1 /* <lambda>null2 * Copyright (C) 2024 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 package com.google.jetpackcamera.core.camera.effects 17 18 import android.graphics.SurfaceTexture 19 import android.opengl.EGL14 20 import android.opengl.EGLConfig 21 import android.opengl.EGLExt 22 import android.opengl.GLES11Ext 23 import android.opengl.GLES20 24 import android.opengl.GLES30 25 import android.util.Log 26 import android.view.Surface 27 import androidx.annotation.WorkerThread 28 import androidx.camera.core.DynamicRange 29 import androidx.graphics.opengl.egl.EGLConfigAttributes 30 import androidx.graphics.opengl.egl.EGLManager 31 import androidx.graphics.opengl.egl.EGLSpec 32 import java.nio.ByteBuffer 33 import java.nio.ByteOrder 34 import java.nio.FloatBuffer 35 36 class ShaderCopy(private val dynamicRange: DynamicRange) : RenderCallbacks { 37 38 // Called on worker thread only 39 private var externalTextureId: Int = -1 40 private var programHandle = -1 41 private var texMatrixLoc = -1 42 private var samplerLoc = -1 43 private var positionLoc = -1 44 private var texCoordLoc = -1 45 private val glExtensions: Set<String> by lazy { 46 checkGlThread() 47 buildSet { 48 GLES20.glGetString(GLES20.GL_EXTENSIONS)?.split(" ")?.also { 49 addAll(it) 50 } 51 } 52 } 53 private val use10bitPipeline: Boolean 54 get() = dynamicRange.bitDepth == DynamicRange.BIT_DEPTH_10_BIT 55 56 override val glThreadName: String 57 get() = TAG 58 59 override val provideEGLSpec: () -> EGLSpec 60 get() = { if (use10bitPipeline) EGLSpec.V14ES3 else EGLSpec.V14 } 61 62 override val initConfig: EGLManager.() -> EGLConfig 63 get() = { 64 checkNotNull( 65 loadConfig( 66 EGLConfigAttributes { 67 if (use10bitPipeline) { 68 TEN_BIT_REQUIRED_EGL_EXTENSIONS.forEach { 69 check(isExtensionSupported(it)) { 70 "Required extension for 10-bit HDR is not " + 71 "supported: $it" 72 } 73 } 74 include(EGLConfigAttributes.RGBA_1010102) 75 EGL14.EGL_RENDERABLE_TYPE to 76 EGLExt.EGL_OPENGL_ES3_BIT_KHR 77 EGL14.EGL_SURFACE_TYPE to 78 (EGL14.EGL_WINDOW_BIT or EGL14.EGL_PBUFFER_BIT) 79 } else { 80 include(EGLConfigAttributes.RGBA_8888) 81 } 82 } 83 ) 84 ) { 85 "Unable to select EGLConfig" 86 } 87 } 88 89 override val initRenderer: () -> Unit 90 get() = { 91 if (use10bitPipeline && glExtensions.contains("GL_KHR_debug")) { 92 GLDebug.enableES3DebugErrorLogging() 93 } 94 95 createProgram( 96 if (use10bitPipeline) { 97 TEN_BIT_VERTEX_SHADER 98 } else { 99 DEFAULT_VERTEX_SHADER 100 }, 101 if (use10bitPipeline) { 102 TEN_BIT_FRAGMENT_SHADER 103 } else { 104 DEFAULT_FRAGMENT_SHADER 105 } 106 ) 107 loadLocations() 108 createTexture() 109 useAndConfigureProgram() 110 } 111 112 override val createSurfaceTexture 113 get() = { width: Int, height: Int -> 114 SurfaceTexture(externalTextureId).apply { 115 setDefaultBufferSize(width, height) 116 } 117 } 118 119 override val createOutputSurface 120 get() = { eglSpec: EGLSpec, 121 config: EGLConfig, 122 surface: Surface, 123 _: Int, 124 _: Int -> 125 eglSpec.eglCreateWindowSurface( 126 config, 127 surface, 128 EGLConfigAttributes { 129 if (use10bitPipeline) { 130 EGL_GL_COLORSPACE_KHR to EGL_GL_COLORSPACE_BT2020_HLG_EXT 131 } 132 } 133 ) 134 } 135 136 override val drawFrame 137 get() = { outputWidth: Int, 138 outputHeight: Int, 139 surfaceTransform: FloatArray -> 140 checkGlThread() 141 GLES20.glViewport( 142 0, 143 0, 144 outputWidth, 145 outputHeight 146 ) 147 GLES20.glScissor( 148 0, 149 0, 150 outputWidth, 151 outputHeight 152 ) 153 154 GLES20.glUniformMatrix4fv( 155 texMatrixLoc, 156 /*count=*/ 157 1, 158 /*transpose=*/ 159 false, 160 surfaceTransform, 161 /*offset=*/ 162 0 163 ) 164 checkGlErrorOrThrow("glUniformMatrix4fv") 165 166 // Draw the rect. 167 GLES20.glDrawArrays( 168 GLES20.GL_TRIANGLE_STRIP, 169 /*firstVertex=*/ 170 0, 171 /*vertexCount=*/ 172 4 173 ) 174 checkGlErrorOrThrow("glDrawArrays") 175 } 176 177 @WorkerThread 178 fun createTexture() { 179 checkGlThread() 180 val textures = IntArray(1) 181 GLES20.glGenTextures(1, textures, 0) 182 checkGlErrorOrThrow("glGenTextures") 183 val texId = textures[0] 184 GLES20.glBindTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, texId) 185 checkGlErrorOrThrow("glBindTexture $texId") 186 GLES20.glTexParameterf( 187 GLES11Ext.GL_TEXTURE_EXTERNAL_OES, 188 GLES20.GL_TEXTURE_MIN_FILTER, 189 GLES20.GL_NEAREST.toFloat() 190 ) 191 GLES20.glTexParameterf( 192 GLES11Ext.GL_TEXTURE_EXTERNAL_OES, 193 GLES20.GL_TEXTURE_MAG_FILTER, 194 GLES20.GL_LINEAR.toFloat() 195 ) 196 GLES20.glTexParameteri( 197 GLES11Ext.GL_TEXTURE_EXTERNAL_OES, 198 GLES20.GL_TEXTURE_WRAP_S, 199 GLES20.GL_CLAMP_TO_EDGE 200 ) 201 GLES20.glTexParameteri( 202 GLES11Ext.GL_TEXTURE_EXTERNAL_OES, 203 GLES20.GL_TEXTURE_WRAP_T, 204 GLES20.GL_CLAMP_TO_EDGE 205 ) 206 checkGlErrorOrThrow("glTexParameter") 207 externalTextureId = texId 208 } 209 210 @WorkerThread 211 fun useAndConfigureProgram() { 212 checkGlThread() 213 // Select the program. 214 GLES20.glUseProgram(programHandle) 215 checkGlErrorOrThrow("glUseProgram") 216 217 // Set the texture. 218 GLES20.glActiveTexture(GLES20.GL_TEXTURE0) 219 GLES20.glBindTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, externalTextureId) 220 GLES20.glUniform1i(samplerLoc, 0) 221 222 if (use10bitPipeline) { 223 val vaos = IntArray(1) 224 GLES30.glGenVertexArrays(1, vaos, 0) 225 GLES30.glBindVertexArray(vaos[0]) 226 checkGlErrorOrThrow("glBindVertexArray") 227 } 228 229 val vbos = IntArray(2) 230 GLES20.glGenBuffers(2, vbos, 0) 231 checkGlErrorOrThrow("glGenBuffers") 232 233 // Connect vertexBuffer to "aPosition". 234 val coordsPerVertex = 2 235 val vertexStride = 0 236 GLES20.glBindBuffer(GLES20.GL_ARRAY_BUFFER, vbos[0]) 237 checkGlErrorOrThrow("glBindBuffer") 238 GLES20.glBufferData( 239 GLES20.GL_ARRAY_BUFFER, 240 VERTEX_BUF.capacity() * SIZEOF_FLOAT, 241 VERTEX_BUF, 242 GLES20.GL_STATIC_DRAW 243 ) 244 checkGlErrorOrThrow("glBufferData") 245 246 // Enable the "aPosition" vertex attribute. 247 GLES20.glEnableVertexAttribArray(positionLoc) 248 checkGlErrorOrThrow("glEnableVertexAttribArray") 249 250 GLES20.glVertexAttribPointer( 251 positionLoc, 252 coordsPerVertex, 253 GLES20.GL_FLOAT, 254 /*normalized=*/ 255 false, 256 vertexStride, 257 0 258 ) 259 checkGlErrorOrThrow("glVertexAttribPointer") 260 261 // Connect texBuffer to "aTextureCoord". 262 val coordsPerTex = 2 263 val texStride = 0 264 GLES20.glBindBuffer(GLES20.GL_ARRAY_BUFFER, vbos[1]) 265 checkGlErrorOrThrow("glBindBuffer") 266 267 GLES20.glBufferData( 268 GLES20.GL_ARRAY_BUFFER, 269 TEX_BUF.capacity() * SIZEOF_FLOAT, 270 TEX_BUF, 271 GLES20.GL_STATIC_DRAW 272 ) 273 checkGlErrorOrThrow("glBufferData") 274 275 // Enable the "aTextureCoord" vertex attribute. 276 GLES20.glEnableVertexAttribArray(texCoordLoc) 277 checkGlErrorOrThrow("glEnableVertexAttribArray") 278 279 GLES20.glVertexAttribPointer( 280 texCoordLoc, 281 coordsPerTex, 282 GLES20.GL_FLOAT, 283 /*normalized=*/ 284 false, 285 texStride, 286 0 287 ) 288 checkGlErrorOrThrow("glVertexAttribPointer") 289 } 290 291 @WorkerThread 292 private fun createProgram(vertShader: String, fragShader: String) { 293 checkGlThread() 294 var vertexShader = -1 295 var fragmentShader = -1 296 var program = -1 297 try { 298 fragmentShader = loadShader( 299 GLES20.GL_FRAGMENT_SHADER, 300 fragShader 301 ) 302 vertexShader = loadShader( 303 GLES20.GL_VERTEX_SHADER, 304 vertShader 305 ) 306 program = GLES20.glCreateProgram() 307 checkGlErrorOrThrow("glCreateProgram") 308 GLES20.glAttachShader(program, vertexShader) 309 checkGlErrorOrThrow("glAttachShader") 310 GLES20.glAttachShader(program, fragmentShader) 311 checkGlErrorOrThrow("glAttachShader") 312 GLES20.glLinkProgram(program) 313 val linkStatus = IntArray(1) 314 GLES20.glGetProgramiv( 315 program, 316 GLES20.GL_LINK_STATUS, 317 linkStatus, 318 /*offset=*/ 319 0 320 ) 321 check(linkStatus[0] == GLES20.GL_TRUE) { 322 "Could not link program: " + GLES20.glGetProgramInfoLog( 323 program 324 ) 325 } 326 programHandle = program 327 } catch (e: Exception) { 328 if (vertexShader != -1) { 329 GLES20.glDeleteShader(vertexShader) 330 } 331 if (fragmentShader != -1) { 332 GLES20.glDeleteShader(fragmentShader) 333 } 334 if (program != -1) { 335 GLES20.glDeleteProgram(program) 336 } 337 throw e 338 } 339 } 340 341 @WorkerThread 342 private fun loadLocations() { 343 checkGlThread() 344 positionLoc = GLES20.glGetAttribLocation(programHandle, "aPosition") 345 checkLocationOrThrow(positionLoc, "aPosition") 346 texCoordLoc = GLES20.glGetAttribLocation(programHandle, "aTextureCoord") 347 checkLocationOrThrow(texCoordLoc, "aTextureCoord") 348 texMatrixLoc = GLES20.glGetUniformLocation(programHandle, "uTexMatrix") 349 checkLocationOrThrow(texMatrixLoc, "uTexMatrix") 350 samplerLoc = GLES20.glGetUniformLocation(programHandle, VAR_TEXTURE) 351 checkLocationOrThrow(samplerLoc, VAR_TEXTURE) 352 } 353 354 @WorkerThread 355 private fun loadShader(shaderType: Int, source: String): Int { 356 checkGlThread() 357 val shader = GLES20.glCreateShader(shaderType) 358 checkGlErrorOrThrow("glCreateShader type=$shaderType") 359 GLES20.glShaderSource(shader, source) 360 GLES20.glCompileShader(shader) 361 val compiled = IntArray(1) 362 GLES20.glGetShaderiv( 363 shader, 364 GLES20.GL_COMPILE_STATUS, 365 compiled, 366 /*offset=*/ 367 0 368 ) 369 check(compiled[0] == GLES20.GL_TRUE) { 370 Log.w(TAG, "Could not compile shader: $source") 371 try { 372 return@check "Could not compile shader type " + 373 "$shaderType: ${GLES20.glGetShaderInfoLog(shader)}" 374 } finally { 375 GLES20.glDeleteShader(shader) 376 } 377 } 378 return shader 379 } 380 381 @WorkerThread 382 private fun checkGlErrorOrThrow(op: String) { 383 val error = GLES20.glGetError() 384 check(error == GLES20.GL_NO_ERROR) { op + ": GL error 0x" + Integer.toHexString(error) } 385 } 386 387 private fun checkLocationOrThrow(location: Int, label: String) { 388 check(location >= 0) { "Unable to locate '$label' in program" } 389 } 390 391 companion object { 392 private const val SIZEOF_FLOAT = 4 393 394 private val VERTEX_BUF = floatArrayOf( 395 // 0 bottom left 396 -1.0f, 397 -1.0f, 398 // 1 bottom right 399 1.0f, 400 -1.0f, 401 // 2 top left 402 -1.0f, 403 1.0f, 404 // 3 top right 405 1.0f, 406 1.0f 407 ).toBuffer() 408 409 private val TEX_BUF = floatArrayOf( 410 // 0 bottom left 411 0.0f, 412 0.0f, 413 // 1 bottom right 414 1.0f, 415 0.0f, 416 // 2 top left 417 0.0f, 418 1.0f, 419 // 3 top right 420 1.0f, 421 1.0f 422 ).toBuffer() 423 424 private const val TAG = "ShaderCopy" 425 private const val GL_THREAD_NAME = TAG 426 427 private const val VAR_TEXTURE_COORD = "vTextureCoord" 428 private val DEFAULT_VERTEX_SHADER = 429 """ 430 uniform mat4 uTexMatrix; 431 attribute vec4 aPosition; 432 attribute vec4 aTextureCoord; 433 varying vec2 $VAR_TEXTURE_COORD; 434 void main() { 435 gl_Position = aPosition; 436 $VAR_TEXTURE_COORD = (uTexMatrix * aTextureCoord).xy; 437 } 438 """.trimIndent() 439 440 private val TEN_BIT_VERTEX_SHADER = 441 """ 442 #version 300 es 443 in vec4 aPosition; 444 in vec4 aTextureCoord; 445 uniform mat4 uTexMatrix; 446 out vec2 $VAR_TEXTURE_COORD; 447 void main() { 448 gl_Position = aPosition; 449 $VAR_TEXTURE_COORD = (uTexMatrix * aTextureCoord).xy; 450 } 451 """.trimIndent() 452 453 private const val VAR_TEXTURE = "sTexture" 454 private val DEFAULT_FRAGMENT_SHADER = 455 """ 456 #extension GL_OES_EGL_image_external : require 457 precision mediump float; 458 varying vec2 $VAR_TEXTURE_COORD; 459 uniform samplerExternalOES $VAR_TEXTURE; 460 void main() { 461 gl_FragColor = texture2D($VAR_TEXTURE, $VAR_TEXTURE_COORD); 462 } 463 """.trimIndent() 464 465 private val TEN_BIT_FRAGMENT_SHADER = 466 """ 467 #version 300 es 468 #extension GL_EXT_YUV_target : require 469 precision mediump float; 470 uniform __samplerExternal2DY2YEXT $VAR_TEXTURE; 471 in vec2 $VAR_TEXTURE_COORD; 472 out vec3 outColor; 473 474 vec3 yuvToRgb(vec3 yuv) { 475 const vec3 yuvOffset = vec3(0.0625, 0.5, 0.5); 476 const mat3 yuvToRgbColorTransform = mat3( 477 1.1689f, 1.1689f, 1.1689f, 478 0.0000f, -0.1881f, 2.1502f, 479 1.6853f, -0.6530f, 0.0000f 480 ); 481 return clamp(yuvToRgbColorTransform * (yuv - yuvOffset), 0.0, 1.0); 482 } 483 484 void main() { 485 outColor = yuvToRgb(texture($VAR_TEXTURE, $VAR_TEXTURE_COORD).xyz); 486 } 487 """.trimIndent() 488 489 private const val EGL_GL_COLORSPACE_KHR = 0x309D 490 private const val EGL_GL_COLORSPACE_BT2020_HLG_EXT = 0x3540 491 492 private val TEN_BIT_REQUIRED_EGL_EXTENSIONS = listOf( 493 "EGL_EXT_gl_colorspace_bt2020_hlg" 494 ) 495 496 private fun FloatArray.toBuffer(): FloatBuffer { 497 val bb = ByteBuffer.allocateDirect(size * SIZEOF_FLOAT) 498 bb.order(ByteOrder.nativeOrder()) 499 val fb = bb.asFloatBuffer() 500 fb.put(this) 501 fb.position(0) 502 return fb 503 } 504 505 private fun checkGlThread() { 506 check(GL_THREAD_NAME == Thread.currentThread().name) 507 } 508 } 509 } 510