1 /*
<lambda>null2  * Copyright 2022 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 package androidx.camera.integration.view
18 
19 import android.graphics.SurfaceTexture
20 import android.graphics.SurfaceTexture.OnFrameAvailableListener
21 import android.os.Handler
22 import android.os.HandlerThread
23 import android.view.Surface
24 import androidx.annotation.VisibleForTesting
25 import androidx.camera.core.DynamicRange
26 import androidx.camera.core.SurfaceOutput
27 import androidx.camera.core.SurfaceProcessor
28 import androidx.camera.core.SurfaceRequest
29 import androidx.camera.core.impl.utils.executor.CameraXExecutors.newHandlerExecutor
30 import androidx.camera.core.processing.OpenGlRenderer
31 import androidx.camera.core.processing.ShaderProvider
32 import androidx.camera.core.processing.util.GLUtils.InputFormat
33 import androidx.core.util.Preconditions.checkState
34 import java.util.concurrent.Executor
35 
36 /**
37  * A processor that applies tone mapping on camera output.
38  *
39  * <p>The thread safety is guaranteed by using the main thread.
40  */
41 class ToneMappingSurfaceProcessor : SurfaceProcessor, OnFrameAvailableListener {
42 
43     companion object {
44         // A fragment shader that applies a yellow hue.
45         private val TONE_MAPPING_SHADER_PROVIDER =
46             object : ShaderProvider {
47                 override fun createFragmentShader(sampler: String, fragCoords: String): String {
48                     return """
49                     #extension GL_OES_EGL_image_external : require
50                     precision mediump float;
51                     uniform samplerExternalOES $sampler;
52                     uniform float uAlphaScale;
53                     varying vec2 $fragCoords;
54                     void main() {
55                       vec4 sampleColor = texture2D($sampler, $fragCoords);
56                       vec4 src = vec4(
57                            sampleColor.r * 0.5 + sampleColor.g * 0.8 + sampleColor.b * 0.3,
58                            sampleColor.r * 0.4 + sampleColor.g * 0.7 + sampleColor.b * 0.2,
59                            sampleColor.r * 0.3 + sampleColor.g * 0.5 + sampleColor.b * 0.1,
60                            1.0);
61                       gl_FragColor = vec4(src.rgb, src.a * uAlphaScale);
62                     }
63                     """
64                 }
65             }
66 
67         private const val GL_THREAD_NAME = "ToneMappingSurfaceProcessor"
68     }
69 
70     private val glThread: HandlerThread = HandlerThread(GL_THREAD_NAME)
71     private var glHandler: Handler
72     private var glExecutor: Executor
73 
74     // Members below are only accessed on GL thread.
75     private val glRenderer: OpenGlRenderer = OpenGlRenderer()
76     private val outputSurfaces: MutableMap<SurfaceOutput, Surface> = mutableMapOf()
77     private val textureTransform: FloatArray = FloatArray(16)
78     private val surfaceTransform: FloatArray = FloatArray(16)
79     private var isReleased = false
80 
81     // For testing only
82     private var surfaceRequested = false
83     private var outputSurfaceProvided = false
84 
85     init {
86         glThread.start()
87         glHandler = Handler(glThread.looper)
88         glExecutor = newHandlerExecutor(glHandler)
89         glExecutor.execute {
90             glRenderer.init(
91                 DynamicRange.SDR,
92                 mapOf(InputFormat.DEFAULT to TONE_MAPPING_SHADER_PROVIDER)
93             )
94         }
95     }
96 
97     override fun onInputSurface(surfaceRequest: SurfaceRequest) {
98         checkGlThread()
99         if (isReleased) {
100             surfaceRequest.willNotProvideSurface()
101             return
102         }
103         surfaceRequested = true
104         val surfaceTexture = SurfaceTexture(glRenderer.textureName)
105         surfaceTexture.setDefaultBufferSize(
106             surfaceRequest.resolution.width,
107             surfaceRequest.resolution.height
108         )
109         val surface = Surface(surfaceTexture)
110         surfaceRequest.provideSurface(surface, glExecutor) {
111             surfaceTexture.setOnFrameAvailableListener(null)
112             surfaceTexture.release()
113             surface.release()
114         }
115         surfaceTexture.setOnFrameAvailableListener(this, glHandler)
116     }
117 
118     override fun onOutputSurface(surfaceOutput: SurfaceOutput) {
119         checkGlThread()
120         outputSurfaceProvided = true
121         if (isReleased) {
122             surfaceOutput.close()
123             return
124         }
125         val surface =
126             surfaceOutput.getSurface(glExecutor) {
127                 surfaceOutput.close()
128                 outputSurfaces.remove(surfaceOutput)?.let { removedSurface ->
129                     glRenderer.unregisterOutputSurface(removedSurface)
130                 }
131             }
132         glRenderer.registerOutputSurface(surface)
133         outputSurfaces[surfaceOutput] = surface
134     }
135 
136     @VisibleForTesting
137     fun isSurfaceRequestedAndProvided(): Boolean {
138         return surfaceRequested && outputSurfaceProvided
139     }
140 
141     fun release() {
142         glExecutor.execute { releaseInternal() }
143     }
144 
145     private fun releaseInternal() {
146         checkGlThread()
147         if (!isReleased) {
148             // Once release is called, we can stop sending frame to output surfaces.
149             for (surfaceOutput in outputSurfaces.keys) {
150                 surfaceOutput.close()
151             }
152             outputSurfaces.clear()
153             glRenderer.release()
154             glThread.quitSafely()
155             isReleased = true
156         }
157     }
158 
159     private fun checkGlThread() {
160         checkState(GL_THREAD_NAME == Thread.currentThread().name)
161     }
162 
163     fun getGlExecutor(): Executor {
164         return glExecutor
165     }
166 
167     override fun onFrameAvailable(surfaceTexture: SurfaceTexture) {
168         checkGlThread()
169         if (isReleased) {
170             return
171         }
172         surfaceTexture.updateTexImage()
173         surfaceTexture.getTransformMatrix(textureTransform)
174         for (entry in outputSurfaces.entries.iterator()) {
175             val surface = entry.value
176             val surfaceOutput = entry.key
177 
178             surfaceOutput.updateTransformMatrix(surfaceTransform, textureTransform)
179             glRenderer.render(surfaceTexture.timestamp, surfaceTransform, surface)
180         }
181     }
182 }
183