1 /*
2  * Copyright 2024 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 package androidx.camera.core.processing.concurrent;
18 
19 import static androidx.camera.core.processing.util.GLUtils.NO_OUTPUT_SURFACE;
20 import static androidx.camera.core.processing.util.GLUtils.checkGlErrorOrThrow;
21 import static androidx.camera.core.processing.util.GLUtils.checkGlThreadOrThrow;
22 import static androidx.camera.core.processing.util.GLUtils.checkInitializedOrThrow;
23 import static androidx.camera.core.processing.util.GLUtils.create4x4IdentityMatrix;
24 import static androidx.camera.core.processing.util.GLUtils.createTexture;
25 
26 import android.graphics.SurfaceTexture;
27 import android.opengl.EGL14;
28 import android.opengl.EGLExt;
29 import android.opengl.GLES20;
30 import android.opengl.GLES30;
31 import android.opengl.Matrix;
32 import android.util.Size;
33 import android.view.Surface;
34 
35 import androidx.annotation.WorkerThread;
36 import androidx.camera.core.CompositionSettings;
37 import androidx.camera.core.DynamicRange;
38 import androidx.camera.core.Logger;
39 import androidx.camera.core.SurfaceOutput;
40 import androidx.camera.core.processing.OpenGlRenderer;
41 import androidx.camera.core.processing.ShaderProvider;
42 import androidx.camera.core.processing.util.GLUtils;
43 import androidx.camera.core.processing.util.GLUtils.InputFormat;
44 import androidx.camera.core.processing.util.GLUtils.SamplerShaderProgram;
45 import androidx.camera.core.processing.util.GraphicDeviceInfo;
46 import androidx.camera.core.processing.util.OutputSurface;
47 import androidx.core.util.Preconditions;
48 
49 import org.jspecify.annotations.NonNull;
50 
51 import java.util.Map;
52 
53 /**
54  * An internal augmented {@link OpenGlRenderer} for dual concurrent cameras.
55  */
56 @WorkerThread
57 public final class DualOpenGlRenderer extends OpenGlRenderer {
58 
59     private static final String TAG = "DualOpenGlRenderer";
60 
61     private int mPrimaryExternalTextureId = -1;
62     private int mSecondaryExternalTextureId = -1;
63 
64     private final @NonNull CompositionSettings mPrimaryCompositionSettings;
65     private final @NonNull CompositionSettings mSecondaryCompositionSettings;
66 
DualOpenGlRenderer( @onNull CompositionSettings primaryCompositionSettings, @NonNull CompositionSettings secondaryCompositionSettings)67     public DualOpenGlRenderer(
68             @NonNull CompositionSettings primaryCompositionSettings,
69             @NonNull CompositionSettings secondaryCompositionSettings) {
70         mPrimaryCompositionSettings = primaryCompositionSettings;
71         mSecondaryCompositionSettings = secondaryCompositionSettings;
72     }
73 
74     @Override
init(@onNull DynamicRange dynamicRange, @NonNull Map<InputFormat, ShaderProvider> shaderProviderOverrides)75     public @NonNull GraphicDeviceInfo init(@NonNull DynamicRange dynamicRange,
76             @NonNull Map<InputFormat, ShaderProvider> shaderProviderOverrides) {
77         GraphicDeviceInfo graphicDeviceInfo = super.init(dynamicRange, shaderProviderOverrides);
78         mPrimaryExternalTextureId = createTexture();
79         mSecondaryExternalTextureId = createTexture();
80         return graphicDeviceInfo;
81     }
82 
83     @Override
release()84     public void release() {
85         super.release();
86         mPrimaryExternalTextureId = -1;
87         mSecondaryExternalTextureId = -1;
88     }
89 
90     /**
91      * Gets the texture name.
92      *
93      * @return the texture name
94      * @throws IllegalStateException if the renderer is not initialized or the caller doesn't run
95      *                               on the GL thread.
96      */
getTextureName(boolean isPrimary)97     public int getTextureName(boolean isPrimary) {
98         checkInitializedOrThrow(mInitialized, true);
99         checkGlThreadOrThrow(mGlThread);
100 
101         return isPrimary ? mPrimaryExternalTextureId : mSecondaryExternalTextureId;
102     }
103 
104     /**
105      * Renders the texture image to the output surface.
106      *
107      * @throws IllegalStateException if the renderer is not initialized, the caller doesn't run
108      *                               on the GL thread or the surface is not registered by
109      *                               {@link #registerOutputSurface(Surface)}.
110      */
render(long timestampNs, @NonNull Surface surface, @NonNull SurfaceOutput surfaceOutput, @NonNull SurfaceTexture primarySurfaceTexture, @NonNull SurfaceTexture secondarySurfaceTexture)111     public void render(long timestampNs,
112             @NonNull Surface surface,
113             @NonNull SurfaceOutput surfaceOutput,
114             @NonNull SurfaceTexture primarySurfaceTexture,
115             @NonNull SurfaceTexture secondarySurfaceTexture) {
116         checkInitializedOrThrow(mInitialized, true);
117         checkGlThreadOrThrow(mGlThread);
118 
119         OutputSurface outputSurface = getOutSurfaceOrThrow(surface);
120 
121         if (outputSurface == NO_OUTPUT_SURFACE) {
122             outputSurface = createOutputSurfaceInternal(surface);
123             if (outputSurface == null) {
124                 return;
125             }
126 
127             mOutputSurfaceMap.put(surface, outputSurface);
128         }
129 
130         if (surface != mCurrentSurface) {
131             makeCurrent(outputSurface.getEglSurface());
132             mCurrentSurface = surface;
133         }
134 
135         GLES30.glClearColor(0.0f, 0.0f, 0.0f, 1.0f);
136         GLES30.glClear(GLES30.GL_COLOR_BUFFER_BIT);
137         // Primary Camera
138         renderInternal(outputSurface, surfaceOutput, primarySurfaceTexture,
139                 mPrimaryCompositionSettings, mPrimaryExternalTextureId, true);
140         // Secondary Camera
141         // Only use primary camera info for output surface
142         renderInternal(outputSurface, surfaceOutput, secondarySurfaceTexture,
143                 mSecondaryCompositionSettings, mSecondaryExternalTextureId, true);
144 
145         EGLExt.eglPresentationTimeANDROID(mEglDisplay, outputSurface.getEglSurface(), timestampNs);
146 
147         if (!EGL14.eglSwapBuffers(mEglDisplay, outputSurface.getEglSurface())) {
148             Logger.w(TAG, "Failed to swap buffers with EGL error: 0x" + Integer.toHexString(
149                     EGL14.eglGetError()));
150             removeOutputSurfaceInternal(surface, false);
151         }
152     }
153 
renderInternal( @onNull OutputSurface outputSurface, @NonNull SurfaceOutput surfaceOutput, @NonNull SurfaceTexture surfaceTexture, @NonNull CompositionSettings compositionSettings, int externalTextureId, boolean isPrimary)154     private void renderInternal(
155             @NonNull OutputSurface outputSurface,
156             @NonNull SurfaceOutput surfaceOutput,
157             @NonNull SurfaceTexture surfaceTexture,
158             @NonNull CompositionSettings compositionSettings,
159             int externalTextureId,
160             boolean isPrimary) {
161         useAndConfigureProgramWithTexture(externalTextureId);
162         GLES20.glViewport(0, 0, outputSurface.getWidth(),
163                 outputSurface.getHeight());
164         GLES20.glScissor(0, 0, outputSurface.getWidth(),
165                 outputSurface.getHeight());
166 
167         float[] textureTransform = new float[16];
168         surfaceTexture.getTransformMatrix(textureTransform);
169 
170         float[] surfaceOutputMatrix = new float[16];
171         surfaceOutput.updateTransformMatrix(
172                 surfaceOutputMatrix, textureTransform, isPrimary);
173 
174         GLUtils.Program2D currentProgram = Preconditions.checkNotNull(mCurrentProgram);
175         if (currentProgram instanceof SamplerShaderProgram) {
176             ((SamplerShaderProgram) currentProgram).updateTextureMatrix(surfaceOutputMatrix);
177         }
178 
179         float[] transTransform = getTransformMatrix(
180                 new Size((int) (outputSurface.getWidth() * compositionSettings.getScale().first),
181                         (int) (outputSurface.getHeight() * compositionSettings.getScale().second)),
182                 new Size(outputSurface.getWidth(), outputSurface.getHeight()),
183                 compositionSettings);
184         currentProgram.updateTransformMatrix(transTransform);
185 
186         currentProgram.updateAlpha(compositionSettings.getAlpha());
187 
188         GLES20.glEnable(GLES20.GL_BLEND);
189         GLES20.glBlendFuncSeparate(
190                 /* srcRGB= */ GLES20.GL_SRC_ALPHA,
191                 /* dstRGB= */ GLES20.GL_ONE_MINUS_SRC_ALPHA,
192                 /* srcAlpha= */ GLES20.GL_ONE,
193                 /* dstAlpha= */ GLES20.GL_ONE_MINUS_SRC_ALPHA);
194 
195         GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, /*firstVertex=*/0, /*vertexCount=*/4);
196         checkGlErrorOrThrow("glDrawArrays");
197 
198         GLES20.glDisable(GLES20.GL_BLEND);
199     }
200 
getTransformMatrix( @onNull Size overlaySize, @NonNull Size backgroundSize, @NonNull CompositionSettings compositionSettings)201     private static float @NonNull [] getTransformMatrix(
202             @NonNull Size overlaySize,
203             @NonNull Size backgroundSize,
204             @NonNull CompositionSettings compositionSettings) {
205         float[] aspectRatioMatrix = create4x4IdentityMatrix();
206         float[] overlayFrameAnchorMatrix = create4x4IdentityMatrix();
207         float[] transformationMatrix = create4x4IdentityMatrix();
208 
209         Matrix.scaleM(
210                 aspectRatioMatrix,
211                 /* mOffset= */ 0,
212                 (float) overlaySize.getWidth() / backgroundSize.getWidth(),
213                 (float) overlaySize.getHeight() / backgroundSize.getHeight(),
214                 /* z= */ 1.0f);
215 
216         // Translate the image.
217         if (compositionSettings.getScale().first != 0.0f
218                 || compositionSettings.getScale().second != 0.0f) {
219             Matrix.translateM(
220                     overlayFrameAnchorMatrix,
221                     /* mOffset= */ 0,
222                     compositionSettings.getOffset().first / compositionSettings.getScale().first,
223                     compositionSettings.getOffset().second / compositionSettings.getScale().second,
224                     /* z= */ 0.0f);
225         }
226 
227         // Correct for aspect ratio of image in output frame.
228         Matrix.multiplyMM(
229                 transformationMatrix,
230                 /* resultOffset= */ 0,
231                 aspectRatioMatrix,
232                 /* lhsOffset= */ 0,
233                 overlayFrameAnchorMatrix,
234                 /* rhsOffset= */ 0);
235 
236         return transformationMatrix;
237     }
238 }
239