1 /*
2  * Copyright 2022 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 package androidx.camera.core.processing;
18 
19 import static android.opengl.GLES11Ext.GL_TEXTURE_EXTERNAL_OES;
20 
21 import static androidx.camera.core.ImageProcessingUtil.copyByteBufferToBitmap;
22 import static androidx.camera.core.processing.util.GLUtils.EMPTY_ATTRIBS;
23 import static androidx.camera.core.processing.util.GLUtils.NO_OUTPUT_SURFACE;
24 import static androidx.camera.core.processing.util.GLUtils.PIXEL_STRIDE;
25 import static androidx.camera.core.processing.util.GLUtils.checkEglErrorOrLog;
26 import static androidx.camera.core.processing.util.GLUtils.checkEglErrorOrThrow;
27 import static androidx.camera.core.processing.util.GLUtils.checkGlErrorOrThrow;
28 import static androidx.camera.core.processing.util.GLUtils.checkGlThreadOrThrow;
29 import static androidx.camera.core.processing.util.GLUtils.checkInitializedOrThrow;
30 import static androidx.camera.core.processing.util.GLUtils.chooseSurfaceAttrib;
31 import static androidx.camera.core.processing.util.GLUtils.createPBufferSurface;
32 import static androidx.camera.core.processing.util.GLUtils.createPrograms;
33 import static androidx.camera.core.processing.util.GLUtils.createTexture;
34 import static androidx.camera.core.processing.util.GLUtils.createWindowSurface;
35 import static androidx.camera.core.processing.util.GLUtils.deleteFbo;
36 import static androidx.camera.core.processing.util.GLUtils.deleteTexture;
37 import static androidx.camera.core.processing.util.GLUtils.generateFbo;
38 import static androidx.camera.core.processing.util.GLUtils.generateTexture;
39 import static androidx.camera.core.processing.util.GLUtils.getGlVersionNumber;
40 import static androidx.camera.core.processing.util.GLUtils.getSurfaceSize;
41 import static androidx.core.util.Preconditions.checkArgument;
42 
43 import static java.util.Objects.requireNonNull;
44 
45 import android.graphics.Bitmap;
46 import android.opengl.EGL14;
47 import android.opengl.EGLConfig;
48 import android.opengl.EGLContext;
49 import android.opengl.EGLDisplay;
50 import android.opengl.EGLExt;
51 import android.opengl.EGLSurface;
52 import android.opengl.GLES20;
53 import android.util.Log;
54 import android.util.Size;
55 import android.view.Surface;
56 
57 import androidx.annotation.WorkerThread;
58 import androidx.camera.core.DynamicRange;
59 import androidx.camera.core.Logger;
60 import androidx.camera.core.SurfaceOutput;
61 import androidx.camera.core.processing.util.GLUtils.InputFormat;
62 import androidx.camera.core.processing.util.GLUtils.Program2D;
63 import androidx.camera.core.processing.util.GLUtils.SamplerShaderProgram;
64 import androidx.camera.core.processing.util.GraphicDeviceInfo;
65 import androidx.camera.core.processing.util.OutputSurface;
66 import androidx.core.util.Pair;
67 import androidx.core.util.Preconditions;
68 
69 import org.jspecify.annotations.NonNull;
70 import org.jspecify.annotations.Nullable;
71 
72 import java.nio.ByteBuffer;
73 import java.util.Collections;
74 import java.util.HashMap;
75 import java.util.Map;
76 import java.util.Objects;
77 import java.util.concurrent.atomic.AtomicBoolean;
78 
79 import javax.microedition.khronos.egl.EGL10;
80 
81 /**
82  * OpenGLRenderer renders texture image to the output surface.
83  *
84  * <p>OpenGLRenderer's methods must run on the same thread, so called GL thread. The GL thread is
85  * locked as the thread running the {@link #init(DynamicRange, Map)} method, otherwise an
86  * {@link IllegalStateException} will be thrown when other methods are called.
87  */
88 @WorkerThread
89 public class OpenGlRenderer {
90 
91     private static final String TAG = "OpenGlRenderer";
92 
93     protected final AtomicBoolean mInitialized = new AtomicBoolean(false);
94     protected final Map<Surface, OutputSurface> mOutputSurfaceMap = new HashMap<>();
95     protected @Nullable Thread mGlThread;
96     protected @NonNull EGLDisplay mEglDisplay = EGL14.EGL_NO_DISPLAY;
97     protected @NonNull EGLContext mEglContext = EGL14.EGL_NO_CONTEXT;
98     protected int @NonNull [] mSurfaceAttrib = EMPTY_ATTRIBS;
99     protected @Nullable EGLConfig mEglConfig;
100     protected @NonNull EGLSurface mTempSurface = EGL14.EGL_NO_SURFACE;
101     protected @Nullable Surface mCurrentSurface;
102     protected @NonNull Map<InputFormat, Program2D> mProgramHandles = Collections.emptyMap();
103     protected @Nullable Program2D mCurrentProgram = null;
104     protected @NonNull InputFormat mCurrentInputformat = InputFormat.UNKNOWN;
105 
106     private int mExternalTextureId = -1;
107 
108     /**
109      * Initializes the OpenGLRenderer
110      *
111      * <p>This is equivalent to calling {@link #init(DynamicRange, Map)} without providing any
112      * shader overrides. Default shaders will be used for the dynamic range specified.
113      */
init(@onNull DynamicRange dynamicRange)114     public @NonNull GraphicDeviceInfo init(@NonNull DynamicRange dynamicRange) {
115         return init(dynamicRange, Collections.emptyMap());
116     }
117 
118     /**
119      * Initializes the OpenGLRenderer
120      *
121      * <p>Initialization must be done before calling other methods, otherwise an
122      * {@link IllegalStateException} will be thrown. Following methods must run on the same
123      * thread as this method, so called GL thread, otherwise an {@link IllegalStateException}
124      * will be thrown.
125      *
126      * @param dynamicRange    the dynamic range used to select default shaders.
127      * @param shaderOverrides specific shader overrides for fragment shaders
128      *                        per {@link InputFormat}.
129      * @return Info about the initialized graphics device.
130      * @throws IllegalStateException    if the renderer is already initialized or failed to be
131      *                                  initialized.
132      * @throws IllegalArgumentException if the ShaderProvider fails to create shader or provides
133      *                                  invalid shader string.
134      */
init(@onNull DynamicRange dynamicRange, @NonNull Map<InputFormat, ShaderProvider> shaderOverrides)135     public @NonNull GraphicDeviceInfo init(@NonNull DynamicRange dynamicRange,
136             @NonNull Map<InputFormat, ShaderProvider> shaderOverrides) {
137         checkInitializedOrThrow(mInitialized, false);
138         GraphicDeviceInfo.Builder infoBuilder = GraphicDeviceInfo.builder();
139         try {
140             if (dynamicRange.is10BitHdr()) {
141                 Pair<String, String> extensions = getExtensionsBeforeInitialized(dynamicRange);
142                 String glExtensions = Preconditions.checkNotNull(extensions.first);
143                 String eglExtensions = Preconditions.checkNotNull(extensions.second);
144                 if (!glExtensions.contains("GL_EXT_YUV_target")) {
145                     Logger.w(TAG, "Device does not support GL_EXT_YUV_target. Fallback to SDR.");
146                     dynamicRange = DynamicRange.SDR;
147                 }
148                 mSurfaceAttrib = chooseSurfaceAttrib(eglExtensions, dynamicRange);
149                 infoBuilder.setGlExtensions(glExtensions);
150                 infoBuilder.setEglExtensions(eglExtensions);
151             }
152             createEglContext(dynamicRange, infoBuilder);
153             createTempSurface();
154             makeCurrent(mTempSurface);
155             infoBuilder.setGlVersion(getGlVersionNumber());
156             mProgramHandles = createPrograms(dynamicRange, shaderOverrides);
157             mExternalTextureId = createTexture();
158             useAndConfigureProgramWithTexture(mExternalTextureId);
159         } catch (IllegalStateException | IllegalArgumentException e) {
160             releaseInternal();
161             throw e;
162         }
163         mGlThread = Thread.currentThread();
164         mInitialized.set(true);
165         return infoBuilder.build();
166     }
167 
168     /**
169      * Releases the OpenGLRenderer
170      *
171      * @throws IllegalStateException if the caller doesn't run on the GL thread.
172      */
release()173     public void release() {
174         if (!mInitialized.getAndSet(false)) {
175             return;
176         }
177         checkGlThreadOrThrow(mGlThread);
178         releaseInternal();
179     }
180 
181     /**
182      * Register the output surface.
183      *
184      * @throws IllegalStateException if the renderer is not initialized or the caller doesn't run
185      *                               on the GL thread.
186      */
registerOutputSurface(@onNull Surface surface)187     public void registerOutputSurface(@NonNull Surface surface) {
188         checkInitializedOrThrow(mInitialized, true);
189         checkGlThreadOrThrow(mGlThread);
190 
191         if (!mOutputSurfaceMap.containsKey(surface)) {
192             mOutputSurfaceMap.put(surface, NO_OUTPUT_SURFACE);
193         }
194     }
195 
196     /**
197      * Unregister the output surface.
198      *
199      * @throws IllegalStateException if the renderer is not initialized or the caller doesn't run
200      *                               on the GL thread.
201      */
unregisterOutputSurface(@onNull Surface surface)202     public void unregisterOutputSurface(@NonNull Surface surface) {
203         checkInitializedOrThrow(mInitialized, true);
204         checkGlThreadOrThrow(mGlThread);
205 
206         removeOutputSurfaceInternal(surface, true);
207     }
208 
209     /**
210      * Gets the texture name.
211      *
212      * @return the texture name
213      * @throws IllegalStateException if the renderer is not initialized or the caller doesn't run
214      *                               on the GL thread.
215      */
getTextureName()216     public int getTextureName() {
217         checkInitializedOrThrow(mInitialized, true);
218         checkGlThreadOrThrow(mGlThread);
219 
220         return mExternalTextureId;
221     }
222 
223     /**
224      * Sets the input format.
225      *
226      * <p>This will ensure the correct sampler is used for the input.
227      *
228      * @param inputFormat The input format for the input texture.
229      * @throws IllegalStateException if the renderer is not initialized or the caller doesn't run
230      *                               on the GL thread.
231      */
setInputFormat(@onNull InputFormat inputFormat)232     public void setInputFormat(@NonNull InputFormat inputFormat) {
233         checkInitializedOrThrow(mInitialized, true);
234         checkGlThreadOrThrow(mGlThread);
235 
236         if (mCurrentInputformat != inputFormat) {
237             mCurrentInputformat = inputFormat;
238             useAndConfigureProgramWithTexture(mExternalTextureId);
239         }
240     }
241 
activateExternalTexture(int externalTextureId)242     private void activateExternalTexture(int externalTextureId) {
243         GLES20.glActiveTexture(GLES20.GL_TEXTURE0);
244         checkGlErrorOrThrow("glActiveTexture");
245 
246         GLES20.glBindTexture(GL_TEXTURE_EXTERNAL_OES, externalTextureId);
247         checkGlErrorOrThrow("glBindTexture");
248     }
249 
250     /**
251      * Renders the texture image to the output surface.
252      *
253      * @throws IllegalStateException if the renderer is not initialized, the caller doesn't run
254      *                               on the GL thread or the surface is not registered by
255      *                               {@link #registerOutputSurface(Surface)}.
256      */
render(long timestampNs, float @NonNull [] textureTransform, @NonNull Surface surface)257     public void render(long timestampNs, float @NonNull [] textureTransform,
258             @NonNull Surface surface) {
259         checkInitializedOrThrow(mInitialized, true);
260         checkGlThreadOrThrow(mGlThread);
261 
262         OutputSurface outputSurface = getOutSurfaceOrThrow(surface);
263 
264         // Workaround situations that out surface is failed to create or needs to be recreated.
265         if (outputSurface == NO_OUTPUT_SURFACE) {
266             outputSurface = createOutputSurfaceInternal(surface);
267             if (outputSurface == null) {
268                 return;
269             }
270 
271             mOutputSurfaceMap.put(surface, outputSurface);
272         }
273 
274         // Set output surface.
275         if (surface != mCurrentSurface) {
276             makeCurrent(outputSurface.getEglSurface());
277             mCurrentSurface = surface;
278             GLES20.glViewport(0, 0, outputSurface.getWidth(), outputSurface.getHeight());
279             GLES20.glScissor(0, 0, outputSurface.getWidth(), outputSurface.getHeight());
280         }
281 
282         // TODO(b/245855601): Upload the matrix to GPU when textureTransform is changed.
283         Program2D program = Preconditions.checkNotNull(mCurrentProgram);
284         if (program instanceof SamplerShaderProgram) {
285             // Copy the texture transformation matrix over.
286             ((SamplerShaderProgram) program).updateTextureMatrix(textureTransform);
287         }
288 
289         // Draw the rect.
290         GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, /*firstVertex=*/0, /*vertexCount=*/4);
291         checkGlErrorOrThrow("glDrawArrays");
292 
293         // Set timestamp
294         EGLExt.eglPresentationTimeANDROID(mEglDisplay, outputSurface.getEglSurface(), timestampNs);
295 
296         // Swap buffer
297         if (!EGL14.eglSwapBuffers(mEglDisplay, outputSurface.getEglSurface())) {
298             Logger.w(TAG, "Failed to swap buffers with EGL error: 0x" + Integer.toHexString(
299                     EGL14.eglGetError()));
300             removeOutputSurfaceInternal(surface, false);
301         }
302     }
303 
304     /**
305      * Takes a snapshot of the current external texture and returns a Bitmap.
306      *
307      * @param size             the size of the output {@link Bitmap}.
308      * @param textureTransform the transformation matrix.
309      *                         See: {@link SurfaceOutput#updateTransformMatrix(float[], float[])}
310      */
snapshot(@onNull Size size, float @NonNull [] textureTransform)311     public @NonNull Bitmap snapshot(@NonNull Size size, float @NonNull [] textureTransform) {
312         // Allocate buffer.
313         ByteBuffer byteBuffer = ByteBuffer.allocateDirect(
314                 size.getWidth() * size.getHeight() * PIXEL_STRIDE);
315         // Take a snapshot.
316         snapshot(byteBuffer, size, textureTransform);
317         // Create a Bitmap and copy the bytes over.
318         Bitmap bitmap = Bitmap.createBitmap(
319                 size.getWidth(), size.getHeight(), Bitmap.Config.ARGB_8888);
320         byteBuffer.rewind();
321         copyByteBufferToBitmap(bitmap, byteBuffer, size.getWidth() * PIXEL_STRIDE);
322         return bitmap;
323     }
324 
325     /**
326      * Takes a snapshot of the current external texture and stores it in the given byte buffer.
327      *
328      * <p> The image is stored as RGBA with pixel stride of 4 bytes and row stride of width * 4
329      * bytes.
330      *
331      * @param byteBuffer       the byte buffer to store the snapshot.
332      * @param size             the size of the output image.
333      * @param textureTransform the transformation matrix.
334      *                         See: {@link SurfaceOutput#updateTransformMatrix(float[], float[])}
335      */
snapshot(@onNull ByteBuffer byteBuffer, @NonNull Size size, float @NonNull [] textureTransform)336     private void snapshot(@NonNull ByteBuffer byteBuffer, @NonNull Size size,
337             float @NonNull [] textureTransform) {
338         checkArgument(byteBuffer.capacity() == size.getWidth() * size.getHeight() * 4,
339                 "ByteBuffer capacity is not equal to width * height * 4.");
340         checkArgument(byteBuffer.isDirect(), "ByteBuffer is not direct.");
341 
342         // Create and initialize intermediate texture.
343         int texture = generateTexture();
344         GLES20.glActiveTexture(GLES20.GL_TEXTURE1);
345         checkGlErrorOrThrow("glActiveTexture");
346         GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, texture);
347         checkGlErrorOrThrow("glBindTexture");
348         // Configure the texture.
349         GLES20.glTexImage2D(GLES20.GL_TEXTURE_2D, 0, GLES20.GL_RGB, size.getWidth(),
350                 size.getHeight(), 0, GLES20.GL_RGB, GLES20.GL_UNSIGNED_BYTE, null);
351         checkGlErrorOrThrow("glTexImage2D");
352         GLES20.glTexParameteri(
353                 GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_MAG_FILTER, GLES20.GL_LINEAR);
354         GLES20.glTexParameteri(
355                 GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_MIN_FILTER, GLES20.GL_LINEAR);
356 
357         // Create FBO.
358         int fbo = generateFbo();
359         GLES20.glBindFramebuffer(GLES20.GL_FRAMEBUFFER, fbo);
360         checkGlErrorOrThrow("glBindFramebuffer");
361 
362         // Attach the intermediate texture to the FBO
363         GLES20.glFramebufferTexture2D(GLES20.GL_FRAMEBUFFER, GLES20.GL_COLOR_ATTACHMENT0,
364                 GLES20.GL_TEXTURE_2D, texture, 0);
365         checkGlErrorOrThrow("glFramebufferTexture2D");
366 
367         // Bind external texture (camera output).
368         GLES20.glActiveTexture(GLES20.GL_TEXTURE0);
369         checkGlErrorOrThrow("glActiveTexture");
370         GLES20.glBindTexture(GL_TEXTURE_EXTERNAL_OES, mExternalTextureId);
371         checkGlErrorOrThrow("glBindTexture");
372 
373         // Set scissor and viewport.
374         mCurrentSurface = null;
375         GLES20.glViewport(0, 0, size.getWidth(), size.getHeight());
376         GLES20.glScissor(0, 0, size.getWidth(), size.getHeight());
377 
378         Program2D program = Preconditions.checkNotNull(mCurrentProgram);
379         if (program instanceof SamplerShaderProgram) {
380             // Upload transform matrix.
381             ((SamplerShaderProgram) program).updateTextureMatrix(textureTransform);
382         }
383 
384         // Draw the external texture to the intermediate texture.
385         GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, /*firstVertex=*/0, /*vertexCount=*/4);
386         checkGlErrorOrThrow("glDrawArrays");
387 
388         // Read the pixels from the framebuffer
389         GLES20.glReadPixels(0, 0, size.getWidth(), size.getHeight(), GLES20.GL_RGBA,
390                 GLES20.GL_UNSIGNED_BYTE,
391                 byteBuffer);
392         checkGlErrorOrThrow("glReadPixels");
393 
394         // Clean up
395         GLES20.glBindFramebuffer(GLES20.GL_FRAMEBUFFER, 0);
396         deleteTexture(texture);
397         deleteFbo(fbo);
398         // Set the external texture to be active.
399         activateExternalTexture(mExternalTextureId);
400     }
401 
402     // Returns a pair of GL extension (first) and EGL extension (second) strings.
getExtensionsBeforeInitialized( @onNull DynamicRange dynamicRangeToInitialize)403     private @NonNull Pair<String, String> getExtensionsBeforeInitialized(
404             @NonNull DynamicRange dynamicRangeToInitialize) {
405         checkInitializedOrThrow(mInitialized, false);
406         try {
407             createEglContext(dynamicRangeToInitialize, /*infoBuilder=*/null);
408             createTempSurface();
409             makeCurrent(mTempSurface);
410             // eglMakeCurrent() has to be called before checking GL_EXTENSIONS.
411             String glExtensions = GLES20.glGetString(GLES20.GL_EXTENSIONS);
412             String eglExtensions = EGL14.eglQueryString(mEglDisplay, EGL14.EGL_EXTENSIONS);
413             return new Pair<>(glExtensions != null ? glExtensions : "", eglExtensions != null
414                     ? eglExtensions : "");
415         } catch (IllegalStateException e) {
416             Logger.w(TAG, "Failed to get GL or EGL extensions: " + e.getMessage(), e);
417             return new Pair<>("", "");
418         } finally {
419             releaseInternal();
420         }
421     }
422 
createEglContext(@onNull DynamicRange dynamicRange, GraphicDeviceInfo.@Nullable Builder infoBuilder)423     private void createEglContext(@NonNull DynamicRange dynamicRange,
424             GraphicDeviceInfo.@Nullable Builder infoBuilder) {
425         mEglDisplay = EGL14.eglGetDisplay(EGL14.EGL_DEFAULT_DISPLAY);
426         if (Objects.equals(mEglDisplay, EGL14.EGL_NO_DISPLAY)) {
427             throw new IllegalStateException("Unable to get EGL14 display");
428         }
429         int[] version = new int[2];
430         if (!EGL14.eglInitialize(mEglDisplay, version, 0, version, 1)) {
431             mEglDisplay = EGL14.EGL_NO_DISPLAY;
432             throw new IllegalStateException("Unable to initialize EGL14");
433         }
434 
435         if (infoBuilder != null) {
436             infoBuilder.setEglVersion(version[0] + "." + version[1]);
437         }
438 
439         int rgbBits = dynamicRange.is10BitHdr() ? 10 : 8;
440         int alphaBits = dynamicRange.is10BitHdr() ? 2 : 8;
441         int renderType = dynamicRange.is10BitHdr() ? EGLExt.EGL_OPENGL_ES3_BIT_KHR
442                 : EGL14.EGL_OPENGL_ES2_BIT;
443         // TODO(b/319277249): It will crash on older Samsung devices for HDR video 10-bit
444         //  because EGLExt.EGL_RECORDABLE_ANDROID is only supported from OneUI 6.1. We need to
445         //  check by GPU Driver version when new OS is release.
446         int recordableAndroid = dynamicRange.is10BitHdr() ? EGL10.EGL_DONT_CARE : EGL14.EGL_TRUE;
447         int[] attribToChooseConfig = {
448                 EGL14.EGL_RED_SIZE, rgbBits,
449                 EGL14.EGL_GREEN_SIZE, rgbBits,
450                 EGL14.EGL_BLUE_SIZE, rgbBits,
451                 EGL14.EGL_ALPHA_SIZE, alphaBits,
452                 EGL14.EGL_DEPTH_SIZE, 0,
453                 EGL14.EGL_STENCIL_SIZE, 0,
454                 EGL14.EGL_RENDERABLE_TYPE, renderType,
455                 EGLExt.EGL_RECORDABLE_ANDROID, recordableAndroid,
456                 EGL14.EGL_SURFACE_TYPE, EGL14.EGL_WINDOW_BIT | EGL14.EGL_PBUFFER_BIT,
457                 EGL14.EGL_NONE
458         };
459         EGLConfig[] configs = new EGLConfig[1];
460         int[] numConfigs = new int[1];
461         if (!EGL14.eglChooseConfig(mEglDisplay, attribToChooseConfig, 0, configs, 0, configs.length,
462                 numConfigs, 0)) {
463             throw new IllegalStateException("Unable to find a suitable EGLConfig");
464         }
465         EGLConfig config = configs[0];
466         int[] attribToCreateContext = {
467                 EGL14.EGL_CONTEXT_CLIENT_VERSION, dynamicRange.is10BitHdr() ? 3 : 2,
468                 EGL14.EGL_NONE
469         };
470         EGLContext context = EGL14.eglCreateContext(mEglDisplay, config, EGL14.EGL_NO_CONTEXT,
471                 attribToCreateContext, 0);
472         checkEglErrorOrThrow("eglCreateContext");
473         mEglConfig = config;
474         mEglContext = context;
475 
476         // Confirm with query.
477         int[] values = new int[1];
478         EGL14.eglQueryContext(mEglDisplay, mEglContext, EGL14.EGL_CONTEXT_CLIENT_VERSION, values,
479                 0);
480         Log.d(TAG, "EGLContext created, client version " + values[0]);
481     }
482 
createTempSurface()483     private void createTempSurface() {
484         mTempSurface = createPBufferSurface(mEglDisplay, requireNonNull(mEglConfig), /*width=*/1,
485                 /*height=*/1);
486     }
487 
makeCurrent(@onNull EGLSurface eglSurface)488     protected void makeCurrent(@NonNull EGLSurface eglSurface) {
489         Preconditions.checkNotNull(mEglDisplay);
490         Preconditions.checkNotNull(mEglContext);
491         if (!EGL14.eglMakeCurrent(mEglDisplay, eglSurface, eglSurface, mEglContext)) {
492             throw new IllegalStateException("eglMakeCurrent failed");
493         }
494     }
495 
useAndConfigureProgramWithTexture(int textureId)496     protected void useAndConfigureProgramWithTexture(int textureId) {
497         Program2D program = mProgramHandles.get(mCurrentInputformat);
498         if (program == null) {
499             throw new IllegalStateException(
500                     "Unable to configure program for input format: " + mCurrentInputformat);
501         }
502         if (mCurrentProgram != program) {
503             mCurrentProgram = program;
504             mCurrentProgram.use();
505             Log.d(TAG, "Using program for input format " + mCurrentInputformat + ": "
506                     + mCurrentProgram);
507         }
508 
509         // Activate the texture
510         activateExternalTexture(textureId);
511     }
512 
releaseInternal()513     private void releaseInternal() {
514         // Delete program
515         for (Program2D program : mProgramHandles.values()) {
516             program.delete();
517         }
518         mProgramHandles = Collections.emptyMap();
519         mCurrentProgram = null;
520 
521         if (!Objects.equals(mEglDisplay, EGL14.EGL_NO_DISPLAY)) {
522             EGL14.eglMakeCurrent(mEglDisplay, EGL14.EGL_NO_SURFACE, EGL14.EGL_NO_SURFACE,
523                     EGL14.EGL_NO_CONTEXT);
524 
525             // Destroy EGLSurfaces
526             for (OutputSurface outputSurface : mOutputSurfaceMap.values()) {
527                 if (!Objects.equals(outputSurface.getEglSurface(), EGL14.EGL_NO_SURFACE)) {
528                     if (!EGL14.eglDestroySurface(mEglDisplay, outputSurface.getEglSurface())) {
529                         checkEglErrorOrLog("eglDestroySurface");
530                     }
531                 }
532             }
533             mOutputSurfaceMap.clear();
534 
535             // Destroy temp surface
536             if (!Objects.equals(mTempSurface, EGL14.EGL_NO_SURFACE)) {
537                 EGL14.eglDestroySurface(mEglDisplay, mTempSurface);
538                 mTempSurface = EGL14.EGL_NO_SURFACE;
539             }
540 
541             // Destroy EGLContext and terminate display
542             if (!Objects.equals(mEglContext, EGL14.EGL_NO_CONTEXT)) {
543                 EGL14.eglDestroyContext(mEglDisplay, mEglContext);
544                 mEglContext = EGL14.EGL_NO_CONTEXT;
545             }
546             EGL14.eglReleaseThread();
547             EGL14.eglTerminate(mEglDisplay);
548             mEglDisplay = EGL14.EGL_NO_DISPLAY;
549         }
550 
551         // Reset other members
552         mEglConfig = null;
553         mExternalTextureId = -1;
554         mCurrentInputformat = InputFormat.UNKNOWN;
555         mCurrentSurface = null;
556         mGlThread = null;
557     }
558 
getOutSurfaceOrThrow(@onNull Surface surface)559     protected @NonNull OutputSurface getOutSurfaceOrThrow(@NonNull Surface surface) {
560         Preconditions.checkState(mOutputSurfaceMap.containsKey(surface),
561                 "The surface is not registered.");
562 
563         return requireNonNull(mOutputSurfaceMap.get(surface));
564     }
565 
createOutputSurfaceInternal(@onNull Surface surface)566     protected @Nullable OutputSurface createOutputSurfaceInternal(@NonNull Surface surface) {
567         EGLSurface eglSurface;
568         try {
569             eglSurface = createWindowSurface(mEglDisplay, requireNonNull(mEglConfig), surface,
570                     mSurfaceAttrib);
571         } catch (IllegalStateException | IllegalArgumentException e) {
572             Logger.w(TAG, "Failed to create EGL surface: " + e.getMessage(), e);
573             return null;
574         }
575 
576         Size size = getSurfaceSize(mEglDisplay, eglSurface);
577         return OutputSurface.of(eglSurface, size.getWidth(), size.getHeight());
578     }
579 
removeOutputSurfaceInternal(@onNull Surface surface, boolean unregister)580     protected void removeOutputSurfaceInternal(@NonNull Surface surface, boolean unregister) {
581         // Unmake current surface.
582         if (mCurrentSurface == surface) {
583             mCurrentSurface = null;
584             makeCurrent(mTempSurface);
585         }
586 
587         // Remove cached EGL surface.
588         OutputSurface removedOutputSurface;
589         if (unregister) {
590             removedOutputSurface = mOutputSurfaceMap.remove(surface);
591         } else {
592             removedOutputSurface = mOutputSurfaceMap.put(surface, NO_OUTPUT_SURFACE);
593         }
594 
595         // Destroy EGL surface.
596         if (removedOutputSurface != null && removedOutputSurface != NO_OUTPUT_SURFACE) {
597             try {
598                 EGL14.eglDestroySurface(mEglDisplay, removedOutputSurface.getEglSurface());
599             } catch (RuntimeException e) {
600                 Logger.w(TAG, "Failed to destroy EGL surface: " + e.getMessage(), e);
601             }
602         }
603     }
604 }
605