1 /*
2  * Copyright 2019 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 package androidx.camera.extensions.impl;
18 
19 import static android.opengl.EGL14.EGL_CONTEXT_CLIENT_VERSION;
20 import static android.opengl.EGL14.EGL_DEFAULT_DISPLAY;
21 import static android.opengl.EGL14.EGL_HEIGHT;
22 import static android.opengl.EGL14.EGL_NONE;
23 import static android.opengl.EGL14.EGL_NO_CONTEXT;
24 import static android.opengl.EGL14.EGL_NO_DISPLAY;
25 import static android.opengl.EGL14.EGL_NO_SURFACE;
26 import static android.opengl.EGL14.EGL_OPENGL_ES2_BIT;
27 import static android.opengl.EGL14.EGL_PBUFFER_BIT;
28 import static android.opengl.EGL14.EGL_RENDERABLE_TYPE;
29 import static android.opengl.EGL14.EGL_SURFACE_TYPE;
30 import static android.opengl.EGL14.EGL_TRUE;
31 import static android.opengl.EGL14.EGL_WIDTH;
32 import static android.opengl.EGL14.EGL_WINDOW_BIT;
33 import static android.opengl.EGLExt.EGL_RECORDABLE_ANDROID;
34 import static android.opengl.GLES20.GL_COLOR_BUFFER_BIT;
35 import static android.opengl.GLES20.GL_COMPILE_STATUS;
36 import static android.opengl.GLES20.GL_FLOAT;
37 import static android.opengl.GLES20.GL_FRAGMENT_SHADER;
38 import static android.opengl.GLES20.GL_LINK_STATUS;
39 import static android.opengl.GLES20.GL_LUMINANCE;
40 import static android.opengl.GLES20.GL_TEXTURE_2D;
41 import static android.opengl.GLES20.GL_TRIANGLES;
42 import static android.opengl.GLES20.GL_UNSIGNED_BYTE;
43 import static android.opengl.GLES20.GL_VERTEX_SHADER;
44 
45 import android.media.Image;
46 import android.opengl.EGL14;
47 import android.opengl.EGLConfig;
48 import android.opengl.EGLContext;
49 import android.opengl.EGLDisplay;
50 import android.opengl.EGLExt;
51 import android.opengl.EGLSurface;
52 import android.opengl.GLES20;
53 import android.util.Log;
54 import android.util.Size;
55 import android.view.Surface;
56 
57 import java.nio.ByteBuffer;
58 import java.nio.ByteOrder;
59 import java.nio.FloatBuffer;
60 import java.util.Objects;
61 
62 /**
63  * A renderer that takes an {@link Image} and renders it to a {@link Surface} that is backed by a
64  * {@link android.graphics.SurfaceTexture}.
65  *
66  * <p> The renderer only takes the Y channel of the input YUV image and copies the same values to
67  * the RGB channels. This is meant only as a demonstration that the preview processing pipeline can
68  * take as input a {@link Image} and write to a {@link android.graphics.SurfaceTexture}. It has only
69  * been tested on a Pixel 2XL.
70  */
71 final class GLImage2SurfaceRenderer {
72     private static final String TAG = "GLImage2SurfaceRenderer";
73 
74     private EGLDisplay mEGLDisplay;
75     private EGLConfig[] mEGLConfigs = new EGLConfig[1];
76     private EGLContext mEGLContext;
77     private EGLSurface mEGLPbufferSurface;
78     private EGLSurface mWindowSurface;
79     private int mTextureId;
80 
81     private int mProgram;
82     private int mPositionHandle;
83 
84     private int mTextureYHandle;
85 
86     private FloatBuffer mVerticesFloatBuffer;
87     private ByteBuffer mVerticesByteBuffer;
88 
89     private Size mInputSize;
90 
91     private static final String sVertexShaderSrc =
92             "attribute vec4 position;"
93                     + "varying vec2 texCoord;"
94                     + "void main() {"
95                     + "    vec2 texCoordUnrotated = (position.xy + vec2(1.0, 1.0)) * 0.5;"
96                     + "    texCoord = vec2(1.0 - texCoordUnrotated.y, 1.0 -texCoordUnrotated.x);"
97                     + "    gl_Position = position;"
98                     + "}";
99 
100     private static final String sFragmentShaderSrc =
101             "precision mediump float;"
102                     + "varying vec2 texCoord;"
103                     + "uniform sampler2D texY;"
104                     + "void main() {"
105                     + "  float y = texture2D(texY, texCoord).r;"
106                     + "  gl_FragColor = vec4(y,y,y, 1.0);"
107                     + "}";
108 
GLImage2SurfaceRenderer()109     GLImage2SurfaceRenderer() {
110         // Initialize
111         mEGLDisplay = EGL14.eglGetDisplay(EGL_DEFAULT_DISPLAY);
112         if (Objects.equals(mEGLDisplay, EGL_NO_DISPLAY)) {
113             throw new RuntimeException("Unable to get GL display");
114         }
115 
116         int[] version = new int[2];
117 
118         boolean initSuccess = EGL14.eglInitialize(mEGLDisplay, version, 0, version, 1);
119 
120         if (!initSuccess) {
121             throw new RuntimeException("Unable to initialize EGL");
122         }
123 
124         int[] configAttribs = {EGL_RENDERABLE_TYPE,
125                 EGL_OPENGL_ES2_BIT,
126                 EGL_SURFACE_TYPE,
127                 EGL_WINDOW_BIT | EGL_PBUFFER_BIT,
128                 EGL_RECORDABLE_ANDROID,
129                 EGL_TRUE,
130                 EGL_NONE};
131 
132         int[] numConfigs = new int[1];
133 
134         boolean eglChooseConfigSuccess = EGL14.eglChooseConfig(
135                 mEGLDisplay,
136                 configAttribs,
137                 0,
138                 mEGLConfigs,
139                 0,
140                 mEGLConfigs.length,
141                 numConfigs,
142                 0);
143 
144         if (!eglChooseConfigSuccess) {
145             throw new RuntimeException("Unable to successfully config egl");
146         }
147 
148         if (numConfigs[0] <= 0) {
149             throw new RuntimeException("Number of configs not greater than 0");
150         }
151 
152         int[] contextAttribs = {EGL_CONTEXT_CLIENT_VERSION, 2, EGL_NONE};
153         mEGLContext = EGL14.eglCreateContext(
154                 mEGLDisplay, mEGLConfigs[0], EGL_NO_CONTEXT, contextAttribs, 0);
155 
156         if (Objects.equals(mEGLContext, EGL_NO_CONTEXT)) {
157             throw new RuntimeException("EGL has no context");
158         }
159 
160         // Create 1x1 pixmap to use as a surface until one is set.
161         int[] pbufferAttribs = {EGL_WIDTH, 1, EGL_HEIGHT, 1, EGL_NONE};
162 
163         mEGLPbufferSurface = EGL14.eglCreatePbufferSurface(mEGLDisplay, mEGLConfigs[0],
164                 pbufferAttribs, 0);
165 
166         if (Objects.equals(mEGLPbufferSurface, EGL_NO_SURFACE)) {
167             throw new RuntimeException("No EGL surface");
168         }
169 
170         EGL14.eglMakeCurrent(mEGLDisplay, mEGLPbufferSurface, mEGLPbufferSurface, mEGLContext);
171 
172         mProgram = createGlProgram();
173 
174         mPositionHandle = GLES20.glGetAttribLocation(mProgram, "position");
175 
176         if (mPositionHandle == -1) {
177             throw new RuntimeException("Unable to position handle");
178         }
179 
180         mTextureYHandle = GLES20.glGetUniformLocation(mProgram, "texY");
181 
182         if (mTextureYHandle == -1) {
183             throw new RuntimeException("Unable to get texture y handle");
184         }
185 
186         int[] textureIds = new int[1];
187         GLES20.glGenTextures(1, textureIds, 0);
188         mTextureId = textureIds[0];
189         GLES20.glUniform1i(mTextureYHandle, 0);
190 
191         initVertexBuffer();
192     }
193 
setInput(Size size)194     void setInput(Size size) {
195         mInputSize = size;
196     }
197 
setWindowSurface(Surface surface, int width, int height)198     void setWindowSurface(Surface surface, int width, int height) {
199         // Destroy previously connected surface
200         destroySurface();
201 
202         // Null surface may have just been passed in to destroy previous surface.
203         if (surface == null) {
204             return;
205         }
206 
207         int[] surfaceAttribs = {
208                 EGL14.EGL_NONE
209         };
210 
211         mWindowSurface = EGL14.eglCreateWindowSurface(mEGLDisplay, mEGLConfigs[0], surface,
212                 surfaceAttribs,
213                 0);
214 
215         if (Objects.equals(mWindowSurface, EGL_NO_SURFACE)) {
216             throw new RuntimeException("Unable to create window surface");
217         }
218 
219         EGL14.eglMakeCurrent(mEGLDisplay, mWindowSurface, mWindowSurface, mEGLContext);
220 
221         GLES20.glViewport(0, 0, width, height);
222         GLES20.glScissor(0, 0, width, height);
223     }
224 
initVertexBuffer()225     void initVertexBuffer() {
226         float[] vertices = {-1.0f, -1.0f, 3.0f, -1.0f, -1.0f, 3.0f};
227         mVerticesByteBuffer = ByteBuffer.allocateDirect(vertices.length * 4).order(
228                 ByteOrder.nativeOrder());
229         mVerticesFloatBuffer = mVerticesByteBuffer.asFloatBuffer();
230         mVerticesFloatBuffer.put(vertices).position(0);
231 
232         int vertexComponents = 2;
233         int vertexType = GL_FLOAT;
234         boolean normalized = false;
235         int vertexStride = 0;
236 
237         GLES20.glVertexAttribPointer(mPositionHandle, vertexComponents, vertexType, normalized,
238                 vertexStride, mVerticesFloatBuffer);
239 
240         GLES20.glEnableVertexAttribArray(mPositionHandle);
241     }
242 
clone(ByteBuffer original)243     public static ByteBuffer clone(ByteBuffer original) {
244         ByteBuffer clone = ByteBuffer.allocate(original.capacity());
245         original.rewind();
246         clone.put(original);
247         original.rewind();
248         clone.flip();
249         return clone;
250     }
251 
renderTexture(Image image)252     void renderTexture(Image image) {
253         GLES20.glUseProgram(mProgram);
254         GLES20.glClearColor(1.0f, 0.0f, 1.0f, 1.0f);
255         GLES20.glClear(GL_COLOR_BUFFER_BIT);
256 
257         // Bind Y texture
258         GLES20.glActiveTexture(GLES20.GL_TEXTURE0);
259         GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, mTextureId);
260 
261         // ByteBuffer needs to be cloned otherwise it might cause failure in emulator.
262         ByteBuffer clonedBuffer = clone(image.getPlanes()[0].getBuffer());
263         GLES20.glTexImage2D(GL_TEXTURE_2D, 0, GL_LUMINANCE, mInputSize.getWidth(),
264                 mInputSize.getHeight(), 0, GL_LUMINANCE, GL_UNSIGNED_BYTE,
265                 clonedBuffer);
266         GLES20.glTexParameterf(GLES20.GL_TEXTURE_2D,
267                 GLES20.GL_TEXTURE_MIN_FILTER, GLES20.GL_LINEAR);
268         GLES20.glTexParameterf(GLES20.GL_TEXTURE_2D,
269                 GLES20.GL_TEXTURE_MAG_FILTER, GLES20.GL_LINEAR);
270         GLES20.glTexParameterf(GLES20.GL_TEXTURE_2D,
271                 GLES20.GL_TEXTURE_WRAP_S, GLES20.GL_CLAMP_TO_EDGE);
272         GLES20.glTexParameterf(GLES20.GL_TEXTURE_2D,
273                 GLES20.GL_TEXTURE_WRAP_T, GLES20.GL_CLAMP_TO_EDGE);
274 
275         GLES20.glDrawArrays(GL_TRIANGLES, 0, 3);
276 
277         EGLExt.eglPresentationTimeANDROID(mEGLDisplay, mWindowSurface, image.getTimestamp());
278 
279         EGL14.eglSwapBuffers(mEGLDisplay, mWindowSurface);
280     }
281 
close()282     void close() {
283         if (mProgram != 0) {
284             GLES20.glDeleteProgram(mProgram);
285             mProgram = 0;
286         }
287 
288         destroySurface();
289         EGL14.eglDestroySurface(mEGLDisplay, mEGLPbufferSurface);
290         EGL14.eglMakeCurrent(mEGLDisplay, EGL_NO_SURFACE, EGL_NO_SURFACE, EGL_NO_CONTEXT);
291         EGL14.eglDestroyContext(mEGLDisplay, mEGLContext);
292     }
293 
getShaderTypeString(int shaderType)294     private String getShaderTypeString(int shaderType) {
295         switch (shaderType) {
296             case GL_VERTEX_SHADER:
297                 return "GL_VERTEX_SHADER";
298             case GL_FRAGMENT_SHADER:
299                 return "GL_FRAGMENT_SHADER";
300             default:
301                 return "<Unknown shader type>";
302         }
303     }
304 
305     // Returns a handle to the shader
compileShader(int shaderType, String shaderSource)306     private int compileShader(int shaderType, String shaderSource) {
307         int shader = GLES20.glCreateShader(shaderType);
308 
309         if (shader == 0) {
310             throw new RuntimeException("Unable to create shader");
311         }
312 
313         GLES20.glShaderSource(shader, shaderSource);
314         GLES20.glCompileShader(shader);
315         int[] compileStatus = new int[1];
316         GLES20.glGetShaderiv(shader, GL_COMPILE_STATUS, compileStatus, 0);
317 
318         if (compileStatus[0] == 0) {
319             String logBuffer = GLES20.glGetShaderInfoLog(shader);
320             Log.e(TAG,
321                     String.format("Unable to compile %s shader: %s",
322                             getShaderTypeString(shaderType),
323                             logBuffer));
324 
325             GLES20.glDeleteShader(shader);
326             shader = 0;
327         }
328 
329         return shader;
330     }
331 
332     // Returns a handle to the output program
createGlProgram()333     private int createGlProgram() {
334         int vertexShader = compileShader(GL_VERTEX_SHADER, sVertexShaderSrc);
335         if (vertexShader == 0) {
336             throw new RuntimeException("Unable to compile vertex shader");
337         }
338 
339         int fragmentShader = compileShader(GL_FRAGMENT_SHADER, sFragmentShaderSrc);
340         if (fragmentShader == 0) {
341             throw new RuntimeException("Unable to compile fragment shader");
342         }
343         int program = GLES20.glCreateProgram();
344         if (program == 0) {
345             throw new RuntimeException("Unable to create GL program");
346         }
347         GLES20.glAttachShader(program, vertexShader);
348         GLES20.glAttachShader(program, fragmentShader);
349         GLES20.glLinkProgram(program);
350 
351         int[] linkStatus = new int[1];
352         GLES20.glGetProgramiv(program, GL_LINK_STATUS, linkStatus, 0);
353         if (linkStatus[0] == 0) {
354             String logBuffer = GLES20.glGetProgramInfoLog(program);
355             Log.e(TAG, String.format("Unable to link program: %s", logBuffer));
356 
357             GLES20.glDeleteProgram(program);
358             program = 0;
359         }
360         if (program == 0) {
361             throw new RuntimeException("Unable to create GL program");
362         }
363         return program;
364     }
365 
destroySurface()366     private void destroySurface() {
367         if (mWindowSurface == null) {
368             return;
369         }
370         EGL14.eglMakeCurrent(mEGLDisplay, mEGLPbufferSurface, mEGLPbufferSurface, mEGLContext);
371         EGL14.eglDestroySurface(mEGLDisplay, mWindowSurface);
372         mWindowSurface = null;
373     }
374 }
375