1 /* 2 * Copyright (C) 2021 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package android.mediav2.common.cts; 18 19 import android.graphics.Bitmap; 20 import android.graphics.SurfaceTexture; 21 import android.opengl.GLES11Ext; 22 import android.opengl.GLES20; 23 import android.opengl.Matrix; 24 import android.util.Log; 25 26 import java.io.FileOutputStream; 27 import java.io.IOException; 28 import java.nio.ByteBuffer; 29 import java.nio.ByteOrder; 30 import java.nio.FloatBuffer; 31 32 /** 33 * Code for rendering a texture onto a surface using OpenGL ES 2.0. 34 */ 35 class TextureRender { 36 private static final String TAG = "TextureRender"; 37 38 private static final int FLOAT_SIZE_BYTES = 4; 39 private static final int TRIANGLE_VERTICES_DATA_STRIDE_BYTES = 5 * FLOAT_SIZE_BYTES; 40 private static final int TRIANGLE_VERTICES_DATA_POS_OFFSET = 0; 41 private static final int TRIANGLE_VERTICES_DATA_UV_OFFSET = 3; 42 private final float[] mTriangleVerticesData = { 43 // X, Y, Z, U, V 44 -1.0f, -1.0f, 0.0f, 0.0f, 0.0f, 45 1.0f, -1.0f, 0.0f, 1.0f, 0.0f, 46 -1.0f, 1.0f, 0.0f, 0.0f, 1.0f, 47 1.0f, 1.0f, 0.0f, 1.0f, 1.0f, 48 }; 49 50 private FloatBuffer mTriangleVertices; 51 52 private static final String VERTEX_SHADER_RGB = 53 "uniform mat4 uMVPMatrix;\n" 54 + "uniform mat4 uSTMatrix;\n" 55 + "attribute vec4 aPosition;\n" 56 + "attribute vec4 aTextureCoord;\n" 57 + "varying vec2 vTextureCoord;\n" 58 + "void main() {\n" 59 + " gl_Position = uMVPMatrix * aPosition;\n" 60 + " vTextureCoord = (uSTMatrix * aTextureCoord).xy;\n" 61 + "}\n"; 62 63 private static final String FRAGMENT_SHADER_RGB = 64 "#extension GL_OES_EGL_image_external : require\n" 65 + "precision mediump float;\n" // highp here doesn't seem to matter 66 + "varying vec2 vTextureCoord;\n" 67 + "uniform samplerExternalOES sTexture;\n" 68 + "void main() {\n" 69 + " gl_FragColor = texture2D(sTexture, vTextureCoord);\n" 70 + "}\n"; 71 72 private static final String VERTEX_SHADER_YUV = 73 "#version 300 es\n" 74 + "uniform mat4 uMVPMatrix;\n" 75 + "uniform mat4 uSTMatrix;\n" 76 + "in vec4 aPosition;\n" 77 + "in vec4 aTextureCoord;\n" 78 + "out vec2 vTextureCoord;\n" 79 + "void main() {\n" 80 + " gl_Position = uMVPMatrix * aPosition;\n" 81 + " vTextureCoord = (uSTMatrix * aTextureCoord).xy;\n" 82 + "}\n"; 83 84 private static final String FRAGMENT_SHADER_YUV = 85 "#version 300 es\n" 86 + "#extension GL_OES_EGL_image_external : require\n" 87 + "#extension GL_EXT_YUV_target : require\n" 88 + "precision mediump float;\n" // highp here doesn't seem to matter 89 + "uniform __samplerExternal2DY2YEXT uTexSampler;\n" 90 + "in vec2 vTextureCoord;\n" 91 + "out vec4 outColor;\n" 92 + "void main() {\n" 93 + " outColor = texture(uTexSampler, vTextureCoord);\n" 94 + "}\n"; 95 96 private float[] mMVPMatrix = new float[16]; 97 private float[] mSTMatrix = new float[16]; 98 99 private int mProgram; 100 private int mTextureID; 101 private int mUMVPMatrixHandle; 102 private int mUSTMatrixHandle; 103 private int mAPositionHandle; 104 private int mATextureHandle; 105 private boolean mUseYuvSampling; 106 TextureRender()107 TextureRender() { 108 mTriangleVertices = ByteBuffer.allocateDirect( 109 mTriangleVerticesData.length * FLOAT_SIZE_BYTES) 110 .order(ByteOrder.nativeOrder()).asFloatBuffer(); 111 mTriangleVertices.put(mTriangleVerticesData).position(0); 112 113 Matrix.setIdentityM(mSTMatrix, 0); 114 mUseYuvSampling = false; 115 } 116 setUseYuvSampling(boolean useYuvSampling)117 public void setUseYuvSampling(boolean useYuvSampling) { 118 mUseYuvSampling = useYuvSampling; 119 } 120 getTextureId()121 public int getTextureId() { 122 return mTextureID; 123 } 124 drawFrame(SurfaceTexture st)125 public void drawFrame(SurfaceTexture st) { 126 checkGlError("onDrawFrame start"); 127 st.getTransformMatrix(mSTMatrix); 128 129 GLES20.glClearColor(0.0f, 1.0f, 0.0f, 1.0f); 130 GLES20.glClear(GLES20.GL_DEPTH_BUFFER_BIT | GLES20.GL_COLOR_BUFFER_BIT); 131 132 GLES20.glUseProgram(mProgram); 133 checkGlError("glUseProgram"); 134 135 GLES20.glActiveTexture(GLES20.GL_TEXTURE0); 136 GLES20.glBindTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, mTextureID); 137 138 mTriangleVertices.position(TRIANGLE_VERTICES_DATA_POS_OFFSET); 139 GLES20.glVertexAttribPointer(mAPositionHandle, 3, GLES20.GL_FLOAT, false, 140 TRIANGLE_VERTICES_DATA_STRIDE_BYTES, mTriangleVertices); 141 checkGlError("glVertexAttribPointer maPosition"); 142 GLES20.glEnableVertexAttribArray(mAPositionHandle); 143 checkGlError("glEnableVertexAttribArray maPositionHandle"); 144 145 mTriangleVertices.position(TRIANGLE_VERTICES_DATA_UV_OFFSET); 146 GLES20.glVertexAttribPointer(mATextureHandle, 2, GLES20.GL_FLOAT, false, 147 TRIANGLE_VERTICES_DATA_STRIDE_BYTES, mTriangleVertices); 148 checkGlError("glVertexAttribPointer maTextureHandle"); 149 GLES20.glEnableVertexAttribArray(mATextureHandle); 150 checkGlError("glEnableVertexAttribArray maTextureHandle"); 151 152 Matrix.setIdentityM(mMVPMatrix, 0); 153 GLES20.glUniformMatrix4fv(mUMVPMatrixHandle, 1, false, mMVPMatrix, 0); 154 GLES20.glUniformMatrix4fv(mUSTMatrixHandle, 1, false, mSTMatrix, 0); 155 156 GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, 0, 4); 157 checkGlError("glDrawArrays"); 158 GLES20.glFinish(); 159 } 160 161 /** 162 * Initializes GL state. Call this after the EGL surface has been created and made current. 163 */ surfaceCreated()164 public void surfaceCreated() { 165 if (!mUseYuvSampling) { 166 mProgram = createProgram(VERTEX_SHADER_RGB, FRAGMENT_SHADER_RGB); 167 } else { 168 mProgram = createProgram(VERTEX_SHADER_YUV, FRAGMENT_SHADER_YUV); 169 } 170 if (mProgram == 0) { 171 throw new RuntimeException("failed creating program"); 172 } 173 mAPositionHandle = GLES20.glGetAttribLocation(mProgram, "aPosition"); 174 checkGlError("glGetAttribLocation aPosition"); 175 if (mAPositionHandle == -1) { 176 throw new RuntimeException("Could not get attrib location for aPosition"); 177 } 178 mATextureHandle = GLES20.glGetAttribLocation(mProgram, "aTextureCoord"); 179 checkGlError("glGetAttribLocation aTextureCoord"); 180 if (mATextureHandle == -1) { 181 throw new RuntimeException("Could not get attrib location for aTextureCoord"); 182 } 183 184 mUMVPMatrixHandle = GLES20.glGetUniformLocation(mProgram, "uMVPMatrix"); 185 checkGlError("glGetUniformLocation uMVPMatrix"); 186 if (mUMVPMatrixHandle == -1) { 187 throw new RuntimeException("Could not get attrib location for uMVPMatrix"); 188 } 189 190 mUSTMatrixHandle = GLES20.glGetUniformLocation(mProgram, "uSTMatrix"); 191 checkGlError("glGetUniformLocation uSTMatrix"); 192 if (mUSTMatrixHandle == -1) { 193 throw new RuntimeException("Could not get attrib location for uSTMatrix"); 194 } 195 196 197 int[] textures = new int[1]; 198 GLES20.glGenTextures(1, textures, 0); 199 200 mTextureID = textures[0]; 201 GLES20.glBindTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, mTextureID); 202 checkGlError("glBindTexture mTextureID"); 203 204 GLES20.glTexParameterf(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_MIN_FILTER, 205 GLES20.GL_NEAREST); 206 GLES20.glTexParameterf(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_MAG_FILTER, 207 GLES20.GL_LINEAR); 208 GLES20.glTexParameteri(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_WRAP_S, 209 GLES20.GL_CLAMP_TO_EDGE); 210 GLES20.glTexParameteri(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_WRAP_T, 211 GLES20.GL_CLAMP_TO_EDGE); 212 checkGlError("glTexParameter"); 213 } 214 215 /** 216 * Replaces the fragment shader. 217 */ changeFragmentShader(String fragmentShader)218 public void changeFragmentShader(String fragmentShader) { 219 GLES20.glDeleteProgram(mProgram); 220 mProgram = createProgram(VERTEX_SHADER_RGB, fragmentShader); 221 if (mProgram == 0) { 222 throw new RuntimeException("failed creating program"); 223 } 224 } 225 loadShader(int shaderType, String source)226 private int loadShader(int shaderType, String source) { 227 int shader = GLES20.glCreateShader(shaderType); 228 checkGlError("glCreateShader type=" + shaderType); 229 GLES20.glShaderSource(shader, source); 230 GLES20.glCompileShader(shader); 231 int[] compiled = new int[1]; 232 GLES20.glGetShaderiv(shader, GLES20.GL_COMPILE_STATUS, compiled, 0); 233 if (compiled[0] == 0) { 234 Log.e(TAG, "Could not compile shader " + shaderType + ":"); 235 Log.e(TAG, " " + GLES20.glGetShaderInfoLog(shader)); 236 GLES20.glDeleteShader(shader); 237 shader = 0; 238 } 239 return shader; 240 } 241 createProgram(String vertexSource, String fragmentSource)242 private int createProgram(String vertexSource, String fragmentSource) { 243 int vertexShader = loadShader(GLES20.GL_VERTEX_SHADER, vertexSource); 244 if (vertexShader == 0) { 245 return 0; 246 } 247 int pixelShader = loadShader(GLES20.GL_FRAGMENT_SHADER, fragmentSource); 248 if (pixelShader == 0) { 249 return 0; 250 } 251 252 int program = GLES20.glCreateProgram(); 253 checkGlError("glCreateProgram"); 254 if (program != 0) { 255 GLES20.glAttachShader(program, vertexShader); 256 checkGlError("glAttachShader"); 257 GLES20.glAttachShader(program, pixelShader); 258 checkGlError("glAttachShader"); 259 GLES20.glLinkProgram(program); 260 int[] linkStatus = new int[1]; 261 GLES20.glGetProgramiv(program, GLES20.GL_LINK_STATUS, linkStatus, 0); 262 if (linkStatus[0] != GLES20.GL_TRUE) { 263 Log.e(TAG, "Could not link program: "); 264 Log.e(TAG, GLES20.glGetProgramInfoLog(program)); 265 GLES20.glDeleteProgram(program); 266 program = 0; 267 } 268 } else { 269 Log.e(TAG, "Could not create program"); 270 } 271 272 return program; 273 } 274 checkGlError(String op)275 public void checkGlError(String op) { 276 int error; 277 while ((error = GLES20.glGetError()) != GLES20.GL_NO_ERROR) { 278 Log.e(TAG, op + ": glError " + error); 279 throw new RuntimeException(op + ": glError " + error); 280 } 281 } 282 283 /** 284 * Saves the current frame to disk as a PNG image. Frame starts from (0,0). 285 * <p> 286 * Useful for debugging. 287 */ saveFrame(String filename, int width, int height)288 public static void saveFrame(String filename, int width, int height) { 289 // glReadPixels gives us a ByteBuffer filled with what is essentially big-endian RGBA 290 // data (i.e. a byte of red, followed by a byte of green...). We need an int[] filled 291 // with native-order ARGB data to feed to Bitmap. 292 // 293 // If we implement this as a series of buf.get() calls, we can spend 2.5 seconds just 294 // copying data around for a 720p frame. It's better to do a bulk get() and then 295 // rearrange the data in memory. (For comparison, the PNG compress takes about 500ms 296 // for a trivial frame.) 297 // 298 // So... we set the ByteBuffer to little-endian, which should turn the bulk IntBuffer 299 // get() into a straight memcpy on most Android devices. Our ints will hold ABGR data. 300 // Swapping B and R gives us ARGB. We need about 30ms for the bulk get(), and another 301 // 270ms for the color swap. 302 // 303 // Making this even more interesting is the upside-down nature of GL, which means we 304 // flip the image vertically here. 305 306 ByteBuffer buf = ByteBuffer.allocateDirect(width * height * 4); 307 buf.order(ByteOrder.LITTLE_ENDIAN); 308 GLES20.glReadPixels(0, 0, width, height, GLES20.GL_RGBA, GLES20.GL_UNSIGNED_BYTE, buf); 309 buf.rewind(); 310 311 int pixelCount = width * height; 312 int[] colors = new int[pixelCount]; 313 buf.asIntBuffer().get(colors); 314 for (int i = 0; i < pixelCount; i++) { 315 int c = colors[i]; 316 colors[i] = (c & 0xff00ff00) | ((c & 0x00ff0000) >> 16) | ((c & 0x000000ff) << 16); 317 } 318 319 FileOutputStream fos = null; 320 try { 321 fos = new FileOutputStream(filename); 322 Bitmap bmp = Bitmap.createBitmap(colors, width, height, Bitmap.Config.ARGB_8888); 323 bmp.compress(Bitmap.CompressFormat.PNG, 90, fos); 324 bmp.recycle(); 325 } catch (IOException ioe) { 326 throw new RuntimeException("Failed to write file " + filename, ioe); 327 } finally { 328 try { 329 if (fos != null) fos.close(); 330 } catch (IOException ioe2) { 331 throw new RuntimeException("Failed to close file " + filename, ioe2); 332 } 333 } 334 Log.d(TAG, "Saved " + width + "x" + height + " frame as '" + filename + "'"); 335 } 336 } 337