1 /* 2 * Copyright 2022 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 package com.google.android.exoplayer2.transformer; 17 18 import static com.google.android.exoplayer2.util.Assertions.checkStateNotNull; 19 20 import android.content.Context; 21 import android.opengl.GLES20; 22 import android.util.Size; 23 import com.google.android.exoplayer2.util.GlProgram; 24 import com.google.android.exoplayer2.util.GlUtil; 25 import java.io.IOException; 26 import org.checkerframework.checker.nullness.qual.MonotonicNonNull; 27 28 /** Copies frames from an external texture and applies color transformations for HDR if needed. */ 29 /* package */ class ExternalCopyFrameProcessor implements GlFrameProcessor { 30 31 static { 32 GlUtil.glAssertionsEnabled = true; 33 } 34 35 private static final String VERTEX_SHADER_TEX_TRANSFORM_PATH = 36 "shaders/vertex_shader_tex_transform_es2.glsl"; 37 private static final String VERTEX_SHADER_TEX_TRANSFORM_ES3_PATH = 38 "shaders/vertex_shader_tex_transform_es3.glsl"; 39 private static final String FRAGMENT_SHADER_COPY_EXTERNAL_PATH = 40 "shaders/fragment_shader_copy_external_es2.glsl"; 41 private static final String FRAGMENT_SHADER_COPY_EXTERNAL_YUV_ES3_PATH = 42 "shaders/fragment_shader_copy_external_yuv_es3.glsl"; 43 // Color transform coefficients from 44 // https://cs.android.com/android/platform/superproject/+/master:frameworks/av/media/libstagefright/colorconversion/ColorConverter.cpp;l=668-670;drc=487adf977a50cac3929eba15fad0d0f461c7ff0f. 45 private static final float[] MATRIX_YUV_TO_BT2020_COLOR_TRANSFORM = { 46 1.168f, 1.168f, 1.168f, 47 0.0f, -0.188f, 2.148f, 48 1.683f, -0.652f, 0.0f, 49 }; 50 51 private final Context context; 52 private final boolean enableExperimentalHdrEditing; 53 54 private @MonotonicNonNull Size size; 55 private @MonotonicNonNull GlProgram glProgram; 56 ExternalCopyFrameProcessor(Context context, boolean enableExperimentalHdrEditing)57 public ExternalCopyFrameProcessor(Context context, boolean enableExperimentalHdrEditing) { 58 this.context = context; 59 this.enableExperimentalHdrEditing = enableExperimentalHdrEditing; 60 } 61 62 @Override initialize(int inputTexId, int inputWidth, int inputHeight)63 public void initialize(int inputTexId, int inputWidth, int inputHeight) throws IOException { 64 size = new Size(inputWidth, inputHeight); 65 // TODO(b/205002913): check the loaded program is consistent with the attributes and uniforms 66 // expected in the code. 67 String vertexShaderFilePath = 68 enableExperimentalHdrEditing 69 ? VERTEX_SHADER_TEX_TRANSFORM_ES3_PATH 70 : VERTEX_SHADER_TEX_TRANSFORM_PATH; 71 String fragmentShaderFilePath = 72 enableExperimentalHdrEditing 73 ? FRAGMENT_SHADER_COPY_EXTERNAL_YUV_ES3_PATH 74 : FRAGMENT_SHADER_COPY_EXTERNAL_PATH; 75 glProgram = new GlProgram(context, vertexShaderFilePath, fragmentShaderFilePath); 76 glProgram.setSamplerTexIdUniform("uTexSampler", inputTexId, /* texUnitIndex= */ 0); 77 // Draw the frame on the entire normalized device coordinate space, from -1 to 1, for x and y. 78 glProgram.setBufferAttribute( 79 "aFramePosition", GlUtil.getNormalizedCoordinateBounds(), GlUtil.RECTANGLE_VERTICES_COUNT); 80 glProgram.setBufferAttribute( 81 "aTexSamplingCoord", GlUtil.getTextureCoordinateBounds(), GlUtil.RECTANGLE_VERTICES_COUNT); 82 if (enableExperimentalHdrEditing) { 83 // In HDR editing mode the decoder output is sampled in YUV. 84 glProgram.setFloatsUniform("uColorTransform", MATRIX_YUV_TO_BT2020_COLOR_TRANSFORM); 85 } 86 } 87 88 @Override getOutputSize()89 public Size getOutputSize() { 90 return checkStateNotNull(size); 91 } 92 93 /** 94 * Sets the texture transform matrix for converting an external surface texture's coordinates to 95 * sampling locations. 96 * 97 * @param textureTransformMatrix The external surface texture's {@linkplain 98 * android.graphics.SurfaceTexture#getTransformMatrix(float[]) transform matrix}. 99 */ setTextureTransformMatrix(float[] textureTransformMatrix)100 public void setTextureTransformMatrix(float[] textureTransformMatrix) { 101 checkStateNotNull(glProgram); 102 glProgram.setFloatsUniform("uTexTransform", textureTransformMatrix); 103 } 104 105 @Override updateProgramAndDraw(long presentationTimeUs)106 public void updateProgramAndDraw(long presentationTimeUs) { 107 checkStateNotNull(glProgram); 108 glProgram.use(); 109 glProgram.bindAttributesAndUniforms(); 110 // The four-vertex triangle strip forms a quad. 111 GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, /* first= */ 0, /* count= */ 4); 112 } 113 114 @Override release()115 public void release() { 116 if (glProgram != null) { 117 glProgram.delete(); 118 } 119 } 120 } 121