1 /* 2 * libjingle 3 * Copyright 2015 Google Inc. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions are met: 7 * 8 * 1. Redistributions of source code must retain the above copyright notice, 9 * this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright notice, 11 * this list of conditions and the following disclaimer in the documentation 12 * and/or other materials provided with the distribution. 13 * 3. The name of the author may not be used to endorse or promote products 14 * derived from this software without specific prior written permission. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED 17 * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 18 * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO 19 * EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 20 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 21 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 22 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 23 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 24 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 25 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 */ 27 28 package org.webrtc; 29 30 import android.opengl.GLES11Ext; 31 import android.opengl.GLES20; 32 33 import org.webrtc.GlShader; 34 import org.webrtc.GlUtil; 35 36 import java.nio.ByteBuffer; 37 import java.nio.FloatBuffer; 38 import java.util.Arrays; 39 import java.util.IdentityHashMap; 40 import java.util.Map; 41 42 /** 43 * Helper class to draw an opaque quad on the target viewport location. Rotation, mirror, and 44 * cropping is specified using a 4x4 texture coordinate transform matrix. The frame input can either 45 * be an OES texture or YUV textures in I420 format. The GL state must be preserved between draw 46 * calls, this is intentional to maximize performance. The function release() must be called 47 * manually to free the resources held by this object. 48 */ 49 public class GlRectDrawer implements RendererCommon.GlDrawer { 50 // Simple vertex shader, used for both YUV and OES. 51 private static final String VERTEX_SHADER_STRING = 52 "varying vec2 interp_tc;\n" 53 + "attribute vec4 in_pos;\n" 54 + "attribute vec4 in_tc;\n" 55 + "\n" 56 + "uniform mat4 texMatrix;\n" 57 + "\n" 58 + "void main() {\n" 59 + " gl_Position = in_pos;\n" 60 + " interp_tc = (texMatrix * in_tc).xy;\n" 61 + "}\n"; 62 63 private static final String YUV_FRAGMENT_SHADER_STRING = 64 "precision mediump float;\n" 65 + "varying vec2 interp_tc;\n" 66 + "\n" 67 + "uniform sampler2D y_tex;\n" 68 + "uniform sampler2D u_tex;\n" 69 + "uniform sampler2D v_tex;\n" 70 + "\n" 71 + "void main() {\n" 72 // CSC according to http://www.fourcc.org/fccyvrgb.php 73 + " float y = texture2D(y_tex, interp_tc).r;\n" 74 + " float u = texture2D(u_tex, interp_tc).r - 0.5;\n" 75 + " float v = texture2D(v_tex, interp_tc).r - 0.5;\n" 76 + " gl_FragColor = vec4(y + 1.403 * v, " 77 + " y - 0.344 * u - 0.714 * v, " 78 + " y + 1.77 * u, 1);\n" 79 + "}\n"; 80 81 private static final String RGB_FRAGMENT_SHADER_STRING = 82 "precision mediump float;\n" 83 + "varying vec2 interp_tc;\n" 84 + "\n" 85 + "uniform sampler2D rgb_tex;\n" 86 + "\n" 87 + "void main() {\n" 88 + " gl_FragColor = texture2D(rgb_tex, interp_tc);\n" 89 + "}\n"; 90 91 private static final String OES_FRAGMENT_SHADER_STRING = 92 "#extension GL_OES_EGL_image_external : require\n" 93 + "precision mediump float;\n" 94 + "varying vec2 interp_tc;\n" 95 + "\n" 96 + "uniform samplerExternalOES oes_tex;\n" 97 + "\n" 98 + "void main() {\n" 99 + " gl_FragColor = texture2D(oes_tex, interp_tc);\n" 100 + "}\n"; 101 102 // Vertex coordinates in Normalized Device Coordinates, i.e. (-1, -1) is bottom-left and (1, 1) is 103 // top-right. 104 private static final FloatBuffer FULL_RECTANGLE_BUF = 105 GlUtil.createFloatBuffer(new float[] { 106 -1.0f, -1.0f, // Bottom left. 107 1.0f, -1.0f, // Bottom right. 108 -1.0f, 1.0f, // Top left. 109 1.0f, 1.0f, // Top right. 110 }); 111 112 // Texture coordinates - (0, 0) is bottom-left and (1, 1) is top-right. 113 private static final FloatBuffer FULL_RECTANGLE_TEX_BUF = 114 GlUtil.createFloatBuffer(new float[] { 115 0.0f, 0.0f, // Bottom left. 116 1.0f, 0.0f, // Bottom right. 117 0.0f, 1.0f, // Top left. 118 1.0f, 1.0f // Top right. 119 }); 120 121 private static class Shader { 122 public final GlShader glShader; 123 public final int texMatrixLocation; 124 Shader(String fragmentShader)125 public Shader(String fragmentShader) { 126 this.glShader = new GlShader(VERTEX_SHADER_STRING, fragmentShader); 127 this.texMatrixLocation = glShader.getUniformLocation("texMatrix"); 128 } 129 } 130 131 // The keys are one of the fragments shaders above. 132 private final Map<String, Shader> shaders = new IdentityHashMap<String, Shader>(); 133 134 /** 135 * Draw an OES texture frame with specified texture transformation matrix. Required resources are 136 * allocated at the first call to this function. 137 */ 138 @Override drawOes(int oesTextureId, float[] texMatrix, int x, int y, int width, int height)139 public void drawOes(int oesTextureId, float[] texMatrix, int x, int y, int width, int height) { 140 prepareShader(OES_FRAGMENT_SHADER_STRING, texMatrix); 141 GLES20.glActiveTexture(GLES20.GL_TEXTURE0); 142 // updateTexImage() may be called from another thread in another EGL context, so we need to 143 // bind/unbind the texture in each draw call so that GLES understads it's a new texture. 144 GLES20.glBindTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, oesTextureId); 145 drawRectangle(x, y, width, height); 146 GLES20.glBindTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, 0); 147 } 148 149 /** 150 * Draw a RGB(A) texture frame with specified texture transformation matrix. Required resources 151 * are allocated at the first call to this function. 152 */ 153 @Override drawRgb(int textureId, float[] texMatrix, int x, int y, int width, int height)154 public void drawRgb(int textureId, float[] texMatrix, int x, int y, int width, int height) { 155 prepareShader(RGB_FRAGMENT_SHADER_STRING, texMatrix); 156 GLES20.glActiveTexture(GLES20.GL_TEXTURE0); 157 GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, textureId); 158 drawRectangle(x, y, width, height); 159 // Unbind the texture as a precaution. 160 GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, 0); 161 } 162 163 /** 164 * Draw a YUV frame with specified texture transformation matrix. Required resources are 165 * allocated at the first call to this function. 166 */ 167 @Override drawYuv(int[] yuvTextures, float[] texMatrix, int x, int y, int width, int height)168 public void drawYuv(int[] yuvTextures, float[] texMatrix, int x, int y, int width, int height) { 169 prepareShader(YUV_FRAGMENT_SHADER_STRING, texMatrix); 170 // Bind the textures. 171 for (int i = 0; i < 3; ++i) { 172 GLES20.glActiveTexture(GLES20.GL_TEXTURE0 + i); 173 GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, yuvTextures[i]); 174 } 175 drawRectangle(x, y, width, height); 176 // Unbind the textures as a precaution.. 177 for (int i = 0; i < 3; ++i) { 178 GLES20.glActiveTexture(GLES20.GL_TEXTURE0 + i); 179 GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, 0); 180 } 181 } 182 drawRectangle(int x, int y, int width, int height)183 private void drawRectangle(int x, int y, int width, int height) { 184 // Draw quad. 185 GLES20.glViewport(x, y, width, height); 186 GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, 0, 4); 187 } 188 prepareShader(String fragmentShader, float[] texMatrix)189 private void prepareShader(String fragmentShader, float[] texMatrix) { 190 final Shader shader; 191 if (shaders.containsKey(fragmentShader)) { 192 shader = shaders.get(fragmentShader); 193 } else { 194 // Lazy allocation. 195 shader = new Shader(fragmentShader); 196 shaders.put(fragmentShader, shader); 197 shader.glShader.useProgram(); 198 // Initialize fragment shader uniform values. 199 if (fragmentShader == YUV_FRAGMENT_SHADER_STRING) { 200 GLES20.glUniform1i(shader.glShader.getUniformLocation("y_tex"), 0); 201 GLES20.glUniform1i(shader.glShader.getUniformLocation("u_tex"), 1); 202 GLES20.glUniform1i(shader.glShader.getUniformLocation("v_tex"), 2); 203 } else if (fragmentShader == RGB_FRAGMENT_SHADER_STRING) { 204 GLES20.glUniform1i(shader.glShader.getUniformLocation("rgb_tex"), 0); 205 } else if (fragmentShader == OES_FRAGMENT_SHADER_STRING) { 206 GLES20.glUniform1i(shader.glShader.getUniformLocation("oes_tex"), 0); 207 } else { 208 throw new IllegalStateException("Unknown fragment shader: " + fragmentShader); 209 } 210 GlUtil.checkNoGLES2Error("Initialize fragment shader uniform values."); 211 // Initialize vertex shader attributes. 212 shader.glShader.setVertexAttribArray("in_pos", 2, FULL_RECTANGLE_BUF); 213 shader.glShader.setVertexAttribArray("in_tc", 2, FULL_RECTANGLE_TEX_BUF); 214 } 215 shader.glShader.useProgram(); 216 // Copy the texture transformation matrix over. 217 GLES20.glUniformMatrix4fv(shader.texMatrixLocation, 1, false, texMatrix, 0); 218 } 219 220 /** 221 * Release all GLES resources. This needs to be done manually, otherwise the resources are leaked. 222 */ 223 @Override release()224 public void release() { 225 for (Shader shader : shaders.values()) { 226 shader.glShader.release(); 227 } 228 shaders.clear(); 229 } 230 } 231