1 /*
2 * Copyright (C) 2016 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 #include "YUVConverter.h"
18
19 #include "DispatchTables.h"
20 #include "host-common/feature_control.h"
21 #include <assert.h>
22 #include <stdio.h>
23 #include <string.h>
24
25 #define FATAL(fmt,...) do { \
26 fprintf(stderr, "%s: FATAL: " fmt "\n", __func__, ##__VA_ARGS__); \
27 assert(false); \
28 } while(0)
29
30 #define YUV_CONVERTER_DEBUG 0
31
32 #if YUV_CONVERTER_DEBUG
33 #define DDD(fmt, ...) \
34 fprintf(stderr, "yuv-converter: %s:%d " fmt "\n", __func__, __LINE__, \
35 ##__VA_ARGS__);
36 #else
37 #define DDD(fmt, ...)
38 #endif
39
40 enum YUVInterleaveDirection {
41 YUVInterleaveDirectionVU = 0,
42 YUVInterleaveDirectionUV = 1,
43 };
44
45 // getYUVOffsets(), given a YUV-formatted buffer that is arranged
46 // according to the spec
47 // https://developer.android.com/reference/android/graphics/ImageFormat.html#YUV
48 // In particular, Android YUV widths are aligned to 16 pixels.
49 // Inputs:
50 // |yv12|: the YUV-formatted buffer
51 // Outputs:
52 // |yoff|: offset into |yv12| of the start of the Y component
53 // |uoff|: offset into |yv12| of the start of the U component
54 // |voff|: offset into |yv12| of the start of the V component
getYUVOffsets(int width,int height,FrameworkFormat format,uint32_t * yoff,uint32_t * uoff,uint32_t * voff,uint32_t * alignwidth,uint32_t * alignwidthc)55 static void getYUVOffsets(int width, int height, FrameworkFormat format,
56 uint32_t* yoff, uint32_t* uoff, uint32_t* voff,
57 uint32_t* alignwidth, uint32_t* alignwidthc) {
58 uint32_t yStride, cStride, cHeight, cSize, align;
59 switch (format) {
60 case FRAMEWORK_FORMAT_YV12:
61 // Luma stride is 32 bytes aligned.
62 align = 32;
63 yStride = (width + (align - 1)) & ~(align - 1);
64 // Chroma stride is 16 bytes aligned.
65 align = 16;
66 cStride = (yStride / 2 + (align - 1)) & ~(align - 1);
67 cHeight = height / 2;
68 cSize = cStride * cHeight;
69 *yoff = 0;
70 *voff = yStride * height;
71 *uoff = (*voff) + cSize;
72 *alignwidth = yStride;
73 *alignwidthc = cStride;
74 break;
75 case FRAMEWORK_FORMAT_YUV_420_888:
76 if (feature_is_enabled(
77 kFeature_YUV420888toNV21)) {
78 align = 1;
79 yStride = (width + (align - 1)) & ~(align - 1);
80 cStride = yStride;
81 cHeight = height / 2;
82 *yoff = 0;
83 *voff = yStride * height;
84 *uoff = (*voff) + 1;
85 *alignwidth = yStride;
86 *alignwidthc = cStride / 2;
87 } else {
88 align = 1;
89 yStride = (width + (align - 1)) & ~(align - 1);
90 cStride = (yStride / 2 + (align - 1)) & ~(align - 1);
91 cHeight = height / 2;
92 cSize = cStride * cHeight;
93 *yoff = 0;
94 *uoff = yStride * height;
95 *voff = (*uoff) + cSize;
96 *alignwidth = yStride;
97 *alignwidthc = cStride;
98 }
99 break;
100 case FRAMEWORK_FORMAT_NV12:
101 align = 1;
102 yStride = width;
103 cStride = yStride;
104 cHeight = height / 2;
105 cSize = cStride * cHeight;
106 *yoff = 0;
107 *uoff = yStride * height;
108 *voff = (*uoff) + 1;
109 *alignwidth = yStride;
110 *alignwidthc = cStride / 2;
111 break;
112 case FRAMEWORK_FORMAT_GL_COMPATIBLE:
113 FATAL("Input not a YUV format! (FRAMEWORK_FORMAT_GL_COMPATIBLE)");
114 default:
115 FATAL("Unknown format: 0x%x", format);
116 }
117 }
118
119 // createYUVGLTex() allocates GPU memory that is enough
120 // to hold the raw data of the YV12 buffer.
121 // The memory is in the form of an OpenGL texture
122 // with one component (GL_LUMINANCE) and
123 // of type GL_UNSIGNED_BYTE.
124 // In order to process all Y, U, V components
125 // simultaneously in conversion, the simple thing to do
126 // is to use multiple texture units, hence
127 // the |texture_unit| argument.
128 // Returns a new OpenGL texture object in |texName_out|
129 // that is to be cleaned up by the caller.
createYUVGLTex(GLenum texture_unit,GLsizei width,GLsizei height,GLuint * texName_out,bool uvInterleaved)130 void YUVConverter::createYUVGLTex(GLenum texture_unit,
131 GLsizei width,
132 GLsizei height,
133 GLuint* texName_out,
134 bool uvInterleaved) {
135 assert(texName_out);
136
137 s_gles2.glActiveTexture(texture_unit);
138 s_gles2.glGenTextures(1, texName_out);
139 s_gles2.glBindTexture(GL_TEXTURE_2D, *texName_out);
140 s_gles2.glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST);
141 s_gles2.glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST);
142 GLint unprevAlignment = 0;
143 s_gles2.glGetIntegerv(GL_UNPACK_ALIGNMENT, &unprevAlignment);
144 s_gles2.glPixelStorei(GL_UNPACK_ALIGNMENT, 1);
145 if (uvInterleaved) {
146 s_gles2.glTexImage2D(GL_TEXTURE_2D, 0, GL_LUMINANCE_ALPHA,
147 width, height, 0,
148 GL_LUMINANCE_ALPHA, GL_UNSIGNED_BYTE,
149 NULL);
150 } else {
151 s_gles2.glTexImage2D(GL_TEXTURE_2D, 0, GL_LUMINANCE,
152 width, height, 0,
153 GL_LUMINANCE, GL_UNSIGNED_BYTE,
154 NULL);
155 }
156 s_gles2.glPixelStorei(GL_UNPACK_ALIGNMENT, unprevAlignment);
157 s_gles2.glActiveTexture(GL_TEXTURE0);
158 }
159
readYUVTex(GLuint tex,void * pixels,bool uvInterleaved)160 static void readYUVTex(GLuint tex, void* pixels, bool uvInterleaved) {
161 GLuint prevTexture = 0;
162 s_gles2.glGetIntegerv(GL_TEXTURE_BINDING_2D, (GLint*)&prevTexture);
163 s_gles2.glBindTexture(GL_TEXTURE_2D, tex);
164 GLint prevAlignment = 0;
165 s_gles2.glGetIntegerv(GL_PACK_ALIGNMENT, &prevAlignment);
166 s_gles2.glPixelStorei(GL_PACK_ALIGNMENT, 1);
167 if (uvInterleaved) {
168 if (s_gles2.glGetTexImage) {
169 s_gles2.glGetTexImage(GL_TEXTURE_2D, 0, GL_LUMINANCE_ALPHA,
170 GL_UNSIGNED_BYTE, pixels);
171 } else {
172 DDD("empty glGetTexImage");
173 }
174 } else {
175 if (s_gles2.glGetTexImage) {
176 s_gles2.glGetTexImage(GL_TEXTURE_2D, 0, GL_LUMINANCE,
177 GL_UNSIGNED_BYTE, pixels);
178 } else {
179 DDD("empty glGetTexImage");
180 }
181 }
182 s_gles2.glPixelStorei(GL_PACK_ALIGNMENT, prevAlignment);
183 s_gles2.glBindTexture(GL_TEXTURE_2D, prevTexture);
184 }
185
186 // subUpdateYUVGLTex() updates a given YUV texture
187 // at the coordinates (x, y, width, height),
188 // with the raw YUV data in |pixels|.
189 // We cannot view the result properly until
190 // after conversion; this is to be used only
191 // as input to the conversion shader.
subUpdateYUVGLTex(GLenum texture_unit,GLuint tex,int x,int y,int width,int height,void * pixels,bool uvInterleaved)192 static void subUpdateYUVGLTex(GLenum texture_unit,
193 GLuint tex,
194 int x, int y, int width, int height,
195 void* pixels, bool uvInterleaved) {
196 s_gles2.glActiveTexture(texture_unit);
197 s_gles2.glBindTexture(GL_TEXTURE_2D, tex);
198 GLint unprevAlignment = 0;
199 s_gles2.glGetIntegerv(GL_UNPACK_ALIGNMENT, &unprevAlignment);
200 s_gles2.glPixelStorei(GL_UNPACK_ALIGNMENT, 1);
201
202 if (uvInterleaved) {
203 s_gles2.glTexSubImage2D(GL_TEXTURE_2D, 0,
204 x, y, width, height,
205 GL_LUMINANCE_ALPHA, GL_UNSIGNED_BYTE,
206 pixels);
207 } else {
208 s_gles2.glTexSubImage2D(GL_TEXTURE_2D, 0,
209 x, y, width, height,
210 GL_LUMINANCE, GL_UNSIGNED_BYTE,
211 pixels);
212 }
213 s_gles2.glPixelStorei(GL_UNPACK_ALIGNMENT, unprevAlignment);
214
215 s_gles2.glActiveTexture(GL_TEXTURE0);
216 }
217
218 // createYUVGLShader() defines the vertex/fragment
219 // shader that does the actual work of converting
220 // YUV to RGB. The resulting program is stored in |program_out|.
createYUVGLShader(GLuint * program_out,GLint * ywidthcutoffloc_out,GLint * cwidthcutoffloc_out,GLint * ysamplerloc_out,GLint * usamplerloc_out,GLint * vsamplerloc_out,GLint * incoordloc_out,GLint * posloc_out)221 static void createYUVGLShader(GLuint* program_out,
222 GLint* ywidthcutoffloc_out,
223 GLint* cwidthcutoffloc_out,
224 GLint* ysamplerloc_out,
225 GLint* usamplerloc_out,
226 GLint* vsamplerloc_out,
227 GLint* incoordloc_out,
228 GLint* posloc_out) {
229 assert(program_out);
230
231 static const char kVShader[] = R"(
232 precision highp float;
233 attribute mediump vec4 position;
234 attribute highp vec2 inCoord;
235 varying highp vec2 outCoord;
236 void main(void) {
237 gl_Position = position;
238 outCoord = inCoord;
239 }
240 )";
241 const GLchar* const kVShaders =
242 static_cast<const GLchar*>(kVShader);
243
244 // Based on:
245 // http://stackoverflow.com/questions/11093061/yv12-to-rgb-using-glsl-in-ios-result-image-attached
246 // + account for 16-pixel alignment using |yWidthCutoff| / |cWidthCutoff|
247 // + use conversion matrix in
248 // frameworks/av/media/libstagefright/colorconversion/ColorConverter.cpp (YUV420p)
249 // + more precision from
250 // https://en.wikipedia.org/wiki/YCbCr#ITU-R_BT.601_conversion
251 static const char kFShader[] = R"(
252 precision highp float;
253 varying highp vec2 outCoord;
254 uniform highp float yWidthCutoff;
255 uniform highp float cWidthCutoff;
256 uniform sampler2D ysampler;
257 uniform sampler2D usampler;
258 uniform sampler2D vsampler;
259 void main(void) {
260 highp vec2 cutoffCoordsY;
261 highp vec2 cutoffCoordsC;
262 highp vec3 yuv;
263 highp vec3 rgb;
264 cutoffCoordsY.x = outCoord.x * yWidthCutoff;
265 cutoffCoordsY.y = outCoord.y;
266 cutoffCoordsC.x = outCoord.x * cWidthCutoff;
267 cutoffCoordsC.y = outCoord.y;
268 yuv[0] = texture2D(ysampler, cutoffCoordsY).r - 0.0625;
269 yuv[1] = 0.96*(texture2D(usampler, cutoffCoordsC).r - 0.5);
270 yuv[2] = texture2D(vsampler, cutoffCoordsC).r - 0.5;
271 highp float yscale = 1.1643835616438356;
272 rgb = mat3(yscale, yscale, yscale,
273 0, -0.39176229009491365, 2.017232142857143,
274 1.5960267857142856, -0.8129676472377708, 0) * yuv;
275 gl_FragColor = vec4(rgb, 1);
276 }
277 )";
278
279 const GLchar* const kFShaders =
280 static_cast<const GLchar*>(kFShader);
281
282 GLuint vshader = s_gles2.glCreateShader(GL_VERTEX_SHADER);
283 GLuint fshader = s_gles2.glCreateShader(GL_FRAGMENT_SHADER);
284
285 const GLint vtextLen = strlen(kVShader);
286 const GLint ftextLen = strlen(kFShaders);
287 s_gles2.glShaderSource(vshader, 1, &kVShaders, &vtextLen);
288 s_gles2.glShaderSource(fshader, 1, &kFShaders, &ftextLen);
289 s_gles2.glCompileShader(vshader);
290 s_gles2.glCompileShader(fshader);
291
292 *program_out = s_gles2.glCreateProgram();
293 s_gles2.glAttachShader(*program_out, vshader);
294 s_gles2.glAttachShader(*program_out, fshader);
295 s_gles2.glLinkProgram(*program_out);
296
297 *ywidthcutoffloc_out = s_gles2.glGetUniformLocation(*program_out, "yWidthCutoff");
298 *cwidthcutoffloc_out = s_gles2.glGetUniformLocation(*program_out, "cWidthCutoff");
299 *ysamplerloc_out = s_gles2.glGetUniformLocation(*program_out, "ysampler");
300 *usamplerloc_out = s_gles2.glGetUniformLocation(*program_out, "usampler");
301 *vsamplerloc_out = s_gles2.glGetUniformLocation(*program_out, "vsampler");
302 *posloc_out = s_gles2.glGetAttribLocation(*program_out, "position");
303 *incoordloc_out = s_gles2.glGetAttribLocation(*program_out, "inCoord");
304
305 s_gles2.glDeleteShader(vshader);
306 s_gles2.glDeleteShader(fshader);
307 }
308
createYUVInterleavedGLShader(GLuint * program_out,GLint * ywidthcutoffloc_out,GLint * cwidthcutoffloc_out,GLint * ysamplerloc_out,GLint * vusamplerloc_out,GLint * incoordloc_out,GLint * posloc_out,YUVInterleaveDirection interleaveDir)309 static void createYUVInterleavedGLShader(GLuint* program_out,
310 GLint* ywidthcutoffloc_out,
311 GLint* cwidthcutoffloc_out,
312 GLint* ysamplerloc_out,
313 GLint* vusamplerloc_out,
314 GLint* incoordloc_out,
315 GLint* posloc_out,
316 YUVInterleaveDirection interleaveDir) {
317 assert(program_out);
318
319 static const char kVShader[] = R"(
320 precision highp float;
321 attribute mediump vec4 position;
322 attribute highp vec2 inCoord;
323 varying highp vec2 outCoord;
324 void main(void) {
325 gl_Position = position;
326 outCoord = inCoord;
327 }
328 )";
329 const GLchar* const kVShaders =
330 static_cast<const GLchar*>(kVShader);
331
332 // Based on:
333 // https://stackoverflow.com/questions/22456884/how-to-render-androids-yuv-nv21-camera-image-on-the-background-in-libgdx-with-o
334 // + account for 16-pixel alignment using |yWidthCutoff| / |cWidthCutoff|
335 // + use conversion matrix in
336 // frameworks/av/media/libstagefright/colorconversion/ColorConverter.cpp (YUV420p)
337 // + more precision from
338 // https://en.wikipedia.org/wiki/YCbCr#ITU-R_BT.601_conversion
339 // UV texture is (width/2*height/2) in size (downsampled by 2 in
340 // both dimensions, each pixel corresponds to 4 pixels of the Y channel)
341 // and each pixel is two bytes. By setting GL_LUMINANCE_ALPHA, OpenGL
342 // puts first byte (V) into R,G and B components and of the texture
343 // and the second byte (U) into the A component of the texture. That's
344 // why we find U and V at A and R respectively in the fragment shader code.
345 // Note that we could have also found V at G or B as well.
346 static const char kFShaderVu[] = R"(
347 precision highp float;
348 varying highp vec2 outCoord;
349 uniform highp float yWidthCutoff;
350 uniform highp float cWidthCutoff;
351 uniform sampler2D ysampler;
352 uniform sampler2D vusampler;
353 void main(void) {
354 highp vec2 cutoffCoordsY;
355 highp vec2 cutoffCoordsC;
356 highp vec3 yuv;
357 highp vec3 rgb;
358 cutoffCoordsY.x = outCoord.x * yWidthCutoff;
359 cutoffCoordsY.y = outCoord.y;
360 cutoffCoordsC.x = outCoord.x * cWidthCutoff;
361 cutoffCoordsC.y = outCoord.y;
362 yuv[0] = texture2D(ysampler, cutoffCoordsY).r - 0.0625;
363 yuv[1] = 0.96 * (texture2D(vusampler, cutoffCoordsC).a - 0.5);
364 yuv[2] = texture2D(vusampler, cutoffCoordsC).r - 0.5;
365 highp float yscale = 1.1643835616438356;
366 rgb = mat3(yscale, yscale, yscale,
367 0, -0.39176229009491365, 2.017232142857143,
368 1.5960267857142856, -0.8129676472377708, 0) * yuv;
369 gl_FragColor = vec4(rgb, 1);
370 }
371 )";
372
373 static const char kFShaderUv[] = R"(
374 precision highp float;
375 varying highp vec2 outCoord;
376 uniform highp float yWidthCutoff;
377 uniform highp float cWidthCutoff;
378 uniform sampler2D ysampler;
379 uniform sampler2D uvsampler;
380 void main(void) {
381 highp vec2 cutoffCoordsY;
382 highp vec2 cutoffCoordsC;
383 highp vec3 yuv;
384 highp vec3 rgb;
385 cutoffCoordsY.x = outCoord.x * yWidthCutoff;
386 cutoffCoordsY.y = outCoord.y;
387 cutoffCoordsC.x = outCoord.x * cWidthCutoff;
388 cutoffCoordsC.y = outCoord.y;
389 yuv[0] = texture2D(ysampler, cutoffCoordsY).r - 0.0625;
390 yuv[1] = 0.96 * (texture2D(uvsampler, cutoffCoordsC).r - 0.5);
391 yuv[2] = (texture2D(uvsampler, cutoffCoordsC).a - 0.5);
392 highp float yscale = 1.1643835616438356;
393 rgb = mat3(yscale, yscale, yscale,
394 0, -0.39176229009491365, 2.017232142857143,
395 1.5960267857142856, -0.8129676472377708, 0) * yuv;
396 gl_FragColor = vec4(rgb, 1);
397 }
398 )";
399
400 const GLchar* const kFShaders =
401 interleaveDir == YUVInterleaveDirectionVU ? kFShaderVu : kFShaderUv;
402
403 GLuint vshader = s_gles2.glCreateShader(GL_VERTEX_SHADER);
404 GLuint fshader = s_gles2.glCreateShader(GL_FRAGMENT_SHADER);
405
406 const GLint vtextLen = strlen(kVShader);
407 const GLint ftextLen = strlen(kFShaders);
408 s_gles2.glShaderSource(vshader, 1, &kVShaders, &vtextLen);
409 s_gles2.glShaderSource(fshader, 1, &kFShaders, &ftextLen);
410 s_gles2.glCompileShader(vshader);
411 s_gles2.glCompileShader(fshader);
412
413 *program_out = s_gles2.glCreateProgram();
414 s_gles2.glAttachShader(*program_out, vshader);
415 s_gles2.glAttachShader(*program_out, fshader);
416 s_gles2.glLinkProgram(*program_out);
417
418 *ywidthcutoffloc_out = s_gles2.glGetUniformLocation(*program_out, "yWidthCutoff");
419 *cwidthcutoffloc_out = s_gles2.glGetUniformLocation(*program_out, "cWidthCutoff");
420 *ysamplerloc_out = s_gles2.glGetUniformLocation(*program_out, "ysampler");
421 *vusamplerloc_out = s_gles2.glGetUniformLocation(
422 *program_out, YUVInterleaveDirectionVU ? "vusampler" : "uvsampler");
423 *posloc_out = s_gles2.glGetAttribLocation(*program_out, "position");
424 *incoordloc_out = s_gles2.glGetAttribLocation(*program_out, "inCoord");
425
426 s_gles2.glDeleteShader(vshader);
427 s_gles2.glDeleteShader(fshader);
428 }
429 // When converting YUV to RGB with shaders,
430 // we are using the OpenGL graphics pipeline to do compute,
431 // so we need to express the place to store the result
432 // with triangles and draw calls.
433 // createYUVGLFullscreenQuad() defines a fullscreen quad
434 // with position and texture coordinates.
435 // The quad will be textured with the resulting RGB colors,
436 // and we will read back the pixels from the framebuffer
437 // to retrieve our RGB result.
createYUVGLFullscreenQuad(GLuint * vbuf_out,GLuint * ibuf_out,int picture_width,int aligned_width)438 static void createYUVGLFullscreenQuad(GLuint* vbuf_out,
439 GLuint* ibuf_out,
440 int picture_width,
441 int aligned_width) {
442 assert(vbuf_out);
443 assert(ibuf_out);
444
445 s_gles2.glGenBuffers(1, vbuf_out);
446 s_gles2.glGenBuffers(1, ibuf_out);
447
448 static const float kVertices[] = {
449 +1, -1, +0, +1, +0,
450 +1, +1, +0, +1, +1,
451 -1, +1, +0, +0, +1,
452 -1, -1, +0, +0, +0,
453 };
454
455 static const GLubyte kIndices[] = { 0, 1, 2, 2, 3, 0 };
456
457 s_gles2.glBindBuffer(GL_ARRAY_BUFFER, *vbuf_out);
458 s_gles2.glBufferData(GL_ARRAY_BUFFER, sizeof(kVertices), kVertices,
459 GL_STATIC_DRAW);
460 s_gles2.glBindBuffer(GL_ELEMENT_ARRAY_BUFFER, *ibuf_out);
461 s_gles2.glBufferData(GL_ELEMENT_ARRAY_BUFFER, sizeof(kIndices), kIndices,
462 GL_STATIC_DRAW);
463 }
464
465 // doYUVConversionDraw() does the actual work of setting up
466 // and submitting draw commands to the GPU.
467 // It uses the textures, shaders, and fullscreen quad defined above
468 // and executes the pipeline on them.
469 // Note, however, that it is up to the caller to dig out
470 // the result of the draw.
doYUVConversionDraw(GLuint program,GLint yWidthCutoffLoc,GLint cWidthCutoffLoc,GLint ySamplerLoc,GLint uSamplerLoc,GLint vSamplerLoc,GLint vuSamplerLoc,GLint inCoordLoc,GLint posLoc,GLuint vbuf,GLuint ibuf,int width,int ywidth,int halfwidth,int cwidth,float yWidthCutoff,float cWidthCutoff,bool uvInterleaved)471 static void doYUVConversionDraw(GLuint program,
472 GLint yWidthCutoffLoc,
473 GLint cWidthCutoffLoc,
474 GLint ySamplerLoc,
475 GLint uSamplerLoc,
476 GLint vSamplerLoc,
477 GLint vuSamplerLoc,
478 GLint inCoordLoc,
479 GLint posLoc,
480 GLuint vbuf, GLuint ibuf,
481 int width, int ywidth,
482 int halfwidth, int cwidth,
483 float yWidthCutoff,
484 float cWidthCutoff,
485 bool uvInterleaved) {
486
487 const GLsizei kVertexAttribStride = 5 * sizeof(GL_FLOAT);
488 const GLvoid* kVertexAttribPosOffset = (GLvoid*)0;
489 const GLvoid* kVertexAttribCoordOffset = (GLvoid*)(3 * sizeof(GL_FLOAT));
490
491 s_gles2.glUseProgram(program);
492
493 s_gles2.glUniform1f(yWidthCutoffLoc, yWidthCutoff);
494 s_gles2.glUniform1f(cWidthCutoffLoc, cWidthCutoff);
495
496 s_gles2.glUniform1i(ySamplerLoc, 0);
497 if (uvInterleaved) {
498 s_gles2.glUniform1i(vuSamplerLoc, 1);
499 } else {
500 s_gles2.glUniform1i(uSamplerLoc, 1);
501 s_gles2.glUniform1i(vSamplerLoc, 2);
502 }
503
504 s_gles2.glBindBuffer(GL_ARRAY_BUFFER, vbuf);
505 s_gles2.glEnableVertexAttribArray(posLoc);
506 s_gles2.glEnableVertexAttribArray(inCoordLoc);
507
508 s_gles2.glVertexAttribPointer(posLoc, 3, GL_FLOAT, false,
509 kVertexAttribStride,
510 kVertexAttribPosOffset);
511 s_gles2.glVertexAttribPointer(inCoordLoc, 2, GL_FLOAT, false,
512 kVertexAttribStride,
513 kVertexAttribCoordOffset);
514
515 s_gles2.glBindBuffer(GL_ELEMENT_ARRAY_BUFFER, ibuf);
516 s_gles2.glDrawElements(GL_TRIANGLES, 6, GL_UNSIGNED_BYTE, 0);
517
518 s_gles2.glDisableVertexAttribArray(posLoc);
519 s_gles2.glDisableVertexAttribArray(inCoordLoc);
520 }
521
522 // initialize(): allocate GPU memory for YUV components,
523 // and create shaders and vertex data.
YUVConverter(int width,int height,FrameworkFormat format)524 YUVConverter::YUVConverter(int width, int height, FrameworkFormat format)
525 : mWidth(width),
526 mHeight(height),
527 mFormat(format),
528 mCbFormat(format) {}
529
init(int width,int height,FrameworkFormat format)530 void YUVConverter::init(int width, int height, FrameworkFormat format) {
531 uint32_t yoff, uoff, voff, ywidth, cwidth, cheight;
532 getYUVOffsets(width, height, mFormat,
533 &yoff, &uoff, &voff,
534 &ywidth, &cwidth);
535 cheight = height / 2;
536
537 mWidth = width;
538 mHeight = height;
539 if (!mYtex)
540 createYUVGLTex(GL_TEXTURE0, ywidth, height, &mYtex, false);
541 switch (mFormat) {
542 case FRAMEWORK_FORMAT_YV12:
543 if (!mUtex)
544 createYUVGLTex(GL_TEXTURE1, cwidth, cheight, &mUtex, false);
545 if (!mVtex)
546 createYUVGLTex(GL_TEXTURE2, cwidth, cheight, &mVtex, false);
547 createYUVGLShader(&mProgram,
548 &mYWidthCutoffLoc,
549 &mCWidthCutoffLoc,
550 &mYSamplerLoc,
551 &mUSamplerLoc,
552 &mVSamplerLoc,
553 &mInCoordLoc,
554 &mPosLoc);
555 break;
556 case FRAMEWORK_FORMAT_YUV_420_888:
557 if (feature_is_enabled(
558 kFeature_YUV420888toNV21)) {
559 if (!mVUtex)
560 createYUVGLTex(GL_TEXTURE1, cwidth, cheight, &mVUtex, true);
561 createYUVInterleavedGLShader(&mProgram,
562 &mYWidthCutoffLoc,
563 &mCWidthCutoffLoc,
564 &mYSamplerLoc,
565 &mVUSamplerLoc,
566 &mInCoordLoc,
567 &mPosLoc,
568 YUVInterleaveDirectionVU);
569 } else {
570 if (!mUtex)
571 createYUVGLTex(GL_TEXTURE1, cwidth, cheight, &mUtex, false);
572 if (!mVtex)
573 createYUVGLTex(GL_TEXTURE2, cwidth, cheight, &mVtex, false);
574 createYUVGLShader(&mProgram,
575 &mYWidthCutoffLoc,
576 &mCWidthCutoffLoc,
577 &mYSamplerLoc,
578 &mUSamplerLoc,
579 &mVSamplerLoc,
580 &mInCoordLoc,
581 &mPosLoc);
582 }
583 break;
584 case FRAMEWORK_FORMAT_NV12:
585 if (!mUVtex)
586 createYUVGLTex(GL_TEXTURE1, cwidth, cheight, &mUVtex, true);
587 createYUVInterleavedGLShader(&mProgram,
588 &mYWidthCutoffLoc,
589 &mCWidthCutoffLoc,
590 &mYSamplerLoc,
591 &mVUSamplerLoc,
592 &mInCoordLoc,
593 &mPosLoc,
594 YUVInterleaveDirectionUV);
595 break;
596 default:
597 FATAL("Unknown format: 0x%x", mFormat);
598 }
599
600 createYUVGLFullscreenQuad(&mVbuf, &mIbuf, width, ywidth);
601 }
602
saveGLState()603 void YUVConverter::saveGLState() {
604 s_gles2.glGetFloatv(GL_VIEWPORT, mCurrViewport);
605 s_gles2.glGetIntegerv(GL_ACTIVE_TEXTURE, &mCurrTexUnit);
606 s_gles2.glGetIntegerv(GL_TEXTURE_BINDING_2D, &mCurrTexBind);
607 s_gles2.glGetIntegerv(GL_CURRENT_PROGRAM, &mCurrProgram);
608 s_gles2.glGetIntegerv(GL_ARRAY_BUFFER_BINDING, &mCurrVbo);
609 s_gles2.glGetIntegerv(GL_ELEMENT_ARRAY_BUFFER_BINDING, &mCurrIbo);
610 }
611
restoreGLState()612 void YUVConverter::restoreGLState() {
613 s_gles2.glViewport(mCurrViewport[0], mCurrViewport[1],
614 mCurrViewport[2], mCurrViewport[3]);
615 s_gles2.glActiveTexture(mCurrTexUnit);
616 s_gles2.glUseProgram(mCurrProgram);
617 s_gles2.glBindBuffer(GL_ARRAY_BUFFER, mCurrVbo);
618 s_gles2.glBindBuffer(GL_ELEMENT_ARRAY_BUFFER, mCurrIbo);
619 }
620
getDataSize()621 uint32_t YUVConverter::getDataSize() {
622 uint32_t align = (mFormat == FRAMEWORK_FORMAT_YV12) ? 16 : 1;
623 uint32_t yStride = (mWidth + (align - 1)) & ~(align - 1);
624 uint32_t uvStride = (yStride / 2 + (align - 1)) & ~(align - 1);
625 uint32_t uvHeight = mHeight / 2;
626 uint32_t dataSize = yStride * mHeight + 2 * (uvHeight * uvStride);
627 return dataSize;
628 }
629
readPixels(uint8_t * pixels,uint32_t pixels_size)630 void YUVConverter::readPixels(uint8_t* pixels, uint32_t pixels_size) {
631 int width = mWidth;
632 int height = mHeight;
633
634 DDD("calling %s %d\n texture: width %d height %d\n", __func__, __LINE__,
635 width, height);
636 {
637 uint32_t align = (mFormat == FRAMEWORK_FORMAT_YV12) ? 16 : 1;
638 uint32_t yStride = (width + (align - 1)) & ~(align - 1);
639 uint32_t uvStride = (yStride / 2 + (align - 1)) & ~(align - 1);
640 uint32_t uvHeight = height / 2;
641 uint32_t dataSize = yStride * height + 2 * (uvHeight * uvStride);
642 if (pixels_size != dataSize) {
643 DDD("failed %s %d\n", __func__, __LINE__);
644 return;
645 }
646 DDD("%s %d reading %d bytes\n", __func__, __LINE__, (int)dataSize);
647 }
648
649 uint32_t yoff, uoff, voff, ywidth, cwidth;
650 getYUVOffsets(width, height, mFormat, &yoff, &uoff, &voff, &ywidth,
651 &cwidth);
652
653 if (mFormat == FRAMEWORK_FORMAT_YUV_420_888) {
654 if (feature_is_enabled(
655 kFeature_YUV420888toNV21)) {
656 readYUVTex(mVUtex, pixels + voff, true);
657 DDD("done");
658 } else {
659 readYUVTex(mUtex, pixels + uoff, false);
660 readYUVTex(mVtex, pixels + voff, false);
661 DDD("done");
662 }
663 } else if (mFormat == FRAMEWORK_FORMAT_NV12) {
664 readYUVTex(mUVtex, pixels + uoff, true);
665 if (mCbFormat == FRAMEWORK_FORMAT_YUV_420_888) {
666 // do a conversion here inplace: NV12 to YUV 420 888
667 uint8_t* scrath_memory = pixels;
668 NV12ToYUV420PlanarInPlaceConvert(width, height, pixels,
669 scrath_memory);
670 DDD("done");
671 }
672 DDD("done");
673 } else if (mFormat == FRAMEWORK_FORMAT_YV12) {
674 readYUVTex(mUtex, pixels + uoff, false);
675 readYUVTex(mVtex, pixels + voff, false);
676 DDD("done");
677 }
678 // read Y the last, because we can might used it as a scratch space
679 readYUVTex(mYtex, pixels + yoff, false);
680 DDD("done");
681 }
682
swapTextures(uint32_t type,uint32_t * textures)683 void YUVConverter::swapTextures(uint32_t type, uint32_t* textures) {
684 if (type == FRAMEWORK_FORMAT_NV12) {
685 mFormat = FRAMEWORK_FORMAT_NV12;
686 std::swap(textures[0], mYtex);
687 std::swap(textures[1], mUVtex);
688 } else if (type == FRAMEWORK_FORMAT_YUV_420_888) {
689 mFormat = FRAMEWORK_FORMAT_YUV_420_888;
690 std::swap(textures[0], mYtex);
691 std::swap(textures[1], mUtex);
692 std::swap(textures[2], mVtex);
693 } else {
694 FATAL("Unknown format: 0x%x", type);
695 }
696 }
697
698 // drawConvert: per-frame updates.
699 // Update YUV textures, then draw the fullscreen
700 // quad set up above, which results in a framebuffer
701 // with the RGB colors.
drawConvert(int x,int y,int width,int height,char * pixels)702 void YUVConverter::drawConvert(int x, int y,
703 int width, int height,
704 char* pixels) {
705 saveGLState();
706 if (pixels && (width != mWidth || height != mHeight)) {
707 reset();
708 }
709
710 if (mProgram == 0) {
711 init(width, height, mFormat);
712 }
713 s_gles2.glViewport(x, y, width, height);
714 uint32_t yoff, uoff, voff, ywidth, cwidth, cheight;
715 getYUVOffsets(width, height, mFormat, &yoff, &uoff, &voff, &ywidth,
716 &cwidth);
717 cheight = height / 2;
718 updateCutoffs(width, ywidth, width / 2, cwidth);
719
720 if (!pixels) {
721 // special case: draw from texture, only support NV12 for now
722 // as cuvid's native format is NV12.
723 // TODO: add more formats if there are such needs in the future.
724 assert(mFormat == FRAMEWORK_FORMAT_NV12);
725 s_gles2.glActiveTexture(GL_TEXTURE1);
726 s_gles2.glBindTexture(GL_TEXTURE_2D, mUVtex);
727 s_gles2.glActiveTexture(GL_TEXTURE0);
728 s_gles2.glBindTexture(GL_TEXTURE_2D, mYtex);
729
730 doYUVConversionDraw(mProgram, mYWidthCutoffLoc, mCWidthCutoffLoc,
731 mYSamplerLoc, mUSamplerLoc, mVSamplerLoc,
732 mVUSamplerLoc, mInCoordLoc, mPosLoc, mVbuf, mIbuf,
733 width, ywidth, width / 2, cwidth, mYWidthCutoff,
734 mCWidthCutoff, true);
735
736 restoreGLState();
737 return;
738 }
739
740 subUpdateYUVGLTex(GL_TEXTURE0, mYtex,
741 x, y, ywidth, height,
742 pixels + yoff, false);
743
744 switch (mFormat) {
745 case FRAMEWORK_FORMAT_YV12:
746 subUpdateYUVGLTex(GL_TEXTURE1, mUtex,
747 x, y, cwidth, cheight,
748 pixels + uoff, false);
749 subUpdateYUVGLTex(GL_TEXTURE2, mVtex,
750 x, y, cwidth, cheight,
751 pixels + voff, false);
752 doYUVConversionDraw(mProgram,
753 mYWidthCutoffLoc,
754 mCWidthCutoffLoc,
755 mYSamplerLoc,
756 mUSamplerLoc,
757 mVSamplerLoc,
758 mVUSamplerLoc,
759 mInCoordLoc,
760 mPosLoc,
761 mVbuf, mIbuf,
762 width, ywidth,
763 width / 2, cwidth,
764 mYWidthCutoff,
765 mCWidthCutoff,
766 false);
767 break;
768 case FRAMEWORK_FORMAT_YUV_420_888:
769 if (feature_is_enabled(
770 kFeature_YUV420888toNV21)) {
771 subUpdateYUVGLTex(GL_TEXTURE1, mVUtex,
772 x, y, cwidth, cheight,
773 pixels + voff, true);
774 doYUVConversionDraw(mProgram,
775 mYWidthCutoffLoc,
776 mCWidthCutoffLoc,
777 mYSamplerLoc,
778 mUSamplerLoc,
779 mVSamplerLoc,
780 mVUSamplerLoc,
781 mInCoordLoc,
782 mPosLoc,
783 mVbuf, mIbuf,
784 width, ywidth,
785 width / 2, cwidth,
786 mYWidthCutoff,
787 mCWidthCutoff,
788 true);
789 } else {
790 subUpdateYUVGLTex(GL_TEXTURE1, mUtex,
791 x, y, cwidth, cheight,
792 pixels + uoff, false);
793 subUpdateYUVGLTex(GL_TEXTURE2, mVtex,
794 x, y, cwidth, cheight,
795 pixels + voff, false);
796 doYUVConversionDraw(mProgram,
797 mYWidthCutoffLoc,
798 mCWidthCutoffLoc,
799 mYSamplerLoc,
800 mUSamplerLoc,
801 mVSamplerLoc,
802 mVUSamplerLoc,
803 mInCoordLoc,
804 mPosLoc,
805 mVbuf, mIbuf,
806 width, ywidth,
807 width / 2, cwidth,
808 mYWidthCutoff,
809 mCWidthCutoff,
810 false);
811 }
812 break;
813 case FRAMEWORK_FORMAT_NV12:
814 subUpdateYUVGLTex(GL_TEXTURE1, mUVtex,
815 x, y, cwidth, cheight,
816 pixels + uoff, true);
817 doYUVConversionDraw(mProgram,
818 mYWidthCutoffLoc,
819 mCWidthCutoffLoc,
820 mYSamplerLoc,
821 mUSamplerLoc,
822 mVSamplerLoc,
823 mVUSamplerLoc,
824 mInCoordLoc,
825 mPosLoc,
826 mVbuf, mIbuf,
827 width, ywidth,
828 width / 2, cwidth,
829 mYWidthCutoff,
830 mCWidthCutoff,
831 true);
832 break;
833 default:
834 FATAL("Unknown format: 0x%x", mFormat);
835 }
836
837 restoreGLState();
838 }
839
updateCutoffs(float width,float ywidth,float halfwidth,float cwidth)840 void YUVConverter::updateCutoffs(float width, float ywidth,
841 float halfwidth, float cwidth) {
842 switch (mFormat) {
843 case FRAMEWORK_FORMAT_YV12:
844 mYWidthCutoff = ((float)width) / ((float)ywidth);
845 mCWidthCutoff = ((float)halfwidth) / ((float)cwidth);
846 break;
847 case FRAMEWORK_FORMAT_YUV_420_888:
848 mYWidthCutoff = 1.0f;
849 mCWidthCutoff = 1.0f;
850 break;
851 case FRAMEWORK_FORMAT_NV12:
852 mYWidthCutoff = 1.0f;
853 mCWidthCutoff = 1.0f;
854 break;
855 case FRAMEWORK_FORMAT_GL_COMPATIBLE:
856 FATAL("Input not a YUV format!");
857 }
858 }
859
reset()860 void YUVConverter::reset() {
861 if (mIbuf) s_gles2.glDeleteBuffers(1, &mIbuf);
862 if (mVbuf) s_gles2.glDeleteBuffers(1, &mVbuf);
863 if (mProgram) s_gles2.glDeleteProgram(mProgram);
864 if (mYtex) s_gles2.glDeleteTextures(1, &mYtex);
865 if (mUtex) s_gles2.glDeleteTextures(1, &mUtex);
866 if (mVtex) s_gles2.glDeleteTextures(1, &mVtex);
867 if (mVUtex) s_gles2.glDeleteTextures(1, &mVUtex);
868 if (mUVtex) s_gles2.glDeleteTextures(1, &mUVtex);
869 mIbuf = 0;
870 mVbuf = 0;
871 mProgram = 0;
872 mYtex = 0;
873 mUtex = 0;
874 mVtex = 0;
875 mVUtex = 0;
876 mUVtex = 0;
877 }
878
~YUVConverter()879 YUVConverter::~YUVConverter() {
880 reset();
881 }
882