1 /*
2 * Copyright (C) 2016 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 #include "YUVConverter.h"
18
19 #include <assert.h>
20 #include <stdio.h>
21 #include <string>
22
23 #include "DispatchTables.h"
24 #include "host-common/feature_control.h"
25 #include "host-common/misc.h"
26
27
28 #define FATAL(fmt,...) do { \
29 fprintf(stderr, "%s: FATAL: " fmt "\n", __func__, ##__VA_ARGS__); \
30 assert(false); \
31 } while(0)
32
33 #define YUV_CONVERTER_DEBUG 0
34
35 #if YUV_CONVERTER_DEBUG
36 #define YUV_DEBUG_LOG(fmt, ...) \
37 fprintf(stderr, "yuv-converter: %s:%d " fmt "\n", __func__, __LINE__, \
38 ##__VA_ARGS__);
39 #else
40 #define YUV_DEBUG_LOG(fmt, ...)
41 #endif
42
isInterleaved(FrameworkFormat format)43 bool isInterleaved(FrameworkFormat format) {
44 switch (format) {
45 case FRAMEWORK_FORMAT_NV12:
46 case FRAMEWORK_FORMAT_P010:
47 return true;
48 case FRAMEWORK_FORMAT_YUV_420_888:
49 return feature_is_enabled(kFeature_YUV420888toNV21);
50 case FRAMEWORK_FORMAT_YV12:
51 return false;
52 default:
53 FATAL("Invalid for format:%d", format);
54 return false;
55 }
56 }
57
58 enum class YUVInterleaveDirection {
59 VU = 0,
60 UV = 1,
61 };
62
getInterleaveDirection(FrameworkFormat format)63 YUVInterleaveDirection getInterleaveDirection(FrameworkFormat format) {
64 if (!isInterleaved(format)) {
65 FATAL("Format:%d not interleaved", format);
66 }
67
68 switch (format) {
69 case FRAMEWORK_FORMAT_NV12:
70 case FRAMEWORK_FORMAT_P010:
71 return YUVInterleaveDirection::UV;
72 case FRAMEWORK_FORMAT_YUV_420_888:
73 if (feature_is_enabled(kFeature_YUV420888toNV21)) {
74 return YUVInterleaveDirection::VU;
75 }
76 FATAL("Format:%d not interleaved", format);
77 return YUVInterleaveDirection::UV;
78 case FRAMEWORK_FORMAT_YV12:
79 default:
80 FATAL("Format:%d not interleaved", format);
81 return YUVInterleaveDirection::UV;
82 }
83 }
84
getGlTextureFormat(FrameworkFormat format,YUVPlane plane)85 GLint getGlTextureFormat(FrameworkFormat format, YUVPlane plane) {
86 switch (format) {
87 case FRAMEWORK_FORMAT_YV12:
88 switch (plane) {
89 case YUVPlane::Y:
90 case YUVPlane::U:
91 case YUVPlane::V:
92 return GL_R8;
93 case YUVPlane::UV:
94 FATAL("Invalid plane:%d for format:%d", plane, format);
95 return 0;
96 }
97 case FRAMEWORK_FORMAT_YUV_420_888:
98 if (feature_is_enabled(kFeature_YUV420888toNV21)) {
99 switch (plane) {
100 case YUVPlane::Y:
101 return GL_R8;
102 case YUVPlane::UV:
103 return GL_RG8;
104 case YUVPlane::U:
105 case YUVPlane::V:
106 FATAL("Invalid plane:%d for format:%d", plane, format);
107 return 0;
108 }
109 } else {
110 switch (plane) {
111 case YUVPlane::Y:
112 case YUVPlane::U:
113 case YUVPlane::V:
114 return GL_R8;
115 case YUVPlane::UV:
116 FATAL("Invalid plane:%d for format:%d", plane, format);
117 return 0;
118 }
119 }
120 case FRAMEWORK_FORMAT_NV12:
121 switch (plane) {
122 case YUVPlane::Y:
123 return GL_R8;
124 case YUVPlane::UV:
125 return GL_RG8;
126 case YUVPlane::U:
127 case YUVPlane::V:
128 FATAL("Invalid plane:%d for format:%d", plane, format);
129 return 0;
130 }
131 case FRAMEWORK_FORMAT_P010:
132 switch (plane) {
133 case YUVPlane::Y:
134 return GL_R16UI;
135 case YUVPlane::UV:
136 return GL_RG16UI;
137 case YUVPlane::U:
138 case YUVPlane::V:
139 FATAL("Invalid plane:%d for format:%d", plane, format);
140 return 0;
141 }
142 default:
143 FATAL("Invalid format:%d", format);
144 return 0;
145 }
146 }
147
getGlPixelFormat(FrameworkFormat format,YUVPlane plane)148 GLenum getGlPixelFormat(FrameworkFormat format, YUVPlane plane) {
149 switch (format) {
150 case FRAMEWORK_FORMAT_YV12:
151 switch (plane) {
152 case YUVPlane::Y:
153 case YUVPlane::U:
154 case YUVPlane::V:
155 return GL_RED;
156 case YUVPlane::UV:
157 FATAL("Invalid plane:%d for format:%d", plane, format);
158 return 0;
159 }
160 case FRAMEWORK_FORMAT_YUV_420_888:
161 if (feature_is_enabled(kFeature_YUV420888toNV21)) {
162 switch (plane) {
163 case YUVPlane::Y:
164 return GL_RED;
165 case YUVPlane::UV:
166 return GL_RG;
167 case YUVPlane::U:
168 case YUVPlane::V:
169 FATAL("Invalid plane:%d for format:%d", plane, format);
170 return 0;
171 }
172 } else {
173 switch (plane) {
174 case YUVPlane::Y:
175 case YUVPlane::U:
176 case YUVPlane::V:
177 return GL_RED;
178 case YUVPlane::UV:
179 FATAL("Invalid plane:%d for format:%d", plane, format);
180 return 0;
181 }
182 }
183 case FRAMEWORK_FORMAT_NV12:
184 switch (plane) {
185 case YUVPlane::Y:
186 return GL_RED;
187 case YUVPlane::UV:
188 return GL_RG;
189 case YUVPlane::U:
190 case YUVPlane::V:
191 FATAL("Invalid plane:%d for format:%d", plane, format);
192 return 0;
193 }
194 case FRAMEWORK_FORMAT_P010:
195 switch (plane) {
196 case YUVPlane::Y:
197 return GL_RED_INTEGER;
198 case YUVPlane::UV:
199 return GL_RG_INTEGER;
200 case YUVPlane::U:
201 case YUVPlane::V:
202 FATAL("Invalid plane:%d for format:%d", plane, format);
203 return 0;
204 }
205 default:
206 FATAL("Invalid format:%d", format);
207 return 0;
208 }
209 }
210
getGlPixelType(FrameworkFormat format,YUVPlane plane)211 GLsizei getGlPixelType(FrameworkFormat format, YUVPlane plane) {
212 switch (format) {
213 case FRAMEWORK_FORMAT_YV12:
214 switch (plane) {
215 case YUVPlane::Y:
216 case YUVPlane::U:
217 case YUVPlane::V:
218 return GL_UNSIGNED_BYTE;
219 case YUVPlane::UV:
220 FATAL("Invalid plane:%d for format:%d", plane, format);
221 return 0;
222 }
223 case FRAMEWORK_FORMAT_YUV_420_888:
224 if (feature_is_enabled(kFeature_YUV420888toNV21)) {
225 switch (plane) {
226 case YUVPlane::Y:
227 case YUVPlane::UV:
228 return GL_UNSIGNED_BYTE;
229 case YUVPlane::U:
230 case YUVPlane::V:
231 FATAL("Invalid plane:%d for format:%d", plane, format);
232 return 0;
233 }
234 } else {
235 switch (plane) {
236 case YUVPlane::Y:
237 case YUVPlane::U:
238 case YUVPlane::V:
239 return GL_UNSIGNED_BYTE;
240 case YUVPlane::UV:
241 FATAL("Invalid plane:%d for format:%d", plane, format);
242 return 0;
243 }
244 }
245 case FRAMEWORK_FORMAT_NV12:
246 switch (plane) {
247 case YUVPlane::Y:
248 case YUVPlane::UV:
249 return GL_UNSIGNED_BYTE;
250 case YUVPlane::U:
251 case YUVPlane::V:
252 FATAL("Invalid plane:%d for format:%d", plane, format);
253 return 0;
254 }
255 case FRAMEWORK_FORMAT_P010:
256 switch (plane) {
257 case YUVPlane::Y:
258 case YUVPlane::UV:
259 return GL_UNSIGNED_SHORT;
260 case YUVPlane::U:
261 case YUVPlane::V:
262 FATAL("Invalid plane:%d for format:%d", plane, format);
263 return 0;
264 }
265 default:
266 FATAL("Invalid format:%d", format);
267 return 0;
268 }
269 }
270
271 // NV12 and YUV420 are all packed
NV12ToYUV420PlanarInPlaceConvert(int nWidth,int nHeight,uint8_t * pFrame,uint8_t * pQuad)272 static void NV12ToYUV420PlanarInPlaceConvert(int nWidth,
273 int nHeight,
274 uint8_t* pFrame,
275 uint8_t* pQuad) {
276 std::vector<uint8_t> tmp;
277 if (pQuad == nullptr) {
278 tmp.resize(nWidth * nHeight / 4);
279 pQuad = tmp.data();
280 }
281 int nPitch = nWidth;
282 uint8_t *puv = pFrame + nPitch * nHeight, *pu = puv,
283 *pv = puv + nPitch * nHeight / 4;
284 for (int y = 0; y < nHeight / 2; y++) {
285 for (int x = 0; x < nWidth / 2; x++) {
286 pu[y * nPitch / 2 + x] = puv[y * nPitch + x * 2];
287 pQuad[y * nWidth / 2 + x] = puv[y * nPitch + x * 2 + 1];
288 }
289 }
290 memcpy(pv, pQuad, nWidth * nHeight / 4);
291 }
292
alignToPower2(uint32_t val,uint32_t align)293 inline uint32_t alignToPower2(uint32_t val, uint32_t align) {
294 return (val + (align - 1)) & ~(align - 1);
295 }
296
297 // getYUVOffsets(), given a YUV-formatted buffer that is arranged
298 // according to the spec
299 // https://developer.android.com/reference/android/graphics/ImageFormat.html#YUV
300 // In particular, Android YUV widths are aligned to 16 pixels.
301 // Inputs:
302 // |yv12|: the YUV-formatted buffer
303 // Outputs:
304 // |yOffset|: offset into |yv12| of the start of the Y component
305 // |uOffset|: offset into |yv12| of the start of the U component
306 // |vOffset|: offset into |yv12| of the start of the V component
getYUVOffsets(int width,int height,FrameworkFormat format,uint32_t * yOffset,uint32_t * uOffset,uint32_t * vOffset,uint32_t * yWidth,uint32_t * cWidth)307 static void getYUVOffsets(int width,
308 int height,
309 FrameworkFormat format,
310 uint32_t* yOffset,
311 uint32_t* uOffset,
312 uint32_t* vOffset,
313 uint32_t* yWidth,
314 uint32_t* cWidth) {
315 uint32_t yStride, cStride, cHeight, cSize;
316 switch (format) {
317 case FRAMEWORK_FORMAT_YV12:
318 // Luma stride is 32 bytes aligned.
319 yStride = alignToPower2(width, 32);
320 // Chroma stride is 16 bytes aligned.
321 cStride = alignToPower2(yStride, 16);
322 cHeight = height / 2;
323 cSize = cStride * cHeight;
324 *yOffset = 0;
325 *vOffset = yStride * height;
326 *uOffset = (*vOffset) + cSize;
327 *yWidth = yStride;
328 *cWidth = cStride;
329 break;
330 case FRAMEWORK_FORMAT_YUV_420_888:
331 if (feature_is_enabled(kFeature_YUV420888toNV21)) {
332 yStride = width;
333 cStride = yStride;
334 cHeight = height / 2;
335 *yOffset = 0;
336 *vOffset = yStride * height;
337 *uOffset = (*vOffset) + 1;
338 *yWidth = yStride;
339 *cWidth = cStride / 2;
340 } else {
341 yStride = width;
342 cStride = yStride / 2;
343 cHeight = height / 2;
344 cSize = cStride * cHeight;
345 *yOffset = 0;
346 *uOffset = yStride * height;
347 *vOffset = (*uOffset) + cSize;
348 *yWidth = yStride;
349 *cWidth = cStride;
350 }
351 break;
352 case FRAMEWORK_FORMAT_NV12:
353 yStride = width;
354 cStride = yStride;
355 cHeight = height / 2;
356 cSize = cStride * cHeight;
357 *yOffset = 0;
358 *uOffset = yStride * height;
359 *vOffset = (*uOffset) + 1;
360 *yWidth = yStride;
361 *cWidth = cStride / 2;
362 break;
363 case FRAMEWORK_FORMAT_P010:
364 *yWidth = width;
365 *cWidth = width / 2;
366 yStride = width * /*bytes per pixel=*/2;
367 cStride = *cWidth * /*bytes per pixel=*/2;
368 cHeight = height / 2;
369 cSize = cStride * cHeight;
370 *yOffset = 0;
371 *uOffset = yStride * height;
372 *vOffset = (*uOffset) + 2;
373 break;
374 case FRAMEWORK_FORMAT_GL_COMPATIBLE:
375 FATAL("Input not a YUV format! (FRAMEWORK_FORMAT_GL_COMPATIBLE)");
376 default:
377 FATAL("Unknown format: 0x%x", format);
378 }
379 }
380
381 // Allocates an OpenGL texture that is large enough for a single plane of
382 // a YUV buffer of the given format and returns the texture name in the
383 // `outTextureName` argument.
createYUVGLTex(GLenum textureUnit,GLsizei width,GLsizei height,FrameworkFormat format,YUVPlane plane,GLuint * outTextureName)384 void YUVConverter::createYUVGLTex(GLenum textureUnit,
385 GLsizei width,
386 GLsizei height,
387 FrameworkFormat format,
388 YUVPlane plane,
389 GLuint* outTextureName) {
390 YUV_DEBUG_LOG("w:%d h:%d format:%d plane:%d", width, height, format, plane);
391
392 s_gles2.glActiveTexture(textureUnit);
393 s_gles2.glGenTextures(1, outTextureName);
394 s_gles2.glBindTexture(GL_TEXTURE_2D, *outTextureName);
395 s_gles2.glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST);
396 s_gles2.glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST);
397 GLint unprevAlignment = 0;
398 s_gles2.glGetIntegerv(GL_UNPACK_ALIGNMENT, &unprevAlignment);
399 s_gles2.glPixelStorei(GL_UNPACK_ALIGNMENT, 1);
400 const GLint textureFormat = getGlTextureFormat(format, plane);
401 const GLenum pixelFormat = getGlPixelFormat(format, plane);
402 const GLenum pixelType = getGlPixelType(format, plane);
403 s_gles2.glTexImage2D(GL_TEXTURE_2D, 0, textureFormat, width, height, 0, pixelFormat, pixelType, NULL);
404 s_gles2.glPixelStorei(GL_UNPACK_ALIGNMENT, unprevAlignment);
405 s_gles2.glActiveTexture(GL_TEXTURE0);
406 }
407
readYUVTex(GLuint tex,FrameworkFormat format,YUVPlane plane,void * pixels)408 static void readYUVTex(GLuint tex, FrameworkFormat format, YUVPlane plane, void* pixels) {
409 YUV_DEBUG_LOG("format%d plane:%d pixels:%p", format, plane, pixels);
410
411 GLuint prevTexture = 0;
412 s_gles2.glGetIntegerv(GL_TEXTURE_BINDING_2D, (GLint*)&prevTexture);
413 s_gles2.glBindTexture(GL_TEXTURE_2D, tex);
414 GLint prevAlignment = 0;
415 s_gles2.glGetIntegerv(GL_PACK_ALIGNMENT, &prevAlignment);
416 s_gles2.glPixelStorei(GL_PACK_ALIGNMENT, 1);
417 const GLenum pixelFormat = getGlPixelFormat(format, plane);
418 const GLenum pixelType = getGlPixelType(format, plane);
419 if (s_gles2.glGetTexImage) {
420 s_gles2.glGetTexImage(GL_TEXTURE_2D, 0, pixelFormat, pixelType, pixels);
421 } else {
422 YUV_DEBUG_LOG("empty glGetTexImage");
423 }
424
425 s_gles2.glPixelStorei(GL_PACK_ALIGNMENT, prevAlignment);
426 s_gles2.glBindTexture(GL_TEXTURE_2D, prevTexture);
427 }
428
429 // Updates a given YUV buffer's plane texture at the coordinates
430 // (x, y, width, height), with the raw YUV data in |pixels|. We
431 // cannot view the result properly until after conversion; this is
432 // to be used only as input to the conversion shader.
subUpdateYUVGLTex(GLenum texture_unit,GLuint tex,int x,int y,int width,int height,FrameworkFormat format,YUVPlane plane,const void * pixels)433 static void subUpdateYUVGLTex(GLenum texture_unit,
434 GLuint tex,
435 int x,
436 int y,
437 int width,
438 int height,
439 FrameworkFormat format,
440 YUVPlane plane,
441 const void* pixels) {
442 YUV_DEBUG_LOG("x:%d y:%d w:%d h:%d format:%d plane:%d", x, y, width, height, format, plane);
443
444 const GLenum pixelFormat = getGlPixelFormat(format, plane);
445 const GLenum pixelType = getGlPixelType(format, plane);
446
447 s_gles2.glActiveTexture(texture_unit);
448 s_gles2.glBindTexture(GL_TEXTURE_2D, tex);
449 GLint unprevAlignment = 0;
450 s_gles2.glGetIntegerv(GL_UNPACK_ALIGNMENT, &unprevAlignment);
451 s_gles2.glPixelStorei(GL_UNPACK_ALIGNMENT, 1);
452 s_gles2.glTexSubImage2D(GL_TEXTURE_2D, 0, x, y, width, height, pixelFormat, pixelType, pixels);
453 s_gles2.glPixelStorei(GL_UNPACK_ALIGNMENT, unprevAlignment);
454 s_gles2.glActiveTexture(GL_TEXTURE0);
455 }
456
createYUVGLShader()457 void YUVConverter::createYUVGLShader() {
458 YUV_DEBUG_LOG("format:%d", mFormat);
459
460 // P010 needs uint samplers.
461 if (mFormat == FRAMEWORK_FORMAT_P010 && !mHasGlsl3Support) {
462 return;
463 }
464
465 static const char kVertShader[] = R"(
466 precision highp float;
467 attribute mediump vec4 aPosition;
468 attribute highp vec2 aTexCoord;
469 varying highp vec2 vTexCoord;
470 void main(void) {
471 gl_Position = aPosition;
472 vTexCoord = aTexCoord;
473 }
474 )";
475
476 static const char kFragShaderVersion3[] = R"(#version 300 es)";
477
478 static const char kFragShaderBegin[] = R"(
479 precision highp float;
480
481 varying highp vec2 vTexCoord;
482
483 uniform highp float uYWidthCutoff;
484 uniform highp float uCWidthCutoff;
485 )";
486
487 static const char kSamplerUniforms[] = R"(
488 uniform sampler2D uSamplerY;
489 uniform sampler2D uSamplerU;
490 uniform sampler2D uSamplerV;
491 )";
492 static const char kSamplerUniformsUint[] = R"(
493 uniform usampler2D uSamplerY;
494 uniform usampler2D uSamplerU;
495 uniform usampler2D uSamplerV;
496 )";
497
498 static const char kFragShaderMainBegin[] = R"(
499 void main(void) {
500 highp vec2 yTexCoords = vTexCoord;
501 highp vec2 uvTexCoords = vTexCoord;
502
503 // For textures with extra padding for alignment (e.g. YV12 pads to 16),
504 // scale the coordinates to only sample from the non-padded area.
505 yTexCoords.x *= uYWidthCutoff;
506 uvTexCoords.y *= uCWidthCutoff;
507
508 highp vec3 yuv;
509 )";
510
511 static const char kSampleY[] = R"(
512 yuv[0] = texture2D(uSamplerY, yTexCoords).r;
513 )";
514 static const char kSampleUV[] = R"(
515 yuv[1] = texture2D(uSamplerU, uvTexCoords).r;
516 yuv[2] = texture2D(uSamplerV, uvTexCoords).r;
517 )";
518 static const char kSampleInterleavedUV[] = R"(
519 // Note: uSamplerU and vSamplerV refer to the same texture.
520 yuv[1] = texture2D(uSamplerU, uvTexCoords).r;
521 yuv[2] = texture2D(uSamplerV, uvTexCoords).g;
522 )";
523 static const char kSampleInterleavedVU[] = R"(
524 // Note: uSamplerU and vSamplerV refer to the same texture.
525 yuv[1] = texture2D(uSamplerU, uvTexCoords).g;
526 yuv[2] = texture2D(uSamplerV, uvTexCoords).r;
527 )";
528
529 static const char kSampleP010[] = R"(
530 uint yRaw = texture(uSamplerY, yTexCoords).r;
531 uint uRaw = texture(uSamplerU, uvTexCoords).r;
532 uint vRaw = texture(uSamplerV, uvTexCoords).g;
533
534 // P010 values are stored in the upper 10-bits of 16-bit unsigned shorts.
535 yuv[0] = float(yRaw >> 6) / 1023.0;
536 yuv[1] = float(uRaw >> 6) / 1023.0;
537 yuv[2] = float(vRaw >> 6) / 1023.0;
538 )";
539
540 static const char kFragShaderMainEnd[] = R"(
541 yuv[0] = yuv[0] - 0.0625;
542 yuv[1] = 0.96 * (yuv[1] - 0.5);
543 yuv[2] = (yuv[2] - 0.5);
544
545 highp float yscale = 1.1643835616438356;
546 highp vec3 rgb = mat3( yscale, yscale, yscale,
547 0, -0.39176229009491365, 2.017232142857143,
548 1.5960267857142856, -0.8129676472377708, 0) * yuv;
549
550 gl_FragColor = vec4(rgb, 1.0);
551 }
552 )";
553
554 std::string vertShaderSource(kVertShader);
555 std::string fragShaderSource;
556
557 if (mFormat == FRAMEWORK_FORMAT_P010) {
558 fragShaderSource += kFragShaderVersion3;
559 }
560
561 fragShaderSource += kFragShaderBegin;
562
563 if (mFormat == FRAMEWORK_FORMAT_P010) {
564 fragShaderSource += kSamplerUniformsUint;
565 } else {
566 fragShaderSource += kSamplerUniforms;
567 }
568
569 fragShaderSource += kFragShaderMainBegin;
570
571 switch (mFormat) {
572 case FRAMEWORK_FORMAT_NV12:
573 case FRAMEWORK_FORMAT_YUV_420_888:
574 case FRAMEWORK_FORMAT_YV12:
575 fragShaderSource += kSampleY;
576 if (isInterleaved(mFormat)) {
577 if (getInterleaveDirection(mFormat) == YUVInterleaveDirection::UV) {
578 fragShaderSource += kSampleInterleavedUV;
579 } else {
580 fragShaderSource += kSampleInterleavedVU;
581 }
582 } else {
583 fragShaderSource += kSampleUV;
584 }
585 break;
586 case FRAMEWORK_FORMAT_P010:
587 fragShaderSource += kSampleP010;
588 break;
589 default:
590 FATAL("%s: invalid format:%d", __FUNCTION__, mFormat);
591 return;
592 }
593
594 fragShaderSource += kFragShaderMainEnd;
595
596 YUV_DEBUG_LOG("format:%d vert-source:%s frag-source:%s", mFormat, vertShaderSource.c_str(), fragShaderSource.c_str());
597
598 const GLchar* const vertShaderSourceChars = vertShaderSource.c_str();
599 const GLchar* const fragShaderSourceChars = fragShaderSource.c_str();
600 const GLint vertShaderSourceLen = vertShaderSource.length();
601 const GLint fragShaderSourceLen = fragShaderSource.length();
602
603 GLuint vertShader = s_gles2.glCreateShader(GL_VERTEX_SHADER);
604 GLuint fragShader = s_gles2.glCreateShader(GL_FRAGMENT_SHADER);
605 s_gles2.glShaderSource(vertShader, 1, &vertShaderSourceChars, &vertShaderSourceLen);
606 s_gles2.glShaderSource(fragShader, 1, &fragShaderSourceChars, &fragShaderSourceLen);
607 s_gles2.glCompileShader(vertShader);
608 s_gles2.glCompileShader(fragShader);
609
610 for (GLuint shader : {vertShader, fragShader}) {
611 GLint status = GL_FALSE;
612 s_gles2.glGetShaderiv(shader, GL_COMPILE_STATUS, &status);
613 if (status == GL_FALSE) {
614 GLchar error[1024];
615 s_gles2.glGetShaderInfoLog(shader, sizeof(error), nullptr, &error[0]);
616 FATAL("Failed to compile YUV conversion shader: %s", error);
617 s_gles2.glDeleteShader(shader);
618 return;
619 }
620 }
621
622 mProgram = s_gles2.glCreateProgram();
623 s_gles2.glAttachShader(mProgram, vertShader);
624 s_gles2.glAttachShader(mProgram, fragShader);
625 s_gles2.glLinkProgram(mProgram);
626
627 GLint status = GL_FALSE;
628 s_gles2.glGetProgramiv(mProgram, GL_LINK_STATUS, &status);
629 if (status == GL_FALSE) {
630 GLchar error[1024];
631 s_gles2.glGetProgramInfoLog(mProgram, sizeof(error), 0, &error[0]);
632 FATAL("Failed to link YUV conversion program: %s", error);
633 s_gles2.glDeleteProgram(mProgram);
634 mProgram = 0;
635 return;
636 }
637
638 mUniformLocYWidthCutoff = s_gles2.glGetUniformLocation(mProgram, "uYWidthCutoff");
639 mUniformLocCWidthCutoff = s_gles2.glGetUniformLocation(mProgram, "uCWidthCutoff");
640 mUniformLocSamplerY = s_gles2.glGetUniformLocation(mProgram, "uSamplerY");
641 mUniformLocSamplerU = s_gles2.glGetUniformLocation(mProgram, "uSamplerU");
642 mUniformLocSamplerV = s_gles2.glGetUniformLocation(mProgram, "uSamplerV");
643 mAttributeLocPos = s_gles2.glGetAttribLocation(mProgram, "aPosition");
644 mAttributeLocTexCoord = s_gles2.glGetAttribLocation(mProgram, "aTexCoord");
645
646 s_gles2.glDeleteShader(vertShader);
647 s_gles2.glDeleteShader(fragShader);
648 }
649
createYUVGLFullscreenQuad()650 void YUVConverter::createYUVGLFullscreenQuad() {
651 s_gles2.glGenBuffers(1, &mQuadVertexBuffer);
652 s_gles2.glGenBuffers(1, &mQuadIndexBuffer);
653
654 static const float kVertices[] = {
655 +1, -1, +0, +1, +0,
656 +1, +1, +0, +1, +1,
657 -1, +1, +0, +0, +1,
658 -1, -1, +0, +0, +0,
659 };
660
661 static const GLubyte kIndices[] = { 0, 1, 2, 2, 3, 0 };
662
663 s_gles2.glBindBuffer(GL_ARRAY_BUFFER, mQuadVertexBuffer);
664 s_gles2.glBufferData(GL_ARRAY_BUFFER, sizeof(kVertices), kVertices, GL_STATIC_DRAW);
665 s_gles2.glBindBuffer(GL_ELEMENT_ARRAY_BUFFER, mQuadIndexBuffer);
666 s_gles2.glBufferData(GL_ELEMENT_ARRAY_BUFFER, sizeof(kIndices), kIndices, GL_STATIC_DRAW);
667 }
668
doYUVConversionDraw(GLuint program,GLint uniformLocYWidthCutoff,GLint uniformLocCWidthCutoff,GLint uniformLocYSampler,GLint uniformLocUSampler,GLint uniformLocVSampler,GLint attributeLocTexCoord,GLint attributeLocPos,GLuint quadVertexBuffer,GLuint quadIndexBuffer,float uYWidthCutoff,float uCWidthCutoff)669 static void doYUVConversionDraw(GLuint program,
670 GLint uniformLocYWidthCutoff,
671 GLint uniformLocCWidthCutoff,
672 GLint uniformLocYSampler,
673 GLint uniformLocUSampler,
674 GLint uniformLocVSampler,
675 GLint attributeLocTexCoord,
676 GLint attributeLocPos,
677 GLuint quadVertexBuffer,
678 GLuint quadIndexBuffer,
679 float uYWidthCutoff,
680 float uCWidthCutoff) {
681 const GLsizei kVertexAttribStride = 5 * sizeof(GL_FLOAT);
682 const GLvoid* kVertexAttribPosOffset = (GLvoid*)0;
683 const GLvoid* kVertexAttribCoordOffset = (GLvoid*)(3 * sizeof(GL_FLOAT));
684
685 s_gles2.glUseProgram(program);
686
687 s_gles2.glUniform1f(uniformLocYWidthCutoff, uYWidthCutoff);
688 s_gles2.glUniform1f(uniformLocCWidthCutoff, uCWidthCutoff);
689
690 s_gles2.glUniform1i(uniformLocYSampler, 0);
691 s_gles2.glUniform1i(uniformLocUSampler, 1);
692 s_gles2.glUniform1i(uniformLocVSampler, 2);
693
694 s_gles2.glBindBuffer(GL_ARRAY_BUFFER, quadVertexBuffer);
695 s_gles2.glEnableVertexAttribArray(attributeLocPos);
696 s_gles2.glEnableVertexAttribArray(attributeLocTexCoord);
697
698 s_gles2.glVertexAttribPointer(attributeLocPos, 3, GL_FLOAT, false,
699 kVertexAttribStride,
700 kVertexAttribPosOffset);
701 s_gles2.glVertexAttribPointer(attributeLocTexCoord, 2, GL_FLOAT, false,
702 kVertexAttribStride,
703 kVertexAttribCoordOffset);
704
705 s_gles2.glBindBuffer(GL_ELEMENT_ARRAY_BUFFER, quadIndexBuffer);
706 s_gles2.glDrawElements(GL_TRIANGLES, 6, GL_UNSIGNED_BYTE, 0);
707
708 s_gles2.glDisableVertexAttribArray(attributeLocPos);
709 s_gles2.glDisableVertexAttribArray(attributeLocTexCoord);
710 }
711
712 // initialize(): allocate GPU memory for YUV components,
713 // and create shaders and vertex data.
YUVConverter(int width,int height,FrameworkFormat format)714 YUVConverter::YUVConverter(int width, int height, FrameworkFormat format)
715 : mWidth(width),
716 mHeight(height),
717 mFormat(format),
718 mColorBufferFormat(format) {}
719
init(int width,int height,FrameworkFormat format)720 void YUVConverter::init(int width, int height, FrameworkFormat format) {
721 YUV_DEBUG_LOG("w:%d h:%d format:%d", width, height, format);
722
723 uint32_t yOffset, uOffset, vOffset, ywidth, cwidth, cheight;
724 getYUVOffsets(width, height, mFormat, &yOffset, &uOffset, &vOffset, &ywidth, &cwidth);
725 cheight = height / 2;
726
727 mWidth = width;
728 mHeight = height;
729 if (!mTextureY) {
730 createYUVGLTex(GL_TEXTURE0, ywidth, height, mFormat, YUVPlane::Y, &mTextureY);
731 }
732 if (isInterleaved(mFormat)) {
733 if (!mTextureU) {
734 createYUVGLTex(GL_TEXTURE1, cwidth, cheight, mFormat, YUVPlane::UV, &mTextureU);
735 mTextureV = mTextureU;
736 }
737 } else {
738 if (!mTextureU) {
739 createYUVGLTex(GL_TEXTURE1, cwidth, cheight, mFormat, YUVPlane::U, &mTextureU);
740 }
741 if (!mTextureV) {
742 createYUVGLTex(GL_TEXTURE2, cwidth, cheight, mFormat, YUVPlane::V, &mTextureV);
743 }
744 }
745
746 int glesMajor;
747 int glesMinor;
748 emugl::getGlesVersion(&glesMajor, &glesMinor);
749 mHasGlsl3Support = glesMajor >= 3;
750 YUV_DEBUG_LOG("YUVConverter has GLSL ES 3 support:%s (major:%d minor:%d", (mHasGlsl3Support ? "yes" : "no"), glesMajor, glesMinor);
751
752 createYUVGLShader();
753 createYUVGLFullscreenQuad();
754 }
755
saveGLState()756 void YUVConverter::saveGLState() {
757 s_gles2.glGetFloatv(GL_VIEWPORT, mCurrViewport);
758 s_gles2.glGetIntegerv(GL_ACTIVE_TEXTURE, &mCurrTexUnit);
759 s_gles2.glGetIntegerv(GL_TEXTURE_BINDING_2D, &mCurrTexBind);
760 s_gles2.glGetIntegerv(GL_CURRENT_PROGRAM, &mCurrProgram);
761 s_gles2.glGetIntegerv(GL_ARRAY_BUFFER_BINDING, &mCurrVbo);
762 s_gles2.glGetIntegerv(GL_ELEMENT_ARRAY_BUFFER_BINDING, &mCurrIbo);
763 }
764
restoreGLState()765 void YUVConverter::restoreGLState() {
766 s_gles2.glViewport(mCurrViewport[0], mCurrViewport[1],
767 mCurrViewport[2], mCurrViewport[3]);
768 s_gles2.glActiveTexture(mCurrTexUnit);
769 s_gles2.glUseProgram(mCurrProgram);
770 s_gles2.glBindBuffer(GL_ARRAY_BUFFER, mCurrVbo);
771 s_gles2.glBindBuffer(GL_ELEMENT_ARRAY_BUFFER, mCurrIbo);
772 }
773
getDataSize()774 uint32_t YUVConverter::getDataSize() {
775 uint32_t align = (mFormat == FRAMEWORK_FORMAT_YV12) ? 16 : 1;
776 uint32_t yStride = (mWidth + (align - 1)) & ~(align - 1);
777 uint32_t uvStride = (yStride / 2 + (align - 1)) & ~(align - 1);
778 uint32_t uvHeight = mHeight / 2;
779 uint32_t dataSize = yStride * mHeight + 2 * (uvHeight * uvStride);
780 return dataSize;
781 }
782
readPixels(uint8_t * pixels,uint32_t pixels_size)783 void YUVConverter::readPixels(uint8_t* pixels, uint32_t pixels_size) {
784 YUV_DEBUG_LOG("w:%d h:%d format:%d pixels:%p pixels-size:%d", mWidth, mHeight, mFormat, pixels, pixels_size);
785
786 uint32_t yOffset, uOffset, vOffset, ywidth, cwidth;
787 getYUVOffsets(mWidth, mHeight, mFormat, &yOffset, &uOffset, &vOffset, &ywidth, &cwidth);
788
789 if (isInterleaved(mFormat)) {
790 readYUVTex(mTextureV, mFormat, YUVPlane::UV, pixels + std::min(uOffset, vOffset));
791 } else {
792 readYUVTex(mTextureU, mFormat, YUVPlane::U, pixels + uOffset);
793 readYUVTex(mTextureV, mFormat, YUVPlane::V, pixels + vOffset);
794 }
795
796 if (mFormat == FRAMEWORK_FORMAT_NV12 && mColorBufferFormat == FRAMEWORK_FORMAT_YUV_420_888) {
797 NV12ToYUV420PlanarInPlaceConvert(mWidth, mHeight, pixels, pixels);
798 }
799
800 // Read the Y plane last because so that we can use it as a scratch space.
801 readYUVTex(mTextureY, mFormat, YUVPlane::Y, pixels + yOffset);
802 }
803
swapTextures(uint32_t type,uint32_t * textures)804 void YUVConverter::swapTextures(uint32_t type, uint32_t* textures) {
805 FrameworkFormat format = static_cast<FrameworkFormat>(type);
806
807 if (isInterleaved(format)) {
808 std::swap(textures[0], mTextureY);
809 std::swap(textures[1], mTextureU);
810 mTextureV = mTextureU;
811 } else {
812 std::swap(textures[0], mTextureY);
813 std::swap(textures[1], mTextureU);
814 std::swap(textures[2], mTextureV);
815 }
816
817 mFormat = format;
818 }
819
drawConvert(int x,int y,int width,int height,const char * pixels)820 void YUVConverter::drawConvert(int x, int y, int width, int height, const char* pixels) {
821 YUV_DEBUG_LOG("x:%d y:%d w:%d h:%d", x, y, width, height);
822
823 saveGLState();
824 if (pixels && (width != mWidth || height != mHeight)) {
825 reset();
826 }
827
828 if (mProgram == 0) {
829 init(width, height, mFormat);
830 }
831
832 if (mFormat == FRAMEWORK_FORMAT_P010 && !mHasGlsl3Support) {
833 // TODO: perhaps fallback to just software conversion.
834 return;
835 }
836
837 s_gles2.glViewport(x, y, width, height);
838 uint32_t yOffset, uOffset, vOffset, ywidth, cwidth, cheight;
839 getYUVOffsets(width, height, mFormat, &yOffset, &uOffset, &vOffset, &ywidth, &cwidth);
840 cheight = height / 2;
841 updateCutoffs(width, ywidth, width / 2, cwidth);
842
843 if (pixels) {
844 subUpdateYUVGLTex(GL_TEXTURE0, mTextureY, x, y, ywidth, height, mFormat, YUVPlane::Y, pixels + yOffset);
845 if (isInterleaved(mFormat)) {
846 subUpdateYUVGLTex(GL_TEXTURE1, mTextureU, x, y, cwidth, cheight, mFormat, YUVPlane::UV, pixels + std::min(uOffset, vOffset));
847 } else {
848 subUpdateYUVGLTex(GL_TEXTURE1, mTextureU, x, y, cwidth, cheight, mFormat, YUVPlane::U, pixels + uOffset);
849 subUpdateYUVGLTex(GL_TEXTURE2, mTextureV, x, y, cwidth, cheight, mFormat, YUVPlane::V, pixels + vOffset);
850 }
851 } else {
852 // special case: draw from texture, only support NV12 for now
853 // as cuvid's native format is NV12.
854 // TODO: add more formats if there are such needs in the future.
855 assert(mFormat == FRAMEWORK_FORMAT_NV12);
856 }
857
858 s_gles2.glActiveTexture(GL_TEXTURE0);
859 s_gles2.glBindTexture(GL_TEXTURE_2D, mTextureY);
860 s_gles2.glActiveTexture(GL_TEXTURE1);
861 s_gles2.glBindTexture(GL_TEXTURE_2D, mTextureU);
862 s_gles2.glActiveTexture(GL_TEXTURE2);
863 s_gles2.glBindTexture(GL_TEXTURE_2D, mTextureV);
864
865 doYUVConversionDraw(mProgram,
866 mUniformLocYWidthCutoff,
867 mUniformLocCWidthCutoff,
868 mUniformLocSamplerY,
869 mUniformLocSamplerU,
870 mUniformLocSamplerV,
871 mAttributeLocTexCoord,
872 mAttributeLocPos,
873 mQuadVertexBuffer,
874 mQuadIndexBuffer,
875 mYWidthCutoff,
876 mCWidthCutoff);
877
878 restoreGLState();
879 }
880
updateCutoffs(float width,float ywidth,float halfwidth,float cwidth)881 void YUVConverter::updateCutoffs(float width, float ywidth,
882 float halfwidth, float cwidth) {
883 switch (mFormat) {
884 case FRAMEWORK_FORMAT_YV12:
885 mYWidthCutoff = ((float)width) / ((float)ywidth);
886 mCWidthCutoff = ((float)halfwidth) / ((float)cwidth);
887 break;
888 case FRAMEWORK_FORMAT_NV12:
889 case FRAMEWORK_FORMAT_P010:
890 case FRAMEWORK_FORMAT_YUV_420_888:
891 mYWidthCutoff = 1.0f;
892 mCWidthCutoff = 1.0f;
893 break;
894 case FRAMEWORK_FORMAT_GL_COMPATIBLE:
895 FATAL("Input not a YUV format!");
896 }
897 }
898
reset()899 void YUVConverter::reset() {
900 if (mQuadIndexBuffer) s_gles2.glDeleteBuffers(1, &mQuadIndexBuffer);
901 if (mQuadVertexBuffer) s_gles2.glDeleteBuffers(1, &mQuadVertexBuffer);
902 if (mProgram) s_gles2.glDeleteProgram(mProgram);
903 if (mTextureY) s_gles2.glDeleteTextures(1, &mTextureY);
904 if (isInterleaved(mFormat)) {
905 if (mTextureU) s_gles2.glDeleteTextures(1, &mTextureU);
906 } else {
907 if (mTextureU) s_gles2.glDeleteTextures(1, &mTextureU);
908 if (mTextureV) s_gles2.glDeleteTextures(1, &mTextureV);
909 }
910 mQuadIndexBuffer = 0;
911 mQuadVertexBuffer = 0;
912 mProgram = 0;
913 mTextureY = 0;
914 mTextureU = 0;
915 mTextureV = 0;
916 }
917
~YUVConverter()918 YUVConverter::~YUVConverter() {
919 reset();
920 }
921