1 /*
2 * Copyright (C) 2016 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 #include "YUVConverter.h"
18
19 #include <assert.h>
20 #include <stdio.h>
21 #include <string>
22
23 #include "OpenGLESDispatch/DispatchTables.h"
24 #include "host-common/feature_control.h"
25 #include "host-common/opengl/misc.h"
26
27 namespace gfxstream {
28 namespace gl {
29
30 #define FATAL(fmt,...) do { \
31 fprintf(stderr, "%s: FATAL: " fmt "\n", __func__, ##__VA_ARGS__); \
32 assert(false); \
33 } while(0)
34
35 #define YUV_CONVERTER_DEBUG 0
36
37 #if YUV_CONVERTER_DEBUG
38 #define YUV_DEBUG_LOG(fmt, ...) \
39 fprintf(stderr, "yuv-converter: %s:%d " fmt "\n", __func__, __LINE__, \
40 ##__VA_ARGS__);
41 #else
42 #define YUV_DEBUG_LOG(fmt, ...)
43 #endif
44
isInterleaved(FrameworkFormat format)45 bool isInterleaved(FrameworkFormat format) {
46 switch (format) {
47 case FRAMEWORK_FORMAT_NV12:
48 case FRAMEWORK_FORMAT_P010:
49 return true;
50 case FRAMEWORK_FORMAT_YUV_420_888:
51 return feature_is_enabled(kFeature_YUV420888toNV21);
52 case FRAMEWORK_FORMAT_YV12:
53 return false;
54 default:
55 FATAL("Invalid for format:%d", format);
56 return false;
57 }
58 }
59
60 enum class YUVInterleaveDirection {
61 VU = 0,
62 UV = 1,
63 };
64
getInterleaveDirection(FrameworkFormat format)65 YUVInterleaveDirection getInterleaveDirection(FrameworkFormat format) {
66 if (!isInterleaved(format)) {
67 FATAL("Format:%d not interleaved", format);
68 }
69
70 switch (format) {
71 case FRAMEWORK_FORMAT_NV12:
72 case FRAMEWORK_FORMAT_P010:
73 return YUVInterleaveDirection::UV;
74 case FRAMEWORK_FORMAT_YUV_420_888:
75 if (feature_is_enabled(kFeature_YUV420888toNV21)) {
76 return YUVInterleaveDirection::VU;
77 }
78 FATAL("Format:%d not interleaved", format);
79 return YUVInterleaveDirection::UV;
80 case FRAMEWORK_FORMAT_YV12:
81 default:
82 FATAL("Format:%d not interleaved", format);
83 return YUVInterleaveDirection::UV;
84 }
85 }
86
getGlTextureFormat(FrameworkFormat format,YUVPlane plane)87 GLint getGlTextureFormat(FrameworkFormat format, YUVPlane plane) {
88 switch (format) {
89 case FRAMEWORK_FORMAT_YV12:
90 switch (plane) {
91 case YUVPlane::Y:
92 case YUVPlane::U:
93 case YUVPlane::V:
94 return GL_R8;
95 case YUVPlane::UV:
96 FATAL("Invalid plane:%d for format:%d", plane, format);
97 return 0;
98 }
99 case FRAMEWORK_FORMAT_YUV_420_888:
100 if (feature_is_enabled(kFeature_YUV420888toNV21)) {
101 switch (plane) {
102 case YUVPlane::Y:
103 return GL_R8;
104 case YUVPlane::UV:
105 return GL_RG8;
106 case YUVPlane::U:
107 case YUVPlane::V:
108 FATAL("Invalid plane:%d for format:%d", plane, format);
109 return 0;
110 }
111 } else {
112 switch (plane) {
113 case YUVPlane::Y:
114 case YUVPlane::U:
115 case YUVPlane::V:
116 return GL_R8;
117 case YUVPlane::UV:
118 FATAL("Invalid plane:%d for format:%d", plane, format);
119 return 0;
120 }
121 }
122 case FRAMEWORK_FORMAT_NV12:
123 switch (plane) {
124 case YUVPlane::Y:
125 return GL_R8;
126 case YUVPlane::UV:
127 return GL_RG8;
128 case YUVPlane::U:
129 case YUVPlane::V:
130 FATAL("Invalid plane:%d for format:%d", plane, format);
131 return 0;
132 }
133 case FRAMEWORK_FORMAT_P010:
134 switch (plane) {
135 case YUVPlane::Y:
136 return GL_R16UI;
137 case YUVPlane::UV:
138 return GL_RG16UI;
139 case YUVPlane::U:
140 case YUVPlane::V:
141 FATAL("Invalid plane:%d for format:%d", plane, format);
142 return 0;
143 }
144 default:
145 FATAL("Invalid format:%d", format);
146 return 0;
147 }
148 }
149
getGlPixelFormat(FrameworkFormat format,YUVPlane plane)150 GLenum getGlPixelFormat(FrameworkFormat format, YUVPlane plane) {
151 switch (format) {
152 case FRAMEWORK_FORMAT_YV12:
153 switch (plane) {
154 case YUVPlane::Y:
155 case YUVPlane::U:
156 case YUVPlane::V:
157 return GL_RED;
158 case YUVPlane::UV:
159 FATAL("Invalid plane:%d for format:%d", plane, format);
160 return 0;
161 }
162 case FRAMEWORK_FORMAT_YUV_420_888:
163 if (feature_is_enabled(kFeature_YUV420888toNV21)) {
164 switch (plane) {
165 case YUVPlane::Y:
166 return GL_RED;
167 case YUVPlane::UV:
168 return GL_RG;
169 case YUVPlane::U:
170 case YUVPlane::V:
171 FATAL("Invalid plane:%d for format:%d", plane, format);
172 return 0;
173 }
174 } else {
175 switch (plane) {
176 case YUVPlane::Y:
177 case YUVPlane::U:
178 case YUVPlane::V:
179 return GL_RED;
180 case YUVPlane::UV:
181 FATAL("Invalid plane:%d for format:%d", plane, format);
182 return 0;
183 }
184 }
185 case FRAMEWORK_FORMAT_NV12:
186 switch (plane) {
187 case YUVPlane::Y:
188 return GL_RED;
189 case YUVPlane::UV:
190 return GL_RG;
191 case YUVPlane::U:
192 case YUVPlane::V:
193 FATAL("Invalid plane:%d for format:%d", plane, format);
194 return 0;
195 }
196 case FRAMEWORK_FORMAT_P010:
197 switch (plane) {
198 case YUVPlane::Y:
199 return GL_RED_INTEGER;
200 case YUVPlane::UV:
201 return GL_RG_INTEGER;
202 case YUVPlane::U:
203 case YUVPlane::V:
204 FATAL("Invalid plane:%d for format:%d", plane, format);
205 return 0;
206 }
207 default:
208 FATAL("Invalid format:%d", format);
209 return 0;
210 }
211 }
212
getGlPixelType(FrameworkFormat format,YUVPlane plane)213 GLsizei getGlPixelType(FrameworkFormat format, YUVPlane plane) {
214 switch (format) {
215 case FRAMEWORK_FORMAT_YV12:
216 switch (plane) {
217 case YUVPlane::Y:
218 case YUVPlane::U:
219 case YUVPlane::V:
220 return GL_UNSIGNED_BYTE;
221 case YUVPlane::UV:
222 FATAL("Invalid plane:%d for format:%d", plane, format);
223 return 0;
224 }
225 case FRAMEWORK_FORMAT_YUV_420_888:
226 if (feature_is_enabled(kFeature_YUV420888toNV21)) {
227 switch (plane) {
228 case YUVPlane::Y:
229 case YUVPlane::UV:
230 return GL_UNSIGNED_BYTE;
231 case YUVPlane::U:
232 case YUVPlane::V:
233 FATAL("Invalid plane:%d for format:%d", plane, format);
234 return 0;
235 }
236 } else {
237 switch (plane) {
238 case YUVPlane::Y:
239 case YUVPlane::U:
240 case YUVPlane::V:
241 return GL_UNSIGNED_BYTE;
242 case YUVPlane::UV:
243 FATAL("Invalid plane:%d for format:%d", plane, format);
244 return 0;
245 }
246 }
247 case FRAMEWORK_FORMAT_NV12:
248 switch (plane) {
249 case YUVPlane::Y:
250 case YUVPlane::UV:
251 return GL_UNSIGNED_BYTE;
252 case YUVPlane::U:
253 case YUVPlane::V:
254 FATAL("Invalid plane:%d for format:%d", plane, format);
255 return 0;
256 }
257 case FRAMEWORK_FORMAT_P010:
258 switch (plane) {
259 case YUVPlane::Y:
260 case YUVPlane::UV:
261 return GL_UNSIGNED_SHORT;
262 case YUVPlane::U:
263 case YUVPlane::V:
264 FATAL("Invalid plane:%d for format:%d", plane, format);
265 return 0;
266 }
267 default:
268 FATAL("Invalid format:%d", format);
269 return 0;
270 }
271 }
272
273 // NV12 and YUV420 are all packed
NV12ToYUV420PlanarInPlaceConvert(int nWidth,int nHeight,uint8_t * pFrame,uint8_t * pQuad)274 static void NV12ToYUV420PlanarInPlaceConvert(int nWidth,
275 int nHeight,
276 uint8_t* pFrame,
277 uint8_t* pQuad) {
278 std::vector<uint8_t> tmp;
279 if (pQuad == nullptr) {
280 tmp.resize(nWidth * nHeight / 4);
281 pQuad = tmp.data();
282 }
283 int nPitch = nWidth;
284 uint8_t *puv = pFrame + nPitch * nHeight, *pu = puv,
285 *pv = puv + nPitch * nHeight / 4;
286 for (int y = 0; y < nHeight / 2; y++) {
287 for (int x = 0; x < nWidth / 2; x++) {
288 pu[y * nPitch / 2 + x] = puv[y * nPitch + x * 2];
289 pQuad[y * nWidth / 2 + x] = puv[y * nPitch + x * 2 + 1];
290 }
291 }
292 memcpy(pv, pQuad, nWidth * nHeight / 4);
293 }
294
alignToPower2(uint32_t val,uint32_t align)295 inline uint32_t alignToPower2(uint32_t val, uint32_t align) {
296 return (val + (align - 1)) & ~(align - 1);
297 }
298
299 // getYUVOffsets(), given a YUV-formatted buffer that is arranged
300 // according to the spec
301 // https://developer.android.com/reference/android/graphics/ImageFormat.html#YUV
302 // In particular, Android YUV widths are aligned to 16 pixels.
303 // Inputs:
304 // |yv12|: the YUV-formatted buffer
305 // Outputs:
306 // |yOffsetBytes|: offset into |yv12| of the start of the Y component
307 // |uOffsetBytes|: offset into |yv12| of the start of the U component
308 // |vOffsetBytes|: offset into |yv12| of the start of the V component
getYUVOffsets(int width,int height,FrameworkFormat format,uint32_t * yWidth,uint32_t * yHeight,uint32_t * yOffsetBytes,uint32_t * yStridePixels,uint32_t * yStrideBytes,uint32_t * uWidth,uint32_t * uHeight,uint32_t * uOffsetBytes,uint32_t * uStridePixels,uint32_t * uStrideBytes,uint32_t * vWidth,uint32_t * vHeight,uint32_t * vOffsetBytes,uint32_t * vStridePixels,uint32_t * vStrideBytes)309 static void getYUVOffsets(int width,
310 int height,
311 FrameworkFormat format,
312 uint32_t* yWidth,
313 uint32_t* yHeight,
314 uint32_t* yOffsetBytes,
315 uint32_t* yStridePixels,
316 uint32_t* yStrideBytes,
317 uint32_t* uWidth,
318 uint32_t* uHeight,
319 uint32_t* uOffsetBytes,
320 uint32_t* uStridePixels,
321 uint32_t* uStrideBytes,
322 uint32_t* vWidth,
323 uint32_t* vHeight,
324 uint32_t* vOffsetBytes,
325 uint32_t* vStridePixels,
326 uint32_t* vStrideBytes) {
327 switch (format) {
328 case FRAMEWORK_FORMAT_YV12: {
329 *yWidth = width;
330 *yHeight = height;
331 *yOffsetBytes = 0;
332 // Luma stride is 32 bytes aligned.
333 *yStridePixels = alignToPower2(width, 32);
334 *yStrideBytes = *yStridePixels;
335
336 // Chroma stride is 16 bytes aligned.
337 *vWidth = width / 2;
338 *vHeight = height / 2;
339 *vOffsetBytes = (*yStrideBytes) * (*yHeight);
340 *vStridePixels = (*yStridePixels) / 2;
341 *vStrideBytes = (*vStridePixels);
342
343 *uWidth = width / 2;
344 *uHeight = height / 2;
345 *uOffsetBytes = (*vOffsetBytes) + ((*vStrideBytes) * (*vHeight));
346 *uStridePixels = (*yStridePixels) / 2;
347 *uStrideBytes = *uStridePixels;
348 break;
349 }
350 case FRAMEWORK_FORMAT_YUV_420_888: {
351 if (feature_is_enabled(kFeature_YUV420888toNV21)) {
352 *yWidth = width;
353 *yHeight = height;
354 *yOffsetBytes = 0;
355 *yStridePixels = width;
356 *yStrideBytes = *yStridePixels;
357
358 *vWidth = width / 2;
359 *vHeight = height / 2;
360 *vOffsetBytes = (*yStrideBytes) * (*yHeight);
361 *vStridePixels = (*yStridePixels) / 2;
362 *vStrideBytes = (*vStridePixels);
363
364 *uWidth = width / 2;
365 *uHeight = height / 2;
366 *uOffsetBytes = (*vOffsetBytes) + 1;
367 *uStridePixels = (*yStridePixels) / 2;
368 *uStrideBytes = *uStridePixels;
369 } else {
370 *yWidth = width;
371 *yHeight = height;
372 *yOffsetBytes = 0;
373 *yStridePixels = width;
374 *yStrideBytes = *yStridePixels;
375
376 *uWidth = width / 2;
377 *uHeight = height / 2;
378 *uOffsetBytes = (*yStrideBytes) * (*yHeight);
379 *uStridePixels = (*yStridePixels) / 2;
380 *uStrideBytes = *uStridePixels;
381
382 *vWidth = width / 2;
383 *vHeight = height / 2;
384 *vOffsetBytes = (*uOffsetBytes) + ((*uStrideBytes) * (*uHeight));
385 *vStridePixels = (*yStridePixels) / 2;
386 *vStrideBytes = (*vStridePixels);
387 }
388 break;
389 }
390 case FRAMEWORK_FORMAT_NV12: {
391 *yWidth = width;
392 *yHeight = height;
393 *yOffsetBytes = 0;
394 *yStridePixels = width;
395 *yStrideBytes = *yStridePixels;
396
397 *uWidth = width / 2;
398 *uHeight = height / 2;
399 *uOffsetBytes = (*yStrideBytes) * (*yHeight);
400 *uStridePixels = (*yStridePixels) / 2;
401 *uStrideBytes = *uStridePixels;
402
403 *vWidth = width / 2;
404 *vHeight = height / 2;
405 *vOffsetBytes = (*uOffsetBytes) + 1;
406 *vStridePixels = (*yStridePixels) / 2;
407 *vStrideBytes = (*vStridePixels);
408 break;
409 }
410 case FRAMEWORK_FORMAT_P010: {
411 *yWidth = width;
412 *yHeight = height;
413 *yOffsetBytes = 0;
414 *yStridePixels = width;
415 *yStrideBytes = (*yStridePixels) * /*bytes per pixel=*/2;
416
417 *uWidth = width / 2;
418 *uHeight = height / 2;
419 *uOffsetBytes = (*yStrideBytes) * (*yHeight);
420 *uStridePixels = (*uWidth);
421 *uStrideBytes = *uStridePixels * /*bytes per pixel=*/2;
422
423 *vWidth = width / 2;
424 *vHeight = height / 2;
425 *vOffsetBytes = (*uOffsetBytes) + 2;
426 *vStridePixels = (*vWidth);
427 *vStrideBytes = (*vStridePixels) * /*bytes per pixel=*/2;
428 break;
429 }
430 case FRAMEWORK_FORMAT_GL_COMPATIBLE: {
431 FATAL("Input not a YUV format! (FRAMEWORK_FORMAT_GL_COMPATIBLE)");
432 }
433 default: {
434 FATAL("Unknown format: 0x%x", format);
435 }
436 }
437 }
438
439 // Allocates an OpenGL texture that is large enough for a single plane of
440 // a YUV buffer of the given format and returns the texture name in the
441 // `outTextureName` argument.
createYUVGLTex(GLenum textureUnit,GLsizei width,GLsizei height,FrameworkFormat format,YUVPlane plane,GLuint * outTextureName)442 void YUVConverter::createYUVGLTex(GLenum textureUnit,
443 GLsizei width,
444 GLsizei height,
445 FrameworkFormat format,
446 YUVPlane plane,
447 GLuint* outTextureName) {
448 YUV_DEBUG_LOG("w:%d h:%d format:%d plane:%d", width, height, format, plane);
449
450 s_gles2.glActiveTexture(textureUnit);
451 s_gles2.glGenTextures(1, outTextureName);
452 s_gles2.glBindTexture(GL_TEXTURE_2D, *outTextureName);
453 s_gles2.glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST);
454 s_gles2.glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST);
455 GLint unprevAlignment = 0;
456 s_gles2.glGetIntegerv(GL_UNPACK_ALIGNMENT, &unprevAlignment);
457 s_gles2.glPixelStorei(GL_UNPACK_ALIGNMENT, 1);
458 const GLint textureFormat = getGlTextureFormat(format, plane);
459 const GLenum pixelFormat = getGlPixelFormat(format, plane);
460 const GLenum pixelType = getGlPixelType(format, plane);
461 s_gles2.glTexImage2D(GL_TEXTURE_2D, 0, textureFormat, width, height, 0, pixelFormat, pixelType, NULL);
462 s_gles2.glPixelStorei(GL_UNPACK_ALIGNMENT, unprevAlignment);
463 s_gles2.glActiveTexture(GL_TEXTURE0);
464 }
465
readYUVTex(GLuint tex,FrameworkFormat format,YUVPlane plane,void * pixels,uint32_t pixelsStride)466 static void readYUVTex(GLuint tex, FrameworkFormat format, YUVPlane plane, void* pixels,
467 uint32_t pixelsStride) {
468 YUV_DEBUG_LOG("format%d plane:%d pixels:%p", format, plane, pixels);
469
470 GLuint prevTexture = 0;
471 s_gles2.glGetIntegerv(GL_TEXTURE_BINDING_2D, (GLint*)&prevTexture);
472 s_gles2.glBindTexture(GL_TEXTURE_2D, tex);
473 GLint prevAlignment = 0;
474 s_gles2.glGetIntegerv(GL_PACK_ALIGNMENT, &prevAlignment);
475 s_gles2.glPixelStorei(GL_PACK_ALIGNMENT, 1);
476 GLint prevStride = 0;
477 s_gles2.glGetIntegerv(GL_PACK_ROW_LENGTH, &prevStride);
478 s_gles2.glPixelStorei(GL_PACK_ROW_LENGTH, pixelsStride);
479
480 const GLenum pixelFormat = getGlPixelFormat(format, plane);
481 const GLenum pixelType = getGlPixelType(format, plane);
482 if (s_gles2.glGetTexImage) {
483 s_gles2.glGetTexImage(GL_TEXTURE_2D, 0, pixelFormat, pixelType, pixels);
484 } else {
485 YUV_DEBUG_LOG("empty glGetTexImage");
486 }
487
488 s_gles2.glPixelStorei(GL_PACK_ROW_LENGTH, prevStride);
489 s_gles2.glPixelStorei(GL_PACK_ALIGNMENT, prevAlignment);
490 s_gles2.glBindTexture(GL_TEXTURE_2D, prevTexture);
491 }
492
493 // Updates a given YUV buffer's plane texture at the coordinates
494 // (x, y, width, height), with the raw YUV data in |pixels|. We
495 // cannot view the result properly until after conversion; this is
496 // to be used only as input to the conversion shader.
subUpdateYUVGLTex(GLenum texture_unit,GLuint tex,int x,int y,int width,int height,FrameworkFormat format,YUVPlane plane,const void * pixels)497 static void subUpdateYUVGLTex(GLenum texture_unit,
498 GLuint tex,
499 int x,
500 int y,
501 int width,
502 int height,
503 FrameworkFormat format,
504 YUVPlane plane,
505 const void* pixels) {
506 YUV_DEBUG_LOG("x:%d y:%d w:%d h:%d format:%d plane:%d", x, y, width, height, format, plane);
507
508 const GLenum pixelFormat = getGlPixelFormat(format, plane);
509 const GLenum pixelType = getGlPixelType(format, plane);
510
511 s_gles2.glActiveTexture(texture_unit);
512 s_gles2.glBindTexture(GL_TEXTURE_2D, tex);
513 GLint unprevAlignment = 0;
514 s_gles2.glGetIntegerv(GL_UNPACK_ALIGNMENT, &unprevAlignment);
515 s_gles2.glPixelStorei(GL_UNPACK_ALIGNMENT, 1);
516 s_gles2.glTexSubImage2D(GL_TEXTURE_2D, 0, x, y, width, height, pixelFormat, pixelType, pixels);
517 s_gles2.glPixelStorei(GL_UNPACK_ALIGNMENT, unprevAlignment);
518 s_gles2.glActiveTexture(GL_TEXTURE0);
519 }
520
createYUVGLShader()521 void YUVConverter::createYUVGLShader() {
522 YUV_DEBUG_LOG("format:%d", mFormat);
523
524 // P010 needs uint samplers.
525 if (mFormat == FRAMEWORK_FORMAT_P010 && !mHasGlsl3Support) {
526 return;
527 }
528
529 static const char kVertShader[] = R"(
530 precision highp float;
531 attribute mediump vec4 aPosition;
532 attribute highp vec2 aTexCoord;
533 varying highp vec2 vTexCoord;
534 void main(void) {
535 gl_Position = aPosition;
536 vTexCoord = aTexCoord;
537 }
538 )";
539
540 static const char kFragShaderVersion3[] = R"(#version 300 es)";
541
542 static const char kFragShaderBegin[] = R"(
543 precision highp float;
544
545 varying highp vec2 vTexCoord;
546
547 uniform highp float uYWidthCutoff;
548 uniform highp float uUVWidthCutoff;
549 )";
550
551 static const char kSamplerUniforms[] = R"(
552 uniform sampler2D uSamplerY;
553 uniform sampler2D uSamplerU;
554 uniform sampler2D uSamplerV;
555 )";
556 static const char kSamplerUniformsUint[] = R"(
557 uniform usampler2D uSamplerY;
558 uniform usampler2D uSamplerU;
559 uniform usampler2D uSamplerV;
560 )";
561
562 static const char kFragShaderMainBegin[] = R"(
563 void main(void) {
564 highp vec2 yTexCoords = vTexCoord;
565 highp vec2 uvTexCoords = vTexCoord;
566
567 // For textures with extra padding for alignment (e.g. YV12 pads to 16),
568 // scale the coordinates to only sample from the non-padded area.
569 yTexCoords.x *= uYWidthCutoff;
570 uvTexCoords.x *= uUVWidthCutoff;
571
572 highp vec3 yuv;
573 )";
574
575 static const char kSampleY[] = R"(
576 yuv[0] = texture2D(uSamplerY, yTexCoords).r;
577 )";
578 static const char kSampleUV[] = R"(
579 yuv[1] = texture2D(uSamplerU, uvTexCoords).r;
580 yuv[2] = texture2D(uSamplerV, uvTexCoords).r;
581 )";
582 static const char kSampleInterleavedUV[] = R"(
583 // Note: uSamplerU and vSamplerV refer to the same texture.
584 yuv[1] = texture2D(uSamplerU, uvTexCoords).r;
585 yuv[2] = texture2D(uSamplerV, uvTexCoords).g;
586 )";
587 static const char kSampleInterleavedVU[] = R"(
588 // Note: uSamplerU and vSamplerV refer to the same texture.
589 yuv[1] = texture2D(uSamplerU, uvTexCoords).g;
590 yuv[2] = texture2D(uSamplerV, uvTexCoords).r;
591 )";
592
593 static const char kSampleP010[] = R"(
594 uint yRaw = texture(uSamplerY, yTexCoords).r;
595 uint uRaw = texture(uSamplerU, uvTexCoords).r;
596 uint vRaw = texture(uSamplerV, uvTexCoords).g;
597
598 // P010 values are stored in the upper 10-bits of 16-bit unsigned shorts.
599 yuv[0] = float(yRaw >> 6) / 1023.0;
600 yuv[1] = float(uRaw >> 6) / 1023.0;
601 yuv[2] = float(vRaw >> 6) / 1023.0;
602 )";
603
604 static const char kFragShaderMainEnd[] = R"(
605 yuv[0] = yuv[0] - 0.0625;
606 yuv[1] = 0.96 * (yuv[1] - 0.5);
607 yuv[2] = (yuv[2] - 0.5);
608
609 highp float yscale = 1.1643835616438356;
610 highp vec3 rgb = mat3( yscale, yscale, yscale,
611 0, -0.39176229009491365, 2.017232142857143,
612 1.5960267857142856, -0.8129676472377708, 0) * yuv;
613
614 gl_FragColor = vec4(rgb, 1.0);
615 }
616 )";
617
618 std::string vertShaderSource(kVertShader);
619 std::string fragShaderSource;
620
621 if (mFormat == FRAMEWORK_FORMAT_P010) {
622 fragShaderSource += kFragShaderVersion3;
623 }
624
625 fragShaderSource += kFragShaderBegin;
626
627 if (mFormat == FRAMEWORK_FORMAT_P010) {
628 fragShaderSource += kSamplerUniformsUint;
629 } else {
630 fragShaderSource += kSamplerUniforms;
631 }
632
633 fragShaderSource += kFragShaderMainBegin;
634
635 switch (mFormat) {
636 case FRAMEWORK_FORMAT_NV12:
637 case FRAMEWORK_FORMAT_YUV_420_888:
638 case FRAMEWORK_FORMAT_YV12:
639 fragShaderSource += kSampleY;
640 if (isInterleaved(mFormat)) {
641 if (getInterleaveDirection(mFormat) == YUVInterleaveDirection::UV) {
642 fragShaderSource += kSampleInterleavedUV;
643 } else {
644 fragShaderSource += kSampleInterleavedVU;
645 }
646 } else {
647 fragShaderSource += kSampleUV;
648 }
649 break;
650 case FRAMEWORK_FORMAT_P010:
651 fragShaderSource += kSampleP010;
652 break;
653 default:
654 FATAL("%s: invalid format:%d", __FUNCTION__, mFormat);
655 return;
656 }
657
658 fragShaderSource += kFragShaderMainEnd;
659
660 YUV_DEBUG_LOG("format:%d vert-source:%s frag-source:%s", mFormat, vertShaderSource.c_str(), fragShaderSource.c_str());
661
662 const GLchar* const vertShaderSourceChars = vertShaderSource.c_str();
663 const GLchar* const fragShaderSourceChars = fragShaderSource.c_str();
664 const GLint vertShaderSourceLen = vertShaderSource.length();
665 const GLint fragShaderSourceLen = fragShaderSource.length();
666
667 GLuint vertShader = s_gles2.glCreateShader(GL_VERTEX_SHADER);
668 GLuint fragShader = s_gles2.glCreateShader(GL_FRAGMENT_SHADER);
669 s_gles2.glShaderSource(vertShader, 1, &vertShaderSourceChars, &vertShaderSourceLen);
670 s_gles2.glShaderSource(fragShader, 1, &fragShaderSourceChars, &fragShaderSourceLen);
671 s_gles2.glCompileShader(vertShader);
672 s_gles2.glCompileShader(fragShader);
673
674 for (GLuint shader : {vertShader, fragShader}) {
675 GLint status = GL_FALSE;
676 s_gles2.glGetShaderiv(shader, GL_COMPILE_STATUS, &status);
677 if (status == GL_FALSE) {
678 GLchar error[1024];
679 s_gles2.glGetShaderInfoLog(shader, sizeof(error), nullptr, &error[0]);
680 FATAL("Failed to compile YUV conversion shader: %s", error);
681 s_gles2.glDeleteShader(shader);
682 return;
683 }
684 }
685
686 mProgram = s_gles2.glCreateProgram();
687 s_gles2.glAttachShader(mProgram, vertShader);
688 s_gles2.glAttachShader(mProgram, fragShader);
689 s_gles2.glLinkProgram(mProgram);
690
691 GLint status = GL_FALSE;
692 s_gles2.glGetProgramiv(mProgram, GL_LINK_STATUS, &status);
693 if (status == GL_FALSE) {
694 GLchar error[1024];
695 s_gles2.glGetProgramInfoLog(mProgram, sizeof(error), 0, &error[0]);
696 FATAL("Failed to link YUV conversion program: %s", error);
697 s_gles2.glDeleteProgram(mProgram);
698 mProgram = 0;
699 return;
700 }
701
702 mUniformLocYWidthCutoff = s_gles2.glGetUniformLocation(mProgram, "uYWidthCutoff");
703 mUniformLocUVWidthCutoff = s_gles2.glGetUniformLocation(mProgram, "uUVWidthCutoff");
704 mUniformLocSamplerY = s_gles2.glGetUniformLocation(mProgram, "uSamplerY");
705 mUniformLocSamplerU = s_gles2.glGetUniformLocation(mProgram, "uSamplerU");
706 mUniformLocSamplerV = s_gles2.glGetUniformLocation(mProgram, "uSamplerV");
707 mAttributeLocPos = s_gles2.glGetAttribLocation(mProgram, "aPosition");
708 mAttributeLocTexCoord = s_gles2.glGetAttribLocation(mProgram, "aTexCoord");
709
710 s_gles2.glDeleteShader(vertShader);
711 s_gles2.glDeleteShader(fragShader);
712 }
713
createYUVGLFullscreenQuad()714 void YUVConverter::createYUVGLFullscreenQuad() {
715 s_gles2.glGenBuffers(1, &mQuadVertexBuffer);
716 s_gles2.glGenBuffers(1, &mQuadIndexBuffer);
717
718 static const float kVertices[] = {
719 +1, -1, +0, +1, +0,
720 +1, +1, +0, +1, +1,
721 -1, +1, +0, +0, +1,
722 -1, -1, +0, +0, +0,
723 };
724
725 static const GLubyte kIndices[] = { 0, 1, 2, 2, 3, 0 };
726
727 s_gles2.glBindBuffer(GL_ARRAY_BUFFER, mQuadVertexBuffer);
728 s_gles2.glBufferData(GL_ARRAY_BUFFER, sizeof(kVertices), kVertices, GL_STATIC_DRAW);
729 s_gles2.glBindBuffer(GL_ELEMENT_ARRAY_BUFFER, mQuadIndexBuffer);
730 s_gles2.glBufferData(GL_ELEMENT_ARRAY_BUFFER, sizeof(kIndices), kIndices, GL_STATIC_DRAW);
731 }
732
doYUVConversionDraw(GLuint program,GLint uniformLocYWidthCutoff,GLint uniformLocUVWidthCutoff,GLint uniformLocYSampler,GLint uniformLocUSampler,GLint uniformLocVSampler,GLint attributeLocTexCoord,GLint attributeLocPos,GLuint quadVertexBuffer,GLuint quadIndexBuffer,float uYWidthCutoff,float uUVWidthCutoff)733 static void doYUVConversionDraw(GLuint program,
734 GLint uniformLocYWidthCutoff,
735 GLint uniformLocUVWidthCutoff,
736 GLint uniformLocYSampler,
737 GLint uniformLocUSampler,
738 GLint uniformLocVSampler,
739 GLint attributeLocTexCoord,
740 GLint attributeLocPos,
741 GLuint quadVertexBuffer,
742 GLuint quadIndexBuffer,
743 float uYWidthCutoff,
744 float uUVWidthCutoff) {
745 const GLsizei kVertexAttribStride = 5 * sizeof(GL_FLOAT);
746 const GLvoid* kVertexAttribPosOffset = (GLvoid*)0;
747 const GLvoid* kVertexAttribCoordOffset = (GLvoid*)(3 * sizeof(GL_FLOAT));
748
749 s_gles2.glUseProgram(program);
750
751 s_gles2.glUniform1f(uniformLocYWidthCutoff, uYWidthCutoff);
752 s_gles2.glUniform1f(uniformLocUVWidthCutoff, uUVWidthCutoff);
753
754 s_gles2.glUniform1i(uniformLocYSampler, 0);
755 s_gles2.glUniform1i(uniformLocUSampler, 1);
756 s_gles2.glUniform1i(uniformLocVSampler, 2);
757
758 s_gles2.glBindBuffer(GL_ARRAY_BUFFER, quadVertexBuffer);
759 s_gles2.glEnableVertexAttribArray(attributeLocPos);
760 s_gles2.glEnableVertexAttribArray(attributeLocTexCoord);
761
762 s_gles2.glVertexAttribPointer(attributeLocPos, 3, GL_FLOAT, false,
763 kVertexAttribStride,
764 kVertexAttribPosOffset);
765 s_gles2.glVertexAttribPointer(attributeLocTexCoord, 2, GL_FLOAT, false,
766 kVertexAttribStride,
767 kVertexAttribCoordOffset);
768
769 s_gles2.glBindBuffer(GL_ELEMENT_ARRAY_BUFFER, quadIndexBuffer);
770 s_gles2.glDrawElements(GL_TRIANGLES, 6, GL_UNSIGNED_BYTE, 0);
771
772 s_gles2.glDisableVertexAttribArray(attributeLocPos);
773 s_gles2.glDisableVertexAttribArray(attributeLocTexCoord);
774 }
775
776 // initialize(): allocate GPU memory for YUV components,
777 // and create shaders and vertex data.
YUVConverter(int width,int height,FrameworkFormat format)778 YUVConverter::YUVConverter(int width, int height, FrameworkFormat format)
779 : mWidth(width),
780 mHeight(height),
781 mFormat(format),
782 mColorBufferFormat(format) {}
783
init(int width,int height,FrameworkFormat format)784 void YUVConverter::init(int width, int height, FrameworkFormat format) {
785 YUV_DEBUG_LOG("w:%d h:%d format:%d", width, height, format);
786
787 uint32_t yWidth, yHeight = 0, yOffsetBytes, yStridePixels = 0, yStrideBytes;
788 uint32_t uWidth, uHeight = 0, uOffsetBytes, uStridePixels = 0, uStrideBytes;
789 uint32_t vWidth, vHeight = 0, vOffsetBytes, vStridePixels = 0, vStrideBytes;
790 getYUVOffsets(width, height, mFormat,
791 &yWidth, &yHeight, &yOffsetBytes, &yStridePixels, &yStrideBytes,
792 &uWidth, &uHeight, &uOffsetBytes, &uStridePixels, &uStrideBytes,
793 &vWidth, &vHeight, &vOffsetBytes, &vStridePixels, &vStrideBytes);
794 mWidth = width;
795 mHeight = height;
796 if (!mTextureY) {
797 createYUVGLTex(GL_TEXTURE0, yStridePixels, yHeight, mFormat, YUVPlane::Y, &mTextureY);
798 }
799 if (isInterleaved(mFormat)) {
800 if (!mTextureU) {
801 createYUVGLTex(GL_TEXTURE1, uStridePixels, uHeight, mFormat, YUVPlane::UV, &mTextureU);
802 mTextureV = mTextureU;
803 }
804 } else {
805 if (!mTextureU) {
806 createYUVGLTex(GL_TEXTURE1, uStridePixels, uHeight, mFormat, YUVPlane::U, &mTextureU);
807 }
808 if (!mTextureV) {
809 createYUVGLTex(GL_TEXTURE2, vStridePixels, vHeight, mFormat, YUVPlane::V, &mTextureV);
810 }
811 }
812
813 int glesMajor;
814 int glesMinor;
815 emugl::getGlesVersion(&glesMajor, &glesMinor);
816 mHasGlsl3Support = glesMajor >= 3;
817 YUV_DEBUG_LOG("YUVConverter has GLSL ES 3 support:%s (major:%d minor:%d", (mHasGlsl3Support ? "yes" : "no"), glesMajor, glesMinor);
818
819 createYUVGLShader();
820 createYUVGLFullscreenQuad();
821 }
822
saveGLState()823 void YUVConverter::saveGLState() {
824 s_gles2.glGetFloatv(GL_VIEWPORT, mCurrViewport);
825 s_gles2.glGetIntegerv(GL_ACTIVE_TEXTURE, &mCurrTexUnit);
826 s_gles2.glGetIntegerv(GL_TEXTURE_BINDING_2D, &mCurrTexBind);
827 s_gles2.glGetIntegerv(GL_CURRENT_PROGRAM, &mCurrProgram);
828 s_gles2.glGetIntegerv(GL_ARRAY_BUFFER_BINDING, &mCurrVbo);
829 s_gles2.glGetIntegerv(GL_ELEMENT_ARRAY_BUFFER_BINDING, &mCurrIbo);
830 }
831
restoreGLState()832 void YUVConverter::restoreGLState() {
833 s_gles2.glViewport(mCurrViewport[0], mCurrViewport[1],
834 mCurrViewport[2], mCurrViewport[3]);
835 s_gles2.glActiveTexture(mCurrTexUnit);
836 s_gles2.glUseProgram(mCurrProgram);
837 s_gles2.glBindBuffer(GL_ARRAY_BUFFER, mCurrVbo);
838 s_gles2.glBindBuffer(GL_ELEMENT_ARRAY_BUFFER, mCurrIbo);
839 }
840
getDataSize()841 uint32_t YUVConverter::getDataSize() {
842 uint32_t align = (mFormat == FRAMEWORK_FORMAT_YV12) ? 16 : 1;
843 uint32_t yStrideBytes = (mWidth + (align - 1)) & ~(align - 1);
844 uint32_t uvStride = (yStrideBytes / 2 + (align - 1)) & ~(align - 1);
845 uint32_t uvHeight = mHeight / 2;
846 uint32_t dataSize = yStrideBytes * mHeight + 2 * (uvHeight * uvStride);
847 YUV_DEBUG_LOG("w:%d h:%d format:%d has data size:%d", mWidth, mHeight, mFormat, dataSize);
848 return dataSize;
849 }
850
readPixels(uint8_t * pixels,uint32_t pixels_size)851 void YUVConverter::readPixels(uint8_t* pixels, uint32_t pixels_size) {
852 YUV_DEBUG_LOG("w:%d h:%d format:%d pixels:%p pixels-size:%d", mWidth, mHeight, mFormat, pixels, pixels_size);
853
854 uint32_t yWidth, yHeight, yOffsetBytes, yStridePixels, yStrideBytes;
855 uint32_t uWidth, uHeight, uOffsetBytes, uStridePixels, uStrideBytes;
856 uint32_t vWidth, vHeight, vOffsetBytes, vStridePixels, vStrideBytes;
857 getYUVOffsets(mWidth, mHeight, mFormat,
858 &yWidth, &yHeight, &yOffsetBytes, &yStridePixels, &yStrideBytes,
859 &uWidth, &uHeight, &uOffsetBytes, &uStridePixels, &uStrideBytes,
860 &vWidth, &vHeight, &vOffsetBytes, &vStridePixels, &vStrideBytes);
861
862 if (isInterleaved(mFormat)) {
863 readYUVTex(mTextureV, mFormat, YUVPlane::UV, pixels + std::min(uOffsetBytes, vOffsetBytes),
864 uStridePixels);
865 } else {
866 readYUVTex(mTextureU, mFormat, YUVPlane::U, pixels + uOffsetBytes, uStridePixels);
867 readYUVTex(mTextureV, mFormat, YUVPlane::V, pixels + vOffsetBytes, vStridePixels);
868 }
869
870 if (mFormat == FRAMEWORK_FORMAT_NV12 && mColorBufferFormat == FRAMEWORK_FORMAT_YUV_420_888) {
871 NV12ToYUV420PlanarInPlaceConvert(mWidth, mHeight, pixels, pixels);
872 }
873
874 // Read the Y plane last because so that we can use it as a scratch space.
875 readYUVTex(mTextureY, mFormat, YUVPlane::Y, pixels + yOffsetBytes, yStridePixels);
876 }
877
swapTextures(FrameworkFormat format,GLuint * textures)878 void YUVConverter::swapTextures(FrameworkFormat format, GLuint* textures) {
879 if (isInterleaved(format)) {
880 std::swap(textures[0], mTextureY);
881 std::swap(textures[1], mTextureU);
882 mTextureV = mTextureU;
883 } else {
884 std::swap(textures[0], mTextureY);
885 std::swap(textures[1], mTextureU);
886 std::swap(textures[2], mTextureV);
887 }
888
889 mFormat = format;
890 mTexturesSwapped = true;
891 }
892
893 // drawConvert: per-frame updates.
894 // Update YUV textures, then draw the fullscreen
895 // quad set up above, which results in a framebuffer
896 // with the RGB colors.
drawConvert(int x,int y,int width,int height,const char * pixels)897 void YUVConverter::drawConvert(int x, int y, int width, int height, const char* pixels) {
898 drawConvertFromFormat(mFormat, x, y, width, height, pixels);
899 }
900
drawConvertFromFormat(FrameworkFormat format,int x,int y,int width,int height,const char * pixels)901 void YUVConverter::drawConvertFromFormat(FrameworkFormat format, int x, int y, int width,
902 int height, const char* pixels) {
903 saveGLState();
904 if (pixels && (width != mWidth || height != mHeight)) {
905 reset();
906 }
907
908 bool uploadFormatChanged = !mTexturesSwapped && pixels && (format != mFormat);
909 bool initNeeded = (mProgram == 0) || uploadFormatChanged;
910
911 if (initNeeded) {
912 if (uploadFormatChanged) {
913 mFormat = format;
914 // TODO: missing cherry-picks, put it back
915 // b/264928117
916 //mCbFormat = format;
917 reset();
918 }
919 init(width, height, mFormat);
920 }
921
922 if (mFormat == FRAMEWORK_FORMAT_P010 && !mHasGlsl3Support) {
923 // TODO: perhaps fallback to just software conversion.
924 return;
925 }
926
927 uint32_t yWidth = 0, yHeight = 0, yOffsetBytes, yStridePixels = 0, yStrideBytes;
928 uint32_t uWidth = 0, uHeight = 0, uOffsetBytes, uStridePixels = 0, uStrideBytes;
929 uint32_t vWidth = 0, vHeight = 0, vOffsetBytes, vStridePixels = 0, vStrideBytes;
930 getYUVOffsets(width, height, mFormat,
931 &yWidth, &yHeight, &yOffsetBytes, &yStridePixels, &yStrideBytes,
932 &uWidth, &uHeight, &uOffsetBytes, &uStridePixels, &uStrideBytes,
933 &vWidth, &vHeight, &vOffsetBytes, &vStridePixels, &vStrideBytes);
934
935 YUV_DEBUG_LOG("Updating YUV textures for drawConvert() "
936 "x:%d y:%d width:%d height:%d "
937 "yWidth:%d yHeight:%d yOffsetBytes:%d yStridePixels:%d yStrideBytes:%d "
938 "uWidth:%d uHeight:%d uOffsetBytes:%d uStridePixels:%d uStrideBytes:%d "
939 "vWidth:%d vHeight:%d vOffsetBytes:%d vStridePixels:%d vStrideBytes:%d ",
940 x, y, width, height,
941 yWidth, yHeight, yOffsetBytes, yStridePixels, yStrideBytes,
942 uWidth, uHeight, uOffsetBytes, uStridePixels, uStrideBytes,
943 vWidth, vHeight, vOffsetBytes, vStridePixels, vStrideBytes);
944
945 s_gles2.glViewport(x, y, width, height);
946
947 updateCutoffs(static_cast<float>(yWidth),
948 static_cast<float>(yStridePixels),
949 static_cast<float>(uWidth),
950 static_cast<float>(uStridePixels));
951
952 if (pixels) {
953 subUpdateYUVGLTex(GL_TEXTURE0, mTextureY, x, y, yStridePixels, yHeight, mFormat, YUVPlane::Y, pixels + yOffsetBytes);
954 if (isInterleaved(mFormat)) {
955 subUpdateYUVGLTex(GL_TEXTURE1, mTextureU, x, y, uStridePixels, uHeight, mFormat, YUVPlane::UV, pixels + std::min(uOffsetBytes, vOffsetBytes));
956 } else {
957 subUpdateYUVGLTex(GL_TEXTURE1, mTextureU, x, y, uStridePixels, uHeight, mFormat, YUVPlane::U, pixels + uOffsetBytes);
958 subUpdateYUVGLTex(GL_TEXTURE2, mTextureV, x, y, vStridePixels, vHeight, mFormat, YUVPlane::V, pixels + vOffsetBytes);
959 }
960 } else {
961 // special case: draw from texture, only support NV12 for now
962 // as cuvid's native format is NV12.
963 // TODO: add more formats if there are such needs in the future.
964 assert(mFormat == FRAMEWORK_FORMAT_NV12);
965 }
966
967 s_gles2.glActiveTexture(GL_TEXTURE0);
968 s_gles2.glBindTexture(GL_TEXTURE_2D, mTextureY);
969 s_gles2.glActiveTexture(GL_TEXTURE1);
970 s_gles2.glBindTexture(GL_TEXTURE_2D, mTextureU);
971 s_gles2.glActiveTexture(GL_TEXTURE2);
972 s_gles2.glBindTexture(GL_TEXTURE_2D, mTextureV);
973
974 doYUVConversionDraw(mProgram,
975 mUniformLocYWidthCutoff,
976 mUniformLocUVWidthCutoff,
977 mUniformLocSamplerY,
978 mUniformLocSamplerU,
979 mUniformLocSamplerV,
980 mAttributeLocTexCoord,
981 mAttributeLocPos,
982 mQuadVertexBuffer,
983 mQuadIndexBuffer,
984 mYWidthCutoff,
985 mUVWidthCutoff);
986
987 restoreGLState();
988 }
989
updateCutoffs(float yWidth,float yStridePixels,float uvWidth,float uvStridePixels)990 void YUVConverter::updateCutoffs(float yWidth, float yStridePixels,
991 float uvWidth, float uvStridePixels) {
992 switch (mFormat) {
993 case FRAMEWORK_FORMAT_YV12:
994 mYWidthCutoff = yWidth / yStridePixels;
995 mUVWidthCutoff = uvWidth / uvStridePixels;
996 break;
997 case FRAMEWORK_FORMAT_NV12:
998 case FRAMEWORK_FORMAT_P010:
999 case FRAMEWORK_FORMAT_YUV_420_888:
1000 mYWidthCutoff = 1.0f;
1001 mUVWidthCutoff = 1.0f;
1002 break;
1003 case FRAMEWORK_FORMAT_GL_COMPATIBLE:
1004 FATAL("Input not a YUV format!");
1005 }
1006 }
1007
reset()1008 void YUVConverter::reset() {
1009 if (mQuadIndexBuffer) s_gles2.glDeleteBuffers(1, &mQuadIndexBuffer);
1010 if (mQuadVertexBuffer) s_gles2.glDeleteBuffers(1, &mQuadVertexBuffer);
1011 if (mProgram) s_gles2.glDeleteProgram(mProgram);
1012 if (mTextureY) s_gles2.glDeleteTextures(1, &mTextureY);
1013 if (isInterleaved(mFormat)) {
1014 if (mTextureU) s_gles2.glDeleteTextures(1, &mTextureU);
1015 } else {
1016 if (mTextureU) s_gles2.glDeleteTextures(1, &mTextureU);
1017 if (mTextureV) s_gles2.glDeleteTextures(1, &mTextureV);
1018 }
1019 mQuadIndexBuffer = 0;
1020 mQuadVertexBuffer = 0;
1021 mProgram = 0;
1022 mTextureY = 0;
1023 mTextureU = 0;
1024 mTextureV = 0;
1025 }
1026
~YUVConverter()1027 YUVConverter::~YUVConverter() {
1028 reset();
1029 }
1030
1031 } // namespace gl
1032 } // namespace gfxstream
1033