1 /* Gstreamer video blending utility functions
2 *
3 * Copied/pasted from gst/videoconvert/videoconvert.c
4 * Copyright (C) 2010 David Schleef <ds@schleef.org>
5 * Copyright (C) 2010 Sebastian Dröge <sebastian.droege@collabora.co.uk>
6 *
7 * Copyright (C) <2011> Intel Corporation
8 * Copyright (C) <2011> Collabora Ltd.
9 * Copyright (C) <2011> Thibault Saunier <thibault.saunier@collabora.com>
10 *
11 * This library is free software; you can redistribute it and/or
12 * modify it under the terms of the GNU Library General Public
13 * License as published by the Free Software Foundation; either
14 * version 2 of the License, or (at your option) any later version.
15 *
16 * This library is distributed in the hope that it will be useful,
17 * but WITHOUT ANY WARRANTY; without even the implied warranty of
18 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
19 * Library General Public License for more details.
20 *
21 * You should have received a copy of the GNU Library General Public
22 * License along with this library; if not, write to the
23 * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
24 * Boston, MA 02110-1301, USA.
25 */
26 #ifdef HAVE_CONFIG_H
27 #include "config.h"
28 #endif
29
30 #include "video-blend.h"
31 #include "video-orc.h"
32
33 #include <string.h>
34
35 #ifndef GST_DISABLE_GST_DEBUG
36
37 #define GST_CAT_DEFAULT ensure_debug_category()
38
39 static GstDebugCategory *
ensure_debug_category(void)40 ensure_debug_category (void)
41 {
42 static gsize cat_gonce = 0;
43
44 if (g_once_init_enter (&cat_gonce)) {
45 gsize cat_done;
46
47 cat_done = (gsize) _gst_debug_category_new ("video-blending", 0,
48 "video blending");
49
50 g_once_init_leave (&cat_gonce, cat_done);
51 }
52
53 return (GstDebugCategory *) cat_gonce;
54 }
55
56 #else
57
58 #define ensure_debug_category() /* NOOP */
59
60 #endif /* GST_DISABLE_GST_DEBUG */
61
62 static void
matrix_identity(guint8 * tmpline,guint width)63 matrix_identity (guint8 * tmpline, guint width)
64 {
65 }
66
67 static void
matrix_prea_rgb_to_yuv(guint8 * tmpline,guint width)68 matrix_prea_rgb_to_yuv (guint8 * tmpline, guint width)
69 {
70 int i;
71 int a, r, g, b;
72 int y, u, v;
73
74 for (i = 0; i < width; i++) {
75 a = tmpline[i * 4 + 0];
76 r = tmpline[i * 4 + 1];
77 g = tmpline[i * 4 + 2];
78 b = tmpline[i * 4 + 3];
79 if (a) {
80 r = (r * 255 + a / 2) / a;
81 g = (g * 255 + a / 2) / a;
82 b = (b * 255 + a / 2) / a;
83 }
84
85 y = (47 * r + 157 * g + 16 * b + 4096) >> 8;
86 u = (-26 * r - 87 * g + 112 * b + 32768) >> 8;
87 v = (112 * r - 102 * g - 10 * b + 32768) >> 8;
88
89 tmpline[i * 4 + 1] = CLAMP (y, 0, 255);
90 tmpline[i * 4 + 2] = CLAMP (u, 0, 255);
91 tmpline[i * 4 + 3] = CLAMP (v, 0, 255);
92 }
93 }
94
95 static void
matrix_rgb_to_yuv(guint8 * tmpline,guint width)96 matrix_rgb_to_yuv (guint8 * tmpline, guint width)
97 {
98 int i;
99 int r, g, b;
100 int y, u, v;
101
102 for (i = 0; i < width; i++) {
103 r = tmpline[i * 4 + 1];
104 g = tmpline[i * 4 + 2];
105 b = tmpline[i * 4 + 3];
106
107 y = (47 * r + 157 * g + 16 * b + 4096) >> 8;
108 u = (-26 * r - 87 * g + 112 * b + 32768) >> 8;
109 v = (112 * r - 102 * g - 10 * b + 32768) >> 8;
110
111 tmpline[i * 4 + 1] = CLAMP (y, 0, 255);
112 tmpline[i * 4 + 2] = CLAMP (u, 0, 255);
113 tmpline[i * 4 + 3] = CLAMP (v, 0, 255);
114 }
115 }
116
117 static void
matrix_yuv_to_rgb(guint8 * tmpline,guint width)118 matrix_yuv_to_rgb (guint8 * tmpline, guint width)
119 {
120 int i;
121 int r, g, b;
122 int y, u, v;
123
124 for (i = 0; i < width; i++) {
125 y = tmpline[i * 4 + 1];
126 u = tmpline[i * 4 + 2];
127 v = tmpline[i * 4 + 3];
128
129 r = (298 * y + 459 * v - 63514) >> 8;
130 g = (298 * y - 55 * u - 136 * v + 19681) >> 8;
131 b = (298 * y + 541 * u - 73988) >> 8;
132
133 tmpline[i * 4 + 1] = CLAMP (r, 0, 255);
134 tmpline[i * 4 + 2] = CLAMP (g, 0, 255);
135 tmpline[i * 4 + 3] = CLAMP (b, 0, 255);
136 }
137 }
138
139 /**
140 * gst_video_blend_scale_linear_RGBA:
141 * @src: the #GstVideoInfo describing the video data in @src_buffer
142 * @src_buffer: the source buffer containing video pixels to scale
143 * @dest_height: the height in pixels to scale the video data in @src_buffer to
144 * @dest_width: the width in pixels to scale the video data in @src_buffer to
145 * @dest: (out): pointer to a #GstVideoInfo structure that will be filled in
146 * with the details for @dest_buffer
147 * @dest_buffer: (out): a pointer to a #GstBuffer variable, which will be
148 * set to a newly-allocated buffer containing the scaled pixels.
149 *
150 * Scales a buffer containing RGBA (or AYUV) video. This is an internal
151 * helper function which is used to scale subtitle overlays, and may be
152 * deprecated in the near future. Use #GstVideoScaler to scale video buffers
153 * instead.
154 */
155 /* returns newly-allocated buffer, which caller must unref */
156 void
gst_video_blend_scale_linear_RGBA(GstVideoInfo * src,GstBuffer * src_buffer,gint dest_height,gint dest_width,GstVideoInfo * dest,GstBuffer ** dest_buffer)157 gst_video_blend_scale_linear_RGBA (GstVideoInfo * src, GstBuffer * src_buffer,
158 gint dest_height, gint dest_width, GstVideoInfo * dest,
159 GstBuffer ** dest_buffer)
160 {
161 const guint8 *src_pixels;
162 int acc;
163 int y_increment;
164 int x_increment;
165 int y1;
166 int i;
167 int j;
168 int x;
169 int dest_size;
170 guint dest_stride;
171 guint src_stride;
172 guint8 *dest_pixels;
173 guint8 *tmpbuf;
174 GstVideoFrame src_frame, dest_frame;
175
176 g_return_if_fail (dest_buffer != NULL);
177
178 gst_video_info_init (dest);
179 if (!gst_video_info_set_format (dest, GST_VIDEO_INFO_FORMAT (src),
180 dest_width, dest_height)) {
181 g_warn_if_reached ();
182 return;
183 }
184
185 tmpbuf = g_malloc (dest_width * 8 * 4);
186
187 *dest_buffer = gst_buffer_new_and_alloc (GST_VIDEO_INFO_SIZE (dest));
188
189 gst_video_frame_map (&src_frame, src, src_buffer, GST_MAP_READ);
190 gst_video_frame_map (&dest_frame, dest, *dest_buffer, GST_MAP_WRITE);
191
192 if (dest_height == 1 || src->height == 1)
193 y_increment = 0;
194 else
195 y_increment = ((src->height - 1) << 16) / (dest_height - 1) - 1;
196
197 if (dest_width == 1 || src->width == 1)
198 x_increment = 0;
199 else
200 x_increment = ((src->width - 1) << 16) / (dest_width - 1) - 1;
201
202 dest_size = dest_stride = dest_width * 4;
203 src_stride = GST_VIDEO_FRAME_PLANE_STRIDE (&src_frame, 0);
204
205 #define LINE(x) ((tmpbuf) + (dest_size)*((x)&1))
206
207 dest_pixels = GST_VIDEO_FRAME_PLANE_DATA (&dest_frame, 0);
208 src_pixels = GST_VIDEO_FRAME_PLANE_DATA (&src_frame, 0);
209
210 acc = 0;
211 video_orc_resample_bilinear_u32 (LINE (0), src_pixels, 0, x_increment,
212 dest_width);
213 y1 = 0;
214 for (i = 0; i < dest_height; i++) {
215 j = acc >> 16;
216 x = acc & 0xffff;
217
218 if (x == 0) {
219 memcpy (dest_pixels + i * dest_stride, LINE (j), dest_size);
220 } else {
221 if (j > y1) {
222 video_orc_resample_bilinear_u32 (LINE (j),
223 src_pixels + j * src_stride, 0, x_increment, dest_width);
224 y1++;
225 }
226 if (j >= y1) {
227 video_orc_resample_bilinear_u32 (LINE (j + 1),
228 src_pixels + (j + 1) * src_stride, 0, x_increment, dest_width);
229 y1++;
230 }
231 video_orc_merge_linear_u8 (dest_pixels + i * dest_stride,
232 LINE (j), LINE (j + 1), (x >> 8), dest_width * 4);
233 }
234
235 acc += y_increment;
236 }
237
238 gst_video_frame_unmap (&src_frame);
239 gst_video_frame_unmap (&dest_frame);
240
241 g_free (tmpbuf);
242 }
243
244 /*
245 * A OVER B alpha compositing operation, with:
246 * max: maximum value a color can have
247 * alphaG: global alpha to apply on the source color
248 * -> only needed for premultiplied source
249 * alphaA: source pixel alpha
250 * colorA: source pixel color
251 * alphaB: destination pixel alpha
252 * colorB: destination pixel color
253 * alphaD: blended pixel alpha
254 * -> only needed for premultiplied destination
255 */
256
257 /* Source non-premultiplied, Destination non-premultiplied */
258 #define OVER00_8BIT(max, alphaG, alphaA, colorA, alphaB, colorB, alphaD) \
259 ((colorA * alphaA + colorB * alphaB * (max - alphaA) / max) / alphaD)
260
261 #define OVER00_16BIT(max, alphaG, alphaA, colorA, alphaB, colorB, alphaD) \
262 ((colorA * alphaA + (guint64) colorB * alphaB * (max - alphaA) / max) / alphaD)
263
264 /* Source premultiplied, Destination non-premultiplied */
265 #define OVER10_8BIT(max, alphaG, alphaA, colorA, alphaB, colorB, alphaD) \
266 ((colorA * alphaG + colorB * alphaB * (max - alphaA) / max) / alphaD)
267
268 #define OVER10_16BIT(max, alphaG, alphaA, colorA, alphaB, colorB, alphaD) \
269 ((colorA * alphaG + (guint64) colorB * alphaB * (max - alphaA) / max) / alphaD)
270
271 /* Source non-premultiplied, Destination premultiplied */
272 #define OVER01(max, alphaG, alphaA, colorA, alphaB, colorB, alphaD) \
273 ((colorA * alphaA + colorB * (max - alphaA)) / max)
274
275 /* Source premultiplied, Destination premultiplied */
276 #define OVER11(max, alphaG, alphaA, colorA, alphaB, colorB, alphaD) \
277 ((colorA * alphaG + colorB * (max - alphaA)) / max)
278
279 #define BLENDC(op, max, global_alpha, aa, ca, ab, cb, dest_alpha) \
280 G_STMT_START { \
281 int c = op((max), (global_alpha), (aa), (ca), (ab), (cb), (dest_alpha)); \
282 cb = MIN(c, (max)); \
283 } G_STMT_END
284
285
286 /**
287 * gst_video_blend:
288 * @dest: The #GstVideoFrame where to blend @src in
289 * @src: the #GstVideoFrame that we want to blend into
290 * @x: The x offset in pixel where the @src image should be blended
291 * @y: the y offset in pixel where the @src image should be blended
292 * @global_alpha: the global_alpha each per-pixel alpha value is multiplied
293 * with
294 *
295 * Lets you blend the @src image into the @dest image
296 */
297 gboolean
gst_video_blend(GstVideoFrame * dest,GstVideoFrame * src,gint x,gint y,gfloat global_alpha)298 gst_video_blend (GstVideoFrame * dest,
299 GstVideoFrame * src, gint x, gint y, gfloat global_alpha)
300 {
301 gint i, j, global_alpha_val, src_width, src_height, dest_width, dest_height;
302 gint src_xoff = 0, src_yoff = 0;
303 guint8 *tmpdestline = NULL, *tmpsrcline = NULL;
304 gboolean src_premultiplied_alpha, dest_premultiplied_alpha;
305 gint bpp;
306 void (*matrix) (guint8 * tmpline, guint width);
307 const GstVideoFormatInfo *sinfo, *dinfo, *dunpackinfo, *sunpackinfo;
308
309 g_assert (dest != NULL);
310 g_assert (src != NULL);
311
312 dest_premultiplied_alpha =
313 GST_VIDEO_INFO_FLAGS (&dest->info) & GST_VIDEO_FLAG_PREMULTIPLIED_ALPHA;
314 src_premultiplied_alpha =
315 GST_VIDEO_INFO_FLAGS (&src->info) & GST_VIDEO_FLAG_PREMULTIPLIED_ALPHA;
316
317 src_width = GST_VIDEO_FRAME_WIDTH (src);
318 src_height = GST_VIDEO_FRAME_HEIGHT (src);
319
320 dest_width = GST_VIDEO_FRAME_WIDTH (dest);
321 dest_height = GST_VIDEO_FRAME_HEIGHT (dest);
322
323 ensure_debug_category ();
324
325 GST_LOG ("blend src %dx%d onto dest %dx%d @ %d,%d", src_width, src_height,
326 dest_width, dest_height, x, y);
327
328 /* In case overlay is completely outside the video, don't render */
329 if (x + src_width <= 0 || y + src_height <= 0
330 || x >= dest_width || y >= dest_height) {
331 goto nothing_to_do;
332 }
333
334 dinfo = gst_video_format_get_info (GST_VIDEO_FRAME_FORMAT (dest));
335 sinfo = gst_video_format_get_info (GST_VIDEO_FRAME_FORMAT (src));
336
337 if (!sinfo || !dinfo)
338 goto failed;
339
340 dunpackinfo = gst_video_format_get_info (dinfo->unpack_format);
341 sunpackinfo = gst_video_format_get_info (sinfo->unpack_format);
342
343 if (dunpackinfo == NULL || sunpackinfo == NULL)
344 goto failed;
345
346 g_assert (GST_VIDEO_FORMAT_INFO_BITS (sunpackinfo) == 8);
347
348 if (GST_VIDEO_FORMAT_INFO_BITS (dunpackinfo) != 8
349 && GST_VIDEO_FORMAT_INFO_BITS (dunpackinfo) != 16)
350 goto unpack_format_not_supported;
351
352 /* Source is always 8 bit but destination might be 8 or 16 bit */
353 bpp = 4 * (GST_VIDEO_FORMAT_INFO_BITS (dunpackinfo) / 8);
354
355 global_alpha_val = (bpp == 4) ? 255.0 * global_alpha : 65535.0 * global_alpha;
356
357 matrix = matrix_identity;
358 if (GST_VIDEO_INFO_IS_RGB (&src->info) != GST_VIDEO_INFO_IS_RGB (&dest->info)) {
359 if (GST_VIDEO_INFO_IS_RGB (&src->info)) {
360 if (src_premultiplied_alpha) {
361 matrix = matrix_prea_rgb_to_yuv;
362 src_premultiplied_alpha = FALSE;
363 } else {
364 matrix = matrix_rgb_to_yuv;
365 }
366 } else {
367 matrix = matrix_yuv_to_rgb;
368 }
369 }
370
371 /* If we're here we know that the overlay image fully or
372 * partially overlaps with the video frame */
373
374 /* adjust src image for negative offsets */
375 if (x < 0) {
376 src_xoff = -x;
377 src_width -= src_xoff;
378 x = 0;
379 }
380
381 if (y < 0) {
382 src_yoff = -y;
383 src_height -= src_yoff;
384 y = 0;
385 }
386
387 /* adjust width/height to render (i.e. clip source image) if the source
388 * image extends beyond the right or bottom border of the video surface */
389 if (x + src_width > dest_width)
390 src_width = dest_width - x;
391
392 if (y + src_height > dest_height)
393 src_height = dest_height - y;
394
395 tmpsrcline = g_malloc (sizeof (guint8) * (src_width + 8) * 4);
396 tmpdestline = g_malloc (sizeof (guint8) * (dest_width + 8) * bpp);
397
398 /* Mainloop doing the needed conversions, and blending */
399 for (i = y; i < y + src_height; i++, src_yoff++) {
400
401 dinfo->unpack_func (dinfo, 0, tmpdestline, dest->data, dest->info.stride,
402 0, i, dest_width);
403 sinfo->unpack_func (sinfo, 0, tmpsrcline, src->data, src->info.stride,
404 src_xoff, src_yoff, src_width);
405
406 /* FIXME: use the x parameter of the unpack func once implemented */
407 tmpdestline += bpp * x;
408
409 matrix (tmpsrcline, src_width);
410
411 #define BLENDLOOP(op, dest_type, max, shift, alpha_val) \
412 G_STMT_START { \
413 for (j = 0; j < src_width * 4; j += 4) { \
414 guint asrc, adst; \
415 guint final_alpha; \
416 dest_type * dest = (dest_type *) tmpdestline; \
417 \
418 asrc = ((guint) tmpsrcline[j]) * alpha_val / max; \
419 asrc = asrc << shift; \
420 if (asrc == 0) \
421 continue; \
422 \
423 adst = dest[j]; \
424 final_alpha = asrc + adst * (max - asrc) / max; \
425 dest[j] = final_alpha; \
426 if (final_alpha == 0) \
427 final_alpha = 1; \
428 \
429 BLENDC (op, max, alpha_val, asrc, tmpsrcline[j + 1] << shift, adst, dest[j + 1], final_alpha); \
430 BLENDC (op, max, alpha_val, asrc, tmpsrcline[j + 2] << shift, adst, dest[j + 2], final_alpha); \
431 BLENDC (op, max, alpha_val, asrc, tmpsrcline[j + 3] << shift, adst, dest[j + 3], final_alpha); \
432 } \
433 } G_STMT_END
434
435 if (bpp == 4) {
436 if (G_LIKELY (global_alpha_val == 255)) {
437 if (src_premultiplied_alpha && dest_premultiplied_alpha) {
438 BLENDLOOP (OVER11, guint8, 255, 0, 255);
439 } else if (!src_premultiplied_alpha && dest_premultiplied_alpha) {
440 BLENDLOOP (OVER01, guint8, 255, 0, 255);
441 } else if (src_premultiplied_alpha && !dest_premultiplied_alpha) {
442 BLENDLOOP (OVER10_8BIT, guint8, 255, 0, 255);
443 } else {
444 BLENDLOOP (OVER00_8BIT, guint8, 255, 0, 255);
445 }
446 } else {
447 if (src_premultiplied_alpha && dest_premultiplied_alpha) {
448 BLENDLOOP (OVER11, guint8, 255, 0, global_alpha_val);
449 } else if (!src_premultiplied_alpha && dest_premultiplied_alpha) {
450 BLENDLOOP (OVER01, guint8, 255, 0, global_alpha_val);
451 } else if (src_premultiplied_alpha && !dest_premultiplied_alpha) {
452 BLENDLOOP (OVER10_8BIT, guint8, 255, 0, global_alpha_val);
453 } else {
454 BLENDLOOP (OVER00_8BIT, guint8, 255, 0, global_alpha_val);
455 }
456 }
457 } else {
458 g_assert (bpp == 8);
459
460 if (G_LIKELY (global_alpha_val == 65535)) {
461 if (src_premultiplied_alpha && dest_premultiplied_alpha) {
462 BLENDLOOP (OVER11, guint16, 65535, 8, 65535);
463 } else if (!src_premultiplied_alpha && dest_premultiplied_alpha) {
464 BLENDLOOP (OVER01, guint16, 65535, 8, 65535);
465 } else if (src_premultiplied_alpha && !dest_premultiplied_alpha) {
466 BLENDLOOP (OVER10_16BIT, guint16, 65535, 8, 65535);
467 } else {
468 BLENDLOOP (OVER00_16BIT, guint16, 65535, 8, 65535);
469 }
470 } else {
471 if (src_premultiplied_alpha && dest_premultiplied_alpha) {
472 BLENDLOOP (OVER11, guint16, 65535, 8, global_alpha_val);
473 } else if (!src_premultiplied_alpha && dest_premultiplied_alpha) {
474 BLENDLOOP (OVER01, guint16, 65535, 8, global_alpha_val);
475 } else if (src_premultiplied_alpha && !dest_premultiplied_alpha) {
476 BLENDLOOP (OVER10_16BIT, guint16, 65535, 8, global_alpha_val);
477 } else {
478 BLENDLOOP (OVER00_16BIT, guint16, 65535, 8, global_alpha_val);
479 }
480 }
481 }
482
483 #undef BLENDLOOP
484
485 /* undo previous pointer adjustments to pass right pointer to g_free */
486 tmpdestline -= bpp * x;
487
488 /* FIXME
489 * #if G_BYTE_ORDER == LITTLE_ENDIAN
490 * video_orc_blend_little (tmpdestline, tmpsrcline, dest->width);
491 * #else
492 * video_orc_blend_big (tmpdestline, tmpsrcline, src->width);
493 * #endif
494 */
495
496 dinfo->pack_func (dinfo, 0, tmpdestline, dest_width,
497 dest->data, dest->info.stride, dest->info.chroma_site, i, dest_width);
498 }
499
500 g_free (tmpdestline);
501 g_free (tmpsrcline);
502
503 return TRUE;
504
505 failed:
506 {
507 GST_WARNING ("Could not do the blending");
508 return FALSE;
509 }
510 unpack_format_not_supported:
511 {
512 GST_FIXME ("video format %s not supported yet for blending",
513 gst_video_format_to_string (dinfo->unpack_format));
514 return FALSE;
515 }
516 nothing_to_do:
517 {
518 GST_LOG
519 ("Overlay completely outside the video surface, hence not rendering");
520 return TRUE;
521 }
522 }
523