1 /*
2 * GStreamer
3 * Copyright (C) 2009 Julien Isorce <julien.isorce@mail.com>
4 * Copyright (C) 2014 Jan Schmidt <jan@centricular.com>
5 * Copyright (C) 2015 Matthew Waters <matthew@centricular.com>
6 *
7 * This library is free software; you can redistribute it and/or
8 * modify it under the terms of the GNU Library General Public
9 * License as published by the Free Software Foundation; either
10 * version 2 of the License, or (at your option) any later version.
11 *
12 * This library is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15 * Library General Public License for more details.
16 *
17 * You should have received a copy of the GNU Library General Public
18 * License along with this library; if not, write to the
19 * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
20 * Boston, MA 02110-1301, USA.
21 */
22
23 /**
24 * SECTION:gstglviewconvert
25 * @title: GstGLViewConvert
26 * @short_description: convert between steroscopic/multiview video formats
27 * @see_also: #GstGLColorConvert, #GstGLContext
28 *
29 * Convert stereoscopic/multiview video using fragment shaders.
30 */
31
32 #ifdef HAVE_CONFIG_H
33 #include "config.h"
34 #endif
35
36 #include "gstglviewconvert.h"
37
38 #include <gst/video/gstvideoaffinetransformationmeta.h>
39
40 #include "gl.h"
41 #include "gstglsl_private.h"
42 #include "gstglutils_private.h"
43
44 #define USING_OPENGL(context) (gst_gl_context_check_gl_version (context, GST_GL_API_OPENGL, 1, 0))
45 #define USING_OPENGL3(context) (gst_gl_context_check_gl_version (context, GST_GL_API_OPENGL3, 3, 1))
46 #define USING_GLES(context) (gst_gl_context_check_gl_version (context, GST_GL_API_GLES, 1, 0))
47 #define USING_GLES2(context) (gst_gl_context_check_gl_version (context, GST_GL_API_GLES2, 2, 0))
48 #define USING_GLES3(context) (gst_gl_context_check_gl_version (context, GST_GL_API_GLES2, 3, 0))
49
50 static GstStaticCaps caps_template =
51 GST_STATIC_CAPS ("video/x-raw(" GST_CAPS_FEATURE_MEMORY_GL_MEMORY "), "
52 "format = (string) RGBA, "
53 "width = " GST_VIDEO_SIZE_RANGE ", "
54 "height = " GST_VIDEO_SIZE_RANGE ", "
55 "framerate = " GST_VIDEO_FPS_RANGE ", "
56 "texture-target = (string) { 2D, rectangle, external-oes } ");
57
58 #define GST_CAT_DEFAULT gst_gl_view_convert_debug
59 GST_DEBUG_CATEGORY_STATIC (GST_CAT_DEFAULT);
60
61 enum
62 {
63 PROP_0,
64 PROP_INPUT_LAYOUT,
65 PROP_INPUT_FLAGS,
66 PROP_OUTPUT_LAYOUT,
67 PROP_OUTPUT_FLAGS,
68 PROP_OUTPUT_DOWNMIX_MODE
69 };
70
71 #define DEFAULT_DOWNMIX GST_GL_STEREO_DOWNMIX_ANAGLYPH_GREEN_MAGENTA_DUBOIS
72
73 struct _GstGLViewConvertPrivate
74 {
75 gboolean result;
76
77 GstVideoMultiviewMode input_mode;
78 GstVideoMultiviewFlags input_flags;
79 GstVideoMultiviewMode output_mode;
80 GstVideoMultiviewFlags output_flags;
81
82 GstBuffer *primary_in;
83 GstBuffer *auxilliary_in;
84
85 GstBuffer *primary_out;
86 GstBuffer *auxilliary_out;
87
88 GstGLMemory *in_tex[GST_VIDEO_MAX_PLANES];
89 GstGLMemory *out_tex[GST_VIDEO_MAX_PLANES];
90 guint n_out_tex;
91
92 GLuint vao;
93 GLuint vertex_buffer;
94 GLuint vbo_indices;
95 GLuint attr_position;
96 GLuint attr_texture;
97 };
98
99 #define DEBUG_INIT \
100 GST_DEBUG_CATEGORY_INIT (gst_gl_view_convert_debug, "glviewconvert", 0, "glviewconvert object");
101
102 G_DEFINE_TYPE_WITH_CODE (GstGLViewConvert, gst_gl_view_convert,
103 GST_TYPE_OBJECT, G_ADD_PRIVATE (GstGLViewConvert) DEBUG_INIT);
104
105 static void gst_gl_view_convert_set_property (GObject * object,
106 guint prop_id, const GValue * value, GParamSpec * pspec);
107 static void gst_gl_view_convert_get_property (GObject * object,
108 guint prop_id, GValue * value, GParamSpec * pspec);
109 static void gst_gl_view_convert_finalize (GObject * object);
110
111 static void _do_view_convert (GstGLContext * context,
112 GstGLViewConvert * viewconvert);
113
114 /* *INDENT-OFF* */
115 /* These match the order and number of DOWNMIX_ANAGLYPH_* modes */
116 static GLfloat downmix_matrices[][2][9] = {
117 { /* Green-Magenta Dubois */
118 {-0.062f, 0.284f, -0.015f, -0.158f, 0.668f, -0.027f, -0.039f, 0.143f, 0.021f},
119 {0.529f, -0.016f, 0.009f, 0.705f, -0.015f, 0.075f, 0.024f, -0.065f, 0.937f}
120 },
121 { /* Red-Cyan Dubois */
122 /* Source of this matrix: http://www.site.uottawa.ca/~edubois/anaglyph/LeastSquaresHowToPhotoshop.pdf */
123 {0.437f, -0.062f, -0.048f, 0.449f, -0.062f, -0.050f, 0.164f, -0.024f, -0.017f},
124 {-0.011f, 0.377f, -0.026f, -0.032f, 0.761f, -0.093f, -0.007f, 0.009f, 1.234f}
125 },
126 { /* Amber-blue Dubois */
127 {1.062f, -0.026f, -0.038f, -0.205f, 0.908f, -0.173f, 0.299f, 0.068f, 0.022f},
128 {-0.016f, 0.006f, 0.094f, -0.123f, 0.062f, 0.185f, -0.017f, -0.017f, 0.911f}
129 }
130 };
131
132 static gfloat identity_matrix[] = {
133 1.0f, 0.0f, 0.0f, 0.0f,
134 0.0f, 1.0f, 0.0f, 0.0f,
135 0.0f, 0.0f, 1.0f, 0.0f,
136 0.0f, 0.0f, 0.0f, 1.0f,
137 };
138 /* *INDENT-ON* */
139
140 #define glsl_OES_extension_string "#extension GL_OES_EGL_image_external : require \n"
141
142 /* *INDENT-OFF* */
143 static const gchar *fragment_header =
144 "uniform sampler2D tex_l;\n"
145 "uniform sampler2D tex_r;\n"
146 "uniform float width;\n"
147 "uniform float height;\n"
148 "uniform mat3 downmix[2];\n"
149 "uniform vec2 tex_scale[2];\n"
150 "uniform vec2 offsets[2];\n";
151
152 static const gchar *frag_input =
153 " vec2 l_tex = v_texcoord * tex_scale[0] + offsets[0];\n"
154 " vec2 r_tex = v_texcoord * tex_scale[1] + offsets[1];\n"
155 " l = texture2D(tex_l, l_tex).rgba;\n"
156 " r = texture2D(tex_r, r_tex).rgba;\n";
157
158 static const gchar *frag_output_downmix =
159 " vec3 lcol = l.rgb * l.a + vec3(1.0-l.a);\n"
160 " vec3 rcol = r.rgb * r.a + vec3(1.0-r.a);\n"
161 " if (l.a + r.a > 0.0) {\n"
162 " lcol = clamp (downmix[0] * lcol, 0.0, 1.0);\n"
163 " rcol = clamp (downmix[1] * rcol, 0.0, 1.0);\n"
164 " gl_FragColor = vec4 (lcol + rcol, 1.0);\n"
165 " } else {\n"
166 " gl_FragColor = vec4 (0.0);\n"
167 " }\n";
168
169 static const gchar *frag_output_left =
170 " gl_FragColor = l;\n";
171
172 static const gchar *frag_output_right =
173 " gl_FragColor = r;\n";
174
175 static const gchar *frag_output_side_by_side =
176 " if (v_texcoord.x < 0.5) {\n"
177 " gl_FragColor = l;\n"
178 " } else {\n"
179 " gl_FragColor = r;\n"
180 " };\n";
181
182 static const gchar *frag_output_top_bottom =
183 "if (v_texcoord.y < 0.5) {\n"
184 " gl_FragColor = l;\n"
185 "} else {\n"
186 " gl_FragColor = r;\n"
187 "};\n";
188
189 static const gchar *frag_output_column_interleaved =
190 "if (int(mod(l_tex.x * width, 2.0)) == 0) {\n"
191 " gl_FragColor = l;\n"
192 "} else {\n"
193 " gl_FragColor = r;\n"
194 "};\n";
195
196 static const gchar *frag_output_row_interleaved =
197 "if (int(mod(l_tex.y * height, 2.0)) == 0) {\n"
198 " gl_FragColor = l;\n"
199 "} else {\n"
200 " gl_FragColor = r;\n"
201 "};\n";
202
203 static const gchar *frag_output_checkerboard =
204 "if (int(mod(l_tex.x * width, 2.0)) == \n"
205 " int(mod(l_tex.y * height, 2.0))) {\n"
206 " gl_FragColor = l;\n"
207 "} else {\n"
208 " gl_FragColor = r;\n"
209 "};\n";
210
211 static const gchar *frag_output_separated =
212 "gl_FragData[0] = l;\n"
213 "gl_FragData[1] = r;\n";
214 /* *INDENT-ON* */
215
216 static const GLfloat vertices[] = {
217 1.0f, -1.0f, 0.0f, 1.0f, 0.0f,
218 -1.0f, -1.0f, 0.0f, 0.0f, 0.0f,
219 -1.0f, 1.0f, 0.0f, 0.0f, 1.0f,
220 1.0f, 1.0f, 0.0f, 1.0f, 1.0f
221 };
222
223 static const GLushort indices[] = { 0, 1, 2, 0, 2, 3 };
224
225 static void
gst_gl_view_convert_class_init(GstGLViewConvertClass * klass)226 gst_gl_view_convert_class_init (GstGLViewConvertClass * klass)
227 {
228 GObjectClass *gobject_class = (GObjectClass *) klass;
229
230 gobject_class->set_property = gst_gl_view_convert_set_property;
231 gobject_class->get_property = gst_gl_view_convert_get_property;
232 gobject_class->finalize = gst_gl_view_convert_finalize;
233
234 g_object_class_install_property (gobject_class, PROP_INPUT_LAYOUT,
235 g_param_spec_enum ("input-mode-override",
236 "Input Multiview Mode Override",
237 "Override any input information about multiview layout",
238 GST_TYPE_VIDEO_MULTIVIEW_MODE,
239 GST_VIDEO_MULTIVIEW_MODE_NONE,
240 G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
241 g_object_class_install_property (gobject_class, PROP_INPUT_FLAGS,
242 g_param_spec_flags ("input-flags-override",
243 "Input Multiview Flags Override",
244 "Override any input information about multiview layout flags",
245 GST_TYPE_VIDEO_MULTIVIEW_FLAGS, GST_VIDEO_MULTIVIEW_FLAGS_NONE,
246 G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
247 g_object_class_install_property (gobject_class, PROP_OUTPUT_LAYOUT,
248 g_param_spec_enum ("output-mode-override",
249 "Output Multiview Mode Override",
250 "Override automatic output mode selection for multiview layout",
251 GST_TYPE_VIDEO_MULTIVIEW_MODE, GST_VIDEO_MULTIVIEW_MODE_NONE,
252 G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
253 g_object_class_install_property (gobject_class, PROP_OUTPUT_FLAGS,
254 g_param_spec_flags ("output-flags-override",
255 "Output Multiview Flags Override",
256 "Override automatic negotiation for output multiview layout flags",
257 GST_TYPE_VIDEO_MULTIVIEW_FLAGS, GST_VIDEO_MULTIVIEW_FLAGS_NONE,
258 G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
259 g_object_class_install_property (gobject_class, PROP_OUTPUT_DOWNMIX_MODE,
260 g_param_spec_enum ("downmix-mode", "Mode for mono downmixed output",
261 "Output anaglyph type to generate when downmixing to mono",
262 GST_TYPE_GL_STEREO_DOWNMIX, DEFAULT_DOWNMIX,
263 G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
264 }
265
266 static void
gst_gl_view_convert_init(GstGLViewConvert * convert)267 gst_gl_view_convert_init (GstGLViewConvert * convert)
268 {
269 convert->priv = gst_gl_view_convert_get_instance_private (convert);
270
271 convert->shader = NULL;
272 convert->downmix_mode = DEFAULT_DOWNMIX;
273 convert->priv->input_mode = GST_VIDEO_MULTIVIEW_MODE_NONE;
274 convert->priv->input_flags = GST_VIDEO_MULTIVIEW_FLAGS_NONE;
275 convert->priv->output_mode = GST_VIDEO_MULTIVIEW_MODE_NONE;
276 convert->priv->output_flags = GST_VIDEO_MULTIVIEW_FLAGS_NONE;
277
278 convert->input_mode_override = GST_VIDEO_MULTIVIEW_MODE_NONE;
279 convert->input_flags_override = GST_VIDEO_MULTIVIEW_FLAGS_NONE;
280 convert->output_mode_override = GST_VIDEO_MULTIVIEW_MODE_NONE;
281 convert->output_flags_override = GST_VIDEO_MULTIVIEW_FLAGS_NONE;
282
283 gst_video_info_init (&convert->in_info);
284 gst_video_info_init (&convert->out_info);
285 }
286
287 static void
gst_gl_view_convert_finalize(GObject * object)288 gst_gl_view_convert_finalize (GObject * object)
289 {
290 GstGLViewConvert *viewconvert;
291
292 viewconvert = GST_GL_VIEW_CONVERT (object);
293
294 gst_gl_view_convert_reset (viewconvert);
295
296 gst_buffer_replace (&viewconvert->priv->primary_in, NULL);
297 gst_buffer_replace (&viewconvert->priv->auxilliary_in, NULL);
298 gst_buffer_replace (&viewconvert->priv->primary_out, NULL);
299 gst_buffer_replace (&viewconvert->priv->auxilliary_out, NULL);
300
301 if (viewconvert->context) {
302 gst_object_unref (viewconvert->context);
303 viewconvert->context = NULL;
304 }
305
306 G_OBJECT_CLASS (gst_gl_view_convert_parent_class)->finalize (object);
307 }
308
309 /**
310 * gst_gl_view_convert_new:
311 *
312 * Returns: (transfer full): a new #GstGLViewConvert
313 *
314 * Since: 1.6
315 */
316 GstGLViewConvert *
gst_gl_view_convert_new(void)317 gst_gl_view_convert_new (void)
318 {
319 GstGLViewConvert *convert;
320
321 convert = g_object_new (GST_TYPE_GL_VIEW_CONVERT, NULL);
322 gst_object_ref_sink (convert);
323
324 return convert;
325 }
326
327 static void
_reset_gl(GstGLContext * context,GstGLViewConvert * viewconvert)328 _reset_gl (GstGLContext * context, GstGLViewConvert * viewconvert)
329 {
330 const GstGLFuncs *gl = context->gl_vtable;
331
332 if (viewconvert->priv->vao) {
333 gl->DeleteVertexArrays (1, &viewconvert->priv->vao);
334 viewconvert->priv->vao = 0;
335 }
336
337 if (viewconvert->priv->vertex_buffer) {
338 gl->DeleteBuffers (1, &viewconvert->priv->vertex_buffer);
339 viewconvert->priv->vertex_buffer = 0;
340 }
341
342 if (viewconvert->priv->vbo_indices) {
343 gl->DeleteBuffers (1, &viewconvert->priv->vbo_indices);
344 viewconvert->priv->vbo_indices = 0;
345 }
346 }
347
348 /**
349 * gst_gl_view_convert_set_context:
350 * @viewconvert: a #GstGLViewConvert
351 * @context: the #GstGLContext to set
352 *
353 * Set @context on @viewconvert
354 *
355 * Since: 1.6
356 */
357 void
gst_gl_view_convert_set_context(GstGLViewConvert * viewconvert,GstGLContext * context)358 gst_gl_view_convert_set_context (GstGLViewConvert * viewconvert,
359 GstGLContext * context)
360 {
361 GstGLContext *old_context = NULL;
362
363 g_return_if_fail (GST_IS_GL_VIEW_CONVERT (viewconvert));
364
365 GST_OBJECT_LOCK (viewconvert);
366 if (context != viewconvert->context) {
367 gst_gl_view_convert_reset (viewconvert);
368 if (viewconvert->context)
369 old_context = viewconvert->context;
370 viewconvert->context = context ? gst_object_ref (context) : NULL;
371 }
372 GST_OBJECT_UNLOCK (viewconvert);
373
374 gst_clear_object (&old_context);
375 }
376
377 static gboolean
_view_convert_set_format(GstGLViewConvert * viewconvert,const GstVideoInfo * in_info,GstGLTextureTarget from_target,const GstVideoInfo * out_info,GstGLTextureTarget to_target)378 _view_convert_set_format (GstGLViewConvert * viewconvert,
379 const GstVideoInfo * in_info, GstGLTextureTarget from_target,
380 const GstVideoInfo * out_info, GstGLTextureTarget to_target)
381 {
382 gboolean passthrough;
383 g_return_val_if_fail (GST_IS_GL_VIEW_CONVERT (viewconvert), FALSE);
384
385 if (gst_video_info_is_equal (in_info, &viewconvert->in_info) &&
386 gst_video_info_is_equal (out_info, &viewconvert->out_info) &&
387 viewconvert->from_texture_target == from_target &&
388 viewconvert->to_texture_target == to_target)
389 return TRUE;
390
391 if (GST_VIDEO_INFO_FORMAT (in_info) != GST_VIDEO_FORMAT_RGBA ||
392 GST_VIDEO_INFO_FORMAT (out_info) != GST_VIDEO_FORMAT_RGBA) {
393 GST_ERROR_OBJECT (viewconvert,
394 "Multiview conversion can currently only be performed on RGBA textures");
395 return FALSE;
396 }
397
398 passthrough = gst_video_info_is_equal (in_info, out_info) &&
399 from_target == to_target;
400
401 if (!passthrough && to_target != GST_GL_TEXTURE_TARGET_2D
402 && to_target != GST_GL_TEXTURE_TARGET_RECTANGLE)
403 return FALSE;
404
405 /* FIXME: Compare what changed and decide if we need a full reset or not */
406 GST_OBJECT_LOCK (viewconvert);
407 gst_gl_view_convert_reset (viewconvert);
408
409 viewconvert->in_info = *in_info;
410 viewconvert->out_info = *out_info;
411 viewconvert->from_texture_target = from_target;
412 viewconvert->to_texture_target = to_target;
413 viewconvert->caps_passthrough = passthrough;
414
415 gst_buffer_replace (&viewconvert->priv->primary_in, NULL);
416 gst_buffer_replace (&viewconvert->priv->auxilliary_in, NULL);
417 gst_buffer_replace (&viewconvert->priv->primary_out, NULL);
418 gst_buffer_replace (&viewconvert->priv->auxilliary_out, NULL);
419 GST_OBJECT_UNLOCK (viewconvert);
420
421 return TRUE;
422 }
423
424 /**
425 * gst_gl_view_convert_set_caps:
426 * @viewconvert: a #GstGLViewConvert
427 * @in_caps: input #GstCaps
428 * @out_caps: output #GstCaps
429 *
430 * Initializes @viewconvert with the information required for conversion.
431 *
432 * Since: 1.6
433 */
434 gboolean
gst_gl_view_convert_set_caps(GstGLViewConvert * viewconvert,GstCaps * in_caps,GstCaps * out_caps)435 gst_gl_view_convert_set_caps (GstGLViewConvert * viewconvert,
436 GstCaps * in_caps, GstCaps * out_caps)
437 {
438 GstVideoInfo in_info, out_info;
439 GstCapsFeatures *in_features, *out_features;
440 GstGLTextureTarget from_target = GST_GL_TEXTURE_TARGET_2D;
441 GstGLTextureTarget to_target = GST_GL_TEXTURE_TARGET_2D;
442
443 g_return_val_if_fail (GST_IS_GL_VIEW_CONVERT (viewconvert), FALSE);
444 g_return_val_if_fail (GST_IS_CAPS (in_caps), FALSE);
445 g_return_val_if_fail (GST_IS_CAPS (out_caps), FALSE);
446
447 GST_INFO_OBJECT (viewconvert,
448 "Configuring multiview conversion from caps %" GST_PTR_FORMAT
449 " to %" GST_PTR_FORMAT, in_caps, out_caps);
450
451 in_features = gst_caps_get_features (in_caps, 0);
452 out_features = gst_caps_get_features (out_caps, 0);
453
454 if (!gst_caps_features_contains (in_features,
455 GST_CAPS_FEATURE_MEMORY_GL_MEMORY))
456 return FALSE;
457 if (!gst_caps_features_contains (out_features,
458 GST_CAPS_FEATURE_MEMORY_GL_MEMORY))
459 return FALSE;
460
461 if (!gst_video_info_from_caps (&in_info, in_caps))
462 return FALSE;
463 if (!gst_video_info_from_caps (&out_info, out_caps))
464 return FALSE;
465
466 {
467 GstStructure *in_s = gst_caps_get_structure (in_caps, 0);
468 GstStructure *out_s = gst_caps_get_structure (out_caps, 0);
469
470 if (gst_structure_has_field_typed (in_s, "texture-target", G_TYPE_STRING)) {
471 from_target =
472 gst_gl_texture_target_from_string (gst_structure_get_string (in_s,
473 "texture-target"));
474 }
475
476 if (gst_structure_has_field_typed (out_s, "texture-target", G_TYPE_STRING)) {
477 to_target =
478 gst_gl_texture_target_from_string (gst_structure_get_string (out_s,
479 "texture-target"));
480 }
481
482 if (to_target == GST_GL_TEXTURE_TARGET_NONE
483 || from_target == GST_GL_TEXTURE_TARGET_NONE)
484 /* invalid caps */
485 return FALSE;
486 }
487
488 return _view_convert_set_format (viewconvert, &in_info, from_target,
489 &out_info, to_target);
490 }
491
492 /* Function that can halve the value
493 * of ints, fractions, int/fraction ranges and lists of ints/fractions */
494 static gboolean
_halve_value(GValue * out,const GValue * in_value)495 _halve_value (GValue * out, const GValue * in_value)
496 {
497 /* Fundamental fixed types first */
498 if (G_VALUE_HOLDS_INT (in_value)) {
499 g_value_init (out, G_TYPE_INT);
500 g_value_set_int (out, MAX (g_value_get_int (in_value) / 2, 1));
501 } else if (GST_VALUE_HOLDS_FRACTION (in_value)) {
502 gint num, den;
503 num = gst_value_get_fraction_numerator (in_value);
504 den = gst_value_get_fraction_denominator (in_value);
505 g_value_init (out, GST_TYPE_FRACTION);
506 /* Don't adjust 'infinite' fractions */
507 if ((num != 1 || den != 2147483647) && (num != 2147483647 || den != 1)) {
508 /* FIXME - could do better approximation when den > G_MAXINT/2? */
509 den = den > G_MAXINT / 2 ? G_MAXINT : den * 2;
510 }
511 gst_value_set_fraction (out, num, den);
512 } else if (GST_VALUE_HOLDS_INT_RANGE (in_value)) {
513 gint range_min = gst_value_get_int_range_min (in_value);
514 gint range_max = gst_value_get_int_range_max (in_value);
515 gint range_step = gst_value_get_int_range_step (in_value);
516 g_value_init (out, GST_TYPE_INT_RANGE);
517 if (range_min != 1)
518 range_min = MAX (1, range_min / 2);
519 if (range_max != G_MAXINT)
520 range_max = MAX (1, range_max / 2);
521 gst_value_set_int_range_step (out, range_min,
522 range_max, MAX (1, range_step / 2));
523 } else if (GST_VALUE_HOLDS_FRACTION_RANGE (in_value)) {
524 GValue min_out = G_VALUE_INIT;
525 GValue max_out = G_VALUE_INIT;
526 const GValue *range_min = gst_value_get_fraction_range_min (in_value);
527 const GValue *range_max = gst_value_get_fraction_range_max (in_value);
528 _halve_value (&min_out, range_min);
529 _halve_value (&max_out, range_max);
530 g_value_init (out, GST_TYPE_FRACTION_RANGE);
531 gst_value_set_fraction_range (out, &min_out, &max_out);
532 g_value_unset (&min_out);
533 g_value_unset (&max_out);
534 } else if (GST_VALUE_HOLDS_LIST (in_value)) {
535 gint i;
536 g_value_init (out, GST_TYPE_LIST);
537 for (i = 0; i < gst_value_list_get_size (in_value); i++) {
538 const GValue *entry;
539 GValue tmp = G_VALUE_INIT;
540
541 entry = gst_value_list_get_value (in_value, i);
542 /* Random list values might not be the right type */
543 if (!_halve_value (&tmp, entry))
544 goto fail;
545 gst_value_list_append_and_take_value (out, &tmp);
546 }
547 } else {
548 return FALSE;
549 }
550
551 return TRUE;
552 fail:
553 g_value_unset (out);
554 return FALSE;
555 }
556
557 static GstStructure *
_halve_structure_field(const GstStructure * in,const gchar * field_name)558 _halve_structure_field (const GstStructure * in, const gchar * field_name)
559 {
560 GstStructure *out;
561 const GValue *in_value = gst_structure_get_value (in, field_name);
562 GValue tmp = G_VALUE_INIT;
563
564 if (G_UNLIKELY (in_value == NULL))
565 return gst_structure_copy (in); /* Field doesn't exist, leave it as is */
566
567 if (!_halve_value (&tmp, in_value))
568 return NULL;
569
570 out = gst_structure_copy (in);
571 gst_structure_set_value (out, field_name, &tmp);
572 g_value_unset (&tmp);
573
574 return out;
575 }
576
577 /* Function that can double the value
578 * of ints, fractions, int/fraction ranges and lists of ints/fractions */
579 static gboolean
_double_value(GValue * out,const GValue * in_value)580 _double_value (GValue * out, const GValue * in_value)
581 {
582 /* Fundamental fixed types first */
583 if (G_VALUE_HOLDS_INT (in_value)) {
584 gint n = g_value_get_int (in_value);
585 g_value_init (out, G_TYPE_INT);
586 if (n <= G_MAXINT / 2)
587 g_value_set_int (out, n * 2);
588 else
589 g_value_set_int (out, G_MAXINT);
590 } else if (GST_VALUE_HOLDS_FRACTION (in_value)) {
591 gint num, den;
592 num = gst_value_get_fraction_numerator (in_value);
593 den = gst_value_get_fraction_denominator (in_value);
594 g_value_init (out, GST_TYPE_FRACTION);
595 /* Don't adjust 'infinite' fractions */
596 if ((num != 1 || den != 2147483647) && (num != 2147483647 || den != 1)) {
597 /* FIXME - could do better approximation when num > G_MAXINT/2? */
598 num = num > G_MAXINT / 2 ? G_MAXINT : num * 2;
599 }
600 gst_value_set_fraction (out, num, den);
601 } else if (GST_VALUE_HOLDS_INT_RANGE (in_value)) {
602 gint range_min = gst_value_get_int_range_min (in_value);
603 gint range_max = gst_value_get_int_range_max (in_value);
604 gint range_step = gst_value_get_int_range_step (in_value);
605 if (range_min != 1) {
606 range_min = MIN (G_MAXINT / 2, range_min);
607 range_min *= 2;
608 }
609 if (range_max != G_MAXINT) {
610 range_max = MIN (G_MAXINT / 2, range_max);
611 range_max *= 2;
612 }
613 range_step = MIN (G_MAXINT / 2, range_step);
614 g_value_init (out, GST_TYPE_INT_RANGE);
615 gst_value_set_int_range_step (out, range_min, range_max, range_step);
616 } else if (GST_VALUE_HOLDS_FRACTION_RANGE (in_value)) {
617 GValue min_out = G_VALUE_INIT;
618 GValue max_out = G_VALUE_INIT;
619 const GValue *range_min = gst_value_get_fraction_range_min (in_value);
620 const GValue *range_max = gst_value_get_fraction_range_max (in_value);
621 _double_value (&min_out, range_min);
622 _double_value (&max_out, range_max);
623 g_value_init (out, GST_TYPE_FRACTION_RANGE);
624 gst_value_set_fraction_range (out, &min_out, &max_out);
625 g_value_unset (&min_out);
626 g_value_unset (&max_out);
627 } else if (GST_VALUE_HOLDS_LIST (in_value)) {
628 gint i;
629 g_value_init (out, GST_TYPE_LIST);
630 for (i = 0; i < gst_value_list_get_size (in_value); i++) {
631 const GValue *entry;
632 GValue tmp = G_VALUE_INIT;
633
634 entry = gst_value_list_get_value (in_value, i);
635 /* Random list values might not be the right type */
636 if (!_double_value (&tmp, entry))
637 goto fail;
638 gst_value_list_append_and_take_value (out, &tmp);
639 }
640 } else {
641 return FALSE;
642 }
643
644 return TRUE;
645 fail:
646 g_value_unset (out);
647 return FALSE;
648 }
649
650 static GstStructure *
_double_structure_field(const GstStructure * in,const gchar * field_name)651 _double_structure_field (const GstStructure * in, const gchar * field_name)
652 {
653 GstStructure *out;
654 const GValue *in_value = gst_structure_get_value (in, field_name);
655 GValue tmp = G_VALUE_INIT;
656
657 if (G_UNLIKELY (in_value == NULL))
658 return gst_structure_copy (in); /* Field doesn't exist, leave it as is */
659
660 if (!_double_value (&tmp, in_value))
661 return NULL;
662
663 out = gst_structure_copy (in);
664 gst_structure_set_value (out, field_name, &tmp);
665 g_value_unset (&tmp);
666
667 return out;
668 }
669
670 /* Return a copy of the caps with the requested field halved in value/range */
671 #if 0
672 static GstCaps *
673 _halve_caps_field (const GstCaps * in, const gchar * field_name)
674 {
675 gint i;
676 GstCaps *out = gst_caps_new_empty ();
677
678 for (i = 0; i < gst_caps_get_size (in); i++) {
679 const GstStructure *cur = gst_caps_get_structure (in, i);
680 GstCapsFeatures *f = gst_caps_get_features (in, i);
681
682 GstStructure *res = _halve_structure_field (cur, field_name);
683 out =
684 gst_caps_merge_structure_full (out, res,
685 f ? gst_caps_features_copy (f) : NULL);
686 }
687
688 return out;
689 }
690 #endif
691
692 /* Return a copy of the caps with the requested field doubled in value/range */
693 static GstCaps *
_double_caps_field(const GstCaps * in,const gchar * field_name)694 _double_caps_field (const GstCaps * in, const gchar * field_name)
695 {
696 gint i;
697 GstCaps *out = gst_caps_new_empty ();
698
699 for (i = 0; i < gst_caps_get_size (in); i++) {
700 const GstStructure *cur = gst_caps_get_structure (in, i);
701 GstCapsFeatures *f = gst_caps_get_features (in, i);
702
703 GstStructure *res = _double_structure_field (cur, field_name);
704 out =
705 gst_caps_merge_structure_full (out, res,
706 f ? gst_caps_features_copy (f) : NULL);
707 }
708
709 return out;
710 }
711
712 /* Takes ownership of the input caps */
713 static GstCaps *
_expand_par_for_half_aspect(GstCaps * in,gboolean vertical_half_aspect)714 _expand_par_for_half_aspect (GstCaps * in, gboolean vertical_half_aspect)
715 {
716
717 guint mview_flags, mview_flags_mask;
718 GstCaps *out;
719 GstStructure *tmp;
720
721 out = gst_caps_new_empty ();
722
723 while (gst_caps_get_size (in) > 0) {
724 GstStructure *s;
725 GstCapsFeatures *features;
726
727 features = gst_caps_get_features (in, 0);
728 if (features)
729 features = gst_caps_features_copy (features);
730
731 s = gst_caps_steal_structure (in, 0);
732
733 if (!gst_structure_get_flagset (s, "multiview-flags", &mview_flags,
734 &mview_flags_mask)) {
735 gst_caps_append_structure_full (out, s, features);
736 continue;
737 }
738 /* If the input doesn't care about the half-aspect flag, allow current PAR in either variant */
739 if ((mview_flags_mask & GST_VIDEO_MULTIVIEW_FLAGS_HALF_ASPECT) == 0) {
740 gst_caps_append_structure_full (out, s, features);
741 continue;
742 }
743 if (!gst_structure_has_field (s, "pixel-aspect-ratio")) {
744 /* No par field, dont-care the half-aspect flag */
745 gst_structure_set (s, "multiview-flags",
746 GST_TYPE_VIDEO_MULTIVIEW_FLAGSET,
747 mview_flags & ~GST_VIDEO_MULTIVIEW_FLAGS_HALF_ASPECT,
748 mview_flags_mask & ~GST_VIDEO_MULTIVIEW_FLAGS_HALF_ASPECT, NULL);
749 gst_caps_append_structure_full (out, s, features);
750 continue;
751 }
752
753 /* Halve or double PAR base on inputs input specified. */
754
755 /* Append a copy with the half-aspect flag as-is */
756 tmp = gst_structure_copy (s);
757 out = gst_caps_merge_structure_full (out, tmp,
758 features ? gst_caps_features_copy (features) : NULL);
759
760 /* and then a copy inverted */
761 if (mview_flags & GST_VIDEO_MULTIVIEW_FLAGS_HALF_ASPECT) {
762 /* Input is half-aspect. Double/halve the PAR, clear the flag */
763 if (vertical_half_aspect)
764 tmp = _halve_structure_field (s, "pixel-aspect-ratio");
765 else
766 tmp = _double_structure_field (s, "pixel-aspect-ratio");
767 /* Clear the flag */
768 gst_structure_set (tmp, "multiview-flags",
769 GST_TYPE_VIDEO_MULTIVIEW_FLAGSET,
770 mview_flags & ~GST_VIDEO_MULTIVIEW_FLAGS_HALF_ASPECT,
771 mview_flags_mask | GST_VIDEO_MULTIVIEW_FLAGS_HALF_ASPECT, NULL);
772 } else {
773 if (vertical_half_aspect)
774 tmp = _double_structure_field (s, "pixel-aspect-ratio");
775 else
776 tmp = _halve_structure_field (s, "pixel-aspect-ratio");
777 /* Set the flag */
778 gst_structure_set (tmp, "multiview-flags",
779 GST_TYPE_VIDEO_MULTIVIEW_FLAGSET,
780 mview_flags | GST_VIDEO_MULTIVIEW_FLAGS_HALF_ASPECT,
781 mview_flags_mask | GST_VIDEO_MULTIVIEW_FLAGS_HALF_ASPECT, NULL);
782 }
783
784 out = gst_caps_merge_structure_full (out, tmp,
785 features ? gst_caps_features_copy (features) : NULL);
786
787 gst_structure_free (s);
788 if (features)
789 gst_caps_features_free (features);
790 }
791
792 gst_caps_unref (in);
793
794 return out;
795 }
796
797 /* If input supports top-bottom or row-interleaved, we may halve height to mono frames.
798 * If input supports left-right, checkerboard, quincunx or column-interleaved,
799 * we may halve width to mono frames.
800 * For output of top-bottom or row-interleaved, we may double the mono height
801 * For output of left-right, checkerboard, quincunx or column-interleaved,
802 * we may double the mono width.
803 * In all cases, if input has half-aspect and output does not, we may double the PAR
804 * And if input does *not* have half-aspect flag and output does not, we may halve the PAR
805 */
806 static GstCaps *
_expand_structure(GstGLViewConvert * viewconvert,GstCaps * out_caps,GstStructure * structure,GstCapsFeatures * features)807 _expand_structure (GstGLViewConvert * viewconvert,
808 GstCaps * out_caps, GstStructure * structure, GstCapsFeatures * features)
809 {
810 GstCaps *expanded_caps, *tmp;
811 GstCaps *mono_caps;
812 const gchar *default_mview_mode_str = NULL;
813 guint mview_flags, mview_flags_mask;
814 const GValue *in_modes;
815 gint i;
816
817 /* Empty caps to accumulate into */
818 expanded_caps = gst_caps_new_empty ();
819
820 /* First, set defaults if multiview flags are missing */
821 default_mview_mode_str =
822 gst_video_multiview_mode_to_caps_string (GST_VIDEO_MULTIVIEW_MODE_MONO);
823
824 mview_flags = GST_VIDEO_MULTIVIEW_FLAGS_NONE;
825 mview_flags_mask = GST_FLAG_SET_MASK_EXACT;
826
827 if (!gst_structure_has_field (structure, "multiview-mode")) {
828 gst_structure_set (structure,
829 "multiview-mode", G_TYPE_STRING, default_mview_mode_str, NULL);
830 }
831 if (!gst_structure_has_field (structure, "multiview-flags")) {
832 gst_structure_set (structure,
833 "multiview-flags", GST_TYPE_VIDEO_MULTIVIEW_FLAGSET, mview_flags,
834 mview_flags_mask, NULL);
835 } else {
836 gst_structure_get_flagset (structure, "multiview-flags",
837 &mview_flags, &mview_flags_mask);
838 }
839
840 in_modes = gst_structure_get_value (structure, "multiview-mode");
841 mono_caps = gst_caps_new_empty ();
842 if (gst_value_intersect (NULL, in_modes,
843 gst_video_multiview_get_mono_modes ())) {
844 GstStructure *new_struct = gst_structure_copy (structure);
845 gst_structure_set_value (new_struct, "multiview-mode",
846 gst_video_multiview_get_mono_modes ());
847 /* Half-aspect makes no sense for mono or unpacked, get rid of it */
848 if (mview_flags & GST_VIDEO_MULTIVIEW_FLAGS_HALF_ASPECT) {
849 gst_structure_set (new_struct, "multiview-flags",
850 GST_TYPE_VIDEO_MULTIVIEW_FLAGSET,
851 mview_flags & ~GST_VIDEO_MULTIVIEW_FLAGS_HALF_ASPECT,
852 mview_flags_mask & ~GST_VIDEO_MULTIVIEW_FLAGS_HALF_ASPECT, NULL);
853 }
854 gst_caps_append_structure_full (mono_caps, new_struct,
855 features ? gst_caps_features_copy (features) : NULL);
856 }
857 if (gst_value_intersect (NULL, in_modes,
858 gst_video_multiview_get_unpacked_modes ())) {
859 GstStructure *new_struct = gst_structure_copy (structure);
860
861 gst_structure_set_value (new_struct, "multiview-mode",
862 gst_video_multiview_get_mono_modes ());
863
864 /* Half-aspect makes no sense for mono or unpacked, get rid of it */
865 if (mview_flags & GST_VIDEO_MULTIVIEW_FLAGS_HALF_ASPECT) {
866 gst_structure_set (new_struct, "multiview-flags",
867 GST_TYPE_VIDEO_MULTIVIEW_FLAGSET,
868 mview_flags & ~GST_VIDEO_MULTIVIEW_FLAGS_HALF_ASPECT,
869 mview_flags_mask & ~GST_VIDEO_MULTIVIEW_FLAGS_HALF_ASPECT, NULL);
870 }
871 gst_caps_append_structure_full (mono_caps, new_struct,
872 features ? gst_caps_features_copy (features) : NULL);
873 }
874
875 if (gst_value_intersect (NULL, in_modes,
876 gst_video_multiview_get_doubled_height_modes ())) {
877 /* Append mono formats with height halved */
878 GstStructure *new_struct = _halve_structure_field (structure, "height");
879 gst_structure_set_value (new_struct, "multiview-mode",
880 gst_video_multiview_get_mono_modes ());
881 /* Normalise the half-aspect flag away */
882 if (mview_flags & GST_VIDEO_MULTIVIEW_FLAGS_HALF_ASPECT) {
883 GstStructure *s =
884 _halve_structure_field (new_struct, "pixel-aspect-ratio");
885 gst_structure_set (structure, "multiview-flags",
886 GST_TYPE_VIDEO_MULTIVIEW_FLAGSET,
887 mview_flags & ~GST_VIDEO_MULTIVIEW_FLAGS_HALF_ASPECT,
888 mview_flags_mask | GST_VIDEO_MULTIVIEW_FLAGS_HALF_ASPECT, NULL);
889 gst_structure_free (new_struct);
890 new_struct = s;
891 }
892 mono_caps = gst_caps_merge_structure_full (mono_caps, new_struct,
893 features ? gst_caps_features_copy (features) : NULL);
894 }
895 if (gst_value_intersect (NULL, in_modes,
896 gst_video_multiview_get_doubled_width_modes ())) {
897 /* Append mono formats with width halved */
898 GstStructure *new_struct = _halve_structure_field (structure, "width");
899 gst_structure_set_value (new_struct, "multiview-mode",
900 gst_video_multiview_get_mono_modes ());
901 /* Normalise the half-aspect flag away */
902 if (mview_flags & GST_VIDEO_MULTIVIEW_FLAGS_HALF_ASPECT) {
903 GstStructure *s =
904 _double_structure_field (new_struct, "pixel-aspect-ratio");
905 gst_structure_set (structure, "multiview-flags",
906 GST_TYPE_VIDEO_MULTIVIEW_FLAGSET,
907 mview_flags & ~GST_VIDEO_MULTIVIEW_FLAGS_HALF_ASPECT,
908 mview_flags_mask | GST_VIDEO_MULTIVIEW_FLAGS_HALF_ASPECT, NULL);
909 gst_structure_free (new_struct);
910 new_struct = s;
911 }
912 mono_caps = gst_caps_merge_structure_full (mono_caps, new_struct,
913 features ? gst_caps_features_copy (features) : NULL);
914 }
915 if (gst_value_intersect (NULL, in_modes,
916 gst_video_multiview_get_doubled_size_modes ())) {
917 /* Append checkerboard/doubled size formats with width & height halved */
918 GstStructure *new_struct_w = _halve_structure_field (structure, "width");
919 GstStructure *new_struct_wh =
920 _halve_structure_field (new_struct_w, "height");
921 gst_structure_free (new_struct_w);
922 gst_structure_set_value (new_struct_wh, "multiview-mode",
923 gst_video_multiview_get_mono_modes ());
924 mono_caps = gst_caps_merge_structure_full (mono_caps, new_struct_wh,
925 features ? gst_caps_features_copy (features) : NULL);
926 }
927
928 /* Everything is normalised now, unset the flags we can change */
929 /* Remove the views field, as these are all 'mono' modes
930 * Need to do this before we expand caps back out to frame packed modes */
931 for (i = 0; i < gst_caps_get_size (mono_caps); i++) {
932 GstStructure *s = gst_caps_get_structure (mono_caps, i);
933 gst_structure_remove_fields (s, "views", NULL);
934 if (gst_structure_get_flagset (s, "multiview-flags", &mview_flags,
935 &mview_flags_mask)) {
936 /* Preserve only the half-aspect and mixed-mono flags, for now.
937 * The rest we can change */
938 mview_flags_mask &=
939 (GST_VIDEO_MULTIVIEW_FLAGS_HALF_ASPECT |
940 GST_VIDEO_MULTIVIEW_FLAGS_MIXED_MONO);
941 gst_structure_set (s, "multiview-flags", GST_TYPE_VIDEO_MULTIVIEW_FLAGSET,
942 mview_flags, mview_flags_mask, NULL);
943 }
944 }
945
946 GST_TRACE_OBJECT (viewconvert,
947 "Collected single-view caps %" GST_PTR_FORMAT, mono_caps);
948 /* Put unpacked and mono modes first. We don't care about flags. Clear them */
949 tmp = gst_caps_copy (mono_caps);
950 for (i = 0; i < gst_caps_get_size (tmp); i++) {
951 GstStructure *s = gst_caps_get_structure (tmp, i);
952 gst_structure_remove_fields (s, "views", NULL);
953 if (gst_structure_get_flagset (s, "multiview-flags", &mview_flags,
954 &mview_flags_mask)) {
955 /* We can change any flags for mono modes - half-aspect and mixed-mono have no meaning */
956 mview_flags_mask = 0;
957 gst_structure_set (s, "multiview-flags", GST_TYPE_VIDEO_MULTIVIEW_FLAGSET,
958 mview_flags, mview_flags_mask, NULL);
959 }
960 }
961 expanded_caps = gst_caps_merge (expanded_caps, tmp);
962
963 /* Unpacked output modes have 2 views, for now */
964 tmp = gst_caps_copy (mono_caps);
965 gst_caps_set_value (tmp, "multiview-mode",
966 gst_video_multiview_get_unpacked_modes ());
967 for (i = 0; i < gst_caps_get_size (tmp); i++) {
968 GstStructure *s = gst_caps_get_structure (tmp, i);
969 gst_structure_set (s, "views", G_TYPE_INT, 2, NULL);
970 if (gst_structure_get_flagset (s, "multiview-flags", &mview_flags,
971 &mview_flags_mask)) {
972 /* We can change any flags for unpacked modes - half-aspect and mixed-mono have no meaning */
973 mview_flags_mask = 0;
974 gst_structure_set (s, "multiview-flags", GST_TYPE_VIDEO_MULTIVIEW_FLAGSET,
975 mview_flags, mview_flags_mask, NULL);
976 }
977 }
978 expanded_caps = gst_caps_merge (expanded_caps, tmp);
979
980 /* Double height output modes */
981 tmp = _double_caps_field (mono_caps, "height");
982 gst_caps_set_value (tmp, "multiview-mode",
983 gst_video_multiview_get_doubled_height_modes ());
984 tmp = _expand_par_for_half_aspect (tmp, TRUE);
985
986 expanded_caps = gst_caps_merge (expanded_caps, tmp);
987
988 /* Double width output modes */
989 tmp = _double_caps_field (mono_caps, "width");
990 gst_caps_set_value (tmp, "multiview-mode",
991 gst_video_multiview_get_doubled_width_modes ());
992 tmp = _expand_par_for_half_aspect (tmp, FALSE);
993
994 expanded_caps = gst_caps_merge (expanded_caps, tmp);
995
996 /* Double size output modes */
997 {
998 GstCaps *tmp_w = _double_caps_field (mono_caps, "width");
999 tmp = _double_caps_field (tmp_w, "height");
1000 gst_caps_unref (tmp_w);
1001 gst_caps_set_value (tmp, "multiview-mode",
1002 gst_video_multiview_get_doubled_size_modes ());
1003 expanded_caps = gst_caps_merge (expanded_caps, tmp);
1004 }
1005
1006 /* We're done with the mono caps now */
1007 gst_caps_unref (mono_caps);
1008
1009 GST_TRACE_OBJECT (viewconvert,
1010 "expanded transform caps now %" GST_PTR_FORMAT, expanded_caps);
1011
1012 if (gst_caps_is_empty (expanded_caps)) {
1013 gst_caps_unref (expanded_caps);
1014 return out_caps;
1015 }
1016 /* Really, we can rescale - so at this point we can append full-range
1017 * height/width/PAR as an unpreferred final option. */
1018 tmp = gst_caps_copy (expanded_caps);
1019 gst_caps_set_simple (tmp, "width", GST_TYPE_INT_RANGE, 1, G_MAXINT,
1020 "height", GST_TYPE_INT_RANGE, 1, G_MAXINT, NULL);
1021
1022 out_caps = gst_caps_merge (out_caps, expanded_caps);
1023 out_caps = gst_caps_merge (out_caps, tmp);
1024 return out_caps;
1025 }
1026
1027 static GstCaps *
_intersect_with_mview_mode(GstCaps * caps,GstVideoMultiviewMode mode,GstVideoMultiviewFlags flags)1028 _intersect_with_mview_mode (GstCaps * caps,
1029 GstVideoMultiviewMode mode, GstVideoMultiviewFlags flags)
1030 {
1031 GstCaps *filter, *result;
1032 const gchar *caps_str;
1033
1034 caps_str = gst_video_multiview_mode_to_caps_string (mode);
1035
1036 filter = gst_caps_new_simple ("video/x-raw",
1037 "multiview-mode", G_TYPE_STRING,
1038 caps_str, "multiview-flags", GST_TYPE_VIDEO_MULTIVIEW_FLAGSET, flags,
1039 GST_FLAG_SET_MASK_EXACT, NULL);
1040
1041 if (mode == GST_VIDEO_MULTIVIEW_MODE_SEPARATED ||
1042 mode == GST_VIDEO_MULTIVIEW_MODE_FRAME_BY_FRAME)
1043 gst_caps_set_simple (filter, "views", G_TYPE_INT, 2, NULL);
1044
1045 gst_caps_set_features (filter, 0, gst_caps_features_new_any ());
1046
1047 GST_DEBUG ("Intersecting target caps %" GST_PTR_FORMAT
1048 " with caps %" GST_PTR_FORMAT, caps, filter);
1049
1050 result = gst_caps_intersect_full (caps, filter, GST_CAPS_INTERSECT_FIRST);
1051 gst_caps_unref (filter);
1052 return result;
1053 }
1054
1055 static GstCaps *
_intersect_with_mview_modes(GstCaps * caps,const GValue * modes)1056 _intersect_with_mview_modes (GstCaps * caps, const GValue * modes)
1057 {
1058 GstCaps *filter, *result;
1059
1060 filter = gst_caps_new_empty_simple ("video/x-raw");
1061
1062 gst_caps_set_value (filter, "multiview-mode", modes);
1063 gst_caps_set_features (filter, 0, gst_caps_features_new_any ());
1064
1065 GST_DEBUG ("Intersecting target caps %" GST_PTR_FORMAT
1066 " with caps %" GST_PTR_FORMAT, caps, filter);
1067
1068 result = gst_caps_intersect_full (caps, filter, GST_CAPS_INTERSECT_FIRST);
1069 gst_caps_unref (filter);
1070 return result;
1071 }
1072
1073 /**
1074 * gst_gl_view_convert_transform_caps:
1075 * @viewconvert: a #GstGLViewConvert
1076 * @direction: a #GstPadDirection
1077 * @caps: (transfer none): the #GstCaps to transform
1078 * @filter: (transfer none): a set of filter #GstCaps
1079 *
1080 * Provides an implementation of #GstBaseTransformClass.transform_caps()
1081 *
1082 * Returns: (transfer full): the converted #GstCaps
1083 *
1084 * Since: 1.6
1085 */
1086 GstCaps *
gst_gl_view_convert_transform_caps(GstGLViewConvert * viewconvert,GstPadDirection direction,GstCaps * caps,GstCaps * filter)1087 gst_gl_view_convert_transform_caps (GstGLViewConvert * viewconvert,
1088 GstPadDirection direction, GstCaps * caps, GstCaps * filter)
1089 {
1090 gint i, n;
1091 GstCaps *base_caps = gst_static_caps_get (&caps_template);
1092 GstCaps *out_caps, *tmp_caps;
1093
1094 g_return_val_if_fail (GST_IS_GL_VIEW_CONVERT (viewconvert), NULL);
1095
1096 GST_DEBUG_OBJECT (viewconvert, "Direction %s "
1097 "input caps %" GST_PTR_FORMAT " filter %" GST_PTR_FORMAT,
1098 direction == GST_PAD_SINK ? "sink" : "src", caps, filter);
1099
1100 /* We can only process GLmemory RGBA caps, start from that */
1101 caps = gst_caps_intersect (caps, base_caps);
1102 gst_caps_unref (base_caps);
1103
1104 /* Change input/output to the formats we can convert to/from,
1105 * but keep the original caps at the start - we will always prefer
1106 * passthrough */
1107 if (direction == GST_PAD_SINK) {
1108 out_caps = gst_caps_copy (caps);
1109 if (viewconvert->input_mode_override != GST_VIDEO_MULTIVIEW_MODE_NONE) {
1110 GstVideoMultiviewMode mode = viewconvert->input_mode_override;
1111 GstVideoMultiviewFlags flags = viewconvert->input_flags_override;
1112
1113 const gchar *caps_str = gst_video_multiview_mode_to_caps_string (mode);
1114 /* Coerce the input caps before transforming, so the sizes come out right */
1115 gst_caps_set_simple (out_caps, "multiview-mode", G_TYPE_STRING,
1116 caps_str, "multiview-flags", GST_TYPE_VIDEO_MULTIVIEW_FLAGSET, flags,
1117 GST_FLAG_SET_MASK_EXACT, NULL);
1118 }
1119 } else {
1120 out_caps = gst_caps_new_empty ();
1121 }
1122
1123 for (i = 0; i < gst_caps_get_size (caps); i++) {
1124 GstStructure *structure = gst_caps_get_structure (caps, i);
1125 GstCapsFeatures *features = gst_caps_get_features (caps, i);
1126 out_caps = _expand_structure (viewconvert, out_caps, structure, features);
1127 }
1128
1129 if (gst_caps_is_empty (out_caps))
1130 goto out;
1131
1132 /* If we have an output mode override, limit things to that */
1133 if (direction == GST_PAD_SINK &&
1134 viewconvert->output_mode_override != GST_VIDEO_MULTIVIEW_MODE_NONE) {
1135
1136 tmp_caps = _intersect_with_mview_mode (out_caps,
1137 viewconvert->output_mode_override, viewconvert->output_flags_override);
1138
1139 gst_caps_unref (out_caps);
1140 out_caps = tmp_caps;
1141 } else if (viewconvert->input_mode_override != GST_VIDEO_MULTIVIEW_MODE_NONE) {
1142 /* Prepend a copy of our preferred input caps in case the peer
1143 * can handle them */
1144 tmp_caps = _intersect_with_mview_mode (out_caps,
1145 viewconvert->input_mode_override, viewconvert->input_flags_override);
1146 out_caps = gst_caps_merge (out_caps, tmp_caps);
1147 }
1148 if (direction == GST_PAD_SRC) {
1149 GstStructure *s;
1150 /* When generating input caps, we also need a copy of the mono caps
1151 * without multiview-mode or flags for backwards compat, at the end */
1152 tmp_caps = _intersect_with_mview_mode (caps,
1153 GST_VIDEO_MULTIVIEW_MODE_MONO, GST_VIDEO_MULTIVIEW_FLAGS_NONE);
1154 if (!gst_caps_is_empty (tmp_caps)) {
1155 s = gst_caps_get_structure (tmp_caps, 0);
1156 gst_structure_remove_fields (s, "multiview-mode", "multiview-flags",
1157 NULL);
1158 out_caps = gst_caps_merge (out_caps, tmp_caps);
1159 } else
1160 gst_caps_unref (tmp_caps);
1161 }
1162 out:
1163 gst_caps_unref (caps);
1164
1165 n = gst_caps_get_size (out_caps);
1166 for (i = 0; i < n; i++) {
1167 GstStructure *s = gst_caps_get_structure (out_caps, i);
1168
1169 gst_structure_remove_fields (s, "texture-target", NULL);
1170 }
1171
1172 GST_DEBUG_OBJECT (viewconvert, "Returning caps %" GST_PTR_FORMAT, out_caps);
1173 return out_caps;
1174 }
1175
1176 static guint
_get_target_bitmask_from_g_value(const GValue * targets)1177 _get_target_bitmask_from_g_value (const GValue * targets)
1178 {
1179 guint new_targets = 0;
1180
1181 if (targets == NULL) {
1182 new_targets = 1 << GST_GL_TEXTURE_TARGET_2D;
1183 } else if (G_TYPE_CHECK_VALUE_TYPE (targets, G_TYPE_STRING)) {
1184 GstGLTextureTarget target;
1185 const gchar *str;
1186
1187 str = g_value_get_string (targets);
1188 target = gst_gl_texture_target_from_string (str);
1189
1190 if (target)
1191 new_targets |= 1 << target;
1192 } else if (G_TYPE_CHECK_VALUE_TYPE (targets, GST_TYPE_LIST)) {
1193 gint j, m;
1194
1195 m = gst_value_list_get_size (targets);
1196 for (j = 0; j < m; j++) {
1197 const GValue *val = gst_value_list_get_value (targets, j);
1198 GstGLTextureTarget target;
1199 const gchar *str;
1200
1201 str = g_value_get_string (val);
1202 target = gst_gl_texture_target_from_string (str);
1203 if (target)
1204 new_targets |= 1 << target;
1205 }
1206 }
1207
1208 return new_targets;
1209 }
1210
1211 static GstCaps *
_fixate_texture_target(GstGLViewConvert * viewconvert,GstPadDirection direction,GstCaps * caps,GstCaps * other)1212 _fixate_texture_target (GstGLViewConvert * viewconvert,
1213 GstPadDirection direction, GstCaps * caps, GstCaps * other)
1214 {
1215 GValue item = G_VALUE_INIT;
1216 const GValue *targets, *other_targets;
1217 guint targets_mask = 0, other_targets_mask = 0, result_mask;
1218 GstStructure *s, *s_other;
1219
1220 other = gst_caps_make_writable (other);
1221 s = gst_caps_get_structure (caps, 0);
1222 s_other = gst_caps_get_structure (other, 0);
1223
1224 other_targets = gst_structure_get_value (s_other, "texture-target");
1225 targets = gst_structure_get_value (s, "texture-target");
1226
1227 targets_mask = _get_target_bitmask_from_g_value (targets);
1228 other_targets_mask = _get_target_bitmask_from_g_value (other_targets);
1229
1230 result_mask = targets_mask & other_targets_mask;
1231 if (result_mask == 0) {
1232 /* nothing we can do here */
1233 return gst_caps_fixate (other);
1234 }
1235
1236 if (direction == GST_PAD_SINK) {
1237 result_mask &=
1238 (1 << GST_GL_TEXTURE_TARGET_2D | 1 << GST_GL_TEXTURE_TARGET_RECTANGLE);
1239 } else {
1240 /* if the src caps has 2D support we can 'convert' to anything */
1241 if (targets_mask & (1 << GST_GL_TEXTURE_TARGET_2D))
1242 result_mask = -1;
1243 else
1244 result_mask = other_targets_mask;
1245 }
1246
1247 g_value_init (&item, G_TYPE_STRING);
1248 if (result_mask & (1 << GST_GL_TEXTURE_TARGET_2D)) {
1249 g_value_set_static_string (&item, GST_GL_TEXTURE_TARGET_2D_STR);
1250 } else if (result_mask & (1 << GST_GL_TEXTURE_TARGET_RECTANGLE)) {
1251 g_value_set_static_string (&item, GST_GL_TEXTURE_TARGET_RECTANGLE_STR);
1252 } else if (result_mask & (1 << GST_GL_TEXTURE_TARGET_EXTERNAL_OES)) {
1253 g_value_set_static_string (&item, GST_GL_TEXTURE_TARGET_EXTERNAL_OES_STR);
1254 }
1255
1256 gst_structure_set_value (s_other, "texture-target", &item);
1257
1258 g_value_unset (&item);
1259
1260 return gst_caps_fixate (other);
1261 }
1262
1263 /**
1264 * gst_gl_view_convert_fixate_caps:
1265 * @viewconvert: a #GstGLViewConvert
1266 * @direction: a #GstPadDirection
1267 * @caps: (transfer none): the #GstCaps of @direction
1268 * @othercaps: (transfer full): the #GstCaps to fixate
1269 *
1270 * Provides an implementation of #GstBaseTransformClass.fixate_caps()
1271 *
1272 * Returns: (transfer full): the fixated #GstCaps
1273 *
1274 * Since: 1.6
1275 */
1276 GstCaps *
gst_gl_view_convert_fixate_caps(GstGLViewConvert * viewconvert,GstPadDirection direction,GstCaps * caps,GstCaps * othercaps)1277 gst_gl_view_convert_fixate_caps (GstGLViewConvert * viewconvert,
1278 GstPadDirection direction, GstCaps * caps, GstCaps * othercaps)
1279 {
1280 GstVideoMultiviewMode mode = viewconvert->output_mode_override;
1281 GstVideoMultiviewFlags flags = viewconvert->output_flags_override;
1282 GstCaps *tmp;
1283
1284 g_return_val_if_fail (GST_IS_GL_VIEW_CONVERT (viewconvert), NULL);
1285
1286 othercaps = gst_caps_make_writable (othercaps);
1287 GST_LOG_OBJECT (viewconvert, "dir %s fixating %" GST_PTR_FORMAT
1288 " against caps %" GST_PTR_FORMAT,
1289 direction == GST_PAD_SINK ? "sink" : "src", othercaps, caps);
1290
1291 if (direction == GST_PAD_SINK) {
1292 if (mode != GST_VIDEO_MULTIVIEW_MODE_NONE) {
1293 /* We have a requested output mode and are fixating source caps, try and enforce it */
1294 tmp = _intersect_with_mview_mode (othercaps, mode, flags);
1295 gst_caps_unref (othercaps);
1296 othercaps = tmp;
1297 } else {
1298 /* See if we can do passthrough */
1299 GstVideoInfo info;
1300
1301 if (gst_video_info_from_caps (&info, caps)) {
1302 GstVideoMultiviewMode mode = GST_VIDEO_INFO_MULTIVIEW_MODE (&info);
1303 GstVideoMultiviewFlags flags = GST_VIDEO_INFO_MULTIVIEW_FLAGS (&info);
1304
1305 if (viewconvert->input_mode_override != GST_VIDEO_MULTIVIEW_MODE_NONE) {
1306 mode = viewconvert->input_mode_override;
1307 flags = viewconvert->input_flags_override;
1308 }
1309
1310 tmp = _intersect_with_mview_mode (othercaps, mode, flags);
1311 if (gst_caps_is_empty (tmp)) {
1312 /* Nope, we can't pass our input caps downstream */
1313 gst_caps_unref (tmp);
1314 } else {
1315 gst_caps_unref (othercaps);
1316 othercaps = tmp;
1317 goto done;
1318 }
1319 }
1320
1321 /* Prefer an unpacked mode for output */
1322 tmp =
1323 _intersect_with_mview_modes (othercaps,
1324 gst_video_multiview_get_unpacked_modes ());
1325 if (!gst_caps_is_empty (tmp)) {
1326 gst_caps_unref (othercaps);
1327 othercaps = tmp;
1328 } else {
1329 gst_caps_unref (tmp);
1330 }
1331 }
1332 } else if (viewconvert->input_mode_override != GST_VIDEO_MULTIVIEW_MODE_NONE) {
1333 /* See if we can coerce the caps into matching input mode/flags,
1334 * in case it doesn't care at all, but allow it not to too */
1335 mode = viewconvert->input_mode_override;
1336 flags = viewconvert->input_flags_override;
1337 tmp = _intersect_with_mview_mode (othercaps, mode, flags);
1338 if (gst_caps_is_empty (tmp)) {
1339 /* Nope, we can pass our input caps downstream */
1340 gst_caps_unref (tmp);
1341 } else {
1342 gst_caps_unref (othercaps);
1343 othercaps = tmp;
1344 }
1345 }
1346
1347 othercaps = _fixate_texture_target (viewconvert, direction, caps, othercaps);
1348
1349 done:
1350 GST_DEBUG_OBJECT (viewconvert, "dir %s fixated to %" GST_PTR_FORMAT
1351 " against caps %" GST_PTR_FORMAT,
1352 direction == GST_PAD_SINK ? "sink" : "src", othercaps, caps);
1353 return othercaps;
1354 }
1355
1356 /**
1357 * gst_gl_view_convert_reset:
1358 * @viewconvert: a #GstGLViewConvert
1359 *
1360 * Reset @viewconvert to the default state. Further operation will require
1361 * setting the caps with gst_gl_view_convert_set_caps().
1362 *
1363 * Since: 1.6
1364 */
1365 void
gst_gl_view_convert_reset(GstGLViewConvert * viewconvert)1366 gst_gl_view_convert_reset (GstGLViewConvert * viewconvert)
1367 {
1368 g_return_if_fail (GST_IS_GL_VIEW_CONVERT (viewconvert));
1369
1370 gst_clear_object (&viewconvert->shader);
1371 gst_clear_object (&viewconvert->fbo);
1372
1373 if (viewconvert->context) {
1374 gst_gl_context_thread_add (viewconvert->context,
1375 (GstGLContextThreadFunc) _reset_gl, viewconvert);
1376 }
1377
1378 viewconvert->initted = FALSE;
1379 viewconvert->reconfigure = FALSE;
1380 }
1381
1382 static void
gst_gl_view_convert_set_property(GObject * object,guint prop_id,const GValue * value,GParamSpec * pspec)1383 gst_gl_view_convert_set_property (GObject * object, guint prop_id,
1384 const GValue * value, GParamSpec * pspec)
1385 {
1386 GstGLViewConvert *convert = GST_GL_VIEW_CONVERT (object);
1387 switch (prop_id) {
1388 case PROP_INPUT_LAYOUT:
1389 convert->input_mode_override = g_value_get_enum (value);
1390 break;
1391 case PROP_INPUT_FLAGS:
1392 convert->input_flags_override = g_value_get_flags (value);
1393 break;
1394 case PROP_OUTPUT_LAYOUT:
1395 convert->output_mode_override = g_value_get_enum (value);
1396 break;
1397 case PROP_OUTPUT_FLAGS:
1398 convert->output_flags_override = g_value_get_flags (value);
1399 break;
1400 case PROP_OUTPUT_DOWNMIX_MODE:
1401 convert->downmix_mode = g_value_get_enum (value);
1402 break;
1403 default:
1404 G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
1405 break;
1406 }
1407 GST_OBJECT_LOCK (convert);
1408 convert->reconfigure = TRUE;
1409 GST_OBJECT_UNLOCK (convert);
1410 }
1411
1412 static void
gst_gl_view_convert_get_property(GObject * object,guint prop_id,GValue * value,GParamSpec * pspec)1413 gst_gl_view_convert_get_property (GObject * object, guint prop_id,
1414 GValue * value, GParamSpec * pspec)
1415 {
1416 GstGLViewConvert *convert = GST_GL_VIEW_CONVERT (object);
1417 switch (prop_id) {
1418 case PROP_INPUT_LAYOUT:
1419 g_value_set_enum (value, convert->input_mode_override);
1420 break;
1421 case PROP_INPUT_FLAGS:
1422 g_value_set_flags (value, convert->input_flags_override);
1423 break;
1424 case PROP_OUTPUT_LAYOUT:
1425 g_value_set_enum (value, convert->output_mode_override);
1426 break;
1427 case PROP_OUTPUT_FLAGS:
1428 g_value_set_flags (value, convert->output_flags_override);
1429 break;
1430 case PROP_OUTPUT_DOWNMIX_MODE:
1431 g_value_set_enum (value, convert->downmix_mode);
1432 break;
1433 default:
1434 G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
1435 break;
1436 }
1437 }
1438
1439 /**
1440 * gst_gl_view_convert_perform:
1441 * @viewconvert: a #GstGLViewConvert
1442 * @inbuf: (transfer none): the #GstGLMemory filled #GstBuffer to convert
1443 *
1444 * Converts the data contained by @inbuf using the formats specified by the
1445 * #GstCaps passed to gst_gl_view_convert_set_caps()
1446 *
1447 * Returns: (transfer full): a converted #GstBuffer or %NULL
1448 *
1449 * Since: 1.6
1450 */
1451 GstBuffer *
gst_gl_view_convert_perform(GstGLViewConvert * viewconvert,GstBuffer * inbuf)1452 gst_gl_view_convert_perform (GstGLViewConvert * viewconvert, GstBuffer * inbuf)
1453 {
1454 GstBuffer *out;
1455
1456 if (gst_gl_view_convert_submit_input_buffer (viewconvert,
1457 GST_BUFFER_IS_DISCONT (inbuf), gst_buffer_ref (inbuf)) != GST_FLOW_OK)
1458 return NULL;
1459 if (gst_gl_view_convert_get_output (viewconvert, &out) != GST_FLOW_OK)
1460 return NULL;
1461
1462 return out;
1463 }
1464
1465 /* called by _init_convert (in the gl thread) */
1466 static gboolean
_init_view_convert_fbo(GstGLViewConvert * viewconvert)1467 _init_view_convert_fbo (GstGLViewConvert * viewconvert)
1468 {
1469 guint out_width, out_height;
1470
1471 out_width = GST_VIDEO_INFO_WIDTH (&viewconvert->out_info);
1472 out_height = GST_VIDEO_INFO_HEIGHT (&viewconvert->out_info);
1473
1474 viewconvert->fbo =
1475 gst_gl_framebuffer_new_with_default_depth (viewconvert->context,
1476 out_width, out_height);
1477
1478 return viewconvert->fbo != NULL;
1479 }
1480
1481 /* free after use */
1482 static gchar *
_get_shader_string(GstGLViewConvert * viewconvert,GstGLShader * shader,GstVideoMultiviewMode in_mode,GstVideoMultiviewMode out_mode,GstGLSLVersion version,GstGLSLProfile profile)1483 _get_shader_string (GstGLViewConvert * viewconvert, GstGLShader * shader,
1484 GstVideoMultiviewMode in_mode, GstVideoMultiviewMode out_mode,
1485 GstGLSLVersion version, GstGLSLProfile profile)
1486 {
1487 const gchar *input_str, *output_str;
1488 gboolean mono_input = FALSE;
1489 gchar *tmp, *tmp2;
1490 GString *str = g_string_new (NULL);
1491 guint n_outputs = 1;
1492
1493 switch (in_mode) {
1494 case GST_VIDEO_MULTIVIEW_MODE_NONE:
1495 case GST_VIDEO_MULTIVIEW_MODE_MONO:
1496 case GST_VIDEO_MULTIVIEW_MODE_LEFT:
1497 case GST_VIDEO_MULTIVIEW_MODE_RIGHT:
1498 mono_input = TRUE;
1499 /* Fall through */
1500 default:
1501 input_str = frag_input;
1502 break;
1503 }
1504
1505 switch (out_mode) {
1506 case GST_VIDEO_MULTIVIEW_MODE_LEFT:
1507 output_str = frag_output_left;
1508 break;
1509 case GST_VIDEO_MULTIVIEW_MODE_RIGHT:
1510 output_str = frag_output_right;
1511 break;
1512 case GST_VIDEO_MULTIVIEW_MODE_SIDE_BY_SIDE_QUINCUNX:
1513 /* FIXME: implement properly with sub-sampling */
1514 case GST_VIDEO_MULTIVIEW_MODE_SIDE_BY_SIDE:
1515 output_str = frag_output_side_by_side;
1516 break;
1517 case GST_VIDEO_MULTIVIEW_MODE_TOP_BOTTOM:
1518 output_str = frag_output_top_bottom;
1519 break;
1520 case GST_VIDEO_MULTIVIEW_MODE_COLUMN_INTERLEAVED:
1521 output_str = frag_output_column_interleaved;
1522 break;
1523 case GST_VIDEO_MULTIVIEW_MODE_ROW_INTERLEAVED:
1524 output_str = frag_output_row_interleaved;
1525 break;
1526 case GST_VIDEO_MULTIVIEW_MODE_SEPARATED:
1527 case GST_VIDEO_MULTIVIEW_MODE_FRAME_BY_FRAME:
1528 output_str = frag_output_separated;
1529 n_outputs = 2;
1530 break;
1531 case GST_VIDEO_MULTIVIEW_MODE_CHECKERBOARD:
1532 output_str = frag_output_checkerboard;
1533 break;
1534 case GST_VIDEO_MULTIVIEW_MODE_NONE:
1535 case GST_VIDEO_MULTIVIEW_MODE_MONO:
1536 default:
1537 if (mono_input)
1538 output_str = frag_output_left;
1539 else
1540 output_str = frag_output_downmix;
1541 break;
1542 }
1543
1544 if (viewconvert->from_texture_target == GST_GL_TEXTURE_TARGET_EXTERNAL_OES)
1545 g_string_append (str, glsl_OES_extension_string);
1546
1547 g_string_append (str,
1548 gst_gl_shader_string_get_highest_precision (viewconvert->context, version,
1549 profile));
1550 g_string_append (str, fragment_header);
1551
1552 /* GL 3.3+ and GL ES 3.x */
1553 if ((profile == GST_GLSL_PROFILE_CORE && version >= GST_GLSL_VERSION_330)
1554 || (profile == GST_GLSL_PROFILE_ES && version >= GST_GLSL_VERSION_300)) {
1555 if (n_outputs > 1) {
1556 gint i;
1557
1558 for (i = 0; i < n_outputs; i++) {
1559 g_string_append_printf (str,
1560 "layout(location = %d) out vec4 fragColor_%d;\n", i, i);
1561 }
1562 } else {
1563 g_string_append (str, "layout (location = 0) out vec4 fragColor;\n");
1564 }
1565 } else if (profile == GST_GLSL_PROFILE_CORE
1566 && version >= GST_GLSL_VERSION_150) {
1567 /* no layout specifiers, use glBindFragDataLocation instead */
1568 if (n_outputs > 1) {
1569 gint i;
1570
1571 for (i = 0; i < n_outputs; i++) {
1572 gchar *var_name = g_strdup_printf ("fragColor_%d", i);
1573 g_string_append_printf (str, "out vec4 %s;\n", var_name);
1574 gst_gl_shader_bind_frag_data_location (shader, i, var_name);
1575 g_free (var_name);
1576 }
1577 } else {
1578 g_string_append (str, "out vec4 fragColor;\n");
1579 gst_gl_shader_bind_frag_data_location (shader, 0, "fragColor");
1580 }
1581 }
1582
1583 {
1584 const gchar *varying = NULL;
1585
1586 if ((profile == GST_GLSL_PROFILE_ES && version >= GST_GLSL_VERSION_300)
1587 || (profile == GST_GLSL_PROFILE_CORE
1588 && version >= GST_GLSL_VERSION_150)) {
1589 varying = "in";
1590 } else {
1591 varying = "varying";
1592 }
1593 g_string_append_printf (str,
1594 "\n%s vec2 v_texcoord;\nvoid main() {\nvec4 l, r;\n", varying);
1595 }
1596
1597 g_string_append (str, input_str);
1598 g_string_append (str, output_str);
1599 g_string_append (str, "\n}");
1600 tmp = g_string_free (str, FALSE);
1601
1602 tmp2 =
1603 _gst_glsl_mangle_shader (tmp, GL_FRAGMENT_SHADER,
1604 GST_GL_TEXTURE_TARGET_2D, viewconvert->from_texture_target,
1605 viewconvert->context, &version, &profile);
1606
1607 return tmp2;
1608 }
1609
1610 static void
_bind_buffer(GstGLViewConvert * viewconvert)1611 _bind_buffer (GstGLViewConvert * viewconvert)
1612 {
1613 const GstGLFuncs *gl = viewconvert->context->gl_vtable;
1614 gl->BindBuffer (GL_ELEMENT_ARRAY_BUFFER, viewconvert->priv->vbo_indices);
1615 gl->BindBuffer (GL_ARRAY_BUFFER, viewconvert->priv->vertex_buffer);
1616 /* Load the vertex position */
1617 gl->VertexAttribPointer (viewconvert->priv->attr_position, 3, GL_FLOAT,
1618 GL_FALSE, 5 * sizeof (GLfloat), (void *) 0);
1619 /* Load the texture coordinate */
1620 gl->VertexAttribPointer (viewconvert->priv->attr_texture, 2, GL_FLOAT,
1621 GL_FALSE, 5 * sizeof (GLfloat), (void *) (3 * sizeof (GLfloat)));
1622 gl->EnableVertexAttribArray (viewconvert->priv->attr_position);
1623 gl->EnableVertexAttribArray (viewconvert->priv->attr_texture);
1624 }
1625
1626 static void
_unbind_buffer(GstGLViewConvert * viewconvert)1627 _unbind_buffer (GstGLViewConvert * viewconvert)
1628 {
1629 const GstGLFuncs *gl = viewconvert->context->gl_vtable;
1630 gl->BindBuffer (GL_ELEMENT_ARRAY_BUFFER, 0);
1631 gl->BindBuffer (GL_ARRAY_BUFFER, 0);
1632 gl->DisableVertexAttribArray (viewconvert->priv->attr_position);
1633 gl->DisableVertexAttribArray (viewconvert->priv->attr_texture);
1634 }
1635
1636 /* Called in the gl thread */
1637 static gboolean
_init_view_convert(GstGLViewConvert * viewconvert)1638 _init_view_convert (GstGLViewConvert * viewconvert)
1639 {
1640 GstGLViewConvertPrivate *priv = viewconvert->priv;
1641 GstVideoMultiviewMode in_mode = priv->input_mode;
1642 GstVideoMultiviewMode out_mode = priv->output_mode;
1643 GstVideoMultiviewFlags in_flags = priv->input_flags;
1644 GstVideoMultiviewFlags out_flags = priv->output_flags;
1645 gfloat tex_scale[2][2] = {
1646 {1., 1.},
1647 {1., 1.}
1648 };
1649 gfloat offsets[2][2] = {
1650 {0., 0.},
1651 {0., 0.}
1652 };
1653 gchar *fragment_source_str;
1654 GstGLFuncs *gl;
1655 gint l_index, r_index;
1656
1657 gl = viewconvert->context->gl_vtable;
1658 if (viewconvert->reconfigure)
1659 gst_gl_view_convert_reset (viewconvert);
1660 if (viewconvert->initted)
1661 return TRUE;
1662
1663 GST_LOG_OBJECT (viewconvert,
1664 "Initializing multiview conversion from %s mode %d flags 0x%x w %u h %u to "
1665 "%s mode %d flags 0x%x w %u h %u",
1666 gst_video_format_to_string (GST_VIDEO_INFO_FORMAT
1667 (&viewconvert->in_info)), in_mode, in_flags,
1668 viewconvert->in_info.width, viewconvert->in_info.height,
1669 gst_video_format_to_string (GST_VIDEO_INFO_FORMAT
1670 (&viewconvert->out_info)), out_mode, out_flags,
1671 viewconvert->out_info.width, viewconvert->out_info.height);
1672
1673 if (!gl->CreateProgramObject && !gl->CreateProgram) {
1674 GST_ERROR_OBJECT (viewconvert, "Cannot perform multiview conversion "
1675 "without OpenGL shaders");
1676 goto error;
1677 }
1678
1679 if (out_mode == GST_VIDEO_MULTIVIEW_MODE_SEPARATED
1680 || out_mode == GST_VIDEO_MULTIVIEW_MODE_FRAME_BY_FRAME) {
1681 if (!gl->DrawBuffers) {
1682 GST_ERROR_OBJECT (viewconvert,
1683 "Separate texture output mode requested however the current "
1684 "OpenGL API does not support drawing to multiple buffers");
1685 goto error;
1686 }
1687 }
1688
1689 if ((in_flags & GST_VIDEO_MULTIVIEW_FLAGS_RIGHT_VIEW_FIRST) ==
1690 (out_flags & GST_VIDEO_MULTIVIEW_FLAGS_RIGHT_VIEW_FIRST)) {
1691 l_index = 0;
1692 r_index = 1;
1693 } else {
1694 GST_LOG_OBJECT (viewconvert, "Switching left/right views");
1695 /* Swap the views */
1696 l_index = 1;
1697 r_index = 0;
1698 }
1699
1700 if (in_mode < GST_VIDEO_MULTIVIEW_MODE_SIDE_BY_SIDE) { /* unknown/mono/left/right single image */
1701 } else if (in_mode == GST_VIDEO_MULTIVIEW_MODE_SIDE_BY_SIDE ||
1702 in_mode == GST_VIDEO_MULTIVIEW_MODE_SIDE_BY_SIDE_QUINCUNX) {
1703 /* Side-by-side input */
1704 offsets[r_index][0] += 0.5 * tex_scale[r_index][0];
1705 tex_scale[0][0] *= 0.5f; /* Half horizontal scale */
1706 tex_scale[1][0] *= 0.5f;
1707 } else if (in_mode == GST_VIDEO_MULTIVIEW_MODE_TOP_BOTTOM) { /* top-bottom */
1708 offsets[r_index][1] += 0.5 * tex_scale[r_index][1];
1709 tex_scale[0][1] *= 0.5f; /* Half vertical scale */
1710 tex_scale[1][1] *= 0.5f;
1711 }
1712
1713 /* Flipped is vertical, flopped is horizontal.
1714 * Adjust and offset per-view scaling. This needs to be done
1715 * after the input scaling already splits the views, before
1716 * adding any output scaling. */
1717 if ((in_flags & GST_VIDEO_MULTIVIEW_FLAGS_LEFT_FLIPPED) !=
1718 (out_flags & GST_VIDEO_MULTIVIEW_FLAGS_LEFT_FLIPPED)) {
1719 offsets[l_index][1] += tex_scale[l_index][1];
1720 tex_scale[l_index][1] *= -1.0;
1721 }
1722 if ((in_flags & GST_VIDEO_MULTIVIEW_FLAGS_LEFT_FLOPPED) !=
1723 (out_flags & GST_VIDEO_MULTIVIEW_FLAGS_LEFT_FLOPPED)) {
1724 offsets[l_index][0] += tex_scale[l_index][0];
1725 tex_scale[l_index][0] *= -1.0;
1726 }
1727 if ((in_flags & GST_VIDEO_MULTIVIEW_FLAGS_RIGHT_FLIPPED) !=
1728 (out_flags & GST_VIDEO_MULTIVIEW_FLAGS_RIGHT_FLIPPED)) {
1729 offsets[r_index][1] += tex_scale[r_index][1];
1730 tex_scale[r_index][1] *= -1.0;
1731 }
1732 if ((in_flags & GST_VIDEO_MULTIVIEW_FLAGS_RIGHT_FLOPPED) !=
1733 (out_flags & GST_VIDEO_MULTIVIEW_FLAGS_RIGHT_FLOPPED)) {
1734 offsets[r_index][0] += tex_scale[r_index][0];
1735 tex_scale[r_index][0] *= -1.0;
1736 }
1737
1738 if (out_mode == GST_VIDEO_MULTIVIEW_MODE_SIDE_BY_SIDE ||
1739 out_mode == GST_VIDEO_MULTIVIEW_MODE_SIDE_BY_SIDE_QUINCUNX) {
1740 /* Side-by-Side */
1741 offsets[1][0] -= tex_scale[1][0];
1742 tex_scale[0][0] *= 2.0f;
1743 tex_scale[1][0] *= 2.0f;
1744 } else if (out_mode == GST_VIDEO_MULTIVIEW_MODE_TOP_BOTTOM) {
1745 offsets[1][1] -= tex_scale[1][1];
1746 tex_scale[0][1] *= 2.0f;
1747 tex_scale[1][1] *= 2.0f;
1748 }
1749
1750 GST_DEBUG_OBJECT (viewconvert,
1751 "Scaling matrix [ %f, %f ] [ %f %f]. Offsets [ %f, %f ] [ %f, %f ]",
1752 tex_scale[0][0], tex_scale[0][1],
1753 tex_scale[1][0], tex_scale[1][1],
1754 offsets[0][0], offsets[0][1], offsets[1][0], offsets[1][1]);
1755
1756 viewconvert->shader = gst_gl_shader_new (viewconvert->context);
1757 {
1758 GstGLSLVersion version;
1759 GstGLSLProfile profile;
1760 GstGLSLStage *vert, *frag;
1761 gchar *tmp, *tmp1, *version_str;
1762 const gchar *strings[2];
1763 GError *error = NULL;
1764
1765 tmp =
1766 _gst_glsl_mangle_shader
1767 (gst_gl_shader_string_vertex_mat4_vertex_transform, GL_VERTEX_SHADER,
1768 GST_GL_TEXTURE_TARGET_2D, viewconvert->from_texture_target,
1769 viewconvert->context, &version, &profile);
1770
1771 tmp1 = gst_glsl_version_profile_to_string (version, profile);
1772 version_str = g_strdup_printf ("#version %s\n", tmp1);
1773 g_free (tmp1);
1774 strings[0] = version_str;
1775
1776 strings[1] = tmp;
1777 vert =
1778 gst_glsl_stage_new_with_strings (viewconvert->context,
1779 GL_VERTEX_SHADER, version, profile, 2, strings);
1780 g_free (tmp);
1781
1782 if (!gst_gl_shader_compile_attach_stage (viewconvert->shader, vert, &error)) {
1783 GST_ERROR_OBJECT (viewconvert, "Failed to compile vertex stage %s",
1784 error->message);
1785 gst_object_unref (viewconvert->shader);
1786 viewconvert->shader = NULL;
1787 g_free (version_str);
1788 goto error;
1789 }
1790
1791 fragment_source_str = _get_shader_string (viewconvert, viewconvert->shader,
1792 in_mode, out_mode, version, profile);
1793 strings[1] = fragment_source_str;
1794
1795 frag =
1796 gst_glsl_stage_new_with_strings (viewconvert->context,
1797 GL_FRAGMENT_SHADER, version, profile, 2, strings);
1798 g_free (version_str);
1799
1800 if (!gst_gl_shader_compile_attach_stage (viewconvert->shader, frag, &error)) {
1801 GST_ERROR_OBJECT (viewconvert, "Failed to compile fragment stage %s",
1802 error->message);
1803 g_free (fragment_source_str);
1804 gst_object_unref (viewconvert->shader);
1805 viewconvert->shader = NULL;
1806 goto error;
1807 }
1808 g_free (fragment_source_str);
1809
1810 if (!gst_gl_shader_link (viewconvert->shader, &error)) {
1811 GST_ERROR_OBJECT (viewconvert, "Failed to link conversion shader %s",
1812 error->message);
1813 gst_object_unref (viewconvert->shader);
1814 viewconvert->shader = NULL;
1815 goto error;
1816 }
1817 }
1818
1819 viewconvert->priv->attr_position =
1820 gst_gl_shader_get_attribute_location (viewconvert->shader, "a_position");
1821 viewconvert->priv->attr_texture =
1822 gst_gl_shader_get_attribute_location (viewconvert->shader, "a_texcoord");
1823 gst_gl_shader_use (viewconvert->shader);
1824 gst_gl_shader_set_uniform_2fv (viewconvert->shader, "tex_scale",
1825 2, tex_scale[0]);
1826 gst_gl_shader_set_uniform_2fv (viewconvert->shader, "offsets", 2, offsets[0]);
1827 gst_gl_shader_set_uniform_1f (viewconvert->shader, "width",
1828 GST_VIDEO_INFO_WIDTH (&viewconvert->out_info));
1829 gst_gl_shader_set_uniform_1f (viewconvert->shader, "height",
1830 GST_VIDEO_INFO_HEIGHT (&viewconvert->out_info));
1831 gst_gl_shader_set_uniform_matrix_3fv (viewconvert->shader, "downmix",
1832 2, FALSE, &downmix_matrices[viewconvert->downmix_mode][0][0]);
1833 gst_gl_shader_set_uniform_matrix_4fv (viewconvert->shader, "u_transformation",
1834 1, FALSE, identity_matrix);
1835 if (in_mode == GST_VIDEO_MULTIVIEW_MODE_SEPARATED ||
1836 in_mode == GST_VIDEO_MULTIVIEW_MODE_FRAME_BY_FRAME) {
1837 gst_gl_shader_set_uniform_1i (viewconvert->shader, "tex_l", l_index);
1838 gst_gl_shader_set_uniform_1i (viewconvert->shader, "tex_r", r_index);
1839 } else {
1840 gst_gl_shader_set_uniform_1i (viewconvert->shader, "tex_l", 0);
1841 gst_gl_shader_set_uniform_1i (viewconvert->shader, "tex_r", 0);
1842 }
1843 gst_gl_context_clear_shader (viewconvert->context);
1844 if (!_init_view_convert_fbo (viewconvert)) {
1845 goto error;
1846 }
1847
1848 if (!viewconvert->priv->vertex_buffer) {
1849 if (gl->GenVertexArrays) {
1850 gl->GenVertexArrays (1, &viewconvert->priv->vao);
1851 gl->BindVertexArray (viewconvert->priv->vao);
1852 }
1853
1854 gl->GenBuffers (1, &viewconvert->priv->vertex_buffer);
1855 gl->BindBuffer (GL_ARRAY_BUFFER, viewconvert->priv->vertex_buffer);
1856 gl->BufferData (GL_ARRAY_BUFFER, 4 * 5 * sizeof (GLfloat), vertices,
1857 GL_STATIC_DRAW);
1858 gl->GenBuffers (1, &viewconvert->priv->vbo_indices);
1859 gl->BindBuffer (GL_ELEMENT_ARRAY_BUFFER, viewconvert->priv->vbo_indices);
1860 gl->BufferData (GL_ELEMENT_ARRAY_BUFFER, sizeof (indices), indices,
1861 GL_STATIC_DRAW);
1862 if (gl->GenVertexArrays) {
1863 _bind_buffer (viewconvert);
1864 gl->BindVertexArray (0);
1865 }
1866
1867 gl->BindBuffer (GL_ARRAY_BUFFER, 0);
1868 gl->BindBuffer (GL_ELEMENT_ARRAY_BUFFER, 0);
1869 }
1870
1871 viewconvert->initted = TRUE;
1872 return TRUE;
1873 error:
1874 return FALSE;
1875 }
1876
1877 static gboolean
_do_view_convert_draw(GstGLContext * context,GstGLViewConvert * viewconvert)1878 _do_view_convert_draw (GstGLContext * context, GstGLViewConvert * viewconvert)
1879 {
1880 GstGLViewConvertPrivate *priv = viewconvert->priv;
1881 GstGLFuncs *gl;
1882 guint out_width, out_height;
1883 gint out_views, i;
1884 GLenum multipleRT[] = {
1885 GL_COLOR_ATTACHMENT0,
1886 GL_COLOR_ATTACHMENT1,
1887 GL_COLOR_ATTACHMENT2
1888 };
1889 GstVideoMultiviewMode in_mode = priv->input_mode;
1890 GstVideoMultiviewMode out_mode = priv->output_mode;
1891 guint from_gl_target =
1892 gst_gl_texture_target_to_gl (viewconvert->from_texture_target);
1893
1894 gl = context->gl_vtable;
1895
1896 gst_gl_framebuffer_bind (viewconvert->fbo);
1897
1898 if (out_mode == GST_VIDEO_MULTIVIEW_MODE_SEPARATED ||
1899 out_mode == GST_VIDEO_MULTIVIEW_MODE_FRAME_BY_FRAME) {
1900 out_views = viewconvert->out_info.views;
1901 } else {
1902 out_views = 1;
1903 }
1904
1905 /* attach the texture to the FBO to renderer to */
1906 for (i = 0; i < out_views; i++) {
1907 GstGLBaseMemory *tex = (GstGLBaseMemory *) priv->out_tex[i];
1908
1909 gst_gl_framebuffer_attach (viewconvert->fbo, GL_COLOR_ATTACHMENT0 + i, tex);
1910 }
1911
1912 if (gl->DrawBuffers)
1913 gl->DrawBuffers (out_views, multipleRT);
1914 else if (gl->DrawBuffer)
1915 gl->DrawBuffer (GL_COLOR_ATTACHMENT0);
1916
1917 gst_gl_framebuffer_get_effective_dimensions (viewconvert->fbo, &out_width,
1918 &out_height);
1919 gl->Viewport (0, 0, out_width, out_height);
1920
1921 gst_gl_shader_use (viewconvert->shader);
1922
1923 /* FIXME: the auxiliary buffer could have a different transform matrix */
1924 {
1925 GstVideoAffineTransformationMeta *af_meta;
1926 gfloat matrix[16];
1927
1928 af_meta =
1929 gst_buffer_get_video_affine_transformation_meta (priv->primary_in);
1930 gst_gl_get_affine_transformation_meta_as_ndc (af_meta, matrix);
1931 gst_gl_shader_set_uniform_matrix_4fv (viewconvert->shader,
1932 "u_transformation", 1, FALSE, matrix);
1933 }
1934
1935 if (gl->BindVertexArray)
1936 gl->BindVertexArray (priv->vao);
1937 _bind_buffer (viewconvert);
1938
1939 if (in_mode == GST_VIDEO_MULTIVIEW_MODE_SEPARATED ||
1940 in_mode == GST_VIDEO_MULTIVIEW_MODE_FRAME_BY_FRAME) {
1941 if (priv->in_tex[1] == NULL) {
1942 GST_ERROR_OBJECT (viewconvert,
1943 "No 2nd view available during conversion!");
1944 return FALSE;
1945 }
1946 gl->ActiveTexture (GL_TEXTURE1);
1947 gl->BindTexture (from_gl_target, priv->in_tex[1]->tex_id);
1948 }
1949
1950 gl->ActiveTexture (GL_TEXTURE0);
1951 gl->BindTexture (from_gl_target, priv->in_tex[0]->tex_id);
1952
1953 gl->ClearColor (0.0, 0.0, 0.0, 1.0);
1954 gl->Clear (GL_COLOR_BUFFER_BIT);
1955
1956 gl->DrawElements (GL_TRIANGLES, 6, GL_UNSIGNED_SHORT, NULL);
1957
1958 if (gl->BindVertexArray)
1959 gl->BindVertexArray (0);
1960 else
1961 _unbind_buffer (viewconvert);
1962 if (gl->DrawBuffer)
1963 gl->DrawBuffer (GL_COLOR_ATTACHMENT0);
1964 /* we are done with the shader */
1965 gst_gl_context_clear_shader (context);
1966 gst_gl_context_clear_framebuffer (context);
1967
1968 return TRUE;
1969 }
1970
1971 static gboolean
_gen_buffer(GstGLViewConvert * viewconvert,GstBuffer ** target)1972 _gen_buffer (GstGLViewConvert * viewconvert, GstBuffer ** target)
1973 {
1974 GstGLVideoAllocationParams *params;
1975 GstGLMemoryAllocator *mem_allocator;
1976 GstAllocator *allocator;
1977 GstVideoMeta *meta;
1978
1979 *target = gst_buffer_new ();
1980
1981 allocator =
1982 GST_ALLOCATOR (gst_gl_memory_allocator_get_default
1983 (viewconvert->context));
1984 mem_allocator = GST_GL_MEMORY_ALLOCATOR (allocator);
1985 params = gst_gl_video_allocation_params_new (viewconvert->context, NULL,
1986 &viewconvert->out_info, 0, NULL, viewconvert->to_texture_target, 0);
1987
1988 if (!gst_gl_memory_setup_buffer (mem_allocator, *target, params, NULL, NULL,
1989 0)) {
1990 gst_gl_allocation_params_free ((GstGLAllocationParams *) params);
1991 gst_object_unref (allocator);
1992 return FALSE;
1993 }
1994 gst_object_unref (allocator);
1995
1996 meta = gst_buffer_add_video_meta_full (*target, 0,
1997 GST_VIDEO_INFO_FORMAT (&viewconvert->out_info),
1998 GST_VIDEO_INFO_WIDTH (&viewconvert->out_info),
1999 GST_VIDEO_INFO_HEIGHT (&viewconvert->out_info),
2000 GST_VIDEO_INFO_N_PLANES (&viewconvert->out_info),
2001 viewconvert->out_info.offset, viewconvert->out_info.stride);
2002
2003 if (params->valign)
2004 gst_video_meta_set_alignment (meta, *params->valign);
2005
2006 gst_gl_allocation_params_free ((GstGLAllocationParams *) params);
2007
2008 return TRUE;
2009 }
2010
2011 static void
_do_view_convert(GstGLContext * context,GstGLViewConvert * viewconvert)2012 _do_view_convert (GstGLContext * context, GstGLViewConvert * viewconvert)
2013 {
2014 GstGLViewConvertPrivate *priv = viewconvert->priv;
2015 guint in_width, in_height, out_width, out_height;
2016 GstMapInfo out_info[GST_VIDEO_MAX_PLANES], in_info[GST_VIDEO_MAX_PLANES];
2017 GstGLMemory *dest_tex[GST_VIDEO_MAX_PLANES];
2018 gboolean res = TRUE;
2019 gint i = 0, j = 0;
2020 gint in_views, out_views;
2021 GstVideoMultiviewMode in_mode;
2022 GstVideoMultiviewMode out_mode;
2023 GstGLSyncMeta *sync_meta;
2024
2025 out_width = GST_VIDEO_INFO_WIDTH (&viewconvert->out_info);
2026 out_height = GST_VIDEO_INFO_HEIGHT (&viewconvert->out_info);
2027 in_width = GST_VIDEO_INFO_WIDTH (&viewconvert->in_info);
2028 in_height = GST_VIDEO_INFO_HEIGHT (&viewconvert->in_info);
2029
2030 g_return_if_fail (priv->primary_out == NULL);
2031 g_return_if_fail (priv->auxilliary_out == NULL);
2032
2033 in_mode = priv->input_mode;
2034 out_mode = priv->output_mode;
2035
2036 if (in_mode == GST_VIDEO_MULTIVIEW_MODE_SEPARATED ||
2037 in_mode == GST_VIDEO_MULTIVIEW_MODE_FRAME_BY_FRAME)
2038 in_views = viewconvert->in_info.views;
2039 else
2040 in_views = 1;
2041
2042 if (out_mode == GST_VIDEO_MULTIVIEW_MODE_SEPARATED ||
2043 out_mode == GST_VIDEO_MULTIVIEW_MODE_FRAME_BY_FRAME)
2044 out_views = viewconvert->out_info.views;
2045 else
2046 out_views = 1;
2047
2048 if (!_init_view_convert (viewconvert)) {
2049 priv->result = FALSE;
2050 return;
2051 }
2052
2053 if (!_gen_buffer (viewconvert, &priv->primary_out)) {
2054 GST_ERROR_OBJECT (viewconvert,
2055 "Failed to setup memory for primary output buffer");
2056 priv->result = FALSE;
2057 return;
2058 }
2059
2060 if (out_mode == GST_VIDEO_MULTIVIEW_MODE_FRAME_BY_FRAME) {
2061 if (!_gen_buffer (viewconvert, &priv->auxilliary_out)) {
2062 GST_ERROR_OBJECT (viewconvert,
2063 "Failed to setup memory for second view output buffer");
2064 priv->result = FALSE;
2065 return;
2066 }
2067 }
2068
2069 for (i = 0; i < in_views; i++) {
2070 if (in_mode == GST_VIDEO_MULTIVIEW_MODE_FRAME_BY_FRAME && i > 0) {
2071 priv->in_tex[i] =
2072 (GstGLMemory *) gst_buffer_peek_memory (priv->auxilliary_in, 0);
2073 } else {
2074 priv->in_tex[i] =
2075 (GstGLMemory *) gst_buffer_peek_memory (priv->primary_in, i);
2076 }
2077 if (!gst_is_gl_memory ((GstMemory *) priv->in_tex[i])) {
2078 GST_ERROR_OBJECT (viewconvert, "input must be GstGLMemory");
2079 res = FALSE;
2080 goto out;
2081 }
2082 if (!gst_memory_map ((GstMemory *) priv->in_tex[i],
2083 &in_info[i], GST_MAP_READ | GST_MAP_GL)) {
2084 GST_ERROR_OBJECT (viewconvert, "failed to map input memory %p",
2085 priv->in_tex[i]);
2086 res = FALSE;
2087 goto out;
2088 }
2089 }
2090
2091 for (j = 0; j < out_views; j++) {
2092 GstGLMemory *out_tex;
2093 guint width, height;
2094 GstVideoInfo temp_info;
2095
2096 if (j > 0 && out_mode == GST_VIDEO_MULTIVIEW_MODE_FRAME_BY_FRAME) {
2097 dest_tex[j] = out_tex =
2098 (GstGLMemory *) gst_buffer_peek_memory (priv->auxilliary_out, 0);
2099 } else {
2100 dest_tex[j] = out_tex =
2101 (GstGLMemory *) gst_buffer_peek_memory (priv->primary_out, j);
2102 }
2103
2104 if (!gst_is_gl_memory ((GstMemory *) out_tex)) {
2105 GST_ERROR_OBJECT (viewconvert, "output must be GstGLMemory");
2106 res = FALSE;
2107 goto out;
2108 }
2109
2110 width = gst_gl_memory_get_texture_width (out_tex);
2111 height = gst_gl_memory_get_texture_height (out_tex);
2112 gst_video_info_set_format (&temp_info, GST_VIDEO_FORMAT_RGBA, width,
2113 height);
2114 if (out_tex->tex_format == GST_GL_LUMINANCE
2115 || out_tex->tex_format == GST_GL_LUMINANCE_ALPHA
2116 || out_width != width || out_height != height) {
2117 /* Luminance formats are not color renderable */
2118 /* rendering to a framebuffer only renders the intersection of all
2119 * the attachments i.e. the smallest attachment size */
2120 if (!priv->out_tex[j]) {
2121 GstGLVideoAllocationParams *params;
2122 GstGLBaseMemoryAllocator *base_mem_allocator;
2123 GstAllocator *allocator;
2124 GstVideoInfo temp_info;
2125
2126 gst_video_info_set_format (&temp_info, GST_VIDEO_FORMAT_RGBA, out_width,
2127 out_height);
2128
2129 allocator =
2130 GST_ALLOCATOR (gst_gl_memory_allocator_get_default (context));
2131 base_mem_allocator = GST_GL_BASE_MEMORY_ALLOCATOR (allocator);
2132 params = gst_gl_video_allocation_params_new (context, NULL, &temp_info,
2133 0, NULL, viewconvert->to_texture_target, GST_GL_RGBA);
2134
2135 priv->out_tex[j] =
2136 (GstGLMemory *) gst_gl_base_memory_alloc (base_mem_allocator,
2137 (GstGLAllocationParams *) params);
2138
2139 gst_gl_allocation_params_free ((GstGLAllocationParams *) params);
2140 gst_object_unref (allocator);
2141 }
2142 } else {
2143 priv->out_tex[j] = out_tex;
2144 }
2145
2146 if (!gst_memory_map ((GstMemory *) priv->out_tex[j],
2147 &out_info[j], GST_MAP_WRITE | GST_MAP_GL)) {
2148 GST_ERROR_OBJECT (viewconvert, "failed to map output memory %p",
2149 priv->out_tex[i]);
2150 res = FALSE;
2151 goto out;
2152 }
2153 }
2154 priv->n_out_tex = out_views;
2155
2156 if (priv->primary_in) {
2157 if ((sync_meta = gst_buffer_get_gl_sync_meta (priv->primary_in))) {
2158 gst_gl_sync_meta_wait (sync_meta, context);
2159 }
2160 }
2161
2162 if (priv->auxilliary_in) {
2163 if ((sync_meta = gst_buffer_get_gl_sync_meta (priv->auxilliary_in))) {
2164 gst_gl_sync_meta_wait (sync_meta, context);
2165 }
2166 }
2167
2168 GST_LOG_OBJECT (viewconvert, "multiview splitting to textures:%p,%p,%p,%p "
2169 "dimensions:%ux%u, from textures:%p,%p,%p,%p dimensions:%ux%u",
2170 priv->out_tex[0], priv->out_tex[1],
2171 priv->out_tex[2], priv->out_tex[3],
2172 out_width, out_height, priv->in_tex[0],
2173 priv->in_tex[1], priv->in_tex[2], priv->in_tex[3], in_width, in_height);
2174
2175 if (!_do_view_convert_draw (context, viewconvert))
2176 res = FALSE;
2177 out:
2178 for (j--; j >= 0; j--) {
2179 GstGLMemory *out_tex;
2180 guint width, height;
2181
2182 out_tex = dest_tex[j];
2183
2184 width = gst_gl_memory_get_texture_width (out_tex);
2185 height = gst_gl_memory_get_texture_height (out_tex);
2186
2187 gst_memory_unmap ((GstMemory *) priv->out_tex[j], &out_info[j]);
2188 if (out_tex != priv->out_tex[j]) {
2189 GstMapInfo to_info, from_info;
2190 if (!gst_memory_map ((GstMemory *) priv->out_tex[j],
2191 &from_info, GST_MAP_READ | GST_MAP_GL)) {
2192 GST_ERROR_OBJECT (viewconvert, "Failed to map intermediate memory");
2193 res = FALSE;
2194 continue;
2195 }
2196 if (!gst_memory_map ((GstMemory *) out_tex, &to_info,
2197 GST_MAP_WRITE | GST_MAP_GL)) {
2198 GST_ERROR_OBJECT (viewconvert, "Failed to map intermediate memory");
2199 res = FALSE;
2200 continue;
2201 }
2202 gst_gl_memory_copy_into (priv->out_tex[j], out_tex->tex_id,
2203 viewconvert->to_texture_target, out_tex->tex_format, width, height);
2204 gst_memory_unmap ((GstMemory *) out_tex, &to_info);
2205 }
2206
2207 priv->out_tex[j] = NULL;
2208 }
2209
2210 for (i--; i >= 0; i--) {
2211 gst_memory_unmap ((GstMemory *) priv->in_tex[i], &in_info[i]);
2212 }
2213
2214 if (!res) {
2215 gst_buffer_replace (&priv->primary_out, NULL);
2216 gst_buffer_replace (&priv->auxilliary_out, NULL);
2217 }
2218
2219 if (priv->primary_out) {
2220 if ((sync_meta = gst_buffer_add_gl_sync_meta (context, priv->primary_out)))
2221 gst_gl_sync_meta_set_sync_point (sync_meta, context);
2222 }
2223
2224 if (priv->auxilliary_out) {
2225 if ((sync_meta =
2226 gst_buffer_add_gl_sync_meta (context, priv->auxilliary_out)))
2227 gst_gl_sync_meta_set_sync_point (sync_meta, context);
2228 }
2229
2230 priv->result = res;
2231 return;
2232 }
2233
2234 /**
2235 * gst_gl_view_convert_submit_input_buffer:
2236 * @viewconvert: a #GstGLViewConvert
2237 * @is_discont: true if we have a discontinuity
2238 * @input: (transfer full): a #GstBuffer
2239 *
2240 * Submit @input to be processed by @viewconvert
2241 *
2242 * Returns: a #GstFlowReturn
2243 *
2244 * Since: 1.6
2245 */
2246 GstFlowReturn
gst_gl_view_convert_submit_input_buffer(GstGLViewConvert * viewconvert,gboolean is_discont,GstBuffer * input)2247 gst_gl_view_convert_submit_input_buffer (GstGLViewConvert * viewconvert,
2248 gboolean is_discont, GstBuffer * input)
2249 {
2250 GstFlowReturn ret = GST_FLOW_OK;
2251 GstVideoMultiviewMode mode;
2252 GstBuffer **target;
2253
2254 if (is_discont) {
2255 gst_buffer_replace (&viewconvert->priv->primary_in, NULL);
2256 gst_buffer_replace (&viewconvert->priv->auxilliary_in, NULL);
2257 }
2258
2259 mode = viewconvert->input_mode_override;
2260 if (mode == GST_VIDEO_MULTIVIEW_MODE_NONE)
2261 mode = GST_VIDEO_INFO_MULTIVIEW_MODE (&viewconvert->in_info);
2262
2263 target = &viewconvert->priv->primary_in;
2264
2265 /* For frame-by-frame mode, we need to collect the 2nd eye into
2266 * our auxiliary buffer */
2267 if (mode == GST_VIDEO_MULTIVIEW_MODE_FRAME_BY_FRAME) {
2268 if (!GST_BUFFER_FLAG_IS_SET (input, GST_VIDEO_BUFFER_FLAG_FIRST_IN_BUNDLE))
2269 target = &viewconvert->priv->auxilliary_in;
2270 }
2271
2272 if (*target)
2273 gst_buffer_unref (*target);
2274 *target = input;
2275
2276 return ret;
2277 }
2278
2279 /**
2280 * gst_gl_view_convert_get_output:
2281 * @viewconvert: a #GstGLViewConvert
2282 * @outbuf_ptr: (out): a #GstBuffer
2283 *
2284 * Retrieve the processed output buffer placing the output in @outbuf_ptr.
2285 *
2286 * Returns: a #GstFlowReturn
2287 *
2288 * Since: 1.6
2289 */
2290 GstFlowReturn
gst_gl_view_convert_get_output(GstGLViewConvert * viewconvert,GstBuffer ** outbuf_ptr)2291 gst_gl_view_convert_get_output (GstGLViewConvert * viewconvert,
2292 GstBuffer ** outbuf_ptr)
2293 {
2294 GstGLViewConvertPrivate *priv = viewconvert->priv;
2295 GstBuffer *outbuf = NULL;
2296 GstFlowReturn ret = GST_FLOW_OK;
2297 GstVideoMultiviewMode in_mode, out_mode;
2298 GstVideoMultiviewFlags in_flags, out_flags;
2299
2300 g_return_val_if_fail (GST_IS_GL_VIEW_CONVERT (viewconvert), GST_FLOW_ERROR);
2301 g_return_val_if_fail (GST_IS_GL_CONTEXT (viewconvert->context),
2302 GST_FLOW_ERROR);
2303
2304 GST_OBJECT_LOCK (viewconvert);
2305
2306 /* See if a buffer is available already */
2307 if (priv->primary_out) {
2308 outbuf = viewconvert->priv->primary_out;
2309 priv->primary_out = NULL;
2310 goto done;
2311 }
2312 if (viewconvert->priv->auxilliary_out) {
2313 outbuf = priv->auxilliary_out;
2314 priv->auxilliary_out = NULL;
2315 goto done;
2316 }
2317
2318 /* Check prereqs before processing a new input buffer */
2319 if (priv->primary_in == NULL)
2320 goto done;
2321
2322 in_mode = viewconvert->input_mode_override;
2323 in_flags = viewconvert->input_flags_override;
2324 if (in_mode == GST_VIDEO_MULTIVIEW_MODE_NONE) {
2325 in_mode = GST_VIDEO_INFO_MULTIVIEW_MODE (&viewconvert->in_info);
2326 in_flags = GST_VIDEO_INFO_MULTIVIEW_FLAGS (&viewconvert->in_info);
2327 }
2328
2329 /* Configured output mode already takes any override
2330 * into account */
2331 out_mode = GST_VIDEO_INFO_MULTIVIEW_MODE (&viewconvert->out_info);
2332 out_flags = GST_VIDEO_INFO_MULTIVIEW_FLAGS (&viewconvert->out_info);
2333
2334 if (in_mode == GST_VIDEO_MULTIVIEW_MODE_FRAME_BY_FRAME) {
2335 /* For frame-by-frame, we need 2 input buffers */
2336 if (priv->auxilliary_in == NULL) {
2337 GST_LOG_OBJECT (viewconvert,
2338 "Can't generate output yet - frame-by-frame mode");
2339 goto done;
2340 }
2341 }
2342
2343 /* Store the current conversion in the priv vars */
2344 priv->input_mode = in_mode;
2345 priv->input_flags = in_flags;
2346 priv->output_mode = out_mode;
2347 priv->output_flags = out_flags;
2348
2349 if (priv->input_mode == priv->output_mode &&
2350 priv->input_flags == priv->output_flags &&
2351 viewconvert->in_info.width == viewconvert->out_info.width &&
2352 viewconvert->in_info.height == viewconvert->out_info.height &&
2353 viewconvert->from_texture_target == viewconvert->to_texture_target) {
2354 /* passthrough - just pass input buffers */
2355 outbuf = gst_buffer_ref (priv->primary_in);
2356 if (in_mode == GST_VIDEO_MULTIVIEW_MODE_FRAME_BY_FRAME)
2357 priv->auxilliary_out = gst_buffer_ref (priv->auxilliary_in);
2358 goto done_clear_input;
2359 }
2360
2361 /* We can't output to OES textures, they're only supported for passthrough */
2362 if (viewconvert->to_texture_target == GST_GL_TEXTURE_TARGET_EXTERNAL_OES) {
2363 ret = GST_FLOW_ERROR;
2364 goto done_clear_input;
2365 }
2366
2367 /* Generate new output buffer(s) */
2368 gst_gl_context_thread_add (viewconvert->context,
2369 (GstGLContextThreadFunc) _do_view_convert, viewconvert);
2370
2371 if (!priv->result) {
2372 if (priv->primary_out)
2373 gst_object_unref (priv->primary_out);
2374 if (priv->auxilliary_out)
2375 gst_object_unref (priv->auxilliary_out);
2376 priv->primary_out = NULL;
2377 priv->auxilliary_out = NULL;
2378 ret = GST_FLOW_ERROR;
2379 goto done_clear_input;
2380 }
2381
2382 outbuf = priv->primary_out;
2383 if (outbuf) {
2384 GstVideoOverlayCompositionMeta *composition_meta;
2385
2386 gst_buffer_copy_into (outbuf, priv->primary_in,
2387 GST_BUFFER_COPY_FLAGS | GST_BUFFER_COPY_TIMESTAMPS, 0, -1);
2388 GST_BUFFER_FLAG_SET (outbuf,
2389 GST_VIDEO_BUFFER_FLAG_FIRST_IN_BUNDLE |
2390 GST_VIDEO_BUFFER_FLAG_MULTIPLE_VIEW);
2391
2392 composition_meta =
2393 gst_buffer_get_video_overlay_composition_meta (priv->primary_in);
2394 if (composition_meta) {
2395 GST_DEBUG ("found video overlay composition meta, applying on output.");
2396 gst_buffer_add_video_overlay_composition_meta
2397 (outbuf, composition_meta->overlay);
2398 }
2399 }
2400
2401 if (priv->auxilliary_out) {
2402 GstVideoOverlayCompositionMeta *composition_meta;
2403
2404 gst_buffer_copy_into (priv->auxilliary_out,
2405 priv->primary_out, GST_BUFFER_COPY_FLAGS, 0, -1);
2406 GST_BUFFER_FLAG_UNSET (priv->auxilliary_out,
2407 GST_VIDEO_BUFFER_FLAG_FIRST_IN_BUNDLE);
2408
2409 composition_meta =
2410 gst_buffer_get_video_overlay_composition_meta (priv->primary_out);
2411 if (composition_meta) {
2412 GST_DEBUG ("found video overlay composition meta, applying on output.");
2413 gst_buffer_add_video_overlay_composition_meta
2414 (priv->auxilliary_out, composition_meta->overlay);
2415 }
2416 }
2417 priv->primary_out = NULL;
2418
2419 done_clear_input:
2420 /* Invalidate input buffers now they've been used */
2421 gst_buffer_replace (&priv->primary_in, NULL);
2422 gst_buffer_replace (&priv->auxilliary_in, NULL);
2423
2424 done:
2425 GST_OBJECT_UNLOCK (viewconvert);
2426 *outbuf_ptr = outbuf;
2427 return ret;
2428 }
2429
2430 #ifndef GST_REMOVE_DEPRECATED
2431 #ifdef GST_DISABLE_DEPRECATED
2432 GST_GL_API GType gst_gl_stereo_downmix_mode_get_type (void);
2433 #endif
2434
2435 GType
gst_gl_stereo_downmix_mode_get_type(void)2436 gst_gl_stereo_downmix_mode_get_type (void)
2437 {
2438 return gst_gl_stereo_downmix_get_type ();
2439 }
2440 #endif
2441