1 /*
2 * GStreamer
3 * Copyright (C) 2004, 2008 Wim Taymans <wim@fluendo.com>
4 * Copyright (C) 2010 Sebastian Dröge <sebastian.droege@collabora.co.uk>
5 * Copyright (C) 2014 Mathieu Duponchelle <mathieu.duponchelle@opencreed.com>
6 * Copyright (C) 2014 Thibault Saunier <tsaunier@gnome.org>
7 * Copyright (C) 2020 Seungha Yang <seungha@centricular.com>
8 *
9 * This library is free software; you can redistribute it and/or
10 * modify it under the terms of the GNU Library General Public
11 * License as published by the Free Software Foundation; either
12 * version 2 of the License, or (at your option) any later version.
13 *
14 * This library is distributed in the hope that it will be useful,
15 * but WITHOUT ANY WARRANTY; without even the implied warranty of
16 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
17 * Library General Public License for more details.
18 *
19 * You should have received a copy of the GNU Library General Public
20 * License along with this library; if not, write to the
21 * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
22 * Boston, MA 02110-1301, USA.
23 */
24
25 /**
26 * SECTION:element-d3d11compositorelement
27 * @title: d3d11compositorelement
28 *
29 * A Direct3D11 based video compositing element.
30 *
31 * Since: 1.20
32 *
33 */
34
35 #ifdef HAVE_CONFIG_H
36 #include "config.h"
37 #endif
38
39 #include "gstd3d11compositor.h"
40 #include "gstd3d11converter.h"
41 #include "gstd3d11shader.h"
42 #include "gstd3d11pluginutils.h"
43 #include <string.h>
44 #include <wrl.h>
45
46 GST_DEBUG_CATEGORY_EXTERN (gst_d3d11_compositor_debug);
47 #define GST_CAT_DEFAULT gst_d3d11_compositor_debug
48
49 /* *INDENT-OFF* */
50 using namespace Microsoft::WRL;
51 /* *INDENT-ON* */
52
53 /**
54 * GstD3D11CompositorBlendOperation:
55 * @GST_D3D11_COMPOSITOR_BLEND_OP_ADD:
56 * Add source 1 and source 2
57 * @GST_D3D11_COMPOSITOR_BLEND_OP_SUBTRACT:
58 * Subtract source 1 from source 2
59 * @GST_D3D11_COMPOSITOR_BLEND_OP_REV_SUBTRACT:
60 * Subtract source 2 from source 1
61 * @GST_D3D11_COMPOSITOR_BLEND_OP_MIN:
62 * Find the minimum of source 1 and source 2
63 * @GST_D3D11_COMPOSITOR_BLEND_OP_MAX:
64 * Find the maximum of source 1 and source 2
65 *
66 * Since: 1.20
67 */
68 GType
gst_d3d11_compositor_blend_operation_get_type(void)69 gst_d3d11_compositor_blend_operation_get_type (void)
70 {
71 static GType blend_operation_type = 0;
72
73 static const GEnumValue blend_operator[] = {
74 {GST_D3D11_COMPOSITOR_BLEND_OP_ADD, "Add source and background",
75 "add"},
76 {GST_D3D11_COMPOSITOR_BLEND_OP_SUBTRACT,
77 "Subtract source from background",
78 "subtract"},
79 {GST_D3D11_COMPOSITOR_BLEND_OP_REV_SUBTRACT,
80 "Subtract background from source",
81 "rev-subtract"},
82 {GST_D3D11_COMPOSITOR_BLEND_OP_MIN,
83 "Minimum of source and background", "min"},
84 {GST_D3D11_COMPOSITOR_BLEND_OP_MAX,
85 "Maximum of source and background", "max"},
86 {0, NULL, NULL},
87 };
88
89 if (!blend_operation_type) {
90 blend_operation_type =
91 g_enum_register_static ("GstD3D11CompositorBlendOperation",
92 blend_operator);
93 }
94 return blend_operation_type;
95 }
96
97 static GstD3D11CompositorBlendOperation
gst_d3d11_compositor_blend_operation_from_native(D3D11_BLEND_OP blend_op)98 gst_d3d11_compositor_blend_operation_from_native (D3D11_BLEND_OP blend_op)
99 {
100 switch (blend_op) {
101 case D3D11_BLEND_OP_ADD:
102 return GST_D3D11_COMPOSITOR_BLEND_OP_ADD;
103 case D3D11_BLEND_OP_SUBTRACT:
104 return GST_D3D11_COMPOSITOR_BLEND_OP_SUBTRACT;
105 case D3D11_BLEND_OP_REV_SUBTRACT:
106 return GST_D3D11_COMPOSITOR_BLEND_OP_REV_SUBTRACT;
107 case D3D11_BLEND_OP_MIN:
108 return GST_D3D11_COMPOSITOR_BLEND_OP_MIN;
109 case D3D11_BLEND_OP_MAX:
110 return GST_D3D11_COMPOSITOR_BLEND_OP_MAX;
111 default:
112 g_assert_not_reached ();
113 break;
114 }
115
116 return GST_D3D11_COMPOSITOR_BLEND_OP_ADD;
117 }
118
119 static D3D11_BLEND_OP
gst_d3d11_compositor_blend_operation_to_native(GstD3D11CompositorBlendOperation op)120 gst_d3d11_compositor_blend_operation_to_native (GstD3D11CompositorBlendOperation
121 op)
122 {
123 switch (op) {
124 case GST_D3D11_COMPOSITOR_BLEND_OP_ADD:
125 return D3D11_BLEND_OP_ADD;
126 case GST_D3D11_COMPOSITOR_BLEND_OP_SUBTRACT:
127 return D3D11_BLEND_OP_SUBTRACT;
128 case GST_D3D11_COMPOSITOR_BLEND_OP_REV_SUBTRACT:
129 return D3D11_BLEND_OP_REV_SUBTRACT;
130 case GST_D3D11_COMPOSITOR_BLEND_OP_MIN:
131 return D3D11_BLEND_OP_MIN;
132 case GST_D3D11_COMPOSITOR_BLEND_OP_MAX:
133 return D3D11_BLEND_OP_MAX;
134 default:
135 g_assert_not_reached ();
136 break;
137 }
138
139 return D3D11_BLEND_OP_ADD;
140 }
141
142 /**
143 * GstD3D11CompositorBlend:
144 * @GST_D3D11_COMPOSITOR_BLEND_ZERO:
145 * The blend factor is (0, 0, 0, 0). No pre-blend operation.
146 * @GST_D3D11_COMPOSITOR_BLEND_ONE:
147 * The blend factor is (1, 1, 1, 1). No pre-blend operation.
148 * @GST_D3D11_COMPOSITOR_BLEND_SRC_COLOR:
149 * The blend factor is (Rs, Gs, Bs, As),
150 * that is color data (RGB) from a pixel shader. No pre-blend operation.
151 * @GST_D3D11_COMPOSITOR_BLEND_INV_SRC_COLOR:
152 * The blend factor is (1 - Rs, 1 - Gs, 1 - Bs, 1 - As),
153 * that is color data (RGB) from a pixel shader.
154 * The pre-blend operation inverts the data, generating 1 - RGB.
155 * @GST_D3D11_COMPOSITOR_BLEND_SRC_ALPHA:
156 * The blend factor is (As, As, As, As),
157 * that is alpha data (A) from a pixel shader. No pre-blend operation.
158 * @GST_D3D11_COMPOSITOR_BLEND_INV_SRC_ALPHA:
159 * The blend factor is ( 1 - As, 1 - As, 1 - As, 1 - As),
160 * that is alpha data (A) from a pixel shader.
161 * The pre-blend operation inverts the data, generating 1 - A.
162 * @GST_D3D11_COMPOSITOR_BLEND_DEST_ALPHA:
163 * The blend factor is (Ad, Ad, Ad, Ad),
164 * that is alpha data from a render target. No pre-blend operation.
165 * @GST_D3D11_COMPOSITOR_BLEND_INV_DEST_ALPHA:
166 * The blend factor is (1 - Ad, 1 - Ad, 1 - Ad, 1 - Ad),
167 * that is alpha data from a render target.
168 * The pre-blend operation inverts the data, generating 1 - A.
169 * @GST_D3D11_COMPOSITOR_BLEND_DEST_COLOR:
170 * The blend factor is (Rd, Gd, Bd, Ad),
171 * that is color data from a render target. No pre-blend operation.
172 * @GST_D3D11_COMPOSITOR_BLEND_INV_DEST_COLOR:
173 * The blend factor is (1 - Rd, 1 - Gd, 1 - Bd, 1 - Ad),
174 * that is color data from a render target.
175 * The pre-blend operation inverts the data, generating 1 - RGB.
176 * @GST_D3D11_COMPOSITOR_BLEND_SRC_ALPHA_SAT:
177 * The blend factor is (f, f, f, 1); where f = min(As, 1 - Ad).
178 * The pre-blend operation clamps the data to 1 or less.
179 * @GST_D3D11_COMPOSITOR_BLEND_BLEND_FACTOR:
180 * The blend factor is the blend factor set with
181 * ID3D11DeviceContext::OMSetBlendState. No pre-blend operation.
182 * @GST_D3D11_COMPOSITOR_BLEND_INV_BLEND_FACTOR:
183 * The blend factor is the blend factor set with
184 * ID3D11DeviceContext::OMSetBlendState.
185 * The pre-blend operation inverts the blend factor,
186 * generating 1 - blend_factor.
187 *
188 * Since: 1.20
189 */
190 GType
gst_d3d11_compositor_blend_get_type(void)191 gst_d3d11_compositor_blend_get_type (void)
192 {
193 static GType blend_type = 0;
194
195 static const GEnumValue blend[] = {
196 {GST_D3D11_COMPOSITOR_BLEND_ZERO,
197 "The blend factor is (0, 0, 0, 0)", "zero"},
198 {GST_D3D11_COMPOSITOR_BLEND_ONE,
199 "The blend factor is (1, 1, 1, 1)", "one"},
200 {GST_D3D11_COMPOSITOR_BLEND_SRC_COLOR,
201 "The blend factor is (Rs, Gs, Bs, As)", "src-color"},
202 {GST_D3D11_COMPOSITOR_BLEND_INV_SRC_COLOR,
203 "The blend factor is (1 - Rs, 1 - Gs, 1 - Bs, 1 - As)",
204 "inv-src-color"},
205 {GST_D3D11_COMPOSITOR_BLEND_SRC_ALPHA,
206 "The blend factor is (As, As, As, As)", "src-alpha"},
207 {GST_D3D11_COMPOSITOR_BLEND_INV_SRC_ALPHA,
208 "The blend factor is (1 - As, 1 - As, 1 - As, 1 - As)",
209 "inv-src-alpha"},
210 {GST_D3D11_COMPOSITOR_BLEND_DEST_ALPHA,
211 "The blend factor is (Ad, Ad, Ad, Ad)", "dest-alpha"},
212 {GST_D3D11_COMPOSITOR_BLEND_INV_DEST_ALPHA,
213 "The blend factor is (1 - Ad, 1 - Ad, 1 - Ad, 1 - Ad)",
214 "inv-dest-alpha"},
215 {GST_D3D11_COMPOSITOR_BLEND_DEST_COLOR,
216 "The blend factor is (Rd, Gd, Bd, Ad)", "dest-color"},
217 {GST_D3D11_COMPOSITOR_BLEND_INV_DEST_COLOR,
218 "The blend factor is (1 - Rd, 1 - Gd, 1 - Bd, 1 - Ad)",
219 "inv-dest-color"},
220 {GST_D3D11_COMPOSITOR_BLEND_SRC_ALPHA_SAT,
221 "The blend factor is (f, f, f, 1); where f = min(As, 1 - Ad)",
222 "src-alpha-sat"},
223 {GST_D3D11_COMPOSITOR_BLEND_BLEND_FACTOR,
224 "User defined blend factor", "blend-factor"},
225 {GST_D3D11_COMPOSITOR_BLEND_INV_BLEND_FACTOR,
226 "Inverse of user defined blend factor", "inv-blend-factor"},
227 {0, NULL, NULL},
228 };
229
230 if (!blend_type) {
231 blend_type = g_enum_register_static ("GstD3D11CompositorBlend", blend);
232 }
233 return blend_type;
234 }
235
236 static GstD3D11CompositorBlend
gst_d3d11_compositor_blend_from_native(D3D11_BLEND blend)237 gst_d3d11_compositor_blend_from_native (D3D11_BLEND blend)
238 {
239 switch (blend) {
240 case D3D11_BLEND_ZERO:
241 return GST_D3D11_COMPOSITOR_BLEND_ZERO;
242 case D3D11_BLEND_ONE:
243 return GST_D3D11_COMPOSITOR_BLEND_ONE;
244 case D3D11_BLEND_SRC_COLOR:
245 return GST_D3D11_COMPOSITOR_BLEND_SRC_COLOR;
246 case D3D11_BLEND_INV_SRC_COLOR:
247 return GST_D3D11_COMPOSITOR_BLEND_INV_SRC_COLOR;
248 case D3D11_BLEND_SRC_ALPHA:
249 return GST_D3D11_COMPOSITOR_BLEND_SRC_ALPHA;
250 case D3D11_BLEND_INV_SRC_ALPHA:
251 return GST_D3D11_COMPOSITOR_BLEND_INV_SRC_ALPHA;
252 case D3D11_BLEND_DEST_ALPHA:
253 return GST_D3D11_COMPOSITOR_BLEND_DEST_ALPHA;
254 case D3D11_BLEND_INV_DEST_ALPHA:
255 return GST_D3D11_COMPOSITOR_BLEND_INV_DEST_ALPHA;
256 case D3D11_BLEND_DEST_COLOR:
257 return GST_D3D11_COMPOSITOR_BLEND_DEST_COLOR;
258 case D3D11_BLEND_INV_DEST_COLOR:
259 return GST_D3D11_COMPOSITOR_BLEND_INV_DEST_COLOR;
260 case D3D11_BLEND_SRC_ALPHA_SAT:
261 return GST_D3D11_COMPOSITOR_BLEND_SRC_ALPHA_SAT;
262 case D3D11_BLEND_BLEND_FACTOR:
263 return GST_D3D11_COMPOSITOR_BLEND_BLEND_FACTOR;
264 case D3D11_BLEND_INV_BLEND_FACTOR:
265 return GST_D3D11_COMPOSITOR_BLEND_INV_BLEND_FACTOR;
266 default:
267 g_assert_not_reached ();
268 break;
269 }
270
271 return GST_D3D11_COMPOSITOR_BLEND_ZERO;
272 }
273
274 static D3D11_BLEND
gst_d3d11_compositor_blend_to_native(GstD3D11CompositorBlend blend)275 gst_d3d11_compositor_blend_to_native (GstD3D11CompositorBlend blend)
276 {
277 switch (blend) {
278 case GST_D3D11_COMPOSITOR_BLEND_ZERO:
279 return D3D11_BLEND_ZERO;
280 case GST_D3D11_COMPOSITOR_BLEND_ONE:
281 return D3D11_BLEND_ONE;
282 case GST_D3D11_COMPOSITOR_BLEND_SRC_COLOR:
283 return D3D11_BLEND_SRC_COLOR;
284 case GST_D3D11_COMPOSITOR_BLEND_INV_SRC_COLOR:
285 return D3D11_BLEND_INV_SRC_COLOR;
286 case GST_D3D11_COMPOSITOR_BLEND_SRC_ALPHA:
287 return D3D11_BLEND_SRC_ALPHA;
288 case GST_D3D11_COMPOSITOR_BLEND_INV_SRC_ALPHA:
289 return D3D11_BLEND_INV_SRC_ALPHA;
290 case GST_D3D11_COMPOSITOR_BLEND_DEST_ALPHA:
291 return D3D11_BLEND_DEST_ALPHA;
292 case GST_D3D11_COMPOSITOR_BLEND_INV_DEST_ALPHA:
293 return D3D11_BLEND_INV_DEST_ALPHA;
294 case GST_D3D11_COMPOSITOR_BLEND_DEST_COLOR:
295 return D3D11_BLEND_DEST_COLOR;
296 case GST_D3D11_COMPOSITOR_BLEND_INV_DEST_COLOR:
297 return D3D11_BLEND_INV_DEST_COLOR;
298 case GST_D3D11_COMPOSITOR_BLEND_SRC_ALPHA_SAT:
299 return D3D11_BLEND_SRC_ALPHA_SAT;
300 case GST_D3D11_COMPOSITOR_BLEND_BLEND_FACTOR:
301 return D3D11_BLEND_BLEND_FACTOR;
302 case GST_D3D11_COMPOSITOR_BLEND_INV_BLEND_FACTOR:
303 return D3D11_BLEND_INV_BLEND_FACTOR;
304 default:
305 g_assert_not_reached ();
306 break;
307 }
308
309 return D3D11_BLEND_ZERO;
310 }
311
312 /**
313 * GstD3D11CompositorBackground:
314 *
315 * Background mode
316 *
317 * Since: 1.20
318 */
319 GType
gst_d3d11_compositor_background_get_type(void)320 gst_d3d11_compositor_background_get_type (void)
321 {
322 static GType compositor_background_type = 0;
323
324 static const GEnumValue compositor_background[] = {
325 {GST_D3D11_COMPOSITOR_BACKGROUND_CHECKER, "Checker pattern", "checker"},
326 {GST_D3D11_COMPOSITOR_BACKGROUND_BLACK, "Black", "black"},
327 {GST_D3D11_COMPOSITOR_BACKGROUND_WHITE, "White", "white"},
328 {GST_D3D11_COMPOSITOR_BACKGROUND_TRANSPARENT,
329 "Transparent Background to enable further compositing", "transparent"},
330 {0, NULL, NULL},
331 };
332
333 if (!compositor_background_type) {
334 compositor_background_type =
335 g_enum_register_static ("GstD3D11CompositorBackground",
336 compositor_background);
337 }
338 return compositor_background_type;
339 }
340
341 /**
342 * GstD3D11CompositorSizingPolicy:
343 *
344 * Sizing policy
345 *
346 * Since: 1.20
347 */
348 GType
gst_d3d11_compositor_sizing_policy_get_type(void)349 gst_d3d11_compositor_sizing_policy_get_type (void)
350 {
351 static GType sizing_policy_type = 0;
352
353 static const GEnumValue sizing_polices[] = {
354 {GST_D3D11_COMPOSITOR_SIZING_POLICY_NONE,
355 "None: Image is scaled to fill configured destination rectangle without "
356 "padding or keeping the aspect ratio", "none"},
357 {GST_D3D11_COMPOSITOR_SIZING_POLICY_KEEP_ASPECT_RATIO,
358 "Keep Aspect Ratio: Image is scaled to fit destination rectangle "
359 "specified by GstCompositorPad:{xpos, ypos, width, height} "
360 "with preserved aspect ratio. Resulting image will be centered in "
361 "the destination rectangle with padding if necessary",
362 "keep-aspect-ratio"},
363 {0, NULL, NULL},
364 };
365
366 if (!sizing_policy_type) {
367 sizing_policy_type =
368 g_enum_register_static ("GstD3D11CompositorSizingPolicy",
369 sizing_polices);
370 }
371 return sizing_policy_type;
372 }
373
374 /* *INDENT-OFF* */
375 static const gchar checker_vs_src[] =
376 "struct VS_INPUT\n"
377 "{\n"
378 " float4 Position : POSITION;\n"
379 "};\n"
380 "\n"
381 "struct VS_OUTPUT\n"
382 "{\n"
383 " float4 Position: SV_POSITION;\n"
384 "};\n"
385 "\n"
386 "VS_OUTPUT main(VS_INPUT input)\n"
387 "{\n"
388 " return input;\n"
389 "}\n";
390
391 static const gchar checker_ps_src[] =
392 "static const float blocksize = 8.0;\n"
393 "static const float4 high = float4(0.667, 0.667, 0.667, 1.0);\n"
394 "static const float4 low = float4(0.333, 0.333, 0.333, 1.0);\n"
395 "struct PS_INPUT\n"
396 "{\n"
397 " float4 Position: SV_POSITION;\n"
398 "};\n"
399 "struct PS_OUTPUT\n"
400 "{\n"
401 " float4 Plane: SV_TARGET;\n"
402 "};\n"
403 "PS_OUTPUT main(PS_INPUT input)\n"
404 "{\n"
405 " PS_OUTPUT output;\n"
406 " if ((input.Position.x % (blocksize * 2.0)) >= blocksize) {\n"
407 " if ((input.Position.y % (blocksize * 2.0)) >= blocksize)\n"
408 " output.Plane = low;\n"
409 " else\n"
410 " output.Plane = high;\n"
411 " } else {\n"
412 " if ((input.Position.y % (blocksize * 2.0)) < blocksize)\n"
413 " output.Plane = low;\n"
414 " else\n"
415 " output.Plane = high;\n"
416 " }\n"
417 " return output;\n"
418 "}\n";
419 /* *INDENT-ON* */
420
421 /**
422 * GstD3D11CompositorPad:
423 *
424 * Since: 1.20
425 */
426 struct _GstD3D11CompositorPad
427 {
428 GstVideoAggregatorConvertPad parent;
429
430 GstD3D11Converter *convert;
431
432 GstBufferPool *fallback_pool;
433 GstBuffer *fallback_buf;
434
435 gboolean position_updated;
436 gboolean alpha_updated;
437 gboolean blend_desc_updated;
438 ID3D11BlendState *blend;
439
440 /* properties */
441 gint xpos;
442 gint ypos;
443 gint width;
444 gint height;
445 gdouble alpha;
446 D3D11_RENDER_TARGET_BLEND_DESC desc;
447 gfloat blend_factor[4];
448 GstD3D11CompositorSizingPolicy sizing_policy;
449 };
450
451 struct _GstD3D11Compositor
452 {
453 GstVideoAggregator parent;
454
455 GstD3D11Device *device;
456
457 GstBufferPool *fallback_pool;
458 GstBuffer *fallback_buf;
459
460 GstD3D11Quad *checker_background;
461 D3D11_VIEWPORT viewport;
462
463 gboolean reconfigured;
464
465 /* properties */
466 gint adapter;
467 GstD3D11CompositorBackground background;
468 };
469
470 enum
471 {
472 PROP_PAD_0,
473 PROP_PAD_XPOS,
474 PROP_PAD_YPOS,
475 PROP_PAD_WIDTH,
476 PROP_PAD_HEIGHT,
477 PROP_PAD_ALPHA,
478 PROP_PAD_BLEND_OP_RGB,
479 PROP_PAD_BLEND_OP_ALPHA,
480 PROP_PAD_BLEND_SRC_RGB,
481 PROP_PAD_BLEND_SRC_ALPHA,
482 PROP_PAD_BLEND_DEST_RGB,
483 PROP_PAD_BLEND_DEST_ALPHA,
484 PROP_PAD_BLEND_FACTOR_RED,
485 PROP_PAD_BLEND_FACTOR_GREEN,
486 PROP_PAD_BLEND_FACTOR_BLUE,
487 PROP_PAD_BLEND_FACTOR_ALPHA,
488 PROP_PAD_SIZING_POLICY,
489 };
490
491 #define DEFAULT_PAD_XPOS 0
492 #define DEFAULT_PAD_YPOS 0
493 #define DEFAULT_PAD_WIDTH 0
494 #define DEFAULT_PAD_HEIGHT 0
495 #define DEFAULT_PAD_ALPHA 1.0
496 #define DEFAULT_PAD_BLEND_OP_RGB GST_D3D11_COMPOSITOR_BLEND_OP_ADD
497 #define DEFAULT_PAD_BLEND_OP_ALPHA GST_D3D11_COMPOSITOR_BLEND_OP_ADD
498 #define DEFAULT_PAD_BLEND_SRC_RGB GST_D3D11_COMPOSITOR_BLEND_SRC_ALPHA
499 #define DEFAULT_PAD_BLEND_SRC_ALPHA GST_D3D11_COMPOSITOR_BLEND_ONE
500 #define DEFAULT_PAD_BLEND_DEST_RGB GST_D3D11_COMPOSITOR_BLEND_INV_SRC_ALPHA
501 #define DEFAULT_PAD_BLEND_DEST_ALPHA GST_D3D11_COMPOSITOR_BLEND_INV_SRC_ALPHA
502 #define DEFAULT_PAD_SIZING_POLICY GST_D3D11_COMPOSITOR_SIZING_POLICY_NONE
503
504 static void gst_d3d11_compositor_pad_set_property (GObject * object,
505 guint prop_id, const GValue * value, GParamSpec * pspec);
506 static void gst_d3d11_compositor_pad_get_property (GObject * object,
507 guint prop_id, GValue * value, GParamSpec * pspec);
508 static gboolean
509 gst_d3d11_compositor_pad_prepare_frame (GstVideoAggregatorPad * pad,
510 GstVideoAggregator * vagg, GstBuffer * buffer,
511 GstVideoFrame * prepared_frame);
512 static void
513 gst_d3d11_compositor_pad_clean_frame (GstVideoAggregatorPad * pad,
514 GstVideoAggregator * vagg, GstVideoFrame * prepared_frame);
515 static void
516 gst_d3d11_compositor_pad_init_blend_options (GstD3D11CompositorPad * pad);
517
518 #define gst_d3d11_compositor_pad_parent_class parent_pad_class
519 G_DEFINE_TYPE (GstD3D11CompositorPad, gst_d3d11_compositor_pad,
520 GST_TYPE_VIDEO_AGGREGATOR_PAD);
521
522 static void
gst_d3d11_compositor_pad_class_init(GstD3D11CompositorPadClass * klass)523 gst_d3d11_compositor_pad_class_init (GstD3D11CompositorPadClass * klass)
524 {
525 GObjectClass *gobject_class = G_OBJECT_CLASS (klass);
526 GstVideoAggregatorPadClass *vaggpadclass =
527 GST_VIDEO_AGGREGATOR_PAD_CLASS (klass);
528
529 gobject_class->set_property = gst_d3d11_compositor_pad_set_property;
530 gobject_class->get_property = gst_d3d11_compositor_pad_get_property;
531
532 g_object_class_install_property (gobject_class, PROP_PAD_XPOS,
533 g_param_spec_int ("xpos", "X Position", "X position of the picture",
534 G_MININT, G_MAXINT, DEFAULT_PAD_XPOS,
535 (GParamFlags) (G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE |
536 G_PARAM_STATIC_STRINGS)));
537
538 g_object_class_install_property (gobject_class, PROP_PAD_YPOS,
539 g_param_spec_int ("ypos", "Y Position", "Y position of the picture",
540 G_MININT, G_MAXINT, DEFAULT_PAD_YPOS,
541 (GParamFlags) (G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE |
542 G_PARAM_STATIC_STRINGS)));
543
544 g_object_class_install_property (gobject_class, PROP_PAD_WIDTH,
545 g_param_spec_int ("width", "Width", "Width of the picture",
546 G_MININT, G_MAXINT, DEFAULT_PAD_WIDTH,
547 (GParamFlags) (G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE |
548 G_PARAM_STATIC_STRINGS)));
549
550 g_object_class_install_property (gobject_class, PROP_PAD_HEIGHT,
551 g_param_spec_int ("height", "Height", "Height of the picture",
552 G_MININT, G_MAXINT, DEFAULT_PAD_HEIGHT,
553 (GParamFlags) (G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE |
554 G_PARAM_STATIC_STRINGS)));
555
556 g_object_class_install_property (gobject_class, PROP_PAD_ALPHA,
557 g_param_spec_double ("alpha", "Alpha", "Alpha of the picture", 0.0, 1.0,
558 DEFAULT_PAD_ALPHA,
559 (GParamFlags) (G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE |
560 G_PARAM_STATIC_STRINGS)));
561
562 g_object_class_install_property (gobject_class, PROP_PAD_BLEND_OP_RGB,
563 g_param_spec_enum ("blend-op-rgb", "Blend Operation RGB",
564 "Blend equation for RGB", GST_TYPE_D3D11_COMPOSITOR_BLEND_OPERATION,
565 DEFAULT_PAD_BLEND_OP_RGB,
566 (GParamFlags) (G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE |
567 G_PARAM_STATIC_STRINGS)));
568
569 g_object_class_install_property (gobject_class, PROP_PAD_BLEND_OP_ALPHA,
570 g_param_spec_enum ("blend-op-alpha", "Blend Operation Alpha",
571 "Blend equation for alpha", GST_TYPE_D3D11_COMPOSITOR_BLEND_OPERATION,
572 DEFAULT_PAD_BLEND_OP_ALPHA,
573 (GParamFlags) (G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE |
574 G_PARAM_STATIC_STRINGS)));
575
576 g_object_class_install_property (gobject_class,
577 PROP_PAD_BLEND_SRC_RGB,
578 g_param_spec_enum ("blend-src-rgb", "Blend Source RGB",
579 "Blend factor for source RGB",
580 GST_TYPE_D3D11_COMPOSITOR_BLEND,
581 DEFAULT_PAD_BLEND_SRC_RGB,
582 (GParamFlags) (G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE |
583 G_PARAM_STATIC_STRINGS)));
584
585 g_object_class_install_property (gobject_class,
586 PROP_PAD_BLEND_SRC_ALPHA,
587 g_param_spec_enum ("blend-src-alpha",
588 "Blend Source Alpha",
589 "Blend factor for source alpha, \"*-color\" values are not allowed",
590 GST_TYPE_D3D11_COMPOSITOR_BLEND,
591 DEFAULT_PAD_BLEND_SRC_ALPHA,
592 (GParamFlags) (G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE |
593 G_PARAM_STATIC_STRINGS)));
594
595 g_object_class_install_property (gobject_class,
596 PROP_PAD_BLEND_DEST_RGB,
597 g_param_spec_enum ("blend-dest-rgb",
598 "Blend Destination RGB",
599 "Blend factor for destination RGB",
600 GST_TYPE_D3D11_COMPOSITOR_BLEND,
601 DEFAULT_PAD_BLEND_DEST_RGB,
602 (GParamFlags) (G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE |
603 G_PARAM_STATIC_STRINGS)));
604
605 g_object_class_install_property (gobject_class,
606 PROP_PAD_BLEND_DEST_ALPHA,
607 g_param_spec_enum ("blend-dest-alpha",
608 "Blend Destination Alpha",
609 "Blend factor for destination alpha, "
610 "\"*-color\" values are not allowed",
611 GST_TYPE_D3D11_COMPOSITOR_BLEND,
612 DEFAULT_PAD_BLEND_DEST_ALPHA,
613 (GParamFlags) (G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE |
614 G_PARAM_STATIC_STRINGS)));
615
616 g_object_class_install_property (gobject_class, PROP_PAD_BLEND_FACTOR_RED,
617 g_param_spec_float ("blend-factor-red", "Blend Factor Red",
618 "Blend factor for red component "
619 "when blend type is \"blend-factor\" or \"inv-blend-factor\"",
620 0.0, 1.0, 1.0,
621 (GParamFlags) (G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE |
622 G_PARAM_STATIC_STRINGS)));
623
624 g_object_class_install_property (gobject_class, PROP_PAD_BLEND_FACTOR_GREEN,
625 g_param_spec_float ("blend-factor-green", "Blend Factor Green",
626 "Blend factor for green component "
627 "when blend type is \"blend-factor\" or \"inv-blend-factor\"",
628 0.0, 1.0, 1.0,
629 (GParamFlags) (G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE |
630 G_PARAM_STATIC_STRINGS)));
631
632 g_object_class_install_property (gobject_class, PROP_PAD_BLEND_FACTOR_BLUE,
633 g_param_spec_float ("blend-factor-blue", "Blend Factor Blue",
634 "Blend factor for blue component "
635 "when blend type is \"blend-factor\" or \"inv-blend-factor\"",
636 0.0, 1.0, 1.0,
637 (GParamFlags) (G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE |
638 G_PARAM_STATIC_STRINGS)));
639
640 g_object_class_install_property (gobject_class, PROP_PAD_BLEND_FACTOR_ALPHA,
641 g_param_spec_float ("blend-factor-alpha", "Blend Factor Alpha",
642 "Blend factor for alpha component "
643 "when blend type is \"blend-factor\" or \"inv-blend-factor\"",
644 0.0, 1.0, 1.0,
645 (GParamFlags) (G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE |
646 G_PARAM_STATIC_STRINGS)));
647
648 g_object_class_install_property (gobject_class, PROP_PAD_SIZING_POLICY,
649 g_param_spec_enum ("sizing-policy", "Sizing policy",
650 "Sizing policy to use for image scaling",
651 GST_TYPE_D3D11_COMPOSITOR_SIZING_POLICY, DEFAULT_PAD_SIZING_POLICY,
652 (GParamFlags) (G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE |
653 G_PARAM_STATIC_STRINGS)));
654
655 vaggpadclass->prepare_frame =
656 GST_DEBUG_FUNCPTR (gst_d3d11_compositor_pad_prepare_frame);
657 vaggpadclass->clean_frame =
658 GST_DEBUG_FUNCPTR (gst_d3d11_compositor_pad_clean_frame);
659
660 gst_type_mark_as_plugin_api (GST_TYPE_D3D11_COMPOSITOR_BLEND,
661 (GstPluginAPIFlags) 0);
662 gst_type_mark_as_plugin_api (GST_TYPE_D3D11_COMPOSITOR_BLEND_OPERATION,
663 (GstPluginAPIFlags) 0);
664 gst_type_mark_as_plugin_api (GST_TYPE_D3D11_COMPOSITOR_SIZING_POLICY,
665 (GstPluginAPIFlags) 0);
666 }
667
668 static void
gst_d3d11_compositor_pad_init(GstD3D11CompositorPad * pad)669 gst_d3d11_compositor_pad_init (GstD3D11CompositorPad * pad)
670 {
671 pad->xpos = DEFAULT_PAD_XPOS;
672 pad->ypos = DEFAULT_PAD_YPOS;
673 pad->width = DEFAULT_PAD_WIDTH;
674 pad->height = DEFAULT_PAD_HEIGHT;
675 pad->alpha = DEFAULT_PAD_ALPHA;
676 pad->sizing_policy = DEFAULT_PAD_SIZING_POLICY;
677
678 gst_d3d11_compositor_pad_init_blend_options (pad);
679 }
680
681 static void
gst_d3d11_compositor_pad_update_blend_function(GstD3D11CompositorPad * pad,D3D11_BLEND * value,GstD3D11CompositorBlend new_value)682 gst_d3d11_compositor_pad_update_blend_function (GstD3D11CompositorPad * pad,
683 D3D11_BLEND * value, GstD3D11CompositorBlend new_value)
684 {
685 D3D11_BLEND temp = gst_d3d11_compositor_blend_to_native (new_value);
686
687 if (temp == *value)
688 return;
689
690 *value = temp;
691 pad->blend_desc_updated = TRUE;
692 }
693
694 static void
gst_d3d11_compositor_pad_update_blend_equation(GstD3D11CompositorPad * pad,D3D11_BLEND_OP * value,GstD3D11CompositorBlendOperation new_value)695 gst_d3d11_compositor_pad_update_blend_equation (GstD3D11CompositorPad * pad,
696 D3D11_BLEND_OP * value, GstD3D11CompositorBlendOperation new_value)
697 {
698 D3D11_BLEND_OP temp =
699 gst_d3d11_compositor_blend_operation_to_native (new_value);
700
701 if (temp == *value)
702 return;
703
704 *value = temp;
705 pad->blend_desc_updated = TRUE;
706 }
707
708 static void
gst_d3d11_compositor_pad_set_property(GObject * object,guint prop_id,const GValue * value,GParamSpec * pspec)709 gst_d3d11_compositor_pad_set_property (GObject * object, guint prop_id,
710 const GValue * value, GParamSpec * pspec)
711 {
712 GstD3D11CompositorPad *pad = GST_D3D11_COMPOSITOR_PAD (object);
713
714 switch (prop_id) {
715 case PROP_PAD_XPOS:
716 pad->xpos = g_value_get_int (value);
717 pad->position_updated = TRUE;
718 break;
719 case PROP_PAD_YPOS:
720 pad->ypos = g_value_get_int (value);
721 pad->position_updated = TRUE;
722 break;
723 case PROP_PAD_WIDTH:
724 pad->width = g_value_get_int (value);
725 pad->position_updated = TRUE;
726 break;
727 case PROP_PAD_HEIGHT:
728 pad->height = g_value_get_int (value);
729 pad->position_updated = TRUE;
730 break;
731 case PROP_PAD_ALPHA:
732 {
733 gdouble alpha = g_value_get_double (value);
734 if (pad->alpha != alpha) {
735 pad->alpha_updated = TRUE;
736 pad->alpha = alpha;
737 }
738 break;
739 }
740 case PROP_PAD_BLEND_OP_RGB:
741 gst_d3d11_compositor_pad_update_blend_equation (pad, &pad->desc.BlendOp,
742 (GstD3D11CompositorBlendOperation) g_value_get_enum (value));
743 break;
744 case PROP_PAD_BLEND_OP_ALPHA:
745 gst_d3d11_compositor_pad_update_blend_equation (pad,
746 &pad->desc.BlendOpAlpha,
747 (GstD3D11CompositorBlendOperation) g_value_get_enum (value));
748 break;
749 case PROP_PAD_BLEND_SRC_RGB:
750 gst_d3d11_compositor_pad_update_blend_function (pad, &pad->desc.SrcBlend,
751 (GstD3D11CompositorBlend) g_value_get_enum (value));
752 break;
753 case PROP_PAD_BLEND_SRC_ALPHA:
754 {
755 GstD3D11CompositorBlend blend =
756 (GstD3D11CompositorBlend) g_value_get_enum (value);
757 if (blend == GST_D3D11_COMPOSITOR_BLEND_SRC_COLOR ||
758 blend == GST_D3D11_COMPOSITOR_BLEND_INV_SRC_COLOR ||
759 blend == GST_D3D11_COMPOSITOR_BLEND_DEST_COLOR ||
760 blend == GST_D3D11_COMPOSITOR_BLEND_INV_DEST_COLOR) {
761 g_warning ("%d is not allowed for %s", blend, pspec->name);
762 } else {
763 gst_d3d11_compositor_pad_update_blend_function (pad,
764 &pad->desc.SrcBlendAlpha, blend);
765 }
766 break;
767 }
768 case PROP_PAD_BLEND_DEST_RGB:
769 gst_d3d11_compositor_pad_update_blend_function (pad, &pad->desc.DestBlend,
770 (GstD3D11CompositorBlend) g_value_get_enum (value));
771 break;
772 case PROP_PAD_BLEND_DEST_ALPHA:
773 {
774 GstD3D11CompositorBlend blend =
775 (GstD3D11CompositorBlend) g_value_get_enum (value);
776 if (blend == GST_D3D11_COMPOSITOR_BLEND_SRC_COLOR ||
777 blend == GST_D3D11_COMPOSITOR_BLEND_INV_SRC_COLOR ||
778 blend == GST_D3D11_COMPOSITOR_BLEND_DEST_COLOR ||
779 blend == GST_D3D11_COMPOSITOR_BLEND_INV_DEST_COLOR) {
780 g_warning ("%d is not allowed for %s", blend, pspec->name);
781 } else {
782 gst_d3d11_compositor_pad_update_blend_function (pad,
783 &pad->desc.DestBlendAlpha, blend);
784 }
785 break;
786 }
787 case PROP_PAD_BLEND_FACTOR_RED:
788 pad->blend_factor[0] = g_value_get_float (value);
789 break;
790 case PROP_PAD_BLEND_FACTOR_GREEN:
791 pad->blend_factor[1] = g_value_get_float (value);
792 break;
793 case PROP_PAD_BLEND_FACTOR_BLUE:
794 pad->blend_factor[2] = g_value_get_float (value);
795 break;
796 case PROP_PAD_BLEND_FACTOR_ALPHA:
797 pad->blend_factor[3] = g_value_get_float (value);
798 break;
799 case PROP_PAD_SIZING_POLICY:
800 pad->sizing_policy =
801 (GstD3D11CompositorSizingPolicy) g_value_get_enum (value);
802 pad->position_updated = TRUE;
803 break;
804 default:
805 G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
806 break;
807 }
808 }
809
810 static void
gst_d3d11_compositor_pad_get_property(GObject * object,guint prop_id,GValue * value,GParamSpec * pspec)811 gst_d3d11_compositor_pad_get_property (GObject * object, guint prop_id,
812 GValue * value, GParamSpec * pspec)
813 {
814 GstD3D11CompositorPad *pad = GST_D3D11_COMPOSITOR_PAD (object);
815
816 switch (prop_id) {
817 case PROP_PAD_XPOS:
818 g_value_set_int (value, pad->xpos);
819 break;
820 case PROP_PAD_YPOS:
821 g_value_set_int (value, pad->ypos);
822 break;
823 case PROP_PAD_WIDTH:
824 g_value_set_int (value, pad->width);
825 break;
826 case PROP_PAD_HEIGHT:
827 g_value_set_int (value, pad->height);
828 break;
829 case PROP_PAD_ALPHA:
830 g_value_set_double (value, pad->alpha);
831 break;
832 case PROP_PAD_BLEND_OP_RGB:
833 g_value_set_enum (value,
834 gst_d3d11_compositor_blend_operation_from_native (pad->desc.BlendOp));
835 break;
836 case PROP_PAD_BLEND_OP_ALPHA:
837 g_value_set_enum (value,
838 gst_d3d11_compositor_blend_operation_from_native (pad->
839 desc.BlendOpAlpha));
840 break;
841 case PROP_PAD_BLEND_SRC_RGB:
842 g_value_set_enum (value,
843 gst_d3d11_compositor_blend_from_native (pad->desc.SrcBlend));
844 break;
845 case PROP_PAD_BLEND_SRC_ALPHA:
846 g_value_set_enum (value,
847 gst_d3d11_compositor_blend_from_native (pad->desc.SrcBlendAlpha));
848 break;
849 case PROP_PAD_BLEND_DEST_RGB:
850 g_value_set_enum (value,
851 gst_d3d11_compositor_blend_from_native (pad->desc.DestBlend));
852 break;
853 case PROP_PAD_BLEND_DEST_ALPHA:
854 g_value_set_enum (value,
855 gst_d3d11_compositor_blend_from_native (pad->desc.DestBlendAlpha));
856 break;
857 case PROP_PAD_BLEND_FACTOR_RED:
858 g_value_set_float (value, pad->blend_factor[0]);
859 break;
860 case PROP_PAD_BLEND_FACTOR_GREEN:
861 g_value_set_float (value, pad->blend_factor[1]);
862 break;
863 case PROP_PAD_BLEND_FACTOR_BLUE:
864 g_value_set_float (value, pad->blend_factor[2]);
865 break;
866 case PROP_PAD_BLEND_FACTOR_ALPHA:
867 g_value_set_float (value, pad->blend_factor[3]);
868 break;
869 case PROP_PAD_SIZING_POLICY:
870 g_value_set_enum (value, pad->sizing_policy);
871 break;
872 default:
873 G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
874 break;
875 }
876 }
877
878 static void
gst_d3d11_compositor_pad_init_blend_options(GstD3D11CompositorPad * pad)879 gst_d3d11_compositor_pad_init_blend_options (GstD3D11CompositorPad * pad)
880 {
881 guint i;
882
883 pad->desc.BlendEnable = TRUE;
884 pad->desc.SrcBlend =
885 gst_d3d11_compositor_blend_to_native (DEFAULT_PAD_BLEND_SRC_RGB);
886 pad->desc.DestBlend =
887 gst_d3d11_compositor_blend_to_native (DEFAULT_PAD_BLEND_DEST_RGB);
888 pad->desc.BlendOp =
889 gst_d3d11_compositor_blend_operation_to_native (DEFAULT_PAD_BLEND_OP_RGB);
890 pad->desc.SrcBlendAlpha =
891 gst_d3d11_compositor_blend_to_native (DEFAULT_PAD_BLEND_SRC_ALPHA);
892 pad->desc.DestBlendAlpha =
893 gst_d3d11_compositor_blend_to_native (DEFAULT_PAD_BLEND_DEST_ALPHA);
894 pad->desc.BlendOpAlpha =
895 gst_d3d11_compositor_blend_operation_to_native
896 (DEFAULT_PAD_BLEND_OP_ALPHA);
897 pad->desc.RenderTargetWriteMask = D3D11_COLOR_WRITE_ENABLE_ALL;
898
899 for (i = 0; i < G_N_ELEMENTS (pad->blend_factor); i++)
900 pad->blend_factor[i] = 1.0f;
901 }
902
903 static gboolean
gst_d3d11_compositor_configure_fallback_pool(GstD3D11Compositor * self,GstVideoInfo * info,gint bind_flags,GstBufferPool ** pool)904 gst_d3d11_compositor_configure_fallback_pool (GstD3D11Compositor * self,
905 GstVideoInfo * info, gint bind_flags, GstBufferPool ** pool)
906 {
907 GstD3D11AllocationParams *d3d11_params;
908 GstBufferPool *new_pool;
909 GstCaps *caps;
910
911 if (*pool) {
912 gst_buffer_pool_set_active (*pool, FALSE);
913 gst_clear_object (pool);
914 }
915
916 caps = gst_video_info_to_caps (info);
917 if (!caps) {
918 GST_ERROR_OBJECT (self, "Couldn't create caps from info");
919 return FALSE;
920 }
921
922 d3d11_params = gst_d3d11_allocation_params_new (self->device,
923 info, (GstD3D11AllocationFlags) 0, bind_flags);
924
925 new_pool = gst_d3d11_buffer_pool_new_with_options (self->device,
926 caps, d3d11_params, 0, 0);
927 gst_caps_unref (caps);
928 gst_d3d11_allocation_params_free (d3d11_params);
929
930 if (!new_pool) {
931 GST_ERROR_OBJECT (self, "Failed to configure fallback pool");
932 return FALSE;
933 }
934
935 gst_buffer_pool_set_active (new_pool, TRUE);
936 *pool = new_pool;
937
938 return TRUE;
939 }
940
941 static gboolean
gst_d3d11_compsitor_prepare_fallback_buffer(GstD3D11Compositor * self,GstVideoInfo * info,gboolean is_input,GstBufferPool ** pool,GstBuffer ** fallback_buffer)942 gst_d3d11_compsitor_prepare_fallback_buffer (GstD3D11Compositor * self,
943 GstVideoInfo * info, gboolean is_input, GstBufferPool ** pool,
944 GstBuffer ** fallback_buffer)
945 {
946 GstBuffer *new_buf = NULL;
947 gint bind_flags = D3D11_BIND_SHADER_RESOURCE;
948 guint i;
949
950 gst_clear_buffer (fallback_buffer);
951
952 if (!is_input)
953 bind_flags = D3D11_BIND_RENDER_TARGET;
954
955 if (*pool == NULL &&
956 !gst_d3d11_compositor_configure_fallback_pool (self, info,
957 bind_flags, pool)) {
958 GST_ERROR_OBJECT (self, "Couldn't configure fallback buffer pool");
959 return FALSE;
960 }
961
962 if (gst_buffer_pool_acquire_buffer (*pool, &new_buf, NULL)
963 != GST_FLOW_OK) {
964 GST_ERROR_OBJECT (self, "Couldn't get fallback buffer from pool");
965 return FALSE;
966 }
967
968 for (i = 0; i < gst_buffer_n_memory (new_buf); i++) {
969 GstD3D11Memory *new_mem =
970 (GstD3D11Memory *) gst_buffer_peek_memory (new_buf, i);
971
972 if (is_input && !gst_d3d11_memory_get_shader_resource_view_size (new_mem)) {
973 GST_ERROR_OBJECT (self, "Couldn't prepare shader resource view");
974 gst_buffer_unref (new_buf);
975 return FALSE;
976 } else if (!is_input &&
977 !gst_d3d11_memory_get_render_target_view_size (new_mem)) {
978 GST_ERROR_OBJECT (self, "Couldn't prepare render target view");
979 gst_buffer_unref (new_buf);
980 return FALSE;
981 }
982 }
983
984 *fallback_buffer = new_buf;
985
986 return TRUE;
987 }
988
989 static gboolean
gst_d3d11_compositor_copy_buffer(GstD3D11Compositor * self,GstVideoInfo * info,GstBuffer * src_buf,GstBuffer * dest_buf,gboolean do_device_copy)990 gst_d3d11_compositor_copy_buffer (GstD3D11Compositor * self,
991 GstVideoInfo * info, GstBuffer * src_buf, GstBuffer * dest_buf,
992 gboolean do_device_copy)
993 {
994 guint i;
995
996 if (do_device_copy) {
997 return gst_d3d11_buffer_copy_into (dest_buf, src_buf, info);
998 } else {
999 GstVideoFrame src_frame, dest_frame;
1000
1001 if (!gst_video_frame_map (&src_frame, info, src_buf,
1002 (GstMapFlags) (GST_MAP_READ | GST_VIDEO_FRAME_MAP_FLAG_NO_REF))) {
1003 GST_ERROR_OBJECT (self, "Couldn't map input buffer");
1004 return FALSE;
1005 }
1006
1007 if (!gst_video_frame_map (&dest_frame, info, dest_buf,
1008 (GstMapFlags) (GST_MAP_WRITE | GST_VIDEO_FRAME_MAP_FLAG_NO_REF))) {
1009 GST_ERROR_OBJECT (self, "Couldn't fallback buffer");
1010 gst_video_frame_unmap (&src_frame);
1011 return FALSE;
1012 }
1013
1014 for (i = 0; i < GST_VIDEO_FRAME_N_PLANES (&src_frame); i++) {
1015 if (!gst_video_frame_copy_plane (&dest_frame, &src_frame, i)) {
1016 GST_ERROR_OBJECT (self, "Couldn't copy %dth plane", i);
1017
1018 gst_video_frame_unmap (&dest_frame);
1019 gst_video_frame_unmap (&src_frame);
1020
1021 return FALSE;
1022 }
1023 }
1024
1025 gst_video_frame_unmap (&dest_frame);
1026 gst_video_frame_unmap (&src_frame);
1027 }
1028
1029 return TRUE;
1030 }
1031
1032 static gboolean
gst_d3d11_compositor_check_d3d11_memory(GstD3D11Compositor * self,GstBuffer * buffer,gboolean is_input,gboolean * view_available)1033 gst_d3d11_compositor_check_d3d11_memory (GstD3D11Compositor * self,
1034 GstBuffer * buffer, gboolean is_input, gboolean * view_available)
1035 {
1036 guint i;
1037 gboolean ret = TRUE;
1038
1039 *view_available = TRUE;
1040
1041 for (i = 0; i < gst_buffer_n_memory (buffer); i++) {
1042 GstMemory *mem = gst_buffer_peek_memory (buffer, i);
1043 GstD3D11Memory *dmem;
1044
1045 if (!gst_is_d3d11_memory (mem)) {
1046 ret = FALSE;
1047 goto done;
1048 }
1049
1050 dmem = (GstD3D11Memory *) mem;
1051 if (dmem->device != self->device) {
1052 ret = FALSE;
1053 goto done;
1054 }
1055
1056 if (is_input) {
1057 if (!gst_d3d11_memory_get_shader_resource_view_size (dmem))
1058 *view_available = FALSE;
1059 } else {
1060 if (!gst_d3d11_memory_get_render_target_view_size (dmem))
1061 *view_available = FALSE;
1062 }
1063 }
1064
1065 done:
1066 if (!ret)
1067 *view_available = FALSE;
1068
1069 return ret;
1070 }
1071
1072 static void
gst_d3d11_compositor_pad_get_output_size(GstD3D11CompositorPad * comp_pad,gint out_par_n,gint out_par_d,gint * width,gint * height,gint * x_offset,gint * y_offset)1073 gst_d3d11_compositor_pad_get_output_size (GstD3D11CompositorPad * comp_pad,
1074 gint out_par_n, gint out_par_d, gint * width, gint * height,
1075 gint * x_offset, gint * y_offset)
1076 {
1077 GstVideoAggregatorPad *vagg_pad = GST_VIDEO_AGGREGATOR_PAD (comp_pad);
1078 gint pad_width, pad_height;
1079 guint dar_n, dar_d;
1080
1081 *x_offset = 0;
1082 *y_offset = 0;
1083 *width = 0;
1084 *height = 0;
1085
1086 /* FIXME: Anything better we can do here? */
1087 if (!vagg_pad->info.finfo
1088 || vagg_pad->info.finfo->format == GST_VIDEO_FORMAT_UNKNOWN) {
1089 GST_DEBUG_OBJECT (comp_pad, "Have no caps yet");
1090 return;
1091 }
1092
1093 pad_width =
1094 comp_pad->width <=
1095 0 ? GST_VIDEO_INFO_WIDTH (&vagg_pad->info) : comp_pad->width;
1096 pad_height =
1097 comp_pad->height <=
1098 0 ? GST_VIDEO_INFO_HEIGHT (&vagg_pad->info) : comp_pad->height;
1099
1100 if (pad_width == 0 || pad_height == 0)
1101 return;
1102
1103 if (!gst_video_calculate_display_ratio (&dar_n, &dar_d, pad_width, pad_height,
1104 GST_VIDEO_INFO_PAR_N (&vagg_pad->info),
1105 GST_VIDEO_INFO_PAR_D (&vagg_pad->info), out_par_n, out_par_d)) {
1106 GST_WARNING_OBJECT (comp_pad, "Cannot calculate display aspect ratio");
1107 return;
1108 }
1109
1110 GST_TRACE_OBJECT (comp_pad, "scaling %ux%u by %u/%u (%u/%u / %u/%u)",
1111 pad_width, pad_height, dar_n, dar_d,
1112 GST_VIDEO_INFO_PAR_N (&vagg_pad->info),
1113 GST_VIDEO_INFO_PAR_D (&vagg_pad->info), out_par_n, out_par_d);
1114
1115 switch (comp_pad->sizing_policy) {
1116 case GST_D3D11_COMPOSITOR_SIZING_POLICY_NONE:
1117 /* Pick either height or width, whichever is an integer multiple of the
1118 * display aspect ratio. However, prefer preserving the height to account
1119 * for interlaced video. */
1120 if (pad_height % dar_n == 0) {
1121 pad_width = gst_util_uint64_scale_int (pad_height, dar_n, dar_d);
1122 } else if (pad_width % dar_d == 0) {
1123 pad_height = gst_util_uint64_scale_int (pad_width, dar_d, dar_n);
1124 } else {
1125 pad_width = gst_util_uint64_scale_int (pad_height, dar_n, dar_d);
1126 }
1127 break;
1128 case GST_D3D11_COMPOSITOR_SIZING_POLICY_KEEP_ASPECT_RATIO:
1129 {
1130 gint from_dar_n, from_dar_d, to_dar_n, to_dar_d, num, den;
1131
1132 /* Calculate DAR again with actual video size */
1133 if (!gst_util_fraction_multiply (GST_VIDEO_INFO_WIDTH (&vagg_pad->info),
1134 GST_VIDEO_INFO_HEIGHT (&vagg_pad->info),
1135 GST_VIDEO_INFO_PAR_N (&vagg_pad->info),
1136 GST_VIDEO_INFO_PAR_D (&vagg_pad->info), &from_dar_n,
1137 &from_dar_d)) {
1138 from_dar_n = from_dar_d = -1;
1139 }
1140
1141 if (!gst_util_fraction_multiply (pad_width, pad_height,
1142 out_par_n, out_par_d, &to_dar_n, &to_dar_d)) {
1143 to_dar_n = to_dar_d = -1;
1144 }
1145
1146 if (from_dar_n != to_dar_n || from_dar_d != to_dar_d) {
1147 /* Calculate new output resolution */
1148 if (from_dar_n != -1 && from_dar_d != -1
1149 && gst_util_fraction_multiply (from_dar_n, from_dar_d,
1150 out_par_d, out_par_n, &num, &den)) {
1151 GstVideoRectangle src_rect, dst_rect, rst_rect;
1152
1153 src_rect.h = gst_util_uint64_scale_int (pad_width, den, num);
1154 if (src_rect.h == 0) {
1155 pad_width = 0;
1156 pad_height = 0;
1157 break;
1158 }
1159
1160 src_rect.x = src_rect.y = 0;
1161 src_rect.w = pad_width;
1162
1163 dst_rect.x = dst_rect.y = 0;
1164 dst_rect.w = pad_width;
1165 dst_rect.h = pad_height;
1166
1167 /* Scale rect to be centered in destination rect */
1168 gst_video_center_rect (&src_rect, &dst_rect, &rst_rect, TRUE);
1169
1170 GST_LOG_OBJECT (comp_pad,
1171 "Re-calculated size %dx%d -> %dx%d (x-offset %d, y-offset %d)",
1172 pad_width, pad_height, rst_rect.w, rst_rect.h, rst_rect.x,
1173 rst_rect.h);
1174
1175 *x_offset = rst_rect.x;
1176 *y_offset = rst_rect.y;
1177 pad_width = rst_rect.w;
1178 pad_height = rst_rect.h;
1179 } else {
1180 GST_WARNING_OBJECT (comp_pad, "Failed to calculate output size");
1181
1182 *x_offset = 0;
1183 *y_offset = 0;
1184 pad_width = 0;
1185 pad_height = 0;
1186 }
1187 }
1188 break;
1189 }
1190 }
1191
1192 *width = pad_width;
1193 *height = pad_height;
1194 }
1195
1196 static GstVideoRectangle
clamp_rectangle(gint x,gint y,gint w,gint h,gint outer_width,gint outer_height)1197 clamp_rectangle (gint x, gint y, gint w, gint h, gint outer_width,
1198 gint outer_height)
1199 {
1200 gint x2 = x + w;
1201 gint y2 = y + h;
1202 GstVideoRectangle clamped;
1203
1204 /* Clamp the x/y coordinates of this frame to the output boundaries to cover
1205 * the case where (say, with negative xpos/ypos or w/h greater than the output
1206 * size) the non-obscured portion of the frame could be outside the bounds of
1207 * the video itself and hence not visible at all */
1208 clamped.x = CLAMP (x, 0, outer_width);
1209 clamped.y = CLAMP (y, 0, outer_height);
1210 clamped.w = CLAMP (x2, 0, outer_width) - clamped.x;
1211 clamped.h = CLAMP (y2, 0, outer_height) - clamped.y;
1212
1213 return clamped;
1214 }
1215
1216 static gboolean
gst_d3d11_compositor_pad_check_frame_obscured(GstVideoAggregatorPad * pad,GstVideoAggregator * vagg)1217 gst_d3d11_compositor_pad_check_frame_obscured (GstVideoAggregatorPad * pad,
1218 GstVideoAggregator * vagg)
1219 {
1220 GstD3D11CompositorPad *cpad = GST_D3D11_COMPOSITOR_PAD (pad);
1221 gint width, height;
1222 GstVideoInfo *info = &vagg->info;
1223 /* The rectangle representing this frame, clamped to the video's boundaries.
1224 * Due to the clamping, this is different from the frame width/height above. */
1225 GstVideoRectangle frame_rect;
1226 gint x_offset, y_offset;
1227
1228 /* There's three types of width/height here:
1229 * 1. GST_VIDEO_FRAME_WIDTH/HEIGHT:
1230 * The frame width/height (same as pad->info.height/width;
1231 * see gst_video_frame_map())
1232 * 2. cpad->width/height:
1233 * The optional pad property for scaling the frame (if zero, the video is
1234 * left unscaled)
1235 */
1236
1237 gst_d3d11_compositor_pad_get_output_size (cpad, GST_VIDEO_INFO_PAR_N (info),
1238 GST_VIDEO_INFO_PAR_D (info), &width, &height, &x_offset, &y_offset);
1239
1240 frame_rect = clamp_rectangle (cpad->xpos + x_offset, cpad->ypos + y_offset,
1241 width, height, GST_VIDEO_INFO_WIDTH (info), GST_VIDEO_INFO_HEIGHT (info));
1242
1243 if (frame_rect.w == 0 || frame_rect.h == 0) {
1244 GST_DEBUG_OBJECT (pad, "Resulting frame is zero-width or zero-height "
1245 "(w: %i, h: %i), skipping", frame_rect.w, frame_rect.h);
1246 return TRUE;
1247 }
1248
1249 return FALSE;
1250 }
1251
1252 static gboolean
gst_d3d11_compositor_pad_prepare_frame(GstVideoAggregatorPad * pad,GstVideoAggregator * vagg,GstBuffer * buffer,GstVideoFrame * prepared_frame)1253 gst_d3d11_compositor_pad_prepare_frame (GstVideoAggregatorPad * pad,
1254 GstVideoAggregator * vagg, GstBuffer * buffer,
1255 GstVideoFrame * prepared_frame)
1256 {
1257 GstD3D11Compositor *self = GST_D3D11_COMPOSITOR (vagg);
1258 GstD3D11CompositorPad *cpad = GST_D3D11_COMPOSITOR_PAD (pad);
1259 GstBuffer *target_buf = buffer;
1260 gboolean do_device_copy = FALSE;
1261
1262 /* Skip this frame */
1263 if (gst_d3d11_compositor_pad_check_frame_obscured (pad, vagg))
1264 return TRUE;
1265
1266 /* Use fallback buffer when input buffer is:
1267 * - non-d3d11 memory
1268 * - or, from different d3d11 device
1269 * - or not bound to shader resource
1270 */
1271 if (!gst_d3d11_compositor_check_d3d11_memory (self,
1272 buffer, TRUE, &do_device_copy) || !do_device_copy) {
1273 if (!gst_d3d11_compsitor_prepare_fallback_buffer (self, &pad->info, TRUE,
1274 &cpad->fallback_pool, &cpad->fallback_buf)) {
1275 GST_ERROR_OBJECT (self, "Couldn't prepare fallback buffer");
1276 return FALSE;
1277 }
1278
1279 if (!gst_d3d11_compositor_copy_buffer (self, &pad->info, buffer,
1280 cpad->fallback_buf, do_device_copy)) {
1281 GST_ERROR_OBJECT (self, "Couldn't copy input buffer to fallback buffer");
1282 gst_clear_buffer (&cpad->fallback_buf);
1283 return FALSE;
1284 }
1285
1286 target_buf = cpad->fallback_buf;
1287 }
1288
1289 if (!gst_video_frame_map (prepared_frame, &pad->info, target_buf,
1290 (GstMapFlags) (GST_MAP_READ | GST_MAP_D3D11))) {
1291 GST_WARNING_OBJECT (pad, "Couldn't map input buffer");
1292 return FALSE;
1293 }
1294
1295 return TRUE;
1296 }
1297
1298 static void
gst_d3d11_compositor_pad_clean_frame(GstVideoAggregatorPad * pad,GstVideoAggregator * vagg,GstVideoFrame * prepared_frame)1299 gst_d3d11_compositor_pad_clean_frame (GstVideoAggregatorPad * pad,
1300 GstVideoAggregator * vagg, GstVideoFrame * prepared_frame)
1301 {
1302 GstD3D11CompositorPad *cpad = GST_D3D11_COMPOSITOR_PAD (pad);
1303
1304 GST_VIDEO_AGGREGATOR_PAD_CLASS (parent_pad_class)->clean_frame (pad,
1305 vagg, prepared_frame);
1306
1307 gst_clear_buffer (&cpad->fallback_buf);
1308 }
1309
1310 static gboolean
gst_d3d11_compositor_pad_setup_converter(GstVideoAggregatorPad * pad,GstVideoAggregator * vagg)1311 gst_d3d11_compositor_pad_setup_converter (GstVideoAggregatorPad * pad,
1312 GstVideoAggregator * vagg)
1313 {
1314 GstD3D11CompositorPad *cpad = GST_D3D11_COMPOSITOR_PAD (pad);
1315 GstD3D11Compositor *self = GST_D3D11_COMPOSITOR (vagg);
1316 RECT rect;
1317 gint width, height;
1318 GstVideoInfo *info = &vagg->info;
1319 GstVideoRectangle frame_rect;
1320 gboolean is_first = FALSE;
1321 gint x_offset, y_offset;
1322 #ifndef GST_DISABLE_GST_DEBUG
1323 guint zorder = 0;
1324 #endif
1325
1326 if (!cpad->convert || self->reconfigured) {
1327 GstStructure *config;
1328
1329 if (cpad->convert)
1330 gst_d3d11_converter_free (cpad->convert);
1331
1332 config = gst_structure_new_empty ("config");
1333 if (cpad->alpha <= 1.0) {
1334 gst_structure_set (config, GST_D3D11_CONVERTER_OPT_ALPHA_VALUE,
1335 G_TYPE_DOUBLE, cpad->alpha, nullptr);
1336 }
1337
1338 cpad->convert =
1339 gst_d3d11_converter_new (self->device, &pad->info, &vagg->info, config);
1340
1341 if (!cpad->convert) {
1342 GST_ERROR_OBJECT (pad, "Couldn't create converter");
1343 return FALSE;
1344 }
1345
1346 is_first = TRUE;
1347 } else if (cpad->alpha_updated) {
1348 GstStructure *config;
1349
1350 config = gst_structure_new_empty ("config");
1351 if (cpad->alpha <= 1.0) {
1352 gst_structure_set (config, GST_D3D11_CONVERTER_OPT_ALPHA_VALUE,
1353 G_TYPE_DOUBLE, cpad->alpha, nullptr);
1354 }
1355
1356 gst_d3d11_converter_update_config (cpad->convert, config);
1357 cpad->alpha_updated = FALSE;
1358 }
1359
1360 if (!cpad->blend || cpad->blend_desc_updated) {
1361 HRESULT hr;
1362 D3D11_BLEND_DESC desc = { 0, };
1363 ID3D11BlendState *blend = NULL;
1364 ID3D11Device *device_handle =
1365 gst_d3d11_device_get_device_handle (self->device);
1366
1367 GST_D3D11_CLEAR_COM (cpad->blend);
1368
1369 desc.AlphaToCoverageEnable = FALSE;
1370 desc.IndependentBlendEnable = FALSE;
1371 desc.RenderTarget[0] = cpad->desc;
1372
1373 hr = device_handle->CreateBlendState (&desc, &blend);
1374 if (!gst_d3d11_result (hr, self->device)) {
1375 GST_ERROR_OBJECT (pad, "Couldn't create blend staten, hr: 0x%x",
1376 (guint) hr);
1377 return FALSE;
1378 }
1379
1380 cpad->blend = blend;
1381 }
1382
1383 if (!is_first && !cpad->position_updated)
1384 return TRUE;
1385
1386 gst_d3d11_compositor_pad_get_output_size (cpad, GST_VIDEO_INFO_PAR_N (info),
1387 GST_VIDEO_INFO_PAR_D (info), &width, &height, &x_offset, &y_offset);
1388
1389 frame_rect = clamp_rectangle (cpad->xpos + x_offset, cpad->ypos + y_offset,
1390 width, height, GST_VIDEO_INFO_WIDTH (info), GST_VIDEO_INFO_HEIGHT (info));
1391
1392 rect.left = frame_rect.x;
1393 rect.top = frame_rect.y;
1394 rect.right = frame_rect.x + frame_rect.w;
1395 rect.bottom = frame_rect.y + frame_rect.h;
1396
1397 #ifndef GST_DISABLE_GST_DEBUG
1398 g_object_get (pad, "zorder", &zorder, NULL);
1399
1400 GST_LOG_OBJECT (pad, "Update position, pad-xpos %d, pad-ypos %d, "
1401 "pad-zorder %d, pad-width %d, pad-height %d, in-resolution %dx%d, "
1402 "out-resoution %dx%d, dst-{left,top,right,bottom} %d-%d-%d-%d",
1403 cpad->xpos, cpad->ypos, zorder, cpad->width, cpad->height,
1404 GST_VIDEO_INFO_WIDTH (&pad->info), GST_VIDEO_INFO_HEIGHT (&pad->info),
1405 GST_VIDEO_INFO_WIDTH (info), GST_VIDEO_INFO_HEIGHT (info),
1406 (gint) rect.left, (gint) rect.top, (gint) rect.right, (gint) rect.bottom);
1407 #endif
1408
1409 cpad->position_updated = FALSE;
1410
1411 return gst_d3d11_converter_update_dest_rect (cpad->convert, &rect);
1412 }
1413
1414 static GstStaticCaps pad_template_caps =
1415 GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE_WITH_FEATURES
1416 (GST_CAPS_FEATURE_MEMORY_D3D11_MEMORY, "{ RGBA, BGRA }"));
1417
1418 enum
1419 {
1420 PROP_0,
1421 PROP_ADAPTER,
1422 PROP_BACKGROUND,
1423 };
1424
1425 #define DEFAULT_ADAPTER -1
1426 #define DEFAULT_BACKGROUND GST_D3D11_COMPOSITOR_BACKGROUND_CHECKER
1427
1428 static void gst_d3d11_compositor_child_proxy_init (gpointer g_iface,
1429 gpointer iface_data);
1430 static void gst_d3d11_compositor_dispose (GObject * object);
1431 static void gst_d3d11_compositor_set_property (GObject * object,
1432 guint prop_id, const GValue * value, GParamSpec * pspec);
1433 static void gst_d3d11_compositor_get_property (GObject * object,
1434 guint prop_id, GValue * value, GParamSpec * pspec);
1435
1436 static GstPad *gst_d3d11_compositor_request_new_pad (GstElement * element,
1437 GstPadTemplate * templ, const gchar * name, const GstCaps * caps);
1438 static void gst_d3d11_compositor_release_pad (GstElement * element,
1439 GstPad * pad);
1440 static void gst_d3d11_compositor_set_context (GstElement * element,
1441 GstContext * context);
1442
1443 static gboolean gst_d3d11_compositor_start (GstAggregator * aggregator);
1444 static gboolean gst_d3d11_compositor_stop (GstAggregator * aggregator);
1445 static gboolean gst_d3d11_compositor_sink_query (GstAggregator * aggregator,
1446 GstAggregatorPad * pad, GstQuery * query);
1447 static gboolean gst_d3d11_compositor_src_query (GstAggregator * aggregator,
1448 GstQuery * query);
1449 static GstCaps *gst_d3d11_compositor_fixate_src_caps (GstAggregator *
1450 aggregator, GstCaps * caps);
1451 static gboolean gst_d3d11_compositor_propose_allocation (GstAggregator *
1452 aggregator, GstAggregatorPad * pad, GstQuery * decide_query,
1453 GstQuery * query);
1454 static gboolean gst_d3d11_compositor_decide_allocation (GstAggregator *
1455 aggregator, GstQuery * query);
1456 static GstFlowReturn
1457 gst_d3d11_compositor_aggregate_frames (GstVideoAggregator * vagg,
1458 GstBuffer * outbuf);
1459 static GstFlowReturn
1460 gst_d3d11_compositor_create_output_buffer (GstVideoAggregator * vagg,
1461 GstBuffer ** outbuffer);
1462
1463 #define gst_d3d11_compositor_parent_class parent_class
1464 G_DEFINE_TYPE_WITH_CODE (GstD3D11Compositor, gst_d3d11_compositor,
1465 GST_TYPE_VIDEO_AGGREGATOR, G_IMPLEMENT_INTERFACE (GST_TYPE_CHILD_PROXY,
1466 gst_d3d11_compositor_child_proxy_init));
1467
1468 static void
gst_d3d11_compositor_class_init(GstD3D11CompositorClass * klass)1469 gst_d3d11_compositor_class_init (GstD3D11CompositorClass * klass)
1470 {
1471 GObjectClass *gobject_class = G_OBJECT_CLASS (klass);
1472 GstElementClass *element_class = GST_ELEMENT_CLASS (klass);
1473 GstAggregatorClass *aggregator_class = GST_AGGREGATOR_CLASS (klass);
1474 GstVideoAggregatorClass *vagg_class = GST_VIDEO_AGGREGATOR_CLASS (klass);
1475 GstCaps *caps;
1476
1477 gobject_class->dispose = gst_d3d11_compositor_dispose;
1478 gobject_class->set_property = gst_d3d11_compositor_set_property;
1479 gobject_class->get_property = gst_d3d11_compositor_get_property;
1480
1481 g_object_class_install_property (gobject_class, PROP_ADAPTER,
1482 g_param_spec_int ("adapter", "Adapter",
1483 "Adapter index for creating device (-1 for default)",
1484 -1, G_MAXINT32, DEFAULT_ADAPTER,
1485 (GParamFlags) (G_PARAM_READWRITE | GST_PARAM_MUTABLE_READY |
1486 G_PARAM_STATIC_STRINGS)));
1487
1488 g_object_class_install_property (gobject_class, PROP_BACKGROUND,
1489 g_param_spec_enum ("background", "Background", "Background type",
1490 GST_TYPE_D3D11_COMPOSITOR_BACKGROUND,
1491 DEFAULT_BACKGROUND,
1492 (GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
1493
1494 element_class->request_new_pad =
1495 GST_DEBUG_FUNCPTR (gst_d3d11_compositor_request_new_pad);
1496 element_class->release_pad =
1497 GST_DEBUG_FUNCPTR (gst_d3d11_compositor_release_pad);
1498 element_class->set_context =
1499 GST_DEBUG_FUNCPTR (gst_d3d11_compositor_set_context);
1500
1501 aggregator_class->start = GST_DEBUG_FUNCPTR (gst_d3d11_compositor_start);
1502 aggregator_class->stop = GST_DEBUG_FUNCPTR (gst_d3d11_compositor_stop);
1503 aggregator_class->sink_query =
1504 GST_DEBUG_FUNCPTR (gst_d3d11_compositor_sink_query);
1505 aggregator_class->src_query =
1506 GST_DEBUG_FUNCPTR (gst_d3d11_compositor_src_query);
1507 aggregator_class->fixate_src_caps =
1508 GST_DEBUG_FUNCPTR (gst_d3d11_compositor_fixate_src_caps);
1509 aggregator_class->propose_allocation =
1510 GST_DEBUG_FUNCPTR (gst_d3d11_compositor_propose_allocation);
1511 aggregator_class->decide_allocation =
1512 GST_DEBUG_FUNCPTR (gst_d3d11_compositor_decide_allocation);
1513
1514 vagg_class->aggregate_frames =
1515 GST_DEBUG_FUNCPTR (gst_d3d11_compositor_aggregate_frames);
1516 vagg_class->create_output_buffer =
1517 GST_DEBUG_FUNCPTR (gst_d3d11_compositor_create_output_buffer);
1518
1519 caps = gst_d3d11_get_updated_template_caps (&pad_template_caps);
1520 gst_element_class_add_pad_template (element_class,
1521 gst_pad_template_new_with_gtype ("sink_%u", GST_PAD_SINK, GST_PAD_REQUEST,
1522 caps, GST_TYPE_D3D11_COMPOSITOR_PAD));
1523
1524 gst_element_class_add_pad_template (element_class,
1525 gst_pad_template_new_with_gtype ("src", GST_PAD_SRC, GST_PAD_ALWAYS,
1526 caps, GST_TYPE_AGGREGATOR_PAD));
1527 gst_caps_unref (caps);
1528
1529 gst_element_class_set_static_metadata (element_class, "Direct3D11 Compositor",
1530 "Filter/Editor/Video/Compositor",
1531 "A Direct3D11 compositor", "Seungha Yang <seungha@centricular.com>");
1532
1533 gst_type_mark_as_plugin_api (GST_TYPE_D3D11_COMPOSITOR_BACKGROUND,
1534 (GstPluginAPIFlags) 0);
1535 gst_type_mark_as_plugin_api (GST_TYPE_D3D11_COMPOSITOR_PAD,
1536 (GstPluginAPIFlags) 0);
1537 }
1538
1539 static void
gst_d3d11_compositor_init(GstD3D11Compositor * self)1540 gst_d3d11_compositor_init (GstD3D11Compositor * self)
1541 {
1542 self->adapter = DEFAULT_ADAPTER;
1543 self->background = DEFAULT_BACKGROUND;
1544 }
1545
1546 static void
gst_d3d11_compositor_dispose(GObject * object)1547 gst_d3d11_compositor_dispose (GObject * object)
1548 {
1549 GstD3D11Compositor *self = GST_D3D11_COMPOSITOR (object);
1550
1551 gst_clear_object (&self->device);
1552 gst_clear_buffer (&self->fallback_buf);
1553 gst_clear_object (&self->fallback_pool);
1554 g_clear_pointer (&self->checker_background, gst_d3d11_quad_free);
1555
1556 G_OBJECT_CLASS (parent_class)->dispose (object);
1557 }
1558
1559 static void
gst_d3d11_compositor_set_property(GObject * object,guint prop_id,const GValue * value,GParamSpec * pspec)1560 gst_d3d11_compositor_set_property (GObject * object,
1561 guint prop_id, const GValue * value, GParamSpec * pspec)
1562 {
1563 GstD3D11Compositor *self = GST_D3D11_COMPOSITOR (object);
1564
1565 switch (prop_id) {
1566 case PROP_ADAPTER:
1567 self->adapter = g_value_get_int (value);
1568 break;
1569 case PROP_BACKGROUND:
1570 self->background =
1571 (GstD3D11CompositorBackground) g_value_get_enum (value);
1572 break;
1573 default:
1574 G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
1575 break;
1576 }
1577 }
1578
1579 static void
gst_d3d11_compositor_get_property(GObject * object,guint prop_id,GValue * value,GParamSpec * pspec)1580 gst_d3d11_compositor_get_property (GObject * object,
1581 guint prop_id, GValue * value, GParamSpec * pspec)
1582 {
1583 GstD3D11Compositor *self = GST_D3D11_COMPOSITOR (object);
1584
1585 switch (prop_id) {
1586 case PROP_ADAPTER:
1587 g_value_set_int (value, self->adapter);
1588 break;
1589 case PROP_BACKGROUND:
1590 g_value_set_enum (value, self->background);
1591 break;
1592 default:
1593 G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
1594 break;
1595 }
1596 }
1597
1598 static GObject *
gst_d3d11_compositor_child_proxy_get_child_by_index(GstChildProxy * proxy,guint index)1599 gst_d3d11_compositor_child_proxy_get_child_by_index (GstChildProxy * proxy,
1600 guint index)
1601 {
1602 GstD3D11Compositor *self = GST_D3D11_COMPOSITOR (proxy);
1603 GObject *obj = NULL;
1604
1605 GST_OBJECT_LOCK (self);
1606 obj = (GObject *) g_list_nth_data (GST_ELEMENT_CAST (self)->sinkpads, index);
1607 if (obj)
1608 gst_object_ref (obj);
1609 GST_OBJECT_UNLOCK (self);
1610
1611 return obj;
1612 }
1613
1614 static guint
gst_d3d11_compositor_child_proxy_get_children_count(GstChildProxy * proxy)1615 gst_d3d11_compositor_child_proxy_get_children_count (GstChildProxy * proxy)
1616 {
1617 GstD3D11Compositor *self = GST_D3D11_COMPOSITOR (proxy);
1618 guint count = 0;
1619
1620 GST_OBJECT_LOCK (self);
1621 count = GST_ELEMENT_CAST (self)->numsinkpads;
1622 GST_OBJECT_UNLOCK (self);
1623 GST_INFO_OBJECT (self, "Children Count: %d", count);
1624
1625 return count;
1626 }
1627
1628 static void
gst_d3d11_compositor_child_proxy_init(gpointer g_iface,gpointer iface_data)1629 gst_d3d11_compositor_child_proxy_init (gpointer g_iface, gpointer iface_data)
1630 {
1631 GstChildProxyInterface *iface = (GstChildProxyInterface *) g_iface;
1632
1633 iface->get_child_by_index =
1634 gst_d3d11_compositor_child_proxy_get_child_by_index;
1635 iface->get_children_count =
1636 gst_d3d11_compositor_child_proxy_get_children_count;
1637 }
1638
1639 static GstPad *
gst_d3d11_compositor_request_new_pad(GstElement * element,GstPadTemplate * templ,const gchar * name,const GstCaps * caps)1640 gst_d3d11_compositor_request_new_pad (GstElement * element,
1641 GstPadTemplate * templ, const gchar * name, const GstCaps * caps)
1642 {
1643 GstPad *pad;
1644
1645 pad = GST_ELEMENT_CLASS (parent_class)->request_new_pad (element,
1646 templ, name, caps);
1647
1648 if (pad == NULL)
1649 goto could_not_create;
1650
1651 gst_child_proxy_child_added (GST_CHILD_PROXY (element), G_OBJECT (pad),
1652 GST_OBJECT_NAME (pad));
1653
1654 GST_DEBUG_OBJECT (element, "Created new pad %s:%s", GST_DEBUG_PAD_NAME (pad));
1655
1656 return pad;
1657
1658 could_not_create:
1659 {
1660 GST_DEBUG_OBJECT (element, "could not create/add pad");
1661 return NULL;
1662 }
1663 }
1664
1665 static gboolean
gst_d3d11_compositor_pad_clear_resource(GstD3D11Compositor * self,GstD3D11CompositorPad * cpad,gpointer user_data)1666 gst_d3d11_compositor_pad_clear_resource (GstD3D11Compositor * self,
1667 GstD3D11CompositorPad * cpad, gpointer user_data)
1668 {
1669 gst_clear_buffer (&cpad->fallback_buf);
1670 if (cpad->fallback_pool) {
1671 gst_buffer_pool_set_active (cpad->fallback_pool, FALSE);
1672 gst_clear_object (&cpad->fallback_pool);
1673 }
1674 g_clear_pointer (&cpad->convert, gst_d3d11_converter_free);
1675 GST_D3D11_CLEAR_COM (cpad->blend);
1676
1677 return TRUE;
1678 }
1679
1680 static void
gst_d3d11_compositor_release_pad(GstElement * element,GstPad * pad)1681 gst_d3d11_compositor_release_pad (GstElement * element, GstPad * pad)
1682 {
1683 GstD3D11Compositor *self = GST_D3D11_COMPOSITOR (element);
1684 GstD3D11CompositorPad *cpad = GST_D3D11_COMPOSITOR_PAD (pad);
1685
1686 GST_DEBUG_OBJECT (self, "Releasing pad %s:%s", GST_DEBUG_PAD_NAME (pad));
1687
1688 gst_child_proxy_child_removed (GST_CHILD_PROXY (self), G_OBJECT (pad),
1689 GST_OBJECT_NAME (pad));
1690
1691 gst_d3d11_compositor_pad_clear_resource (self, cpad, NULL);
1692
1693 GST_ELEMENT_CLASS (parent_class)->release_pad (element, pad);
1694 }
1695
1696 static void
gst_d3d11_compositor_set_context(GstElement * element,GstContext * context)1697 gst_d3d11_compositor_set_context (GstElement * element, GstContext * context)
1698 {
1699 GstD3D11Compositor *self = GST_D3D11_COMPOSITOR (element);
1700
1701 gst_d3d11_handle_set_context (element, context, self->adapter, &self->device);
1702
1703 GST_ELEMENT_CLASS (parent_class)->set_context (element, context);
1704 }
1705
1706 static gboolean
gst_d3d11_compositor_start(GstAggregator * aggregator)1707 gst_d3d11_compositor_start (GstAggregator * aggregator)
1708 {
1709 GstD3D11Compositor *self = GST_D3D11_COMPOSITOR (aggregator);
1710
1711 if (!gst_d3d11_ensure_element_data (GST_ELEMENT_CAST (self),
1712 self->adapter, &self->device)) {
1713 GST_ERROR_OBJECT (self, "Failed to get D3D11 device");
1714 return FALSE;
1715 }
1716
1717 return GST_AGGREGATOR_CLASS (parent_class)->start (aggregator);
1718 }
1719
1720 static gboolean
gst_d3d11_compositor_stop(GstAggregator * aggregator)1721 gst_d3d11_compositor_stop (GstAggregator * aggregator)
1722 {
1723 GstD3D11Compositor *self = GST_D3D11_COMPOSITOR (aggregator);
1724
1725 g_clear_pointer (&self->checker_background, gst_d3d11_quad_free);
1726 gst_clear_object (&self->device);
1727
1728 return GST_AGGREGATOR_CLASS (parent_class)->stop (aggregator);
1729 }
1730
1731 static GstCaps *
gst_d3d11_compositor_sink_getcaps(GstPad * pad,GstCaps * filter)1732 gst_d3d11_compositor_sink_getcaps (GstPad * pad, GstCaps * filter)
1733 {
1734 GstCaps *sinkcaps;
1735 GstCaps *template_caps;
1736 GstCaps *filtered_caps;
1737 GstCaps *returned_caps;
1738
1739 template_caps = gst_pad_get_pad_template_caps (pad);
1740
1741 sinkcaps = gst_pad_get_current_caps (pad);
1742 if (sinkcaps == NULL) {
1743 sinkcaps = gst_caps_ref (template_caps);
1744 } else {
1745 sinkcaps = gst_caps_merge (sinkcaps, gst_caps_ref (template_caps));
1746 }
1747
1748 if (filter) {
1749 filtered_caps = gst_caps_intersect (sinkcaps, filter);
1750 gst_caps_unref (sinkcaps);
1751 } else {
1752 filtered_caps = sinkcaps; /* pass ownership */
1753 }
1754
1755 returned_caps = gst_caps_intersect (filtered_caps, template_caps);
1756
1757 gst_caps_unref (template_caps);
1758 gst_caps_unref (filtered_caps);
1759
1760 GST_DEBUG_OBJECT (pad, "returning %" GST_PTR_FORMAT, returned_caps);
1761
1762 return returned_caps;
1763 }
1764
1765 static gboolean
gst_d3d11_compositor_sink_acceptcaps(GstPad * pad,GstCaps * caps)1766 gst_d3d11_compositor_sink_acceptcaps (GstPad * pad, GstCaps * caps)
1767 {
1768 gboolean ret;
1769 GstCaps *template_caps;
1770
1771 GST_DEBUG_OBJECT (pad, "try accept caps of %" GST_PTR_FORMAT, caps);
1772
1773 template_caps = gst_pad_get_pad_template_caps (pad);
1774 template_caps = gst_caps_make_writable (template_caps);
1775
1776 ret = gst_caps_can_intersect (caps, template_caps);
1777 GST_DEBUG_OBJECT (pad, "%saccepted caps %" GST_PTR_FORMAT,
1778 (ret ? "" : "not "), caps);
1779 gst_caps_unref (template_caps);
1780
1781 return ret;
1782 }
1783
1784 static gboolean
gst_d3d11_compositor_sink_query(GstAggregator * aggregator,GstAggregatorPad * pad,GstQuery * query)1785 gst_d3d11_compositor_sink_query (GstAggregator * aggregator,
1786 GstAggregatorPad * pad, GstQuery * query)
1787 {
1788 GstD3D11Compositor *self = GST_D3D11_COMPOSITOR (aggregator);
1789
1790 switch (GST_QUERY_TYPE (query)) {
1791 case GST_QUERY_CONTEXT:
1792 {
1793 gboolean ret;
1794 ret = gst_d3d11_handle_context_query (GST_ELEMENT (aggregator), query,
1795 self->device);
1796 if (ret)
1797 return TRUE;
1798 break;
1799 }
1800 case GST_QUERY_CAPS:
1801 {
1802 GstCaps *filter, *caps;
1803
1804 gst_query_parse_caps (query, &filter);
1805 caps = gst_d3d11_compositor_sink_getcaps (GST_PAD (pad), filter);
1806 gst_query_set_caps_result (query, caps);
1807 gst_caps_unref (caps);
1808 return TRUE;
1809 }
1810 case GST_QUERY_ACCEPT_CAPS:
1811 {
1812 GstCaps *caps;
1813 gboolean ret;
1814
1815 gst_query_parse_accept_caps (query, &caps);
1816 ret = gst_d3d11_compositor_sink_acceptcaps (GST_PAD (pad), caps);
1817 gst_query_set_accept_caps_result (query, ret);
1818 return TRUE;
1819 }
1820 default:
1821 break;
1822 }
1823
1824 return GST_AGGREGATOR_CLASS (parent_class)->sink_query (aggregator,
1825 pad, query);
1826 }
1827
1828 static gboolean
gst_d3d11_compositor_src_query(GstAggregator * aggregator,GstQuery * query)1829 gst_d3d11_compositor_src_query (GstAggregator * aggregator, GstQuery * query)
1830 {
1831 GstD3D11Compositor *self = GST_D3D11_COMPOSITOR (aggregator);
1832
1833 switch (GST_QUERY_TYPE (query)) {
1834 case GST_QUERY_CONTEXT:
1835 {
1836 gboolean ret;
1837 ret = gst_d3d11_handle_context_query (GST_ELEMENT (aggregator), query,
1838 self->device);
1839 if (ret)
1840 return TRUE;
1841 break;
1842 }
1843 default:
1844 break;
1845 }
1846
1847 return GST_AGGREGATOR_CLASS (parent_class)->src_query (aggregator, query);
1848 }
1849
1850 static GstCaps *
gst_d3d11_compositor_fixate_src_caps(GstAggregator * aggregator,GstCaps * caps)1851 gst_d3d11_compositor_fixate_src_caps (GstAggregator * aggregator,
1852 GstCaps * caps)
1853 {
1854 GstVideoAggregator *vagg = GST_VIDEO_AGGREGATOR (aggregator);
1855 GList *l;
1856 gint best_width = -1, best_height = -1;
1857 gint best_fps_n = -1, best_fps_d = -1;
1858 gint par_n, par_d;
1859 gdouble best_fps = 0.;
1860 GstCaps *ret = NULL;
1861 GstStructure *s;
1862
1863 ret = gst_caps_make_writable (caps);
1864
1865 /* we need this to calculate how large to make the output frame */
1866 s = gst_caps_get_structure (ret, 0);
1867 if (gst_structure_has_field (s, "pixel-aspect-ratio")) {
1868 gst_structure_fixate_field_nearest_fraction (s, "pixel-aspect-ratio", 1, 1);
1869 gst_structure_get_fraction (s, "pixel-aspect-ratio", &par_n, &par_d);
1870 } else {
1871 par_n = par_d = 1;
1872 }
1873
1874 GST_OBJECT_LOCK (vagg);
1875 for (l = GST_ELEMENT (vagg)->sinkpads; l; l = l->next) {
1876 GstVideoAggregatorPad *vaggpad = GST_VIDEO_AGGREGATOR_PAD (l->data);
1877 GstD3D11CompositorPad *cpad = GST_D3D11_COMPOSITOR_PAD (vaggpad);
1878 gint this_width, this_height;
1879 gint width, height;
1880 gint fps_n, fps_d;
1881 gdouble cur_fps;
1882 gint x_offset;
1883 gint y_offset;
1884
1885 fps_n = GST_VIDEO_INFO_FPS_N (&vaggpad->info);
1886 fps_d = GST_VIDEO_INFO_FPS_D (&vaggpad->info);
1887 gst_d3d11_compositor_pad_get_output_size (cpad,
1888 par_n, par_d, &width, &height, &x_offset, &y_offset);
1889
1890 if (width == 0 || height == 0)
1891 continue;
1892
1893 /* {x,y}_offset represent padding size of each top and left area.
1894 * To calculate total resolution, count bottom and right padding area
1895 * as well here */
1896 this_width = width + MAX (cpad->xpos + 2 * x_offset, 0);
1897 this_height = height + MAX (cpad->ypos + 2 * y_offset, 0);
1898
1899 if (best_width < this_width)
1900 best_width = this_width;
1901 if (best_height < this_height)
1902 best_height = this_height;
1903
1904 if (fps_d == 0)
1905 cur_fps = 0.0;
1906 else
1907 gst_util_fraction_to_double (fps_n, fps_d, &cur_fps);
1908
1909 if (best_fps < cur_fps) {
1910 best_fps = cur_fps;
1911 best_fps_n = fps_n;
1912 best_fps_d = fps_d;
1913 }
1914 }
1915 GST_OBJECT_UNLOCK (vagg);
1916
1917 if (best_fps_n <= 0 || best_fps_d <= 0 || best_fps == 0.0) {
1918 best_fps_n = 25;
1919 best_fps_d = 1;
1920 best_fps = 25.0;
1921 }
1922
1923 gst_structure_fixate_field_nearest_int (s, "width", best_width);
1924 gst_structure_fixate_field_nearest_int (s, "height", best_height);
1925 gst_structure_fixate_field_nearest_fraction (s, "framerate", best_fps_n,
1926 best_fps_d);
1927 ret = gst_caps_fixate (ret);
1928
1929 GST_LOG_OBJECT (aggregator, "Fixated caps %" GST_PTR_FORMAT, ret);
1930
1931 return ret;
1932 }
1933
1934 static gboolean
gst_d3d11_compositor_propose_allocation(GstAggregator * aggregator,GstAggregatorPad * pad,GstQuery * decide_query,GstQuery * query)1935 gst_d3d11_compositor_propose_allocation (GstAggregator * aggregator,
1936 GstAggregatorPad * pad, GstQuery * decide_query, GstQuery * query)
1937 {
1938 GstD3D11Compositor *self = GST_D3D11_COMPOSITOR (aggregator);
1939 GstVideoInfo info;
1940 GstBufferPool *pool;
1941 GstCaps *caps;
1942 guint size;
1943
1944 gst_query_parse_allocation (query, &caps, NULL);
1945
1946 if (caps == NULL)
1947 return FALSE;
1948
1949 if (!gst_video_info_from_caps (&info, caps))
1950 return FALSE;
1951
1952 if (gst_query_get_n_allocation_pools (query) == 0) {
1953 GstD3D11AllocationParams *d3d11_params;
1954 GstStructure *config;
1955
1956 d3d11_params = gst_d3d11_allocation_params_new (self->device, &info,
1957 (GstD3D11AllocationFlags) 0, D3D11_BIND_SHADER_RESOURCE);
1958
1959 pool = gst_d3d11_buffer_pool_new_with_options (self->device,
1960 caps, d3d11_params, 0, 0);
1961 gst_d3d11_allocation_params_free (d3d11_params);
1962
1963 if (!pool) {
1964 GST_ERROR_OBJECT (self, "Failed to create buffer pool");
1965 return FALSE;
1966 }
1967
1968 /* d3d11 buffer pool will update buffer size based on allocated texture,
1969 * get size from config again */
1970 config = gst_buffer_pool_get_config (pool);
1971 gst_buffer_pool_config_get_params (config,
1972 nullptr, &size, nullptr, nullptr);
1973 gst_structure_free (config);
1974
1975 gst_query_add_allocation_pool (query, pool, size, 0, 0);
1976 gst_object_unref (pool);
1977 }
1978
1979 gst_query_add_allocation_meta (query, GST_VIDEO_META_API_TYPE, NULL);
1980
1981 return TRUE;
1982 }
1983
1984 static gboolean
gst_d3d11_compositor_decide_allocation(GstAggregator * aggregator,GstQuery * query)1985 gst_d3d11_compositor_decide_allocation (GstAggregator * aggregator,
1986 GstQuery * query)
1987 {
1988 GstD3D11Compositor *self = GST_D3D11_COMPOSITOR (aggregator);
1989 GstCaps *caps;
1990 GstBufferPool *pool = NULL;
1991 guint n, size, min, max;
1992 GstVideoInfo info;
1993 GstStructure *config;
1994 GstD3D11AllocationParams *d3d11_params;
1995
1996 gst_query_parse_allocation (query, &caps, NULL);
1997
1998 if (!caps) {
1999 GST_DEBUG_OBJECT (self, "No output caps");
2000 return FALSE;
2001 }
2002
2003 gst_video_info_from_caps (&info, caps);
2004 n = gst_query_get_n_allocation_pools (query);
2005 if (n > 0)
2006 gst_query_parse_nth_allocation_pool (query, 0, &pool, &size, &min, &max);
2007
2008 /* create our own pool */
2009 if (pool) {
2010 if (!GST_IS_D3D11_BUFFER_POOL (pool)) {
2011 gst_clear_object (&pool);
2012 } else {
2013 GstD3D11BufferPool *dpool = GST_D3D11_BUFFER_POOL (pool);
2014 if (dpool->device != self->device)
2015 gst_clear_object (&pool);
2016 }
2017 }
2018
2019 if (!pool) {
2020 pool = gst_d3d11_buffer_pool_new (self->device);
2021
2022 min = max = 0;
2023 size = (guint) info.size;
2024 }
2025
2026 config = gst_buffer_pool_get_config (pool);
2027 gst_buffer_pool_config_set_params (config, caps, size, min, max);
2028 gst_buffer_pool_config_add_option (config, GST_BUFFER_POOL_OPTION_VIDEO_META);
2029
2030 d3d11_params = gst_buffer_pool_config_get_d3d11_allocation_params (config);
2031 if (!d3d11_params) {
2032 d3d11_params = gst_d3d11_allocation_params_new (self->device,
2033 &info, (GstD3D11AllocationFlags) 0, D3D11_BIND_RENDER_TARGET);
2034 } else {
2035 guint i;
2036
2037 for (i = 0; i < GST_VIDEO_INFO_N_PLANES (&info); i++) {
2038 d3d11_params->desc[i].BindFlags |= D3D11_BIND_RENDER_TARGET;
2039 }
2040 }
2041
2042 gst_buffer_pool_config_set_d3d11_allocation_params (config, d3d11_params);
2043 gst_d3d11_allocation_params_free (d3d11_params);
2044
2045 gst_buffer_pool_set_config (pool, config);
2046
2047 /* d3d11 buffer pool will update buffer size based on allocated texture,
2048 * get size from config again */
2049 config = gst_buffer_pool_get_config (pool);
2050 gst_buffer_pool_config_get_params (config, nullptr, &size, nullptr, nullptr);
2051 gst_structure_free (config);
2052
2053 if (n > 0)
2054 gst_query_set_nth_allocation_pool (query, 0, pool, size, min, max);
2055 else
2056 gst_query_add_allocation_pool (query, pool, size, min, max);
2057 gst_object_unref (pool);
2058
2059 self->reconfigured = TRUE;
2060
2061 return TRUE;
2062 }
2063
2064 typedef struct
2065 {
2066 struct
2067 {
2068 FLOAT x;
2069 FLOAT y;
2070 FLOAT z;
2071 } position;
2072 struct
2073 {
2074 FLOAT u;
2075 FLOAT v;
2076 } texture;
2077 } VertexData;
2078
2079 static GstD3D11Quad *
gst_d3d11_compositor_create_checker_quad(GstD3D11Compositor * self)2080 gst_d3d11_compositor_create_checker_quad (GstD3D11Compositor * self)
2081 {
2082 GstD3D11Quad *quad = NULL;
2083 VertexData *vertex_data;
2084 WORD *indices;
2085 ID3D11Device *device_handle;
2086 ID3D11DeviceContext *context_handle;
2087 D3D11_MAPPED_SUBRESOURCE map;
2088 D3D11_INPUT_ELEMENT_DESC input_desc;
2089 D3D11_BUFFER_DESC buffer_desc;
2090 /* *INDENT-OFF* */
2091 ComPtr<ID3D11Buffer> vertex_buffer;
2092 ComPtr<ID3D11Buffer> index_buffer;
2093 ComPtr<ID3D11PixelShader> ps;
2094 ComPtr<ID3D11VertexShader> vs;
2095 ComPtr<ID3D11InputLayout> layout;
2096 /* *INDENT-ON* */
2097 HRESULT hr;
2098
2099 device_handle = gst_d3d11_device_get_device_handle (self->device);
2100 context_handle = gst_d3d11_device_get_device_context_handle (self->device);
2101
2102 if (!gst_d3d11_create_pixel_shader (self->device, checker_ps_src, &ps)) {
2103 GST_ERROR_OBJECT (self, "Couldn't setup pixel shader");
2104 return NULL;
2105 }
2106
2107 memset (&input_desc, 0, sizeof (D3D11_INPUT_ELEMENT_DESC));
2108 input_desc.SemanticName = "POSITION";
2109 input_desc.SemanticIndex = 0;
2110 input_desc.Format = DXGI_FORMAT_R32G32B32_FLOAT;
2111 input_desc.InputSlot = 0;
2112 input_desc.AlignedByteOffset = D3D11_APPEND_ALIGNED_ELEMENT;
2113 input_desc.InputSlotClass = D3D11_INPUT_PER_VERTEX_DATA;
2114 input_desc.InstanceDataStepRate = 0;
2115
2116 if (!gst_d3d11_create_vertex_shader (self->device, checker_vs_src,
2117 &input_desc, 1, &vs, &layout)) {
2118 GST_ERROR_OBJECT (self, "Couldn't setup vertex shader");
2119 return NULL;
2120 }
2121
2122 memset (&buffer_desc, 0, sizeof (D3D11_BUFFER_DESC));
2123 buffer_desc.Usage = D3D11_USAGE_DYNAMIC;
2124 buffer_desc.ByteWidth = sizeof (VertexData) * 4;
2125 buffer_desc.BindFlags = D3D11_BIND_VERTEX_BUFFER;
2126 buffer_desc.CPUAccessFlags = D3D11_CPU_ACCESS_WRITE;
2127
2128 hr = device_handle->CreateBuffer (&buffer_desc, NULL, &vertex_buffer);
2129 if (!gst_d3d11_result (hr, self->device)) {
2130 GST_ERROR_OBJECT (self,
2131 "Couldn't create vertex buffer, hr: 0x%x", (guint) hr);
2132 return NULL;
2133 }
2134
2135 hr = context_handle->Map (vertex_buffer.Get (),
2136 0, D3D11_MAP_WRITE_DISCARD, 0, &map);
2137
2138 if (!gst_d3d11_result (hr, self->device)) {
2139 GST_ERROR_OBJECT (self, "Couldn't map vertex buffer, hr: 0x%x", (guint) hr);
2140 return NULL;
2141 }
2142
2143 vertex_data = (VertexData *) map.pData;
2144 /* bottom left */
2145 /* bottom left */
2146 vertex_data[0].position.x = -1.0f;
2147 vertex_data[0].position.y = -1.0f;
2148 vertex_data[0].position.z = 0.0f;
2149 vertex_data[0].texture.u = 0.0f;
2150 vertex_data[0].texture.v = 1.0f;
2151
2152 /* top left */
2153 vertex_data[1].position.x = -1.0f;
2154 vertex_data[1].position.y = 1.0f;
2155 vertex_data[1].position.z = 0.0f;
2156 vertex_data[1].texture.u = 0.0f;
2157 vertex_data[1].texture.v = 0.0f;
2158
2159 /* top right */
2160 vertex_data[2].position.x = 1.0f;
2161 vertex_data[2].position.y = 1.0f;
2162 vertex_data[2].position.z = 0.0f;
2163 vertex_data[2].texture.u = 1.0f;
2164 vertex_data[2].texture.v = 0.0f;
2165
2166 /* bottom right */
2167 vertex_data[3].position.x = 1.0f;
2168 vertex_data[3].position.y = -1.0f;
2169 vertex_data[3].position.z = 0.0f;
2170 vertex_data[3].texture.u = 1.0f;
2171 vertex_data[3].texture.v = 1.0f;
2172
2173 context_handle->Unmap (vertex_buffer.Get (), 0);
2174
2175 buffer_desc.Usage = D3D11_USAGE_DYNAMIC;
2176 buffer_desc.ByteWidth = sizeof (WORD) * 6;
2177 buffer_desc.BindFlags = D3D11_BIND_INDEX_BUFFER;
2178 buffer_desc.CPUAccessFlags = D3D11_CPU_ACCESS_WRITE;
2179
2180 hr = device_handle->CreateBuffer (&buffer_desc, NULL, &index_buffer);
2181 if (!gst_d3d11_result (hr, self->device)) {
2182 GST_ERROR_OBJECT (self,
2183 "Couldn't create index buffer, hr: 0x%x", (guint) hr);
2184 return NULL;
2185 }
2186
2187 hr = context_handle->Map (index_buffer.Get (),
2188 0, D3D11_MAP_WRITE_DISCARD, 0, &map);
2189
2190 if (!gst_d3d11_result (hr, self->device)) {
2191 GST_ERROR_OBJECT (self, "Couldn't map index buffer, hr: 0x%x", (guint) hr);
2192 return NULL;
2193 }
2194
2195 indices = (WORD *) map.pData;
2196
2197 /* clockwise indexing */
2198 indices[0] = 0; /* bottom left */
2199 indices[1] = 1; /* top left */
2200 indices[2] = 2; /* top right */
2201
2202 indices[3] = 3; /* bottom right */
2203 indices[4] = 0; /* bottom left */
2204 indices[5] = 2; /* top right */
2205
2206 context_handle->Unmap (index_buffer.Get (), 0);
2207
2208 quad = gst_d3d11_quad_new (self->device,
2209 ps.Get (), vs.Get (), layout.Get (), nullptr, 0,
2210 vertex_buffer.Get (), sizeof (VertexData), index_buffer.Get (),
2211 DXGI_FORMAT_R16_UINT, 6);
2212 if (!quad) {
2213 GST_ERROR_OBJECT (self, "Couldn't setup quad");
2214 return NULL;
2215 }
2216
2217 return quad;
2218 }
2219
2220 static gboolean
gst_d3d11_compositor_draw_background_checker(GstD3D11Compositor * self,ID3D11RenderTargetView * rtv)2221 gst_d3d11_compositor_draw_background_checker (GstD3D11Compositor * self,
2222 ID3D11RenderTargetView * rtv)
2223 {
2224 if (!self->checker_background) {
2225 GstVideoInfo *info = &GST_VIDEO_AGGREGATOR_CAST (self)->info;
2226
2227 self->checker_background = gst_d3d11_compositor_create_checker_quad (self);
2228
2229 if (!self->checker_background)
2230 return FALSE;
2231
2232 self->viewport.TopLeftX = 0;
2233 self->viewport.TopLeftY = 0;
2234 self->viewport.Width = GST_VIDEO_INFO_WIDTH (info);
2235 self->viewport.Height = GST_VIDEO_INFO_HEIGHT (info);
2236 self->viewport.MinDepth = 0.0f;
2237 self->viewport.MaxDepth = 1.0f;
2238 }
2239
2240 return gst_d3d11_draw_quad_unlocked (self->checker_background,
2241 &self->viewport, 1, NULL, 0, &rtv, 1, NULL, NULL, NULL, 0);
2242 }
2243
2244 /* Must be called with d3d11 device lock */
2245 static gboolean
gst_d3d11_compositor_draw_background(GstD3D11Compositor * self,ID3D11RenderTargetView * rtv)2246 gst_d3d11_compositor_draw_background (GstD3D11Compositor * self,
2247 ID3D11RenderTargetView * rtv)
2248 {
2249 ID3D11DeviceContext *device_context =
2250 gst_d3d11_device_get_device_context_handle (self->device);
2251 FLOAT rgba[4] = { 0.0f, 0.0f, 0.0f, 1.0f };
2252
2253 switch (self->background) {
2254 case GST_D3D11_COMPOSITOR_BACKGROUND_CHECKER:
2255 return gst_d3d11_compositor_draw_background_checker (self, rtv);
2256 case GST_D3D11_COMPOSITOR_BACKGROUND_BLACK:
2257 /* {0, 0, 0, 1} */
2258 break;
2259 case GST_D3D11_COMPOSITOR_BACKGROUND_WHITE:
2260 rgba[0] = 1.0f;
2261 rgba[1] = 1.0f;
2262 rgba[2] = 1.0f;
2263 break;
2264 case GST_D3D11_COMPOSITOR_BACKGROUND_TRANSPARENT:
2265 rgba[3] = 0.0f;
2266 break;
2267 default:
2268 g_assert_not_reached ();
2269 return FALSE;
2270 }
2271
2272 device_context->ClearRenderTargetView (rtv, rgba);
2273
2274 return TRUE;
2275 }
2276
2277 static GstFlowReturn
gst_d3d11_compositor_aggregate_frames(GstVideoAggregator * vagg,GstBuffer * outbuf)2278 gst_d3d11_compositor_aggregate_frames (GstVideoAggregator * vagg,
2279 GstBuffer * outbuf)
2280 {
2281 GstD3D11Compositor *self = GST_D3D11_COMPOSITOR (vagg);
2282 GList *iter;
2283 GstBuffer *target_buf = outbuf;
2284 gboolean need_copy = FALSE;
2285 gboolean do_device_copy = FALSE;
2286 GstFlowReturn ret = GST_FLOW_OK;
2287 ID3D11RenderTargetView *rtv[GST_VIDEO_MAX_PLANES] = { NULL, };
2288 guint i, j;
2289 gint view_idx;
2290
2291 /* Use fallback buffer when output buffer is:
2292 * - non-d3d11 memory
2293 * - or, from different d3d11 device
2294 * - or not bound to render target
2295 */
2296 if (!gst_d3d11_compositor_check_d3d11_memory (self,
2297 outbuf, FALSE, &do_device_copy) || !do_device_copy) {
2298 if (!gst_d3d11_compsitor_prepare_fallback_buffer (self, &vagg->info, FALSE,
2299 &self->fallback_pool, &self->fallback_buf)) {
2300 GST_ERROR_OBJECT (self, "Couldn't prepare fallback buffer");
2301 return GST_FLOW_ERROR;
2302 }
2303
2304 GST_TRACE_OBJECT (self, "Will draw on fallback texture");
2305
2306 need_copy = TRUE;
2307 target_buf = self->fallback_buf;
2308 }
2309
2310 view_idx = 0;
2311 for (i = 0; i < gst_buffer_n_memory (target_buf); i++) {
2312 GstMemory *mem = gst_buffer_peek_memory (target_buf, i);
2313 GstD3D11Memory *dmem;
2314 guint rtv_size;
2315
2316 if (!gst_is_d3d11_memory (mem)) {
2317 GST_ERROR_OBJECT (self, "Invalid output memory");
2318 return GST_FLOW_ERROR;
2319 }
2320
2321 dmem = (GstD3D11Memory *) mem;
2322 rtv_size = gst_d3d11_memory_get_render_target_view_size (dmem);
2323 if (!rtv_size) {
2324 GST_ERROR_OBJECT (self, "Render target view is unavailable");
2325 return GST_FLOW_ERROR;
2326 }
2327
2328 for (j = 0; j < rtv_size; j++) {
2329 g_assert (view_idx < GST_VIDEO_MAX_PLANES);
2330
2331 rtv[view_idx] = gst_d3d11_memory_get_render_target_view (dmem, j);
2332 view_idx++;
2333 }
2334
2335 /* Mark need-download for fallback buffer use case */
2336 GST_MINI_OBJECT_FLAG_SET (dmem, GST_D3D11_MEMORY_TRANSFER_NEED_DOWNLOAD);
2337 }
2338
2339 gst_d3d11_device_lock (self->device);
2340 /* XXX: the number of render target view must be one here, since we support
2341 * only RGBA or BGRA */
2342 if (!gst_d3d11_compositor_draw_background (self, rtv[0])) {
2343 GST_ERROR_OBJECT (self, "Couldn't draw background");
2344 gst_d3d11_device_unlock (self->device);
2345 ret = GST_FLOW_ERROR;
2346 goto done;
2347 }
2348
2349 GST_OBJECT_LOCK (self);
2350
2351 for (iter = GST_ELEMENT (vagg)->sinkpads; iter; iter = g_list_next (iter)) {
2352 GstVideoAggregatorPad *pad = GST_VIDEO_AGGREGATOR_PAD (iter->data);
2353 GstD3D11CompositorPad *cpad = GST_D3D11_COMPOSITOR_PAD (pad);
2354 GstVideoFrame *prepared_frame =
2355 gst_video_aggregator_pad_get_prepared_frame (pad);
2356 ID3D11ShaderResourceView *srv[GST_VIDEO_MAX_PLANES] = { NULL, };
2357 GstBuffer *buffer;
2358
2359 if (!prepared_frame)
2360 continue;
2361
2362 if (!gst_d3d11_compositor_pad_setup_converter (pad, vagg)) {
2363 GST_ERROR_OBJECT (self, "Couldn't setup converter");
2364 ret = GST_FLOW_ERROR;
2365 break;
2366 }
2367
2368 buffer = prepared_frame->buffer;
2369
2370 view_idx = 0;
2371 for (i = 0; i < gst_buffer_n_memory (buffer); i++) {
2372 GstD3D11Memory *dmem =
2373 (GstD3D11Memory *) gst_buffer_peek_memory (buffer, i);
2374 guint srv_size = gst_d3d11_memory_get_shader_resource_view_size (dmem);
2375
2376 for (j = 0; j < srv_size; j++) {
2377 g_assert (view_idx < GST_VIDEO_MAX_PLANES);
2378
2379 srv[view_idx] = gst_d3d11_memory_get_shader_resource_view (dmem, j);
2380 view_idx++;
2381 }
2382 }
2383
2384 if (!gst_d3d11_converter_convert_unlocked (cpad->convert, srv, rtv,
2385 cpad->blend, cpad->blend_factor)) {
2386 GST_ERROR_OBJECT (self, "Couldn't convert frame");
2387 ret = GST_FLOW_ERROR;
2388 break;
2389 }
2390 }
2391
2392 self->reconfigured = FALSE;
2393 GST_OBJECT_UNLOCK (self);
2394 gst_d3d11_device_unlock (self->device);
2395
2396 if (ret != GST_FLOW_OK)
2397 goto done;
2398
2399 if (need_copy && !gst_d3d11_compositor_copy_buffer (self, &vagg->info,
2400 target_buf, outbuf, do_device_copy)) {
2401 GST_ERROR_OBJECT (self, "Couldn't copy input buffer to fallback buffer");
2402 ret = GST_FLOW_ERROR;
2403 }
2404
2405 done:
2406 gst_clear_buffer (&self->fallback_buf);
2407
2408 return ret;
2409 }
2410
2411 typedef struct
2412 {
2413 /* without holding ref */
2414 GstD3D11Device *other_device;
2415 gboolean have_same_device;
2416 } DeviceCheckData;
2417
2418 static gboolean
gst_d3d11_compositor_check_device_update(GstElement * agg,GstVideoAggregatorPad * vpad,DeviceCheckData * data)2419 gst_d3d11_compositor_check_device_update (GstElement * agg,
2420 GstVideoAggregatorPad * vpad, DeviceCheckData * data)
2421 {
2422 GstD3D11Compositor *self = GST_D3D11_COMPOSITOR (agg);
2423 GstBuffer *buf;
2424 GstMemory *mem;
2425 GstD3D11Memory *dmem;
2426 gboolean update_device = FALSE;
2427
2428 buf = gst_video_aggregator_pad_get_current_buffer (vpad);
2429 if (!buf)
2430 return TRUE;
2431
2432 /* Ignore gap buffer */
2433 if (GST_BUFFER_FLAG_IS_SET (buf, GST_BUFFER_FLAG_GAP) ||
2434 gst_buffer_get_size (buf) == 0) {
2435 return TRUE;
2436 }
2437
2438 mem = gst_buffer_peek_memory (buf, 0);
2439 /* FIXME: we should be able to accept non-d3d11 memory later once
2440 * we remove intermediate elements (d3d11upload and d3d11colorconvert)
2441 */
2442 if (!gst_is_d3d11_memory (mem)) {
2443 GST_ELEMENT_ERROR (agg, CORE, FAILED, (NULL), ("Invalid memory"));
2444 return FALSE;
2445 }
2446
2447 dmem = GST_D3D11_MEMORY_CAST (mem);
2448
2449 /* We can use existing device */
2450 if (dmem->device == self->device) {
2451 data->have_same_device = TRUE;
2452 return FALSE;
2453 }
2454
2455 if (self->adapter < 0) {
2456 update_device = TRUE;
2457 } else {
2458 guint adapter = 0;
2459
2460 g_object_get (dmem->device, "adapter", &adapter, NULL);
2461 /* The same GPU as what user wanted, update */
2462 if (adapter == (guint) self->adapter)
2463 update_device = TRUE;
2464 }
2465
2466 if (!update_device)
2467 return TRUE;
2468
2469 data->other_device = dmem->device;
2470
2471 /* Keep iterate since there might be one buffer which holds the same device
2472 * as ours */
2473 return TRUE;
2474 }
2475
2476 static GstFlowReturn
gst_d3d11_compositor_create_output_buffer(GstVideoAggregator * vagg,GstBuffer ** outbuffer)2477 gst_d3d11_compositor_create_output_buffer (GstVideoAggregator * vagg,
2478 GstBuffer ** outbuffer)
2479 {
2480 GstD3D11Compositor *self = GST_D3D11_COMPOSITOR (vagg);
2481 DeviceCheckData data;
2482
2483 /* Check whether there is at least one sinkpad which holds d3d11 buffer
2484 * with compatible device, and if not, update our device */
2485 data.other_device = NULL;
2486 data.have_same_device = FALSE;
2487
2488 gst_element_foreach_sink_pad (GST_ELEMENT_CAST (vagg),
2489 (GstElementForeachPadFunc) gst_d3d11_compositor_check_device_update,
2490 &data);
2491
2492 if (data.have_same_device || !data.other_device)
2493 goto done;
2494
2495 /* Clear all device dependent resources */
2496 gst_element_foreach_sink_pad (GST_ELEMENT_CAST (vagg),
2497 (GstElementForeachPadFunc) gst_d3d11_compositor_pad_clear_resource, NULL);
2498
2499 gst_clear_buffer (&self->fallback_buf);
2500 if (self->fallback_pool) {
2501 gst_buffer_pool_set_active (self->fallback_pool, FALSE);
2502 gst_clear_object (&self->fallback_pool);
2503 }
2504 g_clear_pointer (&self->checker_background, gst_d3d11_quad_free);
2505
2506 GST_INFO_OBJECT (self, "Updating device %" GST_PTR_FORMAT " -> %"
2507 GST_PTR_FORMAT, self->device, data.other_device);
2508 gst_object_unref (self->device);
2509 self->device = (GstD3D11Device *) gst_object_ref (data.other_device);
2510
2511 /* We cannot call gst_aggregator_negotiate() here, since GstVideoAggregator
2512 * is holding GST_VIDEO_AGGREGATOR_LOCK() already.
2513 * Mark reconfigure and do reconfigure later */
2514 gst_pad_mark_reconfigure (GST_AGGREGATOR_SRC_PAD (vagg));
2515
2516 return GST_AGGREGATOR_FLOW_NEED_DATA;
2517
2518 done:
2519 return GST_VIDEO_AGGREGATOR_CLASS (parent_class)->create_output_buffer (vagg,
2520 outbuffer);
2521 }
2522