1 /*
2 * GStreamer
3 * Copyright (C) 2019 Matthew Waters <matthew@centricular.com>
4 *
5 * This library is free software; you can redistribute it and/or
6 * modify it under the terms of the GNU Library General Public
7 * License as published by the Free Software Foundation; either
8 * version 2 of the License, or (at your option) any later version.
9 *
10 * This library is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 * Library General Public License for more details.
14 *
15 * You should have received a copy of the GNU Library General Public
16 * License along with this library; if not, write to the
17 * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
18 * Boston, MA 02110-1301, USA.
19 */
20
21 /**
22 * SECTION:element-vulkanimageidentity
23 * @title: vulkanimgeidentity
24 *
25 * vulkanimageidentity produces a vulkan image that is a copy of the input image.
26 */
27
28 #ifdef HAVE_CONFIG_H
29 #include "config.h"
30 #endif
31
32 #include <string.h>
33
34 #include "gstvulkanelements.h"
35 #include "vkviewconvert.h"
36
37 #include "shaders/identity.vert.h"
38 #include "shaders/view_convert.frag.h"
39 #include "gstvulkan-plugins-enumtypes.h"
40
41 GST_DEBUG_CATEGORY (gst_debug_vulkan_view_convert);
42 #define GST_CAT_DEFAULT gst_debug_vulkan_view_convert
43
44 /* *INDENT-OFF* */
45 /* These match the order and number of DOWNMIX_ANAGLYPH_* modes */
46 static float downmix_matrices[][2][12] = {
47 { /* Green-Magenta Dubois */
48 {-0.062f, 0.284f, -0.015f, 0.0, -0.158f, 0.668f, -0.027f, 0.0, -0.039f, 0.143f, 0.021f, 0.0},
49 {0.529f, -0.016f, 0.009f, 0.0, 0.705f, -0.015f, 0.075f, 0.0, 0.024f, -0.065f, 0.937f, 0.0}
50 },
51 { /* Red-Cyan Dubois */
52 /* Source of this matrix: http://www.site.uottawa.ca/~edubois/anaglyph/LeastSquaresHowToPhotoshop.pdf */
53 {0.437f, -0.062f, -0.048f, 0.0, 0.449f, -0.062f, -0.050f, 0.0, 0.164f, -0.024f, -0.017f},
54 {-0.011f, 0.377f, -0.026f, 0.0, -0.032f, 0.761f, -0.093f, 0.0, -0.007f, 0.009f, 1.234f}
55 },
56 { /* Amber-blue Dubois */
57 {1.062f, -0.026f, -0.038f, 0.0, -0.205f, 0.908f, -0.173f, 0.0, 0.299f, 0.068f, 0.022f},
58 {-0.016f, 0.006f, 0.094f, 0.0, -0.123f, 0.062f, 0.185f, 0.0, -0.017f, -0.017f, 0.911f}
59 }
60 };
61 /* *INDENT-ON* */
62
63 struct ViewUpdate
64 {
65 int in_reorder_idx[4];
66 int out_reorder_idx[4];
67 float tex_offset[2][2];
68 float tex_scale[2][2];
69 int tex_size[2];
70 int output_type;
71 int _padding;
72 float downmix[2][12];
73 };
74
75 static void
get_rgb_format_swizzle_order(GstVideoFormat format,gint swizzle[GST_VIDEO_MAX_COMPONENTS])76 get_rgb_format_swizzle_order (GstVideoFormat format,
77 gint swizzle[GST_VIDEO_MAX_COMPONENTS])
78 {
79 const GstVideoFormatInfo *finfo = gst_video_format_get_info (format);
80 int c_i = 0, i;
81
82 g_return_if_fail (finfo->flags & GST_VIDEO_FORMAT_FLAG_RGB
83 || format == GST_VIDEO_FORMAT_AYUV);
84
85 for (i = 0; i < finfo->n_components; i++) {
86 swizzle[c_i++] = finfo->poffset[i];
87 }
88
89 /* special case spaced RGB formats as the space does not contain a poffset
90 * value and we need all four components to be valid in order to swizzle
91 * correctly */
92 if (format == GST_VIDEO_FORMAT_xRGB || format == GST_VIDEO_FORMAT_xBGR) {
93 swizzle[c_i++] = 0;
94 } else if (format == GST_VIDEO_FORMAT_RGBx || format == GST_VIDEO_FORMAT_BGRx) {
95 swizzle[c_i++] = 3;
96 } else {
97 for (i = finfo->n_components; i < GST_VIDEO_MAX_COMPONENTS; i++) {
98 swizzle[c_i++] = -1;
99 }
100 }
101 }
102
103 static void
get_vulkan_rgb_format_swizzle_order(VkFormat format,gint * swizzle,guint swizzle_count,guint offset)104 get_vulkan_rgb_format_swizzle_order (VkFormat format, gint * swizzle,
105 guint swizzle_count, guint offset)
106 {
107 const GstVulkanFormatInfo *finfo = gst_vulkan_format_get_info (format);
108 int i;
109
110 g_return_if_fail (finfo->flags & GST_VULKAN_FORMAT_FLAG_RGB);
111 g_return_if_fail (finfo->n_components <= swizzle_count);
112
113 for (i = 0; i < finfo->n_components; i++) {
114 swizzle[i] = offset + finfo->poffset[i];
115 }
116 for (i = finfo->n_components; i < swizzle_count; i++) {
117 swizzle[i] = -1;
118 }
119 }
120
121 /* given a swizzle index, produce an index such that:
122 *
123 * swizzle[idx[i]] == identity[i] where:
124 * - swizzle is the original swizzle
125 * - idx is the result
126 * - identity = {0, 1, 2,...}
127 * - unset fields are marked by -1
128 */
129 static void
swizzle_identity_order(gint * swizzle,gint * idx)130 swizzle_identity_order (gint * swizzle, gint * idx)
131 {
132 int i;
133
134 for (i = 0; i < GST_VIDEO_MAX_COMPONENTS; i++) {
135 idx[i] = -1;
136 }
137
138 for (i = 0; i < GST_VIDEO_MAX_COMPONENTS; i++) {
139 if (swizzle[i] >= 0 && swizzle[i] < 4 && idx[swizzle[i]] == -1) {
140 idx[swizzle[i]] = i;
141 }
142 }
143 }
144
145 static void
video_format_to_reorder(GstVideoFormat v_format,gint * reorder,gboolean input)146 video_format_to_reorder (GstVideoFormat v_format, gint * reorder,
147 gboolean input)
148 {
149 switch (v_format) {
150 case GST_VIDEO_FORMAT_RGBA:
151 case GST_VIDEO_FORMAT_RGBx:
152 case GST_VIDEO_FORMAT_BGRA:
153 case GST_VIDEO_FORMAT_BGRx:
154 case GST_VIDEO_FORMAT_ARGB:
155 case GST_VIDEO_FORMAT_xRGB:
156 case GST_VIDEO_FORMAT_ABGR:
157 case GST_VIDEO_FORMAT_xBGR:
158 case GST_VIDEO_FORMAT_AYUV:
159 get_rgb_format_swizzle_order (v_format, reorder);
160 break;
161 case GST_VIDEO_FORMAT_UYVY:
162 reorder[0] = 1;
163 reorder[1] = 0;
164 reorder[2] = input ? 3 : 2;
165 reorder[3] = 0;
166 break;
167 case GST_VIDEO_FORMAT_YUY2:
168 reorder[0] = 0;
169 reorder[1] = 1;
170 reorder[2] = 0;
171 reorder[3] = input ? 3 : 2;
172 break;
173 case GST_VIDEO_FORMAT_NV12:
174 reorder[0] = 0;
175 reorder[1] = 1;
176 reorder[2] = 2;
177 reorder[3] = 0;
178 break;
179 default:
180 g_assert_not_reached ();
181 break;
182 }
183
184 GST_TRACE ("swizzle: %u, %u, %u, %u", reorder[0], reorder[1], reorder[2],
185 reorder[3]);
186 }
187
188 static guint
finfo_get_plane_n_components(const GstVideoFormatInfo * finfo,guint plane)189 finfo_get_plane_n_components (const GstVideoFormatInfo * finfo, guint plane)
190 {
191 guint n_components = 0, i;
192
193 switch (finfo->format) {
194 case GST_VIDEO_FORMAT_RGBx:
195 case GST_VIDEO_FORMAT_xRGB:
196 case GST_VIDEO_FORMAT_BGRx:
197 case GST_VIDEO_FORMAT_xBGR:
198 /* fixup spaced RGB formats as we treat the space as a normal alpha
199 * component */
200 return plane == 0 ? 4 : 0;
201 default:
202 break;
203 }
204
205 for (i = 0; i < finfo->n_components; i++) {
206 if (finfo->plane[i] == plane)
207 n_components++;
208 }
209
210 return n_components;
211 }
212
213 static void
get_vulkan_format_swizzle_order(GstVideoFormat v_format,VkFormat vk_format[GST_VIDEO_MAX_PLANES],gint swizzle[GST_VIDEO_MAX_COMPONENTS])214 get_vulkan_format_swizzle_order (GstVideoFormat v_format,
215 VkFormat vk_format[GST_VIDEO_MAX_PLANES],
216 gint swizzle[GST_VIDEO_MAX_COMPONENTS])
217 {
218 const GstVideoFormatInfo *finfo;
219 int i, prev_in_i = 0;
220
221 finfo = gst_video_format_get_info (v_format);
222 for (i = 0; i < finfo->n_planes; i++) {
223 guint plane_components = finfo_get_plane_n_components (finfo, i);
224 get_vulkan_rgb_format_swizzle_order (vk_format[i],
225 &swizzle[prev_in_i], plane_components, prev_in_i);
226 prev_in_i += plane_components;
227 }
228
229 if (v_format == GST_VIDEO_FORMAT_YUY2 || v_format == GST_VIDEO_FORMAT_UYVY) {
230 /* Fixup these packed YUV formats as we use a two component format for
231 * a 4-component pixel and access two samples in the shader */
232 g_assert (swizzle[0] == 0);
233 g_assert (swizzle[1] == 1);
234 swizzle[2] = 2;
235 swizzle[3] = 3;
236 }
237
238 GST_TRACE ("%s: %i, %i, %i, %i", finfo->name, swizzle[0], swizzle[1],
239 swizzle[2], swizzle[3]);
240 }
241
242 static void
calculate_reorder_indexes(GstVideoFormat in_format,GstVulkanImageView * in_views[GST_VIDEO_MAX_COMPONENTS],GstVideoFormat out_format,GstVulkanImageView * out_views[GST_VIDEO_MAX_COMPONENTS],int ret_in[GST_VIDEO_MAX_COMPONENTS],int ret_out[GST_VIDEO_MAX_COMPONENTS])243 calculate_reorder_indexes (GstVideoFormat in_format,
244 GstVulkanImageView * in_views[GST_VIDEO_MAX_COMPONENTS],
245 GstVideoFormat out_format,
246 GstVulkanImageView * out_views[GST_VIDEO_MAX_COMPONENTS],
247 int ret_in[GST_VIDEO_MAX_COMPONENTS], int ret_out[GST_VIDEO_MAX_COMPONENTS])
248 {
249 const GstVideoFormatInfo *in_finfo, *out_finfo;
250 VkFormat in_vk_formats[GST_VIDEO_MAX_COMPONENTS];
251 VkFormat out_vk_formats[GST_VIDEO_MAX_COMPONENTS];
252 int in_vk_order[GST_VIDEO_MAX_COMPONENTS],
253 in_reorder[GST_VIDEO_MAX_COMPONENTS];
254 int out_vk_order[GST_VIDEO_MAX_COMPONENTS],
255 out_reorder[GST_VIDEO_MAX_COMPONENTS];
256 int tmp[GST_VIDEO_MAX_PLANES];
257 int i;
258
259 in_finfo = gst_video_format_get_info (in_format);
260 out_finfo = gst_video_format_get_info (out_format);
261
262 for (i = 0; i < in_finfo->n_planes; i++)
263 in_vk_formats[i] = in_views[i]->image->create_info.format;
264 for (i = 0; i < out_finfo->n_planes; i++)
265 out_vk_formats[i] = out_views[i]->image->create_info.format;
266
267 get_vulkan_format_swizzle_order (in_format, in_vk_formats, in_vk_order);
268 video_format_to_reorder (in_format, in_reorder, TRUE);
269
270 video_format_to_reorder (out_format, out_reorder, FALSE);
271 get_vulkan_format_swizzle_order (out_format, out_vk_formats, out_vk_order);
272
273 for (i = 0; i < GST_VIDEO_MAX_COMPONENTS; i++)
274 tmp[i] = out_vk_order[out_reorder[i]];
275 /* find the identity order for RGBA->$format */
276 GST_TRACE ("pre-invert: %u, %u, %u, %u", tmp[0], tmp[1], tmp[2], tmp[3]);
277 if (out_format == GST_VIDEO_FORMAT_YUY2
278 || out_format == GST_VIDEO_FORMAT_UYVY) {
279 for (i = 0; i < GST_VIDEO_MAX_COMPONENTS; i++)
280 ret_out[i] = tmp[i];
281 } else {
282 swizzle_identity_order (tmp, ret_out);
283 }
284
285 for (i = 0; i < GST_VIDEO_MAX_COMPONENTS; i++)
286 ret_in[i] = in_reorder[in_vk_order[i]];
287 GST_TRACE ("in reorder: %u, %u, %u, %u", ret_in[0], ret_in[1], ret_in[2],
288 ret_in[3]);
289 GST_TRACE ("out reorder: %u, %u, %u, %u", ret_out[0], ret_out[1], ret_out[2],
290 ret_out[3]);
291 }
292
293 static void
update_descriptor_set(GstVulkanViewConvert * conv,GstVulkanImageView ** in_views)294 update_descriptor_set (GstVulkanViewConvert * conv,
295 GstVulkanImageView ** in_views)
296 {
297 GstVulkanVideoFilter *vfilter = GST_VULKAN_VIDEO_FILTER (conv);
298 VkDescriptorImageInfo image_info[GST_VIDEO_MAX_PLANES];
299 VkWriteDescriptorSet writes[GST_VIDEO_MAX_PLANES];
300 GstVideoMultiviewMode in_mode;
301 GstVideoMultiviewFlags in_flags, out_flags;
302 VkImageView views[GST_VIDEO_MAX_PLANES];
303 guint i = 0;
304
305 in_mode = conv->input_mode_override;
306 in_flags = conv->input_flags_override;
307 if (in_mode == GST_VIDEO_MULTIVIEW_MODE_NONE)
308 in_flags = GST_VIDEO_INFO_MULTIVIEW_FLAGS (&vfilter->in_info);
309 out_flags = GST_VIDEO_INFO_MULTIVIEW_FLAGS (&vfilter->out_info);
310
311 for (i = 0; i < GST_VIDEO_INFO_N_PLANES (&vfilter->in_info); i++) {
312 if ((in_flags & GST_VIDEO_MULTIVIEW_FLAGS_RIGHT_VIEW_FIRST) ==
313 (out_flags & GST_VIDEO_MULTIVIEW_FLAGS_RIGHT_VIEW_FIRST)) {
314 views[2 * i] = in_views[i]->view;
315 views[2 * i + 1] = in_views[i]->view;
316 } else {
317 views[2 * i] = in_views[i]->view;
318 views[2 * i + 1] = in_views[i]->view;
319 }
320 }
321
322 for (i = 0; i < GST_VIDEO_INFO_N_PLANES (&vfilter->in_info) * 2; i++) {
323 /* *INDENT-OFF* */
324 image_info[i] = (VkDescriptorImageInfo) {
325 .imageLayout = VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL,
326 .imageView = views[i],
327 .sampler = (VkSampler) conv->quad->sampler->handle
328 };
329
330 g_assert (i < GST_VIDEO_MAX_PLANES);
331
332 writes[i] = (VkWriteDescriptorSet) {
333 .sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET,
334 .pNext = NULL,
335 .dstSet = conv->quad->descriptor_set->set,
336 .dstBinding = i+1,
337 .dstArrayElement = 0,
338 .descriptorType = VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER,
339 .descriptorCount = 1,
340 .pImageInfo = &image_info[i]
341 };
342 /* *INDENT-ON* */
343 }
344 g_assert (i <= G_N_ELEMENTS (writes));
345
346 vkUpdateDescriptorSets (vfilter->device->device, i, writes, 0, NULL);
347 }
348
349 static gboolean
create_uniform_buffer(GstVulkanViewConvert * conv)350 create_uniform_buffer (GstVulkanViewConvert * conv)
351 {
352 GstVulkanVideoFilter *vfilter = GST_VULKAN_VIDEO_FILTER (conv);
353
354 conv->uniform =
355 gst_vulkan_buffer_memory_alloc (vfilter->device,
356 sizeof (struct ViewUpdate),
357 VK_BUFFER_USAGE_TRANSFER_DST_BIT | VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT,
358 VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT |
359 VK_MEMORY_PROPERTY_HOST_COHERENT_BIT);
360
361 return TRUE;
362 }
363
364 static gboolean
update_uniform(GstVulkanViewConvert * conv,GstVulkanImageView ** in_views,GstVulkanImageView ** out_views)365 update_uniform (GstVulkanViewConvert * conv, GstVulkanImageView ** in_views,
366 GstVulkanImageView ** out_views)
367 {
368 GstVulkanVideoFilter *vfilter = GST_VULKAN_VIDEO_FILTER (conv);
369 GstVideoMultiviewMode in_mode, out_mode;
370 GstVideoMultiviewFlags in_flags, out_flags;
371 struct ViewUpdate data;
372 GstMapInfo map_info;
373 guint l_index, r_index;
374 gboolean mono_input = FALSE;
375
376 calculate_reorder_indexes (GST_VIDEO_INFO_FORMAT (&vfilter->in_info),
377 in_views, GST_VIDEO_INFO_FORMAT (&vfilter->out_info),
378 out_views, data.in_reorder_idx, data.out_reorder_idx);
379
380 data.tex_scale[0][0] = data.tex_scale[0][1] = 1.;
381 data.tex_scale[1][0] = data.tex_scale[1][1] = 1.;
382 data.tex_offset[0][0] = data.tex_offset[0][1] = 0.;
383 data.tex_offset[1][0] = data.tex_offset[1][1] = 0.;
384
385 in_mode = conv->input_mode_override;
386 in_flags = conv->input_flags_override;
387 if (in_mode == GST_VIDEO_MULTIVIEW_MODE_NONE) {
388 in_mode = GST_VIDEO_INFO_MULTIVIEW_MODE (&vfilter->in_info);
389 in_flags = GST_VIDEO_INFO_MULTIVIEW_FLAGS (&vfilter->in_info);
390 }
391
392 /* Configured output mode already takes any override
393 * into account */
394 out_mode = GST_VIDEO_INFO_MULTIVIEW_MODE (&vfilter->out_info);
395 out_flags = GST_VIDEO_INFO_MULTIVIEW_FLAGS (&vfilter->out_info);
396
397 if ((in_flags & GST_VIDEO_MULTIVIEW_FLAGS_RIGHT_VIEW_FIRST) ==
398 (out_flags & GST_VIDEO_MULTIVIEW_FLAGS_RIGHT_VIEW_FIRST)) {
399 l_index = 0;
400 r_index = 1;
401 } else {
402 GST_LOG_OBJECT (conv, "Switching left/right views");
403
404 l_index = 1;
405 r_index = 0;
406 }
407
408 if (in_mode < GST_VIDEO_MULTIVIEW_MODE_SIDE_BY_SIDE) { /* unknown/mono/left/right single image */
409 } else if (in_mode == GST_VIDEO_MULTIVIEW_MODE_SIDE_BY_SIDE ||
410 in_mode == GST_VIDEO_MULTIVIEW_MODE_SIDE_BY_SIDE_QUINCUNX) {
411 /* Side-by-side input */
412 data.tex_offset[r_index][0] += 0.5 * data.tex_scale[r_index][0];
413 data.tex_scale[0][0] *= 0.5f; /* Half horizontal scale */
414 data.tex_scale[1][0] *= 0.5f;
415 } else if (in_mode == GST_VIDEO_MULTIVIEW_MODE_TOP_BOTTOM) { /* top-bottom */
416 data.tex_offset[r_index][1] += 0.5 * data.tex_scale[r_index][1];
417 data.tex_scale[0][1] *= 0.5f; /* Half vertical scale */
418 data.tex_scale[1][1] *= 0.5f;
419 }
420
421 /* Flipped is vertical, flopped is horizontal.
422 * Adjust and offset per-view scaling. This needs to be done
423 * after the input scaling already splits the views, before
424 * adding any output scaling. */
425 if ((in_flags & GST_VIDEO_MULTIVIEW_FLAGS_LEFT_FLIPPED) !=
426 (out_flags & GST_VIDEO_MULTIVIEW_FLAGS_LEFT_FLIPPED)) {
427 data.tex_offset[l_index][1] += data.tex_scale[l_index][1];
428 data.tex_scale[l_index][1] *= -1.0;
429 }
430 if ((in_flags & GST_VIDEO_MULTIVIEW_FLAGS_LEFT_FLOPPED) !=
431 (out_flags & GST_VIDEO_MULTIVIEW_FLAGS_LEFT_FLOPPED)) {
432 data.tex_offset[l_index][0] += data.tex_scale[l_index][0];
433 data.tex_scale[l_index][0] *= -1.0;
434 }
435 if ((in_flags & GST_VIDEO_MULTIVIEW_FLAGS_RIGHT_FLIPPED) !=
436 (out_flags & GST_VIDEO_MULTIVIEW_FLAGS_RIGHT_FLIPPED)) {
437 data.tex_offset[r_index][1] += data.tex_scale[r_index][1];
438 data.tex_scale[r_index][1] *= -1.0;
439 }
440 if ((in_flags & GST_VIDEO_MULTIVIEW_FLAGS_RIGHT_FLOPPED) !=
441 (out_flags & GST_VIDEO_MULTIVIEW_FLAGS_RIGHT_FLOPPED)) {
442 data.tex_offset[r_index][0] += data.tex_scale[r_index][0];
443 data.tex_scale[r_index][0] *= -1.0;
444 }
445
446 if (out_mode == GST_VIDEO_MULTIVIEW_MODE_SIDE_BY_SIDE ||
447 out_mode == GST_VIDEO_MULTIVIEW_MODE_SIDE_BY_SIDE_QUINCUNX) {
448 /* Side-by-Side */
449 data.tex_offset[1][0] -= data.tex_scale[1][0];
450 data.tex_scale[0][0] *= 2.0f;
451 data.tex_scale[1][0] *= 2.0f;
452 } else if (out_mode == GST_VIDEO_MULTIVIEW_MODE_TOP_BOTTOM) {
453 data.tex_offset[1][1] -= data.tex_scale[1][1];
454 data.tex_scale[0][1] *= 2.0f;
455 data.tex_scale[1][1] *= 2.0f;
456 }
457
458 GST_DEBUG_OBJECT (conv,
459 "Scaling matrix [ %f, %f ] [ %f %f]. Offsets [ %f, %f ] [ %f, %f ]",
460 data.tex_scale[0][0], data.tex_scale[0][1],
461 data.tex_scale[1][0], data.tex_scale[1][1],
462 data.tex_offset[0][0], data.tex_offset[0][1], data.tex_offset[1][0],
463 data.tex_offset[1][1]);
464
465 if (in_mode == GST_VIDEO_MULTIVIEW_MODE_NONE ||
466 in_mode == GST_VIDEO_MULTIVIEW_MODE_MONO ||
467 in_mode == GST_VIDEO_MULTIVIEW_MODE_LEFT ||
468 in_mode == GST_VIDEO_MULTIVIEW_MODE_RIGHT)
469 mono_input = TRUE;
470
471 data.output_type = out_mode;
472 if (data.output_type == GST_VIDEO_MULTIVIEW_MODE_NONE ||
473 data.output_type == GST_VIDEO_MULTIVIEW_MODE_MONO) {
474 if (mono_input)
475 data.output_type = GST_VIDEO_MULTIVIEW_MODE_LEFT;
476 else
477 data.output_type = GST_VIDEO_MULTIVIEW_MODE_MONO;
478 } else if (data.output_type == GST_VIDEO_MULTIVIEW_MODE_SIDE_BY_SIDE_QUINCUNX) {
479 data.output_type = GST_VIDEO_MULTIVIEW_MODE_SIDE_BY_SIDE;
480 }
481
482 data.tex_size[0] = GST_VIDEO_INFO_WIDTH (&vfilter->out_info);
483 data.tex_size[1] = GST_VIDEO_INFO_HEIGHT (&vfilter->out_info);
484 memcpy (&data.downmix[0], &downmix_matrices[conv->downmix_mode][0],
485 sizeof (data.downmix[0]));
486 memcpy (&data.downmix[1], &downmix_matrices[conv->downmix_mode][1],
487 sizeof (data.downmix[1]));
488
489 if (!gst_memory_map (conv->uniform, &map_info, GST_MAP_WRITE)) {
490 return FALSE;
491 }
492 memcpy (map_info.data, &data, sizeof (data));
493 gst_memory_unmap (conv->uniform, &map_info);
494
495 return TRUE;
496 }
497
498 static GstMemory *
get_uniforms(GstVulkanViewConvert * conv,GstVulkanImageView ** in_views,GstVulkanImageView ** out_views)499 get_uniforms (GstVulkanViewConvert * conv,
500 GstVulkanImageView ** in_views, GstVulkanImageView ** out_views)
501 {
502 if (!conv->uniform) {
503 if (!create_uniform_buffer (conv))
504 return NULL;
505 if (!update_uniform (conv, in_views, out_views)) {
506 gst_memory_unref (conv->uniform);
507 conv->uniform = NULL;
508 return FALSE;
509 }
510 }
511
512 return gst_memory_ref (conv->uniform);
513 }
514
515 static void gst_vulkan_view_convert_set_property (GObject * object,
516 guint prop_id, const GValue * value, GParamSpec * pspec);
517 static void gst_vulkan_view_convert_get_property (GObject * object,
518 guint prop_id, GValue * value, GParamSpec * pspec);
519
520 static gboolean gst_vulkan_view_convert_start (GstBaseTransform * bt);
521 static gboolean gst_vulkan_view_convert_stop (GstBaseTransform * bt);
522
523 static GstCaps *gst_vulkan_view_convert_transform_caps (GstBaseTransform * bt,
524 GstPadDirection direction, GstCaps * caps, GstCaps * filter);
525 static GstCaps *gst_vulkan_view_convert_fixate_caps (GstBaseTransform * bt,
526 GstPadDirection direction, GstCaps * caps, GstCaps * othercaps);
527 static GstFlowReturn gst_vulkan_view_convert_transform (GstBaseTransform * bt,
528 GstBuffer * inbuf, GstBuffer * outbuf);
529 static gboolean gst_vulkan_view_convert_set_caps (GstBaseTransform * bt,
530 GstCaps * in_caps, GstCaps * out_caps);
531
532 static GstStaticPadTemplate gst_vulkan_sink_template =
533 GST_STATIC_PAD_TEMPLATE ("sink",
534 GST_PAD_SINK,
535 GST_PAD_ALWAYS,
536 GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE_WITH_FEATURES
537 (GST_CAPS_FEATURE_MEMORY_VULKAN_IMAGE,
538 "{ BGRA, RGBA }")));
539
540 static GstStaticPadTemplate gst_vulkan_src_template =
541 GST_STATIC_PAD_TEMPLATE ("src",
542 GST_PAD_SRC,
543 GST_PAD_ALWAYS,
544 GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE_WITH_FEATURES
545 (GST_CAPS_FEATURE_MEMORY_VULKAN_IMAGE,
546 "{ BGRA, RGBA }")));
547
548 enum
549 {
550 PROP_0,
551 PROP_INPUT_LAYOUT,
552 PROP_INPUT_FLAGS,
553 PROP_OUTPUT_LAYOUT,
554 PROP_OUTPUT_FLAGS,
555 PROP_OUTPUT_DOWNMIX_MODE
556 };
557
558 #define DEFAULT_DOWNMIX GST_VULKAN_STEREO_DOWNMIX_ANAGLYPH_GREEN_MAGENTA_DUBOIS
559
560 #define gst_vulkan_view_convert_parent_class parent_class
561 G_DEFINE_TYPE_WITH_CODE (GstVulkanViewConvert, gst_vulkan_view_convert,
562 GST_TYPE_VULKAN_VIDEO_FILTER,
563 GST_DEBUG_CATEGORY_INIT (gst_debug_vulkan_view_convert,
564 "vulkanviewconvert", 0, "Vulkan View Convert"));
565 GST_ELEMENT_REGISTER_DEFINE_WITH_CODE (vulkanviewconvert, "vulkanviewconvert",
566 GST_RANK_NONE, GST_TYPE_VULKAN_VIEW_CONVERT, vulkan_element_init (plugin));
567
568 static void
gst_vulkan_view_convert_class_init(GstVulkanViewConvertClass * klass)569 gst_vulkan_view_convert_class_init (GstVulkanViewConvertClass * klass)
570 {
571 GObjectClass *gobject_class;
572 GstElementClass *gstelement_class;
573 GstBaseTransformClass *gstbasetransform_class;
574
575 gobject_class = (GObjectClass *) klass;
576 gstelement_class = (GstElementClass *) klass;
577 gstbasetransform_class = (GstBaseTransformClass *) klass;
578
579 gobject_class->set_property = gst_vulkan_view_convert_set_property;
580 gobject_class->get_property = gst_vulkan_view_convert_get_property;
581
582 g_object_class_install_property (gobject_class, PROP_INPUT_LAYOUT,
583 g_param_spec_enum ("input-mode-override",
584 "Input Multiview Mode Override",
585 "Override any input information about multiview layout",
586 GST_TYPE_VIDEO_MULTIVIEW_MODE,
587 GST_VIDEO_MULTIVIEW_MODE_NONE,
588 G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
589 g_object_class_install_property (gobject_class, PROP_INPUT_FLAGS,
590 g_param_spec_flags ("input-flags-override",
591 "Input Multiview Flags Override",
592 "Override any input information about multiview layout flags",
593 GST_TYPE_VIDEO_MULTIVIEW_FLAGS, GST_VIDEO_MULTIVIEW_FLAGS_NONE,
594 G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
595 g_object_class_install_property (gobject_class, PROP_OUTPUT_LAYOUT,
596 g_param_spec_enum ("output-mode-override",
597 "Output Multiview Mode Override",
598 "Override automatic output mode selection for multiview layout",
599 GST_TYPE_VIDEO_MULTIVIEW_MODE, GST_VIDEO_MULTIVIEW_MODE_NONE,
600 G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
601 g_object_class_install_property (gobject_class, PROP_OUTPUT_FLAGS,
602 g_param_spec_flags ("output-flags-override",
603 "Output Multiview Flags Override",
604 "Override automatic negotiation for output multiview layout flags",
605 GST_TYPE_VIDEO_MULTIVIEW_FLAGS, GST_VIDEO_MULTIVIEW_FLAGS_NONE,
606 G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
607 g_object_class_install_property (gobject_class, PROP_OUTPUT_DOWNMIX_MODE,
608 g_param_spec_enum ("downmix-mode", "Mode for mono downmixed output",
609 "Output anaglyph type to generate when downmixing to mono",
610 GST_TYPE_VULKAN_STEREO_DOWNMIX, DEFAULT_DOWNMIX,
611 G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
612
613 gst_element_class_set_metadata (gstelement_class, "Vulkan View Convert",
614 "Filter/Video/Convert", "A Vulkan View Convert",
615 "Matthew Waters <matthew@centricular.com>");
616
617 gst_type_mark_as_plugin_api (GST_TYPE_VULKAN_STEREO_DOWNMIX, 0);
618 gst_element_class_add_static_pad_template (gstelement_class,
619 &gst_vulkan_sink_template);
620 gst_element_class_add_static_pad_template (gstelement_class,
621 &gst_vulkan_src_template);
622
623 gstbasetransform_class->start =
624 GST_DEBUG_FUNCPTR (gst_vulkan_view_convert_start);
625 gstbasetransform_class->stop =
626 GST_DEBUG_FUNCPTR (gst_vulkan_view_convert_stop);
627 gstbasetransform_class->transform_caps =
628 gst_vulkan_view_convert_transform_caps;
629 gstbasetransform_class->fixate_caps = gst_vulkan_view_convert_fixate_caps;
630 gstbasetransform_class->set_caps = gst_vulkan_view_convert_set_caps;
631 gstbasetransform_class->transform = gst_vulkan_view_convert_transform;
632 }
633
634 static void
gst_vulkan_view_convert_init(GstVulkanViewConvert * conv)635 gst_vulkan_view_convert_init (GstVulkanViewConvert * conv)
636 {
637 conv->downmix_mode = DEFAULT_DOWNMIX;
638
639 conv->input_mode_override = GST_VIDEO_MULTIVIEW_MODE_NONE;
640 conv->input_flags_override = GST_VIDEO_MULTIVIEW_FLAGS_NONE;
641 conv->output_mode_override = GST_VIDEO_MULTIVIEW_MODE_NONE;
642 conv->output_flags_override = GST_VIDEO_MULTIVIEW_FLAGS_NONE;
643 }
644
645 static void
gst_vulkan_view_convert_set_property(GObject * object,guint prop_id,const GValue * value,GParamSpec * pspec)646 gst_vulkan_view_convert_set_property (GObject * object, guint prop_id,
647 const GValue * value, GParamSpec * pspec)
648 {
649 GstVulkanViewConvert *conv = GST_VULKAN_VIEW_CONVERT (object);
650
651 switch (prop_id) {
652 case PROP_INPUT_LAYOUT:
653 conv->input_mode_override = g_value_get_enum (value);
654 gst_base_transform_reconfigure_src (GST_BASE_TRANSFORM (conv));
655 break;
656 case PROP_INPUT_FLAGS:
657 conv->input_flags_override = g_value_get_flags (value);
658 gst_base_transform_reconfigure_src (GST_BASE_TRANSFORM (conv));
659 break;
660 case PROP_OUTPUT_LAYOUT:
661 conv->output_mode_override = g_value_get_enum (value);
662 gst_base_transform_reconfigure_src (GST_BASE_TRANSFORM (conv));
663 break;
664 case PROP_OUTPUT_FLAGS:
665 conv->output_flags_override = g_value_get_flags (value);
666 gst_base_transform_reconfigure_src (GST_BASE_TRANSFORM (conv));
667 break;
668 case PROP_OUTPUT_DOWNMIX_MODE:
669 conv->downmix_mode = g_value_get_enum (value);
670 break;
671 default:
672 G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
673 break;
674 }
675 }
676
677 static void
gst_vulkan_view_convert_get_property(GObject * object,guint prop_id,GValue * value,GParamSpec * pspec)678 gst_vulkan_view_convert_get_property (GObject * object, guint prop_id,
679 GValue * value, GParamSpec * pspec)
680 {
681 GstVulkanViewConvert *conv = GST_VULKAN_VIEW_CONVERT (object);
682
683 switch (prop_id) {
684 case PROP_INPUT_LAYOUT:
685 g_value_set_enum (value, conv->input_mode_override);
686 break;
687 case PROP_INPUT_FLAGS:
688 g_value_set_flags (value, conv->input_flags_override);
689 break;
690 case PROP_OUTPUT_LAYOUT:
691 g_value_set_enum (value, conv->output_mode_override);
692 break;
693 case PROP_OUTPUT_FLAGS:
694 g_value_set_flags (value, conv->output_flags_override);
695 break;
696 case PROP_OUTPUT_DOWNMIX_MODE:
697 g_value_set_enum (value, conv->downmix_mode);
698 break;
699 default:
700 G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
701 break;
702 }
703 }
704
705 /* Function that can halve the value
706 * of ints, fractions, int/fraction ranges and lists of ints/fractions */
707 static gboolean
_halve_value(GValue * out,const GValue * in_value)708 _halve_value (GValue * out, const GValue * in_value)
709 {
710 /* Fundamental fixed types first */
711 if (G_VALUE_HOLDS_INT (in_value)) {
712 g_value_init (out, G_TYPE_INT);
713 g_value_set_int (out, MAX (g_value_get_int (in_value) / 2, 1));
714 } else if (GST_VALUE_HOLDS_FRACTION (in_value)) {
715 gint num, den;
716 num = gst_value_get_fraction_numerator (in_value);
717 den = gst_value_get_fraction_denominator (in_value);
718 g_value_init (out, GST_TYPE_FRACTION);
719 /* Don't adjust 'infinite' fractions */
720 if ((num != 1 || den != 2147483647) && (num != 2147483647 || den != 1)) {
721 /* FIXME - could do better approximation when den > G_MAXINT/2? */
722 den = den > G_MAXINT / 2 ? G_MAXINT : den * 2;
723 }
724 gst_value_set_fraction (out, num, den);
725 } else if (GST_VALUE_HOLDS_INT_RANGE (in_value)) {
726 gint range_min = gst_value_get_int_range_min (in_value);
727 gint range_max = gst_value_get_int_range_max (in_value);
728 gint range_step = gst_value_get_int_range_step (in_value);
729 g_value_init (out, GST_TYPE_INT_RANGE);
730 if (range_min != 1)
731 range_min = MAX (1, range_min / 2);
732 if (range_max != G_MAXINT)
733 range_max = MAX (1, range_max / 2);
734 gst_value_set_int_range_step (out, range_min,
735 range_max, MAX (1, range_step / 2));
736 } else if (GST_VALUE_HOLDS_FRACTION_RANGE (in_value)) {
737 GValue min_out = G_VALUE_INIT;
738 GValue max_out = G_VALUE_INIT;
739 const GValue *range_min = gst_value_get_fraction_range_min (in_value);
740 const GValue *range_max = gst_value_get_fraction_range_max (in_value);
741 _halve_value (&min_out, range_min);
742 _halve_value (&max_out, range_max);
743 g_value_init (out, GST_TYPE_FRACTION_RANGE);
744 gst_value_set_fraction_range (out, &min_out, &max_out);
745 g_value_unset (&min_out);
746 g_value_unset (&max_out);
747 } else if (GST_VALUE_HOLDS_LIST (in_value)) {
748 gint i;
749 g_value_init (out, GST_TYPE_LIST);
750 for (i = 0; i < gst_value_list_get_size (in_value); i++) {
751 const GValue *entry;
752 GValue tmp = G_VALUE_INIT;
753
754 entry = gst_value_list_get_value (in_value, i);
755 /* Random list values might not be the right type */
756 if (!_halve_value (&tmp, entry))
757 goto fail;
758 gst_value_list_append_and_take_value (out, &tmp);
759 }
760 } else {
761 return FALSE;
762 }
763
764 return TRUE;
765 fail:
766 g_value_unset (out);
767 return FALSE;
768 }
769
770 static GstStructure *
_halve_structure_field(const GstStructure * in,const gchar * field_name)771 _halve_structure_field (const GstStructure * in, const gchar * field_name)
772 {
773 GstStructure *out;
774 const GValue *in_value = gst_structure_get_value (in, field_name);
775 GValue tmp = G_VALUE_INIT;
776
777 if (G_UNLIKELY (in_value == NULL))
778 return gst_structure_copy (in); /* Field doesn't exist, leave it as is */
779
780 if (!_halve_value (&tmp, in_value))
781 return NULL;
782
783 out = gst_structure_copy (in);
784 gst_structure_set_value (out, field_name, &tmp);
785 g_value_unset (&tmp);
786
787 return out;
788 }
789
790 /* Function that can double the value
791 * of ints, fractions, int/fraction ranges and lists of ints/fractions */
792 static gboolean
_double_value(GValue * out,const GValue * in_value)793 _double_value (GValue * out, const GValue * in_value)
794 {
795 /* Fundamental fixed types first */
796 if (G_VALUE_HOLDS_INT (in_value)) {
797 gint n = g_value_get_int (in_value);
798 g_value_init (out, G_TYPE_INT);
799 if (n <= G_MAXINT / 2)
800 g_value_set_int (out, n * 2);
801 else
802 g_value_set_int (out, G_MAXINT);
803 } else if (GST_VALUE_HOLDS_FRACTION (in_value)) {
804 gint num, den;
805 num = gst_value_get_fraction_numerator (in_value);
806 den = gst_value_get_fraction_denominator (in_value);
807 g_value_init (out, GST_TYPE_FRACTION);
808 /* Don't adjust 'infinite' fractions */
809 if ((num != 1 || den != 2147483647) && (num != 2147483647 || den != 1)) {
810 /* FIXME - could do better approximation when num > G_MAXINT/2? */
811 num = num > G_MAXINT / 2 ? G_MAXINT : num * 2;
812 }
813 gst_value_set_fraction (out, num, den);
814 } else if (GST_VALUE_HOLDS_INT_RANGE (in_value)) {
815 gint range_min = gst_value_get_int_range_min (in_value);
816 gint range_max = gst_value_get_int_range_max (in_value);
817 gint range_step = gst_value_get_int_range_step (in_value);
818 if (range_min != 1) {
819 range_min = MIN (G_MAXINT / 2, range_min);
820 range_min *= 2;
821 }
822 if (range_max != G_MAXINT) {
823 range_max = MIN (G_MAXINT / 2, range_max);
824 range_max *= 2;
825 }
826 range_step = MIN (G_MAXINT / 2, range_step);
827 g_value_init (out, GST_TYPE_INT_RANGE);
828 gst_value_set_int_range_step (out, range_min, range_max, range_step);
829 } else if (GST_VALUE_HOLDS_FRACTION_RANGE (in_value)) {
830 GValue min_out = G_VALUE_INIT;
831 GValue max_out = G_VALUE_INIT;
832 const GValue *range_min = gst_value_get_fraction_range_min (in_value);
833 const GValue *range_max = gst_value_get_fraction_range_max (in_value);
834 _double_value (&min_out, range_min);
835 _double_value (&max_out, range_max);
836 g_value_init (out, GST_TYPE_FRACTION_RANGE);
837 gst_value_set_fraction_range (out, &min_out, &max_out);
838 g_value_unset (&min_out);
839 g_value_unset (&max_out);
840 } else if (GST_VALUE_HOLDS_LIST (in_value)) {
841 gint i;
842 g_value_init (out, GST_TYPE_LIST);
843 for (i = 0; i < gst_value_list_get_size (in_value); i++) {
844 const GValue *entry;
845 GValue tmp = G_VALUE_INIT;
846
847 entry = gst_value_list_get_value (in_value, i);
848 /* Random list values might not be the right type */
849 if (!_double_value (&tmp, entry))
850 goto fail;
851 gst_value_list_append_and_take_value (out, &tmp);
852 }
853 } else {
854 return FALSE;
855 }
856
857 return TRUE;
858 fail:
859 g_value_unset (out);
860 return FALSE;
861 }
862
863 static GstStructure *
_double_structure_field(const GstStructure * in,const gchar * field_name)864 _double_structure_field (const GstStructure * in, const gchar * field_name)
865 {
866 GstStructure *out;
867 const GValue *in_value = gst_structure_get_value (in, field_name);
868 GValue tmp = G_VALUE_INIT;
869
870 if (G_UNLIKELY (in_value == NULL))
871 return gst_structure_copy (in); /* Field doesn't exist, leave it as is */
872
873 if (!_double_value (&tmp, in_value))
874 return NULL;
875
876 out = gst_structure_copy (in);
877 gst_structure_set_value (out, field_name, &tmp);
878 g_value_unset (&tmp);
879
880 return out;
881 }
882
883 /* Return a copy of the caps with the requested field doubled in value/range */
884 static GstCaps *
_double_caps_field(const GstCaps * in,const gchar * field_name)885 _double_caps_field (const GstCaps * in, const gchar * field_name)
886 {
887 gint i;
888 GstCaps *out = gst_caps_new_empty ();
889
890 for (i = 0; i < gst_caps_get_size (in); i++) {
891 const GstStructure *cur = gst_caps_get_structure (in, i);
892 GstCapsFeatures *f = gst_caps_get_features (in, i);
893
894 GstStructure *res = _double_structure_field (cur, field_name);
895 out =
896 gst_caps_merge_structure_full (out, res,
897 f ? gst_caps_features_copy (f) : NULL);
898 }
899
900 return out;
901 }
902
903 /* Takes ownership of the input caps */
904 static GstCaps *
_expand_par_for_half_aspect(GstCaps * in,gboolean vertical_half_aspect)905 _expand_par_for_half_aspect (GstCaps * in, gboolean vertical_half_aspect)
906 {
907 guint mview_flags, mview_flags_mask;
908 GstCaps *out;
909 GstStructure *tmp;
910
911 out = gst_caps_new_empty ();
912
913 while (gst_caps_get_size (in) > 0) {
914 GstStructure *s;
915 GstCapsFeatures *features;
916
917 features = gst_caps_get_features (in, 0);
918 if (features)
919 features = gst_caps_features_copy (features);
920
921 s = gst_caps_steal_structure (in, 0);
922
923 if (!gst_structure_get_flagset (s, "multiview-flags", &mview_flags,
924 &mview_flags_mask)) {
925 gst_caps_append_structure_full (out, s, features);
926 continue;
927 }
928 /* If the input doesn't care about the half-aspect flag, allow current PAR in either variant */
929 if ((mview_flags_mask & GST_VIDEO_MULTIVIEW_FLAGS_HALF_ASPECT) == 0) {
930 gst_caps_append_structure_full (out, s, features);
931 continue;
932 }
933 if (!gst_structure_has_field (s, "pixel-aspect-ratio")) {
934 /* No par field, dont-care the half-aspect flag */
935 gst_structure_set (s, "multiview-flags",
936 GST_TYPE_VIDEO_MULTIVIEW_FLAGSET,
937 mview_flags & ~GST_VIDEO_MULTIVIEW_FLAGS_HALF_ASPECT,
938 mview_flags_mask & ~GST_VIDEO_MULTIVIEW_FLAGS_HALF_ASPECT, NULL);
939 gst_caps_append_structure_full (out, s, features);
940 continue;
941 }
942
943 /* Halve or double PAR base on inputs input specified. */
944
945 /* Append a copy with the half-aspect flag as-is */
946 tmp = gst_structure_copy (s);
947 out = gst_caps_merge_structure_full (out, tmp,
948 features ? gst_caps_features_copy (features) : NULL);
949
950 /* and then a copy inverted */
951 if (mview_flags & GST_VIDEO_MULTIVIEW_FLAGS_HALF_ASPECT) {
952 /* Input is half-aspect. Double/halve the PAR, clear the flag */
953 if (vertical_half_aspect)
954 tmp = _halve_structure_field (s, "pixel-aspect-ratio");
955 else
956 tmp = _double_structure_field (s, "pixel-aspect-ratio");
957 /* Clear the flag */
958 gst_structure_set (tmp, "multiview-flags",
959 GST_TYPE_VIDEO_MULTIVIEW_FLAGSET,
960 mview_flags & ~GST_VIDEO_MULTIVIEW_FLAGS_HALF_ASPECT,
961 mview_flags_mask | GST_VIDEO_MULTIVIEW_FLAGS_HALF_ASPECT, NULL);
962 } else {
963 if (vertical_half_aspect)
964 tmp = _double_structure_field (s, "pixel-aspect-ratio");
965 else
966 tmp = _halve_structure_field (s, "pixel-aspect-ratio");
967 /* Set the flag */
968 gst_structure_set (tmp, "multiview-flags",
969 GST_TYPE_VIDEO_MULTIVIEW_FLAGSET,
970 mview_flags | GST_VIDEO_MULTIVIEW_FLAGS_HALF_ASPECT,
971 mview_flags_mask | GST_VIDEO_MULTIVIEW_FLAGS_HALF_ASPECT, NULL);
972 }
973
974 out = gst_caps_merge_structure_full (out, tmp,
975 features ? gst_caps_features_copy (features) : NULL);
976
977 gst_structure_free (s);
978 if (features)
979 gst_caps_features_free (features);
980 }
981
982 gst_caps_unref (in);
983
984 return out;
985 }
986
987 /* If input supports top-bottom or row-interleaved, we may halve height to mono frames.
988 * If input supports left-right, checkerboard, quincunx or column-interleaved,
989 * we may halve width to mono frames.
990 * For output of top-bottom or row-interleaved, we may double the mono height
991 * For output of left-right, checkerboard, quincunx or column-interleaved,
992 * we may double the mono width.
993 * In all cases, if input has half-aspect and output does not, we may double the PAR
994 * And if input does *not* have half-aspect flag and output does not, we may halve the PAR
995 */
996 static GstCaps *
_expand_structure(GstVulkanViewConvert * viewconvert,GstCaps * out_caps,GstStructure * structure,GstCapsFeatures * features)997 _expand_structure (GstVulkanViewConvert * viewconvert,
998 GstCaps * out_caps, GstStructure * structure, GstCapsFeatures * features)
999 {
1000 GstCaps *expanded_caps, *tmp;
1001 GstCaps *mono_caps;
1002 const gchar *default_mview_mode_str = NULL;
1003 guint mview_flags, mview_flags_mask;
1004 const GValue *in_modes;
1005 gint i;
1006
1007 /* Empty caps to accumulate into */
1008 expanded_caps = gst_caps_new_empty ();
1009
1010 /* First, set defaults if multiview flags are missing */
1011 default_mview_mode_str =
1012 gst_video_multiview_mode_to_caps_string (GST_VIDEO_MULTIVIEW_MODE_MONO);
1013
1014 mview_flags = GST_VIDEO_MULTIVIEW_FLAGS_NONE;
1015 mview_flags_mask = GST_FLAG_SET_MASK_EXACT;
1016
1017 if (!gst_structure_has_field (structure, "multiview-mode")) {
1018 gst_structure_set (structure,
1019 "multiview-mode", G_TYPE_STRING, default_mview_mode_str, NULL);
1020 }
1021 if (!gst_structure_has_field (structure, "multiview-flags")) {
1022 gst_structure_set (structure,
1023 "multiview-flags", GST_TYPE_VIDEO_MULTIVIEW_FLAGSET, mview_flags,
1024 mview_flags_mask, NULL);
1025 } else {
1026 gst_structure_get_flagset (structure, "multiview-flags",
1027 &mview_flags, &mview_flags_mask);
1028 }
1029
1030 in_modes = gst_structure_get_value (structure, "multiview-mode");
1031 mono_caps = gst_caps_new_empty ();
1032 if (gst_value_intersect (NULL, in_modes,
1033 gst_video_multiview_get_mono_modes ())) {
1034 GstStructure *new_struct = gst_structure_copy (structure);
1035 gst_structure_set_value (new_struct, "multiview-mode",
1036 gst_video_multiview_get_mono_modes ());
1037 /* Half-aspect makes no sense for mono or unpacked, get rid of it */
1038 if (mview_flags & GST_VIDEO_MULTIVIEW_FLAGS_HALF_ASPECT) {
1039 gst_structure_set (new_struct, "multiview-flags",
1040 GST_TYPE_VIDEO_MULTIVIEW_FLAGSET,
1041 mview_flags & ~GST_VIDEO_MULTIVIEW_FLAGS_HALF_ASPECT,
1042 mview_flags_mask & ~GST_VIDEO_MULTIVIEW_FLAGS_HALF_ASPECT, NULL);
1043 }
1044 gst_caps_append_structure_full (mono_caps, new_struct,
1045 features ? gst_caps_features_copy (features) : NULL);
1046 }
1047 if (gst_value_intersect (NULL, in_modes,
1048 gst_video_multiview_get_unpacked_modes ())) {
1049 GstStructure *new_struct = gst_structure_copy (structure);
1050
1051 gst_structure_set_value (new_struct, "multiview-mode",
1052 gst_video_multiview_get_mono_modes ());
1053
1054 /* Half-aspect makes no sense for mono or unpacked, get rid of it */
1055 if (mview_flags & GST_VIDEO_MULTIVIEW_FLAGS_HALF_ASPECT) {
1056 gst_structure_set (new_struct, "multiview-flags",
1057 GST_TYPE_VIDEO_MULTIVIEW_FLAGSET,
1058 mview_flags & ~GST_VIDEO_MULTIVIEW_FLAGS_HALF_ASPECT,
1059 mview_flags_mask & ~GST_VIDEO_MULTIVIEW_FLAGS_HALF_ASPECT, NULL);
1060 }
1061 gst_caps_append_structure_full (mono_caps, new_struct,
1062 features ? gst_caps_features_copy (features) : NULL);
1063 }
1064
1065 if (gst_value_intersect (NULL, in_modes,
1066 gst_video_multiview_get_doubled_height_modes ())) {
1067 /* Append mono formats with height halved */
1068 GstStructure *new_struct = _halve_structure_field (structure, "height");
1069 gst_structure_set_value (new_struct, "multiview-mode",
1070 gst_video_multiview_get_mono_modes ());
1071 /* Normalise the half-aspect flag away */
1072 if (mview_flags & GST_VIDEO_MULTIVIEW_FLAGS_HALF_ASPECT) {
1073 GstStructure *s =
1074 _halve_structure_field (new_struct, "pixel-aspect-ratio");
1075 gst_structure_set (structure, "multiview-flags",
1076 GST_TYPE_VIDEO_MULTIVIEW_FLAGSET,
1077 mview_flags & ~GST_VIDEO_MULTIVIEW_FLAGS_HALF_ASPECT,
1078 mview_flags_mask | GST_VIDEO_MULTIVIEW_FLAGS_HALF_ASPECT, NULL);
1079 gst_structure_free (new_struct);
1080 new_struct = s;
1081 }
1082 mono_caps = gst_caps_merge_structure_full (mono_caps, new_struct,
1083 features ? gst_caps_features_copy (features) : NULL);
1084 }
1085 if (gst_value_intersect (NULL, in_modes,
1086 gst_video_multiview_get_doubled_width_modes ())) {
1087 /* Append mono formats with width halved */
1088 GstStructure *new_struct = _halve_structure_field (structure, "width");
1089 gst_structure_set_value (new_struct, "multiview-mode",
1090 gst_video_multiview_get_mono_modes ());
1091 /* Normalise the half-aspect flag away */
1092 if (mview_flags & GST_VIDEO_MULTIVIEW_FLAGS_HALF_ASPECT) {
1093 GstStructure *s =
1094 _double_structure_field (new_struct, "pixel-aspect-ratio");
1095 gst_structure_set (structure, "multiview-flags",
1096 GST_TYPE_VIDEO_MULTIVIEW_FLAGSET,
1097 mview_flags & ~GST_VIDEO_MULTIVIEW_FLAGS_HALF_ASPECT,
1098 mview_flags_mask | GST_VIDEO_MULTIVIEW_FLAGS_HALF_ASPECT, NULL);
1099 gst_structure_free (new_struct);
1100 new_struct = s;
1101 }
1102 mono_caps = gst_caps_merge_structure_full (mono_caps, new_struct,
1103 features ? gst_caps_features_copy (features) : NULL);
1104 }
1105 if (gst_value_intersect (NULL, in_modes,
1106 gst_video_multiview_get_doubled_size_modes ())) {
1107 /* Append checkerboard/doubled size formats with width & height halved */
1108 GstStructure *new_struct_w = _halve_structure_field (structure, "width");
1109 GstStructure *new_struct_wh =
1110 _halve_structure_field (new_struct_w, "height");
1111 gst_structure_free (new_struct_w);
1112 gst_structure_set_value (new_struct_wh, "multiview-mode",
1113 gst_video_multiview_get_mono_modes ());
1114 mono_caps = gst_caps_merge_structure_full (mono_caps, new_struct_wh,
1115 features ? gst_caps_features_copy (features) : NULL);
1116 }
1117
1118 /* Everything is normalised now, unset the flags we can change */
1119 /* Remove the views field, as these are all 'mono' modes
1120 * Need to do this before we expand caps back out to frame packed modes */
1121 for (i = 0; i < gst_caps_get_size (mono_caps); i++) {
1122 GstStructure *s = gst_caps_get_structure (mono_caps, i);
1123 gst_structure_remove_fields (s, "views", NULL);
1124 if (gst_structure_get_flagset (s, "multiview-flags", &mview_flags,
1125 &mview_flags_mask)) {
1126 /* Preserve only the half-aspect and mixed-mono flags, for now.
1127 * The rest we can change */
1128 mview_flags_mask &=
1129 (GST_VIDEO_MULTIVIEW_FLAGS_HALF_ASPECT |
1130 GST_VIDEO_MULTIVIEW_FLAGS_MIXED_MONO);
1131 gst_structure_set (s, "multiview-flags", GST_TYPE_VIDEO_MULTIVIEW_FLAGSET,
1132 mview_flags, mview_flags_mask, NULL);
1133 }
1134 }
1135
1136 GST_TRACE_OBJECT (viewconvert,
1137 "Collected single-view caps %" GST_PTR_FORMAT, mono_caps);
1138 /* Put unpacked and mono modes first. We don't care about flags. Clear them */
1139 tmp = gst_caps_copy (mono_caps);
1140 for (i = 0; i < gst_caps_get_size (tmp); i++) {
1141 GstStructure *s = gst_caps_get_structure (tmp, i);
1142 gst_structure_remove_fields (s, "views", NULL);
1143 if (gst_structure_get_flagset (s, "multiview-flags", &mview_flags,
1144 &mview_flags_mask)) {
1145 /* We can change any flags for mono modes - half-aspect and mixed-mono have no meaning */
1146 mview_flags_mask = 0;
1147 gst_structure_set (s, "multiview-flags", GST_TYPE_VIDEO_MULTIVIEW_FLAGSET,
1148 mview_flags, mview_flags_mask, NULL);
1149 }
1150 }
1151 expanded_caps = gst_caps_merge (expanded_caps, tmp);
1152
1153 /* Unpacked output modes have 2 views, for now */
1154 tmp = gst_caps_copy (mono_caps);
1155 gst_caps_set_value (tmp, "multiview-mode",
1156 gst_video_multiview_get_unpacked_modes ());
1157 for (i = 0; i < gst_caps_get_size (tmp); i++) {
1158 GstStructure *s = gst_caps_get_structure (tmp, i);
1159 gst_structure_set (s, "views", G_TYPE_INT, 2, NULL);
1160 if (gst_structure_get_flagset (s, "multiview-flags", &mview_flags,
1161 &mview_flags_mask)) {
1162 /* We can change any flags for unpacked modes - half-aspect and mixed-mono have no meaning */
1163 mview_flags_mask = 0;
1164 gst_structure_set (s, "multiview-flags", GST_TYPE_VIDEO_MULTIVIEW_FLAGSET,
1165 mview_flags, mview_flags_mask, NULL);
1166 }
1167 }
1168 expanded_caps = gst_caps_merge (expanded_caps, tmp);
1169
1170 /* Double height output modes */
1171 tmp = _double_caps_field (mono_caps, "height");
1172 gst_caps_set_value (tmp, "multiview-mode",
1173 gst_video_multiview_get_doubled_height_modes ());
1174 tmp = _expand_par_for_half_aspect (tmp, TRUE);
1175
1176 expanded_caps = gst_caps_merge (expanded_caps, tmp);
1177
1178 /* Double width output modes */
1179 tmp = _double_caps_field (mono_caps, "width");
1180 gst_caps_set_value (tmp, "multiview-mode",
1181 gst_video_multiview_get_doubled_width_modes ());
1182 tmp = _expand_par_for_half_aspect (tmp, FALSE);
1183
1184 expanded_caps = gst_caps_merge (expanded_caps, tmp);
1185
1186 /* Double size output modes */
1187 {
1188 GstCaps *tmp_w = _double_caps_field (mono_caps, "width");
1189 tmp = _double_caps_field (tmp_w, "height");
1190 gst_caps_unref (tmp_w);
1191 gst_caps_set_value (tmp, "multiview-mode",
1192 gst_video_multiview_get_doubled_size_modes ());
1193 expanded_caps = gst_caps_merge (expanded_caps, tmp);
1194 }
1195
1196 /* We're done with the mono caps now */
1197 gst_caps_unref (mono_caps);
1198
1199 GST_TRACE_OBJECT (viewconvert,
1200 "expanded transform caps now %" GST_PTR_FORMAT, expanded_caps);
1201
1202 if (gst_caps_is_empty (expanded_caps)) {
1203 gst_caps_unref (expanded_caps);
1204 return out_caps;
1205 }
1206 /* Really, we can rescale - so at this point we can append full-range
1207 * height/width/PAR as an unpreferred final option. */
1208 /* tmp = gst_caps_copy (expanded_caps);
1209 gst_caps_set_simple (tmp, "width", GST_TYPE_INT_RANGE, 1, G_MAXINT,
1210 "height", GST_TYPE_INT_RANGE, 1, G_MAXINT, NULL);
1211 */
1212 out_caps = gst_caps_merge (out_caps, expanded_caps);
1213 /* out_caps = gst_caps_merge (out_caps, tmp);*/
1214 return out_caps;
1215 }
1216
1217 static GstCaps *
_intersect_with_mview_mode(GstCaps * caps,GstVideoMultiviewMode mode,GstVideoMultiviewFlags flags)1218 _intersect_with_mview_mode (GstCaps * caps,
1219 GstVideoMultiviewMode mode, GstVideoMultiviewFlags flags)
1220 {
1221 GstCaps *filter, *result;
1222 const gchar *caps_str;
1223
1224 caps_str = gst_video_multiview_mode_to_caps_string (mode);
1225
1226 filter = gst_caps_new_simple ("video/x-raw",
1227 "multiview-mode", G_TYPE_STRING,
1228 caps_str, "multiview-flags", GST_TYPE_VIDEO_MULTIVIEW_FLAGSET, flags,
1229 GST_FLAG_SET_MASK_EXACT, NULL);
1230
1231 if (mode == GST_VIDEO_MULTIVIEW_MODE_SEPARATED ||
1232 mode == GST_VIDEO_MULTIVIEW_MODE_FRAME_BY_FRAME)
1233 gst_caps_set_simple (filter, "views", G_TYPE_INT, 2, NULL);
1234 gst_caps_set_features (filter, 0, gst_caps_features_new_any ());
1235
1236 GST_DEBUG ("Intersecting target caps %" GST_PTR_FORMAT
1237 " with caps %" GST_PTR_FORMAT, caps, filter);
1238
1239 result = gst_caps_intersect_full (caps, filter, GST_CAPS_INTERSECT_FIRST);
1240 gst_caps_unref (filter);
1241 return result;
1242 }
1243
1244 static GstCaps *
_intersect_with_mview_modes(GstCaps * caps,const GValue * modes)1245 _intersect_with_mview_modes (GstCaps * caps, const GValue * modes)
1246 {
1247 GstCaps *filter, *result;
1248
1249 filter = gst_caps_new_empty_simple ("video/x-raw");
1250
1251 gst_caps_set_value (filter, "multiview-mode", modes);
1252 gst_caps_set_features (filter, 0, gst_caps_features_new_any ());
1253
1254 GST_DEBUG ("Intersecting target caps %" GST_PTR_FORMAT
1255 " with caps %" GST_PTR_FORMAT, caps, filter);
1256
1257 result = gst_caps_intersect_full (caps, filter, GST_CAPS_INTERSECT_FIRST);
1258 gst_caps_unref (filter);
1259 return result;
1260 }
1261
1262 static GstCaps *
gst_vulkan_view_convert_transform_caps(GstBaseTransform * bt,GstPadDirection direction,GstCaps * caps,GstCaps * filter)1263 gst_vulkan_view_convert_transform_caps (GstBaseTransform * bt,
1264 GstPadDirection direction, GstCaps * caps, GstCaps * filter)
1265 {
1266 GstVulkanViewConvert *viewconvert = GST_VULKAN_VIEW_CONVERT (bt);
1267 GstCaps *base_caps =
1268 gst_static_pad_template_get_caps (&gst_vulkan_sink_template);
1269 GstCaps *out_caps, *tmp_caps;
1270 gint i;
1271
1272 GST_DEBUG_OBJECT (viewconvert, "Direction %s "
1273 "input caps %" GST_PTR_FORMAT " filter %" GST_PTR_FORMAT,
1274 direction == GST_PAD_SINK ? "sink" : "src", caps, filter);
1275
1276 /* We can only process VulkanImage caps, start from that */
1277 caps = gst_caps_intersect (caps, base_caps);
1278 gst_caps_unref (base_caps);
1279
1280 /* Change input/output to the formats we can convert to/from,
1281 * but keep the original caps at the start - we will always prefer
1282 * passthrough */
1283 if (direction == GST_PAD_SINK) {
1284 out_caps = gst_caps_copy (caps);
1285 if (viewconvert->input_mode_override != GST_VIDEO_MULTIVIEW_MODE_NONE) {
1286 GstVideoMultiviewMode mode = viewconvert->input_mode_override;
1287 GstVideoMultiviewFlags flags = viewconvert->input_flags_override;
1288
1289 const gchar *caps_str = gst_video_multiview_mode_to_caps_string (mode);
1290 /* Coerce the input caps before transforming, so the sizes come out right */
1291 gst_caps_set_simple (out_caps, "multiview-mode", G_TYPE_STRING,
1292 caps_str, "multiview-flags", GST_TYPE_VIDEO_MULTIVIEW_FLAGSET, flags,
1293 GST_FLAG_SET_MASK_EXACT, NULL);
1294 }
1295 } else {
1296 out_caps = gst_caps_new_empty ();
1297 }
1298
1299 for (i = 0; i < gst_caps_get_size (caps); i++) {
1300 GstStructure *structure = gst_caps_get_structure (caps, i);
1301 GstCapsFeatures *features = gst_caps_get_features (caps, i);
1302 out_caps = _expand_structure (viewconvert, out_caps, structure, features);
1303 }
1304
1305 if (gst_caps_is_empty (out_caps))
1306 goto out;
1307
1308 /* If we have an output mode override, limit things to that */
1309 if (direction == GST_PAD_SINK &&
1310 viewconvert->output_mode_override != GST_VIDEO_MULTIVIEW_MODE_NONE) {
1311
1312 tmp_caps = _intersect_with_mview_mode (out_caps,
1313 viewconvert->output_mode_override, viewconvert->output_flags_override);
1314
1315 gst_caps_unref (out_caps);
1316 out_caps = tmp_caps;
1317 } else if (viewconvert->input_mode_override != GST_VIDEO_MULTIVIEW_MODE_NONE) {
1318 /* Prepend a copy of our preferred input caps in case the peer
1319 * can handle them */
1320 tmp_caps = _intersect_with_mview_mode (out_caps,
1321 viewconvert->input_mode_override, viewconvert->input_flags_override);
1322 out_caps = gst_caps_merge (out_caps, tmp_caps);
1323 }
1324 if (direction == GST_PAD_SRC) {
1325 GstStructure *s;
1326 /* When generating input caps, we also need a copy of the mono caps
1327 * without multiview-mode or flags for backwards compat, at the end */
1328 tmp_caps = _intersect_with_mview_mode (caps,
1329 GST_VIDEO_MULTIVIEW_MODE_MONO, GST_VIDEO_MULTIVIEW_FLAGS_NONE);
1330 if (!gst_caps_is_empty (tmp_caps)) {
1331 s = gst_caps_get_structure (tmp_caps, 0);
1332 gst_structure_remove_fields (s, "multiview-mode", "multiview-flags",
1333 NULL);
1334 out_caps = gst_caps_merge (out_caps, tmp_caps);
1335 } else
1336 gst_caps_unref (tmp_caps);
1337 }
1338 out:
1339 gst_caps_unref (caps);
1340
1341 GST_DEBUG_OBJECT (viewconvert, "Have caps %" GST_PTR_FORMAT
1342 " filtering with caps %" GST_PTR_FORMAT, out_caps, filter);
1343
1344 if (filter) {
1345 GstCaps *tmp =
1346 gst_caps_intersect_full (filter, out_caps, GST_CAPS_INTERSECT_FIRST);
1347 gst_caps_unref (out_caps);
1348 out_caps = tmp;
1349 }
1350
1351 GST_DEBUG_OBJECT (viewconvert, "Returning caps %" GST_PTR_FORMAT, out_caps);
1352 return out_caps;
1353 }
1354
1355 static GstCaps *
fixate_size(GstVulkanViewConvert * viewconvert,GstPadDirection direction,GstCaps * caps,GstCaps * othercaps)1356 fixate_size (GstVulkanViewConvert * viewconvert,
1357 GstPadDirection direction, GstCaps * caps, GstCaps * othercaps)
1358 {
1359 GstStructure *ins, *outs;
1360 const GValue *from_par, *to_par;
1361 GValue fpar = { 0, }, tpar = {
1362 0,};
1363
1364 othercaps = gst_caps_make_writable (othercaps);
1365 othercaps = gst_caps_truncate (othercaps);
1366
1367 GST_DEBUG_OBJECT (viewconvert, "trying to fixate othercaps %" GST_PTR_FORMAT
1368 " based on caps %" GST_PTR_FORMAT, othercaps, caps);
1369
1370 ins = gst_caps_get_structure (caps, 0);
1371 outs = gst_caps_get_structure (othercaps, 0);
1372
1373 from_par = gst_structure_get_value (ins, "pixel-aspect-ratio");
1374 to_par = gst_structure_get_value (outs, "pixel-aspect-ratio");
1375
1376 /* If we're fixating from the sinkpad we always set the PAR and
1377 * assume that missing PAR on the sinkpad means 1/1 and
1378 * missing PAR on the srcpad means undefined
1379 */
1380 if (direction == GST_PAD_SINK) {
1381 if (!from_par) {
1382 g_value_init (&fpar, GST_TYPE_FRACTION);
1383 gst_value_set_fraction (&fpar, 1, 1);
1384 from_par = &fpar;
1385 }
1386 if (!to_par) {
1387 g_value_init (&tpar, GST_TYPE_FRACTION);
1388 gst_value_set_fraction (&tpar, 1, 1);
1389 to_par = &tpar;
1390 }
1391 } else {
1392 if (!to_par) {
1393 g_value_init (&tpar, GST_TYPE_FRACTION);
1394 gst_value_set_fraction (&tpar, 1, 1);
1395 to_par = &tpar;
1396
1397 gst_structure_set (outs, "pixel-aspect-ratio", GST_TYPE_FRACTION, 1, 1,
1398 NULL);
1399 }
1400 if (!from_par) {
1401 g_value_init (&fpar, GST_TYPE_FRACTION);
1402 gst_value_set_fraction (&fpar, 1, 1);
1403 from_par = &fpar;
1404 }
1405 }
1406
1407 /* we have both PAR but they might not be fixated */
1408 {
1409 gint from_w, from_h, from_par_n, from_par_d, to_par_n, to_par_d;
1410 gint w = 0, h = 0;
1411 gint from_dar_n, from_dar_d;
1412 gint num, den;
1413
1414 /* from_par should be fixed */
1415 g_return_val_if_fail (gst_value_is_fixed (from_par), othercaps);
1416
1417 from_par_n = gst_value_get_fraction_numerator (from_par);
1418 from_par_d = gst_value_get_fraction_denominator (from_par);
1419
1420 gst_structure_get_int (ins, "width", &from_w);
1421 gst_structure_get_int (ins, "height", &from_h);
1422
1423 gst_structure_get_int (outs, "width", &w);
1424 gst_structure_get_int (outs, "height", &h);
1425
1426 /* if both width and height are already fixed, we can't do anything
1427 * about it anymore */
1428 if (w && h) {
1429 GST_DEBUG_OBJECT (viewconvert,
1430 "dimensions already set to %dx%d, not fixating", w, h);
1431 if (!gst_value_is_fixed (to_par)) {
1432 GST_DEBUG_OBJECT (viewconvert, "fixating to_par to %dx%d", 1, 1);
1433 if (gst_structure_has_field (outs, "pixel-aspect-ratio"))
1434 gst_structure_fixate_field_nearest_fraction (outs,
1435 "pixel-aspect-ratio", 1, 1);
1436 }
1437 goto done;
1438 }
1439
1440 /* Calculate input DAR */
1441 if (!gst_util_fraction_multiply (from_w, from_h, from_par_n, from_par_d,
1442 &from_dar_n, &from_dar_d)) {
1443 GST_ELEMENT_ERROR (viewconvert, CORE, NEGOTIATION, (NULL),
1444 ("Error calculating the output scaled size - integer overflow"));
1445 goto done;
1446 }
1447
1448 GST_DEBUG_OBJECT (viewconvert, "Input DAR is %d/%d", from_dar_n,
1449 from_dar_d);
1450
1451 /* If either width or height are fixed there's not much we
1452 * can do either except choosing a height or width and PAR
1453 * that matches the DAR as good as possible
1454 */
1455 if (h) {
1456 gint num, den;
1457
1458 GST_DEBUG_OBJECT (viewconvert, "height is fixed (%d)", h);
1459
1460 if (!gst_value_is_fixed (to_par)) {
1461 /* (shortcut) copy-paste (??) of videoscale seems to aim for 1/1,
1462 * so let's make it so ...
1463 * especially if following code assumes fixed */
1464 GST_DEBUG_OBJECT (viewconvert, "fixating to_par to 1x1");
1465 gst_structure_fixate_field_nearest_fraction (outs,
1466 "pixel-aspect-ratio", 1, 1);
1467 to_par = gst_structure_get_value (outs, "pixel-aspect-ratio");
1468 }
1469
1470 /* PAR is fixed, choose the height that is nearest to the
1471 * height with the same DAR */
1472 to_par_n = gst_value_get_fraction_numerator (to_par);
1473 to_par_d = gst_value_get_fraction_denominator (to_par);
1474
1475 GST_DEBUG_OBJECT (viewconvert, "PAR is fixed %d/%d", to_par_n, to_par_d);
1476
1477 if (!gst_util_fraction_multiply (from_dar_n, from_dar_d, to_par_d,
1478 to_par_n, &num, &den)) {
1479 GST_ELEMENT_ERROR (viewconvert, CORE, NEGOTIATION, (NULL),
1480 ("Error calculating the output scaled size - integer overflow"));
1481 goto done;
1482 }
1483
1484 w = (guint) gst_util_uint64_scale_int (h, num, den);
1485 gst_structure_fixate_field_nearest_int (outs, "width", w);
1486
1487 goto done;
1488 } else if (w) {
1489 gint num, den;
1490
1491 GST_DEBUG_OBJECT (viewconvert, "width is fixed (%d)", w);
1492
1493 if (!gst_value_is_fixed (to_par)) {
1494 /* (shortcut) copy-paste (??) of videoscale seems to aim for 1/1,
1495 * so let's make it so ...
1496 * especially if following code assumes fixed */
1497 GST_DEBUG_OBJECT (viewconvert, "fixating to_par to 1x1");
1498 gst_structure_fixate_field_nearest_fraction (outs,
1499 "pixel-aspect-ratio", 1, 1);
1500 to_par = gst_structure_get_value (outs, "pixel-aspect-ratio");
1501 }
1502
1503 /* PAR is fixed, choose the height that is nearest to the
1504 * height with the same DAR */
1505 to_par_n = gst_value_get_fraction_numerator (to_par);
1506 to_par_d = gst_value_get_fraction_denominator (to_par);
1507
1508 GST_DEBUG_OBJECT (viewconvert, "PAR is fixed %d/%d", to_par_n, to_par_d);
1509
1510 if (!gst_util_fraction_multiply (from_dar_n, from_dar_d, to_par_d,
1511 to_par_n, &num, &den)) {
1512 GST_ELEMENT_ERROR (viewconvert, CORE, NEGOTIATION, (NULL),
1513 ("Error calculating the output scaled size - integer overflow"));
1514 goto done;
1515 }
1516
1517 h = (guint) gst_util_uint64_scale_int (w, den, num);
1518 gst_structure_fixate_field_nearest_int (outs, "height", h);
1519
1520 goto done;
1521 } else if (gst_value_is_fixed (to_par)) {
1522 GstStructure *tmp;
1523 gint set_h, set_w, f_h, f_w;
1524
1525 to_par_n = gst_value_get_fraction_numerator (to_par);
1526 to_par_d = gst_value_get_fraction_denominator (to_par);
1527
1528 /* Calculate scale factor for the PAR change */
1529 if (!gst_util_fraction_multiply (from_dar_n, from_dar_d, to_par_n,
1530 to_par_d, &num, &den)) {
1531 GST_ELEMENT_ERROR (viewconvert, CORE, NEGOTIATION, (NULL),
1532 ("Error calculating the output scaled size - integer overflow"));
1533 goto done;
1534 }
1535
1536 /* Try to keep the input height */
1537 tmp = gst_structure_copy (outs);
1538 gst_structure_fixate_field_nearest_int (tmp, "height", from_h);
1539 gst_structure_get_int (tmp, "height", &set_h);
1540
1541 /* This might have failed but try to scale the width
1542 * to keep the DAR nonetheless */
1543 w = (guint) gst_util_uint64_scale_int (set_h, num, den);
1544 gst_structure_fixate_field_nearest_int (tmp, "width", w);
1545 gst_structure_get_int (tmp, "width", &set_w);
1546 gst_structure_free (tmp);
1547
1548 /* We kept the DAR and the height is nearest to the original height */
1549 if (set_w == w) {
1550 gst_structure_set (outs, "width", G_TYPE_INT, set_w, "height",
1551 G_TYPE_INT, set_h, NULL);
1552 goto done;
1553 }
1554
1555 f_h = set_h;
1556 f_w = set_w;
1557
1558 /* If the former failed, try to keep the input width at least */
1559 tmp = gst_structure_copy (outs);
1560 gst_structure_fixate_field_nearest_int (tmp, "width", from_w);
1561 gst_structure_get_int (tmp, "width", &set_w);
1562
1563 /* This might have failed but try to scale the width
1564 * to keep the DAR nonetheless */
1565 h = (guint) gst_util_uint64_scale_int (set_w, den, num);
1566 gst_structure_fixate_field_nearest_int (tmp, "height", h);
1567 gst_structure_get_int (tmp, "height", &set_h);
1568 gst_structure_free (tmp);
1569
1570 /* We kept the DAR and the width is nearest to the original width */
1571 if (set_h == h) {
1572 gst_structure_set (outs, "width", G_TYPE_INT, set_w, "height",
1573 G_TYPE_INT, set_h, NULL);
1574 goto done;
1575 }
1576
1577 /* If all this failed, keep the height that was nearest to the original
1578 * height and the nearest possible width. This changes the DAR but
1579 * there's not much else to do here.
1580 */
1581 gst_structure_set (outs, "width", G_TYPE_INT, f_w, "height", G_TYPE_INT,
1582 f_h, NULL);
1583 goto done;
1584 } else {
1585 GstStructure *tmp;
1586 gint set_h, set_w, set_par_n, set_par_d, tmp2;
1587
1588 /* width, height and PAR are not fixed */
1589
1590 /* First try to keep the height and width as good as possible
1591 * and scale PAR */
1592 tmp = gst_structure_copy (outs);
1593 gst_structure_fixate_field_nearest_int (tmp, "height", from_h);
1594 gst_structure_get_int (tmp, "height", &set_h);
1595 gst_structure_fixate_field_nearest_int (tmp, "width", from_w);
1596 gst_structure_get_int (tmp, "width", &set_w);
1597
1598 if (!gst_util_fraction_multiply (from_dar_n, from_dar_d, set_h, set_w,
1599 &to_par_n, &to_par_d)) {
1600 GST_ELEMENT_ERROR (viewconvert, CORE, NEGOTIATION, (NULL),
1601 ("Error calculating the output scaled size - integer overflow"));
1602 gst_structure_free (tmp);
1603 goto done;
1604 }
1605
1606 if (!gst_structure_has_field (tmp, "pixel-aspect-ratio"))
1607 gst_structure_set_value (tmp, "pixel-aspect-ratio", to_par);
1608 gst_structure_fixate_field_nearest_fraction (tmp, "pixel-aspect-ratio",
1609 to_par_n, to_par_d);
1610 gst_structure_get_fraction (tmp, "pixel-aspect-ratio", &set_par_n,
1611 &set_par_d);
1612 gst_structure_free (tmp);
1613
1614 if (set_par_n == to_par_n && set_par_d == to_par_d) {
1615 gst_structure_set (outs, "width", G_TYPE_INT, set_w, "height",
1616 G_TYPE_INT, set_h, NULL);
1617
1618 if (gst_structure_has_field (outs, "pixel-aspect-ratio") ||
1619 set_par_n != set_par_d)
1620 gst_structure_set (outs, "pixel-aspect-ratio", GST_TYPE_FRACTION,
1621 set_par_n, set_par_d, NULL);
1622 goto done;
1623 }
1624
1625 /* Otherwise try to scale width to keep the DAR with the set
1626 * PAR and height */
1627 if (!gst_util_fraction_multiply (from_dar_n, from_dar_d, set_par_d,
1628 set_par_n, &num, &den)) {
1629 GST_ELEMENT_ERROR (viewconvert, CORE, NEGOTIATION, (NULL),
1630 ("Error calculating the output scaled size - integer overflow"));
1631 goto done;
1632 }
1633
1634 w = (guint) gst_util_uint64_scale_int (set_h, num, den);
1635 tmp = gst_structure_copy (outs);
1636 gst_structure_fixate_field_nearest_int (tmp, "width", w);
1637 gst_structure_get_int (tmp, "width", &tmp2);
1638 gst_structure_free (tmp);
1639
1640 if (tmp2 == w) {
1641 gst_structure_set (outs, "width", G_TYPE_INT, tmp2, "height",
1642 G_TYPE_INT, set_h, NULL);
1643 if (gst_structure_has_field (outs, "pixel-aspect-ratio") ||
1644 set_par_n != set_par_d)
1645 gst_structure_set (outs, "pixel-aspect-ratio", GST_TYPE_FRACTION,
1646 set_par_n, set_par_d, NULL);
1647 goto done;
1648 }
1649
1650 /* ... or try the same with the height */
1651 h = (guint) gst_util_uint64_scale_int (set_w, den, num);
1652 tmp = gst_structure_copy (outs);
1653 gst_structure_fixate_field_nearest_int (tmp, "height", h);
1654 gst_structure_get_int (tmp, "height", &tmp2);
1655 gst_structure_free (tmp);
1656
1657 if (tmp2 == h) {
1658 gst_structure_set (outs, "width", G_TYPE_INT, set_w, "height",
1659 G_TYPE_INT, tmp2, NULL);
1660 if (gst_structure_has_field (outs, "pixel-aspect-ratio") ||
1661 set_par_n != set_par_d)
1662 gst_structure_set (outs, "pixel-aspect-ratio", GST_TYPE_FRACTION,
1663 set_par_n, set_par_d, NULL);
1664 goto done;
1665 }
1666
1667 /* If all fails we can't keep the DAR and take the nearest values
1668 * for everything from the first try */
1669 gst_structure_set (outs, "width", G_TYPE_INT, set_w, "height",
1670 G_TYPE_INT, set_h, NULL);
1671 if (gst_structure_has_field (outs, "pixel-aspect-ratio") ||
1672 set_par_n != set_par_d)
1673 gst_structure_set (outs, "pixel-aspect-ratio", GST_TYPE_FRACTION,
1674 set_par_n, set_par_d, NULL);
1675 }
1676 }
1677
1678 done:
1679 othercaps = gst_caps_fixate (othercaps);
1680
1681 GST_DEBUG_OBJECT (viewconvert, "fixated othercaps to %" GST_PTR_FORMAT,
1682 othercaps);
1683
1684 if (from_par == &fpar)
1685 g_value_unset (&fpar);
1686 if (to_par == &tpar)
1687 g_value_unset (&tpar);
1688
1689 return othercaps;
1690 }
1691
1692 static GstCaps *
gst_vulkan_view_convert_fixate_caps(GstBaseTransform * bt,GstPadDirection direction,GstCaps * caps,GstCaps * othercaps)1693 gst_vulkan_view_convert_fixate_caps (GstBaseTransform * bt,
1694 GstPadDirection direction, GstCaps * caps, GstCaps * othercaps)
1695 {
1696 GstVulkanViewConvert *viewconvert = GST_VULKAN_VIEW_CONVERT (bt);
1697 GstVideoMultiviewMode mode = viewconvert->output_mode_override;
1698 GstVideoMultiviewFlags flags = viewconvert->output_flags_override;
1699 GstCaps *tmp;
1700
1701 othercaps = gst_caps_make_writable (othercaps);
1702 GST_LOG_OBJECT (viewconvert, "dir %s fixating %" GST_PTR_FORMAT
1703 " against caps %" GST_PTR_FORMAT,
1704 direction == GST_PAD_SINK ? "sink" : "src", othercaps, caps);
1705
1706 if (direction == GST_PAD_SINK) {
1707 if (mode != GST_VIDEO_MULTIVIEW_MODE_NONE) {
1708 /* We have a requested output mode and are fixating source caps, try and enforce it */
1709 GST_DEBUG_OBJECT (bt, "fixating multiview mode using the configured "
1710 "output override mode 0x%x and flags 0x%x", mode, flags);
1711 tmp = _intersect_with_mview_mode (othercaps, mode, flags);
1712 gst_caps_unref (othercaps);
1713 othercaps = tmp;
1714 } else {
1715 /* See if we can do passthrough */
1716 GstVideoInfo info;
1717
1718 if (gst_video_info_from_caps (&info, caps)) {
1719 GstVideoMultiviewMode mode = GST_VIDEO_INFO_MULTIVIEW_MODE (&info);
1720 GstVideoMultiviewFlags flags = GST_VIDEO_INFO_MULTIVIEW_FLAGS (&info);
1721
1722 if (viewconvert->input_mode_override != GST_VIDEO_MULTIVIEW_MODE_NONE) {
1723 mode = viewconvert->input_mode_override;
1724 flags = viewconvert->input_flags_override;
1725 }
1726
1727 tmp = _intersect_with_mview_mode (othercaps, mode, flags);
1728 if (gst_caps_is_empty (tmp)) {
1729 /* Nope, we can't pass our input caps downstream */
1730 gst_caps_unref (tmp);
1731 } else {
1732 GST_DEBUG_OBJECT (bt, "can configure a passthrough multiview mode "
1733 "using the input override mode 0x%x and flags 0x%x", mode, flags);
1734 gst_caps_unref (othercaps);
1735 othercaps = tmp;
1736 goto done;
1737 }
1738 }
1739
1740 /* Prefer an unpacked mode for output */
1741 tmp =
1742 _intersect_with_mview_modes (othercaps,
1743 gst_video_multiview_get_unpacked_modes ());
1744 if (!gst_caps_is_empty (tmp)) {
1745 GST_DEBUG_OBJECT (bt, "preferring an unpacked multiview mode");
1746 gst_caps_unref (othercaps);
1747 othercaps = tmp;
1748 } else {
1749 gst_caps_unref (tmp);
1750 }
1751 }
1752 } else if (viewconvert->input_mode_override != GST_VIDEO_MULTIVIEW_MODE_NONE) {
1753 /* See if we can coerce the caps into matching input mode/flags,
1754 * in case it doesn't care at all, but allow it not to too */
1755 mode = viewconvert->input_mode_override;
1756 flags = viewconvert->input_flags_override;
1757 tmp = _intersect_with_mview_mode (othercaps, mode, flags);
1758 if (gst_caps_is_empty (tmp)) {
1759 /* Nope, we can pass our input caps downstream */
1760 gst_caps_unref (tmp);
1761 } else {
1762 GST_DEBUG_OBJECT (bt, "can configure a passthrough multiview mode "
1763 "using the input override mode 0x%x and flags 0x%x", mode, flags);
1764 gst_caps_unref (othercaps);
1765 othercaps = tmp;
1766 }
1767 }
1768
1769 done:
1770 othercaps = fixate_size (viewconvert, direction, caps, othercaps);
1771 othercaps = gst_caps_fixate (othercaps);
1772
1773 GST_DEBUG_OBJECT (viewconvert, "dir %s fixated to %" GST_PTR_FORMAT
1774 " against caps %" GST_PTR_FORMAT,
1775 direction == GST_PAD_SINK ? "sink" : "src", othercaps, caps);
1776 return othercaps;
1777 }
1778
1779 static gboolean
create_descriptor_set_layout(GstVulkanViewConvert * conv,GError ** error)1780 create_descriptor_set_layout (GstVulkanViewConvert * conv, GError ** error)
1781 {
1782 GstVulkanVideoFilter *vfilter = GST_VULKAN_VIDEO_FILTER (conv);
1783 VkDescriptorSetLayoutBinding bindings[GST_VIDEO_MAX_PLANES * 2 + 1] =
1784 { {0,}, };
1785 VkDescriptorSetLayoutCreateInfo layout_info;
1786 VkDescriptorSetLayout descriptor_set_layout;
1787 int descriptor_n = 0;
1788 VkResult err;
1789 int i;
1790
1791 /* *INDENT-OFF* */
1792 bindings[descriptor_n++] = (VkDescriptorSetLayoutBinding) {
1793 .binding = 0,
1794 .descriptorCount = 1,
1795 .descriptorType = VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER,
1796 .pImmutableSamplers = NULL,
1797 .stageFlags = VK_SHADER_STAGE_VERTEX_BIT | VK_SHADER_STAGE_FRAGMENT_BIT
1798 };
1799 for (i = 0; i < GST_VIDEO_INFO_N_PLANES (&vfilter->in_info) * 2; i++) {
1800 bindings[descriptor_n++] = (VkDescriptorSetLayoutBinding) {
1801 .binding = i+1,
1802 .descriptorCount = 1,
1803 .descriptorType = VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER,
1804 .pImmutableSamplers = NULL,
1805 .stageFlags = VK_SHADER_STAGE_FRAGMENT_BIT
1806 };
1807 };
1808
1809 layout_info = (VkDescriptorSetLayoutCreateInfo) {
1810 .sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO,
1811 .pNext = NULL,
1812 .bindingCount = descriptor_n,
1813 .pBindings = bindings
1814 };
1815 /* *INDENT-ON* */
1816
1817 err =
1818 vkCreateDescriptorSetLayout (vfilter->device->device, &layout_info,
1819 NULL, &descriptor_set_layout);
1820 if (gst_vulkan_error_to_g_error (err, error,
1821 "vkCreateDescriptorSetLayout") < 0) {
1822 return FALSE;
1823 }
1824
1825 conv->quad->descriptor_set_layout =
1826 gst_vulkan_handle_new_wrapped (vfilter->device,
1827 GST_VULKAN_HANDLE_TYPE_DESCRIPTOR_SET_LAYOUT,
1828 (GstVulkanHandleTypedef) descriptor_set_layout,
1829 gst_vulkan_handle_free_descriptor_set_layout, NULL);
1830
1831 return TRUE;
1832 }
1833
1834 static gboolean
gst_vulkan_view_convert_start(GstBaseTransform * bt)1835 gst_vulkan_view_convert_start (GstBaseTransform * bt)
1836 {
1837 GstVulkanVideoFilter *vfilter = GST_VULKAN_VIDEO_FILTER (bt);
1838 GstVulkanViewConvert *conv = GST_VULKAN_VIEW_CONVERT (bt);
1839 GstVulkanHandle *vert, *frag;
1840
1841 if (!GST_BASE_TRANSFORM_CLASS (parent_class)->start (bt))
1842 return FALSE;
1843
1844 conv->quad = gst_vulkan_full_screen_quad_new (vfilter->queue);
1845
1846 if (!(vert =
1847 gst_vulkan_create_shader (vfilter->device, identity_vert,
1848 identity_vert_size, NULL))) {
1849 return FALSE;
1850 }
1851 if (!(frag =
1852 gst_vulkan_create_shader (vfilter->device, view_convert_frag,
1853 view_convert_frag_size, NULL))) {
1854 gst_vulkan_handle_unref (vert);
1855 return FALSE;
1856 }
1857
1858 if (!gst_vulkan_full_screen_quad_set_shaders (conv->quad, vert, frag)) {
1859 gst_vulkan_handle_unref (vert);
1860 gst_vulkan_handle_unref (frag);
1861 return FALSE;
1862 }
1863 gst_vulkan_handle_unref (vert);
1864 gst_vulkan_handle_unref (frag);
1865
1866 return TRUE;
1867 }
1868
1869 static gboolean
gst_vulkan_view_convert_set_caps(GstBaseTransform * bt,GstCaps * in_caps,GstCaps * out_caps)1870 gst_vulkan_view_convert_set_caps (GstBaseTransform * bt, GstCaps * in_caps,
1871 GstCaps * out_caps)
1872 {
1873 GstVulkanVideoFilter *vfilter = GST_VULKAN_VIDEO_FILTER (bt);
1874 GstVulkanViewConvert *conv = GST_VULKAN_VIEW_CONVERT (bt);
1875
1876 if (!GST_BASE_TRANSFORM_CLASS (parent_class)->set_caps (bt, in_caps,
1877 out_caps))
1878 return FALSE;
1879
1880 if (!gst_vulkan_full_screen_quad_set_info (conv->quad, &vfilter->in_info,
1881 &vfilter->out_info))
1882 return FALSE;
1883
1884 if (conv->uniform)
1885 gst_memory_unref (conv->uniform);
1886 conv->uniform = NULL;
1887
1888 return TRUE;
1889 }
1890
1891 static gboolean
gst_vulkan_view_convert_stop(GstBaseTransform * bt)1892 gst_vulkan_view_convert_stop (GstBaseTransform * bt)
1893 {
1894 GstVulkanViewConvert *conv = GST_VULKAN_VIEW_CONVERT (bt);
1895
1896 gst_clear_object (&conv->quad);
1897 if (conv->uniform)
1898 gst_memory_unref (conv->uniform);
1899 conv->uniform = NULL;
1900
1901 return GST_BASE_TRANSFORM_CLASS (parent_class)->stop (bt);
1902 }
1903
1904 static GstFlowReturn
gst_vulkan_view_convert_transform(GstBaseTransform * bt,GstBuffer * inbuf,GstBuffer * outbuf)1905 gst_vulkan_view_convert_transform (GstBaseTransform * bt, GstBuffer * inbuf,
1906 GstBuffer * outbuf)
1907 {
1908 GstVulkanVideoFilter *vfilter = GST_VULKAN_VIDEO_FILTER (bt);
1909 GstVulkanViewConvert *conv = GST_VULKAN_VIEW_CONVERT (bt);
1910 GstVulkanImageView *in_img_views[GST_VIDEO_MAX_PLANES] = { NULL, };
1911 GstVulkanImageView *out_img_views[GST_VIDEO_MAX_PLANES] = { NULL, };
1912 GstVulkanCommandBuffer *cmd_buf = NULL;
1913 GstVulkanFence *fence = NULL;
1914 GError *error = NULL;
1915 VkResult err;
1916 int i;
1917
1918 if (!gst_vulkan_full_screen_quad_set_input_buffer (conv->quad, inbuf, &error))
1919 goto error;
1920 if (!gst_vulkan_full_screen_quad_set_output_buffer (conv->quad, outbuf,
1921 &error))
1922 goto error;
1923
1924 fence = gst_vulkan_device_create_fence (vfilter->device, &error);
1925 if (!fence)
1926 goto error;
1927
1928 for (i = 0; i < GST_VIDEO_INFO_N_PLANES (&conv->quad->in_info); i++) {
1929 GstMemory *img_mem = gst_buffer_peek_memory (inbuf, i);
1930 if (!gst_is_vulkan_image_memory (img_mem)) {
1931 g_set_error_literal (&error, GST_VULKAN_ERROR, GST_VULKAN_FAILED,
1932 "Input memory must be a GstVulkanImageMemory");
1933 goto error;
1934 }
1935 in_img_views[i] =
1936 gst_vulkan_get_or_create_image_view ((GstVulkanImageMemory *) img_mem);
1937 gst_vulkan_trash_list_add (conv->quad->trash_list,
1938 gst_vulkan_trash_list_acquire (conv->quad->trash_list, fence,
1939 gst_vulkan_trash_mini_object_unref,
1940 (GstMiniObject *) in_img_views[i]));
1941 }
1942 for (i = 0; i < GST_VIDEO_INFO_N_PLANES (&conv->quad->out_info); i++) {
1943 GstMemory *mem = gst_buffer_peek_memory (outbuf, i);
1944 if (!gst_is_vulkan_image_memory (mem)) {
1945 g_set_error_literal (&error, GST_VULKAN_ERROR, GST_VULKAN_FAILED,
1946 "Output memory must be a GstVulkanImageMemory");
1947 goto error;
1948 }
1949 out_img_views[i] =
1950 gst_vulkan_get_or_create_image_view ((GstVulkanImageMemory *) mem);
1951 gst_vulkan_trash_list_add (conv->quad->trash_list,
1952 gst_vulkan_trash_list_acquire (conv->quad->trash_list, fence,
1953 gst_vulkan_trash_mini_object_unref,
1954 (GstMiniObject *) out_img_views[i]));
1955 }
1956
1957 {
1958 GstMemory *uniforms = get_uniforms (conv, in_img_views, out_img_views);
1959 if (!gst_vulkan_full_screen_quad_set_uniform_buffer (conv->quad, uniforms,
1960 &error))
1961 goto error;
1962 gst_memory_unref (uniforms);
1963 }
1964
1965 if (!conv->quad->descriptor_set_layout)
1966 if (!create_descriptor_set_layout (conv, &error))
1967 goto error;
1968
1969 if (!gst_vulkan_full_screen_quad_prepare_draw (conv->quad, fence, &error))
1970 goto error;
1971
1972 if (!(cmd_buf =
1973 gst_vulkan_command_pool_create (conv->quad->cmd_pool, &error)))
1974 goto error;
1975
1976 {
1977 VkCommandBufferBeginInfo cmd_buf_info = { 0, };
1978
1979 /* *INDENT-OFF* */
1980 cmd_buf_info = (VkCommandBufferBeginInfo) {
1981 .sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO,
1982 .pNext = NULL,
1983 .flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT,
1984 .pInheritanceInfo = NULL
1985 };
1986 /* *INDENT-ON* */
1987
1988 gst_vulkan_command_buffer_lock (cmd_buf);
1989 err = vkBeginCommandBuffer (cmd_buf->cmd, &cmd_buf_info);
1990 if (gst_vulkan_error_to_g_error (err, &error, "vkBeginCommandBuffer") < 0)
1991 goto error;
1992 }
1993
1994 update_descriptor_set (conv, in_img_views);
1995 if (!gst_vulkan_full_screen_quad_fill_command_buffer (conv->quad, cmd_buf,
1996 fence, &error))
1997 goto unlock_error;
1998
1999 err = vkEndCommandBuffer (cmd_buf->cmd);
2000 gst_vulkan_command_buffer_unlock (cmd_buf);
2001 if (gst_vulkan_error_to_g_error (err, &error, "vkEndCommandBuffer") < 0)
2002 goto error;
2003
2004 if (!gst_vulkan_full_screen_quad_submit (conv->quad, cmd_buf, fence, &error))
2005 goto error;
2006
2007 gst_vulkan_fence_unref (fence);
2008
2009 return GST_FLOW_OK;
2010
2011 unlock_error:
2012 if (cmd_buf) {
2013 gst_vulkan_command_buffer_unlock (cmd_buf);
2014 gst_vulkan_command_buffer_unref (cmd_buf);
2015 }
2016
2017 error:
2018 gst_clear_mini_object ((GstMiniObject **) & fence);
2019
2020 GST_ELEMENT_ERROR (bt, LIBRARY, FAILED, ("%s", error->message), (NULL));
2021 g_clear_error (&error);
2022 return GST_FLOW_ERROR;
2023 }
2024