• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 /*
2  * GStreamer
3  * Copyright (C) 2019 Matthew Waters <matthew@centricular.com>
4  *
5  * This library is free software; you can redistribute it and/or
6  * modify it under the terms of the GNU Library General Public
7  * License as published by the Free Software Foundation; either
8  * version 2 of the License, or (at your option) any later version.
9  *
10  * This library is distributed in the hope that it will be useful,
11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
13  * Library General Public License for more details.
14  *
15  * You should have received a copy of the GNU Library General Public
16  * License along with this library; if not, write to the
17  * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
18  * Boston, MA 02110-1301, USA.
19  */
20 
21 /**
22  * SECTION:element-vulkanimageidentity
23  * @title: vulkanimgeidentity
24  *
25  * vulkanimageidentity produces a vulkan image that is a copy of the input image.
26  */
27 
28 #ifdef HAVE_CONFIG_H
29 #include "config.h"
30 #endif
31 
32 #include <string.h>
33 
34 #include "gstvulkanelements.h"
35 #include "vkviewconvert.h"
36 
37 #include "shaders/identity.vert.h"
38 #include "shaders/view_convert.frag.h"
39 #include "gstvulkan-plugins-enumtypes.h"
40 
41 GST_DEBUG_CATEGORY (gst_debug_vulkan_view_convert);
42 #define GST_CAT_DEFAULT gst_debug_vulkan_view_convert
43 
44 /* *INDENT-OFF* */
45 /* These match the order and number of DOWNMIX_ANAGLYPH_* modes */
46 static float downmix_matrices[][2][12] = {
47   {                             /* Green-Magenta Dubois */
48     {-0.062f, 0.284f, -0.015f, 0.0, -0.158f, 0.668f, -0.027f, 0.0, -0.039f, 0.143f, 0.021f, 0.0},
49     {0.529f, -0.016f, 0.009f, 0.0, 0.705f, -0.015f, 0.075f, 0.0, 0.024f, -0.065f, 0.937f, 0.0}
50   },
51   {                             /* Red-Cyan Dubois */
52     /* Source of this matrix: http://www.site.uottawa.ca/~edubois/anaglyph/LeastSquaresHowToPhotoshop.pdf */
53     {0.437f, -0.062f, -0.048f, 0.0, 0.449f, -0.062f, -0.050f, 0.0, 0.164f, -0.024f, -0.017f},
54     {-0.011f, 0.377f, -0.026f, 0.0, -0.032f, 0.761f, -0.093f, 0.0, -0.007f, 0.009f, 1.234f}
55   },
56   {                             /* Amber-blue Dubois */
57     {1.062f, -0.026f, -0.038f, 0.0, -0.205f, 0.908f, -0.173f, 0.0, 0.299f, 0.068f, 0.022f},
58     {-0.016f, 0.006f, 0.094f, 0.0, -0.123f, 0.062f, 0.185f, 0.0, -0.017f, -0.017f, 0.911f}
59   }
60 };
61 /* *INDENT-ON* */
62 
63 struct ViewUpdate
64 {
65   int in_reorder_idx[4];
66   int out_reorder_idx[4];
67   float tex_offset[2][2];
68   float tex_scale[2][2];
69   int tex_size[2];
70   int output_type;
71   int _padding;
72   float downmix[2][12];
73 };
74 
75 static void
get_rgb_format_swizzle_order(GstVideoFormat format,gint swizzle[GST_VIDEO_MAX_COMPONENTS])76 get_rgb_format_swizzle_order (GstVideoFormat format,
77     gint swizzle[GST_VIDEO_MAX_COMPONENTS])
78 {
79   const GstVideoFormatInfo *finfo = gst_video_format_get_info (format);
80   int c_i = 0, i;
81 
82   g_return_if_fail (finfo->flags & GST_VIDEO_FORMAT_FLAG_RGB
83       || format == GST_VIDEO_FORMAT_AYUV);
84 
85   for (i = 0; i < finfo->n_components; i++) {
86     swizzle[c_i++] = finfo->poffset[i];
87   }
88 
89   /* special case spaced RGB formats as the space does not contain a poffset
90    * value and we need all four components to be valid in order to swizzle
91    * correctly */
92   if (format == GST_VIDEO_FORMAT_xRGB || format == GST_VIDEO_FORMAT_xBGR) {
93     swizzle[c_i++] = 0;
94   } else if (format == GST_VIDEO_FORMAT_RGBx || format == GST_VIDEO_FORMAT_BGRx) {
95     swizzle[c_i++] = 3;
96   } else {
97     for (i = finfo->n_components; i < GST_VIDEO_MAX_COMPONENTS; i++) {
98       swizzle[c_i++] = -1;
99     }
100   }
101 }
102 
103 static void
get_vulkan_rgb_format_swizzle_order(VkFormat format,gint * swizzle,guint swizzle_count,guint offset)104 get_vulkan_rgb_format_swizzle_order (VkFormat format, gint * swizzle,
105     guint swizzle_count, guint offset)
106 {
107   const GstVulkanFormatInfo *finfo = gst_vulkan_format_get_info (format);
108   int i;
109 
110   g_return_if_fail (finfo->flags & GST_VULKAN_FORMAT_FLAG_RGB);
111   g_return_if_fail (finfo->n_components <= swizzle_count);
112 
113   for (i = 0; i < finfo->n_components; i++) {
114     swizzle[i] = offset + finfo->poffset[i];
115   }
116   for (i = finfo->n_components; i < swizzle_count; i++) {
117     swizzle[i] = -1;
118   }
119 }
120 
121 /* given a swizzle index, produce an index such that:
122  *
123  * swizzle[idx[i]] == identity[i] where:
124  * - swizzle is the original swizzle
125  * - idx is the result
126  * - identity = {0, 1, 2,...}
127  * - unset fields are marked by -1
128  */
129 static void
swizzle_identity_order(gint * swizzle,gint * idx)130 swizzle_identity_order (gint * swizzle, gint * idx)
131 {
132   int i;
133 
134   for (i = 0; i < GST_VIDEO_MAX_COMPONENTS; i++) {
135     idx[i] = -1;
136   }
137 
138   for (i = 0; i < GST_VIDEO_MAX_COMPONENTS; i++) {
139     if (swizzle[i] >= 0 && swizzle[i] < 4 && idx[swizzle[i]] == -1) {
140       idx[swizzle[i]] = i;
141     }
142   }
143 }
144 
145 static void
video_format_to_reorder(GstVideoFormat v_format,gint * reorder,gboolean input)146 video_format_to_reorder (GstVideoFormat v_format, gint * reorder,
147     gboolean input)
148 {
149   switch (v_format) {
150     case GST_VIDEO_FORMAT_RGBA:
151     case GST_VIDEO_FORMAT_RGBx:
152     case GST_VIDEO_FORMAT_BGRA:
153     case GST_VIDEO_FORMAT_BGRx:
154     case GST_VIDEO_FORMAT_ARGB:
155     case GST_VIDEO_FORMAT_xRGB:
156     case GST_VIDEO_FORMAT_ABGR:
157     case GST_VIDEO_FORMAT_xBGR:
158     case GST_VIDEO_FORMAT_AYUV:
159       get_rgb_format_swizzle_order (v_format, reorder);
160       break;
161     case GST_VIDEO_FORMAT_UYVY:
162       reorder[0] = 1;
163       reorder[1] = 0;
164       reorder[2] = input ? 3 : 2;
165       reorder[3] = 0;
166       break;
167     case GST_VIDEO_FORMAT_YUY2:
168       reorder[0] = 0;
169       reorder[1] = 1;
170       reorder[2] = 0;
171       reorder[3] = input ? 3 : 2;
172       break;
173     case GST_VIDEO_FORMAT_NV12:
174       reorder[0] = 0;
175       reorder[1] = 1;
176       reorder[2] = 2;
177       reorder[3] = 0;
178       break;
179     default:
180       g_assert_not_reached ();
181       break;
182   }
183 
184   GST_TRACE ("swizzle: %u, %u, %u, %u", reorder[0], reorder[1], reorder[2],
185       reorder[3]);
186 }
187 
188 static guint
finfo_get_plane_n_components(const GstVideoFormatInfo * finfo,guint plane)189 finfo_get_plane_n_components (const GstVideoFormatInfo * finfo, guint plane)
190 {
191   guint n_components = 0, i;
192 
193   switch (finfo->format) {
194     case GST_VIDEO_FORMAT_RGBx:
195     case GST_VIDEO_FORMAT_xRGB:
196     case GST_VIDEO_FORMAT_BGRx:
197     case GST_VIDEO_FORMAT_xBGR:
198       /* fixup spaced RGB formats as we treat the space as a normal alpha
199        * component */
200       return plane == 0 ? 4 : 0;
201     default:
202       break;
203   }
204 
205   for (i = 0; i < finfo->n_components; i++) {
206     if (finfo->plane[i] == plane)
207       n_components++;
208   }
209 
210   return n_components;
211 }
212 
213 static void
get_vulkan_format_swizzle_order(GstVideoFormat v_format,VkFormat vk_format[GST_VIDEO_MAX_PLANES],gint swizzle[GST_VIDEO_MAX_COMPONENTS])214 get_vulkan_format_swizzle_order (GstVideoFormat v_format,
215     VkFormat vk_format[GST_VIDEO_MAX_PLANES],
216     gint swizzle[GST_VIDEO_MAX_COMPONENTS])
217 {
218   const GstVideoFormatInfo *finfo;
219   int i, prev_in_i = 0;
220 
221   finfo = gst_video_format_get_info (v_format);
222   for (i = 0; i < finfo->n_planes; i++) {
223     guint plane_components = finfo_get_plane_n_components (finfo, i);
224     get_vulkan_rgb_format_swizzle_order (vk_format[i],
225         &swizzle[prev_in_i], plane_components, prev_in_i);
226     prev_in_i += plane_components;
227   }
228 
229   if (v_format == GST_VIDEO_FORMAT_YUY2 || v_format == GST_VIDEO_FORMAT_UYVY) {
230     /* Fixup these packed YUV formats as we use a two component format for
231      * a 4-component pixel and access two samples in the shader */
232     g_assert (swizzle[0] == 0);
233     g_assert (swizzle[1] == 1);
234     swizzle[2] = 2;
235     swizzle[3] = 3;
236   }
237 
238   GST_TRACE ("%s: %i, %i, %i, %i", finfo->name, swizzle[0], swizzle[1],
239       swizzle[2], swizzle[3]);
240 }
241 
242 static void
calculate_reorder_indexes(GstVideoFormat in_format,GstVulkanImageView * in_views[GST_VIDEO_MAX_COMPONENTS],GstVideoFormat out_format,GstVulkanImageView * out_views[GST_VIDEO_MAX_COMPONENTS],int ret_in[GST_VIDEO_MAX_COMPONENTS],int ret_out[GST_VIDEO_MAX_COMPONENTS])243 calculate_reorder_indexes (GstVideoFormat in_format,
244     GstVulkanImageView * in_views[GST_VIDEO_MAX_COMPONENTS],
245     GstVideoFormat out_format,
246     GstVulkanImageView * out_views[GST_VIDEO_MAX_COMPONENTS],
247     int ret_in[GST_VIDEO_MAX_COMPONENTS], int ret_out[GST_VIDEO_MAX_COMPONENTS])
248 {
249   const GstVideoFormatInfo *in_finfo, *out_finfo;
250   VkFormat in_vk_formats[GST_VIDEO_MAX_COMPONENTS];
251   VkFormat out_vk_formats[GST_VIDEO_MAX_COMPONENTS];
252   int in_vk_order[GST_VIDEO_MAX_COMPONENTS],
253       in_reorder[GST_VIDEO_MAX_COMPONENTS];
254   int out_vk_order[GST_VIDEO_MAX_COMPONENTS],
255       out_reorder[GST_VIDEO_MAX_COMPONENTS];
256   int tmp[GST_VIDEO_MAX_PLANES];
257   int i;
258 
259   in_finfo = gst_video_format_get_info (in_format);
260   out_finfo = gst_video_format_get_info (out_format);
261 
262   for (i = 0; i < in_finfo->n_planes; i++)
263     in_vk_formats[i] = in_views[i]->image->create_info.format;
264   for (i = 0; i < out_finfo->n_planes; i++)
265     out_vk_formats[i] = out_views[i]->image->create_info.format;
266 
267   get_vulkan_format_swizzle_order (in_format, in_vk_formats, in_vk_order);
268   video_format_to_reorder (in_format, in_reorder, TRUE);
269 
270   video_format_to_reorder (out_format, out_reorder, FALSE);
271   get_vulkan_format_swizzle_order (out_format, out_vk_formats, out_vk_order);
272 
273   for (i = 0; i < GST_VIDEO_MAX_COMPONENTS; i++)
274     tmp[i] = out_vk_order[out_reorder[i]];
275   /* find the identity order for RGBA->$format */
276   GST_TRACE ("pre-invert: %u, %u, %u, %u", tmp[0], tmp[1], tmp[2], tmp[3]);
277   if (out_format == GST_VIDEO_FORMAT_YUY2
278       || out_format == GST_VIDEO_FORMAT_UYVY) {
279     for (i = 0; i < GST_VIDEO_MAX_COMPONENTS; i++)
280       ret_out[i] = tmp[i];
281   } else {
282     swizzle_identity_order (tmp, ret_out);
283   }
284 
285   for (i = 0; i < GST_VIDEO_MAX_COMPONENTS; i++)
286     ret_in[i] = in_reorder[in_vk_order[i]];
287   GST_TRACE ("in reorder: %u, %u, %u, %u", ret_in[0], ret_in[1], ret_in[2],
288       ret_in[3]);
289   GST_TRACE ("out reorder: %u, %u, %u, %u", ret_out[0], ret_out[1], ret_out[2],
290       ret_out[3]);
291 }
292 
293 static void
update_descriptor_set(GstVulkanViewConvert * conv,GstVulkanImageView ** in_views)294 update_descriptor_set (GstVulkanViewConvert * conv,
295     GstVulkanImageView ** in_views)
296 {
297   GstVulkanVideoFilter *vfilter = GST_VULKAN_VIDEO_FILTER (conv);
298   VkDescriptorImageInfo image_info[GST_VIDEO_MAX_PLANES];
299   VkWriteDescriptorSet writes[GST_VIDEO_MAX_PLANES];
300   GstVideoMultiviewMode in_mode;
301   GstVideoMultiviewFlags in_flags, out_flags;
302   VkImageView views[GST_VIDEO_MAX_PLANES];
303   guint i = 0;
304 
305   in_mode = conv->input_mode_override;
306   in_flags = conv->input_flags_override;
307   if (in_mode == GST_VIDEO_MULTIVIEW_MODE_NONE)
308     in_flags = GST_VIDEO_INFO_MULTIVIEW_FLAGS (&vfilter->in_info);
309   out_flags = GST_VIDEO_INFO_MULTIVIEW_FLAGS (&vfilter->out_info);
310 
311   for (i = 0; i < GST_VIDEO_INFO_N_PLANES (&vfilter->in_info); i++) {
312     if ((in_flags & GST_VIDEO_MULTIVIEW_FLAGS_RIGHT_VIEW_FIRST) ==
313         (out_flags & GST_VIDEO_MULTIVIEW_FLAGS_RIGHT_VIEW_FIRST)) {
314       views[2 * i] = in_views[i]->view;
315       views[2 * i + 1] = in_views[i]->view;
316     } else {
317       views[2 * i] = in_views[i]->view;
318       views[2 * i + 1] = in_views[i]->view;
319     }
320   }
321 
322   for (i = 0; i < GST_VIDEO_INFO_N_PLANES (&vfilter->in_info) * 2; i++) {
323     /* *INDENT-OFF* */
324     image_info[i] = (VkDescriptorImageInfo) {
325         .imageLayout = VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL,
326         .imageView = views[i],
327         .sampler = (VkSampler) conv->quad->sampler->handle
328     };
329 
330     g_assert (i < GST_VIDEO_MAX_PLANES);
331 
332     writes[i] = (VkWriteDescriptorSet) {
333         .sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET,
334         .pNext = NULL,
335         .dstSet = conv->quad->descriptor_set->set,
336         .dstBinding = i+1,
337         .dstArrayElement = 0,
338         .descriptorType = VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER,
339         .descriptorCount = 1,
340         .pImageInfo = &image_info[i]
341     };
342     /* *INDENT-ON* */
343   }
344   g_assert (i <= G_N_ELEMENTS (writes));
345 
346   vkUpdateDescriptorSets (vfilter->device->device, i, writes, 0, NULL);
347 }
348 
349 static gboolean
create_uniform_buffer(GstVulkanViewConvert * conv)350 create_uniform_buffer (GstVulkanViewConvert * conv)
351 {
352   GstVulkanVideoFilter *vfilter = GST_VULKAN_VIDEO_FILTER (conv);
353 
354   conv->uniform =
355       gst_vulkan_buffer_memory_alloc (vfilter->device,
356       sizeof (struct ViewUpdate),
357       VK_BUFFER_USAGE_TRANSFER_DST_BIT | VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT,
358       VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT |
359       VK_MEMORY_PROPERTY_HOST_COHERENT_BIT);
360 
361   return TRUE;
362 }
363 
364 static gboolean
update_uniform(GstVulkanViewConvert * conv,GstVulkanImageView ** in_views,GstVulkanImageView ** out_views)365 update_uniform (GstVulkanViewConvert * conv, GstVulkanImageView ** in_views,
366     GstVulkanImageView ** out_views)
367 {
368   GstVulkanVideoFilter *vfilter = GST_VULKAN_VIDEO_FILTER (conv);
369   GstVideoMultiviewMode in_mode, out_mode;
370   GstVideoMultiviewFlags in_flags, out_flags;
371   struct ViewUpdate data;
372   GstMapInfo map_info;
373   guint l_index, r_index;
374   gboolean mono_input = FALSE;
375 
376   calculate_reorder_indexes (GST_VIDEO_INFO_FORMAT (&vfilter->in_info),
377       in_views, GST_VIDEO_INFO_FORMAT (&vfilter->out_info),
378       out_views, data.in_reorder_idx, data.out_reorder_idx);
379 
380   data.tex_scale[0][0] = data.tex_scale[0][1] = 1.;
381   data.tex_scale[1][0] = data.tex_scale[1][1] = 1.;
382   data.tex_offset[0][0] = data.tex_offset[0][1] = 0.;
383   data.tex_offset[1][0] = data.tex_offset[1][1] = 0.;
384 
385   in_mode = conv->input_mode_override;
386   in_flags = conv->input_flags_override;
387   if (in_mode == GST_VIDEO_MULTIVIEW_MODE_NONE) {
388     in_mode = GST_VIDEO_INFO_MULTIVIEW_MODE (&vfilter->in_info);
389     in_flags = GST_VIDEO_INFO_MULTIVIEW_FLAGS (&vfilter->in_info);
390   }
391 
392   /* Configured output mode already takes any override
393    * into account */
394   out_mode = GST_VIDEO_INFO_MULTIVIEW_MODE (&vfilter->out_info);
395   out_flags = GST_VIDEO_INFO_MULTIVIEW_FLAGS (&vfilter->out_info);
396 
397   if ((in_flags & GST_VIDEO_MULTIVIEW_FLAGS_RIGHT_VIEW_FIRST) ==
398       (out_flags & GST_VIDEO_MULTIVIEW_FLAGS_RIGHT_VIEW_FIRST)) {
399     l_index = 0;
400     r_index = 1;
401   } else {
402     GST_LOG_OBJECT (conv, "Switching left/right views");
403 
404     l_index = 1;
405     r_index = 0;
406   }
407 
408   if (in_mode < GST_VIDEO_MULTIVIEW_MODE_SIDE_BY_SIDE) {        /* unknown/mono/left/right single image */
409   } else if (in_mode == GST_VIDEO_MULTIVIEW_MODE_SIDE_BY_SIDE ||
410       in_mode == GST_VIDEO_MULTIVIEW_MODE_SIDE_BY_SIDE_QUINCUNX) {
411     /* Side-by-side input */
412     data.tex_offset[r_index][0] += 0.5 * data.tex_scale[r_index][0];
413     data.tex_scale[0][0] *= 0.5f;       /* Half horizontal scale */
414     data.tex_scale[1][0] *= 0.5f;
415   } else if (in_mode == GST_VIDEO_MULTIVIEW_MODE_TOP_BOTTOM) {  /* top-bottom */
416     data.tex_offset[r_index][1] += 0.5 * data.tex_scale[r_index][1];
417     data.tex_scale[0][1] *= 0.5f;       /* Half vertical scale */
418     data.tex_scale[1][1] *= 0.5f;
419   }
420 
421   /* Flipped is vertical, flopped is horizontal.
422    * Adjust and offset per-view scaling. This needs to be done
423    * after the input scaling already splits the views, before
424    * adding any output scaling. */
425   if ((in_flags & GST_VIDEO_MULTIVIEW_FLAGS_LEFT_FLIPPED) !=
426       (out_flags & GST_VIDEO_MULTIVIEW_FLAGS_LEFT_FLIPPED)) {
427     data.tex_offset[l_index][1] += data.tex_scale[l_index][1];
428     data.tex_scale[l_index][1] *= -1.0;
429   }
430   if ((in_flags & GST_VIDEO_MULTIVIEW_FLAGS_LEFT_FLOPPED) !=
431       (out_flags & GST_VIDEO_MULTIVIEW_FLAGS_LEFT_FLOPPED)) {
432     data.tex_offset[l_index][0] += data.tex_scale[l_index][0];
433     data.tex_scale[l_index][0] *= -1.0;
434   }
435   if ((in_flags & GST_VIDEO_MULTIVIEW_FLAGS_RIGHT_FLIPPED) !=
436       (out_flags & GST_VIDEO_MULTIVIEW_FLAGS_RIGHT_FLIPPED)) {
437     data.tex_offset[r_index][1] += data.tex_scale[r_index][1];
438     data.tex_scale[r_index][1] *= -1.0;
439   }
440   if ((in_flags & GST_VIDEO_MULTIVIEW_FLAGS_RIGHT_FLOPPED) !=
441       (out_flags & GST_VIDEO_MULTIVIEW_FLAGS_RIGHT_FLOPPED)) {
442     data.tex_offset[r_index][0] += data.tex_scale[r_index][0];
443     data.tex_scale[r_index][0] *= -1.0;
444   }
445 
446   if (out_mode == GST_VIDEO_MULTIVIEW_MODE_SIDE_BY_SIDE ||
447       out_mode == GST_VIDEO_MULTIVIEW_MODE_SIDE_BY_SIDE_QUINCUNX) {
448     /* Side-by-Side */
449     data.tex_offset[1][0] -= data.tex_scale[1][0];
450     data.tex_scale[0][0] *= 2.0f;
451     data.tex_scale[1][0] *= 2.0f;
452   } else if (out_mode == GST_VIDEO_MULTIVIEW_MODE_TOP_BOTTOM) {
453     data.tex_offset[1][1] -= data.tex_scale[1][1];
454     data.tex_scale[0][1] *= 2.0f;
455     data.tex_scale[1][1] *= 2.0f;
456   }
457 
458   GST_DEBUG_OBJECT (conv,
459       "Scaling matrix [ %f, %f ] [ %f %f]. Offsets [ %f, %f ] [ %f, %f ]",
460       data.tex_scale[0][0], data.tex_scale[0][1],
461       data.tex_scale[1][0], data.tex_scale[1][1],
462       data.tex_offset[0][0], data.tex_offset[0][1], data.tex_offset[1][0],
463       data.tex_offset[1][1]);
464 
465   if (in_mode == GST_VIDEO_MULTIVIEW_MODE_NONE ||
466       in_mode == GST_VIDEO_MULTIVIEW_MODE_MONO ||
467       in_mode == GST_VIDEO_MULTIVIEW_MODE_LEFT ||
468       in_mode == GST_VIDEO_MULTIVIEW_MODE_RIGHT)
469     mono_input = TRUE;
470 
471   data.output_type = out_mode;
472   if (data.output_type == GST_VIDEO_MULTIVIEW_MODE_NONE ||
473       data.output_type == GST_VIDEO_MULTIVIEW_MODE_MONO) {
474     if (mono_input)
475       data.output_type = GST_VIDEO_MULTIVIEW_MODE_LEFT;
476     else
477       data.output_type = GST_VIDEO_MULTIVIEW_MODE_MONO;
478   } else if (data.output_type == GST_VIDEO_MULTIVIEW_MODE_SIDE_BY_SIDE_QUINCUNX) {
479     data.output_type = GST_VIDEO_MULTIVIEW_MODE_SIDE_BY_SIDE;
480   }
481 
482   data.tex_size[0] = GST_VIDEO_INFO_WIDTH (&vfilter->out_info);
483   data.tex_size[1] = GST_VIDEO_INFO_HEIGHT (&vfilter->out_info);
484   memcpy (&data.downmix[0], &downmix_matrices[conv->downmix_mode][0],
485       sizeof (data.downmix[0]));
486   memcpy (&data.downmix[1], &downmix_matrices[conv->downmix_mode][1],
487       sizeof (data.downmix[1]));
488 
489   if (!gst_memory_map (conv->uniform, &map_info, GST_MAP_WRITE)) {
490     return FALSE;
491   }
492   memcpy (map_info.data, &data, sizeof (data));
493   gst_memory_unmap (conv->uniform, &map_info);
494 
495   return TRUE;
496 }
497 
498 static GstMemory *
get_uniforms(GstVulkanViewConvert * conv,GstVulkanImageView ** in_views,GstVulkanImageView ** out_views)499 get_uniforms (GstVulkanViewConvert * conv,
500     GstVulkanImageView ** in_views, GstVulkanImageView ** out_views)
501 {
502   if (!conv->uniform) {
503     if (!create_uniform_buffer (conv))
504       return NULL;
505     if (!update_uniform (conv, in_views, out_views)) {
506       gst_memory_unref (conv->uniform);
507       conv->uniform = NULL;
508       return FALSE;
509     }
510   }
511 
512   return gst_memory_ref (conv->uniform);
513 }
514 
515 static void gst_vulkan_view_convert_set_property (GObject * object,
516     guint prop_id, const GValue * value, GParamSpec * pspec);
517 static void gst_vulkan_view_convert_get_property (GObject * object,
518     guint prop_id, GValue * value, GParamSpec * pspec);
519 
520 static gboolean gst_vulkan_view_convert_start (GstBaseTransform * bt);
521 static gboolean gst_vulkan_view_convert_stop (GstBaseTransform * bt);
522 
523 static GstCaps *gst_vulkan_view_convert_transform_caps (GstBaseTransform * bt,
524     GstPadDirection direction, GstCaps * caps, GstCaps * filter);
525 static GstCaps *gst_vulkan_view_convert_fixate_caps (GstBaseTransform * bt,
526     GstPadDirection direction, GstCaps * caps, GstCaps * othercaps);
527 static GstFlowReturn gst_vulkan_view_convert_transform (GstBaseTransform * bt,
528     GstBuffer * inbuf, GstBuffer * outbuf);
529 static gboolean gst_vulkan_view_convert_set_caps (GstBaseTransform * bt,
530     GstCaps * in_caps, GstCaps * out_caps);
531 
532 static GstStaticPadTemplate gst_vulkan_sink_template =
533 GST_STATIC_PAD_TEMPLATE ("sink",
534     GST_PAD_SINK,
535     GST_PAD_ALWAYS,
536     GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE_WITH_FEATURES
537         (GST_CAPS_FEATURE_MEMORY_VULKAN_IMAGE,
538             "{ BGRA, RGBA }")));
539 
540 static GstStaticPadTemplate gst_vulkan_src_template =
541 GST_STATIC_PAD_TEMPLATE ("src",
542     GST_PAD_SRC,
543     GST_PAD_ALWAYS,
544     GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE_WITH_FEATURES
545         (GST_CAPS_FEATURE_MEMORY_VULKAN_IMAGE,
546             "{ BGRA, RGBA }")));
547 
548 enum
549 {
550   PROP_0,
551   PROP_INPUT_LAYOUT,
552   PROP_INPUT_FLAGS,
553   PROP_OUTPUT_LAYOUT,
554   PROP_OUTPUT_FLAGS,
555   PROP_OUTPUT_DOWNMIX_MODE
556 };
557 
558 #define DEFAULT_DOWNMIX GST_VULKAN_STEREO_DOWNMIX_ANAGLYPH_GREEN_MAGENTA_DUBOIS
559 
560 #define gst_vulkan_view_convert_parent_class parent_class
561 G_DEFINE_TYPE_WITH_CODE (GstVulkanViewConvert, gst_vulkan_view_convert,
562     GST_TYPE_VULKAN_VIDEO_FILTER,
563     GST_DEBUG_CATEGORY_INIT (gst_debug_vulkan_view_convert,
564         "vulkanviewconvert", 0, "Vulkan View Convert"));
565 GST_ELEMENT_REGISTER_DEFINE_WITH_CODE (vulkanviewconvert, "vulkanviewconvert",
566     GST_RANK_NONE, GST_TYPE_VULKAN_VIEW_CONVERT, vulkan_element_init (plugin));
567 
568 static void
gst_vulkan_view_convert_class_init(GstVulkanViewConvertClass * klass)569 gst_vulkan_view_convert_class_init (GstVulkanViewConvertClass * klass)
570 {
571   GObjectClass *gobject_class;
572   GstElementClass *gstelement_class;
573   GstBaseTransformClass *gstbasetransform_class;
574 
575   gobject_class = (GObjectClass *) klass;
576   gstelement_class = (GstElementClass *) klass;
577   gstbasetransform_class = (GstBaseTransformClass *) klass;
578 
579   gobject_class->set_property = gst_vulkan_view_convert_set_property;
580   gobject_class->get_property = gst_vulkan_view_convert_get_property;
581 
582   g_object_class_install_property (gobject_class, PROP_INPUT_LAYOUT,
583       g_param_spec_enum ("input-mode-override",
584           "Input Multiview Mode Override",
585           "Override any input information about multiview layout",
586           GST_TYPE_VIDEO_MULTIVIEW_MODE,
587           GST_VIDEO_MULTIVIEW_MODE_NONE,
588           G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
589   g_object_class_install_property (gobject_class, PROP_INPUT_FLAGS,
590       g_param_spec_flags ("input-flags-override",
591           "Input Multiview Flags Override",
592           "Override any input information about multiview layout flags",
593           GST_TYPE_VIDEO_MULTIVIEW_FLAGS, GST_VIDEO_MULTIVIEW_FLAGS_NONE,
594           G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
595   g_object_class_install_property (gobject_class, PROP_OUTPUT_LAYOUT,
596       g_param_spec_enum ("output-mode-override",
597           "Output Multiview Mode Override",
598           "Override automatic output mode selection for multiview layout",
599           GST_TYPE_VIDEO_MULTIVIEW_MODE, GST_VIDEO_MULTIVIEW_MODE_NONE,
600           G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
601   g_object_class_install_property (gobject_class, PROP_OUTPUT_FLAGS,
602       g_param_spec_flags ("output-flags-override",
603           "Output Multiview Flags Override",
604           "Override automatic negotiation for output multiview layout flags",
605           GST_TYPE_VIDEO_MULTIVIEW_FLAGS, GST_VIDEO_MULTIVIEW_FLAGS_NONE,
606           G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
607   g_object_class_install_property (gobject_class, PROP_OUTPUT_DOWNMIX_MODE,
608       g_param_spec_enum ("downmix-mode", "Mode for mono downmixed output",
609           "Output anaglyph type to generate when downmixing to mono",
610           GST_TYPE_VULKAN_STEREO_DOWNMIX, DEFAULT_DOWNMIX,
611           G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
612 
613   gst_element_class_set_metadata (gstelement_class, "Vulkan View Convert",
614       "Filter/Video/Convert", "A Vulkan View Convert",
615       "Matthew Waters <matthew@centricular.com>");
616 
617   gst_type_mark_as_plugin_api (GST_TYPE_VULKAN_STEREO_DOWNMIX, 0);
618   gst_element_class_add_static_pad_template (gstelement_class,
619       &gst_vulkan_sink_template);
620   gst_element_class_add_static_pad_template (gstelement_class,
621       &gst_vulkan_src_template);
622 
623   gstbasetransform_class->start =
624       GST_DEBUG_FUNCPTR (gst_vulkan_view_convert_start);
625   gstbasetransform_class->stop =
626       GST_DEBUG_FUNCPTR (gst_vulkan_view_convert_stop);
627   gstbasetransform_class->transform_caps =
628       gst_vulkan_view_convert_transform_caps;
629   gstbasetransform_class->fixate_caps = gst_vulkan_view_convert_fixate_caps;
630   gstbasetransform_class->set_caps = gst_vulkan_view_convert_set_caps;
631   gstbasetransform_class->transform = gst_vulkan_view_convert_transform;
632 }
633 
634 static void
gst_vulkan_view_convert_init(GstVulkanViewConvert * conv)635 gst_vulkan_view_convert_init (GstVulkanViewConvert * conv)
636 {
637   conv->downmix_mode = DEFAULT_DOWNMIX;
638 
639   conv->input_mode_override = GST_VIDEO_MULTIVIEW_MODE_NONE;
640   conv->input_flags_override = GST_VIDEO_MULTIVIEW_FLAGS_NONE;
641   conv->output_mode_override = GST_VIDEO_MULTIVIEW_MODE_NONE;
642   conv->output_flags_override = GST_VIDEO_MULTIVIEW_FLAGS_NONE;
643 }
644 
645 static void
gst_vulkan_view_convert_set_property(GObject * object,guint prop_id,const GValue * value,GParamSpec * pspec)646 gst_vulkan_view_convert_set_property (GObject * object, guint prop_id,
647     const GValue * value, GParamSpec * pspec)
648 {
649   GstVulkanViewConvert *conv = GST_VULKAN_VIEW_CONVERT (object);
650 
651   switch (prop_id) {
652     case PROP_INPUT_LAYOUT:
653       conv->input_mode_override = g_value_get_enum (value);
654       gst_base_transform_reconfigure_src (GST_BASE_TRANSFORM (conv));
655       break;
656     case PROP_INPUT_FLAGS:
657       conv->input_flags_override = g_value_get_flags (value);
658       gst_base_transform_reconfigure_src (GST_BASE_TRANSFORM (conv));
659       break;
660     case PROP_OUTPUT_LAYOUT:
661       conv->output_mode_override = g_value_get_enum (value);
662       gst_base_transform_reconfigure_src (GST_BASE_TRANSFORM (conv));
663       break;
664     case PROP_OUTPUT_FLAGS:
665       conv->output_flags_override = g_value_get_flags (value);
666       gst_base_transform_reconfigure_src (GST_BASE_TRANSFORM (conv));
667       break;
668     case PROP_OUTPUT_DOWNMIX_MODE:
669       conv->downmix_mode = g_value_get_enum (value);
670       break;
671     default:
672       G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
673       break;
674   }
675 }
676 
677 static void
gst_vulkan_view_convert_get_property(GObject * object,guint prop_id,GValue * value,GParamSpec * pspec)678 gst_vulkan_view_convert_get_property (GObject * object, guint prop_id,
679     GValue * value, GParamSpec * pspec)
680 {
681   GstVulkanViewConvert *conv = GST_VULKAN_VIEW_CONVERT (object);
682 
683   switch (prop_id) {
684     case PROP_INPUT_LAYOUT:
685       g_value_set_enum (value, conv->input_mode_override);
686       break;
687     case PROP_INPUT_FLAGS:
688       g_value_set_flags (value, conv->input_flags_override);
689       break;
690     case PROP_OUTPUT_LAYOUT:
691       g_value_set_enum (value, conv->output_mode_override);
692       break;
693     case PROP_OUTPUT_FLAGS:
694       g_value_set_flags (value, conv->output_flags_override);
695       break;
696     case PROP_OUTPUT_DOWNMIX_MODE:
697       g_value_set_enum (value, conv->downmix_mode);
698       break;
699     default:
700       G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
701       break;
702   }
703 }
704 
705 /* Function that can halve the value
706  * of ints, fractions, int/fraction ranges and lists of ints/fractions */
707 static gboolean
_halve_value(GValue * out,const GValue * in_value)708 _halve_value (GValue * out, const GValue * in_value)
709 {
710   /* Fundamental fixed types first */
711   if (G_VALUE_HOLDS_INT (in_value)) {
712     g_value_init (out, G_TYPE_INT);
713     g_value_set_int (out, MAX (g_value_get_int (in_value) / 2, 1));
714   } else if (GST_VALUE_HOLDS_FRACTION (in_value)) {
715     gint num, den;
716     num = gst_value_get_fraction_numerator (in_value);
717     den = gst_value_get_fraction_denominator (in_value);
718     g_value_init (out, GST_TYPE_FRACTION);
719     /* Don't adjust 'infinite' fractions */
720     if ((num != 1 || den != 2147483647) && (num != 2147483647 || den != 1)) {
721       /* FIXME - could do better approximation when den > G_MAXINT/2? */
722       den = den > G_MAXINT / 2 ? G_MAXINT : den * 2;
723     }
724     gst_value_set_fraction (out, num, den);
725   } else if (GST_VALUE_HOLDS_INT_RANGE (in_value)) {
726     gint range_min = gst_value_get_int_range_min (in_value);
727     gint range_max = gst_value_get_int_range_max (in_value);
728     gint range_step = gst_value_get_int_range_step (in_value);
729     g_value_init (out, GST_TYPE_INT_RANGE);
730     if (range_min != 1)
731       range_min = MAX (1, range_min / 2);
732     if (range_max != G_MAXINT)
733       range_max = MAX (1, range_max / 2);
734     gst_value_set_int_range_step (out, range_min,
735         range_max, MAX (1, range_step / 2));
736   } else if (GST_VALUE_HOLDS_FRACTION_RANGE (in_value)) {
737     GValue min_out = G_VALUE_INIT;
738     GValue max_out = G_VALUE_INIT;
739     const GValue *range_min = gst_value_get_fraction_range_min (in_value);
740     const GValue *range_max = gst_value_get_fraction_range_max (in_value);
741     _halve_value (&min_out, range_min);
742     _halve_value (&max_out, range_max);
743     g_value_init (out, GST_TYPE_FRACTION_RANGE);
744     gst_value_set_fraction_range (out, &min_out, &max_out);
745     g_value_unset (&min_out);
746     g_value_unset (&max_out);
747   } else if (GST_VALUE_HOLDS_LIST (in_value)) {
748     gint i;
749     g_value_init (out, GST_TYPE_LIST);
750     for (i = 0; i < gst_value_list_get_size (in_value); i++) {
751       const GValue *entry;
752       GValue tmp = G_VALUE_INIT;
753 
754       entry = gst_value_list_get_value (in_value, i);
755       /* Random list values might not be the right type */
756       if (!_halve_value (&tmp, entry))
757         goto fail;
758       gst_value_list_append_and_take_value (out, &tmp);
759     }
760   } else {
761     return FALSE;
762   }
763 
764   return TRUE;
765 fail:
766   g_value_unset (out);
767   return FALSE;
768 }
769 
770 static GstStructure *
_halve_structure_field(const GstStructure * in,const gchar * field_name)771 _halve_structure_field (const GstStructure * in, const gchar * field_name)
772 {
773   GstStructure *out;
774   const GValue *in_value = gst_structure_get_value (in, field_name);
775   GValue tmp = G_VALUE_INIT;
776 
777   if (G_UNLIKELY (in_value == NULL))
778     return gst_structure_copy (in);     /* Field doesn't exist, leave it as is */
779 
780   if (!_halve_value (&tmp, in_value))
781     return NULL;
782 
783   out = gst_structure_copy (in);
784   gst_structure_set_value (out, field_name, &tmp);
785   g_value_unset (&tmp);
786 
787   return out;
788 }
789 
790 /* Function that can double the value
791  * of ints, fractions, int/fraction ranges and lists of ints/fractions */
792 static gboolean
_double_value(GValue * out,const GValue * in_value)793 _double_value (GValue * out, const GValue * in_value)
794 {
795   /* Fundamental fixed types first */
796   if (G_VALUE_HOLDS_INT (in_value)) {
797     gint n = g_value_get_int (in_value);
798     g_value_init (out, G_TYPE_INT);
799     if (n <= G_MAXINT / 2)
800       g_value_set_int (out, n * 2);
801     else
802       g_value_set_int (out, G_MAXINT);
803   } else if (GST_VALUE_HOLDS_FRACTION (in_value)) {
804     gint num, den;
805     num = gst_value_get_fraction_numerator (in_value);
806     den = gst_value_get_fraction_denominator (in_value);
807     g_value_init (out, GST_TYPE_FRACTION);
808     /* Don't adjust 'infinite' fractions */
809     if ((num != 1 || den != 2147483647) && (num != 2147483647 || den != 1)) {
810       /* FIXME - could do better approximation when num > G_MAXINT/2? */
811       num = num > G_MAXINT / 2 ? G_MAXINT : num * 2;
812     }
813     gst_value_set_fraction (out, num, den);
814   } else if (GST_VALUE_HOLDS_INT_RANGE (in_value)) {
815     gint range_min = gst_value_get_int_range_min (in_value);
816     gint range_max = gst_value_get_int_range_max (in_value);
817     gint range_step = gst_value_get_int_range_step (in_value);
818     if (range_min != 1) {
819       range_min = MIN (G_MAXINT / 2, range_min);
820       range_min *= 2;
821     }
822     if (range_max != G_MAXINT) {
823       range_max = MIN (G_MAXINT / 2, range_max);
824       range_max *= 2;
825     }
826     range_step = MIN (G_MAXINT / 2, range_step);
827     g_value_init (out, GST_TYPE_INT_RANGE);
828     gst_value_set_int_range_step (out, range_min, range_max, range_step);
829   } else if (GST_VALUE_HOLDS_FRACTION_RANGE (in_value)) {
830     GValue min_out = G_VALUE_INIT;
831     GValue max_out = G_VALUE_INIT;
832     const GValue *range_min = gst_value_get_fraction_range_min (in_value);
833     const GValue *range_max = gst_value_get_fraction_range_max (in_value);
834     _double_value (&min_out, range_min);
835     _double_value (&max_out, range_max);
836     g_value_init (out, GST_TYPE_FRACTION_RANGE);
837     gst_value_set_fraction_range (out, &min_out, &max_out);
838     g_value_unset (&min_out);
839     g_value_unset (&max_out);
840   } else if (GST_VALUE_HOLDS_LIST (in_value)) {
841     gint i;
842     g_value_init (out, GST_TYPE_LIST);
843     for (i = 0; i < gst_value_list_get_size (in_value); i++) {
844       const GValue *entry;
845       GValue tmp = G_VALUE_INIT;
846 
847       entry = gst_value_list_get_value (in_value, i);
848       /* Random list values might not be the right type */
849       if (!_double_value (&tmp, entry))
850         goto fail;
851       gst_value_list_append_and_take_value (out, &tmp);
852     }
853   } else {
854     return FALSE;
855   }
856 
857   return TRUE;
858 fail:
859   g_value_unset (out);
860   return FALSE;
861 }
862 
863 static GstStructure *
_double_structure_field(const GstStructure * in,const gchar * field_name)864 _double_structure_field (const GstStructure * in, const gchar * field_name)
865 {
866   GstStructure *out;
867   const GValue *in_value = gst_structure_get_value (in, field_name);
868   GValue tmp = G_VALUE_INIT;
869 
870   if (G_UNLIKELY (in_value == NULL))
871     return gst_structure_copy (in);     /* Field doesn't exist, leave it as is */
872 
873   if (!_double_value (&tmp, in_value))
874     return NULL;
875 
876   out = gst_structure_copy (in);
877   gst_structure_set_value (out, field_name, &tmp);
878   g_value_unset (&tmp);
879 
880   return out;
881 }
882 
883 /* Return a copy of the caps with the requested field doubled in value/range */
884 static GstCaps *
_double_caps_field(const GstCaps * in,const gchar * field_name)885 _double_caps_field (const GstCaps * in, const gchar * field_name)
886 {
887   gint i;
888   GstCaps *out = gst_caps_new_empty ();
889 
890   for (i = 0; i < gst_caps_get_size (in); i++) {
891     const GstStructure *cur = gst_caps_get_structure (in, i);
892     GstCapsFeatures *f = gst_caps_get_features (in, i);
893 
894     GstStructure *res = _double_structure_field (cur, field_name);
895     out =
896         gst_caps_merge_structure_full (out, res,
897         f ? gst_caps_features_copy (f) : NULL);
898   }
899 
900   return out;
901 }
902 
903 /* Takes ownership of the input caps  */
904 static GstCaps *
_expand_par_for_half_aspect(GstCaps * in,gboolean vertical_half_aspect)905 _expand_par_for_half_aspect (GstCaps * in, gboolean vertical_half_aspect)
906 {
907   guint mview_flags, mview_flags_mask;
908   GstCaps *out;
909   GstStructure *tmp;
910 
911   out = gst_caps_new_empty ();
912 
913   while (gst_caps_get_size (in) > 0) {
914     GstStructure *s;
915     GstCapsFeatures *features;
916 
917     features = gst_caps_get_features (in, 0);
918     if (features)
919       features = gst_caps_features_copy (features);
920 
921     s = gst_caps_steal_structure (in, 0);
922 
923     if (!gst_structure_get_flagset (s, "multiview-flags", &mview_flags,
924             &mview_flags_mask)) {
925       gst_caps_append_structure_full (out, s, features);
926       continue;
927     }
928     /* If the input doesn't care about the half-aspect flag, allow current PAR in either variant */
929     if ((mview_flags_mask & GST_VIDEO_MULTIVIEW_FLAGS_HALF_ASPECT) == 0) {
930       gst_caps_append_structure_full (out, s, features);
931       continue;
932     }
933     if (!gst_structure_has_field (s, "pixel-aspect-ratio")) {
934       /* No par field, dont-care the half-aspect flag */
935       gst_structure_set (s, "multiview-flags",
936           GST_TYPE_VIDEO_MULTIVIEW_FLAGSET,
937           mview_flags & ~GST_VIDEO_MULTIVIEW_FLAGS_HALF_ASPECT,
938           mview_flags_mask & ~GST_VIDEO_MULTIVIEW_FLAGS_HALF_ASPECT, NULL);
939       gst_caps_append_structure_full (out, s, features);
940       continue;
941     }
942 
943     /* Halve or double PAR base on inputs input specified. */
944 
945     /* Append a copy with the half-aspect flag as-is */
946     tmp = gst_structure_copy (s);
947     out = gst_caps_merge_structure_full (out, tmp,
948         features ? gst_caps_features_copy (features) : NULL);
949 
950     /* and then a copy inverted */
951     if (mview_flags & GST_VIDEO_MULTIVIEW_FLAGS_HALF_ASPECT) {
952       /* Input is half-aspect. Double/halve the PAR, clear the flag */
953       if (vertical_half_aspect)
954         tmp = _halve_structure_field (s, "pixel-aspect-ratio");
955       else
956         tmp = _double_structure_field (s, "pixel-aspect-ratio");
957       /* Clear the flag */
958       gst_structure_set (tmp, "multiview-flags",
959           GST_TYPE_VIDEO_MULTIVIEW_FLAGSET,
960           mview_flags & ~GST_VIDEO_MULTIVIEW_FLAGS_HALF_ASPECT,
961           mview_flags_mask | GST_VIDEO_MULTIVIEW_FLAGS_HALF_ASPECT, NULL);
962     } else {
963       if (vertical_half_aspect)
964         tmp = _double_structure_field (s, "pixel-aspect-ratio");
965       else
966         tmp = _halve_structure_field (s, "pixel-aspect-ratio");
967       /* Set the flag */
968       gst_structure_set (tmp, "multiview-flags",
969           GST_TYPE_VIDEO_MULTIVIEW_FLAGSET,
970           mview_flags | GST_VIDEO_MULTIVIEW_FLAGS_HALF_ASPECT,
971           mview_flags_mask | GST_VIDEO_MULTIVIEW_FLAGS_HALF_ASPECT, NULL);
972     }
973 
974     out = gst_caps_merge_structure_full (out, tmp,
975         features ? gst_caps_features_copy (features) : NULL);
976 
977     gst_structure_free (s);
978     if (features)
979       gst_caps_features_free (features);
980   }
981 
982   gst_caps_unref (in);
983 
984   return out;
985 }
986 
987 /* If input supports top-bottom or row-interleaved, we may halve height to mono frames.
988  * If input supports left-right, checkerboard, quincunx or column-interleaved,
989  * we may halve width to mono frames.
990  * For output of top-bottom or row-interleaved, we may double the mono height
991  * For output of left-right, checkerboard, quincunx or column-interleaved,
992  * we may double the mono width.
993  * In all cases, if input has half-aspect and output does not, we may double the PAR
994  * And if input does *not* have half-aspect flag and output does not, we may halve the PAR
995  */
996 static GstCaps *
_expand_structure(GstVulkanViewConvert * viewconvert,GstCaps * out_caps,GstStructure * structure,GstCapsFeatures * features)997 _expand_structure (GstVulkanViewConvert * viewconvert,
998     GstCaps * out_caps, GstStructure * structure, GstCapsFeatures * features)
999 {
1000   GstCaps *expanded_caps, *tmp;
1001   GstCaps *mono_caps;
1002   const gchar *default_mview_mode_str = NULL;
1003   guint mview_flags, mview_flags_mask;
1004   const GValue *in_modes;
1005   gint i;
1006 
1007   /* Empty caps to accumulate into */
1008   expanded_caps = gst_caps_new_empty ();
1009 
1010   /* First, set defaults if multiview flags are missing */
1011   default_mview_mode_str =
1012       gst_video_multiview_mode_to_caps_string (GST_VIDEO_MULTIVIEW_MODE_MONO);
1013 
1014   mview_flags = GST_VIDEO_MULTIVIEW_FLAGS_NONE;
1015   mview_flags_mask = GST_FLAG_SET_MASK_EXACT;
1016 
1017   if (!gst_structure_has_field (structure, "multiview-mode")) {
1018     gst_structure_set (structure,
1019         "multiview-mode", G_TYPE_STRING, default_mview_mode_str, NULL);
1020   }
1021   if (!gst_structure_has_field (structure, "multiview-flags")) {
1022     gst_structure_set (structure,
1023         "multiview-flags", GST_TYPE_VIDEO_MULTIVIEW_FLAGSET, mview_flags,
1024         mview_flags_mask, NULL);
1025   } else {
1026     gst_structure_get_flagset (structure, "multiview-flags",
1027         &mview_flags, &mview_flags_mask);
1028   }
1029 
1030   in_modes = gst_structure_get_value (structure, "multiview-mode");
1031   mono_caps = gst_caps_new_empty ();
1032   if (gst_value_intersect (NULL, in_modes,
1033           gst_video_multiview_get_mono_modes ())) {
1034     GstStructure *new_struct = gst_structure_copy (structure);
1035     gst_structure_set_value (new_struct, "multiview-mode",
1036         gst_video_multiview_get_mono_modes ());
1037     /* Half-aspect makes no sense for mono or unpacked, get rid of it */
1038     if (mview_flags & GST_VIDEO_MULTIVIEW_FLAGS_HALF_ASPECT) {
1039       gst_structure_set (new_struct, "multiview-flags",
1040           GST_TYPE_VIDEO_MULTIVIEW_FLAGSET,
1041           mview_flags & ~GST_VIDEO_MULTIVIEW_FLAGS_HALF_ASPECT,
1042           mview_flags_mask & ~GST_VIDEO_MULTIVIEW_FLAGS_HALF_ASPECT, NULL);
1043     }
1044     gst_caps_append_structure_full (mono_caps, new_struct,
1045         features ? gst_caps_features_copy (features) : NULL);
1046   }
1047   if (gst_value_intersect (NULL, in_modes,
1048           gst_video_multiview_get_unpacked_modes ())) {
1049     GstStructure *new_struct = gst_structure_copy (structure);
1050 
1051     gst_structure_set_value (new_struct, "multiview-mode",
1052         gst_video_multiview_get_mono_modes ());
1053 
1054     /* Half-aspect makes no sense for mono or unpacked, get rid of it */
1055     if (mview_flags & GST_VIDEO_MULTIVIEW_FLAGS_HALF_ASPECT) {
1056       gst_structure_set (new_struct, "multiview-flags",
1057           GST_TYPE_VIDEO_MULTIVIEW_FLAGSET,
1058           mview_flags & ~GST_VIDEO_MULTIVIEW_FLAGS_HALF_ASPECT,
1059           mview_flags_mask & ~GST_VIDEO_MULTIVIEW_FLAGS_HALF_ASPECT, NULL);
1060     }
1061     gst_caps_append_structure_full (mono_caps, new_struct,
1062         features ? gst_caps_features_copy (features) : NULL);
1063   }
1064 
1065   if (gst_value_intersect (NULL, in_modes,
1066           gst_video_multiview_get_doubled_height_modes ())) {
1067     /* Append mono formats with height halved */
1068     GstStructure *new_struct = _halve_structure_field (structure, "height");
1069     gst_structure_set_value (new_struct, "multiview-mode",
1070         gst_video_multiview_get_mono_modes ());
1071     /* Normalise the half-aspect flag away */
1072     if (mview_flags & GST_VIDEO_MULTIVIEW_FLAGS_HALF_ASPECT) {
1073       GstStructure *s =
1074           _halve_structure_field (new_struct, "pixel-aspect-ratio");
1075       gst_structure_set (structure, "multiview-flags",
1076           GST_TYPE_VIDEO_MULTIVIEW_FLAGSET,
1077           mview_flags & ~GST_VIDEO_MULTIVIEW_FLAGS_HALF_ASPECT,
1078           mview_flags_mask | GST_VIDEO_MULTIVIEW_FLAGS_HALF_ASPECT, NULL);
1079       gst_structure_free (new_struct);
1080       new_struct = s;
1081     }
1082     mono_caps = gst_caps_merge_structure_full (mono_caps, new_struct,
1083         features ? gst_caps_features_copy (features) : NULL);
1084   }
1085   if (gst_value_intersect (NULL, in_modes,
1086           gst_video_multiview_get_doubled_width_modes ())) {
1087     /* Append mono formats with width halved */
1088     GstStructure *new_struct = _halve_structure_field (structure, "width");
1089     gst_structure_set_value (new_struct, "multiview-mode",
1090         gst_video_multiview_get_mono_modes ());
1091     /* Normalise the half-aspect flag away */
1092     if (mview_flags & GST_VIDEO_MULTIVIEW_FLAGS_HALF_ASPECT) {
1093       GstStructure *s =
1094           _double_structure_field (new_struct, "pixel-aspect-ratio");
1095       gst_structure_set (structure, "multiview-flags",
1096           GST_TYPE_VIDEO_MULTIVIEW_FLAGSET,
1097           mview_flags & ~GST_VIDEO_MULTIVIEW_FLAGS_HALF_ASPECT,
1098           mview_flags_mask | GST_VIDEO_MULTIVIEW_FLAGS_HALF_ASPECT, NULL);
1099       gst_structure_free (new_struct);
1100       new_struct = s;
1101     }
1102     mono_caps = gst_caps_merge_structure_full (mono_caps, new_struct,
1103         features ? gst_caps_features_copy (features) : NULL);
1104   }
1105   if (gst_value_intersect (NULL, in_modes,
1106           gst_video_multiview_get_doubled_size_modes ())) {
1107     /* Append checkerboard/doubled size formats with width & height halved */
1108     GstStructure *new_struct_w = _halve_structure_field (structure, "width");
1109     GstStructure *new_struct_wh =
1110         _halve_structure_field (new_struct_w, "height");
1111     gst_structure_free (new_struct_w);
1112     gst_structure_set_value (new_struct_wh, "multiview-mode",
1113         gst_video_multiview_get_mono_modes ());
1114     mono_caps = gst_caps_merge_structure_full (mono_caps, new_struct_wh,
1115         features ? gst_caps_features_copy (features) : NULL);
1116   }
1117 
1118   /* Everything is normalised now, unset the flags we can change */
1119   /* Remove the views field, as these are all 'mono' modes
1120    * Need to do this before we expand caps back out to frame packed modes */
1121   for (i = 0; i < gst_caps_get_size (mono_caps); i++) {
1122     GstStructure *s = gst_caps_get_structure (mono_caps, i);
1123     gst_structure_remove_fields (s, "views", NULL);
1124     if (gst_structure_get_flagset (s, "multiview-flags", &mview_flags,
1125             &mview_flags_mask)) {
1126       /* Preserve only the half-aspect and mixed-mono flags, for now.
1127        * The rest we can change */
1128       mview_flags_mask &=
1129           (GST_VIDEO_MULTIVIEW_FLAGS_HALF_ASPECT |
1130           GST_VIDEO_MULTIVIEW_FLAGS_MIXED_MONO);
1131       gst_structure_set (s, "multiview-flags", GST_TYPE_VIDEO_MULTIVIEW_FLAGSET,
1132           mview_flags, mview_flags_mask, NULL);
1133     }
1134   }
1135 
1136   GST_TRACE_OBJECT (viewconvert,
1137       "Collected single-view caps %" GST_PTR_FORMAT, mono_caps);
1138   /* Put unpacked and mono modes first. We don't care about flags. Clear them */
1139   tmp = gst_caps_copy (mono_caps);
1140   for (i = 0; i < gst_caps_get_size (tmp); i++) {
1141     GstStructure *s = gst_caps_get_structure (tmp, i);
1142     gst_structure_remove_fields (s, "views", NULL);
1143     if (gst_structure_get_flagset (s, "multiview-flags", &mview_flags,
1144             &mview_flags_mask)) {
1145       /* We can change any flags for mono modes - half-aspect and mixed-mono have no meaning */
1146       mview_flags_mask = 0;
1147       gst_structure_set (s, "multiview-flags", GST_TYPE_VIDEO_MULTIVIEW_FLAGSET,
1148           mview_flags, mview_flags_mask, NULL);
1149     }
1150   }
1151   expanded_caps = gst_caps_merge (expanded_caps, tmp);
1152 
1153   /* Unpacked output modes have 2 views, for now */
1154   tmp = gst_caps_copy (mono_caps);
1155   gst_caps_set_value (tmp, "multiview-mode",
1156       gst_video_multiview_get_unpacked_modes ());
1157   for (i = 0; i < gst_caps_get_size (tmp); i++) {
1158     GstStructure *s = gst_caps_get_structure (tmp, i);
1159     gst_structure_set (s, "views", G_TYPE_INT, 2, NULL);
1160     if (gst_structure_get_flagset (s, "multiview-flags", &mview_flags,
1161             &mview_flags_mask)) {
1162       /* We can change any flags for unpacked modes - half-aspect and mixed-mono have no meaning */
1163       mview_flags_mask = 0;
1164       gst_structure_set (s, "multiview-flags", GST_TYPE_VIDEO_MULTIVIEW_FLAGSET,
1165           mview_flags, mview_flags_mask, NULL);
1166     }
1167   }
1168   expanded_caps = gst_caps_merge (expanded_caps, tmp);
1169 
1170   /* Double height output modes */
1171   tmp = _double_caps_field (mono_caps, "height");
1172   gst_caps_set_value (tmp, "multiview-mode",
1173       gst_video_multiview_get_doubled_height_modes ());
1174   tmp = _expand_par_for_half_aspect (tmp, TRUE);
1175 
1176   expanded_caps = gst_caps_merge (expanded_caps, tmp);
1177 
1178   /* Double width output modes */
1179   tmp = _double_caps_field (mono_caps, "width");
1180   gst_caps_set_value (tmp, "multiview-mode",
1181       gst_video_multiview_get_doubled_width_modes ());
1182   tmp = _expand_par_for_half_aspect (tmp, FALSE);
1183 
1184   expanded_caps = gst_caps_merge (expanded_caps, tmp);
1185 
1186   /* Double size output modes */
1187   {
1188     GstCaps *tmp_w = _double_caps_field (mono_caps, "width");
1189     tmp = _double_caps_field (tmp_w, "height");
1190     gst_caps_unref (tmp_w);
1191     gst_caps_set_value (tmp, "multiview-mode",
1192         gst_video_multiview_get_doubled_size_modes ());
1193     expanded_caps = gst_caps_merge (expanded_caps, tmp);
1194   }
1195 
1196   /* We're done with the mono caps now */
1197   gst_caps_unref (mono_caps);
1198 
1199   GST_TRACE_OBJECT (viewconvert,
1200       "expanded transform caps now %" GST_PTR_FORMAT, expanded_caps);
1201 
1202   if (gst_caps_is_empty (expanded_caps)) {
1203     gst_caps_unref (expanded_caps);
1204     return out_caps;
1205   }
1206   /* Really, we can rescale - so at this point we can append full-range
1207    * height/width/PAR as an unpreferred final option. */
1208 /*  tmp = gst_caps_copy (expanded_caps);
1209   gst_caps_set_simple (tmp, "width", GST_TYPE_INT_RANGE, 1, G_MAXINT,
1210       "height", GST_TYPE_INT_RANGE, 1, G_MAXINT, NULL);
1211 */
1212   out_caps = gst_caps_merge (out_caps, expanded_caps);
1213 /*  out_caps = gst_caps_merge (out_caps, tmp);*/
1214   return out_caps;
1215 }
1216 
1217 static GstCaps *
_intersect_with_mview_mode(GstCaps * caps,GstVideoMultiviewMode mode,GstVideoMultiviewFlags flags)1218 _intersect_with_mview_mode (GstCaps * caps,
1219     GstVideoMultiviewMode mode, GstVideoMultiviewFlags flags)
1220 {
1221   GstCaps *filter, *result;
1222   const gchar *caps_str;
1223 
1224   caps_str = gst_video_multiview_mode_to_caps_string (mode);
1225 
1226   filter = gst_caps_new_simple ("video/x-raw",
1227       "multiview-mode", G_TYPE_STRING,
1228       caps_str, "multiview-flags", GST_TYPE_VIDEO_MULTIVIEW_FLAGSET, flags,
1229       GST_FLAG_SET_MASK_EXACT, NULL);
1230 
1231   if (mode == GST_VIDEO_MULTIVIEW_MODE_SEPARATED ||
1232       mode == GST_VIDEO_MULTIVIEW_MODE_FRAME_BY_FRAME)
1233     gst_caps_set_simple (filter, "views", G_TYPE_INT, 2, NULL);
1234   gst_caps_set_features (filter, 0, gst_caps_features_new_any ());
1235 
1236   GST_DEBUG ("Intersecting target caps %" GST_PTR_FORMAT
1237       " with caps %" GST_PTR_FORMAT, caps, filter);
1238 
1239   result = gst_caps_intersect_full (caps, filter, GST_CAPS_INTERSECT_FIRST);
1240   gst_caps_unref (filter);
1241   return result;
1242 }
1243 
1244 static GstCaps *
_intersect_with_mview_modes(GstCaps * caps,const GValue * modes)1245 _intersect_with_mview_modes (GstCaps * caps, const GValue * modes)
1246 {
1247   GstCaps *filter, *result;
1248 
1249   filter = gst_caps_new_empty_simple ("video/x-raw");
1250 
1251   gst_caps_set_value (filter, "multiview-mode", modes);
1252   gst_caps_set_features (filter, 0, gst_caps_features_new_any ());
1253 
1254   GST_DEBUG ("Intersecting target caps %" GST_PTR_FORMAT
1255       " with caps %" GST_PTR_FORMAT, caps, filter);
1256 
1257   result = gst_caps_intersect_full (caps, filter, GST_CAPS_INTERSECT_FIRST);
1258   gst_caps_unref (filter);
1259   return result;
1260 }
1261 
1262 static GstCaps *
gst_vulkan_view_convert_transform_caps(GstBaseTransform * bt,GstPadDirection direction,GstCaps * caps,GstCaps * filter)1263 gst_vulkan_view_convert_transform_caps (GstBaseTransform * bt,
1264     GstPadDirection direction, GstCaps * caps, GstCaps * filter)
1265 {
1266   GstVulkanViewConvert *viewconvert = GST_VULKAN_VIEW_CONVERT (bt);
1267   GstCaps *base_caps =
1268       gst_static_pad_template_get_caps (&gst_vulkan_sink_template);
1269   GstCaps *out_caps, *tmp_caps;
1270   gint i;
1271 
1272   GST_DEBUG_OBJECT (viewconvert, "Direction %s "
1273       "input caps %" GST_PTR_FORMAT " filter %" GST_PTR_FORMAT,
1274       direction == GST_PAD_SINK ? "sink" : "src", caps, filter);
1275 
1276   /* We can only process VulkanImage caps, start from that */
1277   caps = gst_caps_intersect (caps, base_caps);
1278   gst_caps_unref (base_caps);
1279 
1280   /* Change input/output to the formats we can convert to/from,
1281    * but keep the original caps at the start - we will always prefer
1282    * passthrough */
1283   if (direction == GST_PAD_SINK) {
1284     out_caps = gst_caps_copy (caps);
1285     if (viewconvert->input_mode_override != GST_VIDEO_MULTIVIEW_MODE_NONE) {
1286       GstVideoMultiviewMode mode = viewconvert->input_mode_override;
1287       GstVideoMultiviewFlags flags = viewconvert->input_flags_override;
1288 
1289       const gchar *caps_str = gst_video_multiview_mode_to_caps_string (mode);
1290       /* Coerce the input caps before transforming, so the sizes come out right */
1291       gst_caps_set_simple (out_caps, "multiview-mode", G_TYPE_STRING,
1292           caps_str, "multiview-flags", GST_TYPE_VIDEO_MULTIVIEW_FLAGSET, flags,
1293           GST_FLAG_SET_MASK_EXACT, NULL);
1294     }
1295   } else {
1296     out_caps = gst_caps_new_empty ();
1297   }
1298 
1299   for (i = 0; i < gst_caps_get_size (caps); i++) {
1300     GstStructure *structure = gst_caps_get_structure (caps, i);
1301     GstCapsFeatures *features = gst_caps_get_features (caps, i);
1302     out_caps = _expand_structure (viewconvert, out_caps, structure, features);
1303   }
1304 
1305   if (gst_caps_is_empty (out_caps))
1306     goto out;
1307 
1308   /* If we have an output mode override, limit things to that */
1309   if (direction == GST_PAD_SINK &&
1310       viewconvert->output_mode_override != GST_VIDEO_MULTIVIEW_MODE_NONE) {
1311 
1312     tmp_caps = _intersect_with_mview_mode (out_caps,
1313         viewconvert->output_mode_override, viewconvert->output_flags_override);
1314 
1315     gst_caps_unref (out_caps);
1316     out_caps = tmp_caps;
1317   } else if (viewconvert->input_mode_override != GST_VIDEO_MULTIVIEW_MODE_NONE) {
1318     /* Prepend a copy of our preferred input caps in case the peer
1319      * can handle them */
1320     tmp_caps = _intersect_with_mview_mode (out_caps,
1321         viewconvert->input_mode_override, viewconvert->input_flags_override);
1322     out_caps = gst_caps_merge (out_caps, tmp_caps);
1323   }
1324   if (direction == GST_PAD_SRC) {
1325     GstStructure *s;
1326     /* When generating input caps, we also need a copy of the mono caps
1327      * without multiview-mode or flags for backwards compat, at the end */
1328     tmp_caps = _intersect_with_mview_mode (caps,
1329         GST_VIDEO_MULTIVIEW_MODE_MONO, GST_VIDEO_MULTIVIEW_FLAGS_NONE);
1330     if (!gst_caps_is_empty (tmp_caps)) {
1331       s = gst_caps_get_structure (tmp_caps, 0);
1332       gst_structure_remove_fields (s, "multiview-mode", "multiview-flags",
1333           NULL);
1334       out_caps = gst_caps_merge (out_caps, tmp_caps);
1335     } else
1336       gst_caps_unref (tmp_caps);
1337   }
1338 out:
1339   gst_caps_unref (caps);
1340 
1341   GST_DEBUG_OBJECT (viewconvert, "Have caps %" GST_PTR_FORMAT
1342       " filtering with caps %" GST_PTR_FORMAT, out_caps, filter);
1343 
1344   if (filter) {
1345     GstCaps *tmp =
1346         gst_caps_intersect_full (filter, out_caps, GST_CAPS_INTERSECT_FIRST);
1347     gst_caps_unref (out_caps);
1348     out_caps = tmp;
1349   }
1350 
1351   GST_DEBUG_OBJECT (viewconvert, "Returning caps %" GST_PTR_FORMAT, out_caps);
1352   return out_caps;
1353 }
1354 
1355 static GstCaps *
fixate_size(GstVulkanViewConvert * viewconvert,GstPadDirection direction,GstCaps * caps,GstCaps * othercaps)1356 fixate_size (GstVulkanViewConvert * viewconvert,
1357     GstPadDirection direction, GstCaps * caps, GstCaps * othercaps)
1358 {
1359   GstStructure *ins, *outs;
1360   const GValue *from_par, *to_par;
1361   GValue fpar = { 0, }, tpar = {
1362   0,};
1363 
1364   othercaps = gst_caps_make_writable (othercaps);
1365   othercaps = gst_caps_truncate (othercaps);
1366 
1367   GST_DEBUG_OBJECT (viewconvert, "trying to fixate othercaps %" GST_PTR_FORMAT
1368       " based on caps %" GST_PTR_FORMAT, othercaps, caps);
1369 
1370   ins = gst_caps_get_structure (caps, 0);
1371   outs = gst_caps_get_structure (othercaps, 0);
1372 
1373   from_par = gst_structure_get_value (ins, "pixel-aspect-ratio");
1374   to_par = gst_structure_get_value (outs, "pixel-aspect-ratio");
1375 
1376   /* If we're fixating from the sinkpad we always set the PAR and
1377    * assume that missing PAR on the sinkpad means 1/1 and
1378    * missing PAR on the srcpad means undefined
1379    */
1380   if (direction == GST_PAD_SINK) {
1381     if (!from_par) {
1382       g_value_init (&fpar, GST_TYPE_FRACTION);
1383       gst_value_set_fraction (&fpar, 1, 1);
1384       from_par = &fpar;
1385     }
1386     if (!to_par) {
1387       g_value_init (&tpar, GST_TYPE_FRACTION);
1388       gst_value_set_fraction (&tpar, 1, 1);
1389       to_par = &tpar;
1390     }
1391   } else {
1392     if (!to_par) {
1393       g_value_init (&tpar, GST_TYPE_FRACTION);
1394       gst_value_set_fraction (&tpar, 1, 1);
1395       to_par = &tpar;
1396 
1397       gst_structure_set (outs, "pixel-aspect-ratio", GST_TYPE_FRACTION, 1, 1,
1398           NULL);
1399     }
1400     if (!from_par) {
1401       g_value_init (&fpar, GST_TYPE_FRACTION);
1402       gst_value_set_fraction (&fpar, 1, 1);
1403       from_par = &fpar;
1404     }
1405   }
1406 
1407   /* we have both PAR but they might not be fixated */
1408   {
1409     gint from_w, from_h, from_par_n, from_par_d, to_par_n, to_par_d;
1410     gint w = 0, h = 0;
1411     gint from_dar_n, from_dar_d;
1412     gint num, den;
1413 
1414     /* from_par should be fixed */
1415     g_return_val_if_fail (gst_value_is_fixed (from_par), othercaps);
1416 
1417     from_par_n = gst_value_get_fraction_numerator (from_par);
1418     from_par_d = gst_value_get_fraction_denominator (from_par);
1419 
1420     gst_structure_get_int (ins, "width", &from_w);
1421     gst_structure_get_int (ins, "height", &from_h);
1422 
1423     gst_structure_get_int (outs, "width", &w);
1424     gst_structure_get_int (outs, "height", &h);
1425 
1426     /* if both width and height are already fixed, we can't do anything
1427      * about it anymore */
1428     if (w && h) {
1429       GST_DEBUG_OBJECT (viewconvert,
1430           "dimensions already set to %dx%d, not fixating", w, h);
1431       if (!gst_value_is_fixed (to_par)) {
1432         GST_DEBUG_OBJECT (viewconvert, "fixating to_par to %dx%d", 1, 1);
1433         if (gst_structure_has_field (outs, "pixel-aspect-ratio"))
1434           gst_structure_fixate_field_nearest_fraction (outs,
1435               "pixel-aspect-ratio", 1, 1);
1436       }
1437       goto done;
1438     }
1439 
1440     /* Calculate input DAR */
1441     if (!gst_util_fraction_multiply (from_w, from_h, from_par_n, from_par_d,
1442             &from_dar_n, &from_dar_d)) {
1443       GST_ELEMENT_ERROR (viewconvert, CORE, NEGOTIATION, (NULL),
1444           ("Error calculating the output scaled size - integer overflow"));
1445       goto done;
1446     }
1447 
1448     GST_DEBUG_OBJECT (viewconvert, "Input DAR is %d/%d", from_dar_n,
1449         from_dar_d);
1450 
1451     /* If either width or height are fixed there's not much we
1452      * can do either except choosing a height or width and PAR
1453      * that matches the DAR as good as possible
1454      */
1455     if (h) {
1456       gint num, den;
1457 
1458       GST_DEBUG_OBJECT (viewconvert, "height is fixed (%d)", h);
1459 
1460       if (!gst_value_is_fixed (to_par)) {
1461         /* (shortcut) copy-paste (??) of videoscale seems to aim for 1/1,
1462          * so let's make it so ...
1463          * especially if following code assumes fixed */
1464         GST_DEBUG_OBJECT (viewconvert, "fixating to_par to 1x1");
1465         gst_structure_fixate_field_nearest_fraction (outs,
1466             "pixel-aspect-ratio", 1, 1);
1467         to_par = gst_structure_get_value (outs, "pixel-aspect-ratio");
1468       }
1469 
1470       /* PAR is fixed, choose the height that is nearest to the
1471        * height with the same DAR */
1472       to_par_n = gst_value_get_fraction_numerator (to_par);
1473       to_par_d = gst_value_get_fraction_denominator (to_par);
1474 
1475       GST_DEBUG_OBJECT (viewconvert, "PAR is fixed %d/%d", to_par_n, to_par_d);
1476 
1477       if (!gst_util_fraction_multiply (from_dar_n, from_dar_d, to_par_d,
1478               to_par_n, &num, &den)) {
1479         GST_ELEMENT_ERROR (viewconvert, CORE, NEGOTIATION, (NULL),
1480             ("Error calculating the output scaled size - integer overflow"));
1481         goto done;
1482       }
1483 
1484       w = (guint) gst_util_uint64_scale_int (h, num, den);
1485       gst_structure_fixate_field_nearest_int (outs, "width", w);
1486 
1487       goto done;
1488     } else if (w) {
1489       gint num, den;
1490 
1491       GST_DEBUG_OBJECT (viewconvert, "width is fixed (%d)", w);
1492 
1493       if (!gst_value_is_fixed (to_par)) {
1494         /* (shortcut) copy-paste (??) of videoscale seems to aim for 1/1,
1495          * so let's make it so ...
1496          * especially if following code assumes fixed */
1497         GST_DEBUG_OBJECT (viewconvert, "fixating to_par to 1x1");
1498         gst_structure_fixate_field_nearest_fraction (outs,
1499             "pixel-aspect-ratio", 1, 1);
1500         to_par = gst_structure_get_value (outs, "pixel-aspect-ratio");
1501       }
1502 
1503       /* PAR is fixed, choose the height that is nearest to the
1504        * height with the same DAR */
1505       to_par_n = gst_value_get_fraction_numerator (to_par);
1506       to_par_d = gst_value_get_fraction_denominator (to_par);
1507 
1508       GST_DEBUG_OBJECT (viewconvert, "PAR is fixed %d/%d", to_par_n, to_par_d);
1509 
1510       if (!gst_util_fraction_multiply (from_dar_n, from_dar_d, to_par_d,
1511               to_par_n, &num, &den)) {
1512         GST_ELEMENT_ERROR (viewconvert, CORE, NEGOTIATION, (NULL),
1513             ("Error calculating the output scaled size - integer overflow"));
1514         goto done;
1515       }
1516 
1517       h = (guint) gst_util_uint64_scale_int (w, den, num);
1518       gst_structure_fixate_field_nearest_int (outs, "height", h);
1519 
1520       goto done;
1521     } else if (gst_value_is_fixed (to_par)) {
1522       GstStructure *tmp;
1523       gint set_h, set_w, f_h, f_w;
1524 
1525       to_par_n = gst_value_get_fraction_numerator (to_par);
1526       to_par_d = gst_value_get_fraction_denominator (to_par);
1527 
1528       /* Calculate scale factor for the PAR change */
1529       if (!gst_util_fraction_multiply (from_dar_n, from_dar_d, to_par_n,
1530               to_par_d, &num, &den)) {
1531         GST_ELEMENT_ERROR (viewconvert, CORE, NEGOTIATION, (NULL),
1532             ("Error calculating the output scaled size - integer overflow"));
1533         goto done;
1534       }
1535 
1536       /* Try to keep the input height */
1537       tmp = gst_structure_copy (outs);
1538       gst_structure_fixate_field_nearest_int (tmp, "height", from_h);
1539       gst_structure_get_int (tmp, "height", &set_h);
1540 
1541       /* This might have failed but try to scale the width
1542        * to keep the DAR nonetheless */
1543       w = (guint) gst_util_uint64_scale_int (set_h, num, den);
1544       gst_structure_fixate_field_nearest_int (tmp, "width", w);
1545       gst_structure_get_int (tmp, "width", &set_w);
1546       gst_structure_free (tmp);
1547 
1548       /* We kept the DAR and the height is nearest to the original height */
1549       if (set_w == w) {
1550         gst_structure_set (outs, "width", G_TYPE_INT, set_w, "height",
1551             G_TYPE_INT, set_h, NULL);
1552         goto done;
1553       }
1554 
1555       f_h = set_h;
1556       f_w = set_w;
1557 
1558       /* If the former failed, try to keep the input width at least */
1559       tmp = gst_structure_copy (outs);
1560       gst_structure_fixate_field_nearest_int (tmp, "width", from_w);
1561       gst_structure_get_int (tmp, "width", &set_w);
1562 
1563       /* This might have failed but try to scale the width
1564        * to keep the DAR nonetheless */
1565       h = (guint) gst_util_uint64_scale_int (set_w, den, num);
1566       gst_structure_fixate_field_nearest_int (tmp, "height", h);
1567       gst_structure_get_int (tmp, "height", &set_h);
1568       gst_structure_free (tmp);
1569 
1570       /* We kept the DAR and the width is nearest to the original width */
1571       if (set_h == h) {
1572         gst_structure_set (outs, "width", G_TYPE_INT, set_w, "height",
1573             G_TYPE_INT, set_h, NULL);
1574         goto done;
1575       }
1576 
1577       /* If all this failed, keep the height that was nearest to the original
1578        * height and the nearest possible width. This changes the DAR but
1579        * there's not much else to do here.
1580        */
1581       gst_structure_set (outs, "width", G_TYPE_INT, f_w, "height", G_TYPE_INT,
1582           f_h, NULL);
1583       goto done;
1584     } else {
1585       GstStructure *tmp;
1586       gint set_h, set_w, set_par_n, set_par_d, tmp2;
1587 
1588       /* width, height and PAR are not fixed */
1589 
1590       /* First try to keep the height and width as good as possible
1591        * and scale PAR */
1592       tmp = gst_structure_copy (outs);
1593       gst_structure_fixate_field_nearest_int (tmp, "height", from_h);
1594       gst_structure_get_int (tmp, "height", &set_h);
1595       gst_structure_fixate_field_nearest_int (tmp, "width", from_w);
1596       gst_structure_get_int (tmp, "width", &set_w);
1597 
1598       if (!gst_util_fraction_multiply (from_dar_n, from_dar_d, set_h, set_w,
1599               &to_par_n, &to_par_d)) {
1600         GST_ELEMENT_ERROR (viewconvert, CORE, NEGOTIATION, (NULL),
1601             ("Error calculating the output scaled size - integer overflow"));
1602         gst_structure_free (tmp);
1603         goto done;
1604       }
1605 
1606       if (!gst_structure_has_field (tmp, "pixel-aspect-ratio"))
1607         gst_structure_set_value (tmp, "pixel-aspect-ratio", to_par);
1608       gst_structure_fixate_field_nearest_fraction (tmp, "pixel-aspect-ratio",
1609           to_par_n, to_par_d);
1610       gst_structure_get_fraction (tmp, "pixel-aspect-ratio", &set_par_n,
1611           &set_par_d);
1612       gst_structure_free (tmp);
1613 
1614       if (set_par_n == to_par_n && set_par_d == to_par_d) {
1615         gst_structure_set (outs, "width", G_TYPE_INT, set_w, "height",
1616             G_TYPE_INT, set_h, NULL);
1617 
1618         if (gst_structure_has_field (outs, "pixel-aspect-ratio") ||
1619             set_par_n != set_par_d)
1620           gst_structure_set (outs, "pixel-aspect-ratio", GST_TYPE_FRACTION,
1621               set_par_n, set_par_d, NULL);
1622         goto done;
1623       }
1624 
1625       /* Otherwise try to scale width to keep the DAR with the set
1626        * PAR and height */
1627       if (!gst_util_fraction_multiply (from_dar_n, from_dar_d, set_par_d,
1628               set_par_n, &num, &den)) {
1629         GST_ELEMENT_ERROR (viewconvert, CORE, NEGOTIATION, (NULL),
1630             ("Error calculating the output scaled size - integer overflow"));
1631         goto done;
1632       }
1633 
1634       w = (guint) gst_util_uint64_scale_int (set_h, num, den);
1635       tmp = gst_structure_copy (outs);
1636       gst_structure_fixate_field_nearest_int (tmp, "width", w);
1637       gst_structure_get_int (tmp, "width", &tmp2);
1638       gst_structure_free (tmp);
1639 
1640       if (tmp2 == w) {
1641         gst_structure_set (outs, "width", G_TYPE_INT, tmp2, "height",
1642             G_TYPE_INT, set_h, NULL);
1643         if (gst_structure_has_field (outs, "pixel-aspect-ratio") ||
1644             set_par_n != set_par_d)
1645           gst_structure_set (outs, "pixel-aspect-ratio", GST_TYPE_FRACTION,
1646               set_par_n, set_par_d, NULL);
1647         goto done;
1648       }
1649 
1650       /* ... or try the same with the height */
1651       h = (guint) gst_util_uint64_scale_int (set_w, den, num);
1652       tmp = gst_structure_copy (outs);
1653       gst_structure_fixate_field_nearest_int (tmp, "height", h);
1654       gst_structure_get_int (tmp, "height", &tmp2);
1655       gst_structure_free (tmp);
1656 
1657       if (tmp2 == h) {
1658         gst_structure_set (outs, "width", G_TYPE_INT, set_w, "height",
1659             G_TYPE_INT, tmp2, NULL);
1660         if (gst_structure_has_field (outs, "pixel-aspect-ratio") ||
1661             set_par_n != set_par_d)
1662           gst_structure_set (outs, "pixel-aspect-ratio", GST_TYPE_FRACTION,
1663               set_par_n, set_par_d, NULL);
1664         goto done;
1665       }
1666 
1667       /* If all fails we can't keep the DAR and take the nearest values
1668        * for everything from the first try */
1669       gst_structure_set (outs, "width", G_TYPE_INT, set_w, "height",
1670           G_TYPE_INT, set_h, NULL);
1671       if (gst_structure_has_field (outs, "pixel-aspect-ratio") ||
1672           set_par_n != set_par_d)
1673         gst_structure_set (outs, "pixel-aspect-ratio", GST_TYPE_FRACTION,
1674             set_par_n, set_par_d, NULL);
1675     }
1676   }
1677 
1678 done:
1679   othercaps = gst_caps_fixate (othercaps);
1680 
1681   GST_DEBUG_OBJECT (viewconvert, "fixated othercaps to %" GST_PTR_FORMAT,
1682       othercaps);
1683 
1684   if (from_par == &fpar)
1685     g_value_unset (&fpar);
1686   if (to_par == &tpar)
1687     g_value_unset (&tpar);
1688 
1689   return othercaps;
1690 }
1691 
1692 static GstCaps *
gst_vulkan_view_convert_fixate_caps(GstBaseTransform * bt,GstPadDirection direction,GstCaps * caps,GstCaps * othercaps)1693 gst_vulkan_view_convert_fixate_caps (GstBaseTransform * bt,
1694     GstPadDirection direction, GstCaps * caps, GstCaps * othercaps)
1695 {
1696   GstVulkanViewConvert *viewconvert = GST_VULKAN_VIEW_CONVERT (bt);
1697   GstVideoMultiviewMode mode = viewconvert->output_mode_override;
1698   GstVideoMultiviewFlags flags = viewconvert->output_flags_override;
1699   GstCaps *tmp;
1700 
1701   othercaps = gst_caps_make_writable (othercaps);
1702   GST_LOG_OBJECT (viewconvert, "dir %s fixating %" GST_PTR_FORMAT
1703       " against caps %" GST_PTR_FORMAT,
1704       direction == GST_PAD_SINK ? "sink" : "src", othercaps, caps);
1705 
1706   if (direction == GST_PAD_SINK) {
1707     if (mode != GST_VIDEO_MULTIVIEW_MODE_NONE) {
1708       /* We have a requested output mode and are fixating source caps, try and enforce it */
1709       GST_DEBUG_OBJECT (bt, "fixating multiview mode using the configured "
1710           "output override mode 0x%x and flags 0x%x", mode, flags);
1711       tmp = _intersect_with_mview_mode (othercaps, mode, flags);
1712       gst_caps_unref (othercaps);
1713       othercaps = tmp;
1714     } else {
1715       /* See if we can do passthrough */
1716       GstVideoInfo info;
1717 
1718       if (gst_video_info_from_caps (&info, caps)) {
1719         GstVideoMultiviewMode mode = GST_VIDEO_INFO_MULTIVIEW_MODE (&info);
1720         GstVideoMultiviewFlags flags = GST_VIDEO_INFO_MULTIVIEW_FLAGS (&info);
1721 
1722         if (viewconvert->input_mode_override != GST_VIDEO_MULTIVIEW_MODE_NONE) {
1723           mode = viewconvert->input_mode_override;
1724           flags = viewconvert->input_flags_override;
1725         }
1726 
1727         tmp = _intersect_with_mview_mode (othercaps, mode, flags);
1728         if (gst_caps_is_empty (tmp)) {
1729           /* Nope, we can't pass our input caps downstream */
1730           gst_caps_unref (tmp);
1731         } else {
1732           GST_DEBUG_OBJECT (bt, "can configure a passthrough multiview mode "
1733               "using the input override mode 0x%x and flags 0x%x", mode, flags);
1734           gst_caps_unref (othercaps);
1735           othercaps = tmp;
1736           goto done;
1737         }
1738       }
1739 
1740       /* Prefer an unpacked mode for output */
1741       tmp =
1742           _intersect_with_mview_modes (othercaps,
1743           gst_video_multiview_get_unpacked_modes ());
1744       if (!gst_caps_is_empty (tmp)) {
1745         GST_DEBUG_OBJECT (bt, "preferring an unpacked multiview mode");
1746         gst_caps_unref (othercaps);
1747         othercaps = tmp;
1748       } else {
1749         gst_caps_unref (tmp);
1750       }
1751     }
1752   } else if (viewconvert->input_mode_override != GST_VIDEO_MULTIVIEW_MODE_NONE) {
1753     /* See if we can coerce the caps into matching input mode/flags,
1754      * in case it doesn't care at all, but allow it not to too */
1755     mode = viewconvert->input_mode_override;
1756     flags = viewconvert->input_flags_override;
1757     tmp = _intersect_with_mview_mode (othercaps, mode, flags);
1758     if (gst_caps_is_empty (tmp)) {
1759       /* Nope, we can pass our input caps downstream */
1760       gst_caps_unref (tmp);
1761     } else {
1762       GST_DEBUG_OBJECT (bt, "can configure a passthrough multiview mode "
1763           "using the input override mode 0x%x and flags 0x%x", mode, flags);
1764       gst_caps_unref (othercaps);
1765       othercaps = tmp;
1766     }
1767   }
1768 
1769 done:
1770   othercaps = fixate_size (viewconvert, direction, caps, othercaps);
1771   othercaps = gst_caps_fixate (othercaps);
1772 
1773   GST_DEBUG_OBJECT (viewconvert, "dir %s fixated to %" GST_PTR_FORMAT
1774       " against caps %" GST_PTR_FORMAT,
1775       direction == GST_PAD_SINK ? "sink" : "src", othercaps, caps);
1776   return othercaps;
1777 }
1778 
1779 static gboolean
create_descriptor_set_layout(GstVulkanViewConvert * conv,GError ** error)1780 create_descriptor_set_layout (GstVulkanViewConvert * conv, GError ** error)
1781 {
1782   GstVulkanVideoFilter *vfilter = GST_VULKAN_VIDEO_FILTER (conv);
1783   VkDescriptorSetLayoutBinding bindings[GST_VIDEO_MAX_PLANES * 2 + 1] =
1784       { {0,}, };
1785   VkDescriptorSetLayoutCreateInfo layout_info;
1786   VkDescriptorSetLayout descriptor_set_layout;
1787   int descriptor_n = 0;
1788   VkResult err;
1789   int i;
1790 
1791   /* *INDENT-OFF* */
1792   bindings[descriptor_n++] = (VkDescriptorSetLayoutBinding) {
1793       .binding = 0,
1794       .descriptorCount = 1,
1795       .descriptorType = VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER,
1796       .pImmutableSamplers = NULL,
1797       .stageFlags = VK_SHADER_STAGE_VERTEX_BIT | VK_SHADER_STAGE_FRAGMENT_BIT
1798   };
1799   for (i = 0; i < GST_VIDEO_INFO_N_PLANES (&vfilter->in_info) * 2; i++) {
1800     bindings[descriptor_n++] = (VkDescriptorSetLayoutBinding) {
1801       .binding = i+1,
1802       .descriptorCount = 1,
1803       .descriptorType = VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER,
1804       .pImmutableSamplers = NULL,
1805       .stageFlags = VK_SHADER_STAGE_FRAGMENT_BIT
1806     };
1807   };
1808 
1809   layout_info = (VkDescriptorSetLayoutCreateInfo) {
1810       .sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO,
1811       .pNext = NULL,
1812       .bindingCount = descriptor_n,
1813       .pBindings = bindings
1814   };
1815   /* *INDENT-ON* */
1816 
1817   err =
1818       vkCreateDescriptorSetLayout (vfilter->device->device, &layout_info,
1819       NULL, &descriptor_set_layout);
1820   if (gst_vulkan_error_to_g_error (err, error,
1821           "vkCreateDescriptorSetLayout") < 0) {
1822     return FALSE;
1823   }
1824 
1825   conv->quad->descriptor_set_layout =
1826       gst_vulkan_handle_new_wrapped (vfilter->device,
1827       GST_VULKAN_HANDLE_TYPE_DESCRIPTOR_SET_LAYOUT,
1828       (GstVulkanHandleTypedef) descriptor_set_layout,
1829       gst_vulkan_handle_free_descriptor_set_layout, NULL);
1830 
1831   return TRUE;
1832 }
1833 
1834 static gboolean
gst_vulkan_view_convert_start(GstBaseTransform * bt)1835 gst_vulkan_view_convert_start (GstBaseTransform * bt)
1836 {
1837   GstVulkanVideoFilter *vfilter = GST_VULKAN_VIDEO_FILTER (bt);
1838   GstVulkanViewConvert *conv = GST_VULKAN_VIEW_CONVERT (bt);
1839   GstVulkanHandle *vert, *frag;
1840 
1841   if (!GST_BASE_TRANSFORM_CLASS (parent_class)->start (bt))
1842     return FALSE;
1843 
1844   conv->quad = gst_vulkan_full_screen_quad_new (vfilter->queue);
1845 
1846   if (!(vert =
1847           gst_vulkan_create_shader (vfilter->device, identity_vert,
1848               identity_vert_size, NULL))) {
1849     return FALSE;
1850   }
1851   if (!(frag =
1852           gst_vulkan_create_shader (vfilter->device, view_convert_frag,
1853               view_convert_frag_size, NULL))) {
1854     gst_vulkan_handle_unref (vert);
1855     return FALSE;
1856   }
1857 
1858   if (!gst_vulkan_full_screen_quad_set_shaders (conv->quad, vert, frag)) {
1859     gst_vulkan_handle_unref (vert);
1860     gst_vulkan_handle_unref (frag);
1861     return FALSE;
1862   }
1863   gst_vulkan_handle_unref (vert);
1864   gst_vulkan_handle_unref (frag);
1865 
1866   return TRUE;
1867 }
1868 
1869 static gboolean
gst_vulkan_view_convert_set_caps(GstBaseTransform * bt,GstCaps * in_caps,GstCaps * out_caps)1870 gst_vulkan_view_convert_set_caps (GstBaseTransform * bt, GstCaps * in_caps,
1871     GstCaps * out_caps)
1872 {
1873   GstVulkanVideoFilter *vfilter = GST_VULKAN_VIDEO_FILTER (bt);
1874   GstVulkanViewConvert *conv = GST_VULKAN_VIEW_CONVERT (bt);
1875 
1876   if (!GST_BASE_TRANSFORM_CLASS (parent_class)->set_caps (bt, in_caps,
1877           out_caps))
1878     return FALSE;
1879 
1880   if (!gst_vulkan_full_screen_quad_set_info (conv->quad, &vfilter->in_info,
1881           &vfilter->out_info))
1882     return FALSE;
1883 
1884   if (conv->uniform)
1885     gst_memory_unref (conv->uniform);
1886   conv->uniform = NULL;
1887 
1888   return TRUE;
1889 }
1890 
1891 static gboolean
gst_vulkan_view_convert_stop(GstBaseTransform * bt)1892 gst_vulkan_view_convert_stop (GstBaseTransform * bt)
1893 {
1894   GstVulkanViewConvert *conv = GST_VULKAN_VIEW_CONVERT (bt);
1895 
1896   gst_clear_object (&conv->quad);
1897   if (conv->uniform)
1898     gst_memory_unref (conv->uniform);
1899   conv->uniform = NULL;
1900 
1901   return GST_BASE_TRANSFORM_CLASS (parent_class)->stop (bt);
1902 }
1903 
1904 static GstFlowReturn
gst_vulkan_view_convert_transform(GstBaseTransform * bt,GstBuffer * inbuf,GstBuffer * outbuf)1905 gst_vulkan_view_convert_transform (GstBaseTransform * bt, GstBuffer * inbuf,
1906     GstBuffer * outbuf)
1907 {
1908   GstVulkanVideoFilter *vfilter = GST_VULKAN_VIDEO_FILTER (bt);
1909   GstVulkanViewConvert *conv = GST_VULKAN_VIEW_CONVERT (bt);
1910   GstVulkanImageView *in_img_views[GST_VIDEO_MAX_PLANES] = { NULL, };
1911   GstVulkanImageView *out_img_views[GST_VIDEO_MAX_PLANES] = { NULL, };
1912   GstVulkanCommandBuffer *cmd_buf = NULL;
1913   GstVulkanFence *fence = NULL;
1914   GError *error = NULL;
1915   VkResult err;
1916   int i;
1917 
1918   if (!gst_vulkan_full_screen_quad_set_input_buffer (conv->quad, inbuf, &error))
1919     goto error;
1920   if (!gst_vulkan_full_screen_quad_set_output_buffer (conv->quad, outbuf,
1921           &error))
1922     goto error;
1923 
1924   fence = gst_vulkan_device_create_fence (vfilter->device, &error);
1925   if (!fence)
1926     goto error;
1927 
1928   for (i = 0; i < GST_VIDEO_INFO_N_PLANES (&conv->quad->in_info); i++) {
1929     GstMemory *img_mem = gst_buffer_peek_memory (inbuf, i);
1930     if (!gst_is_vulkan_image_memory (img_mem)) {
1931       g_set_error_literal (&error, GST_VULKAN_ERROR, GST_VULKAN_FAILED,
1932           "Input memory must be a GstVulkanImageMemory");
1933       goto error;
1934     }
1935     in_img_views[i] =
1936         gst_vulkan_get_or_create_image_view ((GstVulkanImageMemory *) img_mem);
1937     gst_vulkan_trash_list_add (conv->quad->trash_list,
1938         gst_vulkan_trash_list_acquire (conv->quad->trash_list, fence,
1939             gst_vulkan_trash_mini_object_unref,
1940             (GstMiniObject *) in_img_views[i]));
1941   }
1942   for (i = 0; i < GST_VIDEO_INFO_N_PLANES (&conv->quad->out_info); i++) {
1943     GstMemory *mem = gst_buffer_peek_memory (outbuf, i);
1944     if (!gst_is_vulkan_image_memory (mem)) {
1945       g_set_error_literal (&error, GST_VULKAN_ERROR, GST_VULKAN_FAILED,
1946           "Output memory must be a GstVulkanImageMemory");
1947       goto error;
1948     }
1949     out_img_views[i] =
1950         gst_vulkan_get_or_create_image_view ((GstVulkanImageMemory *) mem);
1951     gst_vulkan_trash_list_add (conv->quad->trash_list,
1952         gst_vulkan_trash_list_acquire (conv->quad->trash_list, fence,
1953             gst_vulkan_trash_mini_object_unref,
1954             (GstMiniObject *) out_img_views[i]));
1955   }
1956 
1957   {
1958     GstMemory *uniforms = get_uniforms (conv, in_img_views, out_img_views);
1959     if (!gst_vulkan_full_screen_quad_set_uniform_buffer (conv->quad, uniforms,
1960             &error))
1961       goto error;
1962     gst_memory_unref (uniforms);
1963   }
1964 
1965   if (!conv->quad->descriptor_set_layout)
1966     if (!create_descriptor_set_layout (conv, &error))
1967       goto error;
1968 
1969   if (!gst_vulkan_full_screen_quad_prepare_draw (conv->quad, fence, &error))
1970     goto error;
1971 
1972   if (!(cmd_buf =
1973           gst_vulkan_command_pool_create (conv->quad->cmd_pool, &error)))
1974     goto error;
1975 
1976   {
1977     VkCommandBufferBeginInfo cmd_buf_info = { 0, };
1978 
1979     /* *INDENT-OFF* */
1980     cmd_buf_info = (VkCommandBufferBeginInfo) {
1981         .sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO,
1982         .pNext = NULL,
1983         .flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT,
1984         .pInheritanceInfo = NULL
1985     };
1986     /* *INDENT-ON* */
1987 
1988     gst_vulkan_command_buffer_lock (cmd_buf);
1989     err = vkBeginCommandBuffer (cmd_buf->cmd, &cmd_buf_info);
1990     if (gst_vulkan_error_to_g_error (err, &error, "vkBeginCommandBuffer") < 0)
1991       goto error;
1992   }
1993 
1994   update_descriptor_set (conv, in_img_views);
1995   if (!gst_vulkan_full_screen_quad_fill_command_buffer (conv->quad, cmd_buf,
1996           fence, &error))
1997     goto unlock_error;
1998 
1999   err = vkEndCommandBuffer (cmd_buf->cmd);
2000   gst_vulkan_command_buffer_unlock (cmd_buf);
2001   if (gst_vulkan_error_to_g_error (err, &error, "vkEndCommandBuffer") < 0)
2002     goto error;
2003 
2004   if (!gst_vulkan_full_screen_quad_submit (conv->quad, cmd_buf, fence, &error))
2005     goto error;
2006 
2007   gst_vulkan_fence_unref (fence);
2008 
2009   return GST_FLOW_OK;
2010 
2011 unlock_error:
2012   if (cmd_buf) {
2013     gst_vulkan_command_buffer_unlock (cmd_buf);
2014     gst_vulkan_command_buffer_unref (cmd_buf);
2015   }
2016 
2017 error:
2018   gst_clear_mini_object ((GstMiniObject **) & fence);
2019 
2020   GST_ELEMENT_ERROR (bt, LIBRARY, FAILED, ("%s", error->message), (NULL));
2021   g_clear_error (&error);
2022   return GST_FLOW_ERROR;
2023 }
2024