1 /*
2 * GStreamer
3 * Copyright (C) 2015 Matthew Waters <matthew@centricular.com>
4 *
5 * This library is free software; you can redistribute it and/or
6 * modify it under the terms of the GNU Library General Public
7 * License as published by the Free Software Foundation; either
8 * version 2 of the License, or (at your option) any later version.
9 *
10 * This library is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 * Library General Public License for more details.
14 *
15 * You should have received a copy of the GNU Library General Public
16 * License along with this library; if not, write to the
17 * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
18 * Boston, MA 02110-1301, USA.
19 */
20
21 #ifdef HAVE_CONFIG_H
22 #include "config.h"
23 #endif
24
25 #include "gstvkutils.h"
26
27 /**
28 * SECTION:vkutils
29 * @title: Vulkan Utils
30 * @short_description: Vulkan utilities
31 * @see_also: #GstVulkanInstance, #GstVulkanDevice
32 */
33
34 GST_DEBUG_CATEGORY_STATIC (GST_CAT_CONTEXT);
35
36 static void
_init_context_debug(void)37 _init_context_debug (void)
38 {
39 #ifndef GST_DISABLE_GST_DEBUG
40 static gsize _init = 0;
41
42 if (g_once_init_enter (&_init)) {
43 GST_DEBUG_CATEGORY_GET (GST_CAT_CONTEXT, "GST_CONTEXT");
44 g_once_init_leave (&_init, 1);
45 }
46 #endif
47 }
48
49 static gboolean
_vk_pad_query(const GValue * item,GValue * value,gpointer user_data)50 _vk_pad_query (const GValue * item, GValue * value, gpointer user_data)
51 {
52 GstPad *pad = g_value_get_object (item);
53 GstQuery *query = user_data;
54 gboolean res;
55
56 _init_context_debug ();
57
58 res = gst_pad_peer_query (pad, query);
59
60 if (res) {
61 g_value_set_boolean (value, TRUE);
62 return FALSE;
63 }
64
65 GST_CAT_INFO_OBJECT (GST_CAT_CONTEXT, pad, "pad peer query failed");
66 return TRUE;
67 }
68
69 /**
70 * gst_vulkan_run_query:
71 * @element: a #GstElement
72 * @query: the #GstQuery to perform
73 * @direction: the #GstPadDirection to perform query on
74 *
75 * Returns: whether @query was answered successfully
76 *
77 * Since: 1.18
78 */
79 gboolean
gst_vulkan_run_query(GstElement * element,GstQuery * query,GstPadDirection direction)80 gst_vulkan_run_query (GstElement * element, GstQuery * query,
81 GstPadDirection direction)
82 {
83 GstIterator *it;
84 GstIteratorFoldFunction func = _vk_pad_query;
85 GValue res = { 0 };
86
87 g_value_init (&res, G_TYPE_BOOLEAN);
88 g_value_set_boolean (&res, FALSE);
89
90 /* Ask neighbor */
91 if (direction == GST_PAD_SRC)
92 it = gst_element_iterate_src_pads (element);
93 else
94 it = gst_element_iterate_sink_pads (element);
95
96 while (gst_iterator_fold (it, func, &res, query) == GST_ITERATOR_RESYNC)
97 gst_iterator_resync (it);
98
99 gst_iterator_free (it);
100
101 return g_value_get_boolean (&res);
102 }
103
104 static GstQuery *
_vulkan_local_context_query(GstElement * element,const gchar * context_type,gboolean set_context)105 _vulkan_local_context_query (GstElement * element,
106 const gchar * context_type, gboolean set_context)
107 {
108 GstQuery *query;
109 GstContext *ctxt;
110
111 _init_context_debug ();
112
113 /* 2a) Query downstream with GST_QUERY_CONTEXT for the context and
114 * check if downstream already has a context of the specific type
115 * 2b) Query upstream as above.
116 */
117 query = gst_query_new_context (context_type);
118 if (gst_vulkan_run_query (element, query, GST_PAD_SRC)) {
119 gst_query_parse_context (query, &ctxt);
120 GST_CAT_INFO_OBJECT (GST_CAT_CONTEXT, element,
121 "found context (%p) in downstream query", ctxt);
122 if (set_context)
123 gst_element_set_context (element, ctxt);
124 } else if (gst_vulkan_run_query (element, query, GST_PAD_SINK)) {
125 gst_query_parse_context (query, &ctxt);
126 GST_CAT_INFO_OBJECT (GST_CAT_CONTEXT, element,
127 "found context (%p) in upstream query", ctxt);
128 if (set_context)
129 gst_element_set_context (element, ctxt);
130 } else {
131 gst_query_unref (query);
132 query = NULL;
133 }
134
135 return query;
136 }
137
138 /**
139 * gst_vulkan_global_context_query:
140 * @element: a #GstElement
141 * @context_type: the context type to query for
142 *
143 * Performs the steps necessary for executing a context query including
144 * posting a message for the application to respond.
145 *
146 * Since: 1.18
147 */
148 void
gst_vulkan_global_context_query(GstElement * element,const gchar * context_type)149 gst_vulkan_global_context_query (GstElement * element,
150 const gchar * context_type)
151 {
152 GstQuery *query;
153 GstMessage *msg;
154
155 if ((query = _vulkan_local_context_query (element, context_type, TRUE))) {
156 gst_query_unref (query);
157 return;
158 }
159
160 /* 3) Post a GST_MESSAGE_NEED_CONTEXT message on the bus with
161 * the required context type and afterwards check if a
162 * usable context was set now as in 1). The message could
163 * be handled by the parent bins of the element and the
164 * application.
165 */
166 GST_CAT_INFO_OBJECT (GST_CAT_CONTEXT, element,
167 "posting need context message");
168 msg = gst_message_new_need_context (GST_OBJECT_CAST (element), context_type);
169 gst_element_post_message (element, msg);
170
171 /*
172 * Whomever responds to the need-context message performs a
173 * GstElement::set_context() with the required context in which the element
174 * is required to update the display_ptr or call gst_vulkan_handle_set_context().
175 */
176 }
177
178 /**
179 * gst_vulkan_local_context_query:
180 * @element: a #GstElement
181 * @context_type: the context type to query for
182 *
183 * Performs the steps necessary for executing a context query between only
184 * other elements in the pipeline
185 *
186 * Since: 1.18
187 */
188 GstQuery *
gst_vulkan_local_context_query(GstElement * element,const gchar * context_type)189 gst_vulkan_local_context_query (GstElement * element,
190 const gchar * context_type)
191 {
192 return _vulkan_local_context_query (element, context_type, FALSE);
193 }
194
195 static void
_vk_display_context_query(GstElement * element,GstVulkanDisplay ** display_ptr)196 _vk_display_context_query (GstElement * element,
197 GstVulkanDisplay ** display_ptr)
198 {
199 gst_vulkan_global_context_query (element,
200 GST_VULKAN_DISPLAY_CONTEXT_TYPE_STR);
201 }
202
203 /* 4) Create a context by itself and post a GST_MESSAGE_HAVE_CONTEXT
204 * message.
205 */
206 /*
207 * @element: (transfer none):
208 * @context: (transfer full):
209 */
210 static void
_vk_context_propagate(GstElement * element,GstContext * context)211 _vk_context_propagate (GstElement * element, GstContext * context)
212 {
213 GstMessage *msg;
214
215 _init_context_debug ();
216
217 gst_element_set_context (element, context);
218
219 GST_CAT_INFO_OBJECT (GST_CAT_CONTEXT, element,
220 "posting have context (%" GST_PTR_FORMAT ") message", context);
221 msg = gst_message_new_have_context (GST_OBJECT_CAST (element), context);
222 gst_element_post_message (GST_ELEMENT_CAST (element), msg);
223 }
224
225 /**
226 * gst_vulkan_ensure_element_data:
227 * @element: a #GstElement
228 * @display_ptr: (inout) (optional): the resulting #GstVulkanDisplay
229 * @instance_ptr: (inout): the resulting #GstVulkanInstance
230 *
231 * Perform the steps necessary for retrieving a #GstVulkanInstance and
232 * (optionally) an #GstVulkanDisplay from the surrounding elements or from
233 * the application using the #GstContext mechanism.
234 *
235 * If the contents of @display_ptr or @instance_ptr are not %NULL, then no
236 * #GstContext query is necessary and no #GstVulkanInstance or #GstVulkanDisplay
237 * retrieval is performed.
238 *
239 * Returns: whether a #GstVulkanInstance exists in @instance_ptr and if
240 * @display_ptr is not %NULL, whether a #GstVulkanDisplay exists in
241 * @display_ptr
242 *
243 * Since: 1.18
244 */
245 gboolean
gst_vulkan_ensure_element_data(GstElement * element,GstVulkanDisplay ** display_ptr,GstVulkanInstance ** instance_ptr)246 gst_vulkan_ensure_element_data (GstElement * element,
247 GstVulkanDisplay ** display_ptr, GstVulkanInstance ** instance_ptr)
248 {
249 g_return_val_if_fail (element != NULL, FALSE);
250 g_return_val_if_fail (instance_ptr != NULL, FALSE);
251
252 /* 1) Check if the element already has a context of the specific
253 * type.
254 */
255 if (!*instance_ptr) {
256 GError *error = NULL;
257 GstContext *context = NULL;
258
259 gst_vulkan_global_context_query (element,
260 GST_VULKAN_INSTANCE_CONTEXT_TYPE_STR);
261
262 /* Neighbour found and it updated the display */
263 if (!*instance_ptr) {
264 /* If no neighboor, or application not interested, use system default */
265 *instance_ptr = gst_vulkan_instance_new ();
266
267 context = gst_context_new (GST_VULKAN_INSTANCE_CONTEXT_TYPE_STR, TRUE);
268 gst_context_set_vulkan_instance (context, *instance_ptr);
269 }
270
271 if (!gst_vulkan_instance_open (*instance_ptr, &error)) {
272 GST_ELEMENT_ERROR (element, RESOURCE, NOT_FOUND,
273 ("Failed to create vulkan instance"), ("%s", error->message));
274 gst_object_unref (*instance_ptr);
275 *instance_ptr = NULL;
276 g_clear_error (&error);
277 return FALSE;
278 }
279
280 if (context)
281 _vk_context_propagate (element, context);
282 }
283
284 /* we don't care about a display */
285 if (!display_ptr)
286 return *instance_ptr != NULL;
287
288 if (!*display_ptr) {
289 _vk_display_context_query (element, display_ptr);
290
291 /* Neighbour found and it updated the display */
292 if (!*display_ptr) {
293 GstContext *context;
294
295 /* instance is required before the display */
296 g_return_val_if_fail (*instance_ptr != NULL, FALSE);
297
298 /* If no neighboor, or application not interested, use system default */
299 *display_ptr = gst_vulkan_display_new (*instance_ptr);
300
301 context = gst_context_new (GST_VULKAN_DISPLAY_CONTEXT_TYPE_STR, TRUE);
302 gst_context_set_vulkan_display (context, *display_ptr);
303
304 _vk_context_propagate (element, context);
305 }
306 }
307
308 return *display_ptr != NULL && *instance_ptr != NULL;
309 }
310
311 /**
312 * gst_vulkan_handle_set_context:
313 * @element: a #GstElement
314 * @context: a #GstContext
315 * @display: (inout) (transfer full) (optional): location of a #GstVulkanDisplay
316 * @instance: (inout) (transfer full): location of a #GstVulkanInstance
317 *
318 * Helper function for implementing #GstElementClass.set_context() in
319 * Vulkan capable elements.
320 *
321 * Retrieve's the #GstVulkanDisplay or #GstVulkanInstance in @context and places
322 * the result in @display or @instance respectively.
323 *
324 * Returns: whether the @display or @instance could be set successfully
325 *
326 * Since: 1.18
327 */
328 gboolean
gst_vulkan_handle_set_context(GstElement * element,GstContext * context,GstVulkanDisplay ** display,GstVulkanInstance ** instance)329 gst_vulkan_handle_set_context (GstElement * element, GstContext * context,
330 GstVulkanDisplay ** display, GstVulkanInstance ** instance)
331 {
332 GstVulkanDisplay *display_replacement = NULL;
333 GstVulkanInstance *instance_replacement = NULL;
334 const gchar *context_type;
335
336 g_return_val_if_fail (instance != NULL, FALSE);
337
338 if (!context)
339 return FALSE;
340
341 context_type = gst_context_get_context_type (context);
342
343 if (display
344 && g_strcmp0 (context_type, GST_VULKAN_DISPLAY_CONTEXT_TYPE_STR) == 0) {
345 if (!gst_context_get_vulkan_display (context, &display_replacement)) {
346 GST_WARNING_OBJECT (element, "Failed to get display from context");
347 return FALSE;
348 }
349 } else if (g_strcmp0 (context_type,
350 GST_VULKAN_INSTANCE_CONTEXT_TYPE_STR) == 0) {
351 if (!gst_context_get_vulkan_instance (context, &instance_replacement)) {
352 GST_WARNING_OBJECT (element, "Failed to get instance from context");
353 return FALSE;
354 }
355 }
356
357 if (display_replacement) {
358 GstVulkanDisplay *old = *display;
359 *display = display_replacement;
360
361 if (old)
362 gst_object_unref (old);
363 }
364
365 if (instance_replacement) {
366 GstVulkanInstance *old = *instance;
367 *instance = instance_replacement;
368
369 if (old)
370 gst_object_unref (old);
371 }
372
373 return TRUE;
374 }
375
376 /**
377 * gst_vulkan_handle_context_query:
378 * @element: a #GstElement
379 * @query: a #GstQuery of type %GST_QUERY_CONTEXT
380 * @display: (transfer none) (nullable): a #GstVulkanDisplay
381 * @instance: (transfer none) (nullable): a #GstVulkanInstance
382 * @device: (transfer none) (nullable): a #GstVulkanInstance
383 *
384 * Returns: Whether the @query was successfully responded to from the passed
385 * @display, @instance, and @device.
386 *
387 * Since: 1.18
388 */
389 gboolean
gst_vulkan_handle_context_query(GstElement * element,GstQuery * query,GstVulkanDisplay * display,GstVulkanInstance * instance,GstVulkanDevice * device)390 gst_vulkan_handle_context_query (GstElement * element, GstQuery * query,
391 GstVulkanDisplay * display, GstVulkanInstance * instance,
392 GstVulkanDevice * device)
393 {
394 if (gst_vulkan_display_handle_context_query (element, query, display))
395 return TRUE;
396 if (gst_vulkan_instance_handle_context_query (element, query, instance))
397 return TRUE;
398 if (gst_vulkan_device_handle_context_query (element, query, device))
399 return TRUE;
400
401 return FALSE;
402 }
403
404 static void
fill_vulkan_image_view_info(VkImage image,VkFormat format,VkImageViewCreateInfo * info)405 fill_vulkan_image_view_info (VkImage image, VkFormat format,
406 VkImageViewCreateInfo * info)
407 {
408 /* *INDENT-OFF* */
409 *info = (VkImageViewCreateInfo) {
410 .sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO,
411 .pNext = NULL,
412 .image = image,
413 .format = format,
414 .viewType = VK_IMAGE_VIEW_TYPE_2D,
415 .flags = 0,
416 .components = (VkComponentMapping) {
417 VK_COMPONENT_SWIZZLE_R,
418 VK_COMPONENT_SWIZZLE_G,
419 VK_COMPONENT_SWIZZLE_B,
420 VK_COMPONENT_SWIZZLE_A
421 },
422 .subresourceRange = (VkImageSubresourceRange) {
423 .aspectMask = VK_IMAGE_ASPECT_COLOR_BIT,
424 .baseMipLevel = 0,
425 .levelCount = 1,
426 .baseArrayLayer = 0,
427 .layerCount = 1,
428 }
429 };
430 /* *INDENT-ON* */
431 }
432
433 static gboolean
find_compatible_view(GstVulkanImageView * view,VkImageViewCreateInfo * info)434 find_compatible_view (GstVulkanImageView * view, VkImageViewCreateInfo * info)
435 {
436 return view->create_info.image == info->image
437 && view->create_info.format == info->format
438 && view->create_info.viewType == info->viewType
439 && view->create_info.flags == info->flags
440 && view->create_info.components.r == info->components.r
441 && view->create_info.components.g == info->components.g
442 && view->create_info.components.b == info->components.b
443 && view->create_info.components.a == info->components.a
444 && view->create_info.subresourceRange.aspectMask ==
445 info->subresourceRange.aspectMask
446 && view->create_info.subresourceRange.baseMipLevel ==
447 info->subresourceRange.baseMipLevel
448 && view->create_info.subresourceRange.levelCount ==
449 info->subresourceRange.levelCount
450 && view->create_info.subresourceRange.baseArrayLayer ==
451 info->subresourceRange.baseArrayLayer
452 && view->create_info.subresourceRange.layerCount ==
453 info->subresourceRange.layerCount;
454 }
455
456 /**
457 * gst_vulkan_get_or_create_image_view
458 * @image: a #GstVulkanImageMemory
459 *
460 * Returns: (transfer full): a #GstVulkanImageView for @image matching the
461 * original layout and format of @image
462 *
463 * Since: 1.18
464 */
465 GstVulkanImageView *
gst_vulkan_get_or_create_image_view(GstVulkanImageMemory * image)466 gst_vulkan_get_or_create_image_view (GstVulkanImageMemory * image)
467 {
468 VkImageViewCreateInfo create_info;
469 GstVulkanImageView *ret = NULL;
470
471 fill_vulkan_image_view_info (image->image, image->create_info.format,
472 &create_info);
473
474 ret = gst_vulkan_image_memory_find_view (image,
475 (GstVulkanImageMemoryFindViewFunc) find_compatible_view, &create_info);
476 if (!ret) {
477 ret = gst_vulkan_image_view_new (image, &create_info);
478 gst_vulkan_image_memory_add_view (image, ret);
479 }
480
481 return ret;
482 }
483
484 #define SPIRV_MAGIC_NUMBER_NE 0x07230203
485 #define SPIRV_MAGIC_NUMBER_OE 0x03022307
486
487 /**
488 * gst_vulkan_create_shader
489 * @device: a #GstVulkanDevice
490 * @code: the SPIR-V shader byte code
491 * @size: length of @code. Must be a multiple of 4
492 * @error: a #GError to fill on failure
493 *
494 * Returns: (transfer full): a #GstVulkanHandle for @image matching the
495 * original layout and format of @image or %NULL
496 *
497 * Since: 1.18
498 */
499 GstVulkanHandle *
gst_vulkan_create_shader(GstVulkanDevice * device,const gchar * code,gsize size,GError ** error)500 gst_vulkan_create_shader (GstVulkanDevice * device, const gchar * code,
501 gsize size, GError ** error)
502 {
503 VkShaderModule shader;
504 VkResult res;
505
506 /* *INDENT-OFF* */
507 VkShaderModuleCreateInfo info = {
508 .sType = VK_STRUCTURE_TYPE_SHADER_MODULE_CREATE_INFO,
509 .pNext = NULL,
510 .flags = 0,
511 .codeSize = size,
512 .pCode = (const guint32 *) code
513 };
514 /* *INDENT-ON* */
515 guint32 first_word;
516 guint32 *new_code = NULL;
517
518 g_return_val_if_fail (size >= 4, VK_NULL_HANDLE);
519 g_return_val_if_fail (size % 4 == 0, VK_NULL_HANDLE);
520
521 first_word = code[0] | code[1] << 8 | code[2] << 16 | code[3] << 24;
522 g_return_val_if_fail (first_word == SPIRV_MAGIC_NUMBER_NE
523 || first_word == SPIRV_MAGIC_NUMBER_OE, VK_NULL_HANDLE);
524 if (first_word == SPIRV_MAGIC_NUMBER_OE) {
525 /* endianness swap... */
526 const guint32 *old_code = (const guint32 *) code;
527 gsize i;
528
529 GST_DEBUG ("performaing endianness conversion on spirv shader of size %"
530 G_GSIZE_FORMAT, size);
531 new_code = g_new0 (guint32, size / 4);
532
533 for (i = 0; i < size / 4; i++) {
534 guint32 old = old_code[i];
535 guint32 new = 0;
536
537 new |= (old & 0xff) << 24;
538 new |= (old & 0xff00) << 8;
539 new |= (old & 0xff0000) >> 8;
540 new |= (old & 0xff000000) >> 24;
541 new_code[i] = new;
542 }
543
544 first_word = ((guint32 *) new_code)[0];
545 g_assert (first_word == SPIRV_MAGIC_NUMBER_NE);
546
547 info.pCode = new_code;
548 }
549
550 res = vkCreateShaderModule (device->device, &info, NULL, &shader);
551 g_free (new_code);
552 if (gst_vulkan_error_to_g_error (res, error, "VkCreateShaderModule") < 0)
553 return NULL;
554
555 return gst_vulkan_handle_new_wrapped (device, GST_VULKAN_HANDLE_TYPE_SHADER,
556 (GstVulkanHandleTypedef) shader, gst_vulkan_handle_free_shader, NULL);
557 }
558