1 /*
2 * GStreamer
3 * Copyright (C) 2012-2014 Matthew Waters <ystree00@gmail.com>
4 * Copyright (C) 2017 Sebastian Dröge <sebastian@centricular.com>
5 *
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Library General Public
8 * License as published by the Free Software Foundation; either
9 * version 2 of the License, or (at your option) any later version.
10 *
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Library General Public License for more details.
15 *
16 * You should have received a copy of the GNU Library General Public
17 * License along with this library; if not, write to the
18 * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
19 * Boston, MA 02110-1301, USA.
20 */
21
22 #ifdef HAVE_CONFIG_H
23 #include "config.h"
24 #endif
25
26 #include <stdio.h>
27
28 #include "gl.h"
29 #include "gstglupload.h"
30 #include "gstglfuncs.h"
31
32 #if GST_GL_HAVE_PLATFORM_EGL
33 #include "egl/gsteglimage.h"
34 #include "egl/gsteglimage_private.h"
35 #include "egl/gstglmemoryegl.h"
36 #include "egl/gstglcontext_egl.h"
37 #endif
38
39 #if GST_GL_HAVE_DMABUF
40 #include <gst/allocators/gstdmabuf.h>
41 #endif
42
43 #if GST_GL_HAVE_VIV_DIRECTVIV
44 #include <gst/allocators/gstphysmemory.h>
45 #include <gst/gl/gstglfuncs.h>
46 #endif
47
48 /**
49 * SECTION:gstglupload
50 * @title: GstGLUpload
51 * @short_description: an object that uploads to GL textures
52 * @see_also: #GstGLDownload, #GstGLMemory
53 *
54 * #GstGLUpload is an object that uploads data from system memory into GL textures.
55 *
56 * A #GstGLUpload can be created with gst_gl_upload_new()
57 */
58
59 #define USING_OPENGL(context) (gst_gl_context_check_gl_version (context, GST_GL_API_OPENGL, 1, 0))
60 #define USING_OPENGL3(context) (gst_gl_context_check_gl_version (context, GST_GL_API_OPENGL3, 3, 1))
61 #define USING_GLES(context) (gst_gl_context_check_gl_version (context, GST_GL_API_GLES, 1, 0))
62 #define USING_GLES2(context) (gst_gl_context_check_gl_version (context, GST_GL_API_GLES2, 2, 0))
63 #define USING_GLES3(context) (gst_gl_context_check_gl_version (context, GST_GL_API_GLES2, 3, 0))
64
65 GST_DEBUG_CATEGORY_STATIC (gst_gl_upload_debug);
66 #define GST_CAT_DEFAULT gst_gl_upload_debug
67
68 static void gst_gl_upload_finalize (GObject * object);
69
70 static GstGLTextureTarget
_caps_get_texture_target(GstCaps * caps,GstGLTextureTarget default_target)71 _caps_get_texture_target (GstCaps * caps, GstGLTextureTarget default_target)
72 {
73 GstGLTextureTarget ret = 0;
74 GstStructure *s = gst_caps_get_structure (caps, 0);
75
76 if (gst_structure_has_field_typed (s, "texture-target", G_TYPE_STRING)) {
77 const gchar *target_str = gst_structure_get_string (s, "texture-target");
78 ret = gst_gl_texture_target_from_string (target_str);
79 }
80
81 if (!ret)
82 ret = default_target;
83
84 return ret;
85 }
86
87 /* Define the maximum number of planes we can upload - handle 2 views per buffer */
88 #define GST_GL_UPLOAD_MAX_PLANES (GST_VIDEO_MAX_PLANES * 2)
89
90 typedef struct _UploadMethod UploadMethod;
91
92 struct _GstGLUploadPrivate
93 {
94 GstVideoInfo in_info;
95 GstVideoInfo out_info;
96 GstCaps *in_caps;
97 GstCaps *out_caps;
98
99 GstBuffer *outbuf;
100
101 /* all method impl pointers */
102 gpointer *upload_impl;
103
104 /* current method */
105 const UploadMethod *method;
106 gpointer method_impl;
107 int method_i;
108
109 /* saved method for reconfigure */
110 int saved_method_i;
111 };
112
113 #define DEBUG_INIT \
114 GST_DEBUG_CATEGORY_INIT (gst_gl_upload_debug, "glupload", 0, "upload");
115
116 G_DEFINE_TYPE_WITH_CODE (GstGLUpload, gst_gl_upload, GST_TYPE_OBJECT,
117 G_ADD_PRIVATE (GstGLUpload) DEBUG_INIT);
118
119 static GstCaps *
_set_caps_features_with_passthrough(const GstCaps * caps,const gchar * feature_name,GstCapsFeatures * passthrough)120 _set_caps_features_with_passthrough (const GstCaps * caps,
121 const gchar * feature_name, GstCapsFeatures * passthrough)
122 {
123 guint i, j, m, n;
124 GstCaps *tmp;
125
126 tmp = gst_caps_new_empty ();
127
128 n = gst_caps_get_size (caps);
129 for (i = 0; i < n; i++) {
130 GstCapsFeatures *features, *orig_features;
131 GstStructure *s = gst_caps_get_structure (caps, i);
132
133 orig_features = gst_caps_get_features (caps, i);
134 features = gst_caps_features_new (feature_name, NULL);
135
136 if (gst_caps_features_is_any (orig_features)) {
137 /* if we have any features, we add both the features with and without @passthrough */
138 gst_caps_append_structure_full (tmp, gst_structure_copy (s),
139 gst_caps_features_copy (features));
140
141 m = gst_caps_features_get_size (passthrough);
142 for (j = 0; j < m; j++) {
143 const gchar *feature = gst_caps_features_get_nth (passthrough, j);
144
145 /* if we already have the features */
146 if (gst_caps_features_contains (features, feature))
147 continue;
148
149 gst_caps_features_add (features, feature);
150 }
151 } else {
152 m = gst_caps_features_get_size (orig_features);
153 for (j = 0; j < m; j++) {
154 const gchar *feature = gst_caps_features_get_nth (orig_features, j);
155
156 /* if we already have the features */
157 if (gst_caps_features_contains (features, feature))
158 continue;
159
160 if (g_strcmp0 (feature, GST_CAPS_FEATURE_MEMORY_SYSTEM_MEMORY) == 0)
161 continue;
162
163 if (gst_caps_features_contains (passthrough, feature)) {
164 gst_caps_features_add (features, feature);
165 }
166 }
167 }
168
169 gst_caps_append_structure_full (tmp, gst_structure_copy (s), features);
170 }
171
172 return tmp;
173 }
174
175 static GstCaps *
_caps_intersect_texture_target(GstCaps * caps,GstGLTextureTarget target_mask)176 _caps_intersect_texture_target (GstCaps * caps, GstGLTextureTarget target_mask)
177 {
178 GValue targets = G_VALUE_INIT;
179 GstCaps *ret, *target;
180
181 target = gst_caps_copy (caps);
182 gst_gl_value_set_texture_target_from_mask (&targets, target_mask);
183 gst_caps_set_value (target, "texture-target", &targets);
184
185 ret = gst_caps_intersect_full (caps, target, GST_CAPS_INTERSECT_FIRST);
186
187 g_value_unset (&targets);
188 gst_caps_unref (target);
189 return ret;
190 }
191
192 typedef enum
193 {
194 METHOD_FLAG_CAN_SHARE_CONTEXT = 1,
195 } GstGLUploadMethodFlags;
196
197 struct _UploadMethod
198 {
199 const gchar *name;
200 GstGLUploadMethodFlags flags;
201
202 GstStaticCaps *input_template_caps;
203
204 gpointer (*new) (GstGLUpload * upload);
205 GstCaps *(*transform_caps) (gpointer impl, GstGLContext * context,
206 GstPadDirection direction, GstCaps * caps);
207 gboolean (*accept) (gpointer impl, GstBuffer * buffer, GstCaps * in_caps,
208 GstCaps * out_caps);
209 void (*propose_allocation) (gpointer impl, GstQuery * decide_query,
210 GstQuery * query);
211 GstGLUploadReturn (*perform) (gpointer impl, GstBuffer * buffer,
212 GstBuffer ** outbuf);
213 void (*free) (gpointer impl);
214 } _UploadMethod;
215
216 struct GLMemoryUpload
217 {
218 GstGLUpload *upload;
219 GstGLTextureTarget input_target;
220 GstGLTextureTarget output_target;
221 };
222
223 static gpointer
_gl_memory_upload_new(GstGLUpload * upload)224 _gl_memory_upload_new (GstGLUpload * upload)
225 {
226 struct GLMemoryUpload *mem = g_new0 (struct GLMemoryUpload, 1);
227
228 mem->upload = upload;
229 mem->input_target = GST_GL_TEXTURE_TARGET_NONE;
230 mem->output_target = GST_GL_TEXTURE_TARGET_NONE;
231
232 return mem;
233 }
234
235 static GstCaps *
_gl_memory_upload_transform_caps(gpointer impl,GstGLContext * context,GstPadDirection direction,GstCaps * caps)236 _gl_memory_upload_transform_caps (gpointer impl, GstGLContext * context,
237 GstPadDirection direction, GstCaps * caps)
238 {
239 struct GLMemoryUpload *upload = impl;
240 GstCapsFeatures *passthrough =
241 gst_caps_features_from_string
242 (GST_CAPS_FEATURE_META_GST_VIDEO_OVERLAY_COMPOSITION);
243 GstCaps *ret;
244
245 ret =
246 _set_caps_features_with_passthrough (caps,
247 GST_CAPS_FEATURE_MEMORY_GL_MEMORY, passthrough);
248
249 gst_caps_features_free (passthrough);
250
251 if (direction == GST_PAD_SINK) {
252 GstCaps *tmp;
253 GstGLTextureTarget target_mask;
254
255 if (upload->input_target != GST_GL_TEXTURE_TARGET_NONE) {
256 target_mask = 1 << upload->input_target;
257 } else {
258 target_mask = 1 << GST_GL_TEXTURE_TARGET_2D |
259 1 << GST_GL_TEXTURE_TARGET_RECTANGLE |
260 1 << GST_GL_TEXTURE_TARGET_EXTERNAL_OES;
261 }
262
263 tmp = _caps_intersect_texture_target (ret, target_mask);
264 gst_caps_unref (ret);
265 ret = tmp;
266 } else {
267 gint i, n;
268
269 n = gst_caps_get_size (ret);
270 for (i = 0; i < n; i++) {
271 GstStructure *s = gst_caps_get_structure (ret, i);
272
273 gst_structure_remove_fields (s, "texture-target", NULL);
274 }
275 }
276
277 return ret;
278 }
279
280 static gboolean
_gl_memory_upload_accept(gpointer impl,GstBuffer * buffer,GstCaps * in_caps,GstCaps * out_caps)281 _gl_memory_upload_accept (gpointer impl, GstBuffer * buffer, GstCaps * in_caps,
282 GstCaps * out_caps)
283 {
284 struct GLMemoryUpload *upload = impl;
285 GstCapsFeatures *features;
286 int i;
287
288 features = gst_caps_get_features (out_caps, 0);
289 if (!gst_caps_features_contains (features, GST_CAPS_FEATURE_MEMORY_GL_MEMORY))
290 return FALSE;
291
292 features = gst_caps_get_features (in_caps, 0);
293 if (!gst_caps_features_contains (features, GST_CAPS_FEATURE_MEMORY_GL_MEMORY)
294 && !gst_caps_features_contains (features,
295 GST_CAPS_FEATURE_MEMORY_SYSTEM_MEMORY))
296 return FALSE;
297
298 if (buffer) {
299 GstVideoInfo *in_info = &upload->upload->priv->in_info;
300 guint expected_memories = GST_VIDEO_INFO_N_PLANES (in_info);
301
302 /* Support stereo views for separated multiview mode */
303 if (GST_VIDEO_INFO_MULTIVIEW_MODE (in_info) ==
304 GST_VIDEO_MULTIVIEW_MODE_SEPARATED)
305 expected_memories *= GST_VIDEO_INFO_VIEWS (in_info);
306
307 if (gst_buffer_n_memory (buffer) != expected_memories)
308 return FALSE;
309
310 for (i = 0; i < expected_memories; i++) {
311 GstMemory *mem = gst_buffer_peek_memory (buffer, i);
312
313 if (!gst_is_gl_memory (mem))
314 return FALSE;
315 }
316 }
317
318 return TRUE;
319 }
320
321 static void
_gl_memory_upload_propose_allocation(gpointer impl,GstQuery * decide_query,GstQuery * query)322 _gl_memory_upload_propose_allocation (gpointer impl, GstQuery * decide_query,
323 GstQuery * query)
324 {
325 struct GLMemoryUpload *upload = impl;
326 GstBufferPool *pool = NULL;
327 guint n_pools, i;
328 GstCaps *caps;
329 GstCapsFeatures *features;
330
331 gst_query_parse_allocation (query, &caps, NULL);
332 if (caps == NULL)
333 goto invalid_caps;
334 features = gst_caps_get_features (caps, 0);
335
336 /* Only offer our custom allocator if that type of memory was negotiated. */
337 if (gst_caps_features_contains (features, GST_CAPS_FEATURE_MEMORY_GL_MEMORY)) {
338 GstAllocator *allocator;
339 GstAllocationParams params;
340 gst_allocation_params_init (¶ms);
341
342 allocator =
343 GST_ALLOCATOR (gst_gl_memory_allocator_get_default (upload->
344 upload->context));
345 gst_query_add_allocation_param (query, allocator, ¶ms);
346 gst_object_unref (allocator);
347
348 #if GST_GL_HAVE_PLATFORM_EGL
349 if (upload->upload->context
350 && gst_gl_context_get_gl_platform (upload->upload->context) ==
351 GST_GL_PLATFORM_EGL) {
352 allocator =
353 GST_ALLOCATOR (gst_allocator_find (GST_GL_MEMORY_EGL_ALLOCATOR_NAME));
354 gst_query_add_allocation_param (query, allocator, ¶ms);
355 gst_object_unref (allocator);
356 }
357 #endif
358 }
359
360 n_pools = gst_query_get_n_allocation_pools (query);
361 for (i = 0; i < n_pools; i++) {
362 gst_query_parse_nth_allocation_pool (query, i, &pool, NULL, NULL, NULL);
363 if (!GST_IS_GL_BUFFER_POOL (pool)) {
364 gst_object_unref (pool);
365 pool = NULL;
366 }
367 }
368
369 if (!pool) {
370 GstStructure *config;
371 GstVideoInfo info;
372 gsize size;
373
374
375 if (!gst_video_info_from_caps (&info, caps))
376 goto invalid_caps;
377
378 pool = gst_gl_buffer_pool_new (upload->upload->context);
379 config = gst_buffer_pool_get_config (pool);
380
381 /* the normal size of a frame */
382 size = info.size;
383 gst_buffer_pool_config_set_params (config, caps, size, 0, 0);
384 gst_buffer_pool_config_add_option (config,
385 GST_BUFFER_POOL_OPTION_GL_SYNC_META);
386 if (upload->upload->priv->out_caps) {
387 GstGLTextureTarget target;
388 const gchar *target_pool_option_str;
389
390 target =
391 _caps_get_texture_target (upload->upload->priv->out_caps,
392 GST_GL_TEXTURE_TARGET_2D);
393 target_pool_option_str =
394 gst_gl_texture_target_to_buffer_pool_option (target);
395 gst_buffer_pool_config_add_option (config, target_pool_option_str);
396 }
397
398 if (!gst_buffer_pool_set_config (pool, config)) {
399 gst_object_unref (pool);
400 goto config_failed;
401 }
402
403 gst_query_add_allocation_pool (query, pool, size, 1, 0);
404 }
405
406 if (pool)
407 gst_object_unref (pool);
408
409 return;
410
411 invalid_caps:
412 {
413 GST_WARNING_OBJECT (upload->upload, "invalid caps specified");
414 return;
415 }
416 config_failed:
417 {
418 GST_WARNING_OBJECT (upload->upload, "failed setting config");
419 return;
420 }
421 }
422
423 static GstGLUploadReturn
_gl_memory_upload_perform(gpointer impl,GstBuffer * buffer,GstBuffer ** outbuf)424 _gl_memory_upload_perform (gpointer impl, GstBuffer * buffer,
425 GstBuffer ** outbuf)
426 {
427 struct GLMemoryUpload *upload = impl;
428 GstGLMemory *gl_mem;
429 int i, n;
430
431 n = gst_buffer_n_memory (buffer);
432 for (i = 0; i < n; i++) {
433 GstMemory *mem = gst_buffer_peek_memory (buffer, i);
434
435 gl_mem = (GstGLMemory *) mem;
436 if (!gst_gl_context_can_share (upload->upload->context,
437 gl_mem->mem.context))
438 return GST_GL_UPLOAD_UNSHARED_GL_CONTEXT;
439
440 if (upload->output_target == GST_GL_TEXTURE_TARGET_NONE &&
441 upload->upload->priv->out_caps) {
442 upload->output_target =
443 _caps_get_texture_target (upload->upload->priv->out_caps,
444 GST_GL_TEXTURE_TARGET_NONE);
445 }
446
447 /* always track the last input texture target so ::transform_caps() can
448 * use it to build the output caps */
449 upload->input_target = gl_mem->tex_target;
450 if (upload->output_target != gl_mem->tex_target) {
451 *outbuf = NULL;
452 return GST_GL_UPLOAD_RECONFIGURE;
453 }
454
455 if (gst_is_gl_memory_pbo (mem))
456 gst_gl_memory_pbo_upload_transfer ((GstGLMemoryPBO *) mem);
457 }
458
459 *outbuf = gst_buffer_ref (buffer);
460
461 return GST_GL_UPLOAD_DONE;
462 }
463
464 static void
_gl_memory_upload_free(gpointer impl)465 _gl_memory_upload_free (gpointer impl)
466 {
467 g_free (impl);
468 }
469
470
471 static GstStaticCaps _gl_memory_upload_caps =
472 GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE_WITH_FEATURES
473 (GST_CAPS_FEATURE_MEMORY_GL_MEMORY, GST_GL_MEMORY_VIDEO_FORMATS_STR));
474
475 static const UploadMethod _gl_memory_upload = {
476 "GLMemory",
477 METHOD_FLAG_CAN_SHARE_CONTEXT,
478 &_gl_memory_upload_caps,
479 &_gl_memory_upload_new,
480 &_gl_memory_upload_transform_caps,
481 &_gl_memory_upload_accept,
482 &_gl_memory_upload_propose_allocation,
483 &_gl_memory_upload_perform,
484 &_gl_memory_upload_free
485 };
486
487 #if GST_GL_HAVE_DMABUF
488 struct DmabufUpload
489 {
490 GstGLUpload *upload;
491
492 GstEGLImage *eglimage[GST_VIDEO_MAX_PLANES];
493 GstGLFormat formats[GST_VIDEO_MAX_PLANES];
494 GstBuffer *outbuf;
495 GstGLVideoAllocationParams *params;
496 guint n_mem;
497
498 gboolean direct;
499 GstGLTextureTarget target;
500 GstVideoInfo out_info;
501 /* only used for pointer comparison */
502 gpointer out_caps;
503 };
504
505 static GstStaticCaps _dma_buf_upload_caps =
506 GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE_WITH_FEATURES
507 (GST_CAPS_FEATURE_MEMORY_DMABUF,
508 GST_GL_MEMORY_VIDEO_FORMATS_STR) ";"
509 GST_VIDEO_CAPS_MAKE (GST_GL_MEMORY_VIDEO_FORMATS_STR));
510
511 static gpointer
_dma_buf_upload_new(GstGLUpload * upload)512 _dma_buf_upload_new (GstGLUpload * upload)
513 {
514 struct DmabufUpload *dmabuf = g_new0 (struct DmabufUpload, 1);
515 dmabuf->upload = upload;
516 dmabuf->target = GST_GL_TEXTURE_TARGET_2D;
517 return dmabuf;
518 }
519
520 static GstCaps *
_dma_buf_upload_transform_caps(gpointer impl,GstGLContext * context,GstPadDirection direction,GstCaps * caps)521 _dma_buf_upload_transform_caps (gpointer impl, GstGLContext * context,
522 GstPadDirection direction, GstCaps * caps)
523 {
524 struct DmabufUpload *dmabuf = impl;
525 GstCapsFeatures *passthrough;
526 GstCaps *ret;
527
528 if (context) {
529 const GstGLFuncs *gl = context->gl_vtable;
530
531 if (!gl->EGLImageTargetTexture2D)
532 return NULL;
533
534 /* Don't propose DMABuf caps feature unless it can be supported */
535 if (gst_gl_context_get_gl_platform (context) != GST_GL_PLATFORM_EGL)
536 return NULL;
537
538 if (!gst_gl_context_check_feature (context, "EGL_KHR_image_base"))
539 return NULL;
540 }
541
542 passthrough = gst_caps_features_from_string
543 (GST_CAPS_FEATURE_META_GST_VIDEO_OVERLAY_COMPOSITION);
544
545 if (direction == GST_PAD_SINK) {
546 GstCaps *tmp;
547
548 ret =
549 _set_caps_features_with_passthrough (caps,
550 GST_CAPS_FEATURE_MEMORY_GL_MEMORY, passthrough);
551
552 tmp = _caps_intersect_texture_target (ret, 1 << GST_GL_TEXTURE_TARGET_2D);
553 gst_caps_unref (ret);
554 ret = tmp;
555 } else {
556 gint i, n;
557 GstCaps *tmp;
558
559 ret =
560 _set_caps_features_with_passthrough (caps,
561 GST_CAPS_FEATURE_MEMORY_DMABUF, passthrough);
562 tmp =
563 _set_caps_features_with_passthrough (caps,
564 GST_CAPS_FEATURE_MEMORY_SYSTEM_MEMORY, passthrough);
565 gst_caps_append (ret, tmp);
566
567 n = gst_caps_get_size (ret);
568 for (i = 0; i < n; i++) {
569 GstStructure *s = gst_caps_get_structure (ret, i);
570
571 gst_structure_remove_fields (s, "texture-target", NULL);
572 }
573 }
574
575 gst_caps_features_free (passthrough);
576
577 GST_DEBUG_OBJECT (dmabuf->upload, "transformed %" GST_PTR_FORMAT " into %"
578 GST_PTR_FORMAT, caps, ret);
579
580 return ret;
581 }
582
583 static GQuark
_eglimage_quark(gint plane)584 _eglimage_quark (gint plane)
585 {
586 static GQuark quark[5] = { 0 };
587 static const gchar *quark_str[] = {
588 "GstGLDMABufEGLImage0",
589 "GstGLDMABufEGLImage1",
590 "GstGLDMABufEGLImage2",
591 "GstGLDMABufEGLImage3",
592 "GstGLDMABufEGLImage",
593 };
594
595 if (!quark[plane])
596 quark[plane] = g_quark_from_static_string (quark_str[plane]);
597
598 return quark[plane];
599 }
600
601 static GstEGLImage *
_get_cached_eglimage(GstMemory * mem,gint plane)602 _get_cached_eglimage (GstMemory * mem, gint plane)
603 {
604 return gst_mini_object_get_qdata (GST_MINI_OBJECT (mem),
605 _eglimage_quark (plane));
606 }
607
608 static void
_set_cached_eglimage(GstMemory * mem,GstEGLImage * eglimage,gint plane)609 _set_cached_eglimage (GstMemory * mem, GstEGLImage * eglimage, gint plane)
610 {
611 return gst_mini_object_set_qdata (GST_MINI_OBJECT (mem),
612 _eglimage_quark (plane), eglimage, (GDestroyNotify) gst_egl_image_unref);
613 }
614
615 static gboolean
_dma_buf_upload_accept(gpointer impl,GstBuffer * buffer,GstCaps * in_caps,GstCaps * out_caps)616 _dma_buf_upload_accept (gpointer impl, GstBuffer * buffer, GstCaps * in_caps,
617 GstCaps * out_caps)
618 {
619 struct DmabufUpload *dmabuf = impl;
620 GstVideoInfo *in_info = &dmabuf->upload->priv->in_info;
621 GstVideoInfo *out_info = &dmabuf->out_info;
622 guint n_planes = GST_VIDEO_INFO_N_PLANES (in_info);
623 GstVideoMeta *meta;
624 guint n_mem;
625 GstMemory *mems[GST_VIDEO_MAX_PLANES];
626 gsize offset[GST_VIDEO_MAX_PLANES];
627 gint fd[GST_VIDEO_MAX_PLANES];
628 guint i;
629
630 n_mem = gst_buffer_n_memory (buffer);
631 meta = gst_buffer_get_video_meta (buffer);
632
633 if (!dmabuf->upload->context->gl_vtable->EGLImageTargetTexture2D)
634 return FALSE;
635
636 /* dmabuf upload is only supported with EGL contexts. */
637 if (gst_gl_context_get_gl_platform (dmabuf->upload->context) !=
638 GST_GL_PLATFORM_EGL)
639 return FALSE;
640
641 if (!gst_gl_context_check_feature (dmabuf->upload->context,
642 "EGL_KHR_image_base")) {
643 GST_DEBUG_OBJECT (dmabuf->upload, "no EGL_KHR_image_base extension");
644 return FALSE;
645 }
646
647 if (dmabuf->target == GST_GL_TEXTURE_TARGET_EXTERNAL_OES &&
648 !gst_gl_context_check_feature (dmabuf->upload->context,
649 "GL_OES_EGL_image_external")) {
650 GST_DEBUG_OBJECT (dmabuf->upload,
651 "no EGL_KHR_image_base_external extension");
652 return FALSE;
653 }
654
655 /* This will eliminate most non-dmabuf out there */
656 if (!gst_is_dmabuf_memory (gst_buffer_peek_memory (buffer, 0))) {
657 GST_DEBUG_OBJECT (dmabuf->upload, "input not dmabuf");
658 return FALSE;
659 }
660
661 /* We cannot have multiple dmabuf per plane */
662 if (n_mem > n_planes) {
663 GST_DEBUG_OBJECT (dmabuf->upload,
664 "number of memory (%u) != number of planes (%u)", n_mem, n_planes);
665 return FALSE;
666 }
667
668 /* Update video info based on video meta */
669 if (meta) {
670 in_info->width = meta->width;
671 in_info->height = meta->height;
672
673 for (i = 0; i < meta->n_planes; i++) {
674 in_info->offset[i] = meta->offset[i];
675 in_info->stride[i] = meta->stride[i];
676 }
677 }
678
679 if (out_caps != dmabuf->out_caps) {
680 dmabuf->out_caps = out_caps;
681 if (!gst_video_info_from_caps (out_info, out_caps))
682 return FALSE;
683 }
684
685 if (dmabuf->params)
686 gst_gl_allocation_params_free ((GstGLAllocationParams *) dmabuf->params);
687 if (!(dmabuf->params =
688 gst_gl_video_allocation_params_new_wrapped_gl_handle (dmabuf->
689 upload->context, NULL, out_info, -1, NULL, dmabuf->target, 0,
690 NULL, NULL, NULL)))
691 return FALSE;
692
693 /* Find and validate all memories */
694 for (i = 0; i < n_planes; i++) {
695 guint plane_size;
696 guint length;
697 guint mem_idx;
698 gsize mem_skip;
699
700 plane_size = gst_gl_get_plane_data_size (in_info, NULL, i);
701
702 if (!gst_buffer_find_memory (buffer, in_info->offset[i], plane_size,
703 &mem_idx, &length, &mem_skip)) {
704 GST_DEBUG_OBJECT (dmabuf->upload, "could not find memory %u", i);
705 return FALSE;
706 }
707
708 /* We can't have more then one dmabuf per plane */
709 if (length != 1) {
710 GST_DEBUG_OBJECT (dmabuf->upload, "data for plane %u spans %u memories",
711 i, length);
712 return FALSE;
713 }
714
715 mems[i] = gst_buffer_peek_memory (buffer, mem_idx);
716
717 /* And all memory found must be dmabuf */
718 if (!gst_is_dmabuf_memory (mems[i])) {
719 GST_DEBUG_OBJECT (dmabuf->upload, "memory %u is not dmabuf", i);
720 return FALSE;
721 }
722
723 offset[i] = mems[i]->offset + mem_skip;
724 fd[i] = gst_dmabuf_memory_get_fd (mems[i]);
725 }
726
727 if (dmabuf->direct) {
728 /* Check if this format is supported by the driver */
729 dmabuf->n_mem = 1;
730 if (!gst_egl_image_check_dmabuf_direct (dmabuf->upload->context, in_info,
731 dmabuf->target)) {
732 GST_DEBUG_OBJECT (dmabuf->upload, "direct check failed");
733 return FALSE;
734 }
735 } else
736 dmabuf->n_mem = n_planes;
737
738 /* Now create an EGLImage for each dmabufs */
739 for (i = 0; i < dmabuf->n_mem; i++) {
740 gint cache_id = dmabuf->direct ? 4 : i;
741
742 /* check if one is cached */
743 dmabuf->eglimage[i] = _get_cached_eglimage (mems[i], cache_id);
744 if (dmabuf->eglimage[i]) {
745 dmabuf->formats[i] = dmabuf->eglimage[i]->format;
746 continue;
747 }
748
749 /* otherwise create one and cache it */
750 if (dmabuf->direct)
751 dmabuf->eglimage[i] =
752 gst_egl_image_from_dmabuf_direct_target (dmabuf->upload->context, fd,
753 offset, in_info, dmabuf->target);
754 else
755 dmabuf->eglimage[i] = gst_egl_image_from_dmabuf (dmabuf->upload->context,
756 fd[i], in_info, i, offset[i]);
757
758 if (!dmabuf->eglimage[i]) {
759 GST_DEBUG_OBJECT (dmabuf->upload, "could not create eglimage");
760 return FALSE;
761 }
762
763 _set_cached_eglimage (mems[i], dmabuf->eglimage[i], cache_id);
764 dmabuf->formats[i] = dmabuf->eglimage[i]->format;
765 }
766
767 return TRUE;
768 }
769
770 static void
_dma_buf_upload_propose_allocation(gpointer impl,GstQuery * decide_query,GstQuery * query)771 _dma_buf_upload_propose_allocation (gpointer impl, GstQuery * decide_query,
772 GstQuery * query)
773 {
774 /* nothing to do for now. */
775 }
776
777 static void
_dma_buf_upload_perform_gl_thread(GstGLContext * context,struct DmabufUpload * dmabuf)778 _dma_buf_upload_perform_gl_thread (GstGLContext * context,
779 struct DmabufUpload *dmabuf)
780 {
781 GstGLMemoryAllocator *allocator;
782
783 allocator =
784 GST_GL_MEMORY_ALLOCATOR (gst_allocator_find
785 (GST_GL_MEMORY_EGL_ALLOCATOR_NAME));
786
787 /* FIXME: buffer pool */
788 dmabuf->outbuf = gst_buffer_new ();
789 gst_gl_memory_setup_buffer (allocator, dmabuf->outbuf, dmabuf->params,
790 dmabuf->formats, (gpointer *) dmabuf->eglimage, dmabuf->n_mem);
791 gst_object_unref (allocator);
792 }
793
794 static GstGLUploadReturn
_dma_buf_upload_perform(gpointer impl,GstBuffer * buffer,GstBuffer ** outbuf)795 _dma_buf_upload_perform (gpointer impl, GstBuffer * buffer, GstBuffer ** outbuf)
796 {
797 struct DmabufUpload *dmabuf = impl;
798
799 /* The direct path sets sinkpad caps to RGBA but this may be incorrect for
800 * the non-direct path, if that path fails to accept. In that case, we need
801 * to reconfigure.
802 */
803 if (!dmabuf->direct &&
804 GST_VIDEO_INFO_FORMAT (&dmabuf->upload->priv->in_info) !=
805 GST_VIDEO_INFO_FORMAT (&dmabuf->out_info))
806 return GST_GL_UPLOAD_RECONFIGURE;
807
808 gst_gl_context_thread_add (dmabuf->upload->context,
809 (GstGLContextThreadFunc) _dma_buf_upload_perform_gl_thread, dmabuf);
810
811 if (!dmabuf->outbuf)
812 return GST_GL_UPLOAD_ERROR;
813
814 gst_buffer_add_parent_buffer_meta (dmabuf->outbuf, buffer);
815
816 *outbuf = dmabuf->outbuf;
817 dmabuf->outbuf = NULL;
818
819 return GST_GL_UPLOAD_DONE;
820 }
821
822 static void
_dma_buf_upload_free(gpointer impl)823 _dma_buf_upload_free (gpointer impl)
824 {
825 struct DmabufUpload *dmabuf = impl;
826
827 if (dmabuf->params)
828 gst_gl_allocation_params_free ((GstGLAllocationParams *) dmabuf->params);
829
830 g_free (impl);
831 }
832
833 static const UploadMethod _dma_buf_upload = {
834 "Dmabuf",
835 0,
836 &_dma_buf_upload_caps,
837 &_dma_buf_upload_new,
838 &_dma_buf_upload_transform_caps,
839 &_dma_buf_upload_accept,
840 &_dma_buf_upload_propose_allocation,
841 &_dma_buf_upload_perform,
842 &_dma_buf_upload_free
843 };
844
845 /* a variant of the DMABuf uploader that relies on HW color conversion instead
846 * of shaders */
847
848 static gpointer
_direct_dma_buf_upload_new(GstGLUpload * upload)849 _direct_dma_buf_upload_new (GstGLUpload * upload)
850 {
851 struct DmabufUpload *dmabuf = _dma_buf_upload_new (upload);
852 dmabuf->direct = TRUE;
853 gst_video_info_init (&dmabuf->out_info);
854 return dmabuf;
855 }
856
857 static GstCaps *
_direct_dma_buf_upload_transform_caps(gpointer impl,GstGLContext * context,GstPadDirection direction,GstCaps * caps)858 _direct_dma_buf_upload_transform_caps (gpointer impl, GstGLContext * context,
859 GstPadDirection direction, GstCaps * caps)
860 {
861 struct DmabufUpload *dmabuf = impl;
862 GstCapsFeatures *passthrough;
863 GstCaps *ret;
864
865 if (context) {
866 const GstGLFuncs *gl = context->gl_vtable;
867
868 if (!gl->EGLImageTargetTexture2D)
869 return NULL;
870
871 /* Don't propose direct DMABuf caps feature unless it can be supported */
872 if (gst_gl_context_get_gl_platform (context) != GST_GL_PLATFORM_EGL)
873 return NULL;
874
875 if (dmabuf->target == GST_GL_TEXTURE_TARGET_EXTERNAL_OES &&
876 !gst_gl_context_check_feature (context, "GL_OES_EGL_image_external"))
877 return NULL;
878 }
879
880 passthrough = gst_caps_features_from_string
881 (GST_CAPS_FEATURE_META_GST_VIDEO_OVERLAY_COMPOSITION);
882
883 if (direction == GST_PAD_SINK) {
884 gint i, n;
885 GstCaps *tmp;
886 GstGLTextureTarget target_mask;
887
888 ret =
889 _set_caps_features_with_passthrough (caps,
890 GST_CAPS_FEATURE_MEMORY_GL_MEMORY, passthrough);
891
892 gst_caps_set_simple (ret, "format", G_TYPE_STRING, "RGBA", NULL);
893
894 n = gst_caps_get_size (ret);
895 for (i = 0; i < n; i++) {
896 GstStructure *s = gst_caps_get_structure (ret, i);
897
898 gst_structure_remove_fields (s, "chroma-site", NULL);
899 gst_structure_remove_fields (s, "colorimetry", NULL);
900 }
901
902 target_mask = 1 << dmabuf->target;
903 tmp = _caps_intersect_texture_target (ret, target_mask);
904 gst_caps_unref (ret);
905 ret = tmp;
906 } else {
907 gint i, n;
908 GstCaps *tmp;
909 GValue formats = G_VALUE_INIT;
910 gchar *format_str = g_strdup (GST_GL_MEMORY_VIDEO_FORMATS_STR);
911
912 ret =
913 _set_caps_features_with_passthrough (caps,
914 GST_CAPS_FEATURE_MEMORY_DMABUF, passthrough);
915 tmp =
916 _set_caps_features_with_passthrough (caps,
917 GST_CAPS_FEATURE_MEMORY_SYSTEM_MEMORY, passthrough);
918 gst_caps_append (ret, tmp);
919
920 g_value_init (&formats, GST_TYPE_LIST);
921 gst_value_deserialize (&formats, format_str);
922 tmp = gst_caps_copy (ret);
923 gst_caps_set_value (tmp, "format", &formats);
924 gst_caps_append (ret, tmp);
925 g_free (format_str);
926 g_value_unset (&formats);
927
928 n = gst_caps_get_size (ret);
929 for (i = 0; i < n; i++) {
930 GstStructure *s = gst_caps_get_structure (ret, i);
931
932 gst_structure_remove_fields (s, "texture-target", NULL);
933 }
934 }
935
936 gst_caps_features_free (passthrough);
937
938 GST_DEBUG_OBJECT (dmabuf->upload, "transformed %" GST_PTR_FORMAT " into %"
939 GST_PTR_FORMAT, caps, ret);
940
941 return ret;
942 }
943
944 static const UploadMethod _direct_dma_buf_upload = {
945 "DirectDmabuf",
946 0,
947 &_dma_buf_upload_caps,
948 &_direct_dma_buf_upload_new,
949 &_direct_dma_buf_upload_transform_caps,
950 &_dma_buf_upload_accept,
951 &_dma_buf_upload_propose_allocation,
952 &_dma_buf_upload_perform,
953 &_dma_buf_upload_free
954 };
955
956 /* a variant of the direct DMABuf uploader that uses external OES textures */
957
958 static gpointer
_direct_dma_buf_external_upload_new(GstGLUpload * upload)959 _direct_dma_buf_external_upload_new (GstGLUpload * upload)
960 {
961 struct DmabufUpload *dmabuf = _direct_dma_buf_upload_new (upload);
962 dmabuf->target = GST_GL_TEXTURE_TARGET_EXTERNAL_OES;
963 return dmabuf;
964 }
965
966 static const UploadMethod _direct_dma_buf_external_upload = {
967 "DirectDmabufExternal",
968 0,
969 &_dma_buf_upload_caps,
970 &_direct_dma_buf_external_upload_new,
971 &_direct_dma_buf_upload_transform_caps,
972 &_dma_buf_upload_accept,
973 &_dma_buf_upload_propose_allocation,
974 &_dma_buf_upload_perform,
975 &_dma_buf_upload_free
976 };
977
978 #endif /* GST_GL_HAVE_DMABUF */
979
980 struct GLUploadMeta
981 {
982 GstGLUpload *upload;
983
984 gboolean result;
985 GstVideoGLTextureUploadMeta *meta;
986 guint texture_ids[GST_GL_UPLOAD_MAX_PLANES];
987 GstBufferPool *pool;
988 };
989
990 static gpointer
_upload_meta_upload_new(GstGLUpload * upload)991 _upload_meta_upload_new (GstGLUpload * upload)
992 {
993 struct GLUploadMeta *meta = g_new0 (struct GLUploadMeta, 1);
994
995 meta->upload = upload;
996 meta->pool = NULL;
997
998 return meta;
999 }
1000
1001 static GstCaps *
_upload_meta_upload_transform_caps(gpointer impl,GstGLContext * context,GstPadDirection direction,GstCaps * caps)1002 _upload_meta_upload_transform_caps (gpointer impl, GstGLContext * context,
1003 GstPadDirection direction, GstCaps * caps)
1004 {
1005 GstCapsFeatures *passthrough =
1006 gst_caps_features_from_string
1007 (GST_CAPS_FEATURE_META_GST_VIDEO_OVERLAY_COMPOSITION);
1008 GstCaps *ret;
1009
1010 if (direction == GST_PAD_SINK) {
1011 GstCaps *tmp;
1012
1013 ret =
1014 _set_caps_features_with_passthrough (caps,
1015 GST_CAPS_FEATURE_MEMORY_GL_MEMORY, passthrough);
1016
1017 tmp = _caps_intersect_texture_target (ret, 1 << GST_GL_TEXTURE_TARGET_2D);
1018 gst_caps_unref (ret);
1019 ret = tmp;
1020 } else {
1021 gint i, n;
1022
1023 ret =
1024 _set_caps_features_with_passthrough (caps,
1025 GST_CAPS_FEATURE_META_GST_VIDEO_GL_TEXTURE_UPLOAD_META, passthrough);
1026 gst_caps_set_simple (ret, "format", G_TYPE_STRING, "RGBA", NULL);
1027
1028 n = gst_caps_get_size (ret);
1029 for (i = 0; i < n; i++) {
1030 GstStructure *s = gst_caps_get_structure (ret, i);
1031
1032 gst_structure_remove_fields (s, "texture-target", NULL);
1033 }
1034 }
1035
1036 gst_caps_features_free (passthrough);
1037
1038 return ret;
1039 }
1040
1041 static gboolean
_upload_meta_upload_accept(gpointer impl,GstBuffer * buffer,GstCaps * in_caps,GstCaps * out_caps)1042 _upload_meta_upload_accept (gpointer impl, GstBuffer * buffer,
1043 GstCaps * in_caps, GstCaps * out_caps)
1044 {
1045 struct GLUploadMeta *upload = impl;
1046 GstCapsFeatures *features;
1047 GstVideoGLTextureUploadMeta *meta;
1048 gboolean ret = TRUE;
1049 GstStructure *config;
1050 gsize size;
1051
1052 features = gst_caps_get_features (in_caps, 0);
1053
1054 if (!gst_caps_features_contains (features,
1055 GST_CAPS_FEATURE_META_GST_VIDEO_GL_TEXTURE_UPLOAD_META))
1056 ret = FALSE;
1057
1058 features = gst_caps_get_features (out_caps, 0);
1059 if (!gst_caps_features_contains (features, GST_CAPS_FEATURE_MEMORY_GL_MEMORY))
1060 ret = FALSE;
1061
1062 if (!ret)
1063 return ret;
1064
1065 if (upload->pool == NULL)
1066 upload->pool = gst_gl_buffer_pool_new (upload->upload->context);
1067
1068 if (!gst_buffer_pool_is_active (upload->pool)) {
1069 config = gst_buffer_pool_get_config (upload->pool);
1070
1071 size = upload->upload->priv->in_info.size;
1072 gst_buffer_pool_config_set_params (config, in_caps, size, 0, 0);
1073
1074 if (!gst_buffer_pool_set_config (upload->pool, config)) {
1075 GST_WARNING_OBJECT (upload->upload, "failed to set bufferpool config");
1076 return FALSE;
1077 }
1078 gst_buffer_pool_set_active (upload->pool, TRUE);
1079 }
1080
1081 if (buffer) {
1082 if ((meta = gst_buffer_get_video_gl_texture_upload_meta (buffer)) == NULL)
1083 return FALSE;
1084
1085 if (meta->texture_type[0] != GST_VIDEO_GL_TEXTURE_TYPE_RGBA) {
1086 GST_FIXME_OBJECT (upload, "only single rgba texture supported");
1087 return FALSE;
1088 }
1089
1090 if (meta->texture_orientation !=
1091 GST_VIDEO_GL_TEXTURE_ORIENTATION_X_NORMAL_Y_NORMAL) {
1092 GST_FIXME_OBJECT (upload, "only x-normal, y-normal textures supported");
1093 return FALSE;
1094 }
1095 }
1096
1097 return TRUE;
1098 }
1099
1100 static void
_upload_meta_upload_propose_allocation(gpointer impl,GstQuery * decide_query,GstQuery * query)1101 _upload_meta_upload_propose_allocation (gpointer impl, GstQuery * decide_query,
1102 GstQuery * query)
1103 {
1104 struct GLUploadMeta *upload = impl;
1105 GstStructure *gl_context;
1106 gchar *platform, *gl_apis;
1107 gpointer handle;
1108
1109 gl_apis =
1110 gst_gl_api_to_string (gst_gl_context_get_gl_api (upload->upload->
1111 context));
1112 platform =
1113 gst_gl_platform_to_string (gst_gl_context_get_gl_platform (upload->
1114 upload->context));
1115 handle = (gpointer) gst_gl_context_get_gl_context (upload->upload->context);
1116
1117 gl_context =
1118 gst_structure_new ("GstVideoGLTextureUploadMeta", "gst.gl.GstGLContext",
1119 GST_TYPE_GL_CONTEXT, upload->upload->context, "gst.gl.context.handle",
1120 G_TYPE_POINTER, handle, "gst.gl.context.type", G_TYPE_STRING, platform,
1121 "gst.gl.context.apis", G_TYPE_STRING, gl_apis, NULL);
1122 gst_query_add_allocation_meta (query,
1123 GST_VIDEO_GL_TEXTURE_UPLOAD_META_API_TYPE, gl_context);
1124
1125 g_free (gl_apis);
1126 g_free (platform);
1127 gst_structure_free (gl_context);
1128 }
1129
1130 /*
1131 * Uploads using gst_video_gl_texture_upload_meta_upload().
1132 * i.e. consumer of GstVideoGLTextureUploadMeta
1133 */
1134 static void
_do_upload_with_meta(GstGLContext * context,struct GLUploadMeta * upload)1135 _do_upload_with_meta (GstGLContext * context, struct GLUploadMeta *upload)
1136 {
1137 if (!gst_video_gl_texture_upload_meta_upload (upload->meta,
1138 upload->texture_ids)) {
1139 upload->result = FALSE;
1140 return;
1141 }
1142
1143 upload->result = TRUE;
1144 }
1145
1146 static GstGLUploadReturn
_upload_meta_upload_perform(gpointer impl,GstBuffer * buffer,GstBuffer ** outbuf)1147 _upload_meta_upload_perform (gpointer impl, GstBuffer * buffer,
1148 GstBuffer ** outbuf)
1149 {
1150 struct GLUploadMeta *upload = impl;
1151 int i;
1152 GstVideoInfo *in_info = &upload->upload->priv->in_info;
1153 guint max_planes = GST_VIDEO_INFO_N_PLANES (in_info);
1154
1155 /* Support stereo views for separated multiview mode */
1156 if (GST_VIDEO_INFO_MULTIVIEW_MODE (in_info) ==
1157 GST_VIDEO_MULTIVIEW_MODE_SEPARATED)
1158 max_planes *= GST_VIDEO_INFO_VIEWS (in_info);
1159
1160 GST_LOG_OBJECT (upload, "Attempting upload with GstVideoGLTextureUploadMeta");
1161
1162 upload->meta = gst_buffer_get_video_gl_texture_upload_meta (buffer);
1163
1164 if (gst_buffer_pool_acquire_buffer (upload->pool, outbuf,
1165 NULL) != GST_FLOW_OK) {
1166 GST_WARNING_OBJECT (upload, "failed to acquire buffer from bufferpool");
1167 return GST_GL_UPLOAD_ERROR;
1168 }
1169
1170 for (i = 0; i < GST_GL_UPLOAD_MAX_PLANES; i++) {
1171 guint tex_id = 0;
1172
1173 if (i < max_planes) {
1174 GstMemory *mem = gst_buffer_peek_memory (*outbuf, i);
1175 tex_id = ((GstGLMemory *) mem)->tex_id;
1176 }
1177
1178 upload->texture_ids[i] = tex_id;
1179 }
1180
1181 GST_LOG ("Uploading with GLTextureUploadMeta with textures "
1182 "%i,%i,%i,%i / %i,%i,%i,%i",
1183 upload->texture_ids[0], upload->texture_ids[1],
1184 upload->texture_ids[2], upload->texture_ids[3],
1185 upload->texture_ids[4], upload->texture_ids[5],
1186 upload->texture_ids[6], upload->texture_ids[7]);
1187
1188 gst_gl_context_thread_add (upload->upload->context,
1189 (GstGLContextThreadFunc) _do_upload_with_meta, upload);
1190
1191 if (!upload->result)
1192 return GST_GL_UPLOAD_ERROR;
1193
1194 return GST_GL_UPLOAD_DONE;
1195 }
1196
1197 static void
_upload_meta_upload_free(gpointer impl)1198 _upload_meta_upload_free (gpointer impl)
1199 {
1200 struct GLUploadMeta *upload = impl;
1201
1202 g_return_if_fail (impl != NULL);
1203
1204 if (upload->pool)
1205 gst_object_unref (upload->pool);
1206
1207 g_free (upload);
1208 }
1209
1210 static GstStaticCaps _upload_meta_upload_caps =
1211 GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE_WITH_FEATURES
1212 (GST_CAPS_FEATURE_META_GST_VIDEO_GL_TEXTURE_UPLOAD_META, "RGBA"));
1213
1214 static const UploadMethod _upload_meta_upload = {
1215 "UploadMeta",
1216 METHOD_FLAG_CAN_SHARE_CONTEXT,
1217 &_upload_meta_upload_caps,
1218 &_upload_meta_upload_new,
1219 &_upload_meta_upload_transform_caps,
1220 &_upload_meta_upload_accept,
1221 &_upload_meta_upload_propose_allocation,
1222 &_upload_meta_upload_perform,
1223 &_upload_meta_upload_free
1224 };
1225
1226 struct RawUploadFrame
1227 {
1228 gint ref_count;
1229 GstVideoFrame frame;
1230 };
1231
1232 struct RawUpload
1233 {
1234 GstGLUpload *upload;
1235 struct RawUploadFrame *in_frame;
1236 GstGLVideoAllocationParams *params;
1237 };
1238
1239 static struct RawUploadFrame *
_raw_upload_frame_new(struct RawUpload * raw,GstBuffer * buffer)1240 _raw_upload_frame_new (struct RawUpload *raw, GstBuffer * buffer)
1241 {
1242 struct RawUploadFrame *frame;
1243 GstVideoInfo *info;
1244 gint i;
1245
1246 if (!buffer)
1247 return NULL;
1248
1249 frame = g_slice_new (struct RawUploadFrame);
1250 frame->ref_count = 1;
1251
1252 if (!gst_video_frame_map (&frame->frame, &raw->upload->priv->in_info,
1253 buffer, GST_MAP_READ)) {
1254 g_slice_free (struct RawUploadFrame, frame);
1255 return NULL;
1256 }
1257
1258 raw->upload->priv->in_info = frame->frame.info;
1259 info = &raw->upload->priv->in_info;
1260
1261 /* Recalculate the offsets (and size) */
1262 info->size = 0;
1263 for (i = 0; i < GST_VIDEO_INFO_N_PLANES (info); i++) {
1264 info->offset[i] = info->size;
1265 info->size += gst_gl_get_plane_data_size (info, NULL, i);
1266 }
1267
1268 return frame;
1269 }
1270
1271 static void
_raw_upload_frame_ref(struct RawUploadFrame * frame)1272 _raw_upload_frame_ref (struct RawUploadFrame *frame)
1273 {
1274 g_atomic_int_inc (&frame->ref_count);
1275 }
1276
1277 static void
_raw_upload_frame_unref(struct RawUploadFrame * frame)1278 _raw_upload_frame_unref (struct RawUploadFrame *frame)
1279 {
1280 if (g_atomic_int_dec_and_test (&frame->ref_count)) {
1281 gst_video_frame_unmap (&frame->frame);
1282 g_slice_free (struct RawUploadFrame, frame);
1283 }
1284 }
1285
1286 static gpointer
_raw_data_upload_new(GstGLUpload * upload)1287 _raw_data_upload_new (GstGLUpload * upload)
1288 {
1289 struct RawUpload *raw = g_new0 (struct RawUpload, 1);
1290
1291 raw->upload = upload;
1292
1293 return raw;
1294 }
1295
1296 static GstCaps *
_raw_data_upload_transform_caps(gpointer impl,GstGLContext * context,GstPadDirection direction,GstCaps * caps)1297 _raw_data_upload_transform_caps (gpointer impl, GstGLContext * context,
1298 GstPadDirection direction, GstCaps * caps)
1299 {
1300 GstCapsFeatures *passthrough =
1301 gst_caps_features_from_string
1302 (GST_CAPS_FEATURE_META_GST_VIDEO_OVERLAY_COMPOSITION);
1303 GstCaps *ret;
1304
1305 if (direction == GST_PAD_SINK) {
1306 GstGLTextureTarget target_mask = 0;
1307 GstCaps *tmp;
1308
1309 ret =
1310 _set_caps_features_with_passthrough (caps,
1311 GST_CAPS_FEATURE_MEMORY_GL_MEMORY, passthrough);
1312
1313 target_mask |= 1 << GST_GL_TEXTURE_TARGET_2D;
1314 target_mask |= 1 << GST_GL_TEXTURE_TARGET_RECTANGLE;
1315 tmp = _caps_intersect_texture_target (ret, target_mask);
1316 gst_caps_unref (ret);
1317 ret = tmp;
1318 } else {
1319 gint i, n;
1320
1321 ret =
1322 _set_caps_features_with_passthrough (caps,
1323 GST_CAPS_FEATURE_MEMORY_SYSTEM_MEMORY, passthrough);
1324
1325 n = gst_caps_get_size (ret);
1326 for (i = 0; i < n; i++) {
1327 GstStructure *s = gst_caps_get_structure (ret, i);
1328
1329 gst_structure_remove_fields (s, "texture-target", NULL);
1330 }
1331 }
1332
1333 gst_caps_features_free (passthrough);
1334
1335 return ret;
1336 }
1337
1338 static gboolean
_raw_data_upload_accept(gpointer impl,GstBuffer * buffer,GstCaps * in_caps,GstCaps * out_caps)1339 _raw_data_upload_accept (gpointer impl, GstBuffer * buffer, GstCaps * in_caps,
1340 GstCaps * out_caps)
1341 {
1342 struct RawUpload *raw = impl;
1343 GstCapsFeatures *features;
1344
1345 features = gst_caps_get_features (out_caps, 0);
1346 if (!gst_caps_features_contains (features, GST_CAPS_FEATURE_MEMORY_GL_MEMORY))
1347 return FALSE;
1348
1349 if (raw->in_frame)
1350 _raw_upload_frame_unref (raw->in_frame);
1351 raw->in_frame = _raw_upload_frame_new (raw, buffer);
1352
1353 if (raw->params)
1354 gst_gl_allocation_params_free ((GstGLAllocationParams *) raw->params);
1355 if (!(raw->params =
1356 gst_gl_video_allocation_params_new_wrapped_data (raw->upload->context,
1357 NULL, &raw->upload->priv->in_info, -1, NULL,
1358 GST_GL_TEXTURE_TARGET_2D, 0, NULL, raw->in_frame,
1359 (GDestroyNotify) _raw_upload_frame_unref)))
1360 return FALSE;
1361
1362 return (raw->in_frame != NULL);
1363 }
1364
1365 static void
_raw_data_upload_propose_allocation(gpointer impl,GstQuery * decide_query,GstQuery * query)1366 _raw_data_upload_propose_allocation (gpointer impl, GstQuery * decide_query,
1367 GstQuery * query)
1368 {
1369 gst_query_add_allocation_meta (query, GST_VIDEO_META_API_TYPE, 0);
1370 }
1371
1372 static GstGLUploadReturn
_raw_data_upload_perform(gpointer impl,GstBuffer * buffer,GstBuffer ** outbuf)1373 _raw_data_upload_perform (gpointer impl, GstBuffer * buffer,
1374 GstBuffer ** outbuf)
1375 {
1376 GstGLBaseMemoryAllocator *allocator;
1377 struct RawUpload *raw = impl;
1378 int i;
1379 GstVideoInfo *in_info = &raw->upload->priv->in_info;
1380 guint n_mem = GST_VIDEO_INFO_N_PLANES (in_info);
1381
1382 allocator =
1383 GST_GL_BASE_MEMORY_ALLOCATOR (gst_gl_memory_allocator_get_default
1384 (raw->upload->context));
1385
1386 /* FIXME Use a buffer pool to cache the generated textures */
1387 *outbuf = gst_buffer_new ();
1388 raw->params->parent.context = raw->upload->context;
1389 if (gst_gl_memory_setup_buffer ((GstGLMemoryAllocator *) allocator, *outbuf,
1390 raw->params, NULL, raw->in_frame->frame.data, n_mem)) {
1391
1392 for (i = 0; i < n_mem; i++)
1393 _raw_upload_frame_ref (raw->in_frame);
1394 gst_buffer_add_gl_sync_meta (raw->upload->context, *outbuf);
1395 } else {
1396 GST_ERROR_OBJECT (raw->upload, "Failed to allocate wrapped texture");
1397 gst_buffer_unref (*outbuf);
1398 gst_object_unref (allocator);
1399 return GST_GL_UPLOAD_ERROR;
1400 }
1401 gst_object_unref (allocator);
1402 _raw_upload_frame_unref (raw->in_frame);
1403 raw->in_frame = NULL;
1404
1405 return GST_GL_UPLOAD_DONE;
1406 }
1407
1408 static void
_raw_data_upload_free(gpointer impl)1409 _raw_data_upload_free (gpointer impl)
1410 {
1411 struct RawUpload *raw = impl;
1412
1413 if (raw->params)
1414 gst_gl_allocation_params_free ((GstGLAllocationParams *) raw->params);
1415
1416 g_free (raw);
1417 }
1418
1419 static GstStaticCaps _raw_data_upload_caps =
1420 GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE (GST_GL_MEMORY_VIDEO_FORMATS_STR));
1421
1422 static const UploadMethod _raw_data_upload = {
1423 "Raw Data",
1424 0,
1425 &_raw_data_upload_caps,
1426 &_raw_data_upload_new,
1427 &_raw_data_upload_transform_caps,
1428 &_raw_data_upload_accept,
1429 &_raw_data_upload_propose_allocation,
1430 &_raw_data_upload_perform,
1431 &_raw_data_upload_free
1432 };
1433
1434 #if GST_GL_HAVE_VIV_DIRECTVIV
1435 #ifndef GL_BGRA_EXT
1436 #define GL_BGRA_EXT 0x80E1
1437 #endif
1438 #ifndef GL_VIV_YV12
1439 #define GL_VIV_YV12 0x8FC0
1440 #endif
1441 #ifndef GL_VIV_NV12
1442 #define GL_VIV_NV12 0x8FC1
1443 #endif
1444 #ifndef GL_VIV_YUY2
1445 #define GL_VIV_YUY2 0x8FC2
1446 #endif
1447 #ifndef GL_VIV_UYVY
1448 #define GL_VIV_UYVY 0x8FC3
1449 #endif
1450 #ifndef GL_VIV_NV21
1451 #define GL_VIV_NV21 0x8FC4
1452 #endif
1453 #ifndef GL_VIV_I420
1454 #define GL_VIV_I420 0x8FC5
1455 #endif
1456
1457 struct DirectVIVUpload
1458 {
1459 GstGLUpload *upload;
1460
1461 GstGLVideoAllocationParams *params;
1462 GstBuffer *inbuf, *outbuf;
1463 void (*TexDirectVIVMap) (GLenum Target, GLsizei Width, GLsizei Height,
1464 GLenum Format, GLvoid ** Logical, const GLuint * Physical);
1465 void (*TexDirectInvalidateVIV) (GLenum Target);
1466 gboolean loaded_functions;
1467 };
1468
1469 #define GST_GL_DIRECTVIV_FORMAT "{RGBA, I420, YV12, NV12, NV21, YUY2, UYVY, BGRA, RGB16}"
1470
1471 static GstStaticCaps _directviv_upload_caps =
1472 GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE (GST_GL_DIRECTVIV_FORMAT));
1473
1474 static gpointer
_directviv_upload_new(GstGLUpload * upload)1475 _directviv_upload_new (GstGLUpload * upload)
1476 {
1477 struct DirectVIVUpload *directviv = g_new0 (struct DirectVIVUpload, 1);
1478 directviv->upload = upload;
1479 directviv->loaded_functions = FALSE;
1480
1481 return directviv;
1482 }
1483
1484 static GstCaps *
_directviv_upload_transform_caps(gpointer impl,GstGLContext * context,GstPadDirection direction,GstCaps * caps)1485 _directviv_upload_transform_caps (gpointer impl, GstGLContext * context,
1486 GstPadDirection direction, GstCaps * caps)
1487 {
1488 GstCapsFeatures *passthrough =
1489 gst_caps_features_from_string
1490 (GST_CAPS_FEATURE_META_GST_VIDEO_OVERLAY_COMPOSITION);
1491 GstCaps *ret;
1492
1493 if (direction == GST_PAD_SINK) {
1494 GstCaps *tmp;
1495
1496 ret =
1497 _set_caps_features_with_passthrough (caps,
1498 GST_CAPS_FEATURE_MEMORY_GL_MEMORY, passthrough);
1499
1500 gst_caps_set_simple (ret, "format", G_TYPE_STRING, "RGBA", NULL);
1501 tmp = _caps_intersect_texture_target (ret, 1 << GST_GL_TEXTURE_TARGET_2D);
1502 gst_caps_unref (ret);
1503 ret = tmp;
1504 } else {
1505 GstCaps *tmp;
1506 tmp = gst_caps_from_string (GST_VIDEO_CAPS_MAKE_WITH_FEATURES
1507 (GST_CAPS_FEATURE_MEMORY_SYSTEM_MEMORY, GST_GL_DIRECTVIV_FORMAT));
1508 ret =
1509 _set_caps_features_with_passthrough (tmp,
1510 GST_CAPS_FEATURE_MEMORY_SYSTEM_MEMORY, passthrough);
1511 gst_caps_unref (tmp);
1512 }
1513
1514 gst_caps_features_free (passthrough);
1515 return ret;
1516 }
1517
1518
1519 static void
_directviv_upload_load_functions_gl_thread(GstGLContext * context,struct DirectVIVUpload * directviv)1520 _directviv_upload_load_functions_gl_thread (GstGLContext * context,
1521 struct DirectVIVUpload *directviv)
1522 {
1523 directviv->TexDirectVIVMap =
1524 gst_gl_context_get_proc_address (context, "glTexDirectVIVMap");
1525 directviv->TexDirectInvalidateVIV =
1526 gst_gl_context_get_proc_address (context, "glTexDirectInvalidateVIV");
1527 }
1528
1529 static gboolean
_directviv_upload_accept(gpointer impl,GstBuffer * buffer,GstCaps * in_caps,GstCaps * out_caps)1530 _directviv_upload_accept (gpointer impl, GstBuffer * buffer, GstCaps * in_caps,
1531 GstCaps * out_caps)
1532 {
1533 struct DirectVIVUpload *directviv = impl;
1534 GstCapsFeatures *features;
1535 guint n_mem;
1536 GstMemory *mem;
1537
1538 if (!directviv->loaded_functions && (!directviv->TexDirectInvalidateVIV ||
1539 !directviv->TexDirectVIVMap)) {
1540 gst_gl_context_thread_add (directviv->upload->context,
1541 (GstGLContextThreadFunc) _directviv_upload_load_functions_gl_thread,
1542 directviv);
1543 directviv->loaded_functions = TRUE;
1544 }
1545 if (!directviv->TexDirectInvalidateVIV || !directviv->TexDirectVIVMap)
1546 return FALSE;
1547
1548 features = gst_caps_get_features (out_caps, 0);
1549 if (!gst_caps_features_contains (features, GST_CAPS_FEATURE_MEMORY_GL_MEMORY))
1550 return FALSE;
1551
1552 if (directviv->params)
1553 gst_gl_allocation_params_free ((GstGLAllocationParams *) directviv->params);
1554 if (!(directviv->params =
1555 gst_gl_video_allocation_params_new (directviv->upload->context, NULL,
1556 &directviv->upload->priv->out_info, -1, NULL,
1557 GST_GL_TEXTURE_TARGET_2D, GST_VIDEO_GL_TEXTURE_TYPE_RGBA)))
1558 return FALSE;
1559
1560 /* We only support a single memory per buffer at this point */
1561 n_mem = gst_buffer_n_memory (buffer);
1562 if (n_mem == 1) {
1563 mem = gst_buffer_peek_memory (buffer, 0);
1564 } else {
1565 mem = NULL;
1566 }
1567
1568 return n_mem == 1 && mem && gst_is_phys_memory (mem);
1569 }
1570
1571 static void
_directviv_upload_propose_allocation(gpointer impl,GstQuery * decide_query,GstQuery * query)1572 _directviv_upload_propose_allocation (gpointer impl, GstQuery * decide_query,
1573 GstQuery * query)
1574 {
1575 }
1576
1577 static GLenum
_directviv_upload_video_format_to_gl_format(GstVideoFormat format)1578 _directviv_upload_video_format_to_gl_format (GstVideoFormat format)
1579 {
1580 switch (format) {
1581 case GST_VIDEO_FORMAT_I420:
1582 return GL_VIV_I420;
1583 case GST_VIDEO_FORMAT_YV12:
1584 return GL_VIV_YV12;
1585 case GST_VIDEO_FORMAT_NV12:
1586 return GL_VIV_NV12;
1587 case GST_VIDEO_FORMAT_NV21:
1588 return GL_VIV_NV21;
1589 case GST_VIDEO_FORMAT_YUY2:
1590 return GL_VIV_YUY2;
1591 case GST_VIDEO_FORMAT_UYVY:
1592 return GL_VIV_UYVY;
1593 case GST_VIDEO_FORMAT_RGB16:
1594 return GL_RGB565;
1595 case GST_VIDEO_FORMAT_RGBA:
1596 return GL_RGBA;
1597 case GST_VIDEO_FORMAT_BGRA:
1598 return GL_BGRA_EXT;
1599 case GST_VIDEO_FORMAT_RGBx:
1600 return GL_RGBA;
1601 case GST_VIDEO_FORMAT_BGRx:
1602 return GL_BGRA_EXT;
1603 default:
1604 return 0;
1605 }
1606 }
1607
1608 typedef struct
1609 {
1610 GstBuffer *buffer;
1611 GstMemory *memory;
1612 GstMapInfo map;
1613 guintptr phys_addr;
1614 } DirectVIVUnmapData;
1615
1616 static void
_directviv_memory_unmap(DirectVIVUnmapData * data)1617 _directviv_memory_unmap (DirectVIVUnmapData * data)
1618 {
1619 gst_memory_unmap (data->memory, &data->map);
1620 gst_memory_unref (data->memory);
1621 gst_buffer_unref (data->buffer);
1622 g_free (data);
1623 }
1624
1625 static void
_directviv_upload_perform_gl_thread(GstGLContext * context,struct DirectVIVUpload * directviv)1626 _directviv_upload_perform_gl_thread (GstGLContext * context,
1627 struct DirectVIVUpload *directviv)
1628 {
1629 static GQuark directviv_unmap_quark = 0;
1630 GstGLMemoryAllocator *allocator;
1631 GstMemory *in_mem;
1632 GstGLMemory *out_gl_mem;
1633 GstVideoInfo *in_info;
1634 DirectVIVUnmapData *unmap_data;
1635 GstVideoMeta *vmeta;
1636 gint width, height, gl_format;
1637 const GstGLFuncs *gl;
1638
1639 if (!directviv_unmap_quark)
1640 directviv_unmap_quark = g_quark_from_static_string ("GstGLDirectVIVUnmap");
1641
1642 gl = context->gl_vtable;
1643
1644 g_assert (gst_buffer_n_memory (directviv->inbuf) == 1);
1645 in_info = &directviv->upload->priv->in_info;
1646 in_mem = gst_buffer_peek_memory (directviv->inbuf, 0);
1647 unmap_data = g_new0 (DirectVIVUnmapData, 1);
1648 if (!gst_memory_map (in_mem, &unmap_data->map, GST_MAP_READ)) {
1649 g_free (unmap_data);
1650 return;
1651 }
1652 unmap_data->phys_addr = gst_phys_memory_get_phys_addr (in_mem);
1653 if (!unmap_data->phys_addr) {
1654 gst_memory_unmap (in_mem, &unmap_data->map);
1655 g_free (unmap_data);
1656 return;
1657 }
1658 unmap_data->memory = gst_memory_ref (in_mem);
1659 unmap_data->buffer = gst_buffer_ref (directviv->inbuf);
1660
1661 allocator =
1662 GST_GL_MEMORY_ALLOCATOR (gst_allocator_find
1663 (GST_GL_MEMORY_PBO_ALLOCATOR_NAME));
1664
1665 /* FIXME: buffer pool */
1666 directviv->outbuf = gst_buffer_new ();
1667 gst_gl_memory_setup_buffer (allocator, directviv->outbuf, directviv->params,
1668 NULL, NULL, 0);
1669 gst_object_unref (allocator);
1670
1671 out_gl_mem = (GstGLMemory *) gst_buffer_peek_memory (directviv->outbuf, 0);
1672
1673 /* Need to keep the input memory and buffer mapped and valid until
1674 * the GL memory is not used anymore */
1675 gst_mini_object_set_qdata ((GstMiniObject *) out_gl_mem,
1676 directviv_unmap_quark, unmap_data,
1677 (GDestroyNotify) _directviv_memory_unmap);
1678 gst_buffer_add_parent_buffer_meta (directviv->outbuf, directviv->inbuf);
1679
1680 /* width/height need to compensate for stride/padding */
1681 vmeta = gst_buffer_get_video_meta (directviv->inbuf);
1682 if (vmeta) {
1683 width = vmeta->stride[0];
1684 if (GST_VIDEO_INFO_N_PLANES (in_info) == 1)
1685 height = gst_memory_get_sizes (in_mem, NULL, NULL) / width;
1686 else
1687 height = vmeta->offset[1] / width;
1688 } else {
1689 width = GST_VIDEO_INFO_PLANE_STRIDE (in_info, 0);
1690 if (GST_VIDEO_INFO_N_PLANES (in_info) == 1)
1691 height = gst_memory_get_sizes (in_mem, NULL, NULL) / width;
1692 else
1693 height = GST_VIDEO_INFO_PLANE_OFFSET (in_info, 1) / width;
1694 }
1695 width /= GST_VIDEO_INFO_COMP_PSTRIDE (in_info, 0);
1696
1697 gl_format =
1698 _directviv_upload_video_format_to_gl_format (GST_VIDEO_INFO_FORMAT
1699 (in_info));
1700
1701 gl->BindTexture (GL_TEXTURE_2D, out_gl_mem->tex_id);
1702 directviv->TexDirectVIVMap (GL_TEXTURE_2D, width, height,
1703 gl_format, (void **) &unmap_data->map.data, &unmap_data->phys_addr);
1704 directviv->TexDirectInvalidateVIV (GL_TEXTURE_2D);
1705 }
1706
1707 static GstGLUploadReturn
_directviv_upload_perform(gpointer impl,GstBuffer * buffer,GstBuffer ** outbuf)1708 _directviv_upload_perform (gpointer impl, GstBuffer * buffer,
1709 GstBuffer ** outbuf)
1710 {
1711 struct DirectVIVUpload *directviv = impl;
1712
1713 directviv->inbuf = buffer;
1714 directviv->outbuf = NULL;
1715 gst_gl_context_thread_add (directviv->upload->context,
1716 (GstGLContextThreadFunc) _directviv_upload_perform_gl_thread, directviv);
1717 directviv->inbuf = NULL;
1718
1719 if (!directviv->outbuf)
1720 return GST_GL_UPLOAD_ERROR;
1721
1722 *outbuf = directviv->outbuf;
1723 directviv->outbuf = NULL;
1724
1725 return GST_GL_UPLOAD_DONE;
1726 }
1727
1728 static void
_directviv_upload_free(gpointer impl)1729 _directviv_upload_free (gpointer impl)
1730 {
1731 struct DirectVIVUpload *directviv = impl;
1732
1733 if (directviv->params)
1734 gst_gl_allocation_params_free ((GstGLAllocationParams *) directviv->params);
1735
1736 g_free (impl);
1737 }
1738
1739 static const UploadMethod _directviv_upload = {
1740 "DirectVIV",
1741 0,
1742 &_directviv_upload_caps,
1743 &_directviv_upload_new,
1744 &_directviv_upload_transform_caps,
1745 &_directviv_upload_accept,
1746 &_directviv_upload_propose_allocation,
1747 &_directviv_upload_perform,
1748 &_directviv_upload_free
1749 };
1750
1751 #endif /* GST_GL_HAVE_VIV_DIRECTVIV */
1752
1753 #if defined(HAVE_NVMM)
1754 #include "nvbuf_utils.h"
1755
1756 struct NVMMUpload
1757 {
1758 GstGLUpload *upload;
1759
1760 GstGLVideoAllocationParams *params;
1761 guint n_mem;
1762
1763 GstGLTextureTarget target;
1764 GstVideoInfo out_info;
1765 /* only used for pointer comparison */
1766 gpointer out_caps;
1767 };
1768
1769 #define GST_CAPS_FEATURE_MEMORY_NVMM "memory:NVMM"
1770
1771 /* FIXME: other formats? */
1772 static GstStaticCaps _nvmm_upload_caps =
1773 GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE_WITH_FEATURES
1774 (GST_CAPS_FEATURE_MEMORY_NVMM,
1775 "RGBA"));
1776
1777 static gpointer
_nvmm_upload_new(GstGLUpload * upload)1778 _nvmm_upload_new (GstGLUpload * upload)
1779 {
1780 struct NVMMUpload *nvmm = g_new0 (struct NVMMUpload, 1);
1781 nvmm->upload = upload;
1782 nvmm->target = GST_GL_TEXTURE_TARGET_EXTERNAL_OES;
1783 return nvmm;
1784 }
1785
1786 static GstCaps *
_nvmm_upload_transform_caps(gpointer impl,GstGLContext * context,GstPadDirection direction,GstCaps * caps)1787 _nvmm_upload_transform_caps (gpointer impl, GstGLContext * context,
1788 GstPadDirection direction, GstCaps * caps)
1789 {
1790 struct NVMMUpload *nvmm = impl;
1791 GstCapsFeatures *passthrough;
1792 GstCaps *ret;
1793
1794 if (context) {
1795 const GstGLFuncs *gl = context->gl_vtable;
1796
1797 if (!gl->EGLImageTargetTexture2D)
1798 return NULL;
1799
1800 /* Don't propose NVMM caps feature unless it can be supported */
1801 if (gst_gl_context_get_gl_platform (context) != GST_GL_PLATFORM_EGL)
1802 return NULL;
1803
1804 if (!gst_gl_context_check_feature (context, "EGL_KHR_image_base"))
1805 return NULL;
1806 }
1807
1808 passthrough = gst_caps_features_from_string
1809 (GST_CAPS_FEATURE_META_GST_VIDEO_OVERLAY_COMPOSITION);
1810
1811 if (direction == GST_PAD_SINK) {
1812 GstCaps *tmp;
1813
1814 ret =
1815 _set_caps_features_with_passthrough (caps,
1816 GST_CAPS_FEATURE_MEMORY_GL_MEMORY, passthrough);
1817
1818 tmp =
1819 _caps_intersect_texture_target (ret,
1820 1 << GST_GL_TEXTURE_TARGET_EXTERNAL_OES);
1821 gst_caps_unref (ret);
1822 ret = tmp;
1823 } else {
1824 gint i, n;
1825
1826 ret =
1827 _set_caps_features_with_passthrough (caps,
1828 GST_CAPS_FEATURE_MEMORY_NVMM, passthrough);
1829
1830 n = gst_caps_get_size (ret);
1831 for (i = 0; i < n; i++) {
1832 GstStructure *s = gst_caps_get_structure (ret, i);
1833
1834 gst_structure_remove_fields (s, "texture-target", NULL);
1835 }
1836 }
1837
1838 gst_caps_features_free (passthrough);
1839
1840 GST_DEBUG_OBJECT (nvmm->upload, "transformed %" GST_PTR_FORMAT " into %"
1841 GST_PTR_FORMAT, caps, ret);
1842
1843 return ret;
1844 }
1845
1846 static gboolean
_nvmm_upload_accept(gpointer impl,GstBuffer * buffer,GstCaps * in_caps,GstCaps * out_caps)1847 _nvmm_upload_accept (gpointer impl, GstBuffer * buffer, GstCaps * in_caps,
1848 GstCaps * out_caps)
1849 {
1850 struct NVMMUpload *nvmm = impl;
1851 GstVideoInfo *in_info = &nvmm->upload->priv->in_info;
1852 GstVideoInfo *out_info = &nvmm->out_info;
1853 GstVideoMeta *meta;
1854 GstMapInfo in_map_info = GST_MAP_INFO_INIT;
1855 guint n_mem;
1856 guint i;
1857
1858 n_mem = gst_buffer_n_memory (buffer);
1859 if (n_mem != 1) {
1860 GST_DEBUG_OBJECT (nvmm->upload, "NVMM uploader only supports "
1861 "1 memory, not %u", n_mem);
1862 return FALSE;
1863 }
1864
1865 meta = gst_buffer_get_video_meta (buffer);
1866
1867 if (!nvmm->upload->context->gl_vtable->EGLImageTargetTexture2D)
1868 return FALSE;
1869
1870 /* NVMM upload is only supported with EGL contexts. */
1871 if (gst_gl_context_get_gl_platform (nvmm->upload->context) !=
1872 GST_GL_PLATFORM_EGL)
1873 return FALSE;
1874
1875 if (!gst_gl_context_check_feature (nvmm->upload->context,
1876 "EGL_KHR_image_base"))
1877 return FALSE;
1878
1879 if (!gst_buffer_map (buffer, &in_map_info, GST_MAP_READ)) {
1880 GST_DEBUG_OBJECT (nvmm->upload, "Failed to map readonly NvBuffer");
1881 return FALSE;
1882 }
1883 if (in_map_info.size != NvBufferGetSize ()) {
1884 GST_DEBUG_OBJECT (nvmm->upload, "Memory size (%" G_GSIZE_FORMAT ") is "
1885 "not the same as what NvBuffer advertises (%u)", in_map_info.size,
1886 NvBufferGetSize ());
1887 gst_buffer_unmap (buffer, &in_map_info);
1888 return FALSE;
1889 }
1890 gst_buffer_unmap (buffer, &in_map_info);
1891
1892 /* Update video info based on video meta */
1893 if (meta) {
1894 in_info->width = meta->width;
1895 in_info->height = meta->height;
1896
1897 for (i = 0; i < meta->n_planes; i++) {
1898 in_info->offset[i] = meta->offset[i];
1899 in_info->stride[i] = meta->stride[i];
1900 }
1901 }
1902
1903 if (out_caps != nvmm->out_caps) {
1904 nvmm->out_caps = out_caps;
1905 if (!gst_video_info_from_caps (out_info, out_caps))
1906 return FALSE;
1907 }
1908
1909 if (nvmm->params)
1910 gst_gl_allocation_params_free ((GstGLAllocationParams *) nvmm->params);
1911 if (!(nvmm->params =
1912 gst_gl_video_allocation_params_new_wrapped_gl_handle (nvmm->
1913 upload->context, NULL, out_info, -1, NULL, nvmm->target, 0, NULL,
1914 NULL, NULL))) {
1915 return FALSE;
1916 }
1917
1918 return TRUE;
1919 }
1920
1921 static void
_nvmm_upload_propose_allocation(gpointer impl,GstQuery * decide_query,GstQuery * query)1922 _nvmm_upload_propose_allocation (gpointer impl, GstQuery * decide_query,
1923 GstQuery * query)
1924 {
1925 /* nothing to do for now. */
1926 }
1927
1928 static void
_egl_image_mem_unref(GstEGLImage * image,GstMemory * mem)1929 _egl_image_mem_unref (GstEGLImage * image, GstMemory * mem)
1930 {
1931 GstGLDisplayEGL *egl_display = NULL;
1932 EGLDisplay display;
1933
1934 egl_display = gst_gl_display_egl_from_gl_display (image->context->display);
1935 if (!egl_display) {
1936 GST_ERROR ("Could not retrieve GstGLDisplayEGL from GstGLDisplay");
1937 return;
1938 }
1939 display =
1940 (EGLDisplay) gst_gl_display_get_handle (GST_GL_DISPLAY (egl_display));
1941
1942 if (NvDestroyEGLImage (display, image->image)) {
1943 GST_ERROR ("Failed to destroy EGLImage %p from NvBuffer", image->image);
1944 } else {
1945 GST_DEBUG ("destroyed EGLImage %p from NvBuffer", image->image);
1946 }
1947
1948 gst_memory_unref (mem);
1949 gst_object_unref (egl_display);
1950 }
1951
1952 static const char *
payload_type_to_string(NvBufferPayloadType ptype)1953 payload_type_to_string (NvBufferPayloadType ptype)
1954 {
1955 switch (ptype) {
1956 case NvBufferPayload_SurfArray:
1957 return "SurfArray";
1958 case NvBufferPayload_MemHandle:
1959 return "MemHandle";
1960 default:
1961 return "<unknown>";
1962 }
1963 }
1964
1965 static const char *
pixel_format_to_string(NvBufferColorFormat fmt)1966 pixel_format_to_string (NvBufferColorFormat fmt)
1967 {
1968 switch (fmt) {
1969 case NvBufferColorFormat_YUV420:
1970 return "YUV420";
1971 case NvBufferColorFormat_YVU420:
1972 return "YVU420";
1973 case NvBufferColorFormat_YUV422:
1974 return "YUV422";
1975 case NvBufferColorFormat_YUV420_ER:
1976 return "YUV420_ER";
1977 case NvBufferColorFormat_YVU420_ER:
1978 return "YVU420_ER";
1979 case NvBufferColorFormat_NV12:
1980 return "NV12";
1981 case NvBufferColorFormat_NV12_ER:
1982 return "NV12_ER";
1983 case NvBufferColorFormat_NV21:
1984 return "NV21";
1985 case NvBufferColorFormat_NV21_ER:
1986 return "NV21_ER";
1987 case NvBufferColorFormat_UYVY:
1988 return "UYVY";
1989 case NvBufferColorFormat_UYVY_ER:
1990 return "UYVY_ER";
1991 case NvBufferColorFormat_VYUY:
1992 return "VYUY";
1993 case NvBufferColorFormat_VYUY_ER:
1994 return "VYUY_ER";
1995 case NvBufferColorFormat_YUYV:
1996 return "YUYV";
1997 case NvBufferColorFormat_YUYV_ER:
1998 return "YUYV_ER";
1999 case NvBufferColorFormat_YVYU:
2000 return "YVYU";
2001 case NvBufferColorFormat_YVYU_ER:
2002 return "YVYU_ER";
2003 case NvBufferColorFormat_ABGR32:
2004 return "ABGR32";
2005 case NvBufferColorFormat_XRGB32:
2006 return "XRGB32";
2007 case NvBufferColorFormat_ARGB32:
2008 return "ARGB32";
2009 case NvBufferColorFormat_NV12_10LE:
2010 return "NV12_10LE";
2011 case NvBufferColorFormat_NV12_10LE_709:
2012 return "NV12_10LE_709";
2013 case NvBufferColorFormat_NV12_10LE_709_ER:
2014 return "NV12_10LE_709_ER";
2015 case NvBufferColorFormat_NV12_10LE_2020:
2016 return "NV12_2020";
2017 case NvBufferColorFormat_NV21_10LE:
2018 return "NV21_10LE";
2019 case NvBufferColorFormat_NV12_12LE:
2020 return "NV12_12LE";
2021 case NvBufferColorFormat_NV12_12LE_2020:
2022 return "NV12_12LE_2020";
2023 case NvBufferColorFormat_NV21_12LE:
2024 return "NV21_12LE";
2025 case NvBufferColorFormat_YUV420_709:
2026 return "YUV420_709";
2027 case NvBufferColorFormat_YUV420_709_ER:
2028 return "YUV420_709_ER";
2029 case NvBufferColorFormat_NV12_709:
2030 return "NV12_709";
2031 case NvBufferColorFormat_NV12_709_ER:
2032 return "NV12_709_ER";
2033 case NvBufferColorFormat_YUV420_2020:
2034 return "YUV420_2020";
2035 case NvBufferColorFormat_NV12_2020:
2036 return "NV12_2020";
2037 case NvBufferColorFormat_SignedR16G16:
2038 return "SignedR16G16";
2039 case NvBufferColorFormat_A32:
2040 return "A32";
2041 case NvBufferColorFormat_YUV444:
2042 return "YUV444";
2043 case NvBufferColorFormat_GRAY8:
2044 return "GRAY8";
2045 case NvBufferColorFormat_NV16:
2046 return "NV16";
2047 case NvBufferColorFormat_NV16_10LE:
2048 return "NV16_10LE";
2049 case NvBufferColorFormat_NV24:
2050 return "NV24";
2051 case NvBufferColorFormat_NV16_ER:
2052 return "NV16_ER";
2053 case NvBufferColorFormat_NV24_ER:
2054 return "NV24_ER";
2055 case NvBufferColorFormat_NV16_709:
2056 return "NV16_709";
2057 case NvBufferColorFormat_NV24_709:
2058 return "NV24_709";
2059 case NvBufferColorFormat_NV16_709_ER:
2060 return "NV16_709_ER";
2061 case NvBufferColorFormat_NV24_709_ER:
2062 return "NV24_709_ER";
2063 case NvBufferColorFormat_NV24_10LE_709:
2064 return "NV24_10LE_709";
2065 case NvBufferColorFormat_NV24_10LE_709_ER:
2066 return "NV24_10LE_709_ER";
2067 case NvBufferColorFormat_NV24_10LE_2020:
2068 return "NV24_10LE_2020";
2069 case NvBufferColorFormat_NV24_12LE_2020:
2070 return "NV24_12LE_2020";
2071 case NvBufferColorFormat_RGBA_10_10_10_2_709:
2072 return "RGBA_10_10_10_2_709";
2073 case NvBufferColorFormat_RGBA_10_10_10_2_2020:
2074 return "RGBA_10_10_10_2_2020";
2075 case NvBufferColorFormat_BGRA_10_10_10_2_709:
2076 return "BGRA_10_10_10_2_709";
2077 case NvBufferColorFormat_BGRA_10_10_10_2_2020:
2078 return "BGRA_10_10_10_2_2020";
2079 case NvBufferColorFormat_Invalid:
2080 return "Invalid";
2081 default:
2082 return "<unknown>";
2083 }
2084 }
2085
2086 static void
dump_nv_buf_params(GstObject * debug_object,NvBufferParamsEx * params)2087 dump_nv_buf_params (GstObject * debug_object, NvBufferParamsEx * params)
2088 {
2089 GST_DEBUG_OBJECT (debug_object, "nvbuffer fd: %u size %i nv_buffer: %p of "
2090 "size %u, payload: (0x%x) %s, pixel format: (0x%x) %s, n_planes: %u, "
2091 "plane 0 { wxh: %ux%u, pitch: %u, offset: %u, psize: %u, layout: %u } "
2092 "plane 1 { wxh: %ux%u, pitch: %u, offset: %u, psize: %u, layout: %u } "
2093 "plane 2 { wxh: %ux%u, pitch: %u, offset: %u, psize: %u, layout: %u }",
2094 params->params.dmabuf_fd, params->params.memsize,
2095 params->params.nv_buffer, params->params.nv_buffer_size,
2096 params->params.payloadType,
2097 payload_type_to_string (params->params.payloadType),
2098 params->params.pixel_format,
2099 pixel_format_to_string (params->params.pixel_format),
2100 params->params.num_planes, params->params.width[0],
2101 params->params.height[0], params->params.pitch[0],
2102 params->params.offset[0], params->params.psize[0],
2103 params->params.offset[0], params->params.width[1],
2104 params->params.height[1], params->params.pitch[1],
2105 params->params.offset[1], params->params.psize[1],
2106 params->params.offset[1], params->params.width[2],
2107 params->params.height[2], params->params.pitch[2],
2108 params->params.offset[2], params->params.psize[2],
2109 params->params.offset[2]);
2110 }
2111
2112 static GstGLUploadReturn
_nvmm_upload_perform(gpointer impl,GstBuffer * buffer,GstBuffer ** outbuf)2113 _nvmm_upload_perform (gpointer impl, GstBuffer * buffer, GstBuffer ** outbuf)
2114 {
2115 struct NVMMUpload *nvmm = impl;
2116 GstGLMemoryAllocator *allocator = NULL;
2117 GstMapInfo in_map_info = GST_MAP_INFO_INIT;
2118 GstGLDisplayEGL *egl_display = NULL;
2119 GstEGLImage *eglimage = NULL;
2120 EGLDisplay display = EGL_NO_DISPLAY;
2121 EGLImageKHR image = EGL_NO_IMAGE;
2122 int in_dmabuf_fd;
2123 NvBufferParamsEx params = { 0, };
2124 GstGLUploadReturn ret = GST_GL_UPLOAD_ERROR;
2125
2126 if (!gst_buffer_map (buffer, &in_map_info, GST_MAP_READ)) {
2127 GST_DEBUG_OBJECT (nvmm->upload, "Failed to map readonly NvBuffer");
2128 goto done;
2129 }
2130
2131 if (ExtractFdFromNvBuffer (in_map_info.data, &in_dmabuf_fd)) {
2132 GST_DEBUG_OBJECT (nvmm->upload, "Failed to extract fd from NvBuffer");
2133 goto done;
2134 }
2135 if (NvBufferGetParamsEx (in_dmabuf_fd, ¶ms)) {
2136 GST_WARNING_OBJECT (nvmm->upload, "Failed to get NvBuffer params");
2137 goto done;
2138 }
2139 dump_nv_buf_params ((GstObject *) nvmm->upload, ¶ms);
2140
2141 egl_display =
2142 gst_gl_display_egl_from_gl_display (nvmm->upload->context->display);
2143 if (!egl_display) {
2144 GST_WARNING ("Failed to retrieve GstGLDisplayEGL from GstGLDisplay");
2145 goto done;
2146 }
2147 display =
2148 (EGLDisplay) gst_gl_display_get_handle (GST_GL_DISPLAY (egl_display));
2149
2150 image = NvEGLImageFromFd (display, in_dmabuf_fd);
2151 if (!image) {
2152 GST_DEBUG_OBJECT (nvmm->upload, "Failed construct EGLImage "
2153 "from NvBuffer fd %i", in_dmabuf_fd);
2154 goto done;
2155 }
2156 GST_DEBUG_OBJECT (nvmm->upload, "constructed EGLImage %p "
2157 "from NvBuffer fd %i", image, in_dmabuf_fd);
2158
2159 eglimage = gst_egl_image_new_wrapped (nvmm->upload->context, image,
2160 GST_GL_RGBA, gst_memory_ref (in_map_info.memory),
2161 (GstEGLImageDestroyNotify) _egl_image_mem_unref);
2162 if (!eglimage) {
2163 GST_WARNING_OBJECT (nvmm->upload, "Failed to wrap constructed "
2164 "EGLImage from NvBuffer");
2165 goto done;
2166 }
2167
2168 gst_buffer_unmap (buffer, &in_map_info);
2169 in_map_info = (GstMapInfo) GST_MAP_INFO_INIT;
2170
2171 allocator =
2172 GST_GL_MEMORY_ALLOCATOR (gst_allocator_find
2173 (GST_GL_MEMORY_EGL_ALLOCATOR_NAME));
2174
2175 /* TODO: buffer pool */
2176 *outbuf = gst_buffer_new ();
2177 if (!gst_gl_memory_setup_buffer (allocator, *outbuf, nvmm->params,
2178 NULL, (gpointer *) & eglimage, 1)) {
2179 GST_WARNING_OBJECT (nvmm->upload, "Failed to setup "
2180 "NVMM -> EGLImage buffer");
2181 goto done;
2182 }
2183 gst_egl_image_unref (eglimage);
2184
2185 gst_buffer_add_parent_buffer_meta (*outbuf, buffer);
2186
2187 /* TODO: NvBuffer has some sync functions that may be more useful here */
2188 {
2189 GstGLSyncMeta *sync_meta;
2190
2191 sync_meta = gst_buffer_add_gl_sync_meta (nvmm->upload->context, *outbuf);
2192 if (sync_meta) {
2193 gst_gl_sync_meta_set_sync_point (sync_meta, nvmm->upload->context);
2194 }
2195 }
2196
2197 ret = GST_GL_UPLOAD_DONE;
2198
2199 done:
2200 if (in_map_info.memory)
2201 gst_buffer_unmap (buffer, &in_map_info);
2202
2203 gst_clear_object (&egl_display);
2204 gst_clear_object (&allocator);
2205
2206 return ret;
2207 }
2208
2209 static void
_nvmm_upload_free(gpointer impl)2210 _nvmm_upload_free (gpointer impl)
2211 {
2212 struct NVMMUpload *nvmm = impl;
2213
2214 if (nvmm->params)
2215 gst_gl_allocation_params_free ((GstGLAllocationParams *) nvmm->params);
2216
2217 g_free (impl);
2218 }
2219
2220 static const UploadMethod _nvmm_upload = {
2221 "NVMM",
2222 0,
2223 &_nvmm_upload_caps,
2224 &_nvmm_upload_new,
2225 &_nvmm_upload_transform_caps,
2226 &_nvmm_upload_accept,
2227 &_nvmm_upload_propose_allocation,
2228 &_nvmm_upload_perform,
2229 &_nvmm_upload_free
2230 };
2231
2232 #endif /* HAVE_NVMM */
2233
2234 static const UploadMethod *upload_methods[] = { &_gl_memory_upload,
2235 #if GST_GL_HAVE_DMABUF
2236 &_direct_dma_buf_upload,
2237 &_direct_dma_buf_external_upload,
2238 &_dma_buf_upload,
2239 #endif
2240 #if GST_GL_HAVE_VIV_DIRECTVIV
2241 &_directviv_upload,
2242 #endif
2243 #if defined(HAVE_NVMM)
2244 &_nvmm_upload,
2245 #endif /* HAVE_NVMM */
2246 &_upload_meta_upload, &_raw_data_upload
2247 };
2248
2249 static GMutex upload_global_lock;
2250
2251 GstCaps *
gst_gl_upload_get_input_template_caps(void)2252 gst_gl_upload_get_input_template_caps (void)
2253 {
2254 GstCaps *ret = NULL;
2255 gint i;
2256
2257 g_mutex_lock (&upload_global_lock);
2258
2259 /* FIXME: cache this and invalidate on changes to upload_methods */
2260 for (i = 0; i < G_N_ELEMENTS (upload_methods); i++) {
2261 GstCaps *template =
2262 gst_static_caps_get (upload_methods[i]->input_template_caps);
2263 ret = ret == NULL ? template : gst_caps_merge (ret, template);
2264 }
2265
2266 ret = gst_caps_simplify (ret);
2267 ret = gst_gl_overlay_compositor_add_caps (ret);
2268 g_mutex_unlock (&upload_global_lock);
2269
2270 return ret;
2271 }
2272
2273 static void
gst_gl_upload_class_init(GstGLUploadClass * klass)2274 gst_gl_upload_class_init (GstGLUploadClass * klass)
2275 {
2276 G_OBJECT_CLASS (klass)->finalize = gst_gl_upload_finalize;
2277 }
2278
2279 static void
gst_gl_upload_init(GstGLUpload * upload)2280 gst_gl_upload_init (GstGLUpload * upload)
2281 {
2282 upload->priv = gst_gl_upload_get_instance_private (upload);
2283 }
2284
2285 /**
2286 * gst_gl_upload_new:
2287 * @context: a #GstGLContext
2288 *
2289 * Returns: (transfer full): a new #GstGLUpload object
2290 */
2291 GstGLUpload *
gst_gl_upload_new(GstGLContext * context)2292 gst_gl_upload_new (GstGLContext * context)
2293 {
2294 GstGLUpload *upload = g_object_new (GST_TYPE_GL_UPLOAD, NULL);
2295 gint i, n;
2296
2297 gst_object_ref_sink (upload);
2298
2299 if (context)
2300 gst_gl_upload_set_context (upload, context);
2301 else
2302 upload->context = NULL;
2303
2304 n = G_N_ELEMENTS (upload_methods);
2305 upload->priv->upload_impl = g_malloc (sizeof (gpointer) * n);
2306 for (i = 0; i < n; i++) {
2307 upload->priv->upload_impl[i] = upload_methods[i]->new (upload);
2308 }
2309
2310 GST_DEBUG_OBJECT (upload, "Created new GLUpload for context %" GST_PTR_FORMAT,
2311 context);
2312
2313 return upload;
2314 }
2315
2316 void
gst_gl_upload_set_context(GstGLUpload * upload,GstGLContext * context)2317 gst_gl_upload_set_context (GstGLUpload * upload, GstGLContext * context)
2318 {
2319 g_return_if_fail (upload != NULL);
2320
2321 gst_object_replace ((GstObject **) & upload->context, (GstObject *) context);
2322 }
2323
2324 static void
gst_gl_upload_finalize(GObject * object)2325 gst_gl_upload_finalize (GObject * object)
2326 {
2327 GstGLUpload *upload;
2328 gint i, n;
2329
2330 upload = GST_GL_UPLOAD (object);
2331
2332 upload->priv->method_i = 0;
2333
2334 if (upload->context) {
2335 gst_object_unref (upload->context);
2336 upload->context = NULL;
2337 }
2338
2339 if (upload->priv->in_caps) {
2340 gst_caps_unref (upload->priv->in_caps);
2341 upload->priv->in_caps = NULL;
2342 }
2343
2344 if (upload->priv->out_caps) {
2345 gst_caps_unref (upload->priv->out_caps);
2346 upload->priv->out_caps = NULL;
2347 }
2348
2349 n = G_N_ELEMENTS (upload_methods);
2350 for (i = 0; i < n; i++) {
2351 if (upload->priv->upload_impl[i])
2352 upload_methods[i]->free (upload->priv->upload_impl[i]);
2353 }
2354 g_free (upload->priv->upload_impl);
2355
2356 G_OBJECT_CLASS (gst_gl_upload_parent_class)->finalize (object);
2357 }
2358
2359 GstCaps *
gst_gl_upload_transform_caps(GstGLUpload * upload,GstGLContext * context,GstPadDirection direction,GstCaps * caps,GstCaps * filter)2360 gst_gl_upload_transform_caps (GstGLUpload * upload, GstGLContext * context,
2361 GstPadDirection direction, GstCaps * caps, GstCaps * filter)
2362 {
2363 GstCaps *result, *tmp;
2364 gint i;
2365
2366 if (upload->priv->method) {
2367 tmp = upload->priv->method->transform_caps (upload->priv->method_impl,
2368 context, direction, caps);
2369 if (tmp) {
2370 if (filter) {
2371 result =
2372 gst_caps_intersect_full (filter, tmp, GST_CAPS_INTERSECT_FIRST);
2373 gst_caps_unref (tmp);
2374 } else {
2375 result = tmp;
2376 }
2377 if (!gst_caps_is_empty (result))
2378 return result;
2379 else
2380 gst_caps_unref (result);
2381 }
2382 }
2383
2384 tmp = gst_caps_new_empty ();
2385
2386 for (i = 0; i < G_N_ELEMENTS (upload_methods); i++) {
2387 GstCaps *tmp2;
2388
2389 tmp2 =
2390 upload_methods[i]->transform_caps (upload->priv->upload_impl[i],
2391 context, direction, caps);
2392
2393 if (tmp2)
2394 tmp = gst_caps_merge (tmp, tmp2);
2395 }
2396
2397 if (filter) {
2398 result = gst_caps_intersect_full (filter, tmp, GST_CAPS_INTERSECT_FIRST);
2399 gst_caps_unref (tmp);
2400 } else {
2401 result = tmp;
2402 }
2403
2404 return result;
2405 }
2406
2407 /**
2408 * gst_gl_upload_propose_allocation:
2409 * @upload: a #GstGLUpload
2410 * @decide_query: (allow-none): a #GstQuery from a decide allocation
2411 * @query: the proposed allocation query
2412 *
2413 * Adds the required allocation parameters to support uploading.
2414 */
2415 void
gst_gl_upload_propose_allocation(GstGLUpload * upload,GstQuery * decide_query,GstQuery * query)2416 gst_gl_upload_propose_allocation (GstGLUpload * upload, GstQuery * decide_query,
2417 GstQuery * query)
2418 {
2419 gint i;
2420
2421 for (i = 0; i < G_N_ELEMENTS (upload_methods); i++)
2422 upload_methods[i]->propose_allocation (upload->priv->upload_impl[i],
2423 decide_query, query);
2424 }
2425
2426 static gboolean
_gst_gl_upload_set_caps_unlocked(GstGLUpload * upload,GstCaps * in_caps,GstCaps * out_caps)2427 _gst_gl_upload_set_caps_unlocked (GstGLUpload * upload, GstCaps * in_caps,
2428 GstCaps * out_caps)
2429 {
2430 g_return_val_if_fail (upload != NULL, FALSE);
2431 g_return_val_if_fail (gst_caps_is_fixed (in_caps), FALSE);
2432
2433 if (upload->priv->in_caps && upload->priv->out_caps
2434 && gst_caps_is_equal (upload->priv->in_caps, in_caps)
2435 && gst_caps_is_equal (upload->priv->out_caps, out_caps))
2436 return TRUE;
2437
2438 gst_caps_replace (&upload->priv->in_caps, in_caps);
2439 gst_caps_replace (&upload->priv->out_caps, out_caps);
2440
2441 gst_video_info_from_caps (&upload->priv->in_info, in_caps);
2442 gst_video_info_from_caps (&upload->priv->out_info, out_caps);
2443
2444 upload->priv->method = NULL;
2445 upload->priv->method_impl = NULL;
2446 upload->priv->method_i = 0;
2447
2448 return TRUE;
2449 }
2450
2451 /**
2452 * gst_gl_upload_set_caps:
2453 * @upload: a #GstGLUpload
2454 * @in_caps: input #GstCaps
2455 * @out_caps: output #GstCaps
2456 *
2457 * Initializes @upload with the information required for upload.
2458 *
2459 * Returns: whether @in_caps and @out_caps could be set on @upload
2460 */
2461 gboolean
gst_gl_upload_set_caps(GstGLUpload * upload,GstCaps * in_caps,GstCaps * out_caps)2462 gst_gl_upload_set_caps (GstGLUpload * upload, GstCaps * in_caps,
2463 GstCaps * out_caps)
2464 {
2465 gboolean ret;
2466
2467 GST_OBJECT_LOCK (upload);
2468 ret = _gst_gl_upload_set_caps_unlocked (upload, in_caps, out_caps);
2469 GST_OBJECT_UNLOCK (upload);
2470
2471 return ret;
2472 }
2473
2474 /**
2475 * gst_gl_upload_get_caps:
2476 * @upload: a #GstGLUpload
2477 * @in_caps: (transfer full) (allow-none) (out): the input #GstCaps
2478 * @out_caps: (transfer full) (allow-none) (out): the output #GstCaps
2479 */
2480 void
gst_gl_upload_get_caps(GstGLUpload * upload,GstCaps ** in_caps,GstCaps ** out_caps)2481 gst_gl_upload_get_caps (GstGLUpload * upload, GstCaps ** in_caps,
2482 GstCaps ** out_caps)
2483 {
2484 GST_OBJECT_LOCK (upload);
2485 if (in_caps)
2486 *in_caps =
2487 upload->priv->in_caps ? gst_caps_ref (upload->priv->in_caps) : NULL;
2488 if (out_caps)
2489 *out_caps =
2490 upload->priv->out_caps ? gst_caps_ref (upload->priv->out_caps) : NULL;
2491 GST_OBJECT_UNLOCK (upload);
2492 }
2493
2494 static gboolean
_upload_find_method(GstGLUpload * upload,gpointer last_impl)2495 _upload_find_method (GstGLUpload * upload, gpointer last_impl)
2496 {
2497 gint method_i;
2498
2499 /* start with the last used method after explicitly reconfiguring to
2500 * negotiate caps for this method */
2501 if (upload->priv->method_i == 0) {
2502 upload->priv->method_i = upload->priv->saved_method_i;
2503 upload->priv->saved_method_i = 0;
2504 }
2505
2506 if (upload->priv->method_i >= G_N_ELEMENTS (upload_methods)) {
2507 if (last_impl)
2508 upload->priv->method_i = 0;
2509 else
2510 return FALSE;
2511 }
2512
2513 method_i = upload->priv->method_i;
2514
2515 if (last_impl == upload->priv->upload_impl[method_i])
2516 return FALSE;
2517
2518 upload->priv->method = upload_methods[method_i];
2519 upload->priv->method_impl = upload->priv->upload_impl[method_i];
2520
2521 GST_DEBUG_OBJECT (upload, "attempting upload with uploader %s",
2522 upload->priv->method->name);
2523
2524 upload->priv->method_i++;
2525
2526 return TRUE;
2527 }
2528
2529 /**
2530 * gst_gl_upload_perform_with_buffer:
2531 * @upload: a #GstGLUpload
2532 * @buffer: input #GstBuffer
2533 * @outbuf_ptr: (out): resulting #GstBuffer
2534 *
2535 * Uploads @buffer using the transformation specified by
2536 * gst_gl_upload_set_caps() creating a new #GstBuffer in @outbuf_ptr.
2537 *
2538 * Returns: whether the upload was successful
2539 */
2540 GstGLUploadReturn
gst_gl_upload_perform_with_buffer(GstGLUpload * upload,GstBuffer * buffer,GstBuffer ** outbuf_ptr)2541 gst_gl_upload_perform_with_buffer (GstGLUpload * upload, GstBuffer * buffer,
2542 GstBuffer ** outbuf_ptr)
2543 {
2544 GstGLUploadReturn ret = GST_GL_UPLOAD_ERROR;
2545 GstBuffer *outbuf = NULL;
2546 gpointer last_impl = upload->priv->method_impl;
2547
2548 g_return_val_if_fail (GST_IS_GL_UPLOAD (upload), FALSE);
2549 g_return_val_if_fail (GST_IS_BUFFER (buffer), FALSE);
2550 g_return_val_if_fail (outbuf_ptr != NULL, FALSE);
2551
2552 GST_OBJECT_LOCK (upload);
2553
2554 #define NEXT_METHOD \
2555 do { \
2556 if (!_upload_find_method (upload, last_impl)) { \
2557 GST_OBJECT_UNLOCK (upload); \
2558 return FALSE; \
2559 } \
2560 goto restart; \
2561 } while (0)
2562
2563 if (!upload->priv->method_impl)
2564 _upload_find_method (upload, last_impl);
2565
2566 restart:
2567 if (!upload->priv->method->accept (upload->priv->method_impl, buffer,
2568 upload->priv->in_caps, upload->priv->out_caps))
2569 NEXT_METHOD;
2570
2571 ret =
2572 upload->priv->method->perform (upload->priv->method_impl, buffer,
2573 &outbuf);
2574 GST_LOG_OBJECT (upload, "uploader %s returned %u, buffer: %p",
2575 upload->priv->method->name, ret, outbuf);
2576 if (ret == GST_GL_UPLOAD_UNSHARED_GL_CONTEXT) {
2577 gint i;
2578
2579 for (i = 0; i < G_N_ELEMENTS (upload_methods); i++) {
2580 if (upload_methods[i] == &_raw_data_upload) {
2581 upload->priv->method = &_raw_data_upload;
2582 upload->priv->method_impl = upload->priv->upload_impl[i];
2583 upload->priv->method_i = i;
2584
2585 break;
2586 }
2587 }
2588
2589 gst_buffer_replace (&outbuf, NULL);
2590 goto restart;
2591 } else if (ret == GST_GL_UPLOAD_DONE || ret == GST_GL_UPLOAD_RECONFIGURE) {
2592 if (last_impl != upload->priv->method_impl) {
2593 GstCaps *caps = gst_gl_upload_transform_caps (upload, upload->context,
2594 GST_PAD_SINK, upload->priv->in_caps, NULL);
2595 if (!gst_caps_is_subset (caps, upload->priv->out_caps)) {
2596 gst_buffer_replace (&outbuf, NULL);
2597 ret = GST_GL_UPLOAD_RECONFIGURE;
2598 }
2599 gst_caps_unref (caps);
2600 }
2601 /* we are done */
2602 } else {
2603 upload->priv->method_impl = NULL;
2604 gst_buffer_replace (&outbuf, NULL);
2605 NEXT_METHOD;
2606 }
2607
2608 if (outbuf && buffer != outbuf)
2609 gst_buffer_copy_into (outbuf, buffer,
2610 GST_BUFFER_COPY_FLAGS | GST_BUFFER_COPY_TIMESTAMPS, 0, -1);
2611 *outbuf_ptr = outbuf;
2612
2613 if (ret == GST_GL_UPLOAD_RECONFIGURE)
2614 upload->priv->saved_method_i = upload->priv->method_i - 1;
2615
2616 GST_OBJECT_UNLOCK (upload);
2617
2618 return ret;
2619
2620 #undef NEXT_METHOD
2621 }
2622