1 /*
2 * GStreamer
3 * Copyright (C) 2012 Matthew Waters <ystree00@gmail.com>
4 *
5 * This library is free software; you can redistribute it and/or
6 * modify it under the terms of the GNU Library General Public
7 * License as published by the Free Software Foundation; either
8 * version 2 of the License, or (at your option) any later version.
9 *
10 * This library is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 * Library General Public License for more details.
14 *
15 * You should have received a copy of the GNU Library General Public
16 * License along with this library; if not, write to the
17 * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
18 * Boston, MA 02110-1301, USA.
19 */
20
21 #ifdef HAVE_CONFIG_H
22 #include "config.h"
23 #endif
24
25 #include <gst/gl/gl.h>
26 #if GST_GL_HAVE_PLATFORM_EGL && GST_GL_HAVE_DMABUF
27 #include <gst/gl/egl/gsteglimage.h>
28 #include <gst/allocators/gstdmabuf.h>
29 #endif
30
31 #include "gstglelements.h"
32 #include "gstgldownloadelement.h"
33
34 GST_DEBUG_CATEGORY_STATIC (gst_gl_download_element_debug);
35 #define GST_CAT_DEFAULT gst_gl_download_element_debug
36
37 #if GST_GL_HAVE_PLATFORM_EGL && defined(HAVE_NVMM)
38 #include <gst/gl/egl/gstgldisplay_egl.h>
39 #include <gst/gl/egl/gstglmemoryegl.h>
40
41 #include "nvbuf_utils.h"
42
43 static const char *
nv_buffer_payload_type_to_string(NvBufferPayloadType ptype)44 nv_buffer_payload_type_to_string (NvBufferPayloadType ptype)
45 {
46 switch (ptype) {
47 case NvBufferPayload_SurfArray:
48 return "SurfArray";
49 case NvBufferPayload_MemHandle:
50 return "MemHandle";
51 default:
52 return "<unknown>";
53 }
54 }
55
56 static const char *
nv_buffer_pixel_format_to_string(NvBufferColorFormat fmt)57 nv_buffer_pixel_format_to_string (NvBufferColorFormat fmt)
58 {
59 switch (fmt) {
60 case NvBufferColorFormat_YUV420:
61 return "YUV420";
62 case NvBufferColorFormat_YVU420:
63 return "YVU420";
64 case NvBufferColorFormat_YUV422:
65 return "YUV422";
66 case NvBufferColorFormat_YUV420_ER:
67 return "YUV420_ER";
68 case NvBufferColorFormat_YVU420_ER:
69 return "YVU420_ER";
70 case NvBufferColorFormat_NV12:
71 return "NV12";
72 case NvBufferColorFormat_NV12_ER:
73 return "NV12_ER";
74 case NvBufferColorFormat_NV21:
75 return "NV21";
76 case NvBufferColorFormat_NV21_ER:
77 return "NV21_ER";
78 case NvBufferColorFormat_UYVY:
79 return "UYVY";
80 case NvBufferColorFormat_UYVY_ER:
81 return "UYVY_ER";
82 case NvBufferColorFormat_VYUY:
83 return "VYUY";
84 case NvBufferColorFormat_VYUY_ER:
85 return "VYUY_ER";
86 case NvBufferColorFormat_YUYV:
87 return "YUYV";
88 case NvBufferColorFormat_YUYV_ER:
89 return "YUYV_ER";
90 case NvBufferColorFormat_YVYU:
91 return "YVYU";
92 case NvBufferColorFormat_YVYU_ER:
93 return "YVYU_ER";
94 case NvBufferColorFormat_ABGR32:
95 return "ABGR32";
96 case NvBufferColorFormat_XRGB32:
97 return "XRGB32";
98 case NvBufferColorFormat_ARGB32:
99 return "ARGB32";
100 case NvBufferColorFormat_NV12_10LE:
101 return "NV12_10LE";
102 case NvBufferColorFormat_NV12_10LE_709:
103 return "NV12_10LE_709";
104 case NvBufferColorFormat_NV12_10LE_709_ER:
105 return "NV12_10LE_709_ER";
106 case NvBufferColorFormat_NV12_10LE_2020:
107 return "NV12_2020";
108 case NvBufferColorFormat_NV21_10LE:
109 return "NV21_10LE";
110 case NvBufferColorFormat_NV12_12LE:
111 return "NV12_12LE";
112 case NvBufferColorFormat_NV12_12LE_2020:
113 return "NV12_12LE_2020";
114 case NvBufferColorFormat_NV21_12LE:
115 return "NV21_12LE";
116 case NvBufferColorFormat_YUV420_709:
117 return "YUV420_709";
118 case NvBufferColorFormat_YUV420_709_ER:
119 return "YUV420_709_ER";
120 case NvBufferColorFormat_NV12_709:
121 return "NV12_709";
122 case NvBufferColorFormat_NV12_709_ER:
123 return "NV12_709_ER";
124 case NvBufferColorFormat_YUV420_2020:
125 return "YUV420_2020";
126 case NvBufferColorFormat_NV12_2020:
127 return "NV12_2020";
128 case NvBufferColorFormat_SignedR16G16:
129 return "SignedR16G16";
130 case NvBufferColorFormat_A32:
131 return "A32";
132 case NvBufferColorFormat_YUV444:
133 return "YUV444";
134 case NvBufferColorFormat_GRAY8:
135 return "GRAY8";
136 case NvBufferColorFormat_NV16:
137 return "NV16";
138 case NvBufferColorFormat_NV16_10LE:
139 return "NV16_10LE";
140 case NvBufferColorFormat_NV24:
141 return "NV24";
142 case NvBufferColorFormat_NV16_ER:
143 return "NV16_ER";
144 case NvBufferColorFormat_NV24_ER:
145 return "NV24_ER";
146 case NvBufferColorFormat_NV16_709:
147 return "NV16_709";
148 case NvBufferColorFormat_NV24_709:
149 return "NV24_709";
150 case NvBufferColorFormat_NV16_709_ER:
151 return "NV16_709_ER";
152 case NvBufferColorFormat_NV24_709_ER:
153 return "NV24_709_ER";
154 case NvBufferColorFormat_NV24_10LE_709:
155 return "NV24_10LE_709";
156 case NvBufferColorFormat_NV24_10LE_709_ER:
157 return "NV24_10LE_709_ER";
158 case NvBufferColorFormat_NV24_10LE_2020:
159 return "NV24_10LE_2020";
160 case NvBufferColorFormat_NV24_12LE_2020:
161 return "NV24_12LE_2020";
162 case NvBufferColorFormat_RGBA_10_10_10_2_709:
163 return "RGBA_10_10_10_2_709";
164 case NvBufferColorFormat_RGBA_10_10_10_2_2020:
165 return "RGBA_10_10_10_2_2020";
166 case NvBufferColorFormat_BGRA_10_10_10_2_709:
167 return "BGRA_10_10_10_2_709";
168 case NvBufferColorFormat_BGRA_10_10_10_2_2020:
169 return "BGRA_10_10_10_2_2020";
170 case NvBufferColorFormat_Invalid:
171 return "Invalid";
172 default:
173 return "<unknown>";
174 }
175 }
176
177 static void
nv_buffer_dump_params(GstObject * debug_object,NvBufferParamsEx * params)178 nv_buffer_dump_params (GstObject * debug_object, NvBufferParamsEx * params)
179 {
180 GST_DEBUG_OBJECT (debug_object, "nvbuffer fd: %u size %i nv_buffer: %p of "
181 "size %u, payload: (0x%x) %s, pixel format: (0x%x) %s, n_planes: %u, "
182 "plane 0 { wxh: %ux%u, pitch: %u, offset: %u, psize: %u, layout: %u } "
183 "plane 1 { wxh: %ux%u, pitch: %u, offset: %u, psize: %u, layout: %u } "
184 "plane 2 { wxh: %ux%u, pitch: %u, offset: %u, psize: %u, layout: %u }",
185 params->params.dmabuf_fd, params->params.memsize,
186 params->params.nv_buffer, params->params.nv_buffer_size,
187 params->params.payloadType,
188 nv_buffer_payload_type_to_string (params->params.payloadType),
189 params->params.pixel_format,
190 nv_buffer_pixel_format_to_string (params->params.pixel_format),
191 params->params.num_planes, params->params.width[0],
192 params->params.height[0], params->params.pitch[0],
193 params->params.offset[0], params->params.psize[0],
194 params->params.offset[0], params->params.width[1],
195 params->params.height[1], params->params.pitch[1],
196 params->params.offset[1], params->params.psize[1],
197 params->params.offset[1], params->params.width[2],
198 params->params.height[2], params->params.pitch[2],
199 params->params.offset[2], params->params.psize[2],
200 params->params.offset[2]);
201 }
202
203 struct _GstMemoryNVMM
204 {
205 GstMemory parent;
206
207 int dmabuf_fd;
208 NvBufferParamsEx params;
209 };
210
211 typedef struct _GstMemoryNVMM GstMemoryNVMM;
212
213 struct _GstAllocatorNVMM
214 {
215 GstAllocator parent;
216 };
217
218 typedef struct _GstAllocatorNVMM GstAllocatorNVMM;
219
220 struct _GstAllocatorNVMMClass
221 {
222 GstAllocatorClass parent_class;
223 };
224
225 typedef struct _GstAllocatorNVMMClass GstAllocatorNVMMClass;
226
227 GType gst_allocator_nvmm_get_type (void);
228 G_DEFINE_TYPE (GstAllocatorNVMM, gst_allocator_nvmm, GST_TYPE_ALLOCATOR);
229
230 static gboolean
gst_memory_nvmm_init(GstMemoryNVMM * nvmm,GstMemoryFlags flags,GstAllocator * allocator,GstMemory * parent,const GstVideoInfo * vinfo)231 gst_memory_nvmm_init (GstMemoryNVMM * nvmm, GstMemoryFlags flags,
232 GstAllocator * allocator, GstMemory * parent, const GstVideoInfo * vinfo)
233 {
234 gsize size = NvBufferGetSize ();
235 NvBufferCreateParams create_params = {
236 .width = GST_VIDEO_INFO_WIDTH (vinfo),
237 .height = GST_VIDEO_INFO_HEIGHT (vinfo),
238 .payloadType = NvBufferPayload_SurfArray,
239 .memsize = GST_VIDEO_INFO_SIZE (vinfo),
240 .layout = NvBufferLayout_BlockLinear,
241 .colorFormat = NvBufferColorFormat_ABGR32,
242 .nvbuf_tag = NvBufferTag_NONE,
243 };
244
245 nvmm->dmabuf_fd = -1;
246
247 if (NvBufferCreateEx (&nvmm->dmabuf_fd, &create_params)) {
248 GST_WARNING_OBJECT (allocator, "Failed to create NvBuffer");
249 return FALSE;
250 }
251
252 if (NvBufferGetParamsEx (nvmm->dmabuf_fd, &nvmm->params)) {
253 GST_WARNING_OBJECT (allocator, "Failed to get NvBuffer params");
254 NvReleaseFd (nvmm->dmabuf_fd);
255 nvmm->dmabuf_fd = -1;
256 return FALSE;
257 }
258 nv_buffer_dump_params ((GstObject *) allocator, &nvmm->params);
259
260 gst_memory_init (&nvmm->parent, flags, allocator, parent, size, 0, 0, size);
261
262 return TRUE;
263 }
264
265 static gpointer
gst_memory_nvmm_map_full(GstMemory * mem,GstMapInfo * info,gsize size)266 gst_memory_nvmm_map_full (GstMemory * mem, GstMapInfo * info, gsize size)
267 {
268 GstMemoryNVMM *nvmm = (GstMemoryNVMM *) mem;
269
270 GST_TRACE ("%p fd:%i map", mem, nvmm->dmabuf_fd);
271
272 // This is what the Nvidia elements do so...
273 return nvmm->params.params.nv_buffer;
274 }
275
276 static void
gst_memory_nvmm_unmap_full(GstMemory * mem,GstMapInfo * info)277 gst_memory_nvmm_unmap_full (GstMemory * mem, GstMapInfo * info)
278 {
279 GstMemoryNVMM *nvmm = (GstMemoryNVMM *) mem;
280
281 GST_TRACE ("%p fd:%i unmap", mem, nvmm->dmabuf_fd);
282 }
283
284 static GstMemory *
gst_memory_nvmm_copy(GstMemory * mem,gssize offset,gssize size)285 gst_memory_nvmm_copy (GstMemory * mem, gssize offset, gssize size)
286 {
287 return NULL;
288 }
289
290 static GstMemory *
gst_memory_nvmm_share(GstMemory * mem,gssize offset,gssize size)291 gst_memory_nvmm_share (GstMemory * mem, gssize offset, gssize size)
292 {
293 return NULL;
294 }
295
296 static gboolean
gst_memory_nvmm_is_span(GstMemory * mem,GstMemory * mem2,gsize * offset)297 gst_memory_nvmm_is_span (GstMemory * mem, GstMemory * mem2, gsize * offset)
298 {
299 return FALSE;
300 }
301
302 static gboolean
gst_is_memory_nvmm(GstMemory * mem)303 gst_is_memory_nvmm (GstMemory * mem)
304 {
305 return mem && mem->allocator
306 && g_type_is_a (G_OBJECT_TYPE (mem->allocator),
307 gst_allocator_nvmm_get_type ());
308 }
309
310 static GstAllocator *_nvmm_allocator;
311
312 static void
init_nvmm_allocator(void)313 init_nvmm_allocator (void)
314 {
315 static gsize _init = 0;
316
317 if (g_once_init_enter (&_init)) {
318 _nvmm_allocator = g_object_new (gst_allocator_nvmm_get_type (), NULL);
319 /* gst_allocator_register ("NvBuffer", _nvmm_allocator); */
320 GST_OBJECT_FLAG_SET (_nvmm_allocator, GST_OBJECT_FLAG_MAY_BE_LEAKED);
321 g_once_init_leave (&_init, 1);
322 }
323 }
324
325 static GstMemory *
gst_allocator_nvmm_alloc(const GstVideoInfo * info)326 gst_allocator_nvmm_alloc (const GstVideoInfo * info)
327 {
328 GstMemoryNVMM *nvmm = g_new0 (GstMemoryNVMM, 1);
329
330 init_nvmm_allocator ();
331
332 if (!gst_memory_nvmm_init (nvmm, 0, _nvmm_allocator, NULL, info)) {
333 g_free (nvmm);
334 return NULL;
335 }
336
337 return (GstMemory *) nvmm;
338 }
339
340 static GstMemory *
_gst_allocator_nvmm_alloc(GstAllocator * alloc,gsize size,GstAllocationParams * params)341 _gst_allocator_nvmm_alloc (GstAllocator * alloc, gsize size,
342 GstAllocationParams * params)
343 {
344 g_warning
345 ("Can't allocate using gst_allocator_alloc(). Use gst_allocator_nvmm_alloc() instead");
346
347 return NULL;
348 }
349
350 static void
_gst_allocator_nvmm_free(GstAllocator * alloc,GstMemory * mem)351 _gst_allocator_nvmm_free (GstAllocator * alloc, GstMemory * mem)
352 {
353 GstMemoryNVMM *nvmm = (GstMemoryNVMM *) mem;
354
355 if (nvmm->dmabuf_fd > 0)
356 NvReleaseFd (nvmm->dmabuf_fd);
357 nvmm->dmabuf_fd = -1;
358
359 g_free (nvmm);
360 }
361
362 static void
gst_allocator_nvmm_class_init(GstAllocatorNVMMClass * klass)363 gst_allocator_nvmm_class_init (GstAllocatorNVMMClass * klass)
364 {
365 GstAllocatorClass *alloc_class = (GstAllocatorClass *) klass;
366
367 alloc_class->alloc = _gst_allocator_nvmm_alloc;
368 alloc_class->free = _gst_allocator_nvmm_free;
369 }
370
371 static void
gst_allocator_nvmm_init(GstAllocatorNVMM * nvmm)372 gst_allocator_nvmm_init (GstAllocatorNVMM * nvmm)
373 {
374 GstAllocator *alloc = (GstAllocator *) nvmm;
375
376 alloc->mem_map_full = gst_memory_nvmm_map_full;
377 alloc->mem_unmap_full = gst_memory_nvmm_unmap_full;
378 alloc->mem_copy = gst_memory_nvmm_copy;
379 alloc->mem_share = gst_memory_nvmm_share;
380 alloc->mem_is_span = gst_memory_nvmm_is_span;
381 }
382
383 GType gst_nvmm_parent_meta_api_get_type (void);
384 #define GST_NVMM_PARENT_META_API_TYPE (gst_nvmm_parent_meta_api_get_type())
385
386 #define gst_buffer_get_nvmm_parent_meta(b) \
387 ((GstNVMMParentMeta*)gst_buffer_get_meta((b),GST_NVMM_PARENT_META_API_TYPE))
388
389 const GstMetaInfo *gst_nvmm_parent_meta_get_info (void);
390 #define GST_NVMM_PARENT_META_INFO (gst_nvmm_parent_meta_get_info())
391
392 /* GstParentBufferMeta but supporting NULL and no copying to avoid accidentally
393 * introducing a circular reference when copying GstMeta's */
394 struct _GstNVMMParentMeta
395 {
396 GstMeta parent;
397
398 GstBuffer *buffer;
399 };
400 typedef struct _GstNVMMParentMeta GstNVMMParentMeta;
401
402 static GstNVMMParentMeta *
gst_buffer_add_nvmm_parent_meta(GstBuffer * buffer,GstBuffer * ref)403 gst_buffer_add_nvmm_parent_meta (GstBuffer * buffer, GstBuffer * ref)
404 {
405 GstNVMMParentMeta *meta;
406
407 g_return_val_if_fail (GST_IS_BUFFER (ref), NULL);
408
409 meta =
410 (GstNVMMParentMeta *) gst_buffer_add_meta (buffer,
411 GST_NVMM_PARENT_META_INFO, NULL);
412
413 if (!meta)
414 return NULL;
415
416 if (ref)
417 meta->buffer = gst_buffer_ref (ref);
418
419 return meta;
420 }
421
422 static gboolean
_gst_nvmm_parent_meta_transform(GstBuffer * dest,GstMeta * meta,GstBuffer * buffer,GQuark type,gpointer data)423 _gst_nvmm_parent_meta_transform (GstBuffer * dest, GstMeta * meta,
424 GstBuffer * buffer, GQuark type, gpointer data)
425 {
426 return FALSE;
427 }
428
429 static void
_gst_nvmm_parent_meta_free(GstNVMMParentMeta * parent_meta,GstBuffer * buffer)430 _gst_nvmm_parent_meta_free (GstNVMMParentMeta * parent_meta, GstBuffer * buffer)
431 {
432 GST_DEBUG ("Dropping reference on buffer %p", parent_meta->buffer);
433 gst_clear_buffer (&parent_meta->buffer);
434 }
435
436 static gboolean
_gst_nvmm_parent_meta_init(GstNVMMParentMeta * parent_meta,gpointer params,GstBuffer * buffer)437 _gst_nvmm_parent_meta_init (GstNVMMParentMeta * parent_meta,
438 gpointer params, GstBuffer * buffer)
439 {
440 parent_meta->buffer = NULL;
441
442 return TRUE;
443 }
444
445 GType
gst_nvmm_parent_meta_api_get_type(void)446 gst_nvmm_parent_meta_api_get_type (void)
447 {
448 static GType type = 0;
449 static const gchar *tags[] = { GST_META_TAG_MEMORY_STR, NULL };
450
451 if (g_once_init_enter (&type)) {
452 GType _type = gst_meta_api_type_register ("GstNVMMParentMetaAPI", tags);
453 g_once_init_leave (&type, _type);
454 }
455
456 return type;
457 }
458
459 const GstMetaInfo *
gst_nvmm_parent_meta_get_info(void)460 gst_nvmm_parent_meta_get_info (void)
461 {
462 static const GstMetaInfo *meta_info = NULL;
463
464 if (g_once_init_enter ((GstMetaInfo **) & meta_info)) {
465 const GstMetaInfo *meta =
466 gst_meta_register (gst_nvmm_parent_meta_api_get_type (),
467 "GstNVMMParentMeta",
468 sizeof (GstNVMMParentMeta),
469 (GstMetaInitFunction) _gst_nvmm_parent_meta_init,
470 (GstMetaFreeFunction) _gst_nvmm_parent_meta_free,
471 _gst_nvmm_parent_meta_transform);
472 g_once_init_leave ((GstMetaInfo **) & meta_info, (GstMetaInfo *) meta);
473 }
474
475 return meta_info;
476 }
477
478 static GstMiniObjectDisposeFunction parent_gst_buffer_dispose = NULL;
479
480 static gboolean
gst_buffer_nvmm_dispose(GstMiniObject * obj)481 gst_buffer_nvmm_dispose (GstMiniObject * obj)
482 {
483 GstBuffer *buf = (GstBuffer *) obj;
484 GstNVMMParentMeta *nv_buf_meta = gst_buffer_get_nvmm_parent_meta (buf);
485
486 GST_TRACE ("nvmm buffer dispose %p, parent_buf_meta %p", obj, nv_buf_meta);
487 if (nv_buf_meta && nv_buf_meta->buffer) {
488 GstNVMMParentMeta *gl_buf_meta;
489
490 gl_buf_meta = gst_buffer_get_nvmm_parent_meta (nv_buf_meta->buffer);
491 if (gl_buf_meta && !gl_buf_meta->buffer) {
492 // reattache the NVMM buffer to the parent buffer
493 GST_LOG ("readding nvmm buffer %p %i, to glmemory buffer %p %i", buf,
494 GST_MINI_OBJECT_REFCOUNT_VALUE (buf), nv_buf_meta->buffer,
495 GST_MINI_OBJECT_REFCOUNT_VALUE (nv_buf_meta->buffer));
496 gl_buf_meta->buffer = gst_buffer_ref (buf);
497 gst_clear_buffer (&nv_buf_meta->buffer);
498 return FALSE;
499 }
500 }
501
502 return parent_gst_buffer_dispose (obj);
503 }
504
505 struct _GstGLBufferPoolNVMMPrivate
506 {
507 GstGLVideoAllocationParams *gl_params;
508 };
509 typedef struct _GstGLBufferPoolNVMMPrivate GstGLBufferPoolNVMMPrivate;
510
511 struct _GstGLBufferPoolNVMM
512 {
513 GstGLBufferPool parent;
514 };
515
516 #define NVMM_POOL_GET_PRIV(obj) gst_gl_buffer_pool_nvmm_get_instance_private((GstGLBufferPoolNVMM *)(obj));
517
518 G_DECLARE_FINAL_TYPE (GstGLBufferPoolNVMM, gst_gl_buffer_pool_nvmm, GST,
519 GL_BUFFER_POOL_NVMM, GstGLBufferPool);
520 G_DEFINE_TYPE_WITH_CODE (GstGLBufferPoolNVMM, gst_gl_buffer_pool_nvmm,
521 GST_TYPE_GL_BUFFER_POOL, G_ADD_PRIVATE (GstGLBufferPoolNVMM));
522
523 static gboolean
gst_gl_buffer_pool_nvmm_set_config(GstBufferPool * pool,GstStructure * config)524 gst_gl_buffer_pool_nvmm_set_config (GstBufferPool * pool, GstStructure * config)
525 {
526 GstGLBufferPoolNVMMPrivate *priv;
527 GstGLBufferPool *glpool = GST_GL_BUFFER_POOL (pool);
528 GstGLVideoAllocationParams *parent_gl_params;
529 GstCaps *caps = NULL;
530 GstVideoInfo vinfo;
531 GstAllocationParams alloc_params;
532
533 priv = NVMM_POOL_GET_PRIV (pool);
534
535 if (!gst_buffer_pool_config_get_allocator (config, NULL, &alloc_params))
536 goto wrong_config;
537
538 if (!gst_buffer_pool_config_get_params (config, &caps, NULL, NULL, NULL))
539 goto wrong_config;
540
541 if (caps == NULL)
542 goto no_caps;
543
544 /* now parse the caps from the config */
545 if (!gst_video_info_from_caps (&vinfo, caps))
546 goto wrong_caps;
547
548 // TODO: fallback to regular GLMemory PBO/GetTexImage downloads?
549 if (GST_VIDEO_INFO_FORMAT (&vinfo) != GST_VIDEO_FORMAT_RGBA)
550 goto wrong_vformat;
551
552 if (!GST_BUFFER_POOL_CLASS (gst_gl_buffer_pool_nvmm_parent_class)->set_config
553 (pool, config))
554 return FALSE;
555
556 parent_gl_params = (GstGLVideoAllocationParams *)
557 gst_gl_buffer_pool_get_gl_allocation_params (glpool);
558
559 if (priv->gl_params)
560 gst_gl_allocation_params_free ((GstGLAllocationParams *) priv->gl_params);
561 priv->gl_params =
562 gst_gl_video_allocation_params_new_wrapped_gl_handle
563 (parent_gl_params->parent.context, parent_gl_params->parent.alloc_params,
564 parent_gl_params->v_info, 0, parent_gl_params->valign,
565 parent_gl_params->target, parent_gl_params->tex_format, NULL, NULL, NULL);
566
567 gst_buffer_pool_config_set_gl_allocation_params (config,
568 (GstGLAllocationParams *) priv->gl_params);
569 gst_gl_allocation_params_free ((GstGLAllocationParams *) parent_gl_params);
570
571 if (!GST_BUFFER_POOL_CLASS (gst_gl_buffer_pool_nvmm_parent_class)->set_config
572 (pool, config))
573 return FALSE;
574
575 return TRUE;
576
577 wrong_config:
578 {
579 GST_WARNING_OBJECT (pool, "invalid config");
580 return FALSE;
581 }
582 no_caps:
583 {
584 GST_WARNING_OBJECT (pool, "no caps in config");
585 return FALSE;
586 }
587 wrong_caps:
588 {
589 GST_WARNING_OBJECT (pool,
590 "failed getting geometry from caps %" GST_PTR_FORMAT, caps);
591 return FALSE;
592 }
593 wrong_vformat:
594 {
595 GST_WARNING_OBJECT (pool, "This pool only deals with RGBA textures");
596 return FALSE;
597 }
598 }
599
600 static void
nv_buffer_egl_image_mem_unref(GstEGLImage * image,GstMemory * mem)601 nv_buffer_egl_image_mem_unref (GstEGLImage * image, GstMemory * mem)
602 {
603 GstGLDisplayEGL *egl_display = NULL;
604 EGLDisplay display;
605
606 egl_display = gst_gl_display_egl_from_gl_display (image->context->display);
607 if (!egl_display) {
608 GST_ERROR ("Could not retrieve GstGLDisplayEGL from GstGLDisplay");
609 return;
610 }
611 display =
612 (EGLDisplay) gst_gl_display_get_handle (GST_GL_DISPLAY (egl_display));
613
614 if (NvDestroyEGLImage (display, image->image)) {
615 GST_ERROR ("Failed to destroy EGLImage %p from NvBuffer", image->image);
616 } else {
617 GST_DEBUG ("destroyed EGLImage %p from NvBuffer", image->image);
618 }
619
620 gst_memory_unref (mem);
621 gst_object_unref (egl_display);
622 }
623
624 static GstFlowReturn
gst_gl_buffer_pool_nvmm_alloc(GstBufferPool * pool,GstBuffer ** outbuf,GstBufferPoolAcquireParams * acquire_params)625 gst_gl_buffer_pool_nvmm_alloc (GstBufferPool * pool, GstBuffer ** outbuf,
626 GstBufferPoolAcquireParams * acquire_params)
627 {
628 GstGLBufferPool *gl_pool = GST_GL_BUFFER_POOL (pool);
629 GstGLBufferPoolNVMMPrivate *priv;
630 GstFlowReturn ret = GST_FLOW_ERROR;
631 GstBuffer *downstream_buf = NULL;
632 GstMapInfo in_map_info = GST_MAP_INFO_INIT;
633 GstGLDisplayEGL *egl_display = NULL;
634 GstEGLImage *eglimage = NULL;
635 EGLDisplay display = EGL_NO_DISPLAY;
636 EGLImageKHR image = EGL_NO_IMAGE;
637 GstGLMemoryAllocator *allocator = NULL;
638 GstMemory *nvmm_mem = NULL;
639 int in_dmabuf_fd;
640
641 priv = NVMM_POOL_GET_PRIV (pool);
642
643 *outbuf = NULL;
644 downstream_buf = gst_buffer_new ();
645 if (!parent_gst_buffer_dispose)
646 parent_gst_buffer_dispose = ((GstMiniObject *) downstream_buf)->dispose;
647 ((GstMiniObject *) downstream_buf)->dispose = gst_buffer_nvmm_dispose;
648
649 nvmm_mem = gst_allocator_nvmm_alloc (priv->gl_params->v_info);
650 if (!nvmm_mem) {
651 GST_WARNING_OBJECT (pool, "Failed to create NVMM GstMemory");
652 return GST_FLOW_ERROR;
653 }
654 gst_buffer_append_memory (downstream_buf, nvmm_mem);
655 in_dmabuf_fd = ((GstMemoryNVMM *) nvmm_mem)->dmabuf_fd;
656
657 egl_display = gst_gl_display_egl_from_gl_display (gl_pool->context->display);
658 if (!egl_display) {
659 GST_WARNING ("Failed to retrieve GstGLDisplayEGL from GstGLDisplay");
660 goto done;
661 }
662 display =
663 (EGLDisplay) gst_gl_display_get_handle (GST_GL_DISPLAY (egl_display));
664
665 image = NvEGLImageFromFd (display, in_dmabuf_fd);
666 if (!image) {
667 GST_DEBUG_OBJECT (pool, "Failed construct EGLImage "
668 "from NvBuffer fd %i", in_dmabuf_fd);
669 goto done;
670 }
671 GST_DEBUG_OBJECT (pool, "constructed EGLImage %p "
672 "from NvBuffer fd %i", image, in_dmabuf_fd);
673
674 eglimage = gst_egl_image_new_wrapped (gl_pool->context, image,
675 GST_GL_RGBA, gst_memory_ref (nvmm_mem),
676 (GstEGLImageDestroyNotify) nv_buffer_egl_image_mem_unref);
677 if (!eglimage) {
678 GST_WARNING_OBJECT (pool, "Failed to wrap constructed "
679 "EGLImage from NvBuffer");
680 goto done;
681 }
682
683 gst_buffer_unmap (downstream_buf, &in_map_info);
684 in_map_info = (GstMapInfo) GST_MAP_INFO_INIT;
685
686 allocator =
687 GST_GL_MEMORY_ALLOCATOR (gst_allocator_find
688 (GST_GL_MEMORY_EGL_ALLOCATOR_NAME));
689
690 /* TODO: buffer pool */
691 *outbuf = gst_buffer_new ();
692 if (!gst_gl_memory_setup_buffer (allocator, *outbuf, priv->gl_params,
693 NULL, (gpointer *) & eglimage, 1)) {
694 GST_WARNING_OBJECT (pool, "Failed to setup NVMM -> EGLImage buffer");
695 goto done;
696 }
697
698 gst_egl_image_unref (eglimage);
699
700 /* TODO: NvBuffer has some sync functions that may be more useful here */
701 {
702 GstGLSyncMeta *sync_meta;
703
704 sync_meta = gst_buffer_add_gl_sync_meta (gl_pool->context, *outbuf);
705 if (sync_meta) {
706 gst_gl_sync_meta_set_sync_point (sync_meta, gl_pool->context);
707 }
708 }
709
710 // possible circular reference here
711 gst_buffer_add_nvmm_parent_meta (*outbuf, downstream_buf);
712 gst_buffer_unref (downstream_buf);
713
714 ret = GST_FLOW_OK;
715
716 done:
717 if (in_map_info.memory)
718 gst_buffer_unmap (downstream_buf, &in_map_info);
719
720 gst_clear_object (&egl_display);
721 gst_clear_object (&allocator);
722
723 return ret;
724 }
725
726 static void
gst_gl_buffer_pool_nvmm_finalize(GObject * object)727 gst_gl_buffer_pool_nvmm_finalize (GObject * object)
728 {
729 GstGLBufferPoolNVMMPrivate *priv = NVMM_POOL_GET_PRIV (object);
730
731 if (priv->gl_params)
732 gst_gl_allocation_params_free ((GstGLAllocationParams *) priv->gl_params);
733 priv->gl_params = NULL;
734
735 G_OBJECT_CLASS (gst_gl_buffer_pool_nvmm_parent_class)->finalize (object);
736 }
737
738 static void
gst_gl_buffer_pool_nvmm_init(GstGLBufferPoolNVMM * pool)739 gst_gl_buffer_pool_nvmm_init (GstGLBufferPoolNVMM * pool)
740 {
741 }
742
743 static void
gst_gl_buffer_pool_nvmm_class_init(GstGLBufferPoolNVMMClass * klass)744 gst_gl_buffer_pool_nvmm_class_init (GstGLBufferPoolNVMMClass * klass)
745 {
746 GstBufferPoolClass *pool_class = (GstBufferPoolClass *) klass;
747 GObjectClass *gobject_class = (GObjectClass *) klass;
748
749 pool_class->set_config = gst_gl_buffer_pool_nvmm_set_config;
750 pool_class->alloc_buffer = gst_gl_buffer_pool_nvmm_alloc;
751
752 gobject_class->finalize = gst_gl_buffer_pool_nvmm_finalize;
753 }
754
755 static GstBufferPool *
gst_gl_buffer_pool_nvmm_new(GstGLContext * context)756 gst_gl_buffer_pool_nvmm_new (GstGLContext * context)
757 {
758 GstGLBufferPoolNVMM *pool;
759 GstGLBufferPool *gl_pool;
760
761 pool = g_object_new (gst_gl_buffer_pool_nvmm_get_type (), NULL);
762 gst_object_ref_sink (pool);
763 gl_pool = GST_GL_BUFFER_POOL (pool);
764 gl_pool->context = gst_object_ref (context);
765
766 GST_LOG_OBJECT (pool, "new NVMM GL buffer pool for context %" GST_PTR_FORMAT,
767 context);
768
769 return GST_BUFFER_POOL_CAST (pool);
770 }
771 #endif /* GST_GL_HAVE_PLATFORM_EGL && defined(HAVE_NVMM) */
772
773 #define gst_gl_download_element_parent_class parent_class
774 G_DEFINE_TYPE_WITH_CODE (GstGLDownloadElement, gst_gl_download_element,
775 GST_TYPE_GL_BASE_FILTER,
776 GST_DEBUG_CATEGORY_INIT (gst_gl_download_element_debug, "gldownloadelement",
777 0, "download element"););
778 GST_ELEMENT_REGISTER_DEFINE_WITH_CODE (gldownload, "gldownload",
779 GST_RANK_NONE, GST_TYPE_GL_DOWNLOAD_ELEMENT, gl_element_init (plugin));
780
781 static gboolean gst_gl_download_element_start (GstBaseTransform * bt);
782 static gboolean gst_gl_download_element_stop (GstBaseTransform * bt);
783 static gboolean gst_gl_download_element_get_unit_size (GstBaseTransform * trans,
784 GstCaps * caps, gsize * size);
785 static GstCaps *gst_gl_download_element_transform_caps (GstBaseTransform * bt,
786 GstPadDirection direction, GstCaps * caps, GstCaps * filter);
787 static GstCaps *gst_gl_download_element_fixate_caps (GstBaseTransform * trans,
788 GstPadDirection direction, GstCaps * caps, GstCaps * othercaps);
789 static gboolean gst_gl_download_element_set_caps (GstBaseTransform * bt,
790 GstCaps * in_caps, GstCaps * out_caps);
791 static GstFlowReturn
792 gst_gl_download_element_prepare_output_buffer (GstBaseTransform * bt,
793 GstBuffer * buffer, GstBuffer ** outbuf);
794 static GstFlowReturn gst_gl_download_element_transform (GstBaseTransform * bt,
795 GstBuffer * buffer, GstBuffer * outbuf);
796 static gboolean gst_gl_download_element_transform_meta (GstBaseTransform * bt,
797 GstBuffer * outbuf, GstMeta * meta, GstBuffer * inbuf);
798 static gboolean gst_gl_download_element_decide_allocation (GstBaseTransform *
799 trans, GstQuery * query);
800 static gboolean gst_gl_download_element_sink_event (GstBaseTransform * bt,
801 GstEvent * event);
802 static gboolean gst_gl_download_element_src_event (GstBaseTransform * bt,
803 GstEvent * event);
804 static gboolean gst_gl_download_element_propose_allocation (GstBaseTransform *
805 bt, GstQuery * decide_query, GstQuery * query);
806 static void gst_gl_download_element_finalize (GObject * object);
807
808 #define GST_CAPS_FEATURE_MEMORY_NVMM "memory:NVMM"
809
810 #if GST_GL_HAVE_PLATFORM_EGL && defined(HAVE_NVMM)
811 #define EXTRA_CAPS_TEMPLATE1 "video/x-raw(" GST_CAPS_FEATURE_MEMORY_NVMM "), format=(string)RGBA; "
812 #else
813 #define EXTRA_CAPS_TEMPLATE1
814 #endif
815
816 #if GST_GL_HAVE_PLATFORM_EGL && GST_GL_HAVE_DMABUF
817 #define EXTRA_CAPS_TEMPLATE2 "video/x-raw(" GST_CAPS_FEATURE_MEMORY_DMABUF "); "
818 #else
819 #define EXTRA_CAPS_TEMPLATE2
820 #endif
821
822 static GstStaticPadTemplate gst_gl_download_element_src_pad_template =
823 GST_STATIC_PAD_TEMPLATE ("src",
824 GST_PAD_SRC,
825 GST_PAD_ALWAYS,
826 GST_STATIC_CAPS (EXTRA_CAPS_TEMPLATE1 EXTRA_CAPS_TEMPLATE2
827 "video/x-raw; video/x-raw(memory:GLMemory)"));
828
829 static GstStaticPadTemplate gst_gl_download_element_sink_pad_template =
830 GST_STATIC_PAD_TEMPLATE ("sink",
831 GST_PAD_SINK,
832 GST_PAD_ALWAYS,
833 GST_STATIC_CAPS ("video/x-raw(memory:GLMemory); video/x-raw"));
834
835 static void
gst_gl_download_element_class_init(GstGLDownloadElementClass * klass)836 gst_gl_download_element_class_init (GstGLDownloadElementClass * klass)
837 {
838 GstBaseTransformClass *bt_class = GST_BASE_TRANSFORM_CLASS (klass);
839 GstElementClass *element_class = GST_ELEMENT_CLASS (klass);
840 GObjectClass *object_class = G_OBJECT_CLASS (klass);
841
842 bt_class->start = gst_gl_download_element_start;
843 bt_class->stop = gst_gl_download_element_stop;
844 bt_class->transform_caps = gst_gl_download_element_transform_caps;
845 bt_class->fixate_caps = gst_gl_download_element_fixate_caps;
846 bt_class->set_caps = gst_gl_download_element_set_caps;
847 bt_class->get_unit_size = gst_gl_download_element_get_unit_size;
848 bt_class->prepare_output_buffer =
849 gst_gl_download_element_prepare_output_buffer;
850 bt_class->transform = gst_gl_download_element_transform;
851 bt_class->decide_allocation = gst_gl_download_element_decide_allocation;
852 bt_class->sink_event = gst_gl_download_element_sink_event;
853 bt_class->src_event = gst_gl_download_element_src_event;
854 bt_class->propose_allocation = gst_gl_download_element_propose_allocation;
855 bt_class->transform_meta = gst_gl_download_element_transform_meta;
856
857 bt_class->passthrough_on_same_caps = TRUE;
858
859 gst_element_class_add_static_pad_template (element_class,
860 &gst_gl_download_element_src_pad_template);
861 gst_element_class_add_static_pad_template (element_class,
862 &gst_gl_download_element_sink_pad_template);
863
864 gst_element_class_set_metadata (element_class,
865 "OpenGL downloader", "Filter/Video",
866 "Downloads data from OpenGL", "Matthew Waters <matthew@centricular.com>");
867
868 object_class->finalize = gst_gl_download_element_finalize;
869 }
870
871 static void
gst_gl_download_element_init(GstGLDownloadElement * download)872 gst_gl_download_element_init (GstGLDownloadElement * download)
873 {
874 gst_base_transform_set_prefer_passthrough (GST_BASE_TRANSFORM (download),
875 TRUE);
876 }
877
878 static gboolean
gst_gl_download_element_start(GstBaseTransform * bt)879 gst_gl_download_element_start (GstBaseTransform * bt)
880 {
881 #if GST_GL_HAVE_PLATFORM_EGL && GST_GL_HAVE_DMABUF
882 GstGLDownloadElement *dl = GST_GL_DOWNLOAD_ELEMENT (bt);
883
884 dl->dmabuf_allocator = gst_dmabuf_allocator_new ();
885 g_atomic_int_set (&dl->try_dmabuf_exports, TRUE);
886 #endif
887
888 return TRUE;
889 }
890
891 static gboolean
gst_gl_download_element_stop(GstBaseTransform * bt)892 gst_gl_download_element_stop (GstBaseTransform * bt)
893 {
894 GstGLDownloadElement *dl = GST_GL_DOWNLOAD_ELEMENT (bt);
895
896 if (dl->dmabuf_allocator) {
897 gst_object_unref (GST_OBJECT (dl->dmabuf_allocator));
898 dl->dmabuf_allocator = NULL;
899 }
900
901 return TRUE;
902 }
903
904 static gboolean
gst_gl_download_element_set_caps(GstBaseTransform * bt,GstCaps * in_caps,GstCaps * out_caps)905 gst_gl_download_element_set_caps (GstBaseTransform * bt, GstCaps * in_caps,
906 GstCaps * out_caps)
907 {
908 GstGLDownloadElement *dl = GST_GL_DOWNLOAD_ELEMENT (bt);
909 GstVideoInfo out_info;
910 GstCapsFeatures *features = NULL;
911
912 if (!gst_video_info_from_caps (&out_info, out_caps))
913 return FALSE;
914
915 features = gst_caps_get_features (out_caps, 0);
916
917 if (gst_caps_features_contains (features, GST_CAPS_FEATURE_MEMORY_GL_MEMORY)) {
918 dl->mode = GST_GL_DOWNLOAD_MODE_PASSTHROUGH;
919 GST_INFO_OBJECT (dl, "caps signal passthrough");
920 #if GST_GL_HAVE_PLATFORM_EGL && defined(HAVE_NVMM)
921 } else if (gst_caps_features_contains (features,
922 GST_CAPS_FEATURE_MEMORY_NVMM)) {
923 dl->mode = GST_GL_DOWNLOAD_MODE_NVMM;
924 GST_INFO_OBJECT (dl, "caps signal NVMM");
925 #endif
926 #if GST_GL_HAVE_PLATFORM_EGL && GST_GL_HAVE_DMABUF
927 } else if (g_atomic_int_get (&dl->try_dmabuf_exports) &&
928 gst_caps_features_contains (features, GST_CAPS_FEATURE_MEMORY_DMABUF)) {
929 dl->mode = GST_GL_DOWNLOAD_MODE_DMABUF_EXPORTS;
930 GST_INFO_OBJECT (dl, "caps signal dma-buf export");
931 #endif
932 } else {
933 /* System Memory */
934 dl->mode = GST_GL_DOWNLOAD_MODE_PBO_TRANSFERS;
935 GST_INFO_OBJECT (dl, "caps signal sysmem download");
936 }
937
938 return TRUE;
939 }
940
941 static GstCaps *
_set_caps_features(const GstCaps * caps,const gchar * feature_name)942 _set_caps_features (const GstCaps * caps, const gchar * feature_name)
943 {
944 GstCaps *tmp = gst_caps_copy (caps);
945 guint n = gst_caps_get_size (tmp);
946 guint i = 0;
947
948 for (i = 0; i < n; i++)
949 gst_caps_set_features (tmp, i,
950 gst_caps_features_from_string (feature_name));
951
952 return tmp;
953 }
954
955 static void
_remove_field(GstCaps * caps,const gchar * field)956 _remove_field (GstCaps * caps, const gchar * field)
957 {
958 guint n = gst_caps_get_size (caps);
959 guint i = 0;
960
961 for (i = 0; i < n; i++) {
962 GstStructure *s = gst_caps_get_structure (caps, i);
963 gst_structure_remove_field (s, field);
964 }
965 }
966
967 static GstCaps *
gst_gl_download_element_transform_caps(GstBaseTransform * bt,GstPadDirection direction,GstCaps * caps,GstCaps * filter)968 gst_gl_download_element_transform_caps (GstBaseTransform * bt,
969 GstPadDirection direction, GstCaps * caps, GstCaps * filter)
970 {
971 GstCaps *result, *tmp;
972
973 if (direction == GST_PAD_SRC) {
974 GstCaps *sys_caps = gst_caps_simplify (_set_caps_features (caps,
975 GST_CAPS_FEATURE_MEMORY_SYSTEM_MEMORY));
976
977 tmp = _set_caps_features (sys_caps, GST_CAPS_FEATURE_MEMORY_GL_MEMORY);
978 tmp = gst_caps_merge (tmp, sys_caps);
979 } else {
980 GstCaps *newcaps;
981 tmp = gst_caps_ref (caps);
982
983 #if GST_GL_HAVE_PLATFORM_EGL && defined(HAVE_NVMM)
984 newcaps = _set_caps_features (caps, GST_CAPS_FEATURE_MEMORY_NVMM);
985 _remove_field (newcaps, "texture-target");
986 // FIXME: RGBA-only?
987 tmp = gst_caps_merge (tmp, newcaps);
988 #endif
989
990 #if GST_GL_HAVE_PLATFORM_EGL && GST_GL_HAVE_DMABUF
991 newcaps = _set_caps_features (caps, GST_CAPS_FEATURE_MEMORY_DMABUF);
992 _remove_field (newcaps, "texture-target");
993 tmp = gst_caps_merge (tmp, newcaps);
994 #endif
995
996 newcaps = _set_caps_features (caps, GST_CAPS_FEATURE_MEMORY_SYSTEM_MEMORY);
997 _remove_field (newcaps, "texture-target");
998 tmp = gst_caps_merge (tmp, newcaps);
999 }
1000
1001 if (filter) {
1002 result = gst_caps_intersect_full (filter, tmp, GST_CAPS_INTERSECT_FIRST);
1003 gst_caps_unref (tmp);
1004 } else {
1005 result = tmp;
1006 }
1007
1008 GST_DEBUG_OBJECT (bt, "returning caps %" GST_PTR_FORMAT, result);
1009
1010 return result;
1011 }
1012
1013 static GstCaps *
gst_gl_download_element_fixate_caps(GstBaseTransform * bt,GstPadDirection direction,GstCaps * caps,GstCaps * othercaps)1014 gst_gl_download_element_fixate_caps (GstBaseTransform * bt,
1015 GstPadDirection direction, GstCaps * caps, GstCaps * othercaps)
1016 {
1017 #if GST_GL_HAVE_PLATFORM_EGL && GST_GL_HAVE_DMABUF
1018 GstGLDownloadElement *dl = GST_GL_DOWNLOAD_ELEMENT (bt);
1019
1020 /* Remove DMABuf features if try_dmabuf_exports is not set */
1021 if (direction == GST_PAD_SINK && !g_atomic_int_get (&dl->try_dmabuf_exports)) {
1022 gint i;
1023
1024 for (i = 0; i < gst_caps_get_size (othercaps); i++) {
1025 GstCapsFeatures *features = gst_caps_get_features (othercaps, i);
1026
1027 if (features && gst_caps_features_contains (features,
1028 GST_CAPS_FEATURE_MEMORY_DMABUF)) {
1029 caps = gst_caps_make_writable (othercaps);
1030 gst_caps_remove_structure (othercaps, i--);
1031 }
1032 }
1033 }
1034 #endif
1035
1036 return GST_BASE_TRANSFORM_CLASS (parent_class)->fixate_caps (bt, direction,
1037 caps, othercaps);
1038 }
1039
1040 static gboolean
gst_gl_download_element_get_unit_size(GstBaseTransform * trans,GstCaps * caps,gsize * size)1041 gst_gl_download_element_get_unit_size (GstBaseTransform * trans, GstCaps * caps,
1042 gsize * size)
1043 {
1044 gboolean ret = FALSE;
1045 GstVideoInfo info;
1046
1047 ret = gst_video_info_from_caps (&info, caps);
1048 if (ret)
1049 *size = GST_VIDEO_INFO_SIZE (&info);
1050
1051 return TRUE;
1052 }
1053
1054 #if GST_GL_HAVE_PLATFORM_EGL && GST_GL_HAVE_DMABUF
1055
1056 struct DmabufInfo
1057 {
1058 GstMemory *dmabuf;
1059 gint stride;
1060 gsize offset;
1061 };
1062
1063 static void
_free_dmabuf_info(struct DmabufInfo * info)1064 _free_dmabuf_info (struct DmabufInfo *info)
1065 {
1066 gst_memory_unref (info->dmabuf);
1067 g_free (info);
1068 }
1069
1070 static GQuark
_dmabuf_info_quark(void)1071 _dmabuf_info_quark (void)
1072 {
1073 static GQuark quark = 0;
1074
1075 if (!quark)
1076 quark = g_quark_from_static_string ("GstGLDownloadDmabufInfo");
1077 return quark;
1078 }
1079
1080 static struct DmabufInfo *
_get_cached_dmabuf_info(GstGLMemory * mem)1081 _get_cached_dmabuf_info (GstGLMemory * mem)
1082 {
1083 return gst_mini_object_get_qdata (GST_MINI_OBJECT (mem),
1084 _dmabuf_info_quark ());
1085 }
1086
1087 static void
_set_cached_dmabuf_info(GstGLMemory * mem,struct DmabufInfo * info)1088 _set_cached_dmabuf_info (GstGLMemory * mem, struct DmabufInfo *info)
1089 {
1090 return gst_mini_object_set_qdata (GST_MINI_OBJECT (mem),
1091 _dmabuf_info_quark (), info, (GDestroyNotify) _free_dmabuf_info);
1092 }
1093
1094 struct DmabufTransfer
1095 {
1096 GstGLDownloadElement *download;
1097 GstGLMemory *glmem;
1098 struct DmabufInfo *info;
1099 };
1100
1101 static void
_create_cached_dmabuf_info(GstGLContext * context,gpointer data)1102 _create_cached_dmabuf_info (GstGLContext * context, gpointer data)
1103 {
1104 struct DmabufTransfer *transfer = (struct DmabufTransfer *) data;
1105 GstEGLImage *image;
1106
1107 image = gst_egl_image_from_texture (context, transfer->glmem, NULL);
1108 if (image) {
1109 int fd;
1110 gint stride;
1111 gsize offset;
1112
1113 if (gst_egl_image_export_dmabuf (image, &fd, &stride, &offset)) {
1114 GstGLDownloadElement *download = transfer->download;
1115 struct DmabufInfo *info;
1116 gsize size;
1117
1118 size =
1119 gst_gl_memory_get_texture_height (transfer->glmem) * stride + offset;
1120
1121 info = g_new0 (struct DmabufInfo, 1);
1122 info->dmabuf =
1123 gst_dmabuf_allocator_alloc (download->dmabuf_allocator, fd, size);
1124 info->stride = stride;
1125 info->offset = offset;
1126
1127 transfer->info = info;
1128 }
1129
1130 gst_egl_image_unref (image);
1131 }
1132 }
1133
1134 static GstBuffer *
_try_export_dmabuf(GstGLDownloadElement * download,GstBuffer * inbuf)1135 _try_export_dmabuf (GstGLDownloadElement * download, GstBuffer * inbuf)
1136 {
1137 GstGLMemory *glmem;
1138 GstBuffer *buffer = NULL;
1139 int i;
1140 gsize offset[GST_VIDEO_MAX_PLANES];
1141 gint stride[GST_VIDEO_MAX_PLANES];
1142 GstCaps *src_caps;
1143 GstVideoInfo out_info;
1144 gsize total_offset;
1145 GstVideoAlignment *alig = NULL;
1146
1147 glmem = GST_GL_MEMORY_CAST (gst_buffer_peek_memory (inbuf, 0));
1148 if (glmem) {
1149 GstGLContext *context = GST_GL_BASE_MEMORY_CAST (glmem)->context;
1150 if (gst_gl_context_get_gl_platform (context) != GST_GL_PLATFORM_EGL)
1151 return NULL;
1152 alig = &glmem->valign;
1153 }
1154
1155 buffer = gst_buffer_new ();
1156 total_offset = 0;
1157
1158 for (i = 0; i < gst_buffer_n_memory (inbuf); i++) {
1159 struct DmabufInfo *info;
1160
1161 glmem = GST_GL_MEMORY_CAST (gst_buffer_peek_memory (inbuf, i));
1162 info = _get_cached_dmabuf_info (glmem);
1163 if (!info) {
1164 GstGLContext *context = GST_GL_BASE_MEMORY_CAST (glmem)->context;
1165 struct DmabufTransfer transfer;
1166
1167 transfer.download = download;
1168 transfer.glmem = glmem;
1169 transfer.info = NULL;
1170 gst_gl_context_thread_add (context, _create_cached_dmabuf_info,
1171 &transfer);
1172 info = transfer.info;
1173
1174 if (info)
1175 _set_cached_dmabuf_info (glmem, info);
1176 }
1177
1178 if (info) {
1179 offset[i] = total_offset + info->offset;
1180 stride[i] = info->stride;
1181 total_offset += gst_memory_get_sizes (info->dmabuf, NULL, NULL);
1182 gst_buffer_insert_memory (buffer, -1, gst_memory_ref (info->dmabuf));
1183 } else {
1184 gst_buffer_unref (buffer);
1185 buffer = NULL;
1186 goto export_complete;
1187 }
1188 }
1189
1190 src_caps = gst_pad_get_current_caps (GST_BASE_TRANSFORM (download)->srcpad);
1191 gst_video_info_from_caps (&out_info, src_caps);
1192 gst_caps_unref (src_caps);
1193
1194 if (download->add_videometa) {
1195 GstVideoMeta *meta;
1196
1197 meta = gst_buffer_add_video_meta_full (buffer, GST_VIDEO_FRAME_FLAG_NONE,
1198 out_info.finfo->format, out_info.width, out_info.height,
1199 out_info.finfo->n_planes, offset, stride);
1200
1201 if (alig)
1202 gst_video_meta_set_alignment (meta, *alig);
1203 } else {
1204 int i;
1205 gboolean match = TRUE;
1206 for (i = 0; i < gst_buffer_n_memory (inbuf); i++) {
1207 if (offset[i] != out_info.offset[i] || stride[i] != out_info.stride[i]) {
1208 match = FALSE;
1209 break;
1210 }
1211 }
1212
1213 if (!match) {
1214 gst_buffer_unref (buffer);
1215 buffer = NULL;
1216 }
1217 }
1218
1219 export_complete:
1220
1221 return buffer;
1222 }
1223 #endif /* GST_GL_HAVE_PLATFORM_EGL && GST_GL_HAVE_DMABUF */
1224
1225 static GstFlowReturn
gst_gl_download_element_prepare_output_buffer(GstBaseTransform * bt,GstBuffer * inbuf,GstBuffer ** outbuf)1226 gst_gl_download_element_prepare_output_buffer (GstBaseTransform * bt,
1227 GstBuffer * inbuf, GstBuffer ** outbuf)
1228 {
1229 GstGLDownloadElement *dl = GST_GL_DOWNLOAD_ELEMENT (bt);
1230 GstBaseTransformClass *bclass = GST_BASE_TRANSFORM_GET_CLASS (bt);
1231 GstGLContext *context = GST_GL_BASE_FILTER (bt)->context;
1232 GstGLSyncMeta *in_sync_meta;
1233 gint i, n;
1234
1235 *outbuf = inbuf;
1236
1237 (void) bclass;
1238
1239 in_sync_meta = gst_buffer_get_gl_sync_meta (inbuf);
1240 if (in_sync_meta)
1241 gst_gl_sync_meta_wait (in_sync_meta, context);
1242
1243 #if GST_GL_HAVE_PLATFORM_EGL && defined(HAVE_NVMM)
1244 if (dl->mode == GST_GL_DOWNLOAD_MODE_NVMM) {
1245 GstNVMMParentMeta *buf_meta = gst_buffer_get_nvmm_parent_meta (inbuf);
1246 GstMemory *mem;
1247 GstMemoryNVMM *nvmm_mem;
1248
1249 if (!buf_meta || !buf_meta->buffer) {
1250 // TODO: remove this restriction with an e.g. copy...
1251 GST_ERROR_OBJECT (dl,
1252 "Cannot push upstream created buffer when outputting NVMM");
1253 return GST_FLOW_ERROR;
1254 }
1255
1256 if (!(mem = gst_buffer_peek_memory (buf_meta->buffer, 0))) {
1257 GST_ERROR_OBJECT (dl, "No memory in buffer?");
1258 return GST_FLOW_ERROR;
1259 }
1260
1261 if (!gst_is_memory_nvmm (mem)) {
1262 GST_ERROR_OBJECT (dl,
1263 "Upstream buffer does not contain an attached NVMM GstMemory");
1264 return GST_FLOW_ERROR;
1265 }
1266 nvmm_mem = (GstMemoryNVMM *) mem;
1267
1268 /* switch up the parent buffer references so that when the NVMM buffer is
1269 * released, the associated EGLImage/OpenGL texture is as well
1270 */
1271 GST_DEBUG_OBJECT (dl, "NVMM buffer fd:%i passed through %" GST_PTR_FORMAT,
1272 nvmm_mem->dmabuf_fd, buf_meta->buffer);
1273 *outbuf = buf_meta->buffer;
1274 bclass->copy_metadata (bt, inbuf, *outbuf);
1275 buf_meta->buffer = NULL;
1276 buf_meta = gst_buffer_get_nvmm_parent_meta (*outbuf);
1277 if (!buf_meta) {
1278 buf_meta = gst_buffer_add_nvmm_parent_meta (*outbuf, inbuf);
1279 } else {
1280 gst_clear_buffer (&buf_meta->buffer);
1281 buf_meta->buffer = gst_buffer_ref (inbuf);
1282 }
1283
1284 return GST_FLOW_OK;
1285 }
1286 #endif
1287 #if GST_GL_HAVE_PLATFORM_EGL && GST_GL_HAVE_DMABUF
1288 if (dl->mode == GST_GL_DOWNLOAD_MODE_DMABUF_EXPORTS) {
1289 GstBuffer *buffer = _try_export_dmabuf (dl, inbuf);
1290
1291 if (buffer) {
1292 if (GST_BASE_TRANSFORM_GET_CLASS (bt)->copy_metadata) {
1293 if (!GST_BASE_TRANSFORM_GET_CLASS (bt)->copy_metadata (bt, inbuf,
1294 buffer)) {
1295 GST_ELEMENT_WARNING (GST_ELEMENT (bt), STREAM, NOT_IMPLEMENTED,
1296 ("could not copy metadata"), (NULL));
1297 }
1298 }
1299
1300 *outbuf = buffer;
1301 } else {
1302 GstCaps *src_caps;
1303 GstCapsFeatures *features;
1304 gboolean ret;
1305
1306 src_caps = gst_pad_get_current_caps (bt->srcpad);
1307 src_caps = gst_caps_make_writable (src_caps);
1308 features = gst_caps_get_features (src_caps, 0);
1309 gst_caps_features_remove (features, GST_CAPS_FEATURE_MEMORY_DMABUF);
1310 g_atomic_int_set (&dl->try_dmabuf_exports, FALSE);
1311 dl->mode = GST_GL_DOWNLOAD_MODE_PBO_TRANSFERS;
1312
1313 ret = gst_base_transform_update_src_caps (bt, src_caps);
1314 gst_caps_unref (src_caps);
1315
1316 if (!ret) {
1317 GST_ERROR_OBJECT (bt, "DMABuf exportation didn't work and system "
1318 "memory is not supported.");
1319 return GST_FLOW_NOT_NEGOTIATED;
1320 }
1321 }
1322 }
1323 #endif
1324
1325 if (dl->mode == GST_GL_DOWNLOAD_MODE_PBO_TRANSFERS) {
1326 n = gst_buffer_n_memory (*outbuf);
1327 for (i = 0; i < n; i++) {
1328 GstMemory *mem = gst_buffer_peek_memory (*outbuf, i);
1329
1330 if (gst_is_gl_memory_pbo (mem))
1331 gst_gl_memory_pbo_download_transfer ((GstGLMemoryPBO *) mem);
1332 }
1333 }
1334
1335 return GST_FLOW_OK;
1336 }
1337
1338 static GstFlowReturn
gst_gl_download_element_transform(GstBaseTransform * bt,GstBuffer * inbuf,GstBuffer * outbuf)1339 gst_gl_download_element_transform (GstBaseTransform * bt,
1340 GstBuffer * inbuf, GstBuffer * outbuf)
1341 {
1342 return GST_FLOW_OK;
1343 }
1344
1345 static gboolean
gst_gl_download_element_transform_meta(GstBaseTransform * bt,GstBuffer * outbuf,GstMeta * meta,GstBuffer * inbuf)1346 gst_gl_download_element_transform_meta (GstBaseTransform * bt,
1347 GstBuffer * outbuf, GstMeta * meta, GstBuffer * inbuf)
1348 {
1349 if (g_type_is_a (meta->info->api, GST_GL_SYNC_META_API_TYPE)) {
1350 GST_LOG_OBJECT (bt, "not copying GstGLSyncMeta onto output buffer");
1351 return FALSE;
1352 }
1353
1354 return GST_BASE_TRANSFORM_CLASS (parent_class)->transform_meta (bt, outbuf,
1355 meta, inbuf);
1356 }
1357
1358 static gboolean
gst_gl_download_element_decide_allocation(GstBaseTransform * trans,GstQuery * query)1359 gst_gl_download_element_decide_allocation (GstBaseTransform * trans,
1360 GstQuery * query)
1361 {
1362 GstGLDownloadElement *download = GST_GL_DOWNLOAD_ELEMENT_CAST (trans);
1363
1364 if (gst_query_find_allocation_meta (query, GST_VIDEO_META_API_TYPE, NULL)) {
1365 download->add_videometa = TRUE;
1366 } else {
1367 download->add_videometa = FALSE;
1368 }
1369
1370 return GST_BASE_TRANSFORM_CLASS (parent_class)->decide_allocation (trans,
1371 query);
1372 }
1373
1374 static gboolean
gst_gl_download_element_sink_event(GstBaseTransform * bt,GstEvent * event)1375 gst_gl_download_element_sink_event (GstBaseTransform * bt, GstEvent * event)
1376 {
1377 GstGLDownloadElement *dl = GST_GL_DOWNLOAD_ELEMENT (bt);
1378
1379 /* Retry exporting whenever we have new caps from upstream */
1380 if (GST_EVENT_TYPE (event) == GST_EVENT_CAPS)
1381 g_atomic_int_set (&dl->try_dmabuf_exports, TRUE);
1382
1383 return GST_BASE_TRANSFORM_CLASS (parent_class)->sink_event (bt, event);
1384 }
1385
1386 static gboolean
gst_gl_download_element_src_event(GstBaseTransform * bt,GstEvent * event)1387 gst_gl_download_element_src_event (GstBaseTransform * bt, GstEvent * event)
1388 {
1389 GstGLDownloadElement *dl = GST_GL_DOWNLOAD_ELEMENT (bt);
1390
1391 /* Retry exporting whenever downstream have changed */
1392 if (GST_EVENT_TYPE (event) == GST_EVENT_RECONFIGURE)
1393 g_atomic_int_set (&dl->try_dmabuf_exports, TRUE);
1394
1395 return GST_BASE_TRANSFORM_CLASS (parent_class)->src_event (bt, event);
1396 }
1397
1398 static gboolean
gst_gl_download_element_propose_allocation(GstBaseTransform * bt,GstQuery * decide_query,GstQuery * query)1399 gst_gl_download_element_propose_allocation (GstBaseTransform * bt,
1400 GstQuery * decide_query, GstQuery * query)
1401 {
1402 GstBufferPool *pool = NULL;
1403 GstCaps *caps;
1404 GstGLContext *context;
1405 GstStructure *config;
1406 GstVideoInfo info;
1407 gsize size;
1408
1409 if (!GST_BASE_TRANSFORM_CLASS (parent_class)->propose_allocation (bt,
1410 decide_query, query))
1411 return FALSE;
1412
1413 gst_query_parse_allocation (query, &caps, NULL);
1414 if (caps == NULL)
1415 goto invalid_caps;
1416
1417 context = GST_GL_BASE_FILTER (bt)->context;
1418 if (!context) {
1419 GST_ERROR_OBJECT (context, "got no GLContext");
1420 return FALSE;
1421 }
1422
1423 if (!gst_video_info_from_caps (&info, caps))
1424 goto invalid_caps;
1425
1426 #if GST_GL_HAVE_PLATFORM_EGL && defined(HAVE_NVMM)
1427 if (!pool && decide_query) {
1428 GstCaps *decide_caps;
1429
1430 gst_query_parse_allocation (decide_query, &decide_caps, NULL);
1431 if (decide_caps && gst_caps_get_size (decide_caps) > 0) {
1432 GstCapsFeatures *features = gst_caps_get_features (decide_caps, 0);
1433
1434 if (gst_caps_features_contains (features, GST_CAPS_FEATURE_MEMORY_NVMM)) {
1435 pool = gst_gl_buffer_pool_nvmm_new (context);
1436 GST_INFO_OBJECT (bt, "have NVMM downstream, proposing NVMM "
1437 "pool %" GST_PTR_FORMAT, pool);
1438 }
1439 }
1440 }
1441 #endif
1442 if (!pool) {
1443 pool = gst_gl_buffer_pool_new (context);
1444 }
1445 config = gst_buffer_pool_get_config (pool);
1446
1447 /* the normal size of a frame */
1448 size = info.size;
1449 gst_buffer_pool_config_set_params (config, caps, size, 0, 0);
1450 gst_buffer_pool_config_add_option (config,
1451 GST_BUFFER_POOL_OPTION_GL_SYNC_META);
1452
1453 if (!gst_buffer_pool_set_config (pool, config)) {
1454 gst_object_unref (pool);
1455 goto config_failed;
1456 }
1457 gst_query_add_allocation_pool (query, pool, size, 1, 0);
1458
1459 gst_object_unref (pool);
1460 return TRUE;
1461
1462 invalid_caps:
1463 {
1464 GST_ERROR_OBJECT (bt, "Invalid Caps specified");
1465 return FALSE;
1466 }
1467 config_failed:
1468 {
1469 GST_ERROR_OBJECT (bt, "failed setting config");
1470 return FALSE;
1471 }
1472 }
1473
1474
1475 static void
gst_gl_download_element_finalize(GObject * object)1476 gst_gl_download_element_finalize (GObject * object)
1477 {
1478 GstGLDownloadElement *download = GST_GL_DOWNLOAD_ELEMENT_CAST (object);
1479
1480 if (download->dmabuf_allocator) {
1481 gst_object_unref (GST_OBJECT (download->dmabuf_allocator));
1482 download->dmabuf_allocator = NULL;
1483 }
1484
1485 G_OBJECT_CLASS (parent_class)->finalize (object);
1486 }
1487