1 /*
2 * Copyright (C) 2017 Ericsson AB. All rights reserved.
3 *
4 * Redistribution and use in source and binary forms, with or without
5 * modification, are permitted provided that the following conditions
6 * are met:
7 *
8 * 1. Redistributions of source code must retain the above copyright
9 * notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 * notice, this list of conditions and the following disclaimer
12 * in the documentation and/or other materials provided with the
13 * distribution.
14 *
15 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
16 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
17 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
18 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
19 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
20 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
21 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
22 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
23 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
24 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
25 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26 */
27
28 #ifdef HAVE_CONFIG_H
29 #include "config.h"
30 #endif
31
32 #include "gstnvdec.h"
33 #include "gstcudautils.h"
34 #include "gstcudabufferpool.h"
35
36 #include <string.h>
37
38 GST_DEBUG_CATEGORY_EXTERN (gst_nvdec_debug);
39 #define GST_CAT_DEFAULT gst_nvdec_debug
40
41 #define DEFAULT_MAX_DISPLAY_DELAY -1
42
43 enum
44 {
45 PROP_0,
46 PROP_MAX_DISPLAY_DELAY,
47 };
48
49 #ifdef HAVE_NVCODEC_GST_GL
50 #define SUPPORTED_GL_APIS (GST_GL_API_OPENGL | GST_GL_API_OPENGL3 | GST_GL_API_GLES2)
51
52 static gboolean
53 gst_nvdec_copy_device_to_gl (GstNvDec * nvdec,
54 CUVIDPARSERDISPINFO * dispinfo, GstBuffer * output_buffer);
55 #endif
56
57 static gboolean
58 gst_nvdec_copy_device_to_memory (GstNvDec * nvdec,
59 CUVIDPARSERDISPINFO * dispinfo, GstBuffer * output_buffer);
60
61 #ifdef HAVE_NVCODEC_GST_GL
62 typedef struct _GstNvDecRegisterResourceData
63 {
64 GstMemory *mem;
65 GstCudaGraphicsResource *resource;
66 GstNvDec *nvdec;
67 gboolean ret;
68 } GstNvDecRegisterResourceData;
69
70 static void
register_cuda_resource(GstGLContext * context,GstNvDecRegisterResourceData * data)71 register_cuda_resource (GstGLContext * context,
72 GstNvDecRegisterResourceData * data)
73 {
74 GstMemory *mem = data->mem;
75 GstCudaGraphicsResource *resource = data->resource;
76 GstNvDec *nvdec = data->nvdec;
77 GstMapInfo map_info = GST_MAP_INFO_INIT;
78 GstGLBuffer *gl_buf_obj;
79
80 data->ret = FALSE;
81
82 if (!gst_cuda_context_push (nvdec->cuda_ctx)) {
83 GST_WARNING_OBJECT (nvdec, "failed to push CUDA context");
84 return;
85 }
86
87 if (gst_memory_map (mem, &map_info, GST_MAP_READ | GST_MAP_GL)) {
88 GstGLMemoryPBO *gl_mem = (GstGLMemoryPBO *) data->mem;
89 gl_buf_obj = gl_mem->pbo;
90
91 GST_LOG_OBJECT (nvdec,
92 "register glbuffer %d to CUDA resource", gl_buf_obj->id);
93
94 /* register resource without read/write only flags, since
95 * downstream CUDA elements (e.g., nvenc) might want to access
96 * this resource later. Instead, use map flags during map/unmap */
97 if (gst_cuda_graphics_resource_register_gl_buffer (resource,
98 gl_buf_obj->id, CU_GRAPHICS_REGISTER_FLAGS_NONE)) {
99 data->ret = TRUE;
100 } else {
101 GST_WARNING_OBJECT (nvdec, "failed to register memory");
102 }
103
104 gst_memory_unmap (mem, &map_info);
105 } else {
106 GST_WARNING_OBJECT (nvdec, "failed to map memory");
107 }
108
109 if (!gst_cuda_context_pop (NULL))
110 GST_WARNING_OBJECT (nvdec, "failed to unlock CUDA context");
111 }
112
113 static GstCudaGraphicsResource *
ensure_cuda_graphics_resource(GstMemory * mem,GstNvDec * nvdec)114 ensure_cuda_graphics_resource (GstMemory * mem, GstNvDec * nvdec)
115 {
116 GQuark quark;
117 GstCudaGraphicsResource *cgr_info;
118 GstNvDecRegisterResourceData data;
119
120 if (!gst_is_gl_memory_pbo (mem)) {
121 GST_WARNING_OBJECT (nvdec, "memory is not GL PBO memory, %s",
122 mem->allocator->mem_type);
123 return NULL;
124 }
125
126 quark = gst_cuda_quark_from_id (GST_CUDA_QUARK_GRAPHICS_RESOURCE);
127
128 cgr_info = gst_mini_object_get_qdata (GST_MINI_OBJECT (mem), quark);
129 if (!cgr_info) {
130 cgr_info = gst_cuda_graphics_resource_new (nvdec->cuda_ctx,
131 GST_OBJECT (GST_GL_BASE_MEMORY_CAST (mem)->context),
132 GST_CUDA_GRAPHICS_RESOURCE_GL_BUFFER);
133 data.mem = mem;
134 data.resource = cgr_info;
135 data.nvdec = nvdec;
136 gst_gl_context_thread_add ((GstGLContext *) cgr_info->graphics_context,
137 (GstGLContextThreadFunc) register_cuda_resource, &data);
138 if (!data.ret) {
139 GST_WARNING_OBJECT (nvdec, "could not register resource");
140 gst_cuda_graphics_resource_free (cgr_info);
141
142 return NULL;
143 }
144
145 gst_mini_object_set_qdata (GST_MINI_OBJECT (mem), quark, cgr_info,
146 (GDestroyNotify) gst_cuda_graphics_resource_free);
147 }
148
149 return cgr_info;
150 }
151 #endif /* HAVE_NVCODEC_GST_GL */
152
153 static gboolean gst_nvdec_open (GstVideoDecoder * decoder);
154 static gboolean gst_nvdec_start (GstVideoDecoder * decoder);
155 static gboolean gst_nvdec_stop (GstVideoDecoder * decoder);
156 static gboolean gst_nvdec_close (GstVideoDecoder * decoder);
157 static gboolean gst_nvdec_set_format (GstVideoDecoder * decoder,
158 GstVideoCodecState * state);
159 static GstFlowReturn gst_nvdec_handle_frame (GstVideoDecoder * decoder,
160 GstVideoCodecFrame * frame);
161 static gboolean gst_nvdec_decide_allocation (GstVideoDecoder * decoder,
162 GstQuery * query);
163 static void gst_nvdec_set_context (GstElement * element, GstContext * context);
164 static gboolean gst_nvdec_src_query (GstVideoDecoder * decoder,
165 GstQuery * query);
166 static gboolean gst_nvdec_flush (GstVideoDecoder * decoder);
167 static GstFlowReturn gst_nvdec_drain (GstVideoDecoder * decoder);
168 static GstFlowReturn gst_nvdec_finish (GstVideoDecoder * decoder);
169 static gboolean gst_nvdec_negotiate (GstVideoDecoder * decoder);
170 #ifdef HAVE_NVCODEC_GST_GL
171 static gboolean gst_nvdec_ensure_gl_context (GstNvDec * nvdec);
172 #endif
173
174 #define gst_nvdec_parent_class parent_class
175 G_DEFINE_ABSTRACT_TYPE (GstNvDec, gst_nvdec, GST_TYPE_VIDEO_DECODER);
176
177 static void
gst_nv_dec_set_property(GObject * object,guint prop_id,const GValue * value,GParamSpec * pspec)178 gst_nv_dec_set_property (GObject * object, guint prop_id, const GValue * value,
179 GParamSpec * pspec)
180 {
181 GstNvDec *nvdec = GST_NVDEC (object);
182
183 switch (prop_id) {
184 case PROP_MAX_DISPLAY_DELAY:
185 nvdec->max_display_delay = g_value_get_int (value);
186 break;
187 default:
188 G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
189 break;
190 }
191 }
192
193 static void
gst_nv_dec_get_property(GObject * object,guint prop_id,GValue * value,GParamSpec * pspec)194 gst_nv_dec_get_property (GObject * object, guint prop_id, GValue * value,
195 GParamSpec * pspec)
196 {
197 GstNvDec *nvdec = GST_NVDEC (object);
198
199 switch (prop_id) {
200 case PROP_MAX_DISPLAY_DELAY:
201 g_value_set_int (value, nvdec->max_display_delay);
202 break;
203 default:
204 G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
205 break;
206 }
207 }
208
209 static void
gst_nvdec_class_init(GstNvDecClass * klass)210 gst_nvdec_class_init (GstNvDecClass * klass)
211 {
212 GObjectClass *gobject_class = G_OBJECT_CLASS (klass);
213 GstVideoDecoderClass *video_decoder_class = GST_VIDEO_DECODER_CLASS (klass);
214 GstElementClass *element_class = GST_ELEMENT_CLASS (klass);
215
216 gobject_class->set_property = gst_nv_dec_set_property;
217 gobject_class->get_property = gst_nv_dec_get_property;
218
219 video_decoder_class->open = GST_DEBUG_FUNCPTR (gst_nvdec_open);
220 video_decoder_class->start = GST_DEBUG_FUNCPTR (gst_nvdec_start);
221 video_decoder_class->stop = GST_DEBUG_FUNCPTR (gst_nvdec_stop);
222 video_decoder_class->close = GST_DEBUG_FUNCPTR (gst_nvdec_close);
223 video_decoder_class->set_format = GST_DEBUG_FUNCPTR (gst_nvdec_set_format);
224 video_decoder_class->handle_frame =
225 GST_DEBUG_FUNCPTR (gst_nvdec_handle_frame);
226 video_decoder_class->decide_allocation =
227 GST_DEBUG_FUNCPTR (gst_nvdec_decide_allocation);
228 video_decoder_class->src_query = GST_DEBUG_FUNCPTR (gst_nvdec_src_query);
229 video_decoder_class->drain = GST_DEBUG_FUNCPTR (gst_nvdec_drain);
230 video_decoder_class->flush = GST_DEBUG_FUNCPTR (gst_nvdec_flush);
231 video_decoder_class->finish = GST_DEBUG_FUNCPTR (gst_nvdec_finish);
232 video_decoder_class->negotiate = GST_DEBUG_FUNCPTR (gst_nvdec_negotiate);
233
234 element_class->set_context = GST_DEBUG_FUNCPTR (gst_nvdec_set_context);
235 gst_type_mark_as_plugin_api (GST_TYPE_NVDEC, 0);
236
237 /**
238 * GstNvDec:max-display-delay:
239 *
240 * Since: 1.20
241 */
242 g_object_class_install_property (gobject_class, PROP_MAX_DISPLAY_DELAY,
243 g_param_spec_int ("max-display-delay", "Max Display Delay",
244 "Improves pipelining of decode with display, 0 means no delay "
245 "(auto = -1)",
246 -1, G_MAXINT, DEFAULT_MAX_DISPLAY_DELAY,
247 G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
248 }
249
250 static void
gst_nvdec_init(GstNvDec * nvdec)251 gst_nvdec_init (GstNvDec * nvdec)
252 {
253 nvdec->max_display_delay = DEFAULT_MAX_DISPLAY_DELAY;
254 gst_video_decoder_set_packetized (GST_VIDEO_DECODER (nvdec), TRUE);
255 gst_video_decoder_set_needs_format (GST_VIDEO_DECODER (nvdec), TRUE);
256 }
257
258 static cudaVideoSurfaceFormat
get_cuda_surface_format_from_gst(GstVideoFormat format)259 get_cuda_surface_format_from_gst (GstVideoFormat format)
260 {
261 switch (format) {
262 case GST_VIDEO_FORMAT_NV12:
263 return cudaVideoSurfaceFormat_NV12;
264 case GST_VIDEO_FORMAT_P010_10LE:
265 case GST_VIDEO_FORMAT_P010_10BE:
266 case GST_VIDEO_FORMAT_P016_LE:
267 case GST_VIDEO_FORMAT_P016_BE:
268 return cudaVideoSurfaceFormat_P016;
269 case GST_VIDEO_FORMAT_Y444:
270 return cudaVideoSurfaceFormat_YUV444;
271 case GST_VIDEO_FORMAT_Y444_16LE:
272 case GST_VIDEO_FORMAT_Y444_16BE:
273 return cudaVideoSurfaceFormat_YUV444_16Bit;
274 default:
275 g_assert_not_reached ();
276 break;
277 }
278
279 return cudaVideoSurfaceFormat_NV12;
280 }
281
282 static guint
calculate_num_decode_surface(cudaVideoCodec codec,guint width,guint height)283 calculate_num_decode_surface (cudaVideoCodec codec, guint width, guint height)
284 {
285 switch (codec) {
286 case cudaVideoCodec_VP9:
287 return 12;
288 case cudaVideoCodec_H264:
289 case cudaVideoCodec_H264_SVC:
290 case cudaVideoCodec_H264_MVC:
291 return 20;
292 case cudaVideoCodec_HEVC:{
293 gint max_dpb_size;
294 gint MaxLumaPS;
295 const gint MaxDpbPicBuf = 6;
296 gint PicSizeInSamplesY;
297
298 /* A.4.1 */
299 MaxLumaPS = 35651584;
300 PicSizeInSamplesY = width * height;
301 if (PicSizeInSamplesY <= (MaxLumaPS >> 2))
302 max_dpb_size = MaxDpbPicBuf * 4;
303 else if (PicSizeInSamplesY <= (MaxLumaPS >> 1))
304 max_dpb_size = MaxDpbPicBuf * 2;
305 else if (PicSizeInSamplesY <= ((3 * MaxLumaPS) >> 2))
306 max_dpb_size = (MaxDpbPicBuf * 4) / 3;
307 else
308 max_dpb_size = MaxDpbPicBuf;
309
310 max_dpb_size = MIN (max_dpb_size, 16);
311
312 return max_dpb_size + 4;
313 }
314 default:
315 break;
316 }
317
318 return 8;
319 }
320
321 static guint
gst_nvdec_get_max_display_delay(GstNvDec * nvdec)322 gst_nvdec_get_max_display_delay (GstNvDec * nvdec)
323 {
324 return nvdec->max_display_delay >= 0 ? nvdec->max_display_delay :
325 (nvdec->is_live ? 0 : 4);
326 }
327
328 static gint64
gst_nvdec_get_latency(GstNvDec * nvdec)329 gst_nvdec_get_latency (GstNvDec * nvdec)
330 {
331 gint fps_n, fps_d;
332
333 if (!nvdec->input_state)
334 return 0;
335 fps_n = GST_VIDEO_INFO_FPS_N (&nvdec->input_state->info);
336 fps_d = GST_VIDEO_INFO_FPS_D (&nvdec->input_state->info);
337
338 /* We assume 25 fps if the input framerate is invalid */
339 if (fps_n < 1 || fps_d < 1) {
340 fps_n = 25;
341 fps_d = 1;
342 }
343
344 return gst_util_uint64_scale_int ((nvdec->num_decode_surface +
345 gst_nvdec_get_max_display_delay (nvdec)) * GST_SECOND, fps_d, fps_n);
346 }
347
348 /* 0: fail, 1: succeeded, > 1: override dpb size of parser
349 * (set by CUVIDPARSERPARAMS::ulMaxNumDecodeSurfaces while creating parser) */
350 static gint CUDAAPI
parser_sequence_callback(GstNvDec * nvdec,CUVIDEOFORMAT * format)351 parser_sequence_callback (GstNvDec * nvdec, CUVIDEOFORMAT * format)
352 {
353 guint width, height;
354 CUVIDDECODECREATEINFO create_info = { 0, };
355 GstVideoFormat out_format;
356 GstVideoInfo *in_info = &nvdec->input_state->info;
357 GstVideoInfo *out_info = &nvdec->out_info;
358 GstVideoInfo prev_out_info = *out_info;
359 GstCudaContext *ctx = nvdec->cuda_ctx;
360 GstStructure *in_s = NULL;
361 gboolean updata = FALSE;
362 guint major_api_ver = 0;
363 guint64 curr_latency, old_latency;
364
365 old_latency = gst_nvdec_get_latency (nvdec);
366 width = format->display_area.right - format->display_area.left;
367 height = format->display_area.bottom - format->display_area.top;
368
369 switch (format->chroma_format) {
370 case cudaVideoChromaFormat_444:
371 if (format->bit_depth_luma_minus8 == 0) {
372 out_format = GST_VIDEO_FORMAT_Y444;
373 } else if (format->bit_depth_luma_minus8 == 2 ||
374 format->bit_depth_luma_minus8 == 4) {
375 #if G_BYTE_ORDER == G_LITTLE_ENDIAN
376 out_format = GST_VIDEO_FORMAT_Y444_16LE;
377 #else
378 out_format = GST_VIDEO_FORMAT_Y444_16BE;
379 #endif
380 } else {
381 GST_ERROR_OBJECT (nvdec, "Unknown 4:4:4 format bitdepth %d",
382 format->bit_depth_luma_minus8 + 8);
383
384 nvdec->last_ret = GST_FLOW_NOT_NEGOTIATED;
385 return 0;
386 }
387 break;
388 case cudaVideoChromaFormat_420:
389 if (format->bit_depth_luma_minus8 == 0) {
390 out_format = GST_VIDEO_FORMAT_NV12;
391 } else if (format->bit_depth_luma_minus8 == 2) {
392 #if G_BYTE_ORDER == G_LITTLE_ENDIAN
393 out_format = GST_VIDEO_FORMAT_P010_10LE;
394 #else
395 out_format = GST_VIDEO_FORMAT_P010_10BE;
396 #endif
397 } else if (format->bit_depth_luma_minus8 == 4) {
398 #if G_BYTE_ORDER == G_LITTLE_ENDIAN
399 out_format = GST_VIDEO_FORMAT_P016_LE;
400 #else
401 out_format = GST_VIDEO_FORMAT_P016_BE;
402 #endif
403 } else {
404 GST_ERROR_OBJECT (nvdec, "Unknown 4:2:0 format bitdepth %d",
405 format->bit_depth_luma_minus8 + 8);
406
407 nvdec->last_ret = GST_FLOW_NOT_NEGOTIATED;
408 return 0;
409 }
410 break;
411 default:
412 GST_ERROR_OBJECT (nvdec, "unhandled chroma format %d, bitdepth %d",
413 format->chroma_format, format->bit_depth_luma_minus8 + 8);
414
415 nvdec->last_ret = GST_FLOW_NOT_NEGOTIATED;
416 return 0;
417 }
418
419 GST_DEBUG_OBJECT (nvdec,
420 "out format: %s", gst_video_format_to_string (out_format));
421
422 GST_DEBUG_OBJECT (nvdec, "width: %u, height: %u", width, height);
423
424 gst_video_info_set_format (out_info, out_format, width, height);
425 GST_VIDEO_INFO_FPS_N (out_info) = GST_VIDEO_INFO_FPS_N (in_info);
426 GST_VIDEO_INFO_FPS_D (out_info) = GST_VIDEO_INFO_FPS_D (in_info);
427
428 if (GST_VIDEO_INFO_FPS_N (out_info) < 1 ||
429 GST_VIDEO_INFO_FPS_D (out_info) < 1) {
430 GST_VIDEO_INFO_FPS_N (out_info) = format->frame_rate.numerator;
431 GST_VIDEO_INFO_FPS_D (out_info) = MAX (1, format->frame_rate.denominator);
432 }
433
434 GST_LOG_OBJECT (nvdec,
435 "Reading colorimetry information full-range %d matrix %d transfer %d primaries %d",
436 format->video_signal_description.video_full_range_flag,
437 format->video_signal_description.matrix_coefficients,
438 format->video_signal_description.transfer_characteristics,
439 format->video_signal_description.color_primaries);
440
441 if (nvdec->input_state->caps)
442 in_s = gst_caps_get_structure (nvdec->input_state->caps, 0);
443
444 /* Set colorimetry when upstream did not provide it */
445 if (in_s && !gst_structure_has_field (in_s, "colorimetry")) {
446 GstVideoColorimetry colorimetry = { 0, };
447
448 if (format->video_signal_description.video_full_range_flag)
449 colorimetry.range = GST_VIDEO_COLOR_RANGE_0_255;
450 else
451 colorimetry.range = GST_VIDEO_COLOR_RANGE_16_235;
452
453 colorimetry.primaries =
454 gst_video_color_primaries_from_iso
455 (format->video_signal_description.color_primaries);
456
457 colorimetry.transfer =
458 gst_video_transfer_function_from_iso
459 (format->video_signal_description.transfer_characteristics);
460
461 colorimetry.matrix =
462 gst_video_color_matrix_from_iso
463 (format->video_signal_description.matrix_coefficients);
464
465 /* Use a colorimetry having at least one valid colorimetry entry,
466 * because we don't know whether the returned
467 * colorimetry (by nvdec) was actually parsed information or not.
468 * Otherwise let GstVideoInfo handle it with default colorimetry */
469 if (colorimetry.primaries != GST_VIDEO_COLOR_PRIMARIES_UNKNOWN ||
470 colorimetry.transfer != GST_VIDEO_TRANSFER_UNKNOWN ||
471 colorimetry.matrix != GST_VIDEO_COLOR_MATRIX_UNKNOWN) {
472 GST_DEBUG_OBJECT (nvdec,
473 "Found valid colorimetry, update output colorimetry");
474 out_info->colorimetry = colorimetry;
475 }
476 } else {
477 out_info->colorimetry = in_info->colorimetry;
478 }
479
480 if (format->progressive_sequence) {
481 out_info->interlace_mode = GST_VIDEO_INTERLACE_MODE_PROGRESSIVE;
482
483 /* nvdec doesn't seem to deal with interlacing with hevc so rely
484 * on upstream's value */
485 if (format->codec == cudaVideoCodec_HEVC) {
486 out_info->interlace_mode = in_info->interlace_mode;
487 }
488 } else {
489 out_info->interlace_mode = GST_VIDEO_INTERLACE_MODE_MIXED;
490 }
491
492 if (gst_cuvid_get_api_version (&major_api_ver, NULL) && major_api_ver >= 9) {
493 /* min_num_decode_surfaces was introduced in nvcodec sdk 9.0 header */
494 nvdec->num_decode_surface = format->min_num_decode_surfaces;
495
496 GST_DEBUG_OBJECT (nvdec,
497 "Num decode surface: %d", nvdec->num_decode_surface);
498 } else {
499 nvdec->num_decode_surface =
500 calculate_num_decode_surface (format->codec, width, height);
501
502 GST_DEBUG_OBJECT (nvdec,
503 "Calculated num decode surface: %d", nvdec->num_decode_surface);
504 }
505
506 /* Update the latency if it has changed */
507 curr_latency = gst_nvdec_get_latency (nvdec);
508 if (old_latency != curr_latency)
509 gst_video_decoder_set_latency (GST_VIDEO_DECODER (nvdec), curr_latency,
510 curr_latency);
511
512 if (!nvdec->decoder || !gst_video_info_is_equal (out_info, &prev_out_info)) {
513 updata = TRUE;
514
515 if (!gst_cuda_context_push (ctx)) {
516 GST_ERROR_OBJECT (nvdec, "failed to lock CUDA context");
517 goto error;
518 }
519
520 if (nvdec->decoder) {
521 GST_DEBUG_OBJECT (nvdec, "destroying decoder");
522 if (!gst_cuda_result (CuvidDestroyDecoder (nvdec->decoder))) {
523 GST_ERROR_OBJECT (nvdec, "failed to destroy decoder");
524 goto error;
525 } else
526 nvdec->decoder = NULL;
527 }
528
529 GST_DEBUG_OBJECT (nvdec, "creating decoder");
530 create_info.ulWidth = width;
531 create_info.ulHeight = height;
532 create_info.ulNumDecodeSurfaces = nvdec->num_decode_surface;
533 create_info.CodecType = format->codec;
534 create_info.ChromaFormat = format->chroma_format;
535 create_info.ulCreationFlags = cudaVideoCreate_Default;
536 create_info.display_area.left = format->display_area.left;
537 create_info.display_area.top = format->display_area.top;
538 create_info.display_area.right = format->display_area.right;
539 create_info.display_area.bottom = format->display_area.bottom;
540 create_info.OutputFormat = get_cuda_surface_format_from_gst (out_format);
541 create_info.bitDepthMinus8 = format->bit_depth_luma_minus8;
542 create_info.DeinterlaceMode = cudaVideoDeinterlaceMode_Weave;
543 create_info.ulTargetWidth = width;
544 create_info.ulTargetHeight = height;
545 create_info.ulNumOutputSurfaces = 1;
546 create_info.target_rect.left = 0;
547 create_info.target_rect.top = 0;
548 create_info.target_rect.right = width;
549 create_info.target_rect.bottom = height;
550
551 if (nvdec->decoder
552 || !gst_cuda_result (CuvidCreateDecoder (&nvdec->decoder,
553 &create_info))) {
554 GST_ERROR_OBJECT (nvdec, "failed to create decoder");
555 goto error;
556 }
557
558 if (!gst_cuda_context_pop (NULL)) {
559 GST_ERROR_OBJECT (nvdec, "failed to unlock CUDA context");
560 goto error;
561 }
562 }
563
564 if (!gst_pad_has_current_caps (GST_VIDEO_DECODER_SRC_PAD (nvdec)) || updata) {
565 if (!gst_video_decoder_negotiate (GST_VIDEO_DECODER (nvdec))) {
566 nvdec->last_ret = GST_FLOW_NOT_NEGOTIATED;
567 return 0;
568 }
569 }
570
571 return nvdec->num_decode_surface;
572
573 error:
574 nvdec->last_ret = GST_FLOW_ERROR;
575 return 0;
576 }
577
578 static gboolean
gst_nvdec_negotiate(GstVideoDecoder * decoder)579 gst_nvdec_negotiate (GstVideoDecoder * decoder)
580 {
581 GstNvDec *nvdec = GST_NVDEC (decoder);
582 GstVideoCodecState *state;
583 GstVideoInfo *vinfo;
584 GstVideoInfo *out_info = &nvdec->out_info;
585 gboolean ret;
586
587 GST_DEBUG_OBJECT (nvdec, "negotiate");
588
589 state = gst_video_decoder_set_output_state (GST_VIDEO_DECODER (nvdec),
590 GST_VIDEO_INFO_FORMAT (out_info), GST_VIDEO_INFO_WIDTH (out_info),
591 GST_VIDEO_INFO_HEIGHT (out_info), nvdec->input_state);
592 vinfo = &state->info;
593
594 /* update output info with CUvidparser provided one */
595 vinfo->interlace_mode = out_info->interlace_mode;
596 vinfo->fps_n = out_info->fps_n;
597 vinfo->fps_d = out_info->fps_d;
598
599 state->caps = gst_video_info_to_caps (&state->info);
600 nvdec->mem_type = GST_NVDEC_MEM_TYPE_SYSTEM;
601
602 {
603 GstCaps *caps;
604 caps = gst_pad_get_allowed_caps (GST_VIDEO_DECODER_SRC_PAD (nvdec));
605 GST_DEBUG_OBJECT (nvdec, "Allowed caps %" GST_PTR_FORMAT, caps);
606
607 if (!caps || gst_caps_is_any (caps)) {
608 GST_DEBUG_OBJECT (nvdec,
609 "cannot determine output format, use system memory");
610 } else {
611 GstCapsFeatures *features;
612 guint size = gst_caps_get_size (caps);
613 guint i;
614 gboolean have_cuda = FALSE;
615 gboolean have_gl = FALSE;
616
617 for (i = 0; i < size; i++) {
618 features = gst_caps_get_features (caps, i);
619 if (features && gst_caps_features_contains (features,
620 GST_CAPS_FEATURE_MEMORY_CUDA_MEMORY)) {
621 GST_DEBUG_OBJECT (nvdec, "found CUDA memory feature");
622 have_cuda = TRUE;
623 break;
624 }
625 #ifdef HAVE_NVCODEC_GST_GL
626 if (nvdec->gl_display &&
627 features && gst_caps_features_contains (features,
628 GST_CAPS_FEATURE_MEMORY_GL_MEMORY)) {
629 GST_DEBUG_OBJECT (nvdec, "found GL memory feature");
630 have_gl = TRUE;
631 }
632 #endif
633 }
634
635 if (have_cuda)
636 nvdec->mem_type = GST_NVDEC_MEM_TYPE_CUDA;
637 else if (have_gl)
638 nvdec->mem_type = GST_NVDEC_MEM_TYPE_GL;
639 }
640 gst_clear_caps (&caps);
641 }
642
643 #ifdef HAVE_NVCODEC_GST_GL
644 if (nvdec->mem_type == GST_NVDEC_MEM_TYPE_GL &&
645 !gst_nvdec_ensure_gl_context (nvdec)) {
646 GST_WARNING_OBJECT (nvdec,
647 "OpenGL context is not CUDA-compatible, fallback to system memory");
648 nvdec->mem_type = GST_NVDEC_MEM_TYPE_SYSTEM;
649 }
650 #endif
651
652 switch (nvdec->mem_type) {
653 case GST_NVDEC_MEM_TYPE_CUDA:
654 GST_DEBUG_OBJECT (nvdec, "use cuda memory");
655 gst_caps_set_features (state->caps, 0,
656 gst_caps_features_new (GST_CAPS_FEATURE_MEMORY_CUDA_MEMORY, NULL));
657 break;
658 #ifdef HAVE_NVCODEC_GST_GL
659 case GST_NVDEC_MEM_TYPE_GL:
660 GST_DEBUG_OBJECT (nvdec, "use gl memory");
661 gst_caps_set_features (state->caps, 0,
662 gst_caps_features_new (GST_CAPS_FEATURE_MEMORY_GL_MEMORY, NULL));
663 gst_caps_set_simple (state->caps, "texture-target", G_TYPE_STRING,
664 "2D", NULL);
665 break;
666 #endif
667 default:
668 GST_DEBUG_OBJECT (nvdec, "use system memory");
669 break;
670 }
671
672 if (nvdec->output_state)
673 gst_video_codec_state_unref (nvdec->output_state);
674
675 nvdec->output_state = state;
676
677 ret = GST_VIDEO_DECODER_CLASS (parent_class)->negotiate (decoder);
678
679 if (!ret) {
680 GST_ERROR_OBJECT (nvdec, "failed to negotiate with downstream");
681 nvdec->last_ret = GST_FLOW_NOT_NEGOTIATED;
682 }
683
684 return ret;
685 }
686
687 static gboolean CUDAAPI
parser_decode_callback(GstNvDec * nvdec,CUVIDPICPARAMS * params)688 parser_decode_callback (GstNvDec * nvdec, CUVIDPICPARAMS * params)
689 {
690 GList *iter, *pending_frames;
691 GstCudaContext *ctx = nvdec->cuda_ctx;
692
693 GST_LOG_OBJECT (nvdec, "picture index: %u", params->CurrPicIdx);
694
695 if (!gst_cuda_context_push (ctx)) {
696 GST_ERROR_OBJECT (nvdec, "failed to lock CUDA context");
697 goto error;
698 }
699
700 if (!gst_cuda_result (CuvidDecodePicture (nvdec->decoder, params))) {
701 GST_ERROR_OBJECT (nvdec, "failed to decode picture");
702 goto error;
703 }
704
705 if (!gst_cuda_context_pop (NULL)) {
706 GST_ERROR_OBJECT (nvdec, "failed to unlock CUDA context");
707 goto error;
708 }
709
710 pending_frames = gst_video_decoder_get_frames (GST_VIDEO_DECODER (nvdec));
711
712 /* NOTE: this decode callback could be invoked multiple times for
713 * one cuvidParseVideoData() call. Most likely it can be related to "decode only"
714 * frame of VPX codec but no document available.
715 * In that case, the last decoded frame seems to be displayed */
716
717 for (iter = pending_frames; iter; iter = g_list_next (iter)) {
718 guint id;
719 GstVideoCodecFrame *frame = (GstVideoCodecFrame *) iter->data;
720 gboolean set_data = FALSE;
721
722 id = GPOINTER_TO_UINT (gst_video_codec_frame_get_user_data (frame));
723 if (G_UNLIKELY (nvdec->state == GST_NVDEC_STATE_DECODE)) {
724 if (id) {
725 GST_LOG_OBJECT (nvdec, "reset the last user data");
726 set_data = TRUE;
727 }
728 } else if (!id) {
729 set_data = TRUE;
730 }
731
732 if (set_data) {
733 gst_video_codec_frame_set_user_data (frame,
734 GUINT_TO_POINTER (params->CurrPicIdx + 1), NULL);
735 break;
736 }
737 }
738
739 nvdec->state = GST_NVDEC_STATE_DECODE;
740
741 g_list_free_full (pending_frames,
742 (GDestroyNotify) gst_video_codec_frame_unref);
743
744 return TRUE;
745
746 error:
747 nvdec->last_ret = GST_FLOW_ERROR;
748 return FALSE;
749 }
750
751 static gboolean CUDAAPI
parser_display_callback(GstNvDec * nvdec,CUVIDPARSERDISPINFO * dispinfo)752 parser_display_callback (GstNvDec * nvdec, CUVIDPARSERDISPINFO * dispinfo)
753 {
754 GList *iter, *pending_frames;
755 GstVideoCodecFrame *frame = NULL;
756 GstBuffer *output_buffer = NULL;
757 GstFlowReturn ret = GST_FLOW_OK;
758 gboolean copy_ret = FALSE;
759
760 GST_LOG_OBJECT (nvdec, "picture index: %u", dispinfo->picture_index);
761
762 pending_frames = gst_video_decoder_get_frames (GST_VIDEO_DECODER (nvdec));
763 for (iter = pending_frames; iter; iter = g_list_next (iter)) {
764 guint id;
765 GstVideoCodecFrame *tmp = (GstVideoCodecFrame *) iter->data;
766
767 id = GPOINTER_TO_UINT (gst_video_codec_frame_get_user_data (tmp));
768 if (id == dispinfo->picture_index + 1) {
769 frame = gst_video_codec_frame_ref (tmp);
770 break;
771 }
772 }
773 g_list_free_full (pending_frames,
774 (GDestroyNotify) gst_video_codec_frame_unref);
775
776 if (G_UNLIKELY (frame == NULL)) {
777 GST_WARNING_OBJECT (nvdec, "no frame for picture index %u",
778 dispinfo->picture_index);
779
780 output_buffer =
781 gst_video_decoder_allocate_output_buffer (GST_VIDEO_DECODER (nvdec));
782
783 if (!output_buffer) {
784 GST_ERROR_OBJECT (nvdec, "Couldn't allocate output buffer");
785 nvdec->last_ret = GST_FLOW_ERROR;
786 return FALSE;
787 }
788
789 GST_BUFFER_PTS (output_buffer) = dispinfo->timestamp;
790 GST_BUFFER_DTS (output_buffer) = GST_CLOCK_TIME_NONE;
791 /* assume buffer duration from framerate */
792 GST_BUFFER_DURATION (output_buffer) =
793 gst_util_uint64_scale (GST_SECOND,
794 GST_VIDEO_INFO_FPS_D (&nvdec->out_info),
795 GST_VIDEO_INFO_FPS_N (&nvdec->out_info));
796 } else {
797 ret = gst_video_decoder_allocate_output_frame (GST_VIDEO_DECODER (nvdec),
798 frame);
799
800 if (ret != GST_FLOW_OK) {
801 GST_WARNING_OBJECT (nvdec, "failed to allocate output frame");
802 nvdec->last_ret = ret;
803 return FALSE;
804 }
805
806 output_buffer = frame->output_buffer;
807
808 if (dispinfo->timestamp != frame->pts) {
809 GST_INFO_OBJECT (nvdec,
810 "timestamp mismatch, diff: %" GST_STIME_FORMAT,
811 GST_STIME_ARGS (GST_CLOCK_DIFF (dispinfo->timestamp, frame->pts)));
812 }
813 }
814
815 #ifdef HAVE_NVCODEC_GST_GL
816 if (nvdec->mem_type == GST_NVDEC_MEM_TYPE_GL) {
817 copy_ret = gst_nvdec_copy_device_to_gl (nvdec, dispinfo, output_buffer);
818
819 /* FIXME: This is the case where OpenGL context of downstream glbufferpool
820 * belongs to non-nvidia (or different device).
821 * There should be enhancement to ensure nvdec has compatible OpenGL context
822 */
823 if (!copy_ret) {
824 GST_WARNING_OBJECT (nvdec,
825 "Couldn't copy frame to GL memory, fallback to system memory");
826 nvdec->mem_type = GST_NVDEC_MEM_TYPE_SYSTEM;
827 }
828 }
829
830 if (!copy_ret)
831 #endif
832 {
833 copy_ret = gst_nvdec_copy_device_to_memory (nvdec, dispinfo, output_buffer);
834 }
835
836 if (!copy_ret) {
837 GST_ERROR_OBJECT (nvdec, "failed to copy decoded picture to output buffer");
838 nvdec->last_ret = GST_FLOW_ERROR;
839
840 if (frame)
841 gst_video_decoder_drop_frame (GST_VIDEO_DECODER (nvdec), frame);
842 else
843 gst_buffer_unref (output_buffer);
844
845 return FALSE;
846 }
847
848 if (!dispinfo->progressive_frame) {
849 GST_BUFFER_FLAG_SET (output_buffer, GST_VIDEO_BUFFER_FLAG_INTERLACED);
850
851 if (dispinfo->top_field_first) {
852 GST_BUFFER_FLAG_SET (output_buffer, GST_VIDEO_BUFFER_FLAG_TFF);
853 }
854
855 if (dispinfo->repeat_first_field == -1) {
856 GST_BUFFER_FLAG_SET (output_buffer, GST_VIDEO_BUFFER_FLAG_ONEFIELD);
857 } else {
858 GST_BUFFER_FLAG_SET (output_buffer, GST_VIDEO_BUFFER_FLAG_RFF);
859 }
860 }
861
862 if (frame) {
863 ret = gst_video_decoder_finish_frame (GST_VIDEO_DECODER (nvdec), frame);
864 } else {
865 ret = gst_pad_push (GST_VIDEO_DECODER_SRC_PAD (nvdec), output_buffer);
866 }
867
868 if (ret != GST_FLOW_OK) {
869 GST_DEBUG_OBJECT (nvdec, "failed to finish frame %s",
870 gst_flow_get_name (ret));
871 nvdec->last_ret = ret;
872 return FALSE;
873 }
874
875 return TRUE;
876 }
877
878 static gboolean
gst_nvdec_open(GstVideoDecoder * decoder)879 gst_nvdec_open (GstVideoDecoder * decoder)
880 {
881 GstNvDec *nvdec = GST_NVDEC (decoder);
882 GstNvDecClass *klass = GST_NVDEC_GET_CLASS (nvdec);
883 CUresult cuda_ret;
884
885 GST_DEBUG_OBJECT (nvdec, "creating CUDA context");
886
887 if (!gst_cuda_ensure_element_context (GST_ELEMENT_CAST (decoder),
888 klass->cuda_device_id, &nvdec->cuda_ctx)) {
889 GST_ERROR_OBJECT (nvdec, "failed to create CUDA context");
890 return FALSE;
891 }
892
893 if (gst_cuda_context_push (nvdec->cuda_ctx)) {
894 cuda_ret = CuStreamCreate (&nvdec->cuda_stream, CU_STREAM_DEFAULT);
895 if (!gst_cuda_result (cuda_ret)) {
896 GST_WARNING_OBJECT (nvdec,
897 "Could not create CUDA stream, will use default stream");
898 nvdec->cuda_stream = NULL;
899 }
900 gst_cuda_context_pop (NULL);
901 }
902 #if HAVE_NVCODEC_GST_GL
903 gst_gl_ensure_element_data (GST_ELEMENT (nvdec),
904 &nvdec->gl_display, &nvdec->other_gl_context);
905 if (nvdec->gl_display)
906 gst_gl_display_filter_gl_api (GST_GL_DISPLAY (nvdec->gl_display),
907 SUPPORTED_GL_APIS);
908 #endif
909
910 return TRUE;
911 }
912
913 static gboolean
gst_nvdec_start(GstVideoDecoder * decoder)914 gst_nvdec_start (GstVideoDecoder * decoder)
915 {
916 GstNvDec *nvdec = GST_NVDEC (decoder);
917 GstNvDecClass *klass = GST_NVDEC_GET_CLASS (nvdec);
918
919 nvdec->state = GST_NVDEC_STATE_INIT;
920 nvdec->last_ret = GST_FLOW_OK;
921 gst_video_info_init (&nvdec->out_info);
922
923 if (klass->codec_type == cudaVideoCodec_H264)
924 nvdec->h264_parser = gst_h264_nal_parser_new ();
925 else if (klass->codec_type == cudaVideoCodec_HEVC)
926 nvdec->h265_parser = gst_h265_parser_new ();
927
928 return TRUE;
929 }
930
931 static gboolean
maybe_destroy_decoder_and_parser(GstNvDec * nvdec)932 maybe_destroy_decoder_and_parser (GstNvDec * nvdec)
933 {
934 gboolean ret = TRUE;
935
936 if (!gst_cuda_context_push (nvdec->cuda_ctx)) {
937 GST_ERROR_OBJECT (nvdec, "failed to lock CUDA context");
938 return FALSE;
939 }
940
941 if (nvdec->decoder) {
942 GST_DEBUG_OBJECT (nvdec, "destroying decoder");
943 ret = gst_cuda_result (CuvidDestroyDecoder (nvdec->decoder));
944 nvdec->decoder = NULL;
945
946 if (!ret)
947 GST_ERROR_OBJECT (nvdec, "failed to destroy decoder");
948 }
949
950 if (nvdec->parser) {
951 GST_DEBUG_OBJECT (nvdec, "destroying parser");
952 if (!gst_cuda_result (CuvidDestroyVideoParser (nvdec->parser))) {
953 GST_ERROR_OBJECT (nvdec, "failed to destroy parser");
954 ret = FALSE;
955 }
956 nvdec->parser = NULL;
957 }
958
959 if (!gst_cuda_context_pop (NULL)) {
960 GST_WARNING_OBJECT (nvdec, "failed to pop CUDA context");
961 }
962
963 return ret;
964 }
965
966 static void
gst_nvdec_clear_codec_data(GstNvDec * self)967 gst_nvdec_clear_codec_data (GstNvDec * self)
968 {
969 GstNvDecClass *klass = GST_NVDEC_GET_CLASS (self);
970 guint i;
971
972 if (klass->codec_type == cudaVideoCodec_HEVC) {
973 for (i = 0; i < G_N_ELEMENTS (self->vps_nals); i++) {
974 gst_clear_buffer (&self->vps_nals[i]);
975 }
976 }
977
978 if (klass->codec_type == cudaVideoCodec_HEVC ||
979 klass->codec_type == cudaVideoCodec_H264) {
980 for (i = 0; i < G_N_ELEMENTS (self->sps_nals); i++) {
981 gst_clear_buffer (&self->sps_nals[i]);
982 }
983
984 for (i = 0; i < G_N_ELEMENTS (self->pps_nals); i++) {
985 gst_clear_buffer (&self->pps_nals[i]);
986 }
987 }
988
989 gst_clear_buffer (&self->codec_data);
990
991 self->need_codec_data = TRUE;
992 }
993
994 static gboolean
gst_nvdec_stop(GstVideoDecoder * decoder)995 gst_nvdec_stop (GstVideoDecoder * decoder)
996 {
997 GstNvDec *nvdec = GST_NVDEC (decoder);
998
999 GST_DEBUG_OBJECT (nvdec, "stop");
1000
1001 if (!maybe_destroy_decoder_and_parser (nvdec))
1002 return FALSE;
1003
1004 #ifdef HAVE_NVCODEC_GST_GL
1005 gst_clear_object (&nvdec->gl_context);
1006 gst_clear_object (&nvdec->other_gl_context);
1007 gst_clear_object (&nvdec->gl_display);
1008 #endif
1009
1010 g_clear_pointer (&nvdec->input_state, gst_video_codec_state_unref);
1011 g_clear_pointer (&nvdec->output_state, gst_video_codec_state_unref);
1012
1013 g_clear_pointer (&nvdec->h264_parser, gst_h264_nal_parser_free);
1014 g_clear_pointer (&nvdec->h265_parser, gst_h265_parser_free);
1015
1016 gst_nvdec_clear_codec_data (nvdec);
1017
1018 return TRUE;
1019 }
1020
1021 static gboolean
gst_nvdec_close(GstVideoDecoder * decoder)1022 gst_nvdec_close (GstVideoDecoder * decoder)
1023 {
1024 GstNvDec *nvdec = GST_NVDEC (decoder);
1025
1026 if (nvdec->cuda_ctx && nvdec->cuda_stream) {
1027 if (gst_cuda_context_push (nvdec->cuda_ctx)) {
1028 gst_cuda_result (CuStreamDestroy (nvdec->cuda_stream));
1029 gst_cuda_context_pop (NULL);
1030 }
1031 }
1032
1033 gst_clear_object (&nvdec->cuda_ctx);
1034 nvdec->cuda_stream = NULL;
1035
1036 return TRUE;
1037 }
1038
1039 static gboolean
gst_nvdec_set_format(GstVideoDecoder * decoder,GstVideoCodecState * state)1040 gst_nvdec_set_format (GstVideoDecoder * decoder, GstVideoCodecState * state)
1041 {
1042 GstNvDec *nvdec = GST_NVDEC (decoder);
1043 GstNvDecClass *klass = GST_NVDEC_GET_CLASS (decoder);
1044 CUVIDPARSERPARAMS parser_params = { 0, };
1045 GstQuery *query;
1046 gboolean ret = TRUE;
1047
1048 GST_DEBUG_OBJECT (nvdec, "set format");
1049
1050 if (nvdec->input_state)
1051 gst_video_codec_state_unref (nvdec->input_state);
1052
1053 nvdec->input_state = gst_video_codec_state_ref (state);
1054
1055 if (!maybe_destroy_decoder_and_parser (nvdec))
1056 return FALSE;
1057
1058 /* Check if pipeline is live */
1059 nvdec->is_live = FALSE;
1060 query = gst_query_new_latency ();
1061 if (gst_pad_peer_query (GST_VIDEO_DECODER_SINK_PAD (decoder), query))
1062 gst_query_parse_latency (query, &nvdec->is_live, NULL, NULL);
1063 gst_query_unref (query);
1064
1065 parser_params.CodecType = klass->codec_type;
1066 /* ulMaxNumDecodeSurfaces will be updated by the return value of
1067 * SequenceCallback */
1068 parser_params.ulMaxNumDecodeSurfaces = 1;
1069 parser_params.ulErrorThreshold = 100;
1070 parser_params.ulMaxDisplayDelay = gst_nvdec_get_max_display_delay (nvdec);
1071 parser_params.ulClockRate = GST_SECOND;
1072 parser_params.pUserData = nvdec;
1073 parser_params.pfnSequenceCallback =
1074 (PFNVIDSEQUENCECALLBACK) parser_sequence_callback;
1075 parser_params.pfnDecodePicture =
1076 (PFNVIDDECODECALLBACK) parser_decode_callback;
1077 parser_params.pfnDisplayPicture =
1078 (PFNVIDDISPLAYCALLBACK) parser_display_callback;
1079
1080 gst_cuda_context_push (nvdec->cuda_ctx);
1081 GST_DEBUG_OBJECT (nvdec, "creating parser");
1082 if (!gst_cuda_result (CuvidCreateVideoParser (&nvdec->parser,
1083 &parser_params))) {
1084 GST_ERROR_OBJECT (nvdec, "failed to create parser");
1085 ret = FALSE;
1086 }
1087
1088 gst_cuda_context_pop (NULL);
1089
1090 /* store codec data */
1091 gst_nvdec_clear_codec_data (nvdec);
1092
1093 if (ret && nvdec->input_state->caps) {
1094 GstStructure *str;
1095
1096 str = gst_caps_get_structure (nvdec->input_state->caps, 0);
1097
1098 if (klass->codec_type == cudaVideoCodec_MPEG4) {
1099 const GValue *codec_data_value;
1100 codec_data_value = gst_structure_get_value (str, "codec_data");
1101 if (codec_data_value && GST_VALUE_HOLDS_BUFFER (codec_data_value)) {
1102 GstBuffer *codec_data = gst_value_get_buffer (codec_data_value);
1103 gst_buffer_replace (&nvdec->codec_data, codec_data);
1104 }
1105 }
1106
1107 /* For all CODEC we get complete picture ... */
1108 nvdec->recv_complete_picture = TRUE;
1109
1110 /* Except for JPEG, for which it depends on the caps */
1111 if (klass->codec_type == cudaVideoCodec_JPEG) {
1112 gboolean parsed;
1113 if (gst_structure_get_boolean (str, "parsed", &parsed))
1114 nvdec->recv_complete_picture = parsed;
1115 else
1116 nvdec->recv_complete_picture = FALSE;
1117 }
1118 }
1119
1120 return ret;
1121 }
1122
1123 #ifdef HAVE_NVCODEC_GST_GL
1124 typedef struct
1125 {
1126 GstNvDec *nvdec;
1127 CUVIDPARSERDISPINFO *dispinfo;
1128 gboolean ret;
1129 GstBuffer *output_buffer;
1130 } GstNvDecCopyToGLData;
1131
1132 static void
copy_video_frame_to_gl_textures(GstGLContext * context,GstNvDecCopyToGLData * data)1133 copy_video_frame_to_gl_textures (GstGLContext * context,
1134 GstNvDecCopyToGLData * data)
1135 {
1136 GstNvDec *nvdec = data->nvdec;
1137 CUVIDPARSERDISPINFO *dispinfo = data->dispinfo;
1138 GstCudaGraphicsResource **resources;
1139 guint num_resources;
1140 CUVIDPROCPARAMS proc_params = { 0, };
1141 guintptr dptr;
1142 guint pitch, i;
1143 CUDA_MEMCPY2D mcpy2d = { 0, };
1144 GstVideoInfo *info = &nvdec->output_state->info;
1145
1146 GST_LOG_OBJECT (nvdec, "picture index: %u", dispinfo->picture_index);
1147
1148 proc_params.progressive_frame = dispinfo->progressive_frame;
1149 proc_params.top_field_first = dispinfo->top_field_first;
1150 proc_params.unpaired_field = dispinfo->repeat_first_field == -1;
1151
1152 data->ret = TRUE;
1153
1154 num_resources = gst_buffer_n_memory (data->output_buffer);
1155 resources = g_newa (GstCudaGraphicsResource *, num_resources);
1156
1157 for (i = 0; i < num_resources; i++) {
1158 GstMemory *mem;
1159
1160 mem = gst_buffer_peek_memory (data->output_buffer, i);
1161 resources[i] = ensure_cuda_graphics_resource (mem, nvdec);
1162 if (!resources[i]) {
1163 GST_WARNING_OBJECT (nvdec, "could not register %dth memory", i);
1164 data->ret = FALSE;
1165
1166 return;
1167 }
1168
1169 /* Need PBO -> texture */
1170 GST_MINI_OBJECT_FLAG_SET (mem, GST_GL_BASE_MEMORY_TRANSFER_NEED_UPLOAD);
1171 }
1172
1173 if (!gst_cuda_context_push (nvdec->cuda_ctx)) {
1174 GST_WARNING_OBJECT (nvdec, "failed to lock CUDA context");
1175 data->ret = FALSE;
1176 return;
1177 }
1178
1179 if (!gst_cuda_result (CuvidMapVideoFrame (nvdec->decoder,
1180 dispinfo->picture_index, &dptr, &pitch, &proc_params))) {
1181 GST_WARNING_OBJECT (nvdec, "failed to map CUDA video frame");
1182 data->ret = FALSE;
1183 goto unlock_cuda_context;
1184 }
1185
1186 mcpy2d.srcMemoryType = CU_MEMORYTYPE_DEVICE;
1187 mcpy2d.srcPitch = pitch;
1188 mcpy2d.dstMemoryType = CU_MEMORYTYPE_DEVICE;
1189
1190 for (i = 0; i < num_resources; i++) {
1191 CUdeviceptr cuda_ptr;
1192 gsize size;
1193 CUgraphicsResource cuda_resource =
1194 gst_cuda_graphics_resource_map (resources[i], nvdec->cuda_stream,
1195 CU_GRAPHICS_MAP_RESOURCE_FLAGS_WRITE_DISCARD);
1196
1197 if (!cuda_resource) {
1198 GST_WARNING_OBJECT (nvdec, "failed to map CUDA resources");
1199 data->ret = FALSE;
1200 goto unmap_video_frame;
1201 }
1202
1203 if (!gst_cuda_result (CuGraphicsResourceGetMappedPointer (&cuda_ptr, &size,
1204 cuda_resource))) {
1205 GST_WARNING_OBJECT (nvdec, "failed to map CUDA resource");
1206 data->ret = FALSE;
1207 break;
1208 }
1209
1210 mcpy2d.dstPitch = GST_VIDEO_INFO_PLANE_STRIDE (info, i);
1211 mcpy2d.WidthInBytes = GST_VIDEO_INFO_COMP_WIDTH (info, i)
1212 * GST_VIDEO_INFO_COMP_PSTRIDE (info, i);
1213
1214 mcpy2d.srcDevice = dptr + (i * pitch * GST_VIDEO_INFO_HEIGHT (info));
1215 mcpy2d.dstDevice = cuda_ptr;
1216 mcpy2d.Height = GST_VIDEO_INFO_COMP_HEIGHT (info, i);
1217
1218 if (!gst_cuda_result (CuMemcpy2DAsync (&mcpy2d, nvdec->cuda_stream))) {
1219 GST_WARNING_OBJECT (nvdec, "memcpy to mapped array failed");
1220 data->ret = FALSE;
1221 }
1222 }
1223
1224 gst_cuda_result (CuStreamSynchronize (nvdec->cuda_stream));
1225
1226 unmap_video_frame:
1227 for (i = 0; i < num_resources; i++) {
1228 gst_cuda_graphics_resource_unmap (resources[i], nvdec->cuda_stream);
1229 }
1230
1231 if (!gst_cuda_result (CuvidUnmapVideoFrame (nvdec->decoder, dptr)))
1232 GST_WARNING_OBJECT (nvdec, "failed to unmap CUDA video frame");
1233
1234 unlock_cuda_context:
1235 if (!gst_cuda_context_pop (NULL))
1236 GST_WARNING_OBJECT (nvdec, "failed to unlock CUDA context");
1237 }
1238
1239 static gboolean
gst_nvdec_copy_device_to_gl(GstNvDec * nvdec,CUVIDPARSERDISPINFO * dispinfo,GstBuffer * output_buffer)1240 gst_nvdec_copy_device_to_gl (GstNvDec * nvdec,
1241 CUVIDPARSERDISPINFO * dispinfo, GstBuffer * output_buffer)
1242 {
1243 GstNvDecCopyToGLData data = { 0, };
1244
1245 data.nvdec = nvdec;
1246 data.dispinfo = dispinfo;
1247 data.output_buffer = output_buffer;
1248
1249 gst_gl_context_thread_add (nvdec->gl_context,
1250 (GstGLContextThreadFunc) copy_video_frame_to_gl_textures, &data);
1251
1252 return data.ret;
1253 }
1254 #endif
1255
1256 static gboolean
gst_nvdec_copy_device_to_memory(GstNvDec * nvdec,CUVIDPARSERDISPINFO * dispinfo,GstBuffer * output_buffer)1257 gst_nvdec_copy_device_to_memory (GstNvDec * nvdec,
1258 CUVIDPARSERDISPINFO * dispinfo, GstBuffer * output_buffer)
1259 {
1260 CUVIDPROCPARAMS params = { 0, };
1261 CUDA_MEMCPY2D copy_params = { 0, };
1262 guintptr dptr;
1263 guint pitch;
1264 GstVideoFrame video_frame;
1265 GstVideoInfo *info = &nvdec->output_state->info;
1266 gint i;
1267 GstMemory *mem;
1268 GstCudaMemory *cuda_mem = NULL;
1269
1270 if (!gst_cuda_context_push (nvdec->cuda_ctx)) {
1271 GST_WARNING_OBJECT (nvdec, "failed to lock CUDA context");
1272 return FALSE;
1273 }
1274
1275 if (nvdec->mem_type == GST_NVDEC_MEM_TYPE_CUDA &&
1276 (mem = gst_buffer_peek_memory (output_buffer, 0)) &&
1277 gst_is_cuda_memory (mem)) {
1278 GstCudaMemory *cmem = GST_CUDA_MEMORY_CAST (mem);
1279
1280 if (cmem->context == nvdec->cuda_ctx ||
1281 gst_cuda_context_get_handle (cmem->context) ==
1282 gst_cuda_context_get_handle (nvdec->cuda_ctx) ||
1283 (gst_cuda_context_can_access_peer (cmem->context, nvdec->cuda_ctx) &&
1284 gst_cuda_context_can_access_peer (nvdec->cuda_ctx,
1285 cmem->context))) {
1286 cuda_mem = cmem;
1287 }
1288 }
1289
1290 if (!cuda_mem) {
1291 if (!gst_video_frame_map (&video_frame, info, output_buffer, GST_MAP_WRITE)) {
1292 GST_ERROR_OBJECT (nvdec, "frame map failure");
1293 gst_cuda_context_pop (NULL);
1294 return FALSE;
1295 }
1296 }
1297
1298 params.progressive_frame = dispinfo->progressive_frame;
1299 params.second_field = dispinfo->repeat_first_field + 1;
1300 params.top_field_first = dispinfo->top_field_first;
1301 params.unpaired_field = dispinfo->repeat_first_field < 0;
1302
1303 if (!gst_cuda_result (CuvidMapVideoFrame (nvdec->decoder,
1304 dispinfo->picture_index, &dptr, &pitch, ¶ms))) {
1305 GST_ERROR_OBJECT (nvdec, "failed to map video frame");
1306 gst_cuda_context_pop (NULL);
1307 return FALSE;
1308 }
1309
1310 copy_params.srcMemoryType = CU_MEMORYTYPE_DEVICE;
1311 copy_params.srcPitch = pitch;
1312 copy_params.dstMemoryType =
1313 cuda_mem ? CU_MEMORYTYPE_DEVICE : CU_MEMORYTYPE_HOST;
1314
1315 for (i = 0; i < GST_VIDEO_INFO_N_PLANES (info); i++) {
1316 copy_params.srcDevice = dptr + (i * pitch * GST_VIDEO_INFO_HEIGHT (info));
1317 if (cuda_mem) {
1318 copy_params.dstDevice = cuda_mem->data + cuda_mem->offset[i];
1319 copy_params.dstPitch = cuda_mem->stride;
1320 } else {
1321 copy_params.dstHost = GST_VIDEO_FRAME_PLANE_DATA (&video_frame, i);
1322 copy_params.dstPitch = GST_VIDEO_FRAME_PLANE_STRIDE (&video_frame, i);
1323 }
1324 copy_params.WidthInBytes = GST_VIDEO_INFO_COMP_WIDTH (info, i)
1325 * GST_VIDEO_INFO_COMP_PSTRIDE (info, i);
1326 copy_params.Height = GST_VIDEO_INFO_COMP_HEIGHT (info, i);
1327
1328 if (!gst_cuda_result (CuMemcpy2DAsync (©_params, nvdec->cuda_stream))) {
1329 GST_ERROR_OBJECT (nvdec, "failed to copy %dth plane", i);
1330 CuvidUnmapVideoFrame (nvdec->decoder, dptr);
1331 if (!cuda_mem)
1332 gst_video_frame_unmap (&video_frame);
1333 gst_cuda_context_pop (NULL);
1334 return FALSE;
1335 }
1336 }
1337
1338 gst_cuda_result (CuStreamSynchronize (nvdec->cuda_stream));
1339
1340 if (!cuda_mem)
1341 gst_video_frame_unmap (&video_frame);
1342
1343 if (!gst_cuda_result (CuvidUnmapVideoFrame (nvdec->decoder, dptr)))
1344 GST_WARNING_OBJECT (nvdec, "failed to unmap video frame");
1345
1346 if (!gst_cuda_context_pop (NULL))
1347 GST_WARNING_OBJECT (nvdec, "failed to unlock CUDA context");
1348
1349 return TRUE;
1350 }
1351
1352 static void
gst_nvdec_store_h264_nal(GstNvDec * self,guint id,GstH264NalUnitType nal_type,GstH264NalUnit * nalu)1353 gst_nvdec_store_h264_nal (GstNvDec * self, guint id,
1354 GstH264NalUnitType nal_type, GstH264NalUnit * nalu)
1355 {
1356 GstBuffer *buf, **store;
1357 guint size = nalu->size, store_size;
1358 static const guint8 start_code[] = { 0, 0, 1 };
1359
1360 if (nal_type == GST_H264_NAL_SPS || nal_type == GST_H264_NAL_SUBSET_SPS) {
1361 store_size = GST_H264_MAX_SPS_COUNT;
1362 store = self->sps_nals;
1363 GST_DEBUG_OBJECT (self, "storing sps %u", id);
1364 } else if (nal_type == GST_H264_NAL_PPS) {
1365 store_size = GST_H264_MAX_PPS_COUNT;
1366 store = self->pps_nals;
1367 GST_DEBUG_OBJECT (self, "storing pps %u", id);
1368 } else {
1369 return;
1370 }
1371
1372 if (id >= store_size) {
1373 GST_DEBUG_OBJECT (self, "unable to store nal, id out-of-range %d", id);
1374 return;
1375 }
1376
1377 buf = gst_buffer_new_allocate (NULL, size + sizeof (start_code), NULL);
1378 gst_buffer_fill (buf, 0, start_code, sizeof (start_code));
1379 gst_buffer_fill (buf, sizeof (start_code), nalu->data + nalu->offset, size);
1380
1381 if (store[id])
1382 gst_buffer_unref (store[id]);
1383
1384 store[id] = buf;
1385 }
1386
1387 static GstBuffer *
gst_nvdec_handle_h264_buffer(GstNvDec * self,GstBuffer * buffer)1388 gst_nvdec_handle_h264_buffer (GstNvDec * self, GstBuffer * buffer)
1389 {
1390 GstH264NalParser *parser = self->h264_parser;
1391 GstH264NalUnit nalu;
1392 GstH264ParserResult pres;
1393 GstMapInfo map;
1394 gboolean have_sps = FALSE;
1395 gboolean have_pps = FALSE;
1396 guint i;
1397 GstBuffer *new_buf;
1398
1399 if (!gst_buffer_map (buffer, &map, GST_MAP_READ)) {
1400 GST_WARNING_OBJECT (self, "Failed to map input buffer");
1401 return gst_buffer_ref (buffer);
1402 }
1403
1404 memset (&nalu, 0, sizeof (GstH264NalUnit));
1405
1406 do {
1407 pres = gst_h264_parser_identify_nalu (parser,
1408 map.data, nalu.offset + nalu.size, map.size, &nalu);
1409
1410 if (pres == GST_H264_PARSER_NO_NAL_END)
1411 pres = GST_H264_PARSER_OK;
1412
1413 switch (nalu.type) {
1414 case GST_H264_NAL_SPS:
1415 case GST_H264_NAL_SUBSET_SPS:{
1416 GstH264SPS sps;
1417
1418 if (nalu.type == GST_H264_NAL_SPS) {
1419 pres = gst_h264_parser_parse_sps (parser, &nalu, &sps);
1420 } else {
1421 pres = gst_h264_parser_parse_subset_sps (parser, &nalu, &sps);
1422 }
1423
1424 if (pres != GST_H264_PARSER_OK)
1425 break;
1426
1427 have_sps = TRUE;
1428 gst_nvdec_store_h264_nal (self, sps.id, nalu.type, &nalu);
1429 gst_h264_sps_clear (&sps);
1430 break;
1431 }
1432 case GST_H264_NAL_PPS:{
1433 GstH264PPS pps;
1434
1435 pres = gst_h264_parser_parse_pps (parser, &nalu, &pps);
1436 if (pres != GST_H264_PARSER_OK)
1437 break;
1438
1439 have_pps = TRUE;
1440 gst_nvdec_store_h264_nal (self, pps.id, nalu.type, &nalu);
1441 gst_h264_pps_clear (&pps);
1442 break;
1443 }
1444 default:
1445 break;
1446 }
1447 } while (pres == GST_H264_PARSER_OK);
1448
1449 gst_buffer_unmap (buffer, &map);
1450
1451 if (!self->need_codec_data || (have_sps && have_pps)) {
1452 self->need_codec_data = FALSE;
1453 return gst_buffer_ref (buffer);
1454 }
1455
1456 new_buf = gst_buffer_new ();
1457 if (!have_sps) {
1458 for (i = 0; i < GST_H264_MAX_SPS_COUNT; i++) {
1459 if (!self->sps_nals[i])
1460 continue;
1461
1462 have_sps = TRUE;
1463 new_buf = gst_buffer_append (new_buf, gst_buffer_ref (self->sps_nals[i]));
1464 }
1465 }
1466
1467 if (!have_pps) {
1468 for (i = 0; i < GST_H264_MAX_PPS_COUNT; i++) {
1469 if (!self->pps_nals[i])
1470 continue;
1471
1472 have_pps = TRUE;
1473 new_buf = gst_buffer_append (new_buf, gst_buffer_ref (self->pps_nals[i]));
1474 }
1475 }
1476
1477 new_buf = gst_buffer_append (new_buf, gst_buffer_ref (buffer));
1478
1479 if (have_sps && have_pps)
1480 self->need_codec_data = FALSE;
1481
1482 return new_buf;
1483 }
1484
1485 static void
gst_nvdec_store_h265_nal(GstNvDec * self,guint id,GstH265NalUnitType nal_type,GstH265NalUnit * nalu)1486 gst_nvdec_store_h265_nal (GstNvDec * self, guint id,
1487 GstH265NalUnitType nal_type, GstH265NalUnit * nalu)
1488 {
1489 GstBuffer *buf, **store;
1490 guint size = nalu->size, store_size;
1491 static const guint8 start_code[] = { 0, 0, 1 };
1492
1493 if (nal_type == GST_H265_NAL_VPS) {
1494 store_size = GST_H265_MAX_VPS_COUNT;
1495 store = self->vps_nals;
1496 GST_DEBUG_OBJECT (self, "storing vps %u", id);
1497 } else if (nal_type == GST_H265_NAL_SPS) {
1498 store_size = GST_H265_MAX_SPS_COUNT;
1499 store = self->sps_nals;
1500 GST_DEBUG_OBJECT (self, "storing sps %u", id);
1501 } else if (nal_type == GST_H265_NAL_PPS) {
1502 store_size = GST_H265_MAX_PPS_COUNT;
1503 store = self->pps_nals;
1504 GST_DEBUG_OBJECT (self, "storing pps %u", id);
1505 } else {
1506 return;
1507 }
1508
1509 if (id >= store_size) {
1510 GST_DEBUG_OBJECT (self, "unable to store nal, id out-of-range %d", id);
1511 return;
1512 }
1513
1514 buf = gst_buffer_new_allocate (NULL, size + sizeof (start_code), NULL);
1515 gst_buffer_fill (buf, 0, start_code, sizeof (start_code));
1516 gst_buffer_fill (buf, sizeof (start_code), nalu->data + nalu->offset, size);
1517
1518 if (store[id])
1519 gst_buffer_unref (store[id]);
1520
1521 store[id] = buf;
1522 }
1523
1524 static GstBuffer *
gst_nvdec_handle_h265_buffer(GstNvDec * self,GstBuffer * buffer)1525 gst_nvdec_handle_h265_buffer (GstNvDec * self, GstBuffer * buffer)
1526 {
1527 GstH265Parser *parser = self->h265_parser;
1528 GstH265NalUnit nalu;
1529 GstH265ParserResult pres;
1530 GstMapInfo map;
1531 gboolean have_vps = FALSE;
1532 gboolean have_sps = FALSE;
1533 gboolean have_pps = FALSE;
1534 GstBuffer *new_buf;
1535 guint i;
1536
1537 if (!gst_buffer_map (buffer, &map, GST_MAP_READ)) {
1538 GST_WARNING_OBJECT (self, "Failed to map input buffer");
1539 return gst_buffer_ref (buffer);
1540 }
1541
1542 memset (&nalu, 0, sizeof (GstH265NalUnit));
1543
1544 do {
1545 pres = gst_h265_parser_identify_nalu (parser,
1546 map.data, nalu.offset + nalu.size, map.size, &nalu);
1547
1548 if (pres == GST_H265_PARSER_NO_NAL_END)
1549 pres = GST_H265_PARSER_OK;
1550
1551 switch (nalu.type) {
1552 case GST_H265_NAL_VPS:{
1553 GstH265VPS vps;
1554
1555 pres = gst_h265_parser_parse_vps (parser, &nalu, &vps);
1556 if (pres != GST_H265_PARSER_OK)
1557 break;
1558
1559 have_vps = TRUE;
1560 gst_nvdec_store_h265_nal (self, vps.id, nalu.type, &nalu);
1561 break;
1562 }
1563 case GST_H265_NAL_SPS:{
1564 GstH265SPS sps;
1565
1566 pres = gst_h265_parser_parse_sps (parser, &nalu, &sps, FALSE);
1567 if (pres != GST_H265_PARSER_OK)
1568 break;
1569
1570 have_sps = TRUE;
1571 gst_nvdec_store_h265_nal (self, sps.id, nalu.type, &nalu);
1572 break;
1573 }
1574 case GST_H265_NAL_PPS:{
1575 GstH265PPS pps;
1576
1577 pres = gst_h265_parser_parse_pps (parser, &nalu, &pps);
1578 if (pres != GST_H265_PARSER_OK)
1579 break;
1580
1581 have_pps = TRUE;
1582 gst_nvdec_store_h265_nal (self, pps.id, nalu.type, &nalu);
1583 break;
1584 }
1585 default:
1586 break;
1587 }
1588 } while (pres == GST_H265_PARSER_OK);
1589
1590 gst_buffer_unmap (buffer, &map);
1591
1592 if (!self->need_codec_data || (have_sps && have_pps)) {
1593 self->need_codec_data = FALSE;
1594 return gst_buffer_ref (buffer);
1595 }
1596
1597 new_buf = gst_buffer_new ();
1598 if (!have_vps) {
1599 for (i = 0; i < GST_H265_MAX_VPS_COUNT; i++) {
1600 if (!self->vps_nals[i])
1601 continue;
1602
1603 new_buf = gst_buffer_append (new_buf, gst_buffer_ref (self->vps_nals[i]));
1604 }
1605 }
1606
1607 if (!have_sps) {
1608 for (i = 0; i < GST_H265_MAX_SPS_COUNT; i++) {
1609 if (!self->sps_nals[i])
1610 continue;
1611
1612 have_sps = TRUE;
1613 new_buf = gst_buffer_append (new_buf, gst_buffer_ref (self->sps_nals[i]));
1614 }
1615 }
1616
1617 if (!have_pps) {
1618 for (i = 0; i < GST_H265_MAX_PPS_COUNT; i++) {
1619 if (!self->pps_nals[i])
1620 continue;
1621
1622 have_pps = TRUE;
1623 new_buf = gst_buffer_append (new_buf, gst_buffer_ref (self->pps_nals[i]));
1624 }
1625 }
1626
1627 if (have_sps && have_pps)
1628 self->need_codec_data = FALSE;
1629
1630 return gst_buffer_append (new_buf, gst_buffer_ref (buffer));
1631 }
1632
1633 static GstBuffer *
gst_nvdec_process_input(GstNvDec * self,GstBuffer * inbuf)1634 gst_nvdec_process_input (GstNvDec * self, GstBuffer * inbuf)
1635 {
1636 GstNvDecClass *klass = GST_NVDEC_GET_CLASS (self);
1637 gboolean parse_nal = FALSE;
1638
1639 if (!GST_BUFFER_FLAG_IS_SET (inbuf, GST_BUFFER_FLAG_DELTA_UNIT) ||
1640 self->need_codec_data) {
1641 parse_nal = TRUE;
1642 }
1643
1644 if (klass->codec_type == cudaVideoCodec_MPEG4 &&
1645 self->codec_data && GST_BUFFER_IS_DISCONT (inbuf)) {
1646 return gst_buffer_append (gst_buffer_ref (self->codec_data),
1647 gst_buffer_ref (inbuf));
1648 } else if (klass->codec_type == cudaVideoCodec_H264 && parse_nal) {
1649 return gst_nvdec_handle_h264_buffer (self, inbuf);
1650 } else if (klass->codec_type == cudaVideoCodec_HEVC && parse_nal) {
1651 return gst_nvdec_handle_h265_buffer (self, inbuf);
1652 }
1653
1654 return gst_buffer_ref (inbuf);
1655 }
1656
1657 static GstFlowReturn
gst_nvdec_handle_frame(GstVideoDecoder * decoder,GstVideoCodecFrame * frame)1658 gst_nvdec_handle_frame (GstVideoDecoder * decoder, GstVideoCodecFrame * frame)
1659 {
1660 GstNvDec *nvdec = GST_NVDEC (decoder);
1661 GstMapInfo map_info = GST_MAP_INFO_INIT;
1662 CUVIDSOURCEDATAPACKET packet = { 0, };
1663 GstBuffer *in_buffer;
1664
1665 GST_LOG_OBJECT (nvdec, "handle frame");
1666
1667 /* initialize with zero to keep track of frames */
1668 gst_video_codec_frame_set_user_data (frame, GUINT_TO_POINTER (0), NULL);
1669
1670 in_buffer = gst_nvdec_process_input (nvdec, frame->input_buffer);
1671
1672 if (!gst_buffer_map (in_buffer, &map_info, GST_MAP_READ)) {
1673 GST_ERROR_OBJECT (nvdec, "failed to map input buffer");
1674 gst_buffer_unref (in_buffer);
1675 gst_video_codec_frame_unref (frame);
1676 return GST_FLOW_ERROR;
1677 }
1678
1679 packet.payload_size = (gulong) map_info.size;
1680 packet.payload = map_info.data;
1681 packet.timestamp = frame->pts;
1682 packet.flags |= CUVID_PKT_TIMESTAMP;
1683
1684 if (nvdec->recv_complete_picture)
1685 packet.flags |= CUVID_PKT_ENDOFPICTURE;
1686
1687 nvdec->state = GST_NVDEC_STATE_PARSE;
1688 nvdec->last_ret = GST_FLOW_OK;
1689
1690 if (!gst_cuda_result (CuvidParseVideoData (nvdec->parser, &packet)))
1691 GST_WARNING_OBJECT (nvdec, "parser failed");
1692
1693 gst_buffer_unmap (in_buffer, &map_info);
1694 gst_buffer_unref (in_buffer);
1695 gst_video_codec_frame_unref (frame);
1696
1697 return nvdec->last_ret;
1698 }
1699
1700 static gboolean
gst_nvdec_flush(GstVideoDecoder * decoder)1701 gst_nvdec_flush (GstVideoDecoder * decoder)
1702 {
1703 GstNvDec *nvdec = GST_NVDEC (decoder);
1704 CUVIDSOURCEDATAPACKET packet = { 0, };
1705
1706 GST_DEBUG_OBJECT (nvdec, "flush");
1707
1708 packet.payload_size = 0;
1709 packet.payload = NULL;
1710 packet.flags = CUVID_PKT_ENDOFSTREAM;
1711
1712 nvdec->state = GST_NVDEC_STATE_PARSE;
1713 nvdec->last_ret = GST_FLOW_OK;
1714
1715 if (nvdec->parser
1716 && !gst_cuda_result (CuvidParseVideoData (nvdec->parser, &packet)))
1717 GST_WARNING_OBJECT (nvdec, "parser failed");
1718
1719 nvdec->need_codec_data = TRUE;
1720
1721 return TRUE;
1722 }
1723
1724 static GstFlowReturn
gst_nvdec_drain(GstVideoDecoder * decoder)1725 gst_nvdec_drain (GstVideoDecoder * decoder)
1726 {
1727 GstNvDec *nvdec = GST_NVDEC (decoder);
1728 CUVIDSOURCEDATAPACKET packet = { 0, };
1729
1730 GST_DEBUG_OBJECT (nvdec, "draining decoder");
1731
1732 packet.payload_size = 0;
1733 packet.payload = NULL;
1734 packet.flags = CUVID_PKT_ENDOFSTREAM;
1735
1736 nvdec->state = GST_NVDEC_STATE_PARSE;
1737 nvdec->last_ret = GST_FLOW_OK;
1738
1739 if (nvdec->parser
1740 && !gst_cuda_result (CuvidParseVideoData (nvdec->parser, &packet)))
1741 GST_WARNING_OBJECT (nvdec, "parser failed");
1742
1743 nvdec->need_codec_data = TRUE;
1744
1745 return nvdec->last_ret;
1746 }
1747
1748 static GstFlowReturn
gst_nvdec_finish(GstVideoDecoder * decoder)1749 gst_nvdec_finish (GstVideoDecoder * decoder)
1750 {
1751 GST_DEBUG_OBJECT (decoder, "finish");
1752
1753 return gst_nvdec_drain (decoder);
1754 }
1755
1756 #ifdef HAVE_NVCODEC_GST_GL
1757 static void
gst_nvdec_check_cuda_device_from_context(GstGLContext * context,gboolean * ret)1758 gst_nvdec_check_cuda_device_from_context (GstGLContext * context,
1759 gboolean * ret)
1760 {
1761 guint device_count = 0;
1762 CUdevice device_list[1] = { 0, };
1763 CUresult cuda_ret;
1764
1765 *ret = FALSE;
1766
1767 cuda_ret = CuGLGetDevices (&device_count,
1768 device_list, 1, CU_GL_DEVICE_LIST_ALL);
1769
1770 if (!gst_cuda_result (cuda_ret) || device_count == 0)
1771 return;
1772
1773 *ret = TRUE;
1774
1775 return;
1776 }
1777
1778 static gboolean
gst_nvdec_ensure_gl_context(GstNvDec * nvdec)1779 gst_nvdec_ensure_gl_context (GstNvDec * nvdec)
1780 {
1781 gboolean ret;
1782
1783 if (!nvdec->gl_display) {
1784 GST_DEBUG_OBJECT (nvdec, "No available OpenGL display");
1785 return FALSE;
1786 }
1787
1788 if (!gst_gl_query_local_gl_context (GST_ELEMENT (nvdec), GST_PAD_SRC,
1789 &nvdec->gl_context)) {
1790 GST_INFO_OBJECT (nvdec, "failed to query local OpenGL context");
1791 if (nvdec->gl_context)
1792 gst_object_unref (nvdec->gl_context);
1793 nvdec->gl_context =
1794 gst_gl_display_get_gl_context_for_thread (nvdec->gl_display, NULL);
1795 if (!nvdec->gl_context
1796 || !gst_gl_display_add_context (nvdec->gl_display, nvdec->gl_context)) {
1797 if (nvdec->gl_context)
1798 gst_object_unref (nvdec->gl_context);
1799 if (!gst_gl_display_create_context (nvdec->gl_display,
1800 nvdec->other_gl_context, &nvdec->gl_context, NULL)) {
1801 GST_ERROR_OBJECT (nvdec, "failed to create OpenGL context");
1802 return FALSE;
1803 }
1804 if (!gst_gl_display_add_context (nvdec->gl_display, nvdec->gl_context)) {
1805 GST_ERROR_OBJECT (nvdec,
1806 "failed to add the OpenGL context to the display");
1807 return FALSE;
1808 }
1809 }
1810 }
1811
1812 if (!gst_gl_context_check_gl_version (nvdec->gl_context,
1813 SUPPORTED_GL_APIS, 3, 0)) {
1814 GST_WARNING_OBJECT (nvdec, "OpenGL context could not support PBO download");
1815 return FALSE;
1816 }
1817
1818 gst_gl_context_thread_add (nvdec->gl_context,
1819 (GstGLContextThreadFunc) gst_nvdec_check_cuda_device_from_context, &ret);
1820
1821 if (!ret) {
1822 GST_WARNING_OBJECT (nvdec, "Current OpenGL context is not CUDA-compatible");
1823 return FALSE;
1824 }
1825
1826 return TRUE;
1827 }
1828
1829 static gboolean
gst_nvdec_ensure_gl_pool(GstNvDec * nvdec,GstQuery * query)1830 gst_nvdec_ensure_gl_pool (GstNvDec * nvdec, GstQuery * query)
1831 {
1832 GstCaps *outcaps;
1833 GstBufferPool *pool = NULL;
1834 guint n, size, min, max;
1835 GstVideoInfo vinfo = { 0, };
1836 GstStructure *config;
1837
1838 GST_DEBUG_OBJECT (nvdec, "decide allocation");
1839
1840 gst_query_parse_allocation (query, &outcaps, NULL);
1841 n = gst_query_get_n_allocation_pools (query);
1842 if (n > 0)
1843 gst_query_parse_nth_allocation_pool (query, 0, &pool, &size, &min, &max);
1844
1845 if (pool && !GST_IS_GL_BUFFER_POOL (pool)) {
1846 gst_object_unref (pool);
1847 pool = NULL;
1848 }
1849
1850 if (!pool) {
1851 GST_DEBUG_OBJECT (nvdec, "no downstream pool, create our pool");
1852 pool = gst_gl_buffer_pool_new (nvdec->gl_context);
1853
1854 if (outcaps)
1855 gst_video_info_from_caps (&vinfo, outcaps);
1856 size = (guint) vinfo.size;
1857 min = max = 0;
1858 }
1859
1860 config = gst_buffer_pool_get_config (pool);
1861 gst_buffer_pool_config_set_params (config, outcaps, size, min, max);
1862 gst_buffer_pool_config_add_option (config, GST_BUFFER_POOL_OPTION_VIDEO_META);
1863 gst_buffer_pool_set_config (pool, config);
1864 if (n > 0)
1865 gst_query_set_nth_allocation_pool (query, 0, pool, size, min, max);
1866 else
1867 gst_query_add_allocation_pool (query, pool, size, min, max);
1868 gst_object_unref (pool);
1869
1870 return TRUE;
1871 }
1872 #endif
1873
1874 static gboolean
gst_nvdec_ensure_cuda_pool(GstNvDec * nvdec,GstQuery * query)1875 gst_nvdec_ensure_cuda_pool (GstNvDec * nvdec, GstQuery * query)
1876 {
1877 GstCaps *outcaps;
1878 GstBufferPool *pool = NULL;
1879 guint n, size, min, max;
1880 GstVideoInfo vinfo = { 0, };
1881 GstStructure *config;
1882
1883 gst_query_parse_allocation (query, &outcaps, NULL);
1884 n = gst_query_get_n_allocation_pools (query);
1885 if (n > 0) {
1886 gst_query_parse_nth_allocation_pool (query, 0, &pool, &size, &min, &max);
1887 if (pool && !GST_IS_CUDA_BUFFER_POOL (pool)) {
1888 gst_object_unref (pool);
1889 pool = NULL;
1890 }
1891 }
1892
1893 if (!pool) {
1894 GST_DEBUG_OBJECT (nvdec, "no downstream pool, create our pool");
1895 pool = gst_cuda_buffer_pool_new (nvdec->cuda_ctx);
1896
1897 if (outcaps)
1898 gst_video_info_from_caps (&vinfo, outcaps);
1899 size = (guint) vinfo.size;
1900 min = max = 0;
1901 }
1902
1903 config = gst_buffer_pool_get_config (pool);
1904 gst_buffer_pool_config_set_params (config, outcaps, size, min, max);
1905 gst_buffer_pool_config_add_option (config, GST_BUFFER_POOL_OPTION_VIDEO_META);
1906 gst_buffer_pool_set_config (pool, config);
1907 if (n > 0)
1908 gst_query_set_nth_allocation_pool (query, 0, pool, size, min, max);
1909 else
1910 gst_query_add_allocation_pool (query, pool, size, min, max);
1911 gst_object_unref (pool);
1912
1913 return TRUE;
1914 }
1915
1916 static gboolean
gst_nvdec_decide_allocation(GstVideoDecoder * decoder,GstQuery * query)1917 gst_nvdec_decide_allocation (GstVideoDecoder * decoder, GstQuery * query)
1918 {
1919 GstNvDec *nvdec = GST_NVDEC (decoder);
1920
1921 GST_DEBUG_OBJECT (nvdec, "decide allocation");
1922
1923 if (nvdec->mem_type == GST_NVDEC_MEM_TYPE_SYSTEM)
1924 goto done;
1925
1926 #ifdef HAVE_NVCODEC_GST_GL
1927 if (nvdec->mem_type == GST_NVDEC_MEM_TYPE_GL) {
1928 if (!gst_nvdec_ensure_gl_pool (nvdec, query))
1929 return FALSE;
1930 } else
1931 #endif
1932 if (!gst_nvdec_ensure_cuda_pool (nvdec, query)) {
1933 return FALSE;
1934 }
1935
1936 done:
1937 return GST_VIDEO_DECODER_CLASS (gst_nvdec_parent_class)->decide_allocation
1938 (decoder, query);
1939 }
1940
1941 static gboolean
gst_nvdec_src_query(GstVideoDecoder * decoder,GstQuery * query)1942 gst_nvdec_src_query (GstVideoDecoder * decoder, GstQuery * query)
1943 {
1944 GstNvDec *nvdec = GST_NVDEC (decoder);
1945
1946 switch (GST_QUERY_TYPE (query)) {
1947 case GST_QUERY_CONTEXT:
1948 if (gst_cuda_handle_context_query (GST_ELEMENT (decoder),
1949 query, nvdec->cuda_ctx)) {
1950 return TRUE;
1951 }
1952 #ifdef HAVE_NVCODEC_GST_GL
1953 if (gst_gl_handle_context_query (GST_ELEMENT (decoder), query,
1954 nvdec->gl_display, nvdec->gl_context, nvdec->other_gl_context)) {
1955 if (nvdec->gl_display)
1956 gst_gl_display_filter_gl_api (GST_GL_DISPLAY (nvdec->gl_display),
1957 SUPPORTED_GL_APIS);
1958 return TRUE;
1959 }
1960 #endif
1961 break;
1962 default:
1963 break;
1964 }
1965
1966 return GST_VIDEO_DECODER_CLASS (gst_nvdec_parent_class)->src_query (decoder,
1967 query);
1968 }
1969
1970 static void
gst_nvdec_set_context(GstElement * element,GstContext * context)1971 gst_nvdec_set_context (GstElement * element, GstContext * context)
1972 {
1973 GstNvDec *nvdec = GST_NVDEC (element);
1974 GstNvDecClass *klass = GST_NVDEC_GET_CLASS (nvdec);
1975
1976 GST_DEBUG_OBJECT (nvdec, "set context %s",
1977 gst_context_get_context_type (context));
1978
1979 if (gst_cuda_handle_set_context (element,
1980 context, klass->cuda_device_id, &nvdec->cuda_ctx)) {
1981 goto done;
1982 }
1983 #ifdef HAVE_NVCODEC_GST_GL
1984 gst_gl_handle_set_context (element, context, &nvdec->gl_display,
1985 &nvdec->other_gl_context);
1986 #endif
1987
1988 done:
1989 GST_ELEMENT_CLASS (gst_nvdec_parent_class)->set_context (element, context);
1990 }
1991
1992 typedef struct
1993 {
1994 GstCaps *sink_caps;
1995 GstCaps *src_caps;
1996 cudaVideoCodec codec_type;
1997 gchar *codec;
1998 guint cuda_device_id;
1999 gboolean is_default;
2000 } GstNvDecClassData;
2001
2002 static void
gst_nvdec_subclass_init(gpointer g_class,gpointer data)2003 gst_nvdec_subclass_init (gpointer g_class, gpointer data)
2004 {
2005 GstElementClass *element_class = GST_ELEMENT_CLASS (g_class);
2006 GstNvDecClass *nvdec_class = GST_NVDEC_CLASS (g_class);
2007 GstNvDecClassData *cdata = data;
2008 gchar *long_name;
2009
2010 if (cdata->is_default) {
2011 long_name = g_strdup_printf ("NVDEC %s Video Decoder", cdata->codec);
2012 } else {
2013 long_name = g_strdup_printf ("NVDEC %s Video Decoder with device %d",
2014 cdata->codec, cdata->cuda_device_id);
2015 }
2016
2017 gst_element_class_set_metadata (element_class, long_name,
2018 "Codec/Decoder/Video/Hardware", "NVDEC video decoder",
2019 "Ericsson AB, http://www.ericsson.com, "
2020 "Seungha Yang <seungha.yang@navercorp.com>");
2021 g_free (long_name);
2022
2023 gst_element_class_add_pad_template (element_class,
2024 gst_pad_template_new ("sink", GST_PAD_SINK, GST_PAD_ALWAYS,
2025 cdata->sink_caps));
2026 gst_element_class_add_pad_template (element_class,
2027 gst_pad_template_new ("src", GST_PAD_SRC, GST_PAD_ALWAYS,
2028 cdata->src_caps));
2029
2030 nvdec_class->codec_type = cdata->codec_type;
2031 nvdec_class->cuda_device_id = cdata->cuda_device_id;
2032
2033 gst_caps_unref (cdata->sink_caps);
2034 gst_caps_unref (cdata->src_caps);
2035 g_free (cdata->codec);
2036 g_free (cdata);
2037 }
2038
2039 static void
gst_nvdec_subclass_register(GstPlugin * plugin,GType type,cudaVideoCodec codec_type,const gchar * codec,guint device_id,guint rank,GstCaps * sink_caps,GstCaps * src_caps)2040 gst_nvdec_subclass_register (GstPlugin * plugin, GType type,
2041 cudaVideoCodec codec_type, const gchar * codec, guint device_id, guint rank,
2042 GstCaps * sink_caps, GstCaps * src_caps)
2043 {
2044 GTypeQuery type_query;
2045 GTypeInfo type_info = { 0, };
2046 GType subtype;
2047 gchar *type_name;
2048 GstNvDecClassData *cdata;
2049 gboolean is_default = TRUE;
2050
2051 cdata = g_new0 (GstNvDecClassData, 1);
2052 cdata->sink_caps = gst_caps_ref (sink_caps);
2053 cdata->src_caps = gst_caps_ref (src_caps);
2054 cdata->codec_type = codec_type;
2055 cdata->codec = g_strdup (codec);
2056 cdata->cuda_device_id = device_id;
2057
2058 g_type_query (type, &type_query);
2059 memset (&type_info, 0, sizeof (type_info));
2060 type_info.class_size = type_query.class_size;
2061 type_info.instance_size = type_query.instance_size;
2062 type_info.class_init = gst_nvdec_subclass_init;
2063 type_info.class_data = cdata;
2064
2065 type_name = g_strdup_printf ("nv%sdec", codec);
2066
2067 if (g_type_from_name (type_name) != 0) {
2068 g_free (type_name);
2069 type_name = g_strdup_printf ("nv%sdevice%ddec", codec, device_id);
2070 is_default = FALSE;
2071 }
2072
2073 cdata->is_default = is_default;
2074 subtype = g_type_register_static (type, type_name, &type_info, 0);
2075
2076 /* make lower rank than default device */
2077 if (rank > 0 && !is_default)
2078 rank--;
2079
2080 if (!gst_element_register (plugin, type_name, rank, subtype))
2081 GST_WARNING ("Failed to register plugin '%s'", type_name);
2082
2083 g_free (type_name);
2084 }
2085
2086 void
gst_nvdec_plugin_init(GstPlugin * plugin,guint device_index,cudaVideoCodec codec,const gchar * codec_name,GstCaps * sink_template,GstCaps * src_template)2087 gst_nvdec_plugin_init (GstPlugin * plugin, guint device_index,
2088 cudaVideoCodec codec, const gchar * codec_name, GstCaps * sink_template,
2089 GstCaps * src_template)
2090 {
2091 gst_nvdec_subclass_register (plugin, GST_TYPE_NVDEC, codec,
2092 codec_name, device_index, GST_RANK_PRIMARY, sink_template, src_template);
2093 }
2094