1 /* GStreamer
2 * Copyright (C) 2010, 2013 Ole André Vadla Ravnås <oleavr@soundrop.com>
3 * Copyright (C) 2012-2016 Alessandro Decina <alessandro.d@gmail.com>
4 *
5 * This library is free software; you can redistribute it and/or
6 * modify it under the terms of the GNU Library General Public
7 * License as published by the Free Software Foundation; either
8 * version 2 of the License, or (at your option) any later version.
9 *
10 * This library is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 * Library General Public License for more details.
14 *
15 * You should have received a copy of the GNU Library General Public
16 * License along with this library; if not, write to the
17 * Free Software Foundation, Inc., 51 Franklin Street, Suite 500,
18 * Boston, MA 02110-1335, USA.
19 */
20 /**
21 * SECTION:element-vtdec
22 * @title: vtdec
23 *
24 * Apple VideoToolbox based decoder which might use a HW or a SW
25 * implementation depending on the device.
26 *
27 * ## Example launch line
28 * |[
29 * gst-launch-1.0 -v filesrc location=file.mov ! qtdemux ! queue ! h264parse ! vtdec ! videoconvert ! autovideosink
30 * ]|
31 * Decode h264 video from a mov file.
32 *
33 */
34
35 /**
36 * SECTION:element-vtdec_hw
37 * @title: vtdec_hw
38 *
39 * Apple VideoToolbox based HW-only decoder.
40 *
41 * ## Example launch line
42 * |[
43 * gst-launch-1.0 -v filesrc location=file.mov ! qtdemux ! queue ! h264parse ! vtdec_hw ! videoconvert ! autovideosink
44 * ]|
45 * Decode h264 video from a mov file.
46 *
47 */
48
49 #ifdef HAVE_CONFIG_H
50 #include "config.h"
51 #endif
52
53 #include <string.h>
54 #include <gst/gst.h>
55 #include <gst/video/video.h>
56 #include <gst/video/gstvideodecoder.h>
57 #include <gst/gl/gstglcontext.h>
58 #include "vtdec.h"
59 #include "vtutil.h"
60 #include "corevideobuffer.h"
61 #include "coremediabuffer.h"
62 #include "videotexturecache-gl.h"
63 #if defined(APPLEMEDIA_MOLTENVK)
64 #include "videotexturecache-vulkan.h"
65 #endif
66
67 GST_DEBUG_CATEGORY_STATIC (gst_vtdec_debug_category);
68 #define GST_CAT_DEFAULT gst_vtdec_debug_category
69
70 enum
71 {
72 /* leave some headroom for new GstVideoCodecFrameFlags flags */
73 VTDEC_FRAME_FLAG_SKIP = (1 << 10),
74 VTDEC_FRAME_FLAG_DROP = (1 << 11),
75 VTDEC_FRAME_FLAG_ERROR = (1 << 12),
76 };
77
78 static void gst_vtdec_finalize (GObject * object);
79
80 static gboolean gst_vtdec_start (GstVideoDecoder * decoder);
81 static gboolean gst_vtdec_stop (GstVideoDecoder * decoder);
82 static gboolean gst_vtdec_negotiate (GstVideoDecoder * decoder);
83 static gboolean gst_vtdec_set_format (GstVideoDecoder * decoder,
84 GstVideoCodecState * state);
85 static gboolean gst_vtdec_flush (GstVideoDecoder * decoder);
86 static GstFlowReturn gst_vtdec_finish (GstVideoDecoder * decoder);
87 static GstFlowReturn gst_vtdec_handle_frame (GstVideoDecoder * decoder,
88 GstVideoCodecFrame * frame);
89
90 static OSStatus gst_vtdec_create_session (GstVtdec * vtdec,
91 GstVideoFormat format, gboolean enable_hardware);
92 static void gst_vtdec_invalidate_session (GstVtdec * vtdec);
93 static CMSampleBufferRef cm_sample_buffer_from_gst_buffer (GstVtdec * vtdec,
94 GstBuffer * buf);
95 static GstFlowReturn gst_vtdec_push_frames_if_needed (GstVtdec * vtdec,
96 gboolean drain, gboolean flush);
97 static CMFormatDescriptionRef create_format_description (GstVtdec * vtdec,
98 CMVideoCodecType cm_format);
99 static CMFormatDescriptionRef
100 create_format_description_from_codec_data (GstVtdec * vtdec,
101 CMVideoCodecType cm_format, GstBuffer * codec_data);
102 static void gst_vtdec_session_output_callback (void
103 *decompression_output_ref_con, void *source_frame_ref_con, OSStatus status,
104 VTDecodeInfoFlags info_flags, CVImageBufferRef image_buffer, CMTime pts,
105 CMTime duration);
106 static gboolean compute_h264_decode_picture_buffer_length (GstVtdec * vtdec,
107 GstBuffer * codec_data, int *length);
108 static gboolean gst_vtdec_compute_reorder_queue_length (GstVtdec * vtdec,
109 CMVideoCodecType cm_format, GstBuffer * codec_data);
110 static void gst_vtdec_set_latency (GstVtdec * vtdec);
111 static void gst_vtdec_set_context (GstElement * element, GstContext * context);
112
113 static GstStaticPadTemplate gst_vtdec_sink_template =
114 GST_STATIC_PAD_TEMPLATE ("sink",
115 GST_PAD_SINK,
116 GST_PAD_ALWAYS,
117 GST_STATIC_CAPS ("video/x-h264, stream-format=avc, alignment=au,"
118 " width=(int)[1, MAX], height=(int)[1, MAX];"
119 "video/mpeg, mpegversion=2, systemstream=false, parsed=true;"
120 "image/jpeg;"
121 "video/x-prores, variant = { (string)standard, (string)hq, (string)lt,"
122 " (string)proxy, (string)4444, (string)4444xq };")
123 );
124
125 /* define EnableHardwareAcceleratedVideoDecoder in < 10.9 */
126 #if defined(MAC_OS_X_VERSION_MAX_ALLOWED) && MAC_OS_X_VERSION_MAX_ALLOWED < 1090
127 const CFStringRef
128 kVTVideoDecoderSpecification_EnableHardwareAcceleratedVideoDecoder =
129 CFSTR ("EnableHardwareAcceleratedVideoDecoder");
130 const CFStringRef
131 kVTVideoDecoderSpecification_RequireHardwareAcceleratedVideoDecoder =
132 CFSTR ("RequireHardwareAcceleratedVideoDecoder");
133 #endif
134
135 #define VIDEO_SRC_CAPS_FORMATS "{ NV12, AYUV64, ARGB64_BE }"
136
137 #define VIDEO_SRC_CAPS_NATIVE \
138 GST_VIDEO_CAPS_MAKE(VIDEO_SRC_CAPS_FORMATS) ";" \
139 GST_VIDEO_CAPS_MAKE_WITH_FEATURES(GST_CAPS_FEATURE_MEMORY_GL_MEMORY,\
140 VIDEO_SRC_CAPS_FORMATS) ", " \
141 "texture-target = (string) rectangle "
142
143 #if defined(APPLEMEDIA_MOLTENVK)
144 #define VIDEO_SRC_CAPS VIDEO_SRC_CAPS_NATIVE "; " \
145 GST_VIDEO_CAPS_MAKE_WITH_FEATURES(GST_CAPS_FEATURE_MEMORY_VULKAN_IMAGE, \
146 VIDEO_SRC_CAPS_FORMATS)
147 #else
148 #define VIDEO_SRC_CAPS VIDEO_SRC_CAPS_NATIVE
149 #endif
150
151 G_DEFINE_TYPE (GstVtdec, gst_vtdec, GST_TYPE_VIDEO_DECODER);
152
153 static void
gst_vtdec_class_init(GstVtdecClass * klass)154 gst_vtdec_class_init (GstVtdecClass * klass)
155 {
156 GObjectClass *gobject_class = G_OBJECT_CLASS (klass);
157 GstElementClass *element_class = GST_ELEMENT_CLASS (klass);
158 GstVideoDecoderClass *video_decoder_class = GST_VIDEO_DECODER_CLASS (klass);
159
160 /* Setting up pads and setting metadata should be moved to
161 base_class_init if you intend to subclass this class. */
162 gst_element_class_add_static_pad_template (element_class,
163 &gst_vtdec_sink_template);
164
165 {
166 GstCaps *caps = gst_caps_from_string (VIDEO_SRC_CAPS);
167 /* RGBA64_LE is kCVPixelFormatType_64RGBALE, only available on macOS 11.3+ */
168 if (GST_VTUTIL_HAVE_64ARGBALE)
169 caps = gst_vtutil_caps_append_video_format (caps, "RGBA64_LE");
170 gst_element_class_add_pad_template (element_class,
171 gst_pad_template_new ("src", GST_PAD_SRC, GST_PAD_ALWAYS, caps));
172 }
173
174 gst_element_class_set_static_metadata (element_class,
175 "Apple VideoToolbox decoder",
176 "Codec/Decoder/Video/Hardware",
177 "Apple VideoToolbox Decoder",
178 "Ole André Vadla Ravnås <oleavr@soundrop.com>; "
179 "Alessandro Decina <alessandro.d@gmail.com>");
180
181 gobject_class->finalize = gst_vtdec_finalize;
182 element_class->set_context = gst_vtdec_set_context;
183 video_decoder_class->start = GST_DEBUG_FUNCPTR (gst_vtdec_start);
184 video_decoder_class->stop = GST_DEBUG_FUNCPTR (gst_vtdec_stop);
185 video_decoder_class->negotiate = GST_DEBUG_FUNCPTR (gst_vtdec_negotiate);
186 video_decoder_class->set_format = GST_DEBUG_FUNCPTR (gst_vtdec_set_format);
187 video_decoder_class->flush = GST_DEBUG_FUNCPTR (gst_vtdec_flush);
188 video_decoder_class->finish = GST_DEBUG_FUNCPTR (gst_vtdec_finish);
189 video_decoder_class->handle_frame =
190 GST_DEBUG_FUNCPTR (gst_vtdec_handle_frame);
191 }
192
193 static void
gst_vtdec_init(GstVtdec * vtdec)194 gst_vtdec_init (GstVtdec * vtdec)
195 {
196 vtdec->reorder_queue = g_async_queue_new ();
197 }
198
199 void
gst_vtdec_finalize(GObject * object)200 gst_vtdec_finalize (GObject * object)
201 {
202 GstVtdec *vtdec = GST_VTDEC (object);
203
204 GST_DEBUG_OBJECT (vtdec, "finalize");
205
206 g_async_queue_unref (vtdec->reorder_queue);
207
208 G_OBJECT_CLASS (gst_vtdec_parent_class)->finalize (object);
209 }
210
211 static gboolean
gst_vtdec_start(GstVideoDecoder * decoder)212 gst_vtdec_start (GstVideoDecoder * decoder)
213 {
214 GstVtdec *vtdec = GST_VTDEC (decoder);
215
216 GST_DEBUG_OBJECT (vtdec, "start");
217
218 if (!vtdec->ctxh)
219 vtdec->ctxh = gst_gl_context_helper_new (GST_ELEMENT (decoder));
220
221 return TRUE;
222 }
223
224 static gboolean
gst_vtdec_stop(GstVideoDecoder * decoder)225 gst_vtdec_stop (GstVideoDecoder * decoder)
226 {
227 GstVtdec *vtdec = GST_VTDEC (decoder);
228
229 gst_vtdec_push_frames_if_needed (vtdec, TRUE, TRUE);
230
231 if (vtdec->input_state)
232 gst_video_codec_state_unref (vtdec->input_state);
233 vtdec->input_state = NULL;
234
235 if (vtdec->session)
236 gst_vtdec_invalidate_session (vtdec);
237
238 if (vtdec->texture_cache)
239 g_object_unref (vtdec->texture_cache);
240 vtdec->texture_cache = NULL;
241
242 if (vtdec->ctxh)
243 gst_gl_context_helper_free (vtdec->ctxh);
244 vtdec->ctxh = NULL;
245
246 if (vtdec->format_description)
247 CFRelease (vtdec->format_description);
248 vtdec->format_description = NULL;
249
250 #if defined(APPLEMEDIA_MOLTENVK)
251 gst_clear_object (&vtdec->device);
252 gst_clear_object (&vtdec->instance);
253 #endif
254
255 GST_DEBUG_OBJECT (vtdec, "stop");
256
257 return TRUE;
258 }
259
260 static void
setup_texture_cache(GstVtdec * vtdec,GstVideoFormat format)261 setup_texture_cache (GstVtdec * vtdec, GstVideoFormat format)
262 {
263 GstVideoCodecState *output_state;
264
265 GST_INFO_OBJECT (vtdec, "setting up texture cache");
266 output_state = gst_video_decoder_get_output_state (GST_VIDEO_DECODER (vtdec));
267 gst_video_texture_cache_set_format (vtdec->texture_cache, format,
268 output_state->caps);
269 gst_video_codec_state_unref (output_state);
270 }
271
272 /*
273 * Unconditionally output a high bit-depth + alpha format when decoding Apple
274 * ProRes video if downstream supports it.
275 * TODO: read src_pix_fmt to get the preferred output format
276 * https://wiki.multimedia.cx/index.php/Apple_ProRes#Frame_header
277 */
278 static GstVideoFormat
get_preferred_video_format(GstStructure * s,gboolean prores)279 get_preferred_video_format (GstStructure * s, gboolean prores)
280 {
281 const GValue *list = gst_structure_get_value (s, "format");
282 guint i, size = gst_value_list_get_size (list);
283 for (i = 0; i < size; i++) {
284 const GValue *value = gst_value_list_get_value (list, i);
285 const char *fmt = g_value_get_string (value);
286 GstVideoFormat vfmt = gst_video_format_from_string (fmt);
287 switch (vfmt) {
288 case GST_VIDEO_FORMAT_NV12:
289 if (!prores)
290 return vfmt;
291 break;
292 case GST_VIDEO_FORMAT_AYUV64:
293 case GST_VIDEO_FORMAT_ARGB64_BE:
294 if (prores)
295 return vfmt;
296 break;
297 case GST_VIDEO_FORMAT_RGBA64_LE:
298 if (GST_VTUTIL_HAVE_64ARGBALE) {
299 if (prores)
300 return vfmt;
301 } else {
302 /* Codepath will never be hit on macOS older than Big Sur (11.3) */
303 g_warn_if_reached ();
304 }
305 break;
306 default:
307 break;
308 }
309 }
310 return GST_VIDEO_FORMAT_UNKNOWN;
311 }
312
313 static gboolean
gst_vtdec_negotiate(GstVideoDecoder * decoder)314 gst_vtdec_negotiate (GstVideoDecoder * decoder)
315 {
316 GstVideoCodecState *output_state = NULL;
317 GstCaps *peercaps = NULL, *caps = NULL, *templcaps = NULL, *prevcaps = NULL;
318 GstVideoFormat format = GST_VIDEO_FORMAT_UNKNOWN;
319 GstVtdec *vtdec;
320 OSStatus err = noErr;
321 GstCapsFeatures *features = NULL;
322 gboolean output_textures = FALSE;
323 #if defined(APPLEMEDIA_MOLTENVK)
324 gboolean output_vulkan = FALSE;
325 #endif
326
327 vtdec = GST_VTDEC (decoder);
328 if (vtdec->session)
329 gst_vtdec_push_frames_if_needed (vtdec, TRUE, FALSE);
330
331 output_state = gst_video_decoder_get_output_state (GST_VIDEO_DECODER (vtdec));
332 if (output_state) {
333 prevcaps = gst_caps_ref (output_state->caps);
334 gst_video_codec_state_unref (output_state);
335 }
336
337 peercaps = gst_pad_peer_query_caps (GST_VIDEO_DECODER_SRC_PAD (vtdec), NULL);
338 if (prevcaps && gst_caps_can_intersect (prevcaps, peercaps)) {
339 /* The hardware decoder can become (temporarily) unavailable across
340 * VTDecompressionSessionCreate/Destroy calls. So if the currently configured
341 * caps are still accepted by downstream we keep them so we don't have to
342 * destroy and recreate the session.
343 */
344 GST_INFO_OBJECT (vtdec,
345 "current and peer caps are compatible, keeping current caps");
346 caps = gst_caps_ref (prevcaps);
347 } else {
348 templcaps =
349 gst_pad_get_pad_template_caps (GST_VIDEO_DECODER_SRC_PAD (decoder));
350 caps =
351 gst_caps_intersect_full (peercaps, templcaps, GST_CAPS_INTERSECT_FIRST);
352 gst_caps_unref (templcaps);
353 }
354 gst_caps_unref (peercaps);
355
356 caps = gst_caps_truncate (gst_caps_make_writable (caps));
357
358 /* Try to use whatever video format downstream prefers */
359 {
360 GstStructure *s = gst_caps_get_structure (caps, 0);
361
362 if (gst_structure_has_field_typed (s, "format", GST_TYPE_LIST)) {
363 GstStructure *is = gst_caps_get_structure (vtdec->input_state->caps, 0);
364 const char *name = gst_structure_get_name (is);
365 format = get_preferred_video_format (s,
366 g_strcmp0 (name, "video/x-prores") == 0);
367 }
368
369 if (format == GST_VIDEO_FORMAT_UNKNOWN) {
370 const char *fmt;
371 gst_structure_fixate_field (s, "format");
372 fmt = gst_structure_get_string (s, "format");
373 if (fmt)
374 format = gst_video_format_from_string (fmt);
375 else
376 /* If all fails, just use NV12 */
377 format = GST_VIDEO_FORMAT_NV12;
378 }
379 }
380
381 features = gst_caps_get_features (caps, 0);
382 if (features)
383 features = gst_caps_features_copy (features);
384
385 output_state = gst_video_decoder_set_output_state (GST_VIDEO_DECODER (vtdec),
386 format, vtdec->video_info.width, vtdec->video_info.height,
387 vtdec->input_state);
388 output_state->caps = gst_video_info_to_caps (&output_state->info);
389 if (features) {
390 gst_caps_set_features (output_state->caps, 0, features);
391 output_textures =
392 gst_caps_features_contains (features,
393 GST_CAPS_FEATURE_MEMORY_GL_MEMORY);
394 if (output_textures)
395 gst_caps_set_simple (output_state->caps, "texture-target", G_TYPE_STRING,
396 #if !HAVE_IOS
397 GST_GL_TEXTURE_TARGET_RECTANGLE_STR,
398 #else
399 GST_GL_TEXTURE_TARGET_2D_STR,
400 #endif
401 NULL);
402
403 #if defined(APPLEMEDIA_MOLTENVK)
404 output_vulkan =
405 gst_caps_features_contains (features,
406 GST_CAPS_FEATURE_MEMORY_VULKAN_IMAGE);
407 #endif
408 }
409 gst_caps_unref (caps);
410
411 if (!prevcaps || !gst_caps_is_equal (prevcaps, output_state->caps)) {
412 gboolean renegotiating = vtdec->session != NULL;
413
414 GST_INFO_OBJECT (vtdec,
415 "negotiated output format %" GST_PTR_FORMAT " previous %"
416 GST_PTR_FORMAT, output_state->caps, prevcaps);
417
418 if (vtdec->session)
419 gst_vtdec_invalidate_session (vtdec);
420
421 err = gst_vtdec_create_session (vtdec, format, TRUE);
422 if (err == noErr) {
423 GST_INFO_OBJECT (vtdec, "using hardware decoder");
424 } else if (err == kVTVideoDecoderNotAvailableNowErr && renegotiating) {
425 GST_WARNING_OBJECT (vtdec, "hw decoder not available anymore");
426 err = gst_vtdec_create_session (vtdec, format, FALSE);
427 }
428
429 if (err != noErr) {
430 GST_ELEMENT_ERROR (vtdec, RESOURCE, FAILED, (NULL),
431 ("VTDecompressionSessionCreate returned %d", (int) err));
432 }
433 }
434
435 if (vtdec->texture_cache != NULL
436 && ((GST_IS_VIDEO_TEXTURE_CACHE_GL (vtdec->texture_cache)
437 && !output_textures)
438 #if defined(APPLEMEDIA_MOLTENVK)
439 || (GST_IS_VIDEO_TEXTURE_CACHE_VULKAN (vtdec->texture_cache)
440 && !output_vulkan)
441 #endif
442 )) {
443 g_object_unref (vtdec->texture_cache);
444 vtdec->texture_cache = NULL;
445 }
446
447 if (err == noErr) {
448 if (output_textures) {
449 GstVideoTextureCacheGL *cache_gl = NULL;
450
451 if (vtdec->texture_cache)
452 cache_gl = GST_VIDEO_TEXTURE_CACHE_GL (vtdec->texture_cache);
453
454 /* call this regardless of whether caps have changed or not since a new
455 * local context could have become available
456 */
457 if (!vtdec->ctxh)
458 vtdec->ctxh = gst_gl_context_helper_new (GST_ELEMENT (vtdec));
459 gst_gl_context_helper_ensure_context (vtdec->ctxh);
460
461 GST_INFO_OBJECT (vtdec, "pushing GL textures, context %p old context %p",
462 vtdec->ctxh->context, cache_gl ? cache_gl->ctx : NULL);
463
464 if (cache_gl && cache_gl->ctx != vtdec->ctxh->context) {
465 g_object_unref (vtdec->texture_cache);
466 vtdec->texture_cache = NULL;
467 }
468 if (!vtdec->texture_cache) {
469 vtdec->texture_cache =
470 gst_video_texture_cache_gl_new (vtdec->ctxh->context);
471 setup_texture_cache (vtdec, format);
472 }
473 }
474 #if defined(APPLEMEDIA_MOLTENVK)
475 if (output_vulkan) {
476 GstVideoTextureCacheVulkan *cache_vulkan = NULL;
477
478 if (vtdec->texture_cache)
479 cache_vulkan = GST_VIDEO_TEXTURE_CACHE_VULKAN (vtdec->texture_cache);
480
481 gst_vulkan_ensure_element_data (GST_ELEMENT (vtdec), NULL,
482 &vtdec->instance);
483
484 if (!gst_vulkan_device_run_context_query (GST_ELEMENT (vtdec),
485 &vtdec->device)) {
486 GError *error = NULL;
487 GST_DEBUG_OBJECT (vtdec, "No device retrieved from peer elements");
488 if (!(vtdec->device =
489 gst_vulkan_instance_create_device (vtdec->instance, &error))) {
490 GST_ELEMENT_ERROR (vtdec, RESOURCE, NOT_FOUND,
491 ("Failed to create vulkan device"), ("%s", error->message));
492 g_clear_error (&error);
493 return FALSE;
494 }
495 }
496
497 GST_INFO_OBJECT (vtdec, "pushing vulkan images, device %" GST_PTR_FORMAT
498 " old device %" GST_PTR_FORMAT, vtdec->device,
499 cache_vulkan ? cache_vulkan->device : NULL);
500
501 if (cache_vulkan && cache_vulkan->device != vtdec->device) {
502 g_object_unref (vtdec->texture_cache);
503 vtdec->texture_cache = NULL;
504 }
505 if (!vtdec->texture_cache) {
506 vtdec->texture_cache =
507 gst_video_texture_cache_vulkan_new (vtdec->device);
508 setup_texture_cache (vtdec, format);
509 }
510 }
511 #endif
512 }
513
514 if (prevcaps)
515 gst_caps_unref (prevcaps);
516
517 if (err != noErr)
518 return FALSE;
519
520 return GST_VIDEO_DECODER_CLASS (gst_vtdec_parent_class)->negotiate (decoder);
521 }
522
523 static gboolean
gst_vtdec_set_format(GstVideoDecoder * decoder,GstVideoCodecState * state)524 gst_vtdec_set_format (GstVideoDecoder * decoder, GstVideoCodecState * state)
525 {
526 GstStructure *structure;
527 CMVideoCodecType cm_format = 0;
528 CMFormatDescriptionRef format_description = NULL;
529 const char *caps_name;
530 GstVtdec *vtdec = GST_VTDEC (decoder);
531
532 GST_DEBUG_OBJECT (vtdec, "set_format");
533
534 structure = gst_caps_get_structure (state->caps, 0);
535 caps_name = gst_structure_get_name (structure);
536 if (!strcmp (caps_name, "video/x-h264")) {
537 cm_format = kCMVideoCodecType_H264;
538 } else if (!strcmp (caps_name, "video/mpeg")) {
539 cm_format = kCMVideoCodecType_MPEG2Video;
540 } else if (!strcmp (caps_name, "image/jpeg")) {
541 cm_format = kCMVideoCodecType_JPEG;
542 } else if (!strcmp (caps_name, "video/x-prores")) {
543 const char *variant = gst_structure_get_string (structure, "variant");
544
545 if (variant)
546 cm_format = gst_vtutil_codec_type_from_prores_variant (variant);
547
548 if (cm_format == GST_kCMVideoCodecType_Some_AppleProRes) {
549 GST_ERROR_OBJECT (vtdec, "Invalid ProRes variant %s", variant);
550 return FALSE;
551 }
552 }
553
554 if (cm_format == kCMVideoCodecType_H264 && state->codec_data == NULL) {
555 GST_INFO_OBJECT (vtdec, "no codec data, wait for one");
556 return TRUE;
557 }
558
559 gst_video_info_from_caps (&vtdec->video_info, state->caps);
560
561 if (!gst_vtdec_compute_reorder_queue_length (vtdec, cm_format,
562 state->codec_data))
563 return FALSE;
564 gst_vtdec_set_latency (vtdec);
565
566 if (state->codec_data) {
567 format_description = create_format_description_from_codec_data (vtdec,
568 cm_format, state->codec_data);
569 } else {
570 format_description = create_format_description (vtdec, cm_format);
571 }
572
573 if (vtdec->format_description)
574 CFRelease (vtdec->format_description);
575 vtdec->format_description = format_description;
576
577 if (vtdec->input_state)
578 gst_video_codec_state_unref (vtdec->input_state);
579 vtdec->input_state = gst_video_codec_state_ref (state);
580
581 return gst_video_decoder_negotiate (decoder);
582 }
583
584 static gboolean
gst_vtdec_flush(GstVideoDecoder * decoder)585 gst_vtdec_flush (GstVideoDecoder * decoder)
586 {
587 GstVtdec *vtdec = GST_VTDEC (decoder);
588
589 GST_DEBUG_OBJECT (vtdec, "flush");
590
591 gst_vtdec_push_frames_if_needed (vtdec, FALSE, TRUE);
592
593 return TRUE;
594 }
595
596 static GstFlowReturn
gst_vtdec_finish(GstVideoDecoder * decoder)597 gst_vtdec_finish (GstVideoDecoder * decoder)
598 {
599 GstVtdec *vtdec = GST_VTDEC (decoder);
600
601 GST_DEBUG_OBJECT (vtdec, "finish");
602
603 return gst_vtdec_push_frames_if_needed (vtdec, TRUE, FALSE);
604 }
605
606 static GstFlowReturn
gst_vtdec_handle_frame(GstVideoDecoder * decoder,GstVideoCodecFrame * frame)607 gst_vtdec_handle_frame (GstVideoDecoder * decoder, GstVideoCodecFrame * frame)
608 {
609 OSStatus status;
610 CMSampleBufferRef cm_sample_buffer = NULL;
611 VTDecodeFrameFlags input_flags;
612 GstVtdec *vtdec = GST_VTDEC (decoder);
613 GstFlowReturn ret = GST_FLOW_OK;
614 int decode_frame_number = frame->decode_frame_number;
615
616 if (vtdec->format_description == NULL) {
617 ret = GST_FLOW_NOT_NEGOTIATED;
618 goto out;
619 }
620
621 GST_LOG_OBJECT (vtdec, "got input frame %d", decode_frame_number);
622
623 ret = gst_vtdec_push_frames_if_needed (vtdec, FALSE, FALSE);
624 if (ret != GST_FLOW_OK)
625 return ret;
626
627 /* don't bother enabling kVTDecodeFrame_EnableTemporalProcessing at all since
628 * it's not mandatory for the underlying VT codec to respect it. KISS and do
629 * reordering ourselves.
630 */
631 input_flags = kVTDecodeFrame_EnableAsynchronousDecompression;
632
633 cm_sample_buffer =
634 cm_sample_buffer_from_gst_buffer (vtdec, frame->input_buffer);
635 status =
636 VTDecompressionSessionDecodeFrame (vtdec->session, cm_sample_buffer,
637 input_flags, frame, NULL);
638 if (status != noErr && FALSE)
639 goto error;
640
641 GST_LOG_OBJECT (vtdec, "submitted input frame %d", decode_frame_number);
642
643 out:
644 if (cm_sample_buffer)
645 CFRelease (cm_sample_buffer);
646 return ret;
647
648 error:
649 GST_ELEMENT_ERROR (vtdec, STREAM, DECODE, (NULL),
650 ("VTDecompressionSessionDecodeFrame returned %d", (int) status));
651 ret = GST_FLOW_ERROR;
652 goto out;
653 }
654
655 static void
gst_vtdec_invalidate_session(GstVtdec * vtdec)656 gst_vtdec_invalidate_session (GstVtdec * vtdec)
657 {
658 g_return_if_fail (vtdec->session);
659
660 VTDecompressionSessionInvalidate (vtdec->session);
661 CFRelease (vtdec->session);
662 vtdec->session = NULL;
663 }
664
665 static OSStatus
gst_vtdec_create_session(GstVtdec * vtdec,GstVideoFormat format,gboolean enable_hardware)666 gst_vtdec_create_session (GstVtdec * vtdec, GstVideoFormat format,
667 gboolean enable_hardware)
668 {
669 CFMutableDictionaryRef output_image_buffer_attrs;
670 VTDecompressionOutputCallbackRecord callback;
671 CFMutableDictionaryRef videoDecoderSpecification;
672 OSStatus status;
673 guint32 cv_format = 0;
674
675 g_return_val_if_fail (vtdec->session == NULL, FALSE);
676
677 switch (format) {
678 case GST_VIDEO_FORMAT_NV12:
679 cv_format = kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange;
680 break;
681 case GST_VIDEO_FORMAT_AYUV64:
682 /* This is fine for now because Apple only ships LE devices */
683 #if G_BYTE_ORDER != G_LITTLE_ENDIAN
684 #error "AYUV64 is NE but kCVPixelFormatType_4444AYpCbCr16 is LE"
685 #endif
686 cv_format = kCVPixelFormatType_4444AYpCbCr16;
687 break;
688 case GST_VIDEO_FORMAT_ARGB64_BE:
689 cv_format = kCVPixelFormatType_64ARGB;
690 break;
691 case GST_VIDEO_FORMAT_RGBA64_LE:
692 if (GST_VTUTIL_HAVE_64ARGBALE)
693 cv_format = kCVPixelFormatType_64RGBALE;
694 else
695 /* Codepath will never be hit on macOS older than Big Sur (11.3) */
696 g_warn_if_reached ();
697 break;
698 default:
699 g_warn_if_reached ();
700 break;
701 }
702
703 videoDecoderSpecification =
704 CFDictionaryCreateMutable (NULL, 0, &kCFTypeDictionaryKeyCallBacks,
705 &kCFTypeDictionaryValueCallBacks);
706
707 /* This is the default on iOS and the key does not exist there */
708 #ifndef HAVE_IOS
709 gst_vtutil_dict_set_boolean (videoDecoderSpecification,
710 kVTVideoDecoderSpecification_EnableHardwareAcceleratedVideoDecoder,
711 enable_hardware);
712 if (enable_hardware && vtdec->require_hardware)
713 gst_vtutil_dict_set_boolean (videoDecoderSpecification,
714 kVTVideoDecoderSpecification_RequireHardwareAcceleratedVideoDecoder,
715 TRUE);
716 #endif
717
718 output_image_buffer_attrs =
719 CFDictionaryCreateMutable (NULL, 0, &kCFTypeDictionaryKeyCallBacks,
720 &kCFTypeDictionaryValueCallBacks);
721 gst_vtutil_dict_set_i32 (output_image_buffer_attrs,
722 kCVPixelBufferPixelFormatTypeKey, cv_format);
723 gst_vtutil_dict_set_i32 (output_image_buffer_attrs, kCVPixelBufferWidthKey,
724 vtdec->video_info.width);
725 gst_vtutil_dict_set_i32 (output_image_buffer_attrs, kCVPixelBufferHeightKey,
726 vtdec->video_info.height);
727
728 callback.decompressionOutputCallback = gst_vtdec_session_output_callback;
729 callback.decompressionOutputRefCon = vtdec;
730
731 status = VTDecompressionSessionCreate (NULL, vtdec->format_description,
732 videoDecoderSpecification, output_image_buffer_attrs, &callback,
733 &vtdec->session);
734
735 if (videoDecoderSpecification)
736 CFRelease (videoDecoderSpecification);
737
738 CFRelease (output_image_buffer_attrs);
739
740 return status;
741 }
742
743 static CMFormatDescriptionRef
create_format_description(GstVtdec * vtdec,CMVideoCodecType cm_format)744 create_format_description (GstVtdec * vtdec, CMVideoCodecType cm_format)
745 {
746 OSStatus status;
747 CMFormatDescriptionRef format_description;
748
749 status = CMVideoFormatDescriptionCreate (NULL,
750 cm_format, vtdec->video_info.width, vtdec->video_info.height,
751 NULL, &format_description);
752 if (status != noErr)
753 return NULL;
754
755 return format_description;
756 }
757
758 static CMFormatDescriptionRef
create_format_description_from_codec_data(GstVtdec * vtdec,CMVideoCodecType cm_format,GstBuffer * codec_data)759 create_format_description_from_codec_data (GstVtdec * vtdec,
760 CMVideoCodecType cm_format, GstBuffer * codec_data)
761 {
762 CMFormatDescriptionRef fmt_desc;
763 CFMutableDictionaryRef extensions, par, atoms;
764 GstMapInfo map;
765 OSStatus status;
766
767 /* Extensions dict */
768 extensions =
769 CFDictionaryCreateMutable (NULL, 0, &kCFTypeDictionaryKeyCallBacks,
770 &kCFTypeDictionaryValueCallBacks);
771 gst_vtutil_dict_set_string (extensions,
772 CFSTR ("CVImageBufferChromaLocationBottomField"), "left");
773 gst_vtutil_dict_set_string (extensions,
774 CFSTR ("CVImageBufferChromaLocationTopField"), "left");
775 gst_vtutil_dict_set_boolean (extensions, CFSTR ("FullRangeVideo"), FALSE);
776
777 /* CVPixelAspectRatio dict */
778 par = CFDictionaryCreateMutable (NULL, 0, &kCFTypeDictionaryKeyCallBacks,
779 &kCFTypeDictionaryValueCallBacks);
780 gst_vtutil_dict_set_i32 (par, CFSTR ("HorizontalSpacing"),
781 vtdec->video_info.par_n);
782 gst_vtutil_dict_set_i32 (par, CFSTR ("VerticalSpacing"),
783 vtdec->video_info.par_d);
784 gst_vtutil_dict_set_object (extensions, CFSTR ("CVPixelAspectRatio"),
785 (CFTypeRef *) par);
786
787 /* SampleDescriptionExtensionAtoms dict */
788 gst_buffer_map (codec_data, &map, GST_MAP_READ);
789 atoms = CFDictionaryCreateMutable (NULL, 0, &kCFTypeDictionaryKeyCallBacks,
790 &kCFTypeDictionaryValueCallBacks);
791 gst_vtutil_dict_set_data (atoms, CFSTR ("avcC"), map.data, map.size);
792 gst_vtutil_dict_set_object (extensions,
793 CFSTR ("SampleDescriptionExtensionAtoms"), (CFTypeRef *) atoms);
794 gst_buffer_unmap (codec_data, &map);
795
796 status = CMVideoFormatDescriptionCreate (NULL,
797 cm_format, vtdec->video_info.width, vtdec->video_info.height,
798 extensions, &fmt_desc);
799
800 if (extensions)
801 CFRelease (extensions);
802
803 if (status == noErr)
804 return fmt_desc;
805 else
806 return NULL;
807 }
808
809 /* Custom FreeBlock function for CMBlockBuffer */
810 static void
cm_block_buffer_freeblock(void * refCon,void * doomedMemoryBlock,size_t sizeInBytes)811 cm_block_buffer_freeblock (void *refCon, void *doomedMemoryBlock,
812 size_t sizeInBytes)
813 {
814 GstMapInfo *info = (GstMapInfo *) refCon;
815
816 gst_memory_unmap (info->memory, info);
817 gst_memory_unref (info->memory);
818 g_slice_free (GstMapInfo, info);
819 }
820
821 static CMBlockBufferRef
cm_block_buffer_from_gst_buffer(GstBuffer * buf,GstMapFlags flags)822 cm_block_buffer_from_gst_buffer (GstBuffer * buf, GstMapFlags flags)
823 {
824 OSStatus status;
825 CMBlockBufferRef bbuf;
826 CMBlockBufferCustomBlockSource blockSource;
827 guint memcount, i;
828
829 /* Initialize custom block source structure */
830 blockSource.version = kCMBlockBufferCustomBlockSourceVersion;
831 blockSource.AllocateBlock = NULL;
832 blockSource.FreeBlock = cm_block_buffer_freeblock;
833
834 /* Determine number of memory blocks */
835 memcount = gst_buffer_n_memory (buf);
836 status = CMBlockBufferCreateEmpty (NULL, memcount, 0, &bbuf);
837 if (status != kCMBlockBufferNoErr) {
838 GST_ERROR ("CMBlockBufferCreateEmpty returned %d", (int) status);
839 return NULL;
840 }
841
842 /* Go over all GstMemory objects and add them to the CMBlockBuffer */
843 for (i = 0; i < memcount; ++i) {
844 GstMemory *mem;
845 GstMapInfo *info;
846
847 mem = gst_buffer_get_memory (buf, i);
848
849 info = g_slice_new (GstMapInfo);
850 if (!gst_memory_map (mem, info, flags)) {
851 GST_ERROR ("failed mapping memory");
852 g_slice_free (GstMapInfo, info);
853 gst_memory_unref (mem);
854 CFRelease (bbuf);
855 return NULL;
856 }
857
858 blockSource.refCon = info;
859 status =
860 CMBlockBufferAppendMemoryBlock (bbuf, info->data, info->size, NULL,
861 &blockSource, 0, info->size, 0);
862 if (status != kCMBlockBufferNoErr) {
863 GST_ERROR ("CMBlockBufferAppendMemoryBlock returned %d", (int) status);
864 gst_memory_unmap (mem, info);
865 g_slice_free (GstMapInfo, info);
866 gst_memory_unref (mem);
867 CFRelease (bbuf);
868 return NULL;
869 }
870 }
871
872 return bbuf;
873 }
874
875 static CMSampleBufferRef
cm_sample_buffer_from_gst_buffer(GstVtdec * vtdec,GstBuffer * buf)876 cm_sample_buffer_from_gst_buffer (GstVtdec * vtdec, GstBuffer * buf)
877 {
878 OSStatus status;
879 CMBlockBufferRef bbuf = NULL;
880 CMSampleBufferRef sbuf = NULL;
881 CMSampleTimingInfo sample_timing;
882 CMSampleTimingInfo time_array[1];
883
884 g_return_val_if_fail (vtdec->format_description, NULL);
885
886 /* create a block buffer */
887 bbuf = cm_block_buffer_from_gst_buffer (buf, GST_MAP_READ);
888 if (bbuf == NULL) {
889 GST_ELEMENT_ERROR (vtdec, RESOURCE, FAILED, (NULL),
890 ("failed creating CMBlockBuffer"));
891 return NULL;
892 }
893
894 /* create a sample buffer */
895 if (GST_BUFFER_DURATION_IS_VALID (buf))
896 sample_timing.duration = CMTimeMake (GST_BUFFER_DURATION (buf), GST_SECOND);
897 else
898 sample_timing.duration = kCMTimeInvalid;
899
900 if (GST_BUFFER_PTS_IS_VALID (buf))
901 sample_timing.presentationTimeStamp =
902 CMTimeMake (GST_BUFFER_PTS (buf), GST_SECOND);
903 else
904 sample_timing.presentationTimeStamp = kCMTimeInvalid;
905
906 if (GST_BUFFER_DTS_IS_VALID (buf))
907 sample_timing.decodeTimeStamp =
908 CMTimeMake (GST_BUFFER_DTS (buf), GST_SECOND);
909 else
910 sample_timing.decodeTimeStamp = kCMTimeInvalid;
911
912 time_array[0] = sample_timing;
913
914 status =
915 CMSampleBufferCreate (NULL, bbuf, TRUE, 0, 0, vtdec->format_description,
916 1, 1, time_array, 0, NULL, &sbuf);
917 CFRelease (bbuf);
918 if (status != noErr) {
919 GST_ELEMENT_ERROR (vtdec, RESOURCE, FAILED, (NULL),
920 ("CMSampleBufferCreate returned %d", (int) status));
921 return NULL;
922 }
923
924 return sbuf;
925 }
926
927 static gint
sort_frames_by_pts(gconstpointer f1,gconstpointer f2,gpointer user_data)928 sort_frames_by_pts (gconstpointer f1, gconstpointer f2, gpointer user_data)
929 {
930 GstVideoCodecFrame *frame1, *frame2;
931 GstClockTime pts1, pts2;
932
933 frame1 = (GstVideoCodecFrame *) f1;
934 frame2 = (GstVideoCodecFrame *) f2;
935 pts1 = pts2 = GST_CLOCK_TIME_NONE;
936 if (frame1->output_buffer)
937 pts1 = GST_BUFFER_PTS (frame1->output_buffer);
938 if (frame2->output_buffer)
939 pts2 = GST_BUFFER_PTS (frame2->output_buffer);
940
941 if (!GST_CLOCK_TIME_IS_VALID (pts1) || !GST_CLOCK_TIME_IS_VALID (pts2))
942 return 0;
943
944 if (pts1 < pts2)
945 return -1;
946 else if (pts1 == pts2)
947 return 0;
948 else
949 return 1;
950 }
951
952 static void
gst_vtdec_session_output_callback(void * decompression_output_ref_con,void * source_frame_ref_con,OSStatus status,VTDecodeInfoFlags info_flags,CVImageBufferRef image_buffer,CMTime pts,CMTime duration)953 gst_vtdec_session_output_callback (void *decompression_output_ref_con,
954 void *source_frame_ref_con, OSStatus status, VTDecodeInfoFlags info_flags,
955 CVImageBufferRef image_buffer, CMTime pts, CMTime duration)
956 {
957 GstVtdec *vtdec = (GstVtdec *) decompression_output_ref_con;
958 GstVideoCodecFrame *frame = (GstVideoCodecFrame *) source_frame_ref_con;
959 GstVideoCodecState *state;
960
961 GST_LOG_OBJECT (vtdec, "got output frame %p %d and VT buffer %p", frame,
962 frame->decode_frame_number, image_buffer);
963
964 frame->output_buffer = NULL;
965
966 if (status != noErr) {
967 GST_ERROR_OBJECT (vtdec, "Error decoding frame %d", (int) status);
968 }
969
970 if (image_buffer) {
971 GstBuffer *buf = NULL;
972
973 /* FIXME: use gst_video_decoder_allocate_output_buffer */
974 state = gst_video_decoder_get_output_state (GST_VIDEO_DECODER (vtdec));
975 if (state == NULL) {
976 GST_WARNING_OBJECT (vtdec, "Output state not configured, release buffer");
977 frame->flags &= VTDEC_FRAME_FLAG_SKIP;
978 } else {
979 buf =
980 gst_core_video_buffer_new (image_buffer, &state->info,
981 vtdec->texture_cache);
982 gst_video_codec_state_unref (state);
983 GST_BUFFER_PTS (buf) = pts.value;
984 GST_BUFFER_DURATION (buf) = duration.value;
985 frame->output_buffer = buf;
986 }
987 } else {
988 if (info_flags & kVTDecodeInfo_FrameDropped) {
989 GST_DEBUG_OBJECT (vtdec, "Frame dropped by video toolbox %p %d",
990 frame, frame->decode_frame_number);
991 frame->flags |= VTDEC_FRAME_FLAG_DROP;
992 } else {
993 GST_DEBUG_OBJECT (vtdec, "Decoded frame is NULL");
994 frame->flags |= VTDEC_FRAME_FLAG_SKIP;
995 }
996 }
997
998 g_async_queue_push_sorted (vtdec->reorder_queue, frame,
999 sort_frames_by_pts, NULL);
1000 }
1001
1002 static GstFlowReturn
gst_vtdec_push_frames_if_needed(GstVtdec * vtdec,gboolean drain,gboolean flush)1003 gst_vtdec_push_frames_if_needed (GstVtdec * vtdec, gboolean drain,
1004 gboolean flush)
1005 {
1006 GstVideoCodecFrame *frame;
1007 GstFlowReturn ret = GST_FLOW_OK;
1008 GstVideoDecoder *decoder = GST_VIDEO_DECODER (vtdec);
1009
1010 /* negotiate now so that we know whether we need to use the GL upload meta or
1011 * not */
1012 if (gst_pad_check_reconfigure (decoder->srcpad)) {
1013 if (!gst_video_decoder_negotiate (decoder)) {
1014 gst_pad_mark_reconfigure (decoder->srcpad);
1015 if (GST_PAD_IS_FLUSHING (decoder->srcpad))
1016 ret = GST_FLOW_FLUSHING;
1017 else
1018 ret = GST_FLOW_NOT_NEGOTIATED;
1019 return ret;
1020 }
1021 }
1022
1023 if (drain)
1024 VTDecompressionSessionWaitForAsynchronousFrames (vtdec->session);
1025
1026 /* push a buffer if there are enough frames to guarantee that we push in PTS
1027 * order
1028 */
1029 while ((g_async_queue_length (vtdec->reorder_queue) >=
1030 vtdec->reorder_queue_length) || drain || flush) {
1031 frame = (GstVideoCodecFrame *) g_async_queue_try_pop (vtdec->reorder_queue);
1032
1033 /* we need to check this in case reorder_queue_length=0 (jpeg for
1034 * example) or we're draining/flushing
1035 */
1036 if (frame) {
1037 if (frame->flags & VTDEC_FRAME_FLAG_ERROR) {
1038 gst_video_decoder_release_frame (decoder, frame);
1039 ret = GST_FLOW_ERROR;
1040 } else if (flush || frame->flags & VTDEC_FRAME_FLAG_SKIP) {
1041 gst_video_decoder_release_frame (decoder, frame);
1042 } else if (frame->flags & VTDEC_FRAME_FLAG_DROP) {
1043 gst_video_decoder_drop_frame (decoder, frame);
1044 } else {
1045 ret = gst_video_decoder_finish_frame (decoder, frame);
1046 }
1047 }
1048
1049 if (!frame || ret != GST_FLOW_OK)
1050 break;
1051 }
1052
1053 return ret;
1054 }
1055
1056 static gboolean
parse_h264_profile_and_level_from_codec_data(GstVtdec * vtdec,GstBuffer * codec_data,int * profile,int * level)1057 parse_h264_profile_and_level_from_codec_data (GstVtdec * vtdec,
1058 GstBuffer * codec_data, int *profile, int *level)
1059 {
1060 GstMapInfo map;
1061 guint8 *data;
1062 gint size;
1063 gboolean ret = TRUE;
1064
1065 gst_buffer_map (codec_data, &map, GST_MAP_READ);
1066 data = map.data;
1067 size = map.size;
1068
1069 /* parse the avcC data */
1070 if (size < 7)
1071 goto avcc_too_small;
1072
1073 /* parse the version, this must be 1 */
1074 if (data[0] != 1)
1075 goto wrong_version;
1076
1077 /* AVCProfileIndication */
1078 /* profile_compat */
1079 /* AVCLevelIndication */
1080 if (profile)
1081 *profile = data[1];
1082
1083 if (level)
1084 *level = data[3];
1085
1086 out:
1087 gst_buffer_unmap (codec_data, &map);
1088
1089 return ret;
1090
1091 avcc_too_small:
1092 GST_ELEMENT_ERROR (vtdec, STREAM, DECODE, (NULL),
1093 ("invalid codec_data buffer length"));
1094 ret = FALSE;
1095 goto out;
1096
1097 wrong_version:
1098 GST_ELEMENT_ERROR (vtdec, STREAM, DECODE, (NULL),
1099 ("wrong avcC version in codec_data"));
1100 ret = FALSE;
1101 goto out;
1102 }
1103
1104 static int
get_dpb_max_mb_s_from_level(GstVtdec * vtdec,int level)1105 get_dpb_max_mb_s_from_level (GstVtdec * vtdec, int level)
1106 {
1107 switch (level) {
1108 case 10:
1109 /* 1b?? */
1110 return 396;
1111 case 11:
1112 return 900;
1113 case 12:
1114 case 13:
1115 case 20:
1116 return 2376;
1117 case 21:
1118 return 4752;
1119 case 22:
1120 case 30:
1121 return 8100;
1122 case 31:
1123 return 18000;
1124 case 32:
1125 return 20480;
1126 case 40:
1127 case 41:
1128 return 32768;
1129 case 42:
1130 return 34816;
1131 case 50:
1132 return 110400;
1133 case 51:
1134 case 52:
1135 return 184320;
1136 default:
1137 GST_ERROR_OBJECT (vtdec, "unknown level %d", level);
1138 return -1;
1139 }
1140 }
1141
1142 static gboolean
gst_vtdec_compute_reorder_queue_length(GstVtdec * vtdec,CMVideoCodecType cm_format,GstBuffer * codec_data)1143 gst_vtdec_compute_reorder_queue_length (GstVtdec * vtdec,
1144 CMVideoCodecType cm_format, GstBuffer * codec_data)
1145 {
1146 if (cm_format == kCMVideoCodecType_H264) {
1147 if (!compute_h264_decode_picture_buffer_length (vtdec, codec_data,
1148 &vtdec->reorder_queue_length)) {
1149 return FALSE;
1150 }
1151 } else {
1152 vtdec->reorder_queue_length = 0;
1153 }
1154
1155 return TRUE;
1156 }
1157
1158 static gboolean
compute_h264_decode_picture_buffer_length(GstVtdec * vtdec,GstBuffer * codec_data,int * length)1159 compute_h264_decode_picture_buffer_length (GstVtdec * vtdec,
1160 GstBuffer * codec_data, int *length)
1161 {
1162 int profile, level;
1163 int dpb_mb_size = 16;
1164 int max_dpb_size_frames = 16;
1165 int max_dpb_mb_s = -1;
1166 int width_in_mb_s = GST_ROUND_UP_16 (vtdec->video_info.width) / dpb_mb_size;
1167 int height_in_mb_s = GST_ROUND_UP_16 (vtdec->video_info.height) / dpb_mb_size;
1168
1169 *length = 0;
1170
1171 if (!parse_h264_profile_and_level_from_codec_data (vtdec, codec_data,
1172 &profile, &level))
1173 return FALSE;
1174
1175 if (vtdec->video_info.width == 0 || vtdec->video_info.height == 0)
1176 return FALSE;
1177
1178 GST_INFO_OBJECT (vtdec, "parsed profile %d, level %d", profile, level);
1179 if (profile == 66) {
1180 /* baseline or constrained-baseline, we don't need to reorder */
1181 return TRUE;
1182 }
1183
1184 max_dpb_mb_s = get_dpb_max_mb_s_from_level (vtdec, level);
1185 if (max_dpb_mb_s == -1) {
1186 GST_ELEMENT_ERROR (vtdec, STREAM, DECODE, (NULL),
1187 ("invalid level in codec_data, could not compute max_dpb_mb_s"));
1188 return FALSE;
1189 }
1190
1191 /* this formula is specified in sections A.3.1.h and A.3.2.f of the 2009
1192 * edition of the standard */
1193 *length = MIN (floor (max_dpb_mb_s / (width_in_mb_s * height_in_mb_s)),
1194 max_dpb_size_frames);
1195 return TRUE;
1196 }
1197
1198 static void
gst_vtdec_set_latency(GstVtdec * vtdec)1199 gst_vtdec_set_latency (GstVtdec * vtdec)
1200 {
1201 GstClockTime frame_duration;
1202 GstClockTime latency;
1203
1204 if (vtdec->video_info.fps_n == 0) {
1205 GST_INFO_OBJECT (vtdec, "Framerate not known, can't set latency");
1206 return;
1207 }
1208
1209 frame_duration = gst_util_uint64_scale (GST_SECOND,
1210 vtdec->video_info.fps_d, vtdec->video_info.fps_n);
1211 latency = frame_duration * vtdec->reorder_queue_length;
1212
1213 GST_INFO_OBJECT (vtdec, "setting latency frames:%d time:%" GST_TIME_FORMAT,
1214 vtdec->reorder_queue_length, GST_TIME_ARGS (latency));
1215 gst_video_decoder_set_latency (GST_VIDEO_DECODER (vtdec), latency, latency);
1216 }
1217
1218 static void
gst_vtdec_set_context(GstElement * element,GstContext * context)1219 gst_vtdec_set_context (GstElement * element, GstContext * context)
1220 {
1221 GstVtdec *vtdec = GST_VTDEC (element);
1222
1223 GST_INFO_OBJECT (element, "setting context %s",
1224 gst_context_get_context_type (context));
1225 if (!vtdec->ctxh)
1226 vtdec->ctxh = gst_gl_context_helper_new (element);
1227 gst_gl_handle_set_context (element, context,
1228 &vtdec->ctxh->display, &vtdec->ctxh->other_context);
1229
1230 #if defined (APPLEMEDIA_MOLTENVK)
1231 gst_vulkan_handle_set_context (element, context, NULL, &vtdec->instance);
1232 #endif
1233
1234 GST_ELEMENT_CLASS (gst_vtdec_parent_class)->set_context (element, context);
1235 }
1236
1237 #ifndef HAVE_IOS
1238 #define GST_TYPE_VTDEC_HW (gst_vtdec_hw_get_type())
1239 #define GST_VTDEC_HW(obj) (G_TYPE_CHECK_INSTANCE_CAST((obj),GST_TYPE_VTDEC_HW,GstVtdecHw))
1240 #define GST_VTDEC_HW_CLASS(klass) (G_TYPE_CHECK_CLASS_CAST((klass),GST_TYPE_VTDEC_HW,GstVtdecHwClass))
1241 #define GST_IS_VTDEC_HW(obj) (G_TYPE_CHECK_INSTANCE_TYPE((obj),GST_TYPE_VTDEC_HW))
1242 #define GST_IS_VTDEC_HW_CLASS(obj) (G_TYPE_CHECK_CLASS_TYPE((klass),GST_TYPE_VTDEC_HW))
1243
1244 typedef GstVtdec GstVtdecHw;
1245 typedef GstVtdecClass GstVtdecHwClass;
1246
1247 GType gst_vtdec_hw_get_type (void);
1248
1249 G_DEFINE_TYPE (GstVtdecHw, gst_vtdec_hw, GST_TYPE_VTDEC);
1250
1251 static void
gst_vtdec_hw_class_init(GstVtdecHwClass * klass)1252 gst_vtdec_hw_class_init (GstVtdecHwClass * klass)
1253 {
1254 gst_element_class_set_static_metadata (GST_ELEMENT_CLASS (klass),
1255 "Apple VideoToolbox decoder (hardware only)",
1256 "Codec/Decoder/Video/Hardware",
1257 "Apple VideoToolbox Decoder",
1258 "Ole André Vadla Ravnås <oleavr@soundrop.com>; "
1259 "Alessandro Decina <alessandro.d@gmail.com>");
1260 }
1261
1262 static void
gst_vtdec_hw_init(GstVtdecHw * vtdec)1263 gst_vtdec_hw_init (GstVtdecHw * vtdec)
1264 {
1265 GST_VTDEC (vtdec)->require_hardware = TRUE;
1266 }
1267
1268 #endif
1269
1270 void
gst_vtdec_register_elements(GstPlugin * plugin)1271 gst_vtdec_register_elements (GstPlugin * plugin)
1272 {
1273 GST_DEBUG_CATEGORY_INIT (gst_vtdec_debug_category, "vtdec", 0,
1274 "debug category for vtdec element");
1275
1276 #ifdef HAVE_IOS
1277 gst_element_register (plugin, "vtdec", GST_RANK_PRIMARY, GST_TYPE_VTDEC);
1278 #else
1279 gst_element_register (plugin, "vtdec_hw", GST_RANK_PRIMARY + 1,
1280 GST_TYPE_VTDEC_HW);
1281 gst_element_register (plugin, "vtdec", GST_RANK_SECONDARY, GST_TYPE_VTDEC);
1282 #endif
1283 }
1284