1 /*
2 * Copyright (C) 2010, 2013 Ole André Vadla Ravnås <oleavr@soundrop.com>
3 * Copyright (C) 2013 Intel Corporation
4 *
5 * This library is free software; you can redistribute it and/or
6 * modify it under the terms of the GNU Library General Public
7 * License as published by the Free Software Foundation; either
8 * version 2 of the License, or (at your option) any later version.
9 *
10 * This library is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 * Library General Public License for more details.
14 *
15 * You should have received a copy of the GNU Library General Public
16 * License along with this library; if not, write to the
17 * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
18 * Boston, MA 02110-1301, USA.
19 */
20
21 /**
22 * SECTION:element-vtenc_h264
23 * @title: vtenc_h264
24 *
25 * Apple VideoToolbox H264 encoder, which can either use HW or a SW
26 * implementation depending on the device.
27 *
28 * ## Example pipeline
29 * |[
30 * gst-launch-1.0 -v videotestsrc ! vtenc_h264 ! qtmux ! filesink location=out.mov
31 * ]| Encode a test video pattern and save it as an MOV file
32 *
33 */
34
35 /**
36 * SECTION:element-vtenc_h264_hw
37 * @title: vtenc_h264_hw
38 *
39 * Apple VideoToolbox H264 HW-only encoder (only available on macOS at
40 * present).
41 *
42 * ## Example pipeline
43 * |[
44 * gst-launch-1.0 -v videotestsrc ! vtenc_h264_hw ! qtmux ! filesink location=out.mov
45 * ]| Encode a test video pattern and save it as an MOV file
46 *
47 */
48
49 /**
50 * SECTION:element-vtenc_prores
51 * @title: vtenc_prores
52 *
53 * Apple VideoToolbox ProRes encoder
54 *
55 * ## Example pipeline
56 * |[
57 * gst-launch-1.0 -v videotestsrc ! vtenc_prores ! qtmux ! filesink location=out.mov
58 * ]| Encode a test video pattern and save it as an MOV file
59 *
60 * Since: 1.20
61 */
62
63 #ifdef HAVE_CONFIG_H
64 #include "config.h"
65 #endif
66
67 #include "vtenc.h"
68
69 #include "coremediabuffer.h"
70 #include "corevideobuffer.h"
71 #include "vtutil.h"
72 #include <gst/pbutils/codec-utils.h>
73
74 #define VTENC_DEFAULT_USAGE 6 /* Profile: Baseline Level: 2.1 */
75 #define VTENC_DEFAULT_BITRATE 0
76 #define VTENC_DEFAULT_FRAME_REORDERING TRUE
77 #define VTENC_DEFAULT_REALTIME FALSE
78 #define VTENC_DEFAULT_QUALITY 0.5
79 #define VTENC_DEFAULT_MAX_KEYFRAME_INTERVAL 0
80 #define VTENC_DEFAULT_MAX_KEYFRAME_INTERVAL_DURATION 0
81 #define VTENC_DEFAULT_PRESERVE_ALPHA TRUE
82
83 GST_DEBUG_CATEGORY (gst_vtenc_debug);
84 #define GST_CAT_DEFAULT (gst_vtenc_debug)
85
86 #define GST_VTENC_CODEC_DETAILS_QDATA \
87 g_quark_from_static_string ("vtenc-codec-details")
88
89 /* define EnableHardwareAcceleratedVideoEncoder in < 10.9 */
90 #if defined(MAC_OS_X_VERSION_MAX_ALLOWED) && MAC_OS_X_VERSION_MAX_ALLOWED < 1090
91 const CFStringRef
92 kVTVideoEncoderSpecification_EnableHardwareAcceleratedVideoEncoder =
93 CFSTR ("EnableHardwareAcceleratedVideoEncoder");
94 const CFStringRef
95 kVTVideoEncoderSpecification_RequireHardwareAcceleratedVideoEncoder =
96 CFSTR ("RequireHardwareAcceleratedVideoEncoder");
97 const CFStringRef kVTCompressionPropertyKey_ProfileLevel =
98 CFSTR ("ProfileLevel");
99 const CFStringRef kVTProfileLevel_H264_Baseline_AutoLevel =
100 CFSTR ("H264_Baseline_AutoLevel");
101 #endif
102
103 #if defined(MAC_OS_X_VERSION_MAX_ALLOWED) && MAC_OS_X_VERSION_MAX_ALLOWED < 1080
104 const CFStringRef kVTCompressionPropertyKey_Quality = CFSTR ("Quality");
105 #endif
106
107 #ifdef HAVE_VIDEOTOOLBOX_10_9_6
108 extern OSStatus
109 VTCompressionSessionPrepareToEncodeFrames (VTCompressionSessionRef session)
110 __attribute__ ((weak_import));
111 #endif
112
113 /* This property key is currently completely undocumented. The only way you can
114 * know about its existence is if Apple tells you. It allows you to tell the
115 * encoder to not preserve alpha even when outputting alpha formats. */
116 const CFStringRef gstVTCodecPropertyKey_PreserveAlphaChannel =
117 CFSTR ("kVTCodecPropertyKey_PreserveAlphaChannel");
118
119 enum
120 {
121 PROP_0,
122 PROP_USAGE,
123 PROP_BITRATE,
124 PROP_ALLOW_FRAME_REORDERING,
125 PROP_REALTIME,
126 PROP_QUALITY,
127 PROP_MAX_KEYFRAME_INTERVAL,
128 PROP_MAX_KEYFRAME_INTERVAL_DURATION,
129 PROP_PRESERVE_ALPHA,
130 };
131
132 typedef struct _GstVTEncFrame GstVTEncFrame;
133
134 struct _GstVTEncFrame
135 {
136 GstBuffer *buf;
137 GstVideoFrame videoframe;
138 };
139
140 static GstElementClass *parent_class = NULL;
141
142 static void gst_vtenc_get_property (GObject * obj, guint prop_id,
143 GValue * value, GParamSpec * pspec);
144 static void gst_vtenc_set_property (GObject * obj, guint prop_id,
145 const GValue * value, GParamSpec * pspec);
146 static void gst_vtenc_finalize (GObject * obj);
147
148 static gboolean gst_vtenc_start (GstVideoEncoder * enc);
149 static gboolean gst_vtenc_stop (GstVideoEncoder * enc);
150 static gboolean gst_vtenc_set_format (GstVideoEncoder * enc,
151 GstVideoCodecState * input_state);
152 static GstFlowReturn gst_vtenc_handle_frame (GstVideoEncoder * enc,
153 GstVideoCodecFrame * frame);
154 static GstFlowReturn gst_vtenc_finish (GstVideoEncoder * enc);
155 static gboolean gst_vtenc_flush (GstVideoEncoder * enc);
156
157 static void gst_vtenc_clear_cached_caps_downstream (GstVTEnc * self);
158
159 static VTCompressionSessionRef gst_vtenc_create_session (GstVTEnc * self);
160 static void gst_vtenc_destroy_session (GstVTEnc * self,
161 VTCompressionSessionRef * session);
162 static void gst_vtenc_session_dump_properties (GstVTEnc * self,
163 VTCompressionSessionRef session);
164 static void gst_vtenc_session_configure_expected_framerate (GstVTEnc * self,
165 VTCompressionSessionRef session, gdouble framerate);
166 static void gst_vtenc_session_configure_max_keyframe_interval (GstVTEnc * self,
167 VTCompressionSessionRef session, gint interval);
168 static void gst_vtenc_session_configure_max_keyframe_interval_duration
169 (GstVTEnc * self, VTCompressionSessionRef session, gdouble duration);
170 static void gst_vtenc_session_configure_bitrate (GstVTEnc * self,
171 VTCompressionSessionRef session, guint bitrate);
172 static OSStatus gst_vtenc_session_configure_property_int (GstVTEnc * self,
173 VTCompressionSessionRef session, CFStringRef name, gint value);
174 static OSStatus gst_vtenc_session_configure_property_double (GstVTEnc * self,
175 VTCompressionSessionRef session, CFStringRef name, gdouble value);
176 static void gst_vtenc_session_configure_allow_frame_reordering (GstVTEnc * self,
177 VTCompressionSessionRef session, gboolean allow_frame_reordering);
178 static void gst_vtenc_session_configure_realtime (GstVTEnc * self,
179 VTCompressionSessionRef session, gboolean realtime);
180
181 static GstFlowReturn gst_vtenc_encode_frame (GstVTEnc * self,
182 GstVideoCodecFrame * frame);
183 static void gst_vtenc_enqueue_buffer (void *outputCallbackRefCon,
184 void *sourceFrameRefCon, OSStatus status, VTEncodeInfoFlags infoFlags,
185 CMSampleBufferRef sampleBuffer);
186 static gboolean gst_vtenc_buffer_is_keyframe (GstVTEnc * self,
187 CMSampleBufferRef sbuf);
188
189
190 #ifndef HAVE_IOS
191 static GstVTEncFrame *gst_vtenc_frame_new (GstBuffer * buf,
192 GstVideoInfo * videoinfo);
193 static void gst_vtenc_frame_free (GstVTEncFrame * frame);
194
195 static void gst_pixel_buffer_release_cb (void *releaseRefCon,
196 const void *dataPtr, size_t dataSize, size_t numberOfPlanes,
197 const void *planeAddresses[]);
198 #endif
199
200 #ifdef HAVE_IOS
201 static GstStaticCaps sink_caps =
202 GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE ("{ NV12, I420 }"));
203 #else
204 static GstStaticCaps sink_caps =
205 GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE
206 ("{ AYUV64, UYVY, NV12, I420, ARGB64_BE }"));
207 #endif
208
209
210 static void
gst_vtenc_base_init(GstVTEncClass * klass)211 gst_vtenc_base_init (GstVTEncClass * klass)
212 {
213 const GstVTEncoderDetails *codec_details =
214 GST_VTENC_CLASS_GET_CODEC_DETAILS (klass);
215 GstElementClass *element_class = GST_ELEMENT_CLASS (klass);
216 const int min_width = 1, max_width = G_MAXINT;
217 const int min_height = 1, max_height = G_MAXINT;
218 const int min_fps_n = 0, max_fps_n = G_MAXINT;
219 const int min_fps_d = 1, max_fps_d = 1;
220 GstCaps *src_caps;
221 gchar *longname, *description;
222
223 longname = g_strdup_printf ("%s encoder", codec_details->name);
224 description = g_strdup_printf ("%s encoder", codec_details->name);
225
226 gst_element_class_set_metadata (element_class, longname,
227 "Codec/Encoder/Video/Hardware", description,
228 "Ole André Vadla Ravnås <oleavr@soundrop.com>, Dominik Röttsches <dominik.rottsches@intel.com>");
229
230 g_free (longname);
231 g_free (description);
232
233 {
234 GstCaps *caps = gst_static_caps_get (&sink_caps);
235 /* RGBA64_LE is kCVPixelFormatType_64RGBALE, only available on macOS 11.3+ */
236 if (GST_VTUTIL_HAVE_64ARGBALE)
237 caps = gst_vtutil_caps_append_video_format (caps, "RGBA64_LE");
238 gst_element_class_add_pad_template (element_class,
239 gst_pad_template_new ("sink", GST_PAD_SINK, GST_PAD_ALWAYS, caps));
240 }
241
242
243 src_caps = gst_caps_new_simple (codec_details->mimetype,
244 "width", GST_TYPE_INT_RANGE, min_width, max_width,
245 "height", GST_TYPE_INT_RANGE, min_height, max_height,
246 "framerate", GST_TYPE_FRACTION_RANGE,
247 min_fps_n, min_fps_d, max_fps_n, max_fps_d, NULL);
248
249 /* Signal our limited interlace support */
250 {
251 G_GNUC_BEGIN_IGNORE_DEPRECATIONS;
252 GValueArray *arr = g_value_array_new (2);
253 GValue val = G_VALUE_INIT;
254
255 g_value_init (&val, G_TYPE_STRING);
256 g_value_set_string (&val, "progressive");
257 arr = g_value_array_append (arr, &val);
258 g_value_set_string (&val, "interleaved");
259 arr = g_value_array_append (arr, &val);
260 G_GNUC_END_IGNORE_DEPRECATIONS;
261 gst_structure_set_list (gst_caps_get_structure (src_caps, 0),
262 "interlace-mode", arr);
263 }
264
265 switch (codec_details->format_id) {
266 case kCMVideoCodecType_H264:
267 gst_structure_set (gst_caps_get_structure (src_caps, 0),
268 "stream-format", G_TYPE_STRING, "avc",
269 "alignment", G_TYPE_STRING, "au", NULL);
270 break;
271 case GST_kCMVideoCodecType_Some_AppleProRes:
272 if (g_strcmp0 (codec_details->mimetype, "video/x-prores") == 0) {
273 G_GNUC_BEGIN_IGNORE_DEPRECATIONS;
274 GValueArray *arr = g_value_array_new (6);
275 GValue val = G_VALUE_INIT;
276
277 g_value_init (&val, G_TYPE_STRING);
278 g_value_set_string (&val, "standard");
279 arr = g_value_array_append (arr, &val);
280 g_value_set_string (&val, "4444xq");
281 arr = g_value_array_append (arr, &val);
282 g_value_set_string (&val, "4444");
283 arr = g_value_array_append (arr, &val);
284 g_value_set_string (&val, "hq");
285 arr = g_value_array_append (arr, &val);
286 g_value_set_string (&val, "lt");
287 arr = g_value_array_append (arr, &val);
288 g_value_set_string (&val, "proxy");
289 arr = g_value_array_append (arr, &val);
290 gst_structure_set_list (gst_caps_get_structure (src_caps, 0),
291 "variant", arr);
292 g_value_array_free (arr);
293 g_value_unset (&val);
294 G_GNUC_END_IGNORE_DEPRECATIONS;
295 break;
296 }
297 /* fall through */
298 default:
299 g_assert_not_reached ();
300 }
301
302 gst_element_class_add_pad_template (element_class,
303 gst_pad_template_new ("src", GST_PAD_SRC, GST_PAD_ALWAYS, src_caps));
304 gst_caps_unref (src_caps);
305 }
306
307 static void
gst_vtenc_class_init(GstVTEncClass * klass)308 gst_vtenc_class_init (GstVTEncClass * klass)
309 {
310 GObjectClass *gobject_class;
311 GstVideoEncoderClass *gstvideoencoder_class;
312
313 gobject_class = (GObjectClass *) klass;
314 gstvideoencoder_class = (GstVideoEncoderClass *) klass;
315
316 parent_class = g_type_class_peek_parent (klass);
317
318 gobject_class->get_property = gst_vtenc_get_property;
319 gobject_class->set_property = gst_vtenc_set_property;
320 gobject_class->finalize = gst_vtenc_finalize;
321
322 gstvideoencoder_class->start = gst_vtenc_start;
323 gstvideoencoder_class->stop = gst_vtenc_stop;
324 gstvideoencoder_class->set_format = gst_vtenc_set_format;
325 gstvideoencoder_class->handle_frame = gst_vtenc_handle_frame;
326 gstvideoencoder_class->finish = gst_vtenc_finish;
327 gstvideoencoder_class->flush = gst_vtenc_flush;
328
329 g_object_class_install_property (gobject_class, PROP_BITRATE,
330 g_param_spec_uint ("bitrate", "Bitrate",
331 "Target video bitrate in kbps (0 = auto)",
332 0, G_MAXUINT, VTENC_DEFAULT_BITRATE,
333 G_PARAM_READWRITE | G_PARAM_CONSTRUCT | G_PARAM_STATIC_STRINGS));
334
335 g_object_class_install_property (gobject_class, PROP_ALLOW_FRAME_REORDERING,
336 g_param_spec_boolean ("allow-frame-reordering", "Allow frame reordering",
337 "Whether to allow frame reordering or not",
338 VTENC_DEFAULT_FRAME_REORDERING,
339 G_PARAM_READWRITE | G_PARAM_CONSTRUCT | G_PARAM_STATIC_STRINGS));
340
341 g_object_class_install_property (gobject_class, PROP_REALTIME,
342 g_param_spec_boolean ("realtime", "Realtime",
343 "Configure the encoder for realtime output",
344 VTENC_DEFAULT_REALTIME,
345 G_PARAM_READWRITE | G_PARAM_CONSTRUCT | G_PARAM_STATIC_STRINGS));
346
347 g_object_class_install_property (gobject_class, PROP_QUALITY,
348 g_param_spec_double ("quality", "Quality",
349 "The desired compression quality",
350 0.0, 1.0, VTENC_DEFAULT_QUALITY,
351 G_PARAM_READWRITE | G_PARAM_CONSTRUCT | G_PARAM_STATIC_STRINGS));
352
353 g_object_class_install_property (gobject_class, PROP_MAX_KEYFRAME_INTERVAL,
354 g_param_spec_int ("max-keyframe-interval", "Max Keyframe Interval",
355 "Maximum number of frames between keyframes (0 = auto)",
356 0, G_MAXINT, VTENC_DEFAULT_MAX_KEYFRAME_INTERVAL,
357 G_PARAM_READWRITE | G_PARAM_CONSTRUCT | G_PARAM_STATIC_STRINGS));
358
359 g_object_class_install_property (gobject_class,
360 PROP_MAX_KEYFRAME_INTERVAL_DURATION,
361 g_param_spec_uint64 ("max-keyframe-interval-duration",
362 "Max Keyframe Interval Duration",
363 "Maximum number of nanoseconds between keyframes (0 = no limit)", 0,
364 G_MAXUINT64, VTENC_DEFAULT_MAX_KEYFRAME_INTERVAL_DURATION,
365 G_PARAM_READWRITE | G_PARAM_CONSTRUCT | G_PARAM_STATIC_STRINGS));
366
367 /*
368 * H264 doesn't support alpha components, so only add the property for prores
369 */
370 if (g_strcmp0 (G_OBJECT_CLASS_NAME (klass), "vtenc_prores") == 0) {
371 /**
372 * vtenc_prores:preserve-alpha
373 *
374 * Preserve non-opaque video alpha values from the input video when
375 * compressing, else treat all alpha component as opaque.
376 *
377 * Since: 1.20
378 */
379 g_object_class_install_property (gobject_class, PROP_PRESERVE_ALPHA,
380 g_param_spec_boolean ("preserve-alpha", "Preserve Video Alpha Values",
381 "Video alpha values (non opaque) need to be preserved",
382 VTENC_DEFAULT_PRESERVE_ALPHA,
383 G_PARAM_READWRITE | G_PARAM_CONSTRUCT | G_PARAM_STATIC_STRINGS));
384 }
385 }
386
387 static void
gst_vtenc_init(GstVTEnc * self)388 gst_vtenc_init (GstVTEnc * self)
389 {
390 GstVTEncClass *klass = (GstVTEncClass *) G_OBJECT_GET_CLASS (self);
391 CFStringRef keyframe_props_keys[] = { kVTEncodeFrameOptionKey_ForceKeyFrame };
392 CFBooleanRef keyframe_props_values[] = { kCFBooleanTrue };
393
394 self->details = GST_VTENC_CLASS_GET_CODEC_DETAILS (klass);
395
396 /* These could be controlled by properties later */
397 self->dump_properties = FALSE;
398 self->dump_attributes = FALSE;
399 self->latency_frames = -1;
400 self->session = NULL;
401 self->profile_level = NULL;
402 self->have_field_order = TRUE;
403
404 self->keyframe_props =
405 CFDictionaryCreate (NULL, (const void **) keyframe_props_keys,
406 (const void **) keyframe_props_values, G_N_ELEMENTS (keyframe_props_keys),
407 &kCFTypeDictionaryKeyCallBacks, &kCFTypeDictionaryValueCallBacks);
408 }
409
410 static void
gst_vtenc_finalize(GObject * obj)411 gst_vtenc_finalize (GObject * obj)
412 {
413 GstVTEnc *self = GST_VTENC_CAST (obj);
414
415 CFRelease (self->keyframe_props);
416
417 G_OBJECT_CLASS (parent_class)->finalize (obj);
418 }
419
420 static guint
gst_vtenc_get_bitrate(GstVTEnc * self)421 gst_vtenc_get_bitrate (GstVTEnc * self)
422 {
423 guint result;
424
425 GST_OBJECT_LOCK (self);
426 result = self->bitrate;
427 GST_OBJECT_UNLOCK (self);
428
429 return result;
430 }
431
432 static void
gst_vtenc_set_bitrate(GstVTEnc * self,guint bitrate)433 gst_vtenc_set_bitrate (GstVTEnc * self, guint bitrate)
434 {
435 GST_OBJECT_LOCK (self);
436
437 self->bitrate = bitrate;
438
439 if (self->session != NULL)
440 gst_vtenc_session_configure_bitrate (self, self->session, bitrate);
441
442 GST_OBJECT_UNLOCK (self);
443 }
444
445 static gboolean
gst_vtenc_get_allow_frame_reordering(GstVTEnc * self)446 gst_vtenc_get_allow_frame_reordering (GstVTEnc * self)
447 {
448 gboolean result;
449
450 GST_OBJECT_LOCK (self);
451 result = self->allow_frame_reordering;
452 GST_OBJECT_UNLOCK (self);
453
454 return result;
455 }
456
457 static void
gst_vtenc_set_allow_frame_reordering(GstVTEnc * self,gboolean allow_frame_reordering)458 gst_vtenc_set_allow_frame_reordering (GstVTEnc * self,
459 gboolean allow_frame_reordering)
460 {
461 GST_OBJECT_LOCK (self);
462 self->allow_frame_reordering = allow_frame_reordering;
463 if (self->session != NULL) {
464 gst_vtenc_session_configure_allow_frame_reordering (self,
465 self->session, allow_frame_reordering);
466 }
467 GST_OBJECT_UNLOCK (self);
468 }
469
470 static gboolean
gst_vtenc_get_realtime(GstVTEnc * self)471 gst_vtenc_get_realtime (GstVTEnc * self)
472 {
473 gboolean result;
474
475 GST_OBJECT_LOCK (self);
476 result = self->realtime;
477 GST_OBJECT_UNLOCK (self);
478
479 return result;
480 }
481
482 static void
gst_vtenc_set_realtime(GstVTEnc * self,gboolean realtime)483 gst_vtenc_set_realtime (GstVTEnc * self, gboolean realtime)
484 {
485 GST_OBJECT_LOCK (self);
486 self->realtime = realtime;
487 if (self->session != NULL)
488 gst_vtenc_session_configure_realtime (self, self->session, realtime);
489 GST_OBJECT_UNLOCK (self);
490 }
491
492 static gdouble
gst_vtenc_get_quality(GstVTEnc * self)493 gst_vtenc_get_quality (GstVTEnc * self)
494 {
495 gdouble result;
496
497 GST_OBJECT_LOCK (self);
498 result = self->quality;
499 GST_OBJECT_UNLOCK (self);
500
501 return result;
502 }
503
504 static void
gst_vtenc_set_quality(GstVTEnc * self,gdouble quality)505 gst_vtenc_set_quality (GstVTEnc * self, gdouble quality)
506 {
507 GST_OBJECT_LOCK (self);
508 self->quality = quality;
509 GST_INFO_OBJECT (self, "setting quality %f", quality);
510 if (self->session != NULL) {
511 gst_vtenc_session_configure_property_double (self, self->session,
512 kVTCompressionPropertyKey_Quality, quality);
513 }
514 GST_OBJECT_UNLOCK (self);
515 }
516
517 static gint
gst_vtenc_get_max_keyframe_interval(GstVTEnc * self)518 gst_vtenc_get_max_keyframe_interval (GstVTEnc * self)
519 {
520 gint result;
521
522 GST_OBJECT_LOCK (self);
523 result = self->max_keyframe_interval;
524 GST_OBJECT_UNLOCK (self);
525
526 return result;
527 }
528
529 static void
gst_vtenc_set_max_keyframe_interval(GstVTEnc * self,gint interval)530 gst_vtenc_set_max_keyframe_interval (GstVTEnc * self, gint interval)
531 {
532 GST_OBJECT_LOCK (self);
533 self->max_keyframe_interval = interval;
534 if (self->session != NULL) {
535 gst_vtenc_session_configure_max_keyframe_interval (self, self->session,
536 interval);
537 }
538 GST_OBJECT_UNLOCK (self);
539 }
540
541 static GstClockTime
gst_vtenc_get_max_keyframe_interval_duration(GstVTEnc * self)542 gst_vtenc_get_max_keyframe_interval_duration (GstVTEnc * self)
543 {
544 GstClockTime result;
545
546 GST_OBJECT_LOCK (self);
547 result = self->max_keyframe_interval_duration;
548 GST_OBJECT_UNLOCK (self);
549
550 return result;
551 }
552
553 static void
gst_vtenc_set_max_keyframe_interval_duration(GstVTEnc * self,GstClockTime interval)554 gst_vtenc_set_max_keyframe_interval_duration (GstVTEnc * self,
555 GstClockTime interval)
556 {
557 GST_OBJECT_LOCK (self);
558 self->max_keyframe_interval_duration = interval;
559 if (self->session != NULL) {
560 gst_vtenc_session_configure_max_keyframe_interval_duration (self,
561 self->session, interval / ((gdouble) GST_SECOND));
562 }
563 GST_OBJECT_UNLOCK (self);
564 }
565
566 static void
gst_vtenc_get_property(GObject * obj,guint prop_id,GValue * value,GParamSpec * pspec)567 gst_vtenc_get_property (GObject * obj, guint prop_id, GValue * value,
568 GParamSpec * pspec)
569 {
570 GstVTEnc *self = GST_VTENC_CAST (obj);
571
572 switch (prop_id) {
573 case PROP_BITRATE:
574 g_value_set_uint (value, gst_vtenc_get_bitrate (self) / 1000);
575 break;
576 case PROP_ALLOW_FRAME_REORDERING:
577 g_value_set_boolean (value, gst_vtenc_get_allow_frame_reordering (self));
578 break;
579 case PROP_REALTIME:
580 g_value_set_boolean (value, gst_vtenc_get_realtime (self));
581 break;
582 case PROP_QUALITY:
583 g_value_set_double (value, gst_vtenc_get_quality (self));
584 break;
585 case PROP_MAX_KEYFRAME_INTERVAL:
586 g_value_set_int (value, gst_vtenc_get_max_keyframe_interval (self));
587 break;
588 case PROP_MAX_KEYFRAME_INTERVAL_DURATION:
589 g_value_set_uint64 (value,
590 gst_vtenc_get_max_keyframe_interval_duration (self));
591 break;
592 case PROP_PRESERVE_ALPHA:
593 g_value_set_boolean (value, self->preserve_alpha);
594 break;
595 default:
596 G_OBJECT_WARN_INVALID_PROPERTY_ID (obj, prop_id, pspec);
597 break;
598 }
599 }
600
601 static void
gst_vtenc_set_property(GObject * obj,guint prop_id,const GValue * value,GParamSpec * pspec)602 gst_vtenc_set_property (GObject * obj, guint prop_id, const GValue * value,
603 GParamSpec * pspec)
604 {
605 GstVTEnc *self = GST_VTENC_CAST (obj);
606
607 switch (prop_id) {
608 case PROP_BITRATE:
609 gst_vtenc_set_bitrate (self, g_value_get_uint (value) * 1000);
610 break;
611 case PROP_ALLOW_FRAME_REORDERING:
612 gst_vtenc_set_allow_frame_reordering (self, g_value_get_boolean (value));
613 break;
614 case PROP_REALTIME:
615 gst_vtenc_set_realtime (self, g_value_get_boolean (value));
616 break;
617 case PROP_QUALITY:
618 gst_vtenc_set_quality (self, g_value_get_double (value));
619 break;
620 case PROP_MAX_KEYFRAME_INTERVAL:
621 gst_vtenc_set_max_keyframe_interval (self, g_value_get_int (value));
622 break;
623 case PROP_MAX_KEYFRAME_INTERVAL_DURATION:
624 gst_vtenc_set_max_keyframe_interval_duration (self,
625 g_value_get_uint64 (value));
626 break;
627 case PROP_PRESERVE_ALPHA:
628 self->preserve_alpha = g_value_get_boolean (value);
629 break;
630 default:
631 G_OBJECT_WARN_INVALID_PROPERTY_ID (obj, prop_id, pspec);
632 break;
633 }
634 }
635
636 static GstFlowReturn
gst_vtenc_finish_encoding(GstVTEnc * self,gboolean is_flushing)637 gst_vtenc_finish_encoding (GstVTEnc * self, gboolean is_flushing)
638 {
639 GST_DEBUG_OBJECT (self,
640 "complete encoding and clean buffer queue, is flushing %d", is_flushing);
641 GstVideoCodecFrame *outframe;
642 GstFlowReturn ret = GST_FLOW_OK;
643 OSStatus vt_status;
644
645 /* We need to unlock the stream lock here because
646 * it can wait for gst_vtenc_enqueue_buffer() to
647 * handle a buffer... which will take the stream
648 * lock from another thread and then deadlock */
649 GST_VIDEO_ENCODER_STREAM_UNLOCK (self);
650 GST_DEBUG_OBJECT (self, "starting VTCompressionSessionCompleteFrames");
651 vt_status =
652 VTCompressionSessionCompleteFrames (self->session,
653 kCMTimePositiveInfinity);
654 GST_DEBUG_OBJECT (self, "VTCompressionSessionCompleteFrames ended");
655 GST_VIDEO_ENCODER_STREAM_LOCK (self);
656 if (vt_status != noErr) {
657 GST_WARNING_OBJECT (self, "VTCompressionSessionCompleteFrames returned %d",
658 (int) vt_status);
659 }
660
661 while ((outframe = g_async_queue_try_pop (self->cur_outframes))) {
662 if (is_flushing) {
663 GST_DEBUG_OBJECT (self, "flushing frame number %d",
664 outframe->system_frame_number);
665 gst_video_codec_frame_unref (outframe);
666 } else {
667 GST_DEBUG_OBJECT (self, "finish frame number %d",
668 outframe->system_frame_number);
669 ret =
670 gst_video_encoder_finish_frame (GST_VIDEO_ENCODER_CAST (self),
671 outframe);
672 }
673 }
674
675 GST_DEBUG_OBJECT (self, "buffer queue cleaned");
676
677 return ret;
678 }
679
680 static gboolean
gst_vtenc_start(GstVideoEncoder * enc)681 gst_vtenc_start (GstVideoEncoder * enc)
682 {
683 GstVTEnc *self = GST_VTENC_CAST (enc);
684
685 self->cur_outframes = g_async_queue_new ();
686
687 return TRUE;
688 }
689
690 static gboolean
gst_vtenc_stop(GstVideoEncoder * enc)691 gst_vtenc_stop (GstVideoEncoder * enc)
692 {
693 GstVTEnc *self = GST_VTENC_CAST (enc);
694
695 GST_VIDEO_ENCODER_STREAM_LOCK (self);
696 gst_vtenc_flush (enc);
697 GST_VIDEO_ENCODER_STREAM_UNLOCK (self);
698
699 GST_OBJECT_LOCK (self);
700 gst_vtenc_destroy_session (self, &self->session);
701 GST_OBJECT_UNLOCK (self);
702
703 if (self->profile_level)
704 CFRelease (self->profile_level);
705 self->profile_level = NULL;
706
707 if (self->input_state)
708 gst_video_codec_state_unref (self->input_state);
709 self->input_state = NULL;
710
711 self->negotiated_width = self->negotiated_height = 0;
712 self->negotiated_fps_n = self->negotiated_fps_d = 0;
713
714 gst_vtenc_clear_cached_caps_downstream (self);
715
716 g_async_queue_unref (self->cur_outframes);
717 self->cur_outframes = NULL;
718
719 return TRUE;
720 }
721
722 static CFStringRef
gst_vtenc_profile_level_key(GstVTEnc * self,const gchar * profile,const gchar * level_arg)723 gst_vtenc_profile_level_key (GstVTEnc * self, const gchar * profile,
724 const gchar * level_arg)
725 {
726 char level[64];
727 gchar *key = NULL;
728 CFStringRef ret = NULL;
729
730 if (profile == NULL)
731 profile = "main";
732 if (level_arg == NULL)
733 level_arg = "AutoLevel";
734 strncpy (level, level_arg, sizeof (level));
735
736 if (!strcmp (profile, "constrained-baseline") ||
737 !strcmp (profile, "baseline")) {
738 profile = "Baseline";
739 } else if (g_str_has_prefix (profile, "high")) {
740 profile = "High";
741 } else if (!strcmp (profile, "main")) {
742 profile = "Main";
743 } else {
744 GST_ERROR_OBJECT (self, "invalid profile: %s", profile);
745 return ret;
746 }
747
748 if (strlen (level) == 1) {
749 level[1] = '_';
750 level[2] = '0';
751 } else if (strlen (level) == 3) {
752 level[1] = '_';
753 }
754
755 key = g_strdup_printf ("H264_%s_%s", profile, level);
756 ret = CFStringCreateWithBytes (NULL, (const guint8 *) key, strlen (key),
757 kCFStringEncodingASCII, 0);
758
759 GST_INFO_OBJECT (self, "negotiated profile and level %s", key);
760
761 g_free (key);
762
763 return ret;
764 }
765
766 static gboolean
gst_vtenc_negotiate_profile_and_level(GstVTEnc * self,GstStructure * s)767 gst_vtenc_negotiate_profile_and_level (GstVTEnc * self, GstStructure * s)
768 {
769 const gchar *profile = gst_structure_get_string (s, "profile");
770 const gchar *level = gst_structure_get_string (s, "level");
771
772 if (self->profile_level)
773 CFRelease (self->profile_level);
774 self->profile_level = gst_vtenc_profile_level_key (self, profile, level);
775 if (self->profile_level == NULL) {
776 GST_ERROR_OBJECT (self, "unsupported h264 profile '%s' or level '%s'",
777 profile, level);
778 return FALSE;
779 }
780
781 return TRUE;
782 }
783
784 static gboolean
gst_vtenc_negotiate_prores_variant(GstVTEnc * self,GstStructure * s)785 gst_vtenc_negotiate_prores_variant (GstVTEnc * self, GstStructure * s)
786 {
787 const char *variant = gst_structure_get_string (s, "variant");
788 CMVideoCodecType codec_type =
789 gst_vtutil_codec_type_from_prores_variant (variant);
790
791 if (codec_type == GST_kCMVideoCodecType_Some_AppleProRes) {
792 GST_ERROR_OBJECT (self, "unsupported prores variant: %s", variant);
793 return FALSE;
794 }
795
796 self->specific_format_id = codec_type;
797 return TRUE;
798 }
799
800 static gboolean
gst_vtenc_negotiate_specific_format_details(GstVideoEncoder * enc)801 gst_vtenc_negotiate_specific_format_details (GstVideoEncoder * enc)
802 {
803 GstVTEnc *self = GST_VTENC_CAST (enc);
804 GstCaps *allowed_caps = NULL;
805 gboolean ret = TRUE;
806
807 allowed_caps = gst_pad_get_allowed_caps (GST_VIDEO_ENCODER_SRC_PAD (enc));
808 if (allowed_caps) {
809 GstStructure *s;
810
811 if (gst_caps_is_empty (allowed_caps)) {
812 GST_ERROR_OBJECT (self, "no allowed downstream caps");
813 goto fail;
814 }
815
816 allowed_caps = gst_caps_make_writable (allowed_caps);
817 allowed_caps = gst_caps_fixate (allowed_caps);
818 s = gst_caps_get_structure (allowed_caps, 0);
819 switch (self->details->format_id) {
820 case kCMVideoCodecType_H264:
821 self->specific_format_id = kCMVideoCodecType_H264;
822 if (!gst_vtenc_negotiate_profile_and_level (self, s))
823 goto fail;
824 break;
825 case GST_kCMVideoCodecType_Some_AppleProRes:
826 if (g_strcmp0 (self->details->mimetype, "video/x-prores") != 0) {
827 GST_ERROR_OBJECT (self, "format_id == %i mimetype must be Apple "
828 "ProRes", GST_kCMVideoCodecType_Some_AppleProRes);
829 goto fail;
830 }
831 if (!gst_vtenc_negotiate_prores_variant (self, s))
832 goto fail;
833 break;
834 default:
835 g_assert_not_reached ();
836 }
837 }
838
839 out:
840 if (allowed_caps)
841 gst_caps_unref (allowed_caps);
842
843 return ret;
844
845 fail:
846 ret = FALSE;
847 goto out;
848 }
849
850 static gboolean
gst_vtenc_set_format(GstVideoEncoder * enc,GstVideoCodecState * state)851 gst_vtenc_set_format (GstVideoEncoder * enc, GstVideoCodecState * state)
852 {
853 GstVTEnc *self = GST_VTENC_CAST (enc);
854 VTCompressionSessionRef session;
855
856 if (self->input_state)
857 gst_video_codec_state_unref (self->input_state);
858 self->input_state = gst_video_codec_state_ref (state);
859
860 self->negotiated_width = state->info.width;
861 self->negotiated_height = state->info.height;
862 self->negotiated_fps_n = state->info.fps_n;
863 self->negotiated_fps_d = state->info.fps_d;
864 self->video_info = state->info;
865
866 GST_OBJECT_LOCK (self);
867 gst_vtenc_destroy_session (self, &self->session);
868 GST_OBJECT_UNLOCK (self);
869
870 gst_vtenc_negotiate_specific_format_details (enc);
871
872 session = gst_vtenc_create_session (self);
873 GST_OBJECT_LOCK (self);
874 self->session = session;
875 GST_OBJECT_UNLOCK (self);
876
877 return session != NULL;
878 }
879
880 static gboolean
gst_vtenc_is_negotiated(GstVTEnc * self)881 gst_vtenc_is_negotiated (GstVTEnc * self)
882 {
883 return self->negotiated_width != 0;
884 }
885
886 /*
887 * When the image is opaque but the output ProRes format has an alpha
888 * component (4 component, 32 bits per pixel), Apple requires that we signal
889 * that it should be ignored by setting the depth to 24 bits per pixel. Not
890 * doing so causes the encoded files to fail validation.
891 *
892 * So we set that in the caps and qtmux sets the depth value in the container,
893 * which will be read by demuxers so that decoders can skip those bytes
894 * entirely. qtdemux does this, but vtdec does not use this information at
895 * present.
896 */
897 static gboolean
gst_vtenc_signal_ignored_alpha_component(GstVTEnc * self)898 gst_vtenc_signal_ignored_alpha_component (GstVTEnc * self)
899 {
900 if (self->preserve_alpha)
901 return FALSE;
902 if (self->specific_format_id == kCMVideoCodecType_AppleProRes4444XQ ||
903 self->specific_format_id == kCMVideoCodecType_AppleProRes4444)
904 return TRUE;
905 return FALSE;
906 }
907
908 static gboolean
gst_vtenc_negotiate_downstream(GstVTEnc * self,CMSampleBufferRef sbuf)909 gst_vtenc_negotiate_downstream (GstVTEnc * self, CMSampleBufferRef sbuf)
910 {
911 gboolean result;
912 GstCaps *caps;
913 GstStructure *s;
914 GstVideoCodecState *state;
915
916 if (self->caps_width == self->negotiated_width &&
917 self->caps_height == self->negotiated_height &&
918 self->caps_fps_n == self->negotiated_fps_n &&
919 self->caps_fps_d == self->negotiated_fps_d) {
920 return TRUE;
921 }
922
923 caps = gst_pad_get_pad_template_caps (GST_VIDEO_ENCODER_SRC_PAD (self));
924 caps = gst_caps_make_writable (caps);
925 s = gst_caps_get_structure (caps, 0);
926 gst_structure_set (s,
927 "width", G_TYPE_INT, self->negotiated_width,
928 "height", G_TYPE_INT, self->negotiated_height,
929 "framerate", GST_TYPE_FRACTION,
930 self->negotiated_fps_n, self->negotiated_fps_d, NULL);
931
932 switch (self->details->format_id) {
933 case kCMVideoCodecType_H264:
934 {
935 CMFormatDescriptionRef fmt;
936 CFDictionaryRef atoms;
937 CFStringRef avccKey;
938 CFDataRef avcc;
939 guint8 *codec_data;
940 gsize codec_data_size;
941 GstBuffer *codec_data_buf;
942 guint8 sps[3];
943
944 fmt = CMSampleBufferGetFormatDescription (sbuf);
945 atoms = CMFormatDescriptionGetExtension (fmt,
946 kCMFormatDescriptionExtension_SampleDescriptionExtensionAtoms);
947 avccKey = CFStringCreateWithCString (NULL, "avcC", kCFStringEncodingUTF8);
948 avcc = CFDictionaryGetValue (atoms, avccKey);
949 CFRelease (avccKey);
950 codec_data_size = CFDataGetLength (avcc);
951 codec_data = g_malloc (codec_data_size);
952 CFDataGetBytes (avcc, CFRangeMake (0, codec_data_size), codec_data);
953 codec_data_buf = gst_buffer_new_wrapped (codec_data, codec_data_size);
954
955 gst_structure_set (s, "codec_data", GST_TYPE_BUFFER, codec_data_buf,
956 NULL);
957
958 sps[0] = codec_data[1];
959 sps[1] = codec_data[2] & ~0xDF;
960 sps[2] = codec_data[3];
961
962 gst_codec_utils_h264_caps_set_level_and_profile (caps, sps, 3);
963
964 gst_buffer_unref (codec_data_buf);
965 }
966 break;
967 case GST_kCMVideoCodecType_Some_AppleProRes:
968 gst_structure_set (s, "variant", G_TYPE_STRING,
969 gst_vtutil_codec_type_to_prores_variant (self->specific_format_id),
970 NULL);
971 if (gst_vtenc_signal_ignored_alpha_component (self))
972 gst_structure_set (s, "depth", G_TYPE_INT, 24, NULL);
973 break;
974 default:
975 g_assert_not_reached ();
976 }
977
978 state =
979 gst_video_encoder_set_output_state (GST_VIDEO_ENCODER_CAST (self), caps,
980 self->input_state);
981 gst_video_codec_state_unref (state);
982 result = gst_video_encoder_negotiate (GST_VIDEO_ENCODER_CAST (self));
983
984 self->caps_width = self->negotiated_width;
985 self->caps_height = self->negotiated_height;
986 self->caps_fps_n = self->negotiated_fps_n;
987 self->caps_fps_d = self->negotiated_fps_d;
988
989 return result;
990 }
991
992 static void
gst_vtenc_clear_cached_caps_downstream(GstVTEnc * self)993 gst_vtenc_clear_cached_caps_downstream (GstVTEnc * self)
994 {
995 self->caps_width = self->caps_height = 0;
996 self->caps_fps_n = self->caps_fps_d = 0;
997 }
998
999 static GstFlowReturn
gst_vtenc_handle_frame(GstVideoEncoder * enc,GstVideoCodecFrame * frame)1000 gst_vtenc_handle_frame (GstVideoEncoder * enc, GstVideoCodecFrame * frame)
1001 {
1002 GstVTEnc *self = GST_VTENC_CAST (enc);
1003
1004 if (!gst_vtenc_is_negotiated (self))
1005 goto not_negotiated;
1006
1007 return gst_vtenc_encode_frame (self, frame);
1008
1009 not_negotiated:
1010 gst_video_codec_frame_unref (frame);
1011 return GST_FLOW_NOT_NEGOTIATED;
1012 }
1013
1014 static GstFlowReturn
gst_vtenc_finish(GstVideoEncoder * enc)1015 gst_vtenc_finish (GstVideoEncoder * enc)
1016 {
1017 GstVTEnc *self = GST_VTENC_CAST (enc);
1018 return gst_vtenc_finish_encoding (self, FALSE);
1019 }
1020
1021 static gboolean
gst_vtenc_flush(GstVideoEncoder * enc)1022 gst_vtenc_flush (GstVideoEncoder * enc)
1023 {
1024 GstVTEnc *self = GST_VTENC_CAST (enc);
1025 GstFlowReturn ret;
1026
1027 ret = gst_vtenc_finish_encoding (self, TRUE);
1028
1029 return (ret == GST_FLOW_OK);
1030 }
1031
1032 static void
gst_vtenc_set_colorimetry(GstVTEnc * self,VTCompressionSessionRef session)1033 gst_vtenc_set_colorimetry (GstVTEnc * self, VTCompressionSessionRef session)
1034 {
1035 OSStatus status;
1036 CFStringRef primaries = NULL, transfer = NULL, matrix = NULL;
1037 GstVideoColorimetry cm = GST_VIDEO_INFO_COLORIMETRY (&self->video_info);
1038
1039 /*
1040 * https://developer.apple.com/documentation/corevideo/cvimagebuffer/image_buffer_ycbcr_matrix_constants
1041 */
1042 switch (cm.matrix) {
1043 case GST_VIDEO_COLOR_MATRIX_BT709:
1044 matrix = kCVImageBufferYCbCrMatrix_ITU_R_709_2;
1045 break;
1046 case GST_VIDEO_COLOR_MATRIX_BT601:
1047 matrix = kCVImageBufferYCbCrMatrix_ITU_R_601_4;
1048 break;
1049 case GST_VIDEO_COLOR_MATRIX_SMPTE240M:
1050 matrix = kCVImageBufferYCbCrMatrix_SMPTE_240M_1995;
1051 break;
1052 case GST_VIDEO_COLOR_MATRIX_BT2020:
1053 matrix = kCVImageBufferYCbCrMatrix_ITU_R_2020;
1054 break;
1055 default:
1056 GST_WARNING_OBJECT (self, "Unsupported color matrix %u", cm.matrix);
1057 }
1058
1059 /*
1060 * https://developer.apple.com/documentation/corevideo/cvimagebuffer/image_buffer_transfer_function_constants
1061 */
1062 switch (cm.transfer) {
1063 case GST_VIDEO_TRANSFER_BT709:
1064 case GST_VIDEO_TRANSFER_BT601:
1065 case GST_VIDEO_TRANSFER_UNKNOWN:
1066 transfer = kCVImageBufferTransferFunction_ITU_R_709_2;
1067 break;
1068 case GST_VIDEO_TRANSFER_SMPTE240M:
1069 transfer = kCVImageBufferTransferFunction_SMPTE_240M_1995;
1070 break;
1071 case GST_VIDEO_TRANSFER_BT2020_12:
1072 transfer = kCVImageBufferTransferFunction_ITU_R_2020;
1073 break;
1074 case GST_VIDEO_TRANSFER_SRGB:
1075 if (__builtin_available (macOS 10.13, *))
1076 transfer = kCVImageBufferTransferFunction_sRGB;
1077 else
1078 GST_WARNING_OBJECT (self, "macOS version is too old, the sRGB transfer "
1079 "function is not available");
1080 break;
1081 case GST_VIDEO_TRANSFER_SMPTE2084:
1082 if (__builtin_available (macOS 10.13, *))
1083 transfer = kCVImageBufferTransferFunction_SMPTE_ST_2084_PQ;
1084 else
1085 GST_WARNING_OBJECT (self, "macOS version is too old, the SMPTE2084 "
1086 "transfer function is not available");
1087 break;
1088 default:
1089 GST_WARNING_OBJECT (self, "Unsupported color transfer %u", cm.transfer);
1090 }
1091
1092 /*
1093 * https://developer.apple.com/documentation/corevideo/cvimagebuffer/image_buffer_color_primaries_constants
1094 */
1095 switch (cm.primaries) {
1096 case GST_VIDEO_COLOR_PRIMARIES_BT709:
1097 primaries = kCVImageBufferColorPrimaries_ITU_R_709_2;
1098 break;
1099 case GST_VIDEO_COLOR_PRIMARIES_SMPTE170M:
1100 case GST_VIDEO_COLOR_PRIMARIES_SMPTE240M:
1101 primaries = kCVImageBufferColorPrimaries_SMPTE_C;
1102 break;
1103 case GST_VIDEO_COLOR_PRIMARIES_BT2020:
1104 primaries = kCVImageBufferColorPrimaries_ITU_R_2020;
1105 break;
1106 case GST_VIDEO_COLOR_PRIMARIES_SMPTERP431:
1107 primaries = kCVImageBufferColorPrimaries_DCI_P3;
1108 break;
1109 case GST_VIDEO_COLOR_PRIMARIES_SMPTEEG432:
1110 primaries = kCVImageBufferColorPrimaries_P3_D65;
1111 break;
1112 case GST_VIDEO_COLOR_PRIMARIES_EBU3213:
1113 primaries = kCVImageBufferColorPrimaries_EBU_3213;
1114 break;
1115 default:
1116 GST_WARNING_OBJECT (self, "Unsupported color primaries %u", cm.primaries);
1117 }
1118
1119 if (primaries) {
1120 status = VTSessionSetProperty (session,
1121 kVTCompressionPropertyKey_ColorPrimaries, primaries);
1122 GST_DEBUG_OBJECT (self, "kVTCompressionPropertyKey_ColorPrimaries =>"
1123 "%d", status);
1124 }
1125
1126 if (transfer) {
1127 status = VTSessionSetProperty (session,
1128 kVTCompressionPropertyKey_TransferFunction, transfer);
1129 GST_DEBUG_OBJECT (self, "kVTCompressionPropertyKey_TransferFunction =>"
1130 "%d", status);
1131 }
1132
1133 if (matrix) {
1134 status = VTSessionSetProperty (session,
1135 kVTCompressionPropertyKey_YCbCrMatrix, matrix);
1136 GST_DEBUG_OBJECT (self, "kVTCompressionPropertyKey_YCbCrMatrix => %d",
1137 status);
1138 }
1139 }
1140
1141 static VTCompressionSessionRef
gst_vtenc_create_session(GstVTEnc * self)1142 gst_vtenc_create_session (GstVTEnc * self)
1143 {
1144 VTCompressionSessionRef session = NULL;
1145 CFMutableDictionaryRef encoder_spec = NULL, pb_attrs = NULL;
1146 OSStatus status;
1147
1148 #if !HAVE_IOS
1149 const GstVTEncoderDetails *codec_details =
1150 GST_VTENC_CLASS_GET_CODEC_DETAILS (G_OBJECT_GET_CLASS (self));
1151
1152 encoder_spec =
1153 CFDictionaryCreateMutable (NULL, 0, &kCFTypeDictionaryKeyCallBacks,
1154 &kCFTypeDictionaryValueCallBacks);
1155 gst_vtutil_dict_set_boolean (encoder_spec,
1156 kVTVideoEncoderSpecification_EnableHardwareAcceleratedVideoEncoder, true);
1157 if (codec_details->require_hardware)
1158 gst_vtutil_dict_set_boolean (encoder_spec,
1159 kVTVideoEncoderSpecification_RequireHardwareAcceleratedVideoEncoder,
1160 TRUE);
1161 #endif
1162
1163 if (self->profile_level) {
1164 pb_attrs = CFDictionaryCreateMutable (NULL, 0,
1165 &kCFTypeDictionaryKeyCallBacks, &kCFTypeDictionaryValueCallBacks);
1166 gst_vtutil_dict_set_i32 (pb_attrs, kCVPixelBufferWidthKey,
1167 self->negotiated_width);
1168 gst_vtutil_dict_set_i32 (pb_attrs, kCVPixelBufferHeightKey,
1169 self->negotiated_height);
1170 }
1171
1172 /* This was set in gst_vtenc_negotiate_specific_format_details() */
1173 g_assert_cmpint (self->specific_format_id, !=, 0);
1174
1175 status = VTCompressionSessionCreate (NULL,
1176 self->negotiated_width, self->negotiated_height,
1177 self->specific_format_id, encoder_spec, pb_attrs, NULL,
1178 gst_vtenc_enqueue_buffer, self, &session);
1179 GST_INFO_OBJECT (self, "VTCompressionSessionCreate for %d x %d => %d",
1180 self->negotiated_width, self->negotiated_height, (int) status);
1181 if (status != noErr) {
1182 GST_ERROR_OBJECT (self, "VTCompressionSessionCreate() returned: %d",
1183 (int) status);
1184 goto beach;
1185 }
1186
1187 if (self->profile_level) {
1188 gst_vtenc_session_configure_expected_framerate (self, session,
1189 (gdouble) self->negotiated_fps_n / (gdouble) self->negotiated_fps_d);
1190
1191 /*
1192 * https://developer.apple.com/documentation/videotoolbox/vtcompressionsession/compression_properties/profile_and_level_constants
1193 */
1194 status = VTSessionSetProperty (session,
1195 kVTCompressionPropertyKey_ProfileLevel, self->profile_level);
1196 GST_DEBUG_OBJECT (self, "kVTCompressionPropertyKey_ProfileLevel => %d",
1197 (int) status);
1198
1199 status = VTSessionSetProperty (session,
1200 kVTCompressionPropertyKey_AllowTemporalCompression, kCFBooleanTrue);
1201 GST_DEBUG_OBJECT (self,
1202 "kVTCompressionPropertyKey_AllowTemporalCompression => %d",
1203 (int) status);
1204
1205 gst_vtenc_session_configure_max_keyframe_interval (self, session,
1206 self->max_keyframe_interval);
1207 gst_vtenc_session_configure_max_keyframe_interval_duration (self, session,
1208 self->max_keyframe_interval_duration / ((gdouble) GST_SECOND));
1209
1210 gst_vtenc_session_configure_bitrate (self, session,
1211 gst_vtenc_get_bitrate (self));
1212 }
1213
1214 /* Force encoder to not preserve alpha with 4444(XQ) ProRes formats if
1215 * requested */
1216 if (!self->preserve_alpha &&
1217 (self->specific_format_id == kCMVideoCodecType_AppleProRes4444XQ ||
1218 self->specific_format_id == kCMVideoCodecType_AppleProRes4444)) {
1219 status = VTSessionSetProperty (session,
1220 gstVTCodecPropertyKey_PreserveAlphaChannel, CFSTR ("NO"));
1221 GST_DEBUG_OBJECT (self, "kVTCodecPropertyKey_PreserveAlphaChannel => %d",
1222 (int) status);
1223 }
1224
1225 gst_vtenc_set_colorimetry (self, session);
1226
1227 /* Interlacing */
1228 switch (GST_VIDEO_INFO_INTERLACE_MODE (&self->video_info)) {
1229 case GST_VIDEO_INTERLACE_MODE_PROGRESSIVE:
1230 gst_vtenc_session_configure_property_int (self, session,
1231 kVTCompressionPropertyKey_FieldCount, 1);
1232 break;
1233 case GST_VIDEO_INTERLACE_MODE_INTERLEAVED:
1234 gst_vtenc_session_configure_property_int (self, session,
1235 kVTCompressionPropertyKey_FieldCount, 2);
1236 switch (GST_VIDEO_INFO_FIELD_ORDER (&self->video_info)) {
1237 case GST_VIDEO_FIELD_ORDER_TOP_FIELD_FIRST:
1238 status = VTSessionSetProperty (session,
1239 kVTCompressionPropertyKey_FieldDetail,
1240 kCMFormatDescriptionFieldDetail_TemporalTopFirst);
1241 GST_DEBUG_OBJECT (self, "kVTCompressionPropertyKey_FieldDetail "
1242 "TemporalTopFirst => %d", (int) status);
1243 break;
1244 case GST_VIDEO_FIELD_ORDER_BOTTOM_FIELD_FIRST:
1245 status = VTSessionSetProperty (session,
1246 kVTCompressionPropertyKey_FieldDetail,
1247 kCMFormatDescriptionFieldDetail_TemporalBottomFirst);
1248 GST_DEBUG_OBJECT (self, "kVTCompressionPropertyKey_FieldDetail "
1249 "TemporalBottomFirst => %d", (int) status);
1250 break;
1251 case GST_VIDEO_FIELD_ORDER_UNKNOWN:
1252 GST_INFO_OBJECT (self, "Unknown field order for interleaved content, "
1253 "will check first buffer");
1254 self->have_field_order = FALSE;
1255 }
1256 break;
1257 default:
1258 /* Caps negotiation should prevent this */
1259 g_assert_not_reached ();
1260 }
1261
1262 gst_vtenc_session_configure_realtime (self, session,
1263 gst_vtenc_get_realtime (self));
1264 gst_vtenc_session_configure_allow_frame_reordering (self, session,
1265 gst_vtenc_get_allow_frame_reordering (self));
1266 gst_vtenc_set_quality (self, self->quality);
1267
1268 if (self->dump_properties) {
1269 gst_vtenc_session_dump_properties (self, session);
1270 self->dump_properties = FALSE;
1271 }
1272 #ifdef HAVE_VIDEOTOOLBOX_10_9_6
1273 if (VTCompressionSessionPrepareToEncodeFrames) {
1274 status = VTCompressionSessionPrepareToEncodeFrames (session);
1275 if (status != noErr) {
1276 GST_ERROR_OBJECT (self,
1277 "VTCompressionSessionPrepareToEncodeFrames() returned: %d",
1278 (int) status);
1279 }
1280 }
1281 #endif
1282
1283 beach:
1284 if (encoder_spec)
1285 CFRelease (encoder_spec);
1286 if (pb_attrs)
1287 CFRelease (pb_attrs);
1288
1289 return session;
1290 }
1291
1292 static void
gst_vtenc_destroy_session(GstVTEnc * self,VTCompressionSessionRef * session)1293 gst_vtenc_destroy_session (GstVTEnc * self, VTCompressionSessionRef * session)
1294 {
1295 VTCompressionSessionInvalidate (*session);
1296 if (*session != NULL) {
1297 CFRelease (*session);
1298 *session = NULL;
1299 }
1300 }
1301
1302 typedef struct
1303 {
1304 GstVTEnc *self;
1305 VTCompressionSessionRef session;
1306 } GstVTDumpPropCtx;
1307
1308 static void
gst_vtenc_session_dump_property(CFStringRef prop_name,CFDictionaryRef prop_attrs,GstVTDumpPropCtx * dpc)1309 gst_vtenc_session_dump_property (CFStringRef prop_name,
1310 CFDictionaryRef prop_attrs, GstVTDumpPropCtx * dpc)
1311 {
1312 gchar *name_str;
1313 CFTypeRef prop_value;
1314 OSStatus status;
1315
1316 name_str = gst_vtutil_string_to_utf8 (prop_name);
1317 if (dpc->self->dump_attributes) {
1318 gchar *attrs_str;
1319
1320 attrs_str = gst_vtutil_object_to_string (prop_attrs);
1321 GST_DEBUG_OBJECT (dpc->self, "%s = %s", name_str, attrs_str);
1322 g_free (attrs_str);
1323 }
1324
1325 status = VTSessionCopyProperty (dpc->session, prop_name, NULL, &prop_value);
1326 if (status == noErr) {
1327 gchar *value_str;
1328
1329 value_str = gst_vtutil_object_to_string (prop_value);
1330 GST_DEBUG_OBJECT (dpc->self, "%s = %s", name_str, value_str);
1331 g_free (value_str);
1332
1333 if (prop_value != NULL)
1334 CFRelease (prop_value);
1335 } else {
1336 GST_DEBUG_OBJECT (dpc->self, "%s = <failed to query: %d>",
1337 name_str, (int) status);
1338 }
1339
1340 g_free (name_str);
1341 }
1342
1343 static void
gst_vtenc_session_dump_properties(GstVTEnc * self,VTCompressionSessionRef session)1344 gst_vtenc_session_dump_properties (GstVTEnc * self,
1345 VTCompressionSessionRef session)
1346 {
1347 GstVTDumpPropCtx dpc = { self, session };
1348 CFDictionaryRef dict;
1349 OSStatus status;
1350
1351 status = VTSessionCopySupportedPropertyDictionary (session, &dict);
1352 if (status != noErr)
1353 goto error;
1354 CFDictionaryApplyFunction (dict,
1355 (CFDictionaryApplierFunction) gst_vtenc_session_dump_property, &dpc);
1356 CFRelease (dict);
1357
1358 return;
1359
1360 error:
1361 GST_WARNING_OBJECT (self, "failed to dump properties");
1362 }
1363
1364 static void
gst_vtenc_session_configure_expected_framerate(GstVTEnc * self,VTCompressionSessionRef session,gdouble framerate)1365 gst_vtenc_session_configure_expected_framerate (GstVTEnc * self,
1366 VTCompressionSessionRef session, gdouble framerate)
1367 {
1368 gst_vtenc_session_configure_property_double (self, session,
1369 kVTCompressionPropertyKey_ExpectedFrameRate, framerate);
1370 }
1371
1372 static void
gst_vtenc_session_configure_max_keyframe_interval(GstVTEnc * self,VTCompressionSessionRef session,gint interval)1373 gst_vtenc_session_configure_max_keyframe_interval (GstVTEnc * self,
1374 VTCompressionSessionRef session, gint interval)
1375 {
1376 gst_vtenc_session_configure_property_int (self, session,
1377 kVTCompressionPropertyKey_MaxKeyFrameInterval, interval);
1378 }
1379
1380 static void
gst_vtenc_session_configure_max_keyframe_interval_duration(GstVTEnc * self,VTCompressionSessionRef session,gdouble duration)1381 gst_vtenc_session_configure_max_keyframe_interval_duration (GstVTEnc * self,
1382 VTCompressionSessionRef session, gdouble duration)
1383 {
1384 gst_vtenc_session_configure_property_double (self, session,
1385 kVTCompressionPropertyKey_MaxKeyFrameIntervalDuration, duration);
1386 }
1387
1388 static void
gst_vtenc_session_configure_bitrate(GstVTEnc * self,VTCompressionSessionRef session,guint bitrate)1389 gst_vtenc_session_configure_bitrate (GstVTEnc * self,
1390 VTCompressionSessionRef session, guint bitrate)
1391 {
1392 gst_vtenc_session_configure_property_int (self, session,
1393 kVTCompressionPropertyKey_AverageBitRate, bitrate);
1394 }
1395
1396 static void
gst_vtenc_session_configure_allow_frame_reordering(GstVTEnc * self,VTCompressionSessionRef session,gboolean allow_frame_reordering)1397 gst_vtenc_session_configure_allow_frame_reordering (GstVTEnc * self,
1398 VTCompressionSessionRef session, gboolean allow_frame_reordering)
1399 {
1400 VTSessionSetProperty (session, kVTCompressionPropertyKey_AllowFrameReordering,
1401 allow_frame_reordering ? kCFBooleanTrue : kCFBooleanFalse);
1402 }
1403
1404 static void
gst_vtenc_session_configure_realtime(GstVTEnc * self,VTCompressionSessionRef session,gboolean realtime)1405 gst_vtenc_session_configure_realtime (GstVTEnc * self,
1406 VTCompressionSessionRef session, gboolean realtime)
1407 {
1408 VTSessionSetProperty (session, kVTCompressionPropertyKey_RealTime,
1409 realtime ? kCFBooleanTrue : kCFBooleanFalse);
1410 }
1411
1412 static OSStatus
gst_vtenc_session_configure_property_int(GstVTEnc * self,VTCompressionSessionRef session,CFStringRef name,gint value)1413 gst_vtenc_session_configure_property_int (GstVTEnc * self,
1414 VTCompressionSessionRef session, CFStringRef name, gint value)
1415 {
1416 CFNumberRef num;
1417 OSStatus status;
1418 gchar name_str[128];
1419
1420 num = CFNumberCreate (NULL, kCFNumberIntType, &value);
1421 status = VTSessionSetProperty (session, name, num);
1422 CFRelease (num);
1423
1424 CFStringGetCString (name, name_str, sizeof (name_str), kCFStringEncodingUTF8);
1425 GST_DEBUG_OBJECT (self, "%s(%d) => %d", name_str, value, (int) status);
1426
1427 return status;
1428 }
1429
1430 static OSStatus
gst_vtenc_session_configure_property_double(GstVTEnc * self,VTCompressionSessionRef session,CFStringRef name,gdouble value)1431 gst_vtenc_session_configure_property_double (GstVTEnc * self,
1432 VTCompressionSessionRef session, CFStringRef name, gdouble value)
1433 {
1434 CFNumberRef num;
1435 OSStatus status;
1436 gchar name_str[128];
1437
1438 num = CFNumberCreate (NULL, kCFNumberDoubleType, &value);
1439 status = VTSessionSetProperty (session, name, num);
1440 CFRelease (num);
1441
1442 CFStringGetCString (name, name_str, sizeof (name_str), kCFStringEncodingUTF8);
1443 GST_DEBUG_OBJECT (self, "%s(%f) => %d", name_str, value, (int) status);
1444
1445 return status;
1446 }
1447
1448 static void
gst_vtenc_update_latency(GstVTEnc * self)1449 gst_vtenc_update_latency (GstVTEnc * self)
1450 {
1451 OSStatus status;
1452 CFNumberRef value;
1453 int frames = 0;
1454 GstClockTime frame_duration;
1455 GstClockTime latency;
1456
1457 if (self->video_info.fps_d == 0) {
1458 GST_INFO_OBJECT (self, "framerate not known, can't set latency");
1459 return;
1460 }
1461
1462 status = VTSessionCopyProperty (self->session,
1463 kVTCompressionPropertyKey_NumberOfPendingFrames, NULL, &value);
1464 if (status != noErr || !value) {
1465 GST_INFO_OBJECT (self, "failed to get NumberOfPendingFrames: %d", status);
1466 return;
1467 }
1468
1469 CFNumberGetValue (value, kCFNumberSInt32Type, &frames);
1470 if (self->latency_frames == -1 || self->latency_frames != frames) {
1471 self->latency_frames = frames;
1472 if (self->video_info.fps_d == 0 || self->video_info.fps_n == 0) {
1473 /* FIXME: Assume 25fps. This is better than reporting no latency at
1474 * all and then later failing in live pipelines
1475 */
1476 frame_duration = gst_util_uint64_scale (GST_SECOND, 1, 25);
1477 } else {
1478 frame_duration = gst_util_uint64_scale (GST_SECOND,
1479 self->video_info.fps_d, self->video_info.fps_n);
1480 }
1481 latency = frame_duration * frames;
1482 GST_INFO_OBJECT (self,
1483 "latency status %d frames %d fps %d/%d time %" GST_TIME_FORMAT, status,
1484 frames, self->video_info.fps_n, self->video_info.fps_d,
1485 GST_TIME_ARGS (latency));
1486 gst_video_encoder_set_latency (GST_VIDEO_ENCODER (self), latency, latency);
1487 }
1488 CFRelease (value);
1489 }
1490
1491 static GstFlowReturn
gst_vtenc_encode_frame(GstVTEnc * self,GstVideoCodecFrame * frame)1492 gst_vtenc_encode_frame (GstVTEnc * self, GstVideoCodecFrame * frame)
1493 {
1494 CMTime ts, duration;
1495 GstCoreMediaMeta *meta;
1496 CVPixelBufferRef pbuf = NULL;
1497 GstVideoCodecFrame *outframe;
1498 OSStatus vt_status;
1499 GstFlowReturn ret = GST_FLOW_OK;
1500 gboolean renegotiated;
1501 CFDictionaryRef frame_props = NULL;
1502
1503 if (GST_VIDEO_CODEC_FRAME_IS_FORCE_KEYFRAME (frame)) {
1504 GST_INFO_OBJECT (self, "received force-keyframe-event, will force intra");
1505 frame_props = self->keyframe_props;
1506 }
1507
1508 ts = CMTimeMake (frame->pts, GST_SECOND);
1509 if (frame->duration != GST_CLOCK_TIME_NONE)
1510 duration = CMTimeMake (frame->duration, GST_SECOND);
1511 else
1512 duration = kCMTimeInvalid;
1513
1514 /* If we don't have field order, we need to pick it up from the first buffer
1515 * that has that information. The encoder session also cannot be reconfigured
1516 * with a new field detail after it has been set, so we encode mixed streams
1517 * with whatever the first buffer's field order is. */
1518 if (!self->have_field_order) {
1519 CFStringRef field_detail = NULL;
1520
1521 if (GST_VIDEO_BUFFER_IS_TOP_FIELD (frame->input_buffer))
1522 field_detail = kCMFormatDescriptionFieldDetail_TemporalTopFirst;
1523 else if (GST_VIDEO_BUFFER_IS_BOTTOM_FIELD (frame->input_buffer))
1524 field_detail = kCMFormatDescriptionFieldDetail_TemporalBottomFirst;
1525
1526 if (field_detail) {
1527 vt_status = VTSessionSetProperty (self->session,
1528 kVTCompressionPropertyKey_FieldDetail, field_detail);
1529 GST_DEBUG_OBJECT (self, "kVTCompressionPropertyKey_FieldDetail => %d",
1530 (int) vt_status);
1531 } else {
1532 GST_WARNING_OBJECT (self, "have interlaced content, but don't know field "
1533 "order yet, skipping buffer");
1534 gst_video_codec_frame_unref (frame);
1535 return GST_FLOW_OK;
1536 }
1537
1538 self->have_field_order = TRUE;
1539 }
1540
1541 meta = gst_buffer_get_core_media_meta (frame->input_buffer);
1542 if (meta != NULL) {
1543 pbuf = gst_core_media_buffer_get_pixel_buffer (frame->input_buffer);
1544 }
1545 #ifdef HAVE_IOS
1546 if (pbuf == NULL) {
1547 GstVideoFrame inframe, outframe;
1548 GstBuffer *outbuf;
1549 OSType pixel_format_type;
1550 CVReturn cv_ret;
1551
1552 /* FIXME: iOS has special stride requirements that we don't know yet.
1553 * Copy into a newly allocated pixelbuffer for now. Probably makes
1554 * sense to create a buffer pool around these at some point.
1555 */
1556
1557 switch (GST_VIDEO_INFO_FORMAT (&self->video_info)) {
1558 case GST_VIDEO_FORMAT_I420:
1559 pixel_format_type = kCVPixelFormatType_420YpCbCr8Planar;
1560 break;
1561 case GST_VIDEO_FORMAT_NV12:
1562 pixel_format_type = kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange;
1563 break;
1564 default:
1565 g_assert_not_reached ();
1566 }
1567
1568 if (!gst_video_frame_map (&inframe, &self->video_info, frame->input_buffer,
1569 GST_MAP_READ)) {
1570 GST_ERROR_OBJECT (self, "failed to map input buffer");
1571 goto cv_error;
1572 }
1573
1574 cv_ret =
1575 CVPixelBufferCreate (NULL, self->negotiated_width,
1576 self->negotiated_height, pixel_format_type, NULL, &pbuf);
1577
1578 if (cv_ret != kCVReturnSuccess) {
1579 GST_ERROR_OBJECT (self, "CVPixelBufferCreate failed: %i", cv_ret);
1580 gst_video_frame_unmap (&inframe);
1581 goto cv_error;
1582 }
1583
1584 outbuf =
1585 gst_core_video_buffer_new ((CVBufferRef) pbuf, &self->video_info, NULL);
1586 if (!gst_video_frame_map (&outframe, &self->video_info, outbuf,
1587 GST_MAP_WRITE)) {
1588 GST_ERROR_OBJECT (self, "Failed to map output buffer");
1589 gst_video_frame_unmap (&inframe);
1590 gst_buffer_unref (outbuf);
1591 CVPixelBufferRelease (pbuf);
1592 goto cv_error;
1593 }
1594
1595 if (!gst_video_frame_copy (&outframe, &inframe)) {
1596 GST_ERROR_OBJECT (self, "Failed to copy output frame");
1597 gst_video_frame_unmap (&inframe);
1598 gst_buffer_unref (outbuf);
1599 CVPixelBufferRelease (pbuf);
1600 goto cv_error;
1601 }
1602
1603 gst_buffer_unref (outbuf);
1604 gst_video_frame_unmap (&inframe);
1605 gst_video_frame_unmap (&outframe);
1606 }
1607 #else
1608 if (pbuf == NULL) {
1609 GstVTEncFrame *vframe;
1610 CVReturn cv_ret;
1611
1612 vframe = gst_vtenc_frame_new (frame->input_buffer, &self->video_info);
1613 if (!vframe) {
1614 GST_ERROR_OBJECT (self, "Failed to create a new input frame");
1615 goto cv_error;
1616 }
1617
1618 {
1619 const size_t num_planes = GST_VIDEO_FRAME_N_PLANES (&vframe->videoframe);
1620 void *plane_base_addresses[GST_VIDEO_MAX_PLANES];
1621 size_t plane_widths[GST_VIDEO_MAX_PLANES];
1622 size_t plane_heights[GST_VIDEO_MAX_PLANES];
1623 size_t plane_bytes_per_row[GST_VIDEO_MAX_PLANES];
1624 OSType pixel_format_type;
1625 size_t i;
1626
1627 for (i = 0; i < num_planes; i++) {
1628 plane_base_addresses[i] =
1629 GST_VIDEO_FRAME_PLANE_DATA (&vframe->videoframe, i);
1630 plane_widths[i] = GST_VIDEO_FRAME_COMP_WIDTH (&vframe->videoframe, i);
1631 plane_heights[i] = GST_VIDEO_FRAME_COMP_HEIGHT (&vframe->videoframe, i);
1632 plane_bytes_per_row[i] =
1633 GST_VIDEO_FRAME_COMP_STRIDE (&vframe->videoframe, i);
1634 plane_bytes_per_row[i] =
1635 GST_VIDEO_FRAME_COMP_STRIDE (&vframe->videoframe, i);
1636 }
1637
1638 switch (GST_VIDEO_INFO_FORMAT (&self->video_info)) {
1639 case GST_VIDEO_FORMAT_ARGB64_BE:
1640 pixel_format_type = kCVPixelFormatType_64ARGB;
1641 break;
1642 case GST_VIDEO_FORMAT_AYUV64:
1643 /* This is fine for now because Apple only ships LE devices */
1644 #if G_BYTE_ORDER != G_LITTLE_ENDIAN
1645 #error "AYUV64 is NE but kCVPixelFormatType_4444AYpCbCr16 is LE"
1646 #endif
1647 pixel_format_type = kCVPixelFormatType_4444AYpCbCr16;
1648 break;
1649 case GST_VIDEO_FORMAT_RGBA64_LE:
1650 if (GST_VTUTIL_HAVE_64ARGBALE)
1651 pixel_format_type = kCVPixelFormatType_64RGBALE;
1652 else
1653 /* Codepath will never be hit on macOS older than Big Sur (11.3) */
1654 g_assert_not_reached ();
1655 break;
1656 case GST_VIDEO_FORMAT_I420:
1657 pixel_format_type = kCVPixelFormatType_420YpCbCr8Planar;
1658 break;
1659 case GST_VIDEO_FORMAT_NV12:
1660 pixel_format_type = kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange;
1661 break;
1662 case GST_VIDEO_FORMAT_UYVY:
1663 pixel_format_type = kCVPixelFormatType_422YpCbCr8;
1664 break;
1665 default:
1666 g_assert_not_reached ();
1667 }
1668
1669 cv_ret = CVPixelBufferCreateWithPlanarBytes (NULL,
1670 self->negotiated_width, self->negotiated_height,
1671 pixel_format_type,
1672 frame,
1673 GST_VIDEO_FRAME_SIZE (&vframe->videoframe),
1674 num_planes,
1675 plane_base_addresses,
1676 plane_widths,
1677 plane_heights,
1678 plane_bytes_per_row, gst_pixel_buffer_release_cb, vframe, NULL,
1679 &pbuf);
1680 if (cv_ret != kCVReturnSuccess) {
1681 GST_ERROR_OBJECT (self, "CVPixelBufferCreateWithPlanarBytes failed: %i",
1682 cv_ret);
1683 gst_vtenc_frame_free (vframe);
1684 goto cv_error;
1685 }
1686 }
1687 }
1688 #endif
1689
1690 /* We need to unlock the stream lock here because
1691 * it can wait for gst_vtenc_enqueue_buffer() to
1692 * handle a buffer... which will take the stream
1693 * lock from another thread and then deadlock */
1694 GST_VIDEO_ENCODER_STREAM_UNLOCK (self);
1695 vt_status = VTCompressionSessionEncodeFrame (self->session,
1696 pbuf, ts, duration, frame_props,
1697 GINT_TO_POINTER (frame->system_frame_number), NULL);
1698 GST_VIDEO_ENCODER_STREAM_LOCK (self);
1699
1700 if (vt_status != noErr) {
1701 GST_WARNING_OBJECT (self, "VTCompressionSessionEncodeFrame returned %d",
1702 (int) vt_status);
1703 }
1704
1705 gst_video_codec_frame_unref (frame);
1706
1707 CVPixelBufferRelease (pbuf);
1708
1709 renegotiated = FALSE;
1710 while ((outframe = g_async_queue_try_pop (self->cur_outframes))) {
1711 if (outframe->output_buffer) {
1712 if (!renegotiated) {
1713 meta = gst_buffer_get_core_media_meta (outframe->output_buffer);
1714 /* Try to renegotiate once */
1715 if (meta) {
1716 if (gst_vtenc_negotiate_downstream (self, meta->sample_buf)) {
1717 renegotiated = TRUE;
1718 } else {
1719 ret = GST_FLOW_NOT_NEGOTIATED;
1720 gst_video_codec_frame_unref (outframe);
1721 /* the rest of the frames will be pop'd and unref'd later */
1722 break;
1723 }
1724 }
1725 }
1726
1727 gst_vtenc_update_latency (self);
1728 }
1729
1730 /* releases frame, even if it has no output buffer (i.e. failed to encode) */
1731 ret =
1732 gst_video_encoder_finish_frame (GST_VIDEO_ENCODER_CAST (self),
1733 outframe);
1734 }
1735
1736 return ret;
1737
1738 cv_error:
1739 {
1740 gst_video_codec_frame_unref (frame);
1741 return GST_FLOW_ERROR;
1742 }
1743 }
1744
1745 static void
gst_vtenc_enqueue_buffer(void * outputCallbackRefCon,void * sourceFrameRefCon,OSStatus status,VTEncodeInfoFlags infoFlags,CMSampleBufferRef sampleBuffer)1746 gst_vtenc_enqueue_buffer (void *outputCallbackRefCon,
1747 void *sourceFrameRefCon,
1748 OSStatus status,
1749 VTEncodeInfoFlags infoFlags, CMSampleBufferRef sampleBuffer)
1750 {
1751 GstVTEnc *self = outputCallbackRefCon;
1752 gboolean is_keyframe;
1753 GstVideoCodecFrame *frame;
1754
1755 frame =
1756 gst_video_encoder_get_frame (GST_VIDEO_ENCODER_CAST (self),
1757 GPOINTER_TO_INT (sourceFrameRefCon));
1758
1759 if (status != noErr) {
1760 if (frame) {
1761 GST_ELEMENT_ERROR (self, LIBRARY, ENCODE, (NULL),
1762 ("Failed to encode frame %d: %d", frame->system_frame_number,
1763 (int) status));
1764 } else {
1765 GST_ELEMENT_ERROR (self, LIBRARY, ENCODE, (NULL),
1766 ("Failed to encode (frame unknown): %d", (int) status));
1767 }
1768 goto beach;
1769 }
1770
1771 if (!frame) {
1772 GST_WARNING_OBJECT (self, "No corresponding frame found!");
1773 goto beach;
1774 }
1775
1776 /* This may happen if we don't have enough bitrate */
1777 if (sampleBuffer == NULL)
1778 goto beach;
1779
1780 is_keyframe = gst_vtenc_buffer_is_keyframe (self, sampleBuffer);
1781
1782 if (is_keyframe) {
1783 GST_VIDEO_CODEC_FRAME_SET_SYNC_POINT (frame);
1784 gst_vtenc_clear_cached_caps_downstream (self);
1785 }
1786
1787 /* We are dealing with block buffers here, so we don't need
1788 * to enable the use of the video meta API on the core media buffer */
1789 frame->output_buffer = gst_core_media_buffer_new (sampleBuffer, FALSE, NULL);
1790
1791 beach:
1792 /* needed anyway so the frame will be released */
1793 if (frame)
1794 g_async_queue_push (self->cur_outframes, frame);
1795 }
1796
1797 static gboolean
gst_vtenc_buffer_is_keyframe(GstVTEnc * self,CMSampleBufferRef sbuf)1798 gst_vtenc_buffer_is_keyframe (GstVTEnc * self, CMSampleBufferRef sbuf)
1799 {
1800 gboolean result = FALSE;
1801 CFArrayRef attachments_for_sample;
1802
1803 attachments_for_sample = CMSampleBufferGetSampleAttachmentsArray (sbuf, 0);
1804 if (attachments_for_sample != NULL) {
1805 CFDictionaryRef attachments;
1806 CFBooleanRef depends_on_others;
1807
1808 attachments = CFArrayGetValueAtIndex (attachments_for_sample, 0);
1809 depends_on_others = CFDictionaryGetValue (attachments,
1810 kCMSampleAttachmentKey_DependsOnOthers);
1811 result = (depends_on_others == kCFBooleanFalse);
1812 }
1813
1814 return result;
1815 }
1816
1817 #ifndef HAVE_IOS
1818 static GstVTEncFrame *
gst_vtenc_frame_new(GstBuffer * buf,GstVideoInfo * video_info)1819 gst_vtenc_frame_new (GstBuffer * buf, GstVideoInfo * video_info)
1820 {
1821 GstVTEncFrame *frame;
1822
1823 frame = g_slice_new (GstVTEncFrame);
1824 frame->buf = gst_buffer_ref (buf);
1825 if (!gst_video_frame_map (&frame->videoframe, video_info, buf, GST_MAP_READ)) {
1826 gst_buffer_unref (frame->buf);
1827 g_slice_free (GstVTEncFrame, frame);
1828 return NULL;
1829 }
1830
1831 return frame;
1832 }
1833
1834 static void
gst_vtenc_frame_free(GstVTEncFrame * frame)1835 gst_vtenc_frame_free (GstVTEncFrame * frame)
1836 {
1837 gst_video_frame_unmap (&frame->videoframe);
1838 gst_buffer_unref (frame->buf);
1839 g_slice_free (GstVTEncFrame, frame);
1840 }
1841
1842 static void
gst_pixel_buffer_release_cb(void * releaseRefCon,const void * dataPtr,size_t dataSize,size_t numberOfPlanes,const void * planeAddresses[])1843 gst_pixel_buffer_release_cb (void *releaseRefCon, const void *dataPtr,
1844 size_t dataSize, size_t numberOfPlanes, const void *planeAddresses[])
1845 {
1846 GstVTEncFrame *frame = (GstVTEncFrame *) releaseRefCon;
1847 gst_vtenc_frame_free (frame);
1848 }
1849 #endif
1850
1851 static void
gst_vtenc_register(GstPlugin * plugin,const GstVTEncoderDetails * codec_details)1852 gst_vtenc_register (GstPlugin * plugin,
1853 const GstVTEncoderDetails * codec_details)
1854 {
1855 GTypeInfo type_info = {
1856 sizeof (GstVTEncClass),
1857 (GBaseInitFunc) gst_vtenc_base_init,
1858 NULL,
1859 (GClassInitFunc) gst_vtenc_class_init,
1860 NULL,
1861 NULL,
1862 sizeof (GstVTEnc),
1863 0,
1864 (GInstanceInitFunc) gst_vtenc_init,
1865 };
1866 gchar *type_name;
1867 GType type;
1868 gboolean result;
1869
1870 type_name = g_strdup_printf ("vtenc_%s", codec_details->element_name);
1871
1872 type =
1873 g_type_register_static (GST_TYPE_VIDEO_ENCODER, type_name, &type_info, 0);
1874
1875 g_type_set_qdata (type, GST_VTENC_CODEC_DETAILS_QDATA,
1876 (gpointer) codec_details);
1877
1878 result = gst_element_register (plugin, type_name, GST_RANK_PRIMARY, type);
1879 if (!result) {
1880 GST_ERROR_OBJECT (plugin, "failed to register element %s", type_name);
1881 }
1882
1883 g_free (type_name);
1884 }
1885
1886 static const GstVTEncoderDetails gst_vtenc_codecs[] = {
1887 {"H.264", "h264", "video/x-h264", kCMVideoCodecType_H264, FALSE},
1888 #ifndef HAVE_IOS
1889 {"H.264 (HW only)", "h264_hw", "video/x-h264", kCMVideoCodecType_H264, TRUE},
1890 #endif
1891 {"Apple ProRes", "prores", "video/x-prores",
1892 GST_kCMVideoCodecType_Some_AppleProRes, FALSE},
1893 };
1894
1895 void
gst_vtenc_register_elements(GstPlugin * plugin)1896 gst_vtenc_register_elements (GstPlugin * plugin)
1897 {
1898 guint i;
1899
1900 GST_DEBUG_CATEGORY_INIT (gst_vtenc_debug, "vtenc",
1901 0, "Apple VideoToolbox Encoder Wrapper");
1902
1903 for (i = 0; i != G_N_ELEMENTS (gst_vtenc_codecs); i++)
1904 gst_vtenc_register (plugin, &gst_vtenc_codecs[i]);
1905 }
1906