1 /*
2 * This file is part of FFmpeg.
3 *
4 * FFmpeg is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU Lesser General Public
6 * License as published by the Free Software Foundation; either
7 * version 2.1 of the License, or (at your option) any later version.
8 *
9 * FFmpeg is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12 * Lesser General Public License for more details.
13 *
14 * You should have received a copy of the GNU Lesser General Public
15 * License along with FFmpeg; if not, write to the Free Software
16 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17 */
18
19 #include "config.h"
20
21 #include <stdint.h>
22 #include <string.h>
23
24 #include <VideoToolbox/VideoToolbox.h>
25
26 #include "buffer.h"
27 #include "buffer_internal.h"
28 #include "common.h"
29 #include "hwcontext.h"
30 #include "hwcontext_internal.h"
31 #include "hwcontext_videotoolbox.h"
32 #include "mem.h"
33 #include "pixfmt.h"
34 #include "pixdesc.h"
35
36 typedef struct VTFramesContext {
37 CVPixelBufferPoolRef pool;
38 } VTFramesContext;
39
40 static const struct {
41 uint32_t cv_fmt;
42 bool full_range;
43 enum AVPixelFormat pix_fmt;
44 } cv_pix_fmts[] = {
45 { kCVPixelFormatType_420YpCbCr8Planar, false, AV_PIX_FMT_YUV420P },
46 { kCVPixelFormatType_422YpCbCr8, false, AV_PIX_FMT_UYVY422 },
47 { kCVPixelFormatType_32BGRA, false, AV_PIX_FMT_BGRA },
48 #ifdef kCFCoreFoundationVersionNumber10_7
49 { kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange, false, AV_PIX_FMT_NV12 },
50 { kCVPixelFormatType_420YpCbCr8BiPlanarFullRange, true, AV_PIX_FMT_NV12 },
51 { kCVPixelFormatType_4444AYpCbCr16, false, AV_PIX_FMT_AYUV64 },
52 #endif
53 #if HAVE_KCVPIXELFORMATTYPE_420YPCBCR10BIPLANARVIDEORANGE
54 { kCVPixelFormatType_420YpCbCr10BiPlanarVideoRange, false, AV_PIX_FMT_P010 },
55 { kCVPixelFormatType_420YpCbCr10BiPlanarFullRange, true, AV_PIX_FMT_P010 },
56 #endif
57 #if HAVE_KCVPIXELFORMATTYPE_422YPCBCR8BIPLANARVIDEORANGE
58 { kCVPixelFormatType_422YpCbCr8BiPlanarVideoRange, false, AV_PIX_FMT_NV16 },
59 { kCVPixelFormatType_422YpCbCr8BiPlanarFullRange, true, AV_PIX_FMT_NV16 },
60 #endif
61 #if HAVE_KCVPIXELFORMATTYPE_422YPCBCR10BIPLANARVIDEORANGE
62 { kCVPixelFormatType_422YpCbCr10BiPlanarVideoRange, false, AV_PIX_FMT_P210 },
63 { kCVPixelFormatType_422YpCbCr10BiPlanarFullRange, true, AV_PIX_FMT_P210 },
64 #endif
65 #if HAVE_KCVPIXELFORMATTYPE_422YPCBCR16BIPLANARVIDEORANGE
66 { kCVPixelFormatType_422YpCbCr16BiPlanarVideoRange, false, AV_PIX_FMT_P216 },
67 #endif
68 #if HAVE_KCVPIXELFORMATTYPE_444YPCBCR8BIPLANARVIDEORANGE
69 { kCVPixelFormatType_444YpCbCr8BiPlanarVideoRange, false, AV_PIX_FMT_NV24 },
70 { kCVPixelFormatType_444YpCbCr8BiPlanarFullRange, true, AV_PIX_FMT_NV24 },
71 #endif
72 #if HAVE_KCVPIXELFORMATTYPE_444YPCBCR10BIPLANARVIDEORANGE
73 { kCVPixelFormatType_444YpCbCr10BiPlanarVideoRange, false, AV_PIX_FMT_P410 },
74 { kCVPixelFormatType_444YpCbCr10BiPlanarFullRange, true, AV_PIX_FMT_P410 },
75 #endif
76 #if HAVE_KCVPIXELFORMATTYPE_444YPCBCR16BIPLANARVIDEORANGE
77 { kCVPixelFormatType_444YpCbCr16BiPlanarVideoRange, false, AV_PIX_FMT_P416 },
78 #endif
79 };
80
81 static const enum AVPixelFormat supported_formats[] = {
82 #ifdef kCFCoreFoundationVersionNumber10_7
83 AV_PIX_FMT_NV12,
84 AV_PIX_FMT_AYUV64,
85 #endif
86 AV_PIX_FMT_YUV420P,
87 AV_PIX_FMT_UYVY422,
88 #if HAVE_KCVPIXELFORMATTYPE_420YPCBCR10BIPLANARVIDEORANGE
89 AV_PIX_FMT_P010,
90 #endif
91 #if HAVE_KCVPIXELFORMATTYPE_422YPCBCR8BIPLANARVIDEORANGE
92 AV_PIX_FMT_NV16,
93 #endif
94 #if HAVE_KCVPIXELFORMATTYPE_422YPCBCR10BIPLANARVIDEORANGE
95 AV_PIX_FMT_P210,
96 #endif
97 #if HAVE_KCVPIXELFORMATTYPE_422YPCBCR16BIPLANARVIDEORANGE
98 AV_PIX_FMT_P216,
99 #endif
100 #if HAVE_KCVPIXELFORMATTYPE_444YPCBCR8BIPLANARVIDEORANGE
101 AV_PIX_FMT_NV24,
102 #endif
103 #if HAVE_KCVPIXELFORMATTYPE_444YPCBCR10BIPLANARVIDEORANGE
104 AV_PIX_FMT_P410,
105 #endif
106 #if HAVE_KCVPIXELFORMATTYPE_444YPCBCR16BIPLANARVIDEORANGE
107 AV_PIX_FMT_P416,
108 #endif
109 AV_PIX_FMT_BGRA,
110 };
111
vt_frames_get_constraints(AVHWDeviceContext * ctx,const void * hwconfig,AVHWFramesConstraints * constraints)112 static int vt_frames_get_constraints(AVHWDeviceContext *ctx,
113 const void *hwconfig,
114 AVHWFramesConstraints *constraints)
115 {
116 int i;
117
118 constraints->valid_sw_formats = av_malloc_array(FF_ARRAY_ELEMS(supported_formats) + 1,
119 sizeof(*constraints->valid_sw_formats));
120 if (!constraints->valid_sw_formats)
121 return AVERROR(ENOMEM);
122
123 for (i = 0; i < FF_ARRAY_ELEMS(supported_formats); i++)
124 constraints->valid_sw_formats[i] = supported_formats[i];
125 constraints->valid_sw_formats[FF_ARRAY_ELEMS(supported_formats)] = AV_PIX_FMT_NONE;
126
127 constraints->valid_hw_formats = av_malloc_array(2, sizeof(*constraints->valid_hw_formats));
128 if (!constraints->valid_hw_formats)
129 return AVERROR(ENOMEM);
130
131 constraints->valid_hw_formats[0] = AV_PIX_FMT_VIDEOTOOLBOX;
132 constraints->valid_hw_formats[1] = AV_PIX_FMT_NONE;
133
134 return 0;
135 }
136
av_map_videotoolbox_format_to_pixfmt(uint32_t cv_fmt)137 enum AVPixelFormat av_map_videotoolbox_format_to_pixfmt(uint32_t cv_fmt)
138 {
139 int i;
140 for (i = 0; i < FF_ARRAY_ELEMS(cv_pix_fmts); i++) {
141 if (cv_pix_fmts[i].cv_fmt == cv_fmt)
142 return cv_pix_fmts[i].pix_fmt;
143 }
144 return AV_PIX_FMT_NONE;
145 }
146
av_map_videotoolbox_format_from_pixfmt(enum AVPixelFormat pix_fmt)147 uint32_t av_map_videotoolbox_format_from_pixfmt(enum AVPixelFormat pix_fmt)
148 {
149 return av_map_videotoolbox_format_from_pixfmt2(pix_fmt, false);
150 }
151
av_map_videotoolbox_format_from_pixfmt2(enum AVPixelFormat pix_fmt,bool full_range)152 uint32_t av_map_videotoolbox_format_from_pixfmt2(enum AVPixelFormat pix_fmt, bool full_range)
153 {
154 int i;
155 for (i = 0; i < FF_ARRAY_ELEMS(cv_pix_fmts); i++) {
156 if (cv_pix_fmts[i].pix_fmt == pix_fmt && cv_pix_fmts[i].full_range == full_range)
157 return cv_pix_fmts[i].cv_fmt;
158 }
159 return 0;
160 }
161
vt_pool_alloc(AVHWFramesContext * ctx)162 static int vt_pool_alloc(AVHWFramesContext *ctx)
163 {
164 VTFramesContext *fctx = ctx->internal->priv;
165 CVReturn err;
166 CFNumberRef w, h, pixfmt;
167 uint32_t cv_pixfmt;
168 CFMutableDictionaryRef attributes, iosurface_properties;
169
170 attributes = CFDictionaryCreateMutable(
171 NULL,
172 2,
173 &kCFTypeDictionaryKeyCallBacks,
174 &kCFTypeDictionaryValueCallBacks);
175
176 cv_pixfmt = av_map_videotoolbox_format_from_pixfmt(ctx->sw_format);
177 pixfmt = CFNumberCreate(NULL, kCFNumberSInt32Type, &cv_pixfmt);
178 CFDictionarySetValue(
179 attributes,
180 kCVPixelBufferPixelFormatTypeKey,
181 pixfmt);
182 CFRelease(pixfmt);
183
184 iosurface_properties = CFDictionaryCreateMutable(
185 NULL,
186 0,
187 &kCFTypeDictionaryKeyCallBacks,
188 &kCFTypeDictionaryValueCallBacks);
189 CFDictionarySetValue(attributes, kCVPixelBufferIOSurfacePropertiesKey, iosurface_properties);
190 CFRelease(iosurface_properties);
191
192 w = CFNumberCreate(NULL, kCFNumberSInt32Type, &ctx->width);
193 h = CFNumberCreate(NULL, kCFNumberSInt32Type, &ctx->height);
194 CFDictionarySetValue(attributes, kCVPixelBufferWidthKey, w);
195 CFDictionarySetValue(attributes, kCVPixelBufferHeightKey, h);
196 CFRelease(w);
197 CFRelease(h);
198
199 err = CVPixelBufferPoolCreate(
200 NULL,
201 NULL,
202 attributes,
203 &fctx->pool);
204 CFRelease(attributes);
205
206 if (err == kCVReturnSuccess)
207 return 0;
208
209 av_log(ctx, AV_LOG_ERROR, "Error creating CVPixelBufferPool: %d\n", err);
210 return AVERROR_EXTERNAL;
211 }
212
videotoolbox_buffer_release(void * opaque,uint8_t * data)213 static void videotoolbox_buffer_release(void *opaque, uint8_t *data)
214 {
215 CVPixelBufferRelease((CVPixelBufferRef)data);
216 }
217
vt_pool_alloc_buffer(void * opaque,size_t size)218 static AVBufferRef *vt_pool_alloc_buffer(void *opaque, size_t size)
219 {
220 CVPixelBufferRef pixbuf;
221 AVBufferRef *buf;
222 CVReturn err;
223 AVHWFramesContext *ctx = opaque;
224 VTFramesContext *fctx = ctx->internal->priv;
225
226 err = CVPixelBufferPoolCreatePixelBuffer(
227 NULL,
228 fctx->pool,
229 &pixbuf
230 );
231 if (err != kCVReturnSuccess) {
232 av_log(ctx, AV_LOG_ERROR, "Failed to create pixel buffer from pool: %d\n", err);
233 return NULL;
234 }
235
236 buf = av_buffer_create((uint8_t *)pixbuf, size,
237 videotoolbox_buffer_release, NULL, 0);
238 if (!buf) {
239 CVPixelBufferRelease(pixbuf);
240 return NULL;
241 }
242 return buf;
243 }
244
vt_frames_uninit(AVHWFramesContext * ctx)245 static void vt_frames_uninit(AVHWFramesContext *ctx)
246 {
247 VTFramesContext *fctx = ctx->internal->priv;
248 if (fctx->pool) {
249 CVPixelBufferPoolRelease(fctx->pool);
250 fctx->pool = NULL;
251 }
252 }
253
vt_frames_init(AVHWFramesContext * ctx)254 static int vt_frames_init(AVHWFramesContext *ctx)
255 {
256 int i, ret;
257
258 for (i = 0; i < FF_ARRAY_ELEMS(supported_formats); i++) {
259 if (ctx->sw_format == supported_formats[i])
260 break;
261 }
262 if (i == FF_ARRAY_ELEMS(supported_formats)) {
263 av_log(ctx, AV_LOG_ERROR, "Pixel format '%s' is not supported\n",
264 av_get_pix_fmt_name(ctx->sw_format));
265 return AVERROR(ENOSYS);
266 }
267
268 if (!ctx->pool) {
269 ctx->internal->pool_internal = av_buffer_pool_init2(
270 sizeof(CVPixelBufferRef), ctx, vt_pool_alloc_buffer, NULL);
271 if (!ctx->internal->pool_internal)
272 return AVERROR(ENOMEM);
273 }
274
275 ret = vt_pool_alloc(ctx);
276 if (ret < 0)
277 return ret;
278
279 return 0;
280 }
281
vt_get_buffer(AVHWFramesContext * ctx,AVFrame * frame)282 static int vt_get_buffer(AVHWFramesContext *ctx, AVFrame *frame)
283 {
284 frame->buf[0] = av_buffer_pool_get(ctx->pool);
285 if (!frame->buf[0])
286 return AVERROR(ENOMEM);
287
288 frame->data[3] = frame->buf[0]->data;
289 frame->format = AV_PIX_FMT_VIDEOTOOLBOX;
290 frame->width = ctx->width;
291 frame->height = ctx->height;
292
293 return 0;
294 }
295
vt_transfer_get_formats(AVHWFramesContext * ctx,enum AVHWFrameTransferDirection dir,enum AVPixelFormat ** formats)296 static int vt_transfer_get_formats(AVHWFramesContext *ctx,
297 enum AVHWFrameTransferDirection dir,
298 enum AVPixelFormat **formats)
299 {
300 enum AVPixelFormat *fmts = av_malloc_array(2, sizeof(*fmts));
301 if (!fmts)
302 return AVERROR(ENOMEM);
303
304 fmts[0] = ctx->sw_format;
305 fmts[1] = AV_PIX_FMT_NONE;
306
307 *formats = fmts;
308 return 0;
309 }
310
vt_unmap(AVHWFramesContext * ctx,HWMapDescriptor * hwmap)311 static void vt_unmap(AVHWFramesContext *ctx, HWMapDescriptor *hwmap)
312 {
313 CVPixelBufferRef pixbuf = (CVPixelBufferRef)hwmap->source->data[3];
314
315 CVPixelBufferUnlockBaseAddress(pixbuf, (uintptr_t)hwmap->priv);
316 }
317
vt_pixbuf_set_par(void * log_ctx,CVPixelBufferRef pixbuf,const AVFrame * src)318 static int vt_pixbuf_set_par(void *log_ctx,
319 CVPixelBufferRef pixbuf, const AVFrame *src)
320 {
321 CFMutableDictionaryRef par = NULL;
322 CFNumberRef num = NULL, den = NULL;
323 AVRational avpar = src->sample_aspect_ratio;
324
325 if (avpar.num == 0)
326 return 0;
327
328 av_reduce(&avpar.num, &avpar.den,
329 avpar.num, avpar.den,
330 0xFFFFFFFF);
331
332 num = CFNumberCreate(kCFAllocatorDefault,
333 kCFNumberIntType,
334 &avpar.num);
335
336 den = CFNumberCreate(kCFAllocatorDefault,
337 kCFNumberIntType,
338 &avpar.den);
339
340 par = CFDictionaryCreateMutable(kCFAllocatorDefault,
341 2,
342 &kCFCopyStringDictionaryKeyCallBacks,
343 &kCFTypeDictionaryValueCallBacks);
344
345 if (!par || !num || !den) {
346 if (par) CFRelease(par);
347 if (num) CFRelease(num);
348 if (den) CFRelease(den);
349 return AVERROR(ENOMEM);
350 }
351
352 CFDictionarySetValue(
353 par,
354 kCVImageBufferPixelAspectRatioHorizontalSpacingKey,
355 num);
356 CFDictionarySetValue(
357 par,
358 kCVImageBufferPixelAspectRatioVerticalSpacingKey,
359 den);
360
361 CVBufferSetAttachment(
362 pixbuf,
363 kCVImageBufferPixelAspectRatioKey,
364 par,
365 kCVAttachmentMode_ShouldPropagate
366 );
367
368 CFRelease(par);
369 CFRelease(num);
370 CFRelease(den);
371
372 return 0;
373 }
374
av_map_videotoolbox_chroma_loc_from_av(enum AVChromaLocation loc)375 CFStringRef av_map_videotoolbox_chroma_loc_from_av(enum AVChromaLocation loc)
376 {
377 switch (loc) {
378 case AVCHROMA_LOC_LEFT:
379 return kCVImageBufferChromaLocation_Left;
380 case AVCHROMA_LOC_CENTER:
381 return kCVImageBufferChromaLocation_Center;
382 case AVCHROMA_LOC_TOP:
383 return kCVImageBufferChromaLocation_Top;
384 case AVCHROMA_LOC_BOTTOM:
385 return kCVImageBufferChromaLocation_Bottom;
386 case AVCHROMA_LOC_TOPLEFT:
387 return kCVImageBufferChromaLocation_TopLeft;
388 case AVCHROMA_LOC_BOTTOMLEFT:
389 return kCVImageBufferChromaLocation_BottomLeft;
390 default:
391 return NULL;
392 }
393 }
394
vt_pixbuf_set_chromaloc(void * log_ctx,CVPixelBufferRef pixbuf,const AVFrame * src)395 static int vt_pixbuf_set_chromaloc(void *log_ctx,
396 CVPixelBufferRef pixbuf, const AVFrame *src)
397 {
398 CFStringRef loc = av_map_videotoolbox_chroma_loc_from_av(src->chroma_location);
399
400 if (loc) {
401 CVBufferSetAttachment(
402 pixbuf,
403 kCVImageBufferChromaLocationTopFieldKey,
404 loc,
405 kCVAttachmentMode_ShouldPropagate);
406 }
407
408 return 0;
409 }
410
av_map_videotoolbox_color_matrix_from_av(enum AVColorSpace space)411 CFStringRef av_map_videotoolbox_color_matrix_from_av(enum AVColorSpace space)
412 {
413 switch (space) {
414 case AVCOL_SPC_BT2020_CL:
415 case AVCOL_SPC_BT2020_NCL:
416 #if HAVE_KCVIMAGEBUFFERYCBCRMATRIX_ITU_R_2020
417 if (__builtin_available(macOS 10.11, iOS 9, *))
418 return kCVImageBufferYCbCrMatrix_ITU_R_2020;
419 #endif
420 return CFSTR("ITU_R_2020");
421 case AVCOL_SPC_BT470BG:
422 case AVCOL_SPC_SMPTE170M:
423 return kCVImageBufferYCbCrMatrix_ITU_R_601_4;
424 case AVCOL_SPC_BT709:
425 return kCVImageBufferYCbCrMatrix_ITU_R_709_2;
426 case AVCOL_SPC_SMPTE240M:
427 return kCVImageBufferYCbCrMatrix_SMPTE_240M_1995;
428 default:
429 #if HAVE_KCVIMAGEBUFFERTRANSFERFUNCTION_ITU_R_2100_HLG
430 if (__builtin_available(macOS 10.13, iOS 11, tvOS 11, watchOS 4, *))
431 return CVYCbCrMatrixGetStringForIntegerCodePoint(space);
432 #endif
433 case AVCOL_SPC_UNSPECIFIED:
434 return NULL;
435 }
436 }
437
av_map_videotoolbox_color_primaries_from_av(enum AVColorPrimaries pri)438 CFStringRef av_map_videotoolbox_color_primaries_from_av(enum AVColorPrimaries pri)
439 {
440 switch (pri) {
441 case AVCOL_PRI_BT2020:
442 #if HAVE_KCVIMAGEBUFFERCOLORPRIMARIES_ITU_R_2020
443 if (__builtin_available(macOS 10.11, iOS 9, *))
444 return kCVImageBufferColorPrimaries_ITU_R_2020;
445 #endif
446 return CFSTR("ITU_R_2020");
447 case AVCOL_PRI_BT709:
448 return kCVImageBufferColorPrimaries_ITU_R_709_2;
449 case AVCOL_PRI_SMPTE170M:
450 return kCVImageBufferColorPrimaries_SMPTE_C;
451 case AVCOL_PRI_BT470BG:
452 return kCVImageBufferColorPrimaries_EBU_3213;
453 default:
454 #if HAVE_KCVIMAGEBUFFERTRANSFERFUNCTION_ITU_R_2100_HLG
455 if (__builtin_available(macOS 10.13, iOS 11, tvOS 11, watchOS 4, *))
456 return CVColorPrimariesGetStringForIntegerCodePoint(pri);
457 #endif
458 case AVCOL_PRI_UNSPECIFIED:
459 return NULL;
460 }
461 }
462
av_map_videotoolbox_color_trc_from_av(enum AVColorTransferCharacteristic trc)463 CFStringRef av_map_videotoolbox_color_trc_from_av(enum AVColorTransferCharacteristic trc)
464 {
465
466 switch (trc) {
467 case AVCOL_TRC_SMPTE2084:
468 #if HAVE_KCVIMAGEBUFFERTRANSFERFUNCTION_SMPTE_ST_2084_PQ
469 if (__builtin_available(macOS 10.13, iOS 11, *))
470 return kCVImageBufferTransferFunction_SMPTE_ST_2084_PQ;
471 #endif
472 return CFSTR("SMPTE_ST_2084_PQ");
473 case AVCOL_TRC_BT2020_10:
474 case AVCOL_TRC_BT2020_12:
475 #if HAVE_KCVIMAGEBUFFERTRANSFERFUNCTION_ITU_R_2020
476 if (__builtin_available(macOS 10.11, iOS 9, *))
477 return kCVImageBufferTransferFunction_ITU_R_2020;
478 #endif
479 return CFSTR("ITU_R_2020");
480 case AVCOL_TRC_BT709:
481 return kCVImageBufferTransferFunction_ITU_R_709_2;
482 case AVCOL_TRC_SMPTE240M:
483 return kCVImageBufferTransferFunction_SMPTE_240M_1995;
484 case AVCOL_TRC_SMPTE428:
485 #if HAVE_KCVIMAGEBUFFERTRANSFERFUNCTION_SMPTE_ST_428_1
486 if (__builtin_available(macOS 10.12, iOS 10, *))
487 return kCVImageBufferTransferFunction_SMPTE_ST_428_1;
488 #endif
489 return CFSTR("SMPTE_ST_428_1");
490 case AVCOL_TRC_ARIB_STD_B67:
491 #if HAVE_KCVIMAGEBUFFERTRANSFERFUNCTION_ITU_R_2100_HLG
492 if (__builtin_available(macOS 10.13, iOS 11, *))
493 return kCVImageBufferTransferFunction_ITU_R_2100_HLG;
494 #endif
495 return CFSTR("ITU_R_2100_HLG");
496 case AVCOL_TRC_GAMMA22:
497 return kCVImageBufferTransferFunction_UseGamma;
498 case AVCOL_TRC_GAMMA28:
499 return kCVImageBufferTransferFunction_UseGamma;
500 default:
501 #if HAVE_KCVIMAGEBUFFERTRANSFERFUNCTION_ITU_R_2100_HLG
502 if (__builtin_available(macOS 10.13, iOS 11, tvOS 11, watchOS 4, *))
503 return CVTransferFunctionGetStringForIntegerCodePoint(trc);
504 #endif
505 case AVCOL_TRC_UNSPECIFIED:
506 return NULL;
507 }
508 }
509
vt_pixbuf_set_colorspace(void * log_ctx,CVPixelBufferRef pixbuf,const AVFrame * src)510 static int vt_pixbuf_set_colorspace(void *log_ctx,
511 CVPixelBufferRef pixbuf, const AVFrame *src)
512 {
513 CFStringRef colormatrix = NULL, colorpri = NULL, colortrc = NULL;
514 Float32 gamma = 0;
515
516 colormatrix = av_map_videotoolbox_color_matrix_from_av(src->colorspace);
517 if (!colormatrix && src->colorspace != AVCOL_SPC_UNSPECIFIED)
518 av_log(log_ctx, AV_LOG_WARNING, "Color space %s is not supported.\n", av_color_space_name(src->colorspace));
519
520 colorpri = av_map_videotoolbox_color_primaries_from_av(src->color_primaries);
521 if (!colorpri && src->color_primaries != AVCOL_PRI_UNSPECIFIED)
522 av_log(log_ctx, AV_LOG_WARNING, "Color primaries %s is not supported.\n", av_color_primaries_name(src->color_primaries));
523
524 colortrc = av_map_videotoolbox_color_trc_from_av(src->color_trc);
525 if (!colortrc && src->color_trc != AVCOL_TRC_UNSPECIFIED)
526 av_log(log_ctx, AV_LOG_WARNING, "Color transfer function %s is not supported.\n", av_color_transfer_name(src->color_trc));
527
528 if (src->color_trc == AVCOL_TRC_GAMMA22)
529 gamma = 2.2;
530 else if (src->color_trc == AVCOL_TRC_GAMMA28)
531 gamma = 2.8;
532
533 if (colormatrix) {
534 CVBufferSetAttachment(
535 pixbuf,
536 kCVImageBufferYCbCrMatrixKey,
537 colormatrix,
538 kCVAttachmentMode_ShouldPropagate);
539 }
540 if (colorpri) {
541 CVBufferSetAttachment(
542 pixbuf,
543 kCVImageBufferColorPrimariesKey,
544 colorpri,
545 kCVAttachmentMode_ShouldPropagate);
546 }
547 if (colortrc) {
548 CVBufferSetAttachment(
549 pixbuf,
550 kCVImageBufferTransferFunctionKey,
551 colortrc,
552 kCVAttachmentMode_ShouldPropagate);
553 }
554 if (gamma != 0) {
555 CFNumberRef gamma_level = CFNumberCreate(NULL, kCFNumberFloat32Type, &gamma);
556 CVBufferSetAttachment(
557 pixbuf,
558 kCVImageBufferGammaLevelKey,
559 gamma_level,
560 kCVAttachmentMode_ShouldPropagate);
561 CFRelease(gamma_level);
562 }
563
564 return 0;
565 }
566
vt_pixbuf_set_attachments(void * log_ctx,CVPixelBufferRef pixbuf,const AVFrame * src)567 static int vt_pixbuf_set_attachments(void *log_ctx,
568 CVPixelBufferRef pixbuf, const AVFrame *src)
569 {
570 int ret;
571 ret = vt_pixbuf_set_par(log_ctx, pixbuf, src);
572 if (ret < 0)
573 return ret;
574 ret = vt_pixbuf_set_colorspace(log_ctx, pixbuf, src);
575 if (ret < 0)
576 return ret;
577 ret = vt_pixbuf_set_chromaloc(log_ctx, pixbuf, src);
578 if (ret < 0)
579 return ret;
580 return 0;
581 }
582
av_vt_pixbuf_set_attachments(void * log_ctx,CVPixelBufferRef pixbuf,const AVFrame * src)583 int av_vt_pixbuf_set_attachments(void *log_ctx,
584 CVPixelBufferRef pixbuf, const AVFrame *src)
585 {
586 return vt_pixbuf_set_attachments(log_ctx, pixbuf, src);
587 }
588
vt_map_frame(AVHWFramesContext * ctx,AVFrame * dst,const AVFrame * src,int flags)589 static int vt_map_frame(AVHWFramesContext *ctx, AVFrame *dst, const AVFrame *src,
590 int flags)
591 {
592 CVPixelBufferRef pixbuf = (CVPixelBufferRef)src->data[3];
593 OSType pixel_format = CVPixelBufferGetPixelFormatType(pixbuf);
594 CVReturn err;
595 uint32_t map_flags = 0;
596 int ret;
597 int i;
598 enum AVPixelFormat format;
599
600 format = av_map_videotoolbox_format_to_pixfmt(pixel_format);
601 if (dst->format != format) {
602 av_log(ctx, AV_LOG_ERROR, "Unsupported or mismatching pixel format: %s\n",
603 av_fourcc2str(pixel_format));
604 return AVERROR_UNKNOWN;
605 }
606
607 if (CVPixelBufferGetWidth(pixbuf) != ctx->width ||
608 CVPixelBufferGetHeight(pixbuf) != ctx->height) {
609 av_log(ctx, AV_LOG_ERROR, "Inconsistent frame dimensions.\n");
610 return AVERROR_UNKNOWN;
611 }
612
613 if (flags == AV_HWFRAME_MAP_READ)
614 map_flags = kCVPixelBufferLock_ReadOnly;
615
616 err = CVPixelBufferLockBaseAddress(pixbuf, map_flags);
617 if (err != kCVReturnSuccess) {
618 av_log(ctx, AV_LOG_ERROR, "Error locking the pixel buffer.\n");
619 return AVERROR_UNKNOWN;
620 }
621
622 if (CVPixelBufferIsPlanar(pixbuf)) {
623 int planes = CVPixelBufferGetPlaneCount(pixbuf);
624 for (i = 0; i < planes; i++) {
625 dst->data[i] = CVPixelBufferGetBaseAddressOfPlane(pixbuf, i);
626 dst->linesize[i] = CVPixelBufferGetBytesPerRowOfPlane(pixbuf, i);
627 }
628 } else {
629 dst->data[0] = CVPixelBufferGetBaseAddress(pixbuf);
630 dst->linesize[0] = CVPixelBufferGetBytesPerRow(pixbuf);
631 }
632
633 ret = ff_hwframe_map_create(src->hw_frames_ctx, dst, src, vt_unmap,
634 (void *)(uintptr_t)map_flags);
635 if (ret < 0)
636 goto unlock;
637
638 return 0;
639
640 unlock:
641 CVPixelBufferUnlockBaseAddress(pixbuf, map_flags);
642 return ret;
643 }
644
vt_transfer_data_from(AVHWFramesContext * hwfc,AVFrame * dst,const AVFrame * src)645 static int vt_transfer_data_from(AVHWFramesContext *hwfc,
646 AVFrame *dst, const AVFrame *src)
647 {
648 AVFrame *map;
649 int err;
650
651 if (dst->width > hwfc->width || dst->height > hwfc->height)
652 return AVERROR(EINVAL);
653
654 map = av_frame_alloc();
655 if (!map)
656 return AVERROR(ENOMEM);
657 map->format = dst->format;
658
659 err = vt_map_frame(hwfc, map, src, AV_HWFRAME_MAP_READ);
660 if (err)
661 goto fail;
662
663 map->width = dst->width;
664 map->height = dst->height;
665
666 err = av_frame_copy(dst, map);
667 if (err)
668 goto fail;
669
670 err = 0;
671 fail:
672 av_frame_free(&map);
673 return err;
674 }
675
vt_transfer_data_to(AVHWFramesContext * hwfc,AVFrame * dst,const AVFrame * src)676 static int vt_transfer_data_to(AVHWFramesContext *hwfc,
677 AVFrame *dst, const AVFrame *src)
678 {
679 AVFrame *map;
680 int err;
681
682 if (src->width > hwfc->width || src->height > hwfc->height)
683 return AVERROR(EINVAL);
684
685 map = av_frame_alloc();
686 if (!map)
687 return AVERROR(ENOMEM);
688 map->format = src->format;
689
690 err = vt_map_frame(hwfc, map, dst, AV_HWFRAME_MAP_WRITE | AV_HWFRAME_MAP_OVERWRITE);
691 if (err)
692 goto fail;
693
694 map->width = src->width;
695 map->height = src->height;
696
697 err = av_frame_copy(map, src);
698 if (err)
699 goto fail;
700
701 err = vt_pixbuf_set_attachments(hwfc, (CVPixelBufferRef)dst->data[3], src);
702 if (err)
703 goto fail;
704
705 err = 0;
706 fail:
707 av_frame_free(&map);
708 return err;
709 }
710
vt_map_from(AVHWFramesContext * hwfc,AVFrame * dst,const AVFrame * src,int flags)711 static int vt_map_from(AVHWFramesContext *hwfc, AVFrame *dst,
712 const AVFrame *src, int flags)
713 {
714 int err;
715
716 if (dst->format == AV_PIX_FMT_NONE)
717 dst->format = hwfc->sw_format;
718 else if (dst->format != hwfc->sw_format)
719 return AVERROR(ENOSYS);
720
721 err = vt_map_frame(hwfc, dst, src, flags);
722 if (err)
723 return err;
724
725 dst->width = src->width;
726 dst->height = src->height;
727
728 err = av_frame_copy_props(dst, src);
729 if (err)
730 return err;
731
732 return 0;
733 }
734
vt_device_create(AVHWDeviceContext * ctx,const char * device,AVDictionary * opts,int flags)735 static int vt_device_create(AVHWDeviceContext *ctx, const char *device,
736 AVDictionary *opts, int flags)
737 {
738 if (device && device[0]) {
739 av_log(ctx, AV_LOG_ERROR, "Device selection unsupported.\n");
740 return AVERROR_UNKNOWN;
741 }
742
743 return 0;
744 }
745
746 const HWContextType ff_hwcontext_type_videotoolbox = {
747 .type = AV_HWDEVICE_TYPE_VIDEOTOOLBOX,
748 .name = "videotoolbox",
749
750 .frames_priv_size = sizeof(VTFramesContext),
751
752 .device_create = vt_device_create,
753 .frames_init = vt_frames_init,
754 .frames_get_buffer = vt_get_buffer,
755 .frames_get_constraints = vt_frames_get_constraints,
756 .frames_uninit = vt_frames_uninit,
757 .transfer_get_formats = vt_transfer_get_formats,
758 .transfer_data_to = vt_transfer_data_to,
759 .transfer_data_from = vt_transfer_data_from,
760 .map_from = vt_map_from,
761
762 .pix_fmts = (const enum AVPixelFormat[]){ AV_PIX_FMT_VIDEOTOOLBOX, AV_PIX_FMT_NONE },
763 };
764