1 #ifndef _EXTFFMPEGDEMUXER_H
2 #define _EXTFFMPEGDEMUXER_H
3 /*
4 * Copyright 2017-2018 NVIDIA Corporation. All rights reserved.
5 *
6 * Licensed under the Apache License, Version 2.0 (the "License");
7 * you may not use this file except in compliance with the License.
8 * You may obtain a copy of the License at
9 *
10 * http://www.apache.org/licenses/LICENSE-2.0
11 *
12 * Unless required by applicable law or agreed to in writing, software
13 * distributed under the License is distributed on an "AS IS" BASIS,
14 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 * See the License for the specific language governing permissions and
16 * limitations under the License.
17 */
18
19 #if ((DE_COMPILER == DE_COMPILER_CLANG || DE_COMPILER == DE_COMPILER_GCC))
20 # define DISABLE_DEPRECATIONS
21 #endif
22
23 #ifdef DISABLE_DEPRECATIONS
24 #pragma GCC diagnostic push
25 #pragma GCC diagnostic ignored "-Wdeprecated-declarations"
26 #pragma GCC diagnostic ignored "-Wconversion"
27 #endif
28
29 extern "C" {
30 #include <libavformat/avformat.h>
31 #include <libavformat/avio.h>
32 #include <libavcodec/avcodec.h>
33 }
34
35 #ifdef DISABLE_DEPRECATIONS
36 #pragma GCC diagnostic pop
37 #endif
38
39 #include <deDefs.h>
40
41 #include <iostream>
42 #include <tcuDefs.hpp>
43
44 #define LOG(X) std::ostream(0)
45
check(int e,int iLine,const char * szFile)46 inline bool check(int e, int iLine, const char *szFile) {
47 if (e < 0) {
48 LOG(ERROR) << "General error " << e << " at line " << iLine << " in file " << szFile;
49 return false;
50 }
51 return true;
52 }
53
54 #define ck(call) check(call, __LINE__, __FILE__)
55
56
57 typedef void* (*pFFMpeg_av_malloc)(size_t size);
58
59 typedef void (*pFFMpeg_av_freep)(void* ptr);
60
61 typedef void (*pFFMpeg_av_init_packet)(AVPacket* pkt);
62
63 typedef void (*pFFMpeg_av_packet_unref)(AVPacket* pkt);
64
65 typedef int (*pFFMpeg_av_bsf_init) (AVBSFContext* ctx);
66
67 typedef int (*pFFMpeg_av_bsf_send_packet)(AVBSFContext* ctx, AVPacket* pkt);
68
69 typedef int (*pFFMpeg_av_bsf_receive_packet)(AVBSFContext* ctx, AVPacket* pkt);
70
71 typedef const AVBitStreamFilter* (*pFFMpeg_av_bsf_get_by_name)(const char* name);
72
73 typedef int (*pFFMpeg_av_bsf_alloc)(const AVBitStreamFilter* filter, AVBSFContext** ctx);
74
75 typedef AVIOContext* (*pFFMpeg_avio_alloc_context) (unsigned char *buffer,
76 int buffer_size,
77 int write_flag,
78 void *opaque,
79 int (*read_packet)(void *opaque, uint8_t *buf, int buf_size),
80 int (*write_packet)(void *opaque, uint8_t *buf, int buf_size),
81 int64_t (*seek)(void *opaque, int64_t offset, int whence));
82
83 typedef int (*pFFMpeg_av_find_best_stream) (AVFormatContext *ic,
84 enum AVMediaType type,
85 int wanted_stream_nb,
86 int related_stream,
87 const AVCodec **decoder_ret,
88 int flags);
89
90 typedef int (*pFFMpeg_av_read_frame)(AVFormatContext *s, AVPacket *pkt);
91
92 typedef AVFormatContext* (*pFFMpeg_avformat_alloc_context)(void);
93
94 typedef int (*pFFMpeg_avformat_network_init)(void);
95
96 typedef int (*pFFMpeg_avformat_find_stream_info)(AVFormatContext *ic, AVDictionary **options);
97
98 typedef int (*pFFMpeg_avformat_open_input)(AVFormatContext **ps, const char *url, const AVInputFormat *fmt, AVDictionary **options);
99
100 typedef void (*pFFMpeg_avformat_close_input)(AVFormatContext **s);
101
102 typedef struct
103 {
104 pFFMpeg_av_malloc av_malloc;
105 pFFMpeg_av_freep av_freep;
106 pFFMpeg_av_init_packet av_init_packet;
107 pFFMpeg_av_packet_unref av_packet_unref;
108 pFFMpeg_av_bsf_init av_bsf_init;
109 pFFMpeg_av_bsf_send_packet av_bsf_send_packet;
110 pFFMpeg_av_bsf_receive_packet av_bsf_receive_packet;
111 pFFMpeg_av_bsf_get_by_name av_bsf_get_by_name;
112 pFFMpeg_av_bsf_alloc av_bsf_alloc;
113 pFFMpeg_avio_alloc_context avio_alloc_context;
114 pFFMpeg_av_find_best_stream av_find_best_stream;
115 pFFMpeg_av_read_frame av_read_frame;
116 pFFMpeg_avformat_alloc_context avformat_alloc_context;
117 pFFMpeg_avformat_network_init avformat_network_init;
118 pFFMpeg_avformat_find_stream_info avformat_find_stream_info;
119 pFFMpeg_avformat_open_input avformat_open_input;
120 pFFMpeg_avformat_close_input avformat_close_input;
121 } FFMpegAPI;
122
123
124 class FFmpegDemuxer {
125 private:
126 AVFormatContext *fmtc = NULL;
127 FFMpegAPI* api = NULL;
128 AVIOContext *avioc = NULL;
129 AVPacket pkt, pktFiltered;
130 AVBSFContext *bsfc = NULL;
131
132 int iVideoStream;
133 bool bMp4;
134 AVCodecID eVideoCodec;
135 int nWidth, nHeight, nBitDepth;
136
137 AVPixelFormat format;
138 /**
139 * Codec-specific bitstream restrictions that the stream conforms to.
140 */
141 int profile;
142 int level;
143
144 /**
145 * Video only. The aspect ratio (width / height) which a single pixel
146 * should have when displayed.
147 *
148 * When the aspect ratio is unknown / undefined, the numerator should be
149 * set to 0 (the denominator may have any value).
150 */
151 AVRational sample_aspect_ratio;
152
153 /**
154 * Video only. The order of the fields in interlaced video.
155 */
156 enum AVFieldOrder field_order;
157
158 /**
159 * Video only. Additional colorspace characteristics.
160 */
161 enum AVColorRange color_range;
162 enum AVColorPrimaries color_primaries;
163 enum AVColorTransferCharacteristic color_trc;
164 enum AVColorSpace color_space;
165 enum AVChromaLocation chroma_location;
166
167 public:
168 class DataProvider {
169 public:
~DataProvider()170 virtual ~DataProvider() {}
171 virtual int GetData(uint8_t *pBuf, int nBuf) = 0;
172 };
173
174 private:
FFmpegDemuxer(AVFormatContext * fmtc_,FFMpegAPI * api_)175 FFmpegDemuxer(AVFormatContext *fmtc_, FFMpegAPI* api_) : fmtc(fmtc_), api(api_) {
176 if (!fmtc) {
177 LOG(ERROR) << "No AVFormatContext provided.";
178 return;
179 }
180
181 LOG(INFO) << "Media format: " << fmtc->iformat->long_name << " (" << fmtc->iformat->name << ")";
182
183 ck(api->avformat_find_stream_info(fmtc, NULL));
184 iVideoStream = api->av_find_best_stream(fmtc, AVMEDIA_TYPE_VIDEO, -1, -1, NULL, 0);
185 if (iVideoStream < 0) {
186 LOG(ERROR) << "FFmpeg error: " << __FILE__ << " " << __LINE__ << " " << "Could not find stream in input file";
187 return;
188 }
189
190 //fmtc->streams[iVideoStream]->need_parsing = AVSTREAM_PARSE_NONE;
191 eVideoCodec = fmtc->streams[iVideoStream]->codecpar->codec_id;
192 nWidth = fmtc->streams[iVideoStream]->codecpar->width;
193 nHeight = fmtc->streams[iVideoStream]->codecpar->height;
194 format = (AVPixelFormat)fmtc->streams[iVideoStream]->codecpar->format;
195 nBitDepth = 8;
196 if (fmtc->streams[iVideoStream]->codecpar->format == AV_PIX_FMT_YUV420P10LE)
197 nBitDepth = 10;
198 if (fmtc->streams[iVideoStream]->codecpar->format == AV_PIX_FMT_YUV420P12LE)
199 nBitDepth = 12;
200
201 bMp4 = (!strcmp(fmtc->iformat->long_name, "QuickTime / MOV") ||
202 !strcmp(fmtc->iformat->long_name, "FLV (Flash Video)") ||
203 !strcmp(fmtc->iformat->long_name, "Matroska / WebM"));
204
205 /**
206 * Codec-specific bitstream restrictions that the stream conforms to.
207 */
208 profile = fmtc->streams[iVideoStream]->codecpar->profile;
209 level = fmtc->streams[iVideoStream]->codecpar->level;
210
211 /**
212 * Video only. The aspect ratio (width / height) which a single pixel
213 * should have when displayed.
214 *
215 * When the aspect ratio is unknown / undefined, the numerator should be
216 * set to 0 (the denominator may have any value).
217 */
218 sample_aspect_ratio = fmtc->streams[iVideoStream]->codecpar->sample_aspect_ratio;
219
220 /**
221 * Video only. The order of the fields in interlaced video.
222 */
223 field_order = fmtc->streams[iVideoStream]->codecpar->field_order;
224
225 /**
226 * Video only. Additional colorspace characteristics.
227 */
228 color_range = fmtc->streams[iVideoStream]->codecpar->color_range;
229 color_primaries = fmtc->streams[iVideoStream]->codecpar->color_primaries;
230 color_trc = fmtc->streams[iVideoStream]->codecpar->color_trc;
231 color_space = fmtc->streams[iVideoStream]->codecpar->color_space;
232 chroma_location = fmtc->streams[iVideoStream]->codecpar->chroma_location;
233
234 api->av_init_packet(&pkt);
235 pkt.data = NULL;
236 pkt.size = 0;
237 api->av_init_packet(&pktFiltered);
238 pktFiltered.data = NULL;
239 pktFiltered.size = 0;
240
241 if (bMp4) {
242 const AVBitStreamFilter *bsf = NULL;
243
244 if (eVideoCodec == AV_CODEC_ID_H264) {
245 bsf = api->av_bsf_get_by_name("h264_mp4toannexb");
246 } else if (eVideoCodec == AV_CODEC_ID_HEVC) {
247 bsf = api->av_bsf_get_by_name("hevc_mp4toannexb");
248 }
249
250 if (!bsf) {
251 LOG(ERROR) << "FFmpeg error: " << __FILE__ << " " << __LINE__ << " " << "av_bsf_get_by_name(): " << eVideoCodec << " failed";
252 return;
253 }
254 ck(api->av_bsf_alloc(bsf, &bsfc));
255 bsfc->par_in = fmtc->streams[iVideoStream]->codecpar;
256 ck(api->av_bsf_init(bsfc));
257 }
258 }
259
CreateFormatContext(DataProvider * pDataProvider,FFMpegAPI * api_)260 AVFormatContext *CreateFormatContext(DataProvider *pDataProvider, FFMpegAPI* api_) {
261 api = api_;
262
263 AVFormatContext *ctx = NULL;
264 if (!(ctx = api->avformat_alloc_context())) {
265 LOG(ERROR) << "FFmpeg error: " << __FILE__ << " " << __LINE__;
266 return NULL;
267 }
268
269 uint8_t *avioc_buffer = NULL;
270 int avioc_buffer_size = 8 * 1024 * 1024;
271 avioc_buffer = (uint8_t *)api->av_malloc(avioc_buffer_size);
272 if (!avioc_buffer) {
273 LOG(ERROR) << "FFmpeg error: " << __FILE__ << " " << __LINE__;
274 return NULL;
275 }
276 avioc = api->avio_alloc_context(avioc_buffer, avioc_buffer_size,
277 0, pDataProvider, &ReadPacket, NULL, NULL);
278 if (!avioc) {
279 LOG(ERROR) << "FFmpeg error: " << __FILE__ << " " << __LINE__;
280 return NULL;
281 }
282 ctx->pb = avioc;
283
284 ck(api->avformat_open_input(&ctx, NULL, NULL, NULL));
285 return ctx;
286 }
287
CreateFormatContext(const char * szFilePath,FFMpegAPI * api_)288 AVFormatContext *CreateFormatContext(const char *szFilePath, FFMpegAPI* api_) {
289 api = api_;
290
291 api->avformat_network_init();
292
293 AVFormatContext *ctx = NULL;
294 ck(api->avformat_open_input(&ctx, szFilePath, NULL, NULL));
295 return ctx;
296 }
297
298 public:
FFmpegDemuxer(const char * szFilePath,FFMpegAPI * api_)299 FFmpegDemuxer(const char *szFilePath, FFMpegAPI* api_) : FFmpegDemuxer(CreateFormatContext(szFilePath, api_), api_) {}
FFmpegDemuxer(DataProvider * pDataProvider,FFMpegAPI * api_)300 FFmpegDemuxer(DataProvider *pDataProvider, FFMpegAPI* api_) : FFmpegDemuxer(CreateFormatContext(pDataProvider, api_), api_) {}
~FFmpegDemuxer()301 ~FFmpegDemuxer() {
302 if (pkt.data) {
303 api->av_packet_unref(&pkt);
304 }
305 if (pktFiltered.data) {
306 api->av_packet_unref(&pktFiltered);
307 }
308
309 api->avformat_close_input(&fmtc);
310 if (avioc) {
311 api->av_freep(&avioc->buffer);
312 api->av_freep(&avioc);
313 }
314 }
GetVideoCodec()315 AVCodecID GetVideoCodec() {
316 return eVideoCodec;
317 }
GetWidth()318 int GetWidth() {
319 return nWidth;
320 }
GetHeight()321 int GetHeight() {
322 return nHeight;
323 }
GetBitDepth()324 int GetBitDepth() {
325 return nBitDepth;
326 }
GetFrameSize()327 int GetFrameSize() {
328 return nBitDepth == 8 ? nWidth * nHeight * 3 / 2: nWidth * nHeight * 3;
329 }
Demux(uint8_t ** ppVideo,int * pnVideoBytes)330 bool Demux(uint8_t **ppVideo, int *pnVideoBytes) {
331 if (!fmtc) {
332 return false;
333 }
334
335 *pnVideoBytes = 0;
336
337 if (pkt.data) {
338 api->av_packet_unref(&pkt);
339 }
340
341 int e = 0;
342 while ((e = api->av_read_frame(fmtc, &pkt)) >= 0 && pkt.stream_index != iVideoStream) {
343 api->av_packet_unref(&pkt);
344 }
345 if (e < 0) {
346 return false;
347 }
348
349 if (bMp4) {
350 if (pktFiltered.data) {
351 api->av_packet_unref(&pktFiltered);
352 }
353 ck(api->av_bsf_send_packet(bsfc, &pkt));
354 ck(api->av_bsf_receive_packet(bsfc, &pktFiltered));
355 *ppVideo = pktFiltered.data;
356 *pnVideoBytes = pktFiltered.size;
357 } else {
358 *ppVideo = pkt.data;
359 *pnVideoBytes = pkt.size;
360 }
361
362 return true;
363 }
364
ReadPacket(void * opaque,uint8_t * pBuf,int nBuf)365 static int ReadPacket(void *opaque, uint8_t *pBuf, int nBuf) {
366 return ((DataProvider *)opaque)->GetData(pBuf, nBuf);
367 }
368
369
DumpStreamParameters()370 void DumpStreamParameters() {
371
372 LOG(INFO) << "Width: " << nWidth << std::endl;
373 LOG(INFO) << "Height: " << nHeight << std::endl;
374 LOG(INFO) << "BitDepth: " << nBitDepth << std::endl;
375 LOG(INFO) << "Profile: " << profile << std::endl;
376 LOG(INFO) << "Level: " << level << std::endl;
377 LOG(INFO) << "Aspect Ration: " << (float)sample_aspect_ratio.num / (float)sample_aspect_ratio.den << std::endl;
378
379 static const char* FieldOrder[] = {
380 "UNKNOWN",
381 "PROGRESSIVE",
382 "TT: Top coded_first, top displayed first",
383 "BB: Bottom coded first, bottom displayed first",
384 "TB: Top coded first, bottom displayed first",
385 "BT: Bottom coded first, top displayed first",
386 };
387 LOG(INFO) << "Field Order: " << FieldOrder[field_order] << std::endl;
388
389 static const char* ColorRange[] = {
390 "UNSPECIFIED",
391 "MPEG: the normal 219*2^(n-8) MPEG YUV ranges",
392 "JPEG: the normal 2^n-1 JPEG YUV ranges",
393 "NB: Not part of ABI",
394 };
395 LOG(INFO) << "Color Range: " << ColorRange[color_range] << std::endl;
396
397 static const char* ColorPrimaries[] = {
398 "RESERVED0",
399 "BT709: also ITU-R BT1361 / IEC 61966-2-4 / SMPTE RP177 Annex B",
400 "UNSPECIFIED",
401 "RESERVED",
402 "BT470M: also FCC Title 47 Code of Federal Regulations 73.682 (a)(20)",
403
404 "BT470BG: also ITU-R BT601-6 625 / ITU-R BT1358 625 / ITU-R BT1700 625 PAL & SECAM",
405 "SMPTE170M: also ITU-R BT601-6 525 / ITU-R BT1358 525 / ITU-R BT1700 NTSC",
406 "SMPTE240M: also ITU-R BT601-6 525 / ITU-R BT1358 525 / ITU-R BT1700 NTSC",
407 "FILM: colour filters using Illuminant C",
408 "BT2020: ITU-R BT2020",
409 "SMPTE428: SMPTE ST 428-1 (CIE 1931 XYZ)",
410 "SMPTE431: SMPTE ST 431-2 (2011) / DCI P3",
411 "SMPTE432: SMPTE ST 432-1 (2010) / P3 D65 / Display P3",
412 "JEDEC_P22: JEDEC P22 phosphors",
413 "NB: Not part of ABI",
414 };
415 LOG(INFO) << "Color Primaries: " << ColorPrimaries[color_primaries] << std::endl;
416
417 static const char* ColorTransferCharacteristic[] = {
418 "RESERVED0",
419 "BT709: also ITU-R BT1361",
420 "UNSPECIFIED",
421 "RESERVED",
422 "GAMMA22: also ITU-R BT470M / ITU-R BT1700 625 PAL & SECAM",
423 "GAMMA28: also ITU-R BT470BG",
424 "SMPTE170M: also ITU-R BT601-6 525 or 625 / ITU-R BT1358 525 or 625 / ITU-R BT1700 NTSC",
425 "SMPTE240M",
426 "LINEAR: Linear transfer characteristics",
427 "LOG: Logarithmic transfer characteristic (100:1 range)",
428 "LOG_SQRT: Logarithmic transfer characteristic (100 * Sqrt(10) : 1 range)",
429 "IEC61966_2_4: IEC 61966-2-4",
430 "BT1361_ECG: ITU-R BT1361 Extended Colour Gamut",
431 "IEC61966_2_1: IEC 61966-2-1 (sRGB or sYCC)",
432 "BT2020_10: ITU-R BT2020 for 10-bit system",
433 "BT2020_12: ITU-R BT2020 for 12-bit system",
434 "SMPTE2084: SMPTE ST 2084 for 10-, 12-, 14- and 16-bit systems",
435 "SMPTE428: SMPTE ST 428-1",
436 "ARIB_STD_B67: ARIB STD-B67, known as Hybrid log-gamma",
437 "NB: Not part of ABI",
438 };
439 LOG(INFO) << "Color Transfer Characteristic: " << ColorTransferCharacteristic[color_trc] << std::endl;
440
441 static const char* ColorSpace[] = {
442 "RGB: order of coefficients is actually GBR, also IEC 61966-2-1 (sRGB)",
443 "BT709: also ITU-R BT1361 / IEC 61966-2-4 xvYCC709 / SMPTE RP177 Annex B",
444 "UNSPECIFIED",
445 "RESERVED",
446 "FCC: FCC Title 47 Code of Federal Regulations 73.682 (a)(20)",
447 "BT470BG: also ITU-R BT601-6 625 / ITU-R BT1358 625 / ITU-R BT1700 625 PAL & SECAM / IEC 61966-2-4 xvYCC601",
448 "SMPTE170M: also ITU-R BT601-6 525 / ITU-R BT1358 525 / ITU-R BT1700 NTSC",
449 "SMPTE240M: functionally identical to above",
450 "YCGCO: Used by Dirac / VC-2 and H.264 FRext, see ITU-T SG16",
451 "BT2020_NCL: ITU-R BT2020 non-constant luminance system",
452 "BT2020_CL: ITU-R BT2020 constant luminance system",
453 "SMPTE2085: SMPTE 2085, Y'D'zD'x",
454 "CHROMA_DERIVED_NCL: Chromaticity-derived non-constant luminance system",
455 "CHROMA_DERIVED_CL: Chromaticity-derived constant luminance system",
456 "ICTCP: ITU-R BT.2100-0, ICtCp",
457 "NB: Not part of ABI",
458 };
459 LOG(INFO) << "Color Space: " << ColorSpace[color_space] << std::endl;
460
461 static const char* ChromaLocation[] = {
462 "UNSPECIFIED",
463 "LEFT: MPEG-2/4 4:2:0, H.264 default for 4:2:0",
464 "CENTER: MPEG-1 4:2:0, JPEG 4:2:0, H.263 4:2:0",
465 "TOPLEFT: ITU-R 601, SMPTE 274M 296M S314M(DV 4:1:1), mpeg2 4:2:2",
466 "TOP",
467 "BOTTOMLEFT",
468 "BOTTOM",
469 "NB:Not part of ABI",
470 };
471 LOG(INFO) << "Chroma Location: " << ChromaLocation[chroma_location] << std::endl;
472 }
473
474 };
475
FFmpeg2NvCodecId(AVCodecID id)476 inline vk::VkVideoCodecOperationFlagBitsKHR FFmpeg2NvCodecId(AVCodecID id) {
477 switch (id) {
478 case AV_CODEC_ID_MPEG1VIDEO : /* assert(false); */ return vk::VkVideoCodecOperationFlagBitsKHR(0);
479 case AV_CODEC_ID_MPEG2VIDEO : /* assert(false); */ return vk::VkVideoCodecOperationFlagBitsKHR(0);
480 case AV_CODEC_ID_MPEG4 : /* assert(false); */ return vk::VkVideoCodecOperationFlagBitsKHR(0);
481 case AV_CODEC_ID_VC1 : /* assert(false); */ return vk::VkVideoCodecOperationFlagBitsKHR(0);
482 case AV_CODEC_ID_H264 : return vk::VK_VIDEO_CODEC_OPERATION_DECODE_H264_BIT_KHR;
483 case AV_CODEC_ID_HEVC : return vk::VK_VIDEO_CODEC_OPERATION_DECODE_H265_BIT_KHR;
484 case AV_CODEC_ID_VP8 : /* assert(false); */ return vk::VkVideoCodecOperationFlagBitsKHR(0);
485 #ifdef VK_EXT_video_decode_vp9
486 case AV_CODEC_ID_VP9 : return VK_VIDEO_CODEC_OPERATION_DECODE_VP9_BIT_KHR;
487 #endif // VK_EXT_video_decode_vp9
488 case AV_CODEC_ID_MJPEG : /* assert(false); */ return vk::VkVideoCodecOperationFlagBitsKHR(0);
489 default : /* assert(false); */ return vk::VkVideoCodecOperationFlagBitsKHR(0);
490 }
491 }
492 #endif /* _EXTFFMPEGDEMUXER_H */
493