1 /*
2 * Android MediaCodec MPEG-2 / H.264 / H.265 / MPEG-4 / VP8 / VP9 decoders
3 *
4 * Copyright (c) 2015-2016 Matthieu Bouron <matthieu.bouron stupeflix.com>
5 *
6 * This file is part of FFmpeg.
7 *
8 * FFmpeg is free software; you can redistribute it and/or
9 * modify it under the terms of the GNU Lesser General Public
10 * License as published by the Free Software Foundation; either
11 * version 2.1 of the License, or (at your option) any later version.
12 *
13 * FFmpeg is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16 * Lesser General Public License for more details.
17 *
18 * You should have received a copy of the GNU Lesser General Public
19 * License along with FFmpeg; if not, write to the Free Software
20 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
21 */
22
23 #include "config_components.h"
24
25 #include <stdint.h>
26 #include <string.h>
27
28 #include "libavutil/avassert.h"
29 #include "libavutil/common.h"
30 #include "libavutil/opt.h"
31 #include "libavutil/intreadwrite.h"
32 #include "libavutil/pixfmt.h"
33 #include "libavutil/internal.h"
34
35 #include "avcodec.h"
36 #include "codec_internal.h"
37 #include "decode.h"
38 #include "h264_parse.h"
39 #include "h264_ps.h"
40 #include "hevc_parse.h"
41 #include "hwconfig.h"
42 #include "internal.h"
43 #include "mediacodec_wrapper.h"
44 #include "mediacodecdec_common.h"
45
46 typedef struct MediaCodecH264DecContext {
47
48 AVClass *avclass;
49
50 MediaCodecDecContext *ctx;
51
52 AVPacket buffered_pkt;
53
54 int delay_flush;
55 int amlogic_mpeg2_api23_workaround;
56
57 } MediaCodecH264DecContext;
58
mediacodec_decode_close(AVCodecContext * avctx)59 static av_cold int mediacodec_decode_close(AVCodecContext *avctx)
60 {
61 MediaCodecH264DecContext *s = avctx->priv_data;
62
63 ff_mediacodec_dec_close(avctx, s->ctx);
64 s->ctx = NULL;
65
66 av_packet_unref(&s->buffered_pkt);
67
68 return 0;
69 }
70
71 #if CONFIG_H264_MEDIACODEC_DECODER || CONFIG_HEVC_MEDIACODEC_DECODER
h2645_ps_to_nalu(const uint8_t * src,int src_size,uint8_t ** out,int * out_size)72 static int h2645_ps_to_nalu(const uint8_t *src, int src_size, uint8_t **out, int *out_size)
73 {
74 int i;
75 int ret = 0;
76 uint8_t *p = NULL;
77 static const uint8_t nalu_header[] = { 0x00, 0x00, 0x00, 0x01 };
78
79 if (!out || !out_size) {
80 return AVERROR(EINVAL);
81 }
82
83 p = av_malloc(sizeof(nalu_header) + src_size);
84 if (!p) {
85 return AVERROR(ENOMEM);
86 }
87
88 *out = p;
89 *out_size = sizeof(nalu_header) + src_size;
90
91 memcpy(p, nalu_header, sizeof(nalu_header));
92 memcpy(p + sizeof(nalu_header), src, src_size);
93
94 /* Escape 0x00, 0x00, 0x0{0-3} pattern */
95 for (i = 4; i < *out_size; i++) {
96 if (i < *out_size - 3 &&
97 p[i + 0] == 0 &&
98 p[i + 1] == 0 &&
99 p[i + 2] <= 3) {
100 uint8_t *new;
101
102 *out_size += 1;
103 new = av_realloc(*out, *out_size);
104 if (!new) {
105 ret = AVERROR(ENOMEM);
106 goto done;
107 }
108 *out = p = new;
109
110 i = i + 2;
111 memmove(p + i + 1, p + i, *out_size - (i + 1));
112 p[i] = 0x03;
113 }
114 }
115 done:
116 if (ret < 0) {
117 av_freep(out);
118 *out_size = 0;
119 }
120
121 return ret;
122 }
123 #endif
124
125 #if CONFIG_H264_MEDIACODEC_DECODER
h264_set_extradata(AVCodecContext * avctx,FFAMediaFormat * format)126 static int h264_set_extradata(AVCodecContext *avctx, FFAMediaFormat *format)
127 {
128 int i;
129 int ret;
130
131 H264ParamSets ps;
132 const PPS *pps = NULL;
133 const SPS *sps = NULL;
134 int is_avc = 0;
135 int nal_length_size = 0;
136
137 memset(&ps, 0, sizeof(ps));
138
139 ret = ff_h264_decode_extradata(avctx->extradata, avctx->extradata_size,
140 &ps, &is_avc, &nal_length_size, 0, avctx);
141 if (ret < 0) {
142 goto done;
143 }
144
145 for (i = 0; i < MAX_PPS_COUNT; i++) {
146 if (ps.pps_list[i]) {
147 pps = (const PPS*)ps.pps_list[i]->data;
148 break;
149 }
150 }
151
152 if (pps) {
153 if (ps.sps_list[pps->sps_id]) {
154 sps = (const SPS*)ps.sps_list[pps->sps_id]->data;
155 }
156 }
157
158 if (pps && sps) {
159 uint8_t *data = NULL;
160 int data_size = 0;
161
162 avctx->profile = ff_h264_get_profile(sps);
163 avctx->level = sps->level_idc;
164
165 if ((ret = h2645_ps_to_nalu(sps->data, sps->data_size, &data, &data_size)) < 0) {
166 goto done;
167 }
168 ff_AMediaFormat_setBuffer(format, "csd-0", (void*)data, data_size);
169 av_freep(&data);
170
171 if ((ret = h2645_ps_to_nalu(pps->data, pps->data_size, &data, &data_size)) < 0) {
172 goto done;
173 }
174 ff_AMediaFormat_setBuffer(format, "csd-1", (void*)data, data_size);
175 av_freep(&data);
176 } else {
177 const int warn = is_avc && (avctx->codec_tag == MKTAG('a','v','c','1') ||
178 avctx->codec_tag == MKTAG('a','v','c','2'));
179 av_log(avctx, warn ? AV_LOG_WARNING : AV_LOG_DEBUG,
180 "Could not extract PPS/SPS from extradata\n");
181 ret = 0;
182 }
183
184 done:
185 ff_h264_ps_uninit(&ps);
186
187 return ret;
188 }
189 #endif
190
191 #if CONFIG_HEVC_MEDIACODEC_DECODER
hevc_set_extradata(AVCodecContext * avctx,FFAMediaFormat * format)192 static int hevc_set_extradata(AVCodecContext *avctx, FFAMediaFormat *format)
193 {
194 int i;
195 int ret;
196
197 HEVCParamSets ps;
198 HEVCSEI sei;
199
200 const HEVCVPS *vps = NULL;
201 const HEVCPPS *pps = NULL;
202 const HEVCSPS *sps = NULL;
203 int is_nalff = 0;
204 int nal_length_size = 0;
205
206 uint8_t *vps_data = NULL;
207 uint8_t *sps_data = NULL;
208 uint8_t *pps_data = NULL;
209 int vps_data_size = 0;
210 int sps_data_size = 0;
211 int pps_data_size = 0;
212
213 memset(&ps, 0, sizeof(ps));
214 memset(&sei, 0, sizeof(sei));
215
216 ret = ff_hevc_decode_extradata(avctx->extradata, avctx->extradata_size,
217 &ps, &sei, &is_nalff, &nal_length_size, 0, 1, avctx);
218 if (ret < 0) {
219 goto done;
220 }
221
222 for (i = 0; i < HEVC_MAX_VPS_COUNT; i++) {
223 if (ps.vps_list[i]) {
224 vps = (const HEVCVPS*)ps.vps_list[i]->data;
225 break;
226 }
227 }
228
229 for (i = 0; i < HEVC_MAX_PPS_COUNT; i++) {
230 if (ps.pps_list[i]) {
231 pps = (const HEVCPPS*)ps.pps_list[i]->data;
232 break;
233 }
234 }
235
236 if (pps) {
237 if (ps.sps_list[pps->sps_id]) {
238 sps = (const HEVCSPS*)ps.sps_list[pps->sps_id]->data;
239 }
240 }
241
242 if (vps && pps && sps) {
243 uint8_t *data;
244 int data_size;
245
246 avctx->profile = sps->ptl.general_ptl.profile_idc;
247 avctx->level = sps->ptl.general_ptl.level_idc;
248
249 if ((ret = h2645_ps_to_nalu(vps->data, vps->data_size, &vps_data, &vps_data_size)) < 0 ||
250 (ret = h2645_ps_to_nalu(sps->data, sps->data_size, &sps_data, &sps_data_size)) < 0 ||
251 (ret = h2645_ps_to_nalu(pps->data, pps->data_size, &pps_data, &pps_data_size)) < 0) {
252 goto done;
253 }
254
255 data_size = vps_data_size + sps_data_size + pps_data_size;
256 data = av_mallocz(data_size);
257 if (!data) {
258 ret = AVERROR(ENOMEM);
259 goto done;
260 }
261
262 memcpy(data , vps_data, vps_data_size);
263 memcpy(data + vps_data_size , sps_data, sps_data_size);
264 memcpy(data + vps_data_size + sps_data_size, pps_data, pps_data_size);
265
266 ff_AMediaFormat_setBuffer(format, "csd-0", data, data_size);
267
268 av_freep(&data);
269 } else {
270 const int warn = is_nalff && avctx->codec_tag == MKTAG('h','v','c','1');
271 av_log(avctx, warn ? AV_LOG_WARNING : AV_LOG_DEBUG,
272 "Could not extract VPS/PPS/SPS from extradata\n");
273 ret = 0;
274 }
275
276 done:
277 ff_hevc_ps_uninit(&ps);
278
279 av_freep(&vps_data);
280 av_freep(&sps_data);
281 av_freep(&pps_data);
282
283 return ret;
284 }
285 #endif
286
287 #if CONFIG_MPEG2_MEDIACODEC_DECODER || \
288 CONFIG_MPEG4_MEDIACODEC_DECODER || \
289 CONFIG_VP8_MEDIACODEC_DECODER || \
290 CONFIG_VP9_MEDIACODEC_DECODER
common_set_extradata(AVCodecContext * avctx,FFAMediaFormat * format)291 static int common_set_extradata(AVCodecContext *avctx, FFAMediaFormat *format)
292 {
293 int ret = 0;
294
295 if (avctx->extradata) {
296 ff_AMediaFormat_setBuffer(format, "csd-0", avctx->extradata, avctx->extradata_size);
297 }
298
299 return ret;
300 }
301 #endif
302
mediacodec_decode_init(AVCodecContext * avctx)303 static av_cold int mediacodec_decode_init(AVCodecContext *avctx)
304 {
305 int ret;
306 int sdk_int;
307
308 const char *codec_mime = NULL;
309
310 FFAMediaFormat *format = NULL;
311 MediaCodecH264DecContext *s = avctx->priv_data;
312
313 format = ff_AMediaFormat_new();
314 if (!format) {
315 av_log(avctx, AV_LOG_ERROR, "Failed to create media format\n");
316 ret = AVERROR_EXTERNAL;
317 goto done;
318 }
319
320 switch (avctx->codec_id) {
321 #if CONFIG_H264_MEDIACODEC_DECODER
322 case AV_CODEC_ID_H264:
323 codec_mime = "video/avc";
324
325 ret = h264_set_extradata(avctx, format);
326 if (ret < 0)
327 goto done;
328 break;
329 #endif
330 #if CONFIG_HEVC_MEDIACODEC_DECODER
331 case AV_CODEC_ID_HEVC:
332 codec_mime = "video/hevc";
333
334 ret = hevc_set_extradata(avctx, format);
335 if (ret < 0)
336 goto done;
337 break;
338 #endif
339 #if CONFIG_MPEG2_MEDIACODEC_DECODER
340 case AV_CODEC_ID_MPEG2VIDEO:
341 codec_mime = "video/mpeg2";
342
343 ret = common_set_extradata(avctx, format);
344 if (ret < 0)
345 goto done;
346 break;
347 #endif
348 #if CONFIG_MPEG4_MEDIACODEC_DECODER
349 case AV_CODEC_ID_MPEG4:
350 codec_mime = "video/mp4v-es",
351
352 ret = common_set_extradata(avctx, format);
353 if (ret < 0)
354 goto done;
355 break;
356 #endif
357 #if CONFIG_VP8_MEDIACODEC_DECODER
358 case AV_CODEC_ID_VP8:
359 codec_mime = "video/x-vnd.on2.vp8";
360
361 ret = common_set_extradata(avctx, format);
362 if (ret < 0)
363 goto done;
364 break;
365 #endif
366 #if CONFIG_VP9_MEDIACODEC_DECODER
367 case AV_CODEC_ID_VP9:
368 codec_mime = "video/x-vnd.on2.vp9";
369
370 ret = common_set_extradata(avctx, format);
371 if (ret < 0)
372 goto done;
373 break;
374 #endif
375 default:
376 av_assert0(0);
377 }
378
379 ff_AMediaFormat_setString(format, "mime", codec_mime);
380 ff_AMediaFormat_setInt32(format, "width", avctx->width);
381 ff_AMediaFormat_setInt32(format, "height", avctx->height);
382
383 s->ctx = av_mallocz(sizeof(*s->ctx));
384 if (!s->ctx) {
385 av_log(avctx, AV_LOG_ERROR, "Failed to allocate MediaCodecDecContext\n");
386 ret = AVERROR(ENOMEM);
387 goto done;
388 }
389
390 s->ctx->delay_flush = s->delay_flush;
391
392 if ((ret = ff_mediacodec_dec_init(avctx, s->ctx, codec_mime, format)) < 0) {
393 s->ctx = NULL;
394 goto done;
395 }
396
397 av_log(avctx, AV_LOG_INFO,
398 "MediaCodec started successfully: codec = %s, ret = %d\n",
399 s->ctx->codec_name, ret);
400
401 sdk_int = ff_Build_SDK_INT(avctx);
402 if (sdk_int <= 23 &&
403 strcmp(s->ctx->codec_name, "OMX.amlogic.mpeg2.decoder.awesome") == 0) {
404 av_log(avctx, AV_LOG_INFO, "Enabling workaround for %s on API=%d\n",
405 s->ctx->codec_name, sdk_int);
406 s->amlogic_mpeg2_api23_workaround = 1;
407 }
408
409 done:
410 if (format) {
411 ff_AMediaFormat_delete(format);
412 }
413
414 if (ret < 0) {
415 mediacodec_decode_close(avctx);
416 }
417
418 return ret;
419 }
420
mediacodec_receive_frame(AVCodecContext * avctx,AVFrame * frame)421 static int mediacodec_receive_frame(AVCodecContext *avctx, AVFrame *frame)
422 {
423 MediaCodecH264DecContext *s = avctx->priv_data;
424 int ret;
425 ssize_t index;
426
427 /* In delay_flush mode, wait until the user has released or rendered
428 all retained frames. */
429 if (s->delay_flush && ff_mediacodec_dec_is_flushing(avctx, s->ctx)) {
430 if (!ff_mediacodec_dec_flush(avctx, s->ctx)) {
431 return AVERROR(EAGAIN);
432 }
433 }
434
435 /* poll for new frame */
436 ret = ff_mediacodec_dec_receive(avctx, s->ctx, frame, false);
437 if (ret != AVERROR(EAGAIN))
438 return ret;
439
440 /* feed decoder */
441 while (1) {
442 if (s->ctx->current_input_buffer < 0) {
443 /* poll for input space */
444 index = ff_AMediaCodec_dequeueInputBuffer(s->ctx->codec, 0);
445 if (index < 0) {
446 /* no space, block for an output frame to appear */
447 return ff_mediacodec_dec_receive(avctx, s->ctx, frame, true);
448 }
449 s->ctx->current_input_buffer = index;
450 }
451
452 /* try to flush any buffered packet data */
453 if (s->buffered_pkt.size > 0) {
454 ret = ff_mediacodec_dec_send(avctx, s->ctx, &s->buffered_pkt, false);
455 if (ret >= 0) {
456 s->buffered_pkt.size -= ret;
457 s->buffered_pkt.data += ret;
458 if (s->buffered_pkt.size <= 0) {
459 av_packet_unref(&s->buffered_pkt);
460 } else {
461 av_log(avctx, AV_LOG_WARNING,
462 "could not send entire packet in single input buffer (%d < %d)\n",
463 ret, s->buffered_pkt.size+ret);
464 }
465 } else if (ret < 0 && ret != AVERROR(EAGAIN)) {
466 return ret;
467 }
468
469 if (s->amlogic_mpeg2_api23_workaround && s->buffered_pkt.size <= 0) {
470 /* fallthrough to fetch next packet regardless of input buffer space */
471 } else {
472 /* poll for space again */
473 continue;
474 }
475 }
476
477 /* fetch new packet or eof */
478 ret = ff_decode_get_packet(avctx, &s->buffered_pkt);
479 if (ret == AVERROR_EOF) {
480 AVPacket null_pkt = { 0 };
481 ret = ff_mediacodec_dec_send(avctx, s->ctx, &null_pkt, true);
482 if (ret < 0)
483 return ret;
484 return ff_mediacodec_dec_receive(avctx, s->ctx, frame, true);
485 } else if (ret == AVERROR(EAGAIN) && s->ctx->current_input_buffer < 0) {
486 return ff_mediacodec_dec_receive(avctx, s->ctx, frame, true);
487 } else if (ret < 0) {
488 return ret;
489 }
490 }
491
492 return AVERROR(EAGAIN);
493 }
494
mediacodec_decode_flush(AVCodecContext * avctx)495 static void mediacodec_decode_flush(AVCodecContext *avctx)
496 {
497 MediaCodecH264DecContext *s = avctx->priv_data;
498
499 av_packet_unref(&s->buffered_pkt);
500
501 ff_mediacodec_dec_flush(avctx, s->ctx);
502 }
503
504 static const AVCodecHWConfigInternal *const mediacodec_hw_configs[] = {
505 &(const AVCodecHWConfigInternal) {
506 .public = {
507 .pix_fmt = AV_PIX_FMT_MEDIACODEC,
508 .methods = AV_CODEC_HW_CONFIG_METHOD_AD_HOC |
509 AV_CODEC_HW_CONFIG_METHOD_HW_DEVICE_CTX,
510 .device_type = AV_HWDEVICE_TYPE_MEDIACODEC,
511 },
512 .hwaccel = NULL,
513 },
514 NULL
515 };
516
517 #define OFFSET(x) offsetof(MediaCodecH264DecContext, x)
518 #define VD AV_OPT_FLAG_VIDEO_PARAM | AV_OPT_FLAG_DECODING_PARAM
519 static const AVOption ff_mediacodec_vdec_options[] = {
520 { "delay_flush", "Delay flush until hw output buffers are returned to the decoder",
521 OFFSET(delay_flush), AV_OPT_TYPE_BOOL, {.i64 = 0}, 0, 1, VD },
522 { NULL }
523 };
524
525 #define DECLARE_MEDIACODEC_VCLASS(short_name) \
526 static const AVClass ff_##short_name##_mediacodec_dec_class = { \
527 .class_name = #short_name "_mediacodec", \
528 .item_name = av_default_item_name, \
529 .option = ff_mediacodec_vdec_options, \
530 .version = LIBAVUTIL_VERSION_INT, \
531 };
532
533 #define DECLARE_MEDIACODEC_VDEC(short_name, full_name, codec_id, bsf) \
534 DECLARE_MEDIACODEC_VCLASS(short_name) \
535 const FFCodec ff_ ## short_name ## _mediacodec_decoder = { \
536 .p.name = #short_name "_mediacodec", \
537 .p.long_name = NULL_IF_CONFIG_SMALL(full_name " Android MediaCodec decoder"), \
538 .p.type = AVMEDIA_TYPE_VIDEO, \
539 .p.id = codec_id, \
540 .p.priv_class = &ff_##short_name##_mediacodec_dec_class, \
541 .priv_data_size = sizeof(MediaCodecH264DecContext), \
542 .init = mediacodec_decode_init, \
543 FF_CODEC_RECEIVE_FRAME_CB(mediacodec_receive_frame), \
544 .flush = mediacodec_decode_flush, \
545 .close = mediacodec_decode_close, \
546 .p.capabilities = AV_CODEC_CAP_DELAY | AV_CODEC_CAP_AVOID_PROBING | AV_CODEC_CAP_HARDWARE, \
547 .caps_internal = FF_CODEC_CAP_SETS_PKT_DTS, \
548 .bsfs = bsf, \
549 .hw_configs = mediacodec_hw_configs, \
550 .p.wrapper_name = "mediacodec", \
551 }; \
552
553 #if CONFIG_H264_MEDIACODEC_DECODER
554 DECLARE_MEDIACODEC_VDEC(h264, "H.264", AV_CODEC_ID_H264, "h264_mp4toannexb")
555 #endif
556
557 #if CONFIG_HEVC_MEDIACODEC_DECODER
558 DECLARE_MEDIACODEC_VDEC(hevc, "H.265", AV_CODEC_ID_HEVC, "hevc_mp4toannexb")
559 #endif
560
561 #if CONFIG_MPEG2_MEDIACODEC_DECODER
562 DECLARE_MEDIACODEC_VDEC(mpeg2, "MPEG-2", AV_CODEC_ID_MPEG2VIDEO, NULL)
563 #endif
564
565 #if CONFIG_MPEG4_MEDIACODEC_DECODER
566 DECLARE_MEDIACODEC_VDEC(mpeg4, "MPEG-4", AV_CODEC_ID_MPEG4, NULL)
567 #endif
568
569 #if CONFIG_VP8_MEDIACODEC_DECODER
570 DECLARE_MEDIACODEC_VDEC(vp8, "VP8", AV_CODEC_ID_VP8, NULL)
571 #endif
572
573 #if CONFIG_VP9_MEDIACODEC_DECODER
574 DECLARE_MEDIACODEC_VDEC(vp9, "VP9", AV_CODEC_ID_VP9, NULL)
575 #endif
576