1 /*
2 * Copyright (C) 2017 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 //#define LOG_NDEBUG 0
18 #define LOG_TAG "FrameDecoder"
19
20 #include "include/FrameDecoder.h"
21 #include "include/FrameCaptureLayer.h"
22 #include "include/HevcUtils.h"
23 #include <binder/MemoryBase.h>
24 #include <binder/MemoryHeapBase.h>
25 #include <gui/Surface.h>
26 #include <inttypes.h>
27 #include <mediadrm/ICrypto.h>
28 #include <media/IMediaSource.h>
29 #include <media/MediaCodecBuffer.h>
30 #include <media/stagefright/foundation/avc_utils.h>
31 #include <media/stagefright/foundation/ADebug.h>
32 #include <media/stagefright/foundation/AMessage.h>
33 #include <media/stagefright/foundation/ColorUtils.h>
34 #include <media/stagefright/ColorConverter.h>
35 #include <media/stagefright/FrameCaptureProcessor.h>
36 #include <media/stagefright/MediaBuffer.h>
37 #include <media/stagefright/MediaCodec.h>
38 #include <media/stagefright/MediaCodecConstants.h>
39 #include <media/stagefright/MediaDefs.h>
40 #include <media/stagefright/MediaErrors.h>
41 #include <media/stagefright/Utils.h>
42 #include <private/media/VideoFrame.h>
43 #include <utils/Log.h>
44
45 namespace android {
46
47 static const int64_t kBufferTimeOutUs = 10000LL; // 10 msec
48 static const size_t kRetryCount = 100; // must be >0
49 static const int64_t kDefaultSampleDurationUs = 33333LL; // 33ms
50
allocVideoFrame(const sp<MetaData> & trackMeta,int32_t width,int32_t height,int32_t tileWidth,int32_t tileHeight,int32_t dstBpp,uint32_t bitDepth,bool allocRotated,bool metaOnly)51 sp<IMemory> allocVideoFrame(const sp<MetaData>& trackMeta,
52 int32_t width, int32_t height, int32_t tileWidth, int32_t tileHeight,
53 int32_t dstBpp, uint32_t bitDepth, bool allocRotated, bool metaOnly) {
54 int32_t rotationAngle;
55 if (!trackMeta->findInt32(kKeyRotation, &rotationAngle)) {
56 rotationAngle = 0; // By default, no rotation
57 }
58 uint32_t type;
59 const void *iccData;
60 size_t iccSize;
61 if (!trackMeta->findData(kKeyIccProfile, &type, &iccData, &iccSize)){
62 iccData = NULL;
63 iccSize = 0;
64 }
65
66 int32_t sarWidth, sarHeight;
67 int32_t displayWidth, displayHeight;
68 if (trackMeta->findInt32(kKeySARWidth, &sarWidth)
69 && trackMeta->findInt32(kKeySARHeight, &sarHeight)
70 && sarHeight != 0) {
71 int32_t multVal;
72 if (width < 0 || sarWidth < 0 ||
73 __builtin_mul_overflow(width, sarWidth, &multVal)) {
74 ALOGE("displayWidth overflow %dx%d", width, sarWidth);
75 return NULL;
76 }
77 displayWidth = (width * sarWidth) / sarHeight;
78 displayHeight = height;
79 } else if (trackMeta->findInt32(kKeyDisplayWidth, &displayWidth)
80 && trackMeta->findInt32(kKeyDisplayHeight, &displayHeight)
81 && displayWidth > 0 && displayHeight > 0
82 && width > 0 && height > 0) {
83 ALOGV("found display size %dx%d", displayWidth, displayHeight);
84 } else {
85 displayWidth = width;
86 displayHeight = height;
87 }
88
89 if (allocRotated) {
90 if (rotationAngle == 90 || rotationAngle == 270) {
91 // swap width and height for 90 & 270 degrees rotation
92 std::swap(width, height);
93 std::swap(displayWidth, displayHeight);
94 std::swap(tileWidth, tileHeight);
95 }
96 // Rotation is already applied.
97 rotationAngle = 0;
98 }
99
100 if (!metaOnly) {
101 int32_t multVal;
102 if (width < 0 || height < 0 || dstBpp < 0 ||
103 __builtin_mul_overflow(dstBpp, width, &multVal) ||
104 __builtin_mul_overflow(multVal, height, &multVal)) {
105 ALOGE("Frame size overflow %dx%d bpp %d", width, height, dstBpp);
106 return NULL;
107 }
108 }
109
110 VideoFrame frame(width, height, displayWidth, displayHeight,
111 tileWidth, tileHeight, rotationAngle, dstBpp, bitDepth, !metaOnly, iccSize);
112
113 size_t size = frame.getFlattenedSize();
114 sp<MemoryHeapBase> heap = new MemoryHeapBase(size, 0, "MetadataRetrieverClient");
115 if (heap == NULL) {
116 ALOGE("failed to create MemoryDealer");
117 return NULL;
118 }
119 sp<IMemory> frameMem = new MemoryBase(heap, 0, size);
120 if (frameMem == NULL || frameMem->unsecurePointer() == NULL) {
121 ALOGE("not enough memory for VideoFrame size=%zu", size);
122 return NULL;
123 }
124 VideoFrame* frameCopy = static_cast<VideoFrame*>(frameMem->unsecurePointer());
125 frameCopy->init(frame, iccData, iccSize);
126
127 return frameMem;
128 }
129
allocVideoFrame(const sp<MetaData> & trackMeta,int32_t width,int32_t height,int32_t tileWidth,int32_t tileHeight,int32_t dstBpp,uint8_t bitDepth,bool allocRotated=false)130 sp<IMemory> allocVideoFrame(const sp<MetaData>& trackMeta,
131 int32_t width, int32_t height, int32_t tileWidth, int32_t tileHeight,
132 int32_t dstBpp, uint8_t bitDepth, bool allocRotated = false) {
133 return allocVideoFrame(trackMeta, width, height, tileWidth, tileHeight, dstBpp, bitDepth,
134 allocRotated, false /*metaOnly*/);
135 }
136
allocMetaFrame(const sp<MetaData> & trackMeta,int32_t width,int32_t height,int32_t tileWidth,int32_t tileHeight,int32_t dstBpp,uint8_t bitDepth)137 sp<IMemory> allocMetaFrame(const sp<MetaData>& trackMeta,
138 int32_t width, int32_t height, int32_t tileWidth, int32_t tileHeight,
139 int32_t dstBpp, uint8_t bitDepth) {
140 return allocVideoFrame(trackMeta, width, height, tileWidth, tileHeight, dstBpp, bitDepth,
141 false /*allocRotated*/, true /*metaOnly*/);
142 }
143
isAvif(const sp<MetaData> & trackMeta)144 bool isAvif(const sp<MetaData> &trackMeta) {
145 const char *mime;
146 return trackMeta->findCString(kKeyMIMEType, &mime)
147 && (!strcasecmp(mime, MEDIA_MIMETYPE_VIDEO_AV1)
148 || !strcasecmp(mime, MEDIA_MIMETYPE_IMAGE_AVIF));
149 }
150
findThumbnailInfo(const sp<MetaData> & trackMeta,int32_t * width,int32_t * height,uint32_t * type=NULL,const void ** data=NULL,size_t * size=NULL)151 bool findThumbnailInfo(
152 const sp<MetaData> &trackMeta, int32_t *width, int32_t *height,
153 uint32_t *type = NULL, const void **data = NULL, size_t *size = NULL) {
154 uint32_t dummyType;
155 const void *dummyData;
156 size_t dummySize;
157 int codecConfigKey = isAvif(trackMeta) ? kKeyThumbnailAV1C : kKeyThumbnailHVCC;
158 return trackMeta->findInt32(kKeyThumbnailWidth, width)
159 && trackMeta->findInt32(kKeyThumbnailHeight, height)
160 && trackMeta->findData(codecConfigKey,
161 type ?: &dummyType, data ?: &dummyData, size ?: &dummySize);
162 }
163
findGridInfo(const sp<MetaData> & trackMeta,int32_t * tileWidth,int32_t * tileHeight,int32_t * gridRows,int32_t * gridCols)164 bool findGridInfo(const sp<MetaData> &trackMeta,
165 int32_t *tileWidth, int32_t *tileHeight, int32_t *gridRows, int32_t *gridCols) {
166 return trackMeta->findInt32(kKeyTileWidth, tileWidth) && (*tileWidth > 0)
167 && trackMeta->findInt32(kKeyTileHeight, tileHeight) && (*tileHeight > 0)
168 && trackMeta->findInt32(kKeyGridRows, gridRows) && (*gridRows > 0)
169 && trackMeta->findInt32(kKeyGridCols, gridCols) && (*gridCols > 0);
170 }
171
getDstColorFormat(android_pixel_format_t colorFormat,OMX_COLOR_FORMATTYPE * dstFormat,ui::PixelFormat * captureFormat,int32_t * dstBpp)172 bool getDstColorFormat(
173 android_pixel_format_t colorFormat,
174 OMX_COLOR_FORMATTYPE *dstFormat,
175 ui::PixelFormat *captureFormat,
176 int32_t *dstBpp) {
177 switch (colorFormat) {
178 case HAL_PIXEL_FORMAT_RGB_565:
179 {
180 *dstFormat = OMX_COLOR_Format16bitRGB565;
181 *captureFormat = ui::PixelFormat::RGB_565;
182 *dstBpp = 2;
183 return true;
184 }
185 case HAL_PIXEL_FORMAT_RGBA_8888:
186 {
187 *dstFormat = OMX_COLOR_Format32BitRGBA8888;
188 *captureFormat = ui::PixelFormat::RGBA_8888;
189 *dstBpp = 4;
190 return true;
191 }
192 case HAL_PIXEL_FORMAT_BGRA_8888:
193 {
194 *dstFormat = OMX_COLOR_Format32bitBGRA8888;
195 *captureFormat = ui::PixelFormat::BGRA_8888;
196 *dstBpp = 4;
197 return true;
198 }
199 case HAL_PIXEL_FORMAT_RGBA_1010102:
200 {
201 *dstFormat = (OMX_COLOR_FORMATTYPE)COLOR_Format32bitABGR2101010;
202 *captureFormat = ui::PixelFormat::RGBA_1010102;
203 *dstBpp = 4;
204 return true;
205 }
206 default:
207 {
208 ALOGE("Unsupported color format: %d", colorFormat);
209 break;
210 }
211 }
212 return false;
213 }
214
215 //static
getMetadataOnly(const sp<MetaData> & trackMeta,int colorFormat,bool thumbnail,uint32_t bitDepth)216 sp<IMemory> FrameDecoder::getMetadataOnly(
217 const sp<MetaData> &trackMeta, int colorFormat, bool thumbnail, uint32_t bitDepth) {
218 OMX_COLOR_FORMATTYPE dstFormat;
219 ui::PixelFormat captureFormat;
220 int32_t dstBpp;
221 if (!getDstColorFormat((android_pixel_format_t)colorFormat,
222 &dstFormat, &captureFormat, &dstBpp)) {
223 return NULL;
224 }
225
226 int32_t width, height, tileWidth = 0, tileHeight = 0;
227 if (thumbnail) {
228 if (!findThumbnailInfo(trackMeta, &width, &height)) {
229 return NULL;
230 }
231 } else {
232 CHECK(trackMeta->findInt32(kKeyWidth, &width));
233 CHECK(trackMeta->findInt32(kKeyHeight, &height));
234
235 int32_t gridRows, gridCols;
236 if (!findGridInfo(trackMeta, &tileWidth, &tileHeight, &gridRows, &gridCols)) {
237 tileWidth = tileHeight = 0;
238 }
239 }
240
241 sp<IMemory> metaMem =
242 allocMetaFrame(trackMeta, width, height, tileWidth, tileHeight, dstBpp, bitDepth);
243 if (metaMem == nullptr) {
244 return NULL;
245 }
246
247 // try to fill sequence meta's duration based on average frame rate,
248 // default to 33ms if frame rate is unavailable.
249 int32_t frameRate;
250 VideoFrame* meta = static_cast<VideoFrame*>(metaMem->unsecurePointer());
251 if (trackMeta->findInt32(kKeyFrameRate, &frameRate) && frameRate > 0) {
252 meta->mDurationUs = 1000000LL / frameRate;
253 } else {
254 meta->mDurationUs = kDefaultSampleDurationUs;
255 }
256 return metaMem;
257 }
258
FrameDecoder(const AString & componentName,const sp<MetaData> & trackMeta,const sp<IMediaSource> & source)259 FrameDecoder::FrameDecoder(
260 const AString &componentName,
261 const sp<MetaData> &trackMeta,
262 const sp<IMediaSource> &source)
263 : mComponentName(componentName),
264 mTrackMeta(trackMeta),
265 mSource(source),
266 mDstFormat(OMX_COLOR_Format16bitRGB565),
267 mDstBpp(2),
268 mHaveMoreInputs(true),
269 mFirstSample(true) {
270 }
271
~FrameDecoder()272 FrameDecoder::~FrameDecoder() {
273 if (mDecoder != NULL) {
274 mDecoder->release();
275 mSource->stop();
276 }
277 }
278
isHDR(const sp<AMessage> & format)279 bool isHDR(const sp<AMessage> &format) {
280 uint32_t standard, transfer;
281 if (!format->findInt32("color-standard", (int32_t*)&standard)) {
282 standard = 0;
283 }
284 if (!format->findInt32("color-transfer", (int32_t*)&transfer)) {
285 transfer = 0;
286 }
287 return standard == ColorUtils::kColorStandardBT2020 &&
288 (transfer == ColorUtils::kColorTransferST2084 ||
289 transfer == ColorUtils::kColorTransferHLG);
290 }
291
init(int64_t frameTimeUs,int option,int colorFormat)292 status_t FrameDecoder::init(
293 int64_t frameTimeUs, int option, int colorFormat) {
294 if (!getDstColorFormat((android_pixel_format_t)colorFormat,
295 &mDstFormat, &mCaptureFormat, &mDstBpp)) {
296 return ERROR_UNSUPPORTED;
297 }
298
299 sp<AMessage> videoFormat = onGetFormatAndSeekOptions(
300 frameTimeUs, option, &mReadOptions, &mSurface);
301 if (videoFormat == NULL) {
302 ALOGE("video format or seek mode not supported");
303 return ERROR_UNSUPPORTED;
304 }
305
306 status_t err;
307 sp<ALooper> looper = new ALooper;
308 looper->start();
309 sp<MediaCodec> decoder = MediaCodec::CreateByComponentName(
310 looper, mComponentName, &err);
311 if (decoder.get() == NULL || err != OK) {
312 ALOGW("Failed to instantiate decoder [%s]", mComponentName.c_str());
313 return (decoder.get() == NULL) ? NO_MEMORY : err;
314 }
315
316 err = decoder->configure(
317 videoFormat, mSurface, NULL /* crypto */, 0 /* flags */);
318 if (err != OK) {
319 ALOGW("configure returned error %d (%s)", err, asString(err));
320 decoder->release();
321 return err;
322 }
323
324 err = decoder->start();
325 if (err != OK) {
326 ALOGW("start returned error %d (%s)", err, asString(err));
327 decoder->release();
328 return err;
329 }
330
331 err = mSource->start();
332 if (err != OK) {
333 ALOGW("source failed to start: %d (%s)", err, asString(err));
334 decoder->release();
335 return err;
336 }
337 mDecoder = decoder;
338
339 return OK;
340 }
341
extractFrame(FrameRect * rect)342 sp<IMemory> FrameDecoder::extractFrame(FrameRect *rect) {
343 status_t err = onExtractRect(rect);
344 if (err == OK) {
345 err = extractInternal();
346 }
347 if (err != OK) {
348 return NULL;
349 }
350
351 return mFrameMemory;
352 }
353
extractInternal()354 status_t FrameDecoder::extractInternal() {
355 status_t err = OK;
356 bool done = false;
357 size_t retriesLeft = kRetryCount;
358 if (!mDecoder) {
359 ALOGE("decoder is not initialized");
360 return NO_INIT;
361 }
362 do {
363 size_t index;
364 int64_t ptsUs = 0LL;
365 uint32_t flags = 0;
366
367 // Queue as many inputs as we possibly can, then block on dequeuing
368 // outputs. After getting each output, come back and queue the inputs
369 // again to keep the decoder busy.
370 while (mHaveMoreInputs) {
371 err = mDecoder->dequeueInputBuffer(&index, 0);
372 if (err != OK) {
373 ALOGV("Timed out waiting for input");
374 if (retriesLeft) {
375 err = OK;
376 }
377 break;
378 }
379 sp<MediaCodecBuffer> codecBuffer;
380 err = mDecoder->getInputBuffer(index, &codecBuffer);
381 if (err != OK) {
382 ALOGE("failed to get input buffer %zu", index);
383 break;
384 }
385
386 MediaBufferBase *mediaBuffer = NULL;
387
388 err = mSource->read(&mediaBuffer, &mReadOptions);
389 mReadOptions.clearSeekTo();
390 if (err != OK) {
391 mHaveMoreInputs = false;
392 if (!mFirstSample && err == ERROR_END_OF_STREAM) {
393 (void)mDecoder->queueInputBuffer(
394 index, 0, 0, 0, MediaCodec::BUFFER_FLAG_EOS);
395 err = OK;
396 } else {
397 ALOGW("Input Error: err=%d", err);
398 }
399 break;
400 }
401
402 if (mediaBuffer->range_length() > codecBuffer->capacity()) {
403 ALOGE("buffer size (%zu) too large for codec input size (%zu)",
404 mediaBuffer->range_length(), codecBuffer->capacity());
405 mHaveMoreInputs = false;
406 err = BAD_VALUE;
407 } else {
408 codecBuffer->setRange(0, mediaBuffer->range_length());
409
410 CHECK(mediaBuffer->meta_data().findInt64(kKeyTime, &ptsUs));
411 memcpy(codecBuffer->data(),
412 (const uint8_t*)mediaBuffer->data() + mediaBuffer->range_offset(),
413 mediaBuffer->range_length());
414
415 onInputReceived(codecBuffer, mediaBuffer->meta_data(), mFirstSample, &flags);
416 mFirstSample = false;
417 }
418
419 mediaBuffer->release();
420
421 if (mHaveMoreInputs) {
422 ALOGV("QueueInput: size=%zu ts=%" PRId64 " us flags=%x",
423 codecBuffer->size(), ptsUs, flags);
424
425 err = mDecoder->queueInputBuffer(
426 index,
427 codecBuffer->offset(),
428 codecBuffer->size(),
429 ptsUs,
430 flags);
431
432 if (flags & MediaCodec::BUFFER_FLAG_EOS) {
433 mHaveMoreInputs = false;
434 }
435 }
436 }
437
438 while (err == OK) {
439 size_t offset, size;
440 // wait for a decoded buffer
441 err = mDecoder->dequeueOutputBuffer(
442 &index,
443 &offset,
444 &size,
445 &ptsUs,
446 &flags,
447 kBufferTimeOutUs);
448
449 if (err == INFO_FORMAT_CHANGED) {
450 ALOGV("Received format change");
451 err = mDecoder->getOutputFormat(&mOutputFormat);
452 } else if (err == INFO_OUTPUT_BUFFERS_CHANGED) {
453 ALOGV("Output buffers changed");
454 err = OK;
455 } else {
456 if (err == -EAGAIN /* INFO_TRY_AGAIN_LATER */ && --retriesLeft > 0) {
457 ALOGV("Timed-out waiting for output.. retries left = %zu", retriesLeft);
458 err = OK;
459 } else if (err == OK) {
460 // If we're seeking with CLOSEST option and obtained a valid targetTimeUs
461 // from the extractor, decode to the specified frame. Otherwise we're done.
462 ALOGV("Received an output buffer, timeUs=%lld", (long long)ptsUs);
463 sp<MediaCodecBuffer> videoFrameBuffer;
464 err = mDecoder->getOutputBuffer(index, &videoFrameBuffer);
465 if (err != OK) {
466 ALOGE("failed to get output buffer %zu", index);
467 break;
468 }
469 if (mSurface != nullptr) {
470 mDecoder->renderOutputBufferAndRelease(index);
471 err = onOutputReceived(videoFrameBuffer, mOutputFormat, ptsUs, &done);
472 } else {
473 err = onOutputReceived(videoFrameBuffer, mOutputFormat, ptsUs, &done);
474 mDecoder->releaseOutputBuffer(index);
475 }
476 } else {
477 ALOGW("Received error %d (%s) instead of output", err, asString(err));
478 done = true;
479 }
480 break;
481 }
482 }
483 } while (err == OK && !done);
484
485 if (err != OK) {
486 ALOGE("failed to get video frame (err %d)", err);
487 }
488
489 return err;
490 }
491
492 //////////////////////////////////////////////////////////////////////
493
VideoFrameDecoder(const AString & componentName,const sp<MetaData> & trackMeta,const sp<IMediaSource> & source)494 VideoFrameDecoder::VideoFrameDecoder(
495 const AString &componentName,
496 const sp<MetaData> &trackMeta,
497 const sp<IMediaSource> &source)
498 : FrameDecoder(componentName, trackMeta, source),
499 mFrame(NULL),
500 mIsAvc(false),
501 mIsHevc(false),
502 mSeekMode(MediaSource::ReadOptions::SEEK_PREVIOUS_SYNC),
503 mTargetTimeUs(-1LL),
504 mDefaultSampleDurationUs(0) {
505 }
506
onGetFormatAndSeekOptions(int64_t frameTimeUs,int seekMode,MediaSource::ReadOptions * options,sp<Surface> * window)507 sp<AMessage> VideoFrameDecoder::onGetFormatAndSeekOptions(
508 int64_t frameTimeUs, int seekMode,
509 MediaSource::ReadOptions *options,
510 sp<Surface> *window) {
511 mSeekMode = static_cast<MediaSource::ReadOptions::SeekMode>(seekMode);
512 if (mSeekMode < MediaSource::ReadOptions::SEEK_PREVIOUS_SYNC ||
513 mSeekMode > MediaSource::ReadOptions::SEEK_FRAME_INDEX) {
514 ALOGE("Unknown seek mode: %d", mSeekMode);
515 return NULL;
516 }
517
518 const char *mime;
519 if (!trackMeta()->findCString(kKeyMIMEType, &mime)) {
520 ALOGE("Could not find mime type");
521 return NULL;
522 }
523
524 mIsAvc = !strcasecmp(mime, MEDIA_MIMETYPE_VIDEO_AVC);
525 mIsHevc = !strcasecmp(mime, MEDIA_MIMETYPE_VIDEO_HEVC);
526
527 if (frameTimeUs < 0) {
528 int64_t thumbNailTime = -1ll;
529 if (!trackMeta()->findInt64(kKeyThumbnailTime, &thumbNailTime)
530 || thumbNailTime < 0) {
531 thumbNailTime = 0;
532 }
533 options->setSeekTo(thumbNailTime, mSeekMode);
534 } else {
535 options->setSeekTo(frameTimeUs, mSeekMode);
536 }
537
538 sp<AMessage> videoFormat;
539 if (convertMetaDataToMessage(trackMeta(), &videoFormat) != OK) {
540 ALOGE("b/23680780");
541 ALOGW("Failed to convert meta data to message");
542 return NULL;
543 }
544
545 if (dstFormat() == COLOR_Format32bitABGR2101010) {
546 videoFormat->setInt32("color-format", COLOR_FormatYUVP010);
547 } else {
548 videoFormat->setInt32("color-format", COLOR_FormatYUV420Flexible);
549 }
550
551 // For the thumbnail extraction case, try to allocate single buffer in both
552 // input and output ports, if seeking to a sync frame. NOTE: This request may
553 // fail if component requires more than that for decoding.
554 bool isSeekingClosest = (mSeekMode == MediaSource::ReadOptions::SEEK_CLOSEST)
555 || (mSeekMode == MediaSource::ReadOptions::SEEK_FRAME_INDEX);
556 if (!isSeekingClosest) {
557 videoFormat->setInt32("android._num-input-buffers", 1);
558 videoFormat->setInt32("android._num-output-buffers", 1);
559 }
560
561 if (isHDR(videoFormat)) {
562 *window = initSurface();
563 if (*window == NULL) {
564 ALOGE("Failed to init surface control for HDR, fallback to non-hdr");
565 } else {
566 videoFormat->setInt32("color-format", OMX_COLOR_FormatAndroidOpaque);
567 }
568 }
569
570 int32_t frameRate;
571 if (trackMeta()->findInt32(kKeyFrameRate, &frameRate) && frameRate > 0) {
572 mDefaultSampleDurationUs = 1000000LL / frameRate;
573 } else {
574 mDefaultSampleDurationUs = kDefaultSampleDurationUs;
575 }
576
577 return videoFormat;
578 }
579
onInputReceived(const sp<MediaCodecBuffer> & codecBuffer,MetaDataBase & sampleMeta,bool firstSample,uint32_t * flags)580 status_t VideoFrameDecoder::onInputReceived(
581 const sp<MediaCodecBuffer> &codecBuffer,
582 MetaDataBase &sampleMeta, bool firstSample, uint32_t *flags) {
583 bool isSeekingClosest = (mSeekMode == MediaSource::ReadOptions::SEEK_CLOSEST)
584 || (mSeekMode == MediaSource::ReadOptions::SEEK_FRAME_INDEX);
585
586 if (firstSample && isSeekingClosest) {
587 sampleMeta.findInt64(kKeyTargetTime, &mTargetTimeUs);
588 ALOGV("Seeking closest: targetTimeUs=%lld", (long long)mTargetTimeUs);
589 }
590
591 if (!isSeekingClosest
592 && ((mIsAvc && IsIDR(codecBuffer->data(), codecBuffer->size()))
593 || (mIsHevc && IsIDR(
594 codecBuffer->data(), codecBuffer->size())))) {
595 // Only need to decode one IDR frame, unless we're seeking with CLOSEST
596 // option, in which case we need to actually decode to targetTimeUs.
597 *flags |= MediaCodec::BUFFER_FLAG_EOS;
598 }
599 int64_t durationUs;
600 if (sampleMeta.findInt64(kKeyDuration, &durationUs)) {
601 mSampleDurations.push_back(durationUs);
602 } else {
603 mSampleDurations.push_back(mDefaultSampleDurationUs);
604 }
605 return OK;
606 }
607
onOutputReceived(const sp<MediaCodecBuffer> & videoFrameBuffer,const sp<AMessage> & outputFormat,int64_t timeUs,bool * done)608 status_t VideoFrameDecoder::onOutputReceived(
609 const sp<MediaCodecBuffer> &videoFrameBuffer,
610 const sp<AMessage> &outputFormat,
611 int64_t timeUs, bool *done) {
612 int64_t durationUs = mDefaultSampleDurationUs;
613 if (!mSampleDurations.empty()) {
614 durationUs = *mSampleDurations.begin();
615 mSampleDurations.erase(mSampleDurations.begin());
616 }
617 bool shouldOutput = (mTargetTimeUs < 0LL) || (timeUs >= mTargetTimeUs);
618
619 // If this is not the target frame, skip color convert.
620 if (!shouldOutput) {
621 *done = false;
622 return OK;
623 }
624
625 *done = true;
626
627 if (outputFormat == NULL) {
628 return ERROR_MALFORMED;
629 }
630
631 int32_t width, height, stride, srcFormat;
632 if (!outputFormat->findInt32("width", &width) ||
633 !outputFormat->findInt32("height", &height) ||
634 !outputFormat->findInt32("color-format", &srcFormat)) {
635 ALOGE("format missing dimension or color: %s",
636 outputFormat->debugString().c_str());
637 return ERROR_MALFORMED;
638 }
639
640 if (!outputFormat->findInt32("stride", &stride)) {
641 if (mCaptureLayer == NULL) {
642 ALOGE("format must have stride for byte buffer mode: %s",
643 outputFormat->debugString().c_str());
644 return ERROR_MALFORMED;
645 }
646 // for surface output, set stride to width, we don't actually need it.
647 stride = width;
648 }
649
650 int32_t crop_left, crop_top, crop_right, crop_bottom;
651 if (!outputFormat->findRect("crop", &crop_left, &crop_top, &crop_right, &crop_bottom)) {
652 crop_left = crop_top = 0;
653 crop_right = width - 1;
654 crop_bottom = height - 1;
655 }
656
657 int32_t slice_height;
658 if (outputFormat->findInt32("slice-height", &slice_height) && slice_height > 0) {
659 height = slice_height;
660 }
661
662 uint32_t bitDepth = 8;
663 if (COLOR_FormatYUVP010 == srcFormat) {
664 bitDepth = 10;
665 }
666
667 if (mFrame == NULL) {
668 sp<IMemory> frameMem = allocVideoFrame(
669 trackMeta(),
670 (crop_right - crop_left + 1),
671 (crop_bottom - crop_top + 1),
672 0,
673 0,
674 dstBpp(),
675 bitDepth,
676 mCaptureLayer != nullptr /*allocRotated*/);
677 if (frameMem == nullptr) {
678 return NO_MEMORY;
679 }
680
681 mFrame = static_cast<VideoFrame*>(frameMem->unsecurePointer());
682
683 setFrame(frameMem);
684 }
685
686 mFrame->mDurationUs = durationUs;
687
688 if (mCaptureLayer != nullptr) {
689 return captureSurface();
690 }
691 ColorConverter converter((OMX_COLOR_FORMATTYPE)srcFormat, dstFormat());
692
693 uint32_t standard, range, transfer;
694 if (!outputFormat->findInt32("color-standard", (int32_t*)&standard)) {
695 standard = 0;
696 }
697 if (!outputFormat->findInt32("color-range", (int32_t*)&range)) {
698 range = 0;
699 }
700 if (!outputFormat->findInt32("color-transfer", (int32_t*)&transfer)) {
701 transfer = 0;
702 }
703 sp<ABuffer> imgObj;
704 if (videoFrameBuffer->meta()->findBuffer("image-data", &imgObj)) {
705 MediaImage2 *imageData = nullptr;
706 imageData = (MediaImage2 *)(imgObj.get()->data());
707 if (imageData != nullptr) {
708 converter.setSrcMediaImage2(*imageData);
709 }
710 }
711 if (srcFormat == COLOR_FormatYUV420Flexible && imgObj.get() == nullptr) {
712 return ERROR_UNSUPPORTED;
713 }
714 converter.setSrcColorSpace(standard, range, transfer);
715 if (converter.isValid()) {
716 converter.convert(
717 (const uint8_t *)videoFrameBuffer->data(),
718 width, height, stride,
719 crop_left, crop_top, crop_right, crop_bottom,
720 mFrame->getFlattenedData(),
721 mFrame->mWidth, mFrame->mHeight, mFrame->mRowBytes,
722 // since the frame is allocated with top-left adjusted,
723 // the dst rect should start at {0,0} as well.
724 0, 0, mFrame->mWidth - 1, mFrame->mHeight - 1);
725 return OK;
726 }
727
728 ALOGE("Unable to convert from format 0x%08x to 0x%08x",
729 srcFormat, dstFormat());
730 return ERROR_UNSUPPORTED;
731 }
732
initSurface()733 sp<Surface> VideoFrameDecoder::initSurface() {
734 // create the consumer listener interface, and hold sp so that this
735 // interface lives as long as the GraphicBufferSource.
736 sp<FrameCaptureLayer> captureLayer = new FrameCaptureLayer();
737 if (captureLayer->init() != OK) {
738 ALOGE("failed to init capture layer");
739 return nullptr;
740 }
741 mCaptureLayer = captureLayer;
742
743 return captureLayer->getSurface();
744 }
745
captureSurface()746 status_t VideoFrameDecoder::captureSurface() {
747 sp<GraphicBuffer> outBuffer;
748 status_t err = mCaptureLayer->capture(
749 captureFormat(), Rect(0, 0, mFrame->mWidth, mFrame->mHeight), &outBuffer);
750
751 if (err != OK) {
752 ALOGE("failed to capture layer (err %d)", err);
753 return err;
754 }
755
756 ALOGV("capture: %dx%d, format %d, stride %d",
757 outBuffer->getWidth(),
758 outBuffer->getHeight(),
759 outBuffer->getPixelFormat(),
760 outBuffer->getStride());
761
762 uint8_t *base;
763 int32_t outBytesPerPixel, outBytesPerStride;
764 err = outBuffer->lock(
765 GraphicBuffer::USAGE_SW_READ_OFTEN,
766 reinterpret_cast<void**>(&base),
767 &outBytesPerPixel,
768 &outBytesPerStride);
769 if (err != OK) {
770 ALOGE("failed to lock graphic buffer: err %d", err);
771 return err;
772 }
773
774 uint8_t *dst = mFrame->getFlattenedData();
775 for (size_t y = 0 ; y < fmin(mFrame->mHeight, outBuffer->getHeight()) ; y++) {
776 memcpy(dst, base, fmin(mFrame->mWidth, outBuffer->getWidth()) * mFrame->mBytesPerPixel);
777 dst += mFrame->mRowBytes;
778 base += outBuffer->getStride() * mFrame->mBytesPerPixel;
779 }
780 outBuffer->unlock();
781 return OK;
782 }
783
784 ////////////////////////////////////////////////////////////////////////
785
MediaImageDecoder(const AString & componentName,const sp<MetaData> & trackMeta,const sp<IMediaSource> & source)786 MediaImageDecoder::MediaImageDecoder(
787 const AString &componentName,
788 const sp<MetaData> &trackMeta,
789 const sp<IMediaSource> &source)
790 : FrameDecoder(componentName, trackMeta, source),
791 mFrame(NULL),
792 mWidth(0),
793 mHeight(0),
794 mGridRows(1),
795 mGridCols(1),
796 mTileWidth(0),
797 mTileHeight(0),
798 mTilesDecoded(0),
799 mTargetTiles(0) {
800 }
801
onGetFormatAndSeekOptions(int64_t frameTimeUs,int,MediaSource::ReadOptions * options,sp<Surface> *)802 sp<AMessage> MediaImageDecoder::onGetFormatAndSeekOptions(
803 int64_t frameTimeUs, int /*seekMode*/,
804 MediaSource::ReadOptions *options, sp<Surface> * /*window*/) {
805 sp<MetaData> overrideMeta;
806 if (frameTimeUs < 0) {
807 uint32_t type;
808 const void *data;
809 size_t size;
810
811 // if we have a stand-alone thumbnail, set up the override meta,
812 // and set seekTo time to -1.
813 if (!findThumbnailInfo(trackMeta(), &mWidth, &mHeight, &type, &data, &size)) {
814 ALOGE("Thumbnail not available");
815 return NULL;
816 }
817 overrideMeta = new MetaData(*(trackMeta()));
818 overrideMeta->remove(kKeyDisplayWidth);
819 overrideMeta->remove(kKeyDisplayHeight);
820 overrideMeta->setInt32(kKeyWidth, mWidth);
821 overrideMeta->setInt32(kKeyHeight, mHeight);
822 // The AV1 codec configuration data is passed via CSD0 to the AV1
823 // decoder.
824 const int codecConfigKey = isAvif(trackMeta()) ? kKeyOpaqueCSD0 : kKeyHVCC;
825 overrideMeta->setData(codecConfigKey, type, data, size);
826 options->setSeekTo(-1);
827 } else {
828 CHECK(trackMeta()->findInt32(kKeyWidth, &mWidth));
829 CHECK(trackMeta()->findInt32(kKeyHeight, &mHeight));
830
831 options->setSeekTo(frameTimeUs);
832 }
833
834 mGridRows = mGridCols = 1;
835 if (overrideMeta == NULL) {
836 // check if we're dealing with a tiled heif
837 int32_t tileWidth, tileHeight, gridRows, gridCols;
838 int32_t widthColsProduct = 0;
839 int32_t heightRowsProduct = 0;
840 if (findGridInfo(trackMeta(), &tileWidth, &tileHeight, &gridRows, &gridCols)) {
841 if (__builtin_mul_overflow(tileWidth, gridCols, &widthColsProduct) ||
842 __builtin_mul_overflow(tileHeight, gridRows, &heightRowsProduct)) {
843 ALOGE("Multiplication overflowed Grid size: %dx%d, Picture size: %dx%d",
844 gridCols, gridRows, tileWidth, tileHeight);
845 return nullptr;
846 }
847 if (mWidth <= widthColsProduct && mHeight <= heightRowsProduct) {
848 ALOGV("grid: %dx%d, tile size: %dx%d, picture size: %dx%d",
849 gridCols, gridRows, tileWidth, tileHeight, mWidth, mHeight);
850
851 overrideMeta = new MetaData(*(trackMeta()));
852 overrideMeta->setInt32(kKeyWidth, tileWidth);
853 overrideMeta->setInt32(kKeyHeight, tileHeight);
854 mTileWidth = tileWidth;
855 mTileHeight = tileHeight;
856 mGridCols = gridCols;
857 mGridRows = gridRows;
858 } else {
859 ALOGW("ignore bad grid: %dx%d, tile size: %dx%d, picture size: %dx%d",
860 gridCols, gridRows, tileWidth, tileHeight, mWidth, mHeight);
861 }
862 }
863 if (overrideMeta == NULL) {
864 overrideMeta = trackMeta();
865 }
866 }
867 mTargetTiles = mGridCols * mGridRows;
868
869 sp<AMessage> videoFormat;
870 if (convertMetaDataToMessage(overrideMeta, &videoFormat) != OK) {
871 ALOGE("b/23680780");
872 ALOGW("Failed to convert meta data to message");
873 return NULL;
874 }
875
876 if (dstFormat() == COLOR_Format32bitABGR2101010) {
877 videoFormat->setInt32("color-format", COLOR_FormatYUVP010);
878 } else {
879 videoFormat->setInt32("color-format", COLOR_FormatYUV420Flexible);
880 }
881
882 if ((mGridRows == 1) && (mGridCols == 1)) {
883 videoFormat->setInt32("android._num-input-buffers", 1);
884 videoFormat->setInt32("android._num-output-buffers", 1);
885 }
886 return videoFormat;
887 }
888
onExtractRect(FrameRect * rect)889 status_t MediaImageDecoder::onExtractRect(FrameRect *rect) {
890 // TODO:
891 // This callback is for verifying whether we can decode the rect,
892 // and if so, set up the internal variables for decoding.
893 // Currently, rect decoding is restricted to sequentially decoding one
894 // row of tiles at a time. We can't decode arbitrary rects, as the image
895 // track doesn't yet support seeking by tiles. So all we do here is to
896 // verify the rect against what we expect.
897 // When seeking by tile is supported, this code should be updated to
898 // set the seek parameters.
899 if (rect == NULL) {
900 if (mTilesDecoded > 0) {
901 return ERROR_UNSUPPORTED;
902 }
903 mTargetTiles = mGridRows * mGridCols;
904 return OK;
905 }
906
907 if (mTileWidth <= 0 || mTileHeight <=0) {
908 return ERROR_UNSUPPORTED;
909 }
910
911 int32_t row = mTilesDecoded / mGridCols;
912 int32_t expectedTop = row * mTileHeight;
913 int32_t expectedBot = (row + 1) * mTileHeight;
914 if (expectedBot > mHeight) {
915 expectedBot = mHeight;
916 }
917 if (rect->left != 0 || rect->top != expectedTop
918 || rect->right != mWidth || rect->bottom != expectedBot) {
919 ALOGE("currently only support sequential decoding of slices");
920 return ERROR_UNSUPPORTED;
921 }
922
923 // advance one row
924 mTargetTiles = mTilesDecoded + mGridCols;
925 return OK;
926 }
927
onOutputReceived(const sp<MediaCodecBuffer> & videoFrameBuffer,const sp<AMessage> & outputFormat,int64_t,bool * done)928 status_t MediaImageDecoder::onOutputReceived(
929 const sp<MediaCodecBuffer> &videoFrameBuffer,
930 const sp<AMessage> &outputFormat, int64_t /*timeUs*/, bool *done) {
931 if (outputFormat == NULL) {
932 return ERROR_MALFORMED;
933 }
934
935 int32_t width, height, stride;
936 if (outputFormat->findInt32("width", &width) == false) {
937 ALOGE("MediaImageDecoder::onOutputReceived:width is missing in outputFormat");
938 return ERROR_MALFORMED;
939 }
940 if (outputFormat->findInt32("height", &height) == false) {
941 ALOGE("MediaImageDecoder::onOutputReceived:height is missing in outputFormat");
942 return ERROR_MALFORMED;
943 }
944 if (outputFormat->findInt32("stride", &stride) == false) {
945 ALOGE("MediaImageDecoder::onOutputReceived:stride is missing in outputFormat");
946 return ERROR_MALFORMED;
947 }
948
949 int32_t srcFormat;
950 CHECK(outputFormat->findInt32("color-format", &srcFormat));
951
952 uint32_t bitDepth = 8;
953 if (COLOR_FormatYUVP010 == srcFormat) {
954 bitDepth = 10;
955 }
956
957 if (mFrame == NULL) {
958 sp<IMemory> frameMem = allocVideoFrame(
959 trackMeta(), mWidth, mHeight, mTileWidth, mTileHeight, dstBpp(), bitDepth);
960
961 if (frameMem == nullptr) {
962 return NO_MEMORY;
963 }
964
965 mFrame = static_cast<VideoFrame*>(frameMem->unsecurePointer());
966
967 setFrame(frameMem);
968 }
969
970 ColorConverter converter((OMX_COLOR_FORMATTYPE)srcFormat, dstFormat());
971
972 uint32_t standard, range, transfer;
973 if (!outputFormat->findInt32("color-standard", (int32_t*)&standard)) {
974 standard = 0;
975 }
976 if (!outputFormat->findInt32("color-range", (int32_t*)&range)) {
977 range = 0;
978 }
979 if (!outputFormat->findInt32("color-transfer", (int32_t*)&transfer)) {
980 transfer = 0;
981 }
982 sp<ABuffer> imgObj;
983 if (videoFrameBuffer->meta()->findBuffer("image-data", &imgObj)) {
984 MediaImage2 *imageData = nullptr;
985 imageData = (MediaImage2 *)(imgObj.get()->data());
986 if (imageData != nullptr) {
987 converter.setSrcMediaImage2(*imageData);
988 }
989 }
990 if (srcFormat == COLOR_FormatYUV420Flexible && imgObj.get() == nullptr) {
991 return ERROR_UNSUPPORTED;
992 }
993 converter.setSrcColorSpace(standard, range, transfer);
994
995 int32_t crop_left, crop_top, crop_right, crop_bottom;
996 if (!outputFormat->findRect("crop", &crop_left, &crop_top, &crop_right, &crop_bottom)) {
997 crop_left = crop_top = 0;
998 crop_right = width - 1;
999 crop_bottom = height - 1;
1000 }
1001
1002 int32_t slice_height;
1003 if (outputFormat->findInt32("slice-height", &slice_height) && slice_height > 0) {
1004 height = slice_height;
1005 }
1006
1007 int32_t crop_width, crop_height;
1008 crop_width = crop_right - crop_left + 1;
1009 crop_height = crop_bottom - crop_top + 1;
1010
1011 int32_t dstLeft, dstTop, dstRight, dstBottom;
1012 dstLeft = mTilesDecoded % mGridCols * crop_width;
1013 dstTop = mTilesDecoded / mGridCols * crop_height;
1014 dstRight = dstLeft + crop_width - 1;
1015 dstBottom = dstTop + crop_height - 1;
1016
1017 // apply crop on bottom-right
1018 // TODO: need to move this into the color converter itself.
1019 if (dstRight >= mWidth) {
1020 crop_right = crop_left + mWidth - dstLeft - 1;
1021 dstRight = mWidth - 1;
1022 }
1023 if (dstBottom >= mHeight) {
1024 crop_bottom = crop_top + mHeight - dstTop - 1;
1025 dstBottom = mHeight - 1;
1026 }
1027
1028 *done = (++mTilesDecoded >= mTargetTiles);
1029
1030 if (converter.isValid()) {
1031 converter.convert(
1032 (const uint8_t *)videoFrameBuffer->data(),
1033 width, height, stride,
1034 crop_left, crop_top, crop_right, crop_bottom,
1035 mFrame->getFlattenedData(),
1036 mFrame->mWidth, mFrame->mHeight, mFrame->mRowBytes,
1037 dstLeft, dstTop, dstRight, dstBottom);
1038 return OK;
1039 }
1040
1041 ALOGE("Unable to convert from format 0x%08x to 0x%08x",
1042 srcFormat, dstFormat());
1043 return ERROR_UNSUPPORTED;
1044 }
1045
1046 } // namespace android
1047