1 /*
2 * Copyright (C) 2017 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 //#define LOG_NDEBUG 0
18 #define LOG_TAG "FrameDecoder"
19
20 #include "include/FrameDecoder.h"
21 #include "include/FrameCaptureLayer.h"
22 #include "include/HevcUtils.h"
23 #include <binder/MemoryBase.h>
24 #include <binder/MemoryHeapBase.h>
25 #include <gui/Surface.h>
26 #include <inttypes.h>
27 #include <mediadrm/ICrypto.h>
28 #include <media/IMediaSource.h>
29 #include <media/MediaCodecBuffer.h>
30 #include <media/stagefright/foundation/avc_utils.h>
31 #include <media/stagefright/foundation/ADebug.h>
32 #include <media/stagefright/foundation/AMessage.h>
33 #include <media/stagefright/foundation/ColorUtils.h>
34 #include <media/stagefright/ColorConverter.h>
35 #include <media/stagefright/FrameCaptureProcessor.h>
36 #include <media/stagefright/MediaBuffer.h>
37 #include <media/stagefright/MediaCodec.h>
38 #include <media/stagefright/MediaCodecConstants.h>
39 #include <media/stagefright/MediaDefs.h>
40 #include <media/stagefright/MediaErrors.h>
41 #include <media/stagefright/Utils.h>
42 #include <private/media/VideoFrame.h>
43 #include <utils/Log.h>
44
45 namespace android {
46
47 static const int64_t kBufferTimeOutUs = 10000LL; // 10 msec
48 static const size_t kRetryCount = 100; // must be >0
49 static const int64_t kDefaultSampleDurationUs = 33333LL; // 33ms
50
allocVideoFrame(const sp<MetaData> & trackMeta,int32_t width,int32_t height,int32_t tileWidth,int32_t tileHeight,int32_t dstBpp,uint32_t bitDepth,bool allocRotated,bool metaOnly)51 sp<IMemory> allocVideoFrame(const sp<MetaData>& trackMeta,
52 int32_t width, int32_t height, int32_t tileWidth, int32_t tileHeight,
53 int32_t dstBpp, uint32_t bitDepth, bool allocRotated, bool metaOnly) {
54 int32_t rotationAngle;
55 if (!trackMeta->findInt32(kKeyRotation, &rotationAngle)) {
56 rotationAngle = 0; // By default, no rotation
57 }
58 uint32_t type;
59 const void *iccData;
60 size_t iccSize;
61 if (!trackMeta->findData(kKeyIccProfile, &type, &iccData, &iccSize)){
62 iccData = NULL;
63 iccSize = 0;
64 }
65
66 int32_t sarWidth, sarHeight;
67 int32_t displayWidth, displayHeight;
68 if (trackMeta->findInt32(kKeySARWidth, &sarWidth)
69 && trackMeta->findInt32(kKeySARHeight, &sarHeight)
70 && sarHeight != 0) {
71 int32_t multVal;
72 if (width < 0 || sarWidth < 0 ||
73 __builtin_mul_overflow(width, sarWidth, &multVal)) {
74 ALOGE("displayWidth overflow %dx%d", width, sarWidth);
75 return NULL;
76 }
77 displayWidth = (width * sarWidth) / sarHeight;
78 displayHeight = height;
79 } else if (trackMeta->findInt32(kKeyDisplayWidth, &displayWidth)
80 && trackMeta->findInt32(kKeyDisplayHeight, &displayHeight)
81 && displayWidth > 0 && displayHeight > 0
82 && width > 0 && height > 0) {
83 ALOGV("found display size %dx%d", displayWidth, displayHeight);
84 } else {
85 displayWidth = width;
86 displayHeight = height;
87 }
88
89 if (allocRotated) {
90 if (rotationAngle == 90 || rotationAngle == 270) {
91 // swap width and height for 90 & 270 degrees rotation
92 std::swap(width, height);
93 std::swap(displayWidth, displayHeight);
94 std::swap(tileWidth, tileHeight);
95 }
96 // Rotation is already applied.
97 rotationAngle = 0;
98 }
99
100 if (!metaOnly) {
101 int32_t multVal;
102 if (width < 0 || height < 0 || dstBpp < 0 ||
103 __builtin_mul_overflow(dstBpp, width, &multVal) ||
104 __builtin_mul_overflow(multVal, height, &multVal)) {
105 ALOGE("Frame size overflow %dx%d bpp %d", width, height, dstBpp);
106 return NULL;
107 }
108 }
109
110 VideoFrame frame(width, height, displayWidth, displayHeight,
111 tileWidth, tileHeight, rotationAngle, dstBpp, bitDepth, !metaOnly, iccSize);
112
113 size_t size = frame.getFlattenedSize();
114 sp<MemoryHeapBase> heap = new MemoryHeapBase(size, 0, "MetadataRetrieverClient");
115 if (heap == NULL) {
116 ALOGE("failed to create MemoryDealer");
117 return NULL;
118 }
119 sp<IMemory> frameMem = new MemoryBase(heap, 0, size);
120 if (frameMem == NULL || frameMem->unsecurePointer() == NULL) {
121 ALOGE("not enough memory for VideoFrame size=%zu", size);
122 return NULL;
123 }
124 VideoFrame* frameCopy = static_cast<VideoFrame*>(frameMem->unsecurePointer());
125 frameCopy->init(frame, iccData, iccSize);
126
127 return frameMem;
128 }
129
allocVideoFrame(const sp<MetaData> & trackMeta,int32_t width,int32_t height,int32_t tileWidth,int32_t tileHeight,int32_t dstBpp,uint8_t bitDepth,bool allocRotated=false)130 sp<IMemory> allocVideoFrame(const sp<MetaData>& trackMeta,
131 int32_t width, int32_t height, int32_t tileWidth, int32_t tileHeight,
132 int32_t dstBpp, uint8_t bitDepth, bool allocRotated = false) {
133 return allocVideoFrame(trackMeta, width, height, tileWidth, tileHeight, dstBpp, bitDepth,
134 allocRotated, false /*metaOnly*/);
135 }
136
allocMetaFrame(const sp<MetaData> & trackMeta,int32_t width,int32_t height,int32_t tileWidth,int32_t tileHeight,int32_t dstBpp,uint8_t bitDepth)137 sp<IMemory> allocMetaFrame(const sp<MetaData>& trackMeta,
138 int32_t width, int32_t height, int32_t tileWidth, int32_t tileHeight,
139 int32_t dstBpp, uint8_t bitDepth) {
140 return allocVideoFrame(trackMeta, width, height, tileWidth, tileHeight, dstBpp, bitDepth,
141 false /*allocRotated*/, true /*metaOnly*/);
142 }
143
isAvif(const sp<MetaData> & trackMeta)144 bool isAvif(const sp<MetaData> &trackMeta) {
145 const char *mime;
146 return trackMeta->findCString(kKeyMIMEType, &mime)
147 && (!strcasecmp(mime, MEDIA_MIMETYPE_VIDEO_AV1)
148 || !strcasecmp(mime, MEDIA_MIMETYPE_IMAGE_AVIF));
149 }
150
findThumbnailInfo(const sp<MetaData> & trackMeta,int32_t * width,int32_t * height,uint32_t * type=NULL,const void ** data=NULL,size_t * size=NULL)151 bool findThumbnailInfo(
152 const sp<MetaData> &trackMeta, int32_t *width, int32_t *height,
153 uint32_t *type = NULL, const void **data = NULL, size_t *size = NULL) {
154 uint32_t dummyType;
155 const void *dummyData;
156 size_t dummySize;
157 int codecConfigKey = isAvif(trackMeta) ? kKeyThumbnailAV1C : kKeyThumbnailHVCC;
158 return trackMeta->findInt32(kKeyThumbnailWidth, width)
159 && trackMeta->findInt32(kKeyThumbnailHeight, height)
160 && trackMeta->findData(codecConfigKey,
161 type ?: &dummyType, data ?: &dummyData, size ?: &dummySize);
162 }
163
findGridInfo(const sp<MetaData> & trackMeta,int32_t * tileWidth,int32_t * tileHeight,int32_t * gridRows,int32_t * gridCols)164 bool findGridInfo(const sp<MetaData> &trackMeta,
165 int32_t *tileWidth, int32_t *tileHeight, int32_t *gridRows, int32_t *gridCols) {
166 return trackMeta->findInt32(kKeyTileWidth, tileWidth) && (*tileWidth > 0)
167 && trackMeta->findInt32(kKeyTileHeight, tileHeight) && (*tileHeight > 0)
168 && trackMeta->findInt32(kKeyGridRows, gridRows) && (*gridRows > 0)
169 && trackMeta->findInt32(kKeyGridCols, gridCols) && (*gridCols > 0);
170 }
171
getDstColorFormat(android_pixel_format_t colorFormat,OMX_COLOR_FORMATTYPE * dstFormat,ui::PixelFormat * captureFormat,int32_t * dstBpp)172 bool getDstColorFormat(
173 android_pixel_format_t colorFormat,
174 OMX_COLOR_FORMATTYPE *dstFormat,
175 ui::PixelFormat *captureFormat,
176 int32_t *dstBpp) {
177 switch (colorFormat) {
178 case HAL_PIXEL_FORMAT_RGB_565:
179 {
180 *dstFormat = OMX_COLOR_Format16bitRGB565;
181 *captureFormat = ui::PixelFormat::RGB_565;
182 *dstBpp = 2;
183 return true;
184 }
185 case HAL_PIXEL_FORMAT_RGBA_8888:
186 {
187 *dstFormat = OMX_COLOR_Format32BitRGBA8888;
188 *captureFormat = ui::PixelFormat::RGBA_8888;
189 *dstBpp = 4;
190 return true;
191 }
192 case HAL_PIXEL_FORMAT_BGRA_8888:
193 {
194 *dstFormat = OMX_COLOR_Format32bitBGRA8888;
195 *captureFormat = ui::PixelFormat::BGRA_8888;
196 *dstBpp = 4;
197 return true;
198 }
199 case HAL_PIXEL_FORMAT_RGBA_1010102:
200 {
201 *dstFormat = (OMX_COLOR_FORMATTYPE)COLOR_Format32bitABGR2101010;
202 *captureFormat = ui::PixelFormat::RGBA_1010102;
203 *dstBpp = 4;
204 return true;
205 }
206 default:
207 {
208 ALOGE("Unsupported color format: %d", colorFormat);
209 break;
210 }
211 }
212 return false;
213 }
214
215 //static
getMetadataOnly(const sp<MetaData> & trackMeta,int colorFormat,bool thumbnail,uint32_t bitDepth)216 sp<IMemory> FrameDecoder::getMetadataOnly(
217 const sp<MetaData> &trackMeta, int colorFormat, bool thumbnail, uint32_t bitDepth) {
218 OMX_COLOR_FORMATTYPE dstFormat;
219 ui::PixelFormat captureFormat;
220 int32_t dstBpp;
221 if (!getDstColorFormat((android_pixel_format_t)colorFormat,
222 &dstFormat, &captureFormat, &dstBpp)) {
223 return NULL;
224 }
225
226 int32_t width, height, tileWidth = 0, tileHeight = 0;
227 if (thumbnail) {
228 if (!findThumbnailInfo(trackMeta, &width, &height)) {
229 return NULL;
230 }
231 } else {
232 CHECK(trackMeta->findInt32(kKeyWidth, &width));
233 CHECK(trackMeta->findInt32(kKeyHeight, &height));
234
235 int32_t gridRows, gridCols;
236 if (!findGridInfo(trackMeta, &tileWidth, &tileHeight, &gridRows, &gridCols)) {
237 tileWidth = tileHeight = 0;
238 }
239 }
240
241 sp<IMemory> metaMem =
242 allocMetaFrame(trackMeta, width, height, tileWidth, tileHeight, dstBpp, bitDepth);
243
244 // try to fill sequence meta's duration based on average frame rate,
245 // default to 33ms if frame rate is unavailable.
246 int32_t frameRate;
247 VideoFrame* meta = static_cast<VideoFrame*>(metaMem->unsecurePointer());
248 if (trackMeta->findInt32(kKeyFrameRate, &frameRate) && frameRate > 0) {
249 meta->mDurationUs = 1000000LL / frameRate;
250 } else {
251 meta->mDurationUs = kDefaultSampleDurationUs;
252 }
253 return metaMem;
254 }
255
FrameDecoder(const AString & componentName,const sp<MetaData> & trackMeta,const sp<IMediaSource> & source)256 FrameDecoder::FrameDecoder(
257 const AString &componentName,
258 const sp<MetaData> &trackMeta,
259 const sp<IMediaSource> &source)
260 : mComponentName(componentName),
261 mTrackMeta(trackMeta),
262 mSource(source),
263 mDstFormat(OMX_COLOR_Format16bitRGB565),
264 mDstBpp(2),
265 mHaveMoreInputs(true),
266 mFirstSample(true) {
267 }
268
~FrameDecoder()269 FrameDecoder::~FrameDecoder() {
270 if (mDecoder != NULL) {
271 mDecoder->release();
272 mSource->stop();
273 }
274 }
275
isHDR(const sp<AMessage> & format)276 bool isHDR(const sp<AMessage> &format) {
277 uint32_t standard, transfer;
278 if (!format->findInt32("color-standard", (int32_t*)&standard)) {
279 standard = 0;
280 }
281 if (!format->findInt32("color-transfer", (int32_t*)&transfer)) {
282 transfer = 0;
283 }
284 return standard == ColorUtils::kColorStandardBT2020 &&
285 (transfer == ColorUtils::kColorTransferST2084 ||
286 transfer == ColorUtils::kColorTransferHLG);
287 }
288
init(int64_t frameTimeUs,int option,int colorFormat)289 status_t FrameDecoder::init(
290 int64_t frameTimeUs, int option, int colorFormat) {
291 if (!getDstColorFormat((android_pixel_format_t)colorFormat,
292 &mDstFormat, &mCaptureFormat, &mDstBpp)) {
293 return ERROR_UNSUPPORTED;
294 }
295
296 sp<AMessage> videoFormat = onGetFormatAndSeekOptions(
297 frameTimeUs, option, &mReadOptions, &mSurface);
298 if (videoFormat == NULL) {
299 ALOGE("video format or seek mode not supported");
300 return ERROR_UNSUPPORTED;
301 }
302
303 status_t err;
304 sp<ALooper> looper = new ALooper;
305 looper->start();
306 sp<MediaCodec> decoder = MediaCodec::CreateByComponentName(
307 looper, mComponentName, &err);
308 if (decoder.get() == NULL || err != OK) {
309 ALOGW("Failed to instantiate decoder [%s]", mComponentName.c_str());
310 return (decoder.get() == NULL) ? NO_MEMORY : err;
311 }
312
313 err = decoder->configure(
314 videoFormat, mSurface, NULL /* crypto */, 0 /* flags */);
315 if (err != OK) {
316 ALOGW("configure returned error %d (%s)", err, asString(err));
317 decoder->release();
318 return err;
319 }
320
321 err = decoder->start();
322 if (err != OK) {
323 ALOGW("start returned error %d (%s)", err, asString(err));
324 decoder->release();
325 return err;
326 }
327
328 err = mSource->start();
329 if (err != OK) {
330 ALOGW("source failed to start: %d (%s)", err, asString(err));
331 decoder->release();
332 return err;
333 }
334 mDecoder = decoder;
335
336 return OK;
337 }
338
extractFrame(FrameRect * rect)339 sp<IMemory> FrameDecoder::extractFrame(FrameRect *rect) {
340 status_t err = onExtractRect(rect);
341 if (err == OK) {
342 err = extractInternal();
343 }
344 if (err != OK) {
345 return NULL;
346 }
347
348 return mFrameMemory;
349 }
350
extractInternal()351 status_t FrameDecoder::extractInternal() {
352 status_t err = OK;
353 bool done = false;
354 size_t retriesLeft = kRetryCount;
355 do {
356 size_t index;
357 int64_t ptsUs = 0LL;
358 uint32_t flags = 0;
359
360 // Queue as many inputs as we possibly can, then block on dequeuing
361 // outputs. After getting each output, come back and queue the inputs
362 // again to keep the decoder busy.
363 while (mHaveMoreInputs) {
364 err = mDecoder->dequeueInputBuffer(&index, 0);
365 if (err != OK) {
366 ALOGV("Timed out waiting for input");
367 if (retriesLeft) {
368 err = OK;
369 }
370 break;
371 }
372 sp<MediaCodecBuffer> codecBuffer;
373 err = mDecoder->getInputBuffer(index, &codecBuffer);
374 if (err != OK) {
375 ALOGE("failed to get input buffer %zu", index);
376 break;
377 }
378
379 MediaBufferBase *mediaBuffer = NULL;
380
381 err = mSource->read(&mediaBuffer, &mReadOptions);
382 mReadOptions.clearSeekTo();
383 if (err != OK) {
384 mHaveMoreInputs = false;
385 if (!mFirstSample && err == ERROR_END_OF_STREAM) {
386 (void)mDecoder->queueInputBuffer(
387 index, 0, 0, 0, MediaCodec::BUFFER_FLAG_EOS);
388 err = OK;
389 } else {
390 ALOGW("Input Error: err=%d", err);
391 }
392 break;
393 }
394
395 if (mediaBuffer->range_length() > codecBuffer->capacity()) {
396 ALOGE("buffer size (%zu) too large for codec input size (%zu)",
397 mediaBuffer->range_length(), codecBuffer->capacity());
398 mHaveMoreInputs = false;
399 err = BAD_VALUE;
400 } else {
401 codecBuffer->setRange(0, mediaBuffer->range_length());
402
403 CHECK(mediaBuffer->meta_data().findInt64(kKeyTime, &ptsUs));
404 memcpy(codecBuffer->data(),
405 (const uint8_t*)mediaBuffer->data() + mediaBuffer->range_offset(),
406 mediaBuffer->range_length());
407
408 onInputReceived(codecBuffer, mediaBuffer->meta_data(), mFirstSample, &flags);
409 mFirstSample = false;
410 }
411
412 mediaBuffer->release();
413
414 if (mHaveMoreInputs) {
415 ALOGV("QueueInput: size=%zu ts=%" PRId64 " us flags=%x",
416 codecBuffer->size(), ptsUs, flags);
417
418 err = mDecoder->queueInputBuffer(
419 index,
420 codecBuffer->offset(),
421 codecBuffer->size(),
422 ptsUs,
423 flags);
424
425 if (flags & MediaCodec::BUFFER_FLAG_EOS) {
426 mHaveMoreInputs = false;
427 }
428 }
429 }
430
431 while (err == OK) {
432 size_t offset, size;
433 // wait for a decoded buffer
434 err = mDecoder->dequeueOutputBuffer(
435 &index,
436 &offset,
437 &size,
438 &ptsUs,
439 &flags,
440 kBufferTimeOutUs);
441
442 if (err == INFO_FORMAT_CHANGED) {
443 ALOGV("Received format change");
444 err = mDecoder->getOutputFormat(&mOutputFormat);
445 } else if (err == INFO_OUTPUT_BUFFERS_CHANGED) {
446 ALOGV("Output buffers changed");
447 err = OK;
448 } else {
449 if (err == -EAGAIN /* INFO_TRY_AGAIN_LATER */ && --retriesLeft > 0) {
450 ALOGV("Timed-out waiting for output.. retries left = %zu", retriesLeft);
451 err = OK;
452 } else if (err == OK) {
453 // If we're seeking with CLOSEST option and obtained a valid targetTimeUs
454 // from the extractor, decode to the specified frame. Otherwise we're done.
455 ALOGV("Received an output buffer, timeUs=%lld", (long long)ptsUs);
456 sp<MediaCodecBuffer> videoFrameBuffer;
457 err = mDecoder->getOutputBuffer(index, &videoFrameBuffer);
458 if (err != OK) {
459 ALOGE("failed to get output buffer %zu", index);
460 break;
461 }
462 if (mSurface != nullptr) {
463 mDecoder->renderOutputBufferAndRelease(index);
464 err = onOutputReceived(videoFrameBuffer, mOutputFormat, ptsUs, &done);
465 } else {
466 err = onOutputReceived(videoFrameBuffer, mOutputFormat, ptsUs, &done);
467 mDecoder->releaseOutputBuffer(index);
468 }
469 } else {
470 ALOGW("Received error %d (%s) instead of output", err, asString(err));
471 done = true;
472 }
473 break;
474 }
475 }
476 } while (err == OK && !done);
477
478 if (err != OK) {
479 ALOGE("failed to get video frame (err %d)", err);
480 }
481
482 return err;
483 }
484
485 //////////////////////////////////////////////////////////////////////
486
VideoFrameDecoder(const AString & componentName,const sp<MetaData> & trackMeta,const sp<IMediaSource> & source)487 VideoFrameDecoder::VideoFrameDecoder(
488 const AString &componentName,
489 const sp<MetaData> &trackMeta,
490 const sp<IMediaSource> &source)
491 : FrameDecoder(componentName, trackMeta, source),
492 mFrame(NULL),
493 mIsAvc(false),
494 mIsHevc(false),
495 mSeekMode(MediaSource::ReadOptions::SEEK_PREVIOUS_SYNC),
496 mTargetTimeUs(-1LL),
497 mDefaultSampleDurationUs(0) {
498 }
499
onGetFormatAndSeekOptions(int64_t frameTimeUs,int seekMode,MediaSource::ReadOptions * options,sp<Surface> * window)500 sp<AMessage> VideoFrameDecoder::onGetFormatAndSeekOptions(
501 int64_t frameTimeUs, int seekMode,
502 MediaSource::ReadOptions *options,
503 sp<Surface> *window) {
504 mSeekMode = static_cast<MediaSource::ReadOptions::SeekMode>(seekMode);
505 if (mSeekMode < MediaSource::ReadOptions::SEEK_PREVIOUS_SYNC ||
506 mSeekMode > MediaSource::ReadOptions::SEEK_FRAME_INDEX) {
507 ALOGE("Unknown seek mode: %d", mSeekMode);
508 return NULL;
509 }
510
511 const char *mime;
512 if (!trackMeta()->findCString(kKeyMIMEType, &mime)) {
513 ALOGE("Could not find mime type");
514 return NULL;
515 }
516
517 mIsAvc = !strcasecmp(mime, MEDIA_MIMETYPE_VIDEO_AVC);
518 mIsHevc = !strcasecmp(mime, MEDIA_MIMETYPE_VIDEO_HEVC);
519
520 if (frameTimeUs < 0) {
521 int64_t thumbNailTime = -1ll;
522 if (!trackMeta()->findInt64(kKeyThumbnailTime, &thumbNailTime)
523 || thumbNailTime < 0) {
524 thumbNailTime = 0;
525 }
526 options->setSeekTo(thumbNailTime, mSeekMode);
527 } else {
528 options->setSeekTo(frameTimeUs, mSeekMode);
529 }
530
531 sp<AMessage> videoFormat;
532 if (convertMetaDataToMessage(trackMeta(), &videoFormat) != OK) {
533 ALOGE("b/23680780");
534 ALOGW("Failed to convert meta data to message");
535 return NULL;
536 }
537
538 if (dstFormat() == COLOR_Format32bitABGR2101010) {
539 videoFormat->setInt32("color-format", COLOR_FormatYUVP010);
540 } else {
541 videoFormat->setInt32("color-format", OMX_COLOR_FormatYUV420Planar);
542 }
543
544 // For the thumbnail extraction case, try to allocate single buffer in both
545 // input and output ports, if seeking to a sync frame. NOTE: This request may
546 // fail if component requires more than that for decoding.
547 bool isSeekingClosest = (mSeekMode == MediaSource::ReadOptions::SEEK_CLOSEST)
548 || (mSeekMode == MediaSource::ReadOptions::SEEK_FRAME_INDEX);
549 if (!isSeekingClosest) {
550 videoFormat->setInt32("android._num-input-buffers", 1);
551 videoFormat->setInt32("android._num-output-buffers", 1);
552 }
553
554 if (isHDR(videoFormat)) {
555 *window = initSurface();
556 if (*window == NULL) {
557 ALOGE("Failed to init surface control for HDR, fallback to non-hdr");
558 } else {
559 videoFormat->setInt32("color-format", OMX_COLOR_FormatAndroidOpaque);
560 }
561 }
562
563 int32_t frameRate;
564 if (trackMeta()->findInt32(kKeyFrameRate, &frameRate) && frameRate > 0) {
565 mDefaultSampleDurationUs = 1000000LL / frameRate;
566 } else {
567 mDefaultSampleDurationUs = kDefaultSampleDurationUs;
568 }
569
570 return videoFormat;
571 }
572
onInputReceived(const sp<MediaCodecBuffer> & codecBuffer,MetaDataBase & sampleMeta,bool firstSample,uint32_t * flags)573 status_t VideoFrameDecoder::onInputReceived(
574 const sp<MediaCodecBuffer> &codecBuffer,
575 MetaDataBase &sampleMeta, bool firstSample, uint32_t *flags) {
576 bool isSeekingClosest = (mSeekMode == MediaSource::ReadOptions::SEEK_CLOSEST)
577 || (mSeekMode == MediaSource::ReadOptions::SEEK_FRAME_INDEX);
578
579 if (firstSample && isSeekingClosest) {
580 sampleMeta.findInt64(kKeyTargetTime, &mTargetTimeUs);
581 ALOGV("Seeking closest: targetTimeUs=%lld", (long long)mTargetTimeUs);
582 }
583
584 if (!isSeekingClosest
585 && ((mIsAvc && IsIDR(codecBuffer->data(), codecBuffer->size()))
586 || (mIsHevc && IsIDR(
587 codecBuffer->data(), codecBuffer->size())))) {
588 // Only need to decode one IDR frame, unless we're seeking with CLOSEST
589 // option, in which case we need to actually decode to targetTimeUs.
590 *flags |= MediaCodec::BUFFER_FLAG_EOS;
591 }
592 int64_t durationUs;
593 if (sampleMeta.findInt64(kKeyDuration, &durationUs)) {
594 mSampleDurations.push_back(durationUs);
595 } else {
596 mSampleDurations.push_back(mDefaultSampleDurationUs);
597 }
598 return OK;
599 }
600
onOutputReceived(const sp<MediaCodecBuffer> & videoFrameBuffer,const sp<AMessage> & outputFormat,int64_t timeUs,bool * done)601 status_t VideoFrameDecoder::onOutputReceived(
602 const sp<MediaCodecBuffer> &videoFrameBuffer,
603 const sp<AMessage> &outputFormat,
604 int64_t timeUs, bool *done) {
605 int64_t durationUs = mDefaultSampleDurationUs;
606 if (!mSampleDurations.empty()) {
607 durationUs = *mSampleDurations.begin();
608 mSampleDurations.erase(mSampleDurations.begin());
609 }
610 bool shouldOutput = (mTargetTimeUs < 0LL) || (timeUs >= mTargetTimeUs);
611
612 // If this is not the target frame, skip color convert.
613 if (!shouldOutput) {
614 *done = false;
615 return OK;
616 }
617
618 *done = true;
619
620 if (outputFormat == NULL) {
621 return ERROR_MALFORMED;
622 }
623
624 int32_t width, height, stride, srcFormat;
625 if (!outputFormat->findInt32("width", &width) ||
626 !outputFormat->findInt32("height", &height) ||
627 !outputFormat->findInt32("color-format", &srcFormat)) {
628 ALOGE("format missing dimension or color: %s",
629 outputFormat->debugString().c_str());
630 return ERROR_MALFORMED;
631 }
632
633 if (!outputFormat->findInt32("stride", &stride)) {
634 if (mCaptureLayer == NULL) {
635 ALOGE("format must have stride for byte buffer mode: %s",
636 outputFormat->debugString().c_str());
637 return ERROR_MALFORMED;
638 }
639 // for surface output, set stride to width, we don't actually need it.
640 stride = width;
641 }
642
643 int32_t crop_left, crop_top, crop_right, crop_bottom;
644 if (!outputFormat->findRect("crop", &crop_left, &crop_top, &crop_right, &crop_bottom)) {
645 crop_left = crop_top = 0;
646 crop_right = width - 1;
647 crop_bottom = height - 1;
648 }
649
650 int32_t slice_height;
651 if (outputFormat->findInt32("slice-height", &slice_height) && slice_height > 0) {
652 height = slice_height;
653 }
654
655 uint32_t bitDepth = 8;
656 if (COLOR_FormatYUVP010 == srcFormat) {
657 bitDepth = 10;
658 }
659
660 if (mFrame == NULL) {
661 sp<IMemory> frameMem = allocVideoFrame(
662 trackMeta(),
663 (crop_right - crop_left + 1),
664 (crop_bottom - crop_top + 1),
665 0,
666 0,
667 dstBpp(),
668 bitDepth,
669 mCaptureLayer != nullptr /*allocRotated*/);
670 if (frameMem == nullptr) {
671 return NO_MEMORY;
672 }
673
674 mFrame = static_cast<VideoFrame*>(frameMem->unsecurePointer());
675
676 setFrame(frameMem);
677 }
678
679 mFrame->mDurationUs = durationUs;
680
681 if (mCaptureLayer != nullptr) {
682 return captureSurface();
683 }
684
685 ColorConverter converter((OMX_COLOR_FORMATTYPE)srcFormat, dstFormat());
686
687 uint32_t standard, range, transfer;
688 if (!outputFormat->findInt32("color-standard", (int32_t*)&standard)) {
689 standard = 0;
690 }
691 if (!outputFormat->findInt32("color-range", (int32_t*)&range)) {
692 range = 0;
693 }
694 if (!outputFormat->findInt32("color-transfer", (int32_t*)&transfer)) {
695 transfer = 0;
696 }
697 converter.setSrcColorSpace(standard, range, transfer);
698
699 if (converter.isValid()) {
700 converter.convert(
701 (const uint8_t *)videoFrameBuffer->data(),
702 width, height, stride,
703 crop_left, crop_top, crop_right, crop_bottom,
704 mFrame->getFlattenedData(),
705 mFrame->mWidth, mFrame->mHeight, mFrame->mRowBytes,
706 // since the frame is allocated with top-left adjusted,
707 // the dst rect should start at {0,0} as well.
708 0, 0, mFrame->mWidth - 1, mFrame->mHeight - 1);
709 return OK;
710 }
711
712 ALOGE("Unable to convert from format 0x%08x to 0x%08x",
713 srcFormat, dstFormat());
714 return ERROR_UNSUPPORTED;
715 }
716
initSurface()717 sp<Surface> VideoFrameDecoder::initSurface() {
718 // create the consumer listener interface, and hold sp so that this
719 // interface lives as long as the GraphicBufferSource.
720 sp<FrameCaptureLayer> captureLayer = new FrameCaptureLayer();
721 if (captureLayer->init() != OK) {
722 ALOGE("failed to init capture layer");
723 return nullptr;
724 }
725 mCaptureLayer = captureLayer;
726
727 return captureLayer->getSurface();
728 }
729
captureSurface()730 status_t VideoFrameDecoder::captureSurface() {
731 sp<GraphicBuffer> outBuffer;
732 status_t err = mCaptureLayer->capture(
733 captureFormat(), Rect(0, 0, mFrame->mWidth, mFrame->mHeight), &outBuffer);
734
735 if (err != OK) {
736 ALOGE("failed to capture layer (err %d)", err);
737 return err;
738 }
739
740 ALOGV("capture: %dx%d, format %d, stride %d",
741 outBuffer->getWidth(),
742 outBuffer->getHeight(),
743 outBuffer->getPixelFormat(),
744 outBuffer->getStride());
745
746 uint8_t *base;
747 int32_t outBytesPerPixel, outBytesPerStride;
748 err = outBuffer->lock(
749 GraphicBuffer::USAGE_SW_READ_OFTEN,
750 reinterpret_cast<void**>(&base),
751 &outBytesPerPixel,
752 &outBytesPerStride);
753 if (err != OK) {
754 ALOGE("failed to lock graphic buffer: err %d", err);
755 return err;
756 }
757
758 uint8_t *dst = mFrame->getFlattenedData();
759 for (size_t y = 0 ; y < fmin(mFrame->mHeight, outBuffer->getHeight()) ; y++) {
760 memcpy(dst, base, fmin(mFrame->mWidth, outBuffer->getWidth()) * mFrame->mBytesPerPixel);
761 dst += mFrame->mRowBytes;
762 base += outBuffer->getStride() * mFrame->mBytesPerPixel;
763 }
764 outBuffer->unlock();
765 return OK;
766 }
767
768 ////////////////////////////////////////////////////////////////////////
769
MediaImageDecoder(const AString & componentName,const sp<MetaData> & trackMeta,const sp<IMediaSource> & source)770 MediaImageDecoder::MediaImageDecoder(
771 const AString &componentName,
772 const sp<MetaData> &trackMeta,
773 const sp<IMediaSource> &source)
774 : FrameDecoder(componentName, trackMeta, source),
775 mFrame(NULL),
776 mWidth(0),
777 mHeight(0),
778 mGridRows(1),
779 mGridCols(1),
780 mTileWidth(0),
781 mTileHeight(0),
782 mTilesDecoded(0),
783 mTargetTiles(0) {
784 }
785
onGetFormatAndSeekOptions(int64_t frameTimeUs,int,MediaSource::ReadOptions * options,sp<Surface> *)786 sp<AMessage> MediaImageDecoder::onGetFormatAndSeekOptions(
787 int64_t frameTimeUs, int /*seekMode*/,
788 MediaSource::ReadOptions *options, sp<Surface> * /*window*/) {
789 sp<MetaData> overrideMeta;
790 if (frameTimeUs < 0) {
791 uint32_t type;
792 const void *data;
793 size_t size;
794
795 // if we have a stand-alone thumbnail, set up the override meta,
796 // and set seekTo time to -1.
797 if (!findThumbnailInfo(trackMeta(), &mWidth, &mHeight, &type, &data, &size)) {
798 ALOGE("Thumbnail not available");
799 return NULL;
800 }
801 overrideMeta = new MetaData(*(trackMeta()));
802 overrideMeta->remove(kKeyDisplayWidth);
803 overrideMeta->remove(kKeyDisplayHeight);
804 overrideMeta->setInt32(kKeyWidth, mWidth);
805 overrideMeta->setInt32(kKeyHeight, mHeight);
806 // The AV1 codec configuration data is passed via CSD0 to the AV1
807 // decoder.
808 const int codecConfigKey = isAvif(trackMeta()) ? kKeyOpaqueCSD0 : kKeyHVCC;
809 overrideMeta->setData(codecConfigKey, type, data, size);
810 options->setSeekTo(-1);
811 } else {
812 CHECK(trackMeta()->findInt32(kKeyWidth, &mWidth));
813 CHECK(trackMeta()->findInt32(kKeyHeight, &mHeight));
814
815 options->setSeekTo(frameTimeUs);
816 }
817
818 mGridRows = mGridCols = 1;
819 if (overrideMeta == NULL) {
820 // check if we're dealing with a tiled heif
821 int32_t tileWidth, tileHeight, gridRows, gridCols;
822 int32_t widthColsProduct = 0;
823 int32_t heightRowsProduct = 0;
824 if (findGridInfo(trackMeta(), &tileWidth, &tileHeight, &gridRows, &gridCols)) {
825 if (__builtin_mul_overflow(tileWidth, gridCols, &widthColsProduct) ||
826 __builtin_mul_overflow(tileHeight, gridRows, &heightRowsProduct)) {
827 ALOGE("Multiplication overflowed Grid size: %dx%d, Picture size: %dx%d",
828 gridCols, gridRows, tileWidth, tileHeight);
829 return nullptr;
830 }
831 if (mWidth <= widthColsProduct && mHeight <= heightRowsProduct) {
832 ALOGV("grid: %dx%d, tile size: %dx%d, picture size: %dx%d",
833 gridCols, gridRows, tileWidth, tileHeight, mWidth, mHeight);
834
835 overrideMeta = new MetaData(*(trackMeta()));
836 overrideMeta->setInt32(kKeyWidth, tileWidth);
837 overrideMeta->setInt32(kKeyHeight, tileHeight);
838 mTileWidth = tileWidth;
839 mTileHeight = tileHeight;
840 mGridCols = gridCols;
841 mGridRows = gridRows;
842 } else {
843 ALOGW("ignore bad grid: %dx%d, tile size: %dx%d, picture size: %dx%d",
844 gridCols, gridRows, tileWidth, tileHeight, mWidth, mHeight);
845 }
846 }
847 if (overrideMeta == NULL) {
848 overrideMeta = trackMeta();
849 }
850 }
851 mTargetTiles = mGridCols * mGridRows;
852
853 sp<AMessage> videoFormat;
854 if (convertMetaDataToMessage(overrideMeta, &videoFormat) != OK) {
855 ALOGE("b/23680780");
856 ALOGW("Failed to convert meta data to message");
857 return NULL;
858 }
859
860 if (dstFormat() == COLOR_Format32bitABGR2101010) {
861 videoFormat->setInt32("color-format", COLOR_FormatYUVP010);
862 } else {
863 videoFormat->setInt32("color-format", OMX_COLOR_FormatYUV420Planar);
864 }
865
866 if ((mGridRows == 1) && (mGridCols == 1)) {
867 videoFormat->setInt32("android._num-input-buffers", 1);
868 videoFormat->setInt32("android._num-output-buffers", 1);
869 }
870 return videoFormat;
871 }
872
onExtractRect(FrameRect * rect)873 status_t MediaImageDecoder::onExtractRect(FrameRect *rect) {
874 // TODO:
875 // This callback is for verifying whether we can decode the rect,
876 // and if so, set up the internal variables for decoding.
877 // Currently, rect decoding is restricted to sequentially decoding one
878 // row of tiles at a time. We can't decode arbitrary rects, as the image
879 // track doesn't yet support seeking by tiles. So all we do here is to
880 // verify the rect against what we expect.
881 // When seeking by tile is supported, this code should be updated to
882 // set the seek parameters.
883 if (rect == NULL) {
884 if (mTilesDecoded > 0) {
885 return ERROR_UNSUPPORTED;
886 }
887 mTargetTiles = mGridRows * mGridCols;
888 return OK;
889 }
890
891 if (mTileWidth <= 0 || mTileHeight <=0) {
892 return ERROR_UNSUPPORTED;
893 }
894
895 int32_t row = mTilesDecoded / mGridCols;
896 int32_t expectedTop = row * mTileHeight;
897 int32_t expectedBot = (row + 1) * mTileHeight;
898 if (expectedBot > mHeight) {
899 expectedBot = mHeight;
900 }
901 if (rect->left != 0 || rect->top != expectedTop
902 || rect->right != mWidth || rect->bottom != expectedBot) {
903 ALOGE("currently only support sequential decoding of slices");
904 return ERROR_UNSUPPORTED;
905 }
906
907 // advance one row
908 mTargetTiles = mTilesDecoded + mGridCols;
909 return OK;
910 }
911
onOutputReceived(const sp<MediaCodecBuffer> & videoFrameBuffer,const sp<AMessage> & outputFormat,int64_t,bool * done)912 status_t MediaImageDecoder::onOutputReceived(
913 const sp<MediaCodecBuffer> &videoFrameBuffer,
914 const sp<AMessage> &outputFormat, int64_t /*timeUs*/, bool *done) {
915 if (outputFormat == NULL) {
916 return ERROR_MALFORMED;
917 }
918
919 int32_t width, height, stride, srcFormat;
920 if (outputFormat->findInt32("width", &width) == false) {
921 ALOGE("MediaImageDecoder::onOutputReceived:width is missing in outputFormat");
922 return ERROR_MALFORMED;
923 }
924 if (outputFormat->findInt32("height", &height) == false) {
925 ALOGE("MediaImageDecoder::onOutputReceived:height is missing in outputFormat");
926 return ERROR_MALFORMED;
927 }
928 if (outputFormat->findInt32("stride", &stride) == false) {
929 ALOGE("MediaImageDecoder::onOutputReceived:stride is missing in outputFormat");
930 return ERROR_MALFORMED;
931 }
932 if (outputFormat->findInt32("color-format", &srcFormat) == false) {
933 ALOGE("MediaImageDecoder::onOutputReceived: color format is missing in outputFormat");
934 return ERROR_MALFORMED;
935 }
936
937 uint32_t bitDepth = 8;
938 if (COLOR_FormatYUVP010 == srcFormat) {
939 bitDepth = 10;
940 }
941
942 if (mFrame == NULL) {
943 sp<IMemory> frameMem = allocVideoFrame(
944 trackMeta(), mWidth, mHeight, mTileWidth, mTileHeight, dstBpp(), bitDepth);
945
946 if (frameMem == nullptr) {
947 return NO_MEMORY;
948 }
949
950 mFrame = static_cast<VideoFrame*>(frameMem->unsecurePointer());
951
952 setFrame(frameMem);
953 }
954
955 ColorConverter converter((OMX_COLOR_FORMATTYPE)srcFormat, dstFormat());
956
957 uint32_t standard, range, transfer;
958 if (!outputFormat->findInt32("color-standard", (int32_t*)&standard)) {
959 standard = 0;
960 }
961 if (!outputFormat->findInt32("color-range", (int32_t*)&range)) {
962 range = 0;
963 }
964 if (!outputFormat->findInt32("color-transfer", (int32_t*)&transfer)) {
965 transfer = 0;
966 }
967 converter.setSrcColorSpace(standard, range, transfer);
968
969 int32_t crop_left, crop_top, crop_right, crop_bottom;
970 if (!outputFormat->findRect("crop", &crop_left, &crop_top, &crop_right, &crop_bottom)) {
971 crop_left = crop_top = 0;
972 crop_right = width - 1;
973 crop_bottom = height - 1;
974 }
975
976 int32_t slice_height;
977 if (outputFormat->findInt32("slice-height", &slice_height) && slice_height > 0) {
978 height = slice_height;
979 }
980
981 int32_t crop_width, crop_height;
982 crop_width = crop_right - crop_left + 1;
983 crop_height = crop_bottom - crop_top + 1;
984
985 int32_t dstLeft, dstTop, dstRight, dstBottom;
986 dstLeft = mTilesDecoded % mGridCols * crop_width;
987 dstTop = mTilesDecoded / mGridCols * crop_height;
988 dstRight = dstLeft + crop_width - 1;
989 dstBottom = dstTop + crop_height - 1;
990
991 // apply crop on bottom-right
992 // TODO: need to move this into the color converter itself.
993 if (dstRight >= mWidth) {
994 crop_right = crop_left + mWidth - dstLeft - 1;
995 dstRight = mWidth - 1;
996 }
997 if (dstBottom >= mHeight) {
998 crop_bottom = crop_top + mHeight - dstTop - 1;
999 dstBottom = mHeight - 1;
1000 }
1001
1002 *done = (++mTilesDecoded >= mTargetTiles);
1003
1004 if (converter.isValid()) {
1005 converter.convert(
1006 (const uint8_t *)videoFrameBuffer->data(),
1007 width, height, stride,
1008 crop_left, crop_top, crop_right, crop_bottom,
1009 mFrame->getFlattenedData(),
1010 mFrame->mWidth, mFrame->mHeight, mFrame->mRowBytes,
1011 dstLeft, dstTop, dstRight, dstBottom);
1012 return OK;
1013 }
1014
1015 ALOGE("Unable to convert from format 0x%08x to 0x%08x",
1016 srcFormat, dstFormat());
1017 return ERROR_UNSUPPORTED;
1018 }
1019
1020 } // namespace android
1021