1 /*
2 * Copyright (C) 2017 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 //#define LOG_NDEBUG 0
18 #define LOG_TAG "FrameDecoder"
19
20 #include "include/FrameDecoder.h"
21 #include "include/FrameCaptureLayer.h"
22 #include "include/HevcUtils.h"
23 #include <binder/MemoryBase.h>
24 #include <binder/MemoryHeapBase.h>
25 #include <gui/Surface.h>
26 #include <inttypes.h>
27 #include <mediadrm/ICrypto.h>
28 #include <media/IMediaSource.h>
29 #include <media/MediaCodecBuffer.h>
30 #include <media/stagefright/foundation/avc_utils.h>
31 #include <media/stagefright/foundation/ADebug.h>
32 #include <media/stagefright/foundation/AMessage.h>
33 #include <media/stagefright/foundation/ColorUtils.h>
34 #include <media/stagefright/ColorConverter.h>
35 #include <media/stagefright/FrameCaptureProcessor.h>
36 #include <media/stagefright/MediaBuffer.h>
37 #include <media/stagefright/MediaCodec.h>
38 #include <media/stagefright/MediaDefs.h>
39 #include <media/stagefright/MediaErrors.h>
40 #include <media/stagefright/Utils.h>
41 #include <private/media/VideoFrame.h>
42 #include <utils/Log.h>
43
44 namespace android {
45
46 static const int64_t kBufferTimeOutUs = 10000LL; // 10 msec
47 static const size_t kRetryCount = 100; // must be >0
48 static const int64_t kDefaultSampleDurationUs = 33333LL; // 33ms
49
allocVideoFrame(const sp<MetaData> & trackMeta,int32_t width,int32_t height,int32_t tileWidth,int32_t tileHeight,int32_t dstBpp,bool allocRotated,bool metaOnly)50 sp<IMemory> allocVideoFrame(const sp<MetaData>& trackMeta,
51 int32_t width, int32_t height, int32_t tileWidth, int32_t tileHeight,
52 int32_t dstBpp, bool allocRotated, bool metaOnly) {
53 int32_t rotationAngle;
54 if (!trackMeta->findInt32(kKeyRotation, &rotationAngle)) {
55 rotationAngle = 0; // By default, no rotation
56 }
57 uint32_t type;
58 const void *iccData;
59 size_t iccSize;
60 if (!trackMeta->findData(kKeyIccProfile, &type, &iccData, &iccSize)){
61 iccData = NULL;
62 iccSize = 0;
63 }
64
65 int32_t sarWidth, sarHeight;
66 int32_t displayWidth, displayHeight;
67 if (trackMeta->findInt32(kKeySARWidth, &sarWidth)
68 && trackMeta->findInt32(kKeySARHeight, &sarHeight)
69 && sarHeight != 0) {
70 int32_t multVal;
71 if (width < 0 || sarWidth < 0 ||
72 __builtin_mul_overflow(width, sarWidth, &multVal)) {
73 ALOGE("displayWidth overflow %dx%d", width, sarWidth);
74 return NULL;
75 }
76 displayWidth = (width * sarWidth) / sarHeight;
77 displayHeight = height;
78 } else if (trackMeta->findInt32(kKeyDisplayWidth, &displayWidth)
79 && trackMeta->findInt32(kKeyDisplayHeight, &displayHeight)
80 && displayWidth > 0 && displayHeight > 0
81 && width > 0 && height > 0) {
82 ALOGV("found display size %dx%d", displayWidth, displayHeight);
83 } else {
84 displayWidth = width;
85 displayHeight = height;
86 }
87
88 if (allocRotated && (rotationAngle == 90 || rotationAngle == 270)) {
89 int32_t tmp;
90 tmp = width; width = height; height = tmp;
91 tmp = displayWidth; displayWidth = displayHeight; displayHeight = tmp;
92 tmp = tileWidth; tileWidth = tileHeight; tileHeight = tmp;
93 rotationAngle = 0;
94 }
95
96 if (!metaOnly) {
97 int32_t multVal;
98 if (width < 0 || height < 0 || dstBpp < 0 ||
99 __builtin_mul_overflow(dstBpp, width, &multVal) ||
100 __builtin_mul_overflow(multVal, height, &multVal)) {
101 ALOGE("Frame size overflow %dx%d bpp %d", width, height, dstBpp);
102 return NULL;
103 }
104 }
105
106 VideoFrame frame(width, height, displayWidth, displayHeight,
107 tileWidth, tileHeight, rotationAngle, dstBpp, !metaOnly, iccSize);
108
109 size_t size = frame.getFlattenedSize();
110 sp<MemoryHeapBase> heap = new MemoryHeapBase(size, 0, "MetadataRetrieverClient");
111 if (heap == NULL) {
112 ALOGE("failed to create MemoryDealer");
113 return NULL;
114 }
115 sp<IMemory> frameMem = new MemoryBase(heap, 0, size);
116 if (frameMem == NULL || frameMem->unsecurePointer() == NULL) {
117 ALOGE("not enough memory for VideoFrame size=%zu", size);
118 return NULL;
119 }
120 VideoFrame* frameCopy = static_cast<VideoFrame*>(frameMem->unsecurePointer());
121 frameCopy->init(frame, iccData, iccSize);
122
123 return frameMem;
124 }
125
allocVideoFrame(const sp<MetaData> & trackMeta,int32_t width,int32_t height,int32_t tileWidth,int32_t tileHeight,int32_t dstBpp,bool allocRotated=false)126 sp<IMemory> allocVideoFrame(const sp<MetaData>& trackMeta,
127 int32_t width, int32_t height, int32_t tileWidth, int32_t tileHeight,
128 int32_t dstBpp, bool allocRotated = false) {
129 return allocVideoFrame(trackMeta, width, height, tileWidth, tileHeight, dstBpp,
130 allocRotated, false /*metaOnly*/);
131 }
132
allocMetaFrame(const sp<MetaData> & trackMeta,int32_t width,int32_t height,int32_t tileWidth,int32_t tileHeight,int32_t dstBpp)133 sp<IMemory> allocMetaFrame(const sp<MetaData>& trackMeta,
134 int32_t width, int32_t height, int32_t tileWidth, int32_t tileHeight,
135 int32_t dstBpp) {
136 return allocVideoFrame(trackMeta, width, height, tileWidth, tileHeight, dstBpp,
137 false /*allocRotated*/, true /*metaOnly*/);
138 }
139
isAvif(const sp<MetaData> & trackMeta)140 bool isAvif(const sp<MetaData> &trackMeta) {
141 const char *mime;
142 return trackMeta->findCString(kKeyMIMEType, &mime)
143 && (!strcasecmp(mime, MEDIA_MIMETYPE_VIDEO_AV1)
144 || !strcasecmp(mime, MEDIA_MIMETYPE_IMAGE_AVIF));
145 }
146
findThumbnailInfo(const sp<MetaData> & trackMeta,int32_t * width,int32_t * height,uint32_t * type=NULL,const void ** data=NULL,size_t * size=NULL)147 bool findThumbnailInfo(
148 const sp<MetaData> &trackMeta, int32_t *width, int32_t *height,
149 uint32_t *type = NULL, const void **data = NULL, size_t *size = NULL) {
150 uint32_t dummyType;
151 const void *dummyData;
152 size_t dummySize;
153 int codecConfigKey = isAvif(trackMeta) ? kKeyThumbnailAV1C : kKeyThumbnailHVCC;
154 return trackMeta->findInt32(kKeyThumbnailWidth, width)
155 && trackMeta->findInt32(kKeyThumbnailHeight, height)
156 && trackMeta->findData(codecConfigKey,
157 type ?: &dummyType, data ?: &dummyData, size ?: &dummySize);
158 }
159
findGridInfo(const sp<MetaData> & trackMeta,int32_t * tileWidth,int32_t * tileHeight,int32_t * gridRows,int32_t * gridCols)160 bool findGridInfo(const sp<MetaData> &trackMeta,
161 int32_t *tileWidth, int32_t *tileHeight, int32_t *gridRows, int32_t *gridCols) {
162 return trackMeta->findInt32(kKeyTileWidth, tileWidth) && (*tileWidth > 0)
163 && trackMeta->findInt32(kKeyTileHeight, tileHeight) && (*tileHeight > 0)
164 && trackMeta->findInt32(kKeyGridRows, gridRows) && (*gridRows > 0)
165 && trackMeta->findInt32(kKeyGridCols, gridCols) && (*gridCols > 0);
166 }
167
getDstColorFormat(android_pixel_format_t colorFormat,OMX_COLOR_FORMATTYPE * dstFormat,ui::PixelFormat * captureFormat,int32_t * dstBpp)168 bool getDstColorFormat(
169 android_pixel_format_t colorFormat,
170 OMX_COLOR_FORMATTYPE *dstFormat,
171 ui::PixelFormat *captureFormat,
172 int32_t *dstBpp) {
173 switch (colorFormat) {
174 case HAL_PIXEL_FORMAT_RGB_565:
175 {
176 *dstFormat = OMX_COLOR_Format16bitRGB565;
177 *captureFormat = ui::PixelFormat::RGB_565;
178 *dstBpp = 2;
179 return true;
180 }
181 case HAL_PIXEL_FORMAT_RGBA_8888:
182 {
183 *dstFormat = OMX_COLOR_Format32BitRGBA8888;
184 *captureFormat = ui::PixelFormat::RGBA_8888;
185 *dstBpp = 4;
186 return true;
187 }
188 case HAL_PIXEL_FORMAT_BGRA_8888:
189 {
190 *dstFormat = OMX_COLOR_Format32bitBGRA8888;
191 *captureFormat = ui::PixelFormat::BGRA_8888;
192 *dstBpp = 4;
193 return true;
194 }
195 default:
196 {
197 ALOGE("Unsupported color format: %d", colorFormat);
198 break;
199 }
200 }
201 return false;
202 }
203
204 //static
getMetadataOnly(const sp<MetaData> & trackMeta,int colorFormat,bool thumbnail)205 sp<IMemory> FrameDecoder::getMetadataOnly(
206 const sp<MetaData> &trackMeta, int colorFormat, bool thumbnail) {
207 OMX_COLOR_FORMATTYPE dstFormat;
208 ui::PixelFormat captureFormat;
209 int32_t dstBpp;
210 if (!getDstColorFormat((android_pixel_format_t)colorFormat,
211 &dstFormat, &captureFormat, &dstBpp)) {
212 return NULL;
213 }
214
215 int32_t width, height, tileWidth = 0, tileHeight = 0;
216 if (thumbnail) {
217 if (!findThumbnailInfo(trackMeta, &width, &height)) {
218 return NULL;
219 }
220 } else {
221 CHECK(trackMeta->findInt32(kKeyWidth, &width));
222 CHECK(trackMeta->findInt32(kKeyHeight, &height));
223
224 int32_t gridRows, gridCols;
225 if (!findGridInfo(trackMeta, &tileWidth, &tileHeight, &gridRows, &gridCols)) {
226 tileWidth = tileHeight = 0;
227 }
228 }
229
230 sp<IMemory> metaMem = allocMetaFrame(trackMeta, width, height, tileWidth, tileHeight, dstBpp);
231
232 // try to fill sequence meta's duration based on average frame rate,
233 // default to 33ms if frame rate is unavailable.
234 int32_t frameRate;
235 VideoFrame* meta = static_cast<VideoFrame*>(metaMem->unsecurePointer());
236 if (trackMeta->findInt32(kKeyFrameRate, &frameRate) && frameRate > 0) {
237 meta->mDurationUs = 1000000LL / frameRate;
238 } else {
239 meta->mDurationUs = kDefaultSampleDurationUs;
240 }
241 return metaMem;
242 }
243
FrameDecoder(const AString & componentName,const sp<MetaData> & trackMeta,const sp<IMediaSource> & source)244 FrameDecoder::FrameDecoder(
245 const AString &componentName,
246 const sp<MetaData> &trackMeta,
247 const sp<IMediaSource> &source)
248 : mComponentName(componentName),
249 mTrackMeta(trackMeta),
250 mSource(source),
251 mDstFormat(OMX_COLOR_Format16bitRGB565),
252 mDstBpp(2),
253 mHaveMoreInputs(true),
254 mFirstSample(true) {
255 }
256
~FrameDecoder()257 FrameDecoder::~FrameDecoder() {
258 if (mDecoder != NULL) {
259 mDecoder->release();
260 mSource->stop();
261 }
262 }
263
isHDR(const sp<AMessage> & format)264 bool isHDR(const sp<AMessage> &format) {
265 uint32_t standard, range, transfer;
266 if (!format->findInt32("color-standard", (int32_t*)&standard)) {
267 standard = 0;
268 }
269 if (!format->findInt32("color-range", (int32_t*)&range)) {
270 range = 0;
271 }
272 if (!format->findInt32("color-transfer", (int32_t*)&transfer)) {
273 transfer = 0;
274 }
275 return standard == ColorUtils::kColorStandardBT2020 &&
276 (transfer == ColorUtils::kColorTransferST2084 ||
277 transfer == ColorUtils::kColorTransferHLG);
278 }
279
init(int64_t frameTimeUs,int option,int colorFormat)280 status_t FrameDecoder::init(
281 int64_t frameTimeUs, int option, int colorFormat) {
282 if (!getDstColorFormat((android_pixel_format_t)colorFormat,
283 &mDstFormat, &mCaptureFormat, &mDstBpp)) {
284 return ERROR_UNSUPPORTED;
285 }
286
287 sp<AMessage> videoFormat = onGetFormatAndSeekOptions(
288 frameTimeUs, option, &mReadOptions, &mSurface);
289 if (videoFormat == NULL) {
290 ALOGE("video format or seek mode not supported");
291 return ERROR_UNSUPPORTED;
292 }
293
294 status_t err;
295 sp<ALooper> looper = new ALooper;
296 looper->start();
297 sp<MediaCodec> decoder = MediaCodec::CreateByComponentName(
298 looper, mComponentName, &err);
299 if (decoder.get() == NULL || err != OK) {
300 ALOGW("Failed to instantiate decoder [%s]", mComponentName.c_str());
301 return (decoder.get() == NULL) ? NO_MEMORY : err;
302 }
303
304 err = decoder->configure(
305 videoFormat, mSurface, NULL /* crypto */, 0 /* flags */);
306 if (err != OK) {
307 ALOGW("configure returned error %d (%s)", err, asString(err));
308 decoder->release();
309 return err;
310 }
311
312 err = decoder->start();
313 if (err != OK) {
314 ALOGW("start returned error %d (%s)", err, asString(err));
315 decoder->release();
316 return err;
317 }
318
319 err = mSource->start();
320 if (err != OK) {
321 ALOGW("source failed to start: %d (%s)", err, asString(err));
322 decoder->release();
323 return err;
324 }
325 mDecoder = decoder;
326
327 return OK;
328 }
329
extractFrame(FrameRect * rect)330 sp<IMemory> FrameDecoder::extractFrame(FrameRect *rect) {
331 status_t err = onExtractRect(rect);
332 if (err == OK) {
333 err = extractInternal();
334 }
335 if (err != OK) {
336 return NULL;
337 }
338
339 return mFrameMemory;
340 }
341
extractInternal()342 status_t FrameDecoder::extractInternal() {
343 status_t err = OK;
344 bool done = false;
345 size_t retriesLeft = kRetryCount;
346 do {
347 size_t index;
348 int64_t ptsUs = 0LL;
349 uint32_t flags = 0;
350
351 // Queue as many inputs as we possibly can, then block on dequeuing
352 // outputs. After getting each output, come back and queue the inputs
353 // again to keep the decoder busy.
354 while (mHaveMoreInputs) {
355 err = mDecoder->dequeueInputBuffer(&index, 0);
356 if (err != OK) {
357 ALOGV("Timed out waiting for input");
358 if (retriesLeft) {
359 err = OK;
360 }
361 break;
362 }
363 sp<MediaCodecBuffer> codecBuffer;
364 err = mDecoder->getInputBuffer(index, &codecBuffer);
365 if (err != OK) {
366 ALOGE("failed to get input buffer %zu", index);
367 break;
368 }
369
370 MediaBufferBase *mediaBuffer = NULL;
371
372 err = mSource->read(&mediaBuffer, &mReadOptions);
373 mReadOptions.clearSeekTo();
374 if (err != OK) {
375 mHaveMoreInputs = false;
376 if (!mFirstSample && err == ERROR_END_OF_STREAM) {
377 (void)mDecoder->queueInputBuffer(
378 index, 0, 0, 0, MediaCodec::BUFFER_FLAG_EOS);
379 err = OK;
380 } else {
381 ALOGW("Input Error: err=%d", err);
382 }
383 break;
384 }
385
386 if (mediaBuffer->range_length() > codecBuffer->capacity()) {
387 ALOGE("buffer size (%zu) too large for codec input size (%zu)",
388 mediaBuffer->range_length(), codecBuffer->capacity());
389 mHaveMoreInputs = false;
390 err = BAD_VALUE;
391 } else {
392 codecBuffer->setRange(0, mediaBuffer->range_length());
393
394 CHECK(mediaBuffer->meta_data().findInt64(kKeyTime, &ptsUs));
395 memcpy(codecBuffer->data(),
396 (const uint8_t*)mediaBuffer->data() + mediaBuffer->range_offset(),
397 mediaBuffer->range_length());
398
399 onInputReceived(codecBuffer, mediaBuffer->meta_data(), mFirstSample, &flags);
400 mFirstSample = false;
401 }
402
403 mediaBuffer->release();
404
405 if (mHaveMoreInputs) {
406 ALOGV("QueueInput: size=%zu ts=%" PRId64 " us flags=%x",
407 codecBuffer->size(), ptsUs, flags);
408
409 err = mDecoder->queueInputBuffer(
410 index,
411 codecBuffer->offset(),
412 codecBuffer->size(),
413 ptsUs,
414 flags);
415
416 if (flags & MediaCodec::BUFFER_FLAG_EOS) {
417 mHaveMoreInputs = false;
418 }
419 }
420 }
421
422 while (err == OK) {
423 size_t offset, size;
424 // wait for a decoded buffer
425 err = mDecoder->dequeueOutputBuffer(
426 &index,
427 &offset,
428 &size,
429 &ptsUs,
430 &flags,
431 kBufferTimeOutUs);
432
433 if (err == INFO_FORMAT_CHANGED) {
434 ALOGV("Received format change");
435 err = mDecoder->getOutputFormat(&mOutputFormat);
436 } else if (err == INFO_OUTPUT_BUFFERS_CHANGED) {
437 ALOGV("Output buffers changed");
438 err = OK;
439 } else {
440 if (err == -EAGAIN /* INFO_TRY_AGAIN_LATER */ && --retriesLeft > 0) {
441 ALOGV("Timed-out waiting for output.. retries left = %zu", retriesLeft);
442 err = OK;
443 } else if (err == OK) {
444 // If we're seeking with CLOSEST option and obtained a valid targetTimeUs
445 // from the extractor, decode to the specified frame. Otherwise we're done.
446 ALOGV("Received an output buffer, timeUs=%lld", (long long)ptsUs);
447 sp<MediaCodecBuffer> videoFrameBuffer;
448 err = mDecoder->getOutputBuffer(index, &videoFrameBuffer);
449 if (err != OK) {
450 ALOGE("failed to get output buffer %zu", index);
451 break;
452 }
453 if (mSurface != nullptr) {
454 mDecoder->renderOutputBufferAndRelease(index);
455 err = onOutputReceived(videoFrameBuffer, mOutputFormat, ptsUs, &done);
456 } else {
457 err = onOutputReceived(videoFrameBuffer, mOutputFormat, ptsUs, &done);
458 mDecoder->releaseOutputBuffer(index);
459 }
460 } else {
461 ALOGW("Received error %d (%s) instead of output", err, asString(err));
462 done = true;
463 }
464 break;
465 }
466 }
467 } while (err == OK && !done);
468
469 if (err != OK) {
470 ALOGE("failed to get video frame (err %d)", err);
471 }
472
473 return err;
474 }
475
476 //////////////////////////////////////////////////////////////////////
477
VideoFrameDecoder(const AString & componentName,const sp<MetaData> & trackMeta,const sp<IMediaSource> & source)478 VideoFrameDecoder::VideoFrameDecoder(
479 const AString &componentName,
480 const sp<MetaData> &trackMeta,
481 const sp<IMediaSource> &source)
482 : FrameDecoder(componentName, trackMeta, source),
483 mFrame(NULL),
484 mIsAvc(false),
485 mIsHevc(false),
486 mSeekMode(MediaSource::ReadOptions::SEEK_PREVIOUS_SYNC),
487 mTargetTimeUs(-1LL),
488 mDefaultSampleDurationUs(0) {
489 }
490
onGetFormatAndSeekOptions(int64_t frameTimeUs,int seekMode,MediaSource::ReadOptions * options,sp<Surface> * window)491 sp<AMessage> VideoFrameDecoder::onGetFormatAndSeekOptions(
492 int64_t frameTimeUs, int seekMode,
493 MediaSource::ReadOptions *options,
494 sp<Surface> *window) {
495 mSeekMode = static_cast<MediaSource::ReadOptions::SeekMode>(seekMode);
496 if (mSeekMode < MediaSource::ReadOptions::SEEK_PREVIOUS_SYNC ||
497 mSeekMode > MediaSource::ReadOptions::SEEK_FRAME_INDEX) {
498 ALOGE("Unknown seek mode: %d", mSeekMode);
499 return NULL;
500 }
501
502 const char *mime;
503 if (!trackMeta()->findCString(kKeyMIMEType, &mime)) {
504 ALOGE("Could not find mime type");
505 return NULL;
506 }
507
508 mIsAvc = !strcasecmp(mime, MEDIA_MIMETYPE_VIDEO_AVC);
509 mIsHevc = !strcasecmp(mime, MEDIA_MIMETYPE_VIDEO_HEVC);
510
511 if (frameTimeUs < 0) {
512 int64_t thumbNailTime = -1ll;
513 if (!trackMeta()->findInt64(kKeyThumbnailTime, &thumbNailTime)
514 || thumbNailTime < 0) {
515 thumbNailTime = 0;
516 }
517 options->setSeekTo(thumbNailTime, mSeekMode);
518 } else {
519 options->setSeekTo(frameTimeUs, mSeekMode);
520 }
521
522 sp<AMessage> videoFormat;
523 if (convertMetaDataToMessage(trackMeta(), &videoFormat) != OK) {
524 ALOGE("b/23680780");
525 ALOGW("Failed to convert meta data to message");
526 return NULL;
527 }
528
529 // TODO: Use Flexible color instead
530 videoFormat->setInt32("color-format", OMX_COLOR_FormatYUV420Planar);
531
532 // For the thumbnail extraction case, try to allocate single buffer in both
533 // input and output ports, if seeking to a sync frame. NOTE: This request may
534 // fail if component requires more than that for decoding.
535 bool isSeekingClosest = (mSeekMode == MediaSource::ReadOptions::SEEK_CLOSEST)
536 || (mSeekMode == MediaSource::ReadOptions::SEEK_FRAME_INDEX);
537 if (!isSeekingClosest) {
538 videoFormat->setInt32("android._num-input-buffers", 1);
539 videoFormat->setInt32("android._num-output-buffers", 1);
540 }
541
542 if (isHDR(videoFormat)) {
543 *window = initSurface();
544 if (*window == NULL) {
545 ALOGE("Failed to init surface control for HDR, fallback to non-hdr");
546 } else {
547 videoFormat->setInt32("color-format", OMX_COLOR_FormatAndroidOpaque);
548 }
549 }
550
551 int32_t frameRate;
552 if (trackMeta()->findInt32(kKeyFrameRate, &frameRate) && frameRate > 0) {
553 mDefaultSampleDurationUs = 1000000LL / frameRate;
554 } else {
555 mDefaultSampleDurationUs = kDefaultSampleDurationUs;
556 }
557
558 return videoFormat;
559 }
560
onInputReceived(const sp<MediaCodecBuffer> & codecBuffer,MetaDataBase & sampleMeta,bool firstSample,uint32_t * flags)561 status_t VideoFrameDecoder::onInputReceived(
562 const sp<MediaCodecBuffer> &codecBuffer,
563 MetaDataBase &sampleMeta, bool firstSample, uint32_t *flags) {
564 bool isSeekingClosest = (mSeekMode == MediaSource::ReadOptions::SEEK_CLOSEST)
565 || (mSeekMode == MediaSource::ReadOptions::SEEK_FRAME_INDEX);
566
567 if (firstSample && isSeekingClosest) {
568 sampleMeta.findInt64(kKeyTargetTime, &mTargetTimeUs);
569 ALOGV("Seeking closest: targetTimeUs=%lld", (long long)mTargetTimeUs);
570 }
571
572 if (!isSeekingClosest
573 && ((mIsAvc && IsIDR(codecBuffer->data(), codecBuffer->size()))
574 || (mIsHevc && IsIDR(
575 codecBuffer->data(), codecBuffer->size())))) {
576 // Only need to decode one IDR frame, unless we're seeking with CLOSEST
577 // option, in which case we need to actually decode to targetTimeUs.
578 *flags |= MediaCodec::BUFFER_FLAG_EOS;
579 }
580 int64_t durationUs;
581 if (sampleMeta.findInt64(kKeyDuration, &durationUs)) {
582 mSampleDurations.push_back(durationUs);
583 } else {
584 mSampleDurations.push_back(mDefaultSampleDurationUs);
585 }
586 return OK;
587 }
588
onOutputReceived(const sp<MediaCodecBuffer> & videoFrameBuffer,const sp<AMessage> & outputFormat,int64_t timeUs,bool * done)589 status_t VideoFrameDecoder::onOutputReceived(
590 const sp<MediaCodecBuffer> &videoFrameBuffer,
591 const sp<AMessage> &outputFormat,
592 int64_t timeUs, bool *done) {
593 int64_t durationUs = mDefaultSampleDurationUs;
594 if (!mSampleDurations.empty()) {
595 durationUs = *mSampleDurations.begin();
596 mSampleDurations.erase(mSampleDurations.begin());
597 }
598 bool shouldOutput = (mTargetTimeUs < 0LL) || (timeUs >= mTargetTimeUs);
599
600 // If this is not the target frame, skip color convert.
601 if (!shouldOutput) {
602 *done = false;
603 return OK;
604 }
605
606 *done = true;
607
608 if (outputFormat == NULL) {
609 return ERROR_MALFORMED;
610 }
611
612 int32_t width, height, stride, srcFormat;
613 if (!outputFormat->findInt32("width", &width) ||
614 !outputFormat->findInt32("height", &height) ||
615 !outputFormat->findInt32("color-format", &srcFormat)) {
616 ALOGE("format missing dimension or color: %s",
617 outputFormat->debugString().c_str());
618 return ERROR_MALFORMED;
619 }
620
621 if (!outputFormat->findInt32("stride", &stride)) {
622 if (mCaptureLayer == NULL) {
623 ALOGE("format must have stride for byte buffer mode: %s",
624 outputFormat->debugString().c_str());
625 return ERROR_MALFORMED;
626 }
627 // for surface output, set stride to width, we don't actually need it.
628 stride = width;
629 }
630
631 int32_t crop_left, crop_top, crop_right, crop_bottom;
632 if (!outputFormat->findRect("crop", &crop_left, &crop_top, &crop_right, &crop_bottom)) {
633 crop_left = crop_top = 0;
634 crop_right = width - 1;
635 crop_bottom = height - 1;
636 }
637
638 if (mFrame == NULL) {
639 sp<IMemory> frameMem = allocVideoFrame(
640 trackMeta(),
641 (crop_right - crop_left + 1),
642 (crop_bottom - crop_top + 1),
643 0,
644 0,
645 dstBpp(),
646 mCaptureLayer != nullptr /*allocRotated*/);
647 if (frameMem == nullptr) {
648 return NO_MEMORY;
649 }
650
651 mFrame = static_cast<VideoFrame*>(frameMem->unsecurePointer());
652
653 setFrame(frameMem);
654 }
655
656 mFrame->mDurationUs = durationUs;
657
658 if (mCaptureLayer != nullptr) {
659 return captureSurface();
660 }
661
662 ColorConverter converter((OMX_COLOR_FORMATTYPE)srcFormat, dstFormat());
663
664 uint32_t standard, range, transfer;
665 if (!outputFormat->findInt32("color-standard", (int32_t*)&standard)) {
666 standard = 0;
667 }
668 if (!outputFormat->findInt32("color-range", (int32_t*)&range)) {
669 range = 0;
670 }
671 if (!outputFormat->findInt32("color-transfer", (int32_t*)&transfer)) {
672 transfer = 0;
673 }
674 converter.setSrcColorSpace(standard, range, transfer);
675
676 if (converter.isValid()) {
677 converter.convert(
678 (const uint8_t *)videoFrameBuffer->data(),
679 width, height, stride,
680 crop_left, crop_top, crop_right, crop_bottom,
681 mFrame->getFlattenedData(),
682 mFrame->mWidth, mFrame->mHeight, mFrame->mRowBytes,
683 // since the frame is allocated with top-left adjusted,
684 // the dst rect should start at {0,0} as well.
685 0, 0, mFrame->mWidth - 1, mFrame->mHeight - 1);
686 return OK;
687 }
688
689 ALOGE("Unable to convert from format 0x%08x to 0x%08x",
690 srcFormat, dstFormat());
691 return ERROR_UNSUPPORTED;
692 }
693
initSurface()694 sp<Surface> VideoFrameDecoder::initSurface() {
695 // create the consumer listener interface, and hold sp so that this
696 // interface lives as long as the GraphicBufferSource.
697 sp<FrameCaptureLayer> captureLayer = new FrameCaptureLayer();
698 if (captureLayer->init() != OK) {
699 ALOGE("failed to init capture layer");
700 return nullptr;
701 }
702 mCaptureLayer = captureLayer;
703
704 return captureLayer->getSurface();
705 }
706
captureSurface()707 status_t VideoFrameDecoder::captureSurface() {
708 sp<GraphicBuffer> outBuffer;
709 status_t err = mCaptureLayer->capture(
710 captureFormat(), Rect(0, 0, mFrame->mWidth, mFrame->mHeight), &outBuffer);
711
712 if (err != OK) {
713 ALOGE("failed to capture layer (err %d)", err);
714 return err;
715 }
716
717 ALOGV("capture: %dx%d, format %d, stride %d",
718 outBuffer->getWidth(),
719 outBuffer->getHeight(),
720 outBuffer->getPixelFormat(),
721 outBuffer->getStride());
722
723 uint8_t *base;
724 int32_t outBytesPerPixel, outBytesPerStride;
725 err = outBuffer->lock(
726 GraphicBuffer::USAGE_SW_READ_OFTEN,
727 reinterpret_cast<void**>(&base),
728 &outBytesPerPixel,
729 &outBytesPerStride);
730 if (err != OK) {
731 ALOGE("failed to lock graphic buffer: err %d", err);
732 return err;
733 }
734
735 uint8_t *dst = mFrame->getFlattenedData();
736 for (size_t y = 0 ; y < fmin(mFrame->mHeight, outBuffer->getHeight()) ; y++) {
737 memcpy(dst, base, fmin(mFrame->mWidth, outBuffer->getWidth()) * mFrame->mBytesPerPixel);
738 dst += mFrame->mRowBytes;
739 base += outBuffer->getStride() * mFrame->mBytesPerPixel;
740 }
741 outBuffer->unlock();
742 return OK;
743 }
744
745 ////////////////////////////////////////////////////////////////////////
746
MediaImageDecoder(const AString & componentName,const sp<MetaData> & trackMeta,const sp<IMediaSource> & source)747 MediaImageDecoder::MediaImageDecoder(
748 const AString &componentName,
749 const sp<MetaData> &trackMeta,
750 const sp<IMediaSource> &source)
751 : FrameDecoder(componentName, trackMeta, source),
752 mFrame(NULL),
753 mWidth(0),
754 mHeight(0),
755 mGridRows(1),
756 mGridCols(1),
757 mTileWidth(0),
758 mTileHeight(0),
759 mTilesDecoded(0),
760 mTargetTiles(0) {
761 }
762
onGetFormatAndSeekOptions(int64_t frameTimeUs,int,MediaSource::ReadOptions * options,sp<Surface> *)763 sp<AMessage> MediaImageDecoder::onGetFormatAndSeekOptions(
764 int64_t frameTimeUs, int /*seekMode*/,
765 MediaSource::ReadOptions *options, sp<Surface> * /*window*/) {
766 sp<MetaData> overrideMeta;
767 if (frameTimeUs < 0) {
768 uint32_t type;
769 const void *data;
770 size_t size;
771
772 // if we have a stand-alone thumbnail, set up the override meta,
773 // and set seekTo time to -1.
774 if (!findThumbnailInfo(trackMeta(), &mWidth, &mHeight, &type, &data, &size)) {
775 ALOGE("Thumbnail not available");
776 return NULL;
777 }
778 overrideMeta = new MetaData(*(trackMeta()));
779 overrideMeta->remove(kKeyDisplayWidth);
780 overrideMeta->remove(kKeyDisplayHeight);
781 overrideMeta->setInt32(kKeyWidth, mWidth);
782 overrideMeta->setInt32(kKeyHeight, mHeight);
783 // The AV1 codec configuration data is passed via CSD0 to the AV1
784 // decoder.
785 const int codecConfigKey = isAvif(trackMeta()) ? kKeyOpaqueCSD0 : kKeyHVCC;
786 overrideMeta->setData(codecConfigKey, type, data, size);
787 options->setSeekTo(-1);
788 } else {
789 CHECK(trackMeta()->findInt32(kKeyWidth, &mWidth));
790 CHECK(trackMeta()->findInt32(kKeyHeight, &mHeight));
791
792 options->setSeekTo(frameTimeUs);
793 }
794
795 mGridRows = mGridCols = 1;
796 if (overrideMeta == NULL) {
797 // check if we're dealing with a tiled heif
798 int32_t tileWidth, tileHeight, gridRows, gridCols;
799 if (findGridInfo(trackMeta(), &tileWidth, &tileHeight, &gridRows, &gridCols)) {
800 if (mWidth <= tileWidth * gridCols && mHeight <= tileHeight * gridRows) {
801 ALOGV("grid: %dx%d, tile size: %dx%d, picture size: %dx%d",
802 gridCols, gridRows, tileWidth, tileHeight, mWidth, mHeight);
803
804 overrideMeta = new MetaData(*(trackMeta()));
805 overrideMeta->setInt32(kKeyWidth, tileWidth);
806 overrideMeta->setInt32(kKeyHeight, tileHeight);
807 mTileWidth = tileWidth;
808 mTileHeight = tileHeight;
809 mGridCols = gridCols;
810 mGridRows = gridRows;
811 } else {
812 ALOGW("ignore bad grid: %dx%d, tile size: %dx%d, picture size: %dx%d",
813 gridCols, gridRows, tileWidth, tileHeight, mWidth, mHeight);
814 }
815 }
816 if (overrideMeta == NULL) {
817 overrideMeta = trackMeta();
818 }
819 }
820 mTargetTiles = mGridCols * mGridRows;
821
822 sp<AMessage> videoFormat;
823 if (convertMetaDataToMessage(overrideMeta, &videoFormat) != OK) {
824 ALOGE("b/23680780");
825 ALOGW("Failed to convert meta data to message");
826 return NULL;
827 }
828
829 // TODO: Use Flexible color instead
830 videoFormat->setInt32("color-format", OMX_COLOR_FormatYUV420Planar);
831
832 if ((mGridRows == 1) && (mGridCols == 1)) {
833 videoFormat->setInt32("android._num-input-buffers", 1);
834 videoFormat->setInt32("android._num-output-buffers", 1);
835 }
836 return videoFormat;
837 }
838
onExtractRect(FrameRect * rect)839 status_t MediaImageDecoder::onExtractRect(FrameRect *rect) {
840 // TODO:
841 // This callback is for verifying whether we can decode the rect,
842 // and if so, set up the internal variables for decoding.
843 // Currently, rect decoding is restricted to sequentially decoding one
844 // row of tiles at a time. We can't decode arbitrary rects, as the image
845 // track doesn't yet support seeking by tiles. So all we do here is to
846 // verify the rect against what we expect.
847 // When seeking by tile is supported, this code should be updated to
848 // set the seek parameters.
849 if (rect == NULL) {
850 if (mTilesDecoded > 0) {
851 return ERROR_UNSUPPORTED;
852 }
853 mTargetTiles = mGridRows * mGridCols;
854 return OK;
855 }
856
857 if (mTileWidth <= 0 || mTileHeight <=0) {
858 return ERROR_UNSUPPORTED;
859 }
860
861 int32_t row = mTilesDecoded / mGridCols;
862 int32_t expectedTop = row * mTileHeight;
863 int32_t expectedBot = (row + 1) * mTileHeight;
864 if (expectedBot > mHeight) {
865 expectedBot = mHeight;
866 }
867 if (rect->left != 0 || rect->top != expectedTop
868 || rect->right != mWidth || rect->bottom != expectedBot) {
869 ALOGE("currently only support sequential decoding of slices");
870 return ERROR_UNSUPPORTED;
871 }
872
873 // advance one row
874 mTargetTiles = mTilesDecoded + mGridCols;
875 return OK;
876 }
877
onOutputReceived(const sp<MediaCodecBuffer> & videoFrameBuffer,const sp<AMessage> & outputFormat,int64_t,bool * done)878 status_t MediaImageDecoder::onOutputReceived(
879 const sp<MediaCodecBuffer> &videoFrameBuffer,
880 const sp<AMessage> &outputFormat, int64_t /*timeUs*/, bool *done) {
881 if (outputFormat == NULL) {
882 return ERROR_MALFORMED;
883 }
884
885 int32_t width, height, stride;
886 CHECK(outputFormat->findInt32("width", &width));
887 CHECK(outputFormat->findInt32("height", &height));
888 CHECK(outputFormat->findInt32("stride", &stride));
889
890 if (mFrame == NULL) {
891 sp<IMemory> frameMem = allocVideoFrame(
892 trackMeta(), mWidth, mHeight, mTileWidth, mTileHeight, dstBpp());
893
894 if (frameMem == nullptr) {
895 return NO_MEMORY;
896 }
897
898 mFrame = static_cast<VideoFrame*>(frameMem->unsecurePointer());
899
900 setFrame(frameMem);
901 }
902
903 int32_t srcFormat;
904 CHECK(outputFormat->findInt32("color-format", &srcFormat));
905
906 ColorConverter converter((OMX_COLOR_FORMATTYPE)srcFormat, dstFormat());
907
908 uint32_t standard, range, transfer;
909 if (!outputFormat->findInt32("color-standard", (int32_t*)&standard)) {
910 standard = 0;
911 }
912 if (!outputFormat->findInt32("color-range", (int32_t*)&range)) {
913 range = 0;
914 }
915 if (!outputFormat->findInt32("color-transfer", (int32_t*)&transfer)) {
916 transfer = 0;
917 }
918 converter.setSrcColorSpace(standard, range, transfer);
919
920 int32_t crop_left, crop_top, crop_right, crop_bottom;
921 if (!outputFormat->findRect("crop", &crop_left, &crop_top, &crop_right, &crop_bottom)) {
922 crop_left = crop_top = 0;
923 crop_right = width - 1;
924 crop_bottom = height - 1;
925 }
926
927 int32_t crop_width, crop_height;
928 crop_width = crop_right - crop_left + 1;
929 crop_height = crop_bottom - crop_top + 1;
930
931 int32_t dstLeft, dstTop, dstRight, dstBottom;
932 dstLeft = mTilesDecoded % mGridCols * crop_width;
933 dstTop = mTilesDecoded / mGridCols * crop_height;
934 dstRight = dstLeft + crop_width - 1;
935 dstBottom = dstTop + crop_height - 1;
936
937 // apply crop on bottom-right
938 // TODO: need to move this into the color converter itself.
939 if (dstRight >= mWidth) {
940 crop_right = crop_left + mWidth - dstLeft - 1;
941 dstRight = mWidth - 1;
942 }
943 if (dstBottom >= mHeight) {
944 crop_bottom = crop_top + mHeight - dstTop - 1;
945 dstBottom = mHeight - 1;
946 }
947
948 *done = (++mTilesDecoded >= mTargetTiles);
949
950 if (converter.isValid()) {
951 converter.convert(
952 (const uint8_t *)videoFrameBuffer->data(),
953 width, height, stride,
954 crop_left, crop_top, crop_right, crop_bottom,
955 mFrame->getFlattenedData(),
956 mFrame->mWidth, mFrame->mHeight, mFrame->mRowBytes,
957 dstLeft, dstTop, dstRight, dstBottom);
958 return OK;
959 }
960
961 ALOGE("Unable to convert from format 0x%08x to 0x%08x",
962 srcFormat, dstFormat());
963 return ERROR_UNSUPPORTED;
964 }
965
966 } // namespace android
967