1 // Copyright (C) 2021 The Android Open Source Project
2 //
3 // Licensed under the Apache License, Version 2.0 (the "License");
4 // you may not use this file except in compliance with the License.
5 // You may obtain a copy of the License at
6 //
7 // http://www.apache.org/licenses/LICENSE-2.0
8 //
9 // Unless required by applicable law or agreed to in writing, software
10 // distributed under the License is distributed on an "AS IS" BASIS,
11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 // See the License for the specific language governing permissions and
13 // limitations under the License.
14
15 #include "host-common/MediaVideoToolBoxUtils.h"
16 #include "host-common/MediaVideoToolBoxVideoHelper.h"
17 #include "host-common/YuvConverter.h"
18 #include "android/utils/debug.h"
19
20 #define MEDIA_VTB_DEBUG 0
21
22 #if MEDIA_VTB_DEBUG
23 #define VTB_DPRINT(fmt, ...) \
24 fprintf(stderr, "media-vtb-video-helper: %s:%d " fmt "\n", __func__, \
25 __LINE__, ##__VA_ARGS__);
26 #else
27 #define VTB_DPRINT(fmt, ...)
28 #endif
29
30 namespace android {
31 namespace emulation {
32
33 using TextureFrame = MediaTexturePool::TextureFrame;
34 using FrameInfo = MediaSnapshotState::FrameInfo;
35 using ColorAspects = MediaSnapshotState::ColorAspects;
36 using H264NaluType = H264NaluParser::H264NaluType;
37
MediaVideoToolBoxVideoHelper(int w,int h,OutputTreatmentMode oMode,FrameStorageMode fMode)38 MediaVideoToolBoxVideoHelper::MediaVideoToolBoxVideoHelper(
39 int w,
40 int h,
41 OutputTreatmentMode oMode,
42 FrameStorageMode fMode)
43 : mOutputWidth(w),
44 mOutputHeight(h),
45 mUseGpuTexture(fMode == FrameStorageMode::USE_GPU_TEXTURE) {
46 mIgnoreDecoderOutput = (oMode == OutputTreatmentMode::IGNORE_RESULT);
47 }
48
~MediaVideoToolBoxVideoHelper()49 MediaVideoToolBoxVideoHelper::~MediaVideoToolBoxVideoHelper() {
50 deInit();
51 }
52
deInit()53 void MediaVideoToolBoxVideoHelper::deInit() {
54 VTB_DPRINT("deInit calling");
55
56 resetDecoderSession();
57
58 resetFormatDesc();
59
60 mVtbReady = false;
61
62 mFfmpegVideoHelper.reset();
63
64 mSavedDecodedFrames.clear();
65 }
66
init()67 bool MediaVideoToolBoxVideoHelper::init() {
68 VTB_DPRINT("init calling");
69 mVtbReady = false;
70 return true;
71 }
72
dumpBytes(const uint8_t * img,size_t szBytes,bool all=false)73 static void dumpBytes(const uint8_t* img, size_t szBytes, bool all = false) {
74 printf("data=");
75 size_t numBytes = szBytes;
76 if (!all) {
77 numBytes = 32;
78 }
79
80 for (size_t i = 0; i < (numBytes > szBytes ? szBytes : numBytes); ++i) {
81 if (i % 8 == 0) {
82 printf("\n");
83 }
84 printf("0x%02x ", img[i]);
85 }
86 printf("\n");
87
88 fflush(stdout);
89 }
90
extractFrameInfo()91 void MediaVideoToolBoxVideoHelper::extractFrameInfo() {
92 // at this moment, ffmpeg should have decoded the first frame to obtain
93 // w/h/colorspace info and number of b frame buffers
94
95 mFfmpegVideoHelper->flush();
96 MediaSnapshotState::FrameInfo frame;
97 bool success = mFfmpegVideoHelper->receiveFrame(&frame);
98 if (success) {
99 mColorAspects = frame.color;
100 mVtbBufferSize = mFfmpegVideoHelper->frameReorderBufferSize();
101 VTB_DPRINT("vtb buffer size is %d", mVtbBufferSize);
102 }
103 }
104
decode(const uint8_t * frame,size_t szBytes,uint64_t inputPts)105 void MediaVideoToolBoxVideoHelper::decode(const uint8_t* frame,
106 size_t szBytes,
107 uint64_t inputPts) {
108 VTB_DPRINT("%s(frame=%p, sz=%zu)", __func__, frame, szBytes);
109
110 ++mTotalFrames;
111 const bool parseOk = parseInputFrames(frame, szBytes);
112 if (!parseOk) {
113 // cannot parse, probably cannot decode either
114 // just fail
115 VTB_DPRINT("Failed to parse frame=%p, sz=%zu, give up.", frame, szBytes);
116 mIsGood = false;
117 return;
118 }
119
120 // has to go in the FIFO order
121 for (int i = 0; i < mInputFrames.size(); ++i) {
122 InputFrame& f = mInputFrames[i];
123 switch (f.type) {
124 case H264NaluType::SPS:
125 mSPS.assign(f.data, f.data + f.size);
126 VTB_DPRINT("this is an SPS frame");
127 // dumpBytes(mSPS.data(), mSPS.size());
128 mVtbReady = false;
129 { // create ffmpeg decoder with only 1 thread to avoid frame delay
130 constexpr int numthreads = 1;
131 mFfmpegVideoHelper.reset(new MediaFfmpegVideoHelper(264, numthreads));
132 mFfmpegVideoHelper->init();
133 }
134 break;
135 case H264NaluType::PPS:
136 VTB_DPRINT("this is an PPS frame");
137 mPPS.assign(f.data, f.data + f.size);
138 // dumpBytes(mPPS.data(), mPPS.size());
139 mVtbReady = false;
140 break;
141 case H264NaluType::SEI:
142 VTB_DPRINT("this is SEI frame");
143 break;
144 case H264NaluType::CodedSliceIDR:
145 VTB_DPRINT("this is an IDR frame");
146 if (mFfmpegVideoHelper) {
147 mFfmpegVideoHelper->decode(frame, szBytes, inputPts);
148 extractFrameInfo();
149 mFfmpegVideoHelper.reset();
150 }
151 if (!mVtbReady) {
152 createCMFormatDescription();
153 Boolean needNewSession = ( VTDecompressionSessionCanAcceptFormatDescription(mDecoderSession, mCmFmtDesc) == false);
154 if (needNewSession) {
155 resetDecoderSession();
156 }
157 if (!mDecoderSession) {
158 // all the previously decoded frames need to be
159 // retrieved
160 flush();
161 recreateDecompressionSession();
162 }
163 if (mIsGood) {
164 mVtbReady = true;
165 }
166 }
167 handleIDRFrame(f.data, f.size, inputPts);
168 break;
169 case H264NaluType::CodedSliceNonIDR:
170 VTB_DPRINT("this is an non-IDR frame");
171 handleIDRFrame(f.data, f.size, inputPts);
172 break;
173 default:
174 VTB_DPRINT("Support for nalu_type=%u not implemented",
175 (uint8_t)f.type);
176 // cannot handle such video, give up so we can fall
177 // back to ffmpeg
178 mIsGood = false;
179 break;
180 }
181 }
182
183 if (mFfmpegVideoHelper) {
184 // we have not reached idr frame yet, keep decoding
185 mFfmpegVideoHelper->decode(frame, szBytes, inputPts);
186 }
187
188 mInputFrames.clear();
189 }
190
flush()191 void MediaVideoToolBoxVideoHelper::flush() {
192 VTB_DPRINT("started flushing");
193 for (auto& iter : mVtbBufferMap) {
194 mSavedDecodedFrames.push_back(std::move(iter.second));
195 }
196 mVtbBufferMap.clear();
197 VTB_DPRINT("done one flushing");
198 }
199
200 //------------------------ private helper functions
201 //-----------------------------------------
202
handleIDRFrame(const uint8_t * ptr,size_t szBytes,uint64_t pts)203 void MediaVideoToolBoxVideoHelper::handleIDRFrame(const uint8_t* ptr,
204 size_t szBytes,
205 uint64_t pts) {
206 uint8_t* fptr = const_cast<uint8_t*>(ptr);
207
208 // We can assume fptr has a valid start code header because it has already
209 // gone through validation in H264NaluParser.
210 uint8_t startHeaderSz = fptr[2] == 1 ? 3 : 4;
211 uint32_t dataSz = szBytes - startHeaderSz;
212 std::unique_ptr<uint8_t> idr(new uint8_t[dataSz + 4]);
213 uint32_t dataSzNl = htonl(dataSz);
214
215 // AVCC format requires us to replace the start code header on this NALU
216 // with the size of the data. Start code is either 0x000001 or 0x00000001.
217 // The size needs to be the first four bytes in network byte order.
218 memcpy(idr.get(), &dataSzNl, 4);
219 memcpy(idr.get() + 4, ptr + startHeaderSz, dataSz);
220
221 CMSampleBufferRef sampleBuf = nullptr;
222 sampleBuf = createSampleBuffer(mCmFmtDesc, (void*)idr.get(), dataSz + 4);
223 if (!sampleBuf) {
224 VTB_DPRINT("%s: Failed to create CMSampleBufferRef", __func__);
225 return;
226 }
227
228 CMSampleBufferSetOutputPresentationTimeStamp(sampleBuf, CMTimeMake(pts, 1));
229
230 OSStatus status;
231 status = VTDecompressionSessionDecodeFrame(mDecoderSession, sampleBuf,
232 0, // decodeFlags
233 NULL, // sourceFrameRefCon
234 0); // infoFlagsOut
235
236 if (status == noErr) {
237 // TODO: this call blocks until the frame has been decoded. Perhaps it
238 // will be more efficient to signal the guest when the frame is ready to
239 // be read instead.
240 status = VTDecompressionSessionWaitForAsynchronousFrames(
241 mDecoderSession);
242 VTB_DPRINT("Success decoding IDR frame");
243 } else {
244 VTB_DPRINT("%s: Failed to decompress frame (err=%d)", __func__, status);
245 }
246
247 CFRelease(sampleBuf);
248 }
249
250 // chop the frame into sub frames that video tool box will consume
251 // one by one
parseInputFrames(const uint8_t * frame,size_t sz)252 bool MediaVideoToolBoxVideoHelper::parseInputFrames(const uint8_t* frame,
253 size_t sz) {
254 mInputFrames.clear();
255 VTB_DPRINT("input frame %d bytes", (int)sz);
256 while (1) {
257 const uint8_t* remainingFrame = parseOneFrame(frame, sz);
258 if (remainingFrame == nullptr)
259 break;
260 int consumed = (remainingFrame - frame);
261 VTB_DPRINT("consumed %d bytes", consumed);
262 frame = remainingFrame;
263 sz = sz - consumed;
264 }
265 return mInputFrames.size() > 0;
266 }
267
parseOneFrame(const uint8_t * frame,size_t szBytes)268 const uint8_t* MediaVideoToolBoxVideoHelper::parseOneFrame(const uint8_t* frame,
269 size_t szBytes) {
270 if (frame == nullptr || szBytes <= 0) {
271 return nullptr;
272 }
273
274 const uint8_t* currentNalu =
275 H264NaluParser::getNextStartCodeHeader(frame, szBytes);
276 if (currentNalu == nullptr) {
277 VTB_DPRINT("No start code header found in this frame");
278 return nullptr;
279 }
280
281 const uint8_t* nextNalu = nullptr;
282 size_t remaining = szBytes - (currentNalu - frame);
283
284 // Figure out the size of |currentNalu|.
285 size_t currentNaluSize = remaining;
286 // 3 is the minimum size of the start code header (3 or 4 bytes).
287 // dumpBytes(currentNalu, currentNaluSize);
288
289 // usually guest sends one frame a time, but sometime, SEI prefixes IDR
290 // frame as one frame from guest, we need to separate SEI out from IDR:
291 // video tool box cannot handle SEI+IDR combo-frame; for other cases, there
292 // is no need to check.
293 if (H264NaluType::SEI == H264NaluParser::getFrameNaluType(frame, szBytes, nullptr)
294 || H264NaluType::SPS == H264NaluParser::getFrameNaluType(frame, szBytes, nullptr)
295 || H264NaluType::PPS == H264NaluParser::getFrameNaluType(frame, szBytes, nullptr)
296
297 ) {
298 nextNalu = H264NaluParser::getNextStartCodeHeader(currentNalu + 3,
299 remaining - 3);
300 if (nextNalu != nullptr) {
301 currentNaluSize = nextNalu - currentNalu;
302 }
303 }
304
305 const uint8_t* remainingFrame = currentNalu + currentNaluSize;
306
307 // |data| is currentNalu, but with the start code header discarded.
308 uint8_t* data = nullptr;
309 H264NaluType naluType = H264NaluParser::getFrameNaluType(
310 currentNalu, currentNaluSize, &data);
311
312 if (naluType == H264NaluType::Undefined)
313 return nullptr;
314
315 size_t dataSize = currentNaluSize - (data - currentNalu);
316 const std::string naluTypeStr = H264NaluParser::naluTypeToString(naluType);
317 VTB_DPRINT("Got frame type=%u (%s)", (uint8_t)naluType,
318 naluTypeStr.c_str());
319
320 if (naluType == H264NaluType::SPS || naluType == H264NaluType::PPS) {
321 mInputFrames.push_back(InputFrame(naluType, data, dataSize));
322 } else {
323 mInputFrames.push_back(
324 InputFrame(naluType, currentNalu, currentNaluSize));
325 }
326 return remainingFrame;
327 }
328
329 // static
videoToolboxDecompressCallback(void * opaque,void * sourceFrameRefCon,OSStatus status,VTDecodeInfoFlags flags,CVPixelBufferRef image_buffer,CMTime pts,CMTime duration)330 void MediaVideoToolBoxVideoHelper::videoToolboxDecompressCallback(
331 void* opaque,
332 void* sourceFrameRefCon,
333 OSStatus status,
334 VTDecodeInfoFlags flags,
335 CVPixelBufferRef image_buffer,
336 CMTime pts,
337 CMTime duration) {
338 VTB_DPRINT("%s", __func__);
339 auto ptr = static_cast<MediaVideoToolBoxVideoHelper*>(opaque);
340
341 // if (ptr->mDecodedFrame) {
342 // CVPixelBufferRelease(ptr->mDecodedFrame);
343 // ptr->mDecodedFrame = nullptr;
344 // }
345
346 if (!image_buffer) {
347 VTB_DPRINT("%s: output image buffer is null", __func__);
348 ptr->mIsGood = false;
349 return;
350 }
351
352 ptr->mOutputPts = pts.value;
353 ptr->mDecodedFrame = CVPixelBufferRetain(image_buffer);
354 CVOpenGLTextureCacheRef _CVGLTextureCache;
355 CVOpenGLTextureRef _CVGLTexture;
356 CGLPixelFormatObj _CGLPixelFormat;
357
358 // Image is ready to be comsumed
359 ptr->copyFrame();
360 ptr->mImageReady = true;
361 VTB_DPRINT("Got decoded frame");
362 CVPixelBufferRelease(ptr->mDecodedFrame);
363 ptr->mDecodedFrame = nullptr;
364 }
365
366 // static
createOutputBufferAttributes(int width,int height,OSType pix_fmt)367 CFDictionaryRef MediaVideoToolBoxVideoHelper::createOutputBufferAttributes(
368 int width,
369 int height,
370 OSType pix_fmt) {
371 CFMutableDictionaryRef buffer_attributes;
372 CFMutableDictionaryRef io_surface_properties;
373 CFNumberRef cv_pix_fmt;
374 CFNumberRef w;
375 CFNumberRef h;
376
377 w = CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt32Type, &width);
378 h = CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt32Type, &height);
379 cv_pix_fmt =
380 CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt32Type, &pix_fmt);
381
382 buffer_attributes = CFDictionaryCreateMutable(
383 kCFAllocatorDefault, 4, &kCFTypeDictionaryKeyCallBacks,
384 &kCFTypeDictionaryValueCallBacks);
385 io_surface_properties = CFDictionaryCreateMutable(
386 kCFAllocatorDefault, 0, &kCFTypeDictionaryKeyCallBacks,
387 &kCFTypeDictionaryValueCallBacks);
388
389 if (pix_fmt) {
390 CFDictionarySetValue(buffer_attributes,
391 kCVPixelBufferPixelFormatTypeKey, cv_pix_fmt);
392 }
393 CFDictionarySetValue(buffer_attributes,
394 kCVPixelBufferIOSurfacePropertiesKey,
395 io_surface_properties);
396 CFDictionarySetValue(buffer_attributes, kCVPixelBufferWidthKey, w);
397 CFDictionarySetValue(buffer_attributes, kCVPixelBufferHeightKey, h);
398 // Not sure if this will work becuase we are passing the pixel buffer back
399 // into the guest
400 CFDictionarySetValue(buffer_attributes,
401 kCVPixelBufferIOSurfaceOpenGLTextureCompatibilityKey,
402 kCFBooleanTrue);
403
404 CFRelease(io_surface_properties);
405 CFRelease(cv_pix_fmt);
406 CFRelease(w);
407 CFRelease(h);
408
409 return buffer_attributes;
410 }
411
412 // static
createSampleBuffer(CMFormatDescriptionRef fmtDesc,void * buffer,size_t sz)413 CMSampleBufferRef MediaVideoToolBoxVideoHelper::createSampleBuffer(
414 CMFormatDescriptionRef fmtDesc,
415 void* buffer,
416 size_t sz) {
417 OSStatus status;
418 CMBlockBufferRef blockBuf = nullptr;
419 CMSampleBufferRef sampleBuf = nullptr;
420
421 status = CMBlockBufferCreateWithMemoryBlock(
422 kCFAllocatorDefault, // structureAllocator
423 buffer, // memoryBlock
424 sz, // blockLength
425 kCFAllocatorNull, // blockAllocator
426 NULL, // customBlockSource
427 0, // offsetToData
428 sz, // dataLength
429 0, // flags
430 &blockBuf);
431
432 if (!status) {
433 status = CMSampleBufferCreate(kCFAllocatorDefault, // allocator
434 blockBuf, // dataBuffer
435 TRUE, // dataReady
436 0, // makeDataReadyCallback
437 0, // makeDataReadyRefCon
438 fmtDesc, // formatDescription
439 1, // numSamples
440 0, // numSampleTimingEntries
441 NULL, // sampleTimingArray
442 0, // numSampleSizeEntries
443 NULL, // sampleSizeArray
444 &sampleBuf);
445 }
446
447 if (blockBuf) {
448 CFRelease(blockBuf);
449 }
450
451 return sampleBuf;
452 }
453
resetDecoderSession()454 void MediaVideoToolBoxVideoHelper::resetDecoderSession() {
455 if (mDecoderSession) {
456 VTDecompressionSessionInvalidate(mDecoderSession);
457 CFRelease(mDecoderSession);
458 mDecoderSession = nullptr;
459 }
460 if (mDecodedFrame) {
461 CVPixelBufferRelease(mDecodedFrame);
462 mDecodedFrame = nullptr;
463 }
464 }
465
getOutputWH()466 void MediaVideoToolBoxVideoHelper::getOutputWH() {
467 if (!mCmFmtDesc)
468 return;
469
470 CMVideoDimensions vsize = CMVideoFormatDescriptionGetDimensions(mCmFmtDesc);
471
472 if (mOutputWidth == vsize.width && mOutputHeight == vsize.height) {
473 VTB_DPRINT("initial w/h is the same as vtb w/h");
474 } else {
475 VTB_DPRINT("initial w/h are not the same as vtb w/h");
476 }
477 mOutputWidth = vsize.width;
478 mOutputHeight = vsize.height;
479 }
480
resetFormatDesc()481 void MediaVideoToolBoxVideoHelper::resetFormatDesc() {
482 if (mCmFmtDesc) {
483 CFRelease(mCmFmtDesc);
484 mCmFmtDesc = nullptr;
485 }
486 }
487
createCMFormatDescription()488 void MediaVideoToolBoxVideoHelper::createCMFormatDescription() {
489 uint8_t* parameterSets[2] = {mSPS.data(), mPPS.data()};
490 size_t parameterSetSizes[2] = {mSPS.size(), mPPS.size()};
491
492 resetFormatDesc();
493
494 OSStatus status = CMVideoFormatDescriptionCreateFromH264ParameterSets(
495 kCFAllocatorDefault, 2, (const uint8_t* const*)parameterSets,
496 parameterSetSizes, 4, &mCmFmtDesc);
497
498 if (status == noErr) {
499 getOutputWH();
500 VTB_DPRINT("Created CMFormatDescription from SPS/PPS sets w %d h %d",
501 mOutputWidth, mOutputHeight);
502 } else {
503 VTB_DPRINT("Unable to create CMFormatDescription (%d)\n", (int)status);
504 }
505 }
506
recreateDecompressionSession()507 void MediaVideoToolBoxVideoHelper::recreateDecompressionSession() {
508 if (mCmFmtDesc == nullptr) {
509 VTB_DPRINT("CMFormatDescription not created. Need sps and pps NALUs.");
510 return;
511 }
512
513 CMVideoCodecType codecType = kCMVideoCodecType_H264;
514 CFMutableDictionaryRef decoder_spec = CFDictionaryCreateMutable(
515 kCFAllocatorDefault, 0, &kCFTypeDictionaryKeyCallBacks,
516 &kCFTypeDictionaryValueCallBacks);
517 // enable hardware acceleration, but allows vtb to fallback to
518 // its software implementation.
519 CFDictionarySetValue(
520 decoder_spec,
521 kVTVideoDecoderSpecification_EnableHardwareAcceleratedVideoDecoder,
522 kCFBooleanTrue);
523
524 CFDictionaryRef bufAttr;
525
526 if (mUseGpuTexture) {
527 // use NV12 format
528 bufAttr = createOutputBufferAttributes(
529 mOutputWidth, mOutputHeight,
530 kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange);
531 } else {
532 bufAttr = createOutputBufferAttributes(
533 mOutputWidth, mOutputHeight,
534 kCVPixelFormatType_420YpCbCr8Planar);
535 }
536
537 VTDecompressionOutputCallbackRecord decoderCb;
538 decoderCb.decompressionOutputCallback = videoToolboxDecompressCallback;
539 decoderCb.decompressionOutputRefCon = this;
540
541 OSStatus status;
542 status = VTDecompressionSessionCreate(
543 NULL, // allocator
544 mCmFmtDesc, // videoFormatDescription
545 decoder_spec, // videoDecoderSpecification
546 bufAttr, // destinationImageBufferAttributes
547 &decoderCb, // outputCallback
548 &mDecoderSession); // decompressionSessionOut
549
550 if (decoder_spec) {
551 CFRelease(decoder_spec);
552 }
553 if (bufAttr) {
554 CFRelease(bufAttr);
555 }
556
557 mIsGood = false;
558 switch (status) {
559 case kVTVideoDecoderNotAvailableNowErr:
560 VTB_DPRINT("VideoToolbox session not available");
561 return;
562 case kVTVideoDecoderUnsupportedDataFormatErr:
563 VTB_DPRINT("VideoToolbox does not support this format");
564 return;
565 case kVTVideoDecoderMalfunctionErr:
566 VTB_DPRINT("VideoToolbox malfunction");
567 return;
568 case kVTVideoDecoderBadDataErr:
569 VTB_DPRINT("VideoToolbox reported invalid data");
570 return;
571 case 0:
572 dprint("VideoToolBox decoding session is created successfully");
573 mIsGood = true;
574 return;
575 default:
576 VTB_DPRINT("Unknown VideoToolbox session creation error %d",
577 status);
578 return;
579 }
580 }
581
copyFrameToTextures()582 void MediaVideoToolBoxVideoHelper::copyFrameToTextures() {
583 // TODO
584 auto startTime = std::chrono::steady_clock::now();
585 TextureFrame texFrame;
586 if (mUseGpuTexture && mTexturePool != nullptr) {
587 VTB_DPRINT("decoded surface is %p w %d h %d",
588 CVPixelBufferGetIOSurface(mDecodedFrame), mOutputWidth,
589 mOutputHeight);
590 auto my_copy_context = media_vtb_utils_copy_context{
591 CVPixelBufferGetIOSurface(mDecodedFrame), mOutputWidth,
592 mOutputHeight};
593 texFrame = mTexturePool->getTextureFrame(mOutputWidth, mOutputHeight);
594 VTB_DPRINT("ask videocore to copy to %d and %d", texFrame.Ytex,
595 texFrame.UVtex);
596 mTexturePool->saveDecodedFrameToTexture(
597 texFrame, &my_copy_context,
598 (void*)media_vtb_utils_nv12_updater);
599 }
600
601 auto elapsed = std::chrono::duration_cast<std::chrono::milliseconds>(
602 std::chrono::steady_clock::now() - startTime);
603 VTB_DPRINT("used %d ms", (int)elapsed.count());
604
605 auto ptspair = std::make_pair(mOutputPts, mTotalFrames);
606 mVtbBufferMap[ptspair] = MediaSnapshotState::FrameInfo{
607 std::vector<uint8_t>(),
608 std::vector<uint32_t>{texFrame.Ytex, texFrame.UVtex},
609 (int)mOutputWidth,
610 (int)mOutputHeight,
611 (uint64_t)(mOutputPts),
612 ColorAspects{mColorAspects}};
613 }
614
copyFrameToCPU()615 void MediaVideoToolBoxVideoHelper::copyFrameToCPU() {
616 int imageSize = CVPixelBufferGetDataSize(mDecodedFrame);
617 int stride = CVPixelBufferGetBytesPerRow(mDecodedFrame);
618
619 mOutBufferSize = mOutputWidth * mOutputHeight * 3 / 2;
620
621 VTB_DPRINT("copying size=%d dimension=[%dx%d] stride=%d", imageSize,
622 mOutputWidth, mOutputHeight, stride);
623
624 // Copies the image data to the guest.
625 mSavedDecodedFrame.resize(mOutputWidth * mOutputHeight * 3 / 2);
626 uint8_t* dst = mSavedDecodedFrame.data();
627
628 CVPixelBufferLockBaseAddress(mDecodedFrame, kCVPixelBufferLock_ReadOnly);
629 if (CVPixelBufferIsPlanar(mDecodedFrame)) {
630 imageSize = 0; // add up the size from the planes
631 int planes = CVPixelBufferGetPlaneCount(mDecodedFrame);
632 for (int i = 0; i < planes; ++i) {
633 void* planeData =
634 CVPixelBufferGetBaseAddressOfPlane(mDecodedFrame, i);
635 int linesize = CVPixelBufferGetBytesPerRowOfPlane(mDecodedFrame, i);
636 int planeWidth = CVPixelBufferGetWidthOfPlane(mDecodedFrame, i);
637 int planeHeight = CVPixelBufferGetHeightOfPlane(mDecodedFrame, i);
638 VTB_DPRINT("plane=%d data=%p linesize=%d pwidth=%d pheight=%d", i,
639 planeData, linesize, planeWidth, planeHeight);
640 // For kCVPixelFormatType_420YpCbCr8Planar, plane 0 is Y, UV planes
641 // are 1 and 2
642 if (planeWidth != mOutputWidth && planeWidth != mOutputWidth / 2) {
643 VTB_DPRINT("ERROR: Unable to determine YUV420 plane type");
644 continue;
645 }
646
647 // Sometimes the buffer stride can be longer than the actual data
648 // width. This means that the extra bytes are just padding and need
649 // to be discarded.
650 if (linesize <= planeWidth) {
651 int sz = planeHeight * planeWidth;
652 imageSize += sz;
653 memcpy(dst, planeData, sz);
654 dst += sz;
655 } else {
656 // Need to copy line by line
657 int sz = planeWidth;
658 for (int j = 0; j < planeHeight; ++j) {
659 uint8_t* ptr = (uint8_t*)planeData;
660 ptr += linesize * j;
661 memcpy(dst, ptr, sz);
662 imageSize += sz;
663 dst += sz;
664 }
665 }
666 }
667 if (imageSize != mOutBufferSize) {
668 VTB_DPRINT(
669 "ERROR: Total size of planes not same as guestSz "
670 "(guestSz=%u, imageSize=%d)",
671 mOutBufferSize, imageSize);
672 }
673 } else {
674 if (imageSize > mOutBufferSize) {
675 VTB_DPRINT(
676 "Buffer size mismatch (guestSz=%u, imageSize=%d). Using "
677 "guestSz instead.",
678 mOutBufferSize, imageSize);
679 imageSize = mOutBufferSize;
680 }
681
682 // IMPORTANT: mDecodedFrame must be locked before accessing the contents
683 // with CPU
684 void* data = CVPixelBufferGetBaseAddress(mDecodedFrame);
685 memcpy(dst, data, imageSize);
686 }
687 CVPixelBufferUnlockBaseAddress(mDecodedFrame, kCVPixelBufferLock_ReadOnly);
688
689 auto ptspair = std::make_pair(mOutputPts, mTotalFrames);
690 mVtbBufferMap[ptspair] = MediaSnapshotState::FrameInfo{
691 std::vector<uint8_t>(), std::vector<uint32_t>{},
692 (int)mOutputWidth, (int)mOutputHeight,
693 (uint64_t)(mOutputPts), ColorAspects{mColorAspects}};
694 mVtbBufferMap[ptspair].data.swap(mSavedDecodedFrame);
695 }
696
copyFrame()697 void MediaVideoToolBoxVideoHelper::copyFrame() {
698 mOutputWidth = CVPixelBufferGetWidth(mDecodedFrame);
699 mOutputHeight = CVPixelBufferGetHeight(mDecodedFrame);
700
701 if (mUseGpuTexture) {
702 copyFrameToTextures();
703 } else {
704 copyFrameToCPU();
705 }
706
707 if (mVtbBufferMap.size() > mVtbBufferSize) {
708 mSavedDecodedFrames.push_back(std::move(mVtbBufferMap.begin()->second));
709 mVtbBufferMap.erase(mVtbBufferMap.begin());
710 }
711 }
712
713 } // namespace emulation
714 } // namespace android
715