1 // Copyright 2023 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4
5 //#define LOG_NDEBUG 0
6 #define ATRACE_TAG ATRACE_TAG_VIDEO
7 #define LOG_TAG "DecodeComponent"
8
9 #include <v4l2_codec2/components/DecodeComponent.h>
10
11 #include <inttypes.h>
12 #include <linux/videodev2.h>
13 #include <stdint.h>
14
15 #include <memory>
16
17 #include <C2.h>
18 #include <C2PlatformSupport.h>
19 #include <Codec2Mapper.h>
20 #include <SimpleC2Interface.h>
21 #include <base/bind.h>
22 #include <base/callback_helpers.h>
23 #include <base/strings/stringprintf.h>
24 #include <base/time/time.h>
25 #include <cutils/properties.h>
26 #include <log/log.h>
27 #include <media/stagefright/foundation/ColorUtils.h>
28 #include <utils/Trace.h>
29
30 #include <v4l2_codec2/common/Common.h>
31 #include <v4l2_codec2/common/H264NalParser.h>
32 #include <v4l2_codec2/common/HEVCNalParser.h>
33 #include <v4l2_codec2/common/VideoTypes.h>
34 #include <v4l2_codec2/components/BitstreamBuffer.h>
35 #include <v4l2_codec2/components/VideoFramePool.h>
36
37 namespace android {
38 namespace {
39
40 // Mask against 30 bits to avoid (undefined) wraparound on signed integer.
frameIndexToBitstreamId(c2_cntr64_t frameIndex)41 int32_t frameIndexToBitstreamId(c2_cntr64_t frameIndex) {
42 return static_cast<int32_t>(frameIndex.peeku() & 0x3FFFFFFF);
43 }
44
parseCodedColorAspects(const C2ConstLinearBlock & input,std::optional<VideoCodec> codec,C2StreamColorAspectsInfo::input * codedAspects)45 bool parseCodedColorAspects(const C2ConstLinearBlock& input, std::optional<VideoCodec> codec,
46 C2StreamColorAspectsInfo::input* codedAspects) {
47 C2ReadView view = input.map().get();
48 NalParser::ColorAspects aspects;
49 std::unique_ptr<NalParser> parser;
50 if (codec == VideoCodec::H264) {
51 parser = std::make_unique<H264NalParser>(view.data(), view.capacity());
52 } else if (codec == VideoCodec::HEVC) {
53 parser = std::make_unique<HEVCNalParser>(view.data(), view.capacity());
54 } else {
55 ALOGV("Unsupported codec for finding color aspects");
56 return false;
57 }
58
59 if (!parser->locateSPS()) {
60 ALOGV("Couldn't find SPS");
61 return false;
62 }
63
64 if (!parser->findCodedColorAspects(&aspects)) {
65 ALOGV("Couldn't find color description in SPS");
66 return false;
67 }
68
69 // Convert ISO color aspects to ColorUtils::ColorAspects.
70 ColorAspects colorAspects;
71 ColorUtils::convertIsoColorAspectsToCodecAspects(
72 aspects.primaries, aspects.transfer, aspects.coeffs, aspects.fullRange, colorAspects);
73 ALOGV("Parsed ColorAspects from bitstream: (R:%d, P:%d, M:%d, T:%d)", colorAspects.mRange,
74 colorAspects.mPrimaries, colorAspects.mMatrixCoeffs, colorAspects.mTransfer);
75
76 // Map ColorUtils::ColorAspects to C2StreamColorAspectsInfo::input parameter.
77 if (!C2Mapper::map(colorAspects.mPrimaries, &codedAspects->primaries)) {
78 codedAspects->primaries = C2Color::PRIMARIES_UNSPECIFIED;
79 }
80 if (!C2Mapper::map(colorAspects.mRange, &codedAspects->range)) {
81 codedAspects->range = C2Color::RANGE_UNSPECIFIED;
82 }
83 if (!C2Mapper::map(colorAspects.mMatrixCoeffs, &codedAspects->matrix)) {
84 codedAspects->matrix = C2Color::MATRIX_UNSPECIFIED;
85 }
86 if (!C2Mapper::map(colorAspects.mTransfer, &codedAspects->transfer)) {
87 codedAspects->transfer = C2Color::TRANSFER_UNSPECIFIED;
88 }
89
90 return true;
91 }
92
isWorkDone(const C2Work & work)93 bool isWorkDone(const C2Work& work) {
94 const int32_t bitstreamId = frameIndexToBitstreamId(work.input.ordinal.frameIndex);
95
96 // Exception: EOS work should be processed by reportEOSWork().
97 // Always return false here no matter the work is actually done.
98 if (work.input.flags & C2FrameData::FLAG_END_OF_STREAM) return false;
99
100 // Work is done when all conditions meet:
101 // 1. mDecoder has released the work's input buffer.
102 // 2. mDecoder has returned the work's output buffer in normal case,
103 // or the input buffer is CSD, or we decide to drop the frame.
104 bool inputReleased = (work.input.buffers.front() == nullptr);
105 bool outputReturned = !work.worklets.front()->output.buffers.empty();
106 bool ignoreOutput = (work.input.flags & C2FrameData::FLAG_CODEC_CONFIG) ||
107 (work.worklets.front()->output.flags & C2FrameData::FLAG_DROP_FRAME);
108 ALOGV("work(%d): inputReleased: %d, outputReturned: %d, ignoreOutput: %d", bitstreamId,
109 inputReleased, outputReturned, ignoreOutput);
110 return inputReleased && (outputReturned || ignoreOutput);
111 }
112
isNoShowFrameWork(const C2Work & work,const C2WorkOrdinalStruct & currOrdinal)113 bool isNoShowFrameWork(const C2Work& work, const C2WorkOrdinalStruct& currOrdinal) {
114 // We consider Work contains no-show frame when all conditions meet:
115 // 1. Work's ordinal is smaller than current ordinal.
116 // 2. Work's output buffer is not returned.
117 // 3. Work is not EOS, CSD, or marked with dropped frame.
118 bool smallOrdinal = (work.input.ordinal.timestamp < currOrdinal.timestamp) &&
119 (work.input.ordinal.frameIndex < currOrdinal.frameIndex);
120 bool outputReturned = !work.worklets.front()->output.buffers.empty();
121 bool specialWork = (work.input.flags & C2FrameData::FLAG_END_OF_STREAM) ||
122 (work.input.flags & C2FrameData::FLAG_CODEC_CONFIG) ||
123 (work.worklets.front()->output.flags & C2FrameData::FLAG_DROP_FRAME);
124 return smallOrdinal && !outputReturned && !specialWork;
125 }
126
127 } // namespace
128
DecodeComponent(uint32_t debugStreamId,const std::string & name,c2_node_id_t id,const std::shared_ptr<DecodeInterface> & intfImpl)129 DecodeComponent::DecodeComponent(uint32_t debugStreamId, const std::string& name, c2_node_id_t id,
130 const std::shared_ptr<DecodeInterface>& intfImpl)
131 : mDebugStreamId(debugStreamId),
132 mIntfImpl(intfImpl),
133 mIntf(std::make_shared<SimpleInterface<DecodeInterface>>(name.c_str(), id, mIntfImpl)) {
134 ALOGV("%s(%s)", __func__, name.c_str());
135 mIsSecure = name.find(".secure") != std::string::npos;
136 }
137
~DecodeComponent()138 DecodeComponent::~DecodeComponent() {
139 ALOGV("%s()", __func__);
140 if (mDecoderThread.IsRunning() && !mDecoderTaskRunner->RunsTasksInCurrentSequence()) {
141 mDecoderTaskRunner->PostTask(FROM_HERE,
142 ::base::BindOnce(&DecodeComponent::releaseTask, mWeakThis));
143 mDecoderThread.Stop();
144 }
145 ALOGV("%s() done", __func__);
146 }
147
start()148 c2_status_t DecodeComponent::start() {
149 ALOGV("%s()", __func__);
150 std::lock_guard<std::mutex> lock(mStartStopLock);
151
152 auto currentState = mComponentState.load();
153 if (currentState != ComponentState::STOPPED) {
154 ALOGE("Could not start at %s state", ComponentStateToString(currentState));
155 return C2_BAD_STATE;
156 }
157
158 if (!mDecoderThread.Start()) {
159 ALOGE("Decoder thread failed to start.");
160 return C2_CORRUPTED;
161 }
162 mDecoderTaskRunner = mDecoderThread.task_runner();
163 mWeakThis = mWeakThisFactory.GetWeakPtr();
164
165 c2_status_t status = C2_CORRUPTED;
166 ::base::WaitableEvent done;
167 mDecoderTaskRunner->PostTask(
168 FROM_HERE, ::base::BindOnce(&DecodeComponent::startTask, mWeakThis,
169 ::base::Unretained(&status), ::base::Unretained(&done)));
170 done.Wait();
171
172 if (status == C2_OK) mComponentState.store(ComponentState::RUNNING);
173 return status;
174 }
175
getVideoFramePool(const ui::Size & size,HalPixelFormat pixelFormat,size_t numBuffers)176 std::unique_ptr<VideoFramePool> DecodeComponent::getVideoFramePool(const ui::Size& size,
177 HalPixelFormat pixelFormat,
178 size_t numBuffers) {
179 ALOGV("%s()", __func__);
180 ALOG_ASSERT(mDecoderTaskRunner->RunsTasksInCurrentSequence());
181
182 auto sharedThis = weak_from_this().lock();
183 if (sharedThis == nullptr) {
184 ALOGE("%s(): DecodeComponent instance is destroyed.", __func__);
185 return nullptr;
186 }
187
188 // (b/157113946): Prevent malicious dynamic resolution change exhausts system memory.
189 constexpr int kMaximumSupportedArea = 4096 * 4096;
190 if (getArea(size).value_or(INT_MAX) > kMaximumSupportedArea) {
191 ALOGE("The output size (%dx%d) is larger than supported size (4096x4096)", size.width,
192 size.height);
193 reportError(C2_BAD_VALUE);
194 return nullptr;
195 }
196
197 // Get block pool ID configured from the client.
198 auto poolId = mIntfImpl->getBlockPoolId();
199 ALOGI("Using C2BlockPool ID = %" PRIu64 " for allocating output buffers", poolId);
200 std::shared_ptr<C2BlockPool> blockPool;
201 auto status = GetCodec2BlockPool(poolId, std::move(sharedThis), &blockPool);
202 if (status != C2_OK) {
203 ALOGE("Graphic block allocator is invalid: %d", status);
204 reportError(status);
205 return nullptr;
206 }
207
208 return VideoFramePool::Create(std::move(blockPool), numBuffers, size, pixelFormat, mIsSecure,
209 mDecoderTaskRunner);
210 }
211
stop()212 c2_status_t DecodeComponent::stop() {
213 ALOGV("%s()", __func__);
214 std::lock_guard<std::mutex> lock(mStartStopLock);
215
216 auto currentState = mComponentState.load();
217 if (currentState != ComponentState::RUNNING && currentState != ComponentState::ERROR) {
218 ALOGE("Could not stop at %s state", ComponentStateToString(currentState));
219 return C2_BAD_STATE;
220 }
221
222 if (mDecoderThread.IsRunning()) {
223 mDecoderTaskRunner->PostTask(FROM_HERE,
224 ::base::BindOnce(&DecodeComponent::stopTask, mWeakThis));
225 mDecoderThread.Stop();
226 mDecoderTaskRunner = nullptr;
227 }
228
229 mComponentState.store(ComponentState::STOPPED);
230 return C2_OK;
231 }
232
stopTask()233 void DecodeComponent::stopTask() {
234 ATRACE_CALL();
235 ALOGV("%s()", __func__);
236 ALOG_ASSERT(mDecoderTaskRunner->RunsTasksInCurrentSequence());
237
238 reportAbandonedWorks();
239 mIsDraining = false;
240
241 releaseTask();
242 }
243
reset()244 c2_status_t DecodeComponent::reset() {
245 ALOGV("%s()", __func__);
246
247 return stop();
248 }
249
release()250 c2_status_t DecodeComponent::release() {
251 ALOGV("%s()", __func__);
252 std::lock_guard<std::mutex> lock(mStartStopLock);
253
254 if (mDecoderThread.IsRunning()) {
255 mDecoderTaskRunner->PostTask(FROM_HERE,
256 ::base::BindOnce(&DecodeComponent::releaseTask, mWeakThis));
257 mDecoderThread.Stop();
258 mDecoderTaskRunner = nullptr;
259 }
260
261 mComponentState.store(ComponentState::RELEASED);
262 return C2_OK;
263 }
264
releaseTask()265 void DecodeComponent::releaseTask() {
266 ATRACE_CALL();
267 ALOGV("%s()", __func__);
268 ALOG_ASSERT(mDecoderTaskRunner->RunsTasksInCurrentSequence());
269
270 mWeakThisFactory.InvalidateWeakPtrs();
271 mDecoder = nullptr;
272 }
273
setListener_vb(const std::shared_ptr<C2Component::Listener> & listener,c2_blocking_t mayBlock)274 c2_status_t DecodeComponent::setListener_vb(const std::shared_ptr<C2Component::Listener>& listener,
275 c2_blocking_t mayBlock) {
276 ALOGV("%s()", __func__);
277
278 auto currentState = mComponentState.load();
279 if (currentState == ComponentState::RELEASED ||
280 (currentState == ComponentState::RUNNING && listener)) {
281 ALOGE("Could not set listener at %s state", ComponentStateToString(currentState));
282 return C2_BAD_STATE;
283 }
284 if (currentState == ComponentState::RUNNING && mayBlock != C2_MAY_BLOCK) {
285 ALOGE("Could not set listener at %s state non-blocking",
286 ComponentStateToString(currentState));
287 return C2_BLOCKING;
288 }
289
290 // If the decoder thread is not running it's safe to update the listener directly.
291 if (!mDecoderThread.IsRunning()) {
292 mListener = listener;
293 return C2_OK;
294 }
295
296 ::base::WaitableEvent done;
297 mDecoderTaskRunner->PostTask(
298 FROM_HERE, ::base::Bind(&DecodeComponent::setListenerTask, mWeakThis, listener, &done));
299 done.Wait();
300 return C2_OK;
301 }
302
setListenerTask(const std::shared_ptr<Listener> & listener,::base::WaitableEvent * done)303 void DecodeComponent::setListenerTask(const std::shared_ptr<Listener>& listener,
304 ::base::WaitableEvent* done) {
305 ALOGV("%s()", __func__);
306 ALOG_ASSERT(mDecoderTaskRunner->RunsTasksInCurrentSequence());
307
308 mListener = listener;
309 done->Signal();
310 }
311
queue_nb(std::list<std::unique_ptr<C2Work>> * const items)312 c2_status_t DecodeComponent::queue_nb(std::list<std::unique_ptr<C2Work>>* const items) {
313 ALOGV("%s()", __func__);
314
315 auto currentState = mComponentState.load();
316 if (currentState != ComponentState::RUNNING) {
317 ALOGE("Could not queue at state: %s", ComponentStateToString(currentState));
318 return C2_BAD_STATE;
319 }
320
321 while (!items->empty()) {
322 if (ATRACE_ENABLED()) {
323 const std::string atraceLabel = ::base::StringPrintf("#%u C2Work", mDebugStreamId);
324 ATRACE_ASYNC_BEGIN(atraceLabel.c_str(),
325 items->front()->input.ordinal.frameIndex.peekull());
326 }
327
328 mDecoderTaskRunner->PostTask(FROM_HERE,
329 ::base::BindOnce(&DecodeComponent::queueTask, mWeakThis,
330 std::move(items->front())));
331 items->pop_front();
332 }
333 return C2_OK;
334 }
335
queueTask(std::unique_ptr<C2Work> work)336 void DecodeComponent::queueTask(std::unique_ptr<C2Work> work) {
337 ATRACE_CALL();
338 ALOGV("%s(): flags=0x%x, index=%llu, timestamp=%llu", __func__, work->input.flags,
339 work->input.ordinal.frameIndex.peekull(), work->input.ordinal.timestamp.peekull());
340 ALOG_ASSERT(mDecoderTaskRunner->RunsTasksInCurrentSequence());
341
342 if (work->worklets.size() != 1u || work->input.buffers.size() > 1u) {
343 ALOGE("Invalid work: worklets.size()=%zu, input.buffers.size()=%zu", work->worklets.size(),
344 work->input.buffers.size());
345 work->result = C2_CORRUPTED;
346 reportWork(std::move(work));
347 return;
348 }
349
350 work->worklets.front()->output.flags = static_cast<C2FrameData::flags_t>(0);
351 work->worklets.front()->output.buffers.clear();
352 work->worklets.front()->output.ordinal = work->input.ordinal;
353 if (work->input.buffers.empty()) {
354 // Client may queue a work with no input buffer for either it's EOS or empty CSD, otherwise
355 // every work must have one input buffer.
356 if ((work->input.flags & C2FrameData::FLAG_END_OF_STREAM) == 0 &&
357 (work->input.flags & C2FrameData::FLAG_CODEC_CONFIG) == 0) {
358 ALOGE("Invalid work: work with no input buffer should be EOS or CSD.");
359 reportError(C2_BAD_VALUE);
360 return;
361 }
362
363 // Emplace a nullptr to unify the check for work done.
364 ALOGV("Got a work with no input buffer! Emplace a nullptr inside.");
365 work->input.buffers.emplace_back(nullptr);
366 }
367
368 mPendingWorks.push(std::move(work));
369 pumpPendingWorks();
370 }
371
pumpPendingWorks()372 void DecodeComponent::pumpPendingWorks() {
373 ATRACE_CALL();
374 ALOGV("%s()", __func__);
375 ALOG_ASSERT(mDecoderTaskRunner->RunsTasksInCurrentSequence());
376
377 auto currentState = mComponentState.load();
378 if (currentState != ComponentState::RUNNING) {
379 ALOGW("Could not pump C2Work at state: %s", ComponentStateToString(currentState));
380 return;
381 }
382
383 while (!mPendingWorks.empty() && !mIsDraining) {
384 std::unique_ptr<C2Work> pendingWork(std::move(mPendingWorks.front()));
385 mPendingWorks.pop();
386
387 const int32_t bitstreamId = frameIndexToBitstreamId(pendingWork->input.ordinal.frameIndex);
388 const bool isCSDWork = pendingWork->input.flags & C2FrameData::FLAG_CODEC_CONFIG;
389 const bool isEmptyWork = pendingWork->input.buffers.front() == nullptr;
390 const bool isEOSWork = pendingWork->input.flags & C2FrameData::FLAG_END_OF_STREAM;
391 const C2Work* work = pendingWork.get();
392 ALOGV("Process C2Work bitstreamId=%d isCSDWork=%d, isEmptyWork=%d", bitstreamId, isCSDWork,
393 isEmptyWork);
394
395 auto res = mWorksAtDecoder.insert(std::make_pair(bitstreamId, std::move(pendingWork)));
396 ALOGW_IF(!res.second, "We already inserted bitstreamId %d to decoder?", bitstreamId);
397
398 if (!isEmptyWork) {
399 if (isCSDWork) {
400 processCSDWork(bitstreamId, work);
401 } else {
402 processWork(bitstreamId, work);
403 }
404 }
405
406 if (isEOSWork) {
407 mDecoder->drain(::base::BindOnce(&DecodeComponent::onDrainDone, mWeakThis));
408 mIsDraining = true;
409 }
410
411 // Directly report the empty CSD work as finished.
412 if (isCSDWork && isEmptyWork) reportWorkIfFinished(bitstreamId);
413 }
414 }
415
processCSDWork(const int32_t bitstreamId,const C2Work * work)416 void DecodeComponent::processCSDWork(const int32_t bitstreamId, const C2Work* work) {
417 // If input.buffers is not empty, the buffer should have meaningful content inside.
418 C2ConstLinearBlock linearBlock = work->input.buffers.front()->data().linearBlocks().front();
419 ALOG_ASSERT(linearBlock.size() > 0u, "Input buffer of work(%d) is empty.", bitstreamId);
420
421 if (mIntfImpl->getVideoCodec() == VideoCodec::VP9) {
422 // The VP9 decoder does not support and does not need the Codec Specific Data (CSD):
423 // https://www.webmproject.org/docs/container/#vp9-codec-feature-metadata-codecprivate.
424 // The most of its content (profile, level, bit depth and chroma subsampling)
425 // can be extracted directly from VP9 bitstream. Ignore CSD if it was passed.
426 reportWorkIfFinished(bitstreamId);
427 return;
428 } else if ((!mIsSecure && mIntfImpl->getVideoCodec() == VideoCodec::H264) ||
429 mIntfImpl->getVideoCodec() == VideoCodec::HEVC) {
430 // Try to parse color aspects from bitstream for CSD work of non-secure H264 codec or HEVC
431 // codec (HEVC will only be CENCv3 which is parseable for secure).
432 C2StreamColorAspectsInfo::input codedAspects = {0u};
433 if (parseCodedColorAspects(linearBlock, mIntfImpl->getVideoCodec(), &codedAspects)) {
434 std::vector<std::unique_ptr<C2SettingResult>> failures;
435 c2_status_t status = mIntfImpl->config({&codedAspects}, C2_MAY_BLOCK, &failures);
436 if (status != C2_OK) {
437 ALOGE("Failed to config color aspects to interface: %d", status);
438 reportError(status);
439 return;
440 }
441 // Record current frame index, color aspects should be updated only for output
442 // buffers whose frame indices are not less than this one.
443 mPendingColorAspectsChange = true;
444 mPendingColorAspectsChangeFrameIndex = work->input.ordinal.frameIndex.peeku();
445 }
446 }
447
448 processWorkBuffer(bitstreamId, linearBlock);
449 }
450
processWork(const int32_t bitstreamId,const C2Work * work)451 void DecodeComponent::processWork(const int32_t bitstreamId, const C2Work* work) {
452 // If input.buffers is not empty, the buffer should have meaningful content inside.
453 C2ConstLinearBlock linearBlock = work->input.buffers.front()->data().linearBlocks().front();
454 ALOG_ASSERT(linearBlock.size() > 0u, "Input buffer of work(%d) is empty.", bitstreamId);
455
456 processWorkBuffer(bitstreamId, linearBlock);
457 }
458
processWorkBuffer(const int32_t bitstreamId,const C2ConstLinearBlock & linearBlock)459 void DecodeComponent::processWorkBuffer(const int32_t bitstreamId,
460 const C2ConstLinearBlock& linearBlock) {
461 std::unique_ptr<ConstBitstreamBuffer> buffer = std::make_unique<ConstBitstreamBuffer>(
462 bitstreamId, linearBlock, linearBlock.offset(), linearBlock.size());
463 if (!buffer) {
464 reportError(C2_CORRUPTED);
465 return;
466 }
467 mDecoder->decode(std::move(buffer),
468 ::base::BindOnce(&DecodeComponent::onDecodeDone, mWeakThis, bitstreamId));
469 }
470
onDecodeDone(int32_t bitstreamId,VideoDecoder::DecodeStatus status)471 void DecodeComponent::onDecodeDone(int32_t bitstreamId, VideoDecoder::DecodeStatus status) {
472 ATRACE_CALL();
473 ALOGV("%s(bitstreamId=%d, status=%s)", __func__, bitstreamId,
474 VideoDecoder::DecodeStatusToString(status));
475 ALOG_ASSERT(mDecoderTaskRunner->RunsTasksInCurrentSequence());
476
477 auto it = mWorksAtDecoder.find(bitstreamId);
478 ALOG_ASSERT(it != mWorksAtDecoder.end());
479 C2Work* work = it->second.get();
480
481 switch (status) {
482 case VideoDecoder::DecodeStatus::kAborted:
483 work->input.buffers.front().reset();
484 work->worklets.front()->output.flags = static_cast<C2FrameData::flags_t>(
485 work->worklets.front()->output.flags & C2FrameData::FLAG_DROP_FRAME);
486 mOutputBitstreamIds.push(bitstreamId);
487
488 pumpReportWork();
489 return;
490
491 case VideoDecoder::DecodeStatus::kError:
492 reportError(C2_CORRUPTED);
493 return;
494
495 case VideoDecoder::DecodeStatus::kOk:
496 // Release the input buffer.
497 work->input.buffers.front().reset();
498
499 // CSD Work doesn't have output buffer, the corresponding onOutputFrameReady() won't be
500 // called. Push the bitstreamId here.
501 if (work->input.flags & C2FrameData::FLAG_CODEC_CONFIG)
502 mOutputBitstreamIds.push(bitstreamId);
503
504 pumpReportWork();
505 return;
506 }
507 }
508
onOutputFrameReady(std::unique_ptr<VideoFrame> frame)509 void DecodeComponent::onOutputFrameReady(std::unique_ptr<VideoFrame> frame) {
510 ALOGV("%s(bitstreamId=%d)", __func__, frame->getBitstreamId());
511 ALOG_ASSERT(mDecoderTaskRunner->RunsTasksInCurrentSequence());
512
513 const int32_t bitstreamId = frame->getBitstreamId();
514 auto it = mWorksAtDecoder.find(bitstreamId);
515 if (it == mWorksAtDecoder.end()) {
516 ALOGE("Work with bitstreamId=%d not found, already abandoned?", bitstreamId);
517 reportError(C2_CORRUPTED);
518 return;
519 }
520 C2Work* work = it->second.get();
521
522 C2ConstGraphicBlock constBlock = std::move(frame)->getGraphicBlock();
523 std::shared_ptr<C2Buffer> buffer = C2Buffer::CreateGraphicBuffer(std::move(constBlock));
524 if (mPendingColorAspectsChange &&
525 work->input.ordinal.frameIndex.peeku() >= mPendingColorAspectsChangeFrameIndex) {
526 mIntfImpl->queryColorAspects(&mCurrentColorAspects);
527 mPendingColorAspectsChange = false;
528 }
529 if (mCurrentColorAspects) {
530 buffer->setInfo(mCurrentColorAspects);
531 }
532 work->worklets.front()->output.buffers.emplace_back(std::move(buffer));
533
534 // Check no-show frame by timestamps for VP8/VP9 cases before reporting the current work.
535 if (mIntfImpl->getVideoCodec() == VideoCodec::VP8 ||
536 mIntfImpl->getVideoCodec() == VideoCodec::VP9) {
537 detectNoShowFrameWorksAndReportIfFinished(work->input.ordinal);
538 }
539
540 mOutputBitstreamIds.push(bitstreamId);
541 pumpReportWork();
542 }
543
detectNoShowFrameWorksAndReportIfFinished(const C2WorkOrdinalStruct & currOrdinal)544 void DecodeComponent::detectNoShowFrameWorksAndReportIfFinished(
545 const C2WorkOrdinalStruct& currOrdinal) {
546 ATRACE_CALL();
547 ALOGV("%s()", __func__);
548 ALOG_ASSERT(mDecoderTaskRunner->RunsTasksInCurrentSequence());
549
550 std::vector<int32_t> noShowFrameBitstreamIds;
551 for (auto& kv : mWorksAtDecoder) {
552 const int32_t bitstreamId = kv.first;
553 const C2Work* work = kv.second.get();
554
555 // A work in mWorksAtDecoder would be considered to have no-show frame if there is no
556 // corresponding output buffer returned while the one of the work with latter timestamp is
557 // already returned. (VD is outputted in display order.)
558 if (isNoShowFrameWork(*work, currOrdinal)) {
559 work->worklets.front()->output.flags = C2FrameData::FLAG_DROP_FRAME;
560
561 // We need to call reportWorkIfFinished() for all detected no-show frame works. However,
562 // we should do it after the detection loop since reportWorkIfFinished() may erase
563 // entries in |mWorksAtDecoder|.
564 noShowFrameBitstreamIds.push_back(bitstreamId);
565 ALOGV("Detected no-show frame work index=%llu timestamp=%llu",
566 work->input.ordinal.frameIndex.peekull(),
567 work->input.ordinal.timestamp.peekull());
568 }
569 }
570
571 // Try to report works with no-show frame.
572 for (const int32_t bitstreamId : noShowFrameBitstreamIds) reportWorkIfFinished(bitstreamId);
573 }
574
pumpReportWork()575 void DecodeComponent::pumpReportWork() {
576 ATRACE_CALL();
577 ALOGV("%s()", __func__);
578 ALOG_ASSERT(mDecoderTaskRunner->RunsTasksInCurrentSequence());
579
580 while (!mOutputBitstreamIds.empty()) {
581 if (!reportWorkIfFinished(mOutputBitstreamIds.front())) break;
582 mOutputBitstreamIds.pop();
583 }
584 }
585
reportWorkIfFinished(int32_t bitstreamId)586 bool DecodeComponent::reportWorkIfFinished(int32_t bitstreamId) {
587 ATRACE_CALL();
588 ALOGV("%s(bitstreamId = %d)", __func__, bitstreamId);
589 ALOG_ASSERT(mDecoderTaskRunner->RunsTasksInCurrentSequence());
590
591 // EOS work will not be reported here. reportEOSWork() does it.
592 if (mIsDraining && mWorksAtDecoder.size() == 1u) {
593 ALOGV("work(bitstreamId = %d) is EOS Work.", bitstreamId);
594 return false;
595 }
596
597 auto it = mWorksAtDecoder.find(bitstreamId);
598 if (it == mWorksAtDecoder.end()) {
599 ALOGI("work(bitstreamId = %d) is dropped, skip.", bitstreamId);
600 return true;
601 }
602
603 if (!isWorkDone(*(it->second))) {
604 ALOGV("work(bitstreamId = %d) is not done yet.", bitstreamId);
605 return false;
606 }
607
608 std::unique_ptr<C2Work> work = std::move(it->second);
609 mWorksAtDecoder.erase(it);
610
611 work->result = C2_OK;
612 work->workletsProcessed = static_cast<uint32_t>(work->worklets.size());
613 // A work with neither flags nor output buffer would be treated as no-corresponding
614 // output by C2 framework, and regain pipeline capacity immediately.
615 if (work->worklets.front()->output.flags & C2FrameData::FLAG_DROP_FRAME)
616 work->worklets.front()->output.flags = static_cast<C2FrameData::flags_t>(0);
617
618 return reportWork(std::move(work));
619 }
620
reportEOSWork()621 bool DecodeComponent::reportEOSWork() {
622 ATRACE_CALL();
623 ALOGV("%s()", __func__);
624 ALOG_ASSERT(mDecoderTaskRunner->RunsTasksInCurrentSequence());
625
626 const auto it =
627 std::find_if(mWorksAtDecoder.begin(), mWorksAtDecoder.end(), [](const auto& kv) {
628 return kv.second->input.flags & C2FrameData::FLAG_END_OF_STREAM;
629 });
630 if (it == mWorksAtDecoder.end()) {
631 ALOGE("Failed to find EOS work.");
632 return false;
633 }
634
635 std::unique_ptr<C2Work> eosWork(std::move(it->second));
636 mWorksAtDecoder.erase(it);
637
638 eosWork->result = C2_OK;
639 eosWork->workletsProcessed = static_cast<uint32_t>(eosWork->worklets.size());
640 eosWork->worklets.front()->output.flags = C2FrameData::FLAG_END_OF_STREAM;
641 if (!eosWork->input.buffers.empty()) eosWork->input.buffers.front().reset();
642
643 if (!mWorksAtDecoder.empty()) {
644 ALOGW("There are remaining works except EOS work. abandon them.");
645 for (const auto& kv : mWorksAtDecoder) {
646 ALOGW("bitstreamId(%d) => Work index=%llu, timestamp=%llu", kv.first,
647 kv.second->input.ordinal.frameIndex.peekull(),
648 kv.second->input.ordinal.timestamp.peekull());
649 }
650 reportAbandonedWorks();
651 }
652
653 return reportWork(std::move(eosWork));
654 }
655
reportWork(std::unique_ptr<C2Work> work)656 bool DecodeComponent::reportWork(std::unique_ptr<C2Work> work) {
657 ATRACE_CALL();
658 ALOGV("%s(work=%llu)", __func__, work->input.ordinal.frameIndex.peekull());
659 ALOG_ASSERT(mDecoderTaskRunner->RunsTasksInCurrentSequence());
660
661 if (!mListener) {
662 ALOGE("mListener is nullptr, setListener_vb() not called?");
663 return false;
664 }
665
666 if (ATRACE_ENABLED()) {
667 const std::string atraceLabel = ::base::StringPrintf("#%u C2Work", mDebugStreamId);
668 ATRACE_ASYNC_END(atraceLabel.c_str(), work->input.ordinal.frameIndex.peekull());
669 }
670 std::list<std::unique_ptr<C2Work>> finishedWorks;
671 finishedWorks.emplace_back(std::move(work));
672 mListener->onWorkDone_nb(weak_from_this(), std::move(finishedWorks));
673 return true;
674 }
675
flush_sm(flush_mode_t mode,std::list<std::unique_ptr<C2Work>> * const)676 c2_status_t DecodeComponent::flush_sm(flush_mode_t mode,
677 std::list<std::unique_ptr<C2Work>>* const /* flushedWork */) {
678 ATRACE_CALL();
679 ALOGV("%s()", __func__);
680
681 auto currentState = mComponentState.load();
682 if (currentState != ComponentState::RUNNING) {
683 ALOGE("Could not flush at state: %s", ComponentStateToString(currentState));
684 return C2_BAD_STATE;
685 }
686 if (mode != FLUSH_COMPONENT) {
687 return C2_OMITTED; // Tunneling is not supported by now
688 }
689
690 mDecoderTaskRunner->PostTask(FROM_HERE,
691 ::base::BindOnce(&DecodeComponent::flushTask, mWeakThis));
692 return C2_OK;
693 }
694
flushTask()695 void DecodeComponent::flushTask() {
696 ATRACE_CALL();
697 ALOGV("%s()", __func__);
698 ALOG_ASSERT(mDecoderTaskRunner->RunsTasksInCurrentSequence());
699
700 mDecoder->flush();
701 reportAbandonedWorks();
702
703 // Pending EOS work will be abandoned here due to component flush if any.
704 mIsDraining = false;
705 }
706
reportAbandonedWorks()707 void DecodeComponent::reportAbandonedWorks() {
708 ALOGV("%s()", __func__);
709 ALOG_ASSERT(mDecoderTaskRunner->RunsTasksInCurrentSequence());
710
711 std::list<std::unique_ptr<C2Work>> abandonedWorks;
712 while (!mPendingWorks.empty()) {
713 abandonedWorks.emplace_back(std::move(mPendingWorks.front()));
714 mPendingWorks.pop();
715 }
716 for (auto& kv : mWorksAtDecoder) {
717 abandonedWorks.emplace_back(std::move(kv.second));
718 }
719 mWorksAtDecoder.clear();
720
721 for (auto& work : abandonedWorks) {
722 // TODO: correlate the definition of flushed work result to framework.
723 work->result = C2_NOT_FOUND;
724 // When the work is abandoned, buffer in input.buffers shall reset by component.
725 if (!work->input.buffers.empty()) {
726 work->input.buffers.front().reset();
727 }
728
729 if (ATRACE_ENABLED()) {
730 const std::string atraceLabel = ::base::StringPrintf("#%u C2Work", mDebugStreamId);
731 ATRACE_ASYNC_END(atraceLabel.c_str(), work->input.ordinal.frameIndex.peekull());
732 }
733 }
734 if (!abandonedWorks.empty()) {
735 if (!mListener) {
736 ALOGE("mListener is nullptr, setListener_vb() not called?");
737 return;
738 }
739 mListener->onWorkDone_nb(weak_from_this(), std::move(abandonedWorks));
740 }
741 }
742
drain_nb(drain_mode_t mode)743 c2_status_t DecodeComponent::drain_nb(drain_mode_t mode) {
744 ALOGV("%s(mode=%u)", __func__, mode);
745
746 auto currentState = mComponentState.load();
747 if (currentState != ComponentState::RUNNING) {
748 ALOGE("Could not drain at state: %s", ComponentStateToString(currentState));
749 return C2_BAD_STATE;
750 }
751
752 switch (mode) {
753 case DRAIN_CHAIN:
754 return C2_OMITTED; // Tunneling is not supported.
755
756 case DRAIN_COMPONENT_NO_EOS:
757 return C2_OK; // Do nothing special.
758
759 case DRAIN_COMPONENT_WITH_EOS:
760 mDecoderTaskRunner->PostTask(FROM_HERE,
761 ::base::BindOnce(&DecodeComponent::drainTask, mWeakThis));
762 return C2_OK;
763 }
764 }
765
drainTask()766 void DecodeComponent::drainTask() {
767 ATRACE_CALL();
768 ALOGV("%s()", __func__);
769 ALOG_ASSERT(mDecoderTaskRunner->RunsTasksInCurrentSequence());
770
771 if (!mPendingWorks.empty()) {
772 ALOGV("Set EOS flag at last queued work.");
773 auto& flags = mPendingWorks.back()->input.flags;
774 flags = static_cast<C2FrameData::flags_t>(flags | C2FrameData::FLAG_END_OF_STREAM);
775 return;
776 }
777
778 if (!mWorksAtDecoder.empty()) {
779 ALOGV("Drain the pending works at the decoder.");
780 mDecoder->drain(::base::BindOnce(&DecodeComponent::onDrainDone, mWeakThis));
781 mIsDraining = true;
782 }
783 }
784
onDrainDone(VideoDecoder::DecodeStatus status)785 void DecodeComponent::onDrainDone(VideoDecoder::DecodeStatus status) {
786 ALOGV("%s(status=%s)", __func__, VideoDecoder::DecodeStatusToString(status));
787 ALOG_ASSERT(mDecoderTaskRunner->RunsTasksInCurrentSequence());
788
789 switch (status) {
790 case VideoDecoder::DecodeStatus::kAborted:
791 return;
792
793 case VideoDecoder::DecodeStatus::kError:
794 reportError(C2_CORRUPTED);
795 return;
796
797 case VideoDecoder::DecodeStatus::kOk:
798 mIsDraining = false;
799 if (!reportEOSWork()) {
800 reportError(C2_CORRUPTED);
801 return;
802 }
803
804 mDecoderTaskRunner->PostTask(
805 FROM_HERE, ::base::BindOnce(&DecodeComponent::pumpPendingWorks, mWeakThis));
806 return;
807 }
808 }
809
reportError(c2_status_t error)810 void DecodeComponent::reportError(c2_status_t error) {
811 ALOGE("%s(error=%u)", __func__, static_cast<uint32_t>(error));
812 ALOG_ASSERT(mDecoderTaskRunner->RunsTasksInCurrentSequence());
813
814 if (mComponentState.load() == ComponentState::ERROR) return;
815 mComponentState.store(ComponentState::ERROR);
816
817 if (!mListener) {
818 ALOGE("mListener is nullptr, setListener_vb() not called?");
819 return;
820 }
821 mListener->onError_nb(weak_from_this(), static_cast<uint32_t>(error));
822 }
823
announce_nb(const std::vector<C2WorkOutline> &)824 c2_status_t DecodeComponent::announce_nb(const std::vector<C2WorkOutline>& /* items */) {
825 return C2_OMITTED; // Tunneling is not supported by now
826 }
827
intf()828 std::shared_ptr<C2ComponentInterface> DecodeComponent::intf() {
829 return mIntf;
830 }
831
832 // static
ComponentStateToString(ComponentState state)833 const char* DecodeComponent::ComponentStateToString(ComponentState state) {
834 switch (state) {
835 case ComponentState::STOPPED:
836 return "STOPPED";
837 case ComponentState::RUNNING:
838 return "RUNNING";
839 case ComponentState::RELEASED:
840 return "RELEASED";
841 case ComponentState::ERROR:
842 return "ERROR";
843 }
844 }
845
846 } // namespace android
847