1 // Copyright 2020 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4
5 //#define LOG_NDEBUG 0
6 #define LOG_TAG "V4L2DecodeComponent"
7
8 #include <v4l2_codec2/components/V4L2DecodeComponent.h>
9
10 #include <inttypes.h>
11 #include <linux/videodev2.h>
12 #include <stdint.h>
13
14 #include <memory>
15
16 #include <C2.h>
17 #include <C2PlatformSupport.h>
18 #include <Codec2Mapper.h>
19 #include <SimpleC2Interface.h>
20 #include <base/bind.h>
21 #include <base/callback_helpers.h>
22 #include <base/time/time.h>
23 #include <cutils/properties.h>
24 #include <log/log.h>
25 #include <media/stagefright/foundation/ColorUtils.h>
26
27 #include <v4l2_codec2/common/Common.h>
28 #include <v4l2_codec2/common/NalParser.h>
29 #include <v4l2_codec2/common/VideoTypes.h>
30 #include <v4l2_codec2/components/BitstreamBuffer.h>
31 #include <v4l2_codec2/components/V4L2Decoder.h>
32 #include <v4l2_codec2/components/VideoFramePool.h>
33 #include <v4l2_codec2/plugin_store/C2VdaBqBlockPool.h>
34
35 namespace android {
36 namespace {
37
38 // CCBC pauses sending input buffers to the component when all the output slots are filled by
39 // pending decoded buffers. If the available output buffers are exhausted before CCBC pauses sending
40 // input buffers, CCodec may timeout due to waiting for a available output buffer.
41 // This function returns the minimum number of output buffers to prevent the buffers from being
42 // exhausted before CCBC pauses sending input buffers.
getMinNumOutputBuffers(VideoCodec codec)43 size_t getMinNumOutputBuffers(VideoCodec codec) {
44 // The constant values copied from CCodecBufferChannel.cpp.
45 // (b/184020290): Check the value still sync when seeing error message from CCodec:
46 // "previous call to queue exceeded timeout".
47 constexpr size_t kSmoothnessFactor = 4;
48 constexpr size_t kRenderingDepth = 3;
49 // Extra number of needed output buffers for V4L2Decoder.
50 constexpr size_t kExtraNumOutputBuffersForDecoder = 2;
51
52 // The total needed number of output buffers at pipeline are:
53 // - MediaCodec output slots: output delay + kSmoothnessFactor
54 // - Surface: kRenderingDepth
55 // - Component: kExtraNumOutputBuffersForDecoder
56 return V4L2DecodeInterface::getOutputDelay(codec) + kSmoothnessFactor + kRenderingDepth +
57 kExtraNumOutputBuffersForDecoder;
58 }
59
60 // Mask against 30 bits to avoid (undefined) wraparound on signed integer.
frameIndexToBitstreamId(c2_cntr64_t frameIndex)61 int32_t frameIndexToBitstreamId(c2_cntr64_t frameIndex) {
62 return static_cast<int32_t>(frameIndex.peeku() & 0x3FFFFFFF);
63 }
64
parseCodedColorAspects(const C2ConstLinearBlock & input,C2StreamColorAspectsInfo::input * codedAspects)65 bool parseCodedColorAspects(const C2ConstLinearBlock& input,
66 C2StreamColorAspectsInfo::input* codedAspects) {
67 C2ReadView view = input.map().get();
68 NalParser parser(view.data(), view.capacity());
69
70 if (!parser.locateSPS()) {
71 ALOGV("Couldn't find SPS");
72 return false;
73 }
74
75 NalParser::ColorAspects aspects;
76 if (!parser.findCodedColorAspects(&aspects)) {
77 ALOGV("Couldn't find color description in SPS");
78 return false;
79 }
80
81 // Convert ISO color aspects to ColorUtils::ColorAspects.
82 ColorAspects colorAspects;
83 ColorUtils::convertIsoColorAspectsToCodecAspects(
84 aspects.primaries, aspects.transfer, aspects.coeffs, aspects.fullRange, colorAspects);
85 ALOGV("Parsed ColorAspects from bitstream: (R:%d, P:%d, M:%d, T:%d)", colorAspects.mRange,
86 colorAspects.mPrimaries, colorAspects.mMatrixCoeffs, colorAspects.mTransfer);
87
88 // Map ColorUtils::ColorAspects to C2StreamColorAspectsInfo::input parameter.
89 if (!C2Mapper::map(colorAspects.mPrimaries, &codedAspects->primaries)) {
90 codedAspects->primaries = C2Color::PRIMARIES_UNSPECIFIED;
91 }
92 if (!C2Mapper::map(colorAspects.mRange, &codedAspects->range)) {
93 codedAspects->range = C2Color::RANGE_UNSPECIFIED;
94 }
95 if (!C2Mapper::map(colorAspects.mMatrixCoeffs, &codedAspects->matrix)) {
96 codedAspects->matrix = C2Color::MATRIX_UNSPECIFIED;
97 }
98 if (!C2Mapper::map(colorAspects.mTransfer, &codedAspects->transfer)) {
99 codedAspects->transfer = C2Color::TRANSFER_UNSPECIFIED;
100 }
101
102 return true;
103 }
104
isWorkDone(const C2Work & work)105 bool isWorkDone(const C2Work& work) {
106 const int32_t bitstreamId = frameIndexToBitstreamId(work.input.ordinal.frameIndex);
107
108 // Exception: EOS work should be processed by reportEOSWork().
109 // Always return false here no matter the work is actually done.
110 if (work.input.flags & C2FrameData::FLAG_END_OF_STREAM) return false;
111
112 // Work is done when all conditions meet:
113 // 1. mDecoder has released the work's input buffer.
114 // 2. mDecoder has returned the work's output buffer in normal case,
115 // or the input buffer is CSD, or we decide to drop the frame.
116 bool inputReleased = (work.input.buffers.front() == nullptr);
117 bool outputReturned = !work.worklets.front()->output.buffers.empty();
118 bool ignoreOutput = (work.input.flags & C2FrameData::FLAG_CODEC_CONFIG) ||
119 (work.worklets.front()->output.flags & C2FrameData::FLAG_DROP_FRAME);
120 ALOGV("work(%d): inputReleased: %d, outputReturned: %d, ignoreOutput: %d", bitstreamId,
121 inputReleased, outputReturned, ignoreOutput);
122 return inputReleased && (outputReturned || ignoreOutput);
123 }
124
isNoShowFrameWork(const C2Work & work,const C2WorkOrdinalStruct & currOrdinal)125 bool isNoShowFrameWork(const C2Work& work, const C2WorkOrdinalStruct& currOrdinal) {
126 // We consider Work contains no-show frame when all conditions meet:
127 // 1. Work's ordinal is smaller than current ordinal.
128 // 2. Work's output buffer is not returned.
129 // 3. Work is not EOS, CSD, or marked with dropped frame.
130 bool smallOrdinal = (work.input.ordinal.timestamp < currOrdinal.timestamp) &&
131 (work.input.ordinal.frameIndex < currOrdinal.frameIndex);
132 bool outputReturned = !work.worklets.front()->output.buffers.empty();
133 bool specialWork = (work.input.flags & C2FrameData::FLAG_END_OF_STREAM) ||
134 (work.input.flags & C2FrameData::FLAG_CODEC_CONFIG) ||
135 (work.worklets.front()->output.flags & C2FrameData::FLAG_DROP_FRAME);
136 return smallOrdinal && !outputReturned && !specialWork;
137 }
138
139 } // namespace
140
141 // static
142 std::atomic<int32_t> V4L2DecodeComponent::sConcurrentInstances = 0;
143
144 // static
create(const std::string & name,c2_node_id_t id,const std::shared_ptr<C2ReflectorHelper> & helper,C2ComponentFactory::ComponentDeleter deleter)145 std::shared_ptr<C2Component> V4L2DecodeComponent::create(
146 const std::string& name, c2_node_id_t id, const std::shared_ptr<C2ReflectorHelper>& helper,
147 C2ComponentFactory::ComponentDeleter deleter) {
148 static const int32_t kMaxConcurrentInstances =
149 property_get_int32("ro.vendor.v4l2_codec2.decode_concurrent_instances", -1);
150 static std::mutex mutex;
151
152 std::lock_guard<std::mutex> lock(mutex);
153
154 if (kMaxConcurrentInstances >= 0 && sConcurrentInstances.load() >= kMaxConcurrentInstances) {
155 ALOGW("Reject to Initialize() due to too many instances: %d", sConcurrentInstances.load());
156 return nullptr;
157 }
158
159 auto intfImpl = std::make_shared<V4L2DecodeInterface>(name, helper);
160 if (intfImpl->status() != C2_OK) {
161 ALOGE("Failed to initialize V4L2DecodeInterface.");
162 return nullptr;
163 }
164
165 return std::shared_ptr<C2Component>(new V4L2DecodeComponent(name, id, helper, intfImpl),
166 deleter);
167 }
168
V4L2DecodeComponent(const std::string & name,c2_node_id_t id,const std::shared_ptr<C2ReflectorHelper> & helper,const std::shared_ptr<V4L2DecodeInterface> & intfImpl)169 V4L2DecodeComponent::V4L2DecodeComponent(const std::string& name, c2_node_id_t id,
170 const std::shared_ptr<C2ReflectorHelper>& helper,
171 const std::shared_ptr<V4L2DecodeInterface>& intfImpl)
172 : mIntfImpl(intfImpl),
173 mIntf(std::make_shared<SimpleInterface<V4L2DecodeInterface>>(name.c_str(), id, mIntfImpl)) {
174 ALOGV("%s(%s)", __func__, name.c_str());
175
176 sConcurrentInstances.fetch_add(1, std::memory_order_relaxed);
177 mIsSecure = name.find(".secure") != std::string::npos;
178 }
179
~V4L2DecodeComponent()180 V4L2DecodeComponent::~V4L2DecodeComponent() {
181 ALOGV("%s()", __func__);
182
183 release();
184
185 sConcurrentInstances.fetch_sub(1, std::memory_order_relaxed);
186 ALOGV("%s() done", __func__);
187 }
188
start()189 c2_status_t V4L2DecodeComponent::start() {
190 ALOGV("%s()", __func__);
191 std::lock_guard<std::mutex> lock(mStartStopLock);
192
193 auto currentState = mComponentState.load();
194 if (currentState != ComponentState::STOPPED) {
195 ALOGE("Could not start at %s state", ComponentStateToString(currentState));
196 return C2_BAD_STATE;
197 }
198
199 if (!mDecoderThread.Start()) {
200 ALOGE("Decoder thread failed to start.");
201 return C2_CORRUPTED;
202 }
203 mDecoderTaskRunner = mDecoderThread.task_runner();
204 mWeakThis = mWeakThisFactory.GetWeakPtr();
205
206 c2_status_t status = C2_CORRUPTED;
207 ::base::WaitableEvent done;
208 mDecoderTaskRunner->PostTask(
209 FROM_HERE, ::base::BindOnce(&V4L2DecodeComponent::startTask, mWeakThis,
210 ::base::Unretained(&status), ::base::Unretained(&done)));
211 done.Wait();
212
213 if (status == C2_OK) mComponentState.store(ComponentState::RUNNING);
214 return status;
215 }
216
startTask(c2_status_t * status,::base::WaitableEvent * done)217 void V4L2DecodeComponent::startTask(c2_status_t* status, ::base::WaitableEvent* done) {
218 ALOGV("%s()", __func__);
219 ALOG_ASSERT(mDecoderTaskRunner->RunsTasksInCurrentSequence());
220
221 ::base::ScopedClosureRunner done_caller(
222 ::base::BindOnce(&::base::WaitableEvent::Signal, ::base::Unretained(done)));
223 *status = C2_CORRUPTED;
224
225 const auto codec = mIntfImpl->getVideoCodec();
226 if (!codec) {
227 ALOGE("Failed to get video codec.");
228 return;
229 }
230 const size_t inputBufferSize = mIntfImpl->getInputBufferSize();
231 const size_t minNumOutputBuffers = getMinNumOutputBuffers(*codec);
232
233 // ::base::Unretained(this) is safe here because |mDecoder| is always destroyed before
234 // |mDecoderThread| is stopped, so |*this| is always valid during |mDecoder|'s lifetime.
235 mDecoder = V4L2Decoder::Create(*codec, inputBufferSize, minNumOutputBuffers,
236 ::base::BindRepeating(&V4L2DecodeComponent::getVideoFramePool,
237 ::base::Unretained(this)),
238 ::base::BindRepeating(&V4L2DecodeComponent::onOutputFrameReady,
239 ::base::Unretained(this)),
240 ::base::BindRepeating(&V4L2DecodeComponent::reportError,
241 ::base::Unretained(this), C2_CORRUPTED),
242 mDecoderTaskRunner);
243 if (!mDecoder) {
244 ALOGE("Failed to create V4L2Decoder for %s", VideoCodecToString(*codec));
245 return;
246 }
247
248 // Get default color aspects on start.
249 if (!mIsSecure && *codec == VideoCodec::H264) {
250 if (mIntfImpl->queryColorAspects(&mCurrentColorAspects) != C2_OK) return;
251 mPendingColorAspectsChange = false;
252 }
253
254 *status = C2_OK;
255 }
256
getVideoFramePool(const ui::Size & size,HalPixelFormat pixelFormat,size_t numBuffers)257 std::unique_ptr<VideoFramePool> V4L2DecodeComponent::getVideoFramePool(const ui::Size& size,
258 HalPixelFormat pixelFormat,
259 size_t numBuffers) {
260 ALOGV("%s()", __func__);
261 ALOG_ASSERT(mDecoderTaskRunner->RunsTasksInCurrentSequence());
262
263 auto sharedThis = weak_from_this().lock();
264 if (sharedThis == nullptr) {
265 ALOGE("%s(): V4L2DecodeComponent instance is destroyed.", __func__);
266 return nullptr;
267 }
268
269 // (b/157113946): Prevent malicious dynamic resolution change exhausts system memory.
270 constexpr int kMaximumSupportedArea = 4096 * 4096;
271 if (getArea(size).value_or(INT_MAX) > kMaximumSupportedArea) {
272 ALOGE("The output size (%dx%d) is larger than supported size (4096x4096)", size.width,
273 size.height);
274 reportError(C2_BAD_VALUE);
275 return nullptr;
276 }
277
278 // Get block pool ID configured from the client.
279 auto poolId = mIntfImpl->getBlockPoolId();
280 ALOGI("Using C2BlockPool ID = %" PRIu64 " for allocating output buffers", poolId);
281 std::shared_ptr<C2BlockPool> blockPool;
282 auto status = GetCodec2BlockPool(poolId, std::move(sharedThis), &blockPool);
283 if (status != C2_OK) {
284 ALOGE("Graphic block allocator is invalid: %d", status);
285 reportError(status);
286 return nullptr;
287 }
288
289 return VideoFramePool::Create(std::move(blockPool), numBuffers, size, pixelFormat, mIsSecure,
290 mDecoderTaskRunner);
291 }
292
stop()293 c2_status_t V4L2DecodeComponent::stop() {
294 ALOGV("%s()", __func__);
295 std::lock_guard<std::mutex> lock(mStartStopLock);
296
297 auto currentState = mComponentState.load();
298 if (currentState != ComponentState::RUNNING && currentState != ComponentState::ERROR) {
299 ALOGE("Could not stop at %s state", ComponentStateToString(currentState));
300 return C2_BAD_STATE;
301 }
302
303 if (mDecoderThread.IsRunning()) {
304 mDecoderTaskRunner->PostTask(FROM_HERE,
305 ::base::BindOnce(&V4L2DecodeComponent::stopTask, mWeakThis));
306 mDecoderThread.Stop();
307 mDecoderTaskRunner = nullptr;
308 }
309
310 mComponentState.store(ComponentState::STOPPED);
311 return C2_OK;
312 }
313
stopTask()314 void V4L2DecodeComponent::stopTask() {
315 ALOGV("%s()", __func__);
316 ALOG_ASSERT(mDecoderTaskRunner->RunsTasksInCurrentSequence());
317
318 reportAbandonedWorks();
319 mIsDraining = false;
320
321 releaseTask();
322 }
323
reset()324 c2_status_t V4L2DecodeComponent::reset() {
325 ALOGV("%s()", __func__);
326
327 return stop();
328 }
329
release()330 c2_status_t V4L2DecodeComponent::release() {
331 ALOGV("%s()", __func__);
332 std::lock_guard<std::mutex> lock(mStartStopLock);
333
334 if (mDecoderThread.IsRunning()) {
335 mDecoderTaskRunner->PostTask(
336 FROM_HERE, ::base::BindOnce(&V4L2DecodeComponent::releaseTask, mWeakThis));
337 mDecoderThread.Stop();
338 mDecoderTaskRunner = nullptr;
339 }
340
341 mComponentState.store(ComponentState::RELEASED);
342 return C2_OK;
343 }
344
releaseTask()345 void V4L2DecodeComponent::releaseTask() {
346 ALOGV("%s()", __func__);
347 ALOG_ASSERT(mDecoderTaskRunner->RunsTasksInCurrentSequence());
348
349 mWeakThisFactory.InvalidateWeakPtrs();
350 mDecoder = nullptr;
351 }
352
setListener_vb(const std::shared_ptr<C2Component::Listener> & listener,c2_blocking_t mayBlock)353 c2_status_t V4L2DecodeComponent::setListener_vb(
354 const std::shared_ptr<C2Component::Listener>& listener, c2_blocking_t mayBlock) {
355 ALOGV("%s()", __func__);
356
357 auto currentState = mComponentState.load();
358 if (currentState == ComponentState::RELEASED ||
359 (currentState == ComponentState::RUNNING && listener)) {
360 ALOGE("Could not set listener at %s state", ComponentStateToString(currentState));
361 return C2_BAD_STATE;
362 }
363 if (currentState == ComponentState::RUNNING && mayBlock != C2_MAY_BLOCK) {
364 ALOGE("Could not set listener at %s state non-blocking",
365 ComponentStateToString(currentState));
366 return C2_BLOCKING;
367 }
368
369 // If the decoder thread is not running it's safe to update the listener directly.
370 if (!mDecoderThread.IsRunning()) {
371 mListener = listener;
372 return C2_OK;
373 }
374
375 ::base::WaitableEvent done;
376 mDecoderTaskRunner->PostTask(FROM_HERE, ::base::Bind(&V4L2DecodeComponent::setListenerTask,
377 mWeakThis, listener, &done));
378 done.Wait();
379 return C2_OK;
380 }
381
setListenerTask(const std::shared_ptr<Listener> & listener,::base::WaitableEvent * done)382 void V4L2DecodeComponent::setListenerTask(const std::shared_ptr<Listener>& listener,
383 ::base::WaitableEvent* done) {
384 ALOGV("%s()", __func__);
385 ALOG_ASSERT(mDecoderTaskRunner->RunsTasksInCurrentSequence());
386
387 mListener = listener;
388 done->Signal();
389 }
390
queue_nb(std::list<std::unique_ptr<C2Work>> * const items)391 c2_status_t V4L2DecodeComponent::queue_nb(std::list<std::unique_ptr<C2Work>>* const items) {
392 ALOGV("%s()", __func__);
393
394 auto currentState = mComponentState.load();
395 if (currentState != ComponentState::RUNNING) {
396 ALOGE("Could not queue at state: %s", ComponentStateToString(currentState));
397 return C2_BAD_STATE;
398 }
399
400 while (!items->empty()) {
401 mDecoderTaskRunner->PostTask(FROM_HERE,
402 ::base::BindOnce(&V4L2DecodeComponent::queueTask, mWeakThis,
403 std::move(items->front())));
404 items->pop_front();
405 }
406 return C2_OK;
407 }
408
queueTask(std::unique_ptr<C2Work> work)409 void V4L2DecodeComponent::queueTask(std::unique_ptr<C2Work> work) {
410 ALOGV("%s(): flags=0x%x, index=%llu, timestamp=%llu", __func__, work->input.flags,
411 work->input.ordinal.frameIndex.peekull(), work->input.ordinal.timestamp.peekull());
412 ALOG_ASSERT(mDecoderTaskRunner->RunsTasksInCurrentSequence());
413
414 if (work->worklets.size() != 1u || work->input.buffers.size() > 1u) {
415 ALOGE("Invalid work: worklets.size()=%zu, input.buffers.size()=%zu", work->worklets.size(),
416 work->input.buffers.size());
417 work->result = C2_CORRUPTED;
418 reportWork(std::move(work));
419 return;
420 }
421
422 work->worklets.front()->output.flags = static_cast<C2FrameData::flags_t>(0);
423 work->worklets.front()->output.buffers.clear();
424 work->worklets.front()->output.ordinal = work->input.ordinal;
425 if (work->input.buffers.empty()) {
426 // Client may queue a work with no input buffer for either it's EOS or empty CSD, otherwise
427 // every work must have one input buffer.
428 if ((work->input.flags & C2FrameData::FLAG_END_OF_STREAM) == 0 &&
429 (work->input.flags & C2FrameData::FLAG_CODEC_CONFIG) == 0) {
430 ALOGE("Invalid work: work with no input buffer should be EOS or CSD.");
431 reportError(C2_BAD_VALUE);
432 return;
433 }
434
435 // Emplace a nullptr to unify the check for work done.
436 ALOGV("Got a work with no input buffer! Emplace a nullptr inside.");
437 work->input.buffers.emplace_back(nullptr);
438 }
439
440 mPendingWorks.push(std::move(work));
441 pumpPendingWorks();
442 }
443
pumpPendingWorks()444 void V4L2DecodeComponent::pumpPendingWorks() {
445 ALOGV("%s()", __func__);
446 ALOG_ASSERT(mDecoderTaskRunner->RunsTasksInCurrentSequence());
447
448 auto currentState = mComponentState.load();
449 if (currentState != ComponentState::RUNNING) {
450 ALOGW("Could not pump C2Work at state: %s", ComponentStateToString(currentState));
451 return;
452 }
453
454 while (!mPendingWorks.empty() && !mIsDraining) {
455 std::unique_ptr<C2Work> pendingWork(std::move(mPendingWorks.front()));
456 mPendingWorks.pop();
457
458 const int32_t bitstreamId = frameIndexToBitstreamId(pendingWork->input.ordinal.frameIndex);
459 const bool isCSDWork = pendingWork->input.flags & C2FrameData::FLAG_CODEC_CONFIG;
460 const bool isEmptyWork = pendingWork->input.buffers.front() == nullptr;
461 const bool isEOSWork = pendingWork->input.flags & C2FrameData::FLAG_END_OF_STREAM;
462 const C2Work* work = pendingWork.get();
463 ALOGV("Process C2Work bitstreamId=%d isCSDWork=%d, isEmptyWork=%d", bitstreamId, isCSDWork,
464 isEmptyWork);
465
466 auto res = mWorksAtDecoder.insert(std::make_pair(bitstreamId, std::move(pendingWork)));
467 ALOGW_IF(!res.second, "We already inserted bitstreamId %d to decoder?", bitstreamId);
468
469 if (!isEmptyWork) {
470 // If input.buffers is not empty, the buffer should have meaningful content inside.
471 C2ConstLinearBlock linearBlock =
472 work->input.buffers.front()->data().linearBlocks().front();
473 ALOG_ASSERT(linearBlock.size() > 0u, "Input buffer of work(%d) is empty.", bitstreamId);
474
475 // Try to parse color aspects from bitstream for CSD work of non-secure H264 codec.
476 if (isCSDWork && !mIsSecure && (mIntfImpl->getVideoCodec() == VideoCodec::H264)) {
477 C2StreamColorAspectsInfo::input codedAspects = {0u};
478 if (parseCodedColorAspects(linearBlock, &codedAspects)) {
479 std::vector<std::unique_ptr<C2SettingResult>> failures;
480 c2_status_t status =
481 mIntfImpl->config({&codedAspects}, C2_MAY_BLOCK, &failures);
482 if (status != C2_OK) {
483 ALOGE("Failed to config color aspects to interface: %d", status);
484 reportError(status);
485 return;
486 }
487
488 // Record current frame index, color aspects should be updated only for output
489 // buffers whose frame indices are not less than this one.
490 mPendingColorAspectsChange = true;
491 mPendingColorAspectsChangeFrameIndex = work->input.ordinal.frameIndex.peeku();
492 }
493 }
494
495 std::unique_ptr<ConstBitstreamBuffer> buffer = std::make_unique<ConstBitstreamBuffer>(
496 bitstreamId, linearBlock, linearBlock.offset(), linearBlock.size());
497 if (!buffer) {
498 reportError(C2_CORRUPTED);
499 return;
500 }
501 mDecoder->decode(std::move(buffer), ::base::BindOnce(&V4L2DecodeComponent::onDecodeDone,
502 mWeakThis, bitstreamId));
503 }
504
505 if (isEOSWork) {
506 mDecoder->drain(::base::BindOnce(&V4L2DecodeComponent::onDrainDone, mWeakThis));
507 mIsDraining = true;
508 }
509
510 // Directly report the empty CSD work as finished.
511 if (isCSDWork && isEmptyWork) reportWorkIfFinished(bitstreamId);
512 }
513 }
514
onDecodeDone(int32_t bitstreamId,VideoDecoder::DecodeStatus status)515 void V4L2DecodeComponent::onDecodeDone(int32_t bitstreamId, VideoDecoder::DecodeStatus status) {
516 ALOGV("%s(bitstreamId=%d, status=%s)", __func__, bitstreamId,
517 VideoDecoder::DecodeStatusToString(status));
518 ALOG_ASSERT(mDecoderTaskRunner->RunsTasksInCurrentSequence());
519
520 auto it = mWorksAtDecoder.find(bitstreamId);
521 ALOG_ASSERT(it != mWorksAtDecoder.end());
522 C2Work* work = it->second.get();
523
524 switch (status) {
525 case VideoDecoder::DecodeStatus::kAborted:
526 work->input.buffers.front().reset();
527 work->worklets.front()->output.flags = static_cast<C2FrameData::flags_t>(
528 work->worklets.front()->output.flags & C2FrameData::FLAG_DROP_FRAME);
529 mOutputBitstreamIds.push(bitstreamId);
530
531 pumpReportWork();
532 return;
533
534 case VideoDecoder::DecodeStatus::kError:
535 reportError(C2_CORRUPTED);
536 return;
537
538 case VideoDecoder::DecodeStatus::kOk:
539 // Release the input buffer.
540 work->input.buffers.front().reset();
541
542 // CSD Work doesn't have output buffer, the corresponding onOutputFrameReady() won't be
543 // called. Push the bitstreamId here.
544 if (work->input.flags & C2FrameData::FLAG_CODEC_CONFIG)
545 mOutputBitstreamIds.push(bitstreamId);
546
547 pumpReportWork();
548 return;
549 }
550 }
551
onOutputFrameReady(std::unique_ptr<VideoFrame> frame)552 void V4L2DecodeComponent::onOutputFrameReady(std::unique_ptr<VideoFrame> frame) {
553 ALOGV("%s(bitstreamId=%d)", __func__, frame->getBitstreamId());
554 ALOG_ASSERT(mDecoderTaskRunner->RunsTasksInCurrentSequence());
555
556 const int32_t bitstreamId = frame->getBitstreamId();
557 auto it = mWorksAtDecoder.find(bitstreamId);
558 if (it == mWorksAtDecoder.end()) {
559 ALOGE("Work with bitstreamId=%d not found, already abandoned?", bitstreamId);
560 reportError(C2_CORRUPTED);
561 return;
562 }
563 C2Work* work = it->second.get();
564
565 C2ConstGraphicBlock constBlock = std::move(frame)->getGraphicBlock();
566 std::shared_ptr<C2Buffer> buffer = C2Buffer::CreateGraphicBuffer(std::move(constBlock));
567 if (mPendingColorAspectsChange &&
568 work->input.ordinal.frameIndex.peeku() >= mPendingColorAspectsChangeFrameIndex) {
569 mIntfImpl->queryColorAspects(&mCurrentColorAspects);
570 mPendingColorAspectsChange = false;
571 }
572 if (mCurrentColorAspects) {
573 buffer->setInfo(mCurrentColorAspects);
574 }
575 work->worklets.front()->output.buffers.emplace_back(std::move(buffer));
576
577 // Check no-show frame by timestamps for VP8/VP9 cases before reporting the current work.
578 if (mIntfImpl->getVideoCodec() == VideoCodec::VP8 ||
579 mIntfImpl->getVideoCodec() == VideoCodec::VP9) {
580 detectNoShowFrameWorksAndReportIfFinished(work->input.ordinal);
581 }
582
583 mOutputBitstreamIds.push(bitstreamId);
584 pumpReportWork();
585 }
586
detectNoShowFrameWorksAndReportIfFinished(const C2WorkOrdinalStruct & currOrdinal)587 void V4L2DecodeComponent::detectNoShowFrameWorksAndReportIfFinished(
588 const C2WorkOrdinalStruct& currOrdinal) {
589 ALOGV("%s()", __func__);
590 ALOG_ASSERT(mDecoderTaskRunner->RunsTasksInCurrentSequence());
591
592 std::vector<int32_t> noShowFrameBitstreamIds;
593 for (auto& kv : mWorksAtDecoder) {
594 const int32_t bitstreamId = kv.first;
595 const C2Work* work = kv.second.get();
596
597 // A work in mWorksAtDecoder would be considered to have no-show frame if there is no
598 // corresponding output buffer returned while the one of the work with latter timestamp is
599 // already returned. (VD is outputted in display order.)
600 if (isNoShowFrameWork(*work, currOrdinal)) {
601 work->worklets.front()->output.flags = C2FrameData::FLAG_DROP_FRAME;
602
603 // We need to call reportWorkIfFinished() for all detected no-show frame works. However,
604 // we should do it after the detection loop since reportWorkIfFinished() may erase
605 // entries in |mWorksAtDecoder|.
606 noShowFrameBitstreamIds.push_back(bitstreamId);
607 ALOGV("Detected no-show frame work index=%llu timestamp=%llu",
608 work->input.ordinal.frameIndex.peekull(),
609 work->input.ordinal.timestamp.peekull());
610 }
611 }
612
613 // Try to report works with no-show frame.
614 for (const int32_t bitstreamId : noShowFrameBitstreamIds) reportWorkIfFinished(bitstreamId);
615 }
616
pumpReportWork()617 void V4L2DecodeComponent::pumpReportWork() {
618 ALOGV("%s()", __func__);
619 ALOG_ASSERT(mDecoderTaskRunner->RunsTasksInCurrentSequence());
620
621 while (!mOutputBitstreamIds.empty()) {
622 if (!reportWorkIfFinished(mOutputBitstreamIds.front())) break;
623 mOutputBitstreamIds.pop();
624 }
625 }
626
reportWorkIfFinished(int32_t bitstreamId)627 bool V4L2DecodeComponent::reportWorkIfFinished(int32_t bitstreamId) {
628 ALOGV("%s(bitstreamId = %d)", __func__, bitstreamId);
629 ALOG_ASSERT(mDecoderTaskRunner->RunsTasksInCurrentSequence());
630
631 // EOS work will not be reported here. reportEOSWork() does it.
632 if (mIsDraining && mWorksAtDecoder.size() == 1u) {
633 ALOGV("work(bitstreamId = %d) is EOS Work.", bitstreamId);
634 return false;
635 }
636
637 auto it = mWorksAtDecoder.find(bitstreamId);
638 if (it == mWorksAtDecoder.end()) {
639 ALOGI("work(bitstreamId = %d) is dropped, skip.", bitstreamId);
640 return true;
641 }
642
643 if (!isWorkDone(*(it->second))) {
644 ALOGV("work(bitstreamId = %d) is not done yet.", bitstreamId);
645 return false;
646 }
647
648 std::unique_ptr<C2Work> work = std::move(it->second);
649 mWorksAtDecoder.erase(it);
650
651 work->result = C2_OK;
652 work->workletsProcessed = static_cast<uint32_t>(work->worklets.size());
653 // A work with neither flags nor output buffer would be treated as no-corresponding
654 // output by C2 framework, and regain pipeline capacity immediately.
655 if (work->worklets.front()->output.flags & C2FrameData::FLAG_DROP_FRAME)
656 work->worklets.front()->output.flags = static_cast<C2FrameData::flags_t>(0);
657
658 return reportWork(std::move(work));
659 }
660
reportEOSWork()661 bool V4L2DecodeComponent::reportEOSWork() {
662 ALOGV("%s()", __func__);
663 ALOG_ASSERT(mDecoderTaskRunner->RunsTasksInCurrentSequence());
664
665 const auto it =
666 std::find_if(mWorksAtDecoder.begin(), mWorksAtDecoder.end(), [](const auto& kv) {
667 return kv.second->input.flags & C2FrameData::FLAG_END_OF_STREAM;
668 });
669 if (it == mWorksAtDecoder.end()) {
670 ALOGE("Failed to find EOS work.");
671 return false;
672 }
673
674 std::unique_ptr<C2Work> eosWork(std::move(it->second));
675 mWorksAtDecoder.erase(it);
676
677 eosWork->result = C2_OK;
678 eosWork->workletsProcessed = static_cast<uint32_t>(eosWork->worklets.size());
679 eosWork->worklets.front()->output.flags = C2FrameData::FLAG_END_OF_STREAM;
680 if (!eosWork->input.buffers.empty()) eosWork->input.buffers.front().reset();
681
682 if (!mWorksAtDecoder.empty()) {
683 ALOGW("There are remaining works except EOS work. abandon them.");
684 for (const auto& kv : mWorksAtDecoder) {
685 ALOGW("bitstreamId(%d) => Work index=%llu, timestamp=%llu", kv.first,
686 kv.second->input.ordinal.frameIndex.peekull(),
687 kv.second->input.ordinal.timestamp.peekull());
688 }
689 reportAbandonedWorks();
690 }
691
692 return reportWork(std::move(eosWork));
693 }
694
reportWork(std::unique_ptr<C2Work> work)695 bool V4L2DecodeComponent::reportWork(std::unique_ptr<C2Work> work) {
696 ALOGV("%s(work=%llu)", __func__, work->input.ordinal.frameIndex.peekull());
697 ALOG_ASSERT(mDecoderTaskRunner->RunsTasksInCurrentSequence());
698
699 if (!mListener) {
700 ALOGE("mListener is nullptr, setListener_vb() not called?");
701 return false;
702 }
703
704 std::list<std::unique_ptr<C2Work>> finishedWorks;
705 finishedWorks.emplace_back(std::move(work));
706 mListener->onWorkDone_nb(weak_from_this(), std::move(finishedWorks));
707 return true;
708 }
709
flush_sm(flush_mode_t mode,std::list<std::unique_ptr<C2Work>> * const)710 c2_status_t V4L2DecodeComponent::flush_sm(
711 flush_mode_t mode, std::list<std::unique_ptr<C2Work>>* const /* flushedWork */) {
712 ALOGV("%s()", __func__);
713
714 auto currentState = mComponentState.load();
715 if (currentState != ComponentState::RUNNING) {
716 ALOGE("Could not flush at state: %s", ComponentStateToString(currentState));
717 return C2_BAD_STATE;
718 }
719 if (mode != FLUSH_COMPONENT) {
720 return C2_OMITTED; // Tunneling is not supported by now
721 }
722
723 mDecoderTaskRunner->PostTask(FROM_HERE,
724 ::base::BindOnce(&V4L2DecodeComponent::flushTask, mWeakThis));
725 return C2_OK;
726 }
727
flushTask()728 void V4L2DecodeComponent::flushTask() {
729 ALOGV("%s()", __func__);
730 ALOG_ASSERT(mDecoderTaskRunner->RunsTasksInCurrentSequence());
731
732 mDecoder->flush();
733 reportAbandonedWorks();
734
735 // Pending EOS work will be abandoned here due to component flush if any.
736 mIsDraining = false;
737 }
738
reportAbandonedWorks()739 void V4L2DecodeComponent::reportAbandonedWorks() {
740 ALOGV("%s()", __func__);
741 ALOG_ASSERT(mDecoderTaskRunner->RunsTasksInCurrentSequence());
742
743 std::list<std::unique_ptr<C2Work>> abandonedWorks;
744 while (!mPendingWorks.empty()) {
745 abandonedWorks.emplace_back(std::move(mPendingWorks.front()));
746 mPendingWorks.pop();
747 }
748 for (auto& kv : mWorksAtDecoder) {
749 abandonedWorks.emplace_back(std::move(kv.second));
750 }
751 mWorksAtDecoder.clear();
752
753 for (auto& work : abandonedWorks) {
754 // TODO: correlate the definition of flushed work result to framework.
755 work->result = C2_NOT_FOUND;
756 // When the work is abandoned, buffer in input.buffers shall reset by component.
757 if (!work->input.buffers.empty()) {
758 work->input.buffers.front().reset();
759 }
760 }
761 if (!abandonedWorks.empty()) {
762 if (!mListener) {
763 ALOGE("mListener is nullptr, setListener_vb() not called?");
764 return;
765 }
766 mListener->onWorkDone_nb(weak_from_this(), std::move(abandonedWorks));
767 }
768 }
769
drain_nb(drain_mode_t mode)770 c2_status_t V4L2DecodeComponent::drain_nb(drain_mode_t mode) {
771 ALOGV("%s(mode=%u)", __func__, mode);
772
773 auto currentState = mComponentState.load();
774 if (currentState != ComponentState::RUNNING) {
775 ALOGE("Could not drain at state: %s", ComponentStateToString(currentState));
776 return C2_BAD_STATE;
777 }
778
779 switch (mode) {
780 case DRAIN_CHAIN:
781 return C2_OMITTED; // Tunneling is not supported.
782
783 case DRAIN_COMPONENT_NO_EOS:
784 return C2_OK; // Do nothing special.
785
786 case DRAIN_COMPONENT_WITH_EOS:
787 mDecoderTaskRunner->PostTask(FROM_HERE,
788 ::base::BindOnce(&V4L2DecodeComponent::drainTask, mWeakThis));
789 return C2_OK;
790 }
791 }
792
drainTask()793 void V4L2DecodeComponent::drainTask() {
794 ALOGV("%s()", __func__);
795 ALOG_ASSERT(mDecoderTaskRunner->RunsTasksInCurrentSequence());
796
797 if (!mPendingWorks.empty()) {
798 ALOGV("Set EOS flag at last queued work.");
799 auto& flags = mPendingWorks.back()->input.flags;
800 flags = static_cast<C2FrameData::flags_t>(flags | C2FrameData::FLAG_END_OF_STREAM);
801 return;
802 }
803
804 if (!mWorksAtDecoder.empty()) {
805 ALOGV("Drain the pending works at the decoder.");
806 mDecoder->drain(::base::BindOnce(&V4L2DecodeComponent::onDrainDone, mWeakThis));
807 mIsDraining = true;
808 }
809 }
810
onDrainDone(VideoDecoder::DecodeStatus status)811 void V4L2DecodeComponent::onDrainDone(VideoDecoder::DecodeStatus status) {
812 ALOGV("%s(status=%s)", __func__, VideoDecoder::DecodeStatusToString(status));
813 ALOG_ASSERT(mDecoderTaskRunner->RunsTasksInCurrentSequence());
814
815 switch (status) {
816 case VideoDecoder::DecodeStatus::kAborted:
817 return;
818
819 case VideoDecoder::DecodeStatus::kError:
820 reportError(C2_CORRUPTED);
821 return;
822
823 case VideoDecoder::DecodeStatus::kOk:
824 mIsDraining = false;
825 if (!reportEOSWork()) {
826 reportError(C2_CORRUPTED);
827 return;
828 }
829
830 mDecoderTaskRunner->PostTask(
831 FROM_HERE, ::base::BindOnce(&V4L2DecodeComponent::pumpPendingWorks, mWeakThis));
832 return;
833 }
834 }
835
reportError(c2_status_t error)836 void V4L2DecodeComponent::reportError(c2_status_t error) {
837 ALOGE("%s(error=%u)", __func__, static_cast<uint32_t>(error));
838 ALOG_ASSERT(mDecoderTaskRunner->RunsTasksInCurrentSequence());
839
840 if (mComponentState.load() == ComponentState::ERROR) return;
841 mComponentState.store(ComponentState::ERROR);
842
843 if (!mListener) {
844 ALOGE("mListener is nullptr, setListener_vb() not called?");
845 return;
846 }
847 mListener->onError_nb(weak_from_this(), static_cast<uint32_t>(error));
848 }
849
announce_nb(const std::vector<C2WorkOutline> &)850 c2_status_t V4L2DecodeComponent::announce_nb(const std::vector<C2WorkOutline>& /* items */) {
851 return C2_OMITTED; // Tunneling is not supported by now
852 }
853
intf()854 std::shared_ptr<C2ComponentInterface> V4L2DecodeComponent::intf() {
855 return mIntf;
856 }
857
858 // static
ComponentStateToString(ComponentState state)859 const char* V4L2DecodeComponent::ComponentStateToString(ComponentState state) {
860 switch (state) {
861 case ComponentState::STOPPED:
862 return "STOPPED";
863 case ComponentState::RUNNING:
864 return "RUNNING";
865 case ComponentState::RELEASED:
866 return "RELEASED";
867 case ComponentState::ERROR:
868 return "ERROR";
869 }
870 }
871
872 } // namespace android
873