1 /*
2 * Copyright 2017, The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 //#define LOG_NDEBUG 0
18 #include <utils/Errors.h>
19 #define LOG_TAG "CCodecBufferChannel"
20 #define ATRACE_TAG ATRACE_TAG_VIDEO
21 #include <utils/Log.h>
22 #include <utils/Trace.h>
23
24 #include <algorithm>
25 #include <atomic>
26 #include <list>
27 #include <numeric>
28 #include <thread>
29 #include <chrono>
30
31 #include <C2AllocatorGralloc.h>
32 #include <C2PlatformSupport.h>
33 #include <C2BlockInternal.h>
34 #include <C2Config.h>
35 #include <C2Debug.h>
36
37 #include <android/hardware/cas/native/1.0/IDescrambler.h>
38 #include <android/hardware/drm/1.0/types.h>
39 #include <android-base/parseint.h>
40 #include <android-base/properties.h>
41 #include <android-base/stringprintf.h>
42 #include <binder/MemoryBase.h>
43 #include <binder/MemoryDealer.h>
44 #include <cutils/properties.h>
45 #include <gui/Surface.h>
46 #include <hidlmemory/FrameworkUtils.h>
47 #include <media/openmax/OMX_Core.h>
48 #include <media/stagefright/foundation/ABuffer.h>
49 #include <media/stagefright/foundation/ALookup.h>
50 #include <media/stagefright/foundation/AMessage.h>
51 #include <media/stagefright/foundation/AUtils.h>
52 #include <media/stagefright/foundation/hexdump.h>
53 #include <media/stagefright/MediaCodecConstants.h>
54 #include <media/stagefright/SkipCutBuffer.h>
55 #include <media/stagefright/SurfaceUtils.h>
56 #include <media/MediaCodecBuffer.h>
57 #include <mediadrm/ICrypto.h>
58 #include <server_configurable_flags/get_flags.h>
59 #include <system/window.h>
60
61 #include "CCodecBufferChannel.h"
62 #include "Codec2Buffer.h"
63
64 namespace android {
65
66 using android::base::StringPrintf;
67 using hardware::hidl_handle;
68 using hardware::hidl_string;
69 using hardware::hidl_vec;
70 using hardware::fromHeap;
71 using hardware::HidlMemory;
72 using server_configurable_flags::GetServerConfigurableFlag;
73
74 using namespace hardware::cas::V1_0;
75 using namespace hardware::cas::native::V1_0;
76
77 using CasStatus = hardware::cas::V1_0::Status;
78 using DrmBufferType = hardware::drm::V1_0::BufferType;
79
80 namespace {
81
82 constexpr size_t kSmoothnessFactor = 4;
83
84 // This is for keeping IGBP's buffer dropping logic in legacy mode other
85 // than making it non-blocking. Do not change this value.
86 const static size_t kDequeueTimeoutNs = 0;
87
areRenderMetricsEnabled()88 static bool areRenderMetricsEnabled() {
89 std::string v = GetServerConfigurableFlag("media_native", "render_metrics_enabled", "false");
90 return v == "true";
91 }
92
93 } // namespace
94
QueueGuard(CCodecBufferChannel::QueueSync & sync)95 CCodecBufferChannel::QueueGuard::QueueGuard(
96 CCodecBufferChannel::QueueSync &sync) : mSync(sync) {
97 Mutex::Autolock l(mSync.mGuardLock);
98 // At this point it's guaranteed that mSync is not under state transition,
99 // as we are holding its mutex.
100
101 Mutexed<CCodecBufferChannel::QueueSync::Counter>::Locked count(mSync.mCount);
102 if (count->value == -1) {
103 mRunning = false;
104 } else {
105 ++count->value;
106 mRunning = true;
107 }
108 }
109
~QueueGuard()110 CCodecBufferChannel::QueueGuard::~QueueGuard() {
111 if (mRunning) {
112 // We are not holding mGuardLock at this point so that QueueSync::stop() can
113 // keep holding the lock until mCount reaches zero.
114 Mutexed<CCodecBufferChannel::QueueSync::Counter>::Locked count(mSync.mCount);
115 --count->value;
116 count->cond.broadcast();
117 }
118 }
119
start()120 void CCodecBufferChannel::QueueSync::start() {
121 Mutex::Autolock l(mGuardLock);
122 // If stopped, it goes to running state; otherwise no-op.
123 Mutexed<Counter>::Locked count(mCount);
124 if (count->value == -1) {
125 count->value = 0;
126 }
127 }
128
stop()129 void CCodecBufferChannel::QueueSync::stop() {
130 Mutex::Autolock l(mGuardLock);
131 Mutexed<Counter>::Locked count(mCount);
132 if (count->value == -1) {
133 // no-op
134 return;
135 }
136 // Holding mGuardLock here blocks creation of additional QueueGuard objects, so
137 // mCount can only decrement. In other words, threads that acquired the lock
138 // are allowed to finish execution but additional threads trying to acquire
139 // the lock at this point will block, and then get QueueGuard at STOPPED
140 // state.
141 while (count->value != 0) {
142 count.waitForCondition(count->cond);
143 }
144 count->value = -1;
145 }
146
147 // Input
148
Input()149 CCodecBufferChannel::Input::Input() : extraBuffers("extra") {}
150
151 // CCodecBufferChannel
152
CCodecBufferChannel(const std::shared_ptr<CCodecCallback> & callback)153 CCodecBufferChannel::CCodecBufferChannel(
154 const std::shared_ptr<CCodecCallback> &callback)
155 : mHeapSeqNum(-1),
156 mCCodecCallback(callback),
157 mFrameIndex(0u),
158 mFirstValidFrameIndex(0u),
159 mAreRenderMetricsEnabled(areRenderMetricsEnabled()),
160 mIsSurfaceToDisplay(false),
161 mHasPresentFenceTimes(false),
162 mRenderingDepth(3u),
163 mMetaMode(MODE_NONE),
164 mInputMetEos(false),
165 mSendEncryptedInfoBuffer(false) {
166 {
167 Mutexed<Input>::Locked input(mInput);
168 input->buffers.reset(new DummyInputBuffers(""));
169 input->extraBuffers.flush();
170 input->inputDelay = 0u;
171 input->pipelineDelay = 0u;
172 input->numSlots = kSmoothnessFactor;
173 input->numExtraSlots = 0u;
174 input->lastFlushIndex = 0u;
175 }
176 {
177 Mutexed<Output>::Locked output(mOutput);
178 output->outputDelay = 0u;
179 output->numSlots = kSmoothnessFactor;
180 output->bounded = false;
181 }
182 {
183 Mutexed<BlockPools>::Locked pools(mBlockPools);
184 pools->outputPoolId = C2BlockPool::BASIC_LINEAR;
185 }
186 std::string value = GetServerConfigurableFlag("media_native", "ccodec_rendering_depth", "3");
187 android::base::ParseInt(value, &mRenderingDepth);
188 mOutputSurface.lock()->maxDequeueBuffers = kSmoothnessFactor + mRenderingDepth;
189 }
190
~CCodecBufferChannel()191 CCodecBufferChannel::~CCodecBufferChannel() {
192 if (mCrypto != nullptr && mHeapSeqNum >= 0) {
193 mCrypto->unsetHeap(mHeapSeqNum);
194 }
195 }
196
setComponent(const std::shared_ptr<Codec2Client::Component> & component)197 void CCodecBufferChannel::setComponent(
198 const std::shared_ptr<Codec2Client::Component> &component) {
199 mComponent = component;
200 mComponentName = component->getName() + StringPrintf("#%d", int(uintptr_t(component.get()) % 997));
201 mName = mComponentName.c_str();
202 }
203
setInputSurface(const std::shared_ptr<InputSurfaceWrapper> & surface)204 status_t CCodecBufferChannel::setInputSurface(
205 const std::shared_ptr<InputSurfaceWrapper> &surface) {
206 ALOGV("[%s] setInputSurface", mName);
207 mInputSurface = surface;
208 return mInputSurface->connect(mComponent);
209 }
210
signalEndOfInputStream()211 status_t CCodecBufferChannel::signalEndOfInputStream() {
212 if (mInputSurface == nullptr) {
213 return INVALID_OPERATION;
214 }
215 return mInputSurface->signalEndOfInputStream();
216 }
217
queueInputBufferInternal(sp<MediaCodecBuffer> buffer,std::shared_ptr<C2LinearBlock> encryptedBlock,size_t blockSize)218 status_t CCodecBufferChannel::queueInputBufferInternal(
219 sp<MediaCodecBuffer> buffer,
220 std::shared_ptr<C2LinearBlock> encryptedBlock,
221 size_t blockSize) {
222 int64_t timeUs;
223 CHECK(buffer->meta()->findInt64("timeUs", &timeUs));
224
225 if (mInputMetEos) {
226 ALOGD("[%s] buffers after EOS ignored (%lld us)", mName, (long long)timeUs);
227 return OK;
228 }
229
230 int32_t flags = 0;
231 int32_t tmp = 0;
232 bool eos = false;
233 bool tunnelFirstFrame = false;
234 if (buffer->meta()->findInt32("eos", &tmp) && tmp) {
235 eos = true;
236 mInputMetEos = true;
237 ALOGV("[%s] input EOS", mName);
238 }
239 if (buffer->meta()->findInt32("csd", &tmp) && tmp) {
240 flags |= C2FrameData::FLAG_CODEC_CONFIG;
241 }
242 if (buffer->meta()->findInt32("tunnel-first-frame", &tmp) && tmp) {
243 tunnelFirstFrame = true;
244 }
245 if (buffer->meta()->findInt32("decode-only", &tmp) && tmp) {
246 flags |= C2FrameData::FLAG_DROP_FRAME;
247 }
248 ALOGV("[%s] queueInputBuffer: buffer->size() = %zu", mName, buffer->size());
249 std::list<std::unique_ptr<C2Work>> items;
250 std::unique_ptr<C2Work> work(new C2Work);
251 work->input.ordinal.timestamp = timeUs;
252 work->input.ordinal.frameIndex = mFrameIndex++;
253 // WORKAROUND: until codecs support handling work after EOS and max output sizing, use timestamp
254 // manipulation to achieve image encoding via video codec, and to constrain encoded output.
255 // Keep client timestamp in customOrdinal
256 work->input.ordinal.customOrdinal = timeUs;
257 work->input.buffers.clear();
258
259 sp<Codec2Buffer> copy;
260 bool usesFrameReassembler = false;
261
262 if (buffer->size() > 0u) {
263 Mutexed<Input>::Locked input(mInput);
264 std::shared_ptr<C2Buffer> c2buffer;
265 if (!input->buffers->releaseBuffer(buffer, &c2buffer, false)) {
266 return -ENOENT;
267 }
268 // TODO: we want to delay copying buffers.
269 if (input->extraBuffers.numComponentBuffers() < input->numExtraSlots) {
270 copy = input->buffers->cloneAndReleaseBuffer(buffer);
271 if (copy != nullptr) {
272 (void)input->extraBuffers.assignSlot(copy);
273 if (!input->extraBuffers.releaseSlot(copy, &c2buffer, false)) {
274 return UNKNOWN_ERROR;
275 }
276 bool released = input->buffers->releaseBuffer(buffer, nullptr, true);
277 ALOGV("[%s] queueInputBuffer: buffer copied; %sreleased",
278 mName, released ? "" : "not ");
279 buffer = copy;
280 } else {
281 ALOGW("[%s] queueInputBuffer: failed to copy a buffer; this may cause input "
282 "buffer starvation on component.", mName);
283 }
284 }
285 if (input->frameReassembler) {
286 usesFrameReassembler = true;
287 input->frameReassembler.process(buffer, &items);
288 } else {
289 int32_t cvo = 0;
290 if (buffer->meta()->findInt32("cvo", &cvo)) {
291 int32_t rotation = cvo % 360;
292 // change rotation to counter-clock wise.
293 rotation = ((rotation <= 0) ? 0 : 360) - rotation;
294
295 Mutexed<OutputSurface>::Locked output(mOutputSurface);
296 uint64_t frameIndex = work->input.ordinal.frameIndex.peeku();
297 output->rotation[frameIndex] = rotation;
298 }
299 work->input.buffers.push_back(c2buffer);
300 if (encryptedBlock) {
301 work->input.infoBuffers.emplace_back(C2InfoBuffer::CreateLinearBuffer(
302 kParamIndexEncryptedBuffer,
303 encryptedBlock->share(0, blockSize, C2Fence())));
304 }
305 }
306 } else if (eos) {
307 Mutexed<Input>::Locked input(mInput);
308 if (input->frameReassembler) {
309 usesFrameReassembler = true;
310 // drain any pending items with eos
311 input->frameReassembler.process(buffer, &items);
312 }
313 flags |= C2FrameData::FLAG_END_OF_STREAM;
314 }
315 if (usesFrameReassembler) {
316 if (!items.empty()) {
317 items.front()->input.configUpdate = std::move(mParamsToBeSet);
318 mFrameIndex = (items.back()->input.ordinal.frameIndex + 1).peek();
319 }
320 } else {
321 work->input.flags = (C2FrameData::flags_t)flags;
322 // TODO: fill info's
323
324 work->input.configUpdate = std::move(mParamsToBeSet);
325 if (tunnelFirstFrame) {
326 C2StreamTunnelHoldRender::input tunnelHoldRender{
327 0u /* stream */,
328 C2_TRUE /* value */
329 };
330 work->input.configUpdate.push_back(C2Param::Copy(tunnelHoldRender));
331 }
332 work->worklets.clear();
333 work->worklets.emplace_back(new C2Worklet);
334
335 items.push_back(std::move(work));
336
337 eos = eos && buffer->size() > 0u;
338 }
339 if (eos) {
340 work.reset(new C2Work);
341 work->input.ordinal.timestamp = timeUs;
342 work->input.ordinal.frameIndex = mFrameIndex++;
343 // WORKAROUND: keep client timestamp in customOrdinal
344 work->input.ordinal.customOrdinal = timeUs;
345 work->input.buffers.clear();
346 work->input.flags = C2FrameData::FLAG_END_OF_STREAM;
347 work->worklets.emplace_back(new C2Worklet);
348 items.push_back(std::move(work));
349 }
350 c2_status_t err = C2_OK;
351 if (!items.empty()) {
352 ScopedTrace trace(ATRACE_TAG, android::base::StringPrintf(
353 "CCodecBufferChannel::queue(%s@ts=%lld)", mName, (long long)timeUs).c_str());
354 {
355 Mutexed<PipelineWatcher>::Locked watcher(mPipelineWatcher);
356 PipelineWatcher::Clock::time_point now = PipelineWatcher::Clock::now();
357 for (const std::unique_ptr<C2Work> &work : items) {
358 watcher->onWorkQueued(
359 work->input.ordinal.frameIndex.peeku(),
360 std::vector(work->input.buffers),
361 now);
362 }
363 }
364 err = mComponent->queue(&items);
365 }
366 if (err != C2_OK) {
367 Mutexed<PipelineWatcher>::Locked watcher(mPipelineWatcher);
368 for (const std::unique_ptr<C2Work> &work : items) {
369 watcher->onWorkDone(work->input.ordinal.frameIndex.peeku());
370 }
371 } else {
372 Mutexed<Input>::Locked input(mInput);
373 bool released = false;
374 if (copy) {
375 released = input->extraBuffers.releaseSlot(copy, nullptr, true);
376 } else if (buffer) {
377 released = input->buffers->releaseBuffer(buffer, nullptr, true);
378 }
379 ALOGV("[%s] queueInputBuffer: buffer%s %sreleased",
380 mName, (buffer == nullptr) ? "(copy)" : "", released ? "" : "not ");
381 }
382
383 feedInputBufferIfAvailableInternal();
384 return err;
385 }
386
setParameters(std::vector<std::unique_ptr<C2Param>> & params)387 status_t CCodecBufferChannel::setParameters(std::vector<std::unique_ptr<C2Param>> ¶ms) {
388 QueueGuard guard(mSync);
389 if (!guard.isRunning()) {
390 ALOGD("[%s] setParameters is only supported in the running state.", mName);
391 return -ENOSYS;
392 }
393 mParamsToBeSet.insert(mParamsToBeSet.end(),
394 std::make_move_iterator(params.begin()),
395 std::make_move_iterator(params.end()));
396 params.clear();
397 return OK;
398 }
399
attachBuffer(const std::shared_ptr<C2Buffer> & c2Buffer,const sp<MediaCodecBuffer> & buffer)400 status_t CCodecBufferChannel::attachBuffer(
401 const std::shared_ptr<C2Buffer> &c2Buffer,
402 const sp<MediaCodecBuffer> &buffer) {
403 if (!buffer->copy(c2Buffer)) {
404 return -ENOSYS;
405 }
406 return OK;
407 }
408
ensureDecryptDestination(size_t size)409 void CCodecBufferChannel::ensureDecryptDestination(size_t size) {
410 if (!mDecryptDestination || mDecryptDestination->size() < size) {
411 sp<IMemoryHeap> heap{new MemoryHeapBase(size * 2)};
412 if (mDecryptDestination && mCrypto && mHeapSeqNum >= 0) {
413 mCrypto->unsetHeap(mHeapSeqNum);
414 }
415 mDecryptDestination = new MemoryBase(heap, 0, size * 2);
416 if (mCrypto) {
417 mHeapSeqNum = mCrypto->setHeap(hardware::fromHeap(heap));
418 }
419 }
420 }
421
getHeapSeqNum(const sp<HidlMemory> & memory)422 int32_t CCodecBufferChannel::getHeapSeqNum(const sp<HidlMemory> &memory) {
423 CHECK(mCrypto);
424 auto it = mHeapSeqNumMap.find(memory);
425 int32_t heapSeqNum = -1;
426 if (it == mHeapSeqNumMap.end()) {
427 heapSeqNum = mCrypto->setHeap(memory);
428 mHeapSeqNumMap.emplace(memory, heapSeqNum);
429 } else {
430 heapSeqNum = it->second;
431 }
432 return heapSeqNum;
433 }
434
attachEncryptedBuffer(const sp<hardware::HidlMemory> & memory,bool secure,const uint8_t * key,const uint8_t * iv,CryptoPlugin::Mode mode,CryptoPlugin::Pattern pattern,size_t offset,const CryptoPlugin::SubSample * subSamples,size_t numSubSamples,const sp<MediaCodecBuffer> & buffer,AString * errorDetailMsg)435 status_t CCodecBufferChannel::attachEncryptedBuffer(
436 const sp<hardware::HidlMemory> &memory,
437 bool secure,
438 const uint8_t *key,
439 const uint8_t *iv,
440 CryptoPlugin::Mode mode,
441 CryptoPlugin::Pattern pattern,
442 size_t offset,
443 const CryptoPlugin::SubSample *subSamples,
444 size_t numSubSamples,
445 const sp<MediaCodecBuffer> &buffer,
446 AString* errorDetailMsg) {
447 static const C2MemoryUsage kSecureUsage{C2MemoryUsage::READ_PROTECTED, 0};
448 static const C2MemoryUsage kDefaultReadWriteUsage{
449 C2MemoryUsage::CPU_READ, C2MemoryUsage::CPU_WRITE};
450
451 size_t size = 0;
452 for (size_t i = 0; i < numSubSamples; ++i) {
453 size += subSamples[i].mNumBytesOfClearData + subSamples[i].mNumBytesOfEncryptedData;
454 }
455 if (size == 0) {
456 buffer->setRange(0, 0);
457 return OK;
458 }
459 std::shared_ptr<C2BlockPool> pool = mBlockPools.lock()->inputPool;
460 std::shared_ptr<C2LinearBlock> block;
461 c2_status_t err = pool->fetchLinearBlock(
462 size,
463 secure ? kSecureUsage : kDefaultReadWriteUsage,
464 &block);
465 if (err != C2_OK) {
466 ALOGI("[%s] attachEncryptedBuffer: fetchLinearBlock failed: size = %zu (%s) err = %d",
467 mName, size, secure ? "secure" : "non-secure", err);
468 return NO_MEMORY;
469 }
470 if (!secure) {
471 ensureDecryptDestination(size);
472 }
473 ssize_t result = -1;
474 ssize_t codecDataOffset = 0;
475 if (mCrypto) {
476 int32_t heapSeqNum = getHeapSeqNum(memory);
477 hardware::drm::V1_0::SharedBuffer src{(uint32_t)heapSeqNum, offset, size};
478 hardware::drm::V1_0::DestinationBuffer dst;
479 if (secure) {
480 dst.type = DrmBufferType::NATIVE_HANDLE;
481 dst.secureMemory = hardware::hidl_handle(block->handle());
482 } else {
483 dst.type = DrmBufferType::SHARED_MEMORY;
484 IMemoryToSharedBuffer(
485 mDecryptDestination, mHeapSeqNum, &dst.nonsecureMemory);
486 }
487 result = mCrypto->decrypt(
488 key, iv, mode, pattern, src, 0, subSamples, numSubSamples,
489 dst, errorDetailMsg);
490 if (result < 0) {
491 ALOGI("[%s] attachEncryptedBuffer: decrypt failed: result = %zd", mName, result);
492 return result;
493 }
494 } else {
495 // Here we cast CryptoPlugin::SubSample to hardware::cas::native::V1_0::SubSample
496 // directly, the structure definitions should match as checked in DescramblerImpl.cpp.
497 hidl_vec<SubSample> hidlSubSamples;
498 hidlSubSamples.setToExternal((SubSample *)subSamples, numSubSamples, false /*own*/);
499
500 hardware::cas::native::V1_0::SharedBuffer src{*memory, offset, size};
501 hardware::cas::native::V1_0::DestinationBuffer dst;
502 if (secure) {
503 dst.type = BufferType::NATIVE_HANDLE;
504 dst.secureMemory = hardware::hidl_handle(block->handle());
505 } else {
506 dst.type = BufferType::SHARED_MEMORY;
507 dst.nonsecureMemory = src;
508 }
509
510 CasStatus status = CasStatus::OK;
511 hidl_string detailedError;
512 ScramblingControl sctrl = ScramblingControl::UNSCRAMBLED;
513
514 if (key != nullptr) {
515 sctrl = (ScramblingControl)key[0];
516 // Adjust for the PES offset
517 codecDataOffset = key[2] | (key[3] << 8);
518 }
519
520 auto returnVoid = mDescrambler->descramble(
521 sctrl,
522 hidlSubSamples,
523 src,
524 0,
525 dst,
526 0,
527 [&status, &result, &detailedError] (
528 CasStatus _status, uint32_t _bytesWritten,
529 const hidl_string& _detailedError) {
530 status = _status;
531 result = (ssize_t)_bytesWritten;
532 detailedError = _detailedError;
533 });
534 if (errorDetailMsg) {
535 errorDetailMsg->setTo(detailedError.c_str(), detailedError.size());
536 }
537 if (!returnVoid.isOk() || status != CasStatus::OK || result < 0) {
538 ALOGI("[%s] descramble failed, trans=%s, status=%d, result=%zd",
539 mName, returnVoid.description().c_str(), status, result);
540 return UNKNOWN_ERROR;
541 }
542
543 if (result < codecDataOffset) {
544 ALOGD("[%s] invalid codec data offset: %zd, result %zd",
545 mName, codecDataOffset, result);
546 return BAD_VALUE;
547 }
548 }
549 if (!secure) {
550 C2WriteView view = block->map().get();
551 if (view.error() != C2_OK) {
552 ALOGI("[%s] attachEncryptedBuffer: block map error: %d (non-secure)",
553 mName, view.error());
554 return UNKNOWN_ERROR;
555 }
556 if (view.size() < result) {
557 ALOGI("[%s] attachEncryptedBuffer: block size too small: size=%u result=%zd "
558 "(non-secure)",
559 mName, view.size(), result);
560 return UNKNOWN_ERROR;
561 }
562 memcpy(view.data(), mDecryptDestination->unsecurePointer(), result);
563 }
564 std::shared_ptr<C2Buffer> c2Buffer{C2Buffer::CreateLinearBuffer(
565 block->share(codecDataOffset, result - codecDataOffset, C2Fence{}))};
566 if (!buffer->copy(c2Buffer)) {
567 ALOGI("[%s] attachEncryptedBuffer: buffer copy failed", mName);
568 return -ENOSYS;
569 }
570 return OK;
571 }
572
queueInputBuffer(const sp<MediaCodecBuffer> & buffer)573 status_t CCodecBufferChannel::queueInputBuffer(const sp<MediaCodecBuffer> &buffer) {
574 QueueGuard guard(mSync);
575 if (!guard.isRunning()) {
576 ALOGD("[%s] No more buffers should be queued at current state.", mName);
577 return -ENOSYS;
578 }
579 return queueInputBufferInternal(buffer);
580 }
581
queueSecureInputBuffer(const sp<MediaCodecBuffer> & buffer,bool secure,const uint8_t * key,const uint8_t * iv,CryptoPlugin::Mode mode,CryptoPlugin::Pattern pattern,const CryptoPlugin::SubSample * subSamples,size_t numSubSamples,AString * errorDetailMsg)582 status_t CCodecBufferChannel::queueSecureInputBuffer(
583 const sp<MediaCodecBuffer> &buffer, bool secure, const uint8_t *key,
584 const uint8_t *iv, CryptoPlugin::Mode mode, CryptoPlugin::Pattern pattern,
585 const CryptoPlugin::SubSample *subSamples, size_t numSubSamples,
586 AString *errorDetailMsg) {
587 QueueGuard guard(mSync);
588 if (!guard.isRunning()) {
589 ALOGD("[%s] No more buffers should be queued at current state.", mName);
590 return -ENOSYS;
591 }
592
593 if (!hasCryptoOrDescrambler()) {
594 return -ENOSYS;
595 }
596 sp<EncryptedLinearBlockBuffer> encryptedBuffer((EncryptedLinearBlockBuffer *)buffer.get());
597
598 std::shared_ptr<C2LinearBlock> block;
599 size_t allocSize = buffer->size();
600 size_t bufferSize = 0;
601 c2_status_t blockRes = C2_OK;
602 bool copied = false;
603 if (mSendEncryptedInfoBuffer) {
604 static const C2MemoryUsage kDefaultReadWriteUsage{
605 C2MemoryUsage::CPU_READ, C2MemoryUsage::CPU_WRITE};
606 constexpr int kAllocGranule0 = 1024 * 64;
607 constexpr int kAllocGranule1 = 1024 * 1024;
608 std::shared_ptr<C2BlockPool> pool = mBlockPools.lock()->inputPool;
609 // round up encrypted sizes to limit fragmentation and encourage buffer reuse
610 if (allocSize <= kAllocGranule1) {
611 bufferSize = align(allocSize, kAllocGranule0);
612 } else {
613 bufferSize = align(allocSize, kAllocGranule1);
614 }
615 blockRes = pool->fetchLinearBlock(
616 bufferSize, kDefaultReadWriteUsage, &block);
617
618 if (blockRes == C2_OK) {
619 C2WriteView view = block->map().get();
620 if (view.error() == C2_OK && view.size() == bufferSize) {
621 copied = true;
622 // TODO: only copy clear sections
623 memcpy(view.data(), buffer->data(), allocSize);
624 }
625 }
626 }
627
628 if (!copied) {
629 block.reset();
630 }
631
632 ssize_t result = -1;
633 ssize_t codecDataOffset = 0;
634 if (numSubSamples == 1
635 && subSamples[0].mNumBytesOfClearData == 0
636 && subSamples[0].mNumBytesOfEncryptedData == 0) {
637 // We don't need to go through crypto or descrambler if the input is empty.
638 result = 0;
639 } else if (mCrypto != nullptr) {
640 hardware::drm::V1_0::DestinationBuffer destination;
641 if (secure) {
642 destination.type = DrmBufferType::NATIVE_HANDLE;
643 destination.secureMemory = hidl_handle(encryptedBuffer->handle());
644 } else {
645 destination.type = DrmBufferType::SHARED_MEMORY;
646 IMemoryToSharedBuffer(
647 mDecryptDestination, mHeapSeqNum, &destination.nonsecureMemory);
648 }
649 hardware::drm::V1_0::SharedBuffer source;
650 encryptedBuffer->fillSourceBuffer(&source);
651 result = mCrypto->decrypt(
652 key, iv, mode, pattern, source, buffer->offset(),
653 subSamples, numSubSamples, destination, errorDetailMsg);
654 if (result < 0) {
655 ALOGI("[%s] decrypt failed: result=%zd", mName, result);
656 return result;
657 }
658 if (destination.type == DrmBufferType::SHARED_MEMORY) {
659 encryptedBuffer->copyDecryptedContent(mDecryptDestination, result);
660 }
661 } else {
662 // Here we cast CryptoPlugin::SubSample to hardware::cas::native::V1_0::SubSample
663 // directly, the structure definitions should match as checked in DescramblerImpl.cpp.
664 hidl_vec<SubSample> hidlSubSamples;
665 hidlSubSamples.setToExternal((SubSample *)subSamples, numSubSamples, false /*own*/);
666
667 hardware::cas::native::V1_0::SharedBuffer srcBuffer;
668 encryptedBuffer->fillSourceBuffer(&srcBuffer);
669
670 DestinationBuffer dstBuffer;
671 if (secure) {
672 dstBuffer.type = BufferType::NATIVE_HANDLE;
673 dstBuffer.secureMemory = hidl_handle(encryptedBuffer->handle());
674 } else {
675 dstBuffer.type = BufferType::SHARED_MEMORY;
676 dstBuffer.nonsecureMemory = srcBuffer;
677 }
678
679 CasStatus status = CasStatus::OK;
680 hidl_string detailedError;
681 ScramblingControl sctrl = ScramblingControl::UNSCRAMBLED;
682
683 if (key != nullptr) {
684 sctrl = (ScramblingControl)key[0];
685 // Adjust for the PES offset
686 codecDataOffset = key[2] | (key[3] << 8);
687 }
688
689 auto returnVoid = mDescrambler->descramble(
690 sctrl,
691 hidlSubSamples,
692 srcBuffer,
693 0,
694 dstBuffer,
695 0,
696 [&status, &result, &detailedError] (
697 CasStatus _status, uint32_t _bytesWritten,
698 const hidl_string& _detailedError) {
699 status = _status;
700 result = (ssize_t)_bytesWritten;
701 detailedError = _detailedError;
702 });
703
704 if (!returnVoid.isOk() || status != CasStatus::OK || result < 0) {
705 ALOGI("[%s] descramble failed, trans=%s, status=%d, result=%zd",
706 mName, returnVoid.description().c_str(), status, result);
707 return UNKNOWN_ERROR;
708 }
709
710 if (result < codecDataOffset) {
711 ALOGD("invalid codec data offset: %zd, result %zd", codecDataOffset, result);
712 return BAD_VALUE;
713 }
714
715 ALOGV("[%s] descramble succeeded, %zd bytes", mName, result);
716
717 if (dstBuffer.type == BufferType::SHARED_MEMORY) {
718 encryptedBuffer->copyDecryptedContentFromMemory(result);
719 }
720 }
721
722 buffer->setRange(codecDataOffset, result - codecDataOffset);
723
724 return queueInputBufferInternal(buffer, block, bufferSize);
725 }
726
feedInputBufferIfAvailable()727 void CCodecBufferChannel::feedInputBufferIfAvailable() {
728 QueueGuard guard(mSync);
729 if (!guard.isRunning()) {
730 ALOGV("[%s] We're not running --- no input buffer reported", mName);
731 return;
732 }
733 feedInputBufferIfAvailableInternal();
734 }
735
feedInputBufferIfAvailableInternal()736 void CCodecBufferChannel::feedInputBufferIfAvailableInternal() {
737 if (mInputMetEos) {
738 return;
739 }
740 {
741 Mutexed<Output>::Locked output(mOutput);
742 if (!output->buffers ||
743 output->buffers->hasPending() ||
744 (!output->bounded && output->buffers->numActiveSlots() >= output->numSlots)) {
745 return;
746 }
747 }
748 size_t numActiveSlots = 0;
749 while (!mPipelineWatcher.lock()->pipelineFull()) {
750 sp<MediaCodecBuffer> inBuffer;
751 size_t index;
752 {
753 Mutexed<Input>::Locked input(mInput);
754 numActiveSlots = input->buffers->numActiveSlots();
755 if (numActiveSlots >= input->numSlots) {
756 break;
757 }
758 if (!input->buffers->requestNewBuffer(&index, &inBuffer)) {
759 ALOGV("[%s] no new buffer available", mName);
760 break;
761 }
762 }
763 ALOGV("[%s] new input index = %zu [%p]", mName, index, inBuffer.get());
764 mCallback->onInputBufferAvailable(index, inBuffer);
765 }
766 ALOGV("[%s] # active slots after feedInputBufferIfAvailable = %zu", mName, numActiveSlots);
767 }
768
renderOutputBuffer(const sp<MediaCodecBuffer> & buffer,int64_t timestampNs)769 status_t CCodecBufferChannel::renderOutputBuffer(
770 const sp<MediaCodecBuffer> &buffer, int64_t timestampNs) {
771 ALOGV("[%s] renderOutputBuffer: %p", mName, buffer.get());
772 std::shared_ptr<C2Buffer> c2Buffer;
773 bool released = false;
774 {
775 Mutexed<Output>::Locked output(mOutput);
776 if (output->buffers) {
777 released = output->buffers->releaseBuffer(buffer, &c2Buffer);
778 }
779 }
780 // NOTE: some apps try to releaseOutputBuffer() with timestamp and/or render
781 // set to true.
782 sendOutputBuffers();
783 // input buffer feeding may have been gated by pending output buffers
784 feedInputBufferIfAvailable();
785 if (!c2Buffer) {
786 if (released) {
787 std::call_once(mRenderWarningFlag, [this] {
788 ALOGW("[%s] The app is calling releaseOutputBuffer() with "
789 "timestamp or render=true with non-video buffers. Apps should "
790 "call releaseOutputBuffer() with render=false for those.",
791 mName);
792 });
793 }
794 return INVALID_OPERATION;
795 }
796
797 #if 0
798 const std::vector<std::shared_ptr<const C2Info>> infoParams = c2Buffer->info();
799 ALOGV("[%s] queuing gfx buffer with %zu infos", mName, infoParams.size());
800 for (const std::shared_ptr<const C2Info> &info : infoParams) {
801 AString res;
802 for (size_t ix = 0; ix + 3 < info->size(); ix += 4) {
803 if (ix) res.append(", ");
804 res.append(*((int32_t*)info.get() + (ix / 4)));
805 }
806 ALOGV(" [%s]", res.c_str());
807 }
808 #endif
809 std::shared_ptr<const C2StreamRotationInfo::output> rotation =
810 std::static_pointer_cast<const C2StreamRotationInfo::output>(
811 c2Buffer->getInfo(C2StreamRotationInfo::output::PARAM_TYPE));
812 bool flip = rotation && (rotation->flip & 1);
813 uint32_t quarters = ((rotation ? rotation->value : 0) / 90) & 3;
814
815 {
816 Mutexed<OutputSurface>::Locked output(mOutputSurface);
817 if (output->surface == nullptr) {
818 ALOGI("[%s] cannot render buffer without surface", mName);
819 return OK;
820 }
821 int64_t frameIndex;
822 buffer->meta()->findInt64("frameIndex", &frameIndex);
823 if (output->rotation.count(frameIndex) != 0) {
824 auto it = output->rotation.find(frameIndex);
825 quarters = (it->second / 90) & 3;
826 output->rotation.erase(it);
827 }
828 }
829
830 uint32_t transform = 0;
831 switch (quarters) {
832 case 0: // no rotation
833 transform = flip ? HAL_TRANSFORM_FLIP_H : 0;
834 break;
835 case 1: // 90 degrees counter-clockwise
836 transform = flip ? (HAL_TRANSFORM_FLIP_V | HAL_TRANSFORM_ROT_90)
837 : HAL_TRANSFORM_ROT_270;
838 break;
839 case 2: // 180 degrees
840 transform = flip ? HAL_TRANSFORM_FLIP_V : HAL_TRANSFORM_ROT_180;
841 break;
842 case 3: // 90 degrees clockwise
843 transform = flip ? (HAL_TRANSFORM_FLIP_H | HAL_TRANSFORM_ROT_90)
844 : HAL_TRANSFORM_ROT_90;
845 break;
846 }
847
848 std::shared_ptr<const C2StreamSurfaceScalingInfo::output> surfaceScaling =
849 std::static_pointer_cast<const C2StreamSurfaceScalingInfo::output>(
850 c2Buffer->getInfo(C2StreamSurfaceScalingInfo::output::PARAM_TYPE));
851 uint32_t videoScalingMode = NATIVE_WINDOW_SCALING_MODE_SCALE_TO_WINDOW;
852 if (surfaceScaling) {
853 videoScalingMode = surfaceScaling->value;
854 }
855
856 // Use dataspace from format as it has the default aspects already applied
857 android_dataspace_t dataSpace = HAL_DATASPACE_UNKNOWN; // this is 0
858 (void)buffer->format()->findInt32("android._dataspace", (int32_t *)&dataSpace);
859
860 // HDR static info
861 std::shared_ptr<const C2StreamHdrStaticInfo::output> hdrStaticInfo =
862 std::static_pointer_cast<const C2StreamHdrStaticInfo::output>(
863 c2Buffer->getInfo(C2StreamHdrStaticInfo::output::PARAM_TYPE));
864
865 // HDR10 plus info
866 std::shared_ptr<const C2StreamHdr10PlusInfo::output> hdr10PlusInfo =
867 std::static_pointer_cast<const C2StreamHdr10PlusInfo::output>(
868 c2Buffer->getInfo(C2StreamHdr10PlusInfo::output::PARAM_TYPE));
869 if (hdr10PlusInfo && hdr10PlusInfo->flexCount() == 0) {
870 hdr10PlusInfo.reset();
871 }
872
873 // HDR dynamic info
874 std::shared_ptr<const C2StreamHdrDynamicMetadataInfo::output> hdrDynamicInfo =
875 std::static_pointer_cast<const C2StreamHdrDynamicMetadataInfo::output>(
876 c2Buffer->getInfo(C2StreamHdrDynamicMetadataInfo::output::PARAM_TYPE));
877 // TODO: make this sticky & enable unset
878 if (hdrDynamicInfo && hdrDynamicInfo->flexCount() == 0) {
879 hdrDynamicInfo.reset();
880 }
881
882 if (hdr10PlusInfo) {
883 // C2StreamHdr10PlusInfo is deprecated; components should use
884 // C2StreamHdrDynamicMetadataInfo
885 // TODO: #metric
886 if (hdrDynamicInfo) {
887 // It is unexpected that C2StreamHdr10PlusInfo and
888 // C2StreamHdrDynamicMetadataInfo is both present.
889 // C2StreamHdrDynamicMetadataInfo takes priority.
890 // TODO: #metric
891 } else {
892 std::shared_ptr<C2StreamHdrDynamicMetadataInfo::output> info =
893 C2StreamHdrDynamicMetadataInfo::output::AllocShared(
894 hdr10PlusInfo->flexCount(),
895 0u,
896 C2Config::HDR_DYNAMIC_METADATA_TYPE_SMPTE_2094_40);
897 memcpy(info->m.data, hdr10PlusInfo->m.value, hdr10PlusInfo->flexCount());
898 hdrDynamicInfo = info;
899 }
900 }
901
902 std::vector<C2ConstGraphicBlock> blocks = c2Buffer->data().graphicBlocks();
903 if (blocks.size() != 1u) {
904 ALOGD("[%s] expected 1 graphic block, but got %zu", mName, blocks.size());
905 return UNKNOWN_ERROR;
906 }
907 const C2ConstGraphicBlock &block = blocks.front();
908 C2Fence c2fence = block.fence();
909 sp<Fence> fence = Fence::NO_FENCE;
910 // TODO: it's not sufficient to just check isHW() and then construct android::fence from it.
911 // Once C2Fence::type() is added, check the exact C2Fence type
912 if (c2fence.isHW()) {
913 int fenceFd = c2fence.fd();
914 fence = sp<Fence>::make(fenceFd);
915 if (!fence) {
916 ALOGE("[%s] Failed to allocate a fence", mName);
917 close(fenceFd);
918 return NO_MEMORY;
919 }
920 }
921
922 // TODO: revisit this after C2Fence implementation.
923 IGraphicBufferProducer::QueueBufferInput qbi(
924 timestampNs,
925 false, // droppable
926 dataSpace,
927 Rect(blocks.front().crop().left,
928 blocks.front().crop().top,
929 blocks.front().crop().right(),
930 blocks.front().crop().bottom()),
931 videoScalingMode,
932 transform,
933 fence, 0);
934 if (hdrStaticInfo || hdrDynamicInfo) {
935 HdrMetadata hdr;
936 if (hdrStaticInfo) {
937 // If mastering max and min luminance fields are 0, do not use them.
938 // It indicates the value may not be present in the stream.
939 if (hdrStaticInfo->mastering.maxLuminance > 0.0f &&
940 hdrStaticInfo->mastering.minLuminance > 0.0f) {
941 struct android_smpte2086_metadata smpte2086_meta = {
942 .displayPrimaryRed = {
943 hdrStaticInfo->mastering.red.x, hdrStaticInfo->mastering.red.y
944 },
945 .displayPrimaryGreen = {
946 hdrStaticInfo->mastering.green.x, hdrStaticInfo->mastering.green.y
947 },
948 .displayPrimaryBlue = {
949 hdrStaticInfo->mastering.blue.x, hdrStaticInfo->mastering.blue.y
950 },
951 .whitePoint = {
952 hdrStaticInfo->mastering.white.x, hdrStaticInfo->mastering.white.y
953 },
954 .maxLuminance = hdrStaticInfo->mastering.maxLuminance,
955 .minLuminance = hdrStaticInfo->mastering.minLuminance,
956 };
957 hdr.validTypes |= HdrMetadata::SMPTE2086;
958 hdr.smpte2086 = smpte2086_meta;
959 }
960 // If the content light level fields are 0, do not use them, it
961 // indicates the value may not be present in the stream.
962 if (hdrStaticInfo->maxCll > 0.0f && hdrStaticInfo->maxFall > 0.0f) {
963 struct android_cta861_3_metadata cta861_meta = {
964 .maxContentLightLevel = hdrStaticInfo->maxCll,
965 .maxFrameAverageLightLevel = hdrStaticInfo->maxFall,
966 };
967 hdr.validTypes |= HdrMetadata::CTA861_3;
968 hdr.cta8613 = cta861_meta;
969 }
970
971 // does not have valid info
972 if (!(hdr.validTypes & (HdrMetadata::SMPTE2086 | HdrMetadata::CTA861_3))) {
973 hdrStaticInfo.reset();
974 }
975 }
976 if (hdrDynamicInfo
977 && hdrDynamicInfo->m.type_ == C2Config::HDR_DYNAMIC_METADATA_TYPE_SMPTE_2094_40) {
978 hdr.validTypes |= HdrMetadata::HDR10PLUS;
979 hdr.hdr10plus.assign(
980 hdrDynamicInfo->m.data,
981 hdrDynamicInfo->m.data + hdrDynamicInfo->flexCount());
982 }
983 qbi.setHdrMetadata(hdr);
984 }
985 SetMetadataToGralloc4Handle(dataSpace, hdrStaticInfo, hdrDynamicInfo, block.handle());
986
987 qbi.setSurfaceDamage(Region::INVALID_REGION); // we don't have dirty regions
988 qbi.getFrameTimestamps = true; // we need to know when a frame is rendered
989 IGraphicBufferProducer::QueueBufferOutput qbo;
990 status_t result = mComponent->queueToOutputSurface(block, qbi, &qbo);
991 if (result != OK) {
992 ALOGI("[%s] queueBuffer failed: %d", mName, result);
993 if (result == NO_INIT) {
994 mCCodecCallback->onError(UNKNOWN_ERROR, ACTION_CODE_FATAL);
995 }
996 return result;
997 }
998
999 if(android::base::GetBoolProperty("debug.stagefright.fps", false)) {
1000 ALOGD("[%s] queue buffer successful", mName);
1001 } else {
1002 ALOGV("[%s] queue buffer successful", mName);
1003 }
1004
1005 int64_t mediaTimeUs = 0;
1006 (void)buffer->meta()->findInt64("timeUs", &mediaTimeUs);
1007 if (mAreRenderMetricsEnabled && mIsSurfaceToDisplay) {
1008 trackReleasedFrame(qbo, mediaTimeUs, timestampNs);
1009 processRenderedFrames(qbo.frameTimestamps);
1010 } else {
1011 // When the surface is an intermediate surface, onFrameRendered is triggered immediately
1012 // when the frame is queued to the non-display surface
1013 mCCodecCallback->onOutputFramesRendered(mediaTimeUs, timestampNs);
1014 }
1015
1016 return OK;
1017 }
1018
initializeFrameTrackingFor(ANativeWindow * window)1019 void CCodecBufferChannel::initializeFrameTrackingFor(ANativeWindow * window) {
1020 mTrackedFrames.clear();
1021
1022 int isSurfaceToDisplay = 0;
1023 window->query(window, NATIVE_WINDOW_QUEUES_TO_WINDOW_COMPOSER, &isSurfaceToDisplay);
1024 mIsSurfaceToDisplay = isSurfaceToDisplay == 1;
1025 // No frame tracking is needed if we're not sending frames to the display
1026 if (!mIsSurfaceToDisplay) {
1027 // Return early so we don't call into SurfaceFlinger (requiring permissions)
1028 return;
1029 }
1030
1031 int hasPresentFenceTimes = 0;
1032 window->query(window, NATIVE_WINDOW_FRAME_TIMESTAMPS_SUPPORTS_PRESENT, &hasPresentFenceTimes);
1033 mHasPresentFenceTimes = hasPresentFenceTimes == 1;
1034 if (!mHasPresentFenceTimes) {
1035 ALOGI("Using latch times for frame rendered signals - present fences not supported");
1036 }
1037 }
1038
trackReleasedFrame(const IGraphicBufferProducer::QueueBufferOutput & qbo,int64_t mediaTimeUs,int64_t desiredRenderTimeNs)1039 void CCodecBufferChannel::trackReleasedFrame(const IGraphicBufferProducer::QueueBufferOutput& qbo,
1040 int64_t mediaTimeUs, int64_t desiredRenderTimeNs) {
1041 // If the render time is earlier than now, then we're suggesting it should be rendered ASAP,
1042 // so track the frame as if the desired render time is now.
1043 int64_t nowNs = systemTime(SYSTEM_TIME_MONOTONIC);
1044 if (desiredRenderTimeNs < nowNs) {
1045 desiredRenderTimeNs = nowNs;
1046 }
1047
1048 // If the render time is more than a second from now, then pretend the frame is supposed to be
1049 // rendered immediately, because that's what SurfaceFlinger heuristics will do. This is a tight
1050 // coupling, but is really the only way to optimize away unnecessary present fence checks in
1051 // processRenderedFrames.
1052 if (desiredRenderTimeNs > nowNs + 1*1000*1000*1000LL) {
1053 desiredRenderTimeNs = nowNs;
1054 }
1055
1056 // We've just queued a frame to the surface, so keep track of it and later check to see if it is
1057 // actually rendered.
1058 TrackedFrame frame;
1059 frame.number = qbo.nextFrameNumber - 1;
1060 frame.mediaTimeUs = mediaTimeUs;
1061 frame.desiredRenderTimeNs = desiredRenderTimeNs;
1062 frame.latchTime = -1;
1063 frame.presentFence = nullptr;
1064 mTrackedFrames.push_back(frame);
1065 }
1066
processRenderedFrames(const FrameEventHistoryDelta & deltas)1067 void CCodecBufferChannel::processRenderedFrames(const FrameEventHistoryDelta& deltas) {
1068 // Grab the latch times and present fences from the frame event deltas
1069 for (const auto& delta : deltas) {
1070 for (auto& frame : mTrackedFrames) {
1071 if (delta.getFrameNumber() == frame.number) {
1072 delta.getLatchTime(&frame.latchTime);
1073 delta.getDisplayPresentFence(&frame.presentFence);
1074 }
1075 }
1076 }
1077
1078 // Scan all frames and check to see if the frames that SHOULD have been rendered by now, have,
1079 // in fact, been rendered.
1080 int64_t nowNs = systemTime(SYSTEM_TIME_MONOTONIC);
1081 while (!mTrackedFrames.empty()) {
1082 TrackedFrame & frame = mTrackedFrames.front();
1083 // Frames that should have been rendered at least 100ms in the past are checked
1084 if (frame.desiredRenderTimeNs > nowNs - 100*1000*1000LL) {
1085 break;
1086 }
1087
1088 // If we don't have a render time by now, then consider the frame as dropped
1089 int64_t renderTimeNs = getRenderTimeNs(frame);
1090 if (renderTimeNs != -1) {
1091 mCCodecCallback->onOutputFramesRendered(frame.mediaTimeUs, renderTimeNs);
1092 }
1093 mTrackedFrames.pop_front();
1094 }
1095 }
1096
getRenderTimeNs(const TrackedFrame & frame)1097 int64_t CCodecBufferChannel::getRenderTimeNs(const TrackedFrame& frame) {
1098 // If the device doesn't have accurate present fence times, then use the latch time as a proxy
1099 if (!mHasPresentFenceTimes) {
1100 if (frame.latchTime == -1) {
1101 ALOGD("no latch time for frame %d", (int) frame.number);
1102 return -1;
1103 }
1104 return frame.latchTime;
1105 }
1106
1107 if (frame.presentFence == nullptr) {
1108 ALOGW("no present fence for frame %d", (int) frame.number);
1109 return -1;
1110 }
1111
1112 nsecs_t actualRenderTimeNs = frame.presentFence->getSignalTime();
1113
1114 if (actualRenderTimeNs == Fence::SIGNAL_TIME_INVALID) {
1115 ALOGW("invalid signal time for frame %d", (int) frame.number);
1116 return -1;
1117 }
1118
1119 if (actualRenderTimeNs == Fence::SIGNAL_TIME_PENDING) {
1120 ALOGD("present fence has not fired for frame %d", (int) frame.number);
1121 return -1;
1122 }
1123
1124 return actualRenderTimeNs;
1125 }
1126
pollForRenderedBuffers()1127 void CCodecBufferChannel::pollForRenderedBuffers() {
1128 FrameEventHistoryDelta delta;
1129 mComponent->pollForRenderedFrames(&delta);
1130 processRenderedFrames(delta);
1131 }
1132
discardBuffer(const sp<MediaCodecBuffer> & buffer)1133 status_t CCodecBufferChannel::discardBuffer(const sp<MediaCodecBuffer> &buffer) {
1134 ALOGV("[%s] discardBuffer: %p", mName, buffer.get());
1135 bool released = false;
1136 {
1137 Mutexed<Input>::Locked input(mInput);
1138 if (input->buffers && input->buffers->releaseBuffer(buffer, nullptr, true)) {
1139 released = true;
1140 }
1141 }
1142 {
1143 Mutexed<Output>::Locked output(mOutput);
1144 if (output->buffers && output->buffers->releaseBuffer(buffer, nullptr)) {
1145 released = true;
1146 }
1147 }
1148 if (released) {
1149 sendOutputBuffers();
1150 feedInputBufferIfAvailable();
1151 } else {
1152 ALOGD("[%s] MediaCodec discarded an unknown buffer", mName);
1153 }
1154 return OK;
1155 }
1156
getInputBufferArray(Vector<sp<MediaCodecBuffer>> * array)1157 void CCodecBufferChannel::getInputBufferArray(Vector<sp<MediaCodecBuffer>> *array) {
1158 array->clear();
1159 Mutexed<Input>::Locked input(mInput);
1160
1161 if (!input->buffers) {
1162 ALOGE("getInputBufferArray: No Input Buffers allocated");
1163 return;
1164 }
1165 if (!input->buffers->isArrayMode()) {
1166 input->buffers = input->buffers->toArrayMode(input->numSlots);
1167 }
1168
1169 input->buffers->getArray(array);
1170 }
1171
getOutputBufferArray(Vector<sp<MediaCodecBuffer>> * array)1172 void CCodecBufferChannel::getOutputBufferArray(Vector<sp<MediaCodecBuffer>> *array) {
1173 array->clear();
1174 Mutexed<Output>::Locked output(mOutput);
1175 if (!output->buffers) {
1176 ALOGE("getOutputBufferArray: No Output Buffers allocated");
1177 return;
1178 }
1179 if (!output->buffers->isArrayMode()) {
1180 output->buffers = output->buffers->toArrayMode(output->numSlots);
1181 }
1182
1183 output->buffers->getArray(array);
1184 }
1185
start(const sp<AMessage> & inputFormat,const sp<AMessage> & outputFormat,bool buffersBoundToCodec)1186 status_t CCodecBufferChannel::start(
1187 const sp<AMessage> &inputFormat,
1188 const sp<AMessage> &outputFormat,
1189 bool buffersBoundToCodec) {
1190 C2StreamBufferTypeSetting::input iStreamFormat(0u);
1191 C2StreamBufferTypeSetting::output oStreamFormat(0u);
1192 C2ComponentKindSetting kind;
1193 C2PortReorderBufferDepthTuning::output reorderDepth;
1194 C2PortReorderKeySetting::output reorderKey;
1195 C2PortActualDelayTuning::input inputDelay(0);
1196 C2PortActualDelayTuning::output outputDelay(0);
1197 C2ActualPipelineDelayTuning pipelineDelay(0);
1198 C2SecureModeTuning secureMode(C2Config::SM_UNPROTECTED);
1199
1200 c2_status_t err = mComponent->query(
1201 {
1202 &iStreamFormat,
1203 &oStreamFormat,
1204 &kind,
1205 &reorderDepth,
1206 &reorderKey,
1207 &inputDelay,
1208 &pipelineDelay,
1209 &outputDelay,
1210 &secureMode,
1211 },
1212 {},
1213 C2_DONT_BLOCK,
1214 nullptr);
1215 if (err == C2_BAD_INDEX) {
1216 if (!iStreamFormat || !oStreamFormat || !kind) {
1217 return UNKNOWN_ERROR;
1218 }
1219 } else if (err != C2_OK) {
1220 return UNKNOWN_ERROR;
1221 }
1222
1223 uint32_t inputDelayValue = inputDelay ? inputDelay.value : 0;
1224 uint32_t pipelineDelayValue = pipelineDelay ? pipelineDelay.value : 0;
1225 uint32_t outputDelayValue = outputDelay ? outputDelay.value : 0;
1226
1227 size_t numInputSlots = inputDelayValue + pipelineDelayValue + kSmoothnessFactor;
1228 size_t numOutputSlots = outputDelayValue + kSmoothnessFactor;
1229
1230 // TODO: get this from input format
1231 bool secure = mComponent->getName().find(".secure") != std::string::npos;
1232
1233 // secure mode is a static parameter (shall not change in the executing state)
1234 mSendEncryptedInfoBuffer = secureMode.value == C2Config::SM_READ_PROTECTED_WITH_ENCRYPTED;
1235
1236 std::shared_ptr<C2AllocatorStore> allocatorStore = GetCodec2PlatformAllocatorStore();
1237 int poolMask = GetCodec2PoolMask();
1238 C2PlatformAllocatorStore::id_t preferredLinearId = GetPreferredLinearAllocatorId(poolMask);
1239
1240 if (inputFormat != nullptr) {
1241 bool graphic = (iStreamFormat.value == C2BufferData::GRAPHIC);
1242 bool audioEncoder = !graphic && (kind.value == C2Component::KIND_ENCODER);
1243 C2Config::api_feature_t apiFeatures = C2Config::api_feature_t(
1244 API_REFLECTION |
1245 API_VALUES |
1246 API_CURRENT_VALUES |
1247 API_DEPENDENCY |
1248 API_SAME_INPUT_BUFFER);
1249 C2StreamAudioFrameSizeInfo::input encoderFrameSize(0u);
1250 C2StreamSampleRateInfo::input sampleRate(0u);
1251 C2StreamChannelCountInfo::input channelCount(0u);
1252 C2StreamPcmEncodingInfo::input pcmEncoding(0u);
1253 std::shared_ptr<C2BlockPool> pool;
1254 {
1255 Mutexed<BlockPools>::Locked pools(mBlockPools);
1256
1257 // set default allocator ID.
1258 pools->inputAllocatorId = (graphic) ? C2PlatformAllocatorStore::GRALLOC
1259 : preferredLinearId;
1260
1261 // query C2PortAllocatorsTuning::input from component. If an allocator ID is obtained
1262 // from component, create the input block pool with given ID. Otherwise, use default IDs.
1263 std::vector<std::unique_ptr<C2Param>> params;
1264 C2ApiFeaturesSetting featuresSetting{apiFeatures};
1265 std::vector<C2Param *> stackParams({&featuresSetting});
1266 if (audioEncoder) {
1267 stackParams.push_back(&encoderFrameSize);
1268 stackParams.push_back(&sampleRate);
1269 stackParams.push_back(&channelCount);
1270 stackParams.push_back(&pcmEncoding);
1271 } else {
1272 encoderFrameSize.invalidate();
1273 sampleRate.invalidate();
1274 channelCount.invalidate();
1275 pcmEncoding.invalidate();
1276 }
1277 err = mComponent->query(stackParams,
1278 { C2PortAllocatorsTuning::input::PARAM_TYPE },
1279 C2_DONT_BLOCK,
1280 ¶ms);
1281 if ((err != C2_OK && err != C2_BAD_INDEX) || params.size() != 1) {
1282 ALOGD("[%s] Query input allocators returned %zu params => %s (%u)",
1283 mName, params.size(), asString(err), err);
1284 } else if (params.size() == 1) {
1285 C2PortAllocatorsTuning::input *inputAllocators =
1286 C2PortAllocatorsTuning::input::From(params[0].get());
1287 if (inputAllocators && inputAllocators->flexCount() > 0) {
1288 std::shared_ptr<C2Allocator> allocator;
1289 // verify allocator IDs and resolve default allocator
1290 allocatorStore->fetchAllocator(inputAllocators->m.values[0], &allocator);
1291 if (allocator) {
1292 pools->inputAllocatorId = allocator->getId();
1293 } else {
1294 ALOGD("[%s] component requested invalid input allocator ID %u",
1295 mName, inputAllocators->m.values[0]);
1296 }
1297 }
1298 }
1299 if (featuresSetting) {
1300 apiFeatures = featuresSetting.value;
1301 }
1302
1303 // TODO: use C2Component wrapper to associate this pool with ourselves
1304 if ((poolMask >> pools->inputAllocatorId) & 1) {
1305 err = CreateCodec2BlockPool(pools->inputAllocatorId, nullptr, &pool);
1306 ALOGD("[%s] Created input block pool with allocatorID %u => poolID %llu - %s (%d)",
1307 mName, pools->inputAllocatorId,
1308 (unsigned long long)(pool ? pool->getLocalId() : 111000111),
1309 asString(err), err);
1310 } else {
1311 err = C2_NOT_FOUND;
1312 }
1313 if (err != C2_OK) {
1314 C2BlockPool::local_id_t inputPoolId =
1315 graphic ? C2BlockPool::BASIC_GRAPHIC : C2BlockPool::BASIC_LINEAR;
1316 err = GetCodec2BlockPool(inputPoolId, nullptr, &pool);
1317 ALOGD("[%s] Using basic input block pool with poolID %llu => got %llu - %s (%d)",
1318 mName, (unsigned long long)inputPoolId,
1319 (unsigned long long)(pool ? pool->getLocalId() : 111000111),
1320 asString(err), err);
1321 if (err != C2_OK) {
1322 return NO_MEMORY;
1323 }
1324 }
1325 pools->inputPool = pool;
1326 }
1327
1328 bool forceArrayMode = false;
1329 Mutexed<Input>::Locked input(mInput);
1330 input->inputDelay = inputDelayValue;
1331 input->pipelineDelay = pipelineDelayValue;
1332 input->numSlots = numInputSlots;
1333 input->extraBuffers.flush();
1334 input->numExtraSlots = 0u;
1335 input->lastFlushIndex = mFrameIndex.load(std::memory_order_relaxed);
1336 if (audioEncoder && encoderFrameSize && sampleRate && channelCount) {
1337 input->frameReassembler.init(
1338 pool,
1339 {C2MemoryUsage::CPU_READ, C2MemoryUsage::CPU_WRITE},
1340 encoderFrameSize.value,
1341 sampleRate.value,
1342 channelCount.value,
1343 pcmEncoding ? pcmEncoding.value : C2Config::PCM_16);
1344 }
1345 bool conforming = (apiFeatures & API_SAME_INPUT_BUFFER);
1346 // For encrypted content, framework decrypts source buffer (ashmem) into
1347 // C2Buffers. Thus non-conforming codecs can process these.
1348 if (!buffersBoundToCodec
1349 && !input->frameReassembler
1350 && (hasCryptoOrDescrambler() || conforming)) {
1351 input->buffers.reset(new SlotInputBuffers(mName));
1352 } else if (graphic) {
1353 if (mInputSurface) {
1354 input->buffers.reset(new DummyInputBuffers(mName));
1355 } else if (mMetaMode == MODE_ANW) {
1356 input->buffers.reset(new GraphicMetadataInputBuffers(mName));
1357 // This is to ensure buffers do not get released prematurely.
1358 // TODO: handle this without going into array mode
1359 forceArrayMode = true;
1360 } else {
1361 input->buffers.reset(new GraphicInputBuffers(mName));
1362 }
1363 } else {
1364 if (hasCryptoOrDescrambler()) {
1365 int32_t capacity = kLinearBufferSize;
1366 (void)inputFormat->findInt32(KEY_MAX_INPUT_SIZE, &capacity);
1367 if ((size_t)capacity > kMaxLinearBufferSize) {
1368 ALOGD("client requested %d, capped to %zu", capacity, kMaxLinearBufferSize);
1369 capacity = kMaxLinearBufferSize;
1370 }
1371 if (mDealer == nullptr) {
1372 mDealer = new MemoryDealer(
1373 align(capacity, MemoryDealer::getAllocationAlignment())
1374 * (numInputSlots + 1),
1375 "EncryptedLinearInputBuffers");
1376 mDecryptDestination = mDealer->allocate((size_t)capacity);
1377 }
1378 if (mCrypto != nullptr && mHeapSeqNum < 0) {
1379 sp<HidlMemory> heap = fromHeap(mDealer->getMemoryHeap());
1380 mHeapSeqNum = mCrypto->setHeap(heap);
1381 } else {
1382 mHeapSeqNum = -1;
1383 }
1384 input->buffers.reset(new EncryptedLinearInputBuffers(
1385 secure, mDealer, mCrypto, mHeapSeqNum, (size_t)capacity,
1386 numInputSlots, mName));
1387 forceArrayMode = true;
1388 } else {
1389 input->buffers.reset(new LinearInputBuffers(mName));
1390 }
1391 }
1392 input->buffers->setFormat(inputFormat);
1393
1394 if (err == C2_OK) {
1395 input->buffers->setPool(pool);
1396 } else {
1397 // TODO: error
1398 }
1399
1400 if (forceArrayMode) {
1401 input->buffers = input->buffers->toArrayMode(numInputSlots);
1402 }
1403 }
1404
1405 if (outputFormat != nullptr) {
1406 sp<IGraphicBufferProducer> outputSurface;
1407 uint32_t outputGeneration;
1408 int maxDequeueCount = 0;
1409 {
1410 Mutexed<OutputSurface>::Locked output(mOutputSurface);
1411 maxDequeueCount = output->maxDequeueBuffers = numOutputSlots +
1412 reorderDepth.value + mRenderingDepth;
1413 outputSurface = output->surface ?
1414 output->surface->getIGraphicBufferProducer() : nullptr;
1415 if (outputSurface) {
1416 output->surface->setMaxDequeuedBufferCount(output->maxDequeueBuffers);
1417 }
1418 outputGeneration = output->generation;
1419 }
1420
1421 bool graphic = (oStreamFormat.value == C2BufferData::GRAPHIC);
1422 C2BlockPool::local_id_t outputPoolId_;
1423 C2BlockPool::local_id_t prevOutputPoolId;
1424
1425 {
1426 Mutexed<BlockPools>::Locked pools(mBlockPools);
1427
1428 prevOutputPoolId = pools->outputPoolId;
1429
1430 // set default allocator ID.
1431 pools->outputAllocatorId = (graphic) ? C2PlatformAllocatorStore::GRALLOC
1432 : preferredLinearId;
1433
1434 // query C2PortAllocatorsTuning::output from component, or use default allocator if
1435 // unsuccessful.
1436 std::vector<std::unique_ptr<C2Param>> params;
1437 err = mComponent->query({ },
1438 { C2PortAllocatorsTuning::output::PARAM_TYPE },
1439 C2_DONT_BLOCK,
1440 ¶ms);
1441 if ((err != C2_OK && err != C2_BAD_INDEX) || params.size() != 1) {
1442 ALOGD("[%s] Query output allocators returned %zu params => %s (%u)",
1443 mName, params.size(), asString(err), err);
1444 } else if (err == C2_OK && params.size() == 1) {
1445 C2PortAllocatorsTuning::output *outputAllocators =
1446 C2PortAllocatorsTuning::output::From(params[0].get());
1447 if (outputAllocators && outputAllocators->flexCount() > 0) {
1448 std::shared_ptr<C2Allocator> allocator;
1449 // verify allocator IDs and resolve default allocator
1450 allocatorStore->fetchAllocator(outputAllocators->m.values[0], &allocator);
1451 if (allocator) {
1452 pools->outputAllocatorId = allocator->getId();
1453 } else {
1454 ALOGD("[%s] component requested invalid output allocator ID %u",
1455 mName, outputAllocators->m.values[0]);
1456 }
1457 }
1458 }
1459
1460 // use bufferqueue if outputting to a surface.
1461 // query C2PortSurfaceAllocatorTuning::output from component, or use default allocator
1462 // if unsuccessful.
1463 if (outputSurface) {
1464 params.clear();
1465 err = mComponent->query({ },
1466 { C2PortSurfaceAllocatorTuning::output::PARAM_TYPE },
1467 C2_DONT_BLOCK,
1468 ¶ms);
1469 if ((err != C2_OK && err != C2_BAD_INDEX) || params.size() != 1) {
1470 ALOGD("[%s] Query output surface allocator returned %zu params => %s (%u)",
1471 mName, params.size(), asString(err), err);
1472 } else if (err == C2_OK && params.size() == 1) {
1473 C2PortSurfaceAllocatorTuning::output *surfaceAllocator =
1474 C2PortSurfaceAllocatorTuning::output::From(params[0].get());
1475 if (surfaceAllocator) {
1476 std::shared_ptr<C2Allocator> allocator;
1477 // verify allocator IDs and resolve default allocator
1478 allocatorStore->fetchAllocator(surfaceAllocator->value, &allocator);
1479 if (allocator) {
1480 pools->outputAllocatorId = allocator->getId();
1481 } else {
1482 ALOGD("[%s] component requested invalid surface output allocator ID %u",
1483 mName, surfaceAllocator->value);
1484 err = C2_BAD_VALUE;
1485 }
1486 }
1487 }
1488 if (pools->outputAllocatorId == C2PlatformAllocatorStore::GRALLOC
1489 && err != C2_OK
1490 && ((poolMask >> C2PlatformAllocatorStore::BUFFERQUEUE) & 1)) {
1491 pools->outputAllocatorId = C2PlatformAllocatorStore::BUFFERQUEUE;
1492 }
1493 }
1494
1495 if ((poolMask >> pools->outputAllocatorId) & 1) {
1496 err = mComponent->createBlockPool(
1497 pools->outputAllocatorId, &pools->outputPoolId, &pools->outputPoolIntf);
1498 ALOGI("[%s] Created output block pool with allocatorID %u => poolID %llu - %s",
1499 mName, pools->outputAllocatorId,
1500 (unsigned long long)pools->outputPoolId,
1501 asString(err));
1502 } else {
1503 err = C2_NOT_FOUND;
1504 }
1505 if (err != C2_OK) {
1506 // use basic pool instead
1507 pools->outputPoolId =
1508 graphic ? C2BlockPool::BASIC_GRAPHIC : C2BlockPool::BASIC_LINEAR;
1509 }
1510
1511 // Configure output block pool ID as parameter C2PortBlockPoolsTuning::output to
1512 // component.
1513 std::unique_ptr<C2PortBlockPoolsTuning::output> poolIdsTuning =
1514 C2PortBlockPoolsTuning::output::AllocUnique({ pools->outputPoolId });
1515
1516 std::vector<std::unique_ptr<C2SettingResult>> failures;
1517 err = mComponent->config({ poolIdsTuning.get() }, C2_MAY_BLOCK, &failures);
1518 ALOGD("[%s] Configured output block pool ids %llu => %s",
1519 mName, (unsigned long long)poolIdsTuning->m.values[0], asString(err));
1520 outputPoolId_ = pools->outputPoolId;
1521 }
1522
1523 if (prevOutputPoolId != C2BlockPool::BASIC_LINEAR
1524 && prevOutputPoolId != C2BlockPool::BASIC_GRAPHIC) {
1525 c2_status_t err = mComponent->destroyBlockPool(prevOutputPoolId);
1526 if (err != C2_OK) {
1527 ALOGW("Failed to clean up previous block pool %llu - %s (%d)\n",
1528 (unsigned long long) prevOutputPoolId, asString(err), err);
1529 }
1530 }
1531
1532 Mutexed<Output>::Locked output(mOutput);
1533 output->outputDelay = outputDelayValue;
1534 output->numSlots = numOutputSlots;
1535 output->bounded = bool(outputSurface);
1536 if (graphic) {
1537 if (outputSurface || !buffersBoundToCodec) {
1538 output->buffers.reset(new GraphicOutputBuffers(mName));
1539 } else {
1540 output->buffers.reset(new RawGraphicOutputBuffers(mName));
1541 }
1542 } else {
1543 output->buffers.reset(new LinearOutputBuffers(mName));
1544 }
1545 output->buffers->setFormat(outputFormat);
1546
1547 output->buffers->clearStash();
1548 if (reorderDepth) {
1549 output->buffers->setReorderDepth(reorderDepth.value);
1550 }
1551 if (reorderKey) {
1552 output->buffers->setReorderKey(reorderKey.value);
1553 }
1554
1555 // Try to set output surface to created block pool if given.
1556 if (outputSurface) {
1557 mComponent->setOutputSurface(
1558 outputPoolId_,
1559 outputSurface,
1560 outputGeneration,
1561 maxDequeueCount);
1562 } else {
1563 // configure CPU read consumer usage
1564 C2StreamUsageTuning::output outputUsage{0u, C2MemoryUsage::CPU_READ};
1565 std::vector<std::unique_ptr<C2SettingResult>> failures;
1566 err = mComponent->config({ &outputUsage }, C2_MAY_BLOCK, &failures);
1567 // do not print error message for now as most components may not yet
1568 // support this setting
1569 ALOGD_IF(err != C2_BAD_INDEX, "[%s] Configured output usage [%#llx]",
1570 mName, (long long)outputUsage.value);
1571 }
1572
1573 if (oStreamFormat.value == C2BufferData::LINEAR) {
1574 if (buffersBoundToCodec) {
1575 // WORKAROUND: if we're using early CSD workaround we convert to
1576 // array mode, to appease apps assuming the output
1577 // buffers to be of the same size.
1578 output->buffers = output->buffers->toArrayMode(numOutputSlots);
1579 }
1580
1581 int32_t channelCount;
1582 int32_t sampleRate;
1583 if (outputFormat->findInt32(KEY_CHANNEL_COUNT, &channelCount)
1584 && outputFormat->findInt32(KEY_SAMPLE_RATE, &sampleRate)) {
1585 int32_t delay = 0;
1586 int32_t padding = 0;;
1587 if (!outputFormat->findInt32("encoder-delay", &delay)) {
1588 delay = 0;
1589 }
1590 if (!outputFormat->findInt32("encoder-padding", &padding)) {
1591 padding = 0;
1592 }
1593 if (delay || padding) {
1594 // We need write access to the buffers, and we're already in
1595 // array mode.
1596 output->buffers->initSkipCutBuffer(delay, padding, sampleRate, channelCount);
1597 }
1598 }
1599 }
1600
1601 int32_t tunneled = 0;
1602 if (!outputFormat->findInt32("android._tunneled", &tunneled)) {
1603 tunneled = 0;
1604 }
1605 mTunneled = (tunneled != 0);
1606 }
1607
1608 // Set up pipeline control. This has to be done after mInputBuffers and
1609 // mOutputBuffers are initialized to make sure that lingering callbacks
1610 // about buffers from the previous generation do not interfere with the
1611 // newly initialized pipeline capacity.
1612
1613 if (inputFormat || outputFormat) {
1614 Mutexed<PipelineWatcher>::Locked watcher(mPipelineWatcher);
1615 watcher->inputDelay(inputDelayValue)
1616 .pipelineDelay(pipelineDelayValue)
1617 .outputDelay(outputDelayValue)
1618 .smoothnessFactor(kSmoothnessFactor);
1619 watcher->flush();
1620 }
1621
1622 mInputMetEos = false;
1623 mSync.start();
1624 return OK;
1625 }
1626
prepareInitialInputBuffers(std::map<size_t,sp<MediaCodecBuffer>> * clientInputBuffers,bool retry)1627 status_t CCodecBufferChannel::prepareInitialInputBuffers(
1628 std::map<size_t, sp<MediaCodecBuffer>> *clientInputBuffers, bool retry) {
1629 if (mInputSurface) {
1630 return OK;
1631 }
1632
1633 size_t numInputSlots = mInput.lock()->numSlots;
1634 int retryCount = 1;
1635 for (; clientInputBuffers->empty() && retryCount >= 0; retryCount--) {
1636 {
1637 Mutexed<Input>::Locked input(mInput);
1638 while (clientInputBuffers->size() < numInputSlots) {
1639 size_t index;
1640 sp<MediaCodecBuffer> buffer;
1641 if (!input->buffers->requestNewBuffer(&index, &buffer)) {
1642 break;
1643 }
1644 clientInputBuffers->emplace(index, buffer);
1645 }
1646 }
1647 if (!retry || (retryCount <= 0)) {
1648 break;
1649 }
1650 if (clientInputBuffers->empty()) {
1651 // wait: buffer may be in transit from component.
1652 std::this_thread::sleep_for(std::chrono::milliseconds(4));
1653 }
1654 }
1655 if (clientInputBuffers->empty()) {
1656 ALOGW("[%s] start: cannot allocate memory at all", mName);
1657 return NO_MEMORY;
1658 } else if (clientInputBuffers->size() < numInputSlots) {
1659 ALOGD("[%s] start: cannot allocate memory for all slots, "
1660 "only %zu buffers allocated",
1661 mName, clientInputBuffers->size());
1662 } else {
1663 ALOGV("[%s] %zu initial input buffers available",
1664 mName, clientInputBuffers->size());
1665 }
1666 return OK;
1667 }
1668
requestInitialInputBuffers(std::map<size_t,sp<MediaCodecBuffer>> && clientInputBuffers)1669 status_t CCodecBufferChannel::requestInitialInputBuffers(
1670 std::map<size_t, sp<MediaCodecBuffer>> &&clientInputBuffers) {
1671 C2StreamBufferTypeSetting::output oStreamFormat(0u);
1672 C2PrependHeaderModeSetting prepend(PREPEND_HEADER_TO_NONE);
1673 c2_status_t err = mComponent->query({ &oStreamFormat, &prepend }, {}, C2_DONT_BLOCK, nullptr);
1674 if (err != C2_OK && err != C2_BAD_INDEX) {
1675 return UNKNOWN_ERROR;
1676 }
1677
1678 std::list<std::unique_ptr<C2Work>> flushedConfigs;
1679 mFlushedConfigs.lock()->swap(flushedConfigs);
1680 if (!flushedConfigs.empty()) {
1681 {
1682 Mutexed<PipelineWatcher>::Locked watcher(mPipelineWatcher);
1683 PipelineWatcher::Clock::time_point now = PipelineWatcher::Clock::now();
1684 for (const std::unique_ptr<C2Work> &work : flushedConfigs) {
1685 watcher->onWorkQueued(
1686 work->input.ordinal.frameIndex.peeku(),
1687 std::vector(work->input.buffers),
1688 now);
1689 }
1690 }
1691 err = mComponent->queue(&flushedConfigs);
1692 if (err != C2_OK) {
1693 ALOGW("[%s] Error while queueing a flushed config", mName);
1694 return UNKNOWN_ERROR;
1695 }
1696 }
1697 if (oStreamFormat.value == C2BufferData::LINEAR &&
1698 (!prepend || prepend.value == PREPEND_HEADER_TO_NONE) &&
1699 !clientInputBuffers.empty()) {
1700 size_t minIndex = clientInputBuffers.begin()->first;
1701 sp<MediaCodecBuffer> minBuffer = clientInputBuffers.begin()->second;
1702 for (const auto &[index, buffer] : clientInputBuffers) {
1703 if (minBuffer->capacity() > buffer->capacity()) {
1704 minIndex = index;
1705 minBuffer = buffer;
1706 }
1707 }
1708 // WORKAROUND: Some apps expect CSD available without queueing
1709 // any input. Queue an empty buffer to get the CSD.
1710 minBuffer->setRange(0, 0);
1711 minBuffer->meta()->clear();
1712 minBuffer->meta()->setInt64("timeUs", 0);
1713 if (queueInputBufferInternal(minBuffer) != OK) {
1714 ALOGW("[%s] Error while queueing an empty buffer to get CSD",
1715 mName);
1716 return UNKNOWN_ERROR;
1717 }
1718 clientInputBuffers.erase(minIndex);
1719 }
1720
1721 for (const auto &[index, buffer] : clientInputBuffers) {
1722 mCallback->onInputBufferAvailable(index, buffer);
1723 }
1724
1725 return OK;
1726 }
1727
stop()1728 void CCodecBufferChannel::stop() {
1729 mSync.stop();
1730 mFirstValidFrameIndex = mFrameIndex.load(std::memory_order_relaxed);
1731 }
1732
stopUseOutputSurface(bool pushBlankBuffer)1733 void CCodecBufferChannel::stopUseOutputSurface(bool pushBlankBuffer) {
1734 sp<Surface> surface = mOutputSurface.lock()->surface;
1735 if (surface) {
1736 C2BlockPool::local_id_t outputPoolId;
1737 {
1738 Mutexed<BlockPools>::Locked pools(mBlockPools);
1739 outputPoolId = pools->outputPoolId;
1740 }
1741 if (mComponent) mComponent->stopUsingOutputSurface(outputPoolId);
1742
1743 if (pushBlankBuffer) {
1744 sp<ANativeWindow> anw = static_cast<ANativeWindow *>(surface.get());
1745 if (anw) {
1746 pushBlankBuffersToNativeWindow(anw.get());
1747 }
1748 }
1749 }
1750 }
1751
reset()1752 void CCodecBufferChannel::reset() {
1753 stop();
1754 if (mInputSurface != nullptr) {
1755 mInputSurface.reset();
1756 }
1757 mPipelineWatcher.lock()->flush();
1758 {
1759 Mutexed<Input>::Locked input(mInput);
1760 input->buffers.reset(new DummyInputBuffers(""));
1761 input->extraBuffers.flush();
1762 }
1763 {
1764 Mutexed<Output>::Locked output(mOutput);
1765 output->buffers.reset();
1766 }
1767 // reset the frames that are being tracked for onFrameRendered callbacks
1768 mTrackedFrames.clear();
1769 }
1770
release()1771 void CCodecBufferChannel::release() {
1772 mComponent.reset();
1773 mInputAllocator.reset();
1774 mOutputSurface.lock()->surface.clear();
1775 {
1776 Mutexed<BlockPools>::Locked blockPools{mBlockPools};
1777 blockPools->inputPool.reset();
1778 blockPools->outputPoolIntf.reset();
1779 }
1780 setCrypto(nullptr);
1781 setDescrambler(nullptr);
1782 }
1783
flush(const std::list<std::unique_ptr<C2Work>> & flushedWork)1784 void CCodecBufferChannel::flush(const std::list<std::unique_ptr<C2Work>> &flushedWork) {
1785 ALOGV("[%s] flush", mName);
1786 std::list<std::unique_ptr<C2Work>> configs;
1787 mInput.lock()->lastFlushIndex = mFrameIndex.load(std::memory_order_relaxed);
1788 {
1789 Mutexed<PipelineWatcher>::Locked watcher(mPipelineWatcher);
1790 for (const std::unique_ptr<C2Work> &work : flushedWork) {
1791 uint64_t frameIndex = work->input.ordinal.frameIndex.peeku();
1792 if (!(work->input.flags & C2FrameData::FLAG_CODEC_CONFIG)) {
1793 watcher->onWorkDone(frameIndex);
1794 continue;
1795 }
1796 if (work->input.buffers.empty()
1797 || work->input.buffers.front() == nullptr
1798 || work->input.buffers.front()->data().linearBlocks().empty()) {
1799 ALOGD("[%s] no linear codec config data found", mName);
1800 watcher->onWorkDone(frameIndex);
1801 continue;
1802 }
1803 std::unique_ptr<C2Work> copy(new C2Work);
1804 copy->input.flags = C2FrameData::flags_t(
1805 work->input.flags | C2FrameData::FLAG_DROP_FRAME);
1806 copy->input.ordinal = work->input.ordinal;
1807 copy->input.ordinal.frameIndex = mFrameIndex++;
1808 for (size_t i = 0; i < work->input.buffers.size(); ++i) {
1809 copy->input.buffers.push_back(watcher->onInputBufferReleased(frameIndex, i));
1810 }
1811 for (const std::unique_ptr<C2Param> ¶m : work->input.configUpdate) {
1812 copy->input.configUpdate.push_back(C2Param::Copy(*param));
1813 }
1814 copy->input.infoBuffers.insert(
1815 copy->input.infoBuffers.begin(),
1816 work->input.infoBuffers.begin(),
1817 work->input.infoBuffers.end());
1818 copy->worklets.emplace_back(new C2Worklet);
1819 configs.push_back(std::move(copy));
1820 watcher->onWorkDone(frameIndex);
1821 ALOGV("[%s] stashed flushed codec config data", mName);
1822 }
1823 }
1824 mFlushedConfigs.lock()->swap(configs);
1825 {
1826 Mutexed<Input>::Locked input(mInput);
1827 input->buffers->flush();
1828 input->extraBuffers.flush();
1829 }
1830 {
1831 Mutexed<Output>::Locked output(mOutput);
1832 if (output->buffers) {
1833 output->buffers->flush(flushedWork);
1834 output->buffers->flushStash();
1835 }
1836 }
1837 }
1838
onWorkDone(std::unique_ptr<C2Work> work,const sp<AMessage> & outputFormat,const C2StreamInitDataInfo::output * initData)1839 void CCodecBufferChannel::onWorkDone(
1840 std::unique_ptr<C2Work> work, const sp<AMessage> &outputFormat,
1841 const C2StreamInitDataInfo::output *initData) {
1842 if (handleWork(std::move(work), outputFormat, initData)) {
1843 feedInputBufferIfAvailable();
1844 }
1845 }
1846
onInputBufferDone(uint64_t frameIndex,size_t arrayIndex)1847 void CCodecBufferChannel::onInputBufferDone(
1848 uint64_t frameIndex, size_t arrayIndex) {
1849 if (mInputSurface) {
1850 return;
1851 }
1852 std::shared_ptr<C2Buffer> buffer =
1853 mPipelineWatcher.lock()->onInputBufferReleased(frameIndex, arrayIndex);
1854 bool newInputSlotAvailable = false;
1855 {
1856 Mutexed<Input>::Locked input(mInput);
1857 if (input->lastFlushIndex >= frameIndex) {
1858 ALOGD("[%s] Ignoring stale input buffer done callback: "
1859 "last flush index = %lld, frameIndex = %lld",
1860 mName, input->lastFlushIndex.peekll(), (long long)frameIndex);
1861 } else {
1862 newInputSlotAvailable = input->buffers->expireComponentBuffer(buffer);
1863 if (!newInputSlotAvailable) {
1864 (void)input->extraBuffers.expireComponentBuffer(buffer);
1865 }
1866 }
1867 }
1868 if (newInputSlotAvailable) {
1869 feedInputBufferIfAvailable();
1870 }
1871 }
1872
handleWork(std::unique_ptr<C2Work> work,const sp<AMessage> & outputFormat,const C2StreamInitDataInfo::output * initData)1873 bool CCodecBufferChannel::handleWork(
1874 std::unique_ptr<C2Work> work,
1875 const sp<AMessage> &outputFormat,
1876 const C2StreamInitDataInfo::output *initData) {
1877 {
1878 Mutexed<Output>::Locked output(mOutput);
1879 if (!output->buffers) {
1880 return false;
1881 }
1882 }
1883
1884 // Whether the output buffer should be reported to the client or not.
1885 bool notifyClient = false;
1886
1887 if (work->result == C2_OK){
1888 notifyClient = true;
1889 } else if (work->result == C2_NOT_FOUND) {
1890 ALOGD("[%s] flushed work; ignored.", mName);
1891 } else {
1892 // C2_OK and C2_NOT_FOUND are the only results that we accept for processing
1893 // the config update.
1894 ALOGD("[%s] work failed to complete: %d", mName, work->result);
1895 mCCodecCallback->onError(work->result, ACTION_CODE_FATAL);
1896 return false;
1897 }
1898
1899 if ((work->input.ordinal.frameIndex -
1900 mFirstValidFrameIndex.load()).peek() < 0) {
1901 // Discard frames from previous generation.
1902 ALOGD("[%s] Discard frames from previous generation.", mName);
1903 notifyClient = false;
1904 }
1905
1906 if (mInputSurface == nullptr && (work->worklets.size() != 1u
1907 || !work->worklets.front()
1908 || !(work->worklets.front()->output.flags &
1909 C2FrameData::FLAG_INCOMPLETE))) {
1910 mPipelineWatcher.lock()->onWorkDone(
1911 work->input.ordinal.frameIndex.peeku());
1912 }
1913
1914 // NOTE: MediaCodec usage supposedly have only one worklet
1915 if (work->worklets.size() != 1u) {
1916 ALOGI("[%s] onWorkDone: incorrect number of worklets: %zu",
1917 mName, work->worklets.size());
1918 mCCodecCallback->onError(UNKNOWN_ERROR, ACTION_CODE_FATAL);
1919 return false;
1920 }
1921
1922 const std::unique_ptr<C2Worklet> &worklet = work->worklets.front();
1923
1924 std::shared_ptr<C2Buffer> buffer;
1925 // NOTE: MediaCodec usage supposedly have only one output stream.
1926 if (worklet->output.buffers.size() > 1u) {
1927 ALOGI("[%s] onWorkDone: incorrect number of output buffers: %zu",
1928 mName, worklet->output.buffers.size());
1929 mCCodecCallback->onError(UNKNOWN_ERROR, ACTION_CODE_FATAL);
1930 return false;
1931 } else if (worklet->output.buffers.size() == 1u) {
1932 buffer = worklet->output.buffers[0];
1933 if (!buffer) {
1934 ALOGD("[%s] onWorkDone: nullptr found in buffers; ignored.", mName);
1935 }
1936 }
1937
1938 std::optional<uint32_t> newInputDelay, newPipelineDelay, newOutputDelay, newReorderDepth;
1939 std::optional<C2Config::ordinal_key_t> newReorderKey;
1940 bool needMaxDequeueBufferCountUpdate = false;
1941 while (!worklet->output.configUpdate.empty()) {
1942 std::unique_ptr<C2Param> param;
1943 worklet->output.configUpdate.back().swap(param);
1944 worklet->output.configUpdate.pop_back();
1945 switch (param->coreIndex().coreIndex()) {
1946 case C2PortReorderBufferDepthTuning::CORE_INDEX: {
1947 C2PortReorderBufferDepthTuning::output reorderDepth;
1948 if (reorderDepth.updateFrom(*param)) {
1949 ALOGV("[%s] onWorkDone: updated reorder depth to %u",
1950 mName, reorderDepth.value);
1951 newReorderDepth = reorderDepth.value;
1952 needMaxDequeueBufferCountUpdate = true;
1953 } else {
1954 ALOGD("[%s] onWorkDone: failed to read reorder depth",
1955 mName);
1956 }
1957 break;
1958 }
1959 case C2PortReorderKeySetting::CORE_INDEX: {
1960 C2PortReorderKeySetting::output reorderKey;
1961 if (reorderKey.updateFrom(*param)) {
1962 newReorderKey = reorderKey.value;
1963 ALOGV("[%s] onWorkDone: updated reorder key to %u",
1964 mName, reorderKey.value);
1965 } else {
1966 ALOGD("[%s] onWorkDone: failed to read reorder key", mName);
1967 }
1968 break;
1969 }
1970 case C2PortActualDelayTuning::CORE_INDEX: {
1971 if (param->isGlobal()) {
1972 C2ActualPipelineDelayTuning pipelineDelay;
1973 if (pipelineDelay.updateFrom(*param)) {
1974 ALOGV("[%s] onWorkDone: updating pipeline delay %u",
1975 mName, pipelineDelay.value);
1976 newPipelineDelay = pipelineDelay.value;
1977 (void)mPipelineWatcher.lock()->pipelineDelay(
1978 pipelineDelay.value);
1979 }
1980 }
1981 if (param->forInput()) {
1982 C2PortActualDelayTuning::input inputDelay;
1983 if (inputDelay.updateFrom(*param)) {
1984 ALOGV("[%s] onWorkDone: updating input delay %u",
1985 mName, inputDelay.value);
1986 newInputDelay = inputDelay.value;
1987 (void)mPipelineWatcher.lock()->inputDelay(
1988 inputDelay.value);
1989 }
1990 }
1991 if (param->forOutput()) {
1992 C2PortActualDelayTuning::output outputDelay;
1993 if (outputDelay.updateFrom(*param)) {
1994 ALOGV("[%s] onWorkDone: updating output delay %u",
1995 mName, outputDelay.value);
1996 (void)mPipelineWatcher.lock()->outputDelay(outputDelay.value);
1997 newOutputDelay = outputDelay.value;
1998 needMaxDequeueBufferCountUpdate = true;
1999
2000 }
2001 }
2002 break;
2003 }
2004 case C2PortTunnelSystemTime::CORE_INDEX: {
2005 C2PortTunnelSystemTime::output frameRenderTime;
2006 if (frameRenderTime.updateFrom(*param)) {
2007 ALOGV("[%s] onWorkDone: frame rendered (sys:%lld ns, media:%lld us)",
2008 mName, (long long)frameRenderTime.value,
2009 (long long)worklet->output.ordinal.timestamp.peekll());
2010 mCCodecCallback->onOutputFramesRendered(
2011 worklet->output.ordinal.timestamp.peek(), frameRenderTime.value);
2012 }
2013 break;
2014 }
2015 case C2StreamTunnelHoldRender::CORE_INDEX: {
2016 C2StreamTunnelHoldRender::output firstTunnelFrameHoldRender;
2017 if (!(worklet->output.flags & C2FrameData::FLAG_INCOMPLETE)) break;
2018 if (!firstTunnelFrameHoldRender.updateFrom(*param)) break;
2019 if (firstTunnelFrameHoldRender.value != C2_TRUE) break;
2020 ALOGV("[%s] onWorkDone: first tunnel frame ready", mName);
2021 mCCodecCallback->onFirstTunnelFrameReady();
2022 break;
2023 }
2024 default:
2025 ALOGV("[%s] onWorkDone: unrecognized config update (%08X)",
2026 mName, param->index());
2027 break;
2028 }
2029 }
2030 if (newInputDelay || newPipelineDelay) {
2031 Mutexed<Input>::Locked input(mInput);
2032 size_t newNumSlots =
2033 newInputDelay.value_or(input->inputDelay) +
2034 newPipelineDelay.value_or(input->pipelineDelay) +
2035 kSmoothnessFactor;
2036 if (input->buffers->isArrayMode()) {
2037 if (input->numSlots >= newNumSlots) {
2038 input->numExtraSlots = 0;
2039 } else {
2040 input->numExtraSlots = newNumSlots - input->numSlots;
2041 }
2042 ALOGV("[%s] onWorkDone: updated number of extra slots to %zu (input array mode)",
2043 mName, input->numExtraSlots);
2044 } else {
2045 input->numSlots = newNumSlots;
2046 }
2047 }
2048 size_t numOutputSlots = 0;
2049 uint32_t reorderDepth = 0;
2050 bool outputBuffersChanged = false;
2051 if (newReorderKey || newReorderDepth || needMaxDequeueBufferCountUpdate) {
2052 Mutexed<Output>::Locked output(mOutput);
2053 if (!output->buffers) {
2054 return false;
2055 }
2056 numOutputSlots = output->numSlots;
2057 if (newReorderKey) {
2058 output->buffers->setReorderKey(newReorderKey.value());
2059 }
2060 if (newReorderDepth) {
2061 output->buffers->setReorderDepth(newReorderDepth.value());
2062 }
2063 reorderDepth = output->buffers->getReorderDepth();
2064 if (newOutputDelay) {
2065 output->outputDelay = newOutputDelay.value();
2066 numOutputSlots = newOutputDelay.value() + kSmoothnessFactor;
2067 if (output->numSlots < numOutputSlots) {
2068 output->numSlots = numOutputSlots;
2069 if (output->buffers->isArrayMode()) {
2070 OutputBuffersArray *array =
2071 (OutputBuffersArray *)output->buffers.get();
2072 ALOGV("[%s] onWorkDone: growing output buffer array to %zu",
2073 mName, numOutputSlots);
2074 array->grow(numOutputSlots);
2075 outputBuffersChanged = true;
2076 }
2077 }
2078 }
2079 numOutputSlots = output->numSlots;
2080 }
2081 if (outputBuffersChanged) {
2082 mCCodecCallback->onOutputBuffersChanged();
2083 }
2084 if (needMaxDequeueBufferCountUpdate) {
2085 int maxDequeueCount = 0;
2086 {
2087 Mutexed<OutputSurface>::Locked output(mOutputSurface);
2088 maxDequeueCount = output->maxDequeueBuffers =
2089 numOutputSlots + reorderDepth + mRenderingDepth;
2090 if (output->surface) {
2091 output->surface->setMaxDequeuedBufferCount(output->maxDequeueBuffers);
2092 }
2093 }
2094 if (maxDequeueCount > 0) {
2095 mComponent->setOutputSurfaceMaxDequeueCount(maxDequeueCount);
2096 }
2097 }
2098
2099 int32_t flags = 0;
2100 if (worklet->output.flags & C2FrameData::FLAG_END_OF_STREAM) {
2101 flags |= BUFFER_FLAG_END_OF_STREAM;
2102 ALOGV("[%s] onWorkDone: output EOS", mName);
2103 }
2104
2105 // WORKAROUND: adjust output timestamp based on client input timestamp and codec
2106 // input timestamp. Codec output timestamp (in the timestamp field) shall correspond to
2107 // the codec input timestamp, but client output timestamp should (reported in timeUs)
2108 // shall correspond to the client input timesamp (in customOrdinal). By using the
2109 // delta between the two, this allows for some timestamp deviation - e.g. if one input
2110 // produces multiple output.
2111 c2_cntr64_t timestamp =
2112 worklet->output.ordinal.timestamp + work->input.ordinal.customOrdinal
2113 - work->input.ordinal.timestamp;
2114 if (mInputSurface != nullptr) {
2115 // When using input surface we need to restore the original input timestamp.
2116 timestamp = work->input.ordinal.customOrdinal;
2117 }
2118 ScopedTrace trace(ATRACE_TAG, android::base::StringPrintf(
2119 "CCodecBufferChannel::onWorkDone(%s@ts=%lld)", mName, timestamp.peekll()).c_str());
2120 ALOGV("[%s] onWorkDone: input %lld, codec %lld => output %lld => %lld",
2121 mName,
2122 work->input.ordinal.customOrdinal.peekll(),
2123 work->input.ordinal.timestamp.peekll(),
2124 worklet->output.ordinal.timestamp.peekll(),
2125 timestamp.peekll());
2126
2127 // csd cannot be re-ordered and will always arrive first.
2128 if (initData != nullptr) {
2129 Mutexed<Output>::Locked output(mOutput);
2130 if (!output->buffers) {
2131 return false;
2132 }
2133 if (outputFormat) {
2134 output->buffers->updateSkipCutBuffer(outputFormat);
2135 output->buffers->setFormat(outputFormat);
2136 }
2137 if (!notifyClient) {
2138 return false;
2139 }
2140 size_t index;
2141 sp<MediaCodecBuffer> outBuffer;
2142 if (output->buffers->registerCsd(initData, &index, &outBuffer) == OK) {
2143 outBuffer->meta()->setInt64("timeUs", timestamp.peek());
2144 outBuffer->meta()->setInt32("flags", BUFFER_FLAG_CODEC_CONFIG);
2145 ALOGV("[%s] onWorkDone: csd index = %zu [%p]", mName, index, outBuffer.get());
2146
2147 // TRICKY: we want popped buffers reported in order, so sending
2148 // the callback while holding the lock here. This assumes that
2149 // onOutputBufferAvailable() does not block. onOutputBufferAvailable()
2150 // callbacks are always sent with the Output lock held.
2151 mCallback->onOutputBufferAvailable(index, outBuffer);
2152 } else {
2153 ALOGD("[%s] onWorkDone: unable to register csd", mName);
2154 output.unlock();
2155 mCCodecCallback->onError(UNKNOWN_ERROR, ACTION_CODE_FATAL);
2156 return false;
2157 }
2158 }
2159
2160 bool drop = false;
2161 if (worklet->output.flags & C2FrameData::FLAG_DROP_FRAME) {
2162 ALOGV("[%s] onWorkDone: drop buffer but keep metadata", mName);
2163 drop = true;
2164 }
2165
2166 // Workaround: if C2FrameData::FLAG_DROP_FRAME is not implemented in
2167 // HAL, the flag is then removed in the corresponding output buffer.
2168 if (work->input.flags & C2FrameData::FLAG_DROP_FRAME) {
2169 flags |= BUFFER_FLAG_DECODE_ONLY;
2170 }
2171
2172 if (notifyClient && !buffer && !flags) {
2173 if (mTunneled && drop && outputFormat) {
2174 ALOGV("[%s] onWorkDone: Keep tunneled, drop frame with format change (%lld)",
2175 mName, work->input.ordinal.frameIndex.peekull());
2176 } else {
2177 ALOGV("[%s] onWorkDone: Not reporting output buffer (%lld)",
2178 mName, work->input.ordinal.frameIndex.peekull());
2179 notifyClient = false;
2180 }
2181 }
2182
2183 if (buffer) {
2184 for (const std::shared_ptr<const C2Info> &info : buffer->info()) {
2185 // TODO: properly translate these to metadata
2186 switch (info->coreIndex().coreIndex()) {
2187 case C2StreamPictureTypeMaskInfo::CORE_INDEX:
2188 if (((C2StreamPictureTypeMaskInfo *)info.get())->value & C2Config::SYNC_FRAME) {
2189 flags |= BUFFER_FLAG_KEY_FRAME;
2190 }
2191 break;
2192 default:
2193 break;
2194 }
2195 }
2196 }
2197
2198 {
2199 Mutexed<Output>::Locked output(mOutput);
2200 if (!output->buffers) {
2201 return false;
2202 }
2203 output->buffers->pushToStash(
2204 buffer,
2205 notifyClient,
2206 timestamp.peek(),
2207 flags,
2208 outputFormat,
2209 worklet->output.ordinal);
2210 }
2211 sendOutputBuffers();
2212 return true;
2213 }
2214
sendOutputBuffers()2215 void CCodecBufferChannel::sendOutputBuffers() {
2216 OutputBuffers::BufferAction action;
2217 size_t index;
2218 sp<MediaCodecBuffer> outBuffer;
2219 std::shared_ptr<C2Buffer> c2Buffer;
2220
2221 constexpr int kMaxReallocTry = 5;
2222 int reallocTryNum = 0;
2223
2224 while (true) {
2225 Mutexed<Output>::Locked output(mOutput);
2226 if (!output->buffers) {
2227 return;
2228 }
2229 action = output->buffers->popFromStashAndRegister(
2230 &c2Buffer, &index, &outBuffer);
2231 if (action != OutputBuffers::REALLOCATE) {
2232 reallocTryNum = 0;
2233 }
2234 switch (action) {
2235 case OutputBuffers::SKIP:
2236 return;
2237 case OutputBuffers::DISCARD:
2238 break;
2239 case OutputBuffers::NOTIFY_CLIENT:
2240 // TRICKY: we want popped buffers reported in order, so sending
2241 // the callback while holding the lock here. This assumes that
2242 // onOutputBufferAvailable() does not block. onOutputBufferAvailable()
2243 // callbacks are always sent with the Output lock held.
2244 mCallback->onOutputBufferAvailable(index, outBuffer);
2245 break;
2246 case OutputBuffers::REALLOCATE:
2247 if (++reallocTryNum > kMaxReallocTry) {
2248 output.unlock();
2249 ALOGE("[%s] sendOutputBuffers: tried %d realloc and failed",
2250 mName, kMaxReallocTry);
2251 mCCodecCallback->onError(UNKNOWN_ERROR, ACTION_CODE_FATAL);
2252 return;
2253 }
2254 if (!output->buffers->isArrayMode()) {
2255 output->buffers =
2256 output->buffers->toArrayMode(output->numSlots);
2257 }
2258 static_cast<OutputBuffersArray*>(output->buffers.get())->
2259 realloc(c2Buffer);
2260 output.unlock();
2261 mCCodecCallback->onOutputBuffersChanged();
2262 break;
2263 case OutputBuffers::RETRY:
2264 ALOGV("[%s] sendOutputBuffers: unable to register output buffer",
2265 mName);
2266 return;
2267 default:
2268 LOG_ALWAYS_FATAL("[%s] sendOutputBuffers: "
2269 "corrupted BufferAction value (%d) "
2270 "returned from popFromStashAndRegister.",
2271 mName, int(action));
2272 return;
2273 }
2274 }
2275 }
2276
setSurface(const sp<Surface> & newSurface,bool pushBlankBuffer)2277 status_t CCodecBufferChannel::setSurface(const sp<Surface> &newSurface, bool pushBlankBuffer) {
2278 static std::atomic_uint32_t surfaceGeneration{0};
2279 uint32_t generation = (getpid() << 10) |
2280 ((surfaceGeneration.fetch_add(1, std::memory_order_relaxed) + 1)
2281 & ((1 << 10) - 1));
2282
2283 sp<IGraphicBufferProducer> producer;
2284 int maxDequeueCount;
2285 sp<Surface> oldSurface;
2286 {
2287 Mutexed<OutputSurface>::Locked outputSurface(mOutputSurface);
2288 maxDequeueCount = outputSurface->maxDequeueBuffers;
2289 oldSurface = outputSurface->surface;
2290 }
2291 if (newSurface) {
2292 newSurface->setScalingMode(NATIVE_WINDOW_SCALING_MODE_SCALE_TO_WINDOW);
2293 newSurface->setDequeueTimeout(kDequeueTimeoutNs);
2294 newSurface->setMaxDequeuedBufferCount(maxDequeueCount);
2295 producer = newSurface->getIGraphicBufferProducer();
2296 producer->setGenerationNumber(generation);
2297 } else {
2298 ALOGE("[%s] setting output surface to null", mName);
2299 return INVALID_OPERATION;
2300 }
2301
2302 std::shared_ptr<Codec2Client::Configurable> outputPoolIntf;
2303 C2BlockPool::local_id_t outputPoolId;
2304 {
2305 Mutexed<BlockPools>::Locked pools(mBlockPools);
2306 outputPoolId = pools->outputPoolId;
2307 outputPoolIntf = pools->outputPoolIntf;
2308 }
2309
2310 if (outputPoolIntf) {
2311 if (mComponent->setOutputSurface(
2312 outputPoolId,
2313 producer,
2314 generation,
2315 maxDequeueCount) != C2_OK) {
2316 ALOGI("[%s] setSurface: component setOutputSurface failed", mName);
2317 return INVALID_OPERATION;
2318 }
2319 }
2320
2321 {
2322 Mutexed<OutputSurface>::Locked output(mOutputSurface);
2323 output->surface = newSurface;
2324 output->generation = generation;
2325 initializeFrameTrackingFor(static_cast<ANativeWindow *>(newSurface.get()));
2326 }
2327
2328 if (oldSurface && pushBlankBuffer) {
2329 // When ReleaseSurface was set from MediaCodec,
2330 // pushing a blank buffer at the end might be necessary.
2331 sp<ANativeWindow> anw = static_cast<ANativeWindow *>(oldSurface.get());
2332 if (anw) {
2333 pushBlankBuffersToNativeWindow(anw.get());
2334 }
2335 }
2336
2337 return OK;
2338 }
2339
elapsed()2340 PipelineWatcher::Clock::duration CCodecBufferChannel::elapsed() {
2341 // Otherwise, component may have stalled work due to input starvation up to
2342 // the sum of the delay in the pipeline.
2343 // TODO(b/231253301): When client pushed EOS, the pipeline could have less
2344 // number of frames.
2345 size_t n = 0;
2346 size_t outputDelay = mOutput.lock()->outputDelay;
2347 {
2348 Mutexed<Input>::Locked input(mInput);
2349 n = input->inputDelay + input->pipelineDelay + outputDelay;
2350 }
2351 return mPipelineWatcher.lock()->elapsed(PipelineWatcher::Clock::now(), n);
2352 }
2353
setMetaMode(MetaMode mode)2354 void CCodecBufferChannel::setMetaMode(MetaMode mode) {
2355 mMetaMode = mode;
2356 }
2357
setCrypto(const sp<ICrypto> & crypto)2358 void CCodecBufferChannel::setCrypto(const sp<ICrypto> &crypto) {
2359 if (mCrypto != nullptr) {
2360 for (std::pair<wp<HidlMemory>, int32_t> entry : mHeapSeqNumMap) {
2361 mCrypto->unsetHeap(entry.second);
2362 }
2363 mHeapSeqNumMap.clear();
2364 if (mHeapSeqNum >= 0) {
2365 mCrypto->unsetHeap(mHeapSeqNum);
2366 mHeapSeqNum = -1;
2367 }
2368 }
2369 mCrypto = crypto;
2370 }
2371
setDescrambler(const sp<IDescrambler> & descrambler)2372 void CCodecBufferChannel::setDescrambler(const sp<IDescrambler> &descrambler) {
2373 mDescrambler = descrambler;
2374 }
2375
getBuffersPixelFormat(bool isEncoder)2376 uint32_t CCodecBufferChannel::getBuffersPixelFormat(bool isEncoder) {
2377 if (isEncoder) {
2378 return getInputBuffersPixelFormat();
2379 } else {
2380 return getOutputBuffersPixelFormat();
2381 }
2382 }
2383
getInputBuffersPixelFormat()2384 uint32_t CCodecBufferChannel::getInputBuffersPixelFormat() {
2385 Mutexed<Input>::Locked input(mInput);
2386 if (input->buffers == nullptr) {
2387 return PIXEL_FORMAT_UNKNOWN;
2388 }
2389 return input->buffers->getPixelFormatIfApplicable();
2390 }
2391
getOutputBuffersPixelFormat()2392 uint32_t CCodecBufferChannel::getOutputBuffersPixelFormat() {
2393 Mutexed<Output>::Locked output(mOutput);
2394 if (output->buffers == nullptr) {
2395 return PIXEL_FORMAT_UNKNOWN;
2396 }
2397 return output->buffers->getPixelFormatIfApplicable();
2398 }
2399
resetBuffersPixelFormat(bool isEncoder)2400 void CCodecBufferChannel::resetBuffersPixelFormat(bool isEncoder) {
2401 if (isEncoder) {
2402 Mutexed<Input>::Locked input(mInput);
2403 if (input->buffers == nullptr) {
2404 return;
2405 }
2406 input->buffers->resetPixelFormatIfApplicable();
2407 } else {
2408 Mutexed<Output>::Locked output(mOutput);
2409 if (output->buffers == nullptr) {
2410 return;
2411 }
2412 output->buffers->resetPixelFormatIfApplicable();
2413 }
2414 }
2415
toStatusT(c2_status_t c2s,c2_operation_t c2op)2416 status_t toStatusT(c2_status_t c2s, c2_operation_t c2op) {
2417 // C2_OK is always translated to OK.
2418 if (c2s == C2_OK) {
2419 return OK;
2420 }
2421
2422 // Operation-dependent translation
2423 // TODO: Add as necessary
2424 switch (c2op) {
2425 case C2_OPERATION_Component_start:
2426 switch (c2s) {
2427 case C2_NO_MEMORY:
2428 return NO_MEMORY;
2429 default:
2430 return UNKNOWN_ERROR;
2431 }
2432 default:
2433 break;
2434 }
2435
2436 // Backup operation-agnostic translation
2437 switch (c2s) {
2438 case C2_BAD_INDEX:
2439 return BAD_INDEX;
2440 case C2_BAD_VALUE:
2441 return BAD_VALUE;
2442 case C2_BLOCKING:
2443 return WOULD_BLOCK;
2444 case C2_DUPLICATE:
2445 return ALREADY_EXISTS;
2446 case C2_NO_INIT:
2447 return NO_INIT;
2448 case C2_NO_MEMORY:
2449 return NO_MEMORY;
2450 case C2_NOT_FOUND:
2451 return NAME_NOT_FOUND;
2452 case C2_TIMED_OUT:
2453 return TIMED_OUT;
2454 case C2_BAD_STATE:
2455 case C2_CANCELED:
2456 case C2_CANNOT_DO:
2457 case C2_CORRUPTED:
2458 case C2_OMITTED:
2459 case C2_REFUSED:
2460 return UNKNOWN_ERROR;
2461 default:
2462 return -static_cast<status_t>(c2s);
2463 }
2464 }
2465
2466 } // namespace android
2467