• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 // Copyright 2017 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4 
5 //#define LOG_NDEBUG 0
6 #define LOG_TAG "C2VDAComponent"
7 
8 #ifdef V4L2_CODEC2_ARC
9 #include <C2VDAAdaptorProxy.h>
10 #else
11 #include <C2VDAAdaptor.h>
12 #endif
13 
14 #define __C2_GENERATE_GLOBAL_VARS__
15 #include <C2VDAAllocatorStore.h>
16 #include <C2VDAComponent.h>
17 #include <C2VDAPixelFormat.h>
18 #include <C2VDASupport.h>  // to getParamReflector from vda store
19 #include <C2VdaBqBlockPool.h>
20 #include <C2VdaPooledBlockPool.h>
21 
22 #include <h264_parser.h>
23 
24 #include <C2AllocatorGralloc.h>
25 #include <C2ComponentFactory.h>
26 #include <C2PlatformSupport.h>
27 #include <Codec2Mapper.h>
28 
29 #include <base/bind.h>
30 #include <base/bind_helpers.h>
31 
32 #include <media/stagefright/MediaDefs.h>
33 #include <media/stagefright/foundation/ColorUtils.h>
34 #include <utils/Log.h>
35 #include <utils/misc.h>
36 
37 #include <inttypes.h>
38 #include <string.h>
39 #include <algorithm>
40 #include <string>
41 
42 #define UNUSED(expr)  \
43     do {              \
44         (void)(expr); \
45     } while (0)
46 
47 namespace android {
48 
49 namespace {
50 
51 // Mask against 30 bits to avoid (undefined) wraparound on signed integer.
frameIndexToBitstreamId(c2_cntr64_t frameIndex)52 int32_t frameIndexToBitstreamId(c2_cntr64_t frameIndex) {
53     return static_cast<int32_t>(frameIndex.peeku() & 0x3FFFFFFF);
54 }
55 
56 // Use basic graphic block pool/allocator as default.
57 const C2BlockPool::local_id_t kDefaultOutputBlockPool = C2BlockPool::BASIC_GRAPHIC;
58 
59 const C2String kH264DecoderName = "c2.vda.avc.decoder";
60 const C2String kVP8DecoderName = "c2.vda.vp8.decoder";
61 const C2String kVP9DecoderName = "c2.vda.vp9.decoder";
62 const C2String kH264SecureDecoderName = "c2.vda.avc.decoder.secure";
63 const C2String kVP8SecureDecoderName = "c2.vda.vp8.decoder.secure";
64 const C2String kVP9SecureDecoderName = "c2.vda.vp9.decoder.secure";
65 
66 const uint32_t kDpbOutputBufferExtraCount = 3;  // Use the same number as ACodec.
67 const int kDequeueRetryDelayUs = 10000;  // Wait time of dequeue buffer retry in microseconds.
68 const int32_t kAllocateBufferMaxRetries = 10;  // Max retry time for fetchGraphicBlock timeout.
69 }  // namespace
70 
adaptorResultToC2Status(VideoDecodeAcceleratorAdaptor::Result result)71 static c2_status_t adaptorResultToC2Status(VideoDecodeAcceleratorAdaptor::Result result) {
72     switch (result) {
73     case VideoDecodeAcceleratorAdaptor::Result::SUCCESS:
74         return C2_OK;
75     case VideoDecodeAcceleratorAdaptor::Result::ILLEGAL_STATE:
76         ALOGE("Got error: ILLEGAL_STATE");
77         return C2_BAD_STATE;
78     case VideoDecodeAcceleratorAdaptor::Result::INVALID_ARGUMENT:
79         ALOGE("Got error: INVALID_ARGUMENT");
80         return C2_BAD_VALUE;
81     case VideoDecodeAcceleratorAdaptor::Result::UNREADABLE_INPUT:
82         ALOGE("Got error: UNREADABLE_INPUT");
83         return C2_BAD_VALUE;
84     case VideoDecodeAcceleratorAdaptor::Result::PLATFORM_FAILURE:
85         ALOGE("Got error: PLATFORM_FAILURE");
86         return C2_CORRUPTED;
87     case VideoDecodeAcceleratorAdaptor::Result::INSUFFICIENT_RESOURCES:
88         ALOGE("Got error: INSUFFICIENT_RESOURCES");
89         return C2_NO_MEMORY;
90     default:
91         ALOGE("Unrecognizable adaptor result (value = %d)...", result);
92         return C2_CORRUPTED;
93     }
94 }
95 
96 // static
ProfileLevelSetter(bool mayBlock,C2P<C2StreamProfileLevelInfo::input> & info)97 C2R C2VDAComponent::IntfImpl::ProfileLevelSetter(bool mayBlock,
98                                                  C2P<C2StreamProfileLevelInfo::input>& info) {
99     (void)mayBlock;
100     return info.F(info.v.profile)
101             .validatePossible(info.v.profile)
102             .plus(info.F(info.v.level).validatePossible(info.v.level));
103 }
104 
105 // static
SizeSetter(bool mayBlock,C2P<C2StreamPictureSizeInfo::output> & videoSize)106 C2R C2VDAComponent::IntfImpl::SizeSetter(bool mayBlock,
107                                          C2P<C2StreamPictureSizeInfo::output>& videoSize) {
108     (void)mayBlock;
109     // TODO: maybe apply block limit?
110     return videoSize.F(videoSize.v.width)
111             .validatePossible(videoSize.v.width)
112             .plus(videoSize.F(videoSize.v.height).validatePossible(videoSize.v.height));
113 }
114 
115 // static
116 template <typename T>
DefaultColorAspectsSetter(bool mayBlock,C2P<T> & def)117 C2R C2VDAComponent::IntfImpl::DefaultColorAspectsSetter(bool mayBlock, C2P<T>& def) {
118     (void)mayBlock;
119     if (def.v.range > C2Color::RANGE_OTHER) {
120         def.set().range = C2Color::RANGE_OTHER;
121     }
122     if (def.v.primaries > C2Color::PRIMARIES_OTHER) {
123         def.set().primaries = C2Color::PRIMARIES_OTHER;
124     }
125     if (def.v.transfer > C2Color::TRANSFER_OTHER) {
126         def.set().transfer = C2Color::TRANSFER_OTHER;
127     }
128     if (def.v.matrix > C2Color::MATRIX_OTHER) {
129         def.set().matrix = C2Color::MATRIX_OTHER;
130     }
131     return C2R::Ok();
132 }
133 
134 // static
MergedColorAspectsSetter(bool mayBlock,C2P<C2StreamColorAspectsInfo::output> & merged,const C2P<C2StreamColorAspectsTuning::output> & def,const C2P<C2StreamColorAspectsInfo::input> & coded)135 C2R C2VDAComponent::IntfImpl::MergedColorAspectsSetter(
136         bool mayBlock, C2P<C2StreamColorAspectsInfo::output>& merged,
137         const C2P<C2StreamColorAspectsTuning::output>& def,
138         const C2P<C2StreamColorAspectsInfo::input>& coded) {
139     (void)mayBlock;
140     // Take coded values for all specified fields, and default values for unspecified ones.
141     merged.set().range = coded.v.range == RANGE_UNSPECIFIED ? def.v.range : coded.v.range;
142     merged.set().primaries =
143             coded.v.primaries == PRIMARIES_UNSPECIFIED ? def.v.primaries : coded.v.primaries;
144     merged.set().transfer =
145             coded.v.transfer == TRANSFER_UNSPECIFIED ? def.v.transfer : coded.v.transfer;
146     merged.set().matrix = coded.v.matrix == MATRIX_UNSPECIFIED ? def.v.matrix : coded.v.matrix;
147     return C2R::Ok();
148 }
149 
IntfImpl(C2String name,const std::shared_ptr<C2ReflectorHelper> & helper)150 C2VDAComponent::IntfImpl::IntfImpl(C2String name, const std::shared_ptr<C2ReflectorHelper>& helper)
151       : C2InterfaceHelper(helper), mInitStatus(C2_OK) {
152     setDerivedInstance(this);
153 
154     // TODO(johnylin): use factory function to determine whether V4L2 stream or slice API is.
155     char inputMime[128];
156     if (name == kH264DecoderName || name == kH264SecureDecoderName) {
157         strcpy(inputMime, MEDIA_MIMETYPE_VIDEO_AVC);
158         mInputCodec = InputCodec::H264;
159         addParameter(
160                 DefineParam(mProfileLevel, C2_PARAMKEY_PROFILE_LEVEL)
161                         .withDefault(new C2StreamProfileLevelInfo::input(
162                                 0u, C2Config::PROFILE_AVC_MAIN, C2Config::LEVEL_AVC_4))
163                         .withFields(
164                                 {C2F(mProfileLevel, profile)
165                                          .oneOf({C2Config::PROFILE_AVC_BASELINE,
166                                                  C2Config::PROFILE_AVC_CONSTRAINED_BASELINE,
167                                                  C2Config::PROFILE_AVC_MAIN,
168                                                  C2Config::PROFILE_AVC_HIGH,
169                                                  C2Config::PROFILE_AVC_CONSTRAINED_HIGH}),
170                                  C2F(mProfileLevel, level)
171                                          .oneOf({C2Config::LEVEL_AVC_1, C2Config::LEVEL_AVC_1B,
172                                                  C2Config::LEVEL_AVC_1_1, C2Config::LEVEL_AVC_1_2,
173                                                  C2Config::LEVEL_AVC_1_3, C2Config::LEVEL_AVC_2,
174                                                  C2Config::LEVEL_AVC_2_1, C2Config::LEVEL_AVC_2_2,
175                                                  C2Config::LEVEL_AVC_3, C2Config::LEVEL_AVC_3_1,
176                                                  C2Config::LEVEL_AVC_3_2, C2Config::LEVEL_AVC_4,
177                                                  C2Config::LEVEL_AVC_4_1, C2Config::LEVEL_AVC_4_2,
178                                                  C2Config::LEVEL_AVC_5, C2Config::LEVEL_AVC_5_1,
179                                                  C2Config::LEVEL_AVC_5_2})})
180                         .withSetter(ProfileLevelSetter)
181                         .build());
182     } else if (name == kVP8DecoderName || name == kVP8SecureDecoderName) {
183         strcpy(inputMime, MEDIA_MIMETYPE_VIDEO_VP8);
184         mInputCodec = InputCodec::VP8;
185         addParameter(DefineParam(mProfileLevel, C2_PARAMKEY_PROFILE_LEVEL)
186                              .withConstValue(new C2StreamProfileLevelInfo::input(
187                                      0u, C2Config::PROFILE_UNUSED, C2Config::LEVEL_UNUSED))
188                              .build());
189     } else if (name == kVP9DecoderName || name == kVP9SecureDecoderName) {
190         strcpy(inputMime, MEDIA_MIMETYPE_VIDEO_VP9);
191         mInputCodec = InputCodec::VP9;
192         addParameter(
193                 DefineParam(mProfileLevel, C2_PARAMKEY_PROFILE_LEVEL)
194                         .withDefault(new C2StreamProfileLevelInfo::input(
195                                 0u, C2Config::PROFILE_VP9_0, C2Config::LEVEL_VP9_5))
196                         .withFields({C2F(mProfileLevel, profile).oneOf({C2Config::PROFILE_VP9_0}),
197                                      C2F(mProfileLevel, level)
198                                              .oneOf({C2Config::LEVEL_VP9_1, C2Config::LEVEL_VP9_1_1,
199                                                      C2Config::LEVEL_VP9_2, C2Config::LEVEL_VP9_2_1,
200                                                      C2Config::LEVEL_VP9_3, C2Config::LEVEL_VP9_3_1,
201                                                      C2Config::LEVEL_VP9_4, C2Config::LEVEL_VP9_4_1,
202                                                      C2Config::LEVEL_VP9_5})})
203                         .withSetter(ProfileLevelSetter)
204                         .build());
205     } else {
206         ALOGE("Invalid component name: %s", name.c_str());
207         mInitStatus = C2_BAD_VALUE;
208         return;
209     }
210     // Get supported profiles from VDA.
211     // TODO: re-think the suitable method of getting supported profiles for both pure Android and
212     //       ARC++.
213     media::VideoDecodeAccelerator::SupportedProfiles supportedProfiles;
214 #ifdef V4L2_CODEC2_ARC
215     supportedProfiles = arc::C2VDAAdaptorProxy::GetSupportedProfiles(mInputCodec);
216 #else
217     supportedProfiles = C2VDAAdaptor::GetSupportedProfiles(mInputCodec);
218 #endif
219     if (supportedProfiles.empty()) {
220         ALOGE("No supported profile from input codec: %d", mInputCodec);
221         mInitStatus = C2_BAD_VALUE;
222         return;
223     }
224 
225     mCodecProfile = supportedProfiles[0].profile;
226 
227     auto minSize = supportedProfiles[0].min_resolution;
228     auto maxSize = supportedProfiles[0].max_resolution;
229 
230     addParameter(
231             DefineParam(mInputFormat, C2_PARAMKEY_INPUT_STREAM_BUFFER_TYPE)
232                     .withConstValue(new C2StreamBufferTypeSetting::input(0u, C2FormatCompressed))
233                     .build());
234 
235     addParameter(DefineParam(mOutputFormat, C2_PARAMKEY_OUTPUT_STREAM_BUFFER_TYPE)
236                          .withConstValue(new C2StreamBufferTypeSetting::output(0u, C2FormatVideo))
237                          .build());
238 
239     addParameter(
240             DefineParam(mInputMediaType, C2_PARAMKEY_INPUT_MEDIA_TYPE)
241                     .withConstValue(AllocSharedString<C2PortMediaTypeSetting::input>(inputMime))
242                     .build());
243 
244     addParameter(DefineParam(mOutputMediaType, C2_PARAMKEY_OUTPUT_MEDIA_TYPE)
245                          .withConstValue(AllocSharedString<C2PortMediaTypeSetting::output>(
246                                  MEDIA_MIMETYPE_VIDEO_RAW))
247                          .build());
248 
249     addParameter(DefineParam(mSize, C2_PARAMKEY_STREAM_PICTURE_SIZE)
250                          .withDefault(new C2StreamPictureSizeInfo::output(0u, 176, 144))
251                          .withFields({
252                                  C2F(mSize, width).inRange(minSize.width(), maxSize.width(), 16),
253                                  C2F(mSize, height).inRange(minSize.height(), maxSize.height(), 16),
254                          })
255                          .withSetter(SizeSetter)
256                          .build());
257 
258     // App may set a smaller value for maximum of input buffer size than actually required
259     // by mistake. C2VDAComponent overrides it if the value specified by app is smaller than
260     // the calculated value in MaxSizeCalculator().
261     // This value is the default maximum of linear buffer size (kLinearBufferSize) in
262     // CCodecBufferChannel.cpp.
263     constexpr static size_t kLinearBufferSize = 1048576;
264     struct LocalCalculator {
265         static C2R MaxSizeCalculator(bool mayBlock, C2P<C2StreamMaxBufferSizeInfo::input>& me,
266                                      const C2P<C2StreamPictureSizeInfo::output>& size) {
267             (void)mayBlock;
268             // TODO: Need larger size?
269             me.set().value = kLinearBufferSize;
270             const uint32_t width = size.v.width;
271             const uint32_t height = size.v.height;
272             // Enlarge the input buffer for 4k video
273             if ((width > 1920 && height > 1080)) {
274                 me.set().value = 4 * kLinearBufferSize;
275             }
276             return C2R::Ok();
277         }
278     };
279     addParameter(DefineParam(mMaxInputSize, C2_PARAMKEY_INPUT_MAX_BUFFER_SIZE)
280                          .withDefault(new C2StreamMaxBufferSizeInfo::input(0u, kLinearBufferSize))
281                          .withFields({
282                                  C2F(mMaxInputSize, value).any(),
283                          })
284                          .calculatedAs(LocalCalculator::MaxSizeCalculator, mSize)
285                          .build());
286 
287     bool secureMode = name.find(".secure") != std::string::npos;
288     C2Allocator::id_t inputAllocators[] = {secureMode ? C2VDAAllocatorStore::SECURE_LINEAR
289                                                       : C2PlatformAllocatorStore::ION};
290 
291     C2Allocator::id_t outputAllocators[] = {C2VDAAllocatorStore::V4L2_BUFFERPOOL};
292 
293     C2Allocator::id_t surfaceAllocator = secureMode ? C2VDAAllocatorStore::SECURE_GRAPHIC
294                                                     : C2VDAAllocatorStore::V4L2_BUFFERQUEUE;
295 
296     addParameter(
297             DefineParam(mInputAllocatorIds, C2_PARAMKEY_INPUT_ALLOCATORS)
298                     .withConstValue(C2PortAllocatorsTuning::input::AllocShared(inputAllocators))
299                     .build());
300 
301     addParameter(
302             DefineParam(mOutputAllocatorIds, C2_PARAMKEY_OUTPUT_ALLOCATORS)
303                     .withConstValue(C2PortAllocatorsTuning::output::AllocShared(outputAllocators))
304                     .build());
305 
306     addParameter(DefineParam(mOutputSurfaceAllocatorId, C2_PARAMKEY_OUTPUT_SURFACE_ALLOCATOR)
307                          .withConstValue(new C2PortSurfaceAllocatorTuning::output(surfaceAllocator))
308                          .build());
309 
310     C2BlockPool::local_id_t outputBlockPools[] = {kDefaultOutputBlockPool};
311 
312     addParameter(
313             DefineParam(mOutputBlockPoolIds, C2_PARAMKEY_OUTPUT_BLOCK_POOLS)
314                     .withDefault(C2PortBlockPoolsTuning::output::AllocShared(outputBlockPools))
315                     .withFields({C2F(mOutputBlockPoolIds, m.values[0]).any(),
316                                  C2F(mOutputBlockPoolIds, m.values).inRange(0, 1)})
317                     .withSetter(Setter<C2PortBlockPoolsTuning::output>::NonStrictValuesWithNoDeps)
318                     .build());
319 
320     addParameter(
321             DefineParam(mDefaultColorAspects, C2_PARAMKEY_DEFAULT_COLOR_ASPECTS)
322                     .withDefault(new C2StreamColorAspectsTuning::output(
323                             0u, C2Color::RANGE_UNSPECIFIED, C2Color::PRIMARIES_UNSPECIFIED,
324                             C2Color::TRANSFER_UNSPECIFIED, C2Color::MATRIX_UNSPECIFIED))
325                     .withFields(
326                             {C2F(mDefaultColorAspects, range)
327                                      .inRange(C2Color::RANGE_UNSPECIFIED, C2Color::RANGE_OTHER),
328                              C2F(mDefaultColorAspects, primaries)
329                                      .inRange(C2Color::PRIMARIES_UNSPECIFIED,
330                                               C2Color::PRIMARIES_OTHER),
331                              C2F(mDefaultColorAspects, transfer)
332                                      .inRange(C2Color::TRANSFER_UNSPECIFIED,
333                                               C2Color::TRANSFER_OTHER),
334                              C2F(mDefaultColorAspects, matrix)
335                                      .inRange(C2Color::MATRIX_UNSPECIFIED, C2Color::MATRIX_OTHER)})
336                     .withSetter(DefaultColorAspectsSetter)
337                     .build());
338 
339     addParameter(
340             DefineParam(mCodedColorAspects, C2_PARAMKEY_VUI_COLOR_ASPECTS)
341                     .withDefault(new C2StreamColorAspectsInfo::input(
342                             0u, C2Color::RANGE_LIMITED, C2Color::PRIMARIES_UNSPECIFIED,
343                             C2Color::TRANSFER_UNSPECIFIED, C2Color::MATRIX_UNSPECIFIED))
344                     .withFields(
345                             {C2F(mCodedColorAspects, range)
346                                      .inRange(C2Color::RANGE_UNSPECIFIED, C2Color::RANGE_OTHER),
347                              C2F(mCodedColorAspects, primaries)
348                                      .inRange(C2Color::PRIMARIES_UNSPECIFIED,
349                                               C2Color::PRIMARIES_OTHER),
350                              C2F(mCodedColorAspects, transfer)
351                                      .inRange(C2Color::TRANSFER_UNSPECIFIED,
352                                               C2Color::TRANSFER_OTHER),
353                              C2F(mCodedColorAspects, matrix)
354                                      .inRange(C2Color::MATRIX_UNSPECIFIED, C2Color::MATRIX_OTHER)})
355                     .withSetter(DefaultColorAspectsSetter)
356                     .build());
357 
358     addParameter(
359             DefineParam(mColorAspects, C2_PARAMKEY_COLOR_ASPECTS)
360                     .withDefault(new C2StreamColorAspectsInfo::output(
361                             0u, C2Color::RANGE_UNSPECIFIED, C2Color::PRIMARIES_UNSPECIFIED,
362                             C2Color::TRANSFER_UNSPECIFIED, C2Color::MATRIX_UNSPECIFIED))
363                     .withFields(
364                             {C2F(mColorAspects, range)
365                                      .inRange(C2Color::RANGE_UNSPECIFIED, C2Color::RANGE_OTHER),
366                              C2F(mColorAspects, primaries)
367                                      .inRange(C2Color::PRIMARIES_UNSPECIFIED,
368                                               C2Color::PRIMARIES_OTHER),
369                              C2F(mColorAspects, transfer)
370                                      .inRange(C2Color::TRANSFER_UNSPECIFIED,
371                                               C2Color::TRANSFER_OTHER),
372                              C2F(mColorAspects, matrix)
373                                      .inRange(C2Color::MATRIX_UNSPECIFIED, C2Color::MATRIX_OTHER)})
374                     .withSetter(MergedColorAspectsSetter, mDefaultColorAspects, mCodedColorAspects)
375                     .build());
376 }
377 
378 ////////////////////////////////////////////////////////////////////////////////
379 #define EXPECT_STATE_OR_RETURN_ON_ERROR(x)                    \
380     do {                                                      \
381         if (mComponentState == ComponentState::ERROR) return; \
382         CHECK_EQ(mComponentState, ComponentState::x);         \
383     } while (0)
384 
385 #define EXPECT_RUNNING_OR_RETURN_ON_ERROR()                       \
386     do {                                                          \
387         if (mComponentState == ComponentState::ERROR) return;     \
388         CHECK_NE(mComponentState, ComponentState::UNINITIALIZED); \
389     } while (0)
390 
VideoFormat(HalPixelFormat pixelFormat,uint32_t minNumBuffers,media::Size codedSize,media::Rect visibleRect)391 C2VDAComponent::VideoFormat::VideoFormat(HalPixelFormat pixelFormat, uint32_t minNumBuffers,
392                                          media::Size codedSize, media::Rect visibleRect)
393       : mPixelFormat(pixelFormat),
394         mMinNumBuffers(minNumBuffers),
395         mCodedSize(codedSize),
396         mVisibleRect(visibleRect) {}
397 
C2VDAComponent(C2String name,c2_node_id_t id,const std::shared_ptr<C2ReflectorHelper> & helper)398 C2VDAComponent::C2VDAComponent(C2String name, c2_node_id_t id,
399                                const std::shared_ptr<C2ReflectorHelper>& helper)
400       : mIntfImpl(std::make_shared<IntfImpl>(name, helper)),
401         mIntf(std::make_shared<SimpleInterface<IntfImpl>>(name.c_str(), id, mIntfImpl)),
402         mThread("C2VDAComponentThread"),
403         mDequeueThread("C2VDAComponentDequeueThread"),
404         mVDAInitResult(VideoDecodeAcceleratorAdaptor::Result::ILLEGAL_STATE),
405         mComponentState(ComponentState::UNINITIALIZED),
406         mPendingOutputEOS(false),
407         mPendingColorAspectsChange(false),
408         mPendingColorAspectsChangeFrameIndex(0),
409         mCodecProfile(media::VIDEO_CODEC_PROFILE_UNKNOWN),
410         mState(State::UNLOADED),
411         mWeakThisFactory(this) {
412     // TODO(johnylin): the client may need to know if init is failed.
413     if (mIntfImpl->status() != C2_OK) {
414         ALOGE("Component interface init failed (err code = %d)", mIntfImpl->status());
415         return;
416     }
417 
418     mSecureMode = name.find(".secure") != std::string::npos;
419     if (!mThread.Start()) {
420         ALOGE("Component thread failed to start.");
421         return;
422     }
423     mTaskRunner = mThread.task_runner();
424     mState.store(State::LOADED);
425 }
426 
~C2VDAComponent()427 C2VDAComponent::~C2VDAComponent() {
428     if (mThread.IsRunning()) {
429         mTaskRunner->PostTask(FROM_HERE,
430                               ::base::Bind(&C2VDAComponent::onDestroy, ::base::Unretained(this)));
431         mThread.Stop();
432     }
433 }
434 
onDestroy()435 void C2VDAComponent::onDestroy() {
436     DCHECK(mTaskRunner->BelongsToCurrentThread());
437     ALOGV("onDestroy");
438     if (mVDAAdaptor.get()) {
439         mVDAAdaptor->destroy();
440         mVDAAdaptor.reset(nullptr);
441     }
442     stopDequeueThread();
443 }
444 
onStart(media::VideoCodecProfile profile,::base::WaitableEvent * done)445 void C2VDAComponent::onStart(media::VideoCodecProfile profile, ::base::WaitableEvent* done) {
446     DCHECK(mTaskRunner->BelongsToCurrentThread());
447     ALOGV("onStart");
448     CHECK_EQ(mComponentState, ComponentState::UNINITIALIZED);
449 
450 #ifdef V4L2_CODEC2_ARC
451     mVDAAdaptor.reset(new arc::C2VDAAdaptorProxy());
452 #else
453     mVDAAdaptor.reset(new C2VDAAdaptor());
454 #endif
455 
456     mVDAInitResult = mVDAAdaptor->initialize(profile, mSecureMode, this);
457     if (mVDAInitResult == VideoDecodeAcceleratorAdaptor::Result::SUCCESS) {
458         mComponentState = ComponentState::STARTED;
459     }
460 
461     if (!mSecureMode && mIntfImpl->getInputCodec() == InputCodec::H264) {
462         // Get default color aspects on start.
463         updateColorAspects();
464         mPendingColorAspectsChange = false;
465     }
466 
467     done->Signal();
468 }
469 
onQueueWork(std::unique_ptr<C2Work> work)470 void C2VDAComponent::onQueueWork(std::unique_ptr<C2Work> work) {
471     DCHECK(mTaskRunner->BelongsToCurrentThread());
472     ALOGV("onQueueWork: flags=0x%x, index=%llu, timestamp=%llu", work->input.flags,
473           work->input.ordinal.frameIndex.peekull(), work->input.ordinal.timestamp.peekull());
474     EXPECT_RUNNING_OR_RETURN_ON_ERROR();
475 
476     uint32_t drainMode = NO_DRAIN;
477     if (work->input.flags & C2FrameData::FLAG_END_OF_STREAM) {
478         drainMode = DRAIN_COMPONENT_WITH_EOS;
479     }
480     mQueue.push({std::move(work), drainMode});
481     // TODO(johnylin): set a maximum size of mQueue and check if mQueue is already full.
482 
483     mTaskRunner->PostTask(FROM_HERE,
484                           ::base::Bind(&C2VDAComponent::onDequeueWork, ::base::Unretained(this)));
485 }
486 
onDequeueWork()487 void C2VDAComponent::onDequeueWork() {
488     DCHECK(mTaskRunner->BelongsToCurrentThread());
489     ALOGV("onDequeueWork");
490     EXPECT_RUNNING_OR_RETURN_ON_ERROR();
491     if (mQueue.empty()) {
492         return;
493     }
494     if (mComponentState == ComponentState::DRAINING ||
495         mComponentState == ComponentState::FLUSHING) {
496         ALOGV("Temporarily stop dequeueing works since component is draining/flushing.");
497         return;
498     }
499     if (mComponentState != ComponentState::STARTED) {
500         ALOGE("Work queue should be empty if the component is not in STARTED state.");
501         return;
502     }
503 
504     // Dequeue a work from mQueue.
505     std::unique_ptr<C2Work> work(std::move(mQueue.front().mWork));
506     auto drainMode = mQueue.front().mDrainMode;
507     mQueue.pop();
508 
509     CHECK_LE(work->input.buffers.size(), 1u);
510     bool isEmptyCSDWork = false;
511     // Use frameIndex as bitstreamId.
512     int32_t bitstreamId = frameIndexToBitstreamId(work->input.ordinal.frameIndex);
513     if (work->input.buffers.empty()) {
514         // Client may queue a work with no input buffer for either it's EOS or empty CSD, otherwise
515         // every work must have one input buffer.
516         isEmptyCSDWork = work->input.flags & C2FrameData::FLAG_CODEC_CONFIG;
517         CHECK(drainMode != NO_DRAIN || isEmptyCSDWork);
518         // Emplace a nullptr to unify the check for work done.
519         ALOGV("Got a work with no input buffer! Emplace a nullptr inside.");
520         work->input.buffers.emplace_back(nullptr);
521     } else {
522         // If input.buffers is not empty, the buffer should have meaningful content inside.
523         C2ConstLinearBlock linearBlock = work->input.buffers.front()->data().linearBlocks().front();
524         CHECK_GT(linearBlock.size(), 0u);
525 
526         // Call parseCodedColorAspects() to try to parse color aspects from bitstream only if:
527         // 1) This is non-secure decoding.
528         // 2) This is H264 codec.
529         // 3) This input is CSD buffer (with flags FLAG_CODEC_CONFIG).
530         if (!mSecureMode && (mIntfImpl->getInputCodec() == InputCodec::H264) &&
531             (work->input.flags & C2FrameData::FLAG_CODEC_CONFIG)) {
532             if (parseCodedColorAspects(linearBlock)) {
533                 // Record current frame index, color aspects should be updated only for output
534                 // buffers whose frame indices are not less than this one.
535                 mPendingColorAspectsChange = true;
536                 mPendingColorAspectsChangeFrameIndex = work->input.ordinal.frameIndex.peeku();
537             }
538         }
539         // Send input buffer to VDA for decode.
540         sendInputBufferToAccelerator(linearBlock, bitstreamId);
541     }
542 
543     CHECK_EQ(work->worklets.size(), 1u);
544     work->worklets.front()->output.flags = static_cast<C2FrameData::flags_t>(0);
545     work->worklets.front()->output.buffers.clear();
546     work->worklets.front()->output.ordinal = work->input.ordinal;
547 
548     if (drainMode != NO_DRAIN) {
549         mVDAAdaptor->flush();
550         mComponentState = ComponentState::DRAINING;
551         mPendingOutputEOS = drainMode == DRAIN_COMPONENT_WITH_EOS;
552     }
553 
554     // Put work to mPendingWorks.
555     mPendingWorks.emplace_back(std::move(work));
556     if (isEmptyCSDWork) {
557         // Directly report the empty CSD work as finished.
558         reportWorkIfFinished(bitstreamId);
559     }
560 
561     if (!mQueue.empty()) {
562         mTaskRunner->PostTask(FROM_HERE, ::base::Bind(&C2VDAComponent::onDequeueWork,
563                                                       ::base::Unretained(this)));
564     }
565 }
566 
onInputBufferDone(int32_t bitstreamId)567 void C2VDAComponent::onInputBufferDone(int32_t bitstreamId) {
568     DCHECK(mTaskRunner->BelongsToCurrentThread());
569     ALOGV("onInputBufferDone: bitstream id=%d", bitstreamId);
570     EXPECT_RUNNING_OR_RETURN_ON_ERROR();
571 
572     C2Work* work = getPendingWorkByBitstreamId(bitstreamId);
573     if (!work) {
574         reportError(C2_CORRUPTED);
575         return;
576     }
577 
578     // When the work is done, the input buffer shall be reset by component.
579     work->input.buffers.front().reset();
580 
581     reportWorkIfFinished(bitstreamId);
582 }
583 
onOutputBufferReturned(std::shared_ptr<C2GraphicBlock> block,uint32_t poolId)584 void C2VDAComponent::onOutputBufferReturned(std::shared_ptr<C2GraphicBlock> block,
585                                             uint32_t poolId) {
586     DCHECK(mTaskRunner->BelongsToCurrentThread());
587     ALOGV("onOutputBufferReturned: pool id=%u", poolId);
588     if (mComponentState == ComponentState::UNINITIALIZED) {
589         // Output buffer is returned from client after component is stopped. Just let the buffer be
590         // released.
591         return;
592     }
593 
594     if (block->width() != static_cast<uint32_t>(mOutputFormat.mCodedSize.width()) ||
595         block->height() != static_cast<uint32_t>(mOutputFormat.mCodedSize.height())) {
596         // Output buffer is returned after we changed output resolution. Just let the buffer be
597         // released.
598         ALOGV("Discard obsolete graphic block: pool id=%u", poolId);
599         return;
600     }
601 
602     GraphicBlockInfo* info = getGraphicBlockByPoolId(poolId);
603     if (!info) {
604         reportError(C2_CORRUPTED);
605         return;
606     }
607     CHECK_EQ(info->mState, GraphicBlockInfo::State::OWNED_BY_CLIENT);
608     info->mGraphicBlock = std::move(block);
609     info->mState = GraphicBlockInfo::State::OWNED_BY_COMPONENT;
610 
611     if (mPendingOutputFormat) {
612         tryChangeOutputFormat();
613     } else {
614         // Do not pass the ownership to accelerator if this buffer will still be reused under
615         // |mPendingBuffersToWork|.
616         auto existingFrame = std::find_if(
617                 mPendingBuffersToWork.begin(), mPendingBuffersToWork.end(),
618                 [id = info->mBlockId](const OutputBufferInfo& o) { return o.mBlockId == id; });
619         bool ownByAccelerator = existingFrame == mPendingBuffersToWork.end();
620         sendOutputBufferToAccelerator(info, ownByAccelerator);
621         sendOutputBufferToWorkIfAny(false /* dropIfUnavailable */);
622     }
623 }
624 
onOutputBufferDone(int32_t pictureBufferId,int32_t bitstreamId)625 void C2VDAComponent::onOutputBufferDone(int32_t pictureBufferId, int32_t bitstreamId) {
626     DCHECK(mTaskRunner->BelongsToCurrentThread());
627     ALOGV("onOutputBufferDone: picture id=%d, bitstream id=%d", pictureBufferId, bitstreamId);
628     EXPECT_RUNNING_OR_RETURN_ON_ERROR();
629 
630     GraphicBlockInfo* info = getGraphicBlockById(pictureBufferId);
631     if (!info) {
632         reportError(C2_CORRUPTED);
633         return;
634     }
635 
636     if (info->mState == GraphicBlockInfo::State::OWNED_BY_ACCELERATOR) {
637         info->mState = GraphicBlockInfo::State::OWNED_BY_COMPONENT;
638     }
639     mPendingBuffersToWork.push_back({bitstreamId, pictureBufferId});
640     sendOutputBufferToWorkIfAny(false /* dropIfUnavailable */);
641 }
642 
sendOutputBufferToWorkIfAny(bool dropIfUnavailable)643 void C2VDAComponent::sendOutputBufferToWorkIfAny(bool dropIfUnavailable) {
644     DCHECK(mTaskRunner->BelongsToCurrentThread());
645 
646     while (!mPendingBuffersToWork.empty()) {
647         auto nextBuffer = mPendingBuffersToWork.front();
648         GraphicBlockInfo* info = getGraphicBlockById(nextBuffer.mBlockId);
649         CHECK_NE(info->mState, GraphicBlockInfo::State::OWNED_BY_ACCELERATOR);
650 
651         C2Work* work = getPendingWorkByBitstreamId(nextBuffer.mBitstreamId);
652         if (!work) {
653             reportError(C2_CORRUPTED);
654             return;
655         }
656 
657         if (info->mState == GraphicBlockInfo::State::OWNED_BY_CLIENT) {
658             // This buffer is the existing frame and still owned by client.
659             if (!dropIfUnavailable &&
660                 std::find(mUndequeuedBlockIds.begin(), mUndequeuedBlockIds.end(),
661                           nextBuffer.mBlockId) == mUndequeuedBlockIds.end()) {
662                 ALOGV("Still waiting for existing frame returned from client...");
663                 return;
664             }
665             ALOGV("Drop this frame...");
666             sendOutputBufferToAccelerator(info, false /* ownByAccelerator */);
667             work->worklets.front()->output.flags = C2FrameData::FLAG_DROP_FRAME;
668         } else {
669             // This buffer is ready to push into the corresponding work.
670             // Output buffer will be passed to client soon along with mListener->onWorkDone_nb().
671             info->mState = GraphicBlockInfo::State::OWNED_BY_CLIENT;
672             mBuffersInClient++;
673             updateUndequeuedBlockIds(info->mBlockId);
674 
675             // Attach output buffer to the work corresponded to bitstreamId.
676             C2ConstGraphicBlock constBlock = info->mGraphicBlock->share(
677                     C2Rect(mOutputFormat.mVisibleRect.width(),
678                            mOutputFormat.mVisibleRect.height()),
679                     C2Fence());
680             MarkBlockPoolDataAsShared(constBlock);
681 
682             std::shared_ptr<C2Buffer> buffer = C2Buffer::CreateGraphicBuffer(std::move(constBlock));
683             if (mPendingColorAspectsChange &&
684                 work->input.ordinal.frameIndex.peeku() >= mPendingColorAspectsChangeFrameIndex) {
685                 updateColorAspects();
686                 mPendingColorAspectsChange = false;
687             }
688             if (mCurrentColorAspects) {
689                 buffer->setInfo(mCurrentColorAspects);
690             }
691             work->worklets.front()->output.buffers.emplace_back(std::move(buffer));
692             info->mGraphicBlock.reset();
693         }
694         reportWorkIfFinished(nextBuffer.mBitstreamId);
695         mPendingBuffersToWork.pop_front();
696     }
697 }
698 
updateUndequeuedBlockIds(int32_t blockId)699 void C2VDAComponent::updateUndequeuedBlockIds(int32_t blockId) {
700     // The size of |mUndequedBlockIds| will always be the minimum buffer count for display.
701     mUndequeuedBlockIds.push_back(blockId);
702     mUndequeuedBlockIds.pop_front();
703 }
704 
onDrain(uint32_t drainMode)705 void C2VDAComponent::onDrain(uint32_t drainMode) {
706     DCHECK(mTaskRunner->BelongsToCurrentThread());
707     ALOGV("onDrain: mode = %u", drainMode);
708     EXPECT_RUNNING_OR_RETURN_ON_ERROR();
709 
710     if (!mQueue.empty()) {
711         // Mark last queued work as "drain-till-here" by setting drainMode. Do not change drainMode
712         // if last work already has one.
713         if (mQueue.back().mDrainMode == NO_DRAIN) {
714             mQueue.back().mDrainMode = drainMode;
715         }
716     } else if (!mPendingWorks.empty()) {
717         // Neglect drain request if component is not in STARTED mode. Otherwise, enters DRAINING
718         // mode and signal VDA flush immediately.
719         if (mComponentState == ComponentState::STARTED) {
720             mVDAAdaptor->flush();
721             mComponentState = ComponentState::DRAINING;
722             mPendingOutputEOS = drainMode == DRAIN_COMPONENT_WITH_EOS;
723         } else {
724             ALOGV("Neglect drain. Component in state: %d", mComponentState);
725         }
726     } else {
727         // Do nothing.
728         ALOGV("No buffers in VDA, drain takes no effect.");
729     }
730 }
731 
onDrainDone()732 void C2VDAComponent::onDrainDone() {
733     DCHECK(mTaskRunner->BelongsToCurrentThread());
734     ALOGV("onDrainDone");
735     if (mComponentState == ComponentState::DRAINING) {
736         mComponentState = ComponentState::STARTED;
737     } else if (mComponentState == ComponentState::STOPPING) {
738         // The client signals stop right before VDA notifies drain done. Let stop process goes.
739         return;
740     } else if (mComponentState != ComponentState::FLUSHING) {
741         // It is reasonable to get onDrainDone in FLUSHING, which means flush is already signaled
742         // and component should still expect onFlushDone callback from VDA.
743         ALOGE("Unexpected state while onDrainDone(). State=%d", mComponentState);
744         reportError(C2_BAD_STATE);
745         return;
746     }
747 
748     // Drop all pending existing frames and return all finished works before drain done.
749     sendOutputBufferToWorkIfAny(true /* dropIfUnavailable */);
750     CHECK(mPendingBuffersToWork.empty());
751 
752     if (mPendingOutputEOS) {
753         // Return EOS work.
754         reportEOSWork();
755     }
756     // mPendingWorks must be empty after draining is finished.
757     CHECK(mPendingWorks.empty());
758 
759     // Work dequeueing was stopped while component draining. Restart it.
760     mTaskRunner->PostTask(FROM_HERE,
761                           ::base::Bind(&C2VDAComponent::onDequeueWork, ::base::Unretained(this)));
762 }
763 
onFlush()764 void C2VDAComponent::onFlush() {
765     DCHECK(mTaskRunner->BelongsToCurrentThread());
766     ALOGV("onFlush");
767     if (mComponentState == ComponentState::FLUSHING ||
768         mComponentState == ComponentState::STOPPING) {
769         return;  // Ignore other flush request when component is flushing or stopping.
770     }
771     EXPECT_RUNNING_OR_RETURN_ON_ERROR();
772 
773     mVDAAdaptor->reset();
774     // Pop all works in mQueue and put into mAbandonedWorks.
775     while (!mQueue.empty()) {
776         mAbandonedWorks.emplace_back(std::move(mQueue.front().mWork));
777         mQueue.pop();
778     }
779     mComponentState = ComponentState::FLUSHING;
780 }
781 
onStop(::base::WaitableEvent * done)782 void C2VDAComponent::onStop(::base::WaitableEvent* done) {
783     DCHECK(mTaskRunner->BelongsToCurrentThread());
784     ALOGV("onStop");
785     EXPECT_RUNNING_OR_RETURN_ON_ERROR();
786 
787     // Do not request VDA reset again before the previous one is done. If reset is already sent by
788     // onFlush(), just regard the following NotifyResetDone callback as for stopping.
789     if (mComponentState != ComponentState::FLUSHING) {
790         mVDAAdaptor->reset();
791     }
792 
793     // Pop all works in mQueue and put into mAbandonedWorks.
794     while (!mQueue.empty()) {
795         mAbandonedWorks.emplace_back(std::move(mQueue.front().mWork));
796         mQueue.pop();
797     }
798 
799     mStopDoneEvent = done;  // restore done event which shoud be signaled in onStopDone().
800     mComponentState = ComponentState::STOPPING;
801 }
802 
onResetDone()803 void C2VDAComponent::onResetDone() {
804     DCHECK(mTaskRunner->BelongsToCurrentThread());
805     if (mComponentState == ComponentState::ERROR) {
806         return;
807     }
808     if (mComponentState == ComponentState::FLUSHING) {
809         onFlushDone();
810     } else if (mComponentState == ComponentState::STOPPING) {
811         onStopDone();
812     } else {
813         reportError(C2_CORRUPTED);
814     }
815 }
816 
onFlushDone()817 void C2VDAComponent::onFlushDone() {
818     ALOGV("onFlushDone");
819     reportAbandonedWorks();
820     mPendingBuffersToWork.clear();
821     mComponentState = ComponentState::STARTED;
822 
823     // Work dequeueing was stopped while component flushing. Restart it.
824     mTaskRunner->PostTask(FROM_HERE,
825                           ::base::Bind(&C2VDAComponent::onDequeueWork, ::base::Unretained(this)));
826 }
827 
onStopDone()828 void C2VDAComponent::onStopDone() {
829     ALOGV("onStopDone");
830     CHECK(mStopDoneEvent);
831 
832     // TODO(johnylin): At this moment, there may be C2Buffer still owned by client, do we need to
833     // do something for them?
834     reportAbandonedWorks();
835     mPendingOutputFormat.reset();
836     mPendingBuffersToWork.clear();
837     if (mVDAAdaptor.get()) {
838         mVDAAdaptor->destroy();
839         mVDAAdaptor.reset(nullptr);
840     }
841 
842     stopDequeueThread();
843     mGraphicBlocks.clear();
844 
845     mStopDoneEvent->Signal();
846     mStopDoneEvent = nullptr;
847     mComponentState = ComponentState::UNINITIALIZED;
848 }
849 
setListener_vb(const std::shared_ptr<C2Component::Listener> & listener,c2_blocking_t mayBlock)850 c2_status_t C2VDAComponent::setListener_vb(const std::shared_ptr<C2Component::Listener>& listener,
851                                            c2_blocking_t mayBlock) {
852     UNUSED(mayBlock);
853     // TODO(johnylin): API says this method must be supported in all states, however I'm quite not
854     //                 sure what is the use case.
855     if (mState.load() != State::LOADED) {
856         return C2_BAD_STATE;
857     }
858     mListener = listener;
859     return C2_OK;
860 }
861 
sendInputBufferToAccelerator(const C2ConstLinearBlock & input,int32_t bitstreamId)862 void C2VDAComponent::sendInputBufferToAccelerator(const C2ConstLinearBlock& input,
863                                                   int32_t bitstreamId) {
864     ALOGV("sendInputBufferToAccelerator");
865     int dupFd = dup(input.handle()->data[0]);
866     if (dupFd < 0) {
867         ALOGE("Failed to dup(%d) input buffer (bitstreamId=%d), errno=%d", input.handle()->data[0],
868               bitstreamId, errno);
869         reportError(C2_CORRUPTED);
870         return;
871     }
872     ALOGV("Decode bitstream ID: %d, offset: %u size: %u", bitstreamId, input.offset(),
873           input.size());
874     mVDAAdaptor->decode(bitstreamId, dupFd, input.offset(), input.size());
875 }
876 
findPendingWorkByBitstreamId(int32_t bitstreamId)877 std::deque<std::unique_ptr<C2Work>>::iterator C2VDAComponent::findPendingWorkByBitstreamId(
878         int32_t bitstreamId) {
879     return std::find_if(mPendingWorks.begin(), mPendingWorks.end(),
880                         [bitstreamId](const std::unique_ptr<C2Work>& w) {
881                             return frameIndexToBitstreamId(w->input.ordinal.frameIndex) ==
882                                    bitstreamId;
883                         });
884 }
885 
getPendingWorkByBitstreamId(int32_t bitstreamId)886 C2Work* C2VDAComponent::getPendingWorkByBitstreamId(int32_t bitstreamId) {
887     auto workIter = findPendingWorkByBitstreamId(bitstreamId);
888     if (workIter == mPendingWorks.end()) {
889         ALOGE("Can't find pending work by bitstream ID: %d", bitstreamId);
890         return nullptr;
891     }
892     return workIter->get();
893 }
894 
getGraphicBlockById(int32_t blockId)895 C2VDAComponent::GraphicBlockInfo* C2VDAComponent::getGraphicBlockById(int32_t blockId) {
896     if (blockId < 0 || blockId >= static_cast<int32_t>(mGraphicBlocks.size())) {
897         ALOGE("getGraphicBlockById failed: id=%d", blockId);
898         return nullptr;
899     }
900     return &mGraphicBlocks[blockId];
901 }
902 
getGraphicBlockByPoolId(uint32_t poolId)903 C2VDAComponent::GraphicBlockInfo* C2VDAComponent::getGraphicBlockByPoolId(uint32_t poolId) {
904     auto blockIter = std::find_if(mGraphicBlocks.begin(), mGraphicBlocks.end(),
905                                   [poolId](const GraphicBlockInfo& gb) {
906                                       return gb.mPoolId == poolId;
907                                   });
908 
909     if (blockIter == mGraphicBlocks.end()) {
910         ALOGE("getGraphicBlockByPoolId failed: poolId=%u", poolId);
911         return nullptr;
912     }
913     return &(*blockIter);
914 }
915 
onOutputFormatChanged(std::unique_ptr<VideoFormat> format)916 void C2VDAComponent::onOutputFormatChanged(std::unique_ptr<VideoFormat> format) {
917     DCHECK(mTaskRunner->BelongsToCurrentThread());
918     ALOGV("onOutputFormatChanged");
919     EXPECT_RUNNING_OR_RETURN_ON_ERROR();
920 
921     ALOGV("New output format(pixel_format=0x%x, min_num_buffers=%u, coded_size=%s, crop_rect=%s)",
922           static_cast<uint32_t>(format->mPixelFormat), format->mMinNumBuffers,
923           format->mCodedSize.ToString().c_str(), format->mVisibleRect.ToString().c_str());
924 
925     for (auto& info : mGraphicBlocks) {
926         if (info.mState == GraphicBlockInfo::State::OWNED_BY_ACCELERATOR)
927             info.mState = GraphicBlockInfo::State::OWNED_BY_COMPONENT;
928     }
929 
930     CHECK(!mPendingOutputFormat);
931     mPendingOutputFormat = std::move(format);
932     tryChangeOutputFormat();
933 }
934 
tryChangeOutputFormat()935 void C2VDAComponent::tryChangeOutputFormat() {
936     DCHECK(mTaskRunner->BelongsToCurrentThread());
937     ALOGV("tryChangeOutputFormat");
938     CHECK(mPendingOutputFormat);
939 
940     // At this point, all output buffers should not be owned by accelerator. The component is not
941     // able to know when a client will release all owned output buffers by now. But it is ok to
942     // leave them to client since componenet won't own those buffers anymore.
943     // TODO(johnylin): we may also set a parameter for component to keep dequeueing buffers and
944     //                 change format only after the component owns most buffers. This may prevent
945     //                 too many buffers are still on client's hand while component starts to
946     //                 allocate more buffers. However, it leads latency on output format change.
947     for (const auto& info : mGraphicBlocks) {
948         CHECK(info.mState != GraphicBlockInfo::State::OWNED_BY_ACCELERATOR);
949     }
950 
951     // Drop all pending existing frames and return all finished works before changing output format.
952     sendOutputBufferToWorkIfAny(true /* dropIfUnavailable */);
953     CHECK(mPendingBuffersToWork.empty());
954 
955     CHECK_EQ(mPendingOutputFormat->mPixelFormat, HalPixelFormat::YCbCr_420_888);
956 
957     mOutputFormat.mPixelFormat = mPendingOutputFormat->mPixelFormat;
958     mOutputFormat.mMinNumBuffers = mPendingOutputFormat->mMinNumBuffers;
959     mOutputFormat.mCodedSize = mPendingOutputFormat->mCodedSize;
960 
961     setOutputFormatCrop(mPendingOutputFormat->mVisibleRect);
962 
963     c2_status_t err = allocateBuffersFromBlockAllocator(
964             mPendingOutputFormat->mCodedSize,
965             static_cast<uint32_t>(mPendingOutputFormat->mPixelFormat));
966     if (err != C2_OK) {
967         reportError(err);
968         return;
969     }
970 
971     for (auto& info : mGraphicBlocks) {
972         sendOutputBufferToAccelerator(&info, true /* ownByAccelerator */);
973     }
974     mPendingOutputFormat.reset();
975 }
976 
allocateBuffersFromBlockAllocator(const media::Size & size,uint32_t pixelFormat)977 c2_status_t C2VDAComponent::allocateBuffersFromBlockAllocator(const media::Size& size,
978                                                               uint32_t pixelFormat) {
979     ALOGV("allocateBuffersFromBlockAllocator(%s, 0x%x)", size.ToString().c_str(), pixelFormat);
980 
981     stopDequeueThread();
982 
983     size_t bufferCount = mOutputFormat.mMinNumBuffers + kDpbOutputBufferExtraCount;
984 
985     // Allocate the output buffers.
986     mVDAAdaptor->assignPictureBuffers(bufferCount);
987 
988     // Get block pool ID configured from the client.
989     std::shared_ptr<C2BlockPool> blockPool;
990     auto poolId = mIntfImpl->getBlockPoolId();
991     ALOGI("Using C2BlockPool ID = %" PRIu64 " for allocating output buffers", poolId);
992     auto err = GetCodec2BlockPool(poolId, shared_from_this(), &blockPool);
993     if (err != C2_OK) {
994         ALOGE("Graphic block allocator is invalid");
995         reportError(err);
996         return err;
997     }
998 
999     mGraphicBlocks.clear();
1000 
1001     bool useBufferQueue = blockPool->getAllocatorId() == C2PlatformAllocatorStore::BUFFERQUEUE;
1002     size_t minBuffersForDisplay = 0;
1003     if (useBufferQueue) {
1004         ALOGV("Bufferqueue-backed block pool is used.");
1005         // Set requested buffer count to C2VdaBqBlockPool.
1006         std::shared_ptr<C2VdaBqBlockPool> bqPool =
1007                 std::static_pointer_cast<C2VdaBqBlockPool>(blockPool);
1008         if (bqPool) {
1009             err = bqPool->requestNewBufferSet(static_cast<int32_t>(bufferCount));
1010             if (err != C2_OK) {
1011                 ALOGE("failed to request new buffer set to block pool: %d", err);
1012                 reportError(err);
1013                 return err;
1014             }
1015             err = bqPool->getMinBuffersForDisplay(&minBuffersForDisplay);
1016             if (err != C2_OK) {
1017                 ALOGE("failed to query minimum undequeued buffer count from block pool: %d", err);
1018                 reportError(err);
1019                 return err;
1020             }
1021         } else {
1022             ALOGE("static_pointer_cast C2VdaBqBlockPool failed...");
1023             reportError(C2_CORRUPTED);
1024             return C2_CORRUPTED;
1025         }
1026     } else {
1027         ALOGV("Bufferpool-backed block pool is used.");
1028         // Set requested buffer count to C2VdaPooledBlockPool.
1029         std::shared_ptr<C2VdaPooledBlockPool> bpPool =
1030                 std::static_pointer_cast<C2VdaPooledBlockPool>(blockPool);
1031         if (bpPool) {
1032             err = bpPool->requestNewBufferSet(static_cast<int32_t>(bufferCount));
1033             if (err != C2_OK) {
1034                 ALOGE("failed to request new buffer set to block pool: %d", err);
1035                 reportError(err);
1036                 return err;
1037             }
1038             minBuffersForDisplay = 0;  // no undequeued buffer restriction for bufferpool.
1039         } else {
1040             ALOGE("static_pointer_cast C2VdaPooledBlockPool failed...");
1041             reportError(C2_CORRUPTED);
1042             return C2_CORRUPTED;
1043         }
1044     }
1045 
1046     ALOGV("Minimum undequeued buffer count = %zu", minBuffersForDisplay);
1047     mUndequeuedBlockIds.resize(minBuffersForDisplay, -1);
1048 
1049     for (size_t i = 0; i < bufferCount; ++i) {
1050         std::shared_ptr<C2GraphicBlock> block;
1051         C2MemoryUsage usage = {
1052                 mSecureMode ? C2MemoryUsage::READ_PROTECTED : C2MemoryUsage::CPU_READ, 0};
1053 
1054         int32_t retries_left = kAllocateBufferMaxRetries;
1055         err = C2_NO_INIT;
1056         while (err != C2_OK) {
1057             err = blockPool->fetchGraphicBlock(size.width(), size.height(), pixelFormat, usage,
1058                                                &block);
1059             if (err == C2_TIMED_OUT && retries_left > 0) {
1060                 ALOGD("allocate buffer timeout, %d retry time(s) left...", retries_left);
1061                 retries_left--;
1062             } else if (err != C2_OK) {
1063                 mGraphicBlocks.clear();
1064                 ALOGE("failed to allocate buffer: %d", err);
1065                 reportError(err);
1066                 return err;
1067             }
1068         }
1069 
1070         uint32_t poolId;
1071         if (useBufferQueue) {
1072             err = C2VdaBqBlockPool::getPoolIdFromGraphicBlock(block, &poolId);
1073         } else {  // use bufferpool
1074             err = C2VdaPooledBlockPool::getPoolIdFromGraphicBlock(block, &poolId);
1075         }
1076         if (err != C2_OK) {
1077             mGraphicBlocks.clear();
1078             ALOGE("failed to getPoolIdFromGraphicBlock: %d", err);
1079             reportError(err);
1080             return err;
1081         }
1082         if (mSecureMode) {
1083             appendSecureOutputBuffer(std::move(block), poolId);
1084         } else {
1085             appendOutputBuffer(std::move(block), poolId);
1086         }
1087     }
1088     mOutputFormat.mMinNumBuffers = bufferCount;
1089 
1090     if (!startDequeueThread(size, pixelFormat, std::move(blockPool),
1091                             true /* resetBuffersInClient */)) {
1092         reportError(C2_CORRUPTED);
1093         return C2_CORRUPTED;
1094     }
1095     return C2_OK;
1096 }
1097 
appendOutputBuffer(std::shared_ptr<C2GraphicBlock> block,uint32_t poolId)1098 void C2VDAComponent::appendOutputBuffer(std::shared_ptr<C2GraphicBlock> block, uint32_t poolId) {
1099     GraphicBlockInfo info;
1100     info.mBlockId = static_cast<int32_t>(mGraphicBlocks.size());
1101     info.mGraphicBlock = std::move(block);
1102     info.mPoolId = poolId;
1103 
1104     C2ConstGraphicBlock constBlock = info.mGraphicBlock->share(
1105             C2Rect(info.mGraphicBlock->width(), info.mGraphicBlock->height()), C2Fence());
1106 
1107     const C2GraphicView& view = constBlock.map().get();
1108     const uint8_t* const* data = view.data();
1109     CHECK_NE(data, nullptr);
1110     const C2PlanarLayout& layout = view.layout();
1111 
1112     ALOGV("allocate graphic buffer: %p, id: %d, size: %dx%d", info.mGraphicBlock->handle(),
1113           info.mBlockId, info.mGraphicBlock->width(), info.mGraphicBlock->height());
1114 
1115     // get offset from data pointers
1116     uint32_t offsets[C2PlanarLayout::MAX_NUM_PLANES];
1117     auto baseAddress = reinterpret_cast<intptr_t>(data[0]);
1118     for (uint32_t i = 0; i < layout.numPlanes; ++i) {
1119         auto planeAddress = reinterpret_cast<intptr_t>(data[i]);
1120         offsets[i] = static_cast<uint32_t>(planeAddress - baseAddress);
1121     }
1122 
1123     bool crcb = false;
1124     if (layout.numPlanes == 3 &&
1125         offsets[C2PlanarLayout::PLANE_U] > offsets[C2PlanarLayout::PLANE_V]) {
1126         // YCrCb format
1127         std::swap(offsets[C2PlanarLayout::PLANE_U], offsets[C2PlanarLayout::PLANE_V]);
1128         crcb = true;
1129     }
1130 
1131     bool semiplanar = false;
1132     uint32_t passedNumPlanes = layout.numPlanes;
1133     if (layout.planes[C2PlanarLayout::PLANE_U].colInc == 2) {  // chroma_step
1134         // Semi-planar format
1135         passedNumPlanes--;
1136         semiplanar = true;
1137     }
1138 
1139     for (uint32_t i = 0; i < passedNumPlanes; ++i) {
1140         ALOGV("plane %u: stride: %d, offset: %u", i, layout.planes[i].rowInc, offsets[i]);
1141     }
1142     info.mPixelFormat = resolveBufferFormat(crcb, semiplanar);
1143     ALOGV("HAL pixel format: 0x%x", static_cast<uint32_t>(info.mPixelFormat));
1144 
1145     ::base::ScopedFD passedHandle(dup(info.mGraphicBlock->handle()->data[0]));
1146     if (!passedHandle.is_valid()) {
1147         ALOGE("Failed to dup(%d), errno=%d", info.mGraphicBlock->handle()->data[0], errno);
1148         reportError(C2_CORRUPTED);
1149         return;
1150     }
1151     std::vector<VideoFramePlane> passedPlanes;
1152     for (uint32_t i = 0; i < passedNumPlanes; ++i) {
1153         CHECK_GT(layout.planes[i].rowInc, 0);
1154         passedPlanes.push_back({offsets[i], static_cast<uint32_t>(layout.planes[i].rowInc)});
1155     }
1156     info.mHandle = std::move(passedHandle);
1157     info.mPlanes = std::move(passedPlanes);
1158 
1159     mGraphicBlocks.push_back(std::move(info));
1160 }
1161 
appendSecureOutputBuffer(std::shared_ptr<C2GraphicBlock> block,uint32_t poolId)1162 void C2VDAComponent::appendSecureOutputBuffer(std::shared_ptr<C2GraphicBlock> block,
1163                                               uint32_t poolId) {
1164 #ifdef V4L2_CODEC2_ARC
1165     const C2Handle* const handle = block->handle();
1166     const int handleFd = handle->data[0];
1167     ::base::ScopedFD passedHandle(dup(handleFd));
1168     if (!passedHandle.is_valid()) {
1169         ALOGE("Failed to dup(%d), errno=%d", handleFd, errno);
1170         reportError(C2_CORRUPTED);
1171         return;
1172     }
1173 
1174     android::HalPixelFormat pixelFormat = getPlatformPixelFormat();
1175     if (pixelFormat == android::HalPixelFormat::UNKNOWN) {
1176         ALOGE("Failed to get pixel format on platform.");
1177         reportError(C2_CORRUPTED);
1178         return;
1179     }
1180     CHECK(pixelFormat == android::HalPixelFormat::YV12 ||
1181           pixelFormat == android::HalPixelFormat::NV12);
1182     ALOGV("HAL pixel format: 0x%x", static_cast<uint32_t>(pixelFormat));
1183 
1184     GraphicBlockInfo info;
1185     info.mBlockId = static_cast<int32_t>(mGraphicBlocks.size());
1186     info.mGraphicBlock = std::move(block);
1187     info.mPoolId = poolId;
1188     info.mHandle = std::move(passedHandle);
1189     info.mPixelFormat = pixelFormat;
1190     // In secure mode, since planes are not referred in Chrome side, empty plane is valid.
1191     info.mPlanes.clear();
1192     mGraphicBlocks.push_back(std::move(info));
1193 #else
1194     ALOGE("appendSecureOutputBuffer() is not supported...");
1195     reportError(C2_OMITTED);
1196 #endif // V4L2_CODEC2_ARC
1197 }
1198 
sendOutputBufferToAccelerator(GraphicBlockInfo * info,bool ownByAccelerator)1199 void C2VDAComponent::sendOutputBufferToAccelerator(GraphicBlockInfo* info, bool ownByAccelerator) {
1200     DCHECK(mTaskRunner->BelongsToCurrentThread());
1201     ALOGV("sendOutputBufferToAccelerator index=%d ownByAccelerator=%d", info->mBlockId,
1202           ownByAccelerator);
1203 
1204     if (ownByAccelerator) {
1205         CHECK_EQ(info->mState, GraphicBlockInfo::State::OWNED_BY_COMPONENT);
1206         info->mState = GraphicBlockInfo::State::OWNED_BY_ACCELERATOR;
1207     }
1208 
1209     // is_valid() is true for the first time the buffer is passed to VDA. In that case, VDA needs to
1210     // import the buffer first.
1211     if (info->mHandle.is_valid()) {
1212         mVDAAdaptor->importBufferForPicture(info->mBlockId, info->mPixelFormat,
1213                                             info->mHandle.release(), info->mPlanes);
1214     } else {
1215         mVDAAdaptor->reusePictureBuffer(info->mBlockId);
1216     }
1217 }
1218 
parseCodedColorAspects(const C2ConstLinearBlock & input)1219 bool C2VDAComponent::parseCodedColorAspects(const C2ConstLinearBlock& input) {
1220     C2ReadView view = input.map().get();
1221     const uint8_t* data = view.data();
1222     const uint32_t size = view.capacity();
1223 
1224     std::unique_ptr<media::H264Parser> h264Parser = std::make_unique<media::H264Parser>();
1225     h264Parser->SetStream(data, static_cast<off_t>(size));
1226     media::H264NALU nalu;
1227     media::H264Parser::Result parRes = h264Parser->AdvanceToNextNALU(&nalu);
1228     if (parRes != media::H264Parser::kEOStream && parRes != media::H264Parser::kOk) {
1229         ALOGE("H264 AdvanceToNextNALU error: %d", static_cast<int>(parRes));
1230         return false;
1231     }
1232     if (nalu.nal_unit_type != media::H264NALU::kSPS) {
1233         ALOGV("NALU is not SPS");
1234         return false;
1235     }
1236 
1237     int spsId;
1238     parRes = h264Parser->ParseSPS(&spsId);
1239     if (parRes != media::H264Parser::kEOStream && parRes != media::H264Parser::kOk) {
1240         ALOGE("H264 ParseSPS error: %d", static_cast<int>(parRes));
1241         return false;
1242     }
1243 
1244     // Parse ISO color aspects from H264 SPS bitstream.
1245     const media::H264SPS* sps = h264Parser->GetSPS(spsId);
1246     if (!sps->colour_description_present_flag) {
1247         ALOGV("No Color Description in SPS");
1248         return false;
1249     }
1250     int32_t primaries = sps->colour_primaries;
1251     int32_t transfer = sps->transfer_characteristics;
1252     int32_t coeffs = sps->matrix_coefficients;
1253     bool fullRange = sps->video_full_range_flag;
1254 
1255     // Convert ISO color aspects to ColorUtils::ColorAspects.
1256     ColorAspects colorAspects;
1257     ColorUtils::convertIsoColorAspectsToCodecAspects(primaries, transfer, coeffs, fullRange,
1258                                                      colorAspects);
1259     ALOGV("Parsed ColorAspects from bitstream: (R:%d, P:%d, M:%d, T:%d)", colorAspects.mRange,
1260           colorAspects.mPrimaries, colorAspects.mMatrixCoeffs, colorAspects.mTransfer);
1261 
1262     // Map ColorUtils::ColorAspects to C2StreamColorAspectsInfo::input parameter.
1263     C2StreamColorAspectsInfo::input codedAspects = {0u};
1264     if (!C2Mapper::map(colorAspects.mPrimaries, &codedAspects.primaries)) {
1265         codedAspects.primaries = C2Color::PRIMARIES_UNSPECIFIED;
1266     }
1267     if (!C2Mapper::map(colorAspects.mRange, &codedAspects.range)) {
1268         codedAspects.range = C2Color::RANGE_UNSPECIFIED;
1269     }
1270     if (!C2Mapper::map(colorAspects.mMatrixCoeffs, &codedAspects.matrix)) {
1271         codedAspects.matrix = C2Color::MATRIX_UNSPECIFIED;
1272     }
1273     if (!C2Mapper::map(colorAspects.mTransfer, &codedAspects.transfer)) {
1274         codedAspects.transfer = C2Color::TRANSFER_UNSPECIFIED;
1275     }
1276     // Configure to interface.
1277     std::vector<std::unique_ptr<C2SettingResult>> failures;
1278     c2_status_t status = mIntfImpl->config({&codedAspects}, C2_MAY_BLOCK, &failures);
1279     if (status != C2_OK) {
1280         ALOGE("Failed to config color aspects to interface, error: %d", status);
1281         return false;
1282     }
1283     return true;
1284 }
1285 
updateColorAspects()1286 c2_status_t C2VDAComponent::updateColorAspects() {
1287     ALOGV("updateColorAspects");
1288     std::unique_ptr<C2StreamColorAspectsInfo::output> colorAspects =
1289             std::make_unique<C2StreamColorAspectsInfo::output>(
1290                     0u, C2Color::RANGE_UNSPECIFIED, C2Color::PRIMARIES_UNSPECIFIED,
1291                     C2Color::TRANSFER_UNSPECIFIED, C2Color::MATRIX_UNSPECIFIED);
1292     c2_status_t status = mIntfImpl->query({colorAspects.get()}, {}, C2_DONT_BLOCK, nullptr);
1293     if (status != C2_OK) {
1294         ALOGE("Failed to query color aspects, error: %d", status);
1295         return status;
1296     }
1297     mCurrentColorAspects = std::move(colorAspects);
1298     return C2_OK;
1299 }
1300 
onVisibleRectChanged(const media::Rect & cropRect)1301 void C2VDAComponent::onVisibleRectChanged(const media::Rect& cropRect) {
1302     DCHECK(mTaskRunner->BelongsToCurrentThread());
1303     ALOGV("onVisibleRectChanged");
1304     EXPECT_RUNNING_OR_RETURN_ON_ERROR();
1305 
1306     // We should make sure there is no pending output format change. That is, the input cropRect is
1307     // corresponding to current output format.
1308     CHECK(mPendingOutputFormat == nullptr);
1309     setOutputFormatCrop(cropRect);
1310 }
1311 
setOutputFormatCrop(const media::Rect & cropRect)1312 void C2VDAComponent::setOutputFormatCrop(const media::Rect& cropRect) {
1313     ALOGV("setOutputFormatCrop(%dx%d)", cropRect.width(), cropRect.height());
1314     // This visible rect should be set as crop window for each C2ConstGraphicBlock passed to
1315     // framework.
1316     mOutputFormat.mVisibleRect = cropRect;
1317 }
1318 
onSurfaceChanged()1319 void C2VDAComponent::onSurfaceChanged() {
1320     DCHECK(mTaskRunner->BelongsToCurrentThread());
1321     ALOGV("onSurfaceChanged");
1322 
1323     if (mComponentState == ComponentState::UNINITIALIZED) {
1324         return;  // Component is already stopped, no need to update graphic blocks.
1325     }
1326 
1327     stopDequeueThread();
1328 
1329     // Get block pool ID configured from the client.
1330     std::shared_ptr<C2BlockPool> blockPool;
1331     auto blockPoolId = mIntfImpl->getBlockPoolId();
1332     ALOGI("Retrieving C2BlockPool ID = %" PRIu64 " for updating output buffers", blockPoolId);
1333     auto err = GetCodec2BlockPool(blockPoolId, shared_from_this(), &blockPool);
1334     if (err != C2_OK) {
1335         ALOGE("Graphic block allocator is invalid");
1336         reportError(err);
1337         return;
1338     }
1339     if (blockPool->getAllocatorId() != C2PlatformAllocatorStore::BUFFERQUEUE) {
1340         ALOGE("Only Bufferqueue-backed block pool would need to change surface.");
1341         reportError(C2_CORRUPTED);
1342         return;
1343     }
1344 
1345     std::shared_ptr<C2VdaBqBlockPool> bqPool =
1346             std::static_pointer_cast<C2VdaBqBlockPool>(blockPool);
1347     if (!bqPool) {
1348         ALOGE("static_pointer_cast C2VdaBqBlockPool failed...");
1349         reportError(C2_CORRUPTED);
1350         return;
1351     }
1352 
1353     size_t minBuffersForDisplay = 0;
1354     err = bqPool->getMinBuffersForDisplay(&minBuffersForDisplay);
1355     if (err != C2_OK) {
1356         ALOGE("failed to query minimum undequeued buffer count from block pool: %d", err);
1357         reportError(err);
1358         return;
1359     }
1360     ALOGV("Minimum undequeued buffer count = %zu", minBuffersForDisplay);
1361     mUndequeuedBlockIds.resize(minBuffersForDisplay, -1);
1362 
1363     for (auto& info : mGraphicBlocks) {
1364         bool willCancel = (info.mGraphicBlock == nullptr);
1365         uint32_t oldSlot = info.mPoolId;
1366         ALOGV("Updating graphic block #%d: slot = %u, willCancel = %d", info.mBlockId, oldSlot,
1367               willCancel);
1368         uint32_t newSlot;
1369         std::shared_ptr<C2GraphicBlock> block;
1370         err = bqPool->updateGraphicBlock(willCancel, oldSlot, &newSlot, &block);
1371         if (err == C2_CANCELED) {
1372             // There may be a chance that a task in task runner before onSurfaceChange triggers
1373             // output format change. If so, block pool will return C2_CANCELED and no need to
1374             // updateGraphicBlock anymore.
1375             return;
1376         }
1377         if (err != C2_OK) {
1378             ALOGE("failed to update graphic block from block pool: %d", err);
1379             reportError(err);
1380             return;
1381         }
1382 
1383         // Update slot index.
1384         info.mPoolId = newSlot;
1385         // Update C2GraphicBlock if |willCancel| is false. Note that although the old C2GraphicBlock
1386         // will be released, the block pool data destructor won't do detachBuffer to new surface
1387         // because the producer ID is not matched.
1388         if (!willCancel) {
1389             info.mGraphicBlock = std::move(block);
1390         }
1391     }
1392 
1393     if (!startDequeueThread(mOutputFormat.mCodedSize,
1394                             static_cast<uint32_t>(mOutputFormat.mPixelFormat), std::move(blockPool),
1395                             false /* resetBuffersInClient */)) {
1396         reportError(C2_CORRUPTED);
1397     }
1398 }
1399 
queue_nb(std::list<std::unique_ptr<C2Work>> * const items)1400 c2_status_t C2VDAComponent::queue_nb(std::list<std::unique_ptr<C2Work>>* const items) {
1401     if (mState.load() != State::RUNNING) {
1402         return C2_BAD_STATE;
1403     }
1404     while (!items->empty()) {
1405         mTaskRunner->PostTask(FROM_HERE,
1406                               ::base::Bind(&C2VDAComponent::onQueueWork, ::base::Unretained(this),
1407                                            ::base::Passed(&items->front())));
1408         items->pop_front();
1409     }
1410     return C2_OK;
1411 }
1412 
announce_nb(const std::vector<C2WorkOutline> & items)1413 c2_status_t C2VDAComponent::announce_nb(const std::vector<C2WorkOutline>& items) {
1414     UNUSED(items);
1415     return C2_OMITTED;  // Tunneling is not supported by now
1416 }
1417 
flush_sm(flush_mode_t mode,std::list<std::unique_ptr<C2Work>> * const flushedWork)1418 c2_status_t C2VDAComponent::flush_sm(flush_mode_t mode,
1419                                      std::list<std::unique_ptr<C2Work>>* const flushedWork) {
1420     if (mode != FLUSH_COMPONENT) {
1421         return C2_OMITTED;  // Tunneling is not supported by now
1422     }
1423     if (mState.load() != State::RUNNING) {
1424         return C2_BAD_STATE;
1425     }
1426     mTaskRunner->PostTask(FROM_HERE, ::base::Bind(&C2VDAComponent::onFlush,
1427                                                   ::base::Unretained(this)));
1428     // Instead of |flushedWork|, abandoned works will be returned via onWorkDone_nb() callback.
1429     return C2_OK;
1430 }
1431 
drain_nb(drain_mode_t mode)1432 c2_status_t C2VDAComponent::drain_nb(drain_mode_t mode) {
1433     if (mode != DRAIN_COMPONENT_WITH_EOS && mode != DRAIN_COMPONENT_NO_EOS) {
1434         return C2_OMITTED;  // Tunneling is not supported by now
1435     }
1436     if (mState.load() != State::RUNNING) {
1437         return C2_BAD_STATE;
1438     }
1439     mTaskRunner->PostTask(FROM_HERE,
1440                           ::base::Bind(&C2VDAComponent::onDrain, ::base::Unretained(this),
1441                                        static_cast<uint32_t>(mode)));
1442     return C2_OK;
1443 }
1444 
start()1445 c2_status_t C2VDAComponent::start() {
1446     // Use mStartStopLock to block other asynchronously start/stop calls.
1447     std::lock_guard<std::mutex> lock(mStartStopLock);
1448 
1449     if (mState.load() != State::LOADED) {
1450         return C2_BAD_STATE;  // start() is only supported when component is in LOADED state.
1451     }
1452 
1453     mCodecProfile = mIntfImpl->getCodecProfile();
1454     ALOGI("get parameter: mCodecProfile = %d", static_cast<int>(mCodecProfile));
1455 
1456     ::base::WaitableEvent done(::base::WaitableEvent::ResetPolicy::AUTOMATIC,
1457                                ::base::WaitableEvent::InitialState::NOT_SIGNALED);
1458     mTaskRunner->PostTask(FROM_HERE,
1459                           ::base::Bind(&C2VDAComponent::onStart, ::base::Unretained(this),
1460                                        mCodecProfile, &done));
1461     done.Wait();
1462     c2_status_t c2Status = adaptorResultToC2Status(mVDAInitResult);
1463     if (c2Status != C2_OK) {
1464         ALOGE("Failed to start component due to VDA error...");
1465         return c2Status;
1466     }
1467     mState.store(State::RUNNING);
1468     return C2_OK;
1469 }
1470 
stop()1471 c2_status_t C2VDAComponent::stop() {
1472     // Use mStartStopLock to block other asynchronously start/stop calls.
1473     std::lock_guard<std::mutex> lock(mStartStopLock);
1474 
1475     auto state = mState.load();
1476     if (!(state == State::RUNNING || state == State::ERROR)) {
1477         return C2_OK;  // Component is already in stopped state.
1478     }
1479 
1480     ::base::WaitableEvent done(::base::WaitableEvent::ResetPolicy::AUTOMATIC,
1481                                ::base::WaitableEvent::InitialState::NOT_SIGNALED);
1482     mTaskRunner->PostTask(FROM_HERE,
1483                           ::base::Bind(&C2VDAComponent::onStop, ::base::Unretained(this), &done));
1484     done.Wait();
1485     mState.store(State::LOADED);
1486     return C2_OK;
1487 }
1488 
reset()1489 c2_status_t C2VDAComponent::reset() {
1490     return stop();
1491     // TODO(johnylin): reset is different than stop that it could be called in any state.
1492     // TODO(johnylin): when reset is called, set ComponentInterface to default values.
1493 }
1494 
release()1495 c2_status_t C2VDAComponent::release() {
1496     return reset();
1497 }
1498 
intf()1499 std::shared_ptr<C2ComponentInterface> C2VDAComponent::intf() {
1500     return mIntf;
1501 }
1502 
providePictureBuffers(uint32_t minNumBuffers,const media::Size & codedSize)1503 void C2VDAComponent::providePictureBuffers(uint32_t minNumBuffers, const media::Size& codedSize) {
1504     // Always use fexible pixel 420 format YCbCr_420_888 in Android.
1505     // Uses coded size for crop rect while it is not available.
1506     auto format = std::make_unique<VideoFormat>(HalPixelFormat::YCbCr_420_888, minNumBuffers,
1507                                                 codedSize, media::Rect(codedSize));
1508 
1509     // Set mRequestedVisibleRect to default.
1510     mRequestedVisibleRect = media::Rect();
1511 
1512     mTaskRunner->PostTask(FROM_HERE, ::base::Bind(&C2VDAComponent::onOutputFormatChanged,
1513                                                   ::base::Unretained(this),
1514                                                   ::base::Passed(&format)));
1515 }
1516 
dismissPictureBuffer(int32_t pictureBufferId)1517 void C2VDAComponent::dismissPictureBuffer(int32_t pictureBufferId) {
1518     UNUSED(pictureBufferId);
1519     // no ops
1520 }
1521 
pictureReady(int32_t pictureBufferId,int32_t bitstreamId,const media::Rect & cropRect)1522 void C2VDAComponent::pictureReady(int32_t pictureBufferId, int32_t bitstreamId,
1523                                   const media::Rect& cropRect) {
1524     UNUSED(pictureBufferId);
1525     UNUSED(bitstreamId);
1526 
1527     if (mRequestedVisibleRect != cropRect) {
1528         mRequestedVisibleRect = cropRect;
1529         mTaskRunner->PostTask(FROM_HERE, ::base::Bind(&C2VDAComponent::onVisibleRectChanged,
1530                                                       ::base::Unretained(this), cropRect));
1531     }
1532 
1533     mTaskRunner->PostTask(FROM_HERE, ::base::Bind(&C2VDAComponent::onOutputBufferDone,
1534                                                   ::base::Unretained(this),
1535                                                   pictureBufferId, bitstreamId));
1536 }
1537 
notifyEndOfBitstreamBuffer(int32_t bitstreamId)1538 void C2VDAComponent::notifyEndOfBitstreamBuffer(int32_t bitstreamId) {
1539     mTaskRunner->PostTask(FROM_HERE, ::base::Bind(&C2VDAComponent::onInputBufferDone,
1540                                                   ::base::Unretained(this), bitstreamId));
1541 }
1542 
notifyFlushDone()1543 void C2VDAComponent::notifyFlushDone() {
1544     mTaskRunner->PostTask(FROM_HERE,
1545                           ::base::Bind(&C2VDAComponent::onDrainDone, ::base::Unretained(this)));
1546 }
1547 
notifyResetDone()1548 void C2VDAComponent::notifyResetDone() {
1549     mTaskRunner->PostTask(FROM_HERE,
1550                           ::base::Bind(&C2VDAComponent::onResetDone, ::base::Unretained(this)));
1551 }
1552 
notifyError(VideoDecodeAcceleratorAdaptor::Result error)1553 void C2VDAComponent::notifyError(VideoDecodeAcceleratorAdaptor::Result error) {
1554     ALOGE("Got notifyError from VDA...");
1555     c2_status_t err = adaptorResultToC2Status(error);
1556     if (err == C2_OK) {
1557         ALOGW("Shouldn't get SUCCESS err code in NotifyError(). Skip it...");
1558         return;
1559     }
1560     reportError(err);
1561 }
1562 
reportWorkIfFinished(int32_t bitstreamId)1563 void C2VDAComponent::reportWorkIfFinished(int32_t bitstreamId) {
1564     DCHECK(mTaskRunner->BelongsToCurrentThread());
1565 
1566     auto workIter = findPendingWorkByBitstreamId(bitstreamId);
1567     if (workIter == mPendingWorks.end()) {
1568         reportError(C2_CORRUPTED);
1569         return;
1570     }
1571 
1572     // EOS work will not be reported here. reportEOSWork() does it.
1573     auto work = workIter->get();
1574     if (isWorkDone(work)) {
1575         if (work->worklets.front()->output.flags & C2FrameData::FLAG_DROP_FRAME) {
1576             // TODO: actually framework does not handle FLAG_DROP_FRAME, use C2_NOT_FOUND result to
1577             //       let framework treat this as flushed work.
1578             work->result = C2_NOT_FOUND;
1579         } else {
1580             work->result = C2_OK;
1581         }
1582         work->workletsProcessed = static_cast<uint32_t>(work->worklets.size());
1583 
1584         ALOGV("Reported finished work index=%llu", work->input.ordinal.frameIndex.peekull());
1585         std::list<std::unique_ptr<C2Work>> finishedWorks;
1586         finishedWorks.emplace_back(std::move(*workIter));
1587         mListener->onWorkDone_nb(shared_from_this(), std::move(finishedWorks));
1588         mPendingWorks.erase(workIter);
1589     }
1590 }
1591 
isWorkDone(const C2Work * work) const1592 bool C2VDAComponent::isWorkDone(const C2Work* work) const {
1593     if (work->input.flags & C2FrameData::FLAG_END_OF_STREAM) {
1594         // This is EOS work and should be processed by reportEOSWork().
1595         return false;
1596     }
1597     if (work->input.buffers.front()) {
1598         // Input buffer is still owned by VDA.
1599         return false;
1600     }
1601     if (mPendingOutputEOS && mPendingWorks.size() == 1u) {
1602         // If mPendingOutputEOS is true, the last returned work should be marked EOS flag and
1603         // returned by reportEOSWork() instead.
1604         return false;
1605     }
1606     if (!(work->input.flags & C2FrameData::FLAG_CODEC_CONFIG) &&
1607         !(work->worklets.front()->output.flags & C2FrameData::FLAG_DROP_FRAME) &&
1608         work->worklets.front()->output.buffers.empty()) {
1609         // Unless the input is CSD or the output is dropped, this work is not done because the
1610         // output buffer is not returned from VDA yet.
1611         return false;
1612     }
1613     return true;  // This work is done.
1614 }
1615 
reportEOSWork()1616 void C2VDAComponent::reportEOSWork() {
1617     ALOGV("reportEOSWork");
1618     DCHECK(mTaskRunner->BelongsToCurrentThread());
1619     // In this moment all works prior to EOS work should be done and returned to listener.
1620     if (mPendingWorks.size() != 1u) {  // only EOS work left
1621         ALOGE("It shouldn't have remaining works in mPendingWorks except EOS work.");
1622         reportError(C2_CORRUPTED);
1623         return;
1624     }
1625 
1626     mPendingOutputEOS = false;
1627 
1628     std::unique_ptr<C2Work> eosWork(std::move(mPendingWorks.front()));
1629     mPendingWorks.pop_front();
1630     if (!eosWork->input.buffers.empty()) {
1631         eosWork->input.buffers.front().reset();
1632     }
1633     eosWork->result = C2_OK;
1634     eosWork->workletsProcessed = static_cast<uint32_t>(eosWork->worklets.size());
1635     eosWork->worklets.front()->output.flags = C2FrameData::FLAG_END_OF_STREAM;
1636 
1637     std::list<std::unique_ptr<C2Work>> finishedWorks;
1638     finishedWorks.emplace_back(std::move(eosWork));
1639     mListener->onWorkDone_nb(shared_from_this(), std::move(finishedWorks));
1640 }
1641 
reportAbandonedWorks()1642 void C2VDAComponent::reportAbandonedWorks() {
1643     DCHECK(mTaskRunner->BelongsToCurrentThread());
1644     std::list<std::unique_ptr<C2Work>> abandonedWorks;
1645 
1646     while (!mPendingWorks.empty()) {
1647         std::unique_ptr<C2Work> work(std::move(mPendingWorks.front()));
1648         mPendingWorks.pop_front();
1649 
1650         // TODO: correlate the definition of flushed work result to framework.
1651         work->result = C2_NOT_FOUND;
1652         // When the work is abandoned, buffer in input.buffers shall reset by component.
1653         if (!work->input.buffers.empty()) {
1654             work->input.buffers.front().reset();
1655         }
1656         abandonedWorks.emplace_back(std::move(work));
1657     }
1658 
1659     for (auto& work : mAbandonedWorks) {
1660         // TODO: correlate the definition of flushed work result to framework.
1661         work->result = C2_NOT_FOUND;
1662         // When the work is abandoned, buffer in input.buffers shall reset by component.
1663         if (!work->input.buffers.empty()) {
1664             work->input.buffers.front().reset();
1665         }
1666         abandonedWorks.emplace_back(std::move(work));
1667     }
1668     mAbandonedWorks.clear();
1669 
1670     // Pending EOS work will be abandoned here due to component flush if any.
1671     mPendingOutputEOS = false;
1672 
1673     if (!abandonedWorks.empty()) {
1674         mListener->onWorkDone_nb(shared_from_this(), std::move(abandonedWorks));
1675     }
1676 }
1677 
reportError(c2_status_t error)1678 void C2VDAComponent::reportError(c2_status_t error) {
1679     mListener->onError_nb(shared_from_this(), static_cast<uint32_t>(error));
1680 }
1681 
startDequeueThread(const media::Size & size,uint32_t pixelFormat,std::shared_ptr<C2BlockPool> blockPool,bool resetBuffersInClient)1682 bool C2VDAComponent::startDequeueThread(const media::Size& size, uint32_t pixelFormat,
1683                                         std::shared_ptr<C2BlockPool> blockPool,
1684                                         bool resetBuffersInClient) {
1685     CHECK(!mDequeueThread.IsRunning());
1686     if (!mDequeueThread.Start()) {
1687         ALOGE("failed to start dequeue thread!!");
1688         return false;
1689     }
1690     mDequeueLoopStop.store(false);
1691     if (resetBuffersInClient) {
1692         mBuffersInClient.store(0u);
1693     }
1694     mDequeueThread.task_runner()->PostTask(
1695             FROM_HERE, ::base::Bind(&C2VDAComponent::dequeueThreadLoop, ::base::Unretained(this),
1696                                     size, pixelFormat, std::move(blockPool)));
1697     return true;
1698 }
1699 
stopDequeueThread()1700 void C2VDAComponent::stopDequeueThread() {
1701     if (mDequeueThread.IsRunning()) {
1702         mDequeueLoopStop.store(true);
1703         mDequeueThread.Stop();
1704     }
1705 }
1706 
dequeueThreadLoop(const media::Size & size,uint32_t pixelFormat,std::shared_ptr<C2BlockPool> blockPool)1707 void C2VDAComponent::dequeueThreadLoop(const media::Size& size, uint32_t pixelFormat,
1708                                        std::shared_ptr<C2BlockPool> blockPool) {
1709     ALOGV("dequeueThreadLoop starts");
1710     DCHECK(mDequeueThread.task_runner()->BelongsToCurrentThread());
1711 
1712     while (!mDequeueLoopStop.load()) {
1713         if (mBuffersInClient.load() == 0) {
1714             ::usleep(kDequeueRetryDelayUs);  // wait for retry
1715             continue;
1716         }
1717         std::shared_ptr<C2GraphicBlock> block;
1718         C2MemoryUsage usage = {
1719                 mSecureMode ? C2MemoryUsage::READ_PROTECTED : C2MemoryUsage::CPU_READ, 0};
1720         auto err = blockPool->fetchGraphicBlock(size.width(), size.height(), pixelFormat, usage,
1721                                                 &block);
1722         if (err == C2_TIMED_OUT) {
1723             // Mutexes often do not care for FIFO. Practically the thread who is locking the mutex
1724             // usually will be granted to lock again right thereafter. To make this loop not too
1725             // bossy, the simpliest way is to add a short delay to the next time acquiring the
1726             // lock. TODO (b/118354314): replace this if there is better solution.
1727             ::usleep(1);
1728             continue;  // wait for retry
1729         }
1730         if (err == C2_BAD_STATE) {
1731             ALOGV("Got informed from block pool surface is changed.");
1732             mTaskRunner->PostTask(FROM_HERE, ::base::Bind(&C2VDAComponent::onSurfaceChanged,
1733                                                           ::base::Unretained(this)));
1734             break;  // terminate the loop, will be resumed after onSurfaceChanged().
1735         }
1736         if (err == C2_OK) {
1737             uint32_t poolId;
1738             if (blockPool->getAllocatorId() == C2PlatformAllocatorStore::BUFFERQUEUE) {
1739                 err = C2VdaBqBlockPool::getPoolIdFromGraphicBlock(block, &poolId);
1740             } else {  // bufferpool
1741                 err = C2VdaPooledBlockPool::getPoolIdFromGraphicBlock(block, &poolId);
1742             }
1743 
1744             if (err != C2_OK) {
1745                 ALOGE("dequeueThreadLoop got error on getPoolIdFromGraphicBlock: %d", err);
1746                 break;
1747             }
1748             mTaskRunner->PostTask(FROM_HERE,
1749                                   ::base::Bind(&C2VDAComponent::onOutputBufferReturned,
1750                                                ::base::Unretained(this), std::move(block), poolId));
1751             mBuffersInClient--;
1752         } else {
1753             ALOGE("dequeueThreadLoop got error: %d", err);
1754             break;
1755         }
1756     }
1757     ALOGV("dequeueThreadLoop terminates");
1758 }
1759 
1760 class C2VDAComponentFactory : public C2ComponentFactory {
1761 public:
C2VDAComponentFactory(C2String decoderName)1762     C2VDAComponentFactory(C2String decoderName)
1763           : mDecoderName(decoderName),
1764             mReflector(std::static_pointer_cast<C2ReflectorHelper>(
1765                     GetCodec2VDAComponentStore()->getParamReflector())){};
1766 
createComponent(c2_node_id_t id,std::shared_ptr<C2Component> * const component,ComponentDeleter deleter)1767     c2_status_t createComponent(c2_node_id_t id, std::shared_ptr<C2Component>* const component,
1768                                 ComponentDeleter deleter) override {
1769         UNUSED(deleter);
1770         *component = std::shared_ptr<C2Component>(new C2VDAComponent(mDecoderName, id, mReflector));
1771         return C2_OK;
1772     }
createInterface(c2_node_id_t id,std::shared_ptr<C2ComponentInterface> * const interface,InterfaceDeleter deleter)1773     c2_status_t createInterface(c2_node_id_t id,
1774                                 std::shared_ptr<C2ComponentInterface>* const interface,
1775                                 InterfaceDeleter deleter) override {
1776         UNUSED(deleter);
1777         *interface =
1778                 std::shared_ptr<C2ComponentInterface>(new SimpleInterface<C2VDAComponent::IntfImpl>(
1779                         mDecoderName.c_str(), id,
1780                         std::make_shared<C2VDAComponent::IntfImpl>(mDecoderName, mReflector)));
1781         return C2_OK;
1782     }
1783     ~C2VDAComponentFactory() override = default;
1784 
1785 private:
1786     const C2String mDecoderName;
1787     std::shared_ptr<C2ReflectorHelper> mReflector;
1788 };
1789 }  // namespace android
1790 
CreateC2VDAH264Factory(bool secureMode)1791 extern "C" ::C2ComponentFactory* CreateC2VDAH264Factory(bool secureMode) {
1792     ALOGV("in %s (secureMode=%d)", __func__, secureMode);
1793     return secureMode ? new ::android::C2VDAComponentFactory(android::kH264SecureDecoderName)
1794                       : new ::android::C2VDAComponentFactory(android::kH264DecoderName);
1795 }
1796 
DestroyC2VDAH264Factory(::C2ComponentFactory * factory)1797 extern "C" void DestroyC2VDAH264Factory(::C2ComponentFactory* factory) {
1798     ALOGV("in %s", __func__);
1799     delete factory;
1800 }
1801 
CreateC2VDAVP8Factory(bool secureMode)1802 extern "C" ::C2ComponentFactory* CreateC2VDAVP8Factory(bool secureMode) {
1803     ALOGV("in %s (secureMode=%d)", __func__, secureMode);
1804     return secureMode ? new ::android::C2VDAComponentFactory(android::kVP8SecureDecoderName)
1805                       : new ::android::C2VDAComponentFactory(android::kVP8DecoderName);
1806 }
1807 
DestroyC2VDAVP8Factory(::C2ComponentFactory * factory)1808 extern "C" void DestroyC2VDAVP8Factory(::C2ComponentFactory* factory) {
1809     ALOGV("in %s", __func__);
1810     delete factory;
1811 }
1812 
CreateC2VDAVP9Factory(bool secureMode)1813 extern "C" ::C2ComponentFactory* CreateC2VDAVP9Factory(bool secureMode) {
1814     ALOGV("in %s (secureMode=%d)", __func__, secureMode);
1815     return secureMode ? new ::android::C2VDAComponentFactory(android::kVP9SecureDecoderName)
1816                       : new ::android::C2VDAComponentFactory(android::kVP9DecoderName);
1817 }
1818 
DestroyC2VDAVP9Factory(::C2ComponentFactory * factory)1819 extern "C" void DestroyC2VDAVP9Factory(::C2ComponentFactory* factory) {
1820     ALOGV("in %s", __func__);
1821     delete factory;
1822 }
1823