• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 /*
2  * Copyright (C) 2010 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 //#define LOG_NDEBUG 0
18 #define LOG_TAG "ACodec"
19 
20 #ifdef __LP64__
21 #define OMX_ANDROID_COMPILE_AS_32BIT_ON_64BIT_PLATFORMS
22 #endif
23 
24 #include <android_media_codec.h>
25 
26 #include <inttypes.h>
27 #include <utils/Trace.h>
28 
29 #include <android/hardware/media/omx/1.0/IGraphicBufferSource.h>
30 
31 #include <gui/Surface.h>
32 
33 #include <media/stagefright/ACodec.h>
34 
35 #include <media/stagefright/foundation/avc_utils.h>
36 #include <media/stagefright/foundation/hexdump.h>
37 #include <media/stagefright/foundation/ABuffer.h>
38 #include <media/stagefright/foundation/ADebug.h>
39 #include <media/stagefright/foundation/AMessage.h>
40 #include <media/stagefright/foundation/AUtils.h>
41 
42 #include <media/stagefright/BufferProducerWrapper.h>
43 #include <media/stagefright/MediaCodec.h>
44 #include <media/stagefright/MediaCodecConstants.h>
45 #include <media/stagefright/MediaDefs.h>
46 #include <media/stagefright/OMXClient.h>
47 #include <media/stagefright/PersistentSurface.h>
48 #include <media/stagefright/RenderedFrameInfo.h>
49 #include <media/stagefright/SurfaceUtils.h>
50 #include <media/hardware/HardwareAPI.h>
51 #include <media/MediaBufferHolder.h>
52 #include <media/OMXBuffer.h>
53 #include <media/omx/1.0/Conversion.h>
54 #include <media/omx/1.0/WOmxNode.h>
55 
56 #include <hidlmemory/mapping.h>
57 
58 #include <media/openmax/OMX_AudioExt.h>
59 #include <media/openmax/OMX_VideoExt.h>
60 #include <media/openmax/OMX_Component.h>
61 #include <media/openmax/OMX_IndexExt.h>
62 #include <media/openmax/OMX_AsString.h>
63 
64 #include "include/ACodecBufferChannel.h"
65 #include "include/DataConverter.h"
66 #include "include/SecureBuffer.h"
67 #include "include/SharedMemoryBuffer.h"
68 #include <media/stagefright/omx/OMXUtils.h>
69 
70 #include <server_configurable_flags/get_flags.h>
71 
72 namespace android {
73 
74 typedef hardware::media::omx::V1_0::IGraphicBufferSource HGraphicBufferSource;
75 
76 using hardware::media::omx::V1_0::Status;
77 using server_configurable_flags::GetServerConfigurableFlag;
78 
79 enum {
80     kMaxIndicesToCheck = 32, // used when enumerating supported formats and profiles
81 };
82 
83 namespace {
84 
85 constexpr char TUNNEL_PEEK_KEY[] = "android._trigger-tunnel-peek";
86 constexpr char TUNNEL_PEEK_SET_LEGACY_KEY[] = "android._tunnel-peek-set-legacy";
87 
88 }
89 
areRenderMetricsEnabled()90 static bool areRenderMetricsEnabled() {
91     std::string v = GetServerConfigurableFlag("media_native", "render_metrics_enabled", "false");
92     return v == "true";
93 }
94 
95 // OMX errors are directly mapped into status_t range if
96 // there is no corresponding MediaError status code.
97 // Use the statusFromOMXError(int32_t omxError) function.
98 //
99 // Currently this is a direct map.
100 // See frameworks/native/include/media/openmax/OMX_Core.h
101 //
102 // Vendor OMX errors     from 0x90000000 - 0x9000FFFF
103 // Extension OMX errors  from 0x8F000000 - 0x90000000
104 // Standard OMX errors   from 0x80001000 - 0x80001024 (0x80001024 current)
105 //
106 
107 // returns true if err is a recognized OMX error code.
108 // as OMX error is OMX_S32, this is an int32_t type
isOMXError(int32_t err)109 static inline bool isOMXError(int32_t err) {
110     return (ERROR_CODEC_MIN <= err && err <= ERROR_CODEC_MAX);
111 }
112 
113 // converts an OMX error to a status_t
statusFromOMXError(int32_t omxError)114 static inline status_t statusFromOMXError(int32_t omxError) {
115     switch (omxError) {
116     case OMX_ErrorInvalidComponentName:
117     case OMX_ErrorComponentNotFound:
118         return NAME_NOT_FOUND; // can trigger illegal argument error for provided names.
119     default:
120         return isOMXError(omxError) ? omxError : 0; // no translation required
121     }
122 }
123 
statusFromBinderStatus(hardware::Return<Status> && status)124 static inline status_t statusFromBinderStatus(hardware::Return<Status> &&status) {
125     if (status.isOk()) {
126         return static_cast<status_t>(status.withDefault(Status::UNKNOWN_ERROR));
127     } else if (status.isDeadObject()) {
128         return DEAD_OBJECT;
129     }
130     // Other exception
131     return UNKNOWN_ERROR;
132 }
133 
134 // checks and converts status_t to a non-side-effect status_t
makeNoSideEffectStatus(status_t err)135 static inline status_t makeNoSideEffectStatus(status_t err) {
136     switch (err) {
137     // the following errors have side effects and may come
138     // from other code modules. Remap for safety reasons.
139     case INVALID_OPERATION:
140     case DEAD_OBJECT:
141         return UNKNOWN_ERROR;
142     default:
143         return err;
144     }
145 }
146 
getVideoBitrateMode(const sp<AMessage> & msg)147 static OMX_VIDEO_CONTROLRATETYPE getVideoBitrateMode(const sp<AMessage> &msg) {
148     int32_t tmp;
149     if (msg->findInt32("bitrate-mode", &tmp)) {
150         // explicitly translate from MediaCodecInfo.EncoderCapabilities.
151         // BITRATE_MODE_* into OMX bitrate mode.
152         switch (tmp) {
153             //BITRATE_MODE_CQ
154             case 0: return OMX_Video_ControlRateConstantQuality;
155             //BITRATE_MODE_VBR
156             case 1: return OMX_Video_ControlRateVariable;
157             //BITRATE_MODE_CBR
158             case 2: return OMX_Video_ControlRateConstant;
159             default: break;
160         }
161     }
162     return OMX_Video_ControlRateVariable;
163 }
164 
findVideoBitrateControlInfo(const sp<AMessage> & msg,OMX_VIDEO_CONTROLRATETYPE * mode,int32_t * bitrate,int32_t * quality)165 static bool findVideoBitrateControlInfo(const sp<AMessage> &msg,
166         OMX_VIDEO_CONTROLRATETYPE *mode, int32_t *bitrate, int32_t *quality) {
167     *mode = getVideoBitrateMode(msg);
168     bool isCQ = (*mode == OMX_Video_ControlRateConstantQuality);
169     return (!isCQ && msg->findInt32("bitrate", bitrate))
170          || (isCQ && msg->findInt32("quality", quality));
171 }
172 
173 struct MessageList : public RefBase {
MessageListandroid::MessageList174     MessageList() {
175     }
~MessageListandroid::MessageList176     virtual ~MessageList() {
177     }
getListandroid::MessageList178     std::list<sp<AMessage> > &getList() { return mList; }
179 private:
180     std::list<sp<AMessage> > mList;
181 
182     DISALLOW_EVIL_CONSTRUCTORS(MessageList);
183 };
184 
getCopyConverter()185 static sp<DataConverter> getCopyConverter() {
186     static pthread_once_t once = PTHREAD_ONCE_INIT; // const-inited
187     static sp<DataConverter> sCopyConverter;        // zero-inited
188     pthread_once(&once, [](){ sCopyConverter = new DataConverter(); });
189     return sCopyConverter;
190 }
191 
192 struct CodecObserver : public BnOMXObserver {
CodecObserverandroid::CodecObserver193     explicit CodecObserver(const sp<AMessage> &msg) : mNotify(msg) {}
194 
195     // from IOMXObserver
onMessagesandroid::CodecObserver196     virtual void onMessages(const std::list<omx_message> &messages) {
197         if (messages.empty()) {
198             return;
199         }
200 
201         sp<AMessage> notify = mNotify->dup();
202         sp<MessageList> msgList = new MessageList();
203         for (std::list<omx_message>::const_iterator it = messages.cbegin();
204               it != messages.cend(); ++it) {
205             const omx_message &omx_msg = *it;
206 
207             sp<AMessage> msg = new AMessage;
208             msg->setInt32("type", omx_msg.type);
209             switch (omx_msg.type) {
210                 case omx_message::EVENT:
211                 {
212                     msg->setInt32("event", omx_msg.u.event_data.event);
213                     msg->setInt32("data1", omx_msg.u.event_data.data1);
214                     msg->setInt32("data2", omx_msg.u.event_data.data2);
215                     break;
216                 }
217 
218                 case omx_message::EMPTY_BUFFER_DONE:
219                 {
220                     msg->setInt32("buffer", omx_msg.u.buffer_data.buffer);
221                     msg->setInt32("fence_fd", omx_msg.fenceFd);
222                     break;
223                 }
224 
225                 case omx_message::FILL_BUFFER_DONE:
226                 {
227                     msg->setInt32(
228                             "buffer", omx_msg.u.extended_buffer_data.buffer);
229                     msg->setInt32(
230                             "range_offset",
231                             omx_msg.u.extended_buffer_data.range_offset);
232                     msg->setInt32(
233                             "range_length",
234                             omx_msg.u.extended_buffer_data.range_length);
235                     msg->setInt32(
236                             "flags",
237                             omx_msg.u.extended_buffer_data.flags);
238                     msg->setInt64(
239                             "timestamp",
240                             omx_msg.u.extended_buffer_data.timestamp);
241                     msg->setInt32(
242                             "fence_fd", omx_msg.fenceFd);
243                     break;
244                 }
245 
246                 case omx_message::FRAME_RENDERED:
247                 {
248                     msg->setInt64(
249                             "media_time_us", omx_msg.u.render_data.timestamp);
250                     msg->setInt64(
251                             "system_nano", omx_msg.u.render_data.nanoTime);
252                     break;
253                 }
254 
255                 default:
256                     ALOGE("Unrecognized message type: %d", omx_msg.type);
257                     break;
258             }
259             msgList->getList().push_back(msg);
260         }
261         notify->setObject("messages", msgList);
262         notify->post();
263     }
264 
265 protected:
~CodecObserverandroid::CodecObserver266     virtual ~CodecObserver() {}
267 
268 private:
269     const sp<AMessage> mNotify;
270 
271     DISALLOW_EVIL_CONSTRUCTORS(CodecObserver);
272 };
273 
274 ////////////////////////////////////////////////////////////////////////////////
275 
276 struct ACodec::BaseState : public AState {
277     explicit BaseState(ACodec *codec, const sp<AState> &parentState = NULL);
278 
279 protected:
280     enum PortMode {
281         KEEP_BUFFERS,
282         RESUBMIT_BUFFERS,
283         FREE_BUFFERS,
284     };
285 
286     ACodec *mCodec;
287 
288     virtual PortMode getPortMode(OMX_U32 portIndex);
289 
290     virtual void stateExited();
291     virtual bool onMessageReceived(const sp<AMessage> &msg);
292 
293     virtual bool onOMXEvent(OMX_EVENTTYPE event, OMX_U32 data1, OMX_U32 data2);
294 
295     virtual void onOutputBufferDrained(const sp<AMessage> &msg);
296     virtual void onInputBufferFilled(const sp<AMessage> &msg);
297 
298     void postFillThisBuffer(BufferInfo *info);
299 
maybePostExtraOutputMetadataBufferRequestandroid::ACodec::BaseState300     void maybePostExtraOutputMetadataBufferRequest() {
301         if (!mPendingExtraOutputMetadataBufferRequest) {
302             (new AMessage(kWhatSubmitExtraOutputMetadataBuffer, mCodec))->post();
303             mPendingExtraOutputMetadataBufferRequest = true;
304         }
305     }
306 
307     void setSurfaceParameters(const sp<AMessage> &msg);
308 
309 private:
310     // Handles an OMX message. Returns true iff message was handled.
311     bool onOMXMessage(const sp<AMessage> &msg);
312 
313     // Handles a list of messages. Returns true iff messages were handled.
314     bool onOMXMessageList(const sp<AMessage> &msg);
315 
316     // returns true iff this message is for this component and the component is alive
317     bool checkOMXMessage(const sp<AMessage> &msg);
318 
319     bool onOMXEmptyBufferDone(IOMX::buffer_id bufferID, int fenceFd);
320 
321     bool onOMXFillBufferDone(
322             IOMX::buffer_id bufferID,
323             size_t rangeOffset, size_t rangeLength,
324             OMX_U32 flags,
325             int64_t timeUs,
326             int fenceFd);
327 
328     virtual bool onOMXFrameRendered(int64_t mediaTimeUs, nsecs_t systemNano);
329 
330     void getMoreInputDataIfPossible();
331 
332     bool mPendingExtraOutputMetadataBufferRequest;
333 
334     DISALLOW_EVIL_CONSTRUCTORS(BaseState);
335 };
336 
337 ////////////////////////////////////////////////////////////////////////////////
338 
339 struct ACodec::DeathNotifier :
340         public IBinder::DeathRecipient,
341         public ::android::hardware::hidl_death_recipient {
DeathNotifierandroid::ACodec::DeathNotifier342     explicit DeathNotifier(const sp<AMessage> &notify)
343         : mNotify(notify) {
344     }
345 
binderDiedandroid::ACodec::DeathNotifier346     virtual void binderDied(const wp<IBinder> &) {
347         mNotify->post();
348     }
349 
serviceDiedandroid::ACodec::DeathNotifier350     virtual void serviceDied(
351             uint64_t /* cookie */,
352             const wp<::android::hidl::base::V1_0::IBase>& /* who */) {
353         mNotify->post();
354     }
355 
356 protected:
~DeathNotifierandroid::ACodec::DeathNotifier357     virtual ~DeathNotifier() {}
358 
359 private:
360     sp<AMessage> mNotify;
361 
362     DISALLOW_EVIL_CONSTRUCTORS(DeathNotifier);
363 };
364 
365 struct ACodec::UninitializedState : public ACodec::BaseState {
366     explicit UninitializedState(ACodec *codec);
367 
368 protected:
369     virtual bool onMessageReceived(const sp<AMessage> &msg);
370     virtual void stateEntered();
371 
372 private:
373     void onSetup(const sp<AMessage> &msg);
374     bool onAllocateComponent(const sp<AMessage> &msg);
375 
376     sp<DeathNotifier> mDeathNotifier;
377 
378     DISALLOW_EVIL_CONSTRUCTORS(UninitializedState);
379 };
380 
381 ////////////////////////////////////////////////////////////////////////////////
382 
383 struct ACodec::LoadedState : public ACodec::BaseState {
384     explicit LoadedState(ACodec *codec);
385 
386 protected:
387     virtual bool onMessageReceived(const sp<AMessage> &msg);
388     virtual void stateEntered();
389 
390 private:
391     friend struct ACodec::UninitializedState;
392 
393     bool onConfigureComponent(const sp<AMessage> &msg);
394     void onCreateInputSurface(const sp<AMessage> &msg);
395     void onSetInputSurface(const sp<AMessage> &msg);
396     void onStart();
397     void onShutdown(bool keepComponentAllocated);
398 
399     status_t setupInputSurface();
400 
401     DISALLOW_EVIL_CONSTRUCTORS(LoadedState);
402 };
403 
404 ////////////////////////////////////////////////////////////////////////////////
405 
406 struct ACodec::LoadedToIdleState : public ACodec::BaseState {
407     explicit LoadedToIdleState(ACodec *codec);
408 
409 protected:
410     virtual bool onMessageReceived(const sp<AMessage> &msg);
411     virtual bool onOMXEvent(OMX_EVENTTYPE event, OMX_U32 data1, OMX_U32 data2);
412     virtual void stateEntered();
413 
414 private:
415     status_t allocateBuffers();
416 
417     DISALLOW_EVIL_CONSTRUCTORS(LoadedToIdleState);
418 };
419 
420 ////////////////////////////////////////////////////////////////////////////////
421 
422 struct ACodec::IdleToExecutingState : public ACodec::BaseState {
423     explicit IdleToExecutingState(ACodec *codec);
424 
425 protected:
426     virtual bool onMessageReceived(const sp<AMessage> &msg);
427     virtual bool onOMXEvent(OMX_EVENTTYPE event, OMX_U32 data1, OMX_U32 data2);
428     virtual void stateEntered();
429 
430 private:
431     DISALLOW_EVIL_CONSTRUCTORS(IdleToExecutingState);
432 };
433 
434 ////////////////////////////////////////////////////////////////////////////////
435 
436 struct ACodec::ExecutingState : public ACodec::BaseState {
437     explicit ExecutingState(ACodec *codec);
438 
439     void submitRegularOutputBuffers();
440     void submitOutputMetaBuffers();
441     void submitOutputBuffers();
442 
443     // Submit output buffers to the decoder, submit input buffers to client
444     // to fill with data.
445     void resume();
446 
447     // Returns true iff input and output buffers are in play.
activeandroid::ACodec::ExecutingState448     bool active() const { return mActive; }
449 
450 protected:
451     virtual PortMode getPortMode(OMX_U32 portIndex);
452     virtual bool onMessageReceived(const sp<AMessage> &msg);
453     virtual void stateEntered();
454 
455     virtual bool onOMXEvent(OMX_EVENTTYPE event, OMX_U32 data1, OMX_U32 data2);
456     virtual bool onOMXFrameRendered(int64_t mediaTimeUs, nsecs_t systemNano);
457 
458 private:
459     bool mActive;
460 
461     DISALLOW_EVIL_CONSTRUCTORS(ExecutingState);
462 };
463 
464 ////////////////////////////////////////////////////////////////////////////////
465 
466 struct ACodec::OutputPortSettingsChangedState : public ACodec::BaseState {
467     explicit OutputPortSettingsChangedState(ACodec *codec);
468 
469 protected:
470     virtual PortMode getPortMode(OMX_U32 portIndex);
471     virtual bool onMessageReceived(const sp<AMessage> &msg);
472     virtual void stateEntered();
473 
474     virtual bool onOMXEvent(OMX_EVENTTYPE event, OMX_U32 data1, OMX_U32 data2);
475     virtual bool onOMXFrameRendered(int64_t mediaTimeUs, nsecs_t systemNano);
476 
477 private:
478     DISALLOW_EVIL_CONSTRUCTORS(OutputPortSettingsChangedState);
479 };
480 
481 ////////////////////////////////////////////////////////////////////////////////
482 
483 struct ACodec::ExecutingToIdleState : public ACodec::BaseState {
484     explicit ExecutingToIdleState(ACodec *codec);
485 
486 protected:
487     virtual bool onMessageReceived(const sp<AMessage> &msg);
488     virtual void stateEntered();
489 
490     virtual bool onOMXEvent(OMX_EVENTTYPE event, OMX_U32 data1, OMX_U32 data2);
491 
492     virtual void onOutputBufferDrained(const sp<AMessage> &msg);
493     virtual void onInputBufferFilled(const sp<AMessage> &msg);
494 
495 private:
496     void changeStateIfWeOwnAllBuffers();
497 
498     bool mComponentNowIdle;
499 
500     DISALLOW_EVIL_CONSTRUCTORS(ExecutingToIdleState);
501 };
502 
503 ////////////////////////////////////////////////////////////////////////////////
504 
505 struct ACodec::IdleToLoadedState : public ACodec::BaseState {
506     explicit IdleToLoadedState(ACodec *codec);
507 
508 protected:
509     virtual bool onMessageReceived(const sp<AMessage> &msg);
510     virtual void stateEntered();
511 
512     virtual bool onOMXEvent(OMX_EVENTTYPE event, OMX_U32 data1, OMX_U32 data2);
513 
514 private:
515     DISALLOW_EVIL_CONSTRUCTORS(IdleToLoadedState);
516 };
517 
518 ////////////////////////////////////////////////////////////////////////////////
519 
520 struct ACodec::FlushingState : public ACodec::BaseState {
521     explicit FlushingState(ACodec *codec);
522 
523 protected:
524     virtual bool onMessageReceived(const sp<AMessage> &msg);
525     virtual void stateEntered();
526 
527     virtual bool onOMXEvent(OMX_EVENTTYPE event, OMX_U32 data1, OMX_U32 data2);
528 
529     virtual void onOutputBufferDrained(const sp<AMessage> &msg);
530     virtual void onInputBufferFilled(const sp<AMessage> &msg);
531 
532 private:
533     bool mFlushComplete[2];
534 
535     void changeStateIfWeOwnAllBuffers();
536 
537     DISALLOW_EVIL_CONSTRUCTORS(FlushingState);
538 };
539 
540 ////////////////////////////////////////////////////////////////////////////////
541 
setWriteFence(int fenceFd,const char * dbg)542 void ACodec::BufferInfo::setWriteFence(int fenceFd, const char *dbg) {
543     if (mFenceFd >= 0) {
544         ALOGW("OVERWRITE OF %s fence %d by write fence %d in %s",
545                 mIsReadFence ? "read" : "write", mFenceFd, fenceFd, dbg);
546     }
547     mFenceFd = fenceFd;
548     mIsReadFence = false;
549 }
550 
setReadFence(int fenceFd,const char * dbg)551 void ACodec::BufferInfo::setReadFence(int fenceFd, const char *dbg) {
552     if (mFenceFd >= 0) {
553         ALOGW("OVERWRITE OF %s fence %d by read fence %d in %s",
554                 mIsReadFence ? "read" : "write", mFenceFd, fenceFd, dbg);
555     }
556     mFenceFd = fenceFd;
557     mIsReadFence = true;
558 }
559 
checkWriteFence(const char * dbg)560 void ACodec::BufferInfo::checkWriteFence(const char *dbg) {
561     if (mFenceFd >= 0 && mIsReadFence) {
562         ALOGD("REUSING read fence %d as write fence in %s", mFenceFd, dbg);
563     }
564 }
565 
checkReadFence(const char * dbg)566 void ACodec::BufferInfo::checkReadFence(const char *dbg) {
567     if (mFenceFd >= 0 && !mIsReadFence) {
568         ALOGD("REUSING write fence %d as read fence in %s", mFenceFd, dbg);
569     }
570 }
571 
572 ////////////////////////////////////////////////////////////////////////////////
573 
ACodec()574 ACodec::ACodec()
575     : mSampleRate(0),
576       mNodeGeneration(0),
577       mAreRenderMetricsEnabled(areRenderMetricsEnabled()),
578       mIsWindowToDisplay(false),
579       mHasPresentFenceTimes(false),
580       mUsingNativeWindow(false),
581       mNativeWindowUsageBits(0),
582       mLastNativeWindowDataSpace(HAL_DATASPACE_UNKNOWN),
583       mIsVideo(false),
584       mIsImage(false),
585       mIsEncoder(false),
586       mFatalError(false),
587       mShutdownInProgress(false),
588       mExplicitShutdown(false),
589       mIsLegacyVP9Decoder(false),
590       mIsStreamCorruptFree(false),
591       mIsLowLatency(false),
592       mEncoderDelay(0),
593       mEncoderPadding(0),
594       mRotationDegrees(0),
595       mChannelMaskPresent(false),
596       mChannelMask(0),
597       mDequeueCounter(0),
598       mMetadataBuffersToSubmit(0),
599       mNumUndequeuedBuffers(0),
600       mRepeatFrameDelayUs(-1LL),
601       mMaxPtsGapUs(0LL),
602       mMaxFps(-1),
603       mFps(-1.0),
604       mCaptureFps(-1.0),
605       mCreateInputBuffersSuspended(false),
606       mTunneled(false),
607       mDescribeColorAspectsIndex((OMX_INDEXTYPE)0),
608       mDescribeHDRStaticInfoIndex((OMX_INDEXTYPE)0),
609       mDescribeHDR10PlusInfoIndex((OMX_INDEXTYPE)0),
610       mStateGeneration(0),
611       mVendorExtensionsStatus(kExtensionsUnchecked) {
612     memset(&mLastHDRStaticInfo, 0, sizeof(mLastHDRStaticInfo));
613 
614     mUninitializedState = new UninitializedState(this);
615     mLoadedState = new LoadedState(this);
616     mLoadedToIdleState = new LoadedToIdleState(this);
617     mIdleToExecutingState = new IdleToExecutingState(this);
618     mExecutingState = new ExecutingState(this);
619 
620     mOutputPortSettingsChangedState =
621         new OutputPortSettingsChangedState(this);
622 
623     mExecutingToIdleState = new ExecutingToIdleState(this);
624     mIdleToLoadedState = new IdleToLoadedState(this);
625     mFlushingState = new FlushingState(this);
626 
627     mPortEOS[kPortIndexInput] = mPortEOS[kPortIndexOutput] = false;
628     mInputEOSResult = OK;
629 
630     mPortMode[kPortIndexInput] = IOMX::kPortModePresetByteBuffer;
631     mPortMode[kPortIndexOutput] = IOMX::kPortModePresetByteBuffer;
632 
633     memset(&mLastNativeWindowCrop, 0, sizeof(mLastNativeWindowCrop));
634 
635     changeState(mUninitializedState);
636 }
637 
~ACodec()638 ACodec::~ACodec() {
639 }
640 
initiateSetup(const sp<AMessage> & msg)641 void ACodec::initiateSetup(const sp<AMessage> &msg) {
642     msg->setWhat(kWhatSetup);
643     msg->setTarget(this);
644     msg->post();
645 }
646 
getBufferChannel()647 std::shared_ptr<BufferChannelBase> ACodec::getBufferChannel() {
648     if (!mBufferChannel) {
649         mBufferChannel = std::make_shared<ACodecBufferChannel>(
650                 new AMessage(kWhatInputBufferFilled, this),
651                 new AMessage(kWhatOutputBufferDrained, this),
652                 new AMessage(kWhatPollForRenderedBuffers, this));
653     }
654     return mBufferChannel;
655 }
656 
signalSetParameters(const sp<AMessage> & params)657 void ACodec::signalSetParameters(const sp<AMessage> &params) {
658     sp<AMessage> msg = new AMessage(kWhatSetParameters, this);
659     msg->setMessage("params", params);
660     msg->post();
661 }
662 
initiateAllocateComponent(const sp<AMessage> & msg)663 void ACodec::initiateAllocateComponent(const sp<AMessage> &msg) {
664     msg->setWhat(kWhatAllocateComponent);
665     msg->setTarget(this);
666     msg->post();
667 }
668 
initiateConfigureComponent(const sp<AMessage> & msg)669 void ACodec::initiateConfigureComponent(const sp<AMessage> &msg) {
670     msg->setWhat(kWhatConfigureComponent);
671     msg->setTarget(this);
672     msg->post();
673 }
674 
setSurface(const sp<Surface> & surface,uint32_t)675 status_t ACodec::setSurface(const sp<Surface> &surface, uint32_t /*generation*/) {
676     sp<AMessage> msg = new AMessage(kWhatSetSurface, this);
677     msg->setObject("surface", surface);
678 
679     sp<AMessage> response;
680     status_t err = msg->postAndAwaitResponse(&response);
681 
682     if (err == OK) {
683         (void)response->findInt32("err", &err);
684     }
685     return err;
686 }
687 
initiateCreateInputSurface()688 void ACodec::initiateCreateInputSurface() {
689     (new AMessage(kWhatCreateInputSurface, this))->post();
690 }
691 
initiateSetInputSurface(const sp<PersistentSurface> & surface)692 void ACodec::initiateSetInputSurface(
693         const sp<PersistentSurface> &surface) {
694     sp<AMessage> msg = new AMessage(kWhatSetInputSurface, this);
695     msg->setObject("input-surface", surface);
696     msg->post();
697 }
698 
signalEndOfInputStream()699 void ACodec::signalEndOfInputStream() {
700     (new AMessage(kWhatSignalEndOfInputStream, this))->post();
701 }
702 
initiateStart()703 void ACodec::initiateStart() {
704     (new AMessage(kWhatStart, this))->post();
705 }
706 
signalFlush()707 void ACodec::signalFlush() {
708     ALOGV("[%s] signalFlush", mComponentName.c_str());
709     (new AMessage(kWhatFlush, this))->post();
710 }
711 
signalResume()712 void ACodec::signalResume() {
713     (new AMessage(kWhatResume, this))->post();
714 }
715 
initiateShutdown(bool keepComponentAllocated)716 void ACodec::initiateShutdown(bool keepComponentAllocated) {
717     sp<AMessage> msg = new AMessage(kWhatShutdown, this);
718     msg->setInt32("keepComponentAllocated", keepComponentAllocated);
719     msg->post();
720     if (!keepComponentAllocated) {
721         // ensure shutdown completes in 3 seconds
722         (new AMessage(kWhatReleaseCodecInstance, this))->post(3000000);
723     }
724 }
725 
signalRequestIDRFrame()726 void ACodec::signalRequestIDRFrame() {
727     (new AMessage(kWhatRequestIDRFrame, this))->post();
728 }
729 
730 // *** NOTE: THE FOLLOWING WORKAROUND WILL BE REMOVED ***
731 // Some codecs may return input buffers before having them processed.
732 // This causes a halt if we already signaled an EOS on the input
733 // port.  For now keep submitting an output buffer if there was an
734 // EOS on the input port, but not yet on the output port.
signalSubmitOutputMetadataBufferIfEOS_workaround()735 void ACodec::signalSubmitOutputMetadataBufferIfEOS_workaround() {
736     if (mPortEOS[kPortIndexInput] && !mPortEOS[kPortIndexOutput] &&
737             mMetadataBuffersToSubmit > 0) {
738         (new AMessage(kWhatSubmitOutputMetadataBufferIfEOS, this))->post();
739     }
740 }
741 
handleSetSurface(const sp<Surface> & surface)742 status_t ACodec::handleSetSurface(const sp<Surface> &surface) {
743     // allow keeping unset surface
744     if (surface == NULL) {
745         if (mNativeWindow != NULL) {
746             ALOGW("cannot unset a surface");
747             return INVALID_OPERATION;
748         }
749         return OK;
750     }
751 
752     // cannot switch from bytebuffers to surface
753     if (mNativeWindow == NULL) {
754         ALOGW("component was not configured with a surface");
755         return INVALID_OPERATION;
756     }
757 
758     ANativeWindow *nativeWindow = surface.get();
759     // if we have not yet started the codec, we can simply set the native window
760     if (mBuffers[kPortIndexInput].size() == 0) {
761         mNativeWindow = surface;
762         initializeFrameTracking();
763         return OK;
764     }
765 
766     // we do not support changing a tunneled surface after start
767     if (mTunneled) {
768         ALOGW("cannot change tunneled surface");
769         return INVALID_OPERATION;
770     }
771 
772     int usageBits = 0;
773     // no need to reconnect as we will not dequeue all buffers
774     status_t err = setupNativeWindowSizeFormatAndUsage(
775             nativeWindow, &usageBits, !storingMetadataInDecodedBuffers());
776     if (err != OK) {
777         return err;
778     }
779 
780     int ignoredFlags = kVideoGrallocUsage;
781     // New output surface is not allowed to add new usage flag except ignored ones.
782     if ((usageBits & ~(mNativeWindowUsageBits | ignoredFlags)) != 0) {
783         ALOGW("cannot change usage from %#x to %#x", mNativeWindowUsageBits, usageBits);
784         return BAD_VALUE;
785     }
786 
787     // get min undequeued count. We cannot switch to a surface that has a higher
788     // undequeued count than we allocated.
789     int minUndequeuedBuffers = 0;
790     err = nativeWindow->query(
791             nativeWindow, NATIVE_WINDOW_MIN_UNDEQUEUED_BUFFERS,
792             &minUndequeuedBuffers);
793     if (err != 0) {
794         ALOGE("NATIVE_WINDOW_MIN_UNDEQUEUED_BUFFERS query failed: %s (%d)",
795                 strerror(-err), -err);
796         return err;
797     }
798     if (minUndequeuedBuffers > (int)mNumUndequeuedBuffers) {
799         ALOGE("new surface holds onto more buffers (%d) than planned for (%zu)",
800                 minUndequeuedBuffers, mNumUndequeuedBuffers);
801         return BAD_VALUE;
802     }
803 
804     // we cannot change the number of output buffers while OMX is running
805     // set up surface to the same count
806     std::vector<BufferInfo> &buffers = mBuffers[kPortIndexOutput];
807     ALOGV("setting up surface for %zu buffers", buffers.size());
808 
809     err = native_window_set_buffer_count(nativeWindow, buffers.size());
810     if (err != 0) {
811         ALOGE("native_window_set_buffer_count failed: %s (%d)", strerror(-err),
812                 -err);
813         return err;
814     }
815 
816     // need to enable allocation when attaching
817     surface->getIGraphicBufferProducer()->allowAllocation(true);
818 
819     // dequeueBuffer cannot time out
820     surface->setDequeueTimeout(-1);
821 
822     // for meta data mode, we move dequeud buffers to the new surface.
823     // for non-meta mode, we must move all registered buffers
824     for (size_t i = 0; i < buffers.size(); ++i) {
825         const BufferInfo &info = buffers[i];
826         // skip undequeued buffers for meta data mode
827         if (storingMetadataInDecodedBuffers()
828                 && info.mStatus == BufferInfo::OWNED_BY_NATIVE_WINDOW) {
829             ALOGV("skipping buffer");
830             continue;
831         }
832         ALOGV("attaching buffer %p", info.mGraphicBuffer->getNativeBuffer());
833 
834         err = surface->attachBuffer(info.mGraphicBuffer->getNativeBuffer());
835         if (err != OK) {
836             ALOGE("failed to attach buffer %p to the new surface: %s (%d)",
837                     info.mGraphicBuffer->getNativeBuffer(),
838                     strerror(-err), -err);
839             return err;
840         }
841     }
842 
843     // cancel undequeued buffers to new surface
844     if (!storingMetadataInDecodedBuffers()) {
845         for (size_t i = 0; i < buffers.size(); ++i) {
846             BufferInfo &info = buffers[i];
847             if (info.mStatus == BufferInfo::OWNED_BY_NATIVE_WINDOW) {
848                 ALOGV("canceling buffer %p", info.mGraphicBuffer->getNativeBuffer());
849                 err = nativeWindow->cancelBuffer(
850                         nativeWindow, info.mGraphicBuffer->getNativeBuffer(), info.mFenceFd);
851                 info.mFenceFd = -1;
852                 if (err != OK) {
853                     ALOGE("failed to cancel buffer %p to the new surface: %s (%d)",
854                             info.mGraphicBuffer->getNativeBuffer(),
855                             strerror(-err), -err);
856                     return err;
857                 }
858             }
859         }
860         // disallow further allocation
861         (void)surface->getIGraphicBufferProducer()->allowAllocation(false);
862     }
863 
864     // push blank buffers to previous window if requested
865     if (mFlags & kFlagPushBlankBuffersToNativeWindowOnShutdown) {
866         pushBlankBuffersToNativeWindow(mNativeWindow.get());
867     }
868 
869     mNativeWindow = nativeWindow;
870     mNativeWindowUsageBits = usageBits;
871     initializeFrameTracking();
872     return OK;
873 }
874 
setPortMode(int32_t portIndex,IOMX::PortMode mode)875 status_t ACodec::setPortMode(int32_t portIndex, IOMX::PortMode mode) {
876     status_t err = mOMXNode->setPortMode(portIndex, mode);
877     if (err != OK) {
878         ALOGE("[%s] setPortMode on %s to %s failed w/ err %d",
879                 mComponentName.c_str(),
880                 portIndex == kPortIndexInput ? "input" : "output",
881                 asString(mode),
882                 err);
883         return err;
884     }
885 
886     mPortMode[portIndex] = mode;
887     return OK;
888 }
889 
allocateBuffersOnPort(OMX_U32 portIndex)890 status_t ACodec::allocateBuffersOnPort(OMX_U32 portIndex) {
891     CHECK(portIndex == kPortIndexInput || portIndex == kPortIndexOutput);
892 
893     CHECK(mAllocator[portIndex] == NULL);
894     CHECK(mBuffers[portIndex].empty());
895 
896     status_t err;
897     if (mNativeWindow != NULL && portIndex == kPortIndexOutput) {
898         if (storingMetadataInDecodedBuffers()) {
899             err = allocateOutputMetadataBuffers();
900         } else {
901             err = allocateOutputBuffersFromNativeWindow();
902         }
903     } else {
904         OMX_PARAM_PORTDEFINITIONTYPE def;
905         InitOMXParams(&def);
906         def.nPortIndex = portIndex;
907 
908         err = mOMXNode->getParameter(
909                 OMX_IndexParamPortDefinition, &def, sizeof(def));
910 
911         if (err == OK) {
912             const IOMX::PortMode &mode = mPortMode[portIndex];
913             size_t bufSize = def.nBufferSize;
914             // Always allocate VideoNativeMetadata if using ANWBuffer.
915             // OMX might use gralloc source internally, but we don't share
916             // metadata buffer with OMX, OMX has its own headers.
917             if (mode == IOMX::kPortModeDynamicANWBuffer) {
918                 bufSize = sizeof(VideoNativeMetadata);
919             } else if (mode == IOMX::kPortModeDynamicNativeHandle) {
920                 bufSize = sizeof(VideoNativeHandleMetadata);
921             }
922 
923             size_t conversionBufferSize = 0;
924 
925             sp<DataConverter> converter = mConverter[portIndex];
926             if (converter != NULL) {
927                 // here we assume conversions of max 4:1, so result fits in int32
928                 if (portIndex == kPortIndexInput) {
929                     conversionBufferSize = converter->sourceSize(bufSize);
930                 } else {
931                     conversionBufferSize = converter->targetSize(bufSize);
932                 }
933             }
934 
935             size_t alignment = 32; // This is the value currently returned by
936                                    // MemoryDealer::getAllocationAlignment().
937                                    // TODO: Fix this when Treble has
938                                    // MemoryHeap/MemoryDealer.
939 
940             ALOGV("[%s] Allocating %u buffers of size %zu (from %u using %s) on %s port",
941                     mComponentName.c_str(),
942                     def.nBufferCountActual, bufSize, def.nBufferSize, asString(mode),
943                     portIndex == kPortIndexInput ? "input" : "output");
944 
945             // verify buffer sizes to avoid overflow in align()
946             if (bufSize == 0 || max(bufSize, conversionBufferSize) > kMaxCodecBufferSize) {
947                 ALOGE("b/22885421");
948                 return NO_MEMORY;
949             }
950 
951             // don't modify bufSize as OMX may not expect it to increase after negotiation
952             size_t alignedSize = align(bufSize, alignment);
953             size_t alignedConvSize = align(conversionBufferSize, alignment);
954             if (def.nBufferCountActual > SIZE_MAX / (alignedSize + alignedConvSize)) {
955                 ALOGE("b/22885421");
956                 return NO_MEMORY;
957             }
958 
959             if (mode != IOMX::kPortModePresetSecureBuffer) {
960                 mAllocator[portIndex] = TAllocator::getService("ashmem");
961                 if (mAllocator[portIndex] == nullptr) {
962                     ALOGE("hidl allocator on port %d is null",
963                             (int)portIndex);
964                     return NO_MEMORY;
965                 }
966                 // TODO: When Treble has MemoryHeap/MemoryDealer, we should
967                 // specify the heap size to be
968                 // def.nBufferCountActual * (alignedSize + alignedConvSize).
969             }
970 
971             const sp<AMessage> &format =
972                     portIndex == kPortIndexInput ? mInputFormat : mOutputFormat;
973             mBuffers[portIndex].reserve(def.nBufferCountActual);
974             for (OMX_U32 i = 0; i < def.nBufferCountActual && err == OK; ++i) {
975                 hidl_memory hidlMemToken;
976                 sp<TMemory> hidlMem;
977                 sp<IMemory> mem;
978 
979                 BufferInfo info;
980                 info.mStatus = BufferInfo::OWNED_BY_US;
981                 info.mFenceFd = -1;
982                 info.mGraphicBuffer = NULL;
983                 info.mNewGraphicBuffer = false;
984 
985                 if (mode == IOMX::kPortModePresetSecureBuffer) {
986                     void *ptr = NULL;
987                     sp<NativeHandle> native_handle;
988                     err = mOMXNode->allocateSecureBuffer(
989                             portIndex, bufSize, &info.mBufferID,
990                             &ptr, &native_handle);
991 
992                     info.mData = (native_handle == NULL)
993                             ? new SecureBuffer(format, ptr, bufSize)
994                             : new SecureBuffer(format, native_handle, bufSize);
995                     info.mCodecData = info.mData;
996                 } else {
997                     bool success;
998                     auto transStatus = mAllocator[portIndex]->allocate(
999                             bufSize,
1000                             [&success, &hidlMemToken](
1001                                     bool s,
1002                                     hidl_memory const& m) {
1003                                 success = s;
1004                                 hidlMemToken = m;
1005                             });
1006 
1007                     if (!transStatus.isOk()) {
1008                         ALOGE("hidl's AshmemAllocator failed at the "
1009                                 "transport: %s",
1010                                 transStatus.description().c_str());
1011                         return NO_MEMORY;
1012                     }
1013                     if (!success) {
1014                         return NO_MEMORY;
1015                     }
1016                     hidlMem = mapMemory(hidlMemToken);
1017                     if (hidlMem == nullptr) {
1018                         return NO_MEMORY;
1019                     }
1020                     err = mOMXNode->useBuffer(
1021                             portIndex, hidlMemToken, &info.mBufferID);
1022 
1023                     if (mode == IOMX::kPortModeDynamicANWBuffer) {
1024                         VideoNativeMetadata* metaData = (VideoNativeMetadata*)(
1025                                 (void*)hidlMem->getPointer());
1026                         metaData->nFenceFd = -1;
1027                     }
1028 
1029                     info.mCodecData = new SharedMemoryBuffer(
1030                             format, hidlMem);
1031                     info.mCodecRef = hidlMem;
1032 
1033                     // if we require conversion, allocate conversion buffer for client use;
1034                     // otherwise, reuse codec buffer
1035                     if (mConverter[portIndex] != NULL) {
1036                         CHECK_GT(conversionBufferSize, (size_t)0);
1037                         bool success;
1038                         mAllocator[portIndex]->allocate(
1039                                 conversionBufferSize,
1040                                 [&success, &hidlMemToken](
1041                                         bool s,
1042                                         hidl_memory const& m) {
1043                                     success = s;
1044                                     hidlMemToken = m;
1045                                 });
1046                         if (!success) {
1047                             return NO_MEMORY;
1048                         }
1049                         hidlMem = mapMemory(hidlMemToken);
1050                         if (hidlMem == nullptr) {
1051                             return NO_MEMORY;
1052                         }
1053                         info.mData = new SharedMemoryBuffer(format, hidlMem);
1054                         info.mMemRef = hidlMem;
1055                     } else {
1056                         info.mData = info.mCodecData;
1057                         info.mMemRef = info.mCodecRef;
1058                     }
1059                 }
1060 
1061                 mBuffers[portIndex].push_back(info);
1062             }
1063         }
1064     }
1065 
1066     if (err != OK) {
1067         return err;
1068     }
1069 
1070     std::vector<ACodecBufferChannel::BufferAndId> array(mBuffers[portIndex].size());
1071     for (size_t i = 0; i < mBuffers[portIndex].size(); ++i) {
1072         array[i] = {mBuffers[portIndex][i].mData, mBuffers[portIndex][i].mBufferID};
1073     }
1074     if (portIndex == kPortIndexInput) {
1075         mBufferChannel->setInputBufferArray(array);
1076     } else if (portIndex == kPortIndexOutput) {
1077         mBufferChannel->setOutputBufferArray(array);
1078     } else {
1079         TRESPASS();
1080     }
1081 
1082     return OK;
1083 }
1084 
setupNativeWindowSizeFormatAndUsage(ANativeWindow * nativeWindow,int * finalUsage,bool reconnect)1085 status_t ACodec::setupNativeWindowSizeFormatAndUsage(
1086         ANativeWindow *nativeWindow /* nonnull */, int *finalUsage /* nonnull */,
1087         bool reconnect) {
1088     OMX_PARAM_PORTDEFINITIONTYPE def;
1089     InitOMXParams(&def);
1090     def.nPortIndex = kPortIndexOutput;
1091 
1092     status_t err = mOMXNode->getParameter(
1093             OMX_IndexParamPortDefinition, &def, sizeof(def));
1094 
1095     if (err != OK) {
1096         return err;
1097     }
1098 
1099     OMX_INDEXTYPE index;
1100     err = mOMXNode->getExtensionIndex(
1101             "OMX.google.android.index.AndroidNativeBufferConsumerUsage",
1102             &index);
1103 
1104     if (err != OK) {
1105         // allow failure
1106         err = OK;
1107     } else {
1108         int usageBits = 0;
1109         if (nativeWindow->query(
1110                 nativeWindow,
1111                 NATIVE_WINDOW_CONSUMER_USAGE_BITS,
1112                 &usageBits) == OK) {
1113             OMX_PARAM_U32TYPE params;
1114             InitOMXParams(&params);
1115             params.nPortIndex = kPortIndexOutput;
1116             params.nU32 = (OMX_U32)usageBits;
1117 
1118             err = mOMXNode->setParameter(index, &params, sizeof(params));
1119 
1120             if (err != OK) {
1121                 ALOGE("Fail to set AndroidNativeBufferConsumerUsage: %d", err);
1122                 return err;
1123             }
1124         }
1125     }
1126 
1127     OMX_U32 usage = 0;
1128     err = mOMXNode->getGraphicBufferUsage(kPortIndexOutput, &usage);
1129     if (err != 0) {
1130         ALOGW("querying usage flags from OMX IL component failed: %d", err);
1131         // XXX: Currently this error is logged, but not fatal.
1132         usage = 0;
1133     }
1134     int omxUsage = usage;
1135 
1136     if (mFlags & kFlagIsGrallocUsageProtected) {
1137         usage |= GRALLOC_USAGE_PROTECTED;
1138     }
1139 
1140     usage |= kVideoGrallocUsage;
1141     *finalUsage = usage;
1142 
1143     memset(&mLastNativeWindowCrop, 0, sizeof(mLastNativeWindowCrop));
1144     mLastNativeWindowDataSpace = HAL_DATASPACE_UNKNOWN;
1145 
1146     ALOGV("gralloc usage: %#x(OMX) => %#x(ACodec)", omxUsage, usage);
1147     return setNativeWindowSizeFormatAndUsage(
1148             nativeWindow,
1149             def.format.video.nFrameWidth,
1150             def.format.video.nFrameHeight,
1151             def.format.video.eColorFormat,
1152             mRotationDegrees,
1153             usage,
1154             reconnect);
1155 }
1156 
configureOutputBuffersFromNativeWindow(OMX_U32 * bufferCount,OMX_U32 * bufferSize,OMX_U32 * minUndequeuedBuffers,bool preregister)1157 status_t ACodec::configureOutputBuffersFromNativeWindow(
1158         OMX_U32 *bufferCount, OMX_U32 *bufferSize,
1159         OMX_U32 *minUndequeuedBuffers, bool preregister) {
1160 
1161     OMX_PARAM_PORTDEFINITIONTYPE def;
1162     InitOMXParams(&def);
1163     def.nPortIndex = kPortIndexOutput;
1164 
1165     status_t err = mOMXNode->getParameter(
1166             OMX_IndexParamPortDefinition, &def, sizeof(def));
1167 
1168     if (err == OK) {
1169         err = setupNativeWindowSizeFormatAndUsage(
1170                 mNativeWindow.get(), &mNativeWindowUsageBits,
1171                 preregister && !mTunneled /* reconnect */);
1172     }
1173     if (err != OK) {
1174         mNativeWindowUsageBits = 0;
1175         return err;
1176     }
1177 
1178     static_cast<Surface *>(mNativeWindow.get())->setDequeueTimeout(-1);
1179 
1180     // Exits here for tunneled video playback codecs -- i.e. skips native window
1181     // buffer allocation step as this is managed by the tunneled OMX omponent
1182     // itself and explicitly sets def.nBufferCountActual to 0.
1183     if (mTunneled) {
1184         ALOGV("Tunneled Playback: skipping native window buffer allocation.");
1185         def.nBufferCountActual = 0;
1186         err = mOMXNode->setParameter(
1187                 OMX_IndexParamPortDefinition, &def, sizeof(def));
1188 
1189         *minUndequeuedBuffers = 0;
1190         *bufferCount = 0;
1191         *bufferSize = 0;
1192         return err;
1193     }
1194 
1195     *minUndequeuedBuffers = 0;
1196     err = mNativeWindow->query(
1197             mNativeWindow.get(), NATIVE_WINDOW_MIN_UNDEQUEUED_BUFFERS,
1198             (int *)minUndequeuedBuffers);
1199 
1200     if (err != 0) {
1201         ALOGE("NATIVE_WINDOW_MIN_UNDEQUEUED_BUFFERS query failed: %s (%d)",
1202                 strerror(-err), -err);
1203         return err;
1204     }
1205 
1206     // FIXME: assume that surface is controlled by app (native window
1207     // returns the number for the case when surface is not controlled by app)
1208     // FIXME2: This means that minUndeqeueudBufs can be 1 larger than reported
1209     // For now, try to allocate 1 more buffer, but don't fail if unsuccessful
1210 
1211     // Use conservative allocation while also trying to reduce starvation
1212     //
1213     // 1. allocate at least nBufferCountMin + minUndequeuedBuffers - that is the
1214     //    minimum needed for the consumer to be able to work
1215     // 2. try to allocate two (2) additional buffers to reduce starvation from
1216     //    the consumer
1217     //    plus an extra buffer to account for incorrect minUndequeuedBufs
1218     for (OMX_U32 extraBuffers = 2 + 1; /* condition inside loop */; extraBuffers--) {
1219         OMX_U32 newBufferCount =
1220             def.nBufferCountMin + *minUndequeuedBuffers + extraBuffers;
1221         def.nBufferCountActual = newBufferCount;
1222         err = mOMXNode->setParameter(
1223                 OMX_IndexParamPortDefinition, &def, sizeof(def));
1224 
1225         if (err == OK) {
1226             *minUndequeuedBuffers += extraBuffers;
1227             break;
1228         }
1229 
1230         ALOGW("[%s] setting nBufferCountActual to %u failed: %d",
1231                 mComponentName.c_str(), newBufferCount, err);
1232         /* exit condition */
1233         if (extraBuffers == 0) {
1234             return err;
1235         }
1236     }
1237 
1238     err = native_window_set_buffer_count(
1239             mNativeWindow.get(), def.nBufferCountActual);
1240 
1241     if (err != 0) {
1242         ALOGE("native_window_set_buffer_count failed: %s (%d)", strerror(-err),
1243                 -err);
1244         return err;
1245     }
1246 
1247     *bufferCount = def.nBufferCountActual;
1248     *bufferSize =  def.nBufferSize;
1249     initializeFrameTracking();
1250     return err;
1251 }
1252 
allocateOutputBuffersFromNativeWindow()1253 status_t ACodec::allocateOutputBuffersFromNativeWindow() {
1254     // This method only handles the non-metadata mode (or simulating legacy
1255     // mode with metadata, which is transparent to ACodec).
1256     CHECK(!storingMetadataInDecodedBuffers());
1257 
1258     OMX_U32 bufferCount, bufferSize, minUndequeuedBuffers;
1259     status_t err = configureOutputBuffersFromNativeWindow(
1260             &bufferCount, &bufferSize, &minUndequeuedBuffers, true /* preregister */);
1261     if (err != 0)
1262         return err;
1263     mNumUndequeuedBuffers = minUndequeuedBuffers;
1264 
1265     static_cast<Surface*>(mNativeWindow.get())
1266             ->getIGraphicBufferProducer()->allowAllocation(true);
1267 
1268     ALOGV("[%s] Allocating %u buffers from a native window of size %u on "
1269          "output port",
1270          mComponentName.c_str(), bufferCount, bufferSize);
1271 
1272     // Dequeue buffers and send them to OMX
1273     mBuffers[kPortIndexOutput].reserve(bufferCount);
1274     for (OMX_U32 i = 0; i < bufferCount; i++) {
1275         ANativeWindowBuffer *buf;
1276         int fenceFd;
1277         err = mNativeWindow->dequeueBuffer(mNativeWindow.get(), &buf, &fenceFd);
1278         if (err != 0) {
1279             ALOGE("dequeueBuffer failed: %s (%d)", strerror(-err), -err);
1280             break;
1281         }
1282 
1283         sp<GraphicBuffer> graphicBuffer(GraphicBuffer::from(buf));
1284         BufferInfo info;
1285         info.mStatus = BufferInfo::OWNED_BY_US;
1286         info.mFenceFd = fenceFd;
1287         info.mIsReadFence = false;
1288         info.mGraphicBuffer = graphicBuffer;
1289         info.mNewGraphicBuffer = false;
1290         info.mDequeuedAt = mDequeueCounter;
1291 
1292         // TODO: We shouln't need to create MediaCodecBuffer. In metadata mode
1293         //       OMX doesn't use the shared memory buffer, but some code still
1294         //       access info.mData. Create an ABuffer as a placeholder.
1295         info.mData = new MediaCodecBuffer(mOutputFormat, new ABuffer(bufferSize));
1296         info.mCodecData = info.mData;
1297 
1298         mBuffers[kPortIndexOutput].push_back(info);
1299 
1300         IOMX::buffer_id bufferId;
1301         err = mOMXNode->useBuffer(kPortIndexOutput, graphicBuffer, &bufferId);
1302         if (err != 0) {
1303             ALOGE("registering GraphicBuffer %u with OMX IL component failed: "
1304                  "%d", i, err);
1305             break;
1306         }
1307 
1308         mBuffers[kPortIndexOutput][i].mBufferID = bufferId;
1309 
1310         ALOGV("[%s] Registered graphic buffer with ID %u (pointer = %p)",
1311              mComponentName.c_str(),
1312              bufferId, graphicBuffer.get());
1313     }
1314 
1315     OMX_U32 cancelStart;
1316     OMX_U32 cancelEnd;
1317 
1318     if (err != OK) {
1319         // If an error occurred while dequeuing we need to cancel any buffers
1320         // that were dequeued. Also cancel all if we're in legacy metadata mode.
1321         cancelStart = 0;
1322         cancelEnd = mBuffers[kPortIndexOutput].size();
1323     } else {
1324         // Return the required minimum undequeued buffers to the native window.
1325         cancelStart = bufferCount - minUndequeuedBuffers;
1326         cancelEnd = bufferCount;
1327     }
1328 
1329     for (OMX_U32 i = cancelStart; i < cancelEnd; i++) {
1330         BufferInfo *info = &mBuffers[kPortIndexOutput][i];
1331         if (info->mStatus == BufferInfo::OWNED_BY_US) {
1332             status_t error = cancelBufferToNativeWindow(info);
1333             if (err == 0) {
1334                 err = error;
1335             }
1336         }
1337     }
1338 
1339     static_cast<Surface*>(mNativeWindow.get())
1340             ->getIGraphicBufferProducer()->allowAllocation(false);
1341 
1342     return err;
1343 }
1344 
allocateOutputMetadataBuffers()1345 status_t ACodec::allocateOutputMetadataBuffers() {
1346     CHECK(storingMetadataInDecodedBuffers());
1347 
1348     OMX_U32 bufferCount, bufferSize, minUndequeuedBuffers;
1349     status_t err = configureOutputBuffersFromNativeWindow(
1350             &bufferCount, &bufferSize, &minUndequeuedBuffers,
1351             mFlags & kFlagPreregisterMetadataBuffers /* preregister */);
1352     if (err != OK)
1353         return err;
1354     mNumUndequeuedBuffers = minUndequeuedBuffers;
1355 
1356     ALOGV("[%s] Allocating %u meta buffers on output port",
1357          mComponentName.c_str(), bufferCount);
1358 
1359     mBuffers[kPortIndexOutput].reserve(bufferCount);
1360     for (OMX_U32 i = 0; i < bufferCount; i++) {
1361         BufferInfo info;
1362         info.mStatus = BufferInfo::OWNED_BY_NATIVE_WINDOW;
1363         info.mFenceFd = -1;
1364         info.mGraphicBuffer = NULL;
1365         info.mNewGraphicBuffer = false;
1366         info.mDequeuedAt = mDequeueCounter;
1367 
1368         info.mData = new MediaCodecBuffer(mOutputFormat, new ABuffer(bufferSize));
1369 
1370         // Initialize fence fd to -1 to avoid warning in freeBuffer().
1371         ((VideoNativeMetadata *)info.mData->base())->nFenceFd = -1;
1372 
1373         info.mCodecData = info.mData;
1374 
1375         err = mOMXNode->useBuffer(kPortIndexOutput, OMXBuffer::sPreset, &info.mBufferID);
1376         mBuffers[kPortIndexOutput].push_back(info);
1377 
1378         ALOGV("[%s] allocated meta buffer with ID %u",
1379                 mComponentName.c_str(), info.mBufferID);
1380     }
1381 
1382     mMetadataBuffersToSubmit = bufferCount - minUndequeuedBuffers;
1383     return err;
1384 }
1385 
submitOutputMetadataBuffer()1386 status_t ACodec::submitOutputMetadataBuffer() {
1387     CHECK(storingMetadataInDecodedBuffers());
1388     if (mMetadataBuffersToSubmit == 0)
1389         return OK;
1390 
1391     BufferInfo *info = dequeueBufferFromNativeWindow();
1392     if (info == NULL) {
1393         return ERROR_IO;
1394     }
1395 
1396     ALOGV("[%s] submitting output meta buffer ID %u for graphic buffer %p",
1397           mComponentName.c_str(), info->mBufferID, info->mGraphicBuffer->handle);
1398 
1399     --mMetadataBuffersToSubmit;
1400     info->checkWriteFence("submitOutputMetadataBuffer");
1401     return fillBuffer(info);
1402 }
1403 
waitForFence(int fd,const char * dbg)1404 status_t ACodec::waitForFence(int fd, const char *dbg ) {
1405     status_t res = OK;
1406     if (fd >= 0) {
1407         sp<Fence> fence = new Fence(fd);
1408         res = fence->wait(IOMX::kFenceTimeoutMs);
1409         ALOGW_IF(res != OK, "FENCE TIMEOUT for %d in %s", fd, dbg);
1410     }
1411     return res;
1412 }
1413 
1414 // static
_asString(BufferInfo::Status s)1415 const char *ACodec::_asString(BufferInfo::Status s) {
1416     switch (s) {
1417         case BufferInfo::OWNED_BY_US:            return "OUR";
1418         case BufferInfo::OWNED_BY_COMPONENT:     return "COMPONENT";
1419         case BufferInfo::OWNED_BY_UPSTREAM:      return "UPSTREAM";
1420         case BufferInfo::OWNED_BY_DOWNSTREAM:    return "DOWNSTREAM";
1421         case BufferInfo::OWNED_BY_NATIVE_WINDOW: return "SURFACE";
1422         case BufferInfo::UNRECOGNIZED:           return "UNRECOGNIZED";
1423         default:                                 return "?";
1424     }
1425 }
1426 
dumpBuffers(OMX_U32 portIndex)1427 void ACodec::dumpBuffers(OMX_U32 portIndex) {
1428     CHECK(portIndex == kPortIndexInput || portIndex == kPortIndexOutput);
1429     ALOGI("[%s] %s port has %zu buffers:", mComponentName.c_str(),
1430             portIndex == kPortIndexInput ? "input" : "output", mBuffers[portIndex].size());
1431     for (size_t i = 0; i < mBuffers[portIndex].size(); ++i) {
1432         const BufferInfo &info = mBuffers[portIndex][i];
1433         ALOGI("  slot %2zu: #%8u %p/%p %s(%d) dequeued:%u",
1434                 i, info.mBufferID, info.mGraphicBuffer.get(),
1435                 info.mGraphicBuffer == NULL ? NULL : info.mGraphicBuffer->getNativeBuffer(),
1436                 _asString(info.mStatus), info.mStatus, info.mDequeuedAt);
1437     }
1438 }
1439 
cancelBufferToNativeWindow(BufferInfo * info)1440 status_t ACodec::cancelBufferToNativeWindow(BufferInfo *info) {
1441     CHECK_EQ((int)info->mStatus, (int)BufferInfo::OWNED_BY_US);
1442 
1443     ALOGV("[%s] Calling cancelBuffer on buffer %u",
1444          mComponentName.c_str(), info->mBufferID);
1445 
1446     info->checkWriteFence("cancelBufferToNativeWindow");
1447     int err = mNativeWindow->cancelBuffer(
1448         mNativeWindow.get(), info->mGraphicBuffer.get(), info->mFenceFd);
1449     info->mFenceFd = -1;
1450 
1451     ALOGW_IF(err != 0, "[%s] can not return buffer %u to native window",
1452             mComponentName.c_str(), info->mBufferID);
1453     // change ownership even if cancelBuffer fails
1454     info->mStatus = BufferInfo::OWNED_BY_NATIVE_WINDOW;
1455 
1456     return err;
1457 }
1458 
onFirstTunnelFrameReady()1459 void ACodec::onFirstTunnelFrameReady() {
1460     mCallback->onFirstTunnelFrameReady();
1461 }
1462 
dequeueBufferFromNativeWindow()1463 ACodec::BufferInfo *ACodec::dequeueBufferFromNativeWindow() {
1464     ANativeWindowBuffer *buf;
1465     CHECK(mNativeWindow.get() != NULL);
1466 
1467     if (mTunneled) {
1468         ALOGW("dequeueBufferFromNativeWindow() should not be called in tunnel"
1469               " video playback mode mode!");
1470         return NULL;
1471     }
1472 
1473     if (mFatalError) {
1474         ALOGW("not dequeuing from native window due to fatal error");
1475         return NULL;
1476     }
1477 
1478     int fenceFd = -1;
1479     do {
1480         status_t err = mNativeWindow->dequeueBuffer(mNativeWindow.get(), &buf, &fenceFd);
1481         if (err != 0) {
1482             ALOGE("dequeueBuffer failed: %s(%d).", asString(err), err);
1483             return NULL;
1484         }
1485 
1486         bool stale = false;
1487         for (size_t i = mBuffers[kPortIndexOutput].size(); i > 0;) {
1488             i--;
1489             BufferInfo *info = &mBuffers[kPortIndexOutput][i];
1490 
1491             if (info->mGraphicBuffer != NULL &&
1492                     info->mGraphicBuffer->handle == buf->handle) {
1493                 // Since consumers can attach buffers to BufferQueues, it is possible
1494                 // that a known yet stale buffer can return from a surface that we
1495                 // once used.  We can simply ignore this as we have already dequeued
1496                 // this buffer properly.  NOTE: this does not eliminate all cases,
1497                 // e.g. it is possible that we have queued the valid buffer to the
1498                 // NW, and a stale copy of the same buffer gets dequeued - which will
1499                 // be treated as the valid buffer by ACodec.
1500                 if (info->mStatus != BufferInfo::OWNED_BY_NATIVE_WINDOW) {
1501                     ALOGI("dequeued stale buffer %p. discarding", buf);
1502                     stale = true;
1503                     break;
1504                 }
1505 
1506                 ALOGV("dequeued buffer #%u with age %u, graphicBuffer %p",
1507                         (unsigned)(info - &mBuffers[kPortIndexOutput][0]),
1508                         mDequeueCounter - info->mDequeuedAt,
1509                         info->mGraphicBuffer->handle);
1510 
1511                 info->mStatus = BufferInfo::OWNED_BY_US;
1512                 info->setWriteFence(fenceFd, "dequeueBufferFromNativeWindow");
1513                 return info;
1514             }
1515         }
1516 
1517         // It is also possible to receive a previously unregistered buffer
1518         // in non-meta mode. These should be treated as stale buffers. The
1519         // same is possible in meta mode, in which case, it will be treated
1520         // as a normal buffer, which is not desirable.
1521         // TODO: fix this.
1522         if (!stale && !storingMetadataInDecodedBuffers()) {
1523             ALOGI("dequeued unrecognized (stale) buffer %p. discarding", buf);
1524             stale = true;
1525         }
1526         if (stale) {
1527             // TODO: detach stale buffer, but there is no API yet to do it.
1528             buf = NULL;
1529         }
1530     } while (buf == NULL);
1531 
1532     // get oldest undequeued buffer
1533     BufferInfo *oldest = NULL;
1534     for (size_t i = mBuffers[kPortIndexOutput].size(); i > 0;) {
1535         i--;
1536         BufferInfo *info = &mBuffers[kPortIndexOutput][i];
1537         if (info->mStatus == BufferInfo::OWNED_BY_NATIVE_WINDOW &&
1538             (oldest == NULL ||
1539              // avoid potential issues from counter rolling over
1540              mDequeueCounter - info->mDequeuedAt >
1541                     mDequeueCounter - oldest->mDequeuedAt)) {
1542             oldest = info;
1543         }
1544     }
1545 
1546     // it is impossible dequeue a buffer when there are no buffers with ANW
1547     CHECK(oldest != NULL);
1548     // it is impossible to dequeue an unknown buffer in non-meta mode, as the
1549     // while loop above does not complete
1550     CHECK(storingMetadataInDecodedBuffers());
1551 
1552     // discard buffer in LRU info and replace with new buffer
1553     oldest->mGraphicBuffer = GraphicBuffer::from(buf);
1554     oldest->mNewGraphicBuffer = true;
1555     oldest->mStatus = BufferInfo::OWNED_BY_US;
1556     oldest->setWriteFence(fenceFd, "dequeueBufferFromNativeWindow for oldest");
1557 
1558     ALOGV("replaced oldest buffer #%u with age %u, graphicBuffer %p",
1559             (unsigned)(oldest - &mBuffers[kPortIndexOutput][0]),
1560             mDequeueCounter - oldest->mDequeuedAt,
1561             oldest->mGraphicBuffer->handle);
1562     return oldest;
1563 }
1564 
initializeFrameTracking()1565 void ACodec::initializeFrameTracking() {
1566     mTrackedFrames.clear();
1567 
1568     int isWindowToDisplay = 0;
1569     mNativeWindow->query(mNativeWindow.get(), NATIVE_WINDOW_QUEUES_TO_WINDOW_COMPOSER,
1570             &isWindowToDisplay);
1571     mIsWindowToDisplay = isWindowToDisplay == 1;
1572     // No frame tracking is needed if we're not sending frames to the display
1573     if (!mIsWindowToDisplay) {
1574         // Return early so we don't call into SurfaceFlinger (requiring permissions)
1575         return;
1576     }
1577 
1578     int hasPresentFenceTimes = 0;
1579     mNativeWindow->query(mNativeWindow.get(), NATIVE_WINDOW_FRAME_TIMESTAMPS_SUPPORTS_PRESENT,
1580             &hasPresentFenceTimes);
1581     mHasPresentFenceTimes = hasPresentFenceTimes == 1;
1582     if (!mHasPresentFenceTimes) {
1583         ALOGI("Using latch times for frame rendered signals - present fences not supported");
1584     }
1585 
1586     status_t err = native_window_enable_frame_timestamps(mNativeWindow.get(), true);
1587     if (err) {
1588         ALOGE("Failed to enable frame timestamps (%d)", err);
1589     }
1590 }
1591 
trackReleasedFrame(int64_t frameId,int64_t mediaTimeUs,int64_t desiredRenderTimeNs)1592 void ACodec::trackReleasedFrame(int64_t frameId, int64_t mediaTimeUs, int64_t desiredRenderTimeNs) {
1593     // If the render time is earlier than now, then we're suggesting it should be rendered ASAP,
1594     // so track the frame as if the desired render time is now.
1595     int64_t nowNs = systemTime(SYSTEM_TIME_MONOTONIC);
1596     if (desiredRenderTimeNs < nowNs) {
1597         desiredRenderTimeNs = nowNs;
1598     }
1599 
1600     // If the render time is more than a second from now, then pretend the frame is supposed to be
1601     // rendered immediately, because that's what SurfaceFlinger heuristics will do. This is a tight
1602     // coupling, but is really the only way to optimize away unnecessary present fence checks in
1603     // processRenderedFrames.
1604     if (desiredRenderTimeNs > nowNs + 1*1000*1000*1000LL) {
1605         desiredRenderTimeNs = nowNs;
1606     }
1607 
1608     // We've just queued a frame to the surface, so keep track of it and later check to see if it is
1609     // actually rendered.
1610     TrackedFrame frame;
1611     frame.id = frameId;
1612     frame.mediaTimeUs = mediaTimeUs;
1613     frame.desiredRenderTimeNs = desiredRenderTimeNs;
1614     mTrackedFrames.push_back(frame);
1615 }
1616 
pollForRenderedFrames()1617 void ACodec::pollForRenderedFrames() {
1618     std::list<RenderedFrameInfo> renderedFrameInfos;
1619     // Scan all frames and check to see if the frames that SHOULD have been rendered by now, have,
1620     // in fact, been rendered.
1621     int64_t nowNs = systemTime(SYSTEM_TIME_MONOTONIC);
1622     while (!mTrackedFrames.empty()) {
1623         TrackedFrame & frame = mTrackedFrames.front();
1624         // Frames that should have been rendered at least 100ms in the past are checked
1625         if (frame.desiredRenderTimeNs > nowNs - 100*1000*1000LL) {
1626             break;
1627         }
1628 
1629         status_t err;
1630         nsecs_t latchOrPresentTimeNs = NATIVE_WINDOW_TIMESTAMP_INVALID;
1631         err = native_window_get_frame_timestamps(mNativeWindow.get(), frame.id,
1632                 /* outRequestedPresentTime */ nullptr, /* outAcquireTime */ nullptr,
1633                 mHasPresentFenceTimes ? nullptr : &latchOrPresentTimeNs, // latch time
1634                 /* outFirstRefreshStartTime */ nullptr, /* outLastRefreshStartTime */ nullptr,
1635                 /* outGpuCompositionDoneTime */ nullptr,
1636                 mHasPresentFenceTimes ? &latchOrPresentTimeNs : nullptr, // display present time,
1637                 /* outDequeueReadyTime */ nullptr, /* outReleaseTime */ nullptr);
1638         if (err) {
1639             ALOGE("Failed to get frame timestamps for %lld: %d", (long long) frame.id, err);
1640         }
1641         // If we don't have a render time by now, then consider the frame as dropped
1642         if (latchOrPresentTimeNs != NATIVE_WINDOW_TIMESTAMP_PENDING &&
1643             latchOrPresentTimeNs != NATIVE_WINDOW_TIMESTAMP_INVALID) {
1644             renderedFrameInfos.push_back(RenderedFrameInfo(frame.mediaTimeUs,
1645                                                            latchOrPresentTimeNs));
1646         }
1647 
1648         mTrackedFrames.pop_front();
1649     }
1650 
1651     if (!renderedFrameInfos.empty()) {
1652         mCallback->onOutputFramesRendered(renderedFrameInfos);
1653     }
1654 }
1655 
freeBuffersOnPort(OMX_U32 portIndex)1656 status_t ACodec::freeBuffersOnPort(OMX_U32 portIndex) {
1657     if (portIndex == kPortIndexInput) {
1658         mBufferChannel->setInputBufferArray({});
1659     } else {
1660         mBufferChannel->setOutputBufferArray({});
1661     }
1662 
1663     status_t err = OK;
1664     for (size_t i = mBuffers[portIndex].size(); i > 0;) {
1665         i--;
1666         status_t err2 = freeBuffer(portIndex, i);
1667         if (err == OK) {
1668             err = err2;
1669         }
1670     }
1671 
1672     mAllocator[portIndex].clear();
1673     return err;
1674 }
1675 
freeOutputBuffersNotOwnedByComponent()1676 status_t ACodec::freeOutputBuffersNotOwnedByComponent() {
1677     status_t err = OK;
1678     for (size_t i = mBuffers[kPortIndexOutput].size(); i > 0;) {
1679         i--;
1680         BufferInfo *info = &mBuffers[kPortIndexOutput][i];
1681 
1682         // At this time some buffers may still be with the component
1683         // or being drained.
1684         if (info->mStatus != BufferInfo::OWNED_BY_COMPONENT &&
1685             info->mStatus != BufferInfo::OWNED_BY_DOWNSTREAM) {
1686             status_t err2 = freeBuffer(kPortIndexOutput, i);
1687             if (err == OK) {
1688                 err = err2;
1689             }
1690         }
1691     }
1692 
1693     return err;
1694 }
1695 
freeBuffer(OMX_U32 portIndex,size_t i)1696 status_t ACodec::freeBuffer(OMX_U32 portIndex, size_t i) {
1697     BufferInfo *info = &mBuffers[portIndex][i];
1698     status_t err = OK;
1699 
1700     // there should not be any fences in the metadata
1701     if (mPortMode[portIndex] == IOMX::kPortModeDynamicANWBuffer && info->mCodecData != NULL
1702             && info->mCodecData->size() >= sizeof(VideoNativeMetadata)) {
1703         int fenceFd = ((VideoNativeMetadata *)info->mCodecData->base())->nFenceFd;
1704         if (fenceFd >= 0) {
1705             ALOGW("unreleased fence (%d) in %s metadata buffer %zu",
1706                     fenceFd, portIndex == kPortIndexInput ? "input" : "output", i);
1707         }
1708     }
1709 
1710     switch (info->mStatus) {
1711         case BufferInfo::OWNED_BY_US:
1712             if (portIndex == kPortIndexOutput && mNativeWindow != NULL) {
1713                 (void)cancelBufferToNativeWindow(info);
1714             }
1715             FALLTHROUGH_INTENDED;
1716 
1717         case BufferInfo::OWNED_BY_NATIVE_WINDOW:
1718             err = mOMXNode->freeBuffer(portIndex, info->mBufferID);
1719             break;
1720 
1721         default:
1722             ALOGE("trying to free buffer not owned by us or ANW (%d)", info->mStatus);
1723             err = FAILED_TRANSACTION;
1724             break;
1725     }
1726 
1727     if (info->mFenceFd >= 0) {
1728         ::close(info->mFenceFd);
1729     }
1730 
1731     // remove buffer even if mOMXNode->freeBuffer fails
1732     mBuffers[portIndex].erase(mBuffers[portIndex].begin() + i);
1733     return err;
1734 }
1735 
findBufferByID(uint32_t portIndex,IOMX::buffer_id bufferID,ssize_t * index)1736 ACodec::BufferInfo *ACodec::findBufferByID(
1737         uint32_t portIndex, IOMX::buffer_id bufferID, ssize_t *index) {
1738     for (size_t i = 0; i < mBuffers[portIndex].size(); ++i) {
1739         BufferInfo *info = &mBuffers[portIndex][i];
1740 
1741         if (info->mBufferID == bufferID) {
1742             if (index != NULL) {
1743                 *index = i;
1744             }
1745             return info;
1746         }
1747     }
1748 
1749     ALOGE("Could not find buffer with ID %u", bufferID);
1750     return NULL;
1751 }
1752 
fillBuffer(BufferInfo * info)1753 status_t ACodec::fillBuffer(BufferInfo *info) {
1754     status_t err;
1755     // Even in dynamic ANW buffer mode, if the graphic buffer is not changing,
1756     // send sPreset instead of the same graphic buffer, so that OMX server
1757     // side doesn't update the meta. In theory it should make no difference,
1758     // however when the same buffer is parcelled again, a new handle could be
1759     // created on server side, and some decoder doesn't recognize the handle
1760     // even if it's the same buffer.
1761     if (!storingMetadataInDecodedBuffers() || !info->mNewGraphicBuffer) {
1762         err = mOMXNode->fillBuffer(
1763             info->mBufferID, OMXBuffer::sPreset, info->mFenceFd);
1764     } else {
1765         err = mOMXNode->fillBuffer(
1766             info->mBufferID, info->mGraphicBuffer, info->mFenceFd);
1767     }
1768 
1769     info->mNewGraphicBuffer = false;
1770     info->mFenceFd = -1;
1771     if (err == OK) {
1772         info->mStatus = BufferInfo::OWNED_BY_COMPONENT;
1773     }
1774     return err;
1775 }
1776 
setComponentRole(bool isEncoder,const char * mime)1777 status_t ACodec::setComponentRole(
1778         bool isEncoder, const char *mime) {
1779     const char *role = GetComponentRole(isEncoder, mime);
1780     if (role == NULL) {
1781         return BAD_VALUE;
1782     }
1783     status_t err = SetComponentRole(mOMXNode, role);
1784     if (err != OK) {
1785         ALOGW("[%s] Failed to set standard component role '%s'.",
1786              mComponentName.c_str(), role);
1787     }
1788     return err;
1789 }
1790 
configureCodec(const char * mime,const sp<AMessage> & msg)1791 status_t ACodec::configureCodec(
1792         const char *mime, const sp<AMessage> &msg) {
1793     int32_t encoder;
1794     if (!msg->findInt32("encoder", &encoder)) {
1795         encoder = false;
1796     }
1797 
1798     sp<AMessage> inputFormat = new AMessage;
1799     sp<AMessage> outputFormat = new AMessage;
1800     mConfigFormat = msg;
1801 
1802     mIsEncoder = encoder;
1803     mIsVideo = !strncasecmp(mime, "video/", 6);
1804     mIsImage = !strncasecmp(mime, "image/", 6);
1805 
1806     mPortMode[kPortIndexInput] = IOMX::kPortModePresetByteBuffer;
1807     mPortMode[kPortIndexOutput] = IOMX::kPortModePresetByteBuffer;
1808 
1809     status_t err = setComponentRole(encoder /* isEncoder */, mime);
1810 
1811     if (err != OK) {
1812         return err;
1813     }
1814 
1815     OMX_VIDEO_CONTROLRATETYPE bitrateMode;
1816     int32_t bitrate = 0, quality;
1817     // FLAC encoder or video encoder in constant quality mode doesn't need a
1818     // bitrate, other encoders do.
1819     if (encoder) {
1820         if (mIsVideo || mIsImage) {
1821             if (!findVideoBitrateControlInfo(msg, &bitrateMode, &bitrate, &quality)) {
1822                 return INVALID_OPERATION;
1823             }
1824         } else if (strcasecmp(mime, MEDIA_MIMETYPE_AUDIO_FLAC)
1825             && !msg->findInt32("bitrate", &bitrate)) {
1826             return INVALID_OPERATION;
1827         }
1828     }
1829 
1830     // propagate bitrate to the output so that the muxer has it
1831     if (encoder && msg->findInt32("bitrate", &bitrate)) {
1832         // Technically ISO spec says that 'bitrate' should be 0 for VBR even though it is the
1833         // average bitrate. We've been setting both bitrate and max-bitrate to this same value.
1834         outputFormat->setInt32("bitrate", bitrate);
1835         outputFormat->setInt32("max-bitrate", bitrate);
1836     }
1837 
1838     int32_t storeMeta;
1839     if (encoder) {
1840         IOMX::PortMode mode = IOMX::kPortModePresetByteBuffer;
1841         if (msg->findInt32("android._input-metadata-buffer-type", &storeMeta)
1842                 && storeMeta != kMetadataBufferTypeInvalid) {
1843             if (storeMeta == kMetadataBufferTypeNativeHandleSource) {
1844                 mode = IOMX::kPortModeDynamicNativeHandle;
1845             } else if (storeMeta == kMetadataBufferTypeANWBuffer ||
1846                     storeMeta == kMetadataBufferTypeGrallocSource) {
1847                 mode = IOMX::kPortModeDynamicANWBuffer;
1848             } else {
1849                 return BAD_VALUE;
1850             }
1851         }
1852         err = setPortMode(kPortIndexInput, mode);
1853         if (err != OK) {
1854             return err;
1855         }
1856 
1857         if (mode != IOMX::kPortModePresetByteBuffer) {
1858             uint32_t usageBits;
1859             if (mOMXNode->getParameter(
1860                     (OMX_INDEXTYPE)OMX_IndexParamConsumerUsageBits,
1861                     &usageBits, sizeof(usageBits)) == OK) {
1862                 inputFormat->setInt32(
1863                         "using-sw-read-often", !!(usageBits & GRALLOC_USAGE_SW_READ_OFTEN));
1864             }
1865         }
1866     }
1867 
1868     int32_t lowLatency = 0;
1869     if (msg->findInt32("low-latency", &lowLatency)) {
1870         err = setLowLatency(lowLatency);
1871         if (err != OK) {
1872             return err;
1873         }
1874     }
1875 
1876     int32_t prependSPSPPS = 0;
1877     if (encoder && mIsVideo
1878             && msg->findInt32("prepend-sps-pps-to-idr-frames", &prependSPSPPS)
1879             && prependSPSPPS != 0) {
1880         OMX_INDEXTYPE index;
1881         err = mOMXNode->getExtensionIndex(
1882                 "OMX.google.android.index.prependSPSPPSToIDRFrames", &index);
1883 
1884         if (err == OK) {
1885             PrependSPSPPSToIDRFramesParams params;
1886             InitOMXParams(&params);
1887             params.bEnable = OMX_TRUE;
1888 
1889             err = mOMXNode->setParameter(index, &params, sizeof(params));
1890         }
1891 
1892         if (err != OK) {
1893             ALOGE("Encoder could not be configured to emit SPS/PPS before "
1894                   "IDR frames. (err %d)", err);
1895 
1896             return err;
1897         }
1898     }
1899 
1900     // Only enable metadata mode on encoder output if encoder can prepend
1901     // sps/pps to idr frames, since in metadata mode the bitstream is in an
1902     // opaque handle, to which we don't have access.
1903     if (encoder && mIsVideo) {
1904         OMX_BOOL enable = (OMX_BOOL) (prependSPSPPS
1905             && msg->findInt32("android._store-metadata-in-buffers-output", &storeMeta)
1906             && storeMeta != 0);
1907         if (mFlags & kFlagIsSecure) {
1908             enable = OMX_TRUE;
1909         }
1910 
1911         err = setPortMode(kPortIndexOutput, enable ?
1912                 IOMX::kPortModePresetSecureBuffer : IOMX::kPortModePresetByteBuffer);
1913         if (err != OK) {
1914             return err;
1915         }
1916 
1917         if (!msg->findInt64(
1918                 KEY_REPEAT_PREVIOUS_FRAME_AFTER, &mRepeatFrameDelayUs)) {
1919             mRepeatFrameDelayUs = -1LL;
1920         }
1921 
1922         if (!msg->findDouble("time-lapse-fps", &mCaptureFps)) {
1923             float captureRate;
1924             if (msg->findAsFloat(KEY_CAPTURE_RATE, &captureRate)) {
1925                 mCaptureFps = captureRate;
1926             } else {
1927                 mCaptureFps = -1.0;
1928             }
1929         }
1930 
1931         if (!msg->findInt32(
1932                 KEY_CREATE_INPUT_SURFACE_SUSPENDED,
1933                 (int32_t*)&mCreateInputBuffersSuspended)) {
1934             mCreateInputBuffersSuspended = false;
1935         }
1936     }
1937 
1938     if (encoder && (mIsVideo || mIsImage)) {
1939         // only allow 32-bit value, since we pass it as U32 to OMX.
1940         if (!msg->findInt64(KEY_MAX_PTS_GAP_TO_ENCODER, &mMaxPtsGapUs)) {
1941             mMaxPtsGapUs = 0LL;
1942         } else if (mMaxPtsGapUs > INT32_MAX || mMaxPtsGapUs < INT32_MIN) {
1943             ALOGW("Unsupported value for max pts gap %lld", (long long) mMaxPtsGapUs);
1944             mMaxPtsGapUs = 0LL;
1945         }
1946 
1947         if (!msg->findFloat(KEY_MAX_FPS_TO_ENCODER, &mMaxFps)) {
1948             mMaxFps = -1;
1949         }
1950 
1951         // notify GraphicBufferSource to allow backward frames
1952         if (mMaxPtsGapUs < 0LL) {
1953             mMaxFps = -1;
1954         }
1955     }
1956 
1957     // NOTE: we only use native window for video decoders
1958     sp<RefBase> obj;
1959     bool haveNativeWindow = msg->findObject("native-window", &obj)
1960             && obj != NULL && mIsVideo && !encoder;
1961     mUsingNativeWindow = haveNativeWindow;
1962     if (mIsVideo && !encoder) {
1963         inputFormat->setInt32("adaptive-playback", false);
1964 
1965         int32_t usageProtected;
1966         if (msg->findInt32("protected", &usageProtected) && usageProtected) {
1967             if (!haveNativeWindow) {
1968                 ALOGE("protected output buffers must be sent to an ANativeWindow");
1969                 return PERMISSION_DENIED;
1970             }
1971             mFlags |= kFlagIsGrallocUsageProtected;
1972             mFlags |= kFlagPushBlankBuffersToNativeWindowOnShutdown;
1973         }
1974     }
1975     if (mFlags & kFlagIsSecure) {
1976         // use native_handles for secure input buffers
1977         err = setPortMode(kPortIndexInput, IOMX::kPortModePresetSecureBuffer);
1978 
1979         if (err != OK) {
1980             ALOGI("falling back to non-native_handles");
1981             setPortMode(kPortIndexInput, IOMX::kPortModePresetByteBuffer);
1982             err = OK; // ignore error for now
1983         }
1984 
1985         OMX_INDEXTYPE index;
1986         if (mOMXNode->getExtensionIndex(
1987                 "OMX.google.android.index.preregisterMetadataBuffers", &index) == OK) {
1988             OMX_CONFIG_BOOLEANTYPE param;
1989             InitOMXParams(&param);
1990             param.bEnabled = OMX_FALSE;
1991             if (mOMXNode->getParameter(index, &param, sizeof(param)) == OK) {
1992                 if (param.bEnabled == OMX_TRUE) {
1993                     mFlags |= kFlagPreregisterMetadataBuffers;
1994                 }
1995             }
1996         }
1997     }
1998     if (haveNativeWindow) {
1999         sp<ANativeWindow> nativeWindow =
2000             static_cast<ANativeWindow *>(static_cast<Surface *>(obj.get()));
2001 
2002         // START of temporary support for automatic FRC - THIS WILL BE REMOVED
2003         int32_t autoFrc;
2004         if (msg->findInt32("auto-frc", &autoFrc)) {
2005             bool enabled = autoFrc;
2006             OMX_CONFIG_BOOLEANTYPE config;
2007             InitOMXParams(&config);
2008             config.bEnabled = (OMX_BOOL)enabled;
2009             status_t temp = mOMXNode->setConfig(
2010                     (OMX_INDEXTYPE)OMX_IndexConfigAutoFramerateConversion,
2011                     &config, sizeof(config));
2012             if (temp == OK) {
2013                 outputFormat->setInt32("auto-frc", enabled);
2014             } else if (enabled) {
2015                 ALOGI("codec does not support requested auto-frc (err %d)", temp);
2016             }
2017         }
2018         // END of temporary support for automatic FRC
2019 
2020         int32_t tunneled;
2021         if (msg->findInt32("feature-tunneled-playback", &tunneled) &&
2022             tunneled != 0) {
2023             ALOGI("Configuring TUNNELED video playback.");
2024             mTunneled = true;
2025 
2026             int32_t audioHwSync = 0;
2027             if (!msg->findInt32("audio-hw-sync", &audioHwSync)) {
2028                 ALOGW("No Audio HW Sync provided for video tunnel");
2029             }
2030             err = configureTunneledVideoPlayback(audioHwSync, nativeWindow);
2031             if (err != OK) {
2032                 ALOGE("configureTunneledVideoPlayback(%d,%p) failed!",
2033                         audioHwSync, nativeWindow.get());
2034                 return err;
2035             }
2036 
2037             int32_t maxWidth = 0, maxHeight = 0;
2038             if (msg->findInt32("max-width", &maxWidth) &&
2039                     msg->findInt32("max-height", &maxHeight)) {
2040 
2041                 err = mOMXNode->prepareForAdaptivePlayback(
2042                         kPortIndexOutput, OMX_TRUE, maxWidth, maxHeight);
2043                 if (err != OK) {
2044                     ALOGW("[%s] prepareForAdaptivePlayback failed w/ err %d",
2045                             mComponentName.c_str(), err);
2046                     // allow failure
2047                     err = OK;
2048                 } else {
2049                     inputFormat->setInt32("max-width", maxWidth);
2050                     inputFormat->setInt32("max-height", maxHeight);
2051                     inputFormat->setInt32("adaptive-playback", true);
2052                 }
2053             }
2054         } else {
2055             ALOGV("Configuring CPU controlled video playback.");
2056             mTunneled = false;
2057 
2058             // Explicity reset the sideband handle of the window for
2059             // non-tunneled video in case the window was previously used
2060             // for a tunneled video playback.
2061             err = native_window_set_sideband_stream(nativeWindow.get(), NULL);
2062             if (err != OK) {
2063                 ALOGE("set_sideband_stream(NULL) failed! (err %d).", err);
2064                 return err;
2065             }
2066 
2067             err = setPortMode(kPortIndexOutput, IOMX::kPortModeDynamicANWBuffer);
2068             if (err != OK) {
2069                 // if adaptive playback has been requested, try JB fallback
2070                 // NOTE: THIS FALLBACK MECHANISM WILL BE REMOVED DUE TO ITS
2071                 // LARGE MEMORY REQUIREMENT
2072 
2073                 // we will not do adaptive playback on software accessed
2074                 // surfaces as they never had to respond to changes in the
2075                 // crop window, and we don't trust that they will be able to.
2076                 int usageBits = 0;
2077                 bool canDoAdaptivePlayback;
2078 
2079                 if (nativeWindow->query(
2080                         nativeWindow.get(),
2081                         NATIVE_WINDOW_CONSUMER_USAGE_BITS,
2082                         &usageBits) != OK) {
2083                     canDoAdaptivePlayback = false;
2084                 } else {
2085                     canDoAdaptivePlayback =
2086                         (usageBits &
2087                                 (GRALLOC_USAGE_SW_READ_MASK |
2088                                  GRALLOC_USAGE_SW_WRITE_MASK)) == 0;
2089                 }
2090 
2091                 int32_t maxWidth = 0, maxHeight = 0;
2092                 if (canDoAdaptivePlayback &&
2093                         msg->findInt32("max-width", &maxWidth) &&
2094                         msg->findInt32("max-height", &maxHeight)) {
2095                     ALOGV("[%s] prepareForAdaptivePlayback(%dx%d)",
2096                             mComponentName.c_str(), maxWidth, maxHeight);
2097 
2098                     err = mOMXNode->prepareForAdaptivePlayback(
2099                             kPortIndexOutput, OMX_TRUE, maxWidth, maxHeight);
2100                     ALOGW_IF(err != OK,
2101                             "[%s] prepareForAdaptivePlayback failed w/ err %d",
2102                             mComponentName.c_str(), err);
2103 
2104                     if (err == OK) {
2105                         inputFormat->setInt32("max-width", maxWidth);
2106                         inputFormat->setInt32("max-height", maxHeight);
2107                         inputFormat->setInt32("adaptive-playback", true);
2108                     }
2109                 }
2110                 // allow failure
2111                 err = OK;
2112             } else {
2113                 ALOGV("[%s] setPortMode on output to %s succeeded",
2114                         mComponentName.c_str(), asString(IOMX::kPortModeDynamicANWBuffer));
2115                 CHECK(storingMetadataInDecodedBuffers());
2116                 inputFormat->setInt32("adaptive-playback", true);
2117             }
2118 
2119             int32_t push;
2120             if (msg->findInt32("push-blank-buffers-on-shutdown", &push)
2121                     && push != 0) {
2122                 mFlags |= kFlagPushBlankBuffersToNativeWindowOnShutdown;
2123             }
2124         }
2125 
2126         int32_t rotationDegrees;
2127         if (msg->findInt32("rotation-degrees", &rotationDegrees)) {
2128             mRotationDegrees = rotationDegrees;
2129         } else {
2130             mRotationDegrees = 0;
2131         }
2132     }
2133 
2134     AudioEncoding pcmEncoding = kAudioEncodingPcm16bit;
2135     (void)msg->findInt32("pcm-encoding", (int32_t*)&pcmEncoding);
2136     // invalid encodings will default to PCM-16bit in setupRawAudioFormat.
2137 
2138     if (mIsVideo || mIsImage) {
2139         // determine need for software renderer
2140         bool usingSwRenderer = false;
2141         if (haveNativeWindow) {
2142             bool requiresSwRenderer = false;
2143             OMX_PARAM_U32TYPE param;
2144             InitOMXParams(&param);
2145             param.nPortIndex = kPortIndexOutput;
2146 
2147             status_t err = mOMXNode->getParameter(
2148                     (OMX_INDEXTYPE)OMX_IndexParamVideoAndroidRequiresSwRenderer,
2149                     &param, sizeof(param));
2150 
2151             if (err == OK && param.nU32 == 1) {
2152                 requiresSwRenderer = true;
2153             }
2154 
2155             if (mComponentName.startsWith("OMX.google.") || requiresSwRenderer) {
2156                 usingSwRenderer = true;
2157                 haveNativeWindow = false;
2158                 (void)setPortMode(kPortIndexOutput, IOMX::kPortModePresetByteBuffer);
2159             } else if (!storingMetadataInDecodedBuffers()) {
2160                 err = setPortMode(kPortIndexOutput, IOMX::kPortModePresetANWBuffer);
2161                 if (err != OK) {
2162                     return err;
2163                 }
2164             }
2165 
2166         }
2167 
2168 
2169         if (encoder) {
2170             err = setupVideoEncoder(mime, msg, outputFormat, inputFormat);
2171         } else {
2172             err = setupVideoDecoder(mime, msg, haveNativeWindow, usingSwRenderer, outputFormat);
2173         }
2174 
2175         if (err != OK) {
2176             return err;
2177         }
2178 
2179         if (haveNativeWindow) {
2180             mNativeWindow = static_cast<Surface *>(obj.get());
2181 
2182             // fallback for devices that do not handle flex-YUV for native buffers
2183             int32_t requestedColorFormat = OMX_COLOR_FormatUnused;
2184             if (msg->findInt32("color-format", &requestedColorFormat) &&
2185                     requestedColorFormat == OMX_COLOR_FormatYUV420Flexible) {
2186                 status_t err = getPortFormat(kPortIndexOutput, outputFormat);
2187                 if (err != OK) {
2188                     return err;
2189                 }
2190                 int32_t colorFormat = OMX_COLOR_FormatUnused;
2191                 OMX_U32 flexibleEquivalent = OMX_COLOR_FormatUnused;
2192                 if (!outputFormat->findInt32("color-format", &colorFormat)) {
2193                     ALOGE("output port did not have a color format (wrong domain?)");
2194                     return BAD_VALUE;
2195                 }
2196                 ALOGD("[%s] Requested output format %#x and got %#x.",
2197                         mComponentName.c_str(), requestedColorFormat, colorFormat);
2198                 if (!IsFlexibleColorFormat(
2199                         mOMXNode, colorFormat, haveNativeWindow, &flexibleEquivalent)
2200                         || flexibleEquivalent != (OMX_U32)requestedColorFormat) {
2201                     // device did not handle flex-YUV request for native window, fall back
2202                     // to SW renderer
2203                     ALOGI("[%s] Falling back to software renderer", mComponentName.c_str());
2204                     mNativeWindow.clear();
2205                     mNativeWindowUsageBits = 0;
2206                     haveNativeWindow = false;
2207                     usingSwRenderer = true;
2208                     // TODO: implement adaptive-playback support for bytebuffer mode.
2209                     // This is done by SW codecs, but most HW codecs don't support it.
2210                     err = setPortMode(kPortIndexOutput, IOMX::kPortModePresetByteBuffer);
2211                     inputFormat->setInt32("adaptive-playback", false);
2212                     if (mFlags & kFlagIsGrallocUsageProtected) {
2213                         // fallback is not supported for protected playback
2214                         err = PERMISSION_DENIED;
2215                     } else if (err == OK) {
2216                         err = setupVideoDecoder(
2217                                 mime, msg, haveNativeWindow, usingSwRenderer, outputFormat);
2218                     }
2219                 }
2220             }
2221         }
2222 
2223         if (usingSwRenderer) {
2224             outputFormat->setInt32("using-sw-renderer", 1);
2225         }
2226     } else if (!strcasecmp(mime, MEDIA_MIMETYPE_AUDIO_MPEG) ||
2227         !strcasecmp(mime, MEDIA_MIMETYPE_AUDIO_MPEG_LAYER_II)) {
2228         int32_t numChannels, sampleRate;
2229         if (!msg->findInt32("channel-count", &numChannels)
2230                 || !msg->findInt32("sample-rate", &sampleRate)) {
2231             // Since we did not always check for these, leave them optional
2232             // and have the decoder figure it all out.
2233             err = OK;
2234         } else {
2235             err = setupRawAudioFormat(
2236                     encoder ? kPortIndexInput : kPortIndexOutput,
2237                     sampleRate,
2238                     numChannels);
2239         }
2240     } else if (!strcasecmp(mime, MEDIA_MIMETYPE_AUDIO_AAC)) {
2241         int32_t numChannels, sampleRate;
2242         if (!msg->findInt32("channel-count", &numChannels)
2243                 || !msg->findInt32("sample-rate", &sampleRate)) {
2244             err = INVALID_OPERATION;
2245         } else {
2246             int32_t isADTS, aacProfile;
2247             int32_t sbrMode;
2248             int32_t maxOutputChannelCount;
2249             int32_t pcmLimiterEnable;
2250             drcParams_t drc;
2251             if (!msg->findInt32("is-adts", &isADTS)) {
2252                 isADTS = 0;
2253             }
2254             if (!msg->findInt32("aac-profile", &aacProfile)) {
2255                 aacProfile = OMX_AUDIO_AACObjectNull;
2256             }
2257             if (!msg->findInt32("aac-sbr-mode", &sbrMode)) {
2258                 sbrMode = -1;
2259             }
2260 
2261             if (!msg->findInt32("aac-max-output-channel_count", &maxOutputChannelCount)) {
2262                 // check non AAC-specific key
2263                 if (!msg->findInt32("max-output-channel-count", &maxOutputChannelCount)) {
2264                     maxOutputChannelCount = -1;
2265                 }
2266             }
2267             if (!msg->findInt32("aac-pcm-limiter-enable", &pcmLimiterEnable)) {
2268                 // value is unknown
2269                 pcmLimiterEnable = -1;
2270             }
2271             if (!msg->findInt32("aac-encoded-target-level", &drc.encodedTargetLevel)) {
2272                 // value is unknown
2273                 drc.encodedTargetLevel = -1;
2274             }
2275             if (!msg->findInt32("aac-drc-cut-level", &drc.drcCut)) {
2276                 // value is unknown
2277                 drc.drcCut = -1;
2278             }
2279             if (!msg->findInt32("aac-drc-boost-level", &drc.drcBoost)) {
2280                 // value is unknown
2281                 drc.drcBoost = -1;
2282             }
2283             if (!msg->findInt32("aac-drc-heavy-compression", &drc.heavyCompression)) {
2284                 // value is unknown
2285                 drc.heavyCompression = -1;
2286             }
2287             if (!msg->findInt32("aac-target-ref-level", &drc.targetRefLevel)) {
2288                 // value is unknown
2289                 drc.targetRefLevel = -2;
2290             }
2291             if (!msg->findInt32("aac-drc-effect-type", &drc.effectType)) {
2292                 // value is unknown
2293                 drc.effectType = -2; // valid values are -1 and over
2294             }
2295             if (!msg->findInt32("aac-drc-album-mode", &drc.albumMode)) {
2296                 // value is unknown
2297                 drc.albumMode = -1; // valid values are 0 and 1
2298             }
2299             if (!msg->findInt32("aac-drc-output-loudness", &drc.outputLoudness)) {
2300                 // value is unknown
2301                 drc.outputLoudness = -1;
2302             }
2303 
2304             err = setupAACCodec(
2305                     encoder, numChannels, sampleRate, bitrate, aacProfile,
2306                     isADTS != 0, sbrMode, maxOutputChannelCount, drc,
2307                     pcmLimiterEnable);
2308         }
2309     } else if (!strcasecmp(mime, MEDIA_MIMETYPE_AUDIO_AMR_NB)) {
2310         err = setupAMRCodec(encoder, false /* isWAMR */, bitrate);
2311     } else if (!strcasecmp(mime, MEDIA_MIMETYPE_AUDIO_AMR_WB)) {
2312         err = setupAMRCodec(encoder, true /* isWAMR */, bitrate);
2313     } else if (!strcasecmp(mime, MEDIA_MIMETYPE_AUDIO_G711_ALAW)
2314             || !strcasecmp(mime, MEDIA_MIMETYPE_AUDIO_G711_MLAW)) {
2315         // These are PCM-like formats with a fixed sample rate but
2316         // a variable number of channels.
2317 
2318         int32_t numChannels;
2319         if (!msg->findInt32("channel-count", &numChannels)) {
2320             err = INVALID_OPERATION;
2321         } else {
2322             int32_t sampleRate;
2323             if (!msg->findInt32("sample-rate", &sampleRate)) {
2324                 sampleRate = 8000;
2325             }
2326             err = setupG711Codec(encoder, sampleRate, numChannels);
2327         }
2328     } else if (!strcasecmp(mime, MEDIA_MIMETYPE_AUDIO_OPUS)) {
2329         int32_t numChannels = 1, sampleRate = 48000;
2330         if (msg->findInt32("channel-count", &numChannels) &&
2331             msg->findInt32("sample-rate", &sampleRate)) {
2332             err = setupOpusCodec(encoder, sampleRate, numChannels);
2333         }
2334     } else if (!strcasecmp(mime, MEDIA_MIMETYPE_AUDIO_FLAC)) {
2335         // numChannels needs to be set to properly communicate PCM values.
2336         int32_t numChannels = 2, sampleRate = 44100, compressionLevel = -1;
2337         if (encoder &&
2338                 (!msg->findInt32("channel-count", &numChannels)
2339                         || !msg->findInt32("sample-rate", &sampleRate))) {
2340             ALOGE("missing channel count or sample rate for FLAC encoder");
2341             err = INVALID_OPERATION;
2342         } else {
2343             if (encoder) {
2344                 if (!msg->findInt32(
2345                             "complexity", &compressionLevel) &&
2346                     !msg->findInt32(
2347                             "flac-compression-level", &compressionLevel)) {
2348                     compressionLevel = 5; // default FLAC compression level
2349                 } else if (compressionLevel < 0) {
2350                     ALOGW("compression level %d outside [0..8] range, "
2351                           "using 0",
2352                           compressionLevel);
2353                     compressionLevel = 0;
2354                 } else if (compressionLevel > 8) {
2355                     ALOGW("compression level %d outside [0..8] range, "
2356                           "using 8",
2357                           compressionLevel);
2358                     compressionLevel = 8;
2359                 }
2360             }
2361             err = setupFlacCodec(
2362                     encoder, numChannels, sampleRate, compressionLevel, pcmEncoding);
2363         }
2364     } else if (!strcasecmp(mime, MEDIA_MIMETYPE_AUDIO_RAW)) {
2365         int32_t numChannels, sampleRate;
2366         if (encoder
2367                 || !msg->findInt32("channel-count", &numChannels)
2368                 || !msg->findInt32("sample-rate", &sampleRate)) {
2369             err = INVALID_OPERATION;
2370         } else {
2371             err = setupRawAudioFormat(kPortIndexInput, sampleRate, numChannels, pcmEncoding);
2372         }
2373     } else if (!strcasecmp(mime, MEDIA_MIMETYPE_AUDIO_AC3)) {
2374         int32_t numChannels;
2375         int32_t sampleRate;
2376         if (!msg->findInt32("channel-count", &numChannels)
2377                 || !msg->findInt32("sample-rate", &sampleRate)) {
2378             err = INVALID_OPERATION;
2379         } else {
2380             err = setupAC3Codec(encoder, numChannels, sampleRate);
2381         }
2382     } else if (!strcasecmp(mime, MEDIA_MIMETYPE_AUDIO_EAC3)) {
2383         int32_t numChannels;
2384         int32_t sampleRate;
2385         if (!msg->findInt32("channel-count", &numChannels)
2386                 || !msg->findInt32("sample-rate", &sampleRate)) {
2387             err = INVALID_OPERATION;
2388         } else {
2389             err = setupEAC3Codec(encoder, numChannels, sampleRate);
2390         }
2391      } else if (!strcasecmp(mime, MEDIA_MIMETYPE_AUDIO_AC4)) {
2392         int32_t numChannels;
2393         int32_t sampleRate;
2394         if (!msg->findInt32("channel-count", &numChannels)
2395                 || !msg->findInt32("sample-rate", &sampleRate)) {
2396             err = INVALID_OPERATION;
2397         } else {
2398             err = setupAC4Codec(encoder, numChannels, sampleRate);
2399         }
2400     }
2401 
2402     if (err != OK) {
2403         return err;
2404     }
2405 
2406     if (!msg->findInt32("encoder-delay", &mEncoderDelay)) {
2407         mEncoderDelay = 0;
2408     }
2409 
2410     if (!msg->findInt32("encoder-padding", &mEncoderPadding)) {
2411         mEncoderPadding = 0;
2412     }
2413 
2414     if (msg->findInt32("channel-mask", &mChannelMask)) {
2415         mChannelMaskPresent = true;
2416     } else {
2417         mChannelMaskPresent = false;
2418     }
2419 
2420     int32_t isCorruptFree = 0;
2421     if (msg->findInt32("corrupt-free", &isCorruptFree)) {
2422         mIsStreamCorruptFree = isCorruptFree == 1 ? true : false;
2423         ALOGV("corrupt-free=[%d]", mIsStreamCorruptFree);
2424     }
2425 
2426     int32_t maxInputSize;
2427     if (msg->findInt32("max-input-size", &maxInputSize)) {
2428         err = setMinBufferSize(kPortIndexInput, (size_t)maxInputSize);
2429         err = OK; // ignore error
2430     } else if (!strcmp("OMX.Nvidia.aac.decoder", mComponentName.c_str())) {
2431         err = setMinBufferSize(kPortIndexInput, 8192);  // XXX
2432         err = OK; // ignore error
2433     }
2434 
2435     int32_t priority;
2436     if (msg->findInt32("priority", &priority)) {
2437         err = setPriority(priority);
2438         err = OK; // ignore error
2439     }
2440 
2441     int32_t rateInt = -1;
2442     float rateFloat = -1;
2443     if (!msg->findFloat("operating-rate", &rateFloat)) {
2444         msg->findInt32("operating-rate", &rateInt);
2445         rateFloat = (float)rateInt;  // 16MHz (FLINTMAX) is OK for upper bound.
2446     }
2447     if (rateFloat > 0) {
2448         err = setOperatingRate(rateFloat, mIsVideo);
2449         err = OK; // ignore errors
2450     }
2451 
2452     if (err == OK) {
2453         err = setVendorParameters(msg);
2454         if (err != OK) {
2455             return err;
2456         }
2457     }
2458 
2459     // NOTE: both mBaseOutputFormat and mOutputFormat are outputFormat to signal first frame.
2460     mBaseOutputFormat = outputFormat;
2461     mLastOutputFormat.clear();
2462 
2463     err = getPortFormat(kPortIndexInput, inputFormat);
2464     if (err == OK) {
2465         err = getPortFormat(kPortIndexOutput, outputFormat);
2466         if (err == OK) {
2467             mInputFormat = inputFormat;
2468             mOutputFormat = outputFormat;
2469         }
2470     }
2471 
2472     // create data converters if needed
2473     if (!mIsVideo && !mIsImage && err == OK) {
2474         AudioEncoding codecPcmEncoding = kAudioEncodingPcm16bit;
2475         if (encoder) {
2476             (void)mInputFormat->findInt32("pcm-encoding", (int32_t*)&codecPcmEncoding);
2477             mConverter[kPortIndexInput] = AudioConverter::Create(pcmEncoding, codecPcmEncoding);
2478             if (mConverter[kPortIndexInput] != NULL) {
2479                 ALOGD("%s: encoder %s input format pcm encoding converter from %d to %d",
2480                         __func__, mComponentName.c_str(), pcmEncoding, codecPcmEncoding);
2481                 mInputFormat->setInt32("pcm-encoding", pcmEncoding);
2482             }
2483         } else {
2484             (void)mOutputFormat->findInt32("pcm-encoding", (int32_t*)&codecPcmEncoding);
2485             mConverter[kPortIndexOutput] = AudioConverter::Create(codecPcmEncoding, pcmEncoding);
2486             if (mConverter[kPortIndexOutput] != NULL) {
2487                 ALOGD("%s: decoder %s output format pcm encoding converter from %d to %d",
2488                         __func__, mComponentName.c_str(), codecPcmEncoding, pcmEncoding);
2489                 mOutputFormat->setInt32("pcm-encoding", pcmEncoding);
2490             }
2491         }
2492     }
2493 
2494     return err;
2495 }
2496 
setLowLatency(int32_t lowLatency)2497 status_t ACodec::setLowLatency(int32_t lowLatency) {
2498     if (mIsEncoder) {
2499         ALOGE("encoder does not support low-latency");
2500         return BAD_VALUE;
2501     }
2502 
2503     OMX_CONFIG_BOOLEANTYPE config;
2504     InitOMXParams(&config);
2505     config.bEnabled = (OMX_BOOL)(lowLatency != 0);
2506     status_t err = mOMXNode->setConfig(
2507             (OMX_INDEXTYPE)OMX_IndexConfigLowLatency,
2508             &config, sizeof(config));
2509     if (err != OK) {
2510         ALOGE("decoder can not set low-latency to %d (err %d)", lowLatency, err);
2511     }
2512     mIsLowLatency = (lowLatency && err == OK);
2513     return err;
2514 }
2515 
setLatency(uint32_t latency)2516 status_t ACodec::setLatency(uint32_t latency) {
2517     OMX_PARAM_U32TYPE config;
2518     InitOMXParams(&config);
2519     config.nPortIndex = kPortIndexInput;
2520     config.nU32 = (OMX_U32)latency;
2521     status_t err = mOMXNode->setConfig(
2522             (OMX_INDEXTYPE)OMX_IndexConfigLatency,
2523             &config, sizeof(config));
2524     return err;
2525 }
2526 
getLatency(uint32_t * latency)2527 status_t ACodec::getLatency(uint32_t *latency) {
2528     OMX_PARAM_U32TYPE config;
2529     InitOMXParams(&config);
2530     config.nPortIndex = kPortIndexInput;
2531     status_t err = mOMXNode->getConfig(
2532             (OMX_INDEXTYPE)OMX_IndexConfigLatency,
2533             &config, sizeof(config));
2534     if (err == OK) {
2535         *latency = config.nU32;
2536     }
2537     return err;
2538 }
2539 
setTunnelPeek(int32_t tunnelPeek)2540 status_t ACodec::setTunnelPeek(int32_t tunnelPeek) {
2541     if (mIsEncoder) {
2542         ALOGE("encoder does not support %s", TUNNEL_PEEK_KEY);
2543         return BAD_VALUE;
2544     }
2545     if (!mTunneled) {
2546         ALOGE("%s is only supported in tunnel mode", TUNNEL_PEEK_KEY);
2547         return BAD_VALUE;
2548     }
2549 
2550     OMX_CONFIG_BOOLEANTYPE tunnelPeekConfig;
2551     InitOMXParams(&tunnelPeekConfig);
2552     tunnelPeekConfig.bEnabled = (OMX_BOOL)(tunnelPeek != 0);
2553     status_t err = mOMXNode->setConfig(
2554             (OMX_INDEXTYPE)OMX_IndexConfigAndroidTunnelPeek,
2555             &tunnelPeekConfig, sizeof(tunnelPeekConfig));
2556     if (err != OK) {
2557         ALOGE("decoder cannot set %s to %d (err %d)",
2558                 TUNNEL_PEEK_KEY, tunnelPeek, err);
2559     }
2560     return err;
2561 }
2562 
setTunnelPeekLegacy(int32_t isLegacy)2563 status_t ACodec::setTunnelPeekLegacy(int32_t isLegacy) {
2564     if (mIsEncoder) {
2565         ALOGE("encoder does not support %s", TUNNEL_PEEK_SET_LEGACY_KEY);
2566         return BAD_VALUE;
2567     }
2568     if (!mTunneled) {
2569         ALOGE("%s is only supported in tunnel mode", TUNNEL_PEEK_SET_LEGACY_KEY);
2570         return BAD_VALUE;
2571     }
2572 
2573     OMX_CONFIG_BOOLEANTYPE tunnelPeekLegacyModeConfig;
2574     InitOMXParams(&tunnelPeekLegacyModeConfig);
2575     tunnelPeekLegacyModeConfig.bEnabled = (OMX_BOOL)(isLegacy != 0);
2576     status_t err = mOMXNode->setConfig(
2577             (OMX_INDEXTYPE)OMX_IndexConfigAndroidTunnelPeekLegacyMode,
2578             &tunnelPeekLegacyModeConfig, sizeof(tunnelPeekLegacyModeConfig));
2579     if (err != OK) {
2580         ALOGE("decoder cannot set video peek legacy mode to %d (err %d)",
2581                 isLegacy,  err);
2582     }
2583     return err;
2584 }
2585 
setAudioPresentation(int32_t presentationId,int32_t programId)2586 status_t ACodec::setAudioPresentation(int32_t presentationId, int32_t programId) {
2587     OMX_AUDIO_CONFIG_ANDROID_AUDIOPRESENTATION config;
2588     InitOMXParams(&config);
2589     config.nPresentationId = (OMX_S32)presentationId;
2590     config.nProgramId = (OMX_S32)programId;
2591     status_t err = mOMXNode->setConfig(
2592             (OMX_INDEXTYPE)OMX_IndexConfigAudioPresentation,
2593             &config, sizeof(config));
2594     return err;
2595 }
2596 
setPriority(int32_t priority)2597 status_t ACodec::setPriority(int32_t priority) {
2598     if (priority < 0) {
2599         return BAD_VALUE;
2600     }
2601     OMX_PARAM_U32TYPE config;
2602     InitOMXParams(&config);
2603     config.nU32 = (OMX_U32)priority;
2604     status_t temp = mOMXNode->setConfig(
2605             (OMX_INDEXTYPE)OMX_IndexConfigPriority,
2606             &config, sizeof(config));
2607     if (temp != OK) {
2608         ALOGI("codec does not support config priority (err %d)", temp);
2609     }
2610     return OK;
2611 }
2612 
setOperatingRate(float rateFloat,bool isVideo)2613 status_t ACodec::setOperatingRate(float rateFloat, bool isVideo) {
2614     if (rateFloat < 0) {
2615         return BAD_VALUE;
2616     }
2617     OMX_U32 rate;
2618     if (isVideo) {
2619         if (rateFloat > 65535) {
2620             return BAD_VALUE;
2621         }
2622         rate = (OMX_U32)(rateFloat * 65536.0f + 0.5f);
2623     } else {
2624         if (rateFloat > (float)UINT_MAX) {
2625             return BAD_VALUE;
2626         }
2627         rate = (OMX_U32)(rateFloat);
2628     }
2629     OMX_PARAM_U32TYPE config;
2630     InitOMXParams(&config);
2631     config.nU32 = rate;
2632     status_t err = mOMXNode->setConfig(
2633             (OMX_INDEXTYPE)OMX_IndexConfigOperatingRate,
2634             &config, sizeof(config));
2635     if (err != OK) {
2636         ALOGI("codec does not support config operating rate (err %d)", err);
2637     }
2638     return OK;
2639 }
2640 
getIntraRefreshPeriod(uint32_t * intraRefreshPeriod)2641 status_t ACodec::getIntraRefreshPeriod(uint32_t *intraRefreshPeriod) {
2642     OMX_VIDEO_CONFIG_ANDROID_INTRAREFRESHTYPE params;
2643     InitOMXParams(&params);
2644     params.nPortIndex = kPortIndexOutput;
2645     status_t err = mOMXNode->getConfig(
2646             (OMX_INDEXTYPE)OMX_IndexConfigAndroidIntraRefresh, &params, sizeof(params));
2647     if (err == OK) {
2648         *intraRefreshPeriod = params.nRefreshPeriod;
2649         return OK;
2650     }
2651 
2652     // Fallback to query through standard OMX index.
2653     OMX_VIDEO_PARAM_INTRAREFRESHTYPE refreshParams;
2654     InitOMXParams(&refreshParams);
2655     refreshParams.nPortIndex = kPortIndexOutput;
2656     refreshParams.eRefreshMode = OMX_VIDEO_IntraRefreshCyclic;
2657     err = mOMXNode->getParameter(
2658             OMX_IndexParamVideoIntraRefresh, &refreshParams, sizeof(refreshParams));
2659     if (err != OK || refreshParams.nCirMBs == 0) {
2660         *intraRefreshPeriod = 0;
2661         return OK;
2662     }
2663 
2664     // Calculate period based on width and height
2665     uint32_t width, height;
2666     OMX_PARAM_PORTDEFINITIONTYPE def;
2667     InitOMXParams(&def);
2668     OMX_VIDEO_PORTDEFINITIONTYPE *video_def = &def.format.video;
2669     def.nPortIndex = kPortIndexOutput;
2670     err = mOMXNode->getParameter(
2671             OMX_IndexParamPortDefinition, &def, sizeof(def));
2672     if (err != OK) {
2673         *intraRefreshPeriod = 0;
2674         return err;
2675     }
2676     width = video_def->nFrameWidth;
2677     height = video_def->nFrameHeight;
2678     // Use H.264/AVC MacroBlock size 16x16
2679     *intraRefreshPeriod = divUp((divUp(width, 16u) * divUp(height, 16u)), refreshParams.nCirMBs);
2680 
2681     return OK;
2682 }
2683 
setIntraRefreshPeriod(uint32_t intraRefreshPeriod,bool inConfigure)2684 status_t ACodec::setIntraRefreshPeriod(uint32_t intraRefreshPeriod, bool inConfigure) {
2685     OMX_VIDEO_CONFIG_ANDROID_INTRAREFRESHTYPE params;
2686     InitOMXParams(&params);
2687     params.nPortIndex = kPortIndexOutput;
2688     params.nRefreshPeriod = intraRefreshPeriod;
2689     status_t err = mOMXNode->setConfig(
2690             (OMX_INDEXTYPE)OMX_IndexConfigAndroidIntraRefresh, &params, sizeof(params));
2691     if (err == OK) {
2692         return OK;
2693     }
2694 
2695     // Only in configure state, a component could invoke setParameter.
2696     if (!inConfigure) {
2697         return INVALID_OPERATION;
2698     } else {
2699         ALOGI("[%s] try falling back to Cyclic", mComponentName.c_str());
2700     }
2701 
2702     OMX_VIDEO_PARAM_INTRAREFRESHTYPE refreshParams;
2703     InitOMXParams(&refreshParams);
2704     refreshParams.nPortIndex = kPortIndexOutput;
2705     refreshParams.eRefreshMode = OMX_VIDEO_IntraRefreshCyclic;
2706 
2707     if (intraRefreshPeriod == 0) {
2708         // 0 means disable intra refresh.
2709         refreshParams.nCirMBs = 0;
2710     } else {
2711         // Calculate macroblocks that need to be intra coded base on width and height
2712         uint32_t width, height;
2713         OMX_PARAM_PORTDEFINITIONTYPE def;
2714         InitOMXParams(&def);
2715         OMX_VIDEO_PORTDEFINITIONTYPE *video_def = &def.format.video;
2716         def.nPortIndex = kPortIndexOutput;
2717         err = mOMXNode->getParameter(
2718                 OMX_IndexParamPortDefinition, &def, sizeof(def));
2719         if (err != OK) {
2720             return err;
2721         }
2722         width = video_def->nFrameWidth;
2723         height = video_def->nFrameHeight;
2724         // Use H.264/AVC MacroBlock size 16x16
2725         refreshParams.nCirMBs = divUp((divUp(width, 16u) * divUp(height, 16u)), intraRefreshPeriod);
2726     }
2727 
2728     err = mOMXNode->setParameter(
2729             OMX_IndexParamVideoIntraRefresh,
2730             &refreshParams, sizeof(refreshParams));
2731     if (err != OK) {
2732         return err;
2733     }
2734 
2735     return OK;
2736 }
2737 
configureTemporalLayers(const sp<AMessage> & msg,bool inConfigure,sp<AMessage> & outputFormat)2738 status_t ACodec::configureTemporalLayers(
2739         const sp<AMessage> &msg, bool inConfigure, sp<AMessage> &outputFormat) {
2740     if (!mIsVideo || !mIsEncoder) {
2741         return INVALID_OPERATION;
2742     }
2743 
2744     AString tsSchema;
2745     if (!msg->findString("ts-schema", &tsSchema)) {
2746         return OK;
2747     }
2748 
2749     unsigned int numLayers = 0;
2750     unsigned int numBLayers = 0;
2751     int tags;
2752     char tmp;
2753     OMX_VIDEO_ANDROID_TEMPORALLAYERINGPATTERNTYPE pattern =
2754         OMX_VIDEO_AndroidTemporalLayeringPatternNone;
2755     if (sscanf(tsSchema.c_str(), "webrtc.vp8.%u-layer%c", &numLayers, &tmp) == 1
2756             && numLayers > 0) {
2757         pattern = OMX_VIDEO_AndroidTemporalLayeringPatternWebRTC;
2758     } else if ((tags = sscanf(tsSchema.c_str(), "android.generic.%u%c%u%c",
2759                     &numLayers, &tmp, &numBLayers, &tmp))
2760             && (tags == 1 || (tags == 3 && tmp == '+'))
2761             && numLayers > 0 && numLayers < UINT32_MAX - numBLayers) {
2762         numLayers += numBLayers;
2763         pattern = OMX_VIDEO_AndroidTemporalLayeringPatternAndroid;
2764     } else {
2765         ALOGI("Ignoring unsupported ts-schema [%s]", tsSchema.c_str());
2766         return BAD_VALUE;
2767     }
2768 
2769     OMX_VIDEO_PARAM_ANDROID_TEMPORALLAYERINGTYPE layerParams;
2770     InitOMXParams(&layerParams);
2771     layerParams.nPortIndex = kPortIndexOutput;
2772 
2773     status_t err = mOMXNode->getParameter(
2774             (OMX_INDEXTYPE)OMX_IndexParamAndroidVideoTemporalLayering,
2775             &layerParams, sizeof(layerParams));
2776 
2777     if (err != OK) {
2778         return err;
2779     } else if (!(layerParams.eSupportedPatterns & pattern)) {
2780         return BAD_VALUE;
2781     }
2782 
2783     numLayers = min(numLayers, layerParams.nLayerCountMax);
2784     numBLayers = min(numBLayers, layerParams.nBLayerCountMax);
2785 
2786     if (!inConfigure) {
2787         OMX_VIDEO_CONFIG_ANDROID_TEMPORALLAYERINGTYPE layerConfig;
2788         InitOMXParams(&layerConfig);
2789         layerConfig.nPortIndex = kPortIndexOutput;
2790         layerConfig.ePattern = pattern;
2791         layerConfig.nPLayerCountActual = numLayers - numBLayers;
2792         layerConfig.nBLayerCountActual = numBLayers;
2793         layerConfig.bBitrateRatiosSpecified = OMX_FALSE;
2794 
2795         err = mOMXNode->setConfig(
2796                 (OMX_INDEXTYPE)OMX_IndexConfigAndroidVideoTemporalLayering,
2797                 &layerConfig, sizeof(layerConfig));
2798     } else {
2799         layerParams.ePattern = pattern;
2800         layerParams.nPLayerCountActual = numLayers - numBLayers;
2801         layerParams.nBLayerCountActual = numBLayers;
2802         layerParams.bBitrateRatiosSpecified = OMX_FALSE;
2803         layerParams.nLayerCountMax = numLayers;
2804         layerParams.nBLayerCountMax = numBLayers;
2805 
2806         err = mOMXNode->setParameter(
2807                 (OMX_INDEXTYPE)OMX_IndexParamAndroidVideoTemporalLayering,
2808                 &layerParams, sizeof(layerParams));
2809     }
2810 
2811     AString configSchema;
2812     if (pattern == OMX_VIDEO_AndroidTemporalLayeringPatternAndroid) {
2813         configSchema = AStringPrintf("android.generic.%u+%u", numLayers - numBLayers, numBLayers);
2814     } else if (pattern == OMX_VIDEO_AndroidTemporalLayeringPatternWebRTC) {
2815         configSchema = AStringPrintf("webrtc.vp8.%u", numLayers);
2816     }
2817 
2818     if (err != OK) {
2819         ALOGW("Failed to set temporal layers to %s (requested %s)",
2820                 configSchema.c_str(), tsSchema.c_str());
2821         return err;
2822     }
2823 
2824     err = mOMXNode->getParameter(
2825             (OMX_INDEXTYPE)OMX_IndexParamAndroidVideoTemporalLayering,
2826             &layerParams, sizeof(layerParams));
2827 
2828     if (err == OK) {
2829         ALOGD("Temporal layers requested:%s configured:%s got:%s(%u: P=%u, B=%u)",
2830                 tsSchema.c_str(), configSchema.c_str(),
2831                 asString(layerParams.ePattern), layerParams.ePattern,
2832                 layerParams.nPLayerCountActual, layerParams.nBLayerCountActual);
2833 
2834         if (outputFormat.get() == mOutputFormat.get()) {
2835             mOutputFormat = mOutputFormat->dup(); // trigger an output format change event
2836         }
2837         // assume we got what we configured
2838         outputFormat->setString("ts-schema", configSchema);
2839     }
2840     return err;
2841 }
2842 
setMinBufferSize(OMX_U32 portIndex,size_t size)2843 status_t ACodec::setMinBufferSize(OMX_U32 portIndex, size_t size) {
2844     OMX_PARAM_PORTDEFINITIONTYPE def;
2845     InitOMXParams(&def);
2846     def.nPortIndex = portIndex;
2847 
2848     status_t err = mOMXNode->getParameter(
2849             OMX_IndexParamPortDefinition, &def, sizeof(def));
2850 
2851     if (err != OK) {
2852         return err;
2853     }
2854 
2855     if (def.nBufferSize >= size) {
2856         return OK;
2857     }
2858 
2859     def.nBufferSize = size;
2860 
2861     err = mOMXNode->setParameter(
2862             OMX_IndexParamPortDefinition, &def, sizeof(def));
2863 
2864     if (err != OK) {
2865         return err;
2866     }
2867 
2868     err = mOMXNode->getParameter(
2869             OMX_IndexParamPortDefinition, &def, sizeof(def));
2870 
2871     if (err != OK) {
2872         return err;
2873     }
2874 
2875     if (def.nBufferSize < size) {
2876         ALOGE("failed to set min buffer size to %zu (is still %u)", size, def.nBufferSize);
2877         return FAILED_TRANSACTION;
2878     }
2879 
2880     return OK;
2881 }
2882 
selectAudioPortFormat(OMX_U32 portIndex,OMX_AUDIO_CODINGTYPE desiredFormat)2883 status_t ACodec::selectAudioPortFormat(
2884         OMX_U32 portIndex, OMX_AUDIO_CODINGTYPE desiredFormat) {
2885     OMX_AUDIO_PARAM_PORTFORMATTYPE format;
2886     InitOMXParams(&format);
2887 
2888     format.nPortIndex = portIndex;
2889     for (OMX_U32 index = 0; index <= kMaxIndicesToCheck; ++index) {
2890         format.nIndex = index;
2891         status_t err = mOMXNode->getParameter(
2892                 OMX_IndexParamAudioPortFormat, &format, sizeof(format));
2893 
2894         if (err != OK) {
2895             return err;
2896         }
2897 
2898         if (format.eEncoding == desiredFormat) {
2899             break;
2900         }
2901 
2902         if (index == kMaxIndicesToCheck) {
2903             ALOGW("[%s] stopping checking formats after %u: %s(%x)",
2904                     mComponentName.c_str(), index,
2905                     asString(format.eEncoding), format.eEncoding);
2906             return ERROR_UNSUPPORTED;
2907         }
2908     }
2909 
2910     return mOMXNode->setParameter(
2911             OMX_IndexParamAudioPortFormat, &format, sizeof(format));
2912 }
2913 
setupAACCodec(bool encoder,int32_t numChannels,int32_t sampleRate,int32_t bitRate,int32_t aacProfile,bool isADTS,int32_t sbrMode,int32_t maxOutputChannelCount,const drcParams_t & drc,int32_t pcmLimiterEnable)2914 status_t ACodec::setupAACCodec(
2915         bool encoder, int32_t numChannels, int32_t sampleRate,
2916         int32_t bitRate, int32_t aacProfile, bool isADTS, int32_t sbrMode,
2917         int32_t maxOutputChannelCount, const drcParams_t& drc,
2918         int32_t pcmLimiterEnable) {
2919     if (encoder && isADTS) {
2920         return -EINVAL;
2921     }
2922 
2923     status_t err = setupRawAudioFormat(
2924             encoder ? kPortIndexInput : kPortIndexOutput,
2925             sampleRate,
2926             numChannels);
2927 
2928     if (err != OK) {
2929         return err;
2930     }
2931 
2932     if (encoder) {
2933         err = selectAudioPortFormat(kPortIndexOutput, OMX_AUDIO_CodingAAC);
2934 
2935         if (err != OK) {
2936             return err;
2937         }
2938 
2939         OMX_PARAM_PORTDEFINITIONTYPE def;
2940         InitOMXParams(&def);
2941         def.nPortIndex = kPortIndexOutput;
2942 
2943         err = mOMXNode->getParameter(
2944                 OMX_IndexParamPortDefinition, &def, sizeof(def));
2945 
2946         if (err != OK) {
2947             return err;
2948         }
2949 
2950         def.format.audio.bFlagErrorConcealment = OMX_TRUE;
2951         def.format.audio.eEncoding = OMX_AUDIO_CodingAAC;
2952 
2953         err = mOMXNode->setParameter(
2954                 OMX_IndexParamPortDefinition, &def, sizeof(def));
2955 
2956         if (err != OK) {
2957             return err;
2958         }
2959 
2960         OMX_AUDIO_PARAM_AACPROFILETYPE profile;
2961         InitOMXParams(&profile);
2962         profile.nPortIndex = kPortIndexOutput;
2963 
2964         err = mOMXNode->getParameter(
2965                 OMX_IndexParamAudioAac, &profile, sizeof(profile));
2966 
2967         if (err != OK) {
2968             return err;
2969         }
2970 
2971         profile.nChannels = numChannels;
2972 
2973         profile.eChannelMode =
2974             (numChannels == 1)
2975                 ? OMX_AUDIO_ChannelModeMono: OMX_AUDIO_ChannelModeStereo;
2976 
2977         profile.nSampleRate = sampleRate;
2978         profile.nBitRate = bitRate;
2979         profile.nAudioBandWidth = 0;
2980         profile.nFrameLength = 0;
2981         profile.nAACtools = OMX_AUDIO_AACToolAll;
2982         profile.nAACERtools = OMX_AUDIO_AACERNone;
2983         profile.eAACProfile = (OMX_AUDIO_AACPROFILETYPE) aacProfile;
2984         profile.eAACStreamFormat = OMX_AUDIO_AACStreamFormatMP4FF;
2985         switch (sbrMode) {
2986         case 0:
2987             // disable sbr
2988             profile.nAACtools &= ~OMX_AUDIO_AACToolAndroidSSBR;
2989             profile.nAACtools &= ~OMX_AUDIO_AACToolAndroidDSBR;
2990             break;
2991         case 1:
2992             // enable single-rate sbr
2993             profile.nAACtools |= OMX_AUDIO_AACToolAndroidSSBR;
2994             profile.nAACtools &= ~OMX_AUDIO_AACToolAndroidDSBR;
2995             break;
2996         case 2:
2997             // enable dual-rate sbr
2998             profile.nAACtools &= ~OMX_AUDIO_AACToolAndroidSSBR;
2999             profile.nAACtools |= OMX_AUDIO_AACToolAndroidDSBR;
3000             break;
3001         case -1:
3002             // enable both modes -> the codec will decide which mode should be used
3003             profile.nAACtools |= OMX_AUDIO_AACToolAndroidSSBR;
3004             profile.nAACtools |= OMX_AUDIO_AACToolAndroidDSBR;
3005             break;
3006         default:
3007             // unsupported sbr mode
3008             return BAD_VALUE;
3009         }
3010 
3011 
3012         err = mOMXNode->setParameter(
3013                 OMX_IndexParamAudioAac, &profile, sizeof(profile));
3014 
3015         if (err != OK) {
3016             return err;
3017         }
3018 
3019         return err;
3020     }
3021 
3022     OMX_AUDIO_PARAM_AACPROFILETYPE profile;
3023     InitOMXParams(&profile);
3024     profile.nPortIndex = kPortIndexInput;
3025 
3026     err = mOMXNode->getParameter(
3027             OMX_IndexParamAudioAac, &profile, sizeof(profile));
3028 
3029     if (err != OK) {
3030         return err;
3031     }
3032 
3033     profile.nChannels = numChannels;
3034     profile.nSampleRate = sampleRate;
3035 
3036     profile.eAACStreamFormat =
3037         isADTS
3038             ? OMX_AUDIO_AACStreamFormatMP4ADTS
3039             : OMX_AUDIO_AACStreamFormatMP4FF;
3040 
3041     OMX_AUDIO_PARAM_ANDROID_AACDRCPRESENTATIONTYPE presentation;
3042     InitOMXParams(&presentation);
3043     presentation.nMaxOutputChannels = maxOutputChannelCount;
3044     presentation.nDrcCut = drc.drcCut;
3045     presentation.nDrcBoost = drc.drcBoost;
3046     presentation.nHeavyCompression = drc.heavyCompression;
3047     presentation.nTargetReferenceLevel = drc.targetRefLevel;
3048     presentation.nEncodedTargetLevel = drc.encodedTargetLevel;
3049     presentation.nPCMLimiterEnable = pcmLimiterEnable;
3050     presentation.nDrcEffectType = drc.effectType;
3051     presentation.nDrcAlbumMode = drc.albumMode;
3052     presentation.nDrcOutputLoudness = drc.outputLoudness;
3053 
3054     status_t res = mOMXNode->setParameter(
3055             OMX_IndexParamAudioAac, &profile, sizeof(profile));
3056     if (res == OK) {
3057         // optional parameters, will not cause configuration failure
3058         if (mOMXNode->setParameter(
3059                 (OMX_INDEXTYPE)OMX_IndexParamAudioAndroidAacDrcPresentation,
3060                 &presentation, sizeof(presentation)) == ERROR_UNSUPPORTED) {
3061             // prior to 9.0 we used a different config structure and index
3062             OMX_AUDIO_PARAM_ANDROID_AACPRESENTATIONTYPE presentation8;
3063             InitOMXParams(&presentation8);
3064             presentation8.nMaxOutputChannels = presentation.nMaxOutputChannels;
3065             presentation8.nDrcCut = presentation.nDrcCut;
3066             presentation8.nDrcBoost = presentation.nDrcBoost;
3067             presentation8.nHeavyCompression = presentation.nHeavyCompression;
3068             presentation8.nTargetReferenceLevel = presentation.nTargetReferenceLevel;
3069             presentation8.nEncodedTargetLevel = presentation.nEncodedTargetLevel;
3070             presentation8.nPCMLimiterEnable = presentation.nPCMLimiterEnable;
3071             (void)mOMXNode->setParameter(
3072                 (OMX_INDEXTYPE)OMX_IndexParamAudioAndroidAacPresentation,
3073                 &presentation8, sizeof(presentation8));
3074         }
3075     } else {
3076         ALOGW("did not set AudioAndroidAacPresentation due to error %d when setting AudioAac", res);
3077     }
3078     mSampleRate = sampleRate;
3079     return res;
3080 }
3081 
setupAC3Codec(bool encoder,int32_t numChannels,int32_t sampleRate)3082 status_t ACodec::setupAC3Codec(
3083         bool encoder, int32_t numChannels, int32_t sampleRate) {
3084     status_t err = setupRawAudioFormat(
3085             encoder ? kPortIndexInput : kPortIndexOutput, sampleRate, numChannels);
3086 
3087     if (err != OK) {
3088         return err;
3089     }
3090 
3091     if (encoder) {
3092         ALOGW("AC3 encoding is not supported.");
3093         return INVALID_OPERATION;
3094     }
3095 
3096     OMX_AUDIO_PARAM_ANDROID_AC3TYPE def;
3097     InitOMXParams(&def);
3098     def.nPortIndex = kPortIndexInput;
3099 
3100     err = mOMXNode->getParameter(
3101             (OMX_INDEXTYPE)OMX_IndexParamAudioAndroidAc3, &def, sizeof(def));
3102 
3103     if (err != OK) {
3104         return err;
3105     }
3106 
3107     def.nChannels = numChannels;
3108     def.nSampleRate = sampleRate;
3109 
3110     return mOMXNode->setParameter(
3111             (OMX_INDEXTYPE)OMX_IndexParamAudioAndroidAc3, &def, sizeof(def));
3112 }
3113 
setupEAC3Codec(bool encoder,int32_t numChannels,int32_t sampleRate)3114 status_t ACodec::setupEAC3Codec(
3115         bool encoder, int32_t numChannels, int32_t sampleRate) {
3116     status_t err = setupRawAudioFormat(
3117             encoder ? kPortIndexInput : kPortIndexOutput, sampleRate, numChannels);
3118 
3119     if (err != OK) {
3120         return err;
3121     }
3122 
3123     if (encoder) {
3124         ALOGW("EAC3 encoding is not supported.");
3125         return INVALID_OPERATION;
3126     }
3127 
3128     OMX_AUDIO_PARAM_ANDROID_EAC3TYPE def;
3129     InitOMXParams(&def);
3130     def.nPortIndex = kPortIndexInput;
3131 
3132     err = mOMXNode->getParameter(
3133             (OMX_INDEXTYPE)OMX_IndexParamAudioAndroidEac3, &def, sizeof(def));
3134 
3135     if (err != OK) {
3136         return err;
3137     }
3138 
3139     def.nChannels = numChannels;
3140     def.nSampleRate = sampleRate;
3141 
3142     return mOMXNode->setParameter(
3143             (OMX_INDEXTYPE)OMX_IndexParamAudioAndroidEac3, &def, sizeof(def));
3144 }
3145 
setupAC4Codec(bool encoder,int32_t numChannels,int32_t sampleRate)3146 status_t ACodec::setupAC4Codec(
3147         bool encoder, int32_t numChannels, int32_t sampleRate) {
3148     status_t err = setupRawAudioFormat(
3149             encoder ? kPortIndexInput : kPortIndexOutput, sampleRate, numChannels);
3150 
3151     if (err != OK) {
3152         return err;
3153     }
3154 
3155     if (encoder) {
3156         ALOGW("AC4 encoding is not supported.");
3157         return INVALID_OPERATION;
3158     }
3159 
3160     OMX_AUDIO_PARAM_ANDROID_AC4TYPE def;
3161     InitOMXParams(&def);
3162     def.nPortIndex = kPortIndexInput;
3163 
3164     err = mOMXNode->getParameter(
3165             (OMX_INDEXTYPE)OMX_IndexParamAudioAndroidAc4, &def, sizeof(def));
3166 
3167     if (err != OK) {
3168         return err;
3169     }
3170 
3171     def.nChannels = numChannels;
3172     def.nSampleRate = sampleRate;
3173 
3174     return mOMXNode->setParameter(
3175             (OMX_INDEXTYPE)OMX_IndexParamAudioAndroidAc4, &def, sizeof(def));
3176 }
3177 
pickModeFromBitRate(bool isAMRWB,int32_t bps)3178 static OMX_AUDIO_AMRBANDMODETYPE pickModeFromBitRate(
3179         bool isAMRWB, int32_t bps) {
3180     if (isAMRWB) {
3181         if (bps <= 6600) {
3182             return OMX_AUDIO_AMRBandModeWB0;
3183         } else if (bps <= 8850) {
3184             return OMX_AUDIO_AMRBandModeWB1;
3185         } else if (bps <= 12650) {
3186             return OMX_AUDIO_AMRBandModeWB2;
3187         } else if (bps <= 14250) {
3188             return OMX_AUDIO_AMRBandModeWB3;
3189         } else if (bps <= 15850) {
3190             return OMX_AUDIO_AMRBandModeWB4;
3191         } else if (bps <= 18250) {
3192             return OMX_AUDIO_AMRBandModeWB5;
3193         } else if (bps <= 19850) {
3194             return OMX_AUDIO_AMRBandModeWB6;
3195         } else if (bps <= 23050) {
3196             return OMX_AUDIO_AMRBandModeWB7;
3197         }
3198 
3199         // 23850 bps
3200         return OMX_AUDIO_AMRBandModeWB8;
3201     } else {  // AMRNB
3202         if (bps <= 4750) {
3203             return OMX_AUDIO_AMRBandModeNB0;
3204         } else if (bps <= 5150) {
3205             return OMX_AUDIO_AMRBandModeNB1;
3206         } else if (bps <= 5900) {
3207             return OMX_AUDIO_AMRBandModeNB2;
3208         } else if (bps <= 6700) {
3209             return OMX_AUDIO_AMRBandModeNB3;
3210         } else if (bps <= 7400) {
3211             return OMX_AUDIO_AMRBandModeNB4;
3212         } else if (bps <= 7950) {
3213             return OMX_AUDIO_AMRBandModeNB5;
3214         } else if (bps <= 10200) {
3215             return OMX_AUDIO_AMRBandModeNB6;
3216         }
3217 
3218         // 12200 bps
3219         return OMX_AUDIO_AMRBandModeNB7;
3220     }
3221 }
3222 
setupAMRCodec(bool encoder,bool isWAMR,int32_t bitrate)3223 status_t ACodec::setupAMRCodec(bool encoder, bool isWAMR, int32_t bitrate) {
3224     OMX_AUDIO_PARAM_AMRTYPE def;
3225     InitOMXParams(&def);
3226     def.nPortIndex = encoder ? kPortIndexOutput : kPortIndexInput;
3227 
3228     status_t err = mOMXNode->getParameter(
3229             OMX_IndexParamAudioAmr, &def, sizeof(def));
3230 
3231     if (err != OK) {
3232         return err;
3233     }
3234 
3235     def.eAMRFrameFormat = OMX_AUDIO_AMRFrameFormatFSF;
3236     def.eAMRBandMode = pickModeFromBitRate(isWAMR, bitrate);
3237 
3238     err = mOMXNode->setParameter(
3239             OMX_IndexParamAudioAmr, &def, sizeof(def));
3240 
3241     if (err != OK) {
3242         return err;
3243     }
3244 
3245     return setupRawAudioFormat(
3246             encoder ? kPortIndexInput : kPortIndexOutput,
3247             isWAMR ? 16000 : 8000 /* sampleRate */,
3248             1 /* numChannels */);
3249 }
3250 
setupG711Codec(bool encoder,int32_t sampleRate,int32_t numChannels)3251 status_t ACodec::setupG711Codec(bool encoder, int32_t sampleRate, int32_t numChannels) {
3252     if (encoder) {
3253         return INVALID_OPERATION;
3254     }
3255 
3256     return setupRawAudioFormat(
3257             kPortIndexInput, sampleRate, numChannels);
3258 }
3259 
setupOpusCodec(bool encoder,int32_t sampleRate,int32_t numChannels)3260 status_t ACodec::setupOpusCodec(bool encoder, int32_t sampleRate, int32_t numChannels) {
3261     if (encoder) {
3262         return INVALID_OPERATION;
3263     }
3264     OMX_AUDIO_PARAM_ANDROID_OPUSTYPE def;
3265     InitOMXParams(&def);
3266     def.nPortIndex = kPortIndexInput;
3267     status_t err = mOMXNode->getParameter(
3268             (OMX_INDEXTYPE)OMX_IndexParamAudioAndroidOpus, &def, sizeof(def));
3269     if (err != OK) {
3270         ALOGE("setupOpusCodec(): Error %d getting OMX_IndexParamAudioAndroidOpus parameter", err);
3271         return err;
3272     }
3273     def.nSampleRate = sampleRate;
3274     def.nChannels = numChannels;
3275     err = mOMXNode->setParameter(
3276            (OMX_INDEXTYPE)OMX_IndexParamAudioAndroidOpus, &def, sizeof(def));
3277     return err;
3278 }
3279 
setupFlacCodec(bool encoder,int32_t numChannels,int32_t sampleRate,int32_t compressionLevel,AudioEncoding encoding)3280 status_t ACodec::setupFlacCodec(
3281         bool encoder, int32_t numChannels, int32_t sampleRate, int32_t compressionLevel,
3282         AudioEncoding encoding) {
3283     if (encoder) {
3284         OMX_AUDIO_PARAM_FLACTYPE def;
3285         InitOMXParams(&def);
3286         def.nPortIndex = kPortIndexOutput;
3287 
3288         // configure compression level
3289         status_t err = mOMXNode->getParameter(OMX_IndexParamAudioFlac, &def, sizeof(def));
3290         if (err != OK) {
3291             ALOGE("setupFlacCodec(): Error %d getting OMX_IndexParamAudioFlac parameter", err);
3292             return err;
3293         }
3294         def.nCompressionLevel = compressionLevel;
3295         err = mOMXNode->setParameter(OMX_IndexParamAudioFlac, &def, sizeof(def));
3296         if (err != OK) {
3297             ALOGE("setupFlacCodec(): Error %d setting OMX_IndexParamAudioFlac parameter", err);
3298             return err;
3299         }
3300     }
3301 
3302     return setupRawAudioFormat(
3303             encoder ? kPortIndexInput : kPortIndexOutput,
3304             sampleRate,
3305             numChannels,
3306             encoding);
3307 }
3308 
setupRawAudioFormat(OMX_U32 portIndex,int32_t sampleRate,int32_t numChannels,AudioEncoding encoding)3309 status_t ACodec::setupRawAudioFormat(
3310         OMX_U32 portIndex, int32_t sampleRate, int32_t numChannels, AudioEncoding encoding) {
3311     OMX_PARAM_PORTDEFINITIONTYPE def;
3312     InitOMXParams(&def);
3313     def.nPortIndex = portIndex;
3314 
3315     status_t err = mOMXNode->getParameter(
3316             OMX_IndexParamPortDefinition, &def, sizeof(def));
3317 
3318     if (err != OK) {
3319         return err;
3320     }
3321 
3322     def.format.audio.eEncoding = OMX_AUDIO_CodingPCM;
3323 
3324     err = mOMXNode->setParameter(
3325             OMX_IndexParamPortDefinition, &def, sizeof(def));
3326 
3327     if (err != OK) {
3328         return err;
3329     }
3330 
3331     OMX_AUDIO_PARAM_PCMMODETYPE pcmParams;
3332     InitOMXParams(&pcmParams);
3333     pcmParams.nPortIndex = portIndex;
3334 
3335     err = mOMXNode->getParameter(
3336             OMX_IndexParamAudioPcm, &pcmParams, sizeof(pcmParams));
3337 
3338     if (err != OK) {
3339         return err;
3340     }
3341 
3342     pcmParams.nChannels = numChannels;
3343     switch (encoding) {
3344         case kAudioEncodingPcm8bit:
3345             pcmParams.eNumData = OMX_NumericalDataUnsigned;
3346             pcmParams.nBitPerSample = 8;
3347             break;
3348         case kAudioEncodingPcmFloat:
3349             pcmParams.eNumData = OMX_NumericalDataFloat;
3350             pcmParams.nBitPerSample = 32;
3351             break;
3352         case kAudioEncodingPcm16bit:
3353             pcmParams.eNumData = OMX_NumericalDataSigned;
3354             pcmParams.nBitPerSample = 16;
3355             break;
3356         default:
3357             return BAD_VALUE;
3358     }
3359     pcmParams.bInterleaved = OMX_TRUE;
3360     pcmParams.nSamplingRate = sampleRate;
3361     pcmParams.ePCMMode = OMX_AUDIO_PCMModeLinear;
3362 
3363     if (getOMXChannelMapping(numChannels, pcmParams.eChannelMapping) != OK) {
3364         ALOGE("%s: incorrect numChannels: %d", __func__, numChannels);
3365         return OMX_ErrorNone;
3366     }
3367 
3368     err = mOMXNode->setParameter(
3369             OMX_IndexParamAudioPcm, &pcmParams, sizeof(pcmParams));
3370     // if we could not set up raw format to non-16-bit, try with 16-bit
3371     // NOTE: we will also verify this via readback, in case codec ignores these fields
3372     if (err != OK && encoding != kAudioEncodingPcm16bit) {
3373         pcmParams.eNumData = OMX_NumericalDataSigned;
3374         pcmParams.nBitPerSample = 16;
3375         err = mOMXNode->setParameter(
3376                 OMX_IndexParamAudioPcm, &pcmParams, sizeof(pcmParams));
3377     }
3378     return err;
3379 }
3380 
configureTunneledVideoPlayback(int32_t audioHwSync,const sp<ANativeWindow> & nativeWindow)3381 status_t ACodec::configureTunneledVideoPlayback(
3382         int32_t audioHwSync, const sp<ANativeWindow> &nativeWindow) {
3383     native_handle_t* sidebandHandle;
3384 
3385     status_t err = mOMXNode->configureVideoTunnelMode(
3386             kPortIndexOutput, OMX_TRUE, audioHwSync, &sidebandHandle);
3387     if (err != OK) {
3388         ALOGE("configureVideoTunnelMode failed! (err %d).", err);
3389         return err;
3390     }
3391 
3392     err = native_window_set_sideband_stream(nativeWindow.get(), sidebandHandle);
3393     if (err != OK) {
3394         ALOGE("native_window_set_sideband_stream(%p) failed! (err %d).",
3395                 sidebandHandle, err);
3396     }
3397 
3398     native_handle_close(sidebandHandle);
3399     native_handle_delete(sidebandHandle);
3400 
3401     return err;
3402 }
3403 
setVideoPortFormatType(OMX_U32 portIndex,OMX_VIDEO_CODINGTYPE compressionFormat,OMX_COLOR_FORMATTYPE colorFormat,bool usingNativeBuffers)3404 status_t ACodec::setVideoPortFormatType(
3405         OMX_U32 portIndex,
3406         OMX_VIDEO_CODINGTYPE compressionFormat,
3407         OMX_COLOR_FORMATTYPE colorFormat,
3408         bool usingNativeBuffers) {
3409     OMX_VIDEO_PARAM_PORTFORMATTYPE format;
3410     InitOMXParams(&format);
3411     format.nPortIndex = portIndex;
3412     format.nIndex = 0;
3413     bool found = false;
3414 
3415     for (OMX_U32 index = 0; index <= kMaxIndicesToCheck; ++index) {
3416         format.nIndex = index;
3417         status_t err = mOMXNode->getParameter(
3418                 OMX_IndexParamVideoPortFormat,
3419                 &format, sizeof(format));
3420 
3421         if (err != OK) {
3422             return err;
3423         }
3424 
3425         // substitute back flexible color format to codec supported format
3426         OMX_U32 flexibleEquivalent;
3427         if (compressionFormat == OMX_VIDEO_CodingUnused
3428                 && IsFlexibleColorFormat(
3429                         mOMXNode, format.eColorFormat, usingNativeBuffers, &flexibleEquivalent)
3430                 && colorFormat == flexibleEquivalent) {
3431             ALOGI("[%s] using color format %#x in place of %#x",
3432                     mComponentName.c_str(), format.eColorFormat, colorFormat);
3433             colorFormat = format.eColorFormat;
3434         }
3435 
3436         // The following assertion is violated by TI's video decoder.
3437         // CHECK_EQ(format.nIndex, index);
3438 
3439         if (!strcmp("OMX.TI.Video.encoder", mComponentName.c_str())) {
3440             if (portIndex == kPortIndexInput
3441                     && colorFormat == format.eColorFormat) {
3442                 // eCompressionFormat does not seem right.
3443                 found = true;
3444                 break;
3445             }
3446             if (portIndex == kPortIndexOutput
3447                     && compressionFormat == format.eCompressionFormat) {
3448                 // eColorFormat does not seem right.
3449                 found = true;
3450                 break;
3451             }
3452         }
3453 
3454         if (format.eCompressionFormat == compressionFormat
3455             && format.eColorFormat == colorFormat) {
3456             found = true;
3457             break;
3458         }
3459 
3460         if (index == kMaxIndicesToCheck) {
3461             ALOGW("[%s] stopping checking formats after %u: %s(%x)/%s(%x)",
3462                     mComponentName.c_str(), index,
3463                     asString(format.eCompressionFormat), format.eCompressionFormat,
3464                     asString(format.eColorFormat), format.eColorFormat);
3465         }
3466     }
3467 
3468     if (!found) {
3469         return UNKNOWN_ERROR;
3470     }
3471 
3472     status_t err = mOMXNode->setParameter(
3473             OMX_IndexParamVideoPortFormat, &format, sizeof(format));
3474 
3475     return err;
3476 }
3477 
3478 // Set optimal output format. OMX component lists output formats in the order
3479 // of preference, but this got more complicated since the introduction of flexible
3480 // YUV formats. We support a legacy behavior for applications that do not use
3481 // surface output, do not specify an output format, but expect a "usable" standard
3482 // OMX format. SW readable and standard formats must be flex-YUV.
3483 //
3484 // Suggested preference order:
3485 // - optimal format for texture rendering (mediaplayer behavior)
3486 // - optimal SW readable & texture renderable format (flex-YUV support)
3487 // - optimal SW readable non-renderable format (flex-YUV bytebuffer support)
3488 // - legacy "usable" standard formats
3489 //
3490 // For legacy support, we prefer a standard format, but will settle for a SW readable
3491 // flex-YUV format.
setSupportedOutputFormat(bool getLegacyFlexibleFormat)3492 status_t ACodec::setSupportedOutputFormat(bool getLegacyFlexibleFormat) {
3493     OMX_VIDEO_PARAM_PORTFORMATTYPE format, legacyFormat;
3494     InitOMXParams(&format);
3495     format.nPortIndex = kPortIndexOutput;
3496 
3497     InitOMXParams(&legacyFormat);
3498     // this field will change when we find a suitable legacy format
3499     legacyFormat.eColorFormat = OMX_COLOR_FormatUnused;
3500 
3501     for (OMX_U32 index = 0; ; ++index) {
3502         format.nIndex = index;
3503         status_t err = mOMXNode->getParameter(
3504                 OMX_IndexParamVideoPortFormat, &format, sizeof(format));
3505         if (err != OK) {
3506             // no more formats, pick legacy format if found
3507             if (legacyFormat.eColorFormat != OMX_COLOR_FormatUnused) {
3508                  memcpy(&format, &legacyFormat, sizeof(format));
3509                  break;
3510             }
3511             return err;
3512         }
3513         if (format.eCompressionFormat != OMX_VIDEO_CodingUnused) {
3514             return OMX_ErrorBadParameter;
3515         }
3516         if (!getLegacyFlexibleFormat) {
3517             break;
3518         }
3519         // standard formats that were exposed to users before
3520         if (format.eColorFormat == OMX_COLOR_FormatYUV420Planar
3521                 || format.eColorFormat == OMX_COLOR_FormatYUV420PackedPlanar
3522                 || format.eColorFormat == OMX_COLOR_FormatYUV420SemiPlanar
3523                 || format.eColorFormat == OMX_COLOR_FormatYUV420PackedSemiPlanar
3524                 || format.eColorFormat == OMX_TI_COLOR_FormatYUV420PackedSemiPlanar) {
3525             break;
3526         }
3527         // find best legacy non-standard format
3528         OMX_U32 flexibleEquivalent;
3529         if (legacyFormat.eColorFormat == OMX_COLOR_FormatUnused
3530                 && IsFlexibleColorFormat(
3531                         mOMXNode, format.eColorFormat, false /* usingNativeBuffers */,
3532                         &flexibleEquivalent)
3533                 && flexibleEquivalent == OMX_COLOR_FormatYUV420Flexible) {
3534             memcpy(&legacyFormat, &format, sizeof(format));
3535         }
3536     }
3537     return mOMXNode->setParameter(
3538             OMX_IndexParamVideoPortFormat, &format, sizeof(format));
3539 }
3540 
3541 static const struct VideoCodingMapEntry {
3542     const char *mMime;
3543     OMX_VIDEO_CODINGTYPE mVideoCodingType;
3544 } kVideoCodingMapEntry[] = {
3545     { MEDIA_MIMETYPE_VIDEO_AVC, OMX_VIDEO_CodingAVC },
3546     { MEDIA_MIMETYPE_VIDEO_HEVC, OMX_VIDEO_CodingHEVC },
3547     { MEDIA_MIMETYPE_VIDEO_MPEG4, OMX_VIDEO_CodingMPEG4 },
3548     { MEDIA_MIMETYPE_VIDEO_H263, OMX_VIDEO_CodingH263 },
3549     { MEDIA_MIMETYPE_VIDEO_MPEG2, OMX_VIDEO_CodingMPEG2 },
3550     { MEDIA_MIMETYPE_VIDEO_VP8, OMX_VIDEO_CodingVP8 },
3551     { MEDIA_MIMETYPE_VIDEO_VP9, OMX_VIDEO_CodingVP9 },
3552     { MEDIA_MIMETYPE_VIDEO_DOLBY_VISION, OMX_VIDEO_CodingDolbyVision },
3553     { MEDIA_MIMETYPE_IMAGE_ANDROID_HEIC, OMX_VIDEO_CodingImageHEIC },
3554     { MEDIA_MIMETYPE_VIDEO_AV1, OMX_VIDEO_CodingAV1 },
3555 };
3556 
GetVideoCodingTypeFromMime(const char * mime,OMX_VIDEO_CODINGTYPE * codingType)3557 static status_t GetVideoCodingTypeFromMime(
3558         const char *mime, OMX_VIDEO_CODINGTYPE *codingType) {
3559     for (size_t i = 0;
3560          i < sizeof(kVideoCodingMapEntry) / sizeof(kVideoCodingMapEntry[0]);
3561          ++i) {
3562         if (!strcasecmp(mime, kVideoCodingMapEntry[i].mMime)) {
3563             *codingType = kVideoCodingMapEntry[i].mVideoCodingType;
3564             return OK;
3565         }
3566     }
3567 
3568     *codingType = OMX_VIDEO_CodingUnused;
3569 
3570     return ERROR_UNSUPPORTED;
3571 }
3572 
GetMimeTypeForVideoCoding(OMX_VIDEO_CODINGTYPE codingType,AString * mime)3573 static status_t GetMimeTypeForVideoCoding(
3574         OMX_VIDEO_CODINGTYPE codingType, AString *mime) {
3575     for (size_t i = 0;
3576          i < sizeof(kVideoCodingMapEntry) / sizeof(kVideoCodingMapEntry[0]);
3577          ++i) {
3578         if (codingType == kVideoCodingMapEntry[i].mVideoCodingType) {
3579             *mime = kVideoCodingMapEntry[i].mMime;
3580             return OK;
3581         }
3582     }
3583 
3584     mime->clear();
3585 
3586     return ERROR_UNSUPPORTED;
3587 }
3588 
setPortBufferNum(OMX_U32 portIndex,int bufferNum)3589 status_t ACodec::setPortBufferNum(OMX_U32 portIndex, int bufferNum) {
3590     OMX_PARAM_PORTDEFINITIONTYPE def;
3591     InitOMXParams(&def);
3592     def.nPortIndex = portIndex;
3593     status_t err;
3594     ALOGD("Setting [%s] %s port buffer number: %d", mComponentName.c_str(),
3595             portIndex == kPortIndexInput ? "input" : "output", bufferNum);
3596     err = mOMXNode->getParameter(
3597         OMX_IndexParamPortDefinition, &def, sizeof(def));
3598     if (err != OK) {
3599         return err;
3600     }
3601     def.nBufferCountActual = bufferNum;
3602     err = mOMXNode->setParameter(
3603         OMX_IndexParamPortDefinition, &def, sizeof(def));
3604     if (err != OK) {
3605         // Component could reject this request.
3606         ALOGW("Fail to set [%s] %s port buffer number: %d", mComponentName.c_str(),
3607             portIndex == kPortIndexInput ? "input" : "output", bufferNum);
3608     }
3609     return OK;
3610 }
3611 
setupVideoDecoder(const char * mime,const sp<AMessage> & msg,bool haveNativeWindow,bool usingSwRenderer,sp<AMessage> & outputFormat)3612 status_t ACodec::setupVideoDecoder(
3613         const char *mime, const sp<AMessage> &msg, bool haveNativeWindow,
3614         bool usingSwRenderer, sp<AMessage> &outputFormat) {
3615     int32_t width, height;
3616     if (!msg->findInt32("width", &width)
3617             || !msg->findInt32("height", &height)) {
3618         return INVALID_OPERATION;
3619     }
3620 
3621     OMX_VIDEO_CODINGTYPE compressionFormat;
3622     status_t err = GetVideoCodingTypeFromMime(mime, &compressionFormat);
3623 
3624     if (err != OK) {
3625         return err;
3626     }
3627 
3628     if (compressionFormat == OMX_VIDEO_CodingHEVC) {
3629         int32_t profile;
3630         if (msg->findInt32("profile", &profile)) {
3631             // verify if Main10 profile is supported at all, and fail
3632             // immediately if it's not supported.
3633             if (profile == OMX_VIDEO_HEVCProfileMain10 ||
3634                 profile == OMX_VIDEO_HEVCProfileMain10HDR10) {
3635                 err = verifySupportForProfileAndLevel(
3636                         kPortIndexInput, profile, 0);
3637                 if (err != OK) {
3638                     return err;
3639                 }
3640             }
3641         }
3642     }
3643 
3644     if (compressionFormat == OMX_VIDEO_CodingVP9) {
3645         OMX_VIDEO_PARAM_PROFILELEVELTYPE params;
3646         InitOMXParams(&params);
3647         params.nPortIndex = kPortIndexInput;
3648         // Check if VP9 decoder advertises supported profiles.
3649         params.nProfileIndex = 0;
3650         status_t err = mOMXNode->getParameter(
3651                 OMX_IndexParamVideoProfileLevelQuerySupported,
3652                 &params, sizeof(params));
3653         mIsLegacyVP9Decoder = err != OK;
3654     }
3655 
3656     err = setVideoPortFormatType(
3657             kPortIndexInput, compressionFormat, OMX_COLOR_FormatUnused);
3658 
3659     if (err != OK) {
3660         return err;
3661     }
3662 
3663     int32_t tmp;
3664     if (msg->findInt32("color-format", &tmp)) {
3665         OMX_COLOR_FORMATTYPE colorFormat =
3666             static_cast<OMX_COLOR_FORMATTYPE>(tmp);
3667         err = setVideoPortFormatType(
3668                 kPortIndexOutput, OMX_VIDEO_CodingUnused, colorFormat, haveNativeWindow);
3669         if (err != OK) {
3670             ALOGW("[%s] does not support color format %d",
3671                   mComponentName.c_str(), colorFormat);
3672             err = setSupportedOutputFormat(!haveNativeWindow /* getLegacyFlexibleFormat */);
3673         }
3674     } else {
3675         err = setSupportedOutputFormat(!haveNativeWindow /* getLegacyFlexibleFormat */);
3676     }
3677 
3678     if (err != OK) {
3679         return err;
3680     }
3681 
3682     // Set the component input buffer number to be |tmp|. If succeed,
3683     // component will set input port buffer number to be |tmp|. If fail,
3684     // component will keep the same buffer number as before.
3685     if (msg->findInt32("android._num-input-buffers", &tmp)) {
3686         err = setPortBufferNum(kPortIndexInput, tmp);
3687         if (err != OK)
3688             return err;
3689     }
3690 
3691     // Set the component output buffer number to be |tmp|. If succeed,
3692     // component will set output port buffer number to be |tmp|. If fail,
3693     // component will keep the same buffer number as before.
3694     if (msg->findInt32("android._num-output-buffers", &tmp)) {
3695         err = setPortBufferNum(kPortIndexOutput, tmp);
3696         if (err != OK)
3697             return err;
3698     }
3699 
3700     int32_t frameRateInt;
3701     float frameRateFloat;
3702     if (!msg->findFloat("frame-rate", &frameRateFloat)) {
3703         if (!msg->findInt32("frame-rate", &frameRateInt)) {
3704             frameRateInt = -1;
3705         }
3706         frameRateFloat = (float)frameRateInt;
3707     }
3708 
3709     err = setVideoFormatOnPort(
3710             kPortIndexInput, width, height, compressionFormat, frameRateFloat);
3711 
3712     if (err != OK) {
3713         return err;
3714     }
3715 
3716     err = setVideoFormatOnPort(
3717             kPortIndexOutput, width, height, OMX_VIDEO_CodingUnused);
3718 
3719     if (err != OK) {
3720         return err;
3721     }
3722 
3723     err = setColorAspectsForVideoDecoder(
3724             width, height, haveNativeWindow | usingSwRenderer, msg, outputFormat);
3725     if (err == ERROR_UNSUPPORTED) { // support is optional
3726         err = OK;
3727     }
3728 
3729     if (err != OK) {
3730         return err;
3731     }
3732 
3733     err = setHDRStaticInfoForVideoCodec(kPortIndexOutput, msg, outputFormat);
3734     if (err == ERROR_UNSUPPORTED) { // support is optional
3735         err = OK;
3736     }
3737     return err;
3738 }
3739 
initDescribeColorAspectsIndex()3740 status_t ACodec::initDescribeColorAspectsIndex() {
3741     status_t err = mOMXNode->getExtensionIndex(
3742             "OMX.google.android.index.describeColorAspects", &mDescribeColorAspectsIndex);
3743     if (err != OK) {
3744         mDescribeColorAspectsIndex = (OMX_INDEXTYPE)0;
3745     }
3746     return err;
3747 }
3748 
setCodecColorAspects(DescribeColorAspectsParams & params,bool verify)3749 status_t ACodec::setCodecColorAspects(DescribeColorAspectsParams &params, bool verify) {
3750     status_t err = ERROR_UNSUPPORTED;
3751     if (mDescribeColorAspectsIndex) {
3752         err = mOMXNode->setConfig(mDescribeColorAspectsIndex, &params, sizeof(params));
3753     }
3754     ALOGV("[%s] setting color aspects (R:%d(%s), P:%d(%s), M:%d(%s), T:%d(%s)) err=%d(%s)",
3755             mComponentName.c_str(),
3756             params.sAspects.mRange, asString(params.sAspects.mRange),
3757             params.sAspects.mPrimaries, asString(params.sAspects.mPrimaries),
3758             params.sAspects.mMatrixCoeffs, asString(params.sAspects.mMatrixCoeffs),
3759             params.sAspects.mTransfer, asString(params.sAspects.mTransfer),
3760             err, asString(err));
3761 
3762     if (verify && err == OK) {
3763         err = getCodecColorAspects(params);
3764     }
3765 
3766     ALOGW_IF(err == ERROR_UNSUPPORTED && mDescribeColorAspectsIndex,
3767             "[%s] setting color aspects failed even though codec advertises support",
3768             mComponentName.c_str());
3769     return err;
3770 }
3771 
setColorAspectsForVideoDecoder(int32_t width,int32_t height,bool usingNativeWindow,const sp<AMessage> & configFormat,sp<AMessage> & outputFormat)3772 status_t ACodec::setColorAspectsForVideoDecoder(
3773         int32_t width, int32_t height, bool usingNativeWindow,
3774         const sp<AMessage> &configFormat, sp<AMessage> &outputFormat) {
3775     DescribeColorAspectsParams params;
3776     InitOMXParams(&params);
3777     params.nPortIndex = kPortIndexOutput;
3778 
3779     getColorAspectsFromFormat(configFormat, params.sAspects);
3780     if (usingNativeWindow) {
3781         setDefaultCodecColorAspectsIfNeeded(params.sAspects, width, height);
3782         // The default aspects will be set back to the output format during the
3783         // getFormat phase of configure(). Set non-Unspecified values back into the
3784         // format, in case component does not support this enumeration.
3785         setColorAspectsIntoFormat(params.sAspects, outputFormat);
3786     }
3787 
3788     (void)initDescribeColorAspectsIndex();
3789 
3790     // communicate color aspects to codec
3791     return setCodecColorAspects(params);
3792 }
3793 
getCodecColorAspects(DescribeColorAspectsParams & params)3794 status_t ACodec::getCodecColorAspects(DescribeColorAspectsParams &params) {
3795     status_t err = ERROR_UNSUPPORTED;
3796     if (mDescribeColorAspectsIndex) {
3797         err = mOMXNode->getConfig(mDescribeColorAspectsIndex, &params, sizeof(params));
3798     }
3799     ALOGV("[%s] got color aspects (R:%d(%s), P:%d(%s), M:%d(%s), T:%d(%s)) err=%d(%s)",
3800             mComponentName.c_str(),
3801             params.sAspects.mRange, asString(params.sAspects.mRange),
3802             params.sAspects.mPrimaries, asString(params.sAspects.mPrimaries),
3803             params.sAspects.mMatrixCoeffs, asString(params.sAspects.mMatrixCoeffs),
3804             params.sAspects.mTransfer, asString(params.sAspects.mTransfer),
3805             err, asString(err));
3806     if (params.bRequestingDataSpace) {
3807         ALOGV("for dataspace %#x", params.nDataSpace);
3808     }
3809     if (err == ERROR_UNSUPPORTED && mDescribeColorAspectsIndex
3810             && !params.bRequestingDataSpace && !params.bDataSpaceChanged) {
3811         ALOGW("[%s] getting color aspects failed even though codec advertises support",
3812                 mComponentName.c_str());
3813     }
3814     return err;
3815 }
3816 
getInputColorAspectsForVideoEncoder(sp<AMessage> & format)3817 status_t ACodec::getInputColorAspectsForVideoEncoder(sp<AMessage> &format) {
3818     DescribeColorAspectsParams params;
3819     InitOMXParams(&params);
3820     params.nPortIndex = kPortIndexInput;
3821     status_t err = getCodecColorAspects(params);
3822     if (err == OK) {
3823         // we only set encoder input aspects if codec supports them
3824         setColorAspectsIntoFormat(params.sAspects, format, true /* force */);
3825     }
3826     return err;
3827 }
3828 
getDataSpace(DescribeColorAspectsParams & params,android_dataspace * dataSpace,bool tryCodec)3829 status_t ACodec::getDataSpace(
3830         DescribeColorAspectsParams &params, android_dataspace *dataSpace /* nonnull */,
3831         bool tryCodec) {
3832     status_t err = OK;
3833     if (tryCodec) {
3834         // request dataspace guidance from codec.
3835         params.bRequestingDataSpace = OMX_TRUE;
3836         err = getCodecColorAspects(params);
3837         params.bRequestingDataSpace = OMX_FALSE;
3838         if (err == OK && params.nDataSpace != HAL_DATASPACE_UNKNOWN) {
3839             *dataSpace = (android_dataspace)params.nDataSpace;
3840             return err;
3841         } else if (err == ERROR_UNSUPPORTED) {
3842             // ignore not-implemented error for dataspace requests
3843             err = OK;
3844         }
3845     }
3846 
3847     // this returns legacy versions if available
3848     *dataSpace = getDataSpaceForColorAspects(params.sAspects, true /* mayexpand */);
3849     ALOGV("[%s] using color aspects (R:%d(%s), P:%d(%s), M:%d(%s), T:%d(%s)) "
3850           "and dataspace %#x",
3851             mComponentName.c_str(),
3852             params.sAspects.mRange, asString(params.sAspects.mRange),
3853             params.sAspects.mPrimaries, asString(params.sAspects.mPrimaries),
3854             params.sAspects.mMatrixCoeffs, asString(params.sAspects.mMatrixCoeffs),
3855             params.sAspects.mTransfer, asString(params.sAspects.mTransfer),
3856             *dataSpace);
3857     return err;
3858 }
3859 
3860 
getColorAspectsAndDataSpaceForVideoDecoder(int32_t width,int32_t height,const sp<AMessage> & configFormat,sp<AMessage> & outputFormat,android_dataspace * dataSpace)3861 status_t ACodec::getColorAspectsAndDataSpaceForVideoDecoder(
3862         int32_t width, int32_t height, const sp<AMessage> &configFormat, sp<AMessage> &outputFormat,
3863         android_dataspace *dataSpace) {
3864     DescribeColorAspectsParams params;
3865     InitOMXParams(&params);
3866     params.nPortIndex = kPortIndexOutput;
3867 
3868     // reset default format and get resulting format
3869     getColorAspectsFromFormat(configFormat, params.sAspects);
3870     if (dataSpace != NULL) {
3871         setDefaultCodecColorAspectsIfNeeded(params.sAspects, width, height);
3872     }
3873     status_t err = setCodecColorAspects(params, true /* readBack */);
3874 
3875     // we always set specified aspects for decoders
3876     setColorAspectsIntoFormat(params.sAspects, outputFormat);
3877 
3878     if (dataSpace != NULL) {
3879         status_t res = getDataSpace(params, dataSpace, err == OK /* tryCodec */);
3880         if (err == OK) {
3881             err = res;
3882         }
3883     }
3884 
3885     return err;
3886 }
3887 
3888 // initial video encoder setup for bytebuffer mode
setColorAspectsForVideoEncoder(const sp<AMessage> & configFormat,sp<AMessage> & outputFormat,sp<AMessage> & inputFormat)3889 status_t ACodec::setColorAspectsForVideoEncoder(
3890         const sp<AMessage> &configFormat, sp<AMessage> &outputFormat, sp<AMessage> &inputFormat) {
3891     // copy config to output format as this is not exposed via getFormat
3892     copyColorConfig(configFormat, outputFormat);
3893 
3894     DescribeColorAspectsParams params;
3895     InitOMXParams(&params);
3896     params.nPortIndex = kPortIndexInput;
3897     getColorAspectsFromFormat(configFormat, params.sAspects);
3898 
3899     (void)initDescribeColorAspectsIndex();
3900 
3901     int32_t usingRecorder;
3902     if (configFormat->findInt32("android._using-recorder", &usingRecorder) && usingRecorder) {
3903         android_dataspace dataSpace = HAL_DATASPACE_BT709;
3904         int32_t width, height;
3905         if (configFormat->findInt32("width", &width)
3906                 && configFormat->findInt32("height", &height)) {
3907             setDefaultCodecColorAspectsIfNeeded(params.sAspects, width, height);
3908             status_t err = getDataSpace(
3909                     params, &dataSpace, mDescribeColorAspectsIndex /* tryCodec */);
3910             if (err != OK) {
3911                 return err;
3912             }
3913             setColorAspectsIntoFormat(params.sAspects, outputFormat);
3914         }
3915         inputFormat->setInt32("android._dataspace", (int32_t)dataSpace);
3916     }
3917 
3918     // communicate color aspects to codec, but do not allow change of the platform aspects
3919     ColorAspects origAspects = params.sAspects;
3920     for (int triesLeft = 2; --triesLeft >= 0; ) {
3921         status_t err = setCodecColorAspects(params, true /* readBack */);
3922         if (err != OK
3923                 || !ColorUtils::checkIfAspectsChangedAndUnspecifyThem(
3924                         params.sAspects, origAspects, true /* usePlatformAspects */)) {
3925             return err;
3926         }
3927         ALOGW_IF(triesLeft == 0, "[%s] Codec repeatedly changed requested ColorAspects.",
3928                 mComponentName.c_str());
3929     }
3930     return OK;
3931 }
3932 
setHDRStaticInfoForVideoCodec(OMX_U32 portIndex,const sp<AMessage> & configFormat,sp<AMessage> & outputFormat)3933 status_t ACodec::setHDRStaticInfoForVideoCodec(
3934         OMX_U32 portIndex, const sp<AMessage> &configFormat, sp<AMessage> &outputFormat) {
3935     CHECK(portIndex == kPortIndexInput || portIndex == kPortIndexOutput);
3936 
3937     DescribeHDRStaticInfoParams params;
3938     InitOMXParams(&params);
3939     params.nPortIndex = portIndex;
3940 
3941     HDRStaticInfo *info = &params.sInfo;
3942     if (getHDRStaticInfoFromFormat(configFormat, info)) {
3943         setHDRStaticInfoIntoFormat(params.sInfo, outputFormat);
3944     }
3945 
3946     (void)initDescribeHDRStaticInfoIndex();
3947 
3948     // communicate HDR static Info to codec
3949     return setHDRStaticInfo(params);
3950 }
3951 
3952 // subsequent initial video encoder setup for surface mode
setInitialColorAspectsForVideoEncoderSurfaceAndGetDataSpace(android_dataspace * dataSpace)3953 status_t ACodec::setInitialColorAspectsForVideoEncoderSurfaceAndGetDataSpace(
3954         android_dataspace *dataSpace /* nonnull */) {
3955     DescribeColorAspectsParams params;
3956     InitOMXParams(&params);
3957     params.nPortIndex = kPortIndexInput;
3958     ColorAspects &aspects = params.sAspects;
3959 
3960     // reset default format and store resulting format into both input and output formats
3961     getColorAspectsFromFormat(mConfigFormat, aspects);
3962     int32_t width, height;
3963     if (mInputFormat->findInt32("width", &width) && mInputFormat->findInt32("height", &height)) {
3964         setDefaultCodecColorAspectsIfNeeded(aspects, width, height);
3965     }
3966     setColorAspectsIntoFormat(aspects, mInputFormat);
3967     setColorAspectsIntoFormat(aspects, mOutputFormat);
3968 
3969     // communicate color aspects to codec, but do not allow any change
3970     ColorAspects origAspects = aspects;
3971     status_t err = OK;
3972     for (int triesLeft = 2; mDescribeColorAspectsIndex && --triesLeft >= 0; ) {
3973         status_t err = setCodecColorAspects(params, true /* readBack */);
3974         if (err != OK || !ColorUtils::checkIfAspectsChangedAndUnspecifyThem(aspects, origAspects)) {
3975             break;
3976         }
3977         ALOGW_IF(triesLeft == 0, "[%s] Codec repeatedly changed requested ColorAspects.",
3978                 mComponentName.c_str());
3979     }
3980 
3981     *dataSpace = HAL_DATASPACE_BT709;
3982     aspects = origAspects; // restore desired color aspects
3983     status_t res = getDataSpace(
3984             params, dataSpace, err == OK && mDescribeColorAspectsIndex /* tryCodec */);
3985     if (err == OK) {
3986         err = res;
3987     }
3988     mInputFormat->setInt32("android._dataspace", (int32_t)*dataSpace);
3989     mInputFormat->setBuffer(
3990             "android._color-aspects", ABuffer::CreateAsCopy(&aspects, sizeof(aspects)));
3991 
3992     // update input format with codec supported color aspects (basically set unsupported
3993     // aspects to Unspecified)
3994     if (err == OK) {
3995         (void)getInputColorAspectsForVideoEncoder(mInputFormat);
3996     }
3997 
3998     ALOGV("set default color aspects, updated input format to %s, output format to %s",
3999             mInputFormat->debugString(4).c_str(), mOutputFormat->debugString(4).c_str());
4000 
4001     return err;
4002 }
4003 
getHDRStaticInfoForVideoCodec(OMX_U32 portIndex,sp<AMessage> & format)4004 status_t ACodec::getHDRStaticInfoForVideoCodec(OMX_U32 portIndex, sp<AMessage> &format) {
4005     CHECK(portIndex == kPortIndexInput || portIndex == kPortIndexOutput);
4006     DescribeHDRStaticInfoParams params;
4007     InitOMXParams(&params);
4008     params.nPortIndex = portIndex;
4009 
4010     status_t err = getHDRStaticInfo(params);
4011     if (err == OK) {
4012         // we only set decodec output HDRStaticInfo if codec supports them
4013         setHDRStaticInfoIntoFormat(params.sInfo, format);
4014     }
4015     return err;
4016 }
4017 
initDescribeHDRStaticInfoIndex()4018 status_t ACodec::initDescribeHDRStaticInfoIndex() {
4019     status_t err = mOMXNode->getExtensionIndex(
4020             "OMX.google.android.index.describeHDRStaticInfo", &mDescribeHDRStaticInfoIndex);
4021     if (err != OK) {
4022         mDescribeHDRStaticInfoIndex = (OMX_INDEXTYPE)0;
4023         return err;
4024     }
4025 
4026     err = mOMXNode->getExtensionIndex(
4027                 "OMX.google.android.index.describeHDR10PlusInfo", &mDescribeHDR10PlusInfoIndex);
4028     if (err != OK) {
4029         mDescribeHDR10PlusInfoIndex = (OMX_INDEXTYPE)0;
4030         return err;
4031     }
4032 
4033     return OK;
4034 }
4035 
setHDRStaticInfo(const DescribeHDRStaticInfoParams & params)4036 status_t ACodec::setHDRStaticInfo(const DescribeHDRStaticInfoParams &params) {
4037     status_t err = ERROR_UNSUPPORTED;
4038     if (mDescribeHDRStaticInfoIndex) {
4039         err = mOMXNode->setConfig(mDescribeHDRStaticInfoIndex, &params, sizeof(params));
4040     }
4041 
4042     const HDRStaticInfo *info = &params.sInfo;
4043     ALOGV("[%s] setting  HDRStaticInfo (R: %u %u, G: %u %u, B: %u, %u, W: %u, %u, "
4044             "MaxDispL: %u, MinDispL: %u, MaxContentL: %u, MaxFrameAvgL: %u)",
4045             mComponentName.c_str(),
4046             info->sType1.mR.x, info->sType1.mR.y, info->sType1.mG.x, info->sType1.mG.y,
4047             info->sType1.mB.x, info->sType1.mB.y, info->sType1.mW.x, info->sType1.mW.y,
4048             info->sType1.mMaxDisplayLuminance, info->sType1.mMinDisplayLuminance,
4049             info->sType1.mMaxContentLightLevel, info->sType1.mMaxFrameAverageLightLevel);
4050 
4051     ALOGW_IF(err == ERROR_UNSUPPORTED && mDescribeHDRStaticInfoIndex,
4052             "[%s] setting HDRStaticInfo failed even though codec advertises support",
4053             mComponentName.c_str());
4054     return err;
4055 }
4056 
getHDRStaticInfo(DescribeHDRStaticInfoParams & params)4057 status_t ACodec::getHDRStaticInfo(DescribeHDRStaticInfoParams &params) {
4058     status_t err = ERROR_UNSUPPORTED;
4059     if (mDescribeHDRStaticInfoIndex) {
4060         err = mOMXNode->getConfig(mDescribeHDRStaticInfoIndex, &params, sizeof(params));
4061     }
4062 
4063     ALOGW_IF(err == ERROR_UNSUPPORTED && mDescribeHDRStaticInfoIndex,
4064             "[%s] getting HDRStaticInfo failed even though codec advertises support",
4065             mComponentName.c_str());
4066     return err;
4067 }
4068 
setupVideoEncoder(const char * mime,const sp<AMessage> & msg,sp<AMessage> & outputFormat,sp<AMessage> & inputFormat)4069 status_t ACodec::setupVideoEncoder(
4070         const char *mime, const sp<AMessage> &msg,
4071         sp<AMessage> &outputFormat, sp<AMessage> &inputFormat) {
4072     int32_t tmp;
4073     if (!msg->findInt32("color-format", &tmp)) {
4074         return INVALID_OPERATION;
4075     }
4076 
4077     OMX_COLOR_FORMATTYPE colorFormat =
4078         static_cast<OMX_COLOR_FORMATTYPE>(tmp);
4079 
4080     status_t err = setVideoPortFormatType(
4081             kPortIndexInput, OMX_VIDEO_CodingUnused, colorFormat);
4082 
4083     if (err != OK) {
4084         ALOGE("[%s] does not support color format %d",
4085               mComponentName.c_str(), colorFormat);
4086 
4087         return err;
4088     }
4089 
4090     /* Input port configuration */
4091 
4092     OMX_PARAM_PORTDEFINITIONTYPE def;
4093     InitOMXParams(&def);
4094 
4095     OMX_VIDEO_PORTDEFINITIONTYPE *video_def = &def.format.video;
4096 
4097     def.nPortIndex = kPortIndexInput;
4098 
4099     err = mOMXNode->getParameter(
4100             OMX_IndexParamPortDefinition, &def, sizeof(def));
4101 
4102     if (err != OK) {
4103         return err;
4104     }
4105 
4106     OMX_VIDEO_CONTROLRATETYPE bitrateMode;
4107     int32_t width, height, bitrate = 0, quality;
4108     if (!msg->findInt32("width", &width)
4109             || !msg->findInt32("height", &height)
4110             || !findVideoBitrateControlInfo(
4111                     msg, &bitrateMode, &bitrate, &quality)) {
4112         return INVALID_OPERATION;
4113     }
4114 
4115     video_def->nFrameWidth = width;
4116     video_def->nFrameHeight = height;
4117 
4118     int32_t stride;
4119     if (!msg->findInt32("stride", &stride)) {
4120         stride = width;
4121     }
4122 
4123     video_def->nStride = stride;
4124 
4125     int32_t sliceHeight;
4126     if (!msg->findInt32("slice-height", &sliceHeight)) {
4127         sliceHeight = height;
4128     }
4129 
4130     video_def->nSliceHeight = sliceHeight;
4131 
4132     def.nBufferSize = (video_def->nStride * video_def->nSliceHeight * 3) / 2;
4133 
4134     float framerate;
4135     if (!msg->findFloat("frame-rate", &framerate)) {
4136         int32_t tmp;
4137         if (!msg->findInt32("frame-rate", &tmp)) {
4138             return INVALID_OPERATION;
4139         }
4140         mFps = (double)tmp;
4141     } else {
4142         mFps = (double)framerate;
4143     }
4144     // propagate framerate to the output so that the muxer has it
4145     outputFormat->setInt32("frame-rate", (int32_t)mFps);
4146 
4147     video_def->xFramerate = (OMX_U32)(mFps * 65536);
4148     video_def->eCompressionFormat = OMX_VIDEO_CodingUnused;
4149     // this is redundant as it was already set up in setVideoPortFormatType
4150     // FIXME for now skip this only for flexible YUV formats
4151     if (colorFormat != OMX_COLOR_FormatYUV420Flexible) {
4152         video_def->eColorFormat = colorFormat;
4153     }
4154 
4155     err = mOMXNode->setParameter(
4156             OMX_IndexParamPortDefinition, &def, sizeof(def));
4157 
4158     if (err != OK) {
4159         ALOGE("[%s] failed to set input port definition parameters.",
4160               mComponentName.c_str());
4161 
4162         return err;
4163     }
4164 
4165     /* Output port configuration */
4166 
4167     OMX_VIDEO_CODINGTYPE compressionFormat;
4168     err = GetVideoCodingTypeFromMime(mime, &compressionFormat);
4169 
4170     if (err != OK) {
4171         return err;
4172     }
4173 
4174     err = setVideoPortFormatType(
4175             kPortIndexOutput, compressionFormat, OMX_COLOR_FormatUnused);
4176 
4177     if (err != OK) {
4178         ALOGE("[%s] does not support compression format %d",
4179              mComponentName.c_str(), compressionFormat);
4180 
4181         return err;
4182     }
4183 
4184     def.nPortIndex = kPortIndexOutput;
4185 
4186     err = mOMXNode->getParameter(
4187             OMX_IndexParamPortDefinition, &def, sizeof(def));
4188 
4189     if (err != OK) {
4190         return err;
4191     }
4192 
4193     video_def->nFrameWidth = width;
4194     video_def->nFrameHeight = height;
4195     video_def->xFramerate = 0;
4196     video_def->nBitrate = bitrate;
4197     video_def->eCompressionFormat = compressionFormat;
4198     video_def->eColorFormat = OMX_COLOR_FormatUnused;
4199 
4200     err = mOMXNode->setParameter(
4201             OMX_IndexParamPortDefinition, &def, sizeof(def));
4202 
4203     if (err != OK) {
4204         ALOGE("[%s] failed to set output port definition parameters.",
4205               mComponentName.c_str());
4206 
4207         return err;
4208     }
4209 
4210     int32_t intraRefreshPeriod = 0;
4211     if (msg->findInt32("intra-refresh-period", &intraRefreshPeriod)
4212             && intraRefreshPeriod >= 0) {
4213         err = setIntraRefreshPeriod((uint32_t)intraRefreshPeriod, true);
4214         if (err != OK) {
4215             ALOGI("[%s] failed setIntraRefreshPeriod. Failure is fine since this key is optional",
4216                     mComponentName.c_str());
4217             err = OK;
4218         }
4219     }
4220 
4221     configureEncoderLatency(msg);
4222 
4223     switch (compressionFormat) {
4224         case OMX_VIDEO_CodingMPEG4:
4225             err = setupMPEG4EncoderParameters(msg);
4226             break;
4227 
4228         case OMX_VIDEO_CodingH263:
4229             err = setupH263EncoderParameters(msg);
4230             break;
4231 
4232         case OMX_VIDEO_CodingAVC:
4233             err = setupAVCEncoderParameters(msg);
4234             break;
4235 
4236         case OMX_VIDEO_CodingHEVC:
4237         case OMX_VIDEO_CodingImageHEIC:
4238             err = setupHEVCEncoderParameters(msg, outputFormat);
4239             break;
4240 
4241         case OMX_VIDEO_CodingVP8:
4242         case OMX_VIDEO_CodingVP9:
4243             err = setupVPXEncoderParameters(msg, outputFormat);
4244             break;
4245 
4246         default:
4247             break;
4248     }
4249 
4250     if (err != OK) {
4251         return err;
4252     }
4253 
4254     // Set up color aspects on input, but propagate them to the output format, as they will
4255     // not be read back from encoder.
4256     err = setColorAspectsForVideoEncoder(msg, outputFormat, inputFormat);
4257     if (err == ERROR_UNSUPPORTED) {
4258         ALOGI("[%s] cannot encode color aspects. Ignoring.", mComponentName.c_str());
4259         err = OK;
4260     }
4261 
4262     if (err != OK) {
4263         return err;
4264     }
4265 
4266     err = setHDRStaticInfoForVideoCodec(kPortIndexInput, msg, outputFormat);
4267     if (err == ERROR_UNSUPPORTED) { // support is optional
4268         ALOGI("[%s] cannot encode HDR static metadata. Ignoring.", mComponentName.c_str());
4269         err = OK;
4270     }
4271 
4272     if (err != OK) {
4273         return err;
4274     }
4275 
4276     switch (compressionFormat) {
4277         case OMX_VIDEO_CodingAVC:
4278         case OMX_VIDEO_CodingHEVC:
4279             err = configureTemporalLayers(msg, true /* inConfigure */, outputFormat);
4280             if (err != OK) {
4281                 err = OK; // ignore failure
4282             }
4283             break;
4284 
4285         case OMX_VIDEO_CodingVP8:
4286         case OMX_VIDEO_CodingVP9:
4287             // TODO: do we need to support android.generic layering? webrtc layering is
4288             // already set up in setupVPXEncoderParameters.
4289             break;
4290 
4291         default:
4292             break;
4293     }
4294 
4295     if (err == OK) {
4296         ALOGI("setupVideoEncoder succeeded");
4297     }
4298 
4299     // Video should be encoded as stand straight because RTP protocol
4300     // can provide rotation information only if CVO is supported.
4301     // This needs to be added to support non-CVO case for video streaming scenario.
4302     int32_t rotation = 0;
4303     if (msg->findInt32("rotation-degrees", &rotation)) {
4304         OMX_CONFIG_ROTATIONTYPE config;
4305         InitOMXParams(&config);
4306         config.nPortIndex = kPortIndexOutput;
4307         status_t err = mOMXNode->getConfig(
4308                 (OMX_INDEXTYPE)OMX_IndexConfigCommonRotate, &config, sizeof(config));
4309         if (err != OK) {
4310             ALOGW("Failed to getConfig of OMX_IndexConfigCommonRotate(err %d)", err);
4311         }
4312         config.nRotation = rotation;
4313         err = mOMXNode->setConfig(
4314                 (OMX_INDEXTYPE)OMX_IndexConfigCommonRotate, &config, sizeof(config));
4315 
4316         ALOGD("Applying encoder-rotation=[%d] to video encoder.", config.nRotation);
4317         if (err != OK) {
4318             ALOGW("Failed to setConfig of OMX_IndexConfigCommonRotate(err %d)", err);
4319         }
4320     }
4321 
4322     return err;
4323 }
4324 
setCyclicIntraMacroblockRefresh(const sp<AMessage> & msg,int32_t mode)4325 status_t ACodec::setCyclicIntraMacroblockRefresh(const sp<AMessage> &msg, int32_t mode) {
4326     OMX_VIDEO_PARAM_INTRAREFRESHTYPE params;
4327     InitOMXParams(&params);
4328     params.nPortIndex = kPortIndexOutput;
4329 
4330     params.eRefreshMode = static_cast<OMX_VIDEO_INTRAREFRESHTYPE>(mode);
4331 
4332     if (params.eRefreshMode == OMX_VIDEO_IntraRefreshCyclic ||
4333             params.eRefreshMode == OMX_VIDEO_IntraRefreshBoth) {
4334         int32_t mbs;
4335         if (!msg->findInt32("intra-refresh-CIR-mbs", &mbs)) {
4336             return INVALID_OPERATION;
4337         }
4338         params.nCirMBs = mbs;
4339     }
4340 
4341     if (params.eRefreshMode == OMX_VIDEO_IntraRefreshAdaptive ||
4342             params.eRefreshMode == OMX_VIDEO_IntraRefreshBoth) {
4343         int32_t mbs;
4344         if (!msg->findInt32("intra-refresh-AIR-mbs", &mbs)) {
4345             return INVALID_OPERATION;
4346         }
4347         params.nAirMBs = mbs;
4348 
4349         int32_t ref;
4350         if (!msg->findInt32("intra-refresh-AIR-ref", &ref)) {
4351             return INVALID_OPERATION;
4352         }
4353         params.nAirRef = ref;
4354     }
4355 
4356     status_t err = mOMXNode->setParameter(
4357             OMX_IndexParamVideoIntraRefresh, &params, sizeof(params));
4358     return err;
4359 }
4360 
setPFramesSpacing(float iFramesInterval,int32_t frameRate,uint32_t BFramesSpacing=0)4361 static OMX_U32 setPFramesSpacing(
4362         float iFramesInterval /* seconds */, int32_t frameRate, uint32_t BFramesSpacing = 0) {
4363     // BFramesSpacing is the number of B frames between I/P frames
4364     // PFramesSpacing (the value to be returned) is the number of P frames between I frames
4365     //
4366     // keyFrameInterval = ((PFramesSpacing + 1) * BFramesSpacing) + PFramesSpacing + 1
4367     //                                     ^^^                            ^^^        ^^^
4368     //                              number of B frames                number of P    I frame
4369     //
4370     //                  = (PFramesSpacing + 1) * (BFramesSpacing + 1)
4371     //
4372     // E.g.
4373     //      I   P   I  : I-interval: 8, nPFrames 1, nBFrames 3
4374     //       BBB BBB
4375 
4376     if (iFramesInterval < 0) { // just 1 key frame
4377         return 0xFFFFFFFE; // don't use maxint as key-frame-interval calculation will add 1
4378     } else if (iFramesInterval == 0) { // just key frames
4379         return 0;
4380     }
4381 
4382     // round down as key-frame-interval is an upper limit
4383     uint32_t keyFrameInterval = uint32_t(frameRate * iFramesInterval);
4384     OMX_U32 ret = keyFrameInterval / (BFramesSpacing + 1);
4385     return ret > 0 ? ret - 1 : 0;
4386 }
4387 
setupMPEG4EncoderParameters(const sp<AMessage> & msg)4388 status_t ACodec::setupMPEG4EncoderParameters(const sp<AMessage> &msg) {
4389     int32_t bitrate;
4390     float iFrameInterval;
4391     if (!msg->findInt32("bitrate", &bitrate)
4392             || !msg->findAsFloat("i-frame-interval", &iFrameInterval)) {
4393         return INVALID_OPERATION;
4394     }
4395 
4396     OMX_VIDEO_CONTROLRATETYPE bitrateMode = getVideoBitrateMode(msg);
4397 
4398     float frameRate;
4399     if (!msg->findFloat("frame-rate", &frameRate)) {
4400         int32_t tmp;
4401         if (!msg->findInt32("frame-rate", &tmp)) {
4402             return INVALID_OPERATION;
4403         }
4404         frameRate = (float)tmp;
4405     }
4406 
4407     OMX_VIDEO_PARAM_MPEG4TYPE mpeg4type;
4408     InitOMXParams(&mpeg4type);
4409     mpeg4type.nPortIndex = kPortIndexOutput;
4410 
4411     status_t err = mOMXNode->getParameter(
4412             OMX_IndexParamVideoMpeg4, &mpeg4type, sizeof(mpeg4type));
4413 
4414     if (err != OK) {
4415         return err;
4416     }
4417 
4418     mpeg4type.nSliceHeaderSpacing = 0;
4419     mpeg4type.bSVH = OMX_FALSE;
4420     mpeg4type.bGov = OMX_FALSE;
4421 
4422     mpeg4type.nAllowedPictureTypes =
4423         OMX_VIDEO_PictureTypeI | OMX_VIDEO_PictureTypeP;
4424 
4425     mpeg4type.nBFrames = 0;
4426     mpeg4type.nPFrames = setPFramesSpacing(iFrameInterval, frameRate, mpeg4type.nBFrames);
4427     if (mpeg4type.nPFrames == 0) {
4428         mpeg4type.nAllowedPictureTypes = OMX_VIDEO_PictureTypeI;
4429     }
4430     mpeg4type.nIDCVLCThreshold = 0;
4431     mpeg4type.bACPred = OMX_TRUE;
4432     mpeg4type.nMaxPacketSize = 256;
4433     mpeg4type.nTimeIncRes = 1000;
4434     mpeg4type.nHeaderExtension = 0;
4435     mpeg4type.bReversibleVLC = OMX_FALSE;
4436 
4437     int32_t profile;
4438     if (msg->findInt32("profile", &profile)) {
4439         int32_t level;
4440         if (!msg->findInt32("level", &level)) {
4441             return INVALID_OPERATION;
4442         }
4443 
4444         err = verifySupportForProfileAndLevel(kPortIndexOutput, profile, level);
4445 
4446         if (err != OK) {
4447             return err;
4448         }
4449 
4450         mpeg4type.eProfile = static_cast<OMX_VIDEO_MPEG4PROFILETYPE>(profile);
4451         mpeg4type.eLevel = static_cast<OMX_VIDEO_MPEG4LEVELTYPE>(level);
4452     }
4453 
4454     err = mOMXNode->setParameter(
4455             OMX_IndexParamVideoMpeg4, &mpeg4type, sizeof(mpeg4type));
4456 
4457     if (err != OK) {
4458         return err;
4459     }
4460 
4461     err = configureBitrate(bitrateMode, bitrate);
4462 
4463     if (err != OK) {
4464         return err;
4465     }
4466 
4467     return setupErrorCorrectionParameters();
4468 }
4469 
setupH263EncoderParameters(const sp<AMessage> & msg)4470 status_t ACodec::setupH263EncoderParameters(const sp<AMessage> &msg) {
4471     int32_t bitrate;
4472     float iFrameInterval;
4473     if (!msg->findInt32("bitrate", &bitrate)
4474             || !msg->findAsFloat("i-frame-interval", &iFrameInterval)) {
4475         return INVALID_OPERATION;
4476     }
4477 
4478     OMX_VIDEO_CONTROLRATETYPE bitrateMode = getVideoBitrateMode(msg);
4479 
4480     float frameRate;
4481     if (!msg->findFloat("frame-rate", &frameRate)) {
4482         int32_t tmp;
4483         if (!msg->findInt32("frame-rate", &tmp)) {
4484             return INVALID_OPERATION;
4485         }
4486         frameRate = (float)tmp;
4487     }
4488 
4489     OMX_VIDEO_PARAM_H263TYPE h263type;
4490     InitOMXParams(&h263type);
4491     h263type.nPortIndex = kPortIndexOutput;
4492 
4493     status_t err = mOMXNode->getParameter(
4494             OMX_IndexParamVideoH263, &h263type, sizeof(h263type));
4495 
4496     if (err != OK) {
4497         return err;
4498     }
4499 
4500     h263type.nAllowedPictureTypes =
4501         OMX_VIDEO_PictureTypeI | OMX_VIDEO_PictureTypeP;
4502 
4503     h263type.nBFrames = 0;
4504     h263type.nPFrames = setPFramesSpacing(iFrameInterval, frameRate, h263type.nBFrames);
4505     if (h263type.nPFrames == 0) {
4506         h263type.nAllowedPictureTypes = OMX_VIDEO_PictureTypeI;
4507     }
4508 
4509     int32_t profile;
4510     if (msg->findInt32("profile", &profile)) {
4511         int32_t level;
4512         if (!msg->findInt32("level", &level)) {
4513             return INVALID_OPERATION;
4514         }
4515 
4516         err = verifySupportForProfileAndLevel(kPortIndexOutput, profile, level);
4517 
4518         if (err != OK) {
4519             return err;
4520         }
4521 
4522         h263type.eProfile = static_cast<OMX_VIDEO_H263PROFILETYPE>(profile);
4523         h263type.eLevel = static_cast<OMX_VIDEO_H263LEVELTYPE>(level);
4524     }
4525 
4526     h263type.bPLUSPTYPEAllowed = OMX_FALSE;
4527     h263type.bForceRoundingTypeToZero = OMX_FALSE;
4528     h263type.nPictureHeaderRepetition = 0;
4529     h263type.nGOBHeaderInterval = 0;
4530 
4531     err = mOMXNode->setParameter(
4532             OMX_IndexParamVideoH263, &h263type, sizeof(h263type));
4533 
4534     if (err != OK) {
4535         return err;
4536     }
4537 
4538     err = configureBitrate(bitrateMode, bitrate);
4539 
4540     if (err != OK) {
4541         return err;
4542     }
4543 
4544     return setupErrorCorrectionParameters();
4545 }
4546 
4547 // static
getAVCLevelFor(int width,int height,int rate,int bitrate,OMX_VIDEO_AVCPROFILEEXTTYPE profile)4548 int /* OMX_VIDEO_AVCLEVELTYPE */ ACodec::getAVCLevelFor(
4549         int width, int height, int rate, int bitrate,
4550         OMX_VIDEO_AVCPROFILEEXTTYPE profile) {
4551     // convert bitrate to main/baseline profile kbps equivalent
4552     switch ((uint32_t)profile) {
4553         case OMX_VIDEO_AVCProfileHigh10:
4554             bitrate = divUp(bitrate, 3000); break;
4555         case OMX_VIDEO_AVCProfileConstrainedHigh:
4556         case OMX_VIDEO_AVCProfileHigh:
4557             bitrate = divUp(bitrate, 1250); break;
4558         default:
4559             bitrate = divUp(bitrate, 1000); break;
4560     }
4561 
4562     // convert size and rate to MBs
4563     width = divUp(width, 16);
4564     height = divUp(height, 16);
4565     int mbs = width * height;
4566     rate *= mbs;
4567     int maxDimension = max(width, height);
4568 
4569     static const int limits[][5] = {
4570         /*    MBps      MB   dim  bitrate        level */
4571         {     1485,     99,   28,     64, OMX_VIDEO_AVCLevel1  },
4572         {     1485,     99,   28,    128, OMX_VIDEO_AVCLevel1b },
4573         {     3000,    396,   56,    192, OMX_VIDEO_AVCLevel11 },
4574         {     6000,    396,   56,    384, OMX_VIDEO_AVCLevel12 },
4575         {    11880,    396,   56,    768, OMX_VIDEO_AVCLevel13 },
4576         {    11880,    396,   56,   2000, OMX_VIDEO_AVCLevel2  },
4577         {    19800,    792,   79,   4000, OMX_VIDEO_AVCLevel21 },
4578         {    20250,   1620,  113,   4000, OMX_VIDEO_AVCLevel22 },
4579         {    40500,   1620,  113,  10000, OMX_VIDEO_AVCLevel3  },
4580         {   108000,   3600,  169,  14000, OMX_VIDEO_AVCLevel31 },
4581         {   216000,   5120,  202,  20000, OMX_VIDEO_AVCLevel32 },
4582         {   245760,   8192,  256,  20000, OMX_VIDEO_AVCLevel4  },
4583         {   245760,   8192,  256,  50000, OMX_VIDEO_AVCLevel41 },
4584         {   522240,   8704,  263,  50000, OMX_VIDEO_AVCLevel42 },
4585         {   589824,  22080,  420, 135000, OMX_VIDEO_AVCLevel5  },
4586         {   983040,  36864,  543, 240000, OMX_VIDEO_AVCLevel51 },
4587         {  2073600,  36864,  543, 240000, OMX_VIDEO_AVCLevel52 },
4588         {  4177920, 139264, 1055, 240000, OMX_VIDEO_AVCLevel6  },
4589         {  8355840, 139264, 1055, 480000, OMX_VIDEO_AVCLevel61 },
4590         { 16711680, 139264, 1055, 800000, OMX_VIDEO_AVCLevel62 },
4591     };
4592 
4593     for (size_t i = 0; i < ARRAY_SIZE(limits); i++) {
4594         const int (&limit)[5] = limits[i];
4595         if (rate <= limit[0] && mbs <= limit[1] && maxDimension <= limit[2]
4596                 && bitrate <= limit[3]) {
4597             return limit[4];
4598         }
4599     }
4600     return 0;
4601 }
4602 
setupAVCEncoderParameters(const sp<AMessage> & msg)4603 status_t ACodec::setupAVCEncoderParameters(const sp<AMessage> &msg) {
4604     int32_t bitrate;
4605     float iFrameInterval;
4606     if (!msg->findInt32("bitrate", &bitrate)
4607             || !msg->findAsFloat("i-frame-interval", &iFrameInterval)) {
4608         return INVALID_OPERATION;
4609     }
4610 
4611     OMX_VIDEO_CONTROLRATETYPE bitrateMode = getVideoBitrateMode(msg);
4612 
4613     float frameRate;
4614     if (!msg->findFloat("frame-rate", &frameRate)) {
4615         int32_t tmp;
4616         if (!msg->findInt32("frame-rate", &tmp)) {
4617             return INVALID_OPERATION;
4618         }
4619         frameRate = (float)tmp;
4620     }
4621 
4622     status_t err = OK;
4623     int32_t intraRefreshMode = 0;
4624     if (msg->findInt32("intra-refresh-mode", &intraRefreshMode)) {
4625         err = setCyclicIntraMacroblockRefresh(msg, intraRefreshMode);
4626         if (err != OK) {
4627             ALOGE("Setting intra macroblock refresh mode (%d) failed: 0x%x",
4628                     err, intraRefreshMode);
4629             return err;
4630         }
4631     }
4632 
4633     OMX_VIDEO_PARAM_AVCTYPE h264type;
4634     InitOMXParams(&h264type);
4635     h264type.nPortIndex = kPortIndexOutput;
4636 
4637     err = mOMXNode->getParameter(
4638             OMX_IndexParamVideoAvc, &h264type, sizeof(h264type));
4639 
4640     if (err != OK) {
4641         return err;
4642     }
4643 
4644     h264type.nAllowedPictureTypes =
4645         OMX_VIDEO_PictureTypeI | OMX_VIDEO_PictureTypeP;
4646 
4647     int32_t profile;
4648     if (msg->findInt32("profile", &profile)) {
4649         int32_t level;
4650         if (!msg->findInt32("level", &level)) {
4651             return INVALID_OPERATION;
4652         }
4653 
4654         err = verifySupportForProfileAndLevel(kPortIndexOutput, profile, level);
4655 
4656         if (err != OK) {
4657             return err;
4658         }
4659 
4660         h264type.eProfile = static_cast<OMX_VIDEO_AVCPROFILETYPE>(profile);
4661         h264type.eLevel = static_cast<OMX_VIDEO_AVCLEVELTYPE>(level);
4662     } else {
4663         h264type.eProfile = OMX_VIDEO_AVCProfileBaseline;
4664 #if 0   /* DON'T YET DEFAULT TO HIGHEST PROFILE */
4665         // Use largest supported profile for AVC recording if profile is not specified.
4666         for (OMX_VIDEO_AVCPROFILETYPE profile : {
4667                 OMX_VIDEO_AVCProfileHigh, OMX_VIDEO_AVCProfileMain }) {
4668             if (verifySupportForProfileAndLevel(kPortIndexOutput, profile, 0) == OK) {
4669                 h264type.eProfile = profile;
4670                 break;
4671             }
4672         }
4673 #endif
4674     }
4675 
4676     ALOGI("setupAVCEncoderParameters with [profile: %s] [level: %s]",
4677             asString(h264type.eProfile), asString(h264type.eLevel));
4678 
4679     if (h264type.eProfile == OMX_VIDEO_AVCProfileBaseline) {
4680         h264type.nSliceHeaderSpacing = 0;
4681         h264type.bUseHadamard = OMX_TRUE;
4682         h264type.nRefFrames = 1;
4683         h264type.nBFrames = 0;
4684         h264type.nPFrames = setPFramesSpacing(iFrameInterval, frameRate, h264type.nBFrames);
4685         if (h264type.nPFrames == 0) {
4686             h264type.nAllowedPictureTypes = OMX_VIDEO_PictureTypeI;
4687         }
4688         h264type.nRefIdx10ActiveMinus1 = 0;
4689         h264type.nRefIdx11ActiveMinus1 = 0;
4690         h264type.bEntropyCodingCABAC = OMX_FALSE;
4691         h264type.bWeightedPPrediction = OMX_FALSE;
4692         h264type.bconstIpred = OMX_FALSE;
4693         h264type.bDirect8x8Inference = OMX_FALSE;
4694         h264type.bDirectSpatialTemporal = OMX_FALSE;
4695         h264type.nCabacInitIdc = 0;
4696     } else if (h264type.eProfile == OMX_VIDEO_AVCProfileMain ||
4697             h264type.eProfile == OMX_VIDEO_AVCProfileHigh) {
4698         h264type.nSliceHeaderSpacing = 0;
4699         h264type.bUseHadamard = OMX_TRUE;
4700         int32_t maxBframes = 0;
4701         (void)msg->findInt32(KEY_MAX_B_FRAMES, &maxBframes);
4702         h264type.nBFrames = uint32_t(maxBframes);
4703         if (mLatency && h264type.nBFrames > *mLatency) {
4704             h264type.nBFrames = *mLatency;
4705         }
4706         h264type.nRefFrames = h264type.nBFrames == 0 ? 1 : 2;
4707 
4708         h264type.nPFrames = setPFramesSpacing(iFrameInterval, frameRate, h264type.nBFrames);
4709         h264type.nAllowedPictureTypes =
4710             OMX_VIDEO_PictureTypeI | OMX_VIDEO_PictureTypeP;
4711         h264type.nRefIdx10ActiveMinus1 = 0;
4712         h264type.nRefIdx11ActiveMinus1 = 0;
4713         h264type.bEntropyCodingCABAC = OMX_TRUE;
4714         h264type.bWeightedPPrediction = OMX_TRUE;
4715         h264type.bconstIpred = OMX_TRUE;
4716         h264type.bDirect8x8Inference = OMX_TRUE;
4717         h264type.bDirectSpatialTemporal = OMX_TRUE;
4718         h264type.nCabacInitIdc = 1;
4719     }
4720 
4721     if (h264type.nBFrames != 0) {
4722         h264type.nAllowedPictureTypes |= OMX_VIDEO_PictureTypeB;
4723     }
4724 
4725     h264type.bEnableUEP = OMX_FALSE;
4726     h264type.bEnableFMO = OMX_FALSE;
4727     h264type.bEnableASO = OMX_FALSE;
4728     h264type.bEnableRS = OMX_FALSE;
4729     h264type.bFrameMBsOnly = OMX_TRUE;
4730     h264type.bMBAFF = OMX_FALSE;
4731     h264type.eLoopFilterMode = OMX_VIDEO_AVCLoopFilterEnable;
4732 
4733     err = mOMXNode->setParameter(
4734             OMX_IndexParamVideoAvc, &h264type, sizeof(h264type));
4735 
4736     if (err != OK) {
4737         return err;
4738     }
4739 
4740     // TRICKY: if we are enabling temporal layering as well, some codecs may not support layering
4741     // when B-frames are enabled. Detect this now so we can disable B frames if temporal layering
4742     // is preferred.
4743     AString tsSchema;
4744     int32_t preferBFrames = (int32_t)false;
4745     if (msg->findString("ts-schema", &tsSchema)
4746             && (!msg->findInt32("android._prefer-b-frames", &preferBFrames) || !preferBFrames)) {
4747         OMX_VIDEO_PARAM_ANDROID_TEMPORALLAYERINGTYPE layering;
4748         InitOMXParams(&layering);
4749         layering.nPortIndex = kPortIndexOutput;
4750         if (mOMXNode->getParameter(
4751                         (OMX_INDEXTYPE)OMX_IndexParamAndroidVideoTemporalLayering,
4752                         &layering, sizeof(layering)) == OK
4753                 && layering.eSupportedPatterns
4754                 && layering.nBLayerCountMax == 0) {
4755             h264type.nBFrames = 0;
4756             h264type.nPFrames = setPFramesSpacing(iFrameInterval, frameRate, h264type.nBFrames);
4757             h264type.nAllowedPictureTypes &= ~OMX_VIDEO_PictureTypeB;
4758             ALOGI("disabling B-frames");
4759             err = mOMXNode->setParameter(
4760                     OMX_IndexParamVideoAvc, &h264type, sizeof(h264type));
4761 
4762             if (err != OK) {
4763                 return err;
4764             }
4765         }
4766     }
4767 
4768     return configureBitrate(bitrateMode, bitrate);
4769 }
4770 
configureImageGrid(const sp<AMessage> & msg,sp<AMessage> & outputFormat)4771 status_t ACodec::configureImageGrid(
4772         const sp<AMessage> &msg, sp<AMessage> &outputFormat) {
4773     int32_t tileWidth, tileHeight, gridRows, gridCols;
4774     OMX_BOOL useGrid = OMX_FALSE;
4775     if (msg->findInt32("tile-width", &tileWidth) &&
4776         msg->findInt32("tile-height", &tileHeight) &&
4777         msg->findInt32("grid-rows", &gridRows) &&
4778         msg->findInt32("grid-cols", &gridCols)) {
4779         useGrid = OMX_TRUE;
4780     } else {
4781         // when bEnabled is false, the tile info is not used,
4782         // but clear out these too.
4783         tileWidth = tileHeight = gridRows = gridCols = 0;
4784     }
4785 
4786     if (!mIsImage && !useGrid) {
4787         return OK;
4788     }
4789 
4790     OMX_VIDEO_PARAM_ANDROID_IMAGEGRIDTYPE gridType;
4791     InitOMXParams(&gridType);
4792     gridType.nPortIndex = kPortIndexOutput;
4793     gridType.bEnabled = useGrid;
4794     gridType.nTileWidth = tileWidth;
4795     gridType.nTileHeight = tileHeight;
4796     gridType.nGridRows = gridRows;
4797     gridType.nGridCols = gridCols;
4798 
4799     ALOGV("sending image grid info to component: bEnabled %d, tile %dx%d, grid %dx%d",
4800             gridType.bEnabled,
4801             gridType.nTileWidth,
4802             gridType.nTileHeight,
4803             gridType.nGridRows,
4804             gridType.nGridCols);
4805 
4806     status_t err = mOMXNode->setParameter(
4807             (OMX_INDEXTYPE)OMX_IndexParamVideoAndroidImageGrid,
4808             &gridType, sizeof(gridType));
4809 
4810     // for video encoders, grid config is only a hint.
4811     if (!mIsImage) {
4812         return OK;
4813     }
4814 
4815     // image encoders must support grid config.
4816     if (err != OK) {
4817         return err;
4818     }
4819 
4820     // query to get the image encoder's real grid config as it might be
4821     // different from the requested, and transfer that to the output.
4822     err = mOMXNode->getParameter(
4823             (OMX_INDEXTYPE)OMX_IndexParamVideoAndroidImageGrid,
4824             &gridType, sizeof(gridType));
4825 
4826     ALOGV("received image grid info from component: bEnabled %d, tile %dx%d, grid %dx%d",
4827             gridType.bEnabled,
4828             gridType.nTileWidth,
4829             gridType.nTileHeight,
4830             gridType.nGridRows,
4831             gridType.nGridCols);
4832 
4833     if (err == OK && gridType.bEnabled) {
4834         outputFormat->setInt32("tile-width", gridType.nTileWidth);
4835         outputFormat->setInt32("tile-height", gridType.nTileHeight);
4836         outputFormat->setInt32("grid-rows", gridType.nGridRows);
4837         outputFormat->setInt32("grid-cols", gridType.nGridCols);
4838     }
4839 
4840     return err;
4841 }
4842 
setupHEVCEncoderParameters(const sp<AMessage> & msg,sp<AMessage> & outputFormat)4843 status_t ACodec::setupHEVCEncoderParameters(
4844         const sp<AMessage> &msg, sp<AMessage> &outputFormat) {
4845     OMX_VIDEO_CONTROLRATETYPE bitrateMode;
4846     int32_t bitrate, quality;
4847     if (!findVideoBitrateControlInfo(msg, &bitrateMode, &bitrate, &quality)) {
4848         return INVALID_OPERATION;
4849     }
4850 
4851     OMX_VIDEO_PARAM_HEVCTYPE hevcType;
4852     InitOMXParams(&hevcType);
4853     hevcType.nPortIndex = kPortIndexOutput;
4854 
4855     status_t err = OK;
4856     err = mOMXNode->getParameter(
4857             (OMX_INDEXTYPE)OMX_IndexParamVideoHevc, &hevcType, sizeof(hevcType));
4858     if (err != OK) {
4859         return err;
4860     }
4861 
4862     int32_t profile;
4863     if (msg->findInt32("profile", &profile)) {
4864         int32_t level;
4865         if (!msg->findInt32("level", &level)) {
4866             return INVALID_OPERATION;
4867         }
4868 
4869         err = verifySupportForProfileAndLevel(kPortIndexOutput, profile, level);
4870         if (err != OK) {
4871             return err;
4872         }
4873 
4874         hevcType.eProfile = static_cast<OMX_VIDEO_HEVCPROFILETYPE>(profile);
4875         hevcType.eLevel = static_cast<OMX_VIDEO_HEVCLEVELTYPE>(level);
4876     }
4877     // TODO: finer control?
4878     if (mIsImage) {
4879         hevcType.nKeyFrameInterval = 1;
4880     } else {
4881         float iFrameInterval;
4882         if (!msg->findAsFloat("i-frame-interval", &iFrameInterval)) {
4883             return INVALID_OPERATION;
4884         }
4885 
4886         float frameRate;
4887         if (!msg->findFloat("frame-rate", &frameRate)) {
4888             int32_t tmp;
4889             if (!msg->findInt32("frame-rate", &tmp)) {
4890                 return INVALID_OPERATION;
4891             }
4892             frameRate = (float)tmp;
4893         }
4894 
4895         hevcType.nKeyFrameInterval =
4896                 setPFramesSpacing(iFrameInterval, frameRate) + 1;
4897     }
4898 
4899 
4900     err = mOMXNode->setParameter(
4901             (OMX_INDEXTYPE)OMX_IndexParamVideoHevc, &hevcType, sizeof(hevcType));
4902     if (err != OK) {
4903         return err;
4904     }
4905 
4906     err = configureImageGrid(msg, outputFormat);
4907 
4908     if (err != OK) {
4909         return err;
4910     }
4911 
4912     return configureBitrate(bitrateMode, bitrate, quality);
4913 }
4914 
setupVPXEncoderParameters(const sp<AMessage> & msg,sp<AMessage> & outputFormat)4915 status_t ACodec::setupVPXEncoderParameters(const sp<AMessage> &msg, sp<AMessage> &outputFormat) {
4916     int32_t bitrate;
4917     float iFrameInterval = 0;
4918     size_t tsLayers = 0;
4919     OMX_VIDEO_ANDROID_VPXTEMPORALLAYERPATTERNTYPE pattern =
4920         OMX_VIDEO_VPXTemporalLayerPatternNone;
4921     static const uint32_t kVp8LayerRateAlloction
4922         [OMX_VIDEO_ANDROID_MAXVP8TEMPORALLAYERS]
4923         [OMX_VIDEO_ANDROID_MAXVP8TEMPORALLAYERS] = {
4924         {100, 100, 100},  // 1 layer
4925         { 60, 100, 100},  // 2 layers {60%, 40%}
4926         { 40,  60, 100},  // 3 layers {40%, 20%, 40%}
4927     };
4928     if (!msg->findInt32("bitrate", &bitrate)) {
4929         return INVALID_OPERATION;
4930     }
4931     msg->findAsFloat("i-frame-interval", &iFrameInterval);
4932 
4933     OMX_VIDEO_CONTROLRATETYPE bitrateMode = getVideoBitrateMode(msg);
4934 
4935     float frameRate;
4936     if (!msg->findFloat("frame-rate", &frameRate)) {
4937         int32_t tmp;
4938         if (!msg->findInt32("frame-rate", &tmp)) {
4939             return INVALID_OPERATION;
4940         }
4941         frameRate = (float)tmp;
4942     }
4943 
4944     AString tsSchema;
4945     OMX_VIDEO_ANDROID_TEMPORALLAYERINGPATTERNTYPE tsType =
4946         OMX_VIDEO_AndroidTemporalLayeringPatternNone;
4947 
4948     if (msg->findString("ts-schema", &tsSchema)) {
4949         unsigned int numLayers = 0;
4950         unsigned int numBLayers = 0;
4951         int tags;
4952         char tmp;
4953         if (sscanf(tsSchema.c_str(), "webrtc.vp8.%u-layer%c", &numLayers, &tmp) == 1
4954                 && numLayers > 0) {
4955             pattern = OMX_VIDEO_VPXTemporalLayerPatternWebRTC;
4956             tsType = OMX_VIDEO_AndroidTemporalLayeringPatternWebRTC;
4957             tsLayers = numLayers;
4958         } else if ((tags = sscanf(tsSchema.c_str(), "android.generic.%u%c%u%c",
4959                         &numLayers, &tmp, &numBLayers, &tmp))
4960                 && (tags == 1 || (tags == 3 && tmp == '+'))
4961                 && numLayers > 0 && numLayers < UINT32_MAX - numBLayers) {
4962             pattern = OMX_VIDEO_VPXTemporalLayerPatternWebRTC;
4963             // VPX does not have a concept of B-frames, so just count all layers
4964             tsType = OMX_VIDEO_AndroidTemporalLayeringPatternAndroid;
4965             tsLayers = numLayers + numBLayers;
4966         } else {
4967             ALOGW("Ignoring unsupported ts-schema [%s]", tsSchema.c_str());
4968         }
4969         tsLayers = min(tsLayers, (size_t)OMX_VIDEO_ANDROID_MAXVP8TEMPORALLAYERS);
4970     }
4971 
4972     OMX_VIDEO_PARAM_ANDROID_VP8ENCODERTYPE vp8type;
4973     InitOMXParams(&vp8type);
4974     vp8type.nPortIndex = kPortIndexOutput;
4975     status_t err = mOMXNode->getParameter(
4976             (OMX_INDEXTYPE)OMX_IndexParamVideoAndroidVp8Encoder,
4977             &vp8type, sizeof(vp8type));
4978 
4979     if (err == OK) {
4980         if (iFrameInterval > 0) {
4981             vp8type.nKeyFrameInterval = setPFramesSpacing(iFrameInterval, frameRate) + 1;
4982         }
4983         vp8type.eTemporalPattern = pattern;
4984         vp8type.nTemporalLayerCount = tsLayers;
4985         if (tsLayers > 0) {
4986             for (size_t i = 0; i < OMX_VIDEO_ANDROID_MAXVP8TEMPORALLAYERS; i++) {
4987                 vp8type.nTemporalLayerBitrateRatio[i] =
4988                     kVp8LayerRateAlloction[tsLayers - 1][i];
4989             }
4990         }
4991         if (bitrateMode == OMX_Video_ControlRateConstant) {
4992             vp8type.nMinQuantizer = 2;
4993             vp8type.nMaxQuantizer = 63;
4994         }
4995 
4996         err = mOMXNode->setParameter(
4997                 (OMX_INDEXTYPE)OMX_IndexParamVideoAndroidVp8Encoder,
4998                 &vp8type, sizeof(vp8type));
4999         if (err != OK) {
5000             ALOGW("Extended VP8 parameters set failed: %d", err);
5001         } else if (tsType == OMX_VIDEO_AndroidTemporalLayeringPatternWebRTC) {
5002             // advertise even single layer WebRTC layering, as it is defined
5003             outputFormat->setString("ts-schema", AStringPrintf("webrtc.vp8.%u-layer", tsLayers));
5004         } else if (tsLayers > 0) {
5005             // tsType == OMX_VIDEO_AndroidTemporalLayeringPatternAndroid
5006             outputFormat->setString("ts-schema", AStringPrintf("android.generic.%u", tsLayers));
5007         }
5008     }
5009 
5010     return configureBitrate(bitrateMode, bitrate);
5011 }
5012 
verifySupportForProfileAndLevel(OMX_U32 portIndex,int32_t profile,int32_t level)5013 status_t ACodec::verifySupportForProfileAndLevel(
5014         OMX_U32 portIndex, int32_t profile, int32_t level) {
5015     OMX_VIDEO_PARAM_PROFILELEVELTYPE params;
5016     InitOMXParams(&params);
5017     params.nPortIndex = portIndex;
5018 
5019     for (OMX_U32 index = 0; index <= kMaxIndicesToCheck; ++index) {
5020         params.nProfileIndex = index;
5021         status_t err = mOMXNode->getParameter(
5022                 OMX_IndexParamVideoProfileLevelQuerySupported,
5023                 &params, sizeof(params));
5024 
5025         if (err != OK) {
5026             return err;
5027         }
5028 
5029         int32_t supportedProfile = static_cast<int32_t>(params.eProfile);
5030         int32_t supportedLevel = static_cast<int32_t>(params.eLevel);
5031 
5032         if (profile == supportedProfile && level <= supportedLevel) {
5033             return OK;
5034         }
5035 
5036         if (index == kMaxIndicesToCheck) {
5037             ALOGW("[%s] stopping checking profiles after %u: %x/%x",
5038                     mComponentName.c_str(), index,
5039                     params.eProfile, params.eLevel);
5040         }
5041     }
5042     return ERROR_UNSUPPORTED;
5043 }
5044 
configureBitrate(OMX_VIDEO_CONTROLRATETYPE bitrateMode,int32_t bitrate,int32_t quality)5045 status_t ACodec::configureBitrate(
5046         OMX_VIDEO_CONTROLRATETYPE bitrateMode, int32_t bitrate, int32_t quality) {
5047     OMX_VIDEO_PARAM_BITRATETYPE bitrateType;
5048     InitOMXParams(&bitrateType);
5049     bitrateType.nPortIndex = kPortIndexOutput;
5050 
5051     status_t err = mOMXNode->getParameter(
5052             OMX_IndexParamVideoBitrate, &bitrateType, sizeof(bitrateType));
5053 
5054     if (err != OK) {
5055         return err;
5056     }
5057 
5058     bitrateType.eControlRate = bitrateMode;
5059 
5060     // write it out explicitly even if it's a union
5061     if (bitrateMode == OMX_Video_ControlRateConstantQuality) {
5062         bitrateType.nQualityFactor = quality;
5063     } else {
5064         bitrateType.nTargetBitrate = bitrate;
5065     }
5066 
5067     return mOMXNode->setParameter(
5068             OMX_IndexParamVideoBitrate, &bitrateType, sizeof(bitrateType));
5069 }
5070 
configureEncoderLatency(const sp<AMessage> & msg)5071 void ACodec::configureEncoderLatency(const sp<AMessage> &msg) {
5072     if (!mIsEncoder || !mIsVideo) {
5073         return;
5074     }
5075 
5076     int32_t latency = 0, bitrateMode;
5077     if (msg->findInt32("latency", &latency) && latency > 0) {
5078         status_t err = setLatency(latency);
5079         if (err != OK) {
5080             ALOGW("[%s] failed setLatency. Failure is fine since this key is optional",
5081                     mComponentName.c_str());
5082             err = OK;
5083         } else {
5084             mLatency = latency;
5085         }
5086     } else if ((!msg->findInt32("bitrate-mode", &bitrateMode) &&
5087             bitrateMode == OMX_Video_ControlRateConstant)) {
5088         // default the latency to be 1 if latency key is not specified or unsupported and bitrateMode
5089         // is CBR.
5090         mLatency = 1;
5091     }
5092 }
5093 
setupErrorCorrectionParameters()5094 status_t ACodec::setupErrorCorrectionParameters() {
5095     OMX_VIDEO_PARAM_ERRORCORRECTIONTYPE errorCorrectionType;
5096     InitOMXParams(&errorCorrectionType);
5097     errorCorrectionType.nPortIndex = kPortIndexOutput;
5098 
5099     status_t err = mOMXNode->getParameter(
5100             OMX_IndexParamVideoErrorCorrection,
5101             &errorCorrectionType, sizeof(errorCorrectionType));
5102 
5103     if (err != OK) {
5104         return OK;  // Optional feature. Ignore this failure
5105     }
5106 
5107     errorCorrectionType.bEnableHEC = OMX_FALSE;
5108     errorCorrectionType.bEnableResync = OMX_TRUE;
5109     errorCorrectionType.nResynchMarkerSpacing = 256;
5110     errorCorrectionType.bEnableDataPartitioning = OMX_FALSE;
5111     errorCorrectionType.bEnableRVLC = OMX_FALSE;
5112 
5113     return mOMXNode->setParameter(
5114             OMX_IndexParamVideoErrorCorrection,
5115             &errorCorrectionType, sizeof(errorCorrectionType));
5116 }
5117 
setVideoFormatOnPort(OMX_U32 portIndex,int32_t width,int32_t height,OMX_VIDEO_CODINGTYPE compressionFormat,float frameRate)5118 status_t ACodec::setVideoFormatOnPort(
5119         OMX_U32 portIndex,
5120         int32_t width, int32_t height, OMX_VIDEO_CODINGTYPE compressionFormat,
5121         float frameRate) {
5122     OMX_PARAM_PORTDEFINITIONTYPE def;
5123     InitOMXParams(&def);
5124     def.nPortIndex = portIndex;
5125 
5126     OMX_VIDEO_PORTDEFINITIONTYPE *video_def = &def.format.video;
5127 
5128     status_t err = mOMXNode->getParameter(
5129             OMX_IndexParamPortDefinition, &def, sizeof(def));
5130     if (err != OK) {
5131         return err;
5132     }
5133 
5134     if (portIndex == kPortIndexInput) {
5135         // XXX Need a (much) better heuristic to compute input buffer sizes.
5136         const size_t X = 64 * 1024;
5137         if (def.nBufferSize < X) {
5138             def.nBufferSize = X;
5139         }
5140     }
5141 
5142     if (def.eDomain != OMX_PortDomainVideo) {
5143         ALOGE("expected video port, got %s(%d)", asString(def.eDomain), def.eDomain);
5144         return FAILED_TRANSACTION;
5145     }
5146 
5147     video_def->nFrameWidth = width;
5148     video_def->nFrameHeight = height;
5149 
5150     if (portIndex == kPortIndexInput) {
5151         video_def->eCompressionFormat = compressionFormat;
5152         video_def->eColorFormat = OMX_COLOR_FormatUnused;
5153         if (frameRate >= 0) {
5154             video_def->xFramerate = (OMX_U32)(frameRate * 65536.0f);
5155         }
5156     }
5157 
5158     err = mOMXNode->setParameter(
5159             OMX_IndexParamPortDefinition, &def, sizeof(def));
5160 
5161     return err;
5162 }
5163 
countBuffersOwnedByComponent(OMX_U32 portIndex) const5164 size_t ACodec::countBuffersOwnedByComponent(OMX_U32 portIndex) const {
5165     size_t n = 0;
5166 
5167     for (size_t i = 0; i < mBuffers[portIndex].size(); ++i) {
5168         const BufferInfo &info = mBuffers[portIndex][i];
5169 
5170         if (info.mStatus == BufferInfo::OWNED_BY_COMPONENT) {
5171             ++n;
5172         }
5173     }
5174 
5175     return n;
5176 }
5177 
countBuffersOwnedByNativeWindow() const5178 size_t ACodec::countBuffersOwnedByNativeWindow() const {
5179     size_t n = 0;
5180 
5181     for (size_t i = 0; i < mBuffers[kPortIndexOutput].size(); ++i) {
5182         const BufferInfo &info = mBuffers[kPortIndexOutput][i];
5183 
5184         if (info.mStatus == BufferInfo::OWNED_BY_NATIVE_WINDOW) {
5185             ++n;
5186         }
5187     }
5188 
5189     return n;
5190 }
5191 
waitUntilAllPossibleNativeWindowBuffersAreReturnedToUs()5192 void ACodec::waitUntilAllPossibleNativeWindowBuffersAreReturnedToUs() {
5193     if (mNativeWindow == NULL) {
5194         return;
5195     }
5196 
5197     while (countBuffersOwnedByNativeWindow() > mNumUndequeuedBuffers
5198             && dequeueBufferFromNativeWindow() != NULL) {
5199         // these buffers will be submitted as regular buffers; account for this
5200         if (storingMetadataInDecodedBuffers() && mMetadataBuffersToSubmit > 0) {
5201             --mMetadataBuffersToSubmit;
5202         }
5203     }
5204 }
5205 
allYourBuffersAreBelongToUs(OMX_U32 portIndex)5206 bool ACodec::allYourBuffersAreBelongToUs(
5207         OMX_U32 portIndex) {
5208     for (size_t i = 0; i < mBuffers[portIndex].size(); ++i) {
5209         BufferInfo *info = &mBuffers[portIndex][i];
5210 
5211         if (info->mStatus != BufferInfo::OWNED_BY_US
5212                 && info->mStatus != BufferInfo::OWNED_BY_NATIVE_WINDOW) {
5213             ALOGV("[%s] Buffer %u on port %u still has status %d",
5214                     mComponentName.c_str(),
5215                     info->mBufferID, portIndex, info->mStatus);
5216             return false;
5217         }
5218     }
5219 
5220     return true;
5221 }
5222 
allYourBuffersAreBelongToUs()5223 bool ACodec::allYourBuffersAreBelongToUs() {
5224     return allYourBuffersAreBelongToUs(kPortIndexInput)
5225         && allYourBuffersAreBelongToUs(kPortIndexOutput);
5226 }
5227 
deferMessage(const sp<AMessage> & msg)5228 void ACodec::deferMessage(const sp<AMessage> &msg) {
5229     mDeferredQueue.push_back(msg);
5230 }
5231 
processDeferredMessages()5232 void ACodec::processDeferredMessages() {
5233     std::list<sp<AMessage>> queue = mDeferredQueue;
5234     mDeferredQueue.clear();
5235 
5236     for(const sp<AMessage> &msg : queue) {
5237         onMessageReceived(msg);
5238     }
5239 }
5240 
getPortFormat(OMX_U32 portIndex,sp<AMessage> & notify)5241 status_t ACodec::getPortFormat(OMX_U32 portIndex, sp<AMessage> &notify) {
5242     const char *niceIndex = portIndex == kPortIndexInput ? "input" : "output";
5243     OMX_PARAM_PORTDEFINITIONTYPE def;
5244     InitOMXParams(&def);
5245     def.nPortIndex = portIndex;
5246 
5247     status_t err = mOMXNode->getParameter(OMX_IndexParamPortDefinition, &def, sizeof(def));
5248     if (err != OK) {
5249         return err;
5250     }
5251 
5252     if (def.eDir != (portIndex == kPortIndexOutput ? OMX_DirOutput : OMX_DirInput)) {
5253         ALOGE("unexpected dir: %s(%d) on %s port", asString(def.eDir), def.eDir, niceIndex);
5254         return BAD_VALUE;
5255     }
5256 
5257     switch (def.eDomain) {
5258         case OMX_PortDomainVideo:
5259         {
5260             OMX_VIDEO_PORTDEFINITIONTYPE *videoDef = &def.format.video;
5261             switch ((int)videoDef->eCompressionFormat) {
5262                 case OMX_VIDEO_CodingUnused:
5263                 {
5264                     CHECK(mIsEncoder ^ (portIndex == kPortIndexOutput));
5265                     notify->setString("mime", MEDIA_MIMETYPE_VIDEO_RAW);
5266 
5267                     notify->setInt32("stride", videoDef->nStride);
5268                     notify->setInt32("slice-height", videoDef->nSliceHeight);
5269                     notify->setInt32("color-format", videoDef->eColorFormat);
5270 
5271                     if (mNativeWindow == NULL) {
5272                         DescribeColorFormat2Params describeParams;
5273                         InitOMXParams(&describeParams);
5274                         describeParams.eColorFormat = videoDef->eColorFormat;
5275                         describeParams.nFrameWidth = videoDef->nFrameWidth;
5276                         describeParams.nFrameHeight = videoDef->nFrameHeight;
5277                         describeParams.nStride = videoDef->nStride;
5278                         describeParams.nSliceHeight = videoDef->nSliceHeight;
5279                         describeParams.bUsingNativeBuffers = OMX_FALSE;
5280 
5281                         if (DescribeColorFormat(mOMXNode, describeParams)) {
5282                             notify->setBuffer(
5283                                     "image-data",
5284                                     ABuffer::CreateAsCopy(
5285                                             &describeParams.sMediaImage,
5286                                             sizeof(describeParams.sMediaImage)));
5287 
5288                             MediaImage2 &img = describeParams.sMediaImage;
5289                             MediaImage2::PlaneInfo *plane = img.mPlane;
5290                             ALOGV("[%s] MediaImage { F(%ux%u) @%u+%d+%d @%u+%d+%d @%u+%d+%d }",
5291                                     mComponentName.c_str(), img.mWidth, img.mHeight,
5292                                     plane[0].mOffset, plane[0].mColInc, plane[0].mRowInc,
5293                                     plane[1].mOffset, plane[1].mColInc, plane[1].mRowInc,
5294                                     plane[2].mOffset, plane[2].mColInc, plane[2].mRowInc);
5295                         }
5296                     }
5297 
5298                     int32_t width = (int32_t)videoDef->nFrameWidth;
5299                     int32_t height = (int32_t)videoDef->nFrameHeight;
5300 
5301                     if (portIndex == kPortIndexOutput) {
5302                         OMX_CONFIG_RECTTYPE rect;
5303                         InitOMXParams(&rect);
5304                         rect.nPortIndex = portIndex;
5305 
5306                         if (mOMXNode->getConfig(
5307                                     (portIndex == kPortIndexOutput ?
5308                                             OMX_IndexConfigCommonOutputCrop :
5309                                             OMX_IndexConfigCommonInputCrop),
5310                                     &rect, sizeof(rect)) != OK) {
5311                             rect.nLeft = 0;
5312                             rect.nTop = 0;
5313                             rect.nWidth = videoDef->nFrameWidth;
5314                             rect.nHeight = videoDef->nFrameHeight;
5315                         }
5316 
5317                         if (rect.nLeft < 0 || rect.nTop < 0 ||
5318                             rect.nWidth == 0 || rect.nHeight == 0 ||
5319                             rect.nLeft + rect.nWidth > videoDef->nFrameWidth ||
5320                             rect.nTop + rect.nHeight > videoDef->nFrameHeight) {
5321                             ALOGE("Wrong cropped rect (%d, %d, %u, %u) vs. frame (%u, %u)",
5322                                     rect.nLeft, rect.nTop,
5323                                     rect.nWidth, rect.nHeight,
5324                                     videoDef->nFrameWidth, videoDef->nFrameHeight);
5325                             return BAD_VALUE;
5326                         }
5327 
5328                         notify->setRect(
5329                                 "crop",
5330                                 rect.nLeft,
5331                                 rect.nTop,
5332                                 rect.nLeft + rect.nWidth - 1,
5333                                 rect.nTop + rect.nHeight - 1);
5334 
5335                         width = rect.nWidth;
5336                         height = rect.nHeight;
5337 
5338                         android_dataspace dataSpace = HAL_DATASPACE_UNKNOWN;
5339                         (void)getColorAspectsAndDataSpaceForVideoDecoder(
5340                                 width, height, mConfigFormat, notify,
5341                                 mUsingNativeWindow ? &dataSpace : NULL);
5342                         if (mUsingNativeWindow) {
5343                             notify->setInt32("android._dataspace", dataSpace);
5344                         }
5345                         (void)getHDRStaticInfoForVideoCodec(kPortIndexOutput, notify);
5346                     } else {
5347                         (void)getInputColorAspectsForVideoEncoder(notify);
5348                         if (mConfigFormat->contains("hdr-static-info")) {
5349                             (void)getHDRStaticInfoForVideoCodec(kPortIndexInput, notify);
5350                         }
5351                         uint32_t latency = 0;
5352                         if (mIsEncoder && !mIsImage &&
5353                                 getLatency(&latency) == OK && latency > 0) {
5354                             notify->setInt32("latency", latency);
5355                         }
5356                     }
5357 
5358                     break;
5359                 }
5360 
5361                 case OMX_VIDEO_CodingVP8:
5362                 case OMX_VIDEO_CodingVP9:
5363                 {
5364                     OMX_VIDEO_PARAM_ANDROID_VP8ENCODERTYPE vp8type;
5365                     InitOMXParams(&vp8type);
5366                     vp8type.nPortIndex = kPortIndexOutput;
5367                     status_t err = mOMXNode->getParameter(
5368                             (OMX_INDEXTYPE)OMX_IndexParamVideoAndroidVp8Encoder,
5369                             &vp8type,
5370                             sizeof(vp8type));
5371 
5372                     if (err == OK) {
5373                         if (vp8type.eTemporalPattern == OMX_VIDEO_VPXTemporalLayerPatternWebRTC
5374                                 && vp8type.nTemporalLayerCount > 0
5375                                 && vp8type.nTemporalLayerCount
5376                                         <= OMX_VIDEO_ANDROID_MAXVP8TEMPORALLAYERS) {
5377                             // advertise as android.generic if we configured for android.generic
5378                             AString origSchema;
5379                             if (notify->findString("ts-schema", &origSchema)
5380                                     && origSchema.startsWith("android.generic")) {
5381                                 notify->setString("ts-schema", AStringPrintf(
5382                                         "android.generic.%u", vp8type.nTemporalLayerCount));
5383                             } else {
5384                                 notify->setString("ts-schema", AStringPrintf(
5385                                         "webrtc.vp8.%u-layer", vp8type.nTemporalLayerCount));
5386                             }
5387                         }
5388                     }
5389                     // Fall through to set up mime.
5390                     FALLTHROUGH_INTENDED;
5391                 }
5392 
5393                 default:
5394                 {
5395                     if (mIsEncoder ^ (portIndex == kPortIndexOutput)) {
5396                         // should be CodingUnused
5397                         ALOGE("Raw port video compression format is %s(%d)",
5398                                 asString(videoDef->eCompressionFormat),
5399                                 videoDef->eCompressionFormat);
5400                         return BAD_VALUE;
5401                     }
5402                     AString mime;
5403                     if (GetMimeTypeForVideoCoding(
5404                         videoDef->eCompressionFormat, &mime) != OK) {
5405                         notify->setString("mime", "application/octet-stream");
5406                     } else {
5407                         notify->setString("mime", mime.c_str());
5408                     }
5409                     uint32_t intraRefreshPeriod = 0;
5410                     if (mIsEncoder && !mIsImage &&
5411                             getIntraRefreshPeriod(&intraRefreshPeriod) == OK
5412                             && intraRefreshPeriod > 0) {
5413                         notify->setInt32("intra-refresh-period", intraRefreshPeriod);
5414                     }
5415                     break;
5416                 }
5417             }
5418             notify->setInt32("width", videoDef->nFrameWidth);
5419             notify->setInt32("height", videoDef->nFrameHeight);
5420             ALOGV("[%s] %s format is %s", mComponentName.c_str(),
5421                     portIndex == kPortIndexInput ? "input" : "output",
5422                     notify->debugString().c_str());
5423 
5424             break;
5425         }
5426 
5427         case OMX_PortDomainAudio:
5428         {
5429             OMX_AUDIO_PORTDEFINITIONTYPE *audioDef = &def.format.audio;
5430 
5431             switch ((int)audioDef->eEncoding) {
5432                 case OMX_AUDIO_CodingPCM:
5433                 {
5434                     OMX_AUDIO_PARAM_PCMMODETYPE params;
5435                     InitOMXParams(&params);
5436                     params.nPortIndex = portIndex;
5437 
5438                     err = mOMXNode->getParameter(
5439                             OMX_IndexParamAudioPcm, &params, sizeof(params));
5440                     if (err != OK) {
5441                         return err;
5442                     }
5443 
5444                     if (params.nChannels <= 0
5445                             || (params.nChannels != 1 && !params.bInterleaved)
5446                             || params.ePCMMode != OMX_AUDIO_PCMModeLinear) {
5447                         ALOGE("unsupported PCM port: %u channels%s, %u-bit",
5448                                 params.nChannels,
5449                                 params.bInterleaved ? " interleaved" : "",
5450                                 params.nBitPerSample);
5451                         return FAILED_TRANSACTION;
5452                     }
5453 
5454                     notify->setString("mime", MEDIA_MIMETYPE_AUDIO_RAW);
5455                     notify->setInt32("channel-count", params.nChannels);
5456                     notify->setInt32("sample-rate", params.nSamplingRate);
5457 
5458                     AudioEncoding encoding = kAudioEncodingPcm16bit;
5459                     if (params.eNumData == OMX_NumericalDataUnsigned
5460                             && params.nBitPerSample == 8u) {
5461                         encoding = kAudioEncodingPcm8bit;
5462                     } else if (params.eNumData == OMX_NumericalDataFloat
5463                             && params.nBitPerSample == 32u) {
5464                         encoding = kAudioEncodingPcmFloat;
5465                     } else if (params.nBitPerSample != 16u
5466                             || params.eNumData != OMX_NumericalDataSigned) {
5467                         ALOGE("unsupported PCM port: %s(%d), %s(%d) mode ",
5468                                 asString(params.eNumData), params.eNumData,
5469                                 asString(params.ePCMMode), params.ePCMMode);
5470                         return FAILED_TRANSACTION;
5471                     }
5472                     notify->setInt32("pcm-encoding", encoding);
5473 
5474                     if (mChannelMaskPresent) {
5475                         notify->setInt32("channel-mask", mChannelMask);
5476                     }
5477 
5478                     if (!mIsEncoder && portIndex == kPortIndexOutput) {
5479                         AString mime;
5480                         if (mConfigFormat->findString("mime", &mime)
5481                                 && !strcasecmp(MEDIA_MIMETYPE_AUDIO_AAC, mime.c_str())) {
5482 
5483                             OMX_AUDIO_PARAM_ANDROID_AACDRCPRESENTATIONTYPE presentation;
5484                             InitOMXParams(&presentation);
5485                             err = mOMXNode->getParameter(
5486                                     (OMX_INDEXTYPE)OMX_IndexParamAudioAndroidAacDrcPresentation,
5487                                     &presentation, sizeof(presentation));
5488                             if (err == OK) {
5489                                 notify->setInt32("aac-encoded-target-level",
5490                                                  presentation.nEncodedTargetLevel);
5491                                 notify->setInt32("aac-drc-cut-level", presentation.nDrcCut);
5492                                 notify->setInt32("aac-drc-boost-level", presentation.nDrcBoost);
5493                                 notify->setInt32("aac-drc-heavy-compression",
5494                                                  presentation.nHeavyCompression);
5495                                 notify->setInt32("aac-target-ref-level",
5496                                                  presentation.nTargetReferenceLevel);
5497                                 notify->setInt32("aac-drc-effect-type",
5498                                                  presentation.nDrcEffectType);
5499                                 notify->setInt32("aac-drc-album-mode", presentation.nDrcAlbumMode);
5500                                 notify->setInt32("aac-drc-output-loudness",
5501                                                  presentation.nDrcOutputLoudness);
5502                             }
5503                         }
5504                     }
5505                     break;
5506                 }
5507 
5508                 case OMX_AUDIO_CodingAAC:
5509                 {
5510                     OMX_AUDIO_PARAM_AACPROFILETYPE params;
5511                     InitOMXParams(&params);
5512                     params.nPortIndex = portIndex;
5513 
5514                     err = mOMXNode->getParameter(
5515                             OMX_IndexParamAudioAac, &params, sizeof(params));
5516                     if (err != OK) {
5517                         return err;
5518                     }
5519 
5520                     notify->setString("mime", MEDIA_MIMETYPE_AUDIO_AAC);
5521                     notify->setInt32("channel-count", params.nChannels);
5522                     notify->setInt32("sample-rate", params.nSampleRate);
5523                     notify->setInt32("bitrate", params.nBitRate);
5524                     notify->setInt32("aac-profile", params.eAACProfile);
5525                     break;
5526                 }
5527 
5528                 case OMX_AUDIO_CodingAMR:
5529                 {
5530                     OMX_AUDIO_PARAM_AMRTYPE params;
5531                     InitOMXParams(&params);
5532                     params.nPortIndex = portIndex;
5533 
5534                     err = mOMXNode->getParameter(
5535                             OMX_IndexParamAudioAmr, &params, sizeof(params));
5536                     if (err != OK) {
5537                         return err;
5538                     }
5539 
5540                     notify->setInt32("channel-count", 1);
5541                     if (params.eAMRBandMode >= OMX_AUDIO_AMRBandModeWB0) {
5542                         notify->setString("mime", MEDIA_MIMETYPE_AUDIO_AMR_WB);
5543                         notify->setInt32("sample-rate", 16000);
5544                     } else {
5545                         notify->setString("mime", MEDIA_MIMETYPE_AUDIO_AMR_NB);
5546                         notify->setInt32("sample-rate", 8000);
5547                     }
5548                     break;
5549                 }
5550 
5551                 case OMX_AUDIO_CodingFLAC:
5552                 {
5553                     OMX_AUDIO_PARAM_FLACTYPE params;
5554                     InitOMXParams(&params);
5555                     params.nPortIndex = portIndex;
5556 
5557                     err = mOMXNode->getParameter(
5558                             OMX_IndexParamAudioFlac, &params, sizeof(params));
5559                     if (err != OK) {
5560                         return err;
5561                     }
5562 
5563                     notify->setString("mime", MEDIA_MIMETYPE_AUDIO_FLAC);
5564                     notify->setInt32("channel-count", params.nChannels);
5565                     notify->setInt32("sample-rate", params.nSampleRate);
5566                     break;
5567                 }
5568 
5569                 case OMX_AUDIO_CodingMP3:
5570                 {
5571                     OMX_AUDIO_PARAM_MP3TYPE params;
5572                     InitOMXParams(&params);
5573                     params.nPortIndex = portIndex;
5574 
5575                     err = mOMXNode->getParameter(
5576                             OMX_IndexParamAudioMp3, &params, sizeof(params));
5577                     if (err != OK) {
5578                         return err;
5579                     }
5580 
5581                     notify->setString("mime", MEDIA_MIMETYPE_AUDIO_MPEG);
5582                     notify->setInt32("channel-count", params.nChannels);
5583                     notify->setInt32("sample-rate", params.nSampleRate);
5584                     break;
5585                 }
5586 
5587                 case OMX_AUDIO_CodingVORBIS:
5588                 {
5589                     OMX_AUDIO_PARAM_VORBISTYPE params;
5590                     InitOMXParams(&params);
5591                     params.nPortIndex = portIndex;
5592 
5593                     err = mOMXNode->getParameter(
5594                             OMX_IndexParamAudioVorbis, &params, sizeof(params));
5595                     if (err != OK) {
5596                         return err;
5597                     }
5598 
5599                     notify->setString("mime", MEDIA_MIMETYPE_AUDIO_VORBIS);
5600                     notify->setInt32("channel-count", params.nChannels);
5601                     notify->setInt32("sample-rate", params.nSampleRate);
5602                     break;
5603                 }
5604 
5605                 case OMX_AUDIO_CodingAndroidAC3:
5606                 {
5607                     OMX_AUDIO_PARAM_ANDROID_AC3TYPE params;
5608                     InitOMXParams(&params);
5609                     params.nPortIndex = portIndex;
5610 
5611                     err = mOMXNode->getParameter(
5612                             (OMX_INDEXTYPE)OMX_IndexParamAudioAndroidAc3,
5613                             &params, sizeof(params));
5614                     if (err != OK) {
5615                         return err;
5616                     }
5617 
5618                     notify->setString("mime", MEDIA_MIMETYPE_AUDIO_AC3);
5619                     notify->setInt32("channel-count", params.nChannels);
5620                     notify->setInt32("sample-rate", params.nSampleRate);
5621                     break;
5622                 }
5623 
5624                 case OMX_AUDIO_CodingAndroidEAC3:
5625                 {
5626                     OMX_AUDIO_PARAM_ANDROID_EAC3TYPE params;
5627                     InitOMXParams(&params);
5628                     params.nPortIndex = portIndex;
5629 
5630                     err = mOMXNode->getParameter(
5631                             (OMX_INDEXTYPE)OMX_IndexParamAudioAndroidEac3,
5632                             &params, sizeof(params));
5633                     if (err != OK) {
5634                         return err;
5635                     }
5636 
5637                     notify->setString("mime", MEDIA_MIMETYPE_AUDIO_EAC3);
5638                     notify->setInt32("channel-count", params.nChannels);
5639                     notify->setInt32("sample-rate", params.nSampleRate);
5640                     break;
5641                 }
5642 
5643                 case OMX_AUDIO_CodingAndroidAC4:
5644                 {
5645                     OMX_AUDIO_PARAM_ANDROID_AC4TYPE params;
5646                     InitOMXParams(&params);
5647                     params.nPortIndex = portIndex;
5648 
5649                     err = mOMXNode->getParameter(
5650                             (OMX_INDEXTYPE)OMX_IndexParamAudioAndroidAc4,
5651                             &params, sizeof(params));
5652                     if (err != OK) {
5653                         return err;
5654                     }
5655 
5656                     notify->setString("mime", MEDIA_MIMETYPE_AUDIO_AC4);
5657                     notify->setInt32("channel-count", params.nChannels);
5658                     notify->setInt32("sample-rate", params.nSampleRate);
5659                     break;
5660                 }
5661 
5662                 case OMX_AUDIO_CodingAndroidOPUS:
5663                 {
5664                     OMX_AUDIO_PARAM_ANDROID_OPUSTYPE params;
5665                     InitOMXParams(&params);
5666                     params.nPortIndex = portIndex;
5667 
5668                     err = mOMXNode->getParameter(
5669                             (OMX_INDEXTYPE)OMX_IndexParamAudioAndroidOpus,
5670                             &params, sizeof(params));
5671                     if (err != OK) {
5672                         return err;
5673                     }
5674 
5675                     notify->setString("mime", MEDIA_MIMETYPE_AUDIO_OPUS);
5676                     notify->setInt32("channel-count", params.nChannels);
5677                     notify->setInt32("sample-rate", params.nSampleRate);
5678                     break;
5679                 }
5680 
5681                 case OMX_AUDIO_CodingG711:
5682                 {
5683                     OMX_AUDIO_PARAM_PCMMODETYPE params;
5684                     InitOMXParams(&params);
5685                     params.nPortIndex = portIndex;
5686 
5687                     err = mOMXNode->getParameter(
5688                             (OMX_INDEXTYPE)OMX_IndexParamAudioPcm, &params, sizeof(params));
5689                     if (err != OK) {
5690                         return err;
5691                     }
5692 
5693                     const char *mime = NULL;
5694                     if (params.ePCMMode == OMX_AUDIO_PCMModeMULaw) {
5695                         mime = MEDIA_MIMETYPE_AUDIO_G711_MLAW;
5696                     } else if (params.ePCMMode == OMX_AUDIO_PCMModeALaw) {
5697                         mime = MEDIA_MIMETYPE_AUDIO_G711_ALAW;
5698                     } else { // params.ePCMMode == OMX_AUDIO_PCMModeLinear
5699                         mime = MEDIA_MIMETYPE_AUDIO_RAW;
5700                     }
5701                     notify->setString("mime", mime);
5702                     notify->setInt32("channel-count", params.nChannels);
5703                     notify->setInt32("sample-rate", params.nSamplingRate);
5704                     notify->setInt32("pcm-encoding", kAudioEncodingPcm16bit);
5705                     break;
5706                 }
5707 
5708                 case OMX_AUDIO_CodingGSMFR:
5709                 {
5710                     OMX_AUDIO_PARAM_PCMMODETYPE params;
5711                     InitOMXParams(&params);
5712                     params.nPortIndex = portIndex;
5713 
5714                     err = mOMXNode->getParameter(
5715                                 OMX_IndexParamAudioPcm, &params, sizeof(params));
5716                     if (err != OK) {
5717                         return err;
5718                     }
5719 
5720                     notify->setString("mime", MEDIA_MIMETYPE_AUDIO_MSGSM);
5721                     notify->setInt32("channel-count", params.nChannels);
5722                     notify->setInt32("sample-rate", params.nSamplingRate);
5723                     break;
5724                 }
5725 
5726                 default:
5727                     ALOGE("Unsupported audio coding: %s(%d)\n",
5728                             asString(audioDef->eEncoding), audioDef->eEncoding);
5729                     return BAD_TYPE;
5730             }
5731             break;
5732         }
5733 
5734         default:
5735             ALOGE("Unsupported domain: %s(%d)", asString(def.eDomain), def.eDomain);
5736             return BAD_TYPE;
5737     }
5738 
5739     return getVendorParameters(portIndex, notify);
5740 }
5741 
getHDR10PlusInfo(size_t paramSizeUsed)5742 DescribeHDR10PlusInfoParams* ACodec::getHDR10PlusInfo(size_t paramSizeUsed) {
5743     if (mDescribeHDR10PlusInfoIndex == 0) {
5744         ALOGE("getHDR10PlusInfo: does not support DescribeHDR10PlusInfoParams");
5745         return nullptr;
5746     }
5747 
5748     size_t newSize = sizeof(DescribeHDR10PlusInfoParams) - 1 +
5749             ((paramSizeUsed > 0) ? paramSizeUsed : 512);
5750     if (mHdr10PlusScratchBuffer == nullptr
5751             || newSize > mHdr10PlusScratchBuffer->size()) {
5752         mHdr10PlusScratchBuffer = new ABuffer(newSize);
5753     }
5754     DescribeHDR10PlusInfoParams *config =
5755             (DescribeHDR10PlusInfoParams *)mHdr10PlusScratchBuffer->data();
5756     InitOMXParams(config);
5757     config->nSize = mHdr10PlusScratchBuffer->size();
5758     config->nPortIndex = 1;
5759     size_t paramSize = config->nSize - sizeof(DescribeHDR10PlusInfoParams) + 1;
5760     config->nParamSize = paramSize;
5761     config->nParamSizeUsed = 0;
5762     status_t err = mOMXNode->getConfig(
5763             (OMX_INDEXTYPE)mDescribeHDR10PlusInfoIndex,
5764             config, config->nSize);
5765     if (err != OK) {
5766         ALOGE("failed to get DescribeHDR10PlusInfoParams (err %d)", err);
5767         return nullptr;
5768     }
5769     if (config->nParamSize != paramSize) {
5770         ALOGE("DescribeHDR10PlusInfoParams alters nParamSize: %u vs %zu",
5771                 config->nParamSize, paramSize);
5772         return nullptr;
5773     }
5774     if (paramSizeUsed > 0 && config->nParamSizeUsed != paramSizeUsed) {
5775         ALOGE("DescribeHDR10PlusInfoParams returns wrong nParamSizeUsed: %u vs %zu",
5776                 config->nParamSizeUsed, paramSizeUsed);
5777         return nullptr;
5778     }
5779     return config;
5780 }
5781 
onConfigUpdate(OMX_INDEXTYPE configIndex)5782 void ACodec::onConfigUpdate(OMX_INDEXTYPE configIndex) {
5783     if (mDescribeHDR10PlusInfoIndex == 0
5784             || configIndex != mDescribeHDR10PlusInfoIndex) {
5785         // mDescribeHDR10PlusInfoIndex is the only update we recognize now
5786         return;
5787     }
5788 
5789     DescribeHDR10PlusInfoParams *config = getHDR10PlusInfo();
5790     if (config == nullptr) {
5791         return;
5792     }
5793     if (config->nParamSizeUsed > config->nParamSize) {
5794         // try again with the size specified
5795         config = getHDR10PlusInfo(config->nParamSizeUsed);
5796         if (config == nullptr) {
5797             return;
5798         }
5799     }
5800 
5801     mOutputFormat = mOutputFormat->dup(); // trigger an output format changed event
5802     mOutputFormat->setBuffer("hdr10-plus-info",
5803             ABuffer::CreateAsCopy(config->nValue, config->nParamSizeUsed));
5804 }
5805 
onDataSpaceChanged(android_dataspace dataSpace,const ColorAspects & aspects)5806 void ACodec::onDataSpaceChanged(android_dataspace dataSpace, const ColorAspects &aspects) {
5807     // aspects are normally communicated in ColorAspects
5808     int32_t range, standard, transfer;
5809     convertCodecColorAspectsToPlatformAspects(aspects, &range, &standard, &transfer);
5810 
5811     int32_t dsRange, dsStandard, dsTransfer;
5812     getColorConfigFromDataSpace(dataSpace, &dsRange, &dsStandard, &dsTransfer);
5813 
5814     // if some aspects are unspecified, use dataspace fields
5815     if (range == 0) {
5816         range = dsRange;
5817     }
5818     if (standard == 0) {
5819         standard = dsStandard;
5820     }
5821     if (transfer == 0) {
5822         transfer = dsTransfer;
5823     }
5824 
5825     mOutputFormat = mOutputFormat->dup(); // trigger an output format changed event
5826     if (range != 0) {
5827         mOutputFormat->setInt32("color-range", range);
5828     }
5829     if (standard != 0) {
5830         mOutputFormat->setInt32("color-standard", standard);
5831     }
5832     if (transfer != 0) {
5833         mOutputFormat->setInt32("color-transfer", transfer);
5834     }
5835 
5836     ALOGD("dataspace changed to %#x (R:%d(%s), P:%d(%s), M:%d(%s), T:%d(%s)) "
5837           "(R:%d(%s), S:%d(%s), T:%d(%s))",
5838             dataSpace,
5839             aspects.mRange, asString(aspects.mRange),
5840             aspects.mPrimaries, asString(aspects.mPrimaries),
5841             aspects.mMatrixCoeffs, asString(aspects.mMatrixCoeffs),
5842             aspects.mTransfer, asString(aspects.mTransfer),
5843             range, asString((ColorRange)range),
5844             standard, asString((ColorStandard)standard),
5845             transfer, asString((ColorTransfer)transfer));
5846 }
5847 
onOutputFormatChanged(sp<const AMessage> expectedFormat)5848 void ACodec::onOutputFormatChanged(sp<const AMessage> expectedFormat) {
5849     // store new output format, at the same time mark that this is no longer the first frame
5850     mOutputFormat = mBaseOutputFormat->dup();
5851 
5852     if (getPortFormat(kPortIndexOutput, mOutputFormat) != OK) {
5853         ALOGE("[%s] Failed to get port format to send format change", mComponentName.c_str());
5854         return;
5855     }
5856 
5857     if (expectedFormat != NULL) {
5858         sp<const AMessage> changes = expectedFormat->changesFrom(mOutputFormat);
5859         sp<const AMessage> to = mOutputFormat->changesFrom(expectedFormat);
5860         if (changes->countEntries() != 0 || to->countEntries() != 0) {
5861             ALOGW("[%s] BAD CODEC: Output format changed unexpectedly from (diff) %s to (diff) %s",
5862                     mComponentName.c_str(),
5863                     changes->debugString(4).c_str(), to->debugString(4).c_str());
5864         }
5865     }
5866 
5867     if (!mIsVideo && !mIsEncoder) {
5868         AudioEncoding pcmEncoding = kAudioEncodingPcm16bit;
5869         (void)mConfigFormat->findInt32("pcm-encoding", (int32_t*)&pcmEncoding);
5870         AudioEncoding codecPcmEncoding = kAudioEncodingPcm16bit;
5871         (void)mOutputFormat->findInt32("pcm-encoding", (int32_t*)&codecPcmEncoding);
5872 
5873         mConverter[kPortIndexOutput] = AudioConverter::Create(codecPcmEncoding, pcmEncoding);
5874         if (mConverter[kPortIndexOutput] != NULL) {
5875             mOutputFormat->setInt32("pcm-encoding", pcmEncoding);
5876         }
5877     }
5878 
5879     if (mTunneled) {
5880         sendFormatChange();
5881     }
5882 }
5883 
sendFormatChange()5884 void ACodec::sendFormatChange() {
5885     AString mime;
5886     CHECK(mOutputFormat->findString("mime", &mime));
5887 
5888     if (mime == MEDIA_MIMETYPE_AUDIO_RAW && (mEncoderDelay || mEncoderPadding)) {
5889         int32_t channelCount, sampleRate;
5890         CHECK(mOutputFormat->findInt32("channel-count", &channelCount));
5891         CHECK(mOutputFormat->findInt32("sample-rate", &sampleRate));
5892         if (mSampleRate != 0 && sampleRate != 0) {
5893             // avoiding 32-bit overflows in intermediate values
5894             mEncoderDelay = (int32_t)((((int64_t)mEncoderDelay) * sampleRate) / mSampleRate);
5895             mEncoderPadding = (int32_t)((((int64_t)mEncoderPadding) * sampleRate) / mSampleRate);
5896             mSampleRate = sampleRate;
5897         }
5898         if (mSkipCutBuffer != NULL) {
5899             size_t prevbufsize = mSkipCutBuffer->size();
5900             if (prevbufsize != 0) {
5901                 ALOGW("Replacing SkipCutBuffer holding %zu bytes", prevbufsize);
5902             }
5903         }
5904         mSkipCutBuffer = new SkipCutBuffer(mEncoderDelay, mEncoderPadding, channelCount);
5905     }
5906 
5907     // mLastOutputFormat is not used when tunneled; doing this just to stay consistent
5908     mLastOutputFormat = mOutputFormat;
5909 }
5910 
signalError(OMX_ERRORTYPE error,status_t internalError)5911 void ACodec::signalError(OMX_ERRORTYPE error, status_t internalError) {
5912     ALOGE("signalError(omxError %#x, internalError %d)", error, internalError);
5913 
5914     if (internalError == UNKNOWN_ERROR) { // find better error code
5915         const status_t omxStatus = statusFromOMXError(error);
5916         if (omxStatus != 0) {
5917             internalError = omxStatus;
5918         } else {
5919             ALOGW("Invalid OMX error %#x", error);
5920         }
5921     }
5922 
5923     mFatalError = true;
5924     mCallback->onError(internalError, ACTION_CODE_FATAL);
5925 }
5926 
requestIDRFrame()5927 status_t ACodec::requestIDRFrame() {
5928     if (!mIsEncoder) {
5929         return ERROR_UNSUPPORTED;
5930     }
5931 
5932     OMX_CONFIG_INTRAREFRESHVOPTYPE params;
5933     InitOMXParams(&params);
5934 
5935     params.nPortIndex = kPortIndexOutput;
5936     params.IntraRefreshVOP = OMX_TRUE;
5937 
5938     return mOMXNode->setConfig(
5939             OMX_IndexConfigVideoIntraVOPRefresh,
5940             &params,
5941             sizeof(params));
5942 }
5943 
5944 ////////////////////////////////////////////////////////////////////////////////
5945 
BaseState(ACodec * codec,const sp<AState> & parentState)5946 ACodec::BaseState::BaseState(ACodec *codec, const sp<AState> &parentState)
5947     : AState(parentState),
5948       mCodec(codec),
5949       mPendingExtraOutputMetadataBufferRequest(false) {
5950 }
5951 
getPortMode(OMX_U32)5952 ACodec::BaseState::PortMode ACodec::BaseState::getPortMode(
5953         OMX_U32 /* portIndex */) {
5954     return KEEP_BUFFERS;
5955 }
5956 
stateExited()5957 void ACodec::BaseState::stateExited() {
5958     ++mCodec->mStateGeneration;
5959 }
5960 
onMessageReceived(const sp<AMessage> & msg)5961 bool ACodec::BaseState::onMessageReceived(const sp<AMessage> &msg) {
5962     switch (msg->what()) {
5963         case kWhatInputBufferFilled:
5964         {
5965             onInputBufferFilled(msg);
5966             break;
5967         }
5968 
5969         case kWhatOutputBufferDrained:
5970         {
5971             onOutputBufferDrained(msg);
5972             break;
5973         }
5974 
5975         case ACodec::kWhatOMXMessageList:
5976         {
5977             return checkOMXMessage(msg) ? onOMXMessageList(msg) : true;
5978         }
5979 
5980         case ACodec::kWhatOMXMessageItem:
5981         {
5982             // no need to check as we already did it for kWhatOMXMessageList
5983             return onOMXMessage(msg);
5984         }
5985 
5986         case ACodec::kWhatOMXMessage:
5987         {
5988             return checkOMXMessage(msg) ? onOMXMessage(msg) : true;
5989         }
5990 
5991         case ACodec::kWhatSetSurface:
5992         {
5993             sp<AReplyToken> replyID;
5994             CHECK(msg->senderAwaitsResponse(&replyID));
5995 
5996             sp<RefBase> obj;
5997             CHECK(msg->findObject("surface", &obj));
5998 
5999             status_t err = mCodec->handleSetSurface(static_cast<Surface *>(obj.get()));
6000 
6001             sp<AMessage> response = new AMessage;
6002             response->setInt32("err", err);
6003             response->postReply(replyID);
6004             break;
6005         }
6006 
6007         case ACodec::kWhatCreateInputSurface:
6008         case ACodec::kWhatSetInputSurface:
6009         case ACodec::kWhatSignalEndOfInputStream:
6010         {
6011             // This may result in an app illegal state exception.
6012             ALOGE("Message 0x%x was not handled", msg->what());
6013             mCodec->signalError(OMX_ErrorUndefined, INVALID_OPERATION);
6014             return true;
6015         }
6016 
6017         case ACodec::kWhatOMXDied:
6018         {
6019             // This will result in kFlagSawMediaServerDie handling in MediaCodec.
6020             ALOGE("OMX/mediaserver died, signalling error!");
6021             mCodec->mGraphicBufferSource.clear();
6022             mCodec->signalError(OMX_ErrorResourcesLost, DEAD_OBJECT);
6023             break;
6024         }
6025 
6026         case ACodec::kWhatReleaseCodecInstance:
6027         {
6028             ALOGI("[%s] forcing the release of codec",
6029                     mCodec->mComponentName.c_str());
6030             status_t err = mCodec->mOMXNode->freeNode();
6031             ALOGE_IF("[%s] failed to release codec instance: err=%d",
6032                        mCodec->mComponentName.c_str(), err);
6033             mCodec->mCallback->onReleaseCompleted();
6034 
6035             mCodec->changeState(mCodec->mUninitializedState);
6036             break;
6037         }
6038 
6039         case ACodec::kWhatForceStateTransition:
6040         {
6041             ALOGV("Already transitioned --- ignore");
6042             break;
6043         }
6044 
6045         case kWhatCheckIfStuck: {
6046             ALOGV("No-op by default");
6047             break;
6048         }
6049 
6050         case kWhatSubmitExtraOutputMetadataBuffer: {
6051             mPendingExtraOutputMetadataBufferRequest = false;
6052             if (getPortMode(kPortIndexOutput) == RESUBMIT_BUFFERS && mCodec->mIsLowLatency) {
6053                 // Decoders often need more than one output buffer to be
6054                 // submitted before processing a single input buffer.
6055                 // For low latency codecs, we don't want to wait for more input
6056                 // to be queued to get those output buffers submitted.
6057                 if (mCodec->submitOutputMetadataBuffer() == OK
6058                         && mCodec->mMetadataBuffersToSubmit > 0) {
6059                     maybePostExtraOutputMetadataBufferRequest();
6060                 }
6061             }
6062             break;
6063         }
6064 
6065         default:
6066             return false;
6067     }
6068 
6069     return true;
6070 }
6071 
checkOMXMessage(const sp<AMessage> & msg)6072 bool ACodec::BaseState::checkOMXMessage(const sp<AMessage> &msg) {
6073     // there is a possibility that this is an outstanding message for a
6074     // codec that we have already destroyed
6075     if (mCodec->mOMXNode == NULL) {
6076         ALOGI("ignoring message as already freed component: %s",
6077                 msg->debugString().c_str());
6078         return false;
6079     }
6080 
6081     int32_t generation;
6082     CHECK(msg->findInt32("generation", (int32_t*)&generation));
6083     if (generation != mCodec->mNodeGeneration) {
6084         ALOGW("Unexpected message for component: %s, gen %u, cur %u",
6085                 msg->debugString().c_str(), generation, mCodec->mNodeGeneration);
6086         return false;
6087     }
6088     return true;
6089 }
6090 
onOMXMessageList(const sp<AMessage> & msg)6091 bool ACodec::BaseState::onOMXMessageList(const sp<AMessage> &msg) {
6092     sp<RefBase> obj;
6093     CHECK(msg->findObject("messages", &obj));
6094     sp<MessageList> msgList = static_cast<MessageList *>(obj.get());
6095     for (std::list<sp<AMessage>>::const_iterator it = msgList->getList().cbegin();
6096           it != msgList->getList().cend(); ++it) {
6097         (*it)->setWhat(ACodec::kWhatOMXMessageItem);
6098         mCodec->handleMessage(*it);
6099     }
6100     return true;
6101 }
6102 
onOMXMessage(const sp<AMessage> & msg)6103 bool ACodec::BaseState::onOMXMessage(const sp<AMessage> &msg) {
6104     int32_t type;
6105     CHECK(msg->findInt32("type", &type));
6106 
6107     switch (type) {
6108         case omx_message::EVENT:
6109         {
6110             int32_t event, data1, data2;
6111             CHECK(msg->findInt32("event", &event));
6112             CHECK(msg->findInt32("data1", &data1));
6113             CHECK(msg->findInt32("data2", &data2));
6114 
6115             if (event == OMX_EventCmdComplete
6116                     && data1 == OMX_CommandFlush
6117                     && data2 == (int32_t)OMX_ALL) {
6118                 // Use of this notification is not consistent across
6119                 // implementations. We'll drop this notification and rely
6120                 // on flush-complete notifications on the individual port
6121                 // indices instead.
6122 
6123                 return true;
6124             }
6125 
6126             return onOMXEvent(
6127                     static_cast<OMX_EVENTTYPE>(event),
6128                     static_cast<OMX_U32>(data1),
6129                     static_cast<OMX_U32>(data2));
6130         }
6131 
6132         case omx_message::EMPTY_BUFFER_DONE:
6133         {
6134             IOMX::buffer_id bufferID;
6135             int32_t fenceFd;
6136 
6137             CHECK(msg->findInt32("buffer", (int32_t*)&bufferID));
6138             CHECK(msg->findInt32("fence_fd", &fenceFd));
6139 
6140             return onOMXEmptyBufferDone(bufferID, fenceFd);
6141         }
6142 
6143         case omx_message::FILL_BUFFER_DONE:
6144         {
6145             IOMX::buffer_id bufferID;
6146             CHECK(msg->findInt32("buffer", (int32_t*)&bufferID));
6147 
6148             int32_t rangeOffset, rangeLength, flags, fenceFd;
6149             int64_t timeUs;
6150 
6151             CHECK(msg->findInt32("range_offset", &rangeOffset));
6152             CHECK(msg->findInt32("range_length", &rangeLength));
6153             CHECK(msg->findInt32("flags", &flags));
6154             CHECK(msg->findInt64("timestamp", &timeUs));
6155             CHECK(msg->findInt32("fence_fd", &fenceFd));
6156 
6157             return onOMXFillBufferDone(
6158                     bufferID,
6159                     (size_t)rangeOffset, (size_t)rangeLength,
6160                     (OMX_U32)flags,
6161                     timeUs,
6162                     fenceFd);
6163         }
6164 
6165         case omx_message::FRAME_RENDERED:
6166         {
6167             int64_t mediaTimeUs, systemNano;
6168 
6169             CHECK(msg->findInt64("media_time_us", &mediaTimeUs));
6170             CHECK(msg->findInt64("system_nano", &systemNano));
6171 
6172             return onOMXFrameRendered(
6173                     mediaTimeUs, systemNano);
6174         }
6175 
6176         default:
6177             ALOGE("Unexpected message type: %d", type);
6178             return false;
6179     }
6180 }
6181 
onOMXFrameRendered(int64_t mediaTimeUs __unused,nsecs_t systemNano __unused)6182 bool ACodec::BaseState::onOMXFrameRendered(
6183         int64_t mediaTimeUs __unused, nsecs_t systemNano __unused) {
6184     // ignore outside of Executing and PortSettingsChanged states
6185     return true;
6186 }
6187 
onOMXEvent(OMX_EVENTTYPE event,OMX_U32 data1,OMX_U32 data2)6188 bool ACodec::BaseState::onOMXEvent(
6189         OMX_EVENTTYPE event, OMX_U32 data1, OMX_U32 data2) {
6190     if (event == OMX_EventDataSpaceChanged) {
6191         ColorAspects aspects = ColorUtils::unpackToColorAspects(data2);
6192 
6193         mCodec->onDataSpaceChanged((android_dataspace)data1, aspects);
6194         return true;
6195     }
6196 
6197     if (event != OMX_EventError) {
6198         ALOGV("[%s] EVENT(%d, 0x%08x, 0x%08x)",
6199              mCodec->mComponentName.c_str(), event, data1, data2);
6200 
6201         return false;
6202     }
6203 
6204     if (mCodec->mIsStreamCorruptFree && data1 == (OMX_U32)OMX_ErrorStreamCorrupt) {
6205         ALOGV("[%s] handle OMX_ErrorStreamCorrupt as a normal operation",
6206                 mCodec->mComponentName.c_str());
6207         return true;
6208     }
6209 
6210     ALOGE("[%s] ERROR(0x%08x)", mCodec->mComponentName.c_str(), data1);
6211 
6212     // verify OMX component sends back an error we expect.
6213     OMX_ERRORTYPE omxError = (OMX_ERRORTYPE)data1;
6214     if (!isOMXError(omxError)) {
6215         ALOGW("Invalid OMX error %#x", omxError);
6216         omxError = OMX_ErrorUndefined;
6217     }
6218     mCodec->signalError(omxError);
6219 
6220     return true;
6221 }
6222 
onOMXEmptyBufferDone(IOMX::buffer_id bufferID,int fenceFd)6223 bool ACodec::BaseState::onOMXEmptyBufferDone(IOMX::buffer_id bufferID, int fenceFd) {
6224     ALOGV("[%s] onOMXEmptyBufferDone %u",
6225          mCodec->mComponentName.c_str(), bufferID);
6226 
6227     BufferInfo *info = mCodec->findBufferByID(kPortIndexInput, bufferID);
6228     BufferInfo::Status status = BufferInfo::getSafeStatus(info);
6229     if (status != BufferInfo::OWNED_BY_COMPONENT) {
6230         ALOGE("Wrong ownership in EBD: %s(%d) buffer #%u", _asString(status), status, bufferID);
6231         mCodec->dumpBuffers(kPortIndexInput);
6232         if (fenceFd >= 0) {
6233             ::close(fenceFd);
6234         }
6235         return false;
6236     }
6237     info->mStatus = BufferInfo::OWNED_BY_US;
6238 
6239     // input buffers cannot take fences, so wait for any fence now
6240     (void)mCodec->waitForFence(fenceFd, "onOMXEmptyBufferDone");
6241     fenceFd = -1;
6242 
6243     // still save fence for completeness
6244     info->setWriteFence(fenceFd, "onOMXEmptyBufferDone");
6245 
6246     // We're in "store-metadata-in-buffers" mode, the underlying
6247     // OMX component had access to data that's implicitly refcounted
6248     // by this "MediaBuffer" object. Now that the OMX component has
6249     // told us that it's done with the input buffer, we can decrement
6250     // the mediaBuffer's reference count.
6251     info->mData->meta()->setObject("mediaBufferHolder", sp<MediaBufferHolder>(nullptr));
6252 
6253     PortMode mode = getPortMode(kPortIndexInput);
6254 
6255     switch (mode) {
6256         case KEEP_BUFFERS:
6257             break;
6258 
6259         case RESUBMIT_BUFFERS:
6260             postFillThisBuffer(info);
6261             break;
6262 
6263         case FREE_BUFFERS:
6264         default:
6265             ALOGE("SHOULD NOT REACH HERE: cannot free empty output buffers");
6266             return false;
6267     }
6268 
6269     return true;
6270 }
6271 
postFillThisBuffer(BufferInfo * info)6272 void ACodec::BaseState::postFillThisBuffer(BufferInfo *info) {
6273     if (mCodec->mPortEOS[kPortIndexInput]) {
6274         return;
6275     }
6276 
6277     CHECK_EQ((int)info->mStatus, (int)BufferInfo::OWNED_BY_US);
6278 
6279     info->mData->setFormat(mCodec->mInputFormat);
6280     mCodec->mBufferChannel->fillThisBuffer(info->mBufferID);
6281     info->mData.clear();
6282     info->mStatus = BufferInfo::OWNED_BY_UPSTREAM;
6283 }
6284 
onInputBufferFilled(const sp<AMessage> & msg)6285 void ACodec::BaseState::onInputBufferFilled(const sp<AMessage> &msg) {
6286     IOMX::buffer_id bufferID;
6287     CHECK(msg->findInt32("buffer-id", (int32_t*)&bufferID));
6288     sp<MediaCodecBuffer> buffer;
6289     int32_t err = OK;
6290     bool eos = false;
6291     PortMode mode = getPortMode(kPortIndexInput);
6292     int32_t discarded = 0;
6293     if (msg->findInt32("discarded", &discarded) && discarded) {
6294         // these are unfilled buffers returned by client
6295         // buffers are returned on MediaCodec.flush
6296         mode = KEEP_BUFFERS;
6297     }
6298     sp<RefBase> obj;
6299     CHECK(msg->findObject("buffer", &obj));
6300     buffer = static_cast<MediaCodecBuffer *>(obj.get());
6301 
6302     int32_t tmp;
6303     if (buffer != NULL && buffer->meta()->findInt32("eos", &tmp) && tmp) {
6304         eos = true;
6305         err = ERROR_END_OF_STREAM;
6306     }
6307 
6308     BufferInfo *info = mCodec->findBufferByID(kPortIndexInput, bufferID);
6309     BufferInfo::Status status = BufferInfo::getSafeStatus(info);
6310     if (status != BufferInfo::OWNED_BY_UPSTREAM) {
6311         ALOGE("Wrong ownership in IBF: %s(%d) buffer #%u", _asString(status), status, bufferID);
6312         mCodec->dumpBuffers(kPortIndexInput);
6313         mCodec->signalError(OMX_ErrorUndefined, FAILED_TRANSACTION);
6314         return;
6315     }
6316 
6317     int32_t cvo;
6318     if (mCodec->mNativeWindow != NULL && buffer != NULL &&
6319             buffer->meta()->findInt32("cvo", &cvo)) {
6320         ALOGV("cvo(%d) found in buffer #%u", cvo, bufferID);
6321         setNativeWindowRotation(mCodec->mNativeWindow.get(), cvo);
6322     }
6323 
6324     info->mStatus = BufferInfo::OWNED_BY_US;
6325     info->mData = buffer;
6326 
6327     switch (mode) {
6328         case KEEP_BUFFERS:
6329         {
6330             if (eos) {
6331                 if (!mCodec->mPortEOS[kPortIndexInput]) {
6332                     mCodec->mPortEOS[kPortIndexInput] = true;
6333                     mCodec->mInputEOSResult = err;
6334                 }
6335             }
6336             break;
6337         }
6338 
6339         case RESUBMIT_BUFFERS:
6340         {
6341             if (buffer != NULL && !mCodec->mPortEOS[kPortIndexInput]) {
6342                 // Do not send empty input buffer w/o EOS to the component.
6343                 if (buffer->size() == 0 && !eos) {
6344                     postFillThisBuffer(info);
6345                     break;
6346                 }
6347 
6348                 int64_t timeUs;
6349                 CHECK(buffer->meta()->findInt64("timeUs", &timeUs));
6350 
6351                 OMX_U32 flags = OMX_BUFFERFLAG_ENDOFFRAME;
6352 
6353                 int32_t isCSD = 0;
6354                 if (buffer->meta()->findInt32("csd", &isCSD) && isCSD != 0) {
6355                     if (mCodec->mIsLegacyVP9Decoder) {
6356                         ALOGV("[%s] is legacy VP9 decoder. Ignore %u codec specific data",
6357                             mCodec->mComponentName.c_str(), bufferID);
6358                         postFillThisBuffer(info);
6359                         break;
6360                     }
6361                     flags |= OMX_BUFFERFLAG_CODECCONFIG;
6362                 }
6363 
6364                 if (eos) {
6365                     flags |= OMX_BUFFERFLAG_EOS;
6366                 }
6367 
6368                 int32_t isDecodeOnly = 0;
6369                 if (buffer->meta()->findInt32("decode-only", &isDecodeOnly) && isDecodeOnly != 0) {
6370                     flags |= OMX_BUFFERFLAG_DECODEONLY;
6371                     mCodec->mDecodeOnlyTimesUs.emplace(timeUs);
6372                 }
6373                 size_t size = buffer->size();
6374                 size_t offset = buffer->offset();
6375                 if (buffer->base() != info->mCodecData->base()) {
6376                     ALOGV("[%s] Needs to copy input data for buffer %u. (%p != %p)",
6377                          mCodec->mComponentName.c_str(),
6378                          bufferID,
6379                          buffer->base(), info->mCodecData->base());
6380 
6381                     sp<DataConverter> converter = mCodec->mConverter[kPortIndexInput];
6382                     if (converter == NULL || isCSD) {
6383                         converter = getCopyConverter();
6384                     }
6385                     status_t err = converter->convert(buffer, info->mCodecData);
6386                     if (err != OK) {
6387                         mCodec->signalError(OMX_ErrorUndefined, err);
6388                         return;
6389                     }
6390                     size = info->mCodecData->size();
6391                 } else {
6392                     info->mCodecData->setRange(offset, size);
6393                 }
6394 
6395                 if (flags & OMX_BUFFERFLAG_CODECCONFIG) {
6396                     ALOGV("[%s] calling emptyBuffer %u w/ codec specific data",
6397                          mCodec->mComponentName.c_str(), bufferID);
6398                 } else if (flags & OMX_BUFFERFLAG_EOS) {
6399                     ALOGV("[%s] calling emptyBuffer %u w/ EOS",
6400                          mCodec->mComponentName.c_str(), bufferID);
6401                 } else {
6402                     if (flags & OMX_BUFFERFLAG_DECODEONLY) {
6403                         ALOGV("[%s] calling emptyBuffer %u w/ decode only flag",
6404                             mCodec->mComponentName.c_str(), bufferID);
6405                     }
6406 #if TRACK_BUFFER_TIMING
6407                     ALOGI("[%s] calling emptyBuffer %u w/ time %lld us",
6408                          mCodec->mComponentName.c_str(), bufferID, (long long)timeUs);
6409 #else
6410                     ALOGV("[%s] calling emptyBuffer %u w/ time %lld us",
6411                          mCodec->mComponentName.c_str(), bufferID, (long long)timeUs);
6412 #endif
6413                 }
6414 
6415 #if TRACK_BUFFER_TIMING
6416                 ACodec::BufferStats stats;
6417                 stats.mEmptyBufferTimeUs = ALooper::GetNowUs();
6418                 stats.mFillBufferDoneTimeUs = -1ll;
6419                 mCodec->mBufferStats.add(timeUs, stats);
6420 #endif
6421 
6422                 if (mCodec->storingMetadataInDecodedBuffers()) {
6423                     // try to submit an output buffer for each input buffer
6424                     PortMode outputMode = getPortMode(kPortIndexOutput);
6425 
6426                     ALOGV("MetadataBuffersToSubmit=%u portMode=%s",
6427                             mCodec->mMetadataBuffersToSubmit,
6428                             (outputMode == FREE_BUFFERS ? "FREE" :
6429                              outputMode == KEEP_BUFFERS ? "KEEP" : "RESUBMIT"));
6430                     if (outputMode == RESUBMIT_BUFFERS) {
6431                         status_t err = mCodec->submitOutputMetadataBuffer();
6432                         if (mCodec->mIsLowLatency
6433                                 && err == OK
6434                                 && mCodec->mMetadataBuffersToSubmit > 0) {
6435                             maybePostExtraOutputMetadataBufferRequest();
6436                         }
6437                     }
6438                 }
6439                 info->checkReadFence("onInputBufferFilled");
6440 
6441                 status_t err2 = OK;
6442                 switch (mCodec->mPortMode[kPortIndexInput]) {
6443                 case IOMX::kPortModePresetByteBuffer:
6444                 case IOMX::kPortModePresetANWBuffer:
6445                 case IOMX::kPortModePresetSecureBuffer:
6446                     {
6447                         err2 = mCodec->mOMXNode->emptyBuffer(
6448                             bufferID, info->mCodecData, flags, timeUs, info->mFenceFd);
6449                     }
6450                     break;
6451 #ifndef OMX_ANDROID_COMPILE_AS_32BIT_ON_64BIT_PLATFORMS
6452                 case IOMX::kPortModeDynamicNativeHandle:
6453                     if (info->mCodecData->size() >= sizeof(VideoNativeHandleMetadata)) {
6454                         VideoNativeHandleMetadata *vnhmd =
6455                             (VideoNativeHandleMetadata*)info->mCodecData->base();
6456                         sp<NativeHandle> handle = NativeHandle::create(
6457                                 vnhmd->pHandle, false /* ownsHandle */);
6458                         err2 = mCodec->mOMXNode->emptyBuffer(
6459                             bufferID, handle, flags, timeUs, info->mFenceFd);
6460                     }
6461                     break;
6462                 case IOMX::kPortModeDynamicANWBuffer:
6463                     if (info->mCodecData->size() >= sizeof(VideoNativeMetadata)) {
6464                         VideoNativeMetadata *vnmd = (VideoNativeMetadata*)info->mCodecData->base();
6465                         sp<GraphicBuffer> graphicBuffer = GraphicBuffer::from(vnmd->pBuffer);
6466                         err2 = mCodec->mOMXNode->emptyBuffer(
6467                             bufferID, graphicBuffer, flags, timeUs, info->mFenceFd);
6468                     }
6469                     break;
6470 #endif
6471                 default:
6472                     ALOGW("Can't marshall %s data in %zu sized buffers in %zu-bit mode",
6473                             asString(mCodec->mPortMode[kPortIndexInput]),
6474                             info->mCodecData->size(),
6475                             sizeof(buffer_handle_t) * 8);
6476                     err2 = ERROR_UNSUPPORTED;
6477                     break;
6478                 }
6479 
6480                 info->mFenceFd = -1;
6481                 if (err2 != OK) {
6482                     mCodec->signalError(OMX_ErrorUndefined, makeNoSideEffectStatus(err2));
6483                     return;
6484                 }
6485                 info->mStatus = BufferInfo::OWNED_BY_COMPONENT;
6486                 // Hold the reference while component is using the buffer.
6487                 info->mData = buffer;
6488 
6489                 if (!eos && err == OK) {
6490                     getMoreInputDataIfPossible();
6491                 } else {
6492                     ALOGV("[%s] Signalled EOS (%d) on the input port",
6493                          mCodec->mComponentName.c_str(), err);
6494 
6495                     mCodec->mPortEOS[kPortIndexInput] = true;
6496                     mCodec->mInputEOSResult = err;
6497                 }
6498             } else if (!mCodec->mPortEOS[kPortIndexInput]) {
6499                 if (err != OK && err != ERROR_END_OF_STREAM) {
6500                     ALOGV("[%s] Signalling EOS on the input port due to error %d",
6501                          mCodec->mComponentName.c_str(), err);
6502                 } else {
6503                     ALOGV("[%s] Signalling EOS on the input port",
6504                          mCodec->mComponentName.c_str());
6505                 }
6506 
6507                 ALOGV("[%s] calling emptyBuffer %u signalling EOS",
6508                      mCodec->mComponentName.c_str(), bufferID);
6509 
6510                 info->checkReadFence("onInputBufferFilled");
6511                 status_t err2 = mCodec->mOMXNode->emptyBuffer(
6512                         bufferID, OMXBuffer::sPreset, OMX_BUFFERFLAG_EOS, 0, info->mFenceFd);
6513                 info->mFenceFd = -1;
6514                 if (err2 != OK) {
6515                     mCodec->signalError(OMX_ErrorUndefined, makeNoSideEffectStatus(err2));
6516                     return;
6517                 }
6518                 info->mStatus = BufferInfo::OWNED_BY_COMPONENT;
6519 
6520                 mCodec->mPortEOS[kPortIndexInput] = true;
6521                 mCodec->mInputEOSResult = err;
6522             }
6523             break;
6524         }
6525 
6526         case FREE_BUFFERS:
6527             break;
6528 
6529         default:
6530             ALOGE("invalid port mode: %d", mode);
6531             break;
6532     }
6533 }
6534 
getMoreInputDataIfPossible()6535 void ACodec::BaseState::getMoreInputDataIfPossible() {
6536     if (mCodec->mPortEOS[kPortIndexInput]) {
6537         return;
6538     }
6539 
6540     BufferInfo *eligible = NULL;
6541 
6542     for (size_t i = 0; i < mCodec->mBuffers[kPortIndexInput].size(); ++i) {
6543         BufferInfo *info = &mCodec->mBuffers[kPortIndexInput][i];
6544 
6545 #if 0
6546         if (info->mStatus == BufferInfo::OWNED_BY_UPSTREAM) {
6547             // There's already a "read" pending.
6548             return;
6549         }
6550 #endif
6551 
6552         if (info->mStatus == BufferInfo::OWNED_BY_US) {
6553             eligible = info;
6554         }
6555     }
6556 
6557     if (eligible == NULL) {
6558         return;
6559     }
6560 
6561     postFillThisBuffer(eligible);
6562 }
6563 
setSurfaceParameters(const sp<AMessage> & msg)6564 void ACodec::BaseState::setSurfaceParameters(const sp<AMessage> &msg) {
6565     sp<AMessage> params;
6566     CHECK(msg->findMessage("params", &params));
6567 
6568     status_t err = mCodec->setSurfaceParameters(params);
6569     if (err != OK) {
6570         ALOGE("[%s] Unable to set input surface parameters (err %d)",
6571                 mCodec->mComponentName.c_str(),
6572                 err);
6573         return;
6574     }
6575 
6576     int64_t timeOffsetUs;
6577     if (params->findInt64(PARAMETER_KEY_OFFSET_TIME, &timeOffsetUs)) {
6578         params->removeEntryAt(params->findEntryByName(PARAMETER_KEY_OFFSET_TIME));
6579 
6580         if (params->countEntries() == 0) {
6581             msg->removeEntryAt(msg->findEntryByName("params"));
6582             return;
6583         }
6584     }
6585 
6586     int64_t skipFramesBeforeUs;
6587     if (params->findInt64("skip-frames-before", &skipFramesBeforeUs)) {
6588         params->removeEntryAt(params->findEntryByName("skip-frames-before"));
6589 
6590         if (params->countEntries() == 0) {
6591             msg->removeEntryAt(msg->findEntryByName("params"));
6592             return;
6593         }
6594     }
6595 
6596     int32_t dropInputFrames;
6597     if (params->findInt32(PARAMETER_KEY_SUSPEND, &dropInputFrames)) {
6598         params->removeEntryAt(params->findEntryByName(PARAMETER_KEY_SUSPEND));
6599 
6600         if (params->countEntries() == 0) {
6601             msg->removeEntryAt(msg->findEntryByName("params"));
6602             return;
6603         }
6604     }
6605 
6606     int64_t stopTimeUs;
6607     if (params->findInt64("stop-time-us", &stopTimeUs)) {
6608         params->removeEntryAt(params->findEntryByName("stop-time-us"));
6609 
6610         if (params->countEntries() == 0) {
6611             msg->removeEntryAt(msg->findEntryByName("params"));
6612             return;
6613         }
6614     }
6615 }
6616 
onOMXFillBufferDone(IOMX::buffer_id bufferID,size_t rangeOffset,size_t rangeLength,OMX_U32 flags,int64_t timeUs,int fenceFd)6617 bool ACodec::BaseState::onOMXFillBufferDone(
6618         IOMX::buffer_id bufferID,
6619         size_t rangeOffset, size_t rangeLength,
6620         OMX_U32 flags,
6621         int64_t timeUs,
6622         int fenceFd) {
6623     ALOGV("[%s] onOMXFillBufferDone %u time %" PRId64 " us, flags = 0x%08x",
6624          mCodec->mComponentName.c_str(), bufferID, timeUs, flags);
6625 
6626     ssize_t index;
6627     status_t err= OK;
6628 
6629 #if TRACK_BUFFER_TIMING
6630     index = mCodec->mBufferStats.indexOfKey(timeUs);
6631     if (index >= 0) {
6632         ACodec::BufferStats *stats = &mCodec->mBufferStats.editValueAt(index);
6633         stats->mFillBufferDoneTimeUs = ALooper::GetNowUs();
6634 
6635         ALOGI("frame PTS %lld: %lld",
6636                 timeUs,
6637                 stats->mFillBufferDoneTimeUs - stats->mEmptyBufferTimeUs);
6638 
6639         mCodec->mBufferStats.removeItemsAt(index);
6640         stats = NULL;
6641     }
6642 #endif
6643 
6644     BufferInfo *info =
6645         mCodec->findBufferByID(kPortIndexOutput, bufferID, &index);
6646     BufferInfo::Status status = BufferInfo::getSafeStatus(info);
6647     if (status != BufferInfo::OWNED_BY_COMPONENT) {
6648         ALOGE("Wrong ownership in FBD: %s(%d) buffer #%u", _asString(status), status, bufferID);
6649         mCodec->dumpBuffers(kPortIndexOutput);
6650         mCodec->signalError(OMX_ErrorUndefined, FAILED_TRANSACTION);
6651         if (fenceFd >= 0) {
6652             ::close(fenceFd);
6653         }
6654         return true;
6655     }
6656 
6657     info->mDequeuedAt = ++mCodec->mDequeueCounter;
6658     info->mStatus = BufferInfo::OWNED_BY_US;
6659 
6660     // byte buffers cannot take fences, so wait for any fence now
6661     if (mCodec->mNativeWindow == NULL) {
6662         (void)mCodec->waitForFence(fenceFd, "onOMXFillBufferDone");
6663         fenceFd = -1;
6664     }
6665     info->setReadFence(fenceFd, "onOMXFillBufferDone");
6666 
6667     PortMode mode = getPortMode(kPortIndexOutput);
6668 
6669     switch (mode) {
6670         case KEEP_BUFFERS:
6671             break;
6672 
6673         case RESUBMIT_BUFFERS:
6674         {
6675             if (rangeLength == 0 && (!(flags & OMX_BUFFERFLAG_EOS)
6676                     || mCodec->mPortEOS[kPortIndexOutput])) {
6677                 ALOGV("[%s] calling fillBuffer %u",
6678                      mCodec->mComponentName.c_str(), info->mBufferID);
6679 
6680                 err = mCodec->fillBuffer(info);
6681                 if (err != OK) {
6682                     mCodec->signalError(OMX_ErrorUndefined, makeNoSideEffectStatus(err));
6683                     return true;
6684                 }
6685                 break;
6686             }
6687 
6688             sp<MediaCodecBuffer> buffer = info->mData;
6689 
6690             if (mCodec->mOutputFormat != mCodec->mLastOutputFormat && rangeLength > 0) {
6691                 // pretend that output format has changed on the first frame (we used to do this)
6692                 if (mCodec->mBaseOutputFormat == mCodec->mOutputFormat) {
6693                     mCodec->onOutputFormatChanged(mCodec->mOutputFormat);
6694                 }
6695                 mCodec->sendFormatChange();
6696             }
6697             buffer->setFormat(mCodec->mOutputFormat);
6698 
6699             if (mCodec->usingSecureBufferOnEncoderOutput()) {
6700                 native_handle_t *handle = NULL;
6701                 sp<SecureBuffer> secureBuffer = static_cast<SecureBuffer *>(buffer.get());
6702                 if (secureBuffer != NULL) {
6703 #ifdef OMX_ANDROID_COMPILE_AS_32BIT_ON_64BIT_PLATFORMS
6704                     // handle is only valid on 32-bit/mediaserver process
6705                     handle = NULL;
6706 #else
6707                     handle = (native_handle_t *)secureBuffer->getDestinationPointer();
6708 #endif
6709                 }
6710                 buffer->meta()->setPointer("handle", handle);
6711                 buffer->meta()->setInt32("rangeOffset", rangeOffset);
6712                 buffer->meta()->setInt32("rangeLength", rangeLength);
6713             } else if (buffer->base() == info->mCodecData->base()) {
6714                 buffer->setRange(rangeOffset, rangeLength);
6715             } else {
6716                 info->mCodecData->setRange(rangeOffset, rangeLength);
6717                 // in this case we know that mConverter is not null
6718                 status_t err = mCodec->mConverter[kPortIndexOutput]->convert(
6719                         info->mCodecData, buffer);
6720                 if (err != OK) {
6721                     mCodec->signalError(OMX_ErrorUndefined, makeNoSideEffectStatus(err));
6722                     return true;
6723                 }
6724             }
6725 #if 0
6726             if (mCodec->mNativeWindow == NULL) {
6727                 if (IsIDR(info->mData->data(), info->mData->size())) {
6728                     ALOGI("IDR frame");
6729                 }
6730             }
6731 #endif
6732 
6733             if (mCodec->mSkipCutBuffer != NULL) {
6734                 mCodec->mSkipCutBuffer->submit(buffer);
6735             }
6736             buffer->meta()->setInt64("timeUs", timeUs);
6737 
6738             info->mData.clear();
6739 
6740             // Workaround: if OMX_BUFFERFLAG_DECODEONLY is not implemented in
6741             // HAL, the flag is then removed in the corresponding output buffer.
6742 
6743             // for all buffers that were marked as DECODE_ONLY, remove their timestamp
6744             // if it is smaller than the timestamp of the buffer that was
6745             // just received
6746             while (!mCodec->mDecodeOnlyTimesUs.empty() &&
6747                    *mCodec->mDecodeOnlyTimesUs.begin() < timeUs) {
6748                     mCodec->mDecodeOnlyTimesUs.erase(mCodec->mDecodeOnlyTimesUs.begin());
6749             }
6750             // if OMX_BUFFERFLAG_DECODEONLY is not implemented in HAL, we need to restore the
6751             // OMX_BUFFERFLAG_DECODEONLY flag to the frames we had saved in the set, the set
6752             // contains the timestamps of buffers that were marked as DECODE_ONLY by the app
6753             if (!mCodec->mDecodeOnlyTimesUs.empty() &&
6754                 *mCodec->mDecodeOnlyTimesUs.begin() == timeUs) {
6755                 mCodec->mDecodeOnlyTimesUs.erase(timeUs);
6756                 // If the app queued the last valid buffer as DECODE_ONLY and queued an additional
6757                 // empty buffer as EOS, it's possible that HAL sets the last valid frame as EOS
6758                 // instead and drops the empty buffer. In such a case, we should not add back
6759                 // the OMX_BUFFERFLAG_DECODEONLY flag to it, as doing so will make it so that the
6760                 // app does not receive the EOS buffer, which breaks the contract of EOS buffers
6761                 if (flags & OMX_BUFFERFLAG_EOS) {
6762                     // Set buffer size to 0, as described by
6763                     // https://developer.android.com/reference/android/media/MediaCodec.BufferInfo?hl=en#size
6764                     // a buffer of size 0 should only be used to carry the EOS flag and should
6765                     // be discarded by the app as it has no data
6766                     buffer->setRange(0, 0);
6767                 } else {
6768                     // re-add the OMX_BUFFERFLAG_DECODEONLY flag to the buffer in case it is
6769                     // not the end of stream buffer
6770                     flags |= OMX_BUFFERFLAG_DECODEONLY;
6771                 }
6772             }
6773             mCodec->mBufferChannel->drainThisBuffer(info->mBufferID, flags);
6774 
6775             info->mStatus = BufferInfo::OWNED_BY_DOWNSTREAM;
6776 
6777             if (flags & OMX_BUFFERFLAG_EOS) {
6778                 ALOGV("[%s] saw output EOS", mCodec->mComponentName.c_str());
6779 
6780                 mCodec->mCallback->onEos(mCodec->mInputEOSResult);
6781                 mCodec->mPortEOS[kPortIndexOutput] = true;
6782             }
6783             break;
6784         }
6785 
6786         case FREE_BUFFERS:
6787             err = mCodec->freeBuffer(kPortIndexOutput, index);
6788             if (err != OK) {
6789                 mCodec->signalError(OMX_ErrorUndefined, makeNoSideEffectStatus(err));
6790                 return true;
6791             }
6792             break;
6793 
6794         default:
6795             ALOGE("Invalid port mode: %d", mode);
6796             return false;
6797     }
6798 
6799     return true;
6800 }
6801 
onOutputBufferDrained(const sp<AMessage> & msg)6802 void ACodec::BaseState::onOutputBufferDrained(const sp<AMessage> &msg) {
6803     IOMX::buffer_id bufferID;
6804     CHECK(msg->findInt32("buffer-id", (int32_t*)&bufferID));
6805     sp<RefBase> obj;
6806     CHECK(msg->findObject("buffer", &obj));
6807     sp<MediaCodecBuffer> buffer = static_cast<MediaCodecBuffer *>(obj.get());
6808     int32_t discarded = 0;
6809     msg->findInt32("discarded", &discarded);
6810 
6811     ssize_t index;
6812     BufferInfo *info = mCodec->findBufferByID(kPortIndexOutput, bufferID, &index);
6813     BufferInfo::Status status = BufferInfo::getSafeStatus(info);
6814     if (status != BufferInfo::OWNED_BY_DOWNSTREAM) {
6815         ALOGE("Wrong ownership in OBD: %s(%d) buffer #%u", _asString(status), status, bufferID);
6816         mCodec->dumpBuffers(kPortIndexOutput);
6817         mCodec->signalError(OMX_ErrorUndefined, FAILED_TRANSACTION);
6818         return;
6819     }
6820     info->mData = buffer;
6821     int32_t render;
6822     if (mCodec->mNativeWindow != NULL
6823             && msg->findInt32("render", &render) && render != 0
6824             && !discarded && buffer->size() != 0) {
6825         ATRACE_NAME("render");
6826         // The client wants this buffer to be rendered.
6827 
6828         android_native_rect_t crop;
6829         if (buffer->format()->findRect("crop", &crop.left, &crop.top, &crop.right, &crop.bottom)) {
6830             // NOTE: native window uses extended right-bottom coordinate
6831             ++crop.right;
6832             ++crop.bottom;
6833             if (memcmp(&crop, &mCodec->mLastNativeWindowCrop, sizeof(crop)) != 0) {
6834                 mCodec->mLastNativeWindowCrop = crop;
6835                 status_t err = native_window_set_crop(mCodec->mNativeWindow.get(), &crop);
6836                 ALOGW_IF(err != NO_ERROR, "failed to set crop: %d", err);
6837             }
6838         }
6839 
6840         int32_t dataSpace;
6841         if (buffer->format()->findInt32("android._dataspace", &dataSpace)
6842                 && dataSpace != mCodec->mLastNativeWindowDataSpace) {
6843             status_t err = native_window_set_buffers_data_space(
6844                     mCodec->mNativeWindow.get(), (android_dataspace)dataSpace);
6845             mCodec->mLastNativeWindowDataSpace = dataSpace;
6846             ALOGW_IF(err != NO_ERROR, "failed to set dataspace: %d", err);
6847         }
6848         if (buffer->format()->contains("hdr-static-info")) {
6849             HDRStaticInfo info;
6850             if (ColorUtils::getHDRStaticInfoFromFormat(buffer->format(), &info)
6851                 && memcmp(&mCodec->mLastHDRStaticInfo, &info, sizeof(info))) {
6852                 setNativeWindowHdrMetadata(mCodec->mNativeWindow.get(), &info);
6853                 mCodec->mLastHDRStaticInfo = info;
6854             }
6855         }
6856 
6857         sp<ABuffer> hdr10PlusInfo;
6858         if (buffer->format()->findBuffer("hdr10-plus-info", &hdr10PlusInfo)
6859                 && hdr10PlusInfo != nullptr && hdr10PlusInfo->size() > 0
6860                 && hdr10PlusInfo != mCodec->mLastHdr10PlusBuffer) {
6861             native_window_set_buffers_hdr10_plus_metadata(mCodec->mNativeWindow.get(),
6862                     hdr10PlusInfo->size(), hdr10PlusInfo->data());
6863             mCodec->mLastHdr10PlusBuffer = hdr10PlusInfo;
6864         }
6865 
6866         int64_t timestampNs = 0;
6867         if (!msg->findInt64("timestampNs", &timestampNs)) {
6868             // use media timestamp if client did not request a specific render timestamp
6869             if (buffer->meta()->findInt64("timeUs", &timestampNs)) {
6870                 ALOGV("using buffer PTS of %lld", (long long)timestampNs);
6871                 timestampNs *= 1000;
6872             }
6873         }
6874 
6875         status_t err;
6876         err = native_window_set_buffers_timestamp(mCodec->mNativeWindow.get(), timestampNs);
6877         ALOGW_IF(err != NO_ERROR, "failed to set buffer timestamp: %d", err);
6878 
6879         uint64_t frameId;
6880         err = native_window_get_next_frame_id(mCodec->mNativeWindow.get(), &frameId);
6881 
6882         info->checkReadFence("onOutputBufferDrained before queueBuffer");
6883         err = mCodec->mNativeWindow->queueBuffer(
6884                     mCodec->mNativeWindow.get(), info->mGraphicBuffer.get(), info->mFenceFd);
6885 
6886         int64_t mediaTimeUs = -1;
6887         buffer->meta()->findInt64("timeUs", &mediaTimeUs);
6888         if (mCodec->mAreRenderMetricsEnabled && mCodec->mIsWindowToDisplay) {
6889             mCodec->trackReleasedFrame(frameId, mediaTimeUs, timestampNs);
6890             mCodec->pollForRenderedFrames();
6891         } else {
6892             // When the surface is an intermediate surface, onFrameRendered is triggered immediately
6893             // when the frame is queued to the non-display surface
6894             mCodec->mCallback->onOutputFramesRendered({RenderedFrameInfo(mediaTimeUs,
6895                                                                          timestampNs)});
6896         }
6897 
6898         info->mFenceFd = -1;
6899         if (err == OK) {
6900             info->mStatus = BufferInfo::OWNED_BY_NATIVE_WINDOW;
6901         } else {
6902             ALOGE("queueBuffer failed in onOutputBufferDrained: %d", err);
6903             mCodec->signalError(OMX_ErrorUndefined, makeNoSideEffectStatus(err));
6904             info->mStatus = BufferInfo::OWNED_BY_US;
6905             // keeping read fence as write fence to avoid clobbering
6906             info->mIsReadFence = false;
6907         }
6908     } else {
6909         if (mCodec->mNativeWindow != NULL && (discarded || buffer->size() != 0)) {
6910             // move read fence into write fence to avoid clobbering
6911             info->mIsReadFence = false;
6912             ATRACE_NAME("frame-drop");
6913         }
6914         info->mStatus = BufferInfo::OWNED_BY_US;
6915     }
6916 
6917     PortMode mode = getPortMode(kPortIndexOutput);
6918 
6919     switch (mode) {
6920         case KEEP_BUFFERS:
6921         {
6922             // XXX fishy, revisit!!! What about the FREE_BUFFERS case below?
6923 
6924             if (info->mStatus == BufferInfo::OWNED_BY_NATIVE_WINDOW) {
6925                 // We cannot resubmit the buffer we just rendered, dequeue
6926                 // the spare instead.
6927 
6928                 info = mCodec->dequeueBufferFromNativeWindow();
6929             }
6930             break;
6931         }
6932 
6933         case RESUBMIT_BUFFERS:
6934         {
6935             if (!mCodec->mPortEOS[kPortIndexOutput]) {
6936                 if (info->mStatus == BufferInfo::OWNED_BY_NATIVE_WINDOW) {
6937                     // We cannot resubmit the buffer we just rendered, dequeue
6938                     // the spare instead.
6939 
6940                     info = mCodec->dequeueBufferFromNativeWindow();
6941                 }
6942 
6943                 if (info != NULL) {
6944                     ALOGV("[%s] calling fillBuffer %u",
6945                          mCodec->mComponentName.c_str(), info->mBufferID);
6946                     info->checkWriteFence("onOutputBufferDrained::RESUBMIT_BUFFERS");
6947                     status_t err = mCodec->fillBuffer(info);
6948                     if (err != OK) {
6949                         mCodec->signalError(OMX_ErrorUndefined, makeNoSideEffectStatus(err));
6950                     }
6951                 }
6952             }
6953             break;
6954         }
6955 
6956         case FREE_BUFFERS:
6957         {
6958             status_t err = mCodec->freeBuffer(kPortIndexOutput, index);
6959             if (err != OK) {
6960                 mCodec->signalError(OMX_ErrorUndefined, makeNoSideEffectStatus(err));
6961             }
6962             break;
6963         }
6964 
6965         default:
6966             ALOGE("Invalid port mode: %d", mode);
6967             return;
6968     }
6969 }
6970 
6971 ////////////////////////////////////////////////////////////////////////////////
6972 
UninitializedState(ACodec * codec)6973 ACodec::UninitializedState::UninitializedState(ACodec *codec)
6974     : BaseState(codec) {
6975 }
6976 
stateEntered()6977 void ACodec::UninitializedState::stateEntered() {
6978     ALOGV("Now uninitialized");
6979 
6980     if (mDeathNotifier != NULL) {
6981         if (mCodec->mOMXNode != NULL) {
6982             auto tOmxNode = mCodec->mOMXNode->getHalInterface<IOmxNode>();
6983             if (tOmxNode) {
6984                 tOmxNode->unlinkToDeath(mDeathNotifier);
6985             }
6986         }
6987         mDeathNotifier.clear();
6988     }
6989 
6990     mCodec->mUsingNativeWindow = false;
6991     mCodec->mNativeWindow.clear();
6992     mCodec->mNativeWindowUsageBits = 0;
6993     mCodec->mOMX.clear();
6994     mCodec->mOMXNode.clear();
6995     mCodec->mFlags = 0;
6996     mCodec->mPortMode[kPortIndexInput] = IOMX::kPortModePresetByteBuffer;
6997     mCodec->mPortMode[kPortIndexOutput] = IOMX::kPortModePresetByteBuffer;
6998     mCodec->mConverter[0].clear();
6999     mCodec->mConverter[1].clear();
7000     mCodec->mComponentName.clear();
7001     mCodec->mDecodeOnlyTimesUs.clear();
7002 }
7003 
onMessageReceived(const sp<AMessage> & msg)7004 bool ACodec::UninitializedState::onMessageReceived(const sp<AMessage> &msg) {
7005     bool handled = false;
7006 
7007     switch (msg->what()) {
7008         case ACodec::kWhatSetup:
7009         {
7010             onSetup(msg);
7011 
7012             handled = true;
7013             break;
7014         }
7015 
7016         case ACodec::kWhatAllocateComponent:
7017         {
7018             onAllocateComponent(msg);
7019             handled = true;
7020             break;
7021         }
7022 
7023         case ACodec::kWhatShutdown:
7024         {
7025             int32_t keepComponentAllocated;
7026             CHECK(msg->findInt32(
7027                         "keepComponentAllocated", &keepComponentAllocated));
7028             ALOGW_IF(keepComponentAllocated,
7029                      "cannot keep component allocated on shutdown in Uninitialized state");
7030             if (keepComponentAllocated) {
7031                 mCodec->mCallback->onStopCompleted();
7032             } else {
7033                 mCodec->mCallback->onReleaseCompleted();
7034             }
7035             handled = true;
7036             break;
7037         }
7038 
7039         case ACodec::kWhatFlush:
7040         {
7041             mCodec->mCallback->onFlushCompleted();
7042             handled = true;
7043             break;
7044         }
7045 
7046         case ACodec::kWhatReleaseCodecInstance:
7047         {
7048             // nothing to do, as we have already signaled shutdown
7049             handled = true;
7050             break;
7051         }
7052 
7053         default:
7054             return BaseState::onMessageReceived(msg);
7055     }
7056 
7057     return handled;
7058 }
7059 
onSetup(const sp<AMessage> & msg)7060 void ACodec::UninitializedState::onSetup(
7061         const sp<AMessage> &msg) {
7062     if (onAllocateComponent(msg)
7063             && mCodec->mLoadedState->onConfigureComponent(msg)) {
7064         mCodec->mLoadedState->onStart();
7065     }
7066 }
7067 
onAllocateComponent(const sp<AMessage> & msg)7068 bool ACodec::UninitializedState::onAllocateComponent(const sp<AMessage> &msg) {
7069     ALOGV("onAllocateComponent");
7070 
7071     CHECK(mCodec->mOMXNode == NULL);
7072     mCodec->mFatalError = false;
7073 
7074     sp<AMessage> notify = new AMessage(kWhatOMXMessageList, mCodec);
7075     notify->setInt32("generation", mCodec->mNodeGeneration + 1);
7076 
7077     sp<RefBase> obj;
7078     CHECK(msg->findObject("codecInfo", &obj));
7079     sp<MediaCodecInfo> info = (MediaCodecInfo *)obj.get();
7080     if (info == nullptr) {
7081         ALOGE("Unexpected nullptr for codec information");
7082         mCodec->signalError(OMX_ErrorUndefined, UNKNOWN_ERROR);
7083         return false;
7084     }
7085     AString owner = (info->getOwnerName() == nullptr) ? "default" : info->getOwnerName();
7086 
7087     AString componentName = info->getCodecName();
7088     // we are no longer using "componentName" as we always pass the codec info for owner.
7089     // CHECK(msg->findString("componentName", &componentName));
7090     AString halName = info->getHalName();
7091 
7092     sp<CodecObserver> observer = new CodecObserver(notify);
7093     sp<IOMX> omx;
7094     sp<IOMXNode> omxNode;
7095 
7096     status_t err = NAME_NOT_FOUND;
7097     OMXClient client;
7098     if (client.connect(owner.c_str()) != OK) {
7099         mCodec->signalError(OMX_ErrorUndefined, NO_INIT);
7100         return false;
7101     }
7102     omx = client.interface();
7103 
7104     pid_t tid = gettid();
7105     int prevPriority = androidGetThreadPriority(tid);
7106     androidSetThreadPriority(tid, ANDROID_PRIORITY_FOREGROUND);
7107     err = omx->allocateNode(halName.c_str(), observer, &omxNode);
7108     androidSetThreadPriority(tid, prevPriority);
7109 
7110     if (err != OK) {
7111         ALOGE("Unable to instantiate codec '%s' for '%s' with err %#x.",
7112                 halName.c_str(), componentName.c_str(), err);
7113 
7114         mCodec->signalError((OMX_ERRORTYPE)err, makeNoSideEffectStatus(err));
7115         return false;
7116     }
7117 
7118     mDeathNotifier = new DeathNotifier(new AMessage(kWhatOMXDied, mCodec));
7119     auto tOmxNode = omxNode->getHalInterface<IOmxNode>();
7120     if (tOmxNode && !tOmxNode->linkToDeath(mDeathNotifier, 0)) {
7121         mDeathNotifier.clear();
7122     }
7123 
7124     ++mCodec->mNodeGeneration;
7125 
7126     mCodec->mComponentName = componentName;
7127     mCodec->mFlags = 0;
7128 
7129     if (componentName.endsWith(".secure")) {
7130         mCodec->mFlags |= kFlagIsSecure;
7131         mCodec->mFlags |= kFlagIsGrallocUsageProtected;
7132         mCodec->mFlags |= kFlagPushBlankBuffersToNativeWindowOnShutdown;
7133     }
7134 
7135     mCodec->mOMX = omx;
7136     mCodec->mOMXNode = omxNode;
7137     mCodec->mCallback->onComponentAllocated(mCodec->mComponentName.c_str());
7138     mCodec->changeState(mCodec->mLoadedState);
7139 
7140     return true;
7141 }
7142 
7143 ////////////////////////////////////////////////////////////////////////////////
7144 
LoadedState(ACodec * codec)7145 ACodec::LoadedState::LoadedState(ACodec *codec)
7146     : BaseState(codec) {
7147 }
7148 
stateEntered()7149 void ACodec::LoadedState::stateEntered() {
7150     ALOGV("[%s] Now Loaded", mCodec->mComponentName.c_str());
7151 
7152     mCodec->mPortEOS[kPortIndexInput] =
7153         mCodec->mPortEOS[kPortIndexOutput] = false;
7154 
7155     mCodec->mInputEOSResult = OK;
7156 
7157     mCodec->mDequeueCounter = 0;
7158     mCodec->mMetadataBuffersToSubmit = 0;
7159     mCodec->mRepeatFrameDelayUs = -1LL;
7160     mCodec->mInputFormat.clear();
7161     mCodec->mOutputFormat.clear();
7162     mCodec->mBaseOutputFormat.clear();
7163     mCodec->mGraphicBufferSource.clear();
7164 
7165     if (mCodec->mShutdownInProgress) {
7166         bool keepComponentAllocated = mCodec->mKeepComponentAllocated;
7167 
7168         mCodec->mShutdownInProgress = false;
7169         mCodec->mKeepComponentAllocated = false;
7170 
7171         onShutdown(keepComponentAllocated);
7172     }
7173     mCodec->mExplicitShutdown = false;
7174 
7175     mCodec->processDeferredMessages();
7176 }
7177 
onShutdown(bool keepComponentAllocated)7178 void ACodec::LoadedState::onShutdown(bool keepComponentAllocated) {
7179     if (!keepComponentAllocated) {
7180         (void)mCodec->mOMXNode->freeNode();
7181 
7182         mCodec->changeState(mCodec->mUninitializedState);
7183     }
7184 
7185     if (mCodec->mExplicitShutdown) {
7186         if (keepComponentAllocated) {
7187             mCodec->mCallback->onStopCompleted();
7188         } else {
7189             mCodec->mCallback->onReleaseCompleted();
7190         }
7191         mCodec->mExplicitShutdown = false;
7192     }
7193 }
7194 
onMessageReceived(const sp<AMessage> & msg)7195 bool ACodec::LoadedState::onMessageReceived(const sp<AMessage> &msg) {
7196     bool handled = false;
7197 
7198     switch (msg->what()) {
7199         case ACodec::kWhatConfigureComponent:
7200         {
7201             onConfigureComponent(msg);
7202             handled = true;
7203             break;
7204         }
7205 
7206         case ACodec::kWhatCreateInputSurface:
7207         {
7208             onCreateInputSurface(msg);
7209             handled = true;
7210             break;
7211         }
7212 
7213         case ACodec::kWhatSetInputSurface:
7214         {
7215             onSetInputSurface(msg);
7216             handled = true;
7217             break;
7218         }
7219 
7220         case ACodec::kWhatStart:
7221         {
7222             onStart();
7223             handled = true;
7224             break;
7225         }
7226 
7227         case ACodec::kWhatShutdown:
7228         {
7229             int32_t keepComponentAllocated;
7230             CHECK(msg->findInt32(
7231                         "keepComponentAllocated", &keepComponentAllocated));
7232 
7233             mCodec->mExplicitShutdown = true;
7234             onShutdown(keepComponentAllocated);
7235 
7236             handled = true;
7237             break;
7238         }
7239 
7240         case ACodec::kWhatFlush:
7241         {
7242             mCodec->mCallback->onFlushCompleted();
7243             handled = true;
7244             break;
7245         }
7246 
7247         default:
7248             return BaseState::onMessageReceived(msg);
7249     }
7250 
7251     return handled;
7252 }
7253 
onConfigureComponent(const sp<AMessage> & msg)7254 bool ACodec::LoadedState::onConfigureComponent(
7255         const sp<AMessage> &msg) {
7256     ALOGV("onConfigureComponent");
7257 
7258     CHECK(mCodec->mOMXNode != NULL);
7259 
7260     status_t err = OK;
7261     AString mime;
7262     if (!msg->findString("mime", &mime)) {
7263         err = BAD_VALUE;
7264     } else {
7265         err = mCodec->configureCodec(mime.c_str(), msg);
7266     }
7267     if (err != OK) {
7268         ALOGE("[%s] configureCodec returning error %d",
7269               mCodec->mComponentName.c_str(), err);
7270 
7271         mCodec->signalError(OMX_ErrorUndefined, makeNoSideEffectStatus(err));
7272         return false;
7273     }
7274 
7275     mCodec->mCallback->onComponentConfigured(mCodec->mInputFormat, mCodec->mOutputFormat);
7276 
7277     return true;
7278 }
7279 
setupInputSurface()7280 status_t ACodec::LoadedState::setupInputSurface() {
7281     if (mCodec->mGraphicBufferSource == NULL) {
7282         return BAD_VALUE;
7283     }
7284 
7285     android_dataspace dataSpace;
7286     status_t err =
7287         mCodec->setInitialColorAspectsForVideoEncoderSurfaceAndGetDataSpace(&dataSpace);
7288     if (err != OK) {
7289         ALOGE("Failed to get default data space");
7290         return err;
7291     }
7292 
7293     err = statusFromBinderStatus(
7294             mCodec->mGraphicBufferSource->configure(
7295                     mCodec->mOMXNode->getHalInterface<IOmxNode>(),
7296                     static_cast<hardware::graphics::common::V1_0::Dataspace>(dataSpace)));
7297     if (err != OK) {
7298         ALOGE("[%s] Unable to configure for node (err %d)",
7299               mCodec->mComponentName.c_str(), err);
7300         return err;
7301     }
7302 
7303     if (mCodec->mRepeatFrameDelayUs > 0LL) {
7304         err = statusFromBinderStatus(
7305                 mCodec->mGraphicBufferSource->setRepeatPreviousFrameDelayUs(
7306                         mCodec->mRepeatFrameDelayUs));
7307 
7308         if (err != OK) {
7309             ALOGE("[%s] Unable to configure option to repeat previous "
7310                   "frames (err %d)",
7311                   mCodec->mComponentName.c_str(), err);
7312             return err;
7313         }
7314     }
7315 
7316     if (mCodec->mIsVideo && mCodec->mMaxPtsGapUs != 0LL) {
7317         OMX_PARAM_U32TYPE maxPtsGapParams;
7318         InitOMXParams(&maxPtsGapParams);
7319         maxPtsGapParams.nPortIndex = kPortIndexInput;
7320         maxPtsGapParams.nU32 = (uint32_t)mCodec->mMaxPtsGapUs;
7321 
7322         err = mCodec->mOMXNode->setParameter(
7323                 (OMX_INDEXTYPE)OMX_IndexParamMaxFrameDurationForBitrateControl,
7324                 &maxPtsGapParams, sizeof(maxPtsGapParams));
7325 
7326         if (err != OK) {
7327             ALOGE("[%s] Unable to configure max timestamp gap (err %d)",
7328                     mCodec->mComponentName.c_str(), err);
7329             return err;
7330         }
7331     }
7332 
7333     if (mCodec->mMaxFps > 0 || mCodec->mMaxPtsGapUs < 0) {
7334         err = statusFromBinderStatus(
7335                 mCodec->mGraphicBufferSource->setMaxFps(mCodec->mMaxFps));
7336 
7337         if (err != OK) {
7338             ALOGE("[%s] Unable to configure max fps (err %d)",
7339                     mCodec->mComponentName.c_str(), err);
7340             return err;
7341         }
7342     }
7343 
7344     if (mCodec->mCaptureFps > 0. && mCodec->mFps > 0.) {
7345         err = statusFromBinderStatus(
7346                 mCodec->mGraphicBufferSource->setTimeLapseConfig(
7347                         mCodec->mFps, mCodec->mCaptureFps));
7348 
7349         if (err != OK) {
7350             ALOGE("[%s] Unable to configure time lapse (err %d)",
7351                     mCodec->mComponentName.c_str(), err);
7352             return err;
7353         }
7354     }
7355 
7356     if (mCodec->mCreateInputBuffersSuspended) {
7357         err = statusFromBinderStatus(
7358                 mCodec->mGraphicBufferSource->setSuspend(true, -1));
7359 
7360         if (err != OK) {
7361             ALOGE("[%s] Unable to configure option to suspend (err %d)",
7362                   mCodec->mComponentName.c_str(), err);
7363             return err;
7364         }
7365     }
7366 
7367     uint32_t usageBits;
7368     if (mCodec->mOMXNode->getParameter(
7369             (OMX_INDEXTYPE)OMX_IndexParamConsumerUsageBits,
7370             &usageBits, sizeof(usageBits)) == OK) {
7371         mCodec->mInputFormat->setInt32(
7372                 "using-sw-read-often", !!(usageBits & GRALLOC_USAGE_SW_READ_OFTEN));
7373     }
7374 
7375     sp<ABuffer> colorAspectsBuffer;
7376     if (mCodec->mInputFormat->findBuffer("android._color-aspects", &colorAspectsBuffer)) {
7377         if (colorAspectsBuffer->size() != sizeof(ColorAspects)) {
7378             return INVALID_OPERATION;
7379         }
7380 
7381         err = statusFromBinderStatus(
7382                 mCodec->mGraphicBufferSource->setColorAspects(
7383                         hardware::media::omx::V1_0::utils::toHardwareColorAspects(
7384                                 *(ColorAspects *)colorAspectsBuffer->base())));
7385 
7386         if (err != OK) {
7387             ALOGE("[%s] Unable to configure color aspects (err %d)",
7388                   mCodec->mComponentName.c_str(), err);
7389             return err;
7390         }
7391     }
7392     return OK;
7393 }
7394 
onCreateInputSurface(const sp<AMessage> &)7395 void ACodec::LoadedState::onCreateInputSurface(
7396         const sp<AMessage> & /* msg */) {
7397     ALOGV("onCreateInputSurface");
7398 
7399     sp<IGraphicBufferProducer> bufferProducer;
7400     sp<HGraphicBufferSource> bufferSource;
7401     status_t err = mCodec->mOMX->createInputSurface(
7402             &bufferProducer, &bufferSource);
7403     mCodec->mGraphicBufferSource = bufferSource;
7404 
7405     if (err == OK) {
7406         err = setupInputSurface();
7407     }
7408 
7409     if (err == OK) {
7410         mCodec->mCallback->onInputSurfaceCreated(
7411                 mCodec->mInputFormat,
7412                 mCodec->mOutputFormat,
7413                 new BufferProducerWrapper(bufferProducer));
7414     } else {
7415         // Can't use mCodec->signalError() here -- MediaCodec won't forward
7416         // the error through because it's in the "configured" state.  We
7417         // send a kWhatInputSurfaceCreated with an error value instead.
7418         ALOGE("[%s] onCreateInputSurface returning error %d",
7419                 mCodec->mComponentName.c_str(), err);
7420         mCodec->mCallback->onInputSurfaceCreationFailed(err);
7421     }
7422 }
7423 
onSetInputSurface(const sp<AMessage> & msg)7424 void ACodec::LoadedState::onSetInputSurface(const sp<AMessage> &msg) {
7425     ALOGV("onSetInputSurface");
7426 
7427     sp<RefBase> obj;
7428     CHECK(msg->findObject("input-surface", &obj));
7429     if (obj == NULL) {
7430         ALOGE("[%s] NULL input surface", mCodec->mComponentName.c_str());
7431         mCodec->mCallback->onInputSurfaceDeclined(BAD_VALUE);
7432         return;
7433     }
7434 
7435     sp<PersistentSurface> surface = static_cast<PersistentSurface *>(obj.get());
7436     sp<HGraphicBufferSource> hgbs = HGraphicBufferSource::castFrom(surface->getHidlTarget());
7437     status_t err = BAD_VALUE;
7438     if (hgbs) {
7439         mCodec->mGraphicBufferSource = hgbs;
7440         err = setupInputSurface();
7441     }
7442 
7443     if (err == OK) {
7444         mCodec->mCallback->onInputSurfaceAccepted(
7445                 mCodec->mInputFormat, mCodec->mOutputFormat);
7446     } else {
7447         // Can't use mCodec->signalError() here -- MediaCodec won't forward
7448         // the error through because it's in the "configured" state.  We
7449         // send a kWhatInputSurfaceAccepted with an error value instead.
7450         ALOGE("[%s] onSetInputSurface returning error %d",
7451                 mCodec->mComponentName.c_str(), err);
7452         mCodec->mCallback->onInputSurfaceDeclined(err);
7453     }
7454 }
7455 
onStart()7456 void ACodec::LoadedState::onStart() {
7457     ALOGV("onStart");
7458 
7459     status_t err = mCodec->mOMXNode->sendCommand(OMX_CommandStateSet, OMX_StateIdle);
7460     if (err != OK) {
7461         mCodec->signalError(OMX_ErrorUndefined, makeNoSideEffectStatus(err));
7462     } else {
7463         mCodec->changeState(mCodec->mLoadedToIdleState);
7464     }
7465 }
7466 
7467 ////////////////////////////////////////////////////////////////////////////////
7468 
LoadedToIdleState(ACodec * codec)7469 ACodec::LoadedToIdleState::LoadedToIdleState(ACodec *codec)
7470     : BaseState(codec) {
7471 }
7472 
stateEntered()7473 void ACodec::LoadedToIdleState::stateEntered() {
7474     ALOGV("[%s] Now Loaded->Idle", mCodec->mComponentName.c_str());
7475 
7476     status_t err;
7477     if ((err = allocateBuffers()) != OK) {
7478         ALOGE("Failed to allocate buffers after transitioning to IDLE state "
7479              "(error 0x%08x)",
7480              err);
7481 
7482         mCodec->signalError(OMX_ErrorUndefined, makeNoSideEffectStatus(err));
7483 
7484         mCodec->mOMXNode->sendCommand(
7485                 OMX_CommandStateSet, OMX_StateLoaded);
7486         if (mCodec->allYourBuffersAreBelongToUs(kPortIndexInput)) {
7487             mCodec->freeBuffersOnPort(kPortIndexInput);
7488         }
7489         if (mCodec->allYourBuffersAreBelongToUs(kPortIndexOutput)) {
7490             mCodec->freeBuffersOnPort(kPortIndexOutput);
7491         }
7492 
7493         mCodec->changeState(mCodec->mLoadedState);
7494     }
7495 }
7496 
allocateBuffers()7497 status_t ACodec::LoadedToIdleState::allocateBuffers() {
7498     status_t err = mCodec->allocateBuffersOnPort(kPortIndexInput);
7499     if (err != OK) {
7500         return err;
7501     }
7502 
7503     err = mCodec->allocateBuffersOnPort(kPortIndexOutput);
7504     if (err != OK) {
7505         return err;
7506     }
7507 
7508     mCodec->mCallback->onStartCompleted();
7509 
7510     return OK;
7511 }
7512 
onMessageReceived(const sp<AMessage> & msg)7513 bool ACodec::LoadedToIdleState::onMessageReceived(const sp<AMessage> &msg) {
7514     switch (msg->what()) {
7515         case kWhatSetParameters:
7516         {
7517             BaseState::setSurfaceParameters(msg);
7518             if (msg->countEntries() > 0) {
7519                 mCodec->deferMessage(msg);
7520             }
7521             return true;
7522         }
7523         case kWhatShutdown:
7524         {
7525             mCodec->deferMessage(msg);
7526             return true;
7527         }
7528 
7529         case kWhatSignalEndOfInputStream:
7530         {
7531             mCodec->onSignalEndOfInputStream();
7532             return true;
7533         }
7534 
7535         case kWhatResume:
7536         {
7537             // We'll be active soon enough.
7538             return true;
7539         }
7540 
7541         case kWhatFlush:
7542         {
7543             // We haven't even started yet, so we're flushed alright...
7544             mCodec->mCallback->onFlushCompleted();
7545             return true;
7546         }
7547 
7548         default:
7549             return BaseState::onMessageReceived(msg);
7550     }
7551 }
7552 
onOMXEvent(OMX_EVENTTYPE event,OMX_U32 data1,OMX_U32 data2)7553 bool ACodec::LoadedToIdleState::onOMXEvent(
7554         OMX_EVENTTYPE event, OMX_U32 data1, OMX_U32 data2) {
7555     switch (event) {
7556         case OMX_EventCmdComplete:
7557         {
7558             status_t err = OK;
7559             if (data1 != (OMX_U32)OMX_CommandStateSet
7560                     || data2 != (OMX_U32)OMX_StateIdle) {
7561                 ALOGE("Unexpected command completion in LoadedToIdleState: %s(%u) %s(%u)",
7562                         asString((OMX_COMMANDTYPE)data1), data1,
7563                         asString((OMX_STATETYPE)data2), data2);
7564                 err = FAILED_TRANSACTION;
7565             }
7566 
7567             if (err == OK) {
7568                 err = mCodec->mOMXNode->sendCommand(
7569                     OMX_CommandStateSet, OMX_StateExecuting);
7570             }
7571 
7572             if (err != OK) {
7573                 mCodec->signalError(OMX_ErrorUndefined, makeNoSideEffectStatus(err));
7574             } else {
7575                 mCodec->changeState(mCodec->mIdleToExecutingState);
7576             }
7577 
7578             return true;
7579         }
7580 
7581         // When Acodec receive an error event at LoadedToIdleState, it will not release
7582         // allocated buffers, which will cause gralloc buffer leak issue. We need to first release
7583         // these buffers and then process the error event
7584         case OMX_EventError:
7585         {
7586             if (mCodec->allYourBuffersAreBelongToUs(kPortIndexInput)) {
7587                 mCodec->freeBuffersOnPort(kPortIndexInput);
7588             }
7589 
7590             if (mCodec->allYourBuffersAreBelongToUs(kPortIndexOutput)) {
7591                 mCodec->freeBuffersOnPort(kPortIndexOutput);
7592             }
7593 
7594             return BaseState::onOMXEvent(event, data1, data2);
7595         }
7596 
7597         default:
7598             return BaseState::onOMXEvent(event, data1, data2);
7599     }
7600 }
7601 
7602 ////////////////////////////////////////////////////////////////////////////////
7603 
IdleToExecutingState(ACodec * codec)7604 ACodec::IdleToExecutingState::IdleToExecutingState(ACodec *codec)
7605     : BaseState(codec) {
7606 }
7607 
stateEntered()7608 void ACodec::IdleToExecutingState::stateEntered() {
7609     ALOGV("[%s] Now Idle->Executing", mCodec->mComponentName.c_str());
7610 }
7611 
onMessageReceived(const sp<AMessage> & msg)7612 bool ACodec::IdleToExecutingState::onMessageReceived(const sp<AMessage> &msg) {
7613     switch (msg->what()) {
7614         case kWhatSetParameters:
7615         {
7616             BaseState::setSurfaceParameters(msg);
7617             if (msg->countEntries() > 0) {
7618                 mCodec->deferMessage(msg);
7619             }
7620             return true;
7621         }
7622         case kWhatShutdown:
7623         {
7624             mCodec->deferMessage(msg);
7625             return true;
7626         }
7627 
7628         case kWhatResume:
7629         {
7630             // We'll be active soon enough.
7631             return true;
7632         }
7633 
7634         case kWhatFlush:
7635         {
7636             // We haven't even started yet, so we're flushed alright...
7637             mCodec->mCallback->onFlushCompleted();
7638             return true;
7639         }
7640 
7641         case kWhatSignalEndOfInputStream:
7642         {
7643             mCodec->onSignalEndOfInputStream();
7644             return true;
7645         }
7646 
7647         default:
7648             return BaseState::onMessageReceived(msg);
7649     }
7650 }
7651 
onOMXEvent(OMX_EVENTTYPE event,OMX_U32 data1,OMX_U32 data2)7652 bool ACodec::IdleToExecutingState::onOMXEvent(
7653         OMX_EVENTTYPE event, OMX_U32 data1, OMX_U32 data2) {
7654     switch (event) {
7655         case OMX_EventCmdComplete:
7656         {
7657             if (data1 != (OMX_U32)OMX_CommandStateSet
7658                     || data2 != (OMX_U32)OMX_StateExecuting) {
7659                 ALOGE("Unexpected command completion in IdleToExecutingState: %s(%u) %s(%u)",
7660                         asString((OMX_COMMANDTYPE)data1), data1,
7661                         asString((OMX_STATETYPE)data2), data2);
7662                 mCodec->signalError(OMX_ErrorUndefined, FAILED_TRANSACTION);
7663                 return true;
7664             }
7665 
7666             mCodec->mExecutingState->resume();
7667             mCodec->changeState(mCodec->mExecutingState);
7668 
7669             return true;
7670         }
7671 
7672         default:
7673             return BaseState::onOMXEvent(event, data1, data2);
7674     }
7675 }
7676 
7677 ////////////////////////////////////////////////////////////////////////////////
7678 
ExecutingState(ACodec * codec)7679 ACodec::ExecutingState::ExecutingState(ACodec *codec)
7680     : BaseState(codec),
7681       mActive(false) {
7682 }
7683 
getPortMode(OMX_U32)7684 ACodec::BaseState::PortMode ACodec::ExecutingState::getPortMode(
7685         OMX_U32 /* portIndex */) {
7686     return RESUBMIT_BUFFERS;
7687 }
7688 
submitOutputMetaBuffers()7689 void ACodec::ExecutingState::submitOutputMetaBuffers() {
7690     // submit as many buffers as there are input buffers with the codec
7691     // in case we are in port reconfiguring
7692     for (size_t i = 0; i < mCodec->mBuffers[kPortIndexInput].size(); ++i) {
7693         BufferInfo *info = &mCodec->mBuffers[kPortIndexInput][i];
7694 
7695         if (info->mStatus == BufferInfo::OWNED_BY_COMPONENT) {
7696             if (mCodec->submitOutputMetadataBuffer() != OK)
7697                 break;
7698         }
7699     }
7700     if (mCodec->mIsLowLatency) {
7701         maybePostExtraOutputMetadataBufferRequest();
7702     }
7703 
7704     // *** NOTE: THE FOLLOWING WORKAROUND WILL BE REMOVED ***
7705     mCodec->signalSubmitOutputMetadataBufferIfEOS_workaround();
7706 }
7707 
submitRegularOutputBuffers()7708 void ACodec::ExecutingState::submitRegularOutputBuffers() {
7709     bool failed = false;
7710     for (size_t i = 0; i < mCodec->mBuffers[kPortIndexOutput].size(); ++i) {
7711         BufferInfo *info = &mCodec->mBuffers[kPortIndexOutput][i];
7712 
7713         if (mCodec->mNativeWindow != NULL) {
7714             if (info->mStatus != BufferInfo::OWNED_BY_US
7715                     && info->mStatus != BufferInfo::OWNED_BY_NATIVE_WINDOW) {
7716                 ALOGE("buffers should be owned by us or the surface");
7717                 failed = true;
7718                 break;
7719             }
7720 
7721             if (info->mStatus == BufferInfo::OWNED_BY_NATIVE_WINDOW) {
7722                 continue;
7723             }
7724         } else {
7725             if (info->mStatus != BufferInfo::OWNED_BY_US) {
7726                 ALOGE("buffers should be owned by us");
7727                 failed = true;
7728                 break;
7729             }
7730         }
7731 
7732         ALOGV("[%s] calling fillBuffer %u", mCodec->mComponentName.c_str(), info->mBufferID);
7733 
7734         info->checkWriteFence("submitRegularOutputBuffers");
7735         status_t err = mCodec->fillBuffer(info);
7736         if (err != OK) {
7737             failed = true;
7738             break;
7739         }
7740     }
7741 
7742     if (failed) {
7743         mCodec->signalError(OMX_ErrorUndefined, FAILED_TRANSACTION);
7744     }
7745 }
7746 
submitOutputBuffers()7747 void ACodec::ExecutingState::submitOutputBuffers() {
7748     submitRegularOutputBuffers();
7749     if (mCodec->storingMetadataInDecodedBuffers()) {
7750         submitOutputMetaBuffers();
7751     }
7752 }
7753 
resume()7754 void ACodec::ExecutingState::resume() {
7755     if (mActive) {
7756         ALOGV("[%s] We're already active, no need to resume.", mCodec->mComponentName.c_str());
7757         return;
7758     }
7759 
7760     submitOutputBuffers();
7761 
7762     // Post all available input buffers
7763     if (mCodec->mBuffers[kPortIndexInput].size() == 0u) {
7764         ALOGW("[%s] we don't have any input buffers to resume", mCodec->mComponentName.c_str());
7765     }
7766 
7767     for (size_t i = 0; i < mCodec->mBuffers[kPortIndexInput].size(); i++) {
7768         BufferInfo *info = &mCodec->mBuffers[kPortIndexInput][i];
7769         if (info->mStatus == BufferInfo::OWNED_BY_US) {
7770             postFillThisBuffer(info);
7771         }
7772     }
7773 
7774     mActive = true;
7775 }
7776 
stateEntered()7777 void ACodec::ExecutingState::stateEntered() {
7778     ALOGV("[%s] Now Executing", mCodec->mComponentName.c_str());
7779     mCodec->processDeferredMessages();
7780 }
7781 
onMessageReceived(const sp<AMessage> & msg)7782 bool ACodec::ExecutingState::onMessageReceived(const sp<AMessage> &msg) {
7783     bool handled = false;
7784 
7785     switch (msg->what()) {
7786         case kWhatShutdown:
7787         {
7788             int32_t keepComponentAllocated;
7789             CHECK(msg->findInt32(
7790                         "keepComponentAllocated", &keepComponentAllocated));
7791 
7792             mCodec->mShutdownInProgress = true;
7793             mCodec->mExplicitShutdown = true;
7794             mCodec->mKeepComponentAllocated = keepComponentAllocated;
7795 
7796             mActive = false;
7797 
7798             status_t err = mCodec->mOMXNode->sendCommand(
7799                     OMX_CommandStateSet, OMX_StateIdle);
7800             if (err != OK) {
7801                 if (keepComponentAllocated) {
7802                     mCodec->signalError(OMX_ErrorUndefined, FAILED_TRANSACTION);
7803                 }
7804                 // TODO: do some recovery here.
7805             } else {
7806                 mCodec->changeState(mCodec->mExecutingToIdleState);
7807             }
7808 
7809             handled = true;
7810             break;
7811         }
7812 
7813         case kWhatFlush:
7814         {
7815             ALOGV("[%s] ExecutingState flushing now "
7816                  "(codec owns %zu/%zu input, %zu/%zu output).",
7817                     mCodec->mComponentName.c_str(),
7818                     mCodec->countBuffersOwnedByComponent(kPortIndexInput),
7819                     mCodec->mBuffers[kPortIndexInput].size(),
7820                     mCodec->countBuffersOwnedByComponent(kPortIndexOutput),
7821                     mCodec->mBuffers[kPortIndexOutput].size());
7822 
7823             mActive = false;
7824 
7825             status_t err = mCodec->mOMXNode->sendCommand(OMX_CommandFlush, OMX_ALL);
7826             if (err != OK) {
7827                 mCodec->signalError(OMX_ErrorUndefined, FAILED_TRANSACTION);
7828             } else {
7829                 mCodec->changeState(mCodec->mFlushingState);
7830             }
7831 
7832             handled = true;
7833             break;
7834         }
7835 
7836         case kWhatResume:
7837         {
7838             resume();
7839 
7840             handled = true;
7841             break;
7842         }
7843 
7844         case kWhatRequestIDRFrame:
7845         {
7846             status_t err = mCodec->requestIDRFrame();
7847             if (err != OK) {
7848                 ALOGW("Requesting an IDR frame failed.");
7849             }
7850 
7851             handled = true;
7852             break;
7853         }
7854 
7855         case kWhatSetParameters:
7856         {
7857             sp<AMessage> params;
7858             CHECK(msg->findMessage("params", &params));
7859 
7860             status_t err = mCodec->setParameters(params);
7861 
7862             sp<AMessage> reply;
7863             if (msg->findMessage("reply", &reply)) {
7864                 reply->setInt32("err", err);
7865                 reply->post();
7866             }
7867 
7868             handled = true;
7869             break;
7870         }
7871 
7872         case ACodec::kWhatSignalEndOfInputStream:
7873         {
7874             mCodec->onSignalEndOfInputStream();
7875             handled = true;
7876             break;
7877         }
7878 
7879         // *** NOTE: THE FOLLOWING WORKAROUND WILL BE REMOVED ***
7880         case kWhatSubmitOutputMetadataBufferIfEOS:
7881         {
7882             if (mCodec->mPortEOS[kPortIndexInput] &&
7883                     !mCodec->mPortEOS[kPortIndexOutput]) {
7884                 status_t err = mCodec->submitOutputMetadataBuffer();
7885                 if (err == OK) {
7886                     mCodec->signalSubmitOutputMetadataBufferIfEOS_workaround();
7887                 }
7888             }
7889             handled = true;
7890             break;
7891         }
7892 
7893         case kWhatPollForRenderedBuffers:
7894         {
7895             mCodec->pollForRenderedFrames();
7896             handled = true;
7897             break;
7898         }
7899 
7900         default:
7901             handled = BaseState::onMessageReceived(msg);
7902             break;
7903     }
7904 
7905     return handled;
7906 }
7907 
setSurfaceParameters(const sp<AMessage> & params)7908 status_t ACodec::setSurfaceParameters(const sp<AMessage> &params) {
7909     int64_t timeOffsetUs;
7910     if (params->findInt64(PARAMETER_KEY_OFFSET_TIME, &timeOffsetUs)) {
7911         if (mGraphicBufferSource == NULL) {
7912             ALOGE("[%s] Invalid to set input buffer time offset without surface",
7913                     mComponentName.c_str());
7914             return INVALID_OPERATION;
7915         }
7916 
7917         status_t err = statusFromBinderStatus(
7918                 mGraphicBufferSource->setTimeOffsetUs(timeOffsetUs));
7919 
7920         if (err != OK) {
7921             ALOGE("[%s] Unable to set input buffer time offset (err %d)",
7922                 mComponentName.c_str(),
7923                 err);
7924             return err;
7925         }
7926     }
7927 
7928     int64_t skipFramesBeforeUs;
7929     if (params->findInt64("skip-frames-before", &skipFramesBeforeUs)) {
7930         if (mGraphicBufferSource == NULL) {
7931             ALOGE("[%s] Invalid to set start time without surface",
7932                     mComponentName.c_str());
7933             return INVALID_OPERATION;
7934         }
7935 
7936         status_t err = statusFromBinderStatus(
7937                 mGraphicBufferSource->setStartTimeUs(skipFramesBeforeUs));
7938 
7939         if (err != OK) {
7940             ALOGE("Failed to set parameter 'skip-frames-before' (err %d)", err);
7941             return err;
7942         }
7943     }
7944 
7945     int32_t dropInputFrames;
7946     if (params->findInt32(PARAMETER_KEY_SUSPEND, &dropInputFrames)) {
7947         if (mGraphicBufferSource == NULL) {
7948             ALOGE("[%s] Invalid to set suspend without surface",
7949                     mComponentName.c_str());
7950             return INVALID_OPERATION;
7951         }
7952 
7953         int64_t suspendStartTimeUs = -1;
7954         (void) params->findInt64(PARAMETER_KEY_SUSPEND_TIME, &suspendStartTimeUs);
7955         status_t err = statusFromBinderStatus(
7956                 mGraphicBufferSource->setSuspend(dropInputFrames != 0, suspendStartTimeUs));
7957 
7958         if (err != OK) {
7959             ALOGE("Failed to set parameter 'drop-input-frames' (err %d)", err);
7960             return err;
7961         }
7962     }
7963 
7964     int64_t stopTimeUs;
7965     if (params->findInt64("stop-time-us", &stopTimeUs)) {
7966         if (mGraphicBufferSource == NULL) {
7967             ALOGE("[%s] Invalid to set stop time without surface",
7968                     mComponentName.c_str());
7969             return INVALID_OPERATION;
7970         }
7971         status_t err = statusFromBinderStatus(
7972                 mGraphicBufferSource->setStopTimeUs(stopTimeUs));
7973 
7974         if (err != OK) {
7975             ALOGE("Failed to set parameter 'stop-time-us' (err %d)", err);
7976             return err;
7977         }
7978 
7979         int64_t stopTimeOffsetUs;
7980         hardware::Return<void> trans = mGraphicBufferSource->getStopTimeOffsetUs(
7981                 [&err, &stopTimeOffsetUs](auto status, auto result) {
7982                     err = static_cast<status_t>(status);
7983                     stopTimeOffsetUs = result;
7984                 });
7985         if (!trans.isOk()) {
7986             err = trans.isDeadObject() ? DEAD_OBJECT : UNKNOWN_ERROR;
7987         }
7988 
7989         if (err != OK) {
7990             ALOGE("Failed to get stop time offset (err %d)", err);
7991             return err;
7992         }
7993         mInputFormat->setInt64("android._stop-time-offset-us", stopTimeOffsetUs);
7994     }
7995 
7996     return OK;
7997 }
7998 
setParameters(const sp<AMessage> & params)7999 status_t ACodec::setParameters(const sp<AMessage> &params) {
8000     status_t err;
8001 
8002     int32_t videoBitrate;
8003     if (params->findInt32("video-bitrate", &videoBitrate)) {
8004         OMX_VIDEO_CONFIG_BITRATETYPE configParams;
8005         InitOMXParams(&configParams);
8006         configParams.nPortIndex = kPortIndexOutput;
8007         configParams.nEncodeBitrate = videoBitrate;
8008 
8009         err = mOMXNode->setConfig(
8010                 OMX_IndexConfigVideoBitrate,
8011                 &configParams,
8012                 sizeof(configParams));
8013 
8014         if (err != OK) {
8015             ALOGE("setConfig(OMX_IndexConfigVideoBitrate, %d) failed w/ err %d",
8016                    videoBitrate, err);
8017 
8018             return err;
8019         }
8020     }
8021 
8022     err = setSurfaceParameters(params);
8023     if (err != OK) {
8024         ALOGE("Failed to set input surface parameters (err %d)", err);
8025         return err;
8026     }
8027 
8028     int32_t tmp;
8029     if (params->findInt32("request-sync", &tmp)) {
8030         err = requestIDRFrame();
8031 
8032         if (err != OK) {
8033             ALOGE("Requesting a sync frame failed w/ err %d", err);
8034             return err;
8035         }
8036     }
8037 
8038     int32_t rateInt = -1;
8039     float rateFloat = -1;
8040     if (!params->findFloat("operating-rate", &rateFloat)) {
8041         params->findInt32("operating-rate", &rateInt);
8042         rateFloat = (float) rateInt; // 16MHz (FLINTMAX) is OK for upper bound.
8043     }
8044     if (rateFloat > 0) {
8045         err = setOperatingRate(rateFloat, mIsVideo);
8046         if (err != OK) {
8047             ALOGI("Failed to set parameter 'operating-rate' (err %d)", err);
8048         }
8049     }
8050 
8051     int32_t intraRefreshPeriod = 0;
8052     if (params->findInt32("intra-refresh-period", &intraRefreshPeriod)
8053             && intraRefreshPeriod > 0) {
8054         err = setIntraRefreshPeriod(intraRefreshPeriod, false);
8055         if (err != OK) {
8056             ALOGI("[%s] failed setIntraRefreshPeriod. Failure is fine since this key is optional",
8057                     mComponentName.c_str());
8058             err = OK;
8059         }
8060     }
8061 
8062     int32_t lowLatency = 0;
8063     if (params->findInt32("low-latency", &lowLatency)) {
8064         err = setLowLatency(lowLatency);
8065         if (err != OK) {
8066             return err;
8067         }
8068     }
8069 
8070     int32_t latency = 0;
8071     if (params->findInt32("latency", &latency) && latency > 0) {
8072         err = setLatency(latency);
8073         if (err != OK) {
8074             ALOGI("[%s] failed setLatency. Failure is fine since this key is optional",
8075                     mComponentName.c_str());
8076             err = OK;
8077         }
8078     }
8079 
8080     int32_t presentationId = -1;
8081     if (params->findInt32("audio-presentation-presentation-id", &presentationId)) {
8082         int32_t programId = -1;
8083         params->findInt32("audio-presentation-program-id", &programId);
8084         err = setAudioPresentation(presentationId, programId);
8085         if (err != OK) {
8086             ALOGI("[%s] failed setAudioPresentation. Failure is fine since this key is optional",
8087                     mComponentName.c_str());
8088             err = OK;
8089         }
8090     }
8091 
8092     sp<ABuffer> hdr10PlusInfo;
8093     if (params->findBuffer("hdr10-plus-info", &hdr10PlusInfo)
8094             && hdr10PlusInfo != nullptr && hdr10PlusInfo->size() > 0) {
8095         (void)setHdr10PlusInfo(hdr10PlusInfo);
8096     }
8097 
8098     // Ignore errors as failure is expected for codecs that aren't video encoders.
8099     (void)configureTemporalLayers(params, false /* inConfigure */, mOutputFormat);
8100 
8101     AString mime;
8102     if (!mIsEncoder
8103             && (mConfigFormat->findString("mime", &mime))
8104             && !strcasecmp(MEDIA_MIMETYPE_AUDIO_AAC, mime.c_str())) {
8105         OMX_AUDIO_PARAM_ANDROID_AACDRCPRESENTATIONTYPE presentation;
8106         InitOMXParams(&presentation);
8107         mOMXNode->getParameter(
8108                     (OMX_INDEXTYPE)OMX_IndexParamAudioAndroidAacDrcPresentation,
8109                     &presentation, sizeof(presentation));
8110         int32_t value32 = 0;
8111         bool updated = false;
8112         if (params->findInt32("aac-pcm-limiter-enable", &value32)) {
8113             presentation.nPCMLimiterEnable = value32;
8114             updated = true;
8115         }
8116         if (params->findInt32("aac-encoded-target-level", &value32)) {
8117             presentation.nEncodedTargetLevel = value32;
8118             updated = true;
8119         }
8120         if (params->findInt32("aac-drc-cut-level", &value32)) {
8121             presentation.nDrcCut = value32;
8122             updated = true;
8123         }
8124         if (params->findInt32("aac-drc-boost-level", &value32)) {
8125             presentation.nDrcBoost = value32;
8126             updated = true;
8127         }
8128         if (params->findInt32("aac-drc-heavy-compression", &value32)) {
8129             presentation.nHeavyCompression = value32;
8130             updated = true;
8131         }
8132         if (params->findInt32("aac-target-ref-level", &value32)) {
8133             presentation.nTargetReferenceLevel = value32;
8134             updated = true;
8135         }
8136         if (params->findInt32("aac-drc-effect-type", &value32)) {
8137             presentation.nDrcEffectType = value32;
8138             updated = true;
8139         }
8140         if (params->findInt32("aac-drc-album-mode", &value32)) {
8141             presentation.nDrcAlbumMode = value32;
8142             updated = true;
8143         }
8144         if (!params->findInt32("aac-drc-output-loudness", &value32)) {
8145             presentation.nDrcOutputLoudness = value32;
8146             updated = true;
8147         }
8148         if (updated) {
8149             mOMXNode->setParameter((OMX_INDEXTYPE)OMX_IndexParamAudioAndroidAacDrcPresentation,
8150                 &presentation, sizeof(presentation));
8151         }
8152     }
8153 
8154     {
8155         int32_t tunnelPeek = 0;
8156         if (params->findInt32(TUNNEL_PEEK_KEY, &tunnelPeek)) {
8157             err = setTunnelPeek(tunnelPeek);
8158             if (err != OK) {
8159                 return err;
8160             }
8161         }
8162     }
8163     {
8164         int32_t tunnelPeekSetLegacy = 0;
8165         if (params->findInt32(TUNNEL_PEEK_SET_LEGACY_KEY, &tunnelPeekSetLegacy)) {
8166             err = setTunnelPeekLegacy(tunnelPeekSetLegacy);
8167             if (err != OK) {
8168                 return err;
8169             }
8170         }
8171     }
8172 
8173     return setVendorParameters(params);
8174 }
8175 
setHdr10PlusInfo(const sp<ABuffer> & hdr10PlusInfo)8176 status_t ACodec::setHdr10PlusInfo(const sp<ABuffer> &hdr10PlusInfo) {
8177     if (mDescribeHDR10PlusInfoIndex == 0) {
8178         ALOGE("setHdr10PlusInfo: does not support DescribeHDR10PlusInfoParams");
8179         return ERROR_UNSUPPORTED;
8180     }
8181     size_t newSize = sizeof(DescribeHDR10PlusInfoParams) + hdr10PlusInfo->size() - 1;
8182     if (mHdr10PlusScratchBuffer == nullptr ||
8183             newSize > mHdr10PlusScratchBuffer->size()) {
8184         mHdr10PlusScratchBuffer = new ABuffer(newSize);
8185     }
8186     DescribeHDR10PlusInfoParams *config =
8187             (DescribeHDR10PlusInfoParams *)mHdr10PlusScratchBuffer->data();
8188     InitOMXParams(config);
8189     config->nPortIndex = 0;
8190     config->nSize = newSize;
8191     config->nParamSize = hdr10PlusInfo->size();
8192     config->nParamSizeUsed = hdr10PlusInfo->size();
8193     memcpy(config->nValue, hdr10PlusInfo->data(), hdr10PlusInfo->size());
8194     status_t err = mOMXNode->setConfig(
8195             (OMX_INDEXTYPE)mDescribeHDR10PlusInfoIndex,
8196             config, config->nSize);
8197     if (err != OK) {
8198         ALOGE("failed to set DescribeHDR10PlusInfoParams (err %d)", err);
8199     }
8200     return OK;
8201 }
8202 
8203 // Removes trailing tags matching |tag| from |key| (e.g. a settings name). |minLength| specifies
8204 // the minimum number of characters to keep in |key| (even if it has trailing tags).
8205 // (Used to remove trailing 'value' tags in settings names, e.g. to normalize
8206 // 'vendor.settingsX.value' to 'vendor.settingsX')
removeTrailingTags(char * key,size_t minLength,const char * tag)8207 static void removeTrailingTags(char *key, size_t minLength, const char *tag) {
8208     size_t length = strlen(key);
8209     size_t tagLength = strlen(tag);
8210     while (length > minLength + tagLength
8211             && !strcmp(key + length - tagLength, tag)
8212             && key[length - tagLength - 1] == '.') {
8213         length -= tagLength + 1;
8214         key[length] = '\0';
8215     }
8216 }
8217 
8218 /**
8219  * Struct encompassing a vendor extension config structure and a potential error status (in case
8220  * the structure is null). Used to iterate through vendor extensions.
8221  */
8222 struct VendorExtension {
8223     OMX_CONFIG_ANDROID_VENDOR_EXTENSIONTYPE *config;  // structure does not own config
8224     status_t status;
8225 
8226     // create based on an error status
VendorExtensionandroid::VendorExtension8227     VendorExtension(status_t s_ = NO_INIT) : config(nullptr), status(s_) { }
8228 
8229     // create based on a successfully retrieved config structure
VendorExtensionandroid::VendorExtension8230     VendorExtension(OMX_CONFIG_ANDROID_VENDOR_EXTENSIONTYPE *c_) : config(c_), status(OK) { }
8231 };
8232 
8233 // class VendorExtensions;
8234 /**
8235  * Forward iterator to enumerate vendor extensions supported by an OMX component.
8236  */
8237 class VendorExtensionIterator {
8238 //private:
8239     static constexpr size_t kLastIndex = ~(size_t)0; // last index marker
8240 
8241     sp<IOMXNode> mNode;                   // component
8242     size_t mIndex;                        // current android extension index
8243     std::unique_ptr<uint8_t[]> mBacking;  // current extension's backing
8244     VendorExtension mCurrent;             // current extension
8245 
VendorExtensionIterator(const sp<IOMXNode> & node,size_t index)8246     VendorExtensionIterator(const sp<IOMXNode> &node, size_t index)
8247         : mNode(node),
8248           mIndex(index) {
8249         mCurrent = retrieve();
8250     }
8251 
8252     friend class VendorExtensions;
8253 
8254 public:
8255     // copy constructor
VendorExtensionIterator(const VendorExtensionIterator & it)8256     VendorExtensionIterator(const VendorExtensionIterator &it)
8257         : VendorExtensionIterator(it.mNode, it.mIndex) { }
8258 
8259     // retrieves the current extension pointed to by this iterator
retrieve()8260     VendorExtension retrieve() {
8261         if (mIndex == kLastIndex) {
8262             return NO_INIT;
8263         }
8264 
8265         // try with one param first, then retry if extension needs more than 1 param
8266         for (size_t paramSizeUsed = 1;; ) {
8267             if (paramSizeUsed > OMX_MAX_ANDROID_VENDOR_PARAMCOUNT) {
8268                 return BAD_VALUE; // this prevents overflow in the following formula
8269             }
8270 
8271             size_t size = sizeof(OMX_CONFIG_ANDROID_VENDOR_EXTENSIONTYPE) +
8272                 (paramSizeUsed - 1) * sizeof(OMX_CONFIG_ANDROID_VENDOR_EXTENSIONTYPE::param);
8273             mBacking.reset(new uint8_t[size]);
8274             if (!mBacking) {
8275                 return NO_MEMORY;
8276             }
8277 
8278             OMX_CONFIG_ANDROID_VENDOR_EXTENSIONTYPE *config =
8279                 reinterpret_cast<OMX_CONFIG_ANDROID_VENDOR_EXTENSIONTYPE *>(mBacking.get());
8280 
8281             InitOMXParams(config);
8282             config->nSize = size;
8283             config->nIndex = mIndex;
8284             config->nParamSizeUsed = paramSizeUsed;
8285             status_t err = mNode->getConfig(
8286                     (OMX_INDEXTYPE)OMX_IndexConfigAndroidVendorExtension, config, size);
8287             if (err == OK && config->nParamCount > paramSizeUsed && paramSizeUsed == 1) {
8288                 // reallocate if we need a bigger config
8289                 paramSizeUsed = config->nParamCount;
8290                 continue;
8291             } else if (err == NOT_ENOUGH_DATA
8292                    || (err != OK && mIndex == 0)) {
8293                 // stop iterator on no-more signal, or if index is not at all supported
8294                 mIndex = kLastIndex;
8295                 return NO_INIT;
8296             } else if (err != OK) {
8297                 return err;
8298             } else if (paramSizeUsed != config->nParamSizeUsed) {
8299                 return BAD_VALUE; // component shall not modify size of nParam
8300             }
8301 
8302             return config;
8303         }
8304     }
8305 
8306     // returns extension pointed to by this iterator
operator *()8307     VendorExtension operator*() {
8308         return mCurrent;
8309     }
8310 
8311     // prefix increment: move to next extension
operator ++()8312     VendorExtensionIterator &operator++() { // prefix
8313         if (mIndex != kLastIndex) {
8314             ++mIndex;
8315             mCurrent = retrieve();
8316         }
8317         return *this;
8318     }
8319 
8320     // iterator equality operators
operator ==(const VendorExtensionIterator & o)8321     bool operator==(const VendorExtensionIterator &o) {
8322         return mNode == o.mNode && mIndex == o.mIndex;
8323     }
8324 
operator !=(const VendorExtensionIterator & o)8325     bool operator!=(const VendorExtensionIterator &o) {
8326         return !(*this == o);
8327     }
8328 };
8329 
8330 /**
8331  * Iterable container for vendor extensions provided by a component
8332  */
8333 class VendorExtensions {
8334 //private:
8335     sp<IOMXNode> mNode;
8336 
8337 public:
VendorExtensions(const sp<IOMXNode> & node)8338     VendorExtensions(const sp<IOMXNode> &node)
8339         : mNode(node) {
8340     }
8341 
begin()8342     VendorExtensionIterator begin() {
8343         return VendorExtensionIterator(mNode, 0);
8344     }
8345 
end()8346     VendorExtensionIterator end() {
8347         return VendorExtensionIterator(mNode, VendorExtensionIterator::kLastIndex);
8348     }
8349 };
8350 
setVendorParameters(const sp<AMessage> & params)8351 status_t ACodec::setVendorParameters(const sp<AMessage> &params) {
8352     std::map<std::string, std::string> vendorKeys; // maps reduced name to actual name
8353     constexpr char prefix[] = "vendor.";
8354     constexpr size_t prefixLength = sizeof(prefix) - 1;
8355     // longest possible vendor param name
8356     char reducedKey[OMX_MAX_STRINGNAME_SIZE + OMX_MAX_STRINGVALUE_SIZE];
8357 
8358     // identify all vendor keys to speed up search later and to detect vendor keys
8359     for (size_t i = params->countEntries(); i; --i) {
8360         AMessage::Type keyType;
8361         const char* key = params->getEntryNameAt(i - 1, &keyType);
8362         if (key != nullptr && !strncmp(key, prefix, prefixLength)
8363                 // it is safe to limit format keys to the max vendor param size as we only
8364                 // shorten parameter names by removing any trailing 'value' tags, and we
8365                 // already remove the vendor prefix.
8366                 && strlen(key + prefixLength) < sizeof(reducedKey)
8367                 && (keyType == AMessage::kTypeInt32
8368                         || keyType == AMessage::kTypeInt64
8369                         || keyType == AMessage::kTypeString)) {
8370             strcpy(reducedKey, key + prefixLength);
8371             removeTrailingTags(reducedKey, 0, "value");
8372             auto existingKey = vendorKeys.find(reducedKey);
8373             if (existingKey != vendorKeys.end()) {
8374                 ALOGW("[%s] vendor parameter '%s' aliases parameter '%s'",
8375                         mComponentName.c_str(), key, existingKey->second.c_str());
8376                 // ignore for now
8377             }
8378             vendorKeys.emplace(reducedKey, key);
8379         }
8380     }
8381 
8382     // don't bother component if we don't have vendor extensions as they may not have implemented
8383     // the android vendor extension support, which will lead to unnecessary OMX failure logs.
8384     if (vendorKeys.empty()) {
8385         return OK;
8386     }
8387 
8388     char key[sizeof(OMX_CONFIG_ANDROID_VENDOR_EXTENSIONTYPE::cName) +
8389             sizeof(OMX_CONFIG_ANDROID_VENDOR_PARAMTYPE::cKey)];
8390 
8391     status_t finalError = OK;
8392 
8393     // don't try again if component does not have vendor extensions
8394     if (mVendorExtensionsStatus == kExtensionsNone) {
8395         return OK;
8396     }
8397 
8398     for (VendorExtension ext : VendorExtensions(mOMXNode)) {
8399         OMX_CONFIG_ANDROID_VENDOR_EXTENSIONTYPE *config = ext.config;
8400         if (config == nullptr) {
8401             return ext.status;
8402         }
8403 
8404         mVendorExtensionsStatus = kExtensionsExist;
8405 
8406         config->cName[sizeof(config->cName) - 1] = '\0'; // null-terminate name
8407         strcpy(key, (const char *)config->cName);
8408         size_t nameLength = strlen(key);
8409         key[nameLength] = '.';
8410 
8411         // don't set vendor extension if client has not provided any of its parameters
8412         // or if client simply unsets parameters that are already unset
8413         bool needToSet = false;
8414         for (size_t paramIndex = 0; paramIndex < config->nParamCount; ++paramIndex) {
8415             // null-terminate param key
8416             config->param[paramIndex].cKey[sizeof(config->param[0].cKey) - 1] = '\0';
8417             strcpy(key + nameLength + 1, (const char *)config->param[paramIndex].cKey);
8418             removeTrailingTags(key, nameLength, "value");
8419             auto existingKey = vendorKeys.find(key);
8420 
8421             // don't touch (e.g. change) parameters that are not specified by client
8422             if (existingKey == vendorKeys.end()) {
8423                 continue;
8424             }
8425 
8426             bool wasSet = config->param[paramIndex].bSet;
8427             switch (config->param[paramIndex].eValueType) {
8428             case OMX_AndroidVendorValueInt32:
8429             {
8430                 int32_t value;
8431                 config->param[paramIndex].bSet =
8432                     (OMX_BOOL)params->findInt32(existingKey->second.c_str(), &value);
8433                 if (config->param[paramIndex].bSet) {
8434                     config->param[paramIndex].nInt32 = value;
8435                 }
8436                 break;
8437             }
8438             case OMX_AndroidVendorValueInt64:
8439             {
8440                 int64_t value;
8441                 config->param[paramIndex].bSet =
8442                     (OMX_BOOL)params->findAsInt64(existingKey->second.c_str(), &value);
8443                 if (config->param[paramIndex].bSet) {
8444                     config->param[paramIndex].nInt64 = value;
8445                 }
8446                 break;
8447             }
8448             case OMX_AndroidVendorValueString:
8449             {
8450                 AString value;
8451                 config->param[paramIndex].bSet =
8452                     (OMX_BOOL)params->findString(existingKey->second.c_str(), &value);
8453                 if (config->param[paramIndex].bSet) {
8454                     size_t dstSize = sizeof(config->param[paramIndex].cString);
8455                     strncpy((char *)config->param[paramIndex].cString, value.c_str(), dstSize - 1);
8456                     // null terminate value
8457                     config->param[paramIndex].cString[dstSize - 1] = '\0';
8458                 }
8459                 break;
8460             }
8461             default:
8462                 ALOGW("[%s] vendor parameter '%s' is not a supported value",
8463                         mComponentName.c_str(), key);
8464                 continue;
8465             }
8466             if (config->param[paramIndex].bSet || wasSet) {
8467                 needToSet = true;
8468             }
8469         }
8470 
8471         if (needToSet) {
8472             status_t err = mOMXNode->setConfig(
8473                     (OMX_INDEXTYPE)OMX_IndexConfigAndroidVendorExtension,
8474                     config, config->nSize);
8475             if (err != OK) {
8476                 key[nameLength] = '\0';
8477                 ALOGW("[%s] failed to set vendor extension '%s'", mComponentName.c_str(), key);
8478                 // try to set each extension, and return first failure
8479                 if (finalError == OK) {
8480                     finalError = err;
8481                 }
8482             }
8483         }
8484     }
8485 
8486     if (mVendorExtensionsStatus == kExtensionsUnchecked) {
8487         mVendorExtensionsStatus = kExtensionsNone;
8488     }
8489 
8490     return finalError;
8491 }
8492 
getVendorParameters(OMX_U32 portIndex,sp<AMessage> & format)8493 status_t ACodec::getVendorParameters(OMX_U32 portIndex, sp<AMessage> &format) {
8494     constexpr char prefix[] = "vendor.";
8495     constexpr size_t prefixLength = sizeof(prefix) - 1;
8496     char key[sizeof(OMX_CONFIG_ANDROID_VENDOR_EXTENSIONTYPE::cName) +
8497             sizeof(OMX_CONFIG_ANDROID_VENDOR_PARAMTYPE::cKey) + prefixLength];
8498     strcpy(key, prefix);
8499 
8500     // don't try again if component does not have vendor extensions
8501     if (mVendorExtensionsStatus == kExtensionsNone) {
8502         return OK;
8503     }
8504 
8505     for (VendorExtension ext : VendorExtensions(mOMXNode)) {
8506         OMX_CONFIG_ANDROID_VENDOR_EXTENSIONTYPE *config = ext.config;
8507         if (config == nullptr) {
8508             return ext.status;
8509         }
8510 
8511         mVendorExtensionsStatus = kExtensionsExist;
8512 
8513         if (config->eDir != (portIndex == kPortIndexInput ? OMX_DirInput : OMX_DirOutput)) {
8514             continue;
8515         }
8516 
8517         config->cName[sizeof(config->cName) - 1] = '\0'; // null-terminate name
8518         strcpy(key + prefixLength, (const char *)config->cName);
8519         size_t nameLength = strlen(key);
8520         key[nameLength] = '.';
8521 
8522         for (size_t paramIndex = 0; paramIndex < config->nParamCount; ++paramIndex) {
8523             // null-terminate param key
8524             config->param[paramIndex].cKey[sizeof(config->param[0].cKey) - 1] = '\0';
8525             strcpy(key + nameLength + 1, (const char *)config->param[paramIndex].cKey);
8526             removeTrailingTags(key, nameLength, "value");
8527             if (config->param[paramIndex].bSet) {
8528                 switch (config->param[paramIndex].eValueType) {
8529                 case OMX_AndroidVendorValueInt32:
8530                 {
8531                     format->setInt32(key, config->param[paramIndex].nInt32);
8532                     break;
8533                 }
8534                 case OMX_AndroidVendorValueInt64:
8535                 {
8536                     format->setInt64(key, config->param[paramIndex].nInt64);
8537                     break;
8538                 }
8539                 case OMX_AndroidVendorValueString:
8540                 {
8541                     config->param[paramIndex].cString[OMX_MAX_STRINGVALUE_SIZE - 1] = '\0';
8542                     format->setString(key, (const char *)config->param[paramIndex].cString);
8543                     break;
8544                 }
8545                 default:
8546                     ALOGW("vendor parameter %s is not a supported value", key);
8547                     continue;
8548                 }
8549             }
8550         }
8551     }
8552 
8553     if (mVendorExtensionsStatus == kExtensionsUnchecked) {
8554         mVendorExtensionsStatus = kExtensionsNone;
8555     }
8556 
8557     return OK;
8558 }
8559 
onSignalEndOfInputStream()8560 void ACodec::onSignalEndOfInputStream() {
8561     status_t err = INVALID_OPERATION;
8562     if (mGraphicBufferSource != NULL) {
8563         err = statusFromBinderStatus(mGraphicBufferSource->signalEndOfInputStream());
8564     }
8565     mCallback->onSignaledInputEOS(err);
8566 }
8567 
forceStateTransition(int generation)8568 void ACodec::forceStateTransition(int generation) {
8569     if (generation != mStateGeneration) {
8570         ALOGV("Ignoring stale force state transition message: #%d (now #%d)",
8571                 generation, mStateGeneration);
8572         return;
8573     }
8574     ALOGE("State machine stuck");
8575     // Error must have already been signalled to the client.
8576 
8577     // Deferred messages will be handled at LoadedState at the end of the
8578     // transition.
8579     mShutdownInProgress = true;
8580     // No shutdown complete callback at the end of the transition.
8581     mExplicitShutdown = false;
8582     mKeepComponentAllocated = true;
8583 
8584     status_t err = mOMXNode->sendCommand(OMX_CommandStateSet, OMX_StateIdle);
8585     if (err != OK) {
8586         // TODO: do some recovery here.
8587     } else {
8588         changeState(mExecutingToIdleState);
8589     }
8590 }
8591 
onOMXFrameRendered(int64_t mediaTimeUs,nsecs_t systemNano)8592 bool ACodec::ExecutingState::onOMXFrameRendered(int64_t mediaTimeUs, nsecs_t systemNano) {
8593     mCodec->mCallback->onOutputFramesRendered({RenderedFrameInfo(mediaTimeUs, systemNano)});
8594     return true;
8595 }
8596 
onOMXEvent(OMX_EVENTTYPE event,OMX_U32 data1,OMX_U32 data2)8597 bool ACodec::ExecutingState::onOMXEvent(
8598         OMX_EVENTTYPE event, OMX_U32 data1, OMX_U32 data2) {
8599     switch (event) {
8600         case OMX_EventPortSettingsChanged:
8601         {
8602             CHECK_EQ(data1, (OMX_U32)kPortIndexOutput);
8603 
8604             mCodec->onOutputFormatChanged();
8605 
8606             if (data2 == 0 || data2 == OMX_IndexParamPortDefinition) {
8607                 mCodec->mMetadataBuffersToSubmit = 0;
8608                 CHECK_EQ(mCodec->mOMXNode->sendCommand(
8609                             OMX_CommandPortDisable, kPortIndexOutput),
8610                          (status_t)OK);
8611 
8612                 mCodec->freeOutputBuffersNotOwnedByComponent();
8613 
8614                 mCodec->changeState(mCodec->mOutputPortSettingsChangedState);
8615             } else if (data2 != OMX_IndexConfigCommonOutputCrop
8616                     && data2 != OMX_IndexConfigAndroidIntraRefresh) {
8617                 ALOGV("[%s] OMX_EventPortSettingsChanged 0x%08x",
8618                      mCodec->mComponentName.c_str(), data2);
8619             }
8620 
8621             return true;
8622         }
8623 
8624         case OMX_EventConfigUpdate:
8625         {
8626             CHECK_EQ(data1, (OMX_U32)kPortIndexOutput);
8627 
8628             mCodec->onConfigUpdate((OMX_INDEXTYPE)data2);
8629 
8630             return true;
8631         }
8632 
8633         case OMX_EventBufferFlag:
8634         {
8635             return true;
8636         }
8637 
8638         case OMX_EventOnFirstTunnelFrameReady:
8639         {
8640             mCodec->onFirstTunnelFrameReady();
8641             return true;
8642         }
8643 
8644         default:
8645             return BaseState::onOMXEvent(event, data1, data2);
8646     }
8647 }
8648 
8649 ////////////////////////////////////////////////////////////////////////////////
8650 
OutputPortSettingsChangedState(ACodec * codec)8651 ACodec::OutputPortSettingsChangedState::OutputPortSettingsChangedState(
8652         ACodec *codec)
8653     : BaseState(codec) {
8654 }
8655 
getPortMode(OMX_U32 portIndex)8656 ACodec::BaseState::PortMode ACodec::OutputPortSettingsChangedState::getPortMode(
8657         OMX_U32 portIndex) {
8658     if (portIndex == kPortIndexOutput) {
8659         return FREE_BUFFERS;
8660     }
8661 
8662     CHECK_EQ(portIndex, (OMX_U32)kPortIndexInput);
8663 
8664     return RESUBMIT_BUFFERS;
8665 }
8666 
onMessageReceived(const sp<AMessage> & msg)8667 bool ACodec::OutputPortSettingsChangedState::onMessageReceived(
8668         const sp<AMessage> &msg) {
8669     bool handled = false;
8670 
8671     switch (msg->what()) {
8672         case kWhatFlush:
8673         case kWhatShutdown: {
8674             if (mCodec->mFatalError) {
8675                 sp<AMessage> msg = new AMessage(ACodec::kWhatForceStateTransition, mCodec);
8676                 msg->setInt32("generation", mCodec->mStateGeneration);
8677                 msg->post(3000000);
8678             }
8679             FALLTHROUGH_INTENDED;
8680         }
8681         case kWhatResume:
8682         {
8683             ALOGV("[%s] Deferring resume", mCodec->mComponentName.c_str());
8684 
8685             mCodec->deferMessage(msg);
8686             handled = true;
8687             break;
8688         }
8689 
8690         case kWhatSetParameters:
8691         {
8692             sp<AMessage> params;
8693             CHECK(msg->findMessage("params", &params));
8694 
8695             sp<ABuffer> hdr10PlusInfo;
8696             if (params->findBuffer("hdr10-plus-info", &hdr10PlusInfo)) {
8697                 if (hdr10PlusInfo != nullptr && hdr10PlusInfo->size() > 0) {
8698                     (void)mCodec->setHdr10PlusInfo(hdr10PlusInfo);
8699                 }
8700                 params->removeEntryAt(params->findEntryByName("hdr10-plus-info"));
8701 
8702                 if (params->countEntries() == 0) {
8703                     msg->removeEntryAt(msg->findEntryByName("params"));
8704                 }
8705             }
8706 
8707             if (msg->countEntries() > 0) {
8708                 mCodec->deferMessage(msg);
8709             }
8710             handled = true;
8711             break;
8712         }
8713 
8714         case kWhatForceStateTransition:
8715         {
8716             int32_t generation = 0;
8717             CHECK(msg->findInt32("generation", &generation));
8718             mCodec->forceStateTransition(generation);
8719 
8720             handled = true;
8721             break;
8722         }
8723 
8724         case kWhatSetSurface:
8725         {
8726             ALOGD("[%s] Deferring setSurface from OutputPortSettingsChangedState",
8727                   mCodec->mComponentName.c_str());
8728 
8729             mCodec->deferMessage(msg);
8730 
8731             handled = true;
8732             break;
8733         }
8734 
8735         case kWhatCheckIfStuck:
8736         {
8737             int32_t generation = 0;
8738             CHECK(msg->findInt32("generation", &generation));
8739             if (generation == mCodec->mStateGeneration) {
8740                 mCodec->signalError(OMX_ErrorUndefined, TIMED_OUT);
8741             }
8742 
8743             handled = true;
8744             break;
8745         }
8746 
8747         default:
8748             handled = BaseState::onMessageReceived(msg);
8749             break;
8750     }
8751 
8752     return handled;
8753 }
8754 
stateEntered()8755 void ACodec::OutputPortSettingsChangedState::stateEntered() {
8756     ALOGV("[%s] Now handling output port settings change",
8757          mCodec->mComponentName.c_str());
8758 
8759     // If we haven't transitioned after 3 seconds, we're probably stuck.
8760     sp<AMessage> msg = new AMessage(ACodec::kWhatCheckIfStuck, mCodec);
8761     msg->setInt32("generation", mCodec->mStateGeneration);
8762     msg->post(3000000);
8763 }
8764 
onOMXFrameRendered(int64_t mediaTimeUs,nsecs_t systemNano)8765 bool ACodec::OutputPortSettingsChangedState::onOMXFrameRendered(
8766         int64_t mediaTimeUs, nsecs_t systemNano) {
8767     mCodec->mCallback->onOutputFramesRendered({RenderedFrameInfo(mediaTimeUs, systemNano)});
8768     return true;
8769 }
8770 
onOMXEvent(OMX_EVENTTYPE event,OMX_U32 data1,OMX_U32 data2)8771 bool ACodec::OutputPortSettingsChangedState::onOMXEvent(
8772         OMX_EVENTTYPE event, OMX_U32 data1, OMX_U32 data2) {
8773     switch (event) {
8774         case OMX_EventCmdComplete:
8775         {
8776             if (data1 == (OMX_U32)OMX_CommandPortDisable) {
8777                 if (data2 != (OMX_U32)kPortIndexOutput) {
8778                     ALOGW("ignoring EventCmdComplete CommandPortDisable for port %u", data2);
8779                     return false;
8780                 }
8781 
8782                 ALOGV("[%s] Output port now disabled.", mCodec->mComponentName.c_str());
8783 
8784                 status_t err = OK;
8785                 if (!mCodec->mBuffers[kPortIndexOutput].empty()) {
8786                     ALOGE("disabled port should be empty, but has %zu buffers",
8787                             mCodec->mBuffers[kPortIndexOutput].size());
8788                     err = FAILED_TRANSACTION;
8789                 } else {
8790                     mCodec->mAllocator[kPortIndexOutput].clear();
8791                 }
8792 
8793                 if (err == OK) {
8794                     err = mCodec->mOMXNode->sendCommand(
8795                             OMX_CommandPortEnable, kPortIndexOutput);
8796                 }
8797 
8798                 if (err == OK) {
8799                     err = mCodec->allocateBuffersOnPort(kPortIndexOutput);
8800                     ALOGE_IF(err != OK, "Failed to allocate output port buffers after port "
8801                             "reconfiguration: (%d)", err);
8802                     mCodec->mCallback->onOutputBuffersChanged();
8803                 }
8804 
8805                 if (err != OK) {
8806                     mCodec->signalError(OMX_ErrorUndefined, makeNoSideEffectStatus(err));
8807                     ALOGE("Error occurred while disabling the output port");
8808                 }
8809 
8810                 return true;
8811             } else if (data1 == (OMX_U32)OMX_CommandPortEnable) {
8812                 if (data2 != (OMX_U32)kPortIndexOutput) {
8813                     ALOGW("ignoring EventCmdComplete OMX_CommandPortEnable for port %u", data2);
8814                     return false;
8815                 }
8816 
8817                 ALOGV("[%s] Output port now reenabled.", mCodec->mComponentName.c_str());
8818 
8819                 if (mCodec->mExecutingState->active()) {
8820                     mCodec->mExecutingState->submitOutputBuffers();
8821                 }
8822 
8823                 mCodec->changeState(mCodec->mExecutingState);
8824 
8825                 return true;
8826             }
8827 
8828             return false;
8829         }
8830 
8831         case OMX_EventConfigUpdate:
8832         {
8833             CHECK_EQ(data1, (OMX_U32)kPortIndexOutput);
8834 
8835             mCodec->onConfigUpdate((OMX_INDEXTYPE)data2);
8836 
8837             return true;
8838         }
8839 
8840         default:
8841             return BaseState::onOMXEvent(event, data1, data2);
8842     }
8843 }
8844 
8845 ////////////////////////////////////////////////////////////////////////////////
8846 
ExecutingToIdleState(ACodec * codec)8847 ACodec::ExecutingToIdleState::ExecutingToIdleState(ACodec *codec)
8848     : BaseState(codec),
8849       mComponentNowIdle(false) {
8850 }
8851 
onMessageReceived(const sp<AMessage> & msg)8852 bool ACodec::ExecutingToIdleState::onMessageReceived(const sp<AMessage> &msg) {
8853     bool handled = false;
8854 
8855     switch (msg->what()) {
8856         case kWhatFlush:
8857         {
8858             // Don't send me a flush request if you previously wanted me
8859             // to shutdown.
8860             ALOGW("Ignoring flush request in ExecutingToIdleState");
8861             break;
8862         }
8863 
8864         case kWhatShutdown:
8865         {
8866             mCodec->deferMessage(msg);
8867             handled = true;
8868             break;
8869         }
8870 
8871         default:
8872             handled = BaseState::onMessageReceived(msg);
8873             break;
8874     }
8875 
8876     return handled;
8877 }
8878 
stateEntered()8879 void ACodec::ExecutingToIdleState::stateEntered() {
8880     ALOGV("[%s] Now Executing->Idle", mCodec->mComponentName.c_str());
8881 
8882     mComponentNowIdle = false;
8883     mCodec->mLastOutputFormat.clear();
8884 }
8885 
onOMXEvent(OMX_EVENTTYPE event,OMX_U32 data1,OMX_U32 data2)8886 bool ACodec::ExecutingToIdleState::onOMXEvent(
8887         OMX_EVENTTYPE event, OMX_U32 data1, OMX_U32 data2) {
8888     switch (event) {
8889         case OMX_EventCmdComplete:
8890         {
8891             if (data1 != (OMX_U32)OMX_CommandStateSet
8892                     || data2 != (OMX_U32)OMX_StateIdle) {
8893                 ALOGE("Unexpected command completion in ExecutingToIdleState: %s(%u) %s(%u)",
8894                         asString((OMX_COMMANDTYPE)data1), data1,
8895                         asString((OMX_STATETYPE)data2), data2);
8896                 mCodec->signalError(OMX_ErrorUndefined, FAILED_TRANSACTION);
8897                 return true;
8898             }
8899 
8900             mComponentNowIdle = true;
8901 
8902             changeStateIfWeOwnAllBuffers();
8903 
8904             return true;
8905         }
8906 
8907         case OMX_EventPortSettingsChanged:
8908         case OMX_EventBufferFlag:
8909         {
8910             // We're shutting down and don't care about this anymore.
8911             return true;
8912         }
8913 
8914         default:
8915             return BaseState::onOMXEvent(event, data1, data2);
8916     }
8917 }
8918 
changeStateIfWeOwnAllBuffers()8919 void ACodec::ExecutingToIdleState::changeStateIfWeOwnAllBuffers() {
8920     if (mComponentNowIdle && mCodec->allYourBuffersAreBelongToUs()) {
8921         status_t err = mCodec->mOMXNode->sendCommand(
8922                 OMX_CommandStateSet, OMX_StateLoaded);
8923         if (err == OK) {
8924             err = mCodec->freeBuffersOnPort(kPortIndexInput);
8925             status_t err2 = mCodec->freeBuffersOnPort(kPortIndexOutput);
8926             if (err == OK) {
8927                 err = err2;
8928             }
8929         }
8930 
8931         if ((mCodec->mFlags & kFlagPushBlankBuffersToNativeWindowOnShutdown)
8932                 && mCodec->mNativeWindow != NULL) {
8933             // We push enough 1x1 blank buffers to ensure that one of
8934             // them has made it to the display.  This allows the OMX
8935             // component teardown to zero out any protected buffers
8936             // without the risk of scanning out one of those buffers.
8937             pushBlankBuffersToNativeWindow(mCodec->mNativeWindow.get());
8938         }
8939 
8940         if (err != OK) {
8941             mCodec->signalError(OMX_ErrorUndefined, FAILED_TRANSACTION);
8942             return;
8943         }
8944 
8945         mCodec->changeState(mCodec->mIdleToLoadedState);
8946     }
8947 }
8948 
onInputBufferFilled(const sp<AMessage> & msg)8949 void ACodec::ExecutingToIdleState::onInputBufferFilled(
8950         const sp<AMessage> &msg) {
8951     BaseState::onInputBufferFilled(msg);
8952 
8953     changeStateIfWeOwnAllBuffers();
8954 }
8955 
onOutputBufferDrained(const sp<AMessage> & msg)8956 void ACodec::ExecutingToIdleState::onOutputBufferDrained(
8957         const sp<AMessage> &msg) {
8958     BaseState::onOutputBufferDrained(msg);
8959 
8960     changeStateIfWeOwnAllBuffers();
8961 }
8962 
8963 ////////////////////////////////////////////////////////////////////////////////
8964 
IdleToLoadedState(ACodec * codec)8965 ACodec::IdleToLoadedState::IdleToLoadedState(ACodec *codec)
8966     : BaseState(codec) {
8967 }
8968 
onMessageReceived(const sp<AMessage> & msg)8969 bool ACodec::IdleToLoadedState::onMessageReceived(const sp<AMessage> &msg) {
8970     bool handled = false;
8971 
8972     switch (msg->what()) {
8973         case kWhatShutdown:
8974         {
8975             mCodec->deferMessage(msg);
8976             handled = true;
8977             break;
8978         }
8979 
8980         case kWhatFlush:
8981         {
8982             // Don't send me a flush request if you previously wanted me
8983             // to shutdown.
8984             ALOGE("Got flush request in IdleToLoadedState");
8985             break;
8986         }
8987 
8988         default:
8989             handled = BaseState::onMessageReceived(msg);
8990             break;
8991     }
8992 
8993     return handled;
8994 }
8995 
stateEntered()8996 void ACodec::IdleToLoadedState::stateEntered() {
8997     ALOGV("[%s] Now Idle->Loaded", mCodec->mComponentName.c_str());
8998 }
8999 
onOMXEvent(OMX_EVENTTYPE event,OMX_U32 data1,OMX_U32 data2)9000 bool ACodec::IdleToLoadedState::onOMXEvent(
9001         OMX_EVENTTYPE event, OMX_U32 data1, OMX_U32 data2) {
9002     switch (event) {
9003         case OMX_EventCmdComplete:
9004         {
9005             if (data1 != (OMX_U32)OMX_CommandStateSet
9006                     || data2 != (OMX_U32)OMX_StateLoaded) {
9007                 ALOGE("Unexpected command completion in IdleToLoadedState: %s(%u) %s(%u)",
9008                         asString((OMX_COMMANDTYPE)data1), data1,
9009                         asString((OMX_STATETYPE)data2), data2);
9010                 mCodec->signalError(OMX_ErrorUndefined, FAILED_TRANSACTION);
9011                 return true;
9012             }
9013 
9014             mCodec->changeState(mCodec->mLoadedState);
9015 
9016             return true;
9017         }
9018 
9019         default:
9020             return BaseState::onOMXEvent(event, data1, data2);
9021     }
9022 }
9023 
9024 ////////////////////////////////////////////////////////////////////////////////
9025 
FlushingState(ACodec * codec)9026 ACodec::FlushingState::FlushingState(ACodec *codec)
9027     : BaseState(codec) {
9028 }
9029 
stateEntered()9030 void ACodec::FlushingState::stateEntered() {
9031     ALOGV("[%s] Now Flushing", mCodec->mComponentName.c_str());
9032 
9033     mFlushComplete[kPortIndexInput] = mFlushComplete[kPortIndexOutput] = false;
9034     mCodec->mDecodeOnlyTimesUs.clear();
9035 
9036     // If we haven't transitioned after 3 seconds, we're probably stuck.
9037     sp<AMessage> msg = new AMessage(ACodec::kWhatCheckIfStuck, mCodec);
9038     msg->setInt32("generation", mCodec->mStateGeneration);
9039     msg->post(3000000);
9040 }
9041 
onMessageReceived(const sp<AMessage> & msg)9042 bool ACodec::FlushingState::onMessageReceived(const sp<AMessage> &msg) {
9043     bool handled = false;
9044 
9045     switch (msg->what()) {
9046         case kWhatShutdown:
9047         {
9048             mCodec->deferMessage(msg);
9049             if (mCodec->mFatalError) {
9050                 sp<AMessage> msg = new AMessage(ACodec::kWhatForceStateTransition, mCodec);
9051                 msg->setInt32("generation", mCodec->mStateGeneration);
9052                 msg->post(3000000);
9053             }
9054             handled = true;
9055             break;
9056         }
9057 
9058         case kWhatFlush:
9059         {
9060             // We're already doing this right now.
9061             handled = true;
9062             break;
9063         }
9064 
9065         case kWhatForceStateTransition:
9066         {
9067             int32_t generation = 0;
9068             CHECK(msg->findInt32("generation", &generation));
9069             mCodec->forceStateTransition(generation);
9070 
9071             handled = true;
9072             break;
9073         }
9074 
9075         case kWhatCheckIfStuck:
9076         {
9077             int32_t generation = 0;
9078             CHECK(msg->findInt32("generation", &generation));
9079             if (generation == mCodec->mStateGeneration) {
9080                 mCodec->signalError(OMX_ErrorUndefined, TIMED_OUT);
9081             }
9082 
9083             handled = true;
9084             break;
9085         }
9086 
9087         default:
9088             handled = BaseState::onMessageReceived(msg);
9089             break;
9090     }
9091 
9092     return handled;
9093 }
9094 
onOMXEvent(OMX_EVENTTYPE event,OMX_U32 data1,OMX_U32 data2)9095 bool ACodec::FlushingState::onOMXEvent(
9096         OMX_EVENTTYPE event, OMX_U32 data1, OMX_U32 data2) {
9097     ALOGV("[%s] FlushingState onOMXEvent(%u,%d)",
9098             mCodec->mComponentName.c_str(), event, (OMX_S32)data1);
9099 
9100     switch (event) {
9101         case OMX_EventCmdComplete:
9102         {
9103             if (data1 != (OMX_U32)OMX_CommandFlush) {
9104                 ALOGE("unexpected EventCmdComplete %s(%d) data2:%d in FlushingState",
9105                         asString((OMX_COMMANDTYPE)data1), data1, data2);
9106                 mCodec->signalError(OMX_ErrorUndefined, FAILED_TRANSACTION);
9107                 return true;
9108             }
9109 
9110             if (data2 == kPortIndexInput || data2 == kPortIndexOutput) {
9111                 if (mFlushComplete[data2]) {
9112                     ALOGW("Flush already completed for %s port",
9113                             data2 == kPortIndexInput ? "input" : "output");
9114                     return true;
9115                 }
9116                 mFlushComplete[data2] = true;
9117 
9118                 if (mFlushComplete[kPortIndexInput] && mFlushComplete[kPortIndexOutput]) {
9119                     changeStateIfWeOwnAllBuffers();
9120                 }
9121             } else if (data2 == OMX_ALL) {
9122                 if (!mFlushComplete[kPortIndexInput] || !mFlushComplete[kPortIndexOutput]) {
9123                     ALOGW("received flush complete event for OMX_ALL before ports have been"
9124                             "flushed (%d/%d)",
9125                             mFlushComplete[kPortIndexInput], mFlushComplete[kPortIndexOutput]);
9126                     return false;
9127                 }
9128 
9129                 changeStateIfWeOwnAllBuffers();
9130             } else {
9131                 ALOGW("data2 not OMX_ALL but %u in EventCmdComplete CommandFlush", data2);
9132             }
9133 
9134             return true;
9135         }
9136 
9137         case OMX_EventPortSettingsChanged:
9138         {
9139             sp<AMessage> msg = new AMessage(kWhatOMXMessage, mCodec);
9140             msg->setInt32("type", omx_message::EVENT);
9141             msg->setInt32("generation", mCodec->mNodeGeneration);
9142             msg->setInt32("event", event);
9143             msg->setInt32("data1", data1);
9144             msg->setInt32("data2", data2);
9145 
9146             ALOGV("[%s] Deferring OMX_EventPortSettingsChanged",
9147                  mCodec->mComponentName.c_str());
9148 
9149             mCodec->deferMessage(msg);
9150 
9151             return true;
9152         }
9153 
9154         default:
9155             return BaseState::onOMXEvent(event, data1, data2);
9156     }
9157 
9158     return true;
9159 }
9160 
onOutputBufferDrained(const sp<AMessage> & msg)9161 void ACodec::FlushingState::onOutputBufferDrained(const sp<AMessage> &msg) {
9162     BaseState::onOutputBufferDrained(msg);
9163 
9164     changeStateIfWeOwnAllBuffers();
9165 }
9166 
onInputBufferFilled(const sp<AMessage> & msg)9167 void ACodec::FlushingState::onInputBufferFilled(const sp<AMessage> &msg) {
9168     BaseState::onInputBufferFilled(msg);
9169 
9170     changeStateIfWeOwnAllBuffers();
9171 }
9172 
changeStateIfWeOwnAllBuffers()9173 void ACodec::FlushingState::changeStateIfWeOwnAllBuffers() {
9174     if (mFlushComplete[kPortIndexInput]
9175             && mFlushComplete[kPortIndexOutput]
9176             && mCodec->allYourBuffersAreBelongToUs()) {
9177         // We now own all buffers except possibly those still queued with
9178         // the native window for rendering. Let's get those back as well.
9179         mCodec->waitUntilAllPossibleNativeWindowBuffersAreReturnedToUs();
9180 
9181         mCodec->mCallback->onFlushCompleted();
9182 
9183         mCodec->mPortEOS[kPortIndexInput] =
9184             mCodec->mPortEOS[kPortIndexOutput] = false;
9185 
9186         mCodec->mInputEOSResult = OK;
9187 
9188         if (mCodec->mSkipCutBuffer != NULL) {
9189             mCodec->mSkipCutBuffer->clear();
9190         }
9191 
9192         mCodec->changeState(mCodec->mExecutingState);
9193     }
9194 }
9195 
queryCapabilities(const char * owner,const char * name,const char * mime,bool isEncoder,MediaCodecInfo::CapabilitiesWriter * caps)9196 status_t ACodec::queryCapabilities(
9197         const char* owner, const char* name, const char* mime, bool isEncoder,
9198         MediaCodecInfo::CapabilitiesWriter* caps) {
9199     const char *role = GetComponentRole(isEncoder, mime);
9200     if (role == NULL) {
9201         return BAD_VALUE;
9202     }
9203 
9204     OMXClient client;
9205     status_t err = client.connect(owner);
9206     if (err != OK) {
9207         return err;
9208     }
9209 
9210     sp<IOMX> omx = client.interface();
9211     sp<CodecObserver> observer = new CodecObserver(new AMessage);
9212     sp<IOMXNode> omxNode;
9213 
9214     err = omx->allocateNode(name, observer, &omxNode);
9215     if (err != OK) {
9216         client.disconnect();
9217         return err;
9218     }
9219 
9220     err = SetComponentRole(omxNode, role);
9221     if (err != OK) {
9222         omxNode->freeNode();
9223         client.disconnect();
9224         return err;
9225     }
9226 
9227     bool isVideo = strncasecmp(mime, "video/", 6) == 0;
9228     bool isImage = strncasecmp(mime, "image/", 6) == 0;
9229 
9230     if (isVideo || isImage) {
9231         OMX_VIDEO_PARAM_PROFILELEVELTYPE param;
9232         InitOMXParams(&param);
9233         param.nPortIndex = isEncoder ? kPortIndexOutput : kPortIndexInput;
9234 
9235         for (OMX_U32 index = 0; index <= kMaxIndicesToCheck; ++index) {
9236             param.nProfileIndex = index;
9237             status_t err = omxNode->getParameter(
9238                     OMX_IndexParamVideoProfileLevelQuerySupported,
9239                     &param, sizeof(param));
9240             if (err != OK) {
9241                 break;
9242             }
9243             caps->addProfileLevel(param.eProfile, param.eLevel);
9244 
9245             // AVC components may not list the constrained profiles explicitly, but
9246             // decoders that support a profile also support its constrained version.
9247             // Encoders must explicitly support constrained profiles.
9248             if (!isEncoder && strcasecmp(mime, MEDIA_MIMETYPE_VIDEO_AVC) == 0) {
9249                 if (param.eProfile == OMX_VIDEO_AVCProfileHigh) {
9250                     caps->addProfileLevel(OMX_VIDEO_AVCProfileConstrainedHigh, param.eLevel);
9251                 } else if (param.eProfile == OMX_VIDEO_AVCProfileBaseline) {
9252                     caps->addProfileLevel(OMX_VIDEO_AVCProfileConstrainedBaseline, param.eLevel);
9253                 }
9254             }
9255 
9256             if (index == kMaxIndicesToCheck) {
9257                 ALOGW("[%s] stopping checking profiles after %u: %x/%x",
9258                         name, index,
9259                         param.eProfile, param.eLevel);
9260             }
9261         }
9262 
9263         // Color format query
9264         // return colors in the order reported by the OMX component
9265         // prefix "flexible" standard ones with the flexible equivalent
9266         OMX_VIDEO_PARAM_PORTFORMATTYPE portFormat;
9267         InitOMXParams(&portFormat);
9268         portFormat.nPortIndex = isEncoder ? kPortIndexInput : kPortIndexOutput;
9269         for (OMX_U32 index = 0; index <= kMaxIndicesToCheck; ++index) {
9270             portFormat.nIndex = index;
9271             status_t err = omxNode->getParameter(
9272                     OMX_IndexParamVideoPortFormat,
9273                     &portFormat, sizeof(portFormat));
9274             if (err != OK) {
9275                 break;
9276             }
9277 
9278             OMX_U32 flexibleEquivalent;
9279             if (IsFlexibleColorFormat(
9280                     omxNode, portFormat.eColorFormat, false /* usingNativeWindow */,
9281                     &flexibleEquivalent)) {
9282                 caps->addColorFormat(flexibleEquivalent);
9283             }
9284             caps->addColorFormat(portFormat.eColorFormat);
9285 
9286             if (index == kMaxIndicesToCheck) {
9287                 ALOGW("[%s] stopping checking formats after %u: %s(%x)",
9288                         name, index,
9289                         asString(portFormat.eColorFormat), portFormat.eColorFormat);
9290             }
9291         }
9292     } else if (strcasecmp(mime, MEDIA_MIMETYPE_AUDIO_AAC) == 0) {
9293         // More audio codecs if they have profiles.
9294         OMX_AUDIO_PARAM_ANDROID_PROFILETYPE param;
9295         InitOMXParams(&param);
9296         param.nPortIndex = isEncoder ? kPortIndexOutput : kPortIndexInput;
9297         for (OMX_U32 index = 0; index <= kMaxIndicesToCheck; ++index) {
9298             param.nProfileIndex = index;
9299             status_t err = omxNode->getParameter(
9300                     (OMX_INDEXTYPE)OMX_IndexParamAudioProfileQuerySupported,
9301                     &param, sizeof(param));
9302             if (err != OK) {
9303                 break;
9304             }
9305             // For audio, level is ignored.
9306             caps->addProfileLevel(param.eProfile, 0 /* level */);
9307 
9308             if (index == kMaxIndicesToCheck) {
9309                 ALOGW("[%s] stopping checking profiles after %u: %x",
9310                         name, index,
9311                         param.eProfile);
9312             }
9313         }
9314 
9315         // NOTE: Without Android extensions, OMX does not provide a way to query
9316         // AAC profile support
9317         if (param.nProfileIndex == 0) {
9318             ALOGW("component %s doesn't support profile query.", name);
9319         }
9320     }
9321 
9322     if (isVideo && !isEncoder) {
9323         native_handle_t *sidebandHandle = NULL;
9324         if (omxNode->configureVideoTunnelMode(
9325                 kPortIndexOutput, OMX_TRUE, 0, &sidebandHandle) == OK) {
9326             // tunneled playback includes adaptive playback
9327         } else {
9328             // tunneled playback is not supported
9329             caps->removeDetail(MediaCodecInfo::Capabilities::FEATURE_TUNNELED_PLAYBACK);
9330             if (omxNode->setPortMode(
9331                     kPortIndexOutput, IOMX::kPortModeDynamicANWBuffer) != OK &&
9332                     omxNode->prepareForAdaptivePlayback(
9333                         kPortIndexOutput, OMX_TRUE,
9334                         1280 /* width */, 720 /* height */) != OK) {
9335                 // adaptive playback is not supported
9336                 caps->removeDetail(MediaCodecInfo::Capabilities::FEATURE_ADAPTIVE_PLAYBACK);
9337             }
9338 
9339             // all non-tunneled video decoders support detached surface mode
9340             if (android::media::codec::provider_->null_output_surface_support() &&
9341                     android::media::codec::provider_->null_output_surface()) {
9342                 caps->addDetail(MediaCodecInfo::Capabilities::FEATURE_DETACHED_SURFACE, 0);
9343             }
9344         }
9345     }
9346 
9347     if (isVideo && isEncoder) {
9348         OMX_VIDEO_CONFIG_ANDROID_INTRAREFRESHTYPE params;
9349         InitOMXParams(&params);
9350         params.nPortIndex = kPortIndexOutput;
9351 
9352         OMX_VIDEO_PARAM_INTRAREFRESHTYPE fallbackParams;
9353         InitOMXParams(&fallbackParams);
9354         fallbackParams.nPortIndex = kPortIndexOutput;
9355         fallbackParams.eRefreshMode = OMX_VIDEO_IntraRefreshCyclic;
9356 
9357         if (omxNode->getConfig(
9358                 (OMX_INDEXTYPE)OMX_IndexConfigAndroidIntraRefresh,
9359                 &params, sizeof(params)) != OK &&
9360                 omxNode->getParameter(
9361                     OMX_IndexParamVideoIntraRefresh, &fallbackParams,
9362                     sizeof(fallbackParams)) != OK) {
9363             // intra refresh is not supported
9364             caps->removeDetail(MediaCodecInfo::Capabilities::FEATURE_INTRA_REFRESH);
9365         }
9366     }
9367 
9368     omxNode->freeNode();
9369     client.disconnect();
9370     return OK;
9371 }
9372 
9373 // These are supposed be equivalent to the logic in
9374 // "audio_channel_out_mask_from_count".
9375 //static
getOMXChannelMapping(size_t numChannels,OMX_AUDIO_CHANNELTYPE map[])9376 status_t ACodec::getOMXChannelMapping(size_t numChannels, OMX_AUDIO_CHANNELTYPE map[]) {
9377     switch (numChannels) {
9378         case 1:
9379             map[0] = OMX_AUDIO_ChannelCF;
9380             break;
9381         case 2:
9382             map[0] = OMX_AUDIO_ChannelLF;
9383             map[1] = OMX_AUDIO_ChannelRF;
9384             break;
9385         case 3:
9386             map[0] = OMX_AUDIO_ChannelLF;
9387             map[1] = OMX_AUDIO_ChannelRF;
9388             map[2] = OMX_AUDIO_ChannelCF;
9389             break;
9390         case 4:
9391             map[0] = OMX_AUDIO_ChannelLF;
9392             map[1] = OMX_AUDIO_ChannelRF;
9393             map[2] = OMX_AUDIO_ChannelLR;
9394             map[3] = OMX_AUDIO_ChannelRR;
9395             break;
9396         case 5:
9397             map[0] = OMX_AUDIO_ChannelLF;
9398             map[1] = OMX_AUDIO_ChannelRF;
9399             map[2] = OMX_AUDIO_ChannelCF;
9400             map[3] = OMX_AUDIO_ChannelLR;
9401             map[4] = OMX_AUDIO_ChannelRR;
9402             break;
9403         case 6:
9404             map[0] = OMX_AUDIO_ChannelLF;
9405             map[1] = OMX_AUDIO_ChannelRF;
9406             map[2] = OMX_AUDIO_ChannelCF;
9407             map[3] = OMX_AUDIO_ChannelLFE;
9408             map[4] = OMX_AUDIO_ChannelLR;
9409             map[5] = OMX_AUDIO_ChannelRR;
9410             break;
9411         case 7:
9412             map[0] = OMX_AUDIO_ChannelLF;
9413             map[1] = OMX_AUDIO_ChannelRF;
9414             map[2] = OMX_AUDIO_ChannelCF;
9415             map[3] = OMX_AUDIO_ChannelLFE;
9416             map[4] = OMX_AUDIO_ChannelLR;
9417             map[5] = OMX_AUDIO_ChannelRR;
9418             map[6] = OMX_AUDIO_ChannelCS;
9419             break;
9420         case 8:
9421             map[0] = OMX_AUDIO_ChannelLF;
9422             map[1] = OMX_AUDIO_ChannelRF;
9423             map[2] = OMX_AUDIO_ChannelCF;
9424             map[3] = OMX_AUDIO_ChannelLFE;
9425             map[4] = OMX_AUDIO_ChannelLR;
9426             map[5] = OMX_AUDIO_ChannelRR;
9427             map[6] = OMX_AUDIO_ChannelLS;
9428             map[7] = OMX_AUDIO_ChannelRS;
9429             break;
9430         default:
9431             return -EINVAL;
9432     }
9433 
9434     return OK;
9435 }
9436 
querySupportedParameters(std::vector<std::string> * names)9437 status_t ACodec::querySupportedParameters(std::vector<std::string> *names) {
9438     if (!names) {
9439         return BAD_VALUE;
9440     }
9441     return OK;
9442 }
9443 
subscribeToParameters(const std::vector<std::string> & names)9444 status_t ACodec::subscribeToParameters([[maybe_unused]] const std::vector<std::string> &names) {
9445     return OK;
9446 }
9447 
unsubscribeFromParameters(const std::vector<std::string> & names)9448 status_t ACodec::unsubscribeFromParameters([[maybe_unused]] const std::vector<std::string> &names) {
9449     return OK;
9450 }
9451 
9452 }  // namespace android
9453