1 /* 2 * Copyright 2012, The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 #ifndef MEDIA_CODEC_H_ 18 19 #define MEDIA_CODEC_H_ 20 21 #include <list> 22 #include <memory> 23 #include <vector> 24 25 #include <gui/IGraphicBufferProducer.h> 26 #include <media/hardware/CryptoAPI.h> 27 #include <media/MediaCodecInfo.h> 28 #include <media/MediaMetrics.h> 29 #include <media/MediaProfiles.h> 30 #include <media/stagefright/foundation/AHandler.h> 31 #include <media/stagefright/foundation/AMessage.h> 32 #include <media/stagefright/foundation/Mutexed.h> 33 #include <media/stagefright/CodecErrorLog.h> 34 #include <media/stagefright/FrameRenderTracker.h> 35 #include <media/stagefright/MediaHistogram.h> 36 #include <media/stagefright/PlaybackDurationAccumulator.h> 37 #include <media/stagefright/ResourceInfo.h> 38 #include <media/stagefright/VideoRenderQualityTracker.h> 39 #include <utils/Vector.h> 40 41 class C2Buffer; 42 class C2GraphicBlock; 43 class C2LinearBlock; 44 45 namespace aidl { 46 namespace android { 47 namespace media { 48 class MediaResourceParcel; 49 class ClientConfigParcel; 50 } // media 51 } // android 52 } // aidl 53 54 namespace android { 55 56 struct ABuffer; 57 struct AMessage; 58 struct AReplyToken; 59 struct AString; 60 struct BatteryChecker; 61 class BufferChannelBase; 62 struct AccessUnitInfo; 63 struct CodecBase; 64 struct CodecCryptoInfo; 65 struct CodecParameterDescriptor; 66 class IBatteryStats; 67 struct ICrypto; 68 class CryptoAsync; 69 class MediaCodecBuffer; 70 class IMemory; 71 struct PersistentSurface; 72 class RenderedFrameInfo; 73 class SoftwareRenderer; 74 class Surface; 75 namespace hardware { 76 namespace cas { 77 namespace native { 78 namespace V1_0 { 79 struct IDescrambler; 80 }}}} 81 82 using hardware::cas::native::V1_0::IDescrambler; 83 using aidl::android::media::MediaResourceParcel; 84 using aidl::android::media::ClientConfigParcel; 85 86 typedef WrapperObject<std::vector<AccessUnitInfo>> BufferInfosWrapper; 87 typedef WrapperObject<std::vector<std::unique_ptr<CodecCryptoInfo>>> CryptoInfosWrapper; 88 89 struct MediaCodec : public AHandler { 90 enum Domain { 91 DOMAIN_UNKNOWN = 0, 92 DOMAIN_VIDEO = 1, 93 DOMAIN_AUDIO = 2, 94 DOMAIN_IMAGE = 3 95 }; 96 97 enum ConfigureFlags { 98 CONFIGURE_FLAG_ENCODE = 1, 99 CONFIGURE_FLAG_USE_BLOCK_MODEL = 2, 100 CONFIGURE_FLAG_USE_CRYPTO_ASYNC = 4, 101 CONFIGURE_FLAG_DETACHED_SURFACE = 8, 102 }; 103 104 enum BufferFlags { 105 BUFFER_FLAG_SYNCFRAME = 1, 106 BUFFER_FLAG_CODECCONFIG = 2, 107 BUFFER_FLAG_EOS = 4, 108 BUFFER_FLAG_PARTIAL_FRAME = 8, 109 BUFFER_FLAG_MUXER_DATA = 16, 110 BUFFER_FLAG_DECODE_ONLY = 32, 111 }; 112 113 enum CVODegree { 114 CVO_DEGREE_0 = 0, 115 CVO_DEGREE_90 = 90, 116 CVO_DEGREE_180 = 180, 117 CVO_DEGREE_270 = 270, 118 }; 119 120 enum { 121 CB_INPUT_AVAILABLE = 1, 122 CB_OUTPUT_AVAILABLE = 2, 123 CB_ERROR = 3, 124 CB_OUTPUT_FORMAT_CHANGED = 4, 125 CB_RESOURCE_RECLAIMED = 5, // deprecated and not used 126 CB_CRYPTO_ERROR = 6, 127 CB_LARGE_FRAME_OUTPUT_AVAILABLE = 7, 128 129 /** Callback ID for when the metrics for this codec have been flushed 130 * due to the start of a new subsession. The associated AMessage will 131 * contain an sp<WrapperObject<std::unique_ptr<mediametrics::Item>>> 132 * Object at the "metrics" key. 133 */ 134 CB_METRICS_FLUSHED = 8, 135 136 /** Callback ID to notify the change in resource requirement 137 * for the codec component. 138 */ 139 CB_REQUIRED_RESOURCES_CHANGED = 9, 140 }; 141 142 static const pid_t kNoPid = -1; 143 static const uid_t kNoUid = -1; 144 145 static sp<MediaCodec> CreateByType( 146 const sp<ALooper> &looper, const AString &mime, bool encoder, status_t *err = NULL, 147 pid_t pid = kNoPid, uid_t uid = kNoUid); 148 149 static sp<MediaCodec> CreateByType( 150 const sp<ALooper> &looper, const AString &mime, bool encoder, status_t *err, 151 pid_t pid, uid_t uid, sp<AMessage> format); 152 153 static sp<MediaCodec> CreateByComponentName( 154 const sp<ALooper> &looper, const AString &name, status_t *err = NULL, 155 pid_t pid = kNoPid, uid_t uid = kNoUid); 156 157 static sp<PersistentSurface> CreatePersistentInputSurface(); 158 159 /** 160 * Get a list of Globally available device codec resources. 161 * 162 * It will return INVALID_OPERATION if: 163 * - HAL does not implement codec availability API 164 * - codec_availability feature flag isn't defined. 165 */ 166 static status_t getGloballyAvailableResources(std::vector<GlobalResourceInfo>& resources); 167 168 status_t configure( 169 const sp<AMessage> &format, 170 const sp<Surface> &nativeWindow, 171 const sp<ICrypto> &crypto, 172 uint32_t flags); 173 174 status_t configure( 175 const sp<AMessage> &format, 176 const sp<Surface> &nativeWindow, 177 const sp<ICrypto> &crypto, 178 const sp<IDescrambler> &descrambler, 179 uint32_t flags); 180 181 /** 182 * Get a list of required codec resources. 183 * 184 * This may only be called after configuring the codec. 185 * 186 * Calling this while the codec wasn't configured, will result in 187 * returning INVALID_OPERATION error code. 188 * It will also return INVALID_OPERATION if: 189 * - HAL does not implement codec availability API 190 * - codec_availability feature flag isn't defined. 191 */ 192 status_t getRequiredResources(std::vector<InstanceResourceInfo>& resources); 193 194 status_t releaseCrypto(); 195 196 status_t setCallback(const sp<AMessage> &callback); 197 198 status_t setOnFrameRenderedNotification(const sp<AMessage> ¬ify); 199 200 status_t setOnFirstTunnelFrameReadyNotification(const sp<AMessage> ¬ify); 201 202 status_t createInputSurface(sp<IGraphicBufferProducer>* bufferProducer); 203 204 status_t setInputSurface(const sp<PersistentSurface> &surface); 205 206 status_t start(); 207 208 // Returns to a state in which the component remains allocated but 209 // unconfigured. 210 status_t stop(); 211 212 // Resets the codec to the INITIALIZED state. Can be called after an error 213 // has occured to make the codec usable. 214 status_t reset(); 215 216 // Client MUST call release before releasing final reference to this 217 // object. 218 status_t release(); 219 220 status_t releaseAsync(const sp<AMessage> ¬ify); 221 222 status_t flush(); 223 224 status_t queueInputBuffer( 225 size_t index, 226 size_t offset, 227 size_t size, 228 int64_t presentationTimeUs, 229 uint32_t flags, 230 AString *errorDetailMsg = NULL); 231 232 status_t queueInputBuffers( 233 size_t index, 234 size_t offset, 235 size_t size, 236 const sp<BufferInfosWrapper> &accessUnitInfo, 237 AString *errorDetailMsg = NULL); 238 239 status_t queueSecureInputBuffer( 240 size_t index, 241 size_t offset, 242 const CryptoPlugin::SubSample *subSamples, 243 size_t numSubSamples, 244 const uint8_t key[16], 245 const uint8_t iv[16], 246 CryptoPlugin::Mode mode, 247 const CryptoPlugin::Pattern &pattern, 248 int64_t presentationTimeUs, 249 uint32_t flags, 250 AString *errorDetailMsg = NULL); 251 252 status_t queueSecureInputBuffers( 253 size_t index, 254 size_t offset, 255 size_t size, 256 const sp<BufferInfosWrapper> &accessUnitInfo, 257 const sp<CryptoInfosWrapper> &cryptoInfos, 258 AString *errorDetailMsg = NULL); 259 260 status_t queueBuffer( 261 size_t index, 262 const std::shared_ptr<C2Buffer> &buffer, 263 const sp<BufferInfosWrapper> &bufferInfos, 264 const sp<AMessage> &tunings, 265 AString *errorDetailMsg = NULL); 266 267 status_t queueEncryptedBuffer( 268 size_t index, 269 const sp<hardware::HidlMemory> &memory, 270 size_t offset, 271 size_t size, 272 const sp<BufferInfosWrapper> &bufferInfos, 273 const sp<CryptoInfosWrapper> &cryptoInfos, 274 const sp<AMessage> &tunings, 275 AString *errorDetailMsg = NULL); 276 277 std::shared_ptr<C2Buffer> decrypt( 278 const std::shared_ptr<C2Buffer> &buffer, 279 const CryptoPlugin::SubSample *subSamples, 280 size_t numSubSamples, 281 const uint8_t key[16], 282 const uint8_t iv[16], 283 CryptoPlugin::Mode mode, 284 const CryptoPlugin::Pattern &pattern); 285 286 status_t dequeueInputBuffer(size_t *index, int64_t timeoutUs = 0ll); 287 288 status_t dequeueOutputBuffer( 289 size_t *index, 290 size_t *offset, 291 size_t *size, 292 int64_t *presentationTimeUs, 293 uint32_t *flags, 294 int64_t timeoutUs = 0ll); 295 296 status_t renderOutputBufferAndRelease(size_t index, int64_t timestampNs); 297 status_t renderOutputBufferAndRelease(size_t index); 298 status_t releaseOutputBuffer(size_t index); 299 300 status_t signalEndOfInputStream(); 301 302 status_t getOutputFormat(sp<AMessage> *format) const; 303 status_t getInputFormat(sp<AMessage> *format) const; 304 305 status_t getInputBuffers(Vector<sp<MediaCodecBuffer> > *buffers) const; 306 status_t getOutputBuffers(Vector<sp<MediaCodecBuffer> > *buffers) const; 307 308 status_t getOutputBuffer(size_t index, sp<MediaCodecBuffer> *buffer); 309 status_t getOutputFormat(size_t index, sp<AMessage> *format); 310 status_t getInputBuffer(size_t index, sp<MediaCodecBuffer> *buffer); 311 312 status_t setSurface(const sp<Surface> &nativeWindow); 313 314 status_t detachOutputSurface(); 315 316 status_t requestIDRFrame(); 317 318 // Notification will be posted once there "is something to do", i.e. 319 // an input/output buffer has become available, a format change is 320 // pending, an error is pending. 321 void requestActivityNotification(const sp<AMessage> ¬ify); 322 323 status_t getName(AString *componentName) const; 324 325 status_t getCodecInfo(sp<MediaCodecInfo> *codecInfo) const; 326 327 status_t getMetrics(mediametrics_handle_t &reply); 328 329 status_t setParameters(const sp<AMessage> ¶ms); 330 331 status_t querySupportedVendorParameters(std::vector<std::string> *names); 332 status_t describeParameter(const std::string &name, CodecParameterDescriptor *desc); 333 status_t subscribeToVendorParameters(const std::vector<std::string> &names); 334 status_t unsubscribeFromVendorParameters(const std::vector<std::string> &names); 335 336 // Create a MediaCodec notification message from a list of rendered or dropped render infos 337 // by adding rendered frame information to a base notification message. Returns the number 338 // of frames that were rendered. 339 static size_t CreateFramesRenderedMessage( 340 const std::list<RenderedFrameInfo> &done, sp<AMessage> &msg); 341 static size_t CreateFramesRenderedMessage( 342 const std::list<FrameRenderTracker::Info> &done, sp<AMessage> &msg); 343 344 static status_t CanFetchLinearBlock( 345 const std::vector<std::string> &names, bool *isCompatible); 346 347 static std::shared_ptr<C2LinearBlock> FetchLinearBlock( 348 size_t capacity, const std::vector<std::string> &names); 349 350 static status_t CanFetchGraphicBlock( 351 const std::vector<std::string> &names, bool *isCompatible); 352 353 static std::shared_ptr<C2GraphicBlock> FetchGraphicBlock( 354 int32_t width, 355 int32_t height, 356 int32_t format, 357 uint64_t usage, 358 const std::vector<std::string> &names); 359 360 template <typename T> 361 struct WrapperObject : public RefBase { WrapperObjectMediaCodec::WrapperObject362 WrapperObject(const T& v) : value(v) {} WrapperObjectMediaCodec::WrapperObject363 WrapperObject(T&& v) : value(std::move(v)) {} 364 T value; 365 }; 366 getErrorLogMediaCodec367 inline CodecErrorLog &getErrorLog() { return mErrorLog; } 368 369 protected: 370 virtual ~MediaCodec(); 371 virtual void onMessageReceived(const sp<AMessage> &msg); 372 373 private: 374 // used by ResourceManagerClient 375 status_t reclaim(bool force = false); 376 friend struct ResourceManagerClient; 377 378 // to create the metrics associated with this codec. 379 // Any error in this function will be captured by the output argument err. 380 mediametrics_handle_t createMediaMetrics(const sp<AMessage>& format, 381 uint32_t flags, 382 status_t* err); 383 384 // Get the required system resources for the current configuration. 385 bool getRequiredSystemResources(); 386 // Convert all dynamic (non-constant) resource types into 387 // constant resource counts. 388 std::vector<InstanceResourceInfo> computeDynamicResources( 389 const std::vector<InstanceResourceInfo>& resources); 390 void updateResourceUsage(const std::vector<InstanceResourceInfo>& oldResources, 391 const std::vector<InstanceResourceInfo>& newResources); 392 393 private: 394 enum State { 395 UNINITIALIZED, 396 INITIALIZING, 397 INITIALIZED, 398 CONFIGURING, 399 CONFIGURED, 400 STARTING, 401 STARTED, 402 FLUSHING, 403 FLUSHED, 404 STOPPING, 405 RELEASING, 406 }; 407 std::string stateString(State state); 408 std::string apiStateString(); 409 410 enum { 411 kPortIndexInput = 0, 412 kPortIndexOutput = 1, 413 }; 414 415 enum { 416 kWhatInit = 'init', 417 kWhatConfigure = 'conf', 418 kWhatSetSurface = 'sSur', 419 kWhatDetachSurface = 'dSur', 420 kWhatCreateInputSurface = 'cisf', 421 kWhatSetInputSurface = 'sisf', 422 kWhatStart = 'strt', 423 kWhatStop = 'stop', 424 kWhatRelease = 'rele', 425 kWhatDequeueInputBuffer = 'deqI', 426 kWhatQueueInputBuffer = 'queI', 427 kWhatDequeueOutputBuffer = 'deqO', 428 kWhatReleaseOutputBuffer = 'relO', 429 kWhatSignalEndOfInputStream = 'eois', 430 kWhatGetBuffers = 'getB', 431 kWhatFlush = 'flus', 432 kWhatGetOutputFormat = 'getO', 433 kWhatGetInputFormat = 'getI', 434 kWhatDequeueInputTimedOut = 'dITO', 435 kWhatDequeueOutputTimedOut = 'dOTO', 436 kWhatCodecNotify = 'codc', 437 kWhatRequestIDRFrame = 'ridr', 438 kWhatRequestActivityNotification = 'racN', 439 kWhatGetName = 'getN', 440 kWhatGetCodecInfo = 'gCoI', 441 kWhatSetParameters = 'setP', 442 kWhatSetCallback = 'setC', 443 kWhatSetNotification = 'setN', 444 kWhatDrmReleaseCrypto = 'rDrm', 445 kWhatCheckBatteryStats = 'chkB', 446 kWhatGetMetrics = 'getM', 447 }; 448 449 enum { 450 kFlagUsesSoftwareRenderer = 1, 451 kFlagOutputFormatChanged = 2, 452 kFlagOutputBuffersChanged = 4, 453 kFlagStickyError = 8, 454 kFlagDequeueInputPending = 16, 455 kFlagDequeueOutputPending = 32, 456 kFlagIsSecure = 64, 457 kFlagSawMediaServerDie = 128, 458 kFlagIsEncoder = 256, 459 // 512 skipped 460 kFlagIsAsync = 1024, 461 kFlagIsComponentAllocated = 2048, 462 kFlagPushBlankBuffersOnShutdown = 4096, 463 kFlagUseBlockModel = 8192, 464 kFlagUseCryptoAsync = 16384, 465 }; 466 467 struct BufferInfo { 468 BufferInfo(); 469 470 sp<MediaCodecBuffer> mData; 471 bool mOwnedByClient; 472 }; 473 474 // This type is used to track the tunnel mode video peek state machine: 475 // 476 // DisabledNoBuffer -> EnabledNoBuffer when tunnel-peek = true 477 // DisabledQueued -> EnabledQueued when tunnel-peek = true 478 // DisabledNoBuffer -> DisabledQueued when first frame queued 479 // EnabledNoBuffer -> DisabledNoBuffer when tunnel-peek = false 480 // EnabledQueued -> DisabledQueued when tunnel-peek = false 481 // EnabledNoBuffer -> EnabledQueued when first frame queued 482 // DisabledNoBuffer -> BufferDecoded when kWhatFirstTunnelFrameReady 483 // DisabledQueued -> BufferDecoded when kWhatFirstTunnelFrameReady 484 // EnabledNoBuffer -> BufferDecoded when kWhatFirstTunnelFrameReady 485 // EnabledQueued -> BufferDecoded when kWhatFirstTunnelFrameReady 486 // BufferDecoded -> BufferRendered when kWhatFrameRendered 487 // <all states> -> EnabledNoBuffer when flush 488 // <all states> -> EnabledNoBuffer when stop then configure then start 489 enum struct TunnelPeekState { 490 kLegacyMode, 491 kDisabledNoBuffer, 492 kEnabledNoBuffer, 493 kDisabledQueued, 494 kEnabledQueued, 495 kBufferDecoded, 496 kBufferRendered, 497 }; 498 499 enum class DequeueOutputResult { 500 kNoBuffer, 501 kDiscardedBuffer, 502 kRepliedWithError, 503 kSuccess, 504 }; 505 506 struct ResourceManagerServiceProxy; 507 508 State mState; 509 bool mReleasedByResourceManager; 510 sp<ALooper> mLooper; 511 sp<ALooper> mCodecLooper; 512 sp<CodecBase> mCodec; 513 AString mComponentName; 514 AString mOwnerName; 515 sp<MediaCodecInfo> mCodecInfo; 516 sp<AReplyToken> mReplyID; 517 std::string mLastReplyOrigin; 518 std::vector<sp<AMessage>> mDeferredMessages; 519 uint32_t mFlags; 520 int64_t mPresentationTimeUs = 0; 521 status_t mStickyError; 522 sp<Surface> mSurface; 523 uint32_t mSurfaceGeneration = 0; 524 SoftwareRenderer *mSoftRenderer; 525 526 // Get the detached BufferQueue surface for a video decoder, and create it 527 // if it did not yet exist. 528 sp<Surface> getOrCreateDetachedSurface(); 529 530 Mutex mMetricsLock; 531 mediametrics_handle_t mMetricsHandle = 0; 532 mediametrics_handle_t mLastMetricsHandle = 0; // only accessed from the looper or destructor 533 bool mMetricsToUpload = false; 534 nsecs_t mLifetimeStartNs = 0; 535 void initMediametrics(); 536 void updateMediametrics(); 537 void flushMediametrics(); 538 void resetMetricsFields(); 539 540 // Reset the metrics fields for a new subsession. 541 void resetSubsessionMetricsFields(); 542 543 // Start a new subsession (for metrics). This includes flushing the current 544 // metrics, notifying the client and resetting the session fields. 545 void handleStartingANewSubsession(); 546 547 void updateEphemeralMediametrics(mediametrics_handle_t item); 548 void updateLowLatency(const sp<AMessage> &msg); 549 void updateCodecImportance(const sp<AMessage>& msg); 550 void updatePictureProfile(const sp<AMessage>& msg, bool applyDefaultProfile); 551 void onGetMetrics(const sp<AMessage>& msg); 552 constexpr const char *asString(TunnelPeekState state, const char *default_string="?"); 553 void updateTunnelPeek(const sp<AMessage> &msg); 554 void processRenderedFrames(const sp<AMessage> &msg); 555 556 inline void initClientConfigParcel(ClientConfigParcel& clientConfig); 557 558 sp<AMessage> mOutputFormat; 559 sp<AMessage> mInputFormat; 560 sp<AMessage> mCallback; 561 sp<AMessage> mOnFrameRenderedNotification; 562 sp<AMessage> mAsyncReleaseCompleteNotification; 563 sp<AMessage> mOnFirstTunnelFrameReadyNotification; 564 565 std::shared_ptr<ResourceManagerServiceProxy> mResourceManagerProxy; 566 567 Domain mDomain; 568 AString mLogSessionId; 569 int32_t mWidth; 570 int32_t mHeight; 571 int32_t mRotationDegrees; 572 int32_t mAllowFrameDroppingBySurface; 573 574 enum { 575 kFlagHasHdrStaticInfo = 1, 576 kFlagHasHdr10PlusInfo = 2, 577 }; 578 uint32_t mHdrInfoFlags; 579 void updateHdrMetrics(bool isConfig); 580 hdr_format getHdrFormat(const AString &mime, const int32_t profile, 581 const int32_t colorTransfer); 582 hdr_format getHdrFormatForEncoder(const AString &mime, const int32_t profile, 583 const int32_t colorTransfer); 584 hdr_format getHdrFormatForDecoder(const AString &mime, const int32_t profile, 585 const int32_t colorTransfer); 586 bool profileSupport10Bits(const AString &mime, const int32_t profile); 587 588 struct ApiUsageMetrics { 589 bool isArrayMode; 590 enum OperationMode { 591 kUnknownMode = 0, 592 kSynchronousMode = 1, 593 kAsynchronousMode = 2, 594 kBlockMode = 3, 595 }; 596 OperationMode operationMode; 597 bool isUsingOutputSurface; 598 struct InputBufferSize { 599 int32_t appMax; // max size configured by the app 600 int32_t usedMax; // max size actually used 601 int32_t codecMax; // max size suggested by the codec 602 } inputBufferSize; 603 } mApiUsageMetrics; 604 struct ReliabilityContextMetrics { 605 int32_t flushCount; 606 int32_t setOutputSurfaceCount; 607 int32_t resolutionChangeCount; 608 } mReliabilityContextMetrics; 609 int32_t mSubsessionCount; 610 611 // initial create parameters 612 AString mInitName; 613 614 // configure parameter 615 sp<AMessage> mConfigureMsg; 616 617 // rewrites the format description during configure() for encoding. 618 // format and flags as they exist within configure() 619 // the (possibly) updated format is returned in place. 620 status_t shapeMediaFormat( 621 const sp<AMessage> &format, 622 uint32_t flags, 623 mediametrics_handle_t handle); 624 625 // populate the format shaper library with information for this codec encoding 626 // for the indicated media type 627 status_t setupFormatShaper(AString mediaType); 628 629 // Used only to synchronize asynchronous getBufferAndFormat 630 // across all the other (synchronous) buffer state change 631 // operations, such as de/queueIn/OutputBuffer, start and 632 // stop/flush/reset/release. 633 Mutex mBufferLock; 634 635 std::list<size_t> mAvailPortBuffers[2]; 636 std::vector<BufferInfo> mPortBuffers[2]; 637 638 int32_t mDequeueInputTimeoutGeneration; 639 sp<AReplyToken> mDequeueInputReplyID; 640 641 int32_t mDequeueOutputTimeoutGeneration; 642 sp<AReplyToken> mDequeueOutputReplyID; 643 644 sp<ICrypto> mCrypto; 645 646 int32_t mTunneledInputWidth; 647 int32_t mTunneledInputHeight; 648 bool mTunneled; 649 TunnelPeekState mTunnelPeekState; 650 bool mTunnelPeekEnabled; 651 652 sp<IDescrambler> mDescrambler; 653 654 std::list<sp<ABuffer> > mCSD; 655 656 sp<AMessage> mActivityNotify; 657 658 bool mHaveInputSurface; 659 bool mHavePendingInputBuffers; 660 bool mCpuBoostRequested; 661 662 std::shared_ptr<BufferChannelBase> mBufferChannel; 663 sp<CryptoAsync> mCryptoAsync; 664 sp<ALooper> mCryptoLooper; 665 666 bool mIsSurfaceToDisplay; 667 bool mAreRenderMetricsEnabled; 668 PlaybackDurationAccumulator mPlaybackDurationAccumulator; 669 VideoRenderQualityTracker mVideoRenderQualityTracker; 670 671 MediaCodec( 672 const sp<ALooper> &looper, pid_t pid, uid_t uid, 673 std::function<sp<CodecBase>(const AString &, const char *)> getCodecBase = nullptr, 674 std::function<status_t(const AString &, sp<MediaCodecInfo> *)> getCodecInfo = nullptr); 675 676 static sp<CodecBase> GetCodecBase(const AString &name, const char *owner = nullptr); 677 678 static status_t PostAndAwaitResponse( 679 const sp<AMessage> &msg, sp<AMessage> *response); 680 681 void PostReplyWithError(const sp<AMessage> &msg, int32_t err); 682 void PostReplyWithError(const sp<AReplyToken> &replyID, int32_t err); 683 684 status_t init(const AString &name); 685 686 void setState(State newState); 687 void returnBuffersToCodec(bool isReclaim = false); 688 void returnBuffersToCodecOnPort(int32_t portIndex, bool isReclaim = false); 689 size_t updateBuffers(int32_t portIndex, const sp<AMessage> &msg); 690 status_t onQueueInputBuffer(const sp<AMessage> &msg); 691 status_t onReleaseOutputBuffer(const sp<AMessage> &msg); 692 BufferInfo *peekNextPortBuffer(int32_t portIndex); 693 ssize_t dequeuePortBuffer(int32_t portIndex); 694 695 status_t getBufferAndFormat( 696 size_t portIndex, size_t index, 697 sp<MediaCodecBuffer> *buffer, sp<AMessage> *format); 698 699 bool handleDequeueInputBuffer(const sp<AReplyToken> &replyID, bool newRequest = false); 700 DequeueOutputResult handleDequeueOutputBuffer( 701 const sp<AReplyToken> &replyID, 702 bool newRequest = false); 703 void cancelPendingDequeueOperations(); 704 705 void extractCSD(const sp<AMessage> &format); 706 status_t queueCSDInputBuffer(size_t bufferIndex); 707 708 status_t handleSetSurface(const sp<Surface> &surface); 709 710 // Common reimplementation of changing the output surface. 711 // Handles setting null surface, which is used during configure and init. 712 // Set |callCodec| to true if the codec needs to be notified (e.g. during executing state). 713 // Setting |onShutdown| to true will avoid extra work, if this is used for detaching on 714 // delayed release. 715 status_t handleSetSurface(const sp<Surface> &surface, bool callCodec, bool onShutdown = false); 716 status_t connectToSurface(const sp<Surface> &surface, uint32_t *generation); 717 status_t disconnectFromSurface(); 718 hasCryptoOrDescramblerMediaCodec719 bool hasCryptoOrDescrambler() { 720 return mCrypto != NULL || mDescrambler != NULL; 721 } 722 723 void postActivityNotificationIfPossible(); 724 725 void onInputBufferAvailable(); 726 void onOutputBufferAvailable(); 727 void onCryptoError(const sp<AMessage> &msg); 728 void onError(status_t err, int32_t actionCode, const char *detail = NULL); 729 void onOutputFormatChanged(); 730 void onRequiredResourcesChanged(); 731 732 status_t onSetParameters(const sp<AMessage> ¶ms); 733 734 status_t amendOutputFormatWithCodecSpecificData(const sp<MediaCodecBuffer> &buffer); 735 void handleOutputFormatChangeIfNeeded(const sp<MediaCodecBuffer> &buffer); 736 bool isExecuting() const; 737 738 uint64_t getGraphicBufferSize(); 739 void requestCpuBoostIfNeeded(); 740 741 bool hasPendingBuffer(int portIndex); 742 bool hasPendingBuffer(); 743 744 void postPendingRepliesAndDeferredMessages(std::string origin, status_t err = OK); 745 void postPendingRepliesAndDeferredMessages(std::string origin, const sp<AMessage> &response); 746 747 /* called to get the last codec error when the sticky flag is set. 748 * if no such codec error is found, returns UNKNOWN_ERROR. 749 */ getStickyErrorMediaCodec750 inline status_t getStickyError() const { 751 return mStickyError != 0 ? mStickyError : UNKNOWN_ERROR; 752 } 753 setStickyErrorMediaCodec754 inline void setStickyError(status_t err) { 755 mFlags |= kFlagStickyError; 756 mStickyError = err; 757 } 758 759 void onReleaseCrypto(const sp<AMessage>& msg); 760 761 void stopCryptoAsync(); 762 763 // managing time-of-flight aka latency 764 typedef struct { 765 int64_t presentationUs; 766 int64_t startedNs; 767 } BufferFlightTiming_t; 768 std::deque<BufferFlightTiming_t> mBuffersInFlight; 769 Mutex mLatencyLock; 770 int64_t mLatencyUnknown; // buffers for which we couldn't calculate latency 771 772 Mutex mOutputStatsLock; 773 int64_t mBytesEncoded = 0; 774 int64_t mEarliestEncodedPtsUs = INT64_MAX; 775 int64_t mLatestEncodedPtsUs = INT64_MIN; 776 int64_t mFramesEncoded = 0; 777 int64_t mBytesInput = 0; 778 int64_t mFramesInput = 0; 779 780 int64_t mNumLowLatencyEnables; // how many times low latency mode is enabled 781 int64_t mNumLowLatencyDisables; // how many times low latency mode is disabled 782 bool mIsLowLatencyModeOn; // is low latency mode on currently 783 int64_t mIndexOfFirstFrameWhenLowLatencyOn; // index of the first frame queued 784 // when low latency is on 785 int64_t mInputBufferCounter; // number of input buffers queued since last reset/flush 786 787 // A rescheduleable message that periodically polls for rendered buffers 788 sp<AMessage> mMsgPollForRenderedBuffers; 789 790 class ReleaseSurface; 791 std::unique_ptr<ReleaseSurface> mDetachedSurface; 792 793 std::list<sp<AMessage>> mLeftover; 794 status_t handleLeftover(size_t index); 795 796 sp<BatteryChecker> mBatteryChecker; 797 798 void statsBufferSent(int64_t presentationUs, const sp<MediaCodecBuffer> &buffer); 799 void statsBufferReceived(int64_t presentationUs, const sp<MediaCodecBuffer> &buffer); 800 bool discardDecodeOnlyOutputBuffer(size_t index); 801 802 enum { 803 // the default shape of our latency histogram buckets 804 // XXX: should these be configurable in some way? 805 kLatencyHistBuckets = 20, 806 kLatencyHistWidth = 2000, 807 kLatencyHistFloor = 2000, 808 809 // how many samples are in the 'recent latency' histogram 810 // 300 frames = 5 sec @ 60fps or ~12 sec @ 24fps 811 kRecentLatencyFrames = 300, 812 813 // how we initialize mRecentSamples 814 kRecentSampleInvalid = -1, 815 }; 816 817 int64_t mRecentSamples[kRecentLatencyFrames]; 818 int mRecentHead; 819 Mutex mRecentLock; 820 821 MediaHistogram<int64_t> mLatencyHist; 822 823 // An unique ID for the codec - Used by the metrics. 824 uint64_t mCodecId = 0; 825 bool mIsHardware = false; 826 827 std::function<sp<CodecBase>(const AString &, const char *)> mGetCodecBase; 828 std::function<status_t(const AString &, sp<MediaCodecInfo> *)> mGetCodecInfo; 829 friend class MediaTestHelper; 830 831 CodecErrorLog mErrorLog; 832 // Required resource info for this codec. 833 Mutexed<std::vector<InstanceResourceInfo>> mRequiredResourceInfo; 834 835 // Default frame-rate. 836 float mFrameRate = 30.0; 837 838 DISALLOW_EVIL_CONSTRUCTORS(MediaCodec); 839 }; 840 841 } // namespace android 842 843 #endif // MEDIA_CODEC_H_ 844