1 /* 2 * Copyright 2012, The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 #ifndef MEDIA_CODEC_H_ 18 19 #define MEDIA_CODEC_H_ 20 21 #include <memory> 22 #include <vector> 23 24 #include <gui/IGraphicBufferProducer.h> 25 #include <media/hardware/CryptoAPI.h> 26 #include <media/MediaCodecInfo.h> 27 #include <media/MediaMetrics.h> 28 #include <media/stagefright/foundation/AHandler.h> 29 #include <media/stagefright/FrameRenderTracker.h> 30 #include <utils/Vector.h> 31 32 class C2Buffer; 33 class C2GraphicBlock; 34 class C2LinearBlock; 35 36 namespace aidl { 37 namespace android { 38 namespace media { 39 class MediaResourceParcel; 40 } // media 41 } // android 42 } // aidl 43 44 namespace android { 45 46 struct ABuffer; 47 struct AMessage; 48 struct AReplyToken; 49 struct AString; 50 struct BatteryChecker; 51 class BufferChannelBase; 52 struct CodecBase; 53 class IBatteryStats; 54 struct ICrypto; 55 class MediaCodecBuffer; 56 class IMemory; 57 struct PersistentSurface; 58 class SoftwareRenderer; 59 class Surface; 60 namespace hardware { 61 namespace cas { 62 namespace native { 63 namespace V1_0 { 64 struct IDescrambler; 65 }}}} 66 67 using hardware::cas::native::V1_0::IDescrambler; 68 using aidl::android::media::MediaResourceParcel; 69 70 struct MediaCodec : public AHandler { 71 enum ConfigureFlags { 72 CONFIGURE_FLAG_ENCODE = 1, 73 CONFIGURE_FLAG_USE_BLOCK_MODEL = 2, 74 }; 75 76 enum BufferFlags { 77 BUFFER_FLAG_SYNCFRAME = 1, 78 BUFFER_FLAG_CODECCONFIG = 2, 79 BUFFER_FLAG_EOS = 4, 80 BUFFER_FLAG_PARTIAL_FRAME = 8, 81 BUFFER_FLAG_MUXER_DATA = 16, 82 }; 83 84 enum { 85 CB_INPUT_AVAILABLE = 1, 86 CB_OUTPUT_AVAILABLE = 2, 87 CB_ERROR = 3, 88 CB_OUTPUT_FORMAT_CHANGED = 4, 89 CB_RESOURCE_RECLAIMED = 5, 90 }; 91 92 static const pid_t kNoPid = -1; 93 static const uid_t kNoUid = -1; 94 95 static sp<MediaCodec> CreateByType( 96 const sp<ALooper> &looper, const AString &mime, bool encoder, status_t *err = NULL, 97 pid_t pid = kNoPid, uid_t uid = kNoUid); 98 99 static sp<MediaCodec> CreateByComponentName( 100 const sp<ALooper> &looper, const AString &name, status_t *err = NULL, 101 pid_t pid = kNoPid, uid_t uid = kNoUid); 102 103 static sp<PersistentSurface> CreatePersistentInputSurface(); 104 105 status_t configure( 106 const sp<AMessage> &format, 107 const sp<Surface> &nativeWindow, 108 const sp<ICrypto> &crypto, 109 uint32_t flags); 110 111 status_t configure( 112 const sp<AMessage> &format, 113 const sp<Surface> &nativeWindow, 114 const sp<ICrypto> &crypto, 115 const sp<IDescrambler> &descrambler, 116 uint32_t flags); 117 118 status_t releaseCrypto(); 119 120 status_t setCallback(const sp<AMessage> &callback); 121 122 status_t setOnFrameRenderedNotification(const sp<AMessage> ¬ify); 123 124 status_t createInputSurface(sp<IGraphicBufferProducer>* bufferProducer); 125 126 status_t setInputSurface(const sp<PersistentSurface> &surface); 127 128 status_t start(); 129 130 // Returns to a state in which the component remains allocated but 131 // unconfigured. 132 status_t stop(); 133 134 // Resets the codec to the INITIALIZED state. Can be called after an error 135 // has occured to make the codec usable. 136 status_t reset(); 137 138 // Client MUST call release before releasing final reference to this 139 // object. 140 status_t release(); 141 142 status_t releaseAsync(const sp<AMessage> ¬ify); 143 144 status_t flush(); 145 146 status_t queueInputBuffer( 147 size_t index, 148 size_t offset, 149 size_t size, 150 int64_t presentationTimeUs, 151 uint32_t flags, 152 AString *errorDetailMsg = NULL); 153 154 status_t queueSecureInputBuffer( 155 size_t index, 156 size_t offset, 157 const CryptoPlugin::SubSample *subSamples, 158 size_t numSubSamples, 159 const uint8_t key[16], 160 const uint8_t iv[16], 161 CryptoPlugin::Mode mode, 162 const CryptoPlugin::Pattern &pattern, 163 int64_t presentationTimeUs, 164 uint32_t flags, 165 AString *errorDetailMsg = NULL); 166 167 status_t queueBuffer( 168 size_t index, 169 const std::shared_ptr<C2Buffer> &buffer, 170 int64_t presentationTimeUs, 171 uint32_t flags, 172 const sp<AMessage> &tunings, 173 AString *errorDetailMsg = NULL); 174 175 status_t queueEncryptedBuffer( 176 size_t index, 177 const sp<hardware::HidlMemory> &memory, 178 size_t offset, 179 const CryptoPlugin::SubSample *subSamples, 180 size_t numSubSamples, 181 const uint8_t key[16], 182 const uint8_t iv[16], 183 CryptoPlugin::Mode mode, 184 const CryptoPlugin::Pattern &pattern, 185 int64_t presentationTimeUs, 186 uint32_t flags, 187 const sp<AMessage> &tunings, 188 AString *errorDetailMsg = NULL); 189 190 std::shared_ptr<C2Buffer> decrypt( 191 const std::shared_ptr<C2Buffer> &buffer, 192 const CryptoPlugin::SubSample *subSamples, 193 size_t numSubSamples, 194 const uint8_t key[16], 195 const uint8_t iv[16], 196 CryptoPlugin::Mode mode, 197 const CryptoPlugin::Pattern &pattern); 198 199 status_t dequeueInputBuffer(size_t *index, int64_t timeoutUs = 0ll); 200 201 status_t dequeueOutputBuffer( 202 size_t *index, 203 size_t *offset, 204 size_t *size, 205 int64_t *presentationTimeUs, 206 uint32_t *flags, 207 int64_t timeoutUs = 0ll); 208 209 status_t renderOutputBufferAndRelease(size_t index, int64_t timestampNs); 210 status_t renderOutputBufferAndRelease(size_t index); 211 status_t releaseOutputBuffer(size_t index); 212 213 status_t signalEndOfInputStream(); 214 215 status_t getOutputFormat(sp<AMessage> *format) const; 216 status_t getInputFormat(sp<AMessage> *format) const; 217 218 status_t getInputBuffers(Vector<sp<MediaCodecBuffer> > *buffers) const; 219 status_t getOutputBuffers(Vector<sp<MediaCodecBuffer> > *buffers) const; 220 221 status_t getOutputBuffer(size_t index, sp<MediaCodecBuffer> *buffer); 222 status_t getOutputFormat(size_t index, sp<AMessage> *format); 223 status_t getInputBuffer(size_t index, sp<MediaCodecBuffer> *buffer); 224 225 status_t setSurface(const sp<Surface> &nativeWindow); 226 227 status_t requestIDRFrame(); 228 229 // Notification will be posted once there "is something to do", i.e. 230 // an input/output buffer has become available, a format change is 231 // pending, an error is pending. 232 void requestActivityNotification(const sp<AMessage> ¬ify); 233 234 status_t getName(AString *componentName) const; 235 236 status_t getCodecInfo(sp<MediaCodecInfo> *codecInfo) const; 237 238 status_t getMetrics(mediametrics_handle_t &reply); 239 240 status_t setParameters(const sp<AMessage> ¶ms); 241 242 // Create a MediaCodec notification message from a list of rendered or dropped render infos 243 // by adding rendered frame information to a base notification message. Returns the number 244 // of frames that were rendered. 245 static size_t CreateFramesRenderedMessage( 246 const std::list<FrameRenderTracker::Info> &done, sp<AMessage> &msg); 247 248 static status_t CanFetchLinearBlock( 249 const std::vector<std::string> &names, bool *isCompatible); 250 251 static std::shared_ptr<C2LinearBlock> FetchLinearBlock( 252 size_t capacity, const std::vector<std::string> &names); 253 254 static status_t CanFetchGraphicBlock( 255 const std::vector<std::string> &names, bool *isCompatible); 256 257 static std::shared_ptr<C2GraphicBlock> FetchGraphicBlock( 258 int32_t width, 259 int32_t height, 260 int32_t format, 261 uint64_t usage, 262 const std::vector<std::string> &names); 263 264 template <typename T> 265 struct WrapperObject : public RefBase { WrapperObjectMediaCodec::WrapperObject266 WrapperObject(const T& v) : value(v) {} WrapperObjectMediaCodec::WrapperObject267 WrapperObject(T&& v) : value(std::move(v)) {} 268 T value; 269 }; 270 271 protected: 272 virtual ~MediaCodec(); 273 virtual void onMessageReceived(const sp<AMessage> &msg); 274 275 private: 276 // used by ResourceManagerClient 277 status_t reclaim(bool force = false); 278 friend struct ResourceManagerClient; 279 280 private: 281 enum State { 282 UNINITIALIZED, 283 INITIALIZING, 284 INITIALIZED, 285 CONFIGURING, 286 CONFIGURED, 287 STARTING, 288 STARTED, 289 FLUSHING, 290 FLUSHED, 291 STOPPING, 292 RELEASING, 293 }; 294 std::string stateString(State state); 295 296 enum { 297 kPortIndexInput = 0, 298 kPortIndexOutput = 1, 299 }; 300 301 enum { 302 kWhatInit = 'init', 303 kWhatConfigure = 'conf', 304 kWhatSetSurface = 'sSur', 305 kWhatCreateInputSurface = 'cisf', 306 kWhatSetInputSurface = 'sisf', 307 kWhatStart = 'strt', 308 kWhatStop = 'stop', 309 kWhatRelease = 'rele', 310 kWhatDequeueInputBuffer = 'deqI', 311 kWhatQueueInputBuffer = 'queI', 312 kWhatDequeueOutputBuffer = 'deqO', 313 kWhatReleaseOutputBuffer = 'relO', 314 kWhatSignalEndOfInputStream = 'eois', 315 kWhatGetBuffers = 'getB', 316 kWhatFlush = 'flus', 317 kWhatGetOutputFormat = 'getO', 318 kWhatGetInputFormat = 'getI', 319 kWhatDequeueInputTimedOut = 'dITO', 320 kWhatDequeueOutputTimedOut = 'dOTO', 321 kWhatCodecNotify = 'codc', 322 kWhatRequestIDRFrame = 'ridr', 323 kWhatRequestActivityNotification = 'racN', 324 kWhatGetName = 'getN', 325 kWhatGetCodecInfo = 'gCoI', 326 kWhatSetParameters = 'setP', 327 kWhatSetCallback = 'setC', 328 kWhatSetNotification = 'setN', 329 kWhatDrmReleaseCrypto = 'rDrm', 330 kWhatCheckBatteryStats = 'chkB', 331 }; 332 333 enum { 334 kFlagUsesSoftwareRenderer = 1, 335 kFlagOutputFormatChanged = 2, 336 kFlagOutputBuffersChanged = 4, 337 kFlagStickyError = 8, 338 kFlagDequeueInputPending = 16, 339 kFlagDequeueOutputPending = 32, 340 kFlagIsSecure = 64, 341 kFlagSawMediaServerDie = 128, 342 kFlagIsEncoder = 256, 343 // 512 skipped 344 kFlagIsAsync = 1024, 345 kFlagIsComponentAllocated = 2048, 346 kFlagPushBlankBuffersOnShutdown = 4096, 347 kFlagUseBlockModel = 8192, 348 }; 349 350 struct BufferInfo { 351 BufferInfo(); 352 353 sp<MediaCodecBuffer> mData; 354 bool mOwnedByClient; 355 }; 356 357 struct ResourceManagerServiceProxy; 358 359 State mState; 360 uid_t mUid; 361 bool mReleasedByResourceManager; 362 sp<ALooper> mLooper; 363 sp<ALooper> mCodecLooper; 364 sp<CodecBase> mCodec; 365 AString mComponentName; 366 AString mOwnerName; 367 sp<MediaCodecInfo> mCodecInfo; 368 sp<AReplyToken> mReplyID; 369 std::string mLastReplyOrigin; 370 std::vector<sp<AMessage>> mDeferredMessages; 371 uint32_t mFlags; 372 status_t mStickyError; 373 sp<Surface> mSurface; 374 SoftwareRenderer *mSoftRenderer; 375 376 mediametrics_handle_t mMetricsHandle = 0; 377 nsecs_t mLifetimeStartNs = 0; 378 void initMediametrics(); 379 void updateMediametrics(); 380 void flushMediametrics(); 381 void updateEphemeralMediametrics(mediametrics_handle_t item); 382 void updateLowLatency(const sp<AMessage> &msg); 383 384 sp<AMessage> mOutputFormat; 385 sp<AMessage> mInputFormat; 386 sp<AMessage> mCallback; 387 sp<AMessage> mOnFrameRenderedNotification; 388 sp<AMessage> mAsyncReleaseCompleteNotification; 389 390 sp<ResourceManagerServiceProxy> mResourceManagerProxy; 391 392 bool mIsVideo; 393 int32_t mVideoWidth; 394 int32_t mVideoHeight; 395 int32_t mRotationDegrees; 396 int32_t mAllowFrameDroppingBySurface; 397 398 // initial create parameters 399 AString mInitName; 400 401 // configure parameter 402 sp<AMessage> mConfigureMsg; 403 404 // Used only to synchronize asynchronous getBufferAndFormat 405 // across all the other (synchronous) buffer state change 406 // operations, such as de/queueIn/OutputBuffer, start and 407 // stop/flush/reset/release. 408 Mutex mBufferLock; 409 410 List<size_t> mAvailPortBuffers[2]; 411 std::vector<BufferInfo> mPortBuffers[2]; 412 413 int32_t mDequeueInputTimeoutGeneration; 414 sp<AReplyToken> mDequeueInputReplyID; 415 416 int32_t mDequeueOutputTimeoutGeneration; 417 sp<AReplyToken> mDequeueOutputReplyID; 418 419 sp<ICrypto> mCrypto; 420 421 sp<IDescrambler> mDescrambler; 422 423 List<sp<ABuffer> > mCSD; 424 425 sp<AMessage> mActivityNotify; 426 427 bool mHaveInputSurface; 428 bool mHavePendingInputBuffers; 429 bool mCpuBoostRequested; 430 431 std::shared_ptr<BufferChannelBase> mBufferChannel; 432 433 MediaCodec( 434 const sp<ALooper> &looper, pid_t pid, uid_t uid, 435 std::function<sp<CodecBase>(const AString &, const char *)> getCodecBase = nullptr, 436 std::function<status_t(const AString &, sp<MediaCodecInfo> *)> getCodecInfo = nullptr); 437 438 static sp<CodecBase> GetCodecBase(const AString &name, const char *owner = nullptr); 439 440 static status_t PostAndAwaitResponse( 441 const sp<AMessage> &msg, sp<AMessage> *response); 442 443 void PostReplyWithError(const sp<AMessage> &msg, int32_t err); 444 void PostReplyWithError(const sp<AReplyToken> &replyID, int32_t err); 445 446 status_t init(const AString &name); 447 448 void setState(State newState); 449 void returnBuffersToCodec(bool isReclaim = false); 450 void returnBuffersToCodecOnPort(int32_t portIndex, bool isReclaim = false); 451 size_t updateBuffers(int32_t portIndex, const sp<AMessage> &msg); 452 status_t onQueueInputBuffer(const sp<AMessage> &msg); 453 status_t onReleaseOutputBuffer(const sp<AMessage> &msg); 454 BufferInfo *peekNextPortBuffer(int32_t portIndex); 455 ssize_t dequeuePortBuffer(int32_t portIndex); 456 457 status_t getBufferAndFormat( 458 size_t portIndex, size_t index, 459 sp<MediaCodecBuffer> *buffer, sp<AMessage> *format); 460 461 bool handleDequeueInputBuffer(const sp<AReplyToken> &replyID, bool newRequest = false); 462 bool handleDequeueOutputBuffer(const sp<AReplyToken> &replyID, bool newRequest = false); 463 void cancelPendingDequeueOperations(); 464 465 void extractCSD(const sp<AMessage> &format); 466 status_t queueCSDInputBuffer(size_t bufferIndex); 467 468 status_t handleSetSurface(const sp<Surface> &surface); 469 status_t connectToSurface(const sp<Surface> &surface); 470 status_t disconnectFromSurface(); 471 hasCryptoOrDescramblerMediaCodec472 bool hasCryptoOrDescrambler() { 473 return mCrypto != NULL || mDescrambler != NULL; 474 } 475 476 void postActivityNotificationIfPossible(); 477 478 void onInputBufferAvailable(); 479 void onOutputBufferAvailable(); 480 void onError(status_t err, int32_t actionCode, const char *detail = NULL); 481 void onOutputFormatChanged(); 482 483 status_t onSetParameters(const sp<AMessage> ¶ms); 484 485 status_t amendOutputFormatWithCodecSpecificData(const sp<MediaCodecBuffer> &buffer); 486 void handleOutputFormatChangeIfNeeded(const sp<MediaCodecBuffer> &buffer); 487 bool isExecuting() const; 488 489 uint64_t getGraphicBufferSize(); 490 void requestCpuBoostIfNeeded(); 491 492 bool hasPendingBuffer(int portIndex); 493 bool hasPendingBuffer(); 494 495 void postPendingRepliesAndDeferredMessages(std::string origin, status_t err = OK); 496 void postPendingRepliesAndDeferredMessages(std::string origin, const sp<AMessage> &response); 497 498 /* called to get the last codec error when the sticky flag is set. 499 * if no such codec error is found, returns UNKNOWN_ERROR. 500 */ getStickyErrorMediaCodec501 inline status_t getStickyError() const { 502 return mStickyError != 0 ? mStickyError : UNKNOWN_ERROR; 503 } 504 setStickyErrorMediaCodec505 inline void setStickyError(status_t err) { 506 mFlags |= kFlagStickyError; 507 mStickyError = err; 508 } 509 510 void onReleaseCrypto(const sp<AMessage>& msg); 511 512 // managing time-of-flight aka latency 513 typedef struct { 514 int64_t presentationUs; 515 int64_t startedNs; 516 } BufferFlightTiming_t; 517 std::deque<BufferFlightTiming_t> mBuffersInFlight; 518 Mutex mLatencyLock; 519 int64_t mLatencyUnknown; // buffers for which we couldn't calculate latency 520 int64_t mNumLowLatencyEnables; // how many times low latency mode is enabled 521 int64_t mNumLowLatencyDisables; // how many times low latency mode is disabled 522 bool mIsLowLatencyModeOn; // is low latency mode on currently 523 int64_t mIndexOfFirstFrameWhenLowLatencyOn; // index of the first frame queued 524 // when low latency is on 525 int64_t mInputBufferCounter; // number of input buffers queued since last reset/flush 526 527 class ReleaseSurface; 528 std::unique_ptr<ReleaseSurface> mReleaseSurface; 529 530 std::list<sp<AMessage>> mLeftover; 531 status_t handleLeftover(size_t index); 532 533 sp<BatteryChecker> mBatteryChecker; 534 535 void statsBufferSent(int64_t presentationUs); 536 void statsBufferReceived(int64_t presentationUs); 537 538 enum { 539 // the default shape of our latency histogram buckets 540 // XXX: should these be configurable in some way? 541 kLatencyHistBuckets = 20, 542 kLatencyHistWidth = 2000, 543 kLatencyHistFloor = 2000, 544 545 // how many samples are in the 'recent latency' histogram 546 // 300 frames = 5 sec @ 60fps or ~12 sec @ 24fps 547 kRecentLatencyFrames = 300, 548 549 // how we initialize mRecentSamples 550 kRecentSampleInvalid = -1, 551 }; 552 553 int64_t mRecentSamples[kRecentLatencyFrames]; 554 int mRecentHead; 555 Mutex mRecentLock; 556 557 class Histogram { 558 public: HistogramMediaCodec559 Histogram() : mFloor(0), mWidth(0), mBelow(0), mAbove(0), 560 mMin(INT64_MAX), mMax(INT64_MIN), mSum(0), mCount(0), 561 mBucketCount(0), mBuckets(NULL) {}; ~HistogramMediaCodec562 ~Histogram() { clear(); }; clearMediaCodec563 void clear() { if (mBuckets != NULL) free(mBuckets); mBuckets = NULL; }; 564 bool setup(int nbuckets, int64_t width, int64_t floor = 0); 565 void insert(int64_t sample); getMinMediaCodec566 int64_t getMin() const { return mMin; } getMaxMediaCodec567 int64_t getMax() const { return mMax; } getCountMediaCodec568 int64_t getCount() const { return mCount; } getSumMediaCodec569 int64_t getSum() const { return mSum; } getAvgMediaCodec570 int64_t getAvg() const { return mSum / (mCount == 0 ? 1 : mCount); } 571 std::string emit(); 572 private: 573 int64_t mFloor, mCeiling, mWidth; 574 int64_t mBelow, mAbove; 575 int64_t mMin, mMax, mSum, mCount; 576 577 int mBucketCount; 578 int64_t *mBuckets; 579 }; 580 581 Histogram mLatencyHist; 582 583 std::function<sp<CodecBase>(const AString &, const char *)> mGetCodecBase; 584 std::function<status_t(const AString &, sp<MediaCodecInfo> *)> mGetCodecInfo; 585 friend class MediaTestHelper; 586 587 DISALLOW_EVIL_CONSTRUCTORS(MediaCodec); 588 }; 589 590 } // namespace android 591 592 #endif // MEDIA_CODEC_H_ 593