1 /* 2 * Copyright 2012, The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 #ifndef MEDIA_CODEC_H_ 18 19 #define MEDIA_CODEC_H_ 20 21 #include <memory> 22 #include <vector> 23 24 #include <gui/IGraphicBufferProducer.h> 25 #include <media/hardware/CryptoAPI.h> 26 #include <media/MediaCodecInfo.h> 27 #include <media/MediaResource.h> 28 #include <media/MediaAnalyticsItem.h> 29 #include <media/stagefright/foundation/AHandler.h> 30 #include <media/stagefright/FrameRenderTracker.h> 31 #include <utils/Vector.h> 32 33 namespace android { 34 35 struct ABuffer; 36 struct AMessage; 37 struct AReplyToken; 38 struct AString; 39 class BufferChannelBase; 40 struct CodecBase; 41 class IBatteryStats; 42 struct ICrypto; 43 class MediaCodecBuffer; 44 class IMemory; 45 class IResourceManagerClient; 46 class IResourceManagerService; 47 struct PersistentSurface; 48 class SoftwareRenderer; 49 class Surface; 50 namespace hardware { 51 namespace cas { 52 namespace native { 53 namespace V1_0 { 54 struct IDescrambler; 55 }}}} 56 using hardware::cas::native::V1_0::IDescrambler; 57 58 struct MediaCodec : public AHandler { 59 enum ConfigureFlags { 60 CONFIGURE_FLAG_ENCODE = 1, 61 }; 62 63 enum BufferFlags { 64 BUFFER_FLAG_SYNCFRAME = 1, 65 BUFFER_FLAG_CODECCONFIG = 2, 66 BUFFER_FLAG_EOS = 4, 67 BUFFER_FLAG_PARTIAL_FRAME = 8, 68 BUFFER_FLAG_MUXER_DATA = 16, 69 }; 70 71 enum { 72 CB_INPUT_AVAILABLE = 1, 73 CB_OUTPUT_AVAILABLE = 2, 74 CB_ERROR = 3, 75 CB_OUTPUT_FORMAT_CHANGED = 4, 76 CB_RESOURCE_RECLAIMED = 5, 77 }; 78 79 static const pid_t kNoPid = -1; 80 static const uid_t kNoUid = -1; 81 82 static sp<MediaCodec> CreateByType( 83 const sp<ALooper> &looper, const AString &mime, bool encoder, status_t *err = NULL, 84 pid_t pid = kNoPid, uid_t uid = kNoUid); 85 86 static sp<MediaCodec> CreateByComponentName( 87 const sp<ALooper> &looper, const AString &name, status_t *err = NULL, 88 pid_t pid = kNoPid, uid_t uid = kNoUid); 89 90 static sp<PersistentSurface> CreatePersistentInputSurface(); 91 92 status_t configure( 93 const sp<AMessage> &format, 94 const sp<Surface> &nativeWindow, 95 const sp<ICrypto> &crypto, 96 uint32_t flags); 97 98 status_t configure( 99 const sp<AMessage> &format, 100 const sp<Surface> &nativeWindow, 101 const sp<ICrypto> &crypto, 102 const sp<IDescrambler> &descrambler, 103 uint32_t flags); 104 105 status_t releaseCrypto(); 106 107 status_t setCallback(const sp<AMessage> &callback); 108 109 status_t setOnFrameRenderedNotification(const sp<AMessage> ¬ify); 110 111 status_t createInputSurface(sp<IGraphicBufferProducer>* bufferProducer); 112 113 status_t setInputSurface(const sp<PersistentSurface> &surface); 114 115 status_t start(); 116 117 // Returns to a state in which the component remains allocated but 118 // unconfigured. 119 status_t stop(); 120 121 // Resets the codec to the INITIALIZED state. Can be called after an error 122 // has occured to make the codec usable. 123 status_t reset(); 124 125 // Client MUST call release before releasing final reference to this 126 // object. 127 status_t release(); 128 129 status_t flush(); 130 131 status_t queueInputBuffer( 132 size_t index, 133 size_t offset, 134 size_t size, 135 int64_t presentationTimeUs, 136 uint32_t flags, 137 AString *errorDetailMsg = NULL); 138 139 status_t queueSecureInputBuffer( 140 size_t index, 141 size_t offset, 142 const CryptoPlugin::SubSample *subSamples, 143 size_t numSubSamples, 144 const uint8_t key[16], 145 const uint8_t iv[16], 146 CryptoPlugin::Mode mode, 147 const CryptoPlugin::Pattern &pattern, 148 int64_t presentationTimeUs, 149 uint32_t flags, 150 AString *errorDetailMsg = NULL); 151 152 status_t dequeueInputBuffer(size_t *index, int64_t timeoutUs = 0ll); 153 154 status_t dequeueOutputBuffer( 155 size_t *index, 156 size_t *offset, 157 size_t *size, 158 int64_t *presentationTimeUs, 159 uint32_t *flags, 160 int64_t timeoutUs = 0ll); 161 162 status_t renderOutputBufferAndRelease(size_t index, int64_t timestampNs); 163 status_t renderOutputBufferAndRelease(size_t index); 164 status_t releaseOutputBuffer(size_t index); 165 166 status_t signalEndOfInputStream(); 167 168 status_t getOutputFormat(sp<AMessage> *format) const; 169 status_t getInputFormat(sp<AMessage> *format) const; 170 171 status_t getInputBuffers(Vector<sp<MediaCodecBuffer> > *buffers) const; 172 status_t getOutputBuffers(Vector<sp<MediaCodecBuffer> > *buffers) const; 173 174 status_t getOutputBuffer(size_t index, sp<MediaCodecBuffer> *buffer); 175 status_t getOutputFormat(size_t index, sp<AMessage> *format); 176 status_t getInputBuffer(size_t index, sp<MediaCodecBuffer> *buffer); 177 178 status_t setSurface(const sp<Surface> &nativeWindow); 179 180 status_t requestIDRFrame(); 181 182 // Notification will be posted once there "is something to do", i.e. 183 // an input/output buffer has become available, a format change is 184 // pending, an error is pending. 185 void requestActivityNotification(const sp<AMessage> ¬ify); 186 187 status_t getName(AString *componentName) const; 188 189 status_t getCodecInfo(sp<MediaCodecInfo> *codecInfo) const; 190 191 status_t getMetrics(MediaAnalyticsItem * &reply); 192 193 status_t setParameters(const sp<AMessage> ¶ms); 194 195 // Create a MediaCodec notification message from a list of rendered or dropped render infos 196 // by adding rendered frame information to a base notification message. Returns the number 197 // of frames that were rendered. 198 static size_t CreateFramesRenderedMessage( 199 const std::list<FrameRenderTracker::Info> &done, sp<AMessage> &msg); 200 201 protected: 202 virtual ~MediaCodec(); 203 virtual void onMessageReceived(const sp<AMessage> &msg); 204 205 private: 206 // used by ResourceManagerClient 207 status_t reclaim(bool force = false); 208 friend struct ResourceManagerClient; 209 210 private: 211 enum State { 212 UNINITIALIZED, 213 INITIALIZING, 214 INITIALIZED, 215 CONFIGURING, 216 CONFIGURED, 217 STARTING, 218 STARTED, 219 FLUSHING, 220 FLUSHED, 221 STOPPING, 222 RELEASING, 223 }; 224 std::string stateString(State state); 225 226 enum { 227 kPortIndexInput = 0, 228 kPortIndexOutput = 1, 229 }; 230 231 enum { 232 kWhatInit = 'init', 233 kWhatConfigure = 'conf', 234 kWhatSetSurface = 'sSur', 235 kWhatCreateInputSurface = 'cisf', 236 kWhatSetInputSurface = 'sisf', 237 kWhatStart = 'strt', 238 kWhatStop = 'stop', 239 kWhatRelease = 'rele', 240 kWhatDequeueInputBuffer = 'deqI', 241 kWhatQueueInputBuffer = 'queI', 242 kWhatDequeueOutputBuffer = 'deqO', 243 kWhatReleaseOutputBuffer = 'relO', 244 kWhatSignalEndOfInputStream = 'eois', 245 kWhatGetBuffers = 'getB', 246 kWhatFlush = 'flus', 247 kWhatGetOutputFormat = 'getO', 248 kWhatGetInputFormat = 'getI', 249 kWhatDequeueInputTimedOut = 'dITO', 250 kWhatDequeueOutputTimedOut = 'dOTO', 251 kWhatCodecNotify = 'codc', 252 kWhatRequestIDRFrame = 'ridr', 253 kWhatRequestActivityNotification = 'racN', 254 kWhatGetName = 'getN', 255 kWhatGetCodecInfo = 'gCoI', 256 kWhatSetParameters = 'setP', 257 kWhatSetCallback = 'setC', 258 kWhatSetNotification = 'setN', 259 kWhatDrmReleaseCrypto = 'rDrm', 260 }; 261 262 enum { 263 kFlagUsesSoftwareRenderer = 1, 264 kFlagOutputFormatChanged = 2, 265 kFlagOutputBuffersChanged = 4, 266 kFlagStickyError = 8, 267 kFlagDequeueInputPending = 16, 268 kFlagDequeueOutputPending = 32, 269 kFlagIsSecure = 64, 270 kFlagSawMediaServerDie = 128, 271 kFlagIsEncoder = 256, 272 // 512 skipped 273 kFlagIsAsync = 1024, 274 kFlagIsComponentAllocated = 2048, 275 kFlagPushBlankBuffersOnShutdown = 4096, 276 }; 277 278 struct BufferInfo { 279 BufferInfo(); 280 281 sp<MediaCodecBuffer> mData; 282 bool mOwnedByClient; 283 }; 284 285 struct ResourceManagerServiceProxy : public IBinder::DeathRecipient { 286 ResourceManagerServiceProxy(pid_t pid); 287 ~ResourceManagerServiceProxy(); 288 289 void init(); 290 291 // implements DeathRecipient 292 virtual void binderDied(const wp<IBinder>& /*who*/); 293 294 void addResource( 295 int64_t clientId, 296 const sp<IResourceManagerClient> &client, 297 const Vector<MediaResource> &resources); 298 299 void removeResource(int64_t clientId); 300 301 bool reclaimResource(const Vector<MediaResource> &resources); 302 303 private: 304 Mutex mLock; 305 sp<IResourceManagerService> mService; 306 pid_t mPid; 307 }; 308 309 State mState; 310 uid_t mUid; 311 bool mReleasedByResourceManager; 312 sp<ALooper> mLooper; 313 sp<ALooper> mCodecLooper; 314 sp<CodecBase> mCodec; 315 AString mComponentName; 316 AString mOwnerName; 317 sp<MediaCodecInfo> mCodecInfo; 318 sp<AReplyToken> mReplyID; 319 uint32_t mFlags; 320 status_t mStickyError; 321 sp<Surface> mSurface; 322 SoftwareRenderer *mSoftRenderer; 323 324 MediaAnalyticsItem *mAnalyticsItem; 325 void initAnalyticsItem(); 326 void updateAnalyticsItem(); 327 void flushAnalyticsItem(); 328 void updateEphemeralAnalytics(MediaAnalyticsItem *item); 329 330 sp<AMessage> mOutputFormat; 331 sp<AMessage> mInputFormat; 332 sp<AMessage> mCallback; 333 sp<AMessage> mOnFrameRenderedNotification; 334 335 sp<IResourceManagerClient> mResourceManagerClient; 336 sp<ResourceManagerServiceProxy> mResourceManagerService; 337 338 bool mBatteryStatNotified; 339 bool mIsVideo; 340 int32_t mVideoWidth; 341 int32_t mVideoHeight; 342 int32_t mRotationDegrees; 343 int32_t mAllowFrameDroppingBySurface; 344 345 // initial create parameters 346 AString mInitName; 347 348 // configure parameter 349 sp<AMessage> mConfigureMsg; 350 351 // Used only to synchronize asynchronous getBufferAndFormat 352 // across all the other (synchronous) buffer state change 353 // operations, such as de/queueIn/OutputBuffer, start and 354 // stop/flush/reset/release. 355 Mutex mBufferLock; 356 357 List<size_t> mAvailPortBuffers[2]; 358 std::vector<BufferInfo> mPortBuffers[2]; 359 360 int32_t mDequeueInputTimeoutGeneration; 361 sp<AReplyToken> mDequeueInputReplyID; 362 363 int32_t mDequeueOutputTimeoutGeneration; 364 sp<AReplyToken> mDequeueOutputReplyID; 365 366 sp<ICrypto> mCrypto; 367 368 sp<IDescrambler> mDescrambler; 369 370 List<sp<ABuffer> > mCSD; 371 372 sp<AMessage> mActivityNotify; 373 374 bool mHaveInputSurface; 375 bool mHavePendingInputBuffers; 376 bool mCpuBoostRequested; 377 378 std::shared_ptr<BufferChannelBase> mBufferChannel; 379 380 MediaCodec(const sp<ALooper> &looper, pid_t pid, uid_t uid); 381 382 static sp<CodecBase> GetCodecBase(const AString &name, const char *owner = nullptr); 383 384 static status_t PostAndAwaitResponse( 385 const sp<AMessage> &msg, sp<AMessage> *response); 386 387 void PostReplyWithError(const sp<AReplyToken> &replyID, int32_t err); 388 389 status_t init(const AString &name); 390 391 void setState(State newState); 392 void returnBuffersToCodec(bool isReclaim = false); 393 void returnBuffersToCodecOnPort(int32_t portIndex, bool isReclaim = false); 394 size_t updateBuffers(int32_t portIndex, const sp<AMessage> &msg); 395 status_t onQueueInputBuffer(const sp<AMessage> &msg); 396 status_t onReleaseOutputBuffer(const sp<AMessage> &msg); 397 ssize_t dequeuePortBuffer(int32_t portIndex); 398 399 status_t getBufferAndFormat( 400 size_t portIndex, size_t index, 401 sp<MediaCodecBuffer> *buffer, sp<AMessage> *format); 402 403 bool handleDequeueInputBuffer(const sp<AReplyToken> &replyID, bool newRequest = false); 404 bool handleDequeueOutputBuffer(const sp<AReplyToken> &replyID, bool newRequest = false); 405 void cancelPendingDequeueOperations(); 406 407 void extractCSD(const sp<AMessage> &format); 408 status_t queueCSDInputBuffer(size_t bufferIndex); 409 410 status_t handleSetSurface(const sp<Surface> &surface); 411 status_t connectToSurface(const sp<Surface> &surface); 412 status_t disconnectFromSurface(); 413 hasCryptoOrDescramblerMediaCodec414 bool hasCryptoOrDescrambler() { 415 return mCrypto != NULL || mDescrambler != NULL; 416 } 417 418 void postActivityNotificationIfPossible(); 419 420 void onInputBufferAvailable(); 421 void onOutputBufferAvailable(); 422 void onError(status_t err, int32_t actionCode, const char *detail = NULL); 423 void onOutputFormatChanged(); 424 425 status_t onSetParameters(const sp<AMessage> ¶ms); 426 427 status_t amendOutputFormatWithCodecSpecificData(const sp<MediaCodecBuffer> &buffer); 428 void updateBatteryStat(); 429 bool isExecuting() const; 430 431 uint64_t getGraphicBufferSize(); 432 void addResource(MediaResource::Type type, MediaResource::SubType subtype, uint64_t value); 433 void requestCpuBoostIfNeeded(); 434 435 bool hasPendingBuffer(int portIndex); 436 bool hasPendingBuffer(); 437 438 /* called to get the last codec error when the sticky flag is set. 439 * if no such codec error is found, returns UNKNOWN_ERROR. 440 */ getStickyErrorMediaCodec441 inline status_t getStickyError() const { 442 return mStickyError != 0 ? mStickyError : UNKNOWN_ERROR; 443 } 444 setStickyErrorMediaCodec445 inline void setStickyError(status_t err) { 446 mFlags |= kFlagStickyError; 447 mStickyError = err; 448 } 449 450 void onReleaseCrypto(const sp<AMessage>& msg); 451 452 // managing time-of-flight aka latency 453 typedef struct { 454 int64_t presentationUs; 455 int64_t startedNs; 456 } BufferFlightTiming_t; 457 std::deque<BufferFlightTiming_t> mBuffersInFlight; 458 Mutex mLatencyLock; 459 int64_t mLatencyUnknown; // buffers for which we couldn't calculate latency 460 461 void statsBufferSent(int64_t presentationUs); 462 void statsBufferReceived(int64_t presentationUs); 463 464 enum { 465 // the default shape of our latency histogram buckets 466 // XXX: should these be configurable in some way? 467 kLatencyHistBuckets = 20, 468 kLatencyHistWidth = 2000, 469 kLatencyHistFloor = 2000, 470 471 // how many samples are in the 'recent latency' histogram 472 // 300 frames = 5 sec @ 60fps or ~12 sec @ 24fps 473 kRecentLatencyFrames = 300, 474 475 // how we initialize mRecentSamples 476 kRecentSampleInvalid = -1, 477 }; 478 479 int64_t mRecentSamples[kRecentLatencyFrames]; 480 int mRecentHead; 481 Mutex mRecentLock; 482 483 class Histogram { 484 public: HistogramMediaCodec485 Histogram() : mFloor(0), mWidth(0), mBelow(0), mAbove(0), 486 mMin(INT64_MAX), mMax(INT64_MIN), mSum(0), mCount(0), 487 mBucketCount(0), mBuckets(NULL) {}; ~HistogramMediaCodec488 ~Histogram() { clear(); }; clearMediaCodec489 void clear() { if (mBuckets != NULL) free(mBuckets); mBuckets = NULL; }; 490 bool setup(int nbuckets, int64_t width, int64_t floor = 0); 491 void insert(int64_t sample); getMinMediaCodec492 int64_t getMin() const { return mMin; } getMaxMediaCodec493 int64_t getMax() const { return mMax; } getCountMediaCodec494 int64_t getCount() const { return mCount; } getSumMediaCodec495 int64_t getSum() const { return mSum; } getAvgMediaCodec496 int64_t getAvg() const { return mSum / (mCount == 0 ? 1 : mCount); } 497 std::string emit(); 498 private: 499 int64_t mFloor, mCeiling, mWidth; 500 int64_t mBelow, mAbove; 501 int64_t mMin, mMax, mSum, mCount; 502 503 int mBucketCount; 504 int64_t *mBuckets; 505 }; 506 507 Histogram mLatencyHist; 508 509 DISALLOW_EVIL_CONSTRUCTORS(MediaCodec); 510 }; 511 512 } // namespace android 513 514 #endif // MEDIA_CODEC_H_ 515