1 /* 2 * Copyright 2012, The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 #ifndef MEDIA_CODEC_H_ 18 19 #define MEDIA_CODEC_H_ 20 21 #include <memory> 22 #include <vector> 23 24 #include <gui/IGraphicBufferProducer.h> 25 #include <media/hardware/CryptoAPI.h> 26 #include <media/MediaCodecInfo.h> 27 #include <media/MediaResource.h> 28 #include <media/MediaAnalyticsItem.h> 29 #include <media/stagefright/foundation/AHandler.h> 30 #include <media/stagefright/FrameRenderTracker.h> 31 #include <utils/Vector.h> 32 33 namespace android { 34 35 struct ABuffer; 36 struct AMessage; 37 struct AReplyToken; 38 struct AString; 39 class BufferChannelBase; 40 struct CodecBase; 41 class IBatteryStats; 42 struct ICrypto; 43 class MediaCodecBuffer; 44 class IMemory; 45 class IResourceManagerClient; 46 class IResourceManagerService; 47 struct PersistentSurface; 48 class SoftwareRenderer; 49 class Surface; 50 namespace hardware { 51 namespace cas { 52 namespace native { 53 namespace V1_0 { 54 struct IDescrambler; 55 }}}} 56 using hardware::cas::native::V1_0::IDescrambler; 57 58 struct MediaCodec : public AHandler { 59 enum ConfigureFlags { 60 CONFIGURE_FLAG_ENCODE = 1, 61 }; 62 63 enum BufferFlags { 64 BUFFER_FLAG_SYNCFRAME = 1, 65 BUFFER_FLAG_CODECCONFIG = 2, 66 BUFFER_FLAG_EOS = 4, 67 }; 68 69 enum { 70 CB_INPUT_AVAILABLE = 1, 71 CB_OUTPUT_AVAILABLE = 2, 72 CB_ERROR = 3, 73 CB_OUTPUT_FORMAT_CHANGED = 4, 74 CB_RESOURCE_RECLAIMED = 5, 75 }; 76 77 static const pid_t kNoPid = -1; 78 static const uid_t kNoUid = -1; 79 80 static sp<MediaCodec> CreateByType( 81 const sp<ALooper> &looper, const AString &mime, bool encoder, status_t *err = NULL, 82 pid_t pid = kNoPid, uid_t uid = kNoUid); 83 84 static sp<MediaCodec> CreateByComponentName( 85 const sp<ALooper> &looper, const AString &name, status_t *err = NULL, 86 pid_t pid = kNoPid, uid_t uid = kNoUid); 87 88 static sp<PersistentSurface> CreatePersistentInputSurface(); 89 90 status_t configure( 91 const sp<AMessage> &format, 92 const sp<Surface> &nativeWindow, 93 const sp<ICrypto> &crypto, 94 uint32_t flags); 95 96 status_t configure( 97 const sp<AMessage> &format, 98 const sp<Surface> &nativeWindow, 99 const sp<ICrypto> &crypto, 100 const sp<IDescrambler> &descrambler, 101 uint32_t flags); 102 103 status_t releaseCrypto(); 104 105 status_t setCallback(const sp<AMessage> &callback); 106 107 status_t setOnFrameRenderedNotification(const sp<AMessage> ¬ify); 108 109 status_t createInputSurface(sp<IGraphicBufferProducer>* bufferProducer); 110 111 status_t setInputSurface(const sp<PersistentSurface> &surface); 112 113 status_t start(); 114 115 // Returns to a state in which the component remains allocated but 116 // unconfigured. 117 status_t stop(); 118 119 // Resets the codec to the INITIALIZED state. Can be called after an error 120 // has occured to make the codec usable. 121 status_t reset(); 122 123 // Client MUST call release before releasing final reference to this 124 // object. 125 status_t release(); 126 127 status_t flush(); 128 129 status_t queueInputBuffer( 130 size_t index, 131 size_t offset, 132 size_t size, 133 int64_t presentationTimeUs, 134 uint32_t flags, 135 AString *errorDetailMsg = NULL); 136 137 status_t queueSecureInputBuffer( 138 size_t index, 139 size_t offset, 140 const CryptoPlugin::SubSample *subSamples, 141 size_t numSubSamples, 142 const uint8_t key[16], 143 const uint8_t iv[16], 144 CryptoPlugin::Mode mode, 145 const CryptoPlugin::Pattern &pattern, 146 int64_t presentationTimeUs, 147 uint32_t flags, 148 AString *errorDetailMsg = NULL); 149 150 status_t dequeueInputBuffer(size_t *index, int64_t timeoutUs = 0ll); 151 152 status_t dequeueOutputBuffer( 153 size_t *index, 154 size_t *offset, 155 size_t *size, 156 int64_t *presentationTimeUs, 157 uint32_t *flags, 158 int64_t timeoutUs = 0ll); 159 160 status_t renderOutputBufferAndRelease(size_t index, int64_t timestampNs); 161 status_t renderOutputBufferAndRelease(size_t index); 162 status_t releaseOutputBuffer(size_t index); 163 164 status_t signalEndOfInputStream(); 165 166 status_t getOutputFormat(sp<AMessage> *format) const; 167 status_t getInputFormat(sp<AMessage> *format) const; 168 169 status_t getInputBuffers(Vector<sp<MediaCodecBuffer> > *buffers) const; 170 status_t getOutputBuffers(Vector<sp<MediaCodecBuffer> > *buffers) const; 171 172 status_t getOutputBuffer(size_t index, sp<MediaCodecBuffer> *buffer); 173 status_t getOutputFormat(size_t index, sp<AMessage> *format); 174 status_t getInputBuffer(size_t index, sp<MediaCodecBuffer> *buffer); 175 176 status_t setSurface(const sp<Surface> &nativeWindow); 177 178 status_t requestIDRFrame(); 179 180 // Notification will be posted once there "is something to do", i.e. 181 // an input/output buffer has become available, a format change is 182 // pending, an error is pending. 183 void requestActivityNotification(const sp<AMessage> ¬ify); 184 185 status_t getName(AString *componentName) const; 186 187 status_t getMetrics(MediaAnalyticsItem * &reply); 188 189 status_t setParameters(const sp<AMessage> ¶ms); 190 191 // Create a MediaCodec notification message from a list of rendered or dropped render infos 192 // by adding rendered frame information to a base notification message. Returns the number 193 // of frames that were rendered. 194 static size_t CreateFramesRenderedMessage( 195 const std::list<FrameRenderTracker::Info> &done, sp<AMessage> &msg); 196 197 protected: 198 virtual ~MediaCodec(); 199 virtual void onMessageReceived(const sp<AMessage> &msg); 200 201 private: 202 // used by ResourceManagerClient 203 status_t reclaim(bool force = false); 204 friend struct ResourceManagerClient; 205 206 private: 207 enum State { 208 UNINITIALIZED, 209 INITIALIZING, 210 INITIALIZED, 211 CONFIGURING, 212 CONFIGURED, 213 STARTING, 214 STARTED, 215 FLUSHING, 216 FLUSHED, 217 STOPPING, 218 RELEASING, 219 }; 220 221 enum { 222 kPortIndexInput = 0, 223 kPortIndexOutput = 1, 224 }; 225 226 enum { 227 kWhatInit = 'init', 228 kWhatConfigure = 'conf', 229 kWhatSetSurface = 'sSur', 230 kWhatCreateInputSurface = 'cisf', 231 kWhatSetInputSurface = 'sisf', 232 kWhatStart = 'strt', 233 kWhatStop = 'stop', 234 kWhatRelease = 'rele', 235 kWhatDequeueInputBuffer = 'deqI', 236 kWhatQueueInputBuffer = 'queI', 237 kWhatDequeueOutputBuffer = 'deqO', 238 kWhatReleaseOutputBuffer = 'relO', 239 kWhatSignalEndOfInputStream = 'eois', 240 kWhatGetBuffers = 'getB', 241 kWhatFlush = 'flus', 242 kWhatGetOutputFormat = 'getO', 243 kWhatGetInputFormat = 'getI', 244 kWhatDequeueInputTimedOut = 'dITO', 245 kWhatDequeueOutputTimedOut = 'dOTO', 246 kWhatCodecNotify = 'codc', 247 kWhatRequestIDRFrame = 'ridr', 248 kWhatRequestActivityNotification = 'racN', 249 kWhatGetName = 'getN', 250 kWhatSetParameters = 'setP', 251 kWhatSetCallback = 'setC', 252 kWhatSetNotification = 'setN', 253 kWhatDrmReleaseCrypto = 'rDrm', 254 }; 255 256 enum { 257 kFlagUsesSoftwareRenderer = 1, 258 kFlagOutputFormatChanged = 2, 259 kFlagOutputBuffersChanged = 4, 260 kFlagStickyError = 8, 261 kFlagDequeueInputPending = 16, 262 kFlagDequeueOutputPending = 32, 263 kFlagIsSecure = 64, 264 kFlagSawMediaServerDie = 128, 265 kFlagIsEncoder = 256, 266 // 512 skipped 267 kFlagIsAsync = 1024, 268 kFlagIsComponentAllocated = 2048, 269 kFlagPushBlankBuffersOnShutdown = 4096, 270 }; 271 272 struct BufferInfo { 273 BufferInfo(); 274 275 sp<MediaCodecBuffer> mData; 276 bool mOwnedByClient; 277 }; 278 279 struct ResourceManagerServiceProxy : public IBinder::DeathRecipient { 280 ResourceManagerServiceProxy(pid_t pid); 281 ~ResourceManagerServiceProxy(); 282 283 void init(); 284 285 // implements DeathRecipient 286 virtual void binderDied(const wp<IBinder>& /*who*/); 287 288 void addResource( 289 int64_t clientId, 290 const sp<IResourceManagerClient> &client, 291 const Vector<MediaResource> &resources); 292 293 void removeResource(int64_t clientId); 294 295 bool reclaimResource(const Vector<MediaResource> &resources); 296 297 private: 298 Mutex mLock; 299 sp<IResourceManagerService> mService; 300 pid_t mPid; 301 }; 302 303 State mState; 304 uid_t mUid; 305 bool mReleasedByResourceManager; 306 sp<ALooper> mLooper; 307 sp<ALooper> mCodecLooper; 308 sp<CodecBase> mCodec; 309 AString mComponentName; 310 sp<AReplyToken> mReplyID; 311 uint32_t mFlags; 312 status_t mStickyError; 313 sp<Surface> mSurface; 314 SoftwareRenderer *mSoftRenderer; 315 316 MediaAnalyticsItem *mAnalyticsItem; 317 void initAnalyticsItem(); 318 void flushAnalyticsItem(); 319 320 sp<AMessage> mOutputFormat; 321 sp<AMessage> mInputFormat; 322 sp<AMessage> mCallback; 323 sp<AMessage> mOnFrameRenderedNotification; 324 325 sp<IResourceManagerClient> mResourceManagerClient; 326 sp<ResourceManagerServiceProxy> mResourceManagerService; 327 328 bool mBatteryStatNotified; 329 bool mIsVideo; 330 int32_t mVideoWidth; 331 int32_t mVideoHeight; 332 int32_t mRotationDegrees; 333 334 // initial create parameters 335 AString mInitName; 336 bool mInitNameIsType; 337 bool mInitIsEncoder; 338 339 // configure parameter 340 sp<AMessage> mConfigureMsg; 341 342 // Used only to synchronize asynchronous getBufferAndFormat 343 // across all the other (synchronous) buffer state change 344 // operations, such as de/queueIn/OutputBuffer, start and 345 // stop/flush/reset/release. 346 Mutex mBufferLock; 347 348 List<size_t> mAvailPortBuffers[2]; 349 std::vector<BufferInfo> mPortBuffers[2]; 350 351 int32_t mDequeueInputTimeoutGeneration; 352 sp<AReplyToken> mDequeueInputReplyID; 353 354 int32_t mDequeueOutputTimeoutGeneration; 355 sp<AReplyToken> mDequeueOutputReplyID; 356 357 sp<ICrypto> mCrypto; 358 359 sp<IDescrambler> mDescrambler; 360 361 List<sp<ABuffer> > mCSD; 362 363 sp<AMessage> mActivityNotify; 364 365 bool mHaveInputSurface; 366 bool mHavePendingInputBuffers; 367 368 std::shared_ptr<BufferChannelBase> mBufferChannel; 369 370 MediaCodec(const sp<ALooper> &looper, pid_t pid, uid_t uid); 371 372 static sp<CodecBase> GetCodecBase(const AString &name, bool nameIsType = false); 373 374 static status_t PostAndAwaitResponse( 375 const sp<AMessage> &msg, sp<AMessage> *response); 376 377 void PostReplyWithError(const sp<AReplyToken> &replyID, int32_t err); 378 379 status_t init(const AString &name, bool nameIsType, bool encoder); 380 381 void setState(State newState); 382 void returnBuffersToCodec(bool isReclaim = false); 383 void returnBuffersToCodecOnPort(int32_t portIndex, bool isReclaim = false); 384 size_t updateBuffers(int32_t portIndex, const sp<AMessage> &msg); 385 status_t onQueueInputBuffer(const sp<AMessage> &msg); 386 status_t onReleaseOutputBuffer(const sp<AMessage> &msg); 387 ssize_t dequeuePortBuffer(int32_t portIndex); 388 389 status_t getBufferAndFormat( 390 size_t portIndex, size_t index, 391 sp<MediaCodecBuffer> *buffer, sp<AMessage> *format); 392 393 bool handleDequeueInputBuffer(const sp<AReplyToken> &replyID, bool newRequest = false); 394 bool handleDequeueOutputBuffer(const sp<AReplyToken> &replyID, bool newRequest = false); 395 void cancelPendingDequeueOperations(); 396 397 void extractCSD(const sp<AMessage> &format); 398 status_t queueCSDInputBuffer(size_t bufferIndex); 399 400 status_t handleSetSurface(const sp<Surface> &surface); 401 status_t connectToSurface(const sp<Surface> &surface); 402 status_t disconnectFromSurface(); 403 hasCryptoOrDescramblerMediaCodec404 bool hasCryptoOrDescrambler() { 405 return mCrypto != NULL || mDescrambler != NULL; 406 } 407 408 void postActivityNotificationIfPossible(); 409 410 void onInputBufferAvailable(); 411 void onOutputBufferAvailable(); 412 void onError(status_t err, int32_t actionCode, const char *detail = NULL); 413 void onOutputFormatChanged(); 414 415 status_t onSetParameters(const sp<AMessage> ¶ms); 416 417 status_t amendOutputFormatWithCodecSpecificData(const sp<MediaCodecBuffer> &buffer); 418 void updateBatteryStat(); 419 bool isExecuting() const; 420 421 uint64_t getGraphicBufferSize(); 422 void addResource(MediaResource::Type type, MediaResource::SubType subtype, uint64_t value); 423 424 bool hasPendingBuffer(int portIndex); 425 bool hasPendingBuffer(); 426 427 /* called to get the last codec error when the sticky flag is set. 428 * if no such codec error is found, returns UNKNOWN_ERROR. 429 */ getStickyErrorMediaCodec430 inline status_t getStickyError() const { 431 return mStickyError != 0 ? mStickyError : UNKNOWN_ERROR; 432 } 433 setStickyErrorMediaCodec434 inline void setStickyError(status_t err) { 435 mFlags |= kFlagStickyError; 436 mStickyError = err; 437 } 438 439 void onReleaseCrypto(const sp<AMessage>& msg); 440 441 DISALLOW_EVIL_CONSTRUCTORS(MediaCodec); 442 }; 443 444 } // namespace android 445 446 #endif // MEDIA_CODEC_H_ 447