1 /* 2 * Copyright 2017, The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 #ifndef CCODEC_BUFFER_CHANNEL_H_ 18 19 #define CCODEC_BUFFER_CHANNEL_H_ 20 21 #include <deque> 22 #include <map> 23 #include <memory> 24 #include <vector> 25 26 #include <C2Buffer.h> 27 #include <C2Component.h> 28 #include <Codec2Mapper.h> 29 30 #include <codec2/hidl/client.h> 31 #include <media/stagefright/foundation/Mutexed.h> 32 #include <media/stagefright/CodecBase.h> 33 34 #include "CCodecBuffers.h" 35 #include "FrameReassembler.h" 36 #include "InputSurfaceWrapper.h" 37 #include "PipelineWatcher.h" 38 39 namespace android { 40 41 class MemoryDealer; 42 43 class CCodecCallback { 44 public: 45 virtual ~CCodecCallback() = default; 46 virtual void onError(status_t err, enum ActionCode actionCode) = 0; 47 virtual void onOutputFramesRendered(int64_t mediaTimeUs, nsecs_t renderTimeNs) = 0; 48 virtual void onOutputBuffersChanged() = 0; 49 virtual void onFirstTunnelFrameReady() = 0; 50 }; 51 52 /** 53 * BufferChannelBase implementation for CCodec. 54 */ 55 class CCodecBufferChannel 56 : public BufferChannelBase, public std::enable_shared_from_this<CCodecBufferChannel> { 57 public: 58 explicit CCodecBufferChannel(const std::shared_ptr<CCodecCallback> &callback); 59 virtual ~CCodecBufferChannel(); 60 61 // BufferChannelBase interface 62 void setCrypto(const sp<ICrypto> &crypto) override; 63 void setDescrambler(const sp<IDescrambler> &descrambler) override; 64 65 virtual status_t queueInputBuffer(const sp<MediaCodecBuffer> &buffer) override; 66 virtual status_t queueSecureInputBuffer( 67 const sp<MediaCodecBuffer> &buffer, 68 bool secure, 69 const uint8_t *key, 70 const uint8_t *iv, 71 CryptoPlugin::Mode mode, 72 CryptoPlugin::Pattern pattern, 73 const CryptoPlugin::SubSample *subSamples, 74 size_t numSubSamples, 75 AString *errorDetailMsg) override; 76 virtual status_t attachBuffer( 77 const std::shared_ptr<C2Buffer> &c2Buffer, 78 const sp<MediaCodecBuffer> &buffer) override; 79 virtual status_t attachEncryptedBuffer( 80 const sp<hardware::HidlMemory> &memory, 81 bool secure, 82 const uint8_t *key, 83 const uint8_t *iv, 84 CryptoPlugin::Mode mode, 85 CryptoPlugin::Pattern pattern, 86 size_t offset, 87 const CryptoPlugin::SubSample *subSamples, 88 size_t numSubSamples, 89 const sp<MediaCodecBuffer> &buffer, 90 AString* errorDetailMsg) override; 91 virtual status_t renderOutputBuffer( 92 const sp<MediaCodecBuffer> &buffer, int64_t timestampNs) override; 93 virtual void pollForRenderedBuffers() override; 94 virtual status_t discardBuffer(const sp<MediaCodecBuffer> &buffer) override; 95 virtual void getInputBufferArray(Vector<sp<MediaCodecBuffer>> *array) override; 96 virtual void getOutputBufferArray(Vector<sp<MediaCodecBuffer>> *array) override; 97 98 // Methods below are interface for CCodec to use. 99 100 /** 101 * Set the component object for buffer processing. 102 */ 103 void setComponent(const std::shared_ptr<Codec2Client::Component> &component); 104 105 /** 106 * Set output graphic surface for rendering. 107 */ 108 status_t setSurface(const sp<Surface> &surface, bool pushBlankBuffer); 109 110 /** 111 * Set GraphicBufferSource object from which the component extracts input 112 * buffers. 113 */ 114 status_t setInputSurface(const std::shared_ptr<InputSurfaceWrapper> &surface); 115 116 /** 117 * Signal EOS to input surface. 118 */ 119 status_t signalEndOfInputStream(); 120 121 /** 122 * Set parameters. 123 */ 124 status_t setParameters(std::vector<std::unique_ptr<C2Param>> ¶ms); 125 126 /** 127 * Start queueing buffers to the component. This object should never queue 128 * buffers before this call has completed. 129 */ 130 status_t start( 131 const sp<AMessage> &inputFormat, 132 const sp<AMessage> &outputFormat, 133 bool buffersBoundToCodec); 134 135 /** 136 * Prepare initial input buffers to be filled by client. 137 * 138 * \param clientInputBuffers[out] pointer to slot index -> buffer map. 139 * On success, it contains prepared 140 * initial input buffers. 141 */ 142 status_t prepareInitialInputBuffers( 143 std::map<size_t, sp<MediaCodecBuffer>> *clientInputBuffers, 144 bool retry = false); 145 146 /** 147 * Request initial input buffers as prepared in clientInputBuffers. 148 * 149 * \param clientInputBuffers[in] slot index -> buffer map with prepared 150 * initial input buffers. 151 */ 152 status_t requestInitialInputBuffers( 153 std::map<size_t, sp<MediaCodecBuffer>> &&clientInputBuffers); 154 155 /** 156 * Stop using buffers of the current output surface for other Codec 157 * instances to use the surface safely. 158 * 159 * \param pushBlankBuffer[in] push a blank buffer at the end if true 160 */ 161 void stopUseOutputSurface(bool pushBlankBuffer); 162 163 /** 164 * Stop queueing buffers to the component. This object should never queue 165 * buffers after this call, until start() is called. 166 */ 167 void stop(); 168 169 /** 170 * Stop queueing buffers to the component and release all buffers. 171 */ 172 void reset(); 173 174 /** 175 * Release all resources. 176 */ 177 void release(); 178 179 void flush(const std::list<std::unique_ptr<C2Work>> &flushedWork); 180 181 /** 182 * Notify input client about work done. 183 * 184 * @param workItems finished work item. 185 * @param outputFormat new output format if it has changed, otherwise nullptr 186 * @param initData new init data (CSD) if it has changed, otherwise nullptr 187 */ 188 void onWorkDone( 189 std::unique_ptr<C2Work> work, const sp<AMessage> &outputFormat, 190 const C2StreamInitDataInfo::output *initData); 191 192 /** 193 * Make an input buffer available for the client as it is no longer needed 194 * by the codec. 195 * 196 * @param frameIndex The index of input work 197 * @param arrayIndex The index of buffer in the input work buffers. 198 */ 199 void onInputBufferDone(uint64_t frameIndex, size_t arrayIndex); 200 201 PipelineWatcher::Clock::duration elapsed(); 202 203 enum MetaMode { 204 MODE_NONE, 205 MODE_ANW, 206 }; 207 208 void setMetaMode(MetaMode mode); 209 210 /** 211 * get pixel format from output buffers. 212 * 213 * @return 0 if no valid pixel format found. 214 */ 215 uint32_t getBuffersPixelFormat(bool isEncoder); 216 217 void resetBuffersPixelFormat(bool isEncoder); 218 219 private: 220 uint32_t getInputBuffersPixelFormat(); 221 222 uint32_t getOutputBuffersPixelFormat(); 223 224 class QueueGuard; 225 226 /** 227 * Special mutex-like object with the following properties: 228 * 229 * - At STOPPED state (initial, or after stop()) 230 * - QueueGuard object gets created at STOPPED state, and the client is 231 * supposed to return immediately. 232 * - At RUNNING state (after start()) 233 * - Each QueueGuard object 234 */ 235 class QueueSync { 236 public: 237 /** 238 * At construction the sync object is in STOPPED state. 239 */ QueueSync()240 inline QueueSync() {} 241 ~QueueSync() = default; 242 243 /** 244 * Transition to RUNNING state when stopped. No-op if already in RUNNING 245 * state. 246 */ 247 void start(); 248 249 /** 250 * At RUNNING state, wait until all QueueGuard object created during 251 * RUNNING state are destroyed, and then transition to STOPPED state. 252 * No-op if already in STOPPED state. 253 */ 254 void stop(); 255 256 private: 257 Mutex mGuardLock; 258 259 struct Counter { CounterCounter260 inline Counter() : value(-1) {} 261 int32_t value; 262 Condition cond; 263 }; 264 Mutexed<Counter> mCount; 265 266 friend class CCodecBufferChannel::QueueGuard; 267 }; 268 269 class QueueGuard { 270 public: 271 QueueGuard(QueueSync &sync); 272 ~QueueGuard(); isRunning()273 inline bool isRunning() { return mRunning; } 274 275 private: 276 QueueSync &mSync; 277 bool mRunning; 278 }; 279 280 struct TrackedFrame { 281 uint64_t number; 282 int64_t mediaTimeUs; 283 int64_t desiredRenderTimeNs; 284 nsecs_t latchTime; 285 sp<Fence> presentFence; 286 }; 287 288 void feedInputBufferIfAvailable(); 289 void feedInputBufferIfAvailableInternal(); 290 status_t queueInputBufferInternal(sp<MediaCodecBuffer> buffer, 291 std::shared_ptr<C2LinearBlock> encryptedBlock = nullptr, 292 size_t blockSize = 0); 293 bool handleWork( 294 std::unique_ptr<C2Work> work, const sp<AMessage> &outputFormat, 295 const C2StreamInitDataInfo::output *initData); 296 void sendOutputBuffers(); 297 void ensureDecryptDestination(size_t size); 298 int32_t getHeapSeqNum(const sp<hardware::HidlMemory> &memory); 299 300 void initializeFrameTrackingFor(ANativeWindow * window); 301 void trackReleasedFrame(const IGraphicBufferProducer::QueueBufferOutput& qbo, 302 int64_t mediaTimeUs, int64_t desiredRenderTimeNs); 303 void processRenderedFrames(const FrameEventHistoryDelta& delta); 304 int64_t getRenderTimeNs(const TrackedFrame& frame); 305 306 QueueSync mSync; 307 sp<MemoryDealer> mDealer; 308 sp<IMemory> mDecryptDestination; 309 int32_t mHeapSeqNum; 310 std::map<wp<hardware::HidlMemory>, int32_t> mHeapSeqNumMap; 311 312 std::shared_ptr<Codec2Client::Component> mComponent; 313 std::string mComponentName; ///< component name for debugging 314 const char *mName; ///< C-string version of component name 315 std::shared_ptr<CCodecCallback> mCCodecCallback; 316 std::shared_ptr<C2BlockPool> mInputAllocator; 317 QueueSync mQueueSync; 318 std::vector<std::unique_ptr<C2Param>> mParamsToBeSet; 319 320 struct Input { 321 Input(); 322 323 std::unique_ptr<InputBuffers> buffers; 324 size_t numSlots; 325 FlexBuffersImpl extraBuffers; 326 size_t numExtraSlots; 327 uint32_t inputDelay; 328 uint32_t pipelineDelay; 329 c2_cntr64_t lastFlushIndex; 330 331 FrameReassembler frameReassembler; 332 }; 333 Mutexed<Input> mInput; 334 struct Output { 335 std::unique_ptr<OutputBuffers> buffers; 336 size_t numSlots; 337 uint32_t outputDelay; 338 // true iff the underlying block pool is bounded --- for example, 339 // a BufferQueue-based block pool would be bounded by the BufferQueue. 340 bool bounded; 341 }; 342 Mutexed<Output> mOutput; 343 Mutexed<std::list<std::unique_ptr<C2Work>>> mFlushedConfigs; 344 345 std::atomic_uint64_t mFrameIndex; 346 std::atomic_uint64_t mFirstValidFrameIndex; 347 348 sp<MemoryDealer> makeMemoryDealer(size_t heapSize); 349 350 std::deque<TrackedFrame> mTrackedFrames; 351 bool mAreRenderMetricsEnabled; 352 bool mIsSurfaceToDisplay; 353 bool mHasPresentFenceTimes; 354 355 struct OutputSurface { 356 sp<Surface> surface; 357 uint32_t generation; 358 int maxDequeueBuffers; 359 std::map<uint64_t, int> rotation; 360 }; 361 Mutexed<OutputSurface> mOutputSurface; 362 int mRenderingDepth; 363 364 struct BlockPools { 365 C2Allocator::id_t inputAllocatorId; 366 std::shared_ptr<C2BlockPool> inputPool; 367 C2Allocator::id_t outputAllocatorId; 368 C2BlockPool::local_id_t outputPoolId; 369 std::shared_ptr<Codec2Client::Configurable> outputPoolIntf; 370 }; 371 Mutexed<BlockPools> mBlockPools; 372 373 std::shared_ptr<InputSurfaceWrapper> mInputSurface; 374 375 MetaMode mMetaMode; 376 377 Mutexed<PipelineWatcher> mPipelineWatcher; 378 379 std::atomic_bool mInputMetEos; 380 std::once_flag mRenderWarningFlag; 381 382 sp<ICrypto> mCrypto; 383 sp<IDescrambler> mDescrambler; 384 hasCryptoOrDescrambler()385 inline bool hasCryptoOrDescrambler() { 386 return mCrypto != nullptr || mDescrambler != nullptr; 387 } 388 std::atomic_bool mSendEncryptedInfoBuffer; 389 390 std::atomic_bool mTunneled; 391 }; 392 393 // Conversion of a c2_status_t value to a status_t value may depend on the 394 // operation that returns the c2_status_t value. 395 enum c2_operation_t { 396 C2_OPERATION_NONE, 397 C2_OPERATION_Component_connectToOmxInputSurface, 398 C2_OPERATION_Component_createBlockPool, 399 C2_OPERATION_Component_destroyBlockPool, 400 C2_OPERATION_Component_disconnectFromInputSurface, 401 C2_OPERATION_Component_drain, 402 C2_OPERATION_Component_flush, 403 C2_OPERATION_Component_queue, 404 C2_OPERATION_Component_release, 405 C2_OPERATION_Component_reset, 406 C2_OPERATION_Component_setOutputSurface, 407 C2_OPERATION_Component_start, 408 C2_OPERATION_Component_stop, 409 C2_OPERATION_ComponentStore_copyBuffer, 410 C2_OPERATION_ComponentStore_createComponent, 411 C2_OPERATION_ComponentStore_createInputSurface, 412 C2_OPERATION_ComponentStore_createInterface, 413 C2_OPERATION_Configurable_config, 414 C2_OPERATION_Configurable_query, 415 C2_OPERATION_Configurable_querySupportedParams, 416 C2_OPERATION_Configurable_querySupportedValues, 417 C2_OPERATION_InputSurface_connectToComponent, 418 C2_OPERATION_InputSurfaceConnection_disconnect, 419 }; 420 421 status_t toStatusT(c2_status_t c2s, c2_operation_t c2op = C2_OPERATION_NONE); 422 423 } // namespace android 424 425 #endif // CCODEC_BUFFER_CHANNEL_H_ 426