1 /* 2 * Copyright (C) 2023 Huawei Device Co., Ltd. 3 * Licensed under the Apache License, Version 2.0 (the "License"); 4 * you may not use this file except in compliance with the License. 5 * You may obtain a copy of the License at 6 * 7 * http://www.apache.org/licenses/LICENSE-2.0 8 * 9 * Unless required by applicable law or agreed to in writing, software 10 * distributed under the License is distributed on an "AS IS" BASIS, 11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 * See the License for the specific language governing permissions and 13 * limitations under the License. 14 */ 15 16 #ifndef HCODEC_HCODEC_H 17 #define HCODEC_HCODEC_H 18 19 #include <queue> 20 #include <functional> 21 #include "securec.h" 22 #include "OMX_Component.h" // third_party/openmax/api/1.1.2 23 #include "codecbase.h" 24 #include "avcodec_errors.h" 25 #include "avsharedmemorybase.h" // foundation/multimedia/av_codec/services/utils/include/ 26 #include "state_machine.h" 27 #include "v1_0/codec_types.h" 28 #include "v1_0/icodec_callback.h" 29 #include "v1_0/icodec_component.h" 30 #include "v1_0/icodec_component_manager.h" 31 #include "type_converter.h" 32 33 namespace OHOS::MediaAVCodec { 34 class HCodec : public CodecBase, protected StateMachine { 35 public: 36 static std::shared_ptr<HCodec> Create(const std::string &name); 37 int32_t SetCallback(const std::shared_ptr<AVCodecCallback> &callback) override; 38 int32_t Configure(const Format &format) override; 39 sptr<Surface> CreateInputSurface() override; 40 int32_t SetInputSurface(sptr<Surface> surface); 41 int32_t SetOutputSurface(sptr<Surface> surface) override; 42 43 int32_t QueueInputBuffer(uint32_t index, const AVCodecBufferInfo &info, AVCodecBufferFlag flag) override; 44 int32_t NotifyEos() override; 45 int32_t ReleaseOutputBuffer(uint32_t index) override; 46 int32_t RenderOutputBuffer(uint32_t index) override; 47 48 int32_t SignalRequestIDRFrame() override; 49 int32_t SetParameter(const Format& format) override; 50 int32_t GetInputFormat(Format& format) override; 51 int32_t GetOutputFormat(Format& format) override; 52 53 int32_t Start() override; 54 int32_t Stop() override; 55 int32_t Flush() override; 56 int32_t Reset() override; 57 int32_t Release() override; 58 59 protected: 60 enum MsgWhat : MsgType { 61 INIT, 62 SET_CALLBACK, 63 CONFIGURE, 64 CREATE_INPUT_SURFACE, 65 SET_INPUT_SURFACE, 66 SET_OUTPUT_SURFACE, 67 START, 68 GET_INPUT_FORMAT, 69 GET_OUTPUT_FORMAT, 70 SET_PARAMETERS, 71 REQUEST_IDR_FRAME, 72 FLUSH, 73 QUEUE_INPUT_BUFFER, 74 NOTIFY_EOS, 75 RELEASE_OUTPUT_BUFFER, 76 RENDER_OUTPUT_BUFFER, 77 STOP, 78 RELEASE, 79 80 INNER_MSG_BEGIN = 1000, 81 CODEC_EVENT, 82 OMX_EMPTY_BUFFER_DONE, 83 OMX_FILL_BUFFER_DONE, 84 GET_BUFFER_FROM_SURFACE, 85 CHECK_IF_STUCK, 86 FORCE_SHUTDOWN, 87 }; 88 89 enum BufferOperationMode { 90 KEEP_BUFFER, 91 RESUBMIT_BUFFER, 92 FREE_BUFFER, 93 }; 94 95 enum class BufferOwner { 96 OWNED_BY_US, 97 OWNED_BY_USER, 98 OWNED_BY_OMX, 99 OWNED_BY_SURFACE, 100 }; 101 102 enum class BufferType { 103 DYNAMIC_SURFACE_BUFFER, 104 PRESET_SURFACE_BUFFER, 105 PRESET_ASHM_BUFFER, 106 }; 107 108 struct PortInfo { 109 uint32_t width; 110 uint32_t height; 111 std::optional<uint32_t> stride; 112 OMX_VIDEO_CODINGTYPE codingType; 113 std::optional<PixelFmt> pixelFmt; 114 double frameRate; 115 std::optional<uint32_t> inputBufSize; 116 }; 117 118 enum DumpMode { 119 DUMP_NONE = 0, 120 DUMP_INPUT = 0b01, 121 DUMP_OUTPUT = 0b10, 122 }; 123 124 struct BufferInfo { 125 bool isInput; 126 BufferOwner owner; 127 std::optional<std::chrono::time_point<std::chrono::steady_clock>> lastOwnerChangeTime; 128 uint32_t bufferId; 129 std::shared_ptr<OHOS::HDI::Codec::V1_0::OmxCodecBuffer> omxBuffer; 130 sptr<SurfaceBuffer> surfaceBuffer; 131 std::shared_ptr<AVSharedMemoryBase> sharedBuffer; 132 bool isImageDataInSharedBuffer = false; 133 134 bool IsValidFrame() const; 135 void Dump(const std::string& prefix, DumpMode dumpMode, const std::optional<PortInfo>& bufferFormat) const; 136 void Dump(const std::string& prefix, const std::optional<PortInfo>& bufferFormat) const; 137 138 private: 139 void DumpSurfaceBuffer(const std::string& prefix) const; 140 void DecideDumpInfo(std::optional<uint32_t>& assumeAlignedH, std::string& suffix, bool& dumpAsVideo) const; 141 void DumpAshmemBuffer(const std::string& prefix, const std::optional<PortInfo>& bufferFormat) const; 142 static constexpr char DUMP_PATH[] = "/data/misc/hcodecdump"; 143 }; 144 145 protected: 146 HCodec(OHOS::HDI::Codec::V1_0::CodecCompCapability caps, OMX_VIDEO_CODINGTYPE codingType, bool isEncoder); 147 ~HCodec() override; 148 static const char* ToString(MsgWhat what); 149 static const char* ToString(BufferOwner owner); 150 void ReplyErrorCode(MsgId id, int32_t err); 151 void ChangeOwner(BufferInfo& info, BufferOwner targetOwner, bool printInfo = false); 152 153 // configure 154 virtual int32_t OnConfigure(const Format &format) = 0; 155 bool GetPixelFmtFromUser(const Format &format); 156 static std::optional<double> GetFrameRateFromUser(const Format &format); 157 int32_t SetVideoPortInfo(OMX_DIRTYPE portIndex, const PortInfo& info); 158 virtual int32_t UpdateInPortFormat() = 0; 159 virtual int32_t UpdateOutPortFormat() = 0; 160 void PrintPortDefinition(const OMX_PARAM_PORTDEFINITIONTYPE& def); 161 OnSetOutputSurface(const sptr<Surface> & surface)162 virtual int32_t OnSetOutputSurface(const sptr<Surface> &surface) { return AVCS_ERR_UNSUPPORT; } OnSetParameters(const Format & format)163 virtual int32_t OnSetParameters(const Format &format) { return AVCS_ERR_OK; } OnCreateInputSurface()164 virtual sptr<Surface> OnCreateInputSurface() { return nullptr; } OnSetInputSurface(sptr<Surface> & inputSurface)165 virtual int32_t OnSetInputSurface(sptr<Surface> &inputSurface) { return AVCS_ERR_UNSUPPORT; } RequestIDRFrame()166 virtual int32_t RequestIDRFrame() { return AVCS_ERR_UNSUPPORT; } 167 168 // start 169 virtual bool ReadyToStart() = 0; 170 virtual int32_t AllocateBuffersOnPort(OMX_DIRTYPE portIndex) = 0; 171 int32_t AllocateSharedBuffers(OMX_DIRTYPE portIndex, bool isImageData); 172 std::shared_ptr<OHOS::HDI::Codec::V1_0::OmxCodecBuffer> AshmemToOmxBuffer( 173 OMX_DIRTYPE portIndex, int32_t fd, uint32_t size); 174 175 virtual int32_t SubmitAllBuffersOwnedByUs() = 0; 176 virtual int32_t SubmitOutputBuffersToOmxNode() = 0; 177 BufferInfo* FindBufferInfoByID(OMX_DIRTYPE portIndex, uint32_t bufferId); 178 std::optional<size_t> FindBufferIndexByID(OMX_DIRTYPE portIndex, uint32_t bufferId); 179 void PrintAllBufferInfo(); 180 virtual void OnGetBufferFromSurface() = 0; 181 uint32_t UserFlagToOmxFlag(AVCodecBufferFlag userFlag); 182 AVCodecBufferFlag OmxFlagToUserFlag(uint32_t omxFlag); 183 184 // input buffer circulation 185 virtual void NotifyUserToFillThisInBuffer(BufferInfo &info); 186 virtual void OnQueueInputBuffer(const MsgInfo &msg, BufferOperationMode mode); 187 void OnQueueInputBuffer(BufferOperationMode mode, BufferInfo* info); 188 virtual void OnSignalEndOfInputStream(const MsgInfo &msg); 189 int32_t NotifyOmxToEmptyThisInBuffer(BufferInfo& info); 190 virtual void OnOMXEmptyBufferDone(uint32_t bufferId, BufferOperationMode mode) = 0; 191 192 // output buffer circulation 193 int32_t NotifyOmxToFillThisOutBuffer(BufferInfo &info); 194 void OnOMXFillBufferDone(const OHOS::HDI::Codec::V1_0::OmxCodecBuffer& omxBuffer, BufferOperationMode mode); 195 void OnOMXFillBufferDone(BufferOperationMode mode, BufferInfo& info, size_t bufferIdx); 196 void UpdateFbdRecord(const BufferInfo& info); 197 void NotifyUserOutBufferAvaliable(BufferInfo &info); 198 void OnReleaseOutputBuffer(const MsgInfo &msg, BufferOperationMode mode); 199 virtual void OnRenderOutputBuffer(const MsgInfo &msg, BufferOperationMode mode); 200 201 // stop/release 202 void ReclaimBuffer(OMX_DIRTYPE portIndex, BufferOwner owner); 203 bool IsAllBufferOwnedByUsOrSurface(OMX_DIRTYPE portIndex); 204 bool IsAllBufferOwnedByUsOrSurface(); 205 void EraseOutBuffersOwnedByUsOrSurface(); 206 void ClearBufferPool(OMX_DIRTYPE portIndex); 207 virtual void EraseBufferFromPool(OMX_DIRTYPE portIndex, size_t i) = 0; 208 void FreeOmxBuffer(OMX_DIRTYPE portIndex, const BufferInfo& info); 209 210 // template 211 template <typename T> InitOMXParam(T & param)212 static inline void InitOMXParam(T& param) 213 { 214 (void)memset_s(¶m, sizeof(T), 0x0, sizeof(T)); 215 param.nSize = sizeof(T); 216 param.nVersion.s.nVersionMajor = 1; 217 } 218 219 template <typename T> InitOMXParamExt(T & param)220 static inline void InitOMXParamExt(T& param) 221 { 222 (void)memset_s(¶m, sizeof(T), 0x0, sizeof(T)); 223 param.size = sizeof(T); 224 param.version.s.nVersionMajor = 1; 225 } 226 227 template <typename T> 228 bool GetParameter(uint32_t index, T& param, bool isCfg = false) 229 { 230 int8_t* p = reinterpret_cast<int8_t*>(¶m); 231 std::vector<int8_t> inVec(p, p + sizeof(T)); 232 std::vector<int8_t> outVec; 233 int32_t ret = isCfg ? compNode_->GetConfig(index, inVec, outVec) : 234 compNode_->GetParameter(index, inVec, outVec); 235 if (ret != HDF_SUCCESS) { 236 return false; 237 } 238 if (outVec.size() != sizeof(T)) { 239 return false; 240 } 241 ret = memcpy_s(¶m, sizeof(T), outVec.data(), outVec.size()); 242 if (ret != EOK) { 243 return false; 244 } 245 return true; 246 } 247 248 template <typename T> 249 bool SetParameter(uint32_t index, const T& param, bool isCfg = false) 250 { 251 const int8_t* p = reinterpret_cast<const int8_t*>(¶m); 252 std::vector<int8_t> inVec(p, p + sizeof(T)); 253 int32_t ret = isCfg ? compNode_->SetConfig(index, inVec) : 254 compNode_->SetParameter(index, inVec); 255 if (ret != HDF_SUCCESS) { 256 return false; 257 } 258 return true; 259 } 260 AlignTo(uint32_t side,uint32_t align)261 static inline uint32_t AlignTo(uint32_t side, uint32_t align) 262 { 263 if (align == 0) { 264 return side; 265 } 266 return (side + align - 1) / align * align; 267 } 268 269 protected: 270 OHOS::HDI::Codec::V1_0::CodecCompCapability caps_; 271 OMX_VIDEO_CODINGTYPE codingType_; 272 bool isEncoder_; 273 std::string componentName_; 274 std::string ctorTime_; 275 bool debugMode_ = false; 276 DumpMode dumpMode_ = DUMP_NONE; 277 sptr<OHOS::HDI::Codec::V1_0::ICodecCallback> compCb_ = nullptr; 278 sptr<OHOS::HDI::Codec::V1_0::ICodecComponent> compNode_ = nullptr; 279 sptr<OHOS::HDI::Codec::V1_0::ICodecComponentManager> compMgr_ = nullptr; 280 uint32_t componentId_ = 0; 281 282 std::shared_ptr<AVCodecCallback> callback_; 283 PixelFmt configuredFmt_; 284 std::shared_ptr<Format> configFormat_; 285 std::shared_ptr<Format> inputFormat_; 286 std::shared_ptr<Format> outputFormat_; 287 std::optional<PortInfo> sharedBufferFormat_; 288 289 std::vector<BufferInfo> inputBufferPool_; 290 std::vector<BufferInfo> outputBufferPool_; 291 bool isBufferCirculating_ = false; 292 bool inputPortEos_ = false; 293 bool outputPortEos_ = false; 294 uint64_t etbCnt_ = 0; 295 uint64_t fbdCnt_ = 0; 296 uint64_t totalCost_ = 0; 297 std::unordered_map<int64_t, std::chrono::time_point<std::chrono::steady_clock>> etbMap_; 298 std::chrono::time_point<std::chrono::steady_clock> firstFbdTime_; 299 300 static constexpr char BUFFER_ID[] = "buffer-id"; 301 static constexpr int TIME_RATIO_S_TO_MS = 1000; 302 static constexpr int TIME_RATIO_MS_TO_US = 1000; 303 static constexpr int TIME_RATIO_S_TO_US = 1000'000; 304 static constexpr uint32_t WAIT_FENCE_MS = 10; 305 306 private: 307 struct BaseState : State { 308 protected: 309 BaseState(HCodec *codec, const std::string &stateName, 310 BufferOperationMode inputMode = KEEP_BUFFER, BufferOperationMode outputMode = KEEP_BUFFER) StateBaseState311 : State(stateName), codec_(codec), inputMode_(inputMode), outputMode_(outputMode) {} 312 void OnMsgReceived(const MsgInfo &info) override; 313 void ReplyErrorCode(MsgId id, int32_t err); 314 void OnCodecEvent(const MsgInfo &info); 315 virtual void OnCodecEvent(OHOS::HDI::Codec::V1_0::CodecEventType event, uint32_t data1, uint32_t data2); 316 void OnGetFormat(const MsgInfo &info); 317 virtual void OnShutDown(const MsgInfo &info) = 0; 318 void OnCheckIfStuck(const MsgInfo &info); 319 void OnForceShutDown(const MsgInfo &info); 320 void OnStateExited() override { codec_->stateGeneration_++; } 321 322 protected: 323 HCodec *codec_; 324 BufferOperationMode inputMode_; 325 BufferOperationMode outputMode_; 326 }; 327 328 struct UninitializedState : BaseState { 329 explicit UninitializedState(HCodec *codec) : BaseState(codec, "Uninitialized") {} 330 private: 331 void OnStateEntered() override; 332 void OnMsgReceived(const MsgInfo &info) override; 333 int32_t OnAllocateComponent(const std::string &name); 334 void OnShutDown(const MsgInfo &info) override; 335 }; 336 337 struct InitializedState : BaseState { 338 explicit InitializedState(HCodec *codec) : BaseState(codec, "Initialized") {} 339 private: 340 void OnStateEntered() override; 341 void ProcessShutDownFromRunning(); 342 void OnMsgReceived(const MsgInfo &info) override; 343 void OnSetCallBack(const MsgInfo &info); 344 void OnConfigure(const MsgInfo &info); 345 void OnSetSurface(const MsgInfo &info, bool isInput); 346 void OnStart(const MsgInfo &info); 347 void OnShutDown(const MsgInfo &info) override; 348 }; 349 350 struct StartingState : BaseState { 351 explicit StartingState(HCodec *codec) : BaseState(codec, "Starting") {} 352 private: 353 void OnStateEntered() override; 354 void OnStateExited() override; 355 void OnMsgReceived(const MsgInfo &info) override; 356 int32_t AllocateBuffers(); 357 void OnCodecEvent(OHOS::HDI::Codec::V1_0::CodecEventType event, uint32_t data1, uint32_t data2) override; 358 void OnShutDown(const MsgInfo &info) override; 359 void ReplyStartMsg(int32_t errCode); 360 bool hasError_ = false; 361 }; 362 363 struct RunningState : BaseState { 364 explicit RunningState(HCodec *codec) : BaseState(codec, "Running", RESUBMIT_BUFFER, RESUBMIT_BUFFER) {} 365 private: 366 void OnStateEntered() override; 367 void OnMsgReceived(const MsgInfo &info) override; 368 void OnCodecEvent(OHOS::HDI::Codec::V1_0::CodecEventType event, uint32_t data1, uint32_t data2) override; 369 void OnShutDown(const MsgInfo &info) override; 370 void OnFlush(const MsgInfo &info); 371 void OnSetParameters(const MsgInfo &info); 372 }; 373 374 struct OutputPortChangedState : BaseState { 375 explicit OutputPortChangedState(HCodec *codec) 376 : BaseState(codec, "OutputPortChanged", RESUBMIT_BUFFER, FREE_BUFFER) {} 377 private: 378 void OnStateEntered() override; 379 void OnMsgReceived(const MsgInfo &info) override; 380 void OnCodecEvent(OHOS::HDI::Codec::V1_0::CodecEventType event, uint32_t data1, uint32_t data2) override; 381 void OnShutDown(const MsgInfo &info) override; 382 void HandleOutputPortDisabled(); 383 void HandleOutputPortEnabled(); 384 void OnFlush(const MsgInfo &info); 385 }; 386 387 struct FlushingState : BaseState { 388 explicit FlushingState(HCodec *codec) : BaseState(codec, "Flushing") {} 389 private: 390 void OnStateEntered() override; 391 void OnMsgReceived(const MsgInfo &info) override; 392 void OnCodecEvent(OHOS::HDI::Codec::V1_0::CodecEventType event, uint32_t data1, uint32_t data2) override; 393 void OnShutDown(const MsgInfo &info) override; 394 void ChangeStateIfWeOwnAllBuffers(); 395 bool IsFlushCompleteOnAllPorts(); 396 int32_t UpdateFlushStatusOnPorts(uint32_t data1, uint32_t data2); 397 bool flushCompleteFlag_[2] {false, false}; 398 }; 399 400 struct StoppingState : BaseState { 401 explicit StoppingState(HCodec *codec) : BaseState(codec, "Stopping"), 402 omxNodeInIdleState_(false), 403 omxNodeIsChangingToLoadedState_(false) {} 404 private: 405 void OnStateEntered() override; 406 void OnMsgReceived(const MsgInfo &info) override; 407 void OnCodecEvent(OHOS::HDI::Codec::V1_0::CodecEventType event, uint32_t data1, uint32_t data2) override; 408 void OnShutDown(const MsgInfo &info) override; 409 void ChangeStateIfWeOwnAllBuffers(); 410 void ChangeOmxNodeToLoadedState(bool forceToFreeBuffer); 411 bool omxNodeInIdleState_; 412 bool omxNodeIsChangingToLoadedState_; 413 }; 414 415 class HdiCallback : public OHOS::HDI::Codec::V1_0::ICodecCallback { 416 public: 417 explicit HdiCallback(HCodec* codec) : codec_(codec) { } 418 virtual ~HdiCallback() = default; 419 int32_t EventHandler(OHOS::HDI::Codec::V1_0::CodecEventType event, 420 const OHOS::HDI::Codec::V1_0::EventInfo& info); 421 int32_t EmptyBufferDone(int64_t appData, const OHOS::HDI::Codec::V1_0::OmxCodecBuffer& buffer); 422 int32_t FillBufferDone(int64_t appData, const OHOS::HDI::Codec::V1_0::OmxCodecBuffer& buffer); 423 private: 424 HCodec* codec_; 425 }; 426 427 private: 428 void InitCreationTime(); 429 int32_t DoSyncCall(MsgWhat msgType, std::function<void(ParamSP)> oper); 430 int32_t DoSyncCallAndGetReply(MsgWhat msgType, std::function<void(ParamSP)> oper, ParamSP &reply); 431 int32_t InitWithName(const std::string &name); 432 void ReleaseComponent(); 433 void CleanUpOmxNode(); 434 void ChangeOmxToTargetState(OHOS::HDI::Codec::V1_0::CodecStateType &state, 435 OHOS::HDI::Codec::V1_0::CodecStateType targetState); 436 bool RollOmxBackToLoaded(); 437 438 int32_t ForceShutdown(int32_t generation); 439 void SignalError(AVCodecErrorType errorType, int32_t errorCode); 440 void DeferMessage(const MsgInfo &info); 441 void ProcessDeferredMessages(); 442 void ReplyToSyncMsgLater(const MsgInfo& msg); 443 bool GetFirstSyncMsgToReply(MsgInfo& msg); 444 445 private: 446 static constexpr size_t MAX_HCODEC_BUFFER_SIZE = 8192 * 4096 * 4; // 8K RGBA 447 static constexpr uint32_t THREE_SECONDS_IN_US = 3'000'000; 448 static constexpr double FRAME_RATE_COEFFICIENT = 65536.0; 449 450 std::shared_ptr<UninitializedState> uninitializedState_; 451 std::shared_ptr<InitializedState> initializedState_; 452 std::shared_ptr<StartingState> startingState_; 453 std::shared_ptr<RunningState> runningState_; 454 std::shared_ptr<OutputPortChangedState> outputPortChangedState_; 455 std::shared_ptr<FlushingState> flushingState_; 456 std::shared_ptr<StoppingState> stoppingState_; 457 458 int32_t stateGeneration_ = 0; 459 bool isShutDownFromRunning_ = false; 460 bool notifyCallerAfterShutdownComplete_ = false; 461 bool keepComponentAllocated_ = false; 462 bool hasFatalError_ = false; 463 std::list<MsgInfo> deferredQueue_; 464 std::map<MsgType, std::queue<std::pair<MsgId, ParamSP>>> syncMsgToReply_; 465 }; // class HCodec 466 } // namespace OHOS::MediaAVCodec 467 #endif // HCODEC_HCODEC_H 468