1 /*
2 * Copyright (c) 2024 Huawei Device Co., Ltd.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at
6 *
7 * http://www.apache.org/licenses/LICENSE-2.0
8 *
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
14 */
15
16 #ifndef IMAGE_CODEC_H
17 #define IMAGE_CODEC_H
18
19 #include <queue>
20 #include <array>
21 #include <functional>
22 #include "securec.h"
23 #include "OMX_Component.h" // third_party/openmax/api/1.1.2
24 #include "param_bundle.h"
25 #include "image_codec_buffer.h"
26 #include "image_codec_common.h"
27 #include "format.h"
28 #include "state_machine.h"
29 #include "type_converter.h"
30 #include "codec_omx_ext.h"
31 #include "hdi_define.h"
32
33 namespace OHOS::ImagePlugin {
34 inline constexpr int TIME_RATIO_S_TO_MS = 1000;
35 inline constexpr double US_TO_MS = 1000.0;
36 inline constexpr double US_TO_S = 1000000.0;
37 inline constexpr uint32_t STRIDE_ALIGNMENT = 32;
38 // for demo: "/data/misc/imagecodecdump"
39 inline constexpr char DUMP_PATH[] = "/data/storage/el2/base/files";
40
GetYuv420Size(uint32_t w,uint32_t h)41 inline uint32_t GetYuv420Size(uint32_t w, uint32_t h)
42 {
43 return w * h * 3 / 2; // 3: nom of ratio, 2: denom of ratio
44 }
45
46 class ImageCodec : protected StateMachine {
47 public:
48 static std::shared_ptr<ImageCodec> Create();
GetComponentName()49 std::string GetComponentName() const { return componentName_; }
50 int32_t SetCallback(const std::shared_ptr<ImageCodecCallback> &callback);
51 int32_t Configure(const Format &format);
52 int32_t QueueInputBuffer(uint32_t index);
53 int32_t ReleaseOutputBuffer(uint32_t index);
54 int32_t GetInputFormat(Format& format);
55 int32_t GetOutputFormat(Format& format);
56 int32_t Start();
57 int32_t Release();
58 int32_t GetOutputBufferUsage(uint64_t& usage);
59 int32_t SetOutputBuffer(sptr<SurfaceBuffer> output);
60 int32_t GetPackedInputCapability(bool& flag);
61 int32_t SetPackedInputFlag(bool flag);
62 protected:
63 enum MsgWhat : MsgType {
64 INIT,
65 SET_CALLBACK,
66 CONFIGURE,
67 START,
68 GET_INPUT_FORMAT,
69 GET_OUTPUT_FORMAT,
70 QUEUE_INPUT_BUFFER,
71 RELEASE_OUTPUT_BUFFER,
72 RELEASE,
73 GET_OUTPUT_BUFFER_USAGE,
74 SET_OUTPUT_BUFFER,
75 GET_PACKED_CAPABILITY,
76 SET_PACKED_INPUT_FLAG,
77
78 INNER_MSG_BEGIN = 1000,
79 CODEC_EVENT,
80 OMX_EMPTY_BUFFER_DONE,
81 OMX_FILL_BUFFER_DONE,
82 CHECK_IF_STUCK,
83 FORCE_SHUTDOWN,
84 };
85
86 enum BufferOperationMode {
87 KEEP_BUFFER,
88 RESUBMIT_BUFFER,
89 FREE_BUFFER,
90 };
91
92 enum BufferOwner {
93 OWNED_BY_US = 0,
94 OWNED_BY_USER = 1,
95 OWNED_BY_OMX = 2,
96 OWNER_CNT = 3,
97 };
98
99 struct PortInfo {
100 uint32_t width;
101 uint32_t height;
102 OMX_VIDEO_CODINGTYPE codingType;
103 std::optional<PixelFmt> pixelFmt;
104 double frameRate;
105 std::optional<uint32_t> inputBufSize;
106 std::optional<uint32_t> bufferCnt;
107 };
108
109 struct BufferInfo {
BufferInfoBufferInfo110 BufferInfo() : lastOwnerChangeTime(std::chrono::steady_clock::now()) {}
111 bool isInput = true;
112 BufferOwner owner = OWNED_BY_US;
113 std::chrono::time_point<std::chrono::steady_clock> lastOwnerChangeTime;
114 uint32_t bufferId = 0;
115 std::shared_ptr<HdiCodecNamespace::OmxCodecBuffer> omxBuffer;
116 sptr<SurfaceBuffer> surfaceBuffer;
117 std::shared_ptr<ImageCodecBuffer> imgCodecBuffer;
118
119 void CleanUpUnusedInfo();
120 void BeginCpuAccess();
121 void EndCpuAccess();
122 bool IsValidFrame() const;
123 void Dump(const std::string& prefix, bool dumpMode) const;
124
125 private:
126 void Dump(const std::string& prefix) const;
127 void DumpSurfaceBuffer(const std::string& prefix) const;
128 void DumpLinearBuffer(const std::string& prefix) const;
129 };
130 protected:
131 ImageCodec(OMX_VIDEO_CODINGTYPE codingType, bool isEncoder);
132 ~ImageCodec() override;
133 static const char* ToString(MsgWhat what);
134 static const char* ToString(BufferOwner owner);
135 void ReplyErrorCode(MsgId id, int32_t err);
136 void PrintAllBufferInfo();
137 std::array<uint32_t, OWNER_CNT> CountOwner(bool isInput);
138 void ChangeOwner(BufferInfo& info, BufferOwner newOwner);
139 void UpdateInputRecord(const BufferInfo& info, std::chrono::time_point<std::chrono::steady_clock> now);
140 void UpdateOutputRecord(const BufferInfo& info, std::chrono::time_point<std::chrono::steady_clock> now);
141
142 // configure
143 virtual int32_t OnConfigure(const Format &format) = 0;
144 bool GetPixelFmtFromUser(const Format &format);
145 static std::optional<double> GetFrameRateFromUser(const Format &format);
146 int32_t SetVideoPortInfo(OMX_DIRTYPE portIndex, const PortInfo& info);
147 virtual int32_t UpdateInPortFormat() = 0;
148 virtual int32_t UpdateOutPortFormat() = 0;
UpdateColorAspects()149 virtual void UpdateColorAspects() {}
150 void PrintPortDefinition(const OMX_PARAM_PORTDEFINITIONTYPE& def);
151 int32_t SetFrameRateAdaptiveMode(const Format &format);
152 int32_t SetProcessName(const Format &format);
153 virtual int32_t ReConfigureOutputBufferCnt() = 0;
154 virtual uint64_t OnGetOutputBufferUsage() = 0;
155 virtual int32_t OnSetOutputBuffer(sptr<SurfaceBuffer> output) = 0;
156 virtual bool OnGetPackedInputCapability() = 0;
157 virtual bool OnSetPackedInputFlag(bool flag) = 0;
158
159 // start
160 virtual bool ReadyToStart() = 0;
161 virtual int32_t AllocateBuffersOnPort(OMX_DIRTYPE portIndex, bool isOutputPortSettingChanged) = 0;
162 virtual void UpdateFormatFromSurfaceBuffer() = 0;
163 int32_t GetPortDefinition(OMX_DIRTYPE portIndex, OMX_PARAM_PORTDEFINITIONTYPE& def);
164 int32_t AllocateSurfaceBuffers(OMX_DIRTYPE portIndex, bool isOutputPortSettingChanged,
165 sptr<SurfaceBuffer> output = nullptr);
166 int32_t AllocateHardwareBuffers(OMX_DIRTYPE portIndex);
167 std::shared_ptr<HdiCodecNamespace::OmxCodecBuffer> SurfaceBufferToOmxBuffer(
168 const sptr<SurfaceBuffer>& surfaceBuffer);
169 std::shared_ptr<HdiCodecNamespace::OmxCodecBuffer> DynamicSurfaceBufferToOmxBuffer();
170
171 virtual int32_t SubmitAllBuffersOwnedByUs() = 0;
SubmitOutputBuffersToOmxNode()172 virtual int32_t SubmitOutputBuffersToOmxNode() { return IC_ERR_UNSUPPORT; }
173 BufferInfo* FindBufferInfoByID(OMX_DIRTYPE portIndex, uint32_t bufferId);
174 std::optional<size_t> FindBufferIndexByID(OMX_DIRTYPE portIndex, uint32_t bufferId);
175
176 // input buffer circulation
177 virtual void NotifyUserToFillThisInBuffer(BufferInfo &info);
178 virtual void OnQueueInputBuffer(const MsgInfo &msg, BufferOperationMode mode);
179 void OnQueueInputBuffer(BufferOperationMode mode, BufferInfo* info);
180 int32_t NotifyOmxToEmptyThisInBuffer(BufferInfo& info);
181 virtual void OnOMXEmptyBufferDone(uint32_t bufferId, BufferOperationMode mode) = 0;
182
183 // output buffer circulation
184 int32_t NotifyOmxToFillThisOutBuffer(BufferInfo &info);
185 void OnOMXFillBufferDone(const HdiCodecNamespace::OmxCodecBuffer& omxBuffer, BufferOperationMode mode);
186 void OnOMXFillBufferDone(BufferOperationMode mode, BufferInfo& info, size_t bufferIdx);
187 void NotifyUserOutBufferAvaliable(BufferInfo &info);
188 void OnReleaseOutputBuffer(const MsgInfo &msg, BufferOperationMode mode);
189
190 // // stop/release
191 void ReclaimBuffer(OMX_DIRTYPE portIndex, BufferOwner owner, bool erase = false);
192 bool IsAllBufferOwnedByUs(OMX_DIRTYPE portIndex);
193 bool IsAllBufferOwnedByUs();
194 void EraseOutBuffersOwnedByUs();
195 void ClearBufferPool(OMX_DIRTYPE portIndex);
196 virtual void EraseBufferFromPool(OMX_DIRTYPE portIndex, size_t i) = 0;
197 void FreeOmxBuffer(OMX_DIRTYPE portIndex, const BufferInfo& info);
198
199 // template
200 template <typename T>
InitOMXParam(T & param)201 static inline void InitOMXParam(T& param)
202 {
203 (void)memset_s(¶m, sizeof(T), 0x0, sizeof(T));
204 param.nSize = sizeof(T);
205 param.nVersion.s.nVersionMajor = 1;
206 }
207
208 template <typename T>
InitOMXParamExt(T & param)209 static inline void InitOMXParamExt(T& param)
210 {
211 (void)memset_s(¶m, sizeof(T), 0x0, sizeof(T));
212 param.size = sizeof(T);
213 param.version.s.nVersionMajor = 1;
214 }
215
216 template <typename T>
217 bool GetParameter(uint32_t index, T& param, bool isCfg = false)
218 {
219 int8_t* p = reinterpret_cast<int8_t*>(¶m);
220 std::vector<int8_t> inVec(p, p + sizeof(T));
221 std::vector<int8_t> outVec;
222 int32_t ret = isCfg ? compNode_->GetConfig(index, inVec, outVec) :
223 compNode_->GetParameter(index, inVec, outVec);
224 if (ret != HDF_SUCCESS) {
225 return false;
226 }
227 if (outVec.size() != sizeof(T)) {
228 return false;
229 }
230 ret = memcpy_s(¶m, sizeof(T), outVec.data(), outVec.size());
231 if (ret != EOK) {
232 return false;
233 }
234 return true;
235 }
236
237 template <typename T>
238 bool SetParameter(uint32_t index, const T& param, bool isCfg = false)
239 {
240 const int8_t* p = reinterpret_cast<const int8_t*>(¶m);
241 std::vector<int8_t> inVec(p, p + sizeof(T));
242 int32_t ret = isCfg ? compNode_->SetConfig(index, inVec) :
243 compNode_->SetParameter(index, inVec);
244 if (ret != HDF_SUCCESS) {
245 return false;
246 }
247 return true;
248 }
249
250 protected:
251 bool isEncoder_;
252 OMX_VIDEO_CODINGTYPE codingType_;
253 uint32_t componentId_ = 0;
254 std::string componentName_;
255 std::string compUniqueStr_;
256 bool is10Bit_ = false;
257 bool debugMode_ = false;
258 bool dumpMode_ = false;
259 bool isPackedInputSupported_ = false;
260 sptr<HdiCodecNamespace::ICodecCallback> compCb_ = nullptr;
261 sptr<HdiCodecNamespace::ICodecComponent> compNode_ = nullptr;
262 sptr<HdiCodecNamespace::ICodecComponentManager> compMgr_ = nullptr;
263
264 std::shared_ptr<ImageCodecCallback> callback_;
265 PixelFmt configuredFmt_{};
266 BufferRequestConfig requestCfg_{};
267 std::shared_ptr<Format> configFormat_;
268 std::shared_ptr<Format> inputFormat_;
269 std::shared_ptr<Format> outputFormat_;
270
271 std::vector<BufferInfo> inputBufferPool_;
272 std::vector<BufferInfo> outputBufferPool_;
273 bool isBufferCirculating_ = false;
274 bool inputPortEos_ = false;
275 bool outputPortEos_ = false;
276
277 struct TotalCntAndCost {
278 uint64_t totalCnt = 0;
279 uint64_t totalCostUs = 0;
280 };
281 std::array<std::array<TotalCntAndCost, OWNER_CNT>, OWNER_CNT> inputHoldTimeRecord_;
282 std::chrono::time_point<std::chrono::steady_clock> firstInTime_;
283 uint64_t inTotalCnt_ = 0;
284 std::array<std::array<TotalCntAndCost, OWNER_CNT>, OWNER_CNT> outputHoldTimeRecord_;
285 std::chrono::time_point<std::chrono::steady_clock> firstOutTime_;
286 TotalCntAndCost outRecord_;
287 std::unordered_map<int64_t, std::chrono::time_point<std::chrono::steady_clock>> inTimeMap_;
288
289 static constexpr char BUFFER_ID[] = "buffer-id";
290 static constexpr uint32_t WAIT_FENCE_MS = 1000;
291 private:
292 struct BaseState : State {
293 protected:
294 BaseState(ImageCodec *codec, const std::string &stateName,
295 BufferOperationMode inputMode = KEEP_BUFFER, BufferOperationMode outputMode = KEEP_BUFFER)
StateBaseState296 : State(stateName), codec_(codec), inputMode_(inputMode), outputMode_(outputMode) {}
297 void OnMsgReceived(const MsgInfo &info) override;
298 void ReplyErrorCode(MsgId id, int32_t err);
299 void OnCodecEvent(const MsgInfo &info);
300 virtual void OnCodecEvent(HdiCodecNamespace::CodecEventType event, uint32_t data1, uint32_t data2);
301 void OnGetFormat(const MsgInfo &info);
302 virtual void OnShutDown(const MsgInfo &info) = 0;
303 void OnCheckIfStuck(const MsgInfo &info);
304 void OnForceShutDown(const MsgInfo &info);
OnStateExitedBaseState305 void OnStateExited() override { codec_->stateGeneration_++; }
306
307 protected:
308 ImageCodec *codec_;
309 BufferOperationMode inputMode_;
310 BufferOperationMode outputMode_;
311 };
312
313 struct UninitializedState : BaseState {
UninitializedStateUninitializedState314 explicit UninitializedState(ImageCodec *codec) : BaseState(codec, "Uninitialized") {}
315 private:
316 void OnStateEntered() override;
317 void OnMsgReceived(const MsgInfo &info) override;
318 int32_t OnAllocateComponent(const std::string &name);
319 void OnShutDown(const MsgInfo &info) override;
320 };
321
322 struct InitializedState : BaseState {
InitializedStateInitializedState323 explicit InitializedState(ImageCodec *codec) : BaseState(codec, "Initialized") {}
324 private:
325 void OnStateEntered() override;
326 void ProcessShutDownFromRunning();
327 void OnMsgReceived(const MsgInfo &info) override;
328 void OnSetCallBack(const MsgInfo &info);
329 void OnConfigure(const MsgInfo &info);
330 void OnGetOutputBufferUsage(const MsgInfo &info);
331 void OnSetOutputBuffer(const MsgInfo &info);
332 void OnGetPackedInputCapability(const MsgInfo &info);
333 void OnSetPackedInputFlag(const MsgInfo &info);
334 void OnStart(const MsgInfo &info);
335 void OnShutDown(const MsgInfo &info) override;
336 };
337
338 struct StartingState : BaseState {
StartingStateStartingState339 explicit StartingState(ImageCodec *codec) : BaseState(codec, "Starting") {}
340 private:
341 void OnStateEntered() override;
342 void OnStateExited() override;
343 void OnMsgReceived(const MsgInfo &info) override;
344 int32_t AllocateBuffers();
345 void OnCodecEvent(HdiCodecNamespace::CodecEventType event, uint32_t data1, uint32_t data2) override;
346 void OnShutDown(const MsgInfo &info) override;
347 void ReplyStartMsg(int32_t errCode);
348 bool hasError_ = false;
349 };
350
351 struct RunningState : BaseState {
RunningStateRunningState352 explicit RunningState(ImageCodec *codec) : BaseState(codec, "Running", RESUBMIT_BUFFER, RESUBMIT_BUFFER) {}
353 private:
354 void OnStateEntered() override;
355 void OnMsgReceived(const MsgInfo &info) override;
356 void OnCodecEvent(HdiCodecNamespace::CodecEventType event, uint32_t data1, uint32_t data2) override;
357 void OnShutDown(const MsgInfo &info) override;
358 };
359
360 struct OutputPortChangedState : BaseState {
OutputPortChangedStateOutputPortChangedState361 explicit OutputPortChangedState(ImageCodec *codec)
362 : BaseState(codec, "OutputPortChanged", RESUBMIT_BUFFER, FREE_BUFFER) {}
363 private:
364 void OnStateEntered() override;
365 void OnMsgReceived(const MsgInfo &info) override;
366 void OnCodecEvent(HdiCodecNamespace::CodecEventType event, uint32_t data1, uint32_t data2) override;
367 void OnShutDown(const MsgInfo &info) override;
368 void HandleOutputPortDisabled();
369 void HandleOutputPortEnabled();
370 };
371
372 struct StoppingState : BaseState {
StoppingStateStoppingState373 explicit StoppingState(ImageCodec *codec) : BaseState(codec, "Stopping"),
374 omxNodeInIdleState_(false),
375 omxNodeIsChangingToLoadedState_(false) {}
376 private:
377 void OnStateEntered() override;
378 void OnMsgReceived(const MsgInfo &info) override;
379 void OnCodecEvent(HdiCodecNamespace::CodecEventType event, uint32_t data1, uint32_t data2) override;
380 void OnShutDown(const MsgInfo &info) override;
381 void ChangeStateIfWeOwnAllBuffers();
382 void ChangeOmxNodeToLoadedState(bool forceToFreeBuffer);
383 bool omxNodeInIdleState_;
384 bool omxNodeIsChangingToLoadedState_;
385 };
386
387 class HdiCallback : public HdiCodecNamespace::ICodecCallback {
388 public:
HdiCallback(ImageCodec * codec)389 explicit HdiCallback(ImageCodec* codec) : codec_(codec) { }
390 virtual ~HdiCallback() = default;
391 int32_t EventHandler(HdiCodecNamespace::CodecEventType event, const HdiCodecNamespace::EventInfo& info);
392 int32_t EmptyBufferDone(int64_t appData, const HdiCodecNamespace::OmxCodecBuffer& buffer);
393 int32_t FillBufferDone(int64_t appData, const HdiCodecNamespace::OmxCodecBuffer& buffer);
394 private:
395 ImageCodec* codec_;
396 };
397 private:
398 int32_t DoSyncCall(MsgWhat msgType, std::function<void(ParamSP)> oper);
399 int32_t DoSyncCallAndGetReply(MsgWhat msgType, std::function<void(ParamSP)> oper, ParamSP &reply);
400 int32_t InitWithName(const std::string &name);
401 void ReleaseComponent();
402 void CleanUpOmxNode();
403 void ChangeOmxToTargetState(HdiCodecNamespace::CodecStateType &state,
404 HdiCodecNamespace::CodecStateType targetState);
405 bool RollOmxBackToLoaded();
406
407 int32_t ForceShutdown(int32_t generation);
408 void SignalError(ImageCodecError err);
409 void DeferMessage(const MsgInfo &info);
410 void ProcessDeferredMessages();
411 void ReplyToSyncMsgLater(const MsgInfo& msg);
412 bool GetFirstSyncMsgToReply(MsgInfo& msg);
413
414 private:
415 static constexpr size_t MAX_IMAGE_CODEC_BUFFER_SIZE = 8192 * 4096 * 4; // 8K RGBA
416 static constexpr uint32_t THREE_SECONDS_IN_US = 3'000'000;
417 static constexpr double FRAME_RATE_COEFFICIENT = 65536.0;
418
419 std::shared_ptr<UninitializedState> uninitializedState_;
420 std::shared_ptr<InitializedState> initializedState_;
421 std::shared_ptr<StartingState> startingState_;
422 std::shared_ptr<RunningState> runningState_;
423 std::shared_ptr<OutputPortChangedState> outputPortChangedState_;
424 std::shared_ptr<StoppingState> stoppingState_;
425
426 int32_t stateGeneration_ = 0;
427 bool isShutDownFromRunning_ = false;
428 bool notifyCallerAfterShutdownComplete_ = false;
429 bool hasFatalError_ = false;
430 std::list<MsgInfo> deferredQueue_;
431 std::map<MsgType, std::queue<std::pair<MsgId, ParamSP>>> syncMsgToReply_;
432 }; // class ImageCodec
433 } // namespace OHOS::ImagePlugin
434
435 #endif // IMAGE_CODEC_H