1 /*
2 * Copyright (c) 2022-2025 Huawei Device Co., Ltd.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at
6 *
7 * http://www.apache.org/licenses/LICENSE-2.0
8 *
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
14 */
15
16 #include "fps_controller_process.h"
17
18 #include "dcamera_utils_tools.h"
19 #include "distributed_camera_errno.h"
20 #include "distributed_hardware_log.h"
21
22 namespace OHOS {
23 namespace DistributedHardware {
~FpsControllerProcess()24 FpsControllerProcess::~FpsControllerProcess()
25 {
26 if (isFpsControllerProcess_) {
27 DHLOGD("~DecodeDataProcess : ReleaseProcessNode.");
28 ReleaseProcessNode();
29 }
30 }
31
InitNode(const VideoConfigParams & sourceConfig,const VideoConfigParams & targetConfig,VideoConfigParams & processedConfig)32 int32_t FpsControllerProcess::InitNode(const VideoConfigParams& sourceConfig, const VideoConfigParams& targetConfig,
33 VideoConfigParams& processedConfig)
34 {
35 if (targetConfig.GetFrameRate() > MAX_TARGET_FRAME_RATE) {
36 DHLOGE("The target framerate : %{public}d is greater than the max framerate : %{public}d.",
37 targetConfig.GetFrameRate(), MAX_TARGET_FRAME_RATE);
38 return DCAMERA_BAD_TYPE;
39 }
40 sourceConfig_ = sourceConfig;
41 targetConfig_ = targetConfig;
42 targetFrameRate_ = targetConfig_.GetFrameRate();
43
44 processedConfig_ = sourceConfig;
45 processedConfig = processedConfig_;
46 isFpsControllerProcess_ = true;
47 return DCAMERA_OK;
48 }
49
ReleaseProcessNode()50 void FpsControllerProcess::ReleaseProcessNode()
51 {
52 DHLOGD("Start release [%{public}zu] node : FPS controller.", nodeRank_);
53 isFpsControllerProcess_ = false;
54 isFirstFrame_ = false;
55 targetFrameRate_ = 0;
56 lastFrameIncomeTimeMs_ = 0;
57 recentFrameTimeSpanMs_ = -1;
58 keepCorrectionCount_ = 0;
59 keepLessThanDoubleCount_ = 0;
60 keepMoreThanDoubleCount_ = 0;
61 frameRateCorrectionFactor_ = 0.0;
62 frameRateOvershootMdf_ = 0;
63 for (int i = 0; i < INCOME_FRAME_TIME_HISTORY_WINDOWS_SIZE; i++) {
64 incomingFrameTimesMs_[i] = 0;
65 }
66
67 if (nextDataProcess_ != nullptr) {
68 nextDataProcess_->ReleaseProcessNode();
69 nextDataProcess_ = nullptr;
70 }
71 DHLOGD("Release [%{public}zu] node : FPS controller end.", nodeRank_);
72 }
73
ProcessData(std::vector<std::shared_ptr<DataBuffer>> & inputBuffers)74 int32_t FpsControllerProcess::ProcessData(std::vector<std::shared_ptr<DataBuffer>>& inputBuffers)
75 {
76 if (inputBuffers.empty() || inputBuffers[0] == nullptr) {
77 DHLOGE("Data buffers is null.");
78 return DCAMERA_BAD_TYPE;
79 }
80 if (!isFpsControllerProcess_) {
81 DHLOGE("Decoder node occurred error.");
82 return DCAMERA_DISABLE_PROCESS;
83 }
84 int64_t timeStampUs = 0;
85 if (!inputBuffers[0]->FindInt64("timeUs", timeStampUs)) {
86 DHLOGE("Find decoder output timestamp failed.");
87 return DCAMERA_BAD_TYPE;
88 }
89
90 std::lock_guard<std::mutex> lck (mtx);
91 int64_t nowTimeMs = GetNowTimeStampMs();
92 UpdateFPSControllerInfo(nowTimeMs);
93
94 float curFrameRate = CalculateFrameRate(nowTimeMs);
95 if (IsDropFrame(curFrameRate)) {
96 DHLOGD("frame control, currect frameRate %{public}f, targetRate %{public}d, drop it",
97 curFrameRate, targetFrameRate_);
98 return DCAMERA_OK;
99 }
100
101 DHLOGD("frame control render PushVideoFrame, frame info width %{public}d height %{public}d, timeStampUs "
102 "%{public}lld, fps %{public}f", sourceConfig_.GetWidth(), sourceConfig_.GetHeight(), (long long)timeStampUs,
103 curFrameRate);
104 return FpsControllerDone(inputBuffers);
105 }
106
UpdateFPSControllerInfo(int64_t nowMs)107 void FpsControllerProcess::UpdateFPSControllerInfo(int64_t nowMs)
108 {
109 DHLOGD("Frame control, update control info.");
110 if (targetFrameRate_ <= 0) {
111 DHLOGD("Frame control, targetFrameRate_ : %{public}d", targetFrameRate_);
112 return;
113 }
114
115 isFirstFrame_ = false;
116 if (lastFrameIncomeTimeMs_ == 0) {
117 DHLOGD("Frame control, income fisrt frame.");
118 isFirstFrame_ = true;
119 }
120 lastFrameIncomeTimeMs_ = nowMs;
121 recentFrameTimeSpanMs_ = nowMs - lastFrameIncomeTimeMs_;
122 DHLOGD("Frame control, lastFrameIncomeTimeMs_ %{public}lld, receive Frame after last frame(ms): %{public}lld",
123 (long long)lastFrameIncomeTimeMs_, (long long)recentFrameTimeSpanMs_);
124 UpdateIncomingFrameTimes(nowMs);
125 UpdateFrameRateCorrectionFactor(nowMs);
126 return;
127 }
128
UpdateFrameRateCorrectionFactor(int64_t nowMs)129 void FpsControllerProcess::UpdateFrameRateCorrectionFactor(int64_t nowMs)
130 {
131 DHLOGD("Frame control, update FPS correction factor.");
132 if (targetFrameRate_ <= 0) {
133 DHLOGD("Frame control, targetFrameRate_ : %{public}d", targetFrameRate_);
134 return;
135 }
136 if (isFirstFrame_) {
137 DHLOGD("No frame rate correction factor when the first frame.");
138 return;
139 }
140
141 const float minDropFrmValue = 0.5;
142 const float maxDropFrmValue = 1.0;
143 const float msPerSecond = 1000;
144 const float maxInstantaneousFrameRateCoefficient = 1.1;
145 float maxInstantaneousFrameRateThreshold = targetFrameRate_ * maxInstantaneousFrameRateCoefficient;
146 if (recentFrameTimeSpanMs_ == 0) {
147 DHLOGE("Frame control, recentFrameTimeSpanMs_ is 0, cannot calculate frame rate.");
148 return;
149 }
150 float instantaneousFrameRate = msPerSecond / recentFrameTimeSpanMs_;
151 if (instantaneousFrameRate < 0) {
152 instantaneousFrameRate = -instantaneousFrameRate;
153 }
154 if (instantaneousFrameRate <= maxInstantaneousFrameRateThreshold) {
155 frameRateCorrectionFactor_ = minDropFrmValue;
156 } else {
157 if (keepCorrectionCount_ >= VIDEO_FRAME_DROP_INTERVAL) {
158 frameRateCorrectionFactor_ = maxDropFrmValue;
159 keepCorrectionCount_ = 0;
160 } else {
161 frameRateCorrectionFactor_ = 0;
162 keepCorrectionCount_++;
163 }
164 DHLOGD("Frame control, instantaneousFrameRate %{public}.3f is more than maxInstantaneousFrameRateThreshold "
165 "%{public}.3f, keepCorrectionCount %{public}d", instantaneousFrameRate,
166 maxInstantaneousFrameRateThreshold, keepCorrectionCount_);
167 }
168
169 DHLOGD("Frame control, targetFramerate %{public}d, maxInstantaneousFrameRateThreshold %{public}.3f,"
170 "instantaneousFrameRate %{public}.3f, frameRateCorrectionFactor %{public}.3f", targetFrameRate_,
171 maxInstantaneousFrameRateThreshold, instantaneousFrameRate, frameRateCorrectionFactor_);
172 return;
173 }
174
UpdateIncomingFrameTimes(int64_t nowMs)175 void FpsControllerProcess::UpdateIncomingFrameTimes(int64_t nowMs)
176 {
177 DHLOGD("Frame control, update incoming frame times array.");
178 if (targetFrameRate_ <= 0) {
179 DHLOGD("Frame control, targetFrameRate_ : %{public}d", targetFrameRate_);
180 return;
181 }
182 if (isFirstFrame_) {
183 incomingFrameTimesMs_[0] = nowMs;
184 return;
185 }
186
187 int64_t intervalNewAndFirst = nowMs - incomingFrameTimesMs_[0];
188 if (intervalNewAndFirst < 0) {
189 intervalNewAndFirst = -intervalNewAndFirst;
190 }
191 if (intervalNewAndFirst > FRMAE_MAX_INTERVAL_TIME_WINDOW_MS) {
192 DHLOGD("frame control, nowMs: %{public}lld mIncomingFrameT[0]: %{public}lld intervalNewAndFirst: "
193 "%{public}lld", (long long)nowMs, (long long)incomingFrameTimesMs_[0], (long long)intervalNewAndFirst);
194 for (int i = 0; i < INCOME_FRAME_TIME_HISTORY_WINDOWS_SIZE; i++) {
195 incomingFrameTimesMs_[i] = 0;
196 }
197 } else {
198 DHLOGD("frame control shift, nowMs: %{public}lld mIncomingFrameT[0]: %{public}lld intervalNewAndFirst: "
199 "%{public}lld", (long long)nowMs, (long long)incomingFrameTimesMs_[0], (long long)intervalNewAndFirst);
200 const int32_t windowLeftNum = 2;
201 for (int i = (INCOME_FRAME_TIME_HISTORY_WINDOWS_SIZE - windowLeftNum); i >= 0; --i) {
202 incomingFrameTimesMs_[i + 1] = incomingFrameTimesMs_[i];
203 }
204 }
205 incomingFrameTimesMs_[0] = nowMs;
206 return;
207 }
208
CalculateFrameRate(int64_t nowMs)209 float FpsControllerProcess::CalculateFrameRate(int64_t nowMs)
210 {
211 DHLOGD("Frame control, calculate frame rate.");
212 if (targetFrameRate_ <= 0) {
213 DHLOGE("Frame control, targetFrameRate_ : %{public}d", targetFrameRate_);
214 return 0.0;
215 }
216
217 int32_t num = 0;
218 int32_t validFramesNumber = 0;
219 if (nowMs < 0) {
220 nowMs = -nowMs;
221 }
222 for (; num < INCOME_FRAME_TIME_HISTORY_WINDOWS_SIZE; num++) {
223 if (incomingFrameTimesMs_[num] <= 0 || nowMs - incomingFrameTimesMs_[num] > FRAME_HISTORY_TIME_WINDOWS_MS) {
224 break;
225 } else {
226 validFramesNumber++;
227 }
228 }
229
230 const float msPerSecond = 1000;
231 const int32_t minValidCalculatedFrameRatesNum = 2;
232 int32_t minIncomingFrameNum = targetFrameRate_ / MIN_INCOME_FRAME_NUM_COEFFICIENT;
233 if (validFramesNumber > minIncomingFrameNum && validFramesNumber > minValidCalculatedFrameRatesNum) {
234 int64_t validTotalTimeInterval = 0;
235 if (num > 0) {
236 validTotalTimeInterval = (nowMs - incomingFrameTimesMs_[num - 1]);
237 }
238 if (validTotalTimeInterval < 0) {
239 validTotalTimeInterval = -validTotalTimeInterval;
240 }
241 if (validTotalTimeInterval > 0) {
242 return validFramesNumber * msPerSecond / validTotalTimeInterval + frameRateCorrectionFactor_;
243 }
244 }
245 return static_cast<float>(validFramesNumber);
246 }
247
IsDropFrame(float incomingFps)248 bool FpsControllerProcess::IsDropFrame(float incomingFps)
249 {
250 DHLOGD("Frame control, IsDropFrame");
251 if (targetFrameRate_ == 0) {
252 DHLOGD("target fps is 0, drop all frame.");
253 return true;
254 }
255 if (incomingFps <= 0) {
256 DHLOGD("incoming fps not more than 0, not drop");
257 return false;
258 }
259 const int32_t incomingFrmRate = static_cast<int32_t>(incomingFps);
260 if (incomingFrmRate > targetFrameRate_) {
261 DHLOGD("incoming fps not more than targetFrameRate_, not drop");
262 return false;
263 }
264 bool isDrop = ReduceFrameRateByUniformStrategy(incomingFrmRate);
265 DHLOGD("drop frame result: %{public}s", isDrop ? "drop" : "no drop");
266 return isDrop;
267 }
268
ReduceFrameRateByUniformStrategy(int32_t incomingFrmRate)269 bool FpsControllerProcess::ReduceFrameRateByUniformStrategy(int32_t incomingFrmRate)
270 {
271 DHLOGD("Frame control, reduce frame rate by uniform rate strategy");
272 if (incomingFrmRate <= targetFrameRate_) {
273 DHLOGD("incoming fps not more than targetFrameRate_, not drop");
274 return false;
275 }
276
277 /*
278 * When the actual incoming frame rate correction value is greater than the target frame
279 * rate, the incoming frames are reduced uniformly.
280 */
281 bool isDrop = false;
282 if (frameRateOvershootMdf_ > INT32_MAX - (incomingFrmRate - targetFrameRate_)) {
283 DHLOGE("Overshoot value is too large, reset to 0.");
284 return false;
285 }
286 int32_t overshoot = frameRateOvershootMdf_ + (incomingFrmRate - targetFrameRate_);
287 if (overshoot < 0) {
288 overshoot = 0;
289 frameRateOvershootMdf_ = 0;
290 }
291 if (overshoot && DOUBLE_MULTIPLE * overshoot < incomingFrmRate) {
292 /*
293 * When the actual input frame rate is less than or equal to twice the target frame rate,
294 * one frame is dropped every (incomingFrmRate / overshoot) frames.
295 */
296 if (keepMoreThanDoubleCount_) {
297 keepMoreThanDoubleCount_ = 0;
298 return true;
299 }
300 const int32_t dropVar = incomingFrmRate / overshoot;
301 if (keepLessThanDoubleCount_ >= dropVar) {
302 isDrop = true;
303 frameRateOvershootMdf_ = -(incomingFrmRate % overshoot) / OVERSHOOT_MODIFY_COEFFICIENT;
304 keepLessThanDoubleCount_ = 1;
305 } else {
306 keepLessThanDoubleCount_++;
307 }
308 } else {
309 /*
310 * When the actual frame rate is more than twice the target frame rate or the overshoot is
311 * equal to 0, one frame is reserved every (overshoot / targetFrameRate_) frames.
312 */
313 keepLessThanDoubleCount_ = 0;
314 const int32_t dropVar = overshoot / targetFrameRate_;
315 if (keepMoreThanDoubleCount_ < dropVar) {
316 isDrop = true;
317 keepMoreThanDoubleCount_++;
318 } else {
319 frameRateOvershootMdf_ = overshoot % targetFrameRate_;
320 isDrop = false;
321 keepMoreThanDoubleCount_ = 0;
322 }
323 }
324 return isDrop;
325 }
326
FpsControllerDone(std::vector<std::shared_ptr<DataBuffer>> & outputBuffers)327 int32_t FpsControllerProcess::FpsControllerDone(std::vector<std::shared_ptr<DataBuffer>>& outputBuffers)
328 {
329 if (outputBuffers.empty()) {
330 DHLOGE("The received data buffers is empty.");
331 return DCAMERA_BAD_VALUE;
332 }
333
334 if (nextDataProcess_ != nullptr) {
335 DHLOGD("Send to the next node of the FpsController for processing.");
336 int32_t err = nextDataProcess_->ProcessData(outputBuffers);
337 if (err != DCAMERA_OK) {
338 DHLOGE("Someone node after the FpsController processes failed.");
339 }
340 return err;
341 }
342 DHLOGD("The current node is the last node, and Output the processed video buffer");
343 std::shared_ptr<DCameraPipelineSource> targetPipelineSource = callbackPipelineSource_.lock();
344 if (targetPipelineSource == nullptr) {
345 DHLOGE("callbackPipelineSource_ is nullptr.");
346 return DCAMERA_BAD_VALUE;
347 }
348 targetPipelineSource->OnProcessedVideoBuffer(outputBuffers[0]);
349 return DCAMERA_OK;
350 }
351
GetProperty(const std::string & propertyName,PropertyCarrier & propertyCarrier)352 int32_t FpsControllerProcess::GetProperty(const std::string& propertyName, PropertyCarrier& propertyCarrier)
353 {
354 return DCAMERA_OK;
355 }
356 } // namespace DistributedHardware
357 } // namespace OHOS
358