1 /*
2 * Copyright (c) 2022-2024 Huawei Device Co., Ltd.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at
6 *
7 * http://www.apache.org/licenses/LICENSE-2.0
8 *
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
14 */
15
16 #include "fps_controller_process.h"
17
18 #include "dcamera_utils_tools.h"
19 #include "distributed_camera_errno.h"
20 #include "distributed_hardware_log.h"
21
22 namespace OHOS {
23 namespace DistributedHardware {
~FpsControllerProcess()24 FpsControllerProcess::~FpsControllerProcess()
25 {
26 if (isFpsControllerProcess_) {
27 DHLOGD("~DecodeDataProcess : ReleaseProcessNode.");
28 ReleaseProcessNode();
29 }
30 }
31
InitNode(const VideoConfigParams & sourceConfig,const VideoConfigParams & targetConfig,VideoConfigParams & processedConfig)32 int32_t FpsControllerProcess::InitNode(const VideoConfigParams& sourceConfig, const VideoConfigParams& targetConfig,
33 VideoConfigParams& processedConfig)
34 {
35 if (targetConfig.GetFrameRate() > MAX_TARGET_FRAME_RATE) {
36 DHLOGE("The target framerate : %{public}d is greater than the max framerate : %{public}d.",
37 targetConfig.GetFrameRate(), MAX_TARGET_FRAME_RATE);
38 return DCAMERA_BAD_TYPE;
39 }
40 sourceConfig_ = sourceConfig;
41 targetConfig_ = targetConfig;
42 targetFrameRate_ = targetConfig_.GetFrameRate();
43
44 processedConfig_ = sourceConfig;
45 processedConfig = processedConfig_;
46 isFpsControllerProcess_ = true;
47 return DCAMERA_OK;
48 }
49
ReleaseProcessNode()50 void FpsControllerProcess::ReleaseProcessNode()
51 {
52 DHLOGD("Start release [%{public}zu] node : FPS controller.", nodeRank_);
53 isFpsControllerProcess_ = false;
54 isFirstFrame_ = false;
55 targetFrameRate_ = 0;
56 lastFrameIncomeTimeMs_ = 0;
57 recentFrameTimeSpanMs_ = -1;
58 keepCorrectionCount_ = 0;
59 keepLessThanDoubleCount_ = 0;
60 keepMoreThanDoubleCount_ = 0;
61 frameRateCorrectionFactor_ = 0.0;
62 frameRateOvershootMdf_ = 0;
63 for (int i = 0; i < INCOME_FRAME_TIME_HISTORY_WINDOWS_SIZE; i++) {
64 incomingFrameTimesMs_[i] = 0;
65 }
66
67 if (nextDataProcess_ != nullptr) {
68 nextDataProcess_->ReleaseProcessNode();
69 nextDataProcess_ = nullptr;
70 }
71 DHLOGD("Release [%{public}zu] node : FPS controller end.", nodeRank_);
72 }
73
ProcessData(std::vector<std::shared_ptr<DataBuffer>> & inputBuffers)74 int32_t FpsControllerProcess::ProcessData(std::vector<std::shared_ptr<DataBuffer>>& inputBuffers)
75 {
76 if (inputBuffers.empty() || inputBuffers[0] == nullptr) {
77 DHLOGE("Data buffers is null.");
78 return DCAMERA_BAD_TYPE;
79 }
80 if (!isFpsControllerProcess_) {
81 DHLOGE("Decoder node occurred error.");
82 return DCAMERA_DISABLE_PROCESS;
83 }
84 int64_t timeStampUs = 0;
85 if (!inputBuffers[0]->FindInt64("timeUs", timeStampUs)) {
86 DHLOGE("Find decoder output timestamp failed.");
87 return DCAMERA_BAD_TYPE;
88 }
89
90 std::lock_guard<std::mutex> lck (mtx);
91 int64_t nowTimeMs = GetNowTimeStampMs();
92 UpdateFPSControllerInfo(nowTimeMs);
93
94 float curFrameRate = CalculateFrameRate(nowTimeMs);
95 if (IsDropFrame(curFrameRate)) {
96 DHLOGD("frame control, currect frameRate %{public}f, targetRate %{public}d, drop it",
97 curFrameRate, targetFrameRate_);
98 return DCAMERA_OK;
99 }
100
101 DHLOGD("frame control render PushVideoFrame, frame info width %{public}d height %{public}d, timeStampUs "
102 "%{public}lld, fps %{public}f", sourceConfig_.GetWidth(), sourceConfig_.GetHeight(), (long long)timeStampUs,
103 curFrameRate);
104 return FpsControllerDone(inputBuffers);
105 }
106
UpdateFPSControllerInfo(int64_t nowMs)107 void FpsControllerProcess::UpdateFPSControllerInfo(int64_t nowMs)
108 {
109 DHLOGD("Frame control, update control info.");
110 if (targetFrameRate_ <= 0) {
111 DHLOGD("Frame control, targetFrameRate_ : %{public}d", targetFrameRate_);
112 return;
113 }
114
115 isFirstFrame_ = false;
116 if (lastFrameIncomeTimeMs_ == 0) {
117 DHLOGD("Frame control, income fisrt frame.");
118 isFirstFrame_ = true;
119 }
120 lastFrameIncomeTimeMs_ = nowMs;
121 recentFrameTimeSpanMs_ = nowMs - lastFrameIncomeTimeMs_;
122 DHLOGD("Frame control, lastFrameIncomeTimeMs_ %{public}lld, receive Frame after last frame(ms): %{public}lld",
123 (long long)lastFrameIncomeTimeMs_, (long long)recentFrameTimeSpanMs_);
124 UpdateIncomingFrameTimes(nowMs);
125 UpdateFrameRateCorrectionFactor(nowMs);
126 return;
127 }
128
UpdateFrameRateCorrectionFactor(int64_t nowMs)129 void FpsControllerProcess::UpdateFrameRateCorrectionFactor(int64_t nowMs)
130 {
131 DHLOGD("Frame control, update FPS correction factor.");
132 if (targetFrameRate_ <= 0) {
133 DHLOGD("Frame control, targetFrameRate_ : %{public}d", targetFrameRate_);
134 return;
135 }
136 if (isFirstFrame_) {
137 DHLOGD("No frame rate correction factor when the first frame.");
138 return;
139 }
140
141 const float minDropFrmValue = 0.5;
142 const float maxDropFrmValue = 1.0;
143 const float msPerSecond = 1000;
144 const float maxInstantaneousFrameRateCoefficient = 1.1;
145 float maxInstantaneousFrameRateThreshold = targetFrameRate_ * maxInstantaneousFrameRateCoefficient;
146 float instantaneousFrameRate = msPerSecond / recentFrameTimeSpanMs_;
147 if (instantaneousFrameRate < 0) {
148 instantaneousFrameRate = -instantaneousFrameRate;
149 }
150 if (instantaneousFrameRate <= maxInstantaneousFrameRateThreshold) {
151 frameRateCorrectionFactor_ = minDropFrmValue;
152 } else {
153 if (keepCorrectionCount_ >= VIDEO_FRAME_DROP_INTERVAL) {
154 frameRateCorrectionFactor_ = maxDropFrmValue;
155 keepCorrectionCount_ = 0;
156 } else {
157 frameRateCorrectionFactor_ = 0;
158 keepCorrectionCount_++;
159 }
160 DHLOGD("Frame control, instantaneousFrameRate %{public}.3f is more than maxInstantaneousFrameRateThreshold "
161 "%{public}.3f, keepCorrectionCount %{public}d", instantaneousFrameRate,
162 maxInstantaneousFrameRateThreshold, keepCorrectionCount_);
163 }
164
165 DHLOGD("Frame control, targetFramerate %{public}d, maxInstantaneousFrameRateThreshold %{public}.3f,"
166 "instantaneousFrameRate %{public}.3f, frameRateCorrectionFactor %{public}.3f", targetFrameRate_,
167 maxInstantaneousFrameRateThreshold, instantaneousFrameRate, frameRateCorrectionFactor_);
168 return;
169 }
170
UpdateIncomingFrameTimes(int64_t nowMs)171 void FpsControllerProcess::UpdateIncomingFrameTimes(int64_t nowMs)
172 {
173 DHLOGD("Frame control, update incoming frame times array.");
174 if (targetFrameRate_ <= 0) {
175 DHLOGD("Frame control, targetFrameRate_ : %{public}d", targetFrameRate_);
176 return;
177 }
178 if (isFirstFrame_) {
179 incomingFrameTimesMs_[0] = nowMs;
180 return;
181 }
182
183 int64_t intervalNewAndFirst = nowMs - incomingFrameTimesMs_[0];
184 if (intervalNewAndFirst < 0) {
185 intervalNewAndFirst = -intervalNewAndFirst;
186 }
187 if (intervalNewAndFirst > FRMAE_MAX_INTERVAL_TIME_WINDOW_MS) {
188 DHLOGD("frame control, nowMs: %{public}lld mIncomingFrameT[0]: %{public}lld intervalNewAndFirst: "
189 "%{public}lld", (long long)nowMs, (long long)incomingFrameTimesMs_[0], (long long)intervalNewAndFirst);
190 for (int i = 0; i < INCOME_FRAME_TIME_HISTORY_WINDOWS_SIZE; i++) {
191 incomingFrameTimesMs_[i] = 0;
192 }
193 } else {
194 DHLOGD("frame control shift, nowMs: %{public}lld mIncomingFrameT[0]: %{public}lld intervalNewAndFirst: "
195 "%{public}lld", (long long)nowMs, (long long)incomingFrameTimesMs_[0], (long long)intervalNewAndFirst);
196 const int32_t windowLeftNum = 2;
197 for (int i = (INCOME_FRAME_TIME_HISTORY_WINDOWS_SIZE - windowLeftNum); i >= 0; --i) {
198 incomingFrameTimesMs_[i + 1] = incomingFrameTimesMs_[i];
199 }
200 }
201 incomingFrameTimesMs_[0] = nowMs;
202 return;
203 }
204
CalculateFrameRate(int64_t nowMs)205 float FpsControllerProcess::CalculateFrameRate(int64_t nowMs)
206 {
207 DHLOGD("Frame control, calculate frame rate.");
208 if (targetFrameRate_ <= 0) {
209 DHLOGE("Frame control, targetFrameRate_ : %{public}d", targetFrameRate_);
210 return 0.0;
211 }
212
213 int32_t num = 0;
214 int32_t validFramesNumber = 0;
215 if (nowMs < 0) {
216 nowMs = -nowMs;
217 }
218 for (; num < INCOME_FRAME_TIME_HISTORY_WINDOWS_SIZE; num++) {
219 if (incomingFrameTimesMs_[num] <= 0 || nowMs - incomingFrameTimesMs_[num] > FRAME_HISTORY_TIME_WINDOWS_MS) {
220 break;
221 } else {
222 validFramesNumber++;
223 }
224 }
225
226 const float msPerSecond = 1000;
227 const int32_t minValidCalculatedFrameRatesNum = 2;
228 int32_t minIncomingFrameNum = targetFrameRate_ / MIN_INCOME_FRAME_NUM_COEFFICIENT;
229 if (validFramesNumber > minIncomingFrameNum && validFramesNumber > minValidCalculatedFrameRatesNum) {
230 int64_t validTotalTimeInterval = 0;
231 if (num > 0) {
232 validTotalTimeInterval = (nowMs - incomingFrameTimesMs_[num - 1]);
233 }
234 if (validTotalTimeInterval < 0) {
235 validTotalTimeInterval = -validTotalTimeInterval;
236 }
237 if (validTotalTimeInterval > 0) {
238 return validFramesNumber * msPerSecond / validTotalTimeInterval + frameRateCorrectionFactor_;
239 }
240 }
241 return static_cast<float>(validFramesNumber);
242 }
243
IsDropFrame(float incomingFps)244 bool FpsControllerProcess::IsDropFrame(float incomingFps)
245 {
246 DHLOGD("Frame control, IsDropFrame");
247 if (targetFrameRate_ == 0) {
248 DHLOGD("target fps is 0, drop all frame.");
249 return true;
250 }
251 if (incomingFps <= 0) {
252 DHLOGD("incoming fps not more than 0, not drop");
253 return false;
254 }
255 const int32_t incomingFrmRate = static_cast<int32_t>(incomingFps);
256 if (incomingFrmRate > targetFrameRate_) {
257 DHLOGD("incoming fps not more than targetFrameRate_, not drop");
258 return false;
259 }
260 bool isDrop = ReduceFrameRateByUniformStrategy(incomingFrmRate);
261 DHLOGD("drop frame result: %{public}s", isDrop ? "drop" : "no drop");
262 return isDrop;
263 }
264
ReduceFrameRateByUniformStrategy(int32_t incomingFrmRate)265 bool FpsControllerProcess::ReduceFrameRateByUniformStrategy(int32_t incomingFrmRate)
266 {
267 DHLOGD("Frame control, reduce frame rate by uniform rate strategy");
268 if (incomingFrmRate > targetFrameRate_) {
269 DHLOGD("incoming fps not more than targetFrameRate_, not drop");
270 return false;
271 }
272
273 /*
274 * When the actual incoming frame rate correction value is greater than the target frame
275 * rate, the incoming frames are reduced uniformly.
276 */
277 bool isDrop = false;
278 int32_t overshoot = frameRateOvershootMdf_ + (incomingFrmRate - targetFrameRate_);
279 if (overshoot < 0) {
280 overshoot = 0;
281 frameRateOvershootMdf_ = 0;
282 }
283 if (overshoot && DOUBLE_MULTIPLE * overshoot < incomingFrmRate) {
284 /*
285 * When the actual input frame rate is less than or equal to twice the target frame rate,
286 * one frame is dropped every (incomingFrmRate / overshoot) frames.
287 */
288 if (keepMoreThanDoubleCount_) {
289 keepMoreThanDoubleCount_ = 0;
290 return true;
291 }
292 const int32_t dropVar = incomingFrmRate / overshoot;
293 if (keepLessThanDoubleCount_ >= dropVar) {
294 isDrop = true;
295 frameRateOvershootMdf_ = -(incomingFrmRate % overshoot) / OVERSHOOT_MODIFY_COEFFICIENT;
296 keepLessThanDoubleCount_ = 1;
297 } else {
298 keepLessThanDoubleCount_++;
299 }
300 } else {
301 /*
302 * When the actual frame rate is more than twice the target frame rate or the overshoot is
303 * equal to 0, one frame is reserved every (overshoot / targetFrameRate_) frames.
304 */
305 keepLessThanDoubleCount_ = 0;
306 const int32_t dropVar = overshoot / targetFrameRate_;
307 if (keepMoreThanDoubleCount_ < dropVar) {
308 isDrop = true;
309 keepMoreThanDoubleCount_++;
310 } else {
311 frameRateOvershootMdf_ = overshoot % targetFrameRate_;
312 isDrop = false;
313 keepMoreThanDoubleCount_ = 0;
314 }
315 }
316 return isDrop;
317 }
318
FpsControllerDone(std::vector<std::shared_ptr<DataBuffer>> & outputBuffers)319 int32_t FpsControllerProcess::FpsControllerDone(std::vector<std::shared_ptr<DataBuffer>>& outputBuffers)
320 {
321 if (outputBuffers.empty()) {
322 DHLOGE("The received data buffers is empty.");
323 return DCAMERA_BAD_VALUE;
324 }
325
326 if (nextDataProcess_ != nullptr) {
327 DHLOGD("Send to the next node of the FpsController for processing.");
328 int32_t err = nextDataProcess_->ProcessData(outputBuffers);
329 if (err != DCAMERA_OK) {
330 DHLOGE("Someone node after the FpsController processes failed.");
331 }
332 return err;
333 }
334 DHLOGD("The current node is the last node, and Output the processed video buffer");
335 std::shared_ptr<DCameraPipelineSource> targetPipelineSource = callbackPipelineSource_.lock();
336 if (targetPipelineSource == nullptr) {
337 DHLOGE("callbackPipelineSource_ is nullptr.");
338 return DCAMERA_BAD_VALUE;
339 }
340 targetPipelineSource->OnProcessedVideoBuffer(outputBuffers[0]);
341 return DCAMERA_OK;
342 }
343
GetProperty(const std::string & propertyName,PropertyCarrier & propertyCarrier)344 int32_t FpsControllerProcess::GetProperty(const std::string& propertyName, PropertyCarrier& propertyCarrier)
345 {
346 return DCAMERA_OK;
347 }
348 } // namespace DistributedHardware
349 } // namespace OHOS
350