1 /*
2 * Copyright (c) 2022-2023 Huawei Device Co., Ltd.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at
6 *
7 * http://www.apache.org/licenses/LICENSE-2.0
8 *
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
14 */
15
16 #include "dcamera_utils_tools.h"
17 #include "distributed_camera_constants.h"
18 #include "distributed_camera_errno.h"
19 #include "distributed_hardware_log.h"
20 #include "scale_convert_process.h"
21 #include "dcamera_frame_info.h"
22
23 namespace OHOS {
24 namespace DistributedHardware {
~ScaleConvertProcess()25 ScaleConvertProcess::~ScaleConvertProcess()
26 {
27 if (isScaleConvert_.load()) {
28 DHLOGI("~ScaleConvertProcess : ReleaseProcessNode");
29 ReleaseProcessNode();
30 }
31 }
32
InitNode(const VideoConfigParams & sourceConfig,const VideoConfigParams & targetConfig,VideoConfigParams & processedConfig)33 int32_t ScaleConvertProcess::InitNode(const VideoConfigParams& sourceConfig, const VideoConfigParams& targetConfig,
34 VideoConfigParams& processedConfig)
35 {
36 DHLOGI("ScaleConvertProcess : InitNode.");
37 sourceConfig_ = sourceConfig;
38 targetConfig_ = targetConfig;
39 processedConfig_ = sourceConfig;
40 processedConfig_.SetWidthAndHeight(targetConfig.GetWidth(), targetConfig.GetHeight());
41 processedConfig_.SetVideoformat(targetConfig.GetVideoformat());
42 processedConfig = processedConfig_;
43
44 if (!IsConvertible(sourceConfig, targetConfig)) {
45 DHLOGI("sourceConfig: Videoformat %d Width %d, Height %d, targetConfig: Videoformat %d Width %d, Height %d.",
46 sourceConfig.GetVideoformat(), sourceConfig.GetWidth(), sourceConfig.GetHeight(),
47 targetConfig.GetVideoformat(), targetConfig.GetWidth(), targetConfig.GetHeight());
48 isScaleConvert_.store(true);
49 return DCAMERA_OK;
50 }
51
52 int32_t ret = av_image_alloc(srcData_, srcLineSize_, sourceConfig_.GetWidth(), sourceConfig_.GetHeight(),
53 GetAVPixelFormat(sourceConfig_.GetVideoformat()), SOURCE_ALIGN);
54 if (ret < DCAMERA_OK) {
55 DHLOGE("Could not allocate source image.");
56 return DCAMERA_BAD_VALUE;
57 }
58
59 dstBuffSize_ = av_image_alloc(dstData_, dstLineSize_, processedConfig_.GetWidth(), processedConfig_.GetHeight(),
60 GetAVPixelFormat(processedConfig_.GetVideoformat()), TARGET_ALIGN);
61 if (dstBuffSize_ < DCAMERA_OK) {
62 DHLOGE("Could not allocate destination image.");
63 return DCAMERA_BAD_VALUE;
64 }
65
66 swsContext_ = sws_getContext(sourceConfig_.GetWidth(), sourceConfig_.GetHeight(),
67 GetAVPixelFormat(sourceConfig_.GetVideoformat()), processedConfig_.GetWidth(), processedConfig_.GetHeight(),
68 GetAVPixelFormat(processedConfig_.GetVideoformat()), SWS_FAST_BILINEAR, nullptr, nullptr, nullptr);
69 if (swsContext_ == nullptr) {
70 DHLOGE("Create SwsContext failed.");
71 return DCAMERA_BAD_VALUE;
72 }
73
74 isScaleConvert_.store(true);
75 return DCAMERA_OK;
76 }
77
IsConvertible(const VideoConfigParams & sourceConfig,const VideoConfigParams & targetConfig)78 bool ScaleConvertProcess::IsConvertible(const VideoConfigParams& sourceConfig, const VideoConfigParams& targetConfig)
79 {
80 return (sourceConfig_.GetWidth() != targetConfig.GetWidth()) ||
81 (sourceConfig_.GetHeight() != targetConfig.GetHeight()) ||
82 (sourceConfig_.GetVideoformat() != targetConfig.GetVideoformat());
83 }
84
ReleaseProcessNode()85 void ScaleConvertProcess::ReleaseProcessNode()
86 {
87 DHLOGI("Start release [%d] node : ScaleConvertNode.", nodeRank_);
88 isScaleConvert_.store(false);
89
90 {
91 std::lock_guard<std::mutex> autoLock(scaleMutex_);
92 if (swsContext_ != nullptr) {
93 av_freep(&srcData_[0]);
94 av_freep(&dstData_[0]);
95 sws_freeContext(swsContext_);
96 swsContext_ = nullptr;
97 }
98 }
99
100 if (nextDataProcess_ != nullptr) {
101 nextDataProcess_->ReleaseProcessNode();
102 nextDataProcess_ = nullptr;
103 }
104 DHLOGI("Release [%d] node : ScaleConvertNode end.", nodeRank_);
105 }
106
ProcessData(std::vector<std::shared_ptr<DataBuffer>> & inputBuffers)107 int ScaleConvertProcess::ProcessData(std::vector<std::shared_ptr<DataBuffer>>& inputBuffers)
108 {
109 int64_t startScaleTime = GetNowTimeStampUs();
110 DHLOGD("Process data in ScaleConvertProcess.");
111 if (!isScaleConvert_.load()) {
112 DHLOGE("Scale Convert node occurred error or start release.");
113 return DCAMERA_DISABLE_PROCESS;
114 }
115
116 if (inputBuffers.empty() || inputBuffers[0] == nullptr) {
117 DHLOGE("The input data buffers is empty.");
118 return DCAMERA_BAD_VALUE;
119 }
120 inputBuffers[0]->frameInfo_.timePonit.startScale = startScaleTime;
121
122 if (!IsConvertible(sourceConfig_, processedConfig_)) {
123 DHLOGD("The target resolution: %dx%d format: %d is the same as the source resolution: %dx%d format: %d",
124 processedConfig_.GetWidth(), processedConfig_.GetHeight(), processedConfig_.GetVideoformat(),
125 sourceConfig_.GetWidth(), sourceConfig_.GetHeight(), sourceConfig_.GetVideoformat());
126 return ConvertDone(inputBuffers);
127 }
128
129 ImageUnitInfo srcImgInfo {Videoformat::YUVI420, 0, 0, 0, 0, 0, 0, nullptr};
130 if ((GetImageUnitInfo(srcImgInfo, inputBuffers[0]) != DCAMERA_OK) || !CheckScaleProcessInputInfo(srcImgInfo)) {
131 DHLOGE("ScaleConvertProcess : srcImgInfo error.");
132 return DCAMERA_BAD_VALUE;
133 }
134
135 std::shared_ptr<DataBuffer> dstBuf = std::make_shared<DataBuffer>(dstBuffSize_);
136 ImageUnitInfo dstImgInfo = { processedConfig_.GetVideoformat(), processedConfig_.GetWidth(),
137 processedConfig_.GetHeight(), processedConfig_.GetWidth(), processedConfig_.GetHeight(),
138 processedConfig_.GetWidth() * processedConfig_.GetHeight(), dstBuf->Size(), dstBuf };
139 if (ScaleConvert(srcImgInfo, dstImgInfo) != DCAMERA_OK) {
140 DHLOGE("ScaleConvertProcess : Scale convert failed.");
141 return DCAMERA_BAD_OPERATE;
142 }
143
144 dstBuf->frameInfo_ = inputBuffers[0]->frameInfo_;
145 dstBuf->SetInt32("Videoformat", static_cast<int32_t>(processedConfig_.GetVideoformat()));
146 dstBuf->SetInt32("alignedWidth", processedConfig_.GetWidth());
147 dstBuf->SetInt32("alignedHeight", processedConfig_.GetHeight());
148 dstBuf->SetInt32("width", processedConfig_.GetWidth());
149 dstBuf->SetInt32("height", processedConfig_.GetHeight());
150
151 std::vector<std::shared_ptr<DataBuffer>> outputBuffers;
152 outputBuffers.push_back(dstBuf);
153 return ConvertDone(outputBuffers);
154 }
155
GetImageUnitInfo(ImageUnitInfo & imgInfo,const std::shared_ptr<DataBuffer> & imgBuf)156 int32_t ScaleConvertProcess::GetImageUnitInfo(ImageUnitInfo& imgInfo, const std::shared_ptr<DataBuffer>& imgBuf)
157 {
158 if (imgBuf == nullptr) {
159 DHLOGE("GetImageUnitInfo failed, imgBuf is nullptr.");
160 return DCAMERA_BAD_VALUE;
161 }
162
163 bool findErr = true;
164 int32_t colorFormat = 0;
165 findErr = findErr && imgBuf->FindInt32("Videoformat", colorFormat);
166 if (!findErr) {
167 DHLOGE("GetImageUnitInfo failed, Videoformat is null.");
168 return DCAMERA_NOT_FOUND;
169 }
170
171 if (colorFormat != static_cast<int32_t>(Videoformat::YUVI420) &&
172 colorFormat != static_cast<int32_t>(Videoformat::NV12) &&
173 colorFormat != static_cast<int32_t>(Videoformat::NV21)) {
174 DHLOGE("GetImageUnitInfo failed, colorFormat %d are not supported.", colorFormat);
175 return DCAMERA_NOT_FOUND;
176 }
177 imgInfo.colorFormat = static_cast<Videoformat>(colorFormat);
178 findErr = findErr && imgBuf->FindInt32("width", imgInfo.width);
179 findErr = findErr && imgBuf->FindInt32("height", imgInfo.height);
180 findErr = findErr && imgBuf->FindInt32("alignedWidth", imgInfo.alignedWidth);
181 findErr = findErr && imgBuf->FindInt32("alignedHeight", imgInfo.alignedHeight);
182 if (!findErr) {
183 DHLOGE("GetImageUnitInfo failed, width %d, height %d, alignedWidth %d, alignedHeight %d.",
184 imgInfo.width, imgInfo.height, imgInfo.alignedWidth, imgInfo.alignedHeight);
185 return DCAMERA_NOT_FOUND;
186 }
187
188 imgInfo.chromaOffset = static_cast<size_t>(imgInfo.alignedWidth * imgInfo.alignedHeight);
189 imgInfo.imgSize = imgBuf->Size();
190 imgInfo.imgData = imgBuf;
191 if (imgInfo.imgData == nullptr) {
192 DHLOGE("Get the imgData of the imgBuf failed.");
193 return DCAMERA_BAD_VALUE;
194 }
195 DHLOGD("ScaleConvertProcess imgBuf info : Videoformat %d, alignedWidth %d, alignedHeight %d, width %d, height %d" +
196 ", chromaOffset %d, imgSize %d.", imgInfo.colorFormat, imgInfo.alignedWidth, imgInfo.alignedHeight,
197 imgInfo.width, imgInfo.height, imgInfo.chromaOffset, imgInfo.imgSize);
198 return DCAMERA_OK;
199 }
200
CheckScaleProcessInputInfo(const ImageUnitInfo & srcImgInfo)201 bool ScaleConvertProcess::CheckScaleProcessInputInfo(const ImageUnitInfo& srcImgInfo)
202 {
203 return srcImgInfo.colorFormat == sourceConfig_.GetVideoformat() &&
204 srcImgInfo.width == sourceConfig_.GetWidth() &&
205 srcImgInfo.height == sourceConfig_.GetHeight() &&
206 IsCorrectImageUnitInfo(srcImgInfo);
207 }
208
CheckScaleConvertInfo(const ImageUnitInfo & srcImgInfo,const ImageUnitInfo & dstImgInfo)209 bool ScaleConvertProcess::CheckScaleConvertInfo(const ImageUnitInfo& srcImgInfo, const ImageUnitInfo& dstImgInfo)
210 {
211 if (srcImgInfo.imgData == nullptr || dstImgInfo.imgData == nullptr) {
212 DHLOGE("The imgData of srcImgInfo or the imgData of dstImgInfo are null!");
213 return false;
214 }
215
216 if (!IsCorrectImageUnitInfo(srcImgInfo)) {
217 DHLOGE("srcImginfo fail: width %d, height %d, alignedWidth %d, alignedHeight %d, chromaOffset %lld, " +
218 "imgSize %lld.", srcImgInfo.width, srcImgInfo.height, srcImgInfo.alignedWidth, srcImgInfo.alignedHeight,
219 srcImgInfo.chromaOffset, srcImgInfo.imgSize);
220 return false;
221 }
222
223 if (!IsCorrectImageUnitInfo(dstImgInfo)) {
224 DHLOGE("dstImginfo fail: width %d, height %d, alignedWidth %d, alignedHeight %d, chromaOffset %lld, " +
225 "imgSize %lld.", dstImgInfo.width, dstImgInfo.height, dstImgInfo.alignedWidth, dstImgInfo.alignedHeight,
226 dstImgInfo.chromaOffset, dstImgInfo.imgSize);
227 return false;
228 }
229
230 if ((dstImgInfo.width == srcImgInfo.alignedWidth) && (dstImgInfo.height == srcImgInfo.alignedHeight) &&
231 (dstImgInfo.colorFormat == srcImgInfo.colorFormat)) {
232 DHLOGE("Comparison ImgInfo fail: dstwidth %d, dstheight %d, srcAlignedWidth %d, srcAlignedHeight %d.",
233 dstImgInfo.width, dstImgInfo.height, srcImgInfo.alignedWidth, srcImgInfo.alignedHeight);
234 return false;
235 }
236
237 return true;
238 }
239
IsCorrectImageUnitInfo(const ImageUnitInfo & imgInfo)240 bool ScaleConvertProcess::IsCorrectImageUnitInfo(const ImageUnitInfo& imgInfo)
241 {
242 size_t expectedImgSize = static_cast<size_t>(imgInfo.alignedWidth * imgInfo.alignedHeight *
243 YUV_BYTES_PER_PIXEL / Y2UV_RATIO);
244 size_t expectedChromaOffset = static_cast<size_t>(imgInfo.alignedWidth * imgInfo.alignedHeight);
245 return (imgInfo.width <= imgInfo.alignedWidth && imgInfo.height <= imgInfo.alignedHeight &&
246 imgInfo.imgSize >= expectedImgSize && imgInfo.chromaOffset == expectedChromaOffset);
247 }
248
ScaleConvert(ImageUnitInfo & srcImgInfo,ImageUnitInfo & dstImgInfo)249 int32_t ScaleConvertProcess::ScaleConvert(ImageUnitInfo& srcImgInfo, ImageUnitInfo& dstImgInfo)
250 {
251 DHLOGD("ScaleConvertProcess : Scale convert start.");
252 if (!CheckScaleConvertInfo(srcImgInfo, dstImgInfo)) {
253 DHLOGE("ScaleConvertProcess : CheckScaleConvertInfo failed.");
254 return DCAMERA_BAD_VALUE;
255 }
256
257 std::lock_guard<std::mutex> autoLock(scaleMutex_);
258 switch (GetAVPixelFormat(srcImgInfo.colorFormat)) {
259 case AV_PIX_FMT_YUV420P: {
260 int32_t ret = CopyYUV420SrcData(srcImgInfo);
261 if (ret != DCAMERA_OK) {
262 DHLOGE("ScaleConvertProcess::ScaleConvert copy yuv420p src data failed.");
263 return ret;
264 }
265 break;
266 }
267 case AV_PIX_FMT_NV12: {
268 int32_t ret = CopyNV12SrcData(srcImgInfo);
269 if (ret != DCAMERA_OK) {
270 DHLOGE("ScaleConvertProcess::ScaleConvert copy nv12 src data failed.");
271 return ret;
272 }
273 break;
274 }
275 case AV_PIX_FMT_NV21: {
276 int32_t ret = CopyNV21SrcData(srcImgInfo);
277 if (ret != DCAMERA_OK) {
278 DHLOGE("ScaleConvertProcess::ScaleConvert copy nv21 src data failed.");
279 return ret;
280 }
281 break;
282 }
283 default:
284 DHLOGE("Unknown pixel format not support.");
285 return DCAMERA_BAD_VALUE;
286 }
287
288 sws_scale(swsContext_, static_cast<const uint8_t * const *>(srcData_), srcLineSize_, 0, srcImgInfo.height,
289 dstData_, dstLineSize_);
290 int32_t ret = memcpy_s(dstImgInfo.imgData->Data(), dstImgInfo.imgSize, dstData_[0], dstBuffSize_);
291 if (ret != EOK) {
292 DHLOGE("ScaleConvertProcess::ScaleConvert copy dst image info failed, ret = %d", ret);
293 return DCAMERA_MEMORY_OPT_ERROR;
294 }
295 return DCAMERA_OK;
296 }
297
CopyYUV420SrcData(const ImageUnitInfo & srcImgInfo)298 int32_t ScaleConvertProcess::CopyYUV420SrcData(const ImageUnitInfo& srcImgInfo)
299 {
300 int32_t ret = memcpy_s(srcData_[0], srcImgInfo.width * srcImgInfo.height,
301 srcImgInfo.imgData->Data(), srcImgInfo.width * srcImgInfo.height);
302 if (ret != EOK) {
303 DHLOGE("ScaleConvertProcess::CopyYUV420SrcData memory copy failed, ret = %d", ret);
304 return DCAMERA_MEMORY_OPT_ERROR;
305 }
306
307 ret = memcpy_s(srcData_[1], srcImgInfo.width * srcImgInfo.height / MEMORY_RATIO_YUV,
308 srcImgInfo.imgData->Data() + srcImgInfo.alignedWidth * srcImgInfo.alignedHeight,
309 srcImgInfo.width * srcImgInfo.height / MEMORY_RATIO_YUV);
310 if (ret != EOK) {
311 DHLOGE("ScaleConvertProcess::CopyYUV420SrcData memory copy failed, ret = %d", ret);
312 return DCAMERA_MEMORY_OPT_ERROR;
313 }
314
315 ret = memcpy_s(srcData_[2], srcImgInfo.width * srcImgInfo.height / MEMORY_RATIO_YUV, // 2: v start address
316 srcImgInfo.imgData->Data() + srcImgInfo.alignedWidth * srcImgInfo.alignedHeight +
317 srcImgInfo.alignedWidth * srcImgInfo.alignedHeight / MEMORY_RATIO_YUV,
318 srcImgInfo.width * srcImgInfo.height / MEMORY_RATIO_YUV);
319 if (ret != EOK) {
320 DHLOGE("ScaleConvertProcess::CopyYUV420SrcData memory copy failed, ret = %d", ret);
321 return DCAMERA_MEMORY_OPT_ERROR;
322 }
323 return DCAMERA_OK;
324 }
325
CopyNV12SrcData(const ImageUnitInfo & srcImgInfo)326 int32_t ScaleConvertProcess::CopyNV12SrcData(const ImageUnitInfo& srcImgInfo)
327 {
328 int32_t ret = memcpy_s(srcData_[0], srcImgInfo.width * srcImgInfo.height,
329 srcImgInfo.imgData->Data(), srcImgInfo.width * srcImgInfo.height);
330 if (ret != EOK) {
331 DHLOGE("ScaleConvertProcess::CopyNV12SrcData memory copy failed, ret = %d", ret);
332 return DCAMERA_MEMORY_OPT_ERROR;
333 }
334
335 ret = memcpy_s(srcData_[1], srcImgInfo.width * srcImgInfo.height / MEMORY_RATIO_NV,
336 srcImgInfo.imgData->Data() + srcImgInfo.alignedWidth * srcImgInfo.alignedHeight,
337 srcImgInfo.width * srcImgInfo.height / MEMORY_RATIO_NV);
338 if (ret != EOK) {
339 DHLOGE("ScaleConvertProcess::CopyNV12SrcData memory copy failed, ret = %d", ret);
340 return DCAMERA_MEMORY_OPT_ERROR;
341 }
342 return DCAMERA_OK;
343 }
344
CopyNV21SrcData(const ImageUnitInfo & srcImgInfo)345 int32_t ScaleConvertProcess::CopyNV21SrcData(const ImageUnitInfo& srcImgInfo)
346 {
347 int32_t ret = memcpy_s(srcData_[0], srcImgInfo.width * srcImgInfo.height,
348 srcImgInfo.imgData->Data(), srcImgInfo.width * srcImgInfo.height);
349 if (ret != EOK) {
350 DHLOGE("ScaleConvertProcess::CopyNV21SrcData memory copy failed, ret = %d", ret);
351 return DCAMERA_MEMORY_OPT_ERROR;
352 }
353
354 ret = memcpy_s(srcData_[1], srcImgInfo.width * srcImgInfo.height / MEMORY_RATIO_NV,
355 srcImgInfo.imgData->Data() + srcImgInfo.alignedWidth * srcImgInfo.alignedHeight,
356 srcImgInfo.width * srcImgInfo.height / MEMORY_RATIO_NV);
357 if (ret != EOK) {
358 DHLOGE("ScaleConvertProcess::CopyNV21SrcData memory copy failed, ret = %d", ret);
359 return DCAMERA_MEMORY_OPT_ERROR;
360 }
361 return DCAMERA_OK;
362 }
363
ConvertDone(std::vector<std::shared_ptr<DataBuffer>> & outputBuffers)364 int32_t ScaleConvertProcess::ConvertDone(std::vector<std::shared_ptr<DataBuffer>>& outputBuffers)
365 {
366 int64_t finishScaleTime = GetNowTimeStampUs();
367 DHLOGD("ScaleConvertProcess : Convert Done.");
368 if (outputBuffers.empty()) {
369 DHLOGE("The received data buffer is empty.");
370 return DCAMERA_BAD_VALUE;
371 }
372 outputBuffers[0]->frameInfo_.timePonit.finishScale = finishScaleTime;
373
374 if (nextDataProcess_ != nullptr) {
375 DHLOGD("Send to the next node of the scale convert for processing.");
376 int32_t err = nextDataProcess_->ProcessData(outputBuffers);
377 if (err != DCAMERA_OK) {
378 DHLOGE("Some node after the scale convert processes failed.");
379 }
380 return err;
381 }
382
383 DHLOGD("The current node is the last noed, and output the processed video buffer.");
384 std::shared_ptr<DCameraPipelineSource> targetPipelineSource = callbackPipelineSource_.lock();
385 if (targetPipelineSource == nullptr) {
386 DHLOGE("callbackPipelineSource_ is nullptr.");
387 return DCAMERA_BAD_VALUE;
388 }
389 targetPipelineSource->OnProcessedVideoBuffer(outputBuffers[0]);
390 return DCAMERA_OK;
391 }
392
GetAVPixelFormat(Videoformat colorFormat)393 AVPixelFormat ScaleConvertProcess::GetAVPixelFormat(Videoformat colorFormat)
394 {
395 AVPixelFormat format;
396 switch (colorFormat) {
397 case Videoformat::NV12:
398 format = AVPixelFormat::AV_PIX_FMT_NV12;
399 break;
400 case Videoformat::NV21:
401 format = AVPixelFormat::AV_PIX_FMT_NV21;
402 break;
403 default:
404 format = AVPixelFormat::AV_PIX_FMT_YUV420P;
405 break;
406 }
407 return format;
408 }
409
GetProperty(const std::string & propertyName,PropertyCarrier & propertyCarrier)410 int32_t ScaleConvertProcess::GetProperty(const std::string& propertyName, PropertyCarrier& propertyCarrier)
411 {
412 return DCAMERA_OK;
413 }
414 } // namespace DistributedHardware
415 } // namespace OHOS
416