1 /*
2 * Copyright (c) 2022 Huawei Device Co., Ltd.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at
6 *
7 * http://www.apache.org/licenses/LICENSE-2.0
8 *
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
14 */
15
16 #include "scale_convert_process.h"
17
18 #include "distributed_hardware_log.h"
19 #include "distributed_camera_errno.h"
20
21 namespace OHOS {
22 namespace DistributedHardware {
~ScaleConvertProcess()23 ScaleConvertProcess::~ScaleConvertProcess()
24 {
25 if (isScaleConvert_.load()) {
26 DHLOGI("~ScaleConvertProcess : ReleaseProcessNode");
27 ReleaseProcessNode();
28 }
29 }
30
InitNode(const VideoConfigParams & sourceConfig,const VideoConfigParams & targetConfig,VideoConfigParams & processedConfig)31 int32_t ScaleConvertProcess::InitNode(const VideoConfigParams& sourceConfig, const VideoConfigParams& targetConfig,
32 VideoConfigParams& processedConfig)
33 {
34 DHLOGI("ScaleConvertProcess : InitNode.");
35 sourceConfig_ = sourceConfig;
36 targetConfig_ = targetConfig;
37 processedConfig_ = sourceConfig;
38 processedConfig_.SetWidthAndHeight(targetConfig.GetWidth(), targetConfig.GetHeight());
39 processedConfig = processedConfig_;
40
41 if (!IsConvertible(sourceConfig, targetConfig)) {
42 DHLOGI("sourceConfig: Videoformat %d Width %d, Height %d, targetConfig: Videoformat %d Width %d, Height %d.",
43 sourceConfig.GetVideoformat(), sourceConfig.GetWidth(), sourceConfig.GetHeight(),
44 targetConfig.GetVideoformat(), targetConfig.GetWidth(), targetConfig.GetHeight());
45 isScaleConvert_.store(true);
46 return DCAMERA_OK;
47 }
48
49 int32_t ret = av_image_alloc(srcData_, srcLineSize_, sourceConfig_.GetWidth(), sourceConfig_.GetHeight(),
50 GetAVPixelFormat(sourceConfig_.GetVideoformat()), SOURCE_ALIGN);
51 if (ret < DCAMERA_OK) {
52 DHLOGE("Could not allocate source image.");
53 return DCAMERA_BAD_VALUE;
54 }
55
56 dstBuffSize_ = av_image_alloc(dstData_, dstLineSize_, processedConfig_.GetWidth(), processedConfig_.GetHeight(),
57 GetAVPixelFormat(processedConfig_.GetVideoformat()), TARGET_ALIGN);
58 if (dstBuffSize_ < DCAMERA_OK) {
59 DHLOGE("Could not allocate destination image.");
60 return DCAMERA_BAD_VALUE;
61 }
62
63 swsContext_ = sws_getContext(sourceConfig_.GetWidth(), sourceConfig_.GetHeight(),
64 GetAVPixelFormat(sourceConfig_.GetVideoformat()), processedConfig_.GetWidth(), processedConfig_.GetHeight(),
65 GetAVPixelFormat(processedConfig_.GetVideoformat()), SWS_FAST_BILINEAR, nullptr, nullptr, nullptr);
66 if (swsContext_ == nullptr) {
67 DHLOGE("Create SwsContext failed.");
68 return DCAMERA_BAD_VALUE;
69 }
70
71 isScaleConvert_.store(true);
72 return DCAMERA_OK;
73 }
74
IsConvertible(const VideoConfigParams & sourceConfig,const VideoConfigParams & targetConfig)75 bool ScaleConvertProcess::IsConvertible(const VideoConfigParams& sourceConfig, const VideoConfigParams& targetConfig)
76 {
77 return (sourceConfig_.GetWidth() != targetConfig.GetWidth()) ||
78 (sourceConfig_.GetHeight() != targetConfig.GetHeight());
79 }
80
ReleaseProcessNode()81 void ScaleConvertProcess::ReleaseProcessNode()
82 {
83 DHLOGI("Start release [%d] node : ScaleConvertNode.", nodeRank_);
84 isScaleConvert_.store(false);
85
86 {
87 std::lock_guard<std::mutex> autoLock(scaleMutex_);
88 if (swsContext_ != nullptr) {
89 av_freep(&srcData_[0]);
90 av_freep(&dstData_[0]);
91 sws_freeContext(swsContext_);
92 swsContext_ = nullptr;
93 }
94 }
95
96 if (nextDataProcess_ != nullptr) {
97 nextDataProcess_->ReleaseProcessNode();
98 nextDataProcess_ = nullptr;
99 }
100 DHLOGI("Release [%d] node : ScaleConvertNode end.", nodeRank_);
101 }
102
ProcessData(std::vector<std::shared_ptr<DataBuffer>> & inputBuffers)103 int ScaleConvertProcess::ProcessData(std::vector<std::shared_ptr<DataBuffer>>& inputBuffers)
104 {
105 DHLOGD("Process data in ScaleConvertProcess.");
106 if (!isScaleConvert_.load()) {
107 DHLOGE("Scale Convert node occurred error or start release.");
108 return DCAMERA_DISABLE_PROCESS;
109 }
110
111 if (inputBuffers.empty() || inputBuffers[0] == nullptr) {
112 DHLOGE("The input data buffers is empty.");
113 return DCAMERA_BAD_VALUE;
114 }
115
116 if (!IsConvertible(sourceConfig_, processedConfig_)) {
117 DHLOGD("The target resolution: %dx%d is the same as the source resolution: %dx%d",
118 processedConfig_.GetWidth(), processedConfig_.GetHeight(),
119 sourceConfig_.GetWidth(), sourceConfig_.GetHeight());
120 return ConvertDone(inputBuffers);
121 }
122
123 int64_t timeStamp = 0;
124 if (!(inputBuffers[0]->FindInt64("timeUs", timeStamp))) {
125 DHLOGE("ScaleConvertProcess : Find inputBuffer timeStamp failed.");
126 return DCAMERA_BAD_VALUE;
127 }
128
129 ImageUnitInfo srcImgInfo {Videoformat::YUVI420, 0, 0, 0, 0, 0, 0, nullptr};
130 if ((GetImageUnitInfo(srcImgInfo, inputBuffers[0]) != DCAMERA_OK) || !CheckScaleProcessInputInfo(srcImgInfo)) {
131 DHLOGE("ScaleConvertProcess : srcImgInfo error.");
132 return DCAMERA_BAD_VALUE;
133 }
134
135 std::shared_ptr<DataBuffer> dstBuf = std::make_shared<DataBuffer>(dstBuffSize_);
136 ImageUnitInfo dstImgInfo = { processedConfig_.GetVideoformat(), processedConfig_.GetWidth(),
137 processedConfig_.GetHeight(), processedConfig_.GetWidth(), processedConfig_.GetHeight(),
138 processedConfig_.GetWidth() * processedConfig_.GetHeight(), dstBuf->Size(), dstBuf->Data() };
139 if (ScaleConvert(srcImgInfo, dstImgInfo) != DCAMERA_OK) {
140 DHLOGE("ScaleConvertProcess : Scale convert failed.");
141 return DCAMERA_BAD_OPERATE;
142 }
143
144 dstBuf->SetInt64("timeUs", timeStamp);
145 dstBuf->SetInt32("Videoformat", static_cast<int32_t>(processedConfig_.GetVideoformat()));
146 dstBuf->SetInt32("alignedWidth", processedConfig_.GetWidth());
147 dstBuf->SetInt32("alignedHeight", processedConfig_.GetHeight());
148 dstBuf->SetInt32("width", processedConfig_.GetWidth());
149 dstBuf->SetInt32("height", processedConfig_.GetHeight());
150
151 std::vector<std::shared_ptr<DataBuffer>> outputBuffers;
152 outputBuffers.push_back(dstBuf);
153 return ConvertDone(outputBuffers);
154 }
155
GetImageUnitInfo(ImageUnitInfo & imgInfo,const std::shared_ptr<DataBuffer> & imgBuf)156 int32_t ScaleConvertProcess::GetImageUnitInfo(ImageUnitInfo& imgInfo, const std::shared_ptr<DataBuffer>& imgBuf)
157 {
158 if (imgBuf == nullptr) {
159 DHLOGE("GetImageUnitInfo failed, imgBuf is nullptr.");
160 return DCAMERA_BAD_VALUE;
161 }
162
163 bool findErr = true;
164 int32_t colorFormat = 0;
165 findErr = findErr && imgBuf->FindInt32("Videoformat", colorFormat);
166 if (!findErr) {
167 DHLOGE("GetImageUnitInfo failed, Videoformat is null.");
168 return DCAMERA_NOT_FOUND;
169 }
170 if (colorFormat != static_cast<int32_t>(Videoformat::YUVI420) &&
171 colorFormat != static_cast<int32_t>(Videoformat::NV12) &&
172 colorFormat != static_cast<int32_t>(Videoformat::NV21)) {
173 DHLOGE("GetImageUnitInfo failed, colorFormat %d are not supported.", colorFormat);
174 return DCAMERA_NOT_FOUND;
175 }
176 imgInfo.colorFormat = static_cast<Videoformat>(colorFormat);
177 findErr = findErr && imgBuf->FindInt32("width", imgInfo.width);
178 findErr = findErr && imgBuf->FindInt32("height", imgInfo.height);
179 findErr = findErr && imgBuf->FindInt32("alignedWidth", imgInfo.alignedWidth);
180 findErr = findErr && imgBuf->FindInt32("alignedHeight", imgInfo.alignedHeight);
181 if (!findErr) {
182 DHLOGE("GetImageUnitInfo failed, width %d, height %d, alignedWidth %d, alignedHeight %d.",
183 imgInfo.width, imgInfo.height, imgInfo.alignedWidth, imgInfo.alignedHeight);
184 return DCAMERA_NOT_FOUND;
185 }
186
187 imgInfo.chromaOffset = static_cast<size_t>(imgInfo.alignedWidth * imgInfo.alignedHeight);
188 imgInfo.imgSize = imgBuf->Size();
189 imgInfo.imgData = imgBuf->Data();
190 if (imgInfo.imgData == nullptr) {
191 DHLOGE("Get the imgData of the imgBuf failed.");
192 return DCAMERA_BAD_VALUE;
193 }
194 DHLOGD("ScaleConvertProcess imgBuf info : Videoformat %d, alignedWidth %d, alignedHeight %d, width %d, height %d" +
195 ", chromaOffset %d, imgSize %d.", imgInfo.colorFormat, imgInfo.width, imgInfo.height, imgInfo.alignedWidth,
196 imgInfo.alignedHeight, imgInfo.chromaOffset, imgInfo.imgSize);
197 return DCAMERA_OK;
198 }
199
CheckScaleProcessInputInfo(const ImageUnitInfo & srcImgInfo)200 bool ScaleConvertProcess::CheckScaleProcessInputInfo(const ImageUnitInfo& srcImgInfo)
201 {
202 return srcImgInfo.colorFormat == sourceConfig_.GetVideoformat() &&
203 srcImgInfo.alignedWidth == sourceConfig_.GetWidth() &&
204 srcImgInfo.alignedHeight == sourceConfig_.GetHeight() &&
205 IsCorrectImageUnitInfo(srcImgInfo);
206 }
207
CheckScaleConvertInfo(const ImageUnitInfo & srcImgInfo,const ImageUnitInfo & dstImgInfo)208 bool ScaleConvertProcess::CheckScaleConvertInfo(const ImageUnitInfo& srcImgInfo, const ImageUnitInfo& dstImgInfo)
209 {
210 if (srcImgInfo.imgData == nullptr || dstImgInfo.imgData == nullptr) {
211 DHLOGE("The imgData of srcImgInfo or the imgData of dstImgInfo are null!");
212 return false;
213 }
214
215 if (!IsCorrectImageUnitInfo(srcImgInfo)) {
216 DHLOGE("srcImginfo fail: width %d, height %d, alignedWidth %d, alignedHeight %d, chromaOffset %lld, " +
217 "imgSize %lld.", srcImgInfo.width, srcImgInfo.height, srcImgInfo.alignedWidth, srcImgInfo.alignedHeight,
218 srcImgInfo.chromaOffset, srcImgInfo.imgSize);
219 return false;
220 }
221
222 if (!IsCorrectImageUnitInfo(dstImgInfo)) {
223 DHLOGE("dstImginfo fail: width %d, height %d, alignedWidth %d, alignedHeight %d, chromaOffset %lld, " +
224 "imgSize %lld.", dstImgInfo.width, dstImgInfo.height, dstImgInfo.alignedWidth, dstImgInfo.alignedHeight,
225 dstImgInfo.chromaOffset, dstImgInfo.imgSize);
226 return false;
227 }
228
229 if ((dstImgInfo.width == srcImgInfo.alignedWidth) && (dstImgInfo.height == srcImgInfo.alignedHeight)) {
230 DHLOGE("Comparison ImgInfo fail: dstwidth %d, dstheight %d, srcAlignedWidth %d, srcAlignedHeight %d.",
231 dstImgInfo.width, dstImgInfo.height, srcImgInfo.alignedWidth, srcImgInfo.alignedHeight);
232 return false;
233 }
234
235 return true;
236 }
237
IsCorrectImageUnitInfo(const ImageUnitInfo & imgInfo)238 bool ScaleConvertProcess::IsCorrectImageUnitInfo(const ImageUnitInfo& imgInfo)
239 {
240 size_t expectedImgSize = static_cast<size_t>(imgInfo.alignedWidth * imgInfo.alignedHeight *
241 YUV_BYTES_PER_PIXEL / Y2UV_RATIO);
242 size_t expectedChromaOffset = static_cast<size_t>(imgInfo.alignedWidth * imgInfo.alignedHeight);
243 return (imgInfo.width <= imgInfo.alignedWidth && imgInfo.height <= imgInfo.alignedHeight &&
244 imgInfo.imgSize >= expectedImgSize && imgInfo.chromaOffset == expectedChromaOffset);
245 }
246
ScaleConvert(const ImageUnitInfo & srcImgInfo,const ImageUnitInfo & dstImgInfo)247 int32_t ScaleConvertProcess::ScaleConvert(const ImageUnitInfo& srcImgInfo, const ImageUnitInfo& dstImgInfo)
248 {
249 DHLOGD("ScaleConvertProcess : Scale convert start.");
250 if (!CheckScaleConvertInfo(srcImgInfo, dstImgInfo)) {
251 DHLOGE("ScaleConvertProcess : CheckScaleConvertInfo failed.");
252 return DCAMERA_BAD_VALUE;
253 }
254
255 std::lock_guard<std::mutex> autoLock(scaleMutex_);
256 switch (GetAVPixelFormat(srcImgInfo.colorFormat)) {
257 case AV_PIX_FMT_YUV420P: {
258 int32_t ret = CopyYUV420SrcData(srcImgInfo);
259 if (ret != DCAMERA_OK) {
260 DHLOGE("ScaleConvertProcess::ScaleConvert copy yuv420p src data failed.");
261 return ret;
262 }
263 break;
264 }
265 case AV_PIX_FMT_NV12: {
266 int32_t ret = CopyNV12SrcData(srcImgInfo);
267 if (ret != DCAMERA_OK) {
268 DHLOGE("ScaleConvertProcess::ScaleConvert copy nv12 src data failed.");
269 return ret;
270 }
271 break;
272 }
273 case AV_PIX_FMT_NV21: {
274 int32_t ret = CopyNV21SrcData(srcImgInfo);
275 if (ret != DCAMERA_OK) {
276 DHLOGE("ScaleConvertProcess::ScaleConvert copy nv21 src data failed.");
277 return ret;
278 }
279 break;
280 }
281 default:
282 DHLOGE("Unknown pixel format not support.");
283 return DCAMERA_BAD_VALUE;
284 }
285
286 sws_scale(swsContext_, (const uint8_t * const *)srcData_, srcLineSize_, 0, srcImgInfo.alignedHeight,
287 dstData_, dstLineSize_);
288 int32_t ret = memcpy_s(dstImgInfo.imgData, dstImgInfo.imgSize, dstData_[0], dstBuffSize_);
289 if (ret != EOK) {
290 DHLOGE("ScaleConvertProcess::ScaleConvert copy dst image info failed, ret = %d", ret);
291 return DCAMERA_MEMORY_OPT_ERROR;
292 }
293 return DCAMERA_OK;
294 }
295
CopyYUV420SrcData(const ImageUnitInfo & srcImgInfo)296 int32_t ScaleConvertProcess::CopyYUV420SrcData(const ImageUnitInfo& srcImgInfo)
297 {
298 int32_t ret = memcpy_s(srcData_[0], srcImgInfo.alignedWidth * srcImgInfo.alignedHeight,
299 srcImgInfo.imgData, srcImgInfo.alignedWidth * srcImgInfo.alignedHeight);
300 if (ret != EOK) {
301 DHLOGE("ScaleConvertProcess::CopyYUV420SrcData memory copy failed, ret = %d", ret);
302 return DCAMERA_MEMORY_OPT_ERROR;
303 }
304
305 ret = memcpy_s(srcData_[1], srcImgInfo.alignedWidth * srcImgInfo.alignedHeight / MEMORY_RATIO_YUV,
306 srcImgInfo.imgData + srcImgInfo.alignedWidth * srcImgInfo.alignedHeight,
307 srcImgInfo.alignedWidth * srcImgInfo.alignedHeight / MEMORY_RATIO_YUV);
308 if (ret != EOK) {
309 DHLOGE("ScaleConvertProcess::CopyYUV420SrcData memory copy failed, ret = %d", ret);
310 return DCAMERA_MEMORY_OPT_ERROR;
311 }
312
313 ret = memcpy_s(srcData_[2], srcImgInfo.alignedWidth * srcImgInfo.alignedHeight / MEMORY_RATIO_YUV,
314 srcImgInfo.imgData + srcImgInfo.alignedWidth * srcImgInfo.alignedHeight +
315 srcImgInfo.alignedWidth * srcImgInfo.alignedHeight / MEMORY_RATIO_YUV,
316 srcImgInfo.alignedWidth * srcImgInfo.alignedHeight / MEMORY_RATIO_YUV);
317 if (ret != EOK) {
318 DHLOGE("ScaleConvertProcess::CopyYUV420SrcData memory copy failed, ret = %d", ret);
319 return DCAMERA_MEMORY_OPT_ERROR;
320 }
321 return DCAMERA_OK;
322 }
323
CopyNV12SrcData(const ImageUnitInfo & srcImgInfo)324 int32_t ScaleConvertProcess::CopyNV12SrcData(const ImageUnitInfo& srcImgInfo)
325 {
326 int32_t ret = memcpy_s(srcData_[0], srcImgInfo.alignedWidth * srcImgInfo.alignedHeight,
327 srcImgInfo.imgData, srcImgInfo.alignedWidth * srcImgInfo.alignedHeight);
328 if (ret != EOK) {
329 DHLOGE("ScaleConvertProcess::CopyNV12SrcData memory copy failed, ret = %d", ret);
330 return DCAMERA_MEMORY_OPT_ERROR;
331 }
332
333 ret = memcpy_s(srcData_[1], srcImgInfo.alignedWidth * srcImgInfo.alignedHeight / MEMORY_RATIO_NV,
334 srcImgInfo.imgData + srcImgInfo.alignedWidth * srcImgInfo.alignedHeight,
335 srcImgInfo.alignedWidth * srcImgInfo.alignedHeight / MEMORY_RATIO_NV);
336 if (ret != EOK) {
337 DHLOGE("ScaleConvertProcess::CopyNV12SrcData memory copy failed, ret = %d", ret);
338 return DCAMERA_MEMORY_OPT_ERROR;
339 }
340 return DCAMERA_OK;
341 }
342
CopyNV21SrcData(const ImageUnitInfo & srcImgInfo)343 int32_t ScaleConvertProcess::CopyNV21SrcData(const ImageUnitInfo& srcImgInfo)
344 {
345 int32_t ret = memcpy_s(srcData_[0], srcImgInfo.alignedWidth * srcImgInfo.alignedHeight,
346 srcImgInfo.imgData, srcImgInfo.alignedWidth * srcImgInfo.alignedHeight);
347 if (ret != EOK) {
348 DHLOGE("ScaleConvertProcess::CopyNV21SrcData memory copy failed, ret = %d", ret);
349 return DCAMERA_MEMORY_OPT_ERROR;
350 }
351
352 ret = memcpy_s(srcData_[1], srcImgInfo.alignedWidth * srcImgInfo.alignedHeight / MEMORY_RATIO_NV,
353 srcImgInfo.imgData + srcImgInfo.alignedWidth * srcImgInfo.alignedHeight,
354 srcImgInfo.alignedWidth * srcImgInfo.alignedHeight / MEMORY_RATIO_NV);
355 if (ret != EOK) {
356 DHLOGE("ScaleConvertProcess::CopyNV21SrcData memory copy failed, ret = %d", ret);
357 return DCAMERA_MEMORY_OPT_ERROR;
358 }
359 return DCAMERA_OK;
360 }
361
ConvertDone(std::vector<std::shared_ptr<DataBuffer>> & outputBuffers)362 int32_t ScaleConvertProcess::ConvertDone(std::vector<std::shared_ptr<DataBuffer>>& outputBuffers)
363 {
364 DHLOGD("ScaleConvertProcess : Convert Done.");
365 if (outputBuffers.empty()) {
366 DHLOGE("The received data buffer is empty.");
367 return DCAMERA_BAD_VALUE;
368 }
369
370 if (nextDataProcess_ != nullptr) {
371 DHLOGD("Send to the next node of the scale convert for processing.");
372 int32_t err = nextDataProcess_->ProcessData(outputBuffers);
373 if (err != DCAMERA_OK) {
374 DHLOGE("Some node after the scale convert processes failed.");
375 }
376 return err;
377 }
378
379 DHLOGD("The current node is the last noed, and output the processed video buffer.");
380 std::shared_ptr<DCameraPipelineSource> targetPipelineSource = callbackPipelineSource_.lock();
381 if (targetPipelineSource == nullptr) {
382 DHLOGE("callbackPipelineSource_ is nullptr.");
383 return DCAMERA_BAD_VALUE;
384 }
385 targetPipelineSource->OnProcessedVideoBuffer(outputBuffers[0]);
386 return DCAMERA_OK;
387 }
388
GetAVPixelFormat(Videoformat colorFormat)389 AVPixelFormat ScaleConvertProcess::GetAVPixelFormat(Videoformat colorFormat)
390 {
391 AVPixelFormat format;
392 switch (colorFormat) {
393 case Videoformat::NV12:
394 format = AVPixelFormat::AV_PIX_FMT_NV12;
395 break;
396 case Videoformat::NV21:
397 format = AVPixelFormat::AV_PIX_FMT_NV21;
398 break;
399 default:
400 format = AVPixelFormat::AV_PIX_FMT_YUV420P;
401 break;
402 }
403 return format;
404 }
405 } // namespace DistributedHardware
406 } // namespace OHOS
407