1 /*
2 * Copyright (c) 2024 Huawei Device Co., Ltd.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at
6 *
7 * http://www.apache.org/licenses/LICENSE-2.0
8 *
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
14 */
15
16 #include "skia_impl.h"
17
18 #include <array>
19 #include <chrono>
20 #include <dlfcn.h>
21
22 #include "detail_enhancer_extension.h"
23 #include "utils.h"
24
25 #include "vpe_log.h"
26
27 namespace OHOS {
28 namespace Media {
29 namespace VideoProcessingEngine {
30
31 namespace {
32 enum ImageFormatType {
33 IMAGE_FORMAT_TYPE_UNKNOWN = 0,
34 IMAGE_FORMAT_TYPE_RGB,
35 IMAGE_FORMAT_TYPE_YUV,
36 };
37
38 constexpr int CHANNEL_Y = 0;
39 constexpr int CHANNEL_UV1 = 1;
40 constexpr int CHANNEL_UV2 = 2;
41 constexpr int CHANNEL_NUM_NV12_NV21 = 2;
42 constexpr int CHANNEL_NUM_YU12_YV12 = 3;
43
GetImageType(const sptr<SurfaceBuffer> & input,const sptr<SurfaceBuffer> & output)44 ImageFormatType GetImageType(const sptr<SurfaceBuffer>& input, const sptr<SurfaceBuffer>& output)
45 {
46 if (input->GetFormat() != output->GetFormat()) {
47 VPE_LOGE("Different format for input and output!");
48 return IMAGE_FORMAT_TYPE_UNKNOWN;
49 }
50 VPE_LOGD("Input format: %{public}d, output format: %{public}d.", input->GetFormat(), output->GetFormat());
51 ImageFormatType imageType = IMAGE_FORMAT_TYPE_UNKNOWN;
52 switch (input->GetFormat()) {
53 case OHOS::GRAPHIC_PIXEL_FMT_RGBA_8888:
54 case OHOS::GRAPHIC_PIXEL_FMT_BGRA_8888:
55 imageType = IMAGE_FORMAT_TYPE_RGB;
56 break;
57 case OHOS::GRAPHIC_PIXEL_FMT_YCBCR_420_SP: // NV12
58 case OHOS::GRAPHIC_PIXEL_FMT_YCRCB_420_SP: // NV21
59 case OHOS::GRAPHIC_PIXEL_FMT_YCBCR_420_P: // YU12
60 case OHOS::GRAPHIC_PIXEL_FMT_YCRCB_420_P: // YV12
61 imageType = IMAGE_FORMAT_TYPE_YUV;
62 break;
63 default:
64 VPE_LOGE("Default unknown type.");
65 break;
66 }
67 return imageType;
68 }
69
GetRGBImageFormat(const sptr<SurfaceBuffer> & surfaceBuffer)70 SkColorType GetRGBImageFormat(const sptr<SurfaceBuffer>& surfaceBuffer)
71 {
72 SkColorType imageFormat;
73 switch (surfaceBuffer->GetFormat()) {
74 case OHOS::GRAPHIC_PIXEL_FMT_RGBA_8888:
75 imageFormat = kRGBA_8888_SkColorType;
76 break;
77 case OHOS::GRAPHIC_PIXEL_FMT_BGRA_8888:
78 imageFormat = kBGRA_8888_SkColorType;
79 break;
80 default:
81 imageFormat = kRGBA_8888_SkColorType;
82 break;
83 }
84 return imageFormat;
85 }
86
PixmapScale(const SkPixmap & inputPixmap,SkPixmap & outputPixmap,SkSamplingOptions options)87 VPEAlgoErrCode PixmapScale(const SkPixmap& inputPixmap, SkPixmap& outputPixmap, SkSamplingOptions options)
88 {
89 if (!inputPixmap.scalePixels(outputPixmap, options)) {
90 return VPE_ALGO_ERR_EXTENSION_PROCESS_FAILED;
91 }
92 return VPE_ALGO_ERR_OK;
93 }
94
RGBScale(const sptr<SurfaceBuffer> & input,const sptr<SurfaceBuffer> & output)95 VPEAlgoErrCode RGBScale(const sptr<SurfaceBuffer>& input, const sptr<SurfaceBuffer>& output)
96 {
97 if (input->GetWidth() <= 0 || input->GetHeight() <= 0 || output->GetWidth() <= 0 || output->GetHeight() <= 0) {
98 VPE_LOGE("Invalid input or output size!");
99 return VPE_ALGO_ERR_INVALID_VAL;
100 }
101 SkImageInfo inputInfo = SkImageInfo::Make(input->GetWidth(), input->GetHeight(), GetRGBImageFormat(input),
102 kPremul_SkAlphaType);
103 SkImageInfo outputInfo = SkImageInfo::Make(output->GetWidth(), output->GetHeight(), GetRGBImageFormat(output),
104 kPremul_SkAlphaType);
105 SkPixmap inputPixmap(inputInfo, input->GetVirAddr(), input->GetStride());
106 SkPixmap outputPixmap(outputInfo, output->GetVirAddr(), output->GetStride());
107
108 SkSamplingOptions scaleOption(SkFilterMode::kNearest);
109 return PixmapScale(inputPixmap, outputPixmap, scaleOption);
110 }
111
ConfigYUVFormat(const sptr<SurfaceBuffer> & buffer,SkYUVAInfo::PlaneConfig & planeConfig,size_t * rowbyte,unsigned char ** pixmapAddr)112 int ConfigYUVFormat(const sptr<SurfaceBuffer>& buffer, SkYUVAInfo::PlaneConfig& planeConfig, size_t* rowbyte,
113 unsigned char** pixmapAddr)
114 {
115 int numPlanes;
116 void* planesInfoPtr = nullptr;
117 buffer->GetPlanesInfo(&planesInfoPtr);
118 auto planesInfo = static_cast<OH_NativeBuffer_Planes*>(planesInfoPtr);
119 switch (buffer->GetFormat()) {
120 case OHOS::GRAPHIC_PIXEL_FMT_YCBCR_420_SP: // NV12
121 planeConfig = SkYUVAInfo::PlaneConfig::kY_UV;
122 rowbyte[CHANNEL_UV1] = planesInfo->planes[CHANNEL_UV1].columnStride;
123 pixmapAddr[CHANNEL_UV1] = pixmapAddr[CHANNEL_Y] +
124 static_cast<unsigned int>(planesInfo->planes[CHANNEL_UV1].offset);
125 numPlanes = CHANNEL_NUM_NV12_NV21;
126 break;
127 case OHOS::GRAPHIC_PIXEL_FMT_YCRCB_420_SP: // NV21
128 planeConfig = SkYUVAInfo::PlaneConfig::kY_VU;
129 rowbyte[CHANNEL_UV1] = planesInfo->planes[CHANNEL_UV2].columnStride;
130 pixmapAddr[CHANNEL_UV1] = pixmapAddr[CHANNEL_Y] +
131 static_cast<unsigned int>(planesInfo->planes[CHANNEL_UV2].offset);
132 numPlanes = CHANNEL_NUM_NV12_NV21;
133 break;
134 case OHOS::GRAPHIC_PIXEL_FMT_YCBCR_420_P: // YU12
135 planeConfig = SkYUVAInfo::PlaneConfig::kY_U_V;
136 rowbyte[CHANNEL_UV1] = planesInfo->planes[CHANNEL_UV1].columnStride;
137 rowbyte[CHANNEL_UV2] = planesInfo->planes[CHANNEL_UV2].columnStride;
138 pixmapAddr[CHANNEL_UV1] = pixmapAddr[CHANNEL_Y] +
139 static_cast<unsigned int>(planesInfo->planes[CHANNEL_UV1].offset);
140 pixmapAddr[CHANNEL_UV2] = pixmapAddr[CHANNEL_Y] +
141 static_cast<unsigned int>(planesInfo->planes[CHANNEL_UV2].offset);
142 numPlanes = CHANNEL_NUM_YU12_YV12;
143 break;
144 case OHOS::GRAPHIC_PIXEL_FMT_YCRCB_420_P: // YV12
145 planeConfig = SkYUVAInfo::PlaneConfig::kY_V_U;
146 rowbyte[CHANNEL_UV1] = planesInfo->planes[CHANNEL_UV2].columnStride;
147 rowbyte[CHANNEL_UV2] = planesInfo->planes[CHANNEL_UV1].columnStride;
148 pixmapAddr[CHANNEL_UV1] = pixmapAddr[CHANNEL_Y] +
149 static_cast<unsigned int>(planesInfo->planes[CHANNEL_UV2].offset);
150 pixmapAddr[CHANNEL_UV2] = pixmapAddr[CHANNEL_Y] +
151 static_cast<unsigned int>(planesInfo->planes[CHANNEL_UV1].offset);
152 numPlanes = CHANNEL_NUM_YU12_YV12;
153 break;
154 default:
155 VPE_LOGD("YUVFormat: default NV12.");
156 planeConfig = SkYUVAInfo::PlaneConfig::kY_UV;
157 rowbyte[CHANNEL_UV1] = planesInfo->planes[CHANNEL_UV1].columnStride;
158 numPlanes = CHANNEL_NUM_NV12_NV21;
159 break;
160 }
161 return numPlanes;
162 }
163
CreateYUVPixmap(const sptr<SurfaceBuffer> & buffer,std::array<SkPixmap,SkYUVAInfo::kMaxPlanes> & pixmaps)164 int CreateYUVPixmap(const sptr<SurfaceBuffer>& buffer, std::array<SkPixmap, SkYUVAInfo::kMaxPlanes>& pixmaps)
165 {
166 SkISize imageSize;
167 imageSize.fWidth = buffer->GetWidth();
168 if (imageSize.fWidth <= 0) {
169 VPE_LOGE("Invalid width!");
170 return 0;
171 }
172 imageSize.fHeight = buffer->GetHeight();
173 if (imageSize.fHeight <= 0) {
174 VPE_LOGE("Invalid height!");
175 return 0;
176 }
177 SkYUVColorSpace yuvColorSpace = SkYUVColorSpace::kRec709_Full_SkYUVColorSpace;
178 SkYUVAInfo::Subsampling subsampling = SkYUVAInfo::Subsampling::k420;
179 SkYUVAInfo::PlaneConfig planeConfig = SkYUVAInfo::PlaneConfig::kY_UV;
180 size_t rowbyte[SkYUVAInfo::kMaxPlanes];
181 unsigned char* pixmapAddr[SkYUVAInfo::kMaxPlanes];
182
183 void* planesInfoPtr = nullptr;
184 buffer->GetPlanesInfo(&planesInfoPtr);
185 auto planesInfo = static_cast<OH_NativeBuffer_Planes*>(planesInfoPtr);
186 if (planesInfoPtr == nullptr) {
187 VPE_LOGD("Planes info is nullptr, configure uv stride with general stride.");
188 for (int i = 0; i < SkYUVAInfo::kMaxPlanes; i++) {
189 rowbyte[i] = static_cast<size_t>(buffer->GetStride());
190 }
191 return 0;
192 }
193
194 rowbyte[CHANNEL_Y] = planesInfo->planes[CHANNEL_Y].columnStride;
195 pixmapAddr[CHANNEL_Y] = static_cast<unsigned char*>(buffer->GetVirAddr());
196 int numPlanes = ConfigYUVFormat(buffer, planeConfig, rowbyte, pixmapAddr);
197
198 const SkYUVAInfo yuvInfo = SkYUVAInfo(imageSize, planeConfig, subsampling, yuvColorSpace);
199 const SkYUVAPixmapInfo pixmapInfo = SkYUVAPixmapInfo(yuvInfo, SkYUVAPixmapInfo::DataType::kUnorm8, rowbyte);
200
201 for (int i = 0; i < numPlanes; i++) {
202 pixmaps[i].reset(pixmapInfo.planeInfo(i), pixmapAddr[i], rowbyte[i]);
203 }
204 return numPlanes;
205 }
206
YUVPixmapScale(const std::array<SkPixmap,SkYUVAInfo::kMaxPlanes> & inputPixmap,std::array<SkPixmap,SkYUVAInfo::kMaxPlanes> & outputPixmap,SkSamplingOptions opt,int numPlanes)207 VPEAlgoErrCode YUVPixmapScale(const std::array<SkPixmap, SkYUVAInfo::kMaxPlanes>& inputPixmap,
208 std::array<SkPixmap, SkYUVAInfo::kMaxPlanes>& outputPixmap, SkSamplingOptions opt, int numPlanes)
209 {
210 for (int i = 0; i < numPlanes; i++) {
211 if (!inputPixmap[i].scalePixels(outputPixmap[i], opt)) {
212 VPE_LOGE("YUV scale failed!");
213 return VPE_ALGO_ERR_EXTENSION_PROCESS_FAILED;
214 }
215 }
216 return VPE_ALGO_ERR_OK;
217 }
218
YUVScale(const sptr<SurfaceBuffer> & input,const sptr<SurfaceBuffer> & output)219 VPEAlgoErrCode YUVScale(const sptr<SurfaceBuffer>& input, const sptr<SurfaceBuffer>& output)
220 {
221 std::array<SkPixmap, SkYUVAInfo::kMaxPlanes> inputPixmap;
222 std::array<SkPixmap, SkYUVAInfo::kMaxPlanes> outputPixmap;
223 int numPlanesInput = CreateYUVPixmap(input, inputPixmap);
224 int numPlanesOutput = CreateYUVPixmap(output, outputPixmap);
225 if (numPlanesInput != numPlanesOutput || numPlanesInput * numPlanesOutput == 0) {
226 VPE_LOGE("Wrong YUV settings!");
227 return VPE_ALGO_ERR_INVALID_VAL;
228 }
229 SkSamplingOptions scaleOption(SkFilterMode::kNearest);
230 return YUVPixmapScale(inputPixmap, outputPixmap, scaleOption, numPlanesInput);
231 }
232
233 constexpr Extension::Rank RANK = Extension::Rank::RANK_DEFAULT;
234 constexpr uint32_t VERSION = 0;
235 } // namespace
236
Create()237 std::unique_ptr<DetailEnhancerBase> Skia::Create()
238 {
239 return std::make_unique<Skia>();
240 }
241
BuildCapabilities()242 DetailEnhancerCapability Skia::BuildCapabilities()
243 {
244 std::vector<uint32_t> levels = { DETAIL_ENH_LEVEL_NONE, DETAIL_ENH_LEVEL_LOW, DETAIL_ENH_LEVEL_MEDIUM,
245 DETAIL_ENH_LEVEL_HIGH_EVE, DETAIL_ENH_LEVEL_HIGH_AISR, DETAIL_ENH_LEVEL_VIDEO};
246 DetailEnhancerCapability capability = { levels, RANK, VERSION };
247 return capability;
248 }
249
Init()250 VPEAlgoErrCode Skia::Init()
251 {
252 return VPE_ALGO_ERR_OK;
253 }
254
Deinit()255 VPEAlgoErrCode Skia::Deinit()
256 {
257 return VPE_ALGO_ERR_OK;
258 }
259
SetParameter(const DetailEnhancerParameters & parameter)260 VPEAlgoErrCode Skia::SetParameter([[maybe_unused]] const DetailEnhancerParameters& parameter)
261 {
262 return VPE_ALGO_ERR_OK;
263 }
264
EnableProtection(bool enable)265 VPEAlgoErrCode Skia::EnableProtection([[maybe_unused]] bool enable)
266 {
267 return VPE_ALGO_ERR_OK;
268 }
269
ResetProtectionStatus()270 VPEAlgoErrCode Skia::ResetProtectionStatus()
271 {
272 return VPE_ALGO_ERR_OK;
273 }
274
Process(const sptr<SurfaceBuffer> & input,const sptr<SurfaceBuffer> & output)275 VPEAlgoErrCode Skia::Process(const sptr<SurfaceBuffer>& input, const sptr<SurfaceBuffer>& output)
276 {
277 VPEAlgoErrCode errCode;
278 ImageFormatType imageType = GetImageType(input, output);
279 if (imageType == IMAGE_FORMAT_TYPE_RGB) {
280 errCode = RGBScale(input, output);
281 } else if (imageType == IMAGE_FORMAT_TYPE_YUV) {
282 errCode = YUVScale(input, output);
283 } else {
284 VPE_LOGE("Unknown image format!");
285 errCode = VPE_ALGO_ERR_INVALID_VAL;
286 }
287 return errCode;
288 }
289
RegisterExtensions()290 static std::vector<std::shared_ptr<Extension::ExtensionBase>> RegisterExtensions()
291 {
292 std::vector<std::shared_ptr<Extension::ExtensionBase>> extensions;
293
294 auto extension = std::make_shared<Extension::DetailEnhancerExtension>();
295 CHECK_AND_RETURN_RET_LOG(extension != nullptr, extensions, "null pointer");
296 extension->info = { Extension::ExtensionType::DETAIL_ENHANCER, "AISR", "0.0.1" };
297 extension->creator = Skia::Create;
298 extension->capabilitiesBuilder = Skia::BuildCapabilities;
299 extensions.push_back(std::static_pointer_cast<Extension::ExtensionBase>(extension));
300
301 return extensions;
302 }
303
304
RegisterSkiaExtensions(uintptr_t extensionListAddr)305 void RegisterSkiaExtensions(uintptr_t extensionListAddr)
306 {
307 Extension::DoRegisterExtensions(extensionListAddr, RegisterExtensions);
308 }
309
310 } // VideoProcessingEngine
311 } // Media
312 } // OHOS