1 /*
2 * Copyright (C) 2023 Huawei Device Co., Ltd.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at
6 *
7 * http://www.apache.org/licenses/LICENSE-2.0
8 *
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
14 */
15
16 #include "ext_encoder.h"
17 #include <algorithm>
18 #include <map>
19
20 #ifdef USE_M133_SKIA
21 #include "include/encode/SkJpegEncoder.h"
22 #include "include/encode/SkPngEncoder.h"
23 #include "include/encode/SkWebpEncoder.h"
24 #include "src/encode/SkImageEncoderFns.h"
25 #else
26 #include "include/core/SkImageEncoder.h"
27 #include "src/images/SkImageEncoderFns.h"
28 #endif
29 #include "include/core/SkBitmap.h"
30 #include "pixel_yuv_utils.h"
31 #ifdef IMAGE_COLORSPACE_FLAG
32 #include "color_space.h"
33 #endif
34 #if !defined(IOS_PLATFORM) && !defined(ANDROID_PLATFORM)
35 #include "astc_codec.h"
36 #endif
37
38 #include "ext_pixel_convert.h"
39 #include "ext_wstream.h"
40 #include "image_data_statistics.h"
41 #include "image_dfx.h"
42 #include "image_format_convert.h"
43 #include "image_func_timer.h"
44 #include "image_fwk_ext_manager.h"
45 #include "image_log.h"
46 #include "image_system_properties.h"
47 #include "image_type_converter.h"
48 #include "image_utils.h"
49 #include "jpeg_mpf_parser.h"
50 #include "media_errors.h"
51 #include "metadata_accessor.h"
52 #include "metadata_accessor_factory.h"
53 #include "pixel_convert_adapter.h"
54 #include "string_ex.h"
55 #if !defined(IOS_PLATFORM) && !defined(ANDROID_PLATFORM)
56 #include "surface_buffer.h"
57 #include "v1_0/buffer_handle_meta_key_type.h"
58 #include "v1_0/cm_color_space.h"
59 #include "v1_0/hdr_static_metadata.h"
60 #include "v2_1/cm_color_space.h"
61 #include "vpe_utils.h"
62 #include "hdr_helper.h"
63 #endif
64 #include "color_utils.h"
65 #include "tiff_parser.h"
66 #include "image_mime_type.h"
67 #include "securec.h"
68 #include "file_packer_stream.h"
69 #include "memory_manager.h"
70 #ifdef HEIF_HW_ENCODE_ENABLE
71 #include "v2_1/icodec_image.h"
72 #include "iremote_object.h"
73 #include "iproxy_broker.h"
74 #endif
75
76 #undef LOG_DOMAIN
77 #define LOG_DOMAIN LOG_TAG_DOMAIN_ID_PLUGIN
78
79 #undef LOG_TAG
80 #define LOG_TAG "ExtEncoder"
81
82 namespace OHOS {
83 namespace ImagePlugin {
84 using namespace Media;
85 #if !defined(IOS_PLATFORM) && !defined(ANDROID_PLATFORM)
86 using namespace HDI::Display::Graphic::Common::V1_0;
87 using CM_ColorSpaceType_V2_1 = OHOS::HDI::Display::Graphic::Common::V2_1::CM_ColorSpaceType;
88 const std::map<PixelFormat, GraphicPixelFormat> SURFACE_FORMAT_MAP = {
89 { PixelFormat::RGBA_8888, GRAPHIC_PIXEL_FMT_RGBA_8888 },
90 { PixelFormat::NV21, GRAPHIC_PIXEL_FMT_YCRCB_420_SP },
91 { PixelFormat::NV12, GRAPHIC_PIXEL_FMT_YCBCR_420_SP },
92 };
93
94 #ifdef IMAGE_COLORSPACE_FLAG
95 static const std::map<CM_ColorSpaceType, ColorManager::ColorSpaceName> CM_COLORSPACE_NAME_MAP = {
96 { CM_COLORSPACE_NONE, ColorManager::NONE },
97 { CM_BT601_EBU_FULL, ColorManager::BT601_EBU },
98 { CM_BT601_SMPTE_C_FULL, ColorManager::BT601_SMPTE_C },
99 { CM_BT709_FULL, ColorManager::BT709 },
100 { CM_BT2020_HLG_FULL, ColorManager::BT2020_HLG },
101 { CM_BT2020_PQ_FULL, ColorManager::BT2020_PQ },
102 { CM_BT601_EBU_LIMIT, ColorManager::BT601_EBU_LIMIT },
103 { CM_BT601_SMPTE_C_LIMIT, ColorManager::BT601_SMPTE_C_LIMIT },
104 { CM_BT709_LIMIT, ColorManager::BT709_LIMIT },
105 { CM_BT2020_HLG_LIMIT, ColorManager::BT2020_HLG_LIMIT },
106 { CM_BT2020_PQ_LIMIT, ColorManager::BT2020_PQ_LIMIT },
107 { CM_SRGB_FULL, ColorManager::SRGB },
108 { CM_P3_FULL, ColorManager::DISPLAY_P3 },
109 { CM_P3_HLG_FULL, ColorManager::P3_HLG },
110 { CM_P3_PQ_FULL, ColorManager::P3_PQ },
111 { CM_ADOBERGB_FULL, ColorManager::ADOBE_RGB },
112 { CM_SRGB_LIMIT, ColorManager::SRGB_LIMIT },
113 { CM_P3_LIMIT, ColorManager::DISPLAY_P3_LIMIT },
114 { CM_P3_HLG_LIMIT, ColorManager::P3_HLG_LIMIT },
115 { CM_P3_PQ_LIMIT, ColorManager::P3_PQ_LIMIT },
116 { CM_ADOBERGB_LIMIT, ColorManager::ADOBE_RGB_LIMIT },
117 { CM_LINEAR_SRGB, ColorManager::LINEAR_SRGB },
118 { CM_LINEAR_BT709, ColorManager::LINEAR_BT709 },
119 { CM_LINEAR_P3, ColorManager::LINEAR_P3 },
120 { CM_LINEAR_BT2020, ColorManager::LINEAR_BT2020 },
121 { CM_DISPLAY_SRGB, ColorManager::DISPLAY_SRGB },
122 { CM_DISPLAY_P3_SRGB, ColorManager::DISPLAY_P3_SRGB },
123 { CM_DISPLAY_P3_HLG, ColorManager::DISPLAY_P3_HLG },
124 { CM_DISPLAY_P3_PQ, ColorManager::DISPLAY_P3_PQ },
125 { CM_DISPLAY_BT2020_SRGB, ColorManager::DISPLAY_BT2020_SRGB },
126 { CM_DISPLAY_BT2020_HLG, ColorManager::DISPLAY_BT2020_HLG },
127 { CM_DISPLAY_BT2020_PQ, ColorManager::DISPLAY_BT2020_PQ },
128 };
129 #endif
130
131 #endif
132
133 namespace {
134 constexpr static uint32_t PRIMARY_IMAGE_ITEM_ID = 1;
135 constexpr static uint32_t GAINMAP_IMAGE_ITEM_ID = 2;
136 constexpr static uint32_t TMAP_IMAGE_ITEM_ID = 3;
137 constexpr static uint32_t EXIF_META_ITEM_ID = 4;
138 constexpr static uint32_t DEPTH_MAP_ITEM_ID = 5;
139 constexpr static uint32_t UNREFOCUS_MAP_ITEM_ID = 6;
140 constexpr static uint32_t LINEAR_MAP_ITEM_ID = 7;
141 constexpr static uint32_t FRAGMENT_MAP_ITEM_ID = 8;
142 const static std::string COMPRESS_TYPE_TMAP = "tmap";
143 const static std::string COMPRESS_TYPE_HEVC = "hevc";
144 const static std::string COMPRESS_TYPE_NONE = "none";
145 const static std::string DEFAULT_ASHMEM_TAG = "Heif Encoder Default";
146 const static std::string ICC_ASHMEM_TAG = "Heif Encoder Property";
147 const static std::string IT35_ASHMEM_TAG = "Heif Encoder IT35";
148 const static std::string OUTPUT_ASHMEM_TAG = "Heif Encoder Output";
149 const static std::string IMAGE_DATA_TAG = "Heif Encoder Image";
150 const static std::string HDR_GAINMAP_TAG = "Heif Encoder Gainmap";
151 const static std::string EXIF_ASHMEM_TAG = "Heif Encoder Exif";
152 const static std::string GAINMAP_IMAGE_ITEM_NAME = "Gain map Image";
153 const static std::string DEPTH_MAP_ITEM_NAME = "Depth Map Image";
154 const static std::string UNREFOCUS_MAP_ITEM_NAME = "Unrefocus Map Image";
155 const static std::string LINEAR_MAP_ITEM_NAME = "Linear Map Image";
156 const static std::string FRAGMENT_MAP_ITEM_NAME = "Fragment Map Image";
157 const static size_t DEFAULT_OUTPUT_SIZE = 35 * 1024 * 1024; // 35M
158 const static uint16_t STATIC_METADATA_COLOR_SCALE = 50000;
159 const static uint16_t STATIC_METADATA_LUM_SCALE = 10000;
160 const static uint8_t INDEX_ZERO = 0;
161 const static uint8_t INDEX_ONE = 1;
162 const static uint8_t INDEX_TWO = 2;
163 constexpr uint32_t DEFAULT_DENOMINATOR = 1000000;
164 constexpr uint8_t GAINMAP_CHANNEL_MULTI = 3;
165 constexpr uint8_t GAINMAP_CHANNEL_SINGLE = 1;
166 constexpr uint8_t EXIF_PRE_SIZE = 6;
167 constexpr uint32_t JPEG_MARKER_TAG_SIZE = 2;
168 constexpr uint32_t DEPTH_MAP_BYTES = sizeof(float); // float16
169 constexpr uint32_t LINEAR_MAP_BYTES = sizeof(short) * 3 / 2; // 16bit yuv420
170 constexpr uint32_t PLACE_HOLDER_LENGTH = 1;
171 constexpr uint32_t LOW_QUALITY_BOUNDARY = 50;
172 constexpr uint32_t RGBA8888_PIXEL_BYTES = 4;
173 constexpr uint32_t HIGHEST_QUALITY = 100;
174 constexpr uint32_t DEFAULT_QUALITY = 75;
175 static constexpr uint32_t TRANSFUNC_OFFSET = 8;
176 static constexpr uint32_t MATRIX_OFFSET = 8;
177 static constexpr uint32_t RANGE_OFFSET = 8;
178
179 // exif/0/0
180 constexpr uint8_t EXIF_PRE_TAG[EXIF_PRE_SIZE] = {
181 0x45, 0x78, 0x69, 0x66, 0x00, 0x00
182 };
183 }
184
185 struct HeifEncodeItemInfo {
186 uint32_t itemId = 0;
187 std::string itemName = "";
188 std::string itemType = "";
189 };
190
191 static const std::map<std::string, SkEncodedImageFormat> FORMAT_NAME = {
192 {IMAGE_BMP_FORMAT, SkEncodedImageFormat::kBMP},
193 {IMAGE_GIF_FORMAT, SkEncodedImageFormat::kGIF},
194 {IMAGE_ICO_FORMAT, SkEncodedImageFormat::kICO},
195 {IMAGE_JPEG_FORMAT, SkEncodedImageFormat::kJPEG},
196 {IMAGE_PNG_FORMAT, SkEncodedImageFormat::kPNG},
197 {IMAGE_WBMP_FORMAT, SkEncodedImageFormat::kWBMP},
198 {IMAGE_WEBP_FORMAT, SkEncodedImageFormat::kWEBP},
199 {IMAGE_HEIF_FORMAT, SkEncodedImageFormat::kHEIF},
200 {IMAGE_HEIC_FORMAT, SkEncodedImageFormat::kHEIF},
201 };
202
203 static const std::map<AuxiliaryPictureType, std::string> DEFAULT_AUXILIARY_TAG_MAP = {
204 {AuxiliaryPictureType::GAINMAP, AUXILIARY_TAG_GAINMAP},
205 {AuxiliaryPictureType::DEPTH_MAP, AUXILIARY_TAG_DEPTH_MAP_BACK},
206 {AuxiliaryPictureType::UNREFOCUS_MAP, AUXILIARY_TAG_UNREFOCUS_MAP},
207 {AuxiliaryPictureType::LINEAR_MAP, AUXILIARY_TAG_LINEAR_MAP},
208 {AuxiliaryPictureType::FRAGMENT_MAP, AUXILIARY_TAG_FRAGMENT_MAP},
209 };
210
211 static const uint8_t NUM_3 = 3;
212 static const uint8_t NUM_4 = 4;
213 static const uint8_t RGBA_BIT_DEPTH = 4;
214
215 static constexpr int32_t MAX_IMAGE_SIZE = 32768;
216 static constexpr int32_t MIN_IMAGE_SIZE = 128;
217 static constexpr int32_t MIN_RGBA_IMAGE_SIZE = 1024;
218
219 #ifdef HEIF_HW_ENCODE_ENABLE
220 using namespace OHOS::HDI::Codec::Image::V2_1;
221 static std::mutex g_codecMtx;
222 static sptr<ICodecImage> g_codecMgr;
223 class CodecHeifDeathRecipient : public IRemoteObject::DeathRecipient {
224 public:
OnRemoteDied(const wptr<OHOS::IRemoteObject> & object)225 void OnRemoteDied(const wptr<OHOS::IRemoteObject>& object) override
226 {
227 IMAGE_LOGW("codec_image_service died");
228 std::lock_guard<std::mutex> lk(g_codecMtx);
229 g_codecMgr = nullptr;
230 }
231 };
232
GetCodecManager()233 static sptr<ICodecImage> GetCodecManager()
234 {
235 std::lock_guard<std::mutex> lk(g_codecMtx);
236 if (g_codecMgr) {
237 return g_codecMgr;
238 }
239 IMAGE_LOGI("need to get ICodecImage");
240 g_codecMgr = ICodecImage::Get();
241 bool cond = g_codecMgr == nullptr;
242 CHECK_ERROR_RETURN_RET_LOG(cond, nullptr, "ICodecImage get failed");
243 bool isDeathRecipientAdded = false;
244 const sptr<OHOS::IRemoteObject> &remote = OHOS::HDI::hdi_objcast<ICodecImage>(g_codecMgr);
245 if (remote) {
246 sptr<CodecHeifDeathRecipient> deathCallBack(new CodecHeifDeathRecipient());
247 isDeathRecipientAdded = remote->AddDeathRecipient(deathCallBack);
248 }
249 if (!isDeathRecipientAdded) {
250 IMAGE_LOGI("failed to add deathRecipient for ICodecImage!");
251 }
252 return g_codecMgr;
253 }
254 #endif
255
ExtEncoder()256 ExtEncoder::ExtEncoder()
257 {
258 }
259
~ExtEncoder()260 ExtEncoder::~ExtEncoder()
261 {
262 }
263
StartEncode(OutputDataStream & outputStream,PlEncodeOptions & option)264 uint32_t ExtEncoder::StartEncode(OutputDataStream &outputStream, PlEncodeOptions &option)
265 {
266 output_ = &outputStream;
267 opts_ = option;
268 return SUCCESS;
269 }
270
AddImage(PixelMap & pixelMap)271 uint32_t ExtEncoder::AddImage(PixelMap &pixelMap)
272 {
273 pixelmap_ = &pixelMap;
274 return SUCCESS;
275 }
276
AddPicture(Media::Picture & picture)277 uint32_t ExtEncoder::AddPicture(Media::Picture &picture)
278 {
279 picture_ = &picture;
280 return SUCCESS;
281 }
282
283 struct TmpBufferHolder {
284 std::unique_ptr<uint8_t[]> buf = nullptr;
285 };
286
287 struct ImageData {
288 uint8_t *dst;
289 uint8_t *pixels;
290 ImageInfo info;
291 };
292
ToSkColorSpace(PixelMap * pixelmap)293 static sk_sp<SkColorSpace> ToSkColorSpace(PixelMap *pixelmap)
294 {
295 #ifdef IMAGE_COLORSPACE_FLAG
296 bool cond = pixelmap->InnerGetGrColorSpacePtr() == nullptr;
297 CHECK_ERROR_RETURN_RET(cond, nullptr);
298
299 cond = pixelmap->InnerGetGrColorSpace().GetColorSpaceName() == ColorManager::ColorSpaceName::NONE;
300 CHECK_ERROR_RETURN_RET(cond, SkColorSpace::MakeSRGB());
301 return pixelmap->InnerGetGrColorSpacePtr()->ToSkColorSpace();
302 #else
303 return nullptr;
304 #endif
305 }
306
ToSkInfo(Media::PixelMap * pixelMap)307 static SkImageInfo ToSkInfo(Media::PixelMap *pixelMap)
308 {
309 ImageInfo info;
310 pixelMap->GetImageInfo(info);
311 SkColorType colorType = ImageTypeConverter::ToSkColorType(info.pixelFormat);
312 SkAlphaType alphaType = ImageTypeConverter::ToSkAlphaType(info.alphaType);
313 sk_sp<SkColorSpace> colorSpace = SkColorSpace::MakeSRGB();
314 #ifdef IMAGE_COLORSPACE_FLAG
315 if (pixelMap->InnerGetGrColorSpacePtr() != nullptr &&
316 pixelMap->InnerGetGrColorSpace().GetColorSpaceName() != ColorManager::ColorSpaceName::NONE) {
317 colorSpace = pixelMap->InnerGetGrColorSpacePtr()->ToSkColorSpace();
318 }
319 #endif
320 return SkImageInfo::Make(info.size.width, info.size.height, colorType, alphaType, colorSpace);
321 }
322
ToHdrEncodeSkColorSpace(Media::PixelMap * pixelmap,sptr<SurfaceBuffer> & buffer,bool sdrIsSRGB,bool isGainmap)323 static sk_sp<SkColorSpace> ToHdrEncodeSkColorSpace(Media::PixelMap *pixelmap,
324 sptr<SurfaceBuffer>& buffer, bool sdrIsSRGB, bool isGainmap)
325 {
326 #ifdef IMAGE_COLORSPACE_FLAG
327 // get graphic colorspace
328 ColorManager::ColorSpaceName graphicColorSpaceName = ColorManager::ColorSpaceName::SRGB;
329 #if !defined(IOS_PLATFORM) && !defined(ANDROID_PLATFORM)
330 CM_ColorSpaceType color;
331 VpeUtils::GetSbColorSpaceType(buffer, color);
332 auto iter = CM_COLORSPACE_NAME_MAP.find(color);
333 if (iter != CM_COLORSPACE_NAME_MAP.end()) {
334 graphicColorSpaceName = iter->second;
335 }
336 #endif
337 skcms_CICP cicp;
338 ColorUtils::ColorSpaceGetCicp(graphicColorSpaceName,
339 #ifdef USE_M133_SKIA
340 cicp.color_primaries, cicp.transfer_characteristics, cicp.matrix_coefficients, cicp.video_full_range_flag);
341 #else
342 cicp.colour_primaries, cicp.transfer_characteristics, cicp.matrix_coefficients, cicp.full_range_flag);
343 #endif
344 sk_sp<SkColorSpace> colorSpace =
345 SkColorSpace::MakeRGB(SkNamedTransferFn::kSRGB, sdrIsSRGB? SkNamedGamut::kSRGB : SkNamedGamut::kDisplayP3);
346 if (isGainmap) {
347 colorSpace = SkColorSpace::MakeRGB(SkNamedTransferFn::kRec2020, SkNamedGamut::kRec2020);
348 if (pixelmap->InnerGetGrColorSpacePtr() != nullptr &&
349 pixelmap->InnerGetGrColorSpace().GetColorSpaceName() != ColorManager::ColorSpaceName::NONE) {
350 colorSpace = pixelmap->InnerGetGrColorSpacePtr()->ToSkColorSpace();
351 }
352 }
353 colorSpace->SetIccCicp(cicp);
354 return colorSpace;
355 #endif
356 }
357
RGBToRGBx(PixelMap * pixelMap,SkImageInfo & skInfo,TmpBufferHolder & holder)358 static uint32_t RGBToRGBx(PixelMap *pixelMap, SkImageInfo &skInfo, TmpBufferHolder &holder)
359 {
360 holder.buf = std::make_unique<uint8_t[]>(skInfo.computeMinByteSize());
361 ExtPixels src = {
362 static_cast<uint8_t*>(pixelMap->GetWritablePixels()),
363 pixelMap->GetCapacity(), pixelMap->GetWidth()*pixelMap->GetHeight(),
364 };
365 ExtPixels dst = {
366 holder.buf.get(), skInfo.computeMinByteSize(), skInfo.width()*skInfo.height(),
367 };
368 return ExtPixelConvert::RGBToRGBx(src, dst);
369 }
370
IsYuvImage(PixelFormat format)371 static bool IsYuvImage(PixelFormat format)
372 {
373 return format == PixelFormat::NV21 || format == PixelFormat::NV12;
374 }
375
YuvToRgbaSkInfo(ImageInfo info,SkImageInfo & skInfo,uint8_t * dstData,Media::PixelMap * pixelMap)376 static uint32_t YuvToRgbaSkInfo(ImageInfo info, SkImageInfo &skInfo, uint8_t * dstData, Media::PixelMap *pixelMap)
377 {
378 uint8_t *srcData = static_cast<uint8_t*>(pixelMap->GetWritablePixels());
379 YUVDataInfo yuvInfo;
380 pixelMap->GetImageYUVInfo(yuvInfo);
381 ConvertDataInfo srcDataInfo = {srcData, static_cast<size_t>(pixelMap->GetByteCount()), info.size,
382 info.pixelFormat, pixelMap->GetColorSpace(), yuvInfo};
383 DestConvertInfo dstDataInfo;
384 dstDataInfo.format = PixelFormat::RGBA_8888;
385 dstDataInfo.allocType = AllocatorType::HEAP_ALLOC;
386 IMAGE_LOGD("HDR-IMAGE yuvToRgba width : %{public}d, height : %{public}d",
387 srcDataInfo.imageSize.width, srcDataInfo.imageSize.height);
388 ImageUtils::DumpDataIfDumpEnabled(reinterpret_cast<const char*>(srcDataInfo.buffer), srcDataInfo.bufferSize,
389 "beforeConvert");
390 bool cond = ImageFormatConvert::ConvertImageFormat(srcDataInfo, dstDataInfo) != SUCCESS;
391 ImageUtils::DumpDataIfDumpEnabled(reinterpret_cast<const char*>(dstDataInfo.buffer), dstDataInfo.bufferSize,
392 "afterConvert");
393 CHECK_ERROR_RETURN_RET_LOG(cond, ERR_IMAGE_ENCODE_FAILED, "YuvToSkInfo Support YUV format RGB convert failed ");
394 cond = memcpy_s(dstData, info.size.width * info.size.height * RGBA8888_PIXEL_BYTES,
395 dstDataInfo.buffer, dstDataInfo.bufferSize) != 0;
396 delete dstDataInfo.buffer;
397 CHECK_ERROR_RETURN_RET_LOG(cond, ERR_IMAGE_ENCODE_FAILED, "YuvToSkInfo memcpy failed ");
398 auto alpha = pixelMap->GetAlphaType();
399 if (alpha == AlphaType::IMAGE_ALPHA_TYPE_UNKNOWN)
400 alpha = AlphaType::IMAGE_ALPHA_TYPE_OPAQUE;
401 SkAlphaType alphaType = ImageTypeConverter::ToSkAlphaType(alpha);
402 auto cs = ToSkColorSpace(pixelMap);
403 skInfo = SkImageInfo::Make(info.size.width, info.size.height, SkColorType::kRGBA_8888_SkColorType, alphaType, cs);
404 IMAGE_LOGD(" YuvToSkInfo: width:%{public}d, height:%{public}d, alpha:%{public}d \n ",
405 info.size.width, info.size.height, (int32_t)alphaType);
406 return SUCCESS;
407 }
408
pixelToSkInfo(ImageData & image,SkImageInfo & skInfo,Media::PixelMap * pixelMap,TmpBufferHolder & holder,SkEncodedImageFormat format)409 static uint32_t pixelToSkInfo(ImageData &image, SkImageInfo &skInfo, Media::PixelMap *pixelMap,
410 TmpBufferHolder &holder, SkEncodedImageFormat format)
411 {
412 skInfo = ToSkInfo(pixelMap);
413 image.pixels = static_cast<uint8_t*>(pixelMap->GetWritablePixels());
414 if (skInfo.colorType() == SkColorType::kRGB_888x_SkColorType &&
415 pixelMap->GetCapacity() < skInfo.computeMinByteSize()) {
416 uint32_t res = RGBToRGBx(pixelMap, skInfo, holder);
417 bool cond = res != SUCCESS;
418 CHECK_ERROR_RETURN_RET_LOG(cond, res, "ExtEncoder::BuildSkBitmap RGB convert failed %{public}d", res);
419 image.pixels = holder.buf.get();
420 skInfo = skInfo.makeColorType(SkColorType::kRGBA_8888_SkColorType);
421 }
422 return SUCCESS;
423 }
424
IsAstc(const std::string & format)425 bool IsAstc(const std::string &format)
426 {
427 return format.find("image/astc") == 0;
428 }
429
CreateAndWriteBlob(MetadataWStream & tStream,PixelMap * pixelmap,SkWStream & outStream,ImageInfo & imageInfo,PlEncodeOptions & opts)430 static uint32_t CreateAndWriteBlob(MetadataWStream &tStream, PixelMap *pixelmap, SkWStream& outStream,
431 ImageInfo &imageInfo, PlEncodeOptions &opts)
432 {
433 ImageFuncTimer imageFuncTimer("insert exif data (%d, %d)", imageInfo.size.width, imageInfo.size.height);
434 auto metadataAccessor =
435 MetadataAccessorFactory::Create(tStream.GetAddr(), tStream.bytesWritten(), BufferMetadataStream::Dynamic);
436 if (metadataAccessor != nullptr) {
437 auto metadataPtr = pixelmap->GetExifMetadata();
438 metadataAccessor->Set(metadataPtr);
439 if (metadataAccessor->Write() == SUCCESS) {
440 if (metadataAccessor->WriteToOutput(outStream)) {
441 return SUCCESS;
442 }
443 }
444 }
445 if (!outStream.write(tStream.GetAddr(), tStream.bytesWritten())) {
446 ReportEncodeFault(imageInfo.size.width, imageInfo.size.height, opts.format, "Failed to encode image");
447 return ERR_IMAGE_ENCODE_FAILED;
448 }
449 return SUCCESS;
450 }
451
IsWideGamutSdrPixelMap(Media::PixelMap * pixelmap)452 bool IsWideGamutSdrPixelMap(Media::PixelMap *pixelmap)
453 {
454 #ifdef IMAGE_COLORSPACE_FLAG
455 return pixelmap->InnerGetGrColorSpace().GetColorSpaceName() ==
456 OHOS::ColorManager::ColorSpaceName::DISPLAY_BT2020_SRGB ? true : false;
457 #else
458 return false;
459 #endif
460 }
461
PixelmapEncode(ExtWStream & wStream)462 uint32_t ExtEncoder::PixelmapEncode(ExtWStream& wStream)
463 {
464 uint32_t error;
465 #if defined(_WIN32) || defined(_APPLE) || defined(IOS_PLATFORM) || defined(ANDROID_PLATFORM)
466 IMAGE_LOGD("pixelmapEncode EncodeImageByPixelMap");
467 error = EncodeImageByPixelMap(pixelmap_, opts_.needsPackProperties, wStream);
468 #else
469 switch (opts_.desiredDynamicRange) {
470 case EncodeDynamicRange::AUTO:
471 if ((pixelmap_->IsHdr() || IsWideGamutSdrPixelMap(pixelmap_)) &&
472 (encodeFormat_ == SkEncodedImageFormat::kJPEG || encodeFormat_ == SkEncodedImageFormat::kHEIF)) {
473 error = EncodeDualVivid(wStream);
474 } else {
475 error = EncodeSdrImage(wStream);
476 }
477 break;
478 case EncodeDynamicRange::SDR:
479 error = EncodeSdrImage(wStream);
480 break;
481 case EncodeDynamicRange::HDR_VIVID_DUAL:
482 error = EncodeDualVivid(wStream);
483 break;
484 case EncodeDynamicRange::HDR_VIVID_SINGLE:
485 error = EncodeSingleVivid(wStream);
486 break;
487 default:
488 error = ERR_IMAGE_ENCODE_FAILED;
489 break;
490 }
491 #endif
492 RecycleResources();
493 return error;
494 }
495 #if !defined(IOS_PLATFORM) && !defined(ANDROID_PLATFORM)
IsSuperFastEncode(const std::string & format)496 bool IsSuperFastEncode(const std::string &format)
497 {
498 return SUT_FORMAT_MAP.find(format) != SUT_FORMAT_MAP.end() ||
499 ASTC_FORMAT_MAP.find(format) != ASTC_FORMAT_MAP.end();
500 }
501 #endif
502
FinalizeEncode()503 uint32_t ExtEncoder::FinalizeEncode()
504 {
505 bool isParameterInvalid = (picture_ == nullptr && pixelmap_ == nullptr) || output_ == nullptr;
506 CHECK_ERROR_RETURN_RET(isParameterInvalid, ERR_IMAGE_INVALID_PARAMETER);
507 ImageDataStatistics imageDataStatistics("[ExtEncoder]FinalizeEncode imageFormat = %s, quality = %d",
508 opts_.format.c_str(), opts_.quality);
509 #if !defined(IOS_PLATFORM) && !defined(ANDROID_PLATFORM)
510 if (IsAstc(opts_.format) || IsSuperFastEncode(opts_.format)) {
511 AstcCodec astcEncoder;
512 astcEncoder.SetAstcEncode(output_, opts_, pixelmap_);
513 return astcEncoder.ASTCEncode();
514 }
515 #endif
516 auto iter = FORMAT_NAME.find(LowerStr(opts_.format));
517 if (iter == FORMAT_NAME.end()) {
518 IMAGE_LOGE("ExtEncoder::FinalizeEncode unsupported format %{public}s", opts_.format.c_str());
519 ReportEncodeFault(0, 0, opts_.format, "Unsupported format:" + opts_.format);
520 return ERR_IMAGE_INVALID_PARAMETER;
521 }
522 #if !defined(_WIN32) && !defined(_APPLE) && !defined(IOS_PLATFORM) && !defined(ANDROID_PLATFORM)
523 if (picture_ != nullptr) {
524 encodeFormat_ = iter->second;
525 return EncodePicture();
526 }
527 #endif
528 ImageInfo imageInfo;
529 pixelmap_->GetImageInfo(imageInfo);
530 imageDataStatistics.AddTitle(", width = %d, height =%d", imageInfo.size.width, imageInfo.size.height);
531 encodeFormat_ = iter->second;
532 ExtWStream wStream(output_);
533 return PixelmapEncode(wStream);
534 }
535
IsHardwareEncodeSupported(const PlEncodeOptions & opts,Media::PixelMap * pixelMap)536 bool ExtEncoder::IsHardwareEncodeSupported(const PlEncodeOptions &opts, Media::PixelMap* pixelMap)
537 {
538 CHECK_ERROR_RETURN_RET_LOG(pixelMap == nullptr, false, "pixelMap is nullptr");
539 bool cond = opts.quality < LOW_QUALITY_BOUNDARY;
540 CHECK_ERROR_RETURN_RET_LOG(cond, false, "%{public}s Low quality use software encode", __func__);
541 bool isSupportedWithRgba = ImageSystemProperties::GetGenThumbWithGpu() &&
542 pixelMap->GetPixelFormat() == PixelFormat::RGBA_8888;
543 bool isSupport = ImageSystemProperties::GetHardWareEncodeEnabled() && opts.format == "image/jpeg" &&
544 (pixelMap->GetWidth() % 2 == 0) && (pixelMap->GetHeight() % 2 == 0) &&
545 (pixelMap->GetPixelFormat() == PixelFormat::NV12 || pixelMap->GetPixelFormat() == PixelFormat::NV21 ||
546 isSupportedWithRgba) &&
547 pixelMap->GetWidth() <= MAX_IMAGE_SIZE && pixelMap->GetHeight() <= MAX_IMAGE_SIZE &&
548 pixelMap->GetWidth() >= MIN_RGBA_IMAGE_SIZE && pixelMap->GetHeight() >= MIN_RGBA_IMAGE_SIZE;
549 CHECK_DEBUG_PRINT_LOG(!isSupport,
550 "hardware encode is not support, dstEncodeFormat:%{public}s, pixelWidth:%{public}d, pixelHeight:%{public}d, "
551 "pixelFormat:%{public}d",
552 opts.format.c_str(), pixelMap->GetWidth(), pixelMap->GetHeight(), pixelMap->GetPixelFormat());
553 return isSupport;
554 }
555
DoHardWareEncode(SkWStream * skStream)556 uint32_t ExtEncoder::DoHardWareEncode(SkWStream* skStream)
557 {
558 static ImageFwkExtManager imageFwkExtManager;
559 if (imageFwkExtManager.doHardWareEncodeFunc_ != nullptr || imageFwkExtManager.LoadImageFwkExtNativeSo()) {
560 int32_t retCode = imageFwkExtManager.doHardWareEncodeFunc_(skStream, opts_, pixelmap_);
561 CHECK_DEBUG_RETURN_RET_LOG(retCode == SUCCESS, SUCCESS, "DoHardWareEncode Success return");
562 IMAGE_LOGE("hardware encode failed, retCode is %{public}d", retCode);
563 ImageInfo imageInfo;
564 pixelmap_->GetImageInfo(imageInfo);
565 ReportEncodeFault(imageInfo.size.width, imageInfo.size.height, opts_.format, "hardware encode failed");
566 } else {
567 IMAGE_LOGE("hardware encode failed because of load native so failed");
568 }
569 return ERR_IMAGE_ENCODE_FAILED;
570 }
571
572 #ifdef USE_M133_SKIA
SkEncodeImage(SkWStream * dst,const SkBitmap & src,SkEncodedImageFormat format,int quality)573 bool ExtEncoder::SkEncodeImage(SkWStream* dst, const SkBitmap& src, SkEncodedImageFormat format, int quality)
574 {
575 SkPixmap pixmap;
576 if (!src.peekPixels(&pixmap)) {
577 return false;
578 }
579 switch (format) {
580 case SkEncodedImageFormat::kJPEG: {
581 SkJpegEncoder::Options opts;
582 opts.fQuality = quality;
583 return SkJpegEncoder::Encode(dst, pixmap, opts);
584 }
585 case SkEncodedImageFormat::kPNG: {
586 SkPngEncoder::Options opts;
587 return SkPngEncoder::Encode(dst, pixmap, opts);
588 }
589 case SkEncodedImageFormat::kWEBP: {
590 SkWebpEncoder::Options opts;
591 if (quality == HIGHEST_QUALITY) {
592 opts.fCompression = SkWebpEncoder::Compression::kLossless;
593 // Note: SkEncodeImage treats 0 quality as the lowest quality
594 // (greatest compression) and 100 as the highest quality (least
595 // compression). For kLossy, this matches libwebp's
596 // interpretation, so it is passed directly to libwebp. But
597 // with kLossless, libwebp always creates the highest quality
598 // image. In this case, fQuality is reinterpreted as how much
599 // effort (time) to put into making a smaller file. This API
600 // does not provide a way to specify this value (though it can
601 // be specified by using SkWebpEncoder::Encode) so we have to
602 // pick one arbitrarily. This value matches that chosen by
603 // blink::ImageEncoder::ComputeWebpOptions as well
604 // WebPConfigInit.
605 opts.fQuality = DEFAULT_QUALITY;
606 } else {
607 opts.fCompression = SkWebpEncoder::Compression::kLossy;
608 opts.fQuality = quality;
609 }
610 return SkWebpEncoder::Encode(dst, pixmap, opts);
611 }
612 default:
613 return false;
614 }
615 }
616 #endif
617
DoEncode(SkWStream * skStream,const SkBitmap & src,const SkEncodedImageFormat & skFormat)618 uint32_t ExtEncoder::DoEncode(SkWStream* skStream, const SkBitmap& src, const SkEncodedImageFormat& skFormat)
619 {
620 ImageFuncTimer imageFuncTimer("%s:(%d, %d)", __func__, pixelmap_->GetWidth(), pixelmap_->GetHeight());
621 ImageInfo imageInfo;
622 pixelmap_->GetImageInfo(imageInfo);
623 if (!SkEncodeImage(skStream, src, skFormat, opts_.quality)) {
624 IMAGE_LOGE("Failed to encode image without exif data");
625 ReportEncodeFault(imageInfo.size.width, imageInfo.size.height, opts_.format, "Failed to encode image");
626 return ERR_IMAGE_ENCODE_FAILED;
627 }
628 return SUCCESS;
629 }
630
HardwareEncode(SkWStream & skStream,bool needExif)631 bool ExtEncoder::HardwareEncode(SkWStream &skStream, bool needExif)
632 {
633 uint32_t retCode = ERR_IMAGE_ENCODE_FAILED;
634 if (IsHardwareEncodeSupported(opts_, pixelmap_)) {
635 if (!needExif || pixelmap_->GetExifMetadata() == nullptr ||
636 pixelmap_->GetExifMetadata()->GetExifData() == nullptr) {
637 retCode = DoHardWareEncode(&skStream);
638 IMAGE_LOGD("HardwareEncode retCode:%{public}d", retCode);
639 return (retCode == SUCCESS);
640 }
641 MetadataWStream tStream;
642 retCode = DoHardWareEncode(&tStream);
643 bool cond = retCode != SUCCESS;
644 CHECK_DEBUG_RETURN_RET_LOG(cond, false, "HardwareEncode failed, retCode:%{public}d", retCode);
645 ImageInfo imageInfo;
646 pixelmap_->GetImageInfo(imageInfo);
647 retCode = CreateAndWriteBlob(tStream, pixelmap_, skStream, imageInfo, opts_);
648 IMAGE_LOGD("HardwareEncode retCode :%{public}d", retCode);
649 return (retCode == SUCCESS);
650 }
651 return (retCode == SUCCESS);
652 }
653
EncodeImageByBitmap(SkBitmap & bitmap,bool needExif,SkWStream & outStream)654 uint32_t ExtEncoder::EncodeImageByBitmap(SkBitmap& bitmap, bool needExif, SkWStream& outStream)
655 {
656 ImageInfo imageInfo;
657 pixelmap_->GetImageInfo(imageInfo);
658 if (!needExif || pixelmap_->GetExifMetadata() == nullptr ||
659 pixelmap_->GetExifMetadata()->GetExifData() == nullptr) {
660 return DoEncode(&outStream, bitmap, encodeFormat_);
661 }
662
663 MetadataWStream tStream;
664 uint32_t errCode = DoEncode(&tStream, bitmap, encodeFormat_);
665 bool cond = errCode != SUCCESS;
666 CHECK_ERROR_RETURN_RET(cond, errCode);
667 return CreateAndWriteBlob(tStream, pixelmap_, outStream, imageInfo, opts_);
668 }
669
EncodeImageByPixelMap(PixelMap * pixelMap,bool needExif,SkWStream & outputStream)670 uint32_t ExtEncoder::EncodeImageByPixelMap(PixelMap* pixelMap, bool needExif, SkWStream& outputStream)
671 {
672 if (encodeFormat_ == SkEncodedImageFormat::kHEIF) {
673 return EncodeHeifByPixelmap(pixelMap, opts_);
674 }
675 SkBitmap bitmap;
676 TmpBufferHolder holder;
677 SkImageInfo skInfo;
678 ImageData imageData;
679 pixelMap->GetImageInfo(imageData.info);
680 uint32_t width = static_cast<uint32_t>(imageData.info.size.width);
681 uint32_t height = static_cast<uint32_t>(imageData.info.size.height);
682
683 bool cond = HardwareEncode(outputStream, needExif) == true;
684 CHECK_DEBUG_RETURN_RET_LOG(cond, SUCCESS, "HardwareEncode Success return");
685 IMAGE_LOGD("HardwareEncode failed or not Supported");
686
687 std::unique_ptr<uint8_t[]> dstData;
688 uint64_t rowStride = 0;
689 if (IsYuvImage(imageData.info.pixelFormat)) {
690 IMAGE_LOGD("YUV format, convert to RGB");
691 dstData = std::make_unique<uint8_t[]>(width * height * NUM_4);
692 if (YuvToRgbaSkInfo(imageData.info, skInfo, dstData.get(), pixelMap) != SUCCESS) {
693 IMAGE_LOGD("YUV format, convert to RGB fail");
694 return ERR_IMAGE_ENCODE_FAILED;
695 }
696 imageData.pixels = dstData.get();
697 rowStride = skInfo.minRowBytes64();
698 } else {
699 dstData = std::make_unique<uint8_t[]>(width * height * NUM_3);
700 imageData.dst = dstData.get();
701 cond = pixelToSkInfo(imageData, skInfo, pixelMap, holder, encodeFormat_) != SUCCESS;
702 CHECK_ERROR_RETURN_RET_LOG(cond, ERR_IMAGE_ENCODE_FAILED,
703 "ExtEncoder::EncodeImageByPixelMap pixel convert failed");
704 rowStride = skInfo.minRowBytes64();
705 #if !defined(IOS_PLATFORM) && !defined(ANDROID_PLATFORM)
706 if (pixelMap->GetAllocatorType() == AllocatorType::DMA_ALLOC) {
707 SurfaceBuffer* sbBuffer = reinterpret_cast<SurfaceBuffer*> (pixelMap->GetFd());
708 rowStride = static_cast<uint64_t>(sbBuffer->GetStride());
709 IMAGE_LOGD("rowStride DMA: %{public}llu", static_cast<unsigned long long>(rowStride));
710 }
711 #endif
712 }
713 if (!bitmap.installPixels(skInfo, imageData.pixels, rowStride)) {
714 IMAGE_LOGE("ExtEncoder::EncodeImageByPixelMap to SkBitmap failed");
715 return ERR_IMAGE_ENCODE_FAILED;
716 }
717 return EncodeImageByBitmap(bitmap, needExif, outputStream);
718 }
719
EncodeHeifByPixelmap(PixelMap * pixelmap,const PlEncodeOptions & opts)720 uint32_t ExtEncoder::EncodeHeifByPixelmap(PixelMap* pixelmap, const PlEncodeOptions& opts)
721 {
722 CHECK_ERROR_RETURN_RET(output_ == nullptr, ERR_IMAGE_INVALID_PARAMETER);
723 Media::PixelFormat format = pixelmap->GetPixelFormat();
724 bool cond = format != PixelFormat::RGBA_8888 && format != PixelFormat::NV12 && format != PixelFormat::NV21;
725 CHECK_ERROR_RETURN_RET_LOG(cond, ERR_IMAGE_INVALID_PARAMETER,
726 "EncodeHeifByPixelmap, invalid format:%{public}d", format);
727 #if !defined(_WIN32) && !defined(_APPLE) && !defined(IOS_PLATFORM) && !defined(ANDROID_PLATFORM) && \
728 defined(HEIF_HW_ENCODE_ENABLE)
729 sptr<SurfaceBuffer> surfaceBuffer;
730 bool needConvertToSurfaceBuffer = pixelmap->GetAllocatorType() != AllocatorType::DMA_ALLOC;
731 if (needConvertToSurfaceBuffer) {
732 surfaceBuffer = ConvertToSurfaceBuffer(pixelmap);
733 cond = surfaceBuffer == nullptr;
734 CHECK_ERROR_RETURN_RET_LOG(cond, ERR_IMAGE_INVALID_PARAMETER,
735 "EncodeHeifByPixelmap ConvertToSurfaceBuffer failed");
736 } else {
737 cond = pixelmap->GetFd() == nullptr;
738 CHECK_ERROR_RETURN_RET_LOG(cond, ERR_IMAGE_INVALID_PARAMETER, "EncodeHeifByPixelmap pixelmap get fd failed");
739 surfaceBuffer = sptr<SurfaceBuffer>(reinterpret_cast<SurfaceBuffer*>(pixelmap->GetFd()));
740 ImageUtils::FlushSurfaceBuffer(surfaceBuffer);
741 }
742 std::vector<ImageItem> inputImgs;
743 std::shared_ptr<ImageItem> primaryItem = AssemblePrimaryImageItem(surfaceBuffer, opts);
744 if (primaryItem == nullptr) {
745 if (needConvertToSurfaceBuffer) {
746 ImageUtils::SurfaceBuffer_Unreference(surfaceBuffer.GetRefPtr());
747 }
748 return ERR_IMAGE_INVALID_PARAMETER;
749 }
750 inputImgs.push_back(*primaryItem);
751 std::vector<MetaItem> inputMetas;
752 std::vector<ItemRef> refs;
753 if (AssembleExifMetaItem(inputMetas)) {
754 AssembleExifRefItem(refs);
755 }
756 uint32_t result = DoHeifEncode(inputImgs, inputMetas, refs);
757 if (needConvertToSurfaceBuffer) {
758 ImageUtils::SurfaceBuffer_Unreference(surfaceBuffer.GetRefPtr());
759 }
760 return result;
761 #endif
762 return ERR_IMAGE_INVALID_PARAMETER;
763 }
764
RecycleResources()765 void ExtEncoder::RecycleResources()
766 {
767 for (std::shared_ptr<AbsMemory> &memory: tmpMemoryList_) {
768 if (memory == nullptr) {
769 continue;
770 }
771 memory->Release();
772 }
773 tmpMemoryList_.clear();
774 }
775
776 #if !defined(_WIN32) && !defined(_APPLE) && !defined(IOS_PLATFORM) && !defined(ANDROID_PLATFORM)
AllocSurfaceBuffer(int32_t width,int32_t height,uint32_t format=GRAPHIC_PIXEL_FMT_RGBA_8888)777 static sptr<SurfaceBuffer> AllocSurfaceBuffer(int32_t width, int32_t height,
778 uint32_t format = GRAPHIC_PIXEL_FMT_RGBA_8888)
779 {
780 sptr<SurfaceBuffer> sb = SurfaceBuffer::Create();
781 BufferRequestConfig requestConfig = {
782 .width = width,
783 .height = height,
784 .strideAlignment = 0x8,
785 .format = format,
786 .usage = BUFFER_USAGE_CPU_READ | BUFFER_USAGE_CPU_WRITE | BUFFER_USAGE_MEM_DMA | BUFFER_USAGE_MEM_MMZ_CACHE,
787 .timeout = 0,
788 };
789 GSError ret = sb->Alloc(requestConfig);
790 if (ret != GSERROR_OK) {
791 return nullptr;
792 }
793 void* nativeBuffer = sb.GetRefPtr();
794 int32_t err = ImageUtils::SurfaceBuffer_Reference(nativeBuffer);
795 bool cond = err != OHOS::GSERROR_OK;
796 CHECK_ERROR_RETURN_RET(cond, nullptr);
797 return sb;
798 }
799
ConvertToSurfaceBuffer(PixelMap * pixelmap)800 sptr<SurfaceBuffer> ExtEncoder::ConvertToSurfaceBuffer(PixelMap* pixelmap)
801 {
802 bool cond = pixelmap->GetHeight() <= 0 || pixelmap->GetWidth() <= 0;
803 CHECK_ERROR_RETURN_RET_LOG(cond, nullptr,
804 "pixelmap height:%{public}d or width:%{public}d error", pixelmap->GetHeight(), pixelmap->GetWidth());
805 uint32_t height = static_cast<uint32_t>(pixelmap->GetHeight());
806 uint32_t width = static_cast<uint32_t>(pixelmap->GetWidth());
807 PixelFormat format = pixelmap->GetPixelFormat();
808 IMAGE_LOGD("ExtEncoder::ConvertToSurfaceBuffer format is %{public}d", format);
809 auto formatSearch = SURFACE_FORMAT_MAP.find(format);
810 cond = formatSearch == SURFACE_FORMAT_MAP.end();
811 CHECK_ERROR_RETURN_RET_LOG(cond, nullptr, "format:[%{public}d] is not in SURFACE_FORMAT_MAP", format);
812 uint32_t graphicFormat = formatSearch->second;
813 sptr<SurfaceBuffer> surfaceBuffer = AllocSurfaceBuffer(width, height, graphicFormat);
814 cond = surfaceBuffer == nullptr || surfaceBuffer->GetStride() < 0;
815 CHECK_ERROR_RETURN_RET_LOG(cond, nullptr, "ConvertToSurfaceBuffer surfaceBuffer is nullptr failed");
816 uint32_t dstStride = static_cast<uint32_t>(surfaceBuffer->GetStride());
817 uint8_t* src = const_cast<uint8_t*>(pixelmap->GetPixels());
818 uint8_t* dst = static_cast<uint8_t*>(surfaceBuffer->GetVirAddr());
819 uint32_t dstSize = surfaceBuffer->GetSize();
820 uint32_t copyHeight = height;
821 uint64_t srcStride = width;
822 if (format == PixelFormat::NV12 || format == PixelFormat::NV21) {
823 const int32_t NUM_2 = 2;
824 copyHeight = height + height / NUM_2;
825 srcStride = width;
826 } else if (format == PixelFormat::RGBA_8888) {
827 copyHeight = height;
828 srcStride = static_cast<uint64_t>(width * NUM_4);
829 }
830 for (uint32_t i = 0; i < copyHeight; i++) {
831 if (memcpy_s(dst, dstSize, src, srcStride) != EOK) {
832 IMAGE_LOGE("ConvertToSurfaceBuffer memcpy failed");
833 ImageUtils::SurfaceBuffer_Unreference(surfaceBuffer.GetRefPtr());
834 return nullptr;
835 }
836 dst += dstStride;
837 dstSize -= dstStride;
838 src += srcStride;
839 }
840 ImageUtils::FlushSurfaceBuffer(surfaceBuffer);
841 return surfaceBuffer;
842 }
843
ConvertPixelMapToDmaBuffer(std::shared_ptr<PixelMap> pixelmap)844 sptr<SurfaceBuffer> ExtEncoder::ConvertPixelMapToDmaBuffer(std::shared_ptr<PixelMap> pixelmap)
845 {
846 sptr<SurfaceBuffer> surfaceBuffer;
847 if (pixelmap->GetAllocatorType() != AllocatorType::DMA_ALLOC) {
848 surfaceBuffer = ConvertToSurfaceBuffer(pixelmap.get());
849 } else {
850 surfaceBuffer = sptr<SurfaceBuffer>(reinterpret_cast<SurfaceBuffer*>(pixelmap->GetFd()));
851 ImageUtils::FlushSurfaceBuffer(surfaceBuffer);
852 }
853 return surfaceBuffer;
854 }
855
FreeBaseAndGainMapSurfaceBuffer(sptr<SurfaceBuffer> & base,sptr<SurfaceBuffer> & gainMap)856 static void FreeBaseAndGainMapSurfaceBuffer(sptr<SurfaceBuffer>& base, sptr<SurfaceBuffer>& gainMap)
857 {
858 ImageUtils::SurfaceBuffer_Unreference(base.GetRefPtr());
859 ImageUtils::SurfaceBuffer_Unreference(gainMap.GetRefPtr());
860 }
861
GetHdrMetadata(sptr<SurfaceBuffer> & hdr,sptr<SurfaceBuffer> & gainmap)862 static HdrMetadata GetHdrMetadata(sptr<SurfaceBuffer>& hdr, sptr<SurfaceBuffer>& gainmap)
863 {
864 std::vector<uint8_t> dynamicMetadata = {};
865 VpeUtils::GetSbDynamicMetadata(hdr, dynamicMetadata);
866 std::vector<uint8_t> staticMetadata = {};
867 VpeUtils::GetSbStaticMetadata(hdr, staticMetadata);
868 HdrMetadata metadata = {
869 .staticMetadata = staticMetadata,
870 .dynamicMetadata = dynamicMetadata
871 };
872 std::vector<uint8_t> extendMetadataVec = {};
873 VpeUtils::GetSbDynamicMetadata(gainmap, extendMetadataVec);
874 if (extendMetadataVec.size() != sizeof(HDRVividExtendMetadata)) {
875 metadata.extendMetaFlag = false;
876 return metadata;
877 }
878 if (memcpy_s(&metadata.extendMeta, sizeof(HDRVividExtendMetadata),
879 extendMetadataVec.data(), extendMetadataVec.size()) != EOK) {
880 metadata.extendMetaFlag = false;
881 } else {
882 metadata.extendMetaFlag = true;
883 }
884 return metadata;
885 }
886
EncodeImageBySurfaceBuffer(sptr<SurfaceBuffer> & surfaceBuffer,SkImageInfo info,bool needExif,SkWStream & outputStream)887 uint32_t ExtEncoder::EncodeImageBySurfaceBuffer(sptr<SurfaceBuffer>& surfaceBuffer, SkImageInfo info,
888 bool needExif, SkWStream& outputStream)
889 {
890 SkBitmap bitmap;
891 ImageInfo imageInfo;
892 uint64_t rowStride = 0;
893 CHECK_ERROR_RETURN_RET_LOG(surfaceBuffer == nullptr, ERR_IMAGE_INVALID_PARAMETER,
894 "EncodeImageBySurfaceBuffer failed, surfaceBuffer is nullptr");
895 auto pixels = surfaceBuffer->GetVirAddr();
896 CHECK_ERROR_RETURN_RET_LOG(pixels == nullptr, ERR_IMAGE_INVALID_PARAMETER,
897 "EncodeImageBySurfaceBuffer failed, pixels is nullptr");
898 /* do hardwareEncode first, if fail then soft encode*/
899 bool cond = HardwareEncode(outputStream, needExif);
900 CHECK_DEBUG_RETURN_RET_LOG(cond, SUCCESS, "HardwareEncode Success return");
901 IMAGE_LOGD("HardwareEncode failed or not Supported");
902
903 pixelmap_->GetImageInfo(imageInfo);
904 cond = !PixelYuvUtils::CheckWidthAndHeightMult(imageInfo.size.width, imageInfo.size.height, RGBA_BIT_DEPTH);
905 CHECK_ERROR_RETURN_RET_LOG(cond, ERR_IMAGE_INVALID_PARAMETER,
906 "EncodeImageBySurfaceBuffer size overflow width(%{public}d), height(%{public}d)",
907 imageInfo.size.width, imageInfo.size.height);
908 std::unique_ptr<uint8_t[]> dstData;
909 if (IsYuvImage(imageInfo.pixelFormat)) {
910 IMAGE_LOGD("EncodeImageBySurfaceBuffer: YUV format, convert to RGB first");
911 dstData = std::make_unique<uint8_t[]>(imageInfo.size.width * imageInfo.size.height * NUM_4);
912 cond = dstData == nullptr;
913 CHECK_ERROR_RETURN_RET_LOG(cond, ERR_IMAGE_ENCODE_FAILED, "Memory allocate fail");
914 cond = YuvToRgbaSkInfo(imageInfo, info, dstData.get(), pixelmap_) != SUCCESS;
915 CHECK_DEBUG_RETURN_RET_LOG(cond, ERR_IMAGE_ENCODE_FAILED, "YUV format, convert to RGB fail");
916 pixels = dstData.get();
917 rowStride = info.minRowBytes64();
918 } else {
919 rowStride = static_cast<uint64_t>(surfaceBuffer->GetStride());
920 }
921 cond = !bitmap.installPixels(info, pixels, rowStride);
922 CHECK_ERROR_RETURN_RET_LOG(cond, ERR_IMAGE_ENCODE_FAILED,
923 "ExtEncoder::EncodeImageBySurfaceBuffer to SkBitmap failed");
924 return EncodeImageByBitmap(bitmap, needExif, outputStream);
925 }
926
GetImageEncodeData(sptr<SurfaceBuffer> & surfaceBuffer,SkImageInfo info,bool needExif)927 sk_sp<SkData> ExtEncoder::GetImageEncodeData(sptr<SurfaceBuffer>& surfaceBuffer, SkImageInfo info, bool needExif)
928 {
929 SkDynamicMemoryWStream stream;
930 if (EncodeImageBySurfaceBuffer(surfaceBuffer, info, needExif, stream) != SUCCESS) {
931 return nullptr;
932 }
933 return stream.detachAsData();
934 }
935
SetHdrColorSpaceType(sptr<SurfaceBuffer> & surfaceBuffer)936 void ExtEncoder::SetHdrColorSpaceType(sptr<SurfaceBuffer>& surfaceBuffer)
937 {
938 if (pixelmap_ == nullptr) {
939 IMAGE_LOGI("SetHdrColorSpaceType pixelmap_ is nullptr");
940 return;
941 }
942 CM_ColorSpaceType colorspaceType;
943 VpeUtils::GetSbColorSpaceType(surfaceBuffer, colorspaceType);
944 if ((colorspaceType & CM_PRIMARIES_MASK) != COLORPRIMARIES_BT2020) {
945 #ifdef IMAGE_COLORSPACE_FLAG
946 ColorManager::ColorSpaceName colorspace = pixelmap_->InnerGetGrColorSpace().GetColorSpaceName();
947 IMAGE_LOGI("ExtEncoder SetHdrColorSpaceType, color is %{public}d", colorspace);
948 colorspaceType = ColorUtils::ConvertToCMColor(colorspace);
949 VpeUtils::SetSbColorSpaceType(surfaceBuffer, colorspaceType);
950 #endif
951 }
952 }
953
DecomposeImage(VpeSurfaceBuffers & buffers,HdrMetadata & metadata,bool onlySdr,bool sdrIsSRGB=false)954 static bool DecomposeImage(VpeSurfaceBuffers& buffers, HdrMetadata& metadata, bool onlySdr, bool sdrIsSRGB = false)
955 {
956 VpeUtils::SetSbMetadataType(buffers.sdr, CM_IMAGE_HDR_VIVID_DUAL);
957 VpeUtils::SetSbColorSpaceType(buffers.sdr, sdrIsSRGB ? CM_SRGB_FULL : CM_P3_FULL);
958 std::unique_ptr<VpeUtils> utils = std::make_unique<VpeUtils>();
959 int32_t res;
960 if (onlySdr) {
961 bool cond = buffers.hdr == nullptr || buffers.sdr == nullptr;
962 CHECK_ERROR_RETURN_RET(cond, false);
963 res = utils->ColorSpaceConverterImageProcess(buffers.hdr, buffers.sdr);
964 } else {
965 VpeUtils::SetSbMetadataType(buffers.gainmap, CM_METADATA_NONE);
966 VpeUtils::SetSbColorSpaceType(buffers.gainmap, sdrIsSRGB ? CM_SRGB_FULL : CM_P3_FULL);
967 res = utils->ColorSpaceConverterDecomposeImage(buffers);
968 }
969 bool cond = res != VPE_ERROR_OK;
970 CHECK_ERROR_RETURN_RET_LOG(cond, false, "DecomposeImage [%{public}d] failed, res = %{public}d", onlySdr, res);
971 if (!onlySdr) {
972 metadata = GetHdrMetadata(buffers.hdr, buffers.gainmap);
973 }
974 return true;
975 }
976
GetSkInfo(PixelMap * pixelMap,bool isGainmap,bool isSRGB=false)977 static SkImageInfo GetSkInfo(PixelMap* pixelMap, bool isGainmap, bool isSRGB = false)
978 {
979 ImageInfo info;
980 pixelMap->GetImageInfo(info);
981 SkColorType colorType = kRGBA_8888_SkColorType;
982 SkAlphaType alphaType = ImageTypeConverter::ToSkAlphaType(info.alphaType);
983 if (alphaType == SkAlphaType::kUnknown_SkAlphaType) {
984 alphaType = SkAlphaType::kOpaque_SkAlphaType;
985 }
986 sk_sp<SkColorSpace> colorSpace =
987 SkColorSpace::MakeRGB(SkNamedTransferFn::kSRGB, isSRGB ? SkNamedGamut::kSRGB : SkNamedGamut::kDisplayP3);
988 int32_t width = info.size.width;
989 int32_t height = info.size.height;
990 if (isGainmap) {
991 const int halfSizeDenominator = 2;
992 width = width / halfSizeDenominator;
993 height = height / halfSizeDenominator;
994 #ifdef IMAGE_COLORSPACE_FLAG
995 if (pixelMap->InnerGetGrColorSpacePtr() != nullptr &&
996 pixelMap->InnerGetGrColorSpace().GetColorSpaceName() != ColorManager::ColorSpaceName::NONE) {
997 colorSpace = pixelMap->InnerGetGrColorSpacePtr()->ToSkColorSpace();
998 }
999 skcms_CICP cicp;
1000 ColorUtils::ColorSpaceGetCicp(pixelMap->InnerGetGrColorSpace().GetColorSpaceName(),
1001 #ifdef USE_M133_SKIA
1002 cicp.color_primaries, cicp.transfer_characteristics, cicp.matrix_coefficients, cicp.video_full_range_flag);
1003 #else
1004 cicp.colour_primaries, cicp.transfer_characteristics, cicp.matrix_coefficients, cicp.full_range_flag);
1005 #endif
1006 colorSpace->SetIccCicp(cicp);
1007 #endif
1008 }
1009 return SkImageInfo::Make(width, height, colorType, alphaType, colorSpace);
1010 }
1011
GetSkInfo(PixelMap * pixelMap,bool isGainmap,sk_sp<SkColorSpace> & colorSpace)1012 static SkImageInfo GetSkInfo(PixelMap* pixelMap, bool isGainmap, sk_sp<SkColorSpace>& colorSpace)
1013 {
1014 ImageInfo info;
1015 pixelMap->GetImageInfo(info);
1016 SkColorType colorType = kRGBA_8888_SkColorType;
1017 SkAlphaType alphaType = ImageTypeConverter::ToSkAlphaType(info.alphaType);
1018 if (alphaType == SkAlphaType::kUnknown_SkAlphaType) {
1019 alphaType = SkAlphaType::kOpaque_SkAlphaType;
1020 }
1021 int32_t width = info.size.width;
1022 int32_t height = info.size.height;
1023 if (isGainmap) {
1024 const int halfSizeDenominator = 2;
1025 width = width / halfSizeDenominator;
1026 height = height / halfSizeDenominator;
1027 }
1028 return SkImageInfo::Make(width, height, colorType, alphaType, colorSpace);
1029 }
1030
EncodeSingleVivid(ExtWStream & outputStream)1031 uint32_t ExtEncoder::EncodeSingleVivid(ExtWStream& outputStream)
1032 {
1033 return ERR_IMAGE_INVALID_PARAMETER;
1034 }
1035
1036 #ifdef HEIF_HW_ENCODE_ENABLE
1037 // format: enum + struct
FillLitePropertyItem(std::vector<uint8_t> & properties,size_t & offset,PropertyType type,const void * payloadData,size_t payloadSize)1038 static bool FillLitePropertyItem(std::vector<uint8_t>& properties, size_t& offset, PropertyType type,
1039 const void* payloadData, size_t payloadSize)
1040 {
1041 uint8_t* memData = properties.data();
1042 bool cond = memData == nullptr;
1043 CHECK_ERROR_RETURN_RET_LOG(cond, false, "FillLitePropertyItem memData is nullptr");
1044 size_t memSize = properties.size();
1045 size_t typeSize = sizeof(type);
1046 size_t endOffset = offset + typeSize + payloadSize;
1047 cond = endOffset > memSize;
1048 CHECK_ERROR_RETURN_RET_LOG(cond, false,
1049 "FillLitePropertyItem, endOffset[%{public}ld] over memSize[%{public}ld]", endOffset, memSize);
1050 bool res = memcpy_s(memData + offset, memSize - offset, &type, typeSize) == EOK &&
1051 memcpy_s(memData + offset + typeSize, memSize - offset - typeSize, payloadData, payloadSize) == EOK;
1052 if (res) {
1053 offset = endOffset;
1054 }
1055 return res;
1056 }
1057
1058 // format: enum + strlen + str
FillLitePropertyItemByString(std::vector<uint8_t> & properties,size_t & offset,PropertyType type,std::string & payloadString)1059 static bool FillLitePropertyItemByString(std::vector<uint8_t>& properties, size_t& offset, PropertyType type,
1060 std::string& payloadString)
1061 {
1062 uint8_t* memData = properties.data();
1063 bool cond = memData == nullptr;
1064 CHECK_ERROR_RETURN_RET_LOG(cond, false, "%{public}s memData is nullptr", __func__);
1065 size_t memSize = properties.size();
1066 cond = payloadString == "";
1067 CHECK_ERROR_RETURN_RET_LOG(cond, false, "%{public}s failed, payloadString is Null", __func__);
1068 size_t lenSize = UINT32_BYTES_NUM;
1069 size_t strLen = payloadString.length();
1070 size_t typeSize = sizeof(type);
1071 size_t endOffset = offset + typeSize + lenSize + strLen;
1072 cond = endOffset >= memSize;
1073 CHECK_ERROR_RETURN_RET_LOG(cond, false,
1074 "%{public}s failed, offset[%{public}ld] over memSize[%{public}ld]", __func__, offset, memSize);
1075 size_t capacitySize = memSize - offset;
1076 const char* payloadData = payloadString.data();
1077 bool res = memcpy_s(memData + offset, capacitySize, &type, typeSize) == EOK &&
1078 memcpy_s(memData + offset + typeSize, capacitySize - typeSize, &strLen, lenSize) == EOK &&
1079 memcpy_s(memData + offset + typeSize + lenSize, capacitySize - typeSize - lenSize, payloadData, strLen) == EOK;
1080 properties[endOffset] = '\0';
1081
1082 if (res) {
1083 offset = endOffset + PLACE_HOLDER_LENGTH;
1084 }
1085 return res;
1086 }
1087
AssembleHdrBaseImageItem(sptr<SurfaceBuffer> & surfaceBuffer,ColorManager::ColorSpaceName color,HdrMetadata & metadata,const PlEncodeOptions & opts)1088 std::shared_ptr<ImageItem> ExtEncoder::AssembleHdrBaseImageItem(sptr<SurfaceBuffer>& surfaceBuffer,
1089 ColorManager::ColorSpaceName color, HdrMetadata& metadata, const PlEncodeOptions& opts)
1090 {
1091 bool cond = surfaceBuffer == nullptr;
1092 CHECK_INFO_RETURN_RET_LOG(cond, nullptr, "AssembleHdrBaseImageItem surfaceBuffer is nullptr");
1093 auto item = std::make_shared<ImageItem>();
1094 item->id = PRIMARY_IMAGE_ITEM_ID;
1095 item->isPrimary = true;
1096 item->isHidden = false;
1097 item->compressType = COMPRESS_TYPE_HEVC;
1098 item->quality = opts.quality;
1099 item->pixelBuffer = sptr<NativeBuffer>::MakeSptr(surfaceBuffer->GetBufferHandle());
1100 item->sharedProperties.fd = -1;
1101 item->pixelSharedBuffer.fd = -1;
1102 ColourInfo colorInfo;
1103 GetColourInfo(color, colorInfo);
1104 ContentLightLevel light;
1105 MasteringDisplayColourVolume colour;
1106 bool hasLight = GetStaticMetadata(metadata, colour, light);
1107 uint32_t propertiesSize = 0;
1108 propertiesSize +=
1109 (sizeof(PropertyType::COLOR_TYPE) + sizeof(ColorType) + sizeof(PropertyType::COLOR_INFO) + sizeof(ColourInfo));
1110 if (hasLight) {
1111 propertiesSize += (sizeof(PropertyType::CONTENT_LIGHT_LEVEL) + sizeof(ContentLightLevel));
1112 }
1113 item->liteProperties.resize(propertiesSize);
1114 size_t offset = 0;
1115 if (!FillNclxColorProperty(item, offset, colorInfo)) {
1116 return nullptr;
1117 }
1118 if (hasLight && (!FillLitePropertyItem(item->liteProperties, offset,
1119 PropertyType::CONTENT_LIGHT_LEVEL, &colour, sizeof(ContentLightLevel)))) {
1120 return nullptr;
1121 }
1122 return item;
1123 }
1124
AssembleGainmapImageItem(sptr<SurfaceBuffer> & surfaceBuffer,ColorManager::ColorSpaceName color,const PlEncodeOptions & opts)1125 std::shared_ptr<ImageItem> ExtEncoder::AssembleGainmapImageItem(sptr<SurfaceBuffer>& surfaceBuffer,
1126 ColorManager::ColorSpaceName color, const PlEncodeOptions &opts)
1127 {
1128 bool cond = surfaceBuffer == nullptr;
1129 CHECK_INFO_RETURN_RET_LOG(cond, nullptr, "AssembleGainmapImageItem surfacebuffer is nullptr");
1130 auto item = std::make_shared<ImageItem>();
1131 item->id = GAINMAP_IMAGE_ITEM_ID;
1132 item->itemName = GAINMAP_IMAGE_ITEM_NAME;
1133 item->isPrimary = false;
1134 item->isHidden = true;
1135 item->compressType = COMPRESS_TYPE_HEVC;
1136 item->quality = opts.quality;
1137 item->sharedProperties.fd = -1;
1138 item->pixelSharedBuffer.fd = -1;
1139 item->pixelBuffer = sptr<NativeBuffer>::MakeSptr(surfaceBuffer->GetBufferHandle());
1140 ColourInfo colorInfo;
1141 GetColourInfo(color, colorInfo);
1142 uint32_t propertiesSize = 0;
1143 propertiesSize +=
1144 (sizeof(PropertyType::COLOR_TYPE) + sizeof(ColorType) + sizeof(PropertyType::COLOR_INFO) + sizeof(ColourInfo));
1145 item->liteProperties.resize(propertiesSize);
1146 size_t offset = 0;
1147 cond = !FillNclxColorProperty(item, offset, colorInfo);
1148 CHECK_ERROR_RETURN_RET(cond, nullptr);
1149 return item;
1150 }
1151
AssembleHeifHdrPicture(sptr<SurfaceBuffer> & mainSptr,bool sdrIsSRGB,std::vector<ImageItem> & inputImgs)1152 uint32_t ExtEncoder::AssembleHeifHdrPicture(
1153 sptr<SurfaceBuffer>& mainSptr, bool sdrIsSRGB, std::vector<ImageItem>& inputImgs)
1154 {
1155 auto gainPixelMap = picture_->GetGainmapPixelMap();
1156 bool cond = gainPixelMap == nullptr || gainPixelMap->GetAllocatorType() != AllocatorType::DMA_ALLOC;
1157 CHECK_ERROR_RETURN_RET_LOG(cond, ERR_IMAGE_INVALID_PARAMETER,
1158 "%{public}s, the gainPixelMap is nullptr or gainPixelMap is nonDMA", __func__);
1159 sptr<SurfaceBuffer> gainMapSptr(reinterpret_cast<SurfaceBuffer*>(gainPixelMap->GetFd()));
1160 ImageUtils::FlushSurfaceBuffer(gainMapSptr);
1161 HdrMetadata metadata = GetHdrMetadata(mainSptr, gainMapSptr);
1162
1163 ColorManager::ColorSpaceName colorspaceName =
1164 sdrIsSRGB ? ColorManager::ColorSpaceName::SRGB : ColorManager::ColorSpaceName::DISPLAY_P3;
1165 std::shared_ptr<ImageItem> primaryItem = AssembleHdrBaseImageItem(mainSptr, colorspaceName, metadata, opts_);
1166 cond = primaryItem == nullptr;
1167 CHECK_ERROR_RETURN_RET_LOG(cond, ERR_IMAGE_INVALID_PARAMETER, "%{public}s, get primary image failed", __func__);
1168 inputImgs.push_back(*primaryItem);
1169 std::shared_ptr<ImageItem> gainmapItem = AssembleGainmapImageItem(gainMapSptr, colorspaceName, opts_);
1170 cond = gainmapItem == nullptr;
1171 CHECK_ERROR_RETURN_RET_LOG(cond, ERR_IMAGE_INVALID_PARAMETER,
1172 "%{public}s, get gainmap image item failed", __func__);
1173 inputImgs.push_back(*gainmapItem);
1174 ColorManager::ColorSpaceName tmapColor = picture_->GetMainPixel()->InnerGetGrColorSpace().GetColorSpaceName();
1175 std::shared_ptr<ImageItem> tmapItem = AssembleTmapImageItem(tmapColor, metadata, opts_);
1176 cond = tmapItem == nullptr;
1177 CHECK_ERROR_RETURN_RET_LOG(cond, ERR_IMAGE_INVALID_PARAMETER, "%{public}s, get tmap image item failed", __func__);
1178 inputImgs.push_back(*tmapItem);
1179 return SUCCESS;
1180 }
1181
AssembleSdrImageItem(sptr<SurfaceBuffer> & surfaceBuffer,SkImageInfo sdrInfo,std::vector<ImageItem> & inputImgs)1182 uint32_t ExtEncoder::AssembleSdrImageItem(
1183 sptr<SurfaceBuffer>& surfaceBuffer, SkImageInfo sdrInfo, std::vector<ImageItem>& inputImgs)
1184 {
1185 bool cond = surfaceBuffer == nullptr;
1186 CHECK_INFO_RETURN_RET_LOG(cond, ERR_IMAGE_INVALID_PARAMETER, "%{public}s surfaceBuffer is nullptr", __func__);
1187 ImageItem item;
1188 #ifdef USE_M133_SKIA
1189 sk_sp<SkData> iccProfile = icc_from_color_space(sdrInfo, nullptr, nullptr);
1190 #else
1191 sk_sp<SkData> iccProfile = icc_from_color_space(sdrInfo);
1192 #endif
1193 cond = !AssembleICCImageProperty(iccProfile, item.sharedProperties);
1194 CHECK_ERROR_RETURN_RET_LOG(cond, ERR_IMAGE_INVALID_PARAMETER,
1195 "%{public}s AssembleICCImageProperty failed", __func__);
1196 item.id = PRIMARY_IMAGE_ITEM_ID;
1197 item.pixelBuffer = sptr<NativeBuffer>::MakeSptr(surfaceBuffer->GetBufferHandle());
1198 item.isPrimary = true;
1199 item.isHidden = false;
1200 item.compressType = COMPRESS_TYPE_HEVC;
1201 item.quality = opts_.quality;
1202 uint32_t litePropertiesSize = (sizeof(PropertyType::COLOR_TYPE) + sizeof(ColorType));
1203 item.liteProperties.resize(litePropertiesSize);
1204 size_t offset = 0;
1205 ColorType colorType = ColorType::RICC;
1206 cond = !FillLitePropertyItem(item.liteProperties, offset, PropertyType::COLOR_TYPE, &colorType, sizeof(ColorType));
1207 CHECK_INFO_RETURN_RET_LOG(cond, ERR_IMAGE_INVALID_PARAMETER, "%{public}s Fill color type failed", __func__);
1208 inputImgs.push_back(item);
1209 return SUCCESS;
1210 }
1211
AssembleHeifAuxiliaryPicture(std::vector<ImageItem> & inputImgs,std::vector<ItemRef> & refs)1212 uint32_t ExtEncoder::AssembleHeifAuxiliaryPicture(std::vector<ImageItem>& inputImgs, std::vector<ItemRef>& refs)
1213 {
1214 bool cond = !picture_;
1215 CHECK_ERROR_RETURN_RET_LOG(cond, ERR_IMAGE_INVALID_PARAMETER, "picture_ is nullptr");
1216 if (picture_->HasAuxiliaryPicture(AuxiliaryPictureType::DEPTH_MAP) &&
1217 AssembleHeifAuxiliaryNoncodingMap(inputImgs, AuxiliaryPictureType::DEPTH_MAP) == SUCCESS) {
1218 AssembleAuxiliaryRefItem(AuxiliaryPictureType::DEPTH_MAP, refs);
1219 }
1220 if (picture_->HasAuxiliaryPicture(AuxiliaryPictureType::UNREFOCUS_MAP) &&
1221 AssembleHeifUnrefocusMap(inputImgs) == SUCCESS) {
1222 AssembleAuxiliaryRefItem(AuxiliaryPictureType::UNREFOCUS_MAP, refs);
1223 }
1224 if (picture_->HasAuxiliaryPicture(AuxiliaryPictureType::LINEAR_MAP) &&
1225 AssembleHeifAuxiliaryNoncodingMap(inputImgs, AuxiliaryPictureType::LINEAR_MAP) == SUCCESS) {
1226 AssembleAuxiliaryRefItem(AuxiliaryPictureType::LINEAR_MAP, refs);
1227 }
1228 if (picture_->HasAuxiliaryPicture(AuxiliaryPictureType::FRAGMENT_MAP) &&
1229 AssembleHeifFragmentMap(inputImgs) == SUCCESS) {
1230 AssembleAuxiliaryRefItem(AuxiliaryPictureType::FRAGMENT_MAP, refs);
1231 }
1232 return SUCCESS;
1233 }
1234
InitAuxiliaryImageItem(uint32_t id,std::string itemName)1235 ImageItem ExtEncoder::InitAuxiliaryImageItem(uint32_t id, std::string itemName)
1236 {
1237 ImageItem item;
1238 item.id = id;
1239 item.itemName = itemName;
1240 item.isPrimary = false;
1241 item.isHidden = true;
1242 item.quality = opts_.quality;
1243 item.sharedProperties.fd = -1;
1244 item.pixelSharedBuffer.fd = -1;
1245 if (id == DEPTH_MAP_ITEM_ID || id == LINEAR_MAP_ITEM_ID) {
1246 item.compressType = COMPRESS_TYPE_NONE;
1247 } else {
1248 item.compressType = COMPRESS_TYPE_HEVC;
1249 }
1250 return item;
1251 }
1252
GetHeifEncodeItemInfo(AuxiliaryPictureType auxType)1253 HeifEncodeItemInfo GetHeifEncodeItemInfo(AuxiliaryPictureType auxType)
1254 {
1255 HeifEncodeItemInfo itemInfo;
1256 switch (auxType) {
1257 case AuxiliaryPictureType::GAINMAP:
1258 itemInfo.itemId = GAINMAP_IMAGE_ITEM_ID;
1259 itemInfo.itemName = GAINMAP_IMAGE_ITEM_NAME;
1260 itemInfo.itemType = HEIF_AUXTTYPE_ID_GAINMAP;
1261 break;
1262 case AuxiliaryPictureType::DEPTH_MAP:
1263 itemInfo.itemId = DEPTH_MAP_ITEM_ID;
1264 itemInfo.itemName = DEPTH_MAP_ITEM_NAME;
1265 itemInfo.itemType = HEIF_AUXTTYPE_ID_DEPTH_MAP;
1266 break;
1267 case AuxiliaryPictureType::UNREFOCUS_MAP:
1268 itemInfo.itemId = UNREFOCUS_MAP_ITEM_ID;
1269 itemInfo.itemName = UNREFOCUS_MAP_ITEM_NAME;
1270 itemInfo.itemType = HEIF_AUXTTYPE_ID_UNREFOCUS_MAP;
1271 break;
1272 case AuxiliaryPictureType::LINEAR_MAP:
1273 itemInfo.itemId = LINEAR_MAP_ITEM_ID;
1274 itemInfo.itemName = LINEAR_MAP_ITEM_NAME;
1275 itemInfo.itemType = HEIF_AUXTTYPE_ID_LINEAR_MAP;
1276 break;
1277 case AuxiliaryPictureType::FRAGMENT_MAP:
1278 itemInfo.itemId = FRAGMENT_MAP_ITEM_ID;
1279 itemInfo.itemName = FRAGMENT_MAP_ITEM_NAME;
1280 itemInfo.itemType = HEIF_AUXTTYPE_ID_FRAGMENT_MAP;
1281 break;
1282 default:
1283 break;
1284 }
1285 return itemInfo;
1286 }
1287
AssembleHeifAuxiliaryNoncodingMap(std::vector<ImageItem> & inputImgs,AuxiliaryPictureType auxType)1288 uint32_t ExtEncoder::AssembleHeifAuxiliaryNoncodingMap(std::vector<ImageItem>& inputImgs, AuxiliaryPictureType auxType)
1289 {
1290 auto auxMap = picture_->GetAuxiliaryPicture(auxType);
1291 if (!auxMap || !auxMap->GetContentPixel() ||
1292 auxMap->GetContentPixel()->GetAllocatorType() != AllocatorType::DMA_ALLOC) {
1293 IMAGE_LOGE("%{public}s The auxMap is nullptr or allocator type is not DMA_ALLOC", __func__);
1294 return ERR_IMAGE_INVALID_PARAMETER;
1295 }
1296 sptr<SurfaceBuffer> auxMapSptr(reinterpret_cast<SurfaceBuffer*>(auxMap->GetContentPixel()->GetFd()));
1297 bool cond = !auxMapSptr;
1298 CHECK_ERROR_RETURN_RET_LOG(cond, ERR_IMAGE_INVALID_PARAMETER, "%{public}s auxMapSptr is nullptr", __func__);
1299
1300 HeifEncodeItemInfo itemInfo = GetHeifEncodeItemInfo(auxType);
1301 auto item = InitAuxiliaryImageItem(itemInfo.itemId, itemInfo.itemName);
1302 std::string auxTypeStr = itemInfo.itemType;
1303 Resolution resolution = {
1304 auxMap->GetContentPixel()->GetWidth(),
1305 auxMap->GetContentPixel()->GetHeight()
1306 };
1307 uint32_t litePropertiesSize =
1308 sizeof(PropertyType::AUX_TYPE) + UINT32_BYTES_NUM + auxTypeStr.length() + PLACE_HOLDER_LENGTH;
1309 litePropertiesSize += (sizeof(PropertyType::IMG_RESOLUTION) + sizeof(Resolution));
1310 size_t offset = 0;
1311 item.liteProperties.resize(litePropertiesSize);
1312 cond = !FillLitePropertyItemByString(item.liteProperties, offset, PropertyType::AUX_TYPE, auxTypeStr);
1313 CHECK_ERROR_RETURN_RET_LOG(cond, ERR_IMAGE_INVALID_PARAMETER, "%{public}s Fill auxiliary type failed", __func__);
1314 cond = !FillLitePropertyItem(item.liteProperties, offset, PropertyType::IMG_RESOLUTION, &resolution,
1315 sizeof(Resolution));
1316 CHECK_ERROR_RETURN_RET_LOG(cond, ERR_IMAGE_INVALID_PARAMETER, "%{public}s Fill color type failed", __func__);
1317 uint32_t capacity = auxMap->GetContentPixel()->GetCapacity();
1318 cond = !FillPixelSharedBuffer(auxMapSptr, capacity, item.pixelSharedBuffer);
1319 CHECK_ERROR_RETURN_RET_LOG(cond, ERR_IMAGE_INVALID_PARAMETER,
1320 "%{public}s Fill pixel shared buffer failed", __func__);
1321 inputImgs.push_back(item);
1322 return SUCCESS;
1323 }
1324
AssembleHeifUnrefocusMap(std::vector<ImageItem> & inputImgs)1325 uint32_t ExtEncoder::AssembleHeifUnrefocusMap(std::vector<ImageItem>& inputImgs)
1326 {
1327 auto unrefocusMap = picture_->GetAuxiliaryPicture(AuxiliaryPictureType::UNREFOCUS_MAP);
1328 bool cond = !unrefocusMap || !unrefocusMap->GetContentPixel();
1329 CHECK_ERROR_RETURN_RET_LOG(cond, ERR_IMAGE_INVALID_PARAMETER, "%{public}s The unrefocusMap is nullptr", __func__);
1330 HeifEncodeItemInfo itemInfo = GetHeifEncodeItemInfo(AuxiliaryPictureType::UNREFOCUS_MAP);
1331 auto item = InitAuxiliaryImageItem(itemInfo.itemId, itemInfo.itemName);
1332 bool sdrIsSRGB = unrefocusMap->GetContentPixel()->GetToSdrColorSpaceIsSRGB();
1333 SkImageInfo unrefocusInfo = GetSkInfo(unrefocusMap->GetContentPixel().get(), false, sdrIsSRGB);
1334 #ifdef USE_M133_SKIA
1335 sk_sp<SkData> iccProfile = icc_from_color_space(unrefocusInfo, nullptr, nullptr);
1336 #else
1337 sk_sp<SkData> iccProfile = icc_from_color_space(unrefocusInfo);
1338 #endif
1339 cond = !AssembleICCImageProperty(iccProfile, item.sharedProperties);
1340 CHECK_ERROR_RETURN_RET_LOG(cond, ERR_IMAGE_INVALID_PARAMETER,
1341 "%{public}s AssembleICCImageProperty failed", __func__);
1342 sptr<SurfaceBuffer> unrefocusMapSptr = ConvertPixelMapToDmaBuffer(unrefocusMap->GetContentPixel());
1343 cond = !unrefocusMapSptr;
1344 CHECK_ERROR_RETURN_RET_LOG(cond, ERR_IMAGE_INVALID_PARAMETER, "%{public}s unrefocusMapSptr is nullptr", __func__);
1345 item.pixelBuffer = sptr<NativeBuffer>::MakeSptr(unrefocusMapSptr->GetBufferHandle());
1346 std::string auxTypeStr = itemInfo.itemType;
1347
1348 uint32_t litePropertiesSize =
1349 sizeof(PropertyType::AUX_TYPE) + UINT32_BYTES_NUM + auxTypeStr.length() + PLACE_HOLDER_LENGTH;
1350 litePropertiesSize += (sizeof(PropertyType::COLOR_TYPE) + sizeof(ColorType));
1351 item.liteProperties.resize(litePropertiesSize);
1352 size_t offset = 0;
1353 cond = !FillLitePropertyItemByString(item.liteProperties, offset, PropertyType::AUX_TYPE, auxTypeStr);
1354 CHECK_ERROR_RETURN_RET_LOG(cond, ERR_IMAGE_INVALID_PARAMETER, "%{public}s Fill auxiliary type failed", __func__);
1355 ColorType colorType = ColorType::RICC;
1356 cond = !FillLitePropertyItem(item.liteProperties, offset, PropertyType::COLOR_TYPE, &colorType, sizeof(ColorType));
1357 CHECK_ERROR_RETURN_RET_LOG(cond, ERR_IMAGE_INVALID_PARAMETER, "%{public}s Fill color type failed", __func__);
1358 inputImgs.push_back(item);
1359 return SUCCESS;
1360 }
1361
GetFragmentRelLocation(std::shared_ptr<AuxiliaryPicture> & fragmentMap)1362 RelativeLocation GetFragmentRelLocation(std::shared_ptr<AuxiliaryPicture> &fragmentMap)
1363 {
1364 RelativeLocation loc;
1365 auto fragmentMeta = fragmentMap->GetMetadata(MetadataType::FRAGMENT);
1366 bool cond = fragmentMeta == nullptr;
1367 CHECK_ERROR_RETURN_RET_LOG(cond, loc, "The fragmentMap has not fragmentMap");
1368 std::string loc_x = "";
1369 std::string loc_y = "";
1370 uint32_t horizontalOffset = 0;
1371 uint32_t verticalOffset = 0;
1372 fragmentMeta->GetValue(FRAGMENT_METADATA_KEY_X, loc_x);
1373 fragmentMeta->GetValue(FRAGMENT_METADATA_KEY_Y, loc_y);
1374 ImageUtils::StrToUint32(loc_x, horizontalOffset);
1375 ImageUtils::StrToUint32(loc_y, verticalOffset);
1376 loc.horizontalOffset = horizontalOffset;
1377 loc.verticalOffset = verticalOffset;
1378 return loc;
1379 }
1380
AssembleHeifFragmentMap(std::vector<ImageItem> & inputImgs)1381 uint32_t ExtEncoder::AssembleHeifFragmentMap(std::vector<ImageItem>& inputImgs)
1382 {
1383 auto fragmentMap = picture_->GetAuxiliaryPicture(AuxiliaryPictureType::FRAGMENT_MAP);
1384 bool cond = !fragmentMap || !fragmentMap->GetContentPixel();
1385 CHECK_ERROR_RETURN_RET_LOG(cond, ERR_IMAGE_INVALID_PARAMETER, "%{public}s The fragmentMap is nullptr", __func__);
1386 sptr<SurfaceBuffer> fragmentMapSptr = ConvertPixelMapToDmaBuffer(fragmentMap->GetContentPixel());
1387 cond = !fragmentMapSptr;
1388 CHECK_ERROR_RETURN_RET_LOG(cond, ERR_IMAGE_INVALID_PARAMETER, "%{public}s fragmentMapSptr is nullptr", __func__);
1389 HeifEncodeItemInfo itemInfo = GetHeifEncodeItemInfo(AuxiliaryPictureType::FRAGMENT_MAP);
1390 auto item = InitAuxiliaryImageItem(itemInfo.itemId, itemInfo.itemName);
1391 bool sdrIsSRGB = fragmentMap->GetContentPixel()->GetToSdrColorSpaceIsSRGB();
1392 SkImageInfo fragmentInfo = GetSkInfo(fragmentMap->GetContentPixel().get(), false, sdrIsSRGB);
1393 #ifdef USE_M133_SKIA
1394 sk_sp<SkData> iccProfile = icc_from_color_space(fragmentInfo, nullptr, nullptr);
1395 #else
1396 sk_sp<SkData> iccProfile = icc_from_color_space(fragmentInfo);
1397 #endif
1398 cond = !AssembleICCImageProperty(iccProfile, item.sharedProperties);
1399 CHECK_ERROR_RETURN_RET_LOG(cond, ERR_IMAGE_INVALID_PARAMETER,
1400 "%{public}s AssembleICCImageProperty failed", __func__);
1401 item.pixelBuffer = sptr<NativeBuffer>::MakeSptr(fragmentMapSptr->GetBufferHandle());
1402 std::string auxTypeStr = itemInfo.itemType;
1403 uint32_t litePropertiesSize =
1404 sizeof(PropertyType::AUX_TYPE) + UINT32_BYTES_NUM + auxTypeStr.length() + PLACE_HOLDER_LENGTH;
1405 litePropertiesSize += (sizeof(PropertyType::COLOR_TYPE) + sizeof(ColorType));
1406 litePropertiesSize += (sizeof(PropertyType::RLOC_INFO) + sizeof(RelativeLocation));
1407 item.liteProperties.resize(litePropertiesSize);
1408 size_t offset = 0;
1409 ColorType colorType = ColorType::RICC;
1410 cond = !FillLitePropertyItem(item.liteProperties, offset, PropertyType::COLOR_TYPE, &colorType, sizeof(ColorType));
1411 CHECK_ERROR_RETURN_RET_LOG(cond, ERR_IMAGE_INVALID_PARAMETER, "%{public}s Fill color type failed", __func__);
1412 cond = !FillLitePropertyItemByString(item.liteProperties, offset, PropertyType::AUX_TYPE, auxTypeStr);
1413 CHECK_ERROR_RETURN_RET_LOG(cond, ERR_IMAGE_INVALID_PARAMETER, "%{public}s Fill auxiliary type failed", __func__);
1414 RelativeLocation loc = GetFragmentRelLocation(fragmentMap);
1415 cond = !FillLitePropertyItem(item.liteProperties, offset, PropertyType::RLOC_INFO, &loc, sizeof(RelativeLocation));
1416 CHECK_ERROR_RETURN_RET_LOG(cond, ERR_IMAGE_INVALID_PARAMETER, "%{public}s Fill auxiliary type failed", __func__);
1417 inputImgs.push_back(item);
1418 return SUCCESS;
1419 }
1420 #endif
1421
DecomposeDualVivid(VpeSurfaceBuffers & buffers,Media::PixelMap * pixelmap,SkEncodedImageFormat format,HdrMetadata & metadata)1422 uint32_t DecomposeDualVivid(VpeSurfaceBuffers& buffers, Media::PixelMap *pixelmap,
1423 SkEncodedImageFormat format, HdrMetadata& metadata)
1424 {
1425 if (pixelmap == nullptr) {
1426 return ERR_IMAGE_INVALID_PARAMETER;
1427 }
1428 if ((!pixelmap->IsHdr() && !IsWideGamutSdrPixelMap(pixelmap)) ||
1429 pixelmap->GetAllocatorType() != AllocatorType::DMA_ALLOC ||
1430 (format != SkEncodedImageFormat::kJPEG && format != SkEncodedImageFormat::kHEIF)) {
1431 return ERR_IMAGE_INVALID_PARAMETER;
1432 }
1433 bool sdrIsSRGB = pixelmap->GetToSdrColorSpaceIsSRGB();
1434 sptr<SurfaceBuffer> hdrSurfaceBuffer(reinterpret_cast<SurfaceBuffer*> (pixelmap->GetFd()));
1435 sptr<SurfaceBuffer> baseSptr = AllocSurfaceBuffer(hdrSurfaceBuffer->GetWidth(),
1436 hdrSurfaceBuffer->GetHeight());
1437 const int halfSizeDenominator = 2;
1438 sptr<SurfaceBuffer> gainmapSptr = AllocSurfaceBuffer(hdrSurfaceBuffer->GetWidth() / halfSizeDenominator,
1439 hdrSurfaceBuffer->GetHeight() / halfSizeDenominator);
1440 bool cond = baseSptr == nullptr || gainmapSptr == nullptr;
1441 CHECK_ERROR_RETURN_RET(cond, IMAGE_RESULT_CREATE_SURFAC_FAILED);
1442 CM_ColorSpaceType colorspaceType;
1443 VpeUtils::GetSbColorSpaceType(hdrSurfaceBuffer, colorspaceType);
1444 if ((colorspaceType & CM_PRIMARIES_MASK) != COLORPRIMARIES_BT2020) {
1445 #ifdef IMAGE_COLORSPACE_FLAG
1446 ColorManager::ColorSpaceName colorspace = pixelmap->InnerGetGrColorSpace().GetColorSpaceName();
1447 IMAGE_LOGI("ExtEncoder SetHdrColorSpaceType, color is %{public}d", colorspace);
1448 colorspaceType = ColorUtils::ConvertToCMColor(colorspace);
1449 VpeUtils::SetSbColorSpaceType(hdrSurfaceBuffer, colorspaceType);
1450 #endif
1451 }
1452 CM_HDR_Metadata_Type hdrMetadataType;
1453 VpeUtils::GetSbMetadataType(hdrSurfaceBuffer, hdrMetadataType);
1454 VpeUtils::SetSbMetadataType(hdrSurfaceBuffer, CM_IMAGE_HDR_VIVID_SINGLE);
1455 buffers.sdr = baseSptr;
1456 buffers.gainmap = gainmapSptr;
1457 buffers.hdr = hdrSurfaceBuffer;
1458 if (!DecomposeImage(buffers, metadata, false, sdrIsSRGB)) {
1459 IMAGE_LOGE("HDR-IMAGE EncodeDualVivid decomposeImage failed");
1460 FreeBaseAndGainMapSurfaceBuffer(baseSptr, gainmapSptr);
1461 return IMAGE_RESULT_CREATE_SURFAC_FAILED;
1462 }
1463 metadata.hdrMetadataType = static_cast<int32_t>(hdrMetadataType);
1464 return SUCCESS;
1465 }
1466
EncodeDualVivid(ExtWStream & outputStream)1467 uint32_t ExtEncoder::EncodeDualVivid(ExtWStream& outputStream)
1468 {
1469 IMAGE_LOGD("ExtEncoder::EncodeDualVivid");
1470 VpeSurfaceBuffers buffers;
1471 HdrMetadata metadata;
1472 if (DecomposeDualVivid(buffers, pixelmap_, encodeFormat_, metadata) != SUCCESS) {
1473 return IMAGE_RESULT_CREATE_SURFAC_FAILED;
1474 }
1475 // get sdr baseInfo
1476 bool sdrIsSRGB = pixelmap_->GetToSdrColorSpaceIsSRGB();
1477 sk_sp<SkColorSpace> colorSpace = ToHdrEncodeSkColorSpace(pixelmap_, buffers.sdr, sdrIsSRGB, false);
1478 SkImageInfo baseInfo = GetSkInfo(pixelmap_, false, colorSpace);
1479 // get gainmap baseInfo
1480 sk_sp<SkColorSpace> gainmapColorSpace = ToHdrEncodeSkColorSpace(pixelmap_, buffers.gainmap, sdrIsSRGB, true);
1481 SkImageInfo gainmapInfo = GetSkInfo(pixelmap_, true, gainmapColorSpace);
1482 uint32_t error;
1483 if (encodeFormat_ == SkEncodedImageFormat::kJPEG) {
1484 sk_sp<SkData> baseImageData = GetImageEncodeData(buffers.sdr, baseInfo, opts_.needsPackProperties);
1485 sk_sp<SkData> gainMapImageData = GetImageEncodeData(buffers.gainmap, gainmapInfo, false);
1486 error = HdrJpegPackerHelper::SpliceHdrStream(baseImageData, gainMapImageData, outputStream, metadata);
1487 } else if (encodeFormat_ == SkEncodedImageFormat::kHEIF) {
1488 error = EncodeHeifDualHdrImage(buffers.sdr, buffers.gainmap, metadata);
1489 } else {
1490 error = ERR_IMAGE_INVALID_PARAMETER;
1491 }
1492 FreeBaseAndGainMapSurfaceBuffer(buffers.sdr, buffers.gainmap);
1493 return error;
1494 }
1495
EncodeSdrImage(ExtWStream & outputStream)1496 uint32_t ExtEncoder::EncodeSdrImage(ExtWStream& outputStream)
1497 {
1498 IMAGE_LOGD("ExtEncoder EncodeSdrImage");
1499 if (!pixelmap_->IsHdr() && !IsWideGamutSdrPixelMap(pixelmap_)) {
1500 return EncodeImageByPixelMap(pixelmap_, opts_.needsPackProperties, outputStream);
1501 }
1502 bool cond = pixelmap_->GetAllocatorType() != AllocatorType::DMA_ALLOC;
1503 CHECK_ERROR_RETURN_RET_LOG(cond, ERR_IMAGE_INVALID_PARAMETER, "pixelmap is 10bit, but not dma buffer");
1504 ImageInfo info;
1505 pixelmap_->GetImageInfo(info);
1506 bool sdrIsSRGB = pixelmap_->GetToSdrColorSpaceIsSRGB();
1507 SkImageInfo baseInfo = GetSkInfo(pixelmap_, false, sdrIsSRGB);
1508 sptr<SurfaceBuffer> baseSptr = AllocSurfaceBuffer(baseInfo.width(), baseInfo.height());
1509 VpeUtils::SetSbMetadataType(baseSptr, CM_IMAGE_HDR_VIVID_DUAL);
1510 VpeUtils::SetSbColorSpaceType(baseSptr, CM_SRGB_FULL);
1511 cond = baseSptr == nullptr;
1512 CHECK_ERROR_RETURN_RET_LOG(cond, IMAGE_RESULT_CREATE_SURFAC_FAILED, "EncodeSdrImage sdr buffer alloc failed");
1513 sptr<SurfaceBuffer> hdrSurfaceBuffer(reinterpret_cast<SurfaceBuffer*>(pixelmap_->GetFd()));
1514 ImageUtils::FlushSurfaceBuffer(hdrSurfaceBuffer);
1515 VpeUtils::SetSbMetadataType(hdrSurfaceBuffer, CM_IMAGE_HDR_VIVID_SINGLE);
1516 SetHdrColorSpaceType(hdrSurfaceBuffer);
1517 VpeSurfaceBuffers buffers = {
1518 .sdr = baseSptr,
1519 .hdr = hdrSurfaceBuffer,
1520 };
1521 HdrMetadata metadata;
1522 if (!DecomposeImage(buffers, metadata, true, sdrIsSRGB)) {
1523 IMAGE_LOGE("EncodeSdrImage decomposeImage failed");
1524 ImageUtils::SurfaceBuffer_Unreference(baseSptr.GetRefPtr());
1525 return IMAGE_RESULT_CREATE_SURFAC_FAILED;
1526 }
1527 uint32_t error;
1528 if (encodeFormat_ == SkEncodedImageFormat::kHEIF) {
1529 error = EncodeHeifSdrImage(baseSptr, baseInfo);
1530 } else {
1531 error = EncodeImageBySurfaceBuffer(baseSptr, baseInfo, opts_.needsPackProperties, outputStream);
1532 }
1533 ImageUtils::SurfaceBuffer_Unreference(baseSptr.GetRefPtr());
1534 return error;
1535 }
1536
EncodeHeifDualHdrImage(sptr<SurfaceBuffer> & sdr,sptr<SurfaceBuffer> & gainmap,Media::HdrMetadata & metadata,bool sdrIsSRGB)1537 uint32_t ExtEncoder::EncodeHeifDualHdrImage(sptr<SurfaceBuffer>& sdr, sptr<SurfaceBuffer>& gainmap,
1538 Media::HdrMetadata& metadata, bool sdrIsSRGB)
1539 {
1540 #ifdef HEIF_HW_ENCODE_ENABLE
1541 std::vector<ImageItem> inputImgs;
1542 ColorSpaceManager colorspaceName =
1543 sdrIsSRGB ? ColorManager::ColorSpaceName::SRGB : ColorManager::ColorSpaceName::DISPLAY_P3;
1544 std::shared_ptr<ImageItem> primaryItem =
1545 AssembleHdrBaseImageItem(sdr, colorspaceName, metadata, opts_);
1546 bool cond = primaryItem == nullptr;
1547 CHECK_ERROR_RETURN_RET_LOG(cond, ERR_IMAGE_INVALID_PARAMETER, "AssmbleHeifDualHdrImage, get primary image failed");
1548 inputImgs.push_back(*primaryItem);
1549 std::shared_ptr<ImageItem> gainmapItem =
1550 AssembleGainmapImageItem(gainmap, colorspaceName, opts_);
1551 cond = gainmapItem == nullptr;
1552 CHECK_ERROR_RETURN_RET_LOG(cond, ERR_IMAGE_INVALID_PARAMETER,
1553 "AssembleDualHdrImage, get gainmap image item failed");
1554 inputImgs.push_back(*gainmapItem);
1555 ColorManager::ColorSpaceName tmapColor = pixelmap_->InnerGetGrColorSpace().GetColorSpaceName();
1556 std::shared_ptr<ImageItem> tmapItem = AssembleTmapImageItem(tmapColor, metadata, opts_);
1557 cond = tmapItem == nullptr;
1558 CHECK_ERROR_RETURN_RET_LOG(cond, ERR_IMAGE_INVALID_PARAMETER, "AssembleDualHdrImage, get tmap image item failed");
1559 inputImgs.push_back(*tmapItem);
1560 std::vector<MetaItem> inputMetas;
1561 std::vector<ItemRef> refs;
1562 if (AssembleExifMetaItem(inputMetas)) {
1563 AssembleExifRefItem(refs);
1564 }
1565 AssembleDualHdrRefItem(refs);
1566 return DoHeifEncode(inputImgs, inputMetas, refs);
1567 #else
1568 return ERR_IMAGE_INVALID_PARAMETER;
1569 #endif
1570 }
1571
EncodeHeifSdrImage(sptr<SurfaceBuffer> & sdr,SkImageInfo sdrInfo)1572 uint32_t ExtEncoder::EncodeHeifSdrImage(sptr<SurfaceBuffer>& sdr, SkImageInfo sdrInfo)
1573 {
1574 #ifdef HEIF_HW_ENCODE_ENABLE
1575 bool cond = sdr == nullptr;
1576 CHECK_ERROR_RETURN_RET_LOG(cond, ERR_IMAGE_INVALID_PARAMETER, "EncodeHeifSdrImage, sdr surfacebuffer is nullptr");
1577 ImageItem item;
1578 item.id = PRIMARY_IMAGE_ITEM_ID;
1579 item.pixelBuffer = sptr<NativeBuffer>::MakeSptr(sdr->GetBufferHandle());
1580 item.isPrimary = true;
1581 item.isHidden = false;
1582 item.compressType = COMPRESS_TYPE_HEVC;
1583 item.quality = opts_.quality;
1584 item.sharedProperties.fd = -1;
1585 item.pixelSharedBuffer.fd = -1;
1586 ImageInfo info;
1587 pixelmap_->GetImageInfo(info);
1588 #ifdef USE_M133_SKIA
1589 sk_sp<SkData> iccProfile = icc_from_color_space(sdrInfo, nullptr, nullptr);
1590 #else
1591 sk_sp<SkData> iccProfile = icc_from_color_space(sdrInfo);
1592 #endif
1593 bool tempRes = AssembleICCImageProperty(iccProfile, item.sharedProperties);
1594 cond = !tempRes;
1595 CHECK_ERROR_RETURN_RET_LOG(cond, ERR_IMAGE_INVALID_PARAMETER, "EncodeSdrImage AssembleICCImageProperty failed");
1596 uint32_t litePropertiesSize = (sizeof(PropertyType::COLOR_TYPE) + sizeof(ColorType));
1597 item.liteProperties.resize(litePropertiesSize);
1598 size_t offset = 0;
1599 ColorType colorType = ColorType::RICC;
1600 cond = !FillLitePropertyItem(item.liteProperties, offset, PropertyType::COLOR_TYPE, &colorType, sizeof(ColorType));
1601 CHECK_ERROR_RETURN_RET_LOG(cond, ERR_IMAGE_INVALID_PARAMETER, "EncodeHeifSdrImage Fill color type failed");
1602 std::vector<ImageItem> inputImgs;
1603 inputImgs.push_back(item);
1604 std::vector<MetaItem> inputMetas;
1605 std::vector<ItemRef> refs;
1606 if (AssembleExifMetaItem(inputMetas)) {
1607 AssembleExifRefItem(refs);
1608 }
1609 return DoHeifEncode(inputImgs, inputMetas, refs);
1610 #else
1611 return ERR_IMAGE_INVALID_PARAMETER;
1612 #endif
1613 }
1614
EncodePicture()1615 uint32_t ExtEncoder::EncodePicture()
1616 {
1617 bool cond = (encodeFormat_ != SkEncodedImageFormat::kJPEG && encodeFormat_ != SkEncodedImageFormat::kHEIF);
1618 CHECK_ERROR_RETURN_RET_LOG(cond, ERR_IMAGE_INVALID_PARAMETER,
1619 "%{public}s: unsupported encode format: %{public}s", __func__, opts_.format.c_str());
1620 if (opts_.isEditScene && encodeFormat_ == SkEncodedImageFormat::kHEIF) {
1621 return EncodeEditScenePicture();
1622 }
1623 if (encodeFormat_ == SkEncodedImageFormat::kJPEG) {
1624 CheckJpegAuxiliaryTagName();
1625 }
1626 ExtWStream wStream(output_);
1627 return EncodeCameraScenePicture(wStream);
1628 }
1629
IsValidSizeForHardwareEncode(int32_t width,int32_t height)1630 static bool IsValidSizeForHardwareEncode(int32_t width, int32_t height)
1631 {
1632 if (ImageUtils::IsEven(width) && ImageUtils::IsInRange(width, MIN_IMAGE_SIZE, MAX_IMAGE_SIZE) &&
1633 ImageUtils::IsEven(height) && ImageUtils::IsInRange(height, MIN_IMAGE_SIZE, MAX_IMAGE_SIZE)) {
1634 return true;
1635 }
1636 IMAGE_LOGD("%{public}s hardware encode is not support, width: %{public}d, height: %{public}d",
1637 __func__, width, height);
1638 return false;
1639 }
1640
IsPictureSupportHardwareEncode()1641 bool ExtEncoder::IsPictureSupportHardwareEncode()
1642 {
1643 bool cond = !ImageSystemProperties::GetHardWareEncodeEnabled();
1644 CHECK_ERROR_RETURN_RET_LOG(cond, false, "%{public}s hardware encode disabled", __func__);
1645 cond = picture_ == nullptr;
1646 CHECK_ERROR_RETURN_RET_LOG(cond, false, "%{public}s picture is null", __func__);
1647
1648 if (opts_.quality < LOW_QUALITY_BOUNDARY) {
1649 IMAGE_LOGE("%{public}s Low quality use jpeg software encode", __func__);
1650 return false;
1651 }
1652 auto mainPixelMap = picture_->GetMainPixel();
1653 if (mainPixelMap == nullptr ||
1654 !IsValidSizeForHardwareEncode(mainPixelMap->GetWidth(), mainPixelMap->GetHeight())) {
1655 IMAGE_LOGI("%{public}s MainPixelMap not support hardware encode", __func__);
1656 return false;
1657 }
1658
1659 const auto& auxTypes = ImageUtils::GetAllAuxiliaryPictureType();
1660 for (const auto& auxType : auxTypes) {
1661 auto auxPicture = picture_->GetAuxiliaryPicture(auxType);
1662 if (auxPicture == nullptr) {
1663 continue;
1664 }
1665 auto auxPixelMap = auxPicture->GetContentPixel();
1666 if (auxPixelMap == nullptr) {
1667 continue;
1668 }
1669 cond = !IsValidSizeForHardwareEncode(auxPixelMap->GetWidth(), auxPixelMap->GetHeight());
1670 CHECK_INFO_RETURN_RET_LOG(cond, false,
1671 "%{public}s auxType: %{public}d not support hardware encode", __func__, auxType);
1672 }
1673 return true;
1674 }
1675
TryHardwareEncodePicture(SkWStream & skStream,std::string & errorMsg)1676 uint32_t ExtEncoder::TryHardwareEncodePicture(SkWStream& skStream, std::string& errorMsg)
1677 {
1678 errorMsg = "Hardware encode picture failed";
1679 if (picture_ == nullptr) {
1680 errorMsg = "picture is nullptr";
1681 return ERR_IMAGE_DATA_ABNORMAL;
1682 }
1683 static ImageFwkExtManager imageFwkExtManager;
1684 if (!imageFwkExtManager.LoadImageFwkExtNativeSo() || imageFwkExtManager.doHardwareEncodePictureFunc_ == nullptr) {
1685 errorMsg = "Load hardware encode library failed";
1686 return ERR_IMAGE_ENCODE_FAILED;
1687 }
1688 return imageFwkExtManager.doHardwareEncodePictureFunc_(&skStream, opts_, picture_);
1689 }
1690
EncodeCameraScenePicture(SkWStream & skStream)1691 uint32_t ExtEncoder::EncodeCameraScenePicture(SkWStream& skStream)
1692 {
1693 uint32_t retCode = ERR_IMAGE_ENCODE_FAILED;
1694 std::string errorMsg = "Unknown encode failed";
1695 if (encodeFormat_ == SkEncodedImageFormat::kHEIF) {
1696 retCode = TryHardwareEncodePicture(skStream, errorMsg);
1697 } else if (encodeFormat_ == SkEncodedImageFormat::kJPEG) {
1698 if (IsPictureSupportHardwareEncode()) {
1699 retCode = TryHardwareEncodePicture(skStream, errorMsg);
1700 } else {
1701 errorMsg = "Not support hardware encode";
1702 }
1703 if (retCode != SUCCESS) {
1704 IMAGE_LOGW("%{public}s, retCode is: %{public}d, try jpeg software encode", errorMsg.c_str(), retCode);
1705 retCode = EncodeJpegPicture(skStream);
1706 errorMsg = "Jpeg software encode picture failed";
1707 }
1708 }
1709 if (retCode != SUCCESS) {
1710 ImageInfo imageInfo;
1711 picture_->GetMainPixel()->GetImageInfo(imageInfo);
1712 IMAGE_LOGE("%{public}s, retCode is: %{public}d", errorMsg.c_str(), retCode);
1713 ReportEncodeFault(imageInfo.size.width, imageInfo.size.height, opts_.format, errorMsg);
1714 }
1715 return retCode;
1716 }
1717
EncodeEditScenePicture()1718 uint32_t ExtEncoder::EncodeEditScenePicture()
1719 {
1720 bool cond = !picture_;
1721 CHECK_ERROR_RETURN_RET_LOG(cond, ERR_IMAGE_DATA_ABNORMAL, "picture_ is nullptr");
1722 auto mainPixelMap = picture_->GetMainPixel();
1723 cond = !mainPixelMap || mainPixelMap->GetAllocatorType() != AllocatorType::DMA_ALLOC;
1724 CHECK_ERROR_RETURN_RET_LOG(cond, ERR_IMAGE_DATA_ABNORMAL,
1725 "MainPixelMap is nullptr or mainPixelMap is not DMA buffer");
1726
1727 bool sdrIsSRGB = mainPixelMap->GetToSdrColorSpaceIsSRGB();
1728 SkImageInfo baseInfo = GetSkInfo(mainPixelMap.get(), false, false);
1729 sptr<SurfaceBuffer> baseSptr(reinterpret_cast<SurfaceBuffer*>(mainPixelMap->GetFd()));
1730 cond = !baseSptr;
1731 CHECK_ERROR_RETURN_RET_LOG(cond, IMAGE_RESULT_CREATE_SURFAC_FAILED, "creat main pixels surfaceBuffer error");
1732 ImageUtils::FlushSurfaceBuffer(baseSptr);
1733 uint32_t errorCode = EncodeHeifPicture(baseSptr, baseInfo, sdrIsSRGB);
1734 RecycleResources();
1735 return errorCode;
1736 }
1737
EncodeHeifPicture(sptr<SurfaceBuffer> & mainSptr,SkImageInfo mainInfo,bool sdrIsSRGB)1738 uint32_t ExtEncoder::EncodeHeifPicture(sptr<SurfaceBuffer>& mainSptr, SkImageInfo mainInfo, bool sdrIsSRGB)
1739 {
1740 #ifdef HEIF_HW_ENCODE_ENABLE
1741 uint32_t error = SUCCESS;
1742 std::vector<ImageItem> inputImgs;
1743 std::vector<MetaItem> inputMetas;
1744 std::vector<ItemRef> refs;
1745 switch (opts_.desiredDynamicRange) {
1746 case EncodeDynamicRange::AUTO:
1747 if (picture_->HasAuxiliaryPicture(AuxiliaryPictureType::GAINMAP) &&
1748 AssembleHeifHdrPicture(mainSptr, sdrIsSRGB, inputImgs) == SUCCESS) {
1749 AssembleDualHdrRefItem(refs);
1750 } else {
1751 inputImgs.clear();
1752 error = AssembleSdrImageItem(mainSptr, mainInfo, inputImgs);
1753 }
1754 break;
1755 case EncodeDynamicRange::SDR:
1756 error = AssembleSdrImageItem(mainSptr, mainInfo, inputImgs);
1757 break;
1758 case EncodeDynamicRange::HDR_VIVID_DUAL:
1759 if (picture_->HasAuxiliaryPicture(AuxiliaryPictureType::GAINMAP)) {
1760 error = AssembleHeifHdrPicture(mainSptr, sdrIsSRGB, inputImgs);
1761 AssembleDualHdrRefItem(refs);
1762 } else {
1763 IMAGE_LOGE("Picture don't has GAINMAP pixels");
1764 error = ERR_IMAGE_ENCODE_FAILED;
1765 }
1766 break;
1767 case EncodeDynamicRange::HDR_VIVID_SINGLE:
1768 IMAGE_LOGE("Heif picture not support HDR_VIVID_SINGLE");
1769 return ERR_IMAGE_ENCODE_FAILED;
1770 default:
1771 return ERR_IMAGE_INVALID_PARAMETER;
1772 }
1773 bool cond = error != SUCCESS;
1774 CHECK_ERROR_RETURN_RET(cond, error);
1775 if (AssembleExifMetaItem(inputMetas)) {
1776 AssembleExifRefItem(refs);
1777 }
1778 error = AssembleHeifAuxiliaryPicture(inputImgs, refs);
1779 cond = error != SUCCESS;
1780 CHECK_ERROR_RETURN_RET(cond, error);
1781 return DoHeifEncode(inputImgs, inputMetas, refs);
1782 #else
1783 return ERR_IMAGE_INVALID_PARAMETER;
1784 #endif
1785 }
1786
CheckJpegAuxiliaryTagName()1787 void ExtEncoder::CheckJpegAuxiliaryTagName()
1788 {
1789 CHECK_ERROR_RETURN(picture_ == nullptr);
1790 auto auxTypes = ImageUtils::GetAllAuxiliaryPictureType();
1791 for (AuxiliaryPictureType auxType : auxTypes) {
1792 auto auxPicture = picture_->GetAuxiliaryPicture(auxType);
1793 if (auxPicture == nullptr) {
1794 continue;
1795 }
1796 AuxiliaryPictureInfo auxInfo = auxPicture->GetAuxiliaryPictureInfo();
1797 auto iter = DEFAULT_AUXILIARY_TAG_MAP.find(auxType);
1798 if (auxInfo.jpegTagName.size() == 0 && iter != DEFAULT_AUXILIARY_TAG_MAP.end()) {
1799 auxInfo.jpegTagName = iter->second;
1800 auxPicture->SetAuxiliaryPictureInfo(auxInfo);
1801 }
1802 }
1803 }
1804
EncodeJpegPicture(SkWStream & skStream)1805 uint32_t ExtEncoder::EncodeJpegPicture(SkWStream& skStream)
1806 {
1807 uint32_t error = ERR_IMAGE_ENCODE_FAILED;
1808 switch (opts_.desiredDynamicRange) {
1809 case EncodeDynamicRange::AUTO:
1810 error = EncodeJpegPictureDualVivid(skStream);
1811 if (error != SUCCESS) {
1812 IMAGE_LOGI("%{public}s jpeg picture encode dual vivid failed, try encode sdr", __func__);
1813 error = EncodeJpegPictureSdr(skStream);
1814 }
1815 break;
1816 case EncodeDynamicRange::SDR:
1817 error = EncodeJpegPictureSdr(skStream);
1818 break;
1819 case EncodeDynamicRange::HDR_VIVID_DUAL:
1820 error = EncodeJpegPictureDualVivid(skStream);
1821 break;
1822 case EncodeDynamicRange::HDR_VIVID_SINGLE:
1823 error = ERR_IMAGE_DECODE_FAILED;
1824 break;
1825 default:
1826 error = ERR_IMAGE_INVALID_PARAMETER;
1827 break;
1828 }
1829 return error;
1830 }
1831
EncodeJpegPictureDualVividInner(SkWStream & skStream,std::shared_ptr<PixelMap> & mainPixelmap,std::shared_ptr<PixelMap> & gainmapPixelmap)1832 uint32_t ExtEncoder::EncodeJpegPictureDualVividInner(SkWStream& skStream, std::shared_ptr<PixelMap>& mainPixelmap,
1833 std::shared_ptr<PixelMap>& gainmapPixelmap)
1834 {
1835 bool mainIsSRGB = mainPixelmap->GetToSdrColorSpaceIsSRGB();
1836 SkImageInfo baseInfo = GetSkInfo(mainPixelmap.get(), false, mainIsSRGB);
1837 sptr<SurfaceBuffer> baseSptr(reinterpret_cast<SurfaceBuffer*>(mainPixelmap->GetFd()));
1838 VpeUtils::SetSbMetadataType(baseSptr, CM_IMAGE_HDR_VIVID_DUAL);
1839 VpeUtils::SetSbColorSpaceType(baseSptr, mainIsSRGB ? CM_SRGB_FULL : CM_P3_FULL);
1840 pixelmap_ = mainPixelmap.get();
1841 sk_sp<SkData> baseImageData = GetImageEncodeData(baseSptr, baseInfo, opts_.needsPackProperties);
1842
1843 bool gainmapIsSRGB = gainmapPixelmap->GetToSdrColorSpaceIsSRGB();
1844 SkImageInfo gainmapInfo = GetSkInfo(gainmapPixelmap.get(), true, gainmapIsSRGB);
1845 ImageInfo tempInfo;
1846 gainmapPixelmap->GetImageInfo(tempInfo);
1847 gainmapInfo = gainmapInfo.makeWH(tempInfo.size.width, tempInfo.size.height);
1848 sptr<SurfaceBuffer> gainMapSptr(reinterpret_cast<SurfaceBuffer*>(gainmapPixelmap->GetFd()));
1849 VpeUtils::SetSbMetadataType(gainMapSptr, CM_METADATA_NONE);
1850 VpeUtils::SetSbColorSpaceType(gainMapSptr, gainmapIsSRGB ? CM_SRGB_FULL : CM_P3_FULL);
1851 pixelmap_ = gainmapPixelmap.get();
1852 sk_sp<SkData> gainMapImageData = GetImageEncodeData(gainMapSptr, gainmapInfo, false);
1853
1854 HdrMetadata hdrMetadata = GetHdrMetadata(baseSptr, gainMapSptr);
1855 SkDynamicMemoryWStream hdrStream;
1856 uint32_t error = HdrJpegPackerHelper::SpliceHdrStream(baseImageData, gainMapImageData, hdrStream, hdrMetadata);
1857 IMAGE_LOGD("%{public}s splice hdr stream result is: %{public}u", __func__, error);
1858 if (error == SUCCESS) {
1859 sk_sp<SkData> hdrSkData = hdrStream.detachAsData();
1860 skStream.write(hdrSkData->data(), hdrSkData->size());
1861 EncodeJpegAuxiliaryPictures(skStream);
1862 }
1863 return error;
1864 }
1865
EncodeJpegPictureDualVivid(SkWStream & skStream)1866 uint32_t ExtEncoder::EncodeJpegPictureDualVivid(SkWStream& skStream)
1867 {
1868 ImageFuncTimer imageFuncTimer("%s enter", __func__);
1869 bool cond = picture_->HasAuxiliaryPicture(AuxiliaryPictureType::GAINMAP);
1870 CHECK_ERROR_RETURN_RET_LOG(!cond, ERR_IMAGE_INVALID_PARAMETER, "%{public}s no gainmap in picture", __func__);
1871 auto mainPixelmap = picture_->GetMainPixel();
1872 auto gainmapPixelmap = picture_->GetGainmapPixelMap();
1873 cond = !mainPixelmap || !gainmapPixelmap;
1874 CHECK_ERROR_RETURN_RET_LOG(cond, ERR_IMAGE_INVALID_PARAMETER,
1875 "%{public}s mainPixelmap or gainmapPixelmap is null", __func__);
1876 AllocatorType mainAllocType = mainPixelmap->GetAllocatorType();
1877 AllocatorType gainmapAllocType = gainmapPixelmap->GetAllocatorType();
1878 cond = mainAllocType != AllocatorType::DMA_ALLOC || gainmapAllocType != AllocatorType::DMA_ALLOC;
1879 CHECK_ERROR_RETURN_RET_LOG(cond, ERR_IMAGE_ENCODE_FAILED,
1880 "%{public}s AllocatorType is not DMA, mainAllocType: %{public}d, gainmapAllocType: %{public}d",
1881 __func__, mainAllocType, gainmapAllocType);
1882 return EncodeJpegPictureDualVividInner(skStream, mainPixelmap, gainmapPixelmap);
1883 }
1884
EncodeJpegPictureSdr(SkWStream & skStream)1885 uint32_t ExtEncoder::EncodeJpegPictureSdr(SkWStream& skStream)
1886 {
1887 ImageFuncTimer imageFuncTimer("%s enter", __func__);
1888 auto mainPixelmap = picture_->GetMainPixel();
1889 bool cond = !mainPixelmap;
1890 CHECK_ERROR_RETURN_RET_LOG(cond, ERR_IMAGE_INVALID_PARAMETER, "%{public}s mainPixelmap is null", __func__);
1891 pixelmap_ = mainPixelmap.get();
1892 uint32_t error = ERR_IMAGE_ENCODE_FAILED;
1893 if (!mainPixelmap->IsHdr()) {
1894 error = EncodeImageByPixelMap(mainPixelmap.get(), opts_.needsPackProperties, skStream);
1895 IMAGE_LOGD("%{public}s encode sdr picture result is: %{public}u", __func__, error);
1896 } else {
1897 cond = mainPixelmap->GetAllocatorType() != AllocatorType::DMA_ALLOC;
1898 CHECK_ERROR_RETURN_RET_LOG(cond, ERR_IMAGE_INVALID_PARAMETER, "pixelmap is 10bit, but not dma buffer");
1899 bool mainIsSRGB = mainPixelmap->GetToSdrColorSpaceIsSRGB();
1900 SkImageInfo baseInfo = GetSkInfo(mainPixelmap.get(), false, mainIsSRGB);
1901 sptr<SurfaceBuffer> baseSptr(reinterpret_cast<SurfaceBuffer*>(mainPixelmap->GetFd()));
1902 VpeUtils::SetSbMetadataType(baseSptr, CM_IMAGE_HDR_VIVID_DUAL);
1903 VpeUtils::SetSbColorSpaceType(baseSptr, mainIsSRGB ? CM_SRGB_FULL : CM_P3_FULL);
1904 error = EncodeImageBySurfaceBuffer(baseSptr, baseInfo, opts_.needsPackProperties, skStream);
1905 IMAGE_LOGD("%{public}s encode hdr picture result is: %{public}u", __func__, error);
1906 }
1907 if (error == SUCCESS) {
1908 EncodeJpegAuxiliaryPictures(skStream);
1909 }
1910 return error;
1911 }
1912
GetMetadataValueInt32(const std::shared_ptr<ImageMetadata> & metadata,const std::string & key,int32_t & outVal)1913 static bool GetMetadataValueInt32(const std::shared_ptr<ImageMetadata>& metadata, const std::string& key,
1914 int32_t& outVal)
1915 {
1916 std::string strVal("");
1917 uint32_t u32Val = 0;
1918 bool cond = metadata == nullptr;
1919 CHECK_ERROR_RETURN_RET_LOG(cond, false, "%{public}s: metadata is nullptr!", __func__);
1920 cond = metadata->GetValue(key, strVal) != SUCCESS || !ImageUtils::StrToUint32(strVal, u32Val);
1921 CHECK_ERROR_RETURN_RET_LOG(cond, false,
1922 "%{public}s: get metadata key[%{public}s]'s value failed!", __func__, key.c_str());
1923 outVal = static_cast<int32_t>(u32Val);
1924 return true;
1925 }
1926
CheckFragmentMetadata(Picture * picture,Media::Rect & outData)1927 static bool CheckFragmentMetadata(Picture* picture, Media::Rect& outData)
1928 {
1929 bool cond = !picture || !picture->GetMainPixel() ||
1930 !picture->HasAuxiliaryPicture(AuxiliaryPictureType::FRAGMENT_MAP);
1931 CHECK_ERROR_RETURN_RET_LOG(cond, false,
1932 "%{public}s: picture or mainPixelMap or fragment picture does not exist!", __func__);
1933 int32_t mainW = picture->GetMainPixel()->GetWidth();
1934 int32_t mainH = picture->GetMainPixel()->GetHeight();
1935
1936 auto fragmentPicture = picture->GetAuxiliaryPicture(AuxiliaryPictureType::FRAGMENT_MAP);
1937 auto fragmentMetadata = fragmentPicture->GetMetadata(MetadataType::FRAGMENT);
1938 cond = fragmentMetadata == nullptr;
1939 CHECK_ERROR_RETURN_RET_LOG(cond, false, "%{public}s: fragmentMetadata is nullptr!", __func__);
1940 cond = !GetMetadataValueInt32(fragmentMetadata, FRAGMENT_METADATA_KEY_X, outData.left) ||
1941 !GetMetadataValueInt32(fragmentMetadata, FRAGMENT_METADATA_KEY_Y, outData.top) ||
1942 !GetMetadataValueInt32(fragmentMetadata, FRAGMENT_METADATA_KEY_WIDTH, outData.width) ||
1943 !GetMetadataValueInt32(fragmentMetadata, FRAGMENT_METADATA_KEY_HEIGHT, outData.height);
1944 CHECK_ERROR_RETURN_RET_LOG(cond, false, "%{public}s: GetMetadataValueInt32 failed!", __func__);
1945 if (!ImageUtils::IsInRange(outData.left, 0, mainW) || !ImageUtils::IsInRange(outData.top, 0, mainH) ||
1946 !ImageUtils::IsInRange(outData.width, 0, mainW) || !ImageUtils::IsInRange(outData.height, 0, mainH) ||
1947 !ImageUtils::IsInRange(outData.left + outData.width, 0, mainW) ||
1948 !ImageUtils::IsInRange(outData.top + outData.height, 0, mainH)) {
1949 IMAGE_LOGW("%{public}s: Fragment Rect is not in main Rect!", __func__);
1950 return false;
1951 }
1952 return true;
1953 }
1954
EncodeJpegAuxiliaryPictures(SkWStream & skStream)1955 void ExtEncoder::EncodeJpegAuxiliaryPictures(SkWStream& skStream)
1956 {
1957 ImageFuncTimer imageFuncTimer("%s enter", __func__);
1958 auto auxTypes = ImageUtils::GetAllAuxiliaryPictureType();
1959 for (AuxiliaryPictureType auxType : auxTypes) {
1960 auto auxPicture = picture_->GetAuxiliaryPicture(auxType);
1961 // Gainmap has been encoded before
1962 if (auxPicture == nullptr || auxType == AuxiliaryPictureType::GAINMAP) {
1963 continue;
1964 }
1965 IMAGE_LOGI("%{public}s try to encode auxiliary picture type: %{public}d", __func__, auxType);
1966 uint32_t error = ERR_IMAGE_ENCODE_FAILED;
1967 size_t writtenSize = skStream.bytesWritten();
1968 if (ImageUtils::IsAuxiliaryPictureEncoded(auxType)) {
1969 error = WriteJpegCodedData(auxPicture, skStream);
1970 } else {
1971 error = WriteJpegUncodedData(auxPicture, skStream);
1972 }
1973 if (error != SUCCESS) {
1974 IMAGE_LOGE("%{public}s encode auxiliary picture type [%{public}d] failed, error: %{public}u",
1975 __func__, auxType, error);
1976 } else {
1977 uint32_t currentDataSize = static_cast<uint32_t>(skStream.bytesWritten() - writtenSize);
1978 WriteJpegAuxiliarySizeAndTag(currentDataSize, auxPicture, skStream);
1979 }
1980 }
1981 }
1982
ConvertBufferToData(sptr<SurfaceBuffer> & surfaceBuffer,std::vector<uint8_t> & pixels)1983 bool ConvertBufferToData(sptr<SurfaceBuffer>& surfaceBuffer, std::vector<uint8_t>& pixels)
1984 {
1985 if (surfaceBuffer == nullptr) {
1986 return false;
1987 }
1988 uint32_t width = static_cast<uint32_t>(surfaceBuffer->GetWidth());
1989 uint32_t height = static_cast<uint32_t>(surfaceBuffer->GetHeight());
1990 uint32_t stride = static_cast<uint32_t>(surfaceBuffer->GetStride());
1991 uint32_t dstStride = width * RGBA8888_PIXEL_BYTES;
1992 uint32_t bufferSize = height * dstStride;
1993 uint8_t *srcBuf = static_cast<uint8_t *>(surfaceBuffer->GetVirAddr());
1994 pixels.resize(bufferSize);
1995 uint8_t *dstBuf = pixels.data();
1996 for (uint32_t i = 0; i < height; ++i) {
1997 if (memcpy_s(dstBuf + i * dstStride, dstStride, srcBuf + i * stride, dstStride) != EOK) {
1998 IMAGE_LOGE("HDR-IMAGE ConvertBufferToData copy memory failed");
1999 return false;
2000 }
2001 }
2002 return true;
2003 }
2004
SetLogResMapColorSpaceType(CM_ColorSpaceType & colorSpaceType,LogResMapMetadata & colorData)2005 void SetLogResMapColorSpaceType(CM_ColorSpaceType& colorSpaceType, LogResMapMetadata& colorData)
2006 {
2007 uint32_t colorSpace = static_cast<uint32_t>(colorSpaceType);
2008 colorData.primaries = static_cast<CM_ColorPrimaries>(colorSpace & CM_PRIMARIES_MASK);
2009 colorData.transfunc = static_cast<CM_TransFunc>((colorSpace & CM_TRANSFUNC_MASK) >> TRANSFUNC_OFFSET);
2010 colorData.matrix = static_cast<CM_Matrix>((colorSpace & CM_MATRIX_MASK) >> MATRIX_OFFSET);
2011 colorData.range = static_cast<CM_Range>((colorSpace & CM_RANGE_MASK) >> RANGE_OFFSET);
2012 }
2013
EncodeLogVideoDataToBlobMetadata(sptr<SurfaceBuffer> & surfaceBuffer,SkWStream & skStream)2014 void ExtEncoder::EncodeLogVideoDataToBlobMetadata(sptr<SurfaceBuffer>& surfaceBuffer, SkWStream& skStream)
2015 {
2016 // write version
2017 const char version[] = RFIMAGE_ID;
2018 std::string versionTag(version, sizeof(version) - 1);
2019 std::vector<uint8_t> versionDataTag(versionTag.begin(), versionTag.end());
2020 skStream.write(versionDataTag.data(), versionDataTag.size());
2021 // write colors
2022 LogResMapMetadata colorData;
2023 CM_ColorSpaceType colorSpaceType;
2024 VpeUtils::GetSbColorSpaceType(surfaceBuffer, colorSpaceType);
2025 SetLogResMapColorSpaceType(colorSpaceType, colorData);
2026 colorData.width = static_cast<uint16_t>(surfaceBuffer->GetWidth());
2027 colorData.height = static_cast<uint16_t>(surfaceBuffer->GetHeight());
2028 colorData.pixelFormat = static_cast<uint16_t>(surfaceBuffer->GetFormat());
2029 if (surfaceBuffer->GetFormat() != GRAPHIC_PIXEL_FMT_RGBA_8888) {
2030 IMAGE_LOGE("HDR-IMAGE encode res-map unsupported format");
2031 return;
2032 }
2033 size_t colorDataSize = sizeof(colorData);
2034 skStream.write(&colorData, colorDataSize);
2035 // write pixels
2036 std::vector<uint8_t> pixels;
2037 if (!ConvertBufferToData(surfaceBuffer, pixels)) {
2038 return;
2039 }
2040 size_t pixelSize = pixels.size();
2041 skStream.write(pixels.data(), pixelSize);
2042 // write size
2043 size_t writeSize = versionDataTag.size() + colorDataSize + pixelSize;
2044 IMAGE_LOGI("HDR-IMAGE try to encode Log-video metadata, datasize %{public}zu", writeSize);
2045 std::vector<uint8_t> dataSize = JpegMpfPacker::PackDataSize(writeSize, false);
2046 skStream.write(dataSize.data(), dataSize.size());
2047 // write tag
2048 const char rawTagName[] = "Res-Map\0";
2049 std::string tagName(rawTagName, sizeof(rawTagName) - 1);
2050 std::vector<uint8_t> dataTag(tagName.begin(), tagName.end());
2051 skStream.write(dataTag.data(), dataTag.size());
2052 }
2053
WriteJpegCodedData(std::shared_ptr<AuxiliaryPicture> & auxPicture,SkWStream & skStream)2054 uint32_t ExtEncoder::WriteJpegCodedData(std::shared_ptr<AuxiliaryPicture>& auxPicture, SkWStream& skStream)
2055 {
2056 auto pixelMap = auxPicture->GetContentPixel();
2057 bool cond = pixelMap == nullptr;
2058 CHECK_ERROR_RETURN_RET(cond, ERR_IMAGE_DATA_ABNORMAL);
2059 pixelmap_ = pixelMap.get();
2060 sk_sp<SkData> skData = nullptr;
2061 if (pixelMap->GetAllocatorType() != AllocatorType::DMA_ALLOC || pixelMap->GetFd() == nullptr) {
2062 SkDynamicMemoryWStream nonDMAStream;
2063 uint32_t error = EncodeImageByPixelMap(pixelmap_, false, nonDMAStream);
2064 IMAGE_LOGD("%{public}s EncodeImageByPixelMap result: %{public}u", __func__, error);
2065 skData = nonDMAStream.detachAsData();
2066 } else {
2067 sptr<SurfaceBuffer> auxSptr(reinterpret_cast<SurfaceBuffer*>(pixelMap->GetFd()));
2068 bool isSRGB = pixelMap->GetToSdrColorSpaceIsSRGB();
2069 SkImageInfo skInfo = GetSkInfo(pixelmap_, false, isSRGB);
2070 skData = GetImageEncodeData(auxSptr, skInfo, false);
2071 }
2072 cond = skData == nullptr;
2073 CHECK_ERROR_RETURN_RET(cond, ERR_IMAGE_ENCODE_FAILED);
2074 cond = auxPicture->GetType() == AuxiliaryPictureType::FRAGMENT_MAP;
2075 CHECK_ERROR_RETURN_RET(cond, SpliceFragmentStream(skStream, skData));
2076 skStream.write(skData->data(), skData->size());
2077 return SUCCESS;
2078 }
2079
SpliceFragmentStream(SkWStream & skStream,sk_sp<SkData> & skData)2080 uint32_t ExtEncoder::SpliceFragmentStream(SkWStream& skStream, sk_sp<SkData>& skData)
2081 {
2082 Media::Rect fragmentMetadata;
2083 bool cond = !CheckFragmentMetadata(picture_, fragmentMetadata);
2084 CHECK_ERROR_RETURN_RET_LOG(cond, ERR_IMAGE_PROPERTY_NOT_EXIST,
2085 "%{public}s: CheckFragmentMetadata failed!", __func__);
2086 const uint8_t* dataBytes = reinterpret_cast<const uint8_t*>(skData->data());
2087 // write JPEG SOI(0xFFD8)
2088 skStream.write(dataBytes, JPEG_MARKER_TAG_SIZE);
2089 std::vector<uint8_t> packedFragmentMetadata = JpegMpfPacker::PackFragmentMetadata(fragmentMetadata);
2090 // write fragment metadata
2091 skStream.write(packedFragmentMetadata.data(), packedFragmentMetadata.size());
2092 // write fragment auxiliary image data
2093 skStream.write(dataBytes + JPEG_MARKER_TAG_SIZE, skData->size() - JPEG_MARKER_TAG_SIZE);
2094 return SUCCESS;
2095 }
2096
WriteJpegUncodedData(std::shared_ptr<AuxiliaryPicture> & auxPicture,SkWStream & skStream)2097 uint32_t ExtEncoder::WriteJpegUncodedData(std::shared_ptr<AuxiliaryPicture>& auxPicture, SkWStream& skStream)
2098 {
2099 auto pixelMap = auxPicture->GetContentPixel();
2100 bool cond = pixelMap == nullptr;
2101 CHECK_ERROR_RETURN_RET(cond, ERR_IMAGE_DATA_ABNORMAL);
2102 void* bytes = nullptr;
2103 uint32_t size = 0;
2104 Size imageSize;
2105 if (pixelMap->GetAllocatorType() != AllocatorType::DMA_ALLOC || pixelMap->GetFd() == nullptr) {
2106 bytes = pixelMap->GetWritablePixels();
2107 size = pixelMap->GetCapacity();
2108 imageSize.width = pixelMap->GetWidth();
2109 imageSize.height = pixelMap->GetHeight();
2110 } else {
2111 auto surfaceBuffer = reinterpret_cast<SurfaceBuffer*>(pixelMap->GetFd());
2112 bytes = surfaceBuffer->GetVirAddr();
2113 size = surfaceBuffer->GetSize();
2114 imageSize.width = surfaceBuffer->GetWidth();
2115 imageSize.height = surfaceBuffer->GetHeight();
2116 }
2117 uint32_t writeSize = (uint32_t)imageSize.width * (uint32_t)imageSize.height;
2118 AuxiliaryPictureInfo auxInfo = auxPicture->GetAuxiliaryPictureInfo();
2119 if (auxInfo.auxiliaryPictureType == AuxiliaryPictureType::DEPTH_MAP) {
2120 writeSize *= DEPTH_MAP_BYTES;
2121 } else if (auxInfo.auxiliaryPictureType == AuxiliaryPictureType::LINEAR_MAP) {
2122 writeSize *= LINEAR_MAP_BYTES;
2123 }
2124 IMAGE_LOGD("%{public}s auxType: %{public}d, width: %{public}d, height: %{public}d, buffer size: %{public}u,"
2125 " write size: %{public}u", __func__, auxInfo.auxiliaryPictureType, imageSize.width, imageSize.height,
2126 size, writeSize);
2127 skStream.write(bytes, std::min(size, writeSize));
2128 return SUCCESS;
2129 }
2130
WriteJpegAuxiliarySizeAndTag(uint32_t size,std::shared_ptr<AuxiliaryPicture> & auxPicture,SkWStream & skStream)2131 void ExtEncoder::WriteJpegAuxiliarySizeAndTag(uint32_t size, std::shared_ptr<AuxiliaryPicture>& auxPicture,
2132 SkWStream& skStream)
2133 {
2134 // Write auxiliary image size(little endian)
2135 std::vector<uint8_t> auxSize = JpegMpfPacker::PackDataSize(size, false);
2136 skStream.write(auxSize.data(), auxSize.size());
2137
2138 // Write auxiliary tag name
2139 AuxiliaryPictureInfo auxInfo = auxPicture->GetAuxiliaryPictureInfo();
2140 std::vector<uint8_t> tagName = JpegMpfPacker::PackAuxiliaryTagName(auxInfo.jpegTagName);
2141 skStream.write(tagName.data(), tagName.size());
2142 }
2143 #endif
2144
2145 #ifdef HEIF_HW_ENCODE_ENABLE
GetBufferSize(size_t contentSize)2146 static size_t GetBufferSize(size_t contentSize)
2147 {
2148 return sizeof(PropertyType) + sizeof(int) + contentSize;
2149 }
2150
2151 // format: enum + int buffersize + buffer
FillImagePropertyItem(const std::shared_ptr<AbsMemory> & mem,const size_t offset,PropertyType type,const void * payloadData,size_t payloadSize)2152 static bool FillImagePropertyItem(const std::shared_ptr<AbsMemory> &mem, const size_t offset,
2153 PropertyType type, const void* payloadData, size_t payloadSize)
2154 {
2155 uint8_t* memData = reinterpret_cast<uint8_t*>(mem->data.data);
2156 size_t memSize = mem->data.size;
2157 IMAGE_LOGD("FileImagePropertyItem memSize is %{public}ld, payloadSize is %{public}ld", memSize, payloadSize);
2158 IMAGE_LOGD("FileImagePropertyItem sizeof(type) is %{public}ld, sizeof(payloadSize) %{public}ld",
2159 sizeof(type), sizeof(payloadSize));
2160 bool cond = payloadSize > INT32_MAX;
2161 CHECK_INFO_RETURN_RET_LOG(cond, false, "payloadSize is over INT32_MAX");
2162 int payloadIntSize = static_cast<int>(payloadSize);
2163 size_t typeSize = sizeof(type);
2164 size_t intSize = sizeof(int);
2165 bool res = (memcpy_s(memData + offset, memSize - offset, &type, typeSize) == EOK) &&
2166 (memcpy_s(memData + offset + typeSize, memSize - offset - typeSize, &payloadIntSize, intSize) == EOK) &&
2167 (memcpy_s(memData + offset + typeSize + intSize, memSize - offset - typeSize - intSize,
2168 payloadData, payloadSize) == EOK);
2169 return res;
2170 }
2171
AllocateNewSharedMem(size_t memorySize,std::string tag)2172 std::shared_ptr<AbsMemory> ExtEncoder::AllocateNewSharedMem(size_t memorySize, std::string tag)
2173 {
2174 MemoryData memoryData;
2175 memoryData.size = memorySize;
2176 memoryData.tag = tag.empty() ? DEFAULT_ASHMEM_TAG.c_str() : tag.c_str();
2177 std::unique_ptr<AbsMemory> memory =
2178 MemoryManager::CreateMemory(AllocatorType::SHARE_MEM_ALLOC, memoryData);
2179 std::shared_ptr<AbsMemory> res = std::move(memory);
2180 return res;
2181 }
2182
GetStaticMetadata(HdrMetadata & metadata,MasteringDisplayColourVolume & color,ContentLightLevel & light)2183 bool ExtEncoder::GetStaticMetadata(HdrMetadata& metadata, MasteringDisplayColourVolume& color, ContentLightLevel& light)
2184 {
2185 HdrStaticMetadata staticMetadata;
2186 size_t staticMetadataSize = sizeof(HdrStaticMetadata);
2187 bool cond = metadata.staticMetadata.size() < staticMetadataSize;
2188 CHECK_INFO_RETURN_RET_LOG(cond, false, "GetStaticMetadata failed");
2189 cond = memcpy_s(&staticMetadata, staticMetadataSize,
2190 metadata.staticMetadata.data(), metadata.staticMetadata.size()) != EOK;
2191 CHECK_INFO_RETURN_RET_LOG(cond, false, "GetStaticMetadata failed, memcpy_s failed");
2192 color.displayPrimariesRX =
2193 (uint16_t)(STATIC_METADATA_COLOR_SCALE * staticMetadata.smpte2086.displayPrimaryRed.x);
2194 color.displayPrimariesRY =
2195 (uint16_t)(STATIC_METADATA_COLOR_SCALE * staticMetadata.smpte2086.displayPrimaryRed.y);
2196 color.displayPrimariesGX =
2197 (uint16_t)(STATIC_METADATA_COLOR_SCALE * staticMetadata.smpte2086.displayPrimaryGreen.x);
2198 color.displayPrimariesGY =
2199 (uint16_t)(STATIC_METADATA_COLOR_SCALE * staticMetadata.smpte2086.displayPrimaryGreen.y);
2200 color.displayPrimariesBX =
2201 (uint16_t)(STATIC_METADATA_COLOR_SCALE * staticMetadata.smpte2086.displayPrimaryBlue.x);
2202 color.displayPrimariesBY =
2203 (uint16_t)(STATIC_METADATA_COLOR_SCALE * staticMetadata.smpte2086.displayPrimaryBlue.y);
2204 color.whitePointX = (uint16_t)(STATIC_METADATA_COLOR_SCALE * staticMetadata.smpte2086.whitePoint.x);
2205 color.whitePointY = (uint16_t)(STATIC_METADATA_COLOR_SCALE * staticMetadata.smpte2086.whitePoint.y);
2206 color.maxDisplayMasteringLuminance = (uint32_t)staticMetadata.smpte2086.maxLuminance;
2207 color.minDisplayMasteringLuminance = (uint32_t)(STATIC_METADATA_LUM_SCALE * staticMetadata.smpte2086.minLuminance);
2208 light.maxContentLightLevel = (uint16_t)staticMetadata.cta861.maxContentLightLevel;
2209 light.maxPicAverageLightLevel = (uint16_t)staticMetadata.cta861.maxFrameAverageLightLevel;
2210 return true;
2211 }
2212
GetToneMapChannel(ISOMetadata & metadata,ToneMapChannel & channel,uint8_t index)2213 bool ExtEncoder::GetToneMapChannel(ISOMetadata& metadata, ToneMapChannel& channel, uint8_t index)
2214 {
2215 bool cond = index > GAINMAP_CHANNEL_MULTI - 1;
2216 CHECK_ERROR_RETURN_RET_LOG(cond, false, "GetToneMapChannel index:[%{public}d] unsupported", index);
2217 channel.gainMapMin.numerator = DEFAULT_DENOMINATOR * metadata.enhanceClippedThreholdMinGainmap[index];
2218 channel.gainMapMin.denominator = DEFAULT_DENOMINATOR;
2219 channel.gainMapMax.numerator = DEFAULT_DENOMINATOR * metadata.enhanceClippedThreholdMaxGainmap[index];
2220 channel.gainMapMax.denominator = DEFAULT_DENOMINATOR;
2221 if (metadata.enhanceMappingGamma[index] > 0.0f) {
2222 channel.gamma.numerator = DEFAULT_DENOMINATOR * metadata.enhanceMappingGamma[index];
2223 } else {
2224 channel.gamma.numerator = 0;
2225 }
2226 channel.gamma.denominator = DEFAULT_DENOMINATOR;
2227 channel.baseOffset.numerator = DEFAULT_DENOMINATOR * metadata.enhanceMappingBaselineOffset[index];
2228 channel.baseOffset.denominator = DEFAULT_DENOMINATOR;
2229 channel.alternateOffset.numerator = DEFAULT_DENOMINATOR * metadata.enhanceMappingAlternateOffset[index];
2230 channel.alternateOffset.denominator = DEFAULT_DENOMINATOR;
2231 IMAGE_LOGD("GetToneMapChannel [%{public}d],gainmapMin:%{public}f, gainmapMax:%{public}f,"
2232 "gamma:%{public}f,baseOffset:%{public}f, alternateOffset:%{public}f", index,
2233 metadata.enhanceClippedThreholdMinGainmap[index], metadata.enhanceClippedThreholdMaxGainmap[index],
2234 metadata.enhanceMappingGamma[index],
2235 metadata.enhanceMappingBaselineOffset[index], metadata.enhanceMappingAlternateOffset[index]);
2236 return true;
2237 }
2238
GetToneMapMetadata(HdrMetadata & metadata,ToneMapMetadata & toneMapMetadata)2239 bool ExtEncoder::GetToneMapMetadata(HdrMetadata& metadata, ToneMapMetadata& toneMapMetadata)
2240 {
2241 ISOMetadata isoMeta = metadata.extendMeta.metaISO;
2242 toneMapMetadata.useBaseColorSpace = (isoMeta.useBaseColorFlag == 1) ? true : false;
2243 toneMapMetadata.channelCnt = (isoMeta.gainmapChannelNum == 0) ? GAINMAP_CHANNEL_SINGLE : GAINMAP_CHANNEL_MULTI;
2244 toneMapMetadata.baseHdrHeadroom.denominator = DEFAULT_DENOMINATOR;
2245 if (isoMeta.baseHeadroom > 0.0f) {
2246 toneMapMetadata.baseHdrHeadroom.numerator = (uint32_t)(isoMeta.baseHeadroom * DEFAULT_DENOMINATOR);
2247 } else {
2248 toneMapMetadata.baseHdrHeadroom.numerator = 0;
2249 }
2250 toneMapMetadata.alternateHdrHeadroom.denominator = DEFAULT_DENOMINATOR;
2251 if (isoMeta.alternateHeadroom > 0.0f) {
2252 toneMapMetadata.alternateHdrHeadroom.numerator = (uint32_t)(isoMeta.alternateHeadroom * DEFAULT_DENOMINATOR);
2253 } else {
2254 toneMapMetadata.alternateHdrHeadroom.numerator = 0;
2255 }
2256 GetToneMapChannel(isoMeta, toneMapMetadata.channels1, INDEX_ZERO);
2257 if (toneMapMetadata.channelCnt == GAINMAP_CHANNEL_MULTI) {
2258 GetToneMapChannel(isoMeta, toneMapMetadata.channels2, INDEX_ONE);
2259 GetToneMapChannel(isoMeta, toneMapMetadata.channels3, INDEX_TWO);
2260 }
2261 IMAGE_LOGD("GetToneMapMetadata useBaseColorSpace:%{public}d, gainmapChannelNum:%{public}d,"
2262 "baseHeadroom:%{public}f,alternateHeadroom:%{public}f", isoMeta.useBaseColorFlag, isoMeta.gainmapChannelNum,
2263 isoMeta.baseHeadroom, isoMeta.alternateHeadroom);
2264 return true;
2265 }
2266
GetColourInfo(ColorManager::ColorSpaceName color,ColourInfo & info)2267 void ExtEncoder::GetColourInfo(ColorManager::ColorSpaceName color, ColourInfo& info)
2268 {
2269 uint8_t fullRangeFlag;
2270 ColorUtils::ColorSpaceGetCicp(color, info.colourPrimaries, info.transferCharacteristics, info.matrixCoefficients,
2271 fullRangeFlag);
2272 // 1 : full range ; 0 : limit range.
2273 info.fullRangeFlag = (fullRangeFlag == 1);
2274 }
2275
AssembleIT35SharedBuffer(HdrMetadata metadata,SharedBuffer & outBuffer)2276 bool ExtEncoder::AssembleIT35SharedBuffer(HdrMetadata metadata, SharedBuffer& outBuffer)
2277 {
2278 std::vector<uint8_t> it35Info;
2279 bool cond = !HdrHeifPackerHelper::PackIT35Info(metadata, it35Info);
2280 CHECK_ERROR_RETURN_RET_LOG(cond, false, "get it35 info failed");
2281 std::shared_ptr<AbsMemory> propertyAshmem =
2282 AllocateNewSharedMem(GetBufferSize(it35Info.size()), IT35_ASHMEM_TAG);
2283 cond = propertyAshmem == nullptr;
2284 CHECK_ERROR_RETURN_RET_LOG(cond, false, "AssembleIT35SharedBuffer it35 alloc failed");
2285 tmpMemoryList_.push_back(propertyAshmem);
2286 cond = !FillImagePropertyItem(propertyAshmem, 0, PropertyType::IT35_INFO, it35Info.data(), it35Info.size());
2287 CHECK_ERROR_RETURN_RET_LOG(cond, false, "AssembleIT35SharedBuffer fill failed");
2288 outBuffer.fd = *static_cast<int *>(propertyAshmem->extend.data);
2289 outBuffer.capacity = propertyAshmem->data.size;
2290 outBuffer.filledLen = propertyAshmem->data.size;
2291 return true;
2292 }
2293
AssembleICCImageProperty(sk_sp<SkData> & iccProfile,SharedBuffer & outBuffer)2294 bool ExtEncoder::AssembleICCImageProperty(sk_sp<SkData>& iccProfile, SharedBuffer& outBuffer)
2295 {
2296 bool cond = iccProfile == nullptr || iccProfile->size() == 0;
2297 CHECK_INFO_RETURN_RET_LOG(cond, false, "AssembleICCImageProperty iccprofile is nullptr");
2298 std::shared_ptr<AbsMemory> propertyAshmem =
2299 AllocateNewSharedMem(GetBufferSize(iccProfile->size()), ICC_ASHMEM_TAG);
2300 cond = propertyAshmem == nullptr;
2301 CHECK_ERROR_RETURN_RET_LOG(cond, false, "AssembleICCImageProperty alloc failed");
2302 tmpMemoryList_.push_back(propertyAshmem);
2303 bool fillRes = FillImagePropertyItem(propertyAshmem, 0, PropertyType::ICC_PROFILE,
2304 iccProfile->data(), iccProfile->size());
2305 if (fillRes) {
2306 outBuffer.fd = *static_cast<int *>(propertyAshmem->extend.data);
2307 outBuffer.capacity = propertyAshmem->data.size;
2308 outBuffer.filledLen = propertyAshmem->data.size;
2309 }
2310 return fillRes;
2311 }
2312
FillNclxColorProperty(std::shared_ptr<ImageItem> & item,size_t & offset,ColourInfo & colorInfo)2313 bool ExtEncoder::FillNclxColorProperty(std::shared_ptr<ImageItem>& item, size_t& offset, ColourInfo& colorInfo)
2314 {
2315 ColorType colorType = ColorType::NCLX;
2316 bool cond = !FillLitePropertyItem(item->liteProperties, offset,
2317 PropertyType::COLOR_TYPE, &colorType, sizeof(ColorType));
2318 CHECK_ERROR_RETURN_RET_LOG(cond, false, "Fill colorType failed");
2319 cond = !FillLitePropertyItem(item->liteProperties, offset,
2320 PropertyType::COLOR_INFO, &colorInfo, sizeof(ColourInfo));
2321 CHECK_INFO_RETURN_RET_LOG(cond, false, "Fill colorInfo failed");
2322 return true;
2323 }
2324
AssembleOutputSharedBuffer(SharedBuffer & outBuffer,std::shared_ptr<AbsMemory> & outMem)2325 bool ExtEncoder::AssembleOutputSharedBuffer(SharedBuffer& outBuffer, std::shared_ptr<AbsMemory>& outMem)
2326 {
2327 bool cond = output_ == nullptr;
2328 CHECK_ERROR_RETURN_RET_LOG(cond, false, "AssembleOutputSharedBuffer output_ is nullptr");
2329 OutputStreamType outType = output_->GetType();
2330 size_t outputCapacity = DEFAULT_OUTPUT_SIZE;
2331 if (outType != OutputStreamType::FILE_PACKER) {
2332 output_->GetCapicity(outputCapacity);
2333 }
2334 std::shared_ptr<AbsMemory> mem = AllocateNewSharedMem(outputCapacity, OUTPUT_ASHMEM_TAG);
2335 cond = mem == nullptr;
2336 CHECK_ERROR_RETURN_RET_LOG(cond, false, "AssembleOutputSharedBuffer alloc out sharemem failed");
2337 outMem = mem;
2338 tmpMemoryList_.push_back(mem);
2339 if (mem->extend.data != nullptr) {
2340 outBuffer.fd = *static_cast<int *>(mem->extend.data);
2341 }
2342 outBuffer.capacity = outputCapacity;
2343 return true;
2344 }
2345
AssembleDualHdrRefItem(std::vector<ItemRef> & refs)2346 void ExtEncoder::AssembleDualHdrRefItem(std::vector<ItemRef>& refs)
2347 {
2348 auto item = std::make_shared<ItemRef>();
2349 item->type = ReferenceType::DIMG;
2350 item->from = TMAP_IMAGE_ITEM_ID;
2351 item->to.resize(INDEX_TWO);
2352 item->to[INDEX_ZERO] = PRIMARY_IMAGE_ITEM_ID;
2353 item->to[INDEX_ONE] = GAINMAP_IMAGE_ITEM_ID;
2354 refs.push_back(*item);
2355 }
2356
DoHeifEncode(std::vector<ImageItem> & inputImgs,std::vector<MetaItem> & inputMetas,std::vector<ItemRef> & refs)2357 uint32_t ExtEncoder::DoHeifEncode(std::vector<ImageItem>& inputImgs, std::vector<MetaItem>& inputMetas,
2358 std::vector<ItemRef>& refs)
2359 {
2360 SharedBuffer outputBuffer {};
2361 std::shared_ptr<AbsMemory> outputAshmem;
2362 bool tempRes = AssembleOutputSharedBuffer(outputBuffer, outputAshmem);
2363 bool cond = !tempRes;
2364 CHECK_ERROR_RETURN_RET_LOG(cond, ERR_IMAGE_INVALID_PARAMETER,
2365 "ExtEncoder::DoHeifEncode alloc sharedbuffer failed");
2366
2367 sptr<ICodecImage> codec = GetCodecManager();
2368 CHECK_ERROR_RETURN_RET(codec == nullptr, ERR_IMAGE_ENCODE_FAILED);
2369 uint32_t outSize = DEFAULT_OUTPUT_SIZE;
2370 int32_t encodeRes = codec->DoHeifEncode(inputImgs, inputMetas, refs, outputBuffer, outSize);
2371 cond = encodeRes != HDF_SUCCESS;
2372 CHECK_ERROR_RETURN_RET_LOG(cond, ERR_IMAGE_ENCODE_FAILED, "ExtEncoder::DoHeifEncode DoHeifEncode failed");
2373 IMAGE_LOGI("ExtEncoder::DoHeifEncode output type is %{public}d", output_->GetType());
2374 bool writeRes = output_->Write(reinterpret_cast<uint8_t *>(outputAshmem->data.data), outSize);
2375 cond = !writeRes;
2376 CHECK_ERROR_RETURN_RET_LOG(cond, ERR_IMAGE_ENCODE_FAILED, "ExtEncoder::DoHeifEncode Write failed");
2377 return SUCCESS;
2378 }
2379
AssembleTmapImageItem(ColorManager::ColorSpaceName color,HdrMetadata metadata,const PlEncodeOptions & opts)2380 std::shared_ptr<ImageItem> ExtEncoder::AssembleTmapImageItem(ColorManager::ColorSpaceName color,
2381 HdrMetadata metadata, const PlEncodeOptions &opts)
2382 {
2383 auto item = std::make_shared<ImageItem>();
2384 item->id = TMAP_IMAGE_ITEM_ID;
2385 item->itemName = "Tone-mapped representation";
2386 item->isPrimary = false;
2387 item->isHidden = false;
2388 item->compressType = COMPRESS_TYPE_TMAP;
2389 item->quality = opts.quality;
2390 item->sharedProperties.fd = -1;
2391 item->pixelSharedBuffer.fd = -1;
2392 ColourInfo colorInfo;
2393 GetColourInfo(color, colorInfo);
2394 ContentLightLevel light;
2395 MasteringDisplayColourVolume colour;
2396 bool hasLight = GetStaticMetadata(metadata, colour, light);
2397 uint32_t propertiesSize = 0;
2398 propertiesSize +=
2399 (sizeof(PropertyType::COLOR_TYPE) + sizeof(ColorType) + sizeof(PropertyType::COLOR_INFO) + sizeof(ColourInfo));
2400 if (hasLight) {
2401 propertiesSize += (sizeof(PropertyType::CONTENT_LIGHT_LEVEL) + sizeof(ContentLightLevel));
2402 }
2403 ToneMapMetadata toneMap;
2404 bool hasToneMap = GetToneMapMetadata(metadata, toneMap);
2405 if (hasToneMap) {
2406 propertiesSize += (sizeof(PropertyType::TONE_MAP_METADATA) + sizeof(ToneMapMetadata));
2407 }
2408 item->liteProperties.resize(propertiesSize);
2409 size_t offset = 0;
2410 if (!FillNclxColorProperty(item, offset, colorInfo)) {
2411 return nullptr;
2412 }
2413 bool cond = hasLight && (!FillLitePropertyItem(item->liteProperties, offset,
2414 PropertyType::CONTENT_LIGHT_LEVEL, &colour, sizeof(ContentLightLevel)));
2415 CHECK_ERROR_RETURN_RET_LOG(cond, nullptr, "AssembleTmapImageItem fill CONTENT_LIGHT_LEVEL failed");
2416 cond = hasToneMap && (!FillLitePropertyItem(item->liteProperties, offset,
2417 PropertyType::TONE_MAP_METADATA, &toneMap, sizeof(ToneMapMetadata)));
2418 CHECK_ERROR_RETURN_RET_LOG(cond, nullptr, "AssembleTmapImageItem fill toneMap failed");
2419 cond = !AssembleIT35SharedBuffer(metadata, item->sharedProperties);
2420 CHECK_ERROR_RETURN_RET_LOG(cond, nullptr, "AssembleTmapImageItem fill it35 failed");
2421 return item;
2422 }
2423
2424 #if !defined(_WIN32) && !defined(_APPLE) && !defined(IOS_PLATFORM) && !defined(ANDROID_PLATFORM)
AssemblePrimaryImageItem(sptr<SurfaceBuffer> & surfaceBuffer,const PlEncodeOptions & opts)2425 std::shared_ptr<ImageItem> ExtEncoder::AssemblePrimaryImageItem(sptr<SurfaceBuffer>& surfaceBuffer,
2426 const PlEncodeOptions &opts)
2427 {
2428 bool cond = pixelmap_ == nullptr || surfaceBuffer == nullptr;
2429 CHECK_ERROR_RETURN_RET_LOG(cond, nullptr, "AssemblePrimaryImageItem surfaceBuffer is nullptr");
2430 auto item = std::make_shared<ImageItem>();
2431 item->id = PRIMARY_IMAGE_ITEM_ID;
2432 item->pixelBuffer = sptr<NativeBuffer>::MakeSptr(surfaceBuffer->GetBufferHandle());
2433 item->isPrimary = true;
2434 item->isHidden = false;
2435 item->compressType = COMPRESS_TYPE_HEVC;
2436 item->quality = opts.quality;
2437 item->sharedProperties.fd = -1;
2438 item->pixelSharedBuffer.fd = -1;
2439 #ifdef USE_M133_SKIA
2440 sk_sp<SkData> iccProfile = icc_from_color_space(ToSkInfo(pixelmap_), nullptr, nullptr);
2441 #else
2442 sk_sp<SkData> iccProfile = icc_from_color_space(ToSkInfo(pixelmap_));
2443 #endif
2444 bool tempRes = AssembleICCImageProperty(iccProfile, item->sharedProperties);
2445 CHECK_ERROR_RETURN_RET(!tempRes, nullptr);
2446 uint32_t litePropertiesSize = 0;
2447 litePropertiesSize += (sizeof(PropertyType::COLOR_TYPE) + sizeof(ColorType));
2448 item->liteProperties.resize(litePropertiesSize);
2449 size_t offset = 0;
2450 ColorType colorType = ColorType::RICC;
2451 cond = !FillLitePropertyItem(item->liteProperties, offset,
2452 PropertyType::COLOR_TYPE, &colorType, sizeof(ColorType));
2453 CHECK_ERROR_RETURN_RET_LOG(cond, nullptr, "AssemblePrimaryImageItem Fill color type failed");
2454 return item;
2455 }
2456 #endif
2457
AssembleExifRefItem(std::vector<ItemRef> & refs)2458 void ExtEncoder::AssembleExifRefItem(std::vector<ItemRef>& refs)
2459 {
2460 auto item = std::make_shared<ItemRef>();
2461 item->type = ReferenceType::CDSC;
2462 item->from = EXIF_META_ITEM_ID;
2463 item->to.resize(INDEX_ONE);
2464 item->to[INDEX_ZERO] = PRIMARY_IMAGE_ITEM_ID;
2465 refs.push_back(*item);
2466 }
2467
FreeExifBlob(uint8_t ** exifBlob)2468 void inline FreeExifBlob(uint8_t** exifBlob)
2469 {
2470 if (exifBlob != nullptr && *exifBlob != nullptr) {
2471 free(*exifBlob);
2472 *exifBlob = nullptr;
2473 }
2474 }
2475
AssembleExifMetaItem(std::vector<MetaItem> & metaItems)2476 bool ExtEncoder::AssembleExifMetaItem(std::vector<MetaItem>& metaItems)
2477 {
2478 if (!opts_.needsPackProperties) {
2479 IMAGE_LOGD("no need encode exif");
2480 return false;
2481 }
2482 ExifData* exifData = nullptr;
2483 if (picture_ != nullptr && picture_->GetExifMetadata() != nullptr &&
2484 picture_->GetExifMetadata()->GetExifData() != nullptr) {
2485 exifData = picture_->GetExifMetadata()->GetExifData();
2486 } else if (pixelmap_ != nullptr && pixelmap_->GetExifMetadata() != nullptr &&
2487 pixelmap_->GetExifMetadata()->GetExifData() != nullptr) {
2488 exifData = pixelmap_->GetExifMetadata()->GetExifData();
2489 } else {
2490 IMAGE_LOGD("no exif");
2491 return false;
2492 }
2493 uint8_t* exifBlob = nullptr;
2494 uint32_t exifSize = 0;
2495 TiffParser::Encode(&exifBlob, exifSize, exifData);
2496 CHECK_ERROR_RETURN_RET_LOG(exifBlob == nullptr, false, "Encode exif data failed");
2497 auto item = std::make_shared<MetaItem>();
2498 item->id = EXIF_META_ITEM_ID;
2499 item->itemName = "exif";
2500 item->data.fd = -1;
2501 std::shared_ptr<AbsMemory> propertyAshmem = AllocateNewSharedMem(exifSize + EXIF_PRE_SIZE, EXIF_ASHMEM_TAG);
2502 if (propertyAshmem == nullptr) {
2503 FreeExifBlob(&exifBlob);
2504 IMAGE_LOGE("AssembleExifMetaItem alloc propertyAshmem failed");
2505 return false;
2506 }
2507 tmpMemoryList_.push_back(propertyAshmem);
2508 uint8_t* memData = reinterpret_cast<uint8_t*>(propertyAshmem->data.data);
2509 size_t memSize = propertyAshmem->data.size;
2510 bool fillRes = (memcpy_s(memData, memSize, EXIF_PRE_TAG, EXIF_PRE_SIZE) == EOK) &&
2511 (memcpy_s(memData + EXIF_PRE_SIZE, memSize - EXIF_PRE_SIZE, exifBlob, exifSize) == EOK);
2512 if (fillRes) {
2513 item->data.fd = *static_cast<int *>(propertyAshmem->extend.data);
2514 item->data.capacity = propertyAshmem->data.size;
2515 item->data.filledLen = propertyAshmem->data.size;
2516 metaItems.push_back(*item);
2517 }
2518 FreeExifBlob(&exifBlob);
2519 return fillRes;
2520 }
2521
FillPixelSharedBuffer(sptr<SurfaceBuffer> sbBuffer,uint32_t capacity,SharedBuffer & outBuffer)2522 bool ExtEncoder::FillPixelSharedBuffer(sptr<SurfaceBuffer> sbBuffer, uint32_t capacity, SharedBuffer& outBuffer)
2523 {
2524 bool cond = sbBuffer == nullptr || capacity == 0;
2525 CHECK_INFO_RETURN_RET_LOG(cond, false, "%{public}s iccprofile is nullptr", __func__);
2526 std::shared_ptr<AbsMemory> sharedMem = AllocateNewSharedMem(capacity, IMAGE_DATA_TAG);
2527 cond = sharedMem == nullptr;
2528 CHECK_ERROR_RETURN_RET_LOG(cond, false, "%{public}s alloc failed", __func__);
2529 tmpMemoryList_.push_back(sharedMem);
2530 uint8_t* memData = reinterpret_cast<uint8_t*>(sharedMem->data.data);
2531 size_t memSize = sharedMem->data.size;
2532 cond = capacity > memSize;
2533 CHECK_ERROR_RETURN_RET_LOG(cond, false,
2534 "%{public}s shared capacity[%{public}d] over memSize[%{public}ld]", __func__, capacity, memSize);
2535 cond = sharedMem->extend.data == nullptr;
2536 CHECK_ERROR_RETURN_RET_LOG(cond, false, "%{public}s sharedMem's extend data is nullptr", __func__);
2537 if (memcpy_s(memData, memSize, sbBuffer->GetVirAddr(), capacity) == EOK) {
2538 outBuffer.fd = *static_cast<int *>(sharedMem->extend.data);
2539 outBuffer.capacity = memSize;
2540 outBuffer.filledLen = memSize;
2541 } else {
2542 IMAGE_LOGE("%{public}s memcpy failed", __func__);
2543 return false;
2544 }
2545 return true;
2546 }
2547
AssembleAuxiliaryRefItem(AuxiliaryPictureType type,std::vector<ItemRef> & refs)2548 void ExtEncoder::AssembleAuxiliaryRefItem(AuxiliaryPictureType type, std::vector<ItemRef>& refs)
2549 {
2550 auto item = std::make_shared<ItemRef>();
2551 item->type = ReferenceType::AUXL;
2552 item->to.resize(INDEX_ONE);
2553 item->to[INDEX_ZERO] = PRIMARY_IMAGE_ITEM_ID;
2554 switch (type) {
2555 case AuxiliaryPictureType::DEPTH_MAP:
2556 item->from = DEPTH_MAP_ITEM_ID;
2557 break;
2558 case AuxiliaryPictureType::UNREFOCUS_MAP:
2559 item->from = UNREFOCUS_MAP_ITEM_ID;
2560 break;
2561 case AuxiliaryPictureType::LINEAR_MAP:
2562 item->from = LINEAR_MAP_ITEM_ID;
2563 break;
2564 case AuxiliaryPictureType::FRAGMENT_MAP:
2565 item->from = FRAGMENT_MAP_ITEM_ID;
2566 break;
2567 default:
2568 break;
2569 }
2570 refs.push_back(*item);
2571 }
2572 #endif
2573 } // namespace ImagePlugin
2574 } // namespace OHOS
2575