1 /*
2 * Copyright (c) 2022-2023 Huawei Device Co., Ltd.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at
6 *
7 * http://www.apache.org/licenses/LICENSE-2.0
8 *
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
14 */
15
16 #include "core/components_ng/image_provider/adapter/flutter_image_provider.h"
17
18 #include <mutex>
19 #include <utility>
20
21 #include "flutter/fml/memory/ref_counted.h"
22
23 #include "base/log/ace_trace.h"
24 #include "base/memory/referenced.h"
25 #ifdef NG_BUILD
26 #include "ace_shell/shell/common/window_manager.h"
27 #include "flutter/lib/ui/io_manager.h"
28 #else
29 #include "flutter/lib/ui/painting/image.h"
30 #endif
31 #include "third_party/skia/include/codec/SkCodec.h"
32 #include "third_party/skia/include/core/SkGraphics.h"
33
34 #include "core/common/container.h"
35 #include "core/common/container_scope.h"
36 #include "core/common/thread_checker.h"
37 #include "core/components_ng/image_provider/adapter/skia_image_data.h"
38 #include "core/components_ng/image_provider/image_object.h"
39 #include "core/components_ng/image_provider/image_utils.h"
40 #include "core/components_ng/image_provider/svg_image_object.h"
41 #include "core/components_ng/render/adapter/skia_canvas_image.h"
42 #include "core/image/flutter_image_cache.h"
43 #include "core/image/image_compressor.h"
44 #include "core/image/image_loader.h"
45 #include "core/pipeline_ng/pipeline_context.h"
46
47 #ifdef NG_BUILD
48 #include "core/components_ng/render/adapter/flutter_canvas_image.h"
49 #endif
50
51 namespace OHOS::Ace::NG {
52 namespace {
53
ApplySizeToSkImage(const sk_sp<SkImage> & rawImage,int32_t dstWidth,int32_t dstHeight,const std::string & srcKey)54 sk_sp<SkImage> ApplySizeToSkImage(
55 const sk_sp<SkImage>& rawImage, int32_t dstWidth, int32_t dstHeight, const std::string& srcKey)
56 {
57 ACE_FUNCTION_TRACE();
58 auto scaledImageInfo =
59 SkImageInfo::Make(dstWidth, dstHeight, rawImage->colorType(), rawImage->alphaType(), rawImage->refColorSpace());
60 SkBitmap scaledBitmap;
61 if (!scaledBitmap.tryAllocPixels(scaledImageInfo)) {
62 LOGE("Could not allocate bitmap when attempting to scale. srcKey: %{private}s, destination size: [%{public}d x"
63 " %{public}d], raw image size: [%{public}d x %{public}d]",
64 srcKey.c_str(), dstWidth, dstHeight, rawImage->width(), rawImage->height());
65 return rawImage;
66 }
67 #ifdef NG_BUILD
68 if (!rawImage->scalePixels(
69 scaledBitmap.pixmap(), SkSamplingOptions(SkFilterMode::kLinear), SkImage::kDisallow_CachingHint)) {
70 #else
71 if (!rawImage->scalePixels(scaledBitmap.pixmap(), kLow_SkFilterQuality, SkImage::kDisallow_CachingHint)) {
72 #endif
73 LOGE("Could not scale pixels srcKey: %{private}s, destination size: [%{public}d x"
74 " %{public}d], raw image size: [%{public}d x %{public}d]",
75 srcKey.c_str(), dstWidth, dstHeight, rawImage->width(), rawImage->height());
76 return rawImage;
77 }
78 // Marking this as immutable makes the MakeFromBitmap call share the pixels instead of copying.
79 scaledBitmap.setImmutable();
80 auto scaledImage = SkImage::MakeFromBitmap(scaledBitmap);
81 CHECK_NULL_RETURN_NOLOG(scaledImage, rawImage);
82 return scaledImage;
83 }
84
85 static sk_sp<SkImage> ResizeSkImage(
86 const sk_sp<SkImage>& rawImage, const std::string& src, const SizeF& resizeTarget, bool forceResize)
87 {
88 if (!resizeTarget.IsPositive()) {
89 LOGE("not valid size! resizeTarget: %{public}s, src: %{public}s", resizeTarget.ToString().c_str(), src.c_str());
90 return rawImage;
91 }
92 int32_t dstWidth = static_cast<int32_t>(resizeTarget.Width() + 0.5);
93 int32_t dstHeight = static_cast<int32_t>(resizeTarget.Height() + 0.5);
94
95 bool needResize = false;
96
97 if (!forceResize) {
98 if (rawImage->width() > dstWidth) {
99 needResize = true;
100 } else {
101 dstWidth = rawImage->width();
102 }
103 if (rawImage->height() > dstHeight) {
104 needResize = true;
105 } else {
106 dstHeight = rawImage->height();
107 }
108 }
109
110 if (!needResize && !forceResize) {
111 return rawImage;
112 }
113 return ApplySizeToSkImage(rawImage, dstWidth, dstHeight, src);
114 }
115
116 } // namespace
117
QueryCanvasImageFromCache(const ImageSourceInfo & src,const SizeF & targetSize)118 RefPtr<CanvasImage> ImageProvider::QueryCanvasImageFromCache(const ImageSourceInfo& src, const SizeF& targetSize)
119 {
120 // Query [CanvasImage] from cache, if hit, notify load success immediately and returns true
121 auto pipelineCtx = PipelineContext::GetCurrentContext();
122 CHECK_NULL_RETURN_NOLOG(pipelineCtx, nullptr);
123 CHECK_NULL_RETURN_NOLOG(pipelineCtx->GetImageCache(), nullptr);
124 auto key = ImageUtils::GenerateImageKey(src, targetSize);
125 auto cache = pipelineCtx->GetImageCache();
126 CHECK_NULL_RETURN(cache, nullptr);
127 auto cacheImage = cache->GetCacheImage(key);
128 CHECK_NULL_RETURN_NOLOG(cacheImage, nullptr);
129 #ifdef NG_BUILD
130 auto canvasImage = cacheImage->imagePtr;
131 #else
132 auto flutterCanvasImage = cacheImage->imagePtr;
133 auto canvasImage = CanvasImage::Create(&flutterCanvasImage);
134 auto skiaCanvasImage = DynamicCast<SkiaCanvasImage>(canvasImage);
135 CHECK_NULL_RETURN(skiaCanvasImage, nullptr);
136 skiaCanvasImage->SetUniqueID(cacheImage->uniqueId);
137 #endif
138 if (canvasImage) {
139 LOGD("[ImageCache][CanvasImage] succeed find canvas image from cache: %{public}s", key.c_str());
140 }
141 return canvasImage;
142 }
143
MakeCanvasImageHelper(const WeakPtr<ImageObject> & objWp,const SizeF & targetSize,const RefPtr<RenderTaskHolder> & renderTaskHolder,bool forceResize,bool sync)144 void ImageProvider::MakeCanvasImageHelper(const WeakPtr<ImageObject>& objWp, const SizeF& targetSize,
145 const RefPtr<RenderTaskHolder>& renderTaskHolder, bool forceResize, bool sync)
146 {
147 auto obj = objWp.Upgrade();
148 CHECK_NULL_VOID(obj && renderTaskHolder);
149 auto flutterRenderTaskHolder = DynamicCast<FlutterRenderTaskHolder>(renderTaskHolder);
150 CHECK_NULL_VOID(flutterRenderTaskHolder);
151 CHECK_NULL_VOID_NOLOG(ImageProvider::PrepareImageData(obj));
152 // resize image
153 auto skiaImageData = DynamicCast<SkiaImageData>(obj->GetData());
154 CHECK_NULL_VOID(skiaImageData && skiaImageData->GetSkData());
155 auto rawImage = SkImage::MakeFromEncoded(skiaImageData->GetSkData());
156 auto key = ImageUtils::GenerateImageKey(obj->GetSourceInfo(), targetSize);
157 if (!rawImage) {
158 std::string errorMessage(
159 "Static image MakeFromEncoded fail! The image format is not supported, please check image format.");
160 ImageProvider::FailCallback(key, errorMessage, sync);
161 return;
162 }
163 // get compressed image for file cache
164 sk_sp<SkImage> image = rawImage;
165 auto compressFileData = ImageLoader::LoadImageDataFromFileCache(key, ".astc");
166 if (!compressFileData) {
167 image = ResizeSkImage(rawImage, obj->GetSourceInfo().GetSrc(), targetSize, forceResize);
168 }
169 CHECK_NULL_VOID(image);
170 // create gpu object
171 flutter::SkiaGPUObject<SkImage> skiaGpuObjSkImage({ image, flutterRenderTaskHolder->unrefQueue });
172 #ifdef NG_BUILD
173 auto canvasImage = CanvasImage::Create();
174 auto flutterImage = AceType::DynamicCast<NG::FlutterCanvasImage>(canvasImage);
175 if (flutterImage) {
176 flutterImage->SetImage(std::move(skiaGpuObjSkImage));
177 }
178 #else
179 // create canvas image
180 auto flutterCanvasImage = flutter::CanvasImage::Create();
181 flutterCanvasImage->set_image(std::move(skiaGpuObjSkImage));
182 auto canvasImage = CanvasImage::Create(&flutterCanvasImage);
183 CHECK_NULL_VOID(canvasImage);
184 ImageProvider::CacheCanvasImage(canvasImage, key);
185 #endif
186 // upload
187 auto uploadTask = [key, sync](const RefPtr<CanvasImage>& canvasImage) {
188 ImageProvider::SuccessCallback(canvasImage, key, sync);
189 };
190 ImageProvider::UploadImageToGPUForRender(
191 canvasImage, std::move(uploadTask), renderTaskHolder, key, targetSize, compressFileData, sync);
192 }
193
MakeCanvasImage(const WeakPtr<ImageObject> & objWp,const WeakPtr<ImageLoadingContext> & ctxWp,const SizeF & targetSize,bool forceResize,bool sync)194 void ImageProvider::MakeCanvasImage(const WeakPtr<ImageObject>& objWp, const WeakPtr<ImageLoadingContext>& ctxWp,
195 const SizeF& targetSize, bool forceResize, bool sync)
196 {
197 auto obj = objWp.Upgrade();
198 CHECK_NULL_VOID(obj);
199 auto key = ImageUtils::GenerateImageKey(obj->GetSourceInfo(), targetSize);
200 // check if same task is already executing
201 if (!RegisterTask(key, ctxWp)) {
202 return;
203 }
204
205 auto renderTaskHolder = CreateRenderTaskHolder();
206 CHECK_NULL_VOID(renderTaskHolder);
207 if (sync) {
208 ImageProvider::MakeCanvasImageHelper(obj, targetSize, renderTaskHolder, forceResize, true);
209 } else {
210 std::scoped_lock<std::mutex> lock(taskMtx_);
211 // wrap with [CancelableCallback] and record in [tasks_] map
212 CancelableCallback<void()> task;
213 task.Reset([objWp, targetSize, renderTaskHolder, forceResize] {
214 MakeCanvasImageHelper(objWp, targetSize, renderTaskHolder, forceResize);
215 });
216 tasks_[key].bgTask_ = task;
217 ImageUtils::PostToBg(task);
218 }
219 }
220
CreateRenderTaskHolder()221 RefPtr<RenderTaskHolder> ImageProvider::CreateRenderTaskHolder()
222 {
223 CHECK_NULL_RETURN(CheckThread(TaskExecutor::TaskType::UI), nullptr);
224 #ifdef NG_BUILD
225 int32_t id = Container::CurrentId();
226 auto currentState = flutter::ace::WindowManager::GetWindow(id);
227 #else
228 auto* currentState = flutter::UIDartState::Current();
229 #endif
230 CHECK_NULL_RETURN(currentState, nullptr);
231 return MakeRefPtr<FlutterRenderTaskHolder>(currentState->GetSkiaUnrefQueue(), currentState->GetIOManager(),
232 currentState->GetTaskRunners().GetIOTaskRunner());
233 }
234
UploadImageToGPUForRender(const RefPtr<CanvasImage> & canvasImage,std::function<void (RefPtr<CanvasImage>)> && callback,const RefPtr<RenderTaskHolder> & renderTaskHolder,const std::string & key,const SizeF & resizeTarget,const RefPtr<ImageData> & data,bool syncLoad)235 void ImageProvider::UploadImageToGPUForRender(const RefPtr<CanvasImage>& canvasImage,
236 std::function<void(RefPtr<CanvasImage>)>&& callback, const RefPtr<RenderTaskHolder>& renderTaskHolder,
237 const std::string& key, const SizeF& resizeTarget, const RefPtr<ImageData>& data, bool syncLoad)
238 {
239 CHECK_NULL_VOID(renderTaskHolder);
240 auto flutterRenderTaskHolder = DynamicCast<FlutterRenderTaskHolder>(renderTaskHolder);
241 CHECK_NULL_VOID(flutterRenderTaskHolder);
242 #ifdef UPLOAD_GPU_DISABLED
243 // If want to dump draw command or gpu disabled, should use CPU image.
244 callback(canvasImage);
245 #else
246 auto skiaCanvasImage = DynamicCast<SkiaCanvasImage>(canvasImage);
247 CHECK_NULL_VOID(skiaCanvasImage);
248 // load compress cache
249 if (data) {
250 int32_t dstWidth = static_cast<int32_t>(resizeTarget.Width() + 0.5);
251 int32_t dstHeight = static_cast<int32_t>(resizeTarget.Height() + 0.5);
252
253 auto skiaImageData = DynamicCast<SkiaImageData>(data);
254 CHECK_NULL_VOID(skiaImageData);
255 auto skdata = skiaImageData->GetSkData();
256 auto stripped = ImageCompressor::StripFileHeader(skdata);
257 LOGI("use astc cache %{public}s %{public}d×%{public}d", key.c_str(), dstWidth, dstHeight);
258 skiaCanvasImage->SetCompressData(stripped, dstWidth, dstHeight);
259 skiaCanvasImage->ReplaceSkImage({ nullptr, flutterRenderTaskHolder->unrefQueue });
260 callback(skiaCanvasImage);
261 return;
262 }
263 if (!ImageCompressor::GetInstance()->CanCompress()) {
264 callback(skiaCanvasImage);
265 return;
266 }
267
268 auto task = [callback, flutterRenderTaskHolder, skiaCanvasImage, id = Container::CurrentId(), src = key] {
269 ContainerScope scope(id);
270 CHECK_NULL_VOID(flutterRenderTaskHolder);
271 auto skImage = skiaCanvasImage->GetCanvasImage();
272 CHECK_NULL_VOID(skImage);
273 auto rasterizedImage = skImage->makeRasterImage();
274 if (!rasterizedImage) {
275 LOGW("Rasterize image failed. callback.");
276 callback(skiaCanvasImage);
277 return;
278 }
279 ACE_DCHECK(!rasterizedImage->isTextureBacked());
280 SkPixmap pixmap;
281 if (!rasterizedImage->peekPixels(&pixmap)) {
282 LOGW("Could not peek pixels of image for texture upload.");
283 callback(skiaCanvasImage);
284 return;
285 }
286 int32_t width = static_cast<int32_t>(pixmap.width());
287 int32_t height = static_cast<int32_t>(pixmap.height());
288 if (ImageCompressor::GetInstance()->CanCompress()) {
289 auto compressData = ImageCompressor::GetInstance()->GpuCompress(src, pixmap, width, height);
290 ImageCompressor::GetInstance()->WriteToFile(src, compressData, { width, height });
291 if (compressData) {
292 // replace skImage of [CanvasImage] with [rasterizedImage]
293 skiaCanvasImage->SetCompressData(compressData, width, height);
294 skiaCanvasImage->ReplaceSkImage({ nullptr, flutterRenderTaskHolder->unrefQueue });
295 } else {
296 skiaCanvasImage->ReplaceSkImage({ rasterizedImage, flutterRenderTaskHolder->unrefQueue });
297 }
298 auto releaseTask = ImageCompressor::GetInstance()->ScheduleReleaseTask();
299 if (flutterRenderTaskHolder->ioTaskRunner) {
300 flutterRenderTaskHolder->ioTaskRunner->PostDelayedTask(
301 releaseTask, fml::TimeDelta::FromMilliseconds(ImageCompressor::releaseTimeMs));
302 } else {
303 ImageUtils::PostToBg(std::move(releaseTask));
304 }
305 }
306 callback(skiaCanvasImage);
307 // Trigger purge cpu bitmap resource, after image upload to gpu.
308 SkGraphics::PurgeResourceCache();
309 };
310 if (syncLoad) {
311 task();
312 } else {
313 ImageUtils::PostToBg(std::move(task));
314 }
315 #endif
316 }
317
CacheCanvasImage(const RefPtr<CanvasImage> & canvasImage,const std::string & key)318 void ImageProvider::CacheCanvasImage(const RefPtr<CanvasImage>& canvasImage, const std::string& key)
319 {
320 auto pipelineCtx = PipelineContext::GetCurrentContext();
321 CHECK_NULL_VOID(pipelineCtx);
322 CHECK_NULL_VOID(pipelineCtx->GetImageCache());
323 #ifdef NG_BUILD
324 pipelineCtx->GetImageCache()->CacheImage(key, std::make_shared<CachedImage>(canvasImage));
325 #else
326 auto skiaCanvasImage = AceType::DynamicCast<SkiaCanvasImage>(canvasImage);
327 CHECK_NULL_VOID_NOLOG(skiaCanvasImage);
328 auto cached = std::make_shared<Ace::CachedImage>(skiaCanvasImage->GetFlutterCanvasImage());
329 cached->uniqueId = skiaCanvasImage->GetUniqueID();
330 pipelineCtx->GetImageCache()->CacheImage(key, cached);
331 #endif
332 }
333
334 } // namespace OHOS::Ace::NG
335