1 /*
2 * Copyright (C) 2018 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 #include "HardwareBitmapUploader.h"
18
19 #include <EGL/egl.h>
20 #include <EGL/eglext.h>
21 #include <GLES2/gl2.h>
22 #include <GLES2/gl2ext.h>
23 #include <GLES3/gl3.h>
24 #include <GrDirectContext.h>
25 #include <GrTypes.h>
26 #include <SkBitmap.h>
27 #include <SkCanvas.h>
28 #include <SkImage.h>
29 #include <SkImageAndroid.h>
30 #include <SkImageInfo.h>
31 #include <SkRefCnt.h>
32 #include <gui/TraceUtils.h>
33 #include <utils/GLUtils.h>
34 #include <utils/NdkUtils.h>
35 #include <utils/Trace.h>
36
37 #include <thread>
38
39 #include "hwui/Bitmap.h"
40 #include "renderthread/EglManager.h"
41 #include "renderthread/VulkanManager.h"
42 #include "thread/ThreadBase.h"
43 #include "utils/TimeUtils.h"
44
45 namespace android::uirenderer {
46
47 static constexpr auto kThreadTimeout = 60000_ms;
48
49 class AHBUploader;
50 // This helper uploader classes allows us to upload using either EGL or Vulkan using the same
51 // interface.
52 static sp<AHBUploader> sUploader = nullptr;
53
54 struct FormatInfo {
55 AHardwareBuffer_Format bufferFormat;
56 GLint format, type;
57 VkFormat vkFormat;
58 bool isSupported = false;
59 bool valid = true;
60 };
61
62 class AHBUploader : public RefBase {
63 public:
~AHBUploader()64 virtual ~AHBUploader() {}
65
destroy()66 void destroy() {
67 std::lock_guard _lock{mLock};
68 LOG_ALWAYS_FATAL_IF(mPendingUploads, "terminate called while uploads in progress");
69 if (mUploadThread) {
70 mUploadThread->requestExit();
71 mUploadThread->join();
72 mUploadThread = nullptr;
73 }
74 onDestroy();
75 }
76
uploadHardwareBitmap(const SkBitmap & bitmap,const FormatInfo & format,AHardwareBuffer * ahb)77 bool uploadHardwareBitmap(const SkBitmap& bitmap, const FormatInfo& format,
78 AHardwareBuffer* ahb) {
79 ATRACE_CALL();
80 beginUpload();
81 bool result = onUploadHardwareBitmap(bitmap, format, ahb);
82 endUpload();
83 return result;
84 }
85
postIdleTimeoutCheck()86 void postIdleTimeoutCheck() {
87 mUploadThread->queue().postDelayed(kThreadTimeout, [this]() { this->idleTimeoutCheck(); });
88 }
89
90 protected:
91 std::mutex mLock;
92 sp<ThreadBase> mUploadThread = nullptr;
93
94 private:
95 virtual void onIdle() = 0;
96 virtual void onDestroy() = 0;
97
98 virtual bool onUploadHardwareBitmap(const SkBitmap& bitmap, const FormatInfo& format,
99 AHardwareBuffer* ahb) = 0;
100 virtual void onBeginUpload() = 0;
101
shouldTimeOutLocked()102 bool shouldTimeOutLocked() {
103 nsecs_t durationSince = systemTime() - mLastUpload;
104 return durationSince > kThreadTimeout;
105 }
106
idleTimeoutCheck()107 void idleTimeoutCheck() {
108 std::lock_guard _lock{mLock};
109 if (mPendingUploads == 0 && shouldTimeOutLocked()) {
110 onIdle();
111 } else {
112 this->postIdleTimeoutCheck();
113 }
114 }
115
beginUpload()116 void beginUpload() {
117 std::lock_guard _lock{mLock};
118 mPendingUploads++;
119
120 if (!mUploadThread) {
121 mUploadThread = new ThreadBase{};
122 }
123 if (!mUploadThread->isRunning()) {
124 mUploadThread->start("GrallocUploadThread");
125 }
126
127 onBeginUpload();
128 }
129
endUpload()130 void endUpload() {
131 std::lock_guard _lock{mLock};
132 mPendingUploads--;
133 mLastUpload = systemTime();
134 }
135
136 int mPendingUploads = 0;
137 nsecs_t mLastUpload = 0;
138 };
139
140 #define FENCE_TIMEOUT 2000000000
141
142 class EGLUploader : public AHBUploader {
143 private:
onDestroy()144 void onDestroy() override {
145 mEglManager.destroy();
146 }
onIdle()147 void onIdle() override {
148 mEglManager.destroy();
149 }
150
onBeginUpload()151 void onBeginUpload() override {
152 if (!mEglManager.hasEglContext()) {
153 mUploadThread->queue().runSync([this]() {
154 this->mEglManager.initialize();
155 glPixelStorei(GL_UNPACK_ALIGNMENT, 1);
156 });
157
158 this->postIdleTimeoutCheck();
159 }
160 }
161
162
getUploadEglDisplay()163 EGLDisplay getUploadEglDisplay() {
164 std::lock_guard _lock{mLock};
165 LOG_ALWAYS_FATAL_IF(!mEglManager.hasEglContext(), "Forgot to begin an upload?");
166 return mEglManager.eglDisplay();
167 }
168
onUploadHardwareBitmap(const SkBitmap & bitmap,const FormatInfo & format,AHardwareBuffer * ahb)169 bool onUploadHardwareBitmap(const SkBitmap& bitmap, const FormatInfo& format,
170 AHardwareBuffer* ahb) override {
171 ATRACE_CALL();
172
173 EGLDisplay display = getUploadEglDisplay();
174
175 LOG_ALWAYS_FATAL_IF(display == EGL_NO_DISPLAY, "Failed to get EGL_DEFAULT_DISPLAY! err=%s",
176 uirenderer::renderthread::EglManager::eglErrorString());
177 // We use an EGLImage to access the content of the buffer
178 // The EGL image is later bound to a 2D texture
179 const EGLClientBuffer clientBuffer = eglGetNativeClientBufferANDROID(ahb);
180 AutoEglImage autoImage(display, clientBuffer);
181 if (autoImage.image == EGL_NO_IMAGE_KHR) {
182 ALOGW("Could not create EGL image, err =%s",
183 uirenderer::renderthread::EglManager::eglErrorString());
184 return false;
185 }
186
187 {
188 ATRACE_FORMAT("CPU -> gralloc transfer (%dx%d)", bitmap.width(), bitmap.height());
189 EGLSyncKHR fence = mUploadThread->queue().runSync([&]() -> EGLSyncKHR {
190 AutoSkiaGlTexture glTexture;
191 glEGLImageTargetTexture2DOES(GL_TEXTURE_2D, autoImage.image);
192 if (GLUtils::dumpGLErrors()) {
193 return EGL_NO_SYNC_KHR;
194 }
195
196 // glTexSubImage2D is synchronous in sense that it memcpy() from pointer that we
197 // provide.
198 // But asynchronous in sense that driver may upload texture onto hardware buffer
199 // when we first use it in drawing
200 glTexSubImage2D(GL_TEXTURE_2D, 0, 0, 0, bitmap.width(), bitmap.height(),
201 format.format, format.type, bitmap.getPixels());
202 if (GLUtils::dumpGLErrors()) {
203 return EGL_NO_SYNC_KHR;
204 }
205
206 EGLSyncKHR uploadFence =
207 eglCreateSyncKHR(eglGetCurrentDisplay(), EGL_SYNC_FENCE_KHR, NULL);
208 if (uploadFence == EGL_NO_SYNC_KHR) {
209 ALOGW("Could not create sync fence %#x", eglGetError());
210 };
211 glFlush();
212 GLUtils::dumpGLErrors();
213 return uploadFence;
214 });
215
216 if (fence == EGL_NO_SYNC_KHR) {
217 return false;
218 }
219 EGLint waitStatus = eglClientWaitSyncKHR(display, fence, 0, FENCE_TIMEOUT);
220 ALOGE_IF(waitStatus != EGL_CONDITION_SATISFIED_KHR,
221 "Failed to wait for the fence %#x", eglGetError());
222
223 eglDestroySyncKHR(display, fence);
224 }
225 return true;
226 }
227
228 renderthread::EglManager mEglManager;
229 };
230
231 class VkUploader : public AHBUploader {
232 private:
onDestroy()233 void onDestroy() override {
234 std::lock_guard _lock{mVkLock};
235 mGrContext.reset();
236 mVulkanManagerStrong.clear();
237 }
onIdle()238 void onIdle() override {
239 onDestroy();
240 }
241
onBeginUpload()242 void onBeginUpload() override {}
243
onUploadHardwareBitmap(const SkBitmap & bitmap,const FormatInfo & format,AHardwareBuffer * ahb)244 bool onUploadHardwareBitmap(const SkBitmap& bitmap, const FormatInfo& format,
245 AHardwareBuffer* ahb) override {
246 bool uploadSucceeded = false;
247 mUploadThread->queue().runSync([this, &uploadSucceeded, bitmap, ahb]() {
248 ATRACE_CALL();
249 std::lock_guard _lock{mVkLock};
250
251 renderthread::VulkanManager* vkManager = getVulkanManager();
252 if (!vkManager->hasVkContext()) {
253 LOG_ALWAYS_FATAL_IF(mGrContext,
254 "GrContext exists with no VulkanManager for vulkan uploads");
255 vkManager->initialize();
256 }
257
258 if (!mGrContext) {
259 GrContextOptions options;
260 mGrContext = vkManager->createContext(options,
261 renderthread::VulkanManager::ContextType::kUploadThread);
262 LOG_ALWAYS_FATAL_IF(!mGrContext, "failed to create GrContext for vulkan uploads");
263 this->postIdleTimeoutCheck();
264 }
265
266 sk_sp<SkImage> image =
267 SkImages::TextureFromAHardwareBufferWithData(mGrContext.get(), bitmap.pixmap(),
268 ahb);
269 mGrContext->submit(GrSyncCpu::kYes);
270
271 uploadSucceeded = (image.get() != nullptr);
272 });
273 return uploadSucceeded;
274 }
275
276 /* must be called on the upload thread after the vkLock has been acquired */
getVulkanManager()277 renderthread::VulkanManager* getVulkanManager() {
278 if (!mVulkanManagerStrong) {
279 mVulkanManagerStrong = mVulkanManagerWeak.promote();
280
281 // create a new manager if we couldn't promote the weak ref
282 if (!mVulkanManagerStrong) {
283 mVulkanManagerStrong = renderthread::VulkanManager::getInstance();
284 mGrContext.reset();
285 mVulkanManagerWeak = mVulkanManagerStrong;
286 }
287 }
288
289 return mVulkanManagerStrong.get();
290 }
291
292 sk_sp<GrDirectContext> mGrContext;
293 sp<renderthread::VulkanManager> mVulkanManagerStrong;
294 wp<renderthread::VulkanManager> mVulkanManagerWeak;
295 std::mutex mVkLock;
296 };
297
checkSupport(AHardwareBuffer_Format format)298 static bool checkSupport(AHardwareBuffer_Format format) {
299 AHardwareBuffer_Desc desc = {
300 .width = 1,
301 .height = 1,
302 .layers = 1,
303 .format = format,
304 .usage = AHARDWAREBUFFER_USAGE_CPU_READ_NEVER | AHARDWAREBUFFER_USAGE_CPU_WRITE_NEVER |
305 AHARDWAREBUFFER_USAGE_GPU_SAMPLED_IMAGE,
306 };
307 UniqueAHardwareBuffer buffer = allocateAHardwareBuffer(desc);
308 return buffer != nullptr;
309 }
310
hasFP16Support()311 bool HardwareBitmapUploader::hasFP16Support() {
312 static bool hasFP16Support = checkSupport(AHARDWAREBUFFER_FORMAT_R16G16B16A16_FLOAT);
313 return hasFP16Support;
314 }
315
has1010102Support()316 bool HardwareBitmapUploader::has1010102Support() {
317 static bool has101012Support = checkSupport(AHARDWAREBUFFER_FORMAT_R10G10B10A2_UNORM);
318 return has101012Support;
319 }
320
hasAlpha8Support()321 bool HardwareBitmapUploader::hasAlpha8Support() {
322 static bool hasAlpha8Support = checkSupport(AHARDWAREBUFFER_FORMAT_R8_UNORM);
323 return hasAlpha8Support;
324 }
325
determineFormat(const SkBitmap & skBitmap,bool usingGL)326 static FormatInfo determineFormat(const SkBitmap& skBitmap, bool usingGL) {
327 FormatInfo formatInfo;
328 switch (skBitmap.info().colorType()) {
329 case kRGBA_8888_SkColorType:
330 formatInfo.isSupported = true;
331 [[fallthrough]];
332 // ARGB_4444 is upconverted to RGBA_8888
333 case kARGB_4444_SkColorType:
334 formatInfo.bufferFormat = AHARDWAREBUFFER_FORMAT_R8G8B8A8_UNORM;
335 formatInfo.format = GL_RGBA;
336 formatInfo.type = GL_UNSIGNED_BYTE;
337 formatInfo.vkFormat = VK_FORMAT_R8G8B8A8_UNORM;
338 break;
339 case kRGBA_F16_SkColorType:
340 formatInfo.isSupported = HardwareBitmapUploader::hasFP16Support();
341 if (formatInfo.isSupported) {
342 formatInfo.type = GL_HALF_FLOAT;
343 formatInfo.bufferFormat = AHARDWAREBUFFER_FORMAT_R16G16B16A16_FLOAT;
344 formatInfo.vkFormat = VK_FORMAT_R16G16B16A16_SFLOAT;
345 } else {
346 formatInfo.type = GL_UNSIGNED_BYTE;
347 formatInfo.bufferFormat = AHARDWAREBUFFER_FORMAT_R8G8B8A8_UNORM;
348 formatInfo.vkFormat = VK_FORMAT_R8G8B8A8_UNORM;
349 }
350 formatInfo.format = GL_RGBA;
351 break;
352 case kRGB_565_SkColorType:
353 formatInfo.isSupported = true;
354 formatInfo.bufferFormat = AHARDWAREBUFFER_FORMAT_R5G6B5_UNORM;
355 formatInfo.format = GL_RGB;
356 formatInfo.type = GL_UNSIGNED_SHORT_5_6_5;
357 formatInfo.vkFormat = VK_FORMAT_R5G6B5_UNORM_PACK16;
358 break;
359 case kGray_8_SkColorType:
360 formatInfo.isSupported = usingGL;
361 formatInfo.bufferFormat = AHARDWAREBUFFER_FORMAT_R8G8B8A8_UNORM;
362 formatInfo.format = GL_LUMINANCE;
363 formatInfo.type = GL_UNSIGNED_BYTE;
364 formatInfo.vkFormat = VK_FORMAT_R8G8B8A8_UNORM;
365 break;
366 case kRGBA_1010102_SkColorType:
367 formatInfo.isSupported = HardwareBitmapUploader::has1010102Support();
368 if (formatInfo.isSupported) {
369 formatInfo.type = GL_UNSIGNED_INT_2_10_10_10_REV;
370 formatInfo.bufferFormat = AHARDWAREBUFFER_FORMAT_R10G10B10A2_UNORM;
371 formatInfo.vkFormat = VK_FORMAT_A2B10G10R10_UNORM_PACK32;
372 } else {
373 formatInfo.type = GL_UNSIGNED_BYTE;
374 formatInfo.bufferFormat = AHARDWAREBUFFER_FORMAT_R8G8B8A8_UNORM;
375 formatInfo.vkFormat = VK_FORMAT_R8G8B8A8_UNORM;
376 }
377 formatInfo.format = GL_RGBA;
378 break;
379 case kAlpha_8_SkColorType:
380 formatInfo.isSupported = HardwareBitmapUploader::hasAlpha8Support();
381 if (formatInfo.isSupported) {
382 formatInfo.bufferFormat = AHARDWAREBUFFER_FORMAT_R8_UNORM;
383 formatInfo.format = GL_RED;
384 formatInfo.type = GL_UNSIGNED_BYTE;
385 formatInfo.vkFormat = VK_FORMAT_R8_UNORM;
386 } else {
387 formatInfo.type = GL_UNSIGNED_BYTE;
388 formatInfo.bufferFormat = AHARDWAREBUFFER_FORMAT_R8G8B8A8_UNORM;
389 formatInfo.vkFormat = VK_FORMAT_R8G8B8A8_UNORM;
390 formatInfo.format = GL_RGBA;
391 }
392 break;
393 default:
394 ALOGW("unable to create hardware bitmap of colortype: %d", skBitmap.info().colorType());
395 formatInfo.valid = false;
396 }
397 return formatInfo;
398 }
399
makeHwCompatible(const FormatInfo & format,const SkBitmap & source)400 static SkBitmap makeHwCompatible(const FormatInfo& format, const SkBitmap& source) {
401 if (format.isSupported) {
402 return source;
403 } else {
404 SkBitmap bitmap;
405 bitmap.allocPixels(source.info().makeColorType(kN32_SkColorType));
406 bitmap.writePixels(source.pixmap());
407 return bitmap;
408 }
409 }
410
411
createUploader(bool usingGL)412 static void createUploader(bool usingGL) {
413 static std::mutex lock;
414 std::lock_guard _lock{lock};
415 if (!sUploader.get()) {
416 if (usingGL) {
417 sUploader = new EGLUploader();
418 } else {
419 sUploader = new VkUploader();
420 }
421 }
422 }
423
allocateHardwareBitmap(const SkBitmap & sourceBitmap)424 sk_sp<Bitmap> HardwareBitmapUploader::allocateHardwareBitmap(const SkBitmap& sourceBitmap) {
425 ATRACE_CALL();
426
427 bool usingGL = uirenderer::Properties::getRenderPipelineType() ==
428 uirenderer::RenderPipelineType::SkiaGL;
429
430 FormatInfo format = determineFormat(sourceBitmap, usingGL);
431 if (!format.valid) {
432 return nullptr;
433 }
434
435 SkBitmap bitmap = makeHwCompatible(format, sourceBitmap);
436 AHardwareBuffer_Desc desc = {
437 .width = static_cast<uint32_t>(bitmap.width()),
438 .height = static_cast<uint32_t>(bitmap.height()),
439 .layers = 1,
440 .format = format.bufferFormat,
441 .usage = AHARDWAREBUFFER_USAGE_CPU_READ_NEVER | AHARDWAREBUFFER_USAGE_CPU_WRITE_NEVER |
442 AHARDWAREBUFFER_USAGE_GPU_SAMPLED_IMAGE,
443 };
444 UniqueAHardwareBuffer ahb = allocateAHardwareBuffer(desc);
445 if (!ahb) {
446 ALOGW("allocateHardwareBitmap() failed in AHardwareBuffer_allocate()");
447 return nullptr;
448 };
449
450 createUploader(usingGL);
451
452 if (!sUploader->uploadHardwareBitmap(bitmap, format, ahb.get())) {
453 return nullptr;
454 }
455 return Bitmap::createFrom(ahb.get(), bitmap.colorType(), bitmap.refColorSpace(),
456 bitmap.alphaType(), Bitmap::computePalette(bitmap));
457 }
458
initialize()459 void HardwareBitmapUploader::initialize() {
460 bool usingGL = uirenderer::Properties::getRenderPipelineType() ==
461 uirenderer::RenderPipelineType::SkiaGL;
462 createUploader(usingGL);
463 }
464
terminate()465 void HardwareBitmapUploader::terminate() {
466 if (sUploader) {
467 sUploader->destroy();
468 }
469 }
470
471 } // namespace android::uirenderer
472