• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 /*
2  * Copyright (C) 2018 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 #include "Readback.h"
18 
19 #include <sync/sync.h>
20 #include <system/window.h>
21 
22 #include <gui/TraceUtils.h>
23 #include "DeferredLayerUpdater.h"
24 #include "Properties.h"
25 #include "hwui/Bitmap.h"
26 #include "pipeline/skia/LayerDrawable.h"
27 #include "renderthread/EglManager.h"
28 #include "renderthread/VulkanManager.h"
29 #include "utils/Color.h"
30 #include "utils/MathUtils.h"
31 #include "utils/NdkUtils.h"
32 
33 using namespace android::uirenderer::renderthread;
34 
35 namespace android {
36 namespace uirenderer {
37 
38 #define ARECT_ARGS(r) float((r).left), float((r).top), float((r).right), float((r).bottom)
39 
copySurfaceInto(ANativeWindow * window,const Rect & inSrcRect,SkBitmap * bitmap)40 CopyResult Readback::copySurfaceInto(ANativeWindow* window, const Rect& inSrcRect,
41                                      SkBitmap* bitmap) {
42     ATRACE_CALL();
43     // Setup the source
44     AHardwareBuffer* rawSourceBuffer;
45     int rawSourceFence;
46     ARect cropRect;
47     uint32_t windowTransform;
48     status_t err = ANativeWindow_getLastQueuedBuffer2(window, &rawSourceBuffer, &rawSourceFence,
49                                                       &cropRect, &windowTransform);
50     base::unique_fd sourceFence(rawSourceFence);
51     // Really this shouldn't ever happen, but better safe than sorry.
52     if (err == UNKNOWN_TRANSACTION) {
53         ALOGW("Readback failed to ANativeWindow_getLastQueuedBuffer2 - who are we talking to?");
54         return copySurfaceIntoLegacy(window, inSrcRect, bitmap);
55     }
56     ALOGV("Using new path, cropRect=" RECT_STRING ", transform=%x", ARECT_ARGS(cropRect),
57           windowTransform);
58 
59     if (err != NO_ERROR) {
60         ALOGW("Failed to get last queued buffer, error = %d", err);
61         return CopyResult::UnknownError;
62     }
63     if (rawSourceBuffer == nullptr) {
64         ALOGW("Surface doesn't have any previously queued frames, nothing to readback from");
65         return CopyResult::SourceEmpty;
66     }
67     UniqueAHardwareBuffer sourceBuffer{rawSourceBuffer};
68     AHardwareBuffer_Desc description;
69     AHardwareBuffer_describe(sourceBuffer.get(), &description);
70     if (description.usage & AHARDWAREBUFFER_USAGE_PROTECTED_CONTENT) {
71         ALOGW("Surface is protected, unable to copy from it");
72         return CopyResult::SourceInvalid;
73     }
74 
75     if (sourceFence != -1 && sync_wait(sourceFence.get(), 500 /* ms */) != NO_ERROR) {
76         ALOGE("Timeout (500ms) exceeded waiting for buffer fence, abandoning readback attempt");
77         return CopyResult::Timeout;
78     }
79 
80     sk_sp<SkColorSpace> colorSpace = DataSpaceToColorSpace(
81             static_cast<android_dataspace>(ANativeWindow_getBuffersDataSpace(window)));
82     sk_sp<SkImage> image =
83             SkImage::MakeFromAHardwareBuffer(sourceBuffer.get(), kPremul_SkAlphaType, colorSpace);
84 
85     if (!image.get()) {
86         return CopyResult::UnknownError;
87     }
88 
89     sk_sp<GrDirectContext> grContext = mRenderThread.requireGrContext();
90 
91     SkRect srcRect = inSrcRect.toSkRect();
92 
93     SkRect imageSrcRect = SkRect::MakeIWH(description.width, description.height);
94     SkISize imageWH = SkISize::Make(description.width, description.height);
95     if (cropRect.left < cropRect.right && cropRect.top < cropRect.bottom) {
96         imageSrcRect =
97                 SkRect::MakeLTRB(cropRect.left, cropRect.top, cropRect.right, cropRect.bottom);
98         imageWH = SkISize::Make(cropRect.right - cropRect.left, cropRect.bottom - cropRect.top);
99 
100         // Chroma channels of YUV420 images are subsampled we may need to shrink the crop region by
101         // a whole texel on each side. Since skia still adds its own 0.5 inset, we apply an
102         // additional 0.5 inset. See GLConsumer::computeTransformMatrix for details.
103         float shrinkAmount = 0.0f;
104         switch (description.format) {
105             // Use HAL formats since some AHB formats are only available in vndk
106             case HAL_PIXEL_FORMAT_YCBCR_420_888:
107             case HAL_PIXEL_FORMAT_YV12:
108             case HAL_PIXEL_FORMAT_IMPLEMENTATION_DEFINED:
109                 shrinkAmount = 0.5f;
110                 break;
111             default:
112                 break;
113         }
114 
115         // Shrink the crop if it has more than 1-px and differs from the buffer size.
116         if (imageWH.width() > 1 && imageWH.width() < (int32_t)description.width)
117             imageSrcRect = imageSrcRect.makeInset(shrinkAmount, 0);
118 
119         if (imageWH.height() > 1 && imageWH.height() < (int32_t)description.height)
120             imageSrcRect = imageSrcRect.makeInset(0, shrinkAmount);
121     }
122 
123     ALOGV("imageSrcRect = " RECT_STRING, SK_RECT_ARGS(imageSrcRect));
124 
125     // Represents the "logical" width/height of the texture. That is, the dimensions of the buffer
126     // after respecting crop & rotate. flipV/flipH still result in the same width & height
127     // so we can ignore those for this.
128     const SkRect textureRect =
129             (windowTransform & NATIVE_WINDOW_TRANSFORM_ROT_90)
130                     ? SkRect::MakeIWH(imageSrcRect.height(), imageSrcRect.width())
131                     : SkRect::MakeIWH(imageSrcRect.width(), imageSrcRect.height());
132 
133     if (srcRect.isEmpty()) {
134         srcRect = textureRect;
135     } else {
136         ALOGV("intersecting " RECT_STRING " with " RECT_STRING, SK_RECT_ARGS(srcRect),
137               SK_RECT_ARGS(textureRect));
138         if (!srcRect.intersect(textureRect)) {
139             return CopyResult::UnknownError;
140         }
141     }
142 
143     sk_sp<SkSurface> tmpSurface =
144             SkSurface::MakeRenderTarget(mRenderThread.getGrContext(), SkBudgeted::kYes,
145                                         bitmap->info(), 0, kTopLeft_GrSurfaceOrigin, nullptr);
146 
147     // if we can't generate a GPU surface that matches the destination bitmap (e.g. 565) then we
148     // attempt to do the intermediate rendering step in 8888
149     if (!tmpSurface.get()) {
150         SkImageInfo tmpInfo = bitmap->info().makeColorType(SkColorType::kN32_SkColorType);
151         tmpSurface = SkSurface::MakeRenderTarget(mRenderThread.getGrContext(), SkBudgeted::kYes,
152                                                  tmpInfo, 0, kTopLeft_GrSurfaceOrigin, nullptr);
153         if (!tmpSurface.get()) {
154             ALOGW("Unable to generate GPU buffer in a format compatible with the provided bitmap");
155             return CopyResult::UnknownError;
156         }
157     }
158 
159     /*
160      * The grand ordering of events.
161      * First we apply the buffer's crop, done by using a srcRect of the crop with a dstRect of the
162      * same width/height as the srcRect but with a 0x0 origin
163      *
164      * Second we apply the window transform via a Canvas matrix. Ordering for that is as follows:
165      *  1) FLIP_H
166      *  2) FLIP_V
167      *  3) ROT_90
168      * as per GLConsumer::computeTransformMatrix
169      *
170      * Third we apply the user's supplied cropping & scale to the output by doing a RectToRect
171      * matrix transform from srcRect to {0,0, bitmapWidth, bitmapHeight}
172      *
173      * Finally we're done messing with this bloody thing for hopefully the last time.
174      *
175      * That's a lie since...
176      * TODO: Do all this same stuff for TextureView as it's strictly more correct & easier
177      * to rationalize. And we can fix the 1-px crop bug.
178      */
179 
180     SkMatrix m;
181     const SkRect imageDstRect = SkRect::Make(imageWH);
182     const float px = imageDstRect.centerX();
183     const float py = imageDstRect.centerY();
184     if (windowTransform & NATIVE_WINDOW_TRANSFORM_FLIP_H) {
185         m.postScale(-1.f, 1.f, px, py);
186     }
187     if (windowTransform & NATIVE_WINDOW_TRANSFORM_FLIP_V) {
188         m.postScale(1.f, -1.f, px, py);
189     }
190     if (windowTransform & NATIVE_WINDOW_TRANSFORM_ROT_90) {
191         m.postRotate(90, 0, 0);
192         m.postTranslate(imageDstRect.height(), 0);
193     }
194 
195     SkSamplingOptions sampling(SkFilterMode::kNearest);
196     ALOGV("Mapping from " RECT_STRING " to " RECT_STRING, SK_RECT_ARGS(srcRect),
197           SK_RECT_ARGS(SkRect::MakeWH(bitmap->width(), bitmap->height())));
198     m.postConcat(SkMatrix::MakeRectToRect(srcRect,
199                                           SkRect::MakeWH(bitmap->width(), bitmap->height()),
200                                           SkMatrix::kFill_ScaleToFit));
201     if (srcRect.width() != bitmap->width() || srcRect.height() != bitmap->height()) {
202         sampling = SkSamplingOptions(SkFilterMode::kLinear);
203     }
204 
205     SkCanvas* canvas = tmpSurface->getCanvas();
206     canvas->save();
207     canvas->concat(m);
208     SkPaint paint;
209     paint.setAlpha(255);
210     paint.setBlendMode(SkBlendMode::kSrc);
211     const bool hasBufferCrop = cropRect.left < cropRect.right && cropRect.top < cropRect.bottom;
212     auto constraint =
213             hasBufferCrop ? SkCanvas::kStrict_SrcRectConstraint : SkCanvas::kFast_SrcRectConstraint;
214     canvas->drawImageRect(image, imageSrcRect, imageDstRect, sampling, &paint, constraint);
215     canvas->restore();
216 
217     if (!tmpSurface->readPixels(*bitmap, 0, 0)) {
218         // if we fail to readback from the GPU directly (e.g. 565) then we attempt to read into
219         // 8888 and then convert that into the destination format before giving up.
220         SkBitmap tmpBitmap;
221         SkImageInfo tmpInfo = bitmap->info().makeColorType(SkColorType::kN32_SkColorType);
222         if (bitmap->info().colorType() == SkColorType::kN32_SkColorType ||
223             !tmpBitmap.tryAllocPixels(tmpInfo) || !tmpSurface->readPixels(tmpBitmap, 0, 0) ||
224             !tmpBitmap.readPixels(bitmap->info(), bitmap->getPixels(), bitmap->rowBytes(), 0, 0)) {
225             ALOGW("Unable to convert content into the provided bitmap");
226             return CopyResult::UnknownError;
227         }
228     }
229 
230     bitmap->notifyPixelsChanged();
231 
232     return CopyResult::Success;
233 }
234 
copySurfaceIntoLegacy(ANativeWindow * window,const Rect & srcRect,SkBitmap * bitmap)235 CopyResult Readback::copySurfaceIntoLegacy(ANativeWindow* window, const Rect& srcRect,
236                                            SkBitmap* bitmap) {
237     // Setup the source
238     AHardwareBuffer* rawSourceBuffer;
239     int rawSourceFence;
240     Matrix4 texTransform;
241     status_t err = ANativeWindow_getLastQueuedBuffer(window, &rawSourceBuffer, &rawSourceFence,
242                                                      texTransform.data);
243     base::unique_fd sourceFence(rawSourceFence);
244     texTransform.invalidateType();
245     if (err != NO_ERROR) {
246         ALOGW("Failed to get last queued buffer, error = %d", err);
247         return CopyResult::UnknownError;
248     }
249     if (rawSourceBuffer == nullptr) {
250         ALOGW("Surface doesn't have any previously queued frames, nothing to readback from");
251         return CopyResult::SourceEmpty;
252     }
253 
254     UniqueAHardwareBuffer sourceBuffer{rawSourceBuffer};
255     AHardwareBuffer_Desc description;
256     AHardwareBuffer_describe(sourceBuffer.get(), &description);
257     if (description.usage & AHARDWAREBUFFER_USAGE_PROTECTED_CONTENT) {
258         ALOGW("Surface is protected, unable to copy from it");
259         return CopyResult::SourceInvalid;
260     }
261 
262     if (sourceFence != -1 && sync_wait(sourceFence.get(), 500 /* ms */) != NO_ERROR) {
263         ALOGE("Timeout (500ms) exceeded waiting for buffer fence, abandoning readback attempt");
264         return CopyResult::Timeout;
265     }
266 
267     sk_sp<SkColorSpace> colorSpace = DataSpaceToColorSpace(
268             static_cast<android_dataspace>(ANativeWindow_getBuffersDataSpace(window)));
269     sk_sp<SkImage> image =
270             SkImage::MakeFromAHardwareBuffer(sourceBuffer.get(), kPremul_SkAlphaType, colorSpace);
271     return copyImageInto(image, srcRect, bitmap);
272 }
273 
copyHWBitmapInto(Bitmap * hwBitmap,SkBitmap * bitmap)274 CopyResult Readback::copyHWBitmapInto(Bitmap* hwBitmap, SkBitmap* bitmap) {
275     LOG_ALWAYS_FATAL_IF(!hwBitmap->isHardware());
276 
277     Rect srcRect;
278     return copyImageInto(hwBitmap->makeImage(), srcRect, bitmap);
279 }
280 
copyLayerInto(DeferredLayerUpdater * deferredLayer,SkBitmap * bitmap)281 CopyResult Readback::copyLayerInto(DeferredLayerUpdater* deferredLayer, SkBitmap* bitmap) {
282     ATRACE_CALL();
283     if (!mRenderThread.getGrContext()) {
284         return CopyResult::UnknownError;
285     }
286 
287     // acquire most recent buffer for drawing
288     deferredLayer->updateTexImage();
289     deferredLayer->apply();
290     const SkRect dstRect = SkRect::MakeIWH(bitmap->width(), bitmap->height());
291     CopyResult copyResult = CopyResult::UnknownError;
292     Layer* layer = deferredLayer->backingLayer();
293     if (layer) {
294         if (copyLayerInto(layer, nullptr, &dstRect, bitmap)) {
295             copyResult = CopyResult::Success;
296         }
297     }
298     return copyResult;
299 }
300 
copyImageInto(const sk_sp<SkImage> & image,SkBitmap * bitmap)301 CopyResult Readback::copyImageInto(const sk_sp<SkImage>& image, SkBitmap* bitmap) {
302     Rect srcRect;
303     return copyImageInto(image, srcRect, bitmap);
304 }
305 
copyImageInto(const sk_sp<SkImage> & image,const Rect & srcRect,SkBitmap * bitmap)306 CopyResult Readback::copyImageInto(const sk_sp<SkImage>& image, const Rect& srcRect,
307                                    SkBitmap* bitmap) {
308     ATRACE_CALL();
309     if (Properties::getRenderPipelineType() == RenderPipelineType::SkiaGL) {
310         mRenderThread.requireGlContext();
311     } else {
312         mRenderThread.requireVkContext();
313     }
314     if (!image.get()) {
315         return CopyResult::UnknownError;
316     }
317     int imgWidth = image->width();
318     int imgHeight = image->height();
319     sk_sp<GrDirectContext> grContext = sk_ref_sp(mRenderThread.getGrContext());
320 
321     CopyResult copyResult = CopyResult::UnknownError;
322 
323     int displayedWidth = imgWidth, displayedHeight = imgHeight;
324     SkRect skiaDestRect = SkRect::MakeWH(bitmap->width(), bitmap->height());
325     SkRect skiaSrcRect = srcRect.toSkRect();
326     if (skiaSrcRect.isEmpty()) {
327         skiaSrcRect = SkRect::MakeIWH(displayedWidth, displayedHeight);
328     }
329     bool srcNotEmpty = skiaSrcRect.intersect(SkRect::MakeIWH(displayedWidth, displayedHeight));
330     if (!srcNotEmpty) {
331         return copyResult;
332     }
333 
334     Layer layer(mRenderThread.renderState(), nullptr, 255, SkBlendMode::kSrc);
335     layer.setSize(displayedWidth, displayedHeight);
336     layer.setImage(image);
337     // Scaling filter is not explicitly set here, because it is done inside copyLayerInfo
338     // after checking the necessity based on the src/dest rect size and the transformation.
339     if (copyLayerInto(&layer, &skiaSrcRect, &skiaDestRect, bitmap)) {
340         copyResult = CopyResult::Success;
341     }
342 
343     return copyResult;
344 }
345 
copyLayerInto(Layer * layer,const SkRect * srcRect,const SkRect * dstRect,SkBitmap * bitmap)346 bool Readback::copyLayerInto(Layer* layer, const SkRect* srcRect, const SkRect* dstRect,
347                              SkBitmap* bitmap) {
348     /* This intermediate surface is present to work around limitations that LayerDrawable expects
349      * to render into a GPU backed canvas.  Additionally, the offscreen buffer solution works around
350      * a scaling issue (b/62262733) that was encountered when sampling from an EGLImage into a
351      * software buffer.
352      */
353     sk_sp<SkSurface> tmpSurface = SkSurface::MakeRenderTarget(mRenderThread.getGrContext(),
354                                                               SkBudgeted::kYes, bitmap->info(), 0,
355                                                               kTopLeft_GrSurfaceOrigin, nullptr);
356 
357     // if we can't generate a GPU surface that matches the destination bitmap (e.g. 565) then we
358     // attempt to do the intermediate rendering step in 8888
359     if (!tmpSurface.get()) {
360         SkImageInfo tmpInfo = bitmap->info().makeColorType(SkColorType::kN32_SkColorType);
361         tmpSurface = SkSurface::MakeRenderTarget(mRenderThread.getGrContext(), SkBudgeted::kYes,
362                                                  tmpInfo, 0, kTopLeft_GrSurfaceOrigin, nullptr);
363         if (!tmpSurface.get()) {
364             ALOGW("Unable to generate GPU buffer in a format compatible with the provided bitmap");
365             return false;
366         }
367     }
368 
369     if (!skiapipeline::LayerDrawable::DrawLayer(mRenderThread.getGrContext(),
370                                                 tmpSurface->getCanvas(), layer, srcRect, dstRect,
371                                                 false)) {
372         ALOGW("Unable to draw content from GPU into the provided bitmap");
373         return false;
374     }
375 
376     if (!tmpSurface->readPixels(*bitmap, 0, 0)) {
377         // if we fail to readback from the GPU directly (e.g. 565) then we attempt to read into
378         // 8888 and then convert that into the destination format before giving up.
379         SkBitmap tmpBitmap;
380         SkImageInfo tmpInfo = bitmap->info().makeColorType(SkColorType::kN32_SkColorType);
381         if (bitmap->info().colorType() == SkColorType::kN32_SkColorType ||
382                 !tmpBitmap.tryAllocPixels(tmpInfo) ||
383                 !tmpSurface->readPixels(tmpBitmap, 0, 0) ||
384                 !tmpBitmap.readPixels(bitmap->info(), bitmap->getPixels(),
385                                       bitmap->rowBytes(), 0, 0)) {
386             ALOGW("Unable to convert content into the provided bitmap");
387             return false;
388         }
389     }
390 
391     bitmap->notifyPixelsChanged();
392     return true;
393 }
394 
395 } /* namespace uirenderer */
396 } /* namespace android */
397