1 /*
2 * Copyright 2018 Google Inc.
3 *
4 * Use of this source code is governed by a BSD-style license that can be
5 * found in the LICENSE file.
6 */
7
8 #include "src/image/SkImage_GpuYUVA.h"
9
10 #include "include/core/SkAlphaType.h"
11 #include "include/core/SkBitmap.h"
12 #include "include/core/SkColorType.h"
13 #include "include/core/SkImageInfo.h"
14 #include "include/core/SkMatrix.h"
15 #include "include/core/SkPixmap.h"
16 #include "include/core/SkSize.h"
17 #include "include/core/SkSurface.h"
18 #include "include/core/SkYUVAInfo.h"
19 #include "include/core/SkYUVAPixmaps.h"
20 #include "include/gpu/GpuTypes.h"
21 #include "include/gpu/GrBackendSurface.h" // IWYU pragma: keep
22 #include "include/gpu/GrContextThreadSafeProxy.h"
23 #include "include/gpu/GrDirectContext.h"
24 #include "include/gpu/GrRecordingContext.h"
25 #include "include/gpu/GrTypes.h"
26 #include "include/gpu/GrYUVABackendTextures.h"
27 #include "include/private/base/SkAssert.h"
28 #include "include/private/gpu/ganesh/GrImageContext.h"
29 #include "include/private/gpu/ganesh/GrTypesPriv.h"
30 #include "src/core/SkImageInfoPriv.h"
31 #include "src/core/SkSamplingPriv.h"
32 #include "src/gpu/RefCntedCallback.h"
33 #include "src/gpu/SkBackingFit.h"
34 #include "src/gpu/Swizzle.h"
35 #include "src/gpu/ganesh/GrCaps.h"
36 #include "src/gpu/ganesh/GrColorInfo.h"
37 #include "src/gpu/ganesh/GrColorSpaceXform.h"
38 #include "src/gpu/ganesh/GrDirectContextPriv.h"
39 #include "src/gpu/ganesh/GrFragmentProcessor.h"
40 #include "src/gpu/ganesh/GrImageContextPriv.h"
41 #include "src/gpu/ganesh/GrImageInfo.h"
42 #include "src/gpu/ganesh/GrProxyProvider.h"
43 #include "src/gpu/ganesh/GrRecordingContextPriv.h"
44 #include "src/gpu/ganesh/GrSamplerState.h"
45 #include "src/gpu/ganesh/GrSurfaceProxy.h"
46 #include "src/gpu/ganesh/GrSurfaceProxyView.h"
47 #include "src/gpu/ganesh/GrTextureProxy.h"
48 #include "src/gpu/ganesh/SkGr.h"
49 #include "src/gpu/ganesh/SurfaceFillContext.h"
50 #include "src/gpu/ganesh/effects/GrBicubicEffect.h"
51 #include "src/gpu/ganesh/effects/GrYUVtoRGBEffect.h"
52 #include "src/image/SkImage_Base.h"
53
54 #include <algorithm>
55 #include <utility>
56
57 enum class SkTileMode;
58 struct SkRect;
59
60 static constexpr auto kAssumedColorType = kRGBA_8888_SkColorType;
61
SkImage_GpuYUVA(sk_sp<GrImageContext> context,uint32_t uniqueID,GrYUVATextureProxies proxies,sk_sp<SkColorSpace> imageColorSpace)62 SkImage_GpuYUVA::SkImage_GpuYUVA(sk_sp<GrImageContext> context,
63 uint32_t uniqueID,
64 GrYUVATextureProxies proxies,
65 sk_sp<SkColorSpace> imageColorSpace)
66 : INHERITED(std::move(context),
67 SkImageInfo::Make(proxies.yuvaInfo().dimensions(),
68 kAssumedColorType,
69 // If an alpha channel is present we always use kPremul. This
70 // is because, although the planar data is always un-premul,
71 // the final interleaved RGBA sample produced in the shader
72 // is premul (and similar if flattened via asView).
73 proxies.yuvaInfo().hasAlpha() ? kPremul_SkAlphaType
74 : kOpaque_SkAlphaType,
75 std::move(imageColorSpace)),
76 uniqueID)
77 , fYUVAProxies(std::move(proxies)) {
78 // The caller should have checked this, just verifying.
79 SkASSERT(fYUVAProxies.isValid());
80 }
81
82 // For onMakeColorSpace()
SkImage_GpuYUVA(sk_sp<GrImageContext> context,const SkImage_GpuYUVA * image,sk_sp<SkColorSpace> targetCS)83 SkImage_GpuYUVA::SkImage_GpuYUVA(sk_sp<GrImageContext> context,
84 const SkImage_GpuYUVA* image,
85 sk_sp<SkColorSpace> targetCS)
86 : INHERITED(std::move(context),
87 image->imageInfo().makeColorSpace(std::move(targetCS)),
88 kNeedNewImageUniqueID)
89 , fYUVAProxies(image->fYUVAProxies)
90 // Since null fFromColorSpace means no GrColorSpaceXform, we turn a null
91 // image->refColorSpace() into an explicit SRGB.
92 , fFromColorSpace(image->colorSpace() ? image->refColorSpace() : SkColorSpace::MakeSRGB()) {
93 }
94
setupMipmapsForPlanes(GrRecordingContext * context) const95 bool SkImage_GpuYUVA::setupMipmapsForPlanes(GrRecordingContext* context) const {
96 if (!context || !fContext->priv().matches(context)) {
97 return false;
98 }
99 if (!context->priv().caps()->mipmapSupport()) {
100 // We succeed in this case by doing nothing.
101 return true;
102 }
103 int n = fYUVAProxies.yuvaInfo().numPlanes();
104 sk_sp<GrSurfaceProxy> newProxies[4];
105 for (int i = 0; i < n; ++i) {
106 auto* t = fYUVAProxies.proxy(i)->asTextureProxy();
107 if (t->mipmapped() == GrMipmapped::kNo && (t->width() > 1 || t->height() > 1)) {
108 auto newView = GrCopyBaseMipMapToView(context, fYUVAProxies.makeView(i));
109 if (!newView) {
110 return false;
111 }
112 SkASSERT(newView.swizzle() == fYUVAProxies.makeView(i).swizzle());
113 newProxies[i] = newView.detachProxy();
114 } else {
115 newProxies[i] = fYUVAProxies.refProxy(i);
116 }
117 }
118 fYUVAProxies = GrYUVATextureProxies(fYUVAProxies.yuvaInfo(),
119 newProxies,
120 fYUVAProxies.textureOrigin());
121 SkASSERT(fYUVAProxies.isValid());
122 return true;
123 }
124
125 //////////////////////////////////////////////////////////////////////////////////////////////////
126
onFlush(GrDirectContext * dContext,const GrFlushInfo & info) const127 GrSemaphoresSubmitted SkImage_GpuYUVA::onFlush(GrDirectContext* dContext,
128 const GrFlushInfo& info) const {
129 if (!fContext->priv().matches(dContext) || dContext->abandoned()) {
130 if (info.fSubmittedProc) {
131 info.fSubmittedProc(info.fSubmittedContext, false);
132 }
133 if (info.fFinishedProc) {
134 info.fFinishedProc(info.fFinishedContext);
135 }
136 return GrSemaphoresSubmitted::kNo;
137 }
138
139 GrSurfaceProxy* proxies[SkYUVAInfo::kMaxPlanes] = {};
140 size_t numProxies = fYUVAProxies.numPlanes();
141 for (size_t i = 0; i < numProxies; ++i) {
142 proxies[i] = fYUVAProxies.proxy(i);
143 }
144 return dContext->priv().flushSurfaces({proxies, numProxies},
145 SkSurface::BackendSurfaceAccess::kNoAccess,
146 info);
147 }
148
onHasMipmaps() const149 bool SkImage_GpuYUVA::onHasMipmaps() const { return fYUVAProxies.mipmapped() == GrMipmapped::kYes; }
150
onTextureSize() const151 size_t SkImage_GpuYUVA::onTextureSize() const {
152 size_t size = 0;
153 for (int i = 0; i < fYUVAProxies.numPlanes(); ++i) {
154 size += fYUVAProxies.proxy(i)->gpuMemorySize();
155 }
156 return size;
157 }
158
onMakeColorTypeAndColorSpace(SkColorType,sk_sp<SkColorSpace> targetCS,GrDirectContext * direct) const159 sk_sp<SkImage> SkImage_GpuYUVA::onMakeColorTypeAndColorSpace(SkColorType,
160 sk_sp<SkColorSpace> targetCS,
161 GrDirectContext* direct) const {
162 // We explicitly ignore color type changes, for now.
163
164 // we may need a mutex here but for now we expect usage to be in a single thread
165 if (fOnMakeColorSpaceTarget &&
166 SkColorSpace::Equals(targetCS.get(), fOnMakeColorSpaceTarget.get())) {
167 return fOnMakeColorSpaceResult;
168 }
169 sk_sp<SkImage> result = sk_sp<SkImage>(new SkImage_GpuYUVA(sk_ref_sp(direct), this, targetCS));
170 if (result) {
171 fOnMakeColorSpaceTarget = targetCS;
172 fOnMakeColorSpaceResult = result;
173 }
174 return result;
175 }
176
onReinterpretColorSpace(sk_sp<SkColorSpace> newCS) const177 sk_sp<SkImage> SkImage_GpuYUVA::onReinterpretColorSpace(sk_sp<SkColorSpace> newCS) const {
178 return sk_sp<SkImage>(new SkImage_GpuYUVA(fContext, this, std::move(newCS)));
179 }
180
onAsView(GrRecordingContext * rContext,GrMipmapped mipmapped,GrImageTexGenPolicy) const181 std::tuple<GrSurfaceProxyView, GrColorType> SkImage_GpuYUVA::onAsView(
182 GrRecordingContext* rContext,
183 GrMipmapped mipmapped,
184 GrImageTexGenPolicy) const {
185 if (!fContext->priv().matches(rContext)) {
186 return {};
187 }
188 auto sfc = rContext->priv().makeSFC(this->imageInfo(),
189 "Image_GpuYUVA_ReinterpretColorSpace",
190 SkBackingFit::kExact,
191 /*sample count*/ 1,
192 mipmapped,
193 GrProtected::kNo,
194 kTopLeft_GrSurfaceOrigin,
195 skgpu::Budgeted::kYes);
196 if (!sfc) {
197 return {};
198 }
199
200 const GrCaps& caps = *rContext->priv().caps();
201 auto fp = GrYUVtoRGBEffect::Make(fYUVAProxies, GrSamplerState::Filter::kNearest, caps);
202 if (fFromColorSpace) {
203 fp = GrColorSpaceXformEffect::Make(std::move(fp),
204 fFromColorSpace.get(), this->alphaType(),
205 this->colorSpace() , this->alphaType());
206 }
207 sfc->fillWithFP(std::move(fp));
208
209 return {sfc->readSurfaceView(), sfc->colorInfo().colorType()};
210 }
211
onAsFragmentProcessor(GrRecordingContext * context,SkSamplingOptions sampling,const SkTileMode tileModes[2],const SkMatrix & m,const SkRect * subset,const SkRect * domain) const212 std::unique_ptr<GrFragmentProcessor> SkImage_GpuYUVA::onAsFragmentProcessor(
213 GrRecordingContext* context,
214 SkSamplingOptions sampling,
215 const SkTileMode tileModes[2],
216 const SkMatrix& m,
217 const SkRect* subset,
218 const SkRect* domain) const {
219 if (!fContext->priv().matches(context)) {
220 return {};
221 }
222 // At least for now we do not attempt aniso filtering on YUVA images.
223 if (sampling.isAniso()) {
224 sampling = SkSamplingPriv::AnisoFallback(fYUVAProxies.mipmapped() == GrMipmapped::kYes);
225 }
226
227 auto wmx = SkTileModeToWrapMode(tileModes[0]);
228 auto wmy = SkTileModeToWrapMode(tileModes[1]);
229 GrSamplerState sampler(wmx, wmy, sampling.filter, sampling.mipmap);
230 if (sampler.mipmapped() == GrMipmapped::kYes && !this->setupMipmapsForPlanes(context)) {
231 sampler = GrSamplerState(sampler.wrapModeX(),
232 sampler.wrapModeY(),
233 sampler.filter(),
234 GrSamplerState::MipmapMode::kNone);
235 }
236
237 const auto& yuvM = sampling.useCubic ? SkMatrix::I() : m;
238 auto fp = GrYUVtoRGBEffect::Make(fYUVAProxies,
239 sampler,
240 *context->priv().caps(),
241 yuvM,
242 subset,
243 domain);
244 if (sampling.useCubic) {
245 fp = GrBicubicEffect::Make(std::move(fp),
246 this->alphaType(),
247 m,
248 sampling.cubic,
249 GrBicubicEffect::Direction::kXY);
250 }
251 if (fFromColorSpace) {
252 fp = GrColorSpaceXformEffect::Make(std::move(fp),
253 fFromColorSpace.get(), this->alphaType(),
254 this->colorSpace() , this->alphaType());
255 }
256 return fp;
257 }
258
259 //////////////////////////////////////////////////////////////////////////////////////////////////
MakeFromYUVATextures(GrRecordingContext * context,const GrYUVABackendTextures & yuvaTextures)260 sk_sp<SkImage> SkImage::MakeFromYUVATextures(GrRecordingContext* context,
261 const GrYUVABackendTextures& yuvaTextures) {
262 return SkImage::MakeFromYUVATextures(context, yuvaTextures, nullptr, nullptr, nullptr);
263 }
264
MakeFromYUVATextures(GrRecordingContext * context,const GrYUVABackendTextures & yuvaTextures,sk_sp<SkColorSpace> imageColorSpace,TextureReleaseProc textureReleaseProc,ReleaseContext releaseContext)265 sk_sp<SkImage> SkImage::MakeFromYUVATextures(GrRecordingContext* context,
266 const GrYUVABackendTextures& yuvaTextures,
267 sk_sp<SkColorSpace> imageColorSpace,
268 TextureReleaseProc textureReleaseProc,
269 ReleaseContext releaseContext) {
270 auto releaseHelper = skgpu::RefCntedCallback::Make(textureReleaseProc, releaseContext);
271
272 GrProxyProvider* proxyProvider = context->priv().proxyProvider();
273 int numPlanes = yuvaTextures.yuvaInfo().numPlanes();
274 sk_sp<GrSurfaceProxy> proxies[SkYUVAInfo::kMaxPlanes];
275 for (int plane = 0; plane < numPlanes; ++plane) {
276 proxies[plane] = proxyProvider->wrapBackendTexture(yuvaTextures.texture(plane),
277 kBorrow_GrWrapOwnership,
278 GrWrapCacheable::kNo,
279 kRead_GrIOType,
280 releaseHelper);
281 if (!proxies[plane]) {
282 return {};
283 }
284 }
285 GrYUVATextureProxies yuvaProxies(yuvaTextures.yuvaInfo(),
286 proxies,
287 yuvaTextures.textureOrigin());
288
289 if (!yuvaProxies.isValid()) {
290 return nullptr;
291 }
292
293 return sk_make_sp<SkImage_GpuYUVA>(sk_ref_sp(context),
294 kNeedNewImageUniqueID,
295 yuvaProxies,
296 imageColorSpace);
297 }
298
MakeFromYUVAPixmaps(GrRecordingContext * context,const SkYUVAPixmaps & pixmaps,GrMipmapped buildMips,bool limitToMaxTextureSize)299 sk_sp<SkImage> SkImage::MakeFromYUVAPixmaps(GrRecordingContext* context,
300 const SkYUVAPixmaps& pixmaps,
301 GrMipmapped buildMips,
302 bool limitToMaxTextureSize) {
303 return SkImage::MakeFromYUVAPixmaps(context, pixmaps, buildMips, limitToMaxTextureSize,
304 nullptr);
305 }
306
MakeFromYUVAPixmaps(GrRecordingContext * context,const SkYUVAPixmaps & pixmaps,GrMipmapped buildMips,bool limitToMaxTextureSize,sk_sp<SkColorSpace> imageColorSpace)307 sk_sp<SkImage> SkImage::MakeFromYUVAPixmaps(GrRecordingContext* context,
308 const SkYUVAPixmaps& pixmaps,
309 GrMipmapped buildMips,
310 bool limitToMaxTextureSize,
311 sk_sp<SkColorSpace> imageColorSpace) {
312 if (!context) {
313 return nullptr; // until we impl this for raster backend
314 }
315
316 if (!pixmaps.isValid()) {
317 return nullptr;
318 }
319
320 if (!context->priv().caps()->mipmapSupport()) {
321 buildMips = GrMipmapped::kNo;
322 }
323
324 // Resize the pixmaps if necessary.
325 int numPlanes = pixmaps.numPlanes();
326 int maxTextureSize = context->priv().caps()->maxTextureSize();
327 int maxDim = std::max(pixmaps.yuvaInfo().width(), pixmaps.yuvaInfo().height());
328
329 SkYUVAPixmaps tempPixmaps;
330 const SkYUVAPixmaps* pixmapsToUpload = &pixmaps;
331 // We assume no plane is larger than the image size (and at least one plane is as big).
332 if (maxDim > maxTextureSize) {
333 if (!limitToMaxTextureSize) {
334 return nullptr;
335 }
336 float scale = static_cast<float>(maxTextureSize)/maxDim;
337 SkISize newDimensions = {
338 std::min(static_cast<int>(pixmaps.yuvaInfo().width() *scale), maxTextureSize),
339 std::min(static_cast<int>(pixmaps.yuvaInfo().height()*scale), maxTextureSize)
340 };
341 SkYUVAInfo newInfo = pixmaps.yuvaInfo().makeDimensions(newDimensions);
342 SkYUVAPixmapInfo newPixmapInfo(newInfo, pixmaps.dataType(), /*row bytes*/ nullptr);
343 tempPixmaps = SkYUVAPixmaps::Allocate(newPixmapInfo);
344 SkSamplingOptions sampling(SkFilterMode::kLinear);
345 if (!tempPixmaps.isValid()) {
346 return nullptr;
347 }
348 for (int i = 0; i < numPlanes; ++i) {
349 if (!pixmaps.plane(i).scalePixels(tempPixmaps.plane(i), sampling)) {
350 return nullptr;
351 }
352 }
353 pixmapsToUpload = &tempPixmaps;
354 }
355
356 // Convert to texture proxies.
357 GrSurfaceProxyView views[SkYUVAInfo::kMaxPlanes];
358 GrColorType pixmapColorTypes[SkYUVAInfo::kMaxPlanes];
359 for (int i = 0; i < numPlanes; ++i) {
360 // Turn the pixmap into a GrTextureProxy
361 SkBitmap bmp;
362 bmp.installPixels(pixmapsToUpload->plane(i));
363 std::tie(views[i], std::ignore) = GrMakeUncachedBitmapProxyView(context, bmp, buildMips);
364 if (!views[i]) {
365 return nullptr;
366 }
367 pixmapColorTypes[i] = SkColorTypeToGrColorType(bmp.colorType());
368 }
369
370 GrYUVATextureProxies yuvaProxies(pixmapsToUpload->yuvaInfo(), views, pixmapColorTypes);
371 SkASSERT(yuvaProxies.isValid());
372 return sk_make_sp<SkImage_GpuYUVA>(sk_ref_sp(context),
373 kNeedNewImageUniqueID,
374 std::move(yuvaProxies),
375 std::move(imageColorSpace));
376 }
377
378 /////////////////////////////////////////////////////////////////////////////////////////////////
379
MakePromiseYUVATexture(sk_sp<GrContextThreadSafeProxy> threadSafeProxy,const GrYUVABackendTextureInfo & backendTextureInfo,sk_sp<SkColorSpace> imageColorSpace,PromiseImageTextureFulfillProc textureFulfillProc,PromiseImageTextureReleaseProc textureReleaseProc,PromiseImageTextureContext textureContexts[])380 sk_sp<SkImage> SkImage::MakePromiseYUVATexture(sk_sp<GrContextThreadSafeProxy> threadSafeProxy,
381 const GrYUVABackendTextureInfo& backendTextureInfo,
382 sk_sp<SkColorSpace> imageColorSpace,
383 PromiseImageTextureFulfillProc textureFulfillProc,
384 PromiseImageTextureReleaseProc textureReleaseProc,
385 PromiseImageTextureContext textureContexts[]) {
386 if (!backendTextureInfo.isValid()) {
387 return nullptr;
388 }
389
390 SkISize planeDimensions[SkYUVAInfo::kMaxPlanes];
391 int n = backendTextureInfo.yuvaInfo().planeDimensions(planeDimensions);
392
393 // Our contract is that we will always call the release proc even on failure.
394 // We use the helper to convey the context, so we need to ensure make doesn't fail.
395 textureReleaseProc = textureReleaseProc ? textureReleaseProc : [](void*) {};
396 sk_sp<skgpu::RefCntedCallback> releaseHelpers[4];
397 for (int i = 0; i < n; ++i) {
398 releaseHelpers[i] = skgpu::RefCntedCallback::Make(textureReleaseProc, textureContexts[i]);
399 }
400
401 if (!threadSafeProxy) {
402 return nullptr;
403 }
404
405 SkAlphaType at = backendTextureInfo.yuvaInfo().hasAlpha() ? kPremul_SkAlphaType
406 : kOpaque_SkAlphaType;
407 SkImageInfo info = SkImageInfo::Make(backendTextureInfo.yuvaInfo().dimensions(),
408 kAssumedColorType, at, imageColorSpace);
409 if (!SkImageInfoIsValid(info)) {
410 return nullptr;
411 }
412
413 // Make a lazy proxy for each plane and wrap in a view.
414 sk_sp<GrSurfaceProxy> proxies[4];
415 for (int i = 0; i < n; ++i) {
416 proxies[i] = SkImage_GpuBase::MakePromiseImageLazyProxy(threadSafeProxy.get(),
417 planeDimensions[i],
418 backendTextureInfo.planeFormat(i),
419 GrMipmapped::kNo,
420 textureFulfillProc,
421 std::move(releaseHelpers[i]));
422 if (!proxies[i]) {
423 return nullptr;
424 }
425 }
426 GrYUVATextureProxies yuvaTextureProxies(backendTextureInfo.yuvaInfo(),
427 proxies,
428 backendTextureInfo.textureOrigin());
429 SkASSERT(yuvaTextureProxies.isValid());
430 sk_sp<GrImageContext> ctx(GrImageContextPriv::MakeForPromiseImage(std::move(threadSafeProxy)));
431 return sk_make_sp<SkImage_GpuYUVA>(std::move(ctx),
432 kNeedNewImageUniqueID,
433 std::move(yuvaTextureProxies),
434 std::move(imageColorSpace));
435 }
436