1 /*
2 * Copyright 2018 Google Inc.
3 *
4 * Use of this source code is governed by a BSD-style license that can be
5 * found in the LICENSE file.
6 */
7
8 #include "src/gpu/ganesh/effects/GrYUVtoRGBEffect.h"
9
10 #include "include/core/SkYUVAInfo.h"
11 #include "src/core/SkYUVMath.h"
12 #include "src/gpu/KeyBuilder.h"
13 #include "src/gpu/ganesh/GrTexture.h"
14 #include "src/gpu/ganesh/GrYUVATextureProxies.h"
15 #include "src/gpu/ganesh/effects/GrMatrixEffect.h"
16 #include "src/gpu/ganesh/effects/GrTextureEffect.h"
17 #include "src/gpu/ganesh/glsl/GrGLSLFragmentShaderBuilder.h"
18 #include "src/gpu/ganesh/glsl/GrGLSLProgramBuilder.h"
19 #include "src/sksl/SkSLUtil.h"
20
border_colors(const GrYUVATextureProxies & yuvaProxies,float planeBorders[4][4])21 static void border_colors(const GrYUVATextureProxies& yuvaProxies, float planeBorders[4][4]) {
22 float m[20];
23 SkColorMatrix_RGB2YUV(yuvaProxies.yuvaInfo().yuvColorSpace(), m);
24 for (int i = 0; i < SkYUVAInfo::kYUVAChannelCount; ++i) {
25 auto [plane, channel] = yuvaProxies.yuvaLocations()[i];
26 if (plane == -1) {
27 return;
28 }
29 auto c = static_cast<int>(channel);
30 planeBorders[plane][c] = m[i*5 + 4];
31 }
32 }
33
Make(const GrYUVATextureProxies & yuvaProxies,GrSamplerState samplerState,const GrCaps & caps,const SkMatrix & localMatrix,const SkRect * subset,const SkRect * domain)34 std::unique_ptr<GrFragmentProcessor> GrYUVtoRGBEffect::Make(const GrYUVATextureProxies& yuvaProxies,
35 GrSamplerState samplerState,
36 const GrCaps& caps,
37 const SkMatrix& localMatrix,
38 const SkRect* subset,
39 const SkRect* domain) {
40 SkASSERT(!subset || SkRect::Make(yuvaProxies.yuvaInfo().dimensions()).contains(*subset));
41
42 int numPlanes = yuvaProxies.yuvaInfo().numPlanes();
43 if (!yuvaProxies.isValid()) {
44 return nullptr;
45 }
46
47 bool usesBorder = samplerState.wrapModeX() == GrSamplerState::WrapMode::kClampToBorder ||
48 samplerState.wrapModeY() == GrSamplerState::WrapMode::kClampToBorder;
49 float planeBorders[4][4] = {};
50 if (usesBorder) {
51 border_colors(yuvaProxies, planeBorders);
52 }
53
54 bool snap[2] = {false, false};
55 std::unique_ptr<GrFragmentProcessor> planeFPs[SkYUVAInfo::kMaxPlanes];
56 for (int i = 0; i < numPlanes; ++i) {
57 bool useSubset = SkToBool(subset);
58 GrSurfaceProxyView view = yuvaProxies.makeView(i);
59 SkMatrix planeMatrix = yuvaProxies.yuvaInfo().originMatrix();
60 // The returned matrix is a view matrix but we need a local matrix.
61 SkAssertResult(planeMatrix.invert(&planeMatrix));
62 SkRect planeSubset;
63 SkRect planeDomain;
64 bool makeLinearWithSnap = false;
65 auto [ssx, ssy] = yuvaProxies.yuvaInfo().planeSubsamplingFactors(i);
66 SkASSERT(ssx > 0 && ssx <= 4);
67 SkASSERT(ssy > 0 && ssy <= 2);
68 float scaleX = 1.f;
69 float scaleY = 1.f;
70 if (ssx > 1 || ssy > 1) {
71 scaleX = 1.f/ssx;
72 scaleY = 1.f/ssy;
73 // We would want to add a translation to this matrix to handle other sitings.
74 SkASSERT(yuvaProxies.yuvaInfo().sitingX() == SkYUVAInfo::Siting::kCentered);
75 SkASSERT(yuvaProxies.yuvaInfo().sitingY() == SkYUVAInfo::Siting::kCentered);
76 planeMatrix.postConcat(SkMatrix::Scale(scaleX, scaleY));
77 if (subset) {
78 planeSubset = {subset->fLeft *scaleX,
79 subset->fTop *scaleY,
80 subset->fRight *scaleX,
81 subset->fBottom*scaleY};
82 } else {
83 planeSubset = SkRect::Make(view.dimensions());
84 }
85 if (domain) {
86 planeDomain = {domain->fLeft *scaleX,
87 domain->fTop *scaleY,
88 domain->fRight *scaleX,
89 domain->fBottom*scaleY};
90 }
91 // If the image is not a multiple of the subsampling then the subsampled plane needs to
92 // be tiled at less than its full width/height. This only matters when the mode is not
93 // clamp.
94 if (samplerState.wrapModeX() != GrSamplerState::WrapMode::kClamp) {
95 int dx = (ssx*view.width() - yuvaProxies.yuvaInfo().width());
96 float maxRight = view.width() - dx*scaleX;
97 if (planeSubset.fRight > maxRight) {
98 planeSubset.fRight = maxRight;
99 useSubset = true;
100 }
101 }
102 if (samplerState.wrapModeY() != GrSamplerState::WrapMode::kClamp) {
103 int dy = (ssy*view.height() - yuvaProxies.yuvaInfo().height());
104 float maxBottom = view.height() - dy*scaleY;
105 if (planeSubset.fBottom > maxBottom) {
106 planeSubset.fBottom = maxBottom;
107 useSubset = true;
108 }
109 }
110 // This promotion of nearest to linear filtering for UV planes exists to mimic
111 // libjpeg[-turbo]'s do_fancy_upsampling option. We will filter the subsampled plane,
112 // however we want to filter at a fixed point for each logical image pixel to simulate
113 // nearest neighbor.
114 if (samplerState.filter() == GrSamplerState::Filter::kNearest) {
115 bool snapX = (ssx != 1),
116 snapY = (ssy != 1);
117 makeLinearWithSnap = snapX || snapY;
118 snap[0] |= snapX;
119 snap[1] |= snapY;
120 if (domain) {
121 // The outer YUVToRGB effect will ensure sampling happens at pixel centers
122 // within this plane.
123 planeDomain = {std::floor(planeDomain.fLeft) + 0.5f,
124 std::floor(planeDomain.fTop) + 0.5f,
125 std::floor(planeDomain.fRight) + 0.5f,
126 std::floor(planeDomain.fBottom) + 0.5f};
127 }
128 }
129 } else {
130 if (subset) {
131 planeSubset = *subset;
132 }
133 if (domain) {
134 planeDomain = *domain;
135 }
136 }
137 if (useSubset) {
138 if (makeLinearWithSnap) {
139 // The plane is subsampled and we have an overall subset on the image. We're
140 // emulating do_fancy_upsampling using linear filtering but snapping look ups to the
141 // y-plane pixel centers. Consider a logical image pixel at the edge of the subset.
142 // When computing the logical pixel color value we should use a 50/50 blend of two
143 // values from the subsampled plane. Depending on where the subset edge falls in
144 // actual subsampled plane, one of those values may come from outside the subset.
145 // Hence, we use this custom inset factory which applies the wrap mode to
146 // planeSubset but allows linear filtering to read pixels from the plane that are
147 // just outside planeSubset.
148 SkRect* domainRect = domain ? &planeDomain : nullptr;
149 planeFPs[i] = GrTextureEffect::MakeCustomLinearFilterInset(std::move(view),
150 kUnknown_SkAlphaType,
151 planeMatrix,
152 samplerState.wrapModeX(),
153 samplerState.wrapModeY(),
154 planeSubset,
155 domainRect,
156 {scaleX/2.f, scaleY/2.f},
157 caps,
158 planeBorders[i]);
159 } else if (domain) {
160 planeFPs[i] = GrTextureEffect::MakeSubset(std::move(view),
161 kUnknown_SkAlphaType,
162 planeMatrix,
163 samplerState,
164 planeSubset,
165 planeDomain,
166 caps,
167 planeBorders[i]);
168 } else {
169 planeFPs[i] = GrTextureEffect::MakeSubset(std::move(view),
170 kUnknown_SkAlphaType,
171 planeMatrix,
172 samplerState,
173 planeSubset,
174 caps,
175 planeBorders[i]);
176 }
177 } else {
178 GrSamplerState planeSampler = samplerState;
179 if (makeLinearWithSnap) {
180 planeSampler = GrSamplerState(samplerState.wrapModeX(),
181 samplerState.wrapModeY(),
182 GrSamplerState::Filter::kLinear,
183 samplerState.mipmapMode());
184 }
185 planeFPs[i] = GrTextureEffect::Make(std::move(view),
186 kUnknown_SkAlphaType,
187 planeMatrix,
188 planeSampler,
189 caps,
190 planeBorders[i]);
191 }
192 }
193 std::unique_ptr<GrFragmentProcessor> fp(
194 new GrYUVtoRGBEffect(planeFPs,
195 numPlanes,
196 yuvaProxies.yuvaLocations(),
197 snap,
198 yuvaProxies.yuvaInfo().yuvColorSpace()));
199 return GrMatrixEffect::Make(localMatrix, std::move(fp));
200 }
201
alpha_type(const SkYUVAInfo::YUVALocations locations)202 static SkAlphaType alpha_type(const SkYUVAInfo::YUVALocations locations) {
203 return locations[SkYUVAInfo::YUVAChannels::kA].fPlane >= 0 ? kPremul_SkAlphaType
204 : kOpaque_SkAlphaType;
205 }
206
GrYUVtoRGBEffect(std::unique_ptr<GrFragmentProcessor> planeFPs[4],int numPlanes,const SkYUVAInfo::YUVALocations & locations,const bool snap[2],SkYUVColorSpace yuvColorSpace)207 GrYUVtoRGBEffect::GrYUVtoRGBEffect(std::unique_ptr<GrFragmentProcessor> planeFPs[4],
208 int numPlanes,
209 const SkYUVAInfo::YUVALocations& locations,
210 const bool snap[2],
211 SkYUVColorSpace yuvColorSpace)
212 : GrFragmentProcessor(kGrYUVtoRGBEffect_ClassID,
213 ModulateForClampedSamplerOptFlags(alpha_type(locations)))
214 , fLocations(locations)
215 , fYUVColorSpace(yuvColorSpace) {
216 std::copy_n(snap, 2, fSnap);
217
218 if (fSnap[0] || fSnap[1]) {
219 // Need this so that we can access coords in SKSL to perform snapping.
220 this->setUsesSampleCoordsDirectly();
221 for (int i = 0; i < numPlanes; ++i) {
222 this->registerChild(std::move(planeFPs[i]), SkSL::SampleUsage::Explicit());
223 }
224 } else {
225 for (int i = 0; i < numPlanes; ++i) {
226 this->registerChild(std::move(planeFPs[i]));
227 }
228 }
229 }
230
231 #if GR_TEST_UTILS
onDumpInfo() const232 SkString GrYUVtoRGBEffect::onDumpInfo() const {
233 SkString str("(");
234 for (int i = 0; i < SkYUVAInfo::kYUVAChannelCount; ++i) {
235 str.appendf("Locations[%d]=%d %d, ",
236 i, fLocations[i].fPlane, static_cast<int>(fLocations[i].fChannel));
237 }
238 str.appendf("YUVColorSpace=%d, snap=(%d, %d))",
239 static_cast<int>(fYUVColorSpace), fSnap[0], fSnap[1]);
240 return str;
241 }
242 #endif
243
onMakeProgramImpl() const244 std::unique_ptr<GrFragmentProcessor::ProgramImpl> GrYUVtoRGBEffect::onMakeProgramImpl() const {
245 class Impl : public ProgramImpl {
246 public:
247 void emitCode(EmitArgs& args) override {
248 GrGLSLFPFragmentBuilder* fragBuilder = args.fFragBuilder;
249 const GrYUVtoRGBEffect& yuvEffect = args.fFp.cast<GrYUVtoRGBEffect>();
250
251 int numPlanes = yuvEffect.numChildProcessors();
252
253 const char* sampleCoords = "";
254 if (yuvEffect.fSnap[0] || yuvEffect.fSnap[1]) {
255 fragBuilder->codeAppendf("float2 snappedCoords = %s;", args.fSampleCoord);
256 if (yuvEffect.fSnap[0]) {
257 fragBuilder->codeAppend("snappedCoords.x = floor(snappedCoords.x) + 0.5;");
258 }
259 if (yuvEffect.fSnap[1]) {
260 fragBuilder->codeAppend("snappedCoords.y = floor(snappedCoords.y) + 0.5;");
261 }
262 sampleCoords = "snappedCoords";
263 }
264
265 fragBuilder->codeAppendf("half4 color;");
266 const bool hasAlpha = yuvEffect.fLocations[SkYUVAInfo::YUVAChannels::kA].fPlane >= 0;
267
268 for (int planeIdx = 0; planeIdx < numPlanes; ++planeIdx) {
269 std::string colorChannel;
270 std::string planeChannel;
271 for (int locIdx = 0; locIdx < (hasAlpha ? 4 : 3); ++locIdx) {
272 auto [yuvPlane, yuvChannel] = yuvEffect.fLocations[locIdx];
273 if (yuvPlane == planeIdx) {
274 colorChannel.push_back("rgba"[locIdx]);
275 planeChannel.push_back("rgba"[static_cast<int>(yuvChannel)]);
276 }
277 }
278
279 SkASSERT(colorChannel.size() == planeChannel.size());
280 if (!colorChannel.empty()) {
281 fragBuilder->codeAppendf(
282 "color.%s = (%s).%s;",
283 colorChannel.c_str(),
284 this->invokeChild(planeIdx, args, sampleCoords).c_str(),
285 planeChannel.c_str());
286 }
287 }
288
289 if (!hasAlpha) {
290 fragBuilder->codeAppendf("color.a = 1;");
291 }
292
293 if (kIdentity_SkYUVColorSpace != yuvEffect.fYUVColorSpace) {
294 fColorSpaceMatrixVar = args.fUniformHandler->addUniform(&yuvEffect,
295 kFragment_GrShaderFlag, SkSLType::kHalf3x3, "colorSpaceMatrix");
296 fColorSpaceTranslateVar = args.fUniformHandler->addUniform(&yuvEffect,
297 kFragment_GrShaderFlag, SkSLType::kHalf3, "colorSpaceTranslate");
298 fragBuilder->codeAppendf(
299 "color.rgb = saturate(color.rgb * %s + %s);",
300 args.fUniformHandler->getUniformCStr(fColorSpaceMatrixVar),
301 args.fUniformHandler->getUniformCStr(fColorSpaceTranslateVar));
302 }
303 if (hasAlpha) {
304 // premultiply alpha
305 fragBuilder->codeAppendf("color.rgb *= color.a;");
306 }
307 fragBuilder->codeAppendf("return color;");
308 }
309
310 private:
311 void onSetData(const GrGLSLProgramDataManager& pdman,
312 const GrFragmentProcessor& proc) override {
313 const GrYUVtoRGBEffect& yuvEffect = proc.cast<GrYUVtoRGBEffect>();
314
315 if (yuvEffect.fYUVColorSpace != kIdentity_SkYUVColorSpace) {
316 SkASSERT(fColorSpaceMatrixVar.isValid());
317 float yuvM[20];
318 SkColorMatrix_YUV2RGB(yuvEffect.fYUVColorSpace, yuvM);
319 // We drop the fourth column entirely since the transformation
320 // should not depend on alpha. The fifth column is sent as a separate
321 // vector. The fourth row is also dropped entirely because alpha should
322 // never be modified.
323 SkASSERT(yuvM[3] == 0 && yuvM[8] == 0 && yuvM[13] == 0 && yuvM[18] == 1);
324 SkASSERT(yuvM[15] == 0 && yuvM[16] == 0 && yuvM[17] == 0 && yuvM[19] == 0);
325 float mtx[9] = {
326 yuvM[ 0], yuvM[ 1], yuvM[ 2],
327 yuvM[ 5], yuvM[ 6], yuvM[ 7],
328 yuvM[10], yuvM[11], yuvM[12],
329 };
330 float v[3] = {yuvM[4], yuvM[9], yuvM[14]};
331 pdman.setMatrix3f(fColorSpaceMatrixVar, mtx);
332 pdman.set3fv(fColorSpaceTranslateVar, 1, v);
333 }
334 }
335
336 UniformHandle fColorSpaceMatrixVar;
337 UniformHandle fColorSpaceTranslateVar;
338 };
339
340 return std::make_unique<Impl>();
341 }
342
onAddToKey(const GrShaderCaps & caps,skgpu::KeyBuilder * b) const343 void GrYUVtoRGBEffect::onAddToKey(const GrShaderCaps& caps, skgpu::KeyBuilder* b) const {
344 uint32_t packed = 0;
345 int i = 0;
346 for (auto [plane, channel] : fLocations) {
347 if (plane < 0) {
348 continue;
349 }
350
351 uint8_t chann = static_cast<int>(channel);
352
353 SkASSERT(plane < 4 && chann < 4);
354
355 packed |= (plane | (chann << 2)) << (i++ * 4);
356 }
357 if (fYUVColorSpace == kIdentity_SkYUVColorSpace) {
358 packed |= 1 << 16;
359 }
360 if (fSnap[0]) {
361 packed |= 1 << 17;
362 }
363 if (fSnap[1]) {
364 packed |= 1 << 18;
365 }
366 b->add32(packed);
367 }
368
onIsEqual(const GrFragmentProcessor & other) const369 bool GrYUVtoRGBEffect::onIsEqual(const GrFragmentProcessor& other) const {
370 const GrYUVtoRGBEffect& that = other.cast<GrYUVtoRGBEffect>();
371
372 return fLocations == that.fLocations &&
373 std::equal(fSnap, fSnap + 2, that.fSnap) &&
374 fYUVColorSpace == that.fYUVColorSpace;
375 }
376
GrYUVtoRGBEffect(const GrYUVtoRGBEffect & src)377 GrYUVtoRGBEffect::GrYUVtoRGBEffect(const GrYUVtoRGBEffect& src)
378 : GrFragmentProcessor(src)
379 , fLocations((src.fLocations))
380 , fYUVColorSpace(src.fYUVColorSpace) {
381 std::copy_n(src.fSnap, 2, fSnap);
382 }
383
clone() const384 std::unique_ptr<GrFragmentProcessor> GrYUVtoRGBEffect::clone() const {
385 return std::unique_ptr<GrFragmentProcessor>(new GrYUVtoRGBEffect(*this));
386 }
387