1 /*
2 * Copyright 2018 Google Inc.
3 *
4 * Use of this source code is governed by a BSD-style license that can be
5 * found in the LICENSE file.
6 */
7
8 #include "src/gpu/effects/GrYUVtoRGBEffect.h"
9
10 #include "include/core/SkYUVAInfo.h"
11 #include "src/core/SkYUVMath.h"
12 #include "src/gpu/GrTexture.h"
13 #include "src/gpu/GrYUVATextureProxies.h"
14 #include "src/gpu/effects/GrMatrixEffect.h"
15 #include "src/gpu/glsl/GrGLSLFragmentProcessor.h"
16 #include "src/gpu/glsl/GrGLSLFragmentShaderBuilder.h"
17 #include "src/gpu/glsl/GrGLSLProgramBuilder.h"
18 #include "src/sksl/SkSLCPP.h"
19 #include "src/sksl/SkSLUtil.h"
20
border_colors(const GrYUVATextureProxies & yuvaProxies,float planeBorders[4][4])21 static void border_colors(const GrYUVATextureProxies& yuvaProxies, float planeBorders[4][4]) {
22 float m[20];
23 SkColorMatrix_RGB2YUV(yuvaProxies.yuvaInfo().yuvColorSpace(), m);
24 for (int i = 0; i < SkYUVAInfo::kYUVAChannelCount; ++i) {
25 auto [plane, channel] = yuvaProxies.yuvaLocations()[i];
26 if (plane == -1) {
27 return;
28 }
29 auto c = static_cast<int>(channel);
30 planeBorders[plane][c] = m[i*5 + 4];
31 }
32 }
33
Make(const GrYUVATextureProxies & yuvaProxies,GrSamplerState samplerState,const GrCaps & caps,const SkMatrix & localMatrix,const SkRect * subset,const SkRect * domain)34 std::unique_ptr<GrFragmentProcessor> GrYUVtoRGBEffect::Make(const GrYUVATextureProxies& yuvaProxies,
35 GrSamplerState samplerState,
36 const GrCaps& caps,
37 const SkMatrix& localMatrix,
38 const SkRect* subset,
39 const SkRect* domain) {
40 int numPlanes = yuvaProxies.yuvaInfo().numPlanes();
41 if (!yuvaProxies.isValid()) {
42 return nullptr;
43 }
44
45 bool usesBorder = samplerState.wrapModeX() == GrSamplerState::WrapMode::kClampToBorder ||
46 samplerState.wrapModeY() == GrSamplerState::WrapMode::kClampToBorder;
47 float planeBorders[4][4] = {};
48 if (usesBorder) {
49 border_colors(yuvaProxies, planeBorders);
50 }
51
52 bool snap[2] = {false, false};
53 std::unique_ptr<GrFragmentProcessor> planeFPs[SkYUVAInfo::kMaxPlanes];
54 for (int i = 0; i < numPlanes; ++i) {
55 GrSurfaceProxyView view = yuvaProxies.makeView(i);
56 SkMatrix planeMatrix = yuvaProxies.yuvaInfo().originMatrix();
57 // The returned matrix is a view matrix but we need a local matrix.
58 SkAssertResult(planeMatrix.invert(&planeMatrix));
59 SkRect planeSubset;
60 SkRect planeDomain;
61 bool makeLinearWithSnap = false;
62 auto [ssx, ssy] = yuvaProxies.yuvaInfo().planeSubsamplingFactors(i);
63 SkASSERT(ssx > 0 && ssx <= 4);
64 SkASSERT(ssy > 0 && ssy <= 2);
65 float scaleX = 1.f;
66 float scaleY = 1.f;
67 if (ssx > 1 || ssy > 1) {
68 // JPEG chroma subsampling of odd dimensions produces U and V planes with the ceiling of
69 // the image size divided by the subsampling factor (2). Our API for creating YUVA
70 // doesn't capture the intended subsampling (and we should fix that). This fixes up 2x
71 // subsampling for images with odd widths/heights (e.g. JPEG 420 or 422).
72 scaleX = 1.f/ssx;
73 scaleY = 1.f/ssy;
74 // We would want to add a translation to this matrix to handle other sitings.
75 SkASSERT(yuvaProxies.yuvaInfo().sitingX() == SkYUVAInfo::Siting::kCentered);
76 SkASSERT(yuvaProxies.yuvaInfo().sitingY() == SkYUVAInfo::Siting::kCentered);
77 planeMatrix.postConcat(SkMatrix::Scale(scaleX, scaleY));
78 if (subset) {
79 planeSubset = {subset->fLeft *scaleX,
80 subset->fTop *scaleY,
81 subset->fRight *scaleX,
82 subset->fBottom*scaleY};
83 }
84 if (domain) {
85 planeDomain = {domain->fLeft *scaleX,
86 domain->fTop *scaleY,
87 domain->fRight *scaleX,
88 domain->fBottom*scaleY};
89 }
90 // This promotion of nearest to linear filtering for UV planes exists to mimic
91 // libjpeg[-turbo]'s do_fancy_upsampling option. We will filter the subsampled plane,
92 // however we want to filter at a fixed point for each logical image pixel to simulate
93 // nearest neighbor.
94 if (samplerState.filter() == GrSamplerState::Filter::kNearest) {
95 bool snapX = (ssx != 1),
96 snapY = (ssy != 1);
97 makeLinearWithSnap = snapX || snapY;
98 snap[0] |= snapX;
99 snap[1] |= snapY;
100 if (domain) {
101 // The outer YUVToRGB effect will ensure sampling happens at pixel centers
102 // within this plane.
103 planeDomain = {std::floor(planeDomain.fLeft) + 0.5f,
104 std::floor(planeDomain.fTop) + 0.5f,
105 std::floor(planeDomain.fRight) + 0.5f,
106 std::floor(planeDomain.fBottom) + 0.5f};
107 }
108 }
109 } else {
110 if (subset) {
111 planeSubset = *subset;
112 }
113 if (domain) {
114 planeDomain = *domain;
115 }
116 }
117 if (subset) {
118 if (makeLinearWithSnap) {
119 // The plane is subsampled and we have an overall subset on the image. We're
120 // emulating do_fancy_upsampling using linear filtering but snapping look ups to the
121 // y-plane pixel centers. Consider a logical image pixel at the edge of the subset.
122 // When computing the logical pixel color value we should use a 50/50 blend of two
123 // values from the subsampled plane. Depending on where the subset edge falls in
124 // actual subsampled plane, one of those values may come from outside the subset.
125 // Hence, we use this custom inset factory which applies the wrap mode to
126 // planeSubset but allows linear filtering to read pixels from the plane that are
127 // just outside planeSubset.
128 SkRect* domainRect = domain ? &planeDomain : nullptr;
129 planeFPs[i] = GrTextureEffect::MakeCustomLinearFilterInset(std::move(view),
130 kUnknown_SkAlphaType,
131 planeMatrix,
132 samplerState.wrapModeX(),
133 samplerState.wrapModeY(),
134 planeSubset,
135 domainRect,
136 {scaleX/2.f, scaleY/2.f},
137 caps,
138 planeBorders[i]);
139 } else if (domain) {
140 planeFPs[i] = GrTextureEffect::MakeSubset(std::move(view),
141 kUnknown_SkAlphaType,
142 planeMatrix,
143 samplerState,
144 planeSubset,
145 planeDomain,
146 caps,
147 planeBorders[i]);
148 } else {
149 planeFPs[i] = GrTextureEffect::MakeSubset(std::move(view),
150 kUnknown_SkAlphaType,
151 planeMatrix,
152 samplerState,
153 planeSubset,
154 caps,
155 planeBorders[i]);
156 }
157 } else {
158 GrSamplerState planeSampler = samplerState;
159 if (makeLinearWithSnap) {
160 planeSampler.setFilterMode(GrSamplerState::Filter::kLinear);
161 }
162 planeFPs[i] = GrTextureEffect::Make(std::move(view),
163 kUnknown_SkAlphaType,
164 planeMatrix,
165 planeSampler,
166 caps,
167 planeBorders[i]);
168 }
169 }
170 std::unique_ptr<GrFragmentProcessor> fp(
171 new GrYUVtoRGBEffect(planeFPs,
172 numPlanes,
173 yuvaProxies.yuvaLocations(),
174 snap,
175 yuvaProxies.yuvaInfo().yuvColorSpace()));
176 return GrMatrixEffect::Make(localMatrix, std::move(fp));
177 }
178
alpha_type(const SkYUVAInfo::YUVALocations locations)179 static SkAlphaType alpha_type(const SkYUVAInfo::YUVALocations locations) {
180 return locations[SkYUVAInfo::YUVAChannels::kA].fPlane >= 0 ? kPremul_SkAlphaType
181 : kOpaque_SkAlphaType;
182 }
183
GrYUVtoRGBEffect(std::unique_ptr<GrFragmentProcessor> planeFPs[4],int numPlanes,const SkYUVAInfo::YUVALocations & locations,const bool snap[2],SkYUVColorSpace yuvColorSpace)184 GrYUVtoRGBEffect::GrYUVtoRGBEffect(std::unique_ptr<GrFragmentProcessor> planeFPs[4],
185 int numPlanes,
186 const SkYUVAInfo::YUVALocations& locations,
187 const bool snap[2],
188 SkYUVColorSpace yuvColorSpace)
189 : GrFragmentProcessor(kGrYUVtoRGBEffect_ClassID,
190 ModulateForClampedSamplerOptFlags(alpha_type(locations)))
191 , fLocations(locations)
192 , fYUVColorSpace(yuvColorSpace) {
193 std::copy_n(snap, 2, fSnap);
194
195 if (fSnap[0] || fSnap[1]) {
196 // Need this so that we can access coords in SKSL to perform snapping.
197 this->setUsesSampleCoordsDirectly();
198 for (int i = 0; i < numPlanes; ++i) {
199 this->registerChild(std::move(planeFPs[i]), SkSL::SampleUsage::Explicit());
200 }
201 } else {
202 for (int i = 0; i < numPlanes; ++i) {
203 this->registerChild(std::move(planeFPs[i]));
204 }
205 }
206 }
207
208 #if GR_TEST_UTILS
onDumpInfo() const209 SkString GrYUVtoRGBEffect::onDumpInfo() const {
210 SkString str("(");
211 for (int i = 0; i < SkYUVAInfo::kYUVAChannelCount; ++i) {
212 str.appendf("Locations[%d]=%d %d, ",
213 i, fLocations[i].fPlane, static_cast<int>(fLocations[i].fChannel));
214 }
215 str.appendf("YUVColorSpace=%d, snap=(%d, %d))",
216 static_cast<int>(fYUVColorSpace), fSnap[0], fSnap[1]);
217 return str;
218 }
219 #endif
220
onMakeProgramImpl() const221 std::unique_ptr<GrGLSLFragmentProcessor> GrYUVtoRGBEffect::onMakeProgramImpl() const {
222 class GrGLSLYUVtoRGBEffect : public GrGLSLFragmentProcessor {
223 public:
224 GrGLSLYUVtoRGBEffect() {}
225
226 void emitCode(EmitArgs& args) override {
227 GrGLSLFPFragmentBuilder* fragBuilder = args.fFragBuilder;
228 const GrYUVtoRGBEffect& yuvEffect = args.fFp.cast<GrYUVtoRGBEffect>();
229
230 int numPlanes = yuvEffect.numChildProcessors();
231
232 const char* sampleCoords = "";
233 if (yuvEffect.fSnap[0] || yuvEffect.fSnap[1]) {
234 fragBuilder->codeAppendf("float2 snappedCoords = %s;", args.fSampleCoord);
235 if (yuvEffect.fSnap[0]) {
236 fragBuilder->codeAppend("snappedCoords.x = floor(snappedCoords.x) + 0.5;");
237 }
238 if (yuvEffect.fSnap[1]) {
239 fragBuilder->codeAppend("snappedCoords.y = floor(snappedCoords.y) + 0.5;");
240 }
241 sampleCoords = "snappedCoords";
242 }
243
244 fragBuilder->codeAppendf("half4 color;");
245 const bool hasAlpha = yuvEffect.fLocations[SkYUVAInfo::YUVAChannels::kA].fPlane >= 0;
246
247 for (int planeIdx = 0; planeIdx < numPlanes; ++planeIdx) {
248 std::string colorChannel;
249 std::string planeChannel;
250 for (int locIdx = 0; locIdx < (hasAlpha ? 4 : 3); ++locIdx) {
251 auto [yuvPlane, yuvChannel] = yuvEffect.fLocations[locIdx];
252 if (yuvPlane == planeIdx) {
253 colorChannel.push_back("rgba"[locIdx]);
254 planeChannel.push_back("rgba"[static_cast<int>(yuvChannel)]);
255 }
256 }
257
258 SkASSERT(colorChannel.size() == planeChannel.size());
259 if (!colorChannel.empty()) {
260 fragBuilder->codeAppendf(
261 "color.%s = (%s).%s;",
262 colorChannel.c_str(),
263 this->invokeChild(planeIdx, args, sampleCoords).c_str(),
264 planeChannel.c_str());
265 }
266 }
267
268 if (!hasAlpha) {
269 fragBuilder->codeAppendf("color.a = 1;");
270 }
271
272 if (kIdentity_SkYUVColorSpace != yuvEffect.fYUVColorSpace) {
273 fColorSpaceMatrixVar = args.fUniformHandler->addUniform(&yuvEffect,
274 kFragment_GrShaderFlag, kHalf3x3_GrSLType, "colorSpaceMatrix");
275 fColorSpaceTranslateVar = args.fUniformHandler->addUniform(&yuvEffect,
276 kFragment_GrShaderFlag, kHalf3_GrSLType, "colorSpaceTranslate");
277 fragBuilder->codeAppendf(
278 "color.rgb = saturate(color.rgb * %s + %s);",
279 args.fUniformHandler->getUniformCStr(fColorSpaceMatrixVar),
280 args.fUniformHandler->getUniformCStr(fColorSpaceTranslateVar));
281 }
282 if (hasAlpha) {
283 // premultiply alpha
284 fragBuilder->codeAppendf("color.rgb *= color.a;");
285 }
286 fragBuilder->codeAppendf("return color;");
287 }
288
289 private:
290 void onSetData(const GrGLSLProgramDataManager& pdman,
291 const GrFragmentProcessor& proc) override {
292 const GrYUVtoRGBEffect& yuvEffect = proc.cast<GrYUVtoRGBEffect>();
293
294 if (yuvEffect.fYUVColorSpace != kIdentity_SkYUVColorSpace) {
295 SkASSERT(fColorSpaceMatrixVar.isValid());
296 float yuvM[20];
297 SkColorMatrix_YUV2RGB(yuvEffect.fYUVColorSpace, yuvM);
298 // We drop the fourth column entirely since the transformation
299 // should not depend on alpha. The fifth column is sent as a separate
300 // vector. The fourth row is also dropped entirely because alpha should
301 // never be modified.
302 SkASSERT(yuvM[3] == 0 && yuvM[8] == 0 && yuvM[13] == 0 && yuvM[18] == 1);
303 SkASSERT(yuvM[15] == 0 && yuvM[16] == 0 && yuvM[17] == 0 && yuvM[19] == 0);
304 float mtx[9] = {
305 yuvM[ 0], yuvM[ 1], yuvM[ 2],
306 yuvM[ 5], yuvM[ 6], yuvM[ 7],
307 yuvM[10], yuvM[11], yuvM[12],
308 };
309 float v[3] = {yuvM[4], yuvM[9], yuvM[14]};
310 pdman.setMatrix3f(fColorSpaceMatrixVar, mtx);
311 pdman.set3fv(fColorSpaceTranslateVar, 1, v);
312 }
313 }
314
315 UniformHandle fColorSpaceMatrixVar;
316 UniformHandle fColorSpaceTranslateVar;
317 };
318
319 return std::make_unique<GrGLSLYUVtoRGBEffect>();
320 }
onGetGLSLProcessorKey(const GrShaderCaps & caps,GrProcessorKeyBuilder * b) const321 void GrYUVtoRGBEffect::onGetGLSLProcessorKey(const GrShaderCaps& caps,
322 GrProcessorKeyBuilder* b) const {
323 uint32_t packed = 0;
324 int i = 0;
325 for (auto [plane, channel] : fLocations) {
326 if (plane < 0) {
327 continue;
328 }
329
330 uint8_t chann = static_cast<int>(channel);
331
332 SkASSERT(plane < 4 && chann < 4);
333
334 packed |= (plane | (chann << 2)) << (i++ * 4);
335 }
336 if (fYUVColorSpace == kIdentity_SkYUVColorSpace) {
337 packed |= 1 << 16;
338 }
339 if (fSnap[0]) {
340 packed |= 1 << 17;
341 }
342 if (fSnap[1]) {
343 packed |= 1 << 18;
344 }
345 b->add32(packed);
346 }
347
onIsEqual(const GrFragmentProcessor & other) const348 bool GrYUVtoRGBEffect::onIsEqual(const GrFragmentProcessor& other) const {
349 const GrYUVtoRGBEffect& that = other.cast<GrYUVtoRGBEffect>();
350
351 return fLocations == that.fLocations &&
352 std::equal(fSnap, fSnap + 2, that.fSnap) &&
353 fYUVColorSpace == that.fYUVColorSpace;
354 }
355
GrYUVtoRGBEffect(const GrYUVtoRGBEffect & src)356 GrYUVtoRGBEffect::GrYUVtoRGBEffect(const GrYUVtoRGBEffect& src)
357 : GrFragmentProcessor(kGrYUVtoRGBEffect_ClassID, src.optimizationFlags())
358 , fLocations((src.fLocations))
359 , fYUVColorSpace(src.fYUVColorSpace) {
360 this->cloneAndRegisterAllChildProcessors(src);
361 if (src.fSnap[0] || src.fSnap[1]) {
362 this->setUsesSampleCoordsDirectly();
363 }
364
365 std::copy_n(src.fSnap, 2, fSnap);
366 }
367
clone() const368 std::unique_ptr<GrFragmentProcessor> GrYUVtoRGBEffect::clone() const {
369 return std::unique_ptr<GrFragmentProcessor>(new GrYUVtoRGBEffect(*this));
370 }
371