1 /*
2 * Copyright 2015 Google Inc.
3 *
4 * Use of this source code is governed by a BSD-style license that can be
5 * found in the LICENSE file.
6 */
7
8 #ifndef SkBlitMask_opts_DEFINED
9 #define SkBlitMask_opts_DEFINED
10
11 #include "include/private/base/SkFeatures.h"
12 #include "src/core/Sk4px.h"
13
14 #if defined(SK_ARM_HAS_NEON)
15 #include <arm_neon.h>
16 #endif
17
18 namespace SK_OPTS_NS {
19
20 #if defined(SK_ARM_HAS_NEON)
21 // The Sk4px versions below will work fine with NEON, but we have had many indications
22 // that it doesn't perform as well as this NEON-specific code.
23
24 #define NEON_A (SK_A32_SHIFT / 8)
25 #define NEON_R (SK_R32_SHIFT / 8)
26 #define NEON_G (SK_G32_SHIFT / 8)
27 #define NEON_B (SK_B32_SHIFT / 8)
28
SkAlpha255To256_neon8(uint8x8_t alpha)29 static inline uint16x8_t SkAlpha255To256_neon8(uint8x8_t alpha) {
30 return vaddw_u8(vdupq_n_u16(1), alpha);
31 }
32
SkAlphaMul_neon8(uint8x8_t color,uint16x8_t scale)33 static inline uint8x8_t SkAlphaMul_neon8(uint8x8_t color, uint16x8_t scale) {
34 return vshrn_n_u16(vmovl_u8(color) * scale, 8);
35 }
36
SkAlphaMulQ_neon8(uint8x8x4_t color,uint16x8_t scale)37 static inline uint8x8x4_t SkAlphaMulQ_neon8(uint8x8x4_t color, uint16x8_t scale) {
38 uint8x8x4_t ret;
39
40 ret.val[0] = SkAlphaMul_neon8(color.val[0], scale);
41 ret.val[1] = SkAlphaMul_neon8(color.val[1], scale);
42 ret.val[2] = SkAlphaMul_neon8(color.val[2], scale);
43 ret.val[3] = SkAlphaMul_neon8(color.val[3], scale);
44
45 return ret;
46 }
47
48
49 template <bool isTranslucent>
blit_mask_d32_a8_neon(void * SK_RESTRICT dst,size_t dstRB,const void * SK_RESTRICT maskPtr,size_t maskRB,SkColor color,int width,int height)50 static void blit_mask_d32_a8_neon(void* SK_RESTRICT dst, size_t dstRB,
51 const void* SK_RESTRICT maskPtr, size_t maskRB,
52 SkColor color, int width, int height) {
53 const SkPMColor pmc = SkPreMultiplyColor(color);
54 const U8CPU colorAlpha = SkGetPackedA32(pmc);
55 SkPMColor* SK_RESTRICT device = (SkPMColor*)dst;
56 const uint8_t* SK_RESTRICT mask = (const uint8_t*)maskPtr;
57 uint8x8x4_t vpmc;
58
59 // Nine patch may set maskRB to 0 to blit the same row repeatedly.
60 ptrdiff_t mask_adjust = (ptrdiff_t)maskRB - width;
61 dstRB -= (width << 2);
62
63 if (width >= 8) {
64 vpmc.val[NEON_A] = vdup_n_u8(colorAlpha);
65 vpmc.val[NEON_R] = vdup_n_u8(SkGetPackedR32(pmc));
66 vpmc.val[NEON_G] = vdup_n_u8(SkGetPackedG32(pmc));
67 vpmc.val[NEON_B] = vdup_n_u8(SkGetPackedB32(pmc));
68 }
69 do {
70 int w = width;
71 while (w >= 8) {
72 uint8x8_t vmask = vld1_u8(mask);
73 uint16x8_t vmask256 = SkAlpha255To256_neon8(vmask);
74 uint16x8_t vscale;
75 if constexpr (isTranslucent) {
76 vscale = vsubw_u8(vdupq_n_u16(256),
77 SkAlphaMul_neon8(vpmc.val[NEON_A], vmask256));
78 } else {
79 vscale = vsubw_u8(vdupq_n_u16(256), vmask);
80 }
81 uint8x8x4_t vdev = vld4_u8((uint8_t*)device);
82
83 vdev.val[NEON_A] = SkAlphaMul_neon8(vpmc.val[NEON_A], vmask256)
84 + SkAlphaMul_neon8(vdev.val[NEON_A], vscale);
85 vdev.val[NEON_R] = SkAlphaMul_neon8(vpmc.val[NEON_R], vmask256)
86 + SkAlphaMul_neon8(vdev.val[NEON_R], vscale);
87 vdev.val[NEON_G] = SkAlphaMul_neon8(vpmc.val[NEON_G], vmask256)
88 + SkAlphaMul_neon8(vdev.val[NEON_G], vscale);
89 vdev.val[NEON_B] = SkAlphaMul_neon8(vpmc.val[NEON_B], vmask256)
90 + SkAlphaMul_neon8(vdev.val[NEON_B], vscale);
91
92 vst4_u8((uint8_t*)device, vdev);
93
94 mask += 8;
95 device += 8;
96 w -= 8;
97 }
98
99 while (w-- > 0) {
100 // These variables aren't actually vectors, but the names are consistent with
101 // the above to make it easier to compare the operations.
102 const U8CPU vmask = *mask++;
103 const U16CPU vmask256 = SkAlpha255To256(vmask);
104 U16CPU vscale;
105 if constexpr (isTranslucent) {
106 vscale = 256 - SkAlphaMulQ(colorAlpha, vmask256);
107 } else {
108 vscale = 256 - vmask;
109 }
110 *device = SkAlphaMulQ(pmc, vmask256)
111 + SkAlphaMulQ(*device, vscale);
112 device += 1;
113 }
114
115 device = (uint32_t*)((char*)device + dstRB);
116 mask += mask_adjust;
117
118 } while (--height != 0);
119 }
120
blit_mask_d32_a8_general(SkPMColor * dst,size_t dstRB,const SkAlpha * mask,size_t maskRB,SkColor color,int w,int h)121 static void blit_mask_d32_a8_general(SkPMColor* dst, size_t dstRB,
122 const SkAlpha* mask, size_t maskRB,
123 SkColor color, int w, int h) {
124 blit_mask_d32_a8_neon<true>(dst, dstRB, mask, maskRB, color, w, h);
125 }
126
127 // As above, but made slightly simpler by requiring that color is opaque.
blit_mask_d32_a8_opaque(SkPMColor * dst,size_t dstRB,const SkAlpha * mask,size_t maskRB,SkColor color,int w,int h)128 static void blit_mask_d32_a8_opaque(SkPMColor* dst, size_t dstRB,
129 const SkAlpha* mask, size_t maskRB,
130 SkColor color, int w, int h) {
131 blit_mask_d32_a8_neon<false>(dst, dstRB, mask, maskRB, color, w, h);
132 }
133
134 // Same as _opaque, but assumes color == SK_ColorBLACK, a very common and even simpler case.
blit_mask_d32_a8_black(SkPMColor * dst,size_t dstRB,const SkAlpha * maskPtr,size_t maskRB,int width,int height)135 static void blit_mask_d32_a8_black(SkPMColor* dst, size_t dstRB,
136 const SkAlpha* maskPtr, size_t maskRB,
137 int width, int height) {
138 SkPMColor* SK_RESTRICT device = (SkPMColor*)dst;
139 const uint8_t* SK_RESTRICT mask = (const uint8_t*)maskPtr;
140
141 // Nine patch may set maskRB to 0 to blit the same row repeatedly.
142 ptrdiff_t mask_adjust = (ptrdiff_t)maskRB - width;
143 dstRB -= (width << 2);
144 do {
145 int w = width;
146 while (w >= 8) {
147 uint8x8_t vmask = vld1_u8(mask);
148 uint16x8_t vscale = vsubw_u8(vdupq_n_u16(256), vmask);
149 uint8x8x4_t vdevice = vld4_u8((uint8_t*)device);
150
151 vdevice = SkAlphaMulQ_neon8(vdevice, vscale);
152 vdevice.val[NEON_A] += vmask;
153
154 vst4_u8((uint8_t*)device, vdevice);
155
156 mask += 8;
157 device += 8;
158 w -= 8;
159 }
160 while (w-- > 0) {
161 // These variables aren't actually vectors, but the names are consistent with
162 // the above to make it easier to compare the operations.
163 const U8CPU vmask = *mask++;
164 const U16CPU vscale = 256 - vmask;
165 *device = SkAlphaMulQ(*device, vscale) +
166 (vmask << SK_A32_SHIFT);
167 device += 1;
168 }
169 device = (uint32_t*)((char*)device + dstRB);
170 mask += mask_adjust;
171 } while (--height != 0);
172 }
173
174 #elif SK_CPU_LSX_LEVEL >= SK_CPU_LSX_LEVEL_LSX
175 #include <lsxintrin.h>
176
177 static __m128i SkAlphaMul_lsx(__m128i x, __m128i y) {
178 __m128i tmp = __lsx_vmul_h(x, y);
179 __m128i mask = __lsx_vreplgr2vr_h(0xff00);
180 return __lsx_vsrlri_h(__lsx_vand_v(tmp, mask), 8);
181 }
182
183 template <bool isColor>
184 static void D32_A8_Opaque_Color_lsx(void* SK_RESTRICT dst, size_t dstRB,
185 const void* SK_RESTRICT maskPtr, size_t maskRB,
186 SkColor color, int width, int height) {
187 SkPMColor pmc = SkPreMultiplyColor(color);
188 SkPMColor* SK_RESTRICT device = (SkPMColor*)dst;
189 const uint8_t* SK_RESTRICT mask = (const uint8_t*)maskPtr;
190 __m128i vpmc_b = __lsx_vldi(0);
191 __m128i vpmc_g = __lsx_vldi(0);
192 __m128i vpmc_r = __lsx_vldi(0);
193 __m128i vpmc_a = __lsx_vldi(0);
194
195 // Nine patch may set maskRB to 0 to blit the same row repeatedly.
196 ptrdiff_t mask_adjust = (ptrdiff_t)maskRB - width;
197 dstRB -= (width << 2);
198
199 if (width >= 8) {
200 vpmc_b = __lsx_vreplgr2vr_h(SkGetPackedB32(pmc));
201 vpmc_g = __lsx_vreplgr2vr_h(SkGetPackedG32(pmc));
202 vpmc_r = __lsx_vreplgr2vr_h(SkGetPackedR32(pmc));
203 vpmc_a = __lsx_vreplgr2vr_h(SkGetPackedA32(pmc));
204 }
205
206 const __m128i zeros = __lsx_vldi(0);
207 __m128i planar = __lsx_vldi(0);
208 planar = __lsx_vinsgr2vr_d(planar, 0x0d0905010c080400, 0);
209 planar = __lsx_vinsgr2vr_d(planar, 0x0f0b07030e0a0602, 1);
210
211 do{
212 int w = width;
213 while(w >= 8){
214 __m128i lo = __lsx_vld(device, 0); // bgra bgra bgra bgra
215 __m128i hi = __lsx_vld(device, 16); // BGRA BGRA BGRA BGRA
216 lo = __lsx_vshuf_b(zeros, lo, planar); // bbbb gggg rrrr aaaa
217 hi = __lsx_vshuf_b(zeros, hi, planar); // BBBB GGGG RRRR AAAA
218 __m128i bg = __lsx_vilvl_w(hi, lo), // bbbb BBBB gggg GGGG
219 ra = __lsx_vilvh_w(hi, lo); // rrrr RRRR aaaa AAAA
220
221 __m128i b = __lsx_vilvl_b(zeros, bg), // _b_b _b_b _B_B _B_B
222 g = __lsx_vilvh_b(zeros, bg), // _g_g _g_g _G_G _G_G
223 r = __lsx_vilvl_b(zeros, ra), // _r_r _r_r _R_R _R_R
224 a = __lsx_vilvh_b(zeros, ra); // _a_a _a_a _A_A _A_A
225
226 __m128i vmask = __lsx_vld(mask, 0);
227 vmask = __lsx_vilvl_b(zeros, vmask);
228 __m128i vscale, vmask256 = __lsx_vadd_h(vmask, __lsx_vreplgr2vr_h(1));
229
230 if (isColor) {
231 __m128i tmp = SkAlphaMul_lsx(vpmc_a, vmask256);
232 vscale = __lsx_vsub_h(__lsx_vreplgr2vr_h(256), tmp);
233 } else {
234 vscale = __lsx_vsub_h(__lsx_vreplgr2vr_h(256), vmask);
235 }
236
237 b = SkAlphaMul_lsx(vpmc_b, vmask256) + SkAlphaMul_lsx(b, vscale);
238 g = SkAlphaMul_lsx(vpmc_g, vmask256) + SkAlphaMul_lsx(g, vscale);
239 r = SkAlphaMul_lsx(vpmc_r, vmask256) + SkAlphaMul_lsx(r, vscale);
240 a = SkAlphaMul_lsx(vpmc_a, vmask256) + SkAlphaMul_lsx(a, vscale);
241
242 bg = __lsx_vor_v(b, __lsx_vslli_h(g, 8)); // bgbg bgbg BGBG BGBG
243 ra = __lsx_vor_v(r, __lsx_vslli_h(a, 8)); // rara rara RARA RARA
244 lo = __lsx_vilvl_h(ra, bg); // bgra bgra bgra bgra
245 hi = __lsx_vilvh_h(ra, bg); // BGRA BGRA BGRA BGRA
246
247 __lsx_vst(lo, device, 0);
248 __lsx_vst(hi, device, 16);
249
250 mask += 8;
251 device += 8;
252 w -= 8;
253 }
254
255 while (w--) {
256 unsigned aa = *mask++;
257 if (isColor) {
258 *device = SkBlendARGB32(pmc, *device, aa);
259 } else {
260 *device = SkAlphaMulQ(pmc, SkAlpha255To256(aa))
261 + SkAlphaMulQ(*device, SkAlpha255To256(255 - aa));
262 }
263 device += 1;
264 }
265
266 device = (uint32_t *)((char*)device + dstRB);
267 mask += mask_adjust;
268
269 } while (--height != 0);
270 }
271
272 static void blit_mask_d32_a8_general(SkPMColor* dst, size_t dstRB,
273 const SkAlpha* mask, size_t maskRB,
274 SkColor color, int w, int h) {
275 D32_A8_Opaque_Color_lsx<true>(dst, dstRB, mask, maskRB, color, w, h);
276 }
277
278 static void blit_mask_d32_a8_opaque(SkPMColor* dst, size_t dstRB,
279 const SkAlpha* mask, size_t maskRB,
280 SkColor color, int w, int h) {
281 D32_A8_Opaque_Color_lsx<false>(dst, dstRB, mask, maskRB, color, w, h);
282 }
283
284 // Same as _opaque, but assumes color == SK_ColorBLACK, a very common and even simpler case.
285 static void blit_mask_d32_a8_black(SkPMColor* dst, size_t dstRB,
286 const SkAlpha* maskPtr, size_t maskRB,
287 int width, int height) {
288 SkPMColor* SK_RESTRICT device = (SkPMColor*)dst;
289 const uint8_t* SK_RESTRICT mask = (const uint8_t*)maskPtr;
290
291 // Nine patch may set maskRB to 0 to blit the same row repeatedly.
292 ptrdiff_t mask_adjust = (ptrdiff_t)maskRB - width;
293 dstRB -= (width << 2);
294 const __m128i zeros = __lsx_vldi(0);
295 __m128i planar = __lsx_vldi(0);
296 planar = __lsx_vinsgr2vr_d(planar, 0x0d0905010c080400, 0);
297 planar = __lsx_vinsgr2vr_d(planar, 0x0f0b07030e0a0602, 1);
298
299 do {
300 int w = width;
301 while (w >= 8) {
302 __m128i vmask = __lsx_vld(mask, 0);
303 vmask = __lsx_vilvl_b(zeros, vmask);
304 __m128i vscale = __lsx_vsub_h(__lsx_vreplgr2vr_h(256), vmask);
305 __m128i lo = __lsx_vld(device, 0); // bgra bgra bgra bgra
306 __m128i hi = __lsx_vld(device, 16); // BGRA BGRA BGRA BGRA
307 lo = __lsx_vshuf_b(zeros, lo, planar); // bbbb gggg rrrr aaaa
308 hi = __lsx_vshuf_b(zeros, hi, planar); // BBBB GGGG RRRR AAAA
309 __m128i bg = __lsx_vilvl_w(hi, lo), // bbbb BBBB gggg GGGG
310 ra = __lsx_vilvh_w(hi, lo); // rrrr RRRR aaaa AAAA
311
312 __m128i b = __lsx_vilvl_b(zeros, bg), // _b_b _b_b _B_B _B_B
313 g = __lsx_vilvh_b(zeros, bg), // _g_g _g_g _G_G _G_G
314 r = __lsx_vilvl_b(zeros, ra), // _r_r _r_r _R_R _R_R
315 a = __lsx_vilvh_b(zeros, ra); // _a_a _a_a _A_A _A_A
316
317 b = SkAlphaMul_lsx(b, vscale);
318 g = SkAlphaMul_lsx(g, vscale);
319 r = SkAlphaMul_lsx(r, vscale);
320 a = SkAlphaMul_lsx(a, vscale);
321
322 a += vmask;
323
324 bg = __lsx_vor_v(b, __lsx_vslli_h(g, 8)); // bgbg bgbg BGBG BGBG
325 ra = __lsx_vor_v(r, __lsx_vslli_h(a, 8)); // rara rara RARA RARA
326 lo = __lsx_vilvl_h(ra, bg); // bgra bgra bgra bgra
327 hi = __lsx_vilvh_h(ra, bg); // BGRA BGRA BGRA BGRA
328
329 __lsx_vst(lo, device, 0);
330 __lsx_vst(hi, device, 16);
331
332 mask += 8;
333 device += 8;
334 w -= 8;
335 }
336
337 while (w-- > 0) {
338 unsigned aa = *mask++;
339 *device = (aa << SK_A32_SHIFT)
340 + SkAlphaMulQ(*device, SkAlpha255To256(255 - aa));
341 device += 1;
342 }
343
344 device = (uint32_t*)((char*)device + dstRB);
345 mask += mask_adjust;
346
347 } while (--height != 0);
348 }
349
350 #else
351 static void blit_mask_d32_a8_general(SkPMColor* dst, size_t dstRB,
352 const SkAlpha* mask, size_t maskRB,
353 SkColor color, int w, int h) {
354 auto s = Sk4px::DupPMColor(SkPreMultiplyColor(color));
355 auto fn = [&](const Sk4px& d, const Sk4px& aa) {
356 // = (s + d(1-sa))aa + d(1-aa)
357 // = s*aa + d(1-sa*aa)
358 auto left = s.approxMulDiv255(aa),
359 right = d.approxMulDiv255(left.alphas().inv());
360 return left + right; // This does not overflow (exhaustively checked).
361 };
362 while (h --> 0) {
363 Sk4px::MapDstAlpha(w, dst, mask, fn);
364 dst += dstRB / sizeof(*dst);
365 mask += maskRB / sizeof(*mask);
366 }
367 }
368
369 // As above, but made slightly simpler by requiring that color is opaque.
370 static void blit_mask_d32_a8_opaque(SkPMColor* dst, size_t dstRB,
371 const SkAlpha* mask, size_t maskRB,
372 SkColor color, int w, int h) {
373 SkASSERT(SkColorGetA(color) == 0xFF);
374 auto s = Sk4px::DupPMColor(SkPreMultiplyColor(color));
375 auto fn = [&](const Sk4px& d, const Sk4px& aa) {
376 // = (s + d(1-sa))aa + d(1-aa)
377 // = s*aa + d(1-sa*aa)
378 // ~~~>
379 // = s*aa + d(1-aa)
380 return s.approxMulDiv255(aa) + d.approxMulDiv255(aa.inv());
381 };
382 while (h --> 0) {
383 Sk4px::MapDstAlpha(w, dst, mask, fn);
384 dst += dstRB / sizeof(*dst);
385 mask += maskRB / sizeof(*mask);
386 }
387 }
388
389 // Same as _opaque, but assumes color == SK_ColorBLACK, a very common and even simpler case.
390 static void blit_mask_d32_a8_black(SkPMColor* dst, size_t dstRB,
391 const SkAlpha* mask, size_t maskRB,
392 int w, int h) {
393 auto fn = [](const Sk4px& d, const Sk4px& aa) {
394 // = (s + d(1-sa))aa + d(1-aa)
395 // = s*aa + d(1-sa*aa)
396 // ~~~>
397 // a = 1*aa + d(1-1*aa) = aa + d(1-aa)
398 // c = 0*aa + d(1-1*aa) = d(1-aa)
399 return (aa & Sk4px(skvx::byte16{0,0,0,255, 0,0,0,255, 0,0,0,255, 0,0,0,255}))
400 + d.approxMulDiv255(aa.inv());
401 };
402 while (h --> 0) {
403 Sk4px::MapDstAlpha(w, dst, mask, fn);
404 dst += dstRB / sizeof(*dst);
405 mask += maskRB / sizeof(*mask);
406 }
407 }
408 #endif
409
blit_mask_d32_a8(SkPMColor * dst,size_t dstRB,const SkAlpha * mask,size_t maskRB,SkColor color,int w,int h)410 /*not static*/ inline void blit_mask_d32_a8(SkPMColor* dst, size_t dstRB,
411 const SkAlpha* mask, size_t maskRB,
412 SkColor color, int w, int h) {
413 if (color == SK_ColorBLACK) {
414 blit_mask_d32_a8_black(dst, dstRB, mask, maskRB, w, h);
415 } else if (SkColorGetA(color) == 0xFF) {
416 blit_mask_d32_a8_opaque(dst, dstRB, mask, maskRB, color, w, h);
417 } else {
418 blit_mask_d32_a8_general(dst, dstRB, mask, maskRB, color, w, h);
419 }
420 }
421
422 } // namespace SK_OPTS_NS
423
424 #endif//SkBlitMask_opts_DEFINED
425