1 /* NEON optimized code (C) COPYRIGHT 2009 Motorola
2 *
3 * Use of this source code is governed by a BSD-style license that can be
4 * found in the LICENSE file.
5 */
6
7 /*
8 * Modifications done in-house at Motorola
9 *
10 * this is a clone of SkBitmapProcState_matrix.h
11 * and has been tuned to work with the NEON unit.
12 *
13 * Still going back and forth between whether this approach
14 * (clone the entire SkBitmapProcState_matrix.h file or
15 * if I should put just the modified routines in here and
16 * then use a construct like #define DONT_DO_THIS_FUNCTION or
17 * something like that...
18 *
19 * This is for the ClampX_ClampY instance
20 *
21 */
22
23
24 #include <arm_neon.h>
25
26 /*
27 * This has been modified on the knowledge that (at the time)
28 * we had the following macro definitions in the parent file
29 *
30 * #define MAKENAME(suffix) ClampX_ClampY ## suffix
31 * #define TILEX_PROCF(fx, max) SkClampMax((fx) >> 16, max)
32 * #define TILEY_PROCF(fy, max) SkClampMax((fy) >> 16, max)
33 * #define TILEX_LOW_BITS(fx, max) (((fx) >> 12) & 0xF)
34 * #define TILEY_LOW_BITS(fy, max) (((fy) >> 12) & 0xF)
35 * #define CHECK_FOR_DECAL
36 */
37
38 /* SkClampMax(val,max) -- bound to 0..max */
39
40 #define SCALE_NOFILTER_NAME MAKENAME(_nofilter_scale_neon)
41 #define SCALE_FILTER_NAME MAKENAME(_filter_scale_neon)
42 #define AFFINE_NOFILTER_NAME MAKENAME(_nofilter_affine_neon)
43 #define AFFINE_FILTER_NAME MAKENAME(_filter_affine_neon)
44 #define PERSP_NOFILTER_NAME MAKENAME(_nofilter_persp_neon)
45 #define PERSP_FILTER_NAME MAKENAME(_filter_persp_neon)
46
47 #define PACK_FILTER_X_NAME MAKENAME(_pack_filter_x)
48 #define PACK_FILTER_Y_NAME MAKENAME(_pack_filter_y)
49
50 #ifndef PREAMBLE
51 #define PREAMBLE(state)
52 #define PREAMBLE_PARAM_X
53 #define PREAMBLE_PARAM_Y
54 #define PREAMBLE_ARG_X
55 #define PREAMBLE_ARG_Y
56 #endif
57
SCALE_NOFILTER_NAME(const SkBitmapProcState & s,uint32_t xy[],int count,int x,int y)58 static void SCALE_NOFILTER_NAME(const SkBitmapProcState& s,
59 uint32_t xy[], int count, int x, int y) {
60 SkASSERT((s.fInvType & ~(SkMatrix::kTranslate_Mask |
61 SkMatrix::kScale_Mask)) == 0);
62
63 PREAMBLE(s);
64 // we store y, x, x, x, x, x
65
66 const unsigned maxX = s.fBitmap->width() - 1;
67 SkFixed fx;
68 {
69 SkPoint pt;
70 s.fInvProc(*s.fInvMatrix, SkIntToScalar(x) + SK_ScalarHalf,
71 SkIntToScalar(y) + SK_ScalarHalf, &pt);
72 fx = SkScalarToFixed(pt.fY);
73 const unsigned maxY = s.fBitmap->height() - 1;
74 *xy++ = TILEY_PROCF(fx, maxY);
75 fx = SkScalarToFixed(pt.fX);
76 }
77
78 if (0 == maxX) {
79 // all of the following X values must be 0
80 memset(xy, 0, count * sizeof(uint16_t));
81 return;
82 }
83
84 const SkFixed dx = s.fInvSx;
85
86 #ifdef CHECK_FOR_DECAL
87 // test if we don't need to apply the tile proc
88 if ((unsigned)(fx >> 16) <= maxX &&
89 (unsigned)((fx + dx * (count - 1)) >> 16) <= maxX) {
90 decal_nofilter_scale_neon(xy, fx, dx, count);
91 return;
92 }
93 #endif
94
95 int i;
96
97 /* very much like done in decal_nofilter, but with
98 * an extra clamping function applied.
99 * TILEX_PROCF(fx,max) SkClampMax((fx)>>16, max)
100 */
101 if (count >= 8) {
102 /* SkFixed is 16.16 fixed point */
103 SkFixed dx2 = dx+dx;
104 SkFixed dx4 = dx2+dx2;
105 SkFixed dx8 = dx4+dx4;
106
107 /* now build fx/fx+dx/fx+2dx/fx+3dx */
108 SkFixed fx1, fx2, fx3;
109 int32x2_t lower, upper;
110 int32x4_t lbase, hbase;
111 int16_t *dst16 = (int16_t *)xy;
112
113 fx1 = fx+dx;
114 fx2 = fx1+dx;
115 fx3 = fx2+dx;
116
117 /* build my template(s) */
118 /* avoid the 'lbase unitialized' warning */
119 lbase = vdupq_n_s32(fx);
120 lbase = vsetq_lane_s32(fx1, lbase, 1);
121 lbase = vsetq_lane_s32(fx2, lbase, 2);
122 lbase = vsetq_lane_s32(fx3, lbase, 3);
123
124 hbase = vaddq_s32(lbase, vdupq_n_s32(dx4));
125
126 /* store & bump */
127 do {
128 int32x4_t lout;
129 int32x4_t hout;
130 int16x8_t hi16;
131
132 /* get the hi 16s of all those 32s */
133 lout = lbase;
134 hout = hbase;
135 /* this sets up all lout's then all hout's in hout */
136 asm ("vuzpq.16 %q0, %q1" : "+w" (lout), "+w" (hout));
137 hi16 = vreinterpretq_s16_s32(hout);
138
139 /* clamp & output */
140 hi16 = vmaxq_s16(hi16, vdupq_n_s16(0));
141 hi16 = vminq_s16(hi16, vdupq_n_s16(maxX));
142 vst1q_s16(dst16, hi16);
143
144 /* but preserving base & on to the next */
145 lbase = vaddq_s32 (lbase, vdupq_n_s32(dx8));
146 hbase = vaddq_s32 (hbase, vdupq_n_s32(dx8));
147 dst16 += 8;
148 count -= 8;
149 fx += dx8;
150 } while (count >= 8);
151 xy = (uint32_t *) dst16;
152 }
153
154 uint16_t* xx = (uint16_t*)xy;
155 for (i = count; i > 0; --i) {
156 *xx++ = TILEX_PROCF(fx, maxX); fx += dx;
157 }
158 }
159
160 // note: we could special-case on a matrix which is skewed in X but not Y.
161 // this would require a more general setup thatn SCALE does, but could use
162 // SCALE's inner loop that only looks at dx
163
AFFINE_NOFILTER_NAME(const SkBitmapProcState & s,uint32_t xy[],int count,int x,int y)164 static void AFFINE_NOFILTER_NAME(const SkBitmapProcState& s,
165 uint32_t xy[], int count, int x, int y) {
166 SkASSERT(s.fInvType & SkMatrix::kAffine_Mask);
167 SkASSERT((s.fInvType & ~(SkMatrix::kTranslate_Mask |
168 SkMatrix::kScale_Mask |
169 SkMatrix::kAffine_Mask)) == 0);
170
171 PREAMBLE(s);
172 SkPoint srcPt;
173 s.fInvProc(*s.fInvMatrix,
174 SkIntToScalar(x) + SK_ScalarHalf,
175 SkIntToScalar(y) + SK_ScalarHalf, &srcPt);
176
177 SkFixed fx = SkScalarToFixed(srcPt.fX);
178 SkFixed fy = SkScalarToFixed(srcPt.fY);
179 SkFixed dx = s.fInvSx;
180 SkFixed dy = s.fInvKy;
181 int maxX = s.fBitmap->width() - 1;
182 int maxY = s.fBitmap->height() - 1;
183
184 /* NEON lets us do an 8x unrolling */
185 if (count >= 8) {
186 /* SkFixed is 16.16 fixed point */
187 SkFixed dx4 = dx * 4;
188 SkFixed dy4 = dy * 4;
189 SkFixed dx8 = dx * 8;
190 SkFixed dy8 = dy * 8;
191
192 int32x4_t xbase, ybase;
193 int32x4_t x2base, y2base;
194 int16_t *dst16 = (int16_t *) xy;
195
196 /* my sets of maxx/maxy for clamping */
197 int32_t maxpair = (maxX&0xffff) | ((maxY&0xffff)<<16);
198 int16x8_t maxXY = vreinterpretq_s16_s32(vdupq_n_s32(maxpair));
199
200 /* now build fx/fx+dx/fx+2dx/fx+3dx */
201 /* avoid the 'xbase unitialized' warning...*/
202 xbase = vdupq_n_s32(fx);
203 xbase = vsetq_lane_s32(fx+dx, xbase, 1);
204 xbase = vsetq_lane_s32(fx+dx+dx, xbase, 2);
205 xbase = vsetq_lane_s32(fx+dx+dx+dx, xbase, 3);
206
207 /* same for fy */
208 /* avoid the 'ybase unitialized' warning...*/
209 ybase = vdupq_n_s32(fy);
210 ybase = vsetq_lane_s32(fy+dy, ybase, 1);
211 ybase = vsetq_lane_s32(fy+dy+dy, ybase, 2);
212 ybase = vsetq_lane_s32(fy+dy+dy+dy, ybase, 3);
213
214 x2base = vaddq_s32(xbase, vdupq_n_s32(dx4));
215 y2base = vaddq_s32(ybase, vdupq_n_s32(dy4));
216
217 /* store & bump */
218 do {
219 int32x4_t xout, yout;
220 int32x4_t x2out, y2out;
221 int16x8_t hi16, hi16_2;
222
223 xout = xbase;
224 yout = ybase;
225
226 /* overlay y's low16 with hi16 from x */
227 /* so we properly shifted xyxyxyxy */
228 yout = vsriq_n_s32(yout, xout, 16);
229 hi16 = vreinterpretq_s16_s32 (yout);
230
231 /* do the clamping; both guys get 0's */
232 hi16 = vmaxq_s16 (hi16, vdupq_n_s16(0));
233 hi16 = vminq_s16 (hi16, maxXY);
234
235 vst1q_s16 (dst16, hi16);
236
237 /* and for the other 4 pieces of this iteration */
238 x2out = x2base;
239 y2out = y2base;
240
241 /* overlay y's low16 with hi16 from x */
242 /* so we properly shifted xyxyxyxy */
243 y2out = vsriq_n_s32(y2out, x2out, 16);
244 hi16_2 = vreinterpretq_s16_s32 (y2out);
245
246 /* do the clamping; both guys get 0's */
247 hi16_2 = vmaxq_s16 (hi16_2, vdupq_n_s16(0));
248 hi16_2 = vminq_s16 (hi16_2, maxXY);
249
250 /* RBE: gcc regenerates dst16+8 all the time instead
251 * of folding it into an addressing mode. *sigh* */
252 vst1q_s16 (dst16+8, hi16_2);
253
254 /* moving base and on to the next */
255 xbase = vaddq_s32 (xbase, vdupq_n_s32 (dx8));
256 ybase = vaddq_s32 (ybase, vdupq_n_s32 (dy8));
257 x2base = vaddq_s32 (x2base, vdupq_n_s32 (dx8));
258 y2base = vaddq_s32 (y2base, vdupq_n_s32 (dy8));
259
260 dst16 += 16; /* 8x32 aka 16x16 */
261 count -= 8;
262 fx += dx8;
263 fy += dy8;
264 } while (count >= 8);
265 xy = (uint32_t *) dst16;
266 }
267
268 for (int i = count; i > 0; --i) {
269 *xy++ = (TILEY_PROCF(fy, maxY) << 16) | TILEX_PROCF(fx, maxX);
270 fx += dx; fy += dy;
271 }
272 }
273
274 #undef DEBUG_PERSP_NOFILTER
275
PERSP_NOFILTER_NAME(const SkBitmapProcState & s,uint32_t * SK_RESTRICT xy,int count,int x,int y)276 static void PERSP_NOFILTER_NAME(const SkBitmapProcState& s,
277 uint32_t* SK_RESTRICT xy,
278 int count, int x, int y) {
279 SkASSERT(s.fInvType & SkMatrix::kPerspective_Mask);
280
281 PREAMBLE(s);
282 /* max{X,Y} are int here, but later shown/assumed to fit in 16 bits */
283 int maxX = s.fBitmap->width() - 1;
284 int maxY = s.fBitmap->height() - 1;
285
286 SkPerspIter iter(*s.fInvMatrix,
287 SkIntToScalar(x) + SK_ScalarHalf,
288 SkIntToScalar(y) + SK_ScalarHalf, count);
289
290 while ((count = iter.next()) != 0) {
291 const SkFixed* SK_RESTRICT srcXY = iter.getXY();
292
293 #if defined(DEBUG_PERSP_NOFILTER)
294 /* debugging stuff */
295 const SkFixed *end_srcXY = srcXY + (count*2);
296 uint32_t *end_xy = xy + (count);
297 const SkFixed *base_srcXY = srcXY;
298 uint32_t *base_xy = xy;
299 int base_count = count;
300 #endif
301
302 #if 1
303 // 2009/9/30: crashes in ApiDemos - Views - Animation - 3D Transition
304 // 2009/10/9: reworked to avoid illegal (but allowed by gas) insn
305
306 /* srcXY is a batch of 32 bit numbers X0,Y0,X1,Y1...
307 * but we immediately discard the low 16 bits...
308 * so what we're going to do is vld4, which will give us
309 * xlo,xhi,ylo,yhi distribution and we can ignore the 'lo'
310 * parts....
311 */
312 if (count >= 8) {
313 int16_t *mysrc = (int16_t *) srcXY;
314 int16_t *mydst = (int16_t *) xy;
315 int16x4_t maxX4 = vdup_n_s16((int16_t)maxX);
316 int16x4_t maxY4 = vdup_n_s16((int16_t)maxY);
317 int16x4_t zero4 = vdup_n_s16(0);
318
319 /* The constructs with local blocks for register assignments
320 * and asm() instructions is to make keep any hard register
321 * assignments to as small a scope as possible. and to avoid
322 * burning call-preserved hard registers on the vld/vst
323 * instructions.
324 */
325
326 do {
327 int16x4_t xlo, xhi, ylo, yhi;
328 int16x4_t x2lo, x2hi, y2lo, y2hi;
329
330 /* vld4 does the de-interleaving for us */
331 {
332 register int16x4_t t_xlo asm("d0");
333 register int16x4_t t_xhi asm("d1");
334 register int16x4_t t_ylo asm("d2");
335 register int16x4_t t_yhi asm("d3");
336
337 asm ("vld4.16 {d0-d3},[%4] /* xlo=%P0 xhi=%P1 ylo=%P2 yhi=%P3 */"
338 : "=w" (t_xlo), "=w" (t_xhi), "=w" (t_ylo), "=w" (t_yhi)
339 : "r" (mysrc)
340 );
341 xlo = t_xlo;
342 xhi = t_xhi;
343 ylo = t_ylo;
344 yhi = t_yhi;
345 }
346
347 /* clamp X>>16 (aka xhi) to 0..maxX */
348 xhi = vmax_s16(xhi, zero4); /* now 0.. */
349 xhi = vmin_s16(xhi, maxX4); /* now 0..maxX */
350
351 /* clamp Y>>16 (aka yhi) to 0..maxY */
352 yhi = vmax_s16(yhi, zero4); /* now 0.. */
353 yhi = vmin_s16(yhi, maxY4); /* now 0..maxY */
354
355 /* deal with the second set of numbers */
356 {
357 register int16x4_t t_xlo asm("d4");
358 register int16x4_t t_xhi asm("d5");
359 register int16x4_t t_ylo asm("d6");
360 register int16x4_t t_yhi asm("d7");
361
362 /* offset == 256 bits == 32 bytes == 8 longs == 16 shorts */
363 asm ("vld4.16 {d4-d7},[%4] /* xlo=%P0 xhi=%P1 ylo=%P2 yhi=%P3 */"
364 : "=w" (t_xlo), "=w" (t_xhi), "=w" (t_ylo), "=w" (t_yhi)
365 : "r" (mysrc+16)
366 );
367 x2lo = t_xlo;
368 x2hi = t_xhi;
369 y2lo = t_ylo;
370 y2hi = t_yhi;
371 }
372
373 /* clamp the second 4 here */
374
375 if (0) { extern void rbe(void); rbe(); }
376
377 /* clamp X>>16 (aka xhi) to 0..maxX */
378 x2hi = vmax_s16(x2hi, zero4); /* now 0.. */
379 x2hi = vmin_s16(x2hi, maxX4); /* now 0..maxX */
380
381 /* clamp Y>>16 (aka yhi) to 0..maxY */
382 y2hi = vmax_s16(y2hi, zero4); /* now 0.. */
383 y2hi = vmin_s16(y2hi, maxY4); /* now 0..maxY */
384
385 /* we're storing as {x,y}s: x is [0], y is [1] */
386 /* we'll use vst2 to make this happen */
387
388 {
389 register int16x4_t out_x asm("d16") = xhi;
390 register int16x4_t out_y asm("d17") = yhi;
391
392 asm ("vst2.16 {d16-d17},[%2] /* xlo=%P0 xhi=%P1 */"
393 :
394 : "w" (out_x), "w" (out_y), "r" (mydst)
395 );
396 }
397 {
398 register int16x4_t out_x asm("d18") = x2hi;
399 register int16x4_t out_y asm("d19") = y2hi;
400
401 asm ("vst2.16 {d18-d19},[%2] /* xlo=%P0 xhi=%P1 */"
402 :
403 : "w" (out_x), "w" (out_y), "r" (mydst+8)
404 );
405 }
406
407 /* XXX: gcc isn't interleaving these with the NEON ops
408 * but i think that all the scoreboarding works out */
409 count -= 8; /* 8 iterations */
410 mysrc += 32; /* 16 longs, aka 32 shorts */
411 mydst += 16; /* 16 shorts, aka 8 longs */
412 } while (count >= 8);
413 /* get xy and srcXY fixed up */
414 srcXY = (const SkFixed *) mysrc;
415 xy = (uint32_t *) mydst;
416 }
417 #endif
418
419 while (--count >= 0) {
420 *xy++ = (TILEY_PROCF(srcXY[1], maxY) << 16) |
421 TILEX_PROCF(srcXY[0], maxX);
422 srcXY += 2;
423 }
424
425 #if defined(DEBUG_PERSP_NOFILTER)
426 /* for checking our NEON-produced results against vanilla code */
427 {
428 int bad = (-1);
429 for (int i = 0; i < base_count; i++) {
430 uint32_t val;
431 val = (TILEY_PROCF (base_srcXY[i * 2 + 1], maxY) << 16) |
432 TILEX_PROCF (base_srcXY[i * 2 + 0], maxX);
433
434 if (val != base_xy[i]) {
435 bad = i;
436 break;
437 }
438 }
439 if (bad >= 0) {
440 SkDebugf("clamp-nofilter-persp failed piece %d\n", bad);
441 SkDebugf(" maxX %08x maxY %08x\n", maxX, maxY);
442 bad -= (bad & 0x7); /* align */
443 for (int i = bad; i < bad + 8; i++) {
444 uint32_t val;
445 val = (TILEY_PROCF (base_srcXY[i * 2 + 1], maxY) << 16) |
446 TILEX_PROCF (base_srcXY[i * 2 + 0], maxX);
447
448 SkDebugf("%d: got %08x want %08x srcXY[0] %08x srcXY[1] %08x\n",
449 i, base_xy[i], val, base_srcXY[i * 2 + 0],
450 base_srcXY[i * 2 + 1]);
451 }
452 SkDebugf ("---\n");
453 }
454
455 if (end_xy != xy) {
456 SkDebugf("xy ended at %08x, should be %08x\n", xy, end_xy);
457 }
458 if (end_srcXY != srcXY) {
459 SkDebugf("srcXY ended at %08x, should be %08x\n", srcXY,
460 end_srcXY);
461 }
462 }
463 #endif
464 }
465 }
466
467 #undef DEBUG_PERSP_NOFILTER
468
469 //////////////////////////////////////////////////////////////////////////////
470
PACK_FILTER_Y_NAME(SkFixed f,unsigned max,SkFixed one PREAMBLE_PARAM_Y)471 static inline uint32_t PACK_FILTER_Y_NAME(SkFixed f, unsigned max,
472 SkFixed one PREAMBLE_PARAM_Y) {
473 unsigned i = TILEY_PROCF(f, max);
474 i = (i << 4) | TILEY_LOW_BITS(f, max);
475 return (i << 14) | (TILEY_PROCF((f + one), max));
476 }
477
PACK_FILTER_X_NAME(SkFixed f,unsigned max,SkFixed one PREAMBLE_PARAM_X)478 static inline uint32_t PACK_FILTER_X_NAME(SkFixed f, unsigned max,
479 SkFixed one PREAMBLE_PARAM_X) {
480 unsigned i = TILEX_PROCF(f, max);
481 i = (i << 4) | TILEX_LOW_BITS(f, max);
482 return (i << 14) | (TILEX_PROCF((f + one), max));
483 }
484
SCALE_FILTER_NAME(const SkBitmapProcState & s,uint32_t xy[],int count,int x,int y)485 static void SCALE_FILTER_NAME(const SkBitmapProcState& s,
486 uint32_t xy[], int count, int x, int y) {
487 SkASSERT((s.fInvType & ~(SkMatrix::kTranslate_Mask |
488 SkMatrix::kScale_Mask)) == 0);
489 SkASSERT(s.fInvKy == 0);
490
491 PREAMBLE(s);
492
493 const unsigned maxX = s.fBitmap->width() - 1;
494 const SkFixed one = s.fFilterOneX;
495 const SkFixed dx = s.fInvSx;
496 SkFixed fx;
497
498 {
499 SkPoint pt;
500 s.fInvProc(*s.fInvMatrix, SkIntToScalar(x) + SK_ScalarHalf,
501 SkIntToScalar(y) + SK_ScalarHalf, &pt);
502 const SkFixed fy = SkScalarToFixed(pt.fY) - (s.fFilterOneY >> 1);
503 const unsigned maxY = s.fBitmap->height() - 1;
504 // compute our two Y values up front
505 *xy++ = PACK_FILTER_Y_NAME(fy, maxY, s.fFilterOneY PREAMBLE_ARG_Y);
506 // now initialize fx
507 fx = SkScalarToFixed(pt.fX) - (one >> 1);
508 }
509
510 #ifdef CHECK_FOR_DECAL
511 // test if we don't need to apply the tile proc
512 if (dx > 0 &&
513 (unsigned)(fx >> 16) <= maxX &&
514 (unsigned)((fx + dx * (count - 1)) >> 16) < maxX) {
515 decal_filter_scale_neon(xy, fx, dx, count);
516 } else
517 #endif
518
519 if (count >= 4) {
520 int32x4_t wide_dx, wide_one;
521 int32x4_t wide_fx, wide_fx1, wide_i, wide_lo;
522 #if 0
523 /* verification hooks -- see below */
524 SkFixed debug_fx = fx;
525 int count_done = 0;
526 #endif
527
528 wide_fx = vdupq_n_s32(fx);
529 wide_fx = vsetq_lane_s32(fx+dx, wide_fx, 1);
530 wide_fx = vsetq_lane_s32(fx+dx+dx, wide_fx, 2);
531 wide_fx = vsetq_lane_s32(fx+dx+dx+dx, wide_fx, 3);
532
533 wide_dx = vdupq_n_s32(dx);
534 wide_one = vdupq_n_s32(one);
535
536 while (count >= 4) {
537 /* original expands to:
538 * unsigned i = SkClampMax((f) >> 16, max);
539 * i = (i << 4) | (((f) >> 12) & 0xF);
540 * return (i << 14) | (SkClampMax(((f + one)) >> 16, max));
541 */
542
543 /* i = SkClampMax(f>>16, maxX) */
544 wide_i = vmaxq_s32(vshrq_n_s32(wide_fx,16), vdupq_n_s32(0));
545 wide_i = vminq_s32(wide_i, vdupq_n_s32(maxX));
546
547 /* i<<4 | TILEX_LOW_BITS(fx) */
548 wide_lo = vshrq_n_s32(wide_fx, 12);
549 wide_i = vsliq_n_s32(wide_lo, wide_i, 4);
550
551 /* i<<14 */
552 wide_i = vshlq_n_s32(wide_i, 14);
553
554 /* SkClampMax(((f + one)) >> 16, max) */
555 wide_fx1 = vaddq_s32(wide_fx, wide_one);
556 wide_fx1 = vmaxq_s32(vshrq_n_s32(wide_fx1,16), vdupq_n_s32(0));
557 wide_fx1 = vminq_s32(wide_fx1, vdupq_n_s32(maxX));
558
559 /* final combination */
560 wide_i = vorrq_s32(wide_i, wide_fx1);
561
562 vst1q_u32(xy, vreinterpretq_u32_s32(wide_i));
563
564 #if 0
565 /* having a verification hook is a good idea */
566 /* use debug_fx, debug_fx+dx, etc. */
567
568 for (int i=0;i<4;i++) {
569 uint32_t want = PACK_FILTER_X_NAME(debug_fx, maxX, one PREAMBLE_ARG_X);
570 if (xy[i] != want)
571 {
572 /* print a nastygram */
573 SkDebugf("clamp-filter-scale fails\n");
574 SkDebugf("got %08x want %08x\n", xy[i], want);
575 SkDebugf("fx %08x debug_fx %08x dx %08x done %d\n",
576 fx, debug_fx, dx, count_done);
577 SkDebugf(" maxX %08x one %08x\n", maxX, one);
578
579 }
580 debug_fx += dx;
581 count_done++;
582 }
583 #endif
584 wide_fx += vdupq_n_s32(dx+dx+dx+dx);
585 fx += dx+dx+dx+dx;
586 xy += 4;
587 count -= 4;
588 }
589 }
590
591 while (--count >= 0) {
592 *xy++ = PACK_FILTER_X_NAME(fx, maxX, one PREAMBLE_ARG_X);
593 fx += dx;
594 }
595 }
596
AFFINE_FILTER_NAME(const SkBitmapProcState & s,uint32_t xy[],int count,int x,int y)597 static void AFFINE_FILTER_NAME(const SkBitmapProcState& s,
598 uint32_t xy[], int count, int x, int y) {
599 SkASSERT(s.fInvType & SkMatrix::kAffine_Mask);
600 SkASSERT((s.fInvType & ~(SkMatrix::kTranslate_Mask |
601 SkMatrix::kScale_Mask |
602 SkMatrix::kAffine_Mask)) == 0);
603
604 PREAMBLE(s);
605 SkPoint srcPt;
606 s.fInvProc(*s.fInvMatrix,
607 SkIntToScalar(x) + SK_ScalarHalf,
608 SkIntToScalar(y) + SK_ScalarHalf, &srcPt);
609
610 SkFixed oneX = s.fFilterOneX;
611 SkFixed oneY = s.fFilterOneY;
612 SkFixed fx = SkScalarToFixed(srcPt.fX) - (oneX >> 1);
613 SkFixed fy = SkScalarToFixed(srcPt.fY) - (oneY >> 1);
614 SkFixed dx = s.fInvSx;
615 SkFixed dy = s.fInvKy;
616 unsigned maxX = s.fBitmap->width() - 1;
617 unsigned maxY = s.fBitmap->height() - 1;
618
619 if (count >= 4) {
620 int32x4_t wide_one, wide_i, wide_lo;
621 int32x4_t wide_dx, wide_fx, wide_onex, wide_fx1;
622 int32x4_t wide_dy, wide_fy, wide_oney, wide_fy1;
623
624 #undef AFFINE_DEBUG
625 #if defined(AFFINE_DEBUG)
626 SkFixed fyp = fy;
627 SkFixed fxp = fx;
628 uint32_t *xyp = xy;
629 int count_done = 0;
630 #endif
631
632 wide_fx = vdupq_n_s32(fx);
633 wide_fx = vsetq_lane_s32(fx+dx, wide_fx, 1);
634 wide_fx = vsetq_lane_s32(fx+dx+dx, wide_fx, 2);
635 wide_fx = vsetq_lane_s32(fx+dx+dx+dx, wide_fx, 3);
636 wide_dx = vdupq_n_s32(dx);
637
638 wide_fy = vdupq_n_s32(fy);
639 wide_fy = vsetq_lane_s32(fy+dy, wide_fy, 1);
640 wide_fy = vsetq_lane_s32(fy+dy+dy, wide_fy, 2);
641 wide_fy = vsetq_lane_s32(fy+dy+dy+dy, wide_fy, 3);
642 wide_dy = vdupq_n_s32(dy);
643
644 wide_onex = vdupq_n_s32(oneX);
645 wide_oney = vdupq_n_s32(oneY);
646
647 while (count >= 4) {
648 int32x4_t wide_x;
649 int32x4_t wide_y;
650
651 /* do the X side, then the Y side, then interleave them */
652
653 /* original expands to:
654 * unsigned i = SkClampMax((f) >> 16, max);
655 * i = (i << 4) | (((f) >> 12) & 0xF);
656 * return (i << 14) | (SkClampMax(((f + one)) >> 16, max));
657 */
658
659 /* i = SkClampMax(f>>16, maxX) */
660 wide_i = vmaxq_s32(vshrq_n_s32(wide_fx,16), vdupq_n_s32(0));
661 wide_i = vminq_s32(wide_i, vdupq_n_s32(maxX));
662
663 /* i<<4 | TILEX_LOW_BITS(fx) */
664 wide_lo = vshrq_n_s32(wide_fx, 12);
665 wide_i = vsliq_n_s32(wide_lo, wide_i, 4);
666
667 /* i<<14 */
668 wide_i = vshlq_n_s32(wide_i, 14);
669
670 /* SkClampMax(((f + one)) >> 16, max) */
671 wide_fx1 = vaddq_s32(wide_fx, wide_onex);
672 wide_fx1 = vmaxq_s32(vshrq_n_s32(wide_fx1,16), vdupq_n_s32(0));
673 wide_fx1 = vminq_s32(wide_fx1, vdupq_n_s32(maxX));
674
675 /* final combination */
676 wide_x = vorrq_s32(wide_i, wide_fx1);
677
678 /* And now the Y side */
679
680 /* i = SkClampMax(f>>16, maxX) */
681 wide_i = vmaxq_s32(vshrq_n_s32(wide_fy,16), vdupq_n_s32(0));
682 wide_i = vminq_s32(wide_i, vdupq_n_s32(maxY));
683
684 /* i<<4 | TILEX_LOW_BITS(fx) */
685 wide_lo = vshrq_n_s32(wide_fy, 12);
686 wide_i = vsliq_n_s32(wide_lo, wide_i, 4);
687
688 /* i<<14 */
689 wide_i = vshlq_n_s32(wide_i, 14);
690
691 /* SkClampMax(((f + one)) >> 16, max) */
692 wide_fy1 = vaddq_s32(wide_fy, wide_oney);
693 wide_fy1 = vmaxq_s32(vshrq_n_s32(wide_fy1,16), vdupq_n_s32(0));
694 wide_fy1 = vminq_s32(wide_fy1, vdupq_n_s32(maxY));
695
696 /* final combination */
697 wide_y = vorrq_s32(wide_i, wide_fy1);
698
699 /* interleave as YXYXYXYX as part of the storing */
700 {
701 /* vst2.32 needs side-by-side registers */
702 register int32x4_t t_x asm("q1");
703 register int32x4_t t_y asm("q0");
704
705 t_x = wide_x; t_y = wide_y;
706 asm ("vst2.32 {q0-q1},[%2] /* y=%q0 x=%q1 */"
707 :
708 : "w" (t_y), "w" (t_x), "r" (xy)
709 );
710 }
711
712 #if defined(AFFINE_DEBUG)
713 /* make sure we're good here -- check the 4 we just output */
714 for (int i = 0; i<4;i++) {
715 uint32_t val;
716 val = PACK_FILTER_Y_NAME(fyp, maxY, oneY PREAMBLE_ARG_Y);
717 if (val != xy[i*2+0]) {
718 /* print a nastygram */
719 SkDebugf("clamp-filter-affine fails\n");
720 SkDebugf("[bad-y] got %08x want %08x\n", xy[i*2+0], val);
721 SkDebugf("fy %08x fxp %08x fyp %08x dx %08x dy %08x done %d\n",
722 fy, fxp, fyp, dx, dy, count_done);
723 SkDebugf(" maxY %08x oneY %08x\n", maxY, oneY);
724 }
725 val = PACK_FILTER_X_NAME(fxp, maxX, oneX PREAMBLE_ARG_X);
726 if (val != xy[i*2+1]) {
727 /* print a nastygram */
728 SkDebugf("clamp-filter-affine fails\n");
729 SkDebugf("[bad-x] got %08x want %08x\n", xy[i*2+1], val);
730 SkDebugf("fx %08x fxp %08x fyp %08x dx %08x dy %08x done %d\n",
731 fx, fxp, fyp, dx, dy, count_done);
732 SkDebugf(" maxX %08x one %08x\n", maxX, oneX);
733 }
734 fyp += dy;
735 fxp += dx;
736 count_done++;
737 }
738 #endif
739
740 wide_fx += vdupq_n_s32(dx+dx+dx+dx);
741 fx += dx+dx+dx+dx;
742 wide_fy += vdupq_n_s32(dy+dy+dy+dy);
743 fy += dy+dy+dy+dy;
744 xy += 8; /* 4 x's, 4 y's */
745 count -= 4;
746 }
747 }
748
749 while (--count >= 0) {
750 /* NB: writing Y/X */
751 *xy++ = PACK_FILTER_Y_NAME(fy, maxY, oneY PREAMBLE_ARG_Y);
752 fy += dy;
753 *xy++ = PACK_FILTER_X_NAME(fx, maxX, oneX PREAMBLE_ARG_X);
754 fx += dx;
755 }
756 }
757
PERSP_FILTER_NAME(const SkBitmapProcState & s,uint32_t * SK_RESTRICT xy,int count,int x,int y)758 static void PERSP_FILTER_NAME(const SkBitmapProcState& s,
759 uint32_t* SK_RESTRICT xy, int count,
760 int x, int y) {
761 SkASSERT(s.fInvType & SkMatrix::kPerspective_Mask);
762
763 PREAMBLE(s);
764 unsigned maxX = s.fBitmap->width() - 1;
765 unsigned maxY = s.fBitmap->height() - 1;
766 SkFixed oneX = s.fFilterOneX;
767 SkFixed oneY = s.fFilterOneY;
768
769 SkPerspIter iter(*s.fInvMatrix,
770 SkIntToScalar(x) + SK_ScalarHalf,
771 SkIntToScalar(y) + SK_ScalarHalf, count);
772
773 while ((count = iter.next()) != 0) {
774 const SkFixed* SK_RESTRICT srcXY = iter.getXY();
775
776 if (count >= 4) {
777 int32x4_t wide_one, wide_i, wide_lo;
778 int32x4_t wide_fx1;
779 int32x4_t wide_fy1;
780 int32x4_t wide_x, wide_y;
781
782 while (count >= 4) {
783 /* RBE: it's good, but:
784 * -- we spill a constant that could be easily regnerated
785 * [perhaps tweak gcc's NEON constant costs?]
786 */
787
788 /* load src: x-y-x-y-x-y-x-y */
789 {
790 register int32x4_t q0 asm ("q0");
791 register int32x4_t q1 asm ("q1");
792 asm ("vld2.32 {q0-q1},[%2] /* x=%q0 y=%q1 */"
793 : "=w" (q0), "=w" (q1)
794 : "r" (srcXY));
795 wide_x = q0; wide_y = q1;
796 }
797
798 /* do the X side, then the Y side, then interleave them */
799
800 wide_x = vsubq_s32(wide_x, vdupq_n_s32 (oneX>>1));
801
802 /* original expands to:
803 * unsigned i = SkClampMax((f) >> 16, max);
804 * i = (i << 4) | (((f) >> 12) & 0xF);
805 * return (i << 14) | (SkClampMax(((f + one)) >> 16, max));
806 */
807
808 /* i = SkClampMax(f>>16, maxX) */
809 wide_i = vmaxq_s32 (vshrq_n_s32 (wide_x, 16), vdupq_n_s32 (0));
810 wide_i = vminq_s32 (wide_i, vdupq_n_s32 (maxX));
811
812 /* i<<4 | TILEX_LOW_BITS(fx) */
813 wide_lo = vshrq_n_s32 (wide_x, 12);
814 wide_i = vsliq_n_s32 (wide_lo, wide_i, 4);
815
816 /* i<<14 */
817 wide_i = vshlq_n_s32 (wide_i, 14);
818
819 /* SkClampMax(((f + one)) >> 16, max) */
820 wide_fx1 = vaddq_s32 (wide_x, vdupq_n_s32(oneX));
821 wide_fx1 = vmaxq_s32 (vshrq_n_s32 (wide_fx1, 16), vdupq_n_s32 (0));
822 wide_fx1 = vminq_s32 (wide_fx1, vdupq_n_s32 (maxX));
823
824 /* final combination */
825 wide_x = vorrq_s32 (wide_i, wide_fx1);
826
827
828 /* And now the Y side */
829
830 wide_y = vsubq_s32(wide_y, vdupq_n_s32 (oneY>>1));
831
832 /* i = SkClampMax(f>>16, maxX) */
833 wide_i = vmaxq_s32 (vshrq_n_s32 (wide_y, 16), vdupq_n_s32 (0));
834 wide_i = vminq_s32 (wide_i, vdupq_n_s32 (maxY));
835
836 /* i<<4 | TILEX_LOW_BITS(fx) */
837 wide_lo = vshrq_n_s32 (wide_y, 12);
838 wide_i = vsliq_n_s32 (wide_lo, wide_i, 4);
839
840 /* i<<14 */
841 wide_i = vshlq_n_s32 (wide_i, 14);
842
843 /* SkClampMax(((f + one)) >> 16, max) */
844
845 /* wide_fy1_1 and wide_fy1_2 are just temporary variables to
846 * work-around an ICE in debug */
847 int32x4_t wide_fy1_1 = vaddq_s32 (wide_y, vdupq_n_s32(oneY));
848 int32x4_t wide_fy1_2 = vmaxq_s32 (vshrq_n_s32 (wide_fy1_1, 16),
849 vdupq_n_s32 (0));
850 wide_fy1 = vminq_s32 (wide_fy1_2, vdupq_n_s32 (maxY));
851
852 /* final combination */
853 wide_y = vorrq_s32 (wide_i, wide_fy1);
854
855 /* switch them around; have to do it this way to get them
856 * in the proper registers to match our instruction */
857
858 /* iteration bookkeeping, ahead of the asm() for scheduling */
859 srcXY += 2*4;
860 count -= 4;
861
862 /* store interleaved as y-x-y-x-y-x-y-x (NB != read order) */
863 {
864 register int32x4_t q0 asm ("q0") = wide_y;
865 register int32x4_t q1 asm ("q1") = wide_x;
866
867 asm ("vst2.32 {q0-q1},[%2] /* y=%q0 x=%q1 */"
868 :
869 : "w" (q0), "w" (q1), "r" (xy));
870 }
871
872 /* on to the next iteration */
873 /* count, srcXY are handled above */
874 xy += 2*4;
875 }
876 }
877
878 /* was do-while; NEON code invalidates original count>0 assumption */
879 while (--count >= 0) {
880 /* NB: we read x/y, we write y/x */
881 *xy++ = PACK_FILTER_Y_NAME(srcXY[1] - (oneY >> 1), maxY,
882 oneY PREAMBLE_ARG_Y);
883 *xy++ = PACK_FILTER_X_NAME(srcXY[0] - (oneX >> 1), maxX,
884 oneX PREAMBLE_ARG_X);
885 srcXY += 2;
886 }
887 }
888 }
889
890 const SkBitmapProcState::MatrixProc MAKENAME(_Procs)[] = {
891 SCALE_NOFILTER_NAME,
892 SCALE_FILTER_NAME,
893 AFFINE_NOFILTER_NAME,
894 AFFINE_FILTER_NAME,
895 PERSP_NOFILTER_NAME,
896 PERSP_FILTER_NAME
897 };
898
899 #undef MAKENAME
900 #undef TILEX_PROCF
901 #undef TILEY_PROCF
902 #ifdef CHECK_FOR_DECAL
903 #undef CHECK_FOR_DECAL
904 #endif
905
906 #undef SCALE_NOFILTER_NAME
907 #undef SCALE_FILTER_NAME
908 #undef AFFINE_NOFILTER_NAME
909 #undef AFFINE_FILTER_NAME
910 #undef PERSP_NOFILTER_NAME
911 #undef PERSP_FILTER_NAME
912
913 #undef PREAMBLE
914 #undef PREAMBLE_PARAM_X
915 #undef PREAMBLE_PARAM_Y
916 #undef PREAMBLE_ARG_X
917 #undef PREAMBLE_ARG_Y
918
919 #undef TILEX_LOW_BITS
920 #undef TILEY_LOW_BITS
921