1 /* NEON optimized code (C) COPYRIGHT 2009 Motorola */
2 /*
3 * Modifications done in-house at Motorola
4 *
5 * this is a clone of SkBitmapProcState_matrix.h
6 * and has been tuned to work with the NEON unit.
7 *
8 * Still going back and forth between whether this approach
9 * (clone the entire SkBitmapProcState_matrix.h file or
10 * if I should put just the modified routines in here and
11 * then use a construct like #define DONT_DO_THIS_FUNCTION or
12 * something like that...
13 *
14 * This is for the ClampX_ClampY instance
15 *
16 */
17
18
19 #if !defined(__ARM_HAVE_NEON)
20 #error this file can be used only when the NEON unit is enabled
21 #endif
22
23 #include <arm_neon.h>
24
25 /*
26 * This has been modified on the knowledge that (at the time)
27 * we had the following macro definitions in the parent file
28 *
29 * #define MAKENAME(suffix) ClampX_ClampY ## suffix
30 * #define TILEX_PROCF(fx, max) SkClampMax((fx) >> 16, max)
31 * #define TILEY_PROCF(fy, max) SkClampMax((fy) >> 16, max)
32 * #define TILEX_LOW_BITS(fx, max) (((fx) >> 12) & 0xF)
33 * #define TILEY_LOW_BITS(fy, max) (((fy) >> 12) & 0xF)
34 * #define CHECK_FOR_DECAL
35 */
36
37 /* SkClampMax(val,max) -- bound to 0..max */
38
39 #define SCALE_NOFILTER_NAME MAKENAME(_nofilter_scale)
40 #define SCALE_FILTER_NAME MAKENAME(_filter_scale)
41 #define AFFINE_NOFILTER_NAME MAKENAME(_nofilter_affine)
42 #define AFFINE_FILTER_NAME MAKENAME(_filter_affine)
43 #define PERSP_NOFILTER_NAME MAKENAME(_nofilter_persp)
44 #define PERSP_FILTER_NAME MAKENAME(_filter_persp)
45
46 #define PACK_FILTER_X_NAME MAKENAME(_pack_filter_x)
47 #define PACK_FILTER_Y_NAME MAKENAME(_pack_filter_y)
48
49 #ifndef PREAMBLE
50 #define PREAMBLE(state)
51 #define PREAMBLE_PARAM_X
52 #define PREAMBLE_PARAM_Y
53 #define PREAMBLE_ARG_X
54 #define PREAMBLE_ARG_Y
55 #endif
56
SCALE_NOFILTER_NAME(const SkBitmapProcState & s,uint32_t xy[],int count,int x,int y)57 static void SCALE_NOFILTER_NAME(const SkBitmapProcState& s,
58 uint32_t xy[], int count, int x, int y) {
59 SkASSERT((s.fInvType & ~(SkMatrix::kTranslate_Mask |
60 SkMatrix::kScale_Mask)) == 0);
61
62 PREAMBLE(s);
63 // we store y, x, x, x, x, x
64
65 const unsigned maxX = s.fBitmap->width() - 1;
66 SkFixed fx;
67 {
68 SkPoint pt;
69 s.fInvProc(*s.fInvMatrix, SkIntToScalar(x) + SK_ScalarHalf,
70 SkIntToScalar(y) + SK_ScalarHalf, &pt);
71 fx = SkScalarToFixed(pt.fY);
72 const unsigned maxY = s.fBitmap->height() - 1;
73 *xy++ = TILEY_PROCF(fx, maxY);
74 fx = SkScalarToFixed(pt.fX);
75 }
76
77 if (0 == maxX) {
78 // all of the following X values must be 0
79 memset(xy, 0, count * sizeof(uint16_t));
80 return;
81 }
82
83 const SkFixed dx = s.fInvSx;
84
85 #ifdef CHECK_FOR_DECAL
86 // test if we don't need to apply the tile proc
87 if ((unsigned)(fx >> 16) <= maxX &&
88 (unsigned)((fx + dx * (count - 1)) >> 16) <= maxX) {
89 decal_nofilter_scale(xy, fx, dx, count);
90 return;
91 }
92 #endif
93
94 int i;
95
96 /* very much like done in decal_nofilter, but with
97 * an extra clamping function applied.
98 * TILEX_PROCF(fx,max) SkClampMax((fx)>>16, max)
99 */
100 if (count >= 8) {
101 /* SkFixed is 16.16 fixed point */
102 SkFixed dx2 = dx+dx;
103 SkFixed dx4 = dx2+dx2;
104 SkFixed dx8 = dx4+dx4;
105
106 /* now build fx/fx+dx/fx+2dx/fx+3dx */
107 SkFixed fx1, fx2, fx3;
108 int32x2_t lower, upper;
109 int32x4_t lbase, hbase;
110 int16_t *dst16 = (int16_t *)xy;
111
112 fx1 = fx+dx;
113 fx2 = fx1+dx;
114 fx3 = fx2+dx;
115
116 /* build my template(s) */
117 /* avoid the 'lbase unitialized' warning */
118 lbase = vdupq_n_s32(fx);
119 lbase = vsetq_lane_s32(fx1, lbase, 1);
120 lbase = vsetq_lane_s32(fx2, lbase, 2);
121 lbase = vsetq_lane_s32(fx3, lbase, 3);
122
123 hbase = vaddq_s32(lbase, vdupq_n_s32(dx4));
124
125 /* store & bump */
126 do {
127 int32x4_t lout;
128 int32x4_t hout;
129 int16x8_t hi16;
130
131 /* get the hi 16s of all those 32s */
132 lout = lbase;
133 hout = hbase;
134 /* this sets up all lout's then all hout's in hout */
135 asm ("vuzpq.16 %q0, %q1" : "+w" (lout), "+w" (hout));
136 hi16 = vreinterpretq_s16_s32(hout);
137
138 /* clamp & output */
139 hi16 = vmaxq_s16(hi16, vdupq_n_s16(0));
140 hi16 = vminq_s16(hi16, vdupq_n_s16(maxX));
141 vst1q_s16(dst16, hi16);
142
143 /* but preserving base & on to the next */
144 lbase = vaddq_s32 (lbase, vdupq_n_s32(dx8));
145 hbase = vaddq_s32 (hbase, vdupq_n_s32(dx8));
146 dst16 += 8;
147 count -= 8;
148 fx += dx8;
149 } while (count >= 8);
150 xy = (uint32_t *) dst16;
151 }
152
153 uint16_t* xx = (uint16_t*)xy;
154 for (i = count; i > 0; --i) {
155 *xx++ = TILEX_PROCF(fx, maxX); fx += dx;
156 }
157 }
158
159 // note: we could special-case on a matrix which is skewed in X but not Y.
160 // this would require a more general setup thatn SCALE does, but could use
161 // SCALE's inner loop that only looks at dx
162
AFFINE_NOFILTER_NAME(const SkBitmapProcState & s,uint32_t xy[],int count,int x,int y)163 static void AFFINE_NOFILTER_NAME(const SkBitmapProcState& s,
164 uint32_t xy[], int count, int x, int y) {
165 SkASSERT(s.fInvType & SkMatrix::kAffine_Mask);
166 SkASSERT((s.fInvType & ~(SkMatrix::kTranslate_Mask |
167 SkMatrix::kScale_Mask |
168 SkMatrix::kAffine_Mask)) == 0);
169
170 PREAMBLE(s);
171 SkPoint srcPt;
172 s.fInvProc(*s.fInvMatrix,
173 SkIntToScalar(x) + SK_ScalarHalf,
174 SkIntToScalar(y) + SK_ScalarHalf, &srcPt);
175
176 SkFixed fx = SkScalarToFixed(srcPt.fX);
177 SkFixed fy = SkScalarToFixed(srcPt.fY);
178 SkFixed dx = s.fInvSx;
179 SkFixed dy = s.fInvKy;
180 int maxX = s.fBitmap->width() - 1;
181 int maxY = s.fBitmap->height() - 1;
182
183 /* NEON lets us do an 8x unrolling */
184 if (count >= 8) {
185 /* SkFixed is 16.16 fixed point */
186 SkFixed dx4 = dx * 4;
187 SkFixed dy4 = dy * 4;
188 SkFixed dx8 = dx * 8;
189 SkFixed dy8 = dy * 8;
190
191 int32x4_t xbase, ybase;
192 int32x4_t x2base, y2base;
193 int16_t *dst16 = (int16_t *) xy;
194
195 /* my sets of maxx/maxy for clamping */
196 int32_t maxpair = (maxX&0xffff) | ((maxY&0xffff)<<16);
197 int16x8_t maxXY = vreinterpretq_s16_s32(vdupq_n_s32(maxpair));
198
199 /* now build fx/fx+dx/fx+2dx/fx+3dx */
200 /* avoid the 'xbase unitialized' warning...*/
201 xbase = vdupq_n_s32(fx);
202 xbase = vsetq_lane_s32(fx+dx, xbase, 1);
203 xbase = vsetq_lane_s32(fx+dx+dx, xbase, 2);
204 xbase = vsetq_lane_s32(fx+dx+dx+dx, xbase, 3);
205
206 /* same for fy */
207 /* avoid the 'ybase unitialized' warning...*/
208 ybase = vdupq_n_s32(fy);
209 ybase = vsetq_lane_s32(fy+dy, ybase, 1);
210 ybase = vsetq_lane_s32(fy+dy+dy, ybase, 2);
211 ybase = vsetq_lane_s32(fy+dy+dy+dy, ybase, 3);
212
213 x2base = vaddq_s32(xbase, vdupq_n_s32(dx4));
214 y2base = vaddq_s32(ybase, vdupq_n_s32(dy4));
215
216 /* store & bump */
217 do {
218 int32x4_t xout, yout;
219 int32x4_t x2out, y2out;
220 int16x8_t hi16, hi16_2;
221
222 xout = xbase;
223 yout = ybase;
224
225 /* overlay y's low16 with hi16 from x */
226 /* so we properly shifted xyxyxyxy */
227 yout = vsriq_n_s32(yout, xout, 16);
228 hi16 = vreinterpretq_s16_s32 (yout);
229
230 /* do the clamping; both guys get 0's */
231 hi16 = vmaxq_s16 (hi16, vdupq_n_s16(0));
232 hi16 = vminq_s16 (hi16, maxXY);
233
234 vst1q_s16 (dst16, hi16);
235
236 /* and for the other 4 pieces of this iteration */
237 x2out = x2base;
238 y2out = y2base;
239
240 /* overlay y's low16 with hi16 from x */
241 /* so we properly shifted xyxyxyxy */
242 y2out = vsriq_n_s32(y2out, x2out, 16);
243 hi16_2 = vreinterpretq_s16_s32 (y2out);
244
245 /* do the clamping; both guys get 0's */
246 hi16_2 = vmaxq_s16 (hi16_2, vdupq_n_s16(0));
247 hi16_2 = vminq_s16 (hi16_2, maxXY);
248
249 /* RBE: gcc regenerates dst16+8 all the time instead
250 * of folding it into an addressing mode. *sigh* */
251 vst1q_s16 (dst16+8, hi16_2);
252
253 /* moving base and on to the next */
254 xbase = vaddq_s32 (xbase, vdupq_n_s32 (dx8));
255 ybase = vaddq_s32 (ybase, vdupq_n_s32 (dy8));
256 x2base = vaddq_s32 (x2base, vdupq_n_s32 (dx8));
257 y2base = vaddq_s32 (y2base, vdupq_n_s32 (dy8));
258
259 dst16 += 16; /* 8x32 aka 16x16 */
260 count -= 8;
261 fx += dx8;
262 fy += dy8;
263 } while (count >= 8);
264 xy = (uint32_t *) dst16;
265 }
266
267 for (int i = count; i > 0; --i) {
268 *xy++ = (TILEY_PROCF(fy, maxY) << 16) | TILEX_PROCF(fx, maxX);
269 fx += dx; fy += dy;
270 }
271 }
272
PERSP_NOFILTER_NAME(const SkBitmapProcState & s,uint32_t * SK_RESTRICT xy,int count,int x,int y)273 static void PERSP_NOFILTER_NAME(const SkBitmapProcState& s,
274 uint32_t* SK_RESTRICT xy,
275 int count, int x, int y) {
276 SkASSERT(s.fInvType & SkMatrix::kPerspective_Mask);
277
278 PREAMBLE(s);
279 /* max{X,Y} are int here, but later shown/assumed to fit in 16 bits */
280 int maxX = s.fBitmap->width() - 1;
281 int maxY = s.fBitmap->height() - 1;
282
283 SkPerspIter iter(*s.fInvMatrix,
284 SkIntToScalar(x) + SK_ScalarHalf,
285 SkIntToScalar(y) + SK_ScalarHalf, count);
286
287 while ((count = iter.next()) != 0) {
288 const SkFixed* SK_RESTRICT srcXY = iter.getXY();
289 #if 0
290 // crashes in ApiDemos - Views - Animation - 3D Transition
291
292 /* srcXY is a batch of 32 bit numbers X0,Y0,X1,Y1...
293 * but we immediately discard the low 16 bits...
294 * so what we're going to do is vld4, which will give us
295 * xlo,xhi,ylo,yhi distribution and we can ignore the 'lo'
296 * parts....
297 */
298 if (count >= 8) {
299 int16_t *mysrc = (int16_t *) srcXY;
300 int16_t *mydst = (int16_t *) xy;
301 int16x4_t maxX4 = vdup_n_s16((int16_t)maxX);
302 int16x4_t maxY4 = vdup_n_s16((int16_t)maxY);
303 int16x4_t zero4 = vdup_n_s16(0);
304 do {
305 register int16x4_t xlo asm("d0");
306 register int16x4_t xhi asm("d1");
307 register int16x4_t ylo asm("d2");
308 register int16x4_t yhi asm("d3");
309 register int16x4_t x2lo asm("d4");
310 register int16x4_t x2hi asm("d5");
311 register int16x4_t y2lo asm("d6");
312 register int16x4_t y2hi asm("d7");
313
314 register int16x4_t out_xhi asm("d8");
315 register int16x4_t out_yhi asm("d9");
316 register int16x4_t out_x2hi asm("d10");
317 register int16x4_t out_y2hi asm("d11");
318
319
320 /* vld4 does the de-interleaving for us */
321 /* dependent on register assignments above */
322 asm ("vld4.16 {d0-d3},[%4] /* xlo=%P0 xhi=%P1 ylo=%P2 yhi=%P3 */"
323 : "=w" (xlo), "=w" (xhi), "=w" (ylo), "=w" (yhi)
324 : "r" (mysrc)
325 );
326
327 /* offset == 256 bits == 32 bytes == 8 longs */
328 asm ("vld4.16 {d4-d7},[%4,#32] /* xlo=%P0 xhi=%P1 ylo=%P2 yhi=%P3 */"
329 : "=w" (x2lo), "=w" (x2hi), "=w" (y2lo), "=w" (y2hi)
330 : "r" (mysrc)
331 );
332
333 /* clamp the first 4 here */
334
335 /* clamp X>>16 (aka xhi) to 0..maxX */
336 xhi = vmax_s16(xhi, zero4); /* now 0.. */
337 out_xhi = vmin_s16(xhi, maxX4); /* now 0..maxX */
338
339 /* clamp Y>>16 (aka yhi) to 0..maxY */
340 yhi = vmax_s16(yhi, zero4); /* now 0.. */
341 out_yhi = vmin_s16(yhi, maxY4); /* now 0..maxY */
342
343 /* clamp the second 4 here */
344
345 /* clamp X>>16 (aka xhi) to 0..maxX */
346 x2hi = vmax_s16(x2hi, zero4); /* now 0.. */
347 out_x2hi = vmin_s16(x2hi, maxX4); /* now 0..maxX */
348
349 /* clamp Y>>16 (aka yhi) to 0..maxY */
350 y2hi = vmax_s16(y2hi, zero4); /* now 0.. */
351 out_y2hi = vmin_s16(y2hi, maxY4); /* now 0..maxY */
352
353 /* we're storing as {x,y}s: x is [0], y is [1] */
354 /* we'll use vst2 to make this happen */
355 /* XXX: could use auto increment! */
356 asm ("vst2.16 {d8-d9},[%2] /* xlo=%P0 xhi=%P1 */"
357 :
358 : "w" (out_xhi), "w" (out_yhi), "r" (mydst)
359 );
360
361 /* offset == 16 bytes == 8 shorts */
362 asm ("vst2.16 {d10-d11},[%2,#16] /* xlo=%P0 xhi=%P1 */"
363 :
364 : "w" (out_x2hi), "w" (out_y2hi), "r" (mydst)
365 );
366
367 /* XXX: gcc isn't interleaving these with the NEON ops
368 * but i think that all the scoreboarding works out */
369 count -= 8; /* 8 iterations */
370 mysrc += 32; /* 16 longs, aka 32 shorts */
371 mydst += 16; /* 16 shorts, aka 8 longs */
372 } while (count >= 8);
373 /* get xy and srcXY fixed up */
374 srcXY = (const SkFixed *) mysrc;
375 xy = (uint32_t *) mydst;
376 }
377 #endif
378 while (--count >= 0) {
379 *xy++ = (TILEY_PROCF(srcXY[1], maxY) << 16) |
380 TILEX_PROCF(srcXY[0], maxX);
381 srcXY += 2;
382 }
383 }
384 }
385
386 //////////////////////////////////////////////////////////////////////////////
387
PACK_FILTER_Y_NAME(SkFixed f,unsigned max,SkFixed one PREAMBLE_PARAM_Y)388 static inline uint32_t PACK_FILTER_Y_NAME(SkFixed f, unsigned max,
389 SkFixed one PREAMBLE_PARAM_Y) {
390 unsigned i = TILEY_PROCF(f, max);
391 i = (i << 4) | TILEY_LOW_BITS(f, max);
392 return (i << 14) | (TILEY_PROCF((f + one), max));
393 }
394
PACK_FILTER_X_NAME(SkFixed f,unsigned max,SkFixed one PREAMBLE_PARAM_X)395 static inline uint32_t PACK_FILTER_X_NAME(SkFixed f, unsigned max,
396 SkFixed one PREAMBLE_PARAM_X) {
397 unsigned i = TILEX_PROCF(f, max);
398 i = (i << 4) | TILEX_LOW_BITS(f, max);
399 return (i << 14) | (TILEX_PROCF((f + one), max));
400 }
401
SCALE_FILTER_NAME(const SkBitmapProcState & s,uint32_t xy[],int count,int x,int y)402 static void SCALE_FILTER_NAME(const SkBitmapProcState& s,
403 uint32_t xy[], int count, int x, int y) {
404 SkASSERT((s.fInvType & ~(SkMatrix::kTranslate_Mask |
405 SkMatrix::kScale_Mask)) == 0);
406 SkASSERT(s.fInvKy == 0);
407
408 PREAMBLE(s);
409
410 const unsigned maxX = s.fBitmap->width() - 1;
411 const SkFixed one = s.fFilterOneX;
412 const SkFixed dx = s.fInvSx;
413 SkFixed fx;
414
415 {
416 SkPoint pt;
417 s.fInvProc(*s.fInvMatrix, SkIntToScalar(x) + SK_ScalarHalf,
418 SkIntToScalar(y) + SK_ScalarHalf, &pt);
419 const SkFixed fy = SkScalarToFixed(pt.fY) - (s.fFilterOneY >> 1);
420 const unsigned maxY = s.fBitmap->height() - 1;
421 // compute our two Y values up front
422 *xy++ = PACK_FILTER_Y_NAME(fy, maxY, s.fFilterOneY PREAMBLE_ARG_Y);
423 // now initialize fx
424 fx = SkScalarToFixed(pt.fX) - (one >> 1);
425 }
426
427 #ifdef CHECK_FOR_DECAL
428 // test if we don't need to apply the tile proc
429 if (dx > 0 &&
430 (unsigned)(fx >> 16) <= maxX &&
431 (unsigned)((fx + dx * (count - 1)) >> 16) < maxX) {
432 decal_filter_scale(xy, fx, dx, count);
433 } else
434 #endif
435
436 if (count >= 4) {
437 int32x4_t wide_dx, wide_one;
438 int32x4_t wide_fx, wide_fx1, wide_i, wide_lo;
439 #if 0
440 /* verification hooks -- see below */
441 SkFixed debug_fx = fx;
442 int count_done = 0;
443 #endif
444
445 wide_fx = vdupq_n_s32(fx);
446 wide_fx = vsetq_lane_s32(fx+dx, wide_fx, 1);
447 wide_fx = vsetq_lane_s32(fx+dx+dx, wide_fx, 2);
448 wide_fx = vsetq_lane_s32(fx+dx+dx+dx, wide_fx, 3);
449
450 wide_dx = vdupq_n_s32(dx);
451 wide_one = vdupq_n_s32(one);
452
453 while (count >= 4) {
454 /* original expands to:
455 * unsigned i = SkClampMax((f) >> 16, max);
456 * i = (i << 4) | (((f) >> 12) & 0xF);
457 * return (i << 14) | (SkClampMax(((f + one)) >> 16, max));
458 */
459
460 /* i = SkClampMax(f>>16, maxX) */
461 wide_i = vmaxq_s32(vshrq_n_s32(wide_fx,16), vdupq_n_s32(0));
462 wide_i = vminq_s32(wide_i, vdupq_n_s32(maxX));
463
464 /* i<<4 | TILEX_LOW_BITS(fx) */
465 wide_lo = vshrq_n_s32(wide_fx, 12);
466 wide_i = vsliq_n_s32(wide_lo, wide_i, 4);
467
468 /* i<<14 */
469 wide_i = vshlq_n_s32(wide_i, 14);
470
471 /* SkClampMax(((f + one)) >> 16, max) */
472 wide_fx1 = vaddq_s32(wide_fx, wide_one);
473 wide_fx1 = vmaxq_s32(vshrq_n_s32(wide_fx1,16), vdupq_n_s32(0));
474 wide_fx1 = vminq_s32(wide_fx1, vdupq_n_s32(maxX));
475
476 /* final combination */
477 wide_i = vorrq_s32(wide_i, wide_fx1);
478
479 vst1q_u32(xy, vreinterpretq_u32_s32(wide_i));
480
481 #if 0
482 /* having a verification hook is a good idea */
483 /* use debug_fx, debug_fx+dx, etc. */
484
485 for (int i=0;i<4;i++) {
486 uint32_t want = PACK_FILTER_X_NAME(debug_fx, maxX, one PREAMBLE_ARG_X);
487 if (xy[i] != want)
488 {
489 /* print a nastygram */
490 SkDebugf("clamp-filter-scale fails\n");
491 SkDebugf("got %08x want %08x\n", xy[i], want);
492 SkDebugf("fx %08x debug_fx %08x dx %08x done %d\n",
493 fx, debug_fx, dx, count_done);
494 SkDebugf(" maxX %08x one %08x\n", maxX, one);
495
496 }
497 debug_fx += dx;
498 count_done++;
499 }
500 #endif
501 wide_fx += vdupq_n_s32(dx+dx+dx+dx);
502 fx += dx+dx+dx+dx;
503 xy += 4;
504 count -= 4;
505 }
506 }
507
508 while (--count >= 0) {
509 *xy++ = PACK_FILTER_X_NAME(fx, maxX, one PREAMBLE_ARG_X);
510 fx += dx;
511 }
512 }
513
AFFINE_FILTER_NAME(const SkBitmapProcState & s,uint32_t xy[],int count,int x,int y)514 static void AFFINE_FILTER_NAME(const SkBitmapProcState& s,
515 uint32_t xy[], int count, int x, int y) {
516 SkASSERT(s.fInvType & SkMatrix::kAffine_Mask);
517 SkASSERT((s.fInvType & ~(SkMatrix::kTranslate_Mask |
518 SkMatrix::kScale_Mask |
519 SkMatrix::kAffine_Mask)) == 0);
520
521 PREAMBLE(s);
522 SkPoint srcPt;
523 s.fInvProc(*s.fInvMatrix,
524 SkIntToScalar(x) + SK_ScalarHalf,
525 SkIntToScalar(y) + SK_ScalarHalf, &srcPt);
526
527 SkFixed oneX = s.fFilterOneX;
528 SkFixed oneY = s.fFilterOneY;
529 SkFixed fx = SkScalarToFixed(srcPt.fX) - (oneX >> 1);
530 SkFixed fy = SkScalarToFixed(srcPt.fY) - (oneY >> 1);
531 SkFixed dx = s.fInvSx;
532 SkFixed dy = s.fInvKy;
533 unsigned maxX = s.fBitmap->width() - 1;
534 unsigned maxY = s.fBitmap->height() - 1;
535
536 if (count >= 4) {
537 int32x4_t wide_one, wide_i, wide_lo;
538 int32x4_t wide_dx, wide_fx, wide_onex, wide_fx1;
539 int32x4_t wide_dy, wide_fy, wide_oney, wide_fy1;
540
541 /* need side-by-side registers for vst2.32 tricks */
542 register int32x4_t wide_x asm("q1");
543 register int32x4_t wide_y asm("q0");
544
545 #undef AFFINE_DEBUG
546 #if defined(AFFINE_DEBUG)
547 SkFixed fyp = fy;
548 SkFixed fxp = fx;
549 uint32_t *xyp = xy;
550 int count_done = 0;
551 #endif
552
553 wide_fx = vdupq_n_s32(fx);
554 wide_fx = vsetq_lane_s32(fx+dx, wide_fx, 1);
555 wide_fx = vsetq_lane_s32(fx+dx+dx, wide_fx, 2);
556 wide_fx = vsetq_lane_s32(fx+dx+dx+dx, wide_fx, 3);
557 wide_dx = vdupq_n_s32(dx);
558
559 wide_fy = vdupq_n_s32(fy);
560 wide_fy = vsetq_lane_s32(fy+dy, wide_fy, 1);
561 wide_fy = vsetq_lane_s32(fy+dy+dy, wide_fy, 2);
562 wide_fy = vsetq_lane_s32(fy+dy+dy+dy, wide_fy, 3);
563 wide_dy = vdupq_n_s32(dy);
564
565 wide_onex = vdupq_n_s32(oneX);
566 wide_oney = vdupq_n_s32(oneY);
567
568 while (count >= 4) {
569 /* do the X side, then the Y side, then interleave them */
570
571 /* original expands to:
572 * unsigned i = SkClampMax((f) >> 16, max);
573 * i = (i << 4) | (((f) >> 12) & 0xF);
574 * return (i << 14) | (SkClampMax(((f + one)) >> 16, max));
575 */
576
577 /* i = SkClampMax(f>>16, maxX) */
578 wide_i = vmaxq_s32(vshrq_n_s32(wide_fx,16), vdupq_n_s32(0));
579 wide_i = vminq_s32(wide_i, vdupq_n_s32(maxX));
580
581 /* i<<4 | TILEX_LOW_BITS(fx) */
582 wide_lo = vshrq_n_s32(wide_fx, 12);
583 wide_i = vsliq_n_s32(wide_lo, wide_i, 4);
584
585 /* i<<14 */
586 wide_i = vshlq_n_s32(wide_i, 14);
587
588 /* SkClampMax(((f + one)) >> 16, max) */
589 wide_fx1 = vaddq_s32(wide_fx, wide_onex);
590 wide_fx1 = vmaxq_s32(vshrq_n_s32(wide_fx1,16), vdupq_n_s32(0));
591 wide_fx1 = vminq_s32(wide_fx1, vdupq_n_s32(maxX));
592
593 /* final combination */
594 wide_x = vorrq_s32(wide_i, wide_fx1);
595
596 /* And now the Y side */
597
598 /* i = SkClampMax(f>>16, maxX) */
599 wide_i = vmaxq_s32(vshrq_n_s32(wide_fy,16), vdupq_n_s32(0));
600 wide_i = vminq_s32(wide_i, vdupq_n_s32(maxY));
601
602 /* i<<4 | TILEX_LOW_BITS(fx) */
603 wide_lo = vshrq_n_s32(wide_fy, 12);
604 wide_i = vsliq_n_s32(wide_lo, wide_i, 4);
605
606 /* i<<14 */
607 wide_i = vshlq_n_s32(wide_i, 14);
608
609 /* SkClampMax(((f + one)) >> 16, max) */
610 wide_fy1 = vaddq_s32(wide_fy, wide_oney);
611 wide_fy1 = vmaxq_s32(vshrq_n_s32(wide_fy1,16), vdupq_n_s32(0));
612 wide_fy1 = vminq_s32(wide_fy1, vdupq_n_s32(maxY));
613
614 /* final combination */
615 wide_y = vorrq_s32(wide_i, wide_fy1);
616
617 /* interleave as YXYXYXYX as part of the storing */
618 asm ("vst2.32 {q0-q1},[%2] /* y=%q0 x=%q1 */"
619 :
620 : "w" (wide_y), "w" (wide_x), "r" (xy)
621 );
622
623 #if defined(AFFINE_DEBUG)
624 /* make sure we're good here -- check the 4 we just output */
625 for (int i = 0; i<4;i++) {
626 uint32_t val;
627 val = PACK_FILTER_Y_NAME(fyp, maxY, oneY PREAMBLE_ARG_Y);
628 if (val != xy[i*2+0]) {
629 /* print a nastygram */
630 SkDebugf("clamp-filter-affine fails\n");
631 SkDebugf("[bad-y] got %08x want %08x\n", xy[i*2+0], val);
632 SkDebugf("fy %08x fxp %08x fyp %08x dx %08x dy %08x done %d\n",
633 fy, fxp, fyp, dx, dy, count_done);
634 SkDebugf(" maxY %08x oneY %08x\n", maxY, oneY);
635 }
636 val = PACK_FILTER_X_NAME(fxp, maxX, oneX PREAMBLE_ARG_X);
637 if (val != xy[i*2+1]) {
638 /* print a nastygram */
639 SkDebugf("clamp-filter-affine fails\n");
640 SkDebugf("[bad-x] got %08x want %08x\n", xy[i*2+1], val);
641 SkDebugf("fx %08x fxp %08x fyp %08x dx %08x dy %08x done %d\n",
642 fx, fxp, fyp, dx, dy, count_done);
643 SkDebugf(" maxX %08x one %08x\n", maxX, oneX);
644 }
645 fyp += dy;
646 fxp += dx;
647 count_done++;
648 }
649 #endif
650
651 wide_fx += vdupq_n_s32(dx+dx+dx+dx);
652 fx += dx+dx+dx+dx;
653 wide_fy += vdupq_n_s32(dy+dy+dy+dy);
654 fy += dy+dy+dy+dy;
655 xy += 8; /* 4 x's, 4 y's */
656 count -= 4;
657 }
658 }
659
660 while (--count >= 0) {
661 /* NB: writing Y/X */
662 *xy++ = PACK_FILTER_Y_NAME(fy, maxY, oneY PREAMBLE_ARG_Y);
663 fy += dy;
664 *xy++ = PACK_FILTER_X_NAME(fx, maxX, oneX PREAMBLE_ARG_X);
665 fx += dx;
666 }
667 }
668
PERSP_FILTER_NAME(const SkBitmapProcState & s,uint32_t * SK_RESTRICT xy,int count,int x,int y)669 static void PERSP_FILTER_NAME(const SkBitmapProcState& s,
670 uint32_t* SK_RESTRICT xy, int count,
671 int x, int y) {
672 SkASSERT(s.fInvType & SkMatrix::kPerspective_Mask);
673
674 PREAMBLE(s);
675 unsigned maxX = s.fBitmap->width() - 1;
676 unsigned maxY = s.fBitmap->height() - 1;
677 SkFixed oneX = s.fFilterOneX;
678 SkFixed oneY = s.fFilterOneY;
679
680 SkPerspIter iter(*s.fInvMatrix,
681 SkIntToScalar(x) + SK_ScalarHalf,
682 SkIntToScalar(y) + SK_ScalarHalf, count);
683
684 while ((count = iter.next()) != 0) {
685 const SkFixed* SK_RESTRICT srcXY = iter.getXY();
686
687 if (count >= 4) {
688 int32x4_t wide_one, wide_i, wide_lo;
689 int32x4_t wide_fx1;
690 int32x4_t wide_fy1;
691 int32x4_t wide_x, wide_y;
692
693 /* need side-by-side regs for vld2/vst2 tricks */
694 register int32x4_t wide_first asm ("q0");
695 register int32x4_t wide_second asm ("q1");
696
697 while (count >= 4) {
698 /* RBE: it's good, but:
699 * -- we spill a constant that could be easily regnerated
700 * [perhaps tweak gcc's NEON constant costs?]
701 */
702
703 /* load src: x-y-x-y-x-y-x-y */
704 asm ("vld2.32 {q0-q1},[%2] /* x=%q0 y=%q1 */"
705 : "=w" (wide_first), "=w" (wide_second)
706 : "r" (srcXY));
707
708 wide_x = wide_first;
709 wide_y = wide_second;
710
711 /* do the X side, then the Y side, then interleave them */
712
713 wide_x = vsubq_s32(wide_x, vdupq_n_s32 (oneX>>1));
714
715 /* original expands to:
716 * unsigned i = SkClampMax((f) >> 16, max);
717 * i = (i << 4) | (((f) >> 12) & 0xF);
718 * return (i << 14) | (SkClampMax(((f + one)) >> 16, max));
719 */
720
721 /* i = SkClampMax(f>>16, maxX) */
722 wide_i = vmaxq_s32 (vshrq_n_s32 (wide_x, 16), vdupq_n_s32 (0));
723 wide_i = vminq_s32 (wide_i, vdupq_n_s32 (maxX));
724
725 /* i<<4 | TILEX_LOW_BITS(fx) */
726 wide_lo = vshrq_n_s32 (wide_x, 12);
727 wide_i = vsliq_n_s32 (wide_lo, wide_i, 4);
728
729 /* i<<14 */
730 wide_i = vshlq_n_s32 (wide_i, 14);
731
732 /* SkClampMax(((f + one)) >> 16, max) */
733 wide_fx1 = vaddq_s32 (wide_x, vdupq_n_s32(oneX));
734 wide_fx1 = vmaxq_s32 (vshrq_n_s32 (wide_fx1, 16), vdupq_n_s32 (0));
735 wide_fx1 = vminq_s32 (wide_fx1, vdupq_n_s32 (maxX));
736
737 /* final combination */
738 wide_x = vorrq_s32 (wide_i, wide_fx1);
739
740
741 /* And now the Y side */
742
743 wide_y = vsubq_s32(wide_y, vdupq_n_s32 (oneY>>1));
744
745 /* i = SkClampMax(f>>16, maxX) */
746 wide_i = vmaxq_s32 (vshrq_n_s32 (wide_y, 16), vdupq_n_s32 (0));
747 wide_i = vminq_s32 (wide_i, vdupq_n_s32 (maxY));
748
749 /* i<<4 | TILEX_LOW_BITS(fx) */
750 wide_lo = vshrq_n_s32 (wide_y, 12);
751 wide_i = vsliq_n_s32 (wide_lo, wide_i, 4);
752
753 /* i<<14 */
754 wide_i = vshlq_n_s32 (wide_i, 14);
755
756 /* SkClampMax(((f + one)) >> 16, max) */
757 wide_fy1 = vaddq_s32 (wide_y, vdupq_n_s32(oneY));
758 wide_fy1 = vmaxq_s32 (vshrq_n_s32 (wide_fy1, 16), vdupq_n_s32 (0));
759 wide_fy1 = vminq_s32 (wide_fy1, vdupq_n_s32 (maxY));
760
761 /* final combination */
762 wide_y = vorrq_s32 (wide_i, wide_fy1);
763
764 /* switch them around; have to do it this way to get them
765 * in the proper registers to match our instruction */
766
767 /* wide_x/wide_y are fixed regs, in wrong order; swap 'em */
768 wide_first = wide_y;
769 wide_second = wide_x;
770
771 /* iteration bookkeeping, ahead of the asm() for scheduling */
772 srcXY += 2*4;
773 count -= 4;
774
775 /* store interleaved as y-x-y-x-y-x-y-x (NB != read order) */
776 asm ("vst2.32 {q0-q1},[%2] /* y=%q0 x=%q1 */"
777 :
778 : "w" (wide_first), "w" (wide_second), "r" (xy));
779
780 /* on to the next iteration */
781 /* count, srcXY are handled above */
782 xy += 2*4;
783 }
784 }
785
786 /* was do-while; NEON code invalidates original count>0 assumption */
787 while (--count >= 0) {
788 /* NB: we read x/y, we write y/x */
789 *xy++ = PACK_FILTER_Y_NAME(srcXY[1] - (oneY >> 1), maxY,
790 oneY PREAMBLE_ARG_Y);
791 *xy++ = PACK_FILTER_X_NAME(srcXY[0] - (oneX >> 1), maxX,
792 oneX PREAMBLE_ARG_X);
793 srcXY += 2;
794 }
795 }
796 }
797
798 static SkBitmapProcState::MatrixProc MAKENAME(_Procs)[] = {
799 SCALE_NOFILTER_NAME,
800 SCALE_FILTER_NAME,
801 AFFINE_NOFILTER_NAME,
802 AFFINE_FILTER_NAME,
803 PERSP_NOFILTER_NAME,
804 PERSP_FILTER_NAME
805 };
806
807 #undef MAKENAME
808 #undef TILEX_PROCF
809 #undef TILEY_PROCF
810 #ifdef CHECK_FOR_DECAL
811 #undef CHECK_FOR_DECAL
812 #endif
813
814 #undef SCALE_NOFILTER_NAME
815 #undef SCALE_FILTER_NAME
816 #undef AFFINE_NOFILTER_NAME
817 #undef AFFINE_FILTER_NAME
818 #undef PERSP_NOFILTER_NAME
819 #undef PERSP_FILTER_NAME
820
821 #undef PREAMBLE
822 #undef PREAMBLE_PARAM_X
823 #undef PREAMBLE_PARAM_Y
824 #undef PREAMBLE_ARG_X
825 #undef PREAMBLE_ARG_Y
826
827 #undef TILEX_LOW_BITS
828 #undef TILEY_LOW_BITS
829