1 /* NEON optimized code (C) COPYRIGHT 2009 Motorola
2 *
3 * Use of this source code is governed by a BSD-style license that can be
4 * found in the LICENSE file.
5 */
6
7 /*
8 * Modifications done in-house at Motorola
9 *
10 * this is a clone of SkBitmapProcState_matrix.h
11 * and has been tuned to work with the NEON unit.
12 *
13 * Still going back and forth between whether this approach
14 * (clone the entire SkBitmapProcState_matrix.h file or
15 * if I should put just the modified routines in here and
16 * then use a construct like #define DONT_DO_THIS_FUNCTION or
17 * something like that...
18 *
19 * This is for the ClampX_ClampY instance
20 *
21 */
22
23
24 #include <arm_neon.h>
25
26 /*
27 * This has been modified on the knowledge that (at the time)
28 * we had the following macro definitions in the parent file
29 *
30 * #define MAKENAME(suffix) ClampX_ClampY ## suffix
31 * #define TILEX_PROCF(fx, max) SkClampMax((fx) >> 16, max)
32 * #define TILEY_PROCF(fy, max) SkClampMax((fy) >> 16, max)
33 * #define TILEX_LOW_BITS(fx, max) (((fx) >> 12) & 0xF)
34 * #define TILEY_LOW_BITS(fy, max) (((fy) >> 12) & 0xF)
35 * #define CHECK_FOR_DECAL
36 */
37
38 /* SkClampMax(val,max) -- bound to 0..max */
39
40 #define SCALE_NOFILTER_NAME MAKENAME(_nofilter_scale)
41 #define SCALE_FILTER_NAME MAKENAME(_filter_scale)
42 #define AFFINE_NOFILTER_NAME MAKENAME(_nofilter_affine)
43 #define AFFINE_FILTER_NAME MAKENAME(_filter_affine)
44 #define PERSP_NOFILTER_NAME MAKENAME(_nofilter_persp)
45 #define PERSP_FILTER_NAME MAKENAME(_filter_persp)
46
47 #define PACK_FILTER_X_NAME MAKENAME(_pack_filter_x)
48 #define PACK_FILTER_Y_NAME MAKENAME(_pack_filter_y)
49
50 #ifndef PREAMBLE
51 #define PREAMBLE(state)
52 #define PREAMBLE_PARAM_X
53 #define PREAMBLE_PARAM_Y
54 #define PREAMBLE_ARG_X
55 #define PREAMBLE_ARG_Y
56 #endif
57
SCALE_NOFILTER_NAME(const SkBitmapProcState & s,uint32_t xy[],int count,int x,int y)58 static void SCALE_NOFILTER_NAME(const SkBitmapProcState& s,
59 uint32_t xy[], int count, int x, int y) {
60 SkASSERT((s.fInvType & ~(SkMatrix::kTranslate_Mask |
61 SkMatrix::kScale_Mask)) == 0);
62
63 PREAMBLE(s);
64 // we store y, x, x, x, x, x
65
66 const unsigned maxX = s.fBitmap->width() - 1;
67 SkFixed fx;
68 {
69 SkPoint pt;
70 s.fInvProc(s.fInvMatrix, SkIntToScalar(x) + SK_ScalarHalf,
71 SkIntToScalar(y) + SK_ScalarHalf, &pt);
72 fx = SkScalarToFixed(pt.fY);
73 const unsigned maxY = s.fBitmap->height() - 1;
74 *xy++ = TILEY_PROCF(fx, maxY);
75 fx = SkScalarToFixed(pt.fX);
76 }
77
78 if (0 == maxX) {
79 // all of the following X values must be 0
80 memset(xy, 0, count * sizeof(uint16_t));
81 return;
82 }
83
84 const SkFixed dx = s.fInvSx;
85
86 #ifdef CHECK_FOR_DECAL
87 // test if we don't need to apply the tile proc
88 if ((unsigned)(fx >> 16) <= maxX &&
89 (unsigned)((fx + dx * (count - 1)) >> 16) <= maxX) {
90 decal_nofilter_scale_neon(xy, fx, dx, count);
91 return;
92 }
93 #endif
94
95 int i;
96
97 /* very much like done in decal_nofilter, but with
98 * an extra clamping function applied.
99 * TILEX_PROCF(fx,max) SkClampMax((fx)>>16, max)
100 */
101 if (count >= 8) {
102 /* SkFixed is 16.16 fixed point */
103 SkFixed dx2 = dx+dx;
104 SkFixed dx4 = dx2+dx2;
105 SkFixed dx8 = dx4+dx4;
106
107 /* now build fx/fx+dx/fx+2dx/fx+3dx */
108 SkFixed fx1, fx2, fx3;
109 int32x4_t lbase, hbase;
110 int16_t *dst16 = (int16_t *)xy;
111
112 fx1 = fx+dx;
113 fx2 = fx1+dx;
114 fx3 = fx2+dx;
115
116 /* build my template(s) */
117 /* avoid the 'lbase unitialized' warning */
118 lbase = vdupq_n_s32(fx);
119 lbase = vsetq_lane_s32(fx1, lbase, 1);
120 lbase = vsetq_lane_s32(fx2, lbase, 2);
121 lbase = vsetq_lane_s32(fx3, lbase, 3);
122
123 hbase = vaddq_s32(lbase, vdupq_n_s32(dx4));
124
125 /* store & bump */
126 do {
127 int32x4_t lout;
128 int32x4_t hout;
129 int16x8_t hi16;
130
131 /* get the hi 16s of all those 32s */
132 lout = lbase;
133 hout = hbase;
134 /* this sets up all lout's then all hout's in hout */
135 asm ("vuzpq.16 %q0, %q1" : "+w" (lout), "+w" (hout));
136 hi16 = vreinterpretq_s16_s32(hout);
137
138 /* clamp & output */
139 hi16 = vmaxq_s16(hi16, vdupq_n_s16(0));
140 hi16 = vminq_s16(hi16, vdupq_n_s16(maxX));
141 vst1q_s16(dst16, hi16);
142
143 /* but preserving base & on to the next */
144 lbase = vaddq_s32 (lbase, vdupq_n_s32(dx8));
145 hbase = vaddq_s32 (hbase, vdupq_n_s32(dx8));
146 dst16 += 8;
147 count -= 8;
148 fx += dx8;
149 } while (count >= 8);
150 xy = (uint32_t *) dst16;
151 }
152
153 uint16_t* xx = (uint16_t*)xy;
154 for (i = count; i > 0; --i) {
155 *xx++ = TILEX_PROCF(fx, maxX); fx += dx;
156 }
157 }
158
159 // note: we could special-case on a matrix which is skewed in X but not Y.
160 // this would require a more general setup thatn SCALE does, but could use
161 // SCALE's inner loop that only looks at dx
162
AFFINE_NOFILTER_NAME(const SkBitmapProcState & s,uint32_t xy[],int count,int x,int y)163 static void AFFINE_NOFILTER_NAME(const SkBitmapProcState& s,
164 uint32_t xy[], int count, int x, int y) {
165 SkASSERT(s.fInvType & SkMatrix::kAffine_Mask);
166 SkASSERT((s.fInvType & ~(SkMatrix::kTranslate_Mask |
167 SkMatrix::kScale_Mask |
168 SkMatrix::kAffine_Mask)) == 0);
169
170 PREAMBLE(s);
171 SkPoint srcPt;
172 s.fInvProc(s.fInvMatrix,
173 SkIntToScalar(x) + SK_ScalarHalf,
174 SkIntToScalar(y) + SK_ScalarHalf, &srcPt);
175
176 SkFixed fx = SkScalarToFixed(srcPt.fX);
177 SkFixed fy = SkScalarToFixed(srcPt.fY);
178 SkFixed dx = s.fInvSx;
179 SkFixed dy = s.fInvKy;
180 int maxX = s.fBitmap->width() - 1;
181 int maxY = s.fBitmap->height() - 1;
182
183 /* NEON lets us do an 8x unrolling */
184 if (count >= 8) {
185 /* SkFixed is 16.16 fixed point */
186 SkFixed dx4 = dx * 4;
187 SkFixed dy4 = dy * 4;
188 SkFixed dx8 = dx * 8;
189 SkFixed dy8 = dy * 8;
190
191 int32x4_t xbase, ybase;
192 int32x4_t x2base, y2base;
193 int16_t *dst16 = (int16_t *) xy;
194
195 /* my sets of maxx/maxy for clamping */
196 int32_t maxpair = (maxX&0xffff) | ((maxY&0xffff)<<16);
197 int16x8_t maxXY = vreinterpretq_s16_s32(vdupq_n_s32(maxpair));
198
199 /* now build fx/fx+dx/fx+2dx/fx+3dx */
200 /* avoid the 'xbase unitialized' warning...*/
201 xbase = vdupq_n_s32(fx);
202 xbase = vsetq_lane_s32(fx+dx, xbase, 1);
203 xbase = vsetq_lane_s32(fx+dx+dx, xbase, 2);
204 xbase = vsetq_lane_s32(fx+dx+dx+dx, xbase, 3);
205
206 /* same for fy */
207 /* avoid the 'ybase unitialized' warning...*/
208 ybase = vdupq_n_s32(fy);
209 ybase = vsetq_lane_s32(fy+dy, ybase, 1);
210 ybase = vsetq_lane_s32(fy+dy+dy, ybase, 2);
211 ybase = vsetq_lane_s32(fy+dy+dy+dy, ybase, 3);
212
213 x2base = vaddq_s32(xbase, vdupq_n_s32(dx4));
214 y2base = vaddq_s32(ybase, vdupq_n_s32(dy4));
215
216 /* store & bump */
217 do {
218 int32x4_t xout, yout;
219 int32x4_t x2out, y2out;
220 int16x8_t hi16, hi16_2;
221
222 xout = xbase;
223 yout = ybase;
224
225 /* overlay y's low16 with hi16 from x */
226 /* so we properly shifted xyxyxyxy */
227 yout = vsriq_n_s32(yout, xout, 16);
228 hi16 = vreinterpretq_s16_s32 (yout);
229
230 /* do the clamping; both guys get 0's */
231 hi16 = vmaxq_s16 (hi16, vdupq_n_s16(0));
232 hi16 = vminq_s16 (hi16, maxXY);
233
234 vst1q_s16 (dst16, hi16);
235
236 /* and for the other 4 pieces of this iteration */
237 x2out = x2base;
238 y2out = y2base;
239
240 /* overlay y's low16 with hi16 from x */
241 /* so we properly shifted xyxyxyxy */
242 y2out = vsriq_n_s32(y2out, x2out, 16);
243 hi16_2 = vreinterpretq_s16_s32 (y2out);
244
245 /* do the clamping; both guys get 0's */
246 hi16_2 = vmaxq_s16 (hi16_2, vdupq_n_s16(0));
247 hi16_2 = vminq_s16 (hi16_2, maxXY);
248
249 /* RBE: gcc regenerates dst16+8 all the time instead
250 * of folding it into an addressing mode. *sigh* */
251 vst1q_s16 (dst16+8, hi16_2);
252
253 /* moving base and on to the next */
254 xbase = vaddq_s32 (xbase, vdupq_n_s32 (dx8));
255 ybase = vaddq_s32 (ybase, vdupq_n_s32 (dy8));
256 x2base = vaddq_s32 (x2base, vdupq_n_s32 (dx8));
257 y2base = vaddq_s32 (y2base, vdupq_n_s32 (dy8));
258
259 dst16 += 16; /* 8x32 aka 16x16 */
260 count -= 8;
261 fx += dx8;
262 fy += dy8;
263 } while (count >= 8);
264 xy = (uint32_t *) dst16;
265 }
266
267 for (int i = count; i > 0; --i) {
268 *xy++ = (TILEY_PROCF(fy, maxY) << 16) | TILEX_PROCF(fx, maxX);
269 fx += dx; fy += dy;
270 }
271 }
272
273 #undef DEBUG_PERSP_NOFILTER
274
PERSP_NOFILTER_NAME(const SkBitmapProcState & s,uint32_t * SK_RESTRICT xy,int count,int x,int y)275 static void PERSP_NOFILTER_NAME(const SkBitmapProcState& s,
276 uint32_t* SK_RESTRICT xy,
277 int count, int x, int y) {
278 SkASSERT(s.fInvType & SkMatrix::kPerspective_Mask);
279
280 PREAMBLE(s);
281 /* max{X,Y} are int here, but later shown/assumed to fit in 16 bits */
282 int maxX = s.fBitmap->width() - 1;
283 int maxY = s.fBitmap->height() - 1;
284
285 SkPerspIter iter(s.fInvMatrix,
286 SkIntToScalar(x) + SK_ScalarHalf,
287 SkIntToScalar(y) + SK_ScalarHalf, count);
288
289 while ((count = iter.next()) != 0) {
290 const SkFixed* SK_RESTRICT srcXY = iter.getXY();
291
292 #if defined(DEBUG_PERSP_NOFILTER)
293 /* debugging stuff */
294 const SkFixed *end_srcXY = srcXY + (count*2);
295 uint32_t *end_xy = xy + (count);
296 const SkFixed *base_srcXY = srcXY;
297 uint32_t *base_xy = xy;
298 int base_count = count;
299 #endif
300
301 #if 1
302 // 2009/9/30: crashes in ApiDemos - Views - Animation - 3D Transition
303 // 2009/10/9: reworked to avoid illegal (but allowed by gas) insn
304
305 /* srcXY is a batch of 32 bit numbers X0,Y0,X1,Y1...
306 * but we immediately discard the low 16 bits...
307 * so what we're going to do is vld4, which will give us
308 * xlo,xhi,ylo,yhi distribution and we can ignore the 'lo'
309 * parts....
310 */
311 if (count >= 8) {
312 int16_t *mysrc = (int16_t *) srcXY;
313 int16_t *mydst = (int16_t *) xy;
314 int16x4_t maxX4 = vdup_n_s16((int16_t)maxX);
315 int16x4_t maxY4 = vdup_n_s16((int16_t)maxY);
316 int16x4_t zero4 = vdup_n_s16(0);
317
318 /* The constructs with local blocks for register assignments
319 * and asm() instructions is to make keep any hard register
320 * assignments to as small a scope as possible. and to avoid
321 * burning call-preserved hard registers on the vld/vst
322 * instructions.
323 */
324
325 do {
326 int16x4_t xhi, yhi;
327 int16x4_t x2hi, y2hi;
328
329 /* vld4 does the de-interleaving for us */
330 {
331 register int16x4_t t_xlo asm("d0");
332 register int16x4_t t_xhi asm("d1");
333 register int16x4_t t_ylo asm("d2");
334 register int16x4_t t_yhi asm("d3");
335
336 asm ("vld4.16 {d0-d3},[%4] /* xlo=%P0 xhi=%P1 ylo=%P2 yhi=%P3 */"
337 : "=w" (t_xlo), "=w" (t_xhi), "=w" (t_ylo), "=w" (t_yhi)
338 : "r" (mysrc)
339 );
340 xhi = t_xhi;
341 yhi = t_yhi;
342 }
343
344 /* clamp X>>16 (aka xhi) to 0..maxX */
345 xhi = vmax_s16(xhi, zero4); /* now 0.. */
346 xhi = vmin_s16(xhi, maxX4); /* now 0..maxX */
347
348 /* clamp Y>>16 (aka yhi) to 0..maxY */
349 yhi = vmax_s16(yhi, zero4); /* now 0.. */
350 yhi = vmin_s16(yhi, maxY4); /* now 0..maxY */
351
352 /* deal with the second set of numbers */
353 {
354 register int16x4_t t_xlo asm("d4");
355 register int16x4_t t_xhi asm("d5");
356 register int16x4_t t_ylo asm("d6");
357 register int16x4_t t_yhi asm("d7");
358
359 /* offset == 256 bits == 32 bytes == 8 longs == 16 shorts */
360 asm ("vld4.16 {d4-d7},[%4] /* xlo=%P0 xhi=%P1 ylo=%P2 yhi=%P3 */"
361 : "=w" (t_xlo), "=w" (t_xhi), "=w" (t_ylo), "=w" (t_yhi)
362 : "r" (mysrc+16)
363 );
364 x2hi = t_xhi;
365 y2hi = t_yhi;
366 }
367
368 /* clamp the second 4 here */
369
370 if (0) { extern void rbe(void); rbe(); }
371
372 /* clamp X>>16 (aka xhi) to 0..maxX */
373 x2hi = vmax_s16(x2hi, zero4); /* now 0.. */
374 x2hi = vmin_s16(x2hi, maxX4); /* now 0..maxX */
375
376 /* clamp Y>>16 (aka yhi) to 0..maxY */
377 y2hi = vmax_s16(y2hi, zero4); /* now 0.. */
378 y2hi = vmin_s16(y2hi, maxY4); /* now 0..maxY */
379
380 /* we're storing as {x,y}s: x is [0], y is [1] */
381 /* we'll use vst2 to make this happen */
382
383 {
384 register int16x4_t out_x asm("d16") = xhi;
385 register int16x4_t out_y asm("d17") = yhi;
386
387 asm ("vst2.16 {d16-d17},[%2] /* xlo=%P0 xhi=%P1 */"
388 :
389 : "w" (out_x), "w" (out_y), "r" (mydst)
390 );
391 }
392 {
393 register int16x4_t out_x asm("d18") = x2hi;
394 register int16x4_t out_y asm("d19") = y2hi;
395
396 asm ("vst2.16 {d18-d19},[%2] /* xlo=%P0 xhi=%P1 */"
397 :
398 : "w" (out_x), "w" (out_y), "r" (mydst+8)
399 );
400 }
401
402 /* XXX: gcc isn't interleaving these with the NEON ops
403 * but i think that all the scoreboarding works out */
404 count -= 8; /* 8 iterations */
405 mysrc += 32; /* 16 longs, aka 32 shorts */
406 mydst += 16; /* 16 shorts, aka 8 longs */
407 } while (count >= 8);
408 /* get xy and srcXY fixed up */
409 srcXY = (const SkFixed *) mysrc;
410 xy = (uint32_t *) mydst;
411 }
412 #endif
413
414 while (--count >= 0) {
415 *xy++ = (TILEY_PROCF(srcXY[1], maxY) << 16) |
416 TILEX_PROCF(srcXY[0], maxX);
417 srcXY += 2;
418 }
419
420 #if defined(DEBUG_PERSP_NOFILTER)
421 /* for checking our NEON-produced results against vanilla code */
422 {
423 int bad = (-1);
424 for (int i = 0; i < base_count; i++) {
425 uint32_t val;
426 val = (TILEY_PROCF (base_srcXY[i * 2 + 1], maxY) << 16) |
427 TILEX_PROCF (base_srcXY[i * 2 + 0], maxX);
428
429 if (val != base_xy[i]) {
430 bad = i;
431 break;
432 }
433 }
434 if (bad >= 0) {
435 SkDebugf("clamp-nofilter-persp failed piece %d\n", bad);
436 SkDebugf(" maxX %08x maxY %08x\n", maxX, maxY);
437 bad -= (bad & 0x7); /* align */
438 for (int i = bad; i < bad + 8; i++) {
439 uint32_t val;
440 val = (TILEY_PROCF (base_srcXY[i * 2 + 1], maxY) << 16) |
441 TILEX_PROCF (base_srcXY[i * 2 + 0], maxX);
442
443 SkDebugf("%d: got %08x want %08x srcXY[0] %08x srcXY[1] %08x\n",
444 i, base_xy[i], val, base_srcXY[i * 2 + 0],
445 base_srcXY[i * 2 + 1]);
446 }
447 SkDebugf ("---\n");
448 }
449
450 if (end_xy != xy) {
451 SkDebugf("xy ended at %08x, should be %08x\n", xy, end_xy);
452 }
453 if (end_srcXY != srcXY) {
454 SkDebugf("srcXY ended at %08x, should be %08x\n", srcXY,
455 end_srcXY);
456 }
457 }
458 #endif
459 }
460 }
461
462 #undef DEBUG_PERSP_NOFILTER
463
464 //////////////////////////////////////////////////////////////////////////////
465
PACK_FILTER_Y_NAME(SkFixed f,unsigned max,SkFixed one PREAMBLE_PARAM_Y)466 static inline uint32_t PACK_FILTER_Y_NAME(SkFixed f, unsigned max,
467 SkFixed one PREAMBLE_PARAM_Y) {
468 unsigned i = TILEY_PROCF(f, max);
469 i = (i << 4) | TILEY_LOW_BITS(f, max);
470 return (i << 14) | (TILEY_PROCF((f + one), max));
471 }
472
PACK_FILTER_X_NAME(SkFixed f,unsigned max,SkFixed one PREAMBLE_PARAM_X)473 static inline uint32_t PACK_FILTER_X_NAME(SkFixed f, unsigned max,
474 SkFixed one PREAMBLE_PARAM_X) {
475 unsigned i = TILEX_PROCF(f, max);
476 i = (i << 4) | TILEX_LOW_BITS(f, max);
477 return (i << 14) | (TILEX_PROCF((f + one), max));
478 }
479
SCALE_FILTER_NAME(const SkBitmapProcState & s,uint32_t xy[],int count,int x,int y)480 static void SCALE_FILTER_NAME(const SkBitmapProcState& s,
481 uint32_t xy[], int count, int x, int y) {
482 SkASSERT((s.fInvType & ~(SkMatrix::kTranslate_Mask |
483 SkMatrix::kScale_Mask)) == 0);
484 SkASSERT(s.fInvKy == 0);
485
486 PREAMBLE(s);
487
488 const unsigned maxX = s.fBitmap->width() - 1;
489 const SkFixed one = s.fFilterOneX;
490 const SkFixed dx = s.fInvSx;
491 SkFixed fx;
492
493 {
494 SkPoint pt;
495 s.fInvProc(s.fInvMatrix, SkIntToScalar(x) + SK_ScalarHalf,
496 SkIntToScalar(y) + SK_ScalarHalf, &pt);
497 const SkFixed fy = SkScalarToFixed(pt.fY) - (s.fFilterOneY >> 1);
498 const unsigned maxY = s.fBitmap->height() - 1;
499 // compute our two Y values up front
500 *xy++ = PACK_FILTER_Y_NAME(fy, maxY, s.fFilterOneY PREAMBLE_ARG_Y);
501 // now initialize fx
502 fx = SkScalarToFixed(pt.fX) - (one >> 1);
503 }
504
505 #ifdef CHECK_FOR_DECAL
506 // test if we don't need to apply the tile proc
507 if (dx > 0 &&
508 (unsigned)(fx >> 16) <= maxX &&
509 (unsigned)((fx + dx * (count - 1)) >> 16) < maxX) {
510 decal_filter_scale_neon(xy, fx, dx, count);
511 } else
512 #endif
513
514 if (count >= 4) {
515 int32x4_t wide_one, wide_fx, wide_fx1, wide_i, wide_lo;
516 #if 0
517 /* verification hooks -- see below */
518 SkFixed debug_fx = fx;
519 int count_done = 0;
520 #endif
521
522 wide_fx = vdupq_n_s32(fx);
523 wide_fx = vsetq_lane_s32(fx+dx, wide_fx, 1);
524 wide_fx = vsetq_lane_s32(fx+dx+dx, wide_fx, 2);
525 wide_fx = vsetq_lane_s32(fx+dx+dx+dx, wide_fx, 3);
526
527 wide_one = vdupq_n_s32(one);
528
529 while (count >= 4) {
530 /* original expands to:
531 * unsigned i = SkClampMax((f) >> 16, max);
532 * i = (i << 4) | (((f) >> 12) & 0xF);
533 * return (i << 14) | (SkClampMax(((f + one)) >> 16, max));
534 */
535
536 /* i = SkClampMax(f>>16, maxX) */
537 wide_i = vmaxq_s32(vshrq_n_s32(wide_fx,16), vdupq_n_s32(0));
538 wide_i = vminq_s32(wide_i, vdupq_n_s32(maxX));
539
540 /* i<<4 | TILEX_LOW_BITS(fx) */
541 wide_lo = vshrq_n_s32(wide_fx, 12);
542 wide_i = vsliq_n_s32(wide_lo, wide_i, 4);
543
544 /* i<<14 */
545 wide_i = vshlq_n_s32(wide_i, 14);
546
547 /* SkClampMax(((f + one)) >> 16, max) */
548 wide_fx1 = vaddq_s32(wide_fx, wide_one);
549 wide_fx1 = vmaxq_s32(vshrq_n_s32(wide_fx1,16), vdupq_n_s32(0));
550 wide_fx1 = vminq_s32(wide_fx1, vdupq_n_s32(maxX));
551
552 /* final combination */
553 wide_i = vorrq_s32(wide_i, wide_fx1);
554
555 vst1q_u32(xy, vreinterpretq_u32_s32(wide_i));
556
557 #if 0
558 /* having a verification hook is a good idea */
559 /* use debug_fx, debug_fx+dx, etc. */
560
561 for (int i=0;i<4;i++) {
562 uint32_t want = PACK_FILTER_X_NAME(debug_fx, maxX, one PREAMBLE_ARG_X);
563 if (xy[i] != want)
564 {
565 /* print a nastygram */
566 SkDebugf("clamp-filter-scale fails\n");
567 SkDebugf("got %08x want %08x\n", xy[i], want);
568 SkDebugf("fx %08x debug_fx %08x dx %08x done %d\n",
569 fx, debug_fx, dx, count_done);
570 SkDebugf(" maxX %08x one %08x\n", maxX, one);
571
572 }
573 debug_fx += dx;
574 count_done++;
575 }
576 #endif
577 wide_fx += vdupq_n_s32(dx+dx+dx+dx);
578 fx += dx+dx+dx+dx;
579 xy += 4;
580 count -= 4;
581 }
582 }
583
584 while (--count >= 0) {
585 *xy++ = PACK_FILTER_X_NAME(fx, maxX, one PREAMBLE_ARG_X);
586 fx += dx;
587 }
588 }
589
AFFINE_FILTER_NAME(const SkBitmapProcState & s,uint32_t xy[],int count,int x,int y)590 static void AFFINE_FILTER_NAME(const SkBitmapProcState& s,
591 uint32_t xy[], int count, int x, int y) {
592 SkASSERT(s.fInvType & SkMatrix::kAffine_Mask);
593 SkASSERT((s.fInvType & ~(SkMatrix::kTranslate_Mask |
594 SkMatrix::kScale_Mask |
595 SkMatrix::kAffine_Mask)) == 0);
596
597 PREAMBLE(s);
598 SkPoint srcPt;
599 s.fInvProc(s.fInvMatrix,
600 SkIntToScalar(x) + SK_ScalarHalf,
601 SkIntToScalar(y) + SK_ScalarHalf, &srcPt);
602
603 SkFixed oneX = s.fFilterOneX;
604 SkFixed oneY = s.fFilterOneY;
605 SkFixed fx = SkScalarToFixed(srcPt.fX) - (oneX >> 1);
606 SkFixed fy = SkScalarToFixed(srcPt.fY) - (oneY >> 1);
607 SkFixed dx = s.fInvSx;
608 SkFixed dy = s.fInvKy;
609 unsigned maxX = s.fBitmap->width() - 1;
610 unsigned maxY = s.fBitmap->height() - 1;
611
612 if (count >= 4) {
613 int32x4_t wide_i, wide_lo;
614 int32x4_t wide_fx, wide_onex, wide_fx1;
615 int32x4_t wide_fy, wide_oney, wide_fy1;
616
617 #undef AFFINE_DEBUG
618 #if defined(AFFINE_DEBUG)
619 SkFixed fyp = fy;
620 SkFixed fxp = fx;
621 uint32_t *xyp = xy;
622 int count_done = 0;
623 #endif
624
625 wide_fx = vdupq_n_s32(fx);
626 wide_fx = vsetq_lane_s32(fx+dx, wide_fx, 1);
627 wide_fx = vsetq_lane_s32(fx+dx+dx, wide_fx, 2);
628 wide_fx = vsetq_lane_s32(fx+dx+dx+dx, wide_fx, 3);
629
630 wide_fy = vdupq_n_s32(fy);
631 wide_fy = vsetq_lane_s32(fy+dy, wide_fy, 1);
632 wide_fy = vsetq_lane_s32(fy+dy+dy, wide_fy, 2);
633 wide_fy = vsetq_lane_s32(fy+dy+dy+dy, wide_fy, 3);
634
635 wide_onex = vdupq_n_s32(oneX);
636 wide_oney = vdupq_n_s32(oneY);
637
638 while (count >= 4) {
639 int32x4_t wide_x;
640 int32x4_t wide_y;
641
642 /* do the X side, then the Y side, then interleave them */
643
644 /* original expands to:
645 * unsigned i = SkClampMax((f) >> 16, max);
646 * i = (i << 4) | (((f) >> 12) & 0xF);
647 * return (i << 14) | (SkClampMax(((f + one)) >> 16, max));
648 */
649
650 /* i = SkClampMax(f>>16, maxX) */
651 wide_i = vmaxq_s32(vshrq_n_s32(wide_fx,16), vdupq_n_s32(0));
652 wide_i = vminq_s32(wide_i, vdupq_n_s32(maxX));
653
654 /* i<<4 | TILEX_LOW_BITS(fx) */
655 wide_lo = vshrq_n_s32(wide_fx, 12);
656 wide_i = vsliq_n_s32(wide_lo, wide_i, 4);
657
658 /* i<<14 */
659 wide_i = vshlq_n_s32(wide_i, 14);
660
661 /* SkClampMax(((f + one)) >> 16, max) */
662 wide_fx1 = vaddq_s32(wide_fx, wide_onex);
663 wide_fx1 = vmaxq_s32(vshrq_n_s32(wide_fx1,16), vdupq_n_s32(0));
664 wide_fx1 = vminq_s32(wide_fx1, vdupq_n_s32(maxX));
665
666 /* final combination */
667 wide_x = vorrq_s32(wide_i, wide_fx1);
668
669 /* And now the Y side */
670
671 /* i = SkClampMax(f>>16, maxX) */
672 wide_i = vmaxq_s32(vshrq_n_s32(wide_fy,16), vdupq_n_s32(0));
673 wide_i = vminq_s32(wide_i, vdupq_n_s32(maxY));
674
675 /* i<<4 | TILEX_LOW_BITS(fx) */
676 wide_lo = vshrq_n_s32(wide_fy, 12);
677 wide_i = vsliq_n_s32(wide_lo, wide_i, 4);
678
679 /* i<<14 */
680 wide_i = vshlq_n_s32(wide_i, 14);
681
682 /* SkClampMax(((f + one)) >> 16, max) */
683 wide_fy1 = vaddq_s32(wide_fy, wide_oney);
684 wide_fy1 = vmaxq_s32(vshrq_n_s32(wide_fy1,16), vdupq_n_s32(0));
685 wide_fy1 = vminq_s32(wide_fy1, vdupq_n_s32(maxY));
686
687 /* final combination */
688 wide_y = vorrq_s32(wide_i, wide_fy1);
689
690 /* interleave as YXYXYXYX as part of the storing */
691 {
692 /* vst2.32 needs side-by-side registers */
693 register int32x4_t t_x asm("q1");
694 register int32x4_t t_y asm("q0");
695
696 t_x = wide_x; t_y = wide_y;
697 asm ("vst2.32 {q0-q1},[%2] /* y=%q0 x=%q1 */"
698 :
699 : "w" (t_y), "w" (t_x), "r" (xy)
700 );
701 }
702
703 #if defined(AFFINE_DEBUG)
704 /* make sure we're good here -- check the 4 we just output */
705 for (int i = 0; i<4;i++) {
706 uint32_t val;
707 val = PACK_FILTER_Y_NAME(fyp, maxY, oneY PREAMBLE_ARG_Y);
708 if (val != xy[i*2+0]) {
709 /* print a nastygram */
710 SkDebugf("clamp-filter-affine fails\n");
711 SkDebugf("[bad-y] got %08x want %08x\n", xy[i*2+0], val);
712 SkDebugf("fy %08x fxp %08x fyp %08x dx %08x dy %08x done %d\n",
713 fy, fxp, fyp, dx, dy, count_done);
714 SkDebugf(" maxY %08x oneY %08x\n", maxY, oneY);
715 }
716 val = PACK_FILTER_X_NAME(fxp, maxX, oneX PREAMBLE_ARG_X);
717 if (val != xy[i*2+1]) {
718 /* print a nastygram */
719 SkDebugf("clamp-filter-affine fails\n");
720 SkDebugf("[bad-x] got %08x want %08x\n", xy[i*2+1], val);
721 SkDebugf("fx %08x fxp %08x fyp %08x dx %08x dy %08x done %d\n",
722 fx, fxp, fyp, dx, dy, count_done);
723 SkDebugf(" maxX %08x one %08x\n", maxX, oneX);
724 }
725 fyp += dy;
726 fxp += dx;
727 count_done++;
728 }
729 #endif
730
731 wide_fx += vdupq_n_s32(dx+dx+dx+dx);
732 fx += dx+dx+dx+dx;
733 wide_fy += vdupq_n_s32(dy+dy+dy+dy);
734 fy += dy+dy+dy+dy;
735 xy += 8; /* 4 x's, 4 y's */
736 count -= 4;
737 }
738 }
739
740 while (--count >= 0) {
741 /* NB: writing Y/X */
742 *xy++ = PACK_FILTER_Y_NAME(fy, maxY, oneY PREAMBLE_ARG_Y);
743 fy += dy;
744 *xy++ = PACK_FILTER_X_NAME(fx, maxX, oneX PREAMBLE_ARG_X);
745 fx += dx;
746 }
747 }
748
PERSP_FILTER_NAME(const SkBitmapProcState & s,uint32_t * SK_RESTRICT xy,int count,int x,int y)749 static void PERSP_FILTER_NAME(const SkBitmapProcState& s,
750 uint32_t* SK_RESTRICT xy, int count,
751 int x, int y) {
752 SkASSERT(s.fInvType & SkMatrix::kPerspective_Mask);
753
754 PREAMBLE(s);
755 unsigned maxX = s.fBitmap->width() - 1;
756 unsigned maxY = s.fBitmap->height() - 1;
757 SkFixed oneX = s.fFilterOneX;
758 SkFixed oneY = s.fFilterOneY;
759
760 SkPerspIter iter(s.fInvMatrix,
761 SkIntToScalar(x) + SK_ScalarHalf,
762 SkIntToScalar(y) + SK_ScalarHalf, count);
763
764 while ((count = iter.next()) != 0) {
765 const SkFixed* SK_RESTRICT srcXY = iter.getXY();
766
767 if (count >= 4) {
768 int32x4_t wide_i, wide_lo;
769 int32x4_t wide_fx1;
770 int32x4_t wide_fy1;
771 int32x4_t wide_x, wide_y;
772
773 while (count >= 4) {
774 /* RBE: it's good, but:
775 * -- we spill a constant that could be easily regnerated
776 * [perhaps tweak gcc's NEON constant costs?]
777 */
778
779 /* load src: x-y-x-y-x-y-x-y */
780 {
781 register int32x4_t q0 asm ("q0");
782 register int32x4_t q1 asm ("q1");
783 asm ("vld2.32 {q0-q1},[%2] /* x=%q0 y=%q1 */"
784 : "=w" (q0), "=w" (q1)
785 : "r" (srcXY));
786 wide_x = q0; wide_y = q1;
787 }
788
789 /* do the X side, then the Y side, then interleave them */
790
791 wide_x = vsubq_s32(wide_x, vdupq_n_s32 (oneX>>1));
792
793 /* original expands to:
794 * unsigned i = SkClampMax((f) >> 16, max);
795 * i = (i << 4) | (((f) >> 12) & 0xF);
796 * return (i << 14) | (SkClampMax(((f + one)) >> 16, max));
797 */
798
799 /* i = SkClampMax(f>>16, maxX) */
800 wide_i = vmaxq_s32 (vshrq_n_s32 (wide_x, 16), vdupq_n_s32 (0));
801 wide_i = vminq_s32 (wide_i, vdupq_n_s32 (maxX));
802
803 /* i<<4 | TILEX_LOW_BITS(fx) */
804 wide_lo = vshrq_n_s32 (wide_x, 12);
805 wide_i = vsliq_n_s32 (wide_lo, wide_i, 4);
806
807 /* i<<14 */
808 wide_i = vshlq_n_s32 (wide_i, 14);
809
810 /* SkClampMax(((f + one)) >> 16, max) */
811 wide_fx1 = vaddq_s32 (wide_x, vdupq_n_s32(oneX));
812 wide_fx1 = vmaxq_s32 (vshrq_n_s32 (wide_fx1, 16), vdupq_n_s32 (0));
813 wide_fx1 = vminq_s32 (wide_fx1, vdupq_n_s32 (maxX));
814
815 /* final combination */
816 wide_x = vorrq_s32 (wide_i, wide_fx1);
817
818
819 /* And now the Y side */
820
821 wide_y = vsubq_s32(wide_y, vdupq_n_s32 (oneY>>1));
822
823 /* i = SkClampMax(f>>16, maxX) */
824 wide_i = vmaxq_s32 (vshrq_n_s32 (wide_y, 16), vdupq_n_s32 (0));
825 wide_i = vminq_s32 (wide_i, vdupq_n_s32 (maxY));
826
827 /* i<<4 | TILEX_LOW_BITS(fx) */
828 wide_lo = vshrq_n_s32 (wide_y, 12);
829 wide_i = vsliq_n_s32 (wide_lo, wide_i, 4);
830
831 /* i<<14 */
832 wide_i = vshlq_n_s32 (wide_i, 14);
833
834 /* SkClampMax(((f + one)) >> 16, max) */
835
836 /* wide_fy1_1 and wide_fy1_2 are just temporary variables to
837 * work-around an ICE in debug */
838 int32x4_t wide_fy1_1 = vaddq_s32 (wide_y, vdupq_n_s32(oneY));
839 int32x4_t wide_fy1_2 = vmaxq_s32 (vshrq_n_s32 (wide_fy1_1, 16),
840 vdupq_n_s32 (0));
841 wide_fy1 = vminq_s32 (wide_fy1_2, vdupq_n_s32 (maxY));
842
843 /* final combination */
844 wide_y = vorrq_s32 (wide_i, wide_fy1);
845
846 /* switch them around; have to do it this way to get them
847 * in the proper registers to match our instruction */
848
849 /* iteration bookkeeping, ahead of the asm() for scheduling */
850 srcXY += 2*4;
851 count -= 4;
852
853 /* store interleaved as y-x-y-x-y-x-y-x (NB != read order) */
854 {
855 register int32x4_t q0 asm ("q0") = wide_y;
856 register int32x4_t q1 asm ("q1") = wide_x;
857
858 asm ("vst2.32 {q0-q1},[%2] /* y=%q0 x=%q1 */"
859 :
860 : "w" (q0), "w" (q1), "r" (xy));
861 }
862
863 /* on to the next iteration */
864 /* count, srcXY are handled above */
865 xy += 2*4;
866 }
867 }
868
869 /* was do-while; NEON code invalidates original count>0 assumption */
870 while (--count >= 0) {
871 /* NB: we read x/y, we write y/x */
872 *xy++ = PACK_FILTER_Y_NAME(srcXY[1] - (oneY >> 1), maxY,
873 oneY PREAMBLE_ARG_Y);
874 *xy++ = PACK_FILTER_X_NAME(srcXY[0] - (oneX >> 1), maxX,
875 oneX PREAMBLE_ARG_X);
876 srcXY += 2;
877 }
878 }
879 }
880
881 const SkBitmapProcState::MatrixProc MAKENAME(_Procs)[] = {
882 SCALE_NOFILTER_NAME,
883 SCALE_FILTER_NAME,
884 AFFINE_NOFILTER_NAME,
885 AFFINE_FILTER_NAME,
886 PERSP_NOFILTER_NAME,
887 PERSP_FILTER_NAME
888 };
889
890 #undef MAKENAME
891 #undef TILEX_PROCF
892 #undef TILEY_PROCF
893 #ifdef CHECK_FOR_DECAL
894 #undef CHECK_FOR_DECAL
895 #endif
896
897 #undef SCALE_NOFILTER_NAME
898 #undef SCALE_FILTER_NAME
899 #undef AFFINE_NOFILTER_NAME
900 #undef AFFINE_FILTER_NAME
901 #undef PERSP_NOFILTER_NAME
902 #undef PERSP_FILTER_NAME
903
904 #undef PREAMBLE
905 #undef PREAMBLE_PARAM_X
906 #undef PREAMBLE_PARAM_Y
907 #undef PREAMBLE_ARG_X
908 #undef PREAMBLE_ARG_Y
909
910 #undef TILEX_LOW_BITS
911 #undef TILEY_LOW_BITS
912