1
2 /*
3 * Copyright 2012 Google Inc.
4 *
5 * Use of this source code is governed by a BSD-style license that can be
6 * found in the LICENSE file.
7 */
8 #include "SkBitmapProcState.h"
9 #include "SkBitmapProcState_filter.h"
10 #include "SkColorPriv.h"
11 #include "SkFilterProc.h"
12 #include "SkPaint.h"
13 #include "SkShader.h" // for tilemodes
14 #include "SkUtilsArm.h"
15
16 // Required to ensure the table is part of the final binary.
17 extern const SkBitmapProcState::SampleProc32 gSkBitmapProcStateSample32_neon[];
18 extern const SkBitmapProcState::SampleProc16 gSkBitmapProcStateSample16_neon[];
19
20 #define NAME_WRAP(x) x ## _neon
21 #include "SkBitmapProcState_filter_neon.h"
22 #include "SkBitmapProcState_procs.h"
23
24 const SkBitmapProcState::SampleProc32 gSkBitmapProcStateSample32_neon[] = {
25 S32_opaque_D32_nofilter_DXDY_neon,
26 S32_alpha_D32_nofilter_DXDY_neon,
27 S32_opaque_D32_nofilter_DX_neon,
28 S32_alpha_D32_nofilter_DX_neon,
29 S32_opaque_D32_filter_DXDY_neon,
30 S32_alpha_D32_filter_DXDY_neon,
31 S32_opaque_D32_filter_DX_neon,
32 S32_alpha_D32_filter_DX_neon,
33
34 S16_opaque_D32_nofilter_DXDY_neon,
35 S16_alpha_D32_nofilter_DXDY_neon,
36 S16_opaque_D32_nofilter_DX_neon,
37 S16_alpha_D32_nofilter_DX_neon,
38 S16_opaque_D32_filter_DXDY_neon,
39 S16_alpha_D32_filter_DXDY_neon,
40 S16_opaque_D32_filter_DX_neon,
41 S16_alpha_D32_filter_DX_neon,
42
43 SI8_opaque_D32_nofilter_DXDY_neon,
44 SI8_alpha_D32_nofilter_DXDY_neon,
45 SI8_opaque_D32_nofilter_DX_neon,
46 SI8_alpha_D32_nofilter_DX_neon,
47 SI8_opaque_D32_filter_DXDY_neon,
48 SI8_alpha_D32_filter_DXDY_neon,
49 SI8_opaque_D32_filter_DX_neon,
50 SI8_alpha_D32_filter_DX_neon,
51
52 S4444_opaque_D32_nofilter_DXDY_neon,
53 S4444_alpha_D32_nofilter_DXDY_neon,
54 S4444_opaque_D32_nofilter_DX_neon,
55 S4444_alpha_D32_nofilter_DX_neon,
56 S4444_opaque_D32_filter_DXDY_neon,
57 S4444_alpha_D32_filter_DXDY_neon,
58 S4444_opaque_D32_filter_DX_neon,
59 S4444_alpha_D32_filter_DX_neon,
60
61 // A8 treats alpha/opauqe the same (equally efficient)
62 SA8_alpha_D32_nofilter_DXDY_neon,
63 SA8_alpha_D32_nofilter_DXDY_neon,
64 SA8_alpha_D32_nofilter_DX_neon,
65 SA8_alpha_D32_nofilter_DX_neon,
66 SA8_alpha_D32_filter_DXDY_neon,
67 SA8_alpha_D32_filter_DXDY_neon,
68 SA8_alpha_D32_filter_DX_neon,
69 SA8_alpha_D32_filter_DX_neon,
70
71 // todo: possibly specialize on opaqueness
72 SG8_alpha_D32_nofilter_DXDY_neon,
73 SG8_alpha_D32_nofilter_DXDY_neon,
74 SG8_alpha_D32_nofilter_DX_neon,
75 SG8_alpha_D32_nofilter_DX_neon,
76 SG8_alpha_D32_filter_DXDY_neon,
77 SG8_alpha_D32_filter_DXDY_neon,
78 SG8_alpha_D32_filter_DX_neon,
79 SG8_alpha_D32_filter_DX_neon,
80 };
81
82 const SkBitmapProcState::SampleProc16 gSkBitmapProcStateSample16_neon[] = {
83 S32_D16_nofilter_DXDY_neon,
84 S32_D16_nofilter_DX_neon,
85 S32_D16_filter_DXDY_neon,
86 S32_D16_filter_DX_neon,
87
88 S16_D16_nofilter_DXDY_neon,
89 S16_D16_nofilter_DX_neon,
90 S16_D16_filter_DXDY_neon,
91 S16_D16_filter_DX_neon,
92
93 SI8_D16_nofilter_DXDY_neon,
94 SI8_D16_nofilter_DX_neon,
95 SI8_D16_filter_DXDY_neon,
96 SI8_D16_filter_DX_neon,
97
98 // Don't support 4444 -> 565
99 NULL, NULL, NULL, NULL,
100 // Don't support A8 -> 565
101 NULL, NULL, NULL, NULL,
102 // Don't support G8 -> 565 (but we could)
103 NULL, NULL, NULL, NULL,
104 };
105
106 ///////////////////////////////////////////////////////////////////////////////
107
108 #include <arm_neon.h>
109 #include "SkConvolver.h"
110
111 // Convolves horizontally along a single row. The row data is given in
112 // |srcData| and continues for the numValues() of the filter.
convolveHorizontally_neon(const unsigned char * srcData,const SkConvolutionFilter1D & filter,unsigned char * outRow,bool hasAlpha)113 void convolveHorizontally_neon(const unsigned char* srcData,
114 const SkConvolutionFilter1D& filter,
115 unsigned char* outRow,
116 bool hasAlpha) {
117 // Loop over each pixel on this row in the output image.
118 int numValues = filter.numValues();
119 for (int outX = 0; outX < numValues; outX++) {
120 uint8x8_t coeff_mask0 = vcreate_u8(0x0100010001000100);
121 uint8x8_t coeff_mask1 = vcreate_u8(0x0302030203020302);
122 uint8x8_t coeff_mask2 = vcreate_u8(0x0504050405040504);
123 uint8x8_t coeff_mask3 = vcreate_u8(0x0706070607060706);
124 // Get the filter that determines the current output pixel.
125 int filterOffset, filterLength;
126 const SkConvolutionFilter1D::ConvolutionFixed* filterValues =
127 filter.FilterForValue(outX, &filterOffset, &filterLength);
128
129 // Compute the first pixel in this row that the filter affects. It will
130 // touch |filterLength| pixels (4 bytes each) after this.
131 const unsigned char* rowToFilter = &srcData[filterOffset * 4];
132
133 // Apply the filter to the row to get the destination pixel in |accum|.
134 int32x4_t accum = vdupq_n_s32(0);
135 for (int filterX = 0; filterX < filterLength >> 2; filterX++) {
136 // Load 4 coefficients
137 int16x4_t coeffs, coeff0, coeff1, coeff2, coeff3;
138 coeffs = vld1_s16(filterValues);
139 coeff0 = vreinterpret_s16_u8(vtbl1_u8(vreinterpret_u8_s16(coeffs), coeff_mask0));
140 coeff1 = vreinterpret_s16_u8(vtbl1_u8(vreinterpret_u8_s16(coeffs), coeff_mask1));
141 coeff2 = vreinterpret_s16_u8(vtbl1_u8(vreinterpret_u8_s16(coeffs), coeff_mask2));
142 coeff3 = vreinterpret_s16_u8(vtbl1_u8(vreinterpret_u8_s16(coeffs), coeff_mask3));
143
144 // Load pixels and calc
145 uint8x16_t pixels = vld1q_u8(rowToFilter);
146 int16x8_t p01_16 = vreinterpretq_s16_u16(vmovl_u8(vget_low_u8(pixels)));
147 int16x8_t p23_16 = vreinterpretq_s16_u16(vmovl_u8(vget_high_u8(pixels)));
148
149 int16x4_t p0_src = vget_low_s16(p01_16);
150 int16x4_t p1_src = vget_high_s16(p01_16);
151 int16x4_t p2_src = vget_low_s16(p23_16);
152 int16x4_t p3_src = vget_high_s16(p23_16);
153
154 int32x4_t p0 = vmull_s16(p0_src, coeff0);
155 int32x4_t p1 = vmull_s16(p1_src, coeff1);
156 int32x4_t p2 = vmull_s16(p2_src, coeff2);
157 int32x4_t p3 = vmull_s16(p3_src, coeff3);
158
159 accum += p0;
160 accum += p1;
161 accum += p2;
162 accum += p3;
163
164 // Advance the pointers
165 rowToFilter += 16;
166 filterValues += 4;
167 }
168 int r = filterLength & 3;
169 if (r) {
170 const uint16_t mask[4][4] = {
171 {0, 0, 0, 0},
172 {0xFFFF, 0, 0, 0},
173 {0xFFFF, 0xFFFF, 0, 0},
174 {0xFFFF, 0xFFFF, 0xFFFF, 0}
175 };
176 uint16x4_t coeffs;
177 int16x4_t coeff0, coeff1, coeff2;
178 coeffs = vld1_u16(reinterpret_cast<const uint16_t*>(filterValues));
179 coeffs &= vld1_u16(&mask[r][0]);
180 coeff0 = vreinterpret_s16_u8(vtbl1_u8(vreinterpret_u8_u16(coeffs), coeff_mask0));
181 coeff1 = vreinterpret_s16_u8(vtbl1_u8(vreinterpret_u8_u16(coeffs), coeff_mask1));
182 coeff2 = vreinterpret_s16_u8(vtbl1_u8(vreinterpret_u8_u16(coeffs), coeff_mask2));
183
184 // Load pixels and calc
185 uint8x16_t pixels = vld1q_u8(rowToFilter);
186 int16x8_t p01_16 = vreinterpretq_s16_u16(vmovl_u8(vget_low_u8(pixels)));
187 int16x8_t p23_16 = vreinterpretq_s16_u16(vmovl_u8(vget_high_u8(pixels)));
188 int32x4_t p0 = vmull_s16(vget_low_s16(p01_16), coeff0);
189 int32x4_t p1 = vmull_s16(vget_high_s16(p01_16), coeff1);
190 int32x4_t p2 = vmull_s16(vget_low_s16(p23_16), coeff2);
191
192 accum += p0;
193 accum += p1;
194 accum += p2;
195 }
196
197 // Bring this value back in range. All of the filter scaling factors
198 // are in fixed point with kShiftBits bits of fractional part.
199 accum = vshrq_n_s32(accum, SkConvolutionFilter1D::kShiftBits);
200
201 // Pack and store the new pixel.
202 int16x4_t accum16 = vqmovn_s32(accum);
203 uint8x8_t accum8 = vqmovun_s16(vcombine_s16(accum16, accum16));
204 vst1_lane_u32(reinterpret_cast<uint32_t*>(outRow), vreinterpret_u32_u8(accum8), 0);
205 outRow += 4;
206 }
207 }
208
209 // Does vertical convolution to produce one output row. The filter values and
210 // length are given in the first two parameters. These are applied to each
211 // of the rows pointed to in the |sourceDataRows| array, with each row
212 // being |pixelWidth| wide.
213 //
214 // The output must have room for |pixelWidth * 4| bytes.
215 template<bool hasAlpha>
convolveVertically_neon(const SkConvolutionFilter1D::ConvolutionFixed * filterValues,int filterLength,unsigned char * const * sourceDataRows,int pixelWidth,unsigned char * outRow)216 void convolveVertically_neon(const SkConvolutionFilter1D::ConvolutionFixed* filterValues,
217 int filterLength,
218 unsigned char* const* sourceDataRows,
219 int pixelWidth,
220 unsigned char* outRow) {
221 int width = pixelWidth & ~3;
222
223 int32x4_t accum0, accum1, accum2, accum3;
224 int16x4_t coeff16;
225
226 // Output four pixels per iteration (16 bytes).
227 for (int outX = 0; outX < width; outX += 4) {
228
229 // Accumulated result for each pixel. 32 bits per RGBA channel.
230 accum0 = accum1 = accum2 = accum3 = vdupq_n_s32(0);
231
232 // Convolve with one filter coefficient per iteration.
233 for (int filterY = 0; filterY < filterLength; filterY++) {
234
235 // Duplicate the filter coefficient 4 times.
236 // [16] cj cj cj cj
237 coeff16 = vdup_n_s16(filterValues[filterY]);
238
239 // Load four pixels (16 bytes) together.
240 // [8] a3 b3 g3 r3 a2 b2 g2 r2 a1 b1 g1 r1 a0 b0 g0 r0
241 uint8x16_t src8 = vld1q_u8(&sourceDataRows[filterY][outX << 2]);
242
243 int16x8_t src16_01 = vreinterpretq_s16_u16(vmovl_u8(vget_low_u8(src8)));
244 int16x8_t src16_23 = vreinterpretq_s16_u16(vmovl_u8(vget_high_u8(src8)));
245 int16x4_t src16_0 = vget_low_s16(src16_01);
246 int16x4_t src16_1 = vget_high_s16(src16_01);
247 int16x4_t src16_2 = vget_low_s16(src16_23);
248 int16x4_t src16_3 = vget_high_s16(src16_23);
249
250 accum0 += vmull_s16(src16_0, coeff16);
251 accum1 += vmull_s16(src16_1, coeff16);
252 accum2 += vmull_s16(src16_2, coeff16);
253 accum3 += vmull_s16(src16_3, coeff16);
254 }
255
256 // Shift right for fixed point implementation.
257 accum0 = vshrq_n_s32(accum0, SkConvolutionFilter1D::kShiftBits);
258 accum1 = vshrq_n_s32(accum1, SkConvolutionFilter1D::kShiftBits);
259 accum2 = vshrq_n_s32(accum2, SkConvolutionFilter1D::kShiftBits);
260 accum3 = vshrq_n_s32(accum3, SkConvolutionFilter1D::kShiftBits);
261
262 // Packing 32 bits |accum| to 16 bits per channel (signed saturation).
263 // [16] a1 b1 g1 r1 a0 b0 g0 r0
264 int16x8_t accum16_0 = vcombine_s16(vqmovn_s32(accum0), vqmovn_s32(accum1));
265 // [16] a3 b3 g3 r3 a2 b2 g2 r2
266 int16x8_t accum16_1 = vcombine_s16(vqmovn_s32(accum2), vqmovn_s32(accum3));
267
268 // Packing 16 bits |accum| to 8 bits per channel (unsigned saturation).
269 // [8] a3 b3 g3 r3 a2 b2 g2 r2 a1 b1 g1 r1 a0 b0 g0 r0
270 uint8x16_t accum8 = vcombine_u8(vqmovun_s16(accum16_0), vqmovun_s16(accum16_1));
271
272 if (hasAlpha) {
273 // Compute the max(ri, gi, bi) for each pixel.
274 // [8] xx a3 b3 g3 xx a2 b2 g2 xx a1 b1 g1 xx a0 b0 g0
275 uint8x16_t a = vreinterpretq_u8_u32(vshrq_n_u32(vreinterpretq_u32_u8(accum8), 8));
276 // [8] xx xx xx max3 xx xx xx max2 xx xx xx max1 xx xx xx max0
277 uint8x16_t b = vmaxq_u8(a, accum8); // Max of r and g
278 // [8] xx xx a3 b3 xx xx a2 b2 xx xx a1 b1 xx xx a0 b0
279 a = vreinterpretq_u8_u32(vshrq_n_u32(vreinterpretq_u32_u8(accum8), 16));
280 // [8] xx xx xx max3 xx xx xx max2 xx xx xx max1 xx xx xx max0
281 b = vmaxq_u8(a, b); // Max of r and g and b.
282 // [8] max3 00 00 00 max2 00 00 00 max1 00 00 00 max0 00 00 00
283 b = vreinterpretq_u8_u32(vshlq_n_u32(vreinterpretq_u32_u8(b), 24));
284
285 // Make sure the value of alpha channel is always larger than maximum
286 // value of color channels.
287 accum8 = vmaxq_u8(b, accum8);
288 } else {
289 // Set value of alpha channels to 0xFF.
290 accum8 = vreinterpretq_u8_u32(vreinterpretq_u32_u8(accum8) | vdupq_n_u32(0xFF000000));
291 }
292
293 // Store the convolution result (16 bytes) and advance the pixel pointers.
294 vst1q_u8(outRow, accum8);
295 outRow += 16;
296 }
297
298 // Process the leftovers when the width of the output is not divisible
299 // by 4, that is at most 3 pixels.
300 int r = pixelWidth & 3;
301 if (r) {
302
303 accum0 = accum1 = accum2 = vdupq_n_s32(0);
304
305 for (int filterY = 0; filterY < filterLength; ++filterY) {
306 coeff16 = vdup_n_s16(filterValues[filterY]);
307
308 // [8] a3 b3 g3 r3 a2 b2 g2 r2 a1 b1 g1 r1 a0 b0 g0 r0
309 uint8x16_t src8 = vld1q_u8(&sourceDataRows[filterY][width << 2]);
310
311 int16x8_t src16_01 = vreinterpretq_s16_u16(vmovl_u8(vget_low_u8(src8)));
312 int16x8_t src16_23 = vreinterpretq_s16_u16(vmovl_u8(vget_high_u8(src8)));
313 int16x4_t src16_0 = vget_low_s16(src16_01);
314 int16x4_t src16_1 = vget_high_s16(src16_01);
315 int16x4_t src16_2 = vget_low_s16(src16_23);
316
317 accum0 += vmull_s16(src16_0, coeff16);
318 accum1 += vmull_s16(src16_1, coeff16);
319 accum2 += vmull_s16(src16_2, coeff16);
320 }
321
322 accum0 = vshrq_n_s32(accum0, SkConvolutionFilter1D::kShiftBits);
323 accum1 = vshrq_n_s32(accum1, SkConvolutionFilter1D::kShiftBits);
324 accum2 = vshrq_n_s32(accum2, SkConvolutionFilter1D::kShiftBits);
325
326 int16x8_t accum16_0 = vcombine_s16(vqmovn_s32(accum0), vqmovn_s32(accum1));
327 int16x8_t accum16_1 = vcombine_s16(vqmovn_s32(accum2), vqmovn_s32(accum2));
328
329 uint8x16_t accum8 = vcombine_u8(vqmovun_s16(accum16_0), vqmovun_s16(accum16_1));
330
331 if (hasAlpha) {
332 // Compute the max(ri, gi, bi) for each pixel.
333 // [8] xx a3 b3 g3 xx a2 b2 g2 xx a1 b1 g1 xx a0 b0 g0
334 uint8x16_t a = vreinterpretq_u8_u32(vshrq_n_u32(vreinterpretq_u32_u8(accum8), 8));
335 // [8] xx xx xx max3 xx xx xx max2 xx xx xx max1 xx xx xx max0
336 uint8x16_t b = vmaxq_u8(a, accum8); // Max of r and g
337 // [8] xx xx a3 b3 xx xx a2 b2 xx xx a1 b1 xx xx a0 b0
338 a = vreinterpretq_u8_u32(vshrq_n_u32(vreinterpretq_u32_u8(accum8), 16));
339 // [8] xx xx xx max3 xx xx xx max2 xx xx xx max1 xx xx xx max0
340 b = vmaxq_u8(a, b); // Max of r and g and b.
341 // [8] max3 00 00 00 max2 00 00 00 max1 00 00 00 max0 00 00 00
342 b = vreinterpretq_u8_u32(vshlq_n_u32(vreinterpretq_u32_u8(b), 24));
343
344 // Make sure the value of alpha channel is always larger than maximum
345 // value of color channels.
346 accum8 = vmaxq_u8(b, accum8);
347 } else {
348 // Set value of alpha channels to 0xFF.
349 accum8 = vreinterpretq_u8_u32(vreinterpretq_u32_u8(accum8) | vdupq_n_u32(0xFF000000));
350 }
351
352 switch(r) {
353 case 1:
354 vst1q_lane_u32(reinterpret_cast<uint32_t*>(outRow), vreinterpretq_u32_u8(accum8), 0);
355 break;
356 case 2:
357 vst1_u32(reinterpret_cast<uint32_t*>(outRow),
358 vreinterpret_u32_u8(vget_low_u8(accum8)));
359 break;
360 case 3:
361 vst1_u32(reinterpret_cast<uint32_t*>(outRow),
362 vreinterpret_u32_u8(vget_low_u8(accum8)));
363 vst1q_lane_u32(reinterpret_cast<uint32_t*>(outRow+8), vreinterpretq_u32_u8(accum8), 2);
364 break;
365 }
366 }
367 }
368
convolveVertically_neon(const SkConvolutionFilter1D::ConvolutionFixed * filterValues,int filterLength,unsigned char * const * sourceDataRows,int pixelWidth,unsigned char * outRow,bool sourceHasAlpha)369 void convolveVertically_neon(const SkConvolutionFilter1D::ConvolutionFixed* filterValues,
370 int filterLength,
371 unsigned char* const* sourceDataRows,
372 int pixelWidth,
373 unsigned char* outRow,
374 bool sourceHasAlpha) {
375 if (sourceHasAlpha) {
376 convolveVertically_neon<true>(filterValues, filterLength,
377 sourceDataRows, pixelWidth,
378 outRow);
379 } else {
380 convolveVertically_neon<false>(filterValues, filterLength,
381 sourceDataRows, pixelWidth,
382 outRow);
383 }
384 }
385
386 // Convolves horizontally along four rows. The row data is given in
387 // |src_data| and continues for the num_values() of the filter.
388 // The algorithm is almost same as |ConvolveHorizontally_SSE2|. Please
389 // refer to that function for detailed comments.
convolve4RowsHorizontally_neon(const unsigned char * srcData[4],const SkConvolutionFilter1D & filter,unsigned char * outRow[4])390 void convolve4RowsHorizontally_neon(const unsigned char* srcData[4],
391 const SkConvolutionFilter1D& filter,
392 unsigned char* outRow[4]) {
393
394 uint8x8_t coeff_mask0 = vcreate_u8(0x0100010001000100);
395 uint8x8_t coeff_mask1 = vcreate_u8(0x0302030203020302);
396 uint8x8_t coeff_mask2 = vcreate_u8(0x0504050405040504);
397 uint8x8_t coeff_mask3 = vcreate_u8(0x0706070607060706);
398 int num_values = filter.numValues();
399
400 int filterOffset, filterLength;
401 // |mask| will be used to decimate all extra filter coefficients that are
402 // loaded by SIMD when |filter_length| is not divisible by 4.
403 // mask[0] is not used in following algorithm.
404 const uint16_t mask[4][4] = {
405 {0, 0, 0, 0},
406 {0xFFFF, 0, 0, 0},
407 {0xFFFF, 0xFFFF, 0, 0},
408 {0xFFFF, 0xFFFF, 0xFFFF, 0}
409 };
410
411 // Output one pixel each iteration, calculating all channels (RGBA) together.
412 for (int outX = 0; outX < num_values; outX++) {
413
414 const SkConvolutionFilter1D::ConvolutionFixed* filterValues =
415 filter.FilterForValue(outX, &filterOffset, &filterLength);
416
417 // four pixels in a column per iteration.
418 int32x4_t accum0 = vdupq_n_s32(0);
419 int32x4_t accum1 = vdupq_n_s32(0);
420 int32x4_t accum2 = vdupq_n_s32(0);
421 int32x4_t accum3 = vdupq_n_s32(0);
422
423 int start = (filterOffset<<2);
424
425 // We will load and accumulate with four coefficients per iteration.
426 for (int filter_x = 0; filter_x < (filterLength >> 2); filter_x++) {
427 int16x4_t coeffs, coeff0, coeff1, coeff2, coeff3;
428
429 coeffs = vld1_s16(filterValues);
430 coeff0 = vreinterpret_s16_u8(vtbl1_u8(vreinterpret_u8_s16(coeffs), coeff_mask0));
431 coeff1 = vreinterpret_s16_u8(vtbl1_u8(vreinterpret_u8_s16(coeffs), coeff_mask1));
432 coeff2 = vreinterpret_s16_u8(vtbl1_u8(vreinterpret_u8_s16(coeffs), coeff_mask2));
433 coeff3 = vreinterpret_s16_u8(vtbl1_u8(vreinterpret_u8_s16(coeffs), coeff_mask3));
434
435 uint8x16_t pixels;
436 int16x8_t p01_16, p23_16;
437 int32x4_t p0, p1, p2, p3;
438
439
440 #define ITERATION(src, accum) \
441 pixels = vld1q_u8(src); \
442 p01_16 = vreinterpretq_s16_u16(vmovl_u8(vget_low_u8(pixels))); \
443 p23_16 = vreinterpretq_s16_u16(vmovl_u8(vget_high_u8(pixels))); \
444 p0 = vmull_s16(vget_low_s16(p01_16), coeff0); \
445 p1 = vmull_s16(vget_high_s16(p01_16), coeff1); \
446 p2 = vmull_s16(vget_low_s16(p23_16), coeff2); \
447 p3 = vmull_s16(vget_high_s16(p23_16), coeff3); \
448 accum += p0; \
449 accum += p1; \
450 accum += p2; \
451 accum += p3
452
453 ITERATION(srcData[0] + start, accum0);
454 ITERATION(srcData[1] + start, accum1);
455 ITERATION(srcData[2] + start, accum2);
456 ITERATION(srcData[3] + start, accum3);
457
458 start += 16;
459 filterValues += 4;
460 }
461
462 int r = filterLength & 3;
463 if (r) {
464 int16x4_t coeffs, coeff0, coeff1, coeff2, coeff3;
465 coeffs = vld1_s16(filterValues);
466 coeffs &= vreinterpret_s16_u16(vld1_u16(&mask[r][0]));
467 coeff0 = vreinterpret_s16_u8(vtbl1_u8(vreinterpret_u8_s16(coeffs), coeff_mask0));
468 coeff1 = vreinterpret_s16_u8(vtbl1_u8(vreinterpret_u8_s16(coeffs), coeff_mask1));
469 coeff2 = vreinterpret_s16_u8(vtbl1_u8(vreinterpret_u8_s16(coeffs), coeff_mask2));
470 coeff3 = vreinterpret_s16_u8(vtbl1_u8(vreinterpret_u8_s16(coeffs), coeff_mask3));
471
472 uint8x16_t pixels;
473 int16x8_t p01_16, p23_16;
474 int32x4_t p0, p1, p2, p3;
475
476 ITERATION(srcData[0] + start, accum0);
477 ITERATION(srcData[1] + start, accum1);
478 ITERATION(srcData[2] + start, accum2);
479 ITERATION(srcData[3] + start, accum3);
480 }
481
482 int16x4_t accum16;
483 uint8x8_t res0, res1, res2, res3;
484
485 #define PACK_RESULT(accum, res) \
486 accum = vshrq_n_s32(accum, SkConvolutionFilter1D::kShiftBits); \
487 accum16 = vqmovn_s32(accum); \
488 res = vqmovun_s16(vcombine_s16(accum16, accum16));
489
490 PACK_RESULT(accum0, res0);
491 PACK_RESULT(accum1, res1);
492 PACK_RESULT(accum2, res2);
493 PACK_RESULT(accum3, res3);
494
495 vst1_lane_u32(reinterpret_cast<uint32_t*>(outRow[0]), vreinterpret_u32_u8(res0), 0);
496 vst1_lane_u32(reinterpret_cast<uint32_t*>(outRow[1]), vreinterpret_u32_u8(res1), 0);
497 vst1_lane_u32(reinterpret_cast<uint32_t*>(outRow[2]), vreinterpret_u32_u8(res2), 0);
498 vst1_lane_u32(reinterpret_cast<uint32_t*>(outRow[3]), vreinterpret_u32_u8(res3), 0);
499 outRow[0] += 4;
500 outRow[1] += 4;
501 outRow[2] += 4;
502 outRow[3] += 4;
503 }
504 }
505
applySIMDPadding_neon(SkConvolutionFilter1D * filter)506 void applySIMDPadding_neon(SkConvolutionFilter1D *filter) {
507 // Padding |paddingCount| of more dummy coefficients after the coefficients
508 // of last filter to prevent SIMD instructions which load 8 or 16 bytes
509 // together to access invalid memory areas. We are not trying to align the
510 // coefficients right now due to the opaqueness of <vector> implementation.
511 // This has to be done after all |AddFilter| calls.
512 for (int i = 0; i < 8; ++i) {
513 filter->addFilterValue(static_cast<SkConvolutionFilter1D::ConvolutionFixed>(0));
514 }
515 }
516
platformConvolutionProcs_arm_neon(SkConvolutionProcs * procs)517 void platformConvolutionProcs_arm_neon(SkConvolutionProcs* procs) {
518 procs->fExtraHorizontalReads = 3;
519 procs->fConvolveVertically = &convolveVertically_neon;
520 procs->fConvolve4RowsHorizontally = &convolve4RowsHorizontally_neon;
521 procs->fConvolveHorizontally = &convolveHorizontally_neon;
522 procs->fApplySIMDPadding = &applySIMDPadding_neon;
523 }
524