1 // Copyright 2012 Google Inc. All Rights Reserved.
2 //
3 // This code is licensed under the same terms as WebM:
4 // Software License Agreement: http://www.webmproject.org/license/software/
5 // Additional IP Rights Grant: http://www.webmproject.org/license/additional/
6 // -----------------------------------------------------------------------------
7 //
8 // ARM NEON version of dsp functions and loop filtering.
9 //
10 // Authors: Somnath Banerjee (somnath@google.com)
11 // Johann Koenig (johannkoenig@google.com)
12
13 #include "./dsp.h"
14
15 #if defined(__cplusplus) || defined(c_plusplus)
16 extern "C" {
17 #endif
18
19 #if defined(WEBP_USE_NEON)
20
21 #include "../dec/vp8i.h"
22
23 #define QRegs "q0", "q1", "q2", "q3", "q4", "q5", "q6", "q7", \
24 "q8", "q9", "q10", "q11", "q12", "q13", "q14", "q15"
25
26 #define FLIP_SIGN_BIT2(a, b, s) \
27 "veor " #a "," #a "," #s " \n" \
28 "veor " #b "," #b "," #s " \n" \
29
30 #define FLIP_SIGN_BIT4(a, b, c, d, s) \
31 FLIP_SIGN_BIT2(a, b, s) \
32 FLIP_SIGN_BIT2(c, d, s) \
33
34 #define NEEDS_FILTER(p1, p0, q0, q1, thresh, mask) \
35 "vabd.u8 q15," #p0 "," #q0 " \n" /* abs(p0 - q0) */ \
36 "vabd.u8 q14," #p1 "," #q1 " \n" /* abs(p1 - q1) */ \
37 "vqadd.u8 q15, q15, q15 \n" /* abs(p0 - q0) * 2 */ \
38 "vshr.u8 q14, q14, #1 \n" /* abs(p1 - q1) / 2 */ \
39 "vqadd.u8 q15, q15, q14 \n" /* abs(p0 - q0) * 2 + abs(p1 - q1) / 2 */ \
40 "vdup.8 q14, " #thresh " \n" \
41 "vcge.u8 " #mask ", q14, q15 \n" /* mask <= thresh */
42
43 #define GET_BASE_DELTA(p1, p0, q0, q1, o) \
44 "vqsub.s8 q15," #q0 "," #p0 " \n" /* (q0 - p0) */ \
45 "vqsub.s8 " #o "," #p1 "," #q1 " \n" /* (p1 - q1) */ \
46 "vqadd.s8 " #o "," #o ", q15 \n" /* (p1 - q1) + 1 * (p0 - q0) */ \
47 "vqadd.s8 " #o "," #o ", q15 \n" /* (p1 - q1) + 2 * (p0 - q0) */ \
48 "vqadd.s8 " #o "," #o ", q15 \n" /* (p1 - q1) + 3 * (p0 - q0) */
49
50 #define DO_SIMPLE_FILTER(p0, q0, fl) \
51 "vmov.i8 q15, #0x03 \n" \
52 "vqadd.s8 q15, q15, " #fl " \n" /* filter1 = filter + 3 */ \
53 "vshr.s8 q15, q15, #3 \n" /* filter1 >> 3 */ \
54 "vqadd.s8 " #p0 "," #p0 ", q15 \n" /* p0 += filter1 */ \
55 \
56 "vmov.i8 q15, #0x04 \n" \
57 "vqadd.s8 q15, q15, " #fl " \n" /* filter1 = filter + 4 */ \
58 "vshr.s8 q15, q15, #3 \n" /* filter2 >> 3 */ \
59 "vqsub.s8 " #q0 "," #q0 ", q15 \n" /* q0 -= filter2 */
60
61 // Applies filter on 2 pixels (p0 and q0)
62 #define DO_FILTER2(p1, p0, q0, q1, thresh) \
63 NEEDS_FILTER(p1, p0, q0, q1, thresh, q9) /* filter mask in q9 */ \
64 "vmov.i8 q10, #0x80 \n" /* sign bit */ \
65 FLIP_SIGN_BIT4(p1, p0, q0, q1, q10) /* convert to signed value */ \
66 GET_BASE_DELTA(p1, p0, q0, q1, q11) /* get filter level */ \
67 "vand q9, q9, q11 \n" /* apply filter mask */ \
68 DO_SIMPLE_FILTER(p0, q0, q9) /* apply filter */ \
69 FLIP_SIGN_BIT2(p0, q0, q10)
70
71 // Load/Store vertical edge
72 #define LOAD8x4(c1, c2, c3, c4, b1, b2, stride) \
73 "vld4.8 {" #c1"[0], " #c2"[0], " #c3"[0], " #c4"[0]}," #b1 "," #stride"\n" \
74 "vld4.8 {" #c1"[1], " #c2"[1], " #c3"[1], " #c4"[1]}," #b2 "," #stride"\n" \
75 "vld4.8 {" #c1"[2], " #c2"[2], " #c3"[2], " #c4"[2]}," #b1 "," #stride"\n" \
76 "vld4.8 {" #c1"[3], " #c2"[3], " #c3"[3], " #c4"[3]}," #b2 "," #stride"\n" \
77 "vld4.8 {" #c1"[4], " #c2"[4], " #c3"[4], " #c4"[4]}," #b1 "," #stride"\n" \
78 "vld4.8 {" #c1"[5], " #c2"[5], " #c3"[5], " #c4"[5]}," #b2 "," #stride"\n" \
79 "vld4.8 {" #c1"[6], " #c2"[6], " #c3"[6], " #c4"[6]}," #b1 "," #stride"\n" \
80 "vld4.8 {" #c1"[7], " #c2"[7], " #c3"[7], " #c4"[7]}," #b2 "," #stride"\n"
81
82 #define STORE8x2(c1, c2, p,stride) \
83 "vst2.8 {" #c1"[0], " #c2"[0]}," #p "," #stride " \n" \
84 "vst2.8 {" #c1"[1], " #c2"[1]}," #p "," #stride " \n" \
85 "vst2.8 {" #c1"[2], " #c2"[2]}," #p "," #stride " \n" \
86 "vst2.8 {" #c1"[3], " #c2"[3]}," #p "," #stride " \n" \
87 "vst2.8 {" #c1"[4], " #c2"[4]}," #p "," #stride " \n" \
88 "vst2.8 {" #c1"[5], " #c2"[5]}," #p "," #stride " \n" \
89 "vst2.8 {" #c1"[6], " #c2"[6]}," #p "," #stride " \n" \
90 "vst2.8 {" #c1"[7], " #c2"[7]}," #p "," #stride " \n"
91
92 //-----------------------------------------------------------------------------
93 // Simple In-loop filtering (Paragraph 15.2)
94
SimpleVFilter16NEON(uint8_t * p,int stride,int thresh)95 static void SimpleVFilter16NEON(uint8_t* p, int stride, int thresh) {
96 __asm__ volatile (
97 "sub %[p], %[p], %[stride], lsl #1 \n" // p -= 2 * stride
98
99 "vld1.u8 {q1}, [%[p]], %[stride] \n" // p1
100 "vld1.u8 {q2}, [%[p]], %[stride] \n" // p0
101 "vld1.u8 {q3}, [%[p]], %[stride] \n" // q0
102 "vld1.u8 {q4}, [%[p]] \n" // q1
103
104 DO_FILTER2(q1, q2, q3, q4, %[thresh])
105
106 "sub %[p], %[p], %[stride], lsl #1 \n" // p -= 2 * stride
107
108 "vst1.u8 {q2}, [%[p]], %[stride] \n" // store op0
109 "vst1.u8 {q3}, [%[p]] \n" // store oq0
110 : [p] "+r"(p)
111 : [stride] "r"(stride), [thresh] "r"(thresh)
112 : "memory", QRegs
113 );
114 }
115
SimpleHFilter16NEON(uint8_t * p,int stride,int thresh)116 static void SimpleHFilter16NEON(uint8_t* p, int stride, int thresh) {
117 __asm__ volatile (
118 "sub r4, %[p], #2 \n" // base1 = p - 2
119 "lsl r6, %[stride], #1 \n" // r6 = 2 * stride
120 "add r5, r4, %[stride] \n" // base2 = base1 + stride
121
122 LOAD8x4(d2, d3, d4, d5, [r4], [r5], r6)
123 LOAD8x4(d6, d7, d8, d9, [r4], [r5], r6)
124 "vswp d3, d6 \n" // p1:q1 p0:q3
125 "vswp d5, d8 \n" // q0:q2 q1:q4
126 "vswp q2, q3 \n" // p1:q1 p0:q2 q0:q3 q1:q4
127
128 DO_FILTER2(q1, q2, q3, q4, %[thresh])
129
130 "sub %[p], %[p], #1 \n" // p - 1
131
132 "vswp d5, d6 \n"
133 STORE8x2(d4, d5, [%[p]], %[stride])
134 STORE8x2(d6, d7, [%[p]], %[stride])
135
136 : [p] "+r"(p)
137 : [stride] "r"(stride), [thresh] "r"(thresh)
138 : "memory", "r4", "r5", "r6", QRegs
139 );
140 }
141
SimpleVFilter16iNEON(uint8_t * p,int stride,int thresh)142 static void SimpleVFilter16iNEON(uint8_t* p, int stride, int thresh) {
143 int k;
144 for (k = 3; k > 0; --k) {
145 p += 4 * stride;
146 SimpleVFilter16NEON(p, stride, thresh);
147 }
148 }
149
SimpleHFilter16iNEON(uint8_t * p,int stride,int thresh)150 static void SimpleHFilter16iNEON(uint8_t* p, int stride, int thresh) {
151 int k;
152 for (k = 3; k > 0; --k) {
153 p += 4;
154 SimpleHFilter16NEON(p, stride, thresh);
155 }
156 }
157
158 //-----------------------------------------------------------------------------
159 // Inverse transforms (Paragraph 14.4)
160
TransformOneNEON(const int16_t * in,uint8_t * dst)161 static void TransformOneNEON(const int16_t *in, uint8_t *dst) {
162 const int kBPS = BPS;
163 const int16_t constants[] = {20091, 17734, 0, 0};
164 /* kC1, kC2. Padded because vld1.16 loads 8 bytes
165 * Technically these are unsigned but vqdmulh is only available in signed.
166 * vqdmulh returns high half (effectively >> 16) but also doubles the value,
167 * changing the >> 16 to >> 15 and requiring an additional >> 1.
168 * We use this to our advantage with kC2. The canonical value is 35468.
169 * However, the high bit is set so treating it as signed will give incorrect
170 * results. We avoid this by down shifting by 1 here to clear the highest bit.
171 * Combined with the doubling effect of vqdmulh we get >> 16.
172 * This can not be applied to kC1 because the lowest bit is set. Down shifting
173 * the constant would reduce precision.
174 */
175
176 /* libwebp uses a trick to avoid some extra addition that libvpx does.
177 * Instead of:
178 * temp2 = ip[12] + ((ip[12] * cospi8sqrt2minus1) >> 16);
179 * libwebp adds 1 << 16 to cospi8sqrt2minus1 (kC1). However, this causes the
180 * same issue with kC1 and vqdmulh that we work around by down shifting kC2
181 */
182
183 /* Adapted from libvpx: vp8/common/arm/neon/shortidct4x4llm_neon.asm */
184 __asm__ volatile (
185 "vld1.16 {q1, q2}, [%[in]] \n"
186 "vld1.16 {d0}, [%[constants]] \n"
187
188 /* d2: in[0]
189 * d3: in[8]
190 * d4: in[4]
191 * d5: in[12]
192 */
193 "vswp d3, d4 \n"
194
195 /* q8 = {in[4], in[12]} * kC1 * 2 >> 16
196 * q9 = {in[4], in[12]} * kC2 >> 16
197 */
198 "vqdmulh.s16 q8, q2, d0[0] \n"
199 "vqdmulh.s16 q9, q2, d0[1] \n"
200
201 /* d22 = a = in[0] + in[8]
202 * d23 = b = in[0] - in[8]
203 */
204 "vqadd.s16 d22, d2, d3 \n"
205 "vqsub.s16 d23, d2, d3 \n"
206
207 /* The multiplication should be x * kC1 >> 16
208 * However, with vqdmulh we get x * kC1 * 2 >> 16
209 * (multiply, double, return high half)
210 * We avoided this in kC2 by pre-shifting the constant.
211 * q8 = in[4]/[12] * kC1 >> 16
212 */
213 "vshr.s16 q8, q8, #1 \n"
214
215 /* Add {in[4], in[12]} back after the multiplication. This is handled by
216 * adding 1 << 16 to kC1 in the libwebp C code.
217 */
218 "vqadd.s16 q8, q2, q8 \n"
219
220 /* d20 = c = in[4]*kC2 - in[12]*kC1
221 * d21 = d = in[4]*kC1 + in[12]*kC2
222 */
223 "vqsub.s16 d20, d18, d17 \n"
224 "vqadd.s16 d21, d19, d16 \n"
225
226 /* d2 = tmp[0] = a + d
227 * d3 = tmp[1] = b + c
228 * d4 = tmp[2] = b - c
229 * d5 = tmp[3] = a - d
230 */
231 "vqadd.s16 d2, d22, d21 \n"
232 "vqadd.s16 d3, d23, d20 \n"
233 "vqsub.s16 d4, d23, d20 \n"
234 "vqsub.s16 d5, d22, d21 \n"
235
236 "vzip.16 q1, q2 \n"
237 "vzip.16 q1, q2 \n"
238
239 "vswp d3, d4 \n"
240
241 /* q8 = {tmp[4], tmp[12]} * kC1 * 2 >> 16
242 * q9 = {tmp[4], tmp[12]} * kC2 >> 16
243 */
244 "vqdmulh.s16 q8, q2, d0[0] \n"
245 "vqdmulh.s16 q9, q2, d0[1] \n"
246
247 /* d22 = a = tmp[0] + tmp[8]
248 * d23 = b = tmp[0] - tmp[8]
249 */
250 "vqadd.s16 d22, d2, d3 \n"
251 "vqsub.s16 d23, d2, d3 \n"
252
253 /* See long winded explanations prior */
254 "vshr.s16 q8, q8, #1 \n"
255 "vqadd.s16 q8, q2, q8 \n"
256
257 /* d20 = c = in[4]*kC2 - in[12]*kC1
258 * d21 = d = in[4]*kC1 + in[12]*kC2
259 */
260 "vqsub.s16 d20, d18, d17 \n"
261 "vqadd.s16 d21, d19, d16 \n"
262
263 /* d2 = tmp[0] = a + d
264 * d3 = tmp[1] = b + c
265 * d4 = tmp[2] = b - c
266 * d5 = tmp[3] = a - d
267 */
268 "vqadd.s16 d2, d22, d21 \n"
269 "vqadd.s16 d3, d23, d20 \n"
270 "vqsub.s16 d4, d23, d20 \n"
271 "vqsub.s16 d5, d22, d21 \n"
272
273 "vld1.32 d6[0], [%[dst]], %[kBPS] \n"
274 "vld1.32 d6[1], [%[dst]], %[kBPS] \n"
275 "vld1.32 d7[0], [%[dst]], %[kBPS] \n"
276 "vld1.32 d7[1], [%[dst]], %[kBPS] \n"
277
278 "sub %[dst], %[dst], %[kBPS], lsl #2 \n"
279
280 /* (val) + 4 >> 3 */
281 "vrshr.s16 d2, d2, #3 \n"
282 "vrshr.s16 d3, d3, #3 \n"
283 "vrshr.s16 d4, d4, #3 \n"
284 "vrshr.s16 d5, d5, #3 \n"
285
286 "vzip.16 q1, q2 \n"
287 "vzip.16 q1, q2 \n"
288
289 /* Must accumulate before saturating */
290 "vmovl.u8 q8, d6 \n"
291 "vmovl.u8 q9, d7 \n"
292
293 "vqadd.s16 q1, q1, q8 \n"
294 "vqadd.s16 q2, q2, q9 \n"
295
296 "vqmovun.s16 d0, q1 \n"
297 "vqmovun.s16 d1, q2 \n"
298
299 "vst1.32 d0[0], [%[dst]], %[kBPS] \n"
300 "vst1.32 d0[1], [%[dst]], %[kBPS] \n"
301 "vst1.32 d1[0], [%[dst]], %[kBPS] \n"
302 "vst1.32 d1[1], [%[dst]] \n"
303
304 : [in] "+r"(in), [dst] "+r"(dst) /* modified registers */
305 : [kBPS] "r"(kBPS), [constants] "r"(constants) /* constants */
306 : "memory", "q0", "q1", "q2", "q8", "q9", "q10", "q11" /* clobbered */
307 );
308 }
309
TransformTwoNEON(const int16_t * in,uint8_t * dst,int do_two)310 static void TransformTwoNEON(const int16_t* in, uint8_t* dst, int do_two) {
311 TransformOneNEON(in, dst);
312 if (do_two) {
313 TransformOneNEON(in + 16, dst + 4);
314 }
315 }
316
TransformWHT(const int16_t * in,int16_t * out)317 static void TransformWHT(const int16_t* in, int16_t* out) {
318 const int kStep = 32; // The store is only incrementing the pointer as if we
319 // had stored a single byte.
320 __asm__ volatile (
321 // part 1
322 // load data into q0, q1
323 "vld1.16 {q0, q1}, [%[in]] \n"
324
325 "vaddl.s16 q2, d0, d3 \n" // a0 = in[0] + in[12]
326 "vaddl.s16 q3, d1, d2 \n" // a1 = in[4] + in[8]
327 "vsubl.s16 q4, d1, d2 \n" // a2 = in[4] - in[8]
328 "vsubl.s16 q5, d0, d3 \n" // a3 = in[0] - in[12]
329
330 "vadd.s32 q0, q2, q3 \n" // tmp[0] = a0 + a1
331 "vsub.s32 q2, q2, q3 \n" // tmp[8] = a0 - a1
332 "vadd.s32 q1, q5, q4 \n" // tmp[4] = a3 + a2
333 "vsub.s32 q3, q5, q4 \n" // tmp[12] = a3 - a2
334
335 // Transpose
336 // q0 = tmp[0, 4, 8, 12], q1 = tmp[2, 6, 10, 14]
337 // q2 = tmp[1, 5, 9, 13], q3 = tmp[3, 7, 11, 15]
338 "vswp d1, d4 \n" // vtrn.64 q0, q2
339 "vswp d3, d6 \n" // vtrn.64 q1, q3
340 "vtrn.32 q0, q1 \n"
341 "vtrn.32 q2, q3 \n"
342
343 "vmov.s32 q4, #3 \n" // dc = 3
344 "vadd.s32 q0, q0, q4 \n" // dc = tmp[0] + 3
345 "vadd.s32 q6, q0, q3 \n" // a0 = dc + tmp[3]
346 "vadd.s32 q7, q1, q2 \n" // a1 = tmp[1] + tmp[2]
347 "vsub.s32 q8, q1, q2 \n" // a2 = tmp[1] - tmp[2]
348 "vsub.s32 q9, q0, q3 \n" // a3 = dc - tmp[3]
349
350 "vadd.s32 q0, q6, q7 \n"
351 "vshrn.s32 d0, q0, #3 \n" // (a0 + a1) >> 3
352 "vadd.s32 q1, q9, q8 \n"
353 "vshrn.s32 d1, q1, #3 \n" // (a3 + a2) >> 3
354 "vsub.s32 q2, q6, q7 \n"
355 "vshrn.s32 d2, q2, #3 \n" // (a0 - a1) >> 3
356 "vsub.s32 q3, q9, q8 \n"
357 "vshrn.s32 d3, q3, #3 \n" // (a3 - a2) >> 3
358
359 // set the results to output
360 "vst1.16 d0[0], [%[out]], %[kStep] \n"
361 "vst1.16 d1[0], [%[out]], %[kStep] \n"
362 "vst1.16 d2[0], [%[out]], %[kStep] \n"
363 "vst1.16 d3[0], [%[out]], %[kStep] \n"
364 "vst1.16 d0[1], [%[out]], %[kStep] \n"
365 "vst1.16 d1[1], [%[out]], %[kStep] \n"
366 "vst1.16 d2[1], [%[out]], %[kStep] \n"
367 "vst1.16 d3[1], [%[out]], %[kStep] \n"
368 "vst1.16 d0[2], [%[out]], %[kStep] \n"
369 "vst1.16 d1[2], [%[out]], %[kStep] \n"
370 "vst1.16 d2[2], [%[out]], %[kStep] \n"
371 "vst1.16 d3[2], [%[out]], %[kStep] \n"
372 "vst1.16 d0[3], [%[out]], %[kStep] \n"
373 "vst1.16 d1[3], [%[out]], %[kStep] \n"
374 "vst1.16 d2[3], [%[out]], %[kStep] \n"
375 "vst1.16 d3[3], [%[out]], %[kStep] \n"
376
377 : [out] "+r"(out) // modified registers
378 : [in] "r"(in), [kStep] "r"(kStep) // constants
379 : "memory", "q0", "q1", "q2", "q3", "q4",
380 "q5", "q6", "q7", "q8", "q9" // clobbered
381 );
382 }
383
384 #endif // WEBP_USE_NEON
385
386 //------------------------------------------------------------------------------
387 // Entry point
388
389 extern void VP8DspInitNEON(void);
390
VP8DspInitNEON(void)391 void VP8DspInitNEON(void) {
392 #if defined(WEBP_USE_NEON)
393 VP8Transform = TransformTwoNEON;
394 VP8TransformWHT = TransformWHT;
395
396 VP8SimpleVFilter16 = SimpleVFilter16NEON;
397 VP8SimpleHFilter16 = SimpleHFilter16NEON;
398 VP8SimpleVFilter16i = SimpleVFilter16iNEON;
399 VP8SimpleHFilter16i = SimpleHFilter16iNEON;
400 #endif // WEBP_USE_NEON
401 }
402
403 #if defined(__cplusplus) || defined(c_plusplus)
404 } // extern "C"
405 #endif
406