1 // Copyright 2012 Google Inc. All Rights Reserved.
2 //
3 // This code is licensed under the same terms as WebM:
4 // Software License Agreement: http://www.webmproject.org/license/software/
5 // Additional IP Rights Grant: http://www.webmproject.org/license/additional/
6 // -----------------------------------------------------------------------------
7 //
8 // ARM NEON version of dsp functions and loop filtering.
9 //
10 // Authors: Somnath Banerjee (somnath@google.com)
11 // Johann Koenig (johannkoenig@google.com)
12
13 #include "./dsp.h"
14
15 #if defined(WEBP_USE_NEON)
16
17 #include "../dec/vp8i.h"
18
19 #if defined(__cplusplus) || defined(c_plusplus)
20 extern "C" {
21 #endif
22
23 #define QRegs "q0", "q1", "q2", "q3", "q4", "q5", "q6", "q7", \
24 "q8", "q9", "q10", "q11", "q12", "q13", "q14", "q15"
25
26 #define FLIP_SIGN_BIT2(a, b, s) \
27 "veor " #a "," #a "," #s " \n" \
28 "veor " #b "," #b "," #s " \n" \
29
30 #define FLIP_SIGN_BIT4(a, b, c, d, s) \
31 FLIP_SIGN_BIT2(a, b, s) \
32 FLIP_SIGN_BIT2(c, d, s) \
33
34 #define NEEDS_FILTER(p1, p0, q0, q1, thresh, mask) \
35 "vabd.u8 q15," #p0 "," #q0 " \n" /* abs(p0 - q0) */ \
36 "vabd.u8 q14," #p1 "," #q1 " \n" /* abs(p1 - q1) */ \
37 "vqadd.u8 q15, q15, q15 \n" /* abs(p0 - q0) * 2 */ \
38 "vshr.u8 q14, q14, #1 \n" /* abs(p1 - q1) / 2 */ \
39 "vqadd.u8 q15, q15, q14 \n" /* abs(p0 - q0) * 2 + abs(p1 - q1) / 2 */ \
40 "vdup.8 q14, " #thresh " \n" \
41 "vcge.u8 " #mask ", q14, q15 \n" /* mask <= thresh */
42
43 #define GET_BASE_DELTA(p1, p0, q0, q1, o) \
44 "vqsub.s8 q15," #q0 "," #p0 " \n" /* (q0 - p0) */ \
45 "vqsub.s8 " #o "," #p1 "," #q1 " \n" /* (p1 - q1) */ \
46 "vqadd.s8 " #o "," #o ", q15 \n" /* (p1 - q1) + 1 * (p0 - q0) */ \
47 "vqadd.s8 " #o "," #o ", q15 \n" /* (p1 - q1) + 2 * (p0 - q0) */ \
48 "vqadd.s8 " #o "," #o ", q15 \n" /* (p1 - q1) + 3 * (p0 - q0) */
49
50 #define DO_SIMPLE_FILTER(p0, q0, fl) \
51 "vmov.i8 q15, #0x03 \n" \
52 "vqadd.s8 q15, q15, " #fl " \n" /* filter1 = filter + 3 */ \
53 "vshr.s8 q15, q15, #3 \n" /* filter1 >> 3 */ \
54 "vqadd.s8 " #p0 "," #p0 ", q15 \n" /* p0 += filter1 */ \
55 \
56 "vmov.i8 q15, #0x04 \n" \
57 "vqadd.s8 q15, q15, " #fl " \n" /* filter1 = filter + 4 */ \
58 "vshr.s8 q15, q15, #3 \n" /* filter2 >> 3 */ \
59 "vqsub.s8 " #q0 "," #q0 ", q15 \n" /* q0 -= filter2 */
60
61 // Applies filter on 2 pixels (p0 and q0)
62 #define DO_FILTER2(p1, p0, q0, q1, thresh) \
63 NEEDS_FILTER(p1, p0, q0, q1, thresh, q9) /* filter mask in q9 */ \
64 "vmov.i8 q10, #0x80 \n" /* sign bit */ \
65 FLIP_SIGN_BIT4(p1, p0, q0, q1, q10) /* convert to signed value */ \
66 GET_BASE_DELTA(p1, p0, q0, q1, q11) /* get filter level */ \
67 "vand q9, q9, q11 \n" /* apply filter mask */ \
68 DO_SIMPLE_FILTER(p0, q0, q9) /* apply filter */ \
69 FLIP_SIGN_BIT2(p0, q0, q10)
70
71 // Load/Store vertical edge
72 #define LOAD8x4(c1, c2, c3, c4, b1, b2, stride) \
73 "vld4.8 {" #c1"[0], " #c2"[0], " #c3"[0], " #c4"[0]}," #b1 "," #stride"\n" \
74 "vld4.8 {" #c1"[1], " #c2"[1], " #c3"[1], " #c4"[1]}," #b2 "," #stride"\n" \
75 "vld4.8 {" #c1"[2], " #c2"[2], " #c3"[2], " #c4"[2]}," #b1 "," #stride"\n" \
76 "vld4.8 {" #c1"[3], " #c2"[3], " #c3"[3], " #c4"[3]}," #b2 "," #stride"\n" \
77 "vld4.8 {" #c1"[4], " #c2"[4], " #c3"[4], " #c4"[4]}," #b1 "," #stride"\n" \
78 "vld4.8 {" #c1"[5], " #c2"[5], " #c3"[5], " #c4"[5]}," #b2 "," #stride"\n" \
79 "vld4.8 {" #c1"[6], " #c2"[6], " #c3"[6], " #c4"[6]}," #b1 "," #stride"\n" \
80 "vld4.8 {" #c1"[7], " #c2"[7], " #c3"[7], " #c4"[7]}," #b2 "," #stride"\n"
81
82 #define STORE8x2(c1, c2, p,stride) \
83 "vst2.8 {" #c1"[0], " #c2"[0]}," #p "," #stride " \n" \
84 "vst2.8 {" #c1"[1], " #c2"[1]}," #p "," #stride " \n" \
85 "vst2.8 {" #c1"[2], " #c2"[2]}," #p "," #stride " \n" \
86 "vst2.8 {" #c1"[3], " #c2"[3]}," #p "," #stride " \n" \
87 "vst2.8 {" #c1"[4], " #c2"[4]}," #p "," #stride " \n" \
88 "vst2.8 {" #c1"[5], " #c2"[5]}," #p "," #stride " \n" \
89 "vst2.8 {" #c1"[6], " #c2"[6]}," #p "," #stride " \n" \
90 "vst2.8 {" #c1"[7], " #c2"[7]}," #p "," #stride " \n"
91
92 //-----------------------------------------------------------------------------
93 // Simple In-loop filtering (Paragraph 15.2)
94
SimpleVFilter16NEON(uint8_t * p,int stride,int thresh)95 static void SimpleVFilter16NEON(uint8_t* p, int stride, int thresh) {
96 __asm__ volatile (
97 "sub %[p], %[p], %[stride], lsl #1 \n" // p -= 2 * stride
98
99 "vld1.u8 {q1}, [%[p]], %[stride] \n" // p1
100 "vld1.u8 {q2}, [%[p]], %[stride] \n" // p0
101 "vld1.u8 {q3}, [%[p]], %[stride] \n" // q0
102 "vld1.u8 {q4}, [%[p]] \n" // q1
103
104 DO_FILTER2(q1, q2, q3, q4, %[thresh])
105
106 "sub %[p], %[p], %[stride], lsl #1 \n" // p -= 2 * stride
107
108 "vst1.u8 {q2}, [%[p]], %[stride] \n" // store op0
109 "vst1.u8 {q3}, [%[p]] \n" // store oq0
110 : [p] "+r"(p)
111 : [stride] "r"(stride), [thresh] "r"(thresh)
112 : "memory", QRegs
113 );
114 }
115
SimpleHFilter16NEON(uint8_t * p,int stride,int thresh)116 static void SimpleHFilter16NEON(uint8_t* p, int stride, int thresh) {
117 __asm__ volatile (
118 "sub r4, %[p], #2 \n" // base1 = p - 2
119 "lsl r6, %[stride], #1 \n" // r6 = 2 * stride
120 "add r5, r4, %[stride] \n" // base2 = base1 + stride
121
122 LOAD8x4(d2, d3, d4, d5, [r4], [r5], r6)
123 LOAD8x4(d6, d7, d8, d9, [r4], [r5], r6)
124 "vswp d3, d6 \n" // p1:q1 p0:q3
125 "vswp d5, d8 \n" // q0:q2 q1:q4
126 "vswp q2, q3 \n" // p1:q1 p0:q2 q0:q3 q1:q4
127
128 DO_FILTER2(q1, q2, q3, q4, %[thresh])
129
130 "sub %[p], %[p], #1 \n" // p - 1
131
132 "vswp d5, d6 \n"
133 STORE8x2(d4, d5, [%[p]], %[stride])
134 STORE8x2(d6, d7, [%[p]], %[stride])
135
136 : [p] "+r"(p)
137 : [stride] "r"(stride), [thresh] "r"(thresh)
138 : "memory", "r4", "r5", "r6", QRegs
139 );
140 }
141
SimpleVFilter16iNEON(uint8_t * p,int stride,int thresh)142 static void SimpleVFilter16iNEON(uint8_t* p, int stride, int thresh) {
143 int k;
144 for (k = 3; k > 0; --k) {
145 p += 4 * stride;
146 SimpleVFilter16NEON(p, stride, thresh);
147 }
148 }
149
SimpleHFilter16iNEON(uint8_t * p,int stride,int thresh)150 static void SimpleHFilter16iNEON(uint8_t* p, int stride, int thresh) {
151 int k;
152 for (k = 3; k > 0; --k) {
153 p += 4;
154 SimpleHFilter16NEON(p, stride, thresh);
155 }
156 }
157
TransformOneNEON(const int16_t * in,uint8_t * dst)158 static void TransformOneNEON(const int16_t *in, uint8_t *dst) {
159 const int kBPS = BPS;
160 const int16_t constants[] = {20091, 17734, 0, 0};
161 /* kC1, kC2. Padded because vld1.16 loads 8 bytes
162 * Technically these are unsigned but vqdmulh is only available in signed.
163 * vqdmulh returns high half (effectively >> 16) but also doubles the value,
164 * changing the >> 16 to >> 15 and requiring an additional >> 1.
165 * We use this to our advantage with kC2. The canonical value is 35468.
166 * However, the high bit is set so treating it as signed will give incorrect
167 * results. We avoid this by down shifting by 1 here to clear the highest bit.
168 * Combined with the doubling effect of vqdmulh we get >> 16.
169 * This can not be applied to kC1 because the lowest bit is set. Down shifting
170 * the constant would reduce precision.
171 */
172
173 /* libwebp uses a trick to avoid some extra addition that libvpx does.
174 * Instead of:
175 * temp2 = ip[12] + ((ip[12] * cospi8sqrt2minus1) >> 16);
176 * libwebp adds 1 << 16 to cospi8sqrt2minus1 (kC1). However, this causes the
177 * same issue with kC1 and vqdmulh that we work around by down shifting kC2
178 */
179
180 /* Adapted from libvpx: vp8/common/arm/neon/shortidct4x4llm_neon.asm */
181 __asm__ volatile (
182 "vld1.16 {q1, q2}, [%[in]] \n"
183 "vld1.16 {d0}, [%[constants]] \n"
184
185 /* d2: in[0]
186 * d3: in[8]
187 * d4: in[4]
188 * d5: in[12]
189 */
190 "vswp d3, d4 \n"
191
192 /* q8 = {in[4], in[12]} * kC1 * 2 >> 16
193 * q9 = {in[4], in[12]} * kC2 >> 16
194 */
195 "vqdmulh.s16 q8, q2, d0[0] \n"
196 "vqdmulh.s16 q9, q2, d0[1] \n"
197
198 /* d22 = a = in[0] + in[8]
199 * d23 = b = in[0] - in[8]
200 */
201 "vqadd.s16 d22, d2, d3 \n"
202 "vqsub.s16 d23, d2, d3 \n"
203
204 /* The multiplication should be x * kC1 >> 16
205 * However, with vqdmulh we get x * kC1 * 2 >> 16
206 * (multiply, double, return high half)
207 * We avoided this in kC2 by pre-shifting the constant.
208 * q8 = in[4]/[12] * kC1 >> 16
209 */
210 "vshr.s16 q8, q8, #1 \n"
211
212 /* Add {in[4], in[12]} back after the multiplication. This is handled by
213 * adding 1 << 16 to kC1 in the libwebp C code.
214 */
215 "vqadd.s16 q8, q2, q8 \n"
216
217 /* d20 = c = in[4]*kC2 - in[12]*kC1
218 * d21 = d = in[4]*kC1 + in[12]*kC2
219 */
220 "vqsub.s16 d20, d18, d17 \n"
221 "vqadd.s16 d21, d19, d16 \n"
222
223 /* d2 = tmp[0] = a + d
224 * d3 = tmp[1] = b + c
225 * d4 = tmp[2] = b - c
226 * d5 = tmp[3] = a - d
227 */
228 "vqadd.s16 d2, d22, d21 \n"
229 "vqadd.s16 d3, d23, d20 \n"
230 "vqsub.s16 d4, d23, d20 \n"
231 "vqsub.s16 d5, d22, d21 \n"
232
233 "vzip.16 q1, q2 \n"
234 "vzip.16 q1, q2 \n"
235
236 "vswp d3, d4 \n"
237
238 /* q8 = {tmp[4], tmp[12]} * kC1 * 2 >> 16
239 * q9 = {tmp[4], tmp[12]} * kC2 >> 16
240 */
241 "vqdmulh.s16 q8, q2, d0[0] \n"
242 "vqdmulh.s16 q9, q2, d0[1] \n"
243
244 /* d22 = a = tmp[0] + tmp[8]
245 * d23 = b = tmp[0] - tmp[8]
246 */
247 "vqadd.s16 d22, d2, d3 \n"
248 "vqsub.s16 d23, d2, d3 \n"
249
250 /* See long winded explanations prior */
251 "vshr.s16 q8, q8, #1 \n"
252 "vqadd.s16 q8, q2, q8 \n"
253
254 /* d20 = c = in[4]*kC2 - in[12]*kC1
255 * d21 = d = in[4]*kC1 + in[12]*kC2
256 */
257 "vqsub.s16 d20, d18, d17 \n"
258 "vqadd.s16 d21, d19, d16 \n"
259
260 /* d2 = tmp[0] = a + d
261 * d3 = tmp[1] = b + c
262 * d4 = tmp[2] = b - c
263 * d5 = tmp[3] = a - d
264 */
265 "vqadd.s16 d2, d22, d21 \n"
266 "vqadd.s16 d3, d23, d20 \n"
267 "vqsub.s16 d4, d23, d20 \n"
268 "vqsub.s16 d5, d22, d21 \n"
269
270 "vld1.32 d6[0], [%[dst]], %[kBPS] \n"
271 "vld1.32 d6[1], [%[dst]], %[kBPS] \n"
272 "vld1.32 d7[0], [%[dst]], %[kBPS] \n"
273 "vld1.32 d7[1], [%[dst]], %[kBPS] \n"
274
275 "sub %[dst], %[dst], %[kBPS], lsl #2 \n"
276
277 /* (val) + 4 >> 3 */
278 "vrshr.s16 d2, d2, #3 \n"
279 "vrshr.s16 d3, d3, #3 \n"
280 "vrshr.s16 d4, d4, #3 \n"
281 "vrshr.s16 d5, d5, #3 \n"
282
283 "vzip.16 q1, q2 \n"
284 "vzip.16 q1, q2 \n"
285
286 /* Must accumulate before saturating */
287 "vmovl.u8 q8, d6 \n"
288 "vmovl.u8 q9, d7 \n"
289
290 "vqadd.s16 q1, q1, q8 \n"
291 "vqadd.s16 q2, q2, q9 \n"
292
293 "vqmovun.s16 d0, q1 \n"
294 "vqmovun.s16 d1, q2 \n"
295
296 "vst1.32 d0[0], [%[dst]], %[kBPS] \n"
297 "vst1.32 d0[1], [%[dst]], %[kBPS] \n"
298 "vst1.32 d1[0], [%[dst]], %[kBPS] \n"
299 "vst1.32 d1[1], [%[dst]] \n"
300
301 : [in] "+r"(in), [dst] "+r"(dst) /* modified registers */
302 : [kBPS] "r"(kBPS), [constants] "r"(constants) /* constants */
303 : "memory", "q0", "q1", "q2", "q8", "q9", "q10", "q11" /* clobbered */
304 );
305 }
306
TransformTwoNEON(const int16_t * in,uint8_t * dst,int do_two)307 static void TransformTwoNEON(const int16_t* in, uint8_t* dst, int do_two) {
308 TransformOneNEON(in, dst);
309 if (do_two) {
310 TransformOneNEON(in + 16, dst + 4);
311 }
312 }
313
314 extern void VP8DspInitNEON(void);
315
VP8DspInitNEON(void)316 void VP8DspInitNEON(void) {
317 VP8Transform = TransformTwoNEON;
318
319 VP8SimpleVFilter16 = SimpleVFilter16NEON;
320 VP8SimpleHFilter16 = SimpleHFilter16NEON;
321 VP8SimpleVFilter16i = SimpleVFilter16iNEON;
322 VP8SimpleHFilter16i = SimpleHFilter16iNEON;
323 }
324
325 #if defined(__cplusplus) || defined(c_plusplus)
326 } // extern "C"
327 #endif
328
329 #endif // WEBP_USE_NEON
330