1 /*
2 * Copyright (c) 2014 The WebM project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11 #include <arm_neon.h>
12 #include "./vpx_config.h"
13 #include "vpx_ports/arm.h"
14
15 #ifdef VPX_INCOMPATIBLE_GCC
write_2x4(unsigned char * dst,int pitch,const uint8x8x2_t result)16 static INLINE void write_2x4(unsigned char *dst, int pitch,
17 const uint8x8x2_t result) {
18 /*
19 * uint8x8x2_t result
20 00 01 02 03 | 04 05 06 07
21 10 11 12 13 | 14 15 16 17
22 ---
23 * after vtrn_u8
24 00 10 02 12 | 04 14 06 16
25 01 11 03 13 | 05 15 07 17
26 */
27 const uint8x8x2_t r01_u8 = vtrn_u8(result.val[0], result.val[1]);
28 const uint16x4_t x_0_4 = vreinterpret_u16_u8(r01_u8.val[0]);
29 const uint16x4_t x_1_5 = vreinterpret_u16_u8(r01_u8.val[1]);
30 vst1_lane_u16((uint16_t *)dst, x_0_4, 0);
31 dst += pitch;
32 vst1_lane_u16((uint16_t *)dst, x_1_5, 0);
33 dst += pitch;
34 vst1_lane_u16((uint16_t *)dst, x_0_4, 1);
35 dst += pitch;
36 vst1_lane_u16((uint16_t *)dst, x_1_5, 1);
37 dst += pitch;
38 vst1_lane_u16((uint16_t *)dst, x_0_4, 2);
39 dst += pitch;
40 vst1_lane_u16((uint16_t *)dst, x_1_5, 2);
41 dst += pitch;
42 vst1_lane_u16((uint16_t *)dst, x_0_4, 3);
43 dst += pitch;
44 vst1_lane_u16((uint16_t *)dst, x_1_5, 3);
45 }
46
write_2x8(unsigned char * dst,int pitch,const uint8x8x2_t result,const uint8x8x2_t result2)47 static INLINE void write_2x8(unsigned char *dst, int pitch,
48 const uint8x8x2_t result,
49 const uint8x8x2_t result2) {
50 write_2x4(dst, pitch, result);
51 dst += pitch * 8;
52 write_2x4(dst, pitch, result2);
53 }
54 #else
write_2x8(unsigned char * dst,int pitch,const uint8x8x2_t result,const uint8x8x2_t result2)55 static INLINE void write_2x8(unsigned char *dst, int pitch,
56 const uint8x8x2_t result,
57 const uint8x8x2_t result2) {
58 vst2_lane_u8(dst, result, 0);
59 dst += pitch;
60 vst2_lane_u8(dst, result, 1);
61 dst += pitch;
62 vst2_lane_u8(dst, result, 2);
63 dst += pitch;
64 vst2_lane_u8(dst, result, 3);
65 dst += pitch;
66 vst2_lane_u8(dst, result, 4);
67 dst += pitch;
68 vst2_lane_u8(dst, result, 5);
69 dst += pitch;
70 vst2_lane_u8(dst, result, 6);
71 dst += pitch;
72 vst2_lane_u8(dst, result, 7);
73 dst += pitch;
74
75 vst2_lane_u8(dst, result2, 0);
76 dst += pitch;
77 vst2_lane_u8(dst, result2, 1);
78 dst += pitch;
79 vst2_lane_u8(dst, result2, 2);
80 dst += pitch;
81 vst2_lane_u8(dst, result2, 3);
82 dst += pitch;
83 vst2_lane_u8(dst, result2, 4);
84 dst += pitch;
85 vst2_lane_u8(dst, result2, 5);
86 dst += pitch;
87 vst2_lane_u8(dst, result2, 6);
88 dst += pitch;
89 vst2_lane_u8(dst, result2, 7);
90 }
91 #endif // VPX_INCOMPATIBLE_GCC
92
93 #ifdef VPX_INCOMPATIBLE_GCC
read_4x8(unsigned char * src,int pitch)94 static INLINE uint8x8x4_t read_4x8(unsigned char *src, int pitch) {
95 uint8x8x4_t x;
96 const uint8x8_t a = vld1_u8(src);
97 const uint8x8_t b = vld1_u8(src + pitch * 1);
98 const uint8x8_t c = vld1_u8(src + pitch * 2);
99 const uint8x8_t d = vld1_u8(src + pitch * 3);
100 const uint8x8_t e = vld1_u8(src + pitch * 4);
101 const uint8x8_t f = vld1_u8(src + pitch * 5);
102 const uint8x8_t g = vld1_u8(src + pitch * 6);
103 const uint8x8_t h = vld1_u8(src + pitch * 7);
104 const uint32x2x2_t r04_u32 =
105 vtrn_u32(vreinterpret_u32_u8(a), vreinterpret_u32_u8(e));
106 const uint32x2x2_t r15_u32 =
107 vtrn_u32(vreinterpret_u32_u8(b), vreinterpret_u32_u8(f));
108 const uint32x2x2_t r26_u32 =
109 vtrn_u32(vreinterpret_u32_u8(c), vreinterpret_u32_u8(g));
110 const uint32x2x2_t r37_u32 =
111 vtrn_u32(vreinterpret_u32_u8(d), vreinterpret_u32_u8(h));
112 const uint16x4x2_t r02_u16 = vtrn_u16(vreinterpret_u16_u32(r04_u32.val[0]),
113 vreinterpret_u16_u32(r26_u32.val[0]));
114 const uint16x4x2_t r13_u16 = vtrn_u16(vreinterpret_u16_u32(r15_u32.val[0]),
115 vreinterpret_u16_u32(r37_u32.val[0]));
116 const uint8x8x2_t r01_u8 = vtrn_u8(vreinterpret_u8_u16(r02_u16.val[0]),
117 vreinterpret_u8_u16(r13_u16.val[0]));
118 const uint8x8x2_t r23_u8 = vtrn_u8(vreinterpret_u8_u16(r02_u16.val[1]),
119 vreinterpret_u8_u16(r13_u16.val[1]));
120 /*
121 * after vtrn_u32
122 00 01 02 03 | 40 41 42 43
123 10 11 12 13 | 50 51 52 53
124 20 21 22 23 | 60 61 62 63
125 30 31 32 33 | 70 71 72 73
126 ---
127 * after vtrn_u16
128 00 01 20 21 | 40 41 60 61
129 02 03 22 23 | 42 43 62 63
130 10 11 30 31 | 50 51 70 71
131 12 13 32 33 | 52 52 72 73
132
133 00 01 20 21 | 40 41 60 61
134 10 11 30 31 | 50 51 70 71
135 02 03 22 23 | 42 43 62 63
136 12 13 32 33 | 52 52 72 73
137 ---
138 * after vtrn_u8
139 00 10 20 30 | 40 50 60 70
140 01 11 21 31 | 41 51 61 71
141 02 12 22 32 | 42 52 62 72
142 03 13 23 33 | 43 53 63 73
143 */
144 x.val[0] = r01_u8.val[0];
145 x.val[1] = r01_u8.val[1];
146 x.val[2] = r23_u8.val[0];
147 x.val[3] = r23_u8.val[1];
148
149 return x;
150 }
151 #else
read_4x8(unsigned char * src,int pitch)152 static INLINE uint8x8x4_t read_4x8(unsigned char *src, int pitch) {
153 uint8x8x4_t x;
154 x.val[0] = x.val[1] = x.val[2] = x.val[3] = vdup_n_u8(0);
155 x = vld4_lane_u8(src, x, 0);
156 src += pitch;
157 x = vld4_lane_u8(src, x, 1);
158 src += pitch;
159 x = vld4_lane_u8(src, x, 2);
160 src += pitch;
161 x = vld4_lane_u8(src, x, 3);
162 src += pitch;
163 x = vld4_lane_u8(src, x, 4);
164 src += pitch;
165 x = vld4_lane_u8(src, x, 5);
166 src += pitch;
167 x = vld4_lane_u8(src, x, 6);
168 src += pitch;
169 x = vld4_lane_u8(src, x, 7);
170 return x;
171 }
172 #endif // VPX_INCOMPATIBLE_GCC
173
vp8_loop_filter_simple_vertical_edge_neon(unsigned char * s,int p,const unsigned char * blimit)174 static INLINE void vp8_loop_filter_simple_vertical_edge_neon(
175 unsigned char *s, int p, const unsigned char *blimit) {
176 unsigned char *src1;
177 uint8x16_t qblimit, q0u8;
178 uint8x16_t q3u8, q4u8, q5u8, q6u8, q7u8, q11u8, q12u8, q14u8, q15u8;
179 int16x8_t q2s16, q13s16, q11s16;
180 int8x8_t d28s8, d29s8;
181 int8x16_t q2s8, q3s8, q10s8, q11s8, q14s8;
182 uint8x8x4_t d0u8x4; // d6, d7, d8, d9
183 uint8x8x4_t d1u8x4; // d10, d11, d12, d13
184 uint8x8x2_t d2u8x2; // d12, d13
185 uint8x8x2_t d3u8x2; // d14, d15
186
187 qblimit = vdupq_n_u8(*blimit);
188
189 src1 = s - 2;
190 d0u8x4 = read_4x8(src1, p);
191 src1 += p * 8;
192 d1u8x4 = read_4x8(src1, p);
193
194 q3u8 = vcombine_u8(d0u8x4.val[0], d1u8x4.val[0]); // d6 d10
195 q4u8 = vcombine_u8(d0u8x4.val[2], d1u8x4.val[2]); // d8 d12
196 q5u8 = vcombine_u8(d0u8x4.val[1], d1u8x4.val[1]); // d7 d11
197 q6u8 = vcombine_u8(d0u8x4.val[3], d1u8x4.val[3]); // d9 d13
198
199 q15u8 = vabdq_u8(q5u8, q4u8);
200 q14u8 = vabdq_u8(q3u8, q6u8);
201
202 q15u8 = vqaddq_u8(q15u8, q15u8);
203 q14u8 = vshrq_n_u8(q14u8, 1);
204 q0u8 = vdupq_n_u8(0x80);
205 q11s16 = vdupq_n_s16(3);
206 q15u8 = vqaddq_u8(q15u8, q14u8);
207
208 q3u8 = veorq_u8(q3u8, q0u8);
209 q4u8 = veorq_u8(q4u8, q0u8);
210 q5u8 = veorq_u8(q5u8, q0u8);
211 q6u8 = veorq_u8(q6u8, q0u8);
212
213 q15u8 = vcgeq_u8(qblimit, q15u8);
214
215 q2s16 = vsubl_s8(vget_low_s8(vreinterpretq_s8_u8(q4u8)),
216 vget_low_s8(vreinterpretq_s8_u8(q5u8)));
217 q13s16 = vsubl_s8(vget_high_s8(vreinterpretq_s8_u8(q4u8)),
218 vget_high_s8(vreinterpretq_s8_u8(q5u8)));
219
220 q14s8 = vqsubq_s8(vreinterpretq_s8_u8(q3u8), vreinterpretq_s8_u8(q6u8));
221
222 q2s16 = vmulq_s16(q2s16, q11s16);
223 q13s16 = vmulq_s16(q13s16, q11s16);
224
225 q11u8 = vdupq_n_u8(3);
226 q12u8 = vdupq_n_u8(4);
227
228 q2s16 = vaddw_s8(q2s16, vget_low_s8(q14s8));
229 q13s16 = vaddw_s8(q13s16, vget_high_s8(q14s8));
230
231 d28s8 = vqmovn_s16(q2s16);
232 d29s8 = vqmovn_s16(q13s16);
233 q14s8 = vcombine_s8(d28s8, d29s8);
234
235 q14s8 = vandq_s8(q14s8, vreinterpretq_s8_u8(q15u8));
236
237 q2s8 = vqaddq_s8(q14s8, vreinterpretq_s8_u8(q11u8));
238 q3s8 = vqaddq_s8(q14s8, vreinterpretq_s8_u8(q12u8));
239 q2s8 = vshrq_n_s8(q2s8, 3);
240 q14s8 = vshrq_n_s8(q3s8, 3);
241
242 q11s8 = vqaddq_s8(vreinterpretq_s8_u8(q5u8), q2s8);
243 q10s8 = vqsubq_s8(vreinterpretq_s8_u8(q4u8), q14s8);
244
245 q6u8 = veorq_u8(vreinterpretq_u8_s8(q11s8), q0u8);
246 q7u8 = veorq_u8(vreinterpretq_u8_s8(q10s8), q0u8);
247
248 d2u8x2.val[0] = vget_low_u8(q6u8); // d12
249 d2u8x2.val[1] = vget_low_u8(q7u8); // d14
250 d3u8x2.val[0] = vget_high_u8(q6u8); // d13
251 d3u8x2.val[1] = vget_high_u8(q7u8); // d15
252
253 src1 = s - 1;
254 write_2x8(src1, p, d2u8x2, d3u8x2);
255 }
256
vp8_loop_filter_bvs_neon(unsigned char * y_ptr,int y_stride,const unsigned char * blimit)257 void vp8_loop_filter_bvs_neon(unsigned char *y_ptr, int y_stride,
258 const unsigned char *blimit) {
259 y_ptr += 4;
260 vp8_loop_filter_simple_vertical_edge_neon(y_ptr, y_stride, blimit);
261 y_ptr += 4;
262 vp8_loop_filter_simple_vertical_edge_neon(y_ptr, y_stride, blimit);
263 y_ptr += 4;
264 vp8_loop_filter_simple_vertical_edge_neon(y_ptr, y_stride, blimit);
265 return;
266 }
267
vp8_loop_filter_mbvs_neon(unsigned char * y_ptr,int y_stride,const unsigned char * blimit)268 void vp8_loop_filter_mbvs_neon(unsigned char *y_ptr, int y_stride,
269 const unsigned char *blimit) {
270 vp8_loop_filter_simple_vertical_edge_neon(y_ptr, y_stride, blimit);
271 return;
272 }
273