1 /*
2 * Copyright (c) 2014 The WebM project authors. All Rights Reserved.
3 * Copyright (c) 2023, Alliance for Open Media. All rights reserved
4 *
5 * This source code is subject to the terms of the BSD 2 Clause License and
6 * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
7 * was not distributed with this source code in the LICENSE file, you can
8 * obtain it at www.aomedia.org/license/software. If the Alliance for Open
9 * Media Patent License 1.0 was not distributed with this source code in the
10 * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
11 */
12
13 #include <arm_neon.h>
14 #include <assert.h>
15
16 #include "config/aom_config.h"
17 #include "config/aom_dsp_rtcd.h"
18
19 #include "aom/aom_integer.h"
20 #include "aom_dsp/aom_dsp_common.h"
21 #include "aom_dsp/aom_filter.h"
22 #include "aom_dsp/arm/mem_neon.h"
23 #include "aom_dsp/arm/transpose_neon.h"
24 #include "aom_ports/mem.h"
25
highbd_convolve8_4_s32(const int16x4_t s0,const int16x4_t s1,const int16x4_t s2,const int16x4_t s3,const int16x4_t s4,const int16x4_t s5,const int16x4_t s6,const int16x4_t s7,const int16x8_t y_filter)26 static INLINE int32x4_t highbd_convolve8_4_s32(
27 const int16x4_t s0, const int16x4_t s1, const int16x4_t s2,
28 const int16x4_t s3, const int16x4_t s4, const int16x4_t s5,
29 const int16x4_t s6, const int16x4_t s7, const int16x8_t y_filter) {
30 const int16x4_t y_filter_lo = vget_low_s16(y_filter);
31 const int16x4_t y_filter_hi = vget_high_s16(y_filter);
32
33 int32x4_t sum = vmull_lane_s16(s0, y_filter_lo, 0);
34 sum = vmlal_lane_s16(sum, s1, y_filter_lo, 1);
35 sum = vmlal_lane_s16(sum, s2, y_filter_lo, 2);
36 sum = vmlal_lane_s16(sum, s3, y_filter_lo, 3);
37 sum = vmlal_lane_s16(sum, s4, y_filter_hi, 0);
38 sum = vmlal_lane_s16(sum, s5, y_filter_hi, 1);
39 sum = vmlal_lane_s16(sum, s6, y_filter_hi, 2);
40 sum = vmlal_lane_s16(sum, s7, y_filter_hi, 3);
41
42 return sum;
43 }
44
highbd_convolve8_4_s32_s16(const int16x4_t s0,const int16x4_t s1,const int16x4_t s2,const int16x4_t s3,const int16x4_t s4,const int16x4_t s5,const int16x4_t s6,const int16x4_t s7,const int16x8_t y_filter)45 static INLINE uint16x4_t highbd_convolve8_4_s32_s16(
46 const int16x4_t s0, const int16x4_t s1, const int16x4_t s2,
47 const int16x4_t s3, const int16x4_t s4, const int16x4_t s5,
48 const int16x4_t s6, const int16x4_t s7, const int16x8_t y_filter) {
49 int32x4_t sum =
50 highbd_convolve8_4_s32(s0, s1, s2, s3, s4, s5, s6, s7, y_filter);
51
52 return vqrshrun_n_s32(sum, FILTER_BITS);
53 }
54
highbd_convolve8_horiz4_s32(const int16x8_t s0,const int16x8_t s1,const int16x8_t x_filter_0_7)55 static INLINE int32x4_t highbd_convolve8_horiz4_s32(
56 const int16x8_t s0, const int16x8_t s1, const int16x8_t x_filter_0_7) {
57 const int16x8_t s2 = vextq_s16(s0, s1, 1);
58 const int16x8_t s3 = vextq_s16(s0, s1, 2);
59 const int16x8_t s4 = vextq_s16(s0, s1, 3);
60 const int16x4_t s0_lo = vget_low_s16(s0);
61 const int16x4_t s1_lo = vget_low_s16(s2);
62 const int16x4_t s2_lo = vget_low_s16(s3);
63 const int16x4_t s3_lo = vget_low_s16(s4);
64 const int16x4_t s4_lo = vget_high_s16(s0);
65 const int16x4_t s5_lo = vget_high_s16(s2);
66 const int16x4_t s6_lo = vget_high_s16(s3);
67 const int16x4_t s7_lo = vget_high_s16(s4);
68
69 return highbd_convolve8_4_s32(s0_lo, s1_lo, s2_lo, s3_lo, s4_lo, s5_lo, s6_lo,
70 s7_lo, x_filter_0_7);
71 }
72
highbd_convolve8_horiz4_s32_s16(const int16x8_t s0,const int16x8_t s1,const int16x8_t x_filter_0_7)73 static INLINE uint16x4_t highbd_convolve8_horiz4_s32_s16(
74 const int16x8_t s0, const int16x8_t s1, const int16x8_t x_filter_0_7) {
75 int32x4_t sum = highbd_convolve8_horiz4_s32(s0, s1, x_filter_0_7);
76
77 return vqrshrun_n_s32(sum, FILTER_BITS);
78 }
79
highbd_convolve8_8_s32(const int16x8_t s0,const int16x8_t s1,const int16x8_t s2,const int16x8_t s3,const int16x8_t s4,const int16x8_t s5,const int16x8_t s6,const int16x8_t s7,const int16x8_t y_filter,int32x4_t * sum0,int32x4_t * sum1)80 static INLINE void highbd_convolve8_8_s32(
81 const int16x8_t s0, const int16x8_t s1, const int16x8_t s2,
82 const int16x8_t s3, const int16x8_t s4, const int16x8_t s5,
83 const int16x8_t s6, const int16x8_t s7, const int16x8_t y_filter,
84 int32x4_t *sum0, int32x4_t *sum1) {
85 const int16x4_t y_filter_lo = vget_low_s16(y_filter);
86 const int16x4_t y_filter_hi = vget_high_s16(y_filter);
87
88 *sum0 = vmull_lane_s16(vget_low_s16(s0), y_filter_lo, 0);
89 *sum0 = vmlal_lane_s16(*sum0, vget_low_s16(s1), y_filter_lo, 1);
90 *sum0 = vmlal_lane_s16(*sum0, vget_low_s16(s2), y_filter_lo, 2);
91 *sum0 = vmlal_lane_s16(*sum0, vget_low_s16(s3), y_filter_lo, 3);
92 *sum0 = vmlal_lane_s16(*sum0, vget_low_s16(s4), y_filter_hi, 0);
93 *sum0 = vmlal_lane_s16(*sum0, vget_low_s16(s5), y_filter_hi, 1);
94 *sum0 = vmlal_lane_s16(*sum0, vget_low_s16(s6), y_filter_hi, 2);
95 *sum0 = vmlal_lane_s16(*sum0, vget_low_s16(s7), y_filter_hi, 3);
96
97 *sum1 = vmull_lane_s16(vget_high_s16(s0), y_filter_lo, 0);
98 *sum1 = vmlal_lane_s16(*sum1, vget_high_s16(s1), y_filter_lo, 1);
99 *sum1 = vmlal_lane_s16(*sum1, vget_high_s16(s2), y_filter_lo, 2);
100 *sum1 = vmlal_lane_s16(*sum1, vget_high_s16(s3), y_filter_lo, 3);
101 *sum1 = vmlal_lane_s16(*sum1, vget_high_s16(s4), y_filter_hi, 0);
102 *sum1 = vmlal_lane_s16(*sum1, vget_high_s16(s5), y_filter_hi, 1);
103 *sum1 = vmlal_lane_s16(*sum1, vget_high_s16(s6), y_filter_hi, 2);
104 *sum1 = vmlal_lane_s16(*sum1, vget_high_s16(s7), y_filter_hi, 3);
105 }
106
highbd_convolve8_horiz8_s32(const int16x8_t s0,const int16x8_t s0_hi,const int16x8_t x_filter_0_7,int32x4_t * sum0,int32x4_t * sum1)107 static INLINE void highbd_convolve8_horiz8_s32(const int16x8_t s0,
108 const int16x8_t s0_hi,
109 const int16x8_t x_filter_0_7,
110 int32x4_t *sum0,
111 int32x4_t *sum1) {
112 const int16x8_t s1 = vextq_s16(s0, s0_hi, 1);
113 const int16x8_t s2 = vextq_s16(s0, s0_hi, 2);
114 const int16x8_t s3 = vextq_s16(s0, s0_hi, 3);
115 const int16x8_t s4 = vextq_s16(s0, s0_hi, 4);
116 const int16x8_t s5 = vextq_s16(s0, s0_hi, 5);
117 const int16x8_t s6 = vextq_s16(s0, s0_hi, 6);
118 const int16x8_t s7 = vextq_s16(s0, s0_hi, 7);
119
120 highbd_convolve8_8_s32(s0, s1, s2, s3, s4, s5, s6, s7, x_filter_0_7, sum0,
121 sum1);
122 }
123
highbd_convolve8_horiz8_s32_s16(const int16x8_t s0,const int16x8_t s1,const int16x8_t x_filter_0_7)124 static INLINE uint16x8_t highbd_convolve8_horiz8_s32_s16(
125 const int16x8_t s0, const int16x8_t s1, const int16x8_t x_filter_0_7) {
126 int32x4_t sum0, sum1;
127 highbd_convolve8_horiz8_s32(s0, s1, x_filter_0_7, &sum0, &sum1);
128
129 return vcombine_u16(vqrshrun_n_s32(sum0, FILTER_BITS),
130 vqrshrun_n_s32(sum1, FILTER_BITS));
131 }
132
highbd_convolve8_8_s32_s16(const int16x8_t s0,const int16x8_t s1,const int16x8_t s2,const int16x8_t s3,const int16x8_t s4,const int16x8_t s5,const int16x8_t s6,const int16x8_t s7,const int16x8_t y_filter)133 static INLINE uint16x8_t highbd_convolve8_8_s32_s16(
134 const int16x8_t s0, const int16x8_t s1, const int16x8_t s2,
135 const int16x8_t s3, const int16x8_t s4, const int16x8_t s5,
136 const int16x8_t s6, const int16x8_t s7, const int16x8_t y_filter) {
137 int32x4_t sum0;
138 int32x4_t sum1;
139 highbd_convolve8_8_s32(s0, s1, s2, s3, s4, s5, s6, s7, y_filter, &sum0,
140 &sum1);
141
142 return vcombine_u16(vqrshrun_n_s32(sum0, FILTER_BITS),
143 vqrshrun_n_s32(sum1, FILTER_BITS));
144 }
145
highbd_convolve_horiz_neon(const uint16_t * src_ptr,ptrdiff_t src_stride,uint16_t * dst_ptr,ptrdiff_t dst_stride,const int16_t * x_filter_ptr,int x_step_q4,int w,int h,int bd)146 static void highbd_convolve_horiz_neon(const uint16_t *src_ptr,
147 ptrdiff_t src_stride, uint16_t *dst_ptr,
148 ptrdiff_t dst_stride,
149 const int16_t *x_filter_ptr,
150 int x_step_q4, int w, int h, int bd) {
151 assert(w >= 4 && h >= 4);
152 const uint16x8_t max = vdupq_n_u16((1 << bd) - 1);
153 const int16x8_t x_filter = vld1q_s16(x_filter_ptr);
154
155 if (w == 4) {
156 const int16_t *s = (const int16_t *)src_ptr;
157 uint16_t *d = dst_ptr;
158
159 do {
160 int16x8_t s0, s1, s2, s3;
161 load_s16_8x2(s, src_stride, &s0, &s2);
162 load_s16_8x2(s + 8, src_stride, &s1, &s3);
163
164 uint16x4_t d0 = highbd_convolve8_horiz4_s32_s16(s0, s1, x_filter);
165 uint16x4_t d1 = highbd_convolve8_horiz4_s32_s16(s2, s3, x_filter);
166
167 uint16x8_t d01 = vcombine_u16(d0, d1);
168 d01 = vminq_u16(d01, max);
169
170 vst1_u16(d + 0 * dst_stride, vget_low_u16(d01));
171 vst1_u16(d + 1 * dst_stride, vget_high_u16(d01));
172
173 s += 2 * src_stride;
174 d += 2 * dst_stride;
175 h -= 2;
176 } while (h > 0);
177 } else {
178 int height = h;
179
180 do {
181 int width = w;
182 const int16_t *s = (const int16_t *)src_ptr;
183 uint16_t *d = dst_ptr;
184 int x_q4 = 0;
185
186 const int16_t *src_x = &s[x_q4 >> SUBPEL_BITS];
187 int16x8_t s0, s2, s4, s6;
188 load_s16_8x4(src_x, src_stride, &s0, &s2, &s4, &s6);
189 src_x += 8;
190
191 do {
192 int16x8_t s1, s3, s5, s7;
193 load_s16_8x4(src_x, src_stride, &s1, &s3, &s5, &s7);
194
195 uint16x8_t d0 = highbd_convolve8_horiz8_s32_s16(s0, s1, x_filter);
196 uint16x8_t d1 = highbd_convolve8_horiz8_s32_s16(s2, s3, x_filter);
197 uint16x8_t d2 = highbd_convolve8_horiz8_s32_s16(s4, s5, x_filter);
198 uint16x8_t d3 = highbd_convolve8_horiz8_s32_s16(s6, s7, x_filter);
199
200 d0 = vminq_u16(d0, max);
201 d1 = vminq_u16(d1, max);
202 d2 = vminq_u16(d2, max);
203 d3 = vminq_u16(d3, max);
204
205 store_u16_8x4(d, dst_stride, d0, d1, d2, d3);
206
207 s0 = s1;
208 s2 = s3;
209 s4 = s5;
210 s6 = s7;
211 src_x += 8;
212 d += 8;
213 width -= 8;
214 x_q4 += 8 * x_step_q4;
215 } while (width > 0);
216 src_ptr += 4 * src_stride;
217 dst_ptr += 4 * dst_stride;
218 height -= 4;
219 } while (height > 0);
220 }
221 }
222
aom_highbd_convolve8_horiz_neon(const uint8_t * src8,ptrdiff_t src_stride,uint8_t * dst8,ptrdiff_t dst_stride,const int16_t * filter_x,int x_step_q4,const int16_t * filter_y,int y_step_q4,int w,int h,int bd)223 void aom_highbd_convolve8_horiz_neon(const uint8_t *src8, ptrdiff_t src_stride,
224 uint8_t *dst8, ptrdiff_t dst_stride,
225 const int16_t *filter_x, int x_step_q4,
226 const int16_t *filter_y, int y_step_q4,
227 int w, int h, int bd) {
228 if (x_step_q4 != 16) {
229 aom_highbd_convolve8_horiz_c(src8, src_stride, dst8, dst_stride, filter_x,
230 x_step_q4, filter_y, y_step_q4, w, h, bd);
231 } else {
232 (void)filter_y;
233 (void)y_step_q4;
234
235 uint16_t *src = CONVERT_TO_SHORTPTR(src8);
236 uint16_t *dst = CONVERT_TO_SHORTPTR(dst8);
237
238 src -= SUBPEL_TAPS / 2 - 1;
239 highbd_convolve_horiz_neon(src, src_stride, dst, dst_stride, filter_x,
240 x_step_q4, w, h, bd);
241 }
242 }
243
highbd_convolve_vert_neon(const uint16_t * src_ptr,ptrdiff_t src_stride,uint16_t * dst_ptr,ptrdiff_t dst_stride,const int16_t * y_filter_ptr,int w,int h,int bd)244 static void highbd_convolve_vert_neon(const uint16_t *src_ptr,
245 ptrdiff_t src_stride, uint16_t *dst_ptr,
246 ptrdiff_t dst_stride,
247 const int16_t *y_filter_ptr, int w, int h,
248 int bd) {
249 assert(w >= 4 && h >= 4);
250 const int16x8_t y_filter = vld1q_s16(y_filter_ptr);
251 const uint16x8_t max = vdupq_n_u16((1 << bd) - 1);
252
253 if (w == 4) {
254 const int16_t *s = (const int16_t *)src_ptr;
255 uint16_t *d = dst_ptr;
256
257 int16x4_t s0, s1, s2, s3, s4, s5, s6;
258 load_s16_4x7(s, src_stride, &s0, &s1, &s2, &s3, &s4, &s5, &s6);
259 s += 7 * src_stride;
260
261 do {
262 int16x4_t s7, s8, s9, s10;
263 load_s16_4x4(s, src_stride, &s7, &s8, &s9, &s10);
264
265 uint16x4_t d0 =
266 highbd_convolve8_4_s32_s16(s0, s1, s2, s3, s4, s5, s6, s7, y_filter);
267 uint16x4_t d1 =
268 highbd_convolve8_4_s32_s16(s1, s2, s3, s4, s5, s6, s7, s8, y_filter);
269 uint16x4_t d2 =
270 highbd_convolve8_4_s32_s16(s2, s3, s4, s5, s6, s7, s8, s9, y_filter);
271 uint16x4_t d3 =
272 highbd_convolve8_4_s32_s16(s3, s4, s5, s6, s7, s8, s9, s10, y_filter);
273
274 uint16x8_t d01 = vcombine_u16(d0, d1);
275 uint16x8_t d23 = vcombine_u16(d2, d3);
276
277 d01 = vminq_u16(d01, max);
278 d23 = vminq_u16(d23, max);
279
280 vst1_u16(d + 0 * dst_stride, vget_low_u16(d01));
281 vst1_u16(d + 1 * dst_stride, vget_high_u16(d01));
282 vst1_u16(d + 2 * dst_stride, vget_low_u16(d23));
283 vst1_u16(d + 3 * dst_stride, vget_high_u16(d23));
284
285 s0 = s4;
286 s1 = s5;
287 s2 = s6;
288 s3 = s7;
289 s4 = s8;
290 s5 = s9;
291 s6 = s10;
292 s += 4 * src_stride;
293 d += 4 * dst_stride;
294 h -= 4;
295 } while (h > 0);
296 } else {
297 do {
298 int height = h;
299 const int16_t *s = (const int16_t *)src_ptr;
300 uint16_t *d = dst_ptr;
301
302 int16x8_t s0, s1, s2, s3, s4, s5, s6;
303 load_s16_8x7(s, src_stride, &s0, &s1, &s2, &s3, &s4, &s5, &s6);
304 s += 7 * src_stride;
305
306 do {
307 int16x8_t s7, s8, s9, s10;
308 load_s16_8x4(s, src_stride, &s7, &s8, &s9, &s10);
309
310 uint16x8_t d0 = highbd_convolve8_8_s32_s16(s0, s1, s2, s3, s4, s5, s6,
311 s7, y_filter);
312 uint16x8_t d1 = highbd_convolve8_8_s32_s16(s1, s2, s3, s4, s5, s6, s7,
313 s8, y_filter);
314 uint16x8_t d2 = highbd_convolve8_8_s32_s16(s2, s3, s4, s5, s6, s7, s8,
315 s9, y_filter);
316 uint16x8_t d3 = highbd_convolve8_8_s32_s16(s3, s4, s5, s6, s7, s8, s9,
317 s10, y_filter);
318
319 d0 = vminq_u16(d0, max);
320 d1 = vminq_u16(d1, max);
321 d2 = vminq_u16(d2, max);
322 d3 = vminq_u16(d3, max);
323
324 store_u16_8x4(d, dst_stride, d0, d1, d2, d3);
325
326 s0 = s4;
327 s1 = s5;
328 s2 = s6;
329 s3 = s7;
330 s4 = s8;
331 s5 = s9;
332 s6 = s10;
333 s += 4 * src_stride;
334 d += 4 * dst_stride;
335 height -= 4;
336 } while (height > 0);
337 src_ptr += 8;
338 dst_ptr += 8;
339 w -= 8;
340 } while (w > 0);
341 }
342 }
343
aom_highbd_convolve8_vert_neon(const uint8_t * src8,ptrdiff_t src_stride,uint8_t * dst8,ptrdiff_t dst_stride,const int16_t * filter_x,int x_step_q4,const int16_t * filter_y,int y_step_q4,int w,int h,int bd)344 void aom_highbd_convolve8_vert_neon(const uint8_t *src8, ptrdiff_t src_stride,
345 uint8_t *dst8, ptrdiff_t dst_stride,
346 const int16_t *filter_x, int x_step_q4,
347 const int16_t *filter_y, int y_step_q4,
348 int w, int h, int bd) {
349 if (y_step_q4 != 16) {
350 aom_highbd_convolve8_vert_c(src8, src_stride, dst8, dst_stride, filter_x,
351 x_step_q4, filter_y, y_step_q4, w, h, bd);
352 } else {
353 (void)filter_x;
354 (void)x_step_q4;
355
356 uint16_t *src = CONVERT_TO_SHORTPTR(src8);
357 uint16_t *dst = CONVERT_TO_SHORTPTR(dst8);
358
359 src -= (SUBPEL_TAPS / 2 - 1) * src_stride;
360 highbd_convolve_vert_neon(src, src_stride, dst, dst_stride, filter_y, w, h,
361 bd);
362 }
363 }
364