1 /*
2 * Copyright (c) 2016, Alliance for Open Media. All rights reserved
3 *
4 * This source code is subject to the terms of the BSD 2 Clause License and
5 * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
6 * was not distributed with this source code in the LICENSE file, you can
7 * obtain it at www.aomedia.org/license/software. If the Alliance for Open
8 * Media Patent License 1.0 was not distributed with this source code in the
9 * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
10 */
11
12 #ifndef AOM_AV1_COMMON_CDEF_BLOCK_SIMD_H_
13 #define AOM_AV1_COMMON_CDEF_BLOCK_SIMD_H_
14
15 #include "config/av1_rtcd.h"
16
17 #include "av1/common/cdef_block.h"
18
19 /* partial A is a 16-bit vector of the form:
20 [x8 x7 x6 x5 x4 x3 x2 x1] and partial B has the form:
21 [0 y1 y2 y3 y4 y5 y6 y7].
22 This function computes (x1^2+y1^2)*C1 + (x2^2+y2^2)*C2 + ...
23 (x7^2+y2^7)*C7 + (x8^2+0^2)*C8 where the C1..C8 constants are in const1
24 and const2. */
fold_mul_and_sum(v128 partiala,v128 partialb,v128 const1,v128 const2)25 static INLINE v128 fold_mul_and_sum(v128 partiala, v128 partialb, v128 const1,
26 v128 const2) {
27 v128 tmp;
28 /* Reverse partial B. */
29 partialb = v128_shuffle_8(
30 partialb, v128_from_32(0x0f0e0100, 0x03020504, 0x07060908, 0x0b0a0d0c));
31 /* Interleave the x and y values of identical indices and pair x8 with 0. */
32 tmp = partiala;
33 partiala = v128_ziplo_16(partialb, partiala);
34 partialb = v128_ziphi_16(partialb, tmp);
35 /* Square and add the corresponding x and y values. */
36 partiala = v128_madd_s16(partiala, partiala);
37 partialb = v128_madd_s16(partialb, partialb);
38 /* Multiply by constant. */
39 partiala = v128_mullo_s32(partiala, const1);
40 partialb = v128_mullo_s32(partialb, const2);
41 /* Sum all results. */
42 partiala = v128_add_32(partiala, partialb);
43 return partiala;
44 }
45
hsum4(v128 x0,v128 x1,v128 x2,v128 x3)46 static INLINE v128 hsum4(v128 x0, v128 x1, v128 x2, v128 x3) {
47 v128 t0, t1, t2, t3;
48 t0 = v128_ziplo_32(x1, x0);
49 t1 = v128_ziplo_32(x3, x2);
50 t2 = v128_ziphi_32(x1, x0);
51 t3 = v128_ziphi_32(x3, x2);
52 x0 = v128_ziplo_64(t1, t0);
53 x1 = v128_ziphi_64(t1, t0);
54 x2 = v128_ziplo_64(t3, t2);
55 x3 = v128_ziphi_64(t3, t2);
56 return v128_add_32(v128_add_32(x0, x1), v128_add_32(x2, x3));
57 }
58
59 /* Computes cost for directions 0, 5, 6 and 7. We can call this function again
60 to compute the remaining directions. */
compute_directions(v128 lines[8],int32_t tmp_cost1[4])61 static INLINE v128 compute_directions(v128 lines[8], int32_t tmp_cost1[4]) {
62 v128 partial4a, partial4b, partial5a, partial5b, partial7a, partial7b;
63 v128 partial6;
64 v128 tmp;
65 /* Partial sums for lines 0 and 1. */
66 partial4a = v128_shl_n_byte(lines[0], 14);
67 partial4b = v128_shr_n_byte(lines[0], 2);
68 partial4a = v128_add_16(partial4a, v128_shl_n_byte(lines[1], 12));
69 partial4b = v128_add_16(partial4b, v128_shr_n_byte(lines[1], 4));
70 tmp = v128_add_16(lines[0], lines[1]);
71 partial5a = v128_shl_n_byte(tmp, 10);
72 partial5b = v128_shr_n_byte(tmp, 6);
73 partial7a = v128_shl_n_byte(tmp, 4);
74 partial7b = v128_shr_n_byte(tmp, 12);
75 partial6 = tmp;
76
77 /* Partial sums for lines 2 and 3. */
78 partial4a = v128_add_16(partial4a, v128_shl_n_byte(lines[2], 10));
79 partial4b = v128_add_16(partial4b, v128_shr_n_byte(lines[2], 6));
80 partial4a = v128_add_16(partial4a, v128_shl_n_byte(lines[3], 8));
81 partial4b = v128_add_16(partial4b, v128_shr_n_byte(lines[3], 8));
82 tmp = v128_add_16(lines[2], lines[3]);
83 partial5a = v128_add_16(partial5a, v128_shl_n_byte(tmp, 8));
84 partial5b = v128_add_16(partial5b, v128_shr_n_byte(tmp, 8));
85 partial7a = v128_add_16(partial7a, v128_shl_n_byte(tmp, 6));
86 partial7b = v128_add_16(partial7b, v128_shr_n_byte(tmp, 10));
87 partial6 = v128_add_16(partial6, tmp);
88
89 /* Partial sums for lines 4 and 5. */
90 partial4a = v128_add_16(partial4a, v128_shl_n_byte(lines[4], 6));
91 partial4b = v128_add_16(partial4b, v128_shr_n_byte(lines[4], 10));
92 partial4a = v128_add_16(partial4a, v128_shl_n_byte(lines[5], 4));
93 partial4b = v128_add_16(partial4b, v128_shr_n_byte(lines[5], 12));
94 tmp = v128_add_16(lines[4], lines[5]);
95 partial5a = v128_add_16(partial5a, v128_shl_n_byte(tmp, 6));
96 partial5b = v128_add_16(partial5b, v128_shr_n_byte(tmp, 10));
97 partial7a = v128_add_16(partial7a, v128_shl_n_byte(tmp, 8));
98 partial7b = v128_add_16(partial7b, v128_shr_n_byte(tmp, 8));
99 partial6 = v128_add_16(partial6, tmp);
100
101 /* Partial sums for lines 6 and 7. */
102 partial4a = v128_add_16(partial4a, v128_shl_n_byte(lines[6], 2));
103 partial4b = v128_add_16(partial4b, v128_shr_n_byte(lines[6], 14));
104 partial4a = v128_add_16(partial4a, lines[7]);
105 tmp = v128_add_16(lines[6], lines[7]);
106 partial5a = v128_add_16(partial5a, v128_shl_n_byte(tmp, 4));
107 partial5b = v128_add_16(partial5b, v128_shr_n_byte(tmp, 12));
108 partial7a = v128_add_16(partial7a, v128_shl_n_byte(tmp, 10));
109 partial7b = v128_add_16(partial7b, v128_shr_n_byte(tmp, 6));
110 partial6 = v128_add_16(partial6, tmp);
111
112 /* Compute costs in terms of partial sums. */
113 partial4a =
114 fold_mul_and_sum(partial4a, partial4b, v128_from_32(210, 280, 420, 840),
115 v128_from_32(105, 120, 140, 168));
116 partial7a =
117 fold_mul_and_sum(partial7a, partial7b, v128_from_32(210, 420, 0, 0),
118 v128_from_32(105, 105, 105, 140));
119 partial5a =
120 fold_mul_and_sum(partial5a, partial5b, v128_from_32(210, 420, 0, 0),
121 v128_from_32(105, 105, 105, 140));
122 partial6 = v128_madd_s16(partial6, partial6);
123 partial6 = v128_mullo_s32(partial6, v128_dup_32(105));
124
125 partial4a = hsum4(partial4a, partial5a, partial6, partial7a);
126 v128_store_unaligned(tmp_cost1, partial4a);
127 return partial4a;
128 }
129
130 /* transpose and reverse the order of the lines -- equivalent to a 90-degree
131 counter-clockwise rotation of the pixels. */
array_reverse_transpose_8x8(v128 * in,v128 * res)132 static INLINE void array_reverse_transpose_8x8(v128 *in, v128 *res) {
133 const v128 tr0_0 = v128_ziplo_16(in[1], in[0]);
134 const v128 tr0_1 = v128_ziplo_16(in[3], in[2]);
135 const v128 tr0_2 = v128_ziphi_16(in[1], in[0]);
136 const v128 tr0_3 = v128_ziphi_16(in[3], in[2]);
137 const v128 tr0_4 = v128_ziplo_16(in[5], in[4]);
138 const v128 tr0_5 = v128_ziplo_16(in[7], in[6]);
139 const v128 tr0_6 = v128_ziphi_16(in[5], in[4]);
140 const v128 tr0_7 = v128_ziphi_16(in[7], in[6]);
141
142 const v128 tr1_0 = v128_ziplo_32(tr0_1, tr0_0);
143 const v128 tr1_1 = v128_ziplo_32(tr0_5, tr0_4);
144 const v128 tr1_2 = v128_ziphi_32(tr0_1, tr0_0);
145 const v128 tr1_3 = v128_ziphi_32(tr0_5, tr0_4);
146 const v128 tr1_4 = v128_ziplo_32(tr0_3, tr0_2);
147 const v128 tr1_5 = v128_ziplo_32(tr0_7, tr0_6);
148 const v128 tr1_6 = v128_ziphi_32(tr0_3, tr0_2);
149 const v128 tr1_7 = v128_ziphi_32(tr0_7, tr0_6);
150
151 res[7] = v128_ziplo_64(tr1_1, tr1_0);
152 res[6] = v128_ziphi_64(tr1_1, tr1_0);
153 res[5] = v128_ziplo_64(tr1_3, tr1_2);
154 res[4] = v128_ziphi_64(tr1_3, tr1_2);
155 res[3] = v128_ziplo_64(tr1_5, tr1_4);
156 res[2] = v128_ziphi_64(tr1_5, tr1_4);
157 res[1] = v128_ziplo_64(tr1_7, tr1_6);
158 res[0] = v128_ziphi_64(tr1_7, tr1_6);
159 }
160
SIMD_FUNC(cdef_find_dir)161 int SIMD_FUNC(cdef_find_dir)(const uint16_t *img, int stride, int32_t *var,
162 int coeff_shift) {
163 int i;
164 int32_t cost[8];
165 int32_t best_cost = 0;
166 int best_dir = 0;
167 v128 lines[8];
168 for (i = 0; i < 8; i++) {
169 lines[i] = v128_load_unaligned(&img[i * stride]);
170 lines[i] =
171 v128_sub_16(v128_shr_s16(lines[i], coeff_shift), v128_dup_16(128));
172 }
173
174 /* Compute "mostly vertical" directions. */
175 v128 dir47 = compute_directions(lines, cost + 4);
176
177 array_reverse_transpose_8x8(lines, lines);
178
179 /* Compute "mostly horizontal" directions. */
180 v128 dir03 = compute_directions(lines, cost);
181
182 v128 max = v128_max_s32(dir03, dir47);
183 max = v128_max_s32(max, v128_align(max, max, 8));
184 max = v128_max_s32(max, v128_align(max, max, 4));
185 best_cost = v128_low_u32(max);
186 v128 t =
187 v128_pack_s32_s16(v128_cmpeq_32(max, dir47), v128_cmpeq_32(max, dir03));
188 best_dir = v128_movemask_8(v128_pack_s16_s8(t, t));
189 best_dir = get_msb(best_dir ^ (best_dir - 1)); // Count trailing zeros
190
191 /* Difference between the optimal variance and the variance along the
192 orthogonal direction. Again, the sum(x^2) terms cancel out. */
193 *var = best_cost - cost[(best_dir + 4) & 7];
194 /* We'd normally divide by 840, but dividing by 1024 is close enough
195 for what we're going to do with this. */
196 *var >>= 10;
197 return best_dir;
198 }
199
200 // Work around compiler out of memory issues with Win32 builds. This issue has
201 // been observed with Visual Studio 2017, 2019, and 2022 (version 17.4).
202 #if defined(_MSC_VER) && defined(_M_IX86) && _MSC_VER < 1940
203 #define CDEF_INLINE static INLINE
204 #else
205 #define CDEF_INLINE SIMD_INLINE
206 #endif
207
208 // sign(a-b) * min(abs(a-b), max(0, threshold - (abs(a-b) >> adjdamp)))
constrain16(v256 a,v256 b,unsigned int threshold,unsigned int adjdamp)209 CDEF_INLINE v256 constrain16(v256 a, v256 b, unsigned int threshold,
210 unsigned int adjdamp) {
211 v256 diff = v256_sub_16(a, b);
212 const v256 sign = v256_shr_n_s16(diff, 15);
213 diff = v256_abs_s16(diff);
214 const v256 s =
215 v256_ssub_u16(v256_dup_16(threshold), v256_shr_u16(diff, adjdamp));
216 return v256_xor(v256_add_16(sign, v256_min_s16(diff, s)), sign);
217 }
218
get_max_primary(const int is_lowbd,v256 * tap,v256 max,v256 cdef_large_value_mask)219 SIMD_INLINE v256 get_max_primary(const int is_lowbd, v256 *tap, v256 max,
220 v256 cdef_large_value_mask) {
221 if (is_lowbd) {
222 v256 max_u8;
223 max_u8 = tap[0];
224 max_u8 = v256_max_u8(max_u8, tap[1]);
225 max_u8 = v256_max_u8(max_u8, tap[2]);
226 max_u8 = v256_max_u8(max_u8, tap[3]);
227 /* The source is 16 bits, however, we only really care about the lower
228 8 bits. The upper 8 bits contain the "large" flag. After the final
229 primary max has been calculated, zero out the upper 8 bits. Use this
230 to find the "16 bit" max. */
231 max = v256_max_s16(max, v256_and(max_u8, cdef_large_value_mask));
232 } else {
233 /* Convert CDEF_VERY_LARGE to 0 before calculating max. */
234 max = v256_max_s16(max, v256_and(tap[0], cdef_large_value_mask));
235 max = v256_max_s16(max, v256_and(tap[1], cdef_large_value_mask));
236 max = v256_max_s16(max, v256_and(tap[2], cdef_large_value_mask));
237 max = v256_max_s16(max, v256_and(tap[3], cdef_large_value_mask));
238 }
239 return max;
240 }
241
get_max_secondary(const int is_lowbd,v256 * tap,v256 max,v256 cdef_large_value_mask)242 SIMD_INLINE v256 get_max_secondary(const int is_lowbd, v256 *tap, v256 max,
243 v256 cdef_large_value_mask) {
244 if (is_lowbd) {
245 v256 max_u8;
246 max_u8 = tap[0];
247 max_u8 = v256_max_u8(max_u8, tap[1]);
248 max_u8 = v256_max_u8(max_u8, tap[2]);
249 max_u8 = v256_max_u8(max_u8, tap[3]);
250 max_u8 = v256_max_u8(max_u8, tap[4]);
251 max_u8 = v256_max_u8(max_u8, tap[5]);
252 max_u8 = v256_max_u8(max_u8, tap[6]);
253 max_u8 = v256_max_u8(max_u8, tap[7]);
254 /* The source is 16 bits, however, we only really care about the lower
255 8 bits. The upper 8 bits contain the "large" flag. After the final
256 primary max has been calculated, zero out the upper 8 bits. Use this
257 to find the "16 bit" max. */
258 max = v256_max_s16(max, v256_and(max_u8, cdef_large_value_mask));
259 } else {
260 /* Convert CDEF_VERY_LARGE to 0 before calculating max. */
261 max = v256_max_s16(max, v256_and(tap[0], cdef_large_value_mask));
262 max = v256_max_s16(max, v256_and(tap[1], cdef_large_value_mask));
263 max = v256_max_s16(max, v256_and(tap[2], cdef_large_value_mask));
264 max = v256_max_s16(max, v256_and(tap[3], cdef_large_value_mask));
265 max = v256_max_s16(max, v256_and(tap[4], cdef_large_value_mask));
266 max = v256_max_s16(max, v256_and(tap[5], cdef_large_value_mask));
267 max = v256_max_s16(max, v256_and(tap[6], cdef_large_value_mask));
268 max = v256_max_s16(max, v256_and(tap[7], cdef_large_value_mask));
269 }
270 return max;
271 }
272
filter_block_4x4(const int is_lowbd,void * dest,int dstride,const uint16_t * in,int pri_strength,int sec_strength,int dir,int pri_damping,int sec_damping,int coeff_shift,int height,int enable_primary,int enable_secondary)273 CDEF_INLINE void filter_block_4x4(const int is_lowbd, void *dest, int dstride,
274 const uint16_t *in, int pri_strength,
275 int sec_strength, int dir, int pri_damping,
276 int sec_damping, int coeff_shift, int height,
277 int enable_primary, int enable_secondary) {
278 uint8_t *dst8 = (uint8_t *)dest;
279 uint16_t *dst16 = (uint16_t *)dest;
280 const int clipping_required = enable_primary && enable_secondary;
281 v256 p0, p1, p2, p3;
282 v256 sum, row, res;
283 v256 max, min;
284 const v256 cdef_large_value_mask = v256_dup_16((uint16_t)~CDEF_VERY_LARGE);
285 const int po1 = cdef_directions[dir][0];
286 const int po2 = cdef_directions[dir][1];
287 const int s1o1 = cdef_directions[dir + 2][0];
288 const int s1o2 = cdef_directions[dir + 2][1];
289 const int s2o1 = cdef_directions[dir - 2][0];
290 const int s2o2 = cdef_directions[dir - 2][1];
291 const int *pri_taps = cdef_pri_taps[(pri_strength >> coeff_shift) & 1];
292 const int *sec_taps = cdef_sec_taps;
293 int i;
294
295 if (enable_primary && pri_strength)
296 pri_damping = AOMMAX(0, pri_damping - get_msb(pri_strength));
297 if (enable_secondary && sec_strength)
298 sec_damping = AOMMAX(0, sec_damping - get_msb(sec_strength));
299
300 for (i = 0; i < height; i += 4) {
301 sum = v256_zero();
302 row = v256_from_v64(v64_load_aligned(&in[(i + 0) * CDEF_BSTRIDE]),
303 v64_load_aligned(&in[(i + 1) * CDEF_BSTRIDE]),
304 v64_load_aligned(&in[(i + 2) * CDEF_BSTRIDE]),
305 v64_load_aligned(&in[(i + 3) * CDEF_BSTRIDE]));
306 max = min = row;
307
308 if (enable_primary) {
309 v256 tap[4];
310 // Primary near taps
311 tap[0] =
312 v256_from_v64(v64_load_unaligned(&in[(i + 0) * CDEF_BSTRIDE + po1]),
313 v64_load_unaligned(&in[(i + 1) * CDEF_BSTRIDE + po1]),
314 v64_load_unaligned(&in[(i + 2) * CDEF_BSTRIDE + po1]),
315 v64_load_unaligned(&in[(i + 3) * CDEF_BSTRIDE + po1]));
316 p0 = constrain16(tap[0], row, pri_strength, pri_damping);
317 tap[1] =
318 v256_from_v64(v64_load_unaligned(&in[(i + 0) * CDEF_BSTRIDE - po1]),
319 v64_load_unaligned(&in[(i + 1) * CDEF_BSTRIDE - po1]),
320 v64_load_unaligned(&in[(i + 2) * CDEF_BSTRIDE - po1]),
321 v64_load_unaligned(&in[(i + 3) * CDEF_BSTRIDE - po1]));
322 p1 = constrain16(tap[1], row, pri_strength, pri_damping);
323
324 // sum += pri_taps[0] * (p0 + p1)
325 sum = v256_add_16(
326 sum, v256_mullo_s16(v256_dup_16(pri_taps[0]), v256_add_16(p0, p1)));
327
328 // Primary far taps
329 tap[2] =
330 v256_from_v64(v64_load_unaligned(&in[(i + 0) * CDEF_BSTRIDE + po2]),
331 v64_load_unaligned(&in[(i + 1) * CDEF_BSTRIDE + po2]),
332 v64_load_unaligned(&in[(i + 2) * CDEF_BSTRIDE + po2]),
333 v64_load_unaligned(&in[(i + 3) * CDEF_BSTRIDE + po2]));
334 p0 = constrain16(tap[2], row, pri_strength, pri_damping);
335 tap[3] =
336 v256_from_v64(v64_load_unaligned(&in[(i + 0) * CDEF_BSTRIDE - po2]),
337 v64_load_unaligned(&in[(i + 1) * CDEF_BSTRIDE - po2]),
338 v64_load_unaligned(&in[(i + 2) * CDEF_BSTRIDE - po2]),
339 v64_load_unaligned(&in[(i + 3) * CDEF_BSTRIDE - po2]));
340 p1 = constrain16(tap[3], row, pri_strength, pri_damping);
341
342 // sum += pri_taps[1] * (p0 + p1)
343 sum = v256_add_16(
344 sum, v256_mullo_s16(v256_dup_16(pri_taps[1]), v256_add_16(p0, p1)));
345 if (clipping_required) {
346 max = get_max_primary(is_lowbd, tap, max, cdef_large_value_mask);
347
348 min = v256_min_s16(min, tap[0]);
349 min = v256_min_s16(min, tap[1]);
350 min = v256_min_s16(min, tap[2]);
351 min = v256_min_s16(min, tap[3]);
352 }
353 }
354
355 if (enable_secondary) {
356 v256 tap[8];
357 // Secondary near taps
358 tap[0] =
359 v256_from_v64(v64_load_unaligned(&in[(i + 0) * CDEF_BSTRIDE + s1o1]),
360 v64_load_unaligned(&in[(i + 1) * CDEF_BSTRIDE + s1o1]),
361 v64_load_unaligned(&in[(i + 2) * CDEF_BSTRIDE + s1o1]),
362 v64_load_unaligned(&in[(i + 3) * CDEF_BSTRIDE + s1o1]));
363 p0 = constrain16(tap[0], row, sec_strength, sec_damping);
364 tap[1] =
365 v256_from_v64(v64_load_unaligned(&in[(i + 0) * CDEF_BSTRIDE - s1o1]),
366 v64_load_unaligned(&in[(i + 1) * CDEF_BSTRIDE - s1o1]),
367 v64_load_unaligned(&in[(i + 2) * CDEF_BSTRIDE - s1o1]),
368 v64_load_unaligned(&in[(i + 3) * CDEF_BSTRIDE - s1o1]));
369 p1 = constrain16(tap[1], row, sec_strength, sec_damping);
370 tap[2] =
371 v256_from_v64(v64_load_unaligned(&in[(i + 0) * CDEF_BSTRIDE + s2o1]),
372 v64_load_unaligned(&in[(i + 1) * CDEF_BSTRIDE + s2o1]),
373 v64_load_unaligned(&in[(i + 2) * CDEF_BSTRIDE + s2o1]),
374 v64_load_unaligned(&in[(i + 3) * CDEF_BSTRIDE + s2o1]));
375 p2 = constrain16(tap[2], row, sec_strength, sec_damping);
376 tap[3] =
377 v256_from_v64(v64_load_unaligned(&in[(i + 0) * CDEF_BSTRIDE - s2o1]),
378 v64_load_unaligned(&in[(i + 1) * CDEF_BSTRIDE - s2o1]),
379 v64_load_unaligned(&in[(i + 2) * CDEF_BSTRIDE - s2o1]),
380 v64_load_unaligned(&in[(i + 3) * CDEF_BSTRIDE - s2o1]));
381 p3 = constrain16(tap[3], row, sec_strength, sec_damping);
382
383 // sum += sec_taps[0] * (p0 + p1 + p2 + p3)
384 sum = v256_add_16(sum, v256_mullo_s16(v256_dup_16(sec_taps[0]),
385 v256_add_16(v256_add_16(p0, p1),
386 v256_add_16(p2, p3))));
387
388 // Secondary far taps
389 tap[4] =
390 v256_from_v64(v64_load_unaligned(&in[(i + 0) * CDEF_BSTRIDE + s1o2]),
391 v64_load_unaligned(&in[(i + 1) * CDEF_BSTRIDE + s1o2]),
392 v64_load_unaligned(&in[(i + 2) * CDEF_BSTRIDE + s1o2]),
393 v64_load_unaligned(&in[(i + 3) * CDEF_BSTRIDE + s1o2]));
394 p0 = constrain16(tap[4], row, sec_strength, sec_damping);
395 tap[5] =
396 v256_from_v64(v64_load_unaligned(&in[(i + 0) * CDEF_BSTRIDE - s1o2]),
397 v64_load_unaligned(&in[(i + 1) * CDEF_BSTRIDE - s1o2]),
398 v64_load_unaligned(&in[(i + 2) * CDEF_BSTRIDE - s1o2]),
399 v64_load_unaligned(&in[(i + 3) * CDEF_BSTRIDE - s1o2]));
400 p1 = constrain16(tap[5], row, sec_strength, sec_damping);
401 tap[6] =
402 v256_from_v64(v64_load_unaligned(&in[(i + 0) * CDEF_BSTRIDE + s2o2]),
403 v64_load_unaligned(&in[(i + 1) * CDEF_BSTRIDE + s2o2]),
404 v64_load_unaligned(&in[(i + 2) * CDEF_BSTRIDE + s2o2]),
405 v64_load_unaligned(&in[(i + 3) * CDEF_BSTRIDE + s2o2]));
406 p2 = constrain16(tap[6], row, sec_strength, sec_damping);
407 tap[7] =
408 v256_from_v64(v64_load_unaligned(&in[(i + 0) * CDEF_BSTRIDE - s2o2]),
409 v64_load_unaligned(&in[(i + 1) * CDEF_BSTRIDE - s2o2]),
410 v64_load_unaligned(&in[(i + 2) * CDEF_BSTRIDE - s2o2]),
411 v64_load_unaligned(&in[(i + 3) * CDEF_BSTRIDE - s2o2]));
412 p3 = constrain16(tap[7], row, sec_strength, sec_damping);
413
414 // sum += sec_taps[1] * (p0 + p1 + p2 + p3)
415 sum = v256_add_16(sum, v256_mullo_s16(v256_dup_16(sec_taps[1]),
416 v256_add_16(v256_add_16(p0, p1),
417 v256_add_16(p2, p3))));
418
419 if (clipping_required) {
420 max = get_max_secondary(is_lowbd, tap, max, cdef_large_value_mask);
421
422 min = v256_min_s16(min, tap[0]);
423 min = v256_min_s16(min, tap[1]);
424 min = v256_min_s16(min, tap[2]);
425 min = v256_min_s16(min, tap[3]);
426 min = v256_min_s16(min, tap[4]);
427 min = v256_min_s16(min, tap[5]);
428 min = v256_min_s16(min, tap[6]);
429 min = v256_min_s16(min, tap[7]);
430 }
431 }
432
433 // res = row + ((sum - (sum < 0) + 8) >> 4)
434 sum = v256_add_16(sum, v256_cmplt_s16(sum, v256_zero()));
435 res = v256_add_16(sum, v256_dup_16(8));
436 res = v256_shr_n_s16(res, 4);
437 res = v256_add_16(row, res);
438 if (clipping_required) {
439 res = v256_min_s16(v256_max_s16(res, min), max);
440 }
441
442 if (is_lowbd) {
443 const v128 res_128 = v256_low_v128(v256_pack_s16_u8(res, res));
444 u32_store_aligned(&dst8[(i + 0) * dstride],
445 v64_high_u32(v128_high_v64(res_128)));
446 u32_store_aligned(&dst8[(i + 1) * dstride],
447 v64_low_u32(v128_high_v64(res_128)));
448 u32_store_aligned(&dst8[(i + 2) * dstride],
449 v64_high_u32(v128_low_v64(res_128)));
450 u32_store_aligned(&dst8[(i + 3) * dstride],
451 v64_low_u32(v128_low_v64(res_128)));
452 } else {
453 v64_store_aligned(&dst16[(i + 0) * dstride],
454 v128_high_v64(v256_high_v128(res)));
455 v64_store_aligned(&dst16[(i + 1) * dstride],
456 v128_low_v64(v256_high_v128(res)));
457 v64_store_aligned(&dst16[(i + 2) * dstride],
458 v128_high_v64(v256_low_v128(res)));
459 v64_store_aligned(&dst16[(i + 3) * dstride],
460 v128_low_v64(v256_low_v128(res)));
461 }
462 }
463 }
464
filter_block_8x8(const int is_lowbd,void * dest,int dstride,const uint16_t * in,int pri_strength,int sec_strength,int dir,int pri_damping,int sec_damping,int coeff_shift,int height,int enable_primary,int enable_secondary)465 CDEF_INLINE void filter_block_8x8(const int is_lowbd, void *dest, int dstride,
466 const uint16_t *in, int pri_strength,
467 int sec_strength, int dir, int pri_damping,
468 int sec_damping, int coeff_shift, int height,
469 int enable_primary, int enable_secondary) {
470 uint8_t *dst8 = (uint8_t *)dest;
471 uint16_t *dst16 = (uint16_t *)dest;
472 const int clipping_required = enable_primary && enable_secondary;
473 int i;
474 v256 sum, p0, p1, p2, p3, row, res;
475 const v256 cdef_large_value_mask = v256_dup_16((uint16_t)~CDEF_VERY_LARGE);
476 v256 max, min;
477 const int po1 = cdef_directions[dir][0];
478 const int po2 = cdef_directions[dir][1];
479 const int s1o1 = cdef_directions[dir + 2][0];
480 const int s1o2 = cdef_directions[dir + 2][1];
481 const int s2o1 = cdef_directions[dir - 2][0];
482 const int s2o2 = cdef_directions[dir - 2][1];
483 const int *pri_taps = cdef_pri_taps[(pri_strength >> coeff_shift) & 1];
484 const int *sec_taps = cdef_sec_taps;
485
486 if (enable_primary && pri_strength)
487 pri_damping = AOMMAX(0, pri_damping - get_msb(pri_strength));
488 if (enable_secondary && sec_strength)
489 sec_damping = AOMMAX(0, sec_damping - get_msb(sec_strength));
490
491 for (i = 0; i < height; i += 2) {
492 v256 tap[8];
493 sum = v256_zero();
494 row = v256_from_v128(v128_load_aligned(&in[i * CDEF_BSTRIDE]),
495 v128_load_aligned(&in[(i + 1) * CDEF_BSTRIDE]));
496
497 min = max = row;
498 if (enable_primary) {
499 // Primary near taps
500 tap[0] = v256_from_v128(
501 v128_load_unaligned(&in[i * CDEF_BSTRIDE + po1]),
502 v128_load_unaligned(&in[(i + 1) * CDEF_BSTRIDE + po1]));
503 tap[1] = v256_from_v128(
504 v128_load_unaligned(&in[i * CDEF_BSTRIDE - po1]),
505 v128_load_unaligned(&in[(i + 1) * CDEF_BSTRIDE - po1]));
506 p0 = constrain16(tap[0], row, pri_strength, pri_damping);
507 p1 = constrain16(tap[1], row, pri_strength, pri_damping);
508
509 // sum += pri_taps[0] * (p0 + p1)
510 sum = v256_add_16(
511 sum, v256_mullo_s16(v256_dup_16(pri_taps[0]), v256_add_16(p0, p1)));
512
513 // Primary far taps
514 tap[2] = v256_from_v128(
515 v128_load_unaligned(&in[i * CDEF_BSTRIDE + po2]),
516 v128_load_unaligned(&in[(i + 1) * CDEF_BSTRIDE + po2]));
517 tap[3] = v256_from_v128(
518 v128_load_unaligned(&in[i * CDEF_BSTRIDE - po2]),
519 v128_load_unaligned(&in[(i + 1) * CDEF_BSTRIDE - po2]));
520 p0 = constrain16(tap[2], row, pri_strength, pri_damping);
521 p1 = constrain16(tap[3], row, pri_strength, pri_damping);
522
523 // sum += pri_taps[1] * (p0 + p1)
524 sum = v256_add_16(
525 sum, v256_mullo_s16(v256_dup_16(pri_taps[1]), v256_add_16(p0, p1)));
526
527 if (clipping_required) {
528 max = get_max_primary(is_lowbd, tap, max, cdef_large_value_mask);
529
530 min = v256_min_s16(min, tap[0]);
531 min = v256_min_s16(min, tap[1]);
532 min = v256_min_s16(min, tap[2]);
533 min = v256_min_s16(min, tap[3]);
534 }
535 // End primary
536 }
537
538 if (enable_secondary) {
539 // Secondary near taps
540 tap[0] = v256_from_v128(
541 v128_load_unaligned(&in[i * CDEF_BSTRIDE + s1o1]),
542 v128_load_unaligned(&in[(i + 1) * CDEF_BSTRIDE + s1o1]));
543 tap[1] = v256_from_v128(
544 v128_load_unaligned(&in[i * CDEF_BSTRIDE - s1o1]),
545 v128_load_unaligned(&in[(i + 1) * CDEF_BSTRIDE - s1o1]));
546 tap[2] = v256_from_v128(
547 v128_load_unaligned(&in[i * CDEF_BSTRIDE + s2o1]),
548 v128_load_unaligned(&in[(i + 1) * CDEF_BSTRIDE + s2o1]));
549 tap[3] = v256_from_v128(
550 v128_load_unaligned(&in[i * CDEF_BSTRIDE - s2o1]),
551 v128_load_unaligned(&in[(i + 1) * CDEF_BSTRIDE - s2o1]));
552 p0 = constrain16(tap[0], row, sec_strength, sec_damping);
553 p1 = constrain16(tap[1], row, sec_strength, sec_damping);
554 p2 = constrain16(tap[2], row, sec_strength, sec_damping);
555 p3 = constrain16(tap[3], row, sec_strength, sec_damping);
556
557 // sum += sec_taps[0] * (p0 + p1 + p2 + p3)
558 sum = v256_add_16(sum, v256_mullo_s16(v256_dup_16(sec_taps[0]),
559 v256_add_16(v256_add_16(p0, p1),
560 v256_add_16(p2, p3))));
561
562 // Secondary far taps
563 tap[4] = v256_from_v128(
564 v128_load_unaligned(&in[i * CDEF_BSTRIDE + s1o2]),
565 v128_load_unaligned(&in[(i + 1) * CDEF_BSTRIDE + s1o2]));
566 tap[5] = v256_from_v128(
567 v128_load_unaligned(&in[i * CDEF_BSTRIDE - s1o2]),
568 v128_load_unaligned(&in[(i + 1) * CDEF_BSTRIDE - s1o2]));
569 tap[6] = v256_from_v128(
570 v128_load_unaligned(&in[i * CDEF_BSTRIDE + s2o2]),
571 v128_load_unaligned(&in[(i + 1) * CDEF_BSTRIDE + s2o2]));
572 tap[7] = v256_from_v128(
573 v128_load_unaligned(&in[i * CDEF_BSTRIDE - s2o2]),
574 v128_load_unaligned(&in[(i + 1) * CDEF_BSTRIDE - s2o2]));
575 p0 = constrain16(tap[4], row, sec_strength, sec_damping);
576 p1 = constrain16(tap[5], row, sec_strength, sec_damping);
577 p2 = constrain16(tap[6], row, sec_strength, sec_damping);
578 p3 = constrain16(tap[7], row, sec_strength, sec_damping);
579
580 // sum += sec_taps[1] * (p0 + p1 + p2 + p3)
581 sum = v256_add_16(sum, v256_mullo_s16(v256_dup_16(sec_taps[1]),
582 v256_add_16(v256_add_16(p0, p1),
583 v256_add_16(p2, p3))));
584
585 if (clipping_required) {
586 max = get_max_secondary(is_lowbd, tap, max, cdef_large_value_mask);
587
588 min = v256_min_s16(min, tap[0]);
589 min = v256_min_s16(min, tap[1]);
590 min = v256_min_s16(min, tap[2]);
591 min = v256_min_s16(min, tap[3]);
592 min = v256_min_s16(min, tap[4]);
593 min = v256_min_s16(min, tap[5]);
594 min = v256_min_s16(min, tap[6]);
595 min = v256_min_s16(min, tap[7]);
596 }
597 // End secondary
598 }
599
600 // res = row + ((sum - (sum < 0) + 8) >> 4)
601 sum = v256_add_16(sum, v256_cmplt_s16(sum, v256_zero()));
602 res = v256_add_16(sum, v256_dup_16(8));
603 res = v256_shr_n_s16(res, 4);
604 res = v256_add_16(row, res);
605 if (clipping_required) {
606 res = v256_min_s16(v256_max_s16(res, min), max);
607 }
608
609 if (is_lowbd) {
610 const v128 res_128 = v256_low_v128(v256_pack_s16_u8(res, res));
611 v64_store_aligned(&dst8[i * dstride], v128_high_v64(res_128));
612 v64_store_aligned(&dst8[(i + 1) * dstride], v128_low_v64(res_128));
613 } else {
614 v128_store_unaligned(&dst16[i * dstride], v256_high_v128(res));
615 v128_store_unaligned(&dst16[(i + 1) * dstride], v256_low_v128(res));
616 }
617 }
618 }
619
copy_block_4xh(const int is_lowbd,void * dest,int dstride,const uint16_t * in,int height)620 SIMD_INLINE void copy_block_4xh(const int is_lowbd, void *dest, int dstride,
621 const uint16_t *in, int height) {
622 uint8_t *dst8 = (uint8_t *)dest;
623 uint16_t *dst16 = (uint16_t *)dest;
624 int i;
625 for (i = 0; i < height; i += 4) {
626 const v128 row0 =
627 v128_from_v64(v64_load_aligned(&in[(i + 0) * CDEF_BSTRIDE]),
628 v64_load_aligned(&in[(i + 1) * CDEF_BSTRIDE]));
629 const v128 row1 =
630 v128_from_v64(v64_load_aligned(&in[(i + 2) * CDEF_BSTRIDE]),
631 v64_load_aligned(&in[(i + 3) * CDEF_BSTRIDE]));
632 if (is_lowbd) {
633 /* Note: v128_pack_s16_u8(). The parameter order is swapped internally */
634 const v128 res_128 = v128_pack_s16_u8(row1, row0);
635 u32_store_aligned(&dst8[(i + 0) * dstride],
636 v64_high_u32(v128_low_v64(res_128)));
637 u32_store_aligned(&dst8[(i + 1) * dstride],
638 v64_low_u32(v128_low_v64(res_128)));
639 u32_store_aligned(&dst8[(i + 2) * dstride],
640 v64_high_u32(v128_high_v64(res_128)));
641 u32_store_aligned(&dst8[(i + 3) * dstride],
642 v64_low_u32(v128_high_v64(res_128)));
643 } else {
644 v64_store_aligned(&dst16[(i + 0) * dstride], v128_high_v64(row0));
645 v64_store_aligned(&dst16[(i + 1) * dstride], v128_low_v64(row0));
646 v64_store_aligned(&dst16[(i + 2) * dstride], v128_high_v64(row1));
647 v64_store_aligned(&dst16[(i + 3) * dstride], v128_low_v64(row1));
648 }
649 }
650 }
651
copy_block_8xh(const int is_lowbd,void * dest,int dstride,const uint16_t * in,int height)652 SIMD_INLINE void copy_block_8xh(const int is_lowbd, void *dest, int dstride,
653 const uint16_t *in, int height) {
654 uint8_t *dst8 = (uint8_t *)dest;
655 uint16_t *dst16 = (uint16_t *)dest;
656 int i;
657 for (i = 0; i < height; i += 2) {
658 const v128 row0 = v128_load_aligned(&in[i * CDEF_BSTRIDE]);
659 const v128 row1 = v128_load_aligned(&in[(i + 1) * CDEF_BSTRIDE]);
660 if (is_lowbd) {
661 /* Note: v128_pack_s16_u8(). The parameter order is swapped internally */
662 const v128 res_128 = v128_pack_s16_u8(row1, row0);
663 v64_store_aligned(&dst8[i * dstride], v128_low_v64(res_128));
664 v64_store_aligned(&dst8[(i + 1) * dstride], v128_high_v64(res_128));
665 } else {
666 v128_store_unaligned(&dst16[i * dstride], row0);
667 v128_store_unaligned(&dst16[(i + 1) * dstride], row1);
668 }
669 }
670 }
671
SIMD_FUNC(cdef_filter_8_0)672 void SIMD_FUNC(cdef_filter_8_0)(void *dest, int dstride, const uint16_t *in,
673 int pri_strength, int sec_strength, int dir,
674 int pri_damping, int sec_damping,
675 int coeff_shift, int block_width,
676 int block_height) {
677 uint8_t *dst8 = (uint8_t *)dest;
678 if (block_width == 8) {
679 filter_block_8x8(/*is_lowbd=*/1, dst8, dstride, in, pri_strength,
680 sec_strength, dir, pri_damping, sec_damping, coeff_shift,
681 block_height, /*enable_primary=*/1,
682 /*enable_secondary=*/1);
683 } else {
684 filter_block_4x4(/*is_lowbd=*/1, dst8, dstride, in, pri_strength,
685 sec_strength, dir, pri_damping, sec_damping, coeff_shift,
686 block_height, /*enable_primary=*/1,
687 /*enable_secondary=*/1);
688 }
689 }
690
SIMD_FUNC(cdef_filter_8_1)691 void SIMD_FUNC(cdef_filter_8_1)(void *dest, int dstride, const uint16_t *in,
692 int pri_strength, int sec_strength, int dir,
693 int pri_damping, int sec_damping,
694 int coeff_shift, int block_width,
695 int block_height) {
696 uint8_t *dst8 = (uint8_t *)dest;
697 if (block_width == 8) {
698 filter_block_8x8(/*is_lowbd=*/1, dst8, dstride, in, pri_strength,
699 sec_strength, dir, pri_damping, sec_damping, coeff_shift,
700 block_height, /*enable_primary=*/1,
701 /*enable_secondary=*/0);
702 } else {
703 filter_block_4x4(/*is_lowbd=*/1, dst8, dstride, in, pri_strength,
704 sec_strength, dir, pri_damping, sec_damping, coeff_shift,
705 block_height, /*enable_primary=*/1,
706 /*enable_secondary=*/0);
707 }
708 }
SIMD_FUNC(cdef_filter_8_2)709 void SIMD_FUNC(cdef_filter_8_2)(void *dest, int dstride, const uint16_t *in,
710 int pri_strength, int sec_strength, int dir,
711 int pri_damping, int sec_damping,
712 int coeff_shift, int block_width,
713 int block_height) {
714 uint8_t *dst8 = (uint8_t *)dest;
715 if (block_width == 8) {
716 filter_block_8x8(/*is_lowbd=*/1, dst8, dstride, in, pri_strength,
717 sec_strength, dir, pri_damping, sec_damping, coeff_shift,
718 block_height, /*enable_primary=*/0,
719 /*enable_secondary=*/1);
720 } else {
721 filter_block_4x4(/*is_lowbd=*/1, dst8, dstride, in, pri_strength,
722 sec_strength, dir, pri_damping, sec_damping, coeff_shift,
723 block_height, /*enable_primary=*/0,
724 /*enable_secondary=*/1);
725 }
726 }
727
SIMD_FUNC(cdef_filter_8_3)728 void SIMD_FUNC(cdef_filter_8_3)(void *dest, int dstride, const uint16_t *in,
729 int pri_strength, int sec_strength, int dir,
730 int pri_damping, int sec_damping,
731 int coeff_shift, int block_width,
732 int block_height) {
733 uint8_t *dst8 = (uint8_t *)dest;
734 (void)pri_strength;
735 (void)sec_strength;
736 (void)dir;
737 (void)pri_damping;
738 (void)sec_damping;
739 (void)coeff_shift;
740 (void)block_width;
741
742 if (block_width == 8) {
743 copy_block_8xh(/*is_lowbd=*/1, dst8, dstride, in, block_height);
744 } else {
745 copy_block_4xh(/*is_lowbd=*/1, dst8, dstride, in, block_height);
746 }
747 }
748
SIMD_FUNC(cdef_filter_16_0)749 void SIMD_FUNC(cdef_filter_16_0)(void *dest, int dstride, const uint16_t *in,
750 int pri_strength, int sec_strength, int dir,
751 int pri_damping, int sec_damping,
752 int coeff_shift, int block_width,
753 int block_height) {
754 uint16_t *dst16 = (uint16_t *)dest;
755 if (block_width == 8) {
756 filter_block_8x8(/*is_lowbd=*/0, dst16, dstride, in, pri_strength,
757 sec_strength, dir, pri_damping, sec_damping, coeff_shift,
758 block_height, /*enable_primary=*/1,
759 /*enable_secondary=*/1);
760 } else {
761 filter_block_4x4(/*is_lowbd=*/0, dst16, dstride, in, pri_strength,
762 sec_strength, dir, pri_damping, sec_damping, coeff_shift,
763 block_height, /*enable_primary=*/1,
764 /*enable_secondary=*/1);
765 }
766 }
767
SIMD_FUNC(cdef_filter_16_1)768 void SIMD_FUNC(cdef_filter_16_1)(void *dest, int dstride, const uint16_t *in,
769 int pri_strength, int sec_strength, int dir,
770 int pri_damping, int sec_damping,
771 int coeff_shift, int block_width,
772 int block_height) {
773 uint16_t *dst16 = (uint16_t *)dest;
774 if (block_width == 8) {
775 filter_block_8x8(/*is_lowbd=*/0, dst16, dstride, in, pri_strength,
776 sec_strength, dir, pri_damping, sec_damping, coeff_shift,
777 block_height, /*enable_primary=*/1,
778 /*enable_secondary=*/0);
779 } else {
780 filter_block_4x4(/*is_lowbd=*/0, dst16, dstride, in, pri_strength,
781 sec_strength, dir, pri_damping, sec_damping, coeff_shift,
782 block_height, /*enable_primary=*/1,
783 /*enable_secondary=*/0);
784 }
785 }
SIMD_FUNC(cdef_filter_16_2)786 void SIMD_FUNC(cdef_filter_16_2)(void *dest, int dstride, const uint16_t *in,
787 int pri_strength, int sec_strength, int dir,
788 int pri_damping, int sec_damping,
789 int coeff_shift, int block_width,
790 int block_height) {
791 uint16_t *dst16 = (uint16_t *)dest;
792 if (block_width == 8) {
793 filter_block_8x8(/*is_lowbd=*/0, dst16, dstride, in, pri_strength,
794 sec_strength, dir, pri_damping, sec_damping, coeff_shift,
795 block_height, /*enable_primary=*/0,
796 /*enable_secondary=*/1);
797 } else {
798 filter_block_4x4(/*is_lowbd=*/0, dst16, dstride, in, pri_strength,
799 sec_strength, dir, pri_damping, sec_damping, coeff_shift,
800 block_height, /*enable_primary=*/0,
801 /*enable_secondary=*/1);
802 }
803 }
804
SIMD_FUNC(cdef_filter_16_3)805 void SIMD_FUNC(cdef_filter_16_3)(void *dest, int dstride, const uint16_t *in,
806 int pri_strength, int sec_strength, int dir,
807 int pri_damping, int sec_damping,
808 int coeff_shift, int block_width,
809 int block_height) {
810 uint16_t *dst16 = (uint16_t *)dest;
811 (void)pri_strength;
812 (void)sec_strength;
813 (void)dir;
814 (void)pri_damping;
815 (void)sec_damping;
816 (void)coeff_shift;
817 (void)block_width;
818 if (block_width == 8) {
819 copy_block_8xh(/*is_lowbd=*/0, dst16, dstride, in, block_height);
820 } else {
821 copy_block_4xh(/*is_lowbd=*/0, dst16, dstride, in, block_height);
822 }
823 }
824
SIMD_FUNC(cdef_copy_rect8_16bit_to_16bit)825 void SIMD_FUNC(cdef_copy_rect8_16bit_to_16bit)(uint16_t *dst, int dstride,
826 const uint16_t *src, int sstride,
827 int width, int height) {
828 int i, j;
829 for (i = 0; i < height; i++) {
830 for (j = 0; j < (width & ~0x7); j += 8) {
831 v128 row = v128_load_unaligned(&src[i * sstride + j]);
832 v128_store_unaligned(&dst[i * dstride + j], row);
833 }
834 for (; j < width; j++) {
835 dst[i * dstride + j] = src[i * sstride + j];
836 }
837 }
838 }
839
840 #undef CDEF_INLINE
841
842 #endif // AOM_AV1_COMMON_CDEF_BLOCK_SIMD_H_
843