1 /*
2 * Copyright (c) 2018 The WebM project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11 #include "./vpx_config.h"
12
13 #include "./vp9_rtcd.h"
14 #include "vpx_dsp/ppc/types_vsx.h"
15
16 // Multiply the packed 16-bit integers in a and b, producing intermediate 32-bit
17 // integers, and return the high 16 bits of the intermediate integers.
18 // (a * b) >> 16
19 // Note: Because this is done in 2 operations, a and b cannot both be UINT16_MIN
vec_mulhi(int16x8_t a,int16x8_t b)20 static INLINE int16x8_t vec_mulhi(int16x8_t a, int16x8_t b) {
21 // madds does ((A * B) >> 15) + C, we need >> 16, so we perform an extra right
22 // shift.
23 return vec_sra(vec_madds(a, b, vec_zeros_s16), vec_ones_u16);
24 }
25
26 // Negate 16-bit integers in a when the corresponding signed 16-bit
27 // integer in b is negative.
vec_sign(int16x8_t a,int16x8_t b)28 static INLINE int16x8_t vec_sign(int16x8_t a, int16x8_t b) {
29 const int16x8_t mask = vec_sra(b, vec_shift_sign_s16);
30 return vec_xor(vec_add(a, mask), mask);
31 }
32
33 // Compare packed 16-bit integers across a, and return the maximum value in
34 // every element. Returns a vector containing the biggest value across vector a.
vec_max_across(int16x8_t a)35 static INLINE int16x8_t vec_max_across(int16x8_t a) {
36 a = vec_max(a, vec_perm(a, a, vec_perm64));
37 a = vec_max(a, vec_perm(a, a, vec_perm32));
38 return vec_max(a, vec_perm(a, a, vec_perm16));
39 }
40
vp9_quantize_fp_vsx(const tran_low_t * coeff_ptr,intptr_t n_coeffs,int skip_block,const int16_t * round_ptr,const int16_t * quant_ptr,tran_low_t * qcoeff_ptr,tran_low_t * dqcoeff_ptr,const int16_t * dequant_ptr,uint16_t * eob_ptr,const int16_t * scan,const int16_t * iscan)41 void vp9_quantize_fp_vsx(const tran_low_t *coeff_ptr, intptr_t n_coeffs,
42 int skip_block, const int16_t *round_ptr,
43 const int16_t *quant_ptr, tran_low_t *qcoeff_ptr,
44 tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr,
45 uint16_t *eob_ptr, const int16_t *scan,
46 const int16_t *iscan) {
47 int16x8_t qcoeff0, qcoeff1, dqcoeff0, dqcoeff1, eob;
48 bool16x8_t zero_coeff0, zero_coeff1;
49
50 int16x8_t round = vec_vsx_ld(0, round_ptr);
51 int16x8_t quant = vec_vsx_ld(0, quant_ptr);
52 int16x8_t dequant = vec_vsx_ld(0, dequant_ptr);
53 int16x8_t coeff0 = vec_vsx_ld(0, coeff_ptr);
54 int16x8_t coeff1 = vec_vsx_ld(16, coeff_ptr);
55 int16x8_t scan0 = vec_vsx_ld(0, iscan);
56 int16x8_t scan1 = vec_vsx_ld(16, iscan);
57
58 (void)scan;
59 (void)skip_block;
60 assert(!skip_block);
61
62 // First set of 8 coeff starts with DC + 7 AC
63 qcoeff0 = vec_mulhi(vec_vaddshs(vec_abs(coeff0), round), quant);
64 zero_coeff0 = vec_cmpeq(qcoeff0, vec_zeros_s16);
65 qcoeff0 = vec_sign(qcoeff0, coeff0);
66 vec_vsx_st(qcoeff0, 0, qcoeff_ptr);
67
68 dqcoeff0 = vec_mladd(qcoeff0, dequant, vec_zeros_s16);
69 vec_vsx_st(dqcoeff0, 0, dqcoeff_ptr);
70
71 // Remove DC value from round and quant
72 round = vec_splat(round, 1);
73 quant = vec_splat(quant, 1);
74
75 // Remove DC value from dequant
76 dequant = vec_splat(dequant, 1);
77
78 // Second set of 8 coeff starts with (all AC)
79 qcoeff1 = vec_mulhi(vec_vaddshs(vec_abs(coeff1), round), quant);
80 zero_coeff1 = vec_cmpeq(qcoeff1, vec_zeros_s16);
81 qcoeff1 = vec_sign(qcoeff1, coeff1);
82 vec_vsx_st(qcoeff1, 16, qcoeff_ptr);
83
84 dqcoeff1 = vec_mladd(qcoeff1, dequant, vec_zeros_s16);
85 vec_vsx_st(dqcoeff1, 16, dqcoeff_ptr);
86
87 eob = vec_max(vec_or(scan0, zero_coeff0), vec_or(scan1, zero_coeff1));
88
89 // We quantize 16 coeff up front (enough for a 4x4) and process 24 coeff per
90 // loop iteration.
91 // for 8x8: 16 + 2 x 24 = 64
92 // for 16x16: 16 + 10 x 24 = 256
93 if (n_coeffs > 16) {
94 int16x8_t coeff2, qcoeff2, dqcoeff2, eob2, scan2;
95 bool16x8_t zero_coeff2;
96
97 int index = 16;
98 int off0 = 32;
99 int off1 = 48;
100 int off2 = 64;
101
102 do {
103 coeff0 = vec_vsx_ld(off0, coeff_ptr);
104 coeff1 = vec_vsx_ld(off1, coeff_ptr);
105 coeff2 = vec_vsx_ld(off2, coeff_ptr);
106 scan0 = vec_vsx_ld(off0, iscan);
107 scan1 = vec_vsx_ld(off1, iscan);
108 scan2 = vec_vsx_ld(off2, iscan);
109
110 qcoeff0 = vec_mulhi(vec_vaddshs(vec_abs(coeff0), round), quant);
111 zero_coeff0 = vec_cmpeq(qcoeff0, vec_zeros_s16);
112 qcoeff0 = vec_sign(qcoeff0, coeff0);
113 vec_vsx_st(qcoeff0, off0, qcoeff_ptr);
114 dqcoeff0 = vec_mladd(qcoeff0, dequant, vec_zeros_s16);
115 vec_vsx_st(dqcoeff0, off0, dqcoeff_ptr);
116
117 qcoeff1 = vec_mulhi(vec_vaddshs(vec_abs(coeff1), round), quant);
118 zero_coeff1 = vec_cmpeq(qcoeff1, vec_zeros_s16);
119 qcoeff1 = vec_sign(qcoeff1, coeff1);
120 vec_vsx_st(qcoeff1, off1, qcoeff_ptr);
121 dqcoeff1 = vec_mladd(qcoeff1, dequant, vec_zeros_s16);
122 vec_vsx_st(dqcoeff1, off1, dqcoeff_ptr);
123
124 qcoeff2 = vec_mulhi(vec_vaddshs(vec_abs(coeff2), round), quant);
125 zero_coeff2 = vec_cmpeq(qcoeff2, vec_zeros_s16);
126 qcoeff2 = vec_sign(qcoeff2, coeff2);
127 vec_vsx_st(qcoeff2, off2, qcoeff_ptr);
128 dqcoeff2 = vec_mladd(qcoeff2, dequant, vec_zeros_s16);
129 vec_vsx_st(dqcoeff2, off2, dqcoeff_ptr);
130
131 eob = vec_max(eob, vec_or(scan0, zero_coeff0));
132 eob2 = vec_max(vec_or(scan1, zero_coeff1), vec_or(scan2, zero_coeff2));
133 eob = vec_max(eob, eob2);
134
135 index += 24;
136 off0 += 48;
137 off1 += 48;
138 off2 += 48;
139 } while (index < n_coeffs);
140 }
141
142 eob = vec_max_across(eob);
143 *eob_ptr = eob[0] + 1;
144 }
145
146 // Sets the value of a 32-bit integers to 1 when the corresponding value in a is
147 // negative.
vec_is_neg(int32x4_t a)148 static INLINE int32x4_t vec_is_neg(int32x4_t a) {
149 return vec_sr(a, vec_shift_sign_s32);
150 }
151
152 // DeQuantization function used for 32x32 blocks. Quantized coeff of 32x32
153 // blocks are twice as big as for other block sizes. As such, using
154 // vec_mladd results in overflow.
dequantize_coeff_32(int16x8_t qcoeff,int16x8_t dequant)155 static INLINE int16x8_t dequantize_coeff_32(int16x8_t qcoeff,
156 int16x8_t dequant) {
157 int32x4_t dqcoeffe = vec_mule(qcoeff, dequant);
158 int32x4_t dqcoeffo = vec_mulo(qcoeff, dequant);
159 // Add 1 if negative to round towards zero because the C uses division.
160 dqcoeffe = vec_add(dqcoeffe, vec_is_neg(dqcoeffe));
161 dqcoeffo = vec_add(dqcoeffo, vec_is_neg(dqcoeffo));
162 dqcoeffe = vec_sra(dqcoeffe, vec_ones_u32);
163 dqcoeffo = vec_sra(dqcoeffo, vec_ones_u32);
164 return (int16x8_t)vec_perm(dqcoeffe, dqcoeffo, vec_perm_odd_even_pack);
165 }
166
vp9_quantize_fp_32x32_vsx(const tran_low_t * coeff_ptr,intptr_t n_coeffs,int skip_block,const int16_t * round_ptr,const int16_t * quant_ptr,tran_low_t * qcoeff_ptr,tran_low_t * dqcoeff_ptr,const int16_t * dequant_ptr,uint16_t * eob_ptr,const int16_t * scan,const int16_t * iscan)167 void vp9_quantize_fp_32x32_vsx(const tran_low_t *coeff_ptr, intptr_t n_coeffs,
168 int skip_block, const int16_t *round_ptr,
169 const int16_t *quant_ptr, tran_low_t *qcoeff_ptr,
170 tran_low_t *dqcoeff_ptr,
171 const int16_t *dequant_ptr, uint16_t *eob_ptr,
172 const int16_t *scan, const int16_t *iscan) {
173 // In stage 1, we quantize 16 coeffs (DC + 15 AC)
174 // In stage 2, we loop 42 times and quantize 24 coeffs per iteration
175 // (32 * 32 - 16) / 24 = 42
176 int num_itr = 42;
177 // Offsets are in bytes, 16 coeffs = 32 bytes
178 int off0 = 32;
179 int off1 = 48;
180 int off2 = 64;
181
182 int16x8_t qcoeff0, qcoeff1, dqcoeff0, dqcoeff1, eob;
183 bool16x8_t mask0, mask1, zero_coeff0, zero_coeff1;
184
185 int16x8_t round = vec_vsx_ld(0, round_ptr);
186 int16x8_t quant = vec_vsx_ld(0, quant_ptr);
187 int16x8_t dequant = vec_vsx_ld(0, dequant_ptr);
188 int16x8_t coeff0 = vec_vsx_ld(0, coeff_ptr);
189 int16x8_t coeff1 = vec_vsx_ld(16, coeff_ptr);
190 int16x8_t scan0 = vec_vsx_ld(0, iscan);
191 int16x8_t scan1 = vec_vsx_ld(16, iscan);
192 int16x8_t thres = vec_sra(dequant, vec_splats((uint16_t)2));
193 int16x8_t abs_coeff0 = vec_abs(coeff0);
194 int16x8_t abs_coeff1 = vec_abs(coeff1);
195
196 (void)scan;
197 (void)skip_block;
198 (void)n_coeffs;
199 assert(!skip_block);
200
201 mask0 = vec_cmpge(abs_coeff0, thres);
202 round = vec_sra(vec_add(round, vec_ones_s16), vec_ones_u16);
203 // First set of 8 coeff starts with DC + 7 AC
204 qcoeff0 = vec_madds(vec_vaddshs(abs_coeff0, round), quant, vec_zeros_s16);
205 qcoeff0 = vec_and(qcoeff0, mask0);
206 zero_coeff0 = vec_cmpeq(qcoeff0, vec_zeros_s16);
207 qcoeff0 = vec_sign(qcoeff0, coeff0);
208 vec_vsx_st(qcoeff0, 0, qcoeff_ptr);
209
210 dqcoeff0 = dequantize_coeff_32(qcoeff0, dequant);
211 vec_vsx_st(dqcoeff0, 0, dqcoeff_ptr);
212
213 // Remove DC value from thres, round, quant and dequant
214 thres = vec_splat(thres, 1);
215 round = vec_splat(round, 1);
216 quant = vec_splat(quant, 1);
217 dequant = vec_splat(dequant, 1);
218
219 mask1 = vec_cmpge(abs_coeff1, thres);
220
221 // Second set of 8 coeff starts with (all AC)
222 qcoeff1 =
223 vec_madds(vec_vaddshs(vec_abs(coeff1), round), quant, vec_zeros_s16);
224 qcoeff1 = vec_and(qcoeff1, mask1);
225 zero_coeff1 = vec_cmpeq(qcoeff1, vec_zeros_s16);
226 qcoeff1 = vec_sign(qcoeff1, coeff1);
227 vec_vsx_st(qcoeff1, 16, qcoeff_ptr);
228
229 dqcoeff1 = dequantize_coeff_32(qcoeff1, dequant);
230 vec_vsx_st(dqcoeff1, 16, dqcoeff_ptr);
231
232 eob = vec_max(vec_or(scan0, zero_coeff0), vec_or(scan1, zero_coeff1));
233
234 do {
235 int16x8_t coeff2, abs_coeff2, qcoeff2, dqcoeff2, eob2, scan2;
236 bool16x8_t zero_coeff2, mask2;
237 coeff0 = vec_vsx_ld(off0, coeff_ptr);
238 coeff1 = vec_vsx_ld(off1, coeff_ptr);
239 coeff2 = vec_vsx_ld(off2, coeff_ptr);
240 scan0 = vec_vsx_ld(off0, iscan);
241 scan1 = vec_vsx_ld(off1, iscan);
242 scan2 = vec_vsx_ld(off2, iscan);
243
244 abs_coeff0 = vec_abs(coeff0);
245 abs_coeff1 = vec_abs(coeff1);
246 abs_coeff2 = vec_abs(coeff2);
247
248 qcoeff0 = vec_madds(vec_vaddshs(abs_coeff0, round), quant, vec_zeros_s16);
249 qcoeff1 = vec_madds(vec_vaddshs(abs_coeff1, round), quant, vec_zeros_s16);
250 qcoeff2 = vec_madds(vec_vaddshs(abs_coeff2, round), quant, vec_zeros_s16);
251
252 mask0 = vec_cmpge(abs_coeff0, thres);
253 mask1 = vec_cmpge(abs_coeff1, thres);
254 mask2 = vec_cmpge(abs_coeff2, thres);
255
256 qcoeff0 = vec_and(qcoeff0, mask0);
257 qcoeff1 = vec_and(qcoeff1, mask1);
258 qcoeff2 = vec_and(qcoeff2, mask2);
259
260 zero_coeff0 = vec_cmpeq(qcoeff0, vec_zeros_s16);
261 zero_coeff1 = vec_cmpeq(qcoeff1, vec_zeros_s16);
262 zero_coeff2 = vec_cmpeq(qcoeff2, vec_zeros_s16);
263
264 qcoeff0 = vec_sign(qcoeff0, coeff0);
265 qcoeff1 = vec_sign(qcoeff1, coeff1);
266 qcoeff2 = vec_sign(qcoeff2, coeff2);
267
268 vec_vsx_st(qcoeff0, off0, qcoeff_ptr);
269 vec_vsx_st(qcoeff1, off1, qcoeff_ptr);
270 vec_vsx_st(qcoeff2, off2, qcoeff_ptr);
271
272 dqcoeff0 = dequantize_coeff_32(qcoeff0, dequant);
273 dqcoeff1 = dequantize_coeff_32(qcoeff1, dequant);
274 dqcoeff2 = dequantize_coeff_32(qcoeff2, dequant);
275
276 vec_vsx_st(dqcoeff0, off0, dqcoeff_ptr);
277 vec_vsx_st(dqcoeff1, off1, dqcoeff_ptr);
278 vec_vsx_st(dqcoeff2, off2, dqcoeff_ptr);
279
280 eob = vec_max(eob, vec_or(scan0, zero_coeff0));
281 eob2 = vec_max(vec_or(scan1, zero_coeff1), vec_or(scan2, zero_coeff2));
282 eob = vec_max(eob, eob2);
283
284 off0 += 48;
285 off1 += 48;
286 off2 += 48;
287 num_itr--;
288 } while (num_itr != 0);
289
290 eob = vec_max_across(eob);
291 *eob_ptr = eob[0] + 1;
292 }
293