1 /*
2 * Copyright (c) 2018 The WebM project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11 #include <assert.h>
12
13 #include "./vpx_dsp_rtcd.h"
14 #include "vpx_dsp/ppc/types_vsx.h"
15
16 // Negate 16-bit integers in a when the corresponding signed 16-bit
17 // integer in b is negative.
vec_sign(int16x8_t a,int16x8_t b)18 static INLINE int16x8_t vec_sign(int16x8_t a, int16x8_t b) {
19 const int16x8_t mask = vec_sra(b, vec_shift_sign_s16);
20 return vec_xor(vec_add(a, mask), mask);
21 }
22
23 // Sets the value of a 32-bit integers to 1 when the corresponding value in a is
24 // negative.
vec_is_neg(int32x4_t a)25 static INLINE int32x4_t vec_is_neg(int32x4_t a) {
26 return vec_sr(a, vec_shift_sign_s32);
27 }
28
29 // Multiply the packed 16-bit integers in a and b, producing intermediate 32-bit
30 // integers, and return the high 16 bits of the intermediate integers.
31 // (a * b) >> 16
vec_mulhi(int16x8_t a,int16x8_t b)32 static INLINE int16x8_t vec_mulhi(int16x8_t a, int16x8_t b) {
33 // madds does ((A * B) >>15) + C, we need >> 16, so we perform an extra right
34 // shift.
35 return vec_sra(vec_madds(a, b, vec_zeros_s16), vec_ones_u16);
36 }
37
38 // Quantization function used for 4x4, 8x8 and 16x16 blocks.
quantize_coeff(int16x8_t coeff,int16x8_t coeff_abs,int16x8_t round,int16x8_t quant,int16x8_t quant_shift,bool16x8_t mask)39 static INLINE int16x8_t quantize_coeff(int16x8_t coeff, int16x8_t coeff_abs,
40 int16x8_t round, int16x8_t quant,
41 int16x8_t quant_shift, bool16x8_t mask) {
42 const int16x8_t rounded = vec_vaddshs(coeff_abs, round);
43 int16x8_t qcoeff = vec_mulhi(rounded, quant);
44 qcoeff = vec_add(qcoeff, rounded);
45 qcoeff = vec_mulhi(qcoeff, quant_shift);
46 qcoeff = vec_sign(qcoeff, coeff);
47 return vec_and(qcoeff, mask);
48 }
49
50 // Quantization function used for 32x32 blocks.
quantize_coeff_32(int16x8_t coeff,int16x8_t coeff_abs,int16x8_t round,int16x8_t quant,int16x8_t quant_shift,bool16x8_t mask)51 static INLINE int16x8_t quantize_coeff_32(int16x8_t coeff, int16x8_t coeff_abs,
52 int16x8_t round, int16x8_t quant,
53 int16x8_t quant_shift,
54 bool16x8_t mask) {
55 const int16x8_t rounded = vec_vaddshs(coeff_abs, round);
56 int16x8_t qcoeff = vec_mulhi(rounded, quant);
57 qcoeff = vec_add(qcoeff, rounded);
58 // 32x32 blocks require an extra multiplication by 2, this compensates for the
59 // extra right shift added in vec_mulhi, as such vec_madds can be used
60 // directly instead of vec_mulhi (((a * b) >> 15) >> 1) << 1 == (a * b >> 15)
61 qcoeff = vec_madds(qcoeff, quant_shift, vec_zeros_s16);
62 qcoeff = vec_sign(qcoeff, coeff);
63 return vec_and(qcoeff, mask);
64 }
65
66 // DeQuantization function used for 32x32 blocks. Quantized coeff of 32x32
67 // blocks are twice as big as for other block sizes. As such, using
68 // vec_mladd results in overflow.
dequantize_coeff_32(int16x8_t qcoeff,int16x8_t dequant)69 static INLINE int16x8_t dequantize_coeff_32(int16x8_t qcoeff,
70 int16x8_t dequant) {
71 int32x4_t dqcoeffe = vec_mule(qcoeff, dequant);
72 int32x4_t dqcoeffo = vec_mulo(qcoeff, dequant);
73 // Add 1 if negative to round towards zero because the C uses division.
74 dqcoeffe = vec_add(dqcoeffe, vec_is_neg(dqcoeffe));
75 dqcoeffo = vec_add(dqcoeffo, vec_is_neg(dqcoeffo));
76 dqcoeffe = vec_sra(dqcoeffe, vec_ones_u32);
77 dqcoeffo = vec_sra(dqcoeffo, vec_ones_u32);
78 return (int16x8_t)vec_perm(dqcoeffe, dqcoeffo, vec_perm_odd_even_pack);
79 }
80
nonzero_scanindex(int16x8_t qcoeff,bool16x8_t mask,const int16_t * iscan_ptr,int index)81 static INLINE int16x8_t nonzero_scanindex(int16x8_t qcoeff, bool16x8_t mask,
82 const int16_t *iscan_ptr, int index) {
83 int16x8_t scan = vec_vsx_ld(index, iscan_ptr);
84 bool16x8_t zero_coeff = vec_cmpeq(qcoeff, vec_zeros_s16);
85 scan = vec_sub(scan, mask);
86 return vec_andc(scan, zero_coeff);
87 }
88
89 // Compare packed 16-bit integers across a, and return the maximum value in
90 // every element. Returns a vector containing the biggest value across vector a.
vec_max_across(int16x8_t a)91 static INLINE int16x8_t vec_max_across(int16x8_t a) {
92 a = vec_max(a, vec_perm(a, a, vec_perm64));
93 a = vec_max(a, vec_perm(a, a, vec_perm32));
94 return vec_max(a, vec_perm(a, a, vec_perm16));
95 }
96
vpx_quantize_b_vsx(const tran_low_t * coeff_ptr,intptr_t n_coeffs,int skip_block,const int16_t * zbin_ptr,const int16_t * round_ptr,const int16_t * quant_ptr,const int16_t * quant_shift_ptr,tran_low_t * qcoeff_ptr,tran_low_t * dqcoeff_ptr,const int16_t * dequant_ptr,uint16_t * eob_ptr,const int16_t * scan_ptr,const int16_t * iscan_ptr)97 void vpx_quantize_b_vsx(const tran_low_t *coeff_ptr, intptr_t n_coeffs,
98 int skip_block, const int16_t *zbin_ptr,
99 const int16_t *round_ptr, const int16_t *quant_ptr,
100 const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr,
101 tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr,
102 uint16_t *eob_ptr, const int16_t *scan_ptr,
103 const int16_t *iscan_ptr) {
104 int16x8_t qcoeff0, qcoeff1, dqcoeff0, dqcoeff1, eob;
105 bool16x8_t zero_mask0, zero_mask1;
106
107 // First set of 8 coeff starts with DC + 7 AC
108 int16x8_t zbin = vec_vsx_ld(0, zbin_ptr);
109 int16x8_t round = vec_vsx_ld(0, round_ptr);
110 int16x8_t quant = vec_vsx_ld(0, quant_ptr);
111 int16x8_t dequant = vec_vsx_ld(0, dequant_ptr);
112 int16x8_t quant_shift = vec_vsx_ld(0, quant_shift_ptr);
113
114 int16x8_t coeff0 = vec_vsx_ld(0, coeff_ptr);
115 int16x8_t coeff1 = vec_vsx_ld(16, coeff_ptr);
116
117 int16x8_t coeff0_abs = vec_abs(coeff0);
118 int16x8_t coeff1_abs = vec_abs(coeff1);
119
120 zero_mask0 = vec_cmpge(coeff0_abs, zbin);
121 zbin = vec_splat(zbin, 1);
122 zero_mask1 = vec_cmpge(coeff1_abs, zbin);
123
124 (void)scan_ptr;
125 (void)skip_block;
126 assert(!skip_block);
127
128 qcoeff0 =
129 quantize_coeff(coeff0, coeff0_abs, round, quant, quant_shift, zero_mask0);
130 vec_vsx_st(qcoeff0, 0, qcoeff_ptr);
131 round = vec_splat(round, 1);
132 quant = vec_splat(quant, 1);
133 quant_shift = vec_splat(quant_shift, 1);
134 qcoeff1 =
135 quantize_coeff(coeff1, coeff1_abs, round, quant, quant_shift, zero_mask1);
136 vec_vsx_st(qcoeff1, 16, qcoeff_ptr);
137
138 dqcoeff0 = vec_mladd(qcoeff0, dequant, vec_zeros_s16);
139 vec_vsx_st(dqcoeff0, 0, dqcoeff_ptr);
140 dequant = vec_splat(dequant, 1);
141 dqcoeff1 = vec_mladd(qcoeff1, dequant, vec_zeros_s16);
142 vec_vsx_st(dqcoeff1, 16, dqcoeff_ptr);
143
144 eob = vec_max(nonzero_scanindex(qcoeff0, zero_mask0, iscan_ptr, 0),
145 nonzero_scanindex(qcoeff1, zero_mask1, iscan_ptr, 16));
146
147 if (n_coeffs > 16) {
148 int index = 16;
149 int off0 = 32;
150 int off1 = 48;
151 int off2 = 64;
152 do {
153 int16x8_t coeff2, coeff2_abs, qcoeff2, dqcoeff2, eob2;
154 bool16x8_t zero_mask2;
155 coeff0 = vec_vsx_ld(off0, coeff_ptr);
156 coeff1 = vec_vsx_ld(off1, coeff_ptr);
157 coeff2 = vec_vsx_ld(off2, coeff_ptr);
158 coeff0_abs = vec_abs(coeff0);
159 coeff1_abs = vec_abs(coeff1);
160 coeff2_abs = vec_abs(coeff2);
161 zero_mask0 = vec_cmpge(coeff0_abs, zbin);
162 zero_mask1 = vec_cmpge(coeff1_abs, zbin);
163 zero_mask2 = vec_cmpge(coeff2_abs, zbin);
164 qcoeff0 = quantize_coeff(coeff0, coeff0_abs, round, quant, quant_shift,
165 zero_mask0);
166 qcoeff1 = quantize_coeff(coeff1, coeff1_abs, round, quant, quant_shift,
167 zero_mask1);
168 qcoeff2 = quantize_coeff(coeff2, coeff2_abs, round, quant, quant_shift,
169 zero_mask2);
170 vec_vsx_st(qcoeff0, off0, qcoeff_ptr);
171 vec_vsx_st(qcoeff1, off1, qcoeff_ptr);
172 vec_vsx_st(qcoeff2, off2, qcoeff_ptr);
173
174 dqcoeff0 = vec_mladd(qcoeff0, dequant, vec_zeros_s16);
175 dqcoeff1 = vec_mladd(qcoeff1, dequant, vec_zeros_s16);
176 dqcoeff2 = vec_mladd(qcoeff2, dequant, vec_zeros_s16);
177
178 vec_vsx_st(dqcoeff0, off0, dqcoeff_ptr);
179 vec_vsx_st(dqcoeff1, off1, dqcoeff_ptr);
180 vec_vsx_st(dqcoeff2, off2, dqcoeff_ptr);
181
182 eob =
183 vec_max(eob, nonzero_scanindex(qcoeff0, zero_mask0, iscan_ptr, off0));
184 eob2 = vec_max(nonzero_scanindex(qcoeff1, zero_mask1, iscan_ptr, off1),
185 nonzero_scanindex(qcoeff2, zero_mask2, iscan_ptr, off2));
186 eob = vec_max(eob, eob2);
187
188 index += 24;
189 off0 += 48;
190 off1 += 48;
191 off2 += 48;
192 } while (index < n_coeffs);
193 }
194
195 eob = vec_max_across(eob);
196 *eob_ptr = eob[0];
197 }
198
vpx_quantize_b_32x32_vsx(const tran_low_t * coeff_ptr,intptr_t n_coeffs,int skip_block,const int16_t * zbin_ptr,const int16_t * round_ptr,const int16_t * quant_ptr,const int16_t * quant_shift_ptr,tran_low_t * qcoeff_ptr,tran_low_t * dqcoeff_ptr,const int16_t * dequant_ptr,uint16_t * eob_ptr,const int16_t * scan_ptr,const int16_t * iscan_ptr)199 void vpx_quantize_b_32x32_vsx(
200 const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block,
201 const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr,
202 const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr,
203 tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr,
204 const int16_t *scan_ptr, const int16_t *iscan_ptr) {
205 // In stage 1, we quantize 16 coeffs (DC + 15 AC)
206 // In stage 2, we loop 42 times and quantize 24 coeffs per iteration
207 // (32 * 32 - 16) / 24 = 42
208 int num_itr = 42;
209 // Offsets are in bytes, 16 coeffs = 32 bytes
210 int off0 = 32;
211 int off1 = 48;
212 int off2 = 64;
213
214 int16x8_t qcoeff0, qcoeff1, eob;
215 bool16x8_t zero_mask0, zero_mask1;
216
217 int16x8_t zbin = vec_vsx_ld(0, zbin_ptr);
218 int16x8_t round = vec_vsx_ld(0, round_ptr);
219 int16x8_t quant = vec_vsx_ld(0, quant_ptr);
220 int16x8_t dequant = vec_vsx_ld(0, dequant_ptr);
221 int16x8_t quant_shift = vec_vsx_ld(0, quant_shift_ptr);
222
223 int16x8_t coeff0 = vec_vsx_ld(0, coeff_ptr);
224 int16x8_t coeff1 = vec_vsx_ld(16, coeff_ptr);
225
226 int16x8_t coeff0_abs = vec_abs(coeff0);
227 int16x8_t coeff1_abs = vec_abs(coeff1);
228
229 (void)scan_ptr;
230 (void)skip_block;
231 (void)n_coeffs;
232 assert(!skip_block);
233
234 // 32x32 quantization requires that zbin and round be divided by 2
235 zbin = vec_sra(vec_add(zbin, vec_ones_s16), vec_ones_u16);
236 round = vec_sra(vec_add(round, vec_ones_s16), vec_ones_u16);
237
238 zero_mask0 = vec_cmpge(coeff0_abs, zbin);
239 zbin = vec_splat(zbin, 1); // remove DC from zbin
240 zero_mask1 = vec_cmpge(coeff1_abs, zbin);
241
242 qcoeff0 = quantize_coeff_32(coeff0, coeff0_abs, round, quant, quant_shift,
243 zero_mask0);
244 round = vec_splat(round, 1); // remove DC from round
245 quant = vec_splat(quant, 1); // remove DC from quant
246 quant_shift = vec_splat(quant_shift, 1); // remove DC from quant_shift
247 qcoeff1 = quantize_coeff_32(coeff1, coeff1_abs, round, quant, quant_shift,
248 zero_mask1);
249
250 vec_vsx_st(qcoeff0, 0, qcoeff_ptr);
251 vec_vsx_st(qcoeff1, 16, qcoeff_ptr);
252
253 vec_vsx_st(dequantize_coeff_32(qcoeff0, dequant), 0, dqcoeff_ptr);
254 dequant = vec_splat(dequant, 1); // remove DC from dequant
255 vec_vsx_st(dequantize_coeff_32(qcoeff1, dequant), 16, dqcoeff_ptr);
256
257 eob = vec_max(nonzero_scanindex(qcoeff0, zero_mask0, iscan_ptr, 0),
258 nonzero_scanindex(qcoeff1, zero_mask1, iscan_ptr, 16));
259
260 do {
261 int16x8_t coeff2, coeff2_abs, qcoeff2, eob2;
262 bool16x8_t zero_mask2;
263
264 coeff0 = vec_vsx_ld(off0, coeff_ptr);
265 coeff1 = vec_vsx_ld(off1, coeff_ptr);
266 coeff2 = vec_vsx_ld(off2, coeff_ptr);
267
268 coeff0_abs = vec_abs(coeff0);
269 coeff1_abs = vec_abs(coeff1);
270 coeff2_abs = vec_abs(coeff2);
271
272 zero_mask0 = vec_cmpge(coeff0_abs, zbin);
273 zero_mask1 = vec_cmpge(coeff1_abs, zbin);
274 zero_mask2 = vec_cmpge(coeff2_abs, zbin);
275
276 qcoeff0 = quantize_coeff_32(coeff0, coeff0_abs, round, quant, quant_shift,
277 zero_mask0);
278 qcoeff1 = quantize_coeff_32(coeff1, coeff1_abs, round, quant, quant_shift,
279 zero_mask1);
280 qcoeff2 = quantize_coeff_32(coeff2, coeff2_abs, round, quant, quant_shift,
281 zero_mask2);
282
283 vec_vsx_st(qcoeff0, off0, qcoeff_ptr);
284 vec_vsx_st(qcoeff1, off1, qcoeff_ptr);
285 vec_vsx_st(qcoeff2, off2, qcoeff_ptr);
286
287 vec_vsx_st(dequantize_coeff_32(qcoeff0, dequant), off0, dqcoeff_ptr);
288 vec_vsx_st(dequantize_coeff_32(qcoeff1, dequant), off1, dqcoeff_ptr);
289 vec_vsx_st(dequantize_coeff_32(qcoeff2, dequant), off2, dqcoeff_ptr);
290
291 eob = vec_max(eob, nonzero_scanindex(qcoeff0, zero_mask0, iscan_ptr, off0));
292 eob2 = vec_max(nonzero_scanindex(qcoeff1, zero_mask1, iscan_ptr, off1),
293 nonzero_scanindex(qcoeff2, zero_mask2, iscan_ptr, off2));
294 eob = vec_max(eob, eob2);
295
296 // 24 int16_t is 48 bytes
297 off0 += 48;
298 off1 += 48;
299 off2 += 48;
300 num_itr--;
301 } while (num_itr != 0);
302
303 eob = vec_max_across(eob);
304 *eob_ptr = eob[0];
305 }
306