1 /*
2 * Copyright (c) 2017 The WebM project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11 #include <assert.h>
12 #include <immintrin.h> // AVX2
13
14 #include "./vp9_rtcd.h"
15 #include "vpx/vpx_integer.h"
16 #include "vpx_dsp/vpx_dsp_common.h"
17 #include "vpx_dsp/x86/bitdepth_conversion_avx2.h"
18 #include "vpx_dsp/x86/quantize_sse2.h"
19
20 // Zero fill 8 positions in the output buffer.
store_zero_tran_low(tran_low_t * a)21 static INLINE void store_zero_tran_low(tran_low_t *a) {
22 const __m256i zero = _mm256_setzero_si256();
23 #if CONFIG_VP9_HIGHBITDEPTH
24 _mm256_storeu_si256((__m256i *)(a), zero);
25 _mm256_storeu_si256((__m256i *)(a + 8), zero);
26 #else
27 _mm256_storeu_si256((__m256i *)(a), zero);
28 #endif
29 }
30
scan_eob_256(const __m256i * iscan_ptr,__m256i * coeff256)31 static INLINE __m256i scan_eob_256(const __m256i *iscan_ptr,
32 __m256i *coeff256) {
33 const __m256i iscan = _mm256_loadu_si256(iscan_ptr);
34 const __m256i zero256 = _mm256_setzero_si256();
35 #if CONFIG_VP9_HIGHBITDEPTH
36 // The _mm256_packs_epi32() in load_tran_low() packs the 64 bit coeff as
37 // B1 A1 B0 A0. Shuffle to B1 B0 A1 A0 in order to scan eob correctly.
38 const __m256i _coeff256 = _mm256_permute4x64_epi64(*coeff256, 0xd8);
39 const __m256i zero_coeff0 = _mm256_cmpeq_epi16(_coeff256, zero256);
40 #else
41 const __m256i zero_coeff0 = _mm256_cmpeq_epi16(*coeff256, zero256);
42 #endif
43 const __m256i nzero_coeff0 = _mm256_cmpeq_epi16(zero_coeff0, zero256);
44 // Add one to convert from indices to counts
45 const __m256i iscan_plus_one = _mm256_sub_epi16(iscan, nzero_coeff0);
46 return _mm256_and_si256(iscan_plus_one, nzero_coeff0);
47 }
48
vp9_quantize_fp_avx2(const tran_low_t * coeff_ptr,intptr_t n_coeffs,int skip_block,const int16_t * round_ptr,const int16_t * quant_ptr,tran_low_t * qcoeff_ptr,tran_low_t * dqcoeff_ptr,const int16_t * dequant_ptr,uint16_t * eob_ptr,const int16_t * scan,const int16_t * iscan)49 void vp9_quantize_fp_avx2(const tran_low_t *coeff_ptr, intptr_t n_coeffs,
50 int skip_block, const int16_t *round_ptr,
51 const int16_t *quant_ptr, tran_low_t *qcoeff_ptr,
52 tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr,
53 uint16_t *eob_ptr, const int16_t *scan,
54 const int16_t *iscan) {
55 __m128i eob;
56 __m256i round256, quant256, dequant256;
57 __m256i eob256, thr256;
58
59 (void)scan;
60 (void)skip_block;
61 assert(!skip_block);
62
63 coeff_ptr += n_coeffs;
64 iscan += n_coeffs;
65 qcoeff_ptr += n_coeffs;
66 dqcoeff_ptr += n_coeffs;
67 n_coeffs = -n_coeffs;
68
69 {
70 __m256i coeff256;
71
72 // Setup global values
73 {
74 const __m128i round = _mm_load_si128((const __m128i *)round_ptr);
75 const __m128i quant = _mm_load_si128((const __m128i *)quant_ptr);
76 const __m128i dequant = _mm_load_si128((const __m128i *)dequant_ptr);
77 round256 = _mm256_castsi128_si256(round);
78 round256 = _mm256_permute4x64_epi64(round256, 0x54);
79
80 quant256 = _mm256_castsi128_si256(quant);
81 quant256 = _mm256_permute4x64_epi64(quant256, 0x54);
82
83 dequant256 = _mm256_castsi128_si256(dequant);
84 dequant256 = _mm256_permute4x64_epi64(dequant256, 0x54);
85 }
86
87 {
88 __m256i qcoeff256;
89 __m256i qtmp256;
90 coeff256 = load_tran_low(coeff_ptr + n_coeffs);
91 qcoeff256 = _mm256_abs_epi16(coeff256);
92 qcoeff256 = _mm256_adds_epi16(qcoeff256, round256);
93 qtmp256 = _mm256_mulhi_epi16(qcoeff256, quant256);
94 qcoeff256 = _mm256_sign_epi16(qtmp256, coeff256);
95 store_tran_low(qcoeff256, qcoeff_ptr + n_coeffs);
96 coeff256 = _mm256_mullo_epi16(qcoeff256, dequant256);
97 store_tran_low(coeff256, dqcoeff_ptr + n_coeffs);
98 }
99
100 eob256 = scan_eob_256((const __m256i *)(iscan + n_coeffs), &coeff256);
101 n_coeffs += 8 * 2;
102 }
103
104 // remove dc constants
105 dequant256 = _mm256_permute2x128_si256(dequant256, dequant256, 0x31);
106 quant256 = _mm256_permute2x128_si256(quant256, quant256, 0x31);
107 round256 = _mm256_permute2x128_si256(round256, round256, 0x31);
108
109 thr256 = _mm256_srai_epi16(dequant256, 1);
110
111 // AC only loop
112 while (n_coeffs < 0) {
113 __m256i coeff256 = load_tran_low(coeff_ptr + n_coeffs);
114 __m256i qcoeff256 = _mm256_abs_epi16(coeff256);
115 int32_t nzflag =
116 _mm256_movemask_epi8(_mm256_cmpgt_epi16(qcoeff256, thr256));
117
118 if (nzflag) {
119 __m256i qtmp256;
120 qcoeff256 = _mm256_adds_epi16(qcoeff256, round256);
121 qtmp256 = _mm256_mulhi_epi16(qcoeff256, quant256);
122 qcoeff256 = _mm256_sign_epi16(qtmp256, coeff256);
123 store_tran_low(qcoeff256, qcoeff_ptr + n_coeffs);
124 coeff256 = _mm256_mullo_epi16(qcoeff256, dequant256);
125 store_tran_low(coeff256, dqcoeff_ptr + n_coeffs);
126 eob256 = _mm256_max_epi16(
127 eob256, scan_eob_256((const __m256i *)(iscan + n_coeffs), &coeff256));
128 } else {
129 store_zero_tran_low(qcoeff_ptr + n_coeffs);
130 store_zero_tran_low(dqcoeff_ptr + n_coeffs);
131 }
132 n_coeffs += 8 * 2;
133 }
134
135 eob = _mm_max_epi16(_mm256_castsi256_si128(eob256),
136 _mm256_extracti128_si256(eob256, 1));
137
138 *eob_ptr = accumulate_eob(eob);
139 }
140