1 /*
2 * Copyright © 2018-2019, VideoLAN and dav1d authors
3 * Copyright © 2018-2019, Two Orioles, LLC
4 * All rights reserved.
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions are met:
8 *
9 * 1. Redistributions of source code must retain the above copyright notice, this
10 * list of conditions and the following disclaimer.
11 *
12 * 2. Redistributions in binary form must reproduce the above copyright notice,
13 * this list of conditions and the following disclaimer in the documentation
14 * and/or other materials provided with the distribution.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
18 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
19 * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
20 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
21 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
22 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
23 * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
24 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
25 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26 */
27
28 #include "config.h"
29
30 #include <stddef.h>
31 #include <stdint.h>
32 #include <string.h>
33
34 #include "common/attributes.h"
35 #include "common/intops.h"
36
37 #include "src/itx.h"
38 #include "src/itx_1d.h"
39
40 static NOINLINE void
inv_txfm_add_c(pixel * dst,const ptrdiff_t stride,coef * const coeff,const int eob,const int w,const int h,const int shift,const itx_1d_fn first_1d_fn,const itx_1d_fn second_1d_fn,const int has_dconly HIGHBD_DECL_SUFFIX)41 inv_txfm_add_c(pixel *dst, const ptrdiff_t stride, coef *const coeff,
42 const int eob, const int w, const int h, const int shift,
43 const itx_1d_fn first_1d_fn, const itx_1d_fn second_1d_fn,
44 const int has_dconly HIGHBD_DECL_SUFFIX)
45 {
46 assert(w >= 4 && w <= 64);
47 assert(h >= 4 && h <= 64);
48 assert(eob >= 0);
49
50 const int is_rect2 = w * 2 == h || h * 2 == w;
51 const int rnd = (1 << shift) >> 1;
52
53 if (eob < has_dconly) {
54 int dc = coeff[0];
55 coeff[0] = 0;
56 if (is_rect2)
57 dc = (dc * 181 + 128) >> 8;
58 dc = (dc * 181 + 128) >> 8;
59 dc = (dc + rnd) >> shift;
60 dc = (dc * 181 + 128 + 2048) >> 12;
61 for (int y = 0; y < h; y++, dst += PXSTRIDE(stride))
62 for (int x = 0; x < w; x++)
63 dst[x] = iclip_pixel(dst[x] + dc);
64 return;
65 }
66
67 const int sh = imin(h, 32), sw = imin(w, 32);
68 #if BITDEPTH == 8
69 const int row_clip_min = INT16_MIN;
70 const int col_clip_min = INT16_MIN;
71 #else
72 const int row_clip_min = (int) ((unsigned) ~bitdepth_max << 7);
73 const int col_clip_min = (int) ((unsigned) ~bitdepth_max << 5);
74 #endif
75 const int row_clip_max = ~row_clip_min;
76 const int col_clip_max = ~col_clip_min;
77
78 int32_t tmp[64 * 64], *c = tmp;
79 for (int y = 0; y < sh; y++, c += w) {
80 if (is_rect2)
81 for (int x = 0; x < sw; x++)
82 c[x] = (coeff[y + x * sh] * 181 + 128) >> 8;
83 else
84 for (int x = 0; x < sw; x++)
85 c[x] = coeff[y + x * sh];
86 first_1d_fn(c, 1, row_clip_min, row_clip_max);
87 }
88
89 memset(coeff, 0, sizeof(*coeff) * sw * sh);
90 for (int i = 0; i < w * sh; i++)
91 tmp[i] = iclip((tmp[i] + rnd) >> shift, col_clip_min, col_clip_max);
92
93 for (int x = 0; x < w; x++)
94 second_1d_fn(&tmp[x], w, col_clip_min, col_clip_max);
95
96 c = tmp;
97 for (int y = 0; y < h; y++, dst += PXSTRIDE(stride))
98 for (int x = 0; x < w; x++)
99 dst[x] = iclip_pixel(dst[x] + ((*c++ + 8) >> 4));
100 }
101
102 #define inv_txfm_fn(type1, type2, w, h, shift, has_dconly) \
103 static void \
104 inv_txfm_add_##type1##_##type2##_##w##x##h##_c(pixel *dst, \
105 const ptrdiff_t stride, \
106 coef *const coeff, \
107 const int eob \
108 HIGHBD_DECL_SUFFIX) \
109 { \
110 inv_txfm_add_c(dst, stride, coeff, eob, w, h, shift, \
111 dav1d_inv_##type1##w##_1d_c, dav1d_inv_##type2##h##_1d_c, \
112 has_dconly HIGHBD_TAIL_SUFFIX); \
113 }
114
115 #define inv_txfm_fn64(w, h, shift) \
116 inv_txfm_fn(dct, dct, w, h, shift, 1)
117
118 #define inv_txfm_fn32(w, h, shift) \
119 inv_txfm_fn64(w, h, shift) \
120 inv_txfm_fn(identity, identity, w, h, shift, 0)
121
122 #define inv_txfm_fn16(w, h, shift) \
123 inv_txfm_fn32(w, h, shift) \
124 inv_txfm_fn(adst, dct, w, h, shift, 0) \
125 inv_txfm_fn(dct, adst, w, h, shift, 0) \
126 inv_txfm_fn(adst, adst, w, h, shift, 0) \
127 inv_txfm_fn(dct, flipadst, w, h, shift, 0) \
128 inv_txfm_fn(flipadst, dct, w, h, shift, 0) \
129 inv_txfm_fn(adst, flipadst, w, h, shift, 0) \
130 inv_txfm_fn(flipadst, adst, w, h, shift, 0) \
131 inv_txfm_fn(flipadst, flipadst, w, h, shift, 0) \
132 inv_txfm_fn(identity, dct, w, h, shift, 0) \
133 inv_txfm_fn(dct, identity, w, h, shift, 0) \
134
135 #define inv_txfm_fn84(w, h, shift) \
136 inv_txfm_fn16(w, h, shift) \
137 inv_txfm_fn(identity, flipadst, w, h, shift, 0) \
138 inv_txfm_fn(flipadst, identity, w, h, shift, 0) \
139 inv_txfm_fn(identity, adst, w, h, shift, 0) \
140 inv_txfm_fn(adst, identity, w, h, shift, 0) \
141
142 inv_txfm_fn84( 4, 4, 0)
143 inv_txfm_fn84( 4, 8, 0)
144 inv_txfm_fn84( 4, 16, 1)
145 inv_txfm_fn84( 8, 4, 0)
146 inv_txfm_fn84( 8, 8, 1)
147 inv_txfm_fn84( 8, 16, 1)
148 inv_txfm_fn32( 8, 32, 2)
149 inv_txfm_fn84(16, 4, 1)
150 inv_txfm_fn84(16, 8, 1)
151 inv_txfm_fn16(16, 16, 2)
152 inv_txfm_fn32(16, 32, 1)
153 inv_txfm_fn64(16, 64, 2)
154 inv_txfm_fn32(32, 8, 2)
155 inv_txfm_fn32(32, 16, 1)
156 inv_txfm_fn32(32, 32, 2)
157 inv_txfm_fn64(32, 64, 1)
158 inv_txfm_fn64(64, 16, 2)
159 inv_txfm_fn64(64, 32, 1)
160 inv_txfm_fn64(64, 64, 2)
161
162 #if !(HAVE_ASM && TRIM_DSP_FUNCTIONS && ( \
163 ARCH_AARCH64 || \
164 (ARCH_ARM && (defined(__ARM_NEON) || defined(__APPLE__) || defined(_WIN32))) \
165 ))
inv_txfm_add_wht_wht_4x4_c(pixel * dst,const ptrdiff_t stride,coef * const coeff,const int eob HIGHBD_DECL_SUFFIX)166 static void inv_txfm_add_wht_wht_4x4_c(pixel *dst, const ptrdiff_t stride,
167 coef *const coeff, const int eob
168 HIGHBD_DECL_SUFFIX)
169 {
170 int32_t tmp[4 * 4], *c = tmp;
171 for (int y = 0; y < 4; y++, c += 4) {
172 for (int x = 0; x < 4; x++)
173 c[x] = coeff[y + x * 4] >> 2;
174 dav1d_inv_wht4_1d_c(c, 1);
175 }
176 memset(coeff, 0, sizeof(*coeff) * 4 * 4);
177
178 for (int x = 0; x < 4; x++)
179 dav1d_inv_wht4_1d_c(&tmp[x], 4);
180
181 c = tmp;
182 for (int y = 0; y < 4; y++, dst += PXSTRIDE(stride))
183 for (int x = 0; x < 4; x++)
184 dst[x] = iclip_pixel(dst[x] + *c++);
185 }
186 #endif
187
188 #if HAVE_ASM
189 #if ARCH_AARCH64 || ARCH_ARM
190 #include "src/arm/itx.h"
191 #elif ARCH_LOONGARCH64
192 #include "src/loongarch/itx.h"
193 #elif ARCH_RISCV
194 #include "src/riscv/itx.h"
195 #elif ARCH_X86
196 #include "src/x86/itx.h"
197 #endif
198 #endif
199
bitfn(dav1d_itx_dsp_init)200 COLD void bitfn(dav1d_itx_dsp_init)(Dav1dInvTxfmDSPContext *const c, int bpc) {
201 #define assign_itx_all_fn64(w, h, pfx) \
202 c->itxfm_add[pfx##TX_##w##X##h][DCT_DCT ] = \
203 inv_txfm_add_dct_dct_##w##x##h##_c
204
205 #define assign_itx_all_fn32(w, h, pfx) \
206 assign_itx_all_fn64(w, h, pfx); \
207 c->itxfm_add[pfx##TX_##w##X##h][IDTX] = \
208 inv_txfm_add_identity_identity_##w##x##h##_c
209
210 #define assign_itx_all_fn16(w, h, pfx) \
211 assign_itx_all_fn32(w, h, pfx); \
212 c->itxfm_add[pfx##TX_##w##X##h][DCT_ADST ] = \
213 inv_txfm_add_adst_dct_##w##x##h##_c; \
214 c->itxfm_add[pfx##TX_##w##X##h][ADST_DCT ] = \
215 inv_txfm_add_dct_adst_##w##x##h##_c; \
216 c->itxfm_add[pfx##TX_##w##X##h][ADST_ADST] = \
217 inv_txfm_add_adst_adst_##w##x##h##_c; \
218 c->itxfm_add[pfx##TX_##w##X##h][ADST_FLIPADST] = \
219 inv_txfm_add_flipadst_adst_##w##x##h##_c; \
220 c->itxfm_add[pfx##TX_##w##X##h][FLIPADST_ADST] = \
221 inv_txfm_add_adst_flipadst_##w##x##h##_c; \
222 c->itxfm_add[pfx##TX_##w##X##h][DCT_FLIPADST] = \
223 inv_txfm_add_flipadst_dct_##w##x##h##_c; \
224 c->itxfm_add[pfx##TX_##w##X##h][FLIPADST_DCT] = \
225 inv_txfm_add_dct_flipadst_##w##x##h##_c; \
226 c->itxfm_add[pfx##TX_##w##X##h][FLIPADST_FLIPADST] = \
227 inv_txfm_add_flipadst_flipadst_##w##x##h##_c; \
228 c->itxfm_add[pfx##TX_##w##X##h][H_DCT] = \
229 inv_txfm_add_dct_identity_##w##x##h##_c; \
230 c->itxfm_add[pfx##TX_##w##X##h][V_DCT] = \
231 inv_txfm_add_identity_dct_##w##x##h##_c
232
233 #define assign_itx_all_fn84(w, h, pfx) \
234 assign_itx_all_fn16(w, h, pfx); \
235 c->itxfm_add[pfx##TX_##w##X##h][H_FLIPADST] = \
236 inv_txfm_add_flipadst_identity_##w##x##h##_c; \
237 c->itxfm_add[pfx##TX_##w##X##h][V_FLIPADST] = \
238 inv_txfm_add_identity_flipadst_##w##x##h##_c; \
239 c->itxfm_add[pfx##TX_##w##X##h][H_ADST] = \
240 inv_txfm_add_adst_identity_##w##x##h##_c; \
241 c->itxfm_add[pfx##TX_##w##X##h][V_ADST] = \
242 inv_txfm_add_identity_adst_##w##x##h##_c; \
243
244 #if !(HAVE_ASM && TRIM_DSP_FUNCTIONS && ( \
245 ARCH_AARCH64 || \
246 (ARCH_ARM && (defined(__ARM_NEON) || defined(__APPLE__) || defined(_WIN32))) \
247 ))
248 c->itxfm_add[TX_4X4][WHT_WHT] = inv_txfm_add_wht_wht_4x4_c;
249 #endif
250 assign_itx_all_fn84( 4, 4, );
251 assign_itx_all_fn84( 4, 8, R);
252 assign_itx_all_fn84( 4, 16, R);
253 assign_itx_all_fn84( 8, 4, R);
254 assign_itx_all_fn84( 8, 8, );
255 assign_itx_all_fn84( 8, 16, R);
256 assign_itx_all_fn32( 8, 32, R);
257 assign_itx_all_fn84(16, 4, R);
258 assign_itx_all_fn84(16, 8, R);
259 assign_itx_all_fn16(16, 16, );
260 assign_itx_all_fn32(16, 32, R);
261 assign_itx_all_fn64(16, 64, R);
262 assign_itx_all_fn32(32, 8, R);
263 assign_itx_all_fn32(32, 16, R);
264 assign_itx_all_fn32(32, 32, );
265 assign_itx_all_fn64(32, 64, R);
266 assign_itx_all_fn64(64, 16, R);
267 assign_itx_all_fn64(64, 32, R);
268 assign_itx_all_fn64(64, 64, );
269
270 #if HAVE_ASM
271 #if ARCH_AARCH64 || ARCH_ARM
272 itx_dsp_init_arm(c, bpc);
273 #endif
274 #if ARCH_LOONGARCH64
275 itx_dsp_init_loongarch(c, bpc);
276 #endif
277 #if ARCH_RISCV
278 itx_dsp_init_riscv(c, bpc);
279 #endif
280 #if ARCH_X86
281 itx_dsp_init_x86(c, bpc);
282 #endif
283 #endif
284 }
285