1 /*
2 * Copyright (c) 2015 The WebM project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11 #include <math.h>
12 #include <stdlib.h>
13 #include <string.h>
14
15 #include "./vpx_dsp_rtcd.h"
16 #include "vpx_dsp/inv_txfm.h"
17
vpx_iwht4x4_16_add_c(const tran_low_t * input,uint8_t * dest,int stride)18 void vpx_iwht4x4_16_add_c(const tran_low_t *input, uint8_t *dest, int stride) {
19 /* 4-point reversible, orthonormal inverse Walsh-Hadamard in 3.5 adds,
20 0.5 shifts per pixel. */
21 int i;
22 tran_low_t output[16];
23 tran_high_t a1, b1, c1, d1, e1;
24 const tran_low_t *ip = input;
25 tran_low_t *op = output;
26
27 for (i = 0; i < 4; i++) {
28 a1 = ip[0] >> UNIT_QUANT_SHIFT;
29 c1 = ip[1] >> UNIT_QUANT_SHIFT;
30 d1 = ip[2] >> UNIT_QUANT_SHIFT;
31 b1 = ip[3] >> UNIT_QUANT_SHIFT;
32 a1 += c1;
33 d1 -= b1;
34 e1 = (a1 - d1) >> 1;
35 b1 = e1 - b1;
36 c1 = e1 - c1;
37 a1 -= b1;
38 d1 += c1;
39 op[0] = WRAPLOW(a1);
40 op[1] = WRAPLOW(b1);
41 op[2] = WRAPLOW(c1);
42 op[3] = WRAPLOW(d1);
43 ip += 4;
44 op += 4;
45 }
46
47 ip = output;
48 for (i = 0; i < 4; i++) {
49 a1 = ip[4 * 0];
50 c1 = ip[4 * 1];
51 d1 = ip[4 * 2];
52 b1 = ip[4 * 3];
53 a1 += c1;
54 d1 -= b1;
55 e1 = (a1 - d1) >> 1;
56 b1 = e1 - b1;
57 c1 = e1 - c1;
58 a1 -= b1;
59 d1 += c1;
60 dest[stride * 0] = clip_pixel_add(dest[stride * 0], WRAPLOW(a1));
61 dest[stride * 1] = clip_pixel_add(dest[stride * 1], WRAPLOW(b1));
62 dest[stride * 2] = clip_pixel_add(dest[stride * 2], WRAPLOW(c1));
63 dest[stride * 3] = clip_pixel_add(dest[stride * 3], WRAPLOW(d1));
64
65 ip++;
66 dest++;
67 }
68 }
69
vpx_iwht4x4_1_add_c(const tran_low_t * input,uint8_t * dest,int stride)70 void vpx_iwht4x4_1_add_c(const tran_low_t *input, uint8_t *dest, int stride) {
71 int i;
72 tran_high_t a1, e1;
73 tran_low_t tmp[4];
74 const tran_low_t *ip = input;
75 tran_low_t *op = tmp;
76
77 a1 = ip[0] >> UNIT_QUANT_SHIFT;
78 e1 = a1 >> 1;
79 a1 -= e1;
80 op[0] = WRAPLOW(a1);
81 op[1] = op[2] = op[3] = WRAPLOW(e1);
82
83 ip = tmp;
84 for (i = 0; i < 4; i++) {
85 e1 = ip[0] >> 1;
86 a1 = ip[0] - e1;
87 dest[stride * 0] = clip_pixel_add(dest[stride * 0], a1);
88 dest[stride * 1] = clip_pixel_add(dest[stride * 1], e1);
89 dest[stride * 2] = clip_pixel_add(dest[stride * 2], e1);
90 dest[stride * 3] = clip_pixel_add(dest[stride * 3], e1);
91 ip++;
92 dest++;
93 }
94 }
95
iadst4_c(const tran_low_t * input,tran_low_t * output)96 void iadst4_c(const tran_low_t *input, tran_low_t *output) {
97 tran_high_t s0, s1, s2, s3, s4, s5, s6, s7;
98 tran_low_t x0 = input[0];
99 tran_low_t x1 = input[1];
100 tran_low_t x2 = input[2];
101 tran_low_t x3 = input[3];
102
103 if (!(x0 | x1 | x2 | x3)) {
104 memset(output, 0, 4 * sizeof(*output));
105 return;
106 }
107
108 // 32-bit result is enough for the following multiplications.
109 s0 = sinpi_1_9 * x0;
110 s1 = sinpi_2_9 * x0;
111 s2 = sinpi_3_9 * x1;
112 s3 = sinpi_4_9 * x2;
113 s4 = sinpi_1_9 * x2;
114 s5 = sinpi_2_9 * x3;
115 s6 = sinpi_4_9 * x3;
116 s7 = WRAPLOW(x0 - x2 + x3);
117
118 s0 = s0 + s3 + s5;
119 s1 = s1 - s4 - s6;
120 s3 = s2;
121 s2 = sinpi_3_9 * s7;
122
123 // 1-D transform scaling factor is sqrt(2).
124 // The overall dynamic range is 14b (input) + 14b (multiplication scaling)
125 // + 1b (addition) = 29b.
126 // Hence the output bit depth is 15b.
127 output[0] = WRAPLOW(dct_const_round_shift(s0 + s3));
128 output[1] = WRAPLOW(dct_const_round_shift(s1 + s3));
129 output[2] = WRAPLOW(dct_const_round_shift(s2));
130 output[3] = WRAPLOW(dct_const_round_shift(s0 + s1 - s3));
131 }
132
idct4_c(const tran_low_t * input,tran_low_t * output)133 void idct4_c(const tran_low_t *input, tran_low_t *output) {
134 int16_t step[4];
135 tran_high_t temp1, temp2;
136
137 // stage 1
138 temp1 = ((int16_t)input[0] + (int16_t)input[2]) * cospi_16_64;
139 temp2 = ((int16_t)input[0] - (int16_t)input[2]) * cospi_16_64;
140 step[0] = WRAPLOW(dct_const_round_shift(temp1));
141 step[1] = WRAPLOW(dct_const_round_shift(temp2));
142 temp1 = (int16_t)input[1] * cospi_24_64 - (int16_t)input[3] * cospi_8_64;
143 temp2 = (int16_t)input[1] * cospi_8_64 + (int16_t)input[3] * cospi_24_64;
144 step[2] = WRAPLOW(dct_const_round_shift(temp1));
145 step[3] = WRAPLOW(dct_const_round_shift(temp2));
146
147 // stage 2
148 output[0] = WRAPLOW(step[0] + step[3]);
149 output[1] = WRAPLOW(step[1] + step[2]);
150 output[2] = WRAPLOW(step[1] - step[2]);
151 output[3] = WRAPLOW(step[0] - step[3]);
152 }
153
vpx_idct4x4_16_add_c(const tran_low_t * input,uint8_t * dest,int stride)154 void vpx_idct4x4_16_add_c(const tran_low_t *input, uint8_t *dest, int stride) {
155 int i, j;
156 tran_low_t out[4 * 4];
157 tran_low_t *outptr = out;
158 tran_low_t temp_in[4], temp_out[4];
159
160 // Rows
161 for (i = 0; i < 4; ++i) {
162 idct4_c(input, outptr);
163 input += 4;
164 outptr += 4;
165 }
166
167 // Columns
168 for (i = 0; i < 4; ++i) {
169 for (j = 0; j < 4; ++j) temp_in[j] = out[j * 4 + i];
170 idct4_c(temp_in, temp_out);
171 for (j = 0; j < 4; ++j) {
172 dest[j * stride + i] = clip_pixel_add(dest[j * stride + i],
173 ROUND_POWER_OF_TWO(temp_out[j], 4));
174 }
175 }
176 }
177
vpx_idct4x4_1_add_c(const tran_low_t * input,uint8_t * dest,int stride)178 void vpx_idct4x4_1_add_c(const tran_low_t *input, uint8_t *dest, int stride) {
179 int i;
180 tran_high_t a1;
181 tran_low_t out =
182 WRAPLOW(dct_const_round_shift((int16_t)input[0] * cospi_16_64));
183
184 out = WRAPLOW(dct_const_round_shift(out * cospi_16_64));
185 a1 = ROUND_POWER_OF_TWO(out, 4);
186
187 for (i = 0; i < 4; i++) {
188 dest[0] = clip_pixel_add(dest[0], a1);
189 dest[1] = clip_pixel_add(dest[1], a1);
190 dest[2] = clip_pixel_add(dest[2], a1);
191 dest[3] = clip_pixel_add(dest[3], a1);
192 dest += stride;
193 }
194 }
195
iadst8_c(const tran_low_t * input,tran_low_t * output)196 void iadst8_c(const tran_low_t *input, tran_low_t *output) {
197 int s0, s1, s2, s3, s4, s5, s6, s7;
198 tran_high_t x0 = input[7];
199 tran_high_t x1 = input[0];
200 tran_high_t x2 = input[5];
201 tran_high_t x3 = input[2];
202 tran_high_t x4 = input[3];
203 tran_high_t x5 = input[4];
204 tran_high_t x6 = input[1];
205 tran_high_t x7 = input[6];
206
207 if (!(x0 | x1 | x2 | x3 | x4 | x5 | x6 | x7)) {
208 memset(output, 0, 8 * sizeof(*output));
209 return;
210 }
211
212 // stage 1
213 s0 = (int)(cospi_2_64 * x0 + cospi_30_64 * x1);
214 s1 = (int)(cospi_30_64 * x0 - cospi_2_64 * x1);
215 s2 = (int)(cospi_10_64 * x2 + cospi_22_64 * x3);
216 s3 = (int)(cospi_22_64 * x2 - cospi_10_64 * x3);
217 s4 = (int)(cospi_18_64 * x4 + cospi_14_64 * x5);
218 s5 = (int)(cospi_14_64 * x4 - cospi_18_64 * x5);
219 s6 = (int)(cospi_26_64 * x6 + cospi_6_64 * x7);
220 s7 = (int)(cospi_6_64 * x6 - cospi_26_64 * x7);
221
222 x0 = WRAPLOW(dct_const_round_shift(s0 + s4));
223 x1 = WRAPLOW(dct_const_round_shift(s1 + s5));
224 x2 = WRAPLOW(dct_const_round_shift(s2 + s6));
225 x3 = WRAPLOW(dct_const_round_shift(s3 + s7));
226 x4 = WRAPLOW(dct_const_round_shift(s0 - s4));
227 x5 = WRAPLOW(dct_const_round_shift(s1 - s5));
228 x6 = WRAPLOW(dct_const_round_shift(s2 - s6));
229 x7 = WRAPLOW(dct_const_round_shift(s3 - s7));
230
231 // stage 2
232 s0 = (int)x0;
233 s1 = (int)x1;
234 s2 = (int)x2;
235 s3 = (int)x3;
236 s4 = (int)(cospi_8_64 * x4 + cospi_24_64 * x5);
237 s5 = (int)(cospi_24_64 * x4 - cospi_8_64 * x5);
238 s6 = (int)(-cospi_24_64 * x6 + cospi_8_64 * x7);
239 s7 = (int)(cospi_8_64 * x6 + cospi_24_64 * x7);
240
241 x0 = WRAPLOW(s0 + s2);
242 x1 = WRAPLOW(s1 + s3);
243 x2 = WRAPLOW(s0 - s2);
244 x3 = WRAPLOW(s1 - s3);
245 x4 = WRAPLOW(dct_const_round_shift(s4 + s6));
246 x5 = WRAPLOW(dct_const_round_shift(s5 + s7));
247 x6 = WRAPLOW(dct_const_round_shift(s4 - s6));
248 x7 = WRAPLOW(dct_const_round_shift(s5 - s7));
249
250 // stage 3
251 s2 = (int)(cospi_16_64 * (x2 + x3));
252 s3 = (int)(cospi_16_64 * (x2 - x3));
253 s6 = (int)(cospi_16_64 * (x6 + x7));
254 s7 = (int)(cospi_16_64 * (x6 - x7));
255
256 x2 = WRAPLOW(dct_const_round_shift(s2));
257 x3 = WRAPLOW(dct_const_round_shift(s3));
258 x6 = WRAPLOW(dct_const_round_shift(s6));
259 x7 = WRAPLOW(dct_const_round_shift(s7));
260
261 output[0] = WRAPLOW(x0);
262 output[1] = WRAPLOW(-x4);
263 output[2] = WRAPLOW(x6);
264 output[3] = WRAPLOW(-x2);
265 output[4] = WRAPLOW(x3);
266 output[5] = WRAPLOW(-x7);
267 output[6] = WRAPLOW(x5);
268 output[7] = WRAPLOW(-x1);
269 }
270
idct8_c(const tran_low_t * input,tran_low_t * output)271 void idct8_c(const tran_low_t *input, tran_low_t *output) {
272 int16_t step1[8], step2[8];
273 tran_high_t temp1, temp2;
274
275 // stage 1
276 step1[0] = (int16_t)input[0];
277 step1[2] = (int16_t)input[4];
278 step1[1] = (int16_t)input[2];
279 step1[3] = (int16_t)input[6];
280 temp1 = (int16_t)input[1] * cospi_28_64 - (int16_t)input[7] * cospi_4_64;
281 temp2 = (int16_t)input[1] * cospi_4_64 + (int16_t)input[7] * cospi_28_64;
282 step1[4] = WRAPLOW(dct_const_round_shift(temp1));
283 step1[7] = WRAPLOW(dct_const_round_shift(temp2));
284 temp1 = (int16_t)input[5] * cospi_12_64 - (int16_t)input[3] * cospi_20_64;
285 temp2 = (int16_t)input[5] * cospi_20_64 + (int16_t)input[3] * cospi_12_64;
286 step1[5] = WRAPLOW(dct_const_round_shift(temp1));
287 step1[6] = WRAPLOW(dct_const_round_shift(temp2));
288
289 // stage 2
290 temp1 = (step1[0] + step1[2]) * cospi_16_64;
291 temp2 = (step1[0] - step1[2]) * cospi_16_64;
292 step2[0] = WRAPLOW(dct_const_round_shift(temp1));
293 step2[1] = WRAPLOW(dct_const_round_shift(temp2));
294 temp1 = step1[1] * cospi_24_64 - step1[3] * cospi_8_64;
295 temp2 = step1[1] * cospi_8_64 + step1[3] * cospi_24_64;
296 step2[2] = WRAPLOW(dct_const_round_shift(temp1));
297 step2[3] = WRAPLOW(dct_const_round_shift(temp2));
298 step2[4] = WRAPLOW(step1[4] + step1[5]);
299 step2[5] = WRAPLOW(step1[4] - step1[5]);
300 step2[6] = WRAPLOW(-step1[6] + step1[7]);
301 step2[7] = WRAPLOW(step1[6] + step1[7]);
302
303 // stage 3
304 step1[0] = WRAPLOW(step2[0] + step2[3]);
305 step1[1] = WRAPLOW(step2[1] + step2[2]);
306 step1[2] = WRAPLOW(step2[1] - step2[2]);
307 step1[3] = WRAPLOW(step2[0] - step2[3]);
308 step1[4] = step2[4];
309 temp1 = (step2[6] - step2[5]) * cospi_16_64;
310 temp2 = (step2[5] + step2[6]) * cospi_16_64;
311 step1[5] = WRAPLOW(dct_const_round_shift(temp1));
312 step1[6] = WRAPLOW(dct_const_round_shift(temp2));
313 step1[7] = step2[7];
314
315 // stage 4
316 output[0] = WRAPLOW(step1[0] + step1[7]);
317 output[1] = WRAPLOW(step1[1] + step1[6]);
318 output[2] = WRAPLOW(step1[2] + step1[5]);
319 output[3] = WRAPLOW(step1[3] + step1[4]);
320 output[4] = WRAPLOW(step1[3] - step1[4]);
321 output[5] = WRAPLOW(step1[2] - step1[5]);
322 output[6] = WRAPLOW(step1[1] - step1[6]);
323 output[7] = WRAPLOW(step1[0] - step1[7]);
324 }
325
vpx_idct8x8_64_add_c(const tran_low_t * input,uint8_t * dest,int stride)326 void vpx_idct8x8_64_add_c(const tran_low_t *input, uint8_t *dest, int stride) {
327 int i, j;
328 tran_low_t out[8 * 8];
329 tran_low_t *outptr = out;
330 tran_low_t temp_in[8], temp_out[8];
331
332 // First transform rows
333 for (i = 0; i < 8; ++i) {
334 idct8_c(input, outptr);
335 input += 8;
336 outptr += 8;
337 }
338
339 // Then transform columns
340 for (i = 0; i < 8; ++i) {
341 for (j = 0; j < 8; ++j) temp_in[j] = out[j * 8 + i];
342 idct8_c(temp_in, temp_out);
343 for (j = 0; j < 8; ++j) {
344 dest[j * stride + i] = clip_pixel_add(dest[j * stride + i],
345 ROUND_POWER_OF_TWO(temp_out[j], 5));
346 }
347 }
348 }
349
vpx_idct8x8_12_add_c(const tran_low_t * input,uint8_t * dest,int stride)350 void vpx_idct8x8_12_add_c(const tran_low_t *input, uint8_t *dest, int stride) {
351 int i, j;
352 tran_low_t out[8 * 8] = { 0 };
353 tran_low_t *outptr = out;
354 tran_low_t temp_in[8], temp_out[8];
355
356 // First transform rows
357 // Only first 4 row has non-zero coefs
358 for (i = 0; i < 4; ++i) {
359 idct8_c(input, outptr);
360 input += 8;
361 outptr += 8;
362 }
363
364 // Then transform columns
365 for (i = 0; i < 8; ++i) {
366 for (j = 0; j < 8; ++j) temp_in[j] = out[j * 8 + i];
367 idct8_c(temp_in, temp_out);
368 for (j = 0; j < 8; ++j) {
369 dest[j * stride + i] = clip_pixel_add(dest[j * stride + i],
370 ROUND_POWER_OF_TWO(temp_out[j], 5));
371 }
372 }
373 }
374
vpx_idct8x8_1_add_c(const tran_low_t * input,uint8_t * dest,int stride)375 void vpx_idct8x8_1_add_c(const tran_low_t *input, uint8_t *dest, int stride) {
376 int i, j;
377 tran_high_t a1;
378 tran_low_t out =
379 WRAPLOW(dct_const_round_shift((int16_t)input[0] * cospi_16_64));
380
381 out = WRAPLOW(dct_const_round_shift(out * cospi_16_64));
382 a1 = ROUND_POWER_OF_TWO(out, 5);
383 for (j = 0; j < 8; ++j) {
384 for (i = 0; i < 8; ++i) dest[i] = clip_pixel_add(dest[i], a1);
385 dest += stride;
386 }
387 }
388
iadst16_c(const tran_low_t * input,tran_low_t * output)389 void iadst16_c(const tran_low_t *input, tran_low_t *output) {
390 tran_high_t s0, s1, s2, s3, s4, s5, s6, s7, s8;
391 tran_high_t s9, s10, s11, s12, s13, s14, s15;
392 tran_high_t x0 = input[15];
393 tran_high_t x1 = input[0];
394 tran_high_t x2 = input[13];
395 tran_high_t x3 = input[2];
396 tran_high_t x4 = input[11];
397 tran_high_t x5 = input[4];
398 tran_high_t x6 = input[9];
399 tran_high_t x7 = input[6];
400 tran_high_t x8 = input[7];
401 tran_high_t x9 = input[8];
402 tran_high_t x10 = input[5];
403 tran_high_t x11 = input[10];
404 tran_high_t x12 = input[3];
405 tran_high_t x13 = input[12];
406 tran_high_t x14 = input[1];
407 tran_high_t x15 = input[14];
408
409 if (!(x0 | x1 | x2 | x3 | x4 | x5 | x6 | x7 | x8 | x9 | x10 | x11 | x12 |
410 x13 | x14 | x15)) {
411 memset(output, 0, 16 * sizeof(*output));
412 return;
413 }
414
415 // stage 1
416 s0 = x0 * cospi_1_64 + x1 * cospi_31_64;
417 s1 = x0 * cospi_31_64 - x1 * cospi_1_64;
418 s2 = x2 * cospi_5_64 + x3 * cospi_27_64;
419 s3 = x2 * cospi_27_64 - x3 * cospi_5_64;
420 s4 = x4 * cospi_9_64 + x5 * cospi_23_64;
421 s5 = x4 * cospi_23_64 - x5 * cospi_9_64;
422 s6 = x6 * cospi_13_64 + x7 * cospi_19_64;
423 s7 = x6 * cospi_19_64 - x7 * cospi_13_64;
424 s8 = x8 * cospi_17_64 + x9 * cospi_15_64;
425 s9 = x8 * cospi_15_64 - x9 * cospi_17_64;
426 s10 = x10 * cospi_21_64 + x11 * cospi_11_64;
427 s11 = x10 * cospi_11_64 - x11 * cospi_21_64;
428 s12 = x12 * cospi_25_64 + x13 * cospi_7_64;
429 s13 = x12 * cospi_7_64 - x13 * cospi_25_64;
430 s14 = x14 * cospi_29_64 + x15 * cospi_3_64;
431 s15 = x14 * cospi_3_64 - x15 * cospi_29_64;
432
433 x0 = WRAPLOW(dct_const_round_shift(s0 + s8));
434 x1 = WRAPLOW(dct_const_round_shift(s1 + s9));
435 x2 = WRAPLOW(dct_const_round_shift(s2 + s10));
436 x3 = WRAPLOW(dct_const_round_shift(s3 + s11));
437 x4 = WRAPLOW(dct_const_round_shift(s4 + s12));
438 x5 = WRAPLOW(dct_const_round_shift(s5 + s13));
439 x6 = WRAPLOW(dct_const_round_shift(s6 + s14));
440 x7 = WRAPLOW(dct_const_round_shift(s7 + s15));
441 x8 = WRAPLOW(dct_const_round_shift(s0 - s8));
442 x9 = WRAPLOW(dct_const_round_shift(s1 - s9));
443 x10 = WRAPLOW(dct_const_round_shift(s2 - s10));
444 x11 = WRAPLOW(dct_const_round_shift(s3 - s11));
445 x12 = WRAPLOW(dct_const_round_shift(s4 - s12));
446 x13 = WRAPLOW(dct_const_round_shift(s5 - s13));
447 x14 = WRAPLOW(dct_const_round_shift(s6 - s14));
448 x15 = WRAPLOW(dct_const_round_shift(s7 - s15));
449
450 // stage 2
451 s0 = x0;
452 s1 = x1;
453 s2 = x2;
454 s3 = x3;
455 s4 = x4;
456 s5 = x5;
457 s6 = x6;
458 s7 = x7;
459 s8 = x8 * cospi_4_64 + x9 * cospi_28_64;
460 s9 = x8 * cospi_28_64 - x9 * cospi_4_64;
461 s10 = x10 * cospi_20_64 + x11 * cospi_12_64;
462 s11 = x10 * cospi_12_64 - x11 * cospi_20_64;
463 s12 = -x12 * cospi_28_64 + x13 * cospi_4_64;
464 s13 = x12 * cospi_4_64 + x13 * cospi_28_64;
465 s14 = -x14 * cospi_12_64 + x15 * cospi_20_64;
466 s15 = x14 * cospi_20_64 + x15 * cospi_12_64;
467
468 x0 = WRAPLOW(s0 + s4);
469 x1 = WRAPLOW(s1 + s5);
470 x2 = WRAPLOW(s2 + s6);
471 x3 = WRAPLOW(s3 + s7);
472 x4 = WRAPLOW(s0 - s4);
473 x5 = WRAPLOW(s1 - s5);
474 x6 = WRAPLOW(s2 - s6);
475 x7 = WRAPLOW(s3 - s7);
476 x8 = WRAPLOW(dct_const_round_shift(s8 + s12));
477 x9 = WRAPLOW(dct_const_round_shift(s9 + s13));
478 x10 = WRAPLOW(dct_const_round_shift(s10 + s14));
479 x11 = WRAPLOW(dct_const_round_shift(s11 + s15));
480 x12 = WRAPLOW(dct_const_round_shift(s8 - s12));
481 x13 = WRAPLOW(dct_const_round_shift(s9 - s13));
482 x14 = WRAPLOW(dct_const_round_shift(s10 - s14));
483 x15 = WRAPLOW(dct_const_round_shift(s11 - s15));
484
485 // stage 3
486 s0 = x0;
487 s1 = x1;
488 s2 = x2;
489 s3 = x3;
490 s4 = x4 * cospi_8_64 + x5 * cospi_24_64;
491 s5 = x4 * cospi_24_64 - x5 * cospi_8_64;
492 s6 = -x6 * cospi_24_64 + x7 * cospi_8_64;
493 s7 = x6 * cospi_8_64 + x7 * cospi_24_64;
494 s8 = x8;
495 s9 = x9;
496 s10 = x10;
497 s11 = x11;
498 s12 = x12 * cospi_8_64 + x13 * cospi_24_64;
499 s13 = x12 * cospi_24_64 - x13 * cospi_8_64;
500 s14 = -x14 * cospi_24_64 + x15 * cospi_8_64;
501 s15 = x14 * cospi_8_64 + x15 * cospi_24_64;
502
503 x0 = WRAPLOW(s0 + s2);
504 x1 = WRAPLOW(s1 + s3);
505 x2 = WRAPLOW(s0 - s2);
506 x3 = WRAPLOW(s1 - s3);
507 x4 = WRAPLOW(dct_const_round_shift(s4 + s6));
508 x5 = WRAPLOW(dct_const_round_shift(s5 + s7));
509 x6 = WRAPLOW(dct_const_round_shift(s4 - s6));
510 x7 = WRAPLOW(dct_const_round_shift(s5 - s7));
511 x8 = WRAPLOW(s8 + s10);
512 x9 = WRAPLOW(s9 + s11);
513 x10 = WRAPLOW(s8 - s10);
514 x11 = WRAPLOW(s9 - s11);
515 x12 = WRAPLOW(dct_const_round_shift(s12 + s14));
516 x13 = WRAPLOW(dct_const_round_shift(s13 + s15));
517 x14 = WRAPLOW(dct_const_round_shift(s12 - s14));
518 x15 = WRAPLOW(dct_const_round_shift(s13 - s15));
519
520 // stage 4
521 s2 = (-cospi_16_64) * (x2 + x3);
522 s3 = cospi_16_64 * (x2 - x3);
523 s6 = cospi_16_64 * (x6 + x7);
524 s7 = cospi_16_64 * (-x6 + x7);
525 s10 = cospi_16_64 * (x10 + x11);
526 s11 = cospi_16_64 * (-x10 + x11);
527 s14 = (-cospi_16_64) * (x14 + x15);
528 s15 = cospi_16_64 * (x14 - x15);
529
530 x2 = WRAPLOW(dct_const_round_shift(s2));
531 x3 = WRAPLOW(dct_const_round_shift(s3));
532 x6 = WRAPLOW(dct_const_round_shift(s6));
533 x7 = WRAPLOW(dct_const_round_shift(s7));
534 x10 = WRAPLOW(dct_const_round_shift(s10));
535 x11 = WRAPLOW(dct_const_round_shift(s11));
536 x14 = WRAPLOW(dct_const_round_shift(s14));
537 x15 = WRAPLOW(dct_const_round_shift(s15));
538
539 output[0] = WRAPLOW(x0);
540 output[1] = WRAPLOW(-x8);
541 output[2] = WRAPLOW(x12);
542 output[3] = WRAPLOW(-x4);
543 output[4] = WRAPLOW(x6);
544 output[5] = WRAPLOW(x14);
545 output[6] = WRAPLOW(x10);
546 output[7] = WRAPLOW(x2);
547 output[8] = WRAPLOW(x3);
548 output[9] = WRAPLOW(x11);
549 output[10] = WRAPLOW(x15);
550 output[11] = WRAPLOW(x7);
551 output[12] = WRAPLOW(x5);
552 output[13] = WRAPLOW(-x13);
553 output[14] = WRAPLOW(x9);
554 output[15] = WRAPLOW(-x1);
555 }
556
idct16_c(const tran_low_t * input,tran_low_t * output)557 void idct16_c(const tran_low_t *input, tran_low_t *output) {
558 int16_t step1[16], step2[16];
559 tran_high_t temp1, temp2;
560
561 // stage 1
562 step1[0] = (int16_t)input[0 / 2];
563 step1[1] = (int16_t)input[16 / 2];
564 step1[2] = (int16_t)input[8 / 2];
565 step1[3] = (int16_t)input[24 / 2];
566 step1[4] = (int16_t)input[4 / 2];
567 step1[5] = (int16_t)input[20 / 2];
568 step1[6] = (int16_t)input[12 / 2];
569 step1[7] = (int16_t)input[28 / 2];
570 step1[8] = (int16_t)input[2 / 2];
571 step1[9] = (int16_t)input[18 / 2];
572 step1[10] = (int16_t)input[10 / 2];
573 step1[11] = (int16_t)input[26 / 2];
574 step1[12] = (int16_t)input[6 / 2];
575 step1[13] = (int16_t)input[22 / 2];
576 step1[14] = (int16_t)input[14 / 2];
577 step1[15] = (int16_t)input[30 / 2];
578
579 // stage 2
580 step2[0] = step1[0];
581 step2[1] = step1[1];
582 step2[2] = step1[2];
583 step2[3] = step1[3];
584 step2[4] = step1[4];
585 step2[5] = step1[5];
586 step2[6] = step1[6];
587 step2[7] = step1[7];
588
589 temp1 = step1[8] * cospi_30_64 - step1[15] * cospi_2_64;
590 temp2 = step1[8] * cospi_2_64 + step1[15] * cospi_30_64;
591 step2[8] = WRAPLOW(dct_const_round_shift(temp1));
592 step2[15] = WRAPLOW(dct_const_round_shift(temp2));
593
594 temp1 = step1[9] * cospi_14_64 - step1[14] * cospi_18_64;
595 temp2 = step1[9] * cospi_18_64 + step1[14] * cospi_14_64;
596 step2[9] = WRAPLOW(dct_const_round_shift(temp1));
597 step2[14] = WRAPLOW(dct_const_round_shift(temp2));
598
599 temp1 = step1[10] * cospi_22_64 - step1[13] * cospi_10_64;
600 temp2 = step1[10] * cospi_10_64 + step1[13] * cospi_22_64;
601 step2[10] = WRAPLOW(dct_const_round_shift(temp1));
602 step2[13] = WRAPLOW(dct_const_round_shift(temp2));
603
604 temp1 = step1[11] * cospi_6_64 - step1[12] * cospi_26_64;
605 temp2 = step1[11] * cospi_26_64 + step1[12] * cospi_6_64;
606 step2[11] = WRAPLOW(dct_const_round_shift(temp1));
607 step2[12] = WRAPLOW(dct_const_round_shift(temp2));
608
609 // stage 3
610 step1[0] = step2[0];
611 step1[1] = step2[1];
612 step1[2] = step2[2];
613 step1[3] = step2[3];
614
615 temp1 = step2[4] * cospi_28_64 - step2[7] * cospi_4_64;
616 temp2 = step2[4] * cospi_4_64 + step2[7] * cospi_28_64;
617 step1[4] = WRAPLOW(dct_const_round_shift(temp1));
618 step1[7] = WRAPLOW(dct_const_round_shift(temp2));
619 temp1 = step2[5] * cospi_12_64 - step2[6] * cospi_20_64;
620 temp2 = step2[5] * cospi_20_64 + step2[6] * cospi_12_64;
621 step1[5] = WRAPLOW(dct_const_round_shift(temp1));
622 step1[6] = WRAPLOW(dct_const_round_shift(temp2));
623
624 step1[8] = WRAPLOW(step2[8] + step2[9]);
625 step1[9] = WRAPLOW(step2[8] - step2[9]);
626 step1[10] = WRAPLOW(-step2[10] + step2[11]);
627 step1[11] = WRAPLOW(step2[10] + step2[11]);
628 step1[12] = WRAPLOW(step2[12] + step2[13]);
629 step1[13] = WRAPLOW(step2[12] - step2[13]);
630 step1[14] = WRAPLOW(-step2[14] + step2[15]);
631 step1[15] = WRAPLOW(step2[14] + step2[15]);
632
633 // stage 4
634 temp1 = (step1[0] + step1[1]) * cospi_16_64;
635 temp2 = (step1[0] - step1[1]) * cospi_16_64;
636 step2[0] = WRAPLOW(dct_const_round_shift(temp1));
637 step2[1] = WRAPLOW(dct_const_round_shift(temp2));
638 temp1 = step1[2] * cospi_24_64 - step1[3] * cospi_8_64;
639 temp2 = step1[2] * cospi_8_64 + step1[3] * cospi_24_64;
640 step2[2] = WRAPLOW(dct_const_round_shift(temp1));
641 step2[3] = WRAPLOW(dct_const_round_shift(temp2));
642 step2[4] = WRAPLOW(step1[4] + step1[5]);
643 step2[5] = WRAPLOW(step1[4] - step1[5]);
644 step2[6] = WRAPLOW(-step1[6] + step1[7]);
645 step2[7] = WRAPLOW(step1[6] + step1[7]);
646
647 step2[8] = step1[8];
648 step2[15] = step1[15];
649 temp1 = -step1[9] * cospi_8_64 + step1[14] * cospi_24_64;
650 temp2 = step1[9] * cospi_24_64 + step1[14] * cospi_8_64;
651 step2[9] = WRAPLOW(dct_const_round_shift(temp1));
652 step2[14] = WRAPLOW(dct_const_round_shift(temp2));
653 temp1 = -step1[10] * cospi_24_64 - step1[13] * cospi_8_64;
654 temp2 = -step1[10] * cospi_8_64 + step1[13] * cospi_24_64;
655 step2[10] = WRAPLOW(dct_const_round_shift(temp1));
656 step2[13] = WRAPLOW(dct_const_round_shift(temp2));
657 step2[11] = step1[11];
658 step2[12] = step1[12];
659
660 // stage 5
661 step1[0] = WRAPLOW(step2[0] + step2[3]);
662 step1[1] = WRAPLOW(step2[1] + step2[2]);
663 step1[2] = WRAPLOW(step2[1] - step2[2]);
664 step1[3] = WRAPLOW(step2[0] - step2[3]);
665 step1[4] = step2[4];
666 temp1 = (step2[6] - step2[5]) * cospi_16_64;
667 temp2 = (step2[5] + step2[6]) * cospi_16_64;
668 step1[5] = WRAPLOW(dct_const_round_shift(temp1));
669 step1[6] = WRAPLOW(dct_const_round_shift(temp2));
670 step1[7] = step2[7];
671
672 step1[8] = WRAPLOW(step2[8] + step2[11]);
673 step1[9] = WRAPLOW(step2[9] + step2[10]);
674 step1[10] = WRAPLOW(step2[9] - step2[10]);
675 step1[11] = WRAPLOW(step2[8] - step2[11]);
676 step1[12] = WRAPLOW(-step2[12] + step2[15]);
677 step1[13] = WRAPLOW(-step2[13] + step2[14]);
678 step1[14] = WRAPLOW(step2[13] + step2[14]);
679 step1[15] = WRAPLOW(step2[12] + step2[15]);
680
681 // stage 6
682 step2[0] = WRAPLOW(step1[0] + step1[7]);
683 step2[1] = WRAPLOW(step1[1] + step1[6]);
684 step2[2] = WRAPLOW(step1[2] + step1[5]);
685 step2[3] = WRAPLOW(step1[3] + step1[4]);
686 step2[4] = WRAPLOW(step1[3] - step1[4]);
687 step2[5] = WRAPLOW(step1[2] - step1[5]);
688 step2[6] = WRAPLOW(step1[1] - step1[6]);
689 step2[7] = WRAPLOW(step1[0] - step1[7]);
690 step2[8] = step1[8];
691 step2[9] = step1[9];
692 temp1 = (-step1[10] + step1[13]) * cospi_16_64;
693 temp2 = (step1[10] + step1[13]) * cospi_16_64;
694 step2[10] = WRAPLOW(dct_const_round_shift(temp1));
695 step2[13] = WRAPLOW(dct_const_round_shift(temp2));
696 temp1 = (-step1[11] + step1[12]) * cospi_16_64;
697 temp2 = (step1[11] + step1[12]) * cospi_16_64;
698 step2[11] = WRAPLOW(dct_const_round_shift(temp1));
699 step2[12] = WRAPLOW(dct_const_round_shift(temp2));
700 step2[14] = step1[14];
701 step2[15] = step1[15];
702
703 // stage 7
704 output[0] = (tran_low_t)WRAPLOW(step2[0] + step2[15]);
705 output[1] = (tran_low_t)WRAPLOW(step2[1] + step2[14]);
706 output[2] = (tran_low_t)WRAPLOW(step2[2] + step2[13]);
707 output[3] = (tran_low_t)WRAPLOW(step2[3] + step2[12]);
708 output[4] = (tran_low_t)WRAPLOW(step2[4] + step2[11]);
709 output[5] = (tran_low_t)WRAPLOW(step2[5] + step2[10]);
710 output[6] = (tran_low_t)WRAPLOW(step2[6] + step2[9]);
711 output[7] = (tran_low_t)WRAPLOW(step2[7] + step2[8]);
712 output[8] = (tran_low_t)WRAPLOW(step2[7] - step2[8]);
713 output[9] = (tran_low_t)WRAPLOW(step2[6] - step2[9]);
714 output[10] = (tran_low_t)WRAPLOW(step2[5] - step2[10]);
715 output[11] = (tran_low_t)WRAPLOW(step2[4] - step2[11]);
716 output[12] = (tran_low_t)WRAPLOW(step2[3] - step2[12]);
717 output[13] = (tran_low_t)WRAPLOW(step2[2] - step2[13]);
718 output[14] = (tran_low_t)WRAPLOW(step2[1] - step2[14]);
719 output[15] = (tran_low_t)WRAPLOW(step2[0] - step2[15]);
720 }
721
vpx_idct16x16_256_add_c(const tran_low_t * input,uint8_t * dest,int stride)722 void vpx_idct16x16_256_add_c(const tran_low_t *input, uint8_t *dest,
723 int stride) {
724 int i, j;
725 tran_low_t out[16 * 16];
726 tran_low_t *outptr = out;
727 tran_low_t temp_in[16], temp_out[16];
728
729 // First transform rows
730 for (i = 0; i < 16; ++i) {
731 idct16_c(input, outptr);
732 input += 16;
733 outptr += 16;
734 }
735
736 // Then transform columns
737 for (i = 0; i < 16; ++i) {
738 for (j = 0; j < 16; ++j) temp_in[j] = out[j * 16 + i];
739 idct16_c(temp_in, temp_out);
740 for (j = 0; j < 16; ++j) {
741 dest[j * stride + i] = clip_pixel_add(dest[j * stride + i],
742 ROUND_POWER_OF_TWO(temp_out[j], 6));
743 }
744 }
745 }
746
vpx_idct16x16_38_add_c(const tran_low_t * input,uint8_t * dest,int stride)747 void vpx_idct16x16_38_add_c(const tran_low_t *input, uint8_t *dest,
748 int stride) {
749 int i, j;
750 tran_low_t out[16 * 16] = { 0 };
751 tran_low_t *outptr = out;
752 tran_low_t temp_in[16], temp_out[16];
753
754 // First transform rows. Since all non-zero dct coefficients are in
755 // upper-left 8x8 area, we only need to calculate first 8 rows here.
756 for (i = 0; i < 8; ++i) {
757 idct16_c(input, outptr);
758 input += 16;
759 outptr += 16;
760 }
761
762 // Then transform columns
763 for (i = 0; i < 16; ++i) {
764 for (j = 0; j < 16; ++j) temp_in[j] = out[j * 16 + i];
765 idct16_c(temp_in, temp_out);
766 for (j = 0; j < 16; ++j) {
767 dest[j * stride + i] = clip_pixel_add(dest[j * stride + i],
768 ROUND_POWER_OF_TWO(temp_out[j], 6));
769 }
770 }
771 }
772
vpx_idct16x16_10_add_c(const tran_low_t * input,uint8_t * dest,int stride)773 void vpx_idct16x16_10_add_c(const tran_low_t *input, uint8_t *dest,
774 int stride) {
775 int i, j;
776 tran_low_t out[16 * 16] = { 0 };
777 tran_low_t *outptr = out;
778 tran_low_t temp_in[16], temp_out[16];
779
780 // First transform rows. Since all non-zero dct coefficients are in
781 // upper-left 4x4 area, we only need to calculate first 4 rows here.
782 for (i = 0; i < 4; ++i) {
783 idct16_c(input, outptr);
784 input += 16;
785 outptr += 16;
786 }
787
788 // Then transform columns
789 for (i = 0; i < 16; ++i) {
790 for (j = 0; j < 16; ++j) temp_in[j] = out[j * 16 + i];
791 idct16_c(temp_in, temp_out);
792 for (j = 0; j < 16; ++j) {
793 dest[j * stride + i] = clip_pixel_add(dest[j * stride + i],
794 ROUND_POWER_OF_TWO(temp_out[j], 6));
795 }
796 }
797 }
798
vpx_idct16x16_1_add_c(const tran_low_t * input,uint8_t * dest,int stride)799 void vpx_idct16x16_1_add_c(const tran_low_t *input, uint8_t *dest, int stride) {
800 int i, j;
801 tran_high_t a1;
802 tran_low_t out =
803 WRAPLOW(dct_const_round_shift((int16_t)input[0] * cospi_16_64));
804
805 out = WRAPLOW(dct_const_round_shift(out * cospi_16_64));
806 a1 = ROUND_POWER_OF_TWO(out, 6);
807 for (j = 0; j < 16; ++j) {
808 for (i = 0; i < 16; ++i) dest[i] = clip_pixel_add(dest[i], a1);
809 dest += stride;
810 }
811 }
812
idct32_c(const tran_low_t * input,tran_low_t * output)813 void idct32_c(const tran_low_t *input, tran_low_t *output) {
814 int16_t step1[32], step2[32];
815 tran_high_t temp1, temp2;
816
817 // stage 1
818 step1[0] = (int16_t)input[0];
819 step1[1] = (int16_t)input[16];
820 step1[2] = (int16_t)input[8];
821 step1[3] = (int16_t)input[24];
822 step1[4] = (int16_t)input[4];
823 step1[5] = (int16_t)input[20];
824 step1[6] = (int16_t)input[12];
825 step1[7] = (int16_t)input[28];
826 step1[8] = (int16_t)input[2];
827 step1[9] = (int16_t)input[18];
828 step1[10] = (int16_t)input[10];
829 step1[11] = (int16_t)input[26];
830 step1[12] = (int16_t)input[6];
831 step1[13] = (int16_t)input[22];
832 step1[14] = (int16_t)input[14];
833 step1[15] = (int16_t)input[30];
834
835 temp1 = (int16_t)input[1] * cospi_31_64 - (int16_t)input[31] * cospi_1_64;
836 temp2 = (int16_t)input[1] * cospi_1_64 + (int16_t)input[31] * cospi_31_64;
837 step1[16] = WRAPLOW(dct_const_round_shift(temp1));
838 step1[31] = WRAPLOW(dct_const_round_shift(temp2));
839
840 temp1 = (int16_t)input[17] * cospi_15_64 - (int16_t)input[15] * cospi_17_64;
841 temp2 = (int16_t)input[17] * cospi_17_64 + (int16_t)input[15] * cospi_15_64;
842 step1[17] = WRAPLOW(dct_const_round_shift(temp1));
843 step1[30] = WRAPLOW(dct_const_round_shift(temp2));
844
845 temp1 = (int16_t)input[9] * cospi_23_64 - (int16_t)input[23] * cospi_9_64;
846 temp2 = (int16_t)input[9] * cospi_9_64 + (int16_t)input[23] * cospi_23_64;
847 step1[18] = WRAPLOW(dct_const_round_shift(temp1));
848 step1[29] = WRAPLOW(dct_const_round_shift(temp2));
849
850 temp1 = (int16_t)input[25] * cospi_7_64 - (int16_t)input[7] * cospi_25_64;
851 temp2 = (int16_t)input[25] * cospi_25_64 + (int16_t)input[7] * cospi_7_64;
852 step1[19] = WRAPLOW(dct_const_round_shift(temp1));
853 step1[28] = WRAPLOW(dct_const_round_shift(temp2));
854
855 temp1 = (int16_t)input[5] * cospi_27_64 - (int16_t)input[27] * cospi_5_64;
856 temp2 = (int16_t)input[5] * cospi_5_64 + (int16_t)input[27] * cospi_27_64;
857 step1[20] = WRAPLOW(dct_const_round_shift(temp1));
858 step1[27] = WRAPLOW(dct_const_round_shift(temp2));
859
860 temp1 = (int16_t)input[21] * cospi_11_64 - (int16_t)input[11] * cospi_21_64;
861 temp2 = (int16_t)input[21] * cospi_21_64 + (int16_t)input[11] * cospi_11_64;
862 step1[21] = WRAPLOW(dct_const_round_shift(temp1));
863 step1[26] = WRAPLOW(dct_const_round_shift(temp2));
864
865 temp1 = (int16_t)input[13] * cospi_19_64 - (int16_t)input[19] * cospi_13_64;
866 temp2 = (int16_t)input[13] * cospi_13_64 + (int16_t)input[19] * cospi_19_64;
867 step1[22] = WRAPLOW(dct_const_round_shift(temp1));
868 step1[25] = WRAPLOW(dct_const_round_shift(temp2));
869
870 temp1 = (int16_t)input[29] * cospi_3_64 - (int16_t)input[3] * cospi_29_64;
871 temp2 = (int16_t)input[29] * cospi_29_64 + (int16_t)input[3] * cospi_3_64;
872 step1[23] = WRAPLOW(dct_const_round_shift(temp1));
873 step1[24] = WRAPLOW(dct_const_round_shift(temp2));
874
875 // stage 2
876 step2[0] = step1[0];
877 step2[1] = step1[1];
878 step2[2] = step1[2];
879 step2[3] = step1[3];
880 step2[4] = step1[4];
881 step2[5] = step1[5];
882 step2[6] = step1[6];
883 step2[7] = step1[7];
884
885 temp1 = step1[8] * cospi_30_64 - step1[15] * cospi_2_64;
886 temp2 = step1[8] * cospi_2_64 + step1[15] * cospi_30_64;
887 step2[8] = WRAPLOW(dct_const_round_shift(temp1));
888 step2[15] = WRAPLOW(dct_const_round_shift(temp2));
889
890 temp1 = step1[9] * cospi_14_64 - step1[14] * cospi_18_64;
891 temp2 = step1[9] * cospi_18_64 + step1[14] * cospi_14_64;
892 step2[9] = WRAPLOW(dct_const_round_shift(temp1));
893 step2[14] = WRAPLOW(dct_const_round_shift(temp2));
894
895 temp1 = step1[10] * cospi_22_64 - step1[13] * cospi_10_64;
896 temp2 = step1[10] * cospi_10_64 + step1[13] * cospi_22_64;
897 step2[10] = WRAPLOW(dct_const_round_shift(temp1));
898 step2[13] = WRAPLOW(dct_const_round_shift(temp2));
899
900 temp1 = step1[11] * cospi_6_64 - step1[12] * cospi_26_64;
901 temp2 = step1[11] * cospi_26_64 + step1[12] * cospi_6_64;
902 step2[11] = WRAPLOW(dct_const_round_shift(temp1));
903 step2[12] = WRAPLOW(dct_const_round_shift(temp2));
904
905 step2[16] = WRAPLOW(step1[16] + step1[17]);
906 step2[17] = WRAPLOW(step1[16] - step1[17]);
907 step2[18] = WRAPLOW(-step1[18] + step1[19]);
908 step2[19] = WRAPLOW(step1[18] + step1[19]);
909 step2[20] = WRAPLOW(step1[20] + step1[21]);
910 step2[21] = WRAPLOW(step1[20] - step1[21]);
911 step2[22] = WRAPLOW(-step1[22] + step1[23]);
912 step2[23] = WRAPLOW(step1[22] + step1[23]);
913 step2[24] = WRAPLOW(step1[24] + step1[25]);
914 step2[25] = WRAPLOW(step1[24] - step1[25]);
915 step2[26] = WRAPLOW(-step1[26] + step1[27]);
916 step2[27] = WRAPLOW(step1[26] + step1[27]);
917 step2[28] = WRAPLOW(step1[28] + step1[29]);
918 step2[29] = WRAPLOW(step1[28] - step1[29]);
919 step2[30] = WRAPLOW(-step1[30] + step1[31]);
920 step2[31] = WRAPLOW(step1[30] + step1[31]);
921
922 // stage 3
923 step1[0] = step2[0];
924 step1[1] = step2[1];
925 step1[2] = step2[2];
926 step1[3] = step2[3];
927
928 temp1 = step2[4] * cospi_28_64 - step2[7] * cospi_4_64;
929 temp2 = step2[4] * cospi_4_64 + step2[7] * cospi_28_64;
930 step1[4] = WRAPLOW(dct_const_round_shift(temp1));
931 step1[7] = WRAPLOW(dct_const_round_shift(temp2));
932 temp1 = step2[5] * cospi_12_64 - step2[6] * cospi_20_64;
933 temp2 = step2[5] * cospi_20_64 + step2[6] * cospi_12_64;
934 step1[5] = WRAPLOW(dct_const_round_shift(temp1));
935 step1[6] = WRAPLOW(dct_const_round_shift(temp2));
936
937 step1[8] = WRAPLOW(step2[8] + step2[9]);
938 step1[9] = WRAPLOW(step2[8] - step2[9]);
939 step1[10] = WRAPLOW(-step2[10] + step2[11]);
940 step1[11] = WRAPLOW(step2[10] + step2[11]);
941 step1[12] = WRAPLOW(step2[12] + step2[13]);
942 step1[13] = WRAPLOW(step2[12] - step2[13]);
943 step1[14] = WRAPLOW(-step2[14] + step2[15]);
944 step1[15] = WRAPLOW(step2[14] + step2[15]);
945
946 step1[16] = step2[16];
947 step1[31] = step2[31];
948 temp1 = -step2[17] * cospi_4_64 + step2[30] * cospi_28_64;
949 temp2 = step2[17] * cospi_28_64 + step2[30] * cospi_4_64;
950 step1[17] = WRAPLOW(dct_const_round_shift(temp1));
951 step1[30] = WRAPLOW(dct_const_round_shift(temp2));
952 temp1 = -step2[18] * cospi_28_64 - step2[29] * cospi_4_64;
953 temp2 = -step2[18] * cospi_4_64 + step2[29] * cospi_28_64;
954 step1[18] = WRAPLOW(dct_const_round_shift(temp1));
955 step1[29] = WRAPLOW(dct_const_round_shift(temp2));
956 step1[19] = step2[19];
957 step1[20] = step2[20];
958 temp1 = -step2[21] * cospi_20_64 + step2[26] * cospi_12_64;
959 temp2 = step2[21] * cospi_12_64 + step2[26] * cospi_20_64;
960 step1[21] = WRAPLOW(dct_const_round_shift(temp1));
961 step1[26] = WRAPLOW(dct_const_round_shift(temp2));
962 temp1 = -step2[22] * cospi_12_64 - step2[25] * cospi_20_64;
963 temp2 = -step2[22] * cospi_20_64 + step2[25] * cospi_12_64;
964 step1[22] = WRAPLOW(dct_const_round_shift(temp1));
965 step1[25] = WRAPLOW(dct_const_round_shift(temp2));
966 step1[23] = step2[23];
967 step1[24] = step2[24];
968 step1[27] = step2[27];
969 step1[28] = step2[28];
970
971 // stage 4
972 temp1 = (step1[0] + step1[1]) * cospi_16_64;
973 temp2 = (step1[0] - step1[1]) * cospi_16_64;
974 step2[0] = WRAPLOW(dct_const_round_shift(temp1));
975 step2[1] = WRAPLOW(dct_const_round_shift(temp2));
976 temp1 = step1[2] * cospi_24_64 - step1[3] * cospi_8_64;
977 temp2 = step1[2] * cospi_8_64 + step1[3] * cospi_24_64;
978 step2[2] = WRAPLOW(dct_const_round_shift(temp1));
979 step2[3] = WRAPLOW(dct_const_round_shift(temp2));
980 step2[4] = WRAPLOW(step1[4] + step1[5]);
981 step2[5] = WRAPLOW(step1[4] - step1[5]);
982 step2[6] = WRAPLOW(-step1[6] + step1[7]);
983 step2[7] = WRAPLOW(step1[6] + step1[7]);
984
985 step2[8] = step1[8];
986 step2[15] = step1[15];
987 temp1 = -step1[9] * cospi_8_64 + step1[14] * cospi_24_64;
988 temp2 = step1[9] * cospi_24_64 + step1[14] * cospi_8_64;
989 step2[9] = WRAPLOW(dct_const_round_shift(temp1));
990 step2[14] = WRAPLOW(dct_const_round_shift(temp2));
991 temp1 = -step1[10] * cospi_24_64 - step1[13] * cospi_8_64;
992 temp2 = -step1[10] * cospi_8_64 + step1[13] * cospi_24_64;
993 step2[10] = WRAPLOW(dct_const_round_shift(temp1));
994 step2[13] = WRAPLOW(dct_const_round_shift(temp2));
995 step2[11] = step1[11];
996 step2[12] = step1[12];
997
998 step2[16] = WRAPLOW(step1[16] + step1[19]);
999 step2[17] = WRAPLOW(step1[17] + step1[18]);
1000 step2[18] = WRAPLOW(step1[17] - step1[18]);
1001 step2[19] = WRAPLOW(step1[16] - step1[19]);
1002 step2[20] = WRAPLOW(-step1[20] + step1[23]);
1003 step2[21] = WRAPLOW(-step1[21] + step1[22]);
1004 step2[22] = WRAPLOW(step1[21] + step1[22]);
1005 step2[23] = WRAPLOW(step1[20] + step1[23]);
1006
1007 step2[24] = WRAPLOW(step1[24] + step1[27]);
1008 step2[25] = WRAPLOW(step1[25] + step1[26]);
1009 step2[26] = WRAPLOW(step1[25] - step1[26]);
1010 step2[27] = WRAPLOW(step1[24] - step1[27]);
1011 step2[28] = WRAPLOW(-step1[28] + step1[31]);
1012 step2[29] = WRAPLOW(-step1[29] + step1[30]);
1013 step2[30] = WRAPLOW(step1[29] + step1[30]);
1014 step2[31] = WRAPLOW(step1[28] + step1[31]);
1015
1016 // stage 5
1017 step1[0] = WRAPLOW(step2[0] + step2[3]);
1018 step1[1] = WRAPLOW(step2[1] + step2[2]);
1019 step1[2] = WRAPLOW(step2[1] - step2[2]);
1020 step1[3] = WRAPLOW(step2[0] - step2[3]);
1021 step1[4] = step2[4];
1022 temp1 = (step2[6] - step2[5]) * cospi_16_64;
1023 temp2 = (step2[5] + step2[6]) * cospi_16_64;
1024 step1[5] = WRAPLOW(dct_const_round_shift(temp1));
1025 step1[6] = WRAPLOW(dct_const_round_shift(temp2));
1026 step1[7] = step2[7];
1027
1028 step1[8] = WRAPLOW(step2[8] + step2[11]);
1029 step1[9] = WRAPLOW(step2[9] + step2[10]);
1030 step1[10] = WRAPLOW(step2[9] - step2[10]);
1031 step1[11] = WRAPLOW(step2[8] - step2[11]);
1032 step1[12] = WRAPLOW(-step2[12] + step2[15]);
1033 step1[13] = WRAPLOW(-step2[13] + step2[14]);
1034 step1[14] = WRAPLOW(step2[13] + step2[14]);
1035 step1[15] = WRAPLOW(step2[12] + step2[15]);
1036
1037 step1[16] = step2[16];
1038 step1[17] = step2[17];
1039 temp1 = -step2[18] * cospi_8_64 + step2[29] * cospi_24_64;
1040 temp2 = step2[18] * cospi_24_64 + step2[29] * cospi_8_64;
1041 step1[18] = WRAPLOW(dct_const_round_shift(temp1));
1042 step1[29] = WRAPLOW(dct_const_round_shift(temp2));
1043 temp1 = -step2[19] * cospi_8_64 + step2[28] * cospi_24_64;
1044 temp2 = step2[19] * cospi_24_64 + step2[28] * cospi_8_64;
1045 step1[19] = WRAPLOW(dct_const_round_shift(temp1));
1046 step1[28] = WRAPLOW(dct_const_round_shift(temp2));
1047 temp1 = -step2[20] * cospi_24_64 - step2[27] * cospi_8_64;
1048 temp2 = -step2[20] * cospi_8_64 + step2[27] * cospi_24_64;
1049 step1[20] = WRAPLOW(dct_const_round_shift(temp1));
1050 step1[27] = WRAPLOW(dct_const_round_shift(temp2));
1051 temp1 = -step2[21] * cospi_24_64 - step2[26] * cospi_8_64;
1052 temp2 = -step2[21] * cospi_8_64 + step2[26] * cospi_24_64;
1053 step1[21] = WRAPLOW(dct_const_round_shift(temp1));
1054 step1[26] = WRAPLOW(dct_const_round_shift(temp2));
1055 step1[22] = step2[22];
1056 step1[23] = step2[23];
1057 step1[24] = step2[24];
1058 step1[25] = step2[25];
1059 step1[30] = step2[30];
1060 step1[31] = step2[31];
1061
1062 // stage 6
1063 step2[0] = WRAPLOW(step1[0] + step1[7]);
1064 step2[1] = WRAPLOW(step1[1] + step1[6]);
1065 step2[2] = WRAPLOW(step1[2] + step1[5]);
1066 step2[3] = WRAPLOW(step1[3] + step1[4]);
1067 step2[4] = WRAPLOW(step1[3] - step1[4]);
1068 step2[5] = WRAPLOW(step1[2] - step1[5]);
1069 step2[6] = WRAPLOW(step1[1] - step1[6]);
1070 step2[7] = WRAPLOW(step1[0] - step1[7]);
1071 step2[8] = step1[8];
1072 step2[9] = step1[9];
1073 temp1 = (-step1[10] + step1[13]) * cospi_16_64;
1074 temp2 = (step1[10] + step1[13]) * cospi_16_64;
1075 step2[10] = WRAPLOW(dct_const_round_shift(temp1));
1076 step2[13] = WRAPLOW(dct_const_round_shift(temp2));
1077 temp1 = (-step1[11] + step1[12]) * cospi_16_64;
1078 temp2 = (step1[11] + step1[12]) * cospi_16_64;
1079 step2[11] = WRAPLOW(dct_const_round_shift(temp1));
1080 step2[12] = WRAPLOW(dct_const_round_shift(temp2));
1081 step2[14] = step1[14];
1082 step2[15] = step1[15];
1083
1084 step2[16] = WRAPLOW(step1[16] + step1[23]);
1085 step2[17] = WRAPLOW(step1[17] + step1[22]);
1086 step2[18] = WRAPLOW(step1[18] + step1[21]);
1087 step2[19] = WRAPLOW(step1[19] + step1[20]);
1088 step2[20] = WRAPLOW(step1[19] - step1[20]);
1089 step2[21] = WRAPLOW(step1[18] - step1[21]);
1090 step2[22] = WRAPLOW(step1[17] - step1[22]);
1091 step2[23] = WRAPLOW(step1[16] - step1[23]);
1092
1093 step2[24] = WRAPLOW(-step1[24] + step1[31]);
1094 step2[25] = WRAPLOW(-step1[25] + step1[30]);
1095 step2[26] = WRAPLOW(-step1[26] + step1[29]);
1096 step2[27] = WRAPLOW(-step1[27] + step1[28]);
1097 step2[28] = WRAPLOW(step1[27] + step1[28]);
1098 step2[29] = WRAPLOW(step1[26] + step1[29]);
1099 step2[30] = WRAPLOW(step1[25] + step1[30]);
1100 step2[31] = WRAPLOW(step1[24] + step1[31]);
1101
1102 // stage 7
1103 step1[0] = WRAPLOW(step2[0] + step2[15]);
1104 step1[1] = WRAPLOW(step2[1] + step2[14]);
1105 step1[2] = WRAPLOW(step2[2] + step2[13]);
1106 step1[3] = WRAPLOW(step2[3] + step2[12]);
1107 step1[4] = WRAPLOW(step2[4] + step2[11]);
1108 step1[5] = WRAPLOW(step2[5] + step2[10]);
1109 step1[6] = WRAPLOW(step2[6] + step2[9]);
1110 step1[7] = WRAPLOW(step2[7] + step2[8]);
1111 step1[8] = WRAPLOW(step2[7] - step2[8]);
1112 step1[9] = WRAPLOW(step2[6] - step2[9]);
1113 step1[10] = WRAPLOW(step2[5] - step2[10]);
1114 step1[11] = WRAPLOW(step2[4] - step2[11]);
1115 step1[12] = WRAPLOW(step2[3] - step2[12]);
1116 step1[13] = WRAPLOW(step2[2] - step2[13]);
1117 step1[14] = WRAPLOW(step2[1] - step2[14]);
1118 step1[15] = WRAPLOW(step2[0] - step2[15]);
1119
1120 step1[16] = step2[16];
1121 step1[17] = step2[17];
1122 step1[18] = step2[18];
1123 step1[19] = step2[19];
1124 temp1 = (-step2[20] + step2[27]) * cospi_16_64;
1125 temp2 = (step2[20] + step2[27]) * cospi_16_64;
1126 step1[20] = WRAPLOW(dct_const_round_shift(temp1));
1127 step1[27] = WRAPLOW(dct_const_round_shift(temp2));
1128 temp1 = (-step2[21] + step2[26]) * cospi_16_64;
1129 temp2 = (step2[21] + step2[26]) * cospi_16_64;
1130 step1[21] = WRAPLOW(dct_const_round_shift(temp1));
1131 step1[26] = WRAPLOW(dct_const_round_shift(temp2));
1132 temp1 = (-step2[22] + step2[25]) * cospi_16_64;
1133 temp2 = (step2[22] + step2[25]) * cospi_16_64;
1134 step1[22] = WRAPLOW(dct_const_round_shift(temp1));
1135 step1[25] = WRAPLOW(dct_const_round_shift(temp2));
1136 temp1 = (-step2[23] + step2[24]) * cospi_16_64;
1137 temp2 = (step2[23] + step2[24]) * cospi_16_64;
1138 step1[23] = WRAPLOW(dct_const_round_shift(temp1));
1139 step1[24] = WRAPLOW(dct_const_round_shift(temp2));
1140 step1[28] = step2[28];
1141 step1[29] = step2[29];
1142 step1[30] = step2[30];
1143 step1[31] = step2[31];
1144
1145 // final stage
1146 output[0] = WRAPLOW(step1[0] + step1[31]);
1147 output[1] = WRAPLOW(step1[1] + step1[30]);
1148 output[2] = WRAPLOW(step1[2] + step1[29]);
1149 output[3] = WRAPLOW(step1[3] + step1[28]);
1150 output[4] = WRAPLOW(step1[4] + step1[27]);
1151 output[5] = WRAPLOW(step1[5] + step1[26]);
1152 output[6] = WRAPLOW(step1[6] + step1[25]);
1153 output[7] = WRAPLOW(step1[7] + step1[24]);
1154 output[8] = WRAPLOW(step1[8] + step1[23]);
1155 output[9] = WRAPLOW(step1[9] + step1[22]);
1156 output[10] = WRAPLOW(step1[10] + step1[21]);
1157 output[11] = WRAPLOW(step1[11] + step1[20]);
1158 output[12] = WRAPLOW(step1[12] + step1[19]);
1159 output[13] = WRAPLOW(step1[13] + step1[18]);
1160 output[14] = WRAPLOW(step1[14] + step1[17]);
1161 output[15] = WRAPLOW(step1[15] + step1[16]);
1162 output[16] = WRAPLOW(step1[15] - step1[16]);
1163 output[17] = WRAPLOW(step1[14] - step1[17]);
1164 output[18] = WRAPLOW(step1[13] - step1[18]);
1165 output[19] = WRAPLOW(step1[12] - step1[19]);
1166 output[20] = WRAPLOW(step1[11] - step1[20]);
1167 output[21] = WRAPLOW(step1[10] - step1[21]);
1168 output[22] = WRAPLOW(step1[9] - step1[22]);
1169 output[23] = WRAPLOW(step1[8] - step1[23]);
1170 output[24] = WRAPLOW(step1[7] - step1[24]);
1171 output[25] = WRAPLOW(step1[6] - step1[25]);
1172 output[26] = WRAPLOW(step1[5] - step1[26]);
1173 output[27] = WRAPLOW(step1[4] - step1[27]);
1174 output[28] = WRAPLOW(step1[3] - step1[28]);
1175 output[29] = WRAPLOW(step1[2] - step1[29]);
1176 output[30] = WRAPLOW(step1[1] - step1[30]);
1177 output[31] = WRAPLOW(step1[0] - step1[31]);
1178 }
1179
vpx_idct32x32_1024_add_c(const tran_low_t * input,uint8_t * dest,int stride)1180 void vpx_idct32x32_1024_add_c(const tran_low_t *input, uint8_t *dest,
1181 int stride) {
1182 int i, j;
1183 tran_low_t out[32 * 32];
1184 tran_low_t *outptr = out;
1185 tran_low_t temp_in[32], temp_out[32];
1186
1187 // Rows
1188 for (i = 0; i < 32; ++i) {
1189 int16_t zero_coeff = 0;
1190 for (j = 0; j < 32; ++j) zero_coeff |= input[j];
1191
1192 if (zero_coeff)
1193 idct32_c(input, outptr);
1194 else
1195 memset(outptr, 0, sizeof(tran_low_t) * 32);
1196 input += 32;
1197 outptr += 32;
1198 }
1199
1200 // Columns
1201 for (i = 0; i < 32; ++i) {
1202 for (j = 0; j < 32; ++j) temp_in[j] = out[j * 32 + i];
1203 idct32_c(temp_in, temp_out);
1204 for (j = 0; j < 32; ++j) {
1205 dest[j * stride + i] = clip_pixel_add(dest[j * stride + i],
1206 ROUND_POWER_OF_TWO(temp_out[j], 6));
1207 }
1208 }
1209 }
1210
vpx_idct32x32_135_add_c(const tran_low_t * input,uint8_t * dest,int stride)1211 void vpx_idct32x32_135_add_c(const tran_low_t *input, uint8_t *dest,
1212 int stride) {
1213 int i, j;
1214 tran_low_t out[32 * 32] = { 0 };
1215 tran_low_t *outptr = out;
1216 tran_low_t temp_in[32], temp_out[32];
1217
1218 // Rows
1219 // Only upper-left 16x16 has non-zero coeff
1220 for (i = 0; i < 16; ++i) {
1221 idct32_c(input, outptr);
1222 input += 32;
1223 outptr += 32;
1224 }
1225
1226 // Columns
1227 for (i = 0; i < 32; ++i) {
1228 for (j = 0; j < 32; ++j) temp_in[j] = out[j * 32 + i];
1229 idct32_c(temp_in, temp_out);
1230 for (j = 0; j < 32; ++j) {
1231 dest[j * stride + i] = clip_pixel_add(dest[j * stride + i],
1232 ROUND_POWER_OF_TWO(temp_out[j], 6));
1233 }
1234 }
1235 }
1236
vpx_idct32x32_34_add_c(const tran_low_t * input,uint8_t * dest,int stride)1237 void vpx_idct32x32_34_add_c(const tran_low_t *input, uint8_t *dest,
1238 int stride) {
1239 int i, j;
1240 tran_low_t out[32 * 32] = { 0 };
1241 tran_low_t *outptr = out;
1242 tran_low_t temp_in[32], temp_out[32];
1243
1244 // Rows
1245 // Only upper-left 8x8 has non-zero coeff
1246 for (i = 0; i < 8; ++i) {
1247 idct32_c(input, outptr);
1248 input += 32;
1249 outptr += 32;
1250 }
1251
1252 // Columns
1253 for (i = 0; i < 32; ++i) {
1254 for (j = 0; j < 32; ++j) temp_in[j] = out[j * 32 + i];
1255 idct32_c(temp_in, temp_out);
1256 for (j = 0; j < 32; ++j) {
1257 dest[j * stride + i] = clip_pixel_add(dest[j * stride + i],
1258 ROUND_POWER_OF_TWO(temp_out[j], 6));
1259 }
1260 }
1261 }
1262
vpx_idct32x32_1_add_c(const tran_low_t * input,uint8_t * dest,int stride)1263 void vpx_idct32x32_1_add_c(const tran_low_t *input, uint8_t *dest, int stride) {
1264 int i, j;
1265 tran_high_t a1;
1266 tran_low_t out =
1267 WRAPLOW(dct_const_round_shift((int16_t)input[0] * cospi_16_64));
1268
1269 out = WRAPLOW(dct_const_round_shift(out * cospi_16_64));
1270 a1 = ROUND_POWER_OF_TWO(out, 6);
1271
1272 for (j = 0; j < 32; ++j) {
1273 for (i = 0; i < 32; ++i) dest[i] = clip_pixel_add(dest[i], a1);
1274 dest += stride;
1275 }
1276 }
1277
1278 #if CONFIG_VP9_HIGHBITDEPTH
1279
1280 // 12 signal input bits + 7 2D forward transform amplify bits + 5 1D inverse
1281 // transform amplify bits + 1 bit for contingency in rounding and quantizing
1282 #define HIGHBD_VALID_TXFM_MAGNITUDE_RANGE (1 << 25)
1283
detect_invalid_highbd_input(const tran_low_t * input,int size)1284 static INLINE int detect_invalid_highbd_input(const tran_low_t *input,
1285 int size) {
1286 int i;
1287 for (i = 0; i < size; ++i)
1288 if (abs(input[i]) >= HIGHBD_VALID_TXFM_MAGNITUDE_RANGE) return 1;
1289 return 0;
1290 }
1291
vpx_highbd_iwht4x4_16_add_c(const tran_low_t * input,uint16_t * dest,int stride,int bd)1292 void vpx_highbd_iwht4x4_16_add_c(const tran_low_t *input, uint16_t *dest,
1293 int stride, int bd) {
1294 /* 4-point reversible, orthonormal inverse Walsh-Hadamard in 3.5 adds,
1295 0.5 shifts per pixel. */
1296 int i;
1297 tran_low_t output[16];
1298 tran_high_t a1, b1, c1, d1, e1;
1299 const tran_low_t *ip = input;
1300 tran_low_t *op = output;
1301
1302 for (i = 0; i < 4; i++) {
1303 a1 = ip[0] >> UNIT_QUANT_SHIFT;
1304 c1 = ip[1] >> UNIT_QUANT_SHIFT;
1305 d1 = ip[2] >> UNIT_QUANT_SHIFT;
1306 b1 = ip[3] >> UNIT_QUANT_SHIFT;
1307 a1 += c1;
1308 d1 -= b1;
1309 e1 = (a1 - d1) >> 1;
1310 b1 = e1 - b1;
1311 c1 = e1 - c1;
1312 a1 -= b1;
1313 d1 += c1;
1314 op[0] = HIGHBD_WRAPLOW(a1, bd);
1315 op[1] = HIGHBD_WRAPLOW(b1, bd);
1316 op[2] = HIGHBD_WRAPLOW(c1, bd);
1317 op[3] = HIGHBD_WRAPLOW(d1, bd);
1318 ip += 4;
1319 op += 4;
1320 }
1321
1322 ip = output;
1323 for (i = 0; i < 4; i++) {
1324 a1 = ip[4 * 0];
1325 c1 = ip[4 * 1];
1326 d1 = ip[4 * 2];
1327 b1 = ip[4 * 3];
1328 a1 += c1;
1329 d1 -= b1;
1330 e1 = (a1 - d1) >> 1;
1331 b1 = e1 - b1;
1332 c1 = e1 - c1;
1333 a1 -= b1;
1334 d1 += c1;
1335 dest[stride * 0] =
1336 highbd_clip_pixel_add(dest[stride * 0], HIGHBD_WRAPLOW(a1, bd), bd);
1337 dest[stride * 1] =
1338 highbd_clip_pixel_add(dest[stride * 1], HIGHBD_WRAPLOW(b1, bd), bd);
1339 dest[stride * 2] =
1340 highbd_clip_pixel_add(dest[stride * 2], HIGHBD_WRAPLOW(c1, bd), bd);
1341 dest[stride * 3] =
1342 highbd_clip_pixel_add(dest[stride * 3], HIGHBD_WRAPLOW(d1, bd), bd);
1343
1344 ip++;
1345 dest++;
1346 }
1347 }
1348
vpx_highbd_iwht4x4_1_add_c(const tran_low_t * input,uint16_t * dest,int stride,int bd)1349 void vpx_highbd_iwht4x4_1_add_c(const tran_low_t *input, uint16_t *dest,
1350 int stride, int bd) {
1351 int i;
1352 tran_high_t a1, e1;
1353 tran_low_t tmp[4];
1354 const tran_low_t *ip = input;
1355 tran_low_t *op = tmp;
1356 (void)bd;
1357
1358 a1 = ip[0] >> UNIT_QUANT_SHIFT;
1359 e1 = a1 >> 1;
1360 a1 -= e1;
1361 op[0] = HIGHBD_WRAPLOW(a1, bd);
1362 op[1] = op[2] = op[3] = HIGHBD_WRAPLOW(e1, bd);
1363
1364 ip = tmp;
1365 for (i = 0; i < 4; i++) {
1366 e1 = ip[0] >> 1;
1367 a1 = ip[0] - e1;
1368 dest[stride * 0] = highbd_clip_pixel_add(dest[stride * 0], a1, bd);
1369 dest[stride * 1] = highbd_clip_pixel_add(dest[stride * 1], e1, bd);
1370 dest[stride * 2] = highbd_clip_pixel_add(dest[stride * 2], e1, bd);
1371 dest[stride * 3] = highbd_clip_pixel_add(dest[stride * 3], e1, bd);
1372 ip++;
1373 dest++;
1374 }
1375 }
1376
vpx_highbd_iadst4_c(const tran_low_t * input,tran_low_t * output,int bd)1377 void vpx_highbd_iadst4_c(const tran_low_t *input, tran_low_t *output, int bd) {
1378 tran_high_t s0, s1, s2, s3, s4, s5, s6, s7;
1379 tran_low_t x0 = input[0];
1380 tran_low_t x1 = input[1];
1381 tran_low_t x2 = input[2];
1382 tran_low_t x3 = input[3];
1383 (void)bd;
1384
1385 if (detect_invalid_highbd_input(input, 4)) {
1386 #if CONFIG_COEFFICIENT_RANGE_CHECKING
1387 assert(0 && "invalid highbd txfm input");
1388 #endif // CONFIG_COEFFICIENT_RANGE_CHECKING
1389 memset(output, 0, sizeof(*output) * 4);
1390 return;
1391 }
1392
1393 if (!(x0 | x1 | x2 | x3)) {
1394 memset(output, 0, 4 * sizeof(*output));
1395 return;
1396 }
1397
1398 s0 = (tran_high_t)sinpi_1_9 * x0;
1399 s1 = (tran_high_t)sinpi_2_9 * x0;
1400 s2 = (tran_high_t)sinpi_3_9 * x1;
1401 s3 = (tran_high_t)sinpi_4_9 * x2;
1402 s4 = (tran_high_t)sinpi_1_9 * x2;
1403 s5 = (tran_high_t)sinpi_2_9 * x3;
1404 s6 = (tran_high_t)sinpi_4_9 * x3;
1405 s7 = (tran_high_t)HIGHBD_WRAPLOW(x0 - x2 + x3, bd);
1406
1407 s0 = s0 + s3 + s5;
1408 s1 = s1 - s4 - s6;
1409 s3 = s2;
1410 s2 = sinpi_3_9 * s7;
1411
1412 // 1-D transform scaling factor is sqrt(2).
1413 // The overall dynamic range is 14b (input) + 14b (multiplication scaling)
1414 // + 1b (addition) = 29b.
1415 // Hence the output bit depth is 15b.
1416 output[0] = HIGHBD_WRAPLOW(dct_const_round_shift(s0 + s3), bd);
1417 output[1] = HIGHBD_WRAPLOW(dct_const_round_shift(s1 + s3), bd);
1418 output[2] = HIGHBD_WRAPLOW(dct_const_round_shift(s2), bd);
1419 output[3] = HIGHBD_WRAPLOW(dct_const_round_shift(s0 + s1 - s3), bd);
1420 }
1421
vpx_highbd_idct4_c(const tran_low_t * input,tran_low_t * output,int bd)1422 void vpx_highbd_idct4_c(const tran_low_t *input, tran_low_t *output, int bd) {
1423 tran_low_t step[4];
1424 tran_high_t temp1, temp2;
1425 (void)bd;
1426
1427 if (detect_invalid_highbd_input(input, 4)) {
1428 #if CONFIG_COEFFICIENT_RANGE_CHECKING
1429 assert(0 && "invalid highbd txfm input");
1430 #endif // CONFIG_COEFFICIENT_RANGE_CHECKING
1431 memset(output, 0, sizeof(*output) * 4);
1432 return;
1433 }
1434
1435 // stage 1
1436 temp1 = (input[0] + input[2]) * (tran_high_t)cospi_16_64;
1437 temp2 = (input[0] - input[2]) * (tran_high_t)cospi_16_64;
1438 step[0] = HIGHBD_WRAPLOW(dct_const_round_shift(temp1), bd);
1439 step[1] = HIGHBD_WRAPLOW(dct_const_round_shift(temp2), bd);
1440 temp1 =
1441 input[1] * (tran_high_t)cospi_24_64 - input[3] * (tran_high_t)cospi_8_64;
1442 temp2 =
1443 input[1] * (tran_high_t)cospi_8_64 + input[3] * (tran_high_t)cospi_24_64;
1444 step[2] = HIGHBD_WRAPLOW(dct_const_round_shift(temp1), bd);
1445 step[3] = HIGHBD_WRAPLOW(dct_const_round_shift(temp2), bd);
1446
1447 // stage 2
1448 output[0] = HIGHBD_WRAPLOW(step[0] + step[3], bd);
1449 output[1] = HIGHBD_WRAPLOW(step[1] + step[2], bd);
1450 output[2] = HIGHBD_WRAPLOW(step[1] - step[2], bd);
1451 output[3] = HIGHBD_WRAPLOW(step[0] - step[3], bd);
1452 }
1453
vpx_highbd_idct4x4_16_add_c(const tran_low_t * input,uint16_t * dest,int stride,int bd)1454 void vpx_highbd_idct4x4_16_add_c(const tran_low_t *input, uint16_t *dest,
1455 int stride, int bd) {
1456 int i, j;
1457 tran_low_t out[4 * 4];
1458 tran_low_t *outptr = out;
1459 tran_low_t temp_in[4], temp_out[4];
1460
1461 // Rows
1462 for (i = 0; i < 4; ++i) {
1463 vpx_highbd_idct4_c(input, outptr, bd);
1464 input += 4;
1465 outptr += 4;
1466 }
1467
1468 // Columns
1469 for (i = 0; i < 4; ++i) {
1470 for (j = 0; j < 4; ++j) temp_in[j] = out[j * 4 + i];
1471 vpx_highbd_idct4_c(temp_in, temp_out, bd);
1472 for (j = 0; j < 4; ++j) {
1473 dest[j * stride + i] = highbd_clip_pixel_add(
1474 dest[j * stride + i], ROUND_POWER_OF_TWO(temp_out[j], 4), bd);
1475 }
1476 }
1477 }
1478
vpx_highbd_idct4x4_1_add_c(const tran_low_t * input,uint16_t * dest,int stride,int bd)1479 void vpx_highbd_idct4x4_1_add_c(const tran_low_t *input, uint16_t *dest,
1480 int stride, int bd) {
1481 int i;
1482 tran_high_t a1;
1483 tran_low_t out = HIGHBD_WRAPLOW(
1484 dct_const_round_shift(input[0] * (tran_high_t)cospi_16_64), bd);
1485
1486 out =
1487 HIGHBD_WRAPLOW(dct_const_round_shift(out * (tran_high_t)cospi_16_64), bd);
1488 a1 = ROUND_POWER_OF_TWO(out, 4);
1489
1490 for (i = 0; i < 4; i++) {
1491 dest[0] = highbd_clip_pixel_add(dest[0], a1, bd);
1492 dest[1] = highbd_clip_pixel_add(dest[1], a1, bd);
1493 dest[2] = highbd_clip_pixel_add(dest[2], a1, bd);
1494 dest[3] = highbd_clip_pixel_add(dest[3], a1, bd);
1495 dest += stride;
1496 }
1497 }
1498
vpx_highbd_iadst8_c(const tran_low_t * input,tran_low_t * output,int bd)1499 void vpx_highbd_iadst8_c(const tran_low_t *input, tran_low_t *output, int bd) {
1500 tran_high_t s0, s1, s2, s3, s4, s5, s6, s7;
1501 tran_low_t x0 = input[7];
1502 tran_low_t x1 = input[0];
1503 tran_low_t x2 = input[5];
1504 tran_low_t x3 = input[2];
1505 tran_low_t x4 = input[3];
1506 tran_low_t x5 = input[4];
1507 tran_low_t x6 = input[1];
1508 tran_low_t x7 = input[6];
1509 (void)bd;
1510
1511 if (detect_invalid_highbd_input(input, 8)) {
1512 #if CONFIG_COEFFICIENT_RANGE_CHECKING
1513 assert(0 && "invalid highbd txfm input");
1514 #endif // CONFIG_COEFFICIENT_RANGE_CHECKING
1515 memset(output, 0, sizeof(*output) * 8);
1516 return;
1517 }
1518
1519 if (!(x0 | x1 | x2 | x3 | x4 | x5 | x6 | x7)) {
1520 memset(output, 0, 8 * sizeof(*output));
1521 return;
1522 }
1523
1524 // stage 1
1525 s0 = (tran_high_t)cospi_2_64 * x0 + (tran_high_t)cospi_30_64 * x1;
1526 s1 = (tran_high_t)cospi_30_64 * x0 - (tran_high_t)cospi_2_64 * x1;
1527 s2 = (tran_high_t)cospi_10_64 * x2 + (tran_high_t)cospi_22_64 * x3;
1528 s3 = (tran_high_t)cospi_22_64 * x2 - (tran_high_t)cospi_10_64 * x3;
1529 s4 = (tran_high_t)cospi_18_64 * x4 + (tran_high_t)cospi_14_64 * x5;
1530 s5 = (tran_high_t)cospi_14_64 * x4 - (tran_high_t)cospi_18_64 * x5;
1531 s6 = (tran_high_t)cospi_26_64 * x6 + (tran_high_t)cospi_6_64 * x7;
1532 s7 = (tran_high_t)cospi_6_64 * x6 - (tran_high_t)cospi_26_64 * x7;
1533
1534 x0 = HIGHBD_WRAPLOW(dct_const_round_shift(s0 + s4), bd);
1535 x1 = HIGHBD_WRAPLOW(dct_const_round_shift(s1 + s5), bd);
1536 x2 = HIGHBD_WRAPLOW(dct_const_round_shift(s2 + s6), bd);
1537 x3 = HIGHBD_WRAPLOW(dct_const_round_shift(s3 + s7), bd);
1538 x4 = HIGHBD_WRAPLOW(dct_const_round_shift(s0 - s4), bd);
1539 x5 = HIGHBD_WRAPLOW(dct_const_round_shift(s1 - s5), bd);
1540 x6 = HIGHBD_WRAPLOW(dct_const_round_shift(s2 - s6), bd);
1541 x7 = HIGHBD_WRAPLOW(dct_const_round_shift(s3 - s7), bd);
1542
1543 // stage 2
1544 s0 = x0;
1545 s1 = x1;
1546 s2 = x2;
1547 s3 = x3;
1548 s4 = (tran_high_t)cospi_8_64 * x4 + (tran_high_t)cospi_24_64 * x5;
1549 s5 = (tran_high_t)cospi_24_64 * x4 - (tran_high_t)cospi_8_64 * x5;
1550 s6 = (tran_high_t)(-cospi_24_64) * x6 + (tran_high_t)cospi_8_64 * x7;
1551 s7 = (tran_high_t)cospi_8_64 * x6 + (tran_high_t)cospi_24_64 * x7;
1552
1553 x0 = HIGHBD_WRAPLOW(s0 + s2, bd);
1554 x1 = HIGHBD_WRAPLOW(s1 + s3, bd);
1555 x2 = HIGHBD_WRAPLOW(s0 - s2, bd);
1556 x3 = HIGHBD_WRAPLOW(s1 - s3, bd);
1557 x4 = HIGHBD_WRAPLOW(dct_const_round_shift(s4 + s6), bd);
1558 x5 = HIGHBD_WRAPLOW(dct_const_round_shift(s5 + s7), bd);
1559 x6 = HIGHBD_WRAPLOW(dct_const_round_shift(s4 - s6), bd);
1560 x7 = HIGHBD_WRAPLOW(dct_const_round_shift(s5 - s7), bd);
1561
1562 // stage 3
1563 s2 = (tran_high_t)cospi_16_64 * (x2 + x3);
1564 s3 = (tran_high_t)cospi_16_64 * (x2 - x3);
1565 s6 = (tran_high_t)cospi_16_64 * (x6 + x7);
1566 s7 = (tran_high_t)cospi_16_64 * (x6 - x7);
1567
1568 x2 = HIGHBD_WRAPLOW(dct_const_round_shift(s2), bd);
1569 x3 = HIGHBD_WRAPLOW(dct_const_round_shift(s3), bd);
1570 x6 = HIGHBD_WRAPLOW(dct_const_round_shift(s6), bd);
1571 x7 = HIGHBD_WRAPLOW(dct_const_round_shift(s7), bd);
1572
1573 output[0] = HIGHBD_WRAPLOW(x0, bd);
1574 output[1] = HIGHBD_WRAPLOW(-x4, bd);
1575 output[2] = HIGHBD_WRAPLOW(x6, bd);
1576 output[3] = HIGHBD_WRAPLOW(-x2, bd);
1577 output[4] = HIGHBD_WRAPLOW(x3, bd);
1578 output[5] = HIGHBD_WRAPLOW(-x7, bd);
1579 output[6] = HIGHBD_WRAPLOW(x5, bd);
1580 output[7] = HIGHBD_WRAPLOW(-x1, bd);
1581 }
1582
vpx_highbd_idct8_c(const tran_low_t * input,tran_low_t * output,int bd)1583 void vpx_highbd_idct8_c(const tran_low_t *input, tran_low_t *output, int bd) {
1584 tran_low_t step1[8], step2[8];
1585 tran_high_t temp1, temp2;
1586
1587 if (detect_invalid_highbd_input(input, 8)) {
1588 #if CONFIG_COEFFICIENT_RANGE_CHECKING
1589 assert(0 && "invalid highbd txfm input");
1590 #endif // CONFIG_COEFFICIENT_RANGE_CHECKING
1591 memset(output, 0, sizeof(*output) * 8);
1592 return;
1593 }
1594
1595 // stage 1
1596 step1[0] = input[0];
1597 step1[2] = input[4];
1598 step1[1] = input[2];
1599 step1[3] = input[6];
1600 temp1 =
1601 input[1] * (tran_high_t)cospi_28_64 - input[7] * (tran_high_t)cospi_4_64;
1602 temp2 =
1603 input[1] * (tran_high_t)cospi_4_64 + input[7] * (tran_high_t)cospi_28_64;
1604 step1[4] = HIGHBD_WRAPLOW(dct_const_round_shift(temp1), bd);
1605 step1[7] = HIGHBD_WRAPLOW(dct_const_round_shift(temp2), bd);
1606 temp1 =
1607 input[5] * (tran_high_t)cospi_12_64 - input[3] * (tran_high_t)cospi_20_64;
1608 temp2 =
1609 input[5] * (tran_high_t)cospi_20_64 + input[3] * (tran_high_t)cospi_12_64;
1610 step1[5] = HIGHBD_WRAPLOW(dct_const_round_shift(temp1), bd);
1611 step1[6] = HIGHBD_WRAPLOW(dct_const_round_shift(temp2), bd);
1612
1613 // stage 2 & stage 3 - even half
1614 vpx_highbd_idct4_c(step1, step1, bd);
1615
1616 // stage 2 - odd half
1617 step2[4] = HIGHBD_WRAPLOW(step1[4] + step1[5], bd);
1618 step2[5] = HIGHBD_WRAPLOW(step1[4] - step1[5], bd);
1619 step2[6] = HIGHBD_WRAPLOW(-step1[6] + step1[7], bd);
1620 step2[7] = HIGHBD_WRAPLOW(step1[6] + step1[7], bd);
1621
1622 // stage 3 - odd half
1623 step1[4] = step2[4];
1624 temp1 = (step2[6] - step2[5]) * (tran_high_t)cospi_16_64;
1625 temp2 = (step2[5] + step2[6]) * (tran_high_t)cospi_16_64;
1626 step1[5] = HIGHBD_WRAPLOW(dct_const_round_shift(temp1), bd);
1627 step1[6] = HIGHBD_WRAPLOW(dct_const_round_shift(temp2), bd);
1628 step1[7] = step2[7];
1629
1630 // stage 4
1631 output[0] = HIGHBD_WRAPLOW(step1[0] + step1[7], bd);
1632 output[1] = HIGHBD_WRAPLOW(step1[1] + step1[6], bd);
1633 output[2] = HIGHBD_WRAPLOW(step1[2] + step1[5], bd);
1634 output[3] = HIGHBD_WRAPLOW(step1[3] + step1[4], bd);
1635 output[4] = HIGHBD_WRAPLOW(step1[3] - step1[4], bd);
1636 output[5] = HIGHBD_WRAPLOW(step1[2] - step1[5], bd);
1637 output[6] = HIGHBD_WRAPLOW(step1[1] - step1[6], bd);
1638 output[7] = HIGHBD_WRAPLOW(step1[0] - step1[7], bd);
1639 }
1640
vpx_highbd_idct8x8_64_add_c(const tran_low_t * input,uint16_t * dest,int stride,int bd)1641 void vpx_highbd_idct8x8_64_add_c(const tran_low_t *input, uint16_t *dest,
1642 int stride, int bd) {
1643 int i, j;
1644 tran_low_t out[8 * 8];
1645 tran_low_t *outptr = out;
1646 tran_low_t temp_in[8], temp_out[8];
1647
1648 // First transform rows
1649 for (i = 0; i < 8; ++i) {
1650 vpx_highbd_idct8_c(input, outptr, bd);
1651 input += 8;
1652 outptr += 8;
1653 }
1654
1655 // Then transform columns
1656 for (i = 0; i < 8; ++i) {
1657 for (j = 0; j < 8; ++j) temp_in[j] = out[j * 8 + i];
1658 vpx_highbd_idct8_c(temp_in, temp_out, bd);
1659 for (j = 0; j < 8; ++j) {
1660 dest[j * stride + i] = highbd_clip_pixel_add(
1661 dest[j * stride + i], ROUND_POWER_OF_TWO(temp_out[j], 5), bd);
1662 }
1663 }
1664 }
1665
vpx_highbd_idct8x8_12_add_c(const tran_low_t * input,uint16_t * dest,int stride,int bd)1666 void vpx_highbd_idct8x8_12_add_c(const tran_low_t *input, uint16_t *dest,
1667 int stride, int bd) {
1668 int i, j;
1669 tran_low_t out[8 * 8] = { 0 };
1670 tran_low_t *outptr = out;
1671 tran_low_t temp_in[8], temp_out[8];
1672
1673 // First transform rows
1674 // Only first 4 row has non-zero coefs
1675 for (i = 0; i < 4; ++i) {
1676 vpx_highbd_idct8_c(input, outptr, bd);
1677 input += 8;
1678 outptr += 8;
1679 }
1680
1681 // Then transform columns
1682 for (i = 0; i < 8; ++i) {
1683 for (j = 0; j < 8; ++j) temp_in[j] = out[j * 8 + i];
1684 vpx_highbd_idct8_c(temp_in, temp_out, bd);
1685 for (j = 0; j < 8; ++j) {
1686 dest[j * stride + i] = highbd_clip_pixel_add(
1687 dest[j * stride + i], ROUND_POWER_OF_TWO(temp_out[j], 5), bd);
1688 }
1689 }
1690 }
1691
vpx_highbd_idct8x8_1_add_c(const tran_low_t * input,uint16_t * dest,int stride,int bd)1692 void vpx_highbd_idct8x8_1_add_c(const tran_low_t *input, uint16_t *dest,
1693 int stride, int bd) {
1694 int i, j;
1695 tran_high_t a1;
1696 tran_low_t out = HIGHBD_WRAPLOW(
1697 dct_const_round_shift(input[0] * (tran_high_t)cospi_16_64), bd);
1698
1699 out =
1700 HIGHBD_WRAPLOW(dct_const_round_shift(out * (tran_high_t)cospi_16_64), bd);
1701 a1 = ROUND_POWER_OF_TWO(out, 5);
1702 for (j = 0; j < 8; ++j) {
1703 for (i = 0; i < 8; ++i) dest[i] = highbd_clip_pixel_add(dest[i], a1, bd);
1704 dest += stride;
1705 }
1706 }
1707
vpx_highbd_iadst16_c(const tran_low_t * input,tran_low_t * output,int bd)1708 void vpx_highbd_iadst16_c(const tran_low_t *input, tran_low_t *output, int bd) {
1709 tran_high_t s0, s1, s2, s3, s4, s5, s6, s7, s8;
1710 tran_high_t s9, s10, s11, s12, s13, s14, s15;
1711 tran_low_t x0 = input[15];
1712 tran_low_t x1 = input[0];
1713 tran_low_t x2 = input[13];
1714 tran_low_t x3 = input[2];
1715 tran_low_t x4 = input[11];
1716 tran_low_t x5 = input[4];
1717 tran_low_t x6 = input[9];
1718 tran_low_t x7 = input[6];
1719 tran_low_t x8 = input[7];
1720 tran_low_t x9 = input[8];
1721 tran_low_t x10 = input[5];
1722 tran_low_t x11 = input[10];
1723 tran_low_t x12 = input[3];
1724 tran_low_t x13 = input[12];
1725 tran_low_t x14 = input[1];
1726 tran_low_t x15 = input[14];
1727 (void)bd;
1728
1729 if (detect_invalid_highbd_input(input, 16)) {
1730 #if CONFIG_COEFFICIENT_RANGE_CHECKING
1731 assert(0 && "invalid highbd txfm input");
1732 #endif // CONFIG_COEFFICIENT_RANGE_CHECKING
1733 memset(output, 0, sizeof(*output) * 16);
1734 return;
1735 }
1736
1737 if (!(x0 | x1 | x2 | x3 | x4 | x5 | x6 | x7 | x8 | x9 | x10 | x11 | x12 |
1738 x13 | x14 | x15)) {
1739 memset(output, 0, 16 * sizeof(*output));
1740 return;
1741 }
1742
1743 // stage 1
1744 s0 = x0 * (tran_high_t)cospi_1_64 + x1 * (tran_high_t)cospi_31_64;
1745 s1 = x0 * (tran_high_t)cospi_31_64 - x1 * (tran_high_t)cospi_1_64;
1746 s2 = x2 * (tran_high_t)cospi_5_64 + x3 * (tran_high_t)cospi_27_64;
1747 s3 = x2 * (tran_high_t)cospi_27_64 - x3 * (tran_high_t)cospi_5_64;
1748 s4 = x4 * (tran_high_t)cospi_9_64 + x5 * (tran_high_t)cospi_23_64;
1749 s5 = x4 * (tran_high_t)cospi_23_64 - x5 * (tran_high_t)cospi_9_64;
1750 s6 = x6 * (tran_high_t)cospi_13_64 + x7 * (tran_high_t)cospi_19_64;
1751 s7 = x6 * (tran_high_t)cospi_19_64 - x7 * (tran_high_t)cospi_13_64;
1752 s8 = x8 * (tran_high_t)cospi_17_64 + x9 * (tran_high_t)cospi_15_64;
1753 s9 = x8 * (tran_high_t)cospi_15_64 - x9 * (tran_high_t)cospi_17_64;
1754 s10 = x10 * (tran_high_t)cospi_21_64 + x11 * (tran_high_t)cospi_11_64;
1755 s11 = x10 * (tran_high_t)cospi_11_64 - x11 * (tran_high_t)cospi_21_64;
1756 s12 = x12 * (tran_high_t)cospi_25_64 + x13 * (tran_high_t)cospi_7_64;
1757 s13 = x12 * (tran_high_t)cospi_7_64 - x13 * (tran_high_t)cospi_25_64;
1758 s14 = x14 * (tran_high_t)cospi_29_64 + x15 * (tran_high_t)cospi_3_64;
1759 s15 = x14 * (tran_high_t)cospi_3_64 - x15 * (tran_high_t)cospi_29_64;
1760
1761 x0 = HIGHBD_WRAPLOW(dct_const_round_shift(s0 + s8), bd);
1762 x1 = HIGHBD_WRAPLOW(dct_const_round_shift(s1 + s9), bd);
1763 x2 = HIGHBD_WRAPLOW(dct_const_round_shift(s2 + s10), bd);
1764 x3 = HIGHBD_WRAPLOW(dct_const_round_shift(s3 + s11), bd);
1765 x4 = HIGHBD_WRAPLOW(dct_const_round_shift(s4 + s12), bd);
1766 x5 = HIGHBD_WRAPLOW(dct_const_round_shift(s5 + s13), bd);
1767 x6 = HIGHBD_WRAPLOW(dct_const_round_shift(s6 + s14), bd);
1768 x7 = HIGHBD_WRAPLOW(dct_const_round_shift(s7 + s15), bd);
1769 x8 = HIGHBD_WRAPLOW(dct_const_round_shift(s0 - s8), bd);
1770 x9 = HIGHBD_WRAPLOW(dct_const_round_shift(s1 - s9), bd);
1771 x10 = HIGHBD_WRAPLOW(dct_const_round_shift(s2 - s10), bd);
1772 x11 = HIGHBD_WRAPLOW(dct_const_round_shift(s3 - s11), bd);
1773 x12 = HIGHBD_WRAPLOW(dct_const_round_shift(s4 - s12), bd);
1774 x13 = HIGHBD_WRAPLOW(dct_const_round_shift(s5 - s13), bd);
1775 x14 = HIGHBD_WRAPLOW(dct_const_round_shift(s6 - s14), bd);
1776 x15 = HIGHBD_WRAPLOW(dct_const_round_shift(s7 - s15), bd);
1777
1778 // stage 2
1779 s0 = x0;
1780 s1 = x1;
1781 s2 = x2;
1782 s3 = x3;
1783 s4 = x4;
1784 s5 = x5;
1785 s6 = x6;
1786 s7 = x7;
1787 s8 = x8 * (tran_high_t)cospi_4_64 + x9 * (tran_high_t)cospi_28_64;
1788 s9 = x8 * (tran_high_t)cospi_28_64 - x9 * (tran_high_t)cospi_4_64;
1789 s10 = x10 * (tran_high_t)cospi_20_64 + x11 * (tran_high_t)cospi_12_64;
1790 s11 = x10 * (tran_high_t)cospi_12_64 - x11 * (tran_high_t)cospi_20_64;
1791 s12 = -x12 * (tran_high_t)cospi_28_64 + x13 * (tran_high_t)cospi_4_64;
1792 s13 = x12 * (tran_high_t)cospi_4_64 + x13 * (tran_high_t)cospi_28_64;
1793 s14 = -x14 * (tran_high_t)cospi_12_64 + x15 * (tran_high_t)cospi_20_64;
1794 s15 = x14 * (tran_high_t)cospi_20_64 + x15 * (tran_high_t)cospi_12_64;
1795
1796 x0 = HIGHBD_WRAPLOW(s0 + s4, bd);
1797 x1 = HIGHBD_WRAPLOW(s1 + s5, bd);
1798 x2 = HIGHBD_WRAPLOW(s2 + s6, bd);
1799 x3 = HIGHBD_WRAPLOW(s3 + s7, bd);
1800 x4 = HIGHBD_WRAPLOW(s0 - s4, bd);
1801 x5 = HIGHBD_WRAPLOW(s1 - s5, bd);
1802 x6 = HIGHBD_WRAPLOW(s2 - s6, bd);
1803 x7 = HIGHBD_WRAPLOW(s3 - s7, bd);
1804 x8 = HIGHBD_WRAPLOW(dct_const_round_shift(s8 + s12), bd);
1805 x9 = HIGHBD_WRAPLOW(dct_const_round_shift(s9 + s13), bd);
1806 x10 = HIGHBD_WRAPLOW(dct_const_round_shift(s10 + s14), bd);
1807 x11 = HIGHBD_WRAPLOW(dct_const_round_shift(s11 + s15), bd);
1808 x12 = HIGHBD_WRAPLOW(dct_const_round_shift(s8 - s12), bd);
1809 x13 = HIGHBD_WRAPLOW(dct_const_round_shift(s9 - s13), bd);
1810 x14 = HIGHBD_WRAPLOW(dct_const_round_shift(s10 - s14), bd);
1811 x15 = HIGHBD_WRAPLOW(dct_const_round_shift(s11 - s15), bd);
1812
1813 // stage 3
1814 s0 = x0;
1815 s1 = x1;
1816 s2 = x2;
1817 s3 = x3;
1818 s4 = x4 * (tran_high_t)cospi_8_64 + x5 * (tran_high_t)cospi_24_64;
1819 s5 = x4 * (tran_high_t)cospi_24_64 - x5 * (tran_high_t)cospi_8_64;
1820 s6 = -x6 * (tran_high_t)cospi_24_64 + x7 * (tran_high_t)cospi_8_64;
1821 s7 = x6 * (tran_high_t)cospi_8_64 + x7 * (tran_high_t)cospi_24_64;
1822 s8 = x8;
1823 s9 = x9;
1824 s10 = x10;
1825 s11 = x11;
1826 s12 = x12 * (tran_high_t)cospi_8_64 + x13 * (tran_high_t)cospi_24_64;
1827 s13 = x12 * (tran_high_t)cospi_24_64 - x13 * (tran_high_t)cospi_8_64;
1828 s14 = -x14 * (tran_high_t)cospi_24_64 + x15 * (tran_high_t)cospi_8_64;
1829 s15 = x14 * (tran_high_t)cospi_8_64 + x15 * (tran_high_t)cospi_24_64;
1830
1831 x0 = HIGHBD_WRAPLOW(s0 + s2, bd);
1832 x1 = HIGHBD_WRAPLOW(s1 + s3, bd);
1833 x2 = HIGHBD_WRAPLOW(s0 - s2, bd);
1834 x3 = HIGHBD_WRAPLOW(s1 - s3, bd);
1835 x4 = HIGHBD_WRAPLOW(dct_const_round_shift(s4 + s6), bd);
1836 x5 = HIGHBD_WRAPLOW(dct_const_round_shift(s5 + s7), bd);
1837 x6 = HIGHBD_WRAPLOW(dct_const_round_shift(s4 - s6), bd);
1838 x7 = HIGHBD_WRAPLOW(dct_const_round_shift(s5 - s7), bd);
1839 x8 = HIGHBD_WRAPLOW(s8 + s10, bd);
1840 x9 = HIGHBD_WRAPLOW(s9 + s11, bd);
1841 x10 = HIGHBD_WRAPLOW(s8 - s10, bd);
1842 x11 = HIGHBD_WRAPLOW(s9 - s11, bd);
1843 x12 = HIGHBD_WRAPLOW(dct_const_round_shift(s12 + s14), bd);
1844 x13 = HIGHBD_WRAPLOW(dct_const_round_shift(s13 + s15), bd);
1845 x14 = HIGHBD_WRAPLOW(dct_const_round_shift(s12 - s14), bd);
1846 x15 = HIGHBD_WRAPLOW(dct_const_round_shift(s13 - s15), bd);
1847
1848 // stage 4
1849 s2 = (tran_high_t)(-cospi_16_64) * (x2 + x3);
1850 s3 = (tran_high_t)cospi_16_64 * (x2 - x3);
1851 s6 = (tran_high_t)cospi_16_64 * (x6 + x7);
1852 s7 = (tran_high_t)cospi_16_64 * (-x6 + x7);
1853 s10 = (tran_high_t)cospi_16_64 * (x10 + x11);
1854 s11 = (tran_high_t)cospi_16_64 * (-x10 + x11);
1855 s14 = (tran_high_t)(-cospi_16_64) * (x14 + x15);
1856 s15 = (tran_high_t)cospi_16_64 * (x14 - x15);
1857
1858 x2 = HIGHBD_WRAPLOW(dct_const_round_shift(s2), bd);
1859 x3 = HIGHBD_WRAPLOW(dct_const_round_shift(s3), bd);
1860 x6 = HIGHBD_WRAPLOW(dct_const_round_shift(s6), bd);
1861 x7 = HIGHBD_WRAPLOW(dct_const_round_shift(s7), bd);
1862 x10 = HIGHBD_WRAPLOW(dct_const_round_shift(s10), bd);
1863 x11 = HIGHBD_WRAPLOW(dct_const_round_shift(s11), bd);
1864 x14 = HIGHBD_WRAPLOW(dct_const_round_shift(s14), bd);
1865 x15 = HIGHBD_WRAPLOW(dct_const_round_shift(s15), bd);
1866
1867 output[0] = HIGHBD_WRAPLOW(x0, bd);
1868 output[1] = HIGHBD_WRAPLOW(-x8, bd);
1869 output[2] = HIGHBD_WRAPLOW(x12, bd);
1870 output[3] = HIGHBD_WRAPLOW(-x4, bd);
1871 output[4] = HIGHBD_WRAPLOW(x6, bd);
1872 output[5] = HIGHBD_WRAPLOW(x14, bd);
1873 output[6] = HIGHBD_WRAPLOW(x10, bd);
1874 output[7] = HIGHBD_WRAPLOW(x2, bd);
1875 output[8] = HIGHBD_WRAPLOW(x3, bd);
1876 output[9] = HIGHBD_WRAPLOW(x11, bd);
1877 output[10] = HIGHBD_WRAPLOW(x15, bd);
1878 output[11] = HIGHBD_WRAPLOW(x7, bd);
1879 output[12] = HIGHBD_WRAPLOW(x5, bd);
1880 output[13] = HIGHBD_WRAPLOW(-x13, bd);
1881 output[14] = HIGHBD_WRAPLOW(x9, bd);
1882 output[15] = HIGHBD_WRAPLOW(-x1, bd);
1883 }
1884
vpx_highbd_idct16_c(const tran_low_t * input,tran_low_t * output,int bd)1885 void vpx_highbd_idct16_c(const tran_low_t *input, tran_low_t *output, int bd) {
1886 tran_low_t step1[16], step2[16];
1887 tran_high_t temp1, temp2;
1888 (void)bd;
1889
1890 if (detect_invalid_highbd_input(input, 16)) {
1891 #if CONFIG_COEFFICIENT_RANGE_CHECKING
1892 assert(0 && "invalid highbd txfm input");
1893 #endif // CONFIG_COEFFICIENT_RANGE_CHECKING
1894 memset(output, 0, sizeof(*output) * 16);
1895 return;
1896 }
1897
1898 // stage 1
1899 step1[0] = input[0 / 2];
1900 step1[1] = input[16 / 2];
1901 step1[2] = input[8 / 2];
1902 step1[3] = input[24 / 2];
1903 step1[4] = input[4 / 2];
1904 step1[5] = input[20 / 2];
1905 step1[6] = input[12 / 2];
1906 step1[7] = input[28 / 2];
1907 step1[8] = input[2 / 2];
1908 step1[9] = input[18 / 2];
1909 step1[10] = input[10 / 2];
1910 step1[11] = input[26 / 2];
1911 step1[12] = input[6 / 2];
1912 step1[13] = input[22 / 2];
1913 step1[14] = input[14 / 2];
1914 step1[15] = input[30 / 2];
1915
1916 // stage 2
1917 step2[0] = step1[0];
1918 step2[1] = step1[1];
1919 step2[2] = step1[2];
1920 step2[3] = step1[3];
1921 step2[4] = step1[4];
1922 step2[5] = step1[5];
1923 step2[6] = step1[6];
1924 step2[7] = step1[7];
1925
1926 temp1 =
1927 step1[8] * (tran_high_t)cospi_30_64 - step1[15] * (tran_high_t)cospi_2_64;
1928 temp2 =
1929 step1[8] * (tran_high_t)cospi_2_64 + step1[15] * (tran_high_t)cospi_30_64;
1930 step2[8] = HIGHBD_WRAPLOW(dct_const_round_shift(temp1), bd);
1931 step2[15] = HIGHBD_WRAPLOW(dct_const_round_shift(temp2), bd);
1932
1933 temp1 = step1[9] * (tran_high_t)cospi_14_64 -
1934 step1[14] * (tran_high_t)cospi_18_64;
1935 temp2 = step1[9] * (tran_high_t)cospi_18_64 +
1936 step1[14] * (tran_high_t)cospi_14_64;
1937 step2[9] = HIGHBD_WRAPLOW(dct_const_round_shift(temp1), bd);
1938 step2[14] = HIGHBD_WRAPLOW(dct_const_round_shift(temp2), bd);
1939
1940 temp1 = step1[10] * (tran_high_t)cospi_22_64 -
1941 step1[13] * (tran_high_t)cospi_10_64;
1942 temp2 = step1[10] * (tran_high_t)cospi_10_64 +
1943 step1[13] * (tran_high_t)cospi_22_64;
1944 step2[10] = HIGHBD_WRAPLOW(dct_const_round_shift(temp1), bd);
1945 step2[13] = HIGHBD_WRAPLOW(dct_const_round_shift(temp2), bd);
1946
1947 temp1 = step1[11] * (tran_high_t)cospi_6_64 -
1948 step1[12] * (tran_high_t)cospi_26_64;
1949 temp2 = step1[11] * (tran_high_t)cospi_26_64 +
1950 step1[12] * (tran_high_t)cospi_6_64;
1951 step2[11] = HIGHBD_WRAPLOW(dct_const_round_shift(temp1), bd);
1952 step2[12] = HIGHBD_WRAPLOW(dct_const_round_shift(temp2), bd);
1953
1954 // stage 3
1955 step1[0] = step2[0];
1956 step1[1] = step2[1];
1957 step1[2] = step2[2];
1958 step1[3] = step2[3];
1959
1960 temp1 =
1961 step2[4] * (tran_high_t)cospi_28_64 - step2[7] * (tran_high_t)cospi_4_64;
1962 temp2 =
1963 step2[4] * (tran_high_t)cospi_4_64 + step2[7] * (tran_high_t)cospi_28_64;
1964 step1[4] = HIGHBD_WRAPLOW(dct_const_round_shift(temp1), bd);
1965 step1[7] = HIGHBD_WRAPLOW(dct_const_round_shift(temp2), bd);
1966 temp1 =
1967 step2[5] * (tran_high_t)cospi_12_64 - step2[6] * (tran_high_t)cospi_20_64;
1968 temp2 =
1969 step2[5] * (tran_high_t)cospi_20_64 + step2[6] * (tran_high_t)cospi_12_64;
1970 step1[5] = HIGHBD_WRAPLOW(dct_const_round_shift(temp1), bd);
1971 step1[6] = HIGHBD_WRAPLOW(dct_const_round_shift(temp2), bd);
1972
1973 step1[8] = HIGHBD_WRAPLOW(step2[8] + step2[9], bd);
1974 step1[9] = HIGHBD_WRAPLOW(step2[8] - step2[9], bd);
1975 step1[10] = HIGHBD_WRAPLOW(-step2[10] + step2[11], bd);
1976 step1[11] = HIGHBD_WRAPLOW(step2[10] + step2[11], bd);
1977 step1[12] = HIGHBD_WRAPLOW(step2[12] + step2[13], bd);
1978 step1[13] = HIGHBD_WRAPLOW(step2[12] - step2[13], bd);
1979 step1[14] = HIGHBD_WRAPLOW(-step2[14] + step2[15], bd);
1980 step1[15] = HIGHBD_WRAPLOW(step2[14] + step2[15], bd);
1981
1982 // stage 4
1983 temp1 = (step1[0] + step1[1]) * (tran_high_t)cospi_16_64;
1984 temp2 = (step1[0] - step1[1]) * (tran_high_t)cospi_16_64;
1985 step2[0] = HIGHBD_WRAPLOW(dct_const_round_shift(temp1), bd);
1986 step2[1] = HIGHBD_WRAPLOW(dct_const_round_shift(temp2), bd);
1987 temp1 =
1988 step1[2] * (tran_high_t)cospi_24_64 - step1[3] * (tran_high_t)cospi_8_64;
1989 temp2 =
1990 step1[2] * (tran_high_t)cospi_8_64 + step1[3] * (tran_high_t)cospi_24_64;
1991 step2[2] = HIGHBD_WRAPLOW(dct_const_round_shift(temp1), bd);
1992 step2[3] = HIGHBD_WRAPLOW(dct_const_round_shift(temp2), bd);
1993 step2[4] = HIGHBD_WRAPLOW(step1[4] + step1[5], bd);
1994 step2[5] = HIGHBD_WRAPLOW(step1[4] - step1[5], bd);
1995 step2[6] = HIGHBD_WRAPLOW(-step1[6] + step1[7], bd);
1996 step2[7] = HIGHBD_WRAPLOW(step1[6] + step1[7], bd);
1997
1998 step2[8] = step1[8];
1999 step2[15] = step1[15];
2000 temp1 = -step1[9] * (tran_high_t)cospi_8_64 +
2001 step1[14] * (tran_high_t)cospi_24_64;
2002 temp2 =
2003 step1[9] * (tran_high_t)cospi_24_64 + step1[14] * (tran_high_t)cospi_8_64;
2004 step2[9] = HIGHBD_WRAPLOW(dct_const_round_shift(temp1), bd);
2005 step2[14] = HIGHBD_WRAPLOW(dct_const_round_shift(temp2), bd);
2006 temp1 = -step1[10] * (tran_high_t)cospi_24_64 -
2007 step1[13] * (tran_high_t)cospi_8_64;
2008 temp2 = -step1[10] * (tran_high_t)cospi_8_64 +
2009 step1[13] * (tran_high_t)cospi_24_64;
2010 step2[10] = HIGHBD_WRAPLOW(dct_const_round_shift(temp1), bd);
2011 step2[13] = HIGHBD_WRAPLOW(dct_const_round_shift(temp2), bd);
2012 step2[11] = step1[11];
2013 step2[12] = step1[12];
2014
2015 // stage 5
2016 step1[0] = HIGHBD_WRAPLOW(step2[0] + step2[3], bd);
2017 step1[1] = HIGHBD_WRAPLOW(step2[1] + step2[2], bd);
2018 step1[2] = HIGHBD_WRAPLOW(step2[1] - step2[2], bd);
2019 step1[3] = HIGHBD_WRAPLOW(step2[0] - step2[3], bd);
2020 step1[4] = step2[4];
2021 temp1 = (step2[6] - step2[5]) * (tran_high_t)cospi_16_64;
2022 temp2 = (step2[5] + step2[6]) * (tran_high_t)cospi_16_64;
2023 step1[5] = HIGHBD_WRAPLOW(dct_const_round_shift(temp1), bd);
2024 step1[6] = HIGHBD_WRAPLOW(dct_const_round_shift(temp2), bd);
2025 step1[7] = step2[7];
2026
2027 step1[8] = HIGHBD_WRAPLOW(step2[8] + step2[11], bd);
2028 step1[9] = HIGHBD_WRAPLOW(step2[9] + step2[10], bd);
2029 step1[10] = HIGHBD_WRAPLOW(step2[9] - step2[10], bd);
2030 step1[11] = HIGHBD_WRAPLOW(step2[8] - step2[11], bd);
2031 step1[12] = HIGHBD_WRAPLOW(-step2[12] + step2[15], bd);
2032 step1[13] = HIGHBD_WRAPLOW(-step2[13] + step2[14], bd);
2033 step1[14] = HIGHBD_WRAPLOW(step2[13] + step2[14], bd);
2034 step1[15] = HIGHBD_WRAPLOW(step2[12] + step2[15], bd);
2035
2036 // stage 6
2037 step2[0] = HIGHBD_WRAPLOW(step1[0] + step1[7], bd);
2038 step2[1] = HIGHBD_WRAPLOW(step1[1] + step1[6], bd);
2039 step2[2] = HIGHBD_WRAPLOW(step1[2] + step1[5], bd);
2040 step2[3] = HIGHBD_WRAPLOW(step1[3] + step1[4], bd);
2041 step2[4] = HIGHBD_WRAPLOW(step1[3] - step1[4], bd);
2042 step2[5] = HIGHBD_WRAPLOW(step1[2] - step1[5], bd);
2043 step2[6] = HIGHBD_WRAPLOW(step1[1] - step1[6], bd);
2044 step2[7] = HIGHBD_WRAPLOW(step1[0] - step1[7], bd);
2045 step2[8] = step1[8];
2046 step2[9] = step1[9];
2047 temp1 = (-step1[10] + step1[13]) * (tran_high_t)cospi_16_64;
2048 temp2 = (step1[10] + step1[13]) * (tran_high_t)cospi_16_64;
2049 step2[10] = HIGHBD_WRAPLOW(dct_const_round_shift(temp1), bd);
2050 step2[13] = HIGHBD_WRAPLOW(dct_const_round_shift(temp2), bd);
2051 temp1 = (-step1[11] + step1[12]) * (tran_high_t)cospi_16_64;
2052 temp2 = (step1[11] + step1[12]) * (tran_high_t)cospi_16_64;
2053 step2[11] = HIGHBD_WRAPLOW(dct_const_round_shift(temp1), bd);
2054 step2[12] = HIGHBD_WRAPLOW(dct_const_round_shift(temp2), bd);
2055 step2[14] = step1[14];
2056 step2[15] = step1[15];
2057
2058 // stage 7
2059 output[0] = HIGHBD_WRAPLOW(step2[0] + step2[15], bd);
2060 output[1] = HIGHBD_WRAPLOW(step2[1] + step2[14], bd);
2061 output[2] = HIGHBD_WRAPLOW(step2[2] + step2[13], bd);
2062 output[3] = HIGHBD_WRAPLOW(step2[3] + step2[12], bd);
2063 output[4] = HIGHBD_WRAPLOW(step2[4] + step2[11], bd);
2064 output[5] = HIGHBD_WRAPLOW(step2[5] + step2[10], bd);
2065 output[6] = HIGHBD_WRAPLOW(step2[6] + step2[9], bd);
2066 output[7] = HIGHBD_WRAPLOW(step2[7] + step2[8], bd);
2067 output[8] = HIGHBD_WRAPLOW(step2[7] - step2[8], bd);
2068 output[9] = HIGHBD_WRAPLOW(step2[6] - step2[9], bd);
2069 output[10] = HIGHBD_WRAPLOW(step2[5] - step2[10], bd);
2070 output[11] = HIGHBD_WRAPLOW(step2[4] - step2[11], bd);
2071 output[12] = HIGHBD_WRAPLOW(step2[3] - step2[12], bd);
2072 output[13] = HIGHBD_WRAPLOW(step2[2] - step2[13], bd);
2073 output[14] = HIGHBD_WRAPLOW(step2[1] - step2[14], bd);
2074 output[15] = HIGHBD_WRAPLOW(step2[0] - step2[15], bd);
2075 }
2076
vpx_highbd_idct16x16_256_add_c(const tran_low_t * input,uint16_t * dest,int stride,int bd)2077 void vpx_highbd_idct16x16_256_add_c(const tran_low_t *input, uint16_t *dest,
2078 int stride, int bd) {
2079 int i, j;
2080 tran_low_t out[16 * 16];
2081 tran_low_t *outptr = out;
2082 tran_low_t temp_in[16], temp_out[16];
2083
2084 // First transform rows
2085 for (i = 0; i < 16; ++i) {
2086 vpx_highbd_idct16_c(input, outptr, bd);
2087 input += 16;
2088 outptr += 16;
2089 }
2090
2091 // Then transform columns
2092 for (i = 0; i < 16; ++i) {
2093 for (j = 0; j < 16; ++j) temp_in[j] = out[j * 16 + i];
2094 vpx_highbd_idct16_c(temp_in, temp_out, bd);
2095 for (j = 0; j < 16; ++j) {
2096 dest[j * stride + i] = highbd_clip_pixel_add(
2097 dest[j * stride + i], ROUND_POWER_OF_TWO(temp_out[j], 6), bd);
2098 }
2099 }
2100 }
2101
vpx_highbd_idct16x16_38_add_c(const tran_low_t * input,uint16_t * dest,int stride,int bd)2102 void vpx_highbd_idct16x16_38_add_c(const tran_low_t *input, uint16_t *dest,
2103 int stride, int bd) {
2104 int i, j;
2105 tran_low_t out[16 * 16] = { 0 };
2106 tran_low_t *outptr = out;
2107 tran_low_t temp_in[16], temp_out[16];
2108
2109 // First transform rows. Since all non-zero dct coefficients are in
2110 // upper-left 8x8 area, we only need to calculate first 8 rows here.
2111 for (i = 0; i < 8; ++i) {
2112 vpx_highbd_idct16_c(input, outptr, bd);
2113 input += 16;
2114 outptr += 16;
2115 }
2116
2117 // Then transform columns
2118 for (i = 0; i < 16; ++i) {
2119 uint16_t *destT = dest;
2120 for (j = 0; j < 16; ++j) temp_in[j] = out[j * 16 + i];
2121 vpx_highbd_idct16_c(temp_in, temp_out, bd);
2122 for (j = 0; j < 16; ++j) {
2123 destT[i] = highbd_clip_pixel_add(destT[i],
2124 ROUND_POWER_OF_TWO(temp_out[j], 6), bd);
2125 destT += stride;
2126 }
2127 }
2128 }
2129
vpx_highbd_idct16x16_10_add_c(const tran_low_t * input,uint16_t * dest,int stride,int bd)2130 void vpx_highbd_idct16x16_10_add_c(const tran_low_t *input, uint16_t *dest,
2131 int stride, int bd) {
2132 int i, j;
2133 tran_low_t out[16 * 16] = { 0 };
2134 tran_low_t *outptr = out;
2135 tran_low_t temp_in[16], temp_out[16];
2136
2137 // First transform rows. Since all non-zero dct coefficients are in
2138 // upper-left 4x4 area, we only need to calculate first 4 rows here.
2139 for (i = 0; i < 4; ++i) {
2140 vpx_highbd_idct16_c(input, outptr, bd);
2141 input += 16;
2142 outptr += 16;
2143 }
2144
2145 // Then transform columns
2146 for (i = 0; i < 16; ++i) {
2147 for (j = 0; j < 16; ++j) temp_in[j] = out[j * 16 + i];
2148 vpx_highbd_idct16_c(temp_in, temp_out, bd);
2149 for (j = 0; j < 16; ++j) {
2150 dest[j * stride + i] = highbd_clip_pixel_add(
2151 dest[j * stride + i], ROUND_POWER_OF_TWO(temp_out[j], 6), bd);
2152 }
2153 }
2154 }
2155
vpx_highbd_idct16x16_1_add_c(const tran_low_t * input,uint16_t * dest,int stride,int bd)2156 void vpx_highbd_idct16x16_1_add_c(const tran_low_t *input, uint16_t *dest,
2157 int stride, int bd) {
2158 int i, j;
2159 tran_high_t a1;
2160 tran_low_t out = HIGHBD_WRAPLOW(
2161 dct_const_round_shift(input[0] * (tran_high_t)cospi_16_64), bd);
2162
2163 out =
2164 HIGHBD_WRAPLOW(dct_const_round_shift(out * (tran_high_t)cospi_16_64), bd);
2165 a1 = ROUND_POWER_OF_TWO(out, 6);
2166 for (j = 0; j < 16; ++j) {
2167 for (i = 0; i < 16; ++i) dest[i] = highbd_clip_pixel_add(dest[i], a1, bd);
2168 dest += stride;
2169 }
2170 }
2171
highbd_idct32_c(const tran_low_t * input,tran_low_t * output,int bd)2172 static void highbd_idct32_c(const tran_low_t *input, tran_low_t *output,
2173 int bd) {
2174 tran_low_t step1[32], step2[32];
2175 tran_high_t temp1, temp2;
2176 (void)bd;
2177
2178 if (detect_invalid_highbd_input(input, 32)) {
2179 #if CONFIG_COEFFICIENT_RANGE_CHECKING
2180 assert(0 && "invalid highbd txfm input");
2181 #endif // CONFIG_COEFFICIENT_RANGE_CHECKING
2182 memset(output, 0, sizeof(*output) * 32);
2183 return;
2184 }
2185
2186 // stage 1
2187 step1[0] = input[0];
2188 step1[1] = input[16];
2189 step1[2] = input[8];
2190 step1[3] = input[24];
2191 step1[4] = input[4];
2192 step1[5] = input[20];
2193 step1[6] = input[12];
2194 step1[7] = input[28];
2195 step1[8] = input[2];
2196 step1[9] = input[18];
2197 step1[10] = input[10];
2198 step1[11] = input[26];
2199 step1[12] = input[6];
2200 step1[13] = input[22];
2201 step1[14] = input[14];
2202 step1[15] = input[30];
2203
2204 temp1 =
2205 input[1] * (tran_high_t)cospi_31_64 - input[31] * (tran_high_t)cospi_1_64;
2206 temp2 =
2207 input[1] * (tran_high_t)cospi_1_64 + input[31] * (tran_high_t)cospi_31_64;
2208 step1[16] = HIGHBD_WRAPLOW(dct_const_round_shift(temp1), bd);
2209 step1[31] = HIGHBD_WRAPLOW(dct_const_round_shift(temp2), bd);
2210
2211 temp1 = input[17] * (tran_high_t)cospi_15_64 -
2212 input[15] * (tran_high_t)cospi_17_64;
2213 temp2 = input[17] * (tran_high_t)cospi_17_64 +
2214 input[15] * (tran_high_t)cospi_15_64;
2215 step1[17] = HIGHBD_WRAPLOW(dct_const_round_shift(temp1), bd);
2216 step1[30] = HIGHBD_WRAPLOW(dct_const_round_shift(temp2), bd);
2217
2218 temp1 =
2219 input[9] * (tran_high_t)cospi_23_64 - input[23] * (tran_high_t)cospi_9_64;
2220 temp2 =
2221 input[9] * (tran_high_t)cospi_9_64 + input[23] * (tran_high_t)cospi_23_64;
2222 step1[18] = HIGHBD_WRAPLOW(dct_const_round_shift(temp1), bd);
2223 step1[29] = HIGHBD_WRAPLOW(dct_const_round_shift(temp2), bd);
2224
2225 temp1 =
2226 input[25] * (tran_high_t)cospi_7_64 - input[7] * (tran_high_t)cospi_25_64;
2227 temp2 =
2228 input[25] * (tran_high_t)cospi_25_64 + input[7] * (tran_high_t)cospi_7_64;
2229 step1[19] = HIGHBD_WRAPLOW(dct_const_round_shift(temp1), bd);
2230 step1[28] = HIGHBD_WRAPLOW(dct_const_round_shift(temp2), bd);
2231
2232 temp1 =
2233 input[5] * (tran_high_t)cospi_27_64 - input[27] * (tran_high_t)cospi_5_64;
2234 temp2 =
2235 input[5] * (tran_high_t)cospi_5_64 + input[27] * (tran_high_t)cospi_27_64;
2236 step1[20] = HIGHBD_WRAPLOW(dct_const_round_shift(temp1), bd);
2237 step1[27] = HIGHBD_WRAPLOW(dct_const_round_shift(temp2), bd);
2238
2239 temp1 = input[21] * (tran_high_t)cospi_11_64 -
2240 input[11] * (tran_high_t)cospi_21_64;
2241 temp2 = input[21] * (tran_high_t)cospi_21_64 +
2242 input[11] * (tran_high_t)cospi_11_64;
2243 step1[21] = HIGHBD_WRAPLOW(dct_const_round_shift(temp1), bd);
2244 step1[26] = HIGHBD_WRAPLOW(dct_const_round_shift(temp2), bd);
2245
2246 temp1 = input[13] * (tran_high_t)cospi_19_64 -
2247 input[19] * (tran_high_t)cospi_13_64;
2248 temp2 = input[13] * (tran_high_t)cospi_13_64 +
2249 input[19] * (tran_high_t)cospi_19_64;
2250 step1[22] = HIGHBD_WRAPLOW(dct_const_round_shift(temp1), bd);
2251 step1[25] = HIGHBD_WRAPLOW(dct_const_round_shift(temp2), bd);
2252
2253 temp1 =
2254 input[29] * (tran_high_t)cospi_3_64 - input[3] * (tran_high_t)cospi_29_64;
2255 temp2 =
2256 input[29] * (tran_high_t)cospi_29_64 + input[3] * (tran_high_t)cospi_3_64;
2257 step1[23] = HIGHBD_WRAPLOW(dct_const_round_shift(temp1), bd);
2258 step1[24] = HIGHBD_WRAPLOW(dct_const_round_shift(temp2), bd);
2259
2260 // stage 2
2261 step2[0] = step1[0];
2262 step2[1] = step1[1];
2263 step2[2] = step1[2];
2264 step2[3] = step1[3];
2265 step2[4] = step1[4];
2266 step2[5] = step1[5];
2267 step2[6] = step1[6];
2268 step2[7] = step1[7];
2269
2270 temp1 =
2271 step1[8] * (tran_high_t)cospi_30_64 - step1[15] * (tran_high_t)cospi_2_64;
2272 temp2 =
2273 step1[8] * (tran_high_t)cospi_2_64 + step1[15] * (tran_high_t)cospi_30_64;
2274 step2[8] = HIGHBD_WRAPLOW(dct_const_round_shift(temp1), bd);
2275 step2[15] = HIGHBD_WRAPLOW(dct_const_round_shift(temp2), bd);
2276
2277 temp1 = step1[9] * (tran_high_t)cospi_14_64 -
2278 step1[14] * (tran_high_t)cospi_18_64;
2279 temp2 = step1[9] * (tran_high_t)cospi_18_64 +
2280 step1[14] * (tran_high_t)cospi_14_64;
2281 step2[9] = HIGHBD_WRAPLOW(dct_const_round_shift(temp1), bd);
2282 step2[14] = HIGHBD_WRAPLOW(dct_const_round_shift(temp2), bd);
2283
2284 temp1 = step1[10] * (tran_high_t)cospi_22_64 -
2285 step1[13] * (tran_high_t)cospi_10_64;
2286 temp2 = step1[10] * (tran_high_t)cospi_10_64 +
2287 step1[13] * (tran_high_t)cospi_22_64;
2288 step2[10] = HIGHBD_WRAPLOW(dct_const_round_shift(temp1), bd);
2289 step2[13] = HIGHBD_WRAPLOW(dct_const_round_shift(temp2), bd);
2290
2291 temp1 = step1[11] * (tran_high_t)cospi_6_64 -
2292 step1[12] * (tran_high_t)cospi_26_64;
2293 temp2 = step1[11] * (tran_high_t)cospi_26_64 +
2294 step1[12] * (tran_high_t)cospi_6_64;
2295 step2[11] = HIGHBD_WRAPLOW(dct_const_round_shift(temp1), bd);
2296 step2[12] = HIGHBD_WRAPLOW(dct_const_round_shift(temp2), bd);
2297
2298 step2[16] = HIGHBD_WRAPLOW(step1[16] + step1[17], bd);
2299 step2[17] = HIGHBD_WRAPLOW(step1[16] - step1[17], bd);
2300 step2[18] = HIGHBD_WRAPLOW(-step1[18] + step1[19], bd);
2301 step2[19] = HIGHBD_WRAPLOW(step1[18] + step1[19], bd);
2302 step2[20] = HIGHBD_WRAPLOW(step1[20] + step1[21], bd);
2303 step2[21] = HIGHBD_WRAPLOW(step1[20] - step1[21], bd);
2304 step2[22] = HIGHBD_WRAPLOW(-step1[22] + step1[23], bd);
2305 step2[23] = HIGHBD_WRAPLOW(step1[22] + step1[23], bd);
2306 step2[24] = HIGHBD_WRAPLOW(step1[24] + step1[25], bd);
2307 step2[25] = HIGHBD_WRAPLOW(step1[24] - step1[25], bd);
2308 step2[26] = HIGHBD_WRAPLOW(-step1[26] + step1[27], bd);
2309 step2[27] = HIGHBD_WRAPLOW(step1[26] + step1[27], bd);
2310 step2[28] = HIGHBD_WRAPLOW(step1[28] + step1[29], bd);
2311 step2[29] = HIGHBD_WRAPLOW(step1[28] - step1[29], bd);
2312 step2[30] = HIGHBD_WRAPLOW(-step1[30] + step1[31], bd);
2313 step2[31] = HIGHBD_WRAPLOW(step1[30] + step1[31], bd);
2314
2315 // stage 3
2316 step1[0] = step2[0];
2317 step1[1] = step2[1];
2318 step1[2] = step2[2];
2319 step1[3] = step2[3];
2320
2321 temp1 =
2322 step2[4] * (tran_high_t)cospi_28_64 - step2[7] * (tran_high_t)cospi_4_64;
2323 temp2 =
2324 step2[4] * (tran_high_t)cospi_4_64 + step2[7] * (tran_high_t)cospi_28_64;
2325 step1[4] = HIGHBD_WRAPLOW(dct_const_round_shift(temp1), bd);
2326 step1[7] = HIGHBD_WRAPLOW(dct_const_round_shift(temp2), bd);
2327 temp1 =
2328 step2[5] * (tran_high_t)cospi_12_64 - step2[6] * (tran_high_t)cospi_20_64;
2329 temp2 =
2330 step2[5] * (tran_high_t)cospi_20_64 + step2[6] * (tran_high_t)cospi_12_64;
2331 step1[5] = HIGHBD_WRAPLOW(dct_const_round_shift(temp1), bd);
2332 step1[6] = HIGHBD_WRAPLOW(dct_const_round_shift(temp2), bd);
2333
2334 step1[8] = HIGHBD_WRAPLOW(step2[8] + step2[9], bd);
2335 step1[9] = HIGHBD_WRAPLOW(step2[8] - step2[9], bd);
2336 step1[10] = HIGHBD_WRAPLOW(-step2[10] + step2[11], bd);
2337 step1[11] = HIGHBD_WRAPLOW(step2[10] + step2[11], bd);
2338 step1[12] = HIGHBD_WRAPLOW(step2[12] + step2[13], bd);
2339 step1[13] = HIGHBD_WRAPLOW(step2[12] - step2[13], bd);
2340 step1[14] = HIGHBD_WRAPLOW(-step2[14] + step2[15], bd);
2341 step1[15] = HIGHBD_WRAPLOW(step2[14] + step2[15], bd);
2342
2343 step1[16] = step2[16];
2344 step1[31] = step2[31];
2345 temp1 = -step2[17] * (tran_high_t)cospi_4_64 +
2346 step2[30] * (tran_high_t)cospi_28_64;
2347 temp2 = step2[17] * (tran_high_t)cospi_28_64 +
2348 step2[30] * (tran_high_t)cospi_4_64;
2349 step1[17] = HIGHBD_WRAPLOW(dct_const_round_shift(temp1), bd);
2350 step1[30] = HIGHBD_WRAPLOW(dct_const_round_shift(temp2), bd);
2351 temp1 = -step2[18] * (tran_high_t)cospi_28_64 -
2352 step2[29] * (tran_high_t)cospi_4_64;
2353 temp2 = -step2[18] * (tran_high_t)cospi_4_64 +
2354 step2[29] * (tran_high_t)cospi_28_64;
2355 step1[18] = HIGHBD_WRAPLOW(dct_const_round_shift(temp1), bd);
2356 step1[29] = HIGHBD_WRAPLOW(dct_const_round_shift(temp2), bd);
2357 step1[19] = step2[19];
2358 step1[20] = step2[20];
2359 temp1 = -step2[21] * (tran_high_t)cospi_20_64 +
2360 step2[26] * (tran_high_t)cospi_12_64;
2361 temp2 = step2[21] * (tran_high_t)cospi_12_64 +
2362 step2[26] * (tran_high_t)cospi_20_64;
2363 step1[21] = HIGHBD_WRAPLOW(dct_const_round_shift(temp1), bd);
2364 step1[26] = HIGHBD_WRAPLOW(dct_const_round_shift(temp2), bd);
2365 temp1 = -step2[22] * (tran_high_t)cospi_12_64 -
2366 step2[25] * (tran_high_t)cospi_20_64;
2367 temp2 = -step2[22] * (tran_high_t)cospi_20_64 +
2368 step2[25] * (tran_high_t)cospi_12_64;
2369 step1[22] = HIGHBD_WRAPLOW(dct_const_round_shift(temp1), bd);
2370 step1[25] = HIGHBD_WRAPLOW(dct_const_round_shift(temp2), bd);
2371 step1[23] = step2[23];
2372 step1[24] = step2[24];
2373 step1[27] = step2[27];
2374 step1[28] = step2[28];
2375
2376 // stage 4
2377 temp1 = (step1[0] + step1[1]) * (tran_high_t)cospi_16_64;
2378 temp2 = (step1[0] - step1[1]) * (tran_high_t)cospi_16_64;
2379 step2[0] = HIGHBD_WRAPLOW(dct_const_round_shift(temp1), bd);
2380 step2[1] = HIGHBD_WRAPLOW(dct_const_round_shift(temp2), bd);
2381 temp1 =
2382 step1[2] * (tran_high_t)cospi_24_64 - step1[3] * (tran_high_t)cospi_8_64;
2383 temp2 =
2384 step1[2] * (tran_high_t)cospi_8_64 + step1[3] * (tran_high_t)cospi_24_64;
2385 step2[2] = HIGHBD_WRAPLOW(dct_const_round_shift(temp1), bd);
2386 step2[3] = HIGHBD_WRAPLOW(dct_const_round_shift(temp2), bd);
2387 step2[4] = HIGHBD_WRAPLOW(step1[4] + step1[5], bd);
2388 step2[5] = HIGHBD_WRAPLOW(step1[4] - step1[5], bd);
2389 step2[6] = HIGHBD_WRAPLOW(-step1[6] + step1[7], bd);
2390 step2[7] = HIGHBD_WRAPLOW(step1[6] + step1[7], bd);
2391
2392 step2[8] = step1[8];
2393 step2[15] = step1[15];
2394 temp1 = -step1[9] * (tran_high_t)cospi_8_64 +
2395 step1[14] * (tran_high_t)cospi_24_64;
2396 temp2 =
2397 step1[9] * (tran_high_t)cospi_24_64 + step1[14] * (tran_high_t)cospi_8_64;
2398 step2[9] = HIGHBD_WRAPLOW(dct_const_round_shift(temp1), bd);
2399 step2[14] = HIGHBD_WRAPLOW(dct_const_round_shift(temp2), bd);
2400 temp1 = -step1[10] * (tran_high_t)cospi_24_64 -
2401 step1[13] * (tran_high_t)cospi_8_64;
2402 temp2 = -step1[10] * (tran_high_t)cospi_8_64 +
2403 step1[13] * (tran_high_t)cospi_24_64;
2404 step2[10] = HIGHBD_WRAPLOW(dct_const_round_shift(temp1), bd);
2405 step2[13] = HIGHBD_WRAPLOW(dct_const_round_shift(temp2), bd);
2406 step2[11] = step1[11];
2407 step2[12] = step1[12];
2408
2409 step2[16] = HIGHBD_WRAPLOW(step1[16] + step1[19], bd);
2410 step2[17] = HIGHBD_WRAPLOW(step1[17] + step1[18], bd);
2411 step2[18] = HIGHBD_WRAPLOW(step1[17] - step1[18], bd);
2412 step2[19] = HIGHBD_WRAPLOW(step1[16] - step1[19], bd);
2413 step2[20] = HIGHBD_WRAPLOW(-step1[20] + step1[23], bd);
2414 step2[21] = HIGHBD_WRAPLOW(-step1[21] + step1[22], bd);
2415 step2[22] = HIGHBD_WRAPLOW(step1[21] + step1[22], bd);
2416 step2[23] = HIGHBD_WRAPLOW(step1[20] + step1[23], bd);
2417
2418 step2[24] = HIGHBD_WRAPLOW(step1[24] + step1[27], bd);
2419 step2[25] = HIGHBD_WRAPLOW(step1[25] + step1[26], bd);
2420 step2[26] = HIGHBD_WRAPLOW(step1[25] - step1[26], bd);
2421 step2[27] = HIGHBD_WRAPLOW(step1[24] - step1[27], bd);
2422 step2[28] = HIGHBD_WRAPLOW(-step1[28] + step1[31], bd);
2423 step2[29] = HIGHBD_WRAPLOW(-step1[29] + step1[30], bd);
2424 step2[30] = HIGHBD_WRAPLOW(step1[29] + step1[30], bd);
2425 step2[31] = HIGHBD_WRAPLOW(step1[28] + step1[31], bd);
2426
2427 // stage 5
2428 step1[0] = HIGHBD_WRAPLOW(step2[0] + step2[3], bd);
2429 step1[1] = HIGHBD_WRAPLOW(step2[1] + step2[2], bd);
2430 step1[2] = HIGHBD_WRAPLOW(step2[1] - step2[2], bd);
2431 step1[3] = HIGHBD_WRAPLOW(step2[0] - step2[3], bd);
2432 step1[4] = step2[4];
2433 temp1 = (step2[6] - step2[5]) * (tran_high_t)cospi_16_64;
2434 temp2 = (step2[5] + step2[6]) * (tran_high_t)cospi_16_64;
2435 step1[5] = HIGHBD_WRAPLOW(dct_const_round_shift(temp1), bd);
2436 step1[6] = HIGHBD_WRAPLOW(dct_const_round_shift(temp2), bd);
2437 step1[7] = step2[7];
2438
2439 step1[8] = HIGHBD_WRAPLOW(step2[8] + step2[11], bd);
2440 step1[9] = HIGHBD_WRAPLOW(step2[9] + step2[10], bd);
2441 step1[10] = HIGHBD_WRAPLOW(step2[9] - step2[10], bd);
2442 step1[11] = HIGHBD_WRAPLOW(step2[8] - step2[11], bd);
2443 step1[12] = HIGHBD_WRAPLOW(-step2[12] + step2[15], bd);
2444 step1[13] = HIGHBD_WRAPLOW(-step2[13] + step2[14], bd);
2445 step1[14] = HIGHBD_WRAPLOW(step2[13] + step2[14], bd);
2446 step1[15] = HIGHBD_WRAPLOW(step2[12] + step2[15], bd);
2447
2448 step1[16] = step2[16];
2449 step1[17] = step2[17];
2450 temp1 = -step2[18] * (tran_high_t)cospi_8_64 +
2451 step2[29] * (tran_high_t)cospi_24_64;
2452 temp2 = step2[18] * (tran_high_t)cospi_24_64 +
2453 step2[29] * (tran_high_t)cospi_8_64;
2454 step1[18] = HIGHBD_WRAPLOW(dct_const_round_shift(temp1), bd);
2455 step1[29] = HIGHBD_WRAPLOW(dct_const_round_shift(temp2), bd);
2456 temp1 = -step2[19] * (tran_high_t)cospi_8_64 +
2457 step2[28] * (tran_high_t)cospi_24_64;
2458 temp2 = step2[19] * (tran_high_t)cospi_24_64 +
2459 step2[28] * (tran_high_t)cospi_8_64;
2460 step1[19] = HIGHBD_WRAPLOW(dct_const_round_shift(temp1), bd);
2461 step1[28] = HIGHBD_WRAPLOW(dct_const_round_shift(temp2), bd);
2462 temp1 = -step2[20] * (tran_high_t)cospi_24_64 -
2463 step2[27] * (tran_high_t)cospi_8_64;
2464 temp2 = -step2[20] * (tran_high_t)cospi_8_64 +
2465 step2[27] * (tran_high_t)cospi_24_64;
2466 step1[20] = HIGHBD_WRAPLOW(dct_const_round_shift(temp1), bd);
2467 step1[27] = HIGHBD_WRAPLOW(dct_const_round_shift(temp2), bd);
2468 temp1 = -step2[21] * (tran_high_t)cospi_24_64 -
2469 step2[26] * (tran_high_t)cospi_8_64;
2470 temp2 = -step2[21] * (tran_high_t)cospi_8_64 +
2471 step2[26] * (tran_high_t)cospi_24_64;
2472 step1[21] = HIGHBD_WRAPLOW(dct_const_round_shift(temp1), bd);
2473 step1[26] = HIGHBD_WRAPLOW(dct_const_round_shift(temp2), bd);
2474 step1[22] = step2[22];
2475 step1[23] = step2[23];
2476 step1[24] = step2[24];
2477 step1[25] = step2[25];
2478 step1[30] = step2[30];
2479 step1[31] = step2[31];
2480
2481 // stage 6
2482 step2[0] = HIGHBD_WRAPLOW(step1[0] + step1[7], bd);
2483 step2[1] = HIGHBD_WRAPLOW(step1[1] + step1[6], bd);
2484 step2[2] = HIGHBD_WRAPLOW(step1[2] + step1[5], bd);
2485 step2[3] = HIGHBD_WRAPLOW(step1[3] + step1[4], bd);
2486 step2[4] = HIGHBD_WRAPLOW(step1[3] - step1[4], bd);
2487 step2[5] = HIGHBD_WRAPLOW(step1[2] - step1[5], bd);
2488 step2[6] = HIGHBD_WRAPLOW(step1[1] - step1[6], bd);
2489 step2[7] = HIGHBD_WRAPLOW(step1[0] - step1[7], bd);
2490 step2[8] = step1[8];
2491 step2[9] = step1[9];
2492 temp1 = (-step1[10] + step1[13]) * (tran_high_t)cospi_16_64;
2493 temp2 = (step1[10] + step1[13]) * (tran_high_t)cospi_16_64;
2494 step2[10] = HIGHBD_WRAPLOW(dct_const_round_shift(temp1), bd);
2495 step2[13] = HIGHBD_WRAPLOW(dct_const_round_shift(temp2), bd);
2496 temp1 = (-step1[11] + step1[12]) * (tran_high_t)cospi_16_64;
2497 temp2 = (step1[11] + step1[12]) * (tran_high_t)cospi_16_64;
2498 step2[11] = HIGHBD_WRAPLOW(dct_const_round_shift(temp1), bd);
2499 step2[12] = HIGHBD_WRAPLOW(dct_const_round_shift(temp2), bd);
2500 step2[14] = step1[14];
2501 step2[15] = step1[15];
2502
2503 step2[16] = HIGHBD_WRAPLOW(step1[16] + step1[23], bd);
2504 step2[17] = HIGHBD_WRAPLOW(step1[17] + step1[22], bd);
2505 step2[18] = HIGHBD_WRAPLOW(step1[18] + step1[21], bd);
2506 step2[19] = HIGHBD_WRAPLOW(step1[19] + step1[20], bd);
2507 step2[20] = HIGHBD_WRAPLOW(step1[19] - step1[20], bd);
2508 step2[21] = HIGHBD_WRAPLOW(step1[18] - step1[21], bd);
2509 step2[22] = HIGHBD_WRAPLOW(step1[17] - step1[22], bd);
2510 step2[23] = HIGHBD_WRAPLOW(step1[16] - step1[23], bd);
2511
2512 step2[24] = HIGHBD_WRAPLOW(-step1[24] + step1[31], bd);
2513 step2[25] = HIGHBD_WRAPLOW(-step1[25] + step1[30], bd);
2514 step2[26] = HIGHBD_WRAPLOW(-step1[26] + step1[29], bd);
2515 step2[27] = HIGHBD_WRAPLOW(-step1[27] + step1[28], bd);
2516 step2[28] = HIGHBD_WRAPLOW(step1[27] + step1[28], bd);
2517 step2[29] = HIGHBD_WRAPLOW(step1[26] + step1[29], bd);
2518 step2[30] = HIGHBD_WRAPLOW(step1[25] + step1[30], bd);
2519 step2[31] = HIGHBD_WRAPLOW(step1[24] + step1[31], bd);
2520
2521 // stage 7
2522 step1[0] = HIGHBD_WRAPLOW(step2[0] + step2[15], bd);
2523 step1[1] = HIGHBD_WRAPLOW(step2[1] + step2[14], bd);
2524 step1[2] = HIGHBD_WRAPLOW(step2[2] + step2[13], bd);
2525 step1[3] = HIGHBD_WRAPLOW(step2[3] + step2[12], bd);
2526 step1[4] = HIGHBD_WRAPLOW(step2[4] + step2[11], bd);
2527 step1[5] = HIGHBD_WRAPLOW(step2[5] + step2[10], bd);
2528 step1[6] = HIGHBD_WRAPLOW(step2[6] + step2[9], bd);
2529 step1[7] = HIGHBD_WRAPLOW(step2[7] + step2[8], bd);
2530 step1[8] = HIGHBD_WRAPLOW(step2[7] - step2[8], bd);
2531 step1[9] = HIGHBD_WRAPLOW(step2[6] - step2[9], bd);
2532 step1[10] = HIGHBD_WRAPLOW(step2[5] - step2[10], bd);
2533 step1[11] = HIGHBD_WRAPLOW(step2[4] - step2[11], bd);
2534 step1[12] = HIGHBD_WRAPLOW(step2[3] - step2[12], bd);
2535 step1[13] = HIGHBD_WRAPLOW(step2[2] - step2[13], bd);
2536 step1[14] = HIGHBD_WRAPLOW(step2[1] - step2[14], bd);
2537 step1[15] = HIGHBD_WRAPLOW(step2[0] - step2[15], bd);
2538
2539 step1[16] = step2[16];
2540 step1[17] = step2[17];
2541 step1[18] = step2[18];
2542 step1[19] = step2[19];
2543 temp1 = (-step2[20] + step2[27]) * (tran_high_t)cospi_16_64;
2544 temp2 = (step2[20] + step2[27]) * (tran_high_t)cospi_16_64;
2545 step1[20] = HIGHBD_WRAPLOW(dct_const_round_shift(temp1), bd);
2546 step1[27] = HIGHBD_WRAPLOW(dct_const_round_shift(temp2), bd);
2547 temp1 = (-step2[21] + step2[26]) * (tran_high_t)cospi_16_64;
2548 temp2 = (step2[21] + step2[26]) * (tran_high_t)cospi_16_64;
2549 step1[21] = HIGHBD_WRAPLOW(dct_const_round_shift(temp1), bd);
2550 step1[26] = HIGHBD_WRAPLOW(dct_const_round_shift(temp2), bd);
2551 temp1 = (-step2[22] + step2[25]) * (tran_high_t)cospi_16_64;
2552 temp2 = (step2[22] + step2[25]) * (tran_high_t)cospi_16_64;
2553 step1[22] = HIGHBD_WRAPLOW(dct_const_round_shift(temp1), bd);
2554 step1[25] = HIGHBD_WRAPLOW(dct_const_round_shift(temp2), bd);
2555 temp1 = (-step2[23] + step2[24]) * (tran_high_t)cospi_16_64;
2556 temp2 = (step2[23] + step2[24]) * (tran_high_t)cospi_16_64;
2557 step1[23] = HIGHBD_WRAPLOW(dct_const_round_shift(temp1), bd);
2558 step1[24] = HIGHBD_WRAPLOW(dct_const_round_shift(temp2), bd);
2559 step1[28] = step2[28];
2560 step1[29] = step2[29];
2561 step1[30] = step2[30];
2562 step1[31] = step2[31];
2563
2564 // final stage
2565 output[0] = HIGHBD_WRAPLOW(step1[0] + step1[31], bd);
2566 output[1] = HIGHBD_WRAPLOW(step1[1] + step1[30], bd);
2567 output[2] = HIGHBD_WRAPLOW(step1[2] + step1[29], bd);
2568 output[3] = HIGHBD_WRAPLOW(step1[3] + step1[28], bd);
2569 output[4] = HIGHBD_WRAPLOW(step1[4] + step1[27], bd);
2570 output[5] = HIGHBD_WRAPLOW(step1[5] + step1[26], bd);
2571 output[6] = HIGHBD_WRAPLOW(step1[6] + step1[25], bd);
2572 output[7] = HIGHBD_WRAPLOW(step1[7] + step1[24], bd);
2573 output[8] = HIGHBD_WRAPLOW(step1[8] + step1[23], bd);
2574 output[9] = HIGHBD_WRAPLOW(step1[9] + step1[22], bd);
2575 output[10] = HIGHBD_WRAPLOW(step1[10] + step1[21], bd);
2576 output[11] = HIGHBD_WRAPLOW(step1[11] + step1[20], bd);
2577 output[12] = HIGHBD_WRAPLOW(step1[12] + step1[19], bd);
2578 output[13] = HIGHBD_WRAPLOW(step1[13] + step1[18], bd);
2579 output[14] = HIGHBD_WRAPLOW(step1[14] + step1[17], bd);
2580 output[15] = HIGHBD_WRAPLOW(step1[15] + step1[16], bd);
2581 output[16] = HIGHBD_WRAPLOW(step1[15] - step1[16], bd);
2582 output[17] = HIGHBD_WRAPLOW(step1[14] - step1[17], bd);
2583 output[18] = HIGHBD_WRAPLOW(step1[13] - step1[18], bd);
2584 output[19] = HIGHBD_WRAPLOW(step1[12] - step1[19], bd);
2585 output[20] = HIGHBD_WRAPLOW(step1[11] - step1[20], bd);
2586 output[21] = HIGHBD_WRAPLOW(step1[10] - step1[21], bd);
2587 output[22] = HIGHBD_WRAPLOW(step1[9] - step1[22], bd);
2588 output[23] = HIGHBD_WRAPLOW(step1[8] - step1[23], bd);
2589 output[24] = HIGHBD_WRAPLOW(step1[7] - step1[24], bd);
2590 output[25] = HIGHBD_WRAPLOW(step1[6] - step1[25], bd);
2591 output[26] = HIGHBD_WRAPLOW(step1[5] - step1[26], bd);
2592 output[27] = HIGHBD_WRAPLOW(step1[4] - step1[27], bd);
2593 output[28] = HIGHBD_WRAPLOW(step1[3] - step1[28], bd);
2594 output[29] = HIGHBD_WRAPLOW(step1[2] - step1[29], bd);
2595 output[30] = HIGHBD_WRAPLOW(step1[1] - step1[30], bd);
2596 output[31] = HIGHBD_WRAPLOW(step1[0] - step1[31], bd);
2597 }
2598
vpx_highbd_idct32x32_1024_add_c(const tran_low_t * input,uint16_t * dest,int stride,int bd)2599 void vpx_highbd_idct32x32_1024_add_c(const tran_low_t *input, uint16_t *dest,
2600 int stride, int bd) {
2601 int i, j;
2602 tran_low_t out[32 * 32];
2603 tran_low_t *outptr = out;
2604 tran_low_t temp_in[32], temp_out[32];
2605
2606 // Rows
2607 for (i = 0; i < 32; ++i) {
2608 tran_low_t zero_coeff = 0;
2609 for (j = 0; j < 32; ++j) zero_coeff |= input[j];
2610
2611 if (zero_coeff)
2612 highbd_idct32_c(input, outptr, bd);
2613 else
2614 memset(outptr, 0, sizeof(tran_low_t) * 32);
2615 input += 32;
2616 outptr += 32;
2617 }
2618
2619 // Columns
2620 for (i = 0; i < 32; ++i) {
2621 for (j = 0; j < 32; ++j) temp_in[j] = out[j * 32 + i];
2622 highbd_idct32_c(temp_in, temp_out, bd);
2623 for (j = 0; j < 32; ++j) {
2624 dest[j * stride + i] = highbd_clip_pixel_add(
2625 dest[j * stride + i], ROUND_POWER_OF_TWO(temp_out[j], 6), bd);
2626 }
2627 }
2628 }
2629
vpx_highbd_idct32x32_135_add_c(const tran_low_t * input,uint16_t * dest,int stride,int bd)2630 void vpx_highbd_idct32x32_135_add_c(const tran_low_t *input, uint16_t *dest,
2631 int stride, int bd) {
2632 int i, j;
2633 tran_low_t out[32 * 32] = { 0 };
2634 tran_low_t *outptr = out;
2635 tran_low_t temp_in[32], temp_out[32];
2636
2637 // Rows
2638 // Only upper-left 16x16 has non-zero coeff
2639 for (i = 0; i < 16; ++i) {
2640 highbd_idct32_c(input, outptr, bd);
2641 input += 32;
2642 outptr += 32;
2643 }
2644
2645 // Columns
2646 for (i = 0; i < 32; ++i) {
2647 uint16_t *destT = dest;
2648 for (j = 0; j < 32; ++j) temp_in[j] = out[j * 32 + i];
2649 highbd_idct32_c(temp_in, temp_out, bd);
2650 for (j = 0; j < 32; ++j) {
2651 destT[i] = highbd_clip_pixel_add(destT[i],
2652 ROUND_POWER_OF_TWO(temp_out[j], 6), bd);
2653 destT += stride;
2654 }
2655 }
2656 }
2657
vpx_highbd_idct32x32_34_add_c(const tran_low_t * input,uint16_t * dest,int stride,int bd)2658 void vpx_highbd_idct32x32_34_add_c(const tran_low_t *input, uint16_t *dest,
2659 int stride, int bd) {
2660 int i, j;
2661 tran_low_t out[32 * 32] = { 0 };
2662 tran_low_t *outptr = out;
2663 tran_low_t temp_in[32], temp_out[32];
2664
2665 // Rows
2666 // Only upper-left 8x8 has non-zero coeff
2667 for (i = 0; i < 8; ++i) {
2668 highbd_idct32_c(input, outptr, bd);
2669 input += 32;
2670 outptr += 32;
2671 }
2672
2673 // Columns
2674 for (i = 0; i < 32; ++i) {
2675 for (j = 0; j < 32; ++j) temp_in[j] = out[j * 32 + i];
2676 highbd_idct32_c(temp_in, temp_out, bd);
2677 for (j = 0; j < 32; ++j) {
2678 dest[j * stride + i] = highbd_clip_pixel_add(
2679 dest[j * stride + i], ROUND_POWER_OF_TWO(temp_out[j], 6), bd);
2680 }
2681 }
2682 }
2683
vpx_highbd_idct32x32_1_add_c(const tran_low_t * input,uint16_t * dest,int stride,int bd)2684 void vpx_highbd_idct32x32_1_add_c(const tran_low_t *input, uint16_t *dest,
2685 int stride, int bd) {
2686 int i, j;
2687 int a1;
2688 tran_low_t out = HIGHBD_WRAPLOW(
2689 dct_const_round_shift(input[0] * (tran_high_t)cospi_16_64), bd);
2690
2691 out =
2692 HIGHBD_WRAPLOW(dct_const_round_shift(out * (tran_high_t)cospi_16_64), bd);
2693 a1 = ROUND_POWER_OF_TWO(out, 6);
2694
2695 for (j = 0; j < 32; ++j) {
2696 for (i = 0; i < 32; ++i) dest[i] = highbd_clip_pixel_add(dest[i], a1, bd);
2697 dest += stride;
2698 }
2699 }
2700
2701 #endif // CONFIG_VP9_HIGHBITDEPTH
2702