• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 /*
2  * AltiVec optimizations for libjpeg-turbo
3  *
4  * Copyright (C) 2014-2015, D. R. Commander.  All Rights Reserved.
5  *
6  * This software is provided 'as-is', without any express or implied
7  * warranty.  In no event will the authors be held liable for any damages
8  * arising from the use of this software.
9  *
10  * Permission is granted to anyone to use this software for any purpose,
11  * including commercial applications, and to alter it and redistribute it
12  * freely, subject to the following restrictions:
13  *
14  * 1. The origin of this software must not be misrepresented; you must not
15  *    claim that you wrote the original software. If you use this software
16  *    in a product, an acknowledgment in the product documentation would be
17  *    appreciated but is not required.
18  * 2. Altered source versions must be plainly marked as such, and must not be
19  *    misrepresented as being the original software.
20  * 3. This notice may not be removed or altered from any source distribution.
21  */
22 
23 /* SLOW INTEGER INVERSE DCT */
24 
25 #include "jsimd_altivec.h"
26 
27 
28 #define F_0_298  2446   /* FIX(0.298631336) */
29 #define F_0_390  3196   /* FIX(0.390180644) */
30 #define F_0_541  4433   /* FIX(0.541196100) */
31 #define F_0_765  6270   /* FIX(0.765366865) */
32 #define F_0_899  7373   /* FIX(0.899976223) */
33 #define F_1_175  9633   /* FIX(1.175875602) */
34 #define F_1_501  12299  /* FIX(1.501321110) */
35 #define F_1_847  15137  /* FIX(1.847759065) */
36 #define F_1_961  16069  /* FIX(1.961570560) */
37 #define F_2_053  16819  /* FIX(2.053119869) */
38 #define F_2_562  20995  /* FIX(2.562915447) */
39 #define F_3_072  25172  /* FIX(3.072711026) */
40 
41 #define CONST_BITS  13
42 #define PASS1_BITS  2
43 #define DESCALE_P1  (CONST_BITS - PASS1_BITS)
44 #define DESCALE_P2  (CONST_BITS + PASS1_BITS + 3)
45 
46 
47 #define DO_IDCT(in, PASS) { \
48   /* Even part \
49    * \
50    * (Original) \
51    * z1 = (z2 + z3) * 0.541196100; \
52    * tmp2 = z1 + z3 * -1.847759065; \
53    * tmp3 = z1 + z2 * 0.765366865; \
54    * \
55    * (This implementation) \
56    * tmp2 = z2 * 0.541196100 + z3 * (0.541196100 - 1.847759065); \
57    * tmp3 = z2 * (0.541196100 + 0.765366865) + z3 * 0.541196100; \
58    */ \
59   \
60   in##26l = vec_mergeh(in##2, in##6); \
61   in##26h = vec_mergel(in##2, in##6); \
62   \
63   tmp3l = vec_msums(in##26l, pw_f130_f054, pd_zero); \
64   tmp3h = vec_msums(in##26h, pw_f130_f054, pd_zero); \
65   tmp2l = vec_msums(in##26l, pw_f054_mf130, pd_zero); \
66   tmp2h = vec_msums(in##26h, pw_f054_mf130, pd_zero); \
67   \
68   tmp0 = vec_add(in##0, in##4); \
69   tmp1 = vec_sub(in##0, in##4); \
70   \
71   tmp0l = vec_unpackh(tmp0); \
72   tmp0h = vec_unpackl(tmp0); \
73   tmp0l = vec_sl(tmp0l, const_bits); \
74   tmp0h = vec_sl(tmp0h, const_bits); \
75   tmp0l = vec_add(tmp0l, pd_descale_p##PASS); \
76   tmp0h = vec_add(tmp0h, pd_descale_p##PASS); \
77   \
78   tmp10l = vec_add(tmp0l, tmp3l); \
79   tmp10h = vec_add(tmp0h, tmp3h); \
80   tmp13l = vec_sub(tmp0l, tmp3l); \
81   tmp13h = vec_sub(tmp0h, tmp3h); \
82   \
83   tmp1l = vec_unpackh(tmp1); \
84   tmp1h = vec_unpackl(tmp1); \
85   tmp1l = vec_sl(tmp1l, const_bits); \
86   tmp1h = vec_sl(tmp1h, const_bits); \
87   tmp1l = vec_add(tmp1l, pd_descale_p##PASS); \
88   tmp1h = vec_add(tmp1h, pd_descale_p##PASS); \
89   \
90   tmp11l = vec_add(tmp1l, tmp2l); \
91   tmp11h = vec_add(tmp1h, tmp2h); \
92   tmp12l = vec_sub(tmp1l, tmp2l); \
93   tmp12h = vec_sub(tmp1h, tmp2h); \
94   \
95   /* Odd part */ \
96   \
97   z3 = vec_add(in##3, in##7); \
98   z4 = vec_add(in##1, in##5); \
99   \
100   /* (Original) \
101    * z5 = (z3 + z4) * 1.175875602; \
102    * z3 = z3 * -1.961570560;  z4 = z4 * -0.390180644; \
103    * z3 += z5;  z4 += z5; \
104    * \
105    * (This implementation) \
106    * z3 = z3 * (1.175875602 - 1.961570560) + z4 * 1.175875602; \
107    * z4 = z3 * 1.175875602 + z4 * (1.175875602 - 0.390180644); \
108    */ \
109   \
110   z34l = vec_mergeh(z3, z4); \
111   z34h = vec_mergel(z3, z4); \
112   \
113   z3l = vec_msums(z34l, pw_mf078_f117, pd_zero); \
114   z3h = vec_msums(z34h, pw_mf078_f117, pd_zero); \
115   z4l = vec_msums(z34l, pw_f117_f078, pd_zero); \
116   z4h = vec_msums(z34h, pw_f117_f078, pd_zero); \
117   \
118   /* (Original) \
119    * z1 = tmp0 + tmp3;  z2 = tmp1 + tmp2; \
120    * tmp0 = tmp0 * 0.298631336;  tmp1 = tmp1 * 2.053119869; \
121    * tmp2 = tmp2 * 3.072711026;  tmp3 = tmp3 * 1.501321110; \
122    * z1 = z1 * -0.899976223;  z2 = z2 * -2.562915447; \
123    * tmp0 += z1 + z3;  tmp1 += z2 + z4; \
124    * tmp2 += z2 + z3;  tmp3 += z1 + z4; \
125    * \
126    * (This implementation) \
127    * tmp0 = tmp0 * (0.298631336 - 0.899976223) + tmp3 * -0.899976223; \
128    * tmp1 = tmp1 * (2.053119869 - 2.562915447) + tmp2 * -2.562915447; \
129    * tmp2 = tmp1 * -2.562915447 + tmp2 * (3.072711026 - 2.562915447); \
130    * tmp3 = tmp0 * -0.899976223 + tmp3 * (1.501321110 - 0.899976223); \
131    * tmp0 += z3;  tmp1 += z4; \
132    * tmp2 += z3;  tmp3 += z4; \
133    */ \
134   \
135   in##71l = vec_mergeh(in##7, in##1); \
136   in##71h = vec_mergel(in##7, in##1); \
137   \
138   tmp0l = vec_msums(in##71l, pw_mf060_mf089, z3l); \
139   tmp0h = vec_msums(in##71h, pw_mf060_mf089, z3h); \
140   tmp3l = vec_msums(in##71l, pw_mf089_f060, z4l); \
141   tmp3h = vec_msums(in##71h, pw_mf089_f060, z4h); \
142   \
143   in##53l = vec_mergeh(in##5, in##3); \
144   in##53h = vec_mergel(in##5, in##3); \
145   \
146   tmp1l = vec_msums(in##53l, pw_mf050_mf256, z4l); \
147   tmp1h = vec_msums(in##53h, pw_mf050_mf256, z4h); \
148   tmp2l = vec_msums(in##53l, pw_mf256_f050, z3l); \
149   tmp2h = vec_msums(in##53h, pw_mf256_f050, z3h); \
150   \
151   /* Final output stage */ \
152   \
153   out0l = vec_add(tmp10l, tmp3l); \
154   out0h = vec_add(tmp10h, tmp3h); \
155   out7l = vec_sub(tmp10l, tmp3l); \
156   out7h = vec_sub(tmp10h, tmp3h); \
157   \
158   out0l = vec_sra(out0l, descale_p##PASS); \
159   out0h = vec_sra(out0h, descale_p##PASS); \
160   out7l = vec_sra(out7l, descale_p##PASS); \
161   out7h = vec_sra(out7h, descale_p##PASS); \
162   \
163   out0 = vec_pack(out0l, out0h); \
164   out7 = vec_pack(out7l, out7h); \
165   \
166   out1l = vec_add(tmp11l, tmp2l); \
167   out1h = vec_add(tmp11h, tmp2h); \
168   out6l = vec_sub(tmp11l, tmp2l); \
169   out6h = vec_sub(tmp11h, tmp2h); \
170   \
171   out1l = vec_sra(out1l, descale_p##PASS); \
172   out1h = vec_sra(out1h, descale_p##PASS); \
173   out6l = vec_sra(out6l, descale_p##PASS); \
174   out6h = vec_sra(out6h, descale_p##PASS); \
175   \
176   out1 = vec_pack(out1l, out1h); \
177   out6 = vec_pack(out6l, out6h); \
178   \
179   out2l = vec_add(tmp12l, tmp1l); \
180   out2h = vec_add(tmp12h, tmp1h); \
181   out5l = vec_sub(tmp12l, tmp1l); \
182   out5h = vec_sub(tmp12h, tmp1h); \
183   \
184   out2l = vec_sra(out2l, descale_p##PASS); \
185   out2h = vec_sra(out2h, descale_p##PASS); \
186   out5l = vec_sra(out5l, descale_p##PASS); \
187   out5h = vec_sra(out5h, descale_p##PASS); \
188   \
189   out2 = vec_pack(out2l, out2h); \
190   out5 = vec_pack(out5l, out5h); \
191   \
192   out3l = vec_add(tmp13l, tmp0l); \
193   out3h = vec_add(tmp13h, tmp0h); \
194   out4l = vec_sub(tmp13l, tmp0l); \
195   out4h = vec_sub(tmp13h, tmp0h); \
196   \
197   out3l = vec_sra(out3l, descale_p##PASS); \
198   out3h = vec_sra(out3h, descale_p##PASS); \
199   out4l = vec_sra(out4l, descale_p##PASS); \
200   out4h = vec_sra(out4h, descale_p##PASS); \
201   \
202   out3 = vec_pack(out3l, out3h); \
203   out4 = vec_pack(out4l, out4h); \
204 }
205 
206 
jsimd_idct_islow_altivec(void * dct_table_,JCOEFPTR coef_block,JSAMPARRAY output_buf,JDIMENSION output_col)207 void jsimd_idct_islow_altivec(void *dct_table_, JCOEFPTR coef_block,
208                               JSAMPARRAY output_buf, JDIMENSION output_col)
209 {
210   short *dct_table = (short *)dct_table_;
211   int *outptr;
212 
213   __vector short row0, row1, row2, row3, row4, row5, row6, row7,
214     col0, col1, col2, col3, col4, col5, col6, col7,
215     quant0, quant1, quant2, quant3, quant4, quant5, quant6, quant7,
216     tmp0, tmp1, tmp2, tmp3, z3, z4,
217     z34l, z34h, col71l, col71h, col26l, col26h, col53l, col53h,
218     row71l, row71h, row26l, row26h, row53l, row53h,
219     out0, out1, out2, out3, out4, out5, out6, out7;
220   __vector int tmp0l, tmp0h, tmp1l, tmp1h, tmp2l, tmp2h, tmp3l, tmp3h,
221     tmp10l, tmp10h, tmp11l, tmp11h, tmp12l, tmp12h, tmp13l, tmp13h,
222     z3l, z3h, z4l, z4h,
223     out0l, out0h, out1l, out1h, out2l, out2h, out3l, out3h, out4l, out4h,
224     out5l, out5h, out6l, out6h, out7l, out7h;
225   __vector signed char outb;
226 
227   /* Constants */
228   __vector short pw_zero = { __8X(0) },
229     pw_f130_f054 = { __4X2(F_0_541 + F_0_765, F_0_541) },
230     pw_f054_mf130 = { __4X2(F_0_541, F_0_541 - F_1_847) },
231     pw_mf078_f117 = { __4X2(F_1_175 - F_1_961, F_1_175) },
232     pw_f117_f078 = { __4X2(F_1_175, F_1_175 - F_0_390) },
233     pw_mf060_mf089 = { __4X2(F_0_298 - F_0_899, -F_0_899) },
234     pw_mf089_f060 = { __4X2(-F_0_899, F_1_501 - F_0_899) },
235     pw_mf050_mf256 = { __4X2(F_2_053 - F_2_562, -F_2_562) },
236     pw_mf256_f050 = { __4X2(-F_2_562, F_3_072 - F_2_562) };
237   __vector unsigned short pass1_bits = { __8X(PASS1_BITS) };
238   __vector int pd_zero = { __4X(0) },
239     pd_descale_p1 = { __4X(1 << (DESCALE_P1 - 1)) },
240     pd_descale_p2 = { __4X(1 << (DESCALE_P2 - 1)) };
241   __vector unsigned int descale_p1 = { __4X(DESCALE_P1) },
242     descale_p2 = { __4X(DESCALE_P2) },
243     const_bits = { __4X(CONST_BITS) };
244   __vector signed char pb_centerjsamp = { __16X(CENTERJSAMPLE) };
245 
246   /* Pass 1: process columns */
247 
248   col0 = vec_ld(0, coef_block);
249   col1 = vec_ld(16, coef_block);
250   col2 = vec_ld(32, coef_block);
251   col3 = vec_ld(48, coef_block);
252   col4 = vec_ld(64, coef_block);
253   col5 = vec_ld(80, coef_block);
254   col6 = vec_ld(96, coef_block);
255   col7 = vec_ld(112, coef_block);
256 
257   tmp1 = vec_or(col1, col2);
258   tmp2 = vec_or(col3, col4);
259   tmp1 = vec_or(tmp1, tmp2);
260   tmp3 = vec_or(col5, col6);
261   tmp3 = vec_or(tmp3, col7);
262   tmp1 = vec_or(tmp1, tmp3);
263 
264   quant0 = vec_ld(0, dct_table);
265   col0 = vec_mladd(col0, quant0, pw_zero);
266 
267   if (vec_all_eq(tmp1, pw_zero)) {
268     /* AC terms all zero */
269 
270     col0 = vec_sl(col0, pass1_bits);
271 
272     row0 = vec_splat(col0, 0);
273     row1 = vec_splat(col0, 1);
274     row2 = vec_splat(col0, 2);
275     row3 = vec_splat(col0, 3);
276     row4 = vec_splat(col0, 4);
277     row5 = vec_splat(col0, 5);
278     row6 = vec_splat(col0, 6);
279     row7 = vec_splat(col0, 7);
280 
281   } else {
282 
283     quant1 = vec_ld(16, dct_table);
284     quant2 = vec_ld(32, dct_table);
285     quant3 = vec_ld(48, dct_table);
286     quant4 = vec_ld(64, dct_table);
287     quant5 = vec_ld(80, dct_table);
288     quant6 = vec_ld(96, dct_table);
289     quant7 = vec_ld(112, dct_table);
290 
291     col1 = vec_mladd(col1, quant1, pw_zero);
292     col2 = vec_mladd(col2, quant2, pw_zero);
293     col3 = vec_mladd(col3, quant3, pw_zero);
294     col4 = vec_mladd(col4, quant4, pw_zero);
295     col5 = vec_mladd(col5, quant5, pw_zero);
296     col6 = vec_mladd(col6, quant6, pw_zero);
297     col7 = vec_mladd(col7, quant7, pw_zero);
298 
299     DO_IDCT(col, 1);
300 
301     TRANSPOSE(out, row);
302   }
303 
304   /* Pass 2: process rows */
305 
306   DO_IDCT(row, 2);
307 
308   TRANSPOSE(out, col);
309 
310   outb = vec_packs(col0, col0);
311   outb = vec_add(outb, pb_centerjsamp);
312   outptr = (int *)(output_buf[0] + output_col);
313   vec_ste((__vector int)outb, 0, outptr);
314   vec_ste((__vector int)outb, 4, outptr);
315 
316   outb = vec_packs(col1, col1);
317   outb = vec_add(outb, pb_centerjsamp);
318   outptr = (int *)(output_buf[1] + output_col);
319   vec_ste((__vector int)outb, 0, outptr);
320   vec_ste((__vector int)outb, 4, outptr);
321 
322   outb = vec_packs(col2, col2);
323   outb = vec_add(outb, pb_centerjsamp);
324   outptr = (int *)(output_buf[2] + output_col);
325   vec_ste((__vector int)outb, 0, outptr);
326   vec_ste((__vector int)outb, 4, outptr);
327 
328   outb = vec_packs(col3, col3);
329   outb = vec_add(outb, pb_centerjsamp);
330   outptr = (int *)(output_buf[3] + output_col);
331   vec_ste((__vector int)outb, 0, outptr);
332   vec_ste((__vector int)outb, 4, outptr);
333 
334   outb = vec_packs(col4, col4);
335   outb = vec_add(outb, pb_centerjsamp);
336   outptr = (int *)(output_buf[4] + output_col);
337   vec_ste((__vector int)outb, 0, outptr);
338   vec_ste((__vector int)outb, 4, outptr);
339 
340   outb = vec_packs(col5, col5);
341   outb = vec_add(outb, pb_centerjsamp);
342   outptr = (int *)(output_buf[5] + output_col);
343   vec_ste((__vector int)outb, 0, outptr);
344   vec_ste((__vector int)outb, 4, outptr);
345 
346   outb = vec_packs(col6, col6);
347   outb = vec_add(outb, pb_centerjsamp);
348   outptr = (int *)(output_buf[6] + output_col);
349   vec_ste((__vector int)outb, 0, outptr);
350   vec_ste((__vector int)outb, 4, outptr);
351 
352   outb = vec_packs(col7, col7);
353   outb = vec_add(outb, pb_centerjsamp);
354   outptr = (int *)(output_buf[7] + output_col);
355   vec_ste((__vector int)outb, 0, outptr);
356   vec_ste((__vector int)outb, 4, outptr);
357 }
358