• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1  // Copyright 2016 Google Inc. All Rights Reserved.
2  //
3  // Use of this source code is governed by a BSD-style license
4  // that can be found in the COPYING file in the root of the source
5  // tree. An additional intellectual property rights grant can be found
6  // in the file PATENTS. All contributing project authors may
7  // be found in the AUTHORS file in the root of the source tree.
8  // -----------------------------------------------------------------------------
9  //
10  // MSA common macros
11  //
12  // Author(s):  Prashant Patil   (prashant.patil@imgtec.com)
13  
14  #ifndef WEBP_DSP_MSA_MACRO_H_
15  #define WEBP_DSP_MSA_MACRO_H_
16  
17  #include <stdint.h>
18  #include <msa.h>
19  
20  #if defined(__clang__)
21    #define CLANG_BUILD
22  #endif
23  
24  #ifdef CLANG_BUILD
25    #define ALPHAVAL  (-1)
26    #define ADDVI_H(a, b)  __msa_addvi_h((v8i16)a, b)
27    #define ADDVI_W(a, b)  __msa_addvi_w((v4i32)a, b)
28    #define SRAI_B(a, b)  __msa_srai_b((v16i8)a, b)
29    #define SRAI_H(a, b)  __msa_srai_h((v8i16)a, b)
30    #define SRAI_W(a, b)  __msa_srai_w((v4i32)a, b)
31    #define SRLI_H(a, b)  __msa_srli_h((v8i16)a, b)
32    #define SLLI_B(a, b)  __msa_slli_b((v4i32)a, b)
33    #define ANDI_B(a, b)  __msa_andi_b((v16u8)a, b)
34    #define ORI_B(a, b)   __msa_ori_b((v16u8)a, b)
35  #else
36    #define ALPHAVAL  (0xff)
37    #define ADDVI_H(a, b)  (a + b)
38    #define ADDVI_W(a, b)  (a + b)
39    #define SRAI_B(a, b)  (a >> b)
40    #define SRAI_H(a, b)  (a >> b)
41    #define SRAI_W(a, b)  (a >> b)
42    #define SRLI_H(a, b)  (a << b)
43    #define SLLI_B(a, b)  (a << b)
44    #define ANDI_B(a, b)  (a & b)
45    #define ORI_B(a, b)   (a | b)
46  #endif
47  
48  #define LD_B(RTYPE, psrc) *((RTYPE*)(psrc))
49  #define LD_UB(...) LD_B(v16u8, __VA_ARGS__)
50  #define LD_SB(...) LD_B(v16i8, __VA_ARGS__)
51  
52  #define LD_H(RTYPE, psrc) *((RTYPE*)(psrc))
53  #define LD_UH(...) LD_H(v8u16, __VA_ARGS__)
54  #define LD_SH(...) LD_H(v8i16, __VA_ARGS__)
55  
56  #define LD_W(RTYPE, psrc) *((RTYPE*)(psrc))
57  #define LD_UW(...) LD_W(v4u32, __VA_ARGS__)
58  #define LD_SW(...) LD_W(v4i32, __VA_ARGS__)
59  
60  #define ST_B(RTYPE, in, pdst) *((RTYPE*)(pdst)) = in
61  #define ST_UB(...) ST_B(v16u8, __VA_ARGS__)
62  #define ST_SB(...) ST_B(v16i8, __VA_ARGS__)
63  
64  #define ST_H(RTYPE, in, pdst) *((RTYPE*)(pdst)) = in
65  #define ST_UH(...) ST_H(v8u16, __VA_ARGS__)
66  #define ST_SH(...) ST_H(v8i16, __VA_ARGS__)
67  
68  #define ST_W(RTYPE, in, pdst) *((RTYPE*)(pdst)) = in
69  #define ST_UW(...) ST_W(v4u32, __VA_ARGS__)
70  #define ST_SW(...) ST_W(v4i32, __VA_ARGS__)
71  
72  #define MSA_LOAD_FUNC(TYPE, INSTR, FUNC_NAME)             \
73    static inline TYPE FUNC_NAME(const void* const psrc) {  \
74      const uint8_t* const psrc_m = (const uint8_t*)psrc;   \
75      TYPE val_m;                                           \
76      asm volatile (                                        \
77        "" #INSTR " %[val_m], %[psrc_m]  \n\t"              \
78        : [val_m] "=r" (val_m)                              \
79        : [psrc_m] "m" (*psrc_m));                          \
80      return val_m;                                         \
81    }
82  
83  #define MSA_LOAD(psrc, FUNC_NAME)  FUNC_NAME(psrc)
84  
85  #define MSA_STORE_FUNC(TYPE, INSTR, FUNC_NAME)               \
86    static inline void FUNC_NAME(TYPE val, void* const pdst) { \
87      uint8_t* const pdst_m = (uint8_t*)pdst;                  \
88      TYPE val_m = val;                                        \
89      asm volatile (                                           \
90        " " #INSTR "  %[val_m],  %[pdst_m]  \n\t"              \
91        : [pdst_m] "=m" (*pdst_m)                              \
92        : [val_m] "r" (val_m));                                \
93    }
94  
95  #define MSA_STORE(val, pdst, FUNC_NAME)  FUNC_NAME(val, pdst)
96  
97  #if (__mips_isa_rev >= 6)
98    MSA_LOAD_FUNC(uint16_t, lh, msa_lh);
99    #define LH(psrc)  MSA_LOAD(psrc, msa_lh)
100    MSA_LOAD_FUNC(uint32_t, lw, msa_lw);
101    #define LW(psrc)  MSA_LOAD(psrc, msa_lw)
102    #if (__mips == 64)
103      MSA_LOAD_FUNC(uint64_t, ld, msa_ld);
104      #define LD(psrc)  MSA_LOAD(psrc, msa_ld)
105    #else  // !(__mips == 64)
106      #define LD(psrc)  ((((uint64_t)MSA_LOAD(psrc + 4, msa_lw)) << 32) | \
107                         MSA_LOAD(psrc, msa_lw))
108    #endif  // (__mips == 64)
109  
110    MSA_STORE_FUNC(uint16_t, sh, msa_sh);
111    #define SH(val, pdst)  MSA_STORE(val, pdst, msa_sh)
112    MSA_STORE_FUNC(uint32_t, sw, msa_sw);
113    #define SW(val, pdst)  MSA_STORE(val, pdst, msa_sw)
114    MSA_STORE_FUNC(uint64_t, sd, msa_sd);
115    #define SD(val, pdst)  MSA_STORE(val, pdst, msa_sd)
116  #else  // !(__mips_isa_rev >= 6)
117    MSA_LOAD_FUNC(uint16_t, ulh, msa_ulh);
118    #define LH(psrc)  MSA_LOAD(psrc, msa_ulh)
119    MSA_LOAD_FUNC(uint32_t, ulw, msa_ulw);
120    #define LW(psrc)  MSA_LOAD(psrc, msa_ulw)
121    #if (__mips == 64)
122      MSA_LOAD_FUNC(uint64_t, uld, msa_uld);
123      #define LD(psrc)  MSA_LOAD(psrc, msa_uld)
124    #else  // !(__mips == 64)
125      #define LD(psrc)  ((((uint64_t)MSA_LOAD(psrc + 4, msa_ulw)) << 32) | \
126                          MSA_LOAD(psrc, msa_ulw))
127    #endif  // (__mips == 64)
128  
129    MSA_STORE_FUNC(uint16_t, ush, msa_ush);
130    #define SH(val, pdst)  MSA_STORE(val, pdst, msa_ush)
131    MSA_STORE_FUNC(uint32_t, usw, msa_usw);
132    #define SW(val, pdst)  MSA_STORE(val, pdst, msa_usw)
133    #define SD(val, pdst) do {                                               \
134      uint8_t* const pdst_sd_m = (uint8_t*)(pdst);                           \
135      const uint32_t val0_m = (uint32_t)(val & 0x00000000FFFFFFFF);          \
136      const uint32_t val1_m = (uint32_t)((val >> 32) & 0x00000000FFFFFFFF);  \
137      SW(val0_m, pdst_sd_m);                                                 \
138      SW(val1_m, pdst_sd_m + 4);                                             \
139    } while (0)
140  #endif  // (__mips_isa_rev >= 6)
141  
142  /* Description : Load 4 words with stride
143   * Arguments   : Inputs  - psrc, stride
144   *               Outputs - out0, out1, out2, out3
145   * Details     : Load word in 'out0' from (psrc)
146   *               Load word in 'out1' from (psrc + stride)
147   *               Load word in 'out2' from (psrc + 2 * stride)
148   *               Load word in 'out3' from (psrc + 3 * stride)
149   */
150  #define LW4(psrc, stride, out0, out1, out2, out3) do {  \
151    const uint8_t* ptmp = (const uint8_t*)psrc;           \
152    out0 = LW(ptmp);                                      \
153    ptmp += stride;                                       \
154    out1 = LW(ptmp);                                      \
155    ptmp += stride;                                       \
156    out2 = LW(ptmp);                                      \
157    ptmp += stride;                                       \
158    out3 = LW(ptmp);                                      \
159  } while (0)
160  
161  /* Description : Store words with stride
162   * Arguments   : Inputs - in0, in1, in2, in3, pdst, stride
163   * Details     : Store word from 'in0' to (pdst)
164   *               Store word from 'in1' to (pdst + stride)
165   *               Store word from 'in2' to (pdst + 2 * stride)
166   *               Store word from 'in3' to (pdst + 3 * stride)
167   */
168  #define SW4(in0, in1, in2, in3, pdst, stride) do {  \
169    uint8_t* ptmp = (uint8_t*)pdst;                   \
170    SW(in0, ptmp);                                    \
171    ptmp += stride;                                   \
172    SW(in1, ptmp);                                    \
173    ptmp += stride;                                   \
174    SW(in2, ptmp);                                    \
175    ptmp += stride;                                   \
176    SW(in3, ptmp);                                    \
177  } while (0)
178  
179  #define SW3(in0, in1, in2, pdst, stride) do {  \
180    uint8_t* ptmp = (uint8_t*)pdst;              \
181    SW(in0, ptmp);                               \
182    ptmp += stride;                              \
183    SW(in1, ptmp);                               \
184    ptmp += stride;                              \
185    SW(in2, ptmp);                               \
186  } while (0)
187  
188  #define SW2(in0, in1, pdst, stride) do {  \
189    uint8_t* ptmp = (uint8_t*)pdst;         \
190    SW(in0, ptmp);                          \
191    ptmp += stride;                         \
192    SW(in1, ptmp);                          \
193  } while (0)
194  
195  /* Description : Store 4 double words with stride
196   * Arguments   : Inputs - in0, in1, in2, in3, pdst, stride
197   * Details     : Store double word from 'in0' to (pdst)
198   *               Store double word from 'in1' to (pdst + stride)
199   *               Store double word from 'in2' to (pdst + 2 * stride)
200   *               Store double word from 'in3' to (pdst + 3 * stride)
201   */
202  #define SD4(in0, in1, in2, in3, pdst, stride) do {  \
203    uint8_t* ptmp = (uint8_t*)pdst;                   \
204    SD(in0, ptmp);                                    \
205    ptmp += stride;                                   \
206    SD(in1, ptmp);                                    \
207    ptmp += stride;                                   \
208    SD(in2, ptmp);                                    \
209    ptmp += stride;                                   \
210    SD(in3, ptmp);                                    \
211  } while (0)
212  
213  /* Description : Load vectors with 16 byte elements with stride
214   * Arguments   : Inputs  - psrc, stride
215   *               Outputs - out0, out1
216   *               Return Type - as per RTYPE
217   * Details     : Load 16 byte elements in 'out0' from (psrc)
218   *               Load 16 byte elements in 'out1' from (psrc + stride)
219   */
220  #define LD_B2(RTYPE, psrc, stride, out0, out1) do {  \
221    out0 = LD_B(RTYPE, psrc);                          \
222    out1 = LD_B(RTYPE, psrc + stride);                 \
223  } while (0)
224  #define LD_UB2(...) LD_B2(v16u8, __VA_ARGS__)
225  #define LD_SB2(...) LD_B2(v16i8, __VA_ARGS__)
226  
227  #define LD_B3(RTYPE, psrc, stride, out0, out1, out2) do {  \
228    LD_B2(RTYPE, psrc, stride, out0, out1);                  \
229    out2 = LD_B(RTYPE, psrc + 2 * stride);                   \
230  } while (0)
231  #define LD_UB3(...) LD_B3(v16u8, __VA_ARGS__)
232  #define LD_SB3(...) LD_B3(v16i8, __VA_ARGS__)
233  
234  #define LD_B4(RTYPE, psrc, stride, out0, out1, out2, out3) do {  \
235    LD_B2(RTYPE, psrc, stride, out0, out1);                        \
236    LD_B2(RTYPE, psrc + 2 * stride , stride, out2, out3);          \
237  } while (0)
238  #define LD_UB4(...) LD_B4(v16u8, __VA_ARGS__)
239  #define LD_SB4(...) LD_B4(v16i8, __VA_ARGS__)
240  
241  #define LD_B8(RTYPE, psrc, stride,                                  \
242                out0, out1, out2, out3, out4, out5, out6, out7) do {  \
243    LD_B4(RTYPE, psrc, stride, out0, out1, out2, out3);               \
244    LD_B4(RTYPE, psrc + 4 * stride, stride, out4, out5, out6, out7);  \
245  } while (0)
246  #define LD_UB8(...) LD_B8(v16u8, __VA_ARGS__)
247  #define LD_SB8(...) LD_B8(v16i8, __VA_ARGS__)
248  
249  /* Description : Load vectors with 8 halfword elements with stride
250   * Arguments   : Inputs  - psrc, stride
251   *               Outputs - out0, out1
252   * Details     : Load 8 halfword elements in 'out0' from (psrc)
253   *               Load 8 halfword elements in 'out1' from (psrc + stride)
254   */
255  #define LD_H2(RTYPE, psrc, stride, out0, out1) do {  \
256    out0 = LD_H(RTYPE, psrc);                          \
257    out1 = LD_H(RTYPE, psrc + stride);                 \
258  } while (0)
259  #define LD_UH2(...) LD_H2(v8u16, __VA_ARGS__)
260  #define LD_SH2(...) LD_H2(v8i16, __VA_ARGS__)
261  
262  /* Description : Load vectors with 4 word elements with stride
263   * Arguments   : Inputs  - psrc, stride
264   *               Outputs - out0, out1, out2, out3
265   * Details     : Load 4 word elements in 'out0' from (psrc + 0 * stride)
266   *               Load 4 word elements in 'out1' from (psrc + 1 * stride)
267   *               Load 4 word elements in 'out2' from (psrc + 2 * stride)
268   *               Load 4 word elements in 'out3' from (psrc + 3 * stride)
269   */
270  #define LD_W2(RTYPE, psrc, stride, out0, out1) do {  \
271    out0 = LD_W(RTYPE, psrc);                          \
272    out1 = LD_W(RTYPE, psrc + stride);                 \
273  } while (0)
274  #define LD_UW2(...) LD_W2(v4u32, __VA_ARGS__)
275  #define LD_SW2(...) LD_W2(v4i32, __VA_ARGS__)
276  
277  #define LD_W3(RTYPE, psrc, stride, out0, out1, out2) do {  \
278    LD_W2(RTYPE, psrc, stride, out0, out1);                  \
279    out2 = LD_W(RTYPE, psrc + 2 * stride);                   \
280  } while (0)
281  #define LD_UW3(...) LD_W3(v4u32, __VA_ARGS__)
282  #define LD_SW3(...) LD_W3(v4i32, __VA_ARGS__)
283  
284  #define LD_W4(RTYPE, psrc, stride, out0, out1, out2, out3) do {  \
285    LD_W2(RTYPE, psrc, stride, out0, out1);                        \
286    LD_W2(RTYPE, psrc + 2 * stride, stride, out2, out3);           \
287  } while (0)
288  #define LD_UW4(...) LD_W4(v4u32, __VA_ARGS__)
289  #define LD_SW4(...) LD_W4(v4i32, __VA_ARGS__)
290  
291  /* Description : Store vectors of 16 byte elements with stride
292   * Arguments   : Inputs - in0, in1, pdst, stride
293   * Details     : Store 16 byte elements from 'in0' to (pdst)
294   *               Store 16 byte elements from 'in1' to (pdst + stride)
295   */
296  #define ST_B2(RTYPE, in0, in1, pdst, stride) do {  \
297    ST_B(RTYPE, in0, pdst);                          \
298    ST_B(RTYPE, in1, pdst + stride);                 \
299  } while (0)
300  #define ST_UB2(...) ST_B2(v16u8, __VA_ARGS__)
301  #define ST_SB2(...) ST_B2(v16i8, __VA_ARGS__)
302  
303  #define ST_B4(RTYPE, in0, in1, in2, in3, pdst, stride) do {  \
304    ST_B2(RTYPE, in0, in1, pdst, stride);                      \
305    ST_B2(RTYPE, in2, in3, pdst + 2 * stride, stride);         \
306  } while (0)
307  #define ST_UB4(...) ST_B4(v16u8, __VA_ARGS__)
308  #define ST_SB4(...) ST_B4(v16i8, __VA_ARGS__)
309  
310  #define ST_B8(RTYPE, in0, in1, in2, in3, in4, in5, in6, in7,    \
311                pdst, stride) do {                                \
312    ST_B4(RTYPE, in0, in1, in2, in3, pdst, stride);               \
313    ST_B4(RTYPE, in4, in5, in6, in7, pdst + 4 * stride, stride);  \
314  } while (0)
315  #define ST_UB8(...) ST_B8(v16u8, __VA_ARGS__)
316  
317  /* Description : Store vectors of 4 word elements with stride
318   * Arguments   : Inputs - in0, in1, in2, in3, pdst, stride
319   * Details     : Store 4 word elements from 'in0' to (pdst + 0 * stride)
320   *               Store 4 word elements from 'in1' to (pdst + 1 * stride)
321   *               Store 4 word elements from 'in2' to (pdst + 2 * stride)
322   *               Store 4 word elements from 'in3' to (pdst + 3 * stride)
323   */
324  #define ST_W2(RTYPE, in0, in1, pdst, stride) do {  \
325    ST_W(RTYPE, in0, pdst);                          \
326    ST_W(RTYPE, in1, pdst + stride);                 \
327  } while (0)
328  #define ST_UW2(...) ST_W2(v4u32, __VA_ARGS__)
329  #define ST_SW2(...) ST_W2(v4i32, __VA_ARGS__)
330  
331  #define ST_W3(RTYPE, in0, in1, in2, pdst, stride) do {  \
332    ST_W2(RTYPE, in0, in1, pdst, stride);                 \
333    ST_W(RTYPE, in2, pdst + 2 * stride);                  \
334  } while (0)
335  #define ST_UW3(...) ST_W3(v4u32, __VA_ARGS__)
336  #define ST_SW3(...) ST_W3(v4i32, __VA_ARGS__)
337  
338  #define ST_W4(RTYPE, in0, in1, in2, in3, pdst, stride) do {  \
339    ST_W2(RTYPE, in0, in1, pdst, stride);                      \
340    ST_W2(RTYPE, in2, in3, pdst + 2 * stride, stride);         \
341  } while (0)
342  #define ST_UW4(...) ST_W4(v4u32, __VA_ARGS__)
343  #define ST_SW4(...) ST_W4(v4i32, __VA_ARGS__)
344  
345  /* Description : Store vectors of 8 halfword elements with stride
346   * Arguments   : Inputs - in0, in1, pdst, stride
347   * Details     : Store 8 halfword elements from 'in0' to (pdst)
348   *               Store 8 halfword elements from 'in1' to (pdst + stride)
349   */
350  #define ST_H2(RTYPE, in0, in1, pdst, stride) do {  \
351    ST_H(RTYPE, in0, pdst);                          \
352    ST_H(RTYPE, in1, pdst + stride);                 \
353  } while (0)
354  #define ST_UH2(...) ST_H2(v8u16, __VA_ARGS__)
355  #define ST_SH2(...) ST_H2(v8i16, __VA_ARGS__)
356  
357  /* Description : Store 2x4 byte block to destination memory from input vector
358   * Arguments   : Inputs - in, stidx, pdst, stride
359   * Details     : Index 'stidx' halfword element from 'in' vector is copied to
360   *               the GP register and stored to (pdst)
361   *               Index 'stidx+1' halfword element from 'in' vector is copied to
362   *               the GP register and stored to (pdst + stride)
363   *               Index 'stidx+2' halfword element from 'in' vector is copied to
364   *               the GP register and stored to (pdst + 2 * stride)
365   *               Index 'stidx+3' halfword element from 'in' vector is copied to
366   *               the GP register and stored to (pdst + 3 * stride)
367   */
368  #define ST2x4_UB(in, stidx, pdst, stride) do {                   \
369    uint8_t* pblk_2x4_m = (uint8_t*)pdst;                          \
370    const uint16_t out0_m = __msa_copy_s_h((v8i16)in, stidx);      \
371    const uint16_t out1_m = __msa_copy_s_h((v8i16)in, stidx + 1);  \
372    const uint16_t out2_m = __msa_copy_s_h((v8i16)in, stidx + 2);  \
373    const uint16_t out3_m = __msa_copy_s_h((v8i16)in, stidx + 3);  \
374    SH(out0_m, pblk_2x4_m);                                        \
375    pblk_2x4_m += stride;                                          \
376    SH(out1_m, pblk_2x4_m);                                        \
377    pblk_2x4_m += stride;                                          \
378    SH(out2_m, pblk_2x4_m);                                        \
379    pblk_2x4_m += stride;                                          \
380    SH(out3_m, pblk_2x4_m);                                        \
381  } while (0)
382  
383  /* Description : Store 4x4 byte block to destination memory from input vector
384   * Arguments   : Inputs - in0, in1, pdst, stride
385   * Details     : 'Idx0' word element from input vector 'in0' is copied to the
386   *               GP register and stored to (pdst)
387   *               'Idx1' word element from input vector 'in0' is copied to the
388   *               GP register and stored to (pdst + stride)
389   *               'Idx2' word element from input vector 'in0' is copied to the
390   *               GP register and stored to (pdst + 2 * stride)
391   *               'Idx3' word element from input vector 'in0' is copied to the
392   *               GP register and stored to (pdst + 3 * stride)
393   */
394  #define ST4x4_UB(in0, in1, idx0, idx1, idx2, idx3, pdst, stride) do {  \
395    uint8_t* const pblk_4x4_m = (uint8_t*)pdst;                          \
396    const uint32_t out0_m = __msa_copy_s_w((v4i32)in0, idx0);            \
397    const uint32_t out1_m = __msa_copy_s_w((v4i32)in0, idx1);            \
398    const uint32_t out2_m = __msa_copy_s_w((v4i32)in1, idx2);            \
399    const uint32_t out3_m = __msa_copy_s_w((v4i32)in1, idx3);            \
400    SW4(out0_m, out1_m, out2_m, out3_m, pblk_4x4_m, stride);             \
401  } while (0)
402  
403  #define ST4x8_UB(in0, in1, pdst, stride) do {                     \
404    uint8_t* const pblk_4x8 = (uint8_t*)pdst;                       \
405    ST4x4_UB(in0, in0, 0, 1, 2, 3, pblk_4x8, stride);               \
406    ST4x4_UB(in1, in1, 0, 1, 2, 3, pblk_4x8 + 4 * stride, stride);  \
407  } while (0)
408  
409  /* Description : Immediate number of elements to slide
410   * Arguments   : Inputs  - in0, in1, slide_val
411   *               Outputs - out
412   *               Return Type - as per RTYPE
413   * Details     : Byte elements from 'in1' vector are slid into 'in0' by
414   *               value specified in the 'slide_val'
415   */
416  #define SLDI_B(RTYPE, in0, in1, slide_val)                      \
417          (RTYPE)__msa_sldi_b((v16i8)in0, (v16i8)in1, slide_val)  \
418  
419  #define SLDI_UB(...) SLDI_B(v16u8, __VA_ARGS__)
420  #define SLDI_SB(...) SLDI_B(v16i8, __VA_ARGS__)
421  #define SLDI_SH(...) SLDI_B(v8i16, __VA_ARGS__)
422  
423  /* Description : Shuffle byte vector elements as per mask vector
424   * Arguments   : Inputs  - in0, in1, in2, in3, mask0, mask1
425   *               Outputs - out0, out1
426   *               Return Type - as per RTYPE
427   * Details     : Byte elements from 'in0' & 'in1' are copied selectively to
428   *               'out0' as per control vector 'mask0'
429   */
430  #define VSHF_B(RTYPE, in0, in1, mask)                              \
431          (RTYPE)__msa_vshf_b((v16i8)mask, (v16i8)in1, (v16i8)in0)
432  
433  #define VSHF_UB(...) VSHF_B(v16u8, __VA_ARGS__)
434  #define VSHF_SB(...) VSHF_B(v16i8, __VA_ARGS__)
435  #define VSHF_UH(...) VSHF_B(v8u16, __VA_ARGS__)
436  #define VSHF_SH(...) VSHF_B(v8i16, __VA_ARGS__)
437  
438  #define VSHF_B2(RTYPE, in0, in1, in2, in3, mask0, mask1, out0, out1) do {  \
439    out0 = VSHF_B(RTYPE, in0, in1, mask0);                                   \
440    out1 = VSHF_B(RTYPE, in2, in3, mask1);                                   \
441  } while (0)
442  #define VSHF_B2_UB(...) VSHF_B2(v16u8, __VA_ARGS__)
443  #define VSHF_B2_SB(...) VSHF_B2(v16i8, __VA_ARGS__)
444  #define VSHF_B2_UH(...) VSHF_B2(v8u16, __VA_ARGS__)
445  #define VSHF_B2_SH(...) VSHF_B2(v8i16, __VA_ARGS__)
446  
447  /* Description : Shuffle halfword vector elements as per mask vector
448   * Arguments   : Inputs  - in0, in1, in2, in3, mask0, mask1
449   *               Outputs - out0, out1
450   *               Return Type - as per RTYPE
451   * Details     : halfword elements from 'in0' & 'in1' are copied selectively to
452   *               'out0' as per control vector 'mask0'
453   */
454  #define VSHF_H2(RTYPE, in0, in1, in2, in3, mask0, mask1, out0, out1) do {  \
455    out0 = (RTYPE)__msa_vshf_h((v8i16)mask0, (v8i16)in1, (v8i16)in0);        \
456    out1 = (RTYPE)__msa_vshf_h((v8i16)mask1, (v8i16)in3, (v8i16)in2);        \
457  } while (0)
458  #define VSHF_H2_UH(...) VSHF_H2(v8u16, __VA_ARGS__)
459  #define VSHF_H2_SH(...) VSHF_H2(v8i16, __VA_ARGS__)
460  
461  /* Description : Dot product of byte vector elements
462   * Arguments   : Inputs  - mult0, mult1, cnst0, cnst1
463   *               Outputs - out0, out1
464   *               Return Type - as per RTYPE
465   * Details     : Signed byte elements from 'mult0' are multiplied with
466   *               signed byte elements from 'cnst0' producing a result
467   *               twice the size of input i.e. signed halfword.
468   *               The multiplication result of adjacent odd-even elements
469   *               are added together and written to the 'out0' vector
470  */
471  #define DOTP_SB2(RTYPE, mult0, mult1, cnst0, cnst1, out0, out1) do {  \
472    out0 = (RTYPE)__msa_dotp_s_h((v16i8)mult0, (v16i8)cnst0);           \
473    out1 = (RTYPE)__msa_dotp_s_h((v16i8)mult1, (v16i8)cnst1);           \
474  } while (0)
475  #define DOTP_SB2_SH(...) DOTP_SB2(v8i16, __VA_ARGS__)
476  
477  /* Description : Dot product of halfword vector elements
478   * Arguments   : Inputs  - mult0, mult1, cnst0, cnst1
479   *               Outputs - out0, out1
480   *               Return Type - as per RTYPE
481   * Details     : Signed halfword elements from 'mult0' are multiplied with
482   *               signed halfword elements from 'cnst0' producing a result
483   *               twice the size of input i.e. signed word.
484   *               The multiplication result of adjacent odd-even elements
485   *               are added together and written to the 'out0' vector
486   */
487  #define DOTP_SH2(RTYPE, mult0, mult1, cnst0, cnst1, out0, out1) do {  \
488    out0 = (RTYPE)__msa_dotp_s_w((v8i16)mult0, (v8i16)cnst0);           \
489    out1 = (RTYPE)__msa_dotp_s_w((v8i16)mult1, (v8i16)cnst1);           \
490  } while (0)
491  #define DOTP_SH2_SW(...) DOTP_SH2(v4i32, __VA_ARGS__)
492  
493  /* Description : Dot product of unsigned word vector elements
494   * Arguments   : Inputs  - mult0, mult1, cnst0, cnst1
495   *               Outputs - out0, out1
496   *               Return Type - as per RTYPE
497   * Details     : Unsigned word elements from 'mult0' are multiplied with
498   *               unsigned word elements from 'cnst0' producing a result
499   *               twice the size of input i.e. unsigned double word.
500   *               The multiplication result of adjacent odd-even elements
501   *               are added together and written to the 'out0' vector
502   */
503  #define DOTP_UW2(RTYPE, mult0, mult1, cnst0, cnst1, out0, out1) do {  \
504    out0 = (RTYPE)__msa_dotp_u_d((v4u32)mult0, (v4u32)cnst0);           \
505    out1 = (RTYPE)__msa_dotp_u_d((v4u32)mult1, (v4u32)cnst1);           \
506  } while (0)
507  #define DOTP_UW2_UD(...) DOTP_UW2(v2u64, __VA_ARGS__)
508  
509  /* Description : Dot product & addition of halfword vector elements
510   * Arguments   : Inputs  - mult0, mult1, cnst0, cnst1
511   *               Outputs - out0, out1
512   *               Return Type - as per RTYPE
513   * Details     : Signed halfword elements from 'mult0' are multiplied with
514   *               signed halfword elements from 'cnst0' producing a result
515   *               twice the size of input i.e. signed word.
516   *               The multiplication result of adjacent odd-even elements
517   *               are added to the 'out0' vector
518   */
519  #define DPADD_SH2(RTYPE, mult0, mult1, cnst0, cnst1, out0, out1) do {      \
520    out0 = (RTYPE)__msa_dpadd_s_w((v4i32)out0, (v8i16)mult0, (v8i16)cnst0);  \
521    out1 = (RTYPE)__msa_dpadd_s_w((v4i32)out1, (v8i16)mult1, (v8i16)cnst1);  \
522  } while (0)
523  #define DPADD_SH2_SW(...) DPADD_SH2(v4i32, __VA_ARGS__)
524  
525  /* Description : Clips all signed halfword elements of input vector
526   *               between 0 & 255
527   * Arguments   : Input/output  - val
528   *               Return Type - signed halfword
529   */
530  #define CLIP_SH_0_255(val) do {                   \
531    const v8i16 max_m = __msa_ldi_h(255);           \
532    val = __msa_maxi_s_h((v8i16)val, 0);            \
533    val = __msa_min_s_h(max_m, (v8i16)val);         \
534  } while (0)
535  
536  #define CLIP_SH2_0_255(in0, in1) do {  \
537    CLIP_SH_0_255(in0);                  \
538    CLIP_SH_0_255(in1);                  \
539  } while (0)
540  
541  #define CLIP_SH4_0_255(in0, in1, in2, in3) do {  \
542    CLIP_SH2_0_255(in0, in1);                      \
543    CLIP_SH2_0_255(in2, in3);                      \
544  } while (0)
545  
546  /* Description : Clips all unsigned halfword elements of input vector
547   *               between 0 & 255
548   * Arguments   : Input  - in
549   *               Output - out_m
550   *               Return Type - unsigned halfword
551   */
552  #define CLIP_UH_0_255(in) do {                    \
553    const v8u16 max_m = (v8u16)__msa_ldi_h(255);    \
554    in = __msa_maxi_u_h((v8u16) in, 0);             \
555    in = __msa_min_u_h((v8u16) max_m, (v8u16) in);  \
556  } while (0)
557  
558  #define CLIP_UH2_0_255(in0, in1) do {  \
559    CLIP_UH_0_255(in0);                  \
560    CLIP_UH_0_255(in1);                  \
561  } while (0)
562  
563  /* Description : Clips all signed word elements of input vector
564   *               between 0 & 255
565   * Arguments   : Input/output  - val
566   *               Return Type - signed word
567   */
568  #define CLIP_SW_0_255(val) do {                   \
569    const v4i32 max_m = __msa_ldi_w(255);           \
570    val = __msa_maxi_s_w((v4i32)val, 0);            \
571    val = __msa_min_s_w(max_m, (v4i32)val);         \
572  } while (0)
573  
574  #define CLIP_SW4_0_255(in0, in1, in2, in3) do {   \
575    CLIP_SW_0_255(in0);                             \
576    CLIP_SW_0_255(in1);                             \
577    CLIP_SW_0_255(in2);                             \
578    CLIP_SW_0_255(in3);                             \
579  } while (0)
580  
581  /* Description : Horizontal addition of 4 signed word elements of input vector
582   * Arguments   : Input  - in       (signed word vector)
583   *               Output - sum_m    (i32 sum)
584   *               Return Type - signed word (GP)
585   * Details     : 4 signed word elements of 'in' vector are added together and
586   *               the resulting integer sum is returned
587   */
func_hadd_sw_s32(v4i32 in)588  static WEBP_INLINE int32_t func_hadd_sw_s32(v4i32 in) {
589    const v2i64 res0_m = __msa_hadd_s_d((v4i32)in, (v4i32)in);
590    const v2i64 res1_m = __msa_splati_d(res0_m, 1);
591    const v2i64 out = res0_m + res1_m;
592    int32_t sum_m = __msa_copy_s_w((v4i32)out, 0);
593    return sum_m;
594  }
595  #define HADD_SW_S32(in) func_hadd_sw_s32(in)
596  
597  /* Description : Horizontal addition of 8 signed halfword elements
598   * Arguments   : Input  - in       (signed halfword vector)
599   *               Output - sum_m    (s32 sum)
600   *               Return Type - signed word
601   * Details     : 8 signed halfword elements of input vector are added
602   *               together and the resulting integer sum is returned
603   */
func_hadd_sh_s32(v8i16 in)604  static WEBP_INLINE int32_t func_hadd_sh_s32(v8i16 in) {
605    const v4i32 res = __msa_hadd_s_w(in, in);
606    const v2i64 res0 = __msa_hadd_s_d(res, res);
607    const v2i64 res1 = __msa_splati_d(res0, 1);
608    const v2i64 res2 = res0 + res1;
609    const int32_t sum_m = __msa_copy_s_w((v4i32)res2, 0);
610    return sum_m;
611  }
612  #define HADD_SH_S32(in) func_hadd_sh_s32(in)
613  
614  /* Description : Horizontal addition of 8 unsigned halfword elements
615   * Arguments   : Input  - in       (unsigned halfword vector)
616   *               Output - sum_m    (u32 sum)
617   *               Return Type - unsigned word
618   * Details     : 8 unsigned halfword elements of input vector are added
619   *               together and the resulting integer sum is returned
620   */
func_hadd_uh_u32(v8u16 in)621  static WEBP_INLINE uint32_t func_hadd_uh_u32(v8u16 in) {
622    uint32_t sum_m;
623    const v4u32 res_m = __msa_hadd_u_w(in, in);
624    v2u64 res0_m = __msa_hadd_u_d(res_m, res_m);
625    v2u64 res1_m = (v2u64)__msa_splati_d((v2i64)res0_m, 1);
626    res0_m = res0_m + res1_m;
627    sum_m = __msa_copy_s_w((v4i32)res0_m, 0);
628    return sum_m;
629  }
630  #define HADD_UH_U32(in) func_hadd_uh_u32(in)
631  
632  /* Description : Horizontal addition of signed half word vector elements
633     Arguments   : Inputs  - in0, in1
634                   Outputs - out0, out1
635                   Return Type - as per RTYPE
636     Details     : Each signed odd half word element from 'in0' is added to
637                   even signed half word element from 'in0' (pairwise) and the
638                   halfword result is written in 'out0'
639  */
640  #define HADD_SH2(RTYPE, in0, in1, out0, out1) do {       \
641    out0 = (RTYPE)__msa_hadd_s_w((v8i16)in0, (v8i16)in0);  \
642    out1 = (RTYPE)__msa_hadd_s_w((v8i16)in1, (v8i16)in1);  \
643  } while (0)
644  #define HADD_SH2_SW(...) HADD_SH2(v4i32, __VA_ARGS__)
645  
646  #define HADD_SH4(RTYPE, in0, in1, in2, in3, out0, out1, out2, out3) do {  \
647    HADD_SH2(RTYPE, in0, in1, out0, out1);                                  \
648    HADD_SH2(RTYPE, in2, in3, out2, out3);                                  \
649  } while (0)
650  #define HADD_SH4_SW(...) HADD_SH4(v4i32, __VA_ARGS__)
651  
652  /* Description : Horizontal subtraction of unsigned byte vector elements
653   * Arguments   : Inputs  - in0, in1
654   *               Outputs - out0, out1
655   *               Return Type - as per RTYPE
656   * Details     : Each unsigned odd byte element from 'in0' is subtracted from
657   *               even unsigned byte element from 'in0' (pairwise) and the
658   *               halfword result is written to 'out0'
659   */
660  #define HSUB_UB2(RTYPE, in0, in1, out0, out1) do {       \
661    out0 = (RTYPE)__msa_hsub_u_h((v16u8)in0, (v16u8)in0);  \
662    out1 = (RTYPE)__msa_hsub_u_h((v16u8)in1, (v16u8)in1);  \
663  } while (0)
664  #define HSUB_UB2_UH(...) HSUB_UB2(v8u16, __VA_ARGS__)
665  #define HSUB_UB2_SH(...) HSUB_UB2(v8i16, __VA_ARGS__)
666  #define HSUB_UB2_SW(...) HSUB_UB2(v4i32, __VA_ARGS__)
667  
668  /* Description : Set element n input vector to GPR value
669   * Arguments   : Inputs - in0, in1, in2, in3
670   *               Output - out
671   *               Return Type - as per RTYPE
672   * Details     : Set element 0 in vector 'out' to value specified in 'in0'
673   */
674  #define INSERT_W2(RTYPE, in0, in1, out) do {        \
675    out = (RTYPE)__msa_insert_w((v4i32)out, 0, in0);  \
676    out = (RTYPE)__msa_insert_w((v4i32)out, 1, in1);  \
677  } while (0)
678  #define INSERT_W2_UB(...) INSERT_W2(v16u8, __VA_ARGS__)
679  #define INSERT_W2_SB(...) INSERT_W2(v16i8, __VA_ARGS__)
680  
681  #define INSERT_W4(RTYPE, in0, in1, in2, in3, out) do {  \
682    out = (RTYPE)__msa_insert_w((v4i32)out, 0, in0);      \
683    out = (RTYPE)__msa_insert_w((v4i32)out, 1, in1);      \
684    out = (RTYPE)__msa_insert_w((v4i32)out, 2, in2);      \
685    out = (RTYPE)__msa_insert_w((v4i32)out, 3, in3);      \
686  } while (0)
687  #define INSERT_W4_UB(...) INSERT_W4(v16u8, __VA_ARGS__)
688  #define INSERT_W4_SB(...) INSERT_W4(v16i8, __VA_ARGS__)
689  #define INSERT_W4_SW(...) INSERT_W4(v4i32, __VA_ARGS__)
690  
691  /* Description : Set element n of double word input vector to GPR value
692   * Arguments   : Inputs - in0, in1
693   *               Output - out
694   *               Return Type - as per RTYPE
695   * Details     : Set element 0 in vector 'out' to GPR value specified in 'in0'
696   *               Set element 1 in vector 'out' to GPR value specified in 'in1'
697   */
698  #define INSERT_D2(RTYPE, in0, in1, out) do {        \
699    out = (RTYPE)__msa_insert_d((v2i64)out, 0, in0);  \
700    out = (RTYPE)__msa_insert_d((v2i64)out, 1, in1);  \
701  } while (0)
702  #define INSERT_D2_UB(...) INSERT_D2(v16u8, __VA_ARGS__)
703  #define INSERT_D2_SB(...) INSERT_D2(v16i8, __VA_ARGS__)
704  
705  /* Description : Interleave even byte elements from vectors
706   * Arguments   : Inputs  - in0, in1, in2, in3
707   *               Outputs - out0, out1
708   *               Return Type - as per RTYPE
709   * Details     : Even byte elements of 'in0' and 'in1' are interleaved
710   *               and written to 'out0'
711   */
712  #define ILVEV_B2(RTYPE, in0, in1, in2, in3, out0, out1) do {  \
713    out0 = (RTYPE)__msa_ilvev_b((v16i8)in1, (v16i8)in0);        \
714    out1 = (RTYPE)__msa_ilvev_b((v16i8)in3, (v16i8)in2);        \
715  } while (0)
716  #define ILVEV_B2_UB(...) ILVEV_B2(v16u8, __VA_ARGS__)
717  #define ILVEV_B2_SB(...) ILVEV_B2(v16i8, __VA_ARGS__)
718  #define ILVEV_B2_UH(...) ILVEV_B2(v8u16, __VA_ARGS__)
719  #define ILVEV_B2_SH(...) ILVEV_B2(v8i16, __VA_ARGS__)
720  #define ILVEV_B2_SD(...) ILVEV_B2(v2i64, __VA_ARGS__)
721  
722  /* Description : Interleave odd byte elements from vectors
723   * Arguments   : Inputs  - in0, in1, in2, in3
724   *               Outputs - out0, out1
725   *               Return Type - as per RTYPE
726   * Details     : Odd byte elements of 'in0' and 'in1' are interleaved
727   *               and written to 'out0'
728   */
729  #define ILVOD_B2(RTYPE, in0, in1, in2, in3, out0, out1) do {  \
730    out0 = (RTYPE)__msa_ilvod_b((v16i8)in1, (v16i8)in0);        \
731    out1 = (RTYPE)__msa_ilvod_b((v16i8)in3, (v16i8)in2);        \
732  } while (0)
733  #define ILVOD_B2_UB(...) ILVOD_B2(v16u8, __VA_ARGS__)
734  #define ILVOD_B2_SB(...) ILVOD_B2(v16i8, __VA_ARGS__)
735  #define ILVOD_B2_UH(...) ILVOD_B2(v8u16, __VA_ARGS__)
736  #define ILVOD_B2_SH(...) ILVOD_B2(v8i16, __VA_ARGS__)
737  #define ILVOD_B2_SD(...) ILVOD_B2(v2i64, __VA_ARGS__)
738  
739  /* Description : Interleave even halfword elements from vectors
740   * Arguments   : Inputs  - in0, in1, in2, in3
741   *               Outputs - out0, out1
742   *               Return Type - as per RTYPE
743   * Details     : Even halfword elements of 'in0' and 'in1' are interleaved
744   *               and written to 'out0'
745   */
746  #define ILVEV_H2(RTYPE, in0, in1, in2, in3, out0, out1) do {  \
747    out0 = (RTYPE)__msa_ilvev_h((v8i16)in1, (v8i16)in0);        \
748    out1 = (RTYPE)__msa_ilvev_h((v8i16)in3, (v8i16)in2);        \
749  } while (0)
750  #define ILVEV_H2_UB(...) ILVEV_H2(v16u8, __VA_ARGS__)
751  #define ILVEV_H2_UH(...) ILVEV_H2(v8u16, __VA_ARGS__)
752  #define ILVEV_H2_SH(...) ILVEV_H2(v8i16, __VA_ARGS__)
753  #define ILVEV_H2_SW(...) ILVEV_H2(v4i32, __VA_ARGS__)
754  
755  /* Description : Interleave odd halfword elements from vectors
756   * Arguments   : Inputs  - in0, in1, in2, in3
757   *               Outputs - out0, out1
758   *               Return Type - as per RTYPE
759   * Details     : Odd halfword elements of 'in0' and 'in1' are interleaved
760   *               and written to 'out0'
761   */
762  #define ILVOD_H2(RTYPE, in0, in1, in2, in3, out0, out1) do {  \
763    out0 = (RTYPE)__msa_ilvod_h((v8i16)in1, (v8i16)in0);        \
764    out1 = (RTYPE)__msa_ilvod_h((v8i16)in3, (v8i16)in2);        \
765  } while (0)
766  #define ILVOD_H2_UB(...) ILVOD_H2(v16u8, __VA_ARGS__)
767  #define ILVOD_H2_UH(...) ILVOD_H2(v8u16, __VA_ARGS__)
768  #define ILVOD_H2_SH(...) ILVOD_H2(v8i16, __VA_ARGS__)
769  #define ILVOD_H2_SW(...) ILVOD_H2(v4i32, __VA_ARGS__)
770  
771  /* Description : Interleave even word elements from vectors
772   * Arguments   : Inputs  - in0, in1, in2, in3
773   *               Outputs - out0, out1
774   *               Return Type - as per RTYPE
775   * Details     : Even word elements of 'in0' and 'in1' are interleaved
776   *               and written to 'out0'
777   */
778  #define ILVEV_W2(RTYPE, in0, in1, in2, in3, out0, out1) do {  \
779    out0 = (RTYPE)__msa_ilvev_w((v4i32)in1, (v4i32)in0);        \
780    out1 = (RTYPE)__msa_ilvev_w((v4i32)in3, (v4i32)in2);        \
781  } while (0)
782  #define ILVEV_W2_UB(...) ILVEV_W2(v16u8, __VA_ARGS__)
783  #define ILVEV_W2_SB(...) ILVEV_W2(v16i8, __VA_ARGS__)
784  #define ILVEV_W2_UH(...) ILVEV_W2(v8u16, __VA_ARGS__)
785  #define ILVEV_W2_SD(...) ILVEV_W2(v2i64, __VA_ARGS__)
786  
787  /* Description : Interleave even-odd word elements from vectors
788   * Arguments   : Inputs  - in0, in1, in2, in3
789   *               Outputs - out0, out1
790   *               Return Type - as per RTYPE
791   * Details     : Even word elements of 'in0' and 'in1' are interleaved
792   *               and written to 'out0'
793   *               Odd word elements of 'in2' and 'in3' are interleaved
794   *               and written to 'out1'
795   */
796  #define ILVEVOD_W2(RTYPE, in0, in1, in2, in3, out0, out1) do {  \
797    out0 = (RTYPE)__msa_ilvev_w((v4i32)in1, (v4i32)in0);          \
798    out1 = (RTYPE)__msa_ilvod_w((v4i32)in3, (v4i32)in2);          \
799  } while (0)
800  #define ILVEVOD_W2_UB(...) ILVEVOD_W2(v16u8, __VA_ARGS__)
801  #define ILVEVOD_W2_UH(...) ILVEVOD_W2(v8u16, __VA_ARGS__)
802  #define ILVEVOD_W2_SH(...) ILVEVOD_W2(v8i16, __VA_ARGS__)
803  #define ILVEVOD_W2_SW(...) ILVEVOD_W2(v4i32, __VA_ARGS__)
804  
805  /* Description : Interleave even-odd half-word elements from vectors
806   * Arguments   : Inputs  - in0, in1, in2, in3
807   *               Outputs - out0, out1
808   *               Return Type - as per RTYPE
809   * Details     : Even half-word elements of 'in0' and 'in1' are interleaved
810   *               and written to 'out0'
811   *               Odd half-word elements of 'in2' and 'in3' are interleaved
812   *               and written to 'out1'
813   */
814  #define ILVEVOD_H2(RTYPE, in0, in1, in2, in3, out0, out1) do {  \
815    out0 = (RTYPE)__msa_ilvev_h((v8i16)in1, (v8i16)in0);          \
816    out1 = (RTYPE)__msa_ilvod_h((v8i16)in3, (v8i16)in2);          \
817  } while (0)
818  #define ILVEVOD_H2_UB(...) ILVEVOD_H2(v16u8, __VA_ARGS__)
819  #define ILVEVOD_H2_UH(...) ILVEVOD_H2(v8u16, __VA_ARGS__)
820  #define ILVEVOD_H2_SH(...) ILVEVOD_H2(v8i16, __VA_ARGS__)
821  #define ILVEVOD_H2_SW(...) ILVEVOD_H2(v4i32, __VA_ARGS__)
822  
823  /* Description : Interleave even double word elements from vectors
824   * Arguments   : Inputs  - in0, in1, in2, in3
825   *               Outputs - out0, out1
826   *               Return Type - as per RTYPE
827   * Details     : Even double word elements of 'in0' and 'in1' are interleaved
828   *               and written to 'out0'
829   */
830  #define ILVEV_D2(RTYPE, in0, in1, in2, in3, out0, out1) do {  \
831    out0 = (RTYPE)__msa_ilvev_d((v2i64)in1, (v2i64)in0);        \
832    out1 = (RTYPE)__msa_ilvev_d((v2i64)in3, (v2i64)in2);        \
833  } while (0)
834  #define ILVEV_D2_UB(...) ILVEV_D2(v16u8, __VA_ARGS__)
835  #define ILVEV_D2_SB(...) ILVEV_D2(v16i8, __VA_ARGS__)
836  #define ILVEV_D2_SW(...) ILVEV_D2(v4i32, __VA_ARGS__)
837  #define ILVEV_D2_SD(...) ILVEV_D2(v2i64, __VA_ARGS__)
838  
839  /* Description : Interleave left half of byte elements from vectors
840   * Arguments   : Inputs  - in0, in1, in2, in3
841   *               Outputs - out0, out1
842   *               Return Type - as per RTYPE
843   * Details     : Left half of byte elements of 'in0' and 'in1' are interleaved
844   *               and written to 'out0'.
845   */
846  #define ILVL_B2(RTYPE, in0, in1, in2, in3, out0, out1) do {  \
847    out0 = (RTYPE)__msa_ilvl_b((v16i8)in0, (v16i8)in1);        \
848    out1 = (RTYPE)__msa_ilvl_b((v16i8)in2, (v16i8)in3);        \
849  } while (0)
850  #define ILVL_B2_UB(...) ILVL_B2(v16u8, __VA_ARGS__)
851  #define ILVL_B2_SB(...) ILVL_B2(v16i8, __VA_ARGS__)
852  #define ILVL_B2_UH(...) ILVL_B2(v8u16, __VA_ARGS__)
853  #define ILVL_B2_SH(...) ILVL_B2(v8i16, __VA_ARGS__)
854  #define ILVL_B2_SW(...) ILVL_B2(v4i32, __VA_ARGS__)
855  
856  /* Description : Interleave right half of byte elements from vectors
857   * Arguments   : Inputs  - in0, in1, in2, in3
858   *               Outputs - out0, out1
859   *               Return Type - as per RTYPE
860   * Details     : Right half of byte elements of 'in0' and 'in1' are interleaved
861   *               and written to out0.
862   */
863  #define ILVR_B2(RTYPE, in0, in1, in2, in3, out0, out1) do {  \
864    out0 = (RTYPE)__msa_ilvr_b((v16i8)in0, (v16i8)in1);        \
865    out1 = (RTYPE)__msa_ilvr_b((v16i8)in2, (v16i8)in3);        \
866  } while (0)
867  #define ILVR_B2_UB(...) ILVR_B2(v16u8, __VA_ARGS__)
868  #define ILVR_B2_SB(...) ILVR_B2(v16i8, __VA_ARGS__)
869  #define ILVR_B2_UH(...) ILVR_B2(v8u16, __VA_ARGS__)
870  #define ILVR_B2_SH(...) ILVR_B2(v8i16, __VA_ARGS__)
871  #define ILVR_B2_SW(...) ILVR_B2(v4i32, __VA_ARGS__)
872  
873  #define ILVR_B4(RTYPE, in0, in1, in2, in3, in4, in5, in6, in7,  \
874                  out0, out1, out2, out3) do {                    \
875    ILVR_B2(RTYPE, in0, in1, in2, in3, out0, out1);               \
876    ILVR_B2(RTYPE, in4, in5, in6, in7, out2, out3);               \
877  } while (0)
878  #define ILVR_B4_UB(...) ILVR_B4(v16u8, __VA_ARGS__)
879  #define ILVR_B4_SB(...) ILVR_B4(v16i8, __VA_ARGS__)
880  #define ILVR_B4_UH(...) ILVR_B4(v8u16, __VA_ARGS__)
881  #define ILVR_B4_SH(...) ILVR_B4(v8i16, __VA_ARGS__)
882  #define ILVR_B4_SW(...) ILVR_B4(v4i32, __VA_ARGS__)
883  
884  /* Description : Interleave right half of halfword elements from vectors
885   * Arguments   : Inputs  - in0, in1, in2, in3
886   *               Outputs - out0, out1
887   *               Return Type - as per RTYPE
888   * Details     : Right half of halfword elements of 'in0' and 'in1' are
889   *               interleaved and written to 'out0'.
890   */
891  #define ILVR_H2(RTYPE, in0, in1, in2, in3, out0, out1) do {  \
892    out0 = (RTYPE)__msa_ilvr_h((v8i16)in0, (v8i16)in1);        \
893    out1 = (RTYPE)__msa_ilvr_h((v8i16)in2, (v8i16)in3);        \
894  } while (0)
895  #define ILVR_H2_UB(...) ILVR_H2(v16u8, __VA_ARGS__)
896  #define ILVR_H2_SH(...) ILVR_H2(v8i16, __VA_ARGS__)
897  #define ILVR_H2_SW(...) ILVR_H2(v4i32, __VA_ARGS__)
898  
899  #define ILVR_H4(RTYPE, in0, in1, in2, in3, in4, in5, in6, in7,  \
900                  out0, out1, out2, out3) do {                    \
901    ILVR_H2(RTYPE, in0, in1, in2, in3, out0, out1);               \
902    ILVR_H2(RTYPE, in4, in5, in6, in7, out2, out3);               \
903  } while (0)
904  #define ILVR_H4_UB(...) ILVR_H4(v16u8, __VA_ARGS__)
905  #define ILVR_H4_SH(...) ILVR_H4(v8i16, __VA_ARGS__)
906  #define ILVR_H4_SW(...) ILVR_H4(v4i32, __VA_ARGS__)
907  
908  /* Description : Interleave right half of double word elements from vectors
909   * Arguments   : Inputs  - in0, in1, in2, in3
910   *               Outputs - out0, out1
911   *               Return Type - as per RTYPE
912   * Details     : Right half of double word elements of 'in0' and 'in1' are
913   *               interleaved and written to 'out0'.
914   */
915  #define ILVR_D2(RTYPE, in0, in1, in2, in3, out0, out1) do {  \
916    out0 = (RTYPE)__msa_ilvr_d((v2i64)in0, (v2i64)in1);        \
917    out1 = (RTYPE)__msa_ilvr_d((v2i64)in2, (v2i64)in3);        \
918  } while (0)
919  #define ILVR_D2_UB(...) ILVR_D2(v16u8, __VA_ARGS__)
920  #define ILVR_D2_SB(...) ILVR_D2(v16i8, __VA_ARGS__)
921  #define ILVR_D2_SH(...) ILVR_D2(v8i16, __VA_ARGS__)
922  
923  #define ILVR_D4(RTYPE, in0, in1, in2, in3, in4, in5, in6, in7,  \
924                  out0, out1, out2, out3) do {                    \
925    ILVR_D2(RTYPE, in0, in1, in2, in3, out0, out1);               \
926    ILVR_D2(RTYPE, in4, in5, in6, in7, out2, out3);               \
927  } while (0)
928  #define ILVR_D4_SB(...) ILVR_D4(v16i8, __VA_ARGS__)
929  #define ILVR_D4_UB(...) ILVR_D4(v16u8, __VA_ARGS__)
930  
931  /* Description : Interleave both left and right half of input vectors
932   * Arguments   : Inputs  - in0, in1
933   *               Outputs - out0, out1
934   *               Return Type - as per RTYPE
935   * Details     : Right half of byte elements from 'in0' and 'in1' are
936   *               interleaved and written to 'out0'
937   */
938  #define ILVRL_B2(RTYPE, in0, in1, out0, out1) do {     \
939    out0 = (RTYPE)__msa_ilvr_b((v16i8)in0, (v16i8)in1);  \
940    out1 = (RTYPE)__msa_ilvl_b((v16i8)in0, (v16i8)in1);  \
941  } while (0)
942  #define ILVRL_B2_UB(...) ILVRL_B2(v16u8, __VA_ARGS__)
943  #define ILVRL_B2_SB(...) ILVRL_B2(v16i8, __VA_ARGS__)
944  #define ILVRL_B2_UH(...) ILVRL_B2(v8u16, __VA_ARGS__)
945  #define ILVRL_B2_SH(...) ILVRL_B2(v8i16, __VA_ARGS__)
946  #define ILVRL_B2_SW(...) ILVRL_B2(v4i32, __VA_ARGS__)
947  
948  #define ILVRL_H2(RTYPE, in0, in1, out0, out1) do {     \
949    out0 = (RTYPE)__msa_ilvr_h((v8i16)in0, (v8i16)in1);  \
950    out1 = (RTYPE)__msa_ilvl_h((v8i16)in0, (v8i16)in1);  \
951  } while (0)
952  #define ILVRL_H2_UB(...) ILVRL_H2(v16u8, __VA_ARGS__)
953  #define ILVRL_H2_SB(...) ILVRL_H2(v16i8, __VA_ARGS__)
954  #define ILVRL_H2_SH(...) ILVRL_H2(v8i16, __VA_ARGS__)
955  #define ILVRL_H2_SW(...) ILVRL_H2(v4i32, __VA_ARGS__)
956  #define ILVRL_H2_UW(...) ILVRL_H2(v4u32, __VA_ARGS__)
957  
958  #define ILVRL_W2(RTYPE, in0, in1, out0, out1) do {     \
959    out0 = (RTYPE)__msa_ilvr_w((v4i32)in0, (v4i32)in1);  \
960    out1 = (RTYPE)__msa_ilvl_w((v4i32)in0, (v4i32)in1);  \
961  } while (0)
962  #define ILVRL_W2_UB(...) ILVRL_W2(v16u8, __VA_ARGS__)
963  #define ILVRL_W2_SH(...) ILVRL_W2(v8i16, __VA_ARGS__)
964  #define ILVRL_W2_SW(...) ILVRL_W2(v4i32, __VA_ARGS__)
965  #define ILVRL_W2_UW(...) ILVRL_W2(v4u32, __VA_ARGS__)
966  
967  /* Description : Pack even byte elements of vector pairs
968   *  Arguments   : Inputs  - in0, in1, in2, in3
969   *                Outputs - out0, out1
970   *                Return Type - as per RTYPE
971   *  Details     : Even byte elements of 'in0' are copied to the left half of
972   *                'out0' & even byte elements of 'in1' are copied to the right
973   *                half of 'out0'.
974   */
975  #define PCKEV_B2(RTYPE, in0, in1, in2, in3, out0, out1) do {  \
976    out0 = (RTYPE)__msa_pckev_b((v16i8)in0, (v16i8)in1);        \
977    out1 = (RTYPE)__msa_pckev_b((v16i8)in2, (v16i8)in3);        \
978  } while (0)
979  #define PCKEV_B2_SB(...) PCKEV_B2(v16i8, __VA_ARGS__)
980  #define PCKEV_B2_UB(...) PCKEV_B2(v16u8, __VA_ARGS__)
981  #define PCKEV_B2_SH(...) PCKEV_B2(v8i16, __VA_ARGS__)
982  #define PCKEV_B2_SW(...) PCKEV_B2(v4i32, __VA_ARGS__)
983  
984  #define PCKEV_B4(RTYPE, in0, in1, in2, in3, in4, in5, in6, in7,  \
985                   out0, out1, out2, out3) do {                    \
986    PCKEV_B2(RTYPE, in0, in1, in2, in3, out0, out1);               \
987    PCKEV_B2(RTYPE, in4, in5, in6, in7, out2, out3);               \
988  } while (0)
989  #define PCKEV_B4_SB(...) PCKEV_B4(v16i8, __VA_ARGS__)
990  #define PCKEV_B4_UB(...) PCKEV_B4(v16u8, __VA_ARGS__)
991  #define PCKEV_B4_SH(...) PCKEV_B4(v8i16, __VA_ARGS__)
992  #define PCKEV_B4_SW(...) PCKEV_B4(v4i32, __VA_ARGS__)
993  
994  /* Description : Pack even halfword elements of vector pairs
995   * Arguments   : Inputs  - in0, in1, in2, in3
996   *               Outputs - out0, out1
997   *               Return Type - as per RTYPE
998   * Details     : Even halfword elements of 'in0' are copied to the left half of
999   *               'out0' & even halfword elements of 'in1' are copied to the
1000   *               right half of 'out0'.
1001   */
1002  #define PCKEV_H2(RTYPE, in0, in1, in2, in3, out0, out1) do {  \
1003    out0 = (RTYPE)__msa_pckev_h((v8i16)in0, (v8i16)in1);        \
1004    out1 = (RTYPE)__msa_pckev_h((v8i16)in2, (v8i16)in3);        \
1005  } while (0)
1006  #define PCKEV_H2_UH(...) PCKEV_H2(v8u16, __VA_ARGS__)
1007  #define PCKEV_H2_SH(...) PCKEV_H2(v8i16, __VA_ARGS__)
1008  #define PCKEV_H2_SW(...) PCKEV_H2(v4i32, __VA_ARGS__)
1009  #define PCKEV_H2_UW(...) PCKEV_H2(v4u32, __VA_ARGS__)
1010  
1011  /* Description : Pack even word elements of vector pairs
1012   * Arguments   : Inputs  - in0, in1, in2, in3
1013   *               Outputs - out0, out1
1014   *               Return Type - as per RTYPE
1015   * Details     : Even word elements of 'in0' are copied to the left half of
1016   *               'out0' & even word elements of 'in1' are copied to the
1017   *               right half of 'out0'.
1018   */
1019  #define PCKEV_W2(RTYPE, in0, in1, in2, in3, out0, out1) do {  \
1020    out0 = (RTYPE)__msa_pckev_w((v4i32)in0, (v4i32)in1);        \
1021    out1 = (RTYPE)__msa_pckev_w((v4i32)in2, (v4i32)in3);        \
1022  } while (0)
1023  #define PCKEV_W2_UH(...) PCKEV_W2(v8u16, __VA_ARGS__)
1024  #define PCKEV_W2_SH(...) PCKEV_W2(v8i16, __VA_ARGS__)
1025  #define PCKEV_W2_SW(...) PCKEV_W2(v4i32, __VA_ARGS__)
1026  #define PCKEV_W2_UW(...) PCKEV_W2(v4u32, __VA_ARGS__)
1027  
1028  /* Description : Pack odd halfword elements of vector pairs
1029   * Arguments   : Inputs  - in0, in1, in2, in3
1030   *               Outputs - out0, out1
1031   *               Return Type - as per RTYPE
1032   * Details     : Odd halfword elements of 'in0' are copied to the left half of
1033   *               'out0' & odd halfword elements of 'in1' are copied to the
1034   *               right half of 'out0'.
1035   */
1036  #define PCKOD_H2(RTYPE, in0, in1, in2, in3, out0, out1) do {  \
1037    out0 = (RTYPE)__msa_pckod_h((v8i16)in0, (v8i16)in1);        \
1038    out1 = (RTYPE)__msa_pckod_h((v8i16)in2, (v8i16)in3);        \
1039  } while (0)
1040  #define PCKOD_H2_UH(...) PCKOD_H2(v8u16, __VA_ARGS__)
1041  #define PCKOD_H2_SH(...) PCKOD_H2(v8i16, __VA_ARGS__)
1042  #define PCKOD_H2_SW(...) PCKOD_H2(v4i32, __VA_ARGS__)
1043  #define PCKOD_H2_UW(...) PCKOD_H2(v4u32, __VA_ARGS__)
1044  
1045  /* Description : Arithmetic immediate shift right all elements of word vector
1046   * Arguments   : Inputs  - in0, in1, shift
1047   *               Outputs - in place operation
1048   *               Return Type - as per input vector RTYPE
1049   * Details     : Each element of vector 'in0' is right shifted by 'shift' and
1050   *               the result is written in-place. 'shift' is a GP variable.
1051   */
1052  #define SRAI_W2(RTYPE, in0, in1, shift_val) do {  \
1053    in0 = (RTYPE)SRAI_W(in0, shift_val);            \
1054    in1 = (RTYPE)SRAI_W(in1, shift_val);            \
1055  } while (0)
1056  #define SRAI_W2_SW(...) SRAI_W2(v4i32, __VA_ARGS__)
1057  #define SRAI_W2_UW(...) SRAI_W2(v4u32, __VA_ARGS__)
1058  
1059  #define SRAI_W4(RTYPE, in0, in1, in2, in3, shift_val) do {  \
1060    SRAI_W2(RTYPE, in0, in1, shift_val);                      \
1061    SRAI_W2(RTYPE, in2, in3, shift_val);                      \
1062  } while (0)
1063  #define SRAI_W4_SW(...) SRAI_W4(v4i32, __VA_ARGS__)
1064  #define SRAI_W4_UW(...) SRAI_W4(v4u32, __VA_ARGS__)
1065  
1066  /* Description : Arithmetic shift right all elements of half-word vector
1067   * Arguments   : Inputs  - in0, in1, shift
1068   *               Outputs - in place operation
1069   *               Return Type - as per input vector RTYPE
1070   * Details     : Each element of vector 'in0' is right shifted by 'shift' and
1071   *               the result is written in-place. 'shift' is a GP variable.
1072   */
1073  #define SRAI_H2(RTYPE, in0, in1, shift_val) do {  \
1074    in0 = (RTYPE)SRAI_H(in0, shift_val);            \
1075    in1 = (RTYPE)SRAI_H(in1, shift_val);            \
1076  } while (0)
1077  #define SRAI_H2_SH(...) SRAI_H2(v8i16, __VA_ARGS__)
1078  #define SRAI_H2_UH(...) SRAI_H2(v8u16, __VA_ARGS__)
1079  
1080  /* Description : Arithmetic rounded shift right all elements of word vector
1081   * Arguments   : Inputs  - in0, in1, shift
1082   *               Outputs - in place operation
1083   *               Return Type - as per input vector RTYPE
1084   * Details     : Each element of vector 'in0' is right shifted by 'shift' and
1085   *               the result is written in-place. 'shift' is a GP variable.
1086   */
1087  #define SRARI_W2(RTYPE, in0, in1, shift) do {     \
1088    in0 = (RTYPE)__msa_srari_w((v4i32)in0, shift);  \
1089    in1 = (RTYPE)__msa_srari_w((v4i32)in1, shift);  \
1090  } while (0)
1091  #define SRARI_W2_SW(...) SRARI_W2(v4i32, __VA_ARGS__)
1092  
1093  #define SRARI_W4(RTYPE, in0, in1, in2, in3, shift) do {  \
1094    SRARI_W2(RTYPE, in0, in1, shift);                      \
1095    SRARI_W2(RTYPE, in2, in3, shift);                      \
1096  } while (0)
1097  #define SRARI_W4_SH(...) SRARI_W4(v8i16, __VA_ARGS__)
1098  #define SRARI_W4_UW(...) SRARI_W4(v4u32, __VA_ARGS__)
1099  #define SRARI_W4_SW(...) SRARI_W4(v4i32, __VA_ARGS__)
1100  
1101  /* Description : Shift right arithmetic rounded double words
1102   * Arguments   : Inputs  - in0, in1, shift
1103   *               Outputs - in place operation
1104   *               Return Type - as per RTYPE
1105   * Details     : Each element of vector 'in0' is shifted right arithmetically by
1106   *               the number of bits in the corresponding element in the vector
1107   *               'shift'. The last discarded bit is added to shifted value for
1108   *               rounding and the result is written in-place.
1109   *               'shift' is a vector.
1110   */
1111  #define SRAR_D2(RTYPE, in0, in1, shift) do {            \
1112    in0 = (RTYPE)__msa_srar_d((v2i64)in0, (v2i64)shift);  \
1113    in1 = (RTYPE)__msa_srar_d((v2i64)in1, (v2i64)shift);  \
1114  } while (0)
1115  #define SRAR_D2_SW(...) SRAR_D2(v4i32, __VA_ARGS__)
1116  #define SRAR_D2_SD(...) SRAR_D2(v2i64, __VA_ARGS__)
1117  #define SRAR_D2_UD(...) SRAR_D2(v2u64, __VA_ARGS__)
1118  
1119  #define SRAR_D4(RTYPE, in0, in1, in2, in3, shift) do {  \
1120    SRAR_D2(RTYPE, in0, in1, shift);                      \
1121    SRAR_D2(RTYPE, in2, in3, shift);                      \
1122  } while (0)
1123  #define SRAR_D4_SD(...) SRAR_D4(v2i64, __VA_ARGS__)
1124  #define SRAR_D4_UD(...) SRAR_D4(v2u64, __VA_ARGS__)
1125  
1126  /* Description : Addition of 2 pairs of half-word vectors
1127   * Arguments   : Inputs  - in0, in1, in2, in3
1128   *               Outputs - out0, out1
1129   * Details     : Each element in 'in0' is added to 'in1' and result is written
1130   *               to 'out0'.
1131   */
1132  #define ADDVI_H2(RTYPE, in0, in1, in2, in3, out0, out1) do {  \
1133    out0 = (RTYPE)ADDVI_H(in0, in1);                            \
1134    out1 = (RTYPE)ADDVI_H(in2, in3);                            \
1135  } while (0)
1136  #define ADDVI_H2_SH(...) ADDVI_H2(v8i16, __VA_ARGS__)
1137  #define ADDVI_H2_UH(...) ADDVI_H2(v8u16, __VA_ARGS__)
1138  
1139  /* Description : Addition of 2 pairs of word vectors
1140   * Arguments   : Inputs  - in0, in1, in2, in3
1141   *               Outputs - out0, out1
1142   * Details     : Each element in 'in0' is added to 'in1' and result is written
1143   *               to 'out0'.
1144   */
1145  #define ADDVI_W2(RTYPE, in0, in1, in2, in3, out0, out1) do {  \
1146    out0 = (RTYPE)ADDVI_W(in0, in1);                            \
1147    out1 = (RTYPE)ADDVI_W(in2, in3);                            \
1148  } while (0)
1149  #define ADDVI_W2_SW(...) ADDVI_W2(v4i32, __VA_ARGS__)
1150  
1151  /* Description : Fill 2 pairs of word vectors with GP registers
1152   * Arguments   : Inputs  - in0, in1
1153   *               Outputs - out0, out1
1154   * Details     : GP register in0 is replicated in each word element of out0
1155   *               GP register in1 is replicated in each word element of out1
1156   */
1157  #define FILL_W2(RTYPE, in0, in1, out0, out1) do {  \
1158    out0 = (RTYPE)__msa_fill_w(in0);                 \
1159    out1 = (RTYPE)__msa_fill_w(in1);                 \
1160  } while (0)
1161  #define FILL_W2_SW(...) FILL_W2(v4i32, __VA_ARGS__)
1162  
1163  /* Description : Addition of 2 pairs of vectors
1164   * Arguments   : Inputs  - in0, in1, in2, in3
1165   *               Outputs - out0, out1
1166   * Details     : Each element in 'in0' is added to 'in1' and result is written
1167   *               to 'out0'.
1168   */
1169  #define ADD2(in0, in1, in2, in3, out0, out1) do {  \
1170    out0 = in0 + in1;                                \
1171    out1 = in2 + in3;                                \
1172  } while (0)
1173  
1174  #define ADD4(in0, in1, in2, in3, in4, in5, in6, in7,  \
1175               out0, out1, out2, out3) do {             \
1176    ADD2(in0, in1, in2, in3, out0, out1);               \
1177    ADD2(in4, in5, in6, in7, out2, out3);               \
1178  } while (0)
1179  
1180  /* Description : Subtraction of 2 pairs of vectors
1181   * Arguments   : Inputs  - in0, in1, in2, in3
1182   *               Outputs - out0, out1
1183   * Details     : Each element in 'in1' is subtracted from 'in0' and result is
1184   *               written to 'out0'.
1185   */
1186  #define SUB2(in0, in1, in2, in3, out0, out1) do {  \
1187    out0 = in0 - in1;                                \
1188    out1 = in2 - in3;                                \
1189  } while (0)
1190  
1191  #define SUB3(in0, in1, in2, in3, in4, in5, out0, out1, out2) do {  \
1192    out0 = in0 - in1;                                                \
1193    out1 = in2 - in3;                                                \
1194    out2 = in4 - in5;                                                \
1195  } while (0)
1196  
1197  #define SUB4(in0, in1, in2, in3, in4, in5, in6, in7,  \
1198               out0, out1, out2, out3) do {             \
1199    out0 = in0 - in1;                                   \
1200    out1 = in2 - in3;                                   \
1201    out2 = in4 - in5;                                   \
1202    out3 = in6 - in7;                                   \
1203  } while (0)
1204  
1205  /* Description : Addition - Subtraction of input vectors
1206   * Arguments   : Inputs  - in0, in1
1207   *               Outputs - out0, out1
1208   * Details     : Each element in 'in1' is added to 'in0' and result is
1209   *               written to 'out0'.
1210   *               Each element in 'in1' is subtracted from 'in0' and result is
1211   *               written to 'out1'.
1212   */
1213  #define ADDSUB2(in0, in1, out0, out1) do {  \
1214    out0 = in0 + in1;                         \
1215    out1 = in0 - in1;                         \
1216  } while (0)
1217  
1218  /* Description : Multiplication of pairs of vectors
1219   * Arguments   : Inputs  - in0, in1, in2, in3
1220   *               Outputs - out0, out1
1221   * Details     : Each element from 'in0' is multiplied with elements from 'in1'
1222   *               and the result is written to 'out0'
1223   */
1224  #define MUL2(in0, in1, in2, in3, out0, out1) do {  \
1225    out0 = in0 * in1;                                \
1226    out1 = in2 * in3;                                \
1227  } while (0)
1228  
1229  #define MUL4(in0, in1, in2, in3, in4, in5, in6, in7,  \
1230               out0, out1, out2, out3) do {             \
1231    MUL2(in0, in1, in2, in3, out0, out1);               \
1232    MUL2(in4, in5, in6, in7, out2, out3);               \
1233  } while (0)
1234  
1235  /* Description : Sign extend halfword elements from right half of the vector
1236   * Arguments   : Input  - in    (halfword vector)
1237   *               Output - out   (sign extended word vector)
1238   *               Return Type - signed word
1239   * Details     : Sign bit of halfword elements from input vector 'in' is
1240   *               extracted and interleaved with same vector 'in0' to generate
1241   *               4 word elements keeping sign intact
1242   */
1243  #define UNPCK_R_SH_SW(in, out) do {                   \
1244    const v8i16 sign_m = __msa_clti_s_h((v8i16)in, 0);  \
1245    out = (v4i32)__msa_ilvr_h(sign_m, (v8i16)in);       \
1246  } while (0)
1247  
1248  /* Description : Sign extend halfword elements from input vector and return
1249   *               the result in pair of vectors
1250   * Arguments   : Input   - in            (halfword vector)
1251   *               Outputs - out0, out1   (sign extended word vectors)
1252   *               Return Type - signed word
1253   * Details     : Sign bit of halfword elements from input vector 'in' is
1254   *               extracted and interleaved right with same vector 'in0' to
1255   *               generate 4 signed word elements in 'out0'
1256   *               Then interleaved left with same vector 'in0' to
1257   *               generate 4 signed word elements in 'out1'
1258   */
1259  #define UNPCK_SH_SW(in, out0, out1) do {              \
1260    const v8i16 tmp_m = __msa_clti_s_h((v8i16)in, 0);   \
1261    ILVRL_H2_SW(tmp_m, in, out0, out1);                 \
1262  } while (0)
1263  
1264  /* Description : Butterfly of 4 input vectors
1265   * Arguments   : Inputs  - in0, in1, in2, in3
1266   *               Outputs - out0, out1, out2, out3
1267   * Details     : Butterfly operation
1268   */
1269  #define BUTTERFLY_4(in0, in1, in2, in3, out0, out1, out2, out3) do {  \
1270    out0 = in0 + in3;                                                   \
1271    out1 = in1 + in2;                                                   \
1272    out2 = in1 - in2;                                                   \
1273    out3 = in0 - in3;                                                   \
1274  } while (0)
1275  
1276  /* Description : Transpose 16x4 block into 4x16 with byte elements in vectors
1277   * Arguments   : Inputs  - in0, in1, in2, in3, in4, in5, in6, in7,
1278   *                         in8, in9, in10, in11, in12, in13, in14, in15
1279   *               Outputs - out0, out1, out2, out3
1280   *               Return Type - unsigned byte
1281   */
1282  #define TRANSPOSE16x4_UB_UB(in0, in1, in2, in3, in4, in5, in6, in7,        \
1283                              in8, in9, in10, in11, in12, in13, in14, in15,  \
1284                              out0, out1, out2, out3) do {                   \
1285    v2i64 tmp0_m, tmp1_m, tmp2_m, tmp3_m, tmp4_m, tmp5_m;                    \
1286    ILVEV_W2_SD(in0, in4, in8, in12, tmp2_m, tmp3_m);                        \
1287    ILVEV_W2_SD(in1, in5, in9, in13, tmp0_m, tmp1_m);                        \
1288    ILVEV_D2_UB(tmp2_m, tmp3_m, tmp0_m, tmp1_m, out1, out3);                 \
1289    ILVEV_W2_SD(in2, in6, in10, in14, tmp4_m, tmp5_m);                       \
1290    ILVEV_W2_SD(in3, in7, in11, in15, tmp0_m, tmp1_m);                       \
1291    ILVEV_D2_SD(tmp4_m, tmp5_m, tmp0_m, tmp1_m, tmp2_m, tmp3_m);             \
1292    ILVEV_B2_SD(out1, out3, tmp2_m, tmp3_m, tmp0_m, tmp1_m);                 \
1293    ILVEVOD_H2_UB(tmp0_m, tmp1_m, tmp0_m, tmp1_m, out0, out2);               \
1294    ILVOD_B2_SD(out1, out3, tmp2_m, tmp3_m, tmp0_m, tmp1_m);                 \
1295    ILVEVOD_H2_UB(tmp0_m, tmp1_m, tmp0_m, tmp1_m, out1, out3);               \
1296  } while (0)
1297  
1298  /* Description : Transpose 16x8 block into 8x16 with byte elements in vectors
1299   * Arguments   : Inputs  - in0, in1, in2, in3, in4, in5, in6, in7,
1300   *                         in8, in9, in10, in11, in12, in13, in14, in15
1301   *               Outputs - out0, out1, out2, out3, out4, out5, out6, out7
1302   *               Return Type - unsigned byte
1303   */
1304  #define TRANSPOSE16x8_UB_UB(in0, in1, in2, in3, in4, in5, in6, in7,        \
1305                              in8, in9, in10, in11, in12, in13, in14, in15,  \
1306                              out0, out1, out2, out3, out4, out5,            \
1307                              out6, out7) do {                               \
1308    v8i16 tmp0_m, tmp1_m, tmp4_m, tmp5_m, tmp6_m, tmp7_m;                    \
1309    v4i32 tmp2_m, tmp3_m;                                                    \
1310    ILVEV_D2_UB(in0, in8, in1, in9, out7, out6);                             \
1311    ILVEV_D2_UB(in2, in10, in3, in11, out5, out4);                           \
1312    ILVEV_D2_UB(in4, in12, in5, in13, out3, out2);                           \
1313    ILVEV_D2_UB(in6, in14, in7, in15, out1, out0);                           \
1314    ILVEV_B2_SH(out7, out6, out5, out4, tmp0_m, tmp1_m);                     \
1315    ILVOD_B2_SH(out7, out6, out5, out4, tmp4_m, tmp5_m);                     \
1316    ILVEV_B2_UB(out3, out2, out1, out0, out5, out7);                         \
1317    ILVOD_B2_SH(out3, out2, out1, out0, tmp6_m, tmp7_m);                     \
1318    ILVEV_H2_SW(tmp0_m, tmp1_m, out5, out7, tmp2_m, tmp3_m);                 \
1319    ILVEVOD_W2_UB(tmp2_m, tmp3_m, tmp2_m, tmp3_m, out0, out4);               \
1320    ILVOD_H2_SW(tmp0_m, tmp1_m, out5, out7, tmp2_m, tmp3_m);                 \
1321    ILVEVOD_W2_UB(tmp2_m, tmp3_m, tmp2_m, tmp3_m, out2, out6);               \
1322    ILVEV_H2_SW(tmp4_m, tmp5_m, tmp6_m, tmp7_m, tmp2_m, tmp3_m);             \
1323    ILVEVOD_W2_UB(tmp2_m, tmp3_m, tmp2_m, tmp3_m, out1, out5);               \
1324    ILVOD_H2_SW(tmp4_m, tmp5_m, tmp6_m, tmp7_m, tmp2_m, tmp3_m);             \
1325    ILVEVOD_W2_UB(tmp2_m, tmp3_m, tmp2_m, tmp3_m, out3, out7);               \
1326  } while (0)
1327  
1328  /* Description : Transpose 4x4 block with word elements in vectors
1329   * Arguments   : Inputs  - in0, in1, in2, in3
1330   *                Outputs - out0, out1, out2, out3
1331   *                Return Type - as per RTYPE
1332   */
1333  #define TRANSPOSE4x4_W(RTYPE, in0, in1, in2, in3,                            \
1334                         out0, out1, out2, out3) do {                          \
1335    v4i32 s0_m, s1_m, s2_m, s3_m;                                              \
1336    ILVRL_W2_SW(in1, in0, s0_m, s1_m);                                         \
1337    ILVRL_W2_SW(in3, in2, s2_m, s3_m);                                         \
1338    out0 = (RTYPE)__msa_ilvr_d((v2i64)s2_m, (v2i64)s0_m);                      \
1339    out1 = (RTYPE)__msa_ilvl_d((v2i64)s2_m, (v2i64)s0_m);                      \
1340    out2 = (RTYPE)__msa_ilvr_d((v2i64)s3_m, (v2i64)s1_m);                      \
1341    out3 = (RTYPE)__msa_ilvl_d((v2i64)s3_m, (v2i64)s1_m);                      \
1342  } while (0)
1343  #define TRANSPOSE4x4_SW_SW(...) TRANSPOSE4x4_W(v4i32, __VA_ARGS__)
1344  
1345  /* Description : Add block 4x4
1346   * Arguments   : Inputs - in0, in1, in2, in3, pdst, stride
1347   * Details     : Least significant 4 bytes from each input vector are added to
1348   *               the destination bytes, clipped between 0-255 and stored.
1349   */
1350  #define ADDBLK_ST4x4_UB(in0, in1, in2, in3, pdst, stride) do {  \
1351    uint32_t src0_m, src1_m, src2_m, src3_m;                      \
1352    v8i16 inp0_m, inp1_m, res0_m, res1_m;                         \
1353    v16i8 dst0_m = { 0 };                                         \
1354    v16i8 dst1_m = { 0 };                                         \
1355    const v16i8 zero_m = { 0 };                                   \
1356    ILVR_D2_SH(in1, in0, in3, in2, inp0_m, inp1_m);               \
1357    LW4(pdst, stride, src0_m, src1_m, src2_m, src3_m);            \
1358    INSERT_W2_SB(src0_m, src1_m, dst0_m);                         \
1359    INSERT_W2_SB(src2_m, src3_m, dst1_m);                         \
1360    ILVR_B2_SH(zero_m, dst0_m, zero_m, dst1_m, res0_m, res1_m);   \
1361    ADD2(res0_m, inp0_m, res1_m, inp1_m, res0_m, res1_m);         \
1362    CLIP_SH2_0_255(res0_m, res1_m);                               \
1363    PCKEV_B2_SB(res0_m, res0_m, res1_m, res1_m, dst0_m, dst1_m);  \
1364    ST4x4_UB(dst0_m, dst1_m, 0, 1, 0, 1, pdst, stride);           \
1365  } while (0)
1366  
1367  /* Description : Pack even byte elements, extract 0 & 2 index words from pair
1368   *               of results and store 4 words in destination memory as per
1369   *               stride
1370   * Arguments   : Inputs - in0, in1, in2, in3, pdst, stride
1371   */
1372  #define PCKEV_ST4x4_UB(in0, in1, in2, in3, pdst, stride) do {  \
1373    v16i8 tmp0_m, tmp1_m;                                        \
1374    PCKEV_B2_SB(in1, in0, in3, in2, tmp0_m, tmp1_m);             \
1375    ST4x4_UB(tmp0_m, tmp1_m, 0, 2, 0, 2, pdst, stride);          \
1376  } while (0)
1377  
1378  /* Description : average with rounding (in0 + in1 + 1) / 2.
1379   * Arguments   : Inputs  - in0, in1, in2, in3,
1380   *               Outputs - out0, out1
1381   *               Return Type - as per RTYPE
1382   * Details     : Each unsigned byte element from 'in0' vector is added with
1383   *               each unsigned byte element from 'in1' vector. Then the average
1384   *               with rounding is calculated and written to 'out0'
1385   */
1386  #define AVER_UB2(RTYPE, in0, in1, in2, in3, out0, out1) do {  \
1387    out0 = (RTYPE)__msa_aver_u_b((v16u8)in0, (v16u8)in1);       \
1388    out1 = (RTYPE)__msa_aver_u_b((v16u8)in2, (v16u8)in3);       \
1389  } while (0)
1390  #define AVER_UB2_UB(...) AVER_UB2(v16u8, __VA_ARGS__)
1391  
1392  #endif  /* WEBP_DSP_MSA_MACRO_H_ */
1393