1 /*
2 * Copyright (c) 2016 The WebM project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11 #ifndef VPX_VPX_DSP_ARM_TRANSPOSE_NEON_H_
12 #define VPX_VPX_DSP_ARM_TRANSPOSE_NEON_H_
13
14 #include <arm_neon.h>
15
16 #include "./vpx_config.h"
17
18 // Transpose 64 bit elements as follows:
19 // a0: 00 01 02 03 04 05 06 07
20 // a1: 16 17 18 19 20 21 22 23
21 //
22 // b0.val[0]: 00 01 02 03 16 17 18 19
23 // b0.val[1]: 04 05 06 07 20 21 22 23
vpx_vtrnq_s64_to_s16(int32x4_t a0,int32x4_t a1)24 static INLINE int16x8x2_t vpx_vtrnq_s64_to_s16(int32x4_t a0, int32x4_t a1) {
25 int16x8x2_t b0;
26 b0.val[0] = vcombine_s16(vreinterpret_s16_s32(vget_low_s32(a0)),
27 vreinterpret_s16_s32(vget_low_s32(a1)));
28 b0.val[1] = vcombine_s16(vreinterpret_s16_s32(vget_high_s32(a0)),
29 vreinterpret_s16_s32(vget_high_s32(a1)));
30 return b0;
31 }
32
vpx_vtrnq_s64_to_s32(int32x4_t a0,int32x4_t a1)33 static INLINE int32x4x2_t vpx_vtrnq_s64_to_s32(int32x4_t a0, int32x4_t a1) {
34 int32x4x2_t b0;
35 b0.val[0] = vcombine_s32(vget_low_s32(a0), vget_low_s32(a1));
36 b0.val[1] = vcombine_s32(vget_high_s32(a0), vget_high_s32(a1));
37 return b0;
38 }
39
vpx_vtrnq_s64(int32x4_t a0,int32x4_t a1)40 static INLINE int64x2x2_t vpx_vtrnq_s64(int32x4_t a0, int32x4_t a1) {
41 int64x2x2_t b0;
42 b0.val[0] = vcombine_s64(vreinterpret_s64_s32(vget_low_s32(a0)),
43 vreinterpret_s64_s32(vget_low_s32(a1)));
44 b0.val[1] = vcombine_s64(vreinterpret_s64_s32(vget_high_s32(a0)),
45 vreinterpret_s64_s32(vget_high_s32(a1)));
46 return b0;
47 }
48
vpx_vtrnq_u64_to_u8(uint32x4_t a0,uint32x4_t a1)49 static INLINE uint8x16x2_t vpx_vtrnq_u64_to_u8(uint32x4_t a0, uint32x4_t a1) {
50 uint8x16x2_t b0;
51 b0.val[0] = vcombine_u8(vreinterpret_u8_u32(vget_low_u32(a0)),
52 vreinterpret_u8_u32(vget_low_u32(a1)));
53 b0.val[1] = vcombine_u8(vreinterpret_u8_u32(vget_high_u32(a0)),
54 vreinterpret_u8_u32(vget_high_u32(a1)));
55 return b0;
56 }
57
vpx_vtrnq_u64_to_u16(uint32x4_t a0,uint32x4_t a1)58 static INLINE uint16x8x2_t vpx_vtrnq_u64_to_u16(uint32x4_t a0, uint32x4_t a1) {
59 uint16x8x2_t b0;
60 b0.val[0] = vcombine_u16(vreinterpret_u16_u32(vget_low_u32(a0)),
61 vreinterpret_u16_u32(vget_low_u32(a1)));
62 b0.val[1] = vcombine_u16(vreinterpret_u16_u32(vget_high_u32(a0)),
63 vreinterpret_u16_u32(vget_high_u32(a1)));
64 return b0;
65 }
66
transpose_u8_4x4(uint8x8_t * a0,uint8x8_t * a1)67 static INLINE void transpose_u8_4x4(uint8x8_t *a0, uint8x8_t *a1) {
68 // Swap 16 bit elements. Goes from:
69 // a0: 00 01 02 03 10 11 12 13
70 // a1: 20 21 22 23 30 31 32 33
71 // to:
72 // b0.val[0]: 00 01 20 21 10 11 30 31
73 // b0.val[1]: 02 03 22 23 12 13 32 33
74
75 const uint16x4x2_t b0 =
76 vtrn_u16(vreinterpret_u16_u8(*a0), vreinterpret_u16_u8(*a1));
77
78 // Swap 32 bit elements resulting in:
79 // c0.val[0]: 00 01 20 21 02 03 22 23
80 // c0.val[1]: 10 11 30 31 12 13 32 33
81
82 const uint32x2x2_t c0 = vtrn_u32(vreinterpret_u32_u16(b0.val[0]),
83 vreinterpret_u32_u16(b0.val[1]));
84
85 // Swap 8 bit elements resulting in:
86 // d0.val[0]: 00 10 20 30 02 12 22 32
87 // d0.val[1]: 01 11 21 31 03 13 23 33
88
89 const uint8x8x2_t d0 =
90 vtrn_u8(vreinterpret_u8_u32(c0.val[0]), vreinterpret_u8_u32(c0.val[1]));
91
92 *a0 = d0.val[0];
93 *a1 = d0.val[1];
94 }
95
transpose_s16_4x4d(int16x4_t * a0,int16x4_t * a1,int16x4_t * a2,int16x4_t * a3)96 static INLINE void transpose_s16_4x4d(int16x4_t *a0, int16x4_t *a1,
97 int16x4_t *a2, int16x4_t *a3) {
98 // Swap 16 bit elements. Goes from:
99 // a0: 00 01 02 03
100 // a1: 10 11 12 13
101 // a2: 20 21 22 23
102 // a3: 30 31 32 33
103 // to:
104 // b0.val[0]: 00 10 02 12
105 // b0.val[1]: 01 11 03 13
106 // b1.val[0]: 20 30 22 32
107 // b1.val[1]: 21 31 23 33
108
109 const int16x4x2_t b0 = vtrn_s16(*a0, *a1);
110 const int16x4x2_t b1 = vtrn_s16(*a2, *a3);
111
112 // Swap 32 bit elements resulting in:
113 // c0.val[0]: 00 10 20 30
114 // c0.val[1]: 02 12 22 32
115 // c1.val[0]: 01 11 21 31
116 // c1.val[1]: 03 13 23 33
117
118 const int32x2x2_t c0 = vtrn_s32(vreinterpret_s32_s16(b0.val[0]),
119 vreinterpret_s32_s16(b1.val[0]));
120 const int32x2x2_t c1 = vtrn_s32(vreinterpret_s32_s16(b0.val[1]),
121 vreinterpret_s32_s16(b1.val[1]));
122
123 *a0 = vreinterpret_s16_s32(c0.val[0]);
124 *a1 = vreinterpret_s16_s32(c1.val[0]);
125 *a2 = vreinterpret_s16_s32(c0.val[1]);
126 *a3 = vreinterpret_s16_s32(c1.val[1]);
127 }
128
transpose_s16_4x4q(int16x8_t * a0,int16x8_t * a1)129 static INLINE void transpose_s16_4x4q(int16x8_t *a0, int16x8_t *a1) {
130 // Swap 32 bit elements. Goes from:
131 // a0: 00 01 02 03 10 11 12 13
132 // a1: 20 21 22 23 30 31 32 33
133 // to:
134 // b0.val[0]: 00 01 20 21 10 11 30 31
135 // b0.val[1]: 02 03 22 23 12 13 32 33
136
137 const int32x4x2_t b0 =
138 vtrnq_s32(vreinterpretq_s32_s16(*a0), vreinterpretq_s32_s16(*a1));
139
140 // Swap 64 bit elements resulting in:
141 // c0: 00 01 20 21 02 03 22 23
142 // c1: 10 11 30 31 12 13 32 33
143
144 const int32x4_t c0 =
145 vcombine_s32(vget_low_s32(b0.val[0]), vget_low_s32(b0.val[1]));
146 const int32x4_t c1 =
147 vcombine_s32(vget_high_s32(b0.val[0]), vget_high_s32(b0.val[1]));
148
149 // Swap 16 bit elements resulting in:
150 // d0.val[0]: 00 10 20 30 02 12 22 32
151 // d0.val[1]: 01 11 21 31 03 13 23 33
152
153 const int16x8x2_t d0 =
154 vtrnq_s16(vreinterpretq_s16_s32(c0), vreinterpretq_s16_s32(c1));
155
156 *a0 = d0.val[0];
157 *a1 = d0.val[1];
158 }
159
transpose_u16_4x4q(uint16x8_t * a0,uint16x8_t * a1)160 static INLINE void transpose_u16_4x4q(uint16x8_t *a0, uint16x8_t *a1) {
161 // Swap 32 bit elements. Goes from:
162 // a0: 00 01 02 03 10 11 12 13
163 // a1: 20 21 22 23 30 31 32 33
164 // to:
165 // b0.val[0]: 00 01 20 21 10 11 30 31
166 // b0.val[1]: 02 03 22 23 12 13 32 33
167
168 const uint32x4x2_t b0 =
169 vtrnq_u32(vreinterpretq_u32_u16(*a0), vreinterpretq_u32_u16(*a1));
170
171 // Swap 64 bit elements resulting in:
172 // c0: 00 01 20 21 02 03 22 23
173 // c1: 10 11 30 31 12 13 32 33
174
175 const uint32x4_t c0 =
176 vcombine_u32(vget_low_u32(b0.val[0]), vget_low_u32(b0.val[1]));
177 const uint32x4_t c1 =
178 vcombine_u32(vget_high_u32(b0.val[0]), vget_high_u32(b0.val[1]));
179
180 // Swap 16 bit elements resulting in:
181 // d0.val[0]: 00 10 20 30 02 12 22 32
182 // d0.val[1]: 01 11 21 31 03 13 23 33
183
184 const uint16x8x2_t d0 =
185 vtrnq_u16(vreinterpretq_u16_u32(c0), vreinterpretq_u16_u32(c1));
186
187 *a0 = d0.val[0];
188 *a1 = d0.val[1];
189 }
190
transpose_u8_4x8(uint8x8_t * a0,uint8x8_t * a1,uint8x8_t * a2,uint8x8_t * a3,const uint8x8_t a4,const uint8x8_t a5,const uint8x8_t a6,const uint8x8_t a7)191 static INLINE void transpose_u8_4x8(uint8x8_t *a0, uint8x8_t *a1, uint8x8_t *a2,
192 uint8x8_t *a3, const uint8x8_t a4,
193 const uint8x8_t a5, const uint8x8_t a6,
194 const uint8x8_t a7) {
195 // Swap 32 bit elements. Goes from:
196 // a0: 00 01 02 03 XX XX XX XX
197 // a1: 10 11 12 13 XX XX XX XX
198 // a2: 20 21 22 23 XX XX XX XX
199 // a3; 30 31 32 33 XX XX XX XX
200 // a4: 40 41 42 43 XX XX XX XX
201 // a5: 50 51 52 53 XX XX XX XX
202 // a6: 60 61 62 63 XX XX XX XX
203 // a7: 70 71 72 73 XX XX XX XX
204 // to:
205 // b0.val[0]: 00 01 02 03 40 41 42 43
206 // b1.val[0]: 10 11 12 13 50 51 52 53
207 // b2.val[0]: 20 21 22 23 60 61 62 63
208 // b3.val[0]: 30 31 32 33 70 71 72 73
209
210 const uint32x2x2_t b0 =
211 vtrn_u32(vreinterpret_u32_u8(*a0), vreinterpret_u32_u8(a4));
212 const uint32x2x2_t b1 =
213 vtrn_u32(vreinterpret_u32_u8(*a1), vreinterpret_u32_u8(a5));
214 const uint32x2x2_t b2 =
215 vtrn_u32(vreinterpret_u32_u8(*a2), vreinterpret_u32_u8(a6));
216 const uint32x2x2_t b3 =
217 vtrn_u32(vreinterpret_u32_u8(*a3), vreinterpret_u32_u8(a7));
218
219 // Swap 16 bit elements resulting in:
220 // c0.val[0]: 00 01 20 21 40 41 60 61
221 // c0.val[1]: 02 03 22 23 42 43 62 63
222 // c1.val[0]: 10 11 30 31 50 51 70 71
223 // c1.val[1]: 12 13 32 33 52 53 72 73
224
225 const uint16x4x2_t c0 = vtrn_u16(vreinterpret_u16_u32(b0.val[0]),
226 vreinterpret_u16_u32(b2.val[0]));
227 const uint16x4x2_t c1 = vtrn_u16(vreinterpret_u16_u32(b1.val[0]),
228 vreinterpret_u16_u32(b3.val[0]));
229
230 // Swap 8 bit elements resulting in:
231 // d0.val[0]: 00 10 20 30 40 50 60 70
232 // d0.val[1]: 01 11 21 31 41 51 61 71
233 // d1.val[0]: 02 12 22 32 42 52 62 72
234 // d1.val[1]: 03 13 23 33 43 53 63 73
235
236 const uint8x8x2_t d0 =
237 vtrn_u8(vreinterpret_u8_u16(c0.val[0]), vreinterpret_u8_u16(c1.val[0]));
238 const uint8x8x2_t d1 =
239 vtrn_u8(vreinterpret_u8_u16(c0.val[1]), vreinterpret_u8_u16(c1.val[1]));
240
241 *a0 = d0.val[0];
242 *a1 = d0.val[1];
243 *a2 = d1.val[0];
244 *a3 = d1.val[1];
245 }
246
transpose_s32_4x4(int32x4_t * a0,int32x4_t * a1,int32x4_t * a2,int32x4_t * a3)247 static INLINE void transpose_s32_4x4(int32x4_t *a0, int32x4_t *a1,
248 int32x4_t *a2, int32x4_t *a3) {
249 // Swap 32 bit elements. Goes from:
250 // a0: 00 01 02 03
251 // a1: 10 11 12 13
252 // a2: 20 21 22 23
253 // a3: 30 31 32 33
254 // to:
255 // b0.val[0]: 00 10 02 12
256 // b0.val[1]: 01 11 03 13
257 // b1.val[0]: 20 30 22 32
258 // b1.val[1]: 21 31 23 33
259
260 const int32x4x2_t b0 = vtrnq_s32(*a0, *a1);
261 const int32x4x2_t b1 = vtrnq_s32(*a2, *a3);
262
263 // Swap 64 bit elements resulting in:
264 // c0.val[0]: 00 10 20 30
265 // c0.val[1]: 02 12 22 32
266 // c1.val[0]: 01 11 21 31
267 // c1.val[1]: 03 13 23 33
268
269 const int32x4x2_t c0 = vpx_vtrnq_s64_to_s32(b0.val[0], b1.val[0]);
270 const int32x4x2_t c1 = vpx_vtrnq_s64_to_s32(b0.val[1], b1.val[1]);
271
272 *a0 = c0.val[0];
273 *a1 = c1.val[0];
274 *a2 = c0.val[1];
275 *a3 = c1.val[1];
276 }
277
transpose_s16_4x8(const int16x4_t a0,const int16x4_t a1,const int16x4_t a2,const int16x4_t a3,const int16x4_t a4,const int16x4_t a5,const int16x4_t a6,const int16x4_t a7,int16x8_t * const o0,int16x8_t * const o1,int16x8_t * const o2,int16x8_t * const o3)278 static INLINE void transpose_s16_4x8(const int16x4_t a0, const int16x4_t a1,
279 const int16x4_t a2, const int16x4_t a3,
280 const int16x4_t a4, const int16x4_t a5,
281 const int16x4_t a6, const int16x4_t a7,
282 int16x8_t *const o0, int16x8_t *const o1,
283 int16x8_t *const o2, int16x8_t *const o3) {
284 // Swap 16 bit elements. Goes from:
285 // a0: 00 01 02 03
286 // a1: 10 11 12 13
287 // a2: 20 21 22 23
288 // a3: 30 31 32 33
289 // a4: 40 41 42 43
290 // a5: 50 51 52 53
291 // a6: 60 61 62 63
292 // a7: 70 71 72 73
293 // to:
294 // b0.val[0]: 00 10 02 12
295 // b0.val[1]: 01 11 03 13
296 // b1.val[0]: 20 30 22 32
297 // b1.val[1]: 21 31 23 33
298 // b2.val[0]: 40 50 42 52
299 // b2.val[1]: 41 51 43 53
300 // b3.val[0]: 60 70 62 72
301 // b3.val[1]: 61 71 63 73
302
303 const int16x4x2_t b0 = vtrn_s16(a0, a1);
304 const int16x4x2_t b1 = vtrn_s16(a2, a3);
305 const int16x4x2_t b2 = vtrn_s16(a4, a5);
306 const int16x4x2_t b3 = vtrn_s16(a6, a7);
307
308 // Swap 32 bit elements resulting in:
309 // c0.val[0]: 00 10 20 30
310 // c0.val[1]: 02 12 22 32
311 // c1.val[0]: 01 11 21 31
312 // c1.val[1]: 03 13 23 33
313 // c2.val[0]: 40 50 60 70
314 // c2.val[1]: 42 52 62 72
315 // c3.val[0]: 41 51 61 71
316 // c3.val[1]: 43 53 63 73
317
318 const int32x2x2_t c0 = vtrn_s32(vreinterpret_s32_s16(b0.val[0]),
319 vreinterpret_s32_s16(b1.val[0]));
320 const int32x2x2_t c1 = vtrn_s32(vreinterpret_s32_s16(b0.val[1]),
321 vreinterpret_s32_s16(b1.val[1]));
322 const int32x2x2_t c2 = vtrn_s32(vreinterpret_s32_s16(b2.val[0]),
323 vreinterpret_s32_s16(b3.val[0]));
324 const int32x2x2_t c3 = vtrn_s32(vreinterpret_s32_s16(b2.val[1]),
325 vreinterpret_s32_s16(b3.val[1]));
326
327 // Swap 64 bit elements resulting in:
328 // o0: 00 10 20 30 40 50 60 70
329 // o1: 01 11 21 31 41 51 61 71
330 // o2: 02 12 22 32 42 52 62 72
331 // o3: 03 13 23 33 43 53 63 73
332
333 *o0 = vcombine_s16(vreinterpret_s16_s32(c0.val[0]),
334 vreinterpret_s16_s32(c2.val[0]));
335 *o1 = vcombine_s16(vreinterpret_s16_s32(c1.val[0]),
336 vreinterpret_s16_s32(c3.val[0]));
337 *o2 = vcombine_s16(vreinterpret_s16_s32(c0.val[1]),
338 vreinterpret_s16_s32(c2.val[1]));
339 *o3 = vcombine_s16(vreinterpret_s16_s32(c1.val[1]),
340 vreinterpret_s16_s32(c3.val[1]));
341 }
342
transpose_s32_4x8(int32x4_t * const a0,int32x4_t * const a1,int32x4_t * const a2,int32x4_t * const a3,int32x4_t * const a4,int32x4_t * const a5,int32x4_t * const a6,int32x4_t * const a7)343 static INLINE void transpose_s32_4x8(int32x4_t *const a0, int32x4_t *const a1,
344 int32x4_t *const a2, int32x4_t *const a3,
345 int32x4_t *const a4, int32x4_t *const a5,
346 int32x4_t *const a6, int32x4_t *const a7) {
347 // Swap 32 bit elements. Goes from:
348 // a0: 00 01 02 03
349 // a1: 10 11 12 13
350 // a2: 20 21 22 23
351 // a3: 30 31 32 33
352 // a4: 40 41 42 43
353 // a5: 50 51 52 53
354 // a6: 60 61 62 63
355 // a7: 70 71 72 73
356 // to:
357 // b0.val[0]: 00 10 02 12
358 // b0.val[1]: 01 11 03 13
359 // b1.val[0]: 20 30 22 32
360 // b1.val[1]: 21 31 23 33
361 // b2.val[0]: 40 50 42 52
362 // b2.val[1]: 41 51 43 53
363 // b3.val[0]: 60 70 62 72
364 // b3.val[1]: 61 71 63 73
365
366 const int32x4x2_t b0 = vtrnq_s32(*a0, *a1);
367 const int32x4x2_t b1 = vtrnq_s32(*a2, *a3);
368 const int32x4x2_t b2 = vtrnq_s32(*a4, *a5);
369 const int32x4x2_t b3 = vtrnq_s32(*a6, *a7);
370
371 // Swap 64 bit elements resulting in:
372 // c0.val[0]: 00 10 20 30
373 // c0.val[1]: 02 12 22 32
374 // c1.val[0]: 01 11 21 31
375 // c1.val[1]: 03 13 23 33
376 // c2.val[0]: 40 50 60 70
377 // c2.val[1]: 42 52 62 72
378 // c3.val[0]: 41 51 61 71
379 // c3.val[1]: 43 53 63 73
380
381 const int64x2x2_t c0 = vpx_vtrnq_s64(b0.val[0], b1.val[0]);
382 const int64x2x2_t c1 = vpx_vtrnq_s64(b0.val[1], b1.val[1]);
383 const int64x2x2_t c2 = vpx_vtrnq_s64(b2.val[0], b3.val[0]);
384 const int64x2x2_t c3 = vpx_vtrnq_s64(b2.val[1], b3.val[1]);
385
386 *a0 = vreinterpretq_s32_s64(c0.val[0]);
387 *a1 = vreinterpretq_s32_s64(c2.val[0]);
388 *a2 = vreinterpretq_s32_s64(c1.val[0]);
389 *a3 = vreinterpretq_s32_s64(c3.val[0]);
390 *a4 = vreinterpretq_s32_s64(c0.val[1]);
391 *a5 = vreinterpretq_s32_s64(c2.val[1]);
392 *a6 = vreinterpretq_s32_s64(c1.val[1]);
393 *a7 = vreinterpretq_s32_s64(c3.val[1]);
394 }
395
transpose_u8_8x4(uint8x8_t * a0,uint8x8_t * a1,uint8x8_t * a2,uint8x8_t * a3)396 static INLINE void transpose_u8_8x4(uint8x8_t *a0, uint8x8_t *a1, uint8x8_t *a2,
397 uint8x8_t *a3) {
398 // Swap 8 bit elements. Goes from:
399 // a0: 00 01 02 03 04 05 06 07
400 // a1: 10 11 12 13 14 15 16 17
401 // a2: 20 21 22 23 24 25 26 27
402 // a3: 30 31 32 33 34 35 36 37
403 // to:
404 // b0.val[0]: 00 10 02 12 04 14 06 16
405 // b0.val[1]: 01 11 03 13 05 15 07 17
406 // b1.val[0]: 20 30 22 32 24 34 26 36
407 // b1.val[1]: 21 31 23 33 25 35 27 37
408
409 const uint8x8x2_t b0 = vtrn_u8(*a0, *a1);
410 const uint8x8x2_t b1 = vtrn_u8(*a2, *a3);
411
412 // Swap 16 bit elements resulting in:
413 // c0.val[0]: 00 10 20 30 04 14 24 34
414 // c0.val[1]: 02 12 22 32 06 16 26 36
415 // c1.val[0]: 01 11 21 31 05 15 25 35
416 // c1.val[1]: 03 13 23 33 07 17 27 37
417
418 const uint16x4x2_t c0 =
419 vtrn_u16(vreinterpret_u16_u8(b0.val[0]), vreinterpret_u16_u8(b1.val[0]));
420 const uint16x4x2_t c1 =
421 vtrn_u16(vreinterpret_u16_u8(b0.val[1]), vreinterpret_u16_u8(b1.val[1]));
422
423 *a0 = vreinterpret_u8_u16(c0.val[0]);
424 *a1 = vreinterpret_u8_u16(c1.val[0]);
425 *a2 = vreinterpret_u8_u16(c0.val[1]);
426 *a3 = vreinterpret_u8_u16(c1.val[1]);
427 }
428
transpose_u16_8x4(uint16x8_t * a0,uint16x8_t * a1,uint16x8_t * a2,uint16x8_t * a3)429 static INLINE void transpose_u16_8x4(uint16x8_t *a0, uint16x8_t *a1,
430 uint16x8_t *a2, uint16x8_t *a3) {
431 // Swap 16 bit elements. Goes from:
432 // a0: 00 01 02 03 04 05 06 07
433 // a1: 10 11 12 13 14 15 16 17
434 // a2: 20 21 22 23 24 25 26 27
435 // a3: 30 31 32 33 34 35 36 37
436 // to:
437 // b0.val[0]: 00 10 02 12 04 14 06 16
438 // b0.val[1]: 01 11 03 13 05 15 07 17
439 // b1.val[0]: 20 30 22 32 24 34 26 36
440 // b1.val[1]: 21 31 23 33 25 35 27 37
441
442 const uint16x8x2_t b0 = vtrnq_u16(*a0, *a1);
443 const uint16x8x2_t b1 = vtrnq_u16(*a2, *a3);
444
445 // Swap 32 bit elements resulting in:
446 // c0.val[0]: 00 10 20 30 04 14 24 34
447 // c0.val[1]: 02 12 22 32 06 16 26 36
448 // c1.val[0]: 01 11 21 31 05 15 25 35
449 // c1.val[1]: 03 13 23 33 07 17 27 37
450
451 const uint32x4x2_t c0 = vtrnq_u32(vreinterpretq_u32_u16(b0.val[0]),
452 vreinterpretq_u32_u16(b1.val[0]));
453 const uint32x4x2_t c1 = vtrnq_u32(vreinterpretq_u32_u16(b0.val[1]),
454 vreinterpretq_u32_u16(b1.val[1]));
455
456 *a0 = vreinterpretq_u16_u32(c0.val[0]);
457 *a1 = vreinterpretq_u16_u32(c1.val[0]);
458 *a2 = vreinterpretq_u16_u32(c0.val[1]);
459 *a3 = vreinterpretq_u16_u32(c1.val[1]);
460 }
461
transpose_s32_8x4(int32x4_t * const a0,int32x4_t * const a1,int32x4_t * const a2,int32x4_t * const a3,int32x4_t * const a4,int32x4_t * const a5,int32x4_t * const a6,int32x4_t * const a7)462 static INLINE void transpose_s32_8x4(int32x4_t *const a0, int32x4_t *const a1,
463 int32x4_t *const a2, int32x4_t *const a3,
464 int32x4_t *const a4, int32x4_t *const a5,
465 int32x4_t *const a6, int32x4_t *const a7) {
466 // Swap 32 bit elements. Goes from:
467 // a0: 00 01 02 03
468 // a1: 04 05 06 07
469 // a2: 10 11 12 13
470 // a3: 14 15 16 17
471 // a4: 20 21 22 23
472 // a5: 24 25 26 27
473 // a6: 30 31 32 33
474 // a7: 34 35 36 37
475 // to:
476 // b0.val[0]: 00 10 02 12
477 // b0.val[1]: 01 11 03 13
478 // b1.val[0]: 04 14 06 16
479 // b1.val[1]: 05 15 07 17
480 // b2.val[0]: 20 30 22 32
481 // b2.val[1]: 21 31 23 33
482 // b3.val[0]: 24 34 26 36
483 // b3.val[1]: 25 35 27 37
484
485 const int32x4x2_t b0 = vtrnq_s32(*a0, *a2);
486 const int32x4x2_t b1 = vtrnq_s32(*a1, *a3);
487 const int32x4x2_t b2 = vtrnq_s32(*a4, *a6);
488 const int32x4x2_t b3 = vtrnq_s32(*a5, *a7);
489
490 // Swap 64 bit elements resulting in:
491 // c0.val[0]: 00 10 20 30
492 // c0.val[1]: 02 12 22 32
493 // c1.val[0]: 01 11 21 31
494 // c1.val[1]: 03 13 23 33
495 // c2.val[0]: 04 14 24 34
496 // c2.val[1]: 06 16 26 36
497 // c3.val[0]: 05 15 25 35
498 // c3.val[1]: 07 17 27 37
499
500 const int64x2x2_t c0 = vpx_vtrnq_s64(b0.val[0], b2.val[0]);
501 const int64x2x2_t c1 = vpx_vtrnq_s64(b0.val[1], b2.val[1]);
502 const int64x2x2_t c2 = vpx_vtrnq_s64(b1.val[0], b3.val[0]);
503 const int64x2x2_t c3 = vpx_vtrnq_s64(b1.val[1], b3.val[1]);
504
505 *a0 = vreinterpretq_s32_s64(c0.val[0]);
506 *a1 = vreinterpretq_s32_s64(c1.val[0]);
507 *a2 = vreinterpretq_s32_s64(c0.val[1]);
508 *a3 = vreinterpretq_s32_s64(c1.val[1]);
509 *a4 = vreinterpretq_s32_s64(c2.val[0]);
510 *a5 = vreinterpretq_s32_s64(c3.val[0]);
511 *a6 = vreinterpretq_s32_s64(c2.val[1]);
512 *a7 = vreinterpretq_s32_s64(c3.val[1]);
513 }
514
515 // Note: Using 'd' registers or 'q' registers has almost identical speed. We use
516 // 'q' registers here to save some instructions.
transpose_u8_8x8(uint8x8_t * a0,uint8x8_t * a1,uint8x8_t * a2,uint8x8_t * a3,uint8x8_t * a4,uint8x8_t * a5,uint8x8_t * a6,uint8x8_t * a7)517 static INLINE void transpose_u8_8x8(uint8x8_t *a0, uint8x8_t *a1, uint8x8_t *a2,
518 uint8x8_t *a3, uint8x8_t *a4, uint8x8_t *a5,
519 uint8x8_t *a6, uint8x8_t *a7) {
520 // Swap 8 bit elements. Goes from:
521 // a0: 00 01 02 03 04 05 06 07
522 // a1: 10 11 12 13 14 15 16 17
523 // a2: 20 21 22 23 24 25 26 27
524 // a3: 30 31 32 33 34 35 36 37
525 // a4: 40 41 42 43 44 45 46 47
526 // a5: 50 51 52 53 54 55 56 57
527 // a6: 60 61 62 63 64 65 66 67
528 // a7: 70 71 72 73 74 75 76 77
529 // to:
530 // b0.val[0]: 00 10 02 12 04 14 06 16 40 50 42 52 44 54 46 56
531 // b0.val[1]: 01 11 03 13 05 15 07 17 41 51 43 53 45 55 47 57
532 // b1.val[0]: 20 30 22 32 24 34 26 36 60 70 62 72 64 74 66 76
533 // b1.val[1]: 21 31 23 33 25 35 27 37 61 71 63 73 65 75 67 77
534
535 const uint8x16x2_t b0 =
536 vtrnq_u8(vcombine_u8(*a0, *a4), vcombine_u8(*a1, *a5));
537 const uint8x16x2_t b1 =
538 vtrnq_u8(vcombine_u8(*a2, *a6), vcombine_u8(*a3, *a7));
539
540 // Swap 16 bit elements resulting in:
541 // c0.val[0]: 00 10 20 30 04 14 24 34 40 50 60 70 44 54 64 74
542 // c0.val[1]: 02 12 22 32 06 16 26 36 42 52 62 72 46 56 66 76
543 // c1.val[0]: 01 11 21 31 05 15 25 35 41 51 61 71 45 55 65 75
544 // c1.val[1]: 03 13 23 33 07 17 27 37 43 53 63 73 47 57 67 77
545
546 const uint16x8x2_t c0 = vtrnq_u16(vreinterpretq_u16_u8(b0.val[0]),
547 vreinterpretq_u16_u8(b1.val[0]));
548 const uint16x8x2_t c1 = vtrnq_u16(vreinterpretq_u16_u8(b0.val[1]),
549 vreinterpretq_u16_u8(b1.val[1]));
550
551 // Unzip 32 bit elements resulting in:
552 // d0.val[0]: 00 10 20 30 40 50 60 70 01 11 21 31 41 51 61 71
553 // d0.val[1]: 04 14 24 34 44 54 64 74 05 15 25 35 45 55 65 75
554 // d1.val[0]: 02 12 22 32 42 52 62 72 03 13 23 33 43 53 63 73
555 // d1.val[1]: 06 16 26 36 46 56 66 76 07 17 27 37 47 57 67 77
556 const uint32x4x2_t d0 = vuzpq_u32(vreinterpretq_u32_u16(c0.val[0]),
557 vreinterpretq_u32_u16(c1.val[0]));
558 const uint32x4x2_t d1 = vuzpq_u32(vreinterpretq_u32_u16(c0.val[1]),
559 vreinterpretq_u32_u16(c1.val[1]));
560
561 *a0 = vreinterpret_u8_u32(vget_low_u32(d0.val[0]));
562 *a1 = vreinterpret_u8_u32(vget_high_u32(d0.val[0]));
563 *a2 = vreinterpret_u8_u32(vget_low_u32(d1.val[0]));
564 *a3 = vreinterpret_u8_u32(vget_high_u32(d1.val[0]));
565 *a4 = vreinterpret_u8_u32(vget_low_u32(d0.val[1]));
566 *a5 = vreinterpret_u8_u32(vget_high_u32(d0.val[1]));
567 *a6 = vreinterpret_u8_u32(vget_low_u32(d1.val[1]));
568 *a7 = vreinterpret_u8_u32(vget_high_u32(d1.val[1]));
569 }
570
571 // Transpose 8x8 to a new location.
transpose_s16_8x8_new(const int16x8_t * a,int16x8_t * b)572 static INLINE void transpose_s16_8x8_new(const int16x8_t *a, int16x8_t *b) {
573 // Swap 16 bit elements.
574 const int16x8x2_t c0 = vtrnq_s16(a[0], a[1]);
575 const int16x8x2_t c1 = vtrnq_s16(a[2], a[3]);
576 const int16x8x2_t c2 = vtrnq_s16(a[4], a[5]);
577 const int16x8x2_t c3 = vtrnq_s16(a[6], a[7]);
578
579 // Swap 32 bit elements.
580 const int32x4x2_t d0 = vtrnq_s32(vreinterpretq_s32_s16(c0.val[0]),
581 vreinterpretq_s32_s16(c1.val[0]));
582 const int32x4x2_t d1 = vtrnq_s32(vreinterpretq_s32_s16(c0.val[1]),
583 vreinterpretq_s32_s16(c1.val[1]));
584 const int32x4x2_t d2 = vtrnq_s32(vreinterpretq_s32_s16(c2.val[0]),
585 vreinterpretq_s32_s16(c3.val[0]));
586 const int32x4x2_t d3 = vtrnq_s32(vreinterpretq_s32_s16(c2.val[1]),
587 vreinterpretq_s32_s16(c3.val[1]));
588
589 // Swap 64 bit elements
590 const int16x8x2_t e0 = vpx_vtrnq_s64_to_s16(d0.val[0], d2.val[0]);
591 const int16x8x2_t e1 = vpx_vtrnq_s64_to_s16(d1.val[0], d3.val[0]);
592 const int16x8x2_t e2 = vpx_vtrnq_s64_to_s16(d0.val[1], d2.val[1]);
593 const int16x8x2_t e3 = vpx_vtrnq_s64_to_s16(d1.val[1], d3.val[1]);
594
595 b[0] = e0.val[0];
596 b[1] = e1.val[0];
597 b[2] = e2.val[0];
598 b[3] = e3.val[0];
599 b[4] = e0.val[1];
600 b[5] = e1.val[1];
601 b[6] = e2.val[1];
602 b[7] = e3.val[1];
603 }
604
transpose_s16_8x8(int16x8_t * a0,int16x8_t * a1,int16x8_t * a2,int16x8_t * a3,int16x8_t * a4,int16x8_t * a5,int16x8_t * a6,int16x8_t * a7)605 static INLINE void transpose_s16_8x8(int16x8_t *a0, int16x8_t *a1,
606 int16x8_t *a2, int16x8_t *a3,
607 int16x8_t *a4, int16x8_t *a5,
608 int16x8_t *a6, int16x8_t *a7) {
609 // Swap 16 bit elements. Goes from:
610 // a0: 00 01 02 03 04 05 06 07
611 // a1: 10 11 12 13 14 15 16 17
612 // a2: 20 21 22 23 24 25 26 27
613 // a3: 30 31 32 33 34 35 36 37
614 // a4: 40 41 42 43 44 45 46 47
615 // a5: 50 51 52 53 54 55 56 57
616 // a6: 60 61 62 63 64 65 66 67
617 // a7: 70 71 72 73 74 75 76 77
618 // to:
619 // b0.val[0]: 00 10 02 12 04 14 06 16
620 // b0.val[1]: 01 11 03 13 05 15 07 17
621 // b1.val[0]: 20 30 22 32 24 34 26 36
622 // b1.val[1]: 21 31 23 33 25 35 27 37
623 // b2.val[0]: 40 50 42 52 44 54 46 56
624 // b2.val[1]: 41 51 43 53 45 55 47 57
625 // b3.val[0]: 60 70 62 72 64 74 66 76
626 // b3.val[1]: 61 71 63 73 65 75 67 77
627
628 const int16x8x2_t b0 = vtrnq_s16(*a0, *a1);
629 const int16x8x2_t b1 = vtrnq_s16(*a2, *a3);
630 const int16x8x2_t b2 = vtrnq_s16(*a4, *a5);
631 const int16x8x2_t b3 = vtrnq_s16(*a6, *a7);
632
633 // Swap 32 bit elements resulting in:
634 // c0.val[0]: 00 10 20 30 04 14 24 34
635 // c0.val[1]: 02 12 22 32 06 16 26 36
636 // c1.val[0]: 01 11 21 31 05 15 25 35
637 // c1.val[1]: 03 13 23 33 07 17 27 37
638 // c2.val[0]: 40 50 60 70 44 54 64 74
639 // c2.val[1]: 42 52 62 72 46 56 66 76
640 // c3.val[0]: 41 51 61 71 45 55 65 75
641 // c3.val[1]: 43 53 63 73 47 57 67 77
642
643 const int32x4x2_t c0 = vtrnq_s32(vreinterpretq_s32_s16(b0.val[0]),
644 vreinterpretq_s32_s16(b1.val[0]));
645 const int32x4x2_t c1 = vtrnq_s32(vreinterpretq_s32_s16(b0.val[1]),
646 vreinterpretq_s32_s16(b1.val[1]));
647 const int32x4x2_t c2 = vtrnq_s32(vreinterpretq_s32_s16(b2.val[0]),
648 vreinterpretq_s32_s16(b3.val[0]));
649 const int32x4x2_t c3 = vtrnq_s32(vreinterpretq_s32_s16(b2.val[1]),
650 vreinterpretq_s32_s16(b3.val[1]));
651
652 // Swap 64 bit elements resulting in:
653 // d0.val[0]: 00 10 20 30 40 50 60 70
654 // d0.val[1]: 04 14 24 34 44 54 64 74
655 // d1.val[0]: 01 11 21 31 41 51 61 71
656 // d1.val[1]: 05 15 25 35 45 55 65 75
657 // d2.val[0]: 02 12 22 32 42 52 62 72
658 // d2.val[1]: 06 16 26 36 46 56 66 76
659 // d3.val[0]: 03 13 23 33 43 53 63 73
660 // d3.val[1]: 07 17 27 37 47 57 67 77
661 const int16x8x2_t d0 = vpx_vtrnq_s64_to_s16(c0.val[0], c2.val[0]);
662 const int16x8x2_t d1 = vpx_vtrnq_s64_to_s16(c1.val[0], c3.val[0]);
663 const int16x8x2_t d2 = vpx_vtrnq_s64_to_s16(c0.val[1], c2.val[1]);
664 const int16x8x2_t d3 = vpx_vtrnq_s64_to_s16(c1.val[1], c3.val[1]);
665
666 *a0 = d0.val[0];
667 *a1 = d1.val[0];
668 *a2 = d2.val[0];
669 *a3 = d3.val[0];
670 *a4 = d0.val[1];
671 *a5 = d1.val[1];
672 *a6 = d2.val[1];
673 *a7 = d3.val[1];
674 }
675
transpose_u16_8x8(uint16x8_t * a0,uint16x8_t * a1,uint16x8_t * a2,uint16x8_t * a3,uint16x8_t * a4,uint16x8_t * a5,uint16x8_t * a6,uint16x8_t * a7)676 static INLINE void transpose_u16_8x8(uint16x8_t *a0, uint16x8_t *a1,
677 uint16x8_t *a2, uint16x8_t *a3,
678 uint16x8_t *a4, uint16x8_t *a5,
679 uint16x8_t *a6, uint16x8_t *a7) {
680 // Swap 16 bit elements. Goes from:
681 // a0: 00 01 02 03 04 05 06 07
682 // a1: 10 11 12 13 14 15 16 17
683 // a2: 20 21 22 23 24 25 26 27
684 // a3: 30 31 32 33 34 35 36 37
685 // a4: 40 41 42 43 44 45 46 47
686 // a5: 50 51 52 53 54 55 56 57
687 // a6: 60 61 62 63 64 65 66 67
688 // a7: 70 71 72 73 74 75 76 77
689 // to:
690 // b0.val[0]: 00 10 02 12 04 14 06 16
691 // b0.val[1]: 01 11 03 13 05 15 07 17
692 // b1.val[0]: 20 30 22 32 24 34 26 36
693 // b1.val[1]: 21 31 23 33 25 35 27 37
694 // b2.val[0]: 40 50 42 52 44 54 46 56
695 // b2.val[1]: 41 51 43 53 45 55 47 57
696 // b3.val[0]: 60 70 62 72 64 74 66 76
697 // b3.val[1]: 61 71 63 73 65 75 67 77
698
699 const uint16x8x2_t b0 = vtrnq_u16(*a0, *a1);
700 const uint16x8x2_t b1 = vtrnq_u16(*a2, *a3);
701 const uint16x8x2_t b2 = vtrnq_u16(*a4, *a5);
702 const uint16x8x2_t b3 = vtrnq_u16(*a6, *a7);
703
704 // Swap 32 bit elements resulting in:
705 // c0.val[0]: 00 10 20 30 04 14 24 34
706 // c0.val[1]: 02 12 22 32 06 16 26 36
707 // c1.val[0]: 01 11 21 31 05 15 25 35
708 // c1.val[1]: 03 13 23 33 07 17 27 37
709 // c2.val[0]: 40 50 60 70 44 54 64 74
710 // c2.val[1]: 42 52 62 72 46 56 66 76
711 // c3.val[0]: 41 51 61 71 45 55 65 75
712 // c3.val[1]: 43 53 63 73 47 57 67 77
713
714 const uint32x4x2_t c0 = vtrnq_u32(vreinterpretq_u32_u16(b0.val[0]),
715 vreinterpretq_u32_u16(b1.val[0]));
716 const uint32x4x2_t c1 = vtrnq_u32(vreinterpretq_u32_u16(b0.val[1]),
717 vreinterpretq_u32_u16(b1.val[1]));
718 const uint32x4x2_t c2 = vtrnq_u32(vreinterpretq_u32_u16(b2.val[0]),
719 vreinterpretq_u32_u16(b3.val[0]));
720 const uint32x4x2_t c3 = vtrnq_u32(vreinterpretq_u32_u16(b2.val[1]),
721 vreinterpretq_u32_u16(b3.val[1]));
722
723 // Swap 64 bit elements resulting in:
724 // d0.val[0]: 00 10 20 30 40 50 60 70
725 // d0.val[1]: 04 14 24 34 44 54 64 74
726 // d1.val[0]: 01 11 21 31 41 51 61 71
727 // d1.val[1]: 05 15 25 35 45 55 65 75
728 // d2.val[0]: 02 12 22 32 42 52 62 72
729 // d2.val[1]: 06 16 26 36 46 56 66 76
730 // d3.val[0]: 03 13 23 33 43 53 63 73
731 // d3.val[1]: 07 17 27 37 47 57 67 77
732 const uint16x8x2_t d0 = vpx_vtrnq_u64_to_u16(c0.val[0], c2.val[0]);
733 const uint16x8x2_t d1 = vpx_vtrnq_u64_to_u16(c1.val[0], c3.val[0]);
734 const uint16x8x2_t d2 = vpx_vtrnq_u64_to_u16(c0.val[1], c2.val[1]);
735 const uint16x8x2_t d3 = vpx_vtrnq_u64_to_u16(c1.val[1], c3.val[1]);
736
737 *a0 = d0.val[0];
738 *a1 = d1.val[0];
739 *a2 = d2.val[0];
740 *a3 = d3.val[0];
741 *a4 = d0.val[1];
742 *a5 = d1.val[1];
743 *a6 = d2.val[1];
744 *a7 = d3.val[1];
745 }
746
transpose_s32_8x8(int32x4x2_t * a0,int32x4x2_t * a1,int32x4x2_t * a2,int32x4x2_t * a3,int32x4x2_t * a4,int32x4x2_t * a5,int32x4x2_t * a6,int32x4x2_t * a7)747 static INLINE void transpose_s32_8x8(int32x4x2_t *a0, int32x4x2_t *a1,
748 int32x4x2_t *a2, int32x4x2_t *a3,
749 int32x4x2_t *a4, int32x4x2_t *a5,
750 int32x4x2_t *a6, int32x4x2_t *a7) {
751 // Swap 32 bit elements. Goes from:
752 // a0: 00 01 02 03 04 05 06 07
753 // a1: 10 11 12 13 14 15 16 17
754 // a2: 20 21 22 23 24 25 26 27
755 // a3: 30 31 32 33 34 35 36 37
756 // a4: 40 41 42 43 44 45 46 47
757 // a5: 50 51 52 53 54 55 56 57
758 // a6: 60 61 62 63 64 65 66 67
759 // a7: 70 71 72 73 74 75 76 77
760 // to:
761 // b0: 00 10 02 12 01 11 03 13
762 // b1: 20 30 22 32 21 31 23 33
763 // b2: 40 50 42 52 41 51 43 53
764 // b3: 60 70 62 72 61 71 63 73
765 // b4: 04 14 06 16 05 15 07 17
766 // b5: 24 34 26 36 25 35 27 37
767 // b6: 44 54 46 56 45 55 47 57
768 // b7: 64 74 66 76 65 75 67 77
769
770 const int32x4x2_t b0 = vtrnq_s32(a0->val[0], a1->val[0]);
771 const int32x4x2_t b1 = vtrnq_s32(a2->val[0], a3->val[0]);
772 const int32x4x2_t b2 = vtrnq_s32(a4->val[0], a5->val[0]);
773 const int32x4x2_t b3 = vtrnq_s32(a6->val[0], a7->val[0]);
774 const int32x4x2_t b4 = vtrnq_s32(a0->val[1], a1->val[1]);
775 const int32x4x2_t b5 = vtrnq_s32(a2->val[1], a3->val[1]);
776 const int32x4x2_t b6 = vtrnq_s32(a4->val[1], a5->val[1]);
777 const int32x4x2_t b7 = vtrnq_s32(a6->val[1], a7->val[1]);
778
779 // Swap 64 bit elements resulting in:
780 // c0: 00 10 20 30 02 12 22 32
781 // c1: 01 11 21 31 03 13 23 33
782 // c2: 40 50 60 70 42 52 62 72
783 // c3: 41 51 61 71 43 53 63 73
784 // c4: 04 14 24 34 06 16 26 36
785 // c5: 05 15 25 35 07 17 27 37
786 // c6: 44 54 64 74 46 56 66 76
787 // c7: 45 55 65 75 47 57 67 77
788 const int32x4x2_t c0 = vpx_vtrnq_s64_to_s32(b0.val[0], b1.val[0]);
789 const int32x4x2_t c1 = vpx_vtrnq_s64_to_s32(b0.val[1], b1.val[1]);
790 const int32x4x2_t c2 = vpx_vtrnq_s64_to_s32(b2.val[0], b3.val[0]);
791 const int32x4x2_t c3 = vpx_vtrnq_s64_to_s32(b2.val[1], b3.val[1]);
792 const int32x4x2_t c4 = vpx_vtrnq_s64_to_s32(b4.val[0], b5.val[0]);
793 const int32x4x2_t c5 = vpx_vtrnq_s64_to_s32(b4.val[1], b5.val[1]);
794 const int32x4x2_t c6 = vpx_vtrnq_s64_to_s32(b6.val[0], b7.val[0]);
795 const int32x4x2_t c7 = vpx_vtrnq_s64_to_s32(b6.val[1], b7.val[1]);
796
797 // Swap 128 bit elements resulting in:
798 // a0: 00 10 20 30 40 50 60 70
799 // a1: 01 11 21 31 41 51 61 71
800 // a2: 02 12 22 32 42 52 62 72
801 // a3: 03 13 23 33 43 53 63 73
802 // a4: 04 14 24 34 44 54 64 74
803 // a5: 05 15 25 35 45 55 65 75
804 // a6: 06 16 26 36 46 56 66 76
805 // a7: 07 17 27 37 47 57 67 77
806 a0->val[0] = c0.val[0];
807 a0->val[1] = c2.val[0];
808 a1->val[0] = c1.val[0];
809 a1->val[1] = c3.val[0];
810 a2->val[0] = c0.val[1];
811 a2->val[1] = c2.val[1];
812 a3->val[0] = c1.val[1];
813 a3->val[1] = c3.val[1];
814 a4->val[0] = c4.val[0];
815 a4->val[1] = c6.val[0];
816 a5->val[0] = c5.val[0];
817 a5->val[1] = c7.val[0];
818 a6->val[0] = c4.val[1];
819 a6->val[1] = c6.val[1];
820 a7->val[0] = c5.val[1];
821 a7->val[1] = c7.val[1];
822 }
823
824 // Helper transpose function for highbd FDCT variants
transpose_s32_8x8_2(int32x4_t * left,int32x4_t * right,int32x4_t * out_left,int32x4_t * out_right)825 static INLINE void transpose_s32_8x8_2(int32x4_t *left /*[8]*/,
826 int32x4_t *right /*[8]*/,
827 int32x4_t *out_left /*[8]*/,
828 int32x4_t *out_right /*[8]*/) {
829 int32x4x2_t out[8];
830
831 out[0].val[0] = left[0];
832 out[0].val[1] = right[0];
833 out[1].val[0] = left[1];
834 out[1].val[1] = right[1];
835 out[2].val[0] = left[2];
836 out[2].val[1] = right[2];
837 out[3].val[0] = left[3];
838 out[3].val[1] = right[3];
839 out[4].val[0] = left[4];
840 out[4].val[1] = right[4];
841 out[5].val[0] = left[5];
842 out[5].val[1] = right[5];
843 out[6].val[0] = left[6];
844 out[6].val[1] = right[6];
845 out[7].val[0] = left[7];
846 out[7].val[1] = right[7];
847
848 transpose_s32_8x8(&out[0], &out[1], &out[2], &out[3], &out[4], &out[5],
849 &out[6], &out[7]);
850
851 out_left[0] = out[0].val[0];
852 out_left[1] = out[1].val[0];
853 out_left[2] = out[2].val[0];
854 out_left[3] = out[3].val[0];
855 out_left[4] = out[4].val[0];
856 out_left[5] = out[5].val[0];
857 out_left[6] = out[6].val[0];
858 out_left[7] = out[7].val[0];
859 out_right[0] = out[0].val[1];
860 out_right[1] = out[1].val[1];
861 out_right[2] = out[2].val[1];
862 out_right[3] = out[3].val[1];
863 out_right[4] = out[4].val[1];
864 out_right[5] = out[5].val[1];
865 out_right[6] = out[6].val[1];
866 out_right[7] = out[7].val[1];
867 }
868
transpose_u8_16x8(const uint8x16_t i0,const uint8x16_t i1,const uint8x16_t i2,const uint8x16_t i3,const uint8x16_t i4,const uint8x16_t i5,const uint8x16_t i6,const uint8x16_t i7,uint8x8_t * o0,uint8x8_t * o1,uint8x8_t * o2,uint8x8_t * o3,uint8x8_t * o4,uint8x8_t * o5,uint8x8_t * o6,uint8x8_t * o7,uint8x8_t * o8,uint8x8_t * o9,uint8x8_t * o10,uint8x8_t * o11,uint8x8_t * o12,uint8x8_t * o13,uint8x8_t * o14,uint8x8_t * o15)869 static INLINE void transpose_u8_16x8(
870 const uint8x16_t i0, const uint8x16_t i1, const uint8x16_t i2,
871 const uint8x16_t i3, const uint8x16_t i4, const uint8x16_t i5,
872 const uint8x16_t i6, const uint8x16_t i7, uint8x8_t *o0, uint8x8_t *o1,
873 uint8x8_t *o2, uint8x8_t *o3, uint8x8_t *o4, uint8x8_t *o5, uint8x8_t *o6,
874 uint8x8_t *o7, uint8x8_t *o8, uint8x8_t *o9, uint8x8_t *o10, uint8x8_t *o11,
875 uint8x8_t *o12, uint8x8_t *o13, uint8x8_t *o14, uint8x8_t *o15) {
876 // Swap 8 bit elements. Goes from:
877 // i0: 00 01 02 03 04 05 06 07 08 09 0A 0B 0C 0D 0E 0F
878 // i1: 10 11 12 13 14 15 16 17 18 19 1A 1B 1C 1D 1E 1F
879 // i2: 20 21 22 23 24 25 26 27 28 29 2A 2B 2C 2D 2E 2F
880 // i3: 30 31 32 33 34 35 36 37 38 39 3A 3B 3C 3D 3E 3F
881 // i4: 40 41 42 43 44 45 46 47 48 49 4A 4B 4C 4D 4E 4F
882 // i5: 50 51 52 53 54 55 56 57 58 59 5A 5B 5C 5D 5E 5F
883 // i6: 60 61 62 63 64 65 66 67 68 69 6A 6B 6C 6D 6E 6F
884 // i7: 70 71 72 73 74 75 76 77 78 79 7A 7B 7C 7D 7E 7F
885 // to:
886 // b0.val[0]: 00 10 02 12 04 14 06 16 08 18 0A 1A 0C 1C 0E 1E
887 // b0.val[1]: 01 11 03 13 05 15 07 17 09 19 0B 1B 0D 1D 0F 1F
888 // b1.val[0]: 20 30 22 32 24 34 26 36 28 38 2A 3A 2C 3C 2E 3E
889 // b1.val[1]: 21 31 23 33 25 35 27 37 29 39 2B 3B 2D 3D 2F 3F
890 // b2.val[0]: 40 50 42 52 44 54 46 56 48 58 4A 5A 4C 5C 4E 5E
891 // b2.val[1]: 41 51 43 53 45 55 47 57 49 59 4B 5B 4D 5D 4F 5F
892 // b3.val[0]: 60 70 62 72 64 74 66 76 68 78 6A 7A 6C 7C 6E 7E
893 // b3.val[1]: 61 71 63 73 65 75 67 77 69 79 6B 7B 6D 7D 6F 7F
894 const uint8x16x2_t b0 = vtrnq_u8(i0, i1);
895 const uint8x16x2_t b1 = vtrnq_u8(i2, i3);
896 const uint8x16x2_t b2 = vtrnq_u8(i4, i5);
897 const uint8x16x2_t b3 = vtrnq_u8(i6, i7);
898
899 // Swap 16 bit elements resulting in:
900 // c0.val[0]: 00 10 20 30 04 14 24 34 08 18 28 38 0C 1C 2C 3C
901 // c0.val[1]: 02 12 22 32 06 16 26 36 0A 1A 2A 3A 0E 1E 2E 3E
902 // c1.val[0]: 01 11 21 31 05 15 25 35 09 19 29 39 0D 1D 2D 3D
903 // c1.val[1]: 03 13 23 33 07 17 27 37 0B 1B 2B 3B 0F 1F 2F 3F
904 // c2.val[0]: 40 50 60 70 44 54 64 74 48 58 68 78 4C 5C 6C 7C
905 // c2.val[1]: 42 52 62 72 46 56 66 76 4A 5A 6A 7A 4E 5E 6E 7E
906 // c3.val[0]: 41 51 61 71 45 55 65 75 49 59 69 79 4D 5D 6D 7D
907 // c3.val[1]: 43 53 63 73 47 57 67 77 4B 5B 6B 7B 4F 5F 6F 7F
908 const uint16x8x2_t c0 = vtrnq_u16(vreinterpretq_u16_u8(b0.val[0]),
909 vreinterpretq_u16_u8(b1.val[0]));
910 const uint16x8x2_t c1 = vtrnq_u16(vreinterpretq_u16_u8(b0.val[1]),
911 vreinterpretq_u16_u8(b1.val[1]));
912 const uint16x8x2_t c2 = vtrnq_u16(vreinterpretq_u16_u8(b2.val[0]),
913 vreinterpretq_u16_u8(b3.val[0]));
914 const uint16x8x2_t c3 = vtrnq_u16(vreinterpretq_u16_u8(b2.val[1]),
915 vreinterpretq_u16_u8(b3.val[1]));
916
917 // Swap 32 bit elements resulting in:
918 // d0.val[0]: 00 10 20 30 40 50 60 70 08 18 28 38 48 58 68 78
919 // d0.val[1]: 04 14 24 34 44 54 64 74 0C 1C 2C 3C 4C 5C 6C 7C
920 // d1.val[0]: 02 12 22 32 42 52 62 72 0A 1A 2A 3A 4A 5A 6A 7A
921 // d1.val[1]: 06 16 26 36 46 56 66 76 0E 1E 2E 3E 4E 5E 6E 7E
922 // d2.val[0]: 01 11 21 31 41 51 61 71 09 19 29 39 49 59 69 79
923 // d2.val[1]: 05 15 25 35 45 55 65 75 0D 1D 2D 3D 4D 5D 6D 7D
924 // d3.val[0]: 03 13 23 33 43 53 63 73 0B 1B 2B 3B 4B 5B 6B 7B
925 // d3.val[1]: 07 17 27 37 47 57 67 77 0F 1F 2F 3F 4F 5F 6F 7F
926 const uint32x4x2_t d0 = vtrnq_u32(vreinterpretq_u32_u16(c0.val[0]),
927 vreinterpretq_u32_u16(c2.val[0]));
928 const uint32x4x2_t d1 = vtrnq_u32(vreinterpretq_u32_u16(c0.val[1]),
929 vreinterpretq_u32_u16(c2.val[1]));
930 const uint32x4x2_t d2 = vtrnq_u32(vreinterpretq_u32_u16(c1.val[0]),
931 vreinterpretq_u32_u16(c3.val[0]));
932 const uint32x4x2_t d3 = vtrnq_u32(vreinterpretq_u32_u16(c1.val[1]),
933 vreinterpretq_u32_u16(c3.val[1]));
934
935 // Output:
936 // o0 : 00 10 20 30 40 50 60 70
937 // o1 : 01 11 21 31 41 51 61 71
938 // o2 : 02 12 22 32 42 52 62 72
939 // o3 : 03 13 23 33 43 53 63 73
940 // o4 : 04 14 24 34 44 54 64 74
941 // o5 : 05 15 25 35 45 55 65 75
942 // o6 : 06 16 26 36 46 56 66 76
943 // o7 : 07 17 27 37 47 57 67 77
944 // o8 : 08 18 28 38 48 58 68 78
945 // o9 : 09 19 29 39 49 59 69 79
946 // o10: 0A 1A 2A 3A 4A 5A 6A 7A
947 // o11: 0B 1B 2B 3B 4B 5B 6B 7B
948 // o12: 0C 1C 2C 3C 4C 5C 6C 7C
949 // o13: 0D 1D 2D 3D 4D 5D 6D 7D
950 // o14: 0E 1E 2E 3E 4E 5E 6E 7E
951 // o15: 0F 1F 2F 3F 4F 5F 6F 7F
952 *o0 = vget_low_u8(vreinterpretq_u8_u32(d0.val[0]));
953 *o1 = vget_low_u8(vreinterpretq_u8_u32(d2.val[0]));
954 *o2 = vget_low_u8(vreinterpretq_u8_u32(d1.val[0]));
955 *o3 = vget_low_u8(vreinterpretq_u8_u32(d3.val[0]));
956 *o4 = vget_low_u8(vreinterpretq_u8_u32(d0.val[1]));
957 *o5 = vget_low_u8(vreinterpretq_u8_u32(d2.val[1]));
958 *o6 = vget_low_u8(vreinterpretq_u8_u32(d1.val[1]));
959 *o7 = vget_low_u8(vreinterpretq_u8_u32(d3.val[1]));
960 *o8 = vget_high_u8(vreinterpretq_u8_u32(d0.val[0]));
961 *o9 = vget_high_u8(vreinterpretq_u8_u32(d2.val[0]));
962 *o10 = vget_high_u8(vreinterpretq_u8_u32(d1.val[0]));
963 *o11 = vget_high_u8(vreinterpretq_u8_u32(d3.val[0]));
964 *o12 = vget_high_u8(vreinterpretq_u8_u32(d0.val[1]));
965 *o13 = vget_high_u8(vreinterpretq_u8_u32(d2.val[1]));
966 *o14 = vget_high_u8(vreinterpretq_u8_u32(d1.val[1]));
967 *o15 = vget_high_u8(vreinterpretq_u8_u32(d3.val[1]));
968 }
969
transpose_u8_8x16(const uint8x8_t i0,const uint8x8_t i1,const uint8x8_t i2,const uint8x8_t i3,const uint8x8_t i4,const uint8x8_t i5,const uint8x8_t i6,const uint8x8_t i7,const uint8x8_t i8,const uint8x8_t i9,const uint8x8_t i10,const uint8x8_t i11,const uint8x8_t i12,const uint8x8_t i13,const uint8x8_t i14,const uint8x8_t i15,uint8x16_t * o0,uint8x16_t * o1,uint8x16_t * o2,uint8x16_t * o3,uint8x16_t * o4,uint8x16_t * o5,uint8x16_t * o6,uint8x16_t * o7)970 static INLINE void transpose_u8_8x16(
971 const uint8x8_t i0, const uint8x8_t i1, const uint8x8_t i2,
972 const uint8x8_t i3, const uint8x8_t i4, const uint8x8_t i5,
973 const uint8x8_t i6, const uint8x8_t i7, const uint8x8_t i8,
974 const uint8x8_t i9, const uint8x8_t i10, const uint8x8_t i11,
975 const uint8x8_t i12, const uint8x8_t i13, const uint8x8_t i14,
976 const uint8x8_t i15, uint8x16_t *o0, uint8x16_t *o1, uint8x16_t *o2,
977 uint8x16_t *o3, uint8x16_t *o4, uint8x16_t *o5, uint8x16_t *o6,
978 uint8x16_t *o7) {
979 // Combine 8 bit elements. Goes from:
980 // i0 : 00 01 02 03 04 05 06 07
981 // i1 : 10 11 12 13 14 15 16 17
982 // i2 : 20 21 22 23 24 25 26 27
983 // i3 : 30 31 32 33 34 35 36 37
984 // i4 : 40 41 42 43 44 45 46 47
985 // i5 : 50 51 52 53 54 55 56 57
986 // i6 : 60 61 62 63 64 65 66 67
987 // i7 : 70 71 72 73 74 75 76 77
988 // i8 : 80 81 82 83 84 85 86 87
989 // i9 : 90 91 92 93 94 95 96 97
990 // i10: A0 A1 A2 A3 A4 A5 A6 A7
991 // i11: B0 B1 B2 B3 B4 B5 B6 B7
992 // i12: C0 C1 C2 C3 C4 C5 C6 C7
993 // i13: D0 D1 D2 D3 D4 D5 D6 D7
994 // i14: E0 E1 E2 E3 E4 E5 E6 E7
995 // i15: F0 F1 F2 F3 F4 F5 F6 F7
996 // to:
997 // a0: 00 01 02 03 04 05 06 07 80 81 82 83 84 85 86 87
998 // a1: 10 11 12 13 14 15 16 17 90 91 92 93 94 95 96 97
999 // a2: 20 21 22 23 24 25 26 27 A0 A1 A2 A3 A4 A5 A6 A7
1000 // a3: 30 31 32 33 34 35 36 37 B0 B1 B2 B3 B4 B5 B6 B7
1001 // a4: 40 41 42 43 44 45 46 47 C0 C1 C2 C3 C4 C5 C6 C7
1002 // a5: 50 51 52 53 54 55 56 57 D0 D1 D2 D3 D4 D5 D6 D7
1003 // a6: 60 61 62 63 64 65 66 67 E0 E1 E2 E3 E4 E5 E6 E7
1004 // a7: 70 71 72 73 74 75 76 77 F0 F1 F2 F3 F4 F5 F6 F7
1005 const uint8x16_t a0 = vcombine_u8(i0, i8);
1006 const uint8x16_t a1 = vcombine_u8(i1, i9);
1007 const uint8x16_t a2 = vcombine_u8(i2, i10);
1008 const uint8x16_t a3 = vcombine_u8(i3, i11);
1009 const uint8x16_t a4 = vcombine_u8(i4, i12);
1010 const uint8x16_t a5 = vcombine_u8(i5, i13);
1011 const uint8x16_t a6 = vcombine_u8(i6, i14);
1012 const uint8x16_t a7 = vcombine_u8(i7, i15);
1013
1014 // Swap 8 bit elements resulting in:
1015 // b0.val[0]: 00 10 02 12 04 14 06 16 80 90 82 92 84 94 86 96
1016 // b0.val[1]: 01 11 03 13 05 15 07 17 81 91 83 93 85 95 87 97
1017 // b1.val[0]: 20 30 22 32 24 34 26 36 A0 B0 A2 B2 A4 B4 A6 B6
1018 // b1.val[1]: 21 31 23 33 25 35 27 37 A1 B1 A3 B3 A5 B5 A7 B7
1019 // b2.val[0]: 40 50 42 52 44 54 46 56 C0 D0 C2 D2 C4 D4 C6 D6
1020 // b2.val[1]: 41 51 43 53 45 55 47 57 C1 D1 C3 D3 C5 D5 C7 D7
1021 // b3.val[0]: 60 70 62 72 64 74 66 76 E0 F0 E2 F2 E4 F4 E6 F6
1022 // b3.val[1]: 61 71 63 73 65 75 67 77 E1 F1 E3 F3 E5 F5 E7 F7
1023 const uint8x16x2_t b0 = vtrnq_u8(a0, a1);
1024 const uint8x16x2_t b1 = vtrnq_u8(a2, a3);
1025 const uint8x16x2_t b2 = vtrnq_u8(a4, a5);
1026 const uint8x16x2_t b3 = vtrnq_u8(a6, a7);
1027
1028 // Swap 16 bit elements resulting in:
1029 // c0.val[0]: 00 10 20 30 04 14 24 34 80 90 A0 B0 84 94 A4 B4
1030 // c0.val[1]: 02 12 22 32 06 16 26 36 82 92 A2 B2 86 96 A6 B6
1031 // c1.val[0]: 01 11 21 31 05 15 25 35 81 91 A1 B1 85 95 A5 B5
1032 // c1.val[1]: 03 13 23 33 07 17 27 37 83 93 A3 B3 87 97 A7 B7
1033 // c2.val[0]: 40 50 60 70 44 54 64 74 C0 D0 E0 F0 C4 D4 E4 F4
1034 // c2.val[1]: 42 52 62 72 46 56 66 76 C2 D2 E2 F2 C6 D6 E6 F6
1035 // c3.val[0]: 41 51 61 71 45 55 65 75 C1 D1 E1 F1 C5 D5 E5 F5
1036 // c3.val[1]: 43 53 63 73 47 57 67 77 C3 D3 E3 F3 C7 D7 E7 F7
1037 const uint16x8x2_t c0 = vtrnq_u16(vreinterpretq_u16_u8(b0.val[0]),
1038 vreinterpretq_u16_u8(b1.val[0]));
1039 const uint16x8x2_t c1 = vtrnq_u16(vreinterpretq_u16_u8(b0.val[1]),
1040 vreinterpretq_u16_u8(b1.val[1]));
1041 const uint16x8x2_t c2 = vtrnq_u16(vreinterpretq_u16_u8(b2.val[0]),
1042 vreinterpretq_u16_u8(b3.val[0]));
1043 const uint16x8x2_t c3 = vtrnq_u16(vreinterpretq_u16_u8(b2.val[1]),
1044 vreinterpretq_u16_u8(b3.val[1]));
1045
1046 // Swap 32 bit elements resulting in:
1047 // d0.val[0]: 00 10 20 30 40 50 60 70 80 90 A0 B0 C0 D0 E0 F0
1048 // d0.val[1]: 04 14 24 34 44 54 64 74 84 94 A4 B4 C4 D4 E4 F4
1049 // d1.val[0]: 02 12 22 32 42 52 62 72 82 92 A2 B2 C2 D2 E2 F2
1050 // d1.val[1]: 06 16 26 36 46 56 66 76 86 96 A6 B6 C6 D6 E6 F6
1051 // d2.val[0]: 01 11 21 31 41 51 61 71 81 91 A1 B1 C1 D1 E1 F1
1052 // d2.val[1]: 05 15 25 35 45 55 65 75 85 95 A5 B5 C5 D5 E5 F5
1053 // d3.val[0]: 03 13 23 33 43 53 63 73 83 93 A3 B3 C3 D3 E3 F3
1054 // d3.val[1]: 07 17 27 37 47 57 67 77 87 97 A7 B7 C7 D7 E7 F7
1055 const uint32x4x2_t d0 = vtrnq_u32(vreinterpretq_u32_u16(c0.val[0]),
1056 vreinterpretq_u32_u16(c2.val[0]));
1057 const uint32x4x2_t d1 = vtrnq_u32(vreinterpretq_u32_u16(c0.val[1]),
1058 vreinterpretq_u32_u16(c2.val[1]));
1059 const uint32x4x2_t d2 = vtrnq_u32(vreinterpretq_u32_u16(c1.val[0]),
1060 vreinterpretq_u32_u16(c3.val[0]));
1061 const uint32x4x2_t d3 = vtrnq_u32(vreinterpretq_u32_u16(c1.val[1]),
1062 vreinterpretq_u32_u16(c3.val[1]));
1063
1064 // Output:
1065 // o0: 00 10 20 30 40 50 60 70 80 90 A0 B0 C0 D0 E0 F0
1066 // o1: 01 11 21 31 41 51 61 71 81 91 A1 B1 C1 D1 E1 F1
1067 // o2: 02 12 22 32 42 52 62 72 82 92 A2 B2 C2 D2 E2 F2
1068 // o3: 03 13 23 33 43 53 63 73 83 93 A3 B3 C3 D3 E3 F3
1069 // o4: 04 14 24 34 44 54 64 74 84 94 A4 B4 C4 D4 E4 F4
1070 // o5: 05 15 25 35 45 55 65 75 85 95 A5 B5 C5 D5 E5 F5
1071 // o6: 06 16 26 36 46 56 66 76 86 96 A6 B6 C6 D6 E6 F6
1072 // o7: 07 17 27 37 47 57 67 77 87 97 A7 B7 C7 D7 E7 F7
1073 *o0 = vreinterpretq_u8_u32(d0.val[0]);
1074 *o1 = vreinterpretq_u8_u32(d2.val[0]);
1075 *o2 = vreinterpretq_u8_u32(d1.val[0]);
1076 *o3 = vreinterpretq_u8_u32(d3.val[0]);
1077 *o4 = vreinterpretq_u8_u32(d0.val[1]);
1078 *o5 = vreinterpretq_u8_u32(d2.val[1]);
1079 *o6 = vreinterpretq_u8_u32(d1.val[1]);
1080 *o7 = vreinterpretq_u8_u32(d3.val[1]);
1081 }
1082
transpose_u8_16x16(const uint8x16_t i0,const uint8x16_t i1,const uint8x16_t i2,const uint8x16_t i3,const uint8x16_t i4,const uint8x16_t i5,const uint8x16_t i6,const uint8x16_t i7,const uint8x16_t i8,const uint8x16_t i9,const uint8x16_t i10,const uint8x16_t i11,const uint8x16_t i12,const uint8x16_t i13,const uint8x16_t i14,const uint8x16_t i15,uint8x16_t * o0,uint8x16_t * o1,uint8x16_t * o2,uint8x16_t * o3,uint8x16_t * o4,uint8x16_t * o5,uint8x16_t * o6,uint8x16_t * o7,uint8x16_t * o8,uint8x16_t * o9,uint8x16_t * o10,uint8x16_t * o11,uint8x16_t * o12,uint8x16_t * o13,uint8x16_t * o14,uint8x16_t * o15)1083 static INLINE void transpose_u8_16x16(
1084 const uint8x16_t i0, const uint8x16_t i1, const uint8x16_t i2,
1085 const uint8x16_t i3, const uint8x16_t i4, const uint8x16_t i5,
1086 const uint8x16_t i6, const uint8x16_t i7, const uint8x16_t i8,
1087 const uint8x16_t i9, const uint8x16_t i10, const uint8x16_t i11,
1088 const uint8x16_t i12, const uint8x16_t i13, const uint8x16_t i14,
1089 const uint8x16_t i15, uint8x16_t *o0, uint8x16_t *o1, uint8x16_t *o2,
1090 uint8x16_t *o3, uint8x16_t *o4, uint8x16_t *o5, uint8x16_t *o6,
1091 uint8x16_t *o7, uint8x16_t *o8, uint8x16_t *o9, uint8x16_t *o10,
1092 uint8x16_t *o11, uint8x16_t *o12, uint8x16_t *o13, uint8x16_t *o14,
1093 uint8x16_t *o15) {
1094 // Swap 8 bit elements. Goes from:
1095 // i0: 00 01 02 03 04 05 06 07 08 09 0A 0B 0C 0D 0E 0F
1096 // i1: 10 11 12 13 14 15 16 17 18 19 1A 1B 1C 1D 1E 1F
1097 // i2: 20 21 22 23 24 25 26 27 28 29 2A 2B 2C 2D 2E 2F
1098 // i3: 30 31 32 33 34 35 36 37 38 39 3A 3B 3C 3D 3E 3F
1099 // i4: 40 41 42 43 44 45 46 47 48 49 4A 4B 4C 4D 4E 4F
1100 // i5: 50 51 52 53 54 55 56 57 58 59 5A 5B 5C 5D 5E 5F
1101 // i6: 60 61 62 63 64 65 66 67 68 69 6A 6B 6C 6D 6E 6F
1102 // i7: 70 71 72 73 74 75 76 77 78 79 7A 7B 7C 7D 7E 7F
1103 // i8: 80 81 82 83 84 85 86 87 88 89 8A 8B 8C 8D 8E 8F
1104 // i9: 90 91 92 93 94 95 96 97 98 99 9A 9B 9C 9D 9E 9F
1105 // i10: A0 A1 A2 A3 A4 A5 A6 A7 A8 A9 AA AB AC AD AE AF
1106 // i11: B0 B1 B2 B3 B4 B5 B6 B7 B8 B9 BA BB BC BD BE BF
1107 // i12: C0 C1 C2 C3 C4 C5 C6 C7 C8 C9 CA CB CC CD CE CF
1108 // i13: D0 D1 D2 D3 D4 D5 D6 D7 D8 D9 DA DB DC DD DE DF
1109 // i14: E0 E1 E2 E3 E4 E5 E6 E7 E8 E9 EA EB EC ED EE EF
1110 // i15: F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 FA FB FC FD FE FF
1111 // to:
1112 // b0.val[0]: 00 10 02 12 04 14 06 16 08 18 0A 1A 0C 1C 0E 1E
1113 // b0.val[1]: 01 11 03 13 05 15 07 17 09 19 0B 1B 0D 1D 0F 1F
1114 // b1.val[0]: 20 30 22 32 24 34 26 36 28 38 2A 3A 2C 3C 2E 3E
1115 // b1.val[1]: 21 31 23 33 25 35 27 37 29 39 2B 3B 2D 3D 2F 3F
1116 // b2.val[0]: 40 50 42 52 44 54 46 56 48 58 4A 5A 4C 5C 4E 5E
1117 // b2.val[1]: 41 51 43 53 45 55 47 57 49 59 4B 5B 4D 5D 4F 5F
1118 // b3.val[0]: 60 70 62 72 64 74 66 76 68 78 6A 7A 6C 7C 6E 7E
1119 // b3.val[1]: 61 71 63 73 65 75 67 77 69 79 6B 7B 6D 7D 6F 7F
1120 // b4.val[0]: 80 90 82 92 84 94 86 96 88 98 8A 9A 8C 9C 8E 9E
1121 // b4.val[1]: 81 91 83 93 85 95 87 97 89 99 8B 9B 8D 9D 8F 9F
1122 // b5.val[0]: A0 B0 A2 B2 A4 B4 A6 B6 A8 B8 AA BA AC BC AE BE
1123 // b5.val[1]: A1 B1 A3 B3 A5 B5 A7 B7 A9 B9 AB BB AD BD AF BF
1124 // b6.val[0]: C0 D0 C2 D2 C4 D4 C6 D6 C8 D8 CA DA CC DC CE DE
1125 // b6.val[1]: C1 D1 C3 D3 C5 D5 C7 D7 C9 D9 CB DB CD DD CF DF
1126 // b7.val[0]: E0 F0 E2 F2 E4 F4 E6 F6 E8 F8 EA FA EC FC EE FE
1127 // b7.val[1]: E1 F1 E3 F3 E5 F5 E7 F7 E9 F9 EB FB ED FD EF FF
1128 const uint8x16x2_t b0 = vtrnq_u8(i0, i1);
1129 const uint8x16x2_t b1 = vtrnq_u8(i2, i3);
1130 const uint8x16x2_t b2 = vtrnq_u8(i4, i5);
1131 const uint8x16x2_t b3 = vtrnq_u8(i6, i7);
1132 const uint8x16x2_t b4 = vtrnq_u8(i8, i9);
1133 const uint8x16x2_t b5 = vtrnq_u8(i10, i11);
1134 const uint8x16x2_t b6 = vtrnq_u8(i12, i13);
1135 const uint8x16x2_t b7 = vtrnq_u8(i14, i15);
1136
1137 // Swap 16 bit elements resulting in:
1138 // c0.val[0]: 00 10 20 30 04 14 24 34 08 18 28 38 0C 1C 2C 3C
1139 // c0.val[1]: 02 12 22 32 06 16 26 36 0A 1A 2A 3A 0E 1E 2E 3E
1140 // c1.val[0]: 01 11 21 31 05 15 25 35 09 19 29 39 0D 1D 2D 3D
1141 // c1.val[1]: 03 13 23 33 07 17 27 37 0B 1B 2B 3B 0F 1F 2F 3F
1142 // c2.val[0]: 40 50 60 70 44 54 64 74 48 58 68 78 4C 5C 6C 7C
1143 // c2.val[1]: 42 52 62 72 46 56 66 76 4A 5A 6A 7A 4E 5E 6E 7E
1144 // c3.val[0]: 41 51 61 71 45 55 65 75 49 59 69 79 4D 5D 6D 7D
1145 // c3.val[1]: 43 53 63 73 47 57 67 77 4B 5B 6B 7B 4F 5F 6F 7F
1146 // c4.val[0]: 80 90 A0 B0 84 94 A4 B4 88 98 A8 B8 8C 9C AC BC
1147 // c4.val[1]: 82 92 A2 B2 86 96 A6 B6 8A 9A AA BA 8E 9E AE BE
1148 // c5.val[0]: 81 91 A1 B1 85 95 A5 B5 89 99 A9 B9 8D 9D AD BD
1149 // c5.val[1]: 83 93 A3 B3 87 97 A7 B7 8B 9B AB BB 8F 9F AF BF
1150 // c6.val[0]: C0 D0 E0 F0 C4 D4 E4 F4 C8 D8 E8 F8 CC DC EC FC
1151 // c6.val[1]: C2 D2 E2 F2 C6 D6 E6 F6 CA DA EA FA CE DE EE FE
1152 // c7.val[0]: C1 D1 E1 F1 C5 D5 E5 F5 C9 D9 E9 F9 CD DD ED FD
1153 // c7.val[1]: C3 D3 E3 F3 C7 D7 E7 F7 CB DB EB FB CF DF EF FF
1154 const uint16x8x2_t c0 = vtrnq_u16(vreinterpretq_u16_u8(b0.val[0]),
1155 vreinterpretq_u16_u8(b1.val[0]));
1156 const uint16x8x2_t c1 = vtrnq_u16(vreinterpretq_u16_u8(b0.val[1]),
1157 vreinterpretq_u16_u8(b1.val[1]));
1158 const uint16x8x2_t c2 = vtrnq_u16(vreinterpretq_u16_u8(b2.val[0]),
1159 vreinterpretq_u16_u8(b3.val[0]));
1160 const uint16x8x2_t c3 = vtrnq_u16(vreinterpretq_u16_u8(b2.val[1]),
1161 vreinterpretq_u16_u8(b3.val[1]));
1162 const uint16x8x2_t c4 = vtrnq_u16(vreinterpretq_u16_u8(b4.val[0]),
1163 vreinterpretq_u16_u8(b5.val[0]));
1164 const uint16x8x2_t c5 = vtrnq_u16(vreinterpretq_u16_u8(b4.val[1]),
1165 vreinterpretq_u16_u8(b5.val[1]));
1166 const uint16x8x2_t c6 = vtrnq_u16(vreinterpretq_u16_u8(b6.val[0]),
1167 vreinterpretq_u16_u8(b7.val[0]));
1168 const uint16x8x2_t c7 = vtrnq_u16(vreinterpretq_u16_u8(b6.val[1]),
1169 vreinterpretq_u16_u8(b7.val[1]));
1170
1171 // Swap 32 bit elements resulting in:
1172 // d0.val[0]: 00 10 20 30 40 50 60 70 08 18 28 38 48 58 68 78
1173 // d0.val[1]: 04 14 24 34 44 54 64 74 0C 1C 2C 3C 4C 5C 6C 7C
1174 // d1.val[0]: 02 12 22 32 42 52 62 72 0A 1A 2A 3A 4A 5A 6A 7A
1175 // d1.val[1]: 06 16 26 36 46 56 66 76 0E 1E 2E 3E 4E 5E 6E 7E
1176 // d2.val[0]: 01 11 21 31 41 51 61 71 09 19 29 39 49 59 69 79
1177 // d2.val[1]: 05 15 25 35 45 55 65 75 0D 1D 2D 3D 4D 5D 6D 7D
1178 // d3.val[0]: 03 13 23 33 43 53 63 73 0B 1B 2B 3B 4B 5B 6B 7B
1179 // d3.val[1]: 07 17 27 37 47 57 67 77 0F 1F 2F 3F 4F 5F 6F 7F
1180 // d4.val[0]: 80 90 A0 B0 C0 D0 E0 F0 88 98 A8 B8 C8 D8 E8 F8
1181 // d4.val[1]: 84 94 A4 B4 C4 D4 E4 F4 8C 9C AC BC CC DC EC FC
1182 // d5.val[0]: 82 92 A2 B2 C2 D2 E2 F2 8A 9A AA BA CA DA EA FA
1183 // d5.val[1]: 86 96 A6 B6 C6 D6 E6 F6 8E 9E AE BE CE DE EE FE
1184 // d6.val[0]: 81 91 A1 B1 C1 D1 E1 F1 89 99 A9 B9 C9 D9 E9 F9
1185 // d6.val[1]: 85 95 A5 B5 C5 D5 E5 F5 8D 9D AD BD CD DD ED FD
1186 // d7.val[0]: 83 93 A3 B3 C3 D3 E3 F3 8B 9B AB BB CB DB EB FB
1187 // d7.val[1]: 87 97 A7 B7 C7 D7 E7 F7 8F 9F AF BF CF DF EF FF
1188 const uint32x4x2_t d0 = vtrnq_u32(vreinterpretq_u32_u16(c0.val[0]),
1189 vreinterpretq_u32_u16(c2.val[0]));
1190 const uint32x4x2_t d1 = vtrnq_u32(vreinterpretq_u32_u16(c0.val[1]),
1191 vreinterpretq_u32_u16(c2.val[1]));
1192 const uint32x4x2_t d2 = vtrnq_u32(vreinterpretq_u32_u16(c1.val[0]),
1193 vreinterpretq_u32_u16(c3.val[0]));
1194 const uint32x4x2_t d3 = vtrnq_u32(vreinterpretq_u32_u16(c1.val[1]),
1195 vreinterpretq_u32_u16(c3.val[1]));
1196 const uint32x4x2_t d4 = vtrnq_u32(vreinterpretq_u32_u16(c4.val[0]),
1197 vreinterpretq_u32_u16(c6.val[0]));
1198 const uint32x4x2_t d5 = vtrnq_u32(vreinterpretq_u32_u16(c4.val[1]),
1199 vreinterpretq_u32_u16(c6.val[1]));
1200 const uint32x4x2_t d6 = vtrnq_u32(vreinterpretq_u32_u16(c5.val[0]),
1201 vreinterpretq_u32_u16(c7.val[0]));
1202 const uint32x4x2_t d7 = vtrnq_u32(vreinterpretq_u32_u16(c5.val[1]),
1203 vreinterpretq_u32_u16(c7.val[1]));
1204
1205 // Swap 64 bit elements resulting in:
1206 // e0.val[0]: 00 10 20 30 40 50 60 70 80 90 A0 B0 C0 D0 E0 F0
1207 // e0.val[1]: 08 18 28 38 48 58 68 78 88 98 A8 B8 C8 D8 E8 F8
1208 // e1.val[0]: 01 11 21 31 41 51 61 71 84 94 A4 B4 C4 D4 E4 F4
1209 // e1.val[1]: 09 19 29 39 49 59 69 79 89 99 A9 B9 C9 D9 E9 F9
1210 // e2.val[0]: 02 12 22 32 42 52 62 72 82 92 A2 B2 C2 D2 E2 F2
1211 // e2.val[1]: 0A 1A 2A 3A 4A 5A 6A 7A 8A 9A AA BA CA DA EA FA
1212 // e3.val[0]: 03 13 23 33 43 53 63 73 86 96 A6 B6 C6 D6 E6 F6
1213 // e3.val[1]: 0B 1B 2B 3B 4B 5B 6B 7B 8B 9B AB BB CB DB EB FB
1214 // e4.val[0]: 04 14 24 34 44 54 64 74 81 91 A1 B1 C1 D1 E1 F1
1215 // e4.val[1]: 0C 1C 2C 3C 4C 5C 6C 7C 8C 9C AC BC CC DC EC FC
1216 // e5.val[0]: 05 15 25 35 45 55 65 75 85 95 A5 B5 C5 D5 E5 F5
1217 // e5.val[1]: 0D 1D 2D 3D 4D 5D 6D 7D 8D 9D AD BD CD DD ED FD
1218 // e6.val[0]: 06 16 26 36 46 56 66 76 83 93 A3 B3 C3 D3 E3 F3
1219 // e6.val[1]: 0E 1E 2E 3E 4E 5E 6E 7E 8E 9E AE BE CE DE EE FE
1220 // e7.val[0]: 07 17 27 37 47 57 67 77 87 97 A7 B7 C7 D7 E7 F7
1221 // e7.val[1]: 0F 1F 2F 3F 4F 5F 6F 7F 8F 9F AF BF CF DF EF FF
1222 const uint8x16x2_t e0 = vpx_vtrnq_u64_to_u8(d0.val[0], d4.val[0]);
1223 const uint8x16x2_t e1 = vpx_vtrnq_u64_to_u8(d2.val[0], d6.val[0]);
1224 const uint8x16x2_t e2 = vpx_vtrnq_u64_to_u8(d1.val[0], d5.val[0]);
1225 const uint8x16x2_t e3 = vpx_vtrnq_u64_to_u8(d3.val[0], d7.val[0]);
1226 const uint8x16x2_t e4 = vpx_vtrnq_u64_to_u8(d0.val[1], d4.val[1]);
1227 const uint8x16x2_t e5 = vpx_vtrnq_u64_to_u8(d2.val[1], d6.val[1]);
1228 const uint8x16x2_t e6 = vpx_vtrnq_u64_to_u8(d1.val[1], d5.val[1]);
1229 const uint8x16x2_t e7 = vpx_vtrnq_u64_to_u8(d3.val[1], d7.val[1]);
1230
1231 // Output:
1232 // o0 : 00 10 20 30 40 50 60 70 80 90 A0 B0 C0 D0 E0 F0
1233 // o1 : 01 11 21 31 41 51 61 71 84 94 A4 B4 C4 D4 E4 F4
1234 // o2 : 02 12 22 32 42 52 62 72 82 92 A2 B2 C2 D2 E2 F2
1235 // o3 : 03 13 23 33 43 53 63 73 86 96 A6 B6 C6 D6 E6 F6
1236 // o4 : 04 14 24 34 44 54 64 74 81 91 A1 B1 C1 D1 E1 F1
1237 // o5 : 05 15 25 35 45 55 65 75 85 95 A5 B5 C5 D5 E5 F5
1238 // o6 : 06 16 26 36 46 56 66 76 83 93 A3 B3 C3 D3 E3 F3
1239 // o7 : 07 17 27 37 47 57 67 77 87 97 A7 B7 C7 D7 E7 F7
1240 // o8 : 08 18 28 38 48 58 68 78 88 98 A8 B8 C8 D8 E8 F8
1241 // o9 : 09 19 29 39 49 59 69 79 89 99 A9 B9 C9 D9 E9 F9
1242 // o10: 0A 1A 2A 3A 4A 5A 6A 7A 8A 9A AA BA CA DA EA FA
1243 // o11: 0B 1B 2B 3B 4B 5B 6B 7B 8B 9B AB BB CB DB EB FB
1244 // o12: 0C 1C 2C 3C 4C 5C 6C 7C 8C 9C AC BC CC DC EC FC
1245 // o13: 0D 1D 2D 3D 4D 5D 6D 7D 8D 9D AD BD CD DD ED FD
1246 // o14: 0E 1E 2E 3E 4E 5E 6E 7E 8E 9E AE BE CE DE EE FE
1247 // o15: 0F 1F 2F 3F 4F 5F 6F 7F 8F 9F AF BF CF DF EF FF
1248 *o0 = e0.val[0];
1249 *o1 = e1.val[0];
1250 *o2 = e2.val[0];
1251 *o3 = e3.val[0];
1252 *o4 = e4.val[0];
1253 *o5 = e5.val[0];
1254 *o6 = e6.val[0];
1255 *o7 = e7.val[0];
1256 *o8 = e0.val[1];
1257 *o9 = e1.val[1];
1258 *o10 = e2.val[1];
1259 *o11 = e3.val[1];
1260 *o12 = e4.val[1];
1261 *o13 = e5.val[1];
1262 *o14 = e6.val[1];
1263 *o15 = e7.val[1];
1264 }
1265
transpose_s16_16x16(int16x8_t * in0,int16x8_t * in1)1266 static INLINE void transpose_s16_16x16(int16x8_t *in0, int16x8_t *in1) {
1267 int16x8_t t[8];
1268
1269 // transpose the 4 8x8 quadrants separately but first swap quadrants 2 and 3.
1270 t[0] = in0[8];
1271 t[1] = in0[9];
1272 t[2] = in0[10];
1273 t[3] = in0[11];
1274 t[4] = in0[12];
1275 t[5] = in0[13];
1276 t[6] = in0[14];
1277 t[7] = in0[15];
1278 in0[8] = in1[0];
1279 in0[9] = in1[1];
1280 in0[10] = in1[2];
1281 in0[11] = in1[3];
1282 in0[12] = in1[4];
1283 in0[13] = in1[5];
1284 in0[14] = in1[6];
1285 in0[15] = in1[7];
1286 in1[0] = t[0];
1287 in1[1] = t[1];
1288 in1[2] = t[2];
1289 in1[3] = t[3];
1290 in1[4] = t[4];
1291 in1[5] = t[5];
1292 in1[6] = t[6];
1293 in1[7] = t[7];
1294
1295 transpose_s16_8x8(&in0[0], &in0[1], &in0[2], &in0[3], &in0[4], &in0[5],
1296 &in0[6], &in0[7]);
1297 transpose_s16_8x8(&in0[8], &in0[9], &in0[10], &in0[11], &in0[12], &in0[13],
1298 &in0[14], &in0[15]);
1299 transpose_s16_8x8(&in1[0], &in1[1], &in1[2], &in1[3], &in1[4], &in1[5],
1300 &in1[6], &in1[7]);
1301 transpose_s16_8x8(&in1[8], &in1[9], &in1[10], &in1[11], &in1[12], &in1[13],
1302 &in1[14], &in1[15]);
1303 }
1304
load_and_transpose_u8_4x8(const uint8_t * a,const int a_stride,uint8x8_t * a0,uint8x8_t * a1,uint8x8_t * a2,uint8x8_t * a3)1305 static INLINE void load_and_transpose_u8_4x8(const uint8_t *a,
1306 const int a_stride, uint8x8_t *a0,
1307 uint8x8_t *a1, uint8x8_t *a2,
1308 uint8x8_t *a3) {
1309 uint8x8_t a4, a5, a6, a7;
1310 *a0 = vld1_u8(a);
1311 a += a_stride;
1312 *a1 = vld1_u8(a);
1313 a += a_stride;
1314 *a2 = vld1_u8(a);
1315 a += a_stride;
1316 *a3 = vld1_u8(a);
1317 a += a_stride;
1318 a4 = vld1_u8(a);
1319 a += a_stride;
1320 a5 = vld1_u8(a);
1321 a += a_stride;
1322 a6 = vld1_u8(a);
1323 a += a_stride;
1324 a7 = vld1_u8(a);
1325
1326 transpose_u8_4x8(a0, a1, a2, a3, a4, a5, a6, a7);
1327 }
1328
load_and_transpose_u8_8x8(const uint8_t * a,const int a_stride,uint8x8_t * a0,uint8x8_t * a1,uint8x8_t * a2,uint8x8_t * a3,uint8x8_t * a4,uint8x8_t * a5,uint8x8_t * a6,uint8x8_t * a7)1329 static INLINE void load_and_transpose_u8_8x8(const uint8_t *a,
1330 const int a_stride, uint8x8_t *a0,
1331 uint8x8_t *a1, uint8x8_t *a2,
1332 uint8x8_t *a3, uint8x8_t *a4,
1333 uint8x8_t *a5, uint8x8_t *a6,
1334 uint8x8_t *a7) {
1335 *a0 = vld1_u8(a);
1336 a += a_stride;
1337 *a1 = vld1_u8(a);
1338 a += a_stride;
1339 *a2 = vld1_u8(a);
1340 a += a_stride;
1341 *a3 = vld1_u8(a);
1342 a += a_stride;
1343 *a4 = vld1_u8(a);
1344 a += a_stride;
1345 *a5 = vld1_u8(a);
1346 a += a_stride;
1347 *a6 = vld1_u8(a);
1348 a += a_stride;
1349 *a7 = vld1_u8(a);
1350
1351 transpose_u8_8x8(a0, a1, a2, a3, a4, a5, a6, a7);
1352 }
1353
transpose_and_store_u8_8x8(uint8_t * a,const int a_stride,uint8x8_t a0,uint8x8_t a1,uint8x8_t a2,uint8x8_t a3,uint8x8_t a4,uint8x8_t a5,uint8x8_t a6,uint8x8_t a7)1354 static INLINE void transpose_and_store_u8_8x8(uint8_t *a, const int a_stride,
1355 uint8x8_t a0, uint8x8_t a1,
1356 uint8x8_t a2, uint8x8_t a3,
1357 uint8x8_t a4, uint8x8_t a5,
1358 uint8x8_t a6, uint8x8_t a7) {
1359 transpose_u8_8x8(&a0, &a1, &a2, &a3, &a4, &a5, &a6, &a7);
1360
1361 vst1_u8(a, a0);
1362 a += a_stride;
1363 vst1_u8(a, a1);
1364 a += a_stride;
1365 vst1_u8(a, a2);
1366 a += a_stride;
1367 vst1_u8(a, a3);
1368 a += a_stride;
1369 vst1_u8(a, a4);
1370 a += a_stride;
1371 vst1_u8(a, a5);
1372 a += a_stride;
1373 vst1_u8(a, a6);
1374 a += a_stride;
1375 vst1_u8(a, a7);
1376 }
1377
load_and_transpose_s16_8x8(const int16_t * a,const int a_stride,int16x8_t * a0,int16x8_t * a1,int16x8_t * a2,int16x8_t * a3,int16x8_t * a4,int16x8_t * a5,int16x8_t * a6,int16x8_t * a7)1378 static INLINE void load_and_transpose_s16_8x8(const int16_t *a,
1379 const int a_stride, int16x8_t *a0,
1380 int16x8_t *a1, int16x8_t *a2,
1381 int16x8_t *a3, int16x8_t *a4,
1382 int16x8_t *a5, int16x8_t *a6,
1383 int16x8_t *a7) {
1384 *a0 = vld1q_s16(a);
1385 a += a_stride;
1386 *a1 = vld1q_s16(a);
1387 a += a_stride;
1388 *a2 = vld1q_s16(a);
1389 a += a_stride;
1390 *a3 = vld1q_s16(a);
1391 a += a_stride;
1392 *a4 = vld1q_s16(a);
1393 a += a_stride;
1394 *a5 = vld1q_s16(a);
1395 a += a_stride;
1396 *a6 = vld1q_s16(a);
1397 a += a_stride;
1398 *a7 = vld1q_s16(a);
1399
1400 transpose_s16_8x8(a0, a1, a2, a3, a4, a5, a6, a7);
1401 }
1402
load_and_transpose_s32_8x8(const int32_t * a,const int a_stride,int32x4x2_t * const a0,int32x4x2_t * const a1,int32x4x2_t * const a2,int32x4x2_t * const a3,int32x4x2_t * const a4,int32x4x2_t * const a5,int32x4x2_t * const a6,int32x4x2_t * const a7)1403 static INLINE void load_and_transpose_s32_8x8(
1404 const int32_t *a, const int a_stride, int32x4x2_t *const a0,
1405 int32x4x2_t *const a1, int32x4x2_t *const a2, int32x4x2_t *const a3,
1406 int32x4x2_t *const a4, int32x4x2_t *const a5, int32x4x2_t *const a6,
1407 int32x4x2_t *const a7) {
1408 a0->val[0] = vld1q_s32(a);
1409 a0->val[1] = vld1q_s32(a + 4);
1410 a += a_stride;
1411 a1->val[0] = vld1q_s32(a);
1412 a1->val[1] = vld1q_s32(a + 4);
1413 a += a_stride;
1414 a2->val[0] = vld1q_s32(a);
1415 a2->val[1] = vld1q_s32(a + 4);
1416 a += a_stride;
1417 a3->val[0] = vld1q_s32(a);
1418 a3->val[1] = vld1q_s32(a + 4);
1419 a += a_stride;
1420 a4->val[0] = vld1q_s32(a);
1421 a4->val[1] = vld1q_s32(a + 4);
1422 a += a_stride;
1423 a5->val[0] = vld1q_s32(a);
1424 a5->val[1] = vld1q_s32(a + 4);
1425 a += a_stride;
1426 a6->val[0] = vld1q_s32(a);
1427 a6->val[1] = vld1q_s32(a + 4);
1428 a += a_stride;
1429 a7->val[0] = vld1q_s32(a);
1430 a7->val[1] = vld1q_s32(a + 4);
1431
1432 transpose_s32_8x8(a0, a1, a2, a3, a4, a5, a6, a7);
1433 }
1434 #endif // VPX_VPX_DSP_ARM_TRANSPOSE_NEON_H_
1435