1 /*
2 * Copyright (c) 2015 The WebM project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11 #ifndef VPX_DSP_X86_TRANSPOSE_SSE2_H_
12 #define VPX_DSP_X86_TRANSPOSE_SSE2_H_
13
14 #include <emmintrin.h> // SSE2
15
16 #include "./vpx_config.h"
17
transpose_8bit_4x4(const __m128i * const in)18 static INLINE __m128i transpose_8bit_4x4(const __m128i *const in) {
19 // Unpack 16 bit elements. Goes from:
20 // in[0]: 00 01 02 03
21 // in[1]: 10 11 12 13
22 // in[2]: 20 21 22 23
23 // in[3]: 30 31 32 33
24 // to:
25 // a0: 00 10 01 11 02 12 03 13
26 // a1: 20 30 21 31 22 32 23 33
27 const __m128i a0 = _mm_unpacklo_epi8(in[0], in[1]);
28 const __m128i a1 = _mm_unpacklo_epi8(in[2], in[3]);
29
30 // Unpack 32 bit elements resulting in:
31 // 00 10 20 30 01 11 21 31 02 12 22 32 03 13 23 33
32 return _mm_unpacklo_epi16(a0, a1);
33 }
34
transpose_8bit_8x8(const __m128i * const in,__m128i * const out)35 static INLINE void transpose_8bit_8x8(const __m128i *const in,
36 __m128i *const out) {
37 // Unpack 8 bit elements. Goes from:
38 // in[0]: 00 01 02 03 04 05 06 07
39 // in[1]: 10 11 12 13 14 15 16 17
40 // in[2]: 20 21 22 23 24 25 26 27
41 // in[3]: 30 31 32 33 34 35 36 37
42 // in[4]: 40 41 42 43 44 45 46 47
43 // in[5]: 50 51 52 53 54 55 56 57
44 // in[6]: 60 61 62 63 64 65 66 67
45 // in[7]: 70 71 72 73 74 75 76 77
46 // to:
47 // a0: 00 10 01 11 02 12 03 13 04 14 05 15 06 16 07 17
48 // a1: 20 30 21 31 22 32 23 33 24 34 25 35 26 36 27 37
49 // a2: 40 50 41 51 42 52 43 53 44 54 45 55 46 56 47 57
50 // a3: 60 70 61 71 62 72 63 73 64 74 65 75 66 76 67 77
51 const __m128i a0 = _mm_unpacklo_epi8(in[0], in[1]);
52 const __m128i a1 = _mm_unpacklo_epi8(in[2], in[3]);
53 const __m128i a2 = _mm_unpacklo_epi8(in[4], in[5]);
54 const __m128i a3 = _mm_unpacklo_epi8(in[6], in[7]);
55
56 // Unpack 16 bit elements resulting in:
57 // b0: 00 10 20 30 01 11 21 31 02 12 22 32 03 13 23 33
58 // b1: 40 50 60 70 41 51 61 71 42 52 62 72 43 53 63 73
59 // b2: 04 14 24 34 05 15 25 35 06 16 26 36 07 17 27 37
60 // b3: 44 54 64 74 45 55 65 75 46 56 66 76 47 57 67 77
61 const __m128i b0 = _mm_unpacklo_epi16(a0, a1);
62 const __m128i b1 = _mm_unpackhi_epi16(a0, a1);
63 const __m128i b2 = _mm_unpacklo_epi16(a2, a3);
64 const __m128i b3 = _mm_unpackhi_epi16(a2, a3);
65
66 // Unpack 32 bit elements resulting in:
67 // c0: 00 10 20 30 40 50 60 70 01 11 21 31 41 51 61 71
68 // c1: 02 12 22 32 42 52 62 72 03 13 23 33 43 53 63 73
69 // c2: 04 14 24 34 44 54 64 74 05 15 25 35 45 55 65 75
70 // c3: 06 16 26 36 46 56 66 76 07 17 27 37 47 57 67 77
71 const __m128i c0 = _mm_unpacklo_epi32(b0, b2);
72 const __m128i c1 = _mm_unpackhi_epi32(b0, b2);
73 const __m128i c2 = _mm_unpacklo_epi32(b1, b3);
74 const __m128i c3 = _mm_unpackhi_epi32(b1, b3);
75
76 // Unpack 64 bit elements resulting in:
77 // out[0]: 00 10 20 30 40 50 60 70
78 // out[1]: 01 11 21 31 41 51 61 71
79 // out[2]: 02 12 22 32 42 52 62 72
80 // out[3]: 03 13 23 33 43 53 63 73
81 // out[4]: 04 14 24 34 44 54 64 74
82 // out[5]: 05 15 25 35 45 55 65 75
83 // out[6]: 06 16 26 36 46 56 66 76
84 // out[7]: 07 17 27 37 47 57 67 77
85 out[0] = _mm_unpacklo_epi64(c0, c0);
86 out[1] = _mm_unpackhi_epi64(c0, c0);
87 out[2] = _mm_unpacklo_epi64(c1, c1);
88 out[3] = _mm_unpackhi_epi64(c1, c1);
89 out[4] = _mm_unpacklo_epi64(c2, c2);
90 out[5] = _mm_unpackhi_epi64(c2, c2);
91 out[6] = _mm_unpacklo_epi64(c3, c3);
92 out[7] = _mm_unpackhi_epi64(c3, c3);
93 }
94
transpose_16bit_4x4(const __m128i * const in,__m128i * const out)95 static INLINE void transpose_16bit_4x4(const __m128i *const in,
96 __m128i *const out) {
97 // Unpack 16 bit elements. Goes from:
98 // in[0]: 00 01 02 03 XX XX XX XX
99 // in[1]: 10 11 12 13 XX XX XX XX
100 // in[2]: 20 21 22 23 XX XX XX XX
101 // in[3]: 30 31 32 33 XX XX XX XX
102 // to:
103 // a0: 00 10 01 11 02 12 03 13
104 // a1: 20 30 21 31 22 32 23 33
105 const __m128i a0 = _mm_unpacklo_epi16(in[0], in[1]);
106 const __m128i a1 = _mm_unpacklo_epi16(in[2], in[3]);
107
108 // Unpack 32 bit elements resulting in:
109 // out[0]: 00 10 20 30 01 11 21 31
110 // out[1]: 02 12 22 32 03 13 23 33
111 out[0] = _mm_unpacklo_epi32(a0, a1);
112 out[1] = _mm_unpackhi_epi32(a0, a1);
113 }
114
transpose_16bit_4x8(const __m128i * const in,__m128i * const out)115 static INLINE void transpose_16bit_4x8(const __m128i *const in,
116 __m128i *const out) {
117 // Unpack 16 bit elements. Goes from:
118 // in[0]: 00 01 02 03 XX XX XX XX
119 // in[1]: 10 11 12 13 XX XX XX XX
120 // in[2]: 20 21 22 23 XX XX XX XX
121 // in[3]: 30 31 32 33 XX XX XX XX
122 // in[4]: 40 41 42 43 XX XX XX XX
123 // in[5]: 50 51 52 53 XX XX XX XX
124 // in[6]: 60 61 62 63 XX XX XX XX
125 // in[7]: 70 71 72 73 XX XX XX XX
126 // to:
127 // a0: 00 10 01 11 02 12 03 13
128 // a1: 20 30 21 31 22 32 23 33
129 // a2: 40 50 41 51 42 52 43 53
130 // a3: 60 70 61 71 62 72 63 73
131 const __m128i a0 = _mm_unpacklo_epi16(in[0], in[1]);
132 const __m128i a1 = _mm_unpacklo_epi16(in[2], in[3]);
133 const __m128i a2 = _mm_unpacklo_epi16(in[4], in[5]);
134 const __m128i a3 = _mm_unpacklo_epi16(in[6], in[7]);
135
136 // Unpack 32 bit elements resulting in:
137 // b0: 00 10 20 30 01 11 21 31
138 // b1: 40 50 60 70 41 51 61 71
139 // b2: 02 12 22 32 03 13 23 33
140 // b3: 42 52 62 72 43 53 63 73
141 const __m128i b0 = _mm_unpacklo_epi32(a0, a1);
142 const __m128i b1 = _mm_unpacklo_epi32(a2, a3);
143 const __m128i b2 = _mm_unpackhi_epi32(a0, a1);
144 const __m128i b3 = _mm_unpackhi_epi32(a2, a3);
145
146 // Unpack 64 bit elements resulting in:
147 // out[0]: 00 10 20 30 40 50 60 70
148 // out[1]: 01 11 21 31 41 51 61 71
149 // out[2]: 02 12 22 32 42 52 62 72
150 // out[3]: 03 13 23 33 43 53 63 73
151 out[0] = _mm_unpacklo_epi64(b0, b1);
152 out[1] = _mm_unpackhi_epi64(b0, b1);
153 out[2] = _mm_unpacklo_epi64(b2, b3);
154 out[3] = _mm_unpackhi_epi64(b2, b3);
155 }
156
transpose_16bit_8x8(const __m128i * const in,__m128i * const out)157 static INLINE void transpose_16bit_8x8(const __m128i *const in,
158 __m128i *const out) {
159 // Unpack 16 bit elements. Goes from:
160 // in[0]: 00 01 02 03 04 05 06 07
161 // in[1]: 10 11 12 13 14 15 16 17
162 // in[2]: 20 21 22 23 24 25 26 27
163 // in[3]: 30 31 32 33 34 35 36 37
164 // in[4]: 40 41 42 43 44 45 46 47
165 // in[5]: 50 51 52 53 54 55 56 57
166 // in[6]: 60 61 62 63 64 65 66 67
167 // in[7]: 70 71 72 73 74 75 76 77
168 // to:
169 // a0: 00 10 01 11 02 12 03 13
170 // a1: 20 30 21 31 22 32 23 33
171 // a2: 40 50 41 51 42 52 43 53
172 // a3: 60 70 61 71 62 72 63 73
173 // a4: 04 14 05 15 06 16 07 17
174 // a5: 24 34 25 35 26 36 27 37
175 // a6: 44 54 45 55 46 56 47 57
176 // a7: 64 74 65 75 66 76 67 77
177 const __m128i a0 = _mm_unpacklo_epi16(in[0], in[1]);
178 const __m128i a1 = _mm_unpacklo_epi16(in[2], in[3]);
179 const __m128i a2 = _mm_unpacklo_epi16(in[4], in[5]);
180 const __m128i a3 = _mm_unpacklo_epi16(in[6], in[7]);
181 const __m128i a4 = _mm_unpackhi_epi16(in[0], in[1]);
182 const __m128i a5 = _mm_unpackhi_epi16(in[2], in[3]);
183 const __m128i a6 = _mm_unpackhi_epi16(in[4], in[5]);
184 const __m128i a7 = _mm_unpackhi_epi16(in[6], in[7]);
185
186 // Unpack 32 bit elements resulting in:
187 // b0: 00 10 20 30 01 11 21 31
188 // b1: 40 50 60 70 41 51 61 71
189 // b2: 04 14 24 34 05 15 25 35
190 // b3: 44 54 64 74 45 55 65 75
191 // b4: 02 12 22 32 03 13 23 33
192 // b5: 42 52 62 72 43 53 63 73
193 // b6: 06 16 26 36 07 17 27 37
194 // b7: 46 56 66 76 47 57 67 77
195 const __m128i b0 = _mm_unpacklo_epi32(a0, a1);
196 const __m128i b1 = _mm_unpacklo_epi32(a2, a3);
197 const __m128i b2 = _mm_unpacklo_epi32(a4, a5);
198 const __m128i b3 = _mm_unpacklo_epi32(a6, a7);
199 const __m128i b4 = _mm_unpackhi_epi32(a0, a1);
200 const __m128i b5 = _mm_unpackhi_epi32(a2, a3);
201 const __m128i b6 = _mm_unpackhi_epi32(a4, a5);
202 const __m128i b7 = _mm_unpackhi_epi32(a6, a7);
203
204 // Unpack 64 bit elements resulting in:
205 // out[0]: 00 10 20 30 40 50 60 70
206 // out[1]: 01 11 21 31 41 51 61 71
207 // out[2]: 02 12 22 32 42 52 62 72
208 // out[3]: 03 13 23 33 43 53 63 73
209 // out[4]: 04 14 24 34 44 54 64 74
210 // out[5]: 05 15 25 35 45 55 65 75
211 // out[6]: 06 16 26 36 46 56 66 76
212 // out[7]: 07 17 27 37 47 57 67 77
213 out[0] = _mm_unpacklo_epi64(b0, b1);
214 out[1] = _mm_unpackhi_epi64(b0, b1);
215 out[2] = _mm_unpacklo_epi64(b4, b5);
216 out[3] = _mm_unpackhi_epi64(b4, b5);
217 out[4] = _mm_unpacklo_epi64(b2, b3);
218 out[5] = _mm_unpackhi_epi64(b2, b3);
219 out[6] = _mm_unpacklo_epi64(b6, b7);
220 out[7] = _mm_unpackhi_epi64(b6, b7);
221 }
222
223 // Transpose in-place
transpose_16bit_16x16(__m128i * const left,__m128i * const right)224 static INLINE void transpose_16bit_16x16(__m128i *const left,
225 __m128i *const right) {
226 __m128i tbuf[8];
227 transpose_16bit_8x8(left, left);
228 transpose_16bit_8x8(right, tbuf);
229 transpose_16bit_8x8(left + 8, right);
230 transpose_16bit_8x8(right + 8, right + 8);
231
232 left[8] = tbuf[0];
233 left[9] = tbuf[1];
234 left[10] = tbuf[2];
235 left[11] = tbuf[3];
236 left[12] = tbuf[4];
237 left[13] = tbuf[5];
238 left[14] = tbuf[6];
239 left[15] = tbuf[7];
240 }
241
transpose_32bit_4x4(const __m128i * const in,__m128i * const out)242 static INLINE void transpose_32bit_4x4(const __m128i *const in,
243 __m128i *const out) {
244 // Unpack 32 bit elements. Goes from:
245 // in[0]: 00 01 02 03
246 // in[1]: 10 11 12 13
247 // in[2]: 20 21 22 23
248 // in[3]: 30 31 32 33
249 // to:
250 // a0: 00 10 01 11
251 // a1: 20 30 21 31
252 // a2: 02 12 03 13
253 // a3: 22 32 23 33
254
255 const __m128i a0 = _mm_unpacklo_epi32(in[0], in[1]);
256 const __m128i a1 = _mm_unpacklo_epi32(in[2], in[3]);
257 const __m128i a2 = _mm_unpackhi_epi32(in[0], in[1]);
258 const __m128i a3 = _mm_unpackhi_epi32(in[2], in[3]);
259
260 // Unpack 64 bit elements resulting in:
261 // out[0]: 00 10 20 30
262 // out[1]: 01 11 21 31
263 // out[2]: 02 12 22 32
264 // out[3]: 03 13 23 33
265 out[0] = _mm_unpacklo_epi64(a0, a1);
266 out[1] = _mm_unpackhi_epi64(a0, a1);
267 out[2] = _mm_unpacklo_epi64(a2, a3);
268 out[3] = _mm_unpackhi_epi64(a2, a3);
269 }
270
transpose_32bit_4x4x2(const __m128i * const in,__m128i * const out)271 static INLINE void transpose_32bit_4x4x2(const __m128i *const in,
272 __m128i *const out) {
273 // Unpack 32 bit elements. Goes from:
274 // in[0]: 00 01 02 03
275 // in[1]: 10 11 12 13
276 // in[2]: 20 21 22 23
277 // in[3]: 30 31 32 33
278 // in[4]: 04 05 06 07
279 // in[5]: 14 15 16 17
280 // in[6]: 24 25 26 27
281 // in[7]: 34 35 36 37
282 // to:
283 // a0: 00 10 01 11
284 // a1: 20 30 21 31
285 // a2: 02 12 03 13
286 // a3: 22 32 23 33
287 // a4: 04 14 05 15
288 // a5: 24 34 25 35
289 // a6: 06 16 07 17
290 // a7: 26 36 27 37
291 const __m128i a0 = _mm_unpacklo_epi32(in[0], in[1]);
292 const __m128i a1 = _mm_unpacklo_epi32(in[2], in[3]);
293 const __m128i a2 = _mm_unpackhi_epi32(in[0], in[1]);
294 const __m128i a3 = _mm_unpackhi_epi32(in[2], in[3]);
295 const __m128i a4 = _mm_unpacklo_epi32(in[4], in[5]);
296 const __m128i a5 = _mm_unpacklo_epi32(in[6], in[7]);
297 const __m128i a6 = _mm_unpackhi_epi32(in[4], in[5]);
298 const __m128i a7 = _mm_unpackhi_epi32(in[6], in[7]);
299
300 // Unpack 64 bit elements resulting in:
301 // out[0]: 00 10 20 30
302 // out[1]: 01 11 21 31
303 // out[2]: 02 12 22 32
304 // out[3]: 03 13 23 33
305 // out[4]: 04 14 24 34
306 // out[5]: 05 15 25 35
307 // out[6]: 06 16 26 36
308 // out[7]: 07 17 27 37
309 out[0] = _mm_unpacklo_epi64(a0, a1);
310 out[1] = _mm_unpackhi_epi64(a0, a1);
311 out[2] = _mm_unpacklo_epi64(a2, a3);
312 out[3] = _mm_unpackhi_epi64(a2, a3);
313 out[4] = _mm_unpacklo_epi64(a4, a5);
314 out[5] = _mm_unpackhi_epi64(a4, a5);
315 out[6] = _mm_unpacklo_epi64(a6, a7);
316 out[7] = _mm_unpackhi_epi64(a6, a7);
317 }
318
transpose_32bit_8x4(const __m128i * const in,__m128i * const out)319 static INLINE void transpose_32bit_8x4(const __m128i *const in,
320 __m128i *const out) {
321 // Unpack 32 bit elements. Goes from:
322 // in[0]: 00 01 02 03
323 // in[1]: 04 05 06 07
324 // in[2]: 10 11 12 13
325 // in[3]: 14 15 16 17
326 // in[4]: 20 21 22 23
327 // in[5]: 24 25 26 27
328 // in[6]: 30 31 32 33
329 // in[7]: 34 35 36 37
330 // to:
331 // a0: 00 10 01 11
332 // a1: 20 30 21 31
333 // a2: 02 12 03 13
334 // a3: 22 32 23 33
335 // a4: 04 14 05 15
336 // a5: 24 34 25 35
337 // a6: 06 16 07 17
338 // a7: 26 36 27 37
339 const __m128i a0 = _mm_unpacklo_epi32(in[0], in[2]);
340 const __m128i a1 = _mm_unpacklo_epi32(in[4], in[6]);
341 const __m128i a2 = _mm_unpackhi_epi32(in[0], in[2]);
342 const __m128i a3 = _mm_unpackhi_epi32(in[4], in[6]);
343 const __m128i a4 = _mm_unpacklo_epi32(in[1], in[3]);
344 const __m128i a5 = _mm_unpacklo_epi32(in[5], in[7]);
345 const __m128i a6 = _mm_unpackhi_epi32(in[1], in[3]);
346 const __m128i a7 = _mm_unpackhi_epi32(in[5], in[7]);
347
348 // Unpack 64 bit elements resulting in:
349 // out[0]: 00 10 20 30
350 // out[1]: 01 11 21 31
351 // out[2]: 02 12 22 32
352 // out[3]: 03 13 23 33
353 // out[4]: 04 14 24 34
354 // out[5]: 05 15 25 35
355 // out[6]: 06 16 26 36
356 // out[7]: 07 17 27 37
357 out[0] = _mm_unpacklo_epi64(a0, a1);
358 out[1] = _mm_unpackhi_epi64(a0, a1);
359 out[2] = _mm_unpacklo_epi64(a2, a3);
360 out[3] = _mm_unpackhi_epi64(a2, a3);
361 out[4] = _mm_unpacklo_epi64(a4, a5);
362 out[5] = _mm_unpackhi_epi64(a4, a5);
363 out[6] = _mm_unpacklo_epi64(a6, a7);
364 out[7] = _mm_unpackhi_epi64(a6, a7);
365 }
366
367 #endif // VPX_DSP_X86_TRANSPOSE_SSE2_H_
368