1 /*
2 * Copyright 2019 The libgav1 Authors
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 #ifndef LIBGAV1_SRC_DSP_X86_TRANSPOSE_SSE4_H_
18 #define LIBGAV1_SRC_DSP_X86_TRANSPOSE_SSE4_H_
19
20 #include "src/utils/compiler_attributes.h"
21 #include "src/utils/cpu.h"
22
23 #if LIBGAV1_ENABLE_SSE4_1
24 #include <emmintrin.h>
25
26 namespace libgav1 {
27 namespace dsp {
28
Transpose4x4_U8(const __m128i * const in)29 LIBGAV1_ALWAYS_INLINE __m128i Transpose4x4_U8(const __m128i* const in) {
30 // Unpack 8 bit elements. Goes from:
31 // in[0]: 00 01 02 03
32 // in[1]: 10 11 12 13
33 // in[2]: 20 21 22 23
34 // in[3]: 30 31 32 33
35 // to:
36 // a0: 00 10 01 11 02 12 03 13
37 // a1: 20 30 21 31 22 32 23 33
38 const __m128i a0 = _mm_unpacklo_epi8(in[0], in[1]);
39 const __m128i a1 = _mm_unpacklo_epi8(in[2], in[3]);
40
41 // Unpack 32 bit elements resulting in:
42 // 00 10 20 30 01 11 21 31 02 12 22 32 03 13 23 33
43 return _mm_unpacklo_epi16(a0, a1);
44 }
45
Transpose8x8To4x16_U8(const __m128i * const in,__m128i * out)46 LIBGAV1_ALWAYS_INLINE void Transpose8x8To4x16_U8(const __m128i* const in,
47 __m128i* out) {
48 // Unpack 8 bit elements. Goes from:
49 // in[0]: 00 01 02 03 04 05 06 07
50 // in[1]: 10 11 12 13 14 15 16 17
51 // in[2]: 20 21 22 23 24 25 26 27
52 // in[3]: 30 31 32 33 34 35 36 37
53 // in[4]: 40 41 42 43 44 45 46 47
54 // in[5]: 50 51 52 53 54 55 56 57
55 // in[6]: 60 61 62 63 64 65 66 67
56 // in[7]: 70 71 72 73 74 75 76 77
57 // to:
58 // a0: 00 10 01 11 02 12 03 13 04 14 05 15 06 16 07 17
59 // a1: 20 30 21 31 22 32 23 33 24 34 25 35 26 36 27 37
60 // a2: 40 50 41 51 42 52 43 53 44 54 45 55 46 56 47 57
61 // a3: 60 70 61 71 62 72 63 73 64 74 65 75 66 76 67 77
62 const __m128i a0 = _mm_unpacklo_epi8(in[0], in[1]);
63 const __m128i a1 = _mm_unpacklo_epi8(in[2], in[3]);
64 const __m128i a2 = _mm_unpacklo_epi8(in[4], in[5]);
65 const __m128i a3 = _mm_unpacklo_epi8(in[6], in[7]);
66
67 // b0: 00 10 20 30 01 11 21 31 02 12 22 32 03 13 23 33
68 // b1: 40 50 60 70 41 51 61 71 42 52 62 72 43 53 63 73
69 // b2: 04 14 24 34 05 15 25 35 06 16 26 36 07 17 27 37
70 // b3: 44 54 64 74 45 55 65 75 46 56 66 76 47 57 67 77
71 const __m128i b0 = _mm_unpacklo_epi16(a0, a1);
72 const __m128i b1 = _mm_unpacklo_epi16(a2, a3);
73 const __m128i b2 = _mm_unpackhi_epi16(a0, a1);
74 const __m128i b3 = _mm_unpackhi_epi16(a2, a3);
75
76 // out[0]: 00 10 20 30 40 50 60 70 01 11 21 31 41 51 61 71
77 // out[1]: 02 12 22 32 42 52 62 72 03 13 23 33 43 53 63 73
78 // out[2]: 04 14 24 34 44 54 64 74 05 15 25 35 45 55 65 75
79 // out[3]: 06 16 26 36 46 56 66 76 07 17 27 37 47 57 67 77
80 out[0] = _mm_unpacklo_epi32(b0, b1);
81 out[1] = _mm_unpackhi_epi32(b0, b1);
82 out[2] = _mm_unpacklo_epi32(b2, b3);
83 out[3] = _mm_unpackhi_epi32(b2, b3);
84 }
85
Transpose4x4_U16(const __m128i * in,__m128i * out)86 LIBGAV1_ALWAYS_INLINE void Transpose4x4_U16(const __m128i* in, __m128i* out) {
87 // Unpack 16 bit elements. Goes from:
88 // in[0]: 00 01 02 03 XX XX XX XX
89 // in[1]: 10 11 12 13 XX XX XX XX
90 // in[2]: 20 21 22 23 XX XX XX XX
91 // in[3]: 30 31 32 33 XX XX XX XX
92 // to:
93 // ba: 00 10 01 11 02 12 03 13
94 // dc: 20 30 21 31 22 32 23 33
95 const __m128i ba = _mm_unpacklo_epi16(in[0], in[1]);
96 const __m128i dc = _mm_unpacklo_epi16(in[2], in[3]);
97 // Unpack 32 bit elements resulting in:
98 // dcba_lo: 00 10 20 30 01 11 21 31
99 // dcba_hi: 02 12 22 32 03 13 23 33
100 const __m128i dcba_lo = _mm_unpacklo_epi32(ba, dc);
101 const __m128i dcba_hi = _mm_unpackhi_epi32(ba, dc);
102 // Assign or shift right by 8 bytes resulting in:
103 // out[0]: 00 10 20 30 01 11 21 31
104 // out[1]: 01 11 21 31 XX XX XX XX
105 // out[2]: 02 12 22 32 03 13 23 33
106 // out[3]: 03 13 23 33 XX XX XX XX
107 out[0] = dcba_lo;
108 out[1] = _mm_srli_si128(dcba_lo, 8);
109 out[2] = dcba_hi;
110 out[3] = _mm_srli_si128(dcba_hi, 8);
111 }
112
Transpose4x8To8x4_U16(const __m128i * in,__m128i * out)113 LIBGAV1_ALWAYS_INLINE void Transpose4x8To8x4_U16(const __m128i* in,
114 __m128i* out) {
115 // Unpack 16 bit elements. Goes from:
116 // in[0]: 00 01 02 03 XX XX XX XX
117 // in[1]: 10 11 12 13 XX XX XX XX
118 // in[2]: 20 21 22 23 XX XX XX XX
119 // in[3]: 30 31 32 33 XX XX XX XX
120 // in[4]: 40 41 42 43 XX XX XX XX
121 // in[5]: 50 51 52 53 XX XX XX XX
122 // in[6]: 60 61 62 63 XX XX XX XX
123 // in[7]: 70 71 72 73 XX XX XX XX
124 // to:
125 // a0: 00 10 01 11 02 12 03 13
126 // a1: 20 30 21 31 22 32 23 33
127 // a2: 40 50 41 51 42 52 43 53
128 // a3: 60 70 61 71 62 72 63 73
129 const __m128i a0 = _mm_unpacklo_epi16(in[0], in[1]);
130 const __m128i a1 = _mm_unpacklo_epi16(in[2], in[3]);
131 const __m128i a2 = _mm_unpacklo_epi16(in[4], in[5]);
132 const __m128i a3 = _mm_unpacklo_epi16(in[6], in[7]);
133
134 // Unpack 32 bit elements resulting in:
135 // b0: 00 10 20 30 01 11 21 31
136 // b1: 40 50 60 70 41 51 61 71
137 // b2: 02 12 22 32 03 13 23 33
138 // b3: 42 52 62 72 43 53 63 73
139 const __m128i b0 = _mm_unpacklo_epi32(a0, a1);
140 const __m128i b1 = _mm_unpacklo_epi32(a2, a3);
141 const __m128i b2 = _mm_unpackhi_epi32(a0, a1);
142 const __m128i b3 = _mm_unpackhi_epi32(a2, a3);
143
144 // Unpack 64 bit elements resulting in:
145 // out[0]: 00 10 20 30 40 50 60 70
146 // out[1]: 01 11 21 31 41 51 61 71
147 // out[2]: 02 12 22 32 42 52 62 72
148 // out[3]: 03 13 23 33 43 53 63 73
149 out[0] = _mm_unpacklo_epi64(b0, b1);
150 out[1] = _mm_unpackhi_epi64(b0, b1);
151 out[2] = _mm_unpacklo_epi64(b2, b3);
152 out[3] = _mm_unpackhi_epi64(b2, b3);
153 }
154
Transpose8x4To4x8_U16(const __m128i * in,__m128i * out)155 LIBGAV1_ALWAYS_INLINE void Transpose8x4To4x8_U16(const __m128i* in,
156 __m128i* out) {
157 // Unpack 16 bit elements. Goes from:
158 // in[0]: 00 01 02 03 04 05 06 07
159 // in[1]: 10 11 12 13 14 15 16 17
160 // in[2]: 20 21 22 23 24 25 26 27
161 // in[3]: 30 31 32 33 34 35 36 37
162
163 // to:
164 // a0: 00 10 01 11 02 12 03 13
165 // a1: 20 30 21 31 22 32 23 33
166 // a4: 04 14 05 15 06 16 07 17
167 // a5: 24 34 25 35 26 36 27 37
168 const __m128i a0 = _mm_unpacklo_epi16(in[0], in[1]);
169 const __m128i a1 = _mm_unpacklo_epi16(in[2], in[3]);
170 const __m128i a4 = _mm_unpackhi_epi16(in[0], in[1]);
171 const __m128i a5 = _mm_unpackhi_epi16(in[2], in[3]);
172
173 // Unpack 32 bit elements resulting in:
174 // b0: 00 10 20 30 01 11 21 31
175 // b2: 04 14 24 34 05 15 25 35
176 // b4: 02 12 22 32 03 13 23 33
177 // b6: 06 16 26 36 07 17 27 37
178 const __m128i b0 = _mm_unpacklo_epi32(a0, a1);
179 const __m128i b2 = _mm_unpacklo_epi32(a4, a5);
180 const __m128i b4 = _mm_unpackhi_epi32(a0, a1);
181 const __m128i b6 = _mm_unpackhi_epi32(a4, a5);
182
183 // Unpack 64 bit elements resulting in:
184 // out[0]: 00 10 20 30 XX XX XX XX
185 // out[1]: 01 11 21 31 XX XX XX XX
186 // out[2]: 02 12 22 32 XX XX XX XX
187 // out[3]: 03 13 23 33 XX XX XX XX
188 // out[4]: 04 14 24 34 XX XX XX XX
189 // out[5]: 05 15 25 35 XX XX XX XX
190 // out[6]: 06 16 26 36 XX XX XX XX
191 // out[7]: 07 17 27 37 XX XX XX XX
192 const __m128i zeros = _mm_setzero_si128();
193 out[0] = _mm_unpacklo_epi64(b0, zeros);
194 out[1] = _mm_unpackhi_epi64(b0, zeros);
195 out[2] = _mm_unpacklo_epi64(b4, zeros);
196 out[3] = _mm_unpackhi_epi64(b4, zeros);
197 out[4] = _mm_unpacklo_epi64(b2, zeros);
198 out[5] = _mm_unpackhi_epi64(b2, zeros);
199 out[6] = _mm_unpacklo_epi64(b6, zeros);
200 out[7] = _mm_unpackhi_epi64(b6, zeros);
201 }
202
Transpose8x8_U16(const __m128i * const in,__m128i * const out)203 LIBGAV1_ALWAYS_INLINE void Transpose8x8_U16(const __m128i* const in,
204 __m128i* const out) {
205 // Unpack 16 bit elements. Goes from:
206 // in[0]: 00 01 02 03 04 05 06 07
207 // in[1]: 10 11 12 13 14 15 16 17
208 // in[2]: 20 21 22 23 24 25 26 27
209 // in[3]: 30 31 32 33 34 35 36 37
210 // in[4]: 40 41 42 43 44 45 46 47
211 // in[5]: 50 51 52 53 54 55 56 57
212 // in[6]: 60 61 62 63 64 65 66 67
213 // in[7]: 70 71 72 73 74 75 76 77
214 // to:
215 // a0: 00 10 01 11 02 12 03 13
216 // a1: 20 30 21 31 22 32 23 33
217 // a2: 40 50 41 51 42 52 43 53
218 // a3: 60 70 61 71 62 72 63 73
219 // a4: 04 14 05 15 06 16 07 17
220 // a5: 24 34 25 35 26 36 27 37
221 // a6: 44 54 45 55 46 56 47 57
222 // a7: 64 74 65 75 66 76 67 77
223 const __m128i a0 = _mm_unpacklo_epi16(in[0], in[1]);
224 const __m128i a1 = _mm_unpacklo_epi16(in[2], in[3]);
225 const __m128i a2 = _mm_unpacklo_epi16(in[4], in[5]);
226 const __m128i a3 = _mm_unpacklo_epi16(in[6], in[7]);
227 const __m128i a4 = _mm_unpackhi_epi16(in[0], in[1]);
228 const __m128i a5 = _mm_unpackhi_epi16(in[2], in[3]);
229 const __m128i a6 = _mm_unpackhi_epi16(in[4], in[5]);
230 const __m128i a7 = _mm_unpackhi_epi16(in[6], in[7]);
231
232 // Unpack 32 bit elements resulting in:
233 // b0: 00 10 20 30 01 11 21 31
234 // b1: 40 50 60 70 41 51 61 71
235 // b2: 04 14 24 34 05 15 25 35
236 // b3: 44 54 64 74 45 55 65 75
237 // b4: 02 12 22 32 03 13 23 33
238 // b5: 42 52 62 72 43 53 63 73
239 // b6: 06 16 26 36 07 17 27 37
240 // b7: 46 56 66 76 47 57 67 77
241 const __m128i b0 = _mm_unpacklo_epi32(a0, a1);
242 const __m128i b1 = _mm_unpacklo_epi32(a2, a3);
243 const __m128i b2 = _mm_unpacklo_epi32(a4, a5);
244 const __m128i b3 = _mm_unpacklo_epi32(a6, a7);
245 const __m128i b4 = _mm_unpackhi_epi32(a0, a1);
246 const __m128i b5 = _mm_unpackhi_epi32(a2, a3);
247 const __m128i b6 = _mm_unpackhi_epi32(a4, a5);
248 const __m128i b7 = _mm_unpackhi_epi32(a6, a7);
249
250 // Unpack 64 bit elements resulting in:
251 // out[0]: 00 10 20 30 40 50 60 70
252 // out[1]: 01 11 21 31 41 51 61 71
253 // out[2]: 02 12 22 32 42 52 62 72
254 // out[3]: 03 13 23 33 43 53 63 73
255 // out[4]: 04 14 24 34 44 54 64 74
256 // out[5]: 05 15 25 35 45 55 65 75
257 // out[6]: 06 16 26 36 46 56 66 76
258 // out[7]: 07 17 27 37 47 57 67 77
259 out[0] = _mm_unpacklo_epi64(b0, b1);
260 out[1] = _mm_unpackhi_epi64(b0, b1);
261 out[2] = _mm_unpacklo_epi64(b4, b5);
262 out[3] = _mm_unpackhi_epi64(b4, b5);
263 out[4] = _mm_unpacklo_epi64(b2, b3);
264 out[5] = _mm_unpackhi_epi64(b2, b3);
265 out[6] = _mm_unpacklo_epi64(b6, b7);
266 out[7] = _mm_unpackhi_epi64(b6, b7);
267 }
268
269 } // namespace dsp
270 } // namespace libgav1
271
272 #endif // LIBGAV1_ENABLE_SSE4_1
273 #endif // LIBGAV1_SRC_DSP_X86_TRANSPOSE_SSE4_H_
274