1 /*
2 * Copyright 2019 The libgav1 Authors
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 #ifndef LIBGAV1_SRC_DSP_X86_TRANSPOSE_SSE4_H_
18 #define LIBGAV1_SRC_DSP_X86_TRANSPOSE_SSE4_H_
19
20 #include "src/utils/compiler_attributes.h"
21 #include "src/utils/cpu.h"
22
23 #if LIBGAV1_TARGETING_SSE4_1
24 #include <emmintrin.h>
25
26 namespace libgav1 {
27 namespace dsp {
28
Transpose2x16_U16(const __m128i * const in,__m128i * const out)29 LIBGAV1_ALWAYS_INLINE void Transpose2x16_U16(const __m128i* const in,
30 __m128i* const out) {
31 // Unpack 16 bit elements. Goes from:
32 // in[0]: 00 01 10 11 20 21 30 31
33 // in[1]: 40 41 50 51 60 61 70 71
34 // in[2]: 80 81 90 91 a0 a1 b0 b1
35 // in[3]: c0 c1 d0 d1 e0 e1 f0 f1
36 // to:
37 // a0: 00 40 01 41 10 50 11 51
38 // a1: 20 60 21 61 30 70 31 71
39 // a2: 80 c0 81 c1 90 d0 91 d1
40 // a3: a0 e0 a1 e1 b0 f0 b1 f1
41 const __m128i a0 = _mm_unpacklo_epi16(in[0], in[1]);
42 const __m128i a1 = _mm_unpackhi_epi16(in[0], in[1]);
43 const __m128i a2 = _mm_unpacklo_epi16(in[2], in[3]);
44 const __m128i a3 = _mm_unpackhi_epi16(in[2], in[3]);
45 // b0: 00 20 40 60 01 21 41 61
46 // b1: 10 30 50 70 11 31 51 71
47 // b2: 80 a0 c0 e0 81 a1 c1 e1
48 // b3: 90 b0 d0 f0 91 b1 d1 f1
49 const __m128i b0 = _mm_unpacklo_epi16(a0, a1);
50 const __m128i b1 = _mm_unpackhi_epi16(a0, a1);
51 const __m128i b2 = _mm_unpacklo_epi16(a2, a3);
52 const __m128i b3 = _mm_unpackhi_epi16(a2, a3);
53 // out[0]: 00 10 20 30 40 50 60 70
54 // out[1]: 01 11 21 31 41 51 61 71
55 // out[2]: 80 90 a0 b0 c0 d0 e0 f0
56 // out[3]: 81 91 a1 b1 c1 d1 e1 f1
57 out[0] = _mm_unpacklo_epi16(b0, b1);
58 out[1] = _mm_unpackhi_epi16(b0, b1);
59 out[2] = _mm_unpacklo_epi16(b2, b3);
60 out[3] = _mm_unpackhi_epi16(b2, b3);
61 }
62
Transpose4x4_U8(const __m128i * const in)63 LIBGAV1_ALWAYS_INLINE __m128i Transpose4x4_U8(const __m128i* const in) {
64 // Unpack 8 bit elements. Goes from:
65 // in[0]: 00 01 02 03
66 // in[1]: 10 11 12 13
67 // in[2]: 20 21 22 23
68 // in[3]: 30 31 32 33
69 // to:
70 // a0: 00 10 01 11 02 12 03 13
71 // a1: 20 30 21 31 22 32 23 33
72 const __m128i a0 = _mm_unpacklo_epi8(in[0], in[1]);
73 const __m128i a1 = _mm_unpacklo_epi8(in[2], in[3]);
74
75 // Unpack 32 bit elements resulting in:
76 // 00 10 20 30 01 11 21 31 02 12 22 32 03 13 23 33
77 return _mm_unpacklo_epi16(a0, a1);
78 }
79
Transpose8x8To4x16_U8(const __m128i * const in,__m128i * out)80 LIBGAV1_ALWAYS_INLINE void Transpose8x8To4x16_U8(const __m128i* const in,
81 __m128i* out) {
82 // Unpack 8 bit elements. Goes from:
83 // in[0]: 00 01 02 03 04 05 06 07
84 // in[1]: 10 11 12 13 14 15 16 17
85 // in[2]: 20 21 22 23 24 25 26 27
86 // in[3]: 30 31 32 33 34 35 36 37
87 // in[4]: 40 41 42 43 44 45 46 47
88 // in[5]: 50 51 52 53 54 55 56 57
89 // in[6]: 60 61 62 63 64 65 66 67
90 // in[7]: 70 71 72 73 74 75 76 77
91 // to:
92 // a0: 00 10 01 11 02 12 03 13 04 14 05 15 06 16 07 17
93 // a1: 20 30 21 31 22 32 23 33 24 34 25 35 26 36 27 37
94 // a2: 40 50 41 51 42 52 43 53 44 54 45 55 46 56 47 57
95 // a3: 60 70 61 71 62 72 63 73 64 74 65 75 66 76 67 77
96 const __m128i a0 = _mm_unpacklo_epi8(in[0], in[1]);
97 const __m128i a1 = _mm_unpacklo_epi8(in[2], in[3]);
98 const __m128i a2 = _mm_unpacklo_epi8(in[4], in[5]);
99 const __m128i a3 = _mm_unpacklo_epi8(in[6], in[7]);
100
101 // b0: 00 10 20 30 01 11 21 31 02 12 22 32 03 13 23 33
102 // b1: 40 50 60 70 41 51 61 71 42 52 62 72 43 53 63 73
103 // b2: 04 14 24 34 05 15 25 35 06 16 26 36 07 17 27 37
104 // b3: 44 54 64 74 45 55 65 75 46 56 66 76 47 57 67 77
105 const __m128i b0 = _mm_unpacklo_epi16(a0, a1);
106 const __m128i b1 = _mm_unpacklo_epi16(a2, a3);
107 const __m128i b2 = _mm_unpackhi_epi16(a0, a1);
108 const __m128i b3 = _mm_unpackhi_epi16(a2, a3);
109
110 // out[0]: 00 10 20 30 40 50 60 70 01 11 21 31 41 51 61 71
111 // out[1]: 02 12 22 32 42 52 62 72 03 13 23 33 43 53 63 73
112 // out[2]: 04 14 24 34 44 54 64 74 05 15 25 35 45 55 65 75
113 // out[3]: 06 16 26 36 46 56 66 76 07 17 27 37 47 57 67 77
114 out[0] = _mm_unpacklo_epi32(b0, b1);
115 out[1] = _mm_unpackhi_epi32(b0, b1);
116 out[2] = _mm_unpacklo_epi32(b2, b3);
117 out[3] = _mm_unpackhi_epi32(b2, b3);
118 }
119
Transpose4x4_U16(const __m128i * in,__m128i * out)120 LIBGAV1_ALWAYS_INLINE void Transpose4x4_U16(const __m128i* in, __m128i* out) {
121 // Unpack 16 bit elements. Goes from:
122 // in[0]: 00 01 02 03 XX XX XX XX
123 // in[1]: 10 11 12 13 XX XX XX XX
124 // in[2]: 20 21 22 23 XX XX XX XX
125 // in[3]: 30 31 32 33 XX XX XX XX
126 // to:
127 // ba: 00 10 01 11 02 12 03 13
128 // dc: 20 30 21 31 22 32 23 33
129 const __m128i ba = _mm_unpacklo_epi16(in[0], in[1]);
130 const __m128i dc = _mm_unpacklo_epi16(in[2], in[3]);
131 // Unpack 32 bit elements resulting in:
132 // dcba_lo: 00 10 20 30 01 11 21 31
133 // dcba_hi: 02 12 22 32 03 13 23 33
134 const __m128i dcba_lo = _mm_unpacklo_epi32(ba, dc);
135 const __m128i dcba_hi = _mm_unpackhi_epi32(ba, dc);
136 // Assign or shift right by 8 bytes resulting in:
137 // out[0]: 00 10 20 30 01 11 21 31
138 // out[1]: 01 11 21 31 XX XX XX XX
139 // out[2]: 02 12 22 32 03 13 23 33
140 // out[3]: 03 13 23 33 XX XX XX XX
141 out[0] = dcba_lo;
142 out[1] = _mm_srli_si128(dcba_lo, 8);
143 out[2] = dcba_hi;
144 out[3] = _mm_srli_si128(dcba_hi, 8);
145 }
146
Transpose4x8To8x4_U16(const __m128i * in,__m128i * out)147 LIBGAV1_ALWAYS_INLINE void Transpose4x8To8x4_U16(const __m128i* in,
148 __m128i* out) {
149 // Unpack 16 bit elements. Goes from:
150 // in[0]: 00 01 02 03 XX XX XX XX
151 // in[1]: 10 11 12 13 XX XX XX XX
152 // in[2]: 20 21 22 23 XX XX XX XX
153 // in[3]: 30 31 32 33 XX XX XX XX
154 // in[4]: 40 41 42 43 XX XX XX XX
155 // in[5]: 50 51 52 53 XX XX XX XX
156 // in[6]: 60 61 62 63 XX XX XX XX
157 // in[7]: 70 71 72 73 XX XX XX XX
158 // to:
159 // a0: 00 10 01 11 02 12 03 13
160 // a1: 20 30 21 31 22 32 23 33
161 // a2: 40 50 41 51 42 52 43 53
162 // a3: 60 70 61 71 62 72 63 73
163 const __m128i a0 = _mm_unpacklo_epi16(in[0], in[1]);
164 const __m128i a1 = _mm_unpacklo_epi16(in[2], in[3]);
165 const __m128i a2 = _mm_unpacklo_epi16(in[4], in[5]);
166 const __m128i a3 = _mm_unpacklo_epi16(in[6], in[7]);
167
168 // Unpack 32 bit elements resulting in:
169 // b0: 00 10 20 30 01 11 21 31
170 // b1: 40 50 60 70 41 51 61 71
171 // b2: 02 12 22 32 03 13 23 33
172 // b3: 42 52 62 72 43 53 63 73
173 const __m128i b0 = _mm_unpacklo_epi32(a0, a1);
174 const __m128i b1 = _mm_unpacklo_epi32(a2, a3);
175 const __m128i b2 = _mm_unpackhi_epi32(a0, a1);
176 const __m128i b3 = _mm_unpackhi_epi32(a2, a3);
177
178 // Unpack 64 bit elements resulting in:
179 // out[0]: 00 10 20 30 40 50 60 70
180 // out[1]: 01 11 21 31 41 51 61 71
181 // out[2]: 02 12 22 32 42 52 62 72
182 // out[3]: 03 13 23 33 43 53 63 73
183 out[0] = _mm_unpacklo_epi64(b0, b1);
184 out[1] = _mm_unpackhi_epi64(b0, b1);
185 out[2] = _mm_unpacklo_epi64(b2, b3);
186 out[3] = _mm_unpackhi_epi64(b2, b3);
187 }
188
Transpose8x4To4x8_U16(const __m128i * in,__m128i * out)189 LIBGAV1_ALWAYS_INLINE void Transpose8x4To4x8_U16(const __m128i* in,
190 __m128i* out) {
191 // Unpack 16 bit elements. Goes from:
192 // in[0]: 00 01 02 03 04 05 06 07
193 // in[1]: 10 11 12 13 14 15 16 17
194 // in[2]: 20 21 22 23 24 25 26 27
195 // in[3]: 30 31 32 33 34 35 36 37
196
197 // to:
198 // a0: 00 10 01 11 02 12 03 13
199 // a1: 20 30 21 31 22 32 23 33
200 // a4: 04 14 05 15 06 16 07 17
201 // a5: 24 34 25 35 26 36 27 37
202 const __m128i a0 = _mm_unpacklo_epi16(in[0], in[1]);
203 const __m128i a1 = _mm_unpacklo_epi16(in[2], in[3]);
204 const __m128i a4 = _mm_unpackhi_epi16(in[0], in[1]);
205 const __m128i a5 = _mm_unpackhi_epi16(in[2], in[3]);
206
207 // Unpack 32 bit elements resulting in:
208 // b0: 00 10 20 30 01 11 21 31
209 // b2: 04 14 24 34 05 15 25 35
210 // b4: 02 12 22 32 03 13 23 33
211 // b6: 06 16 26 36 07 17 27 37
212 const __m128i b0 = _mm_unpacklo_epi32(a0, a1);
213 const __m128i b2 = _mm_unpacklo_epi32(a4, a5);
214 const __m128i b4 = _mm_unpackhi_epi32(a0, a1);
215 const __m128i b6 = _mm_unpackhi_epi32(a4, a5);
216
217 // Unpack 64 bit elements resulting in:
218 // out[0]: 00 10 20 30 XX XX XX XX
219 // out[1]: 01 11 21 31 XX XX XX XX
220 // out[2]: 02 12 22 32 XX XX XX XX
221 // out[3]: 03 13 23 33 XX XX XX XX
222 // out[4]: 04 14 24 34 XX XX XX XX
223 // out[5]: 05 15 25 35 XX XX XX XX
224 // out[6]: 06 16 26 36 XX XX XX XX
225 // out[7]: 07 17 27 37 XX XX XX XX
226 const __m128i zeros = _mm_setzero_si128();
227 out[0] = _mm_unpacklo_epi64(b0, zeros);
228 out[1] = _mm_unpackhi_epi64(b0, zeros);
229 out[2] = _mm_unpacklo_epi64(b4, zeros);
230 out[3] = _mm_unpackhi_epi64(b4, zeros);
231 out[4] = _mm_unpacklo_epi64(b2, zeros);
232 out[5] = _mm_unpackhi_epi64(b2, zeros);
233 out[6] = _mm_unpacklo_epi64(b6, zeros);
234 out[7] = _mm_unpackhi_epi64(b6, zeros);
235 }
236
Transpose8x8_U16(const __m128i * const in,__m128i * const out)237 LIBGAV1_ALWAYS_INLINE void Transpose8x8_U16(const __m128i* const in,
238 __m128i* const out) {
239 // Unpack 16 bit elements. Goes from:
240 // in[0]: 00 01 02 03 04 05 06 07
241 // in[1]: 10 11 12 13 14 15 16 17
242 // in[2]: 20 21 22 23 24 25 26 27
243 // in[3]: 30 31 32 33 34 35 36 37
244 // in[4]: 40 41 42 43 44 45 46 47
245 // in[5]: 50 51 52 53 54 55 56 57
246 // in[6]: 60 61 62 63 64 65 66 67
247 // in[7]: 70 71 72 73 74 75 76 77
248 // to:
249 // a0: 00 10 01 11 02 12 03 13
250 // a1: 20 30 21 31 22 32 23 33
251 // a2: 40 50 41 51 42 52 43 53
252 // a3: 60 70 61 71 62 72 63 73
253 // a4: 04 14 05 15 06 16 07 17
254 // a5: 24 34 25 35 26 36 27 37
255 // a6: 44 54 45 55 46 56 47 57
256 // a7: 64 74 65 75 66 76 67 77
257 const __m128i a0 = _mm_unpacklo_epi16(in[0], in[1]);
258 const __m128i a1 = _mm_unpacklo_epi16(in[2], in[3]);
259 const __m128i a2 = _mm_unpacklo_epi16(in[4], in[5]);
260 const __m128i a3 = _mm_unpacklo_epi16(in[6], in[7]);
261 const __m128i a4 = _mm_unpackhi_epi16(in[0], in[1]);
262 const __m128i a5 = _mm_unpackhi_epi16(in[2], in[3]);
263 const __m128i a6 = _mm_unpackhi_epi16(in[4], in[5]);
264 const __m128i a7 = _mm_unpackhi_epi16(in[6], in[7]);
265
266 // Unpack 32 bit elements resulting in:
267 // b0: 00 10 20 30 01 11 21 31
268 // b1: 40 50 60 70 41 51 61 71
269 // b2: 04 14 24 34 05 15 25 35
270 // b3: 44 54 64 74 45 55 65 75
271 // b4: 02 12 22 32 03 13 23 33
272 // b5: 42 52 62 72 43 53 63 73
273 // b6: 06 16 26 36 07 17 27 37
274 // b7: 46 56 66 76 47 57 67 77
275 const __m128i b0 = _mm_unpacklo_epi32(a0, a1);
276 const __m128i b1 = _mm_unpacklo_epi32(a2, a3);
277 const __m128i b2 = _mm_unpacklo_epi32(a4, a5);
278 const __m128i b3 = _mm_unpacklo_epi32(a6, a7);
279 const __m128i b4 = _mm_unpackhi_epi32(a0, a1);
280 const __m128i b5 = _mm_unpackhi_epi32(a2, a3);
281 const __m128i b6 = _mm_unpackhi_epi32(a4, a5);
282 const __m128i b7 = _mm_unpackhi_epi32(a6, a7);
283
284 // Unpack 64 bit elements resulting in:
285 // out[0]: 00 10 20 30 40 50 60 70
286 // out[1]: 01 11 21 31 41 51 61 71
287 // out[2]: 02 12 22 32 42 52 62 72
288 // out[3]: 03 13 23 33 43 53 63 73
289 // out[4]: 04 14 24 34 44 54 64 74
290 // out[5]: 05 15 25 35 45 55 65 75
291 // out[6]: 06 16 26 36 46 56 66 76
292 // out[7]: 07 17 27 37 47 57 67 77
293 out[0] = _mm_unpacklo_epi64(b0, b1);
294 out[1] = _mm_unpackhi_epi64(b0, b1);
295 out[2] = _mm_unpacklo_epi64(b4, b5);
296 out[3] = _mm_unpackhi_epi64(b4, b5);
297 out[4] = _mm_unpacklo_epi64(b2, b3);
298 out[5] = _mm_unpackhi_epi64(b2, b3);
299 out[6] = _mm_unpacklo_epi64(b6, b7);
300 out[7] = _mm_unpackhi_epi64(b6, b7);
301 }
302
303 } // namespace dsp
304 } // namespace libgav1
305
306 #endif // LIBGAV1_TARGETING_SSE4_1
307 #endif // LIBGAV1_SRC_DSP_X86_TRANSPOSE_SSE4_H_
308