1 // Auto-generated file. Do not edit!
2 // Template: src/x32-transpose/sse2.c.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2021 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10 #include <immintrin.h>
11
12 #include <assert.h>
13
14 #include <xnnpack/common.h>
15 #include <xnnpack/math.h>
16 #include <xnnpack/transpose.h>
17
xnn_x16_transpose_ukernel__8x8_multi_switch_sse2(const uint16_t * input,uint16_t * output,size_t input_stride,size_t output_stride,size_t block_width,size_t block_height)18 void xnn_x16_transpose_ukernel__8x8_multi_switch_sse2(
19 const uint16_t* input,
20 uint16_t* output,
21 size_t input_stride,
22 size_t output_stride,
23 size_t block_width,
24 size_t block_height)
25 {
26 assert(output_stride >= block_height * sizeof(uint16_t));
27 assert(input_stride >= block_width * sizeof(uint16_t));
28
29 const size_t tile_height = 8;
30 const size_t tile_width = 8;
31 const size_t tile_hbytes = tile_height * sizeof(uint16_t);
32 const size_t tile_wbytes = tile_width * sizeof(uint16_t);
33 const size_t input_reset = tile_wbytes - round_down_po2(block_height, tile_height) * input_stride;
34 const size_t input_offset = tile_height * input_stride;
35 const size_t output_reset = tile_width * output_stride - round_down_po2(block_height, 2) * sizeof(uint16_t);
36
37 const uint16_t* i0 = input;
38 const uint16_t* i1 = (const uint16_t*) ((uintptr_t) i0 + input_stride);
39 const uint16_t* i2 = (const uint16_t*) ((uintptr_t) i1 + input_stride);
40 const uint16_t* i3 = (const uint16_t*) ((uintptr_t) i2 + input_stride);
41 const uint16_t* i4 = (const uint16_t*) ((uintptr_t) i3 + input_stride);
42 const uint16_t* i5 = (const uint16_t*) ((uintptr_t) i4 + input_stride);
43 const uint16_t* i6 = (const uint16_t*) ((uintptr_t) i5 + input_stride);
44 const uint16_t* i7 = (const uint16_t*) ((uintptr_t) i6 + input_stride);
45 uint16_t* o = (uint16_t*) output;
46 const size_t minus_output_stride = -output_stride;
47
48 do {
49 const size_t rem = min(block_width - 1, 7);
50 const size_t oN_stride = rem * output_stride;
51 size_t bh = block_height;
52 for (; bh >= 8; bh -= 8) {
53 const __m128i v3_0 = _mm_loadu_si128((const __m128i*) i0);
54 i0 = (uint16_t*) ((uintptr_t) i0 + input_offset);
55 const __m128i v3_1 = _mm_loadu_si128((const __m128i*) i1);
56 i1 = (uint16_t*) ((uintptr_t) i1 + input_offset);
57 const __m128i v3_2 = _mm_loadu_si128((const __m128i*) i2);
58 i2 = (uint16_t*) ((uintptr_t) i2 + input_offset);
59 const __m128i v3_3 = _mm_loadu_si128((const __m128i*) i3);
60 i3 = (uint16_t*) ((uintptr_t) i3 + input_offset);
61 const __m128i v3_4 = _mm_loadu_si128((const __m128i*) i4);
62 i4 = (uint16_t*) ((uintptr_t) i4 + input_offset);
63 const __m128i v3_5 = _mm_loadu_si128((const __m128i*) i5);
64 i5 = (uint16_t*) ((uintptr_t) i5 + input_offset);
65 const __m128i v3_6 = _mm_loadu_si128((const __m128i*) i6);
66 i6 = (uint16_t*) ((uintptr_t) i6 + input_offset);
67 const __m128i v3_7 = _mm_loadu_si128((const __m128i*) i7);
68 i7 = (uint16_t*) ((uintptr_t) i7 + input_offset);
69
70 const __m128i v2_0 = _mm_unpacklo_epi16(v3_0, v3_1);
71 const __m128i v2_1 = _mm_unpackhi_epi16(v3_0, v3_1);
72 const __m128i v2_2 = _mm_unpacklo_epi16(v3_2, v3_3);
73 const __m128i v2_3 = _mm_unpackhi_epi16(v3_2, v3_3);
74 const __m128i v2_4 = _mm_unpacklo_epi16(v3_4, v3_5);
75 const __m128i v2_5 = _mm_unpackhi_epi16(v3_4, v3_5);
76 const __m128i v2_6 = _mm_unpacklo_epi16(v3_6, v3_7);
77 const __m128i v2_7 = _mm_unpackhi_epi16(v3_6, v3_7);
78
79 const __m128i v1_0 = _mm_unpacklo_epi32(v2_0, v2_2);
80 const __m128i v1_1 = _mm_unpackhi_epi32(v2_0, v2_2);
81 const __m128i v1_2 = _mm_unpacklo_epi32(v2_1, v2_3);
82 const __m128i v1_3 = _mm_unpackhi_epi32(v2_1, v2_3);
83 const __m128i v1_4 = _mm_unpacklo_epi32(v2_4, v2_6);
84 const __m128i v1_5 = _mm_unpackhi_epi32(v2_4, v2_6);
85 const __m128i v1_6 = _mm_unpacklo_epi32(v2_5, v2_7);
86 const __m128i v1_7 = _mm_unpackhi_epi32(v2_5, v2_7);
87
88 const __m128i v0_0 = _mm_unpacklo_epi64(v1_0, v1_4);
89 const __m128i v0_1 = _mm_unpackhi_epi64(v1_0, v1_4);
90 const __m128i v0_2 = _mm_unpacklo_epi64(v1_1, v1_5);
91 const __m128i v0_3 = _mm_unpackhi_epi64(v1_1, v1_5);
92 const __m128i v0_4 = _mm_unpacklo_epi64(v1_2, v1_6);
93 const __m128i v0_5 = _mm_unpackhi_epi64(v1_2, v1_6);
94 const __m128i v0_6 = _mm_unpacklo_epi64(v1_3, v1_7);
95 const __m128i v0_7 = _mm_unpackhi_epi64(v1_3, v1_7);
96
97
98 uint16_t* oN = (uint16_t*) ((uintptr_t) o + oN_stride);
99 switch (rem) {
100 case 7:
101 _mm_storeu_si128((__m128i*) oN, v0_7);
102 oN = (uint16_t*) ((uintptr_t) oN + minus_output_stride);
103 case 6:
104 _mm_storeu_si128((__m128i*) oN, v0_6);
105 oN = (uint16_t*) ((uintptr_t) oN + minus_output_stride);
106 case 5:
107 _mm_storeu_si128((__m128i*) oN, v0_5);
108 oN = (uint16_t*) ((uintptr_t) oN + minus_output_stride);
109 case 4:
110 _mm_storeu_si128((__m128i*) oN, v0_4);
111 oN = (uint16_t*) ((uintptr_t) oN + minus_output_stride);
112 case 3:
113 _mm_storeu_si128((__m128i*) oN, v0_3);
114 oN = (uint16_t*) ((uintptr_t) oN + minus_output_stride);
115 case 2:
116 _mm_storeu_si128((__m128i*) oN, v0_2);
117 oN = (uint16_t*) ((uintptr_t) oN + minus_output_stride);
118 case 1:
119 _mm_storeu_si128((__m128i*) oN, v0_1);
120 case 0:
121 _mm_storeu_si128((__m128i*) o, v0_0);
122 o = (uint16_t*) ((uintptr_t) o + tile_hbytes);
123 break;
124 default:
125 XNN_UNREACHABLE;
126 }
127 }
128 if (bh != 0) {
129 const __m128i v3_0 = _mm_loadu_si128((const __m128i*) i0);
130 if XNN_UNPREDICTABLE(bh < 2) {
131 i1 = i0;
132 }
133 const __m128i v3_1 = _mm_loadu_si128((const __m128i*) i1);
134 if XNN_UNPREDICTABLE(bh <= 2) {
135 i2 = i0;
136 }
137 const __m128i v3_2 = _mm_loadu_si128((const __m128i*) i2);
138 if XNN_UNPREDICTABLE(bh < 4) {
139 i3 = i0;
140 }
141 const __m128i v3_3 = _mm_loadu_si128((const __m128i*) i3);
142 if XNN_UNPREDICTABLE(bh <= 4) {
143 i4 = i0;
144 }
145 const __m128i v3_4 = _mm_loadu_si128((const __m128i*) i4);
146 if XNN_UNPREDICTABLE(bh < 6) {
147 i5 = i0;
148 }
149 const __m128i v3_5 = _mm_loadu_si128((const __m128i*) i5);
150 if XNN_UNPREDICTABLE(bh <= 6) {
151 i6 = i0;
152 }
153 const __m128i v3_6 = _mm_loadu_si128((const __m128i*) i6);
154 const __m128i v3_7 = _mm_undefined_si128();
155
156 const __m128i v2_0 = _mm_unpacklo_epi16(v3_0, v3_1);
157 const __m128i v2_1 = _mm_unpackhi_epi16(v3_0, v3_1);
158 const __m128i v2_2 = _mm_unpacklo_epi16(v3_2, v3_3);
159 const __m128i v2_3 = _mm_unpackhi_epi16(v3_2, v3_3);
160 const __m128i v2_4 = _mm_unpacklo_epi16(v3_4, v3_5);
161 const __m128i v2_5 = _mm_unpackhi_epi16(v3_4, v3_5);
162 const __m128i v2_6 = _mm_unpacklo_epi16(v3_6, v3_7);
163 const __m128i v2_7 = _mm_unpackhi_epi16(v3_6, v3_7);
164
165 const __m128i v1_0 = _mm_unpacklo_epi32(v2_0, v2_2);
166 const __m128i v1_1 = _mm_unpackhi_epi32(v2_0, v2_2);
167 const __m128i v1_2 = _mm_unpacklo_epi32(v2_1, v2_3);
168 const __m128i v1_3 = _mm_unpackhi_epi32(v2_1, v2_3);
169 const __m128i v1_4 = _mm_unpacklo_epi32(v2_4, v2_6);
170 const __m128i v1_5 = _mm_unpackhi_epi32(v2_4, v2_6);
171 const __m128i v1_6 = _mm_unpacklo_epi32(v2_5, v2_7);
172 const __m128i v1_7 = _mm_unpackhi_epi32(v2_5, v2_7);
173
174 __m128i v0_0 = _mm_unpacklo_epi64(v1_0, v1_4);
175 __m128i v0_1 = _mm_unpackhi_epi64(v1_0, v1_4);
176 __m128i v0_2 = _mm_unpacklo_epi64(v1_1, v1_5);
177 __m128i v0_3 = _mm_unpackhi_epi64(v1_1, v1_5);
178 __m128i v0_4 = _mm_unpacklo_epi64(v1_2, v1_6);
179 __m128i v0_5 = _mm_unpackhi_epi64(v1_2, v1_6);
180 __m128i v0_6 = _mm_unpacklo_epi64(v1_3, v1_7);
181 __m128i v0_7 = _mm_unpackhi_epi64(v1_3, v1_7);
182
183
184 if (bh & 4) {
185 uint16_t* oN = (uint16_t*) ((uintptr_t) o + oN_stride);
186 switch (rem) {
187 case 7:
188 _mm_storel_epi64((__m128i*) oN, v0_7);
189 oN = (uint16_t*) ((uintptr_t) oN + minus_output_stride);
190 case 6:
191 _mm_storel_epi64((__m128i*) oN, v0_6);
192 oN = (uint16_t*) ((uintptr_t) oN + minus_output_stride);
193 case 5:
194 _mm_storel_epi64((__m128i*) oN, v0_5);
195 oN = (uint16_t*) ((uintptr_t) oN + minus_output_stride);
196 case 4:
197 _mm_storel_epi64((__m128i*) oN, v0_4);
198 oN = (uint16_t*) ((uintptr_t) oN + minus_output_stride);
199 case 3:
200 _mm_storel_epi64((__m128i*) oN, v0_3);
201 oN = (uint16_t*) ((uintptr_t) oN + minus_output_stride);
202 case 2:
203 _mm_storel_epi64((__m128i*) oN, v0_2);
204 oN = (uint16_t*) ((uintptr_t) oN + minus_output_stride);
205 case 1:
206 _mm_storel_epi64((__m128i*) oN, v0_1);
207 case 0:
208 _mm_storel_epi64((__m128i*) o, v0_0);
209 break;
210 default:
211 XNN_UNREACHABLE;
212 }
213 o += 4;
214 v0_0 = _mm_unpackhi_epi64(v0_0, v0_0);
215 v0_1 = _mm_unpackhi_epi64(v0_1, v0_1);
216 v0_2 = _mm_unpackhi_epi64(v0_2, v0_2);
217 v0_3 = _mm_unpackhi_epi64(v0_3, v0_3);
218 v0_4 = _mm_unpackhi_epi64(v0_4, v0_4);
219 v0_5 = _mm_unpackhi_epi64(v0_5, v0_5);
220 v0_6 = _mm_unpackhi_epi64(v0_6, v0_6);
221 v0_7 = _mm_unpackhi_epi64(v0_7, v0_7);
222 }
223
224 if (bh & 2) {
225 uint16_t* oN = (uint16_t*) ((uintptr_t) o + oN_stride);
226 switch (rem) {
227 case 7:
228 *((int*) oN) = _mm_cvtsi128_si32(v0_7);
229 oN = (uint16_t*) ((uintptr_t) oN + minus_output_stride);
230 case 6:
231 *((int*) oN) = _mm_cvtsi128_si32(v0_6);
232 oN = (uint16_t*) ((uintptr_t) oN + minus_output_stride);
233 case 5:
234 *((int*) oN) = _mm_cvtsi128_si32(v0_5);
235 oN = (uint16_t*) ((uintptr_t) oN + minus_output_stride);
236 case 4:
237 *((int*) oN) = _mm_cvtsi128_si32(v0_4);
238 oN = (uint16_t*) ((uintptr_t) oN + minus_output_stride);
239 case 3:
240 *((int*) oN) = _mm_cvtsi128_si32(v0_3);
241 oN = (uint16_t*) ((uintptr_t) oN + minus_output_stride);
242 case 2:
243 *((int*) oN) = _mm_cvtsi128_si32(v0_2);
244 oN = (uint16_t*) ((uintptr_t) oN + minus_output_stride);
245 case 1:
246 *((int*) oN) = _mm_cvtsi128_si32(v0_1);
247 case 0:
248 *((int*) o) = _mm_cvtsi128_si32(v0_0);
249 break;
250 default:
251 XNN_UNREACHABLE;
252 }
253 o += 2;
254 v0_0 = _mm_srli_epi64(v0_0, 32);
255 v0_1 = _mm_srli_epi64(v0_1, 32);
256 v0_2 = _mm_srli_epi64(v0_2, 32);
257 v0_3 = _mm_srli_epi64(v0_3, 32);
258 v0_4 = _mm_srli_epi64(v0_4, 32);
259 v0_5 = _mm_srli_epi64(v0_5, 32);
260 v0_6 = _mm_srli_epi64(v0_6, 32);
261 v0_7 = _mm_srli_epi64(v0_7, 32);
262 }
263 if (bh & 1) {
264 uint16_t* oN = (uint16_t*) ((uintptr_t) o + oN_stride);
265 switch (rem) {
266 case 7:
267 *((uint16_t*) oN) = (uint16_t) _mm_cvtsi128_si32(v0_7);
268 oN = (uint16_t*) ((uintptr_t) oN + minus_output_stride);
269 case 6:
270 *((uint16_t*) oN) = (uint16_t) _mm_cvtsi128_si32(v0_6);
271 oN = (uint16_t*) ((uintptr_t) oN + minus_output_stride);
272 case 5:
273 *((uint16_t*) oN) = (uint16_t) _mm_cvtsi128_si32(v0_5);
274 oN = (uint16_t*) ((uintptr_t) oN + minus_output_stride);
275 case 4:
276 *((uint16_t*) oN) = (uint16_t) _mm_cvtsi128_si32(v0_4);
277 oN = (uint16_t*) ((uintptr_t) oN + minus_output_stride);
278 case 3:
279 *((uint16_t*) oN) = (uint16_t) _mm_cvtsi128_si32(v0_3);
280 oN = (uint16_t*) ((uintptr_t) oN + minus_output_stride);
281 case 2:
282 *((uint16_t*) oN) = (uint16_t) _mm_cvtsi128_si32(v0_2);
283 oN = (uint16_t*) ((uintptr_t) oN + minus_output_stride);
284 case 1:
285 *((uint16_t*) oN) = (uint16_t) _mm_cvtsi128_si32(v0_1);
286 case 0:
287 *((uint16_t*) o) = (uint16_t) _mm_cvtsi128_si32(v0_0);
288 break;
289 default:
290 XNN_UNREACHABLE;
291 }
292 }
293 }
294
295 i0 = (const uint16_t*) ((uintptr_t) i0 + input_reset);
296 i1 = (const uint16_t*) ((uintptr_t) i0 + input_stride);
297 i2 = (const uint16_t*) ((uintptr_t) i1 + input_stride);
298 i3 = (const uint16_t*) ((uintptr_t) i2 + input_stride);
299 i4 = (const uint16_t*) ((uintptr_t) i3 + input_stride);
300 i5 = (const uint16_t*) ((uintptr_t) i4 + input_stride);
301 i6 = (const uint16_t*) ((uintptr_t) i5 + input_stride);
302 i7 = (const uint16_t*) ((uintptr_t) i6 + input_stride);
303 o = (uint16_t*) ((uintptr_t) o + output_reset);
304 block_width = doz(block_width, tile_width);
305 } while (block_width != 0);
306 }
307