1 // Copyright 2019 Google LLC
2 //
3 // This source code is licensed under the BSD-style license found in the
4 // LICENSE file in the root directory of this source tree.
5
6 #include <assert.h>
7
8 #include <emmintrin.h>
9
10 #include <xnnpack/zip.h>
11
12
xnn_x32_zip_x3_ukernel__sse2(size_t n,const uint32_t * input,uint32_t * output)13 void xnn_x32_zip_x3_ukernel__sse2(
14 size_t n,
15 const uint32_t* input,
16 uint32_t* output)
17 {
18 assert(n != 0);
19 assert(n % 4 == 0);
20
21 const float* x = (const float*) input;
22 const float* y = (const float*) ((uintptr_t) x + n);
23 const float* z = (const float*) ((uintptr_t) y + n);
24 float* o = (float*) output;
25
26 while (n >= 16) {
27 // vx = ( x3, x2, x1, x0 )
28 const __m128 vx = _mm_loadu_ps(x);
29 x += 4;
30 // vy = ( y3, y2, y1, y0 )
31 const __m128 vy = _mm_loadu_ps(y);
32 y += 4;
33 // vz = ( z3, z2, z1, z0 )
34 const __m128 vz = _mm_loadu_ps(z);
35 z += 4;
36
37 // vxy = ( y2, y0, x2, x0 )
38 const __m128 vxy = _mm_shuffle_ps(vx, vy, _MM_SHUFFLE(2, 0, 2, 0));
39 // vyz = ( z3, z1, y3, y1 )
40 const __m128 vyz = _mm_shuffle_ps(vy, vz, _MM_SHUFFLE(3, 1, 3, 1));
41 // vzx = ( x3, x1, z2, z0 )
42 const __m128 vzx = _mm_shuffle_ps(vz, vx, _MM_SHUFFLE(3, 1, 2, 0));
43
44 // vxyz0 = ( x1, z0, y0, x0 )
45 const __m128 vxyz0 = _mm_shuffle_ps(vxy, vzx, _MM_SHUFFLE(2, 0, 2, 0));
46 // vxyz1 = ( y2, x2, z1, y1 )
47 const __m128 vxyz1 = _mm_shuffle_ps(vyz, vxy, _MM_SHUFFLE(3, 1, 2, 0));
48 // vxyz2 = ( z3, y3, x3, z2 )
49 const __m128 vxyz2 = _mm_shuffle_ps(vzx, vyz, _MM_SHUFFLE(3, 1, 3, 1));
50
51 _mm_storeu_ps(o, vxyz0);
52 _mm_storeu_ps(o + 4, vxyz1);
53 _mm_storeu_ps(o + 8, vxyz2);
54 o += 12;
55 n -= 16;
56 }
57 if XNN_UNLIKELY(n != 0) {
58 if (n & 8) {
59 // vx = ( -, -, x1, x0 )
60 const __m128 vx = _mm_castpd_ps(_mm_load_sd((const double*) x));
61 x += 2;
62 // vy = ( -, -, y1, y0 )
63 const __m128 vy = _mm_castpd_ps(_mm_load_sd((const double*) y));
64 y += 2;
65 // vz = ( -, -, z1, z0 )
66 const __m128 vz = _mm_castpd_ps(_mm_load_sd((const double*) z));
67 z += 2;
68
69 // vxy = ( y1, x1, y0, x0 )
70 const __m128 vxy = _mm_unpacklo_ps(vx, vy);
71 // vzx = ( x1, z1, x0, z0 )
72 const __m128 vzx = _mm_unpacklo_ps(vz, vx);
73 // vyz = ( z1, y1, z0, y0 )
74 const __m128 vyz = _mm_unpacklo_ps(vy, vz);
75
76 _mm_storeu_ps(o, _mm_shuffle_ps(vxy, vzx, _MM_SHUFFLE(3, 0, 1, 0)));
77 _mm_storeh_pi((__m64*) (o + 4), vyz);
78 o += 6;
79 }
80 if (n & 4) {
81 const __m128 vx = _mm_load_ss(x);
82 const __m128 vy = _mm_load_ss(y);
83 const __m128 vz = _mm_load_ss(z);
84 _mm_store_ss(o, vx);
85 _mm_store_ss(o + 1, vy);
86 _mm_store_ss(o + 2, vz);
87 }
88 }
89 }
90