1 // Copyright 2019 Google LLC
2 //
3 // This source code is licensed under the BSD-style license found in the
4 // LICENSE file in the root directory of this source tree.
5
6 #include <assert.h>
7
8 #include <xmmintrin.h>
9
10 #include <xnnpack/dwconv.h>
11 #include <xnnpack/math.h>
12
13
xnn_f32_dwconv_spchw_ukernel_3x3s2p1__sse(size_t m,size_t n,const float * input,const float * weights,float * output,size_t input_tuple_stride,size_t output_tuple_stride,size_t input_width_stride,size_t output_width_stride,const union xnn_f32_spchw_params params[restrict static1])14 void xnn_f32_dwconv_spchw_ukernel_3x3s2p1__sse(
15 size_t m,
16 size_t n,
17 const float* input,
18 const float* weights,
19 float* output,
20 size_t input_tuple_stride,
21 size_t output_tuple_stride,
22 size_t input_width_stride,
23 size_t output_width_stride,
24 const union xnn_f32_spchw_params params[restrict static 1])
25 {
26 assert(n != 0);
27
28 const __m128 vmask_even = _mm_load_ps((const float*) params->sse.mask_even);
29 const __m128 vmask_odd = _mm_load_ps((const float*) params->sse.mask_odd);
30 const __m128 vmax = _mm_load_ps(params->sse.max);
31 const __m128 vmin = _mm_load_ps(params->sse.min);
32
33 const size_t input_width_increment = input_width_stride * 2 - n / 8 * input_tuple_stride * 2;
34 const size_t output_width_increment = output_width_stride - n / 8 * output_tuple_stride;
35
36 // No vertical padding.
37 const float* i0 = input;
38 const float* i1 = (const float*) ((uintptr_t) i0 + input_width_stride);
39 const float* i2 = (const float*) ((uintptr_t) i1 + input_width_stride);
40
41 const __m128 vbias = _mm_load1_ps(weights);
42 const __m128 vk00 = _mm_load1_ps(weights + 1);
43 const __m128 vk01 = _mm_load1_ps(weights + 2);
44 const __m128 vk02 = _mm_load1_ps(weights + 3);
45 const __m128 vk10 = _mm_load1_ps(weights + 4);
46 const __m128 vk11 = _mm_load1_ps(weights + 5);
47 const __m128 vk12 = _mm_load1_ps(weights + 6);
48 const __m128 vk20 = _mm_load1_ps(weights + 7);
49 const __m128 vk21 = _mm_load1_ps(weights + 8);
50 const __m128 vk22 = _mm_load1_ps(weights + 9);
51
52 do {
53 __m128 vi0x7531 = _mm_setzero_ps();
54 __m128 vi1x7531 = _mm_setzero_ps();
55 __m128 vi2x7531 = _mm_setzero_ps();
56
57 size_t k = n;
58 for (; k >= 8; k -= 8) {
59 __m128 vo8ACEp0 = vbias;
60
61 const __m128 vi0x89AB = _mm_loadu_ps(i0);
62 i0 = (const float*) ((uintptr_t) i0 + input_tuple_stride);
63 const __m128 vi1x89AB = _mm_loadu_ps(i1);
64 i1 = (const float*) ((uintptr_t) i1 + input_tuple_stride);
65 const __m128 vi2x89AB = _mm_loadu_ps(i2);
66 i2 = (const float*) ((uintptr_t) i2 + input_tuple_stride);
67
68 const __m128 vi0xCDEF = _mm_loadu_ps(i0);
69 i0 = (const float*) ((uintptr_t) i0 + input_tuple_stride);
70 const __m128 vi1xCDEF = _mm_loadu_ps(i1);
71 i1 = (const float*) ((uintptr_t) i1 + input_tuple_stride);
72 const __m128 vi2xCDEF = _mm_loadu_ps(i2);
73 i2 = (const float*) ((uintptr_t) i2 + input_tuple_stride);
74
75 const __m128 vi0x8ACE = _mm_shuffle_ps(vi0x89AB, vi0xCDEF, _MM_SHUFFLE(2, 0, 2, 0));
76 const __m128 vi0x9BDF = _mm_shuffle_ps(vi0x89AB, vi0xCDEF, _MM_SHUFFLE(3, 1, 3, 1));
77 const __m128 vi1x8ACE = _mm_shuffle_ps(vi1x89AB, vi1xCDEF, _MM_SHUFFLE(2, 0, 2, 0));
78 const __m128 vi1x9BDF = _mm_shuffle_ps(vi1x89AB, vi1xCDEF, _MM_SHUFFLE(3, 1, 3, 1));
79 const __m128 vi2x8ACE = _mm_shuffle_ps(vi2x89AB, vi2xCDEF, _MM_SHUFFLE(2, 0, 2, 0));
80 const __m128 vi2x9BDF = _mm_shuffle_ps(vi2x89AB, vi2xCDEF, _MM_SHUFFLE(3, 1, 3, 1));
81
82 vo8ACEp0 = _mm_add_ps(vo8ACEp0, _mm_mul_ps(vi0x8ACE, vk01));
83 __m128 vo8ACEp1 = _mm_mul_ps(vi1x8ACE, vk11);
84 __m128 vo8ACEp2 = _mm_mul_ps(vi2x8ACE, vk21);
85
86 const __m128 vi0xF9BD = _mm_shuffle_ps(vi0x9BDF, vi0x9BDF, _MM_SHUFFLE(2, 1, 0, 3));
87 const __m128 vi1xF9BD = _mm_shuffle_ps(vi1x9BDF, vi1x9BDF, _MM_SHUFFLE(2, 1, 0, 3));
88 const __m128 vi2xF9BD = _mm_shuffle_ps(vi2x9BDF, vi2x9BDF, _MM_SHUFFLE(2, 1, 0, 3));
89
90 vo8ACEp0 = _mm_add_ps(vo8ACEp0, _mm_mul_ps(vi0x9BDF, vk02));
91 vo8ACEp1 = _mm_add_ps(vo8ACEp1, _mm_mul_ps(vi1x9BDF, vk12));
92 vo8ACEp2 = _mm_add_ps(vo8ACEp2, _mm_mul_ps(vi2x9BDF, vk22));
93
94 const __m128 vi0x7BDF = _mm_move_ss(vi0xF9BD, vi0x7531);
95 const __m128 vi1x7BDF = _mm_move_ss(vi1xF9BD, vi1x7531);
96 const __m128 vi2x7BDF = _mm_move_ss(vi2xF9BD, vi2x7531);
97
98 vi0x7531 = vi0xF9BD;
99 vi1x7531 = vi1xF9BD;
100 vi2x7531 = vi2xF9BD;
101
102 vo8ACEp0 = _mm_add_ps(vo8ACEp0, _mm_mul_ps(vi0x7BDF, vk00));
103 vo8ACEp1 = _mm_add_ps(vo8ACEp1, _mm_mul_ps(vi1x7BDF, vk10));
104 vo8ACEp2 = _mm_add_ps(vo8ACEp2, _mm_mul_ps(vi2x7BDF, vk20));
105
106 __m128 vo = _mm_add_ps(vo8ACEp0, vo8ACEp1);
107 vo = _mm_add_ps(vo, vo8ACEp2);
108
109 vo = _mm_max_ps(vo, vmin);
110 vo = _mm_min_ps(vo, vmax);
111
112 _mm_storeu_ps(output, vo);
113 output = (float*) ((uintptr_t) output + output_tuple_stride);
114 }
115 // Last block has 0-7 pixels to process.
116 assert(k < 8);
117 if XNN_LIKELY(k != 0) {
118 __m128 vo8ACEp0 = vbias;
119
120 const __m128 vi0x89AB = _mm_loadu_ps(i0);
121 const __m128 vi1x89AB = _mm_loadu_ps(i1);
122 const __m128 vi2x89AB = _mm_loadu_ps(i2);
123
124 const __m128 vi0xCDEF = _mm_loadu_ps((const float*) ((uintptr_t) i0 + input_tuple_stride));
125 const __m128 vi1xCDEF = _mm_loadu_ps((const float*) ((uintptr_t) i1 + input_tuple_stride));
126 const __m128 vi2xCDEF = _mm_loadu_ps((const float*) ((uintptr_t) i2 + input_tuple_stride));
127
128 const __m128 vi0x8ACE = _mm_and_ps(vmask_even, _mm_shuffle_ps(vi0x89AB, vi0xCDEF, _MM_SHUFFLE(2, 0, 2, 0)));
129 const __m128 vi0x9BDF = _mm_and_ps(vmask_odd, _mm_shuffle_ps(vi0x89AB, vi0xCDEF, _MM_SHUFFLE(3, 1, 3, 1)));
130 const __m128 vi1x8ACE = _mm_and_ps(vmask_even, _mm_shuffle_ps(vi1x89AB, vi1xCDEF, _MM_SHUFFLE(2, 0, 2, 0)));
131 const __m128 vi1x9BDF = _mm_and_ps(vmask_odd, _mm_shuffle_ps(vi1x89AB, vi1xCDEF, _MM_SHUFFLE(3, 1, 3, 1)));
132 const __m128 vi2x8ACE = _mm_and_ps(vmask_even, _mm_shuffle_ps(vi2x89AB, vi2xCDEF, _MM_SHUFFLE(2, 0, 2, 0)));
133 const __m128 vi2x9BDF = _mm_and_ps(vmask_odd, _mm_shuffle_ps(vi2x89AB, vi2xCDEF, _MM_SHUFFLE(3, 1, 3, 1)));
134
135 vo8ACEp0 = _mm_add_ps(vo8ACEp0, _mm_mul_ps(vi0x8ACE, vk01));
136 __m128 vo8ACEp1 = _mm_mul_ps(vi1x8ACE, vk11);
137 __m128 vo8ACEp2 = _mm_mul_ps(vi2x8ACE, vk21);
138
139 const __m128 vi0xF9BD = _mm_shuffle_ps(vi0x9BDF, vi0x9BDF, _MM_SHUFFLE(2, 1, 0, 3));
140 const __m128 vi1xF9BD = _mm_shuffle_ps(vi1x9BDF, vi1x9BDF, _MM_SHUFFLE(2, 1, 0, 3));
141 const __m128 vi2xF9BD = _mm_shuffle_ps(vi2x9BDF, vi2x9BDF, _MM_SHUFFLE(2, 1, 0, 3));
142
143 vo8ACEp0 = _mm_add_ps(vo8ACEp0, _mm_mul_ps(vi0x9BDF, vk02));
144 vo8ACEp1 = _mm_add_ps(vo8ACEp1, _mm_mul_ps(vi1x9BDF, vk12));
145 vo8ACEp2 = _mm_add_ps(vo8ACEp2, _mm_mul_ps(vi2x9BDF, vk22));
146
147 const __m128 vi0x7BDF = _mm_move_ss(vi0xF9BD, vi0x7531);
148 const __m128 vi1x7BDF = _mm_move_ss(vi1xF9BD, vi1x7531);
149 const __m128 vi2x7BDF = _mm_move_ss(vi2xF9BD, vi2x7531);
150
151 vo8ACEp0 = _mm_add_ps(vo8ACEp0, _mm_mul_ps(vi0x7BDF, vk00));
152 vo8ACEp1 = _mm_add_ps(vo8ACEp1, _mm_mul_ps(vi1x7BDF, vk10));
153 vo8ACEp2 = _mm_add_ps(vo8ACEp2, _mm_mul_ps(vi2x7BDF, vk20));
154
155 __m128 vo = _mm_add_ps(vo8ACEp0, vo8ACEp1);
156 vo = _mm_add_ps(vo, vo8ACEp2);
157
158 vo = _mm_max_ps(vo, vmin);
159 vo = _mm_min_ps(vo, vmax);
160
161 if (k == 7) {
162 _mm_storeu_ps(output, vo);
163 } else {
164 float* output_lo = output;
165 k += 1;
166 if (k & 4) {
167 _mm_storel_pi((__m64*) output_lo, vo);
168 output_lo += 2;
169 vo = _mm_movehl_ps(vo, vo);
170 }
171 if (k & 2) {
172 _mm_store_ss(output_lo, vo);
173 }
174 }
175 }
176
177 i0 = (const float*) ((uintptr_t) i0 + input_width_increment);
178 i1 = (const float*) ((uintptr_t) i1 + input_width_increment);
179 i2 = (const float*) ((uintptr_t) i2 + input_width_increment);
180 output = (float*) ((uintptr_t) output + output_width_increment);
181 } while (--m != 0);
182 }
183