1 // Auto-generated file. Do not edit!
2 // Template: src/f32-f16-vcvt/sse.c.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2021 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10 #include <assert.h>
11
12 #include <smmintrin.h>
13
14 #include <xnnpack/common.h>
15 #include <xnnpack/vcvt.h>
16
17
xnn_f32_f16_vcvt_ukernel__sse41_x24(size_t n,const float * input,void * output,const union xnn_f32_f16_cvt_params params[restrict XNN_MIN_ELEMENTS (1)])18 void xnn_f32_f16_vcvt_ukernel__sse41_x24(
19 size_t n,
20 const float* input,
21 void* output,
22 const union xnn_f32_f16_cvt_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_OOB_READS
23 {
24 assert(n != 0);
25 assert(n % sizeof(float) == 0);
26 assert(input != NULL);
27 assert(output != NULL);
28
29 const __m128 vnonsign_mask = _mm_load_ps((const float*) params->sse2.nonsign_mask);
30 const __m128i vexp_bias = _mm_load_si128((const __m128i*) params->sse2.exp_bias);
31 const __m128 vscale_to_inf = _mm_load_ps(params->sse2.scale_to_inf);
32 const __m128i vexpw_max = _mm_load_si128((const __m128i*) params->sse2.expw_max);
33 const __m128 vscale_to_zero = _mm_load_ps(params->sse2.scale_to_zero);
34 const __m128i vbias_min = _mm_load_si128((const __m128i*) params->sse2.bias_min);
35 const __m128i vmanth_mask = _mm_load_si128((const __m128i*) params->sse2.manth_mask);
36 const __m128i vexph_mask = _mm_load_si128((const __m128i*) params->sse2.exph_mask);
37 const __m128i vnanh = _mm_load_si128((const __m128i*) params->sse2.nanh);
38
39 uint16_t* o = (uint16_t*) output;
40 for (; n >= 24 * sizeof(float); n -= 24 * sizeof(float)) {
41 const __m128 vx0 = _mm_loadu_ps(input);
42 const __m128 vx1 = _mm_loadu_ps(input + 4);
43 const __m128 vx2 = _mm_loadu_ps(input + 8);
44 const __m128 vx3 = _mm_loadu_ps(input + 12);
45 const __m128 vx4 = _mm_loadu_ps(input + 16);
46 const __m128 vx5 = _mm_loadu_ps(input + 20);
47 input += 24;
48
49 const __m128 vabsx0 = _mm_and_ps(vx0, vnonsign_mask);
50 const __m128 vabsx1 = _mm_and_ps(vx1, vnonsign_mask);
51 const __m128 vabsx2 = _mm_and_ps(vx2, vnonsign_mask);
52 const __m128 vabsx3 = _mm_and_ps(vx3, vnonsign_mask);
53 const __m128 vabsx4 = _mm_and_ps(vx4, vnonsign_mask);
54 const __m128 vabsx5 = _mm_and_ps(vx5, vnonsign_mask);
55
56 const __m128 vsignx0 = _mm_xor_ps(vx0, vabsx0);
57 const __m128 vsignx1 = _mm_xor_ps(vx1, vabsx1);
58 const __m128 vsignx2 = _mm_xor_ps(vx2, vabsx2);
59 const __m128 vsignx3 = _mm_xor_ps(vx3, vabsx3);
60 const __m128 vsignx4 = _mm_xor_ps(vx4, vabsx4);
61 const __m128 vsignx5 = _mm_xor_ps(vx5, vabsx5);
62
63 __m128i vbias0 = _mm_add_epi32(_mm_castps_si128(vabsx0), vexp_bias);
64 __m128i vbias1 = _mm_add_epi32(_mm_castps_si128(vabsx1), vexp_bias);
65 __m128i vbias2 = _mm_add_epi32(_mm_castps_si128(vabsx2), vexp_bias);
66 __m128i vbias3 = _mm_add_epi32(_mm_castps_si128(vabsx3), vexp_bias);
67 __m128i vbias4 = _mm_add_epi32(_mm_castps_si128(vabsx4), vexp_bias);
68 __m128i vbias5 = _mm_add_epi32(_mm_castps_si128(vabsx5), vexp_bias);
69
70 __m128 vf0 = _mm_mul_ps(vabsx0, vscale_to_inf);
71 __m128 vf1 = _mm_mul_ps(vabsx1, vscale_to_inf);
72 __m128 vf2 = _mm_mul_ps(vabsx2, vscale_to_inf);
73 __m128 vf3 = _mm_mul_ps(vabsx3, vscale_to_inf);
74 __m128 vf4 = _mm_mul_ps(vabsx4, vscale_to_inf);
75 __m128 vf5 = _mm_mul_ps(vabsx5, vscale_to_inf);
76
77 const __m128i vnanmaskw0 = _mm_cmpgt_epi32(_mm_castps_si128(vabsx0), vexpw_max);
78 const __m128i vnanmaskw1 = _mm_cmpgt_epi32(_mm_castps_si128(vabsx1), vexpw_max);
79 const __m128i vnanmaskw2 = _mm_cmpgt_epi32(_mm_castps_si128(vabsx2), vexpw_max);
80 const __m128i vnanmaskw3 = _mm_cmpgt_epi32(_mm_castps_si128(vabsx3), vexpw_max);
81 const __m128i vnanmaskw4 = _mm_cmpgt_epi32(_mm_castps_si128(vabsx4), vexpw_max);
82 const __m128i vnanmaskw5 = _mm_cmpgt_epi32(_mm_castps_si128(vabsx5), vexpw_max);
83
84 vbias0 = _mm_and_si128(vbias0, vexpw_max);
85 vbias1 = _mm_and_si128(vbias1, vexpw_max);
86 vbias2 = _mm_and_si128(vbias2, vexpw_max);
87 vbias3 = _mm_and_si128(vbias3, vexpw_max);
88 vbias4 = _mm_and_si128(vbias4, vexpw_max);
89 vbias5 = _mm_and_si128(vbias5, vexpw_max);
90
91 vf0 = _mm_mul_ps(vf0, vscale_to_zero);
92 vf1 = _mm_mul_ps(vf1, vscale_to_zero);
93 vf2 = _mm_mul_ps(vf2, vscale_to_zero);
94 vf3 = _mm_mul_ps(vf3, vscale_to_zero);
95 vf4 = _mm_mul_ps(vf4, vscale_to_zero);
96 vf5 = _mm_mul_ps(vf5, vscale_to_zero);
97
98 const __m128i vnanmaskh0 = _mm_packs_epi32(vnanmaskw0, vnanmaskw1);
99 const __m128i vnanmaskh1 = _mm_packs_epi32(vnanmaskw2, vnanmaskw3);
100 const __m128i vnanmaskh2 = _mm_packs_epi32(vnanmaskw4, vnanmaskw5);
101
102 const __m128i vsignh0 = _mm_packs_epi32(_mm_castps_si128(vsignx0), _mm_castps_si128(vsignx1));
103 const __m128i vsignh1 = _mm_packs_epi32(_mm_castps_si128(vsignx2), _mm_castps_si128(vsignx3));
104 const __m128i vsignh2 = _mm_packs_epi32(_mm_castps_si128(vsignx4), _mm_castps_si128(vsignx5));
105
106 vbias0 = _mm_max_epi16(vbias0, vbias_min);
107 vbias1 = _mm_max_epi16(vbias1, vbias_min);
108 vbias2 = _mm_max_epi16(vbias2, vbias_min);
109 vbias3 = _mm_max_epi16(vbias3, vbias_min);
110 vbias4 = _mm_max_epi16(vbias4, vbias_min);
111 vbias5 = _mm_max_epi16(vbias5, vbias_min);
112
113
114 vf0 = _mm_add_ps(vf0, _mm_castsi128_ps(vbias0));
115 vf1 = _mm_add_ps(vf1, _mm_castsi128_ps(vbias1));
116 vf2 = _mm_add_ps(vf2, _mm_castsi128_ps(vbias2));
117 vf3 = _mm_add_ps(vf3, _mm_castsi128_ps(vbias3));
118 vf4 = _mm_add_ps(vf4, _mm_castsi128_ps(vbias4));
119 vf5 = _mm_add_ps(vf5, _mm_castsi128_ps(vbias5));
120
121
122 __m128i vexpw0 = _mm_srli_epi32(_mm_castps_si128(vf0), 13);
123 __m128i vexpw1 = _mm_srli_epi32(_mm_castps_si128(vf1), 13);
124 __m128i vexpw2 = _mm_srli_epi32(_mm_castps_si128(vf2), 13);
125 __m128i vexpw3 = _mm_srli_epi32(_mm_castps_si128(vf3), 13);
126 __m128i vexpw4 = _mm_srli_epi32(_mm_castps_si128(vf4), 13);
127 __m128i vexpw5 = _mm_srli_epi32(_mm_castps_si128(vf5), 13);
128
129 const __m128i vmantw0 = _mm_and_si128(_mm_castps_si128(vf0), vmanth_mask);
130 const __m128i vmantw1 = _mm_and_si128(_mm_castps_si128(vf1), vmanth_mask);
131 const __m128i vmantw2 = _mm_and_si128(_mm_castps_si128(vf2), vmanth_mask);
132 const __m128i vmantw3 = _mm_and_si128(_mm_castps_si128(vf3), vmanth_mask);
133 const __m128i vmantw4 = _mm_and_si128(_mm_castps_si128(vf4), vmanth_mask);
134 const __m128i vmantw5 = _mm_and_si128(_mm_castps_si128(vf5), vmanth_mask);
135
136 vexpw0 = _mm_and_si128(vexpw0, vexph_mask);
137 vexpw1 = _mm_and_si128(vexpw1, vexph_mask);
138 vexpw2 = _mm_and_si128(vexpw2, vexph_mask);
139 vexpw3 = _mm_and_si128(vexpw3, vexph_mask);
140 vexpw4 = _mm_and_si128(vexpw4, vexph_mask);
141 vexpw5 = _mm_and_si128(vexpw5, vexph_mask);
142
143 const __m128i vnonsignw0 = _mm_add_epi32(vmantw0, vexpw0);
144 const __m128i vnonsignw1 = _mm_add_epi32(vmantw1, vexpw1);
145 const __m128i vnonsignw2 = _mm_add_epi32(vmantw2, vexpw2);
146 const __m128i vnonsignw3 = _mm_add_epi32(vmantw3, vexpw3);
147 const __m128i vnonsignw4 = _mm_add_epi32(vmantw4, vexpw4);
148 const __m128i vnonsignw5 = _mm_add_epi32(vmantw5, vexpw5);
149
150 const __m128i vnonsignh0 = _mm_packs_epi32(vnonsignw0, vnonsignw1);
151 const __m128i vnonsignh1 = _mm_packs_epi32(vnonsignw2, vnonsignw3);
152 const __m128i vnonsignh2 = _mm_packs_epi32(vnonsignw4, vnonsignw5);
153
154 const __m128i vabsh0 = _mm_blendv_epi8(vnonsignh0, vnanh, vnanmaskh0);
155 const __m128i vabsh1 = _mm_blendv_epi8(vnonsignh1, vnanh, vnanmaskh1);
156 const __m128i vabsh2 = _mm_blendv_epi8(vnonsignh2, vnanh, vnanmaskh2);
157
158 const __m128i vh0 = _mm_or_si128(vabsh0, vsignh0);
159 const __m128i vh1 = _mm_or_si128(vabsh1, vsignh1);
160 const __m128i vh2 = _mm_or_si128(vabsh2, vsignh2);
161
162 _mm_storeu_si128((__m128i*) o, vh0);
163 _mm_storeu_si128((__m128i*) (o + 8), vh1);
164 _mm_storeu_si128((__m128i*) (o + 16), vh2);
165 o += 24;
166 }
167 for (; n >= 8 * sizeof(float); n -= 8 * sizeof(float)) {
168 const __m128 vx_lo = _mm_loadu_ps(input);
169 const __m128 vx_hi = _mm_loadu_ps(input + 4);
170 input += 8;
171
172 const __m128 vabsx_lo = _mm_and_ps(vx_lo, vnonsign_mask);
173 const __m128 vabsx_hi = _mm_and_ps(vx_hi, vnonsign_mask);
174
175 const __m128 vsignx_lo = _mm_xor_ps(vx_lo, vabsx_lo);
176 const __m128 vsignx_hi = _mm_xor_ps(vx_hi, vabsx_hi);
177 __m128i vbias_lo = _mm_add_epi32(_mm_castps_si128(vabsx_lo), vexp_bias);
178 __m128i vbias_hi = _mm_add_epi32(_mm_castps_si128(vabsx_hi), vexp_bias);
179 __m128 vf_lo = _mm_mul_ps(vabsx_lo, vscale_to_inf);
180 __m128 vf_hi = _mm_mul_ps(vabsx_hi, vscale_to_inf);
181 const __m128i vnanmaskw_lo = _mm_cmpgt_epi32(_mm_castps_si128(vabsx_lo), vexpw_max);
182 const __m128i vnanmaskw_hi = _mm_cmpgt_epi32(_mm_castps_si128(vabsx_hi), vexpw_max);
183
184 vbias_lo = _mm_and_si128(vbias_lo, vexpw_max);
185 vbias_hi = _mm_and_si128(vbias_hi, vexpw_max);
186 vf_lo = _mm_mul_ps(vf_lo, vscale_to_zero);
187 vf_hi = _mm_mul_ps(vf_hi, vscale_to_zero);
188 const __m128i vnanmaskh = _mm_packs_epi32(vnanmaskw_lo, vnanmaskw_hi);
189 const __m128i vsignh = _mm_packs_epi32(_mm_castps_si128(vsignx_lo), _mm_castps_si128(vsignx_hi));
190
191 vbias_lo = _mm_max_epi16(vbias_lo, vbias_min);
192 vbias_hi = _mm_max_epi16(vbias_hi, vbias_min);
193
194 vf_lo = _mm_add_ps(vf_lo, _mm_castsi128_ps(vbias_lo));
195 vf_hi = _mm_add_ps(vf_hi, _mm_castsi128_ps(vbias_hi));
196
197 __m128i vexpw_lo = _mm_srli_epi32(_mm_castps_si128(vf_lo), 13);
198 __m128i vexpw_hi = _mm_srli_epi32(_mm_castps_si128(vf_hi), 13);
199 const __m128i vmantw_lo = _mm_and_si128(_mm_castps_si128(vf_lo), vmanth_mask);
200 const __m128i vmantw_hi = _mm_and_si128(_mm_castps_si128(vf_hi), vmanth_mask);
201
202 vexpw_lo = _mm_and_si128(vexpw_lo, vexph_mask);
203 vexpw_hi = _mm_and_si128(vexpw_hi, vexph_mask);
204
205 const __m128i vnonsignw_lo = _mm_add_epi32(vmantw_lo, vexpw_lo);
206 const __m128i vnonsignw_hi = _mm_add_epi32(vmantw_hi, vexpw_hi);
207
208 const __m128i vnonsignh = _mm_packs_epi32(vnonsignw_lo, vnonsignw_hi);
209
210 const __m128i vabsh = _mm_blendv_epi8(vnonsignh, vnanh, vnanmaskh);
211
212 const __m128i vh = _mm_or_si128(vabsh, vsignh);
213
214 _mm_storeu_si128((__m128i*) o, vh);
215 o += 8;
216 }
217 if XNN_UNPREDICTABLE(n != 0) {
218 const __m128 vx_lo = _mm_loadu_ps(input);
219 const float* input_hi = (const float*) ((uintptr_t) input + (n & (4 * sizeof(float))));
220 const __m128 vx_hi = _mm_loadu_ps(input_hi);
221
222 const __m128 vabsx_lo = _mm_and_ps(vx_lo, vnonsign_mask);
223 const __m128 vabsx_hi = _mm_and_ps(vx_hi, vnonsign_mask);
224
225 const __m128 vsignx_lo = _mm_xor_ps(vx_lo, vabsx_lo);
226 const __m128 vsignx_hi = _mm_xor_ps(vx_hi, vabsx_hi);
227 __m128i vbias_lo = _mm_add_epi32(_mm_castps_si128(vabsx_lo), vexp_bias);
228 __m128i vbias_hi = _mm_add_epi32(_mm_castps_si128(vabsx_hi), vexp_bias);
229 __m128 vf_lo = _mm_mul_ps(vabsx_lo, vscale_to_inf);
230 __m128 vf_hi = _mm_mul_ps(vabsx_hi, vscale_to_inf);
231 const __m128i vnanmaskw_lo = _mm_cmpgt_epi32(_mm_castps_si128(vabsx_lo), vexpw_max);
232 const __m128i vnanmaskw_hi = _mm_cmpgt_epi32(_mm_castps_si128(vabsx_hi), vexpw_max);
233
234 vbias_lo = _mm_and_si128(vbias_lo, vexpw_max);
235 vbias_hi = _mm_and_si128(vbias_hi, vexpw_max);
236 vf_lo = _mm_mul_ps(vf_lo, vscale_to_zero);
237 vf_hi = _mm_mul_ps(vf_hi, vscale_to_zero);
238 const __m128i vnanmaskh = _mm_packs_epi32(vnanmaskw_lo, vnanmaskw_hi);
239 const __m128i vsignh = _mm_packs_epi32(_mm_castps_si128(vsignx_lo), _mm_castps_si128(vsignx_hi));
240
241 vbias_lo = _mm_max_epi16(vbias_lo, vbias_min);
242 vbias_hi = _mm_max_epi16(vbias_hi, vbias_min);
243
244 vf_lo = _mm_add_ps(vf_lo, _mm_castsi128_ps(vbias_lo));
245 vf_hi = _mm_add_ps(vf_hi, _mm_castsi128_ps(vbias_hi));
246
247 __m128i vexpw_lo = _mm_srli_epi32(_mm_castps_si128(vf_lo), 13);
248 __m128i vexpw_hi = _mm_srli_epi32(_mm_castps_si128(vf_hi), 13);
249 const __m128i vmantw_lo = _mm_and_si128(_mm_castps_si128(vf_lo), vmanth_mask);
250 const __m128i vmantw_hi = _mm_and_si128(_mm_castps_si128(vf_hi), vmanth_mask);
251
252 vexpw_lo = _mm_and_si128(vexpw_lo, vexph_mask);
253 vexpw_hi = _mm_and_si128(vexpw_hi, vexph_mask);
254
255 const __m128i vnonsignw_lo = _mm_add_epi32(vmantw_lo, vexpw_lo);
256 const __m128i vnonsignw_hi = _mm_add_epi32(vmantw_hi, vexpw_hi);
257
258 const __m128i vnonsignh = _mm_packs_epi32(vnonsignw_lo, vnonsignw_hi);
259
260 const __m128i vabsh = _mm_blendv_epi8(vnonsignh, vnanh, vnanmaskh);
261
262 __m128i vh = _mm_or_si128(vabsh, vsignh);
263
264 if (n & (4 * sizeof(float))) {
265 _mm_storel_epi64((__m128i*) o, vh);
266 vh = _mm_unpackhi_epi64(vh, vh);
267 o += 4;
268 }
269 if (n & (2 * sizeof(float))) {
270 *((uint32_t*) o) = (uint32_t) _mm_cvtsi128_si32(vh);
271 vh = _mm_srli_epi64(vh, 32);
272 o += 2;
273 }
274 if (n & (1 * sizeof(float))) {
275 *o = (uint16_t) _mm_extract_epi16(vh, 0);
276 }
277 }
278 }
279