1 // Auto-generated file. Do not edit!
2 // Template: src/f16-spmm/neonfp16arith.c.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2019 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10 #include <assert.h>
11
12 #include <arm_neon.h>
13
14 #include <xnnpack/spmm.h>
15
16
xnn_f16_spmm_ukernel_32x1__neonfp16arith_unroll2(uint32_t m,uint32_t n,const void * restrict input,const void * restrict weights,const int32_t * restrict widx_dmap,const uint32_t * restrict nidx_nnzmap,void * restrict output,const struct xnn_f16_output_params params[restrict static1])17 void xnn_f16_spmm_ukernel_32x1__neonfp16arith_unroll2(
18 uint32_t m,
19 uint32_t n,
20 const void*restrict input,
21 const void*restrict weights,
22 const int32_t*restrict widx_dmap,
23 const uint32_t*restrict nidx_nnzmap,
24 void*restrict output,
25 const struct xnn_f16_output_params params[restrict static 1])
26 {
27 assert(m != 0);
28
29 const __fp16*restrict a = input;
30 __fp16*restrict c = output;
31
32 const float16x8_t vscale = vld1q_dup_f16((const __fp16*) ¶ms->scale);
33 const float16x8_t vmax = vld1q_dup_f16((const __fp16*) ¶ms->max);
34 const float16x8_t vmin = vld1q_dup_f16((const __fp16*) ¶ms->min);
35
36 size_t i = m;
37 while XNN_LIKELY(i >= 32) {
38 const __fp16*restrict w = weights;
39 const int32_t* dmap = widx_dmap;
40 const uint32_t* nnzmap = nidx_nnzmap;
41 size_t j = n;
42 do {
43 uint32_t nnz = *nnzmap++;
44 float16x8_t vacc01234567x0 = vld1q_dup_f16(w); w += 1;
45 float16x8_t vacc01234567x1 = vmovq_n_f16(0.0f);
46 float16x8_t vacc89ABCDEFx0 = vacc01234567x0;
47 float16x8_t vacc89ABCDEFx1 = vmovq_n_f16(0.0f);
48 float16x8_t vaccGHIJKLMNx0 = vacc01234567x0;
49 float16x8_t vaccGHIJKLMNx1 = vmovq_n_f16(0.0f);
50 float16x8_t vaccOPQRSTUVx0 = vacc01234567x0;
51 float16x8_t vaccOPQRSTUVx1 = vmovq_n_f16(0.0f);
52 for (; nnz >= 2; nnz -= 2) {
53 const intptr_t diff0 = dmap[0];
54 const intptr_t diff1 = dmap[1];
55 dmap += 2;
56 const float16x8_t va01234567x0 = vld1q_f16(a);
57 const float16x8_t va89ABCDEFx0 = vld1q_f16(a + 8);
58 const float16x8_t vaGHIJKLMNx0 = vld1q_f16(a + 16);
59 const float16x8_t vaOPQRSTUVx0 = vld1q_f16(a + 24);
60 a = (const __fp16*restrict) ((uintptr_t) a + (uintptr_t) diff0);
61 const float16x8_t vb0 = vld1q_dup_f16(w); w += 1;
62 vacc01234567x0 = vfmaq_f16(vacc01234567x0, va01234567x0, vb0);
63 vacc89ABCDEFx0 = vfmaq_f16(vacc89ABCDEFx0, va89ABCDEFx0, vb0);
64 vaccGHIJKLMNx0 = vfmaq_f16(vaccGHIJKLMNx0, vaGHIJKLMNx0, vb0);
65 vaccOPQRSTUVx0 = vfmaq_f16(vaccOPQRSTUVx0, vaOPQRSTUVx0, vb0);
66 const float16x8_t va01234567x1 = vld1q_f16(a);
67 const float16x8_t va89ABCDEFx1 = vld1q_f16(a + 8);
68 const float16x8_t vaGHIJKLMNx1 = vld1q_f16(a + 16);
69 const float16x8_t vaOPQRSTUVx1 = vld1q_f16(a + 24);
70 a = (const __fp16*restrict) ((uintptr_t) a + (uintptr_t) diff1);
71 const float16x8_t vb1 = vld1q_dup_f16(w); w += 1;
72 vacc01234567x1 = vfmaq_f16(vacc01234567x1, va01234567x1, vb1);
73 vacc89ABCDEFx1 = vfmaq_f16(vacc89ABCDEFx1, va89ABCDEFx1, vb1);
74 vaccGHIJKLMNx1 = vfmaq_f16(vaccGHIJKLMNx1, vaGHIJKLMNx1, vb1);
75 vaccOPQRSTUVx1 = vfmaq_f16(vaccOPQRSTUVx1, vaOPQRSTUVx1, vb1);
76 }
77 float16x8_t vacc01234567 = vacc01234567x0;
78 float16x8_t vacc89ABCDEF = vacc89ABCDEFx0;
79 float16x8_t vaccGHIJKLMN = vaccGHIJKLMNx0;
80 float16x8_t vaccOPQRSTUV = vaccOPQRSTUVx0;
81 vacc01234567 = vaddq_f16(vacc01234567, vacc01234567x1);
82 vacc89ABCDEF = vaddq_f16(vacc89ABCDEF, vacc89ABCDEFx1);
83 vaccGHIJKLMN = vaddq_f16(vaccGHIJKLMN, vaccGHIJKLMNx1);
84 vaccOPQRSTUV = vaddq_f16(vaccOPQRSTUV, vaccOPQRSTUVx1);
85 if XNN_LIKELY(nnz != 0) {
86 do {
87 const intptr_t diff = *dmap++;
88 const float16x8_t va01234567 = vld1q_f16(a);
89 const float16x8_t va89ABCDEF = vld1q_f16(a + 8);
90 const float16x8_t vaGHIJKLMN = vld1q_f16(a + 16);
91 const float16x8_t vaOPQRSTUV = vld1q_f16(a + 24);
92 a = (const __fp16*restrict) ((uintptr_t) a + (uintptr_t) diff);
93 const float16x8_t vb = vld1q_dup_f16(w); w += 1;
94 vacc01234567 = vfmaq_f16(vacc01234567, va01234567, vb);
95 vacc89ABCDEF = vfmaq_f16(vacc89ABCDEF, va89ABCDEF, vb);
96 vaccGHIJKLMN = vfmaq_f16(vaccGHIJKLMN, vaGHIJKLMN, vb);
97 vaccOPQRSTUV = vfmaq_f16(vaccOPQRSTUV, vaOPQRSTUV, vb);
98 } while (--nnz != 0);
99 }
100 float16x8_t vout01234567 = vmulq_f16(vacc01234567, vscale);
101 float16x8_t vout89ABCDEF = vmulq_f16(vacc89ABCDEF, vscale);
102 float16x8_t voutGHIJKLMN = vmulq_f16(vaccGHIJKLMN, vscale);
103 float16x8_t voutOPQRSTUV = vmulq_f16(vaccOPQRSTUV, vscale);
104 vout01234567 = vminq_f16(vout01234567, vmax);
105 vout89ABCDEF = vminq_f16(vout89ABCDEF, vmax);
106 voutGHIJKLMN = vminq_f16(voutGHIJKLMN, vmax);
107 voutOPQRSTUV = vminq_f16(voutOPQRSTUV, vmax);
108 vout01234567 = vmaxq_f16(vout01234567, vmin);
109 vout89ABCDEF = vmaxq_f16(vout89ABCDEF, vmin);
110 voutGHIJKLMN = vmaxq_f16(voutGHIJKLMN, vmin);
111 voutOPQRSTUV = vmaxq_f16(voutOPQRSTUV, vmin);
112 vst1q_f16(c, vout01234567);
113 vst1q_f16(c + 8, vout89ABCDEF);
114 vst1q_f16(c + 16, voutGHIJKLMN);
115 vst1q_f16(c + 24, voutOPQRSTUV);
116 c += m;
117 } while (--j != 0);
118 c -= m * n;
119 c += 32;
120 a += 32;
121 i -= 32;
122 }
123 if XNN_UNLIKELY(i != 0) {
124 if (i & 16) {
125 const __fp16*restrict w = weights;
126 const int32_t* dmap = widx_dmap;
127 const uint32_t* nnzmap = nidx_nnzmap;
128 size_t j = n;
129 do {
130 uint32_t nnz = *nnzmap++;
131 float16x8_t vacc01234567 = vld1q_dup_f16(w); w += 1;
132 float16x8_t vacc89ABCDEF = vacc01234567;
133 if XNN_LIKELY(nnz != 0) {
134 do {
135 const intptr_t diff = *dmap++;
136 const float16x8_t va01234567 = vld1q_f16(a);
137 const float16x8_t va89ABCDEF = vld1q_f16(a + 8);
138 a = (const __fp16*restrict) ((uintptr_t) a + (uintptr_t) diff);
139 const float16x8_t vb = vld1q_dup_f16(w); w += 1;
140 vacc01234567 = vfmaq_f16(vacc01234567, va01234567, vb);
141 vacc89ABCDEF = vfmaq_f16(vacc89ABCDEF, va89ABCDEF, vb);
142 } while (--nnz != 0);
143 }
144 float16x8_t vout01234567 = vminq_f16(vacc01234567, vmax);
145 float16x8_t vout89ABCDEF = vminq_f16(vacc89ABCDEF, vmax);
146 vout01234567 = vmaxq_f16(vout01234567, vmin);
147 vout89ABCDEF = vmaxq_f16(vout89ABCDEF, vmin);
148 vst1q_f16(c, vout01234567);
149 vst1q_f16(c + 8, vout89ABCDEF);
150 c += m;
151 } while (--j != 0);
152 c -= m * n;
153 c += 16;
154 a += 16;
155 }
156 if (i & 8) {
157 const __fp16*restrict w = weights;
158 const int32_t* dmap = widx_dmap;
159 const uint32_t* nnzmap = nidx_nnzmap;
160 size_t j = n;
161 do {
162 uint32_t nnz = *nnzmap++;
163 float16x8_t vacc01234567 = vld1q_dup_f16(w); w += 1;
164 if XNN_LIKELY(nnz != 0) {
165 do {
166 const intptr_t diff = *dmap++;
167 const float16x8_t va01234567 = vld1q_f16(a);
168 a = (const __fp16*restrict) ((uintptr_t) a + (uintptr_t) diff);
169 const float16x8_t vb = vld1q_dup_f16(w); w += 1;
170 vacc01234567 = vfmaq_f16(vacc01234567, va01234567, vb);
171 } while (--nnz != 0);
172 }
173 float16x8_t vout01234567 = vminq_f16(vacc01234567, vmax);
174 vout01234567 = vmaxq_f16(vout01234567, vmin);
175 vst1q_f16(c, vout01234567);
176 c += m;
177 } while (--j != 0);
178 c -= m * n;
179 c += 8;
180 a += 8;
181 }
182 if (i & 4) {
183 const __fp16*restrict w = weights;
184 const int32_t* dmap = widx_dmap;
185 const uint32_t* nnzmap = nidx_nnzmap;
186 size_t j = n;
187 do {
188 uint32_t nnz = *nnzmap++;
189 float16x4_t vacc0123 = vld1_dup_f16(w); w += 1;
190 if XNN_LIKELY(nnz != 0) {
191 do {
192 const intptr_t diff = *dmap++;
193 const float16x4_t va0123 = vld1_f16(a);
194 a = (const __fp16*restrict) ((uintptr_t) a + (uintptr_t) diff);
195 const float16x4_t vb = vld1_dup_f16(w); w += 1;
196 vacc0123 = vfma_f16(vacc0123, va0123, vb);
197 } while (--nnz != 0);
198 }
199 float16x4_t vout0123 = vmin_f16(vacc0123, vget_low_f16(vmax));
200 vout0123 = vmax_f16(vout0123, vget_low_f16(vmin));
201 vst1_f16(c, vout0123);
202 c += m;
203 } while (--j != 0);
204 c -= m * n;
205 c += 4;
206 a += 4;
207 }
208 if (i & 2) {
209 const __fp16*restrict w = weights;
210 const int32_t* dmap = widx_dmap;
211 const uint32_t* nnzmap = nidx_nnzmap;
212 size_t j = n;
213 do {
214 uint32_t nnz = *nnzmap++;
215 float16x4_t vacc01 = vld1_dup_f16(w); w += 1;
216 if XNN_LIKELY(nnz != 0) {
217 do {
218 const intptr_t diff = *dmap++;
219 const float16x4_t va01 = vreinterpret_f32_f16(vld1_dup_f32(__builtin_assume_aligned(a, 1)));
220 a = (const __fp16*restrict) ((uintptr_t) a + (uintptr_t) diff);
221 const float16x4_t vb = vld1_dup_f16(w); w += 1;
222 vacc01 = vfma_f16(vacc01, va01, vb);
223 } while (--nnz != 0);
224 }
225 float16x4_t vout01 = vmin_f16(vacc01, vget_low_f16(vmax));
226 vout01 = vmax_f16(vout01, vget_low_f16(vmin));
227 vst1_lane_f32(__builtin_assume_aligned(c, 1), vreinterpret_f16_f32(vout01), 0);
228 c += m;
229 } while (--j != 0);
230 c -= m * n;
231 c += 2;
232 a += 2;
233 }
234 if (i & 1) {
235 const __fp16*restrict w = weights;
236 const int32_t* dmap = widx_dmap;
237 const uint32_t* nnzmap = nidx_nnzmap;
238 size_t j = n;
239 do {
240 uint32_t nnz = *nnzmap++;
241 float16x4_t vacc0 = vld1_dup_f16(w); w += 1;
242 if XNN_LIKELY(nnz != 0) {
243 do {
244 const intptr_t diff = *dmap++;
245 const float16x4_t va0 = vld1_dup_f16(a);
246 a = (const __fp16*restrict) ((uintptr_t) a + (uintptr_t) diff);
247 const float16x4_t vb = vld1_dup_f16(w); w += 1;
248 vacc0 = vfma_f16(vacc0, va0, vb);
249 } while (--nnz != 0);
250 }
251 float16x4_t vout0 = vmin_f16(vacc0, vget_low_f16(vmax));
252 vout0 = vmax_f16(vout0, vget_low_f16(vmin));
253 vst1_lane_f16(c, vout0, 0);
254 c += m;
255 } while (--j != 0);
256 c -= m * n;
257 c += 1;
258 a += 1;
259 }
260 }
261 }
262