1 // Auto-generated file. Do not edit!
2 // Template: src/f16-spmm/neonfp16arith.c.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2019 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10 #include <assert.h>
11
12 #include <arm_neon.h>
13
14 #include <xnnpack/spmm.h>
15
16
xnn_f16_spmm_minmax_ukernel_24x1__neonfp16arith_x2(size_t mc,size_t nc,const void * restrict input,const void * restrict weights,const int32_t * restrict widx_dmap,const uint32_t * restrict nidx_nnzmap,void * restrict output,size_t output_stride,const struct xnn_f16_scaleminmax_params params[restrict XNN_MIN_ELEMENTS (1)])17 void xnn_f16_spmm_minmax_ukernel_24x1__neonfp16arith_x2(
18 size_t mc,
19 size_t nc,
20 const void*restrict input,
21 const void*restrict weights,
22 const int32_t*restrict widx_dmap,
23 const uint32_t*restrict nidx_nnzmap,
24 void*restrict output,
25 size_t output_stride,
26 const struct xnn_f16_scaleminmax_params params[restrict XNN_MIN_ELEMENTS(1)])
27 {
28 assert(mc != 0);
29 assert(mc % sizeof(__fp16) == 0);
30 assert(nc != 0);
31
32 const __fp16*restrict i = (const __fp16*) input;
33 __fp16*restrict o = (__fp16*) output;
34
35 const float16x8_t vscale = vld1q_dup_f16((const __fp16*) ¶ms->scale);
36 const float16x8_t vmax = vld1q_dup_f16((const __fp16*) ¶ms->max);
37 const float16x8_t vmin = vld1q_dup_f16((const __fp16*) ¶ms->min);
38
39 size_t output_decrement = output_stride * nc - 24 * sizeof(__fp16);
40 while XNN_LIKELY(mc >= 24 * sizeof(__fp16)) {
41 const __fp16*restrict w = (const __fp16*) weights;
42 const int32_t* dmap = widx_dmap;
43 const uint32_t* nnzmap = nidx_nnzmap;
44 size_t n = nc;
45 do {
46 uint32_t nnz = *nnzmap++;
47 float16x8_t vacc01234567x0 = vld1q_dup_f16(w); w += 1;
48 float16x8_t vacc01234567x1 = vmovq_n_f16(0.0f);
49 float16x8_t vacc89ABCDEFx0 = vacc01234567x0;
50 float16x8_t vacc89ABCDEFx1 = vmovq_n_f16(0.0f);
51 float16x8_t vaccGHIJKLMNx0 = vacc01234567x0;
52 float16x8_t vaccGHIJKLMNx1 = vmovq_n_f16(0.0f);
53 for (; nnz >= 2; nnz -= 2) {
54 const intptr_t diff0 = dmap[0];
55 const intptr_t diff1 = dmap[1];
56 dmap += 2;
57 const float16x8_t va01234567x0 = vld1q_f16(i);
58 const float16x8_t va89ABCDEFx0 = vld1q_f16(i + 8);
59 const float16x8_t vaGHIJKLMNx0 = vld1q_f16(i + 16);
60 i = (const __fp16*restrict) ((uintptr_t) i + (uintptr_t) diff0);
61 const float16x8_t vb0 = vld1q_dup_f16(w); w += 1;
62 vacc01234567x0 = vfmaq_f16(vacc01234567x0, va01234567x0, vb0);
63 vacc89ABCDEFx0 = vfmaq_f16(vacc89ABCDEFx0, va89ABCDEFx0, vb0);
64 vaccGHIJKLMNx0 = vfmaq_f16(vaccGHIJKLMNx0, vaGHIJKLMNx0, vb0);
65 const float16x8_t va01234567x1 = vld1q_f16(i);
66 const float16x8_t va89ABCDEFx1 = vld1q_f16(i + 8);
67 const float16x8_t vaGHIJKLMNx1 = vld1q_f16(i + 16);
68 i = (const __fp16*restrict) ((uintptr_t) i + (uintptr_t) diff1);
69 const float16x8_t vb1 = vld1q_dup_f16(w); w += 1;
70 vacc01234567x1 = vfmaq_f16(vacc01234567x1, va01234567x1, vb1);
71 vacc89ABCDEFx1 = vfmaq_f16(vacc89ABCDEFx1, va89ABCDEFx1, vb1);
72 vaccGHIJKLMNx1 = vfmaq_f16(vaccGHIJKLMNx1, vaGHIJKLMNx1, vb1);
73 }
74 float16x8_t vacc01234567 = vacc01234567x0;
75 float16x8_t vacc89ABCDEF = vacc89ABCDEFx0;
76 float16x8_t vaccGHIJKLMN = vaccGHIJKLMNx0;
77 vacc01234567 = vaddq_f16(vacc01234567, vacc01234567x1);
78 vacc89ABCDEF = vaddq_f16(vacc89ABCDEF, vacc89ABCDEFx1);
79 vaccGHIJKLMN = vaddq_f16(vaccGHIJKLMN, vaccGHIJKLMNx1);
80 if XNN_LIKELY(nnz != 0) {
81 do {
82 const intptr_t diff = *dmap++;
83 const float16x8_t va01234567 = vld1q_f16(i);
84 const float16x8_t va89ABCDEF = vld1q_f16(i + 8);
85 const float16x8_t vaGHIJKLMN = vld1q_f16(i + 16);
86 i = (const __fp16*restrict) ((uintptr_t) i + (uintptr_t) diff);
87 const float16x8_t vb = vld1q_dup_f16(w); w += 1;
88 vacc01234567 = vfmaq_f16(vacc01234567, va01234567, vb);
89 vacc89ABCDEF = vfmaq_f16(vacc89ABCDEF, va89ABCDEF, vb);
90 vaccGHIJKLMN = vfmaq_f16(vaccGHIJKLMN, vaGHIJKLMN, vb);
91 } while (--nnz != 0);
92 }
93 float16x8_t vout01234567 = vmulq_f16(vacc01234567, vscale);
94 float16x8_t vout89ABCDEF = vmulq_f16(vacc89ABCDEF, vscale);
95 float16x8_t voutGHIJKLMN = vmulq_f16(vaccGHIJKLMN, vscale);
96 vout01234567 = vminq_f16(vout01234567, vmax);
97 vout89ABCDEF = vminq_f16(vout89ABCDEF, vmax);
98 voutGHIJKLMN = vminq_f16(voutGHIJKLMN, vmax);
99 vout01234567 = vmaxq_f16(vout01234567, vmin);
100 vout89ABCDEF = vmaxq_f16(vout89ABCDEF, vmin);
101 voutGHIJKLMN = vmaxq_f16(voutGHIJKLMN, vmin);
102 vst1q_f16(o, vout01234567);
103 vst1q_f16(o + 8, vout89ABCDEF);
104 vst1q_f16(o + 16, voutGHIJKLMN);
105 o = (__fp16*restrict) ((uintptr_t) o + output_stride);
106 } while (--n != 0);
107 o = (__fp16*restrict) ((uintptr_t) o - output_decrement);
108 i += 24;
109 mc -= 24 * sizeof(__fp16);
110 }
111 if XNN_UNLIKELY(mc != 0) {
112 output_decrement += 8 * sizeof(__fp16);
113 if (mc & (16 * sizeof(__fp16))) {
114 const __fp16*restrict w = (const __fp16*) weights;
115 const int32_t* dmap = widx_dmap;
116 const uint32_t* nnzmap = nidx_nnzmap;
117 size_t n = nc;
118 do {
119 uint32_t nnz = *nnzmap++;
120 float16x8_t vacc01234567 = vld1q_dup_f16(w); w += 1;
121 float16x8_t vacc89ABCDEF = vacc01234567;
122 if XNN_LIKELY(nnz != 0) {
123 do {
124 const intptr_t diff = *dmap++;
125 const float16x8_t va01234567 = vld1q_f16(i);
126 const float16x8_t va89ABCDEF = vld1q_f16(i + 8);
127 i = (const __fp16*restrict) ((uintptr_t) i + (uintptr_t) diff);
128 const float16x8_t vb = vld1q_dup_f16(w); w += 1;
129 vacc01234567 = vfmaq_f16(vacc01234567, va01234567, vb);
130 vacc89ABCDEF = vfmaq_f16(vacc89ABCDEF, va89ABCDEF, vb);
131 } while (--nnz != 0);
132 }
133 float16x8_t vout01234567 = vminq_f16(vacc01234567, vmax);
134 float16x8_t vout89ABCDEF = vminq_f16(vacc89ABCDEF, vmax);
135 vout01234567 = vmaxq_f16(vout01234567, vmin);
136 vout89ABCDEF = vmaxq_f16(vout89ABCDEF, vmin);
137 vst1q_f16(o, vout01234567);
138 vst1q_f16(o + 8, vout89ABCDEF);
139 o = (__fp16*restrict) ((uintptr_t) o + output_stride);
140 } while (--n != 0);
141 o = (__fp16*restrict) ((uintptr_t) o - output_decrement);
142 i += 16;
143 }
144 output_decrement += 8 * sizeof(__fp16);
145 if (mc & (8 * sizeof(__fp16))) {
146 const __fp16*restrict w = (const __fp16*) weights;
147 const int32_t* dmap = widx_dmap;
148 const uint32_t* nnzmap = nidx_nnzmap;
149 size_t n = nc;
150 do {
151 uint32_t nnz = *nnzmap++;
152 float16x8_t vacc01234567 = vld1q_dup_f16(w); w += 1;
153 if XNN_LIKELY(nnz != 0) {
154 do {
155 const intptr_t diff = *dmap++;
156 const float16x8_t va01234567 = vld1q_f16(i);
157 i = (const __fp16*restrict) ((uintptr_t) i + (uintptr_t) diff);
158 const float16x8_t vb = vld1q_dup_f16(w); w += 1;
159 vacc01234567 = vfmaq_f16(vacc01234567, va01234567, vb);
160 } while (--nnz != 0);
161 }
162 float16x8_t vout01234567 = vminq_f16(vacc01234567, vmax);
163 vout01234567 = vmaxq_f16(vout01234567, vmin);
164 vst1q_f16(o, vout01234567);
165 o = (__fp16*restrict) ((uintptr_t) o + output_stride);
166 } while (--n != 0);
167 o = (__fp16*restrict) ((uintptr_t) o - output_decrement);
168 i += 8;
169 }
170 output_decrement += 4 * sizeof(__fp16);
171 if (mc & (4 * sizeof(__fp16))) {
172 const __fp16*restrict w = (const __fp16*) weights;
173 const int32_t* dmap = widx_dmap;
174 const uint32_t* nnzmap = nidx_nnzmap;
175 size_t n = nc;
176 do {
177 uint32_t nnz = *nnzmap++;
178 float16x4_t vacc0123 = vld1_dup_f16(w); w += 1;
179 if XNN_LIKELY(nnz != 0) {
180 do {
181 const intptr_t diff = *dmap++;
182 const float16x4_t va0123 = vld1_f16(i);
183 i = (const __fp16*restrict) ((uintptr_t) i + (uintptr_t) diff);
184 const float16x4_t vb = vld1_dup_f16(w); w += 1;
185 vacc0123 = vfma_f16(vacc0123, va0123, vb);
186 } while (--nnz != 0);
187 }
188 float16x4_t vout0123 = vmin_f16(vacc0123, vget_low_f16(vmax));
189 vout0123 = vmax_f16(vout0123, vget_low_f16(vmin));
190 vst1_f16(o, vout0123);
191 o = (__fp16*restrict) ((uintptr_t) o + output_stride);
192 } while (--n != 0);
193 o = (__fp16*restrict) ((uintptr_t) o - output_decrement);
194 i += 4;
195 }
196 output_decrement += 2 * sizeof(__fp16);
197 if (mc & (2 * sizeof(__fp16))) {
198 const __fp16*restrict w = (const __fp16*) weights;
199 const int32_t* dmap = widx_dmap;
200 const uint32_t* nnzmap = nidx_nnzmap;
201 size_t n = nc;
202 do {
203 uint32_t nnz = *nnzmap++;
204 float16x4_t vacc01 = vld1_dup_f16(w); w += 1;
205 if XNN_LIKELY(nnz != 0) {
206 do {
207 const intptr_t diff = *dmap++;
208 const float16x4_t va01 = vreinterpret_f16_f32(vld1_dup_f32(__builtin_assume_aligned(i, 1)));
209 i = (const __fp16*restrict) ((uintptr_t) i + (uintptr_t) diff);
210 const float16x4_t vb = vld1_dup_f16(w); w += 1;
211 vacc01 = vfma_f16(vacc01, va01, vb);
212 } while (--nnz != 0);
213 }
214 float16x4_t vout01 = vmin_f16(vacc01, vget_low_f16(vmax));
215 vout01 = vmax_f16(vout01, vget_low_f16(vmin));
216 vst1_lane_f32(__builtin_assume_aligned(o, 1), vreinterpret_f32_f16(vout01), 0);
217 o = (__fp16*restrict) ((uintptr_t) o + output_stride);
218 } while (--n != 0);
219 o = (__fp16*restrict) ((uintptr_t) o - output_decrement);
220 i += 2;
221 }
222 output_decrement += 1 * sizeof(__fp16);
223 if (mc & (1 * sizeof(__fp16))) {
224 const __fp16*restrict w = (const __fp16*) weights;
225 const int32_t* dmap = widx_dmap;
226 const uint32_t* nnzmap = nidx_nnzmap;
227 size_t n = nc;
228 do {
229 uint32_t nnz = *nnzmap++;
230 float16x4_t vacc0 = vld1_dup_f16(w); w += 1;
231 if XNN_LIKELY(nnz != 0) {
232 do {
233 const intptr_t diff = *dmap++;
234 const float16x4_t va0 = vld1_dup_f16(i);
235 i = (const __fp16*restrict) ((uintptr_t) i + (uintptr_t) diff);
236 const float16x4_t vb = vld1_dup_f16(w); w += 1;
237 vacc0 = vfma_f16(vacc0, va0, vb);
238 } while (--nnz != 0);
239 }
240 float16x4_t vout0 = vmin_f16(vacc0, vget_low_f16(vmax));
241 vout0 = vmax_f16(vout0, vget_low_f16(vmin));
242 vst1_lane_f16(o, vout0, 0);
243 o = (__fp16*restrict) ((uintptr_t) o + output_stride);
244 } while (--n != 0);
245 o = (__fp16*restrict) ((uintptr_t) o - output_decrement);
246 i += 1;
247 }
248 }
249 }
250