1 // Auto-generated file. Do not edit!
2 // Template: src/f32-spmm/neon.c.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2019 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10 #include <assert.h>
11
12 #include <arm_neon.h>
13
14 #include <xnnpack/spmm.h>
15
16
xnn_f32_spmm_ukernel_16x1__neonfma_unroll2(uint32_t m,uint32_t n,const float * restrict a,const float * restrict weights,const int32_t * restrict widx_dmap,const uint32_t * restrict nidx_nnzmap,float * restrict c,const union xnn_f32_output_params params[restrict static1])17 void xnn_f32_spmm_ukernel_16x1__neonfma_unroll2(
18 uint32_t m,
19 uint32_t n,
20 const float*restrict a,
21 const float*restrict weights,
22 const int32_t*restrict widx_dmap,
23 const uint32_t*restrict nidx_nnzmap,
24 float*restrict c,
25 const union xnn_f32_output_params params[restrict static 1])
26 {
27 assert(m != 0);
28
29 const float32x4_t vmin = vld1q_dup_f32(¶ms->scalar.min);
30 const float32x4_t vmax = vld1q_dup_f32(¶ms->scalar.max);
31 size_t i = m;
32 while XNN_LIKELY(i >= 16) {
33 const float*restrict w = weights;
34 const int32_t* dmap = widx_dmap;
35 const uint32_t* nnzmap = nidx_nnzmap;
36 size_t j = n;
37 do {
38 uint32_t nnz = *nnzmap++;
39 float32x4_t vacc0123x0 = vld1q_dup_f32(w); w += 1;
40 float32x4_t vacc0123x1 = vmovq_n_f32(0.0f);
41 float32x4_t vacc4567x0 = vacc0123x0;
42 float32x4_t vacc4567x1 = vmovq_n_f32(0.0f);
43 float32x4_t vacc89ABx0 = vacc0123x0;
44 float32x4_t vacc89ABx1 = vmovq_n_f32(0.0f);
45 float32x4_t vaccCDEFx0 = vacc0123x0;
46 float32x4_t vaccCDEFx1 = vmovq_n_f32(0.0f);
47 for (; nnz >= 2; nnz -= 2) {
48 const intptr_t diff0 = dmap[0];
49 const intptr_t diff1 = dmap[1];
50 dmap += 2;
51 const float32x4_t va0123x0 = vld1q_f32(a);
52 const float32x4_t va4567x0 = vld1q_f32(a + 4);
53 const float32x4_t va89ABx0 = vld1q_f32(a + 8);
54 const float32x4_t vaCDEFx0 = vld1q_f32(a + 12);
55 __builtin_prefetch(a + 16);
56 a = (const float*restrict) ((uintptr_t) a + (uintptr_t) diff0);
57 const float32x4_t vb0 = vld1q_dup_f32(w); w += 1;
58 vacc0123x0 = vfmaq_f32(vacc0123x0, va0123x0, vb0);
59 vacc4567x0 = vfmaq_f32(vacc4567x0, va4567x0, vb0);
60 vacc89ABx0 = vfmaq_f32(vacc89ABx0, va89ABx0, vb0);
61 vaccCDEFx0 = vfmaq_f32(vaccCDEFx0, vaCDEFx0, vb0);
62 const float32x4_t va0123x1 = vld1q_f32(a);
63 const float32x4_t va4567x1 = vld1q_f32(a + 4);
64 const float32x4_t va89ABx1 = vld1q_f32(a + 8);
65 const float32x4_t vaCDEFx1 = vld1q_f32(a + 12);
66 __builtin_prefetch(a + 16);
67 a = (const float*restrict) ((uintptr_t) a + (uintptr_t) diff1);
68 const float32x4_t vb1 = vld1q_dup_f32(w); w += 1;
69 vacc0123x1 = vfmaq_f32(vacc0123x1, va0123x1, vb1);
70 vacc4567x1 = vfmaq_f32(vacc4567x1, va4567x1, vb1);
71 vacc89ABx1 = vfmaq_f32(vacc89ABx1, va89ABx1, vb1);
72 vaccCDEFx1 = vfmaq_f32(vaccCDEFx1, vaCDEFx1, vb1);
73 }
74 float32x4_t vacc0123 = vacc0123x0;
75 float32x4_t vacc4567 = vacc4567x0;
76 float32x4_t vacc89AB = vacc89ABx0;
77 float32x4_t vaccCDEF = vaccCDEFx0;
78 vacc0123 = vaddq_f32(vacc0123, vacc0123x1);
79 vacc4567 = vaddq_f32(vacc4567, vacc4567x1);
80 vacc89AB = vaddq_f32(vacc89AB, vacc89ABx1);
81 vaccCDEF = vaddq_f32(vaccCDEF, vaccCDEFx1);
82 if XNN_LIKELY(nnz != 0) {
83 do {
84 const intptr_t diff = *dmap++;
85 const float32x4_t va0123 = vld1q_f32(a);
86 const float32x4_t va4567 = vld1q_f32(a + 4);
87 const float32x4_t va89AB = vld1q_f32(a + 8);
88 const float32x4_t vaCDEF = vld1q_f32(a + 12);
89 __builtin_prefetch(a + 16);
90 a = (const float*restrict) ((uintptr_t) a + (uintptr_t) diff);
91 const float32x4_t vb = vld1q_dup_f32(w); w += 1;
92 vacc0123 = vfmaq_f32(vacc0123, va0123, vb);
93 vacc4567 = vfmaq_f32(vacc4567, va4567, vb);
94 vacc89AB = vfmaq_f32(vacc89AB, va89AB, vb);
95 vaccCDEF = vfmaq_f32(vaccCDEF, vaCDEF, vb);
96 } while (--nnz != 0);
97 }
98 float32x4_t vout0123 = vminq_f32(vacc0123, vmax);
99 float32x4_t vout4567 = vminq_f32(vacc4567, vmax);
100 float32x4_t vout89AB = vminq_f32(vacc89AB, vmax);
101 float32x4_t voutCDEF = vminq_f32(vaccCDEF, vmax);
102 vout0123 = vmaxq_f32(vout0123, vmin);
103 vout4567 = vmaxq_f32(vout4567, vmin);
104 vout89AB = vmaxq_f32(vout89AB, vmin);
105 voutCDEF = vmaxq_f32(voutCDEF, vmin);
106 vst1q_f32(c, vout0123);
107 vst1q_f32(c + 4, vout4567);
108 vst1q_f32(c + 8, vout89AB);
109 vst1q_f32(c + 12, voutCDEF);
110 c += m;
111 } while (--j != 0);
112 c -= m * n;
113 c += 16;
114 a += 16;
115 i -= 16;
116 }
117 if XNN_UNLIKELY(i != 0) {
118 if (i & 8) {
119 const float*restrict w = weights;
120 const int32_t* dmap = widx_dmap;
121 const uint32_t* nnzmap = nidx_nnzmap;
122 size_t j = n;
123 do {
124 uint32_t nnz = *nnzmap++;
125 float32x4_t vacc0123 = vld1q_dup_f32(w); w += 1;
126 float32x4_t vacc4567 = vacc0123;
127 if XNN_LIKELY(nnz != 0) {
128 do {
129 const intptr_t diff = *dmap++;
130 const float32x4_t va0123 = vld1q_f32(a);
131 const float32x4_t va4567 = vld1q_f32(a + 4);
132 a = (const float*restrict) ((uintptr_t) a + (uintptr_t) diff);
133 const float32x4_t vb = vld1q_dup_f32(w); w += 1;
134 vacc0123 = vfmaq_f32(vacc0123, va0123, vb);
135 vacc4567 = vfmaq_f32(vacc4567, va4567, vb);
136 } while (--nnz != 0);
137 }
138 float32x4_t vout0123 = vminq_f32(vacc0123, vmax);
139 float32x4_t vout4567 = vminq_f32(vacc4567, vmax);
140 vout0123 = vmaxq_f32(vout0123, vmin);
141 vout4567 = vmaxq_f32(vout4567, vmin);
142 vst1q_f32(c, vout0123);
143 vst1q_f32(c + 4, vout4567);
144 c += m;
145 } while (--j != 0);
146 c -= m * n;
147 c += 8;
148 a += 8;
149 }
150 if (i & 4) {
151 const float*restrict w = weights;
152 const int32_t* dmap = widx_dmap;
153 const uint32_t* nnzmap = nidx_nnzmap;
154 size_t j = n;
155 do {
156 uint32_t nnz = *nnzmap++;
157 float32x4_t vacc0123 = vld1q_dup_f32(w); w += 1;
158 if XNN_LIKELY(nnz != 0) {
159 do {
160 const intptr_t diff = *dmap++;
161 const float32x4_t va0123 = vld1q_f32(a);
162 a = (const float*restrict) ((uintptr_t) a + (uintptr_t) diff);
163 const float32x4_t vb = vld1q_dup_f32(w); w += 1;
164 vacc0123 = vfmaq_f32(vacc0123, va0123, vb);
165 } while (--nnz != 0);
166 }
167 float32x4_t vout0123 = vminq_f32(vacc0123, vmax);
168 vout0123 = vmaxq_f32(vout0123, vmin);
169 vst1q_f32(c, vout0123);
170 c += m;
171 } while (--j != 0);
172 c -= m * n;
173 c += 4;
174 a += 4;
175 }
176 if (i & 2) {
177 const float*restrict w = weights;
178 const int32_t* dmap = widx_dmap;
179 const uint32_t* nnzmap = nidx_nnzmap;
180 size_t j = n;
181 do {
182 uint32_t nnz = *nnzmap++;
183 float32x2_t vacc01 = vld1_dup_f32(w); w += 1;
184 if XNN_LIKELY(nnz != 0) {
185 do {
186 const intptr_t diff = *dmap++;
187 const float32x2_t va01 = vld1_f32(a);
188 a = (const float*restrict) ((uintptr_t) a + (uintptr_t) diff);
189 const float32x2_t vb = vld1_dup_f32(w); w += 1;
190 vacc01 = vfma_f32(vacc01, va01, vb);
191 } while (--nnz != 0);
192 }
193 float32x2_t vout01 = vmin_f32(vacc01, vget_low_f32(vmax));
194 vout01 = vmax_f32(vout01, vget_low_f32(vmin));
195 vst1_f32(c, vout01);
196 c += m;
197 } while (--j != 0);
198 c -= m * n;
199 c += 2;
200 a += 2;
201 }
202 if (i & 1) {
203 const float*restrict w = weights;
204 const int32_t* dmap = widx_dmap;
205 const uint32_t* nnzmap = nidx_nnzmap;
206 size_t j = n;
207 do {
208 uint32_t nnz = *nnzmap++;
209 float32x2_t vacc0 = vld1_dup_f32(w); w += 1;
210 if XNN_LIKELY(nnz != 0) {
211 do {
212 const intptr_t diff = *dmap++;
213 const float32x2_t va0 = vld1_dup_f32(a);
214 a = (const float*restrict) ((uintptr_t) a + (uintptr_t) diff);
215 const float32x2_t vb = vld1_dup_f32(w); w += 1;
216 vacc0 = vfma_f32(vacc0, va0, vb);
217 } while (--nnz != 0);
218 }
219 float32x2_t vout0 = vmin_f32(vacc0, vget_low_f32(vmax));
220 vout0 = vmax_f32(vout0, vget_low_f32(vmin));
221 vst1_lane_f32(c, vout0, 0);
222 c += m;
223 } while (--j != 0);
224 c -= m * n;
225 c += 1;
226 a += 1;
227 }
228 }
229 }
230