1 // Auto-generated file. Do not edit!
2 // Template: src/f32-spmm/neon-pipelined.c.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2019 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10 #include <assert.h>
11
12 #include <arm_neon.h>
13
14 #include <xnnpack/spmm.h>
15
16
xnn_f32_spmm_ukernel_16x1__neonfma_pipelined(uint32_t m,uint32_t n,const float * restrict a,const float * restrict weights,const int32_t * restrict widx_dmap,const uint32_t * restrict nidx_nnzmap,float * restrict c,const union xnn_f32_output_params params[restrict static1])17 void xnn_f32_spmm_ukernel_16x1__neonfma_pipelined(
18 uint32_t m,
19 uint32_t n,
20 const float*restrict a,
21 const float*restrict weights,
22 const int32_t*restrict widx_dmap,
23 const uint32_t*restrict nidx_nnzmap,
24 float*restrict c,
25 const union xnn_f32_output_params params[restrict static 1])
26 {
27 assert(m != 0);
28
29 const float32x4_t vmin = vld1q_dup_f32(¶ms->scalar.min);
30 const float32x4_t vmax = vld1q_dup_f32(¶ms->scalar.max);
31 size_t i = m;
32 while XNN_LIKELY(i >= 16) {
33 const float*restrict w = weights;
34 const int32_t* dmap = widx_dmap;
35 const uint32_t* nnzmap = nidx_nnzmap;
36 float32x4_t vw = vld1q_dup_f32(w); w += 1;
37 intptr_t diff = *dmap++;
38 float32x4_t va0123 = vld1q_f32(a);
39 float32x4_t va4567 = vld1q_f32(a + 4);
40 float32x4_t va89AB = vld1q_f32(a + 8);
41 float32x4_t vaCDEF = vld1q_f32(a + 12);
42 __builtin_prefetch(a + 16);
43 size_t j = n;
44 do {
45 uint32_t nnz = *nnzmap++;
46 float32x4_t vacc0123 = vw;
47 float32x4_t vacc4567 = vw;
48 float32x4_t vacc89AB = vw;
49 float32x4_t vaccCDEF = vw;
50 vw = vld1q_dup_f32(w); w += 1;
51 if XNN_LIKELY(nnz != 0) {
52 do {
53 vacc0123 = vfmaq_f32(vacc0123, va0123, vw);
54 vacc4567 = vfmaq_f32(vacc4567, va4567, vw);
55 vacc89AB = vfmaq_f32(vacc89AB, va89AB, vw);
56 vaccCDEF = vfmaq_f32(vaccCDEF, vaCDEF, vw);
57 a = (const float*restrict) ((uintptr_t) a + (uintptr_t) diff);
58
59 diff = *dmap++;
60 vw = vld1q_dup_f32(w); w += 1;
61 va0123 = vld1q_f32(a);
62 va4567 = vld1q_f32(a + 4);
63 va89AB = vld1q_f32(a + 8);
64 vaCDEF = vld1q_f32(a + 12);
65 __builtin_prefetch(a + 16);
66 } while (--nnz != 0);
67 }
68 float32x4_t vout0123 = vminq_f32(vacc0123, vmax);
69 float32x4_t vout4567 = vminq_f32(vacc4567, vmax);
70 float32x4_t vout89AB = vminq_f32(vacc89AB, vmax);
71 float32x4_t voutCDEF = vminq_f32(vaccCDEF, vmax);
72 vout0123 = vmaxq_f32(vout0123, vmin);
73 vout4567 = vmaxq_f32(vout4567, vmin);
74 vout89AB = vmaxq_f32(vout89AB, vmin);
75 voutCDEF = vmaxq_f32(voutCDEF, vmin);
76 vst1q_f32(c, vout0123);
77 vst1q_f32(c + 4, vout4567);
78 vst1q_f32(c + 8, vout89AB);
79 vst1q_f32(c + 12, voutCDEF);
80 c += m;
81 } while (--j != 0);
82 c -= m * n;
83 c += 16;
84 a += 16;
85 i -= 16;
86 }
87 if XNN_UNLIKELY(i != 0) {
88 if (i & 8) {
89 const float*restrict w = weights;
90 const int32_t* dmap = widx_dmap;
91 const uint32_t* nnzmap = nidx_nnzmap;
92 size_t j = n;
93 do {
94 uint32_t nnz = *nnzmap++;
95 float32x4_t vacc0123 = vld1q_dup_f32(w); w += 1;
96 float32x4_t vacc4567 = vacc0123;
97 if XNN_LIKELY(nnz != 0) {
98 do {
99 const intptr_t diff = *dmap++;
100 const float32x4_t va0123 = vld1q_f32(a);
101 const float32x4_t va4567 = vld1q_f32(a + 4);
102 a = (const float*restrict) ((uintptr_t) a + (uintptr_t) diff);
103 const float32x4_t vb = vld1q_dup_f32(w); w += 1;
104 vacc0123 = vfmaq_f32(vacc0123, va0123, vb);
105 vacc4567 = vfmaq_f32(vacc4567, va4567, vb);
106 } while (--nnz != 0);
107 }
108 float32x4_t vout0123 = vminq_f32(vacc0123, vmax);
109 float32x4_t vout4567 = vminq_f32(vacc4567, vmax);
110 vout0123 = vmaxq_f32(vout0123, vmin);
111 vout4567 = vmaxq_f32(vout4567, vmin);
112 vst1q_f32(c, vout0123);
113 vst1q_f32(c + 4, vout4567);
114 c += m;
115 } while (--j != 0);
116 c -= m * n;
117 c += 8;
118 a += 8;
119 }
120 if (i & 4) {
121 const float*restrict w = weights;
122 const int32_t* dmap = widx_dmap;
123 const uint32_t* nnzmap = nidx_nnzmap;
124 size_t j = n;
125 do {
126 uint32_t nnz = *nnzmap++;
127 float32x4_t vacc0123 = vld1q_dup_f32(w); w += 1;
128 if XNN_LIKELY(nnz != 0) {
129 do {
130 const intptr_t diff = *dmap++;
131 const float32x4_t va0123 = vld1q_f32(a);
132 a = (const float*restrict) ((uintptr_t) a + (uintptr_t) diff);
133 const float32x4_t vb = vld1q_dup_f32(w); w += 1;
134 vacc0123 = vfmaq_f32(vacc0123, va0123, vb);
135 } while (--nnz != 0);
136 }
137 float32x4_t vout0123 = vminq_f32(vacc0123, vmax);
138 vout0123 = vmaxq_f32(vout0123, vmin);
139 vst1q_f32(c, vout0123);
140 c += m;
141 } while (--j != 0);
142 c -= m * n;
143 c += 4;
144 a += 4;
145 }
146 if (i & 2) {
147 const float*restrict w = weights;
148 const int32_t* dmap = widx_dmap;
149 const uint32_t* nnzmap = nidx_nnzmap;
150 size_t j = n;
151 do {
152 uint32_t nnz = *nnzmap++;
153 float32x2_t vacc01 = vld1_dup_f32(w); w += 1;
154 if XNN_LIKELY(nnz != 0) {
155 do {
156 const intptr_t diff = *dmap++;
157 const float32x2_t va01 = vld1_f32(a);
158 a = (const float*restrict) ((uintptr_t) a + (uintptr_t) diff);
159 const float32x2_t vb = vld1_dup_f32(w); w += 1;
160 vacc01 = vfma_f32(vacc01, va01, vb);
161 } while (--nnz != 0);
162 }
163 float32x2_t vout01 = vmin_f32(vacc01, vget_low_f32(vmax));
164 vout01 = vmax_f32(vout01, vget_low_f32(vmin));
165 vst1_f32(c, vout01);
166 c += m;
167 } while (--j != 0);
168 c -= m * n;
169 c += 2;
170 a += 2;
171 }
172 if (i & 1) {
173 const float*restrict w = weights;
174 const int32_t* dmap = widx_dmap;
175 const uint32_t* nnzmap = nidx_nnzmap;
176 size_t j = n;
177 do {
178 uint32_t nnz = *nnzmap++;
179 float32x2_t vacc0 = vld1_dup_f32(w); w += 1;
180 if XNN_LIKELY(nnz != 0) {
181 do {
182 const intptr_t diff = *dmap++;
183 const float32x2_t va0 = vld1_dup_f32(a);
184 a = (const float*restrict) ((uintptr_t) a + (uintptr_t) diff);
185 const float32x2_t vb = vld1_dup_f32(w); w += 1;
186 vacc0 = vfma_f32(vacc0, va0, vb);
187 } while (--nnz != 0);
188 }
189 float32x2_t vout0 = vmin_f32(vacc0, vget_low_f32(vmax));
190 vout0 = vmax_f32(vout0, vget_low_f32(vmin));
191 vst1_lane_f32(c, vout0, 0);
192 c += m;
193 } while (--j != 0);
194 c -= m * n;
195 c += 1;
196 a += 1;
197 }
198 }
199 }
200