1 // Auto-generated file. Do not edit!
2 // Template: src/f32-spmm/scalar-pipelined.c.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2019 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10 #include <assert.h>
11
12 #include <xnnpack/math.h>
13 #include <xnnpack/spmm.h>
14
15
xnn_f32_spmm_minmax_ukernel_8x1__scalar_pipelined(size_t mc,size_t nc,const float * restrict input,const float * restrict weights,const int32_t * restrict widx_dmap,const uint32_t * restrict nidx_nnzmap,float * restrict output,size_t output_stride,const union xnn_f32_minmax_params params[restrict XNN_MIN_ELEMENTS (1)])16 void xnn_f32_spmm_minmax_ukernel_8x1__scalar_pipelined(
17 size_t mc,
18 size_t nc,
19 const float*restrict input,
20 const float*restrict weights,
21 const int32_t*restrict widx_dmap,
22 const uint32_t*restrict nidx_nnzmap,
23 float*restrict output,
24 size_t output_stride,
25 const union xnn_f32_minmax_params params[restrict XNN_MIN_ELEMENTS(1)])
26 {
27 assert(mc != 0);
28 assert(mc % sizeof(float) == 0);
29 assert(nc != 0);
30
31 const float vmin = params->scalar.min;
32 const float vmax = params->scalar.max;
33 size_t output_decrement = output_stride * nc - 8 * sizeof(float);
34 while XNN_LIKELY(mc >= 8 * sizeof(float)) {
35 const float*restrict w = weights;
36 const int32_t* dmap = widx_dmap;
37 const uint32_t* nnzmap = nidx_nnzmap;
38 float vw = *w++;
39 intptr_t diff = *dmap++;
40 float vi0 = input[0];
41 float vi1 = input[1];
42 float vi2 = input[2];
43 float vi3 = input[3];
44 float vi4 = input[4];
45 float vi5 = input[5];
46 float vi6 = input[6];
47 float vi7 = input[7];
48 size_t n = nc;
49 do {
50 uint32_t nnz = *nnzmap++;
51 float vacc0 = vw;
52 float vacc1 = vw;
53 float vacc2 = vw;
54 float vacc3 = vw;
55 float vacc4 = vw;
56 float vacc5 = vw;
57 float vacc6 = vw;
58 float vacc7 = vw;
59 vw = *w++;
60 if XNN_LIKELY(nnz != 0) {
61 do {
62 vacc0 += vi0 * vw;
63 vacc1 += vi1 * vw;
64 vacc2 += vi2 * vw;
65 vacc3 += vi3 * vw;
66 vacc4 += vi4 * vw;
67 vacc5 += vi5 * vw;
68 vacc6 += vi6 * vw;
69 vacc7 += vi7 * vw;
70 input = (const float*restrict) ((uintptr_t) input + (uintptr_t) diff);
71
72 diff = *dmap++;
73 vw = *w++;
74 vi0 = input[0];
75 vi1 = input[1];
76 vi2 = input[2];
77 vi3 = input[3];
78 vi4 = input[4];
79 vi5 = input[5];
80 vi6 = input[6];
81 vi7 = input[7];
82 } while (--nnz != 0);
83 }
84 float vout0 = math_min_f32(vacc0, vmax);
85 float vout1 = math_min_f32(vacc1, vmax);
86 float vout2 = math_min_f32(vacc2, vmax);
87 float vout3 = math_min_f32(vacc3, vmax);
88 float vout4 = math_min_f32(vacc4, vmax);
89 float vout5 = math_min_f32(vacc5, vmax);
90 float vout6 = math_min_f32(vacc6, vmax);
91 float vout7 = math_min_f32(vacc7, vmax);
92 vout0 = math_max_f32(vout0, vmin);
93 vout1 = math_max_f32(vout1, vmin);
94 vout2 = math_max_f32(vout2, vmin);
95 vout3 = math_max_f32(vout3, vmin);
96 vout4 = math_max_f32(vout4, vmin);
97 vout5 = math_max_f32(vout5, vmin);
98 vout6 = math_max_f32(vout6, vmin);
99 vout7 = math_max_f32(vout7, vmin);
100 output[0] = vout0;
101 output[1] = vout1;
102 output[2] = vout2;
103 output[3] = vout3;
104 output[4] = vout4;
105 output[5] = vout5;
106 output[6] = vout6;
107 output[7] = vout7;
108 output = (float*restrict) ((uintptr_t) output + output_stride);
109 } while (--n != 0);
110 output = (float*restrict) ((uintptr_t) output - output_decrement);
111 input += 8;
112 mc -= 8 * sizeof(float);
113 }
114 if XNN_UNLIKELY(mc != 0) {
115 output_decrement += 4 * sizeof(float);
116 if (mc & (4 * sizeof(float))) {
117 const float*restrict w = weights;
118 const int32_t* dmap = widx_dmap;
119 const uint32_t* nnzmap = nidx_nnzmap;
120 float vw = *w++;
121 intptr_t diff = *dmap++;
122 float vi0 = input[0];
123 float vi1 = input[1];
124 float vi2 = input[2];
125 float vi3 = input[3];
126 size_t n = nc;
127 do {
128 uint32_t nnz = *nnzmap++;
129 float vacc0 = vw;
130 float vacc1 = vw;
131 float vacc2 = vw;
132 float vacc3 = vw;
133 vw = *w++;
134 if XNN_LIKELY(nnz != 0) {
135 do {
136 vacc0 += vi0 * vw;
137 vacc1 += vi1 * vw;
138 vacc2 += vi2 * vw;
139 vacc3 += vi3 * vw;
140 input = (const float*restrict) ((uintptr_t) input + (uintptr_t) diff);
141
142 diff = *dmap++;
143 vw = *w++;
144 vi0 = input[0];
145 vi1 = input[1];
146 vi2 = input[2];
147 vi3 = input[3];
148 } while (--nnz != 0);
149 }
150 float vout0 = math_min_f32(vacc0, vmax);
151 float vout1 = math_min_f32(vacc1, vmax);
152 float vout2 = math_min_f32(vacc2, vmax);
153 float vout3 = math_min_f32(vacc3, vmax);
154 vout0 = math_max_f32(vout0, vmin);
155 vout1 = math_max_f32(vout1, vmin);
156 vout2 = math_max_f32(vout2, vmin);
157 vout3 = math_max_f32(vout3, vmin);
158 output[0] = vout0;
159 output[1] = vout1;
160 output[2] = vout2;
161 output[3] = vout3;
162 output = (float*restrict) ((uintptr_t) output + output_stride);
163 } while (--n != 0);
164 output = (float*restrict) ((uintptr_t) output - output_decrement);
165 input += 4;
166 }
167 output_decrement += 2 * sizeof(float);
168 if (mc & (2 * sizeof(float))) {
169 const float*restrict w = weights;
170 const int32_t* dmap = widx_dmap;
171 const uint32_t* nnzmap = nidx_nnzmap;
172 float vw = *w++;
173 intptr_t diff = *dmap++;
174 float vi0 = input[0];
175 float vi1 = input[1];
176 size_t n = nc;
177 do {
178 uint32_t nnz = *nnzmap++;
179 float vacc0 = vw;
180 float vacc1 = vw;
181 vw = *w++;
182 if XNN_LIKELY(nnz != 0) {
183 do {
184 vacc0 += vi0 * vw;
185 vacc1 += vi1 * vw;
186 input = (const float*restrict) ((uintptr_t) input + (uintptr_t) diff);
187
188 diff = *dmap++;
189 vw = *w++;
190 vi0 = input[0];
191 vi1 = input[1];
192 } while (--nnz != 0);
193 }
194 float vout0 = math_min_f32(vacc0, vmax);
195 float vout1 = math_min_f32(vacc1, vmax);
196 vout0 = math_max_f32(vout0, vmin);
197 vout1 = math_max_f32(vout1, vmin);
198 output[0] = vout0;
199 output[1] = vout1;
200 output = (float*restrict) ((uintptr_t) output + output_stride);
201 } while (--n != 0);
202 output = (float*restrict) ((uintptr_t) output - output_decrement);
203 input += 2;
204 }
205 output_decrement += 1 * sizeof(float);
206 if (mc & (1 * sizeof(float))) {
207 const float*restrict w = weights;
208 const int32_t* dmap = widx_dmap;
209 const uint32_t* nnzmap = nidx_nnzmap;
210 float vw = *w++;
211 intptr_t diff = *dmap++;
212 float vi0 = input[0];
213 size_t n = nc;
214 do {
215 uint32_t nnz = *nnzmap++;
216 float vacc0 = vw;
217 vw = *w++;
218 if XNN_LIKELY(nnz != 0) {
219 do {
220 vacc0 += vi0 * vw;
221 input = (const float*restrict) ((uintptr_t) input + (uintptr_t) diff);
222
223 diff = *dmap++;
224 vw = *w++;
225 vi0 = input[0];
226 } while (--nnz != 0);
227 }
228 float vout0 = math_min_f32(vacc0, vmax);
229 vout0 = math_max_f32(vout0, vmin);
230 output[0] = vout0;
231 output = (float*restrict) ((uintptr_t) output + output_stride);
232 } while (--n != 0);
233 output = (float*restrict) ((uintptr_t) output - output_decrement);
234 input += 1;
235 }
236 }
237 }
238