1 // Auto-generated file. Do not edit!
2 // Template: src/qs8-dwconv/unipass-scalar.c.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2021 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10 #include <assert.h>
11
12 #include <fp16.h>
13
14 #include <xnnpack/dwconv.h>
15 #include <xnnpack/math.h>
16
17
xnn_qs8_dwconv_minmax_fp32_ukernel_up4x9__scalar_fmagic(size_t channels,size_t output_width,const int8_t ** input,const void * weights,int8_t * output,size_t input_stride,size_t output_increment,size_t input_offset,const int8_t * zero,const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS (1)])18 void xnn_qs8_dwconv_minmax_fp32_ukernel_up4x9__scalar_fmagic(
19 size_t channels,
20 size_t output_width,
21 const int8_t** input,
22 const void* weights,
23 int8_t* output,
24 size_t input_stride,
25 size_t output_increment,
26 size_t input_offset,
27 const int8_t* zero,
28 const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)])
29 {
30 assert(channels != 0);
31 assert(output_width != 0);
32
33 const float vscale = params->fp32_scalar_fmagic.scale;
34 const float voutput_min_less_zero_point = params->fp32_scalar_fmagic.output_min_less_zero_point;
35 const float voutput_max_less_zero_point = params->fp32_scalar_fmagic.output_max_less_zero_point;
36 const float vmagic_bias = params->fp32_scalar_fmagic.magic_bias;
37 const int32_t vmagic_bias_less_output_zero_point = params->fp32_scalar_fmagic.magic_bias_less_output_zero_point;
38 do {
39 const int8_t* i0 = input[0];
40 assert(i0 != NULL);
41 if XNN_UNPREDICTABLE(i0 != zero) {
42 i0 = (const int8_t*) ((uintptr_t) i0 + input_offset);
43 }
44 const int8_t* i1 = input[1];
45 assert(i1 != NULL);
46 if XNN_UNPREDICTABLE(i1 != zero) {
47 i1 = (const int8_t*) ((uintptr_t) i1 + input_offset);
48 }
49 const int8_t* i2 = input[2];
50 assert(i2 != NULL);
51 if XNN_UNPREDICTABLE(i2 != zero) {
52 i2 = (const int8_t*) ((uintptr_t) i2 + input_offset);
53 }
54 const int8_t* i3 = input[3];
55 assert(i3 != NULL);
56 if XNN_UNPREDICTABLE(i3 != zero) {
57 i3 = (const int8_t*) ((uintptr_t) i3 + input_offset);
58 }
59 const int8_t* i4 = input[4];
60 assert(i4 != NULL);
61 if XNN_UNPREDICTABLE(i4 != zero) {
62 i4 = (const int8_t*) ((uintptr_t) i4 + input_offset);
63 }
64 const int8_t* i5 = input[5];
65 assert(i5 != NULL);
66 if XNN_UNPREDICTABLE(i5 != zero) {
67 i5 = (const int8_t*) ((uintptr_t) i5 + input_offset);
68 }
69 const int8_t* i6 = input[6];
70 assert(i6 != NULL);
71 if XNN_UNPREDICTABLE(i6 != zero) {
72 i6 = (const int8_t*) ((uintptr_t) i6 + input_offset);
73 }
74 const int8_t* i7 = input[7];
75 assert(i7 != NULL);
76 if XNN_UNPREDICTABLE(i7 != zero) {
77 i7 = (const int8_t*) ((uintptr_t) i7 + input_offset);
78 }
79 const int8_t* i8 = input[8];
80 assert(i8 != NULL);
81 if XNN_UNPREDICTABLE(i8 != zero) {
82 i8 = (const int8_t*) ((uintptr_t) i8 + input_offset);
83 }
84 input = (const int8_t**) ((uintptr_t) input + input_stride);
85
86 size_t c = channels;
87 const void* w = weights;
88 for (; c >= 4; c -= 4) {
89 int32_t vacc0 = ((const int32_t*) w)[0];
90 int32_t vacc1 = ((const int32_t*) w)[1];
91 int32_t vacc2 = ((const int32_t*) w)[2];
92 int32_t vacc3 = ((const int32_t*) w)[3];
93
94
95 const int32_t vi0x0 = (int32_t) i0[0];
96 const int32_t vi0x1 = (int32_t) i0[1];
97 const int32_t vi0x2 = (int32_t) i0[2];
98 const int32_t vi0x3 = (int32_t) i0[3];
99 i0 += 4;
100
101 const int32_t vk0x0 = (int32_t) ((const int8_t*) ((uintptr_t) w + 4 * sizeof(int32_t)))[0];
102 const int32_t vk0x1 = (int32_t) ((const int8_t*) ((uintptr_t) w + 4 * sizeof(int32_t)))[1];
103 const int32_t vk0x2 = (int32_t) ((const int8_t*) ((uintptr_t) w + 4 * sizeof(int32_t)))[2];
104 const int32_t vk0x3 = (int32_t) ((const int8_t*) ((uintptr_t) w + 4 * sizeof(int32_t)))[3];
105
106 vacc0 += vi0x0 * vk0x0;
107 vacc1 += vi0x1 * vk0x1;
108 vacc2 += vi0x2 * vk0x2;
109 vacc3 += vi0x3 * vk0x3;
110
111 const int32_t vi1x0 = (int32_t) i1[0];
112 const int32_t vi1x1 = (int32_t) i1[1];
113 const int32_t vi1x2 = (int32_t) i1[2];
114 const int32_t vi1x3 = (int32_t) i1[3];
115 i1 += 4;
116
117 const int32_t vk1x0 = (int32_t) ((const int8_t*) ((uintptr_t) w + 4 * sizeof(int32_t)))[4];
118 const int32_t vk1x1 = (int32_t) ((const int8_t*) ((uintptr_t) w + 4 * sizeof(int32_t)))[5];
119 const int32_t vk1x2 = (int32_t) ((const int8_t*) ((uintptr_t) w + 4 * sizeof(int32_t)))[6];
120 const int32_t vk1x3 = (int32_t) ((const int8_t*) ((uintptr_t) w + 4 * sizeof(int32_t)))[7];
121
122 vacc0 += vi1x0 * vk1x0;
123 vacc1 += vi1x1 * vk1x1;
124 vacc2 += vi1x2 * vk1x2;
125 vacc3 += vi1x3 * vk1x3;
126
127 const int32_t vi2x0 = (int32_t) i2[0];
128 const int32_t vi2x1 = (int32_t) i2[1];
129 const int32_t vi2x2 = (int32_t) i2[2];
130 const int32_t vi2x3 = (int32_t) i2[3];
131 i2 += 4;
132
133 const int32_t vk2x0 = (int32_t) ((const int8_t*) ((uintptr_t) w + 4 * sizeof(int32_t)))[8];
134 const int32_t vk2x1 = (int32_t) ((const int8_t*) ((uintptr_t) w + 4 * sizeof(int32_t)))[9];
135 const int32_t vk2x2 = (int32_t) ((const int8_t*) ((uintptr_t) w + 4 * sizeof(int32_t)))[10];
136 const int32_t vk2x3 = (int32_t) ((const int8_t*) ((uintptr_t) w + 4 * sizeof(int32_t)))[11];
137
138 vacc0 += vi2x0 * vk2x0;
139 vacc1 += vi2x1 * vk2x1;
140 vacc2 += vi2x2 * vk2x2;
141 vacc3 += vi2x3 * vk2x3;
142
143 const int32_t vi3x0 = (int32_t) i3[0];
144 const int32_t vi3x1 = (int32_t) i3[1];
145 const int32_t vi3x2 = (int32_t) i3[2];
146 const int32_t vi3x3 = (int32_t) i3[3];
147 i3 += 4;
148
149 const int32_t vk3x0 = (int32_t) ((const int8_t*) ((uintptr_t) w + 4 * sizeof(int32_t)))[12];
150 const int32_t vk3x1 = (int32_t) ((const int8_t*) ((uintptr_t) w + 4 * sizeof(int32_t)))[13];
151 const int32_t vk3x2 = (int32_t) ((const int8_t*) ((uintptr_t) w + 4 * sizeof(int32_t)))[14];
152 const int32_t vk3x3 = (int32_t) ((const int8_t*) ((uintptr_t) w + 4 * sizeof(int32_t)))[15];
153
154 vacc0 += vi3x0 * vk3x0;
155 vacc1 += vi3x1 * vk3x1;
156 vacc2 += vi3x2 * vk3x2;
157 vacc3 += vi3x3 * vk3x3;
158
159 const int32_t vi4x0 = (int32_t) i4[0];
160 const int32_t vi4x1 = (int32_t) i4[1];
161 const int32_t vi4x2 = (int32_t) i4[2];
162 const int32_t vi4x3 = (int32_t) i4[3];
163 i4 += 4;
164
165 const int32_t vk4x0 = (int32_t) ((const int8_t*) ((uintptr_t) w + 4 * sizeof(int32_t)))[16];
166 const int32_t vk4x1 = (int32_t) ((const int8_t*) ((uintptr_t) w + 4 * sizeof(int32_t)))[17];
167 const int32_t vk4x2 = (int32_t) ((const int8_t*) ((uintptr_t) w + 4 * sizeof(int32_t)))[18];
168 const int32_t vk4x3 = (int32_t) ((const int8_t*) ((uintptr_t) w + 4 * sizeof(int32_t)))[19];
169
170 vacc0 += vi4x0 * vk4x0;
171 vacc1 += vi4x1 * vk4x1;
172 vacc2 += vi4x2 * vk4x2;
173 vacc3 += vi4x3 * vk4x3;
174
175 const int32_t vi5x0 = (int32_t) i5[0];
176 const int32_t vi5x1 = (int32_t) i5[1];
177 const int32_t vi5x2 = (int32_t) i5[2];
178 const int32_t vi5x3 = (int32_t) i5[3];
179 i5 += 4;
180
181 const int32_t vk5x0 = (int32_t) ((const int8_t*) ((uintptr_t) w + 4 * sizeof(int32_t)))[20];
182 const int32_t vk5x1 = (int32_t) ((const int8_t*) ((uintptr_t) w + 4 * sizeof(int32_t)))[21];
183 const int32_t vk5x2 = (int32_t) ((const int8_t*) ((uintptr_t) w + 4 * sizeof(int32_t)))[22];
184 const int32_t vk5x3 = (int32_t) ((const int8_t*) ((uintptr_t) w + 4 * sizeof(int32_t)))[23];
185
186 vacc0 += vi5x0 * vk5x0;
187 vacc1 += vi5x1 * vk5x1;
188 vacc2 += vi5x2 * vk5x2;
189 vacc3 += vi5x3 * vk5x3;
190
191 const int32_t vi6x0 = (int32_t) i6[0];
192 const int32_t vi6x1 = (int32_t) i6[1];
193 const int32_t vi6x2 = (int32_t) i6[2];
194 const int32_t vi6x3 = (int32_t) i6[3];
195 i6 += 4;
196
197 const int32_t vk6x0 = (int32_t) ((const int8_t*) ((uintptr_t) w + 4 * sizeof(int32_t)))[24];
198 const int32_t vk6x1 = (int32_t) ((const int8_t*) ((uintptr_t) w + 4 * sizeof(int32_t)))[25];
199 const int32_t vk6x2 = (int32_t) ((const int8_t*) ((uintptr_t) w + 4 * sizeof(int32_t)))[26];
200 const int32_t vk6x3 = (int32_t) ((const int8_t*) ((uintptr_t) w + 4 * sizeof(int32_t)))[27];
201
202 vacc0 += vi6x0 * vk6x0;
203 vacc1 += vi6x1 * vk6x1;
204 vacc2 += vi6x2 * vk6x2;
205 vacc3 += vi6x3 * vk6x3;
206
207 const int32_t vi7x0 = (int32_t) i7[0];
208 const int32_t vi7x1 = (int32_t) i7[1];
209 const int32_t vi7x2 = (int32_t) i7[2];
210 const int32_t vi7x3 = (int32_t) i7[3];
211 i7 += 4;
212
213 const int32_t vk7x0 = (int32_t) ((const int8_t*) ((uintptr_t) w + 4 * sizeof(int32_t)))[28];
214 const int32_t vk7x1 = (int32_t) ((const int8_t*) ((uintptr_t) w + 4 * sizeof(int32_t)))[29];
215 const int32_t vk7x2 = (int32_t) ((const int8_t*) ((uintptr_t) w + 4 * sizeof(int32_t)))[30];
216 const int32_t vk7x3 = (int32_t) ((const int8_t*) ((uintptr_t) w + 4 * sizeof(int32_t)))[31];
217
218 vacc0 += vi7x0 * vk7x0;
219 vacc1 += vi7x1 * vk7x1;
220 vacc2 += vi7x2 * vk7x2;
221 vacc3 += vi7x3 * vk7x3;
222
223 const int32_t vi8x0 = (int32_t) i8[0];
224 const int32_t vi8x1 = (int32_t) i8[1];
225 const int32_t vi8x2 = (int32_t) i8[2];
226 const int32_t vi8x3 = (int32_t) i8[3];
227 i8 += 4;
228
229 const int32_t vk8x0 = (int32_t) ((const int8_t*) ((uintptr_t) w + 4 * sizeof(int32_t)))[32];
230 const int32_t vk8x1 = (int32_t) ((const int8_t*) ((uintptr_t) w + 4 * sizeof(int32_t)))[33];
231 const int32_t vk8x2 = (int32_t) ((const int8_t*) ((uintptr_t) w + 4 * sizeof(int32_t)))[34];
232 const int32_t vk8x3 = (int32_t) ((const int8_t*) ((uintptr_t) w + 4 * sizeof(int32_t)))[35];
233
234 vacc0 += vi8x0 * vk8x0;
235 vacc1 += vi8x1 * vk8x1;
236 vacc2 += vi8x2 * vk8x2;
237 vacc3 += vi8x3 * vk8x3;
238
239 w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t) + 36 * sizeof(int8_t));
240
241 float vfpacc0 = (float) vacc0;
242 float vfpacc1 = (float) vacc1;
243 float vfpacc2 = (float) vacc2;
244 float vfpacc3 = (float) vacc3;
245
246 vfpacc0 *= vscale;
247 vfpacc1 *= vscale;
248 vfpacc2 *= vscale;
249 vfpacc3 *= vscale;
250
251 vfpacc0 = math_max_f32(vfpacc0, voutput_min_less_zero_point);
252 vfpacc1 = math_max_f32(vfpacc1, voutput_min_less_zero_point);
253 vfpacc2 = math_max_f32(vfpacc2, voutput_min_less_zero_point);
254 vfpacc3 = math_max_f32(vfpacc3, voutput_min_less_zero_point);
255
256 vfpacc0 = math_min_f32(vfpacc0, voutput_max_less_zero_point);
257 vfpacc1 = math_min_f32(vfpacc1, voutput_max_less_zero_point);
258 vfpacc2 = math_min_f32(vfpacc2, voutput_max_less_zero_point);
259 vfpacc3 = math_min_f32(vfpacc3, voutput_max_less_zero_point);
260
261 vfpacc0 += vmagic_bias;
262 vfpacc1 += vmagic_bias;
263 vfpacc2 += vmagic_bias;
264 vfpacc3 += vmagic_bias;
265
266 int32_t vout0 = (int32_t) fp32_to_bits(vfpacc0) - vmagic_bias_less_output_zero_point;
267 int32_t vout1 = (int32_t) fp32_to_bits(vfpacc1) - vmagic_bias_less_output_zero_point;
268 int32_t vout2 = (int32_t) fp32_to_bits(vfpacc2) - vmagic_bias_less_output_zero_point;
269 int32_t vout3 = (int32_t) fp32_to_bits(vfpacc3) - vmagic_bias_less_output_zero_point;
270
271 output[0] = (int8_t) vout0;
272 output[1] = (int8_t) vout1;
273 output[2] = (int8_t) vout2;
274 output[3] = (int8_t) vout3;
275 output += 4;
276 }
277 if XNN_UNLIKELY(c != 0) {
278 const int8_t* k = (const int8_t*) ((uintptr_t) w + 4 * sizeof(int32_t));
279 do {
280 int32_t vacc = *((const int32_t*) w);
281 w = (const void*) ((uintptr_t) w + sizeof(int32_t));
282
283 const int32_t vi0 = (int32_t) *i0++;
284 const int32_t vk0 = (int32_t) k[0];
285 vacc += vi0 * vk0;
286 const int32_t vi1 = (int32_t) *i1++;
287 const int32_t vk1 = (int32_t) k[4];
288 vacc += vi1 * vk1;
289 const int32_t vi2 = (int32_t) *i2++;
290 const int32_t vk2 = (int32_t) k[8];
291 vacc += vi2 * vk2;
292 const int32_t vi3 = (int32_t) *i3++;
293 const int32_t vk3 = (int32_t) k[12];
294 vacc += vi3 * vk3;
295 const int32_t vi4 = (int32_t) *i4++;
296 const int32_t vk4 = (int32_t) k[16];
297 vacc += vi4 * vk4;
298 const int32_t vi5 = (int32_t) *i5++;
299 const int32_t vk5 = (int32_t) k[20];
300 vacc += vi5 * vk5;
301 const int32_t vi6 = (int32_t) *i6++;
302 const int32_t vk6 = (int32_t) k[24];
303 vacc += vi6 * vk6;
304 const int32_t vi7 = (int32_t) *i7++;
305 const int32_t vk7 = (int32_t) k[28];
306 vacc += vi7 * vk7;
307 const int32_t vi8 = (int32_t) *i8++;
308 const int32_t vk8 = (int32_t) k[32];
309 vacc += vi8 * vk8;
310 k += 1;
311
312 float vfpacc = (float) vacc * vscale;
313
314 vfpacc = math_max_f32(vfpacc, voutput_min_less_zero_point);
315 vfpacc = math_min_f32(vfpacc, voutput_max_less_zero_point);
316 vfpacc += vmagic_bias;
317 int32_t vout = (int32_t) fp32_to_bits(vfpacc) - vmagic_bias_less_output_zero_point;
318
319 *output++ = (int8_t) vout;
320 } while (--c != 0);
321 }
322
323 output = (int8_t*) ((uintptr_t) output + output_increment);
324 } while (--output_width != 0);
325 }
326