1 // Auto-generated file. Do not edit!
2 // Template: src/qs8-dwconv/unipass-neon-mul8.c.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2020 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10 #include <assert.h>
11
12 #include <arm_neon.h>
13
14 #include <xnnpack/dwconv.h>
15
16
xnn_qs8_dwconv_minmax_rndnu_ukernel_up16x9__neon_mul8_ld64(size_t channels,size_t output_width,const int8_t ** input,const void * weights,int8_t * output,size_t input_stride,size_t output_increment,size_t input_offset,const int8_t * zero,const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS (1)])17 void xnn_qs8_dwconv_minmax_rndnu_ukernel_up16x9__neon_mul8_ld64(
18 size_t channels,
19 size_t output_width,
20 const int8_t** input,
21 const void* weights,
22 int8_t* output,
23 size_t input_stride,
24 size_t output_increment,
25 size_t input_offset,
26 const int8_t* zero,
27 const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_OOB_READS
28 {
29 assert(channels != 0);
30 assert(output_width != 0);
31
32 const int32x4_t vright_pre_shift = vld1q_dup_s32(¶ms->rndnu_neon.right_pre_shift);
33 const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->rndnu_neon.multiplier);
34 const int32x4_t vright_post_shift = vld1q_dup_s32(¶ms->rndnu_neon.right_post_shift);
35 const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->rndnu_neon.output_zero_point);
36 const int8x16_t voutput_min = vld1q_dup_s8(¶ms->rndnu_neon.output_min);
37 const int8x16_t voutput_max = vld1q_dup_s8(¶ms->rndnu_neon.output_max);
38 do {
39 const int8_t* i0 = input[0];
40 assert(i0 != NULL);
41 if XNN_UNPREDICTABLE(i0 != zero) {
42 i0 = (const int8_t*) ((uintptr_t) i0 + input_offset);
43 }
44 const int8_t* i1 = input[1];
45 assert(i1 != NULL);
46 if XNN_UNPREDICTABLE(i1 != zero) {
47 i1 = (const int8_t*) ((uintptr_t) i1 + input_offset);
48 }
49 const int8_t* i2 = input[2];
50 assert(i2 != NULL);
51 if XNN_UNPREDICTABLE(i2 != zero) {
52 i2 = (const int8_t*) ((uintptr_t) i2 + input_offset);
53 }
54 const int8_t* i3 = input[3];
55 assert(i3 != NULL);
56 if XNN_UNPREDICTABLE(i3 != zero) {
57 i3 = (const int8_t*) ((uintptr_t) i3 + input_offset);
58 }
59 const int8_t* i4 = input[4];
60 assert(i4 != NULL);
61 if XNN_UNPREDICTABLE(i4 != zero) {
62 i4 = (const int8_t*) ((uintptr_t) i4 + input_offset);
63 }
64 const int8_t* i5 = input[5];
65 assert(i5 != NULL);
66 if XNN_UNPREDICTABLE(i5 != zero) {
67 i5 = (const int8_t*) ((uintptr_t) i5 + input_offset);
68 }
69 const int8_t* i6 = input[6];
70 assert(i6 != NULL);
71 if XNN_UNPREDICTABLE(i6 != zero) {
72 i6 = (const int8_t*) ((uintptr_t) i6 + input_offset);
73 }
74 const int8_t* i7 = input[7];
75 assert(i7 != NULL);
76 if XNN_UNPREDICTABLE(i7 != zero) {
77 i7 = (const int8_t*) ((uintptr_t) i7 + input_offset);
78 }
79 const int8_t* i8 = input[8];
80 assert(i8 != NULL);
81 if XNN_UNPREDICTABLE(i8 != zero) {
82 i8 = (const int8_t*) ((uintptr_t) i8 + input_offset);
83 }
84 input = (const int8_t**) ((uintptr_t) input + input_stride);
85
86 size_t c = channels;
87 const void* w = weights;
88 for (; c >= 16; c -= 16) {
89 int32x4_t vacc0123 = vld1q_s32(w); w = (const void*) ((const int32_t*) w + 4);
90 int32x4_t vacc4567 = vld1q_s32(w); w = (const void*) ((const int32_t*) w + 4);
91 int32x4_t vacc89AB = vld1q_s32(w); w = (const void*) ((const int32_t*) w + 4);
92 int32x4_t vaccCDEF = vld1q_s32(w); w = (const void*) ((const int32_t*) w + 4);
93
94 const int8x8_t vi0x01234567 = vld1_s8(i0); i0 += 8;
95 const int8x8_t vk0x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
96 const int8x8_t vi0x89ABCDEF = vld1_s8(i0); i0 += 8;
97 const int8x8_t vk0x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
98
99 int16x8_t vprod01234567 = vmull_s8(vi0x01234567, vk0x01234567);
100 int16x8_t vprod89ABCDEF = vmull_s8(vi0x89ABCDEF, vk0x89ABCDEF);
101
102 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
103 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
104 vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
105 vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
106 const int8x8_t vi1x01234567 = vld1_s8(i1); i1 += 8;
107 const int8x8_t vk1x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
108 const int8x8_t vi1x89ABCDEF = vld1_s8(i1); i1 += 8;
109 const int8x8_t vk1x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
110
111 vprod01234567 = vmull_s8(vi1x01234567, vk1x01234567);
112 vprod89ABCDEF = vmull_s8(vi1x89ABCDEF, vk1x89ABCDEF);
113
114 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
115 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
116 vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
117 vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
118 const int8x8_t vi2x01234567 = vld1_s8(i2); i2 += 8;
119 const int8x8_t vk2x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
120 const int8x8_t vi2x89ABCDEF = vld1_s8(i2); i2 += 8;
121 const int8x8_t vk2x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
122
123 vprod01234567 = vmull_s8(vi2x01234567, vk2x01234567);
124 vprod89ABCDEF = vmull_s8(vi2x89ABCDEF, vk2x89ABCDEF);
125
126 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
127 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
128 vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
129 vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
130 const int8x8_t vi3x01234567 = vld1_s8(i3); i3 += 8;
131 const int8x8_t vk3x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
132 const int8x8_t vi3x89ABCDEF = vld1_s8(i3); i3 += 8;
133 const int8x8_t vk3x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
134
135 vprod01234567 = vmull_s8(vi3x01234567, vk3x01234567);
136 vprod89ABCDEF = vmull_s8(vi3x89ABCDEF, vk3x89ABCDEF);
137
138 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
139 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
140 vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
141 vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
142 const int8x8_t vi4x01234567 = vld1_s8(i4); i4 += 8;
143 const int8x8_t vk4x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
144 const int8x8_t vi4x89ABCDEF = vld1_s8(i4); i4 += 8;
145 const int8x8_t vk4x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
146
147 vprod01234567 = vmull_s8(vi4x01234567, vk4x01234567);
148 vprod89ABCDEF = vmull_s8(vi4x89ABCDEF, vk4x89ABCDEF);
149
150 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
151 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
152 vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
153 vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
154 const int8x8_t vi5x01234567 = vld1_s8(i5); i5 += 8;
155 const int8x8_t vk5x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
156 const int8x8_t vi5x89ABCDEF = vld1_s8(i5); i5 += 8;
157 const int8x8_t vk5x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
158
159 vprod01234567 = vmull_s8(vi5x01234567, vk5x01234567);
160 vprod89ABCDEF = vmull_s8(vi5x89ABCDEF, vk5x89ABCDEF);
161
162 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
163 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
164 vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
165 vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
166 const int8x8_t vi6x01234567 = vld1_s8(i6); i6 += 8;
167 const int8x8_t vk6x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
168 const int8x8_t vi6x89ABCDEF = vld1_s8(i6); i6 += 8;
169 const int8x8_t vk6x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
170
171 vprod01234567 = vmull_s8(vi6x01234567, vk6x01234567);
172 vprod89ABCDEF = vmull_s8(vi6x89ABCDEF, vk6x89ABCDEF);
173
174 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
175 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
176 vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
177 vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
178 const int8x8_t vi7x01234567 = vld1_s8(i7); i7 += 8;
179 const int8x8_t vk7x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
180 const int8x8_t vi7x89ABCDEF = vld1_s8(i7); i7 += 8;
181 const int8x8_t vk7x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
182
183 vprod01234567 = vmull_s8(vi7x01234567, vk7x01234567);
184 vprod89ABCDEF = vmull_s8(vi7x89ABCDEF, vk7x89ABCDEF);
185
186 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
187 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
188 vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
189 vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
190 const int8x8_t vi8x01234567 = vld1_s8(i8); i8 += 8;
191 const int8x8_t vk8x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
192 const int8x8_t vi8x89ABCDEF = vld1_s8(i8); i8 += 8;
193 const int8x8_t vk8x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
194
195 vprod01234567 = vmull_s8(vi8x01234567, vk8x01234567);
196 vprod89ABCDEF = vmull_s8(vi8x89ABCDEF, vk8x89ABCDEF);
197
198 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
199 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
200 vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
201 vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
202
203 vacc0123 = vqshlq_s32(vacc0123, vright_pre_shift);
204 vacc4567 = vqshlq_s32(vacc4567, vright_pre_shift);
205 vacc89AB = vqshlq_s32(vacc89AB, vright_pre_shift);
206 vaccCDEF = vqshlq_s32(vaccCDEF, vright_pre_shift);
207
208 vacc0123 = vqdmulhq_s32(vacc0123, vmultiplier);
209 vacc4567 = vqdmulhq_s32(vacc4567, vmultiplier);
210 vacc89AB = vqdmulhq_s32(vacc89AB, vmultiplier);
211 vaccCDEF = vqdmulhq_s32(vaccCDEF, vmultiplier);
212
213 vacc0123 = vrshlq_s32(vacc0123, vright_post_shift);
214 vacc4567 = vrshlq_s32(vacc4567, vright_post_shift);
215 vacc89AB = vrshlq_s32(vacc89AB, vright_post_shift);
216 vaccCDEF = vrshlq_s32(vaccCDEF, vright_post_shift);
217
218 #if XNN_ARCH_ARM64
219 int16x8_t vacc01234567 = vqmovn_high_s32(vqmovn_s32(vacc0123), vacc4567);
220 int16x8_t vacc89ABCDEF = vqmovn_high_s32(vqmovn_s32(vacc89AB), vaccCDEF);
221
222 vacc01234567 = vqaddq_s16(vacc01234567, voutput_zero_point);
223 vacc89ABCDEF = vqaddq_s16(vacc89ABCDEF, voutput_zero_point);
224
225 int8x16_t vout0123456789ABCDEF = vqmovn_high_s16(vqmovn_s16(vacc01234567), vacc89ABCDEF);
226 #else // !XNN_ARCH_ARM64
227 int16x8_t vacc01234567 = vcombine_s16(vqmovn_s32(vacc0123), vqmovn_s32(vacc4567));
228 int16x8_t vacc89ABCDEF = vcombine_s16(vqmovn_s32(vacc89AB), vqmovn_s32(vaccCDEF));
229
230 vacc01234567 = vqaddq_s16(vacc01234567, voutput_zero_point);
231 vacc89ABCDEF = vqaddq_s16(vacc89ABCDEF, voutput_zero_point);
232
233 int8x16_t vout0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc01234567), vqmovn_s16(vacc89ABCDEF));
234 #endif // !XNN_ARCH_ARM64
235
236 vout0123456789ABCDEF = vmaxq_s8(vout0123456789ABCDEF, voutput_min);
237
238 vout0123456789ABCDEF = vminq_s8(vout0123456789ABCDEF, voutput_max);
239
240 vst1q_s8(output, vout0123456789ABCDEF); output += 16;
241 }
242 if XNN_UNLIKELY(c != 0) {
243 const int8_t* k = (const int8_t*) ((const int32_t*) w + 16);
244 do {
245 int32x4_t vacc0123 = vld1q_s32(w); w = (const void*) ((const int32_t*) w + 4);
246 int32x4_t vacc4567 = vld1q_s32(w); w = (const void*) ((const int32_t*) w + 4);
247
248 const int8x8_t vi0x01234567 = vld1_s8(i0); i0 += 8;
249 const int8x8_t vk0x01234567 = vld1_s8(k); k += 8;
250
251 int16x8_t vprod01234567 = vmull_s8(vi0x01234567, vk0x01234567);
252
253 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
254 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
255 const int8x8_t vi1x01234567 = vld1_s8(i1); i1 += 8;
256 const int8x8_t vk1x01234567 = vld1_s8((const void*) (k + 8));
257
258 vprod01234567 = vmull_s8(vi1x01234567, vk1x01234567);
259
260 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
261 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
262 const int8x8_t vi2x01234567 = vld1_s8(i2); i2 += 8;
263 const int8x8_t vk2x01234567 = vld1_s8((const void*) (k + 24));
264
265 vprod01234567 = vmull_s8(vi2x01234567, vk2x01234567);
266
267 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
268 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
269 const int8x8_t vi3x01234567 = vld1_s8(i3); i3 += 8;
270 const int8x8_t vk3x01234567 = vld1_s8((const void*) (k + 40));
271
272 vprod01234567 = vmull_s8(vi3x01234567, vk3x01234567);
273
274 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
275 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
276 const int8x8_t vi4x01234567 = vld1_s8(i4); i4 += 8;
277 const int8x8_t vk4x01234567 = vld1_s8((const void*) (k + 56));
278
279 vprod01234567 = vmull_s8(vi4x01234567, vk4x01234567);
280
281 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
282 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
283 const int8x8_t vi5x01234567 = vld1_s8(i5); i5 += 8;
284 const int8x8_t vk5x01234567 = vld1_s8((const void*) (k + 72));
285
286 vprod01234567 = vmull_s8(vi5x01234567, vk5x01234567);
287
288 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
289 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
290 const int8x8_t vi6x01234567 = vld1_s8(i6); i6 += 8;
291 const int8x8_t vk6x01234567 = vld1_s8((const void*) (k + 88));
292
293 vprod01234567 = vmull_s8(vi6x01234567, vk6x01234567);
294
295 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
296 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
297 const int8x8_t vi7x01234567 = vld1_s8(i7); i7 += 8;
298 const int8x8_t vk7x01234567 = vld1_s8((const void*) (k + 104));
299
300 vprod01234567 = vmull_s8(vi7x01234567, vk7x01234567);
301
302 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
303 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
304 const int8x8_t vi8x01234567 = vld1_s8(i8); i8 += 8;
305 const int8x8_t vk8x01234567 = vld1_s8((const void*) (k + 120));
306
307 vprod01234567 = vmull_s8(vi8x01234567, vk8x01234567);
308
309 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
310 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
311
312 vacc0123 = vqshlq_s32(vacc0123, vright_pre_shift);
313 vacc4567 = vqshlq_s32(vacc4567, vright_pre_shift);
314
315 vacc0123 = vqdmulhq_s32(vacc0123, vmultiplier);
316 vacc4567 = vqdmulhq_s32(vacc4567, vmultiplier);
317
318 vacc0123 = vrshlq_s32(vacc0123, vright_post_shift);
319 vacc4567 = vrshlq_s32(vacc4567, vright_post_shift);
320
321 #if XNN_ARCH_ARM64
322 int16x8_t vacc01234567 = vqmovn_high_s32(vqmovn_s32(vacc0123), vacc4567);
323 #else
324 int16x8_t vacc01234567 = vcombine_s16(vqmovn_s32(vacc0123), vqmovn_s32(vacc4567));
325 #endif
326 vacc01234567 = vqaddq_s16(vacc01234567, voutput_zero_point);
327
328 int8x8_t vout01234567 = vqmovn_s16(vacc01234567);
329 vout01234567 = vmax_s8(vout01234567, vget_low_s8(voutput_min));
330 vout01234567 = vmin_s8(vout01234567, vget_low_s8(voutput_max));
331
332 if XNN_LIKELY(c >= 8) {
333 vst1_s8(output, vout01234567); output += 8;
334 c -= 8;
335 } else {
336 if (c & 4) {
337 vst1_lane_u32((void*) output, vreinterpret_u32_s8(vout01234567), 0); output += 4;
338 vout01234567 = vext_s8(vout01234567, vout01234567, 4);
339 }
340 if (c & 2) {
341 vst1_lane_u16((void*) output, vreinterpret_u16_s8(vout01234567), 0); output += 2;
342 vout01234567 = vext_s8(vout01234567, vout01234567, 2);
343 }
344 if (c & 1) {
345 vst1_lane_s8(output, vout01234567, 0); output += 1;
346 }
347 c = 0;
348 }
349 } while (c != 0);
350 }
351
352 output = (int8_t*) ((uintptr_t) output + output_increment);
353 } while (--output_width != 0);
354 }
355