1 // Auto-generated file. Do not edit!
2 // Template: src/f16-dwconv/up-neonfp16arith.c.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2020 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10 #include <assert.h>
11
12 #include <arm_neon.h>
13
14 #include <xnnpack/dwconv.h>
15
16
xnn_f16_dwconv_minmax_ukernel_up32x25__neonfp16arith(size_t channels,size_t output_width,const void ** input,const void * weights,void * output_ptr,size_t input_stride,size_t output_increment,size_t input_offset,const void * zero,const union xnn_f16_minmax_params params[restrict XNN_MIN_ELEMENTS (1)])17 void xnn_f16_dwconv_minmax_ukernel_up32x25__neonfp16arith(
18 size_t channels,
19 size_t output_width,
20 const void** input,
21 const void* weights,
22 void* output_ptr,
23 size_t input_stride,
24 size_t output_increment,
25 size_t input_offset,
26 const void* zero,
27 const union xnn_f16_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_OOB_READS
28 {
29 assert(channels != 0);
30 assert(output_width != 0);
31
32 __fp16* output = (__fp16*) output_ptr;
33 const float16x8_t vmax = vreinterpretq_f16_u16(vld1q_dup_u16(¶ms->neon.max));
34 const float16x8_t vmin = vreinterpretq_f16_u16(vld1q_dup_u16(¶ms->neon.min));
35 do {
36 const __fp16* i0 = (const __fp16*) input[0];
37 assert(i0 != NULL);
38 if XNN_UNPREDICTABLE(i0 != (const __fp16*) zero) {
39 i0 = (const __fp16*) ((uintptr_t) i0 + input_offset);
40 }
41 const __fp16* i1 = (const __fp16*) input[1];
42 assert(i1 != NULL);
43 if XNN_UNPREDICTABLE(i1 != (const __fp16*) zero) {
44 i1 = (const __fp16*) ((uintptr_t) i1 + input_offset);
45 }
46 const __fp16* i2 = (const __fp16*) input[2];
47 assert(i2 != NULL);
48 if XNN_UNPREDICTABLE(i2 != (const __fp16*) zero) {
49 i2 = (const __fp16*) ((uintptr_t) i2 + input_offset);
50 }
51 const __fp16* i3 = (const __fp16*) input[3];
52 assert(i3 != NULL);
53 if XNN_UNPREDICTABLE(i3 != (const __fp16*) zero) {
54 i3 = (const __fp16*) ((uintptr_t) i3 + input_offset);
55 }
56 const __fp16* i4 = (const __fp16*) input[4];
57 assert(i4 != NULL);
58 if XNN_UNPREDICTABLE(i4 != (const __fp16*) zero) {
59 i4 = (const __fp16*) ((uintptr_t) i4 + input_offset);
60 }
61 const __fp16* i5 = (const __fp16*) input[5];
62 assert(i5 != NULL);
63 if XNN_UNPREDICTABLE(i5 != (const __fp16*) zero) {
64 i5 = (const __fp16*) ((uintptr_t) i5 + input_offset);
65 }
66 const __fp16* i6 = (const __fp16*) input[6];
67 assert(i6 != NULL);
68 if XNN_UNPREDICTABLE(i6 != (const __fp16*) zero) {
69 i6 = (const __fp16*) ((uintptr_t) i6 + input_offset);
70 }
71 const __fp16* i7 = (const __fp16*) input[7];
72 assert(i7 != NULL);
73 if XNN_UNPREDICTABLE(i7 != (const __fp16*) zero) {
74 i7 = (const __fp16*) ((uintptr_t) i7 + input_offset);
75 }
76 const __fp16* i8 = (const __fp16*) input[8];
77 assert(i8 != NULL);
78 if XNN_UNPREDICTABLE(i8 != (const __fp16*) zero) {
79 i8 = (const __fp16*) ((uintptr_t) i8 + input_offset);
80 }
81 const __fp16* i9 = (const __fp16*) input[9];
82 assert(i9 != NULL);
83 if XNN_UNPREDICTABLE(i9 != (const __fp16*) zero) {
84 i9 = (const __fp16*) ((uintptr_t) i9 + input_offset);
85 }
86 const __fp16* i10 = (const __fp16*) input[10];
87 assert(i10 != NULL);
88 if XNN_UNPREDICTABLE(i10 != (const __fp16*) zero) {
89 i10 = (const __fp16*) ((uintptr_t) i10 + input_offset);
90 }
91 const __fp16* i11 = (const __fp16*) input[11];
92 assert(i11 != NULL);
93 if XNN_UNPREDICTABLE(i11 != (const __fp16*) zero) {
94 i11 = (const __fp16*) ((uintptr_t) i11 + input_offset);
95 }
96 const __fp16* i12 = (const __fp16*) input[12];
97 assert(i12 != NULL);
98 if XNN_UNPREDICTABLE(i12 != (const __fp16*) zero) {
99 i12 = (const __fp16*) ((uintptr_t) i12 + input_offset);
100 }
101 const __fp16* i13 = (const __fp16*) input[13];
102 assert(i13 != NULL);
103 if XNN_UNPREDICTABLE(i13 != (const __fp16*) zero) {
104 i13 = (const __fp16*) ((uintptr_t) i13 + input_offset);
105 }
106 const __fp16* i14 = (const __fp16*) input[14];
107 assert(i14 != NULL);
108 if XNN_UNPREDICTABLE(i14 != (const __fp16*) zero) {
109 i14 = (const __fp16*) ((uintptr_t) i14 + input_offset);
110 }
111 const __fp16* i15 = (const __fp16*) input[15];
112 assert(i15 != NULL);
113 if XNN_UNPREDICTABLE(i15 != (const __fp16*) zero) {
114 i15 = (const __fp16*) ((uintptr_t) i15 + input_offset);
115 }
116 const __fp16* i16 = (const __fp16*) input[16];
117 assert(i16 != NULL);
118 if XNN_UNPREDICTABLE(i16 != (const __fp16*) zero) {
119 i16 = (const __fp16*) ((uintptr_t) i16 + input_offset);
120 }
121 const __fp16* i17 = (const __fp16*) input[17];
122 assert(i17 != NULL);
123 if XNN_UNPREDICTABLE(i17 != (const __fp16*) zero) {
124 i17 = (const __fp16*) ((uintptr_t) i17 + input_offset);
125 }
126 const __fp16* i18 = (const __fp16*) input[18];
127 assert(i18 != NULL);
128 if XNN_UNPREDICTABLE(i18 != (const __fp16*) zero) {
129 i18 = (const __fp16*) ((uintptr_t) i18 + input_offset);
130 }
131 const __fp16* i19 = (const __fp16*) input[19];
132 assert(i19 != NULL);
133 if XNN_UNPREDICTABLE(i19 != (const __fp16*) zero) {
134 i19 = (const __fp16*) ((uintptr_t) i19 + input_offset);
135 }
136 const __fp16* i20 = (const __fp16*) input[20];
137 assert(i20 != NULL);
138 if XNN_UNPREDICTABLE(i20 != (const __fp16*) zero) {
139 i20 = (const __fp16*) ((uintptr_t) i20 + input_offset);
140 }
141 const __fp16* i21 = (const __fp16*) input[21];
142 assert(i21 != NULL);
143 if XNN_UNPREDICTABLE(i21 != (const __fp16*) zero) {
144 i21 = (const __fp16*) ((uintptr_t) i21 + input_offset);
145 }
146 const __fp16* i22 = (const __fp16*) input[22];
147 assert(i22 != NULL);
148 if XNN_UNPREDICTABLE(i22 != (const __fp16*) zero) {
149 i22 = (const __fp16*) ((uintptr_t) i22 + input_offset);
150 }
151 const __fp16* i23 = (const __fp16*) input[23];
152 assert(i23 != NULL);
153 if XNN_UNPREDICTABLE(i23 != (const __fp16*) zero) {
154 i23 = (const __fp16*) ((uintptr_t) i23 + input_offset);
155 }
156 const __fp16* i24 = (const __fp16*) input[24];
157 assert(i24 != NULL);
158 if XNN_UNPREDICTABLE(i24 != (const __fp16*) zero) {
159 i24 = (const __fp16*) ((uintptr_t) i24 + input_offset);
160 }
161
162 input = (const void**) ((uintptr_t) input + input_stride);
163
164 size_t c = channels;
165 const __fp16* w = (const __fp16*) weights;
166 for (; c >= 32; c -= 32) {
167 float16x8_t vacc01234567p0 = vld1q_f16(w); w += 8;
168 float16x8_t vacc89ABCDEFp0 = vld1q_f16(w); w += 8;
169 float16x8_t vaccGHIJKLMNp0 = vld1q_f16(w); w += 8;
170 float16x8_t vaccOPQRSTUVp0 = vld1q_f16(w); w += 8;
171
172
173 const float16x8_t vi0x01234567 = vld1q_f16(i0); i0 += 8;
174 const float16x8_t vi0x89ABCDEF = vld1q_f16(i0); i0 += 8;
175 const float16x8_t vi0xGHIJKLMN = vld1q_f16(i0); i0 += 8;
176 const float16x8_t vi0xOPQRSTUV = vld1q_f16(i0); i0 += 8;
177 const float16x8_t vk0x01234567 = vld1q_f16(w); w += 8;
178 const float16x8_t vk0x89ABCDEF = vld1q_f16(w); w += 8;
179 const float16x8_t vk0xGHIJKLMN = vld1q_f16(w); w += 8;
180 const float16x8_t vk0xOPQRSTUV = vld1q_f16(w); w += 8;
181 vacc01234567p0 = vfmaq_f16(vacc01234567p0, vi0x01234567, vk0x01234567);
182 vacc89ABCDEFp0 = vfmaq_f16(vacc89ABCDEFp0, vi0x89ABCDEF, vk0x89ABCDEF);
183 vaccGHIJKLMNp0 = vfmaq_f16(vaccGHIJKLMNp0, vi0xGHIJKLMN, vk0xGHIJKLMN);
184 vaccOPQRSTUVp0 = vfmaq_f16(vaccOPQRSTUVp0, vi0xOPQRSTUV, vk0xOPQRSTUV);
185
186 const float16x8_t vi1x01234567 = vld1q_f16(i1); i1 += 8;
187 const float16x8_t vi1x89ABCDEF = vld1q_f16(i1); i1 += 8;
188 const float16x8_t vi1xGHIJKLMN = vld1q_f16(i1); i1 += 8;
189 const float16x8_t vi1xOPQRSTUV = vld1q_f16(i1); i1 += 8;
190 const float16x8_t vk1x01234567 = vld1q_f16(w); w += 8;
191 const float16x8_t vk1x89ABCDEF = vld1q_f16(w); w += 8;
192 const float16x8_t vk1xGHIJKLMN = vld1q_f16(w); w += 8;
193 const float16x8_t vk1xOPQRSTUV = vld1q_f16(w); w += 8;
194 vacc01234567p0 = vfmaq_f16(vacc01234567p0, vi1x01234567, vk1x01234567);
195 vacc89ABCDEFp0 = vfmaq_f16(vacc89ABCDEFp0, vi1x89ABCDEF, vk1x89ABCDEF);
196 vaccGHIJKLMNp0 = vfmaq_f16(vaccGHIJKLMNp0, vi1xGHIJKLMN, vk1xGHIJKLMN);
197 vaccOPQRSTUVp0 = vfmaq_f16(vaccOPQRSTUVp0, vi1xOPQRSTUV, vk1xOPQRSTUV);
198
199 const float16x8_t vi2x01234567 = vld1q_f16(i2); i2 += 8;
200 const float16x8_t vi2x89ABCDEF = vld1q_f16(i2); i2 += 8;
201 const float16x8_t vi2xGHIJKLMN = vld1q_f16(i2); i2 += 8;
202 const float16x8_t vi2xOPQRSTUV = vld1q_f16(i2); i2 += 8;
203 const float16x8_t vk2x01234567 = vld1q_f16(w); w += 8;
204 const float16x8_t vk2x89ABCDEF = vld1q_f16(w); w += 8;
205 const float16x8_t vk2xGHIJKLMN = vld1q_f16(w); w += 8;
206 const float16x8_t vk2xOPQRSTUV = vld1q_f16(w); w += 8;
207 vacc01234567p0 = vfmaq_f16(vacc01234567p0, vi2x01234567, vk2x01234567);
208 vacc89ABCDEFp0 = vfmaq_f16(vacc89ABCDEFp0, vi2x89ABCDEF, vk2x89ABCDEF);
209 vaccGHIJKLMNp0 = vfmaq_f16(vaccGHIJKLMNp0, vi2xGHIJKLMN, vk2xGHIJKLMN);
210 vaccOPQRSTUVp0 = vfmaq_f16(vaccOPQRSTUVp0, vi2xOPQRSTUV, vk2xOPQRSTUV);
211
212 const float16x8_t vi3x01234567 = vld1q_f16(i3); i3 += 8;
213 const float16x8_t vi3x89ABCDEF = vld1q_f16(i3); i3 += 8;
214 const float16x8_t vi3xGHIJKLMN = vld1q_f16(i3); i3 += 8;
215 const float16x8_t vi3xOPQRSTUV = vld1q_f16(i3); i3 += 8;
216 const float16x8_t vk3x01234567 = vld1q_f16(w); w += 8;
217 const float16x8_t vk3x89ABCDEF = vld1q_f16(w); w += 8;
218 const float16x8_t vk3xGHIJKLMN = vld1q_f16(w); w += 8;
219 const float16x8_t vk3xOPQRSTUV = vld1q_f16(w); w += 8;
220 vacc01234567p0 = vfmaq_f16(vacc01234567p0, vi3x01234567, vk3x01234567);
221 vacc89ABCDEFp0 = vfmaq_f16(vacc89ABCDEFp0, vi3x89ABCDEF, vk3x89ABCDEF);
222 vaccGHIJKLMNp0 = vfmaq_f16(vaccGHIJKLMNp0, vi3xGHIJKLMN, vk3xGHIJKLMN);
223 vaccOPQRSTUVp0 = vfmaq_f16(vaccOPQRSTUVp0, vi3xOPQRSTUV, vk3xOPQRSTUV);
224
225 const float16x8_t vi4x01234567 = vld1q_f16(i4); i4 += 8;
226 const float16x8_t vi4x89ABCDEF = vld1q_f16(i4); i4 += 8;
227 const float16x8_t vi4xGHIJKLMN = vld1q_f16(i4); i4 += 8;
228 const float16x8_t vi4xOPQRSTUV = vld1q_f16(i4); i4 += 8;
229 const float16x8_t vk4x01234567 = vld1q_f16(w); w += 8;
230 const float16x8_t vk4x89ABCDEF = vld1q_f16(w); w += 8;
231 const float16x8_t vk4xGHIJKLMN = vld1q_f16(w); w += 8;
232 const float16x8_t vk4xOPQRSTUV = vld1q_f16(w); w += 8;
233 vacc01234567p0 = vfmaq_f16(vacc01234567p0, vi4x01234567, vk4x01234567);
234 vacc89ABCDEFp0 = vfmaq_f16(vacc89ABCDEFp0, vi4x89ABCDEF, vk4x89ABCDEF);
235 vaccGHIJKLMNp0 = vfmaq_f16(vaccGHIJKLMNp0, vi4xGHIJKLMN, vk4xGHIJKLMN);
236 vaccOPQRSTUVp0 = vfmaq_f16(vaccOPQRSTUVp0, vi4xOPQRSTUV, vk4xOPQRSTUV);
237
238 const float16x8_t vi5x01234567 = vld1q_f16(i5); i5 += 8;
239 const float16x8_t vi5x89ABCDEF = vld1q_f16(i5); i5 += 8;
240 const float16x8_t vi5xGHIJKLMN = vld1q_f16(i5); i5 += 8;
241 const float16x8_t vi5xOPQRSTUV = vld1q_f16(i5); i5 += 8;
242 const float16x8_t vk5x01234567 = vld1q_f16(w); w += 8;
243 const float16x8_t vk5x89ABCDEF = vld1q_f16(w); w += 8;
244 const float16x8_t vk5xGHIJKLMN = vld1q_f16(w); w += 8;
245 const float16x8_t vk5xOPQRSTUV = vld1q_f16(w); w += 8;
246 vacc01234567p0 = vfmaq_f16(vacc01234567p0, vi5x01234567, vk5x01234567);
247 vacc89ABCDEFp0 = vfmaq_f16(vacc89ABCDEFp0, vi5x89ABCDEF, vk5x89ABCDEF);
248 vaccGHIJKLMNp0 = vfmaq_f16(vaccGHIJKLMNp0, vi5xGHIJKLMN, vk5xGHIJKLMN);
249 vaccOPQRSTUVp0 = vfmaq_f16(vaccOPQRSTUVp0, vi5xOPQRSTUV, vk5xOPQRSTUV);
250
251 const float16x8_t vi6x01234567 = vld1q_f16(i6); i6 += 8;
252 const float16x8_t vi6x89ABCDEF = vld1q_f16(i6); i6 += 8;
253 const float16x8_t vi6xGHIJKLMN = vld1q_f16(i6); i6 += 8;
254 const float16x8_t vi6xOPQRSTUV = vld1q_f16(i6); i6 += 8;
255 const float16x8_t vk6x01234567 = vld1q_f16(w); w += 8;
256 const float16x8_t vk6x89ABCDEF = vld1q_f16(w); w += 8;
257 const float16x8_t vk6xGHIJKLMN = vld1q_f16(w); w += 8;
258 const float16x8_t vk6xOPQRSTUV = vld1q_f16(w); w += 8;
259 vacc01234567p0 = vfmaq_f16(vacc01234567p0, vi6x01234567, vk6x01234567);
260 vacc89ABCDEFp0 = vfmaq_f16(vacc89ABCDEFp0, vi6x89ABCDEF, vk6x89ABCDEF);
261 vaccGHIJKLMNp0 = vfmaq_f16(vaccGHIJKLMNp0, vi6xGHIJKLMN, vk6xGHIJKLMN);
262 vaccOPQRSTUVp0 = vfmaq_f16(vaccOPQRSTUVp0, vi6xOPQRSTUV, vk6xOPQRSTUV);
263
264 const float16x8_t vi7x01234567 = vld1q_f16(i7); i7 += 8;
265 const float16x8_t vi7x89ABCDEF = vld1q_f16(i7); i7 += 8;
266 const float16x8_t vi7xGHIJKLMN = vld1q_f16(i7); i7 += 8;
267 const float16x8_t vi7xOPQRSTUV = vld1q_f16(i7); i7 += 8;
268 const float16x8_t vk7x01234567 = vld1q_f16(w); w += 8;
269 const float16x8_t vk7x89ABCDEF = vld1q_f16(w); w += 8;
270 const float16x8_t vk7xGHIJKLMN = vld1q_f16(w); w += 8;
271 const float16x8_t vk7xOPQRSTUV = vld1q_f16(w); w += 8;
272 vacc01234567p0 = vfmaq_f16(vacc01234567p0, vi7x01234567, vk7x01234567);
273 vacc89ABCDEFp0 = vfmaq_f16(vacc89ABCDEFp0, vi7x89ABCDEF, vk7x89ABCDEF);
274 vaccGHIJKLMNp0 = vfmaq_f16(vaccGHIJKLMNp0, vi7xGHIJKLMN, vk7xGHIJKLMN);
275 vaccOPQRSTUVp0 = vfmaq_f16(vaccOPQRSTUVp0, vi7xOPQRSTUV, vk7xOPQRSTUV);
276
277 const float16x8_t vi8x01234567 = vld1q_f16(i8); i8 += 8;
278 const float16x8_t vi8x89ABCDEF = vld1q_f16(i8); i8 += 8;
279 const float16x8_t vi8xGHIJKLMN = vld1q_f16(i8); i8 += 8;
280 const float16x8_t vi8xOPQRSTUV = vld1q_f16(i8); i8 += 8;
281 const float16x8_t vk8x01234567 = vld1q_f16(w); w += 8;
282 const float16x8_t vk8x89ABCDEF = vld1q_f16(w); w += 8;
283 const float16x8_t vk8xGHIJKLMN = vld1q_f16(w); w += 8;
284 const float16x8_t vk8xOPQRSTUV = vld1q_f16(w); w += 8;
285 vacc01234567p0 = vfmaq_f16(vacc01234567p0, vi8x01234567, vk8x01234567);
286 vacc89ABCDEFp0 = vfmaq_f16(vacc89ABCDEFp0, vi8x89ABCDEF, vk8x89ABCDEF);
287 vaccGHIJKLMNp0 = vfmaq_f16(vaccGHIJKLMNp0, vi8xGHIJKLMN, vk8xGHIJKLMN);
288 vaccOPQRSTUVp0 = vfmaq_f16(vaccOPQRSTUVp0, vi8xOPQRSTUV, vk8xOPQRSTUV);
289
290 const float16x8_t vi9x01234567 = vld1q_f16(i9); i9 += 8;
291 const float16x8_t vi9x89ABCDEF = vld1q_f16(i9); i9 += 8;
292 const float16x8_t vi9xGHIJKLMN = vld1q_f16(i9); i9 += 8;
293 const float16x8_t vi9xOPQRSTUV = vld1q_f16(i9); i9 += 8;
294 const float16x8_t vk9x01234567 = vld1q_f16(w); w += 8;
295 const float16x8_t vk9x89ABCDEF = vld1q_f16(w); w += 8;
296 const float16x8_t vk9xGHIJKLMN = vld1q_f16(w); w += 8;
297 const float16x8_t vk9xOPQRSTUV = vld1q_f16(w); w += 8;
298 vacc01234567p0 = vfmaq_f16(vacc01234567p0, vi9x01234567, vk9x01234567);
299 vacc89ABCDEFp0 = vfmaq_f16(vacc89ABCDEFp0, vi9x89ABCDEF, vk9x89ABCDEF);
300 vaccGHIJKLMNp0 = vfmaq_f16(vaccGHIJKLMNp0, vi9xGHIJKLMN, vk9xGHIJKLMN);
301 vaccOPQRSTUVp0 = vfmaq_f16(vaccOPQRSTUVp0, vi9xOPQRSTUV, vk9xOPQRSTUV);
302
303 const float16x8_t vi10x01234567 = vld1q_f16(i10); i10 += 8;
304 const float16x8_t vi10x89ABCDEF = vld1q_f16(i10); i10 += 8;
305 const float16x8_t vi10xGHIJKLMN = vld1q_f16(i10); i10 += 8;
306 const float16x8_t vi10xOPQRSTUV = vld1q_f16(i10); i10 += 8;
307 const float16x8_t vk10x01234567 = vld1q_f16(w); w += 8;
308 const float16x8_t vk10x89ABCDEF = vld1q_f16(w); w += 8;
309 const float16x8_t vk10xGHIJKLMN = vld1q_f16(w); w += 8;
310 const float16x8_t vk10xOPQRSTUV = vld1q_f16(w); w += 8;
311 vacc01234567p0 = vfmaq_f16(vacc01234567p0, vi10x01234567, vk10x01234567);
312 vacc89ABCDEFp0 = vfmaq_f16(vacc89ABCDEFp0, vi10x89ABCDEF, vk10x89ABCDEF);
313 vaccGHIJKLMNp0 = vfmaq_f16(vaccGHIJKLMNp0, vi10xGHIJKLMN, vk10xGHIJKLMN);
314 vaccOPQRSTUVp0 = vfmaq_f16(vaccOPQRSTUVp0, vi10xOPQRSTUV, vk10xOPQRSTUV);
315
316 const float16x8_t vi11x01234567 = vld1q_f16(i11); i11 += 8;
317 const float16x8_t vi11x89ABCDEF = vld1q_f16(i11); i11 += 8;
318 const float16x8_t vi11xGHIJKLMN = vld1q_f16(i11); i11 += 8;
319 const float16x8_t vi11xOPQRSTUV = vld1q_f16(i11); i11 += 8;
320 const float16x8_t vk11x01234567 = vld1q_f16(w); w += 8;
321 const float16x8_t vk11x89ABCDEF = vld1q_f16(w); w += 8;
322 const float16x8_t vk11xGHIJKLMN = vld1q_f16(w); w += 8;
323 const float16x8_t vk11xOPQRSTUV = vld1q_f16(w); w += 8;
324 vacc01234567p0 = vfmaq_f16(vacc01234567p0, vi11x01234567, vk11x01234567);
325 vacc89ABCDEFp0 = vfmaq_f16(vacc89ABCDEFp0, vi11x89ABCDEF, vk11x89ABCDEF);
326 vaccGHIJKLMNp0 = vfmaq_f16(vaccGHIJKLMNp0, vi11xGHIJKLMN, vk11xGHIJKLMN);
327 vaccOPQRSTUVp0 = vfmaq_f16(vaccOPQRSTUVp0, vi11xOPQRSTUV, vk11xOPQRSTUV);
328
329 const float16x8_t vi12x01234567 = vld1q_f16(i12); i12 += 8;
330 const float16x8_t vi12x89ABCDEF = vld1q_f16(i12); i12 += 8;
331 const float16x8_t vi12xGHIJKLMN = vld1q_f16(i12); i12 += 8;
332 const float16x8_t vi12xOPQRSTUV = vld1q_f16(i12); i12 += 8;
333 const float16x8_t vk12x01234567 = vld1q_f16(w); w += 8;
334 const float16x8_t vk12x89ABCDEF = vld1q_f16(w); w += 8;
335 const float16x8_t vk12xGHIJKLMN = vld1q_f16(w); w += 8;
336 const float16x8_t vk12xOPQRSTUV = vld1q_f16(w); w += 8;
337 vacc01234567p0 = vfmaq_f16(vacc01234567p0, vi12x01234567, vk12x01234567);
338 vacc89ABCDEFp0 = vfmaq_f16(vacc89ABCDEFp0, vi12x89ABCDEF, vk12x89ABCDEF);
339 vaccGHIJKLMNp0 = vfmaq_f16(vaccGHIJKLMNp0, vi12xGHIJKLMN, vk12xGHIJKLMN);
340 vaccOPQRSTUVp0 = vfmaq_f16(vaccOPQRSTUVp0, vi12xOPQRSTUV, vk12xOPQRSTUV);
341
342 const float16x8_t vi13x01234567 = vld1q_f16(i13); i13 += 8;
343 const float16x8_t vi13x89ABCDEF = vld1q_f16(i13); i13 += 8;
344 const float16x8_t vi13xGHIJKLMN = vld1q_f16(i13); i13 += 8;
345 const float16x8_t vi13xOPQRSTUV = vld1q_f16(i13); i13 += 8;
346 const float16x8_t vk13x01234567 = vld1q_f16(w); w += 8;
347 const float16x8_t vk13x89ABCDEF = vld1q_f16(w); w += 8;
348 const float16x8_t vk13xGHIJKLMN = vld1q_f16(w); w += 8;
349 const float16x8_t vk13xOPQRSTUV = vld1q_f16(w); w += 8;
350 vacc01234567p0 = vfmaq_f16(vacc01234567p0, vi13x01234567, vk13x01234567);
351 vacc89ABCDEFp0 = vfmaq_f16(vacc89ABCDEFp0, vi13x89ABCDEF, vk13x89ABCDEF);
352 vaccGHIJKLMNp0 = vfmaq_f16(vaccGHIJKLMNp0, vi13xGHIJKLMN, vk13xGHIJKLMN);
353 vaccOPQRSTUVp0 = vfmaq_f16(vaccOPQRSTUVp0, vi13xOPQRSTUV, vk13xOPQRSTUV);
354
355 const float16x8_t vi14x01234567 = vld1q_f16(i14); i14 += 8;
356 const float16x8_t vi14x89ABCDEF = vld1q_f16(i14); i14 += 8;
357 const float16x8_t vi14xGHIJKLMN = vld1q_f16(i14); i14 += 8;
358 const float16x8_t vi14xOPQRSTUV = vld1q_f16(i14); i14 += 8;
359 const float16x8_t vk14x01234567 = vld1q_f16(w); w += 8;
360 const float16x8_t vk14x89ABCDEF = vld1q_f16(w); w += 8;
361 const float16x8_t vk14xGHIJKLMN = vld1q_f16(w); w += 8;
362 const float16x8_t vk14xOPQRSTUV = vld1q_f16(w); w += 8;
363 vacc01234567p0 = vfmaq_f16(vacc01234567p0, vi14x01234567, vk14x01234567);
364 vacc89ABCDEFp0 = vfmaq_f16(vacc89ABCDEFp0, vi14x89ABCDEF, vk14x89ABCDEF);
365 vaccGHIJKLMNp0 = vfmaq_f16(vaccGHIJKLMNp0, vi14xGHIJKLMN, vk14xGHIJKLMN);
366 vaccOPQRSTUVp0 = vfmaq_f16(vaccOPQRSTUVp0, vi14xOPQRSTUV, vk14xOPQRSTUV);
367
368 const float16x8_t vi15x01234567 = vld1q_f16(i15); i15 += 8;
369 const float16x8_t vi15x89ABCDEF = vld1q_f16(i15); i15 += 8;
370 const float16x8_t vi15xGHIJKLMN = vld1q_f16(i15); i15 += 8;
371 const float16x8_t vi15xOPQRSTUV = vld1q_f16(i15); i15 += 8;
372 const float16x8_t vk15x01234567 = vld1q_f16(w); w += 8;
373 const float16x8_t vk15x89ABCDEF = vld1q_f16(w); w += 8;
374 const float16x8_t vk15xGHIJKLMN = vld1q_f16(w); w += 8;
375 const float16x8_t vk15xOPQRSTUV = vld1q_f16(w); w += 8;
376 vacc01234567p0 = vfmaq_f16(vacc01234567p0, vi15x01234567, vk15x01234567);
377 vacc89ABCDEFp0 = vfmaq_f16(vacc89ABCDEFp0, vi15x89ABCDEF, vk15x89ABCDEF);
378 vaccGHIJKLMNp0 = vfmaq_f16(vaccGHIJKLMNp0, vi15xGHIJKLMN, vk15xGHIJKLMN);
379 vaccOPQRSTUVp0 = vfmaq_f16(vaccOPQRSTUVp0, vi15xOPQRSTUV, vk15xOPQRSTUV);
380
381 const float16x8_t vi16x01234567 = vld1q_f16(i16); i16 += 8;
382 const float16x8_t vi16x89ABCDEF = vld1q_f16(i16); i16 += 8;
383 const float16x8_t vi16xGHIJKLMN = vld1q_f16(i16); i16 += 8;
384 const float16x8_t vi16xOPQRSTUV = vld1q_f16(i16); i16 += 8;
385 const float16x8_t vk16x01234567 = vld1q_f16(w); w += 8;
386 const float16x8_t vk16x89ABCDEF = vld1q_f16(w); w += 8;
387 const float16x8_t vk16xGHIJKLMN = vld1q_f16(w); w += 8;
388 const float16x8_t vk16xOPQRSTUV = vld1q_f16(w); w += 8;
389 vacc01234567p0 = vfmaq_f16(vacc01234567p0, vi16x01234567, vk16x01234567);
390 vacc89ABCDEFp0 = vfmaq_f16(vacc89ABCDEFp0, vi16x89ABCDEF, vk16x89ABCDEF);
391 vaccGHIJKLMNp0 = vfmaq_f16(vaccGHIJKLMNp0, vi16xGHIJKLMN, vk16xGHIJKLMN);
392 vaccOPQRSTUVp0 = vfmaq_f16(vaccOPQRSTUVp0, vi16xOPQRSTUV, vk16xOPQRSTUV);
393
394 const float16x8_t vi17x01234567 = vld1q_f16(i17); i17 += 8;
395 const float16x8_t vi17x89ABCDEF = vld1q_f16(i17); i17 += 8;
396 const float16x8_t vi17xGHIJKLMN = vld1q_f16(i17); i17 += 8;
397 const float16x8_t vi17xOPQRSTUV = vld1q_f16(i17); i17 += 8;
398 const float16x8_t vk17x01234567 = vld1q_f16(w); w += 8;
399 const float16x8_t vk17x89ABCDEF = vld1q_f16(w); w += 8;
400 const float16x8_t vk17xGHIJKLMN = vld1q_f16(w); w += 8;
401 const float16x8_t vk17xOPQRSTUV = vld1q_f16(w); w += 8;
402 vacc01234567p0 = vfmaq_f16(vacc01234567p0, vi17x01234567, vk17x01234567);
403 vacc89ABCDEFp0 = vfmaq_f16(vacc89ABCDEFp0, vi17x89ABCDEF, vk17x89ABCDEF);
404 vaccGHIJKLMNp0 = vfmaq_f16(vaccGHIJKLMNp0, vi17xGHIJKLMN, vk17xGHIJKLMN);
405 vaccOPQRSTUVp0 = vfmaq_f16(vaccOPQRSTUVp0, vi17xOPQRSTUV, vk17xOPQRSTUV);
406
407 const float16x8_t vi18x01234567 = vld1q_f16(i18); i18 += 8;
408 const float16x8_t vi18x89ABCDEF = vld1q_f16(i18); i18 += 8;
409 const float16x8_t vi18xGHIJKLMN = vld1q_f16(i18); i18 += 8;
410 const float16x8_t vi18xOPQRSTUV = vld1q_f16(i18); i18 += 8;
411 const float16x8_t vk18x01234567 = vld1q_f16(w); w += 8;
412 const float16x8_t vk18x89ABCDEF = vld1q_f16(w); w += 8;
413 const float16x8_t vk18xGHIJKLMN = vld1q_f16(w); w += 8;
414 const float16x8_t vk18xOPQRSTUV = vld1q_f16(w); w += 8;
415 vacc01234567p0 = vfmaq_f16(vacc01234567p0, vi18x01234567, vk18x01234567);
416 vacc89ABCDEFp0 = vfmaq_f16(vacc89ABCDEFp0, vi18x89ABCDEF, vk18x89ABCDEF);
417 vaccGHIJKLMNp0 = vfmaq_f16(vaccGHIJKLMNp0, vi18xGHIJKLMN, vk18xGHIJKLMN);
418 vaccOPQRSTUVp0 = vfmaq_f16(vaccOPQRSTUVp0, vi18xOPQRSTUV, vk18xOPQRSTUV);
419
420 const float16x8_t vi19x01234567 = vld1q_f16(i19); i19 += 8;
421 const float16x8_t vi19x89ABCDEF = vld1q_f16(i19); i19 += 8;
422 const float16x8_t vi19xGHIJKLMN = vld1q_f16(i19); i19 += 8;
423 const float16x8_t vi19xOPQRSTUV = vld1q_f16(i19); i19 += 8;
424 const float16x8_t vk19x01234567 = vld1q_f16(w); w += 8;
425 const float16x8_t vk19x89ABCDEF = vld1q_f16(w); w += 8;
426 const float16x8_t vk19xGHIJKLMN = vld1q_f16(w); w += 8;
427 const float16x8_t vk19xOPQRSTUV = vld1q_f16(w); w += 8;
428 vacc01234567p0 = vfmaq_f16(vacc01234567p0, vi19x01234567, vk19x01234567);
429 vacc89ABCDEFp0 = vfmaq_f16(vacc89ABCDEFp0, vi19x89ABCDEF, vk19x89ABCDEF);
430 vaccGHIJKLMNp0 = vfmaq_f16(vaccGHIJKLMNp0, vi19xGHIJKLMN, vk19xGHIJKLMN);
431 vaccOPQRSTUVp0 = vfmaq_f16(vaccOPQRSTUVp0, vi19xOPQRSTUV, vk19xOPQRSTUV);
432
433 const float16x8_t vi20x01234567 = vld1q_f16(i20); i20 += 8;
434 const float16x8_t vi20x89ABCDEF = vld1q_f16(i20); i20 += 8;
435 const float16x8_t vi20xGHIJKLMN = vld1q_f16(i20); i20 += 8;
436 const float16x8_t vi20xOPQRSTUV = vld1q_f16(i20); i20 += 8;
437 const float16x8_t vk20x01234567 = vld1q_f16(w); w += 8;
438 const float16x8_t vk20x89ABCDEF = vld1q_f16(w); w += 8;
439 const float16x8_t vk20xGHIJKLMN = vld1q_f16(w); w += 8;
440 const float16x8_t vk20xOPQRSTUV = vld1q_f16(w); w += 8;
441 vacc01234567p0 = vfmaq_f16(vacc01234567p0, vi20x01234567, vk20x01234567);
442 vacc89ABCDEFp0 = vfmaq_f16(vacc89ABCDEFp0, vi20x89ABCDEF, vk20x89ABCDEF);
443 vaccGHIJKLMNp0 = vfmaq_f16(vaccGHIJKLMNp0, vi20xGHIJKLMN, vk20xGHIJKLMN);
444 vaccOPQRSTUVp0 = vfmaq_f16(vaccOPQRSTUVp0, vi20xOPQRSTUV, vk20xOPQRSTUV);
445
446 const float16x8_t vi21x01234567 = vld1q_f16(i21); i21 += 8;
447 const float16x8_t vi21x89ABCDEF = vld1q_f16(i21); i21 += 8;
448 const float16x8_t vi21xGHIJKLMN = vld1q_f16(i21); i21 += 8;
449 const float16x8_t vi21xOPQRSTUV = vld1q_f16(i21); i21 += 8;
450 const float16x8_t vk21x01234567 = vld1q_f16(w); w += 8;
451 const float16x8_t vk21x89ABCDEF = vld1q_f16(w); w += 8;
452 const float16x8_t vk21xGHIJKLMN = vld1q_f16(w); w += 8;
453 const float16x8_t vk21xOPQRSTUV = vld1q_f16(w); w += 8;
454 vacc01234567p0 = vfmaq_f16(vacc01234567p0, vi21x01234567, vk21x01234567);
455 vacc89ABCDEFp0 = vfmaq_f16(vacc89ABCDEFp0, vi21x89ABCDEF, vk21x89ABCDEF);
456 vaccGHIJKLMNp0 = vfmaq_f16(vaccGHIJKLMNp0, vi21xGHIJKLMN, vk21xGHIJKLMN);
457 vaccOPQRSTUVp0 = vfmaq_f16(vaccOPQRSTUVp0, vi21xOPQRSTUV, vk21xOPQRSTUV);
458
459 const float16x8_t vi22x01234567 = vld1q_f16(i22); i22 += 8;
460 const float16x8_t vi22x89ABCDEF = vld1q_f16(i22); i22 += 8;
461 const float16x8_t vi22xGHIJKLMN = vld1q_f16(i22); i22 += 8;
462 const float16x8_t vi22xOPQRSTUV = vld1q_f16(i22); i22 += 8;
463 const float16x8_t vk22x01234567 = vld1q_f16(w); w += 8;
464 const float16x8_t vk22x89ABCDEF = vld1q_f16(w); w += 8;
465 const float16x8_t vk22xGHIJKLMN = vld1q_f16(w); w += 8;
466 const float16x8_t vk22xOPQRSTUV = vld1q_f16(w); w += 8;
467 vacc01234567p0 = vfmaq_f16(vacc01234567p0, vi22x01234567, vk22x01234567);
468 vacc89ABCDEFp0 = vfmaq_f16(vacc89ABCDEFp0, vi22x89ABCDEF, vk22x89ABCDEF);
469 vaccGHIJKLMNp0 = vfmaq_f16(vaccGHIJKLMNp0, vi22xGHIJKLMN, vk22xGHIJKLMN);
470 vaccOPQRSTUVp0 = vfmaq_f16(vaccOPQRSTUVp0, vi22xOPQRSTUV, vk22xOPQRSTUV);
471
472 const float16x8_t vi23x01234567 = vld1q_f16(i23); i23 += 8;
473 const float16x8_t vi23x89ABCDEF = vld1q_f16(i23); i23 += 8;
474 const float16x8_t vi23xGHIJKLMN = vld1q_f16(i23); i23 += 8;
475 const float16x8_t vi23xOPQRSTUV = vld1q_f16(i23); i23 += 8;
476 const float16x8_t vk23x01234567 = vld1q_f16(w); w += 8;
477 const float16x8_t vk23x89ABCDEF = vld1q_f16(w); w += 8;
478 const float16x8_t vk23xGHIJKLMN = vld1q_f16(w); w += 8;
479 const float16x8_t vk23xOPQRSTUV = vld1q_f16(w); w += 8;
480 vacc01234567p0 = vfmaq_f16(vacc01234567p0, vi23x01234567, vk23x01234567);
481 vacc89ABCDEFp0 = vfmaq_f16(vacc89ABCDEFp0, vi23x89ABCDEF, vk23x89ABCDEF);
482 vaccGHIJKLMNp0 = vfmaq_f16(vaccGHIJKLMNp0, vi23xGHIJKLMN, vk23xGHIJKLMN);
483 vaccOPQRSTUVp0 = vfmaq_f16(vaccOPQRSTUVp0, vi23xOPQRSTUV, vk23xOPQRSTUV);
484
485 const float16x8_t vi24x01234567 = vld1q_f16(i24); i24 += 8;
486 const float16x8_t vi24x89ABCDEF = vld1q_f16(i24); i24 += 8;
487 const float16x8_t vi24xGHIJKLMN = vld1q_f16(i24); i24 += 8;
488 const float16x8_t vi24xOPQRSTUV = vld1q_f16(i24); i24 += 8;
489 const float16x8_t vk24x01234567 = vld1q_f16(w); w += 8;
490 const float16x8_t vk24x89ABCDEF = vld1q_f16(w); w += 8;
491 const float16x8_t vk24xGHIJKLMN = vld1q_f16(w); w += 8;
492 const float16x8_t vk24xOPQRSTUV = vld1q_f16(w); w += 8;
493 vacc01234567p0 = vfmaq_f16(vacc01234567p0, vi24x01234567, vk24x01234567);
494 vacc89ABCDEFp0 = vfmaq_f16(vacc89ABCDEFp0, vi24x89ABCDEF, vk24x89ABCDEF);
495 vaccGHIJKLMNp0 = vfmaq_f16(vaccGHIJKLMNp0, vi24xGHIJKLMN, vk24xGHIJKLMN);
496 vaccOPQRSTUVp0 = vfmaq_f16(vaccOPQRSTUVp0, vi24xOPQRSTUV, vk24xOPQRSTUV);
497
498
499 float16x8_t vacc01234567 = vmaxq_f16(vacc01234567p0, vmin);
500 float16x8_t vacc89ABCDEF = vmaxq_f16(vacc89ABCDEFp0, vmin);
501 float16x8_t vaccGHIJKLMN = vmaxq_f16(vaccGHIJKLMNp0, vmin);
502 float16x8_t vaccOPQRSTUV = vmaxq_f16(vaccOPQRSTUVp0, vmin);
503 vacc01234567 = vminq_f16(vacc01234567, vmax);
504 vacc89ABCDEF = vminq_f16(vacc89ABCDEF, vmax);
505 vaccGHIJKLMN = vminq_f16(vaccGHIJKLMN, vmax);
506 vaccOPQRSTUV = vminq_f16(vaccOPQRSTUV, vmax);
507
508 vst1q_f16(output, vacc01234567); output += 8;
509 vst1q_f16(output, vacc89ABCDEF); output += 8;
510 vst1q_f16(output, vaccGHIJKLMN); output += 8;
511 vst1q_f16(output, vaccOPQRSTUV); output += 8;
512 }
513 for (; c >= 8; c -= 8) {
514 float16x8_t vacc01234567p0 = vld1q_f16(w); w += 8;
515
516
517 const float16x8_t vi0x01234567 = vld1q_f16(i0); i0 += 8;
518 const float16x8_t vk0x01234567 = vld1q_f16(w + 24);
519 vacc01234567p0 = vfmaq_f16(vacc01234567p0, vi0x01234567, vk0x01234567);
520
521 const float16x8_t vi1x01234567 = vld1q_f16(i1); i1 += 8;
522 const float16x8_t vk1x01234567 = vld1q_f16(w + 56);
523 vacc01234567p0 = vfmaq_f16(vacc01234567p0, vi1x01234567, vk1x01234567);
524
525 const float16x8_t vi2x01234567 = vld1q_f16(i2); i2 += 8;
526 const float16x8_t vk2x01234567 = vld1q_f16(w + 88);
527 vacc01234567p0 = vfmaq_f16(vacc01234567p0, vi2x01234567, vk2x01234567);
528
529 const float16x8_t vi3x01234567 = vld1q_f16(i3); i3 += 8;
530 const float16x8_t vk3x01234567 = vld1q_f16(w + 120);
531 vacc01234567p0 = vfmaq_f16(vacc01234567p0, vi3x01234567, vk3x01234567);
532
533 const float16x8_t vi4x01234567 = vld1q_f16(i4); i4 += 8;
534 const float16x8_t vk4x01234567 = vld1q_f16(w + 152);
535 vacc01234567p0 = vfmaq_f16(vacc01234567p0, vi4x01234567, vk4x01234567);
536
537 const float16x8_t vi5x01234567 = vld1q_f16(i5); i5 += 8;
538 const float16x8_t vk5x01234567 = vld1q_f16(w + 184);
539 vacc01234567p0 = vfmaq_f16(vacc01234567p0, vi5x01234567, vk5x01234567);
540
541 const float16x8_t vi6x01234567 = vld1q_f16(i6); i6 += 8;
542 const float16x8_t vk6x01234567 = vld1q_f16(w + 216);
543 vacc01234567p0 = vfmaq_f16(vacc01234567p0, vi6x01234567, vk6x01234567);
544
545 const float16x8_t vi7x01234567 = vld1q_f16(i7); i7 += 8;
546 const float16x8_t vk7x01234567 = vld1q_f16(w + 248);
547 vacc01234567p0 = vfmaq_f16(vacc01234567p0, vi7x01234567, vk7x01234567);
548
549 const float16x8_t vi8x01234567 = vld1q_f16(i8); i8 += 8;
550 const float16x8_t vk8x01234567 = vld1q_f16(w + 280);
551 vacc01234567p0 = vfmaq_f16(vacc01234567p0, vi8x01234567, vk8x01234567);
552
553 const float16x8_t vi9x01234567 = vld1q_f16(i9); i9 += 8;
554 const float16x8_t vk9x01234567 = vld1q_f16(w + 312);
555 vacc01234567p0 = vfmaq_f16(vacc01234567p0, vi9x01234567, vk9x01234567);
556
557 const float16x8_t vi10x01234567 = vld1q_f16(i10); i10 += 8;
558 const float16x8_t vk10x01234567 = vld1q_f16(w + 344);
559 vacc01234567p0 = vfmaq_f16(vacc01234567p0, vi10x01234567, vk10x01234567);
560
561 const float16x8_t vi11x01234567 = vld1q_f16(i11); i11 += 8;
562 const float16x8_t vk11x01234567 = vld1q_f16(w + 376);
563 vacc01234567p0 = vfmaq_f16(vacc01234567p0, vi11x01234567, vk11x01234567);
564
565 const float16x8_t vi12x01234567 = vld1q_f16(i12); i12 += 8;
566 const float16x8_t vk12x01234567 = vld1q_f16(w + 408);
567 vacc01234567p0 = vfmaq_f16(vacc01234567p0, vi12x01234567, vk12x01234567);
568
569 const float16x8_t vi13x01234567 = vld1q_f16(i13); i13 += 8;
570 const float16x8_t vk13x01234567 = vld1q_f16(w + 440);
571 vacc01234567p0 = vfmaq_f16(vacc01234567p0, vi13x01234567, vk13x01234567);
572
573 const float16x8_t vi14x01234567 = vld1q_f16(i14); i14 += 8;
574 const float16x8_t vk14x01234567 = vld1q_f16(w + 472);
575 vacc01234567p0 = vfmaq_f16(vacc01234567p0, vi14x01234567, vk14x01234567);
576
577 const float16x8_t vi15x01234567 = vld1q_f16(i15); i15 += 8;
578 const float16x8_t vk15x01234567 = vld1q_f16(w + 504);
579 vacc01234567p0 = vfmaq_f16(vacc01234567p0, vi15x01234567, vk15x01234567);
580
581 const float16x8_t vi16x01234567 = vld1q_f16(i16); i16 += 8;
582 const float16x8_t vk16x01234567 = vld1q_f16(w + 536);
583 vacc01234567p0 = vfmaq_f16(vacc01234567p0, vi16x01234567, vk16x01234567);
584
585 const float16x8_t vi17x01234567 = vld1q_f16(i17); i17 += 8;
586 const float16x8_t vk17x01234567 = vld1q_f16(w + 568);
587 vacc01234567p0 = vfmaq_f16(vacc01234567p0, vi17x01234567, vk17x01234567);
588
589 const float16x8_t vi18x01234567 = vld1q_f16(i18); i18 += 8;
590 const float16x8_t vk18x01234567 = vld1q_f16(w + 600);
591 vacc01234567p0 = vfmaq_f16(vacc01234567p0, vi18x01234567, vk18x01234567);
592
593 const float16x8_t vi19x01234567 = vld1q_f16(i19); i19 += 8;
594 const float16x8_t vk19x01234567 = vld1q_f16(w + 632);
595 vacc01234567p0 = vfmaq_f16(vacc01234567p0, vi19x01234567, vk19x01234567);
596
597 const float16x8_t vi20x01234567 = vld1q_f16(i20); i20 += 8;
598 const float16x8_t vk20x01234567 = vld1q_f16(w + 664);
599 vacc01234567p0 = vfmaq_f16(vacc01234567p0, vi20x01234567, vk20x01234567);
600
601 const float16x8_t vi21x01234567 = vld1q_f16(i21); i21 += 8;
602 const float16x8_t vk21x01234567 = vld1q_f16(w + 696);
603 vacc01234567p0 = vfmaq_f16(vacc01234567p0, vi21x01234567, vk21x01234567);
604
605 const float16x8_t vi22x01234567 = vld1q_f16(i22); i22 += 8;
606 const float16x8_t vk22x01234567 = vld1q_f16(w + 728);
607 vacc01234567p0 = vfmaq_f16(vacc01234567p0, vi22x01234567, vk22x01234567);
608
609 const float16x8_t vi23x01234567 = vld1q_f16(i23); i23 += 8;
610 const float16x8_t vk23x01234567 = vld1q_f16(w + 760);
611 vacc01234567p0 = vfmaq_f16(vacc01234567p0, vi23x01234567, vk23x01234567);
612
613 const float16x8_t vi24x01234567 = vld1q_f16(i24); i24 += 8;
614 const float16x8_t vk24x01234567 = vld1q_f16(w + 792);
615 vacc01234567p0 = vfmaq_f16(vacc01234567p0, vi24x01234567, vk24x01234567);
616
617
618 float16x8_t vacc01234567 = vmaxq_f16(vacc01234567p0, vmin);
619 vacc01234567 = vminq_f16(vacc01234567, vmax);
620
621 vst1q_f16(output, vacc01234567); output += 8;
622 }
623 if XNN_UNLIKELY(c != 0) {
624 float16x8_t vacc01234567p0 = vld1q_f16(w);
625
626
627 const float16x8_t vi0x01234567 = vld1q_f16(i0);
628 const float16x8_t vk0x01234567 = vld1q_f16(w + 32);
629 vacc01234567p0 = vfmaq_f16(vacc01234567p0, vi0x01234567, vk0x01234567);
630
631 const float16x8_t vi1x01234567 = vld1q_f16(i1);
632 const float16x8_t vk1x01234567 = vld1q_f16(w + 64);
633 vacc01234567p0 = vfmaq_f16(vacc01234567p0, vi1x01234567, vk1x01234567);
634
635 const float16x8_t vi2x01234567 = vld1q_f16(i2);
636 const float16x8_t vk2x01234567 = vld1q_f16(w + 96);
637 vacc01234567p0 = vfmaq_f16(vacc01234567p0, vi2x01234567, vk2x01234567);
638
639 const float16x8_t vi3x01234567 = vld1q_f16(i3);
640 const float16x8_t vk3x01234567 = vld1q_f16(w + 128);
641 vacc01234567p0 = vfmaq_f16(vacc01234567p0, vi3x01234567, vk3x01234567);
642
643 const float16x8_t vi4x01234567 = vld1q_f16(i4);
644 const float16x8_t vk4x01234567 = vld1q_f16(w + 160);
645 vacc01234567p0 = vfmaq_f16(vacc01234567p0, vi4x01234567, vk4x01234567);
646
647 const float16x8_t vi5x01234567 = vld1q_f16(i5);
648 const float16x8_t vk5x01234567 = vld1q_f16(w + 192);
649 vacc01234567p0 = vfmaq_f16(vacc01234567p0, vi5x01234567, vk5x01234567);
650
651 const float16x8_t vi6x01234567 = vld1q_f16(i6);
652 const float16x8_t vk6x01234567 = vld1q_f16(w + 224);
653 vacc01234567p0 = vfmaq_f16(vacc01234567p0, vi6x01234567, vk6x01234567);
654
655 const float16x8_t vi7x01234567 = vld1q_f16(i7);
656 const float16x8_t vk7x01234567 = vld1q_f16(w + 256);
657 vacc01234567p0 = vfmaq_f16(vacc01234567p0, vi7x01234567, vk7x01234567);
658
659 const float16x8_t vi8x01234567 = vld1q_f16(i8);
660 const float16x8_t vk8x01234567 = vld1q_f16(w + 288);
661 vacc01234567p0 = vfmaq_f16(vacc01234567p0, vi8x01234567, vk8x01234567);
662
663 const float16x8_t vi9x01234567 = vld1q_f16(i9);
664 const float16x8_t vk9x01234567 = vld1q_f16(w + 320);
665 vacc01234567p0 = vfmaq_f16(vacc01234567p0, vi9x01234567, vk9x01234567);
666
667 const float16x8_t vi10x01234567 = vld1q_f16(i10);
668 const float16x8_t vk10x01234567 = vld1q_f16(w + 352);
669 vacc01234567p0 = vfmaq_f16(vacc01234567p0, vi10x01234567, vk10x01234567);
670
671 const float16x8_t vi11x01234567 = vld1q_f16(i11);
672 const float16x8_t vk11x01234567 = vld1q_f16(w + 384);
673 vacc01234567p0 = vfmaq_f16(vacc01234567p0, vi11x01234567, vk11x01234567);
674
675 const float16x8_t vi12x01234567 = vld1q_f16(i12);
676 const float16x8_t vk12x01234567 = vld1q_f16(w + 416);
677 vacc01234567p0 = vfmaq_f16(vacc01234567p0, vi12x01234567, vk12x01234567);
678
679 const float16x8_t vi13x01234567 = vld1q_f16(i13);
680 const float16x8_t vk13x01234567 = vld1q_f16(w + 448);
681 vacc01234567p0 = vfmaq_f16(vacc01234567p0, vi13x01234567, vk13x01234567);
682
683 const float16x8_t vi14x01234567 = vld1q_f16(i14);
684 const float16x8_t vk14x01234567 = vld1q_f16(w + 480);
685 vacc01234567p0 = vfmaq_f16(vacc01234567p0, vi14x01234567, vk14x01234567);
686
687 const float16x8_t vi15x01234567 = vld1q_f16(i15);
688 const float16x8_t vk15x01234567 = vld1q_f16(w + 512);
689 vacc01234567p0 = vfmaq_f16(vacc01234567p0, vi15x01234567, vk15x01234567);
690
691 const float16x8_t vi16x01234567 = vld1q_f16(i16);
692 const float16x8_t vk16x01234567 = vld1q_f16(w + 544);
693 vacc01234567p0 = vfmaq_f16(vacc01234567p0, vi16x01234567, vk16x01234567);
694
695 const float16x8_t vi17x01234567 = vld1q_f16(i17);
696 const float16x8_t vk17x01234567 = vld1q_f16(w + 576);
697 vacc01234567p0 = vfmaq_f16(vacc01234567p0, vi17x01234567, vk17x01234567);
698
699 const float16x8_t vi18x01234567 = vld1q_f16(i18);
700 const float16x8_t vk18x01234567 = vld1q_f16(w + 608);
701 vacc01234567p0 = vfmaq_f16(vacc01234567p0, vi18x01234567, vk18x01234567);
702
703 const float16x8_t vi19x01234567 = vld1q_f16(i19);
704 const float16x8_t vk19x01234567 = vld1q_f16(w + 640);
705 vacc01234567p0 = vfmaq_f16(vacc01234567p0, vi19x01234567, vk19x01234567);
706
707 const float16x8_t vi20x01234567 = vld1q_f16(i20);
708 const float16x8_t vk20x01234567 = vld1q_f16(w + 672);
709 vacc01234567p0 = vfmaq_f16(vacc01234567p0, vi20x01234567, vk20x01234567);
710
711 const float16x8_t vi21x01234567 = vld1q_f16(i21);
712 const float16x8_t vk21x01234567 = vld1q_f16(w + 704);
713 vacc01234567p0 = vfmaq_f16(vacc01234567p0, vi21x01234567, vk21x01234567);
714
715 const float16x8_t vi22x01234567 = vld1q_f16(i22);
716 const float16x8_t vk22x01234567 = vld1q_f16(w + 736);
717 vacc01234567p0 = vfmaq_f16(vacc01234567p0, vi22x01234567, vk22x01234567);
718
719 const float16x8_t vi23x01234567 = vld1q_f16(i23);
720 const float16x8_t vk23x01234567 = vld1q_f16(w + 768);
721 vacc01234567p0 = vfmaq_f16(vacc01234567p0, vi23x01234567, vk23x01234567);
722
723 const float16x8_t vi24x01234567 = vld1q_f16(i24);
724 const float16x8_t vk24x01234567 = vld1q_f16(w + 800);
725 vacc01234567p0 = vfmaq_f16(vacc01234567p0, vi24x01234567, vk24x01234567);
726
727
728 float16x8_t vacc01234567 = vmaxq_f16(vacc01234567p0, vmin);
729 vacc01234567 = vminq_f16(vacc01234567, vmax);
730
731 float16x4_t vacc0123 = vget_low_f16(vacc01234567);
732 if (c & 4) {
733 vst1_f16(output, vacc0123); output += 4;
734 vacc0123 = vget_high_f16(vacc01234567);
735 }
736 if (c & 2) {
737 vst1_lane_u32((void*) output, vreinterpret_u32_f16(vacc0123), 0); output += 2;
738 vacc0123 = vext_f16(vacc0123, vacc0123, 2);
739 }
740 if (c & 1) {
741 vst1_lane_f16(output, vacc0123, 0); output += 1;
742 }
743 }
744
745 output = (__fp16*) ((uintptr_t) output + output_increment);
746 } while (--output_width != 0);
747 }
748