1 // Auto-generated file. Do not edit!
2 // Template: src/f16-spmm/neonfp16arith.c.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2019 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10 #include <assert.h>
11
12 #include <arm_neon.h>
13
14 #include <xnnpack/spmm.h>
15
16
xnn_f16_spmm_ukernel_16x1__neonfp16arith_unroll2(uint32_t m,uint32_t n,const void * restrict input,const void * restrict weights,const int32_t * restrict widx_dmap,const uint32_t * restrict nidx_nnzmap,void * restrict output,const struct xnn_f16_output_params params[restrict static1])17 void xnn_f16_spmm_ukernel_16x1__neonfp16arith_unroll2(
18 uint32_t m,
19 uint32_t n,
20 const void*restrict input,
21 const void*restrict weights,
22 const int32_t*restrict widx_dmap,
23 const uint32_t*restrict nidx_nnzmap,
24 void*restrict output,
25 const struct xnn_f16_output_params params[restrict static 1])
26 {
27 assert(m != 0);
28
29 const __fp16*restrict a = input;
30 __fp16*restrict c = output;
31
32 const float16x8_t vscale = vld1q_dup_f16((const __fp16*) ¶ms->scale);
33 const float16x8_t vmax = vld1q_dup_f16((const __fp16*) ¶ms->max);
34 const float16x8_t vmin = vld1q_dup_f16((const __fp16*) ¶ms->min);
35
36 size_t i = m;
37 while XNN_LIKELY(i >= 16) {
38 const __fp16*restrict w = weights;
39 const int32_t* dmap = widx_dmap;
40 const uint32_t* nnzmap = nidx_nnzmap;
41 size_t j = n;
42 do {
43 uint32_t nnz = *nnzmap++;
44 float16x8_t vacc01234567x0 = vld1q_dup_f16(w); w += 1;
45 float16x8_t vacc01234567x1 = vmovq_n_f16(0.0f);
46 float16x8_t vacc89ABCDEFx0 = vacc01234567x0;
47 float16x8_t vacc89ABCDEFx1 = vmovq_n_f16(0.0f);
48 for (; nnz >= 2; nnz -= 2) {
49 const intptr_t diff0 = dmap[0];
50 const intptr_t diff1 = dmap[1];
51 dmap += 2;
52 const float16x8_t va01234567x0 = vld1q_f16(a);
53 const float16x8_t va89ABCDEFx0 = vld1q_f16(a + 8);
54 a = (const __fp16*restrict) ((uintptr_t) a + (uintptr_t) diff0);
55 const float16x8_t vb0 = vld1q_dup_f16(w); w += 1;
56 vacc01234567x0 = vfmaq_f16(vacc01234567x0, va01234567x0, vb0);
57 vacc89ABCDEFx0 = vfmaq_f16(vacc89ABCDEFx0, va89ABCDEFx0, vb0);
58 const float16x8_t va01234567x1 = vld1q_f16(a);
59 const float16x8_t va89ABCDEFx1 = vld1q_f16(a + 8);
60 a = (const __fp16*restrict) ((uintptr_t) a + (uintptr_t) diff1);
61 const float16x8_t vb1 = vld1q_dup_f16(w); w += 1;
62 vacc01234567x1 = vfmaq_f16(vacc01234567x1, va01234567x1, vb1);
63 vacc89ABCDEFx1 = vfmaq_f16(vacc89ABCDEFx1, va89ABCDEFx1, vb1);
64 }
65 float16x8_t vacc01234567 = vacc01234567x0;
66 float16x8_t vacc89ABCDEF = vacc89ABCDEFx0;
67 vacc01234567 = vaddq_f16(vacc01234567, vacc01234567x1);
68 vacc89ABCDEF = vaddq_f16(vacc89ABCDEF, vacc89ABCDEFx1);
69 if XNN_LIKELY(nnz != 0) {
70 do {
71 const intptr_t diff = *dmap++;
72 const float16x8_t va01234567 = vld1q_f16(a);
73 const float16x8_t va89ABCDEF = vld1q_f16(a + 8);
74 a = (const __fp16*restrict) ((uintptr_t) a + (uintptr_t) diff);
75 const float16x8_t vb = vld1q_dup_f16(w); w += 1;
76 vacc01234567 = vfmaq_f16(vacc01234567, va01234567, vb);
77 vacc89ABCDEF = vfmaq_f16(vacc89ABCDEF, va89ABCDEF, vb);
78 } while (--nnz != 0);
79 }
80 float16x8_t vout01234567 = vmulq_f16(vacc01234567, vscale);
81 float16x8_t vout89ABCDEF = vmulq_f16(vacc89ABCDEF, vscale);
82 vout01234567 = vminq_f16(vout01234567, vmax);
83 vout89ABCDEF = vminq_f16(vout89ABCDEF, vmax);
84 vout01234567 = vmaxq_f16(vout01234567, vmin);
85 vout89ABCDEF = vmaxq_f16(vout89ABCDEF, vmin);
86 vst1q_f16(c, vout01234567);
87 vst1q_f16(c + 8, vout89ABCDEF);
88 c += m;
89 } while (--j != 0);
90 c -= m * n;
91 c += 16;
92 a += 16;
93 i -= 16;
94 }
95 if XNN_UNLIKELY(i != 0) {
96 if (i & 8) {
97 const __fp16*restrict w = weights;
98 const int32_t* dmap = widx_dmap;
99 const uint32_t* nnzmap = nidx_nnzmap;
100 size_t j = n;
101 do {
102 uint32_t nnz = *nnzmap++;
103 float16x8_t vacc01234567 = vld1q_dup_f16(w); w += 1;
104 if XNN_LIKELY(nnz != 0) {
105 do {
106 const intptr_t diff = *dmap++;
107 const float16x8_t va01234567 = vld1q_f16(a);
108 a = (const __fp16*restrict) ((uintptr_t) a + (uintptr_t) diff);
109 const float16x8_t vb = vld1q_dup_f16(w); w += 1;
110 vacc01234567 = vfmaq_f16(vacc01234567, va01234567, vb);
111 } while (--nnz != 0);
112 }
113 float16x8_t vout01234567 = vminq_f16(vacc01234567, vmax);
114 vout01234567 = vmaxq_f16(vout01234567, vmin);
115 vst1q_f16(c, vout01234567);
116 c += m;
117 } while (--j != 0);
118 c -= m * n;
119 c += 8;
120 a += 8;
121 }
122 if (i & 4) {
123 const __fp16*restrict w = weights;
124 const int32_t* dmap = widx_dmap;
125 const uint32_t* nnzmap = nidx_nnzmap;
126 size_t j = n;
127 do {
128 uint32_t nnz = *nnzmap++;
129 float16x4_t vacc0123 = vld1_dup_f16(w); w += 1;
130 if XNN_LIKELY(nnz != 0) {
131 do {
132 const intptr_t diff = *dmap++;
133 const float16x4_t va0123 = vld1_f16(a);
134 a = (const __fp16*restrict) ((uintptr_t) a + (uintptr_t) diff);
135 const float16x4_t vb = vld1_dup_f16(w); w += 1;
136 vacc0123 = vfma_f16(vacc0123, va0123, vb);
137 } while (--nnz != 0);
138 }
139 float16x4_t vout0123 = vmin_f16(vacc0123, vget_low_f16(vmax));
140 vout0123 = vmax_f16(vout0123, vget_low_f16(vmin));
141 vst1_f16(c, vout0123);
142 c += m;
143 } while (--j != 0);
144 c -= m * n;
145 c += 4;
146 a += 4;
147 }
148 if (i & 2) {
149 const __fp16*restrict w = weights;
150 const int32_t* dmap = widx_dmap;
151 const uint32_t* nnzmap = nidx_nnzmap;
152 size_t j = n;
153 do {
154 uint32_t nnz = *nnzmap++;
155 float16x4_t vacc01 = vld1_dup_f16(w); w += 1;
156 if XNN_LIKELY(nnz != 0) {
157 do {
158 const intptr_t diff = *dmap++;
159 const float16x4_t va01 = vreinterpret_f32_f16(vld1_dup_f32(__builtin_assume_aligned(a, 1)));
160 a = (const __fp16*restrict) ((uintptr_t) a + (uintptr_t) diff);
161 const float16x4_t vb = vld1_dup_f16(w); w += 1;
162 vacc01 = vfma_f16(vacc01, va01, vb);
163 } while (--nnz != 0);
164 }
165 float16x4_t vout01 = vmin_f16(vacc01, vget_low_f16(vmax));
166 vout01 = vmax_f16(vout01, vget_low_f16(vmin));
167 vst1_lane_f32(__builtin_assume_aligned(c, 1), vreinterpret_f16_f32(vout01), 0);
168 c += m;
169 } while (--j != 0);
170 c -= m * n;
171 c += 2;
172 a += 2;
173 }
174 if (i & 1) {
175 const __fp16*restrict w = weights;
176 const int32_t* dmap = widx_dmap;
177 const uint32_t* nnzmap = nidx_nnzmap;
178 size_t j = n;
179 do {
180 uint32_t nnz = *nnzmap++;
181 float16x4_t vacc0 = vld1_dup_f16(w); w += 1;
182 if XNN_LIKELY(nnz != 0) {
183 do {
184 const intptr_t diff = *dmap++;
185 const float16x4_t va0 = vld1_dup_f16(a);
186 a = (const __fp16*restrict) ((uintptr_t) a + (uintptr_t) diff);
187 const float16x4_t vb = vld1_dup_f16(w); w += 1;
188 vacc0 = vfma_f16(vacc0, va0, vb);
189 } while (--nnz != 0);
190 }
191 float16x4_t vout0 = vmin_f16(vacc0, vget_low_f16(vmax));
192 vout0 = vmax_f16(vout0, vget_low_f16(vmin));
193 vst1_lane_f16(c, vout0, 0);
194 c += m;
195 } while (--j != 0);
196 c -= m * n;
197 c += 1;
198 a += 1;
199 }
200 }
201 }
202