1 // Auto-generated file. Do not edit!
2 // Template: src/f32-spmm/neon.c.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2019 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10 #include <assert.h>
11
12 #include <arm_neon.h>
13
14 #include <xnnpack/spmm.h>
15
16
xnn_f32_spmm_ukernel_8x1__neonfma_unroll2(uint32_t m,uint32_t n,const float * restrict a,const float * restrict weights,const int32_t * restrict widx_dmap,const uint32_t * restrict nidx_nnzmap,float * restrict c,const union xnn_f32_output_params params[restrict static1])17 void xnn_f32_spmm_ukernel_8x1__neonfma_unroll2(
18 uint32_t m,
19 uint32_t n,
20 const float*restrict a,
21 const float*restrict weights,
22 const int32_t*restrict widx_dmap,
23 const uint32_t*restrict nidx_nnzmap,
24 float*restrict c,
25 const union xnn_f32_output_params params[restrict static 1])
26 {
27 assert(m != 0);
28
29 const float32x4_t vmin = vld1q_dup_f32(¶ms->scalar.min);
30 const float32x4_t vmax = vld1q_dup_f32(¶ms->scalar.max);
31 size_t i = m;
32 while XNN_LIKELY(i >= 8) {
33 const float*restrict w = weights;
34 const int32_t* dmap = widx_dmap;
35 const uint32_t* nnzmap = nidx_nnzmap;
36 size_t j = n;
37 do {
38 uint32_t nnz = *nnzmap++;
39 float32x4_t vacc0123x0 = vld1q_dup_f32(w); w += 1;
40 float32x4_t vacc0123x1 = vmovq_n_f32(0.0f);
41 float32x4_t vacc4567x0 = vacc0123x0;
42 float32x4_t vacc4567x1 = vmovq_n_f32(0.0f);
43 for (; nnz >= 2; nnz -= 2) {
44 const intptr_t diff0 = dmap[0];
45 const intptr_t diff1 = dmap[1];
46 dmap += 2;
47 const float32x4_t va0123x0 = vld1q_f32(a);
48 const float32x4_t va4567x0 = vld1q_f32(a + 4);
49 a = (const float*restrict) ((uintptr_t) a + (uintptr_t) diff0);
50 const float32x4_t vb0 = vld1q_dup_f32(w); w += 1;
51 vacc0123x0 = vfmaq_f32(vacc0123x0, va0123x0, vb0);
52 vacc4567x0 = vfmaq_f32(vacc4567x0, va4567x0, vb0);
53 const float32x4_t va0123x1 = vld1q_f32(a);
54 const float32x4_t va4567x1 = vld1q_f32(a + 4);
55 a = (const float*restrict) ((uintptr_t) a + (uintptr_t) diff1);
56 const float32x4_t vb1 = vld1q_dup_f32(w); w += 1;
57 vacc0123x1 = vfmaq_f32(vacc0123x1, va0123x1, vb1);
58 vacc4567x1 = vfmaq_f32(vacc4567x1, va4567x1, vb1);
59 }
60 float32x4_t vacc0123 = vacc0123x0;
61 float32x4_t vacc4567 = vacc4567x0;
62 vacc0123 = vaddq_f32(vacc0123, vacc0123x1);
63 vacc4567 = vaddq_f32(vacc4567, vacc4567x1);
64 if XNN_LIKELY(nnz != 0) {
65 do {
66 const intptr_t diff = *dmap++;
67 const float32x4_t va0123 = vld1q_f32(a);
68 const float32x4_t va4567 = vld1q_f32(a + 4);
69 a = (const float*restrict) ((uintptr_t) a + (uintptr_t) diff);
70 const float32x4_t vb = vld1q_dup_f32(w); w += 1;
71 vacc0123 = vfmaq_f32(vacc0123, va0123, vb);
72 vacc4567 = vfmaq_f32(vacc4567, va4567, vb);
73 } while (--nnz != 0);
74 }
75 float32x4_t vout0123 = vminq_f32(vacc0123, vmax);
76 float32x4_t vout4567 = vminq_f32(vacc4567, vmax);
77 vout0123 = vmaxq_f32(vout0123, vmin);
78 vout4567 = vmaxq_f32(vout4567, vmin);
79 vst1q_f32(c, vout0123);
80 vst1q_f32(c + 4, vout4567);
81 c += m;
82 } while (--j != 0);
83 c -= m * n;
84 c += 8;
85 a += 8;
86 i -= 8;
87 }
88 if XNN_UNLIKELY(i != 0) {
89 if (i & 4) {
90 const float*restrict w = weights;
91 const int32_t* dmap = widx_dmap;
92 const uint32_t* nnzmap = nidx_nnzmap;
93 size_t j = n;
94 do {
95 uint32_t nnz = *nnzmap++;
96 float32x4_t vacc0123 = vld1q_dup_f32(w); w += 1;
97 if XNN_LIKELY(nnz != 0) {
98 do {
99 const intptr_t diff = *dmap++;
100 const float32x4_t va0123 = vld1q_f32(a);
101 a = (const float*restrict) ((uintptr_t) a + (uintptr_t) diff);
102 const float32x4_t vb = vld1q_dup_f32(w); w += 1;
103 vacc0123 = vfmaq_f32(vacc0123, va0123, vb);
104 } while (--nnz != 0);
105 }
106 float32x4_t vout0123 = vminq_f32(vacc0123, vmax);
107 vout0123 = vmaxq_f32(vout0123, vmin);
108 vst1q_f32(c, vout0123);
109 c += m;
110 } while (--j != 0);
111 c -= m * n;
112 c += 4;
113 a += 4;
114 }
115 if (i & 2) {
116 const float*restrict w = weights;
117 const int32_t* dmap = widx_dmap;
118 const uint32_t* nnzmap = nidx_nnzmap;
119 size_t j = n;
120 do {
121 uint32_t nnz = *nnzmap++;
122 float32x2_t vacc01 = vld1_dup_f32(w); w += 1;
123 if XNN_LIKELY(nnz != 0) {
124 do {
125 const intptr_t diff = *dmap++;
126 const float32x2_t va01 = vld1_f32(a);
127 a = (const float*restrict) ((uintptr_t) a + (uintptr_t) diff);
128 const float32x2_t vb = vld1_dup_f32(w); w += 1;
129 vacc01 = vfma_f32(vacc01, va01, vb);
130 } while (--nnz != 0);
131 }
132 float32x2_t vout01 = vmin_f32(vacc01, vget_low_f32(vmax));
133 vout01 = vmax_f32(vout01, vget_low_f32(vmin));
134 vst1_f32(c, vout01);
135 c += m;
136 } while (--j != 0);
137 c -= m * n;
138 c += 2;
139 a += 2;
140 }
141 if (i & 1) {
142 const float*restrict w = weights;
143 const int32_t* dmap = widx_dmap;
144 const uint32_t* nnzmap = nidx_nnzmap;
145 size_t j = n;
146 do {
147 uint32_t nnz = *nnzmap++;
148 float32x2_t vacc0 = vld1_dup_f32(w); w += 1;
149 if XNN_LIKELY(nnz != 0) {
150 do {
151 const intptr_t diff = *dmap++;
152 const float32x2_t va0 = vld1_dup_f32(a);
153 a = (const float*restrict) ((uintptr_t) a + (uintptr_t) diff);
154 const float32x2_t vb = vld1_dup_f32(w); w += 1;
155 vacc0 = vfma_f32(vacc0, va0, vb);
156 } while (--nnz != 0);
157 }
158 float32x2_t vout0 = vmin_f32(vacc0, vget_low_f32(vmax));
159 vout0 = vmax_f32(vout0, vget_low_f32(vmin));
160 vst1_lane_f32(c, vout0, 0);
161 c += m;
162 } while (--j != 0);
163 c -= m * n;
164 c += 1;
165 a += 1;
166 }
167 }
168 }
169