1 // Copyright 2019 Google LLC
2 //
3 // This source code is licensed under the BSD-style license found in the
4 // LICENSE file in the root directory of this source tree.
5
6 #include <assert.h>
7
8 #include <psimd.h>
9
10 #include <xnnpack/avgpool.h>
11
12
xnn_f32_avgpool_ukernel_mp9p8q__psimd(size_t n,size_t ks,size_t kc,const float ** input,const float * zero,float * buffer,float * output,size_t input_increment,size_t output_increment,const union xnn_f32_avgpool_params params[restrict static1])13 void xnn_f32_avgpool_ukernel_mp9p8q__psimd(
14 size_t n,
15 size_t ks,
16 size_t kc,
17 const float** input,
18 const float* zero,
19 float* buffer,
20 float* output,
21 size_t input_increment,
22 size_t output_increment,
23 const union xnn_f32_avgpool_params params[restrict static 1])
24 {
25 assert(n != 0);
26 assert(ks > 9);
27 assert(kc != 0);
28
29 const psimd_f32 vmultiplier = psimd_load_splat_f32(¶ms->scalar.multiplier);
30 const psimd_f32 voutput_min = psimd_load_splat_f32(¶ms->scalar.output_min);
31 const psimd_f32 voutput_max = psimd_load_splat_f32(¶ms->scalar.output_max);
32
33 do {
34 {
35 const float* i0 = *input++;
36 const float* i1 = *input++;
37 const float* i2 = *input++;
38 const float* i3 = *input++;
39 const float* i4 = *input++;
40 const float* i5 = *input++;
41 const float* i6 = *input++;
42 const float* i7 = *input++;
43 const float* i8 = *input++;
44
45 float* b = buffer;
46 for (size_t k = 0; k < kc; k += 4) {
47 const psimd_f32 vi0 = psimd_load_f32(i0);
48 i0 += 4;
49 const psimd_f32 vi1 = psimd_load_f32(i1);
50 i1 += 4;
51 const psimd_f32 vi2 = psimd_load_f32(i2);
52 i2 += 4;
53 const psimd_f32 vi3 = psimd_load_f32(i3);
54 i3 += 4;
55 const psimd_f32 vi4 = psimd_load_f32(i4);
56 i4 += 4;
57 const psimd_f32 vi5 = psimd_load_f32(i5);
58 i5 += 4;
59 const psimd_f32 vi6 = psimd_load_f32(i6);
60 i6 += 4;
61 const psimd_f32 vi7 = psimd_load_f32(i7);
62 i7 += 4;
63 const psimd_f32 vi8 = psimd_load_f32(i8);
64 i8 += 4;
65
66 const psimd_f32 vsum01 = psimd_add_f32(vi0, vi1);
67 const psimd_f32 vsum23 = psimd_add_f32(vi2, vi3);
68 const psimd_f32 vsum45 = psimd_add_f32(vi4, vi5);
69 const psimd_f32 vsum67 = psimd_add_f32(vi6, vi7);
70 const psimd_f32 vsum018 = psimd_add_f32(vsum01, vi8);
71 const psimd_f32 vsum2345 = psimd_add_f32(vsum23, vsum45);
72 const psimd_f32 vsum01678 = psimd_add_f32(vsum018, vsum67);
73 const psimd_f32 vsum = psimd_add_f32(vsum2345, vsum01678);
74
75 psimd_store_f32(b, vsum);
76 b += 4;
77 }
78 }
79
80 size_t m = ks;
81 for (m -= 9; m > 8; m -= 8) {
82 const float* i0 = *input++;
83 const float* i1 = *input++;
84 const float* i2 = *input++;
85 const float* i3 = *input++;
86 const float* i4 = *input++;
87 const float* i5 = *input++;
88 const float* i6 = *input++;
89 const float* i7 = *input++;
90
91 float* b = buffer;
92 for (size_t k = 0; k < kc; k += 4) {
93 const psimd_f32 vi0 = psimd_load_f32(i0);
94 i0 += 4;
95 const psimd_f32 vi1 = psimd_load_f32(i1);
96 i1 += 4;
97 const psimd_f32 vi2 = psimd_load_f32(i2);
98 i2 += 4;
99 const psimd_f32 vi3 = psimd_load_f32(i3);
100 i3 += 4;
101 const psimd_f32 vi4 = psimd_load_f32(i4);
102 i4 += 4;
103 const psimd_f32 vi5 = psimd_load_f32(i5);
104 i5 += 4;
105 const psimd_f32 vi6 = psimd_load_f32(i6);
106 i6 += 4;
107 const psimd_f32 vi7 = psimd_load_f32(i7);
108 i7 += 4;
109 const psimd_f32 vacc = psimd_load_f32(b);
110
111 const psimd_f32 vsum01 = psimd_add_f32(vi0, vi1);
112 const psimd_f32 vsum23 = psimd_add_f32(vi2, vi3);
113 const psimd_f32 vsum45 = psimd_add_f32(vi4, vi5);
114 const psimd_f32 vsum67 = psimd_add_f32(vi6, vi7);
115 const psimd_f32 vsum01a = psimd_add_f32(vsum01, vacc);
116 const psimd_f32 vsum2345 = psimd_add_f32(vsum23, vsum45);
117 const psimd_f32 vsum0167a = psimd_add_f32(vsum01a, vsum67);
118 const psimd_f32 vsum = psimd_add_f32(vsum2345, vsum0167a);
119
120 psimd_store_f32(b, vsum);
121 b += 4;
122 }
123 }
124
125 {
126 const float* i0 = input[0];
127 const float* i1 = input[1];
128 const float* i2 = input[2];
129 const float* i3 = input[3];
130 const float* i4 = input[4];
131 const float* i5 = input[5];
132 const float* i6 = input[6];
133 const float* i7 = input[7];
134 input = (const float**) ((uintptr_t) input + input_increment);
135 if (m < 2) {
136 i1 = zero;
137 }
138 if (m <= 2) {
139 i2 = zero;
140 }
141 if (m < 4) {
142 i3 = zero;
143 }
144 if (m <= 4) {
145 i4 = zero;
146 }
147 if (m < 6) {
148 i5 = zero;
149 }
150 if (m <= 6) {
151 i6 = zero;
152 }
153 if (m != 8) {
154 i7 = zero;
155 }
156
157 size_t k = kc;
158 float* b = buffer;
159 while (k >= 4) {
160 const psimd_f32 vi0 = psimd_load_f32(i0);
161 i0 += 4;
162 const psimd_f32 vi1 = psimd_load_f32(i1);
163 i1 += 4;
164 const psimd_f32 vi2 = psimd_load_f32(i2);
165 i2 += 4;
166 const psimd_f32 vi3 = psimd_load_f32(i3);
167 i3 += 4;
168 const psimd_f32 vi4 = psimd_load_f32(i4);
169 i4 += 4;
170 const psimd_f32 vi5 = psimd_load_f32(i5);
171 i5 += 4;
172 const psimd_f32 vi6 = psimd_load_f32(i6);
173 i6 += 4;
174 const psimd_f32 vi7 = psimd_load_f32(i7);
175 i7 += 4;
176 const psimd_f32 vacc = psimd_load_f32(b);
177 b += 4;
178
179 const psimd_f32 vsum01 = psimd_add_f32(vi0, vi1);
180 const psimd_f32 vsum23 = psimd_add_f32(vi2, vi3);
181 const psimd_f32 vsum45 = psimd_add_f32(vi4, vi5);
182 const psimd_f32 vsum67 = psimd_add_f32(vi6, vi7);
183 const psimd_f32 vsum01a = psimd_add_f32(vsum01, vacc);
184 const psimd_f32 vsum2345 = psimd_add_f32(vsum23, vsum45);
185 const psimd_f32 vsum0167a = psimd_add_f32(vsum01a, vsum67);
186 const psimd_f32 vsum = psimd_add_f32(vsum2345, vsum0167a);
187
188 psimd_f32 vout = psimd_mul_f32(vsum, vmultiplier);
189 vout = psimd_max_f32(vout, voutput_min);
190 vout = psimd_min_f32(vout, voutput_max);
191
192 psimd_store_f32(output, vout);
193 output += 4;
194
195 k -= 4;
196 }
197 if (k != 0) {
198 const psimd_f32 vi0 = psimd_load_f32(i0);
199 const psimd_f32 vi1 = psimd_load_f32(i1);
200 const psimd_f32 vi2 = psimd_load_f32(i2);
201 const psimd_f32 vi3 = psimd_load_f32(i3);
202 const psimd_f32 vi4 = psimd_load_f32(i4);
203 const psimd_f32 vi5 = psimd_load_f32(i5);
204 const psimd_f32 vi6 = psimd_load_f32(i6);
205 const psimd_f32 vi7 = psimd_load_f32(i7);
206 const psimd_f32 vacc = psimd_load_f32(b);
207
208 const psimd_f32 vsum01 = psimd_add_f32(vi0, vi1);
209 const psimd_f32 vsum23 = psimd_add_f32(vi2, vi3);
210 const psimd_f32 vsum45 = psimd_add_f32(vi4, vi5);
211 const psimd_f32 vsum67 = psimd_add_f32(vi6, vi7);
212 const psimd_f32 vsum01a = psimd_add_f32(vsum01, vacc);
213 const psimd_f32 vsum2345 = psimd_add_f32(vsum23, vsum45);
214 const psimd_f32 vsum0167a = psimd_add_f32(vsum01a, vsum67);
215 const psimd_f32 vsum = psimd_add_f32(vsum2345, vsum0167a);
216
217 psimd_f32 vout = psimd_mul_f32(vsum, vmultiplier);
218 vout = psimd_max_f32(vout, voutput_min);
219 vout = psimd_min_f32(vout, voutput_max);
220
221 if (k & 2) {
222 psimd_store2_f32(output, vout);
223 output += 2;
224 vout = psimd_concat_hi_f32(vout, vout);
225 }
226 if (k & 1) {
227 psimd_store1_f32(output, vout);
228 output += 1;
229 }
230 }
231 }
232 output = (float*) ((uintptr_t) output + output_increment);
233 } while (--n != 0);
234 }
235