1 // Copyright 2021 Google LLC
2 //
3 // This source code is licensed under the BSD-style license found in the
4 // LICENSE file in the root directory of this source tree.
5
6 #include <assert.h>
7
8 #include <arm_neon.h>
9
10 #include <xnnpack/maxpool.h>
11
12
xnn_s8_maxpool_minmax_ukernel_9p8x__neon_c16(size_t output_pixels,size_t kernel_elements,size_t channels,const int8_t ** input,size_t input_offset,int8_t * output,size_t input_increment,size_t output_increment,const union xnn_s8_minmax_params params[restrict XNN_MIN_ELEMENTS (1)])13 void xnn_s8_maxpool_minmax_ukernel_9p8x__neon_c16(
14 size_t output_pixels,
15 size_t kernel_elements,
16 size_t channels,
17 const int8_t** input,
18 size_t input_offset,
19 int8_t* output,
20 size_t input_increment,
21 size_t output_increment,
22 const union xnn_s8_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_OOB_READS
23 {
24 assert(output_pixels != 0);
25 assert(kernel_elements != 0);
26 assert(channels != 0);
27
28 const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.max);
29 const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.min);
30 do {
31 int8_t* o = output;
32 {
33 const int8_t* i0 = *input++;
34 const int8_t* i1 = *input++;
35 const int8_t* i2 = *input++;
36 const int8_t* i3 = *input++;
37 const int8_t* i4 = *input++;
38 const int8_t* i5 = *input++;
39 const int8_t* i6 = *input++;
40 const int8_t* i7 = *input++;
41 const int8_t* i8 = *input++;
42 i0 = (const int8_t*) ((uintptr_t) i0 + input_offset);
43 i1 = (const int8_t*) ((uintptr_t) i1 + input_offset);
44 i2 = (const int8_t*) ((uintptr_t) i2 + input_offset);
45 i3 = (const int8_t*) ((uintptr_t) i3 + input_offset);
46 i4 = (const int8_t*) ((uintptr_t) i4 + input_offset);
47 i5 = (const int8_t*) ((uintptr_t) i5 + input_offset);
48 i6 = (const int8_t*) ((uintptr_t) i6 + input_offset);
49 i7 = (const int8_t*) ((uintptr_t) i7 + input_offset);
50 i8 = (const int8_t*) ((uintptr_t) i8 + input_offset);
51 if (kernel_elements < 2) {
52 i1 = i0;
53 }
54 if (kernel_elements <= 2) {
55 i2 = i0;
56 }
57 if (kernel_elements < 4) {
58 i3 = i0;
59 }
60 if (kernel_elements <= 4) {
61 i4 = i0;
62 }
63 if (kernel_elements < 6) {
64 i5 = i0;
65 }
66 if (kernel_elements <= 6) {
67 i6 = i0;
68 }
69 if (kernel_elements < 8) {
70 i7 = i0;
71 }
72 if (kernel_elements <= 8) {
73 i8 = i0;
74 }
75
76 size_t c = channels;
77 for (; c >= 16; c -= 16) {
78 const int8x16_t vi0 = vld1q_s8(i0); i0 += 16;
79 const int8x16_t vi1 = vld1q_s8(i1); i1 += 16;
80 const int8x16_t vi2 = vld1q_s8(i2); i2 += 16;
81 const int8x16_t vi3 = vld1q_s8(i3); i3 += 16;
82 const int8x16_t vi4 = vld1q_s8(i4); i4 += 16;
83 const int8x16_t vi5 = vld1q_s8(i5); i5 += 16;
84 const int8x16_t vi6 = vld1q_s8(i6); i6 += 16;
85 const int8x16_t vi7 = vld1q_s8(i7); i7 += 16;
86 const int8x16_t vi8 = vld1q_s8(i8); i8 += 16;
87
88 const int8x16_t vmax018 = vmaxq_s8(vmaxq_s8(vi0, vi1), vi8);
89 const int8x16_t vmax23 = vmaxq_s8(vi2, vi3);
90 const int8x16_t vmax45 = vmaxq_s8(vi4, vi5);
91 const int8x16_t vmax67 = vmaxq_s8(vi6, vi7);
92
93 const int8x16_t vmax2345 = vmaxq_s8(vmax23, vmax45);
94 const int8x16_t vmax01678 = vmaxq_s8(vmax018, vmax67);
95 int8x16_t vout = vmaxq_s8(vmax2345, vmax01678);
96 vout = vmaxq_s8(vout, voutput_min);
97 vout = vminq_s8(vout, voutput_max);
98
99 vst1q_s8(o, vout); o += 16;
100 }
101 if (c != 0) {
102 const int8x16_t vi0 = vld1q_s8(i0);
103 const int8x16_t vi1 = vld1q_s8(i1);
104 const int8x16_t vi2 = vld1q_s8(i2);
105 const int8x16_t vi3 = vld1q_s8(i3);
106 const int8x16_t vi4 = vld1q_s8(i4);
107 const int8x16_t vi5 = vld1q_s8(i5);
108 const int8x16_t vi6 = vld1q_s8(i6);
109 const int8x16_t vi7 = vld1q_s8(i7);
110 const int8x16_t vi8 = vld1q_s8(i8);
111
112 const int8x16_t vmax018 = vmaxq_s8(vmaxq_s8(vi0, vi1), vi8);
113 const int8x16_t vmax23 = vmaxq_s8(vi2, vi3);
114 const int8x16_t vmax45 = vmaxq_s8(vi4, vi5);
115 const int8x16_t vmax67 = vmaxq_s8(vi6, vi7);
116
117 const int8x16_t vmax2345 = vmaxq_s8(vmax23, vmax45);
118 const int8x16_t vmax01678 = vmaxq_s8(vmax018, vmax67);
119 int8x16_t vout = vmaxq_s8(vmax2345, vmax01678);
120 vout = vmaxq_s8(vout, voutput_min);
121 vout = vminq_s8(vout, voutput_max);
122
123 int8x8_t vout_lo = vget_low_s8(vout);
124 if (c & 8) {
125 vst1_s8(o, vout_lo); o += 8;
126 vout_lo = vget_high_s8(vout);
127 }
128 if (c & 4) {
129 vst1_lane_u32((void*) o, vreinterpret_u32_s8(vout_lo), 0); o += 4;
130 vout_lo = vext_s8(vout_lo, vout_lo, 4);
131 }
132 if (c & 2) {
133 vst1_lane_u16((void*) o, vreinterpret_u16_s8(vout_lo), 0); o += 2;
134 vout_lo = vext_s8(vout_lo, vout_lo, 2);
135 }
136 if (c & 1) {
137 vst1_lane_s8(o, vout_lo, 0); o += 1;
138 }
139 }
140 }
141
142 for (ptrdiff_t k = (ptrdiff_t) kernel_elements - 9; k > 0; k -= 8) {
143 const int8_t* i0 = *input++;
144 const int8_t* i1 = *input++;
145 const int8_t* i2 = *input++;
146 const int8_t* i3 = *input++;
147 const int8_t* i4 = *input++;
148 const int8_t* i5 = *input++;
149 const int8_t* i6 = *input++;
150 const int8_t* i7 = *input++;
151 i0 = (const int8_t*) ((uintptr_t) i0 + input_offset);
152 i1 = (const int8_t*) ((uintptr_t) i1 + input_offset);
153 i2 = (const int8_t*) ((uintptr_t) i2 + input_offset);
154 i3 = (const int8_t*) ((uintptr_t) i3 + input_offset);
155 i4 = (const int8_t*) ((uintptr_t) i4 + input_offset);
156 i5 = (const int8_t*) ((uintptr_t) i5 + input_offset);
157 i6 = (const int8_t*) ((uintptr_t) i6 + input_offset);
158 i7 = (const int8_t*) ((uintptr_t) i7 + input_offset);
159 if (k < 2) {
160 i1 = i0;
161 }
162 if (k <= 2) {
163 i2 = i0;
164 }
165 if (k < 4) {
166 i3 = i0;
167 }
168 if (k <= 4) {
169 i4 = i0;
170 }
171 if (k < 6) {
172 i5 = i0;
173 }
174 if (k <= 6) {
175 i6 = i0;
176 }
177 if (k < 8) {
178 i7 = i0;
179 }
180
181 o = output;
182 size_t c = channels;
183 for (; c >= 16; c -= 16) {
184 const int8x16_t vi0 = vld1q_s8(i0); i0 += 16;
185 const int8x16_t vi1 = vld1q_s8(i1); i1 += 16;
186 const int8x16_t vi2 = vld1q_s8(i2); i2 += 16;
187 const int8x16_t vi3 = vld1q_s8(i3); i3 += 16;
188 const int8x16_t vi4 = vld1q_s8(i4); i4 += 16;
189 const int8x16_t vi5 = vld1q_s8(i5); i5 += 16;
190 const int8x16_t vi6 = vld1q_s8(i6); i6 += 16;
191 const int8x16_t vi7 = vld1q_s8(i7); i7 += 16;
192 const int8x16_t vo = vld1q_s8(o);
193
194 const int8x16_t vmax01 = vmaxq_s8(vmaxq_s8(vi0, vi1), vo);
195 const int8x16_t vmax23 = vmaxq_s8(vi2, vi3);
196 const int8x16_t vmax45 = vmaxq_s8(vi4, vi5);
197 const int8x16_t vmax67 = vmaxq_s8(vi6, vi7);
198
199 const int8x16_t vmax2345 = vmaxq_s8(vmax23, vmax45);
200 const int8x16_t vmax0167 = vmaxq_s8(vmax01, vmax67);
201 int8x16_t vout = vmaxq_s8(vmax2345, vmax0167);
202 vout = vmaxq_s8(vout, voutput_min);
203 vout = vminq_s8(vout, voutput_max);
204
205 vst1q_s8(o, vout); o += 16;
206 }
207 if (c != 0) {
208 const int8x16_t vi0 = vld1q_s8(i0);
209 const int8x16_t vi1 = vld1q_s8(i1);
210 const int8x16_t vi2 = vld1q_s8(i2);
211 const int8x16_t vi3 = vld1q_s8(i3);
212 const int8x16_t vi4 = vld1q_s8(i4);
213 const int8x16_t vi5 = vld1q_s8(i5);
214 const int8x16_t vi6 = vld1q_s8(i6);
215 const int8x16_t vi7 = vld1q_s8(i7);
216 const int8x16_t vo = vld1q_s8(o);
217
218 const int8x16_t vmax01 = vmaxq_s8(vmaxq_s8(vi0, vi1), vo);
219 const int8x16_t vmax23 = vmaxq_s8(vi2, vi3);
220 const int8x16_t vmax45 = vmaxq_s8(vi4, vi5);
221 const int8x16_t vmax67 = vmaxq_s8(vi6, vi7);
222
223 const int8x16_t vmax2345 = vmaxq_s8(vmax23, vmax45);
224 const int8x16_t vmax0167 = vmaxq_s8(vmax01, vmax67);
225 int8x16_t vout = vmaxq_s8(vmax2345, vmax0167);
226 vout = vmaxq_s8(vout, voutput_min);
227 vout = vminq_s8(vout, voutput_max);
228
229 int8x8_t vout_lo = vget_low_s8(vout);
230 if (c & 8) {
231 vst1_s8(o, vout_lo); o += 8;
232 vout_lo = vget_high_s8(vout);
233 }
234 if (c & 4) {
235 vst1_lane_u32((void*) o, vreinterpret_u32_s8(vout_lo), 0); o += 4;
236 vout_lo = vext_s8(vout_lo, vout_lo, 4);
237 }
238 if (c & 2) {
239 vst1_lane_u16((void*) o, vreinterpret_u16_s8(vout_lo), 0); o += 2;
240 vout_lo = vext_s8(vout_lo, vout_lo, 2);
241 }
242 if (c & 1) {
243 vst1_lane_s8(o, vout_lo, 0); o += 1;
244 }
245 }
246 }
247 input = (const int8_t**) ((uintptr_t) input + input_increment);
248 output = (int8_t*) ((uintptr_t) o + output_increment);
249 } while (--output_pixels != 0);
250 }
251