1 // Auto-generated file. Do not edit!
2 // Template: src/f16-ibilinear-chw/neonfp16arith.c.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2022 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10 #include <assert.h>
11
12 #include <arm_neon.h>
13
14 #include <xnnpack/ibilinear.h>
15
16
xnn_f16_ibilinear_chw_ukernel__neonfp16arith_p8(size_t output_pixels,size_t channels,const void ** restrict input,size_t input_offset,const void * restrict weights,void * restrict output,size_t input_increment)17 void xnn_f16_ibilinear_chw_ukernel__neonfp16arith_p8(
18 size_t output_pixels,
19 size_t channels,
20 const void**restrict input,
21 size_t input_offset,
22 const void*restrict weights,
23 void*restrict output,
24 size_t input_increment) XNN_OOB_READS
25 {
26 assert(output_pixels != 0);
27 assert(channels != 0);
28 assert(input_increment % sizeof(__fp16) == 0);
29
30 __fp16* o = (__fp16*) output;
31 do {
32 const __fp16** i = (const __fp16**)input;
33 const __fp16* w = weights;
34 size_t p = output_pixels;
35
36 for (; p >= 8; p -= 8) {
37 const __fp16* itl0 = (const __fp16*) ((uintptr_t) i[0] + input_offset);
38 const __fp16* ibl0 = (const __fp16*) ((uintptr_t) i[1] + input_offset);
39 const __fp16* itl1 = (const __fp16*) ((uintptr_t) i[2] + input_offset);
40 const __fp16* ibl1 = (const __fp16*) ((uintptr_t) i[3] + input_offset);
41 const __fp16* itl2 = (const __fp16*) ((uintptr_t) i[4] + input_offset);
42 const __fp16* ibl2 = (const __fp16*) ((uintptr_t) i[5] + input_offset);
43 const __fp16* itl3 = (const __fp16*) ((uintptr_t) i[6] + input_offset);
44 const __fp16* ibl3 = (const __fp16*) ((uintptr_t) i[7] + input_offset);
45 const __fp16* itl4 = (const __fp16*) ((uintptr_t) i[8] + input_offset);
46 const __fp16* ibl4 = (const __fp16*) ((uintptr_t) i[9] + input_offset);
47 const __fp16* itl5 = (const __fp16*) ((uintptr_t) i[10] + input_offset);
48 const __fp16* ibl5 = (const __fp16*) ((uintptr_t) i[11] + input_offset);
49 const __fp16* itl6 = (const __fp16*) ((uintptr_t) i[12] + input_offset);
50 const __fp16* ibl6 = (const __fp16*) ((uintptr_t) i[13] + input_offset);
51 const __fp16* itl7 = (const __fp16*) ((uintptr_t) i[14] + input_offset);
52 const __fp16* ibl7 = (const __fp16*) ((uintptr_t) i[15] + input_offset);
53 i += 2 * 8;
54
55 const float16x4x2_t vw0123 = vld2_f16(w + 0);
56 const float16x4x2_t vw4567 = vld2_f16(w + 8);
57 w += 2 * 8;
58
59 float16x8_t vtltr0123 = vmovq_n_f16(0); // vmov for uninitialized var warning
60 float16x8_t vblbr0123 = vmovq_n_f16(0);
61 vtltr0123 = vreinterpretq_f16_u32(vld1q_lane_u32((const void*) itl0, vreinterpretq_u32_f16(vtltr0123), 0));
62 vblbr0123 = vreinterpretq_f16_u32(vld1q_lane_u32((const void*) ibl0, vreinterpretq_u32_f16(vblbr0123), 0));
63 vtltr0123 = vreinterpretq_f16_u32(vld1q_lane_u32((const void*) itl1, vreinterpretq_u32_f16(vtltr0123), 1));
64 vblbr0123 = vreinterpretq_f16_u32(vld1q_lane_u32((const void*) ibl1, vreinterpretq_u32_f16(vblbr0123), 1));
65 vtltr0123 = vreinterpretq_f16_u32(vld1q_lane_u32((const void*) itl2, vreinterpretq_u32_f16(vtltr0123), 2));
66 vblbr0123 = vreinterpretq_f16_u32(vld1q_lane_u32((const void*) ibl2, vreinterpretq_u32_f16(vblbr0123), 2));
67 vtltr0123 = vreinterpretq_f16_u32(vld1q_lane_u32((const void*) itl3, vreinterpretq_u32_f16(vtltr0123), 3));
68 vblbr0123 = vreinterpretq_f16_u32(vld1q_lane_u32((const void*) ibl3, vreinterpretq_u32_f16(vblbr0123), 3));
69 float16x8_t vtltr4567 = vmovq_n_f16(0); // vmov for uninitialized var warning
70 float16x8_t vblbr4567 = vmovq_n_f16(0);
71 vtltr4567 = vreinterpretq_f16_u32(vld1q_lane_u32((const void*) itl4, vreinterpretq_u32_f16(vtltr4567), 0));
72 vblbr4567 = vreinterpretq_f16_u32(vld1q_lane_u32((const void*) ibl4, vreinterpretq_u32_f16(vblbr4567), 0));
73 vtltr4567 = vreinterpretq_f16_u32(vld1q_lane_u32((const void*) itl5, vreinterpretq_u32_f16(vtltr4567), 1));
74 vblbr4567 = vreinterpretq_f16_u32(vld1q_lane_u32((const void*) ibl5, vreinterpretq_u32_f16(vblbr4567), 1));
75 vtltr4567 = vreinterpretq_f16_u32(vld1q_lane_u32((const void*) itl6, vreinterpretq_u32_f16(vtltr4567), 2));
76 vblbr4567 = vreinterpretq_f16_u32(vld1q_lane_u32((const void*) ibl6, vreinterpretq_u32_f16(vblbr4567), 2));
77 vtltr4567 = vreinterpretq_f16_u32(vld1q_lane_u32((const void*) itl7, vreinterpretq_u32_f16(vtltr4567), 3));
78 vblbr4567 = vreinterpretq_f16_u32(vld1q_lane_u32((const void*) ibl7, vreinterpretq_u32_f16(vblbr4567), 3));
79
80 const float16x8_t valphah01234567 = vcombine_f16(vw0123.val[0], vw4567.val[0]);
81 const float16x8_t valphav01234567 = vcombine_f16(vw0123.val[1], vw4567.val[1]);
82
83 const float16x8_t vldrd0123 = vsubq_f16(vblbr0123, vtltr0123);
84 const float16x8_t vldrd4567 = vsubq_f16(vblbr4567, vtltr4567);
85
86 const float16x8x2_t vld_t01234567 = vuzpq_f16(vldrd0123, vldrd4567);
87 const float16x8_t vld01234567 = vld_t01234567.val[0];
88 const float16x8_t vrd01234567 = vld_t01234567.val[1];
89
90 const float16x8x2_t vtl_t01234567 = vuzpq_f16(vtltr0123, vtltr4567);
91 const float16x8_t vtl01234567 = vtl_t01234567.val[0];
92 const float16x8_t vtr01234567 = vtl_t01234567.val[1];
93
94 const float16x8_t vl01234567 = vfmaq_f16(vtl01234567, vld01234567, valphav01234567);
95 const float16x8_t vr01234567 = vfmaq_f16(vtr01234567, vrd01234567, valphav01234567);
96
97 const float16x8_t vd01234567 = vsubq_f16(vr01234567, vl01234567);
98 const float16x8_t vo01234567 = vfmaq_f16(vl01234567, vd01234567, valphah01234567);
99
100 vst1q_f16(o + 0, vo01234567);
101 o += 8;
102 }
103
104 for (; p >= 4; p -= 4) {
105 const __fp16* itl0 = (const __fp16*) ((uintptr_t) i[0] + input_offset);
106 const __fp16* ibl0 = (const __fp16*) ((uintptr_t) i[1] + input_offset);
107 const __fp16* itl1 = (const __fp16*) ((uintptr_t) i[2] + input_offset);
108 const __fp16* ibl1 = (const __fp16*) ((uintptr_t) i[3] + input_offset);
109 const __fp16* itl2 = (const __fp16*) ((uintptr_t) i[4] + input_offset);
110 const __fp16* ibl2 = (const __fp16*) ((uintptr_t) i[5] + input_offset);
111 const __fp16* itl3 = (const __fp16*) ((uintptr_t) i[6] + input_offset);
112 const __fp16* ibl3 = (const __fp16*) ((uintptr_t) i[7] + input_offset);
113 i += 8;
114
115 const float16x4x2_t vw = vld2_f16(w);
116 w += 8;
117
118 float16x8_t vtltr = vmovq_n_f16(0); // vmov for uninitialized var warning
119 float16x8_t vblbr = vmovq_n_f16(0);
120 vtltr = vreinterpretq_f16_u32(vld1q_lane_u32((const void*) itl0, vreinterpretq_u32_f16(vtltr), 0));
121 vblbr = vreinterpretq_f16_u32(vld1q_lane_u32((const void*) ibl0, vreinterpretq_u32_f16(vblbr), 0));
122 vtltr = vreinterpretq_f16_u32(vld1q_lane_u32((const void*) itl1, vreinterpretq_u32_f16(vtltr), 1));
123 vblbr = vreinterpretq_f16_u32(vld1q_lane_u32((const void*) ibl1, vreinterpretq_u32_f16(vblbr), 1));
124 vtltr = vreinterpretq_f16_u32(vld1q_lane_u32((const void*) itl2, vreinterpretq_u32_f16(vtltr), 2));
125 vblbr = vreinterpretq_f16_u32(vld1q_lane_u32((const void*) ibl2, vreinterpretq_u32_f16(vblbr), 2));
126 vtltr = vreinterpretq_f16_u32(vld1q_lane_u32((const void*) itl3, vreinterpretq_u32_f16(vtltr), 3));
127 vblbr = vreinterpretq_f16_u32(vld1q_lane_u32((const void*) ibl3, vreinterpretq_u32_f16(vblbr), 3));
128
129 const float16x4_t valphah = vw.val[0];
130 const float16x4_t valphav = vw.val[1];
131
132 const float16x8_t vldrd = vsubq_f16(vblbr, vtltr);
133
134 const float16x4x2_t vld_t = vuzp_f16(vget_low_f16(vldrd), vget_high_f16(vldrd));
135 const float16x4_t vld = vld_t.val[0];
136 const float16x4_t vrd = vld_t.val[1];
137
138 const float16x4x2_t vtl_t = vuzp_f16(vget_low_f16(vtltr), vget_high_f16(vtltr));
139 const float16x4_t vtl = vtl_t.val[0];
140 const float16x4_t vtr = vtl_t.val[1];
141
142 const float16x4_t vl = vfma_f16(vtl, vld, valphav);
143 const float16x4_t vr = vfma_f16(vtr, vrd, valphav);
144
145 const float16x4_t vd = vsub_f16(vr, vl);
146 const float16x4_t vo = vfma_f16(vl, vd, valphah);
147
148 vst1_f16(o, vo);
149 o += 4;
150 }
151
152 if XNN_UNLIKELY(p != 0) {
153 if (p & 2) {
154 const __fp16* itl0 = (const __fp16*) ((uintptr_t) i[0] + input_offset);
155 const __fp16* ibl0 = (const __fp16*) ((uintptr_t) i[1] + input_offset);
156 const __fp16* itl1 = (const __fp16*) ((uintptr_t) i[2] + input_offset);
157 const __fp16* ibl1 = (const __fp16*) ((uintptr_t) i[3] + input_offset);
158 i += 4;
159
160 const float16x4_t vw = vld1_f16(w);
161 w += 4;
162
163 const float16x4x2_t vwhv = vuzp_f16(vw, vw);
164 const float16x4_t valphah = vwhv.val[0];
165 const float16x4_t valphav = vwhv.val[1];
166
167 float16x4_t vtltr = vmov_n_f16(0); // vmov for uninitialized var warning
168 float16x4_t vblbr = vmov_n_f16(0);
169
170 vtltr = vreinterpret_f16_u32(vld1_lane_u32((const void*) itl0, vreinterpret_u32_f16(vtltr), 0));
171 vblbr = vreinterpret_f16_u32(vld1_lane_u32((const void*) ibl0, vreinterpret_u32_f16(vblbr), 0));
172 vtltr = vreinterpret_f16_u32(vld1_lane_u32((const void*) itl1, vreinterpret_u32_f16(vtltr), 1));
173 vblbr = vreinterpret_f16_u32(vld1_lane_u32((const void*) ibl1, vreinterpret_u32_f16(vblbr), 1));
174
175 const float16x4_t vldrd = vsub_f16(vblbr, vtltr);
176
177 const float16x4x2_t vld_t = vuzp_f16(vldrd, vldrd);
178 const float16x4_t vld = vld_t.val[0];
179 const float16x4_t vrd = vld_t.val[1];
180
181 const float16x4x2_t vtl_t = vuzp_f16(vtltr, vtltr);
182 const float16x4_t vtl = vtl_t.val[0];
183 const float16x4_t vtr = vtl_t.val[1];
184
185 const float16x4_t vl = vfma_f16(vtl, vld, valphav);
186 const float16x4_t vr = vfma_f16(vtr, vrd, valphav);
187
188 const float16x4_t vd = vsub_f16(vr, vl);
189 const float16x4_t vo = vfma_f16(vl, vd, valphah);
190
191 vst1_lane_u32((void*) o, vreinterpret_u32_f16(vo), 0);
192 o += 2;
193 }
194
195 if (p & 1) {
196 // We are computing the following formula:
197 // result = (1 - alpha_h) * (1 - alpha_v) * top_left +
198 // alpha_h * (1 - alpha_v) * top_right +
199 // (1 - alpha_h) * alpha_v * bottom_left +
200 // alpha_h * alpha_v * bottom_right.
201 //
202 // Rearranging gives
203 // result = left + alpha_h * (right - left),
204 // where
205 // left = top_left + alpha_v * (bottom_left - top_left),
206 // right = top_right + alpha_v * (bottom_right - top_right).
207
208 const __fp16* itl = (const __fp16*) ((uintptr_t) i[0] + input_offset);
209 const __fp16* ibl = (const __fp16*) ((uintptr_t) i[1] + input_offset);
210 i += 2;
211
212 float16x4_t vw = vmov_n_f16(0);
213 vw = vreinterpret_f16_u32(vld1_lane_u32((const void*) w, vreinterpret_u32_f16(vw), 0));
214 w += 2;
215
216 const float16x4x2_t vwhv = vuzp_f16(vw, vw);
217 const float16x4_t valphah = vwhv.val[0];
218 const float16x4_t valphav = vwhv.val[1];
219
220 float16x4_t vtltr = vmov_n_f16(0); // vmov for uninitialized var warning
221 float16x4_t vblbr = vmov_n_f16(0);
222
223 vtltr = vreinterpret_f16_u32(vld1_lane_u32((const void*) itl, vreinterpret_u32_f16(vtltr), 0));
224 vblbr = vreinterpret_f16_u32(vld1_lane_u32((const void*) ibl, vreinterpret_u32_f16(vblbr), 0));
225
226 const float16x4_t vldrd = vsub_f16(vblbr, vtltr);
227
228 const float16x4x2_t vld_t = vuzp_f16(vldrd, vldrd);
229 const float16x4_t vld = vld_t.val[0];
230 const float16x4_t vrd = vld_t.val[1];
231
232 const float16x4x2_t vtl_t = vuzp_f16(vtltr, vtltr);
233 const float16x4_t vtl = vtl_t.val[0];
234 const float16x4_t vtr = vtl_t.val[1];
235
236 const float16x4_t vl = vfma_f16(vtl, vld, valphav);
237 const float16x4_t vr = vfma_f16(vtr, vrd, valphav);
238
239 const float16x4_t vd = vsub_f16(vr, vl);
240 const float16x4_t vo = vfma_f16(vl, vd, valphah);
241
242 vst1_lane_f16(o, vo, 0);
243 o += 1;
244 }
245 }
246
247 input_offset += input_increment;
248 } while (--channels != 0);
249 }
250