1 // Auto-generated file. Do not edit!
2 // Template: src/cs16-bfly4/scalar.c.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2022 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10 #include <assert.h>
11 #include <stddef.h>
12 #include <stdint.h>
13
14 #include <xnnpack/math.h>
15 #include <xnnpack/fft.h>
16
17
xnn_cs16_bfly4_ukernel__scalar_x3(size_t samples,int16_t * data,const size_t stride,const int16_t * twiddle)18 void xnn_cs16_bfly4_ukernel__scalar_x3(
19 size_t samples,
20 int16_t* data,
21 const size_t stride,
22 const int16_t* twiddle)
23 {
24 const int16_t* tw1 = twiddle;
25 const int16_t* tw2 = twiddle;
26 const int16_t* tw3 = twiddle;
27 int16_t* data0 = data;
28 int16_t* data1 = data + samples * 2;
29 int16_t* data2 = data + samples * 4;
30 int16_t* data3 = data + samples * 6;
31
32 assert(samples != 0);
33 assert(data != NULL);
34 assert(stride != 0);
35 assert(twiddle != NULL);
36
37 for (; samples >= 3; samples -= 3) {
38 int32_t vout0r0 = (int32_t) data0[0];
39 int32_t vout0i0 = (int32_t) data0[1];
40 int32_t vout0r1 = (int32_t) data0[2];
41 int32_t vout0i1 = (int32_t) data0[3];
42 int32_t vout0r2 = (int32_t) data0[4];
43 int32_t vout0i2 = (int32_t) data0[5];
44 int32_t vout1r0 = (int32_t) data1[0];
45 int32_t vout1i0 = (int32_t) data1[1];
46 int32_t vout1r1 = (int32_t) data1[2];
47 int32_t vout1i1 = (int32_t) data1[3];
48 int32_t vout1r2 = (int32_t) data1[4];
49 int32_t vout1i2 = (int32_t) data1[5];
50 int32_t vout2r0 = (int32_t) data2[0];
51 int32_t vout2i0 = (int32_t) data2[1];
52 int32_t vout2r1 = (int32_t) data2[2];
53 int32_t vout2i1 = (int32_t) data2[3];
54 int32_t vout2r2 = (int32_t) data2[4];
55 int32_t vout2i2 = (int32_t) data2[5];
56 int32_t vout3r0 = (int32_t) data3[0];
57 int32_t vout3i0 = (int32_t) data3[1];
58 int32_t vout3r1 = (int32_t) data3[2];
59 int32_t vout3i1 = (int32_t) data3[3];
60 int32_t vout3r2 = (int32_t) data3[4];
61 int32_t vout3i2 = (int32_t) data3[5];
62
63 const int32_t vtw1r0 = (const int32_t) tw1[0];
64 const int32_t vtw1i0 = (const int32_t) tw1[1];
65 tw1 += stride * 2;
66 const int32_t vtw1r1 = (const int32_t) tw1[0];
67 const int32_t vtw1i1 = (const int32_t) tw1[1];
68 tw1 += stride * 2;
69 const int32_t vtw1r2 = (const int32_t) tw1[0];
70 const int32_t vtw1i2 = (const int32_t) tw1[1];
71 tw1 += stride * 2;
72 const int32_t vtw2r0 = (const int32_t) tw2[0];
73 const int32_t vtw2i0 = (const int32_t) tw2[1];
74 tw2 += stride * 4;
75 const int32_t vtw2r1 = (const int32_t) tw2[0];
76 const int32_t vtw2i1 = (const int32_t) tw2[1];
77 tw2 += stride * 4;
78 const int32_t vtw2r2 = (const int32_t) tw2[0];
79 const int32_t vtw2i2 = (const int32_t) tw2[1];
80 tw2 += stride * 4;
81 const int32_t vtw3r0 = (const int32_t) tw3[0];
82 const int32_t vtw3i0 = (const int32_t) tw3[1];
83 tw3 += stride * 6;
84 const int32_t vtw3r1 = (const int32_t) tw3[0];
85 const int32_t vtw3i1 = (const int32_t) tw3[1];
86 tw3 += stride * 6;
87 const int32_t vtw3r2 = (const int32_t) tw3[0];
88 const int32_t vtw3i2 = (const int32_t) tw3[1];
89 tw3 += stride * 6;
90
91 // Note 32767 / 4 = 8191. Should be 8192.
92 vout0r0 = math_asr_s32(vout0r0 * 8191 + 16384, 15);
93 vout0r1 = math_asr_s32(vout0r1 * 8191 + 16384, 15);
94 vout0r2 = math_asr_s32(vout0r2 * 8191 + 16384, 15);
95 vout0i0 = math_asr_s32(vout0i0 * 8191 + 16384, 15);
96 vout0i1 = math_asr_s32(vout0i1 * 8191 + 16384, 15);
97 vout0i2 = math_asr_s32(vout0i2 * 8191 + 16384, 15);
98 vout1r0 = math_asr_s32(vout1r0 * 8191 + 16384, 15);
99 vout1r1 = math_asr_s32(vout1r1 * 8191 + 16384, 15);
100 vout1r2 = math_asr_s32(vout1r2 * 8191 + 16384, 15);
101 vout1i0 = math_asr_s32(vout1i0 * 8191 + 16384, 15);
102 vout1i1 = math_asr_s32(vout1i1 * 8191 + 16384, 15);
103 vout1i2 = math_asr_s32(vout1i2 * 8191 + 16384, 15);
104 vout2r0 = math_asr_s32(vout2r0 * 8191 + 16384, 15);
105 vout2r1 = math_asr_s32(vout2r1 * 8191 + 16384, 15);
106 vout2r2 = math_asr_s32(vout2r2 * 8191 + 16384, 15);
107 vout2i0 = math_asr_s32(vout2i0 * 8191 + 16384, 15);
108 vout2i1 = math_asr_s32(vout2i1 * 8191 + 16384, 15);
109 vout2i2 = math_asr_s32(vout2i2 * 8191 + 16384, 15);
110 vout3r0 = math_asr_s32(vout3r0 * 8191 + 16384, 15);
111 vout3r1 = math_asr_s32(vout3r1 * 8191 + 16384, 15);
112 vout3r2 = math_asr_s32(vout3r2 * 8191 + 16384, 15);
113 vout3i0 = math_asr_s32(vout3i0 * 8191 + 16384, 15);
114 vout3i1 = math_asr_s32(vout3i1 * 8191 + 16384, 15);
115 vout3i2 = math_asr_s32(vout3i2 * 8191 + 16384, 15);
116
117 const int32_t vtmp0r0 = math_asr_s32(vout1r0 * vtw1r0 - vout1i0 * vtw1i0 + 16384, 15);
118 const int32_t vtmp0r1 = math_asr_s32(vout1r1 * vtw1r1 - vout1i1 * vtw1i1 + 16384, 15);
119 const int32_t vtmp0r2 = math_asr_s32(vout1r2 * vtw1r2 - vout1i2 * vtw1i2 + 16384, 15);
120 const int32_t vtmp0i0 = math_asr_s32(vout1r0 * vtw1i0 + vout1i0 * vtw1r0 + 16384, 15);
121 const int32_t vtmp0i1 = math_asr_s32(vout1r1 * vtw1i1 + vout1i1 * vtw1r1 + 16384, 15);
122 const int32_t vtmp0i2 = math_asr_s32(vout1r2 * vtw1i2 + vout1i2 * vtw1r2 + 16384, 15);
123 const int32_t vtmp1r0 = math_asr_s32(vout2r0 * vtw2r0 - vout2i0 * vtw2i0 + 16384, 15);
124 const int32_t vtmp1r1 = math_asr_s32(vout2r1 * vtw2r1 - vout2i1 * vtw2i1 + 16384, 15);
125 const int32_t vtmp1r2 = math_asr_s32(vout2r2 * vtw2r2 - vout2i2 * vtw2i2 + 16384, 15);
126 const int32_t vtmp1i0 = math_asr_s32(vout2r0 * vtw2i0 + vout2i0 * vtw2r0 + 16384, 15);
127 const int32_t vtmp1i1 = math_asr_s32(vout2r1 * vtw2i1 + vout2i1 * vtw2r1 + 16384, 15);
128 const int32_t vtmp1i2 = math_asr_s32(vout2r2 * vtw2i2 + vout2i2 * vtw2r2 + 16384, 15);
129 const int32_t vtmp2r0 = math_asr_s32(vout3r0 * vtw3r0 - vout3i0 * vtw3i0 + 16384, 15);
130 const int32_t vtmp2r1 = math_asr_s32(vout3r1 * vtw3r1 - vout3i1 * vtw3i1 + 16384, 15);
131 const int32_t vtmp2r2 = math_asr_s32(vout3r2 * vtw3r2 - vout3i2 * vtw3i2 + 16384, 15);
132 const int32_t vtmp2i0 = math_asr_s32(vout3r0 * vtw3i0 + vout3i0 * vtw3r0 + 16384, 15);
133 const int32_t vtmp2i1 = math_asr_s32(vout3r1 * vtw3i1 + vout3i1 * vtw3r1 + 16384, 15);
134 const int32_t vtmp2i2 = math_asr_s32(vout3r2 * vtw3i2 + vout3i2 * vtw3r2 + 16384, 15);
135
136 const int32_t vtmp5r0 = vout0r0 - vtmp1r0;
137 const int32_t vtmp5r1 = vout0r1 - vtmp1r1;
138 const int32_t vtmp5r2 = vout0r2 - vtmp1r2;
139 const int32_t vtmp5i0 = vout0i0 - vtmp1i0;
140 const int32_t vtmp5i1 = vout0i1 - vtmp1i1;
141 const int32_t vtmp5i2 = vout0i2 - vtmp1i2;
142 vout0r0 += vtmp1r0;
143 vout0r1 += vtmp1r1;
144 vout0r2 += vtmp1r2;
145 vout0i0 += vtmp1i0;
146 vout0i1 += vtmp1i1;
147 vout0i2 += vtmp1i2;
148 const int32_t vtmp3r0 = vtmp0r0 + vtmp2r0;
149 const int32_t vtmp3r1 = vtmp0r1 + vtmp2r1;
150 const int32_t vtmp3r2 = vtmp0r2 + vtmp2r2;
151 const int32_t vtmp3i0 = vtmp0i0 + vtmp2i0;
152 const int32_t vtmp3i1 = vtmp0i1 + vtmp2i1;
153 const int32_t vtmp3i2 = vtmp0i2 + vtmp2i2;
154 const int32_t vtmp4r0 = vtmp0r0 - vtmp2r0;
155 const int32_t vtmp4r1 = vtmp0r1 - vtmp2r1;
156 const int32_t vtmp4r2 = vtmp0r2 - vtmp2r2;
157 const int32_t vtmp4i0 = vtmp0i0 - vtmp2i0;
158 const int32_t vtmp4i1 = vtmp0i1 - vtmp2i1;
159 const int32_t vtmp4i2 = vtmp0i2 - vtmp2i2;
160 vout2r0 = vout0r0 - vtmp3r0;
161 vout2r1 = vout0r1 - vtmp3r1;
162 vout2r2 = vout0r2 - vtmp3r2;
163 vout2i0 = vout0i0 - vtmp3i0;
164 vout2i1 = vout0i1 - vtmp3i1;
165 vout2i2 = vout0i2 - vtmp3i2;
166 vout0r0 += vtmp3r0;
167 vout0r1 += vtmp3r1;
168 vout0r2 += vtmp3r2;
169 vout0i0 += vtmp3i0;
170 vout0i1 += vtmp3i1;
171 vout0i2 += vtmp3i2;
172 vout1r0 = vtmp5r0 + vtmp4i0;
173 vout1r1 = vtmp5r1 + vtmp4i1;
174 vout1r2 = vtmp5r2 + vtmp4i2;
175 vout1i0 = vtmp5i0 - vtmp4r0;
176 vout1i1 = vtmp5i1 - vtmp4r1;
177 vout1i2 = vtmp5i2 - vtmp4r2;
178 vout3r0 = vtmp5r0 - vtmp4i0;
179 vout3r1 = vtmp5r1 - vtmp4i1;
180 vout3r2 = vtmp5r2 - vtmp4i2;
181 vout3i0 = vtmp5i0 + vtmp4r0;
182 vout3i1 = vtmp5i1 + vtmp4r1;
183 vout3i2 = vtmp5i2 + vtmp4r2;
184
185 data0[0] = (int16_t) vout0r0;
186 data0[1] = (int16_t) vout0i0;
187 data0[2] = (int16_t) vout0r1;
188 data0[3] = (int16_t) vout0i1;
189 data0[4] = (int16_t) vout0r2;
190 data0[5] = (int16_t) vout0i2;
191 data0 += 3 * 2;
192 data1[0] = (int16_t) vout1r0;
193 data1[1] = (int16_t) vout1i0;
194 data1[2] = (int16_t) vout1r1;
195 data1[3] = (int16_t) vout1i1;
196 data1[4] = (int16_t) vout1r2;
197 data1[5] = (int16_t) vout1i2;
198 data1 += 3 * 2;
199 data2[0] = (int16_t) vout2r0;
200 data2[1] = (int16_t) vout2i0;
201 data2[2] = (int16_t) vout2r1;
202 data2[3] = (int16_t) vout2i1;
203 data2[4] = (int16_t) vout2r2;
204 data2[5] = (int16_t) vout2i2;
205 data2 += 3 * 2;
206 data3[0] = (int16_t) vout3r0;
207 data3[1] = (int16_t) vout3i0;
208 data3[2] = (int16_t) vout3r1;
209 data3[3] = (int16_t) vout3i1;
210 data3[4] = (int16_t) vout3r2;
211 data3[5] = (int16_t) vout3i2;
212 data3 += 3 * 2;
213 }
214
215 if XNN_UNLIKELY(samples != 0) {
216 do {
217 int32_t vout0r = (int32_t) data0[0];
218 int32_t vout0i = (int32_t) data0[1];
219 int32_t vout1r = (int32_t) data1[0];
220 int32_t vout1i = (int32_t) data1[1];
221 int32_t vout2r = (int32_t) data2[0];
222 int32_t vout2i = (int32_t) data2[1];
223 int32_t vout3r = (int32_t) data3[0];
224 int32_t vout3i = (int32_t) data3[1];
225
226 const int32_t vtw1r = (const int32_t) tw1[0];
227 const int32_t vtw1i = (const int32_t) tw1[1];
228 const int32_t vtw2r = (const int32_t) tw2[0];
229 const int32_t vtw2i = (const int32_t) tw2[1];
230 const int32_t vtw3r = (const int32_t) tw3[0];
231 const int32_t vtw3i = (const int32_t) tw3[1];
232 tw1 += stride * 2;
233 tw2 += stride * 4;
234 tw3 += stride * 6;
235
236 // Note 32767 / 4 = 8191. Should be 8192.
237 vout0r = math_asr_s32(vout0r * 8191 + 16384, 15);
238 vout0i = math_asr_s32(vout0i * 8191 + 16384, 15);
239 vout1r = math_asr_s32(vout1r * 8191 + 16384, 15);
240 vout1i = math_asr_s32(vout1i * 8191 + 16384, 15);
241 vout2r = math_asr_s32(vout2r * 8191 + 16384, 15);
242 vout2i = math_asr_s32(vout2i * 8191 + 16384, 15);
243 vout3r = math_asr_s32(vout3r * 8191 + 16384, 15);
244 vout3i = math_asr_s32(vout3i * 8191 + 16384, 15);
245
246 const int32_t vtmp0r = math_asr_s32(vout1r * vtw1r - vout1i * vtw1i + 16384, 15);
247 const int32_t vtmp0i = math_asr_s32(vout1r * vtw1i + vout1i * vtw1r + 16384, 15);
248 const int32_t vtmp1r = math_asr_s32(vout2r * vtw2r - vout2i * vtw2i + 16384, 15);
249 const int32_t vtmp1i = math_asr_s32(vout2r * vtw2i + vout2i * vtw2r + 16384, 15);
250 const int32_t vtmp2r = math_asr_s32(vout3r * vtw3r - vout3i * vtw3i + 16384, 15);
251 const int32_t vtmp2i = math_asr_s32(vout3r * vtw3i + vout3i * vtw3r + 16384, 15);
252
253 const int32_t vtmp5r = vout0r - vtmp1r;
254 const int32_t vtmp5i = vout0i - vtmp1i;
255 vout0r += vtmp1r;
256 vout0i += vtmp1i;
257 const int32_t vtmp3r = vtmp0r + vtmp2r;
258 const int32_t vtmp3i = vtmp0i + vtmp2i;
259 const int32_t vtmp4r = vtmp0r - vtmp2r;
260 const int32_t vtmp4i = vtmp0i - vtmp2i;
261 vout2r = vout0r - vtmp3r;
262 vout2i = vout0i - vtmp3i;
263
264 vout0r += vtmp3r;
265 vout0i += vtmp3i;
266
267 vout1r = vtmp5r + vtmp4i;
268 vout1i = vtmp5i - vtmp4r;
269 vout3r = vtmp5r - vtmp4i;
270 vout3i = vtmp5i + vtmp4r;
271
272 data0[0] = (int16_t) vout0r;
273 data0[1] = (int16_t) vout0i;
274 data1[0] = (int16_t) vout1r;
275 data1[1] = (int16_t) vout1i;
276 data2[0] = (int16_t) vout2r;
277 data2[1] = (int16_t) vout2i;
278 data3[0] = (int16_t) vout3r;
279 data3[1] = (int16_t) vout3i;
280 data0 += 2;
281 data1 += 2;
282 data2 += 2;
283 data3 += 2;
284 } while(--samples != 0);
285 }
286 }
287