1 // Auto-generated file. Do not edit!
2 // Template: src/f32-dwconv/up-psimd.c.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2019 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10 #include <assert.h>
11
12 #include <psimd.h>
13
14 #include <xnnpack/dwconv.h>
15
16
xnn_f32_dwconv_ukernel_up4x9__psimd_acc2(size_t channels,size_t output_width,const float ** input,const float * weights,float * output,size_t input_stride,size_t output_increment,const union xnn_f32_output_params params[restrict static1])17 void xnn_f32_dwconv_ukernel_up4x9__psimd_acc2(
18 size_t channels,
19 size_t output_width,
20 const float** input,
21 const float* weights,
22 float* output,
23 size_t input_stride,
24 size_t output_increment,
25 const union xnn_f32_output_params params[restrict static 1])
26 {
27 assert(channels != 0);
28 assert(output_width != 0);
29
30 const psimd_f32 vmax = psimd_load_splat_f32(¶ms->scalar.max);
31 const psimd_f32 vmin = psimd_load_splat_f32(¶ms->scalar.min);
32 do {
33 const float* i0 = input[0];
34 assert(i0 != NULL);
35 const float* i1 = input[1];
36 assert(i1 != NULL);
37 const float* i2 = input[2];
38 assert(i2 != NULL);
39 const float* i3 = input[3];
40 assert(i3 != NULL);
41 const float* i4 = input[4];
42 assert(i4 != NULL);
43 const float* i5 = input[5];
44 assert(i5 != NULL);
45 const float* i6 = input[6];
46 assert(i6 != NULL);
47 const float* i7 = input[7];
48 assert(i7 != NULL);
49 const float* i8 = input[8];
50 assert(i8 != NULL);
51 input = (const float**) ((uintptr_t) input + input_stride);
52
53 size_t c = channels;
54 const float* w = weights;
55 for (; c >= 4; c -= 4) {
56 psimd_f32 vacc0123p0 = psimd_load_f32(w);
57
58
59 const psimd_f32 vi0x0123 = psimd_load_f32(i0);
60 i0 += 4;
61
62 const psimd_f32 vk0x0123 = psimd_load_f32(w + 4);
63 vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi0x0123, vk0x0123);
64
65 const psimd_f32 vi1x0123 = psimd_load_f32(i1);
66 i1 += 4;
67
68 const psimd_f32 vk1x0123 = psimd_load_f32(w + 8);
69 psimd_f32 vacc0123p1 = psimd_mul_f32(vi1x0123, vk1x0123);
70
71 const psimd_f32 vi2x0123 = psimd_load_f32(i2);
72 i2 += 4;
73
74 const psimd_f32 vk2x0123 = psimd_load_f32(w + 12);
75 vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi2x0123, vk2x0123);
76
77 const psimd_f32 vi3x0123 = psimd_load_f32(i3);
78 i3 += 4;
79
80 const psimd_f32 vk3x0123 = psimd_load_f32(w + 16);
81 vacc0123p1 = psimd_qfma_f32(vacc0123p1, vi3x0123, vk3x0123);
82
83 const psimd_f32 vi4x0123 = psimd_load_f32(i4);
84 i4 += 4;
85
86 const psimd_f32 vk4x0123 = psimd_load_f32(w + 20);
87 vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi4x0123, vk4x0123);
88
89 const psimd_f32 vi5x0123 = psimd_load_f32(i5);
90 i5 += 4;
91
92 const psimd_f32 vk5x0123 = psimd_load_f32(w + 24);
93 vacc0123p1 = psimd_qfma_f32(vacc0123p1, vi5x0123, vk5x0123);
94
95 const psimd_f32 vi6x0123 = psimd_load_f32(i6);
96 i6 += 4;
97
98 const psimd_f32 vk6x0123 = psimd_load_f32(w + 28);
99 vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi6x0123, vk6x0123);
100
101 const psimd_f32 vi7x0123 = psimd_load_f32(i7);
102 i7 += 4;
103
104 const psimd_f32 vk7x0123 = psimd_load_f32(w + 32);
105 vacc0123p1 = psimd_qfma_f32(vacc0123p1, vi7x0123, vk7x0123);
106
107 const psimd_f32 vi8x0123 = psimd_load_f32(i8);
108 i8 += 4;
109
110 const psimd_f32 vk8x0123 = psimd_load_f32(w + 36);
111 vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi8x0123, vk8x0123);
112
113 w += 40;
114
115 // Add up all accumulators to vacc0123p0
116 vacc0123p0 = psimd_add_f32(vacc0123p0, vacc0123p1);
117
118 psimd_f32 vacc0123 = psimd_max_f32(vacc0123p0, vmin);
119 vacc0123 = psimd_min_f32(vacc0123, vmax);
120
121 psimd_store_f32(output, vacc0123);
122 output += 4;
123 }
124 if XNN_UNLIKELY(c != 0) {
125 psimd_f32 vacc0123p0 = psimd_load_f32(w);
126
127 const psimd_f32 vi0x0123 = psimd_load_f32(i0);
128 const psimd_f32 vk0x0123 = psimd_load_f32(w + 4);
129 vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi0x0123, vk0x0123);
130
131 const psimd_f32 vi1x0123 = psimd_load_f32(i1);
132 const psimd_f32 vk1x0123 = psimd_load_f32(w + 8);
133 psimd_f32 vacc0123p1 = psimd_mul_f32(vi1x0123, vk1x0123);
134
135 const psimd_f32 vi2x0123 = psimd_load_f32(i2);
136 const psimd_f32 vk2x0123 = psimd_load_f32(w + 12);
137 vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi2x0123, vk2x0123);
138
139 const psimd_f32 vi3x0123 = psimd_load_f32(i3);
140 const psimd_f32 vk3x0123 = psimd_load_f32(w + 16);
141 vacc0123p1 = psimd_qfma_f32(vacc0123p1, vi3x0123, vk3x0123);
142
143 const psimd_f32 vi4x0123 = psimd_load_f32(i4);
144 const psimd_f32 vk4x0123 = psimd_load_f32(w + 20);
145 vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi4x0123, vk4x0123);
146
147 const psimd_f32 vi5x0123 = psimd_load_f32(i5);
148 const psimd_f32 vk5x0123 = psimd_load_f32(w + 24);
149 vacc0123p1 = psimd_qfma_f32(vacc0123p1, vi5x0123, vk5x0123);
150
151 const psimd_f32 vi6x0123 = psimd_load_f32(i6);
152 const psimd_f32 vk6x0123 = psimd_load_f32(w + 28);
153 vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi6x0123, vk6x0123);
154
155 const psimd_f32 vi7x0123 = psimd_load_f32(i7);
156 const psimd_f32 vk7x0123 = psimd_load_f32(w + 32);
157 vacc0123p1 = psimd_qfma_f32(vacc0123p1, vi7x0123, vk7x0123);
158
159 const psimd_f32 vi8x0123 = psimd_load_f32(i8);
160 const psimd_f32 vk8x0123 = psimd_load_f32(w + 36);
161 vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi8x0123, vk8x0123);
162
163 // Add up all accumulators to vacc0123p0
164 vacc0123p0 = psimd_add_f32(vacc0123p0, vacc0123p1);
165
166 psimd_f32 vacc0123 = psimd_max_f32(vacc0123p0, vmin);
167 vacc0123 = psimd_min_f32(vacc0123, vmax);
168
169 if (c & 2) {
170 psimd_store2_f32(output, vacc0123);
171 vacc0123 = psimd_concat_hi_f32(vacc0123, vacc0123);
172 output += 2;
173 }
174 if (c & 1) {
175 psimd_store1_f32(output, vacc0123);
176 output += 1;
177 }
178 }
179
180 output = (float*) ((uintptr_t) output + output_increment);
181 } while (--output_width != 0);
182 }
183