1 // Copyright 2019 Google LLC
2 //
3 // This source code is licensed under the BSD-style license found in the
4 // LICENSE file in the root directory of this source tree.
5
6 #include <algorithm>
7 #include <cfloat>
8 #include <cmath>
9 #include <functional>
10 #include <random>
11 #include <vector>
12
13 #include <cpuinfo.h>
14
15 #include <benchmark/benchmark.h>
16 #include "bench/dwconv.h"
17 #include "bench/utils.h"
18 #include <xnnpack/AlignedAllocator.h>
19 #include <xnnpack/common.h>
20 #include <xnnpack/dwconv.h>
21 #include <xnnpack/indirection.h>
22 #include <xnnpack/operator.h>
23 #include <xnnpack/pack.h>
24 #include <xnnpack/params-init.h>
25 #include <xnnpack/params.h>
26
27
DWConvBenchmark(benchmark::State & state,xnn_f32_dwconv_up_ukernel_function dwconv,uint32_t cr,uint32_t kr,benchmark::utils::IsaCheckFunction isa_check=nullptr)28 static void DWConvBenchmark(benchmark::State& state,
29 xnn_f32_dwconv_up_ukernel_function dwconv,
30 uint32_t cr, uint32_t kr,
31 benchmark::utils::IsaCheckFunction isa_check = nullptr)
32 {
33 if (!cpuinfo_initialize()) {
34 state.SkipWithError("cpuinfo initialization failed");
35 return;
36 }
37 if (isa_check && !isa_check(state)) {
38 return;
39 }
40
41 const size_t input_height = state.range(0);
42 const size_t input_width = state.range(1);
43 const size_t kernel_height = state.range(2);
44 const size_t kernel_width = state.range(3);
45 const size_t padding_height = state.range(4);
46 const size_t padding_width = state.range(5);
47 const size_t subsampling = state.range(6);
48 const size_t dilation = state.range(7);
49 const size_t channels = state.range(8);
50
51 const size_t kernel_size = kernel_height * kernel_width;
52 if (kernel_size != kr) {
53 state.SkipWithError("kernel size mismatch");
54 return;
55 }
56
57 std::random_device random_device;
58 auto rng = std::mt19937(random_device());
59 auto f32rng = std::bind(std::uniform_real_distribution<float>(0.0f, 1.0f), rng);
60
61 const size_t effective_kernel_height = (kernel_height - 1) * dilation + 1;
62 const size_t effective_kernel_width = (kernel_width - 1) * dilation + 1;
63 const size_t padding_left = padding_width / 2;
64 const size_t padding_top = padding_height / 2;
65 const size_t output_height = (input_height + padding_height - effective_kernel_height) / subsampling + 1;
66 const size_t output_width = (input_width + padding_width - effective_kernel_width) / subsampling + 1;
67 const size_t output_size = output_height * output_width;
68 const size_t step_width = dilation == 1 ? subsampling : kernel_width;
69 const size_t step_height = kernel_size + (output_width - 1) * step_width * kernel_height;
70
71 const size_t c_stride = benchmark::utils::RoundUp<size_t>(channels, cr);
72
73 std::vector<float> a(channels * input_height * input_width + XNN_EXTRA_BYTES / sizeof(float));
74 std::generate(a.begin(), a.end(), std::ref(f32rng));
75 std::vector<float> k(channels * kernel_height * kernel_width);
76 std::generate(k.begin(), k.end(), std::ref(f32rng));
77 std::vector<float> b(channels);
78 std::generate(b.begin(), b.end(), std::ref(f32rng));
79
80 std::vector<float> z(channels + XNN_EXTRA_BYTES / sizeof(float));
81
82 const size_t w_elements = (kernel_size + 1) * c_stride;
83 const size_t i_elements = output_height * step_height;
84 const size_t c_elements = output_size * channels;
85 const size_t num_buffers = 1 +
86 benchmark::utils::DivideRoundUp<size_t>(benchmark::utils::GetMaxCacheSize(),
87 sizeof(float) * (w_elements + c_elements) + sizeof(void*) * i_elements);
88
89 std::vector<float, AlignedAllocator<float, 32>> w(w_elements * num_buffers);
90 std::fill(w.begin(), w.end(), 0.0f);
91 xnn_pack_f32_dwconv_ghw_w(kernel_height, kernel_width, channels, cr,
92 k.data(), b.data(), w.data());
93 for (size_t n = 1; n < num_buffers; n++) {
94 std::copy(w.cbegin(), w.cbegin() + w_elements, w.begin() + n * w_elements);
95 }
96
97 std::vector<const float*> i(i_elements * num_buffers);
98 xnn_operator convolution_op = { };
99 convolution_op.indirection_buffer = reinterpret_cast<const void**>(i.data());
100 convolution_op.input = a.data();
101 convolution_op.input_pixel_stride = channels;
102 convolution_op.zero_buffer = z.data();
103 convolution_op.batch_size = 1;
104 convolution_op.input_height = input_height;
105 convolution_op.input_width = input_width;
106 convolution_op.output_height = output_height;
107 convolution_op.output_width = output_width;
108 convolution_op.kernel_height = kernel_height;
109 convolution_op.kernel_width = kernel_width;
110 convolution_op.stride_height = subsampling;
111 convolution_op.stride_width = subsampling;
112 convolution_op.dilation_height = dilation;
113 convolution_op.dilation_width = dilation;
114 convolution_op.padding_top = padding_top;
115 convolution_op.padding_left = padding_left;
116
117 xnn_indirection_init_dwconv2d(&convolution_op, 0, step_height, step_width, 2 /* log2(sizeof(float)) */);
118 for (size_t n = 1; n < num_buffers; n++) {
119 std::copy(i.cbegin(), i.cbegin() + i_elements, i.begin() + n * i_elements);
120 }
121
122 std::vector<float> c(c_elements * num_buffers);
123 std::fill(c.begin(), c.end(), std::nanf(""));
124
125 xnn_f32_output_params output_params =
126 xnn_init_f32_output_params(-std::numeric_limits<float>::infinity(), +std::numeric_limits<float>::infinity());
127
128 size_t buffer_index = 0;
129 for (auto _ : state) {
130 state.PauseTiming();
131 benchmark::utils::PrefetchToL1(a.data(), a.size() * sizeof(float));
132 buffer_index = (buffer_index + 1) % num_buffers;
133 state.ResumeTiming();
134
135 for (uint32_t y = 0; y < output_height; y++) {
136 dwconv(channels, output_width,
137 i.data() + buffer_index * i_elements + step_height * y,
138 w.data() + buffer_index * w_elements,
139 c.data() + buffer_index * c_elements + y * output_width * channels,
140 kernel_height * step_width * sizeof(void*), 0,
141 &output_params);
142 }
143 }
144
145 state.counters["Freq"] = benchmark::utils::GetCurrentCpuFrequency();
146 state.counters["FLOPS"] = benchmark::Counter(
147 uint64_t(state.iterations()) * 2 * output_size * channels * kernel_size,
148 benchmark::Counter::kIsRate);
149
150 state.counters["BYTES"] = benchmark::Counter(
151 uint64_t(state.iterations()) * (output_size + input_height * input_width + kernel_size + 1 /* bias */) * channels * sizeof(float),
152 benchmark::Counter::kIsRate);
153 }
154
155 #if XNN_ARCH_ARM64 && XNN_ENABLE_ASSEMBLY
f32_dwconv_4x9__aarch64_neonfma(benchmark::State & state,const char * net)156 static void f32_dwconv_4x9__aarch64_neonfma(benchmark::State& state, const char* net) {
157 DWConvBenchmark(state, xnn_f32_dwconv_ukernel_up4x9__neon, 4, 9);
158 }
159
f32_dwconv_4x9__aarch64_neonfma_cortex_a55(benchmark::State & state,const char * net)160 static void f32_dwconv_4x9__aarch64_neonfma_cortex_a55(benchmark::State& state, const char* net) {
161 DWConvBenchmark(state, xnn_f32_dwconv_ukernel_up4x9__neonfma, 4, 9);
162 }
163
164 BENCHMARK_DWCONV(f32_dwconv_4x9__aarch64_neonfma)
BENCHMARK_DWCONV(f32_dwconv_4x9__aarch64_neonfma_cortex_a55)165 BENCHMARK_DWCONV(f32_dwconv_4x9__aarch64_neonfma_cortex_a55)
166 #endif // XNN_ARCH_ARM64
167
168
169 #if XNN_ARCH_ARM || XNN_ARCH_ARM64
170 static void f32_dwconv_4x9__neon(benchmark::State& state, const char* net) {
171 DWConvBenchmark(state, xnn_f32_dwconv_ukernel_up4x9__neon, 4, 9,
172 benchmark::utils::CheckNEON);
173 }
174
f32_dwconv_4x9__neonfma(benchmark::State & state,const char * net)175 static void f32_dwconv_4x9__neonfma(benchmark::State& state, const char* net) {
176 DWConvBenchmark(state, xnn_f32_dwconv_ukernel_up4x9__neonfma, 4, 9,
177 benchmark::utils::CheckNEONFMA);
178 }
179
f32_dwconv_8x9__neonfma(benchmark::State & state,const char * net)180 static void f32_dwconv_8x9__neonfma(benchmark::State& state, const char* net) {
181 DWConvBenchmark(state, xnn_f32_dwconv_ukernel_up8x9__neonfma, 8, 9,
182 benchmark::utils::CheckNEONFMA);
183 }
184
185 BENCHMARK_DWCONV(f32_dwconv_4x9__neon)
BENCHMARK_DWCONV(f32_dwconv_4x9__neonfma)186 BENCHMARK_DWCONV(f32_dwconv_4x9__neonfma)
187 BENCHMARK_DWCONV(f32_dwconv_8x9__neonfma)
188 #endif // XNN_ARCH_ARM || XNN_ARCH_ARM64
189
190
191 #if XNN_ARCH_X86 || XNN_ARCH_X86_64
192 static void f32_dwconv_4x4__sse(benchmark::State& state, const char* net) {
193 DWConvBenchmark(state, xnn_f32_dwconv_ukernel_up4x4__sse, 4, 4);
194 }
195
f32_dwconv_4x9__sse(benchmark::State & state,const char * net)196 static void f32_dwconv_4x9__sse(benchmark::State& state, const char* net) {
197 DWConvBenchmark(state, xnn_f32_dwconv_ukernel_up4x9__sse, 4, 9);
198 }
199
f32_dwconv_4x25__sse(benchmark::State & state,const char * net)200 static void f32_dwconv_4x25__sse(benchmark::State& state, const char* net) {
201 DWConvBenchmark(state, xnn_f32_dwconv_ukernel_up4x25__sse, 4, 25);
202 }
203
204 BENCHMARK_DWCONV(f32_dwconv_4x4__sse)
BENCHMARK_DWCONV(f32_dwconv_4x9__sse)205 BENCHMARK_DWCONV(f32_dwconv_4x9__sse)
206 BENCHMARK_DWCONV(f32_dwconv_4x25__sse)
207 #endif // XNN_ARCH_X86 || XNN_ARCH_X86_64
208
209
210 #if !XNN_ARCH_WASM && !XNN_ARCH_ASMJS
211 static void f32_dwconv_4x4__psimd(benchmark::State& state, const char* net) {
212 DWConvBenchmark(state, xnn_f32_dwconv_ukernel_up4x4__psimd, 4, 4);
213 }
214
f32_dwconv_4x9__psimd(benchmark::State & state,const char * net)215 static void f32_dwconv_4x9__psimd(benchmark::State& state, const char* net) {
216 DWConvBenchmark(state, xnn_f32_dwconv_ukernel_up4x9__psimd, 4, 9);
217 }
218
f32_dwconv_4x25__psimd(benchmark::State & state,const char * net)219 static void f32_dwconv_4x25__psimd(benchmark::State& state, const char* net) {
220 DWConvBenchmark(state, xnn_f32_dwconv_ukernel_up4x25__psimd, 4, 25);
221 }
222
223 BENCHMARK_DWCONV(f32_dwconv_4x4__psimd)
BENCHMARK_DWCONV(f32_dwconv_4x9__psimd)224 BENCHMARK_DWCONV(f32_dwconv_4x9__psimd)
225 BENCHMARK_DWCONV(f32_dwconv_4x25__psimd)
226 #endif // !XNN_ARCH_WASM && !XNN_ARCH_ASMJS
227
228
229 static void f32_dwconv_1x4__scalar(benchmark::State& state, const char* net) {
230 DWConvBenchmark(state, xnn_f32_dwconv_ukernel_up1x4__scalar, 1, 4);
231 }
232
f32_dwconv_1x9__scalar(benchmark::State & state,const char * net)233 static void f32_dwconv_1x9__scalar(benchmark::State& state, const char* net) {
234 DWConvBenchmark(state, xnn_f32_dwconv_ukernel_up1x9__scalar, 1, 9);
235 }
236
f32_dwconv_1x25__scalar(benchmark::State & state,const char * net)237 static void f32_dwconv_1x25__scalar(benchmark::State& state, const char* net) {
238 DWConvBenchmark(state, xnn_f32_dwconv_ukernel_up1x25__scalar, 1, 25);
239 }
240
241 BENCHMARK_DWCONV(f32_dwconv_1x4__scalar)
242 BENCHMARK_DWCONV(f32_dwconv_1x9__scalar)
243 BENCHMARK_DWCONV(f32_dwconv_1x25__scalar)
244
245 #ifndef XNNPACK_BENCHMARK_NO_MAIN
246 BENCHMARK_MAIN();
247 #endif
248