1 // Copyright (c) Facebook, Inc. and its affiliates.
2 // All rights reserved.
3 //
4 // This source code is licensed under the BSD-style license found in the
5 // LICENSE file in the root directory of this source tree.
6
7 #include <algorithm>
8 #include <cmath>
9 #include <functional>
10 #include <random>
11 #include <vector>
12
13 #include <xnnpack.h>
14
15 #include <benchmark/benchmark.h>
16 #include "bench/utils.h"
17 #ifdef BENCHMARK_TENSORFLOW_LITE
18 #include "flatbuffers/include/flatbuffers/flatbuffers.h"
19 #include "tensorflow/lite/interpreter.h"
20 #include "tensorflow/lite/kernels/register.h"
21 #include "tensorflow/lite/model.h"
22 #include "tensorflow/lite/schema/schema_generated.h"
23 #include "tensorflow/lite/version.h"
24 #endif // BENCHMARK_TENSORFLOW_LITE
25
26 #ifndef XNN_NO_QU8_OPERATORS
xnnpack_softmax_qu8(benchmark::State & state)27 static void xnnpack_softmax_qu8(benchmark::State& state) {
28 const size_t batch_size = static_cast<size_t>(state.range(0));
29 const size_t channels = static_cast<size_t>(state.range(1));
30
31 std::random_device random_device;
32 auto rng = std::mt19937(random_device());
33 auto u8rng = std::bind(std::uniform_int_distribution<uint32_t>(0, std::numeric_limits<uint8_t>::max()), std::ref(rng));
34
35 std::vector<uint8_t> input(batch_size * channels);
36 std::vector<uint8_t> output(batch_size * channels);
37 std::generate(input.begin(), input.end(), std::ref(u8rng));
38 std::fill(output.begin(), output.end(), 0xA5);
39
40 xnn_status status = xnn_initialize(nullptr /* allocator */);
41 if (status != xnn_status_success) {
42 state.SkipWithError("failed to initialize XNNPACK");
43 return;
44 }
45
46 xnn_operator_t softmax_op = nullptr;
47 status = xnn_create_softmax_nc_qu8(
48 channels, channels /* input stride */, channels /* output stride */,
49 1.0f /* input scale */,
50 0 /* output zero point */, 1.0f / 256.0f /* output scale */,
51 0 /* flags */, &softmax_op);
52 if (status != xnn_status_success || softmax_op == nullptr) {
53 state.SkipWithError("failed to create SoftMax operator");
54 return;
55 }
56
57 status = xnn_setup_softmax_nc_qu8(
58 softmax_op,
59 batch_size,
60 input.data(), output.data(),
61 nullptr /* thread pool */);
62 if (status != xnn_status_success) {
63 state.SkipWithError("failed to setup SoftMax operator");
64 return;
65 }
66
67 for (auto _ : state) {
68 status = xnn_run_operator(softmax_op, nullptr /* thread pool */);
69 if (status != xnn_status_success) {
70 state.SkipWithError("failed to run SoftMax operator");
71 return;
72 }
73 }
74
75 status = xnn_delete_operator(softmax_op);
76 if (status != xnn_status_success) {
77 state.SkipWithError("failed to delete SoftMax operator");
78 return;
79 }
80
81 const uint64_t cpu_frequency = benchmark::utils::GetCurrentCpuFrequency();
82 if (cpu_frequency != 0) {
83 state.counters["cpufreq"] = cpu_frequency;
84 }
85
86 const size_t elements_per_iteration = batch_size * channels;
87 state.counters["elements"] =
88 benchmark::Counter(uint64_t(state.iterations()) * elements_per_iteration, benchmark::Counter::kIsRate);
89
90 const size_t bytes_per_iteration = 2 * elements_per_iteration * sizeof(uint8_t);
91 state.counters["bytes"] =
92 benchmark::Counter(uint64_t(state.iterations()) * bytes_per_iteration, benchmark::Counter::kIsRate);
93 }
94
xnnpack_softmax_f32(benchmark::State & state)95 static void xnnpack_softmax_f32(benchmark::State& state) {
96 const size_t batch_size = static_cast<size_t>(state.range(0));
97 const size_t channels = static_cast<size_t>(state.range(1));
98
99 std::random_device random_device;
100 auto rng = std::mt19937(random_device());
101 auto f32rng = std::bind(std::uniform_real_distribution<float>(-100.0f, 100.0f), std::ref(rng));
102
103 std::vector<float> input(batch_size * channels + XNN_EXTRA_BYTES / sizeof(float));
104 std::vector<float> output(batch_size * channels);
105 std::generate(input.begin(), input.end(), std::ref(f32rng));
106 std::fill(output.begin(), output.end(), std::nanf(""));
107
108 xnn_status status = xnn_initialize(nullptr /* allocator */);
109 if (status != xnn_status_success) {
110 state.SkipWithError("failed to initialize XNNPACK");
111 return;
112 }
113
114 xnn_operator_t softmax_op = nullptr;
115 status = xnn_create_softmax_nc_f32(
116 channels, channels /* input stride */, channels /* output stride */,
117 0 /* flags */, &softmax_op);
118 if (status != xnn_status_success || softmax_op == nullptr) {
119 state.SkipWithError("failed to create SoftMax operator");
120 return;
121 }
122
123 status = xnn_setup_softmax_nc_f32(
124 softmax_op,
125 batch_size,
126 input.data(), output.data(),
127 nullptr /* thread pool */);
128 if (status != xnn_status_success) {
129 state.SkipWithError("failed to setup SoftMax operator");
130 return;
131 }
132
133 for (auto _ : state) {
134 status = xnn_run_operator(softmax_op, nullptr /* thread pool */);
135 if (status != xnn_status_success) {
136 state.SkipWithError("failed to run SoftMax operator");
137 return;
138 }
139 }
140
141 status = xnn_delete_operator(softmax_op);
142 if (status != xnn_status_success) {
143 state.SkipWithError("failed to delete SoftMax operator");
144 return;
145 }
146
147 const uint64_t cpu_frequency = benchmark::utils::GetCurrentCpuFrequency();
148 if (cpu_frequency != 0) {
149 state.counters["cpufreq"] = cpu_frequency;
150 }
151
152 const size_t elements_per_iteration = batch_size * channels;
153 state.counters["elements"] =
154 benchmark::Counter(uint64_t(state.iterations()) * elements_per_iteration, benchmark::Counter::kIsRate);
155
156 const size_t bytes_per_iteration = 2 * elements_per_iteration * sizeof(float);
157 state.counters["bytes"] =
158 benchmark::Counter(uint64_t(state.iterations()) * bytes_per_iteration, benchmark::Counter::kIsRate);
159 }
160 #endif // XNN_NO_QU8_OPERATORS
161
162 #ifdef BENCHMARK_TENSORFLOW_LITE
tflite_softmax_f32(benchmark::State & state)163 static void tflite_softmax_f32(benchmark::State& state) {
164 const size_t batch_size = state.range(0);
165 const size_t channels = state.range(1);
166
167 std::random_device random_device;
168 auto rng = std::mt19937(random_device());
169 auto f32rng = std::bind(std::uniform_real_distribution<float>(-100.0f, 100.0f), std::ref(rng));
170
171 flatbuffers::FlatBufferBuilder builder;
172 flatbuffers::Offset<tflite::OperatorCode> operator_code =
173 tflite::CreateOperatorCode(builder, tflite::BuiltinOperator_SOFTMAX);
174
175 flatbuffers::Offset<tflite::SoftmaxOptions> softmax_options =
176 tflite::CreateSoftmaxOptions(builder, 1.0f /* beta */);
177
178 flatbuffers::Offset<tflite::Buffer> buffers[1] = {
179 tflite::CreateBuffer(builder, builder.CreateVector({})),
180 };
181
182 const int32_t input_shape[4] = {
183 static_cast<int32_t>(batch_size),
184 static_cast<int32_t>(1 /* height */),
185 static_cast<int32_t>(1 /* width */),
186 static_cast<int32_t>(channels)
187 };
188 const int32_t output_shape[4] = {
189 static_cast<int32_t>(batch_size),
190 static_cast<int32_t>(1 /* height */),
191 static_cast<int32_t>(1 /* width */),
192 static_cast<int32_t>(channels)
193 };
194
195 flatbuffers::Offset<tflite::Tensor> tensors[2] = {
196 tflite::CreateTensor(builder,
197 builder.CreateVector<int32_t>(input_shape, 4),
198 tflite::TensorType_FLOAT32),
199 tflite::CreateTensor(builder,
200 builder.CreateVector<int32_t>(output_shape, 4),
201 tflite::TensorType_FLOAT32),
202 };
203
204 const int32_t op_inputs[1] = { 0 };
205 const int32_t op_outputs[1] = { 1 };
206 flatbuffers::Offset<tflite::Operator> op = tflite::CreateOperator(
207 builder,
208 0 /* opcode_index */,
209 builder.CreateVector<int32_t>(op_inputs, 1),
210 builder.CreateVector<int32_t>(op_outputs, 1),
211 tflite::BuiltinOptions_SoftmaxOptions, softmax_options.Union());
212
213 const int32_t graph_inputs[1] = { 0 };
214 const int32_t graph_outputs[1] = { 1 };
215 flatbuffers::Offset<tflite::SubGraph> subgraph = tflite::CreateSubGraph(
216 builder,
217 builder.CreateVector(tensors, 2),
218 builder.CreateVector<int32_t>(graph_inputs, 1),
219 builder.CreateVector<int32_t>(graph_outputs, 1),
220 builder.CreateVector(&op, 1));
221
222 flatbuffers::Offset<flatbuffers::String> description = builder.CreateString("Softmax model");
223
224 flatbuffers::Offset<tflite::Model> model_buffer = tflite::CreateModel(builder,
225 TFLITE_SCHEMA_VERSION,
226 builder.CreateVector(&operator_code, 1),
227 builder.CreateVector(&subgraph, 1),
228 description,
229 builder.CreateVector(buffers, 1));
230
231 builder.Finish(model_buffer);
232
233 const tflite::Model* model = tflite::GetModel(builder.GetBufferPointer());
234 tflite::ops::builtin::BuiltinOpResolver resolver;
235 tflite::InterpreterBuilder interpreterBuilder(model, resolver);
236 std::unique_ptr<tflite::Interpreter> interpreter;
237 if (interpreterBuilder(&interpreter) != kTfLiteOk) {
238 state.SkipWithError("failed to create TFLite interpreter");
239 return;
240 }
241 if (interpreter == nullptr) {
242 state.SkipWithError("TFLite interpreter is null");
243 return;
244 }
245 interpreter->SetNumThreads(1);
246
247 if (interpreter->AllocateTensors() != kTfLiteOk) {
248 state.SkipWithError("failed to allocate tensors");
249 return;
250 }
251
252 std::generate(
253 interpreter->typed_tensor<float>(0),
254 interpreter->typed_tensor<float>(0) + batch_size * channels,
255 std::ref(f32rng));
256
257 for (auto _ : state) {
258 if (interpreter->Invoke() != kTfLiteOk) {
259 state.SkipWithError("failed to invoke TFLite interpreter");
260 return;
261 }
262 }
263
264 const uint64_t cpu_frequency = benchmark::utils::GetCurrentCpuFrequency();
265 if (cpu_frequency != 0) {
266 state.counters["cpufreq"] = cpu_frequency;
267 }
268
269 const size_t elements_per_iteration = batch_size * channels;
270 state.counters["elements"] =
271 benchmark::Counter(uint64_t(state.iterations()) * elements_per_iteration, benchmark::Counter::kIsRate);
272
273 const size_t bytes_per_iteration = 2 * elements_per_iteration * sizeof(float);
274 state.counters["bytes"] =
275 benchmark::Counter(uint64_t(state.iterations()) * bytes_per_iteration, benchmark::Counter::kIsRate);
276
277 interpreter.reset();
278 }
279 #endif // BENCHMARK_TENSORFLOW_LITE
280
CharacteristicArguments(benchmark::internal::Benchmark * b)281 static void CharacteristicArguments(benchmark::internal::Benchmark* b)
282 {
283 b->ArgNames({"N", "C"});
284
285 // CIFAR-10
286 b->Args({1, 10});
287 // CIFAR-100 */
288 b->Args({1, 100});
289 // ImageNet-1K
290 b->Args({1, 1000});
291 // ImageNet-1K+1
292 b->Args({1, 1001});
293 // ImageNet-22K
294 b->Args({1, 21841});
295 // ADE20K
296 b->Args({257 * 257, 151});
297 }
298
299 #ifndef XNN_NO_QU8_OPERATORS
300 BENCHMARK(xnnpack_softmax_qu8)->Apply(CharacteristicArguments)->UseRealTime();
301 #endif // XNN_NO_QU8_OPERATORS
302
303 BENCHMARK(xnnpack_softmax_f32)->Apply(CharacteristicArguments)->UseRealTime();
304 #ifdef BENCHMARK_TENSORFLOW_LITE
305 BENCHMARK(tflite_softmax_f32)->Apply(CharacteristicArguments)->UseRealTime();
306 #endif // BENCHMARK_TENSORFLOW_LITE
307
308 #ifndef XNNPACK_BENCHMARK_NO_MAIN
309 BENCHMARK_MAIN();
310 #endif
311