1 // Copyright (c) Facebook, Inc. and its affiliates.
2 // All rights reserved.
3 //
4 // This source code is licensed under the BSD-style license found in the
5 // LICENSE file in the root directory of this source tree.
6
7 #include <algorithm>
8 #include <cmath>
9 #include <functional>
10 #include <random>
11 #include <vector>
12
13 #include <xnnpack.h>
14
15 #include <benchmark/benchmark.h>
16 #include "bench/utils.h"
17 #ifdef BENCHMARK_TENSORFLOW_LITE
18 #include "flatbuffers/include/flatbuffers/flatbuffers.h"
19 #include "tensorflow/lite/interpreter.h"
20 #include "tensorflow/lite/kernels/register.h"
21 #include "tensorflow/lite/model.h"
22 #include "tensorflow/lite/schema/schema_generated.h"
23 #include "tensorflow/lite/version.h"
24 #endif // BENCHMARK_TENSORFLOW_LITE
25
26
xnnpack_softmax_q8(benchmark::State & state)27 static void xnnpack_softmax_q8(benchmark::State& state) {
28 const size_t batch_size = static_cast<size_t>(state.range(0));
29 const size_t channels = static_cast<size_t>(state.range(1));
30
31 std::random_device random_device;
32 auto rng = std::mt19937(random_device());
33 auto u8rng = std::bind(std::uniform_int_distribution<uint8_t>(), rng);
34
35 std::vector<uint8_t> input(batch_size * channels);
36 std::vector<uint8_t> output(batch_size * channels);
37 std::generate(input.begin(), input.end(), std::ref(u8rng));
38 std::fill(output.begin(), output.end(), 0xA5);
39
40 xnn_status status = xnn_initialize(nullptr /* allocator */);
41 if (status != xnn_status_success) {
42 state.SkipWithError("failed to initialize XNNPACK");
43 return;
44 }
45
46 xnn_operator_t softmax_op = nullptr;
47 status = xnn_create_softmax_nc_q8(
48 channels, channels /* input stride */, channels /* output stride */,
49 1.0f /* input scale */,
50 0 /* output zero point */, 1.0f / 256.0f /* output scale */,
51 0 /* flags */, &softmax_op);
52 if (status != xnn_status_success || softmax_op == nullptr) {
53 state.SkipWithError("failed to create SoftMax operator");
54 return;
55 }
56
57 status = xnn_setup_softmax_nc_q8(
58 softmax_op,
59 batch_size,
60 input.data(), output.data(),
61 nullptr /* thread pool */);
62 if (status != xnn_status_success) {
63 state.SkipWithError("failed to setup SoftMax operator");
64 return;
65 }
66
67 for (auto _ : state) {
68 status = xnn_run_operator(softmax_op, nullptr /* thread pool */);
69 if (status != xnn_status_success) {
70 state.SkipWithError("failed to run SoftMax operator");
71 return;
72 }
73 }
74
75 status = xnn_delete_operator(softmax_op);
76 if (status != xnn_status_success) {
77 state.SkipWithError("failed to delete SoftMax operator");
78 return;
79 }
80
81 state.counters["Freq"] = benchmark::utils::GetCurrentCpuFrequency();
82
83 const size_t elements_per_iteration = batch_size * channels;
84 state.counters["elements"] =
85 benchmark::Counter(uint64_t(state.iterations()) * elements_per_iteration, benchmark::Counter::kIsRate);
86
87 const size_t bytes_per_iteration = 2 * elements_per_iteration * sizeof(uint8_t);
88 state.counters["bytes"] =
89 benchmark::Counter(uint64_t(state.iterations()) * bytes_per_iteration, benchmark::Counter::kIsRate);
90 }
91
xnnpack_softmax_f32(benchmark::State & state)92 static void xnnpack_softmax_f32(benchmark::State& state) {
93 const size_t batch_size = static_cast<size_t>(state.range(0));
94 const size_t channels = static_cast<size_t>(state.range(1));
95
96 std::random_device random_device;
97 auto rng = std::mt19937(random_device());
98 auto f32rng = std::bind(std::uniform_real_distribution<float>(-100.0f, 100.0f), rng);
99
100 std::vector<float> input(batch_size * channels + XNN_EXTRA_BYTES / sizeof(float));
101 std::vector<float> output(batch_size * channels);
102 std::generate(input.begin(), input.end(), std::ref(f32rng));
103 std::fill(output.begin(), output.end(), std::nanf(""));
104
105 xnn_status status = xnn_initialize(nullptr /* allocator */);
106 if (status != xnn_status_success) {
107 state.SkipWithError("failed to initialize XNNPACK");
108 return;
109 }
110
111 xnn_operator_t softmax_op = nullptr;
112 status = xnn_create_softmax_nc_f32(
113 channels, channels /* input stride */, channels /* output stride */,
114 0 /* flags */, &softmax_op);
115 if (status != xnn_status_success || softmax_op == nullptr) {
116 state.SkipWithError("failed to create SoftMax operator");
117 return;
118 }
119
120 status = xnn_setup_softmax_nc_f32(
121 softmax_op,
122 batch_size,
123 input.data(), output.data(),
124 nullptr /* thread pool */);
125 if (status != xnn_status_success) {
126 state.SkipWithError("failed to setup SoftMax operator");
127 return;
128 }
129
130 for (auto _ : state) {
131 status = xnn_run_operator(softmax_op, nullptr /* thread pool */);
132 if (status != xnn_status_success) {
133 state.SkipWithError("failed to run SoftMax operator");
134 return;
135 }
136 }
137
138 status = xnn_delete_operator(softmax_op);
139 if (status != xnn_status_success) {
140 state.SkipWithError("failed to delete SoftMax operator");
141 return;
142 }
143
144 state.counters["Freq"] = benchmark::utils::GetCurrentCpuFrequency();
145
146 const size_t elements_per_iteration = batch_size * channels;
147 state.counters["elements"] =
148 benchmark::Counter(uint64_t(state.iterations()) * elements_per_iteration, benchmark::Counter::kIsRate);
149
150 const size_t bytes_per_iteration = 2 * elements_per_iteration * sizeof(float);
151 state.counters["bytes"] =
152 benchmark::Counter(uint64_t(state.iterations()) * bytes_per_iteration, benchmark::Counter::kIsRate);
153 }
154
155 #ifdef BENCHMARK_TENSORFLOW_LITE
tflite_softmax_f32(benchmark::State & state)156 static void tflite_softmax_f32(benchmark::State& state) {
157 const size_t batch_size = state.range(0);
158 const size_t channels = state.range(1);
159
160 std::random_device random_device;
161 auto rng = std::mt19937(random_device());
162 auto f32rng = std::bind(std::uniform_real_distribution<float>(-100.0f, 100.0f), rng);
163
164 flatbuffers::FlatBufferBuilder builder;
165 flatbuffers::Offset<tflite::OperatorCode> operator_code =
166 tflite::CreateOperatorCode(builder, tflite::BuiltinOperator_SOFTMAX);
167
168 flatbuffers::Offset<tflite::SoftmaxOptions> softmax_options =
169 tflite::CreateSoftmaxOptions(builder, 1.0f /* beta */);
170
171 flatbuffers::Offset<tflite::Buffer> buffers[1] = {
172 tflite::CreateBuffer(builder, builder.CreateVector({})),
173 };
174
175 const int32_t input_shape[4] = {
176 static_cast<int32_t>(batch_size),
177 static_cast<int32_t>(1 /* height */),
178 static_cast<int32_t>(1 /* width */),
179 static_cast<int32_t>(channels)
180 };
181 const int32_t output_shape[4] = {
182 static_cast<int32_t>(batch_size),
183 static_cast<int32_t>(1 /* height */),
184 static_cast<int32_t>(1 /* width */),
185 static_cast<int32_t>(channels)
186 };
187
188 flatbuffers::Offset<tflite::Tensor> tensors[2] = {
189 tflite::CreateTensor(builder,
190 builder.CreateVector<int32_t>(input_shape, 4),
191 tflite::TensorType_FLOAT32),
192 tflite::CreateTensor(builder,
193 builder.CreateVector<int32_t>(output_shape, 4),
194 tflite::TensorType_FLOAT32),
195 };
196
197 const int32_t op_inputs[1] = { 0 };
198 const int32_t op_outputs[1] = { 1 };
199 flatbuffers::Offset<tflite::Operator> op = tflite::CreateOperator(
200 builder,
201 0 /* opcode_index */,
202 builder.CreateVector<int32_t>(op_inputs, 1),
203 builder.CreateVector<int32_t>(op_outputs, 1),
204 tflite::BuiltinOptions_SoftmaxOptions, softmax_options.Union());
205
206 const int32_t graph_inputs[1] = { 0 };
207 const int32_t graph_outputs[1] = { 1 };
208 flatbuffers::Offset<tflite::SubGraph> subgraph = tflite::CreateSubGraph(
209 builder,
210 builder.CreateVector(tensors, 2),
211 builder.CreateVector<int32_t>(graph_inputs, 1),
212 builder.CreateVector<int32_t>(graph_outputs, 1),
213 builder.CreateVector(&op, 1));
214
215 flatbuffers::Offset<flatbuffers::String> description = builder.CreateString("Softmax model");
216
217 flatbuffers::Offset<tflite::Model> model_buffer = tflite::CreateModel(builder,
218 TFLITE_SCHEMA_VERSION,
219 builder.CreateVector(&operator_code, 1),
220 builder.CreateVector(&subgraph, 1),
221 description,
222 builder.CreateVector(buffers, 1));
223
224 builder.Finish(model_buffer);
225
226 const tflite::Model* model = tflite::GetModel(builder.GetBufferPointer());
227 tflite::ops::builtin::BuiltinOpResolver resolver;
228 tflite::InterpreterBuilder interpreterBuilder(model, resolver);
229 std::unique_ptr<tflite::Interpreter> interpreter;
230 if (interpreterBuilder(&interpreter) != kTfLiteOk) {
231 state.SkipWithError("failed to create TFLite interpreter");
232 return;
233 }
234 if (interpreter == nullptr) {
235 state.SkipWithError("TFLite interpreter is null");
236 return;
237 }
238 interpreter->SetNumThreads(1);
239
240 if (interpreter->AllocateTensors() != kTfLiteOk) {
241 state.SkipWithError("failed to allocate tensors");
242 return;
243 }
244
245 std::generate(
246 interpreter->typed_tensor<float>(0),
247 interpreter->typed_tensor<float>(0) + batch_size * channels,
248 std::ref(f32rng));
249
250 for (auto _ : state) {
251 if (interpreter->Invoke() != kTfLiteOk) {
252 state.SkipWithError("failed to invoke TFLite interpreter");
253 return;
254 }
255 }
256
257 state.counters["Freq"] = benchmark::utils::GetCurrentCpuFrequency();
258
259 const size_t elements_per_iteration = batch_size * channels;
260 state.counters["elements"] =
261 benchmark::Counter(uint64_t(state.iterations()) * elements_per_iteration, benchmark::Counter::kIsRate);
262
263 const size_t bytes_per_iteration = 2 * elements_per_iteration * sizeof(float);
264 state.counters["bytes"] =
265 benchmark::Counter(uint64_t(state.iterations()) * bytes_per_iteration, benchmark::Counter::kIsRate);
266
267 interpreter.reset();
268 }
269 #endif // BENCHMARK_TENSORFLOW_LITE
270
CharacteristicArguments(benchmark::internal::Benchmark * b)271 static void CharacteristicArguments(benchmark::internal::Benchmark* b)
272 {
273 b->ArgNames({"N", "C"});
274
275 // CIFAR-10
276 b->Args({1, 10});
277 // CIFAR-100 */
278 b->Args({1, 100});
279 // ImageNet-1K
280 b->Args({1, 1000});
281 // ImageNet-1K+1
282 b->Args({1, 1001});
283 // ImageNet-22K
284 b->Args({1, 21841});
285 }
286
287 BENCHMARK(xnnpack_softmax_q8)->Apply(CharacteristicArguments)->UseRealTime();
288 BENCHMARK(xnnpack_softmax_f32)->Apply(CharacteristicArguments)->UseRealTime();
289 #ifdef BENCHMARK_TENSORFLOW_LITE
290 BENCHMARK(tflite_softmax_f32)->Apply(CharacteristicArguments)->UseRealTime();
291 #endif // BENCHMARK_TENSORFLOW_LITE
292
293 #ifndef XNNPACK_BENCHMARK_NO_MAIN
294 BENCHMARK_MAIN();
295 #endif
296