1 /* Copyright 2020 The TensorFlow Authors. All Rights Reserved.
2
3 Licensed under the Apache License, Version 2.0 (the "License");
4 you may not use this file except in compliance with the License.
5 You may obtain a copy of the License at
6
7 http://www.apache.org/licenses/LICENSE-2.0
8
9 Unless required by applicable law or agreed to in writing, software
10 distributed under the License is distributed on an "AS IS" BASIS,
11 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 See the License for the specific language governing permissions and
13 limitations under the License.
14 ==============================================================================*/
15
16 #include "tensorflow/lite/delegates/xnnpack/depthwise_conv_2d_tester.h"
17
18 #include <array>
19 #include <cstdint>
20 #include <functional>
21 #include <random>
22 #include <vector>
23
24 #include <gtest/gtest.h>
25 #include <fp16.h>
26 #include "flatbuffers/flatbuffers.h" // from @flatbuffers
27 #include "tensorflow/lite/interpreter.h"
28 #include "tensorflow/lite/kernels/register.h"
29 #include "tensorflow/lite/model.h"
30 #include "tensorflow/lite/schema/schema_conversion_utils.h"
31 #include "tensorflow/lite/schema/schema_generated.h"
32 #include "tensorflow/lite/version.h"
33
34 namespace tflite {
35 namespace xnnpack {
36
Test(TfLiteDelegate * delegate) const37 void DepthwiseConv2DTester::Test(TfLiteDelegate* delegate) const {
38 std::vector<char> buffer = CreateTfLiteModel();
39 const Model* model = GetModel(buffer.data());
40
41 std::unique_ptr<Interpreter> delegate_interpreter;
42 ASSERT_EQ(
43 InterpreterBuilder(
44 model,
45 ::tflite::ops::builtin::BuiltinOpResolverWithoutDefaultDelegates())(
46 &delegate_interpreter),
47 kTfLiteOk);
48 std::unique_ptr<Interpreter> default_interpreter;
49 ASSERT_EQ(
50 InterpreterBuilder(
51 model,
52 ::tflite::ops::builtin::BuiltinOpResolverWithoutDefaultDelegates())(
53 &default_interpreter),
54 kTfLiteOk);
55
56 ASSERT_TRUE(delegate_interpreter);
57 ASSERT_TRUE(default_interpreter);
58
59 ASSERT_EQ(delegate_interpreter->inputs().size(), 1);
60 ASSERT_EQ(default_interpreter->inputs().size(), 1);
61
62 ASSERT_EQ(delegate_interpreter->outputs().size(), 1);
63 ASSERT_EQ(default_interpreter->outputs().size(), 1);
64
65 ASSERT_EQ(delegate_interpreter->AllocateTensors(), kTfLiteOk);
66 ASSERT_EQ(default_interpreter->AllocateTensors(), kTfLiteOk);
67
68 ASSERT_EQ(delegate_interpreter->ModifyGraphWithDelegate(delegate), kTfLiteOk);
69
70 std::random_device random_device;
71 auto rng = std::mt19937(random_device());
72 auto input_rng =
73 std::bind(std::uniform_real_distribution<float>(), std::ref(rng));
74 float* default_input_data = default_interpreter->typed_tensor<float>(
75 default_interpreter->inputs()[0]);
76 std::generate(default_input_data,
77 default_input_data + BatchSize() * InputHeight() *
78 InputWidth() * InputChannels(),
79 input_rng);
80
81 float* delegate_input_data = delegate_interpreter->typed_tensor<float>(
82 delegate_interpreter->inputs()[0]);
83 std::copy(default_input_data,
84 default_input_data +
85 BatchSize() * InputHeight() * InputWidth() * InputChannels(),
86 delegate_input_data);
87
88 ASSERT_EQ(default_interpreter->Invoke(), kTfLiteOk);
89 ASSERT_EQ(delegate_interpreter->Invoke(), kTfLiteOk);
90
91 float* default_output_data = default_interpreter->typed_tensor<float>(
92 default_interpreter->outputs()[0]);
93 float* delegate_output_data = delegate_interpreter->typed_tensor<float>(
94 delegate_interpreter->outputs()[0]);
95
96 for (int32_t i = 0; i < BatchSize(); i++) {
97 for (int32_t y = 0; y < OutputHeight(); y++) {
98 for (int32_t x = 0; x < OutputWidth(); x++) {
99 for (int32_t c = 0; c < OutputChannels(); c++) {
100 const int32_t index = ((i * OutputHeight() + y) * OutputWidth() + x) *
101 OutputChannels() +
102 c;
103 ASSERT_NEAR(default_output_data[index], delegate_output_data[index],
104 std::abs(default_output_data[index]) * 3.0e-6f)
105 << "batch " << i << " / " << BatchSize() << ", y position " << y
106 << " / " << OutputHeight() << ", x position " << x << " / "
107 << OutputWidth() << ", channel " << c << " / "
108 << OutputChannels();
109 }
110 }
111 }
112 }
113 }
114
CreateTfLiteModel() const115 std::vector<char> DepthwiseConv2DTester::CreateTfLiteModel() const {
116 std::random_device random_device;
117 auto rng = std::mt19937(random_device());
118 auto range_rng = std::bind(
119 std::uniform_real_distribution<float>(-25.0f, 25.0f), std::ref(rng));
120
121 flatbuffers::FlatBufferBuilder builder;
122 std::vector<flatbuffers::Offset<OperatorCode>> operator_codes{
123 {CreateOperatorCode(builder, BuiltinOperator_DEPTHWISE_CONV_2D)}};
124 std::vector<flatbuffers::Offset<tflite::Operator>> operators;
125 std::vector<flatbuffers::Offset<tflite::Buffer>> buffers{
126 {CreateBuffer(builder, builder.CreateVector({}))}};
127
128 if (FP16Weights()) {
129 operator_codes.emplace_back(
130 CreateOperatorCode(builder, BuiltinOperator_DEQUANTIZE));
131
132 std::vector<uint16_t> filter_data(KernelHeight() * KernelWidth() *
133 OutputChannels());
134 std::vector<uint16_t> bias_data(OutputChannels());
135 for (int32_t ic = 0; ic < InputChannels(); ic++) {
136 // Use the same range of all-positive or all-negative values to generate
137 // all pixels within the same batch index & channel, but different ranges
138 // for different channels or batches. This ensures that no catastrophic
139 // cancellation occur, but test covers both positive and negative inputs.
140 const float range = range_rng();
141 auto value_rng =
142 std::bind(fp16_ieee_from_fp32_value,
143 std::bind(std::uniform_real_distribution<float>(
144 std::min(range, 0.0f), std::max(range, 0.0f)),
145 std::ref(rng)));
146 for (int32_t m = 0; m < DepthMultiplier(); m++) {
147 const int32_t oc = ic * DepthMultiplier() + m;
148 bias_data[oc] = value_rng();
149 for (int32_t y = 0; y < KernelHeight(); y++) {
150 for (int32_t x = 0; x < KernelWidth(); x++) {
151 const int32_t index =
152 (y * KernelWidth() + x) * OutputChannels() + oc;
153 filter_data[index] = value_rng();
154 }
155 }
156 }
157 }
158
159 buffers.emplace_back(CreateBuffer(
160 builder, builder.CreateVector(
161 reinterpret_cast<const uint8_t*>(filter_data.data()),
162 sizeof(uint16_t) * filter_data.size())));
163 buffers.emplace_back(CreateBuffer(
164 builder,
165 builder.CreateVector(reinterpret_cast<const uint8_t*>(bias_data.data()),
166 sizeof(uint16_t) * bias_data.size())));
167
168 const std::array<int32_t, 1> dequantize_filter_inputs{{0}};
169 const std::array<int32_t, 1> dequantize_filter_outputs{{3}};
170 operators.emplace_back(CreateOperator(
171 builder, /*opcode_index=*/1,
172 builder.CreateVector<int32_t>(dequantize_filter_inputs.data(),
173 dequantize_filter_inputs.size()),
174 builder.CreateVector<int32_t>(dequantize_filter_outputs.data(),
175 dequantize_filter_outputs.size())));
176 const std::array<int32_t, 1> dequantize_bias_inputs{{1}};
177 const std::array<int32_t, 1> dequantize_bias_outputs{{4}};
178 operators.emplace_back(CreateOperator(
179 builder, /*opcode_index=*/1,
180 builder.CreateVector<int32_t>(dequantize_bias_inputs.data(),
181 dequantize_bias_inputs.size()),
182 builder.CreateVector<int32_t>(dequantize_bias_outputs.data(),
183 dequantize_bias_outputs.size())));
184 } else {
185 std::vector<float> filter_data(KernelHeight() * KernelWidth() *
186 OutputChannels());
187 std::vector<float> bias_data(OutputChannels());
188 for (int32_t ic = 0; ic < InputChannels(); ic++) {
189 // Use the same range of all-positive or all-negative values to generate
190 // all pixels within the same batch index & channel, but different ranges
191 // for different channels or batches. This ensures that no catastrophic
192 // cancellation occur, but test covers both positive and negative inputs.
193 const float range = range_rng();
194 auto value_rng =
195 std::bind(std::uniform_real_distribution<float>(
196 std::min(range, 0.0f), std::max(range, 0.0f)),
197 std::ref(rng));
198 for (int32_t m = 0; m < DepthMultiplier(); m++) {
199 const int32_t oc = ic * DepthMultiplier() + m;
200 bias_data[oc] = value_rng();
201 for (int32_t y = 0; y < KernelHeight(); y++) {
202 for (int32_t x = 0; x < KernelWidth(); x++) {
203 const int32_t index =
204 (y * KernelWidth() + x) * OutputChannels() + oc;
205 filter_data[index] = value_rng();
206 }
207 }
208 }
209 }
210
211 buffers.emplace_back(CreateBuffer(
212 builder, builder.CreateVector(
213 reinterpret_cast<const uint8_t*>(filter_data.data()),
214 sizeof(float) * filter_data.size())));
215 buffers.emplace_back(CreateBuffer(
216 builder,
217 builder.CreateVector(reinterpret_cast<const uint8_t*>(bias_data.data()),
218 sizeof(float) * bias_data.size())));
219
220 if (SparseWeights()) {
221 operator_codes.emplace_back(
222 CreateOperatorCode(builder, BuiltinOperator_DENSIFY));
223 const std::array<int32_t, 1> densify_filter_inputs{{0}};
224 const std::array<int32_t, 1> densify_filter_outputs{{2}};
225 operators.emplace_back(CreateOperator(
226 builder, /*opcode_index=*/1,
227 builder.CreateVector<int32_t>(densify_filter_inputs.data(),
228 densify_filter_inputs.size()),
229 builder.CreateVector<int32_t>(densify_filter_outputs.data(),
230 densify_filter_outputs.size())));
231 }
232 }
233
234 const std::array<int32_t, 4> input_shape{
235 {BatchSize(), InputHeight(), InputWidth(), InputChannels()}};
236 const std::array<int32_t, 4> output_shape{
237 {BatchSize(), OutputHeight(), OutputWidth(), OutputChannels()}};
238 const std::array<int32_t, 4> filter_shape{
239 {1, KernelHeight(), KernelWidth(), OutputChannels()}};
240 const std::array<int32_t, 1> bias_shape{{OutputChannels()}};
241
242 std::vector<flatbuffers::Offset<tflite::Tensor>> tensors;
243 if (FP16Weights()) {
244 tensors.emplace_back(CreateTensor(
245 builder,
246 builder.CreateVector<int32_t>(filter_shape.data(), filter_shape.size()),
247 TensorType_FLOAT16, /*buffer=*/1));
248 tensors.emplace_back(CreateTensor(
249 builder,
250 builder.CreateVector<int32_t>(bias_shape.data(), bias_shape.size()),
251 TensorType_FLOAT16, /*buffer=*/2));
252 } else if (SparseWeights()) {
253 // Sparse tensor in TFLite can be in different formats. Here we choose the
254 // simplest configuration that
255 // 1. all dimensions are dense,
256 // 2. in-order traversal, and
257 // 3. no block configuration.
258 int dims_count = filter_shape.size();
259 std::vector<flatbuffers::Offset<DimensionMetadata>> dim_metadata(
260 dims_count);
261 std::vector<int> traversal_order(dims_count);
262 for (int i = 0; i < dims_count; i++) {
263 traversal_order[i] = i;
264 dim_metadata[i] = CreateDimensionMetadata(builder, DimensionType_DENSE,
265 filter_shape[i]);
266 }
267 flatbuffers::Offset<SparsityParameters> sparsity_param =
268 CreateSparsityParameters(builder, builder.CreateVector(traversal_order),
269 0, builder.CreateVector(dim_metadata));
270 tensors.emplace_back(CreateTensor(
271 builder,
272 builder.CreateVector<int32_t>(filter_shape.data(), filter_shape.size()),
273 TensorType_FLOAT32, /*buffer=*/1, /*name=*/0, /*quantization=*/0,
274 /*is_variable=*/false, /*sparsity=*/sparsity_param));
275 }
276 tensors.emplace_back(CreateTensor(
277 builder,
278 builder.CreateVector<int32_t>(input_shape.data(), input_shape.size()),
279 TensorType_FLOAT32));
280 tensors.emplace_back(CreateTensor(
281 builder,
282 builder.CreateVector<int32_t>(filter_shape.data(), filter_shape.size()),
283 TensorType_FLOAT32, /*buffer=*/FP16Weights() || SparseWeights() ? 0 : 1));
284 tensors.emplace_back(CreateTensor(
285 builder,
286 builder.CreateVector<int32_t>(bias_shape.data(), bias_shape.size()),
287 TensorType_FLOAT32, /*buffer=*/FP16Weights() ? 0 : 2));
288 tensors.emplace_back(CreateTensor(
289 builder,
290 builder.CreateVector<int32_t>(output_shape.data(), output_shape.size()),
291 TensorType_FLOAT32));
292
293 const std::array<int32_t, 3> op_inputs{
294 {static_cast<int>(tensors.size()) - 4,
295 static_cast<int>(tensors.size()) - 3,
296 static_cast<int>(tensors.size()) - 2}};
297 const std::array<int32_t, 1> op_outputs{
298 {static_cast<int>(tensors.size()) - 1}};
299
300 flatbuffers::Offset<DepthwiseConv2DOptions> depthwise_conv2d_options =
301 CreateDepthwiseConv2DOptions(
302 builder, Padding(), StrideWidth(), StrideHeight(), DepthMultiplier(),
303 Activation(), DilationWidth(), DilationHeight());
304 operators.emplace_back(CreateOperator(
305 builder, /*opcode_index=*/0,
306 builder.CreateVector<int32_t>(op_inputs.data(), op_inputs.size()),
307 builder.CreateVector<int32_t>(op_outputs.data(), op_outputs.size()),
308 BuiltinOptions_DepthwiseConv2DOptions, depthwise_conv2d_options.Union()));
309
310 const std::array<int32_t, 1> subgraph_inputs{
311 {static_cast<int>(tensors.size()) - 4}};
312 const std::array<int32_t, 1> subgraph_outputs{
313 {static_cast<int>(tensors.size()) - 1}};
314 flatbuffers::Offset<SubGraph> subgraph = CreateSubGraph(
315 builder, builder.CreateVector(tensors.data(), tensors.size()),
316 builder.CreateVector<int32_t>(subgraph_inputs.data(),
317 subgraph_inputs.size()),
318 builder.CreateVector<int32_t>(subgraph_outputs.data(),
319 subgraph_outputs.size()),
320 builder.CreateVector(operators.data(), operators.size()));
321
322 flatbuffers::Offset<flatbuffers::String> description =
323 builder.CreateString("DepthwiseConv2D model");
324
325 flatbuffers::Offset<Model> model_buffer = CreateModel(
326 builder, TFLITE_SCHEMA_VERSION,
327 builder.CreateVector(operator_codes.data(), operator_codes.size()),
328 builder.CreateVector(&subgraph, 1), description,
329 builder.CreateVector(buffers.data(), buffers.size()));
330
331 builder.Finish(model_buffer);
332
333 return std::vector<char>(builder.GetBufferPointer(),
334 builder.GetBufferPointer() + builder.GetSize());
335 }
336
337 } // namespace xnnpack
338 } // namespace tflite
339