1 /* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
2
3 Licensed under the Apache License, Version 2.0 (the "License");
4 you may not use this file except in compliance with the License.
5 You may obtain a copy of the License at
6
7 http://www.apache.org/licenses/LICENSE-2.0
8
9 Unless required by applicable law or agreed to in writing, software
10 distributed under the License is distributed on an "AS IS" BASIS,
11 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 See the License for the specific language governing permissions and
13 limitations under the License.
14 ==============================================================================*/
15 #include "tensorflow/lite/kernels/test_util.h"
16
17 #include <stddef.h>
18 #include <stdint.h>
19
20 #include <algorithm>
21 #include <complex>
22 #include <functional>
23 #include <map>
24 #include <memory>
25 #include <optional>
26 #include <string>
27 #include <tuple>
28 #include <utility>
29 #include <vector>
30
31 #include <gmock/gmock.h>
32 #include <gtest/gtest.h>
33 #include "flatbuffers/flatbuffers.h" // from @flatbuffers
34 #include "tensorflow/core/platform/logging.h"
35 #include "tensorflow/lite/c/common.h"
36 #include "tensorflow/lite/core/api/op_resolver.h"
37 #include "tensorflow/lite/core/subgraph.h"
38 #include "tensorflow/lite/delegates/nnapi/acceleration_test_util.h"
39 #include "tensorflow/lite/delegates/nnapi/nnapi_delegate.h"
40 #include "tensorflow/lite/interpreter.h"
41 #include "tensorflow/lite/kernels/acceleration_test_util.h"
42 #include "tensorflow/lite/kernels/register.h"
43 #include "tensorflow/lite/kernels/test_delegate_providers.h"
44 #include "tensorflow/lite/model.h"
45 #include "tensorflow/lite/nnapi/nnapi_implementation.h"
46 #include "tensorflow/lite/schema/schema_conversion_utils.h"
47 #include "tensorflow/lite/schema/schema_generated.h"
48 #include "tensorflow/lite/simple_planner.h"
49 #include "tensorflow/lite/string_type.h"
50 #include "tensorflow/lite/string_util.h"
51 #include "tensorflow/lite/tools/logging.h"
52 #include "tensorflow/lite/tools/versioning/op_version.h"
53 #include "tensorflow/lite/version.h"
54
55 namespace tflite {
56
57 using ::testing::FloatNear;
58 using ::testing::Matcher;
59
ArrayFloatNear(const std::vector<float> & values,float max_abs_error)60 std::vector<Matcher<float>> ArrayFloatNear(const std::vector<float>& values,
61 float max_abs_error) {
62 std::vector<Matcher<float>> matchers;
63 matchers.reserve(values.size());
64 for (const float& v : values) {
65 matchers.emplace_back(FloatNear(v, max_abs_error));
66 }
67 return matchers;
68 }
69
ArrayComplex64Near(const std::vector<std::complex<float>> & values,float max_abs_error)70 std::vector<Matcher<std::complex<float>>> ArrayComplex64Near(
71 const std::vector<std::complex<float>>& values, float max_abs_error) {
72 std::vector<Matcher<std::complex<float>>> matchers;
73 matchers.reserve(values.size());
74 for (const std::complex<float>& v : values) {
75 matchers.emplace_back(
76 AllOf(::testing::Property(&std::complex<float>::real,
77 FloatNear(v.real(), max_abs_error)),
78 ::testing::Property(&std::complex<float>::imag,
79 FloatNear(v.imag(), max_abs_error))));
80 }
81 return matchers;
82 }
83
AddInput(const TensorData & t)84 int SingleOpModel::AddInput(const TensorData& t) {
85 int id = 0;
86 if (t.per_channel_quantization) {
87 id = AddTensorPerChannelQuant(t);
88 } else {
89 id = AddTensor<float>(t, {});
90 }
91 inputs_.push_back(id);
92 return id;
93 }
94
AddVariableInput(const TensorData & t)95 int SingleOpModel::AddVariableInput(const TensorData& t) {
96 int id = 0;
97 if (t.per_channel_quantization) {
98 id = AddTensorPerChannelQuant(t);
99 } else {
100 id = AddTensor<float>(t, {}, true);
101 }
102 inputs_.push_back(id);
103 return id;
104 }
105
AddIntermediate(TensorType type,const std::vector<float> & scale,const std::vector<int64_t> & zero_point)106 int SingleOpModel::AddIntermediate(TensorType type,
107 const std::vector<float>& scale,
108 const std::vector<int64_t>& zero_point) {
109 // Currently supports only int16 intermediate types.
110 int id = tensors_.size();
111 flatbuffers::Offset<QuantizationParameters> q_params =
112 CreateQuantizationParameters(builder_, /*min=*/0, /*max=*/0,
113 builder_.CreateVector<float>(scale),
114 builder_.CreateVector<int64_t>(zero_point));
115 tensors_.push_back(CreateTensor(builder_, builder_.CreateVector<int>({}),
116 type,
117 /*buffer=*/0,
118 /*name=*/0, q_params, false));
119 intermediates_.push_back(id);
120 return id;
121 }
122
AddNullInput()123 int SingleOpModel::AddNullInput() {
124 int id = kTfLiteOptionalTensor;
125 inputs_.push_back(id);
126 return id;
127 }
128
AddOutput(const TensorData & t)129 int SingleOpModel::AddOutput(const TensorData& t) {
130 int id = AddTensor<float>(t, {});
131 outputs_.push_back(id);
132 return id;
133 }
134
SetBuiltinOp(BuiltinOperator type,BuiltinOptions builtin_options_type,flatbuffers::Offset<void> builtin_options)135 void SingleOpModel::SetBuiltinOp(BuiltinOperator type,
136 BuiltinOptions builtin_options_type,
137 flatbuffers::Offset<void> builtin_options) {
138 opcodes_.push_back(CreateOperatorCode(builder_, type, 0, 0));
139 operators_.push_back(CreateOperator(
140 builder_, /*opcode_index=*/0, builder_.CreateVector<int32_t>(inputs_),
141 builder_.CreateVector<int32_t>(outputs_), builtin_options_type,
142 builtin_options,
143 /*custom_options=*/0, CustomOptionsFormat_FLEXBUFFERS, 0,
144 builder_.CreateVector<int32_t>(intermediates_)));
145 }
146
SetCustomOp(const string & name,const std::vector<uint8_t> & custom_option,const std::function<TfLiteRegistration * ()> & registration)147 void SingleOpModel::SetCustomOp(
148 const string& name, const std::vector<uint8_t>& custom_option,
149 const std::function<TfLiteRegistration*()>& registration) {
150 custom_registrations_[name] = registration;
151 opcodes_.push_back(
152 CreateOperatorCodeDirect(builder_, BuiltinOperator_CUSTOM, name.data()));
153 operators_.push_back(CreateOperator(
154 builder_, /*opcode_index=*/0, builder_.CreateVector<int32_t>(inputs_),
155 builder_.CreateVector<int32_t>(outputs_), BuiltinOptions_NONE, 0,
156 builder_.CreateVector<uint8_t>(custom_option),
157 CustomOptionsFormat_FLEXBUFFERS));
158 }
159
AllocateAndDelegate(bool apply_delegate)160 void SingleOpModel::AllocateAndDelegate(bool apply_delegate) {
161 CHECK(interpreter_->AllocateTensors() == kTfLiteOk)
162 << "Cannot allocate tensors";
163 interpreter_->ResetVariableTensors();
164
165 // In some rare cases a test may need to postpone modifying the graph with
166 // a delegate, e.g. if tensors are not fully specified. In such cases the
167 // test has to explicitly call ApplyDelegate() when necessary.
168 if (apply_delegate) ApplyDelegate();
169 }
170
BuildInterpreter(std::vector<std::vector<int>> input_shapes,int num_threads,bool allow_fp32_relax_to_fp16,bool apply_delegate,bool allocate_and_delegate)171 void SingleOpModel::BuildInterpreter(std::vector<std::vector<int>> input_shapes,
172 int num_threads,
173 bool allow_fp32_relax_to_fp16,
174 bool apply_delegate,
175 bool allocate_and_delegate) {
176 auto opcodes = builder_.CreateVector(opcodes_);
177 auto operators = builder_.CreateVector(operators_);
178 auto tensors = builder_.CreateVector(tensors_);
179 auto inputs = builder_.CreateVector<int32_t>(inputs_);
180 auto outputs = builder_.CreateVector<int32_t>(outputs_);
181 // Create a single subgraph
182 std::vector<flatbuffers::Offset<SubGraph>> subgraphs;
183 auto subgraph = CreateSubGraph(builder_, tensors, inputs, outputs, operators);
184 subgraphs.push_back(subgraph);
185 auto subgraphs_flatbuffer = builder_.CreateVector(subgraphs);
186
187 auto buffers = builder_.CreateVector(buffers_);
188 auto description = builder_.CreateString("programmatic model");
189 builder_.Finish(CreateModel(builder_, TFLITE_SCHEMA_VERSION, opcodes,
190 subgraphs_flatbuffer, description, buffers));
191
192 uint8_t* buffer_pointer = builder_.GetBufferPointer();
193 UpdateOpVersion(buffer_pointer);
194
195 bool use_simple_allocator =
196 tflite::KernelTestDelegateProviders::Get()->ConstParams().Get<bool>(
197 tflite::KernelTestDelegateProviders::kUseSimpleAllocator);
198
199 if (!resolver_) {
200 // If we have a manually-set TfLite delegate, we assume the intention of
201 // the test is to test against the particular delegate, hence bypassing
202 // applying TfLite default delegates (i.e. the XNNPACK delegate).
203 bool bypass_default_delegates = (delegate_ != nullptr);
204 if (!bypass_default_delegates) {
205 // Check if any delegates are specified via the commandline flags.
206 const auto specified_delegates =
207 tflite::KernelTestDelegateProviders::Get()->CreateAllDelegates();
208 if (!specified_delegates.empty()) {
209 bypass_default_delegates = true;
210 }
211 }
212 MutableOpResolver* resolver =
213 (bypass_default_delegates || use_simple_allocator)
214 ? new ops::builtin::BuiltinOpResolverWithoutDefaultDelegates()
215 : new ops::builtin::BuiltinOpResolver();
216 for (const auto& reg : custom_registrations_) {
217 resolver->AddCustom(reg.first.data(), reg.second());
218 }
219 resolver_ = std::unique_ptr<OpResolver>(resolver);
220 }
221 CHECK(InterpreterBuilder(GetModel(buffer_pointer), *resolver_)(
222 &interpreter_, num_threads) == kTfLiteOk);
223
224 CHECK(interpreter_ != nullptr);
225
226 if (use_simple_allocator) {
227 LOG(INFO) << "Use SimplePlanner.\n";
228 tflite::Subgraph& primary_subgraph = interpreter_->primary_subgraph();
229 auto memory_planner = new SimplePlanner(
230 &primary_subgraph.context_,
231 std::unique_ptr<GraphInfo>(primary_subgraph.CreateGraphInfo()));
232 primary_subgraph.memory_planner_.reset(memory_planner);
233 memory_planner->PlanAllocations();
234 }
235
236 for (size_t i = 0; i < input_shapes.size(); ++i) {
237 const int input_idx = interpreter_->inputs()[i];
238 if (input_idx == kTfLiteOptionalTensor) continue;
239 const auto& shape = input_shapes[i];
240 if (shape.empty()) continue;
241 CHECK(interpreter_->ResizeInputTensor(input_idx, shape) == kTfLiteOk);
242 }
243
244 interpreter_->SetAllowFp16PrecisionForFp32(allow_fp32_relax_to_fp16);
245
246 if (allocate_and_delegate) {
247 AllocateAndDelegate(apply_delegate);
248 }
249 }
250
ApplyDelegate()251 TfLiteStatus SingleOpModel::ApplyDelegate() {
252 if (delegate_) {
253 TFLITE_LOG(WARN) << "Having a manually-set TfLite delegate, and bypassing "
254 "KernelTestDelegateProviders";
255 TF_LITE_ENSURE_STATUS(interpreter_->ModifyGraphWithDelegate(delegate_));
256 ++num_applied_delegates_;
257 } else {
258 auto* delegate_providers = tflite::KernelTestDelegateProviders::Get();
259 // Most TFLite NNAPI delegation tests have been written to run against the
260 // NNAPI CPU path. We'll enable that for tests. However, need to first check
261 // if the parameter is present - it will not be if the NNAPI delegate
262 // provider is not linked into the test.
263 if (delegate_providers->ConstParams().HasParam("disable_nnapi_cpu")) {
264 delegate_providers->MutableParams()->Set("disable_nnapi_cpu", false);
265 }
266 for (auto& one : delegate_providers->CreateAllDelegates()) {
267 // The raw ptr always points to the actual TfLiteDegate object.
268 auto* delegate_raw_ptr = one.delegate.get();
269 TF_LITE_ENSURE_STATUS(
270 interpreter_->ModifyGraphWithDelegate(std::move(one.delegate)));
271 // Note: 'delegate_' is always set to the last successfully applied one.
272 delegate_ = delegate_raw_ptr;
273 ++num_applied_delegates_;
274 }
275 }
276 return kTfLiteOk;
277 }
278
Invoke()279 void SingleOpModel::Invoke() { ASSERT_EQ(interpreter_->Invoke(), kTfLiteOk); }
280
InvokeUnchecked()281 TfLiteStatus SingleOpModel::InvokeUnchecked() { return interpreter_->Invoke(); }
282
BuildInterpreter(std::vector<std::vector<int>> input_shapes)283 void SingleOpModel::BuildInterpreter(
284 std::vector<std::vector<int>> input_shapes) {
285 BuildInterpreter(input_shapes, /*num_threads=*/-1,
286 /*allow_fp32_relax_to_fp16=*/false,
287 /*apply_delegate=*/true, /*allocate_and_delegate=*/true);
288 }
289
290 // static
GetForceUseNnapi()291 bool SingleOpModel::GetForceUseNnapi() {
292 const auto& delegate_params =
293 tflite::KernelTestDelegateProviders::Get()->ConstParams();
294 // It's possible this library isn't linked with the nnapi delegate provider
295 // lib.
296 return delegate_params.HasParam("use_nnapi") &&
297 delegate_params.Get<bool>("use_nnapi");
298 }
299
GetTensorSize(int index) const300 int32_t SingleOpModel::GetTensorSize(int index) const {
301 TfLiteTensor* t = interpreter_->tensor(index);
302 CHECK(t);
303 int total_size = 1;
304 for (int i = 0; i < t->dims->size; ++i) {
305 total_size *= t->dims->data[i];
306 }
307 return total_size;
308 }
309
310 template <>
ExtractVector(int index) const311 std::vector<string> SingleOpModel::ExtractVector(int index) const {
312 TfLiteTensor* tensor_ptr = interpreter_->tensor(index);
313 CHECK(tensor_ptr != nullptr);
314 const int num_strings = GetStringCount(tensor_ptr);
315 std::vector<string> result;
316 result.reserve(num_strings);
317 for (int i = 0; i < num_strings; ++i) {
318 const auto str = GetString(tensor_ptr, i);
319 result.emplace_back(str.str, str.len);
320 }
321 return result;
322 }
323
324 namespace {
325
326 // Returns the number of partitions associated, as result of a call to
327 // ModifyGraphWithDelegate, to the given delegate.
CountPartitionsDelegatedTo(Subgraph * subgraph,const TfLiteDelegate * delegate)328 int CountPartitionsDelegatedTo(Subgraph* subgraph,
329 const TfLiteDelegate* delegate) {
330 return std::count_if(
331 subgraph->nodes_and_registration().begin(),
332 subgraph->nodes_and_registration().end(),
333 [delegate](
334 std::pair<TfLiteNode, TfLiteRegistration> node_and_registration) {
335 return node_and_registration.first.delegate == delegate;
336 });
337 }
338
339 // Returns the number of partitions associated, as result of a call to
340 // ModifyGraphWithDelegate, to the given delegate.
CountPartitionsDelegatedTo(Interpreter * interpreter,const TfLiteDelegate * delegate)341 int CountPartitionsDelegatedTo(Interpreter* interpreter,
342 const TfLiteDelegate* delegate) {
343 int result = 0;
344 for (int i = 0; i < interpreter->subgraphs_size(); i++) {
345 Subgraph* subgraph = interpreter->subgraph(i);
346
347 result += CountPartitionsDelegatedTo(subgraph, delegate);
348 }
349
350 return result;
351 }
352
353 // Returns the number of nodes that will be executed on the CPU
CountPartitionsExecutedByCpuKernel(const Interpreter * interpreter)354 int CountPartitionsExecutedByCpuKernel(const Interpreter* interpreter) {
355 int result = 0;
356 for (int node_idx : interpreter->execution_plan()) {
357 TfLiteNode node;
358 TfLiteRegistration reg;
359 std::tie(node, reg) = *(interpreter->node_and_registration(node_idx));
360
361 if (node.delegate == nullptr) {
362 ++result;
363 }
364 }
365
366 return result;
367 }
368
369 } // namespace
370
ExpectOpAcceleratedWithNnapi(const std::string & test_id)371 void SingleOpModel::ExpectOpAcceleratedWithNnapi(const std::string& test_id) {
372 std::optional<NnapiAccelerationTestParams> validation_params =
373 GetNnapiAccelerationTestParam(test_id);
374 if (!validation_params.has_value()) {
375 return;
376 }
377
378 // If we have multiple delegates applied, we would skip this check at the
379 // moment.
380 if (num_applied_delegates_ > 1) {
381 TFLITE_LOG(WARN) << "Skipping ExpectOpAcceleratedWithNnapi as "
382 << num_applied_delegates_
383 << " delegates have been successfully applied.";
384 return;
385 }
386 TFLITE_LOG(INFO) << "Validating acceleration";
387 const NnApi* nnapi = NnApiImplementation();
388 if (nnapi && nnapi->nnapi_exists &&
389 nnapi->android_sdk_version >=
390 validation_params.value().MinAndroidSdkVersion()) {
391 EXPECT_EQ(CountPartitionsDelegatedTo(interpreter_.get(), delegate_), 1)
392 << "Expecting operation to be accelerated but cannot find a partition "
393 "associated to the NNAPI delegate";
394 EXPECT_GT(num_applied_delegates_, 0) << "No delegates were applied.";
395 }
396 }
397
ValidateAcceleration()398 void SingleOpModel::ValidateAcceleration() {
399 if (GetForceUseNnapi()) {
400 ExpectOpAcceleratedWithNnapi(GetCurrentTestId());
401 }
402 }
403
CountOpsExecutedByCpuKernel()404 int SingleOpModel::CountOpsExecutedByCpuKernel() {
405 return CountPartitionsExecutedByCpuKernel(interpreter_.get());
406 }
407
~SingleOpModel()408 SingleOpModel::~SingleOpModel() { ValidateAcceleration(); }
409
AddBuiltinOp(BuiltinOperator type,BuiltinOptions builtin_options_type,const flatbuffers::Offset<void> & builtin_options,const std::vector<int32_t> & inputs,const std::vector<int32_t> & outputs)410 void MultiOpModel::AddBuiltinOp(
411 BuiltinOperator type, BuiltinOptions builtin_options_type,
412 const flatbuffers::Offset<void>& builtin_options,
413 const std::vector<int32_t>& inputs, const std::vector<int32_t>& outputs) {
414 opcodes_.push_back(CreateOperatorCode(builder_, type, 0, 0));
415 const int opcode_index = opcodes_.size() - 1;
416 operators_.push_back(CreateOperator(
417 builder_, opcode_index, builder_.CreateVector<int32_t>(inputs),
418 builder_.CreateVector<int32_t>(outputs), builtin_options_type,
419 builtin_options,
420 /*custom_options=*/0, CustomOptionsFormat_FLEXBUFFERS));
421 }
422
AddCustomOp(const string & name,const std::vector<uint8_t> & custom_option,const std::function<TfLiteRegistration * ()> & registration,const std::vector<int32_t> & inputs,const std::vector<int32_t> & outputs)423 void MultiOpModel::AddCustomOp(
424 const string& name, const std::vector<uint8_t>& custom_option,
425 const std::function<TfLiteRegistration*()>& registration,
426 const std::vector<int32_t>& inputs, const std::vector<int32_t>& outputs) {
427 custom_registrations_[name] = registration;
428 opcodes_.push_back(
429 CreateOperatorCodeDirect(builder_, BuiltinOperator_CUSTOM, name.data()));
430 const int opcode_index = opcodes_.size() - 1;
431 operators_.push_back(CreateOperator(
432 builder_, opcode_index, builder_.CreateVector<int32_t>(inputs),
433 builder_.CreateVector<int32_t>(outputs), BuiltinOptions_NONE, 0,
434 builder_.CreateVector<uint8_t>(custom_option),
435 CustomOptionsFormat_FLEXBUFFERS));
436 }
437 } // namespace tflite
438