1 /* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
2
3 Licensed under the Apache License, Version 2.0 (the "License");
4 you may not use this file except in compliance with the License.
5 You may obtain a copy of the License at
6
7 http://www.apache.org/licenses/LICENSE-2.0
8
9 Unless required by applicable law or agreed to in writing, software
10 distributed under the License is distributed on an "AS IS" BASIS,
11 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 See the License for the specific language governing permissions and
13 limitations under the License.
14 ==============================================================================*/
15 #include "tensorflow/lite/kernels/test_util.h"
16
17 #include <stddef.h>
18 #include <stdint.h>
19
20 #include <algorithm>
21 #include <complex>
22 #include <functional>
23 #include <map>
24 #include <memory>
25 #include <optional>
26 #include <string>
27 #include <tuple>
28 #include <utility>
29 #include <vector>
30
31 #include <gmock/gmock.h>
32 #include <gtest/gtest.h>
33 #include "flatbuffers/flatbuffers.h" // from @flatbuffers
34 #include "tensorflow/core/platform/logging.h"
35 #include "tensorflow/lite/c/common.h"
36 #include "tensorflow/lite/core/api/op_resolver.h"
37 #include "tensorflow/lite/core/subgraph.h"
38 #include "tensorflow/lite/delegates/nnapi/acceleration_test_util.h"
39 #include "tensorflow/lite/delegates/nnapi/nnapi_delegate.h"
40 #include "tensorflow/lite/interpreter.h"
41 #include "tensorflow/lite/kernels/acceleration_test_util.h"
42 #include "tensorflow/lite/kernels/register.h"
43 #include "tensorflow/lite/kernels/test_delegate_providers.h"
44 #include "tensorflow/lite/model.h"
45 #include "tensorflow/lite/nnapi/nnapi_implementation.h"
46 #include "tensorflow/lite/schema/schema_conversion_utils.h"
47 #include "tensorflow/lite/schema/schema_generated.h"
48 #include "tensorflow/lite/simple_planner.h"
49 #include "tensorflow/lite/string_type.h"
50 #include "tensorflow/lite/string_util.h"
51 #include "tensorflow/lite/tools/logging.h"
52 #include "tensorflow/lite/tools/versioning/op_version.h"
53 #include "tensorflow/lite/version.h"
54
55 namespace tflite {
56
57 using ::testing::FloatNear;
58 using ::testing::Matcher;
59
ArrayFloatNear(const std::vector<float> & values,float max_abs_error)60 std::vector<Matcher<float>> ArrayFloatNear(const std::vector<float>& values,
61 float max_abs_error) {
62 std::vector<Matcher<float>> matchers;
63 matchers.reserve(values.size());
64 for (const float& v : values) {
65 matchers.emplace_back(FloatNear(v, max_abs_error));
66 }
67 return matchers;
68 }
69
ArrayComplex64Near(const std::vector<std::complex<float>> & values,float max_abs_error)70 std::vector<Matcher<std::complex<float>>> ArrayComplex64Near(
71 const std::vector<std::complex<float>>& values, float max_abs_error) {
72 std::vector<Matcher<std::complex<float>>> matchers;
73 matchers.reserve(values.size());
74 for (const std::complex<float>& v : values) {
75 matchers.emplace_back(
76 AllOf(::testing::Property(&std::complex<float>::real,
77 FloatNear(v.real(), max_abs_error)),
78 ::testing::Property(&std::complex<float>::imag,
79 FloatNear(v.imag(), max_abs_error))));
80 }
81 return matchers;
82 }
83
AddInput(const TensorData & t)84 int SingleOpModel::AddInput(const TensorData& t) {
85 int id = 0;
86 if (t.per_channel_quantization) {
87 id = AddTensorPerChannelQuant(t);
88 } else {
89 id = AddTensor<float>(t, nullptr, 0);
90 }
91 inputs_.push_back(id);
92 return id;
93 }
94
AddVariableInput(const TensorData & t)95 int SingleOpModel::AddVariableInput(const TensorData& t) {
96 int id = 0;
97 if (t.per_channel_quantization) {
98 id = AddTensorPerChannelQuant(t);
99 } else {
100 id = AddTensor<float>(t, nullptr, 0, true);
101 }
102 inputs_.push_back(id);
103 return id;
104 }
105
AddIntermediate(TensorType type,const std::vector<float> & scale,const std::vector<int64_t> & zero_point)106 int SingleOpModel::AddIntermediate(TensorType type,
107 const std::vector<float>& scale,
108 const std::vector<int64_t>& zero_point) {
109 // Currently supports only int16 intermediate types.
110 int id = tensors_.size();
111 flatbuffers::Offset<QuantizationParameters> q_params =
112 CreateQuantizationParameters(builder_, /*min=*/0, /*max=*/0,
113 builder_.CreateVector<float>(scale),
114 builder_.CreateVector<int64_t>(zero_point));
115 std::vector<int> empty;
116 tensors_.push_back(CreateTensor(builder_, builder_.CreateVector<int>(empty),
117 type,
118 /*buffer=*/0,
119 /*name=*/0, q_params, false));
120 intermediates_.push_back(id);
121 return id;
122 }
123
AddNullInput()124 int SingleOpModel::AddNullInput() {
125 int id = kTfLiteOptionalTensor;
126 inputs_.push_back(id);
127 return id;
128 }
129
AddOutput(const TensorData & t)130 int SingleOpModel::AddOutput(const TensorData& t) {
131 int id = 0;
132 if (t.per_channel_quantization) {
133 id = AddTensorPerChannelQuant(t);
134 } else {
135 id = AddTensor<float>(t, nullptr, 0);
136 }
137 outputs_.push_back(id);
138 return id;
139 }
140
SetBuiltinOp(BuiltinOperator type,BuiltinOptions builtin_options_type,flatbuffers::Offset<void> builtin_options)141 void SingleOpModel::SetBuiltinOp(BuiltinOperator type,
142 BuiltinOptions builtin_options_type,
143 flatbuffers::Offset<void> builtin_options) {
144 opcodes_.push_back(CreateOperatorCode(builder_, type, 0, 0));
145 operators_.push_back(CreateOperator(
146 builder_, /*opcode_index=*/0, builder_.CreateVector<int32_t>(inputs_),
147 builder_.CreateVector<int32_t>(outputs_), builtin_options_type,
148 builtin_options,
149 /*custom_options=*/0, CustomOptionsFormat_FLEXBUFFERS, 0,
150 builder_.CreateVector<int32_t>(intermediates_)));
151 }
152
SetCustomOp(const string & name,const std::vector<uint8_t> & custom_option,const std::function<TfLiteRegistration * ()> & registration)153 void SingleOpModel::SetCustomOp(
154 const string& name, const std::vector<uint8_t>& custom_option,
155 const std::function<TfLiteRegistration*()>& registration) {
156 custom_registrations_[name] = registration;
157 opcodes_.push_back(
158 CreateOperatorCodeDirect(builder_, BuiltinOperator_CUSTOM, name.data()));
159 operators_.push_back(CreateOperator(
160 builder_, /*opcode_index=*/0, builder_.CreateVector<int32_t>(inputs_),
161 builder_.CreateVector<int32_t>(outputs_), BuiltinOptions_NONE, 0,
162 builder_.CreateVector<uint8_t>(custom_option),
163 CustomOptionsFormat_FLEXBUFFERS));
164 }
165
AllocateAndDelegate(bool apply_delegate)166 void SingleOpModel::AllocateAndDelegate(bool apply_delegate) {
167 CHECK(interpreter_->AllocateTensors() == kTfLiteOk)
168 << "Cannot allocate tensors";
169 interpreter_->ResetVariableTensors();
170
171 // In some rare cases a test may need to postpone modifying the graph with
172 // a delegate, e.g. if tensors are not fully specified. In such cases the
173 // test has to explicitly call ApplyDelegate() when necessary.
174 if (apply_delegate) ApplyDelegate();
175 }
176
BuildInterpreter(std::vector<std::vector<int>> input_shapes,int num_threads,bool allow_fp32_relax_to_fp16,bool apply_delegate,bool allocate_and_delegate)177 void SingleOpModel::BuildInterpreter(std::vector<std::vector<int>> input_shapes,
178 int num_threads,
179 bool allow_fp32_relax_to_fp16,
180 bool apply_delegate,
181 bool allocate_and_delegate) {
182 input_shapes_ = input_shapes;
183 allow_fp32_relax_to_fp16_ = allow_fp32_relax_to_fp16;
184 apply_delegate_ = apply_delegate;
185 allocate_and_delegate_ = allocate_and_delegate;
186
187 auto opcodes = builder_.CreateVector(opcodes_);
188 auto operators = builder_.CreateVector(operators_);
189 auto tensors = builder_.CreateVector(tensors_);
190 auto inputs = builder_.CreateVector<int32_t>(inputs_);
191 auto outputs = builder_.CreateVector<int32_t>(outputs_);
192 // Create a single subgraph
193 std::vector<flatbuffers::Offset<SubGraph>> subgraphs;
194 auto subgraph = CreateSubGraph(builder_, tensors, inputs, outputs, operators);
195 subgraphs.push_back(subgraph);
196 auto subgraphs_flatbuffer = builder_.CreateVector(subgraphs);
197
198 auto buffers = builder_.CreateVector(buffers_);
199 auto description = builder_.CreateString("programmatic model");
200 builder_.Finish(CreateModel(builder_, TFLITE_SCHEMA_VERSION, opcodes,
201 subgraphs_flatbuffer, description, buffers));
202
203 uint8_t* buffer_pointer = builder_.GetBufferPointer();
204 UpdateOpVersion(buffer_pointer);
205
206 bool use_simple_allocator =
207 tflite::KernelTestDelegateProviders::Get()->ConstParams().Get<bool>(
208 tflite::KernelTestDelegateProviders::kUseSimpleAllocator);
209
210 if (!resolver_) {
211 if (!bypass_default_delegates_) {
212 // Check if any delegates are specified via the commandline flags. We also
213 // assume the intention of the test is to test against a particular
214 // delegate, hence bypassing applying TfLite default delegates (i.e. the
215 // XNNPACK delegate).
216 const auto specified_delegates =
217 tflite::KernelTestDelegateProviders::Get()->CreateAllDelegates();
218 if (!specified_delegates.empty()) {
219 bypass_default_delegates_ = true;
220 }
221 }
222 MutableOpResolver* resolver =
223 (bypass_default_delegates_ || use_simple_allocator)
224 ? new ops::builtin::BuiltinOpResolverWithoutDefaultDelegates()
225 : new ops::builtin::BuiltinOpResolver();
226 for (const auto& reg : custom_registrations_) {
227 resolver->AddCustom(reg.first.data(), reg.second());
228 }
229 resolver_ = std::unique_ptr<OpResolver>(resolver);
230 }
231 CHECK(InterpreterBuilder(GetModel(buffer_pointer), *resolver_)(
232 &interpreter_, num_threads) == kTfLiteOk);
233
234 CHECK(interpreter_ != nullptr);
235
236 if (use_simple_allocator) {
237 LOG(INFO) << "Use SimplePlanner.\n";
238 tflite::Subgraph& primary_subgraph = interpreter_->primary_subgraph();
239 auto memory_planner = new SimplePlanner(
240 &primary_subgraph.context_,
241 std::unique_ptr<GraphInfo>(primary_subgraph.CreateGraphInfo()));
242 primary_subgraph.memory_planner_.reset(memory_planner);
243 memory_planner->PlanAllocations();
244 }
245
246 for (size_t i = 0; i < input_shapes.size(); ++i) {
247 const int input_idx = interpreter_->inputs()[i];
248 if (input_idx == kTfLiteOptionalTensor) continue;
249 const auto& shape = input_shapes[i];
250 if (shape.empty()) continue;
251 CHECK(interpreter_->ResizeInputTensor(input_idx, shape) == kTfLiteOk);
252 }
253
254 interpreter_->SetAllowFp16PrecisionForFp32(allow_fp32_relax_to_fp16);
255
256 if (allocate_and_delegate) {
257 AllocateAndDelegate(apply_delegate);
258 }
259 }
260
ApplyDelegate()261 TfLiteStatus SingleOpModel::ApplyDelegate() {
262 if (delegate_) {
263 TFLITE_LOG(WARN) << "Having a manually-set TfLite delegate, and bypassing "
264 "KernelTestDelegateProviders";
265 TF_LITE_ENSURE_STATUS(interpreter_->ModifyGraphWithDelegate(delegate_));
266 ++num_applied_delegates_;
267 } else {
268 auto* delegate_providers = tflite::KernelTestDelegateProviders::Get();
269 // Most TFLite NNAPI delegation tests have been written to run against the
270 // NNAPI CPU path. We'll enable that for tests. However, need to first check
271 // if the parameter is present - it will not be if the NNAPI delegate
272 // provider is not linked into the test.
273 if (delegate_providers->ConstParams().HasParam("disable_nnapi_cpu")) {
274 delegate_providers->MutableParams()->Set("disable_nnapi_cpu", false);
275 }
276 for (auto& one : delegate_providers->CreateAllDelegates()) {
277 // The raw ptr always points to the actual TfLiteDegate object.
278 auto* delegate_raw_ptr = one.delegate.get();
279 TF_LITE_ENSURE_STATUS(
280 interpreter_->ModifyGraphWithDelegate(std::move(one.delegate)));
281 // Note: 'delegate_' is always set to the last successfully applied one.
282 delegate_ = delegate_raw_ptr;
283 ++num_applied_delegates_;
284 }
285 }
286 return kTfLiteOk;
287 }
288
Invoke()289 TfLiteStatus SingleOpModel::Invoke() { return interpreter_->Invoke(); }
290
BuildInterpreter(std::vector<std::vector<int>> input_shapes)291 void SingleOpModel::BuildInterpreter(
292 std::vector<std::vector<int>> input_shapes) {
293 BuildInterpreter(input_shapes, /*num_threads=*/-1,
294 /*allow_fp32_relax_to_fp16=*/false,
295 /*apply_delegate=*/true, /*allocate_and_delegate=*/true);
296 }
297
298 // static
GetForceUseNnapi()299 bool SingleOpModel::GetForceUseNnapi() {
300 const auto& delegate_params =
301 tflite::KernelTestDelegateProviders::Get()->ConstParams();
302 // It's possible this library isn't linked with the nnapi delegate provider
303 // lib.
304 return delegate_params.HasParam("use_nnapi") &&
305 delegate_params.Get<bool>("use_nnapi");
306 }
307
GetTensorSize(int index) const308 int32_t SingleOpModel::GetTensorSize(int index) const {
309 TfLiteTensor* t = interpreter_->tensor(index);
310 CHECK(t);
311 int total_size = 1;
312 for (int i = 0; i < t->dims->size; ++i) {
313 total_size *= t->dims->data[i];
314 }
315 return total_size;
316 }
317
318 template <>
ExtractVector(int index) const319 std::vector<string> SingleOpModel::ExtractVector(int index) const {
320 TfLiteTensor* tensor_ptr = interpreter_->tensor(index);
321 CHECK(tensor_ptr != nullptr);
322 const int num_strings = GetStringCount(tensor_ptr);
323 std::vector<string> result;
324 result.reserve(num_strings);
325 for (int i = 0; i < num_strings; ++i) {
326 const auto str = GetString(tensor_ptr, i);
327 result.emplace_back(str.str, str.len);
328 }
329 return result;
330 }
331
332 namespace {
333
334 // Returns the number of partitions associated, as result of a call to
335 // ModifyGraphWithDelegate, to the given delegate.
CountPartitionsDelegatedTo(Subgraph * subgraph,const TfLiteDelegate * delegate)336 int CountPartitionsDelegatedTo(Subgraph* subgraph,
337 const TfLiteDelegate* delegate) {
338 return std::count_if(
339 subgraph->nodes_and_registration().begin(),
340 subgraph->nodes_and_registration().end(),
341 [delegate](
342 std::pair<TfLiteNode, TfLiteRegistration> node_and_registration) {
343 return node_and_registration.first.delegate == delegate;
344 });
345 }
346
347 // Returns the number of partitions associated, as result of a call to
348 // ModifyGraphWithDelegate, to the given delegate.
CountPartitionsDelegatedTo(Interpreter * interpreter,const TfLiteDelegate * delegate)349 int CountPartitionsDelegatedTo(Interpreter* interpreter,
350 const TfLiteDelegate* delegate) {
351 int result = 0;
352 for (int i = 0; i < interpreter->subgraphs_size(); i++) {
353 Subgraph* subgraph = interpreter->subgraph(i);
354
355 result += CountPartitionsDelegatedTo(subgraph, delegate);
356 }
357
358 return result;
359 }
360
361 // Returns the number of nodes that will be executed on the CPU
CountPartitionsExecutedByCpuKernel(const Interpreter * interpreter)362 int CountPartitionsExecutedByCpuKernel(const Interpreter* interpreter) {
363 int result = 0;
364 for (int node_idx : interpreter->execution_plan()) {
365 TfLiteNode node;
366 TfLiteRegistration reg;
367 std::tie(node, reg) = *(interpreter->node_and_registration(node_idx));
368
369 if (node.delegate == nullptr) {
370 ++result;
371 }
372 }
373
374 return result;
375 }
376
377 } // namespace
378
ExpectOpAcceleratedWithNnapi(const std::string & test_id)379 void SingleOpModel::ExpectOpAcceleratedWithNnapi(const std::string& test_id) {
380 std::optional<NnapiAccelerationTestParams> validation_params =
381 GetNnapiAccelerationTestParam(test_id);
382 if (!validation_params.has_value()) {
383 return;
384 }
385
386 // If we have multiple delegates applied, we would skip this check at the
387 // moment.
388 if (num_applied_delegates_ > 1) {
389 TFLITE_LOG(WARN) << "Skipping ExpectOpAcceleratedWithNnapi as "
390 << num_applied_delegates_
391 << " delegates have been successfully applied.";
392 return;
393 }
394 TFLITE_LOG(INFO) << "Validating acceleration";
395 const NnApi* nnapi = NnApiImplementation();
396 if (nnapi && nnapi->nnapi_exists &&
397 nnapi->android_sdk_version >=
398 validation_params.value().MinAndroidSdkVersion()) {
399 EXPECT_EQ(CountPartitionsDelegatedTo(interpreter_.get(), delegate_), 1)
400 << "Expecting operation to be accelerated but cannot find a partition "
401 "associated to the NNAPI delegate";
402 EXPECT_GT(num_applied_delegates_, 0) << "No delegates were applied.";
403 }
404 }
405
ValidateAcceleration()406 void SingleOpModel::ValidateAcceleration() {
407 if (GetForceUseNnapi()) {
408 ExpectOpAcceleratedWithNnapi(GetCurrentTestId());
409 }
410 }
411
CountOpsExecutedByCpuKernel()412 int SingleOpModel::CountOpsExecutedByCpuKernel() {
413 return CountPartitionsExecutedByCpuKernel(interpreter_.get());
414 }
415
~SingleOpModel()416 SingleOpModel::~SingleOpModel() { ValidateAcceleration(); }
417
AddBuiltinOp(BuiltinOperator type,BuiltinOptions builtin_options_type,const flatbuffers::Offset<void> & builtin_options,const std::vector<int32_t> & inputs,const std::vector<int32_t> & outputs)418 void MultiOpModel::AddBuiltinOp(
419 BuiltinOperator type, BuiltinOptions builtin_options_type,
420 const flatbuffers::Offset<void>& builtin_options,
421 const std::vector<int32_t>& inputs, const std::vector<int32_t>& outputs) {
422 opcodes_.push_back(CreateOperatorCode(builder_, type, 0, 0));
423 const int opcode_index = opcodes_.size() - 1;
424 operators_.push_back(CreateOperator(
425 builder_, opcode_index, builder_.CreateVector<int32_t>(inputs),
426 builder_.CreateVector<int32_t>(outputs), builtin_options_type,
427 builtin_options,
428 /*custom_options=*/0, CustomOptionsFormat_FLEXBUFFERS));
429 }
430
AddCustomOp(const string & name,const std::vector<uint8_t> & custom_option,const std::function<TfLiteRegistration * ()> & registration,const std::vector<int32_t> & inputs,const std::vector<int32_t> & outputs)431 void MultiOpModel::AddCustomOp(
432 const string& name, const std::vector<uint8_t>& custom_option,
433 const std::function<TfLiteRegistration*()>& registration,
434 const std::vector<int32_t>& inputs, const std::vector<int32_t>& outputs) {
435 custom_registrations_[name] = registration;
436 opcodes_.push_back(
437 CreateOperatorCodeDirect(builder_, BuiltinOperator_CUSTOM, name.data()));
438 const int opcode_index = opcodes_.size() - 1;
439 operators_.push_back(CreateOperator(
440 builder_, opcode_index, builder_.CreateVector<int32_t>(inputs),
441 builder_.CreateVector<int32_t>(outputs), BuiltinOptions_NONE, 0,
442 builder_.CreateVector<uint8_t>(custom_option),
443 CustomOptionsFormat_FLEXBUFFERS));
444 }
445 } // namespace tflite
446