1 /* Copyright 2020 The TensorFlow Authors. All Rights Reserved.
2
3 Licensed under the Apache License, Version 2.0 (the "License");
4 you may not use this file except in compliance with the License.
5 You may obtain a copy of the License at
6
7 http://www.apache.org/licenses/LICENSE-2.0
8
9 Unless required by applicable law or agreed to in writing, software
10 distributed under the License is distributed on an "AS IS" BASIS,
11 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 See the License for the specific language governing permissions and
13 limitations under the License.
14 ==============================================================================*/
15 #include <string>
16
17 #include "tensorflow/lite/delegates/nnapi/nnapi_delegate.h"
18 #include "tensorflow/lite/nnapi/nnapi_implementation.h"
19 #include "tensorflow/lite/nnapi/nnapi_util.h"
20 #include "tensorflow/lite/tools/delegates/delegate_provider.h"
21
22 namespace tflite {
23 namespace tools {
24
25 class NnapiDelegateProvider : public DelegateProvider {
26 public:
NnapiDelegateProvider()27 NnapiDelegateProvider() {
28 default_params_.AddParam("use_nnapi", ToolParam::Create<bool>(false));
29 default_params_.AddParam("nnapi_execution_preference",
30 ToolParam::Create<std::string>(""));
31 default_params_.AddParam("nnapi_execution_priority",
32 ToolParam::Create<std::string>(""));
33 default_params_.AddParam("nnapi_accelerator_name",
34 ToolParam::Create<std::string>(""));
35 default_params_.AddParam("disable_nnapi_cpu",
36 ToolParam::Create<bool>(true));
37 default_params_.AddParam("nnapi_allow_fp16",
38 ToolParam::Create<bool>(false));
39 }
40
41 std::vector<Flag> CreateFlags(ToolParams* params) const final;
42
43 void LogParams(const ToolParams& params, bool verbose) const final;
44
45 TfLiteDelegatePtr CreateTfLiteDelegate(const ToolParams& params) const final;
46
GetName() const47 std::string GetName() const final { return "NNAPI"; }
48 };
49 REGISTER_DELEGATE_PROVIDER(NnapiDelegateProvider);
50
CreateFlags(ToolParams * params) const51 std::vector<Flag> NnapiDelegateProvider::CreateFlags(ToolParams* params) const {
52 std::vector<Flag> flags = {
53 CreateFlag<bool>("use_nnapi", params, "use nnapi delegate api"),
54 CreateFlag<std::string>("nnapi_execution_preference", params,
55 "execution preference for nnapi delegate. Should "
56 "be one of the following: fast_single_answer, "
57 "sustained_speed, low_power, undefined"),
58 CreateFlag<std::string>("nnapi_execution_priority", params,
59 "The model execution priority in nnapi, and it "
60 "should be one of the following: default, low, "
61 "medium and high. This requires Android 11+."),
62 CreateFlag<std::string>(
63 "nnapi_accelerator_name", params,
64 "the name of the nnapi accelerator to use (requires Android Q+)"),
65 CreateFlag<bool>("disable_nnapi_cpu", params,
66 "Disable the NNAPI CPU device"),
67 CreateFlag<bool>("nnapi_allow_fp16", params,
68 "Allow fp32 computation to be run in fp16")};
69
70 return flags;
71 }
72
LogParams(const ToolParams & params,bool verbose) const73 void NnapiDelegateProvider::LogParams(const ToolParams& params,
74 bool verbose) const {
75 LOG_TOOL_PARAM(params, bool, "use_nnapi", "Use NNAPI", verbose);
76 if (!params.Get<bool>("use_nnapi")) return;
77
78 LOG_TOOL_PARAM(params, std::string, "nnapi_execution_preference",
79 "NNAPI execution preference", verbose);
80 LOG_TOOL_PARAM(params, std::string, "nnapi_execution_priority",
81 "Model execution priority in nnapi", verbose);
82 LOG_TOOL_PARAM(params, std::string, "nnapi_accelerator_name",
83 "NNAPI accelerator name", verbose);
84
85 std::string string_device_names_list = nnapi::GetStringDeviceNamesList();
86 // Print available devices when possible as it's informative.
87 if (!string_device_names_list.empty()) {
88 TFLITE_LOG(INFO) << "NNAPI accelerators available: ["
89 << string_device_names_list << "]";
90 }
91
92 LOG_TOOL_PARAM(params, bool, "disable_nnapi_cpu", "Disable NNAPI cpu",
93 verbose);
94 LOG_TOOL_PARAM(params, bool, "nnapi_allow_fp16", "Allow fp16 in NNAPI",
95 verbose);
96 }
97
CreateTfLiteDelegate(const ToolParams & params) const98 TfLiteDelegatePtr NnapiDelegateProvider::CreateTfLiteDelegate(
99 const ToolParams& params) const {
100 TfLiteDelegatePtr delegate(nullptr, [](TfLiteDelegate*) {});
101 if (params.Get<bool>("use_nnapi")) {
102 StatefulNnApiDelegate::Options options;
103 std::string accelerator_name =
104 params.Get<std::string>("nnapi_accelerator_name");
105 if (!accelerator_name.empty()) {
106 options.accelerator_name = accelerator_name.c_str();
107 } else {
108 options.disallow_nnapi_cpu = params.Get<bool>("disable_nnapi_cpu");
109 }
110
111 if (params.Get<bool>("nnapi_allow_fp16")) {
112 options.allow_fp16 = true;
113 }
114
115 std::string string_execution_preference =
116 params.Get<std::string>("nnapi_execution_preference");
117 // Only set execution preference if user explicitly passes one. Otherwise,
118 // leave it as whatever NNAPI has as the default.
119 if (!string_execution_preference.empty()) {
120 tflite::StatefulNnApiDelegate::Options::ExecutionPreference
121 execution_preference =
122 tflite::StatefulNnApiDelegate::Options::kUndefined;
123 if (string_execution_preference == "low_power") {
124 execution_preference =
125 tflite::StatefulNnApiDelegate::Options::kLowPower;
126 } else if (string_execution_preference == "sustained_speed") {
127 execution_preference =
128 tflite::StatefulNnApiDelegate::Options::kSustainedSpeed;
129 } else if (string_execution_preference == "fast_single_answer") {
130 execution_preference =
131 tflite::StatefulNnApiDelegate::Options::kFastSingleAnswer;
132 } else if (string_execution_preference == "undefined") {
133 execution_preference =
134 tflite::StatefulNnApiDelegate::Options::kUndefined;
135 } else {
136 TFLITE_LOG(WARN) << "The provided value ("
137 << string_execution_preference
138 << ") is not a valid nnapi execution preference.";
139 }
140 options.execution_preference = execution_preference;
141 }
142
143 std::string string_execution_priority =
144 params.Get<std::string>("nnapi_execution_priority");
145 // Only set execution priority if user explicitly passes one. Otherwise,
146 // leave it as whatever NNAPI has as the default.
147 if (!string_execution_priority.empty()) {
148 int execution_priority = 0;
149 if (string_execution_priority == "default") {
150 execution_priority = ANEURALNETWORKS_PRIORITY_DEFAULT;
151 } else if (string_execution_priority == "low") {
152 execution_priority = ANEURALNETWORKS_PRIORITY_LOW;
153 } else if (string_execution_priority == "medium") {
154 execution_priority = ANEURALNETWORKS_PRIORITY_MEDIUM;
155 } else if (string_execution_priority == "high") {
156 execution_priority = ANEURALNETWORKS_PRIORITY_HIGH;
157 } else {
158 TFLITE_LOG(WARN) << "The provided value (" << string_execution_priority
159 << ") is not a valid nnapi execution priority.";
160 }
161 options.execution_priority = execution_priority;
162 }
163
164 int max_delegated_partitions = params.Get<int>("max_delegated_partitions");
165 if (max_delegated_partitions >= 0) {
166 options.max_number_delegated_partitions = max_delegated_partitions;
167 }
168 const auto* nnapi_impl = NnApiImplementation();
169 if (!nnapi_impl->nnapi_exists) {
170 TFLITE_LOG(WARN) << "NNAPI acceleration is unsupported on this platform.";
171 return delegate;
172 }
173 return TfLiteDelegatePtr(
174 new StatefulNnApiDelegate(nnapi_impl, options),
175 [](TfLiteDelegate* delegate) {
176 delete reinterpret_cast<StatefulNnApiDelegate*>(delegate);
177 });
178 } else if (!params.Get<std::string>("nnapi_accelerator_name").empty()) {
179 TFLITE_LOG(WARN)
180 << "`--use_nnapi=true` must be set for the provided NNAPI accelerator ("
181 << params.Get<std::string>("nnapi_accelerator_name") << ") to be used.";
182 } else if (!params.Get<std::string>("nnapi_execution_preference").empty()) {
183 TFLITE_LOG(WARN) << "`--use_nnapi=true` must be set for the provided NNAPI "
184 "execution preference ("
185 << params.Get<std::string>("nnapi_execution_preference")
186 << ") to be used.";
187 }
188 return delegate;
189 }
190
191 } // namespace tools
192 } // namespace tflite
193