1 /* Copyright 2020 The TensorFlow Authors. All Rights Reserved.
2
3 Licensed under the Apache License, Version 2.0 (the "License");
4 you may not use this file except in compliance with the License.
5 You may obtain a copy of the License at
6
7 http://www.apache.org/licenses/LICENSE-2.0
8
9 Unless required by applicable law or agreed to in writing, software
10 distributed under the License is distributed on an "AS IS" BASIS,
11 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 See the License for the specific language governing permissions and
13 limitations under the License.
14 ==============================================================================*/
15 #include <string>
16
17 #include "tensorflow/lite/tools/delegates/delegate_provider.h"
18 #include "tensorflow/lite/tools/evaluation/utils.h"
19 #if TFLITE_SUPPORTS_GPU_DELEGATE
20 #include "tensorflow/lite/delegates/gpu/delegate.h"
21 #elif defined(__APPLE__)
22 #include "TargetConditionals.h"
23 #if TARGET_OS_IPHONE && !TARGET_IPHONE_SIMULATOR
24 // Only enable metal delegate when using a real iPhone device.
25 #define REAL_IPHONE_DEVICE
26 #include "tensorflow/lite/delegates/gpu/metal_delegate.h"
27 #endif
28 #endif
29
30 namespace tflite {
31 namespace tools {
32
33 class GpuDelegateProvider : public DelegateProvider {
34 public:
GpuDelegateProvider()35 GpuDelegateProvider() {
36 default_params_.AddParam("use_gpu", ToolParam::Create<bool>(false));
37 #if TFLITE_SUPPORTS_GPU_DELEGATE || defined(REAL_IPHONE_DEVICE)
38 default_params_.AddParam("gpu_precision_loss_allowed",
39 ToolParam::Create<bool>(true));
40 default_params_.AddParam("gpu_experimental_enable_quant",
41 ToolParam::Create<bool>(true));
42 #endif
43 #if TFLITE_SUPPORTS_GPU_DELEGATE
44 default_params_.AddParam("gpu_backend", ToolParam::Create<std::string>(""));
45 #endif
46 #if defined(REAL_IPHONE_DEVICE)
47 default_params_.AddParam("gpu_wait_type",
48 ToolParam::Create<std::string>(""));
49 #endif
50 }
51
52 std::vector<Flag> CreateFlags(ToolParams* params) const final;
53
54 void LogParams(const ToolParams& params, bool verbose) const final;
55
56 TfLiteDelegatePtr CreateTfLiteDelegate(const ToolParams& params) const final;
57
GetName() const58 std::string GetName() const final { return "GPU"; }
59 };
60 REGISTER_DELEGATE_PROVIDER(GpuDelegateProvider);
61
CreateFlags(ToolParams * params) const62 std::vector<Flag> GpuDelegateProvider::CreateFlags(ToolParams* params) const {
63 std::vector<Flag> flags = {
64 CreateFlag<bool>("use_gpu", params, "use gpu"),
65 #if TFLITE_SUPPORTS_GPU_DELEGATE || defined(REAL_IPHONE_DEVICE)
66 CreateFlag<bool>("gpu_precision_loss_allowed", params,
67 "Allow to process computation in lower precision than "
68 "FP32 in GPU. By default, it's enabled."),
69 CreateFlag<bool>("gpu_experimental_enable_quant", params,
70 "Whether to enable the GPU delegate to run quantized "
71 "models or not. By default, it's enabled."),
72 #endif
73 #if TFLITE_SUPPORTS_GPU_DELEGATE
74 CreateFlag<std::string>(
75 "gpu_backend", params,
76 "Force the GPU delegate to use a particular backend for execution, and "
77 "fail if unsuccessful. Should be one of: cl, gl"),
78 #endif
79 #if defined(REAL_IPHONE_DEVICE)
80 CreateFlag<std::string>(
81 "gpu_wait_type", params,
82 "GPU wait type. Should be one of the following: passive, active, "
83 "do_not_wait, aggressive"),
84 #endif
85 };
86 return flags;
87 }
88
LogParams(const ToolParams & params,bool verbose) const89 void GpuDelegateProvider::LogParams(const ToolParams& params,
90 bool verbose) const {
91 LOG_TOOL_PARAM(params, bool, "use_gpu", "Use gpu", verbose);
92 #if TFLITE_SUPPORTS_GPU_DELEGATE || defined(REAL_IPHONE_DEVICE)
93 LOG_TOOL_PARAM(params, bool, "gpu_precision_loss_allowed",
94 "Allow lower precision in gpu", verbose);
95 LOG_TOOL_PARAM(params, bool, "gpu_experimental_enable_quant",
96 "Enable running quant models in gpu", verbose);
97 #endif
98 #if TFLITE_SUPPORTS_GPU_DELEGATE
99 LOG_TOOL_PARAM(params, std::string, "gpu_backend", "GPU backend", verbose);
100 #endif
101 #if defined(REAL_IPHONE_DEVICE)
102 LOG_TOOL_PARAM(params, std::string, "gpu_wait_type", "GPU delegate wait type",
103 verbose);
104 #endif
105 }
106
CreateTfLiteDelegate(const ToolParams & params) const107 TfLiteDelegatePtr GpuDelegateProvider::CreateTfLiteDelegate(
108 const ToolParams& params) const {
109 TfLiteDelegatePtr delegate(nullptr, [](TfLiteDelegate*) {});
110
111 if (params.Get<bool>("use_gpu")) {
112 #if TFLITE_SUPPORTS_GPU_DELEGATE
113 TfLiteGpuDelegateOptionsV2 gpu_opts = TfLiteGpuDelegateOptionsV2Default();
114 if (params.Get<bool>("gpu_precision_loss_allowed")) {
115 gpu_opts.inference_priority1 = TFLITE_GPU_INFERENCE_PRIORITY_MIN_LATENCY;
116 gpu_opts.inference_priority2 =
117 TFLITE_GPU_INFERENCE_PRIORITY_MIN_MEMORY_USAGE;
118 gpu_opts.inference_priority3 =
119 TFLITE_GPU_INFERENCE_PRIORITY_MAX_PRECISION;
120 }
121 if (params.Get<bool>("gpu_experimental_enable_quant")) {
122 gpu_opts.experimental_flags |= TFLITE_GPU_EXPERIMENTAL_FLAGS_ENABLE_QUANT;
123 }
124 std::string gpu_backend = params.Get<std::string>("gpu_backend");
125 if (!gpu_backend.empty()) {
126 if (gpu_backend == "cl") {
127 gpu_opts.experimental_flags |= TFLITE_GPU_EXPERIMENTAL_FLAGS_CL_ONLY;
128 } else if (gpu_backend == "gl") {
129 gpu_opts.experimental_flags |= TFLITE_GPU_EXPERIMENTAL_FLAGS_GL_ONLY;
130 }
131 }
132 gpu_opts.max_delegated_partitions =
133 params.Get<int>("max_delegated_partitions");
134 delegate = evaluation::CreateGPUDelegate(&gpu_opts);
135 #elif defined(REAL_IPHONE_DEVICE)
136 TFLGpuDelegateOptions gpu_opts = {0};
137 gpu_opts.allow_precision_loss =
138 params.Get<bool>("gpu_precision_loss_allowed");
139 gpu_opts.enable_quantization =
140 params.Get<bool>("gpu_experimental_enable_quant");
141
142 std::string string_gpu_wait_type = params.Get<std::string>("gpu_wait_type");
143 if (!string_gpu_wait_type.empty()) {
144 TFLGpuDelegateWaitType wait_type = TFLGpuDelegateWaitTypePassive;
145 if (string_gpu_wait_type == "passive") {
146 wait_type = TFLGpuDelegateWaitTypePassive;
147 } else if (string_gpu_wait_type == "active") {
148 wait_type = TFLGpuDelegateWaitTypeActive;
149 } else if (string_gpu_wait_type == "do_not_wait") {
150 wait_type = TFLGpuDelegateWaitTypeDoNotWait;
151 } else if (string_gpu_wait_type == "aggressive") {
152 wait_type = TFLGpuDelegateWaitTypeAggressive;
153 }
154 gpu_opts.wait_type = wait_type;
155 }
156 delegate = TfLiteDelegatePtr(TFLGpuDelegateCreate(&gpu_opts),
157 &TFLGpuDelegateDelete);
158 #else
159 TFLITE_LOG(WARN) << "The GPU delegate compile options are only supported "
160 "on Android or iOS platforms or when the tool was "
161 "built with -DCL_DELEGATE_NO_GL.";
162 delegate = evaluation::CreateGPUDelegate();
163 #endif
164
165 if (!delegate.get()) {
166 TFLITE_LOG(WARN) << "GPU acceleration is unsupported on this platform.";
167 }
168 }
169 return delegate;
170 }
171
172 } // namespace tools
173 } // namespace tflite
174