1 // 2 // Copyright © 2020-2023 Arm Ltd and Contributors. All rights reserved. 3 // SPDX-License-Identifier: MIT 4 // 5 6 #pragma once 7 8 #include <armnn/ArmNN.hpp> 9 #include <armnn/Optional.hpp> 10 11 #include <string> 12 #include <vector> 13 14 namespace armnnDelegate 15 { 16 17 struct DelegateOptionsImpl; 18 19 class DelegateOptions 20 { 21 public: 22 ~DelegateOptions(); 23 DelegateOptions(); 24 DelegateOptions(const DelegateOptions& other); 25 26 DelegateOptions(armnn::Compute computeDevice, 27 const std::vector<armnn::BackendOptions>& backendOptions = {}, 28 armnn::Optional<armnn::LogSeverity> logSeverityLevel = armnn::EmptyOptional()); 29 30 DelegateOptions(const std::vector<armnn::BackendId>& backends, 31 const std::vector<armnn::BackendOptions>& backendOptions = {}, 32 armnn::Optional<armnn::LogSeverity> logSeverityLevel = armnn::EmptyOptional()); 33 34 DelegateOptions(armnn::Compute computeDevice, 35 const armnn::OptimizerOptionsOpaque& optimizerOptions, 36 const armnn::Optional<armnn::LogSeverity>& logSeverityLevel = armnn::EmptyOptional(), 37 const armnn::Optional<armnn::DebugCallbackFunction>& func = armnn::EmptyOptional()); 38 39 DelegateOptions(const std::vector<armnn::BackendId>& backends, 40 const armnn::OptimizerOptionsOpaque& optimizerOptions, 41 const armnn::Optional<armnn::LogSeverity>& logSeverityLevel = armnn::EmptyOptional(), 42 const armnn::Optional<armnn::DebugCallbackFunction>& func = armnn::EmptyOptional()); 43 44 /** 45 * This constructor processes delegate options in form of command line arguments. 46 * It works in conjunction with the TfLite external delegate plugin. 47 * 48 * Available options: 49 * 50 * Option key: "backends" \n 51 * Possible values: ["EthosNPU"/"GpuAcc"/"CpuAcc"/"CpuRef"] \n 52 * Descriptions: A comma separated list without whitespaces of 53 * backends which should be used for execution. Falls 54 * back to next backend in list if previous doesn't 55 * provide support for operation. e.g. "GpuAcc,CpuAcc" 56 * 57 * Option key: "dynamic-backends-path" \n 58 * Possible values: [filenameString] \n 59 * Descriptions: This is the directory that will be searched for any dynamic backends. 60 * 61 * Option key: "logging-severity" \n 62 * Possible values: ["trace"/"debug"/"info"/"warning"/"error"/"fatal"] \n 63 * Description: Sets the logging severity level for ArmNN. Logging 64 * is turned off if this option is not provided. 65 * 66 * Option key: "gpu-tuning-level" \n 67 * Possible values: ["0"/"1"/"2"/"3"] \n 68 * Description: 0=UseOnly(default), 1=RapidTuning, 2=NormalTuning, 69 * 3=ExhaustiveTuning. Requires option gpu-tuning-file. 70 * 1,2 and 3 will create a tuning-file, 0 will apply the 71 * tunings from an existing file 72 * 73 * Option key: "gpu-mlgo-tuning-file" \n 74 * Possible values: [filenameString] \n 75 * Description: File name for the MLGO tuning file 76 * 77 * Option key: "gpu-tuning-file" \n 78 * Possible values: [filenameString] \n 79 * Description: File name for the tuning file. 80 * 81 * Option key: "gpu-enable-profiling" \n 82 * Possible values: ["true"/"false"] \n 83 * Description: Enables GPU profiling 84 * 85 * Option key: "gpu-kernel-profiling-enabled" \n 86 * Possible values: ["true"/"false"] \n 87 * Description: Enables GPU kernel profiling 88 * 89 * Option key: "save-cached-network" \n 90 * Possible values: ["true"/"false"] \n 91 * Description: Enables saving of the cached network to a file, 92 * specified with the cached-network-filepath option 93 * 94 * Option key: "cached-network-filepath" \n 95 * Possible values: [filenameString] \n 96 * Description: If non-empty, the given file will be used to load/save the cached network. 97 * If save-cached-network is given then the cached network will be saved to the given file. 98 * To save the cached network a file must already exist. 99 * If save-cached-network is not given then the cached network will be loaded from the given file. 100 * This will remove initial compilation time of kernels and speed up the first execution. 101 * 102 * Option key: "enable-fast-math" \n 103 * Possible values: ["true"/"false"] \n 104 * Description: Enables fast_math options in backends that support it 105 * 106 * Option key: "number-of-threads" \n 107 * Possible values: ["1"-"64"] \n 108 * Description: Assign the number of threads used by the CpuAcc backend. 109 * Default is set to 0 (Backend will decide number of threads to use). 110 * 111 * Option key: "reduce-fp32-to-fp16" \n 112 * Possible values: ["true"/"false"] \n 113 * Description: Reduce Fp32 data to Fp16 for faster processing 114 * 115 * Option key: "reduce-fp32-to-bf16" \n 116 * Possible values: ["true"/"false"] \n 117 * Description: This option is currently ignored. Please enable Fast Math in the CpuAcc or GpuAcc backends. 118 * 119 * Option key: "debug-data" \n 120 * Possible values: ["true"/"false"] \n 121 * Description: Add debug data for easier troubleshooting 122 * 123 * Option key: "memory-import" \n 124 * Possible values: ["true"/"false"] \n 125 * Description: Enable memory import 126 * 127 * Option key: "enable-internal-profiling" \n 128 * Possible values: ["true"/"false"] \n 129 * Description: Enable the internal profiling feature. 130 * 131 * Option key: "internal-profiling-detail" \n 132 * Possible values: [1/2] \n 133 * Description: Set the detail on the internal profiling. 1 = DetailsWithEvents, 2 = DetailsOnly. 134 * 135 * Option key: "enable-external-profiling" \n 136 * Possible values: ["true"/"false"] \n 137 * Description: Enable the external profiling feature. 138 * 139 * Option key: "timeline-profiling" \n 140 * Possible values: ["true"/"false"] \n 141 * Description: Indicates whether external timeline profiling is enabled or not. 142 * 143 * Option key: "outgoing-capture-file" \n 144 * Possible values: [filenameString] \n 145 * Description: Path to a file in which outgoing timeline profiling messages will be stored. 146 * 147 * Option key: "incoming-capture-file" \n 148 * Possible values: [filenameString] \n 149 * Description: Path to a file in which incoming timeline profiling messages will be stored. 150 * 151 * Option key: "file-only-external-profiling" \n 152 * Possible values: ["true"/"false"] \n 153 * Description: Enable profiling output to file only. 154 * 155 * Option key: "counter-capture-period" \n 156 * Possible values: Integer, Default is 10000u 157 * Description: Value in microseconds of the profiling capture period. \n 158 * 159 * Option key: "profiling-file-format" \n 160 * Possible values: String of ["binary"] \n 161 * Description: The format of the file used for outputting profiling data. Currently on "binary" is supported. 162 * 163 * Option key: "serialize-to-dot" \n 164 * Possible values: [filenameString] \n 165 * Description: Serialize the optimized network to the file specified in "dot" format. 166 * 167 * Option key: "infer-output-shape" \n 168 * Possible values: ["true"/"false"] \n 169 * Description: Infers output tensor shape from input tensor shape and validate where applicable. 170 * 171 * Option key: "allow-expanded-dims" \n 172 * Possible values: ["true"/"false"] \n 173 * Description: If true will disregard dimensions with a size of 1 when validating tensor shapes but tensor 174 * sizes must still match. \n 175 * This is an Experimental parameter that is incompatible with "infer-output-shape". \n 176 * This parameter may be removed in a later update. 177 * 178 * Option key: "disable-tflite-runtime-fallback" \n 179 * Possible values: ["true"/"false"] \n 180 * Description: Disable TfLite Runtime fallback in the Arm NN TfLite delegate. 181 * An exception will be thrown if unsupported operators are encountered. 182 * This option is only for testing purposes. 183 * 184 * @param[in] option_keys Delegate option names 185 * @param[in] options_values Delegate option values 186 * @param[in] num_options Number of delegate options 187 * @param[in,out] report_error Error callback function 188 * 189 */ 190 DelegateOptions(char const* const* options_keys, 191 char const* const* options_values, 192 size_t num_options, 193 void (*report_error)(const char*)); 194 195 const std::vector<armnn::BackendId>& GetBackends() const; 196 197 void SetBackends(const std::vector<armnn::BackendId>& backends); 198 199 void SetDynamicBackendsPath(const std::string& dynamicBackendsPath); 200 201 const std::string& GetDynamicBackendsPath() const; 202 203 void SetGpuProfilingState(bool gpuProfilingState); 204 205 bool GetGpuProfilingState(); 206 207 const std::vector<armnn::BackendOptions>& GetBackendOptions() const; 208 209 /// Appends a backend option to the list of backend options 210 void AddBackendOption(const armnn::BackendOptions& option); 211 212 /// Sets the severity level for logging within ArmNN that will be used on creation of the delegate 213 void SetLoggingSeverity(const armnn::LogSeverity& level); 214 void SetLoggingSeverity(const std::string& level); 215 216 /// Returns the severity level for logging within ArmNN 217 armnn::LogSeverity GetLoggingSeverity(); 218 219 bool IsLoggingEnabled(); 220 221 const armnn::OptimizerOptionsOpaque& GetOptimizerOptions() const; 222 223 void SetOptimizerOptions(const armnn::OptimizerOptionsOpaque& optimizerOptions); 224 225 const armnn::Optional<armnn::DebugCallbackFunction>& GetDebugCallbackFunction() const; 226 227 void SetInternalProfilingParams(bool internalProfilingState, 228 const armnn::ProfilingDetailsMethod& internalProfilingDetail); 229 230 bool GetInternalProfilingState() const; 231 232 const armnn::ProfilingDetailsMethod& GetInternalProfilingDetail() const; 233 234 void SetSerializeToDot(const std::string& serializeToDotFile); 235 236 const std::string& GetSerializeToDot() const; 237 238 /// @Note: This might overwrite options that were set with other setter functions of DelegateOptions 239 void SetRuntimeOptions(const armnn::IRuntime::CreationOptions& runtimeOptions); 240 241 const armnn::IRuntime::CreationOptions& GetRuntimeOptions(); 242 243 void DisableTfLiteRuntimeFallback(bool fallbackState); 244 245 bool TfLiteRuntimeFallbackDisabled(); 246 247 private: 248 std::unique_ptr<armnnDelegate::DelegateOptionsImpl> p_DelegateOptionsImpl; 249 250 }; 251 252 } // namespace armnnDelegate 253