1 /* 2 * Copyright (C) 2017 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 #ifndef ANDROID_ML_NN_RUNTIME_MANAGER_H 18 #define ANDROID_ML_NN_RUNTIME_MANAGER_H 19 20 #include "HalInterfaces.h" 21 #include "Utils.h" 22 #include "VersionedInterfaces.h" 23 24 #include <android-base/macros.h> 25 #include <map> 26 #include <unordered_set> 27 #include <vector> 28 29 namespace android { 30 namespace nn { 31 32 // A unified interface for actual driver devices as well as the CPU 33 class Device { 34 public: ~Device()35 virtual ~Device() {} 36 37 // Get the handle of underlying VersionedIDevice, if any 38 virtual VersionedIDevice* getInterface() = 0; 39 40 // Introspection methods returning device information 41 virtual const char* getName() const = 0; 42 virtual const char* getVersionString() const = 0; 43 virtual int64_t getFeatureLevel() = 0; 44 virtual int32_t getType() const = 0; 45 virtual hidl_vec<Extension> getSupportedExtensions() const = 0; 46 47 // If hidlModel is not compliant with the HAL version of the driver device, 48 // then the behavior depends on whether or not a non-nullptr slicer is 49 // provided. 50 // 51 // If there is no slicer, then no operations are supported. 52 // 53 // If there is a slicer, and it successfully slices the model, then some 54 // operations may be supported. 55 // 56 // See the IModelSlicer class in Utils.h for more details. 57 virtual void getSupportedOperations(const Model& hidlModel, IModelSlicer* slicer, 58 hidl_vec<bool>* supportedOperations) = 0; getSupportedOperations(const Model & hidlModel,hidl_vec<bool> * supportedOperations)59 void getSupportedOperations(const Model& hidlModel, hidl_vec<bool>* supportedOperations) { 60 return getSupportedOperations(hidlModel, nullptr, supportedOperations); 61 } 62 63 virtual PerformanceInfo getPerformance(OperandType type) const = 0; 64 virtual PerformanceInfo getRelaxedFloat32toFloat16PerformanceScalar() const = 0; 65 virtual PerformanceInfo getRelaxedFloat32toFloat16PerformanceTensor() const = 0; 66 virtual std::pair<uint32_t, uint32_t> getNumberOfCacheFilesNeeded() const = 0; 67 bool isCachingSupported() const; 68 69 virtual int prepareModel( 70 const Model& hidlModel, ExecutionPreference executionPreference, 71 const hidl_vec<hidl_handle>& modelCache, const hidl_vec<hidl_handle>& dataCache, 72 const hidl_array<uint8_t, ANEURALNETWORKS_BYTE_SIZE_OF_CACHE_TOKEN>& token, 73 std::shared_ptr<VersionedIPreparedModel>* preparedModel) = 0; 74 virtual int prepareModelFromCache( 75 const hidl_vec<hidl_handle>& modelCache, const hidl_vec<hidl_handle>& dataCache, 76 const hidl_array<uint8_t, ANEURALNETWORKS_BYTE_SIZE_OF_CACHE_TOKEN>& token, 77 std::shared_ptr<VersionedIPreparedModel>* preparedModel) = 0; 78 }; 79 80 // Manages the NN HAL devices. Only one instance of this class will exist. 81 // Use get() to retrieve it. 82 class DeviceManager { 83 public: getDrivers()84 const std::vector<std::shared_ptr<Device>>& getDrivers() const { 85 if (mSetCpuOnly || mDebugNNCpuOnly) { 86 return mDevicesCpuOnly; 87 } 88 return mDevices; 89 } 90 91 // For testing only: setUseCpuOnly(bool useCpuOnly)92 void setUseCpuOnly(bool useCpuOnly) { mSetCpuOnly = useCpuOnly; } getUseCpuOnly()93 bool getUseCpuOnly() const { return mSetCpuOnly; } setSyncExecHal(bool val)94 void setSyncExecHal(bool val) { 95 mSyncExecHal = val; 96 mSyncExecHalSetter = true; 97 } 98 syncExecCpu()99 bool syncExecCpu() const { return mSyncExecCpu; } syncExecHal()100 bool syncExecHal() const { return mSyncExecHal; } syncExecRuntime()101 bool syncExecRuntime() const { return mSyncExecRuntime; } 102 103 // How to handle graph partitioning? 104 // 0 - Don't do graph partitioning. 105 // 1 - Do graph partitioning; but fall back to non-partitioned 106 // execution if there is a partitioning failure. 107 // 2 - Do graph partitioning, and rely on it; there is no fallback. 108 enum { 109 kPartitioningNo = 0, 110 kPartitioningWithFallback = 1, 111 kPartitioningWithoutFallback = 2 112 }; getPartitioning()113 uint32_t getPartitioning() const { return mPartitioning; } partitioningAllowsFallback(uint32_t partitioning)114 static bool partitioningAllowsFallback(uint32_t partitioning) { 115 return partitioning == kPartitioningWithFallback; 116 } 117 strictSlicing()118 bool strictSlicing() const { return mStrictSlicing; } 119 120 // Returns the singleton manager. 121 static DeviceManager* get(); 122 123 // Returns the singleton Cpu device. 124 static std::shared_ptr<Device> getCpuDevice(); 125 126 // The forTest_* functions below are solely intended for use by unit tests. 127 128 // Returns all devices (ignores the cpu-only flags). forTest_getDevices()129 std::vector<std::shared_ptr<Device>> forTest_getDevices() const { return mDevices; } 130 131 // Sets the device list (does not affect cpu-only queries). forTest_setDevices(std::vector<std::shared_ptr<Device>> devices)132 void forTest_setDevices(std::vector<std::shared_ptr<Device>> devices) { 133 mDevices = std::move(devices); 134 } 135 136 // Register a test device. forTest_registerDevice(const char * name,const sp<V1_0::IDevice> & device)137 void forTest_registerDevice(const char* name, const sp<V1_0::IDevice>& device) { 138 registerDevice(name, device); 139 } 140 141 // Re-initialize the list of available devices. forTest_reInitializeDeviceList()142 void forTest_reInitializeDeviceList() { 143 mDevices.clear(); 144 mDevicesCpuOnly.clear(); 145 findAvailableDevices(); 146 } 147 148 // Make a test device 149 static std::shared_ptr<Device> forTest_makeDriverDevice(const std::string& name, 150 const sp<V1_0::IDevice>& device); 151 forTest_isCpuDevice(const ANeuralNetworksDevice * device)152 bool forTest_isCpuDevice(const ANeuralNetworksDevice* device) const { 153 return reinterpret_cast<const Device*>(device) == getCpuDevice().get(); 154 } 155 156 private: 157 // Builds the list of available drivers and queries their capabilities. 158 DeviceManager(); 159 160 // Adds a device for the manager to use. 161 void registerDevice(const char* name, const sp<V1_0::IDevice>& device); 162 163 void findAvailableDevices(); 164 165 // List of all the devices we discovered (including CpuDevice). 166 std::vector<std::shared_ptr<Device>> mDevices; 167 168 // We set this one to have CpuDevice only. To be used when m*CpuOnly is true. 169 std::vector<std::shared_ptr<Device>> mDevicesCpuOnly; 170 171 // If either of these is true, we'll ignore the drivers that are 172 // on the device and run everything on the CPU. 173 bool mSetCpuOnly = false; // set by setUseCpuOnly() 174 bool mDebugNNCpuOnly = false; // derived from system property debug.nn.cpuonly 175 176 // synchronous execution 177 bool mSyncExecCpu = true; 178 bool mSyncExecHal = true; // Call executeSynchronously() when available on device. 179 bool mSyncExecHalSetter = false; // Has mSyncExecHal been set by setSyncExecHal()? 180 // If so, don't allow the setting to be overridden 181 // by system property debug.nn.syncexec-hal 182 bool mSyncExecRuntime = false; 183 184 static const uint32_t kPartitioningDefault = kPartitioningWithFallback; 185 uint32_t mPartitioning = kPartitioningDefault; 186 187 bool mStrictSlicing = false; 188 }; 189 190 } // namespace nn 191 } // namespace android 192 193 #endif // ANDROID_ML_NN_RUNTIME_MANAGER_H 194