1 /* 2 * Copyright (c) 2023 Huawei Device Co., Ltd. 3 * Licensed under the Apache License, Version 2.0 (the "License"); 4 * you may not use this file except in compliance with the License. 5 * You may obtain a copy of the License at 6 * 7 * http://www.apache.org/licenses/LICENSE-2.0 8 * 9 * Unless required by applicable law or agreed to in writing, software 10 * distributed under the License is distributed on an "AS IS" BASIS, 11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 * See the License for the specific language governing permissions and 13 * limitations under the License. 14 */ 15 16 #ifndef NEURAL_NETWORK_RUNTIME_NNEXECUTOR_H 17 #define NEURAL_NETWORK_RUNTIME_NNEXECUTOR_H 18 19 #include <mutex> 20 #include "executor.h" 21 #include "device.h" 22 #include "prepared_model.h" 23 #include "nn_tensor.h" 24 #include "log.h" 25 26 #include "event_handler.h" 27 #include "event_runner.h" 28 29 #include <chrono> 30 namespace OHOS { 31 namespace NeuralNetworkRuntime { 32 class NNExecutor : public Executor { 33 public: 34 NNExecutor(size_t backendID, 35 std::shared_ptr<Device> device, 36 std::shared_ptr<PreparedModel> preparedModel, 37 const std::vector<std::pair<std::shared_ptr<TensorDesc>, OH_NN_TensorType>>& inputTensorDescs, 38 const std::vector<std::pair<std::shared_ptr<TensorDesc>, OH_NN_TensorType>>& outputTensorDescs, 39 std::string cachePath, uint32_t cacheVersion, ExtensionConfig extensionConfig, bool enableFp16, 40 OH_NN_PerformanceMode performance, OH_NN_Priority priority); 41 ~NNExecutor() override; 42 43 OH_NN_ReturnCode GetInputDimRange(size_t inputIndex, 44 size_t** minInputDims, 45 size_t** maxInputDims, 46 size_t* shapeNum) const override; 47 OH_NN_ReturnCode GetOutputShape(uint32_t outputIndex, int32_t** shape, uint32_t* shapeNum) const override; 48 49 size_t GetInputNum() const override; 50 size_t GetOutputNum() const override; 51 NN_TensorDesc* CreateInputTensorDesc(size_t index) const override; 52 NN_TensorDesc* CreateOutputTensorDesc(size_t index) const override; 53 54 OH_NN_ReturnCode SetOnRunDone(NN_OnRunDone onRunDone) override; 55 OH_NN_ReturnCode SetOnServiceDied(NN_OnServiceDied onServiceDied) override; 56 OH_NN_ReturnCode RunSync(NN_Tensor* inputTensors[], 57 size_t inputSize, 58 NN_Tensor* outputTensors[], 59 size_t outputSize) override; 60 OH_NN_ReturnCode RunAsync(NN_Tensor* inputTensors[], 61 size_t inputSize, 62 NN_Tensor* outputTensors[], 63 size_t outputSize, 64 int32_t timeout, 65 void* userData) override; 66 OH_NN_ReturnCode GetModelID(uint32_t& modelId) const override; 67 size_t GetBackendID() override; 68 OH_NN_ReturnCode SetExtensionConfig(const std::unordered_map<std::string, std::vector<char>>& configs) override; 69 ExecutorConfig* GetExecutorConfig() const override; 70 71 // The following APIs are compatible with older versions 72 OH_NN_ReturnCode SetInput(uint32_t index, const OH_NN_Tensor& nnTensor, const void* buffer, size_t length); 73 OH_NN_ReturnCode SetInputFromMemory(uint32_t index, const OH_NN_Tensor& nnTensor, const OH_NN_Memory& memory); 74 OH_NN_ReturnCode SetOutput(uint32_t index, void* buffer, size_t length); 75 OH_NN_ReturnCode SetOutputFromMemory(uint32_t index, const OH_NN_Memory& memory); 76 77 OH_NN_ReturnCode CreateInputMemory(uint32_t index, size_t length, OH_NN_Memory** memory); 78 OH_NN_ReturnCode CreateOutputMemory(uint32_t index, size_t length, OH_NN_Memory** memory); 79 OH_NN_ReturnCode DestroyInputMemory(uint32_t index, OH_NN_Memory** memory); 80 OH_NN_ReturnCode DestroyOutputMemory(uint32_t index, OH_NN_Memory** memory); 81 82 OH_NN_ReturnCode Run(); 83 84 bool DeinitModel(std::string mode) override; 85 OH_NN_ReturnCode SetDeinitModelCallBack() override; 86 OH_NN_ReturnCode UnSetDeinitModelCallBack() override; 87 88 private: 89 OH_NN_ReturnCode GetInputDimVec() const; 90 OH_NN_ReturnCode CheckInputDimRanges(NN_Tensor* inputTensors[], size_t inputSize); 91 92 // The following APIs are compatible with older versions 93 OH_NN_ReturnCode Run(const std::vector<std::shared_ptr<NNTensor>>& inputTensors, 94 std::vector<std::shared_ptr<NNTensor>>& outputTensors); 95 bool CompareAttribute( 96 const std::pair<std::shared_ptr<TensorDesc>, OH_NN_TensorType>& tensorDesc, const NNTensor& tensor) const; 97 std::shared_ptr<NNTensor> BuildNNTensorFromDesc( 98 const std::pair<std::shared_ptr<TensorDesc>, OH_NN_TensorType>& tensorDesc); 99 OH_NN_ReturnCode BuildInputTensor(uint32_t index, const OH_NN_Tensor& nnTensor, 100 std::shared_ptr<NNTensor> inputTensor) const; 101 OH_NN_ReturnCode SetInputTensorWithCurrentBuffer(uint32_t index, std::shared_ptr<NNTensor> inputTensor, 102 const void* buffer, size_t dataLength, size_t curBufferLength); 103 void SetInputTensorWithNewBuffer(uint32_t index, std::shared_ptr<NNTensor> inputTensor, 104 const void* inputBuffer, size_t length, bool isInnerMem); 105 OH_NN_ReturnCode CheckInputDimRanges(uint32_t index, const OH_NN_Tensor& nnTensor) const; 106 OH_NN_ReturnCode DeserializedTensorsFromBuffer( 107 const Buffer& buffer, std::vector<std::pair<std::shared_ptr<TensorDesc>, OH_NN_TensorType>>& tensorDescs); 108 OH_NN_ReturnCode Reload(); 109 OH_NN_ReturnCode ReinitScheduling(uint32_t hiaimodelID, bool* needModelLatency, const char* cachePath); 110 OH_NN_ReturnCode DeinitScheduling(uint32_t hiaimodelID); 111 OH_NN_ReturnCode GetNNRtModelIDFromCache(const std::string& path, const std::string& modelName, 112 size_t& nnrtModelID); 113 114 private: 115 size_t m_backendID {0}; 116 std::shared_ptr<Device> m_device {nullptr}; 117 std::shared_ptr<PreparedModel> m_preparedModel {nullptr}; 118 std::vector<std::pair<std::shared_ptr<TensorDesc>, OH_NN_TensorType>> m_inputTensorDescs; 119 std::vector<std::pair<std::shared_ptr<TensorDesc>, OH_NN_TensorType>> m_outputTensorDescs; 120 std::string m_cachePath; 121 uint32_t m_cacheVersion {0}; 122 ExtensionConfig m_extensionConfig; 123 bool m_enableFp16 {false}; 124 OH_NN_PerformanceMode m_performance {OH_NN_PERFORMANCE_NONE}; 125 OH_NN_Priority m_priority {OH_NN_PRIORITY_NONE}; 126 uint32_t m_originHiaiModelId; 127 128 // The following parameters are provided for compatibility with older versions 129 struct ExeTensor { 130 std::shared_ptr<NNTensor> tensor {nullptr}; 131 void* userBuffer {nullptr}; 132 size_t userBufferLength {0}; 133 bool isInnerMem {false}; 134 }; 135 bool m_isRun {false}; 136 ExecutorConfig* m_executorConfig {nullptr}; 137 std::unordered_map<int, ExeTensor> m_inputTensors; 138 std::unordered_map<int, ExeTensor> m_outputTensors; 139 std::unordered_map<int, std::vector<void*>> m_inputCreatedMem; 140 std::unordered_map<int, std::vector<void*>> m_outputCreatedMem; 141 mutable std::vector<std::vector<size_t>> m_minInputDimsVec; 142 mutable std::vector<std::vector<size_t>> m_maxInputDimsVec; 143 144 std::shared_ptr<OHOS::AppExecFwk::EventRunner> m_autoUnloadRunner; 145 std::shared_ptr<OHOS::AppExecFwk::EventHandler> m_autoUnloadHandler; 146 uint64_t m_executorid; 147 std::mutex m_mutex; 148 }; 149 } // namespace NeuralNetworkRuntime 150 } // namespace OHOS 151 #endif // NEURAL_NETWORK_RUNTIME_NNEXECUTOR_H 152