1 /** 2 * Copyright 2022 Huawei Technologies Co., Ltd 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 #ifndef MINDSPORE_NNR_DELEGATE_H 17 #define MINDSPORE_NNR_DELEGATE_H 18 19 #include <vector> 20 #include <map> 21 #include "include/api/delegate.h" 22 #include "include/model.h" 23 #include "src/litert/inner_context.h" 24 #include "nnrt_model_kernel.h" 25 #include "hiai_foundation_wrapper.h" 26 #include "extension_options_parser.h" 27 #include "schema/model_generated.h" 28 #include "neural_network_runtime/neural_network_runtime_type.h" 29 #include "neural_network_runtime/neural_network_runtime.h" 30 #include "neural_network_runtime_inner.h" 31 32 namespace mindspore { 33 namespace lite { 34 struct NNRTOpRange { 35 /* NNRT kernel range in DelegateModel: [begin_iter_, end_iter_) */ 36 KernelIter begin_iter_; 37 KernelIter end_iter_; 38 /* NNRT node range in lite_graph_: [begin_index_, end_index_) */ 39 size_t begin_index_; 40 size_t end_index_; 41 }; 42 43 class NNRTDelegate : public Delegate { 44 public: 45 NNRTDelegate() = default; NNRTDelegate(const NNRtDeviceInfo & nnrt_device_info)46 NNRTDelegate(const NNRtDeviceInfo &nnrt_device_info) : nnrt_device_info_(nnrt_device_info) {} 47 ~NNRTDelegate() override; 48 Status Init() override; 49 Status Build(DelegateModel<schema::Primitive> *model) override; 50 void ShallowCopyLiteGraph(const lite::LiteGraph &liteGraph); 51 void FreeLiteGraph(lite::LiteGraph **liteGraph); SetMetaGraph(const void * meta_graph)52 void SetMetaGraph(const void *meta_graph) { 53 meta_graph_ = meta_graph; 54 } SetDequantTensors(std::vector<Tensor * > * src_tensors)55 void SetDequantTensors(std::vector<Tensor *> *src_tensors) { 56 dequant_src_tensors_ = src_tensors; 57 } 58 static std::vector<NNRTOpRange> GetNNRTSubgraphRanges(DelegateModel<schema::Primitive> *model, 59 const std::vector<bool> &op_supports); 60 61 private: 62 void InitExtensionOptions(); 63 Status BuildNormalModel(DelegateModel<schema::Primitive> *model); 64 OH_NNModel *CreateFullNNModel(); 65 std::vector<bool> QueryOpSupports(OH_NNModel *nn_model); 66 Status CreateLiteGraphForNNRTSubgraph( 67 const std::vector<NNRTOpRange> &nnrt_op_ranges, 68 std::vector<LiteGraph *> *sub_lite_graphs); 69 Status CreateNNRTSubgraphKernels( 70 DelegateModel<schema::Primitive> *model, 71 const std::vector<LiteGraph *> &sub_lite_graphs, 72 const std::vector<NNRTOpRange> &nnrt_subgraph_ranges, 73 std::vector<NNRTModelKernel *> *nnrt_subgraph_kernels); 74 void ReplaceNNRTKernelsInDelegateModel(DelegateModel<schema::Primitive> *model, 75 const std::vector<NNRTOpRange> &nnrt_subgraph_ranges, 76 const std::vector<NNRTModelKernel *> &nnrt_subgraph_kernels); 77 Status PrepareInputs(DelegateModel<schema::Primitive> *model, OH_NNExecutor *oh_nn_executor); 78 Status PrepareOutputs(DelegateModel<schema::Primitive> *model, OH_NNExecutor *oh_nn_executor); 79 Status InitNNCompilation(OH_NNCompilation *nn_compilation) const; 80 static OH_NN_DataType CastToNNRTDataType(mindspore::DataType data_type); 81 bool IsCustomModel() const; 82 Status DequantLiteGraph(LiteGraph *lite_graph); 83 int DequantNodeInputs(LiteGraph::Node *node); 84 schema::Tensor *TensorToSchemaTensor(Tensor *lite_tensor, schema::Tensor *schema_tensor); 85 86 #ifdef SUPPORT_NNRT_METAGRAPH 87 bool CheckNPUPrefix(const std::string prefix_name) const; 88 bool IsKirinNPUWithOnlineInference() const; 89 bool IsKirinNPUWithOfflineInference() const; 90 Status BuildKirinNPUModel(DelegateModel<schema::Primitive> *model); 91 Status BuildOfflineModel(DelegateModel<schema::Primitive> *model); 92 Status CreateFullModelKernel(DelegateModel<schema::Primitive> *model, OH_NNModel *nn_model); 93 #endif 94 95 NNRtDeviceInfo nnrt_device_info_; 96 LiteGraph *lite_graph_ = nullptr; 97 const void *meta_graph_ = nullptr; 98 nnrt::ExtensionOptions extension_options_; 99 std::vector<OH_NNExecutor *> nn_executor_list_; 100 std::vector<Tensor *> *dequant_src_tensors_; 101 std::map<uint32_t, schema::Tensor *> dequant_schema_tensors_; 102 std::map<schema::Tensor *, void *> dequant_schema_tensors_buffer_map_; 103 std::vector<schema::Tensor *> replaced_schema_tensors_; 104 void *hiai_handle_{nullptr}; 105 }; 106 } // namespace lite 107 } // namespace mindspore 108 109 #endif // MINDSPORE_NNR_DELEGATE_H 110