1 /**
2 * Copyright 2022 Huawei Technologies Co., Ltd
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16 #include "nnrt_delegate.h"
17 #include "checker/primitive_check.h"
18 #include "src/common/log_adapter.h"
19 #include "interfaces/kits/c/neural_network_runtime.h"
20 #include "interfaces/innerkits/c/neural_network_runtime_inner.h"
21 #include "nnrt_model_kernel.h"
22
Build(DelegateModel<schema::Primitive> * model)23 mindspore::Status mindspore::NNRTDelegate::Build(DelegateModel<schema::Primitive> *model) {
24 if (this->nnrt_lite_graph == nullptr) {
25 MS_LOG(ERROR) << "nnrt_lite_graph is nullptr.";
26 return mindspore::kLiteError;
27 }
28 if (this->nnrt_lite_graph->sub_graphs_.empty()) {
29 // must have at lease one subgraph
30 MS_LOG(ERROR) << "must have at lease one subgraph";
31 return mindspore::kLiteError;
32 }
33 OH_NN_ReturnCode ret_code;
34 OH_NNModel *oh_nnmodel = OH_NNModel_Construct();
35 if (oh_nnmodel == nullptr) {
36 MS_LOG(ERROR) << "Construct NNModel failed, oh_nnmodel is nullptr.";
37 return mindspore::kLiteError;
38 }
39
40 ret_code = OH_NNModel_BuildFromLiteGraph(oh_nnmodel, this->nnrt_lite_graph);
41 if (ret_code != OH_NN_SUCCESS) {
42 MS_LOG(ERROR) << "Build NNModel failed, OH_NN_ReturnCode = " << ret_code;
43 OH_NNModel_Destroy(&oh_nnmodel);
44 return mindspore::kLiteError;
45 }
46 MS_LOG(INFO) << "NNRTDelegate creates NNModel success.";
47
48 OH_NNCompilation *oh_nn_compilation = nullptr;
49 oh_nn_compilation = OH_NNCompilation_Construct(oh_nnmodel);
50
51 if (oh_nn_compilation == nullptr) {
52 MS_LOG(ERROR) << "Construct NNCompilation failed";
53 OH_NNModel_Destroy(&oh_nnmodel);
54 return mindspore::kLiteError;
55 }
56 MS_LOG(INFO) << "NNRTDelegate creates NNCompilation success.";
57
58 const size_t *allDevicesID = nullptr;
59 uint32_t device_count = 0;
60 ret_code = OH_NNDevice_GetAllDevicesID(&allDevicesID, &device_count);
61 if (ret_code != OH_NN_SUCCESS) {
62 MS_LOG(ERROR) << "NNModel GetAllDevicesID failed, OH_NN_ReturnCode = " << ret_code;
63 OH_NNCompilation_Destroy(&oh_nn_compilation);
64 OH_NNModel_Destroy(&oh_nnmodel);
65 return mindspore::kLiteError;
66 }
67
68 if (device_count <= 0) {
69 MS_LOG(WARNING) << "No NNRt Device found, fall back to CPU. ";
70 // OH_NNCompilation_Destroy(&oh_nn_compilation);
71 // OH_NNModel_Destroy(&oh_nnmodel);
72 return mindspore::kSuccess;
73 }
74 MS_LOG(INFO) << "NNRTDelegate GetAllDevicesID success.";
75
76 // check if model ops are supported
77 const bool *issupported = nullptr;
78 uint32_t op_count = 0;
79 ret_code = OH_NNModel_GetAvailableOperations(oh_nnmodel, allDevicesID[0], &issupported, &op_count);
80 if (ret_code != OH_NN_SUCCESS) {
81 MS_LOG(ERROR) << "NNModel GetAvailableOperations failed, OH_NN_ReturnCode = " << ret_code
82 << ", maybe due to dataParcel data length limitaion. Fall back to CPU.";
83 OH_NNCompilation_Destroy(&oh_nn_compilation);
84 OH_NNModel_Destroy(&oh_nnmodel);
85 return mindspore::kSuccess;
86 }
87 uint32_t supported_op_count = 0;
88 for (uint32_t i = 0; i < op_count; i++) {
89 if (issupported[i]) {
90 supported_op_count++;
91 }
92 }
93 if (op_count != supported_op_count) {
94 MS_LOG(WARNING) << "this model has " << op_count << "ops, but NNRT only support " << supported_op_count
95 << " ops, fall back to CPU.";
96 // must support all op, else fall back to CPU
97 OH_NNCompilation_Destroy(&oh_nn_compilation);
98 OH_NNModel_Destroy(&oh_nnmodel);
99 return mindspore::kSuccess;
100 }
101 MS_LOG(INFO) << "NNRtDelegate supports all op in this model.";
102
103 ret_code = OH_NNCompilation_SetDevice(oh_nn_compilation, allDevicesID[0]);
104
105 if (ret_code != OH_NN_SUCCESS) {
106 MS_LOG(ERROR) << "NNCompilation SetDevice failed, OH_NN_ReturnCode = " << ret_code;
107 OH_NNCompilation_Destroy(&oh_nn_compilation);
108 OH_NNModel_Destroy(&oh_nnmodel);
109 return mindspore::kLiteError;
110 }
111
112 ret_code = OH_NNCompilation_Build(oh_nn_compilation);
113
114 if (ret_code != OH_NN_SUCCESS) {
115 MS_LOG(ERROR) << "Build NNCompilation failed, OH_NN_ReturnCode = " << ret_code;
116 OH_NNCompilation_Destroy(&oh_nn_compilation);
117 OH_NNModel_Destroy(&oh_nnmodel);
118 return mindspore::kLiteError;
119 }
120
121 MS_LOG(DEBUG) << "NNRTDelegate SetDevice success.";
122
123 OH_NNExecutor *oh_nn_executor = nullptr;
124 oh_nn_executor = OH_NNExecutor_Construct(oh_nn_compilation);
125 if (oh_nn_executor == nullptr) {
126 MS_LOG(ERROR) << "Construct NNCompilation SetDevice failed, OH_NN_ReturnCode = " << ret_code;
127 OH_NNCompilation_Destroy(&oh_nn_compilation);
128 OH_NNModel_Destroy(&oh_nnmodel);
129 return mindspore::kLiteError;
130 }
131 MS_LOG(DEBUG) << "NNRTDelegate creates NNExecutor success.";
132 mindspore::Status prepare_data_ret;
133 auto nnr_model_kernel = new (std::nothrow) NNRTModelKernel(oh_nn_executor, model->inputs(), model->outputs());
134 if (nnr_model_kernel == nullptr) {
135 MS_LOG(ERROR) << "new NNRTModelKernel failed";
136 return mindspore::kLiteError;
137 }
138 OH_NNCompilation_Destroy(&oh_nn_compilation);
139 OH_NNModel_Destroy(&oh_nnmodel);
140 KernelIter from = model->BeginKernelIterator();
141 KernelIter end = model->EndKernelIterator();
142 model->Replace(from, end, nnr_model_kernel);
143
144 MS_LOG(INFO) << "NNRTDelegate build success.";
145 return mindspore::kSuccess;
146 }
147
Init()148 mindspore::Status mindspore::NNRTDelegate::Init() {
149 MS_LOG(DEBUG) << "NNRTDelegate init success.";
150 return mindspore::kSuccess;
151 }
PrepareInputs(DelegateModel<schema::Primitive> * model,OH_NNExecutor * oh_nn_executor)152 mindspore::Status mindspore::NNRTDelegate::PrepareInputs(DelegateModel<schema::Primitive> *model,
153 OH_NNExecutor *oh_nn_executor) {
154 auto input_tensors = model->inputs();
155 for (size_t i = 0; i < input_tensors.size(); i++) {
156 auto tensor = input_tensors[i];
157 auto tensor_shape = tensor.Shape();
158 auto tmp_quant_param = tensor.QuantParams();
159 OH_NN_QuantParam *quant_param = nullptr;
160 std::vector<uint32_t> bit_num;
161 std::vector<double> scale;
162 std::vector<int32_t> zero_point;
163 if (!tmp_quant_param.empty()) {
164 quant_param = new (std::nothrow) OH_NN_QuantParam;
165 if (quant_param == nullptr) {
166 MS_LOG(ERROR) << "new OH_NN_QuantParam failed.";
167 return mindspore::kLiteError;
168 }
169 for (auto qparam : tmp_quant_param) {
170 bit_num.emplace_back(qparam.bit_num);
171 scale.emplace_back(qparam.scale);
172 zero_point.emplace_back(qparam.zero_point);
173 }
174 quant_param->quantCount = tmp_quant_param.size();
175 quant_param->numBits = bit_num.data();
176 quant_param->scale = scale.data();
177 quant_param->zeroPoint = zero_point.data();
178 }
179 auto oprend = new (std::nothrow) OH_NN_Tensor;
180 if (oprend == nullptr) {
181 MS_LOG(ERROR) << "new OH_NN_Tensor Failed";
182 return mindspore::kLiteError;
183 }
184 oprend->dataType = ConvertDataType(tensor.DataType());
185 oprend->dimensionCount = tensor_shape.size();
186
187 std::vector<int32_t> dimensions_list;
188 for (auto shape : tensor_shape) {
189 if (shape < INT32_MAX) {
190 dimensions_list.emplace_back(static_cast<int32_t>(shape));
191 } else {
192 MS_LOG(ERROR) << "NNExecutor SetInput failed,tensor dimension is is too large, max dim = " << INT32_MAX
193 << ", but get dimension = " << shape;
194 return mindspore::kLiteError;
195 }
196 }
197 oprend->dimensions = dimensions_list.data();
198 oprend->quantParam = quant_param;
199 oprend->type = OH_NN_TENSOR;
200 OH_NN_ReturnCode ret_code =
201 OH_NNExecutor_SetInput(oh_nn_executor, i, oprend, tensor.MutableData(), tensor.DataSize());
202 delete (oprend);
203
204 if (!tmp_quant_param.empty()) {
205 delete (quant_param);
206 quant_param = nullptr;
207 }
208
209 if (ret_code != OH_NN_SUCCESS) {
210 MS_LOG(ERROR) << "NNExecutor SetInput failed, current input tensor is" << tensor.Name()
211 << "OH_NN_ReturnCode = " << ret_code;
212 return mindspore::kLiteError;
213 }
214 }
215
216 return mindspore::kSuccess;
217 }
ConvertDataType(mindspore::DataType data_type)218 OH_NN_DataType mindspore::NNRTDelegate::ConvertDataType(mindspore::DataType data_type) {
219 OH_NN_DataType oh_data_type;
220 switch (data_type) {
221 case mindspore::DataType::kTypeUnknown:
222 case mindspore::DataType::kObjectTypeString:
223 case mindspore::DataType::kObjectTypeList:
224 case mindspore::DataType::kObjectTypeTuple:
225 case mindspore::DataType::kObjectTypeTensorType:
226 case mindspore::DataType::kNumberTypeBegin:
227 case mindspore::DataType::kNumberTypeEnd:
228 case mindspore::DataType::kInvalidType:
229 oh_data_type = OH_NN_UNKNOWN;
230 break;
231 case mindspore::DataType::kNumberTypeBool:
232 oh_data_type = OH_NN_BOOL;
233 break;
234 case mindspore::DataType::kNumberTypeInt8:
235 oh_data_type = OH_NN_INT8;
236 break;
237 case mindspore::DataType::kNumberTypeInt16:
238 oh_data_type = OH_NN_INT16;
239 break;
240 case mindspore::DataType::kNumberTypeInt32:
241 oh_data_type = OH_NN_INT32;
242 break;
243 case mindspore::DataType::kNumberTypeInt64:
244 oh_data_type = OH_NN_INT64;
245 break;
246 case mindspore::DataType::kNumberTypeUInt8:
247 oh_data_type = OH_NN_UINT8;
248 break;
249 case mindspore::DataType::kNumberTypeUInt16:
250 oh_data_type = OH_NN_UINT16;
251 break;
252 case mindspore::DataType::kNumberTypeUInt32:
253 oh_data_type = OH_NN_UINT32;
254 break;
255 case mindspore::DataType::kNumberTypeUInt64:
256 oh_data_type = OH_NN_UINT64;
257 break;
258 case mindspore::DataType::kNumberTypeFloat16:
259 oh_data_type = OH_NN_FLOAT16;
260 break;
261 case mindspore::DataType::kNumberTypeFloat32:
262 oh_data_type = OH_NN_FLOAT32;
263 break;
264 case mindspore::DataType::kNumberTypeFloat64:
265 oh_data_type = OH_NN_FLOAT64;
266 break;
267 default: {
268 oh_data_type = OH_NN_UNKNOWN;
269 }
270 }
271 return oh_data_type;
272 }
273
PrepareOutputs(DelegateModel<schema::Primitive> * model,OH_NNExecutor * oh_nn_executor)274 mindspore::Status mindspore::NNRTDelegate::PrepareOutputs(DelegateModel<schema::Primitive> *model,
275 OH_NNExecutor *oh_nn_executor) {
276 auto output_tensors = model->outputs();
277 for (size_t i = 0; i < output_tensors.size(); i++) {
278 auto tensor = output_tensors[i];
279 OH_NN_ReturnCode ret_code = OH_NNExecutor_SetOutput(oh_nn_executor, i, tensor.MutableData(), tensor.DataSize());
280 if (ret_code != OH_NN_SUCCESS) {
281 MS_LOG(ERROR) << "NNExecutor SetOutput failed, current out tensor is" << tensor.Name()
282 << ", OH_NN_ReturnCode = " << ret_code;
283 return mindspore::kLiteError;
284 }
285 }
286 return mindspore::kSuccess;
287 }
288
ShallowCopyLiteGraph(const mindspore::lite::LiteGraph & lite_graph)289 void mindspore::NNRTDelegate::ShallowCopyLiteGraph(const mindspore::lite::LiteGraph &lite_graph) {
290 Status ret;
291 for (auto node : lite_graph.all_nodes_) {
292 ret = lite::CheckPrimitiveSupported(static_cast<const schema::Primitive *>(node->primitive_));
293 if (ret == mindspore::kLiteError) {
294 MS_LOG(ERROR) << " primitive supported check failed.";
295 return;
296 }
297 }
298 std::vector<LiteGraph::Node *> node_list;
299 node_list.reserve(lite_graph.all_nodes_.size());
300 // copy node
301 for (auto node : lite_graph.all_nodes_) {
302 auto new_node = new (std::nothrow) LiteGraph::Node;
303 if (new_node == nullptr) {
304 MS_LOG(ERROR) << " new LiteGraph::Node failed.";
305 return;
306 }
307 new_node->name_ = node->name_;
308 new_node->op_type_ = node->op_type_;
309 new_node->node_type_ = node->node_type_;
310 new_node->primitive_ = node->primitive_;
311 new_node->base_operator_ = node->base_operator_;
312 new_node->input_indices_ = node->input_indices_;
313 new_node->output_indices_ = node->output_indices_;
314 new_node->quant_type_ = node->quant_type_;
315 new_node->device_type_ = node->device_type_;
316 node_list.emplace_back(new_node);
317 }
318 // copy subgraph
319 std::vector<LiteGraph::SubGraph *> subgraph_list;
320 for (auto subgraph : lite_graph.sub_graphs_) {
321 auto new_subgraph = new (std::nothrow) LiteGraph::SubGraph;
322 if (new_subgraph == nullptr) {
323 MS_LOG(ERROR) << "new LiteGraph::Subgraph failed.";
324 return;
325 }
326 new_subgraph->name_ = subgraph->name_;
327 new_subgraph->input_indices_ = subgraph->input_indices_;
328 new_subgraph->output_indices_ = subgraph->output_indices_;
329 new_subgraph->node_indices_ = subgraph->node_indices_;
330 subgraph_list.emplace_back(new_subgraph);
331 }
332 for (auto tensor : lite_graph.all_tensors_) {
333 ret = lite::CheckTensorSupported(static_cast<const schema::Tensor *>(tensor));
334 if (ret == mindspore::kLiteError) {
335 MS_LOG(ERROR) << "tensor supported check failed.";
336 return;
337 }
338 }
339
340 nnrt_lite_graph = new (std::nothrow) lite::LiteGraph();
341 if (nnrt_lite_graph == nullptr) {
342 MS_LOG(ERROR) << "new LiteGraph failed.";
343 return;
344 }
345
346 nnrt_lite_graph->name_ = lite_graph.name_;
347 nnrt_lite_graph->version_ = lite_graph.version_;
348 nnrt_lite_graph->input_indices_ = lite_graph.input_indices_;
349 nnrt_lite_graph->output_indices_ = lite_graph.output_indices_;
350 nnrt_lite_graph->all_tensors_ = lite_graph.all_tensors_;
351 nnrt_lite_graph->all_nodes_ = node_list;
352 nnrt_lite_graph->sub_graphs_ = subgraph_list;
353 MS_LOG(INFO) << "ShallowCopyLiteGraph success.";
354 }
355
~NNRTDelegate()356 mindspore::NNRTDelegate::~NNRTDelegate() {
357 if (this->nnrt_lite_graph != nullptr) {
358 MS_LOG(ERROR) << "Delete NNRTDelegate.";
359 }
360 };
361