1 /*
2 * Copyright (c) 2023 Huawei Device Co., Ltd.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at
6 *
7 * http://www.apache.org/licenses/LICENSE-2.0
8 *
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
14 */
15
16
17 #include "nnexecutor.h"
18 #include "nntensor.h"
19 #include "nncompiled_cache.h"
20 #include "cpp_type.h"
21 #include "neural_network_runtime_inner.h"
22 #include "nnrt_client.h"
23 #include "log.h"
24
25 #include "securec.h"
26 #include "utils.h"
27 #include "scoped_trace.h"
28 #include "transform.h"
29
30 namespace OHOS {
31 constexpr size_t EXTENSION_MAX_SIZE = 200;
32 constexpr int AUTOUNLOAD_TIME = 15 * 60 * 1000;
33
34 namespace NeuralNetworkRuntime {
35 constexpr int CACHE_INPUT_TENSORDESC_OFFSET = 2;
36 constexpr int CACHE_OUTPUT_TENSORDESC_OFFSET = 1;
37 constexpr size_t CHECK_SUM_ZERO = 0;
38 constexpr size_t CHECK_SUM_ONE = 1;
39 constexpr size_t CHECK_SUM_TWO = 2;
40 constexpr int32_t NUMBER_CACHE_INFO_MEMBERS = 3;
41
42 struct SerializedTensorDesc {
43 public:
44 SerializedTensorDesc() = default;
45 ~SerializedTensorDesc() = default;
46
CopyFromTensorDescOHOS::NeuralNetworkRuntime::SerializedTensorDesc47 OH_NN_ReturnCode CopyFromTensorDesc(const std::pair<std::shared_ptr<TensorDesc>, OH_NN_TensorType>& tensorDesc)
48 {
49 if (tensorDesc.first == nullptr) {
50 LOGE("CopyFromTensorDesc failed, tensor desc is nullptr.");
51 return OH_NN_NULL_PTR;
52 }
53 OH_NN_ReturnCode ret = tensorDesc.first->GetDataType(&m_dataType);
54 if (ret != OH_NN_SUCCESS) {
55 LOGE("CopyFromTensorDesc failed, error happened when getting data type from tensor desc.");
56 return ret;
57 }
58
59 ret = tensorDesc.first->GetFormat(&m_format);
60 if (ret != OH_NN_SUCCESS) {
61 LOGE("CopyFromTensorDesc failed, error happened when getting format from tensor desc.");
62 return ret;
63 }
64
65 ret = tensorDesc.first->GetShape(&m_shape, &m_shapeNum);
66 if (ret != OH_NN_SUCCESS) {
67 LOGE("CopyFromTensorDesc failed, error happened when getting shape from tensor desc.");
68 return ret;
69 }
70
71 ret = tensorDesc.first->GetName(&m_name);
72 if (ret != OH_NN_SUCCESS) {
73 LOGE("CopyFromTensorDesc failed, error happened when getting name from tensor desc.");
74 return ret;
75 }
76
77 m_tensorType = tensorDesc.second;
78
79 return ret;
80 }
81
CopyToTensorDescOHOS::NeuralNetworkRuntime::SerializedTensorDesc82 OH_NN_ReturnCode CopyToTensorDesc(TensorDesc& tensorDesc) const
83 {
84 OH_NN_ReturnCode ret = tensorDesc.SetDataType(m_dataType);
85 if (ret != OH_NN_SUCCESS) {
86 LOGE("CopyToTensorDesc failed, error happened when setting data type to tensor desc.");
87 return ret;
88 }
89
90 ret = tensorDesc.SetFormat(m_format);
91 if (ret != OH_NN_SUCCESS) {
92 LOGE("CopyToTensorDesc failed, error happened when setting format to tensor desc.");
93 return ret;
94 }
95
96 ret = tensorDesc.SetShape(m_shape, m_shapeNum);
97 if (ret != OH_NN_SUCCESS) {
98 LOGE("CopyToTensorDesc failed, error happened when setting shape to tensor desc.");
99 return ret;
100 }
101
102 ret = tensorDesc.SetName(m_name);
103 if (ret != OH_NN_SUCCESS) {
104 LOGE("CopyToTensorDesc failed, error happened when setting name to tensor desc.");
105 }
106
107 return ret;
108 }
109
110 public:
111 OH_NN_DataType m_dataType{OH_NN_UNKNOWN};
112 OH_NN_Format m_format{OH_NN_FORMAT_NONE};
113 OH_NN_TensorType m_tensorType{OH_NN_TENSOR};
114 size_t m_shapeNum{0};
115 int32_t* m_shape{nullptr};
116 const char* m_name{nullptr}; // null-terminated
117 };
118 const size_t SIZE_OF_DATATYPE = sizeof(SerializedTensorDesc::m_dataType);
119 const size_t SIZE_OF_FORMAT = sizeof(SerializedTensorDesc::m_format);
120 const size_t SIZE_OF_TENSOR_TYPE = sizeof(SerializedTensorDesc::m_tensorType);
121 const size_t SIZE_OF_SHAPE_NUM = sizeof(SerializedTensorDesc::m_shapeNum);
122
GenRandom(void)123 uint64_t GenRandom(void)
124 {
125 uint64_t random = 0;
126 int fd = open("/dev/random", O_RDONLY);
127 if (fd >= 0) {
128 read(fd, &random, sizeof(random));
129 close(fd);
130 }
131 return random;
132 }
133
NNExecutor(size_t backendID,std::shared_ptr<Device> device,std::shared_ptr<PreparedModel> preparedModel,const std::vector<std::pair<std::shared_ptr<TensorDesc>,OH_NN_TensorType>> & inputTensorDescs,const std::vector<std::pair<std::shared_ptr<TensorDesc>,OH_NN_TensorType>> & outputTensorDescs,std::string cachePath,uint32_t cacheVersion,ExtensionConfig extensionConfig,bool enableFp16,OH_NN_PerformanceMode performance,OH_NN_Priority priority)134 NNExecutor::NNExecutor(size_t backendID, std::shared_ptr<Device> device, std::shared_ptr<PreparedModel> preparedModel,
135 const std::vector<std::pair<std::shared_ptr<TensorDesc>, OH_NN_TensorType>>& inputTensorDescs,
136 const std::vector<std::pair<std::shared_ptr<TensorDesc>, OH_NN_TensorType>>& outputTensorDescs,
137 std::string cachePath, uint32_t cacheVersion, ExtensionConfig extensionConfig, bool enableFp16,
138 OH_NN_PerformanceMode performance, OH_NN_Priority priority)
139 : m_backendID(backendID),
140 m_device(device),
141 m_preparedModel(preparedModel),
142 m_inputTensorDescs(inputTensorDescs),
143 m_outputTensorDescs(outputTensorDescs),
144 m_cachePath(cachePath),
145 m_cacheVersion(cacheVersion),
146 m_extensionConfig(extensionConfig),
147 m_enableFp16(enableFp16),
148 m_performance(performance),
149 m_priority(priority) {
150 m_executorid = GenRandom();
151 m_autoUnloadRunner = OHOS::AppExecFwk::EventRunner::Create
152 ("nnexecutor_autounload" + std::to_string(m_executorid));
153 m_autoUnloadHandler = std::make_shared<OHOS::AppExecFwk::EventHandler>(m_autoUnloadRunner);
154 auto AutoUnloadTask = [this]() {
155 DeinitModel("DelayUnload");
156 };
157 m_autoUnloadHandler->PostTask(AutoUnloadTask,
158 "nnexecutor_autounload" + std::to_string(m_executorid), AUTOUNLOAD_TIME);
159
160 GetModelID(m_originHiaiModelId);
161 LOGI("manualload pid=%{public}d originHiaiModelId=%{public}d",
162 getpid(), m_originHiaiModelId);
163 }
164
GetInputDimVec() const165 OH_NN_ReturnCode NNExecutor::GetInputDimVec() const
166 {
167 std::vector<std::vector<uint32_t>> minInputDimsVec;
168 std::vector<std::vector<uint32_t>> maxInputDimsVec;
169 OH_NN_ReturnCode oldRet = m_preparedModel->GetInputDimRanges(minInputDimsVec, maxInputDimsVec);
170 if (oldRet != OH_NN_SUCCESS) {
171 LOGW("GetInputDimVec failed, current version don't support get input dim ranges.");
172 return OH_NN_OPERATION_FORBIDDEN;
173 }
174 size_t inputSize = minInputDimsVec.size();
175 if (inputSize != maxInputDimsVec.size()) {
176 LOGE("GetInputDimVece failed, size of minInputDimsVec is not equal to maxInputDimsVec.");
177 return OH_NN_INVALID_PARAMETER;
178 }
179 for (size_t i = 0; i < inputSize; i++) {
180 std::vector<size_t> minInputDimVec;
181 std::vector<size_t> maxInputDimVec;
182 size_t minInputDimVecSize = minInputDimsVec[i].size();
183 if (minInputDimVecSize != maxInputDimsVec[i].size()) {
184 LOGE("GetInputDimVec failed, size of the min input dims is not equal to the max input"
185 " dims of the %{public}zuth input.", i);
186 return OH_NN_INVALID_PARAMETER;
187 }
188 for (size_t j = 0; j < minInputDimVecSize; j++) {
189 minInputDimVec.emplace_back(static_cast<size_t>(minInputDimsVec[i][j]));
190 maxInputDimVec.emplace_back(static_cast<size_t>(maxInputDimsVec[i][j]));
191 }
192 m_minInputDimsVec.emplace_back(minInputDimVec);
193 m_maxInputDimsVec.emplace_back(maxInputDimVec);
194 }
195 return OH_NN_SUCCESS;
196 }
197
GetInputDimRange(size_t inputIndex,size_t ** minInputDims,size_t ** maxInputDims,size_t * shapeNum) const198 OH_NN_ReturnCode NNExecutor::GetInputDimRange(
199 size_t inputIndex, size_t** minInputDims, size_t** maxInputDims, size_t* shapeNum) const
200 {
201 if (minInputDims == nullptr) {
202 LOGE("NNExecutor::GetInputDimRange failed, minInputDims is nullptr.");
203 return OH_NN_INVALID_PARAMETER;
204 }
205 if (maxInputDims == nullptr) {
206 LOGE("NNExecutor::GetInputDimRange failed, maxInputDims is nullptr.");
207 return OH_NN_INVALID_PARAMETER;
208 }
209 if (shapeNum == nullptr) {
210 LOGE("NNExecutor::GetInputDimRange failed, shapeNum is nullptr.");
211 return OH_NN_INVALID_PARAMETER;
212 }
213
214 if (m_minInputDimsVec.empty()) {
215 OH_NN_ReturnCode ret = GetInputDimVec();
216 if (ret != OH_NN_SUCCESS) {
217 LOGE("NNExecutor::GetInputDimRange failed, GetInputDimVec failed.");
218 return ret;
219 }
220 }
221
222 if (inputIndex >= m_minInputDimsVec.size()) {
223 LOGE("NNExecutor::GetInputDimRange failed, inputIndex[%{public}zu] is out of range.", inputIndex);
224 return OH_NN_INVALID_PARAMETER;
225 }
226
227 *shapeNum = m_minInputDimsVec[inputIndex].size();
228 if (*shapeNum != m_maxInputDimsVec[inputIndex].size()) {
229 LOGE("NNExecutor::GetInputDimRange failed, size of the min input dims is not equal to the max input"
230 " dims of the %{public}zuth input.", inputIndex);
231 return OH_NN_INVALID_PARAMETER;
232 }
233 *minInputDims = m_minInputDimsVec[inputIndex].data();
234 *maxInputDims = m_maxInputDimsVec[inputIndex].data();
235 return OH_NN_SUCCESS;
236 }
237
GetOutputShape(uint32_t outputIndex,int32_t ** shape,uint32_t * shapeNum) const238 OH_NN_ReturnCode NNExecutor::GetOutputShape(uint32_t outputIndex, int32_t** shape, uint32_t* shapeNum) const
239 {
240 if (outputIndex >= m_outputTensorDescs.size()) {
241 LOGE("NNExecutor::GetOutputShape failed, outputIndex must be smaller than m_outputTensorDescs.size.");
242 return OH_NN_INVALID_PARAMETER;
243 }
244 if (m_outputTensorDescs[outputIndex].first == nullptr) {
245 LOGE("NNExecutor::GetOutputShape failed, tensor desc of output %{public}u is nullptr.", outputIndex);
246 return OH_NN_INVALID_PARAMETER;
247 }
248
249 auto tensorDesc = m_outputTensorDescs[outputIndex].first;
250 size_t shapeNumTmp = 0;
251 auto ret = tensorDesc->GetShape(shape, &shapeNumTmp);
252 if (ret != OH_NN_SUCCESS) {
253 LOGE("NNExecutor::GetOutputShape failed, failed to get shape from tensor desc.");
254 return ret;
255 }
256 *shapeNum = static_cast<uint32_t>(shapeNumTmp);
257
258 return OH_NN_SUCCESS;
259 }
260
GetInputNum() const261 size_t NNExecutor::GetInputNum() const
262 {
263 return m_inputTensorDescs.size();
264 }
265
GetOutputNum() const266 size_t NNExecutor::GetOutputNum() const
267 {
268 return m_outputTensorDescs.size();
269 }
270
CreateInputTensorDesc(size_t index) const271 NN_TensorDesc* NNExecutor::CreateInputTensorDesc(size_t index) const
272 {
273 if (index >= m_inputTensorDescs.size()) {
274 LOGE("NNExecutor::CreateInputTensorDesc failed, index must be smaller than m_inputTensorDescs.size.");
275 return nullptr;
276 }
277 if (m_inputTensorDescs[index].first == nullptr) {
278 LOGE("NNExecutor::CreateInputTensorDesc failed, tensor desc of input %{public}zu is nullptr.", index);
279 return nullptr;
280 }
281
282 TensorDesc* tensorDescImpl = new (std::nothrow) TensorDesc();
283 if (tensorDescImpl == nullptr) {
284 LOGE("NNExecutor::CreateInputTensorDesc failed, failed to create tensor desc.");
285 return nullptr;
286 }
287
288 // Copy the member attributes to new tensor description
289 *tensorDescImpl = *(m_inputTensorDescs[index].first.get());
290
291 return reinterpret_cast<NN_TensorDesc*>(tensorDescImpl);
292 }
293
CreateOutputTensorDesc(size_t index) const294 NN_TensorDesc* NNExecutor::CreateOutputTensorDesc(size_t index) const
295 {
296 if (index >= m_outputTensorDescs.size()) {
297 LOGE("NNExecutor::CreateOutputTensorDesc failed, index must be smaller than m_outputTensorDescs.size.");
298 return nullptr;
299 }
300 if (m_outputTensorDescs[index].first == nullptr) {
301 LOGE("NNExecutor::CreateOutputTensorDesc failed, tensor desc of output %{public}zu is nullptr.", index);
302 return nullptr;
303 }
304
305 TensorDesc* tensorDescImpl = new (std::nothrow) TensorDesc();
306 if (tensorDescImpl == nullptr) {
307 LOGE("NNExecutor::CreateOutputTensorDesc failed, failed to create tensor desc.");
308 return nullptr;
309 }
310
311 // Copy the member attributes to new tensor description
312 *tensorDescImpl = *(m_outputTensorDescs[index].first.get());
313
314 return reinterpret_cast<NN_TensorDesc*>(tensorDescImpl);
315 }
316
SetExtensionConfig(const std::unordered_map<std::string,std::vector<char>> & configs)317 OH_NN_ReturnCode NNExecutor::SetExtensionConfig(const std::unordered_map<std::string, std::vector<char>>& configs)
318 {
319 if (m_executorConfig == nullptr) {
320 m_executorConfig = new (std::nothrow) ExecutorConfig();
321 if (m_executorConfig == nullptr) {
322 LOGE("[NNExecutor] SetExtensionConfig, m_executorConfig create failed.");
323 return OH_NN_FAILED;
324 }
325 }
326
327 if (configs.size() > EXTENSION_MAX_SIZE) {
328 LOGE("[NNExecutor] SetExtensionConfig, configs size more than 200.");
329 return OH_NN_FAILED;
330 }
331
332 for (auto config : configs) {
333 char* configData = reinterpret_cast<char*>(config.second.data());
334 if (configData == nullptr) {
335 LOGE("[NNExecutor] SetExtensionConfig, key: %s, configData is nullptr.", config.first.c_str());
336 return OH_NN_FAILED;
337 }
338
339 if (!config.first.compare("callingPid")) {
340 m_executorConfig->callingPid = std::atoi(configData);
341 LOGD("[NNExecutor] SetExtensionConfig, callingPid: %{public}d.", m_executorConfig->callingPid);
342 }
343
344 if (!config.first.compare("hiaiModelId")) {
345 m_executorConfig->hiaiModelId = std::atoi(configData);
346 LOGD("[NNExecutor] SetExtensionConfig, hiaiModelId: %{public}d.", m_executorConfig->hiaiModelId);
347 }
348
349 if (!config.first.compare("isNeedModelLatency")) {
350 m_executorConfig->isNeedModelLatency = static_cast<bool>(*configData);
351 LOGD("[NNExecutor] SetExtensionConfig, isNeedModelLatency: %{public}d.",
352 m_executorConfig->isNeedModelLatency);
353 }
354 }
355
356 return OH_NN_SUCCESS;
357 }
358
GetExecutorConfig() const359 ExecutorConfig* NNExecutor::GetExecutorConfig() const
360 {
361 return m_executorConfig;
362 }
363
SetOnRunDone(NN_OnRunDone onRunDone)364 OH_NN_ReturnCode NNExecutor::SetOnRunDone(NN_OnRunDone onRunDone)
365 {
366 LOGE("NNExecutor::SetOnRunDone failed, SetOnRunDone is not supported.");
367 return OH_NN_OPERATION_FORBIDDEN;
368 }
369
SetOnServiceDied(NN_OnServiceDied onServiceDied)370 OH_NN_ReturnCode NNExecutor::SetOnServiceDied(NN_OnServiceDied onServiceDied)
371 {
372 LOGE("NNExecutor::SetOnServiceDied failed, SetOnServiceDied is not supported.");
373 return OH_NN_OPERATION_FORBIDDEN;
374 }
375
ReleaseDescShape(std::vector<SerializedTensorDesc> & immediateTensorDescs)376 void ReleaseDescShape(std::vector<SerializedTensorDesc>& immediateTensorDescs)
377 {
378 for (auto desc : immediateTensorDescs) {
379 delete[] desc.m_shape;
380 }
381 immediateTensorDescs.clear();
382 }
383
DeserializedTensorsFromBuffer(const Buffer & buffer,std::vector<std::pair<std::shared_ptr<TensorDesc>,OH_NN_TensorType>> & tensorDescs)384 OH_NN_ReturnCode NNExecutor::DeserializedTensorsFromBuffer(
385 const Buffer& buffer, std::vector<std::pair<std::shared_ptr<TensorDesc>, OH_NN_TensorType>>& tensorDescs)
386 {
387 std::vector<SerializedTensorDesc> immediateTensorDescs;
388 const char* ptr = static_cast<const char*>(buffer.data);
389 const char* end = ptr + buffer.length;
390 while (ptr < end) {
391 SerializedTensorDesc desc;
392
393 auto memRet = memcpy_s(&desc.m_dataType, SIZE_OF_DATATYPE, ptr, sizeof(desc.m_dataType));
394 if (memRet != EOK) {
395 LOGE("[NNExecutor] DeserializedTensorsFromBuffer failed, failed to memcpy_s data type.");
396 ReleaseDescShape(immediateTensorDescs);
397 return OH_NN_MEMORY_ERROR;
398 }
399 ptr += sizeof(desc.m_dataType);
400
401 memRet = memcpy_s(&desc.m_format, SIZE_OF_FORMAT, ptr, sizeof(desc.m_format));
402 if (memRet != EOK) {
403 LOGE("[NNExecutor] DeserializedTensorsFromBuffer failed, failed to memcpy_s format.");
404 ReleaseDescShape(immediateTensorDescs);
405 return OH_NN_MEMORY_ERROR;
406 }
407 ptr += sizeof(desc.m_format);
408
409 memRet = memcpy_s(&desc.m_tensorType, SIZE_OF_TENSOR_TYPE, ptr, sizeof(desc.m_tensorType));
410 if (memRet != EOK) {
411 LOGE("[NNExecutor] DeserializedTensorsFromBuffer failed, failed to memcpy_s tensor type.");
412 ReleaseDescShape(immediateTensorDescs);
413 return OH_NN_MEMORY_ERROR;
414 }
415 ptr += sizeof(desc.m_tensorType);
416
417 memRet = memcpy_s(&desc.m_shapeNum, SIZE_OF_SHAPE_NUM, ptr, sizeof(desc.m_shapeNum));
418 if (memRet != EOK) {
419 LOGE("[NNExecutor] DeserializedTensorsFromBuffer failed, failed to memcpy_s shape num.");
420 ReleaseDescShape(immediateTensorDescs);
421 return OH_NN_MEMORY_ERROR;
422 }
423 ptr += sizeof(desc.m_shapeNum);
424
425 desc.m_shape = new (std::nothrow) int32_t[desc.m_shapeNum];
426 if (desc.m_shape == nullptr) {
427 LOGE("[NNExecutor] DeserializedTensorsFromBuffer failed, failed to create shape buffer.");
428 ReleaseDescShape(immediateTensorDescs);
429 return OH_NN_NULL_PTR;
430 }
431 memRet = memcpy_s(desc.m_shape, desc.m_shapeNum * sizeof(int32_t), ptr, desc.m_shapeNum * sizeof(int32_t));
432 if (memRet != EOK) {
433 LOGE("[NNExecutor] DeserializedTensorsFromBuffer failed, failed to memcpy_s shape.");
434 ReleaseDescShape(immediateTensorDescs);
435 return OH_NN_MEMORY_ERROR;
436 }
437 ptr += desc.m_shapeNum * sizeof(int32_t);
438
439 desc.m_name = ptr;
440 ptr += std::strlen(desc.m_name) + 1; // +1 for null terminator
441
442 immediateTensorDescs.push_back(desc);
443 }
444
445 OH_NN_ReturnCode ret {OH_NN_SUCCESS};
446 for (const auto& immediateTensorDesc : immediateTensorDescs) {
447 std::pair<std::shared_ptr<TensorDesc>, OH_NN_TensorType> tensorDescPair;
448 tensorDescPair.first = CreateSharedPtr<TensorDesc>();
449 if (tensorDescPair.first == nullptr) {
450 LOGE("[NNExecutor] DeserializedTensorsFromBuffer failed, failed to create tensor desc.");
451 tensorDescs.clear();
452 ReleaseDescShape(immediateTensorDescs);
453 return OH_NN_NULL_PTR;
454 }
455 ret = immediateTensorDesc.CopyToTensorDesc(*(tensorDescPair.first.get()));
456 if (ret != OH_NN_SUCCESS) {
457 LOGE("[NNExecutor] DeserializedTensorsFromBuffer failed, error happened when copying "
458 "SerializedTensorDesc to TensorDesc.");
459 tensorDescs.clear();
460 ReleaseDescShape(immediateTensorDescs);
461 return ret;
462 }
463 tensorDescPair.second = immediateTensorDesc.m_tensorType;
464
465 tensorDescs.emplace_back(tensorDescPair);
466 }
467
468 ReleaseDescShape(immediateTensorDescs);
469 return ret;
470 }
471
Reload()472 OH_NN_ReturnCode NNExecutor::Reload()
473 {
474 if (m_cachePath.empty()) {
475 LOGE("[NNExecutor] RestoreFromCacheFile failed, path is empty.");
476 return OH_NN_INVALID_PARAMETER;
477 }
478
479 if (m_cacheVersion == INVALID_CAHCE_VERSION) {
480 LOGE("[NNExecutor] RestoreFromCacheFile failed, cache version is invalid. Please set a valid cache version.");
481 return OH_NN_INVALID_PARAMETER;
482 }
483
484 if (m_preparedModel != nullptr) {
485 LOGE("[NNExecutor] RestoreFromCacheFile failed, m_preparedModel is not nullptr.");
486 return OH_NN_FAILED;
487 }
488
489 NNCompiledCache compiledCache;
490 OH_NN_ReturnCode ret = compiledCache.SetBackend(m_backendID);
491 if (ret != OH_NN_SUCCESS) {
492 LOGE("[NNExecutor] RestoreFromCacheFile failed, fail to set backend.");
493 return ret;
494 }
495
496 std::vector<Buffer> caches;
497 compiledCache.SetModelName(m_extensionConfig.modelName);
498 ret = compiledCache.Restore(m_cachePath, m_cacheVersion, caches);
499 if (ret != OH_NN_SUCCESS) {
500 LOGE("[NNExecutor] RestoreFromCacheFile failed, error happened when restoring model cache.");
501 compiledCache.ReleaseCacheBuffer(caches);
502 return ret;
503 }
504
505 size_t cacheNum = caches.size();
506 std::vector<std::pair<std::shared_ptr<TensorDesc>, OH_NN_TensorType>> inputTensorDescs;
507 ret = DeserializedTensorsFromBuffer(caches[cacheNum - CACHE_INPUT_TENSORDESC_OFFSET], inputTensorDescs);
508 if (ret != OH_NN_SUCCESS) {
509 LOGE("[NNExecutor] RestoreFromCacheFile failed, error happened when deserializing input tensor desc.");
510 compiledCache.ReleaseCacheBuffer(caches);
511 return ret;
512 }
513
514 std::vector<std::pair<std::shared_ptr<TensorDesc>, OH_NN_TensorType>> outputTensorDescs;
515 ret = DeserializedTensorsFromBuffer(caches[cacheNum - CACHE_OUTPUT_TENSORDESC_OFFSET], outputTensorDescs);
516 if (ret != OH_NN_SUCCESS) {
517 LOGE("[NNExecutor] RestoreFromCacheFile failed, error happened when deserializing output tensor desc.");
518 compiledCache.ReleaseCacheBuffer(caches);
519 return ret;
520 }
521
522 ModelConfig config;
523 config.enableFloat16 = m_enableFp16;
524 config.mode = m_performance;
525 config.priority = m_priority;
526 config.extensionConfig.isNpuFmShared = m_extensionConfig.isNpuFmShared;
527 std::vector<Buffer> modelOnlyCaches(caches.begin(), caches.end() - CACHE_INPUT_TENSORDESC_OFFSET);
528 bool isUpdatable = false;
529 ret = m_device->PrepareModelFromModelCache(modelOnlyCaches, config, m_preparedModel, isUpdatable);
530 if (ret != OH_NN_SUCCESS) {
531 LOGE("[NNExecutor] RestoreFromCacheFile failed, error happened when preparing model from cache.");
532 compiledCache.ReleaseCacheBuffer(caches);
533 return ret;
534 }
535
536 compiledCache.ReleaseCacheBuffer(caches);
537
538 m_inputTensorDescs = inputTensorDescs;
539 m_outputTensorDescs = outputTensorDescs;
540 LOGI("[NNExecutor] Restore model cache successfully.");
541 return OH_NN_SUCCESS;
542 }
543
RunSync(NN_Tensor * inputTensors[],size_t inputSize,NN_Tensor * outputTensors[],size_t outputSize)544 OH_NN_ReturnCode NNExecutor::RunSync(NN_Tensor* inputTensors[], size_t inputSize,
545 NN_Tensor* outputTensors[], size_t outputSize)
546 {
547 std::lock_guard<std::mutex> lock(m_mutex);
548 {
549 uint32_t modelId;
550 GetModelID(modelId);
551 m_autoUnloadHandler->RemoveTask("nnexecutor_autounload" + std::to_string(m_executorid));
552 if (m_inputTensorDescs.size() != inputSize) {
553 LOGE("NNExecutor::RunSync failed, inputSize:%{public}zu is not equal to model input size:%{public}zu",
554 inputSize, m_inputTensorDescs.size());
555 return OH_NN_INVALID_PARAMETER;
556 }
557 if (m_outputTensorDescs.size() != outputSize) {
558 LOGE("NNExecutor::RunSync failed, outputSize:%{public}zu is not equal to model output size:%{public}zu",
559 outputSize, m_outputTensorDescs.size());
560 return OH_NN_INVALID_PARAMETER;
561 }
562
563 if (m_preparedModel == nullptr) {
564 if (Reload() != OH_NN_SUCCESS) {
565 return OH_NN_INVALID_PARAMETER;
566 }
567 auto _ret = GetModelID(modelId);
568 LOGI("AutoReload pid=%{public}d originHiaiModelId=%{public}d hiaiModelId=%{public}d",
569 getpid(), m_originHiaiModelId, modelId);
570 if (_ret != OH_NN_SUCCESS) {
571 LOGW("GetModelID failed, some error happen when get model id for device.");
572 }
573 _ret = ReinitScheduling(modelId, &m_executorConfig->isNeedModelLatency, m_cachePath.c_str());
574 if (_ret != OH_NN_SUCCESS) {
575 LOGW("ReinitScheduling failed, some error happen when ReinitScheduling model.");
576 }
577 _ret = SetDeinitModelCallBack();
578 if (_ret != OH_NN_SUCCESS) {
579 LOGW("SetDeinitModelCallBack failed, some error happen when ReinitScheduling model.");
580 }
581 }
582
583 OH_NN_ReturnCode ret {OH_NN_FAILED};
584 ret = CheckInputDimRanges(inputTensors, inputSize);
585 if (ret != OH_NN_OPERATION_FORBIDDEN && ret != OH_NN_SUCCESS) {
586 LOGE("NNExecutor::RunSync failed, failed to check input dim ranges.");
587 return ret;
588 }
589
590 OHOS::NeuralNetworkRuntime::IOTensor tensor;
591 std::vector<NN_Tensor*> inputTensorsVec;
592 for (size_t i = 0; i < inputSize; ++i) {
593 if (inputTensors[i] == nullptr) {
594 LOGE("NNExecutor::RunSync failed, input[%{public}zu] is nullptr.", i);
595 return OH_NN_INVALID_PARAMETER;
596 }
597 inputTensorsVec.emplace_back(inputTensors[i]);
598 }
599
600 std::vector<NN_Tensor*> outputTensorsVec;
601 for (size_t i = 0; i < outputSize; ++i) {
602 if (outputTensors[i] == nullptr) {
603 LOGE("NNExecutor::RunSync failed, output[%{public}zu] is nullptr.", i);
604 return OH_NN_INVALID_PARAMETER;
605 }
606 outputTensorsVec.emplace_back(outputTensors[i]);
607 }
608
609 std::vector<std::vector<int32_t>> outputsDims;
610 std::vector<bool> isSufficientDataBuffer;
611
612 ret = m_preparedModel->Run(inputTensorsVec, outputTensorsVec, outputsDims, isSufficientDataBuffer);
613 if (ret != OH_NN_SUCCESS) {
614 LOGE("NNExecutor::RunSync failed, failed to run in prepared model.");
615 return ret;
616 }
617
618 // Set the output NNTensor2_0's dimensions from output IOTensor if it is dynamic.
619 // NNTensor2_0::SetDimensions will check if the tensor buffer is enough for the new dimensions.
620 if (outputsDims.size() != outputSize) {
621 LOGE("NNExecutor::RunSync failed, size of outputsDims is not equal to outputTensors.");
622 return OH_NN_INVALID_PARAMETER;
623 }
624 for (size_t i = 0; i < outputSize; ++i) {
625 NNTensor2_0* nnTensor = reinterpret_cast<NNTensor2_0*>(outputTensors[i]);
626 TensorDesc* nnTensorDesc = nnTensor->GetTensorDesc();
627 if (nnTensorDesc == nullptr) {
628 LOGE("NNExecutor::RunSync failed, failed to get desc from tensor.");
629 return OH_NN_NULL_PTR;
630 }
631 ret = nnTensorDesc->SetShape(outputsDims[i].data(), outputsDims[i].size());
632 if (ret != OH_NN_SUCCESS) {
633 LOGE("NNExecutor::RunSync failed, error happened when setting output tensor's dimensions,"
634 " output id: %zu.", i);
635 return ret;
636 }
637 ret = m_outputTensorDescs[i].first->SetShape(outputsDims[i].data(), outputsDims[i].size());
638 if (ret != OH_NN_SUCCESS) {
639 LOGE("NNExecutor::RunSync failed, error happened when setting inner output tensor's dimensions,"
640 " output id: %zu.", i);
641 return ret;
642 }
643 }
644 }
645 auto AutoUnloadTask = [this]() {
646 DeinitModel("DelayUnload");
647 };
648 m_autoUnloadHandler->PostTask(AutoUnloadTask,
649 "nnexecutor_autounload" + std::to_string(m_executorid), AUTOUNLOAD_TIME);
650
651 return OH_NN_SUCCESS;
652 }
653
RunAsync(NN_Tensor * inputTensors[],size_t inputSize,NN_Tensor * outputTensors[],size_t outputSize,int32_t timeout,void * userData)654 OH_NN_ReturnCode NNExecutor::RunAsync(NN_Tensor* inputTensors[], size_t inputSize,
655 NN_Tensor* outputTensors[], size_t outputSize, int32_t timeout, void* userData)
656 {
657 LOGE("NNExecutor::RunAsync failed, RunAsync is not supported.");
658 return OH_NN_OPERATION_FORBIDDEN;
659 }
660
GetModelID(uint32_t & modelId) const661 OH_NN_ReturnCode NNExecutor::GetModelID(uint32_t& modelId) const
662 {
663 if (m_preparedModel != nullptr) {
664 OH_NN_ReturnCode ret = m_preparedModel->GetModelID(modelId);
665 if (ret != OH_NN_SUCCESS) {
666 LOGE("GetModelID failed, some error happen when get model id for device.");
667 return ret;
668 }
669 return OH_NN_SUCCESS;
670 }
671
672 return OH_NN_OPERATION_FORBIDDEN;
673 }
674
GetBackendID()675 size_t NNExecutor::GetBackendID()
676 {
677 return m_backendID;
678 }
679
CheckInputDimRanges(NN_Tensor * inputTensors[],size_t inputSize)680 OH_NN_ReturnCode NNExecutor::CheckInputDimRanges(NN_Tensor* inputTensors[], size_t inputSize)
681 {
682 std::vector<std::vector<uint32_t>> minInputDims;
683 std::vector<std::vector<uint32_t>> maxInputDims;
684 OH_NN_ReturnCode oldRet = m_preparedModel->GetInputDimRanges(minInputDims, maxInputDims);
685 if (oldRet != OH_NN_SUCCESS) {
686 return OH_NN_OPERATION_FORBIDDEN;
687 }
688
689 if (inputSize != minInputDims.size()) {
690 LOGE("NNExecutor::CheckInputDimRanges failed, size of minInputDims:%{public}zu is not equal to "
691 "inputSize:%{public}zu.", minInputDims.size(), inputSize);
692 return OH_NN_INVALID_PARAMETER;
693 }
694
695 if (inputSize != maxInputDims.size()) {
696 LOGE("NNExecutor::CheckInputDimRanges failed, size of maxInputDims:%{public}zu is not equal to "
697 "inputSize:%{public}zu.", maxInputDims.size(), inputSize);
698 return OH_NN_INVALID_PARAMETER;
699 }
700
701 const NNTensor2_0* nnTensor = nullptr;
702 OH_NN_ReturnCode ret {OH_NN_FAILED};
703 for (size_t i = 0; i < inputSize; ++i) {
704 const std::vector<uint32_t>& minSingleInputDims = minInputDims[i];
705 const std::vector<uint32_t>& maxSingleInputDims = maxInputDims[i];
706 nnTensor = reinterpret_cast<const NNTensor2_0*>(inputTensors[i]);
707 if (nnTensor == nullptr) {
708 LOGE("NNExecutor::CheckInputDimRanges failed, input %{public}zu is nullptr.", i);
709 return OH_NN_NULL_PTR;
710 }
711 ret = nnTensor->CheckDimRanges(minSingleInputDims, maxSingleInputDims);
712 if (ret != OH_NN_SUCCESS) {
713 LOGE("NNExecutor::CheckInputDimRanges failed, failed to check input dim ranges of input %{public}zu", i);
714 return ret;
715 }
716 }
717
718 return OH_NN_SUCCESS;
719 }
720
CompareAttribute(const std::pair<std::shared_ptr<TensorDesc>,OH_NN_TensorType> & tensorDesc,const NNTensor & tensor) const721 bool NNExecutor::CompareAttribute(
722 const std::pair<std::shared_ptr<TensorDesc>, OH_NN_TensorType>& tensorDesc, const NNTensor& tensor) const
723 {
724 OH_NN_DataType dataType;
725 auto ret = tensorDesc.first->GetDataType(&dataType);
726 if (ret != OH_NN_SUCCESS) {
727 LOGE("CompareAttribute failed, failed to get data type from tensor desc.");
728 return false;
729 }
730 if (dataType != tensor.GetDataType()) {
731 LOGI("Tensors have different data type: %d and %d.", dataType, tensor.GetDataType());
732 return false;
733 }
734
735 int32_t* shape {nullptr};
736 size_t shapeNum {0};
737 ret = tensorDesc.first->GetShape(&shape, &shapeNum);
738 if (ret != OH_NN_SUCCESS) {
739 LOGE("CompareAttribute failed, failed to get shape from tensor desc.");
740 return false;
741 }
742 const std::vector<int32_t> dimensions = tensor.GetDimensions();
743 if (shapeNum != dimensions.size()) {
744 LOGI("Tensors have differents dimension counts: %zu and %zu.", shapeNum, dimensions.size());
745 return false;
746 }
747
748 size_t dimensionsSize = dimensions.size();
749 for (size_t i = 0; i < dimensionsSize; i++) {
750 if ((shape[i] != -1) && (shape[i] != dimensions[i])) {
751 LOGI("Tensors have different dimension: dimension index: %zu, dimension value: %d and %d.",
752 i, shape[i], dimensions[i]);
753 return false;
754 }
755 }
756
757 if (tensorDesc.second != tensor.GetType()) {
758 LOGI("Tensors have different type: %{public}d and %{public}d.", tensorDesc.second, tensor.GetType());
759 return false;
760 }
761
762 return true;
763 }
764
BuildInputTensor(uint32_t index,const OH_NN_Tensor & nnTensor,std::shared_ptr<NNTensor> inputTensor) const765 OH_NN_ReturnCode NNExecutor::BuildInputTensor(uint32_t index, const OH_NN_Tensor& nnTensor,
766 std::shared_ptr<NNTensor> inputTensor) const
767 {
768 // Note: inputs have only shapes info.
769 if (index >= m_inputTensorDescs.size()) {
770 LOGE("BuildInputTensor failed, input index is out of range.");
771 return OH_NN_INVALID_PARAMETER;
772 }
773 if (m_inputTensorDescs[index].first == nullptr) {
774 LOGE("BuildInputTensor failed, tensor desc of input %{public}u is nullptr.", index);
775 return OH_NN_INVALID_PARAMETER;
776 }
777
778 // Build a tensor from nnTensor.
779 auto ret = inputTensor->BuildFromOHNNTensor(nnTensor);
780 if (ret != OH_NN_SUCCESS) {
781 LOGE("BuildInputTensor failed, please check input nnTensor.");
782 return ret;
783 }
784
785 if (inputTensor->IsDynamicShape()) {
786 LOGE("BuildInputTensor failed, input nnTensor should has certain dimensions which cannot contain -1.");
787 return OH_NN_INVALID_PARAMETER;
788 }
789
790 OH_NN_Format format;
791 ret = m_inputTensorDescs[index].first->GetFormat(&format);
792 if (ret != OH_NN_SUCCESS) {
793 LOGE("BuildInputTensor failed, failed to get tensor format from desc.");
794 return ret;
795 }
796 inputTensor->SetFormat(format);
797
798 if (!CompareAttribute(m_inputTensorDescs[index], *inputTensor)) {
799 LOGE("BuildInputTensor failed, input has different attributes from the one in the constructed model.");
800 return OH_NN_INVALID_PARAMETER;
801 }
802
803 const char* name {nullptr};
804 ret = m_inputTensorDescs[index].first->GetName(&name);
805 if (ret != OH_NN_SUCCESS) {
806 LOGE("BuildInputTensor failed, failed to get tensor name from desc.");
807 return ret;
808 }
809 inputTensor->SetName(name);
810 return OH_NN_SUCCESS;
811 }
812
813
SetInputTensorWithCurrentBuffer(uint32_t index,std::shared_ptr<NNTensor> inputTensor,const void * buffer,size_t dataLength,size_t curBufferLength)814 OH_NN_ReturnCode NNExecutor::SetInputTensorWithCurrentBuffer(uint32_t index,
815 std::shared_ptr<NNTensor> inputTensor, const void* buffer, size_t dataLength, size_t curBufferLength)
816 {
817 void* curBuffer = m_inputTensors[index].tensor->GetBuffer();
818 errno_t status = memcpy_s(curBuffer, dataLength, buffer, dataLength);
819 // Current buffer inside m_inputTensors is managed by executor, no need to release if memcpy failed.
820 if (status != EOK) {
821 LOGE("SetInputTensorWithCurrentBuffe failed, copy data from user buffer to device buffer failed. "
822 "Error code: %d.", status);
823 return OH_NN_MEMORY_ERROR;
824 }
825
826 // Set the new tensor with the buffer of current tensor
827 inputTensor->SetBuffer(curBuffer, curBufferLength);
828
829 // The memory is reused here. Thus, current tensor's buffer must set to nullptr, in case the memory is released
830 // twice.
831 m_inputTensors[index].tensor->SetBuffer(nullptr, 0);
832
833 // Set to the new tensor, and release current one.
834 m_inputTensors[index].tensor = inputTensor;
835 return OH_NN_SUCCESS;
836 }
837
838
SetInputTensorWithNewBuffer(uint32_t index,std::shared_ptr<NNTensor> inputTensor,const void * inputBuffer,size_t length,bool isInnerMem)839 void NNExecutor::SetInputTensorWithNewBuffer(uint32_t index,
840 std::shared_ptr<NNTensor> inputTensor, const void* inputBuffer, size_t length, bool isInnerMem)
841 {
842 // Release the memory inside the tensor first, if it is allocated by Executor during SetInput().
843 if (m_inputTensors.find(index) != m_inputTensors.end()) {
844 if (m_inputTensors[index].isInnerMem) {
845 void* curBuffer = m_inputTensors[index].tensor->GetBuffer();
846 m_device->ReleaseBuffer(curBuffer);
847 }
848 // Set current tensor's buffer to nullptr in case the NNTensor release the driver memory in destruction.
849 m_inputTensors[index].tensor->SetBuffer(nullptr, 0);
850 }
851
852 // Set new input tensor data buffer
853 inputTensor->SetBuffer(inputBuffer, length);
854
855 // Create or update the input tensor
856 ExeTensor exeTensor{inputTensor, nullptr, 0, isInnerMem};
857 m_inputTensors[index] = exeTensor;
858 }
859
860
CheckInputDimRanges(uint32_t index,const OH_NN_Tensor & nnTensor) const861 OH_NN_ReturnCode NNExecutor::CheckInputDimRanges(uint32_t index, const OH_NN_Tensor& nnTensor) const
862 {
863 std::vector<std::vector<uint32_t>> minInputDims;
864 std::vector<std::vector<uint32_t>> maxInputDims;
865 auto ret = m_preparedModel->GetInputDimRanges(minInputDims, maxInputDims);
866 if (ret != OH_NN_SUCCESS) {
867 LOGE("Get the dimension ranges of input %u failed. ErrorCode=%d", index, ret);
868 return ret;
869 }
870
871 if (index >= minInputDims.size()) {
872 LOGE("index is %u, which exceeds the size of minInputDims:%zu.", index, minInputDims.size());
873 return OH_NN_INVALID_PARAMETER;
874 }
875
876 if (index >= maxInputDims.size()) {
877 LOGE("index is %u, which exceeds the size of maxInputDims:%zu.", index, maxInputDims.size());
878 return OH_NN_INVALID_PARAMETER;
879 }
880
881 const std::vector<uint32_t>& minSingleInputDims = minInputDims[index];
882 const std::vector<uint32_t>& maxSingleInputDims = maxInputDims[index];
883
884 std::vector<int32_t> tensorShape = ConstructVectorFromArray(nnTensor.dimensions, nnTensor.dimensionCount);
885 size_t tensorShapeSize = tensorShape.size();
886 if (minSingleInputDims.size() != tensorShapeSize || maxSingleInputDims.size() != tensorShapeSize) {
887 LOGE("Size of minSingleInputDims, maxSingleInputDims and tensorShape of input %u are not equal.", index);
888 return OH_NN_INVALID_PARAMETER;
889 }
890
891 for (size_t j = 0; j < tensorShapeSize; ++j) {
892 // Dimensions cannot be negative
893 if (tensorShape[j] < 0) {
894 LOGE("Dimension %zu of input %u is %d.", j, index, tensorShape[j]);
895 return OH_NN_INVALID_PARAMETER;
896 }
897 uint32_t dim = static_cast<uint32_t>(tensorShape[j]);
898 if (dim < minSingleInputDims[j] || dim > maxSingleInputDims[j]) {
899 LOGE("Dimension %zu of input %u is %u, which is out of range [%u, %u]",
900 j, index, dim, minSingleInputDims[j], maxSingleInputDims[j]);
901 return OH_NN_INVALID_PARAMETER;
902 }
903 }
904
905 return OH_NN_SUCCESS;
906 }
907
908
SetInput(uint32_t index,const OH_NN_Tensor & nnTensor,const void * buffer,size_t length)909 OH_NN_ReturnCode NNExecutor::SetInput(uint32_t index, const OH_NN_Tensor& nnTensor, const void* buffer, size_t length)
910 {
911 auto nnRet = CheckInputDimRanges(index, nnTensor);
912 if (nnRet == OH_NN_OPERATION_FORBIDDEN) {
913 LOGI("Skip input dimension bounds check.");
914 } else if (nnRet != OH_NN_SUCCESS) {
915 LOGE("SetInput failed, Check the range of the %uth input dimension ranges failed.", index);
916 return nnRet;
917 }
918
919 std::shared_ptr<NNTensor> inputTensor = CreateSharedPtr<NNTensor>();
920 if (inputTensor == nullptr) {
921 LOGE("SetInput failed, error happened when creating NNTensor.");
922 return OH_NN_MEMORY_ERROR;
923 }
924
925 auto ret = BuildInputTensor(index, nnTensor, inputTensor);
926 if (ret != OH_NN_SUCCESS) {
927 LOGE("SetInput failed, please check input index or nnTensor.");
928 return ret;
929 }
930
931 // dataLength will be larger than 0 after BuildInputTensor()
932 size_t dataLength = inputTensor->GetDataLength();
933 if (length == 0 || length < dataLength) {
934 LOGE("SetInput failed, the given buffer length is too small to store the input nnTensor data.");
935 return OH_NN_INVALID_PARAMETER;
936 }
937
938 // Get length of current buffer if it is allocate by SetInput() before.
939 size_t curBufferLength = 0;
940 if ((m_inputTensors.find(index) != m_inputTensors.end()) && (m_inputTensors[index].isInnerMem)) {
941 curBufferLength = m_inputTensors[index].tensor->GetBufferLength();
942 }
943
944 // (dataLength <= curBufferLength) returns true if and only if current buffer is allocated by SetInput() before
945 // and is larger than user buffer.
946 if (dataLength <= curBufferLength) {
947 ret = SetInputTensorWithCurrentBuffer(index, inputTensor, buffer, dataLength, curBufferLength);
948 if (ret != OH_NN_SUCCESS) {
949 LOGE("SetInput failed, error happened when setting input with current buffer.");
950 return ret;
951 }
952 m_isRun = false;
953 return OH_NN_SUCCESS;
954 }
955
956 /**
957 * Buffer needs to allocated or reallocated if:
958 *
959 * - Current buffer is not enough.
960 * - SetInput() has not been called for the input before.
961 * - The buffer held in m_inputTensors is allocated and set by CreateInputMemory() and SetInputFromMemory().
962 */
963 void* inputBuffer = m_device->AllocateTensorBuffer(length, inputTensor);
964 if (inputBuffer == nullptr) {
965 LOGE("SetInput failed, error happened when allocating input device buffer.");
966 return OH_NN_MEMORY_ERROR;
967 }
968
969 errno_t status = memcpy_s(inputBuffer, dataLength, buffer, dataLength);
970 if (status != EOK) {
971 LOGE("SetInput failed, copy data from user buffer failed. Error code: %d.", status);
972 m_device->ReleaseBuffer(inputBuffer);
973 return OH_NN_MEMORY_ERROR;
974 }
975
976 SetInputTensorWithNewBuffer(index, inputTensor, inputBuffer, length, true);
977 m_isRun = false;
978 return OH_NN_SUCCESS;
979 }
980
SetInputFromMemory(uint32_t index,const OH_NN_Tensor & nnTensor,const OH_NN_Memory & memory)981 OH_NN_ReturnCode NNExecutor::SetInputFromMemory(
982 uint32_t index, const OH_NN_Tensor& nnTensor, const OH_NN_Memory& memory)
983 {
984 auto nnRet = CheckInputDimRanges(index, nnTensor);
985 if (nnRet == OH_NN_OPERATION_FORBIDDEN) {
986 LOGI("Skip input dimension bounds check.");
987 } else if (nnRet != OH_NN_SUCCESS) {
988 LOGE("SetInputFromMemory failed, Check the range of the %uth input dimension ranges failed.", index);
989 return nnRet;
990 }
991
992 // Build a input tensor
993 std::shared_ptr<NNTensor> inputTensor = CreateSharedPtr<NNTensor>();
994 if (inputTensor == nullptr) {
995 LOGE("SetInputFromMemory failed, error happened when creating NNTensor.");
996 return OH_NN_MEMORY_ERROR;
997 }
998
999 auto ret = BuildInputTensor(index, nnTensor, inputTensor);
1000 if (ret != OH_NN_SUCCESS) {
1001 LOGE("SetInputFromMemory failed, please check input index or nnTensor");
1002 return ret;
1003 }
1004
1005 // check data length
1006 size_t dataLength = inputTensor->GetDataLength();
1007 if (memory.length == 0 || memory.length < dataLength) {
1008 LOGE("SetInputFromMemory failed,"
1009 " the length in the given memory is too small to store the input nnTensor data.");
1010 return OH_NN_INVALID_PARAMETER;
1011 }
1012
1013 SetInputTensorWithNewBuffer(index, inputTensor, const_cast<const void*>(memory.data), memory.length, false);
1014 m_isRun = false;
1015 return OH_NN_SUCCESS;
1016 }
1017
BuildNNTensorFromDesc(const std::pair<std::shared_ptr<TensorDesc>,OH_NN_TensorType> & tensorDesc)1018 std::shared_ptr<NNTensor> NNExecutor::BuildNNTensorFromDesc(
1019 const std::pair<std::shared_ptr<TensorDesc>, OH_NN_TensorType>& tensorDesc)
1020 {
1021 std::shared_ptr<NNTensor> tensor = CreateSharedPtr<NNTensor>();
1022 if (tensor == nullptr) {
1023 LOGE("BuildNNTensorFromDesc failed, error happened when creating NNTensor.");
1024 return nullptr;
1025 }
1026
1027 // Build a tensor from nnTensor.
1028 NN_TensorDesc* tensorDescCast = reinterpret_cast<NN_TensorDesc*>(tensorDesc.first.get());
1029 auto ret = tensor->BuildFromTensorDesc(tensorDescCast);
1030 if (ret != OH_NN_SUCCESS) {
1031 LOGE("BuildNNTensorFromDesc failed, please check input nnTensor.");
1032 return nullptr;
1033 }
1034
1035 OH_NN_Format format;
1036 tensorDesc.first->GetFormat(&format);
1037 if (ret != OH_NN_SUCCESS) {
1038 LOGE("BuildNNTensorFromDesc failed, failed to get tensor format from desc.");
1039 return nullptr;
1040 }
1041 tensor->SetFormat(format);
1042
1043 ret = tensor->SetTensorType(tensorDesc.second);
1044 if (ret != OH_NN_SUCCESS) {
1045 LOGE("BuildNNTensorFromDesc failed, failed to set tensor type.");
1046 return nullptr;
1047 }
1048
1049 if (!CompareAttribute(tensorDesc, *tensor)) {
1050 LOGE("BuildNNTensorFromDesc failed, input has different attributes from the one in the constructed model.");
1051 return nullptr;
1052 }
1053
1054 const char* name {nullptr};
1055 ret = tensorDesc.first->GetName(&name);
1056 if (ret != OH_NN_SUCCESS) {
1057 LOGE("BuildNNTensorFromDesc failed, failed to get tensor name from desc.");
1058 return nullptr;
1059 }
1060 tensor->SetName(name);
1061 return tensor;
1062 }
1063
SetOutput(uint32_t index,void * buffer,size_t length)1064 OH_NN_ReturnCode NNExecutor::SetOutput(uint32_t index, void* buffer, size_t length)
1065 {
1066 if (index >= m_outputTensorDescs.size()) {
1067 LOGE("SetOutput failed, output index is out of range.");
1068 return OH_NN_INVALID_PARAMETER;
1069 }
1070 if (m_outputTensorDescs[index].first == nullptr) {
1071 LOGE("NNExecutor::SetOutput failed, tensor desc of output %{public}u is nullptr.", index);
1072 return OH_NN_INVALID_PARAMETER;
1073 }
1074
1075 size_t dataLength {0};
1076 auto ret = m_outputTensorDescs[index].first->GetByteSize(&dataLength);
1077 if (ret != OH_NN_SUCCESS) {
1078 LOGE("SetOutputFromMemory failed, failed to get byte size from tensor desc.");
1079 return ret;
1080 }
1081 if (length == 0 || length < dataLength) {
1082 LOGE("SetOutput failed, the given buffer length is too small to store the output tensor data.");
1083 return OH_NN_INVALID_PARAMETER;
1084 }
1085
1086 // If output tensor does not exist, or inner device buffer size is not enough,
1087 // or device buffer is set by SetOutputFromMemory() before,
1088 // allocate a new device buffer and set it to output tensor, and update the user buffer.
1089 if (m_outputTensors.find(index) != m_outputTensors.end()) {
1090 if (m_outputTensors[index].isInnerMem) {
1091 size_t curBufferLength = m_outputTensors[index].tensor->GetBufferLength();
1092 if (length <= curBufferLength) {
1093 // If current device buffer size is enough, only update the user buffer.
1094 m_outputTensors[index].userBuffer = buffer;
1095 m_outputTensors[index].userBufferLength = length;
1096 m_isRun = false;
1097 return OH_NN_SUCCESS;
1098 } else {
1099 // If current device buffer size is not enough,
1100 // release current device buffer and then allocate a new one below.
1101 void* curBuffer = m_outputTensors[index].tensor->GetBuffer();
1102 m_device->ReleaseBuffer(curBuffer);
1103 }
1104 }
1105 } else {
1106 // If output tensor does not exist, create a new null output tensor.
1107 ExeTensor exeTensor;
1108 m_outputTensors[index] = exeTensor;
1109 m_outputTensors[index].tensor = BuildNNTensorFromDesc(m_outputTensorDescs[index]);
1110 if (m_outputTensors[index].tensor == nullptr) {
1111 LOGE("SetOutput failed, failed to build nntensor from desc.");
1112 return OH_NN_NULL_PTR;
1113 }
1114 }
1115
1116 void* deviceOutputBuffer = m_device->AllocateTensorBuffer(length, m_outputTensorDescs[index].first);
1117 if (deviceOutputBuffer == nullptr) {
1118 LOGE("SetOutput failed, allocating output device buffer failed.");
1119 return OH_NN_MEMORY_ERROR;
1120 }
1121
1122 m_outputTensors[index].tensor->SetBuffer(deviceOutputBuffer, length);
1123 m_outputTensors[index].userBuffer = buffer;
1124 m_outputTensors[index].userBufferLength = length;
1125 m_outputTensors[index].isInnerMem = true;
1126 m_isRun = false;
1127 return OH_NN_SUCCESS;
1128 }
1129
1130
SetOutputFromMemory(uint32_t index,const OH_NN_Memory & memory)1131 OH_NN_ReturnCode NNExecutor::SetOutputFromMemory(uint32_t index, const OH_NN_Memory& memory)
1132 {
1133 if (index >= m_outputTensorDescs.size()) {
1134 LOGE("SetOutputFromMemory failed, output index is out of range.");
1135 return OH_NN_INVALID_PARAMETER;
1136 }
1137 if (m_outputTensorDescs[index].first == nullptr) {
1138 LOGE("NNExecutor::SetOutputFromMemory failed, tensor desc of output %{public}u is nullptr.", index);
1139 return OH_NN_INVALID_PARAMETER;
1140 }
1141
1142 size_t dataLength {0};
1143 auto ret = m_outputTensorDescs[index].first->GetByteSize(&dataLength);
1144 if (ret != OH_NN_SUCCESS) {
1145 LOGE("SetOutputFromMemory failed, failed to get byte size from tensor desc.");
1146 return ret;
1147 }
1148 if (memory.length == 0 || memory.length < dataLength) {
1149 LOGE("SetOutputFromMemory failed, the memory is too small to store the output tensor data.");
1150 return OH_NN_INVALID_PARAMETER;
1151 }
1152
1153 if (m_outputTensors.find(index) != m_outputTensors.end()) {
1154 if (m_outputTensors[index].isInnerMem) {
1155 // If it is inner buffer, releate it
1156 void* curBuffer = m_outputTensors[index].tensor->GetBuffer();
1157 m_device->ReleaseBuffer(curBuffer);
1158 }
1159 } else {
1160 // If output tensor does not exist, create a new null output tensor.
1161 ExeTensor exeTensor;
1162 m_outputTensors[index] = exeTensor;
1163 m_outputTensors[index].tensor = BuildNNTensorFromDesc(m_outputTensorDescs[index]);
1164 if (m_outputTensors[index].tensor == nullptr) {
1165 LOGE("SetOutputFromMemory failed, failed to build nntensor from desc.");
1166 return OH_NN_NULL_PTR;
1167 }
1168 }
1169
1170 // Set the output tensor with memory
1171 m_outputTensors[index].tensor->SetBuffer(const_cast<const void*>(memory.data), memory.length);
1172 m_outputTensors[index].userBuffer = nullptr;
1173 m_outputTensors[index].userBufferLength = 0;
1174 m_outputTensors[index].isInnerMem = false;
1175 m_isRun = false;
1176 return OH_NN_SUCCESS;
1177 }
1178
CreateInputMemory(uint32_t index,size_t length,OH_NN_Memory ** memory)1179 OH_NN_ReturnCode NNExecutor::CreateInputMemory(uint32_t index, size_t length, OH_NN_Memory** memory)
1180 {
1181 if (index >= m_inputTensorDescs.size()) {
1182 LOGE("CreateInputMemory failed, input index is out of range.");
1183 return OH_NN_INVALID_PARAMETER;
1184 }
1185 if (m_inputTensorDescs[index].first == nullptr) {
1186 LOGE("CreateInputMemory failed, tensor desc of input %{public}u is nullptr.", index);
1187 return OH_NN_INVALID_PARAMETER;
1188 }
1189
1190 // Allocate device buffer
1191 void* deviceInputBuffer = m_device->AllocateTensorBuffer(length, m_inputTensorDescs[index].first);
1192 if (deviceInputBuffer == nullptr) {
1193 LOGE("CreateInputMemory failed, allocating intput device buffer failed.");
1194 return OH_NN_MEMORY_ERROR;
1195 }
1196
1197 *memory = new(std::nothrow) OH_NN_Memory{deviceInputBuffer, length};
1198 if (*memory == nullptr) {
1199 LOGE("CreateInputMemory failed, constructing OH_NN_Memory failed.");
1200 m_device->ReleaseBuffer(deviceInputBuffer);
1201 return OH_NN_MEMORY_ERROR;
1202 }
1203
1204 // Save the buffer address for check when destroying it.
1205 m_inputCreatedMem[index].emplace_back(deviceInputBuffer);
1206
1207 return OH_NN_SUCCESS;
1208 }
1209
1210
DestroyInputMemory(uint32_t index,OH_NN_Memory ** memory)1211 OH_NN_ReturnCode NNExecutor::DestroyInputMemory(uint32_t index, OH_NN_Memory** memory)
1212 {
1213 if (index >= m_inputTensorDescs.size()) {
1214 LOGE("DestroyInputMemory failed, input index is out of range.");
1215 return OH_NN_INVALID_PARAMETER;
1216 }
1217
1218 if (m_inputCreatedMem.find(index) == m_inputCreatedMem.end()) {
1219 LOGE("DestroyInputMemory failed, the memory has not been created with the index.");
1220 return OH_NN_INVALID_PARAMETER;
1221 }
1222
1223 std::vector<void*>& inputCreatedMem = m_inputCreatedMem[index];
1224 auto pos = std::find(inputCreatedMem.begin(), inputCreatedMem.end(), (*memory)->data);
1225 if (pos == inputCreatedMem.end()) {
1226 LOGE("DestroyInputMemory failed, the index does not match the memory.");
1227 return OH_NN_INVALID_PARAMETER;
1228 }
1229
1230 auto ret = m_device->ReleaseBuffer((*memory)->data);
1231 if (ret != OH_NN_SUCCESS) {
1232 LOGE("Release input buffer failed.");
1233 return ret;
1234 }
1235
1236 inputCreatedMem.erase(pos);
1237 delete *memory;
1238 *memory = nullptr;
1239
1240 return OH_NN_SUCCESS;
1241 }
1242
1243
CreateOutputMemory(uint32_t index,size_t length,OH_NN_Memory ** memory)1244 OH_NN_ReturnCode NNExecutor::CreateOutputMemory(uint32_t index, size_t length, OH_NN_Memory** memory)
1245 {
1246 if (index >= m_outputTensorDescs.size()) {
1247 LOGE("CreateOutputMemory failed, output index is out of range.");
1248 return OH_NN_INVALID_PARAMETER;
1249 }
1250 if (m_outputTensorDescs[index].first == nullptr) {
1251 LOGE("NNExecutor::CreateOutputMemory failed, tensor desc of output %{public}u is nullptr.", index);
1252 return OH_NN_INVALID_PARAMETER;
1253 }
1254
1255 // Allocate device buffer
1256 void* deviceOutputBuffer = m_device->AllocateTensorBuffer(length, m_outputTensorDescs[index].first);
1257 if (deviceOutputBuffer == nullptr) {
1258 LOGE("CreateOutputMemory failed, allocating output device buffer failed.");
1259 return OH_NN_MEMORY_ERROR;
1260 }
1261
1262 *memory = new(std::nothrow) OH_NN_Memory{deviceOutputBuffer, length};
1263 if (*memory == nullptr) {
1264 LOGE("CreateOutputMemory failed, constructing OH_NN_Memory failed.");
1265 m_device->ReleaseBuffer(deviceOutputBuffer);
1266 return OH_NN_MEMORY_ERROR;
1267 }
1268
1269 // Save the buffer address for check when destroying it.
1270 m_outputCreatedMem[index].emplace_back(deviceOutputBuffer);
1271
1272 return OH_NN_SUCCESS;
1273 }
1274
1275
DestroyOutputMemory(uint32_t index,OH_NN_Memory ** memory)1276 OH_NN_ReturnCode NNExecutor::DestroyOutputMemory(uint32_t index, OH_NN_Memory** memory)
1277 {
1278 if (index >= m_outputTensorDescs.size()) {
1279 LOGE("DestroyOutputMemory failed, output index is out of range.");
1280 return OH_NN_INVALID_PARAMETER;
1281 }
1282
1283 if (m_outputCreatedMem.find(index) == m_outputCreatedMem.end()) {
1284 LOGE("DestroyOutputMemory failed, the memory has not been created with the index.");
1285 return OH_NN_INVALID_PARAMETER;
1286 }
1287
1288 std::vector<void*>& outputCreatedMem = m_outputCreatedMem[index];
1289 auto pos = std::find(outputCreatedMem.begin(), outputCreatedMem.end(), (*memory)->data);
1290 if (pos == outputCreatedMem.end()) {
1291 LOGE("DestroyOutputMemory failed, the index does not match the memory.");
1292 return OH_NN_INVALID_PARAMETER;
1293 }
1294
1295 auto ret = m_device->ReleaseBuffer((*memory)->data);
1296 if (ret != OH_NN_SUCCESS) {
1297 LOGE("Release output buffer failed.");
1298 return ret;
1299 }
1300
1301 outputCreatedMem.erase(pos);
1302 delete *memory;
1303 *memory = nullptr;
1304
1305 return OH_NN_SUCCESS;
1306 }
1307
Run(const std::vector<std::shared_ptr<NNTensor>> & inputTensors,std::vector<std::shared_ptr<NNTensor>> & outputTensors)1308 OH_NN_ReturnCode NNExecutor::Run(const std::vector<std::shared_ptr<NNTensor>>& inputTensors,
1309 std::vector<std::shared_ptr<NNTensor>>& outputTensors)
1310 {
1311 OH_NN_ReturnCode ret {OH_NN_FAILED};
1312 IOTensor tensor;
1313 std::vector<IOTensor> inputIOTensors;
1314 size_t inputSize = inputTensors.size();
1315 size_t outputSize = outputTensors.size();
1316 for (size_t i = 0; i < inputSize; ++i) {
1317 inputTensors[i]->ConvertToIOTensor(tensor);
1318 inputIOTensors.emplace_back(std::move(tensor));
1319 }
1320
1321 std::vector<IOTensor> outputIOTensors;
1322 for (size_t i = 0; i < outputSize; ++i) {
1323 outputTensors[i]->ConvertToIOTensor(tensor);
1324 outputIOTensors.emplace_back(std::move(tensor));
1325 }
1326
1327 std::vector<std::vector<int32_t>> outputsDims;
1328 std::vector<bool> isSufficientDataBuffer;
1329 ret = m_preparedModel->Run(inputIOTensors, outputIOTensors, outputsDims, isSufficientDataBuffer);
1330 if (ret != OH_NN_SUCCESS) {
1331 LOGE("PrepardModel Run() failed.");
1332 return ret;
1333 }
1334
1335 // Set the output NNTensor's dimensions from output IOTensor if it is dynamic.
1336 // NNTensor::SetDimensions will check if the tensor buffer is enough for the new dimensions.
1337 if (outputsDims.size() != outputSize) {
1338 LOGE("ExecutionPlan run failed, size of outputsDims is not equal to outputTensors.");
1339 return OH_NN_INVALID_PARAMETER;
1340 }
1341 for (size_t i = 0; i < outputSize; ++i) {
1342 ret = outputTensors[i]->SetDimensions(outputsDims[i]);
1343 if (ret != OH_NN_SUCCESS) {
1344 LOGE("Run failed, error happened when setting output tensor's dimensions, output id: %zu.", i);
1345 return ret;
1346 }
1347 ret = m_outputTensorDescs[i].first->SetShape(outputsDims[i].data(), outputsDims[i].size());
1348 if (ret != OH_NN_SUCCESS) {
1349 LOGE("Run failed, error happened when setting inner output tensor's dimensions,"
1350 " output id: %zu.", i);
1351 return ret;
1352 }
1353 }
1354
1355 return OH_NN_SUCCESS;
1356 }
1357
Run()1358 OH_NN_ReturnCode NNExecutor::Run()
1359 {
1360 NNRT_TRACE_NAME("Execution");
1361 if (m_inputTensorDescs.size() != m_inputTensors.size()) {
1362 LOGE("Run failed, some input tensors have not been set.");
1363 return OH_NN_INVALID_PARAMETER;
1364 }
1365 if (m_outputTensorDescs.size() != m_outputTensors.size()) {
1366 LOGE("Run failed, some output tensors have not been set.");
1367 return OH_NN_INVALID_PARAMETER;
1368 }
1369
1370 // Build the NNTensor pointer vector: inputTensors and outputTensors
1371 std::vector<std::shared_ptr<NNTensor>> inputTensors;
1372 std::vector<std::shared_ptr<NNTensor>> outputTensors;
1373 size_t inputSize = m_inputTensors.size();
1374 size_t outputSize = m_outputTensors.size();
1375 for (size_t i = 0; i < inputSize; ++i) {
1376 inputTensors.emplace_back(m_inputTensors[i].tensor);
1377 }
1378 for (size_t i = 0; i < outputSize; ++i) {
1379 outputTensors.emplace_back(m_outputTensors[i].tensor);
1380 }
1381
1382 // Predict
1383 auto ret = Run(inputTensors, outputTensors);
1384 if (ret != OH_NN_SUCCESS) {
1385 LOGE("Run failed, error happened when executing the inference.");
1386 return ret;
1387 }
1388
1389 errno_t status{EOK};
1390 // Copy inner device buffer to user buffer if using SetOutput()
1391 for (size_t i = 0; i < outputSize; ++i) {
1392 if (m_outputTensors[i].isInnerMem) {
1393 auto size = outputTensors[i]->GetDataLength();
1394 if (size > m_outputTensors[i].userBufferLength) {
1395 LOGE("Output buffer size is not enough. Your size=%zu, but actual output size=%zu.",
1396 m_outputTensors[i].userBufferLength, size);
1397 return OH_NN_INVALID_PARAMETER;
1398 }
1399
1400 void* deviceBuffer = outputTensors[i]->GetBuffer();
1401 if (deviceBuffer == nullptr) {
1402 LOGE("Output buffer is nullptr.");
1403 return OH_NN_FAILED;
1404 }
1405
1406 status = memcpy_s(m_outputTensors[i].userBuffer, m_outputTensors[i].userBufferLength, deviceBuffer, size);
1407 if (status != EOK) {
1408 LOGE("Run failed, memory copy from device buffer to user buffer failed. Error code: %d.", status);
1409 return OH_NN_MEMORY_ERROR;
1410 }
1411 }
1412 }
1413
1414 m_isRun = true;
1415 return OH_NN_SUCCESS;
1416 }
1417
~NNExecutor()1418 NNExecutor::~NNExecutor()
1419 {
1420 for (auto& it : m_inputTensors) {
1421 if ((it.second).isInnerMem) {
1422 m_device->ReleaseBuffer((it.second).tensor->GetBuffer());
1423 }
1424 (it.second).tensor->SetBuffer(nullptr, 0);
1425 (it.second).tensor.reset();
1426 (it.second).userBuffer = nullptr;
1427 }
1428 m_inputTensors.clear();
1429
1430 for (auto& it : m_outputTensors) {
1431 if ((it.second).isInnerMem) {
1432 m_device->ReleaseBuffer((it.second).tensor->GetBuffer());
1433 }
1434 (it.second).tensor->SetBuffer(nullptr, 0);
1435 (it.second).tensor.reset();
1436 (it.second).userBuffer = nullptr;
1437 }
1438 m_outputTensors.clear();
1439
1440 for (auto& it : m_inputCreatedMem) {
1441 it.second.clear();
1442 }
1443 m_inputCreatedMem.clear();
1444
1445 for (auto& it : m_outputCreatedMem) {
1446 it.second.clear();
1447 }
1448 m_outputCreatedMem.clear();
1449
1450 if (m_executorConfig != nullptr) {
1451 delete m_executorConfig;
1452 m_executorConfig = nullptr;
1453 }
1454
1455 UnSetDeinitModelCallBack();
1456
1457 uint32_t modelId;
1458 GetModelID(modelId);
1459 LOGI("manualUnload pid=%{public}d originHiaiModelId=%{public}d hiaiModelId=%{public}d",
1460 getpid(), m_originHiaiModelId, modelId);
1461 }
1462
SetDeinitModelCallBack()1463 OH_NN_ReturnCode NNExecutor::SetDeinitModelCallBack()
1464 {
1465 NNRtServiceApi& nnrtService = NNRtServiceApi::GetInstance();
1466 if (!nnrtService.IsServiceAvaliable()) {
1467 LOGW("SetDeinitModelCallBack failed, fail to get nnrt service, skip SetDeinitModelCallBack.");
1468 return OH_NN_SUCCESS;
1469 }
1470
1471 if (nnrtService.SetDeinitModelCallBack == nullptr) {
1472 LOGE("SetDeinitModelCallBack failed, nnrtService SetDeinitModelCallBack func is nullptr.");
1473 return OH_NN_INVALID_PARAMETER;
1474 }
1475
1476 if (m_preparedModel == nullptr) {
1477 LOGE("SetDeinitModelCallBack failed, m_preparedModel is nullptr.");
1478 return OH_NN_INVALID_PARAMETER;
1479 }
1480
1481 int ret = nnrtService.SetDeinitModelCallBack(m_executorid, reinterpret_cast<Executor*>(this));
1482 if (ret != static_cast<int>(OH_NN_SUCCESS)) {
1483 LOGE("SetDeinitModelCallBack failed, some error happened when SetDeinitModelCallBack.");
1484 return static_cast<OH_NN_ReturnCode>(ret);
1485 }
1486
1487 return OH_NN_SUCCESS;
1488 }
1489
UnSetDeinitModelCallBack()1490 OH_NN_ReturnCode NNExecutor::UnSetDeinitModelCallBack()
1491 {
1492 NNRtServiceApi& nnrtService = NNRtServiceApi::GetInstance();
1493 if (!nnrtService.IsServiceAvaliable()) {
1494 LOGW("UnSetDeinitModelCallBack failed, fail to get nnrt service, skip UnSetDeinitModelCallBack.");
1495 return OH_NN_SUCCESS;
1496 }
1497
1498 if (nnrtService.UnSetDeinitModelCallBack == nullptr) {
1499 LOGE("UnSetDeinitModelCallBack failed, nnrtService UnSetDeinitModelCallBack func is nullptr.");
1500 return OH_NN_INVALID_PARAMETER;
1501 }
1502
1503 int ret = nnrtService.UnSetDeinitModelCallBack(m_executorid);
1504 if (ret != static_cast<int>(OH_NN_SUCCESS)) {
1505 LOGE("UnSetDeinitModelCallBack failed, some error happened when UnSetDeinitModelCallBack.");
1506 return static_cast<OH_NN_ReturnCode>(ret);
1507 }
1508
1509 return OH_NN_SUCCESS;
1510 }
1511
GetNNRtModelIDFromCache(const std::string & path,const std::string & modelName,size_t & nnrtModelID)1512 OH_NN_ReturnCode NNExecutor::GetNNRtModelIDFromCache(const std::string& path, const std::string& modelName,
1513 size_t& nnrtModelID)
1514 {
1515 if (path.empty()) {
1516 LOGE("GetNNRtmodelIDFromCache failed, path is empty");
1517 return OH_NN_INVALID_PARAMETER;
1518 }
1519
1520 if (modelName.empty()) {
1521 LOGE("GetNNRtmodelIDFromCache failed, modelName is empty");
1522 return OH_NN_INVALID_PARAMETER;
1523 }
1524
1525 if (!std::filesystem::is_directory(path)) {
1526 LOGW("GetNNRtmodelIDFromCache cvache path is not directory.");
1527 nnrtModelID = std::hash<std::string>{}(path);
1528 return OH_NN_SUCCESS;
1529 }
1530
1531 std::string modelPath = path + "/" + modelName + "cache_info.nncache";
1532 char modelCachePath[PATH_MAX];
1533 if (realpath(modelPath.c_str(), modelCachePath) == nullptr) {
1534 LOGE("GetNNRtmodelIDFromCache fail to get real path of cacheDir.");
1535 return OH_NN_INVALID_PARAMETER;
1536 }
1537
1538 NNCompiledCache compiledCache;
1539 NNCompiledCacheInfo cacheInfo;
1540 OH_NN_ReturnCode retCode = compiledCache.SetBackend(m_backendID);
1541 if (retCode != OH_NN_SUCCESS) {
1542 LOGE("GetNNRtmodelIDFromCache failed, fail to set backend.");
1543 return retCode;
1544 }
1545
1546 retCode = compiledCache.CheckCacheInfo(cacheInfo, modelCachePath);
1547 if (retCode != OH_NN_SUCCESS) {
1548 LOGE("GetNNRtmodelIDFromCache failed, fail to CheckCacheInfo.");
1549 return retCode;
1550 }
1551
1552 if (cacheInfo.modelCheckSum.size() != NUMBER_CACHE_INFO_MEMBERS) {
1553 LOGE("GetNNRtmodelIDFromCache failed, fail to modelCheckSum.");
1554 return OH_NN_INVALID_PARAMETER;
1555 }
1556
1557 std::string cacheStr = std::to_string(cacheInfo.modelCheckSum[CHECK_SUM_ZERO]) +
1558 std::to_string(cacheInfo.modelCheckSum[CHECK_SUM_ONE]) +
1559 std::to_string(cacheInfo.modelCheckSum[CHECK_SUM_TWO]);
1560 nnrtModelID = std::hash<std::string>{}(cacheStr);
1561
1562 return OH_NN_SUCCESS;
1563 }
1564
ReinitScheduling(uint32_t hiaimodelID,bool * needModelLatency,const char * cachePath)1565 OH_NN_ReturnCode NNExecutor::ReinitScheduling(uint32_t hiaimodelID, bool* needModelLatency, const char* cachePath)
1566 {
1567 NNRtServiceApi& nnrtService = NNRtServiceApi::GetInstance();
1568 if (!nnrtService.IsServiceAvaliable()) {
1569 LOGE("[HiaiExecutorImpl] ReinitScheduling failed, fail to get nnrt service, skip ReinitScheduling.");
1570 return OH_NN_SUCCESS;
1571 }
1572
1573 if (nnrtService.AutoReinitSetModelID == nullptr) {
1574 LOGE("[HiaiExecutorImpl] ReinitScheduling failed, nnrtService AutoReinitSetModelId func is nullptr.");
1575 return OH_NN_INVALID_PARAMETER;
1576 }
1577
1578 size_t nnrtmodelID = 0;
1579 OH_NN_ReturnCode retCode = GetNNRtModelIDFromCache(m_cachePath, m_extensionConfig.modelName, nnrtmodelID);
1580 if ((retCode != OH_NN_SUCCESS) || (nnrtmodelID == 0)) {
1581 LOGE("[HiaiExecutorImpl] ReinitScheduling is failedm fail to GetNNRtModelIDFromCache.");
1582 return OH_NN_INVALID_PARAMETER;
1583 }
1584
1585 int ret = nnrtService.AutoReinitSetModelID(m_originHiaiModelId, hiaimodelID, nnrtmodelID);
1586 if (ret != static_cast<int>(OH_NN_SUCCESS)) {
1587 LOGE("[HiaiExecutorImpl] ReinitScheduling failed, some error happened when AutoReinitSetModelID.");
1588 return OH_NN_INVALID_PARAMETER;
1589 }
1590
1591 if (nnrtService.IsSupportScheduling == nullptr) {
1592 LOGE("[HiaiExecutorImpl] ReinitScheduling failed, nnrtService IsSupportScheduling func is nullptr.");
1593 return OH_NN_INVALID_PARAMETER;
1594 }
1595
1596 bool supportStat = false;
1597 ret = nnrtService.IsSupportScheduling(&supportStat);
1598 if (ret != static_cast<int>(OH_NN_SUCCESS)) {
1599 LOGE("ReinitScheduling failed, some error happened when judge if support scheduling.");
1600 return OH_NN_INVALID_PARAMETER;
1601 }
1602 if (!supportStat) {
1603 LOGW("device not support scheduling, jumper over scheduling.");
1604 return OH_NN_SUCCESS;
1605 }
1606
1607 if (nnrtService.AutoReinitScheduling == nullptr) {
1608 LOGE("ReinitScheduling failed, nnrtService IsSupportScheduling func is nullptr.");
1609 return OH_NN_INVALID_PARAMETER;
1610 }
1611
1612 ret = nnrtService.AutoReinitScheduling(m_originHiaiModelId, hiaimodelID, needModelLatency, cachePath);
1613 if (ret != static_cast<int>(OH_NN_SUCCESS)) {
1614 LOGE("ReinitScheduling failed, some error happened when scheduling.");
1615 return OH_NN_INVALID_PARAMETER;
1616 }
1617
1618 return OH_NN_SUCCESS;
1619 }
1620
DeinitScheduling(uint32_t hiaimodelID)1621 OH_NN_ReturnCode NNExecutor::DeinitScheduling(uint32_t hiaimodelID)
1622 {
1623 NNRtServiceApi& nnrtService = NNRtServiceApi::GetInstance();
1624 if (nnrtService.AutoUnload == nullptr) {
1625 LOGE("[HiaiExecutorImpl] AutoUnload failed, nnrtService AutoUnload func is nullptr.");
1626 return OH_NN_INVALID_PARAMETER;
1627 }
1628 int ret = nnrtService.AutoUnload(m_originHiaiModelId, hiaimodelID);
1629 if (ret != static_cast<int>(OH_NN_SUCCESS)) {
1630 LOGE("[HiaiExecutorImpl] AutoUnload failed, some error happen when AutoUnload hiaiModelId.");
1631 return OH_NN_INVALID_PARAMETER;
1632 }
1633
1634 return OH_NN_SUCCESS;
1635 }
1636
DeinitModel(std::string mode)1637 bool NNExecutor::DeinitModel(std::string mode)
1638 {
1639 if (m_preparedModel == nullptr) {
1640 return false;
1641 }
1642
1643 std::lock_guard<std::mutex> lock(m_mutex);
1644
1645 if (m_preparedModel != nullptr &&
1646 OH_NNModel_HasCache(m_cachePath.c_str(),
1647 m_extensionConfig.modelName.c_str(),
1648 m_cacheVersion)) {
1649 uint32_t modelId;
1650 auto _ret = GetModelID(modelId);
1651 if (_ret != OH_NN_SUCCESS) {
1652 LOGW("GetModelID failed, some error happen when get model id for device.");
1653 }
1654
1655 _ret = DeinitScheduling(modelId);
1656 if (_ret != OH_NN_SUCCESS) {
1657 LOGW("DeinitScheduling failed, some error happen when DeinitScheduling model.");
1658 }
1659 m_preparedModel.reset();
1660 if (mode == "FrozenDeinit") {
1661 m_autoUnloadHandler->RemoveTask("nnexecutor_autounload" + std::to_string(m_executorid));
1662 LOGI("FrozenDeinit pid=%{public}d originHiaiModelId=%{public}d hiaiModelId=%{public}d",
1663 getpid(), m_originHiaiModelId, modelId);
1664 } else {
1665 LOGI("AutoUnload pid=%{public}d originHiaiModelId=%{public}d hiaiModelId=%{public}d",
1666 getpid(), m_originHiaiModelId, modelId);
1667 }
1668 }
1669
1670 return true;
1671 }
1672 } // namespace NeuralNetworkRuntime
1673 } // namespace OHOS