1From 40b0edbf17e4335b40b51887fabd53aea2f1ef2d Mon Sep 17 00:00:00 2001 2From: chengfeng27 <chengfeng27@huawei.com> 3Date: Wed, 12 Jun 2024 11:46:33 +0800 4Subject: [PATCH] adapter HiAI Foundation NPU 5 6--- 7 mindspore/lite/BUILD.gn | 7 + 8 mindspore/lite/src/litert/c_api/context_c.cc | 11 ++ 9 .../delegate/nnrt/extension_options_parser.cc | 90 ++++++++++++ 10 .../delegate/nnrt/extension_options_parser.h | 46 ++++++ 11 .../delegate/nnrt/hiai_foundation_wrapper.cc | 64 +++++++++ 12 .../delegate/nnrt/hiai_foundation_wrapper.h | 47 +++++++ 13 .../src/litert/delegate/nnrt/nnrt_delegate.cc | 133 ++++++++++++++---- 14 .../src/litert/delegate/nnrt/nnrt_delegate.h | 15 +- 15 8 files changed, 383 insertions(+), 30 deletions(-) 16 create mode 100644 mindspore/lite/src/litert/delegate/nnrt/extension_options_parser.cc 17 create mode 100644 mindspore/lite/src/litert/delegate/nnrt/extension_options_parser.h 18 create mode 100644 mindspore/lite/src/litert/delegate/nnrt/hiai_foundation_wrapper.cc 19 create mode 100644 mindspore/lite/src/litert/delegate/nnrt/hiai_foundation_wrapper.h 20 21diff --git a/mindspore/lite/BUILD.gn b/mindspore/lite/BUILD.gn 22index 6f7f85e9..467cdb6a 100644 23--- a/mindspore/lite/BUILD.gn 24+++ b/mindspore/lite/BUILD.gn 25@@ -445,6 +445,8 @@ ohos_shared_library("mindspore_lib") { 26 "src/litert/delegate/nnrt/nnrt_delegate.cc", 27 "src/litert/delegate/nnrt/nnrt_model_kernel.cc", 28 "src/litert/delegate/nnrt/nnrt_allocator.cc", 29+ "src/litert/delegate/nnrt/hiai_foundation_wrapper.cc", 30+ "src/litert/delegate/nnrt/extension_options_parser.cc", 31 ] 32 include_dirs += [ 33 "src/delegate/nnrt/include", 34@@ -510,6 +512,11 @@ ohos_shared_library("mindspore_ndk") { 35 "ENABLE_HI_APP_EVENT", 36 ] 37 38+ if (mindspore_feature_nnrt_metagraph) { 39+ defines += [ "SUPPORT_NNRT_METAGRAPH" ] 40+ print("enabled feature: mindspore_feature_nnrt_metagraph") 41+ } 42+ 43 configs = [ 44 ":mindspore_api", 45 ":disable_android", 46diff --git a/mindspore/lite/src/litert/c_api/context_c.cc b/mindspore/lite/src/litert/c_api/context_c.cc 47index bde0460c..6b6a50d5 100644 48--- a/mindspore/lite/src/litert/c_api/context_c.cc 49+++ b/mindspore/lite/src/litert/c_api/context_c.cc 50@@ -18,6 +18,9 @@ 51 #include <string.h> 52 #include "src/litert/c_api/type_c_private.h" 53 #include "src/common/log_adapter.h" 54+#ifdef SUPPORT_NNRT_METAGRAPH 55+#include "src/litert/delegate/nnrt/hiai_foundation_wrapper.h" 56+#endif 57 #ifdef SUPPORT_NNRT 58 #include "interfaces/kits/c/neural_network_runtime/neural_network_runtime.h" 59 #endif 60@@ -300,6 +303,14 @@ NNRTDeviceDesc *OH_AI_GetAllNNRTDeviceDescs(size_t *num) { 61 return nullptr; 62 } 63 #ifdef SUPPORT_NNRT 64+#ifdef SUPPORT_NNRT_METAGRAPH 65+ void *hiai_handle_{nullptr}; 66+ auto ret_load = mindspore::lite::LoadHiaiFLibraryFromPath(&hiai_handle_); 67+ if (!ret_load || hiai_handle_ == nullptr) { 68+ MS_LOG(ERROR) << "Load HiAI_Foundation so failed."; 69+ return nullptr; 70+ } 71+#endif 72 *num = 0; 73 74 const size_t *all_device_ids; 75diff --git a/mindspore/lite/src/litert/delegate/nnrt/extension_options_parser.cc b/mindspore/lite/src/litert/delegate/nnrt/extension_options_parser.cc 76new file mode 100644 77index 00000000..98343898 78--- /dev/null 79+++ b/mindspore/lite/src/litert/delegate/nnrt/extension_options_parser.cc 80@@ -0,0 +1,90 @@ 81+/** 82+ * Copyright 2024 Huawei Technologies Co., Ltd 83+ * 84+ * Licensed under the Apache License, Version 2.0 (the "License"); 85+ * you may not use this file except in compliance with the License. 86+ * You may obtain a copy of the License at 87+ * 88+ * http://www.apache.org/licenses/LICENSE-2.0 89+ * 90+ * Unless required by applicable law or agreed to in writing, software 91+ * distributed under the License is distributed on an "AS IS" BASIS, 92+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 93+ * See the License for the specific language governing permissions and 94+ * limitations under the License. 95+ */ 96+ 97+#include "extension_options_parser.h" 98+#include "stdlib.h" 99+#include <map> 100+ 101+namespace mindspore::lite::nnrt { 102+namespace { 103+const std::map<std::string, mindspore::lite::HiAI_BandMode> kBandModeMap = { 104+ {"HIAI_BANDMODE_UNSET", mindspore::lite::HIAI_BANDMODE_UNSET}, 105+ {"HIAI_BANDMODE_LOW", mindspore::lite::HIAI_BANDMODE_LOW}, 106+ {"HIAI_BANDMODE_NORMAL", mindspore::lite::HIAI_BANDMODE_NORMAL}, 107+ {"HIAI_BANDMODE_HIGH", mindspore::lite::HIAI_BANDMODE_HIGH}, 108+}; 109+const std::string kCachePath = "CachePath"; 110+const std::string kCacheVersion = "CacheVersion"; 111+const std::string kBandMode = "BandMode"; 112+const std::string kQuantConfigData = "QuantConfigData"; 113+} // namespace 114+ 115+int ExtensionOptionsParser::Parse(const std::vector<Extension> &extensions, ExtensionOptions *param) { 116+ MS_CHECK_TRUE_RET(param != nullptr, RET_ERROR); 117+ 118+ DoParseCachePath(extensions, ¶m->cache_path_); 119+ DoParseCacheVersion(extensions, ¶m->cache_version_); 120+ DoParseBondMode(extensions, ¶m->band_mode); 121+ DoParseQuantConfig(extensions, ¶m->quant_config, ¶m->quant_config_size); 122+ return RET_OK; 123+} 124+ 125+void ExtensionOptionsParser::DoParseCachePath(const std::vector<Extension> &extensions, std::string *cache_path) { 126+ MS_CHECK_TRUE_RET_VOID(cache_path != nullptr); 127+ auto iter_config = std::find_if(extensions.begin(), extensions.end(), [](const Extension &extension) { 128+ return extension.name == kCachePath; 129+ }); 130+ if (iter_config != extensions.end()) { 131+ *cache_path = std::string(iter_config->value.begin(), iter_config->value.end()); 132+ } 133+} 134+ 135+void ExtensionOptionsParser::DoParseCacheVersion(const std::vector<Extension> &extensions, uint32_t *cache_version) { 136+ MS_CHECK_TRUE_RET_VOID(cache_version != nullptr); 137+ auto iter_config = std::find_if(extensions.begin(), extensions.end(), [](const Extension &extension) { 138+ return extension.name == kCacheVersion; 139+ }); 140+ if (iter_config != extensions.end()) { 141+ std::string version_str = std::string(iter_config->value.begin(), iter_config->value.end()); 142+ *cache_version = static_cast<uint32_t>(std::atol(version_str.c_str())); 143+ } 144+} 145+ 146+void ExtensionOptionsParser::DoParseBondMode(const std::vector<Extension> &extensions, mindspore::lite::HiAI_BandMode *band_mode) { 147+ MS_CHECK_TRUE_RET_VOID(band_mode != nullptr); 148+ auto iter_config = std::find_if(extensions.begin(), extensions.end(), [](const Extension &extension) { 149+ return extension.name == kBandMode; 150+ }); 151+ if (iter_config != extensions.end()) { 152+ auto iter = kBandModeMap.find(std::string(iter_config->value.begin(), iter_config->value.end())); 153+ if (iter != kBandModeMap.end()) { 154+ *band_mode = iter->second; 155+ } 156+ } 157+} 158+ 159+void ExtensionOptionsParser::DoParseQuantConfig(const std::vector<Extension> &extensions, void **quant_config, size_t *num) { 160+ MS_CHECK_TRUE_RET_VOID(quant_config != nullptr); 161+ MS_CHECK_TRUE_RET_VOID(num != nullptr); 162+ auto iter_config = std::find_if(extensions.begin(), extensions.end(), [](const Extension &extension) { 163+ return extension.name == kQuantConfigData; 164+ }); 165+ if (iter_config != extensions.end()) { 166+ *quant_config = static_cast<void *>(const_cast<uint8_t *>(iter_config->value.data())); 167+ *num = iter_config->value.size(); 168+ } 169+} 170+} // mindspore::lite::nnrt 171\ No newline at end of file 172diff --git a/mindspore/lite/src/litert/delegate/nnrt/extension_options_parser.h b/mindspore/lite/src/litert/delegate/nnrt/extension_options_parser.h 173new file mode 100644 174index 00000000..792805a4 175--- /dev/null 176+++ b/mindspore/lite/src/litert/delegate/nnrt/extension_options_parser.h 177@@ -0,0 +1,46 @@ 178+/** 179+ * Copyright 2024 Huawei Technologies Co., Ltd 180+ * 181+ * Licensed under the Apache License, Version 2.0 (the "License"); 182+ * you may not use this file except in compliance with the License. 183+ * You may obtain a copy of the License at 184+ * 185+ * http://www.apache.org/licenses/LICENSE-2.0 186+ * 187+ * Unless required by applicable law or agreed to in writing, software 188+ * distributed under the License is distributed on an "AS IS" BASIS, 189+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 190+ * See the License for the specific language governing permissions and 191+ * limitations under the License. 192+ */ 193+ 194+#ifndef MINDSPORE_LITE_EXTENSION_OPTIONS_PARSER_H 195+#define MINDSPORE_LITE_EXTENSION_OPTIONS_PARSER_H 196+ 197+#include <vector> 198+#include "src/litert/inner_context.h" 199+#include "hiai_foundation_wrapper.h" 200+ 201+namespace mindspore::lite::nnrt { 202+struct ExtensionOptions { 203+ std::string cache_path_ = ""; 204+ uint32_t cache_version_ = 0; 205+ mindspore::lite::HiAI_BandMode band_mode{HIAI_BANDMODE_UNSET}; 206+ void *quant_config; 207+ size_t quant_config_size = 0; 208+}; 209+ 210+class ExtensionOptionsParser { 211+public: 212+ static int Parse(const std::vector<Extension> &extensions, ExtensionOptions *param); 213+ 214+private: 215+ static void DoParseBondMode(const std::vector<Extension> &extensions, mindspore::lite::HiAI_BandMode *band_mode); 216+ static void DoParseQuantConfig(const std::vector<Extension> &extensions, void **quant_config, size_t *num); 217+ static void DoParseCachePath(const std::vector<Extension> &extensions, std::string *cache_path); 218+ static void DoParseCacheVersion(const std::vector<Extension> &extensions, uint32_t *cache_version); 219+}; 220+ 221+} // namespace mindspore::lite::nnrt 222+ 223+#endif // MINDSPORE_LITE_EXTENSION_OPTIONS_PARSER_H 224diff --git a/mindspore/lite/src/litert/delegate/nnrt/hiai_foundation_wrapper.cc b/mindspore/lite/src/litert/delegate/nnrt/hiai_foundation_wrapper.cc 225new file mode 100644 226index 00000000..e7a52827 227--- /dev/null 228+++ b/mindspore/lite/src/litert/delegate/nnrt/hiai_foundation_wrapper.cc 229@@ -0,0 +1,64 @@ 230+/** 231+ * Copyright 2024 Huawei Technologies Co., Ltd 232+ * 233+ * Licensed under the Apache License, Version 2.0 (the "License"); 234+ * you may not use this file except in compliance with the License. 235+ * You may obtain a copy of the License at 236+ * 237+ * http://www.apache.org/licenses/LICENSE-2.0 238+ * 239+ * Unless required by applicable law or agreed to in writing, software 240+ * distributed under the License is distributed on an "AS IS" BASIS, 241+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 242+ * See the License for the specific language governing permissions and 243+ * limitations under the License. 244+ */ 245+ 246+#include "hiai_foundation_wrapper.h" 247+#include "dlfcn.h" 248+#include "src/common/log.h" 249+ 250+namespace mindspore::lite { 251+static const char *HIAI_F_LIB = "libhiai_foundation.so"; 252+ 253+bool UnLoadHiaiFLibrary(void *handle) { 254+ if (handle != nullptr) { 255+ if (dlclose(handle) != 0) { 256+ MS_LOG(WARNING) << "dlclose failed, error: " << dlerror(); 257+ return false; 258+ } 259+ return true; 260+ } 261+ return true; 262+} 263+ 264+bool LoadHiaiFLibraryFromPath(void **handle_ptr) { 265+ if (handle_ptr == nullptr) { 266+ return false; 267+ } 268+ 269+ *handle_ptr = dlopen(HIAI_F_LIB, RTLD_NOW | RTLD_LOCAL); 270+ if (*handle_ptr == nullptr) { 271+ return false; 272+ } 273+ 274+// load function ptr use dlopen and dlsym. 275+#define LOAD_HIAIF_FUNCTION_PTR(func_name) \ 276+ func_name = reinterpret_cast<func_name##Func>(dlsym(*handle_ptr, #func_name)); \ 277+ if (func_name == nullptr) { \ 278+ MS_LOG(ERROR) << "load func (" << #func_name << ") from (" << HIAI_F_LIB << ") failed!"; \ 279+ UnLoadHiaiFLibrary(*handle_ptr); \ 280+ return false; \ 281+ } 282+ 283+ LOAD_HIAIF_FUNCTION_PTR(HMS_HiAIOptions_SetQuantConfig); 284+ LOAD_HIAIF_FUNCTION_PTR(HMS_HiAIOptions_SetBandMode); 285+ return true; 286+} 287+ 288+#define HIAIF_DEFINE_FUNC_PTR(func) func##Func func = nullptr 289+HIAIF_DEFINE_FUNC_PTR(HMS_HiAIOptions_SetQuantConfig); 290+HIAIF_DEFINE_FUNC_PTR(HMS_HiAIOptions_SetBandMode); 291+ 292+#undef LOAD_HIAIF_FUNCTION_PTR 293+} // mindspore::lite 294\ No newline at end of file 295diff --git a/mindspore/lite/src/litert/delegate/nnrt/hiai_foundation_wrapper.h b/mindspore/lite/src/litert/delegate/nnrt/hiai_foundation_wrapper.h 296new file mode 100644 297index 00000000..9231940d 298--- /dev/null 299+++ b/mindspore/lite/src/litert/delegate/nnrt/hiai_foundation_wrapper.h 300@@ -0,0 +1,47 @@ 301+/** 302+ * Copyright 2024 Huawei Technologies Co., Ltd 303+ * 304+ * Licensed under the Apache License, Version 2.0 (the "License"); 305+ * you may not use this file except in compliance with the License. 306+ * You may obtain a copy of the License at 307+ * 308+ * http://www.apache.org/licenses/LICENSE-2.0 309+ * 310+ * Unless required by applicable law or agreed to in writing, software 311+ * distributed under the License is distributed on an "AS IS" BASIS, 312+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 313+ * See the License for the specific language governing permissions and 314+ * limitations under the License. 315+ */ 316+ 317+#ifndef LITE_HIAI_FOUNDATION_WRAPPER_H 318+#define LITE_HIAI_FOUNDATION_WRAPPER_H 319+ 320+#include <string> 321+#include "interfaces/kits/c/neural_network_runtime/neural_network_runtime.h" 322+ 323+namespace mindspore::lite { 324+bool LoadHiaiFLibraryFromPath(void **handle_ptr); 325+bool UnLoadHiaiFLibrary(void *handle); 326+ 327+typedef enum { 328+ /** Automatically adjusted by the system. */ 329+ HIAI_BANDMODE_UNSET = 0, 330+ /** Low bandwidth mode. */ 331+ HIAI_BANDMODE_LOW = 1, 332+ /** Medium bandwidth mode. */ 333+ HIAI_BANDMODE_NORMAL = 2, 334+ /** High bandwidth mode. */ 335+ HIAI_BANDMODE_HIGH = 3, 336+} HiAI_BandMode; 337+ 338+using HMS_HiAIOptions_SetQuantConfigFunc = OH_NN_ReturnCode (*)(OH_NNCompilation*, void*, size_t); 339+using HMS_HiAIOptions_SetBandModeFunc = OH_NN_ReturnCode (*)(OH_NNCompilation*, HiAI_BandMode); 340+ 341+#define HIAIF_DECLARE_FUNC_PTR(func) extern func##Func func 342+HIAIF_DECLARE_FUNC_PTR(HMS_HiAIOptions_SetQuantConfig); 343+HIAIF_DECLARE_FUNC_PTR(HMS_HiAIOptions_SetBandMode); 344+#undef HIAIF_DECLARE_FUNC_PTR 345+} // mindspore::lite 346+ 347+#endif // LITE_HIAI_FOUNDATION_WRAPPER_H 348diff --git a/mindspore/lite/src/litert/delegate/nnrt/nnrt_delegate.cc b/mindspore/lite/src/litert/delegate/nnrt/nnrt_delegate.cc 349index a949c910..17abd0ed 100644 350--- a/mindspore/lite/src/litert/delegate/nnrt/nnrt_delegate.cc 351+++ b/mindspore/lite/src/litert/delegate/nnrt/nnrt_delegate.cc 352@@ -29,26 +29,20 @@ 353 354 namespace mindspore { 355 namespace lite { 356-void NNRTDelegate::InitCachePath() { 357- static const std::string kCachePathName = "CachePath"; 358- static const std::string kCacheVersion = "CacheVersion"; 359- 360- const auto &extensions = nnrt_device_info_.extensions_; 361- 362- auto iter_path = std::find_if(extensions.begin(), extensions.end(), [](const Extension &extension) { 363- return extension.name == kCachePathName; 364- }); 365- if (iter_path != extensions.end()) { 366- cache_path_ = std::string(iter_path->value.begin(), iter_path->value.end()); 367+Status NNRTDelegate::Init() { 368+#ifdef SUPPORT_NNRT_METAGRAPH 369+ auto ret = mindspore::lite::LoadHiaiFLibraryFromPath(&hiai_handle_); 370+ if (!ret || hiai_handle_ == nullptr) { 371+ MS_LOG(ERROR) << "Load HiAI_Foundation so failed."; 372+ return kLiteError; 373 } 374+#endif 375+ return kSuccess; 376+} 377 378- auto iter_version = std::find_if(extensions.begin(), extensions.end(), [](const Extension &extension) { 379- return extension.name == kCacheVersion; 380- }); 381- if (iter_version != extensions.end()) { 382- std::string version_str = std::string(iter_version->value.begin(), iter_version->value.end()); 383- cache_version_ = static_cast<uint32_t>(std::atol(version_str.c_str())); 384- } 385+void NNRTDelegate::InitExtensionOptions() { 386+ const auto &extensions = nnrt_device_info_.extensions_; 387+ mindspore::lite::nnrt::ExtensionOptionsParser::Parse(extensions, &extension_options_); 388 } 389 390 Status NNRTDelegate::Build(DelegateModel<schema::Primitive> *model) { 391@@ -59,11 +53,15 @@ Status NNRTDelegate::Build(DelegateModel<schema::Primitive> *model) { 392 return kLiteError; 393 } 394 #ifdef SUPPORT_NNRT_METAGRAPH 395- if (IsKirinNPU()) { 396- MS_LOG(DEBUG) << "Choose to build nnrt model with Metagraph"; 397- InitCachePath(); 398+ InitExtensionOptions(); 399+ if (IsKirinNPUWithOnlineInference()) { 400+ MS_LOG(DEBUG) << "Choose to build online inference model"; 401 return BuildKirinNPUModel(model); 402 } 403+ if (IsKirinNPUWithOfflineInference()) { 404+ MS_LOG(DEBUG) << "Choose to build offline inference model"; 405+ return BuildOfflineModel(model); 406+ } 407 #endif 408 409 return BuildNormalModel(model); 410@@ -88,8 +86,8 @@ bool NNRTDelegate::IsCustomModel() const { 411 } 412 413 #ifdef SUPPORT_NNRT_METAGRAPH 414-bool NNRTDelegate::IsKirinNPU() const { 415- const std::string kirin_npu_name_prefix = "NPU_"; 416+bool NNRTDelegate::CheckNPUPrefix(const std::string prefix_name) const { 417+ const std::string kirin_npu_name_prefix = prefix_name; 418 auto device_id = nnrt_device_info_.device_id_; 419 const char *device_name; 420 auto ret = OH_NNDevice_GetName(device_id, &device_name); 421@@ -105,6 +103,14 @@ bool NNRTDelegate::IsKirinNPU() const { 422 return true; 423 } 424 425+bool NNRTDelegate::IsKirinNPUWithOnlineInference() const { 426+ return CheckNPUPrefix("NPU_"); 427+} 428+ 429+bool NNRTDelegate::IsKirinNPUWithOfflineInference() const { 430+ return CheckNPUPrefix("HIAI_F"); 431+} 432+ 433 Status NNRTDelegate::BuildKirinNPUModel(DelegateModel<schema::Primitive> *model) { 434 OH_NNModel *nn_model = OH_NNModel_Construct(); 435 if (nn_model == nullptr) { 436@@ -142,6 +148,64 @@ Status NNRTDelegate::BuildKirinNPUModel(DelegateModel<schema::Primitive> *model) 437 return kSuccess; 438 } 439 440+namespace { 441+constexpr int32_t kNum2 = 2; 442+} 443+ 444+Status NNRTDelegate::BuildOfflineModel(DelegateModel<schema::Primitive> *model) { 445+ if (!IsCustomModel()) { 446+ MS_LOG(ERROR) << "not third party model"; 447+ return kLiteNullptr; 448+ } 449+ 450+ auto node = lite_graph_->all_nodes_[0]; 451+ MS_CHECK_TRUE_RET(node != nullptr, kLiteError); 452+ auto input_num = node->input_indices_.size(); 453+ // at least one input and one OM model buffer(as the last constant input) 454+ MS_CHECK_TRUE_RET(input_num >= kNum2, kLiteError); 455+ MS_CHECK_TRUE_RET(lite_graph_->all_tensors_.size() >= kNum2, kLiteError); 456+ auto tensor = lite_graph_->all_tensors_[node->input_indices_[input_num - 1]]; 457+ MS_CHECK_TRUE_RET(tensor != nullptr, kLiteError); 458+ MS_CHECK_TRUE_RET(tensor->data() != nullptr, kLiteError); 459+ const uint8_t *model_buf = static_cast<const uint8_t *>(tensor->data()->data()); 460+ size_t model_size = tensor->data()->size(); 461+ 462+ OH_NNCompilation *nn_compilation = OH_NNCompilation_ConstructWithOfflineModelBuffer(model_buf, model_size); 463+ if (nn_compilation == nullptr) { 464+ MS_LOG(ERROR) << "Construct Offline NNCompilation failed"; 465+ return kLiteError; 466+ } 467+ MS_LOG(DEBUG) << "NNRTDelegate creates NNCompilation success."; 468+ 469+ auto ret_code = InitNNCompilation(nn_compilation); 470+ if (ret_code != kSuccess) { 471+ MS_LOG(ERROR) << "Init NNCompilation failed"; 472+ OH_NNCompilation_Destroy(&nn_compilation); 473+ return kLiteError; 474+ } 475+ MS_LOG(DEBUG) << "HiAI F InitNNCompilation success"; 476+ 477+ OH_NNExecutor *nn_executor = nullptr; 478+ nn_executor = OH_NNExecutor_Construct(nn_compilation); 479+ if (nn_executor == nullptr) { 480+ MS_LOG(ERROR) << "Construct NNExecutor failed, ret: " << ret_code; 481+ OH_NNCompilation_Destroy(&nn_compilation); 482+ return kLiteError; 483+ } 484+ OH_NNCompilation_Destroy(&nn_compilation); 485+ 486+ auto nnrt_model_kernel = new (std::nothrow) NNRTModelKernel(nn_executor, nnrt_device_info_.device_id_, model->inputs(), model->outputs()); 487+ if (nnrt_model_kernel == nullptr) { 488+ OH_NNExecutor_Destroy(&nn_executor); 489+ MS_LOG(ERROR) << "new NNRTModelKernel failed"; 490+ return kLiteError; 491+ } 492+ nn_executor_list_.push_back(nn_executor); 493+ 494+ (void)model->Replace(model->BeginKernelIterator(), model->EndKernelIterator(), nnrt_model_kernel); 495+ return kSuccess; 496+} 497+ 498 Status NNRTDelegate::CreateFullModelKernel(DelegateModel<schema::Primitive> *model, OH_NNModel *nn_model) { 499 OH_NNCompilation *nn_compilation = OH_NNCompilation_Construct(nn_model); 500 if (nn_compilation == nullptr) { 501@@ -473,14 +537,33 @@ Status NNRTDelegate::InitNNCompilation(OH_NNCompilation *nn_compilation) const { 502 return kLiteError; 503 } 504 505- if (!cache_path_.empty()) { // Set cache path if user indeed set it. 506- ret_code = OH_NNCompilation_SetCache(nn_compilation, cache_path_.c_str(), cache_version_); 507+ if (!extension_options_.cache_path_.empty()) { // Set cache path if user indeed set it. 508+ ret_code = OH_NNCompilation_SetCache(nn_compilation, extension_options_.cache_path_.c_str(), 509+ extension_options_.cache_version_); 510 if ((ret_code != OH_NN_SUCCESS) && (ret_code != OH_NN_OPERATION_FORBIDDEN)) { 511 MS_LOG(ERROR) << "NNCompilation set cache failed, ret: " << ret_code; 512 return kLiteError; 513 } 514 } 515 516+#ifdef SUPPORT_NNRT_METAGRAPH 517+ ret_code = mindspore::lite::HMS_HiAIOptions_SetBandMode(nn_compilation, extension_options_.band_mode); 518+ if ((ret_code != OH_NN_SUCCESS) && (ret_code != OH_NN_OPERATION_FORBIDDEN)) { 519+ MS_LOG(ERROR) << "NNCompilation set BandMode failed, ret: " << ret_code; 520+ return kLiteError; 521+ } 522+ 523+ if (extension_options_.quant_config != nullptr && extension_options_.quant_config_size != 0) { 524+ ret_code = mindspore::lite::HMS_HiAIOptions_SetQuantConfig(nn_compilation, 525+ extension_options_.quant_config, 526+ extension_options_.quant_config_size); 527+ if ((ret_code != OH_NN_SUCCESS) && (ret_code != OH_NN_OPERATION_FORBIDDEN)) { 528+ MS_LOG(ERROR) << "NNCompilation set QuantConfig failed, ret: " << ret_code; 529+ return kLiteError; 530+ } 531+ } 532+#endif 533+ 534 ret_code = OH_NNCompilation_Build(nn_compilation); 535 if (ret_code != OH_NN_SUCCESS) { 536 MS_LOG(ERROR) << "Build NNCompilation failed, ret: " << ret_code; 537diff --git a/mindspore/lite/src/litert/delegate/nnrt/nnrt_delegate.h b/mindspore/lite/src/litert/delegate/nnrt/nnrt_delegate.h 538index db2f0ee7..c1adc9f0 100644 539--- a/mindspore/lite/src/litert/delegate/nnrt/nnrt_delegate.h 540+++ b/mindspore/lite/src/litert/delegate/nnrt/nnrt_delegate.h 541@@ -22,6 +22,8 @@ 542 #include "include/model.h" 543 #include "src/litert/inner_context.h" 544 #include "nnrt_model_kernel.h" 545+#include "hiai_foundation_wrapper.h" 546+#include "extension_options_parser.h" 547 #include "schema/model_generated.h" 548 #include "interfaces/kits/c/neural_network_runtime/neural_network_runtime_type.h" 549 #include "interfaces/kits/c/neural_network_runtime/neural_network_runtime.h" 550@@ -43,7 +45,7 @@ class NNRTDelegate : public Delegate { 551 NNRTDelegate() = default; 552 NNRTDelegate(const NNRtDeviceInfo &nnrt_device_info) : nnrt_device_info_(nnrt_device_info) {} 553 ~NNRTDelegate() override; 554- Status Init() override { return kSuccess; } 555+ Status Init() override; 556 Status Build(DelegateModel<schema::Primitive> *model) override; 557 void ShallowCopyLiteGraph(const lite::LiteGraph &liteGraph); 558 void FreeLiteGraph(lite::LiteGraph **liteGraph); 559@@ -57,7 +59,7 @@ class NNRTDelegate : public Delegate { 560 const std::vector<bool> &op_supports); 561 562 private: 563- void InitCachePath(); 564+ void InitExtensionOptions(); 565 Status BuildNormalModel(DelegateModel<schema::Primitive> *model); 566 OH_NNModel *CreateFullNNModel(); 567 std::vector<bool> QueryOpSupports(OH_NNModel *nn_model); 568@@ -82,21 +84,24 @@ class NNRTDelegate : public Delegate { 569 schema::Tensor *TensorToSchemaTensor(Tensor *lite_tensor, schema::Tensor *schema_tensor); 570 571 #ifdef SUPPORT_NNRT_METAGRAPH 572- bool IsKirinNPU() const; 573+ bool CheckNPUPrefix(const std::string prefix_name) const; 574+ bool IsKirinNPUWithOnlineInference() const; 575+ bool IsKirinNPUWithOfflineInference() const; 576 Status BuildKirinNPUModel(DelegateModel<schema::Primitive> *model); 577+ Status BuildOfflineModel(DelegateModel<schema::Primitive> *model); 578 Status CreateFullModelKernel(DelegateModel<schema::Primitive> *model, OH_NNModel *nn_model); 579 #endif 580 581 NNRtDeviceInfo nnrt_device_info_; 582 LiteGraph *lite_graph_ = nullptr; 583 const void *meta_graph_ = nullptr; 584- std::string cache_path_ = ""; 585- uint32_t cache_version_ = 0; 586+ nnrt::ExtensionOptions extension_options_; 587 std::vector<OH_NNExecutor *> nn_executor_list_; 588 std::vector<Tensor *> *dequant_src_tensors_; 589 std::map<uint32_t, schema::Tensor *> dequant_schema_tensors_; 590 std::map<schema::Tensor *, void *> dequant_schema_tensors_buffer_map_; 591 std::vector<schema::Tensor *> replaced_schema_tensors_; 592+ void *hiai_handle_{nullptr}; 593 }; 594 } // namespace lite 595 } // namespace mindspore 596-- 5972.17.1 598 599