1From 17b5a08e7c53b25a912bcc9d65cc47febb7fc835 Mon Sep 17 00:00:00 2001 2From: zhangyanhui <zhangyanhui17@huawei.com> 3Date: Tue, 30 Jan 2024 16:23:22 +0800 4Subject: [PATCH] 0010-micro 5 6--- 7 .../lite/include/registry/deobf_processor.h | 2 +- 8 .../config_parser/config_file_parser.cc | 8 +-- 9 .../config_parser/config_file_parser.h | 2 +- 10 .../config_parser/micro_param_parser.cc | 8 +-- 11 .../config_parser/micro_param_parser.h | 3 +- 12 mindspore/lite/tools/converter/converter.cc | 10 ++-- 13 .../lite/tools/converter/micro/coder/coder.cc | 1 + 14 .../lite/tools/converter/micro/coder/config.h | 9 ++++ 15 .../generator/component/common_component.cc | 49 ++++++++++++++++--- 16 .../component/const_blocks/calib_output.cc | 19 +++++++ 17 .../component/const_blocks/mtensor.cc | 2 +- 18 11 files changed, 90 insertions(+), 23 deletions(-) 19 20diff --git a/mindspore/lite/include/registry/deobf_processor.h b/mindspore/lite/include/registry/deobf_processor.h 21index 5987233f..c70735b2 100644 22--- a/mindspore/lite/include/registry/deobf_processor.h 23+++ b/mindspore/lite/include/registry/deobf_processor.h 24@@ -21,7 +21,7 @@ 25 #include <string> 26 #include <numeric> 27 #include "src/common/prim_util.h" 28-#include "src/common/log.h" 29+#include "src/common/log_adapter.h" 30 #include "include/model.h" 31 #include "schema/inner/model_generated.h" 32 33diff --git a/mindspore/lite/tools/converter/config_parser/config_file_parser.cc b/mindspore/lite/tools/converter/config_parser/config_file_parser.cc 34index 7b47fb8c..eed1c1ef 100644 35--- a/mindspore/lite/tools/converter/config_parser/config_file_parser.cc 36+++ b/mindspore/lite/tools/converter/config_parser/config_file_parser.cc 37@@ -230,7 +230,7 @@ int ParseInputShapeTemplate(const std::string &shape_template, std::set<std::str 38 return RET_OK; 39 } 40 41-int ParseDynmiacDimTemplate(const std::string &dims_template, std::set<std::string> *dynamic_symbols, 42+int ParseDynamicDimTemplate(const std::string &dims_template, std::set<std::string> *dynamic_symbols, 43 MicroParamString *micro_param_string) { 44 // the dynamic_dim_params config is like: d0:[1,3~6];d1:[1~8] 45 auto dim_info_vec = SplitStringToVector(dims_template, ';'); 46@@ -259,7 +259,7 @@ int ParseDynmiacDimTemplate(const std::string &dims_template, std::set<std::stri 47 MS_LOG(ERROR) << "the dynamic_dim_params range value must be greater than 0"; 48 return RET_INPUT_PARAM_INVALID; 49 } 50- micro_param_string->dynamic_symbols_map[symbol] += continuous_dim[0] + ","; 51+ micro_param_string->dynamic_symbols_map[symbol].emplace_back(std::stoi(continuous_dim[0])); 52 continue; 53 } 54 if (!IsNumber(continuous_dim[0]) || std::stoi(continuous_dim[0]) <= 0 || !IsNumber(continuous_dim[1]) || 55@@ -270,7 +270,7 @@ int ParseDynmiacDimTemplate(const std::string &dims_template, std::set<std::stri 56 auto start = std::stoi(continuous_dim[0]); 57 auto end = std::stoi(continuous_dim[1]); 58 for (auto i = start; i <= end; ++i) { 59- micro_param_string->dynamic_symbols_map[symbol] += std::to_string(i) + ","; 60+ micro_param_string->dynamic_symbols_map[symbol].emplace_back(i); 61 } 62 } 63 } 64@@ -597,7 +597,7 @@ int ConfigFileParser::ParseMicroParamString(const std::map<std::string, std::map 65 const std::string dynamic_dims = "dynamic_dim_params"; 66 if (!dynamic_symbols.empty() && map.find(dynamic_dims) != map.end()) { 67 const auto &dims_template = map.at(dynamic_dims); 68- ParseDynmiacDimTemplate(dims_template, &dynamic_symbols, µ_param_string_); 69+ ParseDynamicDimTemplate(dims_template, &dynamic_symbols, µ_param_string_); 70 } 71 std::map<std::string, std::string &> parse_map{ 72 {"target", micro_param_string_.target}, 73diff --git a/mindspore/lite/tools/converter/config_parser/config_file_parser.h b/mindspore/lite/tools/converter/config_parser/config_file_parser.h 74index 163782b7..b6267c0c 100644 75--- a/mindspore/lite/tools/converter/config_parser/config_file_parser.h 76+++ b/mindspore/lite/tools/converter/config_parser/config_file_parser.h 77@@ -110,7 +110,7 @@ struct MicroParamString { 78 std::string changeable_weights_name; 79 std::string inputs_shape; 80 std::string dynamic_dim_params; 81- std::map<std::string, std::string> dynamic_symbols_map; 82+ std::map<std::string, std::vector<int>> dynamic_symbols_map; 83 }; 84 85 struct ThirdPartyModelString { 86diff --git a/mindspore/lite/tools/converter/config_parser/micro_param_parser.cc b/mindspore/lite/tools/converter/config_parser/micro_param_parser.cc 87index 903f2863..da5b46eb 100644 88--- a/mindspore/lite/tools/converter/config_parser/micro_param_parser.cc 89+++ b/mindspore/lite/tools/converter/config_parser/micro_param_parser.cc 90@@ -117,7 +117,7 @@ STATUS MicroParamParser::ParseChangeableWeightsName(const std::string &changeabl 91 } 92 93 STATUS MicroParamParser::ParseGraphInputsShapeTemplate(const std::string &graph_inputs_shape_template, 94- const std::map<std::string, std::string> &dynamic_symbols_map, 95+ const std::map<std::string, std::vector<int>> &dynamic_symbols_map, 96 micro::MicroParam *micro_param) { 97 MS_LOG(DEBUG) << "Micro record inputs shape: " << graph_inputs_shape_template; 98 if (!graph_inputs_shape_template.empty()) { 99@@ -145,9 +145,8 @@ STATUS MicroParamParser::ParseGraphInputsShapeTemplate(const std::string &graph_ 100 for (const auto &item : dynamic_symbols_map) { 101 symbols_index[item.first] = index++; 102 symbols.push_back(item.first); 103- auto num_str_list = SplitStringToVector(item.second, ','); 104- for (const auto &num_str : num_str_list) { 105- symbols_to_num[item.first].push_back(std::stoi(num_str)); 106+ for (const auto &num : item.second) { 107+ symbols_to_num[item.first].push_back(num); 108 } 109 if (symbols_to_num[item.first].empty()) { 110 MS_LOG(ERROR) << "Micro param invalid, dynamic symbol must have value."; 111@@ -158,6 +157,7 @@ STATUS MicroParamParser::ParseGraphInputsShapeTemplate(const std::string &graph_ 112 } 113 micro_param->dynamic_symbols = symbols; 114 micro_param->dynamic_symbols_num = scene_num_by_symbol; 115+ micro_param->dynamic_symbols_map = dynamic_symbols_map; 116 std::vector<size_t> post_multi(symbols.size(), 1); 117 for (int i = static_cast<int>(post_multi.size()) - 2; i >= 0; --i) { 118 post_multi[i] = post_multi[i + 1] * scene_num_by_symbol[i + 1]; 119diff --git a/mindspore/lite/tools/converter/config_parser/micro_param_parser.h b/mindspore/lite/tools/converter/config_parser/micro_param_parser.h 120index eb95c571..fe818b90 100644 121--- a/mindspore/lite/tools/converter/config_parser/micro_param_parser.h 122+++ b/mindspore/lite/tools/converter/config_parser/micro_param_parser.h 123@@ -17,6 +17,7 @@ 124 #define MINDSPORE_LITE_TOOLS_CONVERTER_CONFIG_PARSER_MICRO_PARAM_PARSER_H_ 125 126 #include <string> 127+#include <vector> 128 #include "tools/converter/config_parser/config_file_parser.h" 129 #include "tools/converter/micro/coder/config.h" 130 #include "include/errorcode.h" 131@@ -38,7 +39,7 @@ class MicroParamParser { 132 STATUS ParseKeepOriginalWeight(const std::string &keep_weight, micro::MicroParam *micro_param); 133 STATUS ParseChangeableWeightsName(const std::string &changeable_weights_name, micro::MicroParam *micro_param); 134 STATUS ParseGraphInputsShapeTemplate(const std::string &graph_inputs_shape_template, 135- const std::map<std::string, std::string> &dynamic_symbols_map, 136+ const std::map<std::string, std::vector<int>> &dynamic_symbols_map, 137 micro::MicroParam *micro_param); 138 }; 139 } // namespace lite 140diff --git a/mindspore/lite/tools/converter/converter.cc b/mindspore/lite/tools/converter/converter.cc 141index 4703e889..e2a5e6a1 100644 142--- a/mindspore/lite/tools/converter/converter.cc 143+++ b/mindspore/lite/tools/converter/converter.cc 144@@ -455,10 +455,12 @@ int ConverterImpl::InitConfigParam(const std::shared_ptr<ConverterPara> ¶m, 145 MS_LOG(ERROR) << "Parse mixed bit weight quant param failed."; 146 return ret; 147 } 148- ret = lite::ThirdPartyParamParser::Parse(config_parser.GetThirdPartyModelString(), ¶m->thirdPartyModelParam); 149- if (ret != RET_OK) { 150- MS_LOG(ERROR) << "Parse third party param failed."; 151- return ret; 152+ if (param->fmk_type == converter::kFmkTypeThirdParty) { 153+ ret = lite::ThirdPartyParamParser::Parse(config_parser.GetThirdPartyModelString(), ¶m->thirdPartyModelParam); 154+ if (ret != RET_OK) { 155+ MS_LOG(ERROR) << "Parse third party param failed."; 156+ return ret; 157+ } 158 } 159 ret = InitExtendedIntegrationInfo(param, config_parser); 160 if (ret != RET_OK) { 161diff --git a/mindspore/lite/tools/converter/micro/coder/coder.cc b/mindspore/lite/tools/converter/micro/coder/coder.cc 162index a502500d..065cbdfd 100644 163--- a/mindspore/lite/tools/converter/micro/coder/coder.cc 164+++ b/mindspore/lite/tools/converter/micro/coder/coder.cc 165@@ -238,6 +238,7 @@ int Coder::Init(const MicroParam ¶m) const { 166 config->set_graph_inputs_shape_infos(param.graph_inputs_shape_infos); 167 config->set_dynamic_symbols(param.dynamic_symbols); 168 config->set_dynamic_symbols_num(param.dynamic_symbols_num); 169+ config->set_dynamic_symbols_map(param.dynamic_symbols_map); 170 config->set_user_graph_inputs_template(param.graph_inputs_template); 171 172 auto print_parameter = [](auto name, auto value) { 173diff --git a/mindspore/lite/tools/converter/micro/coder/config.h b/mindspore/lite/tools/converter/micro/coder/config.h 174index fb90a2fc..3afea051 100644 175--- a/mindspore/lite/tools/converter/micro/coder/config.h 176+++ b/mindspore/lite/tools/converter/micro/coder/config.h 177@@ -18,6 +18,8 @@ 178 #define MINDSPORE_LITE_TOOLS_CONVERTER_MICRO_CODER_CONFIG_H_ 179 180 #include <string> 181+#include <map> 182+#include <vector> 183 184 namespace mindspore::lite::micro { 185 enum Target { kX86 = 0, kCortex_M = 1, kARM32 = 2, kARM64 = 3, kAllTargets = 4, kTargetUnknown = 99 }; 186@@ -38,6 +40,7 @@ struct MicroParam { 187 std::map<std::string, std::vector<std::string>> graph_inputs_origin_info; 188 std::vector<std::string> dynamic_symbols; 189 std::vector<size_t> dynamic_symbols_num; 190+ std::map<std::string, std::vector<int>> dynamic_symbols_map; 191 std::vector<std::vector<std::vector<int>>> graph_inputs_shape_infos; 192 std::map<std::string, std::vector<std::vector<int>>> inputs_shape_by_scenes; 193 }; 194@@ -84,6 +87,11 @@ class Configurator { 195 } 196 const std::vector<size_t> &dynamic_symbols_num() const { return dynamic_symbols_num_; } 197 198+ void set_dynamic_symbols_map(const std::map<std::string, std::vector<int>> &dynamic_symbols_map) { 199+ dynamic_symbols_map_ = dynamic_symbols_map; 200+ } 201+ const std::map<std::string, std::vector<int>> &dynamic_symbols_map() const { return dynamic_symbols_map_; } 202+ 203 void set_user_graph_inputs_template(const std::vector<std::vector<std::string>> &graph_inputs_template) { 204 user_graph_inputs_template_ = graph_inputs_template; 205 } 206@@ -110,6 +118,7 @@ class Configurator { 207 std::string changeable_weights_name_; 208 std::vector<std::string> dynamic_symbols_; 209 std::vector<size_t> dynamic_symbols_num_; 210+ std::map<std::string, std::vector<int>> dynamic_symbols_map_; 211 std::vector<std::vector<std::vector<int>>> graph_inputs_shape_infos_; 212 std::vector<std::vector<std::string>> user_graph_inputs_template_; 213 }; 214diff --git a/mindspore/lite/tools/converter/micro/coder/generator/component/common_component.cc b/mindspore/lite/tools/converter/micro/coder/generator/component/common_component.cc 215index 62c2f668..b9ab846e 100644 216--- a/mindspore/lite/tools/converter/micro/coder/generator/component/common_component.cc 217+++ b/mindspore/lite/tools/converter/micro/coder/generator/component/common_component.cc 218@@ -492,6 +492,19 @@ void CodeMSModelResize(std::ofstream &ofs, const std::unique_ptr<CoderContext> & 219 for (auto &item : symbol_to_indexes) { 220 ofs << " int dim" << index << " = shape_infos[" << item.second[0] << "].shape[" << item.second[1] << "];\n"; 221 inner_to_outer[item.first] = "dim" + std::to_string(index); 222+ std::string cur_dim_symbol; 223+ for (std::map<std::string, std::string>::iterator it = user_to_inner.begin(); it != user_to_inner.end(); it++) { 224+ if (it->second == item.first) { 225+ cur_dim_symbol = it->first; 226+ break; 227+ } 228+ } 229+ auto dynamic_dim_range = config.dynamic_symbols_map().at(cur_dim_symbol); 230+ ofs << " int dim" << index << "_range[" << dynamic_dim_range.size() << "] = {"; 231+ for (const auto dim : dynamic_dim_range) { 232+ ofs << dim << ", "; 233+ } 234+ ofs << "};\n"; 235 ++index; 236 } 237 std::string condition; 238@@ -508,18 +521,40 @@ void CodeMSModelResize(std::ofstream &ofs, const std::unique_ptr<CoderContext> & 239 for (size_t i = 0; i < symbol_to_indexes.size(); ++i) { 240 ofs << " store" + std::to_string(ctx->GetCurModelIndex()) + "_" << i << " = dim" << i << ";\n"; 241 } 242+ auto &dynamic_symbols = config.dynamic_symbols(); 243+ int id = 0; 244+ for (auto &symbol : dynamic_symbols) { 245+ auto cur_dim = inner_to_outer[user_to_inner[symbol]]; 246+ auto dim_list = cur_dim + "_range"; 247+ ofs << " int index" << id << " = 0;\n"; 248+ ofs << " for (int i = 0; i < sizeof(" << dim_list << ") / sizeof(" << dim_list << "[0]); i++) {\n" 249+ << " if (" << dim_list << "[i] == " << cur_dim << ") {\n" 250+ << " index" << id << " = i;\n" 251+ << " break;\n" 252+ << " }\n" 253+ << " }\n"; 254+ id++; 255+ } 256 ofs << " if (" << kBufferPrefixName << " != NULL) {\n"; 257 ofs << " free(" << kBufferPrefixName << ");\n"; 258+ ofs << " " << kBufferPrefixName << " = NULL;\n"; 259 ofs << " }\n"; 260- std::string real_array_index; 261- auto &dynamic_symbols = config.dynamic_symbols(); 262- for (auto &symbol : dynamic_symbols) { 263- real_array_index += "[" + inner_to_outer[user_to_inner[symbol]] + " - 1]"; 264+ ofs << " " << kBufferPrefixName << " = malloc(buffer_sizes"; 265+ for (size_t i = 0; i < dynamic_symbols.size(); i++) { 266+ ofs << "[index" << i << "]"; 267 } 268- ofs << " " << kBufferPrefixName << " = malloc(buffer_sizes" << real_array_index << ");\n"; 269+ ofs << ");\n"; 270 ofs << " micro_model->runtime_buffer = " << kBufferPrefixName << ";\n"; 271- ofs << " " << kShapePrefixName << " = &shapes" << real_array_index << "[0];\n"; 272- ofs << " " << kOffsetPrefixName << " = &offsets" << real_array_index << "[0];\n"; 273+ ofs << " " << kShapePrefixName << " = &shapes"; 274+ for (size_t i = 0; i < dynamic_symbols.size(); i++) { 275+ ofs << "[index" << i << "]"; 276+ } 277+ ofs << "[0];\n"; 278+ ofs << " " << kOffsetPrefixName << " = &offsets"; 279+ for (size_t i = 0; i < dynamic_symbols.size(); i++) { 280+ ofs << "[index" << i << "]"; 281+ } 282+ ofs << "[0];\n"; 283 ofs << " OH_AI_TensorHandleArray outputs = OH_AI_ModelGetOutputs(model);\n"; 284 for (size_t i = 0; i < ctx->graph_outputs().size(); ++i) { 285 ofs << " OH_AI_TensorSetData(outputs.handle_list[" << i << "], NULL);\n"; 286diff --git a/mindspore/lite/tools/converter/micro/coder/generator/component/const_blocks/calib_output.cc b/mindspore/lite/tools/converter/micro/coder/generator/component/const_blocks/calib_output.cc 287index 66af9069..bd020d83 100644 288--- a/mindspore/lite/tools/converter/micro/coder/generator/component/const_blocks/calib_output.cc 289+++ b/mindspore/lite/tools/converter/micro/coder/generator/component/const_blocks/calib_output.cc 290@@ -80,6 +80,9 @@ const char *calib_source = R"RAW(/** 291 #include <stdlib.h> 292 #include <math.h> 293 #include <string.h> 294+#ifdef ENABLE_FP16 295+#include <arm_neon.h> 296+#endif 297 298 #define kToleranceVal 0.0001f 299 #define kMaxOutput 5 300@@ -196,6 +199,22 @@ int CompareOutputs(OH_AI_TensorHandleArray outputs, CalibTensor **calib_tensors, 301 } 302 break; 303 } 304+#ifdef ENABLE_FP16 305+ case OH_AI_DATATYPE_NUMBERTYPE_FLOAT16: { 306+ float16_t *float16_output = (float16_t *)output->data; 307+ for (size_t j = 0; j < elements; ++j) { 308+ if (isnan(float16_output[j]) || isinf(float16_output[j]) || isnan(calib[i].data_[j]) || 309+ isinf(calib[i].data_[j])) { 310+ printf("error, output data is nan or inf\n"); 311+ return OH_AI_STATUS_LITE_ERROR; 312+ } 313+ dot += float16_output[j] * calib[i].data_[j]; 314+ normx += float16_output[j] * float16_output[j]; 315+ normy += calib[i].data_[j] * calib[i].data_[j]; 316+ } 317+ break; 318+ } 319+#endif 320 case OH_AI_DATATYPE_NUMBERTYPE_INT8: { 321 int8_t *int_output = (int8_t *)output->data; 322 for (size_t j = 0; j < elements; ++j) { 323diff --git a/mindspore/lite/tools/converter/micro/coder/generator/component/const_blocks/mtensor.cc b/mindspore/lite/tools/converter/micro/coder/generator/component/const_blocks/mtensor.cc 324index e4581829..8d8e6768 100644 325--- a/mindspore/lite/tools/converter/micro/coder/generator/component/const_blocks/mtensor.cc 326+++ b/mindspore/lite/tools/converter/micro/coder/generator/component/const_blocks/mtensor.cc 327@@ -315,12 +315,12 @@ void *TransformInput(OH_AI_TensorHandle tensor, int expect_type, bool *type_chan 328 if (type_trans_mode == TypeTransMode_UNSUPPORT) { 329 return NULL; 330 } 331+#ifdef ENABLE_FP16 332 int shape_size = micro_tensor->ndim; 333 int num = 1; 334 for (int i = 0; i < shape_size; ++i) { 335 num *= micro_tensor->shape[i]; 336 } 337-#ifdef ENABLE_FP16 338 if (type_trans_mode == TypeTransMode_FP32_TO_FP16) { 339 void *expect_input_fp16 = (void *)malloc(DataTypeSize(expect_type) * num); 340 Fp32CastToFp16((float *)micro_tensor->data, (float16_t *)expect_input_fp16, num); 341-- 3422.25.1 343 344