1 /**
2 * Copyright 2021 Huawei Technologies Co., Ltd
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16 #include "debug/env_config_parser.h"
17 #include <algorithm>
18 #include <fstream>
19 #include "nlohmann/json.hpp"
20 #include "utils/log_adapter.h"
21 #include "debug/common.h"
22 #include "utils/ms_context.h"
23 #include "utils/convert_utils_base.h"
24
25 namespace {
26 #ifdef ENABLE_DUMP_IR
27 constexpr auto ENV_RDR_ENABLE = "MS_RDR_ENABLE";
28 constexpr auto ENV_RDR_PATH = "MS_RDR_PATH";
29 constexpr auto KEY_RDR_SETTINGS = "rdr";
30 constexpr auto KEY_PATH = "path";
31 constexpr auto KEY_ENABLE = "enable";
32 #endif
33 constexpr auto KEY_MEM_REUSE_SETTINGS = "sys";
34 constexpr auto KEY_MEM_REUSE = "mem_reuse";
35 } // namespace
36
37 namespace mindspore {
38 #ifdef ENABLE_DUMP_IR
GetRdrEnableFromEnv()39 std::optional<bool> GetRdrEnableFromEnv() {
40 // get environment variable to configure RDR
41 std::string env_enable_str = common::GetEnv(ENV_RDR_ENABLE);
42 if (!env_enable_str.empty()) {
43 (void)std::transform(env_enable_str.begin(), env_enable_str.end(), env_enable_str.begin(), ::tolower);
44 if (env_enable_str != "0" && env_enable_str != "1") {
45 MS_LOG(WARNING) << "The environment variable '" << ENV_RDR_ENABLE << "' should be 0 or 1.";
46 }
47 if (env_enable_str == "1") {
48 return true;
49 }
50 return false;
51 }
52 return std::nullopt;
53 }
54
GetRdrPathFromEnv()55 std::optional<std::string> GetRdrPathFromEnv() {
56 // get environment variable to configure RDR
57 std::string path = common::GetEnv(ENV_RDR_PATH);
58 if (!path.empty()) {
59 std::string err_msg = "RDR path parse from environment variable failed. Please check the settings about '" +
60 std::string(ENV_RDR_PATH) + "' in environment variables.";
61 if (!Common::IsPathValid(path, MAX_DIRECTORY_LENGTH, err_msg)) {
62 return std::string("");
63 }
64 return path;
65 }
66 return std::nullopt;
67 }
68 #endif
69
CheckJsonStringType(const nlohmann::json & content,const std::string & setting_key,const std::string & key) const70 bool EnvConfigParser::CheckJsonStringType(const nlohmann::json &content, const std::string &setting_key,
71 const std::string &key) const {
72 if (!content.is_string()) {
73 MS_LOG(WARNING) << "Json Parse Failed. The '" << key << "' in '" << setting_key << "' should be string."
74 << " Please check the config file '" << config_file_ << "' set by 'env_config_path' in context.";
75 return false;
76 }
77 return true;
78 }
79
CheckJsonKeyExist(const nlohmann::json & content,const std::string & setting_key,const std::string & key) const80 std::optional<nlohmann::detail::iter_impl<const nlohmann::json>> EnvConfigParser::CheckJsonKeyExist(
81 const nlohmann::json &content, const std::string &setting_key, const std::string &key) const {
82 auto iter = content.find(key);
83 if (iter == content.end()) {
84 MS_LOG(WARNING) << "Check json failed, '" << key << "' not found in '" << setting_key << "'."
85 << " Please check the config file '" << config_file_ << "' set by 'env_config_path' in context.";
86 return std::nullopt;
87 }
88 return iter;
89 }
90
GetIfstreamString(const std::ifstream & ifstream) const91 std::string EnvConfigParser::GetIfstreamString(const std::ifstream &ifstream) const {
92 std::stringstream buffer;
93 buffer << ifstream.rdbuf();
94 return buffer.str();
95 }
96
ParseFromEnv()97 void EnvConfigParser::ParseFromEnv() {
98 #ifdef ENABLE_DUMP_IR
99 // Get RDR seetings from environment variables
100 auto rdr_enable_env = GetRdrEnableFromEnv();
101 if (rdr_enable_env.has_value()) {
102 has_rdr_setting_ = true;
103 rdr_enabled_ = rdr_enable_env.value();
104 }
105 auto path_env = GetRdrPathFromEnv();
106 if (path_env.has_value()) {
107 has_rdr_setting_ = true;
108 std::string path = path_env.value();
109 if (!path.empty()) {
110 if (path.back() != '/') {
111 path += '/';
112 }
113 rdr_path_ = path;
114 }
115 }
116 #endif
117 }
118
ParseFromFile()119 void EnvConfigParser::ParseFromFile() {
120 auto context = MsContext::GetInstance();
121 MS_EXCEPTION_IF_NULL(context);
122 auto config_file = context->get_param<std::string>(MS_CTX_ENV_CONFIG_PATH);
123 if (config_file.empty()) {
124 MS_LOG(INFO) << "The 'env_config_path' in 'mindspore.context.set_context(env_config_path={path})' is empty.";
125 return;
126 }
127 config_file_ = config_file;
128 std::ifstream json_file(config_file_);
129 if (!json_file.is_open()) {
130 MS_LOG(WARNING) << "Env config file:" << config_file_ << " open failed."
131 << " Please check the config file '" << config_file_ << "' set by 'env_config_path' in context."
132 << ErrnoToString(errno);
133 return;
134 }
135
136 nlohmann::json j;
137 try {
138 json_file >> j;
139 } catch (nlohmann::json::parse_error &e) {
140 MS_LOG(WARNING) << "Env config json contents '" << GetIfstreamString(json_file) << "' in config file '"
141 << config_file_ << "' set by 'env_config_path' in context.";
142 return;
143 }
144
145 // convert json to string
146 std::stringstream ss;
147 ss << j;
148 std::string cfg = ss.str();
149 MS_LOG(INFO) << "Env config json:" << cfg;
150
151 #ifdef ENABLE_DUMP_IR
152 ParseRdrSetting(j);
153 #endif
154 ParseMemReuseSetting(j);
155
156 ConfigToString();
157 }
158
Parse()159 void EnvConfigParser::Parse() {
160 std::lock_guard<std::mutex> guard(lock_);
161 if (already_parsed_) {
162 return;
163 }
164 already_parsed_ = true;
165 ParseFromEnv();
166 ParseFromFile();
167 }
168
ParseMemReuseSetting(const nlohmann::json & content)169 void EnvConfigParser::ParseMemReuseSetting(const nlohmann::json &content) {
170 auto sys_setting = content.find(KEY_MEM_REUSE_SETTINGS);
171 if (sys_setting == content.end()) {
172 MS_LOG(INFO) << "The '" << KEY_MEM_REUSE_SETTINGS << "' isn't existed. Please check the config file '"
173 << config_file_ << "' set by 'env_config_path' in context.";
174 return;
175 }
176 auto sys_memreuse = CheckJsonKeyExist(*sys_setting, KEY_MEM_REUSE_SETTINGS, KEY_MEM_REUSE);
177 if (sys_memreuse.has_value()) {
178 ParseSysMemReuse(**sys_memreuse);
179 }
180 }
181
ParseSysMemReuse(const nlohmann::json & content)182 void EnvConfigParser::ParseSysMemReuse(const nlohmann::json &content) {
183 if (!content.is_boolean()) {
184 MS_LOG(INFO) << "the json object parses failed. 'enable' in " << KEY_MEM_REUSE_SETTINGS << " should be boolean."
185 << " Please check the config file '" << config_file_ << "' set by 'env_config_path' in context.";
186 return;
187 }
188 sys_memreuse_ = content;
189 }
190
191 #ifdef ENABLE_DUMP_IR
ParseRdrSetting(const nlohmann::json & content)192 void EnvConfigParser::ParseRdrSetting(const nlohmann::json &content) {
193 auto rdr_setting = content.find(KEY_RDR_SETTINGS);
194 if (rdr_setting == content.end()) {
195 MS_LOG(WARNING) << "The '" << KEY_RDR_SETTINGS << "' not exists. Please check the config file '" << config_file_
196 << "' set by 'env_config_path' in context.";
197 return;
198 }
199
200 has_rdr_setting_ = true;
201
202 auto rdr_enable = CheckJsonKeyExist(*rdr_setting, KEY_RDR_SETTINGS, KEY_ENABLE);
203 if (rdr_enable.has_value()) {
204 ParseRdrEnable(**rdr_enable);
205 }
206
207 auto rdr_path = CheckJsonKeyExist(*rdr_setting, KEY_RDR_SETTINGS, KEY_PATH);
208 if (rdr_path.has_value()) {
209 ParseRdrPath(**rdr_path);
210 }
211 }
212
ParseRdrPath(const nlohmann::json & content)213 void EnvConfigParser::ParseRdrPath(const nlohmann::json &content) {
214 std::string err_msg = "RDR path parse failed. The RDR path will be a default value: '" + rdr_path_ +
215 "'. Please check the settings about '" + KEY_RDR_SETTINGS + "' in config file '" +
216 config_file_ + "' set by 'env_config_path' in context.";
217
218 if (!CheckJsonStringType(content, KEY_RDR_SETTINGS, KEY_PATH)) {
219 MS_LOG(WARNING) << err_msg;
220 return;
221 }
222
223 std::string path = content;
224 if (!Common::IsPathValid(path, MAX_DIRECTORY_LENGTH, err_msg)) {
225 return;
226 }
227
228 if (path.back() != '/') {
229 path += '/';
230 }
231 rdr_path_ = path;
232 }
233
ParseRdrEnable(const nlohmann::json & content)234 void EnvConfigParser::ParseRdrEnable(const nlohmann::json &content) {
235 if (!content.is_boolean()) {
236 MS_LOG(WARNING) << "Json parse failed. 'enable' in " << KEY_RDR_SETTINGS << " should be boolean."
237 << " Please check the config file '" << config_file_ << "' set by 'env_config_path' in context.";
238 return;
239 }
240 rdr_enabled_ = content;
241 }
242 #endif
243
ConfigToString()244 void EnvConfigParser::ConfigToString() {
245 std::string cur_config;
246 #ifdef ENABLE_DUMP_IR
247 cur_config.append("After parsed, rdr path: ");
248 cur_config.append(rdr_path_);
249 cur_config.append(", rdr_enable: ");
250 std::string rdr_enable_flag = rdr_enabled_ ? "1" : "0";
251 (void)cur_config.append(rdr_enable_flag);
252 #endif
253 MS_LOG(INFO) << cur_config;
254 }
255 } // namespace mindspore
256