1 /**
2 * Copyright 2021-2022 Huawei Technologies Co., Ltd
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16 #include "include/common/debug/env_config_parser.h"
17 #include <algorithm>
18 #include <fstream>
19 #include "nlohmann/json.hpp"
20 #include "utils/log_adapter.h"
21 #include "include/common/debug/common.h"
22 #include "utils/ms_context.h"
23
24 namespace {
25 #ifdef ENABLE_DUMP_IR
26 constexpr auto ENV_RDR_ENABLE = "MS_RDR_ENABLE";
27 constexpr auto ENV_RDR_MODE = "MS_RDR_MODE";
28 constexpr auto ENV_RDR_PATH = "MS_RDR_PATH";
29 constexpr auto KEY_RDR_SETTINGS = "rdr";
30 constexpr auto KEY_ENABLE = "enable";
31 constexpr auto KEY_MODE = "mode";
32 constexpr auto KEY_PATH = "path";
33 #endif
34 constexpr auto KEY_MEM_REUSE_SETTINGS = "sys";
35 constexpr auto KEY_MEM_REUSE = "mem_reuse";
36 } // namespace
37
38 namespace mindspore {
GetInstance()39 EnvConfigParser &EnvConfigParser::GetInstance() {
40 static EnvConfigParser instance = EnvConfigParser();
41 instance.Parse();
42 return instance;
43 }
44
45 #ifdef ENABLE_DUMP_IR
GetRdrEnableFromEnv()46 std::optional<bool> GetRdrEnableFromEnv() {
47 // get environment variable to configure RDR
48 std::string env_enable_str = common::GetEnv(ENV_RDR_ENABLE);
49 if (!env_enable_str.empty()) {
50 (void)std::transform(env_enable_str.begin(), env_enable_str.end(), env_enable_str.begin(), ::tolower);
51 if (env_enable_str != "0" && env_enable_str != "1") {
52 MS_LOG(WARNING) << "The environment variable '" << ENV_RDR_ENABLE << "' should be 0 or 1.";
53 }
54 if (env_enable_str == "1") {
55 return true;
56 }
57 return false;
58 }
59 return std::nullopt;
60 }
61
GetRdrModeFromEnv()62 std::optional<int> GetRdrModeFromEnv() {
63 // get environment variable to configure RDR
64 std::string env_mode_str = common::GetEnv(ENV_RDR_MODE);
65 if (!env_mode_str.empty()) {
66 (void)std::transform(env_mode_str.begin(), env_mode_str.end(), env_mode_str.begin(), ::tolower);
67 if (env_mode_str != "1" && env_mode_str != "2") {
68 MS_LOG(WARNING) << "The environment variable '" << ENV_RDR_MODE << "' should be 1 or 2.";
69 }
70 if (env_mode_str == "2") {
71 return Normal;
72 }
73 return Exceptional;
74 }
75 return std::nullopt;
76 }
77
GetRdrPathFromEnv()78 std::optional<std::string> GetRdrPathFromEnv() {
79 // get environment variable to configure RDR
80 std::string path = common::GetEnv(ENV_RDR_PATH);
81 if (!path.empty()) {
82 std::string err_msg = "RDR path parse from environment variable failed. Please check the settings about '" +
83 std::string(ENV_RDR_PATH) + "' in environment variables.";
84 if (!Common::IsPathValid(path, MAX_DIRECTORY_LENGTH, err_msg)) {
85 return std::string("");
86 }
87 return path;
88 }
89 return std::nullopt;
90 }
91 #endif
92
CheckJsonStringType(const nlohmann::json & content,const std::string & setting_key,const std::string & key) const93 bool EnvConfigParser::CheckJsonStringType(const nlohmann::json &content, const std::string &setting_key,
94 const std::string &key) const {
95 if (!content.is_string()) {
96 MS_LOG(WARNING) << "Json Parse Failed. The '" << key << "' in '" << setting_key << "' should be string."
97 << " Please check the config file '" << config_file_ << "' set by 'env_config_path' in context.";
98 return false;
99 }
100 return true;
101 }
102
CheckJsonKeyExist(const nlohmann::json & content,const std::string & setting_key,const std::string & key) const103 std::optional<nlohmann::detail::iter_impl<const nlohmann::json>> EnvConfigParser::CheckJsonKeyExist(
104 const nlohmann::json &content, const std::string &setting_key, const std::string &key) const {
105 auto iter = content.find(key);
106 if (iter == content.end()) {
107 MS_LOG(WARNING) << "Check json failed, '" << key << "' not found in '" << setting_key << "'."
108 << " Please check the config file '" << config_file_ << "' set by 'env_config_path' in context.";
109 return std::nullopt;
110 }
111 return iter;
112 }
113
GetIfstreamString(const std::ifstream & ifstream) const114 std::string EnvConfigParser::GetIfstreamString(const std::ifstream &ifstream) const {
115 std::stringstream buffer;
116 buffer << ifstream.rdbuf();
117 return buffer.str();
118 }
119
ParseFromEnv()120 void EnvConfigParser::ParseFromEnv() {
121 #ifdef ENABLE_DUMP_IR
122 // Get RDR seetings from environment variables
123 auto rdr_enable_env = GetRdrEnableFromEnv();
124 if (rdr_enable_env.has_value()) {
125 has_rdr_setting_ = true;
126 rdr_enabled_ = rdr_enable_env.value();
127 }
128 auto rdr_mode_env = GetRdrModeFromEnv();
129 if (rdr_mode_env.has_value()) {
130 has_rdr_setting_ = true;
131 rdr_mode_ = rdr_mode_env.value();
132 }
133 auto path_env = GetRdrPathFromEnv();
134 if (path_env.has_value()) {
135 has_rdr_setting_ = true;
136 std::string path = path_env.value();
137 if (!path.empty()) {
138 if (path.back() != '/') {
139 path += '/';
140 }
141 rdr_path_ = path;
142 }
143 }
144 #endif
145 }
146
ParseFromFile()147 void EnvConfigParser::ParseFromFile() {
148 auto context = MsContext::GetInstance();
149 MS_EXCEPTION_IF_NULL(context);
150 auto config_file = context->get_param<std::string>(MS_CTX_ENV_CONFIG_PATH);
151 if (config_file.empty()) {
152 MS_LOG(INFO) << "The 'env_config_path' in 'mindspore.context.set_context(env_config_path={path})' is empty.";
153 return;
154 }
155 config_file_ = config_file;
156 std::ifstream json_file(config_file_);
157 if (!json_file.is_open()) {
158 MS_LOG(WARNING) << "Env config file:" << config_file_ << " open failed."
159 << " Please check the config file '" << config_file_ << "' set by 'env_config_path' in context."
160 << ErrnoToString(errno);
161 return;
162 }
163
164 nlohmann::json j;
165 try {
166 json_file >> j;
167 } catch (nlohmann::json::parse_error &e) {
168 MS_LOG(WARNING) << "Env config json contents '" << GetIfstreamString(json_file) << "' in config file '"
169 << config_file_ << "' set by 'env_config_path' in context.";
170 return;
171 }
172
173 // convert json to string
174 std::stringstream ss;
175 ss << j;
176 std::string cfg = ss.str();
177 MS_LOG(INFO) << "Env config json:" << cfg;
178
179 #ifdef ENABLE_DUMP_IR
180 ParseRdrSetting(j);
181 #endif
182 ParseMemReuseSetting(j);
183
184 ConfigToString();
185 }
186
Parse()187 void EnvConfigParser::Parse() {
188 std::lock_guard<std::mutex> guard(lock_);
189 if (already_parsed_) {
190 return;
191 }
192 already_parsed_ = true;
193 ParseFromEnv();
194 ParseFromFile();
195 }
196
ParseMemReuseSetting(const nlohmann::json & content)197 void EnvConfigParser::ParseMemReuseSetting(const nlohmann::json &content) {
198 auto sys_setting = content.find(KEY_MEM_REUSE_SETTINGS);
199 if (sys_setting == content.end()) {
200 MS_LOG(INFO) << "The '" << KEY_MEM_REUSE_SETTINGS << "' isn't existed. Please check the config file '"
201 << config_file_ << "' set by 'env_config_path' in context.";
202 return;
203 }
204 auto sys_memreuse = CheckJsonKeyExist(*sys_setting, KEY_MEM_REUSE_SETTINGS, KEY_MEM_REUSE);
205 if (sys_memreuse.has_value()) {
206 ParseSysMemReuse(**sys_memreuse);
207 }
208 }
209
ParseSysMemReuse(const nlohmann::json & content)210 void EnvConfigParser::ParseSysMemReuse(const nlohmann::json &content) {
211 if (!content.is_boolean()) {
212 MS_LOG(INFO) << "the json object parses failed. 'enable' in " << KEY_MEM_REUSE_SETTINGS << " should be boolean."
213 << " Please check the config file '" << config_file_ << "' set by 'env_config_path' in context.";
214 return;
215 }
216 sys_memreuse_ = content;
217 }
218
219 #ifdef ENABLE_DUMP_IR
ParseRdrSetting(const nlohmann::json & content)220 void EnvConfigParser::ParseRdrSetting(const nlohmann::json &content) {
221 auto rdr_setting = content.find(KEY_RDR_SETTINGS);
222 if (rdr_setting == content.end()) {
223 MS_LOG(WARNING) << "The '" << KEY_RDR_SETTINGS << "' not exists. Please check the config file '" << config_file_
224 << "' set by 'env_config_path' in context.";
225 return;
226 }
227
228 has_rdr_setting_ = true;
229
230 auto rdr_enable = CheckJsonKeyExist(*rdr_setting, KEY_RDR_SETTINGS, KEY_ENABLE);
231 if (rdr_enable.has_value()) {
232 ParseRdrEnable(**rdr_enable);
233 }
234
235 auto rdr_mode = CheckJsonKeyExist(*rdr_setting, KEY_RDR_SETTINGS, KEY_MODE);
236 if (rdr_mode.has_value()) {
237 ParseRdrMode(**rdr_mode);
238 }
239
240 auto rdr_path = CheckJsonKeyExist(*rdr_setting, KEY_RDR_SETTINGS, KEY_PATH);
241 if (rdr_path.has_value()) {
242 ParseRdrPath(**rdr_path);
243 }
244 }
245
ParseRdrEnable(const nlohmann::json & content)246 void EnvConfigParser::ParseRdrEnable(const nlohmann::json &content) {
247 if (!content.is_boolean()) {
248 MS_LOG(WARNING) << "Json parse failed. 'enable' in " << KEY_RDR_SETTINGS << " should be boolean."
249 << " Please check the config file '" << config_file_ << "' set by 'env_config_path' in context.";
250 return;
251 }
252 rdr_enabled_ = content;
253 }
254
ParseRdrMode(const nlohmann::json & content)255 void EnvConfigParser::ParseRdrMode(const nlohmann::json &content) {
256 if (content != Exceptional && content != Normal) {
257 MS_LOG(WARNING) << "Json parse failed. 'mode' in " << KEY_RDR_SETTINGS << " should be 1 or 2."
258 << " Please check the config file '" << config_file_ << "' set by 'env_config_path' in context.";
259 return;
260 }
261 rdr_mode_ = content;
262 }
263
ParseRdrPath(const nlohmann::json & content)264 void EnvConfigParser::ParseRdrPath(const nlohmann::json &content) {
265 std::string err_msg = "RDR path parse failed. The RDR path will be a default value: '" + rdr_path_ +
266 "'. Please check the settings about '" + KEY_RDR_SETTINGS + "' in config file '" +
267 config_file_ + "' set by 'env_config_path' in context.";
268
269 if (!CheckJsonStringType(content, KEY_RDR_SETTINGS, KEY_PATH)) {
270 MS_LOG(WARNING) << err_msg;
271 return;
272 }
273
274 std::string path = content;
275 if (!Common::IsPathValid(path, MAX_DIRECTORY_LENGTH, err_msg)) {
276 return;
277 }
278
279 if (path.back() != '/') {
280 path += '/';
281 }
282 rdr_path_ = path;
283 }
284 #endif
285
ConfigToString()286 void EnvConfigParser::ConfigToString() {
287 std::string cur_config;
288 #ifdef ENABLE_DUMP_IR
289 (void)cur_config.append("After parsed, ");
290 (void)cur_config.append("rdr_enable: ");
291 std::string rdr_enable_flag = rdr_enabled_ ? "1" : "0";
292 (void)cur_config.append(rdr_enable_flag);
293 (void)cur_config.append(", rdr mode: ");
294 (void)cur_config.append(std::to_string(rdr_mode_));
295 (void)cur_config.append(", rdr path: ");
296 (void)cur_config.append(rdr_path_);
297 #endif
298 MS_LOG(INFO) << cur_config;
299 }
300 } // namespace mindspore
301