1 /**
2 * Copyright 2021 Huawei Technologies Co., Ltd
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16 #include "include/common/debug/rdr/recorder_manager.h"
17 #include <utility>
18 #include "include/common/debug/rdr/base_recorder.h"
19 #include "include/common/debug/env_config_parser.h"
20
21 namespace mindspore {
Instance()22 RecorderManager &RecorderManager::Instance() {
23 static RecorderManager manager{};
24 manager.UpdateRdrEnable();
25 return manager;
26 }
27
UpdateRdrEnable()28 void RecorderManager::UpdateRdrEnable() {
29 static bool updated = false;
30 if (updated) {
31 return;
32 }
33 auto &config_parser = mindspore::EnvConfigParser::GetInstance();
34 rdr_enable_ = config_parser.RdrEnabled();
35 rdr_mode_ = config_parser.RdrMode();
36 if (config_parser.HasRdrSetting()) {
37 #ifdef __linux__
38 if (!rdr_enable_) {
39 MS_LOG(WARNING) << "Not enabling RDR. You can enable RDR through configuration file or environment variables.";
40 }
41 #else
42 if (rdr_enable_) {
43 MS_LOG(WARNING) << "The RDR only supports linux os currently.";
44 }
45 rdr_enable_ = false;
46 #endif
47 }
48 updated = true;
49 }
50
RecordObject(const BaseRecorderPtr & recorder)51 bool RecorderManager::RecordObject(const BaseRecorderPtr &recorder) {
52 if (!rdr_enable_) {
53 return false;
54 }
55 if (recorder == nullptr) {
56 MS_LOG(ERROR) << "Register recorder module with nullptr.";
57 return false;
58 }
59 std::string module = recorder->GetModule();
60 std::string name = recorder->GetName();
61 std::pair<std::string, std::string> recorder_key(module, name);
62 std::lock_guard<std::mutex> lock(mtx_);
63 recorder_container_[recorder_key] = recorder;
64 MS_LOG(INFO) << "RDR record object " << name << " in module \"" << module << "\".";
65 return true;
66 }
67
GetRecorder(std::string module,std::string name)68 BaseRecorderPtr RecorderManager::GetRecorder(std::string module, std::string name) {
69 if (!rdr_enable_) {
70 return nullptr;
71 }
72 std::lock_guard<std::mutex> lock(mtx_);
73 std::pair<std::string, std::string> recorder_key(module, name);
74 auto item = recorder_container_.find(recorder_key);
75 if (item != recorder_container_.end()) {
76 return item->second;
77 }
78 return nullptr;
79 }
80
RdrEnable() const81 bool RecorderManager::RdrEnable() const {
82 std::lock_guard<std::mutex> lock(mtx_);
83 return rdr_enable_;
84 }
85
CheckRdrMemIsRecord() const86 bool RecorderManager::CheckRdrMemIsRecord() const {
87 if (!rdr_enable_) {
88 return false;
89 }
90 std::lock_guard<std::mutex> lock(mtx_);
91 return rdr_has_record_mem_;
92 }
93
SetRdrMemIsRecord(bool is_enable)94 void RecorderManager::SetRdrMemIsRecord(bool is_enable) {
95 if (!rdr_enable_) {
96 return;
97 }
98 std::lock_guard<std::mutex> lock(mtx_);
99 rdr_has_record_mem_ = is_enable;
100 }
101
TriggerAll()102 void RecorderManager::TriggerAll() {
103 if (!rdr_enable_) {
104 return;
105 }
106 bool trigger = false;
107 std::lock_guard<std::mutex> lock(mtx_);
108 for (auto iter = recorder_container_.begin(); iter != recorder_container_.end(); ++iter) {
109 iter->second->Export();
110 trigger = true;
111 }
112 if (!trigger) {
113 MS_LOG(WARNING) << "There is no recorder to export.";
114 } else {
115 if (rdr_mode_ != Normal) {
116 MS_LOG(INFO) << "RDR exports all recorders in abnormal end scenario.";
117 }
118 }
119 }
120
Snapshot()121 void RecorderManager::Snapshot() {
122 if (!rdr_enable_ || rdr_mode_ != Normal) {
123 return;
124 }
125 RecorderManager::TriggerAll();
126 MS_LOG(INFO) << "RDR exports all recorders in normal end scenario.";
127 // Prevent duplicate data export by ClearResAtexit() in exceptional scenario.
128 ClearAll();
129 }
130
ClearAll()131 void RecorderManager::ClearAll() {
132 if (!rdr_enable_) {
133 return;
134 }
135 std::lock_guard<std::mutex> lock(mtx_);
136 recorder_container_.clear();
137 rdr_has_record_mem_ = false;
138 MS_LOG(INFO) << "RDR clear all recorders.";
139 }
140
141 namespace RDR {
TriggerAll()142 void TriggerAll() { mindspore::RecorderManager::Instance().TriggerAll(); }
143
Snapshot()144 void Snapshot() { mindspore::RecorderManager::Instance().Snapshot(); }
145
ResetRecorder()146 void ResetRecorder() { mindspore::RecorderManager::Instance().ClearAll(); }
147 } // namespace RDR
148 } // namespace mindspore
149