1 /**
2 * Copyright 2023-2024 Huawei Technologies Co., Ltd
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16 #include "common/debug/profiler/profiling_data_dumper.h"
17 #include <algorithm>
18 #include <mutex>
19 #include <utility>
20 #include "kernel/kernel.h"
21 #include "ops/structure_op_name.h"
22 #include "include/backend/anf_runtime_algorithm.h"
23 #include "include/common/utils/anfalgo.h"
24 #include "include/common/utils/utils.h"
25 #include "utils/ms_context.h"
26 #include "nlohmann/json.hpp"
27 #include "include/backend/debug/profiler/profiling.h"
28
29 namespace mindspore {
30 namespace profiler {
31 namespace ascend {
32
33 #if !defined(_WIN32) && !defined(_WIN64) && !defined(__ANDROID__) && !defined(ANDROID) && !defined(__APPLE__)
IsFileExist(const std::string & path)34 bool Utils::IsFileExist(const std::string &path) {
35 if (path.empty() || path.size() > PATH_MAX) {
36 return false;
37 }
38 return (access(path.c_str(), F_OK) == 0) ? true : false;
39 }
40
IsFileWritable(const std::string & path)41 bool Utils::IsFileWritable(const std::string &path) {
42 if (path.empty() || path.size() > PATH_MAX) {
43 return false;
44 }
45 return (access(path.c_str(), W_OK) == 0) ? true : false;
46 }
47
IsDir(const std::string & path)48 bool Utils::IsDir(const std::string &path) {
49 if (path.empty() || path.size() > PATH_MAX) {
50 return false;
51 }
52 struct stat st = {0};
53 int ret = lstat(path.c_str(), &st);
54 if (ret != 0) {
55 return false;
56 }
57 return S_ISDIR(st.st_mode) ? true : false;
58 }
59
CreateDir(const std::string & path)60 bool Utils::CreateDir(const std::string &path) {
61 if (path.empty() || path.size() > PATH_MAX) {
62 return false;
63 }
64 if (IsFileExist(path)) {
65 return IsDir(path) ? true : false;
66 }
67 size_t pos = 0;
68 static const int DEFAULT_MKDIR_MODE = S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH;
69 while ((pos = path.find_first_of('/', pos)) != std::string::npos) {
70 std::string base_dir = path.substr(0, ++pos);
71 if (IsFileExist(base_dir)) {
72 if (IsDir(base_dir)) {
73 continue;
74 } else {
75 return false;
76 }
77 }
78 if (mkdir(base_dir.c_str(), DEFAULT_MKDIR_MODE) != 0) {
79 return false;
80 }
81 }
82 return (mkdir(path.c_str(), DEFAULT_MKDIR_MODE) == 0) ? true : false;
83 }
84
RealPath(const std::string & path)85 std::string Utils::RealPath(const std::string &path) {
86 if (path.empty() || path.size() > PATH_MAX) {
87 return "";
88 }
89 char realPath[PATH_MAX] = {0};
90 if (realpath(path.c_str(), realPath) == nullptr) {
91 return "";
92 }
93 return std::string(realPath);
94 }
95
RelativeToAbsPath(const std::string & path)96 std::string Utils::RelativeToAbsPath(const std::string &path) {
97 if (path.empty() || path.size() > PATH_MAX) {
98 return "";
99 }
100 if (path[0] != '/') {
101 char pwd_path[PATH_MAX] = {0};
102 if (getcwd(pwd_path, PATH_MAX) != nullptr) {
103 return std::string(pwd_path) + "/" + path;
104 }
105 return "";
106 }
107 return std::string(path);
108 }
109
DirName(const std::string & path)110 std::string Utils::DirName(const std::string &path) {
111 if (path.empty()) {
112 return "";
113 }
114 std::string temp_path = std::string(path.begin(), path.end());
115 char *path_c = dirname(const_cast<char *>(temp_path.data()));
116 return path_c ? std::string(path_c) : "";
117 }
118
GetClockMonotonicRawNs()119 uint64_t Utils::GetClockMonotonicRawNs() {
120 struct timespec ts = {0};
121 clock_gettime(CLOCK_MONOTONIC_RAW, &ts);
122 return static_cast<uint64_t>(ts.tv_sec) * 1000000000 +
123 static_cast<uint64_t>(ts.tv_nsec); // To convert to nanoseconds, it needs to be 1000000000.
124 }
125
CreateDumpFile(const std::string & path)126 bool Utils::CreateDumpFile(const std::string &path) {
127 if (path.empty() || path.size() > PATH_MAX || !CreateDir(DirName(path))) {
128 return false;
129 }
130 std::ofstream output_file(path);
131 output_file.close();
132 if (chmod(path.c_str(), S_IRUSR | S_IWUSR | S_IRGRP) == -1) {
133 MS_LOG(WARNING) << "chmod failed, path: " << path;
134 return false;
135 }
136 return true;
137 }
138
IsSoftLink(const std::string & path)139 bool Utils::IsSoftLink(const std::string &path) {
140 if (path.empty() || path.size() > PATH_MAX || !IsFileExist(path)) {
141 return false;
142 }
143 struct stat st {};
144 if (lstat(path.c_str(), &st) != 0) {
145 return false;
146 }
147 return S_ISLNK(st.st_mode);
148 }
149
GetTid()150 uint64_t Utils::GetTid() {
151 #if !defined(_WIN32) && !defined(_WIN64) && !defined(__ANDROID__) && !defined(ANDROID) && !defined(__APPLE__)
152 static thread_local uint64_t tid = static_cast<uint64_t>(syscall(SYS_gettid));
153 #else
154 static thread_local uint64_t tid = 0;
155 #endif
156 return tid;
157 }
158
GetPid()159 uint64_t Utils::GetPid() {
160 static thread_local uint64_t pid = static_cast<uint64_t>(getpid());
161 return pid;
162 }
163
164 template <typename T>
Init(size_t capacity)165 void RingBuffer<T>::Init(size_t capacity) {
166 capacity_ = capacity;
167 data_queue_.resize(capacity);
168 is_inited_ = true;
169 is_quit_ = false;
170 }
171
172 template <typename T>
UnInit()173 void RingBuffer<T>::UnInit() {
174 if (is_inited_) {
175 data_queue_.clear();
176 read_index_ = 0;
177 write_index_ = 0;
178 idle_write_index_ = 0;
179 capacity_ = 0;
180 is_quit_ = true;
181 is_inited_ = false;
182 }
183 }
184
185 template <typename T>
Size()186 size_t RingBuffer<T>::Size() {
187 size_t curr_read_index = read_index_.load(std::memory_order_acquire);
188 size_t curr_write_index = write_index_.load(std::memory_order_acquire);
189 if (curr_read_index >= curr_write_index) {
190 return 0;
191 }
192 return curr_write_index - curr_read_index;
193 }
194
195 template <typename T>
Full()196 bool RingBuffer<T>::Full() {
197 size_t curr_write_index = write_index_.load(std::memory_order_acquire);
198 if (curr_write_index >= capacity_) {
199 return true;
200 } else {
201 return false;
202 }
203 }
204
205 template <typename T>
Push(T data)206 bool RingBuffer<T>::Push(T data) {
207 size_t curr_write_index = 0;
208 curr_write_index = write_index_.fetch_add(1, std::memory_order_acquire);
209 if (curr_write_index >= capacity_) {
210 return false;
211 }
212 data_queue_[curr_write_index] = std::move(data);
213 return true;
214 }
215
216 template <typename T>
Pop()217 T RingBuffer<T>::Pop() {
218 if (!is_inited_) {
219 return nullptr;
220 }
221 size_t curr_read_index = read_index_.fetch_add(1, std::memory_order_acquire);
222 size_t curr_write_index = write_index_.load(std::memory_order_acquire);
223 if (curr_read_index >= curr_write_index || curr_read_index >= capacity_) {
224 return nullptr;
225 }
226 T data = std::move(data_queue_[curr_read_index]);
227 return data;
228 }
229
230 template <typename T>
Reset()231 void RingBuffer<T>::Reset() {
232 write_index_ = 0;
233 read_index_ = 0;
234 }
235
ProfilingDataDumper()236 ProfilingDataDumper::ProfilingDataDumper() : path_(""), start_(false), init_(false) {}
237
~ProfilingDataDumper()238 ProfilingDataDumper::~ProfilingDataDumper() { UnInit(); }
239
GetInstance()240 ProfilingDataDumper &ProfilingDataDumper::GetInstance() {
241 static ProfilingDataDumper instance;
242 return instance;
243 }
244
Init(const std::string & path,size_t capacity)245 void ProfilingDataDumper::Init(const std::string &path, size_t capacity) {
246 MS_LOG(INFO) << "init profiling data dumper, capacity: " << capacity;
247 path_ = path;
248 data_chunk_buf_.Init(capacity);
249 init_.store(true);
250 }
251
UnInit()252 void ProfilingDataDumper::UnInit() {
253 MS_LOG(INFO) << "uninit profiling data dumper.";
254 if (init_.load()) {
255 init_.store(false);
256 start_.store(false);
257 for (auto &f : fd_map_) {
258 if (f.second != nullptr) {
259 fclose(f.second);
260 f.second = nullptr;
261 }
262 }
263 fd_map_.clear();
264 }
265 }
266
Start()267 void ProfilingDataDumper::Start() {
268 MS_LOG(INFO) << "start profiling data dumper.";
269 if (!init_.load() || !Utils::CreateDir(path_)) {
270 MS_LOG(ERROR) << "init_.load() " << !init_.load() << !Utils::CreateDir(path_);
271 return;
272 }
273 start_.store(true);
274 }
275
Stop()276 void ProfilingDataDumper::Stop() {
277 MS_LOG(INFO) << "stop profiling data dumper.";
278 if (start_.load() == true) {
279 start_.store(false);
280 }
281 Flush();
282 }
283
GatherAndDumpData()284 void ProfilingDataDumper::GatherAndDumpData() {
285 std::map<std::string, std::vector<uint8_t>> dataMap;
286 uint64_t batchSize = 0;
287 while (batchSize < kBatchMaxLen) {
288 std::unique_ptr<BaseReportData> data = data_chunk_buf_.Pop();
289 if (data == nullptr) {
290 break;
291 }
292 std::vector<uint8_t> encodeData = data->encode();
293 batchSize += encodeData.size();
294 const std::string &key = data->tag;
295 auto iter = dataMap.find(key);
296 if (iter == dataMap.end()) {
297 dataMap.insert({key, encodeData});
298 } else {
299 iter->second.insert(iter->second.end(), encodeData.cbegin(), encodeData.cend());
300 }
301 }
302 if (dataMap.size() > 0) {
303 Dump(dataMap);
304 }
305 }
306
Flush()307 void ProfilingDataDumper::Flush() {
308 MS_LOG(INFO) << "data_chunk_buf_.Size: " << data_chunk_buf_.Size();
309 while (data_chunk_buf_.Size() > 0) {
310 GatherAndDumpData();
311 }
312 data_chunk_buf_.Reset();
313 }
314
Report(std::unique_ptr<BaseReportData> data)315 void ProfilingDataDumper::Report(std::unique_ptr<BaseReportData> data) {
316 if (!start_.load() || data == nullptr) {
317 return;
318 }
319 int i = 0;
320 while (is_flush_.load() && i < 10) {
321 usleep(kMaxWaitTimeUs);
322 i++;
323 }
324 if (!data_chunk_buf_.Push(std::move(data))) {
325 is_flush_.store(true);
326 std::lock_guard<std::mutex> flush_lock_(flush_mutex_);
327 if (data_chunk_buf_.Full()) {
328 Flush();
329 }
330 is_flush_.store(false);
331 if (!data_chunk_buf_.Push(std::move(data))) {
332 MS_LOG(ERROR) << "profiling data Report failed.";
333 }
334 }
335 }
336
Dump(const std::map<std::string,std::vector<uint8_t>> & dataMap)337 void ProfilingDataDumper::Dump(const std::map<std::string, std::vector<uint8_t>> &dataMap) {
338 for (auto &data : dataMap) {
339 FILE *fd = nullptr;
340 const std::string dump_file = path_ + "/" + data.first;
341
342 auto iter = fd_map_.find(dump_file);
343 if (iter == fd_map_.end()) {
344 if (!Utils::IsFileExist(dump_file) && !Utils::CreateDumpFile(dump_file)) {
345 MS_LOG(WARNING) << "create file failed, dump_file: " << dump_file;
346 continue;
347 }
348 fd = fopen(dump_file.c_str(), "ab");
349 if (fd == nullptr) {
350 MS_LOG(WARNING) << "create file failed, dump_file: " << dump_file;
351 continue;
352 }
353 fd_map_.insert({dump_file, fd});
354 } else {
355 fd = iter->second;
356 }
357 fwrite(reinterpret_cast<const char *>(data.second.data()), sizeof(char), data.second.size(), fd);
358 fflush(fd);
359 }
360 }
361 #else
362 bool Utils::IsFileExist(const std::string &path) { MS_LOG(INTERNAL_EXCEPTION) << "profiler not support cpu windows."; }
363
364 bool Utils::IsFileWritable(const std::string &path) {
365 MS_LOG(INTERNAL_EXCEPTION) << "profiler not support cpu windows.";
366 }
367 bool Utils::IsDir(const std::string &path) { MS_LOG(INTERNAL_EXCEPTION) << "profiler not support cpu windows."; }
368 bool Utils::CreateDir(const std::string &path) { MS_LOG(INTERNAL_EXCEPTION) << "profiler not support cpu windows."; }
369
370 std::string Utils::RealPath(const std::string &path) {
371 MS_LOG(INTERNAL_EXCEPTION) << "profiler not support cpu windows.";
372 }
373
374 std::string Utils::RelativeToAbsPath(const std::string &path) {
375 MS_LOG(INTERNAL_EXCEPTION) << "profiler not support cpu windows.";
376 }
377
378 std::string Utils::DirName(const std::string &path) {
379 MS_LOG(INTERNAL_EXCEPTION) << "profiler not support cpu windows.";
380 }
381
382 uint64_t Utils::GetClockMonotonicRawNs() { MS_LOG(INTERNAL_EXCEPTION) << "profiler not support cpu windows."; }
383
384 bool Utils::CreateDumpFile(const std::string &path) {
385 MS_LOG(INTERNAL_EXCEPTION) << "profiler not support cpu windows.";
386 }
387
388 bool Utils::IsSoftLink(const std::string &path) { MS_LOG(INTERNAL_EXCEPTION) << "profiler not support cpu windows."; }
389
390 uint64_t Utils::GetTid() { MS_LOG(INTERNAL_EXCEPTION) << "profiler not support cpu windows."; }
391
392 uint64_t Utils::GetPid() { MS_LOG(INTERNAL_EXCEPTION) << "profiler not support cpu windows."; }
393
394 template <typename T>
395 void RingBuffer<T>::Init(size_t capacity) {
396 MS_LOG(INTERNAL_EXCEPTION) << "profiler not support cpu windows.";
397 }
398
399 template <typename T>
400 void RingBuffer<T>::UnInit() {
401 MS_LOG(INTERNAL_EXCEPTION) << "profiler not support cpu windows.";
402 }
403
404 template <typename T>
405 size_t RingBuffer<T>::Size() {
406 MS_LOG(INTERNAL_EXCEPTION) << "profiler not support cpu windows.";
407 }
408
409 template <typename T>
410 bool RingBuffer<T>::Full() {
411 MS_LOG(INTERNAL_EXCEPTION) << "profiler not support cpu windows.";
412 }
413
414 template <typename T>
415 bool RingBuffer<T>::Push(T data) {
416 MS_LOG(INTERNAL_EXCEPTION) << "profiler not support cpu windows.";
417 }
418
419 template <typename T>
420 T RingBuffer<T>::Pop() {
421 MS_LOG(INTERNAL_EXCEPTION) << "profiler not support cpu windows.";
422 }
423
424 template <typename T>
425 void RingBuffer<T>::Reset() {
426 MS_LOG(INTERNAL_EXCEPTION) << "profiler not support cpu windows.";
427 }
428
429 ProfilingDataDumper::ProfilingDataDumper() : path_(""), start_(false), init_(false) {
430 MS_LOG(INTERNAL_EXCEPTION) << "profiler not support cpu windows.";
431 }
432
433 ProfilingDataDumper::~ProfilingDataDumper() { MS_LOG(INTERNAL_EXCEPTION) << "profiler not support cpu windows."; }
434
435 ProfilingDataDumper &ProfilingDataDumper::GetInstance() {
436 MS_LOG(INTERNAL_EXCEPTION) << "profiler not support cpu windows.";
437 }
438
439 void ProfilingDataDumper::Init(const std::string &path, size_t capacity) {
440 MS_LOG(INTERNAL_EXCEPTION) << "profiler not support cpu windows.";
441 }
442
443 void ProfilingDataDumper::UnInit() { MS_LOG(INTERNAL_EXCEPTION) << "profiler not support cpu windows."; }
444
445 void ProfilingDataDumper::Start() { MS_LOG(INTERNAL_EXCEPTION) << "profiler not support cpu windows."; }
446
447 void ProfilingDataDumper::Stop() { MS_LOG(INTERNAL_EXCEPTION) << "profiler not support cpu windows."; }
448
449 void ProfilingDataDumper::GatherAndDumpData() { MS_LOG(INTERNAL_EXCEPTION) << "profiler not support cpu windows."; }
450
451 void ProfilingDataDumper::Flush() { MS_LOG(INTERNAL_EXCEPTION) << "profiler not support cpu windows."; }
452
453 void ProfilingDataDumper::Report(std::unique_ptr<BaseReportData> data) {
454 MS_LOG(INTERNAL_EXCEPTION) << "profiler not support cpu windows.";
455 }
456
457 void ProfilingDataDumper::Dump(const std::map<std::string, std::vector<uint8_t>> &dataMap) {
458 MS_LOG(INTERNAL_EXCEPTION) << "profiler not support cpu windows.";
459 }
460 #endif
461 } // namespace ascend
462 } // namespace profiler
463 } // namespace mindspore
464