1 /** 2 * Copyright 2021 Huawei Technologies Co., Ltd 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 #ifndef MINDSPORE_CCSRC_PROFILER_DEVICE_DATA_SAVER_H 18 #define MINDSPORE_CCSRC_PROFILER_DEVICE_DATA_SAVER_H 19 #include <iostream> 20 #include <algorithm> 21 #include <unordered_map> 22 #include <vector> 23 #include <string> 24 #include <memory> 25 #include "include/backend/debug/profiler/profiling.h" 26 #include "utils/log_adapter.h" 27 #include "mindspore/core/utils/file_utils.h" 28 29 namespace mindspore { 30 namespace profiler { 31 struct CurKernelInputInfo { 32 uint32_t input_id; 33 std::string shape; 34 }; 35 struct CurKernelInfo { 36 std::string op_type; 37 std::string op_name; 38 std::vector<CurKernelInputInfo> cur_kernel_all_inputs_info; 39 uint32_t graph_id; 40 }; 41 struct OpDetailInfo { 42 std::string op_type_; 43 std::string op_name_; 44 std::string op_full_name_; 45 std::shared_ptr<OpInfo> op_info_{nullptr}; 46 float op_avg_time_{0}; 47 float proportion_{0}; 48 49 OpDetailInfo() = default; 50 OpDetailInfo(const std::shared_ptr<OpInfo> op_info, float proportion); 51 GetCpuHeaderOpDetailInfo52 std::string GetCpuHeader() const { 53 return "op_side,op_type,op_name,full_op_name,op_occurrences,op_total_time(ms)," 54 "op_avg_time(ms),total_proportion,subgraph,pid"; 55 } GetGpuHeaderOpDetailInfo56 std::string GetGpuHeader() const { 57 return "op_side,op_type,op_name,op_full_name,op_occurrences,op_total_time(us),op_avg_time(us),total_proportion," 58 "cuda_activity_cost_time(us),cuda_activity_call_count"; 59 } 60 OutputCpuOpDetailInfoOpDetailInfo61 void OutputCpuOpDetailInfo(std::ostream &os) const { 62 os << "Host," << op_type_ << ',' << op_name_ << ',' << op_full_name_ << ',' << op_info_->op_count << ',' 63 << op_info_->op_host_cost_time << ',' << op_avg_time_ << ',' << proportion_ << ",Default," << op_info_->pid 64 << std::endl; 65 } 66 OutputGpuOpDetailInfoOpDetailInfo67 void OutputGpuOpDetailInfo(std::ostream &os) const { 68 os << "Device," << op_type_ << ',' << op_name_ << ',' << op_full_name_ << ',' << op_info_->op_count << ',' 69 << op_info_->op_host_cost_time << ',' << op_avg_time_ << ',' << proportion_ << ',' 70 << op_info_->cupti_activity_time << ',' << op_info_->op_kernel_count << std::endl; 71 } 72 }; 73 74 struct OpType { 75 std::string op_type_; 76 int count_{0}; 77 int step_{0}; 78 float total_time_{0}; 79 float avg_time_{0}; 80 float proportion_{0}; 81 GetCpuHeaderOpType82 std::string GetCpuHeader() const { 83 return "op_type,type_occurrences,execution_frequency(per-step)," 84 "total_compute_time,avg_time(ms),percent"; 85 } GetGpuHeaderOpType86 std::string GetGpuHeader() const { return "op_type,type_occurrences,total_time(us),total_proportion,avg_time(us)"; } 87 OutputCpuOpTypeInfoOpType88 void OutputCpuOpTypeInfo(std::ostream &os) const { 89 if (step_ == 0) { 90 MS_LOG(ERROR) << "The run step can not be 0."; 91 return; 92 } 93 if (count_ == 0) { 94 MS_LOG(ERROR) << "The num of operation type can not be 0."; 95 return; 96 } 97 os << op_type_ << ',' << count_ << ',' << count_ / step_ << ',' << total_time_ << ',' << total_time_ / count_ << ',' 98 << proportion_ << std::endl; 99 } 100 OutputGpuOpTypeInfoOpType101 void OutputGpuOpTypeInfo(std::ostream &os) const { 102 os << op_type_ << ',' << count_ << ',' << total_time_ << ',' << proportion_ << ',' << avg_time_ << std::endl; 103 } 104 105 OpType &operator+=(const OpType &other) { 106 this->count_ += other.count_; 107 this->total_time_ += other.total_time_; 108 this->proportion_ += other.proportion_; 109 return *this; 110 } 111 }; 112 113 using OpTimestampInfo = std::unordered_map<std::string, std::vector<StartDuration>>; // <op_full_name, StartDuration> 114 using OpInfoMap = std::unordered_map<std::string, OpInfo>; 115 using OpTypeInfos = std::unordered_map<std::string, OpType>; // <op_full_name, Optype> 116 using OpDetailInfos = std::vector<OpDetailInfo>; 117 using MemoryInfoList = std::vector<MemoryPoolInfo>; 118 119 class BACKEND_EXPORT DataSaver { 120 public: 121 DataSaver() = default; 122 123 virtual ~DataSaver() = default; 124 125 void ParseOpInfo(const OpInfoMap &op_info_maps); 126 127 void ParseMemoryInfo(const MemoryInfoList &memory_info_list); 128 129 void WriteFrameWork(const std::string &base_dir, const std::vector<CurKernelInfo> &all_kernel_info_); 130 131 OpTimestampInfo op_timestamps_map_; 132 133 protected: 134 void AddOpDetailInfoForType(const OpDetailInfo &op_detail_info); 135 136 float GetTotalOpTime(const OpInfoMap &op_info_maps) const; 137 138 void WriteOpType(const std::string &saver_base_dir); 139 140 void WriteOpDetail(const std::string &saver_base_dir); 141 142 void WriteOpTimestamp(const std::string &saver_base_dir); 143 144 void WriteMemoryData(const std::string &saver_base_dir); 145 146 void ChangeFileMode(const std::string &file_path) const; 147 148 OpTypeInfos op_type_infos_; 149 OpDetailInfos op_detail_infos_; 150 std::string op_side_; 151 std::string device_id_; 152 std::string memory_info_; 153 }; 154 } // namespace profiler 155 } // namespace mindspore 156 157 #endif // MINDSPORE_CCSRC_PROFILER_DEVICE_DATA_SAVER_H 158