1 /** 2 * Copyright 2019-2021 Huawei Technologies Co., Ltd 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 #ifndef MINDSPORE_CCSRC_RUNTIME_DEVICE_ASCEND_PROFILING_PROFILING_UTILS_H_ 17 #define MINDSPORE_CCSRC_RUNTIME_DEVICE_ASCEND_PROFILING_PROFILING_UTILS_H_ 18 19 #include <map> 20 #include <memory> 21 #include <string> 22 #include <vector> 23 #include <set> 24 #include <unordered_map> 25 #include "backend/session/kernel_graph.h" 26 #include "utils/contract.h" 27 #include "runtime/device/ascend/profiling/reporter/profiling_desc.h" 28 29 namespace mindspore { 30 namespace device { 31 namespace ascend { 32 struct ProfilingTraceInfo { 33 // (trace_begin) -> FP -> BP -> (trace_bp_end) -> OPTIMIZER -> (trace_iter_end) 34 std::string trace_begin; 35 std::set<std::string> trace_bp_end; 36 std::set<std::string> trace_iter_end; 37 38 // profiling specific op, such as AllReduce; 39 std::set<std::string> trace_custom_node; 40 IsValidProfilingTraceInfo41 bool IsValid() const { return !(trace_begin.empty() || trace_iter_end.empty()); } 42 }; 43 44 struct ProfilingContent { 45 // true -send data from device to host and finish profiling 46 bool notify; 47 uint64_t profiler_trace_id; 48 uint32_t flags; 49 }; 50 51 class ProfilingUtils { 52 public: 53 ProfilingUtils() = default; 54 ~ProfilingUtils() = default; 55 56 static void InsertProfilingTraceFp(const AnfNodePtr &anf_node, const ProfilingTraceInfo &profiling_trace_info, 57 NotNull<session::KernelGraph *> graph_ptr, 58 NotNull<std::vector<CNodePtr> *> kernel_list); 59 static void InsertProfilingTraceJobId(const AnfNodePtr &anf_node, NotNull<session::KernelGraph *> graph_ptr, 60 NotNull<std::vector<CNodePtr> *> kernel_list); 61 static void InsertProfilingTraceIterEnd(const AnfNodePtr &anf_node, const ProfilingTraceInfo &profiling_trace_info, 62 NotNull<session::KernelGraph *> graph_ptr, 63 NotNull<std::vector<CNodePtr> *> kernel_list); 64 static void InsertProfilingTraceBpEnd(const mindspore::AnfNodePtr &anf_node, 65 const ProfilingTraceInfo &profiling_trace_info, 66 NotNull<session::KernelGraph *> graph_ptr, 67 NotNull<std::vector<mindspore::CNodePtr> *> kernel_list); 68 static void SetGraphProfilingCNode(uint32_t graph_id, const std::vector<CNodePtr> &profiling_cnode_list); 69 static void SetGraphKernelName(uint32_t graph_id, const std::vector<std::string> &kernel_names); 70 // Save graph information to Framework file 71 static void ReportProfilingData(const std::vector<uint32_t> &task_ids, const std::vector<uint32_t> &stream_ids, 72 const session::KernelGraph &graph); 73 // Generate profiling trace 74 static ProfilingTraceInfo GenerateProfilingTrace(const session::KernelGraph &kernel_graph); 75 76 // Insert two profiling trace points, one in front and one behind 77 static void InsertProfilingCustomOp(const mindspore::AnfNodePtr &anf_node, 78 const ProfilingTraceInfo &profiling_trace_info, 79 NotNull<session::KernelGraph *> graph_ptr, 80 NotNull<std::vector<mindspore::CNodePtr> *> kernel_list); 81 graph_kernel_name()82 static std::map<uint32_t, std::vector<std::string>> graph_kernel_name() { return graph_kernel_name_; } 83 84 inline static constexpr char kProfiling[] = "Profiling"; 85 inline static constexpr char kNotify[] = "notify"; 86 inline static constexpr char kProfilerTraceId[] = "profiler_trace_id"; 87 inline static constexpr char kFlags[] = "flags"; 88 89 private: 90 static NotNull<CNodePtr> CreateProfilingCNode(const ProfilingContent &profiling_content, 91 NotNull<session::KernelGraph *> graph_ptr); 92 static CNodePtr CreateProfilingCNodeWithStream(const AnfNodePtr &anf_node, const ProfilingContent &profiling_content, 93 NotNull<session::KernelGraph *> graph_ptr); 94 static void GetTraceBegin(const session::KernelGraph &kernel_graph, const nlohmann::json &option, 95 ProfilingTraceInfo *trace_info); 96 static void GetTraceBpEnd(const session::KernelGraph &kernel_graph, const nlohmann::json &option, 97 ProfilingTraceInfo *trace_info); 98 static void GetTraceIterEnd(const session::KernelGraph &kernel_graph, ProfilingTraceInfo *trace_info); 99 static std::string GetGraphLastKernelName(const session::KernelGraph &kernel_graph); 100 static void GetTraceHccl(const session::KernelGraph &kernel_graph, NotNull<ProfilingTraceInfo *> profiling_trace); 101 static void GetCNodeOutputRealNode(const std::string &node_name, const session::KernelGraph &kernel_graph, 102 NotNull<std::set<std::string> *> getnext_outputs); 103 104 static bool ValidComputeGraph(const session::KernelGraph &kernel_graph); 105 static void SaveProfilingPoint(uint32_t graph_id, const std::string &node_name, uint32_t point_id); 106 107 // graph id --> (kernel name list) 108 inline static std::map<uint32_t, std::vector<CNodePtr>> graph_profiling_cnode_; 109 inline static std::map<uint32_t, std::vector<std::string>> graph_kernel_name_; 110 inline static std::map<uint32_t, std::vector<std::shared_ptr<ProfDesc>>> graph_point_; 111 inline static uint32_t custom_node_index_; 112 }; 113 } // namespace ascend 114 } // namespace device 115 } // namespace mindspore 116 #endif // MINDSPORE_CCSRC_RUNTIME_DEVICE_ASCEND_PROFILING_PROFILING_UTILS_H_ 117