1 /* Copyright 2020 The TensorFlow Authors. All Rights Reserved.
2
3 Licensed under the Apache License, Version 2.0 (the "License");
4 you may not use this file except in compliance with the License.
5 You may obtain a copy of the License at
6
7 http://www.apache.org/licenses/LICENSE-2.0
8
9 Unless required by applicable law or agreed to in writing, software
10 distributed under the License is distributed on an "AS IS" BASIS,
11 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 See the License for the specific language governing permissions and
13 limitations under the License.
14 ==============================================================================*/
15
16 #include "tensorflow/core/profiler/convert/xplane_to_tools_data.h"
17
18 #include <string>
19 #include <utility>
20 #include <vector>
21
22 #include "absl/strings/str_format.h"
23 #include "absl/strings/string_view.h"
24 #include "tensorflow/core/platform/errors.h"
25 #include "tensorflow/core/platform/logging.h"
26 #include "tensorflow/core/platform/statusor.h"
27 #include "tensorflow/core/profiler/convert/hlo_to_tools_data.h"
28 #include "tensorflow/core/profiler/convert/op_stats_to_input_pipeline_analysis.h"
29 #include "tensorflow/core/profiler/convert/op_stats_to_op_profile.h"
30 #include "tensorflow/core/profiler/convert/op_stats_to_overview_page.h"
31 #include "tensorflow/core/profiler/convert/op_stats_to_pod_viewer.h"
32 #include "tensorflow/core/profiler/convert/op_stats_to_tf_stats.h"
33 #include "tensorflow/core/profiler/convert/tool_options.h"
34 #include "tensorflow/core/profiler/convert/xplane_to_hlo.h"
35 #include "tensorflow/core/profiler/convert/xplane_to_memory_profile.h"
36 #include "tensorflow/core/profiler/convert/xplane_to_op_stats.h"
37 #include "tensorflow/core/profiler/convert/xplane_to_tf_data_stats.h"
38 #include "tensorflow/core/profiler/convert/xplane_to_trace_events.h"
39 #include "tensorflow/core/profiler/protobuf/hardware_types.pb.h"
40 #include "tensorflow/core/profiler/protobuf/input_pipeline.pb.h"
41 #include "tensorflow/core/profiler/protobuf/kernel_stats.pb.h"
42 #include "tensorflow/core/profiler/protobuf/op_profile.pb.h"
43 #include "tensorflow/core/profiler/protobuf/op_stats.pb.h"
44 #include "tensorflow/core/profiler/protobuf/overview_page.pb.h"
45 #include "tensorflow/core/profiler/protobuf/pod_viewer.pb.h"
46 #include "tensorflow/core/profiler/protobuf/tf_data_stats.pb.h"
47 #include "tensorflow/core/profiler/protobuf/tf_stats.pb.h"
48 #include "tensorflow/core/profiler/protobuf/xplane.pb.h"
49 #include "tensorflow/core/profiler/utils/hardware_type_utils.h"
50 #include "tensorflow/core/profiler/utils/xplane_schema.h"
51 #include "tensorflow/core/profiler/utils/xplane_utils.h"
52
53 namespace tensorflow {
54 namespace profiler {
55
56 namespace {
57
ConvertXSpaceToTraceEvents(const std::vector<XSpace> & xspaces)58 StatusOr<std::string> ConvertXSpaceToTraceEvents(
59 const std::vector<XSpace>& xspaces) {
60 if (xspaces.size() != 1) {
61 return errors::InvalidArgument(
62 "Trace events tool expects only 1 XSpace path but gets ",
63 xspaces.size());
64 }
65
66 std::string content;
67 ConvertXSpaceToTraceEventsString(xspaces[0], &content);
68 return content;
69 }
70
ConvertMultiXSpacesToOverviewPage(const std::vector<XSpace> & xspaces)71 StatusOr<std::string> ConvertMultiXSpacesToOverviewPage(
72 const std::vector<XSpace>& xspaces) {
73 OpStatsOptions options;
74 options.generate_kernel_stats_db = true;
75 options.generate_op_metrics_db = true;
76 options.generate_step_db = true;
77 OpStats combined_op_stats;
78 TF_RETURN_IF_ERROR(ConvertMultiXSpacesToCombinedOpStats(xspaces, options,
79 &combined_op_stats));
80
81 // TODO(profiler): xspace should tell whether this is sampling mode.
82 OverviewPage overview_page_db =
83 ConvertOpStatsToOverviewPage(combined_op_stats);
84 return overview_page_db.SerializeAsString();
85 }
86
ConvertMultiXSpacesToInputPipeline(const std::vector<XSpace> & xspaces)87 StatusOr<std::string> ConvertMultiXSpacesToInputPipeline(
88 const std::vector<XSpace>& xspaces) {
89 OpStatsOptions options;
90 options.generate_op_metrics_db = true;
91 options.generate_step_db = true;
92 OpStats combined_op_stats;
93 TF_RETURN_IF_ERROR(ConvertMultiXSpacesToCombinedOpStats(xspaces, options,
94 &combined_op_stats));
95 return ConvertOpStatsToInputPipelineAnalysis(combined_op_stats)
96 .SerializeAsString();
97 }
98
ConvertMultiXSpacesToTfStats(const std::vector<XSpace> & xspaces)99 StatusOr<std::string> ConvertMultiXSpacesToTfStats(
100 const std::vector<XSpace>& xspaces) {
101 OpStatsOptions options;
102 options.generate_op_metrics_db = true;
103 options.generate_kernel_stats_db = true;
104 OpStats combined_op_stats;
105 TF_RETURN_IF_ERROR(ConvertMultiXSpacesToCombinedOpStats(xspaces, options,
106 &combined_op_stats));
107 return ConvertOpStatsToTfStats(combined_op_stats).SerializeAsString();
108 }
109
ConvertMultiXSpacesToKernelStats(const std::vector<XSpace> & xspaces)110 StatusOr<std::string> ConvertMultiXSpacesToKernelStats(
111 const std::vector<XSpace>& xspaces) {
112 OpStatsOptions options;
113 options.generate_kernel_stats_db = true;
114 OpStats combined_op_stats;
115 TF_RETURN_IF_ERROR(ConvertMultiXSpacesToCombinedOpStats(xspaces, options,
116 &combined_op_stats));
117 return combined_op_stats.kernel_stats_db().SerializeAsString();
118 }
119
ConvertXSpaceToMemoryProfile(const std::vector<XSpace> & xspaces)120 StatusOr<std::string> ConvertXSpaceToMemoryProfile(
121 const std::vector<XSpace>& xspaces) {
122 if (xspaces.size() != 1) {
123 return errors::InvalidArgument(
124 "Memory profile tool expects only 1 XSpace path but gets ",
125 xspaces.size());
126 }
127 std::string json_output;
128 TF_RETURN_IF_ERROR(
129 ConvertXSpaceToMemoryProfileJson(xspaces[0], &json_output));
130 return json_output;
131 }
132
ConvertMultiXSpacesToPodViewer(const std::vector<XSpace> & xspaces)133 StatusOr<std::string> ConvertMultiXSpacesToPodViewer(
134 const std::vector<XSpace>& xspaces) {
135 OpStatsOptions options;
136 options.generate_op_metrics_db = true;
137 options.generate_step_db = true;
138 OpStats combined_op_stats;
139 TF_RETURN_IF_ERROR(ConvertMultiXSpacesToCombinedOpStats(xspaces, options,
140 &combined_op_stats));
141
142 std::string json_output;
143 protobuf::util::JsonPrintOptions opts;
144 opts.always_print_primitive_fields = true;
145 auto encode_status = protobuf::util::MessageToJsonString(
146 ConvertOpStatsToPodViewer(combined_op_stats), &json_output, opts);
147 if (!encode_status.ok()) {
148 const auto& error_message = encode_status.message();
149 return errors::Internal(
150 "Could not convert pod viewer proto to json. Error: ",
151 absl::string_view(error_message.data(), error_message.length()));
152 }
153 return json_output;
154 }
155
ConvertMultiXSpacesToTfDataBottleneckAnalysis(const std::vector<XSpace> & xspaces,const std::vector<std::string> & filenames)156 StatusOr<std::string> ConvertMultiXSpacesToTfDataBottleneckAnalysis(
157 const std::vector<XSpace>& xspaces,
158 const std::vector<std::string>& filenames) {
159 CombinedTfDataStats combined_tf_data_stats;
160 CombinedTfDataStatsBuilder builder(&combined_tf_data_stats);
161
162 std::vector<XSpace> mutable_xspaces = xspaces;
163
164 for (int idx = 0; idx < mutable_xspaces.size(); ++idx) {
165 XPlane* host_plane =
166 FindMutablePlaneWithName(&mutable_xspaces[idx], kHostThreadsPlaneName);
167 if (host_plane == nullptr) {
168 return errors::InvalidArgument(
169 "Could not find host XPlane for tf data stats: ", filenames[idx]);
170 }
171 absl::string_view host_name = mutable_xspaces[idx].hostnames_size()
172 ? mutable_xspaces[idx].hostnames(0)
173 : filenames[idx];
174 builder.Add(host_name, host_plane);
175 }
176 builder.Finalize();
177 return combined_tf_data_stats.SerializeAsString();
178 }
179
ConvertMultiXSpacesToOpProfileViewer(const std::vector<XSpace> & xspaces)180 StatusOr<std::string> ConvertMultiXSpacesToOpProfileViewer(
181 const std::vector<XSpace>& xspaces) {
182 OpStatsOptions options;
183 options.generate_op_metrics_db = true;
184 OpStats combined_op_stats;
185 TF_RETURN_IF_ERROR(ConvertMultiXSpacesToCombinedOpStats(xspaces, options,
186 &combined_op_stats));
187
188 tensorflow::profiler::op_profile::Profile profile;
189 ConvertOpStatsToOpProfile(
190 combined_op_stats,
191 ParseHardwareType(combined_op_stats.run_environment().device_type()),
192 profile);
193
194 return profile.SerializeAsString();
195 }
196 } // namespace
197
ConvertMultiXSpacesToToolData(const std::vector<XSpace> & xspaces,const std::vector<std::string> & filenames,const absl::string_view tool_name,const ToolOptions & options)198 StatusOr<std::string> ConvertMultiXSpacesToToolData(
199 const std::vector<XSpace>& xspaces,
200 const std::vector<std::string>& filenames,
201 const absl::string_view tool_name, const ToolOptions& options) {
202 if (tool_name == "trace_viewer") {
203 return ConvertXSpaceToTraceEvents(xspaces);
204 } else if (tool_name == "overview_page") {
205 return ConvertMultiXSpacesToOverviewPage(xspaces);
206 } else if (tool_name == "input_pipeline_analyzer") {
207 return ConvertMultiXSpacesToInputPipeline(xspaces);
208 } else if (tool_name == "tensorflow_stats") {
209 return ConvertMultiXSpacesToTfStats(xspaces);
210 } else if (tool_name == "kernel_stats") {
211 return ConvertMultiXSpacesToKernelStats(xspaces);
212 } else if (tool_name == "memory_profile") {
213 return ConvertXSpaceToMemoryProfile(xspaces);
214 } else if (tool_name == "pod_viewer") {
215 return ConvertMultiXSpacesToPodViewer(xspaces);
216 } else if (tool_name == "tf_data_bottleneck_analysis") {
217 return ConvertMultiXSpacesToTfDataBottleneckAnalysis(xspaces, filenames);
218 } else if (tool_name == "hlo_proto") {
219 // <hlo_proto> is a special tool name to generate HLO proto files from
220 // XSpace and store them in profile repository, this method does not return
221 // actual tool data.
222 TF_RETURN_IF_ERROR(
223 GetHloProtoFromMultiXSpaceAndSaveToFile(xspaces, filenames));
224 return std::string();
225 } else if (tool_name == "op_profile") {
226 return ConvertMultiXSpacesToOpProfileViewer(xspaces);
227 } else if (tool_name == "memory_viewer" || tool_name == "graph_viewer") {
228 return ConvertHloProtoToToolData(filenames, tool_name, options);
229 } else {
230 return errors::InvalidArgument(
231 "Can not find tool: ", tool_name,
232 ". Please update to the latest version of Tensorflow.");
233 }
234 }
235
236 } // namespace profiler
237 } // namespace tensorflow
238