• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 /* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
2 
3 Licensed under the Apache License, Version 2.0 (the "License");
4 you may not use this file except in compliance with the License.
5 You may obtain a copy of the License at
6 
7     http://www.apache.org/licenses/LICENSE-2.0
8 
9 Unless required by applicable law or agreed to in writing, software
10 distributed under the License is distributed on an "AS IS" BASIS,
11 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 See the License for the specific language governing permissions and
13 limitations under the License.
14 ==============================================================================*/
15 
16 #include "tensorflow/core/util/stats_calculator.h"
17 
18 #include <iomanip>
19 #include <map>
20 #include <queue>
21 #include <sstream>
22 #include <string>
23 
24 namespace tensorflow {
25 
StatsCalculator(const StatSummarizerOptions & options)26 StatsCalculator::StatsCalculator(const StatSummarizerOptions& options)
27     : options_(options) {}
28 
GetShortSummary() const29 std::string StatsCalculator::GetShortSummary() const {
30   std::stringstream stream;
31   stream << "Timings (microseconds): ";
32   run_total_us_.OutputToStream(&stream);
33   stream << std::endl;
34 
35   stream << "Memory (bytes): ";
36   memory_.OutputToStream(&stream);
37   stream << std::endl;
38 
39   stream << details_.size() << " nodes observed" << std::endl;
40   return stream.str();
41 }
42 
InitField(std::ostream & stream,int width)43 std::ostream& InitField(std::ostream& stream, int width) {
44   stream << "\t" << std::right << std::setw(width) << std::fixed
45          << std::setprecision(3);
46   return stream;
47 }
48 
HeaderString(const std::string & title) const49 std::string StatsCalculator::HeaderString(const std::string& title) const {
50   std::stringstream stream;
51 
52   stream << "============================== " << title
53          << " ==============================" << std::endl;
54   if (options_.format_as_csv) {
55     stream << "node type, start, first, avg_ms, %, cdf%, mem KB, times called, "
56               "name";
57   } else {
58     InitField(stream, 24) << "[node type]";
59     InitField(stream, 17) << "[start]";
60     InitField(stream, 9) << "[first]";
61     InitField(stream, 9) << "[avg ms]";
62     InitField(stream, 8) << "[%]";
63     InitField(stream, 8) << "[cdf%]";
64     InitField(stream, 10) << "[mem KB]";
65     InitField(stream, 9) << "[times called]";
66     stream << "\t"
67            << "[Name]";
68   }
69   return stream.str();
70 }
71 
ColumnString(const Detail & detail,const int64_t cumulative_stat_on_node,const Stat<int64_t> & stat) const72 std::string StatsCalculator::ColumnString(const Detail& detail,
73                                           const int64_t cumulative_stat_on_node,
74                                           const Stat<int64_t>& stat) const {
75   const double start_ms = detail.start_us.avg() / 1000.0;
76   const double first_time_ms = detail.rel_end_us.first() / 1000.0;
77   const double avg_time_ms = detail.rel_end_us.avg() / 1000.0;
78   const double percentage = detail.rel_end_us.sum() * 100.0 / stat.sum();
79   const double cdf_percentage = (cumulative_stat_on_node * 100.0f) / stat.sum();
80   const int64_t times_called = detail.times_called / num_runs();
81 
82   std::stringstream stream;
83   if (options_.format_as_csv) {
84     std::string name(detail.name);
85     std::replace(name.begin(), name.end(), ',', '\t');
86     stream << detail.type << ", " << start_ms << ", " << first_time_ms << ", "
87            << avg_time_ms << ", " << percentage << "%, " << cdf_percentage
88            << "%, " << detail.mem_used.newest() / 1000.0 << ", " << times_called
89            << ", " << name;
90   } else {
91     InitField(stream, 24) << detail.type;
92     InitField(stream, 17) << start_ms;
93     InitField(stream, 9) << first_time_ms;
94     InitField(stream, 9) << avg_time_ms;
95     InitField(stream, 7) << percentage << "%";
96     InitField(stream, 7) << cdf_percentage << "%";
97     InitField(stream, 10) << detail.mem_used.newest() / 1000.0;
98     InitField(stream, 9) << times_called;
99     stream << "\t" << detail.name;
100   }
101 
102   return stream.str();
103 }
104 
OrderNodesByMetric(SortingMetric metric,std::vector<const Detail * > * details) const105 void StatsCalculator::OrderNodesByMetric(
106     SortingMetric metric, std::vector<const Detail*>* details) const {
107   std::priority_queue<std::pair<std::string, const Detail*>> sorted_list;
108   const int num_nodes = details_.size();
109 
110   for (const auto& det : details_) {
111     const Detail* detail = &(det.second);
112     std::stringstream stream;
113     stream << std::setw(20) << std::right << std::setprecision(10)
114            << std::fixed;
115 
116     switch (metric) {
117       case BY_NAME:
118         stream << detail->name;
119         break;
120       case BY_RUN_ORDER:
121         stream << num_nodes - detail->run_order;
122         break;
123       case BY_TIME:
124         stream << detail->rel_end_us.avg();
125         break;
126       case BY_MEMORY:
127         stream << detail->mem_used.avg();
128         break;
129       case BY_TYPE:
130         stream << detail->type;
131         break;
132       default:
133         stream << "";
134         break;
135     }
136 
137     sorted_list.emplace(stream.str(), detail);
138   }
139 
140   while (!sorted_list.empty()) {
141     auto entry = sorted_list.top();
142     sorted_list.pop();
143     details->push_back(entry.second);
144   }
145 }
146 
ComputeStatsByType(std::map<std::string,int64_t> * node_type_map_count,std::map<std::string,int64_t> * node_type_map_time,std::map<std::string,int64_t> * node_type_map_memory,std::map<std::string,int64_t> * node_type_map_times_called,int64_t * accumulated_us) const147 void StatsCalculator::ComputeStatsByType(
148     std::map<std::string, int64_t>* node_type_map_count,
149     std::map<std::string, int64_t>* node_type_map_time,
150     std::map<std::string, int64_t>* node_type_map_memory,
151     std::map<std::string, int64_t>* node_type_map_times_called,
152     int64_t* accumulated_us) const {
153   int64_t run_count = run_total_us_.count();
154 
155   for (const auto& det : details_) {
156     const std::string node_name = det.first;
157     const Detail& detail = det.second;
158 
159     int64_t curr_time_val =
160         static_cast<int64_t>(detail.rel_end_us.sum() / run_count);
161     *accumulated_us += curr_time_val;
162 
163     int64_t curr_memory_val = detail.mem_used.newest();
164 
165     const std::string& node_type = detail.type;
166 
167     (*node_type_map_count)[node_type] += 1;
168     (*node_type_map_time)[node_type] += curr_time_val;
169     (*node_type_map_memory)[node_type] += curr_memory_val;
170     (*node_type_map_times_called)[node_type] += detail.times_called / run_count;
171   }
172 }
173 
GetStatsByNodeType() const174 std::string StatsCalculator::GetStatsByNodeType() const {
175   std::stringstream stream;
176 
177   stream << "Number of nodes executed: " << details_.size() << std::endl;
178 
179   stream << "============================== Summary by node type "
180             "=============================="
181          << std::endl;
182 
183   std::map<std::string, int64_t> node_type_map_count;
184   std::map<std::string, int64_t> node_type_map_time;
185   std::map<std::string, int64_t> node_type_map_memory;
186   std::map<std::string, int64_t> node_type_map_times_called;
187   int64_t accumulated_us = 0;
188 
189   ComputeStatsByType(&node_type_map_count, &node_type_map_time,
190                      &node_type_map_memory, &node_type_map_times_called,
191                      &accumulated_us);
192 
193   // Sort them.
194   std::priority_queue<std::pair<int64_t, std::pair<std::string, int64_t>>>
195       timings;
196   for (const auto& node_type : node_type_map_time) {
197     const int64_t mem_used = node_type_map_memory[node_type.first];
198     timings.emplace(node_type.second,
199                     std::pair<std::string, int64_t>(node_type.first, mem_used));
200   }
201 
202   if (options_.format_as_csv) {
203     stream << "node type, count, avg_ms, avg %, cdf %, mem KB, times called\n";
204   } else {
205     InitField(stream, 24) << "[Node type]";
206     InitField(stream, 9) << "[count]";
207     InitField(stream, 10) << "[avg ms]";
208     InitField(stream, 11) << "[avg %]";
209     InitField(stream, 11) << "[cdf %]";
210     InitField(stream, 10) << "[mem KB]";
211     InitField(stream, 10) << "[times called]";
212     stream << std::endl;
213   }
214 
215   float cdf = 0.0f;
216   while (!timings.empty()) {
217     auto entry = timings.top();
218     timings.pop();
219 
220     const std::string node_type = entry.second.first;
221     const float memory = entry.second.second / 1000.0f;
222 
223     const int64_t node_type_total_us = entry.first;
224     const float time_per_run_ms = node_type_total_us / 1000.0f;
225 
226     const float percentage =
227         ((entry.first / static_cast<float>(accumulated_us)) * 100.0f);
228     cdf += percentage;
229 
230     if (options_.format_as_csv) {
231       stream << node_type << ", " << node_type_map_count[node_type] << ", "
232              << time_per_run_ms << ", " << percentage << "%, " << cdf << "%, "
233              << memory << ", " << node_type_map_times_called[node_type]
234              << std::endl;
235     } else {
236       InitField(stream, 24) << node_type;
237       InitField(stream, 9) << node_type_map_count[node_type];
238       InitField(stream, 10) << time_per_run_ms;
239       InitField(stream, 10) << percentage << "%";
240       InitField(stream, 10) << cdf << "%";
241       InitField(stream, 10) << memory;
242       InitField(stream, 9) << node_type_map_times_called[node_type];
243       stream << std::endl;
244     }
245   }
246   stream << std::endl;
247   return stream.str();
248 }
249 
GetStatsByMetric(const std::string & title,SortingMetric sorting_metric,int num_stats) const250 std::string StatsCalculator::GetStatsByMetric(const std::string& title,
251                                               SortingMetric sorting_metric,
252                                               int num_stats) const {
253   std::vector<const Detail*> details;
254   OrderNodesByMetric(sorting_metric, &details);
255 
256   double cumulative_stat_on_node = 0;
257 
258   std::stringstream stream;
259   stream << HeaderString(title) << std::endl;
260   int stat_num = 0;
261   for (auto detail : details) {
262     ++stat_num;
263     if (num_stats > 0 && stat_num > num_stats) {
264       break;
265     }
266 
267     // TODO(andrewharp): Make this keep track of the particular metric for cdf.
268     cumulative_stat_on_node += detail->rel_end_us.sum();
269     stream << ColumnString(*detail, cumulative_stat_on_node, run_total_us_)
270            << std::endl;
271   }
272   stream << std::endl;
273   return stream.str();
274 }
275 
GetOutputString() const276 std::string StatsCalculator::GetOutputString() const {
277   std::stringstream stream;
278   if (options_.show_run_order) {
279     stream << GetStatsByMetric("Run Order", BY_RUN_ORDER,
280                                options_.run_order_limit);
281   }
282   if (options_.show_time) {
283     stream << GetStatsByMetric("Top by Computation Time", BY_TIME,
284                                options_.time_limit);
285   }
286   if (options_.show_memory) {
287     stream << GetStatsByMetric("Top by Memory Use", BY_MEMORY,
288                                options_.memory_limit);
289   }
290   if (options_.show_type) {
291     stream << GetStatsByNodeType();
292   }
293   if (options_.show_summary) {
294     stream << GetShortSummary() << std::endl;
295   }
296   return stream.str();
297 }
298 
AddNodeStats(const std::string & name,const std::string & type,int64_t run_order,int64_t start_us,int64_t rel_end_us,int64_t mem_used)299 void StatsCalculator::AddNodeStats(const std::string& name,
300                                    const std::string& type, int64_t run_order,
301                                    int64_t start_us, int64_t rel_end_us,
302                                    int64_t mem_used) {
303   Detail* detail = nullptr;
304   if (details_.find(name) == details_.end()) {
305     details_.insert({name, {}});
306     detail = &details_.at(name);
307     detail->type = type;
308     detail->name = name;
309     detail->run_order = run_order;
310   } else {
311     detail = &details_.at(name);
312   }
313   detail->start_us.UpdateStat(start_us);
314   detail->rel_end_us.UpdateStat(rel_end_us);
315   detail->mem_used.UpdateStat(mem_used);
316   detail->times_called++;
317 }
318 
319 }  // namespace tensorflow
320