• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 /*
2  *  Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
3  *
4  *  Use of this source code is governed by a BSD-style license
5  *  that can be found in the LICENSE file in the root of the source
6  *  tree. An additional intellectual property rights grant can be found
7  *  in the file PATENTS.  All contributing project authors may
8  *  be found in the AUTHORS file in the root of the source tree.
9  */
10 
11 #include "test/testsupport/perf_test.h"
12 
13 #include <stdio.h>
14 
15 #include <fstream>
16 #include <set>
17 #include <sstream>
18 #include <vector>
19 
20 #include "rtc_base/checks.h"
21 #include "rtc_base/synchronization/mutex.h"
22 #include "test/testsupport/perf_test_histogram_writer.h"
23 
24 namespace webrtc {
25 namespace test {
26 
27 namespace {
28 
UnitWithDirection(const std::string & units,webrtc::test::ImproveDirection improve_direction)29 std::string UnitWithDirection(
30     const std::string& units,
31     webrtc::test::ImproveDirection improve_direction) {
32   switch (improve_direction) {
33     case webrtc::test::ImproveDirection::kNone:
34       return units;
35     case webrtc::test::ImproveDirection::kSmallerIsBetter:
36       return units + "_smallerIsBetter";
37     case webrtc::test::ImproveDirection::kBiggerIsBetter:
38       return units + "_biggerIsBetter";
39   }
40 }
41 
42 template <typename Container>
OutputListToStream(std::ostream * ostream,const Container & values)43 void OutputListToStream(std::ostream* ostream, const Container& values) {
44   const char* sep = "";
45   for (const auto& v : values) {
46     (*ostream) << sep << v;
47     sep = ",";
48   }
49 }
50 
51 struct PlottableCounter {
52   std::string graph_name;
53   std::string trace_name;
54   webrtc::SamplesStatsCounter counter;
55   std::string units;
56 };
57 
58 class PlottableCounterPrinter {
59  public:
PlottableCounterPrinter()60   PlottableCounterPrinter() : output_(stdout) {}
61 
SetOutput(FILE * output)62   void SetOutput(FILE* output) {
63     MutexLock lock(&mutex_);
64     output_ = output;
65   }
66 
AddCounter(const std::string & graph_name,const std::string & trace_name,const webrtc::SamplesStatsCounter & counter,const std::string & units)67   void AddCounter(const std::string& graph_name,
68                   const std::string& trace_name,
69                   const webrtc::SamplesStatsCounter& counter,
70                   const std::string& units) {
71     MutexLock lock(&mutex_);
72     plottable_counters_.push_back({graph_name, trace_name, counter, units});
73   }
74 
Print(const std::vector<std::string> & desired_graphs_raw) const75   void Print(const std::vector<std::string>& desired_graphs_raw) const {
76     std::set<std::string> desired_graphs(desired_graphs_raw.begin(),
77                                          desired_graphs_raw.end());
78     MutexLock lock(&mutex_);
79     for (auto& counter : plottable_counters_) {
80       if (!desired_graphs.empty()) {
81         auto it = desired_graphs.find(counter.graph_name);
82         if (it == desired_graphs.end()) {
83           continue;
84         }
85       }
86 
87       std::ostringstream value_stream;
88       value_stream.precision(8);
89       value_stream << R"({"graph_name":")" << counter.graph_name << R"(",)";
90       value_stream << R"("trace_name":")" << counter.trace_name << R"(",)";
91       value_stream << R"("units":")" << counter.units << R"(",)";
92       if (!counter.counter.IsEmpty()) {
93         value_stream << R"("mean":)" << counter.counter.GetAverage() << ',';
94         value_stream << R"("std":)" << counter.counter.GetStandardDeviation()
95                      << ',';
96       }
97       value_stream << R"("samples":[)";
98       const char* sep = "";
99       for (const auto& sample : counter.counter.GetTimedSamples()) {
100         value_stream << sep << R"({"time":)" << sample.time.us() << ','
101                      << R"("value":)" << sample.value << '}';
102         sep = ",";
103       }
104       value_stream << "]}";
105 
106       fprintf(output_, "PLOTTABLE_DATA: %s\n", value_stream.str().c_str());
107     }
108   }
109 
110  private:
111   mutable Mutex mutex_;
112   std::vector<PlottableCounter> plottable_counters_ RTC_GUARDED_BY(&mutex_);
113   FILE* output_ RTC_GUARDED_BY(&mutex_);
114 };
115 
GetPlottableCounterPrinter()116 PlottableCounterPrinter& GetPlottableCounterPrinter() {
117   static PlottableCounterPrinter* printer_ = new PlottableCounterPrinter();
118   return *printer_;
119 }
120 
121 class ResultsLinePrinter {
122  public:
ResultsLinePrinter()123   ResultsLinePrinter() : output_(stdout) {}
124 
SetOutput(FILE * output)125   void SetOutput(FILE* output) {
126     MutexLock lock(&mutex_);
127     output_ = output;
128   }
129 
PrintResult(const std::string & graph_name,const std::string & trace_name,const double value,const std::string & units,bool important,ImproveDirection improve_direction)130   void PrintResult(const std::string& graph_name,
131                    const std::string& trace_name,
132                    const double value,
133                    const std::string& units,
134                    bool important,
135                    ImproveDirection improve_direction) {
136     std::ostringstream value_stream;
137     value_stream.precision(8);
138     value_stream << value;
139 
140     PrintResultImpl(graph_name, trace_name, value_stream.str(), std::string(),
141                     std::string(), UnitWithDirection(units, improve_direction),
142                     important);
143   }
144 
PrintResultMeanAndError(const std::string & graph_name,const std::string & trace_name,const double mean,const double error,const std::string & units,bool important,ImproveDirection improve_direction)145   void PrintResultMeanAndError(const std::string& graph_name,
146                                const std::string& trace_name,
147                                const double mean,
148                                const double error,
149                                const std::string& units,
150                                bool important,
151                                ImproveDirection improve_direction) {
152     std::ostringstream value_stream;
153     value_stream.precision(8);
154     value_stream << mean << ',' << error;
155     PrintResultImpl(graph_name, trace_name, value_stream.str(), "{", "}",
156                     UnitWithDirection(units, improve_direction), important);
157   }
158 
PrintResultList(const std::string & graph_name,const std::string & trace_name,const rtc::ArrayView<const double> values,const std::string & units,const bool important,webrtc::test::ImproveDirection improve_direction)159   void PrintResultList(const std::string& graph_name,
160                        const std::string& trace_name,
161                        const rtc::ArrayView<const double> values,
162                        const std::string& units,
163                        const bool important,
164                        webrtc::test::ImproveDirection improve_direction) {
165     std::ostringstream value_stream;
166     value_stream.precision(8);
167     OutputListToStream(&value_stream, values);
168     PrintResultImpl(graph_name, trace_name, value_stream.str(), "[", "]", units,
169                     important);
170   }
171 
172  private:
PrintResultImpl(const std::string & graph_name,const std::string & trace_name,const std::string & values,const std::string & prefix,const std::string & suffix,const std::string & units,bool important)173   void PrintResultImpl(const std::string& graph_name,
174                        const std::string& trace_name,
175                        const std::string& values,
176                        const std::string& prefix,
177                        const std::string& suffix,
178                        const std::string& units,
179                        bool important) {
180     MutexLock lock(&mutex_);
181     // <*>RESULT <graph_name>: <trace_name>= <value> <units>
182     // <*>RESULT <graph_name>: <trace_name>= {<mean>, <std deviation>} <units>
183     // <*>RESULT <graph_name>: <trace_name>= [<value>,value,value,...,] <units>
184     fprintf(output_, "%sRESULT %s: %s= %s%s%s %s\n", important ? "*" : "",
185             graph_name.c_str(), trace_name.c_str(), prefix.c_str(),
186             values.c_str(), suffix.c_str(), units.c_str());
187   }
188 
189   Mutex mutex_;
190   FILE* output_ RTC_GUARDED_BY(&mutex_);
191 };
192 
GetResultsLinePrinter()193 ResultsLinePrinter& GetResultsLinePrinter() {
194   static ResultsLinePrinter* const printer_ = new ResultsLinePrinter();
195   return *printer_;
196 }
197 
GetPerfWriter()198 PerfTestResultWriter& GetPerfWriter() {
199   static PerfTestResultWriter* writer = CreateHistogramWriter();
200   return *writer;
201 }
202 
203 }  // namespace
204 
ClearPerfResults()205 void ClearPerfResults() {
206   GetPerfWriter().ClearResults();
207 }
208 
SetPerfResultsOutput(FILE * output)209 void SetPerfResultsOutput(FILE* output) {
210   GetPlottableCounterPrinter().SetOutput(output);
211   GetResultsLinePrinter().SetOutput(output);
212 }
213 
GetPerfResults()214 std::string GetPerfResults() {
215   return GetPerfWriter().Serialize();
216 }
217 
PrintPlottableResults(const std::vector<std::string> & desired_graphs)218 void PrintPlottableResults(const std::vector<std::string>& desired_graphs) {
219   GetPlottableCounterPrinter().Print(desired_graphs);
220 }
221 
WritePerfResults(const std::string & output_path)222 bool WritePerfResults(const std::string& output_path) {
223   std::string results = GetPerfResults();
224   FILE* output = fopen(output_path.c_str(), "wb");
225   if (output == NULL) {
226     printf("Failed to write to %s.\n", output_path.c_str());
227     return false;
228   }
229   size_t written =
230       fwrite(results.c_str(), sizeof(char), results.size(), output);
231   fclose(output);
232 
233   if (written != results.size()) {
234     long expected = results.size();
235     printf("Wrote %zu, tried to write %lu\n", written, expected);
236     return false;
237   }
238 
239   return true;
240 }
241 
PrintResult(const std::string & measurement,const std::string & modifier,const std::string & trace,const double value,const std::string & units,bool important,ImproveDirection improve_direction)242 void PrintResult(const std::string& measurement,
243                  const std::string& modifier,
244                  const std::string& trace,
245                  const double value,
246                  const std::string& units,
247                  bool important,
248                  ImproveDirection improve_direction) {
249   std::string graph_name = measurement + modifier;
250   RTC_CHECK(std::isfinite(value))
251       << "Expected finite value for graph " << graph_name << ", trace name "
252       << trace << ", units " << units << ", got " << value;
253   GetPerfWriter().LogResult(graph_name, trace, value, units, important,
254                             improve_direction);
255   GetResultsLinePrinter().PrintResult(graph_name, trace, value, units,
256                                       important, improve_direction);
257 }
258 
PrintResult(const std::string & measurement,const std::string & modifier,const std::string & trace,const SamplesStatsCounter & counter,const std::string & units,const bool important,ImproveDirection improve_direction)259 void PrintResult(const std::string& measurement,
260                  const std::string& modifier,
261                  const std::string& trace,
262                  const SamplesStatsCounter& counter,
263                  const std::string& units,
264                  const bool important,
265                  ImproveDirection improve_direction) {
266   std::string graph_name = measurement + modifier;
267   GetPlottableCounterPrinter().AddCounter(graph_name, trace, counter, units);
268 
269   double mean = counter.IsEmpty() ? 0 : counter.GetAverage();
270   double error = counter.IsEmpty() ? 0 : counter.GetStandardDeviation();
271   PrintResultMeanAndError(measurement, modifier, trace, mean, error, units,
272                           important, improve_direction);
273 }
274 
PrintResultMeanAndError(const std::string & measurement,const std::string & modifier,const std::string & trace,const double mean,const double error,const std::string & units,bool important,ImproveDirection improve_direction)275 void PrintResultMeanAndError(const std::string& measurement,
276                              const std::string& modifier,
277                              const std::string& trace,
278                              const double mean,
279                              const double error,
280                              const std::string& units,
281                              bool important,
282                              ImproveDirection improve_direction) {
283   RTC_CHECK(std::isfinite(mean));
284   RTC_CHECK(std::isfinite(error));
285 
286   std::string graph_name = measurement + modifier;
287   GetPerfWriter().LogResultMeanAndError(graph_name, trace, mean, error, units,
288                                         important, improve_direction);
289   GetResultsLinePrinter().PrintResultMeanAndError(
290       graph_name, trace, mean, error, units, important, improve_direction);
291 }
292 
PrintResultList(const std::string & measurement,const std::string & modifier,const std::string & trace,const rtc::ArrayView<const double> values,const std::string & units,bool important,ImproveDirection improve_direction)293 void PrintResultList(const std::string& measurement,
294                      const std::string& modifier,
295                      const std::string& trace,
296                      const rtc::ArrayView<const double> values,
297                      const std::string& units,
298                      bool important,
299                      ImproveDirection improve_direction) {
300   for (double v : values) {
301     RTC_CHECK(std::isfinite(v));
302   }
303 
304   std::string graph_name = measurement + modifier;
305   GetPerfWriter().LogResultList(graph_name, trace, values, units, important,
306                                 improve_direction);
307   GetResultsLinePrinter().PrintResultList(graph_name, trace, values, units,
308                                           important, improve_direction);
309 }
310 
311 }  // namespace test
312 }  // namespace webrtc
313