1 /*
2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11 #include "test/testsupport/perf_test.h"
12
13 #include <stdio.h>
14
15 #include <fstream>
16 #include <set>
17 #include <sstream>
18 #include <vector>
19
20 #include "rtc_base/checks.h"
21 #include "rtc_base/synchronization/mutex.h"
22 #include "test/testsupport/perf_test_histogram_writer.h"
23
24 namespace webrtc {
25 namespace test {
26
27 namespace {
28
UnitWithDirection(const std::string & units,webrtc::test::ImproveDirection improve_direction)29 std::string UnitWithDirection(
30 const std::string& units,
31 webrtc::test::ImproveDirection improve_direction) {
32 switch (improve_direction) {
33 case webrtc::test::ImproveDirection::kNone:
34 return units;
35 case webrtc::test::ImproveDirection::kSmallerIsBetter:
36 return units + "_smallerIsBetter";
37 case webrtc::test::ImproveDirection::kBiggerIsBetter:
38 return units + "_biggerIsBetter";
39 }
40 }
41
42 template <typename Container>
OutputListToStream(std::ostream * ostream,const Container & values)43 void OutputListToStream(std::ostream* ostream, const Container& values) {
44 const char* sep = "";
45 for (const auto& v : values) {
46 (*ostream) << sep << v;
47 sep = ",";
48 }
49 }
50
51 struct PlottableCounter {
52 std::string graph_name;
53 std::string trace_name;
54 webrtc::SamplesStatsCounter counter;
55 std::string units;
56 };
57
58 class PlottableCounterPrinter {
59 public:
PlottableCounterPrinter()60 PlottableCounterPrinter() : output_(stdout) {}
61
SetOutput(FILE * output)62 void SetOutput(FILE* output) {
63 MutexLock lock(&mutex_);
64 output_ = output;
65 }
66
AddCounter(const std::string & graph_name,const std::string & trace_name,const webrtc::SamplesStatsCounter & counter,const std::string & units)67 void AddCounter(const std::string& graph_name,
68 const std::string& trace_name,
69 const webrtc::SamplesStatsCounter& counter,
70 const std::string& units) {
71 MutexLock lock(&mutex_);
72 plottable_counters_.push_back({graph_name, trace_name, counter, units});
73 }
74
Print(const std::vector<std::string> & desired_graphs_raw) const75 void Print(const std::vector<std::string>& desired_graphs_raw) const {
76 std::set<std::string> desired_graphs(desired_graphs_raw.begin(),
77 desired_graphs_raw.end());
78 MutexLock lock(&mutex_);
79 for (auto& counter : plottable_counters_) {
80 if (!desired_graphs.empty()) {
81 auto it = desired_graphs.find(counter.graph_name);
82 if (it == desired_graphs.end()) {
83 continue;
84 }
85 }
86
87 std::ostringstream value_stream;
88 value_stream.precision(8);
89 value_stream << R"({"graph_name":")" << counter.graph_name << R"(",)";
90 value_stream << R"("trace_name":")" << counter.trace_name << R"(",)";
91 value_stream << R"("units":")" << counter.units << R"(",)";
92 if (!counter.counter.IsEmpty()) {
93 value_stream << R"("mean":)" << counter.counter.GetAverage() << ',';
94 value_stream << R"("std":)" << counter.counter.GetStandardDeviation()
95 << ',';
96 }
97 value_stream << R"("samples":[)";
98 const char* sep = "";
99 for (const auto& sample : counter.counter.GetTimedSamples()) {
100 value_stream << sep << R"({"time":)" << sample.time.us() << ','
101 << R"("value":)" << sample.value << '}';
102 sep = ",";
103 }
104 value_stream << "]}";
105
106 fprintf(output_, "PLOTTABLE_DATA: %s\n", value_stream.str().c_str());
107 }
108 }
109
110 private:
111 mutable Mutex mutex_;
112 std::vector<PlottableCounter> plottable_counters_ RTC_GUARDED_BY(&mutex_);
113 FILE* output_ RTC_GUARDED_BY(&mutex_);
114 };
115
GetPlottableCounterPrinter()116 PlottableCounterPrinter& GetPlottableCounterPrinter() {
117 static PlottableCounterPrinter* printer_ = new PlottableCounterPrinter();
118 return *printer_;
119 }
120
121 class ResultsLinePrinter {
122 public:
ResultsLinePrinter()123 ResultsLinePrinter() : output_(stdout) {}
124
SetOutput(FILE * output)125 void SetOutput(FILE* output) {
126 MutexLock lock(&mutex_);
127 output_ = output;
128 }
129
PrintResult(const std::string & graph_name,const std::string & trace_name,const double value,const std::string & units,bool important,ImproveDirection improve_direction)130 void PrintResult(const std::string& graph_name,
131 const std::string& trace_name,
132 const double value,
133 const std::string& units,
134 bool important,
135 ImproveDirection improve_direction) {
136 std::ostringstream value_stream;
137 value_stream.precision(8);
138 value_stream << value;
139
140 PrintResultImpl(graph_name, trace_name, value_stream.str(), std::string(),
141 std::string(), UnitWithDirection(units, improve_direction),
142 important);
143 }
144
PrintResultMeanAndError(const std::string & graph_name,const std::string & trace_name,const double mean,const double error,const std::string & units,bool important,ImproveDirection improve_direction)145 void PrintResultMeanAndError(const std::string& graph_name,
146 const std::string& trace_name,
147 const double mean,
148 const double error,
149 const std::string& units,
150 bool important,
151 ImproveDirection improve_direction) {
152 std::ostringstream value_stream;
153 value_stream.precision(8);
154 value_stream << mean << ',' << error;
155 PrintResultImpl(graph_name, trace_name, value_stream.str(), "{", "}",
156 UnitWithDirection(units, improve_direction), important);
157 }
158
PrintResultList(const std::string & graph_name,const std::string & trace_name,const rtc::ArrayView<const double> values,const std::string & units,const bool important,webrtc::test::ImproveDirection improve_direction)159 void PrintResultList(const std::string& graph_name,
160 const std::string& trace_name,
161 const rtc::ArrayView<const double> values,
162 const std::string& units,
163 const bool important,
164 webrtc::test::ImproveDirection improve_direction) {
165 std::ostringstream value_stream;
166 value_stream.precision(8);
167 OutputListToStream(&value_stream, values);
168 PrintResultImpl(graph_name, trace_name, value_stream.str(), "[", "]", units,
169 important);
170 }
171
172 private:
PrintResultImpl(const std::string & graph_name,const std::string & trace_name,const std::string & values,const std::string & prefix,const std::string & suffix,const std::string & units,bool important)173 void PrintResultImpl(const std::string& graph_name,
174 const std::string& trace_name,
175 const std::string& values,
176 const std::string& prefix,
177 const std::string& suffix,
178 const std::string& units,
179 bool important) {
180 MutexLock lock(&mutex_);
181 // <*>RESULT <graph_name>: <trace_name>= <value> <units>
182 // <*>RESULT <graph_name>: <trace_name>= {<mean>, <std deviation>} <units>
183 // <*>RESULT <graph_name>: <trace_name>= [<value>,value,value,...,] <units>
184 fprintf(output_, "%sRESULT %s: %s= %s%s%s %s\n", important ? "*" : "",
185 graph_name.c_str(), trace_name.c_str(), prefix.c_str(),
186 values.c_str(), suffix.c_str(), units.c_str());
187 }
188
189 Mutex mutex_;
190 FILE* output_ RTC_GUARDED_BY(&mutex_);
191 };
192
GetResultsLinePrinter()193 ResultsLinePrinter& GetResultsLinePrinter() {
194 static ResultsLinePrinter* const printer_ = new ResultsLinePrinter();
195 return *printer_;
196 }
197
GetPerfWriter()198 PerfTestResultWriter& GetPerfWriter() {
199 static PerfTestResultWriter* writer = CreateHistogramWriter();
200 return *writer;
201 }
202
203 } // namespace
204
ClearPerfResults()205 void ClearPerfResults() {
206 GetPerfWriter().ClearResults();
207 }
208
SetPerfResultsOutput(FILE * output)209 void SetPerfResultsOutput(FILE* output) {
210 GetPlottableCounterPrinter().SetOutput(output);
211 GetResultsLinePrinter().SetOutput(output);
212 }
213
GetPerfResults()214 std::string GetPerfResults() {
215 return GetPerfWriter().Serialize();
216 }
217
PrintPlottableResults(const std::vector<std::string> & desired_graphs)218 void PrintPlottableResults(const std::vector<std::string>& desired_graphs) {
219 GetPlottableCounterPrinter().Print(desired_graphs);
220 }
221
WritePerfResults(const std::string & output_path)222 bool WritePerfResults(const std::string& output_path) {
223 std::string results = GetPerfResults();
224 FILE* output = fopen(output_path.c_str(), "wb");
225 if (output == NULL) {
226 printf("Failed to write to %s.\n", output_path.c_str());
227 return false;
228 }
229 size_t written =
230 fwrite(results.c_str(), sizeof(char), results.size(), output);
231 fclose(output);
232
233 if (written != results.size()) {
234 long expected = results.size();
235 printf("Wrote %zu, tried to write %lu\n", written, expected);
236 return false;
237 }
238
239 return true;
240 }
241
PrintResult(const std::string & measurement,const std::string & modifier,const std::string & trace,const double value,const std::string & units,bool important,ImproveDirection improve_direction)242 void PrintResult(const std::string& measurement,
243 const std::string& modifier,
244 const std::string& trace,
245 const double value,
246 const std::string& units,
247 bool important,
248 ImproveDirection improve_direction) {
249 std::string graph_name = measurement + modifier;
250 RTC_CHECK(std::isfinite(value))
251 << "Expected finite value for graph " << graph_name << ", trace name "
252 << trace << ", units " << units << ", got " << value;
253 GetPerfWriter().LogResult(graph_name, trace, value, units, important,
254 improve_direction);
255 GetResultsLinePrinter().PrintResult(graph_name, trace, value, units,
256 important, improve_direction);
257 }
258
PrintResult(const std::string & measurement,const std::string & modifier,const std::string & trace,const SamplesStatsCounter & counter,const std::string & units,const bool important,ImproveDirection improve_direction)259 void PrintResult(const std::string& measurement,
260 const std::string& modifier,
261 const std::string& trace,
262 const SamplesStatsCounter& counter,
263 const std::string& units,
264 const bool important,
265 ImproveDirection improve_direction) {
266 std::string graph_name = measurement + modifier;
267 GetPlottableCounterPrinter().AddCounter(graph_name, trace, counter, units);
268
269 double mean = counter.IsEmpty() ? 0 : counter.GetAverage();
270 double error = counter.IsEmpty() ? 0 : counter.GetStandardDeviation();
271 PrintResultMeanAndError(measurement, modifier, trace, mean, error, units,
272 important, improve_direction);
273 }
274
PrintResultMeanAndError(const std::string & measurement,const std::string & modifier,const std::string & trace,const double mean,const double error,const std::string & units,bool important,ImproveDirection improve_direction)275 void PrintResultMeanAndError(const std::string& measurement,
276 const std::string& modifier,
277 const std::string& trace,
278 const double mean,
279 const double error,
280 const std::string& units,
281 bool important,
282 ImproveDirection improve_direction) {
283 RTC_CHECK(std::isfinite(mean));
284 RTC_CHECK(std::isfinite(error));
285
286 std::string graph_name = measurement + modifier;
287 GetPerfWriter().LogResultMeanAndError(graph_name, trace, mean, error, units,
288 important, improve_direction);
289 GetResultsLinePrinter().PrintResultMeanAndError(
290 graph_name, trace, mean, error, units, important, improve_direction);
291 }
292
PrintResultList(const std::string & measurement,const std::string & modifier,const std::string & trace,const rtc::ArrayView<const double> values,const std::string & units,bool important,ImproveDirection improve_direction)293 void PrintResultList(const std::string& measurement,
294 const std::string& modifier,
295 const std::string& trace,
296 const rtc::ArrayView<const double> values,
297 const std::string& units,
298 bool important,
299 ImproveDirection improve_direction) {
300 for (double v : values) {
301 RTC_CHECK(std::isfinite(v));
302 }
303
304 std::string graph_name = measurement + modifier;
305 GetPerfWriter().LogResultList(graph_name, trace, values, units, important,
306 improve_direction);
307 GetResultsLinePrinter().PrintResultList(graph_name, trace, values, units,
308 important, improve_direction);
309 }
310
311 } // namespace test
312 } // namespace webrtc
313