1 /*
2 * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11 #include "test/testsupport/perf_test_histogram_writer.h"
12
13 #include <stdlib.h>
14
15 #include <map>
16 #include <memory>
17
18 #include "absl/strings/string_view.h"
19 #include "api/numerics/samples_stats_counter.h"
20 #include "rtc_base/logging.h"
21 #include "rtc_base/strings/string_builder.h"
22 #include "rtc_base/synchronization/mutex.h"
23 #include "third_party/catapult/tracing/tracing/value/diagnostics/reserved_infos.h"
24 #include "third_party/catapult/tracing/tracing/value/histogram.h"
25
26 namespace webrtc {
27 namespace test {
28
29 namespace {
30
31 namespace proto = catapult::tracing::tracing::proto;
32
AsJsonString(const std::string string)33 std::string AsJsonString(const std::string string) {
34 return "\"" + string + "\"";
35 }
36
37 class PerfTestHistogramWriter : public PerfTestResultWriter {
38 public:
PerfTestHistogramWriter()39 PerfTestHistogramWriter() : mutex_() {}
ClearResults()40 void ClearResults() override {
41 MutexLock lock(&mutex_);
42 histograms_.clear();
43 }
44
LogResult(absl::string_view graph_name,absl::string_view trace_name,const double value,absl::string_view units,const bool important,ImproveDirection improve_direction)45 void LogResult(absl::string_view graph_name,
46 absl::string_view trace_name,
47 const double value,
48 absl::string_view units,
49 const bool important,
50 ImproveDirection improve_direction) override {
51 (void)important;
52 AddSample(graph_name, trace_name, value, units, improve_direction);
53 }
LogResultMeanAndError(absl::string_view graph_name,absl::string_view trace_name,const double mean,const double error,absl::string_view units,const bool important,ImproveDirection improve_direction)54 void LogResultMeanAndError(absl::string_view graph_name,
55 absl::string_view trace_name,
56 const double mean,
57 const double error,
58 absl::string_view units,
59 const bool important,
60 ImproveDirection improve_direction) override {
61 RTC_LOG(LS_WARNING) << "Discarding stddev, not supported by histograms";
62 (void)error;
63 (void)important;
64
65 AddSample(graph_name, trace_name, mean, units, improve_direction);
66 }
LogResultList(absl::string_view graph_name,absl::string_view trace_name,const rtc::ArrayView<const double> values,absl::string_view units,const bool important,ImproveDirection improve_direction)67 void LogResultList(absl::string_view graph_name,
68 absl::string_view trace_name,
69 const rtc::ArrayView<const double> values,
70 absl::string_view units,
71 const bool important,
72 ImproveDirection improve_direction) override {
73 (void)important;
74 for (double value : values) {
75 AddSample(graph_name, trace_name, value, units, improve_direction);
76 }
77 }
Serialize() const78 std::string Serialize() const override {
79 proto::HistogramSet histogram_set;
80
81 MutexLock lock(&mutex_);
82 for (const auto& histogram : histograms_) {
83 std::unique_ptr<proto::Histogram> proto = histogram.second->toProto();
84 histogram_set.mutable_histograms()->AddAllocated(proto.release());
85 }
86
87 std::string output;
88 bool ok = histogram_set.SerializeToString(&output);
89 RTC_DCHECK(ok) << "Failed to serialize histogram set to string";
90 return output;
91 }
92
93 private:
AddSample(absl::string_view original_graph_name,absl::string_view trace_name,const double value,absl::string_view units,ImproveDirection improve_direction)94 void AddSample(absl::string_view original_graph_name,
95 absl::string_view trace_name,
96 const double value,
97 absl::string_view units,
98 ImproveDirection improve_direction) {
99 // WebRTC annotates the units into the metric name when they are not
100 // supported by the Histogram API.
101 std::string graph_name(original_graph_name);
102 if (units == "dB") {
103 graph_name += "_dB";
104 } else if (units == "fps") {
105 graph_name += "_fps";
106 } else if (units == "%") {
107 graph_name += "_%";
108 }
109
110 // Lookup on graph name + trace name (or measurement + story in catapult
111 // parlance). There should be several histograms with the same measurement
112 // if they're for different stories.
113 rtc::StringBuilder measurement_and_story;
114 measurement_and_story << graph_name << trace_name;
115 MutexLock lock(&mutex_);
116 if (histograms_.count(measurement_and_story.str()) == 0) {
117 proto::UnitAndDirection unit = ParseUnit(units, improve_direction);
118 std::unique_ptr<catapult::HistogramBuilder> builder =
119 std::make_unique<catapult::HistogramBuilder>(graph_name, unit);
120
121 // Set all summary options as false - we don't want to generate
122 // metric_std, metric_count, and so on for all metrics.
123 builder->SetSummaryOptions(proto::SummaryOptions());
124 histograms_[measurement_and_story.str()] = std::move(builder);
125
126 proto::Diagnostic stories;
127 proto::GenericSet* generic_set = stories.mutable_generic_set();
128 generic_set->add_values(AsJsonString(std::string(trace_name)));
129 histograms_[measurement_and_story.str()]->AddDiagnostic(
130 catapult::kStoriesDiagnostic, stories);
131 }
132
133 if (units == "bps") {
134 // Bps has been interpreted as bits per second in WebRTC tests.
135 histograms_[measurement_and_story.str()]->AddSample(value / 8);
136 } else {
137 histograms_[measurement_and_story.str()]->AddSample(value);
138 }
139 }
140
ParseUnit(absl::string_view units,ImproveDirection improve_direction)141 proto::UnitAndDirection ParseUnit(absl::string_view units,
142 ImproveDirection improve_direction) {
143 RTC_DCHECK(units.find('_') == std::string::npos)
144 << "The unit_bigger|smallerIsBetter syntax isn't supported in WebRTC, "
145 "use the enum instead.";
146
147 proto::UnitAndDirection result;
148 result.set_improvement_direction(ParseDirection(improve_direction));
149 if (units == "bps") {
150 result.set_unit(proto::BYTES_PER_SECOND);
151 } else if (units == "dB") {
152 result.set_unit(proto::UNITLESS);
153 } else if (units == "fps") {
154 result.set_unit(proto::HERTZ);
155 } else if (units == "frames") {
156 result.set_unit(proto::COUNT);
157 } else if (units == "ms") {
158 result.set_unit(proto::MS_BEST_FIT_FORMAT);
159 } else if (units == "%") {
160 result.set_unit(proto::UNITLESS);
161 } else {
162 proto::Unit unit = catapult::UnitFromJsonUnit(std::string(units));
163
164 // UnitFromJsonUnit returns UNITLESS if it doesn't recognize the unit.
165 if (unit == proto::UNITLESS && units != "unitless") {
166 RTC_LOG(LS_WARNING) << "Unit " << units << " is unsupported.";
167 }
168
169 result.set_unit(unit);
170 }
171 return result;
172 }
173
ParseDirection(ImproveDirection improve_direction)174 proto::ImprovementDirection ParseDirection(
175 ImproveDirection improve_direction) {
176 switch (improve_direction) {
177 case ImproveDirection::kNone:
178 return proto::NOT_SPECIFIED;
179 case ImproveDirection::kSmallerIsBetter:
180 return proto::SMALLER_IS_BETTER;
181 case ImproveDirection::kBiggerIsBetter:
182 return proto::BIGGER_IS_BETTER;
183 default:
184 RTC_DCHECK_NOTREACHED() << "Invalid enum value " << improve_direction;
185 }
186 }
187
188 private:
189 mutable Mutex mutex_;
190 std::map<std::string, std::unique_ptr<catapult::HistogramBuilder>> histograms_
191 RTC_GUARDED_BY(&mutex_);
192 };
193
194 } // namespace
195
CreateHistogramWriter()196 PerfTestResultWriter* CreateHistogramWriter() {
197 return new PerfTestHistogramWriter();
198 }
199
200 } // namespace test
201 } // namespace webrtc
202