1 /* 2 * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 11 #ifndef TEST_TESTSUPPORT_PERF_RESULT_REPORTER_H_ 12 #define TEST_TESTSUPPORT_PERF_RESULT_REPORTER_H_ 13 14 #include <string> 15 #include <unordered_map> 16 17 #include "absl/types/optional.h" 18 #include "api/array_view.h" 19 #include "test/testsupport/perf_test.h" 20 21 namespace webrtc { 22 namespace test { 23 24 // These match the units in histogram.proto (in third_party/catapult). 25 enum class Unit { 26 kMs, 27 kMsBestFitFormat, 28 kMsTs, 29 kNPercent, 30 kSizeInBytes, 31 kBytesPerSecond, 32 kHertz, 33 kUnitless, 34 kCount, 35 kSigma, 36 }; 37 38 struct MetricInfo { 39 Unit unit; 40 ImproveDirection improve_direction; 41 }; 42 43 // A helper class for using the perf test printing functions safely, as 44 // otherwise it's easy to accidentally mix up arguments to produce usable but 45 // malformed perf data. See https://crbug.com/923564. 46 // 47 // Sample usage: 48 // auto reporter = PerfResultReporter("ramp_up_time", "bwe_15s"); 49 // reporter.RegisterImportantMetric( 50 // "_turn_over_tcp", Unit::kMs, ImproveDirection::kBiggerIsBetter); 51 // reporter.RegisterImportantMetric("_cpu_time", Unit::kMs); 52 // ... 53 // reporter.AddResult("turn_over_tcp", GetTurnOverTcpTime()); 54 // reporter.AddResult("turn_over_udp", GetTurnOverUdpTime()); 55 // 56 // This will show in the dashboard as 57 // (test binary name) > (bot) > ramp_up_time_turn_over_tcp > bwe_15s. 58 // (test binary name) > (bot) > ramp_up_time_turn_over_udp > bwe_15s. 59 // 60 // If you add more reporters that cover other user stories, they will show up 61 // as separate subtests (e.g. next to bwe_15s). 62 class PerfResultReporter { 63 public: 64 PerfResultReporter(const std::string& metric_basename, 65 const std::string& story_name); 66 ~PerfResultReporter(); 67 68 void RegisterMetric(const std::string& metric_suffix, Unit unit); 69 void RegisterMetric(const std::string& metric_suffix, 70 Unit unit, 71 ImproveDirection improve_direction); 72 void AddResult(const std::string& metric_suffix, size_t value) const; 73 void AddResult(const std::string& metric_suffix, double value) const; 74 75 void AddResultList(const std::string& metric_suffix, 76 rtc::ArrayView<const double> values) const; 77 78 // Users should prefer AddResultList if possible, as otherwise the min/max 79 // values reported on the perf dashboard aren't useful. 80 // |mean_and_error| should be a comma-separated string of mean then 81 // error/stddev, e.g. "2.4,0.5". 82 void AddResultMeanAndError(const std::string& metric_suffix, 83 const double mean, 84 const double error); 85 86 // Returns the metric info if it has been registered. 87 absl::optional<MetricInfo> GetMetricInfo( 88 const std::string& metric_suffix) const; 89 90 private: 91 MetricInfo GetMetricInfoOrFail(const std::string& metric_suffix) const; 92 93 std::string metric_basename_; 94 std::string story_name_; 95 std::unordered_map<std::string, MetricInfo> metric_map_; 96 }; 97 98 } // namespace test 99 } // namespace webrtc 100 101 #endif // TEST_TESTSUPPORT_PERF_RESULT_REPORTER_H_ 102