1 /* 2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 11 #ifndef TEST_TESTSUPPORT_PERF_TEST_H_ 12 #define TEST_TESTSUPPORT_PERF_TEST_H_ 13 14 #include <sstream> 15 #include <string> 16 #include <vector> 17 18 #include "api/array_view.h" 19 #include "rtc_base/numerics/samples_stats_counter.h" 20 21 namespace webrtc { 22 namespace test { 23 24 enum class ImproveDirection { 25 // Direction is undefined. 26 kNone, 27 // Smaller value is better. 28 kSmallerIsBetter, 29 // Bigger value is better. 30 kBiggerIsBetter, 31 }; 32 33 // Prints a performance test result. 34 // 35 // For example, 36 // PrintResult("ramp_up_time_", "turn_over_tcp", 37 // "bwe_15s", 1234.2, "ms", false); 38 // 39 // will show up in the http://chromeperf.appspot.com under 40 // 41 // (test binary name) > (bot) > ramp_up_time_turn_over_tcp > bwe_15s. 42 // 43 // The |measurement| + |modifier| is what we're measuring. |user_story| is the 44 // scenario we're testing under. 45 // 46 // The binary this runs in must be hooked up as a perf test in the WebRTC 47 // recipes for this to actually be uploaded to chromeperf.appspot.com. 48 void PrintResult(const std::string& measurement, 49 const std::string& modifier, 50 const std::string& user_story, 51 const double value, 52 const std::string& units, 53 bool important, 54 ImproveDirection improve_direction = ImproveDirection::kNone); 55 56 // Like PrintResult(), but prints a (mean, standard deviation) result pair. 57 // The |<values>| should be two comma-separated numbers, the mean and 58 // standard deviation (or other error metric) of the measurement. 59 // DEPRECATED: soon unsupported. 60 void PrintResultMeanAndError( 61 const std::string& measurement, 62 const std::string& modifier, 63 const std::string& user_story, 64 const double mean, 65 const double error, 66 const std::string& units, 67 bool important, 68 ImproveDirection improve_direction = ImproveDirection::kNone); 69 70 // Like PrintResult(), but prints an entire list of results. The |values| 71 // will generally be a list of comma-separated numbers. A typical 72 // post-processing step might produce plots of their mean and standard 73 // deviation. 74 void PrintResultList( 75 const std::string& measurement, 76 const std::string& modifier, 77 const std::string& user_story, 78 rtc::ArrayView<const double> values, 79 const std::string& units, 80 bool important, 81 ImproveDirection improve_direction = ImproveDirection::kNone); 82 83 // Like PrintResult(), but prints a (mean, standard deviation) from stats 84 // counter. Also add specified metric to the plotable metrics output. 85 void PrintResult(const std::string& measurement, 86 const std::string& modifier, 87 const std::string& user_story, 88 const SamplesStatsCounter& counter, 89 const std::string& units, 90 const bool important, 91 ImproveDirection improve_direction = ImproveDirection::kNone); 92 93 // Returns a string-encoded proto as described in 94 // tracing/tracing/proto/histogram.proto in 95 // https://github.com/catapult-project/catapult/blob/master/. 96 // If you want to print the proto in human readable format, use 97 // tracing/bin/proto2json from third_party/catapult in your WebRTC checkout. 98 std::string GetPerfResults(); 99 100 // Print into stdout plottable metrics for further post processing. 101 // |desired_graphs| - list of metrics, that should be plotted. If empty - all 102 // available metrics will be plotted. If some of |desired_graphs| are missing 103 // they will be skipped. 104 void PrintPlottableResults(const std::vector<std::string>& desired_graphs); 105 106 // Call GetPerfResults() and write its output to a file. Returns false if we 107 // failed to write to the file. If you want to print the proto in human readable 108 // format, use tracing/bin/proto2json from third_party/catapult in your WebRTC 109 // checkout. 110 bool WritePerfResults(const std::string& output_path); 111 112 // By default, human-readable perf results are printed to stdout. Set the FILE* 113 // to where they should be printing instead. These results are not used to 114 // upload to the dashboard, however - this is only through WritePerfResults. 115 void SetPerfResultsOutput(FILE* output); 116 117 // Only for use by tests. 118 void ClearPerfResults(); 119 120 } // namespace test 121 } // namespace webrtc 122 123 #endif // TEST_TESTSUPPORT_PERF_TEST_H_ 124