• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 /*
2  *  Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
3  *
4  *  Use of this source code is governed by a BSD-style license
5  *  that can be found in the LICENSE file in the root of the source
6  *  tree. An additional intellectual property rights grant can be found
7  *  in the file PATENTS.  All contributing project authors may
8  *  be found in the AUTHORS file in the root of the source tree.
9  */
10 
11 #ifndef TEST_TESTSUPPORT_PERF_TEST_H_
12 #define TEST_TESTSUPPORT_PERF_TEST_H_
13 
14 #include <sstream>
15 #include <string>
16 #include <vector>
17 
18 #include "api/array_view.h"
19 #include "rtc_base/numerics/samples_stats_counter.h"
20 
21 namespace webrtc {
22 namespace test {
23 
24 enum class ImproveDirection {
25   // Direction is undefined.
26   kNone,
27   // Smaller value is better.
28   kSmallerIsBetter,
29   // Bigger value is better.
30   kBiggerIsBetter,
31 };
32 
33 // Prints a performance test result.
34 //
35 // For example,
36 // PrintResult("ramp_up_time_", "turn_over_tcp",
37 //             "bwe_15s", 1234.2, "ms", false);
38 //
39 // will show up in the http://chromeperf.appspot.com under
40 //
41 // (test binary name) > (bot) > ramp_up_time_turn_over_tcp > bwe_15s.
42 //
43 // The |measurement| + |modifier| is what we're measuring. |user_story| is the
44 // scenario we're testing under.
45 //
46 // The binary this runs in must be hooked up as a perf test in the WebRTC
47 // recipes for this to actually be uploaded to chromeperf.appspot.com.
48 void PrintResult(const std::string& measurement,
49                  const std::string& modifier,
50                  const std::string& user_story,
51                  const double value,
52                  const std::string& units,
53                  bool important,
54                  ImproveDirection improve_direction = ImproveDirection::kNone);
55 
56 // Like PrintResult(), but prints a (mean, standard deviation) result pair.
57 // The |<values>| should be two comma-separated numbers, the mean and
58 // standard deviation (or other error metric) of the measurement.
59 // DEPRECATED: soon unsupported.
60 void PrintResultMeanAndError(
61     const std::string& measurement,
62     const std::string& modifier,
63     const std::string& user_story,
64     const double mean,
65     const double error,
66     const std::string& units,
67     bool important,
68     ImproveDirection improve_direction = ImproveDirection::kNone);
69 
70 // Like PrintResult(), but prints an entire list of results. The |values|
71 // will generally be a list of comma-separated numbers. A typical
72 // post-processing step might produce plots of their mean and standard
73 // deviation.
74 void PrintResultList(
75     const std::string& measurement,
76     const std::string& modifier,
77     const std::string& user_story,
78     rtc::ArrayView<const double> values,
79     const std::string& units,
80     bool important,
81     ImproveDirection improve_direction = ImproveDirection::kNone);
82 
83 // Like PrintResult(), but prints a (mean, standard deviation) from stats
84 // counter. Also add specified metric to the plotable metrics output.
85 void PrintResult(const std::string& measurement,
86                  const std::string& modifier,
87                  const std::string& user_story,
88                  const SamplesStatsCounter& counter,
89                  const std::string& units,
90                  const bool important,
91                  ImproveDirection improve_direction = ImproveDirection::kNone);
92 
93 // Returns a string-encoded proto as described in
94 // tracing/tracing/proto/histogram.proto in
95 // https://github.com/catapult-project/catapult/blob/master/.
96 // If you want to print the proto in human readable format, use
97 // tracing/bin/proto2json from third_party/catapult in your WebRTC checkout.
98 std::string GetPerfResults();
99 
100 // Print into stdout plottable metrics for further post processing.
101 // |desired_graphs| - list of metrics, that should be plotted. If empty - all
102 // available metrics will be plotted. If some of |desired_graphs| are missing
103 // they will be skipped.
104 void PrintPlottableResults(const std::vector<std::string>& desired_graphs);
105 
106 // Call GetPerfResults() and write its output to a file. Returns false if we
107 // failed to write to the file. If you want to print the proto in human readable
108 // format, use tracing/bin/proto2json from third_party/catapult in your WebRTC
109 // checkout.
110 bool WritePerfResults(const std::string& output_path);
111 
112 // By default, human-readable perf results are printed to stdout. Set the FILE*
113 // to where they should be printing instead. These results are not used to
114 // upload to the dashboard, however - this is only through WritePerfResults.
115 void SetPerfResultsOutput(FILE* output);
116 
117 // Only for use by tests.
118 void ClearPerfResults();
119 
120 }  // namespace test
121 }  // namespace webrtc
122 
123 #endif  // TEST_TESTSUPPORT_PERF_TEST_H_
124