1 // Copyright 2015 Google Inc. All rights reserved.
2 //
3 // Licensed under the Apache License, Version 2.0 (the "License");
4 // you may not use this file except in compliance with the License.
5 // You may obtain a copy of the License at
6 //
7 // http://www.apache.org/licenses/LICENSE-2.0
8 //
9 // Unless required by applicable law or agreed to in writing, software
10 // distributed under the License is distributed on an "AS IS" BASIS,
11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 // See the License for the specific language governing permissions and
13 // limitations under the License.
14 #ifndef BENCHMARK_REPORTER_H_
15 #define BENCHMARK_REPORTER_H_
16
17 #include <cassert>
18 #include <iosfwd>
19 #include <string>
20 #include <utility>
21 #include <vector>
22 #include <set>
23
24 #include "benchmark_api.h" // For forward declaration of BenchmarkReporter
25
26 namespace benchmark {
27
28 // Interface for custom benchmark result printers.
29 // By default, benchmark reports are printed to stdout. However an application
30 // can control the destination of the reports by calling
31 // RunSpecifiedBenchmarks and passing it a custom reporter object.
32 // The reporter object must implement the following interface.
33 class BenchmarkReporter {
34 public:
35 struct Context {
36 int num_cpus;
37 double mhz_per_cpu;
38 bool cpu_scaling_enabled;
39
40 // The number of chars in the longest benchmark name.
41 size_t name_field_width;
42 };
43
44 struct Run {
RunRun45 Run()
46 : error_occurred(false),
47 iterations(1),
48 time_unit(kNanosecond),
49 real_accumulated_time(0),
50 cpu_accumulated_time(0),
51 bytes_per_second(0),
52 items_per_second(0),
53 max_heapbytes_used(0),
54 complexity(oNone),
55 complexity_lambda(),
56 complexity_n(0),
57 report_big_o(false),
58 report_rms(false),
59 counters() {}
60
61 std::string benchmark_name;
62 std::string report_label; // Empty if not set by benchmark.
63 bool error_occurred;
64 std::string error_message;
65
66 int64_t iterations;
67 TimeUnit time_unit;
68 double real_accumulated_time;
69 double cpu_accumulated_time;
70
71 // Return a value representing the real time per iteration in the unit
72 // specified by 'time_unit'.
73 // NOTE: If 'iterations' is zero the returned value represents the
74 // accumulated time.
75 double GetAdjustedRealTime() const;
76
77 // Return a value representing the cpu time per iteration in the unit
78 // specified by 'time_unit'.
79 // NOTE: If 'iterations' is zero the returned value represents the
80 // accumulated time.
81 double GetAdjustedCPUTime() const;
82
83 // Zero if not set by benchmark.
84 double bytes_per_second;
85 double items_per_second;
86
87 // This is set to 0.0 if memory tracing is not enabled.
88 double max_heapbytes_used;
89
90 // Keep track of arguments to compute asymptotic complexity
91 BigO complexity;
92 BigOFunc* complexity_lambda;
93 int complexity_n;
94
95 // Inform print function whether the current run is a complexity report
96 bool report_big_o;
97 bool report_rms;
98
99 UserCounters counters;
100 };
101
102 // Construct a BenchmarkReporter with the output stream set to 'std::cout'
103 // and the error stream set to 'std::cerr'
104 BenchmarkReporter();
105
106 // Called once for every suite of benchmarks run.
107 // The parameter "context" contains information that the
108 // reporter may wish to use when generating its report, for example the
109 // platform under which the benchmarks are running. The benchmark run is
110 // never started if this function returns false, allowing the reporter
111 // to skip runs based on the context information.
112 virtual bool ReportContext(const Context& context) = 0;
113
114 // Called once for each group of benchmark runs, gives information about
115 // cpu-time and heap memory usage during the benchmark run. If the group
116 // of runs contained more than two entries then 'report' contains additional
117 // elements representing the mean and standard deviation of those runs.
118 // Additionally if this group of runs was the last in a family of benchmarks
119 // 'reports' contains additional entries representing the asymptotic
120 // complexity and RMS of that benchmark family.
121 virtual void ReportRuns(const std::vector<Run>& report) = 0;
122
123 // Called once and only once after ever group of benchmarks is run and
124 // reported.
Finalize()125 virtual void Finalize() {}
126
127 // REQUIRES: The object referenced by 'out' is valid for the lifetime
128 // of the reporter.
SetOutputStream(std::ostream * out)129 void SetOutputStream(std::ostream* out) {
130 assert(out);
131 output_stream_ = out;
132 }
133
134 // REQUIRES: The object referenced by 'err' is valid for the lifetime
135 // of the reporter.
SetErrorStream(std::ostream * err)136 void SetErrorStream(std::ostream* err) {
137 assert(err);
138 error_stream_ = err;
139 }
140
GetOutputStream()141 std::ostream& GetOutputStream() const { return *output_stream_; }
142
GetErrorStream()143 std::ostream& GetErrorStream() const { return *error_stream_; }
144
145 virtual ~BenchmarkReporter();
146
147 // Write a human readable string to 'out' representing the specified
148 // 'context'.
149 // REQUIRES: 'out' is non-null.
150 static void PrintBasicContext(std::ostream* out, Context const& context);
151
152 private:
153 std::ostream* output_stream_;
154 std::ostream* error_stream_;
155 };
156
157 // Simple reporter that outputs benchmark data to the console. This is the
158 // default reporter used by RunSpecifiedBenchmarks().
159 class ConsoleReporter : public BenchmarkReporter {
160 public:
161 enum OutputOptions {
162 OO_None = 0,
163 OO_Color = 1,
164 OO_Tabular = 2,
165 OO_ColorTabular = OO_Color|OO_Tabular,
166 OO_Defaults = OO_ColorTabular
167 };
168 explicit ConsoleReporter(OutputOptions opts_ = OO_Defaults)
output_options_(opts_)169 : output_options_(opts_), name_field_width_(0),
170 prev_counters_(), printed_header_(false) {}
171
172 virtual bool ReportContext(const Context& context);
173 virtual void ReportRuns(const std::vector<Run>& reports);
174
175 protected:
176 virtual void PrintRunData(const Run& report);
177 virtual void PrintHeader(const Run& report);
178
179 OutputOptions output_options_;
180 size_t name_field_width_;
181 UserCounters prev_counters_;
182 bool printed_header_;
183 };
184
185 class JSONReporter : public BenchmarkReporter {
186 public:
JSONReporter()187 JSONReporter() : first_report_(true) {}
188 virtual bool ReportContext(const Context& context);
189 virtual void ReportRuns(const std::vector<Run>& reports);
190 virtual void Finalize();
191
192 private:
193 void PrintRunData(const Run& report);
194
195 bool first_report_;
196 };
197
198 class CSVReporter : public BenchmarkReporter {
199 public:
CSVReporter()200 CSVReporter() : printed_header_(false) {}
201 virtual bool ReportContext(const Context& context);
202 virtual void ReportRuns(const std::vector<Run>& reports);
203
204 private:
205 void PrintRunData(const Run& report);
206
207 bool printed_header_;
208 std::set< std::string > user_counter_names_;
209 };
210
GetTimeUnitString(TimeUnit unit)211 inline const char* GetTimeUnitString(TimeUnit unit) {
212 switch (unit) {
213 case kMillisecond:
214 return "ms";
215 case kMicrosecond:
216 return "us";
217 case kNanosecond:
218 default:
219 return "ns";
220 }
221 }
222
GetTimeUnitMultiplier(TimeUnit unit)223 inline double GetTimeUnitMultiplier(TimeUnit unit) {
224 switch (unit) {
225 case kMillisecond:
226 return 1e3;
227 case kMicrosecond:
228 return 1e6;
229 case kNanosecond:
230 default:
231 return 1e9;
232 }
233 }
234
235 } // end namespace benchmark
236 #endif // BENCHMARK_REPORTER_H_
237