1
2 #undef NDEBUG
3 #include <utility>
4
5 #include "benchmark/benchmark.h"
6 #include "output_test.h"
7
8 // ========================================================================= //
9 // ---------------------- Testing Prologue Output -------------------------- //
10 // ========================================================================= //
11
12 ADD_CASES(TC_ConsoleOut,
13 {{"^[-]+$", MR_Next},
14 {"^Benchmark %s Time %s CPU %s Iterations$", MR_Next},
15 {"^[-]+$", MR_Next}});
AddContextCases()16 static int AddContextCases() {
17 AddCases(TC_ConsoleErr,
18 {
19 {"%int[-/]%int[-/]%int %int:%int:%int$", MR_Default},
20 {"Run on \\(%int X %float MHz CPU s\\)", MR_Next},
21 });
22 AddCases(TC_JSONOut, {{"^\\{", MR_Default},
23 {"\"context\":", MR_Next},
24 {"\"date\": \"", MR_Next},
25 {"\"num_cpus\": %int,$", MR_Next},
26 {"\"mhz_per_cpu\": %float,$", MR_Next},
27 {"\"cpu_scaling_enabled\": ", MR_Next},
28 {"\"caches\": \\[$", MR_Next}});
29 auto const& Caches = benchmark::CPUInfo::Get().caches;
30 if (!Caches.empty()) {
31 AddCases(TC_ConsoleErr, {{"CPU Caches:$", MR_Next}});
32 }
33 for (size_t I = 0; I < Caches.size(); ++I) {
34 std::string num_caches_str =
35 Caches[I].num_sharing != 0 ? " \\(x%int\\)$" : "$";
36 AddCases(
37 TC_ConsoleErr,
38 {{"L%int (Data|Instruction|Unified) %intK" + num_caches_str, MR_Next}});
39 AddCases(TC_JSONOut, {{"\\{$", MR_Next},
40 {"\"type\": \"", MR_Next},
41 {"\"level\": %int,$", MR_Next},
42 {"\"size\": %int,$", MR_Next},
43 {"\"num_sharing\": %int$", MR_Next},
44 {"}[,]{0,1}$", MR_Next}});
45 }
46
47 AddCases(TC_JSONOut, {{"],$"}});
48 return 0;
49 }
50 int dummy_register = AddContextCases();
51 ADD_CASES(TC_CSVOut, {{"%csv_header"}});
52
53 // ========================================================================= //
54 // ------------------------ Testing Basic Output --------------------------- //
55 // ========================================================================= //
56
BM_basic(benchmark::State & state)57 void BM_basic(benchmark::State& state) {
58 for (auto _ : state) {
59 }
60 }
61 BENCHMARK(BM_basic);
62
63 ADD_CASES(TC_ConsoleOut, {{"^BM_basic %console_report$"}});
64 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_basic\",$"},
65 {"\"iterations\": %int,$", MR_Next},
66 {"\"real_time\": %float,$", MR_Next},
67 {"\"cpu_time\": %float,$", MR_Next},
68 {"\"time_unit\": \"ns\"$", MR_Next},
69 {"}", MR_Next}});
70 ADD_CASES(TC_CSVOut, {{"^\"BM_basic\",%csv_report$"}});
71
72 // ========================================================================= //
73 // ------------------------ Testing Bytes per Second Output ---------------- //
74 // ========================================================================= //
75
BM_bytes_per_second(benchmark::State & state)76 void BM_bytes_per_second(benchmark::State& state) {
77 for (auto _ : state) {
78 }
79 state.SetBytesProcessed(1);
80 }
81 BENCHMARK(BM_bytes_per_second);
82
83 ADD_CASES(TC_ConsoleOut,
84 {{"^BM_bytes_per_second %console_report +%float[kM]{0,1}B/s$"}});
85 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_bytes_per_second\",$"},
86 {"\"iterations\": %int,$", MR_Next},
87 {"\"real_time\": %float,$", MR_Next},
88 {"\"cpu_time\": %float,$", MR_Next},
89 {"\"time_unit\": \"ns\",$", MR_Next},
90 {"\"bytes_per_second\": %float$", MR_Next},
91 {"}", MR_Next}});
92 ADD_CASES(TC_CSVOut, {{"^\"BM_bytes_per_second\",%csv_bytes_report$"}});
93
94 // ========================================================================= //
95 // ------------------------ Testing Items per Second Output ---------------- //
96 // ========================================================================= //
97
BM_items_per_second(benchmark::State & state)98 void BM_items_per_second(benchmark::State& state) {
99 for (auto _ : state) {
100 }
101 state.SetItemsProcessed(1);
102 }
103 BENCHMARK(BM_items_per_second);
104
105 ADD_CASES(TC_ConsoleOut,
106 {{"^BM_items_per_second %console_report +%float[kM]{0,1} items/s$"}});
107 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_items_per_second\",$"},
108 {"\"iterations\": %int,$", MR_Next},
109 {"\"real_time\": %float,$", MR_Next},
110 {"\"cpu_time\": %float,$", MR_Next},
111 {"\"time_unit\": \"ns\",$", MR_Next},
112 {"\"items_per_second\": %float$", MR_Next},
113 {"}", MR_Next}});
114 ADD_CASES(TC_CSVOut, {{"^\"BM_items_per_second\",%csv_items_report$"}});
115
116 // ========================================================================= //
117 // ------------------------ Testing Label Output --------------------------- //
118 // ========================================================================= //
119
BM_label(benchmark::State & state)120 void BM_label(benchmark::State& state) {
121 for (auto _ : state) {
122 }
123 state.SetLabel("some label");
124 }
125 BENCHMARK(BM_label);
126
127 ADD_CASES(TC_ConsoleOut, {{"^BM_label %console_report some label$"}});
128 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_label\",$"},
129 {"\"iterations\": %int,$", MR_Next},
130 {"\"real_time\": %float,$", MR_Next},
131 {"\"cpu_time\": %float,$", MR_Next},
132 {"\"time_unit\": \"ns\",$", MR_Next},
133 {"\"label\": \"some label\"$", MR_Next},
134 {"}", MR_Next}});
135 ADD_CASES(TC_CSVOut, {{"^\"BM_label\",%csv_label_report_begin\"some "
136 "label\"%csv_label_report_end$"}});
137
138 // ========================================================================= //
139 // ------------------------ Testing Error Output --------------------------- //
140 // ========================================================================= //
141
BM_error(benchmark::State & state)142 void BM_error(benchmark::State& state) {
143 state.SkipWithError("message");
144 for (auto _ : state) {
145 }
146 }
147 BENCHMARK(BM_error);
148 ADD_CASES(TC_ConsoleOut, {{"^BM_error[ ]+ERROR OCCURRED: 'message'$"}});
149 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_error\",$"},
150 {"\"error_occurred\": true,$", MR_Next},
151 {"\"error_message\": \"message\",$", MR_Next}});
152
153 ADD_CASES(TC_CSVOut, {{"^\"BM_error\",,,,,,,,true,\"message\"$"}});
154
155 // ========================================================================= //
156 // ------------------------ Testing No Arg Name Output -----------------------
157 // //
158 // ========================================================================= //
159
BM_no_arg_name(benchmark::State & state)160 void BM_no_arg_name(benchmark::State& state) {
161 for (auto _ : state) {
162 }
163 }
164 BENCHMARK(BM_no_arg_name)->Arg(3);
165 ADD_CASES(TC_ConsoleOut, {{"^BM_no_arg_name/3 %console_report$"}});
166 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_no_arg_name/3\",$"}});
167 ADD_CASES(TC_CSVOut, {{"^\"BM_no_arg_name/3\",%csv_report$"}});
168
169 // ========================================================================= //
170 // ------------------------ Testing Arg Name Output ----------------------- //
171 // ========================================================================= //
172
BM_arg_name(benchmark::State & state)173 void BM_arg_name(benchmark::State& state) {
174 for (auto _ : state) {
175 }
176 }
177 BENCHMARK(BM_arg_name)->ArgName("first")->Arg(3);
178 ADD_CASES(TC_ConsoleOut, {{"^BM_arg_name/first:3 %console_report$"}});
179 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_arg_name/first:3\",$"}});
180 ADD_CASES(TC_CSVOut, {{"^\"BM_arg_name/first:3\",%csv_report$"}});
181
182 // ========================================================================= //
183 // ------------------------ Testing Arg Names Output ----------------------- //
184 // ========================================================================= //
185
BM_arg_names(benchmark::State & state)186 void BM_arg_names(benchmark::State& state) {
187 for (auto _ : state) {
188 }
189 }
190 BENCHMARK(BM_arg_names)->Args({2, 5, 4})->ArgNames({"first", "", "third"});
191 ADD_CASES(TC_ConsoleOut,
192 {{"^BM_arg_names/first:2/5/third:4 %console_report$"}});
193 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_arg_names/first:2/5/third:4\",$"}});
194 ADD_CASES(TC_CSVOut, {{"^\"BM_arg_names/first:2/5/third:4\",%csv_report$"}});
195
196 // ========================================================================= //
197 // ----------------------- Testing Complexity Output ----------------------- //
198 // ========================================================================= //
199
BM_Complexity_O1(benchmark::State & state)200 void BM_Complexity_O1(benchmark::State& state) {
201 for (auto _ : state) {
202 }
203 state.SetComplexityN(state.range(0));
204 }
205 BENCHMARK(BM_Complexity_O1)->Range(1, 1 << 18)->Complexity(benchmark::o1);
206 SET_SUBSTITUTIONS({{"%bigOStr", "[ ]* %float \\([0-9]+\\)"},
207 {"%RMS", "[ ]*[0-9]+ %"}});
208 ADD_CASES(TC_ConsoleOut, {{"^BM_Complexity_O1_BigO %bigOStr %bigOStr[ ]*$"},
209 {"^BM_Complexity_O1_RMS %RMS %RMS[ ]*$"}});
210
211 // ========================================================================= //
212 // ----------------------- Testing Aggregate Output ------------------------ //
213 // ========================================================================= //
214
215 // Test that non-aggregate data is printed by default
BM_Repeat(benchmark::State & state)216 void BM_Repeat(benchmark::State& state) {
217 for (auto _ : state) {
218 }
219 }
220 // need two repetitions min to be able to output any aggregate output
221 BENCHMARK(BM_Repeat)->Repetitions(2);
222 ADD_CASES(TC_ConsoleOut, {{"^BM_Repeat/repeats:2 %console_report$"},
223 {"^BM_Repeat/repeats:2 %console_report$"},
224 {"^BM_Repeat/repeats:2_mean %console_report$"},
225 {"^BM_Repeat/repeats:2_median %console_report$"},
226 {"^BM_Repeat/repeats:2_stddev %console_report$"}});
227 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Repeat/repeats:2\",$"},
228 {"\"name\": \"BM_Repeat/repeats:2\",$"},
229 {"\"name\": \"BM_Repeat/repeats:2_mean\",$"},
230 {"\"name\": \"BM_Repeat/repeats:2_median\",$"},
231 {"\"name\": \"BM_Repeat/repeats:2_stddev\",$"}});
232 ADD_CASES(TC_CSVOut, {{"^\"BM_Repeat/repeats:2\",%csv_report$"},
233 {"^\"BM_Repeat/repeats:2\",%csv_report$"},
234 {"^\"BM_Repeat/repeats:2_mean\",%csv_report$"},
235 {"^\"BM_Repeat/repeats:2_median\",%csv_report$"},
236 {"^\"BM_Repeat/repeats:2_stddev\",%csv_report$"}});
237 // but for two repetitions, mean and median is the same, so let's repeat..
238 BENCHMARK(BM_Repeat)->Repetitions(3);
239 ADD_CASES(TC_ConsoleOut, {{"^BM_Repeat/repeats:3 %console_report$"},
240 {"^BM_Repeat/repeats:3 %console_report$"},
241 {"^BM_Repeat/repeats:3 %console_report$"},
242 {"^BM_Repeat/repeats:3_mean %console_report$"},
243 {"^BM_Repeat/repeats:3_median %console_report$"},
244 {"^BM_Repeat/repeats:3_stddev %console_report$"}});
245 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Repeat/repeats:3\",$"},
246 {"\"name\": \"BM_Repeat/repeats:3\",$"},
247 {"\"name\": \"BM_Repeat/repeats:3\",$"},
248 {"\"name\": \"BM_Repeat/repeats:3_mean\",$"},
249 {"\"name\": \"BM_Repeat/repeats:3_median\",$"},
250 {"\"name\": \"BM_Repeat/repeats:3_stddev\",$"}});
251 ADD_CASES(TC_CSVOut, {{"^\"BM_Repeat/repeats:3\",%csv_report$"},
252 {"^\"BM_Repeat/repeats:3\",%csv_report$"},
253 {"^\"BM_Repeat/repeats:3\",%csv_report$"},
254 {"^\"BM_Repeat/repeats:3_mean\",%csv_report$"},
255 {"^\"BM_Repeat/repeats:3_median\",%csv_report$"},
256 {"^\"BM_Repeat/repeats:3_stddev\",%csv_report$"}});
257 // median differs between even/odd number of repetitions, so just to be sure
258 BENCHMARK(BM_Repeat)->Repetitions(4);
259 ADD_CASES(TC_ConsoleOut, {{"^BM_Repeat/repeats:4 %console_report$"},
260 {"^BM_Repeat/repeats:4 %console_report$"},
261 {"^BM_Repeat/repeats:4 %console_report$"},
262 {"^BM_Repeat/repeats:4 %console_report$"},
263 {"^BM_Repeat/repeats:4_mean %console_report$"},
264 {"^BM_Repeat/repeats:4_median %console_report$"},
265 {"^BM_Repeat/repeats:4_stddev %console_report$"}});
266 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Repeat/repeats:4\",$"},
267 {"\"name\": \"BM_Repeat/repeats:4\",$"},
268 {"\"name\": \"BM_Repeat/repeats:4\",$"},
269 {"\"name\": \"BM_Repeat/repeats:4\",$"},
270 {"\"name\": \"BM_Repeat/repeats:4_mean\",$"},
271 {"\"name\": \"BM_Repeat/repeats:4_median\",$"},
272 {"\"name\": \"BM_Repeat/repeats:4_stddev\",$"}});
273 ADD_CASES(TC_CSVOut, {{"^\"BM_Repeat/repeats:4\",%csv_report$"},
274 {"^\"BM_Repeat/repeats:4\",%csv_report$"},
275 {"^\"BM_Repeat/repeats:4\",%csv_report$"},
276 {"^\"BM_Repeat/repeats:4\",%csv_report$"},
277 {"^\"BM_Repeat/repeats:4_mean\",%csv_report$"},
278 {"^\"BM_Repeat/repeats:4_median\",%csv_report$"},
279 {"^\"BM_Repeat/repeats:4_stddev\",%csv_report$"}});
280
281 // Test that a non-repeated test still prints non-aggregate results even when
282 // only-aggregate reports have been requested
BM_RepeatOnce(benchmark::State & state)283 void BM_RepeatOnce(benchmark::State& state) {
284 for (auto _ : state) {
285 }
286 }
287 BENCHMARK(BM_RepeatOnce)->Repetitions(1)->ReportAggregatesOnly();
288 ADD_CASES(TC_ConsoleOut, {{"^BM_RepeatOnce/repeats:1 %console_report$"}});
289 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_RepeatOnce/repeats:1\",$"}});
290 ADD_CASES(TC_CSVOut, {{"^\"BM_RepeatOnce/repeats:1\",%csv_report$"}});
291
292 // Test that non-aggregate data is not reported
BM_SummaryRepeat(benchmark::State & state)293 void BM_SummaryRepeat(benchmark::State& state) {
294 for (auto _ : state) {
295 }
296 }
297 BENCHMARK(BM_SummaryRepeat)->Repetitions(3)->ReportAggregatesOnly();
298 ADD_CASES(TC_ConsoleOut,
299 {{".*BM_SummaryRepeat/repeats:3 ", MR_Not},
300 {"^BM_SummaryRepeat/repeats:3_mean %console_report$"},
301 {"^BM_SummaryRepeat/repeats:3_median %console_report$"},
302 {"^BM_SummaryRepeat/repeats:3_stddev %console_report$"}});
303 ADD_CASES(TC_JSONOut, {{".*BM_SummaryRepeat/repeats:3 ", MR_Not},
304 {"\"name\": \"BM_SummaryRepeat/repeats:3_mean\",$"},
305 {"\"name\": \"BM_SummaryRepeat/repeats:3_median\",$"},
306 {"\"name\": \"BM_SummaryRepeat/repeats:3_stddev\",$"}});
307 ADD_CASES(TC_CSVOut, {{".*BM_SummaryRepeat/repeats:3 ", MR_Not},
308 {"^\"BM_SummaryRepeat/repeats:3_mean\",%csv_report$"},
309 {"^\"BM_SummaryRepeat/repeats:3_median\",%csv_report$"},
310 {"^\"BM_SummaryRepeat/repeats:3_stddev\",%csv_report$"}});
311
BM_RepeatTimeUnit(benchmark::State & state)312 void BM_RepeatTimeUnit(benchmark::State& state) {
313 for (auto _ : state) {
314 }
315 }
316 BENCHMARK(BM_RepeatTimeUnit)
317 ->Repetitions(3)
318 ->ReportAggregatesOnly()
319 ->Unit(benchmark::kMicrosecond);
320 ADD_CASES(TC_ConsoleOut,
321 {{".*BM_RepeatTimeUnit/repeats:3 ", MR_Not},
322 {"^BM_RepeatTimeUnit/repeats:3_mean %console_us_report$"},
323 {"^BM_RepeatTimeUnit/repeats:3_median %console_us_report$"},
324 {"^BM_RepeatTimeUnit/repeats:3_stddev %console_us_report$"}});
325 ADD_CASES(TC_JSONOut, {{".*BM_RepeatTimeUnit/repeats:3 ", MR_Not},
326 {"\"name\": \"BM_RepeatTimeUnit/repeats:3_mean\",$"},
327 {"\"time_unit\": \"us\",?$"},
328 {"\"name\": \"BM_RepeatTimeUnit/repeats:3_median\",$"},
329 {"\"time_unit\": \"us\",?$"},
330 {"\"name\": \"BM_RepeatTimeUnit/repeats:3_stddev\",$"},
331 {"\"time_unit\": \"us\",?$"}});
332 ADD_CASES(TC_CSVOut,
333 {{".*BM_RepeatTimeUnit/repeats:3 ", MR_Not},
334 {"^\"BM_RepeatTimeUnit/repeats:3_mean\",%csv_us_report$"},
335 {"^\"BM_RepeatTimeUnit/repeats:3_median\",%csv_us_report$"},
336 {"^\"BM_RepeatTimeUnit/repeats:3_stddev\",%csv_us_report$"}});
337
338 // ========================================================================= //
339 // -------------------- Testing user-provided statistics ------------------- //
340 // ========================================================================= //
341
__anon8f5086f70102(const std::vector<double>& v) 342 const auto UserStatistics = [](const std::vector<double>& v) {
343 return v.back();
344 };
BM_UserStats(benchmark::State & state)345 void BM_UserStats(benchmark::State& state) {
346 for (auto _ : state) {
347 }
348 }
349 BENCHMARK(BM_UserStats)
350 ->Repetitions(3)
351 ->ComputeStatistics("", UserStatistics);
352 // check that user-provided stats is calculated, and is after the default-ones
353 // empty string as name is intentional, it would sort before anything else
354 ADD_CASES(TC_ConsoleOut, {{"^BM_UserStats/repeats:3 %console_report$"},
355 {"^BM_UserStats/repeats:3 %console_report$"},
356 {"^BM_UserStats/repeats:3 %console_report$"},
357 {"^BM_UserStats/repeats:3_mean %console_report$"},
358 {"^BM_UserStats/repeats:3_median %console_report$"},
359 {"^BM_UserStats/repeats:3_stddev %console_report$"},
360 {"^BM_UserStats/repeats:3_ %console_report$"}});
361 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_UserStats/repeats:3\",$"},
362 {"\"name\": \"BM_UserStats/repeats:3\",$"},
363 {"\"name\": \"BM_UserStats/repeats:3\",$"},
364 {"\"name\": \"BM_UserStats/repeats:3_mean\",$"},
365 {"\"name\": \"BM_UserStats/repeats:3_median\",$"},
366 {"\"name\": \"BM_UserStats/repeats:3_stddev\",$"},
367 {"\"name\": \"BM_UserStats/repeats:3_\",$"}});
368 ADD_CASES(TC_CSVOut, {{"^\"BM_UserStats/repeats:3\",%csv_report$"},
369 {"^\"BM_UserStats/repeats:3\",%csv_report$"},
370 {"^\"BM_UserStats/repeats:3\",%csv_report$"},
371 {"^\"BM_UserStats/repeats:3_mean\",%csv_report$"},
372 {"^\"BM_UserStats/repeats:3_median\",%csv_report$"},
373 {"^\"BM_UserStats/repeats:3_stddev\",%csv_report$"},
374 {"^\"BM_UserStats/repeats:3_\",%csv_report$"}});
375
376 // ========================================================================= //
377 // --------------------------- TEST CASES END ------------------------------ //
378 // ========================================================================= //
379
main(int argc,char * argv[])380 int main(int argc, char* argv[]) { RunOutputTests(argc, argv); }
381