1
2 #undef NDEBUG
3 #include <utility>
4
5 #include "benchmark/benchmark.h"
6 #include "output_test.h"
7
8 // ========================================================================= //
9 // ---------------------- Testing Prologue Output -------------------------- //
10 // ========================================================================= //
11
12 ADD_CASES(TC_ConsoleOut, {{"^[-]+$", MR_Next},
13 {"^Benchmark %s Time %s CPU %s Iterations$", MR_Next},
14 {"^[-]+$", MR_Next}});
AddContextCases()15 static int AddContextCases() {
16 AddCases(TC_ConsoleErr,
17 {
18 {"^%int-%int-%intT%int:%int:%int[-+]%int:%int$", MR_Default},
19 {"Running .*/reporter_output_test(\\.exe)?$", MR_Next},
20 {"Run on \\(%int X %float MHz CPU s?\\)", MR_Next},
21 });
22 AddCases(TC_JSONOut,
23 {{"^\\{", MR_Default},
24 {"\"context\":", MR_Next},
25 {"\"date\": \"", MR_Next},
26 {"\"host_name\":", MR_Next},
27 {"\"executable\": \".*(/|\\\\)reporter_output_test(\\.exe)?\",",
28 MR_Next},
29 {"\"num_cpus\": %int,$", MR_Next},
30 {"\"mhz_per_cpu\": %float,$", MR_Next},
31 {"\"caches\": \\[$", MR_Default}});
32 auto const& Info = benchmark::CPUInfo::Get();
33 auto const& Caches = Info.caches;
34 if (!Caches.empty()) {
35 AddCases(TC_ConsoleErr, {{"CPU Caches:$", MR_Next}});
36 }
37 for (size_t I = 0; I < Caches.size(); ++I) {
38 std::string num_caches_str =
39 Caches[I].num_sharing != 0 ? " \\(x%int\\)$" : "$";
40 AddCases(TC_ConsoleErr,
41 {{"L%int (Data|Instruction|Unified) %int KiB" + num_caches_str,
42 MR_Next}});
43 AddCases(TC_JSONOut, {{"\\{$", MR_Next},
44 {"\"type\": \"", MR_Next},
45 {"\"level\": %int,$", MR_Next},
46 {"\"size\": %int,$", MR_Next},
47 {"\"num_sharing\": %int$", MR_Next},
48 {"}[,]{0,1}$", MR_Next}});
49 }
50 AddCases(TC_JSONOut, {{"],$"}});
51 auto const& LoadAvg = Info.load_avg;
52 if (!LoadAvg.empty()) {
53 AddCases(TC_ConsoleErr,
54 {{"Load Average: (%float, ){0,2}%float$", MR_Next}});
55 }
56 AddCases(TC_JSONOut, {{"\"load_avg\": \\[(%float,?){0,3}],$", MR_Next}});
57 return 0;
58 }
59 int dummy_register = AddContextCases();
60 ADD_CASES(TC_CSVOut, {{"%csv_header"}});
61
62 // ========================================================================= //
63 // ------------------------ Testing Basic Output --------------------------- //
64 // ========================================================================= //
65
BM_basic(benchmark::State & state)66 void BM_basic(benchmark::State& state) {
67 for (auto _ : state) {
68 }
69 }
70 BENCHMARK(BM_basic);
71
72 ADD_CASES(TC_ConsoleOut, {{"^BM_basic %console_report$"}});
73 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_basic\",$"},
74 {"\"run_name\": \"BM_basic\",$", MR_Next},
75 {"\"run_type\": \"iteration\",$", MR_Next},
76 {"\"repetitions\": 0,$", MR_Next},
77 {"\"repetition_index\": 0,$", MR_Next},
78 {"\"threads\": 1,$", MR_Next},
79 {"\"iterations\": %int,$", MR_Next},
80 {"\"real_time\": %float,$", MR_Next},
81 {"\"cpu_time\": %float,$", MR_Next},
82 {"\"time_unit\": \"ns\"$", MR_Next},
83 {"}", MR_Next}});
84 ADD_CASES(TC_CSVOut, {{"^\"BM_basic\",%csv_report$"}});
85
86 // ========================================================================= //
87 // ------------------------ Testing Bytes per Second Output ---------------- //
88 // ========================================================================= //
89
BM_bytes_per_second(benchmark::State & state)90 void BM_bytes_per_second(benchmark::State& state) {
91 for (auto _ : state) {
92 // This test requires a non-zero CPU time to avoid divide-by-zero
93 benchmark::DoNotOptimize(state.iterations());
94 }
95 state.SetBytesProcessed(1);
96 }
97 BENCHMARK(BM_bytes_per_second);
98
99 ADD_CASES(TC_ConsoleOut, {{"^BM_bytes_per_second %console_report "
100 "bytes_per_second=%float[kM]{0,1}/s$"}});
101 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_bytes_per_second\",$"},
102 {"\"run_name\": \"BM_bytes_per_second\",$", MR_Next},
103 {"\"run_type\": \"iteration\",$", MR_Next},
104 {"\"repetitions\": 0,$", MR_Next},
105 {"\"repetition_index\": 0,$", MR_Next},
106 {"\"threads\": 1,$", MR_Next},
107 {"\"iterations\": %int,$", MR_Next},
108 {"\"real_time\": %float,$", MR_Next},
109 {"\"cpu_time\": %float,$", MR_Next},
110 {"\"time_unit\": \"ns\",$", MR_Next},
111 {"\"bytes_per_second\": %float$", MR_Next},
112 {"}", MR_Next}});
113 ADD_CASES(TC_CSVOut, {{"^\"BM_bytes_per_second\",%csv_bytes_report$"}});
114
115 // ========================================================================= //
116 // ------------------------ Testing Items per Second Output ---------------- //
117 // ========================================================================= //
118
BM_items_per_second(benchmark::State & state)119 void BM_items_per_second(benchmark::State& state) {
120 for (auto _ : state) {
121 // This test requires a non-zero CPU time to avoid divide-by-zero
122 benchmark::DoNotOptimize(state.iterations());
123 }
124 state.SetItemsProcessed(1);
125 }
126 BENCHMARK(BM_items_per_second);
127
128 ADD_CASES(TC_ConsoleOut, {{"^BM_items_per_second %console_report "
129 "items_per_second=%float[kM]{0,1}/s$"}});
130 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_items_per_second\",$"},
131 {"\"run_name\": \"BM_items_per_second\",$", MR_Next},
132 {"\"run_type\": \"iteration\",$", MR_Next},
133 {"\"repetitions\": 0,$", MR_Next},
134 {"\"repetition_index\": 0,$", MR_Next},
135 {"\"threads\": 1,$", MR_Next},
136 {"\"iterations\": %int,$", MR_Next},
137 {"\"real_time\": %float,$", MR_Next},
138 {"\"cpu_time\": %float,$", MR_Next},
139 {"\"time_unit\": \"ns\",$", MR_Next},
140 {"\"items_per_second\": %float$", MR_Next},
141 {"}", MR_Next}});
142 ADD_CASES(TC_CSVOut, {{"^\"BM_items_per_second\",%csv_items_report$"}});
143
144 // ========================================================================= //
145 // ------------------------ Testing Label Output --------------------------- //
146 // ========================================================================= //
147
BM_label(benchmark::State & state)148 void BM_label(benchmark::State& state) {
149 for (auto _ : state) {
150 }
151 state.SetLabel("some label");
152 }
153 BENCHMARK(BM_label);
154
155 ADD_CASES(TC_ConsoleOut, {{"^BM_label %console_report some label$"}});
156 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_label\",$"},
157 {"\"run_name\": \"BM_label\",$", MR_Next},
158 {"\"run_type\": \"iteration\",$", MR_Next},
159 {"\"repetitions\": 0,$", MR_Next},
160 {"\"repetition_index\": 0,$", MR_Next},
161 {"\"threads\": 1,$", MR_Next},
162 {"\"iterations\": %int,$", MR_Next},
163 {"\"real_time\": %float,$", MR_Next},
164 {"\"cpu_time\": %float,$", MR_Next},
165 {"\"time_unit\": \"ns\",$", MR_Next},
166 {"\"label\": \"some label\"$", MR_Next},
167 {"}", MR_Next}});
168 ADD_CASES(TC_CSVOut, {{"^\"BM_label\",%csv_label_report_begin\"some "
169 "label\"%csv_label_report_end$"}});
170
171 // ========================================================================= //
172 // ------------------------ Testing Error Output --------------------------- //
173 // ========================================================================= //
174
BM_error(benchmark::State & state)175 void BM_error(benchmark::State& state) {
176 state.SkipWithError("message");
177 for (auto _ : state) {
178 }
179 }
180 BENCHMARK(BM_error);
181 ADD_CASES(TC_ConsoleOut, {{"^BM_error[ ]+ERROR OCCURRED: 'message'$"}});
182 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_error\",$"},
183 {"\"run_name\": \"BM_error\",$", MR_Next},
184 {"\"run_type\": \"iteration\",$", MR_Next},
185 {"\"repetitions\": 0,$", MR_Next},
186 {"\"repetition_index\": 0,$", MR_Next},
187 {"\"threads\": 1,$", MR_Next},
188 {"\"error_occurred\": true,$", MR_Next},
189 {"\"error_message\": \"message\",$", MR_Next}});
190
191 ADD_CASES(TC_CSVOut, {{"^\"BM_error\",,,,,,,,true,\"message\"$"}});
192
193 // ========================================================================= //
194 // ------------------------ Testing No Arg Name Output -----------------------
195 // //
196 // ========================================================================= //
197
BM_no_arg_name(benchmark::State & state)198 void BM_no_arg_name(benchmark::State& state) {
199 for (auto _ : state) {
200 }
201 }
202 BENCHMARK(BM_no_arg_name)->Arg(3);
203 ADD_CASES(TC_ConsoleOut, {{"^BM_no_arg_name/3 %console_report$"}});
204 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_no_arg_name/3\",$"},
205 {"\"run_name\": \"BM_no_arg_name/3\",$", MR_Next},
206 {"\"run_type\": \"iteration\",$", MR_Next},
207 {"\"repetitions\": 0,$", MR_Next},
208 {"\"repetition_index\": 0,$", MR_Next},
209 {"\"threads\": 1,$", MR_Next}});
210 ADD_CASES(TC_CSVOut, {{"^\"BM_no_arg_name/3\",%csv_report$"}});
211
212 // ========================================================================= //
213 // ------------------------ Testing Arg Name Output ----------------------- //
214 // ========================================================================= //
215
BM_arg_name(benchmark::State & state)216 void BM_arg_name(benchmark::State& state) {
217 for (auto _ : state) {
218 }
219 }
220 BENCHMARK(BM_arg_name)->ArgName("first")->Arg(3);
221 ADD_CASES(TC_ConsoleOut, {{"^BM_arg_name/first:3 %console_report$"}});
222 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_arg_name/first:3\",$"},
223 {"\"run_name\": \"BM_arg_name/first:3\",$", MR_Next},
224 {"\"run_type\": \"iteration\",$", MR_Next},
225 {"\"repetitions\": 0,$", MR_Next},
226 {"\"repetition_index\": 0,$", MR_Next},
227 {"\"threads\": 1,$", MR_Next}});
228 ADD_CASES(TC_CSVOut, {{"^\"BM_arg_name/first:3\",%csv_report$"}});
229
230 // ========================================================================= //
231 // ------------------------ Testing Arg Names Output ----------------------- //
232 // ========================================================================= //
233
BM_arg_names(benchmark::State & state)234 void BM_arg_names(benchmark::State& state) {
235 for (auto _ : state) {
236 }
237 }
238 BENCHMARK(BM_arg_names)->Args({2, 5, 4})->ArgNames({"first", "", "third"});
239 ADD_CASES(TC_ConsoleOut,
240 {{"^BM_arg_names/first:2/5/third:4 %console_report$"}});
241 ADD_CASES(TC_JSONOut,
242 {{"\"name\": \"BM_arg_names/first:2/5/third:4\",$"},
243 {"\"run_name\": \"BM_arg_names/first:2/5/third:4\",$", MR_Next},
244 {"\"run_type\": \"iteration\",$", MR_Next},
245 {"\"repetitions\": 0,$", MR_Next},
246 {"\"repetition_index\": 0,$", MR_Next},
247 {"\"threads\": 1,$", MR_Next}});
248 ADD_CASES(TC_CSVOut, {{"^\"BM_arg_names/first:2/5/third:4\",%csv_report$"}});
249
250 // ========================================================================= //
251 // ------------------------ Testing Big Args Output ------------------------ //
252 // ========================================================================= //
253
BM_BigArgs(benchmark::State & state)254 void BM_BigArgs(benchmark::State& state) {
255 for (auto _ : state) {
256 }
257 }
258 BENCHMARK(BM_BigArgs)->RangeMultiplier(2)->Range(1U << 30U, 1U << 31U);
259 ADD_CASES(TC_ConsoleOut, {{"^BM_BigArgs/1073741824 %console_report$"},
260 {"^BM_BigArgs/2147483648 %console_report$"}});
261
262 // ========================================================================= //
263 // ----------------------- Testing Complexity Output ----------------------- //
264 // ========================================================================= //
265
BM_Complexity_O1(benchmark::State & state)266 void BM_Complexity_O1(benchmark::State& state) {
267 for (auto _ : state) {
268 // This test requires a non-zero CPU time to avoid divide-by-zero
269 benchmark::DoNotOptimize(state.iterations());
270 }
271 state.SetComplexityN(state.range(0));
272 }
273 BENCHMARK(BM_Complexity_O1)->Range(1, 1 << 18)->Complexity(benchmark::o1);
274 SET_SUBSTITUTIONS({{"%bigOStr", "[ ]* %float \\([0-9]+\\)"},
275 {"%RMS", "[ ]*[0-9]+ %"}});
276 ADD_CASES(TC_ConsoleOut, {{"^BM_Complexity_O1_BigO %bigOStr %bigOStr[ ]*$"},
277 {"^BM_Complexity_O1_RMS %RMS %RMS[ ]*$"}});
278
279 // ========================================================================= //
280 // ----------------------- Testing Aggregate Output ------------------------ //
281 // ========================================================================= //
282
283 // Test that non-aggregate data is printed by default
BM_Repeat(benchmark::State & state)284 void BM_Repeat(benchmark::State& state) {
285 for (auto _ : state) {
286 }
287 }
288 // need two repetitions min to be able to output any aggregate output
289 BENCHMARK(BM_Repeat)->Repetitions(2);
290 ADD_CASES(TC_ConsoleOut,
291 {{"^BM_Repeat/repeats:2 %console_report$"},
292 {"^BM_Repeat/repeats:2 %console_report$"},
293 {"^BM_Repeat/repeats:2_mean %console_time_only_report [ ]*2$"},
294 {"^BM_Repeat/repeats:2_median %console_time_only_report [ ]*2$"},
295 {"^BM_Repeat/repeats:2_stddev %console_time_only_report [ ]*2$"}});
296 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Repeat/repeats:2\",$"},
297 {"\"run_name\": \"BM_Repeat/repeats:2\"", MR_Next},
298 {"\"run_type\": \"iteration\",$", MR_Next},
299 {"\"repetitions\": 2,$", MR_Next},
300 {"\"repetition_index\": 0,$", MR_Next},
301 {"\"threads\": 1,$", MR_Next},
302 {"\"name\": \"BM_Repeat/repeats:2\",$"},
303 {"\"run_name\": \"BM_Repeat/repeats:2\",$", MR_Next},
304 {"\"run_type\": \"iteration\",$", MR_Next},
305 {"\"repetitions\": 2,$", MR_Next},
306 {"\"repetition_index\": 1,$", MR_Next},
307 {"\"threads\": 1,$", MR_Next},
308 {"\"name\": \"BM_Repeat/repeats:2_mean\",$"},
309 {"\"run_name\": \"BM_Repeat/repeats:2\",$", MR_Next},
310 {"\"run_type\": \"aggregate\",$", MR_Next},
311 {"\"repetitions\": 2,$", MR_Next},
312 {"\"threads\": 1,$", MR_Next},
313 {"\"aggregate_name\": \"mean\",$", MR_Next},
314 {"\"iterations\": 2,$", MR_Next},
315 {"\"name\": \"BM_Repeat/repeats:2_median\",$"},
316 {"\"run_name\": \"BM_Repeat/repeats:2\",$", MR_Next},
317 {"\"run_type\": \"aggregate\",$", MR_Next},
318 {"\"repetitions\": 2,$", MR_Next},
319 {"\"threads\": 1,$", MR_Next},
320 {"\"aggregate_name\": \"median\",$", MR_Next},
321 {"\"iterations\": 2,$", MR_Next},
322 {"\"name\": \"BM_Repeat/repeats:2_stddev\",$"},
323 {"\"run_name\": \"BM_Repeat/repeats:2\",$", MR_Next},
324 {"\"run_type\": \"aggregate\",$", MR_Next},
325 {"\"repetitions\": 2,$", MR_Next},
326 {"\"threads\": 1,$", MR_Next},
327 {"\"aggregate_name\": \"stddev\",$", MR_Next},
328 {"\"iterations\": 2,$", MR_Next}});
329 ADD_CASES(TC_CSVOut, {{"^\"BM_Repeat/repeats:2\",%csv_report$"},
330 {"^\"BM_Repeat/repeats:2\",%csv_report$"},
331 {"^\"BM_Repeat/repeats:2_mean\",%csv_report$"},
332 {"^\"BM_Repeat/repeats:2_median\",%csv_report$"},
333 {"^\"BM_Repeat/repeats:2_stddev\",%csv_report$"}});
334 // but for two repetitions, mean and median is the same, so let's repeat..
335 BENCHMARK(BM_Repeat)->Repetitions(3);
336 ADD_CASES(TC_ConsoleOut,
337 {{"^BM_Repeat/repeats:3 %console_report$"},
338 {"^BM_Repeat/repeats:3 %console_report$"},
339 {"^BM_Repeat/repeats:3 %console_report$"},
340 {"^BM_Repeat/repeats:3_mean %console_time_only_report [ ]*3$"},
341 {"^BM_Repeat/repeats:3_median %console_time_only_report [ ]*3$"},
342 {"^BM_Repeat/repeats:3_stddev %console_time_only_report [ ]*3$"}});
343 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Repeat/repeats:3\",$"},
344 {"\"run_name\": \"BM_Repeat/repeats:3\",$", MR_Next},
345 {"\"run_type\": \"iteration\",$", MR_Next},
346 {"\"repetitions\": 3,$", MR_Next},
347 {"\"repetition_index\": 0,$", MR_Next},
348 {"\"threads\": 1,$", MR_Next},
349 {"\"name\": \"BM_Repeat/repeats:3\",$"},
350 {"\"run_name\": \"BM_Repeat/repeats:3\",$", MR_Next},
351 {"\"run_type\": \"iteration\",$", MR_Next},
352 {"\"repetitions\": 3,$", MR_Next},
353 {"\"repetition_index\": 1,$", MR_Next},
354 {"\"threads\": 1,$", MR_Next},
355 {"\"name\": \"BM_Repeat/repeats:3\",$"},
356 {"\"run_name\": \"BM_Repeat/repeats:3\",$", MR_Next},
357 {"\"run_type\": \"iteration\",$", MR_Next},
358 {"\"repetitions\": 3,$", MR_Next},
359 {"\"repetition_index\": 2,$", MR_Next},
360 {"\"threads\": 1,$", MR_Next},
361 {"\"name\": \"BM_Repeat/repeats:3_mean\",$"},
362 {"\"run_name\": \"BM_Repeat/repeats:3\",$", MR_Next},
363 {"\"run_type\": \"aggregate\",$", MR_Next},
364 {"\"repetitions\": 3,$", MR_Next},
365 {"\"threads\": 1,$", MR_Next},
366 {"\"aggregate_name\": \"mean\",$", MR_Next},
367 {"\"iterations\": 3,$", MR_Next},
368 {"\"name\": \"BM_Repeat/repeats:3_median\",$"},
369 {"\"run_name\": \"BM_Repeat/repeats:3\",$", MR_Next},
370 {"\"run_type\": \"aggregate\",$", MR_Next},
371 {"\"repetitions\": 3,$", MR_Next},
372 {"\"threads\": 1,$", MR_Next},
373 {"\"aggregate_name\": \"median\",$", MR_Next},
374 {"\"iterations\": 3,$", MR_Next},
375 {"\"name\": \"BM_Repeat/repeats:3_stddev\",$"},
376 {"\"run_name\": \"BM_Repeat/repeats:3\",$", MR_Next},
377 {"\"run_type\": \"aggregate\",$", MR_Next},
378 {"\"repetitions\": 3,$", MR_Next},
379 {"\"threads\": 1,$", MR_Next},
380 {"\"aggregate_name\": \"stddev\",$", MR_Next},
381 {"\"iterations\": 3,$", MR_Next}});
382 ADD_CASES(TC_CSVOut, {{"^\"BM_Repeat/repeats:3\",%csv_report$"},
383 {"^\"BM_Repeat/repeats:3\",%csv_report$"},
384 {"^\"BM_Repeat/repeats:3\",%csv_report$"},
385 {"^\"BM_Repeat/repeats:3_mean\",%csv_report$"},
386 {"^\"BM_Repeat/repeats:3_median\",%csv_report$"},
387 {"^\"BM_Repeat/repeats:3_stddev\",%csv_report$"}});
388 // median differs between even/odd number of repetitions, so just to be sure
389 BENCHMARK(BM_Repeat)->Repetitions(4);
390 ADD_CASES(TC_ConsoleOut,
391 {{"^BM_Repeat/repeats:4 %console_report$"},
392 {"^BM_Repeat/repeats:4 %console_report$"},
393 {"^BM_Repeat/repeats:4 %console_report$"},
394 {"^BM_Repeat/repeats:4 %console_report$"},
395 {"^BM_Repeat/repeats:4_mean %console_time_only_report [ ]*4$"},
396 {"^BM_Repeat/repeats:4_median %console_time_only_report [ ]*4$"},
397 {"^BM_Repeat/repeats:4_stddev %console_time_only_report [ ]*4$"}});
398 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Repeat/repeats:4\",$"},
399 {"\"run_name\": \"BM_Repeat/repeats:4\",$", MR_Next},
400 {"\"run_type\": \"iteration\",$", MR_Next},
401 {"\"repetitions\": 4,$", MR_Next},
402 {"\"repetition_index\": 0,$", MR_Next},
403 {"\"threads\": 1,$", MR_Next},
404 {"\"name\": \"BM_Repeat/repeats:4\",$"},
405 {"\"run_name\": \"BM_Repeat/repeats:4\",$", MR_Next},
406 {"\"run_type\": \"iteration\",$", MR_Next},
407 {"\"repetitions\": 4,$", MR_Next},
408 {"\"repetition_index\": 1,$", MR_Next},
409 {"\"threads\": 1,$", MR_Next},
410 {"\"name\": \"BM_Repeat/repeats:4\",$"},
411 {"\"run_name\": \"BM_Repeat/repeats:4\",$", MR_Next},
412 {"\"run_type\": \"iteration\",$", MR_Next},
413 {"\"repetitions\": 4,$", MR_Next},
414 {"\"repetition_index\": 2,$", MR_Next},
415 {"\"threads\": 1,$", MR_Next},
416 {"\"name\": \"BM_Repeat/repeats:4\",$"},
417 {"\"run_name\": \"BM_Repeat/repeats:4\",$", MR_Next},
418 {"\"run_type\": \"iteration\",$", MR_Next},
419 {"\"repetitions\": 4,$", MR_Next},
420 {"\"repetition_index\": 3,$", MR_Next},
421 {"\"threads\": 1,$", MR_Next},
422 {"\"name\": \"BM_Repeat/repeats:4_mean\",$"},
423 {"\"run_name\": \"BM_Repeat/repeats:4\",$", MR_Next},
424 {"\"run_type\": \"aggregate\",$", MR_Next},
425 {"\"repetitions\": 4,$", MR_Next},
426 {"\"threads\": 1,$", MR_Next},
427 {"\"aggregate_name\": \"mean\",$", MR_Next},
428 {"\"iterations\": 4,$", MR_Next},
429 {"\"name\": \"BM_Repeat/repeats:4_median\",$"},
430 {"\"run_name\": \"BM_Repeat/repeats:4\",$", MR_Next},
431 {"\"run_type\": \"aggregate\",$", MR_Next},
432 {"\"repetitions\": 4,$", MR_Next},
433 {"\"threads\": 1,$", MR_Next},
434 {"\"aggregate_name\": \"median\",$", MR_Next},
435 {"\"iterations\": 4,$", MR_Next},
436 {"\"name\": \"BM_Repeat/repeats:4_stddev\",$"},
437 {"\"run_name\": \"BM_Repeat/repeats:4\",$", MR_Next},
438 {"\"run_type\": \"aggregate\",$", MR_Next},
439 {"\"repetitions\": 4,$", MR_Next},
440 {"\"threads\": 1,$", MR_Next},
441 {"\"aggregate_name\": \"stddev\",$", MR_Next},
442 {"\"iterations\": 4,$", MR_Next}});
443 ADD_CASES(TC_CSVOut, {{"^\"BM_Repeat/repeats:4\",%csv_report$"},
444 {"^\"BM_Repeat/repeats:4\",%csv_report$"},
445 {"^\"BM_Repeat/repeats:4\",%csv_report$"},
446 {"^\"BM_Repeat/repeats:4\",%csv_report$"},
447 {"^\"BM_Repeat/repeats:4_mean\",%csv_report$"},
448 {"^\"BM_Repeat/repeats:4_median\",%csv_report$"},
449 {"^\"BM_Repeat/repeats:4_stddev\",%csv_report$"}});
450
451 // Test that a non-repeated test still prints non-aggregate results even when
452 // only-aggregate reports have been requested
BM_RepeatOnce(benchmark::State & state)453 void BM_RepeatOnce(benchmark::State& state) {
454 for (auto _ : state) {
455 }
456 }
457 BENCHMARK(BM_RepeatOnce)->Repetitions(1)->ReportAggregatesOnly();
458 ADD_CASES(TC_ConsoleOut, {{"^BM_RepeatOnce/repeats:1 %console_report$"}});
459 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_RepeatOnce/repeats:1\",$"},
460 {"\"run_name\": \"BM_RepeatOnce/repeats:1\",$", MR_Next},
461 {"\"run_type\": \"iteration\",$", MR_Next},
462 {"\"repetitions\": 1,$", MR_Next},
463 {"\"repetition_index\": 0,$", MR_Next},
464 {"\"threads\": 1,$", MR_Next}});
465 ADD_CASES(TC_CSVOut, {{"^\"BM_RepeatOnce/repeats:1\",%csv_report$"}});
466
467 // Test that non-aggregate data is not reported
BM_SummaryRepeat(benchmark::State & state)468 void BM_SummaryRepeat(benchmark::State& state) {
469 for (auto _ : state) {
470 }
471 }
472 BENCHMARK(BM_SummaryRepeat)->Repetitions(3)->ReportAggregatesOnly();
473 ADD_CASES(
474 TC_ConsoleOut,
475 {{".*BM_SummaryRepeat/repeats:3 ", MR_Not},
476 {"^BM_SummaryRepeat/repeats:3_mean %console_time_only_report [ ]*3$"},
477 {"^BM_SummaryRepeat/repeats:3_median %console_time_only_report [ ]*3$"},
478 {"^BM_SummaryRepeat/repeats:3_stddev %console_time_only_report [ ]*3$"}});
479 ADD_CASES(TC_JSONOut,
480 {{".*BM_SummaryRepeat/repeats:3 ", MR_Not},
481 {"\"name\": \"BM_SummaryRepeat/repeats:3_mean\",$"},
482 {"\"run_name\": \"BM_SummaryRepeat/repeats:3\",$", MR_Next},
483 {"\"run_type\": \"aggregate\",$", MR_Next},
484 {"\"repetitions\": 3,$", MR_Next},
485 {"\"threads\": 1,$", MR_Next},
486 {"\"aggregate_name\": \"mean\",$", MR_Next},
487 {"\"iterations\": 3,$", MR_Next},
488 {"\"name\": \"BM_SummaryRepeat/repeats:3_median\",$"},
489 {"\"run_name\": \"BM_SummaryRepeat/repeats:3\",$", MR_Next},
490 {"\"run_type\": \"aggregate\",$", MR_Next},
491 {"\"repetitions\": 3,$", MR_Next},
492 {"\"threads\": 1,$", MR_Next},
493 {"\"aggregate_name\": \"median\",$", MR_Next},
494 {"\"iterations\": 3,$", MR_Next},
495 {"\"name\": \"BM_SummaryRepeat/repeats:3_stddev\",$"},
496 {"\"run_name\": \"BM_SummaryRepeat/repeats:3\",$", MR_Next},
497 {"\"run_type\": \"aggregate\",$", MR_Next},
498 {"\"repetitions\": 3,$", MR_Next},
499 {"\"threads\": 1,$", MR_Next},
500 {"\"aggregate_name\": \"stddev\",$", MR_Next},
501 {"\"iterations\": 3,$", MR_Next}});
502 ADD_CASES(TC_CSVOut, {{".*BM_SummaryRepeat/repeats:3 ", MR_Not},
503 {"^\"BM_SummaryRepeat/repeats:3_mean\",%csv_report$"},
504 {"^\"BM_SummaryRepeat/repeats:3_median\",%csv_report$"},
505 {"^\"BM_SummaryRepeat/repeats:3_stddev\",%csv_report$"}});
506
507 // Test that non-aggregate data is not displayed.
508 // NOTE: this test is kinda bad. we are only testing the display output.
509 // But we don't check that the file output still contains everything...
BM_SummaryDisplay(benchmark::State & state)510 void BM_SummaryDisplay(benchmark::State& state) {
511 for (auto _ : state) {
512 }
513 }
514 BENCHMARK(BM_SummaryDisplay)->Repetitions(2)->DisplayAggregatesOnly();
515 ADD_CASES(
516 TC_ConsoleOut,
517 {{".*BM_SummaryDisplay/repeats:2 ", MR_Not},
518 {"^BM_SummaryDisplay/repeats:2_mean %console_time_only_report [ ]*2$"},
519 {"^BM_SummaryDisplay/repeats:2_median %console_time_only_report [ ]*2$"},
520 {"^BM_SummaryDisplay/repeats:2_stddev %console_time_only_report [ ]*2$"}});
521 ADD_CASES(TC_JSONOut,
522 {{".*BM_SummaryDisplay/repeats:2 ", MR_Not},
523 {"\"name\": \"BM_SummaryDisplay/repeats:2_mean\",$"},
524 {"\"run_name\": \"BM_SummaryDisplay/repeats:2\",$", MR_Next},
525 {"\"run_type\": \"aggregate\",$", MR_Next},
526 {"\"repetitions\": 2,$", MR_Next},
527 {"\"threads\": 1,$", MR_Next},
528 {"\"aggregate_name\": \"mean\",$", MR_Next},
529 {"\"iterations\": 2,$", MR_Next},
530 {"\"name\": \"BM_SummaryDisplay/repeats:2_median\",$"},
531 {"\"run_name\": \"BM_SummaryDisplay/repeats:2\",$", MR_Next},
532 {"\"run_type\": \"aggregate\",$", MR_Next},
533 {"\"repetitions\": 2,$", MR_Next},
534 {"\"threads\": 1,$", MR_Next},
535 {"\"aggregate_name\": \"median\",$", MR_Next},
536 {"\"iterations\": 2,$", MR_Next},
537 {"\"name\": \"BM_SummaryDisplay/repeats:2_stddev\",$"},
538 {"\"run_name\": \"BM_SummaryDisplay/repeats:2\",$", MR_Next},
539 {"\"run_type\": \"aggregate\",$", MR_Next},
540 {"\"repetitions\": 2,$", MR_Next},
541 {"\"threads\": 1,$", MR_Next},
542 {"\"aggregate_name\": \"stddev\",$", MR_Next},
543 {"\"iterations\": 2,$", MR_Next}});
544 ADD_CASES(TC_CSVOut,
545 {{".*BM_SummaryDisplay/repeats:2 ", MR_Not},
546 {"^\"BM_SummaryDisplay/repeats:2_mean\",%csv_report$"},
547 {"^\"BM_SummaryDisplay/repeats:2_median\",%csv_report$"},
548 {"^\"BM_SummaryDisplay/repeats:2_stddev\",%csv_report$"}});
549
550 // Test repeats with custom time unit.
BM_RepeatTimeUnit(benchmark::State & state)551 void BM_RepeatTimeUnit(benchmark::State& state) {
552 for (auto _ : state) {
553 }
554 }
555 BENCHMARK(BM_RepeatTimeUnit)
556 ->Repetitions(3)
557 ->ReportAggregatesOnly()
558 ->Unit(benchmark::kMicrosecond);
559 ADD_CASES(
560 TC_ConsoleOut,
561 {{".*BM_RepeatTimeUnit/repeats:3 ", MR_Not},
562 {"^BM_RepeatTimeUnit/repeats:3_mean %console_us_time_only_report [ ]*3$"},
563 {"^BM_RepeatTimeUnit/repeats:3_median %console_us_time_only_report [ "
564 "]*3$"},
565 {"^BM_RepeatTimeUnit/repeats:3_stddev %console_us_time_only_report [ "
566 "]*3$"}});
567 ADD_CASES(TC_JSONOut,
568 {{".*BM_RepeatTimeUnit/repeats:3 ", MR_Not},
569 {"\"name\": \"BM_RepeatTimeUnit/repeats:3_mean\",$"},
570 {"\"run_name\": \"BM_RepeatTimeUnit/repeats:3\",$", MR_Next},
571 {"\"run_type\": \"aggregate\",$", MR_Next},
572 {"\"repetitions\": 3,$", MR_Next},
573 {"\"threads\": 1,$", MR_Next},
574 {"\"aggregate_name\": \"mean\",$", MR_Next},
575 {"\"iterations\": 3,$", MR_Next},
576 {"\"time_unit\": \"us\",?$"},
577 {"\"name\": \"BM_RepeatTimeUnit/repeats:3_median\",$"},
578 {"\"run_name\": \"BM_RepeatTimeUnit/repeats:3\",$", MR_Next},
579 {"\"run_type\": \"aggregate\",$", MR_Next},
580 {"\"repetitions\": 3,$", MR_Next},
581 {"\"threads\": 1,$", MR_Next},
582 {"\"aggregate_name\": \"median\",$", MR_Next},
583 {"\"iterations\": 3,$", MR_Next},
584 {"\"time_unit\": \"us\",?$"},
585 {"\"name\": \"BM_RepeatTimeUnit/repeats:3_stddev\",$"},
586 {"\"run_name\": \"BM_RepeatTimeUnit/repeats:3\",$", MR_Next},
587 {"\"run_type\": \"aggregate\",$", MR_Next},
588 {"\"repetitions\": 3,$", MR_Next},
589 {"\"threads\": 1,$", MR_Next},
590 {"\"aggregate_name\": \"stddev\",$", MR_Next},
591 {"\"iterations\": 3,$", MR_Next},
592 {"\"time_unit\": \"us\",?$"}});
593 ADD_CASES(TC_CSVOut,
594 {{".*BM_RepeatTimeUnit/repeats:3 ", MR_Not},
595 {"^\"BM_RepeatTimeUnit/repeats:3_mean\",%csv_us_report$"},
596 {"^\"BM_RepeatTimeUnit/repeats:3_median\",%csv_us_report$"},
597 {"^\"BM_RepeatTimeUnit/repeats:3_stddev\",%csv_us_report$"}});
598
599 // ========================================================================= //
600 // -------------------- Testing user-provided statistics ------------------- //
601 // ========================================================================= //
602
__anone1aba2660102(const std::vector<double>& v) 603 const auto UserStatistics = [](const std::vector<double>& v) {
604 return v.back();
605 };
BM_UserStats(benchmark::State & state)606 void BM_UserStats(benchmark::State& state) {
607 for (auto _ : state) {
608 state.SetIterationTime(150 / 10e8);
609 }
610 }
611 // clang-format off
612 BENCHMARK(BM_UserStats)
613 ->Repetitions(3)
614 ->Iterations(5)
615 ->UseManualTime()
616 ->ComputeStatistics("", UserStatistics);
617 // clang-format on
618
619 // check that user-provided stats is calculated, and is after the default-ones
620 // empty string as name is intentional, it would sort before anything else
621 ADD_CASES(TC_ConsoleOut, {{"^BM_UserStats/iterations:5/repeats:3/manual_time [ "
622 "]* 150 ns %time [ ]*5$"},
623 {"^BM_UserStats/iterations:5/repeats:3/manual_time [ "
624 "]* 150 ns %time [ ]*5$"},
625 {"^BM_UserStats/iterations:5/repeats:3/manual_time [ "
626 "]* 150 ns %time [ ]*5$"},
627 {"^BM_UserStats/iterations:5/repeats:3/"
628 "manual_time_mean [ ]* 150 ns %time [ ]*3$"},
629 {"^BM_UserStats/iterations:5/repeats:3/"
630 "manual_time_median [ ]* 150 ns %time [ ]*3$"},
631 {"^BM_UserStats/iterations:5/repeats:3/"
632 "manual_time_stddev [ ]* 0.000 ns %time [ ]*3$"},
633 {"^BM_UserStats/iterations:5/repeats:3/manual_time_ "
634 "[ ]* 150 ns %time [ ]*3$"}});
635 ADD_CASES(
636 TC_JSONOut,
637 {{"\"name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$"},
638 {"\"run_name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$",
639 MR_Next},
640 {"\"run_type\": \"iteration\",$", MR_Next},
641 {"\"repetitions\": 3,$", MR_Next},
642 {"\"repetition_index\": 0,$", MR_Next},
643 {"\"threads\": 1,$", MR_Next},
644 {"\"iterations\": 5,$", MR_Next},
645 {"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next},
646 {"\"name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$"},
647 {"\"run_name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$",
648 MR_Next},
649 {"\"run_type\": \"iteration\",$", MR_Next},
650 {"\"repetitions\": 3,$", MR_Next},
651 {"\"repetition_index\": 1,$", MR_Next},
652 {"\"threads\": 1,$", MR_Next},
653 {"\"iterations\": 5,$", MR_Next},
654 {"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next},
655 {"\"name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$"},
656 {"\"run_name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$",
657 MR_Next},
658 {"\"run_type\": \"iteration\",$", MR_Next},
659 {"\"repetitions\": 3,$", MR_Next},
660 {"\"repetition_index\": 2,$", MR_Next},
661 {"\"threads\": 1,$", MR_Next},
662 {"\"iterations\": 5,$", MR_Next},
663 {"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next},
664 {"\"name\": \"BM_UserStats/iterations:5/repeats:3/manual_time_mean\",$"},
665 {"\"run_name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$",
666 MR_Next},
667 {"\"run_type\": \"aggregate\",$", MR_Next},
668 {"\"repetitions\": 3,$", MR_Next},
669 {"\"threads\": 1,$", MR_Next},
670 {"\"aggregate_name\": \"mean\",$", MR_Next},
671 {"\"iterations\": 3,$", MR_Next},
672 {"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next},
673 {"\"name\": \"BM_UserStats/iterations:5/repeats:3/manual_time_median\",$"},
674 {"\"run_name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$",
675 MR_Next},
676 {"\"run_type\": \"aggregate\",$", MR_Next},
677 {"\"repetitions\": 3,$", MR_Next},
678 {"\"threads\": 1,$", MR_Next},
679 {"\"aggregate_name\": \"median\",$", MR_Next},
680 {"\"iterations\": 3,$", MR_Next},
681 {"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next},
682 {"\"name\": \"BM_UserStats/iterations:5/repeats:3/manual_time_stddev\",$"},
683 {"\"run_name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$",
684 MR_Next},
685 {"\"run_type\": \"aggregate\",$", MR_Next},
686 {"\"repetitions\": 3,$", MR_Next},
687 {"\"threads\": 1,$", MR_Next},
688 {"\"aggregate_name\": \"stddev\",$", MR_Next},
689 {"\"iterations\": 3,$", MR_Next},
690 {"\"real_time\": %float,$", MR_Next},
691 {"\"name\": \"BM_UserStats/iterations:5/repeats:3/manual_time_\",$"},
692 {"\"run_name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$",
693 MR_Next},
694 {"\"run_type\": \"aggregate\",$", MR_Next},
695 {"\"repetitions\": 3,$", MR_Next},
696 {"\"threads\": 1,$", MR_Next},
697 {"\"aggregate_name\": \"\",$", MR_Next},
698 {"\"iterations\": 3,$", MR_Next},
699 {"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next}});
700 ADD_CASES(
701 TC_CSVOut,
702 {{"^\"BM_UserStats/iterations:5/repeats:3/manual_time\",%csv_report$"},
703 {"^\"BM_UserStats/iterations:5/repeats:3/manual_time\",%csv_report$"},
704 {"^\"BM_UserStats/iterations:5/repeats:3/manual_time\",%csv_report$"},
705 {"^\"BM_UserStats/iterations:5/repeats:3/manual_time_mean\",%csv_report$"},
706 {"^\"BM_UserStats/iterations:5/repeats:3/"
707 "manual_time_median\",%csv_report$"},
708 {"^\"BM_UserStats/iterations:5/repeats:3/"
709 "manual_time_stddev\",%csv_report$"},
710 {"^\"BM_UserStats/iterations:5/repeats:3/manual_time_\",%csv_report$"}});
711
712 // ========================================================================= //
713 // ------------------------- Testing StrEscape JSON ------------------------ //
714 // ========================================================================= //
715 #if 0 // enable when csv testing code correctly handles multi-line fields
716 void BM_JSON_Format(benchmark::State& state) {
717 state.SkipWithError("val\b\f\n\r\t\\\"with\"es,capes");
718 for (auto _ : state) {
719 }
720 }
721 BENCHMARK(BM_JSON_Format);
722 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_JSON_Format\",$"},
723 {"\"run_name\": \"BM_JSON_Format\",$", MR_Next},
724 {"\"run_type\": \"iteration\",$", MR_Next},
725 {"\"repetitions\": 0,$", MR_Next},
726 {"\"repetition_index\": 0,$", MR_Next},
727 {"\"threads\": 1,$", MR_Next},
728 {"\"error_occurred\": true,$", MR_Next},
729 {R"("error_message": "val\\b\\f\\n\\r\\t\\\\\\"with\\"es,capes",$)", MR_Next}});
730 #endif
731 // ========================================================================= //
732 // -------------------------- Testing CsvEscape ---------------------------- //
733 // ========================================================================= //
734
BM_CSV_Format(benchmark::State & state)735 void BM_CSV_Format(benchmark::State& state) {
736 state.SkipWithError("\"freedom\"");
737 for (auto _ : state) {
738 }
739 }
740 BENCHMARK(BM_CSV_Format);
741 ADD_CASES(TC_CSVOut, {{"^\"BM_CSV_Format\",,,,,,,,true,\"\"\"freedom\"\"\"$"}});
742
743 // ========================================================================= //
744 // --------------------------- TEST CASES END ------------------------------ //
745 // ========================================================================= //
746
main(int argc,char * argv[])747 int main(int argc, char* argv[]) { RunOutputTests(argc, argv); }
748