1
2 #undef NDEBUG
3 #include <numeric>
4 #include <utility>
5
6 #include "benchmark/benchmark.h"
7 #include "output_test.h"
8
9 // ========================================================================= //
10 // ---------------------- Testing Prologue Output -------------------------- //
11 // ========================================================================= //
12
13 ADD_CASES(TC_ConsoleOut, {{"^[-]+$", MR_Next},
14 {"^Benchmark %s Time %s CPU %s Iterations$", MR_Next},
15 {"^[-]+$", MR_Next}});
AddContextCases()16 static int AddContextCases() {
17 AddCases(TC_ConsoleErr,
18 {
19 {"^%int-%int-%intT%int:%int:%int[-+]%int:%int$", MR_Default},
20 {"Running .*(/|\\\\)reporter_output_test(\\.exe)?$", MR_Next},
21 {"Run on \\(%int X %float MHz CPU s?\\)", MR_Next},
22 });
23 AddCases(TC_JSONOut,
24 {{"^\\{", MR_Default},
25 {"\"context\":", MR_Next},
26 {"\"date\": \"", MR_Next},
27 {"\"host_name\":", MR_Next},
28 {"\"executable\": \".*(/|\\\\)reporter_output_test(\\.exe)?\",",
29 MR_Next},
30 {"\"num_cpus\": %int,$", MR_Next},
31 {"\"mhz_per_cpu\": %float,$", MR_Next},
32 {"\"caches\": \\[$", MR_Default}});
33 auto const& Info = benchmark::CPUInfo::Get();
34 auto const& Caches = Info.caches;
35 if (!Caches.empty()) {
36 AddCases(TC_ConsoleErr, {{"CPU Caches:$", MR_Next}});
37 }
38 for (size_t I = 0; I < Caches.size(); ++I) {
39 std::string num_caches_str =
40 Caches[I].num_sharing != 0 ? " \\(x%int\\)$" : "$";
41 AddCases(TC_ConsoleErr,
42 {{"L%int (Data|Instruction|Unified) %int KiB" + num_caches_str,
43 MR_Next}});
44 AddCases(TC_JSONOut, {{"\\{$", MR_Next},
45 {"\"type\": \"", MR_Next},
46 {"\"level\": %int,$", MR_Next},
47 {"\"size\": %int,$", MR_Next},
48 {"\"num_sharing\": %int$", MR_Next},
49 {"}[,]{0,1}$", MR_Next}});
50 }
51 AddCases(TC_JSONOut, {{"],$"}});
52 auto const& LoadAvg = Info.load_avg;
53 if (!LoadAvg.empty()) {
54 AddCases(TC_ConsoleErr,
55 {{"Load Average: (%float, ){0,2}%float$", MR_Next}});
56 }
57 AddCases(TC_JSONOut, {{"\"load_avg\": \\[(%float,?){0,3}],$", MR_Next}});
58 return 0;
59 }
60 int dummy_register = AddContextCases();
61 ADD_CASES(TC_CSVOut, {{"%csv_header"}});
62
63 // ========================================================================= //
64 // ------------------------ Testing Basic Output --------------------------- //
65 // ========================================================================= //
66
BM_basic(benchmark::State & state)67 void BM_basic(benchmark::State& state) {
68 for (auto _ : state) {
69 }
70 }
71 BENCHMARK(BM_basic);
72
73 ADD_CASES(TC_ConsoleOut, {{"^BM_basic %console_report$"}});
74 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_basic\",$"},
75 {"\"family_index\": 0,$", MR_Next},
76 {"\"per_family_instance_index\": 0,$", MR_Next},
77 {"\"run_name\": \"BM_basic\",$", MR_Next},
78 {"\"run_type\": \"iteration\",$", MR_Next},
79 {"\"repetitions\": 1,$", MR_Next},
80 {"\"repetition_index\": 0,$", MR_Next},
81 {"\"threads\": 1,$", MR_Next},
82 {"\"iterations\": %int,$", MR_Next},
83 {"\"real_time\": %float,$", MR_Next},
84 {"\"cpu_time\": %float,$", MR_Next},
85 {"\"time_unit\": \"ns\"$", MR_Next},
86 {"}", MR_Next}});
87 ADD_CASES(TC_CSVOut, {{"^\"BM_basic\",%csv_report$"}});
88
89 // ========================================================================= //
90 // ------------------------ Testing Bytes per Second Output ---------------- //
91 // ========================================================================= //
92
BM_bytes_per_second(benchmark::State & state)93 void BM_bytes_per_second(benchmark::State& state) {
94 for (auto _ : state) {
95 // This test requires a non-zero CPU time to avoid divide-by-zero
96 auto iterations = state.iterations();
97 benchmark::DoNotOptimize(iterations);
98 }
99 state.SetBytesProcessed(1);
100 }
101 BENCHMARK(BM_bytes_per_second);
102
103 ADD_CASES(TC_ConsoleOut, {{"^BM_bytes_per_second %console_report "
104 "bytes_per_second=%float[kM]{0,1}/s$"}});
105 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_bytes_per_second\",$"},
106 {"\"family_index\": 1,$", MR_Next},
107 {"\"per_family_instance_index\": 0,$", MR_Next},
108 {"\"run_name\": \"BM_bytes_per_second\",$", MR_Next},
109 {"\"run_type\": \"iteration\",$", MR_Next},
110 {"\"repetitions\": 1,$", MR_Next},
111 {"\"repetition_index\": 0,$", MR_Next},
112 {"\"threads\": 1,$", MR_Next},
113 {"\"iterations\": %int,$", MR_Next},
114 {"\"real_time\": %float,$", MR_Next},
115 {"\"cpu_time\": %float,$", MR_Next},
116 {"\"time_unit\": \"ns\",$", MR_Next},
117 {"\"bytes_per_second\": %float$", MR_Next},
118 {"}", MR_Next}});
119 ADD_CASES(TC_CSVOut, {{"^\"BM_bytes_per_second\",%csv_bytes_report$"}});
120
121 // ========================================================================= //
122 // ------------------------ Testing Items per Second Output ---------------- //
123 // ========================================================================= //
124
BM_items_per_second(benchmark::State & state)125 void BM_items_per_second(benchmark::State& state) {
126 for (auto _ : state) {
127 // This test requires a non-zero CPU time to avoid divide-by-zero
128 auto iterations = state.iterations();
129 benchmark::DoNotOptimize(iterations);
130 }
131 state.SetItemsProcessed(1);
132 }
133 BENCHMARK(BM_items_per_second);
134
135 ADD_CASES(TC_ConsoleOut, {{"^BM_items_per_second %console_report "
136 "items_per_second=%float[kM]{0,1}/s$"}});
137 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_items_per_second\",$"},
138 {"\"family_index\": 2,$", MR_Next},
139 {"\"per_family_instance_index\": 0,$", MR_Next},
140 {"\"run_name\": \"BM_items_per_second\",$", MR_Next},
141 {"\"run_type\": \"iteration\",$", MR_Next},
142 {"\"repetitions\": 1,$", MR_Next},
143 {"\"repetition_index\": 0,$", MR_Next},
144 {"\"threads\": 1,$", MR_Next},
145 {"\"iterations\": %int,$", MR_Next},
146 {"\"real_time\": %float,$", MR_Next},
147 {"\"cpu_time\": %float,$", MR_Next},
148 {"\"time_unit\": \"ns\",$", MR_Next},
149 {"\"items_per_second\": %float$", MR_Next},
150 {"}", MR_Next}});
151 ADD_CASES(TC_CSVOut, {{"^\"BM_items_per_second\",%csv_items_report$"}});
152
153 // ========================================================================= //
154 // ------------------------ Testing Label Output --------------------------- //
155 // ========================================================================= //
156
BM_label(benchmark::State & state)157 void BM_label(benchmark::State& state) {
158 for (auto _ : state) {
159 }
160 state.SetLabel("some label");
161 }
162 BENCHMARK(BM_label);
163
164 ADD_CASES(TC_ConsoleOut, {{"^BM_label %console_report some label$"}});
165 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_label\",$"},
166 {"\"family_index\": 3,$", MR_Next},
167 {"\"per_family_instance_index\": 0,$", MR_Next},
168 {"\"run_name\": \"BM_label\",$", MR_Next},
169 {"\"run_type\": \"iteration\",$", MR_Next},
170 {"\"repetitions\": 1,$", MR_Next},
171 {"\"repetition_index\": 0,$", MR_Next},
172 {"\"threads\": 1,$", MR_Next},
173 {"\"iterations\": %int,$", MR_Next},
174 {"\"real_time\": %float,$", MR_Next},
175 {"\"cpu_time\": %float,$", MR_Next},
176 {"\"time_unit\": \"ns\",$", MR_Next},
177 {"\"label\": \"some label\"$", MR_Next},
178 {"}", MR_Next}});
179 ADD_CASES(TC_CSVOut, {{"^\"BM_label\",%csv_label_report_begin\"some "
180 "label\"%csv_label_report_end$"}});
181
182 // ========================================================================= //
183 // ------------------------ Testing Time Label Output ---------------------- //
184 // ========================================================================= //
185
BM_time_label_nanosecond(benchmark::State & state)186 void BM_time_label_nanosecond(benchmark::State& state) {
187 for (auto _ : state) {
188 }
189 }
190 BENCHMARK(BM_time_label_nanosecond)->Unit(benchmark::kNanosecond);
191
192 ADD_CASES(TC_ConsoleOut, {{"^BM_time_label_nanosecond %console_report$"}});
193 ADD_CASES(TC_JSONOut,
194 {{"\"name\": \"BM_time_label_nanosecond\",$"},
195 {"\"family_index\": 4,$", MR_Next},
196 {"\"per_family_instance_index\": 0,$", MR_Next},
197 {"\"run_name\": \"BM_time_label_nanosecond\",$", MR_Next},
198 {"\"run_type\": \"iteration\",$", MR_Next},
199 {"\"repetitions\": 1,$", MR_Next},
200 {"\"repetition_index\": 0,$", MR_Next},
201 {"\"threads\": 1,$", MR_Next},
202 {"\"iterations\": %int,$", MR_Next},
203 {"\"real_time\": %float,$", MR_Next},
204 {"\"cpu_time\": %float,$", MR_Next},
205 {"\"time_unit\": \"ns\"$", MR_Next},
206 {"}", MR_Next}});
207 ADD_CASES(TC_CSVOut, {{"^\"BM_time_label_nanosecond\",%csv_report$"}});
208
BM_time_label_microsecond(benchmark::State & state)209 void BM_time_label_microsecond(benchmark::State& state) {
210 for (auto _ : state) {
211 }
212 }
213 BENCHMARK(BM_time_label_microsecond)->Unit(benchmark::kMicrosecond);
214
215 ADD_CASES(TC_ConsoleOut, {{"^BM_time_label_microsecond %console_us_report$"}});
216 ADD_CASES(TC_JSONOut,
217 {{"\"name\": \"BM_time_label_microsecond\",$"},
218 {"\"family_index\": 5,$", MR_Next},
219 {"\"per_family_instance_index\": 0,$", MR_Next},
220 {"\"run_name\": \"BM_time_label_microsecond\",$", MR_Next},
221 {"\"run_type\": \"iteration\",$", MR_Next},
222 {"\"repetitions\": 1,$", MR_Next},
223 {"\"repetition_index\": 0,$", MR_Next},
224 {"\"threads\": 1,$", MR_Next},
225 {"\"iterations\": %int,$", MR_Next},
226 {"\"real_time\": %float,$", MR_Next},
227 {"\"cpu_time\": %float,$", MR_Next},
228 {"\"time_unit\": \"us\"$", MR_Next},
229 {"}", MR_Next}});
230 ADD_CASES(TC_CSVOut, {{"^\"BM_time_label_microsecond\",%csv_us_report$"}});
231
BM_time_label_millisecond(benchmark::State & state)232 void BM_time_label_millisecond(benchmark::State& state) {
233 for (auto _ : state) {
234 }
235 }
236 BENCHMARK(BM_time_label_millisecond)->Unit(benchmark::kMillisecond);
237
238 ADD_CASES(TC_ConsoleOut, {{"^BM_time_label_millisecond %console_ms_report$"}});
239 ADD_CASES(TC_JSONOut,
240 {{"\"name\": \"BM_time_label_millisecond\",$"},
241 {"\"family_index\": 6,$", MR_Next},
242 {"\"per_family_instance_index\": 0,$", MR_Next},
243 {"\"run_name\": \"BM_time_label_millisecond\",$", MR_Next},
244 {"\"run_type\": \"iteration\",$", MR_Next},
245 {"\"repetitions\": 1,$", MR_Next},
246 {"\"repetition_index\": 0,$", MR_Next},
247 {"\"threads\": 1,$", MR_Next},
248 {"\"iterations\": %int,$", MR_Next},
249 {"\"real_time\": %float,$", MR_Next},
250 {"\"cpu_time\": %float,$", MR_Next},
251 {"\"time_unit\": \"ms\"$", MR_Next},
252 {"}", MR_Next}});
253 ADD_CASES(TC_CSVOut, {{"^\"BM_time_label_millisecond\",%csv_ms_report$"}});
254
BM_time_label_second(benchmark::State & state)255 void BM_time_label_second(benchmark::State& state) {
256 for (auto _ : state) {
257 }
258 }
259 BENCHMARK(BM_time_label_second)->Unit(benchmark::kSecond);
260
261 ADD_CASES(TC_ConsoleOut, {{"^BM_time_label_second %console_s_report$"}});
262 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_time_label_second\",$"},
263 {"\"family_index\": 7,$", MR_Next},
264 {"\"per_family_instance_index\": 0,$", MR_Next},
265 {"\"run_name\": \"BM_time_label_second\",$", MR_Next},
266 {"\"run_type\": \"iteration\",$", MR_Next},
267 {"\"repetitions\": 1,$", MR_Next},
268 {"\"repetition_index\": 0,$", MR_Next},
269 {"\"threads\": 1,$", MR_Next},
270 {"\"iterations\": %int,$", MR_Next},
271 {"\"real_time\": %float,$", MR_Next},
272 {"\"cpu_time\": %float,$", MR_Next},
273 {"\"time_unit\": \"s\"$", MR_Next},
274 {"}", MR_Next}});
275 ADD_CASES(TC_CSVOut, {{"^\"BM_time_label_second\",%csv_s_report$"}});
276
277 // ========================================================================= //
278 // ------------------------ Testing Error Output --------------------------- //
279 // ========================================================================= //
280
BM_error(benchmark::State & state)281 void BM_error(benchmark::State& state) {
282 state.SkipWithError("message");
283 for (auto _ : state) {
284 }
285 }
286 BENCHMARK(BM_error);
287 ADD_CASES(TC_ConsoleOut, {{"^BM_error[ ]+ERROR OCCURRED: 'message'$"}});
288 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_error\",$"},
289 {"\"family_index\": 8,$", MR_Next},
290 {"\"per_family_instance_index\": 0,$", MR_Next},
291 {"\"run_name\": \"BM_error\",$", MR_Next},
292 {"\"run_type\": \"iteration\",$", MR_Next},
293 {"\"repetitions\": 1,$", MR_Next},
294 {"\"repetition_index\": 0,$", MR_Next},
295 {"\"threads\": 1,$", MR_Next},
296 {"\"error_occurred\": true,$", MR_Next},
297 {"\"error_message\": \"message\",$", MR_Next}});
298
299 ADD_CASES(TC_CSVOut, {{"^\"BM_error\",,,,,,,,true,\"message\"$"}});
300
301 // ========================================================================= //
302 // ------------------------ Testing No Arg Name Output -----------------------
303 // //
304 // ========================================================================= //
305
BM_no_arg_name(benchmark::State & state)306 void BM_no_arg_name(benchmark::State& state) {
307 for (auto _ : state) {
308 }
309 }
310 BENCHMARK(BM_no_arg_name)->Arg(3);
311 ADD_CASES(TC_ConsoleOut, {{"^BM_no_arg_name/3 %console_report$"}});
312 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_no_arg_name/3\",$"},
313 {"\"family_index\": 9,$", MR_Next},
314 {"\"per_family_instance_index\": 0,$", MR_Next},
315 {"\"run_name\": \"BM_no_arg_name/3\",$", MR_Next},
316 {"\"run_type\": \"iteration\",$", MR_Next},
317 {"\"repetitions\": 1,$", MR_Next},
318 {"\"repetition_index\": 0,$", MR_Next},
319 {"\"threads\": 1,$", MR_Next}});
320 ADD_CASES(TC_CSVOut, {{"^\"BM_no_arg_name/3\",%csv_report$"}});
321
322 // ========================================================================= //
323 // ------------------------ Testing Arg Name Output ------------------------ //
324 // ========================================================================= //
325
BM_arg_name(benchmark::State & state)326 void BM_arg_name(benchmark::State& state) {
327 for (auto _ : state) {
328 }
329 }
330 BENCHMARK(BM_arg_name)->ArgName("first")->Arg(3);
331 ADD_CASES(TC_ConsoleOut, {{"^BM_arg_name/first:3 %console_report$"}});
332 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_arg_name/first:3\",$"},
333 {"\"family_index\": 10,$", MR_Next},
334 {"\"per_family_instance_index\": 0,$", MR_Next},
335 {"\"run_name\": \"BM_arg_name/first:3\",$", MR_Next},
336 {"\"run_type\": \"iteration\",$", MR_Next},
337 {"\"repetitions\": 1,$", MR_Next},
338 {"\"repetition_index\": 0,$", MR_Next},
339 {"\"threads\": 1,$", MR_Next}});
340 ADD_CASES(TC_CSVOut, {{"^\"BM_arg_name/first:3\",%csv_report$"}});
341
342 // ========================================================================= //
343 // ------------------------ Testing Arg Names Output ----------------------- //
344 // ========================================================================= //
345
BM_arg_names(benchmark::State & state)346 void BM_arg_names(benchmark::State& state) {
347 for (auto _ : state) {
348 }
349 }
350 BENCHMARK(BM_arg_names)->Args({2, 5, 4})->ArgNames({"first", "", "third"});
351 ADD_CASES(TC_ConsoleOut,
352 {{"^BM_arg_names/first:2/5/third:4 %console_report$"}});
353 ADD_CASES(TC_JSONOut,
354 {{"\"name\": \"BM_arg_names/first:2/5/third:4\",$"},
355 {"\"family_index\": 11,$", MR_Next},
356 {"\"per_family_instance_index\": 0,$", MR_Next},
357 {"\"run_name\": \"BM_arg_names/first:2/5/third:4\",$", MR_Next},
358 {"\"run_type\": \"iteration\",$", MR_Next},
359 {"\"repetitions\": 1,$", MR_Next},
360 {"\"repetition_index\": 0,$", MR_Next},
361 {"\"threads\": 1,$", MR_Next}});
362 ADD_CASES(TC_CSVOut, {{"^\"BM_arg_names/first:2/5/third:4\",%csv_report$"}});
363
364 // ========================================================================= //
365 // ------------------------ Testing Name Output ---------------------------- //
366 // ========================================================================= //
367
BM_name(benchmark::State & state)368 void BM_name(benchmark::State& state) {
369 for (auto _ : state) {
370 }
371 }
372 BENCHMARK(BM_name)->Name("BM_custom_name");
373
374 ADD_CASES(TC_ConsoleOut, {{"^BM_custom_name %console_report$"}});
375 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_custom_name\",$"},
376 {"\"family_index\": 12,$", MR_Next},
377 {"\"per_family_instance_index\": 0,$", MR_Next},
378 {"\"run_name\": \"BM_custom_name\",$", MR_Next},
379 {"\"run_type\": \"iteration\",$", MR_Next},
380 {"\"repetitions\": 1,$", MR_Next},
381 {"\"repetition_index\": 0,$", MR_Next},
382 {"\"threads\": 1,$", MR_Next},
383 {"\"iterations\": %int,$", MR_Next},
384 {"\"real_time\": %float,$", MR_Next},
385 {"\"cpu_time\": %float,$", MR_Next},
386 {"\"time_unit\": \"ns\"$", MR_Next},
387 {"}", MR_Next}});
388 ADD_CASES(TC_CSVOut, {{"^\"BM_custom_name\",%csv_report$"}});
389
390 // ========================================================================= //
391 // ------------------------ Testing Big Args Output ------------------------ //
392 // ========================================================================= //
393
BM_BigArgs(benchmark::State & state)394 void BM_BigArgs(benchmark::State& state) {
395 for (auto _ : state) {
396 }
397 }
398 BENCHMARK(BM_BigArgs)->RangeMultiplier(2)->Range(1U << 30U, 1U << 31U);
399 ADD_CASES(TC_ConsoleOut, {{"^BM_BigArgs/1073741824 %console_report$"},
400 {"^BM_BigArgs/2147483648 %console_report$"}});
401
402 // ========================================================================= //
403 // ----------------------- Testing Complexity Output ----------------------- //
404 // ========================================================================= //
405
BM_Complexity_O1(benchmark::State & state)406 void BM_Complexity_O1(benchmark::State& state) {
407 for (auto _ : state) {
408 // This test requires a non-zero CPU time to avoid divide-by-zero
409 auto iterations = state.iterations();
410 benchmark::DoNotOptimize(iterations);
411 }
412 state.SetComplexityN(state.range(0));
413 }
414 BENCHMARK(BM_Complexity_O1)->Range(1, 1 << 18)->Complexity(benchmark::o1);
415 SET_SUBSTITUTIONS({{"%bigOStr", "[ ]* %float \\([0-9]+\\)"},
416 {"%RMS", "[ ]*[0-9]+ %"}});
417 ADD_CASES(TC_ConsoleOut, {{"^BM_Complexity_O1_BigO %bigOStr %bigOStr[ ]*$"},
418 {"^BM_Complexity_O1_RMS %RMS %RMS[ ]*$"}});
419
420 // ========================================================================= //
421 // ----------------------- Testing Aggregate Output ------------------------ //
422 // ========================================================================= //
423
424 // Test that non-aggregate data is printed by default
BM_Repeat(benchmark::State & state)425 void BM_Repeat(benchmark::State& state) {
426 for (auto _ : state) {
427 }
428 }
429 // need two repetitions min to be able to output any aggregate output
430 BENCHMARK(BM_Repeat)->Repetitions(2);
431 ADD_CASES(TC_ConsoleOut,
432 {{"^BM_Repeat/repeats:2 %console_report$"},
433 {"^BM_Repeat/repeats:2 %console_report$"},
434 {"^BM_Repeat/repeats:2_mean %console_time_only_report [ ]*2$"},
435 {"^BM_Repeat/repeats:2_median %console_time_only_report [ ]*2$"},
436 {"^BM_Repeat/repeats:2_stddev %console_time_only_report [ ]*2$"}});
437 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Repeat/repeats:2\",$"},
438 {"\"family_index\": 15,$", MR_Next},
439 {"\"per_family_instance_index\": 0,$", MR_Next},
440 {"\"run_name\": \"BM_Repeat/repeats:2\"", MR_Next},
441 {"\"run_type\": \"iteration\",$", MR_Next},
442 {"\"repetitions\": 2,$", MR_Next},
443 {"\"repetition_index\": 0,$", MR_Next},
444 {"\"threads\": 1,$", MR_Next},
445 {"\"name\": \"BM_Repeat/repeats:2\",$"},
446 {"\"family_index\": 15,$", MR_Next},
447 {"\"per_family_instance_index\": 0,$", MR_Next},
448 {"\"run_name\": \"BM_Repeat/repeats:2\",$", MR_Next},
449 {"\"run_type\": \"iteration\",$", MR_Next},
450 {"\"repetitions\": 2,$", MR_Next},
451 {"\"repetition_index\": 1,$", MR_Next},
452 {"\"threads\": 1,$", MR_Next},
453 {"\"name\": \"BM_Repeat/repeats:2_mean\",$"},
454 {"\"family_index\": 15,$", MR_Next},
455 {"\"per_family_instance_index\": 0,$", MR_Next},
456 {"\"run_name\": \"BM_Repeat/repeats:2\",$", MR_Next},
457 {"\"run_type\": \"aggregate\",$", MR_Next},
458 {"\"repetitions\": 2,$", MR_Next},
459 {"\"threads\": 1,$", MR_Next},
460 {"\"aggregate_name\": \"mean\",$", MR_Next},
461 {"\"aggregate_unit\": \"time\",$", MR_Next},
462 {"\"iterations\": 2,$", MR_Next},
463 {"\"name\": \"BM_Repeat/repeats:2_median\",$"},
464 {"\"family_index\": 15,$", MR_Next},
465 {"\"per_family_instance_index\": 0,$", MR_Next},
466 {"\"run_name\": \"BM_Repeat/repeats:2\",$", MR_Next},
467 {"\"run_type\": \"aggregate\",$", MR_Next},
468 {"\"repetitions\": 2,$", MR_Next},
469 {"\"threads\": 1,$", MR_Next},
470 {"\"aggregate_name\": \"median\",$", MR_Next},
471 {"\"aggregate_unit\": \"time\",$", MR_Next},
472 {"\"iterations\": 2,$", MR_Next},
473 {"\"name\": \"BM_Repeat/repeats:2_stddev\",$"},
474 {"\"family_index\": 15,$", MR_Next},
475 {"\"per_family_instance_index\": 0,$", MR_Next},
476 {"\"run_name\": \"BM_Repeat/repeats:2\",$", MR_Next},
477 {"\"run_type\": \"aggregate\",$", MR_Next},
478 {"\"repetitions\": 2,$", MR_Next},
479 {"\"threads\": 1,$", MR_Next},
480 {"\"aggregate_name\": \"stddev\",$", MR_Next},
481 {"\"aggregate_unit\": \"time\",$", MR_Next},
482 {"\"iterations\": 2,$", MR_Next}});
483 ADD_CASES(TC_CSVOut, {{"^\"BM_Repeat/repeats:2\",%csv_report$"},
484 {"^\"BM_Repeat/repeats:2\",%csv_report$"},
485 {"^\"BM_Repeat/repeats:2_mean\",%csv_report$"},
486 {"^\"BM_Repeat/repeats:2_median\",%csv_report$"},
487 {"^\"BM_Repeat/repeats:2_stddev\",%csv_report$"}});
488 // but for two repetitions, mean and median is the same, so let's repeat..
489 BENCHMARK(BM_Repeat)->Repetitions(3);
490 ADD_CASES(TC_ConsoleOut,
491 {{"^BM_Repeat/repeats:3 %console_report$"},
492 {"^BM_Repeat/repeats:3 %console_report$"},
493 {"^BM_Repeat/repeats:3 %console_report$"},
494 {"^BM_Repeat/repeats:3_mean %console_time_only_report [ ]*3$"},
495 {"^BM_Repeat/repeats:3_median %console_time_only_report [ ]*3$"},
496 {"^BM_Repeat/repeats:3_stddev %console_time_only_report [ ]*3$"}});
497 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Repeat/repeats:3\",$"},
498 {"\"family_index\": 16,$", MR_Next},
499 {"\"per_family_instance_index\": 0,$", MR_Next},
500 {"\"run_name\": \"BM_Repeat/repeats:3\",$", MR_Next},
501 {"\"run_type\": \"iteration\",$", MR_Next},
502 {"\"repetitions\": 3,$", MR_Next},
503 {"\"repetition_index\": 0,$", MR_Next},
504 {"\"threads\": 1,$", MR_Next},
505 {"\"name\": \"BM_Repeat/repeats:3\",$"},
506 {"\"family_index\": 16,$", MR_Next},
507 {"\"per_family_instance_index\": 0,$", MR_Next},
508 {"\"run_name\": \"BM_Repeat/repeats:3\",$", MR_Next},
509 {"\"run_type\": \"iteration\",$", MR_Next},
510 {"\"repetitions\": 3,$", MR_Next},
511 {"\"repetition_index\": 1,$", MR_Next},
512 {"\"threads\": 1,$", MR_Next},
513 {"\"name\": \"BM_Repeat/repeats:3\",$"},
514 {"\"family_index\": 16,$", MR_Next},
515 {"\"per_family_instance_index\": 0,$", MR_Next},
516 {"\"run_name\": \"BM_Repeat/repeats:3\",$", MR_Next},
517 {"\"run_type\": \"iteration\",$", MR_Next},
518 {"\"repetitions\": 3,$", MR_Next},
519 {"\"repetition_index\": 2,$", MR_Next},
520 {"\"threads\": 1,$", MR_Next},
521 {"\"name\": \"BM_Repeat/repeats:3_mean\",$"},
522 {"\"family_index\": 16,$", MR_Next},
523 {"\"per_family_instance_index\": 0,$", MR_Next},
524 {"\"run_name\": \"BM_Repeat/repeats:3\",$", MR_Next},
525 {"\"run_type\": \"aggregate\",$", MR_Next},
526 {"\"repetitions\": 3,$", MR_Next},
527 {"\"threads\": 1,$", MR_Next},
528 {"\"aggregate_name\": \"mean\",$", MR_Next},
529 {"\"aggregate_unit\": \"time\",$", MR_Next},
530 {"\"iterations\": 3,$", MR_Next},
531 {"\"name\": \"BM_Repeat/repeats:3_median\",$"},
532 {"\"family_index\": 16,$", MR_Next},
533 {"\"per_family_instance_index\": 0,$", MR_Next},
534 {"\"run_name\": \"BM_Repeat/repeats:3\",$", MR_Next},
535 {"\"run_type\": \"aggregate\",$", MR_Next},
536 {"\"repetitions\": 3,$", MR_Next},
537 {"\"threads\": 1,$", MR_Next},
538 {"\"aggregate_name\": \"median\",$", MR_Next},
539 {"\"aggregate_unit\": \"time\",$", MR_Next},
540 {"\"iterations\": 3,$", MR_Next},
541 {"\"name\": \"BM_Repeat/repeats:3_stddev\",$"},
542 {"\"family_index\": 16,$", MR_Next},
543 {"\"per_family_instance_index\": 0,$", MR_Next},
544 {"\"run_name\": \"BM_Repeat/repeats:3\",$", MR_Next},
545 {"\"run_type\": \"aggregate\",$", MR_Next},
546 {"\"repetitions\": 3,$", MR_Next},
547 {"\"threads\": 1,$", MR_Next},
548 {"\"aggregate_name\": \"stddev\",$", MR_Next},
549 {"\"aggregate_unit\": \"time\",$", MR_Next},
550 {"\"iterations\": 3,$", MR_Next}});
551 ADD_CASES(TC_CSVOut, {{"^\"BM_Repeat/repeats:3\",%csv_report$"},
552 {"^\"BM_Repeat/repeats:3\",%csv_report$"},
553 {"^\"BM_Repeat/repeats:3\",%csv_report$"},
554 {"^\"BM_Repeat/repeats:3_mean\",%csv_report$"},
555 {"^\"BM_Repeat/repeats:3_median\",%csv_report$"},
556 {"^\"BM_Repeat/repeats:3_stddev\",%csv_report$"}});
557 // median differs between even/odd number of repetitions, so just to be sure
558 BENCHMARK(BM_Repeat)->Repetitions(4);
559 ADD_CASES(TC_ConsoleOut,
560 {{"^BM_Repeat/repeats:4 %console_report$"},
561 {"^BM_Repeat/repeats:4 %console_report$"},
562 {"^BM_Repeat/repeats:4 %console_report$"},
563 {"^BM_Repeat/repeats:4 %console_report$"},
564 {"^BM_Repeat/repeats:4_mean %console_time_only_report [ ]*4$"},
565 {"^BM_Repeat/repeats:4_median %console_time_only_report [ ]*4$"},
566 {"^BM_Repeat/repeats:4_stddev %console_time_only_report [ ]*4$"}});
567 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Repeat/repeats:4\",$"},
568 {"\"family_index\": 17,$", MR_Next},
569 {"\"per_family_instance_index\": 0,$", MR_Next},
570 {"\"run_name\": \"BM_Repeat/repeats:4\",$", MR_Next},
571 {"\"run_type\": \"iteration\",$", MR_Next},
572 {"\"repetitions\": 4,$", MR_Next},
573 {"\"repetition_index\": 0,$", MR_Next},
574 {"\"threads\": 1,$", MR_Next},
575 {"\"name\": \"BM_Repeat/repeats:4\",$"},
576 {"\"family_index\": 17,$", MR_Next},
577 {"\"per_family_instance_index\": 0,$", MR_Next},
578 {"\"run_name\": \"BM_Repeat/repeats:4\",$", MR_Next},
579 {"\"run_type\": \"iteration\",$", MR_Next},
580 {"\"repetitions\": 4,$", MR_Next},
581 {"\"repetition_index\": 1,$", MR_Next},
582 {"\"threads\": 1,$", MR_Next},
583 {"\"name\": \"BM_Repeat/repeats:4\",$"},
584 {"\"family_index\": 17,$", MR_Next},
585 {"\"per_family_instance_index\": 0,$", MR_Next},
586 {"\"run_name\": \"BM_Repeat/repeats:4\",$", MR_Next},
587 {"\"run_type\": \"iteration\",$", MR_Next},
588 {"\"repetitions\": 4,$", MR_Next},
589 {"\"repetition_index\": 2,$", MR_Next},
590 {"\"threads\": 1,$", MR_Next},
591 {"\"name\": \"BM_Repeat/repeats:4\",$"},
592 {"\"family_index\": 17,$", MR_Next},
593 {"\"per_family_instance_index\": 0,$", MR_Next},
594 {"\"run_name\": \"BM_Repeat/repeats:4\",$", MR_Next},
595 {"\"run_type\": \"iteration\",$", MR_Next},
596 {"\"repetitions\": 4,$", MR_Next},
597 {"\"repetition_index\": 3,$", MR_Next},
598 {"\"threads\": 1,$", MR_Next},
599 {"\"name\": \"BM_Repeat/repeats:4_mean\",$"},
600 {"\"family_index\": 17,$", MR_Next},
601 {"\"per_family_instance_index\": 0,$", MR_Next},
602 {"\"run_name\": \"BM_Repeat/repeats:4\",$", MR_Next},
603 {"\"run_type\": \"aggregate\",$", MR_Next},
604 {"\"repetitions\": 4,$", MR_Next},
605 {"\"threads\": 1,$", MR_Next},
606 {"\"aggregate_name\": \"mean\",$", MR_Next},
607 {"\"aggregate_unit\": \"time\",$", MR_Next},
608 {"\"iterations\": 4,$", MR_Next},
609 {"\"name\": \"BM_Repeat/repeats:4_median\",$"},
610 {"\"family_index\": 17,$", MR_Next},
611 {"\"per_family_instance_index\": 0,$", MR_Next},
612 {"\"run_name\": \"BM_Repeat/repeats:4\",$", MR_Next},
613 {"\"run_type\": \"aggregate\",$", MR_Next},
614 {"\"repetitions\": 4,$", MR_Next},
615 {"\"threads\": 1,$", MR_Next},
616 {"\"aggregate_name\": \"median\",$", MR_Next},
617 {"\"aggregate_unit\": \"time\",$", MR_Next},
618 {"\"iterations\": 4,$", MR_Next},
619 {"\"name\": \"BM_Repeat/repeats:4_stddev\",$"},
620 {"\"family_index\": 17,$", MR_Next},
621 {"\"per_family_instance_index\": 0,$", MR_Next},
622 {"\"run_name\": \"BM_Repeat/repeats:4\",$", MR_Next},
623 {"\"run_type\": \"aggregate\",$", MR_Next},
624 {"\"repetitions\": 4,$", MR_Next},
625 {"\"threads\": 1,$", MR_Next},
626 {"\"aggregate_name\": \"stddev\",$", MR_Next},
627 {"\"aggregate_unit\": \"time\",$", MR_Next},
628 {"\"iterations\": 4,$", MR_Next}});
629 ADD_CASES(TC_CSVOut, {{"^\"BM_Repeat/repeats:4\",%csv_report$"},
630 {"^\"BM_Repeat/repeats:4\",%csv_report$"},
631 {"^\"BM_Repeat/repeats:4\",%csv_report$"},
632 {"^\"BM_Repeat/repeats:4\",%csv_report$"},
633 {"^\"BM_Repeat/repeats:4_mean\",%csv_report$"},
634 {"^\"BM_Repeat/repeats:4_median\",%csv_report$"},
635 {"^\"BM_Repeat/repeats:4_stddev\",%csv_report$"}});
636
637 // Test that a non-repeated test still prints non-aggregate results even when
638 // only-aggregate reports have been requested
BM_RepeatOnce(benchmark::State & state)639 void BM_RepeatOnce(benchmark::State& state) {
640 for (auto _ : state) {
641 }
642 }
643 BENCHMARK(BM_RepeatOnce)->Repetitions(1)->ReportAggregatesOnly();
644 ADD_CASES(TC_ConsoleOut, {{"^BM_RepeatOnce/repeats:1 %console_report$"}});
645 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_RepeatOnce/repeats:1\",$"},
646 {"\"family_index\": 18,$", MR_Next},
647 {"\"per_family_instance_index\": 0,$", MR_Next},
648 {"\"run_name\": \"BM_RepeatOnce/repeats:1\",$", MR_Next},
649 {"\"run_type\": \"iteration\",$", MR_Next},
650 {"\"repetitions\": 1,$", MR_Next},
651 {"\"repetition_index\": 0,$", MR_Next},
652 {"\"threads\": 1,$", MR_Next}});
653 ADD_CASES(TC_CSVOut, {{"^\"BM_RepeatOnce/repeats:1\",%csv_report$"}});
654
655 // Test that non-aggregate data is not reported
BM_SummaryRepeat(benchmark::State & state)656 void BM_SummaryRepeat(benchmark::State& state) {
657 for (auto _ : state) {
658 }
659 }
660 BENCHMARK(BM_SummaryRepeat)->Repetitions(3)->ReportAggregatesOnly();
661 ADD_CASES(
662 TC_ConsoleOut,
663 {{".*BM_SummaryRepeat/repeats:3 ", MR_Not},
664 {"^BM_SummaryRepeat/repeats:3_mean %console_time_only_report [ ]*3$"},
665 {"^BM_SummaryRepeat/repeats:3_median %console_time_only_report [ ]*3$"},
666 {"^BM_SummaryRepeat/repeats:3_stddev %console_time_only_report [ ]*3$"}});
667 ADD_CASES(TC_JSONOut,
668 {{".*BM_SummaryRepeat/repeats:3 ", MR_Not},
669 {"\"name\": \"BM_SummaryRepeat/repeats:3_mean\",$"},
670 {"\"family_index\": 19,$", MR_Next},
671 {"\"per_family_instance_index\": 0,$", MR_Next},
672 {"\"run_name\": \"BM_SummaryRepeat/repeats:3\",$", MR_Next},
673 {"\"run_type\": \"aggregate\",$", MR_Next},
674 {"\"repetitions\": 3,$", MR_Next},
675 {"\"threads\": 1,$", MR_Next},
676 {"\"aggregate_name\": \"mean\",$", MR_Next},
677 {"\"aggregate_unit\": \"time\",$", MR_Next},
678 {"\"iterations\": 3,$", MR_Next},
679 {"\"name\": \"BM_SummaryRepeat/repeats:3_median\",$"},
680 {"\"family_index\": 19,$", MR_Next},
681 {"\"per_family_instance_index\": 0,$", MR_Next},
682 {"\"run_name\": \"BM_SummaryRepeat/repeats:3\",$", MR_Next},
683 {"\"run_type\": \"aggregate\",$", MR_Next},
684 {"\"repetitions\": 3,$", MR_Next},
685 {"\"threads\": 1,$", MR_Next},
686 {"\"aggregate_name\": \"median\",$", MR_Next},
687 {"\"aggregate_unit\": \"time\",$", MR_Next},
688 {"\"iterations\": 3,$", MR_Next},
689 {"\"name\": \"BM_SummaryRepeat/repeats:3_stddev\",$"},
690 {"\"family_index\": 19,$", MR_Next},
691 {"\"per_family_instance_index\": 0,$", MR_Next},
692 {"\"run_name\": \"BM_SummaryRepeat/repeats:3\",$", MR_Next},
693 {"\"run_type\": \"aggregate\",$", MR_Next},
694 {"\"repetitions\": 3,$", MR_Next},
695 {"\"threads\": 1,$", MR_Next},
696 {"\"aggregate_name\": \"stddev\",$", MR_Next},
697 {"\"aggregate_unit\": \"time\",$", MR_Next},
698 {"\"iterations\": 3,$", MR_Next}});
699 ADD_CASES(TC_CSVOut, {{".*BM_SummaryRepeat/repeats:3 ", MR_Not},
700 {"^\"BM_SummaryRepeat/repeats:3_mean\",%csv_report$"},
701 {"^\"BM_SummaryRepeat/repeats:3_median\",%csv_report$"},
702 {"^\"BM_SummaryRepeat/repeats:3_stddev\",%csv_report$"}});
703
704 // Test that non-aggregate data is not displayed.
705 // NOTE: this test is kinda bad. we are only testing the display output.
706 // But we don't check that the file output still contains everything...
BM_SummaryDisplay(benchmark::State & state)707 void BM_SummaryDisplay(benchmark::State& state) {
708 for (auto _ : state) {
709 }
710 }
711 BENCHMARK(BM_SummaryDisplay)->Repetitions(2)->DisplayAggregatesOnly();
712 ADD_CASES(
713 TC_ConsoleOut,
714 {{".*BM_SummaryDisplay/repeats:2 ", MR_Not},
715 {"^BM_SummaryDisplay/repeats:2_mean %console_time_only_report [ ]*2$"},
716 {"^BM_SummaryDisplay/repeats:2_median %console_time_only_report [ ]*2$"},
717 {"^BM_SummaryDisplay/repeats:2_stddev %console_time_only_report [ ]*2$"}});
718 ADD_CASES(TC_JSONOut,
719 {{".*BM_SummaryDisplay/repeats:2 ", MR_Not},
720 {"\"name\": \"BM_SummaryDisplay/repeats:2_mean\",$"},
721 {"\"family_index\": 20,$", MR_Next},
722 {"\"per_family_instance_index\": 0,$", MR_Next},
723 {"\"run_name\": \"BM_SummaryDisplay/repeats:2\",$", MR_Next},
724 {"\"run_type\": \"aggregate\",$", MR_Next},
725 {"\"repetitions\": 2,$", MR_Next},
726 {"\"threads\": 1,$", MR_Next},
727 {"\"aggregate_name\": \"mean\",$", MR_Next},
728 {"\"aggregate_unit\": \"time\",$", MR_Next},
729 {"\"iterations\": 2,$", MR_Next},
730 {"\"name\": \"BM_SummaryDisplay/repeats:2_median\",$"},
731 {"\"family_index\": 20,$", MR_Next},
732 {"\"per_family_instance_index\": 0,$", MR_Next},
733 {"\"run_name\": \"BM_SummaryDisplay/repeats:2\",$", MR_Next},
734 {"\"run_type\": \"aggregate\",$", MR_Next},
735 {"\"repetitions\": 2,$", MR_Next},
736 {"\"threads\": 1,$", MR_Next},
737 {"\"aggregate_name\": \"median\",$", MR_Next},
738 {"\"aggregate_unit\": \"time\",$", MR_Next},
739 {"\"iterations\": 2,$", MR_Next},
740 {"\"name\": \"BM_SummaryDisplay/repeats:2_stddev\",$"},
741 {"\"family_index\": 20,$", MR_Next},
742 {"\"per_family_instance_index\": 0,$", MR_Next},
743 {"\"run_name\": \"BM_SummaryDisplay/repeats:2\",$", MR_Next},
744 {"\"run_type\": \"aggregate\",$", MR_Next},
745 {"\"repetitions\": 2,$", MR_Next},
746 {"\"threads\": 1,$", MR_Next},
747 {"\"aggregate_name\": \"stddev\",$", MR_Next},
748 {"\"aggregate_unit\": \"time\",$", MR_Next},
749 {"\"iterations\": 2,$", MR_Next}});
750 ADD_CASES(TC_CSVOut,
751 {{".*BM_SummaryDisplay/repeats:2 ", MR_Not},
752 {"^\"BM_SummaryDisplay/repeats:2_mean\",%csv_report$"},
753 {"^\"BM_SummaryDisplay/repeats:2_median\",%csv_report$"},
754 {"^\"BM_SummaryDisplay/repeats:2_stddev\",%csv_report$"}});
755
756 // Test repeats with custom time unit.
BM_RepeatTimeUnit(benchmark::State & state)757 void BM_RepeatTimeUnit(benchmark::State& state) {
758 for (auto _ : state) {
759 }
760 }
761 BENCHMARK(BM_RepeatTimeUnit)
762 ->Repetitions(3)
763 ->ReportAggregatesOnly()
764 ->Unit(benchmark::kMicrosecond);
765 ADD_CASES(
766 TC_ConsoleOut,
767 {{".*BM_RepeatTimeUnit/repeats:3 ", MR_Not},
768 {"^BM_RepeatTimeUnit/repeats:3_mean %console_us_time_only_report [ ]*3$"},
769 {"^BM_RepeatTimeUnit/repeats:3_median %console_us_time_only_report [ "
770 "]*3$"},
771 {"^BM_RepeatTimeUnit/repeats:3_stddev %console_us_time_only_report [ "
772 "]*3$"}});
773 ADD_CASES(TC_JSONOut,
774 {{".*BM_RepeatTimeUnit/repeats:3 ", MR_Not},
775 {"\"name\": \"BM_RepeatTimeUnit/repeats:3_mean\",$"},
776 {"\"family_index\": 21,$", MR_Next},
777 {"\"per_family_instance_index\": 0,$", MR_Next},
778 {"\"run_name\": \"BM_RepeatTimeUnit/repeats:3\",$", MR_Next},
779 {"\"run_type\": \"aggregate\",$", MR_Next},
780 {"\"repetitions\": 3,$", MR_Next},
781 {"\"threads\": 1,$", MR_Next},
782 {"\"aggregate_name\": \"mean\",$", MR_Next},
783 {"\"aggregate_unit\": \"time\",$", MR_Next},
784 {"\"iterations\": 3,$", MR_Next},
785 {"\"time_unit\": \"us\",?$"},
786 {"\"name\": \"BM_RepeatTimeUnit/repeats:3_median\",$"},
787 {"\"family_index\": 21,$", MR_Next},
788 {"\"per_family_instance_index\": 0,$", MR_Next},
789 {"\"run_name\": \"BM_RepeatTimeUnit/repeats:3\",$", MR_Next},
790 {"\"run_type\": \"aggregate\",$", MR_Next},
791 {"\"repetitions\": 3,$", MR_Next},
792 {"\"threads\": 1,$", MR_Next},
793 {"\"aggregate_name\": \"median\",$", MR_Next},
794 {"\"aggregate_unit\": \"time\",$", MR_Next},
795 {"\"iterations\": 3,$", MR_Next},
796 {"\"time_unit\": \"us\",?$"},
797 {"\"name\": \"BM_RepeatTimeUnit/repeats:3_stddev\",$"},
798 {"\"family_index\": 21,$", MR_Next},
799 {"\"per_family_instance_index\": 0,$", MR_Next},
800 {"\"run_name\": \"BM_RepeatTimeUnit/repeats:3\",$", MR_Next},
801 {"\"run_type\": \"aggregate\",$", MR_Next},
802 {"\"repetitions\": 3,$", MR_Next},
803 {"\"threads\": 1,$", MR_Next},
804 {"\"aggregate_name\": \"stddev\",$", MR_Next},
805 {"\"aggregate_unit\": \"time\",$", MR_Next},
806 {"\"iterations\": 3,$", MR_Next},
807 {"\"time_unit\": \"us\",?$"}});
808 ADD_CASES(TC_CSVOut,
809 {{".*BM_RepeatTimeUnit/repeats:3 ", MR_Not},
810 {"^\"BM_RepeatTimeUnit/repeats:3_mean\",%csv_us_report$"},
811 {"^\"BM_RepeatTimeUnit/repeats:3_median\",%csv_us_report$"},
812 {"^\"BM_RepeatTimeUnit/repeats:3_stddev\",%csv_us_report$"}});
813
814 // ========================================================================= //
815 // -------------------- Testing user-provided statistics ------------------- //
816 // ========================================================================= //
817
__anon387e2a0e0102(const std::vector<double>& v) 818 const auto UserStatistics = [](const std::vector<double>& v) {
819 return v.back();
820 };
BM_UserStats(benchmark::State & state)821 void BM_UserStats(benchmark::State& state) {
822 for (auto _ : state) {
823 state.SetIterationTime(150 / 10e8);
824 }
825 }
826 // clang-format off
827 BENCHMARK(BM_UserStats)
828 ->Repetitions(3)
829 ->Iterations(5)
830 ->UseManualTime()
831 ->ComputeStatistics("", UserStatistics);
832 // clang-format on
833
834 // check that user-provided stats is calculated, and is after the default-ones
835 // empty string as name is intentional, it would sort before anything else
836 ADD_CASES(TC_ConsoleOut, {{"^BM_UserStats/iterations:5/repeats:3/manual_time [ "
837 "]* 150 ns %time [ ]*5$"},
838 {"^BM_UserStats/iterations:5/repeats:3/manual_time [ "
839 "]* 150 ns %time [ ]*5$"},
840 {"^BM_UserStats/iterations:5/repeats:3/manual_time [ "
841 "]* 150 ns %time [ ]*5$"},
842 {"^BM_UserStats/iterations:5/repeats:3/"
843 "manual_time_mean [ ]* 150 ns %time [ ]*3$"},
844 {"^BM_UserStats/iterations:5/repeats:3/"
845 "manual_time_median [ ]* 150 ns %time [ ]*3$"},
846 {"^BM_UserStats/iterations:5/repeats:3/"
847 "manual_time_stddev [ ]* 0.000 ns %time [ ]*3$"},
848 {"^BM_UserStats/iterations:5/repeats:3/manual_time_ "
849 "[ ]* 150 ns %time [ ]*3$"}});
850 ADD_CASES(
851 TC_JSONOut,
852 {{"\"name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$"},
853 {"\"family_index\": 22,$", MR_Next},
854 {"\"per_family_instance_index\": 0,$", MR_Next},
855 {"\"run_name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$",
856 MR_Next},
857 {"\"run_type\": \"iteration\",$", MR_Next},
858 {"\"repetitions\": 3,$", MR_Next},
859 {"\"repetition_index\": 0,$", MR_Next},
860 {"\"threads\": 1,$", MR_Next},
861 {"\"iterations\": 5,$", MR_Next},
862 {"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next},
863 {"\"name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$"},
864 {"\"family_index\": 22,$", MR_Next},
865 {"\"per_family_instance_index\": 0,$", MR_Next},
866 {"\"run_name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$",
867 MR_Next},
868 {"\"run_type\": \"iteration\",$", MR_Next},
869 {"\"repetitions\": 3,$", MR_Next},
870 {"\"repetition_index\": 1,$", MR_Next},
871 {"\"threads\": 1,$", MR_Next},
872 {"\"iterations\": 5,$", MR_Next},
873 {"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next},
874 {"\"name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$"},
875 {"\"family_index\": 22,$", MR_Next},
876 {"\"per_family_instance_index\": 0,$", MR_Next},
877 {"\"run_name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$",
878 MR_Next},
879 {"\"run_type\": \"iteration\",$", MR_Next},
880 {"\"repetitions\": 3,$", MR_Next},
881 {"\"repetition_index\": 2,$", MR_Next},
882 {"\"threads\": 1,$", MR_Next},
883 {"\"iterations\": 5,$", MR_Next},
884 {"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next},
885 {"\"name\": \"BM_UserStats/iterations:5/repeats:3/manual_time_mean\",$"},
886 {"\"family_index\": 22,$", MR_Next},
887 {"\"per_family_instance_index\": 0,$", MR_Next},
888 {"\"run_name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$",
889 MR_Next},
890 {"\"run_type\": \"aggregate\",$", MR_Next},
891 {"\"repetitions\": 3,$", MR_Next},
892 {"\"threads\": 1,$", MR_Next},
893 {"\"aggregate_name\": \"mean\",$", MR_Next},
894 {"\"aggregate_unit\": \"time\",$", MR_Next},
895 {"\"iterations\": 3,$", MR_Next},
896 {"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next},
897 {"\"name\": \"BM_UserStats/iterations:5/repeats:3/manual_time_median\",$"},
898 {"\"family_index\": 22,$", MR_Next},
899 {"\"per_family_instance_index\": 0,$", MR_Next},
900 {"\"run_name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$",
901 MR_Next},
902 {"\"run_type\": \"aggregate\",$", MR_Next},
903 {"\"repetitions\": 3,$", MR_Next},
904 {"\"threads\": 1,$", MR_Next},
905 {"\"aggregate_name\": \"median\",$", MR_Next},
906 {"\"aggregate_unit\": \"time\",$", MR_Next},
907 {"\"iterations\": 3,$", MR_Next},
908 {"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next},
909 {"\"name\": \"BM_UserStats/iterations:5/repeats:3/manual_time_stddev\",$"},
910 {"\"family_index\": 22,$", MR_Next},
911 {"\"per_family_instance_index\": 0,$", MR_Next},
912 {"\"run_name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$",
913 MR_Next},
914 {"\"run_type\": \"aggregate\",$", MR_Next},
915 {"\"repetitions\": 3,$", MR_Next},
916 {"\"threads\": 1,$", MR_Next},
917 {"\"aggregate_name\": \"stddev\",$", MR_Next},
918 {"\"aggregate_unit\": \"time\",$", MR_Next},
919 {"\"iterations\": 3,$", MR_Next},
920 {"\"real_time\": %float,$", MR_Next},
921 {"\"name\": \"BM_UserStats/iterations:5/repeats:3/manual_time_\",$"},
922 {"\"family_index\": 22,$", MR_Next},
923 {"\"per_family_instance_index\": 0,$", MR_Next},
924 {"\"run_name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$",
925 MR_Next},
926 {"\"run_type\": \"aggregate\",$", MR_Next},
927 {"\"repetitions\": 3,$", MR_Next},
928 {"\"threads\": 1,$", MR_Next},
929 {"\"aggregate_name\": \"\",$", MR_Next},
930 {"\"aggregate_unit\": \"time\",$", MR_Next},
931 {"\"iterations\": 3,$", MR_Next},
932 {"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next}});
933 ADD_CASES(
934 TC_CSVOut,
935 {{"^\"BM_UserStats/iterations:5/repeats:3/manual_time\",%csv_report$"},
936 {"^\"BM_UserStats/iterations:5/repeats:3/manual_time\",%csv_report$"},
937 {"^\"BM_UserStats/iterations:5/repeats:3/manual_time\",%csv_report$"},
938 {"^\"BM_UserStats/iterations:5/repeats:3/manual_time_mean\",%csv_report$"},
939 {"^\"BM_UserStats/iterations:5/repeats:3/"
940 "manual_time_median\",%csv_report$"},
941 {"^\"BM_UserStats/iterations:5/repeats:3/"
942 "manual_time_stddev\",%csv_report$"},
943 {"^\"BM_UserStats/iterations:5/repeats:3/manual_time_\",%csv_report$"}});
944
945 // ========================================================================= //
946 // ------------- Testing relative standard deviation statistics ------------ //
947 // ========================================================================= //
948
__anon387e2a0e0202(const std::vector<double>&) 949 const auto UserPercentStatistics = [](const std::vector<double>&) {
950 return 1. / 100.;
951 };
BM_UserPercentStats(benchmark::State & state)952 void BM_UserPercentStats(benchmark::State& state) {
953 for (auto _ : state) {
954 state.SetIterationTime(150 / 10e8);
955 }
956 }
957 // clang-format off
958 BENCHMARK(BM_UserPercentStats)
959 ->Repetitions(3)
960 ->Iterations(5)
961 ->UseManualTime()
962 ->Unit(benchmark::TimeUnit::kNanosecond)
963 ->ComputeStatistics("", UserPercentStatistics, benchmark::StatisticUnit::kPercentage);
964 // clang-format on
965
966 // check that UserPercent-provided stats is calculated, and is after the
967 // default-ones empty string as name is intentional, it would sort before
968 // anything else
969 ADD_CASES(TC_ConsoleOut,
970 {{"^BM_UserPercentStats/iterations:5/repeats:3/manual_time [ "
971 "]* 150 ns %time [ ]*5$"},
972 {"^BM_UserPercentStats/iterations:5/repeats:3/manual_time [ "
973 "]* 150 ns %time [ ]*5$"},
974 {"^BM_UserPercentStats/iterations:5/repeats:3/manual_time [ "
975 "]* 150 ns %time [ ]*5$"},
976 {"^BM_UserPercentStats/iterations:5/repeats:3/"
977 "manual_time_mean [ ]* 150 ns %time [ ]*3$"},
978 {"^BM_UserPercentStats/iterations:5/repeats:3/"
979 "manual_time_median [ ]* 150 ns %time [ ]*3$"},
980 {"^BM_UserPercentStats/iterations:5/repeats:3/"
981 "manual_time_stddev [ ]* 0.000 ns %time [ ]*3$"},
982 {"^BM_UserPercentStats/iterations:5/repeats:3/manual_time_ "
983 "[ ]* 1.00 % [ ]* 1.00 %[ ]*3$"}});
984 ADD_CASES(
985 TC_JSONOut,
986 {{"\"name\": \"BM_UserPercentStats/iterations:5/repeats:3/manual_time\",$"},
987 {"\"family_index\": 23,$", MR_Next},
988 {"\"per_family_instance_index\": 0,$", MR_Next},
989 {"\"run_name\": "
990 "\"BM_UserPercentStats/iterations:5/repeats:3/manual_time\",$",
991 MR_Next},
992 {"\"run_type\": \"iteration\",$", MR_Next},
993 {"\"repetitions\": 3,$", MR_Next},
994 {"\"repetition_index\": 0,$", MR_Next},
995 {"\"threads\": 1,$", MR_Next},
996 {"\"iterations\": 5,$", MR_Next},
997 {"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next},
998 {"\"name\": \"BM_UserPercentStats/iterations:5/repeats:3/manual_time\",$"},
999 {"\"family_index\": 23,$", MR_Next},
1000 {"\"per_family_instance_index\": 0,$", MR_Next},
1001 {"\"run_name\": "
1002 "\"BM_UserPercentStats/iterations:5/repeats:3/manual_time\",$",
1003 MR_Next},
1004 {"\"run_type\": \"iteration\",$", MR_Next},
1005 {"\"repetitions\": 3,$", MR_Next},
1006 {"\"repetition_index\": 1,$", MR_Next},
1007 {"\"threads\": 1,$", MR_Next},
1008 {"\"iterations\": 5,$", MR_Next},
1009 {"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next},
1010 {"\"name\": \"BM_UserPercentStats/iterations:5/repeats:3/manual_time\",$"},
1011 {"\"family_index\": 23,$", MR_Next},
1012 {"\"per_family_instance_index\": 0,$", MR_Next},
1013 {"\"run_name\": "
1014 "\"BM_UserPercentStats/iterations:5/repeats:3/manual_time\",$",
1015 MR_Next},
1016 {"\"run_type\": \"iteration\",$", MR_Next},
1017 {"\"repetitions\": 3,$", MR_Next},
1018 {"\"repetition_index\": 2,$", MR_Next},
1019 {"\"threads\": 1,$", MR_Next},
1020 {"\"iterations\": 5,$", MR_Next},
1021 {"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next},
1022 {"\"name\": "
1023 "\"BM_UserPercentStats/iterations:5/repeats:3/manual_time_mean\",$"},
1024 {"\"family_index\": 23,$", MR_Next},
1025 {"\"per_family_instance_index\": 0,$", MR_Next},
1026 {"\"run_name\": "
1027 "\"BM_UserPercentStats/iterations:5/repeats:3/manual_time\",$",
1028 MR_Next},
1029 {"\"run_type\": \"aggregate\",$", MR_Next},
1030 {"\"repetitions\": 3,$", MR_Next},
1031 {"\"threads\": 1,$", MR_Next},
1032 {"\"aggregate_name\": \"mean\",$", MR_Next},
1033 {"\"aggregate_unit\": \"time\",$", MR_Next},
1034 {"\"iterations\": 3,$", MR_Next},
1035 {"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next},
1036 {"\"name\": "
1037 "\"BM_UserPercentStats/iterations:5/repeats:3/manual_time_median\",$"},
1038 {"\"family_index\": 23,$", MR_Next},
1039 {"\"per_family_instance_index\": 0,$", MR_Next},
1040 {"\"run_name\": "
1041 "\"BM_UserPercentStats/iterations:5/repeats:3/manual_time\",$",
1042 MR_Next},
1043 {"\"run_type\": \"aggregate\",$", MR_Next},
1044 {"\"repetitions\": 3,$", MR_Next},
1045 {"\"threads\": 1,$", MR_Next},
1046 {"\"aggregate_name\": \"median\",$", MR_Next},
1047 {"\"aggregate_unit\": \"time\",$", MR_Next},
1048 {"\"iterations\": 3,$", MR_Next},
1049 {"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next},
1050 {"\"name\": "
1051 "\"BM_UserPercentStats/iterations:5/repeats:3/manual_time_stddev\",$"},
1052 {"\"family_index\": 23,$", MR_Next},
1053 {"\"per_family_instance_index\": 0,$", MR_Next},
1054 {"\"run_name\": "
1055 "\"BM_UserPercentStats/iterations:5/repeats:3/manual_time\",$",
1056 MR_Next},
1057 {"\"run_type\": \"aggregate\",$", MR_Next},
1058 {"\"repetitions\": 3,$", MR_Next},
1059 {"\"threads\": 1,$", MR_Next},
1060 {"\"aggregate_name\": \"stddev\",$", MR_Next},
1061 {"\"aggregate_unit\": \"time\",$", MR_Next},
1062 {"\"iterations\": 3,$", MR_Next},
1063 {"\"real_time\": %float,$", MR_Next},
1064 {"\"name\": "
1065 "\"BM_UserPercentStats/iterations:5/repeats:3/manual_time_\",$"},
1066 {"\"family_index\": 23,$", MR_Next},
1067 {"\"per_family_instance_index\": 0,$", MR_Next},
1068 {"\"run_name\": "
1069 "\"BM_UserPercentStats/iterations:5/repeats:3/manual_time\",$",
1070 MR_Next},
1071 {"\"run_type\": \"aggregate\",$", MR_Next},
1072 {"\"repetitions\": 3,$", MR_Next},
1073 {"\"threads\": 1,$", MR_Next},
1074 {"\"aggregate_name\": \"\",$", MR_Next},
1075 {"\"aggregate_unit\": \"percentage\",$", MR_Next},
1076 {"\"iterations\": 3,$", MR_Next},
1077 {"\"real_time\": 1\\.(0)*e-(0)*2,$", MR_Next}});
1078 ADD_CASES(TC_CSVOut, {{"^\"BM_UserPercentStats/iterations:5/repeats:3/"
1079 "manual_time\",%csv_report$"},
1080 {"^\"BM_UserPercentStats/iterations:5/repeats:3/"
1081 "manual_time\",%csv_report$"},
1082 {"^\"BM_UserPercentStats/iterations:5/repeats:3/"
1083 "manual_time\",%csv_report$"},
1084 {"^\"BM_UserPercentStats/iterations:5/repeats:3/"
1085 "manual_time_mean\",%csv_report$"},
1086 {"^\"BM_UserPercentStats/iterations:5/repeats:3/"
1087 "manual_time_median\",%csv_report$"},
1088 {"^\"BM_UserPercentStats/iterations:5/repeats:3/"
1089 "manual_time_stddev\",%csv_report$"},
1090 {"^\"BM_UserPercentStats/iterations:5/repeats:3/"
1091 "manual_time_\",%csv_report$"}});
1092
1093 // ========================================================================= //
1094 // ------------------------- Testing StrEscape JSON ------------------------ //
1095 // ========================================================================= //
1096 #if 0 // enable when csv testing code correctly handles multi-line fields
1097 void BM_JSON_Format(benchmark::State& state) {
1098 state.SkipWithError("val\b\f\n\r\t\\\"with\"es,capes");
1099 for (auto _ : state) {
1100 }
1101 }
1102 BENCHMARK(BM_JSON_Format);
1103 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_JSON_Format\",$"},
1104 {"\"family_index\": 23,$", MR_Next},
1105 {"\"per_family_instance_index\": 0,$", MR_Next},
1106 {"\"run_name\": \"BM_JSON_Format\",$", MR_Next},
1107 {"\"run_type\": \"iteration\",$", MR_Next},
1108 {"\"repetitions\": 1,$", MR_Next},
1109 {"\"repetition_index\": 0,$", MR_Next},
1110 {"\"threads\": 1,$", MR_Next},
1111 {"\"error_occurred\": true,$", MR_Next},
1112 {R"("error_message": "val\\b\\f\\n\\r\\t\\\\\\"with\\"es,capes",$)", MR_Next}});
1113 #endif
1114 // ========================================================================= //
1115 // -------------------------- Testing CsvEscape ---------------------------- //
1116 // ========================================================================= //
1117
BM_CSV_Format(benchmark::State & state)1118 void BM_CSV_Format(benchmark::State& state) {
1119 state.SkipWithError("\"freedom\"");
1120 for (auto _ : state) {
1121 }
1122 }
1123 BENCHMARK(BM_CSV_Format);
1124 ADD_CASES(TC_CSVOut, {{"^\"BM_CSV_Format\",,,,,,,,true,\"\"\"freedom\"\"\"$"}});
1125
1126 // ========================================================================= //
1127 // --------------------------- TEST CASES END ------------------------------ //
1128 // ========================================================================= //
1129
main(int argc,char * argv[])1130 int main(int argc, char* argv[]) { RunOutputTests(argc, argv); }
1131