1
2 #undef NDEBUG
3 #include <numeric>
4 #include <utility>
5
6 #include "benchmark/benchmark.h"
7 #include "output_test.h"
8
9 // ========================================================================= //
10 // ---------------------- Testing Prologue Output -------------------------- //
11 // ========================================================================= //
12
13 ADD_CASES(TC_ConsoleOut, {{"^[-]+$", MR_Next},
14 {"^Benchmark %s Time %s CPU %s Iterations$", MR_Next},
15 {"^[-]+$", MR_Next}});
AddContextCases()16 static int AddContextCases() {
17 AddCases(TC_ConsoleErr,
18 {
19 {"^%int-%int-%intT%int:%int:%int[-+]%int:%int$", MR_Default},
20 {"Running .*/reporter_output_test(\\.exe)?$", MR_Next},
21 {"Run on \\(%int X %float MHz CPU s?\\)", MR_Next},
22 });
23 AddCases(TC_JSONOut,
24 {{"^\\{", MR_Default},
25 {"\"context\":", MR_Next},
26 {"\"date\": \"", MR_Next},
27 {"\"host_name\":", MR_Next},
28 {"\"executable\": \".*(/|\\\\)reporter_output_test(\\.exe)?\",",
29 MR_Next},
30 {"\"num_cpus\": %int,$", MR_Next},
31 {"\"mhz_per_cpu\": %float,$", MR_Next},
32 {"\"caches\": \\[$", MR_Default}});
33 auto const& Info = benchmark::CPUInfo::Get();
34 auto const& Caches = Info.caches;
35 if (!Caches.empty()) {
36 AddCases(TC_ConsoleErr, {{"CPU Caches:$", MR_Next}});
37 }
38 for (size_t I = 0; I < Caches.size(); ++I) {
39 std::string num_caches_str =
40 Caches[I].num_sharing != 0 ? " \\(x%int\\)$" : "$";
41 AddCases(TC_ConsoleErr,
42 {{"L%int (Data|Instruction|Unified) %int KiB" + num_caches_str,
43 MR_Next}});
44 AddCases(TC_JSONOut, {{"\\{$", MR_Next},
45 {"\"type\": \"", MR_Next},
46 {"\"level\": %int,$", MR_Next},
47 {"\"size\": %int,$", MR_Next},
48 {"\"num_sharing\": %int$", MR_Next},
49 {"}[,]{0,1}$", MR_Next}});
50 }
51 AddCases(TC_JSONOut, {{"],$"}});
52 auto const& LoadAvg = Info.load_avg;
53 if (!LoadAvg.empty()) {
54 AddCases(TC_ConsoleErr,
55 {{"Load Average: (%float, ){0,2}%float$", MR_Next}});
56 }
57 AddCases(TC_JSONOut, {{"\"load_avg\": \\[(%float,?){0,3}],$", MR_Next}});
58 return 0;
59 }
60 int dummy_register = AddContextCases();
61 ADD_CASES(TC_CSVOut, {{"%csv_header"}});
62
63 // ========================================================================= //
64 // ------------------------ Testing Basic Output --------------------------- //
65 // ========================================================================= //
66
BM_basic(benchmark::State & state)67 void BM_basic(benchmark::State& state) {
68 for (auto _ : state) {
69 }
70 }
71 BENCHMARK(BM_basic);
72
73 ADD_CASES(TC_ConsoleOut, {{"^BM_basic %console_report$"}});
74 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_basic\",$"},
75 {"\"family_index\": 0,$", MR_Next},
76 {"\"per_family_instance_index\": 0,$", MR_Next},
77 {"\"run_name\": \"BM_basic\",$", MR_Next},
78 {"\"run_type\": \"iteration\",$", MR_Next},
79 {"\"repetitions\": 1,$", MR_Next},
80 {"\"repetition_index\": 0,$", MR_Next},
81 {"\"threads\": 1,$", MR_Next},
82 {"\"iterations\": %int,$", MR_Next},
83 {"\"real_time\": %float,$", MR_Next},
84 {"\"cpu_time\": %float,$", MR_Next},
85 {"\"time_unit\": \"ns\"$", MR_Next},
86 {"}", MR_Next}});
87 ADD_CASES(TC_CSVOut, {{"^\"BM_basic\",%csv_report$"}});
88
89 // ========================================================================= //
90 // ------------------------ Testing Bytes per Second Output ---------------- //
91 // ========================================================================= //
92
BM_bytes_per_second(benchmark::State & state)93 void BM_bytes_per_second(benchmark::State& state) {
94 for (auto _ : state) {
95 // This test requires a non-zero CPU time to avoid divide-by-zero
96 benchmark::DoNotOptimize(state.iterations());
97 }
98 state.SetBytesProcessed(1);
99 }
100 BENCHMARK(BM_bytes_per_second);
101
102 ADD_CASES(TC_ConsoleOut, {{"^BM_bytes_per_second %console_report "
103 "bytes_per_second=%float[kM]{0,1}/s$"}});
104 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_bytes_per_second\",$"},
105 {"\"family_index\": 1,$", MR_Next},
106 {"\"per_family_instance_index\": 0,$", MR_Next},
107 {"\"run_name\": \"BM_bytes_per_second\",$", MR_Next},
108 {"\"run_type\": \"iteration\",$", MR_Next},
109 {"\"repetitions\": 1,$", MR_Next},
110 {"\"repetition_index\": 0,$", MR_Next},
111 {"\"threads\": 1,$", MR_Next},
112 {"\"iterations\": %int,$", MR_Next},
113 {"\"real_time\": %float,$", MR_Next},
114 {"\"cpu_time\": %float,$", MR_Next},
115 {"\"time_unit\": \"ns\",$", MR_Next},
116 {"\"bytes_per_second\": %float$", MR_Next},
117 {"}", MR_Next}});
118 ADD_CASES(TC_CSVOut, {{"^\"BM_bytes_per_second\",%csv_bytes_report$"}});
119
120 // ========================================================================= //
121 // ------------------------ Testing Items per Second Output ---------------- //
122 // ========================================================================= //
123
BM_items_per_second(benchmark::State & state)124 void BM_items_per_second(benchmark::State& state) {
125 for (auto _ : state) {
126 // This test requires a non-zero CPU time to avoid divide-by-zero
127 benchmark::DoNotOptimize(state.iterations());
128 }
129 state.SetItemsProcessed(1);
130 }
131 BENCHMARK(BM_items_per_second);
132
133 ADD_CASES(TC_ConsoleOut, {{"^BM_items_per_second %console_report "
134 "items_per_second=%float[kM]{0,1}/s$"}});
135 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_items_per_second\",$"},
136 {"\"family_index\": 2,$", MR_Next},
137 {"\"per_family_instance_index\": 0,$", MR_Next},
138 {"\"run_name\": \"BM_items_per_second\",$", MR_Next},
139 {"\"run_type\": \"iteration\",$", MR_Next},
140 {"\"repetitions\": 1,$", MR_Next},
141 {"\"repetition_index\": 0,$", MR_Next},
142 {"\"threads\": 1,$", MR_Next},
143 {"\"iterations\": %int,$", MR_Next},
144 {"\"real_time\": %float,$", MR_Next},
145 {"\"cpu_time\": %float,$", MR_Next},
146 {"\"time_unit\": \"ns\",$", MR_Next},
147 {"\"items_per_second\": %float$", MR_Next},
148 {"}", MR_Next}});
149 ADD_CASES(TC_CSVOut, {{"^\"BM_items_per_second\",%csv_items_report$"}});
150
151 // ========================================================================= //
152 // ------------------------ Testing Label Output --------------------------- //
153 // ========================================================================= //
154
BM_label(benchmark::State & state)155 void BM_label(benchmark::State& state) {
156 for (auto _ : state) {
157 }
158 state.SetLabel("some label");
159 }
160 BENCHMARK(BM_label);
161
162 ADD_CASES(TC_ConsoleOut, {{"^BM_label %console_report some label$"}});
163 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_label\",$"},
164 {"\"family_index\": 3,$", MR_Next},
165 {"\"per_family_instance_index\": 0,$", MR_Next},
166 {"\"run_name\": \"BM_label\",$", MR_Next},
167 {"\"run_type\": \"iteration\",$", MR_Next},
168 {"\"repetitions\": 1,$", MR_Next},
169 {"\"repetition_index\": 0,$", MR_Next},
170 {"\"threads\": 1,$", MR_Next},
171 {"\"iterations\": %int,$", MR_Next},
172 {"\"real_time\": %float,$", MR_Next},
173 {"\"cpu_time\": %float,$", MR_Next},
174 {"\"time_unit\": \"ns\",$", MR_Next},
175 {"\"label\": \"some label\"$", MR_Next},
176 {"}", MR_Next}});
177 ADD_CASES(TC_CSVOut, {{"^\"BM_label\",%csv_label_report_begin\"some "
178 "label\"%csv_label_report_end$"}});
179
180 // ========================================================================= //
181 // ------------------------ Testing Time Label Output ---------------------- //
182 // ========================================================================= //
183
BM_time_label_nanosecond(benchmark::State & state)184 void BM_time_label_nanosecond(benchmark::State& state) {
185 for (auto _ : state) {
186 }
187 }
188 BENCHMARK(BM_time_label_nanosecond)->Unit(benchmark::kNanosecond);
189
190 ADD_CASES(TC_ConsoleOut, {{"^BM_time_label_nanosecond %console_report$"}});
191 ADD_CASES(TC_JSONOut,
192 {{"\"name\": \"BM_time_label_nanosecond\",$"},
193 {"\"family_index\": 4,$", MR_Next},
194 {"\"per_family_instance_index\": 0,$", MR_Next},
195 {"\"run_name\": \"BM_time_label_nanosecond\",$", MR_Next},
196 {"\"run_type\": \"iteration\",$", MR_Next},
197 {"\"repetitions\": 1,$", MR_Next},
198 {"\"repetition_index\": 0,$", MR_Next},
199 {"\"threads\": 1,$", MR_Next},
200 {"\"iterations\": %int,$", MR_Next},
201 {"\"real_time\": %float,$", MR_Next},
202 {"\"cpu_time\": %float,$", MR_Next},
203 {"\"time_unit\": \"ns\"$", MR_Next},
204 {"}", MR_Next}});
205 ADD_CASES(TC_CSVOut, {{"^\"BM_time_label_nanosecond\",%csv_report$"}});
206
BM_time_label_microsecond(benchmark::State & state)207 void BM_time_label_microsecond(benchmark::State& state) {
208 for (auto _ : state) {
209 }
210 }
211 BENCHMARK(BM_time_label_microsecond)->Unit(benchmark::kMicrosecond);
212
213 ADD_CASES(TC_ConsoleOut, {{"^BM_time_label_microsecond %console_us_report$"}});
214 ADD_CASES(TC_JSONOut,
215 {{"\"name\": \"BM_time_label_microsecond\",$"},
216 {"\"family_index\": 5,$", MR_Next},
217 {"\"per_family_instance_index\": 0,$", MR_Next},
218 {"\"run_name\": \"BM_time_label_microsecond\",$", MR_Next},
219 {"\"run_type\": \"iteration\",$", MR_Next},
220 {"\"repetitions\": 1,$", MR_Next},
221 {"\"repetition_index\": 0,$", MR_Next},
222 {"\"threads\": 1,$", MR_Next},
223 {"\"iterations\": %int,$", MR_Next},
224 {"\"real_time\": %float,$", MR_Next},
225 {"\"cpu_time\": %float,$", MR_Next},
226 {"\"time_unit\": \"us\"$", MR_Next},
227 {"}", MR_Next}});
228 ADD_CASES(TC_CSVOut, {{"^\"BM_time_label_microsecond\",%csv_us_report$"}});
229
BM_time_label_millisecond(benchmark::State & state)230 void BM_time_label_millisecond(benchmark::State& state) {
231 for (auto _ : state) {
232 }
233 }
234 BENCHMARK(BM_time_label_millisecond)->Unit(benchmark::kMillisecond);
235
236 ADD_CASES(TC_ConsoleOut, {{"^BM_time_label_millisecond %console_ms_report$"}});
237 ADD_CASES(TC_JSONOut,
238 {{"\"name\": \"BM_time_label_millisecond\",$"},
239 {"\"family_index\": 6,$", MR_Next},
240 {"\"per_family_instance_index\": 0,$", MR_Next},
241 {"\"run_name\": \"BM_time_label_millisecond\",$", MR_Next},
242 {"\"run_type\": \"iteration\",$", MR_Next},
243 {"\"repetitions\": 1,$", MR_Next},
244 {"\"repetition_index\": 0,$", MR_Next},
245 {"\"threads\": 1,$", MR_Next},
246 {"\"iterations\": %int,$", MR_Next},
247 {"\"real_time\": %float,$", MR_Next},
248 {"\"cpu_time\": %float,$", MR_Next},
249 {"\"time_unit\": \"ms\"$", MR_Next},
250 {"}", MR_Next}});
251 ADD_CASES(TC_CSVOut, {{"^\"BM_time_label_millisecond\",%csv_ms_report$"}});
252
BM_time_label_second(benchmark::State & state)253 void BM_time_label_second(benchmark::State& state) {
254 for (auto _ : state) {
255 }
256 }
257 BENCHMARK(BM_time_label_second)->Unit(benchmark::kSecond);
258
259 ADD_CASES(TC_ConsoleOut, {{"^BM_time_label_second %console_s_report$"}});
260 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_time_label_second\",$"},
261 {"\"family_index\": 7,$", MR_Next},
262 {"\"per_family_instance_index\": 0,$", MR_Next},
263 {"\"run_name\": \"BM_time_label_second\",$", MR_Next},
264 {"\"run_type\": \"iteration\",$", MR_Next},
265 {"\"repetitions\": 1,$", MR_Next},
266 {"\"repetition_index\": 0,$", MR_Next},
267 {"\"threads\": 1,$", MR_Next},
268 {"\"iterations\": %int,$", MR_Next},
269 {"\"real_time\": %float,$", MR_Next},
270 {"\"cpu_time\": %float,$", MR_Next},
271 {"\"time_unit\": \"s\"$", MR_Next},
272 {"}", MR_Next}});
273 ADD_CASES(TC_CSVOut, {{"^\"BM_time_label_second\",%csv_s_report$"}});
274
275 // ========================================================================= //
276 // ------------------------ Testing Error Output --------------------------- //
277 // ========================================================================= //
278
BM_error(benchmark::State & state)279 void BM_error(benchmark::State& state) {
280 state.SkipWithError("message");
281 for (auto _ : state) {
282 }
283 }
284 BENCHMARK(BM_error);
285 ADD_CASES(TC_ConsoleOut, {{"^BM_error[ ]+ERROR OCCURRED: 'message'$"}});
286 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_error\",$"},
287 {"\"family_index\": 8,$", MR_Next},
288 {"\"per_family_instance_index\": 0,$", MR_Next},
289 {"\"run_name\": \"BM_error\",$", MR_Next},
290 {"\"run_type\": \"iteration\",$", MR_Next},
291 {"\"repetitions\": 1,$", MR_Next},
292 {"\"repetition_index\": 0,$", MR_Next},
293 {"\"threads\": 1,$", MR_Next},
294 {"\"error_occurred\": true,$", MR_Next},
295 {"\"error_message\": \"message\",$", MR_Next}});
296
297 ADD_CASES(TC_CSVOut, {{"^\"BM_error\",,,,,,,,true,\"message\"$"}});
298
299 // ========================================================================= //
300 // ------------------------ Testing No Arg Name Output -----------------------
301 // //
302 // ========================================================================= //
303
BM_no_arg_name(benchmark::State & state)304 void BM_no_arg_name(benchmark::State& state) {
305 for (auto _ : state) {
306 }
307 }
308 BENCHMARK(BM_no_arg_name)->Arg(3);
309 ADD_CASES(TC_ConsoleOut, {{"^BM_no_arg_name/3 %console_report$"}});
310 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_no_arg_name/3\",$"},
311 {"\"family_index\": 9,$", MR_Next},
312 {"\"per_family_instance_index\": 0,$", MR_Next},
313 {"\"run_name\": \"BM_no_arg_name/3\",$", MR_Next},
314 {"\"run_type\": \"iteration\",$", MR_Next},
315 {"\"repetitions\": 1,$", MR_Next},
316 {"\"repetition_index\": 0,$", MR_Next},
317 {"\"threads\": 1,$", MR_Next}});
318 ADD_CASES(TC_CSVOut, {{"^\"BM_no_arg_name/3\",%csv_report$"}});
319
320 // ========================================================================= //
321 // ------------------------ Testing Arg Name Output ----------------------- //
322 // ========================================================================= //
323
BM_arg_name(benchmark::State & state)324 void BM_arg_name(benchmark::State& state) {
325 for (auto _ : state) {
326 }
327 }
328 BENCHMARK(BM_arg_name)->ArgName("first")->Arg(3);
329 ADD_CASES(TC_ConsoleOut, {{"^BM_arg_name/first:3 %console_report$"}});
330 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_arg_name/first:3\",$"},
331 {"\"family_index\": 10,$", MR_Next},
332 {"\"per_family_instance_index\": 0,$", MR_Next},
333 {"\"run_name\": \"BM_arg_name/first:3\",$", MR_Next},
334 {"\"run_type\": \"iteration\",$", MR_Next},
335 {"\"repetitions\": 1,$", MR_Next},
336 {"\"repetition_index\": 0,$", MR_Next},
337 {"\"threads\": 1,$", MR_Next}});
338 ADD_CASES(TC_CSVOut, {{"^\"BM_arg_name/first:3\",%csv_report$"}});
339
340 // ========================================================================= //
341 // ------------------------ Testing Arg Names Output ----------------------- //
342 // ========================================================================= //
343
BM_arg_names(benchmark::State & state)344 void BM_arg_names(benchmark::State& state) {
345 for (auto _ : state) {
346 }
347 }
348 BENCHMARK(BM_arg_names)->Args({2, 5, 4})->ArgNames({"first", "", "third"});
349 ADD_CASES(TC_ConsoleOut,
350 {{"^BM_arg_names/first:2/5/third:4 %console_report$"}});
351 ADD_CASES(TC_JSONOut,
352 {{"\"name\": \"BM_arg_names/first:2/5/third:4\",$"},
353 {"\"family_index\": 11,$", MR_Next},
354 {"\"per_family_instance_index\": 0,$", MR_Next},
355 {"\"run_name\": \"BM_arg_names/first:2/5/third:4\",$", MR_Next},
356 {"\"run_type\": \"iteration\",$", MR_Next},
357 {"\"repetitions\": 1,$", MR_Next},
358 {"\"repetition_index\": 0,$", MR_Next},
359 {"\"threads\": 1,$", MR_Next}});
360 ADD_CASES(TC_CSVOut, {{"^\"BM_arg_names/first:2/5/third:4\",%csv_report$"}});
361
362 // ========================================================================= //
363 // ------------------------ Testing Name Output ---------------------------- //
364 // ========================================================================= //
365
BM_name(benchmark::State & state)366 void BM_name(benchmark::State& state) {
367 for (auto _ : state) {
368 }
369 }
370 BENCHMARK(BM_name)->Name("BM_custom_name");
371
372 ADD_CASES(TC_ConsoleOut, {{"^BM_custom_name %console_report$"}});
373 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_custom_name\",$"},
374 {"\"family_index\": 12,$", MR_Next},
375 {"\"per_family_instance_index\": 0,$", MR_Next},
376 {"\"run_name\": \"BM_custom_name\",$", MR_Next},
377 {"\"run_type\": \"iteration\",$", MR_Next},
378 {"\"repetitions\": 1,$", MR_Next},
379 {"\"repetition_index\": 0,$", MR_Next},
380 {"\"threads\": 1,$", MR_Next},
381 {"\"iterations\": %int,$", MR_Next},
382 {"\"real_time\": %float,$", MR_Next},
383 {"\"cpu_time\": %float,$", MR_Next},
384 {"\"time_unit\": \"ns\"$", MR_Next},
385 {"}", MR_Next}});
386 ADD_CASES(TC_CSVOut, {{"^\"BM_custom_name\",%csv_report$"}});
387
388 // ========================================================================= //
389 // ------------------------ Testing Big Args Output ------------------------ //
390 // ========================================================================= //
391
BM_BigArgs(benchmark::State & state)392 void BM_BigArgs(benchmark::State& state) {
393 for (auto _ : state) {
394 }
395 }
396 BENCHMARK(BM_BigArgs)->RangeMultiplier(2)->Range(1U << 30U, 1U << 31U);
397 ADD_CASES(TC_ConsoleOut, {{"^BM_BigArgs/1073741824 %console_report$"},
398 {"^BM_BigArgs/2147483648 %console_report$"}});
399
400 // ========================================================================= //
401 // ----------------------- Testing Complexity Output ----------------------- //
402 // ========================================================================= //
403
BM_Complexity_O1(benchmark::State & state)404 void BM_Complexity_O1(benchmark::State& state) {
405 for (auto _ : state) {
406 // This test requires a non-zero CPU time to avoid divide-by-zero
407 benchmark::DoNotOptimize(state.iterations());
408 }
409 state.SetComplexityN(state.range(0));
410 }
411 BENCHMARK(BM_Complexity_O1)->Range(1, 1 << 18)->Complexity(benchmark::o1);
412 SET_SUBSTITUTIONS({{"%bigOStr", "[ ]* %float \\([0-9]+\\)"},
413 {"%RMS", "[ ]*[0-9]+ %"}});
414 ADD_CASES(TC_ConsoleOut, {{"^BM_Complexity_O1_BigO %bigOStr %bigOStr[ ]*$"},
415 {"^BM_Complexity_O1_RMS %RMS %RMS[ ]*$"}});
416
417 // ========================================================================= //
418 // ----------------------- Testing Aggregate Output ------------------------ //
419 // ========================================================================= //
420
421 // Test that non-aggregate data is printed by default
BM_Repeat(benchmark::State & state)422 void BM_Repeat(benchmark::State& state) {
423 for (auto _ : state) {
424 }
425 }
426 // need two repetitions min to be able to output any aggregate output
427 BENCHMARK(BM_Repeat)->Repetitions(2);
428 ADD_CASES(TC_ConsoleOut,
429 {{"^BM_Repeat/repeats:2 %console_report$"},
430 {"^BM_Repeat/repeats:2 %console_report$"},
431 {"^BM_Repeat/repeats:2_mean %console_time_only_report [ ]*2$"},
432 {"^BM_Repeat/repeats:2_median %console_time_only_report [ ]*2$"},
433 {"^BM_Repeat/repeats:2_stddev %console_time_only_report [ ]*2$"}});
434 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Repeat/repeats:2\",$"},
435 {"\"family_index\": 15,$", MR_Next},
436 {"\"per_family_instance_index\": 0,$", MR_Next},
437 {"\"run_name\": \"BM_Repeat/repeats:2\"", MR_Next},
438 {"\"run_type\": \"iteration\",$", MR_Next},
439 {"\"repetitions\": 2,$", MR_Next},
440 {"\"repetition_index\": 0,$", MR_Next},
441 {"\"threads\": 1,$", MR_Next},
442 {"\"name\": \"BM_Repeat/repeats:2\",$"},
443 {"\"family_index\": 15,$", MR_Next},
444 {"\"per_family_instance_index\": 0,$", MR_Next},
445 {"\"run_name\": \"BM_Repeat/repeats:2\",$", MR_Next},
446 {"\"run_type\": \"iteration\",$", MR_Next},
447 {"\"repetitions\": 2,$", MR_Next},
448 {"\"repetition_index\": 1,$", MR_Next},
449 {"\"threads\": 1,$", MR_Next},
450 {"\"name\": \"BM_Repeat/repeats:2_mean\",$"},
451 {"\"family_index\": 15,$", MR_Next},
452 {"\"per_family_instance_index\": 0,$", MR_Next},
453 {"\"run_name\": \"BM_Repeat/repeats:2\",$", MR_Next},
454 {"\"run_type\": \"aggregate\",$", MR_Next},
455 {"\"repetitions\": 2,$", MR_Next},
456 {"\"threads\": 1,$", MR_Next},
457 {"\"aggregate_name\": \"mean\",$", MR_Next},
458 {"\"aggregate_unit\": \"time\",$", MR_Next},
459 {"\"iterations\": 2,$", MR_Next},
460 {"\"name\": \"BM_Repeat/repeats:2_median\",$"},
461 {"\"family_index\": 15,$", MR_Next},
462 {"\"per_family_instance_index\": 0,$", MR_Next},
463 {"\"run_name\": \"BM_Repeat/repeats:2\",$", MR_Next},
464 {"\"run_type\": \"aggregate\",$", MR_Next},
465 {"\"repetitions\": 2,$", MR_Next},
466 {"\"threads\": 1,$", MR_Next},
467 {"\"aggregate_name\": \"median\",$", MR_Next},
468 {"\"aggregate_unit\": \"time\",$", MR_Next},
469 {"\"iterations\": 2,$", MR_Next},
470 {"\"name\": \"BM_Repeat/repeats:2_stddev\",$"},
471 {"\"family_index\": 15,$", MR_Next},
472 {"\"per_family_instance_index\": 0,$", MR_Next},
473 {"\"run_name\": \"BM_Repeat/repeats:2\",$", MR_Next},
474 {"\"run_type\": \"aggregate\",$", MR_Next},
475 {"\"repetitions\": 2,$", MR_Next},
476 {"\"threads\": 1,$", MR_Next},
477 {"\"aggregate_name\": \"stddev\",$", MR_Next},
478 {"\"aggregate_unit\": \"time\",$", MR_Next},
479 {"\"iterations\": 2,$", MR_Next}});
480 ADD_CASES(TC_CSVOut, {{"^\"BM_Repeat/repeats:2\",%csv_report$"},
481 {"^\"BM_Repeat/repeats:2\",%csv_report$"},
482 {"^\"BM_Repeat/repeats:2_mean\",%csv_report$"},
483 {"^\"BM_Repeat/repeats:2_median\",%csv_report$"},
484 {"^\"BM_Repeat/repeats:2_stddev\",%csv_report$"}});
485 // but for two repetitions, mean and median is the same, so let's repeat..
486 BENCHMARK(BM_Repeat)->Repetitions(3);
487 ADD_CASES(TC_ConsoleOut,
488 {{"^BM_Repeat/repeats:3 %console_report$"},
489 {"^BM_Repeat/repeats:3 %console_report$"},
490 {"^BM_Repeat/repeats:3 %console_report$"},
491 {"^BM_Repeat/repeats:3_mean %console_time_only_report [ ]*3$"},
492 {"^BM_Repeat/repeats:3_median %console_time_only_report [ ]*3$"},
493 {"^BM_Repeat/repeats:3_stddev %console_time_only_report [ ]*3$"}});
494 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Repeat/repeats:3\",$"},
495 {"\"family_index\": 16,$", MR_Next},
496 {"\"per_family_instance_index\": 0,$", MR_Next},
497 {"\"run_name\": \"BM_Repeat/repeats:3\",$", MR_Next},
498 {"\"run_type\": \"iteration\",$", MR_Next},
499 {"\"repetitions\": 3,$", MR_Next},
500 {"\"repetition_index\": 0,$", MR_Next},
501 {"\"threads\": 1,$", MR_Next},
502 {"\"name\": \"BM_Repeat/repeats:3\",$"},
503 {"\"family_index\": 16,$", MR_Next},
504 {"\"per_family_instance_index\": 0,$", MR_Next},
505 {"\"run_name\": \"BM_Repeat/repeats:3\",$", MR_Next},
506 {"\"run_type\": \"iteration\",$", MR_Next},
507 {"\"repetitions\": 3,$", MR_Next},
508 {"\"repetition_index\": 1,$", MR_Next},
509 {"\"threads\": 1,$", MR_Next},
510 {"\"name\": \"BM_Repeat/repeats:3\",$"},
511 {"\"family_index\": 16,$", MR_Next},
512 {"\"per_family_instance_index\": 0,$", MR_Next},
513 {"\"run_name\": \"BM_Repeat/repeats:3\",$", MR_Next},
514 {"\"run_type\": \"iteration\",$", MR_Next},
515 {"\"repetitions\": 3,$", MR_Next},
516 {"\"repetition_index\": 2,$", MR_Next},
517 {"\"threads\": 1,$", MR_Next},
518 {"\"name\": \"BM_Repeat/repeats:3_mean\",$"},
519 {"\"family_index\": 16,$", MR_Next},
520 {"\"per_family_instance_index\": 0,$", MR_Next},
521 {"\"run_name\": \"BM_Repeat/repeats:3\",$", MR_Next},
522 {"\"run_type\": \"aggregate\",$", MR_Next},
523 {"\"repetitions\": 3,$", MR_Next},
524 {"\"threads\": 1,$", MR_Next},
525 {"\"aggregate_name\": \"mean\",$", MR_Next},
526 {"\"aggregate_unit\": \"time\",$", MR_Next},
527 {"\"iterations\": 3,$", MR_Next},
528 {"\"name\": \"BM_Repeat/repeats:3_median\",$"},
529 {"\"family_index\": 16,$", MR_Next},
530 {"\"per_family_instance_index\": 0,$", MR_Next},
531 {"\"run_name\": \"BM_Repeat/repeats:3\",$", MR_Next},
532 {"\"run_type\": \"aggregate\",$", MR_Next},
533 {"\"repetitions\": 3,$", MR_Next},
534 {"\"threads\": 1,$", MR_Next},
535 {"\"aggregate_name\": \"median\",$", MR_Next},
536 {"\"aggregate_unit\": \"time\",$", MR_Next},
537 {"\"iterations\": 3,$", MR_Next},
538 {"\"name\": \"BM_Repeat/repeats:3_stddev\",$"},
539 {"\"family_index\": 16,$", MR_Next},
540 {"\"per_family_instance_index\": 0,$", MR_Next},
541 {"\"run_name\": \"BM_Repeat/repeats:3\",$", MR_Next},
542 {"\"run_type\": \"aggregate\",$", MR_Next},
543 {"\"repetitions\": 3,$", MR_Next},
544 {"\"threads\": 1,$", MR_Next},
545 {"\"aggregate_name\": \"stddev\",$", MR_Next},
546 {"\"aggregate_unit\": \"time\",$", MR_Next},
547 {"\"iterations\": 3,$", MR_Next}});
548 ADD_CASES(TC_CSVOut, {{"^\"BM_Repeat/repeats:3\",%csv_report$"},
549 {"^\"BM_Repeat/repeats:3\",%csv_report$"},
550 {"^\"BM_Repeat/repeats:3\",%csv_report$"},
551 {"^\"BM_Repeat/repeats:3_mean\",%csv_report$"},
552 {"^\"BM_Repeat/repeats:3_median\",%csv_report$"},
553 {"^\"BM_Repeat/repeats:3_stddev\",%csv_report$"}});
554 // median differs between even/odd number of repetitions, so just to be sure
555 BENCHMARK(BM_Repeat)->Repetitions(4);
556 ADD_CASES(TC_ConsoleOut,
557 {{"^BM_Repeat/repeats:4 %console_report$"},
558 {"^BM_Repeat/repeats:4 %console_report$"},
559 {"^BM_Repeat/repeats:4 %console_report$"},
560 {"^BM_Repeat/repeats:4 %console_report$"},
561 {"^BM_Repeat/repeats:4_mean %console_time_only_report [ ]*4$"},
562 {"^BM_Repeat/repeats:4_median %console_time_only_report [ ]*4$"},
563 {"^BM_Repeat/repeats:4_stddev %console_time_only_report [ ]*4$"}});
564 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Repeat/repeats:4\",$"},
565 {"\"family_index\": 17,$", MR_Next},
566 {"\"per_family_instance_index\": 0,$", MR_Next},
567 {"\"run_name\": \"BM_Repeat/repeats:4\",$", MR_Next},
568 {"\"run_type\": \"iteration\",$", MR_Next},
569 {"\"repetitions\": 4,$", MR_Next},
570 {"\"repetition_index\": 0,$", MR_Next},
571 {"\"threads\": 1,$", MR_Next},
572 {"\"name\": \"BM_Repeat/repeats:4\",$"},
573 {"\"family_index\": 17,$", MR_Next},
574 {"\"per_family_instance_index\": 0,$", MR_Next},
575 {"\"run_name\": \"BM_Repeat/repeats:4\",$", MR_Next},
576 {"\"run_type\": \"iteration\",$", MR_Next},
577 {"\"repetitions\": 4,$", MR_Next},
578 {"\"repetition_index\": 1,$", MR_Next},
579 {"\"threads\": 1,$", MR_Next},
580 {"\"name\": \"BM_Repeat/repeats:4\",$"},
581 {"\"family_index\": 17,$", MR_Next},
582 {"\"per_family_instance_index\": 0,$", MR_Next},
583 {"\"run_name\": \"BM_Repeat/repeats:4\",$", MR_Next},
584 {"\"run_type\": \"iteration\",$", MR_Next},
585 {"\"repetitions\": 4,$", MR_Next},
586 {"\"repetition_index\": 2,$", MR_Next},
587 {"\"threads\": 1,$", MR_Next},
588 {"\"name\": \"BM_Repeat/repeats:4\",$"},
589 {"\"family_index\": 17,$", MR_Next},
590 {"\"per_family_instance_index\": 0,$", MR_Next},
591 {"\"run_name\": \"BM_Repeat/repeats:4\",$", MR_Next},
592 {"\"run_type\": \"iteration\",$", MR_Next},
593 {"\"repetitions\": 4,$", MR_Next},
594 {"\"repetition_index\": 3,$", MR_Next},
595 {"\"threads\": 1,$", MR_Next},
596 {"\"name\": \"BM_Repeat/repeats:4_mean\",$"},
597 {"\"family_index\": 17,$", MR_Next},
598 {"\"per_family_instance_index\": 0,$", MR_Next},
599 {"\"run_name\": \"BM_Repeat/repeats:4\",$", MR_Next},
600 {"\"run_type\": \"aggregate\",$", MR_Next},
601 {"\"repetitions\": 4,$", MR_Next},
602 {"\"threads\": 1,$", MR_Next},
603 {"\"aggregate_name\": \"mean\",$", MR_Next},
604 {"\"aggregate_unit\": \"time\",$", MR_Next},
605 {"\"iterations\": 4,$", MR_Next},
606 {"\"name\": \"BM_Repeat/repeats:4_median\",$"},
607 {"\"family_index\": 17,$", MR_Next},
608 {"\"per_family_instance_index\": 0,$", MR_Next},
609 {"\"run_name\": \"BM_Repeat/repeats:4\",$", MR_Next},
610 {"\"run_type\": \"aggregate\",$", MR_Next},
611 {"\"repetitions\": 4,$", MR_Next},
612 {"\"threads\": 1,$", MR_Next},
613 {"\"aggregate_name\": \"median\",$", MR_Next},
614 {"\"aggregate_unit\": \"time\",$", MR_Next},
615 {"\"iterations\": 4,$", MR_Next},
616 {"\"name\": \"BM_Repeat/repeats:4_stddev\",$"},
617 {"\"family_index\": 17,$", MR_Next},
618 {"\"per_family_instance_index\": 0,$", MR_Next},
619 {"\"run_name\": \"BM_Repeat/repeats:4\",$", MR_Next},
620 {"\"run_type\": \"aggregate\",$", MR_Next},
621 {"\"repetitions\": 4,$", MR_Next},
622 {"\"threads\": 1,$", MR_Next},
623 {"\"aggregate_name\": \"stddev\",$", MR_Next},
624 {"\"aggregate_unit\": \"time\",$", MR_Next},
625 {"\"iterations\": 4,$", MR_Next}});
626 ADD_CASES(TC_CSVOut, {{"^\"BM_Repeat/repeats:4\",%csv_report$"},
627 {"^\"BM_Repeat/repeats:4\",%csv_report$"},
628 {"^\"BM_Repeat/repeats:4\",%csv_report$"},
629 {"^\"BM_Repeat/repeats:4\",%csv_report$"},
630 {"^\"BM_Repeat/repeats:4_mean\",%csv_report$"},
631 {"^\"BM_Repeat/repeats:4_median\",%csv_report$"},
632 {"^\"BM_Repeat/repeats:4_stddev\",%csv_report$"}});
633
634 // Test that a non-repeated test still prints non-aggregate results even when
635 // only-aggregate reports have been requested
BM_RepeatOnce(benchmark::State & state)636 void BM_RepeatOnce(benchmark::State& state) {
637 for (auto _ : state) {
638 }
639 }
640 BENCHMARK(BM_RepeatOnce)->Repetitions(1)->ReportAggregatesOnly();
641 ADD_CASES(TC_ConsoleOut, {{"^BM_RepeatOnce/repeats:1 %console_report$"}});
642 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_RepeatOnce/repeats:1\",$"},
643 {"\"family_index\": 18,$", MR_Next},
644 {"\"per_family_instance_index\": 0,$", MR_Next},
645 {"\"run_name\": \"BM_RepeatOnce/repeats:1\",$", MR_Next},
646 {"\"run_type\": \"iteration\",$", MR_Next},
647 {"\"repetitions\": 1,$", MR_Next},
648 {"\"repetition_index\": 0,$", MR_Next},
649 {"\"threads\": 1,$", MR_Next}});
650 ADD_CASES(TC_CSVOut, {{"^\"BM_RepeatOnce/repeats:1\",%csv_report$"}});
651
652 // Test that non-aggregate data is not reported
BM_SummaryRepeat(benchmark::State & state)653 void BM_SummaryRepeat(benchmark::State& state) {
654 for (auto _ : state) {
655 }
656 }
657 BENCHMARK(BM_SummaryRepeat)->Repetitions(3)->ReportAggregatesOnly();
658 ADD_CASES(
659 TC_ConsoleOut,
660 {{".*BM_SummaryRepeat/repeats:3 ", MR_Not},
661 {"^BM_SummaryRepeat/repeats:3_mean %console_time_only_report [ ]*3$"},
662 {"^BM_SummaryRepeat/repeats:3_median %console_time_only_report [ ]*3$"},
663 {"^BM_SummaryRepeat/repeats:3_stddev %console_time_only_report [ ]*3$"}});
664 ADD_CASES(TC_JSONOut,
665 {{".*BM_SummaryRepeat/repeats:3 ", MR_Not},
666 {"\"name\": \"BM_SummaryRepeat/repeats:3_mean\",$"},
667 {"\"family_index\": 19,$", MR_Next},
668 {"\"per_family_instance_index\": 0,$", MR_Next},
669 {"\"run_name\": \"BM_SummaryRepeat/repeats:3\",$", MR_Next},
670 {"\"run_type\": \"aggregate\",$", MR_Next},
671 {"\"repetitions\": 3,$", MR_Next},
672 {"\"threads\": 1,$", MR_Next},
673 {"\"aggregate_name\": \"mean\",$", MR_Next},
674 {"\"aggregate_unit\": \"time\",$", MR_Next},
675 {"\"iterations\": 3,$", MR_Next},
676 {"\"name\": \"BM_SummaryRepeat/repeats:3_median\",$"},
677 {"\"family_index\": 19,$", MR_Next},
678 {"\"per_family_instance_index\": 0,$", MR_Next},
679 {"\"run_name\": \"BM_SummaryRepeat/repeats:3\",$", MR_Next},
680 {"\"run_type\": \"aggregate\",$", MR_Next},
681 {"\"repetitions\": 3,$", MR_Next},
682 {"\"threads\": 1,$", MR_Next},
683 {"\"aggregate_name\": \"median\",$", MR_Next},
684 {"\"aggregate_unit\": \"time\",$", MR_Next},
685 {"\"iterations\": 3,$", MR_Next},
686 {"\"name\": \"BM_SummaryRepeat/repeats:3_stddev\",$"},
687 {"\"family_index\": 19,$", MR_Next},
688 {"\"per_family_instance_index\": 0,$", MR_Next},
689 {"\"run_name\": \"BM_SummaryRepeat/repeats:3\",$", MR_Next},
690 {"\"run_type\": \"aggregate\",$", MR_Next},
691 {"\"repetitions\": 3,$", MR_Next},
692 {"\"threads\": 1,$", MR_Next},
693 {"\"aggregate_name\": \"stddev\",$", MR_Next},
694 {"\"aggregate_unit\": \"time\",$", MR_Next},
695 {"\"iterations\": 3,$", MR_Next}});
696 ADD_CASES(TC_CSVOut, {{".*BM_SummaryRepeat/repeats:3 ", MR_Not},
697 {"^\"BM_SummaryRepeat/repeats:3_mean\",%csv_report$"},
698 {"^\"BM_SummaryRepeat/repeats:3_median\",%csv_report$"},
699 {"^\"BM_SummaryRepeat/repeats:3_stddev\",%csv_report$"}});
700
701 // Test that non-aggregate data is not displayed.
702 // NOTE: this test is kinda bad. we are only testing the display output.
703 // But we don't check that the file output still contains everything...
BM_SummaryDisplay(benchmark::State & state)704 void BM_SummaryDisplay(benchmark::State& state) {
705 for (auto _ : state) {
706 }
707 }
708 BENCHMARK(BM_SummaryDisplay)->Repetitions(2)->DisplayAggregatesOnly();
709 ADD_CASES(
710 TC_ConsoleOut,
711 {{".*BM_SummaryDisplay/repeats:2 ", MR_Not},
712 {"^BM_SummaryDisplay/repeats:2_mean %console_time_only_report [ ]*2$"},
713 {"^BM_SummaryDisplay/repeats:2_median %console_time_only_report [ ]*2$"},
714 {"^BM_SummaryDisplay/repeats:2_stddev %console_time_only_report [ ]*2$"}});
715 ADD_CASES(TC_JSONOut,
716 {{".*BM_SummaryDisplay/repeats:2 ", MR_Not},
717 {"\"name\": \"BM_SummaryDisplay/repeats:2_mean\",$"},
718 {"\"family_index\": 20,$", MR_Next},
719 {"\"per_family_instance_index\": 0,$", MR_Next},
720 {"\"run_name\": \"BM_SummaryDisplay/repeats:2\",$", MR_Next},
721 {"\"run_type\": \"aggregate\",$", MR_Next},
722 {"\"repetitions\": 2,$", MR_Next},
723 {"\"threads\": 1,$", MR_Next},
724 {"\"aggregate_name\": \"mean\",$", MR_Next},
725 {"\"aggregate_unit\": \"time\",$", MR_Next},
726 {"\"iterations\": 2,$", MR_Next},
727 {"\"name\": \"BM_SummaryDisplay/repeats:2_median\",$"},
728 {"\"family_index\": 20,$", MR_Next},
729 {"\"per_family_instance_index\": 0,$", MR_Next},
730 {"\"run_name\": \"BM_SummaryDisplay/repeats:2\",$", MR_Next},
731 {"\"run_type\": \"aggregate\",$", MR_Next},
732 {"\"repetitions\": 2,$", MR_Next},
733 {"\"threads\": 1,$", MR_Next},
734 {"\"aggregate_name\": \"median\",$", MR_Next},
735 {"\"aggregate_unit\": \"time\",$", MR_Next},
736 {"\"iterations\": 2,$", MR_Next},
737 {"\"name\": \"BM_SummaryDisplay/repeats:2_stddev\",$"},
738 {"\"family_index\": 20,$", MR_Next},
739 {"\"per_family_instance_index\": 0,$", MR_Next},
740 {"\"run_name\": \"BM_SummaryDisplay/repeats:2\",$", MR_Next},
741 {"\"run_type\": \"aggregate\",$", MR_Next},
742 {"\"repetitions\": 2,$", MR_Next},
743 {"\"threads\": 1,$", MR_Next},
744 {"\"aggregate_name\": \"stddev\",$", MR_Next},
745 {"\"aggregate_unit\": \"time\",$", MR_Next},
746 {"\"iterations\": 2,$", MR_Next}});
747 ADD_CASES(TC_CSVOut,
748 {{".*BM_SummaryDisplay/repeats:2 ", MR_Not},
749 {"^\"BM_SummaryDisplay/repeats:2_mean\",%csv_report$"},
750 {"^\"BM_SummaryDisplay/repeats:2_median\",%csv_report$"},
751 {"^\"BM_SummaryDisplay/repeats:2_stddev\",%csv_report$"}});
752
753 // Test repeats with custom time unit.
BM_RepeatTimeUnit(benchmark::State & state)754 void BM_RepeatTimeUnit(benchmark::State& state) {
755 for (auto _ : state) {
756 }
757 }
758 BENCHMARK(BM_RepeatTimeUnit)
759 ->Repetitions(3)
760 ->ReportAggregatesOnly()
761 ->Unit(benchmark::kMicrosecond);
762 ADD_CASES(
763 TC_ConsoleOut,
764 {{".*BM_RepeatTimeUnit/repeats:3 ", MR_Not},
765 {"^BM_RepeatTimeUnit/repeats:3_mean %console_us_time_only_report [ ]*3$"},
766 {"^BM_RepeatTimeUnit/repeats:3_median %console_us_time_only_report [ "
767 "]*3$"},
768 {"^BM_RepeatTimeUnit/repeats:3_stddev %console_us_time_only_report [ "
769 "]*3$"}});
770 ADD_CASES(TC_JSONOut,
771 {{".*BM_RepeatTimeUnit/repeats:3 ", MR_Not},
772 {"\"name\": \"BM_RepeatTimeUnit/repeats:3_mean\",$"},
773 {"\"family_index\": 21,$", MR_Next},
774 {"\"per_family_instance_index\": 0,$", MR_Next},
775 {"\"run_name\": \"BM_RepeatTimeUnit/repeats:3\",$", MR_Next},
776 {"\"run_type\": \"aggregate\",$", MR_Next},
777 {"\"repetitions\": 3,$", MR_Next},
778 {"\"threads\": 1,$", MR_Next},
779 {"\"aggregate_name\": \"mean\",$", MR_Next},
780 {"\"aggregate_unit\": \"time\",$", MR_Next},
781 {"\"iterations\": 3,$", MR_Next},
782 {"\"time_unit\": \"us\",?$"},
783 {"\"name\": \"BM_RepeatTimeUnit/repeats:3_median\",$"},
784 {"\"family_index\": 21,$", MR_Next},
785 {"\"per_family_instance_index\": 0,$", MR_Next},
786 {"\"run_name\": \"BM_RepeatTimeUnit/repeats:3\",$", MR_Next},
787 {"\"run_type\": \"aggregate\",$", MR_Next},
788 {"\"repetitions\": 3,$", MR_Next},
789 {"\"threads\": 1,$", MR_Next},
790 {"\"aggregate_name\": \"median\",$", MR_Next},
791 {"\"aggregate_unit\": \"time\",$", MR_Next},
792 {"\"iterations\": 3,$", MR_Next},
793 {"\"time_unit\": \"us\",?$"},
794 {"\"name\": \"BM_RepeatTimeUnit/repeats:3_stddev\",$"},
795 {"\"family_index\": 21,$", MR_Next},
796 {"\"per_family_instance_index\": 0,$", MR_Next},
797 {"\"run_name\": \"BM_RepeatTimeUnit/repeats:3\",$", MR_Next},
798 {"\"run_type\": \"aggregate\",$", MR_Next},
799 {"\"repetitions\": 3,$", MR_Next},
800 {"\"threads\": 1,$", MR_Next},
801 {"\"aggregate_name\": \"stddev\",$", MR_Next},
802 {"\"aggregate_unit\": \"time\",$", MR_Next},
803 {"\"iterations\": 3,$", MR_Next},
804 {"\"time_unit\": \"us\",?$"}});
805 ADD_CASES(TC_CSVOut,
806 {{".*BM_RepeatTimeUnit/repeats:3 ", MR_Not},
807 {"^\"BM_RepeatTimeUnit/repeats:3_mean\",%csv_us_report$"},
808 {"^\"BM_RepeatTimeUnit/repeats:3_median\",%csv_us_report$"},
809 {"^\"BM_RepeatTimeUnit/repeats:3_stddev\",%csv_us_report$"}});
810
811 // ========================================================================= //
812 // -------------------- Testing user-provided statistics ------------------- //
813 // ========================================================================= //
814
__anon3970b7c20102(const std::vector<double>& v) 815 const auto UserStatistics = [](const std::vector<double>& v) {
816 return v.back();
817 };
BM_UserStats(benchmark::State & state)818 void BM_UserStats(benchmark::State& state) {
819 for (auto _ : state) {
820 state.SetIterationTime(150 / 10e8);
821 }
822 }
823 // clang-format off
824 BENCHMARK(BM_UserStats)
825 ->Repetitions(3)
826 ->Iterations(5)
827 ->UseManualTime()
828 ->ComputeStatistics("", UserStatistics);
829 // clang-format on
830
831 // check that user-provided stats is calculated, and is after the default-ones
832 // empty string as name is intentional, it would sort before anything else
833 ADD_CASES(TC_ConsoleOut, {{"^BM_UserStats/iterations:5/repeats:3/manual_time [ "
834 "]* 150 ns %time [ ]*5$"},
835 {"^BM_UserStats/iterations:5/repeats:3/manual_time [ "
836 "]* 150 ns %time [ ]*5$"},
837 {"^BM_UserStats/iterations:5/repeats:3/manual_time [ "
838 "]* 150 ns %time [ ]*5$"},
839 {"^BM_UserStats/iterations:5/repeats:3/"
840 "manual_time_mean [ ]* 150 ns %time [ ]*3$"},
841 {"^BM_UserStats/iterations:5/repeats:3/"
842 "manual_time_median [ ]* 150 ns %time [ ]*3$"},
843 {"^BM_UserStats/iterations:5/repeats:3/"
844 "manual_time_stddev [ ]* 0.000 ns %time [ ]*3$"},
845 {"^BM_UserStats/iterations:5/repeats:3/manual_time_ "
846 "[ ]* 150 ns %time [ ]*3$"}});
847 ADD_CASES(
848 TC_JSONOut,
849 {{"\"name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$"},
850 {"\"family_index\": 22,$", MR_Next},
851 {"\"per_family_instance_index\": 0,$", MR_Next},
852 {"\"run_name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$",
853 MR_Next},
854 {"\"run_type\": \"iteration\",$", MR_Next},
855 {"\"repetitions\": 3,$", MR_Next},
856 {"\"repetition_index\": 0,$", MR_Next},
857 {"\"threads\": 1,$", MR_Next},
858 {"\"iterations\": 5,$", MR_Next},
859 {"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next},
860 {"\"name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$"},
861 {"\"family_index\": 22,$", MR_Next},
862 {"\"per_family_instance_index\": 0,$", MR_Next},
863 {"\"run_name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$",
864 MR_Next},
865 {"\"run_type\": \"iteration\",$", MR_Next},
866 {"\"repetitions\": 3,$", MR_Next},
867 {"\"repetition_index\": 1,$", MR_Next},
868 {"\"threads\": 1,$", MR_Next},
869 {"\"iterations\": 5,$", MR_Next},
870 {"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next},
871 {"\"name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$"},
872 {"\"family_index\": 22,$", MR_Next},
873 {"\"per_family_instance_index\": 0,$", MR_Next},
874 {"\"run_name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$",
875 MR_Next},
876 {"\"run_type\": \"iteration\",$", MR_Next},
877 {"\"repetitions\": 3,$", MR_Next},
878 {"\"repetition_index\": 2,$", MR_Next},
879 {"\"threads\": 1,$", MR_Next},
880 {"\"iterations\": 5,$", MR_Next},
881 {"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next},
882 {"\"name\": \"BM_UserStats/iterations:5/repeats:3/manual_time_mean\",$"},
883 {"\"family_index\": 22,$", MR_Next},
884 {"\"per_family_instance_index\": 0,$", MR_Next},
885 {"\"run_name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$",
886 MR_Next},
887 {"\"run_type\": \"aggregate\",$", MR_Next},
888 {"\"repetitions\": 3,$", MR_Next},
889 {"\"threads\": 1,$", MR_Next},
890 {"\"aggregate_name\": \"mean\",$", MR_Next},
891 {"\"aggregate_unit\": \"time\",$", MR_Next},
892 {"\"iterations\": 3,$", MR_Next},
893 {"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next},
894 {"\"name\": \"BM_UserStats/iterations:5/repeats:3/manual_time_median\",$"},
895 {"\"family_index\": 22,$", MR_Next},
896 {"\"per_family_instance_index\": 0,$", MR_Next},
897 {"\"run_name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$",
898 MR_Next},
899 {"\"run_type\": \"aggregate\",$", MR_Next},
900 {"\"repetitions\": 3,$", MR_Next},
901 {"\"threads\": 1,$", MR_Next},
902 {"\"aggregate_name\": \"median\",$", MR_Next},
903 {"\"aggregate_unit\": \"time\",$", MR_Next},
904 {"\"iterations\": 3,$", MR_Next},
905 {"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next},
906 {"\"name\": \"BM_UserStats/iterations:5/repeats:3/manual_time_stddev\",$"},
907 {"\"family_index\": 22,$", MR_Next},
908 {"\"per_family_instance_index\": 0,$", MR_Next},
909 {"\"run_name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$",
910 MR_Next},
911 {"\"run_type\": \"aggregate\",$", MR_Next},
912 {"\"repetitions\": 3,$", MR_Next},
913 {"\"threads\": 1,$", MR_Next},
914 {"\"aggregate_name\": \"stddev\",$", MR_Next},
915 {"\"aggregate_unit\": \"time\",$", MR_Next},
916 {"\"iterations\": 3,$", MR_Next},
917 {"\"real_time\": %float,$", MR_Next},
918 {"\"name\": \"BM_UserStats/iterations:5/repeats:3/manual_time_\",$"},
919 {"\"family_index\": 22,$", MR_Next},
920 {"\"per_family_instance_index\": 0,$", MR_Next},
921 {"\"run_name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$",
922 MR_Next},
923 {"\"run_type\": \"aggregate\",$", MR_Next},
924 {"\"repetitions\": 3,$", MR_Next},
925 {"\"threads\": 1,$", MR_Next},
926 {"\"aggregate_name\": \"\",$", MR_Next},
927 {"\"aggregate_unit\": \"time\",$", MR_Next},
928 {"\"iterations\": 3,$", MR_Next},
929 {"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next}});
930 ADD_CASES(
931 TC_CSVOut,
932 {{"^\"BM_UserStats/iterations:5/repeats:3/manual_time\",%csv_report$"},
933 {"^\"BM_UserStats/iterations:5/repeats:3/manual_time\",%csv_report$"},
934 {"^\"BM_UserStats/iterations:5/repeats:3/manual_time\",%csv_report$"},
935 {"^\"BM_UserStats/iterations:5/repeats:3/manual_time_mean\",%csv_report$"},
936 {"^\"BM_UserStats/iterations:5/repeats:3/"
937 "manual_time_median\",%csv_report$"},
938 {"^\"BM_UserStats/iterations:5/repeats:3/"
939 "manual_time_stddev\",%csv_report$"},
940 {"^\"BM_UserStats/iterations:5/repeats:3/manual_time_\",%csv_report$"}});
941
942 // ========================================================================= //
943 // ------------- Testing relative standard deviation statistics ------------ //
944 // ========================================================================= //
945
__anon3970b7c20202(const std::vector<double>&) 946 const auto UserPercentStatistics = [](const std::vector<double>&) {
947 return 1. / 100.;
948 };
BM_UserPercentStats(benchmark::State & state)949 void BM_UserPercentStats(benchmark::State& state) {
950 for (auto _ : state) {
951 state.SetIterationTime(150 / 10e8);
952 }
953 }
954 // clang-format off
955 BENCHMARK(BM_UserPercentStats)
956 ->Repetitions(3)
957 ->Iterations(5)
958 ->UseManualTime()
959 ->Unit(benchmark::TimeUnit::kNanosecond)
960 ->ComputeStatistics("", UserPercentStatistics, benchmark::StatisticUnit::kPercentage);
961 // clang-format on
962
963 // check that UserPercent-provided stats is calculated, and is after the
964 // default-ones empty string as name is intentional, it would sort before
965 // anything else
966 ADD_CASES(TC_ConsoleOut,
967 {{"^BM_UserPercentStats/iterations:5/repeats:3/manual_time [ "
968 "]* 150 ns %time [ ]*5$"},
969 {"^BM_UserPercentStats/iterations:5/repeats:3/manual_time [ "
970 "]* 150 ns %time [ ]*5$"},
971 {"^BM_UserPercentStats/iterations:5/repeats:3/manual_time [ "
972 "]* 150 ns %time [ ]*5$"},
973 {"^BM_UserPercentStats/iterations:5/repeats:3/"
974 "manual_time_mean [ ]* 150 ns %time [ ]*3$"},
975 {"^BM_UserPercentStats/iterations:5/repeats:3/"
976 "manual_time_median [ ]* 150 ns %time [ ]*3$"},
977 {"^BM_UserPercentStats/iterations:5/repeats:3/"
978 "manual_time_stddev [ ]* 0.000 ns %time [ ]*3$"},
979 {"^BM_UserPercentStats/iterations:5/repeats:3/manual_time_ "
980 "[ ]* 1.00 % [ ]* 1.00 %[ ]*3$"}});
981 ADD_CASES(
982 TC_JSONOut,
983 {{"\"name\": \"BM_UserPercentStats/iterations:5/repeats:3/manual_time\",$"},
984 {"\"family_index\": 23,$", MR_Next},
985 {"\"per_family_instance_index\": 0,$", MR_Next},
986 {"\"run_name\": "
987 "\"BM_UserPercentStats/iterations:5/repeats:3/manual_time\",$",
988 MR_Next},
989 {"\"run_type\": \"iteration\",$", MR_Next},
990 {"\"repetitions\": 3,$", MR_Next},
991 {"\"repetition_index\": 0,$", MR_Next},
992 {"\"threads\": 1,$", MR_Next},
993 {"\"iterations\": 5,$", MR_Next},
994 {"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next},
995 {"\"name\": \"BM_UserPercentStats/iterations:5/repeats:3/manual_time\",$"},
996 {"\"family_index\": 23,$", MR_Next},
997 {"\"per_family_instance_index\": 0,$", MR_Next},
998 {"\"run_name\": "
999 "\"BM_UserPercentStats/iterations:5/repeats:3/manual_time\",$",
1000 MR_Next},
1001 {"\"run_type\": \"iteration\",$", MR_Next},
1002 {"\"repetitions\": 3,$", MR_Next},
1003 {"\"repetition_index\": 1,$", MR_Next},
1004 {"\"threads\": 1,$", MR_Next},
1005 {"\"iterations\": 5,$", MR_Next},
1006 {"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next},
1007 {"\"name\": \"BM_UserPercentStats/iterations:5/repeats:3/manual_time\",$"},
1008 {"\"family_index\": 23,$", MR_Next},
1009 {"\"per_family_instance_index\": 0,$", MR_Next},
1010 {"\"run_name\": "
1011 "\"BM_UserPercentStats/iterations:5/repeats:3/manual_time\",$",
1012 MR_Next},
1013 {"\"run_type\": \"iteration\",$", MR_Next},
1014 {"\"repetitions\": 3,$", MR_Next},
1015 {"\"repetition_index\": 2,$", MR_Next},
1016 {"\"threads\": 1,$", MR_Next},
1017 {"\"iterations\": 5,$", MR_Next},
1018 {"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next},
1019 {"\"name\": "
1020 "\"BM_UserPercentStats/iterations:5/repeats:3/manual_time_mean\",$"},
1021 {"\"family_index\": 23,$", MR_Next},
1022 {"\"per_family_instance_index\": 0,$", MR_Next},
1023 {"\"run_name\": "
1024 "\"BM_UserPercentStats/iterations:5/repeats:3/manual_time\",$",
1025 MR_Next},
1026 {"\"run_type\": \"aggregate\",$", MR_Next},
1027 {"\"repetitions\": 3,$", MR_Next},
1028 {"\"threads\": 1,$", MR_Next},
1029 {"\"aggregate_name\": \"mean\",$", MR_Next},
1030 {"\"aggregate_unit\": \"time\",$", MR_Next},
1031 {"\"iterations\": 3,$", MR_Next},
1032 {"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next},
1033 {"\"name\": "
1034 "\"BM_UserPercentStats/iterations:5/repeats:3/manual_time_median\",$"},
1035 {"\"family_index\": 23,$", MR_Next},
1036 {"\"per_family_instance_index\": 0,$", MR_Next},
1037 {"\"run_name\": "
1038 "\"BM_UserPercentStats/iterations:5/repeats:3/manual_time\",$",
1039 MR_Next},
1040 {"\"run_type\": \"aggregate\",$", MR_Next},
1041 {"\"repetitions\": 3,$", MR_Next},
1042 {"\"threads\": 1,$", MR_Next},
1043 {"\"aggregate_name\": \"median\",$", MR_Next},
1044 {"\"aggregate_unit\": \"time\",$", MR_Next},
1045 {"\"iterations\": 3,$", MR_Next},
1046 {"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next},
1047 {"\"name\": "
1048 "\"BM_UserPercentStats/iterations:5/repeats:3/manual_time_stddev\",$"},
1049 {"\"family_index\": 23,$", MR_Next},
1050 {"\"per_family_instance_index\": 0,$", MR_Next},
1051 {"\"run_name\": "
1052 "\"BM_UserPercentStats/iterations:5/repeats:3/manual_time\",$",
1053 MR_Next},
1054 {"\"run_type\": \"aggregate\",$", MR_Next},
1055 {"\"repetitions\": 3,$", MR_Next},
1056 {"\"threads\": 1,$", MR_Next},
1057 {"\"aggregate_name\": \"stddev\",$", MR_Next},
1058 {"\"aggregate_unit\": \"time\",$", MR_Next},
1059 {"\"iterations\": 3,$", MR_Next},
1060 {"\"real_time\": %float,$", MR_Next},
1061 {"\"name\": "
1062 "\"BM_UserPercentStats/iterations:5/repeats:3/manual_time_\",$"},
1063 {"\"family_index\": 23,$", MR_Next},
1064 {"\"per_family_instance_index\": 0,$", MR_Next},
1065 {"\"run_name\": "
1066 "\"BM_UserPercentStats/iterations:5/repeats:3/manual_time\",$",
1067 MR_Next},
1068 {"\"run_type\": \"aggregate\",$", MR_Next},
1069 {"\"repetitions\": 3,$", MR_Next},
1070 {"\"threads\": 1,$", MR_Next},
1071 {"\"aggregate_name\": \"\",$", MR_Next},
1072 {"\"aggregate_unit\": \"percentage\",$", MR_Next},
1073 {"\"iterations\": 3,$", MR_Next},
1074 {"\"real_time\": 1\\.(0)*e-(0)*2,$", MR_Next}});
1075 ADD_CASES(TC_CSVOut, {{"^\"BM_UserPercentStats/iterations:5/repeats:3/"
1076 "manual_time\",%csv_report$"},
1077 {"^\"BM_UserPercentStats/iterations:5/repeats:3/"
1078 "manual_time\",%csv_report$"},
1079 {"^\"BM_UserPercentStats/iterations:5/repeats:3/"
1080 "manual_time\",%csv_report$"},
1081 {"^\"BM_UserPercentStats/iterations:5/repeats:3/"
1082 "manual_time_mean\",%csv_report$"},
1083 {"^\"BM_UserPercentStats/iterations:5/repeats:3/"
1084 "manual_time_median\",%csv_report$"},
1085 {"^\"BM_UserPercentStats/iterations:5/repeats:3/"
1086 "manual_time_stddev\",%csv_report$"},
1087 {"^\"BM_UserPercentStats/iterations:5/repeats:3/"
1088 "manual_time_\",%csv_report$"}});
1089
1090 // ========================================================================= //
1091 // ------------------------- Testing StrEscape JSON ------------------------ //
1092 // ========================================================================= //
1093 #if 0 // enable when csv testing code correctly handles multi-line fields
1094 void BM_JSON_Format(benchmark::State& state) {
1095 state.SkipWithError("val\b\f\n\r\t\\\"with\"es,capes");
1096 for (auto _ : state) {
1097 }
1098 }
1099 BENCHMARK(BM_JSON_Format);
1100 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_JSON_Format\",$"},
1101 {"\"family_index\": 23,$", MR_Next},
1102 {"\"per_family_instance_index\": 0,$", MR_Next},
1103 {"\"run_name\": \"BM_JSON_Format\",$", MR_Next},
1104 {"\"run_type\": \"iteration\",$", MR_Next},
1105 {"\"repetitions\": 1,$", MR_Next},
1106 {"\"repetition_index\": 0,$", MR_Next},
1107 {"\"threads\": 1,$", MR_Next},
1108 {"\"error_occurred\": true,$", MR_Next},
1109 {R"("error_message": "val\\b\\f\\n\\r\\t\\\\\\"with\\"es,capes",$)", MR_Next}});
1110 #endif
1111 // ========================================================================= //
1112 // -------------------------- Testing CsvEscape ---------------------------- //
1113 // ========================================================================= //
1114
BM_CSV_Format(benchmark::State & state)1115 void BM_CSV_Format(benchmark::State& state) {
1116 state.SkipWithError("\"freedom\"");
1117 for (auto _ : state) {
1118 }
1119 }
1120 BENCHMARK(BM_CSV_Format);
1121 ADD_CASES(TC_CSVOut, {{"^\"BM_CSV_Format\",,,,,,,,true,\"\"\"freedom\"\"\"$"}});
1122
1123 // ========================================================================= //
1124 // --------------------------- TEST CASES END ------------------------------ //
1125 // ========================================================================= //
1126
main(int argc,char * argv[])1127 int main(int argc, char* argv[]) { RunOutputTests(argc, argv); }
1128