1
2 #undef NDEBUG
3
4 #include "benchmark/benchmark.h"
5 #include "output_test.h"
6
7 // @todo: <jpmag> this checks the full output at once; the rule for
8 // CounterSet1 was failing because it was not matching "^[-]+$".
9 // @todo: <jpmag> check that the counters are vertically aligned.
10 ADD_CASES(TC_ConsoleOut,
11 {
12 // keeping these lines long improves readability, so:
13 // clang-format off
14 {"^[-]+$", MR_Next},
15 {"^Benchmark %s Time %s CPU %s Iterations %s Bar %s Bat %s Baz %s Foo %s Frob %s Lob$", MR_Next},
16 {"^[-]+$", MR_Next},
17 {"^BM_Counters_Tabular/repeats:2/threads:1 %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
18 {"^BM_Counters_Tabular/repeats:2/threads:1 %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
19 {"^BM_Counters_Tabular/repeats:2/threads:1_mean %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
20 {"^BM_Counters_Tabular/repeats:2/threads:1_median %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
21 {"^BM_Counters_Tabular/repeats:2/threads:1_stddev %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
22 {"^BM_Counters_Tabular/repeats:2/threads:1_cv %console_percentage_report [ ]*%percentage[ ]*% [ ]*%percentage[ ]*% [ ]*%percentage[ ]*% [ ]*%percentage[ ]*% [ ]*%percentage[ ]*% [ ]*%percentage[ ]*%$", MR_Next},
23 {"^BM_Counters_Tabular/repeats:2/threads:2 %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
24 {"^BM_Counters_Tabular/repeats:2/threads:2 %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
25 {"^BM_Counters_Tabular/repeats:2/threads:2_mean %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
26 {"^BM_Counters_Tabular/repeats:2/threads:2_median %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
27 {"^BM_Counters_Tabular/repeats:2/threads:2_stddev %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
28 {"^BM_Counters_Tabular/repeats:2/threads:2_cv %console_percentage_report [ ]*%percentage[ ]*% [ ]*%percentage[ ]*% [ ]*%percentage[ ]*% [ ]*%percentage[ ]*% [ ]*%percentage[ ]*% [ ]*%percentage[ ]*%$", MR_Next},
29 {"^BM_CounterRates_Tabular/threads:%int %console_report [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s$", MR_Next},
30 {"^BM_CounterRates_Tabular/threads:%int %console_report [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s$", MR_Next},
31 {"^BM_CounterRates_Tabular/threads:%int %console_report [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s$", MR_Next},
32 {"^BM_CounterRates_Tabular/threads:%int %console_report [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s$", MR_Next},
33 {"^BM_CounterRates_Tabular/threads:%int %console_report [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s$", MR_Next},
34 {"^[-]+$", MR_Next},
35 {"^Benchmark %s Time %s CPU %s Iterations %s Bar %s Baz %s Foo$", MR_Next},
36 {"^[-]+$", MR_Next},
37 {"^BM_CounterSet0_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
38 {"^BM_CounterSet0_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
39 {"^BM_CounterSet0_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
40 {"^BM_CounterSet0_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
41 {"^BM_CounterSet0_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
42 {"^BM_CounterSet1_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
43 {"^BM_CounterSet1_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
44 {"^BM_CounterSet1_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
45 {"^BM_CounterSet1_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
46 {"^BM_CounterSet1_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
47 {"^[-]+$", MR_Next},
48 {"^Benchmark %s Time %s CPU %s Iterations %s Bat %s Baz %s Foo$", MR_Next},
49 {"^[-]+$", MR_Next},
50 {"^BM_CounterSet2_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
51 {"^BM_CounterSet2_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
52 {"^BM_CounterSet2_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
53 {"^BM_CounterSet2_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
54 {"^BM_CounterSet2_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$"},
55 // clang-format on
56 });
57 ADD_CASES(TC_CSVOut, {{"%csv_header,"
58 "\"Bar\",\"Bat\",\"Baz\",\"Foo\",\"Frob\",\"Lob\""}});
59
60 // ========================================================================= //
61 // ------------------------- Tabular Counters Output ----------------------- //
62 // ========================================================================= //
63
BM_Counters_Tabular(benchmark::State & state)64 void BM_Counters_Tabular(benchmark::State& state) {
65 for (auto _ : state) {
66 }
67 namespace bm = benchmark;
68 state.counters.insert({
69 {"Foo", {1, bm::Counter::kAvgThreads}},
70 {"Bar", {2, bm::Counter::kAvgThreads}},
71 {"Baz", {4, bm::Counter::kAvgThreads}},
72 {"Bat", {8, bm::Counter::kAvgThreads}},
73 {"Frob", {16, bm::Counter::kAvgThreads}},
74 {"Lob", {32, bm::Counter::kAvgThreads}},
75 });
76 }
77 BENCHMARK(BM_Counters_Tabular)->ThreadRange(1, 2)->Repetitions(2);
78 ADD_CASES(TC_JSONOut,
79 {{"\"name\": \"BM_Counters_Tabular/repeats:2/threads:1\",$"},
80 {"\"family_index\": 0,$", MR_Next},
81 {"\"per_family_instance_index\": 0,$", MR_Next},
82 {"\"run_name\": \"BM_Counters_Tabular/repeats:2/threads:1\",$",
83 MR_Next},
84 {"\"run_type\": \"iteration\",$", MR_Next},
85 {"\"repetitions\": 2,$", MR_Next},
86 {"\"repetition_index\": 0,$", MR_Next},
87 {"\"threads\": 1,$", MR_Next},
88 {"\"iterations\": %int,$", MR_Next},
89 {"\"real_time\": %float,$", MR_Next},
90 {"\"cpu_time\": %float,$", MR_Next},
91 {"\"time_unit\": \"ns\",$", MR_Next},
92 {"\"Bar\": %float,$", MR_Next},
93 {"\"Bat\": %float,$", MR_Next},
94 {"\"Baz\": %float,$", MR_Next},
95 {"\"Foo\": %float,$", MR_Next},
96 {"\"Frob\": %float,$", MR_Next},
97 {"\"Lob\": %float$", MR_Next},
98 {"}", MR_Next}});
99 ADD_CASES(TC_JSONOut,
100 {{"\"name\": \"BM_Counters_Tabular/repeats:2/threads:1\",$"},
101 {"\"family_index\": 0,$", MR_Next},
102 {"\"per_family_instance_index\": 0,$", MR_Next},
103 {"\"run_name\": \"BM_Counters_Tabular/repeats:2/threads:1\",$",
104 MR_Next},
105 {"\"run_type\": \"iteration\",$", MR_Next},
106 {"\"repetitions\": 2,$", MR_Next},
107 {"\"repetition_index\": 1,$", MR_Next},
108 {"\"threads\": 1,$", MR_Next},
109 {"\"iterations\": %int,$", MR_Next},
110 {"\"real_time\": %float,$", MR_Next},
111 {"\"cpu_time\": %float,$", MR_Next},
112 {"\"time_unit\": \"ns\",$", MR_Next},
113 {"\"Bar\": %float,$", MR_Next},
114 {"\"Bat\": %float,$", MR_Next},
115 {"\"Baz\": %float,$", MR_Next},
116 {"\"Foo\": %float,$", MR_Next},
117 {"\"Frob\": %float,$", MR_Next},
118 {"\"Lob\": %float$", MR_Next},
119 {"}", MR_Next}});
120 ADD_CASES(TC_JSONOut,
121 {{"\"name\": \"BM_Counters_Tabular/repeats:2/threads:1_mean\",$"},
122 {"\"family_index\": 0,$", MR_Next},
123 {"\"per_family_instance_index\": 0,$", MR_Next},
124 {"\"run_name\": \"BM_Counters_Tabular/repeats:2/threads:1\",$",
125 MR_Next},
126 {"\"run_type\": \"aggregate\",$", MR_Next},
127 {"\"repetitions\": 2,$", MR_Next},
128 {"\"threads\": 1,$", MR_Next},
129 {"\"aggregate_name\": \"mean\",$", MR_Next},
130 {"\"aggregate_unit\": \"time\",$", MR_Next},
131 {"\"iterations\": %int,$", MR_Next},
132 {"\"real_time\": %float,$", MR_Next},
133 {"\"cpu_time\": %float,$", MR_Next},
134 {"\"time_unit\": \"ns\",$", MR_Next},
135 {"\"Bar\": %float,$", MR_Next},
136 {"\"Bat\": %float,$", MR_Next},
137 {"\"Baz\": %float,$", MR_Next},
138 {"\"Foo\": %float,$", MR_Next},
139 {"\"Frob\": %float,$", MR_Next},
140 {"\"Lob\": %float$", MR_Next},
141 {"}", MR_Next}});
142 ADD_CASES(TC_JSONOut,
143 {{"\"name\": \"BM_Counters_Tabular/repeats:2/threads:1_median\",$"},
144 {"\"family_index\": 0,$", MR_Next},
145 {"\"per_family_instance_index\": 0,$", MR_Next},
146 {"\"run_name\": \"BM_Counters_Tabular/repeats:2/threads:1\",$",
147 MR_Next},
148 {"\"run_type\": \"aggregate\",$", MR_Next},
149 {"\"repetitions\": 2,$", MR_Next},
150 {"\"threads\": 1,$", MR_Next},
151 {"\"aggregate_name\": \"median\",$", MR_Next},
152 {"\"aggregate_unit\": \"time\",$", MR_Next},
153 {"\"iterations\": %int,$", MR_Next},
154 {"\"real_time\": %float,$", MR_Next},
155 {"\"cpu_time\": %float,$", MR_Next},
156 {"\"time_unit\": \"ns\",$", MR_Next},
157 {"\"Bar\": %float,$", MR_Next},
158 {"\"Bat\": %float,$", MR_Next},
159 {"\"Baz\": %float,$", MR_Next},
160 {"\"Foo\": %float,$", MR_Next},
161 {"\"Frob\": %float,$", MR_Next},
162 {"\"Lob\": %float$", MR_Next},
163 {"}", MR_Next}});
164 ADD_CASES(TC_JSONOut,
165 {{"\"name\": \"BM_Counters_Tabular/repeats:2/threads:1_stddev\",$"},
166 {"\"family_index\": 0,$", MR_Next},
167 {"\"per_family_instance_index\": 0,$", MR_Next},
168 {"\"run_name\": \"BM_Counters_Tabular/repeats:2/threads:1\",$",
169 MR_Next},
170 {"\"run_type\": \"aggregate\",$", MR_Next},
171 {"\"repetitions\": 2,$", MR_Next},
172 {"\"threads\": 1,$", MR_Next},
173 {"\"aggregate_name\": \"stddev\",$", MR_Next},
174 {"\"aggregate_unit\": \"time\",$", MR_Next},
175 {"\"iterations\": %int,$", MR_Next},
176 {"\"real_time\": %float,$", MR_Next},
177 {"\"cpu_time\": %float,$", MR_Next},
178 {"\"time_unit\": \"ns\",$", MR_Next},
179 {"\"Bar\": %float,$", MR_Next},
180 {"\"Bat\": %float,$", MR_Next},
181 {"\"Baz\": %float,$", MR_Next},
182 {"\"Foo\": %float,$", MR_Next},
183 {"\"Frob\": %float,$", MR_Next},
184 {"\"Lob\": %float$", MR_Next},
185 {"}", MR_Next}});
186 ADD_CASES(TC_JSONOut,
187 {{"\"name\": \"BM_Counters_Tabular/repeats:2/threads:1_cv\",$"},
188 {"\"family_index\": 0,$", MR_Next},
189 {"\"per_family_instance_index\": 0,$", MR_Next},
190 {"\"run_name\": \"BM_Counters_Tabular/repeats:2/threads:1\",$",
191 MR_Next},
192 {"\"run_type\": \"aggregate\",$", MR_Next},
193 {"\"repetitions\": 2,$", MR_Next},
194 {"\"threads\": 1,$", MR_Next},
195 {"\"aggregate_name\": \"cv\",$", MR_Next},
196 {"\"aggregate_unit\": \"percentage\",$", MR_Next},
197 {"\"iterations\": %int,$", MR_Next},
198 {"\"real_time\": %float,$", MR_Next},
199 {"\"cpu_time\": %float,$", MR_Next},
200 {"\"time_unit\": \"ns\",$", MR_Next},
201 {"\"Bar\": %float,$", MR_Next},
202 {"\"Bat\": %float,$", MR_Next},
203 {"\"Baz\": %float,$", MR_Next},
204 {"\"Foo\": %float,$", MR_Next},
205 {"\"Frob\": %float,$", MR_Next},
206 {"\"Lob\": %float$", MR_Next},
207 {"}", MR_Next}});
208
209 ADD_CASES(TC_JSONOut,
210 {{"\"name\": \"BM_Counters_Tabular/repeats:2/threads:2\",$"},
211 {"\"family_index\": 0,$", MR_Next},
212 {"\"per_family_instance_index\": 1,$", MR_Next},
213 {"\"run_name\": \"BM_Counters_Tabular/repeats:2/threads:2\",$",
214 MR_Next},
215 {"\"run_type\": \"iteration\",$", MR_Next},
216 {"\"repetitions\": 2,$", MR_Next},
217 {"\"repetition_index\": 0,$", MR_Next},
218 {"\"threads\": 2,$", MR_Next},
219 {"\"iterations\": %int,$", MR_Next},
220 {"\"real_time\": %float,$", MR_Next},
221 {"\"cpu_time\": %float,$", MR_Next},
222 {"\"time_unit\": \"ns\",$", MR_Next},
223 {"\"Bar\": %float,$", MR_Next},
224 {"\"Bat\": %float,$", MR_Next},
225 {"\"Baz\": %float,$", MR_Next},
226 {"\"Foo\": %float,$", MR_Next},
227 {"\"Frob\": %float,$", MR_Next},
228 {"\"Lob\": %float$", MR_Next},
229 {"}", MR_Next}});
230 ADD_CASES(TC_JSONOut,
231 {{"\"name\": \"BM_Counters_Tabular/repeats:2/threads:2\",$"},
232 {"\"family_index\": 0,$", MR_Next},
233 {"\"per_family_instance_index\": 1,$", MR_Next},
234 {"\"run_name\": \"BM_Counters_Tabular/repeats:2/threads:2\",$",
235 MR_Next},
236 {"\"run_type\": \"iteration\",$", MR_Next},
237 {"\"repetitions\": 2,$", MR_Next},
238 {"\"repetition_index\": 1,$", MR_Next},
239 {"\"threads\": 2,$", MR_Next},
240 {"\"iterations\": %int,$", MR_Next},
241 {"\"real_time\": %float,$", MR_Next},
242 {"\"cpu_time\": %float,$", MR_Next},
243 {"\"time_unit\": \"ns\",$", MR_Next},
244 {"\"Bar\": %float,$", MR_Next},
245 {"\"Bat\": %float,$", MR_Next},
246 {"\"Baz\": %float,$", MR_Next},
247 {"\"Foo\": %float,$", MR_Next},
248 {"\"Frob\": %float,$", MR_Next},
249 {"\"Lob\": %float$", MR_Next},
250 {"}", MR_Next}});
251 ADD_CASES(TC_JSONOut,
252 {{"\"name\": \"BM_Counters_Tabular/repeats:2/threads:2_median\",$"},
253 {"\"family_index\": 0,$", MR_Next},
254 {"\"per_family_instance_index\": 1,$", MR_Next},
255 {"\"run_name\": \"BM_Counters_Tabular/repeats:2/threads:2\",$",
256 MR_Next},
257 {"\"run_type\": \"aggregate\",$", MR_Next},
258 {"\"repetitions\": 2,$", MR_Next},
259 {"\"threads\": 2,$", MR_Next},
260 {"\"aggregate_name\": \"median\",$", MR_Next},
261 {"\"aggregate_unit\": \"time\",$", MR_Next},
262 {"\"iterations\": %int,$", MR_Next},
263 {"\"real_time\": %float,$", MR_Next},
264 {"\"cpu_time\": %float,$", MR_Next},
265 {"\"time_unit\": \"ns\",$", MR_Next},
266 {"\"Bar\": %float,$", MR_Next},
267 {"\"Bat\": %float,$", MR_Next},
268 {"\"Baz\": %float,$", MR_Next},
269 {"\"Foo\": %float,$", MR_Next},
270 {"\"Frob\": %float,$", MR_Next},
271 {"\"Lob\": %float$", MR_Next},
272 {"}", MR_Next}});
273 ADD_CASES(TC_JSONOut,
274 {{"\"name\": \"BM_Counters_Tabular/repeats:2/threads:2_stddev\",$"},
275 {"\"family_index\": 0,$", MR_Next},
276 {"\"per_family_instance_index\": 1,$", MR_Next},
277 {"\"run_name\": \"BM_Counters_Tabular/repeats:2/threads:2\",$",
278 MR_Next},
279 {"\"run_type\": \"aggregate\",$", MR_Next},
280 {"\"repetitions\": 2,$", MR_Next},
281 {"\"threads\": 2,$", MR_Next},
282 {"\"aggregate_name\": \"stddev\",$", MR_Next},
283 {"\"aggregate_unit\": \"time\",$", MR_Next},
284 {"\"iterations\": %int,$", MR_Next},
285 {"\"real_time\": %float,$", MR_Next},
286 {"\"cpu_time\": %float,$", MR_Next},
287 {"\"time_unit\": \"ns\",$", MR_Next},
288 {"\"Bar\": %float,$", MR_Next},
289 {"\"Bat\": %float,$", MR_Next},
290 {"\"Baz\": %float,$", MR_Next},
291 {"\"Foo\": %float,$", MR_Next},
292 {"\"Frob\": %float,$", MR_Next},
293 {"\"Lob\": %float$", MR_Next},
294 {"}", MR_Next}});
295 ADD_CASES(TC_JSONOut,
296 {{"\"name\": \"BM_Counters_Tabular/repeats:2/threads:2_cv\",$"},
297 {"\"family_index\": 0,$", MR_Next},
298 {"\"per_family_instance_index\": 1,$", MR_Next},
299 {"\"run_name\": \"BM_Counters_Tabular/repeats:2/threads:2\",$",
300 MR_Next},
301 {"\"run_type\": \"aggregate\",$", MR_Next},
302 {"\"repetitions\": 2,$", MR_Next},
303 {"\"threads\": 2,$", MR_Next},
304 {"\"aggregate_name\": \"cv\",$", MR_Next},
305 {"\"aggregate_unit\": \"percentage\",$", MR_Next},
306 {"\"iterations\": %int,$", MR_Next},
307 {"\"real_time\": %float,$", MR_Next},
308 {"\"cpu_time\": %float,$", MR_Next},
309 {"\"time_unit\": \"ns\",$", MR_Next},
310 {"\"Bar\": %float,$", MR_Next},
311 {"\"Bat\": %float,$", MR_Next},
312 {"\"Baz\": %float,$", MR_Next},
313 {"\"Foo\": %float,$", MR_Next},
314 {"\"Frob\": %float,$", MR_Next},
315 {"\"Lob\": %float$", MR_Next},
316 {"}", MR_Next}});
317 ADD_CASES(TC_CSVOut,
318 {{"^\"BM_Counters_Tabular/repeats:2/threads:1\",%csv_report,"
319 "%float,%float,%float,%float,%float,%float$"}});
320 ADD_CASES(TC_CSVOut,
321 {{"^\"BM_Counters_Tabular/repeats:2/threads:1\",%csv_report,"
322 "%float,%float,%float,%float,%float,%float$"}});
323 ADD_CASES(TC_CSVOut,
324 {{"^\"BM_Counters_Tabular/repeats:2/threads:1_mean\",%csv_report,"
325 "%float,%float,%float,%float,%float,%float$"}});
326 ADD_CASES(TC_CSVOut,
327 {{"^\"BM_Counters_Tabular/repeats:2/threads:1_median\",%csv_report,"
328 "%float,%float,%float,%float,%float,%float$"}});
329 ADD_CASES(TC_CSVOut,
330 {{"^\"BM_Counters_Tabular/repeats:2/threads:1_stddev\",%csv_report,"
331 "%float,%float,%float,%float,%float,%float$"}});
332 ADD_CASES(TC_CSVOut,
333 {{"^\"BM_Counters_Tabular/repeats:2/threads:1_cv\",%csv_report,"
334 "%float,%float,%float,%float,%float,%float$"}});
335 ADD_CASES(TC_CSVOut,
336 {{"^\"BM_Counters_Tabular/repeats:2/threads:2\",%csv_report,"
337 "%float,%float,%float,%float,%float,%float$"}});
338 ADD_CASES(TC_CSVOut,
339 {{"^\"BM_Counters_Tabular/repeats:2/threads:2\",%csv_report,"
340 "%float,%float,%float,%float,%float,%float$"}});
341 ADD_CASES(TC_CSVOut,
342 {{"^\"BM_Counters_Tabular/repeats:2/threads:2_mean\",%csv_report,"
343 "%float,%float,%float,%float,%float,%float$"}});
344 ADD_CASES(TC_CSVOut,
345 {{"^\"BM_Counters_Tabular/repeats:2/threads:2_median\",%csv_report,"
346 "%float,%float,%float,%float,%float,%float$"}});
347 ADD_CASES(TC_CSVOut,
348 {{"^\"BM_Counters_Tabular/repeats:2/threads:2_stddev\",%csv_report,"
349 "%float,%float,%float,%float,%float,%float$"}});
350 ADD_CASES(TC_CSVOut,
351 {{"^\"BM_Counters_Tabular/repeats:2/threads:2_cv\",%csv_report,"
352 "%float,%float,%float,%float,%float,%float$"}});
353 // VS2013 does not allow this function to be passed as a lambda argument
354 // to CHECK_BENCHMARK_RESULTS()
CheckTabular(Results const & e)355 void CheckTabular(Results const& e) {
356 CHECK_COUNTER_VALUE(e, int, "Foo", EQ, 1);
357 CHECK_COUNTER_VALUE(e, int, "Bar", EQ, 2);
358 CHECK_COUNTER_VALUE(e, int, "Baz", EQ, 4);
359 CHECK_COUNTER_VALUE(e, int, "Bat", EQ, 8);
360 CHECK_COUNTER_VALUE(e, int, "Frob", EQ, 16);
361 CHECK_COUNTER_VALUE(e, int, "Lob", EQ, 32);
362 }
363 CHECK_BENCHMARK_RESULTS("BM_Counters_Tabular/repeats:2/threads:1$",
364 &CheckTabular);
365 CHECK_BENCHMARK_RESULTS("BM_Counters_Tabular/repeats:2/threads:2$",
366 &CheckTabular);
367
368 // ========================================================================= //
369 // -------------------- Tabular+Rate Counters Output ----------------------- //
370 // ========================================================================= //
371
BM_CounterRates_Tabular(benchmark::State & state)372 void BM_CounterRates_Tabular(benchmark::State& state) {
373 for (auto _ : state) {
374 // This test requires a non-zero CPU time to avoid divide-by-zero
375 benchmark::DoNotOptimize(state.iterations());
376 }
377 namespace bm = benchmark;
378 state.counters.insert({
379 {"Foo", {1, bm::Counter::kAvgThreadsRate}},
380 {"Bar", {2, bm::Counter::kAvgThreadsRate}},
381 {"Baz", {4, bm::Counter::kAvgThreadsRate}},
382 {"Bat", {8, bm::Counter::kAvgThreadsRate}},
383 {"Frob", {16, bm::Counter::kAvgThreadsRate}},
384 {"Lob", {32, bm::Counter::kAvgThreadsRate}},
385 });
386 }
387 BENCHMARK(BM_CounterRates_Tabular)->ThreadRange(1, 16);
388 ADD_CASES(TC_JSONOut,
389 {{"\"name\": \"BM_CounterRates_Tabular/threads:%int\",$"},
390 {"\"family_index\": 1,$", MR_Next},
391 {"\"per_family_instance_index\": 0,$", MR_Next},
392 {"\"run_name\": \"BM_CounterRates_Tabular/threads:%int\",$",
393 MR_Next},
394 {"\"run_type\": \"iteration\",$", MR_Next},
395 {"\"repetitions\": 1,$", MR_Next},
396 {"\"repetition_index\": 0,$", MR_Next},
397 {"\"threads\": 1,$", MR_Next},
398 {"\"iterations\": %int,$", MR_Next},
399 {"\"real_time\": %float,$", MR_Next},
400 {"\"cpu_time\": %float,$", MR_Next},
401 {"\"time_unit\": \"ns\",$", MR_Next},
402 {"\"Bar\": %float,$", MR_Next},
403 {"\"Bat\": %float,$", MR_Next},
404 {"\"Baz\": %float,$", MR_Next},
405 {"\"Foo\": %float,$", MR_Next},
406 {"\"Frob\": %float,$", MR_Next},
407 {"\"Lob\": %float$", MR_Next},
408 {"}", MR_Next}});
409 ADD_CASES(TC_CSVOut, {{"^\"BM_CounterRates_Tabular/threads:%int\",%csv_report,"
410 "%float,%float,%float,%float,%float,%float$"}});
411 // VS2013 does not allow this function to be passed as a lambda argument
412 // to CHECK_BENCHMARK_RESULTS()
CheckTabularRate(Results const & e)413 void CheckTabularRate(Results const& e) {
414 double t = e.DurationCPUTime();
415 CHECK_FLOAT_COUNTER_VALUE(e, "Foo", EQ, 1. / t, 0.001);
416 CHECK_FLOAT_COUNTER_VALUE(e, "Bar", EQ, 2. / t, 0.001);
417 CHECK_FLOAT_COUNTER_VALUE(e, "Baz", EQ, 4. / t, 0.001);
418 CHECK_FLOAT_COUNTER_VALUE(e, "Bat", EQ, 8. / t, 0.001);
419 CHECK_FLOAT_COUNTER_VALUE(e, "Frob", EQ, 16. / t, 0.001);
420 CHECK_FLOAT_COUNTER_VALUE(e, "Lob", EQ, 32. / t, 0.001);
421 }
422 CHECK_BENCHMARK_RESULTS("BM_CounterRates_Tabular/threads:%int",
423 &CheckTabularRate);
424
425 // ========================================================================= //
426 // ------------------------- Tabular Counters Output ----------------------- //
427 // ========================================================================= //
428
429 // set only some of the counters
BM_CounterSet0_Tabular(benchmark::State & state)430 void BM_CounterSet0_Tabular(benchmark::State& state) {
431 for (auto _ : state) {
432 }
433 namespace bm = benchmark;
434 state.counters.insert({
435 {"Foo", {10, bm::Counter::kAvgThreads}},
436 {"Bar", {20, bm::Counter::kAvgThreads}},
437 {"Baz", {40, bm::Counter::kAvgThreads}},
438 });
439 }
440 BENCHMARK(BM_CounterSet0_Tabular)->ThreadRange(1, 16);
441 ADD_CASES(TC_JSONOut,
442 {{"\"name\": \"BM_CounterSet0_Tabular/threads:%int\",$"},
443 {"\"family_index\": 2,$", MR_Next},
444 {"\"per_family_instance_index\": 0,$", MR_Next},
445 {"\"run_name\": \"BM_CounterSet0_Tabular/threads:%int\",$", MR_Next},
446 {"\"run_type\": \"iteration\",$", MR_Next},
447 {"\"repetitions\": 1,$", MR_Next},
448 {"\"repetition_index\": 0,$", MR_Next},
449 {"\"threads\": 1,$", MR_Next},
450 {"\"iterations\": %int,$", MR_Next},
451 {"\"real_time\": %float,$", MR_Next},
452 {"\"cpu_time\": %float,$", MR_Next},
453 {"\"time_unit\": \"ns\",$", MR_Next},
454 {"\"Bar\": %float,$", MR_Next},
455 {"\"Baz\": %float,$", MR_Next},
456 {"\"Foo\": %float$", MR_Next},
457 {"}", MR_Next}});
458 ADD_CASES(TC_CSVOut, {{"^\"BM_CounterSet0_Tabular/threads:%int\",%csv_report,"
459 "%float,,%float,%float,,"}});
460 // VS2013 does not allow this function to be passed as a lambda argument
461 // to CHECK_BENCHMARK_RESULTS()
CheckSet0(Results const & e)462 void CheckSet0(Results const& e) {
463 CHECK_COUNTER_VALUE(e, int, "Foo", EQ, 10);
464 CHECK_COUNTER_VALUE(e, int, "Bar", EQ, 20);
465 CHECK_COUNTER_VALUE(e, int, "Baz", EQ, 40);
466 }
467 CHECK_BENCHMARK_RESULTS("BM_CounterSet0_Tabular", &CheckSet0);
468
469 // again.
BM_CounterSet1_Tabular(benchmark::State & state)470 void BM_CounterSet1_Tabular(benchmark::State& state) {
471 for (auto _ : state) {
472 }
473 namespace bm = benchmark;
474 state.counters.insert({
475 {"Foo", {15, bm::Counter::kAvgThreads}},
476 {"Bar", {25, bm::Counter::kAvgThreads}},
477 {"Baz", {45, bm::Counter::kAvgThreads}},
478 });
479 }
480 BENCHMARK(BM_CounterSet1_Tabular)->ThreadRange(1, 16);
481 ADD_CASES(TC_JSONOut,
482 {{"\"name\": \"BM_CounterSet1_Tabular/threads:%int\",$"},
483 {"\"family_index\": 3,$", MR_Next},
484 {"\"per_family_instance_index\": 0,$", MR_Next},
485 {"\"run_name\": \"BM_CounterSet1_Tabular/threads:%int\",$", MR_Next},
486 {"\"run_type\": \"iteration\",$", MR_Next},
487 {"\"repetitions\": 1,$", MR_Next},
488 {"\"repetition_index\": 0,$", MR_Next},
489 {"\"threads\": 1,$", MR_Next},
490 {"\"iterations\": %int,$", MR_Next},
491 {"\"real_time\": %float,$", MR_Next},
492 {"\"cpu_time\": %float,$", MR_Next},
493 {"\"time_unit\": \"ns\",$", MR_Next},
494 {"\"Bar\": %float,$", MR_Next},
495 {"\"Baz\": %float,$", MR_Next},
496 {"\"Foo\": %float$", MR_Next},
497 {"}", MR_Next}});
498 ADD_CASES(TC_CSVOut, {{"^\"BM_CounterSet1_Tabular/threads:%int\",%csv_report,"
499 "%float,,%float,%float,,"}});
500 // VS2013 does not allow this function to be passed as a lambda argument
501 // to CHECK_BENCHMARK_RESULTS()
CheckSet1(Results const & e)502 void CheckSet1(Results const& e) {
503 CHECK_COUNTER_VALUE(e, int, "Foo", EQ, 15);
504 CHECK_COUNTER_VALUE(e, int, "Bar", EQ, 25);
505 CHECK_COUNTER_VALUE(e, int, "Baz", EQ, 45);
506 }
507 CHECK_BENCHMARK_RESULTS("BM_CounterSet1_Tabular/threads:%int", &CheckSet1);
508
509 // ========================================================================= //
510 // ------------------------- Tabular Counters Output ----------------------- //
511 // ========================================================================= //
512
513 // set only some of the counters, different set now.
BM_CounterSet2_Tabular(benchmark::State & state)514 void BM_CounterSet2_Tabular(benchmark::State& state) {
515 for (auto _ : state) {
516 }
517 namespace bm = benchmark;
518 state.counters.insert({
519 {"Foo", {10, bm::Counter::kAvgThreads}},
520 {"Bat", {30, bm::Counter::kAvgThreads}},
521 {"Baz", {40, bm::Counter::kAvgThreads}},
522 });
523 }
524 BENCHMARK(BM_CounterSet2_Tabular)->ThreadRange(1, 16);
525 ADD_CASES(TC_JSONOut,
526 {{"\"name\": \"BM_CounterSet2_Tabular/threads:%int\",$"},
527 {"\"family_index\": 4,$", MR_Next},
528 {"\"per_family_instance_index\": 0,$", MR_Next},
529 {"\"run_name\": \"BM_CounterSet2_Tabular/threads:%int\",$", MR_Next},
530 {"\"run_type\": \"iteration\",$", MR_Next},
531 {"\"repetitions\": 1,$", MR_Next},
532 {"\"repetition_index\": 0,$", MR_Next},
533 {"\"threads\": 1,$", MR_Next},
534 {"\"iterations\": %int,$", MR_Next},
535 {"\"real_time\": %float,$", MR_Next},
536 {"\"cpu_time\": %float,$", MR_Next},
537 {"\"time_unit\": \"ns\",$", MR_Next},
538 {"\"Bat\": %float,$", MR_Next},
539 {"\"Baz\": %float,$", MR_Next},
540 {"\"Foo\": %float$", MR_Next},
541 {"}", MR_Next}});
542 ADD_CASES(TC_CSVOut, {{"^\"BM_CounterSet2_Tabular/threads:%int\",%csv_report,"
543 ",%float,%float,%float,,"}});
544 // VS2013 does not allow this function to be passed as a lambda argument
545 // to CHECK_BENCHMARK_RESULTS()
CheckSet2(Results const & e)546 void CheckSet2(Results const& e) {
547 CHECK_COUNTER_VALUE(e, int, "Foo", EQ, 10);
548 CHECK_COUNTER_VALUE(e, int, "Bat", EQ, 30);
549 CHECK_COUNTER_VALUE(e, int, "Baz", EQ, 40);
550 }
551 CHECK_BENCHMARK_RESULTS("BM_CounterSet2_Tabular", &CheckSet2);
552
553 // ========================================================================= //
554 // --------------------------- TEST CASES END ------------------------------ //
555 // ========================================================================= //
556
main(int argc,char * argv[])557 int main(int argc, char* argv[]) { RunOutputTests(argc, argv); }
558