1
2 #undef NDEBUG
3
4 #include "benchmark/benchmark.h"
5 #include "output_test.h"
6
7 // @todo: <jpmag> this checks the full output at once; the rule for
8 // CounterSet1 was failing because it was not matching "^[-]+$".
9 // @todo: <jpmag> check that the counters are vertically aligned.
10 ADD_CASES(
11 TC_ConsoleOut,
12 {
13 // keeping these lines long improves readability, so:
14 // clang-format off
15 {"^[-]+$", MR_Next},
16 {"^Benchmark %s Time %s CPU %s Iterations %s Bar %s Bat %s Baz %s Foo %s Frob %s Lob$", MR_Next},
17 {"^[-]+$", MR_Next},
18 {"^BM_Counters_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
19 {"^BM_Counters_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
20 {"^BM_Counters_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
21 {"^BM_Counters_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
22 {"^BM_Counters_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
23 {"^BM_CounterRates_Tabular/threads:%int %console_report [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s$", MR_Next},
24 {"^BM_CounterRates_Tabular/threads:%int %console_report [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s$", MR_Next},
25 {"^BM_CounterRates_Tabular/threads:%int %console_report [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s$", MR_Next},
26 {"^BM_CounterRates_Tabular/threads:%int %console_report [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s$", MR_Next},
27 {"^BM_CounterRates_Tabular/threads:%int %console_report [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s$", MR_Next},
28 {"^[-]+$", MR_Next},
29 {"^Benchmark %s Time %s CPU %s Iterations %s Bar %s Baz %s Foo$", MR_Next},
30 {"^[-]+$", MR_Next},
31 {"^BM_CounterSet0_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
32 {"^BM_CounterSet0_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
33 {"^BM_CounterSet0_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
34 {"^BM_CounterSet0_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
35 {"^BM_CounterSet0_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
36 {"^BM_CounterSet1_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
37 {"^BM_CounterSet1_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
38 {"^BM_CounterSet1_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
39 {"^BM_CounterSet1_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
40 {"^BM_CounterSet1_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
41 {"^[-]+$", MR_Next},
42 {"^Benchmark %s Time %s CPU %s Iterations %s Bat %s Baz %s Foo$", MR_Next},
43 {"^[-]+$", MR_Next},
44 {"^BM_CounterSet2_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
45 {"^BM_CounterSet2_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
46 {"^BM_CounterSet2_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
47 {"^BM_CounterSet2_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
48 {"^BM_CounterSet2_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$"},
49 // clang-format on
50 });
51 ADD_CASES(TC_CSVOut, {{"%csv_header,"
52 "\"Bar\",\"Bat\",\"Baz\",\"Foo\",\"Frob\",\"Lob\""}});
53
54 // ========================================================================= //
55 // ------------------------- Tabular Counters Output ----------------------- //
56 // ========================================================================= //
57
BM_Counters_Tabular(benchmark::State & state)58 void BM_Counters_Tabular(benchmark::State& state) {
59 for (auto _ : state) {
60 }
61 namespace bm = benchmark;
62 state.counters.insert({
63 {"Foo", {1, bm::Counter::kAvgThreads}},
64 {"Bar", {2, bm::Counter::kAvgThreads}},
65 {"Baz", {4, bm::Counter::kAvgThreads}},
66 {"Bat", {8, bm::Counter::kAvgThreads}},
67 {"Frob", {16, bm::Counter::kAvgThreads}},
68 {"Lob", {32, bm::Counter::kAvgThreads}},
69 });
70 }
71 BENCHMARK(BM_Counters_Tabular)->ThreadRange(1, 16);
72 ADD_CASES(TC_JSONOut,
73 {{"\"name\": \"BM_Counters_Tabular/threads:%int\",$"},
74 {"\"run_name\": \"BM_Counters_Tabular/threads:%int\",$", MR_Next},
75 {"\"run_type\": \"iteration\",$", MR_Next},
76 {"\"iterations\": %int,$", MR_Next},
77 {"\"real_time\": %float,$", MR_Next},
78 {"\"cpu_time\": %float,$", MR_Next},
79 {"\"time_unit\": \"ns\",$", MR_Next},
80 {"\"Bar\": %float,$", MR_Next},
81 {"\"Bat\": %float,$", MR_Next},
82 {"\"Baz\": %float,$", MR_Next},
83 {"\"Foo\": %float,$", MR_Next},
84 {"\"Frob\": %float,$", MR_Next},
85 {"\"Lob\": %float$", MR_Next},
86 {"}", MR_Next}});
87 ADD_CASES(TC_CSVOut, {{"^\"BM_Counters_Tabular/threads:%int\",%csv_report,"
88 "%float,%float,%float,%float,%float,%float$"}});
89 // VS2013 does not allow this function to be passed as a lambda argument
90 // to CHECK_BENCHMARK_RESULTS()
CheckTabular(Results const & e)91 void CheckTabular(Results const& e) {
92 CHECK_COUNTER_VALUE(e, int, "Foo", EQ, 1);
93 CHECK_COUNTER_VALUE(e, int, "Bar", EQ, 2);
94 CHECK_COUNTER_VALUE(e, int, "Baz", EQ, 4);
95 CHECK_COUNTER_VALUE(e, int, "Bat", EQ, 8);
96 CHECK_COUNTER_VALUE(e, int, "Frob", EQ, 16);
97 CHECK_COUNTER_VALUE(e, int, "Lob", EQ, 32);
98 }
99 CHECK_BENCHMARK_RESULTS("BM_Counters_Tabular/threads:%int", &CheckTabular);
100
101 // ========================================================================= //
102 // -------------------- Tabular+Rate Counters Output ----------------------- //
103 // ========================================================================= //
104
BM_CounterRates_Tabular(benchmark::State & state)105 void BM_CounterRates_Tabular(benchmark::State& state) {
106 for (auto _ : state) {
107 }
108 namespace bm = benchmark;
109 state.counters.insert({
110 {"Foo", {1, bm::Counter::kAvgThreadsRate}},
111 {"Bar", {2, bm::Counter::kAvgThreadsRate}},
112 {"Baz", {4, bm::Counter::kAvgThreadsRate}},
113 {"Bat", {8, bm::Counter::kAvgThreadsRate}},
114 {"Frob", {16, bm::Counter::kAvgThreadsRate}},
115 {"Lob", {32, bm::Counter::kAvgThreadsRate}},
116 });
117 }
118 BENCHMARK(BM_CounterRates_Tabular)->ThreadRange(1, 16);
119 ADD_CASES(TC_JSONOut,
120 {{"\"name\": \"BM_CounterRates_Tabular/threads:%int\",$"},
121 {"\"run_name\": \"BM_CounterRates_Tabular/threads:%int\",$",
122 MR_Next},
123 {"\"run_type\": \"iteration\",$", MR_Next},
124 {"\"iterations\": %int,$", MR_Next},
125 {"\"real_time\": %float,$", MR_Next},
126 {"\"cpu_time\": %float,$", MR_Next},
127 {"\"time_unit\": \"ns\",$", MR_Next},
128 {"\"Bar\": %float,$", MR_Next},
129 {"\"Bat\": %float,$", MR_Next},
130 {"\"Baz\": %float,$", MR_Next},
131 {"\"Foo\": %float,$", MR_Next},
132 {"\"Frob\": %float,$", MR_Next},
133 {"\"Lob\": %float$", MR_Next},
134 {"}", MR_Next}});
135 ADD_CASES(TC_CSVOut, {{"^\"BM_CounterRates_Tabular/threads:%int\",%csv_report,"
136 "%float,%float,%float,%float,%float,%float$"}});
137 // VS2013 does not allow this function to be passed as a lambda argument
138 // to CHECK_BENCHMARK_RESULTS()
CheckTabularRate(Results const & e)139 void CheckTabularRate(Results const& e) {
140 double t = e.DurationCPUTime();
141 CHECK_FLOAT_COUNTER_VALUE(e, "Foo", EQ, 1. / t, 0.001);
142 CHECK_FLOAT_COUNTER_VALUE(e, "Bar", EQ, 2. / t, 0.001);
143 CHECK_FLOAT_COUNTER_VALUE(e, "Baz", EQ, 4. / t, 0.001);
144 CHECK_FLOAT_COUNTER_VALUE(e, "Bat", EQ, 8. / t, 0.001);
145 CHECK_FLOAT_COUNTER_VALUE(e, "Frob", EQ, 16. / t, 0.001);
146 CHECK_FLOAT_COUNTER_VALUE(e, "Lob", EQ, 32. / t, 0.001);
147 }
148 CHECK_BENCHMARK_RESULTS("BM_CounterRates_Tabular/threads:%int",
149 &CheckTabularRate);
150
151 // ========================================================================= //
152 // ------------------------- Tabular Counters Output ----------------------- //
153 // ========================================================================= //
154
155 // set only some of the counters
BM_CounterSet0_Tabular(benchmark::State & state)156 void BM_CounterSet0_Tabular(benchmark::State& state) {
157 for (auto _ : state) {
158 }
159 namespace bm = benchmark;
160 state.counters.insert({
161 {"Foo", {10, bm::Counter::kAvgThreads}},
162 {"Bar", {20, bm::Counter::kAvgThreads}},
163 {"Baz", {40, bm::Counter::kAvgThreads}},
164 });
165 }
166 BENCHMARK(BM_CounterSet0_Tabular)->ThreadRange(1, 16);
167 ADD_CASES(TC_JSONOut,
168 {{"\"name\": \"BM_CounterSet0_Tabular/threads:%int\",$"},
169 {"\"run_name\": \"BM_CounterSet0_Tabular/threads:%int\",$", MR_Next},
170 {"\"run_type\": \"iteration\",$", MR_Next},
171 {"\"iterations\": %int,$", MR_Next},
172 {"\"real_time\": %float,$", MR_Next},
173 {"\"cpu_time\": %float,$", MR_Next},
174 {"\"time_unit\": \"ns\",$", MR_Next},
175 {"\"Bar\": %float,$", MR_Next},
176 {"\"Baz\": %float,$", MR_Next},
177 {"\"Foo\": %float$", MR_Next},
178 {"}", MR_Next}});
179 ADD_CASES(TC_CSVOut, {{"^\"BM_CounterSet0_Tabular/threads:%int\",%csv_report,"
180 "%float,,%float,%float,,"}});
181 // VS2013 does not allow this function to be passed as a lambda argument
182 // to CHECK_BENCHMARK_RESULTS()
CheckSet0(Results const & e)183 void CheckSet0(Results const& e) {
184 CHECK_COUNTER_VALUE(e, int, "Foo", EQ, 10);
185 CHECK_COUNTER_VALUE(e, int, "Bar", EQ, 20);
186 CHECK_COUNTER_VALUE(e, int, "Baz", EQ, 40);
187 }
188 CHECK_BENCHMARK_RESULTS("BM_CounterSet0_Tabular", &CheckSet0);
189
190 // again.
BM_CounterSet1_Tabular(benchmark::State & state)191 void BM_CounterSet1_Tabular(benchmark::State& state) {
192 for (auto _ : state) {
193 }
194 namespace bm = benchmark;
195 state.counters.insert({
196 {"Foo", {15, bm::Counter::kAvgThreads}},
197 {"Bar", {25, bm::Counter::kAvgThreads}},
198 {"Baz", {45, bm::Counter::kAvgThreads}},
199 });
200 }
201 BENCHMARK(BM_CounterSet1_Tabular)->ThreadRange(1, 16);
202 ADD_CASES(TC_JSONOut,
203 {{"\"name\": \"BM_CounterSet1_Tabular/threads:%int\",$"},
204 {"\"run_name\": \"BM_CounterSet1_Tabular/threads:%int\",$", MR_Next},
205 {"\"run_type\": \"iteration\",$", MR_Next},
206 {"\"iterations\": %int,$", MR_Next},
207 {"\"real_time\": %float,$", MR_Next},
208 {"\"cpu_time\": %float,$", MR_Next},
209 {"\"time_unit\": \"ns\",$", MR_Next},
210 {"\"Bar\": %float,$", MR_Next},
211 {"\"Baz\": %float,$", MR_Next},
212 {"\"Foo\": %float$", MR_Next},
213 {"}", MR_Next}});
214 ADD_CASES(TC_CSVOut, {{"^\"BM_CounterSet1_Tabular/threads:%int\",%csv_report,"
215 "%float,,%float,%float,,"}});
216 // VS2013 does not allow this function to be passed as a lambda argument
217 // to CHECK_BENCHMARK_RESULTS()
CheckSet1(Results const & e)218 void CheckSet1(Results const& e) {
219 CHECK_COUNTER_VALUE(e, int, "Foo", EQ, 15);
220 CHECK_COUNTER_VALUE(e, int, "Bar", EQ, 25);
221 CHECK_COUNTER_VALUE(e, int, "Baz", EQ, 45);
222 }
223 CHECK_BENCHMARK_RESULTS("BM_CounterSet1_Tabular/threads:%int", &CheckSet1);
224
225 // ========================================================================= //
226 // ------------------------- Tabular Counters Output ----------------------- //
227 // ========================================================================= //
228
229 // set only some of the counters, different set now.
BM_CounterSet2_Tabular(benchmark::State & state)230 void BM_CounterSet2_Tabular(benchmark::State& state) {
231 for (auto _ : state) {
232 }
233 namespace bm = benchmark;
234 state.counters.insert({
235 {"Foo", {10, bm::Counter::kAvgThreads}},
236 {"Bat", {30, bm::Counter::kAvgThreads}},
237 {"Baz", {40, bm::Counter::kAvgThreads}},
238 });
239 }
240 BENCHMARK(BM_CounterSet2_Tabular)->ThreadRange(1, 16);
241 ADD_CASES(TC_JSONOut,
242 {{"\"name\": \"BM_CounterSet2_Tabular/threads:%int\",$"},
243 {"\"run_name\": \"BM_CounterSet2_Tabular/threads:%int\",$", MR_Next},
244 {"\"run_type\": \"iteration\",$", MR_Next},
245 {"\"iterations\": %int,$", MR_Next},
246 {"\"real_time\": %float,$", MR_Next},
247 {"\"cpu_time\": %float,$", MR_Next},
248 {"\"time_unit\": \"ns\",$", MR_Next},
249 {"\"Bat\": %float,$", MR_Next},
250 {"\"Baz\": %float,$", MR_Next},
251 {"\"Foo\": %float$", MR_Next},
252 {"}", MR_Next}});
253 ADD_CASES(TC_CSVOut, {{"^\"BM_CounterSet2_Tabular/threads:%int\",%csv_report,"
254 ",%float,%float,%float,,"}});
255 // VS2013 does not allow this function to be passed as a lambda argument
256 // to CHECK_BENCHMARK_RESULTS()
CheckSet2(Results const & e)257 void CheckSet2(Results const& e) {
258 CHECK_COUNTER_VALUE(e, int, "Foo", EQ, 10);
259 CHECK_COUNTER_VALUE(e, int, "Bat", EQ, 30);
260 CHECK_COUNTER_VALUE(e, int, "Baz", EQ, 40);
261 }
262 CHECK_BENCHMARK_RESULTS("BM_CounterSet2_Tabular", &CheckSet2);
263
264 // ========================================================================= //
265 // --------------------------- TEST CASES END ------------------------------ //
266 // ========================================================================= //
267
main(int argc,char * argv[])268 int main(int argc, char* argv[]) { RunOutputTests(argc, argv); }
269