1
2 #undef NDEBUG
3 #include <utility>
4
5 #include "benchmark/benchmark.h"
6 #include "output_test.h"
7
8 // ========================================================================= //
9 // ---------------------- Testing Prologue Output -------------------------- //
10 // ========================================================================= //
11
12 ADD_CASES(TC_ConsoleOut,
13 {{"^[-]+$", MR_Next},
14 {"^Benchmark %s Time %s CPU %s Iterations$", MR_Next},
15 {"^[-]+$", MR_Next}});
16 ADD_CASES(TC_CSVOut, {{"%csv_header"}});
17
18 // ========================================================================= //
19 // ------------------------ Testing Basic Output --------------------------- //
20 // ========================================================================= //
21
BM_basic(benchmark::State & state)22 void BM_basic(benchmark::State& state) {
23 while (state.KeepRunning()) {
24 }
25 }
26 BENCHMARK(BM_basic);
27
28 ADD_CASES(TC_ConsoleOut, {{"^BM_basic %console_report$"}});
29 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_basic\",$"},
30 {"\"iterations\": %int,$", MR_Next},
31 {"\"real_time\": %int,$", MR_Next},
32 {"\"cpu_time\": %int,$", MR_Next},
33 {"\"time_unit\": \"ns\"$", MR_Next},
34 {"}", MR_Next}});
35 ADD_CASES(TC_CSVOut, {{"^\"BM_basic\",%csv_report$"}});
36
37 // ========================================================================= //
38 // ------------------------ Testing Bytes per Second Output ---------------- //
39 // ========================================================================= //
40
BM_bytes_per_second(benchmark::State & state)41 void BM_bytes_per_second(benchmark::State& state) {
42 while (state.KeepRunning()) {
43 }
44 state.SetBytesProcessed(1);
45 }
46 BENCHMARK(BM_bytes_per_second);
47
48 ADD_CASES(TC_ConsoleOut,
49 {{"^BM_bytes_per_second %console_report +%floatB/s$"}});
50 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_bytes_per_second\",$"},
51 {"\"iterations\": %int,$", MR_Next},
52 {"\"real_time\": %int,$", MR_Next},
53 {"\"cpu_time\": %int,$", MR_Next},
54 {"\"time_unit\": \"ns\",$", MR_Next},
55 {"\"bytes_per_second\": %int$", MR_Next},
56 {"}", MR_Next}});
57 ADD_CASES(TC_CSVOut, {{"^\"BM_bytes_per_second\",%csv_bytes_report$"}});
58
59 // ========================================================================= //
60 // ------------------------ Testing Items per Second Output ---------------- //
61 // ========================================================================= //
62
BM_items_per_second(benchmark::State & state)63 void BM_items_per_second(benchmark::State& state) {
64 while (state.KeepRunning()) {
65 }
66 state.SetItemsProcessed(1);
67 }
68 BENCHMARK(BM_items_per_second);
69
70 ADD_CASES(TC_ConsoleOut,
71 {{"^BM_items_per_second %console_report +%float items/s$"}});
72 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_items_per_second\",$"},
73 {"\"iterations\": %int,$", MR_Next},
74 {"\"real_time\": %int,$", MR_Next},
75 {"\"cpu_time\": %int,$", MR_Next},
76 {"\"time_unit\": \"ns\",$", MR_Next},
77 {"\"items_per_second\": %int$", MR_Next},
78 {"}", MR_Next}});
79 ADD_CASES(TC_CSVOut, {{"^\"BM_items_per_second\",%csv_items_report$"}});
80
81 // ========================================================================= //
82 // ------------------------ Testing Label Output --------------------------- //
83 // ========================================================================= //
84
BM_label(benchmark::State & state)85 void BM_label(benchmark::State& state) {
86 while (state.KeepRunning()) {
87 }
88 state.SetLabel("some label");
89 }
90 BENCHMARK(BM_label);
91
92 ADD_CASES(TC_ConsoleOut, {{"^BM_label %console_report some label$"}});
93 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_label\",$"},
94 {"\"iterations\": %int,$", MR_Next},
95 {"\"real_time\": %int,$", MR_Next},
96 {"\"cpu_time\": %int,$", MR_Next},
97 {"\"time_unit\": \"ns\",$", MR_Next},
98 {"\"label\": \"some label\"$", MR_Next},
99 {"}", MR_Next}});
100 ADD_CASES(TC_CSVOut, {{"^\"BM_label\",%csv_label_report_begin\"some "
101 "label\"%csv_label_report_end$"}});
102
103 // ========================================================================= //
104 // ------------------------ Testing Error Output --------------------------- //
105 // ========================================================================= //
106
BM_error(benchmark::State & state)107 void BM_error(benchmark::State& state) {
108 state.SkipWithError("message");
109 while (state.KeepRunning()) {
110 }
111 }
112 BENCHMARK(BM_error);
113 ADD_CASES(TC_ConsoleOut, {{"^BM_error[ ]+ERROR OCCURRED: 'message'$"}});
114 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_error\",$"},
115 {"\"error_occurred\": true,$", MR_Next},
116 {"\"error_message\": \"message\",$", MR_Next}});
117
118 ADD_CASES(TC_CSVOut, {{"^\"BM_error\",,,,,,,,true,\"message\"$"}});
119
120 // ========================================================================= //
121 // ------------------------ Testing No Arg Name Output -----------------------
122 // //
123 // ========================================================================= //
124
BM_no_arg_name(benchmark::State & state)125 void BM_no_arg_name(benchmark::State& state) {
126 while (state.KeepRunning()) {
127 }
128 }
129 BENCHMARK(BM_no_arg_name)->Arg(3);
130 ADD_CASES(TC_ConsoleOut, {{"^BM_no_arg_name/3 %console_report$"}});
131 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_no_arg_name/3\",$"}});
132 ADD_CASES(TC_CSVOut, {{"^\"BM_no_arg_name/3\",%csv_report$"}});
133
134 // ========================================================================= //
135 // ------------------------ Testing Arg Name Output ----------------------- //
136 // ========================================================================= //
137
BM_arg_name(benchmark::State & state)138 void BM_arg_name(benchmark::State& state) {
139 while (state.KeepRunning()) {
140 }
141 }
142 BENCHMARK(BM_arg_name)->ArgName("first")->Arg(3);
143 ADD_CASES(TC_ConsoleOut, {{"^BM_arg_name/first:3 %console_report$"}});
144 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_arg_name/first:3\",$"}});
145 ADD_CASES(TC_CSVOut, {{"^\"BM_arg_name/first:3\",%csv_report$"}});
146
147 // ========================================================================= //
148 // ------------------------ Testing Arg Names Output ----------------------- //
149 // ========================================================================= //
150
BM_arg_names(benchmark::State & state)151 void BM_arg_names(benchmark::State& state) {
152 while (state.KeepRunning()) {
153 }
154 }
155 BENCHMARK(BM_arg_names)->Args({2, 5, 4})->ArgNames({"first", "", "third"});
156 ADD_CASES(TC_ConsoleOut,
157 {{"^BM_arg_names/first:2/5/third:4 %console_report$"}});
158 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_arg_names/first:2/5/third:4\",$"}});
159 ADD_CASES(TC_CSVOut, {{"^\"BM_arg_names/first:2/5/third:4\",%csv_report$"}});
160
161 // ========================================================================= //
162 // ----------------------- Testing Complexity Output ----------------------- //
163 // ========================================================================= //
164
BM_Complexity_O1(benchmark::State & state)165 void BM_Complexity_O1(benchmark::State& state) {
166 while (state.KeepRunning()) {
167 }
168 state.SetComplexityN(state.range(0));
169 }
170 BENCHMARK(BM_Complexity_O1)->Range(1, 1 << 18)->Complexity(benchmark::o1);
171 SET_SUBSTITUTIONS({{"%bigOStr", "[ ]* %float \\([0-9]+\\)"},
172 {"%RMS", "[ ]*[0-9]+ %"}});
173 ADD_CASES(TC_ConsoleOut, {{"^BM_Complexity_O1_BigO %bigOStr %bigOStr[ ]*$"},
174 {"^BM_Complexity_O1_RMS %RMS %RMS[ ]*$"}});
175
176 // ========================================================================= //
177 // ----------------------- Testing Aggregate Output ------------------------ //
178 // ========================================================================= //
179
180 // Test that non-aggregate data is printed by default
BM_Repeat(benchmark::State & state)181 void BM_Repeat(benchmark::State& state) {
182 while (state.KeepRunning()) {
183 }
184 }
185 BENCHMARK(BM_Repeat)->Repetitions(3);
186 ADD_CASES(TC_ConsoleOut, {{"^BM_Repeat/repeats:3 %console_report$"},
187 {"^BM_Repeat/repeats:3 %console_report$"},
188 {"^BM_Repeat/repeats:3 %console_report$"},
189 {"^BM_Repeat/repeats:3_mean %console_report$"},
190 {"^BM_Repeat/repeats:3_stddev %console_report$"}});
191 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Repeat/repeats:3\",$"},
192 {"\"name\": \"BM_Repeat/repeats:3\",$"},
193 {"\"name\": \"BM_Repeat/repeats:3\",$"},
194 {"\"name\": \"BM_Repeat/repeats:3_mean\",$"},
195 {"\"name\": \"BM_Repeat/repeats:3_stddev\",$"}});
196 ADD_CASES(TC_CSVOut, {{"^\"BM_Repeat/repeats:3\",%csv_report$"},
197 {"^\"BM_Repeat/repeats:3\",%csv_report$"},
198 {"^\"BM_Repeat/repeats:3\",%csv_report$"},
199 {"^\"BM_Repeat/repeats:3_mean\",%csv_report$"},
200 {"^\"BM_Repeat/repeats:3_stddev\",%csv_report$"}});
201
202 // Test that a non-repeated test still prints non-aggregate results even when
203 // only-aggregate reports have been requested
BM_RepeatOnce(benchmark::State & state)204 void BM_RepeatOnce(benchmark::State& state) {
205 while (state.KeepRunning()) {
206 }
207 }
208 BENCHMARK(BM_RepeatOnce)->Repetitions(1)->ReportAggregatesOnly();
209 ADD_CASES(TC_ConsoleOut, {{"^BM_RepeatOnce/repeats:1 %console_report$"}});
210 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_RepeatOnce/repeats:1\",$"}});
211 ADD_CASES(TC_CSVOut, {{"^\"BM_RepeatOnce/repeats:1\",%csv_report$"}});
212
213 // Test that non-aggregate data is not reported
BM_SummaryRepeat(benchmark::State & state)214 void BM_SummaryRepeat(benchmark::State& state) {
215 while (state.KeepRunning()) {
216 }
217 }
218 BENCHMARK(BM_SummaryRepeat)->Repetitions(3)->ReportAggregatesOnly();
219 ADD_CASES(TC_ConsoleOut,
220 {{".*BM_SummaryRepeat/repeats:3 ", MR_Not},
221 {"^BM_SummaryRepeat/repeats:3_mean %console_report$"},
222 {"^BM_SummaryRepeat/repeats:3_stddev %console_report$"}});
223 ADD_CASES(TC_JSONOut, {{".*BM_SummaryRepeat/repeats:3 ", MR_Not},
224 {"\"name\": \"BM_SummaryRepeat/repeats:3_mean\",$"},
225 {"\"name\": \"BM_SummaryRepeat/repeats:3_stddev\",$"}});
226 ADD_CASES(TC_CSVOut, {{".*BM_SummaryRepeat/repeats:3 ", MR_Not},
227 {"^\"BM_SummaryRepeat/repeats:3_mean\",%csv_report$"},
228 {"^\"BM_SummaryRepeat/repeats:3_stddev\",%csv_report$"}});
229
BM_RepeatTimeUnit(benchmark::State & state)230 void BM_RepeatTimeUnit(benchmark::State& state) {
231 while (state.KeepRunning()) {
232 }
233 }
234 BENCHMARK(BM_RepeatTimeUnit)
235 ->Repetitions(3)
236 ->ReportAggregatesOnly()
237 ->Unit(benchmark::kMicrosecond);
238 ADD_CASES(TC_ConsoleOut,
239 {{".*BM_RepeatTimeUnit/repeats:3 ", MR_Not},
240 {"^BM_RepeatTimeUnit/repeats:3_mean %console_us_report$"},
241 {"^BM_RepeatTimeUnit/repeats:3_stddev %console_us_report$"}});
242 ADD_CASES(TC_JSONOut, {{".*BM_RepeatTimeUnit/repeats:3 ", MR_Not},
243 {"\"name\": \"BM_RepeatTimeUnit/repeats:3_mean\",$"},
244 {"\"time_unit\": \"us\",?$"},
245 {"\"name\": \"BM_RepeatTimeUnit/repeats:3_stddev\",$"},
246 {"\"time_unit\": \"us\",?$"}});
247 ADD_CASES(TC_CSVOut,
248 {{".*BM_RepeatTimeUnit/repeats:3 ", MR_Not},
249 {"^\"BM_RepeatTimeUnit/repeats:3_mean\",%csv_us_report$"},
250 {"^\"BM_RepeatTimeUnit/repeats:3_stddev\",%csv_us_report$"}});
251
252 // ========================================================================= //
253 // --------------------------- TEST CASES END ------------------------------ //
254 // ========================================================================= //
255
main(int argc,char * argv[])256 int main(int argc, char* argv[]) { RunOutputTests(argc, argv); }
257