1
2 #undef NDEBUG
3 #include <cassert>
4 #include <vector>
5
6 #include "../src/check.h" // NOTE: check.h is for internal use only!
7 #include "benchmark/benchmark.h"
8
9 namespace {
10
11 class TestReporter : public benchmark::ConsoleReporter {
12 public:
ReportContext(const Context & context)13 virtual bool ReportContext(const Context& context) BENCHMARK_OVERRIDE {
14 return ConsoleReporter::ReportContext(context);
15 };
16
ReportRuns(const std::vector<Run> & report)17 virtual void ReportRuns(const std::vector<Run>& report) BENCHMARK_OVERRIDE {
18 all_runs_.insert(all_runs_.end(), begin(report), end(report));
19 ConsoleReporter::ReportRuns(report);
20 }
21
TestReporter()22 TestReporter() {}
~TestReporter()23 virtual ~TestReporter() {}
24
25 mutable std::vector<Run> all_runs_;
26 };
27
28 struct TestCase {
29 std::string name;
30 bool error_occurred;
31 std::string error_message;
32
33 typedef benchmark::BenchmarkReporter::Run Run;
34
CheckRun__anond738587a0111::TestCase35 void CheckRun(Run const& run) const {
36 BM_CHECK(name == run.benchmark_name())
37 << "expected " << name << " got " << run.benchmark_name();
38 BM_CHECK(error_occurred == run.error_occurred);
39 BM_CHECK(error_message == run.error_message);
40 if (error_occurred) {
41 // BM_CHECK(run.iterations == 0);
42 } else {
43 BM_CHECK(run.iterations != 0);
44 }
45 }
46 };
47
48 std::vector<TestCase> ExpectedResults;
49
AddCases(const char * base_name,std::initializer_list<TestCase> const & v)50 int AddCases(const char* base_name, std::initializer_list<TestCase> const& v) {
51 for (auto TC : v) {
52 TC.name = base_name + TC.name;
53 ExpectedResults.push_back(std::move(TC));
54 }
55 return 0;
56 }
57
58 #define CONCAT(x, y) CONCAT2(x, y)
59 #define CONCAT2(x, y) x##y
60 #define ADD_CASES(...) int CONCAT(dummy, __LINE__) = AddCases(__VA_ARGS__)
61
62 } // end namespace
63
BM_error_no_running(benchmark::State & state)64 void BM_error_no_running(benchmark::State& state) {
65 state.SkipWithError("error message");
66 }
67 BENCHMARK(BM_error_no_running);
68 ADD_CASES("BM_error_no_running", {{"", true, "error message"}});
69
BM_error_before_running(benchmark::State & state)70 void BM_error_before_running(benchmark::State& state) {
71 state.SkipWithError("error message");
72 while (state.KeepRunning()) {
73 assert(false);
74 }
75 }
76 BENCHMARK(BM_error_before_running);
77 ADD_CASES("BM_error_before_running", {{"", true, "error message"}});
78
BM_error_before_running_batch(benchmark::State & state)79 void BM_error_before_running_batch(benchmark::State& state) {
80 state.SkipWithError("error message");
81 while (state.KeepRunningBatch(17)) {
82 assert(false);
83 }
84 }
85 BENCHMARK(BM_error_before_running_batch);
86 ADD_CASES("BM_error_before_running_batch", {{"", true, "error message"}});
87
BM_error_before_running_range_for(benchmark::State & state)88 void BM_error_before_running_range_for(benchmark::State& state) {
89 state.SkipWithError("error message");
90 for (auto _ : state) {
91 assert(false);
92 }
93 }
94 BENCHMARK(BM_error_before_running_range_for);
95 ADD_CASES("BM_error_before_running_range_for", {{"", true, "error message"}});
96
BM_error_during_running(benchmark::State & state)97 void BM_error_during_running(benchmark::State& state) {
98 int first_iter = true;
99 while (state.KeepRunning()) {
100 if (state.range(0) == 1 && state.thread_index() <= (state.threads() / 2)) {
101 assert(first_iter);
102 first_iter = false;
103 state.SkipWithError("error message");
104 } else {
105 state.PauseTiming();
106 state.ResumeTiming();
107 }
108 }
109 }
110 BENCHMARK(BM_error_during_running)->Arg(1)->Arg(2)->ThreadRange(1, 8);
111 ADD_CASES("BM_error_during_running", {{"/1/threads:1", true, "error message"},
112 {"/1/threads:2", true, "error message"},
113 {"/1/threads:4", true, "error message"},
114 {"/1/threads:8", true, "error message"},
115 {"/2/threads:1", false, ""},
116 {"/2/threads:2", false, ""},
117 {"/2/threads:4", false, ""},
118 {"/2/threads:8", false, ""}});
119
BM_error_during_running_ranged_for(benchmark::State & state)120 void BM_error_during_running_ranged_for(benchmark::State& state) {
121 assert(state.max_iterations > 3 && "test requires at least a few iterations");
122 bool first_iter = true;
123 // NOTE: Users should not write the for loop explicitly.
124 for (auto It = state.begin(), End = state.end(); It != End; ++It) {
125 if (state.range(0) == 1) {
126 assert(first_iter);
127 first_iter = false;
128 (void)first_iter;
129 state.SkipWithError("error message");
130 // Test the unfortunate but documented behavior that the ranged-for loop
131 // doesn't automatically terminate when SkipWithError is set.
132 assert(++It != End);
133 break; // Required behavior
134 }
135 }
136 }
137 BENCHMARK(BM_error_during_running_ranged_for)->Arg(1)->Arg(2)->Iterations(5);
138 ADD_CASES("BM_error_during_running_ranged_for",
139 {{"/1/iterations:5", true, "error message"},
140 {"/2/iterations:5", false, ""}});
141
BM_error_after_running(benchmark::State & state)142 void BM_error_after_running(benchmark::State& state) {
143 for (auto _ : state) {
144 benchmark::DoNotOptimize(state.iterations());
145 }
146 if (state.thread_index() <= (state.threads() / 2))
147 state.SkipWithError("error message");
148 }
149 BENCHMARK(BM_error_after_running)->ThreadRange(1, 8);
150 ADD_CASES("BM_error_after_running", {{"/threads:1", true, "error message"},
151 {"/threads:2", true, "error message"},
152 {"/threads:4", true, "error message"},
153 {"/threads:8", true, "error message"}});
154
BM_error_while_paused(benchmark::State & state)155 void BM_error_while_paused(benchmark::State& state) {
156 bool first_iter = true;
157 while (state.KeepRunning()) {
158 if (state.range(0) == 1 && state.thread_index() <= (state.threads() / 2)) {
159 assert(first_iter);
160 first_iter = false;
161 state.PauseTiming();
162 state.SkipWithError("error message");
163 } else {
164 state.PauseTiming();
165 state.ResumeTiming();
166 }
167 }
168 }
169 BENCHMARK(BM_error_while_paused)->Arg(1)->Arg(2)->ThreadRange(1, 8);
170 ADD_CASES("BM_error_while_paused", {{"/1/threads:1", true, "error message"},
171 {"/1/threads:2", true, "error message"},
172 {"/1/threads:4", true, "error message"},
173 {"/1/threads:8", true, "error message"},
174 {"/2/threads:1", false, ""},
175 {"/2/threads:2", false, ""},
176 {"/2/threads:4", false, ""},
177 {"/2/threads:8", false, ""}});
178
main(int argc,char * argv[])179 int main(int argc, char* argv[]) {
180 benchmark::Initialize(&argc, argv);
181
182 TestReporter test_reporter;
183 benchmark::RunSpecifiedBenchmarks(&test_reporter);
184
185 typedef benchmark::BenchmarkReporter::Run Run;
186 auto EB = ExpectedResults.begin();
187
188 for (Run const& run : test_reporter.all_runs_) {
189 assert(EB != ExpectedResults.end());
190 EB->CheckRun(run);
191 ++EB;
192 }
193 assert(EB == ExpectedResults.end());
194
195 return 0;
196 }
197