1 #include <iostream>
2 #include <map>
3 #include <memory>
4 #include <sstream>
5
6 #include "../src/check.h" // NOTE: check.h is for internal use only!
7 #include "../src/re.h" // NOTE: re.h is for internal use only
8 #include "output_test.h"
9
10 // ========================================================================= //
11 // ------------------------------ Internals -------------------------------- //
12 // ========================================================================= //
13 namespace internal {
14 namespace {
15
16 using TestCaseList = std::vector<TestCase>;
17
18 // Use a vector because the order elements are added matters during iteration.
19 // std::map/unordered_map don't guarantee that.
20 // For example:
21 // SetSubstitutions({{"%HelloWorld", "Hello"}, {"%Hello", "Hi"}});
22 // Substitute("%HelloWorld") // Always expands to Hello.
23 using SubMap = std::vector<std::pair<std::string, std::string>>;
24
GetTestCaseList(TestCaseID ID)25 TestCaseList& GetTestCaseList(TestCaseID ID) {
26 // Uses function-local statics to ensure initialization occurs
27 // before first use.
28 static TestCaseList lists[TC_NumID];
29 return lists[ID];
30 }
31
GetSubstitutions()32 SubMap& GetSubstitutions() {
33 // Don't use 'dec_re' from header because it may not yet be initialized.
34 static std::string dec_re = "[0-9]*[.]?[0-9]+([eE][-+][0-9]+)?";
35 static SubMap map = {
36 {"%float", "[0-9]*[.]?[0-9]+([eE][-+][0-9]+)?"},
37 {"%int", "[ ]*[0-9]+"},
38 {" %s ", "[ ]+"},
39 {"%time", "[ ]*[0-9]{1,5} ns"},
40 {"%console_report", "[ ]*[0-9]{1,5} ns [ ]*[0-9]{1,5} ns [ ]*[0-9]+"},
41 {"%console_us_report", "[ ]*[0-9] us [ ]*[0-9] us [ ]*[0-9]+"},
42 {"%csv_report", "[0-9]+," + dec_re + "," + dec_re + ",ns,,,,,"},
43 {"%csv_us_report", "[0-9]+," + dec_re + "," + dec_re + ",us,,,,,"},
44 {"%csv_bytes_report",
45 "[0-9]+," + dec_re + "," + dec_re + ",ns," + dec_re + ",,,,"},
46 {"%csv_items_report",
47 "[0-9]+," + dec_re + "," + dec_re + ",ns,," + dec_re + ",,,"},
48 {"%csv_label_report_begin", "[0-9]+," + dec_re + "," + dec_re + ",ns,,,"},
49 {"%csv_label_report_end", ",,"}};
50 return map;
51 }
52
PerformSubstitutions(std::string source)53 std::string PerformSubstitutions(std::string source) {
54 SubMap const& subs = GetSubstitutions();
55 using SizeT = std::string::size_type;
56 for (auto const& KV : subs) {
57 SizeT pos;
58 SizeT next_start = 0;
59 while ((pos = source.find(KV.first, next_start)) != std::string::npos) {
60 next_start = pos + KV.second.size();
61 source.replace(pos, KV.first.size(), KV.second);
62 }
63 }
64 return source;
65 }
66
CheckCase(std::stringstream & remaining_output,TestCase const & TC,TestCaseList const & not_checks)67 void CheckCase(std::stringstream& remaining_output, TestCase const& TC,
68 TestCaseList const& not_checks) {
69 std::string first_line;
70 bool on_first = true;
71 std::string line;
72 while (remaining_output.eof() == false) {
73 CHECK(remaining_output.good());
74 std::getline(remaining_output, line);
75 if (on_first) {
76 first_line = line;
77 on_first = false;
78 }
79 for (const auto& NC : not_checks) {
80 CHECK(!NC.regex->Match(line))
81 << "Unexpected match for line \"" << line << "\" for MR_Not regex \""
82 << NC.regex_str << "\""
83 << "\n actual regex string \"" << TC.substituted_regex << "\""
84 << "\n started matching near: " << first_line;
85 }
86 if (TC.regex->Match(line)) return;
87 CHECK(TC.match_rule != MR_Next)
88 << "Expected line \"" << line << "\" to match regex \"" << TC.regex_str
89 << "\""
90 << "\n actual regex string \"" << TC.substituted_regex << "\""
91 << "\n started matching near: " << first_line;
92 }
93 CHECK(remaining_output.eof() == false)
94 << "End of output reached before match for regex \"" << TC.regex_str
95 << "\" was found"
96 << "\n actual regex string \"" << TC.substituted_regex << "\""
97 << "\n started matching near: " << first_line;
98 }
99
CheckCases(TestCaseList const & checks,std::stringstream & output)100 void CheckCases(TestCaseList const& checks, std::stringstream& output) {
101 std::vector<TestCase> not_checks;
102 for (size_t i = 0; i < checks.size(); ++i) {
103 const auto& TC = checks[i];
104 if (TC.match_rule == MR_Not) {
105 not_checks.push_back(TC);
106 continue;
107 }
108 CheckCase(output, TC, not_checks);
109 not_checks.clear();
110 }
111 }
112
113 class TestReporter : public benchmark::BenchmarkReporter {
114 public:
TestReporter(std::vector<benchmark::BenchmarkReporter * > reps)115 TestReporter(std::vector<benchmark::BenchmarkReporter*> reps)
116 : reporters_(reps) {}
117
ReportContext(const Context & context)118 virtual bool ReportContext(const Context& context) {
119 bool last_ret = false;
120 bool first = true;
121 for (auto rep : reporters_) {
122 bool new_ret = rep->ReportContext(context);
123 CHECK(first || new_ret == last_ret)
124 << "Reports return different values for ReportContext";
125 first = false;
126 last_ret = new_ret;
127 }
128 (void)first;
129 return last_ret;
130 }
131
ReportRuns(const std::vector<Run> & report)132 void ReportRuns(const std::vector<Run>& report) {
133 for (auto rep : reporters_) rep->ReportRuns(report);
134 }
Finalize()135 void Finalize() {
136 for (auto rep : reporters_) rep->Finalize();
137 }
138
139 private:
140 std::vector<benchmark::BenchmarkReporter *> reporters_;
141 };
142 }
143 } // end namespace internal
144
145 // ========================================================================= //
146 // -------------------------- Public API Definitions------------------------ //
147 // ========================================================================= //
148
TestCase(std::string re,int rule)149 TestCase::TestCase(std::string re, int rule)
150 : regex_str(std::move(re)),
151 match_rule(rule),
152 substituted_regex(internal::PerformSubstitutions(regex_str)),
153 regex(std::make_shared<benchmark::Regex>()) {
154 std::string err_str;
155 regex->Init(substituted_regex,& err_str);
156 CHECK(err_str.empty()) << "Could not construct regex \"" << substituted_regex
157 << "\""
158 << "\n originally \"" << regex_str << "\""
159 << "\n got error: " << err_str;
160 }
161
AddCases(TestCaseID ID,std::initializer_list<TestCase> il)162 int AddCases(TestCaseID ID, std::initializer_list<TestCase> il) {
163 auto& L = internal::GetTestCaseList(ID);
164 L.insert(L.end(), il);
165 return 0;
166 }
167
SetSubstitutions(std::initializer_list<std::pair<std::string,std::string>> il)168 int SetSubstitutions(
169 std::initializer_list<std::pair<std::string, std::string>> il) {
170 auto& subs = internal::GetSubstitutions();
171 for (auto KV : il) {
172 bool exists = false;
173 KV.second = internal::PerformSubstitutions(KV.second);
174 for (auto& EKV : subs) {
175 if (EKV.first == KV.first) {
176 EKV.second = std::move(KV.second);
177 exists = true;
178 break;
179 }
180 }
181 if (!exists) subs.push_back(std::move(KV));
182 }
183 return 0;
184 }
185
RunOutputTests(int argc,char * argv[])186 void RunOutputTests(int argc, char* argv[]) {
187 using internal::GetTestCaseList;
188 benchmark::Initialize(&argc, argv);
189 benchmark::ConsoleReporter CR(benchmark::ConsoleReporter::OO_None);
190 benchmark::JSONReporter JR;
191 benchmark::CSVReporter CSVR;
192 struct ReporterTest {
193 const char* name;
194 std::vector<TestCase>& output_cases;
195 std::vector<TestCase>& error_cases;
196 benchmark::BenchmarkReporter& reporter;
197 std::stringstream out_stream;
198 std::stringstream err_stream;
199
200 ReporterTest(const char* n, std::vector<TestCase>& out_tc,
201 std::vector<TestCase>& err_tc,
202 benchmark::BenchmarkReporter& br)
203 : name(n), output_cases(out_tc), error_cases(err_tc), reporter(br) {
204 reporter.SetOutputStream(&out_stream);
205 reporter.SetErrorStream(&err_stream);
206 }
207 } TestCases[] = {
208 {"ConsoleReporter", GetTestCaseList(TC_ConsoleOut),
209 GetTestCaseList(TC_ConsoleErr), CR},
210 {"JSONReporter", GetTestCaseList(TC_JSONOut), GetTestCaseList(TC_JSONErr),
211 JR},
212 {"CSVReporter", GetTestCaseList(TC_CSVOut), GetTestCaseList(TC_CSVErr),
213 CSVR},
214 };
215
216 // Create the test reporter and run the benchmarks.
217 std::cout << "Running benchmarks...\n";
218 internal::TestReporter test_rep({&CR, &JR, &CSVR});
219 benchmark::RunSpecifiedBenchmarks(&test_rep);
220
221 for (auto& rep_test : TestCases) {
222 std::string msg = std::string("\nTesting ") + rep_test.name + " Output\n";
223 std::string banner(msg.size() - 1, '-');
224 std::cout << banner << msg << banner << "\n";
225
226 std::cerr << rep_test.err_stream.str();
227 std::cout << rep_test.out_stream.str();
228
229 internal::CheckCases(rep_test.error_cases, rep_test.err_stream);
230 internal::CheckCases(rep_test.output_cases, rep_test.out_stream);
231
232 std::cout << "\n";
233 }
234 }
235