1 #ifndef TEST_OUTPUT_TEST_H
2 #define TEST_OUTPUT_TEST_H
3
4 #undef NDEBUG
5 #include <functional>
6 #include <initializer_list>
7 #include <memory>
8 #include <sstream>
9 #include <string>
10 #include <utility>
11 #include <vector>
12
13 #include "../src/re.h"
14 #include "benchmark/benchmark.h"
15
16 #define CONCAT2(x, y) x##y
17 #define CONCAT(x, y) CONCAT2(x, y)
18
19 #define ADD_CASES(...) int CONCAT(dummy, __LINE__) = ::AddCases(__VA_ARGS__)
20
21 #define SET_SUBSTITUTIONS(...) \
22 int CONCAT(dummy, __LINE__) = ::SetSubstitutions(__VA_ARGS__)
23
24 enum MatchRules {
25 MR_Default, // Skip non-matching lines until a match is found.
26 MR_Next, // Match must occur on the next line.
27 MR_Not // No line between the current position and the next match matches
28 // the regex
29 };
30
31 struct TestCase {
32 TestCase(std::string re, int rule = MR_Default);
33
34 std::string regex_str;
35 int match_rule;
36 std::string substituted_regex;
37 std::shared_ptr<benchmark::Regex> regex;
38 };
39
40 enum TestCaseID {
41 TC_ConsoleOut,
42 TC_ConsoleErr,
43 TC_JSONOut,
44 TC_JSONErr,
45 TC_CSVOut,
46 TC_CSVErr,
47
48 TC_NumID // PRIVATE
49 };
50
51 // Add a list of test cases to be run against the output specified by
52 // 'ID'
53 int AddCases(TestCaseID ID, std::initializer_list<TestCase> il);
54
55 // Add or set a list of substitutions to be performed on constructed regex's
56 // See 'output_test_helper.cc' for a list of default substitutions.
57 int SetSubstitutions(
58 std::initializer_list<std::pair<std::string, std::string>> il);
59
60 // Run all output tests.
61 void RunOutputTests(int argc, char* argv[]);
62
63 // Count the number of 'pat' substrings in the 'haystack' string.
64 int SubstrCnt(const std::string& haystack, const std::string& pat);
65
66 // Run registered benchmarks with file reporter enabled, and return the content
67 // outputted by the file reporter.
68 std::string GetFileReporterOutput(int argc, char* argv[]);
69
70 // ========================================================================= //
71 // ------------------------- Results checking ------------------------------ //
72 // ========================================================================= //
73
74 // Call this macro to register a benchmark for checking its results. This
75 // should be all that's needed. It subscribes a function to check the (CSV)
76 // results of a benchmark. This is done only after verifying that the output
77 // strings are really as expected.
78 // bm_name_pattern: a name or a regex pattern which will be matched against
79 // all the benchmark names. Matching benchmarks
80 // will be the subject of a call to checker_function
81 // checker_function: should be of type ResultsCheckFn (see below)
82 #define CHECK_BENCHMARK_RESULTS(bm_name_pattern, checker_function) \
83 size_t CONCAT(dummy, __LINE__) = AddChecker(bm_name_pattern, checker_function)
84
85 struct Results;
86 typedef std::function<void(Results const&)> ResultsCheckFn;
87
88 size_t AddChecker(const char* bm_name_pattern, const ResultsCheckFn& fn);
89
90 // Class holding the results of a benchmark.
91 // It is passed in calls to checker functions.
92 struct Results {
93 // the benchmark name
94 std::string name;
95 // the benchmark fields
96 std::map<std::string, std::string> values;
97
ResultsResults98 Results(const std::string& n) : name(n) {}
99
100 int NumThreads() const;
101
102 double NumIterations() const;
103
104 typedef enum { kCpuTime, kRealTime } BenchmarkTime;
105
106 // get cpu_time or real_time in seconds
107 double GetTime(BenchmarkTime which) const;
108
109 // get the real_time duration of the benchmark in seconds.
110 // it is better to use fuzzy float checks for this, as the float
111 // ASCII formatting is lossy.
DurationRealTimeResults112 double DurationRealTime() const {
113 return NumIterations() * GetTime(kRealTime);
114 }
115 // get the cpu_time duration of the benchmark in seconds
DurationCPUTimeResults116 double DurationCPUTime() const { return NumIterations() * GetTime(kCpuTime); }
117
118 // get the string for a result by name, or nullptr if the name
119 // is not found
GetResults120 const std::string* Get(const char* entry_name) const {
121 auto it = values.find(entry_name);
122 if (it == values.end()) return nullptr;
123 return &it->second;
124 }
125
126 // get a result by name, parsed as a specific type.
127 // NOTE: for counters, use GetCounterAs instead.
128 template <class T>
129 T GetAs(const char* entry_name) const;
130
131 // counters are written as doubles, so they have to be read first
132 // as a double, and only then converted to the asked type.
133 template <class T>
GetCounterAsResults134 T GetCounterAs(const char* entry_name) const {
135 double dval = GetAs<double>(entry_name);
136 T tval = static_cast<T>(dval);
137 return tval;
138 }
139 };
140
141 template <class T>
GetAs(const char * entry_name)142 T Results::GetAs(const char* entry_name) const {
143 auto* sv = Get(entry_name);
144 BM_CHECK(sv != nullptr && !sv->empty());
145 std::stringstream ss;
146 ss << *sv;
147 T out;
148 ss >> out;
149 BM_CHECK(!ss.fail());
150 return out;
151 }
152
153 //----------------------------------
154 // Macros to help in result checking. Do not use them with arguments causing
155 // side-effects.
156
157 // clang-format off
158
159 #define CHECK_RESULT_VALUE_IMPL(entry, getfn, var_type, var_name, relationship, value) \
160 CONCAT(BM_CHECK_, relationship) \
161 (entry.getfn< var_type >(var_name), (value)) << "\n" \
162 << __FILE__ << ":" << __LINE__ << ": " << (entry).name << ":\n" \
163 << __FILE__ << ":" << __LINE__ << ": " \
164 << "expected (" << #var_type << ")" << (var_name) \
165 << "=" << (entry).getfn< var_type >(var_name) \
166 << " to be " #relationship " to " << (value) << "\n"
167
168 // check with tolerance. eps_factor is the tolerance window, which is
169 // interpreted relative to value (eg, 0.1 means 10% of value).
170 #define CHECK_FLOAT_RESULT_VALUE_IMPL(entry, getfn, var_type, var_name, relationship, value, eps_factor) \
171 CONCAT(BM_CHECK_FLOAT_, relationship) \
172 (entry.getfn< var_type >(var_name), (value), (eps_factor) * (value)) << "\n" \
173 << __FILE__ << ":" << __LINE__ << ": " << (entry).name << ":\n" \
174 << __FILE__ << ":" << __LINE__ << ": " \
175 << "expected (" << #var_type << ")" << (var_name) \
176 << "=" << (entry).getfn< var_type >(var_name) \
177 << " to be " #relationship " to " << (value) << "\n" \
178 << __FILE__ << ":" << __LINE__ << ": " \
179 << "with tolerance of " << (eps_factor) * (value) \
180 << " (" << (eps_factor)*100. << "%), " \
181 << "but delta was " << ((entry).getfn< var_type >(var_name) - (value)) \
182 << " (" << (((entry).getfn< var_type >(var_name) - (value)) \
183 / \
184 ((value) > 1.e-5 || value < -1.e-5 ? value : 1.e-5)*100.) \
185 << "%)"
186
187 #define CHECK_RESULT_VALUE(entry, var_type, var_name, relationship, value) \
188 CHECK_RESULT_VALUE_IMPL(entry, GetAs, var_type, var_name, relationship, value)
189
190 #define CHECK_COUNTER_VALUE(entry, var_type, var_name, relationship, value) \
191 CHECK_RESULT_VALUE_IMPL(entry, GetCounterAs, var_type, var_name, relationship, value)
192
193 #define CHECK_FLOAT_RESULT_VALUE(entry, var_name, relationship, value, eps_factor) \
194 CHECK_FLOAT_RESULT_VALUE_IMPL(entry, GetAs, double, var_name, relationship, value, eps_factor)
195
196 #define CHECK_FLOAT_COUNTER_VALUE(entry, var_name, relationship, value, eps_factor) \
197 CHECK_FLOAT_RESULT_VALUE_IMPL(entry, GetCounterAs, double, var_name, relationship, value, eps_factor)
198
199 // clang-format on
200
201 // ========================================================================= //
202 // --------------------------- Misc Utilities ------------------------------ //
203 // ========================================================================= //
204
205 namespace {
206
207 const char* const dec_re = "[0-9]*[.]?[0-9]+([eE][-+][0-9]+)?";
208
209 } // end namespace
210
211 #endif // TEST_OUTPUT_TEST_H
212