1 /* 2 * Created by Joachim on 16/04/2019. 3 * Adapted from donated nonius code. 4 * 5 * Distributed under the Boost Software License, Version 1.0. (See accompanying 6 * file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) 7 */ 8 9 #include "catch.hpp" 10 #if defined(CATCH_CONFIG_ENABLE_BENCHMARKING) 11 namespace { 12 struct manual_clock { 13 public: 14 using duration = std::chrono::nanoseconds; 15 using time_point = std::chrono::time_point<manual_clock, duration>; 16 using rep = duration::rep; 17 using period = duration::period; 18 enum { is_steady = true }; 19 now__anon53caa19e0111::manual_clock20 static time_point now() { 21 return time_point(duration(tick())); 22 } 23 advance__anon53caa19e0111::manual_clock24 static void advance(int ticks = 1) { 25 tick() += ticks; 26 } 27 28 private: tick__anon53caa19e0111::manual_clock29 static rep& tick() { 30 static rep the_tick = 0; 31 return the_tick; 32 } 33 }; 34 35 struct counting_clock { 36 public: 37 using duration = std::chrono::nanoseconds; 38 using time_point = std::chrono::time_point<counting_clock, duration>; 39 using rep = duration::rep; 40 using period = duration::period; 41 enum { is_steady = true }; 42 now__anon53caa19e0111::counting_clock43 static time_point now() { 44 static rep ticks = 0; 45 return time_point(duration(ticks += rate())); 46 } 47 set_rate__anon53caa19e0111::counting_clock48 static void set_rate(rep new_rate) { rate() = new_rate; } 49 50 private: rate__anon53caa19e0111::counting_clock51 static rep& rate() { 52 static rep the_rate = 1; 53 return the_rate; 54 } 55 }; 56 57 struct TestChronometerModel : Catch::Benchmark::Detail::ChronometerConcept { 58 int started = 0; 59 int finished = 0; 60 start__anon53caa19e0111::TestChronometerModel61 void start() override { ++started; } finish__anon53caa19e0111::TestChronometerModel62 void finish() override { ++finished; } 63 }; 64 } // namespace 65 66 TEST_CASE("warmup", "[benchmark]") { 67 auto rate = 1000; 68 counting_clock::set_rate(rate); 69 70 auto start = counting_clock::now(); 71 auto iterations = Catch::Benchmark::Detail::warmup<counting_clock>(); 72 auto end = counting_clock::now(); 73 74 REQUIRE((iterations * rate) > Catch::Benchmark::Detail::warmup_time.count()); 75 REQUIRE((end - start) > Catch::Benchmark::Detail::warmup_time); 76 } 77 78 TEST_CASE("resolution", "[benchmark]") { 79 auto rate = 1000; 80 counting_clock::set_rate(rate); 81 82 size_t count = 10; 83 auto res = Catch::Benchmark::Detail::resolution<counting_clock>(static_cast<int>(count)); 84 85 REQUIRE(res.size() == count); 86 87 for (size_t i = 1; i < count; ++i) { 88 REQUIRE(res[i] == rate); 89 } 90 } 91 92 TEST_CASE("estimate_clock_resolution", "[benchmark]") { 93 auto rate = 1000; 94 counting_clock::set_rate(rate); 95 96 int iters = 160000; 97 auto res = Catch::Benchmark::Detail::estimate_clock_resolution<counting_clock>(iters); 98 99 REQUIRE(res.mean.count() == rate); 100 REQUIRE(res.outliers.total() == 0); 101 } 102 103 TEST_CASE("benchmark function call", "[benchmark]") { 104 SECTION("without chronometer") { 105 auto called = 0; 106 auto model = TestChronometerModel{}; 107 auto meter = Catch::Benchmark::Chronometer{ model, 1 }; __anon53caa19e0402null108 auto fn = Catch::Benchmark::Detail::BenchmarkFunction{ [&] { 109 CHECK(model.started == 1); 110 CHECK(model.finished == 0); 111 ++called; 112 } }; 113 114 fn(meter); 115 116 CHECK(model.started == 1); 117 CHECK(model.finished == 1); 118 CHECK(called == 1); 119 } 120 121 SECTION("with chronometer") { 122 auto called = 0; 123 auto model = TestChronometerModel{}; 124 auto meter = Catch::Benchmark::Chronometer{ model, 1 }; __anon53caa19e0502(Catch::Benchmark::Chronometer) 125 auto fn = Catch::Benchmark::Detail::BenchmarkFunction{ [&](Catch::Benchmark::Chronometer) { 126 CHECK(model.started == 0); 127 CHECK(model.finished == 0); 128 ++called; 129 } }; 130 131 fn(meter); 132 133 CHECK(model.started == 0); 134 CHECK(model.finished == 0); 135 CHECK(called == 1); 136 } 137 } 138 139 TEST_CASE("uniform samples", "[benchmark]") { 140 std::vector<double> samples(100); 141 std::fill(samples.begin(), samples.end(), 23); 142 143 using it = std::vector<double>::iterator; __anon53caa19e0602(it a, it b) 144 auto e = Catch::Benchmark::Detail::bootstrap(0.95, samples.begin(), samples.end(), samples, [](it a, it b) { 145 auto sum = std::accumulate(a, b, 0.); 146 return sum / (b - a); 147 }); 148 CHECK(e.point == 23); 149 CHECK(e.upper_bound == 23); 150 CHECK(e.lower_bound == 23); 151 CHECK(e.confidence_interval == 0.95); 152 } 153 154 155 TEST_CASE("normal_cdf", "[benchmark]") { 156 using Catch::Benchmark::Detail::normal_cdf; 157 CHECK(normal_cdf(0.000000) == Approx(0.50000000000000000)); 158 CHECK(normal_cdf(1.000000) == Approx(0.84134474606854293)); 159 CHECK(normal_cdf(-1.000000) == Approx(0.15865525393145705)); 160 CHECK(normal_cdf(2.809729) == Approx(0.99752083845315409)); 161 CHECK(normal_cdf(-1.352570) == Approx(0.08809652095066035)); 162 } 163 164 TEST_CASE("erfc_inv", "[benchmark]") { 165 using Catch::Benchmark::Detail::erfc_inv; 166 CHECK(erfc_inv(1.103560) == Approx(-0.09203687623843015)); 167 CHECK(erfc_inv(1.067400) == Approx(-0.05980291115763361)); 168 CHECK(erfc_inv(0.050000) == Approx(1.38590382434967796)); 169 } 170 171 TEST_CASE("normal_quantile", "[benchmark]") { 172 using Catch::Benchmark::Detail::normal_quantile; 173 CHECK(normal_quantile(0.551780) == Approx(0.13015979861484198)); 174 CHECK(normal_quantile(0.533700) == Approx(0.08457408802851875)); 175 CHECK(normal_quantile(0.025000) == Approx(-1.95996398454005449)); 176 } 177 178 179 TEST_CASE("mean", "[benchmark]") { 180 std::vector<double> x{ 10., 20., 14., 16., 30., 24. }; 181 182 auto m = Catch::Benchmark::Detail::mean(x.begin(), x.end()); 183 184 REQUIRE(m == 19.); 185 } 186 187 TEST_CASE("weighted_average_quantile", "[benchmark]") { 188 std::vector<double> x{ 10., 20., 14., 16., 30., 24. }; 189 190 auto q1 = Catch::Benchmark::Detail::weighted_average_quantile(1, 4, x.begin(), x.end()); 191 auto med = Catch::Benchmark::Detail::weighted_average_quantile(1, 2, x.begin(), x.end()); 192 auto q3 = Catch::Benchmark::Detail::weighted_average_quantile(3, 4, x.begin(), x.end()); 193 194 REQUIRE(q1 == 14.5); 195 REQUIRE(med == 18.); 196 REQUIRE(q3 == 23.); 197 } 198 199 TEST_CASE("classify_outliers", "[benchmark]") { __anon53caa19e0702(Catch::Benchmark::OutlierClassification o, int los, int lom, int him, int his) 200 auto require_outliers = [](Catch::Benchmark::OutlierClassification o, int los, int lom, int him, int his) { 201 REQUIRE(o.low_severe == los); 202 REQUIRE(o.low_mild == lom); 203 REQUIRE(o.high_mild == him); 204 REQUIRE(o.high_severe == his); 205 REQUIRE(o.total() == los + lom + him + his); 206 }; 207 208 SECTION("none") { 209 std::vector<double> x{ 10., 20., 14., 16., 30., 24. }; 210 211 auto o = Catch::Benchmark::Detail::classify_outliers(x.begin(), x.end()); 212 213 REQUIRE(o.samples_seen == static_cast<int>(x.size())); 214 require_outliers(o, 0, 0, 0, 0); 215 } 216 SECTION("low severe") { 217 std::vector<double> x{ -12., 20., 14., 16., 30., 24. }; 218 219 auto o = Catch::Benchmark::Detail::classify_outliers(x.begin(), x.end()); 220 221 REQUIRE(o.samples_seen == static_cast<int>(x.size())); 222 require_outliers(o, 1, 0, 0, 0); 223 } 224 SECTION("low mild") { 225 std::vector<double> x{ 1., 20., 14., 16., 30., 24. }; 226 227 auto o = Catch::Benchmark::Detail::classify_outliers(x.begin(), x.end()); 228 229 REQUIRE(o.samples_seen == static_cast<int>(x.size())); 230 require_outliers(o, 0, 1, 0, 0); 231 } 232 SECTION("high mild") { 233 std::vector<double> x{ 10., 20., 14., 16., 36., 24. }; 234 235 auto o = Catch::Benchmark::Detail::classify_outliers(x.begin(), x.end()); 236 237 REQUIRE(o.samples_seen == static_cast<int>(x.size())); 238 require_outliers(o, 0, 0, 1, 0); 239 } 240 SECTION("high severe") { 241 std::vector<double> x{ 10., 20., 14., 16., 49., 24. }; 242 243 auto o = Catch::Benchmark::Detail::classify_outliers(x.begin(), x.end()); 244 245 REQUIRE(o.samples_seen == static_cast<int>(x.size())); 246 require_outliers(o, 0, 0, 0, 1); 247 } 248 SECTION("mixed") { 249 std::vector<double> x{ -20., 20., 14., 16., 39., 24. }; 250 251 auto o = Catch::Benchmark::Detail::classify_outliers(x.begin(), x.end()); 252 253 REQUIRE(o.samples_seen == static_cast<int>(x.size())); 254 require_outliers(o, 1, 0, 1, 0); 255 } 256 } 257 258 TEST_CASE("analyse", "[benchmark]") { 259 Catch::ConfigData data{}; 260 data.benchmarkConfidenceInterval = 0.95; 261 data.benchmarkNoAnalysis = false; 262 data.benchmarkResamples = 1000; 263 data.benchmarkSamples = 99; 264 Catch::Config config{data}; 265 266 using Duration = Catch::Benchmark::FloatDuration<Catch::Benchmark::default_clock>; 267 268 Catch::Benchmark::Environment<Duration> env; 269 std::vector<Duration> samples(99); 270 for (size_t i = 0; i < samples.size(); ++i) { 271 samples[i] = Duration(23 + (i % 3 - 1)); 272 } 273 274 auto analysis = Catch::Benchmark::Detail::analyse(config, env, samples.begin(), samples.end()); 275 CHECK(analysis.mean.point.count() == 23); 276 CHECK(analysis.mean.lower_bound.count() < 23); 277 CHECK(analysis.mean.lower_bound.count() > 22); 278 CHECK(analysis.mean.upper_bound.count() > 23); 279 CHECK(analysis.mean.upper_bound.count() < 24); 280 281 CHECK(analysis.standard_deviation.point.count() > 0.5); 282 CHECK(analysis.standard_deviation.point.count() < 1); 283 CHECK(analysis.standard_deviation.lower_bound.count() > 0.5); 284 CHECK(analysis.standard_deviation.lower_bound.count() < 1); 285 CHECK(analysis.standard_deviation.upper_bound.count() > 0.5); 286 CHECK(analysis.standard_deviation.upper_bound.count() < 1); 287 288 CHECK(analysis.outliers.total() == 0); 289 CHECK(analysis.outliers.low_mild == 0); 290 CHECK(analysis.outliers.low_severe == 0); 291 CHECK(analysis.outliers.high_mild == 0); 292 CHECK(analysis.outliers.high_severe == 0); 293 CHECK(analysis.outliers.samples_seen == samples.size()); 294 295 CHECK(analysis.outlier_variance < 0.5); 296 CHECK(analysis.outlier_variance > 0); 297 } 298 299 TEST_CASE("analyse no analysis", "[benchmark]") { 300 Catch::ConfigData data{}; 301 data.benchmarkConfidenceInterval = 0.95; 302 data.benchmarkNoAnalysis = true; 303 data.benchmarkResamples = 1000; 304 data.benchmarkSamples = 99; 305 Catch::Config config{ data }; 306 307 using Duration = Catch::Benchmark::FloatDuration<Catch::Benchmark::default_clock>; 308 309 Catch::Benchmark::Environment<Duration> env; 310 std::vector<Duration> samples(99); 311 for (size_t i = 0; i < samples.size(); ++i) { 312 samples[i] = Duration(23 + (i % 3 - 1)); 313 } 314 315 auto analysis = Catch::Benchmark::Detail::analyse(config, env, samples.begin(), samples.end()); 316 CHECK(analysis.mean.point.count() == 23); 317 CHECK(analysis.mean.lower_bound.count() == 23); 318 CHECK(analysis.mean.upper_bound.count() == 23); 319 320 CHECK(analysis.standard_deviation.point.count() == 0); 321 CHECK(analysis.standard_deviation.lower_bound.count() == 0); 322 CHECK(analysis.standard_deviation.upper_bound.count() == 0); 323 324 CHECK(analysis.outliers.total() == 0); 325 CHECK(analysis.outliers.low_mild == 0); 326 CHECK(analysis.outliers.low_severe == 0); 327 CHECK(analysis.outliers.high_mild == 0); 328 CHECK(analysis.outliers.high_severe == 0); 329 CHECK(analysis.outliers.samples_seen == 0); 330 331 CHECK(analysis.outlier_variance == 0); 332 } 333 334 TEST_CASE("run_for_at_least, int", "[benchmark]") { 335 manual_clock::duration time(100); 336 337 int old_x = 1; __anon53caa19e0802(int x) 338 auto Timing = Catch::Benchmark::Detail::run_for_at_least<manual_clock>(time, 1, [&old_x](int x) -> int { 339 CHECK(x >= old_x); 340 manual_clock::advance(x); 341 old_x = x; 342 return x + 17; 343 }); 344 345 REQUIRE(Timing.elapsed >= time); 346 REQUIRE(Timing.result == Timing.iterations + 17); 347 REQUIRE(Timing.iterations >= time.count()); 348 } 349 350 TEST_CASE("run_for_at_least, chronometer", "[benchmark]") { 351 manual_clock::duration time(100); 352 353 int old_runs = 1; __anon53caa19e0902(Catch::Benchmark::Chronometer meter) 354 auto Timing = Catch::Benchmark::Detail::run_for_at_least<manual_clock>(time, 1, [&old_runs](Catch::Benchmark::Chronometer meter) -> int { 355 CHECK(meter.runs() >= old_runs); 356 manual_clock::advance(100); 357 meter.measure([] { 358 manual_clock::advance(1); 359 }); 360 old_runs = meter.runs(); 361 return meter.runs() + 17; 362 }); 363 364 REQUIRE(Timing.elapsed >= time); 365 REQUIRE(Timing.result == Timing.iterations + 17); 366 REQUIRE(Timing.iterations >= time.count()); 367 } 368 369 370 TEST_CASE("measure", "[benchmark]") { __anon53caa19e0b02(int x) 371 auto r = Catch::Benchmark::Detail::measure<manual_clock>([](int x) -> int { 372 CHECK(x == 17); 373 manual_clock::advance(42); 374 return 23; 375 }, 17); __anon53caa19e0c02(int x) 376 auto s = Catch::Benchmark::Detail::measure<manual_clock>([](int x) -> int { 377 CHECK(x == 23); 378 manual_clock::advance(69); 379 return 17; 380 }, 23); 381 382 CHECK(r.elapsed.count() == 42); 383 CHECK(r.result == 23); 384 CHECK(r.iterations == 1); 385 386 CHECK(s.elapsed.count() == 69); 387 CHECK(s.result == 17); 388 CHECK(s.iterations == 1); 389 } 390 391 TEST_CASE("run benchmark", "[benchmark]") { 392 counting_clock::set_rate(1000); 393 auto start = counting_clock::now(); 394 __anon53caa19e0d02() 395 Catch::Benchmark::Benchmark bench{ "Test Benchmark", [](Catch::Benchmark::Chronometer meter) { 396 counting_clock::set_rate(100000); 397 meter.measure([] { return counting_clock::now(); }); 398 } }; 399 400 bench.run<counting_clock>(); 401 auto end = counting_clock::now(); 402 403 CHECK((end - start).count() == 2867251000); 404 } 405 #endif // CATCH_CONFIG_ENABLE_BENCHMARKING 406