1 #![allow(deprecated)]
2
3 use crate::analysis;
4 use crate::connection::OutgoingMessage;
5 use crate::measurement::{Measurement, WallTime};
6 use crate::report::{BenchmarkId, Report, ReportContext};
7 use crate::routine::{Function, Routine};
8 use crate::{Bencher, Criterion, DurationExt, Mode, PlotConfiguration, SamplingMode, Throughput};
9 use std::cell::RefCell;
10 use std::fmt::Debug;
11 use std::marker::Sized;
12 use std::time::Duration;
13
14 // TODO: Move the benchmark config stuff to a separate module for easier use.
15
16 /// Struct containing all of the configuration options for a benchmark.
17 pub struct BenchmarkConfig {
18 pub confidence_level: f64,
19 pub measurement_time: Duration,
20 pub noise_threshold: f64,
21 pub nresamples: usize,
22 pub sample_size: usize,
23 pub significance_level: f64,
24 pub warm_up_time: Duration,
25 pub sampling_mode: SamplingMode,
26 }
27
28 /// Struct representing a partially-complete per-benchmark configuration.
29 #[derive(Clone)]
30 pub(crate) struct PartialBenchmarkConfig {
31 pub(crate) confidence_level: Option<f64>,
32 pub(crate) measurement_time: Option<Duration>,
33 pub(crate) noise_threshold: Option<f64>,
34 pub(crate) nresamples: Option<usize>,
35 pub(crate) sample_size: Option<usize>,
36 pub(crate) significance_level: Option<f64>,
37 pub(crate) warm_up_time: Option<Duration>,
38 pub(crate) sampling_mode: Option<SamplingMode>,
39 pub(crate) plot_config: PlotConfiguration,
40 }
41
42 impl Default for PartialBenchmarkConfig {
default() -> Self43 fn default() -> Self {
44 PartialBenchmarkConfig {
45 confidence_level: None,
46 measurement_time: None,
47 noise_threshold: None,
48 nresamples: None,
49 sample_size: None,
50 significance_level: None,
51 warm_up_time: None,
52 plot_config: PlotConfiguration::default(),
53 sampling_mode: None,
54 }
55 }
56 }
57
58 impl PartialBenchmarkConfig {
to_complete(&self, defaults: &BenchmarkConfig) -> BenchmarkConfig59 pub(crate) fn to_complete(&self, defaults: &BenchmarkConfig) -> BenchmarkConfig {
60 BenchmarkConfig {
61 confidence_level: self.confidence_level.unwrap_or(defaults.confidence_level),
62 measurement_time: self.measurement_time.unwrap_or(defaults.measurement_time),
63 noise_threshold: self.noise_threshold.unwrap_or(defaults.noise_threshold),
64 nresamples: self.nresamples.unwrap_or(defaults.nresamples),
65 sample_size: self.sample_size.unwrap_or(defaults.sample_size),
66 significance_level: self
67 .significance_level
68 .unwrap_or(defaults.significance_level),
69 warm_up_time: self.warm_up_time.unwrap_or(defaults.warm_up_time),
70 sampling_mode: self.sampling_mode.unwrap_or(defaults.sampling_mode),
71 }
72 }
73 }
74
75 pub(crate) struct NamedRoutine<T, M: Measurement = WallTime> {
76 pub id: String,
77 pub(crate) f: Box<RefCell<dyn Routine<M, T>>>,
78 }
79
80 /// Structure representing a benchmark (or group of benchmarks)
81 /// which take one parameter.
82 #[doc(hidden)]
83 #[deprecated(since = "0.3.4", note = "Please use BenchmarkGroups instead.")]
84 pub struct ParameterizedBenchmark<T: Debug, M: Measurement = WallTime> {
85 config: PartialBenchmarkConfig,
86 values: Vec<T>,
87 routines: Vec<NamedRoutine<T, M>>,
88 throughput: Option<Box<dyn Fn(&T) -> Throughput>>,
89 }
90
91 /// Structure representing a benchmark (or group of benchmarks)
92 /// which takes no parameters.
93 #[doc(hidden)]
94 #[deprecated(since = "0.3.4", note = "Please use BenchmarkGroups instead.")]
95 pub struct Benchmark<M: Measurement = WallTime> {
96 config: PartialBenchmarkConfig,
97 routines: Vec<NamedRoutine<(), M>>,
98 throughput: Option<Throughput>,
99 }
100
101 /// Common trait for `Benchmark` and `ParameterizedBenchmark`. Not intended to be
102 /// used outside of Criterion.rs.
103 #[doc(hidden)]
104 pub trait BenchmarkDefinition<M: Measurement = WallTime>: Sized {
105 #[doc(hidden)]
run(self, group_id: &str, c: &mut Criterion<M>)106 fn run(self, group_id: &str, c: &mut Criterion<M>);
107 }
108
109 macro_rules! benchmark_config {
110 ($type:tt) => {
111 /// Changes the size of the sample for this benchmark
112 ///
113 /// A bigger sample should yield more accurate results if paired with a sufficiently large
114 /// measurement time.
115 ///
116 /// Sample size must be at least 10.
117 ///
118 /// # Panics
119 ///
120 /// Panics if n < 10.
121 pub fn sample_size(mut self, n: usize) -> Self {
122 assert!(n >= 10);
123
124 self.config.sample_size = Some(n);
125 self
126 }
127
128 /// Changes the warm up time for this benchmark
129 ///
130 /// # Panics
131 ///
132 /// Panics if the input duration is zero
133 pub fn warm_up_time(mut self, dur: Duration) -> Self {
134 assert!(dur.to_nanos() > 0);
135
136 self.config.warm_up_time = Some(dur);
137 self
138 }
139
140 /// Changes the target measurement time for this benchmark. Criterion will attempt
141 /// to spend approximately this amount of time measuring the benchmark.
142 /// With a longer time, the measurement will become more resilient to transitory peak loads
143 /// caused by external programs.
144 ///
145 /// # Panics
146 ///
147 /// Panics if the input duration in zero
148 pub fn measurement_time(mut self, dur: Duration) -> Self {
149 assert!(dur.to_nanos() > 0);
150
151 self.config.measurement_time = Some(dur);
152 self
153 }
154
155 /// Changes the number of resamples for this benchmark
156 ///
157 /// Number of resamples to use for the
158 /// [bootstrap](http://en.wikipedia.org/wiki/Bootstrapping_(statistics)#Case_resampling)
159 ///
160 /// A larger number of resamples reduces the random sampling errors, which are inherent to the
161 /// bootstrap method, but also increases the analysis time.
162 ///
163 /// # Panics
164 ///
165 /// Panics if the number of resamples is set to zero
166 pub fn nresamples(mut self, n: usize) -> Self {
167 assert!(n > 0);
168 if n <= 1000 {
169 println!("\nWarning: It is not recommended to reduce nresamples below 1000.");
170 }
171
172 self.config.nresamples = Some(n);
173 self
174 }
175
176 /// Changes the default noise threshold for this benchmark. The noise threshold
177 /// is used to filter out small changes in performance, even if they are statistically
178 /// significant. Sometimes benchmarking the same code twice will result in small but
179 /// statistically significant differences solely because of noise. This provides a way to filter
180 /// out some of these false positives at the cost of making it harder to detect small changes
181 /// to the true performance of the benchmark.
182 ///
183 /// The default is 0.01, meaning that changes smaller than 1% will be ignored.
184 ///
185 /// # Panics
186 ///
187 /// Panics if the threshold is set to a negative value
188 pub fn noise_threshold(mut self, threshold: f64) -> Self {
189 assert!(threshold >= 0.0);
190
191 self.config.noise_threshold = Some(threshold);
192 self
193 }
194
195 /// Changes the default confidence level for this benchmark. The confidence
196 /// level is the desired probability that the true runtime lies within the estimated
197 /// [confidence interval](https://en.wikipedia.org/wiki/Confidence_interval). The default is
198 /// 0.95, meaning that the confidence interval should capture the true value 95% of the time.
199 ///
200 /// # Panics
201 ///
202 /// Panics if the confidence level is set to a value outside the `(0, 1)` range
203 pub fn confidence_level(mut self, cl: f64) -> Self {
204 assert!(cl > 0.0 && cl < 1.0);
205 if cl < 0.5 {
206 println!("\nWarning: It is not recommended to reduce confidence level below 0.5.");
207 }
208
209 self.config.confidence_level = Some(cl);
210 self
211 }
212
213 /// Changes the default [significance level](https://en.wikipedia.org/wiki/Statistical_significance)
214 /// for this benchmark. This is used to perform a
215 /// [hypothesis test](https://en.wikipedia.org/wiki/Statistical_hypothesis_testing) to see if
216 /// the measurements from this run are different from the measured performance of the last run.
217 /// The significance level is the desired probability that two measurements of identical code
218 /// will be considered 'different' due to noise in the measurements. The default value is 0.05,
219 /// meaning that approximately 5% of identical benchmarks will register as different due to
220 /// noise.
221 ///
222 /// This presents a trade-off. By setting the significance level closer to 0.0, you can increase
223 /// the statistical robustness against noise, but it also weakens Criterion.rs' ability to
224 /// detect small but real changes in the performance. By setting the significance level
225 /// closer to 1.0, Criterion.rs will be more able to detect small true changes, but will also
226 /// report more spurious differences.
227 ///
228 /// See also the noise threshold setting.
229 ///
230 /// # Panics
231 ///
232 /// Panics if the significance level is set to a value outside the `(0, 1)` range
233 pub fn significance_level(mut self, sl: f64) -> Self {
234 assert!(sl > 0.0 && sl < 1.0);
235
236 self.config.significance_level = Some(sl);
237 self
238 }
239
240 /// Changes the plot configuration for this benchmark.
241 pub fn plot_config(mut self, new_config: PlotConfiguration) -> Self {
242 self.config.plot_config = new_config;
243 self
244 }
245
246 /// Changes the sampling mode for this benchmark.
247 pub fn sampling_mode(mut self, new_mode: SamplingMode) -> Self {
248 self.config.sampling_mode = Some(new_mode);
249 self
250 }
251 };
252 }
253
254 impl<M> Benchmark<M>
255 where
256 M: Measurement + 'static,
257 {
258 benchmark_config!(Benchmark);
259
260 /// Create a new benchmark group and adds the given function to it.
261 ///
262 /// # Example
263 ///
264 /// ```rust
265 /// # #[macro_use] extern crate criterion;
266 /// # use criterion::*;
267 ///
268 /// fn bench(c: &mut Criterion) {
269 /// // One-time setup goes here
270 /// c.bench(
271 /// "my_group",
272 /// Benchmark::new("my_function", |b| b.iter(|| {
273 /// // Code to benchmark goes here
274 /// })),
275 /// );
276 /// }
277 ///
278 /// criterion_group!(benches, bench);
279 /// criterion_main!(benches);
280 /// ```
new<S, F>(id: S, f: F) -> Benchmark<M> where S: Into<String>, F: FnMut(&mut Bencher<'_, M>) + 'static,281 pub fn new<S, F>(id: S, f: F) -> Benchmark<M>
282 where
283 S: Into<String>,
284 F: FnMut(&mut Bencher<'_, M>) + 'static,
285 {
286 Benchmark {
287 config: PartialBenchmarkConfig::default(),
288 routines: vec![],
289 throughput: None,
290 }
291 .with_function(id, f)
292 }
293
294 /// Add a function to the benchmark group.
with_function<S, F>(mut self, id: S, mut f: F) -> Benchmark<M> where S: Into<String>, F: FnMut(&mut Bencher<'_, M>) + 'static,295 pub fn with_function<S, F>(mut self, id: S, mut f: F) -> Benchmark<M>
296 where
297 S: Into<String>,
298 F: FnMut(&mut Bencher<'_, M>) + 'static,
299 {
300 let routine = NamedRoutine {
301 id: id.into(),
302 f: Box::new(RefCell::new(Function::new(move |b, _| f(b)))),
303 };
304 self.routines.push(routine);
305 self
306 }
307
308 /// Set the input size for this benchmark group. Used for reporting the
309 /// throughput.
throughput(mut self, throughput: Throughput) -> Benchmark<M>310 pub fn throughput(mut self, throughput: Throughput) -> Benchmark<M> {
311 self.throughput = Some(throughput);
312 self
313 }
314 }
315
316 impl<M: Measurement> BenchmarkDefinition<M> for Benchmark<M> {
run(self, group_id: &str, c: &mut Criterion<M>)317 fn run(self, group_id: &str, c: &mut Criterion<M>) {
318 let report_context = ReportContext {
319 output_directory: c.output_directory.clone(),
320 plot_config: self.config.plot_config.clone(),
321 };
322
323 let config = self.config.to_complete(&c.config);
324 let num_routines = self.routines.len();
325
326 let mut all_ids = vec![];
327 let mut any_matched = false;
328
329 if let Some(conn) = &c.connection {
330 conn.send(&OutgoingMessage::BeginningBenchmarkGroup { group: group_id })
331 .unwrap();
332 }
333
334 for routine in self.routines {
335 let function_id = if num_routines == 1 && group_id == routine.id {
336 None
337 } else {
338 Some(routine.id)
339 };
340
341 let mut id = BenchmarkId::new(
342 group_id.to_owned(),
343 function_id,
344 None,
345 self.throughput.clone(),
346 );
347
348 id.ensure_directory_name_unique(&c.all_directories);
349 c.all_directories.insert(id.as_directory_name().to_owned());
350 id.ensure_title_unique(&c.all_titles);
351 c.all_titles.insert(id.as_title().to_owned());
352
353 let do_run = c.filter_matches(id.id());
354 any_matched |= do_run;
355
356 execute_benchmark(
357 do_run,
358 &id,
359 c,
360 &config,
361 &mut *routine.f.borrow_mut(),
362 &report_context,
363 &(),
364 self.throughput.clone(),
365 );
366
367 all_ids.push(id);
368 }
369
370 if let Some(conn) = &c.connection {
371 conn.send(&OutgoingMessage::FinishedBenchmarkGroup { group: group_id })
372 .unwrap();
373 conn.serve_value_formatter(c.measurement.formatter())
374 .unwrap();
375 }
376
377 if all_ids.len() > 1 && any_matched && c.mode.is_benchmark() {
378 c.report
379 .summarize(&report_context, &all_ids, c.measurement.formatter());
380 }
381 if any_matched {
382 c.report.group_separator();
383 }
384 }
385 }
386
387 impl<T, M> ParameterizedBenchmark<T, M>
388 where
389 T: Debug + 'static,
390 M: Measurement + 'static,
391 {
392 benchmark_config!(ParameterizedBenchmark);
393
with_functions( functions: Vec<NamedRoutine<T, M>>, parameters: Vec<T>, ) -> ParameterizedBenchmark<T, M>394 pub(crate) fn with_functions(
395 functions: Vec<NamedRoutine<T, M>>,
396 parameters: Vec<T>,
397 ) -> ParameterizedBenchmark<T, M> {
398 ParameterizedBenchmark {
399 config: PartialBenchmarkConfig::default(),
400 values: parameters,
401 routines: functions,
402 throughput: None,
403 }
404 }
405
406 /// Create a new parameterized benchmark group and adds the given function
407 /// to it.
408 /// The function under test must follow the setup - bench - teardown pattern:
409 ///
410 /// # Example
411 ///
412 /// ```rust
413 /// # #[macro_use] extern crate criterion;
414 /// # use criterion::*;
415 ///
416 /// fn bench(c: &mut Criterion) {
417 /// let parameters = vec![1u64, 2u64, 3u64];
418 ///
419 /// // One-time setup goes here
420 /// c.bench(
421 /// "my_group",
422 /// ParameterizedBenchmark::new(
423 /// "my_function",
424 /// |b, param| b.iter(|| {
425 /// // Code to benchmark using param goes here
426 /// }),
427 /// parameters
428 /// )
429 /// );
430 /// }
431 ///
432 /// criterion_group!(benches, bench);
433 /// criterion_main!(benches);
434 /// ```
new<S, F, I>(id: S, f: F, parameters: I) -> ParameterizedBenchmark<T, M> where S: Into<String>, F: FnMut(&mut Bencher<'_, M>, &T) + 'static, I: IntoIterator<Item = T>,435 pub fn new<S, F, I>(id: S, f: F, parameters: I) -> ParameterizedBenchmark<T, M>
436 where
437 S: Into<String>,
438 F: FnMut(&mut Bencher<'_, M>, &T) + 'static,
439 I: IntoIterator<Item = T>,
440 {
441 ParameterizedBenchmark {
442 config: PartialBenchmarkConfig::default(),
443 values: parameters.into_iter().collect(),
444 routines: vec![],
445 throughput: None,
446 }
447 .with_function(id, f)
448 }
449
450 /// Add a function to the benchmark group.
with_function<S, F>(mut self, id: S, f: F) -> ParameterizedBenchmark<T, M> where S: Into<String>, F: FnMut(&mut Bencher<'_, M>, &T) + 'static,451 pub fn with_function<S, F>(mut self, id: S, f: F) -> ParameterizedBenchmark<T, M>
452 where
453 S: Into<String>,
454 F: FnMut(&mut Bencher<'_, M>, &T) + 'static,
455 {
456 let routine = NamedRoutine {
457 id: id.into(),
458 f: Box::new(RefCell::new(Function::new(f))),
459 };
460 self.routines.push(routine);
461 self
462 }
463
464 /// Use the given function to calculate the input size for a given input.
throughput<F>(mut self, throughput: F) -> ParameterizedBenchmark<T, M> where F: Fn(&T) -> Throughput + 'static,465 pub fn throughput<F>(mut self, throughput: F) -> ParameterizedBenchmark<T, M>
466 where
467 F: Fn(&T) -> Throughput + 'static,
468 {
469 self.throughput = Some(Box::new(throughput));
470 self
471 }
472 }
473 impl<T, M> BenchmarkDefinition<M> for ParameterizedBenchmark<T, M>
474 where
475 T: Debug + 'static,
476 M: Measurement + 'static,
477 {
run(self, group_id: &str, c: &mut Criterion<M>)478 fn run(self, group_id: &str, c: &mut Criterion<M>) {
479 let report_context = ReportContext {
480 output_directory: c.output_directory.clone(),
481 plot_config: self.config.plot_config.clone(),
482 };
483
484 let config = self.config.to_complete(&c.config);
485 let num_parameters = self.values.len();
486 let num_routines = self.routines.len();
487
488 let mut all_ids = vec![];
489 let mut any_matched = false;
490
491 if let Some(conn) = &c.connection {
492 conn.send(&OutgoingMessage::BeginningBenchmarkGroup { group: group_id })
493 .unwrap();
494 }
495
496 for routine in self.routines {
497 for value in &self.values {
498 let function_id = if num_routines == 1 && group_id == routine.id {
499 None
500 } else {
501 Some(routine.id.clone())
502 };
503
504 let value_str = if num_parameters == 1 {
505 None
506 } else {
507 Some(format!("{:?}", value))
508 };
509
510 let throughput = self.throughput.as_ref().map(|func| func(value));
511 let mut id = BenchmarkId::new(
512 group_id.to_owned(),
513 function_id,
514 value_str,
515 throughput.clone(),
516 );
517
518 id.ensure_directory_name_unique(&c.all_directories);
519 c.all_directories.insert(id.as_directory_name().to_owned());
520 id.ensure_title_unique(&c.all_titles);
521 c.all_titles.insert(id.as_title().to_owned());
522
523 let do_run = c.filter_matches(id.id());
524 any_matched |= do_run;
525
526 execute_benchmark(
527 do_run,
528 &id,
529 c,
530 &config,
531 &mut *routine.f.borrow_mut(),
532 &report_context,
533 value,
534 throughput,
535 );
536
537 all_ids.push(id);
538 }
539 }
540
541 if let Some(conn) = &c.connection {
542 conn.send(&OutgoingMessage::FinishedBenchmarkGroup { group: group_id })
543 .unwrap();
544 conn.serve_value_formatter(c.measurement.formatter())
545 .unwrap();
546 }
547
548 if all_ids.len() > 1 && any_matched && c.mode.is_benchmark() {
549 c.report
550 .summarize(&report_context, &all_ids, c.measurement.formatter());
551 }
552 if any_matched {
553 c.report.group_separator();
554 }
555 }
556 }
557
558 #[cfg_attr(feature = "cargo-clippy", allow(clippy::too_many_arguments))]
execute_benchmark<T, M>( do_run: bool, id: &BenchmarkId, c: &Criterion<M>, config: &BenchmarkConfig, routine: &mut dyn Routine<M, T>, report_context: &ReportContext, parameter: &T, throughput: Option<Throughput>, ) where T: Debug, M: Measurement,559 fn execute_benchmark<T, M>(
560 do_run: bool,
561 id: &BenchmarkId,
562 c: &Criterion<M>,
563 config: &BenchmarkConfig,
564 routine: &mut dyn Routine<M, T>,
565 report_context: &ReportContext,
566 parameter: &T,
567 throughput: Option<Throughput>,
568 ) where
569 T: Debug,
570 M: Measurement,
571 {
572 match c.mode {
573 Mode::Benchmark => {
574 if let Some(conn) = &c.connection {
575 if do_run {
576 conn.send(&OutgoingMessage::BeginningBenchmark { id: id.into() })
577 .unwrap();
578 } else {
579 conn.send(&OutgoingMessage::SkippingBenchmark { id: id.into() })
580 .unwrap();
581 }
582 }
583
584 if do_run {
585 analysis::common(
586 id,
587 routine,
588 config,
589 c,
590 report_context,
591 parameter,
592 throughput,
593 );
594 }
595 }
596 Mode::List => {
597 if do_run {
598 println!("{}: bench", id);
599 }
600 }
601 Mode::Test => {
602 if do_run {
603 // In test mode, run the benchmark exactly once, then exit.
604 c.report.test_start(id, report_context);
605 routine.test(&c.measurement, parameter);
606 c.report.test_pass(id, report_context);
607 }
608 }
609 Mode::Profile(duration) => {
610 if do_run {
611 routine.profile(&c.measurement, id, c, report_context, duration, parameter);
612 }
613 }
614 }
615 }
616