1 use crate::stats::univariate::outliers::tukey::LabeledSample;
2 use crate::{csv_report::FileCsvReport, stats::bivariate::regression::Slope};
3 use crate::{html::Html, stats::bivariate::Data};
4
5 use crate::estimate::{ChangeDistributions, ChangeEstimates, Distributions, Estimate, Estimates};
6 use crate::format;
7 use crate::measurement::ValueFormatter;
8 use crate::stats::univariate::Sample;
9 use crate::stats::Distribution;
10 use crate::{PlotConfiguration, Throughput};
11 use std::cell::Cell;
12 use std::cmp;
13 use std::collections::HashSet;
14 use std::fmt;
15 use std::io::stdout;
16 use std::io::Write;
17 use std::path::{Path, PathBuf};
18
19 const MAX_DIRECTORY_NAME_LEN: usize = 64;
20 const MAX_TITLE_LEN: usize = 100;
21
22 pub(crate) struct ComparisonData {
23 pub p_value: f64,
24 pub t_distribution: Distribution<f64>,
25 pub t_value: f64,
26 pub relative_estimates: ChangeEstimates,
27 pub relative_distributions: ChangeDistributions,
28 pub significance_threshold: f64,
29 pub noise_threshold: f64,
30 pub base_iter_counts: Vec<f64>,
31 pub base_sample_times: Vec<f64>,
32 pub base_avg_times: Vec<f64>,
33 pub base_estimates: Estimates,
34 }
35
36 pub(crate) struct MeasurementData<'a> {
37 pub data: Data<'a, f64, f64>,
38 pub avg_times: LabeledSample<'a, f64>,
39 pub absolute_estimates: Estimates,
40 pub distributions: Distributions,
41 pub comparison: Option<ComparisonData>,
42 pub throughput: Option<Throughput>,
43 }
44 impl<'a> MeasurementData<'a> {
iter_counts(&self) -> &Sample<f64>45 pub fn iter_counts(&self) -> &Sample<f64> {
46 self.data.x()
47 }
48
sample_times(&self) -> &Sample<f64>49 pub fn sample_times(&self) -> &Sample<f64> {
50 self.data.y()
51 }
52 }
53
54 #[derive(Debug, Clone, Copy, Eq, PartialEq)]
55 pub enum ValueType {
56 Bytes,
57 Elements,
58 Value,
59 }
60
61 #[derive(Clone, Serialize, Deserialize, PartialEq)]
62 pub struct BenchmarkId {
63 pub group_id: String,
64 pub function_id: Option<String>,
65 pub value_str: Option<String>,
66 pub throughput: Option<Throughput>,
67 full_id: String,
68 directory_name: String,
69 title: String,
70 }
71
truncate_to_character_boundary(s: &mut String, max_len: usize)72 fn truncate_to_character_boundary(s: &mut String, max_len: usize) {
73 let mut boundary = cmp::min(max_len, s.len());
74 while !s.is_char_boundary(boundary) {
75 boundary -= 1;
76 }
77 s.truncate(boundary);
78 }
79
make_filename_safe(string: &str) -> String80 pub fn make_filename_safe(string: &str) -> String {
81 let mut string = string.replace(
82 &['?', '"', '/', '\\', '*', '<', '>', ':', '|', '^'][..],
83 "_",
84 );
85
86 // Truncate to last character boundary before max length...
87 truncate_to_character_boundary(&mut string, MAX_DIRECTORY_NAME_LEN);
88
89 if cfg!(target_os = "windows") {
90 {
91 string = string
92 // On Windows, spaces in the end of the filename are ignored and will be trimmed.
93 //
94 // Without trimming ourselves, creating a directory `dir ` will silently create
95 // `dir` instead, but then operations on files like `dir /file` will fail.
96 //
97 // Also note that it's important to do this *after* trimming to MAX_DIRECTORY_NAME_LEN,
98 // otherwise it can trim again to a name with a trailing space.
99 .trim_end()
100 // On Windows, file names are not case-sensitive, so lowercase everything.
101 .to_lowercase();
102 }
103 }
104
105 string
106 }
107
108 impl BenchmarkId {
new( group_id: String, function_id: Option<String>, value_str: Option<String>, throughput: Option<Throughput>, ) -> BenchmarkId109 pub fn new(
110 group_id: String,
111 function_id: Option<String>,
112 value_str: Option<String>,
113 throughput: Option<Throughput>,
114 ) -> BenchmarkId {
115 let full_id = match (&function_id, &value_str) {
116 (&Some(ref func), &Some(ref val)) => format!("{}/{}/{}", group_id, func, val),
117 (&Some(ref func), &None) => format!("{}/{}", group_id, func),
118 (&None, &Some(ref val)) => format!("{}/{}", group_id, val),
119 (&None, &None) => group_id.clone(),
120 };
121
122 let mut title = full_id.clone();
123 truncate_to_character_boundary(&mut title, MAX_TITLE_LEN);
124 if title != full_id {
125 title.push_str("...");
126 }
127
128 let directory_name = match (&function_id, &value_str) {
129 (&Some(ref func), &Some(ref val)) => format!(
130 "{}/{}/{}",
131 make_filename_safe(&group_id),
132 make_filename_safe(func),
133 make_filename_safe(val)
134 ),
135 (&Some(ref func), &None) => format!(
136 "{}/{}",
137 make_filename_safe(&group_id),
138 make_filename_safe(func)
139 ),
140 (&None, &Some(ref val)) => format!(
141 "{}/{}",
142 make_filename_safe(&group_id),
143 make_filename_safe(val)
144 ),
145 (&None, &None) => make_filename_safe(&group_id),
146 };
147
148 BenchmarkId {
149 group_id,
150 function_id,
151 value_str,
152 throughput,
153 full_id,
154 directory_name,
155 title,
156 }
157 }
158
id(&self) -> &str159 pub fn id(&self) -> &str {
160 &self.full_id
161 }
162
as_title(&self) -> &str163 pub fn as_title(&self) -> &str {
164 &self.title
165 }
166
as_directory_name(&self) -> &str167 pub fn as_directory_name(&self) -> &str {
168 &self.directory_name
169 }
170
as_number(&self) -> Option<f64>171 pub fn as_number(&self) -> Option<f64> {
172 match self.throughput {
173 Some(Throughput::Bytes(n)) | Some(Throughput::Elements(n)) => Some(n as f64),
174 None => self
175 .value_str
176 .as_ref()
177 .and_then(|string| string.parse::<f64>().ok()),
178 }
179 }
180
value_type(&self) -> Option<ValueType>181 pub fn value_type(&self) -> Option<ValueType> {
182 match self.throughput {
183 Some(Throughput::Bytes(_)) => Some(ValueType::Bytes),
184 Some(Throughput::Elements(_)) => Some(ValueType::Elements),
185 None => self
186 .value_str
187 .as_ref()
188 .and_then(|string| string.parse::<f64>().ok())
189 .map(|_| ValueType::Value),
190 }
191 }
192
ensure_directory_name_unique(&mut self, existing_directories: &HashSet<String>)193 pub fn ensure_directory_name_unique(&mut self, existing_directories: &HashSet<String>) {
194 if !existing_directories.contains(self.as_directory_name()) {
195 return;
196 }
197
198 let mut counter = 2;
199 loop {
200 let new_dir_name = format!("{}_{}", self.as_directory_name(), counter);
201 if !existing_directories.contains(&new_dir_name) {
202 self.directory_name = new_dir_name;
203 return;
204 }
205 counter += 1;
206 }
207 }
208
ensure_title_unique(&mut self, existing_titles: &HashSet<String>)209 pub fn ensure_title_unique(&mut self, existing_titles: &HashSet<String>) {
210 if !existing_titles.contains(self.as_title()) {
211 return;
212 }
213
214 let mut counter = 2;
215 loop {
216 let new_title = format!("{} #{}", self.as_title(), counter);
217 if !existing_titles.contains(&new_title) {
218 self.title = new_title;
219 return;
220 }
221 counter += 1;
222 }
223 }
224 }
225 impl fmt::Display for BenchmarkId {
fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result226 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
227 f.write_str(self.as_title())
228 }
229 }
230 impl fmt::Debug for BenchmarkId {
fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result231 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
232 fn format_opt(opt: &Option<String>) -> String {
233 match *opt {
234 Some(ref string) => format!("\"{}\"", string),
235 None => "None".to_owned(),
236 }
237 }
238
239 write!(
240 f,
241 "BenchmarkId {{ group_id: \"{}\", function_id: {}, value_str: {}, throughput: {:?} }}",
242 self.group_id,
243 format_opt(&self.function_id),
244 format_opt(&self.value_str),
245 self.throughput,
246 )
247 }
248 }
249
250 pub struct ReportContext {
251 pub output_directory: PathBuf,
252 pub plot_config: PlotConfiguration,
253 }
254 impl ReportContext {
report_path<P: AsRef<Path> + ?Sized>(&self, id: &BenchmarkId, file_name: &P) -> PathBuf255 pub fn report_path<P: AsRef<Path> + ?Sized>(&self, id: &BenchmarkId, file_name: &P) -> PathBuf {
256 let mut path = self.output_directory.clone();
257 path.push(id.as_directory_name());
258 path.push("report");
259 path.push(file_name);
260 path
261 }
262 }
263
264 pub(crate) trait Report {
test_start(&self, _id: &BenchmarkId, _context: &ReportContext)265 fn test_start(&self, _id: &BenchmarkId, _context: &ReportContext) {}
test_pass(&self, _id: &BenchmarkId, _context: &ReportContext)266 fn test_pass(&self, _id: &BenchmarkId, _context: &ReportContext) {}
267
benchmark_start(&self, _id: &BenchmarkId, _context: &ReportContext)268 fn benchmark_start(&self, _id: &BenchmarkId, _context: &ReportContext) {}
profile(&self, _id: &BenchmarkId, _context: &ReportContext, _profile_ns: f64)269 fn profile(&self, _id: &BenchmarkId, _context: &ReportContext, _profile_ns: f64) {}
warmup(&self, _id: &BenchmarkId, _context: &ReportContext, _warmup_ns: f64)270 fn warmup(&self, _id: &BenchmarkId, _context: &ReportContext, _warmup_ns: f64) {}
terminated(&self, _id: &BenchmarkId, _context: &ReportContext)271 fn terminated(&self, _id: &BenchmarkId, _context: &ReportContext) {}
analysis(&self, _id: &BenchmarkId, _context: &ReportContext)272 fn analysis(&self, _id: &BenchmarkId, _context: &ReportContext) {}
measurement_start( &self, _id: &BenchmarkId, _context: &ReportContext, _sample_count: u64, _estimate_ns: f64, _iter_count: u64, )273 fn measurement_start(
274 &self,
275 _id: &BenchmarkId,
276 _context: &ReportContext,
277 _sample_count: u64,
278 _estimate_ns: f64,
279 _iter_count: u64,
280 ) {
281 }
measurement_complete( &self, _id: &BenchmarkId, _context: &ReportContext, _measurements: &MeasurementData<'_>, _formatter: &dyn ValueFormatter, )282 fn measurement_complete(
283 &self,
284 _id: &BenchmarkId,
285 _context: &ReportContext,
286 _measurements: &MeasurementData<'_>,
287 _formatter: &dyn ValueFormatter,
288 ) {
289 }
summarize( &self, _context: &ReportContext, _all_ids: &[BenchmarkId], _formatter: &dyn ValueFormatter, )290 fn summarize(
291 &self,
292 _context: &ReportContext,
293 _all_ids: &[BenchmarkId],
294 _formatter: &dyn ValueFormatter,
295 ) {
296 }
final_summary(&self, _context: &ReportContext)297 fn final_summary(&self, _context: &ReportContext) {}
group_separator(&self)298 fn group_separator(&self) {}
299 }
300
301 pub(crate) struct Reports {
302 pub(crate) cli_enabled: bool,
303 pub(crate) cli: CliReport,
304 pub(crate) bencher_enabled: bool,
305 pub(crate) bencher: BencherReport,
306 pub(crate) csv_enabled: bool,
307 pub(crate) csv: FileCsvReport,
308 pub(crate) html_enabled: bool,
309 pub(crate) html: Html,
310 }
311 macro_rules! reports_impl {
312 (fn $name:ident(&self, $($argn:ident: $argt:ty),*)) => {
313 fn $name(&self, $($argn: $argt),* ) {
314 if self.cli_enabled {
315 self.cli.$name($($argn),*);
316 }
317 if self.bencher_enabled {
318 self.bencher.$name($($argn),*);
319 }
320 if self.csv_enabled {
321 self.csv.$name($($argn),*);
322 }
323 if self.html_enabled {
324 self.html.$name($($argn),*);
325 }
326 }
327 };
328 }
329
330 impl Report for Reports {
331 reports_impl!(fn test_start(&self, id: &BenchmarkId, context: &ReportContext));
332 reports_impl!(fn test_pass(&self, id: &BenchmarkId, context: &ReportContext));
333 reports_impl!(fn benchmark_start(&self, id: &BenchmarkId, context: &ReportContext));
334 reports_impl!(fn profile(&self, id: &BenchmarkId, context: &ReportContext, profile_ns: f64));
335 reports_impl!(fn warmup(&self, id: &BenchmarkId, context: &ReportContext, warmup_ns: f64));
336 reports_impl!(fn terminated(&self, id: &BenchmarkId, context: &ReportContext));
337 reports_impl!(fn analysis(&self, id: &BenchmarkId, context: &ReportContext));
338 reports_impl!(fn measurement_start(
339 &self,
340 id: &BenchmarkId,
341 context: &ReportContext,
342 sample_count: u64,
343 estimate_ns: f64,
344 iter_count: u64
345 ));
346 reports_impl!(
347 fn measurement_complete(
348 &self,
349 id: &BenchmarkId,
350 context: &ReportContext,
351 measurements: &MeasurementData<'_>,
352 formatter: &dyn ValueFormatter
353 ));
354 reports_impl!(
355 fn summarize(
356 &self,
357 context: &ReportContext,
358 all_ids: &[BenchmarkId],
359 formatter: &dyn ValueFormatter
360 ));
361
362 reports_impl!(fn final_summary(&self, context: &ReportContext));
363 reports_impl!(fn group_separator(&self, ));
364 }
365
366 pub(crate) struct CliReport {
367 pub enable_text_overwrite: bool,
368 pub enable_text_coloring: bool,
369 pub verbose: bool,
370
371 last_line_len: Cell<usize>,
372 }
373 impl CliReport {
new( enable_text_overwrite: bool, enable_text_coloring: bool, verbose: bool, ) -> CliReport374 pub fn new(
375 enable_text_overwrite: bool,
376 enable_text_coloring: bool,
377 verbose: bool,
378 ) -> CliReport {
379 CliReport {
380 enable_text_overwrite,
381 enable_text_coloring,
382 verbose,
383
384 last_line_len: Cell::new(0),
385 }
386 }
387
text_overwrite(&self)388 fn text_overwrite(&self) {
389 if self.enable_text_overwrite {
390 print!("\r");
391 for _ in 0..self.last_line_len.get() {
392 print!(" ");
393 }
394 print!("\r");
395 }
396 }
397
398 // Passing a String is the common case here.
399 #[cfg_attr(feature = "cargo-clippy", allow(clippy::needless_pass_by_value))]
print_overwritable(&self, s: String)400 fn print_overwritable(&self, s: String) {
401 if self.enable_text_overwrite {
402 self.last_line_len.set(s.len());
403 print!("{}", s);
404 stdout().flush().unwrap();
405 } else {
406 println!("{}", s);
407 }
408 }
409
green(&self, s: String) -> String410 fn green(&self, s: String) -> String {
411 if self.enable_text_coloring {
412 format!("\x1B[32m{}\x1B[39m", s)
413 } else {
414 s
415 }
416 }
417
yellow(&self, s: String) -> String418 fn yellow(&self, s: String) -> String {
419 if self.enable_text_coloring {
420 format!("\x1B[33m{}\x1B[39m", s)
421 } else {
422 s
423 }
424 }
425
red(&self, s: String) -> String426 fn red(&self, s: String) -> String {
427 if self.enable_text_coloring {
428 format!("\x1B[31m{}\x1B[39m", s)
429 } else {
430 s
431 }
432 }
433
bold(&self, s: String) -> String434 fn bold(&self, s: String) -> String {
435 if self.enable_text_coloring {
436 format!("\x1B[1m{}\x1B[22m", s)
437 } else {
438 s
439 }
440 }
441
faint(&self, s: String) -> String442 fn faint(&self, s: String) -> String {
443 if self.enable_text_coloring {
444 format!("\x1B[2m{}\x1B[22m", s)
445 } else {
446 s
447 }
448 }
449
outliers(&self, sample: &LabeledSample<'_, f64>)450 pub fn outliers(&self, sample: &LabeledSample<'_, f64>) {
451 let (los, lom, _, him, his) = sample.count();
452 let noutliers = los + lom + him + his;
453 let sample_size = sample.len();
454
455 if noutliers == 0 {
456 return;
457 }
458
459 let percent = |n: usize| 100. * n as f64 / sample_size as f64;
460
461 println!(
462 "{}",
463 self.yellow(format!(
464 "Found {} outliers among {} measurements ({:.2}%)",
465 noutliers,
466 sample_size,
467 percent(noutliers)
468 ))
469 );
470
471 let print = |n, label| {
472 if n != 0 {
473 println!(" {} ({:.2}%) {}", n, percent(n), label);
474 }
475 };
476
477 print(los, "low severe");
478 print(lom, "low mild");
479 print(him, "high mild");
480 print(his, "high severe");
481 }
482 }
483 impl Report for CliReport {
test_start(&self, id: &BenchmarkId, _: &ReportContext)484 fn test_start(&self, id: &BenchmarkId, _: &ReportContext) {
485 println!("Testing {}", id);
486 }
test_pass(&self, _: &BenchmarkId, _: &ReportContext)487 fn test_pass(&self, _: &BenchmarkId, _: &ReportContext) {
488 println!("Success");
489 }
490
benchmark_start(&self, id: &BenchmarkId, _: &ReportContext)491 fn benchmark_start(&self, id: &BenchmarkId, _: &ReportContext) {
492 self.print_overwritable(format!("Benchmarking {}", id));
493 }
494
profile(&self, id: &BenchmarkId, _: &ReportContext, warmup_ns: f64)495 fn profile(&self, id: &BenchmarkId, _: &ReportContext, warmup_ns: f64) {
496 self.text_overwrite();
497 self.print_overwritable(format!(
498 "Benchmarking {}: Profiling for {}",
499 id,
500 format::time(warmup_ns)
501 ));
502 }
503
warmup(&self, id: &BenchmarkId, _: &ReportContext, warmup_ns: f64)504 fn warmup(&self, id: &BenchmarkId, _: &ReportContext, warmup_ns: f64) {
505 self.text_overwrite();
506 self.print_overwritable(format!(
507 "Benchmarking {}: Warming up for {}",
508 id,
509 format::time(warmup_ns)
510 ));
511 }
512
terminated(&self, id: &BenchmarkId, _: &ReportContext)513 fn terminated(&self, id: &BenchmarkId, _: &ReportContext) {
514 self.text_overwrite();
515 println!("Benchmarking {}: Complete (Analysis Disabled)", id);
516 }
517
analysis(&self, id: &BenchmarkId, _: &ReportContext)518 fn analysis(&self, id: &BenchmarkId, _: &ReportContext) {
519 self.text_overwrite();
520 self.print_overwritable(format!("Benchmarking {}: Analyzing", id));
521 }
522
measurement_start( &self, id: &BenchmarkId, _: &ReportContext, sample_count: u64, estimate_ns: f64, iter_count: u64, )523 fn measurement_start(
524 &self,
525 id: &BenchmarkId,
526 _: &ReportContext,
527 sample_count: u64,
528 estimate_ns: f64,
529 iter_count: u64,
530 ) {
531 self.text_overwrite();
532 let iter_string = if self.verbose {
533 format!("{} iterations", iter_count)
534 } else {
535 format::iter_count(iter_count)
536 };
537
538 self.print_overwritable(format!(
539 "Benchmarking {}: Collecting {} samples in estimated {} ({})",
540 id,
541 sample_count,
542 format::time(estimate_ns),
543 iter_string
544 ));
545 }
546
measurement_complete( &self, id: &BenchmarkId, _: &ReportContext, meas: &MeasurementData<'_>, formatter: &dyn ValueFormatter, )547 fn measurement_complete(
548 &self,
549 id: &BenchmarkId,
550 _: &ReportContext,
551 meas: &MeasurementData<'_>,
552 formatter: &dyn ValueFormatter,
553 ) {
554 self.text_overwrite();
555
556 let typical_estimate = &meas.absolute_estimates.typical();
557
558 {
559 let mut id = id.as_title().to_owned();
560
561 if id.len() > 23 {
562 println!("{}", self.green(id.clone()));
563 id.clear();
564 }
565 let id_len = id.len();
566
567 println!(
568 "{}{}time: [{} {} {}]",
569 self.green(id),
570 " ".repeat(24 - id_len),
571 self.faint(
572 formatter.format_value(typical_estimate.confidence_interval.lower_bound)
573 ),
574 self.bold(formatter.format_value(typical_estimate.point_estimate)),
575 self.faint(
576 formatter.format_value(typical_estimate.confidence_interval.upper_bound)
577 )
578 );
579 }
580
581 if let Some(ref throughput) = meas.throughput {
582 println!(
583 "{}thrpt: [{} {} {}]",
584 " ".repeat(24),
585 self.faint(formatter.format_throughput(
586 throughput,
587 typical_estimate.confidence_interval.upper_bound
588 )),
589 self.bold(formatter.format_throughput(throughput, typical_estimate.point_estimate)),
590 self.faint(formatter.format_throughput(
591 throughput,
592 typical_estimate.confidence_interval.lower_bound
593 )),
594 )
595 }
596
597 if let Some(ref comp) = meas.comparison {
598 let different_mean = comp.p_value < comp.significance_threshold;
599 let mean_est = &comp.relative_estimates.mean;
600 let point_estimate = mean_est.point_estimate;
601 let mut point_estimate_str = format::change(point_estimate, true);
602 // The change in throughput is related to the change in timing. Reducing the timing by
603 // 50% increases the throughput by 100%.
604 let to_thrpt_estimate = |ratio: f64| 1.0 / (1.0 + ratio) - 1.0;
605 let mut thrpt_point_estimate_str =
606 format::change(to_thrpt_estimate(point_estimate), true);
607 let explanation_str: String;
608
609 if !different_mean {
610 explanation_str = "No change in performance detected.".to_owned();
611 } else {
612 let comparison = compare_to_threshold(&mean_est, comp.noise_threshold);
613 match comparison {
614 ComparisonResult::Improved => {
615 point_estimate_str = self.green(self.bold(point_estimate_str));
616 thrpt_point_estimate_str = self.green(self.bold(thrpt_point_estimate_str));
617 explanation_str =
618 format!("Performance has {}.", self.green("improved".to_owned()));
619 }
620 ComparisonResult::Regressed => {
621 point_estimate_str = self.red(self.bold(point_estimate_str));
622 thrpt_point_estimate_str = self.red(self.bold(thrpt_point_estimate_str));
623 explanation_str =
624 format!("Performance has {}.", self.red("regressed".to_owned()));
625 }
626 ComparisonResult::NonSignificant => {
627 explanation_str = "Change within noise threshold.".to_owned();
628 }
629 }
630 }
631
632 if meas.throughput.is_some() {
633 println!("{}change:", " ".repeat(17));
634
635 println!(
636 "{}time: [{} {} {}] (p = {:.2} {} {:.2})",
637 " ".repeat(24),
638 self.faint(format::change(
639 mean_est.confidence_interval.lower_bound,
640 true
641 )),
642 point_estimate_str,
643 self.faint(format::change(
644 mean_est.confidence_interval.upper_bound,
645 true
646 )),
647 comp.p_value,
648 if different_mean { "<" } else { ">" },
649 comp.significance_threshold
650 );
651 println!(
652 "{}thrpt: [{} {} {}]",
653 " ".repeat(24),
654 self.faint(format::change(
655 to_thrpt_estimate(mean_est.confidence_interval.upper_bound),
656 true
657 )),
658 thrpt_point_estimate_str,
659 self.faint(format::change(
660 to_thrpt_estimate(mean_est.confidence_interval.lower_bound),
661 true
662 )),
663 );
664 } else {
665 println!(
666 "{}change: [{} {} {}] (p = {:.2} {} {:.2})",
667 " ".repeat(24),
668 self.faint(format::change(
669 mean_est.confidence_interval.lower_bound,
670 true
671 )),
672 point_estimate_str,
673 self.faint(format::change(
674 mean_est.confidence_interval.upper_bound,
675 true
676 )),
677 comp.p_value,
678 if different_mean { "<" } else { ">" },
679 comp.significance_threshold
680 );
681 }
682
683 println!("{}{}", " ".repeat(24), explanation_str);
684 }
685
686 self.outliers(&meas.avg_times);
687
688 if self.verbose {
689 let format_short_estimate = |estimate: &Estimate| -> String {
690 format!(
691 "[{} {}]",
692 formatter.format_value(estimate.confidence_interval.lower_bound),
693 formatter.format_value(estimate.confidence_interval.upper_bound)
694 )
695 };
696
697 let data = &meas.data;
698 if let Some(slope_estimate) = meas.absolute_estimates.slope.as_ref() {
699 println!(
700 "{:<7}{} {:<15}[{:0.7} {:0.7}]",
701 "slope",
702 format_short_estimate(slope_estimate),
703 "R^2",
704 Slope(slope_estimate.confidence_interval.lower_bound).r_squared(data),
705 Slope(slope_estimate.confidence_interval.upper_bound).r_squared(data),
706 );
707 }
708 println!(
709 "{:<7}{} {:<15}{}",
710 "mean",
711 format_short_estimate(&meas.absolute_estimates.mean),
712 "std. dev.",
713 format_short_estimate(&meas.absolute_estimates.std_dev),
714 );
715 println!(
716 "{:<7}{} {:<15}{}",
717 "median",
718 format_short_estimate(&meas.absolute_estimates.median),
719 "med. abs. dev.",
720 format_short_estimate(&meas.absolute_estimates.median_abs_dev),
721 );
722 }
723 }
724
group_separator(&self)725 fn group_separator(&self) {
726 println!();
727 }
728 }
729
730 pub struct BencherReport;
731 impl Report for BencherReport {
measurement_start( &self, id: &BenchmarkId, _context: &ReportContext, _sample_count: u64, _estimate_ns: f64, _iter_count: u64, )732 fn measurement_start(
733 &self,
734 id: &BenchmarkId,
735 _context: &ReportContext,
736 _sample_count: u64,
737 _estimate_ns: f64,
738 _iter_count: u64,
739 ) {
740 print!("test {} ... ", id);
741 }
742
measurement_complete( &self, _id: &BenchmarkId, _: &ReportContext, meas: &MeasurementData<'_>, formatter: &dyn ValueFormatter, )743 fn measurement_complete(
744 &self,
745 _id: &BenchmarkId,
746 _: &ReportContext,
747 meas: &MeasurementData<'_>,
748 formatter: &dyn ValueFormatter,
749 ) {
750 let mut values = [
751 meas.absolute_estimates.median.point_estimate,
752 meas.absolute_estimates.std_dev.point_estimate,
753 ];
754 let unit = formatter.scale_for_machines(&mut values);
755
756 println!(
757 "bench: {:>11} {}/iter (+/- {})",
758 format::integer(values[0]),
759 unit,
760 format::integer(values[1])
761 );
762 }
763
group_separator(&self)764 fn group_separator(&self) {
765 println!();
766 }
767 }
768
769 enum ComparisonResult {
770 Improved,
771 Regressed,
772 NonSignificant,
773 }
774
compare_to_threshold(estimate: &Estimate, noise: f64) -> ComparisonResult775 fn compare_to_threshold(estimate: &Estimate, noise: f64) -> ComparisonResult {
776 let ci = &estimate.confidence_interval;
777 let lb = ci.lower_bound;
778 let ub = ci.upper_bound;
779
780 if lb < -noise && ub < -noise {
781 ComparisonResult::Improved
782 } else if lb > noise && ub > noise {
783 ComparisonResult::Regressed
784 } else {
785 ComparisonResult::NonSignificant
786 }
787 }
788
789 #[cfg(test)]
790 mod test {
791 use super::*;
792
793 #[test]
test_make_filename_safe_replaces_characters()794 fn test_make_filename_safe_replaces_characters() {
795 let input = "?/\\*\"";
796 let safe = make_filename_safe(input);
797 assert_eq!("_____", &safe);
798 }
799
800 #[test]
test_make_filename_safe_truncates_long_strings()801 fn test_make_filename_safe_truncates_long_strings() {
802 let input = "this is a very long string. it is too long to be safe as a directory name, and so it needs to be truncated. what a long string this is.";
803 let safe = make_filename_safe(input);
804 assert!(input.len() > MAX_DIRECTORY_NAME_LEN);
805 assert_eq!(&input[0..MAX_DIRECTORY_NAME_LEN], &safe);
806 }
807
808 #[test]
test_make_filename_safe_respects_character_boundaries()809 fn test_make_filename_safe_respects_character_boundaries() {
810 let input = "✓✓✓✓✓✓✓✓✓✓✓✓✓✓✓✓✓✓✓✓✓✓✓✓✓✓✓✓✓✓✓✓✓✓";
811 let safe = make_filename_safe(input);
812 assert!(safe.len() < MAX_DIRECTORY_NAME_LEN);
813 }
814
815 #[test]
test_benchmark_id_make_directory_name_unique()816 fn test_benchmark_id_make_directory_name_unique() {
817 let existing_id = BenchmarkId::new(
818 "group".to_owned(),
819 Some("function".to_owned()),
820 Some("value".to_owned()),
821 None,
822 );
823 let mut directories = HashSet::new();
824 directories.insert(existing_id.as_directory_name().to_owned());
825
826 let mut new_id = existing_id.clone();
827 new_id.ensure_directory_name_unique(&directories);
828 assert_eq!("group/function/value_2", new_id.as_directory_name());
829 directories.insert(new_id.as_directory_name().to_owned());
830
831 new_id = existing_id.clone();
832 new_id.ensure_directory_name_unique(&directories);
833 assert_eq!("group/function/value_3", new_id.as_directory_name());
834 directories.insert(new_id.as_directory_name().to_owned());
835 }
836 #[test]
test_benchmark_id_make_long_directory_name_unique()837 fn test_benchmark_id_make_long_directory_name_unique() {
838 let long_name = (0..MAX_DIRECTORY_NAME_LEN).map(|_| 'a').collect::<String>();
839 let existing_id = BenchmarkId::new(long_name, None, None, None);
840 let mut directories = HashSet::new();
841 directories.insert(existing_id.as_directory_name().to_owned());
842
843 let mut new_id = existing_id.clone();
844 new_id.ensure_directory_name_unique(&directories);
845 assert_ne!(existing_id.as_directory_name(), new_id.as_directory_name());
846 }
847 }
848