1 use super::*;
2
3 use crate::{
4 bench::Bencher,
5 console::OutputLocation,
6 formatters::PrettyFormatter,
7 options::OutputFormat,
8 test::{
9 filter_tests,
10 parse_opts,
11 run_test,
12 DynTestFn,
13 DynTestName,
14 MetricMap,
15 RunIgnored,
16 RunStrategy,
17 ShouldPanic,
18 StaticTestName,
19 TestDesc,
20 TestDescAndFn,
21 TestOpts,
22 TrIgnored,
23 TrOk,
24 // FIXME (introduced by #65251)
25 // ShouldPanic, StaticTestName, TestDesc, TestDescAndFn, TestOpts, TestTimeOptions,
26 // TestType, TrFailedMsg, TrIgnored, TrOk,
27 },
28 time::{TestTimeOptions, TimeThreshold},
29 };
30 use std::sync::mpsc::channel;
31 use std::time::Duration;
32
33 impl TestOpts {
new() -> TestOpts34 fn new() -> TestOpts {
35 TestOpts {
36 list: false,
37 filters: vec![],
38 filter_exact: false,
39 force_run_in_process: false,
40 exclude_should_panic: false,
41 run_ignored: RunIgnored::No,
42 run_tests: false,
43 bench_benchmarks: false,
44 logfile: None,
45 nocapture: false,
46 color: AutoColor,
47 format: OutputFormat::Pretty,
48 shuffle: false,
49 shuffle_seed: None,
50 test_threads: None,
51 skip: vec![],
52 time_options: None,
53 options: Options::new(),
54 fail_fast: false,
55 }
56 }
57 }
58
one_ignored_one_unignored_test() -> Vec<TestDescAndFn>59 fn one_ignored_one_unignored_test() -> Vec<TestDescAndFn> {
60 vec![
61 TestDescAndFn {
62 desc: TestDesc {
63 name: StaticTestName("1"),
64 ignore: true,
65 ignore_message: None,
66 source_file: "",
67 start_line: 0,
68 start_col: 0,
69 end_line: 0,
70 end_col: 0,
71 should_panic: ShouldPanic::No,
72 compile_fail: false,
73 no_run: false,
74 test_type: TestType::Unknown,
75 },
76 testfn: DynTestFn(Box::new(move || Ok(()))),
77 },
78 TestDescAndFn {
79 desc: TestDesc {
80 name: StaticTestName("2"),
81 ignore: false,
82 ignore_message: None,
83 source_file: "",
84 start_line: 0,
85 start_col: 0,
86 end_line: 0,
87 end_col: 0,
88 should_panic: ShouldPanic::No,
89 compile_fail: false,
90 no_run: false,
91 test_type: TestType::Unknown,
92 },
93 testfn: DynTestFn(Box::new(move || Ok(()))),
94 },
95 ]
96 }
97
98 #[test]
do_not_run_ignored_tests()99 pub fn do_not_run_ignored_tests() {
100 fn f() -> Result<(), String> {
101 panic!();
102 }
103 let desc = TestDescAndFn {
104 desc: TestDesc {
105 name: StaticTestName("whatever"),
106 ignore: true,
107 ignore_message: None,
108 source_file: "",
109 start_line: 0,
110 start_col: 0,
111 end_line: 0,
112 end_col: 0,
113 should_panic: ShouldPanic::No,
114 compile_fail: false,
115 no_run: false,
116 test_type: TestType::Unknown,
117 },
118 testfn: DynTestFn(Box::new(f)),
119 };
120 let (tx, rx) = channel();
121 run_test(&TestOpts::new(), false, TestId(0), desc, RunStrategy::InProcess, tx);
122 let result = rx.recv().unwrap().result;
123 assert_ne!(result, TrOk);
124 }
125
126 #[test]
ignored_tests_result_in_ignored()127 pub fn ignored_tests_result_in_ignored() {
128 fn f() -> Result<(), String> {
129 Ok(())
130 }
131 let desc = TestDescAndFn {
132 desc: TestDesc {
133 name: StaticTestName("whatever"),
134 ignore: true,
135 ignore_message: None,
136 source_file: "",
137 start_line: 0,
138 start_col: 0,
139 end_line: 0,
140 end_col: 0,
141 should_panic: ShouldPanic::No,
142 compile_fail: false,
143 no_run: false,
144 test_type: TestType::Unknown,
145 },
146 testfn: DynTestFn(Box::new(f)),
147 };
148 let (tx, rx) = channel();
149 run_test(&TestOpts::new(), false, TestId(0), desc, RunStrategy::InProcess, tx);
150 let result = rx.recv().unwrap().result;
151 assert_eq!(result, TrIgnored);
152 }
153
154 // FIXME: Re-enable emscripten once it can catch panics again (introduced by #65251)
155 #[test]
156 #[cfg(not(target_os = "emscripten"))]
157 #[cfg_attr(not(panic = "unwind"), ignore = "test requires unwinding support")]
test_should_panic()158 fn test_should_panic() {
159 fn f() -> Result<(), String> {
160 panic!();
161 }
162 let desc = TestDescAndFn {
163 desc: TestDesc {
164 name: StaticTestName("whatever"),
165 ignore: false,
166 ignore_message: None,
167 source_file: "",
168 start_line: 0,
169 start_col: 0,
170 end_line: 0,
171 end_col: 0,
172 should_panic: ShouldPanic::Yes,
173 compile_fail: false,
174 no_run: false,
175 test_type: TestType::Unknown,
176 },
177 testfn: DynTestFn(Box::new(f)),
178 };
179 let (tx, rx) = channel();
180 run_test(&TestOpts::new(), false, TestId(0), desc, RunStrategy::InProcess, tx);
181 let result = rx.recv().unwrap().result;
182 assert_eq!(result, TrOk);
183 }
184
185 // FIXME: Re-enable emscripten once it can catch panics again (introduced by #65251)
186 #[test]
187 #[cfg(not(target_os = "emscripten"))]
188 #[cfg_attr(not(panic = "unwind"), ignore = "test requires unwinding support")]
test_should_panic_good_message()189 fn test_should_panic_good_message() {
190 fn f() -> Result<(), String> {
191 panic!("an error message");
192 }
193 let desc = TestDescAndFn {
194 desc: TestDesc {
195 name: StaticTestName("whatever"),
196 ignore: false,
197 ignore_message: None,
198 source_file: "",
199 start_line: 0,
200 start_col: 0,
201 end_line: 0,
202 end_col: 0,
203 should_panic: ShouldPanic::YesWithMessage("error message"),
204 compile_fail: false,
205 no_run: false,
206 test_type: TestType::Unknown,
207 },
208 testfn: DynTestFn(Box::new(f)),
209 };
210 let (tx, rx) = channel();
211 run_test(&TestOpts::new(), false, TestId(0), desc, RunStrategy::InProcess, tx);
212 let result = rx.recv().unwrap().result;
213 assert_eq!(result, TrOk);
214 }
215
216 // FIXME: Re-enable emscripten once it can catch panics again (introduced by #65251)
217 #[test]
218 #[cfg(not(target_os = "emscripten"))]
219 #[cfg_attr(not(panic = "unwind"), ignore = "test requires unwinding support")]
test_should_panic_bad_message()220 fn test_should_panic_bad_message() {
221 use crate::tests::TrFailedMsg;
222 fn f() -> Result<(), String> {
223 panic!("an error message");
224 }
225 let expected = "foobar";
226 let failed_msg = r#"panic did not contain expected string
227 panic message: `"an error message"`,
228 expected substring: `"foobar"`"#;
229 let desc = TestDescAndFn {
230 desc: TestDesc {
231 name: StaticTestName("whatever"),
232 ignore: false,
233 ignore_message: None,
234 source_file: "",
235 start_line: 0,
236 start_col: 0,
237 end_line: 0,
238 end_col: 0,
239 should_panic: ShouldPanic::YesWithMessage(expected),
240 compile_fail: false,
241 no_run: false,
242 test_type: TestType::Unknown,
243 },
244 testfn: DynTestFn(Box::new(f)),
245 };
246 let (tx, rx) = channel();
247 run_test(&TestOpts::new(), false, TestId(0), desc, RunStrategy::InProcess, tx);
248 let result = rx.recv().unwrap().result;
249 assert_eq!(result, TrFailedMsg(failed_msg.to_string()));
250 }
251
252 // FIXME: Re-enable emscripten once it can catch panics again (introduced by #65251)
253 #[test]
254 #[cfg(not(target_os = "emscripten"))]
255 #[cfg_attr(not(panic = "unwind"), ignore = "test requires unwinding support")]
test_should_panic_non_string_message_type()256 fn test_should_panic_non_string_message_type() {
257 use crate::tests::TrFailedMsg;
258 use std::any::TypeId;
259 fn f() -> Result<(), String> {
260 std::panic::panic_any(1i32);
261 }
262 let expected = "foobar";
263 let failed_msg = format!(
264 r#"expected panic with string value,
265 found non-string value: `{:?}`
266 expected substring: `"foobar"`"#,
267 TypeId::of::<i32>()
268 );
269 let desc = TestDescAndFn {
270 desc: TestDesc {
271 name: StaticTestName("whatever"),
272 ignore: false,
273 ignore_message: None,
274 source_file: "",
275 start_line: 0,
276 start_col: 0,
277 end_line: 0,
278 end_col: 0,
279 should_panic: ShouldPanic::YesWithMessage(expected),
280 compile_fail: false,
281 no_run: false,
282 test_type: TestType::Unknown,
283 },
284 testfn: DynTestFn(Box::new(f)),
285 };
286 let (tx, rx) = channel();
287 run_test(&TestOpts::new(), false, TestId(0), desc, RunStrategy::InProcess, tx);
288 let result = rx.recv().unwrap().result;
289 assert_eq!(result, TrFailedMsg(failed_msg));
290 }
291
292 // FIXME: Re-enable emscripten once it can catch panics again (introduced by #65251)
293 #[test]
294 #[cfg(not(target_os = "emscripten"))]
295 #[cfg_attr(not(panic = "unwind"), ignore = "test requires unwinding support")]
test_should_panic_but_succeeds()296 fn test_should_panic_but_succeeds() {
297 let should_panic_variants = [ShouldPanic::Yes, ShouldPanic::YesWithMessage("error message")];
298
299 for &should_panic in should_panic_variants.iter() {
300 fn f() -> Result<(), String> {
301 Ok(())
302 }
303 let desc = TestDescAndFn {
304 desc: TestDesc {
305 name: StaticTestName("whatever"),
306 ignore: false,
307 ignore_message: None,
308 source_file: "",
309 start_line: 0,
310 start_col: 0,
311 end_line: 0,
312 end_col: 0,
313 should_panic,
314 compile_fail: false,
315 no_run: false,
316 test_type: TestType::Unknown,
317 },
318 testfn: DynTestFn(Box::new(f)),
319 };
320 let (tx, rx) = channel();
321 run_test(&TestOpts::new(), false, TestId(0), desc, RunStrategy::InProcess, tx);
322 let result = rx.recv().unwrap().result;
323 assert_eq!(
324 result,
325 TrFailedMsg("test did not panic as expected".to_string()),
326 "should_panic == {:?}",
327 should_panic
328 );
329 }
330 }
331
report_time_test_template(report_time: bool) -> Option<TestExecTime>332 fn report_time_test_template(report_time: bool) -> Option<TestExecTime> {
333 fn f() -> Result<(), String> {
334 Ok(())
335 }
336 let desc = TestDescAndFn {
337 desc: TestDesc {
338 name: StaticTestName("whatever"),
339 ignore: false,
340 ignore_message: None,
341 source_file: "",
342 start_line: 0,
343 start_col: 0,
344 end_line: 0,
345 end_col: 0,
346 should_panic: ShouldPanic::No,
347 compile_fail: false,
348 no_run: false,
349 test_type: TestType::Unknown,
350 },
351 testfn: DynTestFn(Box::new(f)),
352 };
353 let time_options = if report_time { Some(TestTimeOptions::default()) } else { None };
354
355 let test_opts = TestOpts { time_options, ..TestOpts::new() };
356 let (tx, rx) = channel();
357 run_test(&test_opts, false, TestId(0), desc, RunStrategy::InProcess, tx);
358 let exec_time = rx.recv().unwrap().exec_time;
359 exec_time
360 }
361
362 #[test]
test_should_not_report_time()363 fn test_should_not_report_time() {
364 let exec_time = report_time_test_template(false);
365 assert!(exec_time.is_none());
366 }
367
368 #[test]
test_should_report_time()369 fn test_should_report_time() {
370 let exec_time = report_time_test_template(true);
371 assert!(exec_time.is_some());
372 }
373
time_test_failure_template(test_type: TestType) -> TestResult374 fn time_test_failure_template(test_type: TestType) -> TestResult {
375 fn f() -> Result<(), String> {
376 Ok(())
377 }
378 let desc = TestDescAndFn {
379 desc: TestDesc {
380 name: StaticTestName("whatever"),
381 ignore: false,
382 ignore_message: None,
383 source_file: "",
384 start_line: 0,
385 start_col: 0,
386 end_line: 0,
387 end_col: 0,
388 should_panic: ShouldPanic::No,
389 compile_fail: false,
390 no_run: false,
391 test_type,
392 },
393 testfn: DynTestFn(Box::new(f)),
394 };
395 // `Default` will initialize all the thresholds to 0 milliseconds.
396 let mut time_options = TestTimeOptions::default();
397 time_options.error_on_excess = true;
398
399 let test_opts = TestOpts { time_options: Some(time_options), ..TestOpts::new() };
400 let (tx, rx) = channel();
401 run_test(&test_opts, false, TestId(0), desc, RunStrategy::InProcess, tx);
402 let result = rx.recv().unwrap().result;
403
404 result
405 }
406
407 #[test]
test_error_on_exceed()408 fn test_error_on_exceed() {
409 let types = [TestType::UnitTest, TestType::IntegrationTest, TestType::DocTest];
410
411 for test_type in types.iter() {
412 let result = time_test_failure_template(*test_type);
413
414 assert_eq!(result, TestResult::TrTimedFail);
415 }
416
417 // Check that for unknown tests thresholds aren't applied.
418 let result = time_test_failure_template(TestType::Unknown);
419 assert_eq!(result, TestResult::TrOk);
420 }
421
typed_test_desc(test_type: TestType) -> TestDesc422 fn typed_test_desc(test_type: TestType) -> TestDesc {
423 TestDesc {
424 name: StaticTestName("whatever"),
425 ignore: false,
426 ignore_message: None,
427 source_file: "",
428 start_line: 0,
429 start_col: 0,
430 end_line: 0,
431 end_col: 0,
432 should_panic: ShouldPanic::No,
433 compile_fail: false,
434 no_run: false,
435 test_type,
436 }
437 }
438
test_exec_time(millis: u64) -> TestExecTime439 fn test_exec_time(millis: u64) -> TestExecTime {
440 TestExecTime(Duration::from_millis(millis))
441 }
442
443 #[test]
test_time_options_threshold()444 fn test_time_options_threshold() {
445 let unit = TimeThreshold::new(Duration::from_millis(50), Duration::from_millis(100));
446 let integration = TimeThreshold::new(Duration::from_millis(500), Duration::from_millis(1000));
447 let doc = TimeThreshold::new(Duration::from_millis(5000), Duration::from_millis(10000));
448
449 let options = TestTimeOptions {
450 error_on_excess: false,
451 unit_threshold: unit.clone(),
452 integration_threshold: integration.clone(),
453 doctest_threshold: doc.clone(),
454 };
455
456 let test_vector = [
457 (TestType::UnitTest, unit.warn.as_millis() - 1, false, false),
458 (TestType::UnitTest, unit.warn.as_millis(), true, false),
459 (TestType::UnitTest, unit.critical.as_millis(), true, true),
460 (TestType::IntegrationTest, integration.warn.as_millis() - 1, false, false),
461 (TestType::IntegrationTest, integration.warn.as_millis(), true, false),
462 (TestType::IntegrationTest, integration.critical.as_millis(), true, true),
463 (TestType::DocTest, doc.warn.as_millis() - 1, false, false),
464 (TestType::DocTest, doc.warn.as_millis(), true, false),
465 (TestType::DocTest, doc.critical.as_millis(), true, true),
466 ];
467
468 for (test_type, time, expected_warn, expected_critical) in test_vector.iter() {
469 let test_desc = typed_test_desc(*test_type);
470 let exec_time = test_exec_time(*time as u64);
471
472 assert_eq!(options.is_warn(&test_desc, &exec_time), *expected_warn);
473 assert_eq!(options.is_critical(&test_desc, &exec_time), *expected_critical);
474 }
475 }
476
477 #[test]
parse_ignored_flag()478 fn parse_ignored_flag() {
479 let args = vec!["progname".to_string(), "filter".to_string(), "--ignored".to_string()];
480 let opts = parse_opts(&args).unwrap().unwrap();
481 assert_eq!(opts.run_ignored, RunIgnored::Only);
482 }
483
484 #[test]
parse_show_output_flag()485 fn parse_show_output_flag() {
486 let args = vec!["progname".to_string(), "filter".to_string(), "--show-output".to_string()];
487 let opts = parse_opts(&args).unwrap().unwrap();
488 assert!(opts.options.display_output);
489 }
490
491 #[test]
parse_include_ignored_flag()492 fn parse_include_ignored_flag() {
493 let args = vec!["progname".to_string(), "filter".to_string(), "--include-ignored".to_string()];
494 let opts = parse_opts(&args).unwrap().unwrap();
495 assert_eq!(opts.run_ignored, RunIgnored::Yes);
496 }
497
498 #[test]
filter_for_ignored_option()499 pub fn filter_for_ignored_option() {
500 // When we run ignored tests the test filter should filter out all the
501 // unignored tests and flip the ignore flag on the rest to false
502
503 let mut opts = TestOpts::new();
504 opts.run_tests = true;
505 opts.run_ignored = RunIgnored::Only;
506
507 let tests = one_ignored_one_unignored_test();
508 let filtered = filter_tests(&opts, tests);
509
510 assert_eq!(filtered.len(), 1);
511 assert_eq!(filtered[0].desc.name.to_string(), "1");
512 assert!(!filtered[0].desc.ignore);
513 }
514
515 #[test]
run_include_ignored_option()516 pub fn run_include_ignored_option() {
517 // When we "--include-ignored" tests, the ignore flag should be set to false on
518 // all tests and no test filtered out
519
520 let mut opts = TestOpts::new();
521 opts.run_tests = true;
522 opts.run_ignored = RunIgnored::Yes;
523
524 let tests = one_ignored_one_unignored_test();
525 let filtered = filter_tests(&opts, tests);
526
527 assert_eq!(filtered.len(), 2);
528 assert!(!filtered[0].desc.ignore);
529 assert!(!filtered[1].desc.ignore);
530 }
531
532 #[test]
exclude_should_panic_option()533 pub fn exclude_should_panic_option() {
534 let mut opts = TestOpts::new();
535 opts.run_tests = true;
536 opts.exclude_should_panic = true;
537
538 let mut tests = one_ignored_one_unignored_test();
539 tests.push(TestDescAndFn {
540 desc: TestDesc {
541 name: StaticTestName("3"),
542 ignore: false,
543 ignore_message: None,
544 source_file: "",
545 start_line: 0,
546 start_col: 0,
547 end_line: 0,
548 end_col: 0,
549 should_panic: ShouldPanic::Yes,
550 compile_fail: false,
551 no_run: false,
552 test_type: TestType::Unknown,
553 },
554 testfn: DynTestFn(Box::new(move || Ok(()))),
555 });
556
557 let filtered = filter_tests(&opts, tests);
558
559 assert_eq!(filtered.len(), 2);
560 assert!(filtered.iter().all(|test| test.desc.should_panic == ShouldPanic::No));
561 }
562
563 #[test]
exact_filter_match()564 pub fn exact_filter_match() {
565 fn tests() -> Vec<TestDescAndFn> {
566 ["base", "base::test", "base::test1", "base::test2"]
567 .into_iter()
568 .map(|name| TestDescAndFn {
569 desc: TestDesc {
570 name: StaticTestName(name),
571 ignore: false,
572 ignore_message: None,
573 source_file: "",
574 start_line: 0,
575 start_col: 0,
576 end_line: 0,
577 end_col: 0,
578 should_panic: ShouldPanic::No,
579 compile_fail: false,
580 no_run: false,
581 test_type: TestType::Unknown,
582 },
583 testfn: DynTestFn(Box::new(move || Ok(()))),
584 })
585 .collect()
586 }
587
588 let substr =
589 filter_tests(&TestOpts { filters: vec!["base".into()], ..TestOpts::new() }, tests());
590 assert_eq!(substr.len(), 4);
591
592 let substr =
593 filter_tests(&TestOpts { filters: vec!["bas".into()], ..TestOpts::new() }, tests());
594 assert_eq!(substr.len(), 4);
595
596 let substr =
597 filter_tests(&TestOpts { filters: vec!["::test".into()], ..TestOpts::new() }, tests());
598 assert_eq!(substr.len(), 3);
599
600 let substr =
601 filter_tests(&TestOpts { filters: vec!["base::test".into()], ..TestOpts::new() }, tests());
602 assert_eq!(substr.len(), 3);
603
604 let substr = filter_tests(
605 &TestOpts { filters: vec!["test1".into(), "test2".into()], ..TestOpts::new() },
606 tests(),
607 );
608 assert_eq!(substr.len(), 2);
609
610 let exact = filter_tests(
611 &TestOpts { filters: vec!["base".into()], filter_exact: true, ..TestOpts::new() },
612 tests(),
613 );
614 assert_eq!(exact.len(), 1);
615
616 let exact = filter_tests(
617 &TestOpts { filters: vec!["bas".into()], filter_exact: true, ..TestOpts::new() },
618 tests(),
619 );
620 assert_eq!(exact.len(), 0);
621
622 let exact = filter_tests(
623 &TestOpts { filters: vec!["::test".into()], filter_exact: true, ..TestOpts::new() },
624 tests(),
625 );
626 assert_eq!(exact.len(), 0);
627
628 let exact = filter_tests(
629 &TestOpts { filters: vec!["base::test".into()], filter_exact: true, ..TestOpts::new() },
630 tests(),
631 );
632 assert_eq!(exact.len(), 1);
633
634 let exact = filter_tests(
635 &TestOpts {
636 filters: vec!["base".into(), "base::test".into()],
637 filter_exact: true,
638 ..TestOpts::new()
639 },
640 tests(),
641 );
642 assert_eq!(exact.len(), 2);
643 }
644
sample_tests() -> Vec<TestDescAndFn>645 fn sample_tests() -> Vec<TestDescAndFn> {
646 let names = vec![
647 "sha1::test".to_string(),
648 "isize::test_to_str".to_string(),
649 "isize::test_pow".to_string(),
650 "test::do_not_run_ignored_tests".to_string(),
651 "test::ignored_tests_result_in_ignored".to_string(),
652 "test::first_free_arg_should_be_a_filter".to_string(),
653 "test::parse_ignored_flag".to_string(),
654 "test::parse_include_ignored_flag".to_string(),
655 "test::filter_for_ignored_option".to_string(),
656 "test::run_include_ignored_option".to_string(),
657 "test::sort_tests".to_string(),
658 ];
659 fn testfn() -> Result<(), String> {
660 Ok(())
661 }
662 let mut tests = Vec::new();
663 for name in &names {
664 let test = TestDescAndFn {
665 desc: TestDesc {
666 name: DynTestName((*name).clone()),
667 ignore: false,
668 ignore_message: None,
669 source_file: "",
670 start_line: 0,
671 start_col: 0,
672 end_line: 0,
673 end_col: 0,
674 should_panic: ShouldPanic::No,
675 compile_fail: false,
676 no_run: false,
677 test_type: TestType::Unknown,
678 },
679 testfn: DynTestFn(Box::new(testfn)),
680 };
681 tests.push(test);
682 }
683 tests
684 }
685
686 #[test]
shuffle_tests()687 pub fn shuffle_tests() {
688 let mut opts = TestOpts::new();
689 opts.shuffle = true;
690
691 let shuffle_seed = get_shuffle_seed(&opts).unwrap();
692
693 let left =
694 sample_tests().into_iter().enumerate().map(|(i, e)| (TestId(i), e)).collect::<Vec<_>>();
695 let mut right =
696 sample_tests().into_iter().enumerate().map(|(i, e)| (TestId(i), e)).collect::<Vec<_>>();
697
698 assert!(left.iter().zip(&right).all(|(a, b)| a.1.desc.name == b.1.desc.name));
699
700 helpers::shuffle::shuffle_tests(shuffle_seed, right.as_mut_slice());
701
702 assert!(left.iter().zip(right).any(|(a, b)| a.1.desc.name != b.1.desc.name));
703 }
704
705 #[test]
shuffle_tests_with_seed()706 pub fn shuffle_tests_with_seed() {
707 let mut opts = TestOpts::new();
708 opts.shuffle = true;
709
710 let shuffle_seed = get_shuffle_seed(&opts).unwrap();
711
712 let mut left =
713 sample_tests().into_iter().enumerate().map(|(i, e)| (TestId(i), e)).collect::<Vec<_>>();
714 let mut right =
715 sample_tests().into_iter().enumerate().map(|(i, e)| (TestId(i), e)).collect::<Vec<_>>();
716
717 helpers::shuffle::shuffle_tests(shuffle_seed, left.as_mut_slice());
718 helpers::shuffle::shuffle_tests(shuffle_seed, right.as_mut_slice());
719
720 assert!(left.iter().zip(right).all(|(a, b)| a.1.desc.name == b.1.desc.name));
721 }
722
723 #[test]
order_depends_on_more_than_seed()724 pub fn order_depends_on_more_than_seed() {
725 let mut opts = TestOpts::new();
726 opts.shuffle = true;
727
728 let shuffle_seed = get_shuffle_seed(&opts).unwrap();
729
730 let mut left_tests = sample_tests();
731 let mut right_tests = sample_tests();
732
733 left_tests.pop();
734 right_tests.remove(0);
735
736 let mut left =
737 left_tests.into_iter().enumerate().map(|(i, e)| (TestId(i), e)).collect::<Vec<_>>();
738 let mut right =
739 right_tests.into_iter().enumerate().map(|(i, e)| (TestId(i), e)).collect::<Vec<_>>();
740
741 assert_eq!(left.len(), right.len());
742
743 assert!(left.iter().zip(&right).all(|(a, b)| a.0 == b.0));
744
745 helpers::shuffle::shuffle_tests(shuffle_seed, left.as_mut_slice());
746 helpers::shuffle::shuffle_tests(shuffle_seed, right.as_mut_slice());
747
748 assert!(left.iter().zip(right).any(|(a, b)| a.0 != b.0));
749 }
750
751 #[test]
test_metricmap_compare()752 pub fn test_metricmap_compare() {
753 let mut m1 = MetricMap::new();
754 let mut m2 = MetricMap::new();
755 m1.insert_metric("in-both-noise", 1000.0, 200.0);
756 m2.insert_metric("in-both-noise", 1100.0, 200.0);
757
758 m1.insert_metric("in-first-noise", 1000.0, 2.0);
759 m2.insert_metric("in-second-noise", 1000.0, 2.0);
760
761 m1.insert_metric("in-both-want-downwards-but-regressed", 1000.0, 10.0);
762 m2.insert_metric("in-both-want-downwards-but-regressed", 2000.0, 10.0);
763
764 m1.insert_metric("in-both-want-downwards-and-improved", 2000.0, 10.0);
765 m2.insert_metric("in-both-want-downwards-and-improved", 1000.0, 10.0);
766
767 m1.insert_metric("in-both-want-upwards-but-regressed", 2000.0, -10.0);
768 m2.insert_metric("in-both-want-upwards-but-regressed", 1000.0, -10.0);
769
770 m1.insert_metric("in-both-want-upwards-and-improved", 1000.0, -10.0);
771 m2.insert_metric("in-both-want-upwards-and-improved", 2000.0, -10.0);
772 }
773
774 #[test]
test_bench_once_no_iter()775 pub fn test_bench_once_no_iter() {
776 fn f(_: &mut Bencher) -> Result<(), String> {
777 Ok(())
778 }
779 bench::run_once(f).unwrap();
780 }
781
782 #[test]
test_bench_once_iter()783 pub fn test_bench_once_iter() {
784 fn f(b: &mut Bencher) -> Result<(), String> {
785 b.iter(|| {});
786 Ok(())
787 }
788 bench::run_once(f).unwrap();
789 }
790
791 #[test]
test_bench_no_iter()792 pub fn test_bench_no_iter() {
793 fn f(_: &mut Bencher) -> Result<(), String> {
794 Ok(())
795 }
796
797 let (tx, rx) = channel();
798
799 let desc = TestDesc {
800 name: StaticTestName("f"),
801 ignore: false,
802 ignore_message: None,
803 source_file: "",
804 start_line: 0,
805 start_col: 0,
806 end_line: 0,
807 end_col: 0,
808 should_panic: ShouldPanic::No,
809 compile_fail: false,
810 no_run: false,
811 test_type: TestType::Unknown,
812 };
813
814 crate::bench::benchmark(TestId(0), desc, tx, true, f);
815 rx.recv().unwrap();
816 }
817
818 #[test]
test_bench_iter()819 pub fn test_bench_iter() {
820 fn f(b: &mut Bencher) -> Result<(), String> {
821 b.iter(|| {});
822 Ok(())
823 }
824
825 let (tx, rx) = channel();
826
827 let desc = TestDesc {
828 name: StaticTestName("f"),
829 ignore: false,
830 ignore_message: None,
831 source_file: "",
832 start_line: 0,
833 start_col: 0,
834 end_line: 0,
835 end_col: 0,
836 should_panic: ShouldPanic::No,
837 compile_fail: false,
838 no_run: false,
839 test_type: TestType::Unknown,
840 };
841
842 crate::bench::benchmark(TestId(0), desc, tx, true, f);
843 rx.recv().unwrap();
844 }
845
846 #[test]
should_sort_failures_before_printing_them()847 fn should_sort_failures_before_printing_them() {
848 let test_a = TestDesc {
849 name: StaticTestName("a"),
850 ignore: false,
851 ignore_message: None,
852 source_file: "",
853 start_line: 0,
854 start_col: 0,
855 end_line: 0,
856 end_col: 0,
857 should_panic: ShouldPanic::No,
858 compile_fail: false,
859 no_run: false,
860 test_type: TestType::Unknown,
861 };
862
863 let test_b = TestDesc {
864 name: StaticTestName("b"),
865 ignore: false,
866 ignore_message: None,
867 source_file: "",
868 start_line: 0,
869 start_col: 0,
870 end_line: 0,
871 end_col: 0,
872 should_panic: ShouldPanic::No,
873 compile_fail: false,
874 no_run: false,
875 test_type: TestType::Unknown,
876 };
877
878 let mut out = PrettyFormatter::new(OutputLocation::Raw(Vec::new()), false, 10, false, None);
879
880 let st = console::ConsoleTestState {
881 log_out: None,
882 total: 0,
883 passed: 0,
884 failed: 0,
885 ignored: 0,
886 filtered_out: 0,
887 measured: 0,
888 exec_time: None,
889 metrics: MetricMap::new(),
890 failures: vec![(test_b, Vec::new()), (test_a, Vec::new())],
891 options: Options::new(),
892 not_failures: Vec::new(),
893 ignores: Vec::new(),
894 time_failures: Vec::new(),
895 };
896
897 out.write_failures(&st).unwrap();
898 let s = match out.output_location() {
899 &OutputLocation::Raw(ref m) => String::from_utf8_lossy(&m[..]),
900 &OutputLocation::Pretty(_) => unreachable!(),
901 };
902
903 let apos = s.find("a").unwrap();
904 let bpos = s.find("b").unwrap();
905 assert!(apos < bpos);
906 }
907
908 #[test]
909 #[cfg(not(target_os = "emscripten"))]
test_dyn_bench_returning_err_fails_when_run_as_test()910 fn test_dyn_bench_returning_err_fails_when_run_as_test() {
911 fn f(_: &mut Bencher) -> Result<(), String> {
912 Result::Err("An error".into())
913 }
914 let desc = TestDescAndFn {
915 desc: TestDesc {
916 name: StaticTestName("whatever"),
917 ignore: false,
918 ignore_message: None,
919 source_file: "",
920 start_line: 0,
921 start_col: 0,
922 end_line: 0,
923 end_col: 0,
924 should_panic: ShouldPanic::No,
925 compile_fail: false,
926 no_run: false,
927 test_type: TestType::Unknown,
928 },
929 testfn: DynBenchFn(Box::new(f)),
930 };
931 let (tx, rx) = channel();
932 let notify = move |event: TestEvent| {
933 if let TestEvent::TeResult(result) = event {
934 tx.send(result).unwrap();
935 }
936 Ok(())
937 };
938 run_tests(&TestOpts { run_tests: true, ..TestOpts::new() }, vec![desc], notify).unwrap();
939 let result = rx.recv().unwrap().result;
940 assert_eq!(result, TrFailed);
941 }
942