• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/env python3
2"""
3Command line tool to bisect failing CPython tests.
4
5Find the test_os test method which alters the environment:
6
7    ./python -m test.bisect_cmd --fail-env-changed test_os
8
9Find a reference leak in "test_os", write the list of failing tests into the
10"bisect" file:
11
12    ./python -m test.bisect_cmd -o bisect -R 3:3 test_os
13
14Load an existing list of tests from a file using -i option:
15
16    ./python -m test --list-cases -m FileTests test_os > tests
17    ./python -m test.bisect_cmd -i tests test_os
18"""
19
20import argparse
21import datetime
22import os.path
23import math
24import random
25import subprocess
26import sys
27import tempfile
28import time
29
30
31def write_tests(filename, tests):
32    with open(filename, "w") as fp:
33        for name in tests:
34            print(name, file=fp)
35        fp.flush()
36
37
38def write_output(filename, tests):
39    if not filename:
40        return
41    print("Writing %s tests into %s" % (len(tests), filename))
42    write_tests(filename, tests)
43    return filename
44
45
46def format_shell_args(args):
47    return ' '.join(args)
48
49
50def list_cases(args):
51    cmd = [sys.executable, '-m', 'test', '--list-cases']
52    cmd.extend(args.test_args)
53    proc = subprocess.run(cmd,
54                          stdout=subprocess.PIPE,
55                          universal_newlines=True)
56    exitcode = proc.returncode
57    if exitcode:
58        cmd = format_shell_args(cmd)
59        print("Failed to list tests: %s failed with exit code %s"
60              % (cmd, exitcode))
61        sys.exit(exitcode)
62    tests = proc.stdout.splitlines()
63    return tests
64
65
66def run_tests(args, tests, huntrleaks=None):
67    tmp = tempfile.mktemp()
68    try:
69        write_tests(tmp, tests)
70
71        cmd = [sys.executable, '-m', 'test', '--matchfile', tmp]
72        cmd.extend(args.test_args)
73        print("+ %s" % format_shell_args(cmd))
74        proc = subprocess.run(cmd)
75        return proc.returncode
76    finally:
77        if os.path.exists(tmp):
78            os.unlink(tmp)
79
80
81def parse_args():
82    parser = argparse.ArgumentParser()
83    parser.add_argument('-i', '--input',
84                        help='Test names produced by --list-tests written '
85                             'into a file. If not set, run --list-tests')
86    parser.add_argument('-o', '--output',
87                        help='Result of the bisection')
88    parser.add_argument('-n', '--max-tests', type=int, default=1,
89                        help='Maximum number of tests to stop the bisection '
90                             '(default: 1)')
91    parser.add_argument('-N', '--max-iter', type=int, default=100,
92                        help='Maximum number of bisection iterations '
93                             '(default: 100)')
94    # FIXME: document that following arguments are test arguments
95
96    args, test_args = parser.parse_known_args()
97    args.test_args = test_args
98    return args
99
100
101def main():
102    args = parse_args()
103
104    if args.input:
105        with open(args.input) as fp:
106            tests = [line.strip() for line in fp]
107    else:
108        tests = list_cases(args)
109
110    print("Start bisection with %s tests" % len(tests))
111    print("Test arguments: %s" % format_shell_args(args.test_args))
112    print("Bisection will stop when getting %s or less tests "
113          "(-n/--max-tests option), or after %s iterations "
114          "(-N/--max-iter option)"
115          % (args.max_tests, args.max_iter))
116    output = write_output(args.output, tests)
117    print()
118
119    start_time = time.monotonic()
120    iteration = 1
121    try:
122        while len(tests) > args.max_tests and iteration <= args.max_iter:
123            ntest = len(tests)
124            ntest = max(ntest // 2, 1)
125            subtests = random.sample(tests, ntest)
126
127            print("[+] Iteration %s: run %s tests/%s"
128                  % (iteration, len(subtests), len(tests)))
129            print()
130
131            exitcode = run_tests(args, subtests)
132
133            print("ran %s tests/%s" % (ntest, len(tests)))
134            print("exit", exitcode)
135            if exitcode:
136                print("Tests failed: continuing with this subtest")
137                tests = subtests
138                output = write_output(args.output, tests)
139            else:
140                print("Tests succeeded: skipping this subtest, trying a new subset")
141            print()
142            iteration += 1
143    except KeyboardInterrupt:
144        print()
145        print("Bisection interrupted!")
146        print()
147
148    print("Tests (%s):" % len(tests))
149    for test in tests:
150        print("* %s" % test)
151    print()
152
153    if output:
154        print("Output written into %s" % output)
155
156    dt = math.ceil(time.monotonic() - start_time)
157    if len(tests) <= args.max_tests:
158        print("Bisection completed in %s iterations and %s"
159              % (iteration, datetime.timedelta(seconds=dt)))
160        sys.exit(1)
161    else:
162        print("Bisection failed after %s iterations and %s"
163              % (iteration, datetime.timedelta(seconds=dt)))
164
165
166if __name__ == "__main__":
167    main()
168