• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/env python2
2"""
3Command line tool to bisect failing CPython tests.
4
5Find the test_os test method which alters the environment:
6
7    ./python -m test.bisect --fail-env-changed test_os
8
9Find a reference leak in "test_os", write the list of failing tests into the
10"bisect" file:
11
12    ./python -m test.bisect -o bisect -R 3:3 test_os
13
14Load an existing list of tests from a file using -i option:
15
16    ./python -m test --list-cases -m FileTests test_os > tests
17    ./python -m test.bisect -i tests test_os
18"""
19from __future__ import print_function
20
21import argparse
22import datetime
23import os.path
24import math
25import random
26import subprocess
27import sys
28import tempfile
29import time
30
31
32def write_tests(filename, tests):
33    with open(filename, "w") as fp:
34        for name in tests:
35            print(name, file=fp)
36        fp.flush()
37
38
39def write_output(filename, tests):
40    if not filename:
41        return
42    print("Write %s tests into %s" % (len(tests), filename))
43    write_tests(filename, tests)
44    return filename
45
46
47def format_shell_args(args):
48    return ' '.join(args)
49
50
51def list_cases(args):
52    cmd = [sys.executable, '-m', 'test', '--list-cases']
53    cmd.extend(args.test_args)
54    proc = subprocess.Popen(cmd,
55                            stdout=subprocess.PIPE,
56                            universal_newlines=True)
57    try:
58        stdout = proc.communicate()[0]
59    except:
60        proc.stdout.close()
61        proc.kill()
62        proc.wait()
63        raise
64    exitcode = proc.wait()
65    if exitcode:
66        cmd = format_shell_args(cmd)
67        print("Failed to list tests: %s failed with exit code %s"
68              % (cmd, exitcode))
69        sys.exit(exitcode)
70    tests = stdout.splitlines()
71    return tests
72
73
74def run_tests(args, tests, huntrleaks=None):
75    tmp = tempfile.mktemp()
76    try:
77        write_tests(tmp, tests)
78
79        cmd = [sys.executable, '-m', 'test', '--matchfile', tmp]
80        cmd.extend(args.test_args)
81        print("+ %s" % format_shell_args(cmd))
82        proc = subprocess.Popen(cmd)
83        try:
84            exitcode = proc.wait()
85        except:
86            proc.kill()
87            proc.wait()
88            raise
89        return exitcode
90    finally:
91        if os.path.exists(tmp):
92            os.unlink(tmp)
93
94
95def parse_args():
96    parser = argparse.ArgumentParser()
97    parser.add_argument('-i', '--input',
98                        help='Test names produced by --list-tests written '
99                             'into a file. If not set, run --list-tests')
100    parser.add_argument('-o', '--output',
101                        help='Result of the bisection')
102    parser.add_argument('-n', '--max-tests', type=int, default=1,
103                        help='Maximum number of tests to stop the bisection '
104                             '(default: 1)')
105    parser.add_argument('-N', '--max-iter', type=int, default=100,
106                        help='Maximum number of bisection iterations '
107                             '(default: 100)')
108    # FIXME: document that following arguments are test arguments
109
110    args, test_args = parser.parse_known_args()
111    args.test_args = test_args
112    return args
113
114
115def main():
116    args = parse_args()
117
118    if args.input:
119        with open(args.input) as fp:
120            tests = [line.strip() for line in fp]
121    else:
122        tests = list_cases(args)
123
124    print("Start bisection with %s tests" % len(tests))
125    print("Test arguments: %s" % format_shell_args(args.test_args))
126    print("Bisection will stop when getting %s or less tests "
127          "(-n/--max-tests option), or after %s iterations "
128          "(-N/--max-iter option)"
129          % (args.max_tests, args.max_iter))
130    output = write_output(args.output, tests)
131    print()
132
133    start_time = time.time()
134    iteration = 1
135    try:
136        while len(tests) > args.max_tests and iteration <= args.max_iter:
137            ntest = len(tests)
138            ntest = max(ntest // 2, 1)
139            subtests = random.sample(tests, ntest)
140
141            print("[+] Iteration %s: run %s tests/%s"
142                  % (iteration, len(subtests), len(tests)))
143            print()
144
145            exitcode = run_tests(args, subtests)
146
147            print("ran %s tests/%s" % (ntest, len(tests)))
148            print("exit", exitcode)
149            if exitcode:
150                print("Tests failed: use this new subtest")
151                tests = subtests
152                output = write_output(args.output, tests)
153            else:
154                print("Tests succeeded: skip this subtest, try a new subbset")
155            print()
156            iteration += 1
157    except KeyboardInterrupt:
158        print()
159        print("Bisection interrupted!")
160        print()
161
162    print("Tests (%s):" % len(tests))
163    for test in tests:
164        print("* %s" % test)
165    print()
166
167    if output:
168        print("Output written into %s" % output)
169
170    dt = math.ceil(time.time() - start_time)
171    if len(tests) <= args.max_tests:
172        print("Bisection completed in %s iterations and %s"
173              % (iteration, datetime.timedelta(seconds=dt)))
174        sys.exit(1)
175    else:
176        print("Bisection failed after %s iterations and %s"
177              % (iteration, datetime.timedelta(seconds=dt)))
178
179
180if __name__ == "__main__":
181    main()
182