1 #!/usr/bin/env python3
2 """
3 Command line tool to bisect failing CPython tests.
4
5 Find the test_os test method which alters the environment:
6
7 ./python -m test.bisect_cmd --fail-env-changed test_os
8
9 Find a reference leak in "test_os", write the list of failing tests into the
10 "bisect" file:
11
12 ./python -m test.bisect_cmd -o bisect -R 3:3 test_os
13
14 Load an existing list of tests from a file using -i option:
15
16 ./python -m test --list-cases -m FileTests test_os > tests
17 ./python -m test.bisect_cmd -i tests test_os
18 """
19
20 import argparse
21 import datetime
22 import os.path
23 import math
24 import random
25 import subprocess
26 import sys
27 import tempfile
28 import time
29
30
31 def write_tests(filename, tests):
32 with open(filename, "w") as fp:
33 for name in tests:
34 print(name, file=fp)
35 fp.flush()
36
37
38 def write_output(filename, tests):
39 if not filename:
40 return
41 print("Writing %s tests into %s" % (len(tests), filename))
42 write_tests(filename, tests)
43 return filename
44
45
46 def format_shell_args(args):
47 return ' '.join(args)
48
49
50 def python_cmd():
51 cmd = [sys.executable]
52 cmd.extend(subprocess._args_from_interpreter_flags())
53 cmd.extend(subprocess._optim_args_from_interpreter_flags())
54 return cmd
55
56
57 def list_cases(args):
58 cmd = python_cmd()
59 cmd.extend(['-m', 'test', '--list-cases'])
60 cmd.extend(args.test_args)
61 proc = subprocess.run(cmd,
62 stdout=subprocess.PIPE,
63 universal_newlines=True)
64 exitcode = proc.returncode
65 if exitcode:
66 cmd = format_shell_args(cmd)
67 print("Failed to list tests: %s failed with exit code %s"
68 % (cmd, exitcode))
69 sys.exit(exitcode)
70 tests = proc.stdout.splitlines()
71 return tests
72
73
74 def run_tests(args, tests, huntrleaks=None):
75 tmp = tempfile.mktemp()
76 try:
77 write_tests(tmp, tests)
78
79 cmd = python_cmd()
80 cmd.extend(['-m', 'test', '--matchfile', tmp])
81 cmd.extend(args.test_args)
82 print("+ %s" % format_shell_args(cmd))
83 proc = subprocess.run(cmd)
84 return proc.returncode
85 finally:
86 if os.path.exists(tmp):
87 os.unlink(tmp)
88
89
90 def parse_args():
91 parser = argparse.ArgumentParser()
92 parser.add_argument('-i', '--input',
93 help='Test names produced by --list-tests written '
94 'into a file. If not set, run --list-tests')
95 parser.add_argument('-o', '--output',
96 help='Result of the bisection')
97 parser.add_argument('-n', '--max-tests', type=int, default=1,
98 help='Maximum number of tests to stop the bisection '
99 '(default: 1)')
100 parser.add_argument('-N', '--max-iter', type=int, default=100,
101 help='Maximum number of bisection iterations '
102 '(default: 100)')
103 # FIXME: document that following arguments are test arguments
104
105 args, test_args = parser.parse_known_args()
106 args.test_args = test_args
107 return args
108
109
110 def main():
111 args = parse_args()
112 for opt in ('-w', '--rerun', '--verbose2'):
113 if opt in args.test_args:
114 print(f"WARNING: {opt} option should not be used to bisect!")
115 print()
116
117 if args.input:
118 with open(args.input) as fp:
119 tests = [line.strip() for line in fp]
120 else:
121 tests = list_cases(args)
122
123 print("Start bisection with %s tests" % len(tests))
124 print("Test arguments: %s" % format_shell_args(args.test_args))
125 print("Bisection will stop when getting %s or less tests "
126 "(-n/--max-tests option), or after %s iterations "
127 "(-N/--max-iter option)"
128 % (args.max_tests, args.max_iter))
129 output = write_output(args.output, tests)
130 print()
131
132 start_time = time.monotonic()
133 iteration = 1
134 try:
135 while len(tests) > args.max_tests and iteration <= args.max_iter:
136 ntest = len(tests)
137 ntest = max(ntest // 2, 1)
138 subtests = random.sample(tests, ntest)
139
140 print("[+] Iteration %s: run %s tests/%s"
141 % (iteration, len(subtests), len(tests)))
142 print()
143
144 exitcode = run_tests(args, subtests)
145
146 print("ran %s tests/%s" % (ntest, len(tests)))
147 print("exit", exitcode)
148 if exitcode:
149 print("Tests failed: continuing with this subtest")
150 tests = subtests
151 output = write_output(args.output, tests)
152 else:
153 print("Tests succeeded: skipping this subtest, trying a new subset")
154 print()
155 iteration += 1
156 except KeyboardInterrupt:
157 print()
158 print("Bisection interrupted!")
159 print()
160
161 print("Tests (%s):" % len(tests))
162 for test in tests:
163 print("* %s" % test)
164 print()
165
166 if output:
167 print("Output written into %s" % output)
168
169 dt = math.ceil(time.monotonic() - start_time)
170 if len(tests) <= args.max_tests:
171 print("Bisection completed in %s iterations and %s"
172 % (iteration, datetime.timedelta(seconds=dt)))
173 sys.exit(1)
174 else:
175 print("Bisection failed after %s iterations and %s"
176 % (iteration, datetime.timedelta(seconds=dt)))
177
178
179 if __name__ == "__main__":
180 main()