python (3.11.7)
1 #!/usr/bin/env python3
2 """
3 Command line tool to bisect failing CPython tests.
4
5 Find the test_os test method which alters the environment:
6
7 ./python -m test.bisect_cmd --fail-env-changed test_os
8
9 Find a reference leak in "test_os", write the list of failing tests into the
10 "bisect" file:
11
12 ./python -m test.bisect_cmd -o bisect -R 3:3 test_os
13
14 Load an existing list of tests from a file using -i option:
15
16 ./python -m test --list-cases -m FileTests test_os > tests
17 ./python -m test.bisect_cmd -i tests test_os
18 """
19
20 import argparse
21 import datetime
22 import os.path
23 import math
24 import random
25 import subprocess
26 import sys
27 import tempfile
28 import time
29
30
31 def write_tests(filename, tests):
32 with open(filename, "w") as fp:
33 for name in tests:
34 print(name, file=fp)
35 fp.flush()
36
37
38 def write_output(filename, tests):
39 if not filename:
40 return
41 print("Writing %s tests into %s" % (len(tests), filename))
42 write_tests(filename, tests)
43 return filename
44
45
46 def format_shell_args(args):
47 return ' '.join(args)
48
49
50 def python_cmd():
51 cmd = [sys.executable]
52 cmd.extend(subprocess._args_from_interpreter_flags())
53 cmd.extend(subprocess._optim_args_from_interpreter_flags())
54 return cmd
55
56
57 def list_cases(args):
58 cmd = python_cmd()
59 cmd.extend(['-m', 'test', '--list-cases'])
60 cmd.extend(args.test_args)
61 proc = subprocess.run(cmd,
62 stdout=subprocess.PIPE,
63 universal_newlines=True)
64 exitcode = proc.returncode
65 if exitcode:
66 cmd = format_shell_args(cmd)
67 print("Failed to list tests: %s failed with exit code %s"
68 % (cmd, exitcode))
69 sys.exit(exitcode)
70 tests = proc.stdout.splitlines()
71 return tests
72
73
74 def run_tests(args, tests, huntrleaks=None):
75 tmp = tempfile.mktemp()
76 try:
77 write_tests(tmp, tests)
78
79 cmd = python_cmd()
80 cmd.extend(['-m', 'test', '--matchfile', tmp])
81 cmd.extend(args.test_args)
82 print("+ %s" % format_shell_args(cmd))
83 proc = subprocess.run(cmd)
84 return proc.returncode
85 finally:
86 if os.path.exists(tmp):
87 os.unlink(tmp)
88
89
90 def parse_args():
91 parser = argparse.ArgumentParser()
92 parser.add_argument('-i', '--input',
93 help='Test names produced by --list-tests written '
94 'into a file. If not set, run --list-tests')
95 parser.add_argument('-o', '--output',
96 help='Result of the bisection')
97 parser.add_argument('-n', '--max-tests', type=int, default=1,
98 help='Maximum number of tests to stop the bisection '
99 '(default: 1)')
100 parser.add_argument('-N', '--max-iter', type=int, default=100,
101 help='Maximum number of bisection iterations '
102 '(default: 100)')
103 # FIXME: document that following arguments are test arguments
104
105 args, test_args = parser.parse_known_args()
106 args.test_args = test_args
107 return args
108
109
110 def main():
111 args = parse_args()
112 if '-w' in args.test_args or '--verbose2' in args.test_args:
113 print("WARNING: -w/--verbose2 option should not be used to bisect!")
114 print()
115
116 if args.input:
117 with open(args.input) as fp:
118 tests = [line.strip() for line in fp]
119 else:
120 tests = list_cases(args)
121
122 print("Start bisection with %s tests" % len(tests))
123 print("Test arguments: %s" % format_shell_args(args.test_args))
124 print("Bisection will stop when getting %s or less tests "
125 "(-n/--max-tests option), or after %s iterations "
126 "(-N/--max-iter option)"
127 % (args.max_tests, args.max_iter))
128 output = write_output(args.output, tests)
129 print()
130
131 start_time = time.monotonic()
132 iteration = 1
133 try:
134 while len(tests) > args.max_tests and iteration <= args.max_iter:
135 ntest = len(tests)
136 ntest = max(ntest // 2, 1)
137 subtests = random.sample(tests, ntest)
138
139 print("[+] Iteration %s: run %s tests/%s"
140 % (iteration, len(subtests), len(tests)))
141 print()
142
143 exitcode = run_tests(args, subtests)
144
145 print("ran %s tests/%s" % (ntest, len(tests)))
146 print("exit", exitcode)
147 if exitcode:
148 print("Tests failed: continuing with this subtest")
149 tests = subtests
150 output = write_output(args.output, tests)
151 else:
152 print("Tests succeeded: skipping this subtest, trying a new subset")
153 print()
154 iteration += 1
155 except KeyboardInterrupt:
156 print()
157 print("Bisection interrupted!")
158 print()
159
160 print("Tests (%s):" % len(tests))
161 for test in tests:
162 print("* %s" % test)
163 print()
164
165 if output:
166 print("Output written into %s" % output)
167
168 dt = math.ceil(time.monotonic() - start_time)
169 if len(tests) <= args.max_tests:
170 print("Bisection completed in %s iterations and %s"
171 % (iteration, datetime.timedelta(seconds=dt)))
172 sys.exit(1)
173 else:
174 print("Bisection failed after %s iterations and %s"
175 % (iteration, datetime.timedelta(seconds=dt)))
176
177
178 if __name__ == "__main__":
179 main()
Browse
Usage
Versions