1 import argparse
2 import os
3 import shlex
4 import sys
5 from test.support import os_helper
6
7
8 USAGE = """\
9 python -m test [options] [test_name1 [test_name2 ...]]
10 python path/to/Lib/test/regrtest.py [options] [test_name1 [test_name2 ...]]
11 """
12
13 DESCRIPTION = """\
14 Run Python regression tests.
15
16 If no arguments or options are provided, finds all files matching
17 the pattern "test_*" in the Lib/test subdirectory and runs
18 them in alphabetical order (but see -M and -u, below, for exceptions).
19
20 For more rigorous testing, it is useful to use the following
21 command line:
22
23 python -E -Wd -m test [options] [test_name1 ...]
24 """
25
26 EPILOG = """\
27 Additional option details:
28
29 -r randomizes test execution order. You can use --randseed=int to provide an
30 int seed value for the randomizer; this is useful for reproducing troublesome
31 test orders.
32
33 -s On the first invocation of regrtest using -s, the first test file found
34 or the first test file given on the command line is run, and the name of
35 the next test is recorded in a file named pynexttest. If run from the
36 Python build directory, pynexttest is located in the 'build' subdirectory,
37 otherwise it is located in tempfile.gettempdir(). On subsequent runs,
38 the test in pynexttest is run, and the next test is written to pynexttest.
39 When the last test has been run, pynexttest is deleted. In this way it
40 is possible to single step through the test files. This is useful when
41 doing memory analysis on the Python interpreter, which process tends to
42 consume too many resources to run the full regression test non-stop.
43
44 -S is used to continue running tests after an aborted run. It will
45 maintain the order a standard run (ie, this assumes -r is not used).
46 This is useful after the tests have prematurely stopped for some external
47 reason and you want to start running from where you left off rather
48 than starting from the beginning.
49
50 -f reads the names of tests from the file given as f's argument, one
51 or more test names per line. Whitespace is ignored. Blank lines and
52 lines beginning with '#' are ignored. This is especially useful for
53 whittling down failures involving interactions among tests.
54
55 -L causes the leaks(1) command to be run just before exit if it exists.
56 leaks(1) is available on Mac OS X and presumably on some other
57 FreeBSD-derived systems.
58
59 -R runs each test several times and examines sys.gettotalrefcount() to
60 see if the test appears to be leaking references. The argument should
61 be of the form stab:run:fname where 'stab' is the number of times the
62 test is run to let gettotalrefcount settle down, 'run' is the number
63 of times further it is run and 'fname' is the name of the file the
64 reports are written to. These parameters all have defaults (5, 4 and
65 "reflog.txt" respectively), and the minimal invocation is '-R :'.
66
67 -M runs tests that require an exorbitant amount of memory. These tests
68 typically try to ascertain containers keep working when containing more than
69 2 billion objects, which only works on 64-bit systems. There are also some
70 tests that try to exhaust the address space of the process, which only makes
71 sense on 32-bit systems with at least 2Gb of memory. The passed-in memlimit,
72 which is a string in the form of '2.5Gb', determines how much memory the
73 tests will limit themselves to (but they may go slightly over.) The number
74 shouldn't be more memory than the machine has (including swap memory). You
75 should also keep in mind that swap memory is generally much, much slower
76 than RAM, and setting memlimit to all available RAM or higher will heavily
77 tax the machine. On the other hand, it is no use running these tests with a
78 limit of less than 2.5Gb, and many require more than 20Gb. Tests that expect
79 to use more than memlimit memory will be skipped. The big-memory tests
80 generally run very, very long.
81
82 -u is used to specify which special resource intensive tests to run,
83 such as those requiring large file support or network connectivity.
84 The argument is a comma-separated list of words indicating the
85 resources to test. Currently only the following are defined:
86
87 all - Enable all special resources.
88
89 none - Disable all special resources (this is the default).
90
91 audio - Tests that use the audio device. (There are known
92 cases of broken audio drivers that can crash Python or
93 even the Linux kernel.)
94
95 curses - Tests that use curses and will modify the terminal's
96 state and output modes.
97
98 largefile - It is okay to run some test that may create huge
99 files. These tests can take a long time and may
100 consume >2 GiB of disk space temporarily.
101
102 network - It is okay to run tests that use external network
103 resource, e.g. testing SSL support for sockets.
104
105 decimal - Test the decimal module against a large suite that
106 verifies compliance with standards.
107
108 cpu - Used for certain CPU-heavy tests.
109
110 walltime - Long running but not CPU-bound tests.
111
112 subprocess Run all tests for the subprocess module.
113
114 urlfetch - It is okay to download files required on testing.
115
116 gui - Run tests that require a running GUI.
117
118 tzdata - Run tests that require timezone data.
119
120 To enable all resources except one, use '-uall,-<resource>'. For
121 example, to run all the tests except for the gui tests, give the
122 option '-uall,-gui'.
123
124 --matchfile filters tests using a text file, one pattern per line.
125 Pattern examples:
126
127 - test method: test_stat_attributes
128 - test class: FileTests
129 - test identifier: test_os.FileTests.test_stat_attributes
130 """
131
132
133 ALL_RESOURCES = ('audio', 'curses', 'largefile', 'network',
134 'decimal', 'cpu', 'subprocess', 'urlfetch', 'gui', 'walltime')
135
136 # Other resources excluded from --use=all:
137 #
138 # - extralagefile (ex: test_zipfile64): really too slow to be enabled
139 # "by default"
140 # - tzdata: while needed to validate fully test_datetime, it makes
141 # test_datetime too slow (15-20 min on some buildbots) and so is disabled by
142 # default (see bpo-30822).
143 RESOURCE_NAMES = ALL_RESOURCES + ('extralargefile', 'tzdata')
144
145
146 class ESC[4;38;5;81mNamespace(ESC[4;38;5;149margparseESC[4;38;5;149m.ESC[4;38;5;149mNamespace):
147 def __init__(self, **kwargs) -> None:
148 self.testdir = None
149 self.verbose = 0
150 self.quiet = False
151 self.exclude = False
152 self.single = False
153 self.randomize = False
154 self.fromfile = None
155 self.fail_env_changed = False
156 self.use_resources = None
157 self.trace = False
158 self.coverdir = 'coverage'
159 self.runleaks = False
160 self.huntrleaks = False
161 self.rerun = False
162 self.verbose3 = False
163 self.print_slow = False
164 self.random_seed = None
165 self.use_mp = None
166 self.forever = False
167 self.header = False
168 self.failfast = False
169 self.match_tests = None
170 self.ignore_tests = None
171 self.pgo = False
172 self.pgo_extended = False
173
174 super().__init__(**kwargs)
175
176
177 class ESC[4;38;5;81m_ArgParser(ESC[4;38;5;149margparseESC[4;38;5;149m.ESC[4;38;5;149mArgumentParser):
178
179 def error(self, message):
180 super().error(message + "\nPass -h or --help for complete help.")
181
182
183 def _create_parser():
184 # Set prog to prevent the uninformative "__main__.py" from displaying in
185 # error messages when using "python -m test ...".
186 parser = _ArgParser(prog='regrtest.py',
187 usage=USAGE,
188 description=DESCRIPTION,
189 epilog=EPILOG,
190 add_help=False,
191 formatter_class=argparse.RawDescriptionHelpFormatter)
192
193 # Arguments with this clause added to its help are described further in
194 # the epilog's "Additional option details" section.
195 more_details = ' See the section at bottom for more details.'
196
197 group = parser.add_argument_group('General options')
198 # We add help explicitly to control what argument group it renders under.
199 group.add_argument('-h', '--help', action='help',
200 help='show this help message and exit')
201 group.add_argument('--timeout', metavar='TIMEOUT', type=float,
202 help='dump the traceback and exit if a test takes '
203 'more than TIMEOUT seconds; disabled if TIMEOUT '
204 'is negative or equals to zero')
205 group.add_argument('--wait', action='store_true',
206 help='wait for user input, e.g., allow a debugger '
207 'to be attached')
208 group.add_argument('--worker-args', metavar='ARGS')
209 group.add_argument('-S', '--start', metavar='START',
210 help='the name of the test at which to start.' +
211 more_details)
212 group.add_argument('-p', '--python', metavar='PYTHON',
213 help='Command to run Python test subprocesses with.')
214
215 group = parser.add_argument_group('Verbosity')
216 group.add_argument('-v', '--verbose', action='count',
217 help='run tests in verbose mode with output to stdout')
218 group.add_argument('-w', '--rerun', action='store_true',
219 help='re-run failed tests in verbose mode')
220 group.add_argument('--verbose2', action='store_true', dest='rerun',
221 help='deprecated alias to --rerun')
222 group.add_argument('-W', '--verbose3', action='store_true',
223 help='display test output on failure')
224 group.add_argument('-q', '--quiet', action='store_true',
225 help='no output unless one or more tests fail')
226 group.add_argument('-o', '--slowest', action='store_true', dest='print_slow',
227 help='print the slowest 10 tests')
228 group.add_argument('--header', action='store_true',
229 help='print header with interpreter info')
230
231 group = parser.add_argument_group('Selecting tests')
232 group.add_argument('-r', '--randomize', action='store_true',
233 help='randomize test execution order.' + more_details)
234 group.add_argument('--randseed', metavar='SEED',
235 dest='random_seed', type=int,
236 help='pass a random seed to reproduce a previous '
237 'random run')
238 group.add_argument('-f', '--fromfile', metavar='FILE',
239 help='read names of tests to run from a file.' +
240 more_details)
241 group.add_argument('-x', '--exclude', action='store_true',
242 help='arguments are tests to *exclude*')
243 group.add_argument('-s', '--single', action='store_true',
244 help='single step through a set of tests.' +
245 more_details)
246 group.add_argument('-m', '--match', metavar='PAT',
247 dest='match_tests', action='append',
248 help='match test cases and methods with glob pattern PAT')
249 group.add_argument('-i', '--ignore', metavar='PAT',
250 dest='ignore_tests', action='append',
251 help='ignore test cases and methods with glob pattern PAT')
252 group.add_argument('--matchfile', metavar='FILENAME',
253 dest='match_filename',
254 help='similar to --match but get patterns from a '
255 'text file, one pattern per line')
256 group.add_argument('--ignorefile', metavar='FILENAME',
257 dest='ignore_filename',
258 help='similar to --matchfile but it receives patterns '
259 'from text file to ignore')
260 group.add_argument('-G', '--failfast', action='store_true',
261 help='fail as soon as a test fails (only with -v or -W)')
262 group.add_argument('-u', '--use', metavar='RES1,RES2,...',
263 action='append', type=resources_list,
264 help='specify which special resource intensive tests '
265 'to run.' + more_details)
266 group.add_argument('-M', '--memlimit', metavar='LIMIT',
267 help='run very large memory-consuming tests.' +
268 more_details)
269 group.add_argument('--testdir', metavar='DIR',
270 type=relative_filename,
271 help='execute test files in the specified directory '
272 '(instead of the Python stdlib test suite)')
273
274 group = parser.add_argument_group('Special runs')
275 group.add_argument('-L', '--runleaks', action='store_true',
276 help='run the leaks(1) command just before exit.' +
277 more_details)
278 group.add_argument('-R', '--huntrleaks', metavar='RUNCOUNTS',
279 type=huntrleaks,
280 help='search for reference leaks (needs debug build, '
281 'very slow).' + more_details)
282 group.add_argument('-j', '--multiprocess', metavar='PROCESSES',
283 dest='use_mp', type=int,
284 help='run PROCESSES processes at once')
285 group.add_argument('-T', '--coverage', action='store_true',
286 dest='trace',
287 help='turn on code coverage tracing using the trace '
288 'module')
289 group.add_argument('-D', '--coverdir', metavar='DIR',
290 type=relative_filename,
291 help='directory where coverage files are put')
292 group.add_argument('-N', '--nocoverdir',
293 action='store_const', const=None, dest='coverdir',
294 help='put coverage files alongside modules')
295 group.add_argument('-t', '--threshold', metavar='THRESHOLD',
296 type=int,
297 help='call gc.set_threshold(THRESHOLD)')
298 group.add_argument('-n', '--nowindows', action='store_true',
299 help='suppress error message boxes on Windows')
300 group.add_argument('-F', '--forever', action='store_true',
301 help='run the specified tests in a loop, until an '
302 'error happens; imply --failfast')
303 group.add_argument('--list-tests', action='store_true',
304 help="only write the name of tests that will be run, "
305 "don't execute them")
306 group.add_argument('--list-cases', action='store_true',
307 help='only write the name of test cases that will be run'
308 ' , don\'t execute them')
309 group.add_argument('-P', '--pgo', dest='pgo', action='store_true',
310 help='enable Profile Guided Optimization (PGO) training')
311 group.add_argument('--pgo-extended', action='store_true',
312 help='enable extended PGO training (slower training)')
313 group.add_argument('--fail-env-changed', action='store_true',
314 help='if a test file alters the environment, mark '
315 'the test as failed')
316 group.add_argument('--fail-rerun', action='store_true',
317 help='if a test failed and then passed when re-run, '
318 'mark the tests as failed')
319
320 group.add_argument('--junit-xml', dest='xmlpath', metavar='FILENAME',
321 help='writes JUnit-style XML results to the specified '
322 'file')
323 group.add_argument('--tempdir', metavar='PATH',
324 help='override the working directory for the test run')
325 group.add_argument('--cleanup', action='store_true',
326 help='remove old test_python_* directories')
327 return parser
328
329
330 def relative_filename(string):
331 # CWD is replaced with a temporary dir before calling main(), so we
332 # join it with the saved CWD so it ends up where the user expects.
333 return os.path.join(os_helper.SAVEDCWD, string)
334
335
336 def huntrleaks(string):
337 args = string.split(':')
338 if len(args) not in (2, 3):
339 raise argparse.ArgumentTypeError(
340 'needs 2 or 3 colon-separated arguments')
341 nwarmup = int(args[0]) if args[0] else 5
342 ntracked = int(args[1]) if args[1] else 4
343 fname = args[2] if len(args) > 2 and args[2] else 'reflog.txt'
344 return nwarmup, ntracked, fname
345
346
347 def resources_list(string):
348 u = [x.lower() for x in string.split(',')]
349 for r in u:
350 if r == 'all' or r == 'none':
351 continue
352 if r[0] == '-':
353 r = r[1:]
354 if r not in RESOURCE_NAMES:
355 raise argparse.ArgumentTypeError('invalid resource: ' + r)
356 return u
357
358
359 def _parse_args(args, **kwargs):
360 # Defaults
361 ns = Namespace()
362 for k, v in kwargs.items():
363 if not hasattr(ns, k):
364 raise TypeError('%r is an invalid keyword argument '
365 'for this function' % k)
366 setattr(ns, k, v)
367 if ns.use_resources is None:
368 ns.use_resources = []
369
370 parser = _create_parser()
371 # Issue #14191: argparse doesn't support "intermixed" positional and
372 # optional arguments. Use parse_known_args() as workaround.
373 ns.args = parser.parse_known_args(args=args, namespace=ns)[1]
374 for arg in ns.args:
375 if arg.startswith('-'):
376 parser.error("unrecognized arguments: %s" % arg)
377 sys.exit(1)
378
379 if ns.single and ns.fromfile:
380 parser.error("-s and -f don't go together!")
381 if ns.use_mp is not None and ns.trace:
382 parser.error("-T and -j don't go together!")
383 if ns.python is not None:
384 if ns.use_mp is None:
385 parser.error("-p requires -j!")
386 # The "executable" may be two or more parts, e.g. "node python.js"
387 ns.python = shlex.split(ns.python)
388 if ns.failfast and not (ns.verbose or ns.verbose3):
389 parser.error("-G/--failfast needs either -v or -W")
390 if ns.pgo and (ns.verbose or ns.rerun or ns.verbose3):
391 parser.error("--pgo/-v don't go together!")
392 if ns.pgo_extended:
393 ns.pgo = True # pgo_extended implies pgo
394
395 if ns.nowindows:
396 print("Warning: the --nowindows (-n) option is deprecated. "
397 "Use -vv to display assertions in stderr.", file=sys.stderr)
398
399 if ns.quiet:
400 ns.verbose = 0
401 if ns.timeout is not None:
402 if ns.timeout <= 0:
403 ns.timeout = None
404 if ns.use_mp is not None:
405 if ns.use_mp <= 0:
406 # Use all cores + extras for tests that like to sleep
407 ns.use_mp = 2 + (os.cpu_count() or 1)
408 if ns.use:
409 for a in ns.use:
410 for r in a:
411 if r == 'all':
412 ns.use_resources[:] = ALL_RESOURCES
413 continue
414 if r == 'none':
415 del ns.use_resources[:]
416 continue
417 remove = False
418 if r[0] == '-':
419 remove = True
420 r = r[1:]
421 if remove:
422 if r in ns.use_resources:
423 ns.use_resources.remove(r)
424 elif r not in ns.use_resources:
425 ns.use_resources.append(r)
426 if ns.random_seed is not None:
427 ns.randomize = True
428 if ns.verbose:
429 ns.header = True
430 if ns.huntrleaks and ns.verbose3:
431 ns.verbose3 = False
432 print("WARNING: Disable --verbose3 because it's incompatible with "
433 "--huntrleaks: see http://bugs.python.org/issue27103",
434 file=sys.stderr)
435 if ns.match_filename:
436 if ns.match_tests is None:
437 ns.match_tests = []
438 with open(ns.match_filename) as fp:
439 for line in fp:
440 ns.match_tests.append(line.strip())
441 if ns.ignore_filename:
442 if ns.ignore_tests is None:
443 ns.ignore_tests = []
444 with open(ns.ignore_filename) as fp:
445 for line in fp:
446 ns.ignore_tests.append(line.strip())
447 if ns.forever:
448 # --forever implies --failfast
449 ns.failfast = True
450
451 return ns