python (3.11.7)
1 import argparse
2 import os.path
3 import shlex
4 import sys
5 from test.support import os_helper
6 from .utils import ALL_RESOURCES, RESOURCE_NAMES, TestFilter
7
8
9 USAGE = """\
10 python -m test [options] [test_name1 [test_name2 ...]]
11 python path/to/Lib/test/regrtest.py [options] [test_name1 [test_name2 ...]]
12 """
13
14 DESCRIPTION = """\
15 Run Python regression tests.
16
17 If no arguments or options are provided, finds all files matching
18 the pattern "test_*" in the Lib/test subdirectory and runs
19 them in alphabetical order (but see -M and -u, below, for exceptions).
20
21 For more rigorous testing, it is useful to use the following
22 command line:
23
24 python -E -Wd -m test [options] [test_name1 ...]
25 """
26
27 EPILOG = """\
28 Additional option details:
29
30 -r randomizes test execution order. You can use --randseed=int to provide an
31 int seed value for the randomizer. The randseed value will be used
32 to set seeds for all random usages in tests
33 (including randomizing the tests order if -r is set).
34 By default we always set random seed, but do not randomize test order.
35
36 -s On the first invocation of regrtest using -s, the first test file found
37 or the first test file given on the command line is run, and the name of
38 the next test is recorded in a file named pynexttest. If run from the
39 Python build directory, pynexttest is located in the 'build' subdirectory,
40 otherwise it is located in tempfile.gettempdir(). On subsequent runs,
41 the test in pynexttest is run, and the next test is written to pynexttest.
42 When the last test has been run, pynexttest is deleted. In this way it
43 is possible to single step through the test files. This is useful when
44 doing memory analysis on the Python interpreter, which process tends to
45 consume too many resources to run the full regression test non-stop.
46
47 -S is used to continue running tests after an aborted run. It will
48 maintain the order a standard run (ie, this assumes -r is not used).
49 This is useful after the tests have prematurely stopped for some external
50 reason and you want to start running from where you left off rather
51 than starting from the beginning.
52
53 -f reads the names of tests from the file given as f's argument, one
54 or more test names per line. Whitespace is ignored. Blank lines and
55 lines beginning with '#' are ignored. This is especially useful for
56 whittling down failures involving interactions among tests.
57
58 -L causes the leaks(1) command to be run just before exit if it exists.
59 leaks(1) is available on Mac OS X and presumably on some other
60 FreeBSD-derived systems.
61
62 -R runs each test several times and examines sys.gettotalrefcount() to
63 see if the test appears to be leaking references. The argument should
64 be of the form stab:run:fname where 'stab' is the number of times the
65 test is run to let gettotalrefcount settle down, 'run' is the number
66 of times further it is run and 'fname' is the name of the file the
67 reports are written to. These parameters all have defaults (5, 4 and
68 "reflog.txt" respectively), and the minimal invocation is '-R :'.
69
70 -M runs tests that require an exorbitant amount of memory. These tests
71 typically try to ascertain containers keep working when containing more than
72 2 billion objects, which only works on 64-bit systems. There are also some
73 tests that try to exhaust the address space of the process, which only makes
74 sense on 32-bit systems with at least 2Gb of memory. The passed-in memlimit,
75 which is a string in the form of '2.5Gb', determines how much memory the
76 tests will limit themselves to (but they may go slightly over.) The number
77 shouldn't be more memory than the machine has (including swap memory). You
78 should also keep in mind that swap memory is generally much, much slower
79 than RAM, and setting memlimit to all available RAM or higher will heavily
80 tax the machine. On the other hand, it is no use running these tests with a
81 limit of less than 2.5Gb, and many require more than 20Gb. Tests that expect
82 to use more than memlimit memory will be skipped. The big-memory tests
83 generally run very, very long.
84
85 -u is used to specify which special resource intensive tests to run,
86 such as those requiring large file support or network connectivity.
87 The argument is a comma-separated list of words indicating the
88 resources to test. Currently only the following are defined:
89
90 all - Enable all special resources.
91
92 none - Disable all special resources (this is the default).
93
94 audio - Tests that use the audio device. (There are known
95 cases of broken audio drivers that can crash Python or
96 even the Linux kernel.)
97
98 curses - Tests that use curses and will modify the terminal's
99 state and output modes.
100
101 largefile - It is okay to run some test that may create huge
102 files. These tests can take a long time and may
103 consume >2 GiB of disk space temporarily.
104
105 network - It is okay to run tests that use external network
106 resource, e.g. testing SSL support for sockets.
107
108 decimal - Test the decimal module against a large suite that
109 verifies compliance with standards.
110
111 cpu - Used for certain CPU-heavy tests.
112
113 walltime - Long running but not CPU-bound tests.
114
115 subprocess Run all tests for the subprocess module.
116
117 urlfetch - It is okay to download files required on testing.
118
119 gui - Run tests that require a running GUI.
120
121 tzdata - Run tests that require timezone data.
122
123 To enable all resources except one, use '-uall,-<resource>'. For
124 example, to run all the tests except for the gui tests, give the
125 option '-uall,-gui'.
126
127 --matchfile filters tests using a text file, one pattern per line.
128 Pattern examples:
129
130 - test method: test_stat_attributes
131 - test class: FileTests
132 - test identifier: test_os.FileTests.test_stat_attributes
133 """
134
135
136 class ESC[4;38;5;81mNamespace(ESC[4;38;5;149margparseESC[4;38;5;149m.ESC[4;38;5;149mNamespace):
137 def __init__(self, **kwargs) -> None:
138 self.ci = False
139 self.testdir = None
140 self.verbose = 0
141 self.quiet = False
142 self.exclude = False
143 self.cleanup = False
144 self.wait = False
145 self.list_cases = False
146 self.list_tests = False
147 self.single = False
148 self.randomize = False
149 self.fromfile = None
150 self.fail_env_changed = False
151 self.use_resources = None
152 self.trace = False
153 self.coverdir = 'coverage'
154 self.runleaks = False
155 self.huntrleaks: tuple[int, int, str] | None = None
156 self.rerun = False
157 self.verbose3 = False
158 self.print_slow = False
159 self.random_seed = None
160 self.use_mp = None
161 self.forever = False
162 self.header = False
163 self.failfast = False
164 self.match_tests: TestFilter = []
165 self.pgo = False
166 self.pgo_extended = False
167 self.worker_json = None
168 self.start = None
169 self.timeout = None
170 self.memlimit = None
171 self.threshold = None
172 self.fail_rerun = False
173 self.tempdir = None
174 self._add_python_opts = True
175
176 super().__init__(**kwargs)
177
178
179 class ESC[4;38;5;81m_ArgParser(ESC[4;38;5;149margparseESC[4;38;5;149m.ESC[4;38;5;149mArgumentParser):
180
181 def error(self, message):
182 super().error(message + "\nPass -h or --help for complete help.")
183
184
185 class ESC[4;38;5;81mFilterAction(ESC[4;38;5;149margparseESC[4;38;5;149m.ESC[4;38;5;149mAction):
186 def __call__(self, parser, namespace, value, option_string=None):
187 items = getattr(namespace, self.dest)
188 items.append((value, self.const))
189
190
191 class ESC[4;38;5;81mFromFileFilterAction(ESC[4;38;5;149margparseESC[4;38;5;149m.ESC[4;38;5;149mAction):
192 def __call__(self, parser, namespace, value, option_string=None):
193 items = getattr(namespace, self.dest)
194 with open(value, encoding='utf-8') as fp:
195 for line in fp:
196 items.append((line.strip(), self.const))
197
198
199 def _create_parser():
200 # Set prog to prevent the uninformative "__main__.py" from displaying in
201 # error messages when using "python -m test ...".
202 parser = _ArgParser(prog='regrtest.py',
203 usage=USAGE,
204 description=DESCRIPTION,
205 epilog=EPILOG,
206 add_help=False,
207 formatter_class=argparse.RawDescriptionHelpFormatter)
208 parser.set_defaults(match_tests=[])
209
210 # Arguments with this clause added to its help are described further in
211 # the epilog's "Additional option details" section.
212 more_details = ' See the section at bottom for more details.'
213
214 group = parser.add_argument_group('General options')
215 # We add help explicitly to control what argument group it renders under.
216 group.add_argument('-h', '--help', action='help',
217 help='show this help message and exit')
218 group.add_argument('--fast-ci', action='store_true',
219 help='Fast Continuous Integration (CI) mode used by '
220 'GitHub Actions')
221 group.add_argument('--slow-ci', action='store_true',
222 help='Slow Continuous Integration (CI) mode used by '
223 'buildbot workers')
224 group.add_argument('--timeout', metavar='TIMEOUT',
225 help='dump the traceback and exit if a test takes '
226 'more than TIMEOUT seconds; disabled if TIMEOUT '
227 'is negative or equals to zero')
228 group.add_argument('--wait', action='store_true',
229 help='wait for user input, e.g., allow a debugger '
230 'to be attached')
231 group.add_argument('-S', '--start', metavar='START',
232 help='the name of the test at which to start.' +
233 more_details)
234 group.add_argument('-p', '--python', metavar='PYTHON',
235 help='Command to run Python test subprocesses with.')
236 group.add_argument('--randseed', metavar='SEED',
237 dest='random_seed', type=int,
238 help='pass a global random seed')
239
240 group = parser.add_argument_group('Verbosity')
241 group.add_argument('-v', '--verbose', action='count',
242 help='run tests in verbose mode with output to stdout')
243 group.add_argument('-w', '--rerun', action='store_true',
244 help='re-run failed tests in verbose mode')
245 group.add_argument('--verbose2', action='store_true', dest='rerun',
246 help='deprecated alias to --rerun')
247 group.add_argument('-W', '--verbose3', action='store_true',
248 help='display test output on failure')
249 group.add_argument('-q', '--quiet', action='store_true',
250 help='no output unless one or more tests fail')
251 group.add_argument('-o', '--slowest', action='store_true', dest='print_slow',
252 help='print the slowest 10 tests')
253 group.add_argument('--header', action='store_true',
254 help='print header with interpreter info')
255
256 group = parser.add_argument_group('Selecting tests')
257 group.add_argument('-r', '--randomize', action='store_true',
258 help='randomize test execution order.' + more_details)
259 group.add_argument('-f', '--fromfile', metavar='FILE',
260 help='read names of tests to run from a file.' +
261 more_details)
262 group.add_argument('-x', '--exclude', action='store_true',
263 help='arguments are tests to *exclude*')
264 group.add_argument('-s', '--single', action='store_true',
265 help='single step through a set of tests.' +
266 more_details)
267 group.add_argument('-m', '--match', metavar='PAT',
268 dest='match_tests', action=FilterAction, const=True,
269 help='match test cases and methods with glob pattern PAT')
270 group.add_argument('-i', '--ignore', metavar='PAT',
271 dest='match_tests', action=FilterAction, const=False,
272 help='ignore test cases and methods with glob pattern PAT')
273 group.add_argument('--matchfile', metavar='FILENAME',
274 dest='match_tests',
275 action=FromFileFilterAction, const=True,
276 help='similar to --match but get patterns from a '
277 'text file, one pattern per line')
278 group.add_argument('--ignorefile', metavar='FILENAME',
279 dest='match_tests',
280 action=FromFileFilterAction, const=False,
281 help='similar to --matchfile but it receives patterns '
282 'from text file to ignore')
283 group.add_argument('-G', '--failfast', action='store_true',
284 help='fail as soon as a test fails (only with -v or -W)')
285 group.add_argument('-u', '--use', metavar='RES1,RES2,...',
286 action='append', type=resources_list,
287 help='specify which special resource intensive tests '
288 'to run.' + more_details)
289 group.add_argument('-M', '--memlimit', metavar='LIMIT',
290 help='run very large memory-consuming tests.' +
291 more_details)
292 group.add_argument('--testdir', metavar='DIR',
293 type=relative_filename,
294 help='execute test files in the specified directory '
295 '(instead of the Python stdlib test suite)')
296
297 group = parser.add_argument_group('Special runs')
298 group.add_argument('-L', '--runleaks', action='store_true',
299 help='run the leaks(1) command just before exit.' +
300 more_details)
301 group.add_argument('-R', '--huntrleaks', metavar='RUNCOUNTS',
302 type=huntrleaks,
303 help='search for reference leaks (needs debug build, '
304 'very slow).' + more_details)
305 group.add_argument('-j', '--multiprocess', metavar='PROCESSES',
306 dest='use_mp', type=int,
307 help='run PROCESSES processes at once')
308 group.add_argument('-T', '--coverage', action='store_true',
309 dest='trace',
310 help='turn on code coverage tracing using the trace '
311 'module')
312 group.add_argument('-D', '--coverdir', metavar='DIR',
313 type=relative_filename,
314 help='directory where coverage files are put')
315 group.add_argument('-N', '--nocoverdir',
316 action='store_const', const=None, dest='coverdir',
317 help='put coverage files alongside modules')
318 group.add_argument('-t', '--threshold', metavar='THRESHOLD',
319 type=int,
320 help='call gc.set_threshold(THRESHOLD)')
321 group.add_argument('-n', '--nowindows', action='store_true',
322 help='suppress error message boxes on Windows')
323 group.add_argument('-F', '--forever', action='store_true',
324 help='run the specified tests in a loop, until an '
325 'error happens; imply --failfast')
326 group.add_argument('--list-tests', action='store_true',
327 help="only write the name of tests that will be run, "
328 "don't execute them")
329 group.add_argument('--list-cases', action='store_true',
330 help='only write the name of test cases that will be run'
331 ' , don\'t execute them')
332 group.add_argument('-P', '--pgo', dest='pgo', action='store_true',
333 help='enable Profile Guided Optimization (PGO) training')
334 group.add_argument('--pgo-extended', action='store_true',
335 help='enable extended PGO training (slower training)')
336 group.add_argument('--fail-env-changed', action='store_true',
337 help='if a test file alters the environment, mark '
338 'the test as failed')
339 group.add_argument('--fail-rerun', action='store_true',
340 help='if a test failed and then passed when re-run, '
341 'mark the tests as failed')
342
343 group.add_argument('--junit-xml', dest='xmlpath', metavar='FILENAME',
344 help='writes JUnit-style XML results to the specified '
345 'file')
346 group.add_argument('--tempdir', metavar='PATH',
347 help='override the working directory for the test run')
348 group.add_argument('--cleanup', action='store_true',
349 help='remove old test_python_* directories')
350 group.add_argument('--dont-add-python-opts', dest='_add_python_opts',
351 action='store_false',
352 help="internal option, don't use it")
353 return parser
354
355
356 def relative_filename(string):
357 # CWD is replaced with a temporary dir before calling main(), so we
358 # join it with the saved CWD so it ends up where the user expects.
359 return os.path.join(os_helper.SAVEDCWD, string)
360
361
362 def huntrleaks(string):
363 args = string.split(':')
364 if len(args) not in (2, 3):
365 raise argparse.ArgumentTypeError(
366 'needs 2 or 3 colon-separated arguments')
367 nwarmup = int(args[0]) if args[0] else 5
368 ntracked = int(args[1]) if args[1] else 4
369 fname = args[2] if len(args) > 2 and args[2] else 'reflog.txt'
370 return nwarmup, ntracked, fname
371
372
373 def resources_list(string):
374 u = [x.lower() for x in string.split(',')]
375 for r in u:
376 if r == 'all' or r == 'none':
377 continue
378 if r[0] == '-':
379 r = r[1:]
380 if r not in RESOURCE_NAMES:
381 raise argparse.ArgumentTypeError('invalid resource: ' + r)
382 return u
383
384
385 def _parse_args(args, **kwargs):
386 # Defaults
387 ns = Namespace()
388 for k, v in kwargs.items():
389 if not hasattr(ns, k):
390 raise TypeError('%r is an invalid keyword argument '
391 'for this function' % k)
392 setattr(ns, k, v)
393 if ns.use_resources is None:
394 ns.use_resources = []
395
396 parser = _create_parser()
397 # Issue #14191: argparse doesn't support "intermixed" positional and
398 # optional arguments. Use parse_known_args() as workaround.
399 ns.args = parser.parse_known_args(args=args, namespace=ns)[1]
400 for arg in ns.args:
401 if arg.startswith('-'):
402 parser.error("unrecognized arguments: %s" % arg)
403
404 if ns.timeout is not None:
405 # Support "--timeout=" (no value) so Makefile.pre.pre TESTTIMEOUT
406 # can be used by "make buildbottest" and "make test".
407 if ns.timeout != "":
408 try:
409 ns.timeout = float(ns.timeout)
410 except ValueError:
411 parser.error(f"invalid timeout value: {ns.timeout!r}")
412 else:
413 ns.timeout = None
414
415 # Continuous Integration (CI): common options for fast/slow CI modes
416 if ns.slow_ci or ns.fast_ci:
417 # Similar to options:
418 #
419 # -j0 --randomize --fail-env-changed --fail-rerun --rerun
420 # --slowest --verbose3
421 if ns.use_mp is None:
422 ns.use_mp = 0
423 ns.randomize = True
424 ns.fail_env_changed = True
425 ns.fail_rerun = True
426 if ns.python is None:
427 ns.rerun = True
428 ns.print_slow = True
429 ns.verbose3 = True
430 else:
431 ns._add_python_opts = False
432
433 # When both --slow-ci and --fast-ci options are present,
434 # --slow-ci has the priority
435 if ns.slow_ci:
436 # Similar to: -u "all" --timeout=1200
437 if ns.use is None:
438 ns.use = []
439 ns.use.insert(0, ['all'])
440 if ns.timeout is None:
441 ns.timeout = 1200 # 20 minutes
442 elif ns.fast_ci:
443 # Similar to: -u "all,-cpu" --timeout=600
444 if ns.use is None:
445 ns.use = []
446 ns.use.insert(0, ['all', '-cpu'])
447 if ns.timeout is None:
448 ns.timeout = 600 # 10 minutes
449
450 if ns.single and ns.fromfile:
451 parser.error("-s and -f don't go together!")
452 if ns.use_mp is not None and ns.trace:
453 parser.error("-T and -j don't go together!")
454 if ns.python is not None:
455 if ns.use_mp is None:
456 parser.error("-p requires -j!")
457 # The "executable" may be two or more parts, e.g. "node python.js"
458 ns.python = shlex.split(ns.python)
459 if ns.failfast and not (ns.verbose or ns.verbose3):
460 parser.error("-G/--failfast needs either -v or -W")
461 if ns.pgo and (ns.verbose or ns.rerun or ns.verbose3):
462 parser.error("--pgo/-v don't go together!")
463 if ns.pgo_extended:
464 ns.pgo = True # pgo_extended implies pgo
465
466 if ns.nowindows:
467 print("Warning: the --nowindows (-n) option is deprecated. "
468 "Use -vv to display assertions in stderr.", file=sys.stderr)
469
470 if ns.quiet:
471 ns.verbose = 0
472 if ns.timeout is not None:
473 if ns.timeout <= 0:
474 ns.timeout = None
475 if ns.use:
476 for a in ns.use:
477 for r in a:
478 if r == 'all':
479 ns.use_resources[:] = ALL_RESOURCES
480 continue
481 if r == 'none':
482 del ns.use_resources[:]
483 continue
484 remove = False
485 if r[0] == '-':
486 remove = True
487 r = r[1:]
488 if remove:
489 if r in ns.use_resources:
490 ns.use_resources.remove(r)
491 elif r not in ns.use_resources:
492 ns.use_resources.append(r)
493 if ns.random_seed is not None:
494 ns.randomize = True
495 if ns.verbose:
496 ns.header = True
497 # When -jN option is used, a worker process does not use --verbose3
498 # and so -R 3:3 -jN --verbose3 just works as expected: there is no false
499 # alarm about memory leak.
500 if ns.huntrleaks and ns.verbose3 and ns.use_mp is None:
501 ns.verbose3 = False
502 # run_single_test() replaces sys.stdout with io.StringIO if verbose3
503 # is true. In this case, huntrleaks sees an write into StringIO as
504 # a memory leak, whereas it is not (gh-71290).
505 print("WARNING: Disable --verbose3 because it's incompatible with "
506 "--huntrleaks without -jN option",
507 file=sys.stderr)
508 if ns.forever:
509 # --forever implies --failfast
510 ns.failfast = True
511
512 if ns.huntrleaks:
513 warmup, repetitions, _ = ns.huntrleaks
514 if warmup < 1 or repetitions < 1:
515 msg = ("Invalid values for the --huntrleaks/-R parameters. The "
516 "number of warmups and repetitions must be at least 1 "
517 "each (1:1).")
518 print(msg, file=sys.stderr, flush=True)
519 sys.exit(2)
520
521 return ns
Browse
Usage
Versions