python (3.11.7)
1 """distutils.util
2
3 Miscellaneous utility functions -- anything that doesn't fit into
4 one of the other *util.py modules.
5 """
6
7 import importlib.util
8 import os
9 import re
10 import string
11 import subprocess
12 import sys
13 import sysconfig
14 import functools
15
16 from distutils.errors import DistutilsPlatformError, DistutilsByteCompileError
17 from distutils.dep_util import newer
18 from distutils.spawn import spawn
19 from distutils import log
20
21
22 def get_host_platform():
23 """
24 Return a string that identifies the current platform. Use this
25 function to distinguish platform-specific build directories and
26 platform-specific built distributions.
27 """
28
29 # This function initially exposed platforms as defined in Python 3.9
30 # even with older Python versions when distutils was split out.
31 # Now it delegates to stdlib sysconfig, but maintains compatibility.
32
33 if sys.version_info < (3, 8):
34 if os.name == 'nt':
35 if '(arm)' in sys.version.lower():
36 return 'win-arm32'
37 if '(arm64)' in sys.version.lower():
38 return 'win-arm64'
39
40 if sys.version_info < (3, 9):
41 if os.name == "posix" and hasattr(os, 'uname'):
42 osname, host, release, version, machine = os.uname()
43 if osname[:3] == "aix":
44 from .py38compat import aix_platform
45
46 return aix_platform(osname, version, release)
47
48 return sysconfig.get_platform()
49
50
51 def get_platform():
52 if os.name == 'nt':
53 TARGET_TO_PLAT = {
54 'x86': 'win32',
55 'x64': 'win-amd64',
56 'arm': 'win-arm32',
57 'arm64': 'win-arm64',
58 }
59 target = os.environ.get('VSCMD_ARG_TGT_ARCH')
60 return TARGET_TO_PLAT.get(target) or get_host_platform()
61 return get_host_platform()
62
63
64 if sys.platform == 'darwin':
65 _syscfg_macosx_ver = None # cache the version pulled from sysconfig
66 MACOSX_VERSION_VAR = 'MACOSX_DEPLOYMENT_TARGET'
67
68
69 def _clear_cached_macosx_ver():
70 """For testing only. Do not call."""
71 global _syscfg_macosx_ver
72 _syscfg_macosx_ver = None
73
74
75 def get_macosx_target_ver_from_syscfg():
76 """Get the version of macOS latched in the Python interpreter configuration.
77 Returns the version as a string or None if can't obtain one. Cached."""
78 global _syscfg_macosx_ver
79 if _syscfg_macosx_ver is None:
80 from distutils import sysconfig
81
82 ver = sysconfig.get_config_var(MACOSX_VERSION_VAR) or ''
83 if ver:
84 _syscfg_macosx_ver = ver
85 return _syscfg_macosx_ver
86
87
88 def get_macosx_target_ver():
89 """Return the version of macOS for which we are building.
90
91 The target version defaults to the version in sysconfig latched at time
92 the Python interpreter was built, unless overridden by an environment
93 variable. If neither source has a value, then None is returned"""
94
95 syscfg_ver = get_macosx_target_ver_from_syscfg()
96 env_ver = os.environ.get(MACOSX_VERSION_VAR)
97
98 if env_ver:
99 # Validate overridden version against sysconfig version, if have both.
100 # Ensure that the deployment target of the build process is not less
101 # than 10.3 if the interpreter was built for 10.3 or later. This
102 # ensures extension modules are built with correct compatibility
103 # values, specifically LDSHARED which can use
104 # '-undefined dynamic_lookup' which only works on >= 10.3.
105 if (
106 syscfg_ver
107 and split_version(syscfg_ver) >= [10, 3]
108 and split_version(env_ver) < [10, 3]
109 ):
110 my_msg = (
111 '$' + MACOSX_VERSION_VAR + ' mismatch: '
112 'now "%s" but "%s" during configure; '
113 'must use 10.3 or later' % (env_ver, syscfg_ver)
114 )
115 raise DistutilsPlatformError(my_msg)
116 return env_ver
117 return syscfg_ver
118
119
120 def split_version(s):
121 """Convert a dot-separated string into a list of numbers for comparisons"""
122 return [int(n) for n in s.split('.')]
123
124
125 def convert_path(pathname):
126 """Return 'pathname' as a name that will work on the native filesystem,
127 i.e. split it on '/' and put it back together again using the current
128 directory separator. Needed because filenames in the setup script are
129 always supplied in Unix style, and have to be converted to the local
130 convention before we can actually use them in the filesystem. Raises
131 ValueError on non-Unix-ish systems if 'pathname' either starts or
132 ends with a slash.
133 """
134 if os.sep == '/':
135 return pathname
136 if not pathname:
137 return pathname
138 if pathname[0] == '/':
139 raise ValueError("path '%s' cannot be absolute" % pathname)
140 if pathname[-1] == '/':
141 raise ValueError("path '%s' cannot end with '/'" % pathname)
142
143 paths = pathname.split('/')
144 while '.' in paths:
145 paths.remove('.')
146 if not paths:
147 return os.curdir
148 return os.path.join(*paths)
149
150
151 # convert_path ()
152
153
154 def change_root(new_root, pathname):
155 """Return 'pathname' with 'new_root' prepended. If 'pathname' is
156 relative, this is equivalent to "os.path.join(new_root,pathname)".
157 Otherwise, it requires making 'pathname' relative and then joining the
158 two, which is tricky on DOS/Windows and Mac OS.
159 """
160 if os.name == 'posix':
161 if not os.path.isabs(pathname):
162 return os.path.join(new_root, pathname)
163 else:
164 return os.path.join(new_root, pathname[1:])
165
166 elif os.name == 'nt':
167 (drive, path) = os.path.splitdrive(pathname)
168 if path[0] == '\\':
169 path = path[1:]
170 return os.path.join(new_root, path)
171
172 raise DistutilsPlatformError(f"nothing known about platform '{os.name}'")
173
174
175 @functools.lru_cache()
176 def check_environ():
177 """Ensure that 'os.environ' has all the environment variables we
178 guarantee that users can use in config files, command-line options,
179 etc. Currently this includes:
180 HOME - user's home directory (Unix only)
181 PLAT - description of the current platform, including hardware
182 and OS (see 'get_platform()')
183 """
184 if os.name == 'posix' and 'HOME' not in os.environ:
185 try:
186 import pwd
187
188 os.environ['HOME'] = pwd.getpwuid(os.getuid())[5]
189 except (ImportError, KeyError):
190 # bpo-10496: if the current user identifier doesn't exist in the
191 # password database, do nothing
192 pass
193
194 if 'PLAT' not in os.environ:
195 os.environ['PLAT'] = get_platform()
196
197
198 def subst_vars(s, local_vars):
199 """
200 Perform variable substitution on 'string'.
201 Variables are indicated by format-style braces ("{var}").
202 Variable is substituted by the value found in the 'local_vars'
203 dictionary or in 'os.environ' if it's not in 'local_vars'.
204 'os.environ' is first checked/augmented to guarantee that it contains
205 certain values: see 'check_environ()'. Raise ValueError for any
206 variables not found in either 'local_vars' or 'os.environ'.
207 """
208 check_environ()
209 lookup = dict(os.environ)
210 lookup.update((name, str(value)) for name, value in local_vars.items())
211 try:
212 return _subst_compat(s).format_map(lookup)
213 except KeyError as var:
214 raise ValueError(f"invalid variable {var}")
215
216
217 def _subst_compat(s):
218 """
219 Replace shell/Perl-style variable substitution with
220 format-style. For compatibility.
221 """
222
223 def _subst(match):
224 return f'{{{match.group(1)}}}'
225
226 repl = re.sub(r'\$([a-zA-Z_][a-zA-Z_0-9]*)', _subst, s)
227 if repl != s:
228 import warnings
229
230 warnings.warn(
231 "shell/Perl-style substitions are deprecated",
232 DeprecationWarning,
233 )
234 return repl
235
236
237 def grok_environment_error(exc, prefix="error: "):
238 # Function kept for backward compatibility.
239 # Used to try clever things with EnvironmentErrors,
240 # but nowadays str(exception) produces good messages.
241 return prefix + str(exc)
242
243
244 # Needed by 'split_quoted()'
245 _wordchars_re = _squote_re = _dquote_re = None
246
247
248 def _init_regex():
249 global _wordchars_re, _squote_re, _dquote_re
250 _wordchars_re = re.compile(r'[^\\\'\"%s ]*' % string.whitespace)
251 _squote_re = re.compile(r"'(?:[^'\\]|\\.)*'")
252 _dquote_re = re.compile(r'"(?:[^"\\]|\\.)*"')
253
254
255 def split_quoted(s):
256 """Split a string up according to Unix shell-like rules for quotes and
257 backslashes. In short: words are delimited by spaces, as long as those
258 spaces are not escaped by a backslash, or inside a quoted string.
259 Single and double quotes are equivalent, and the quote characters can
260 be backslash-escaped. The backslash is stripped from any two-character
261 escape sequence, leaving only the escaped character. The quote
262 characters are stripped from any quoted string. Returns a list of
263 words.
264 """
265
266 # This is a nice algorithm for splitting up a single string, since it
267 # doesn't require character-by-character examination. It was a little
268 # bit of a brain-bender to get it working right, though...
269 if _wordchars_re is None:
270 _init_regex()
271
272 s = s.strip()
273 words = []
274 pos = 0
275
276 while s:
277 m = _wordchars_re.match(s, pos)
278 end = m.end()
279 if end == len(s):
280 words.append(s[:end])
281 break
282
283 if s[end] in string.whitespace:
284 # unescaped, unquoted whitespace: now
285 # we definitely have a word delimiter
286 words.append(s[:end])
287 s = s[end:].lstrip()
288 pos = 0
289
290 elif s[end] == '\\':
291 # preserve whatever is being escaped;
292 # will become part of the current word
293 s = s[:end] + s[end + 1 :]
294 pos = end + 1
295
296 else:
297 if s[end] == "'": # slurp singly-quoted string
298 m = _squote_re.match(s, end)
299 elif s[end] == '"': # slurp doubly-quoted string
300 m = _dquote_re.match(s, end)
301 else:
302 raise RuntimeError("this can't happen (bad char '%c')" % s[end])
303
304 if m is None:
305 raise ValueError("bad string (mismatched %s quotes?)" % s[end])
306
307 (beg, end) = m.span()
308 s = s[:beg] + s[beg + 1 : end - 1] + s[end:]
309 pos = m.end() - 2
310
311 if pos >= len(s):
312 words.append(s)
313 break
314
315 return words
316
317
318 # split_quoted ()
319
320
321 def execute(func, args, msg=None, verbose=0, dry_run=0):
322 """Perform some action that affects the outside world (eg. by
323 writing to the filesystem). Such actions are special because they
324 are disabled by the 'dry_run' flag. This method takes care of all
325 that bureaucracy for you; all you have to do is supply the
326 function to call and an argument tuple for it (to embody the
327 "external action" being performed), and an optional message to
328 print.
329 """
330 if msg is None:
331 msg = "{}{!r}".format(func.__name__, args)
332 if msg[-2:] == ',)': # correct for singleton tuple
333 msg = msg[0:-2] + ')'
334
335 log.info(msg)
336 if not dry_run:
337 func(*args)
338
339
340 def strtobool(val):
341 """Convert a string representation of truth to true (1) or false (0).
342
343 True values are 'y', 'yes', 't', 'true', 'on', and '1'; false values
344 are 'n', 'no', 'f', 'false', 'off', and '0'. Raises ValueError if
345 'val' is anything else.
346 """
347 val = val.lower()
348 if val in ('y', 'yes', 't', 'true', 'on', '1'):
349 return 1
350 elif val in ('n', 'no', 'f', 'false', 'off', '0'):
351 return 0
352 else:
353 raise ValueError("invalid truth value {!r}".format(val))
354
355
356 def byte_compile( # noqa: C901
357 py_files,
358 optimize=0,
359 force=0,
360 prefix=None,
361 base_dir=None,
362 verbose=1,
363 dry_run=0,
364 direct=None,
365 ):
366 """Byte-compile a collection of Python source files to .pyc
367 files in a __pycache__ subdirectory. 'py_files' is a list
368 of files to compile; any files that don't end in ".py" are silently
369 skipped. 'optimize' must be one of the following:
370 0 - don't optimize
371 1 - normal optimization (like "python -O")
372 2 - extra optimization (like "python -OO")
373 If 'force' is true, all files are recompiled regardless of
374 timestamps.
375
376 The source filename encoded in each bytecode file defaults to the
377 filenames listed in 'py_files'; you can modify these with 'prefix' and
378 'basedir'. 'prefix' is a string that will be stripped off of each
379 source filename, and 'base_dir' is a directory name that will be
380 prepended (after 'prefix' is stripped). You can supply either or both
381 (or neither) of 'prefix' and 'base_dir', as you wish.
382
383 If 'dry_run' is true, doesn't actually do anything that would
384 affect the filesystem.
385
386 Byte-compilation is either done directly in this interpreter process
387 with the standard py_compile module, or indirectly by writing a
388 temporary script and executing it. Normally, you should let
389 'byte_compile()' figure out to use direct compilation or not (see
390 the source for details). The 'direct' flag is used by the script
391 generated in indirect mode; unless you know what you're doing, leave
392 it set to None.
393 """
394
395 # nothing is done if sys.dont_write_bytecode is True
396 if sys.dont_write_bytecode:
397 raise DistutilsByteCompileError('byte-compiling is disabled.')
398
399 # First, if the caller didn't force us into direct or indirect mode,
400 # figure out which mode we should be in. We take a conservative
401 # approach: choose direct mode *only* if the current interpreter is
402 # in debug mode and optimize is 0. If we're not in debug mode (-O
403 # or -OO), we don't know which level of optimization this
404 # interpreter is running with, so we can't do direct
405 # byte-compilation and be certain that it's the right thing. Thus,
406 # always compile indirectly if the current interpreter is in either
407 # optimize mode, or if either optimization level was requested by
408 # the caller.
409 if direct is None:
410 direct = __debug__ and optimize == 0
411
412 # "Indirect" byte-compilation: write a temporary script and then
413 # run it with the appropriate flags.
414 if not direct:
415 try:
416 from tempfile import mkstemp
417
418 (script_fd, script_name) = mkstemp(".py")
419 except ImportError:
420 from tempfile import mktemp
421
422 (script_fd, script_name) = None, mktemp(".py")
423 log.info("writing byte-compilation script '%s'", script_name)
424 if not dry_run:
425 if script_fd is not None:
426 script = os.fdopen(script_fd, "w")
427 else:
428 script = open(script_name, "w")
429
430 with script:
431 script.write(
432 """\
433 from distutils.util import byte_compile
434 files = [
435 """
436 )
437
438 # XXX would be nice to write absolute filenames, just for
439 # safety's sake (script should be more robust in the face of
440 # chdir'ing before running it). But this requires abspath'ing
441 # 'prefix' as well, and that breaks the hack in build_lib's
442 # 'byte_compile()' method that carefully tacks on a trailing
443 # slash (os.sep really) to make sure the prefix here is "just
444 # right". This whole prefix business is rather delicate -- the
445 # problem is that it's really a directory, but I'm treating it
446 # as a dumb string, so trailing slashes and so forth matter.
447
448 script.write(",\n".join(map(repr, py_files)) + "]\n")
449 script.write(
450 """
451 byte_compile(files, optimize=%r, force=%r,
452 prefix=%r, base_dir=%r,
453 verbose=%r, dry_run=0,
454 direct=1)
455 """
456 % (optimize, force, prefix, base_dir, verbose)
457 )
458
459 cmd = [sys.executable]
460 cmd.extend(subprocess._optim_args_from_interpreter_flags())
461 cmd.append(script_name)
462 spawn(cmd, dry_run=dry_run)
463 execute(os.remove, (script_name,), "removing %s" % script_name, dry_run=dry_run)
464
465 # "Direct" byte-compilation: use the py_compile module to compile
466 # right here, right now. Note that the script generated in indirect
467 # mode simply calls 'byte_compile()' in direct mode, a weird sort of
468 # cross-process recursion. Hey, it works!
469 else:
470 from py_compile import compile
471
472 for file in py_files:
473 if file[-3:] != ".py":
474 # This lets us be lazy and not filter filenames in
475 # the "install_lib" command.
476 continue
477
478 # Terminology from the py_compile module:
479 # cfile - byte-compiled file
480 # dfile - purported source filename (same as 'file' by default)
481 if optimize >= 0:
482 opt = '' if optimize == 0 else optimize
483 cfile = importlib.util.cache_from_source(file, optimization=opt)
484 else:
485 cfile = importlib.util.cache_from_source(file)
486 dfile = file
487 if prefix:
488 if file[: len(prefix)] != prefix:
489 raise ValueError(
490 "invalid prefix: filename %r doesn't start with %r"
491 % (file, prefix)
492 )
493 dfile = dfile[len(prefix) :]
494 if base_dir:
495 dfile = os.path.join(base_dir, dfile)
496
497 cfile_base = os.path.basename(cfile)
498 if direct:
499 if force or newer(file, cfile):
500 log.info("byte-compiling %s to %s", file, cfile_base)
501 if not dry_run:
502 compile(file, cfile, dfile)
503 else:
504 log.debug("skipping byte-compilation of %s to %s", file, cfile_base)
505
506
507 def rfc822_escape(header):
508 """Return a version of the string escaped for inclusion in an
509 RFC-822 header, by ensuring there are 8 spaces space after each newline.
510 """
511 lines = header.split('\n')
512 sep = '\n' + 8 * ' '
513 return sep.join(lines)
Browse
Usage
Versions