1 """Module/script to byte-compile all .py files to .pyc files.
2
3 When called as a script with arguments, this compiles the directories
4 given as arguments recursively; the -l option prevents it from
5 recursing into directories.
6
7 Without arguments, it compiles all modules on sys.path, without
8 recursing into subdirectories. (Even though it should do so for
9 packages -- for now, you'll have to deal with packages separately.)
10
11 See module py_compile for details of the actual byte-compilation.
12 """
13 import os
14 import sys
15 import importlib.util
16 import py_compile
17 import struct
18 import filecmp
19
20 from functools import partial
21 from pathlib import Path
22
23 __all__ = ["compile_dir","compile_file","compile_path"]
24
25 def _walk_dir(dir, maxlevels, quiet=0):
26 if quiet < 2 and isinstance(dir, os.PathLike):
27 dir = os.fspath(dir)
28 if not quiet:
29 print('Listing {!r}...'.format(dir))
30 try:
31 names = os.listdir(dir)
32 except OSError:
33 if quiet < 2:
34 print("Can't list {!r}".format(dir))
35 names = []
36 names.sort()
37 for name in names:
38 if name == '__pycache__':
39 continue
40 fullname = os.path.join(dir, name)
41 if not os.path.isdir(fullname):
42 yield fullname
43 elif (maxlevels > 0 and name != os.curdir and name != os.pardir and
44 os.path.isdir(fullname) and not os.path.islink(fullname)):
45 yield from _walk_dir(fullname, maxlevels=maxlevels - 1,
46 quiet=quiet)
47
48 def compile_dir(dir, maxlevels=None, ddir=None, force=False,
49 rx=None, quiet=0, legacy=False, optimize=-1, workers=1,
50 invalidation_mode=None, *, stripdir=None,
51 prependdir=None, limit_sl_dest=None, hardlink_dupes=False):
52 """Byte-compile all modules in the given directory tree.
53
54 Arguments (only dir is required):
55
56 dir: the directory to byte-compile
57 maxlevels: maximum recursion level (default `sys.getrecursionlimit()`)
58 ddir: the directory that will be prepended to the path to the
59 file as it is compiled into each byte-code file.
60 force: if True, force compilation, even if timestamps are up-to-date
61 quiet: full output with False or 0, errors only with 1,
62 no output with 2
63 legacy: if True, produce legacy pyc paths instead of PEP 3147 paths
64 optimize: int or list of optimization levels or -1 for level of
65 the interpreter. Multiple levels leads to multiple compiled
66 files each with one optimization level.
67 workers: maximum number of parallel workers
68 invalidation_mode: how the up-to-dateness of the pyc will be checked
69 stripdir: part of path to left-strip from source file path
70 prependdir: path to prepend to beginning of original file path, applied
71 after stripdir
72 limit_sl_dest: ignore symlinks if they are pointing outside of
73 the defined path
74 hardlink_dupes: hardlink duplicated pyc files
75 """
76 ProcessPoolExecutor = None
77 if ddir is not None and (stripdir is not None or prependdir is not None):
78 raise ValueError(("Destination dir (ddir) cannot be used "
79 "in combination with stripdir or prependdir"))
80 if ddir is not None:
81 stripdir = dir
82 prependdir = ddir
83 ddir = None
84 if workers < 0:
85 raise ValueError('workers must be greater or equal to 0')
86 if workers != 1:
87 # Check if this is a system where ProcessPoolExecutor can function.
88 from concurrent.futures.process import _check_system_limits
89 try:
90 _check_system_limits()
91 except NotImplementedError:
92 workers = 1
93 else:
94 from concurrent.futures import ProcessPoolExecutor
95 if maxlevels is None:
96 maxlevels = sys.getrecursionlimit()
97 files = _walk_dir(dir, quiet=quiet, maxlevels=maxlevels)
98 success = True
99 if workers != 1 and ProcessPoolExecutor is not None:
100 # If workers == 0, let ProcessPoolExecutor choose
101 workers = workers or None
102 with ProcessPoolExecutor(max_workers=workers) as executor:
103 results = executor.map(partial(compile_file,
104 ddir=ddir, force=force,
105 rx=rx, quiet=quiet,
106 legacy=legacy,
107 optimize=optimize,
108 invalidation_mode=invalidation_mode,
109 stripdir=stripdir,
110 prependdir=prependdir,
111 limit_sl_dest=limit_sl_dest,
112 hardlink_dupes=hardlink_dupes),
113 files)
114 success = min(results, default=True)
115 else:
116 for file in files:
117 if not compile_file(file, ddir, force, rx, quiet,
118 legacy, optimize, invalidation_mode,
119 stripdir=stripdir, prependdir=prependdir,
120 limit_sl_dest=limit_sl_dest,
121 hardlink_dupes=hardlink_dupes):
122 success = False
123 return success
124
125 def compile_file(fullname, ddir=None, force=False, rx=None, quiet=0,
126 legacy=False, optimize=-1,
127 invalidation_mode=None, *, stripdir=None, prependdir=None,
128 limit_sl_dest=None, hardlink_dupes=False):
129 """Byte-compile one file.
130
131 Arguments (only fullname is required):
132
133 fullname: the file to byte-compile
134 ddir: if given, the directory name compiled in to the
135 byte-code file.
136 force: if True, force compilation, even if timestamps are up-to-date
137 quiet: full output with False or 0, errors only with 1,
138 no output with 2
139 legacy: if True, produce legacy pyc paths instead of PEP 3147 paths
140 optimize: int or list of optimization levels or -1 for level of
141 the interpreter. Multiple levels leads to multiple compiled
142 files each with one optimization level.
143 invalidation_mode: how the up-to-dateness of the pyc will be checked
144 stripdir: part of path to left-strip from source file path
145 prependdir: path to prepend to beginning of original file path, applied
146 after stripdir
147 limit_sl_dest: ignore symlinks if they are pointing outside of
148 the defined path.
149 hardlink_dupes: hardlink duplicated pyc files
150 """
151
152 if ddir is not None and (stripdir is not None or prependdir is not None):
153 raise ValueError(("Destination dir (ddir) cannot be used "
154 "in combination with stripdir or prependdir"))
155
156 success = True
157 fullname = os.fspath(fullname)
158 stripdir = os.fspath(stripdir) if stripdir is not None else None
159 name = os.path.basename(fullname)
160
161 dfile = None
162
163 if ddir is not None:
164 dfile = os.path.join(ddir, name)
165
166 if stripdir is not None:
167 fullname_parts = fullname.split(os.path.sep)
168 stripdir_parts = stripdir.split(os.path.sep)
169 ddir_parts = list(fullname_parts)
170
171 for spart, opart in zip(stripdir_parts, fullname_parts):
172 if spart == opart:
173 ddir_parts.remove(spart)
174
175 dfile = os.path.join(*ddir_parts)
176
177 if prependdir is not None:
178 if dfile is None:
179 dfile = os.path.join(prependdir, fullname)
180 else:
181 dfile = os.path.join(prependdir, dfile)
182
183 if isinstance(optimize, int):
184 optimize = [optimize]
185
186 # Use set() to remove duplicates.
187 # Use sorted() to create pyc files in a deterministic order.
188 optimize = sorted(set(optimize))
189
190 if hardlink_dupes and len(optimize) < 2:
191 raise ValueError("Hardlinking of duplicated bytecode makes sense "
192 "only for more than one optimization level")
193
194 if rx is not None:
195 mo = rx.search(fullname)
196 if mo:
197 return success
198
199 if limit_sl_dest is not None and os.path.islink(fullname):
200 if Path(limit_sl_dest).resolve() not in Path(fullname).resolve().parents:
201 return success
202
203 opt_cfiles = {}
204
205 if os.path.isfile(fullname):
206 for opt_level in optimize:
207 if legacy:
208 opt_cfiles[opt_level] = fullname + 'c'
209 else:
210 if opt_level >= 0:
211 opt = opt_level if opt_level >= 1 else ''
212 cfile = (importlib.util.cache_from_source(
213 fullname, optimization=opt))
214 opt_cfiles[opt_level] = cfile
215 else:
216 cfile = importlib.util.cache_from_source(fullname)
217 opt_cfiles[opt_level] = cfile
218
219 head, tail = name[:-3], name[-3:]
220 if tail == '.py':
221 if not force:
222 try:
223 mtime = int(os.stat(fullname).st_mtime)
224 expect = struct.pack('<4sLL', importlib.util.MAGIC_NUMBER,
225 0, mtime & 0xFFFF_FFFF)
226 for cfile in opt_cfiles.values():
227 with open(cfile, 'rb') as chandle:
228 actual = chandle.read(12)
229 if expect != actual:
230 break
231 else:
232 return success
233 except OSError:
234 pass
235 if not quiet:
236 print('Compiling {!r}...'.format(fullname))
237 try:
238 for index, opt_level in enumerate(optimize):
239 cfile = opt_cfiles[opt_level]
240 ok = py_compile.compile(fullname, cfile, dfile, True,
241 optimize=opt_level,
242 invalidation_mode=invalidation_mode)
243 if index > 0 and hardlink_dupes:
244 previous_cfile = opt_cfiles[optimize[index - 1]]
245 if filecmp.cmp(cfile, previous_cfile, shallow=False):
246 os.unlink(cfile)
247 os.link(previous_cfile, cfile)
248 except py_compile.PyCompileError as err:
249 success = False
250 if quiet >= 2:
251 return success
252 elif quiet:
253 print('*** Error compiling {!r}...'.format(fullname))
254 else:
255 print('*** ', end='')
256 # escape non-printable characters in msg
257 encoding = sys.stdout.encoding or sys.getdefaultencoding()
258 msg = err.msg.encode(encoding, errors='backslashreplace').decode(encoding)
259 print(msg)
260 except (SyntaxError, UnicodeError, OSError) as e:
261 success = False
262 if quiet >= 2:
263 return success
264 elif quiet:
265 print('*** Error compiling {!r}...'.format(fullname))
266 else:
267 print('*** ', end='')
268 print(e.__class__.__name__ + ':', e)
269 else:
270 if ok == 0:
271 success = False
272 return success
273
274 def compile_path(skip_curdir=1, maxlevels=0, force=False, quiet=0,
275 legacy=False, optimize=-1,
276 invalidation_mode=None):
277 """Byte-compile all module on sys.path.
278
279 Arguments (all optional):
280
281 skip_curdir: if true, skip current directory (default True)
282 maxlevels: max recursion level (default 0)
283 force: as for compile_dir() (default False)
284 quiet: as for compile_dir() (default 0)
285 legacy: as for compile_dir() (default False)
286 optimize: as for compile_dir() (default -1)
287 invalidation_mode: as for compiler_dir()
288 """
289 success = True
290 for dir in sys.path:
291 if (not dir or dir == os.curdir) and skip_curdir:
292 if quiet < 2:
293 print('Skipping current directory')
294 else:
295 success = success and compile_dir(
296 dir,
297 maxlevels,
298 None,
299 force,
300 quiet=quiet,
301 legacy=legacy,
302 optimize=optimize,
303 invalidation_mode=invalidation_mode,
304 )
305 return success
306
307
308 def main():
309 """Script main program."""
310 import argparse
311
312 parser = argparse.ArgumentParser(
313 description='Utilities to support installing Python libraries.')
314 parser.add_argument('-l', action='store_const', const=0,
315 default=None, dest='maxlevels',
316 help="don't recurse into subdirectories")
317 parser.add_argument('-r', type=int, dest='recursion',
318 help=('control the maximum recursion level. '
319 'if `-l` and `-r` options are specified, '
320 'then `-r` takes precedence.'))
321 parser.add_argument('-f', action='store_true', dest='force',
322 help='force rebuild even if timestamps are up to date')
323 parser.add_argument('-q', action='count', dest='quiet', default=0,
324 help='output only error messages; -qq will suppress '
325 'the error messages as well.')
326 parser.add_argument('-b', action='store_true', dest='legacy',
327 help='use legacy (pre-PEP3147) compiled file locations')
328 parser.add_argument('-d', metavar='DESTDIR', dest='ddir', default=None,
329 help=('directory to prepend to file paths for use in '
330 'compile-time tracebacks and in runtime '
331 'tracebacks in cases where the source file is '
332 'unavailable'))
333 parser.add_argument('-s', metavar='STRIPDIR', dest='stripdir',
334 default=None,
335 help=('part of path to left-strip from path '
336 'to source file - for example buildroot. '
337 '`-d` and `-s` options cannot be '
338 'specified together.'))
339 parser.add_argument('-p', metavar='PREPENDDIR', dest='prependdir',
340 default=None,
341 help=('path to add as prefix to path '
342 'to source file - for example / to make '
343 'it absolute when some part is removed '
344 'by `-s` option. '
345 '`-d` and `-p` options cannot be '
346 'specified together.'))
347 parser.add_argument('-x', metavar='REGEXP', dest='rx', default=None,
348 help=('skip files matching the regular expression; '
349 'the regexp is searched for in the full path '
350 'of each file considered for compilation'))
351 parser.add_argument('-i', metavar='FILE', dest='flist',
352 help=('add all the files and directories listed in '
353 'FILE to the list considered for compilation; '
354 'if "-", names are read from stdin'))
355 parser.add_argument('compile_dest', metavar='FILE|DIR', nargs='*',
356 help=('zero or more file and directory names '
357 'to compile; if no arguments given, defaults '
358 'to the equivalent of -l sys.path'))
359 parser.add_argument('-j', '--workers', default=1,
360 type=int, help='Run compileall concurrently')
361 invalidation_modes = [mode.name.lower().replace('_', '-')
362 for mode in py_compile.PycInvalidationMode]
363 parser.add_argument('--invalidation-mode',
364 choices=sorted(invalidation_modes),
365 help=('set .pyc invalidation mode; defaults to '
366 '"checked-hash" if the SOURCE_DATE_EPOCH '
367 'environment variable is set, and '
368 '"timestamp" otherwise.'))
369 parser.add_argument('-o', action='append', type=int, dest='opt_levels',
370 help=('Optimization levels to run compilation with. '
371 'Default is -1 which uses the optimization level '
372 'of the Python interpreter itself (see -O).'))
373 parser.add_argument('-e', metavar='DIR', dest='limit_sl_dest',
374 help='Ignore symlinks pointing outsite of the DIR')
375 parser.add_argument('--hardlink-dupes', action='store_true',
376 dest='hardlink_dupes',
377 help='Hardlink duplicated pyc files')
378
379 args = parser.parse_args()
380 compile_dests = args.compile_dest
381
382 if args.rx:
383 import re
384 args.rx = re.compile(args.rx)
385
386 if args.limit_sl_dest == "":
387 args.limit_sl_dest = None
388
389 if args.recursion is not None:
390 maxlevels = args.recursion
391 else:
392 maxlevels = args.maxlevels
393
394 if args.opt_levels is None:
395 args.opt_levels = [-1]
396
397 if len(args.opt_levels) == 1 and args.hardlink_dupes:
398 parser.error(("Hardlinking of duplicated bytecode makes sense "
399 "only for more than one optimization level."))
400
401 if args.ddir is not None and (
402 args.stripdir is not None or args.prependdir is not None
403 ):
404 parser.error("-d cannot be used in combination with -s or -p")
405
406 # if flist is provided then load it
407 if args.flist:
408 try:
409 with (sys.stdin if args.flist=='-' else
410 open(args.flist, encoding="utf-8")) as f:
411 for line in f:
412 compile_dests.append(line.strip())
413 except OSError:
414 if args.quiet < 2:
415 print("Error reading file list {}".format(args.flist))
416 return False
417
418 if args.invalidation_mode:
419 ivl_mode = args.invalidation_mode.replace('-', '_').upper()
420 invalidation_mode = py_compile.PycInvalidationMode[ivl_mode]
421 else:
422 invalidation_mode = None
423
424 success = True
425 try:
426 if compile_dests:
427 for dest in compile_dests:
428 if os.path.isfile(dest):
429 if not compile_file(dest, args.ddir, args.force, args.rx,
430 args.quiet, args.legacy,
431 invalidation_mode=invalidation_mode,
432 stripdir=args.stripdir,
433 prependdir=args.prependdir,
434 optimize=args.opt_levels,
435 limit_sl_dest=args.limit_sl_dest,
436 hardlink_dupes=args.hardlink_dupes):
437 success = False
438 else:
439 if not compile_dir(dest, maxlevels, args.ddir,
440 args.force, args.rx, args.quiet,
441 args.legacy, workers=args.workers,
442 invalidation_mode=invalidation_mode,
443 stripdir=args.stripdir,
444 prependdir=args.prependdir,
445 optimize=args.opt_levels,
446 limit_sl_dest=args.limit_sl_dest,
447 hardlink_dupes=args.hardlink_dupes):
448 success = False
449 return success
450 else:
451 return compile_path(legacy=args.legacy, force=args.force,
452 quiet=args.quiet,
453 invalidation_mode=invalidation_mode)
454 except KeyboardInterrupt:
455 if args.quiet < 2:
456 print("\n[interrupted]")
457 return False
458 return True
459
460
461 if __name__ == '__main__':
462 exit_status = int(not main())
463 sys.exit(exit_status)