1 """
2 Generates a layout of Python for Windows from a build.
3
4 See python make_layout.py --help for usage.
5 """
6
7 __author__ = "Steve Dower <steve.dower@python.org>"
8 __version__ = "3.8"
9
10 import argparse
11 import functools
12 import os
13 import re
14 import shutil
15 import subprocess
16 import sys
17 import tempfile
18 import zipfile
19
20 from pathlib import Path
21
22 if __name__ == "__main__":
23 # Started directly, so enable relative imports
24 __path__ = [str(Path(__file__).resolve().parent)]
25
26 from .support.appxmanifest import *
27 from .support.catalog import *
28 from .support.constants import *
29 from .support.filesets import *
30 from .support.logging import *
31 from .support.options import *
32 from .support.pip import *
33 from .support.props import *
34 from .support.nuspec import *
35
36 TEST_PYDS_ONLY = FileStemSet("xxlimited", "xxlimited_35", "_ctypes_test", "_test*")
37 TEST_DIRS_ONLY = FileNameSet("test", "tests")
38
39 IDLE_DIRS_ONLY = FileNameSet("idlelib")
40
41 TCLTK_PYDS_ONLY = FileStemSet("tcl*", "tk*", "_tkinter")
42 TCLTK_DIRS_ONLY = FileNameSet("tkinter", "turtledemo")
43 TCLTK_FILES_ONLY = FileNameSet("turtle.py")
44
45 VENV_DIRS_ONLY = FileNameSet("venv", "ensurepip")
46
47 EXCLUDE_FROM_PYDS = FileStemSet("python*", "pyshellext", "vcruntime*")
48 EXCLUDE_FROM_LIB = FileNameSet("*.pyc", "__pycache__", "*.pickle")
49 EXCLUDE_FROM_PACKAGED_LIB = FileNameSet("readme.txt")
50 EXCLUDE_FROM_COMPILE = FileNameSet("badsyntax_*", "bad_*")
51 EXCLUDE_FROM_CATALOG = FileSuffixSet(".exe", ".pyd", ".dll")
52
53 REQUIRED_DLLS = FileStemSet("libcrypto*", "libssl*", "libffi*")
54
55 LIB2TO3_GRAMMAR_FILES = FileNameSet("Grammar.txt", "PatternGrammar.txt")
56
57 PY_FILES = FileSuffixSet(".py")
58 PYC_FILES = FileSuffixSet(".pyc")
59 CAT_FILES = FileSuffixSet(".cat")
60 CDF_FILES = FileSuffixSet(".cdf")
61
62 DATA_DIRS = FileNameSet("data")
63
64 TOOLS_DIRS = FileNameSet("scripts", "i18n", "demo", "parser")
65 TOOLS_FILES = FileSuffixSet(".py", ".pyw", ".txt")
66
67
68 def copy_if_modified(src, dest):
69 try:
70 dest_stat = os.stat(dest)
71 except FileNotFoundError:
72 do_copy = True
73 else:
74 src_stat = os.stat(src)
75 do_copy = (
76 src_stat.st_mtime != dest_stat.st_mtime
77 or src_stat.st_size != dest_stat.st_size
78 )
79
80 if do_copy:
81 shutil.copy2(src, dest)
82
83
84 def get_lib_layout(ns):
85 def _c(f):
86 if f in EXCLUDE_FROM_LIB:
87 return False
88 if f.is_dir():
89 if f in TEST_DIRS_ONLY:
90 return ns.include_tests
91 if f in TCLTK_DIRS_ONLY:
92 return ns.include_tcltk
93 if f in IDLE_DIRS_ONLY:
94 return ns.include_idle
95 if f in VENV_DIRS_ONLY:
96 return ns.include_venv
97 else:
98 if f in TCLTK_FILES_ONLY:
99 return ns.include_tcltk
100 return True
101
102 for dest, src in rglob(ns.source / "Lib", "**/*", _c):
103 yield dest, src
104
105
106 def get_tcltk_lib(ns):
107 if not ns.include_tcltk:
108 return
109
110 tcl_lib = os.getenv("TCL_LIBRARY")
111 if not tcl_lib or not os.path.isdir(tcl_lib):
112 try:
113 with open(ns.build / "TCL_LIBRARY.env", "r", encoding="utf-8-sig") as f:
114 tcl_lib = f.read().strip()
115 except FileNotFoundError:
116 pass
117 if not tcl_lib or not os.path.isdir(tcl_lib):
118 log_warning("Failed to find TCL_LIBRARY")
119 return
120
121 for dest, src in rglob(Path(tcl_lib).parent, "**/*"):
122 yield "tcl/{}".format(dest), src
123
124
125 def get_layout(ns):
126 def in_build(f, dest="", new_name=None):
127 n, _, x = f.rpartition(".")
128 n = new_name or n
129 src = ns.build / f
130 if ns.debug and src not in REQUIRED_DLLS:
131 if not src.stem.endswith("_d"):
132 src = src.parent / (src.stem + "_d" + src.suffix)
133 if not n.endswith("_d"):
134 n += "_d"
135 f = n + "." + x
136 yield dest + n + "." + x, src
137 if ns.include_symbols:
138 pdb = src.with_suffix(".pdb")
139 if pdb.is_file():
140 yield dest + n + ".pdb", pdb
141 if ns.include_dev:
142 lib = src.with_suffix(".lib")
143 if lib.is_file():
144 yield "libs/" + n + ".lib", lib
145
146 if ns.include_appxmanifest:
147 yield from in_build("python_uwp.exe", new_name="python{}".format(VER_DOT))
148 yield from in_build("pythonw_uwp.exe", new_name="pythonw{}".format(VER_DOT))
149 # For backwards compatibility, but we don't reference these ourselves.
150 yield from in_build("python_uwp.exe", new_name="python")
151 yield from in_build("pythonw_uwp.exe", new_name="pythonw")
152 else:
153 yield from in_build("python.exe", new_name="python")
154 yield from in_build("pythonw.exe", new_name="pythonw")
155
156 yield from in_build(PYTHON_DLL_NAME)
157
158 if ns.include_launchers and ns.include_appxmanifest:
159 if ns.include_pip:
160 yield from in_build("python_uwp.exe", new_name="pip{}".format(VER_DOT))
161 if ns.include_idle:
162 yield from in_build("pythonw_uwp.exe", new_name="idle{}".format(VER_DOT))
163
164 if ns.include_stable:
165 yield from in_build(PYTHON_STABLE_DLL_NAME)
166
167 found_any = False
168 for dest, src in rglob(ns.build, "vcruntime*.dll"):
169 found_any = True
170 yield dest, src
171 if not found_any:
172 log_error("Failed to locate vcruntime DLL in the build.")
173
174 yield "LICENSE.txt", ns.build / "LICENSE.txt"
175
176 for dest, src in rglob(ns.build, ("*.pyd", "*.dll")):
177 if src.stem.endswith("_d") != bool(ns.debug) and src not in REQUIRED_DLLS:
178 continue
179 if src in EXCLUDE_FROM_PYDS:
180 continue
181 if src in TEST_PYDS_ONLY and not ns.include_tests:
182 continue
183 if src in TCLTK_PYDS_ONLY and not ns.include_tcltk:
184 continue
185
186 yield from in_build(src.name, dest="" if ns.flat_dlls else "DLLs/")
187
188 if ns.zip_lib:
189 zip_name = PYTHON_ZIP_NAME
190 yield zip_name, ns.temp / zip_name
191 else:
192 for dest, src in get_lib_layout(ns):
193 yield "Lib/{}".format(dest), src
194
195 if ns.include_venv:
196 yield from in_build("venvlauncher.exe", "Lib/venv/scripts/nt/", "python")
197 yield from in_build("venvwlauncher.exe", "Lib/venv/scripts/nt/", "pythonw")
198
199 if ns.include_tools:
200
201 def _c(d):
202 if d.is_dir():
203 return d in TOOLS_DIRS
204 return d in TOOLS_FILES
205
206 for dest, src in rglob(ns.source / "Tools", "**/*", _c):
207 yield "Tools/{}".format(dest), src
208
209 if ns.include_underpth:
210 yield PYTHON_PTH_NAME, ns.temp / PYTHON_PTH_NAME
211
212 if ns.include_dev:
213
214 for dest, src in rglob(ns.source / "Include", "**/*.h"):
215 yield "include/{}".format(dest), src
216 src = ns.source / "PC" / "pyconfig.h"
217 yield "include/pyconfig.h", src
218
219 for dest, src in get_tcltk_lib(ns):
220 yield dest, src
221
222 if ns.include_pip:
223 for dest, src in get_pip_layout(ns):
224 if not isinstance(src, tuple) and (
225 src in EXCLUDE_FROM_LIB or src in EXCLUDE_FROM_PACKAGED_LIB
226 ):
227 continue
228 yield dest, src
229
230 if ns.include_chm:
231 for dest, src in rglob(ns.doc_build / "htmlhelp", PYTHON_CHM_NAME):
232 yield "Doc/{}".format(dest), src
233
234 if ns.include_html_doc:
235 for dest, src in rglob(ns.doc_build / "html", "**/*"):
236 yield "Doc/html/{}".format(dest), src
237
238 if ns.include_props:
239 for dest, src in get_props_layout(ns):
240 yield dest, src
241
242 if ns.include_nuspec:
243 for dest, src in get_nuspec_layout(ns):
244 yield dest, src
245
246 for dest, src in get_appx_layout(ns):
247 yield dest, src
248
249 if ns.include_cat:
250 if ns.flat_dlls:
251 yield ns.include_cat.name, ns.include_cat
252 else:
253 yield "DLLs/{}".format(ns.include_cat.name), ns.include_cat
254
255
256 def _compile_one_py(src, dest, name, optimize, checked=True):
257 import py_compile
258
259 if dest is not None:
260 dest = str(dest)
261
262 mode = (
263 py_compile.PycInvalidationMode.CHECKED_HASH
264 if checked
265 else py_compile.PycInvalidationMode.UNCHECKED_HASH
266 )
267
268 try:
269 return Path(
270 py_compile.compile(
271 str(src),
272 dest,
273 str(name),
274 doraise=True,
275 optimize=optimize,
276 invalidation_mode=mode,
277 )
278 )
279 except py_compile.PyCompileError:
280 log_warning("Failed to compile {}", src)
281 return None
282
283
284 # name argument added to address bpo-37641
285 def _py_temp_compile(src, name, ns, dest_dir=None, checked=True):
286 if not ns.precompile or src not in PY_FILES or src.parent in DATA_DIRS:
287 return None
288 dest = (dest_dir or ns.temp) / (src.stem + ".pyc")
289 return _compile_one_py(src, dest, name, optimize=2, checked=checked)
290
291
292 def _write_to_zip(zf, dest, src, ns, checked=True):
293 pyc = _py_temp_compile(src, dest, ns, checked=checked)
294 if pyc:
295 try:
296 zf.write(str(pyc), dest.with_suffix(".pyc"))
297 finally:
298 try:
299 pyc.unlink()
300 except:
301 log_exception("Failed to delete {}", pyc)
302 return
303
304 if src in LIB2TO3_GRAMMAR_FILES:
305 from lib2to3.pgen2.driver import load_grammar
306
307 tmp = ns.temp / src.name
308 try:
309 shutil.copy(src, tmp)
310 load_grammar(str(tmp))
311 for f in ns.temp.glob(src.stem + "*.pickle"):
312 zf.write(str(f), str(dest.parent / f.name))
313 try:
314 f.unlink()
315 except:
316 log_exception("Failed to delete {}", f)
317 except:
318 log_exception("Failed to compile {}", src)
319 finally:
320 try:
321 tmp.unlink()
322 except:
323 log_exception("Failed to delete {}", tmp)
324
325 zf.write(str(src), str(dest))
326
327
328 def generate_source_files(ns):
329 if ns.zip_lib:
330 zip_name = PYTHON_ZIP_NAME
331 zip_path = ns.temp / zip_name
332 if zip_path.is_file():
333 zip_path.unlink()
334 elif zip_path.is_dir():
335 log_error(
336 "Cannot create zip file because a directory exists by the same name"
337 )
338 return
339 log_info("Generating {} in {}", zip_name, ns.temp)
340 ns.temp.mkdir(parents=True, exist_ok=True)
341 with zipfile.ZipFile(zip_path, "w", zipfile.ZIP_DEFLATED) as zf:
342 for dest, src in get_lib_layout(ns):
343 _write_to_zip(zf, dest, src, ns, checked=False)
344
345 if ns.include_underpth:
346 log_info("Generating {} in {}", PYTHON_PTH_NAME, ns.temp)
347 ns.temp.mkdir(parents=True, exist_ok=True)
348 with open(ns.temp / PYTHON_PTH_NAME, "w", encoding="utf-8") as f:
349 if ns.zip_lib:
350 print(PYTHON_ZIP_NAME, file=f)
351 if ns.include_pip:
352 print("packages", file=f)
353 else:
354 print("Lib", file=f)
355 print("Lib/site-packages", file=f)
356 if not ns.flat_dlls:
357 print("DLLs", file=f)
358 print(".", file=f)
359 print(file=f)
360 print("# Uncomment to run site.main() automatically", file=f)
361 print("#import site", file=f)
362
363 if ns.include_pip:
364 log_info("Extracting pip")
365 extract_pip_files(ns)
366
367
368 def _create_zip_file(ns):
369 if not ns.zip:
370 return None
371
372 if ns.zip.is_file():
373 try:
374 ns.zip.unlink()
375 except OSError:
376 log_exception("Unable to remove {}", ns.zip)
377 sys.exit(8)
378 elif ns.zip.is_dir():
379 log_error("Cannot create ZIP file because {} is a directory", ns.zip)
380 sys.exit(8)
381
382 ns.zip.parent.mkdir(parents=True, exist_ok=True)
383 return zipfile.ZipFile(ns.zip, "w", zipfile.ZIP_DEFLATED)
384
385
386 def copy_files(files, ns):
387 if ns.copy:
388 ns.copy.mkdir(parents=True, exist_ok=True)
389
390 try:
391 total = len(files)
392 except TypeError:
393 total = None
394 count = 0
395
396 zip_file = _create_zip_file(ns)
397 try:
398 need_compile = []
399 in_catalog = []
400
401 for dest, src in files:
402 count += 1
403 if count % 10 == 0:
404 if total:
405 log_info("Processed {:>4} of {} files", count, total)
406 else:
407 log_info("Processed {} files", count)
408 log_debug("Processing {!s}", src)
409
410 if isinstance(src, tuple):
411 src, content = src
412 if ns.copy:
413 log_debug("Copy {} -> {}", src, ns.copy / dest)
414 (ns.copy / dest).parent.mkdir(parents=True, exist_ok=True)
415 with open(ns.copy / dest, "wb") as f:
416 f.write(content)
417 if ns.zip:
418 log_debug("Zip {} into {}", src, ns.zip)
419 zip_file.writestr(str(dest), content)
420 continue
421
422 if (
423 ns.precompile
424 and src in PY_FILES
425 and src not in EXCLUDE_FROM_COMPILE
426 and src.parent not in DATA_DIRS
427 and os.path.normcase(str(dest)).startswith(os.path.normcase("Lib"))
428 ):
429 if ns.copy:
430 need_compile.append((dest, ns.copy / dest))
431 else:
432 (ns.temp / "Lib" / dest).parent.mkdir(parents=True, exist_ok=True)
433 copy_if_modified(src, ns.temp / "Lib" / dest)
434 need_compile.append((dest, ns.temp / "Lib" / dest))
435
436 if src not in EXCLUDE_FROM_CATALOG:
437 in_catalog.append((src.name, src))
438
439 if ns.copy:
440 log_debug("Copy {} -> {}", src, ns.copy / dest)
441 (ns.copy / dest).parent.mkdir(parents=True, exist_ok=True)
442 try:
443 copy_if_modified(src, ns.copy / dest)
444 except shutil.SameFileError:
445 pass
446
447 if ns.zip:
448 log_debug("Zip {} into {}", src, ns.zip)
449 zip_file.write(src, str(dest))
450
451 if need_compile:
452 for dest, src in need_compile:
453 compiled = [
454 _compile_one_py(src, None, dest, optimize=0),
455 _compile_one_py(src, None, dest, optimize=1),
456 _compile_one_py(src, None, dest, optimize=2),
457 ]
458 for c in compiled:
459 if not c:
460 continue
461 cdest = Path(dest).parent / Path(c).relative_to(src.parent)
462 if ns.zip:
463 log_debug("Zip {} into {}", c, ns.zip)
464 zip_file.write(c, str(cdest))
465 in_catalog.append((cdest.name, cdest))
466
467 if ns.catalog:
468 # Just write out the CDF now. Compilation and signing is
469 # an extra step
470 log_info("Generating {}", ns.catalog)
471 ns.catalog.parent.mkdir(parents=True, exist_ok=True)
472 write_catalog(ns.catalog, in_catalog)
473
474 finally:
475 if zip_file:
476 zip_file.close()
477
478
479 def main():
480 parser = argparse.ArgumentParser()
481 parser.add_argument("-v", help="Increase verbosity", action="count")
482 parser.add_argument(
483 "-s",
484 "--source",
485 metavar="dir",
486 help="The directory containing the repository root",
487 type=Path,
488 default=None,
489 )
490 parser.add_argument(
491 "-b", "--build", metavar="dir", help="Specify the build directory", type=Path
492 )
493 parser.add_argument(
494 "--arch",
495 metavar="architecture",
496 help="Specify the target architecture",
497 type=str,
498 default=None,
499 )
500 parser.add_argument(
501 "--doc-build",
502 metavar="dir",
503 help="Specify the docs build directory",
504 type=Path,
505 default=None,
506 )
507 parser.add_argument(
508 "--copy",
509 metavar="directory",
510 help="The name of the directory to copy an extracted layout to",
511 type=Path,
512 default=None,
513 )
514 parser.add_argument(
515 "--zip",
516 metavar="file",
517 help="The ZIP file to write all files to",
518 type=Path,
519 default=None,
520 )
521 parser.add_argument(
522 "--catalog",
523 metavar="file",
524 help="The CDF file to write catalog entries to",
525 type=Path,
526 default=None,
527 )
528 parser.add_argument(
529 "--log",
530 metavar="file",
531 help="Write all operations to the specified file",
532 type=Path,
533 default=None,
534 )
535 parser.add_argument(
536 "-t",
537 "--temp",
538 metavar="file",
539 help="A temporary working directory",
540 type=Path,
541 default=None,
542 )
543 parser.add_argument(
544 "-d", "--debug", help="Include debug build", action="store_true"
545 )
546 parser.add_argument(
547 "-p",
548 "--precompile",
549 help="Include .pyc files instead of .py",
550 action="store_true",
551 )
552 parser.add_argument(
553 "-z", "--zip-lib", help="Include library in a ZIP file", action="store_true"
554 )
555 parser.add_argument(
556 "--flat-dlls", help="Does not create a DLLs directory", action="store_true"
557 )
558 parser.add_argument(
559 "-a",
560 "--include-all",
561 help="Include all optional components",
562 action="store_true",
563 )
564 parser.add_argument(
565 "--include-cat",
566 metavar="file",
567 help="Specify the catalog file to include",
568 type=Path,
569 default=None,
570 )
571 for opt, help in get_argparse_options():
572 parser.add_argument(opt, help=help, action="store_true")
573
574 ns = parser.parse_args()
575 update_presets(ns)
576
577 ns.source = ns.source or (Path(__file__).resolve().parent.parent.parent)
578 ns.build = ns.build or Path(sys.executable).parent
579 ns.temp = ns.temp or Path(tempfile.mkdtemp())
580 ns.doc_build = ns.doc_build or (ns.source / "Doc" / "build")
581 if not ns.source.is_absolute():
582 ns.source = (Path.cwd() / ns.source).resolve()
583 if not ns.build.is_absolute():
584 ns.build = (Path.cwd() / ns.build).resolve()
585 if not ns.temp.is_absolute():
586 ns.temp = (Path.cwd() / ns.temp).resolve()
587 if not ns.doc_build.is_absolute():
588 ns.doc_build = (Path.cwd() / ns.doc_build).resolve()
589 if ns.include_cat and not ns.include_cat.is_absolute():
590 ns.include_cat = (Path.cwd() / ns.include_cat).resolve()
591 if not ns.arch:
592 ns.arch = "amd64" if sys.maxsize > 2 ** 32 else "win32"
593
594 if ns.copy and not ns.copy.is_absolute():
595 ns.copy = (Path.cwd() / ns.copy).resolve()
596 if ns.zip and not ns.zip.is_absolute():
597 ns.zip = (Path.cwd() / ns.zip).resolve()
598 if ns.catalog and not ns.catalog.is_absolute():
599 ns.catalog = (Path.cwd() / ns.catalog).resolve()
600
601 configure_logger(ns)
602
603 log_info(
604 """OPTIONS
605 Source: {ns.source}
606 Build: {ns.build}
607 Temp: {ns.temp}
608 Arch: {ns.arch}
609
610 Copy to: {ns.copy}
611 Zip to: {ns.zip}
612 Catalog: {ns.catalog}""",
613 ns=ns,
614 )
615
616 if ns.arch not in ("win32", "amd64", "arm32", "arm64"):
617 log_error("--arch is not a valid value (win32, amd64, arm32, arm64)")
618 return 4
619 if ns.arch in ("arm32", "arm64"):
620 for n in ("include_idle", "include_tcltk"):
621 if getattr(ns, n):
622 log_warning(f"Disabling --{n.replace('_', '-')} on unsupported platform")
623 setattr(ns, n, False)
624
625 if ns.include_idle and not ns.include_tcltk:
626 log_warning("Assuming --include-tcltk to support --include-idle")
627 ns.include_tcltk = True
628
629 try:
630 generate_source_files(ns)
631 files = list(get_layout(ns))
632 copy_files(files, ns)
633 except KeyboardInterrupt:
634 log_info("Interrupted by Ctrl+C")
635 return 3
636 except SystemExit:
637 raise
638 except:
639 log_exception("Unhandled error")
640
641 if error_was_logged():
642 log_error("Errors occurred.")
643 return 1
644
645
646 if __name__ == "__main__":
647 sys.exit(int(main() or 0))