1 """
2 Generates a layout of Python for Windows from a build.
3
4 See python make_layout.py --help for usage.
5 """
6
7 __author__ = "Steve Dower <steve.dower@python.org>"
8 __version__ = "3.8"
9
10 import argparse
11 import os
12 import shutil
13 import sys
14 import tempfile
15 import zipfile
16
17 from pathlib import Path
18
19 if __name__ == "__main__":
20 # Started directly, so enable relative imports
21 __path__ = [str(Path(__file__).resolve().parent)]
22
23 from .support.appxmanifest import *
24 from .support.catalog import *
25 from .support.constants import *
26 from .support.filesets import *
27 from .support.logging import *
28 from .support.options import *
29 from .support.pip import *
30 from .support.props import *
31 from .support.nuspec import *
32
33 TEST_PYDS_ONLY = FileStemSet("xxlimited", "xxlimited_35", "_ctypes_test", "_test*")
34 TEST_DIRS_ONLY = FileNameSet("test", "tests")
35
36 IDLE_DIRS_ONLY = FileNameSet("idlelib")
37
38 TCLTK_PYDS_ONLY = FileStemSet("tcl*", "tk*", "_tkinter", "zlib1")
39 TCLTK_DIRS_ONLY = FileNameSet("tkinter", "turtledemo")
40 TCLTK_FILES_ONLY = FileNameSet("turtle.py")
41
42 VENV_DIRS_ONLY = FileNameSet("venv", "ensurepip")
43
44 EXCLUDE_FROM_PYDS = FileStemSet("python*", "pyshellext", "vcruntime*")
45 EXCLUDE_FROM_LIB = FileNameSet("*.pyc", "__pycache__", "*.pickle")
46 EXCLUDE_FROM_PACKAGED_LIB = FileNameSet("readme.txt")
47 EXCLUDE_FROM_COMPILE = FileNameSet("badsyntax_*", "bad_*")
48 EXCLUDE_FROM_CATALOG = FileSuffixSet(".exe", ".pyd", ".dll")
49
50 REQUIRED_DLLS = FileStemSet("libcrypto*", "libssl*", "libffi*")
51
52 LIB2TO3_GRAMMAR_FILES = FileNameSet("Grammar.txt", "PatternGrammar.txt")
53
54 PY_FILES = FileSuffixSet(".py")
55 PYC_FILES = FileSuffixSet(".pyc")
56 CAT_FILES = FileSuffixSet(".cat")
57 CDF_FILES = FileSuffixSet(".cdf")
58
59 DATA_DIRS = FileNameSet("data")
60
61 TOOLS_DIRS = FileNameSet("scripts", "i18n", "parser")
62 TOOLS_FILES = FileSuffixSet(".py", ".pyw", ".txt")
63
64
65 def copy_if_modified(src, dest):
66 try:
67 dest_stat = os.stat(dest)
68 except FileNotFoundError:
69 do_copy = True
70 else:
71 src_stat = os.stat(src)
72 do_copy = (
73 src_stat.st_mtime != dest_stat.st_mtime
74 or src_stat.st_size != dest_stat.st_size
75 )
76
77 if do_copy:
78 shutil.copy2(src, dest)
79
80
81 def get_lib_layout(ns):
82 def _c(f):
83 if f in EXCLUDE_FROM_LIB:
84 return False
85 if f.is_dir():
86 if f in TEST_DIRS_ONLY:
87 return ns.include_tests
88 if f in TCLTK_DIRS_ONLY:
89 return ns.include_tcltk
90 if f in IDLE_DIRS_ONLY:
91 return ns.include_idle
92 if f in VENV_DIRS_ONLY:
93 return ns.include_venv
94 else:
95 if f in TCLTK_FILES_ONLY:
96 return ns.include_tcltk
97 return True
98
99 for dest, src in rglob(ns.source / "Lib", "**/*", _c):
100 yield dest, src
101
102
103 def get_tcltk_lib(ns):
104 if not ns.include_tcltk:
105 return
106
107 tcl_lib = os.getenv("TCL_LIBRARY")
108 if not tcl_lib or not os.path.isdir(tcl_lib):
109 try:
110 with open(ns.build / "TCL_LIBRARY.env", "r", encoding="utf-8-sig") as f:
111 tcl_lib = f.read().strip()
112 except FileNotFoundError:
113 pass
114 if not tcl_lib or not os.path.isdir(tcl_lib):
115 log_warning("Failed to find TCL_LIBRARY")
116 return
117
118 for dest, src in rglob(Path(tcl_lib).parent, "**/*"):
119 yield "tcl/{}".format(dest), src
120
121
122 def get_layout(ns):
123 def in_build(f, dest="", new_name=None):
124 n, _, x = f.rpartition(".")
125 n = new_name or n
126 src = ns.build / f
127 if ns.debug and src not in REQUIRED_DLLS:
128 if not src.stem.endswith("_d"):
129 src = src.parent / (src.stem + "_d" + src.suffix)
130 if not n.endswith("_d"):
131 n += "_d"
132 f = n + "." + x
133 yield dest + n + "." + x, src
134 if ns.include_symbols:
135 pdb = src.with_suffix(".pdb")
136 if pdb.is_file():
137 yield dest + n + ".pdb", pdb
138 if ns.include_dev:
139 lib = src.with_suffix(".lib")
140 if lib.is_file():
141 yield "libs/" + n + ".lib", lib
142
143 if ns.include_appxmanifest:
144 yield from in_build("python_uwp.exe", new_name="python{}".format(VER_DOT))
145 yield from in_build("pythonw_uwp.exe", new_name="pythonw{}".format(VER_DOT))
146 # For backwards compatibility, but we don't reference these ourselves.
147 yield from in_build("python_uwp.exe", new_name="python")
148 yield from in_build("pythonw_uwp.exe", new_name="pythonw")
149 else:
150 yield from in_build("python.exe", new_name="python")
151 yield from in_build("pythonw.exe", new_name="pythonw")
152
153 yield from in_build(PYTHON_DLL_NAME)
154
155 if ns.include_launchers and ns.include_appxmanifest:
156 if ns.include_pip:
157 yield from in_build("python_uwp.exe", new_name="pip{}".format(VER_DOT))
158 if ns.include_idle:
159 yield from in_build("pythonw_uwp.exe", new_name="idle{}".format(VER_DOT))
160
161 if ns.include_stable:
162 yield from in_build(PYTHON_STABLE_DLL_NAME)
163
164 found_any = False
165 for dest, src in rglob(ns.build, "vcruntime*.dll"):
166 found_any = True
167 yield dest, src
168 if not found_any:
169 log_error("Failed to locate vcruntime DLL in the build.")
170
171 yield "LICENSE.txt", ns.build / "LICENSE.txt"
172
173 for dest, src in rglob(ns.build, ("*.pyd", "*.dll")):
174 if src.stem.endswith("_d") != bool(ns.debug) and src not in REQUIRED_DLLS:
175 continue
176 if src in EXCLUDE_FROM_PYDS:
177 continue
178 if src in TEST_PYDS_ONLY and not ns.include_tests:
179 continue
180 if src in TCLTK_PYDS_ONLY and not ns.include_tcltk:
181 continue
182
183 yield from in_build(src.name, dest="" if ns.flat_dlls else "DLLs/")
184
185 if ns.zip_lib:
186 zip_name = PYTHON_ZIP_NAME
187 yield zip_name, ns.temp / zip_name
188 else:
189 for dest, src in get_lib_layout(ns):
190 yield "Lib/{}".format(dest), src
191
192 if ns.include_venv:
193 yield from in_build("venvlauncher.exe", "Lib/venv/scripts/nt/", "python")
194 yield from in_build("venvwlauncher.exe", "Lib/venv/scripts/nt/", "pythonw")
195
196 if ns.include_tools:
197
198 def _c(d):
199 if d.is_dir():
200 return d in TOOLS_DIRS
201 return d in TOOLS_FILES
202
203 for dest, src in rglob(ns.source / "Tools", "**/*", _c):
204 yield "Tools/{}".format(dest), src
205
206 if ns.include_underpth:
207 yield PYTHON_PTH_NAME, ns.temp / PYTHON_PTH_NAME
208
209 if ns.include_dev:
210
211 for dest, src in rglob(ns.source / "Include", "**/*.h"):
212 yield "include/{}".format(dest), src
213 src = ns.source / "PC" / "pyconfig.h"
214 yield "include/pyconfig.h", src
215
216 for dest, src in get_tcltk_lib(ns):
217 yield dest, src
218
219 if ns.include_pip:
220 for dest, src in get_pip_layout(ns):
221 if not isinstance(src, tuple) and (
222 src in EXCLUDE_FROM_LIB or src in EXCLUDE_FROM_PACKAGED_LIB
223 ):
224 continue
225 yield dest, src
226
227 if ns.include_chm:
228 for dest, src in rglob(ns.doc_build / "htmlhelp", PYTHON_CHM_NAME):
229 yield "Doc/{}".format(dest), src
230
231 if ns.include_html_doc:
232 for dest, src in rglob(ns.doc_build / "html", "**/*"):
233 yield "Doc/html/{}".format(dest), src
234
235 if ns.include_props:
236 for dest, src in get_props_layout(ns):
237 yield dest, src
238
239 if ns.include_nuspec:
240 for dest, src in get_nuspec_layout(ns):
241 yield dest, src
242
243 for dest, src in get_appx_layout(ns):
244 yield dest, src
245
246 if ns.include_cat:
247 if ns.flat_dlls:
248 yield ns.include_cat.name, ns.include_cat
249 else:
250 yield "DLLs/{}".format(ns.include_cat.name), ns.include_cat
251
252
253 def _compile_one_py(src, dest, name, optimize, checked=True):
254 import py_compile
255
256 if dest is not None:
257 dest = str(dest)
258
259 mode = (
260 py_compile.PycInvalidationMode.CHECKED_HASH
261 if checked
262 else py_compile.PycInvalidationMode.UNCHECKED_HASH
263 )
264
265 try:
266 return Path(
267 py_compile.compile(
268 str(src),
269 dest,
270 str(name),
271 doraise=True,
272 optimize=optimize,
273 invalidation_mode=mode,
274 )
275 )
276 except py_compile.PyCompileError:
277 log_warning("Failed to compile {}", src)
278 return None
279
280
281 # name argument added to address bpo-37641
282 def _py_temp_compile(src, name, ns, dest_dir=None, checked=True):
283 if not ns.precompile or src not in PY_FILES or src.parent in DATA_DIRS:
284 return None
285 dest = (dest_dir or ns.temp) / (src.stem + ".pyc")
286 return _compile_one_py(src, dest, name, optimize=2, checked=checked)
287
288
289 def _write_to_zip(zf, dest, src, ns, checked=True):
290 pyc = _py_temp_compile(src, dest, ns, checked=checked)
291 if pyc:
292 try:
293 zf.write(str(pyc), dest.with_suffix(".pyc"))
294 finally:
295 try:
296 pyc.unlink()
297 except:
298 log_exception("Failed to delete {}", pyc)
299 return
300
301 if src in LIB2TO3_GRAMMAR_FILES:
302 from lib2to3.pgen2.driver import load_grammar
303
304 tmp = ns.temp / src.name
305 try:
306 shutil.copy(src, tmp)
307 load_grammar(str(tmp))
308 for f in ns.temp.glob(src.stem + "*.pickle"):
309 zf.write(str(f), str(dest.parent / f.name))
310 try:
311 f.unlink()
312 except:
313 log_exception("Failed to delete {}", f)
314 except:
315 log_exception("Failed to compile {}", src)
316 finally:
317 try:
318 tmp.unlink()
319 except:
320 log_exception("Failed to delete {}", tmp)
321
322 zf.write(str(src), str(dest))
323
324
325 def generate_source_files(ns):
326 if ns.zip_lib:
327 zip_name = PYTHON_ZIP_NAME
328 zip_path = ns.temp / zip_name
329 if zip_path.is_file():
330 zip_path.unlink()
331 elif zip_path.is_dir():
332 log_error(
333 "Cannot create zip file because a directory exists by the same name"
334 )
335 return
336 log_info("Generating {} in {}", zip_name, ns.temp)
337 ns.temp.mkdir(parents=True, exist_ok=True)
338 with zipfile.ZipFile(zip_path, "w", zipfile.ZIP_DEFLATED) as zf:
339 for dest, src in get_lib_layout(ns):
340 _write_to_zip(zf, dest, src, ns, checked=False)
341
342 if ns.include_underpth:
343 log_info("Generating {} in {}", PYTHON_PTH_NAME, ns.temp)
344 ns.temp.mkdir(parents=True, exist_ok=True)
345 with open(ns.temp / PYTHON_PTH_NAME, "w", encoding="utf-8") as f:
346 if ns.zip_lib:
347 print(PYTHON_ZIP_NAME, file=f)
348 if ns.include_pip:
349 print("packages", file=f)
350 else:
351 print("Lib", file=f)
352 print("Lib/site-packages", file=f)
353 if not ns.flat_dlls:
354 print("DLLs", file=f)
355 print(".", file=f)
356 print(file=f)
357 print("# Uncomment to run site.main() automatically", file=f)
358 print("#import site", file=f)
359
360 if ns.include_pip:
361 log_info("Extracting pip")
362 extract_pip_files(ns)
363
364
365 def _create_zip_file(ns):
366 if not ns.zip:
367 return None
368
369 if ns.zip.is_file():
370 try:
371 ns.zip.unlink()
372 except OSError:
373 log_exception("Unable to remove {}", ns.zip)
374 sys.exit(8)
375 elif ns.zip.is_dir():
376 log_error("Cannot create ZIP file because {} is a directory", ns.zip)
377 sys.exit(8)
378
379 ns.zip.parent.mkdir(parents=True, exist_ok=True)
380 return zipfile.ZipFile(ns.zip, "w", zipfile.ZIP_DEFLATED)
381
382
383 def copy_files(files, ns):
384 if ns.copy:
385 ns.copy.mkdir(parents=True, exist_ok=True)
386
387 try:
388 total = len(files)
389 except TypeError:
390 total = None
391 count = 0
392
393 zip_file = _create_zip_file(ns)
394 try:
395 need_compile = []
396 in_catalog = []
397
398 for dest, src in files:
399 count += 1
400 if count % 10 == 0:
401 if total:
402 log_info("Processed {:>4} of {} files", count, total)
403 else:
404 log_info("Processed {} files", count)
405 log_debug("Processing {!s}", src)
406
407 if isinstance(src, tuple):
408 src, content = src
409 if ns.copy:
410 log_debug("Copy {} -> {}", src, ns.copy / dest)
411 (ns.copy / dest).parent.mkdir(parents=True, exist_ok=True)
412 with open(ns.copy / dest, "wb") as f:
413 f.write(content)
414 if ns.zip:
415 log_debug("Zip {} into {}", src, ns.zip)
416 zip_file.writestr(str(dest), content)
417 continue
418
419 if (
420 ns.precompile
421 and src in PY_FILES
422 and src not in EXCLUDE_FROM_COMPILE
423 and src.parent not in DATA_DIRS
424 and os.path.normcase(str(dest)).startswith(os.path.normcase("Lib"))
425 ):
426 if ns.copy:
427 need_compile.append((dest, ns.copy / dest))
428 else:
429 (ns.temp / "Lib" / dest).parent.mkdir(parents=True, exist_ok=True)
430 copy_if_modified(src, ns.temp / "Lib" / dest)
431 need_compile.append((dest, ns.temp / "Lib" / dest))
432
433 if src not in EXCLUDE_FROM_CATALOG:
434 in_catalog.append((src.name, src))
435
436 if ns.copy:
437 log_debug("Copy {} -> {}", src, ns.copy / dest)
438 (ns.copy / dest).parent.mkdir(parents=True, exist_ok=True)
439 try:
440 copy_if_modified(src, ns.copy / dest)
441 except shutil.SameFileError:
442 pass
443
444 if ns.zip:
445 log_debug("Zip {} into {}", src, ns.zip)
446 zip_file.write(src, str(dest))
447
448 if need_compile:
449 for dest, src in need_compile:
450 compiled = [
451 _compile_one_py(src, None, dest, optimize=0),
452 _compile_one_py(src, None, dest, optimize=1),
453 _compile_one_py(src, None, dest, optimize=2),
454 ]
455 for c in compiled:
456 if not c:
457 continue
458 cdest = Path(dest).parent / Path(c).relative_to(src.parent)
459 if ns.zip:
460 log_debug("Zip {} into {}", c, ns.zip)
461 zip_file.write(c, str(cdest))
462 in_catalog.append((cdest.name, cdest))
463
464 if ns.catalog:
465 # Just write out the CDF now. Compilation and signing is
466 # an extra step
467 log_info("Generating {}", ns.catalog)
468 ns.catalog.parent.mkdir(parents=True, exist_ok=True)
469 write_catalog(ns.catalog, in_catalog)
470
471 finally:
472 if zip_file:
473 zip_file.close()
474
475
476 def main():
477 parser = argparse.ArgumentParser()
478 parser.add_argument("-v", help="Increase verbosity", action="count")
479 parser.add_argument(
480 "-s",
481 "--source",
482 metavar="dir",
483 help="The directory containing the repository root",
484 type=Path,
485 default=None,
486 )
487 parser.add_argument(
488 "-b", "--build", metavar="dir", help="Specify the build directory", type=Path
489 )
490 parser.add_argument(
491 "--arch",
492 metavar="architecture",
493 help="Specify the target architecture",
494 type=str,
495 default=None,
496 )
497 parser.add_argument(
498 "--doc-build",
499 metavar="dir",
500 help="Specify the docs build directory",
501 type=Path,
502 default=None,
503 )
504 parser.add_argument(
505 "--copy",
506 metavar="directory",
507 help="The name of the directory to copy an extracted layout to",
508 type=Path,
509 default=None,
510 )
511 parser.add_argument(
512 "--zip",
513 metavar="file",
514 help="The ZIP file to write all files to",
515 type=Path,
516 default=None,
517 )
518 parser.add_argument(
519 "--catalog",
520 metavar="file",
521 help="The CDF file to write catalog entries to",
522 type=Path,
523 default=None,
524 )
525 parser.add_argument(
526 "--log",
527 metavar="file",
528 help="Write all operations to the specified file",
529 type=Path,
530 default=None,
531 )
532 parser.add_argument(
533 "-t",
534 "--temp",
535 metavar="file",
536 help="A temporary working directory",
537 type=Path,
538 default=None,
539 )
540 parser.add_argument(
541 "-d", "--debug", help="Include debug build", action="store_true"
542 )
543 parser.add_argument(
544 "-p",
545 "--precompile",
546 help="Include .pyc files instead of .py",
547 action="store_true",
548 )
549 parser.add_argument(
550 "-z", "--zip-lib", help="Include library in a ZIP file", action="store_true"
551 )
552 parser.add_argument(
553 "--flat-dlls", help="Does not create a DLLs directory", action="store_true"
554 )
555 parser.add_argument(
556 "-a",
557 "--include-all",
558 help="Include all optional components",
559 action="store_true",
560 )
561 parser.add_argument(
562 "--include-cat",
563 metavar="file",
564 help="Specify the catalog file to include",
565 type=Path,
566 default=None,
567 )
568 for opt, help in get_argparse_options():
569 parser.add_argument(opt, help=help, action="store_true")
570
571 ns = parser.parse_args()
572 update_presets(ns)
573
574 ns.source = ns.source or (Path(__file__).resolve().parent.parent.parent)
575 ns.build = ns.build or Path(sys.executable).parent
576 ns.temp = ns.temp or Path(tempfile.mkdtemp())
577 ns.doc_build = ns.doc_build or (ns.source / "Doc" / "build")
578 if not ns.source.is_absolute():
579 ns.source = (Path.cwd() / ns.source).resolve()
580 if not ns.build.is_absolute():
581 ns.build = (Path.cwd() / ns.build).resolve()
582 if not ns.temp.is_absolute():
583 ns.temp = (Path.cwd() / ns.temp).resolve()
584 if not ns.doc_build.is_absolute():
585 ns.doc_build = (Path.cwd() / ns.doc_build).resolve()
586 if ns.include_cat and not ns.include_cat.is_absolute():
587 ns.include_cat = (Path.cwd() / ns.include_cat).resolve()
588 if not ns.arch:
589 ns.arch = "amd64" if sys.maxsize > 2 ** 32 else "win32"
590
591 if ns.copy and not ns.copy.is_absolute():
592 ns.copy = (Path.cwd() / ns.copy).resolve()
593 if ns.zip and not ns.zip.is_absolute():
594 ns.zip = (Path.cwd() / ns.zip).resolve()
595 if ns.catalog and not ns.catalog.is_absolute():
596 ns.catalog = (Path.cwd() / ns.catalog).resolve()
597
598 configure_logger(ns)
599
600 log_info(
601 """OPTIONS
602 Source: {ns.source}
603 Build: {ns.build}
604 Temp: {ns.temp}
605 Arch: {ns.arch}
606
607 Copy to: {ns.copy}
608 Zip to: {ns.zip}
609 Catalog: {ns.catalog}""",
610 ns=ns,
611 )
612
613 if ns.arch not in ("win32", "amd64", "arm32", "arm64"):
614 log_error("--arch is not a valid value (win32, amd64, arm32, arm64)")
615 return 4
616 if ns.arch in ("arm32", "arm64"):
617 for n in ("include_idle", "include_tcltk"):
618 if getattr(ns, n):
619 log_warning(f"Disabling --{n.replace('_', '-')} on unsupported platform")
620 setattr(ns, n, False)
621
622 if ns.include_idle and not ns.include_tcltk:
623 log_warning("Assuming --include-tcltk to support --include-idle")
624 ns.include_tcltk = True
625
626 try:
627 generate_source_files(ns)
628 files = list(get_layout(ns))
629 copy_files(files, ns)
630 except KeyboardInterrupt:
631 log_info("Interrupted by Ctrl+C")
632 return 3
633 except SystemExit:
634 raise
635 except:
636 log_exception("Unhandled error")
637
638 if error_was_logged():
639 log_error("Errors occurred.")
640 return 1
641
642
643 if __name__ == "__main__":
644 sys.exit(int(main() or 0))