1 #!/usr/bin/env python3
2 """Build script for Python on WebAssembly platforms.
3
4 $ ./Tools/wasm/wasm_builder.py emscripten-browser build repl
5 $ ./Tools/wasm/wasm_builder.py emscripten-node-dl build test
6 $ ./Tools/wasm/wasm_builder.py wasi build test
7
8 Primary build targets are "emscripten-node-dl" (NodeJS, dynamic linking),
9 "emscripten-browser", and "wasi".
10
11 Emscripten builds require a recent Emscripten SDK. The tools looks for an
12 activated EMSDK environment (". /path/to/emsdk_env.sh"). System packages
13 (Debian, Homebrew) are not supported.
14
15 WASI builds require WASI SDK and wasmtime. The tool looks for 'WASI_SDK_PATH'
16 and falls back to /opt/wasi-sdk.
17
18 The 'build' Python interpreter must be rebuilt every time Python's byte code
19 changes.
20
21 ./Tools/wasm/wasm_builder.py --clean build build
22
23 """
24 import argparse
25 import enum
26 import dataclasses
27 import logging
28 import os
29 import pathlib
30 import re
31 import shlex
32 import shutil
33 import socket
34 import subprocess
35 import sys
36 import sysconfig
37 import tempfile
38 import time
39 import warnings
40 import webbrowser
41
42 # for Python 3.8
43 from typing import Any, Callable, Dict, Iterable, List, Optional, Tuple, Union
44
45 logger = logging.getLogger("wasm_build")
46
47 SRCDIR = pathlib.Path(__file__).parent.parent.parent.absolute()
48 WASMTOOLS = SRCDIR / "Tools" / "wasm"
49 BUILDDIR = SRCDIR / "builddir"
50 CONFIGURE = SRCDIR / "configure"
51 SETUP_LOCAL = SRCDIR / "Modules" / "Setup.local"
52
53 HAS_CCACHE = shutil.which("ccache") is not None
54
55 # path to WASI-SDK root
56 WASI_SDK_PATH = pathlib.Path(os.environ.get("WASI_SDK_PATH", "/opt/wasi-sdk"))
57
58 # path to Emscripten SDK config file.
59 # auto-detect's EMSDK in /opt/emsdk without ". emsdk_env.sh".
60 EM_CONFIG = pathlib.Path(os.environ.setdefault("EM_CONFIG", "/opt/emsdk/.emscripten"))
61 EMSDK_MIN_VERSION = (3, 1, 19)
62 EMSDK_BROKEN_VERSION = {
63 (3, 1, 14): "https://github.com/emscripten-core/emscripten/issues/17338",
64 (3, 1, 16): "https://github.com/emscripten-core/emscripten/issues/17393",
65 (3, 1, 20): "https://github.com/emscripten-core/emscripten/issues/17720",
66 }
67 _MISSING = pathlib.PurePath("MISSING")
68
69 WASM_WEBSERVER = WASMTOOLS / "wasm_webserver.py"
70
71 CLEAN_SRCDIR = f"""
72 Builds require a clean source directory. Please use a clean checkout or
73 run "make clean -C '{SRCDIR}'".
74 """
75
76 INSTALL_NATIVE = """
77 Builds require a C compiler (gcc, clang), make, pkg-config, and development
78 headers for dependencies like zlib.
79
80 Debian/Ubuntu: sudo apt install build-essential git curl pkg-config zlib1g-dev
81 Fedora/CentOS: sudo dnf install gcc make git-core curl pkgconfig zlib-devel
82 """
83
84 INSTALL_EMSDK = """
85 wasm32-emscripten builds need Emscripten SDK. Please follow instructions at
86 https://emscripten.org/docs/getting_started/downloads.html how to install
87 Emscripten and how to activate the SDK with "emsdk_env.sh".
88
89 git clone https://github.com/emscripten-core/emsdk.git /path/to/emsdk
90 cd /path/to/emsdk
91 ./emsdk install latest
92 ./emsdk activate latest
93 source /path/to/emsdk_env.sh
94 """
95
96 INSTALL_WASI_SDK = """
97 wasm32-wasi builds need WASI SDK. Please fetch the latest SDK from
98 https://github.com/WebAssembly/wasi-sdk/releases and install it to
99 "/opt/wasi-sdk". Alternatively you can install the SDK in a different location
100 and point the environment variable WASI_SDK_PATH to the root directory
101 of the SDK. The SDK is available for Linux x86_64, macOS x86_64, and MinGW.
102 """
103
104 INSTALL_WASMTIME = """
105 wasm32-wasi tests require wasmtime on PATH. Please follow instructions at
106 https://wasmtime.dev/ to install wasmtime.
107 """
108
109
110 def parse_emconfig(
111 emconfig: pathlib.Path = EM_CONFIG,
112 ) -> Tuple[pathlib.PurePath, pathlib.PurePath]:
113 """Parse EM_CONFIG file and lookup EMSCRIPTEN_ROOT and NODE_JS.
114
115 The ".emscripten" config file is a Python snippet that uses "EM_CONFIG"
116 environment variable. EMSCRIPTEN_ROOT is the "upstream/emscripten"
117 subdirectory with tools like "emconfigure".
118 """
119 if not emconfig.exists():
120 return _MISSING, _MISSING
121 with open(emconfig, encoding="utf-8") as f:
122 code = f.read()
123 # EM_CONFIG file is a Python snippet
124 local: Dict[str, Any] = {}
125 exec(code, globals(), local)
126 emscripten_root = pathlib.Path(local["EMSCRIPTEN_ROOT"])
127 node_js = pathlib.Path(local["NODE_JS"])
128 return emscripten_root, node_js
129
130
131 EMSCRIPTEN_ROOT, NODE_JS = parse_emconfig()
132
133
134 def read_python_version(configure: pathlib.Path = CONFIGURE) -> str:
135 """Read PACKAGE_VERSION from configure script
136
137 configure and configure.ac are the canonical source for major and
138 minor version number.
139 """
140 version_re = re.compile(r"^PACKAGE_VERSION='(\d\.\d+)'")
141 with configure.open(encoding="utf-8") as f:
142 for line in f:
143 mo = version_re.match(line)
144 if mo:
145 return mo.group(1)
146 raise ValueError(f"PACKAGE_VERSION not found in {configure}")
147
148
149 PYTHON_VERSION = read_python_version()
150
151
152 class ESC[4;38;5;81mConditionError(ESC[4;38;5;149mValueError):
153 def __init__(self, info: str, text: str):
154 self.info = info
155 self.text = text
156
157 def __str__(self):
158 return f"{type(self).__name__}: '{self.info}'\n{self.text}"
159
160
161 class ESC[4;38;5;81mMissingDependency(ESC[4;38;5;149mConditionError):
162 pass
163
164
165 class ESC[4;38;5;81mDirtySourceDirectory(ESC[4;38;5;149mConditionError):
166 pass
167
168
169 @dataclasses.dataclass
170 class ESC[4;38;5;81mPlatform:
171 """Platform-specific settings
172
173 - CONFIG_SITE override
174 - configure wrapper (e.g. emconfigure)
175 - make wrapper (e.g. emmake)
176 - additional environment variables
177 - check function to verify SDK
178 """
179
180 name: str
181 pythonexe: str
182 config_site: Optional[pathlib.PurePath]
183 configure_wrapper: Optional[pathlib.PurePath]
184 make_wrapper: Optional[pathlib.PurePath]
185 environ: dict
186 check: Callable[[], None]
187 # Used for build_emports().
188 ports: Optional[pathlib.PurePath]
189 cc: Optional[pathlib.PurePath]
190
191 def getenv(self, profile: "BuildProfile") -> dict:
192 return self.environ.copy()
193
194
195 def _check_clean_src():
196 candidates = [
197 SRCDIR / "Programs" / "python.o",
198 SRCDIR / "Python" / "frozen_modules" / "importlib._bootstrap.h",
199 ]
200 for candidate in candidates:
201 if candidate.exists():
202 raise DirtySourceDirectory(os.fspath(candidate), CLEAN_SRCDIR)
203
204
205 def _check_native():
206 if not any(shutil.which(cc) for cc in ["cc", "gcc", "clang"]):
207 raise MissingDependency("cc", INSTALL_NATIVE)
208 if not shutil.which("make"):
209 raise MissingDependency("make", INSTALL_NATIVE)
210 if sys.platform == "linux":
211 # skip pkg-config check on macOS
212 if not shutil.which("pkg-config"):
213 raise MissingDependency("pkg-config", INSTALL_NATIVE)
214 # zlib is needed to create zip files
215 for devel in ["zlib"]:
216 try:
217 subprocess.check_call(["pkg-config", "--exists", devel])
218 except subprocess.CalledProcessError:
219 raise MissingDependency(devel, INSTALL_NATIVE) from None
220 _check_clean_src()
221
222
223 NATIVE = Platform(
224 "native",
225 # macOS has python.exe
226 pythonexe=sysconfig.get_config_var("BUILDPYTHON") or "python",
227 config_site=None,
228 configure_wrapper=None,
229 ports=None,
230 cc=None,
231 make_wrapper=None,
232 environ={},
233 check=_check_native,
234 )
235
236
237 def _check_emscripten():
238 if EMSCRIPTEN_ROOT is _MISSING:
239 raise MissingDependency("Emscripten SDK EM_CONFIG", INSTALL_EMSDK)
240 # sanity check
241 emconfigure = EMSCRIPTEN.configure_wrapper
242 if not emconfigure.exists():
243 raise MissingDependency(os.fspath(emconfigure), INSTALL_EMSDK)
244 # version check
245 version_txt = EMSCRIPTEN_ROOT / "emscripten-version.txt"
246 if not version_txt.exists():
247 raise MissingDependency(os.fspath(version_txt), INSTALL_EMSDK)
248 with open(version_txt) as f:
249 version = f.read().strip().strip('"')
250 if version.endswith("-git"):
251 # git / upstream / tot-upstream installation
252 version = version[:-4]
253 version_tuple = tuple(int(v) for v in version.split("."))
254 if version_tuple < EMSDK_MIN_VERSION:
255 raise ConditionError(
256 os.fspath(version_txt),
257 f"Emscripten SDK {version} in '{EMSCRIPTEN_ROOT}' is older than "
258 "minimum required version "
259 f"{'.'.join(str(v) for v in EMSDK_MIN_VERSION)}.",
260 )
261 broken = EMSDK_BROKEN_VERSION.get(version_tuple)
262 if broken is not None:
263 raise ConditionError(
264 os.fspath(version_txt),
265 (
266 f"Emscripten SDK {version} in '{EMSCRIPTEN_ROOT}' has known "
267 f"bugs, see {broken}."
268 ),
269 )
270 if os.environ.get("PKG_CONFIG_PATH"):
271 warnings.warn(
272 "PKG_CONFIG_PATH is set and not empty. emconfigure overrides "
273 "this environment variable. Use EM_PKG_CONFIG_PATH instead."
274 )
275 _check_clean_src()
276
277
278 EMSCRIPTEN = Platform(
279 "emscripten",
280 pythonexe="python.js",
281 config_site=WASMTOOLS / "config.site-wasm32-emscripten",
282 configure_wrapper=EMSCRIPTEN_ROOT / "emconfigure",
283 ports=EMSCRIPTEN_ROOT / "embuilder",
284 cc=EMSCRIPTEN_ROOT / "emcc",
285 make_wrapper=EMSCRIPTEN_ROOT / "emmake",
286 environ={
287 # workaround for https://github.com/emscripten-core/emscripten/issues/17635
288 "TZ": "UTC",
289 "EM_COMPILER_WRAPPER": "ccache" if HAS_CCACHE else None,
290 "PATH": [EMSCRIPTEN_ROOT, os.environ["PATH"]],
291 },
292 check=_check_emscripten,
293 )
294
295
296 def _check_wasi():
297 wasm_ld = WASI_SDK_PATH / "bin" / "wasm-ld"
298 if not wasm_ld.exists():
299 raise MissingDependency(os.fspath(wasm_ld), INSTALL_WASI_SDK)
300 wasmtime = shutil.which("wasmtime")
301 if wasmtime is None:
302 raise MissingDependency("wasmtime", INSTALL_WASMTIME)
303 _check_clean_src()
304
305
306 WASI = Platform(
307 "wasi",
308 pythonexe="python.wasm",
309 config_site=WASMTOOLS / "config.site-wasm32-wasi",
310 configure_wrapper=WASMTOOLS / "wasi-env",
311 ports=None,
312 cc=WASI_SDK_PATH / "bin" / "clang",
313 make_wrapper=None,
314 environ={
315 "WASI_SDK_PATH": WASI_SDK_PATH,
316 # workaround for https://github.com/python/cpython/issues/95952
317 "HOSTRUNNER": (
318 "wasmtime run "
319 "--env PYTHONPATH=/{relbuilddir}/build/lib.wasi-wasm32-{version}:/Lib "
320 "--mapdir /::{srcdir} --"
321 ),
322 "PATH": [WASI_SDK_PATH / "bin", os.environ["PATH"]],
323 },
324 check=_check_wasi,
325 )
326
327
328 class ESC[4;38;5;81mHost(ESC[4;38;5;149menumESC[4;38;5;149m.ESC[4;38;5;149mEnum):
329 """Target host triplet"""
330
331 wasm32_emscripten = "wasm32-unknown-emscripten"
332 wasm64_emscripten = "wasm64-unknown-emscripten"
333 wasm32_wasi = "wasm32-unknown-wasi"
334 wasm64_wasi = "wasm64-unknown-wasi"
335 # current platform
336 build = sysconfig.get_config_var("BUILD_GNU_TYPE")
337
338 @property
339 def platform(self) -> Platform:
340 if self.is_emscripten:
341 return EMSCRIPTEN
342 elif self.is_wasi:
343 return WASI
344 else:
345 return NATIVE
346
347 @property
348 def is_emscripten(self) -> bool:
349 cls = type(self)
350 return self in {cls.wasm32_emscripten, cls.wasm64_emscripten}
351
352 @property
353 def is_wasi(self) -> bool:
354 cls = type(self)
355 return self in {cls.wasm32_wasi, cls.wasm64_wasi}
356
357 def get_extra_paths(self) -> Iterable[pathlib.PurePath]:
358 """Host-specific os.environ["PATH"] entries.
359
360 Emscripten's Node version 14.x works well for wasm32-emscripten.
361 wasm64-emscripten requires more recent v8 version, e.g. node 16.x.
362 Attempt to use system's node command.
363 """
364 cls = type(self)
365 if self == cls.wasm32_emscripten:
366 return [NODE_JS.parent]
367 elif self == cls.wasm64_emscripten:
368 # TODO: look for recent node
369 return []
370 else:
371 return []
372
373 @property
374 def emport_args(self) -> List[str]:
375 """Host-specific port args (Emscripten)."""
376 cls = type(self)
377 if self is cls.wasm64_emscripten:
378 return ["-sMEMORY64=1"]
379 elif self is cls.wasm32_emscripten:
380 return ["-sMEMORY64=0"]
381 else:
382 return []
383
384 @property
385 def embuilder_args(self) -> List[str]:
386 """Host-specific embuilder args (Emscripten)."""
387 cls = type(self)
388 if self is cls.wasm64_emscripten:
389 return ["--wasm64"]
390 else:
391 return []
392
393
394 class ESC[4;38;5;81mEmscriptenTarget(ESC[4;38;5;149menumESC[4;38;5;149m.ESC[4;38;5;149mEnum):
395 """Emscripten-specific targets (--with-emscripten-target)"""
396
397 browser = "browser"
398 browser_debug = "browser-debug"
399 node = "node"
400 node_debug = "node-debug"
401
402 @property
403 def is_browser(self):
404 cls = type(self)
405 return self in {cls.browser, cls.browser_debug}
406
407 @property
408 def emport_args(self) -> List[str]:
409 """Target-specific port args."""
410 cls = type(self)
411 if self in {cls.browser_debug, cls.node_debug}:
412 # some libs come in debug and non-debug builds
413 return ["-O0"]
414 else:
415 return ["-O2"]
416
417
418 class ESC[4;38;5;81mSupportLevel(ESC[4;38;5;149menumESC[4;38;5;149m.ESC[4;38;5;149mEnum):
419 supported = "tier 3, supported"
420 working = "working, unsupported"
421 experimental = "experimental, may be broken"
422 broken = "broken / unavailable"
423
424 def __bool__(self):
425 cls = type(self)
426 return self in {cls.supported, cls.working}
427
428
429 @dataclasses.dataclass
430 class ESC[4;38;5;81mBuildProfile:
431 name: str
432 support_level: SupportLevel
433 host: Host
434 target: Union[EmscriptenTarget, None] = None
435 dynamic_linking: Union[bool, None] = None
436 pthreads: Union[bool, None] = None
437 default_testopts: str = "-j2"
438
439 @property
440 def is_browser(self) -> bool:
441 """Is this a browser build?"""
442 return self.target is not None and self.target.is_browser
443
444 @property
445 def builddir(self) -> pathlib.Path:
446 """Path to build directory"""
447 return BUILDDIR / self.name
448
449 @property
450 def python_cmd(self) -> pathlib.Path:
451 """Path to python executable"""
452 return self.builddir / self.host.platform.pythonexe
453
454 @property
455 def makefile(self) -> pathlib.Path:
456 """Path to Makefile"""
457 return self.builddir / "Makefile"
458
459 @property
460 def configure_cmd(self) -> List[str]:
461 """Generate configure command"""
462 # use relative path, so WASI tests can find lib prefix.
463 # pathlib.Path.relative_to() does not work here.
464 configure = os.path.relpath(CONFIGURE, self.builddir)
465 cmd = [configure, "-C"]
466 platform = self.host.platform
467 if platform.configure_wrapper:
468 cmd.insert(0, os.fspath(platform.configure_wrapper))
469
470 cmd.append(f"--host={self.host.value}")
471 cmd.append(f"--build={Host.build.value}")
472
473 if self.target is not None:
474 assert self.host.is_emscripten
475 cmd.append(f"--with-emscripten-target={self.target.value}")
476
477 if self.dynamic_linking is not None:
478 assert self.host.is_emscripten
479 opt = "enable" if self.dynamic_linking else "disable"
480 cmd.append(f"--{opt}-wasm-dynamic-linking")
481
482 if self.pthreads is not None:
483 opt = "enable" if self.pthreads else "disable"
484 cmd.append(f"--{opt}-wasm-pthreads")
485
486 if self.host != Host.build:
487 cmd.append(f"--with-build-python={BUILD.python_cmd}")
488
489 if platform.config_site is not None:
490 cmd.append(f"CONFIG_SITE={platform.config_site}")
491
492 return cmd
493
494 @property
495 def make_cmd(self) -> List[str]:
496 """Generate make command"""
497 cmd = ["make"]
498 platform = self.host.platform
499 if platform.make_wrapper:
500 cmd.insert(0, os.fspath(platform.make_wrapper))
501 return cmd
502
503 def getenv(self) -> dict:
504 """Generate environ dict for platform"""
505 env = os.environ.copy()
506 env.setdefault("MAKEFLAGS", f"-j{os.cpu_count()}")
507 platenv = self.host.platform.getenv(self)
508 for key, value in platenv.items():
509 if value is None:
510 env.pop(key, None)
511 elif key == "PATH":
512 # list of path items, prefix with extra paths
513 new_path: List[pathlib.PurePath] = []
514 new_path.extend(self.host.get_extra_paths())
515 new_path.extend(value)
516 env[key] = os.pathsep.join(os.fspath(p) for p in new_path)
517 elif isinstance(value, str):
518 env[key] = value.format(
519 relbuilddir=self.builddir.relative_to(SRCDIR),
520 srcdir=SRCDIR,
521 version=PYTHON_VERSION,
522 )
523 else:
524 env[key] = value
525 return env
526
527 def _run_cmd(
528 self,
529 cmd: Iterable[str],
530 args: Iterable[str] = (),
531 cwd: Optional[pathlib.Path] = None,
532 ):
533 cmd = list(cmd)
534 cmd.extend(args)
535 if cwd is None:
536 cwd = self.builddir
537 logger.info('Running "%s" in "%s"', shlex.join(cmd), cwd)
538 return subprocess.check_call(
539 cmd,
540 cwd=os.fspath(cwd),
541 env=self.getenv(),
542 )
543
544 def _check_execute(self):
545 if self.is_browser:
546 raise ValueError(f"Cannot execute on {self.target}")
547
548 def run_build(self, *args):
549 """Run configure (if necessary) and make"""
550 if not self.makefile.exists():
551 logger.info("Makefile not found, running configure")
552 self.run_configure(*args)
553 self.run_make("all", *args)
554
555 def run_configure(self, *args):
556 """Run configure script to generate Makefile"""
557 os.makedirs(self.builddir, exist_ok=True)
558 return self._run_cmd(self.configure_cmd, args)
559
560 def run_make(self, *args):
561 """Run make (defaults to build all)"""
562 return self._run_cmd(self.make_cmd, args)
563
564 def run_pythoninfo(self, *args):
565 """Run 'make pythoninfo'"""
566 self._check_execute()
567 return self.run_make("pythoninfo", *args)
568
569 def run_test(self, target: str, testopts: Optional[str] = None):
570 """Run buildbottests"""
571 self._check_execute()
572 if testopts is None:
573 testopts = self.default_testopts
574 return self.run_make(target, f"TESTOPTS={testopts}")
575
576 def run_py(self, *args):
577 """Run Python with hostrunner"""
578 self._check_execute()
579 self.run_make(
580 "--eval", f"run: all; $(HOSTRUNNER) ./$(PYTHON) {shlex.join(args)}", "run"
581 )
582
583 def run_browser(self, bind="127.0.0.1", port=8000):
584 """Run WASM webserver and open build in browser"""
585 relbuilddir = self.builddir.relative_to(SRCDIR)
586 url = f"http://{bind}:{port}/{relbuilddir}/python.html"
587 args = [
588 sys.executable,
589 os.fspath(WASM_WEBSERVER),
590 "--bind",
591 bind,
592 "--port",
593 str(port),
594 ]
595 srv = subprocess.Popen(args, cwd=SRCDIR)
596 # wait for server
597 end = time.monotonic() + 3.0
598 while time.monotonic() < end and srv.returncode is None:
599 try:
600 with socket.create_connection((bind, port), timeout=0.1) as _:
601 pass
602 except OSError:
603 time.sleep(0.01)
604 else:
605 break
606
607 webbrowser.open(url)
608
609 try:
610 srv.wait()
611 except KeyboardInterrupt:
612 pass
613
614 def clean(self, all: bool = False):
615 """Clean build directory"""
616 if all:
617 if self.builddir.exists():
618 shutil.rmtree(self.builddir)
619 elif self.makefile.exists():
620 self.run_make("clean")
621
622 def build_emports(self, force: bool = False):
623 """Pre-build emscripten ports."""
624 platform = self.host.platform
625 if platform.ports is None or platform.cc is None:
626 raise ValueError("Need ports and CC command")
627
628 embuilder_cmd = [os.fspath(platform.ports)]
629 embuilder_cmd.extend(self.host.embuilder_args)
630 if force:
631 embuilder_cmd.append("--force")
632
633 ports_cmd = [os.fspath(platform.cc)]
634 ports_cmd.extend(self.host.emport_args)
635 if self.target:
636 ports_cmd.extend(self.target.emport_args)
637
638 if self.dynamic_linking:
639 # Trigger PIC build.
640 ports_cmd.append("-sMAIN_MODULE")
641 embuilder_cmd.append("--pic")
642
643 if self.pthreads:
644 # Trigger multi-threaded build.
645 ports_cmd.append("-sUSE_PTHREADS")
646
647 # Pre-build libbz2, libsqlite3, libz, and some system libs.
648 ports_cmd.extend(["-sUSE_ZLIB", "-sUSE_BZIP2", "-sUSE_SQLITE3"])
649 # Multi-threaded sqlite3 has different suffix
650 embuilder_cmd.extend(
651 ["build", "bzip2", "sqlite3-mt" if self.pthreads else "sqlite3", "zlib"]
652 )
653
654 self._run_cmd(embuilder_cmd, cwd=SRCDIR)
655
656 with tempfile.TemporaryDirectory(suffix="-py-emport") as tmpdir:
657 tmppath = pathlib.Path(tmpdir)
658 main_c = tmppath / "main.c"
659 main_js = tmppath / "main.js"
660 with main_c.open("w") as f:
661 f.write("int main(void) { return 0; }\n")
662 args = [
663 os.fspath(main_c),
664 "-o",
665 os.fspath(main_js),
666 ]
667 self._run_cmd(ports_cmd, args, cwd=tmppath)
668
669
670 # native build (build Python)
671 BUILD = BuildProfile(
672 "build",
673 support_level=SupportLevel.working,
674 host=Host.build,
675 )
676
677 _profiles = [
678 BUILD,
679 # wasm32-emscripten
680 BuildProfile(
681 "emscripten-browser",
682 support_level=SupportLevel.supported,
683 host=Host.wasm32_emscripten,
684 target=EmscriptenTarget.browser,
685 dynamic_linking=True,
686 ),
687 BuildProfile(
688 "emscripten-browser-debug",
689 support_level=SupportLevel.working,
690 host=Host.wasm32_emscripten,
691 target=EmscriptenTarget.browser_debug,
692 dynamic_linking=True,
693 ),
694 BuildProfile(
695 "emscripten-node-dl",
696 support_level=SupportLevel.supported,
697 host=Host.wasm32_emscripten,
698 target=EmscriptenTarget.node,
699 dynamic_linking=True,
700 ),
701 BuildProfile(
702 "emscripten-node-dl-debug",
703 support_level=SupportLevel.working,
704 host=Host.wasm32_emscripten,
705 target=EmscriptenTarget.node_debug,
706 dynamic_linking=True,
707 ),
708 BuildProfile(
709 "emscripten-node-pthreads",
710 support_level=SupportLevel.supported,
711 host=Host.wasm32_emscripten,
712 target=EmscriptenTarget.node,
713 pthreads=True,
714 ),
715 BuildProfile(
716 "emscripten-node-pthreads-debug",
717 support_level=SupportLevel.working,
718 host=Host.wasm32_emscripten,
719 target=EmscriptenTarget.node_debug,
720 pthreads=True,
721 ),
722 # Emscripten build with both pthreads and dynamic linking is crashing.
723 BuildProfile(
724 "emscripten-node-dl-pthreads-debug",
725 support_level=SupportLevel.broken,
726 host=Host.wasm32_emscripten,
727 target=EmscriptenTarget.node_debug,
728 dynamic_linking=True,
729 pthreads=True,
730 ),
731 # wasm64-emscripten (requires Emscripten >= 3.1.21)
732 BuildProfile(
733 "wasm64-emscripten-node-debug",
734 support_level=SupportLevel.experimental,
735 host=Host.wasm64_emscripten,
736 target=EmscriptenTarget.node_debug,
737 # MEMORY64 is not compatible with dynamic linking
738 dynamic_linking=False,
739 pthreads=False,
740 ),
741 # wasm32-wasi
742 BuildProfile(
743 "wasi",
744 support_level=SupportLevel.supported,
745 host=Host.wasm32_wasi,
746 ),
747 # wasm32-wasi-threads
748 BuildProfile(
749 "wasi-threads",
750 support_level=SupportLevel.experimental,
751 host=Host.wasm32_wasi,
752 pthreads=True,
753 ),
754 # no SDK available yet
755 # BuildProfile(
756 # "wasm64-wasi",
757 # support_level=SupportLevel.broken,
758 # host=Host.wasm64_wasi,
759 # ),
760 ]
761
762 PROFILES = {p.name: p for p in _profiles}
763
764 parser = argparse.ArgumentParser(
765 "wasm_build.py",
766 description=__doc__,
767 formatter_class=argparse.RawTextHelpFormatter,
768 )
769
770 parser.add_argument(
771 "--clean",
772 "-c",
773 help="Clean build directories first",
774 action="store_true",
775 )
776
777 parser.add_argument(
778 "--verbose",
779 "-v",
780 help="Verbose logging",
781 action="store_true",
782 )
783
784 parser.add_argument(
785 "--silent",
786 help="Run configure and make in silent mode",
787 action="store_true",
788 )
789
790 parser.add_argument(
791 "--testopts",
792 help=(
793 "Additional test options for 'test' and 'hostrunnertest', e.g. "
794 "--testopts='-v test_os'."
795 ),
796 default=None,
797 )
798
799 # Don't list broken and experimental variants in help
800 platforms_choices = list(p.name for p in _profiles) + ["cleanall"]
801 platforms_help = list(p.name for p in _profiles if p.support_level) + ["cleanall"]
802 parser.add_argument(
803 "platform",
804 metavar="PLATFORM",
805 help=f"Build platform: {', '.join(platforms_help)}",
806 choices=platforms_choices,
807 )
808
809 ops = dict(
810 build="auto build (build 'build' Python, emports, configure, compile)",
811 configure="run ./configure",
812 compile="run 'make all'",
813 pythoninfo="run 'make pythoninfo'",
814 test="run 'make buildbottest TESTOPTS=...' (supports parallel tests)",
815 hostrunnertest="run 'make hostrunnertest TESTOPTS=...'",
816 repl="start interactive REPL / webserver + browser session",
817 clean="run 'make clean'",
818 cleanall="remove all build directories",
819 emports="build Emscripten port with embuilder (only Emscripten)",
820 )
821 ops_help = "\n".join(f"{op:16s} {help}" for op, help in ops.items())
822 parser.add_argument(
823 "ops",
824 metavar="OP",
825 help=f"operation (default: build)\n\n{ops_help}",
826 choices=tuple(ops),
827 default="build",
828 nargs="*",
829 )
830
831
832 def main():
833 args = parser.parse_args()
834 logging.basicConfig(
835 level=logging.INFO if args.verbose else logging.ERROR,
836 format="%(message)s",
837 )
838
839 if args.platform == "cleanall":
840 for builder in PROFILES.values():
841 builder.clean(all=True)
842 parser.exit(0)
843
844 # additional configure and make args
845 cm_args = ("--silent",) if args.silent else ()
846
847 # nargs=* with default quirk
848 if args.ops == "build":
849 args.ops = ["build"]
850
851 builder = PROFILES[args.platform]
852 try:
853 builder.host.platform.check()
854 except ConditionError as e:
855 parser.error(str(e))
856
857 if args.clean:
858 builder.clean(all=False)
859
860 # hack for WASI
861 if builder.host.is_wasi and not SETUP_LOCAL.exists():
862 SETUP_LOCAL.touch()
863
864 # auto-build
865 if "build" in args.ops:
866 # check and create build Python
867 if builder is not BUILD:
868 logger.info("Auto-building 'build' Python.")
869 try:
870 BUILD.host.platform.check()
871 except ConditionError as e:
872 parser.error(str(e))
873 if args.clean:
874 BUILD.clean(all=False)
875 BUILD.run_build(*cm_args)
876 # build Emscripten ports with embuilder
877 if builder.host.is_emscripten and "emports" not in args.ops:
878 builder.build_emports()
879
880 for op in args.ops:
881 logger.info("\n*** %s %s", args.platform, op)
882 if op == "build":
883 builder.run_build(*cm_args)
884 elif op == "configure":
885 builder.run_configure(*cm_args)
886 elif op == "compile":
887 builder.run_make("all", *cm_args)
888 elif op == "pythoninfo":
889 builder.run_pythoninfo(*cm_args)
890 elif op == "repl":
891 if builder.is_browser:
892 builder.run_browser()
893 else:
894 builder.run_py()
895 elif op == "test":
896 builder.run_test("buildbottest", testopts=args.testopts)
897 elif op == "hostrunnertest":
898 builder.run_test("hostrunnertest", testopts=args.testopts)
899 elif op == "clean":
900 builder.clean(all=False)
901 elif op == "cleanall":
902 builder.clean(all=True)
903 elif op == "emports":
904 builder.build_emports(force=args.clean)
905 else:
906 raise ValueError(op)
907
908 print(builder.builddir)
909 parser.exit(0)
910
911
912 if __name__ == "__main__":
913 main()