1 #!/usr/bin/env python3
2 """Build script for Python on WebAssembly platforms.
3
4 $ ./Tools/wasm/wasm_builder.py emscripten-browser build repl
5 $ ./Tools/wasm/wasm_builder.py emscripten-node-dl build test
6 $ ./Tools/wasm/wasm_builder.py wasi build test
7
8 Primary build targets are "emscripten-node-dl" (NodeJS, dynamic linking),
9 "emscripten-browser", and "wasi".
10
11 Emscripten builds require a recent Emscripten SDK. The tools looks for an
12 activated EMSDK environment (". /path/to/emsdk_env.sh"). System packages
13 (Debian, Homebrew) are not supported.
14
15 WASI builds require WASI SDK and wasmtime. The tool looks for 'WASI_SDK_PATH'
16 and falls back to /opt/wasi-sdk.
17
18 The 'build' Python interpreter must be rebuilt every time Python's byte code
19 changes.
20
21 ./Tools/wasm/wasm_builder.py --clean build build
22
23 """
24 import argparse
25 import enum
26 import dataclasses
27 import logging
28 import os
29 import pathlib
30 import re
31 import shlex
32 import shutil
33 import socket
34 import subprocess
35 import sys
36 import sysconfig
37 import tempfile
38 import time
39 import warnings
40 import webbrowser
41
42 # for Python 3.8
43 from typing import Any, Callable, Dict, Iterable, List, Optional, Tuple, Union
44
45 logger = logging.getLogger("wasm_build")
46
47 SRCDIR = pathlib.Path(__file__).parent.parent.parent.absolute()
48 WASMTOOLS = SRCDIR / "Tools" / "wasm"
49 BUILDDIR = SRCDIR / "builddir"
50 CONFIGURE = SRCDIR / "configure"
51 SETUP_LOCAL = SRCDIR / "Modules" / "Setup.local"
52
53 HAS_CCACHE = shutil.which("ccache") is not None
54
55 # path to WASI-SDK root
56 WASI_SDK_PATH = pathlib.Path(os.environ.get("WASI_SDK_PATH", "/opt/wasi-sdk"))
57
58 # path to Emscripten SDK config file.
59 # auto-detect's EMSDK in /opt/emsdk without ". emsdk_env.sh".
60 EM_CONFIG = pathlib.Path(os.environ.setdefault("EM_CONFIG", "/opt/emsdk/.emscripten"))
61 EMSDK_MIN_VERSION = (3, 1, 19)
62 EMSDK_BROKEN_VERSION = {
63 (3, 1, 14): "https://github.com/emscripten-core/emscripten/issues/17338",
64 (3, 1, 16): "https://github.com/emscripten-core/emscripten/issues/17393",
65 (3, 1, 20): "https://github.com/emscripten-core/emscripten/issues/17720",
66 }
67 _MISSING = pathlib.PurePath("MISSING")
68
69 WASM_WEBSERVER = WASMTOOLS / "wasm_webserver.py"
70
71 CLEAN_SRCDIR = f"""
72 Builds require a clean source directory. Please use a clean checkout or
73 run "make clean -C '{SRCDIR}'".
74 """
75
76 INSTALL_NATIVE = f"""
77 Builds require a C compiler (gcc, clang), make, pkg-config, and development
78 headers for dependencies like zlib.
79
80 Debian/Ubuntu: sudo apt install build-essential git curl pkg-config zlib1g-dev
81 Fedora/CentOS: sudo dnf install gcc make git-core curl pkgconfig zlib-devel
82 """
83
84 INSTALL_EMSDK = """
85 wasm32-emscripten builds need Emscripten SDK. Please follow instructions at
86 https://emscripten.org/docs/getting_started/downloads.html how to install
87 Emscripten and how to activate the SDK with "emsdk_env.sh".
88
89 git clone https://github.com/emscripten-core/emsdk.git /path/to/emsdk
90 cd /path/to/emsdk
91 ./emsdk install latest
92 ./emsdk activate latest
93 source /path/to/emsdk_env.sh
94 """
95
96 INSTALL_WASI_SDK = """
97 wasm32-wasi builds need WASI SDK. Please fetch the latest SDK from
98 https://github.com/WebAssembly/wasi-sdk/releases and install it to
99 "/opt/wasi-sdk". Alternatively you can install the SDK in a different location
100 and point the environment variable WASI_SDK_PATH to the root directory
101 of the SDK. The SDK is available for Linux x86_64, macOS x86_64, and MinGW.
102 """
103
104 INSTALL_WASMTIME = """
105 wasm32-wasi tests require wasmtime on PATH. Please follow instructions at
106 https://wasmtime.dev/ to install wasmtime.
107 """
108
109
110 def parse_emconfig(
111 emconfig: pathlib.Path = EM_CONFIG,
112 ) -> Tuple[pathlib.PurePath, pathlib.PurePath]:
113 """Parse EM_CONFIG file and lookup EMSCRIPTEN_ROOT and NODE_JS.
114
115 The ".emscripten" config file is a Python snippet that uses "EM_CONFIG"
116 environment variable. EMSCRIPTEN_ROOT is the "upstream/emscripten"
117 subdirectory with tools like "emconfigure".
118 """
119 if not emconfig.exists():
120 return _MISSING, _MISSING
121 with open(emconfig, encoding="utf-8") as f:
122 code = f.read()
123 # EM_CONFIG file is a Python snippet
124 local: Dict[str, Any] = {}
125 exec(code, globals(), local)
126 emscripten_root = pathlib.Path(local["EMSCRIPTEN_ROOT"])
127 node_js = pathlib.Path(local["NODE_JS"])
128 return emscripten_root, node_js
129
130
131 EMSCRIPTEN_ROOT, NODE_JS = parse_emconfig()
132
133
134 def read_python_version(configure: pathlib.Path = CONFIGURE) -> str:
135 """Read PACKAGE_VERSION from configure script
136
137 configure and configure.ac are the canonical source for major and
138 minor version number.
139 """
140 version_re = re.compile("^PACKAGE_VERSION='(\d\.\d+)'")
141 with configure.open(encoding="utf-8") as f:
142 for line in f:
143 mo = version_re.match(line)
144 if mo:
145 return mo.group(1)
146 raise ValueError(f"PACKAGE_VERSION not found in {configure}")
147
148
149 PYTHON_VERSION = read_python_version()
150
151
152 class ESC[4;38;5;81mConditionError(ESC[4;38;5;149mValueError):
153 def __init__(self, info: str, text: str):
154 self.info = info
155 self.text = text
156
157 def __str__(self):
158 return f"{type(self).__name__}: '{self.info}'\n{self.text}"
159
160
161 class ESC[4;38;5;81mMissingDependency(ESC[4;38;5;149mConditionError):
162 pass
163
164
165 class ESC[4;38;5;81mDirtySourceDirectory(ESC[4;38;5;149mConditionError):
166 pass
167
168
169 @dataclasses.dataclass
170 class ESC[4;38;5;81mPlatform:
171 """Platform-specific settings
172
173 - CONFIG_SITE override
174 - configure wrapper (e.g. emconfigure)
175 - make wrapper (e.g. emmake)
176 - additional environment variables
177 - check function to verify SDK
178 """
179
180 name: str
181 pythonexe: str
182 config_site: Optional[pathlib.PurePath]
183 configure_wrapper: Optional[pathlib.PurePath]
184 make_wrapper: Optional[pathlib.PurePath]
185 environ: dict
186 check: Callable[[], None]
187 # Used for build_emports().
188 ports: Optional[pathlib.PurePath]
189 cc: Optional[pathlib.PurePath]
190
191 def getenv(self, profile: "BuildProfile") -> dict:
192 return self.environ.copy()
193
194
195 def _check_clean_src():
196 candidates = [
197 SRCDIR / "Programs" / "python.o",
198 SRCDIR / "Python" / "frozen_modules" / "importlib._bootstrap.h",
199 ]
200 for candidate in candidates:
201 if candidate.exists():
202 raise DirtySourceDirectory(os.fspath(candidate), CLEAN_SRCDIR)
203
204
205 def _check_native():
206 if not any(shutil.which(cc) for cc in ["cc", "gcc", "clang"]):
207 raise MissingDependency("cc", INSTALL_NATIVE)
208 if not shutil.which("make"):
209 raise MissingDependency("make", INSTALL_NATIVE)
210 if sys.platform == "linux":
211 # skip pkg-config check on macOS
212 if not shutil.which("pkg-config"):
213 raise MissingDependency("pkg-config", INSTALL_NATIVE)
214 # zlib is needed to create zip files
215 for devel in ["zlib"]:
216 try:
217 subprocess.check_call(["pkg-config", "--exists", devel])
218 except subprocess.CalledProcessError:
219 raise MissingDependency(devel, INSTALL_NATIVE) from None
220 _check_clean_src()
221
222
223 NATIVE = Platform(
224 "native",
225 # macOS has python.exe
226 pythonexe=sysconfig.get_config_var("BUILDPYTHON") or "python",
227 config_site=None,
228 configure_wrapper=None,
229 ports=None,
230 cc=None,
231 make_wrapper=None,
232 environ={},
233 check=_check_native,
234 )
235
236
237 def _check_emscripten():
238 if EMSCRIPTEN_ROOT is _MISSING:
239 raise MissingDependency("Emscripten SDK EM_CONFIG", INSTALL_EMSDK)
240 # sanity check
241 emconfigure = EMSCRIPTEN.configure_wrapper
242 if not emconfigure.exists():
243 raise MissingDependency(os.fspath(emconfigure), INSTALL_EMSDK)
244 # version check
245 version_txt = EMSCRIPTEN_ROOT / "emscripten-version.txt"
246 if not version_txt.exists():
247 raise MissingDependency(os.fspath(version_txt), INSTALL_EMSDK)
248 with open(version_txt) as f:
249 version = f.read().strip().strip('"')
250 if version.endswith("-git"):
251 # git / upstream / tot-upstream installation
252 version = version[:-4]
253 version_tuple = tuple(int(v) for v in version.split("."))
254 if version_tuple < EMSDK_MIN_VERSION:
255 raise ConditionError(
256 os.fspath(version_txt),
257 f"Emscripten SDK {version} in '{EMSCRIPTEN_ROOT}' is older than "
258 "minimum required version "
259 f"{'.'.join(str(v) for v in EMSDK_MIN_VERSION)}.",
260 )
261 broken = EMSDK_BROKEN_VERSION.get(version_tuple)
262 if broken is not None:
263 raise ConditionError(
264 os.fspath(version_txt),
265 (
266 f"Emscripten SDK {version} in '{EMSCRIPTEN_ROOT}' has known "
267 f"bugs, see {broken}."
268 ),
269 )
270 if os.environ.get("PKG_CONFIG_PATH"):
271 warnings.warn(
272 "PKG_CONFIG_PATH is set and not empty. emconfigure overrides "
273 "this environment variable. Use EM_PKG_CONFIG_PATH instead."
274 )
275 _check_clean_src()
276
277
278 EMSCRIPTEN = Platform(
279 "emscripten",
280 pythonexe="python.js",
281 config_site=WASMTOOLS / "config.site-wasm32-emscripten",
282 configure_wrapper=EMSCRIPTEN_ROOT / "emconfigure",
283 ports=EMSCRIPTEN_ROOT / "embuilder",
284 cc=EMSCRIPTEN_ROOT / "emcc",
285 make_wrapper=EMSCRIPTEN_ROOT / "emmake",
286 environ={
287 # workaround for https://github.com/emscripten-core/emscripten/issues/17635
288 "TZ": "UTC",
289 "EM_COMPILER_WRAPPER": "ccache" if HAS_CCACHE else None,
290 "PATH": [EMSCRIPTEN_ROOT, os.environ["PATH"]],
291 },
292 check=_check_emscripten,
293 )
294
295
296 def _check_wasi():
297 wasm_ld = WASI_SDK_PATH / "bin" / "wasm-ld"
298 if not wasm_ld.exists():
299 raise MissingDependency(os.fspath(wasm_ld), INSTALL_WASI_SDK)
300 wasmtime = shutil.which("wasmtime")
301 if wasmtime is None:
302 raise MissingDependency("wasmtime", INSTALL_WASMTIME)
303 _check_clean_src()
304
305
306 WASI = Platform(
307 "wasi",
308 pythonexe="python.wasm",
309 config_site=WASMTOOLS / "config.site-wasm32-wasi",
310 configure_wrapper=WASMTOOLS / "wasi-env",
311 ports=None,
312 cc=WASI_SDK_PATH / "bin" / "clang",
313 make_wrapper=None,
314 environ={
315 "WASI_SDK_PATH": WASI_SDK_PATH,
316 # workaround for https://github.com/python/cpython/issues/95952
317 "HOSTRUNNER": (
318 "wasmtime run "
319 "--env PYTHONPATH=/{relbuilddir}/build/lib.wasi-wasm32-{version}:/Lib "
320 "--mapdir /::{srcdir} --"
321 ),
322 "PATH": [WASI_SDK_PATH / "bin", os.environ["PATH"]],
323 },
324 check=_check_wasi,
325 )
326
327
328 class ESC[4;38;5;81mHost(ESC[4;38;5;149menumESC[4;38;5;149m.ESC[4;38;5;149mEnum):
329 """Target host triplet"""
330
331 wasm32_emscripten = "wasm32-unknown-emscripten"
332 wasm64_emscripten = "wasm64-unknown-emscripten"
333 wasm32_wasi = "wasm32-unknown-wasi"
334 wasm64_wasi = "wasm64-unknown-wasi"
335 # current platform
336 build = sysconfig.get_config_var("BUILD_GNU_TYPE")
337
338 @property
339 def platform(self) -> Platform:
340 if self.is_emscripten:
341 return EMSCRIPTEN
342 elif self.is_wasi:
343 return WASI
344 else:
345 return NATIVE
346
347 @property
348 def is_emscripten(self) -> bool:
349 cls = type(self)
350 return self in {cls.wasm32_emscripten, cls.wasm64_emscripten}
351
352 @property
353 def is_wasi(self) -> bool:
354 cls = type(self)
355 return self in {cls.wasm32_wasi, cls.wasm64_wasi}
356
357 def get_extra_paths(self) -> Iterable[pathlib.PurePath]:
358 """Host-specific os.environ["PATH"] entries.
359
360 Emscripten's Node version 14.x works well for wasm32-emscripten.
361 wasm64-emscripten requires more recent v8 version, e.g. node 16.x.
362 Attempt to use system's node command.
363 """
364 cls = type(self)
365 if self == cls.wasm32_emscripten:
366 return [NODE_JS.parent]
367 elif self == cls.wasm64_emscripten:
368 # TODO: look for recent node
369 return []
370 else:
371 return []
372
373 @property
374 def emport_args(self) -> List[str]:
375 """Host-specific port args (Emscripten)."""
376 cls = type(self)
377 if self is cls.wasm64_emscripten:
378 return ["-sMEMORY64=1"]
379 elif self is cls.wasm32_emscripten:
380 return ["-sMEMORY64=0"]
381 else:
382 return []
383
384 @property
385 def embuilder_args(self) -> List[str]:
386 """Host-specific embuilder args (Emscripten)."""
387 cls = type(self)
388 if self is cls.wasm64_emscripten:
389 return ["--wasm64"]
390 else:
391 return []
392
393
394 class ESC[4;38;5;81mEmscriptenTarget(ESC[4;38;5;149menumESC[4;38;5;149m.ESC[4;38;5;149mEnum):
395 """Emscripten-specific targets (--with-emscripten-target)"""
396
397 browser = "browser"
398 browser_debug = "browser-debug"
399 node = "node"
400 node_debug = "node-debug"
401
402 @property
403 def is_browser(self):
404 cls = type(self)
405 return self in {cls.browser, cls.browser_debug}
406
407 @property
408 def emport_args(self) -> List[str]:
409 """Target-specific port args."""
410 cls = type(self)
411 if self in {cls.browser_debug, cls.node_debug}:
412 # some libs come in debug and non-debug builds
413 return ["-O0"]
414 else:
415 return ["-O2"]
416
417
418 class ESC[4;38;5;81mSupportLevel(ESC[4;38;5;149menumESC[4;38;5;149m.ESC[4;38;5;149mEnum):
419 supported = "tier 3, supported"
420 working = "working, unsupported"
421 experimental = "experimental, may be broken"
422 broken = "broken / unavailable"
423
424 def __bool__(self):
425 cls = type(self)
426 return self in {cls.supported, cls.working}
427
428
429 @dataclasses.dataclass
430 class ESC[4;38;5;81mBuildProfile:
431 name: str
432 support_level: SupportLevel
433 host: Host
434 target: Union[EmscriptenTarget, None] = None
435 dynamic_linking: Union[bool, None] = None
436 pthreads: Union[bool, None] = None
437 default_testopts: str = "-j2"
438
439 @property
440 def is_browser(self) -> bool:
441 """Is this a browser build?"""
442 return self.target is not None and self.target.is_browser
443
444 @property
445 def builddir(self) -> pathlib.Path:
446 """Path to build directory"""
447 return BUILDDIR / self.name
448
449 @property
450 def python_cmd(self) -> pathlib.Path:
451 """Path to python executable"""
452 return self.builddir / self.host.platform.pythonexe
453
454 @property
455 def makefile(self) -> pathlib.Path:
456 """Path to Makefile"""
457 return self.builddir / "Makefile"
458
459 @property
460 def configure_cmd(self) -> List[str]:
461 """Generate configure command"""
462 # use relative path, so WASI tests can find lib prefix.
463 # pathlib.Path.relative_to() does not work here.
464 configure = os.path.relpath(CONFIGURE, self.builddir)
465 cmd = [configure, "-C"]
466 platform = self.host.platform
467 if platform.configure_wrapper:
468 cmd.insert(0, os.fspath(platform.configure_wrapper))
469
470 cmd.append(f"--host={self.host.value}")
471 cmd.append(f"--build={Host.build.value}")
472
473 if self.target is not None:
474 assert self.host.is_emscripten
475 cmd.append(f"--with-emscripten-target={self.target.value}")
476
477 if self.dynamic_linking is not None:
478 assert self.host.is_emscripten
479 opt = "enable" if self.dynamic_linking else "disable"
480 cmd.append(f"--{opt}-wasm-dynamic-linking")
481
482 if self.pthreads is not None:
483 assert self.host.is_emscripten
484 opt = "enable" if self.pthreads else "disable"
485 cmd.append(f"--{opt}-wasm-pthreads")
486
487 if self.host != Host.build:
488 cmd.append(f"--with-build-python={BUILD.python_cmd}")
489
490 if platform.config_site is not None:
491 cmd.append(f"CONFIG_SITE={platform.config_site}")
492
493 return cmd
494
495 @property
496 def make_cmd(self) -> List[str]:
497 """Generate make command"""
498 cmd = ["make"]
499 platform = self.host.platform
500 if platform.make_wrapper:
501 cmd.insert(0, os.fspath(platform.make_wrapper))
502 return cmd
503
504 def getenv(self) -> dict:
505 """Generate environ dict for platform"""
506 env = os.environ.copy()
507 env.setdefault("MAKEFLAGS", f"-j{os.cpu_count()}")
508 platenv = self.host.platform.getenv(self)
509 for key, value in platenv.items():
510 if value is None:
511 env.pop(key, None)
512 elif key == "PATH":
513 # list of path items, prefix with extra paths
514 new_path: List[pathlib.PurePath] = []
515 new_path.extend(self.host.get_extra_paths())
516 new_path.extend(value)
517 env[key] = os.pathsep.join(os.fspath(p) for p in new_path)
518 elif isinstance(value, str):
519 env[key] = value.format(
520 relbuilddir=self.builddir.relative_to(SRCDIR),
521 srcdir=SRCDIR,
522 version=PYTHON_VERSION,
523 )
524 else:
525 env[key] = value
526 return env
527
528 def _run_cmd(
529 self,
530 cmd: Iterable[str],
531 args: Iterable[str] = (),
532 cwd: Optional[pathlib.Path] = None,
533 ):
534 cmd = list(cmd)
535 cmd.extend(args)
536 if cwd is None:
537 cwd = self.builddir
538 logger.info('Running "%s" in "%s"', shlex.join(cmd), cwd)
539 return subprocess.check_call(
540 cmd,
541 cwd=os.fspath(cwd),
542 env=self.getenv(),
543 )
544
545 def _check_execute(self):
546 if self.is_browser:
547 raise ValueError(f"Cannot execute on {self.target}")
548
549 def run_build(self, *args):
550 """Run configure (if necessary) and make"""
551 if not self.makefile.exists():
552 logger.info("Makefile not found, running configure")
553 self.run_configure(*args)
554 self.run_make("all", *args)
555
556 def run_configure(self, *args):
557 """Run configure script to generate Makefile"""
558 os.makedirs(self.builddir, exist_ok=True)
559 return self._run_cmd(self.configure_cmd, args)
560
561 def run_make(self, *args):
562 """Run make (defaults to build all)"""
563 return self._run_cmd(self.make_cmd, args)
564
565 def run_pythoninfo(self, *args):
566 """Run 'make pythoninfo'"""
567 self._check_execute()
568 return self.run_make("pythoninfo", *args)
569
570 def run_test(self, target: str, testopts: Optional[str] = None):
571 """Run buildbottests"""
572 self._check_execute()
573 if testopts is None:
574 testopts = self.default_testopts
575 return self.run_make(target, f"TESTOPTS={testopts}")
576
577 def run_py(self, *args):
578 """Run Python with hostrunner"""
579 self._check_execute()
580 self.run_make(
581 "--eval", f"run: all; $(HOSTRUNNER) ./$(PYTHON) {shlex.join(args)}", "run"
582 )
583
584 def run_browser(self, bind="127.0.0.1", port=8000):
585 """Run WASM webserver and open build in browser"""
586 relbuilddir = self.builddir.relative_to(SRCDIR)
587 url = f"http://{bind}:{port}/{relbuilddir}/python.html"
588 args = [
589 sys.executable,
590 os.fspath(WASM_WEBSERVER),
591 "--bind",
592 bind,
593 "--port",
594 str(port),
595 ]
596 srv = subprocess.Popen(args, cwd=SRCDIR)
597 # wait for server
598 end = time.monotonic() + 3.0
599 while time.monotonic() < end and srv.returncode is None:
600 try:
601 with socket.create_connection((bind, port), timeout=0.1) as s:
602 pass
603 except OSError:
604 time.sleep(0.01)
605 else:
606 break
607
608 webbrowser.open(url)
609
610 try:
611 srv.wait()
612 except KeyboardInterrupt:
613 pass
614
615 def clean(self, all: bool = False):
616 """Clean build directory"""
617 if all:
618 if self.builddir.exists():
619 shutil.rmtree(self.builddir)
620 elif self.makefile.exists():
621 self.run_make("clean")
622
623 def build_emports(self, force: bool = False):
624 """Pre-build emscripten ports."""
625 platform = self.host.platform
626 if platform.ports is None or platform.cc is None:
627 raise ValueError("Need ports and CC command")
628
629 embuilder_cmd = [os.fspath(platform.ports)]
630 embuilder_cmd.extend(self.host.embuilder_args)
631 if force:
632 embuilder_cmd.append("--force")
633
634 ports_cmd = [os.fspath(platform.cc)]
635 ports_cmd.extend(self.host.emport_args)
636 if self.target:
637 ports_cmd.extend(self.target.emport_args)
638
639 if self.dynamic_linking:
640 # Trigger PIC build.
641 ports_cmd.append("-sMAIN_MODULE")
642 embuilder_cmd.append("--pic")
643
644 if self.pthreads:
645 # Trigger multi-threaded build.
646 ports_cmd.append("-sUSE_PTHREADS")
647
648 # Pre-build libbz2, libsqlite3, libz, and some system libs.
649 ports_cmd.extend(["-sUSE_ZLIB", "-sUSE_BZIP2", "-sUSE_SQLITE3"])
650 # Multi-threaded sqlite3 has different suffix
651 embuilder_cmd.extend(
652 ["build", "bzip2", "sqlite3-mt" if self.pthreads else "sqlite3", "zlib"]
653 )
654
655 self._run_cmd(embuilder_cmd, cwd=SRCDIR)
656
657 with tempfile.TemporaryDirectory(suffix="-py-emport") as tmpdir:
658 tmppath = pathlib.Path(tmpdir)
659 main_c = tmppath / "main.c"
660 main_js = tmppath / "main.js"
661 with main_c.open("w") as f:
662 f.write("int main(void) { return 0; }\n")
663 args = [
664 os.fspath(main_c),
665 "-o",
666 os.fspath(main_js),
667 ]
668 self._run_cmd(ports_cmd, args, cwd=tmppath)
669
670
671 # native build (build Python)
672 BUILD = BuildProfile(
673 "build",
674 support_level=SupportLevel.working,
675 host=Host.build,
676 )
677
678 _profiles = [
679 BUILD,
680 # wasm32-emscripten
681 BuildProfile(
682 "emscripten-browser",
683 support_level=SupportLevel.supported,
684 host=Host.wasm32_emscripten,
685 target=EmscriptenTarget.browser,
686 dynamic_linking=True,
687 ),
688 BuildProfile(
689 "emscripten-browser-debug",
690 support_level=SupportLevel.working,
691 host=Host.wasm32_emscripten,
692 target=EmscriptenTarget.browser_debug,
693 dynamic_linking=True,
694 ),
695 BuildProfile(
696 "emscripten-node-dl",
697 support_level=SupportLevel.supported,
698 host=Host.wasm32_emscripten,
699 target=EmscriptenTarget.node,
700 dynamic_linking=True,
701 ),
702 BuildProfile(
703 "emscripten-node-dl-debug",
704 support_level=SupportLevel.working,
705 host=Host.wasm32_emscripten,
706 target=EmscriptenTarget.node_debug,
707 dynamic_linking=True,
708 ),
709 BuildProfile(
710 "emscripten-node-pthreads",
711 support_level=SupportLevel.supported,
712 host=Host.wasm32_emscripten,
713 target=EmscriptenTarget.node,
714 pthreads=True,
715 ),
716 BuildProfile(
717 "emscripten-node-pthreads-debug",
718 support_level=SupportLevel.working,
719 host=Host.wasm32_emscripten,
720 target=EmscriptenTarget.node_debug,
721 pthreads=True,
722 ),
723 # Emscripten build with both pthreads and dynamic linking is crashing.
724 BuildProfile(
725 "emscripten-node-dl-pthreads-debug",
726 support_level=SupportLevel.broken,
727 host=Host.wasm32_emscripten,
728 target=EmscriptenTarget.node_debug,
729 dynamic_linking=True,
730 pthreads=True,
731 ),
732 # wasm64-emscripten (requires Emscripten >= 3.1.21)
733 BuildProfile(
734 "wasm64-emscripten-node-debug",
735 support_level=SupportLevel.experimental,
736 host=Host.wasm64_emscripten,
737 target=EmscriptenTarget.node_debug,
738 # MEMORY64 is not compatible with dynamic linking
739 dynamic_linking=False,
740 pthreads=False,
741 ),
742 # wasm32-wasi
743 BuildProfile(
744 "wasi",
745 support_level=SupportLevel.supported,
746 host=Host.wasm32_wasi,
747 ),
748 # no SDK available yet
749 # BuildProfile(
750 # "wasm64-wasi",
751 # support_level=SupportLevel.broken,
752 # host=Host.wasm64_wasi,
753 # ),
754 ]
755
756 PROFILES = {p.name: p for p in _profiles}
757
758 parser = argparse.ArgumentParser(
759 "wasm_build.py",
760 description=__doc__,
761 formatter_class=argparse.RawTextHelpFormatter,
762 )
763
764 parser.add_argument(
765 "--clean",
766 "-c",
767 help="Clean build directories first",
768 action="store_true",
769 )
770
771 parser.add_argument(
772 "--verbose",
773 "-v",
774 help="Verbose logging",
775 action="store_true",
776 )
777
778 parser.add_argument(
779 "--silent",
780 help="Run configure and make in silent mode",
781 action="store_true",
782 )
783
784 parser.add_argument(
785 "--testopts",
786 help=(
787 "Additional test options for 'test' and 'hostrunnertest', e.g. "
788 "--testopts='-v test_os'."
789 ),
790 default=None,
791 )
792
793 # Don't list broken and experimental variants in help
794 platforms_choices = list(p.name for p in _profiles) + ["cleanall"]
795 platforms_help = list(p.name for p in _profiles if p.support_level) + ["cleanall"]
796 parser.add_argument(
797 "platform",
798 metavar="PLATFORM",
799 help=f"Build platform: {', '.join(platforms_help)}",
800 choices=platforms_choices,
801 )
802
803 ops = dict(
804 build="auto build (build 'build' Python, emports, configure, compile)",
805 configure="run ./configure",
806 compile="run 'make all'",
807 pythoninfo="run 'make pythoninfo'",
808 test="run 'make buildbottest TESTOPTS=...' (supports parallel tests)",
809 hostrunnertest="run 'make hostrunnertest TESTOPTS=...'",
810 repl="start interactive REPL / webserver + browser session",
811 clean="run 'make clean'",
812 cleanall="remove all build directories",
813 emports="build Emscripten port with embuilder (only Emscripten)",
814 )
815 ops_help = "\n".join(f"{op:16s} {help}" for op, help in ops.items())
816 parser.add_argument(
817 "ops",
818 metavar="OP",
819 help=f"operation (default: build)\n\n{ops_help}",
820 choices=tuple(ops),
821 default="build",
822 nargs="*",
823 )
824
825
826 def main():
827 args = parser.parse_args()
828 logging.basicConfig(
829 level=logging.INFO if args.verbose else logging.ERROR,
830 format="%(message)s",
831 )
832
833 if args.platform == "cleanall":
834 for builder in PROFILES.values():
835 builder.clean(all=True)
836 parser.exit(0)
837
838 # additional configure and make args
839 cm_args = ("--silent",) if args.silent else ()
840
841 # nargs=* with default quirk
842 if args.ops == "build":
843 args.ops = ["build"]
844
845 builder = PROFILES[args.platform]
846 try:
847 builder.host.platform.check()
848 except ConditionError as e:
849 parser.error(str(e))
850
851 if args.clean:
852 builder.clean(all=False)
853
854 # hack for WASI
855 if builder.host.is_wasi and not SETUP_LOCAL.exists():
856 SETUP_LOCAL.touch()
857
858 # auto-build
859 if "build" in args.ops:
860 # check and create build Python
861 if builder is not BUILD:
862 logger.info("Auto-building 'build' Python.")
863 try:
864 BUILD.host.platform.check()
865 except ConditionError as e:
866 parser.error(str(e))
867 if args.clean:
868 BUILD.clean(all=False)
869 BUILD.run_build(*cm_args)
870 # build Emscripten ports with embuilder
871 if builder.host.is_emscripten and "emports" not in args.ops:
872 builder.build_emports()
873
874 for op in args.ops:
875 logger.info("\n*** %s %s", args.platform, op)
876 if op == "build":
877 builder.run_build(*cm_args)
878 elif op == "configure":
879 builder.run_configure(*cm_args)
880 elif op == "compile":
881 builder.run_make("all", *cm_args)
882 elif op == "pythoninfo":
883 builder.run_pythoninfo(*cm_args)
884 elif op == "repl":
885 if builder.is_browser:
886 builder.run_browser()
887 else:
888 builder.run_py()
889 elif op == "test":
890 builder.run_test("buildbottest", testopts=args.testopts)
891 elif op == "hostrunnertest":
892 builder.run_test("hostrunnertest", testopts=args.testopts)
893 elif op == "clean":
894 builder.clean(all=False)
895 elif op == "cleanall":
896 builder.clean(all=True)
897 elif op == "emports":
898 builder.build_emports(force=args.clean)
899 else:
900 raise ValueError(op)
901
902 print(builder.builddir)
903 parser.exit(0)
904
905
906 if __name__ == "__main__":
907 main()