python (3.11.7)
1 """
2 Load setuptools configuration from ``pyproject.toml`` files.
3
4 **PRIVATE MODULE**: API reserved for setuptools internal usage only.
5 """
6 import logging
7 import os
8 import warnings
9 from contextlib import contextmanager
10 from functools import partial
11 from typing import TYPE_CHECKING, Callable, Dict, Optional, Mapping, Union
12
13 from setuptools.errors import FileError, OptionError
14
15 from . import expand as _expand
16 from ._apply_pyprojecttoml import apply as _apply
17 from ._apply_pyprojecttoml import _PREVIOUSLY_DEFINED, _WouldIgnoreField
18
19 if TYPE_CHECKING:
20 from setuptools.dist import Distribution # noqa
21
22 _Path = Union[str, os.PathLike]
23 _logger = logging.getLogger(__name__)
24
25
26 def load_file(filepath: _Path) -> dict:
27 from setuptools.extern import tomli # type: ignore
28
29 with open(filepath, "rb") as file:
30 return tomli.load(file)
31
32
33 def validate(config: dict, filepath: _Path) -> bool:
34 from . import _validate_pyproject as validator
35
36 trove_classifier = validator.FORMAT_FUNCTIONS.get("trove-classifier")
37 if hasattr(trove_classifier, "_disable_download"):
38 # Improve reproducibility by default. See issue 31 for validate-pyproject.
39 trove_classifier._disable_download() # type: ignore
40
41 try:
42 return validator.validate(config)
43 except validator.ValidationError as ex:
44 summary = f"configuration error: {ex.summary}"
45 if ex.name.strip("`") != "project":
46 # Probably it is just a field missing/misnamed, not worthy the verbosity...
47 _logger.debug(summary)
48 _logger.debug(ex.details)
49
50 error = f"invalid pyproject.toml config: {ex.name}."
51 raise ValueError(f"{error}\n{summary}") from None
52
53
54 def apply_configuration(
55 dist: "Distribution",
56 filepath: _Path,
57 ignore_option_errors=False,
58 ) -> "Distribution":
59 """Apply the configuration from a ``pyproject.toml`` file into an existing
60 distribution object.
61 """
62 config = read_configuration(filepath, True, ignore_option_errors, dist)
63 return _apply(dist, config, filepath)
64
65
66 def read_configuration(
67 filepath: _Path,
68 expand=True,
69 ignore_option_errors=False,
70 dist: Optional["Distribution"] = None,
71 ):
72 """Read given configuration file and returns options from it as a dict.
73
74 :param str|unicode filepath: Path to configuration file in the ``pyproject.toml``
75 format.
76
77 :param bool expand: Whether to expand directives and other computed values
78 (i.e. post-process the given configuration)
79
80 :param bool ignore_option_errors: Whether to silently ignore
81 options, values of which could not be resolved (e.g. due to exceptions
82 in directives such as file:, attr:, etc.).
83 If False exceptions are propagated as expected.
84
85 :param Distribution|None: Distribution object to which the configuration refers.
86 If not given a dummy object will be created and discarded after the
87 configuration is read. This is used for auto-discovery of packages in the case
88 a dynamic configuration (e.g. ``attr`` or ``cmdclass``) is expanded.
89 When ``expand=False`` this object is simply ignored.
90
91 :rtype: dict
92 """
93 filepath = os.path.abspath(filepath)
94
95 if not os.path.isfile(filepath):
96 raise FileError(f"Configuration file {filepath!r} does not exist.")
97
98 asdict = load_file(filepath) or {}
99 project_table = asdict.get("project", {})
100 tool_table = asdict.get("tool", {})
101 setuptools_table = tool_table.get("setuptools", {})
102 if not asdict or not (project_table or setuptools_table):
103 return {} # User is not using pyproject to configure setuptools
104
105 if setuptools_table:
106 # TODO: Remove the following once the feature stabilizes:
107 msg = "Support for `[tool.setuptools]` in `pyproject.toml` is still *beta*."
108 warnings.warn(msg, _BetaConfiguration)
109
110 # There is an overall sense in the community that making include_package_data=True
111 # the default would be an improvement.
112 # `ini2toml` backfills include_package_data=False when nothing is explicitly given,
113 # therefore setting a default here is backwards compatible.
114 orig_setuptools_table = setuptools_table.copy()
115 if dist and getattr(dist, "include_package_data") is not None:
116 setuptools_table.setdefault("include-package-data", dist.include_package_data)
117 else:
118 setuptools_table.setdefault("include-package-data", True)
119 # Persist changes:
120 asdict["tool"] = tool_table
121 tool_table["setuptools"] = setuptools_table
122
123 try:
124 # Don't complain about unrelated errors (e.g. tools not using the "tool" table)
125 subset = {"project": project_table, "tool": {"setuptools": setuptools_table}}
126 validate(subset, filepath)
127 except Exception as ex:
128 # TODO: Remove the following once the feature stabilizes:
129 if _skip_bad_config(project_table, orig_setuptools_table, dist):
130 return {}
131 # TODO: After the previous statement is removed the try/except can be replaced
132 # by the _ignore_errors context manager.
133 if ignore_option_errors:
134 _logger.debug(f"ignored error: {ex.__class__.__name__} - {ex}")
135 else:
136 raise # re-raise exception
137
138 if expand:
139 root_dir = os.path.dirname(filepath)
140 return expand_configuration(asdict, root_dir, ignore_option_errors, dist)
141
142 return asdict
143
144
145 def _skip_bad_config(
146 project_cfg: dict, setuptools_cfg: dict, dist: Optional["Distribution"]
147 ) -> bool:
148 """Be temporarily forgiving with invalid ``pyproject.toml``"""
149 # See pypa/setuptools#3199 and pypa/cibuildwheel#1064
150
151 if dist is None or (
152 dist.metadata.name is None
153 and dist.metadata.version is None
154 and dist.install_requires is None
155 ):
156 # It seems that the build is not getting any configuration from other places
157 return False
158
159 if setuptools_cfg:
160 # If `[tool.setuptools]` is set, then `pyproject.toml` config is intentional
161 return False
162
163 given_config = set(project_cfg.keys())
164 popular_subset = {"name", "version", "python_requires", "requires-python"}
165 if given_config <= popular_subset:
166 # It seems that the docs in cibuildtool has been inadvertently encouraging users
167 # to create `pyproject.toml` files that are not compliant with the standards.
168 # Let's be forgiving for the time being.
169 warnings.warn(_InvalidFile.message(), _InvalidFile, stacklevel=2)
170 return True
171
172 return False
173
174
175 def expand_configuration(
176 config: dict,
177 root_dir: Optional[_Path] = None,
178 ignore_option_errors: bool = False,
179 dist: Optional["Distribution"] = None,
180 ) -> dict:
181 """Given a configuration with unresolved fields (e.g. dynamic, cmdclass, ...)
182 find their final values.
183
184 :param dict config: Dict containing the configuration for the distribution
185 :param str root_dir: Top-level directory for the distribution/project
186 (the same directory where ``pyproject.toml`` is place)
187 :param bool ignore_option_errors: see :func:`read_configuration`
188 :param Distribution|None: Distribution object to which the configuration refers.
189 If not given a dummy object will be created and discarded after the
190 configuration is read. Used in the case a dynamic configuration
191 (e.g. ``attr`` or ``cmdclass``).
192
193 :rtype: dict
194 """
195 return _ConfigExpander(config, root_dir, ignore_option_errors, dist).expand()
196
197
198 class ESC[4;38;5;81m_ConfigExpander:
199 def __init__(
200 self,
201 config: dict,
202 root_dir: Optional[_Path] = None,
203 ignore_option_errors: bool = False,
204 dist: Optional["Distribution"] = None,
205 ):
206 self.config = config
207 self.root_dir = root_dir or os.getcwd()
208 self.project_cfg = config.get("project", {})
209 self.dynamic = self.project_cfg.get("dynamic", [])
210 self.setuptools_cfg = config.get("tool", {}).get("setuptools", {})
211 self.dynamic_cfg = self.setuptools_cfg.get("dynamic", {})
212 self.ignore_option_errors = ignore_option_errors
213 self._dist = dist
214
215 def _ensure_dist(self) -> "Distribution":
216 from setuptools.dist import Distribution
217
218 attrs = {"src_root": self.root_dir, "name": self.project_cfg.get("name", None)}
219 return self._dist or Distribution(attrs)
220
221 def _process_field(self, container: dict, field: str, fn: Callable):
222 if field in container:
223 with _ignore_errors(self.ignore_option_errors):
224 container[field] = fn(container[field])
225
226 def _canonic_package_data(self, field="package-data"):
227 package_data = self.setuptools_cfg.get(field, {})
228 return _expand.canonic_package_data(package_data)
229
230 def expand(self):
231 self._expand_packages()
232 self._canonic_package_data()
233 self._canonic_package_data("exclude-package-data")
234
235 # A distribution object is required for discovering the correct package_dir
236 dist = self._ensure_dist()
237 ctx = _EnsurePackagesDiscovered(dist, self.project_cfg, self.setuptools_cfg)
238 with ctx as ensure_discovered:
239 package_dir = ensure_discovered.package_dir
240 self._expand_data_files()
241 self._expand_cmdclass(package_dir)
242 self._expand_all_dynamic(dist, package_dir)
243
244 return self.config
245
246 def _expand_packages(self):
247 packages = self.setuptools_cfg.get("packages")
248 if packages is None or isinstance(packages, (list, tuple)):
249 return
250
251 find = packages.get("find")
252 if isinstance(find, dict):
253 find["root_dir"] = self.root_dir
254 find["fill_package_dir"] = self.setuptools_cfg.setdefault("package-dir", {})
255 with _ignore_errors(self.ignore_option_errors):
256 self.setuptools_cfg["packages"] = _expand.find_packages(**find)
257
258 def _expand_data_files(self):
259 data_files = partial(_expand.canonic_data_files, root_dir=self.root_dir)
260 self._process_field(self.setuptools_cfg, "data-files", data_files)
261
262 def _expand_cmdclass(self, package_dir: Mapping[str, str]):
263 root_dir = self.root_dir
264 cmdclass = partial(_expand.cmdclass, package_dir=package_dir, root_dir=root_dir)
265 self._process_field(self.setuptools_cfg, "cmdclass", cmdclass)
266
267 def _expand_all_dynamic(self, dist: "Distribution", package_dir: Mapping[str, str]):
268 special = ( # need special handling
269 "version",
270 "readme",
271 "entry-points",
272 "scripts",
273 "gui-scripts",
274 "classifiers",
275 "dependencies",
276 "optional-dependencies",
277 )
278 # `_obtain` functions are assumed to raise appropriate exceptions/warnings.
279 obtained_dynamic = {
280 field: self._obtain(dist, field, package_dir)
281 for field in self.dynamic
282 if field not in special
283 }
284 obtained_dynamic.update(
285 self._obtain_entry_points(dist, package_dir) or {},
286 version=self._obtain_version(dist, package_dir),
287 readme=self._obtain_readme(dist),
288 classifiers=self._obtain_classifiers(dist),
289 dependencies=self._obtain_dependencies(dist),
290 optional_dependencies=self._obtain_optional_dependencies(dist),
291 )
292 # `None` indicates there is nothing in `tool.setuptools.dynamic` but the value
293 # might have already been set by setup.py/extensions, so avoid overwriting.
294 updates = {k: v for k, v in obtained_dynamic.items() if v is not None}
295 self.project_cfg.update(updates)
296
297 def _ensure_previously_set(self, dist: "Distribution", field: str):
298 previous = _PREVIOUSLY_DEFINED[field](dist)
299 if previous is None and not self.ignore_option_errors:
300 msg = (
301 f"No configuration found for dynamic {field!r}.\n"
302 "Some dynamic fields need to be specified via `tool.setuptools.dynamic`"
303 "\nothers must be specified via the equivalent attribute in `setup.py`."
304 )
305 raise OptionError(msg)
306
307 def _expand_directive(
308 self, specifier: str, directive, package_dir: Mapping[str, str]
309 ):
310 with _ignore_errors(self.ignore_option_errors):
311 root_dir = self.root_dir
312 if "file" in directive:
313 return _expand.read_files(directive["file"], root_dir)
314 if "attr" in directive:
315 return _expand.read_attr(directive["attr"], package_dir, root_dir)
316 raise ValueError(f"invalid `{specifier}`: {directive!r}")
317 return None
318
319 def _obtain(self, dist: "Distribution", field: str, package_dir: Mapping[str, str]):
320 if field in self.dynamic_cfg:
321 return self._expand_directive(
322 f"tool.setuptools.dynamic.{field}",
323 self.dynamic_cfg[field],
324 package_dir,
325 )
326 self._ensure_previously_set(dist, field)
327 return None
328
329 def _obtain_version(self, dist: "Distribution", package_dir: Mapping[str, str]):
330 # Since plugins can set version, let's silently skip if it cannot be obtained
331 if "version" in self.dynamic and "version" in self.dynamic_cfg:
332 return _expand.version(self._obtain(dist, "version", package_dir))
333 return None
334
335 def _obtain_readme(self, dist: "Distribution") -> Optional[Dict[str, str]]:
336 if "readme" not in self.dynamic:
337 return None
338
339 dynamic_cfg = self.dynamic_cfg
340 if "readme" in dynamic_cfg:
341 return {
342 "text": self._obtain(dist, "readme", {}),
343 "content-type": dynamic_cfg["readme"].get("content-type", "text/x-rst"),
344 }
345
346 self._ensure_previously_set(dist, "readme")
347 return None
348
349 def _obtain_entry_points(
350 self, dist: "Distribution", package_dir: Mapping[str, str]
351 ) -> Optional[Dict[str, dict]]:
352 fields = ("entry-points", "scripts", "gui-scripts")
353 if not any(field in self.dynamic for field in fields):
354 return None
355
356 text = self._obtain(dist, "entry-points", package_dir)
357 if text is None:
358 return None
359
360 groups = _expand.entry_points(text)
361 expanded = {"entry-points": groups}
362
363 def _set_scripts(field: str, group: str):
364 if group in groups:
365 value = groups.pop(group)
366 if field not in self.dynamic:
367 msg = _WouldIgnoreField.message(field, value)
368 warnings.warn(msg, _WouldIgnoreField)
369 # TODO: Don't set field when support for pyproject.toml stabilizes
370 # instead raise an error as specified in PEP 621
371 expanded[field] = value
372
373 _set_scripts("scripts", "console_scripts")
374 _set_scripts("gui-scripts", "gui_scripts")
375
376 return expanded
377
378 def _obtain_classifiers(self, dist: "Distribution"):
379 if "classifiers" in self.dynamic:
380 value = self._obtain(dist, "classifiers", {})
381 if value:
382 return value.splitlines()
383 return None
384
385 def _obtain_dependencies(self, dist: "Distribution"):
386 if "dependencies" in self.dynamic:
387 value = self._obtain(dist, "dependencies", {})
388 if value:
389 return _parse_requirements_list(value)
390 return None
391
392 def _obtain_optional_dependencies(self, dist: "Distribution"):
393 if "optional-dependencies" not in self.dynamic:
394 return None
395 if "optional-dependencies" in self.dynamic_cfg:
396 optional_dependencies_map = self.dynamic_cfg["optional-dependencies"]
397 assert isinstance(optional_dependencies_map, dict)
398 return {
399 group: _parse_requirements_list(self._expand_directive(
400 f"tool.setuptools.dynamic.optional-dependencies.{group}",
401 directive,
402 {},
403 ))
404 for group, directive in optional_dependencies_map.items()
405 }
406 self._ensure_previously_set(dist, "optional-dependencies")
407 return None
408
409
410 def _parse_requirements_list(value):
411 return [
412 line
413 for line in value.splitlines()
414 if line.strip() and not line.strip().startswith("#")
415 ]
416
417
418 @contextmanager
419 def _ignore_errors(ignore_option_errors: bool):
420 if not ignore_option_errors:
421 yield
422 return
423
424 try:
425 yield
426 except Exception as ex:
427 _logger.debug(f"ignored error: {ex.__class__.__name__} - {ex}")
428
429
430 class ESC[4;38;5;81m_EnsurePackagesDiscovered(ESC[4;38;5;149m_expandESC[4;38;5;149m.ESC[4;38;5;149mEnsurePackagesDiscovered):
431 def __init__(
432 self, distribution: "Distribution", project_cfg: dict, setuptools_cfg: dict
433 ):
434 super().__init__(distribution)
435 self._project_cfg = project_cfg
436 self._setuptools_cfg = setuptools_cfg
437
438 def __enter__(self):
439 """When entering the context, the values of ``packages``, ``py_modules`` and
440 ``package_dir`` that are missing in ``dist`` are copied from ``setuptools_cfg``.
441 """
442 dist, cfg = self._dist, self._setuptools_cfg
443 package_dir: Dict[str, str] = cfg.setdefault("package-dir", {})
444 package_dir.update(dist.package_dir or {})
445 dist.package_dir = package_dir # needs to be the same object
446
447 dist.set_defaults._ignore_ext_modules() # pyproject.toml-specific behaviour
448
449 # Set `name`, `py_modules` and `packages` in dist to short-circuit
450 # auto-discovery, but avoid overwriting empty lists purposefully set by users.
451 if dist.metadata.name is None:
452 dist.metadata.name = self._project_cfg.get("name")
453 if dist.py_modules is None:
454 dist.py_modules = cfg.get("py-modules")
455 if dist.packages is None:
456 dist.packages = cfg.get("packages")
457
458 return super().__enter__()
459
460 def __exit__(self, exc_type, exc_value, traceback):
461 """When exiting the context, if values of ``packages``, ``py_modules`` and
462 ``package_dir`` are missing in ``setuptools_cfg``, copy from ``dist``.
463 """
464 # If anything was discovered set them back, so they count in the final config.
465 self._setuptools_cfg.setdefault("packages", self._dist.packages)
466 self._setuptools_cfg.setdefault("py-modules", self._dist.py_modules)
467 return super().__exit__(exc_type, exc_value, traceback)
468
469
470 class ESC[4;38;5;81m_BetaConfiguration(ESC[4;38;5;149mUserWarning):
471 """Explicitly inform users that some `pyproject.toml` configuration is *beta*"""
472
473
474 class ESC[4;38;5;81m_InvalidFile(ESC[4;38;5;149mUserWarning):
475 """The given `pyproject.toml` file is invalid and would be ignored.
476 !!\n\n
477 ############################
478 # Invalid `pyproject.toml` #
479 ############################
480
481 Any configurations in `pyproject.toml` will be ignored.
482 Please note that future releases of setuptools will halt the build process
483 if an invalid file is given.
484
485 To prevent setuptools from considering `pyproject.toml` please
486 DO NOT include the `[project]` or `[tool.setuptools]` tables in your file.
487 \n\n!!
488 """
489
490 @classmethod
491 def message(cls):
492 from inspect import cleandoc
493 return cleandoc(cls.__doc__)
Browse
Usage
Versions