python (3.11.7)
1 from functools import partial
2 from glob import glob
3 from distutils.util import convert_path
4 import distutils.command.build_py as orig
5 import os
6 import fnmatch
7 import textwrap
8 import io
9 import distutils.errors
10 import itertools
11 import stat
12 import warnings
13 from pathlib import Path
14 from typing import Dict, Iterable, Iterator, List, Optional, Tuple
15
16 from setuptools._deprecation_warning import SetuptoolsDeprecationWarning
17 from setuptools.extern.more_itertools import unique_everseen
18
19
20 def make_writable(target):
21 os.chmod(target, os.stat(target).st_mode | stat.S_IWRITE)
22
23
24 class ESC[4;38;5;81mbuild_py(ESC[4;38;5;149morigESC[4;38;5;149m.ESC[4;38;5;149mbuild_py):
25 """Enhanced 'build_py' command that includes data files with packages
26
27 The data files are specified via a 'package_data' argument to 'setup()'.
28 See 'setuptools.dist.Distribution' for more details.
29
30 Also, this version of the 'build_py' command allows you to specify both
31 'py_modules' and 'packages' in the same setup operation.
32 """
33 editable_mode: bool = False
34 existing_egg_info_dir: Optional[str] = None #: Private API, internal use only.
35
36 def finalize_options(self):
37 orig.build_py.finalize_options(self)
38 self.package_data = self.distribution.package_data
39 self.exclude_package_data = self.distribution.exclude_package_data or {}
40 if 'data_files' in self.__dict__:
41 del self.__dict__['data_files']
42 self.__updated_files = []
43
44 def copy_file(self, infile, outfile, preserve_mode=1, preserve_times=1,
45 link=None, level=1):
46 # Overwrite base class to allow using links
47 if link:
48 infile = str(Path(infile).resolve())
49 outfile = str(Path(outfile).resolve())
50 return super().copy_file(infile, outfile, preserve_mode, preserve_times,
51 link, level)
52
53 def run(self):
54 """Build modules, packages, and copy data files to build directory"""
55 if not (self.py_modules or self.packages) or self.editable_mode:
56 return
57
58 if self.py_modules:
59 self.build_modules()
60
61 if self.packages:
62 self.build_packages()
63 self.build_package_data()
64
65 # Only compile actual .py files, using our base class' idea of what our
66 # output files are.
67 self.byte_compile(orig.build_py.get_outputs(self, include_bytecode=0))
68
69 def __getattr__(self, attr):
70 "lazily compute data files"
71 if attr == 'data_files':
72 self.data_files = self._get_data_files()
73 return self.data_files
74 return orig.build_py.__getattr__(self, attr)
75
76 def build_module(self, module, module_file, package):
77 outfile, copied = orig.build_py.build_module(self, module, module_file, package)
78 if copied:
79 self.__updated_files.append(outfile)
80 return outfile, copied
81
82 def _get_data_files(self):
83 """Generate list of '(package,src_dir,build_dir,filenames)' tuples"""
84 self.analyze_manifest()
85 return list(map(self._get_pkg_data_files, self.packages or ()))
86
87 def get_data_files_without_manifest(self):
88 """
89 Generate list of ``(package,src_dir,build_dir,filenames)`` tuples,
90 but without triggering any attempt to analyze or build the manifest.
91 """
92 # Prevent eventual errors from unset `manifest_files`
93 # (that would otherwise be set by `analyze_manifest`)
94 self.__dict__.setdefault('manifest_files', {})
95 return list(map(self._get_pkg_data_files, self.packages or ()))
96
97 def _get_pkg_data_files(self, package):
98 # Locate package source directory
99 src_dir = self.get_package_dir(package)
100
101 # Compute package build directory
102 build_dir = os.path.join(*([self.build_lib] + package.split('.')))
103
104 # Strip directory from globbed filenames
105 filenames = [
106 os.path.relpath(file, src_dir)
107 for file in self.find_data_files(package, src_dir)
108 ]
109 return package, src_dir, build_dir, filenames
110
111 def find_data_files(self, package, src_dir):
112 """Return filenames for package's data files in 'src_dir'"""
113 patterns = self._get_platform_patterns(
114 self.package_data,
115 package,
116 src_dir,
117 )
118 globs_expanded = map(partial(glob, recursive=True), patterns)
119 # flatten the expanded globs into an iterable of matches
120 globs_matches = itertools.chain.from_iterable(globs_expanded)
121 glob_files = filter(os.path.isfile, globs_matches)
122 files = itertools.chain(
123 self.manifest_files.get(package, []),
124 glob_files,
125 )
126 return self.exclude_data_files(package, src_dir, files)
127
128 def get_outputs(self, include_bytecode=1) -> List[str]:
129 """See :class:`setuptools.commands.build.SubCommand`"""
130 if self.editable_mode:
131 return list(self.get_output_mapping().keys())
132 return super().get_outputs(include_bytecode)
133
134 def get_output_mapping(self) -> Dict[str, str]:
135 """See :class:`setuptools.commands.build.SubCommand`"""
136 mapping = itertools.chain(
137 self._get_package_data_output_mapping(),
138 self._get_module_mapping(),
139 )
140 return dict(sorted(mapping, key=lambda x: x[0]))
141
142 def _get_module_mapping(self) -> Iterator[Tuple[str, str]]:
143 """Iterate over all modules producing (dest, src) pairs."""
144 for (package, module, module_file) in self.find_all_modules():
145 package = package.split('.')
146 filename = self.get_module_outfile(self.build_lib, package, module)
147 yield (filename, module_file)
148
149 def _get_package_data_output_mapping(self) -> Iterator[Tuple[str, str]]:
150 """Iterate over package data producing (dest, src) pairs."""
151 for package, src_dir, build_dir, filenames in self.data_files:
152 for filename in filenames:
153 target = os.path.join(build_dir, filename)
154 srcfile = os.path.join(src_dir, filename)
155 yield (target, srcfile)
156
157 def build_package_data(self):
158 """Copy data files into build directory"""
159 for target, srcfile in self._get_package_data_output_mapping():
160 self.mkpath(os.path.dirname(target))
161 _outf, _copied = self.copy_file(srcfile, target)
162 make_writable(target)
163
164 def analyze_manifest(self):
165 self.manifest_files = mf = {}
166 if not self.distribution.include_package_data:
167 return
168 src_dirs = {}
169 for package in self.packages or ():
170 # Locate package source directory
171 src_dirs[assert_relative(self.get_package_dir(package))] = package
172
173 if (
174 getattr(self, 'existing_egg_info_dir', None)
175 and Path(self.existing_egg_info_dir, "SOURCES.txt").exists()
176 ):
177 egg_info_dir = self.existing_egg_info_dir
178 manifest = Path(egg_info_dir, "SOURCES.txt")
179 files = manifest.read_text(encoding="utf-8").splitlines()
180 else:
181 self.run_command('egg_info')
182 ei_cmd = self.get_finalized_command('egg_info')
183 egg_info_dir = ei_cmd.egg_info
184 files = ei_cmd.filelist.files
185
186 check = _IncludePackageDataAbuse()
187 for path in self._filter_build_files(files, egg_info_dir):
188 d, f = os.path.split(assert_relative(path))
189 prev = None
190 oldf = f
191 while d and d != prev and d not in src_dirs:
192 prev = d
193 d, df = os.path.split(d)
194 f = os.path.join(df, f)
195 if d in src_dirs:
196 if f == oldf:
197 if check.is_module(f):
198 continue # it's a module, not data
199 else:
200 importable = check.importable_subpackage(src_dirs[d], f)
201 if importable:
202 check.warn(importable)
203 mf.setdefault(src_dirs[d], []).append(path)
204
205 def _filter_build_files(self, files: Iterable[str], egg_info: str) -> Iterator[str]:
206 """
207 ``build_meta`` may try to create egg_info outside of the project directory,
208 and this can be problematic for certain plugins (reported in issue #3500).
209
210 Extensions might also include between their sources files created on the
211 ``build_lib`` and ``build_temp`` directories.
212
213 This function should filter this case of invalid files out.
214 """
215 build = self.get_finalized_command("build")
216 build_dirs = (egg_info, self.build_lib, build.build_temp, build.build_base)
217 norm_dirs = [os.path.normpath(p) for p in build_dirs if p]
218
219 for file in files:
220 norm_path = os.path.normpath(file)
221 if not os.path.isabs(file) or all(d not in norm_path for d in norm_dirs):
222 yield file
223
224 def get_data_files(self):
225 pass # Lazily compute data files in _get_data_files() function.
226
227 def check_package(self, package, package_dir):
228 """Check namespace packages' __init__ for declare_namespace"""
229 try:
230 return self.packages_checked[package]
231 except KeyError:
232 pass
233
234 init_py = orig.build_py.check_package(self, package, package_dir)
235 self.packages_checked[package] = init_py
236
237 if not init_py or not self.distribution.namespace_packages:
238 return init_py
239
240 for pkg in self.distribution.namespace_packages:
241 if pkg == package or pkg.startswith(package + '.'):
242 break
243 else:
244 return init_py
245
246 with io.open(init_py, 'rb') as f:
247 contents = f.read()
248 if b'declare_namespace' not in contents:
249 raise distutils.errors.DistutilsError(
250 "Namespace package problem: %s is a namespace package, but "
251 "its\n__init__.py does not call declare_namespace()! Please "
252 'fix it.\n(See the setuptools manual under '
253 '"Namespace Packages" for details.)\n"' % (package,)
254 )
255 return init_py
256
257 def initialize_options(self):
258 self.packages_checked = {}
259 orig.build_py.initialize_options(self)
260 self.editable_mode = False
261 self.existing_egg_info_dir = None
262
263 def get_package_dir(self, package):
264 res = orig.build_py.get_package_dir(self, package)
265 if self.distribution.src_root is not None:
266 return os.path.join(self.distribution.src_root, res)
267 return res
268
269 def exclude_data_files(self, package, src_dir, files):
270 """Filter filenames for package's data files in 'src_dir'"""
271 files = list(files)
272 patterns = self._get_platform_patterns(
273 self.exclude_package_data,
274 package,
275 src_dir,
276 )
277 match_groups = (fnmatch.filter(files, pattern) for pattern in patterns)
278 # flatten the groups of matches into an iterable of matches
279 matches = itertools.chain.from_iterable(match_groups)
280 bad = set(matches)
281 keepers = (fn for fn in files if fn not in bad)
282 # ditch dupes
283 return list(unique_everseen(keepers))
284
285 @staticmethod
286 def _get_platform_patterns(spec, package, src_dir):
287 """
288 yield platform-specific path patterns (suitable for glob
289 or fn_match) from a glob-based spec (such as
290 self.package_data or self.exclude_package_data)
291 matching package in src_dir.
292 """
293 raw_patterns = itertools.chain(
294 spec.get('', []),
295 spec.get(package, []),
296 )
297 return (
298 # Each pattern has to be converted to a platform-specific path
299 os.path.join(src_dir, convert_path(pattern))
300 for pattern in raw_patterns
301 )
302
303
304 def assert_relative(path):
305 if not os.path.isabs(path):
306 return path
307 from distutils.errors import DistutilsSetupError
308
309 msg = (
310 textwrap.dedent(
311 """
312 Error: setup script specifies an absolute path:
313
314 %s
315
316 setup() arguments must *always* be /-separated paths relative to the
317 setup.py directory, *never* absolute paths.
318 """
319 ).lstrip()
320 % path
321 )
322 raise DistutilsSetupError(msg)
323
324
325 class ESC[4;38;5;81m_IncludePackageDataAbuse:
326 """Inform users that package or module is included as 'data file'"""
327
328 MESSAGE = """\
329 Installing {importable!r} as data is deprecated, please list it in `packages`.
330 !!\n\n
331 ############################
332 # Package would be ignored #
333 ############################
334 Python recognizes {importable!r} as an importable package,
335 but it is not listed in the `packages` configuration of setuptools.
336
337 {importable!r} has been automatically added to the distribution only
338 because it may contain data files, but this behavior is likely to change
339 in future versions of setuptools (and therefore is considered deprecated).
340
341 Please make sure that {importable!r} is included as a package by using
342 the `packages` configuration field or the proper discovery methods
343 (for example by using `find_namespace_packages(...)`/`find_namespace:`
344 instead of `find_packages(...)`/`find:`).
345
346 You can read more about "package discovery" and "data files" on setuptools
347 documentation page.
348 \n\n!!
349 """
350
351 def __init__(self):
352 self._already_warned = set()
353
354 def is_module(self, file):
355 return file.endswith(".py") and file[:-len(".py")].isidentifier()
356
357 def importable_subpackage(self, parent, file):
358 pkg = Path(file).parent
359 parts = list(itertools.takewhile(str.isidentifier, pkg.parts))
360 if parts:
361 return ".".join([parent, *parts])
362 return None
363
364 def warn(self, importable):
365 if importable not in self._already_warned:
366 msg = textwrap.dedent(self.MESSAGE).format(importable=importable)
367 warnings.warn(msg, SetuptoolsDeprecationWarning, stacklevel=2)
368 self._already_warned.add(importable)
Browse
Usage
Versions