python (3.11.7)
1 """distutils.filelist
2
3 Provides the FileList class, used for poking about the filesystem
4 and building lists of files.
5 """
6
7 import os
8 import re
9 import fnmatch
10 import functools
11
12 from distutils.util import convert_path
13 from distutils.errors import DistutilsTemplateError, DistutilsInternalError
14 from distutils import log
15
16
17 class ESC[4;38;5;81mFileList:
18 """A list of files built by on exploring the filesystem and filtered by
19 applying various patterns to what we find there.
20
21 Instance attributes:
22 dir
23 directory from which files will be taken -- only used if
24 'allfiles' not supplied to constructor
25 files
26 list of filenames currently being built/filtered/manipulated
27 allfiles
28 complete list of files under consideration (ie. without any
29 filtering applied)
30 """
31
32 def __init__(self, warn=None, debug_print=None):
33 # ignore argument to FileList, but keep them for backwards
34 # compatibility
35 self.allfiles = None
36 self.files = []
37
38 def set_allfiles(self, allfiles):
39 self.allfiles = allfiles
40
41 def findall(self, dir=os.curdir):
42 self.allfiles = findall(dir)
43
44 def debug_print(self, msg):
45 """Print 'msg' to stdout if the global DEBUG (taken from the
46 DISTUTILS_DEBUG environment variable) flag is true.
47 """
48 from distutils.debug import DEBUG
49
50 if DEBUG:
51 print(msg)
52
53 # Collection methods
54
55 def append(self, item):
56 self.files.append(item)
57
58 def extend(self, items):
59 self.files.extend(items)
60
61 def sort(self):
62 # Not a strict lexical sort!
63 sortable_files = sorted(map(os.path.split, self.files))
64 self.files = []
65 for sort_tuple in sortable_files:
66 self.files.append(os.path.join(*sort_tuple))
67
68 # Other miscellaneous utility methods
69
70 def remove_duplicates(self):
71 # Assumes list has been sorted!
72 for i in range(len(self.files) - 1, 0, -1):
73 if self.files[i] == self.files[i - 1]:
74 del self.files[i]
75
76 # "File template" methods
77
78 def _parse_template_line(self, line):
79 words = line.split()
80 action = words[0]
81
82 patterns = dir = dir_pattern = None
83
84 if action in ('include', 'exclude', 'global-include', 'global-exclude'):
85 if len(words) < 2:
86 raise DistutilsTemplateError(
87 "'%s' expects <pattern1> <pattern2> ..." % action
88 )
89 patterns = [convert_path(w) for w in words[1:]]
90 elif action in ('recursive-include', 'recursive-exclude'):
91 if len(words) < 3:
92 raise DistutilsTemplateError(
93 "'%s' expects <dir> <pattern1> <pattern2> ..." % action
94 )
95 dir = convert_path(words[1])
96 patterns = [convert_path(w) for w in words[2:]]
97 elif action in ('graft', 'prune'):
98 if len(words) != 2:
99 raise DistutilsTemplateError(
100 "'%s' expects a single <dir_pattern>" % action
101 )
102 dir_pattern = convert_path(words[1])
103 else:
104 raise DistutilsTemplateError("unknown action '%s'" % action)
105
106 return (action, patterns, dir, dir_pattern)
107
108 def process_template_line(self, line): # noqa: C901
109 # Parse the line: split it up, make sure the right number of words
110 # is there, and return the relevant words. 'action' is always
111 # defined: it's the first word of the line. Which of the other
112 # three are defined depends on the action; it'll be either
113 # patterns, (dir and patterns), or (dir_pattern).
114 (action, patterns, dir, dir_pattern) = self._parse_template_line(line)
115
116 # OK, now we know that the action is valid and we have the
117 # right number of words on the line for that action -- so we
118 # can proceed with minimal error-checking.
119 if action == 'include':
120 self.debug_print("include " + ' '.join(patterns))
121 for pattern in patterns:
122 if not self.include_pattern(pattern, anchor=1):
123 log.warn("warning: no files found matching '%s'", pattern)
124
125 elif action == 'exclude':
126 self.debug_print("exclude " + ' '.join(patterns))
127 for pattern in patterns:
128 if not self.exclude_pattern(pattern, anchor=1):
129 log.warn(
130 (
131 "warning: no previously-included files "
132 "found matching '%s'"
133 ),
134 pattern,
135 )
136
137 elif action == 'global-include':
138 self.debug_print("global-include " + ' '.join(patterns))
139 for pattern in patterns:
140 if not self.include_pattern(pattern, anchor=0):
141 log.warn(
142 (
143 "warning: no files found matching '%s' "
144 "anywhere in distribution"
145 ),
146 pattern,
147 )
148
149 elif action == 'global-exclude':
150 self.debug_print("global-exclude " + ' '.join(patterns))
151 for pattern in patterns:
152 if not self.exclude_pattern(pattern, anchor=0):
153 log.warn(
154 (
155 "warning: no previously-included files matching "
156 "'%s' found anywhere in distribution"
157 ),
158 pattern,
159 )
160
161 elif action == 'recursive-include':
162 self.debug_print("recursive-include {} {}".format(dir, ' '.join(patterns)))
163 for pattern in patterns:
164 if not self.include_pattern(pattern, prefix=dir):
165 msg = (
166 "warning: no files found matching '%s' " "under directory '%s'"
167 )
168 log.warn(msg, pattern, dir)
169
170 elif action == 'recursive-exclude':
171 self.debug_print("recursive-exclude {} {}".format(dir, ' '.join(patterns)))
172 for pattern in patterns:
173 if not self.exclude_pattern(pattern, prefix=dir):
174 log.warn(
175 (
176 "warning: no previously-included files matching "
177 "'%s' found under directory '%s'"
178 ),
179 pattern,
180 dir,
181 )
182
183 elif action == 'graft':
184 self.debug_print("graft " + dir_pattern)
185 if not self.include_pattern(None, prefix=dir_pattern):
186 log.warn("warning: no directories found matching '%s'", dir_pattern)
187
188 elif action == 'prune':
189 self.debug_print("prune " + dir_pattern)
190 if not self.exclude_pattern(None, prefix=dir_pattern):
191 log.warn(
192 ("no previously-included directories found " "matching '%s'"),
193 dir_pattern,
194 )
195 else:
196 raise DistutilsInternalError(
197 "this cannot happen: invalid action '%s'" % action
198 )
199
200 # Filtering/selection methods
201
202 def include_pattern(self, pattern, anchor=1, prefix=None, is_regex=0):
203 """Select strings (presumably filenames) from 'self.files' that
204 match 'pattern', a Unix-style wildcard (glob) pattern. Patterns
205 are not quite the same as implemented by the 'fnmatch' module: '*'
206 and '?' match non-special characters, where "special" is platform-
207 dependent: slash on Unix; colon, slash, and backslash on
208 DOS/Windows; and colon on Mac OS.
209
210 If 'anchor' is true (the default), then the pattern match is more
211 stringent: "*.py" will match "foo.py" but not "foo/bar.py". If
212 'anchor' is false, both of these will match.
213
214 If 'prefix' is supplied, then only filenames starting with 'prefix'
215 (itself a pattern) and ending with 'pattern', with anything in between
216 them, will match. 'anchor' is ignored in this case.
217
218 If 'is_regex' is true, 'anchor' and 'prefix' are ignored, and
219 'pattern' is assumed to be either a string containing a regex or a
220 regex object -- no translation is done, the regex is just compiled
221 and used as-is.
222
223 Selected strings will be added to self.files.
224
225 Return True if files are found, False otherwise.
226 """
227 # XXX docstring lying about what the special chars are?
228 files_found = False
229 pattern_re = translate_pattern(pattern, anchor, prefix, is_regex)
230 self.debug_print("include_pattern: applying regex r'%s'" % pattern_re.pattern)
231
232 # delayed loading of allfiles list
233 if self.allfiles is None:
234 self.findall()
235
236 for name in self.allfiles:
237 if pattern_re.search(name):
238 self.debug_print(" adding " + name)
239 self.files.append(name)
240 files_found = True
241 return files_found
242
243 def exclude_pattern(self, pattern, anchor=1, prefix=None, is_regex=0):
244 """Remove strings (presumably filenames) from 'files' that match
245 'pattern'. Other parameters are the same as for
246 'include_pattern()', above.
247 The list 'self.files' is modified in place.
248 Return True if files are found, False otherwise.
249 """
250 files_found = False
251 pattern_re = translate_pattern(pattern, anchor, prefix, is_regex)
252 self.debug_print("exclude_pattern: applying regex r'%s'" % pattern_re.pattern)
253 for i in range(len(self.files) - 1, -1, -1):
254 if pattern_re.search(self.files[i]):
255 self.debug_print(" removing " + self.files[i])
256 del self.files[i]
257 files_found = True
258 return files_found
259
260
261 # Utility functions
262
263
264 def _find_all_simple(path):
265 """
266 Find all files under 'path'
267 """
268 all_unique = _UniqueDirs.filter(os.walk(path, followlinks=True))
269 results = (
270 os.path.join(base, file) for base, dirs, files in all_unique for file in files
271 )
272 return filter(os.path.isfile, results)
273
274
275 class ESC[4;38;5;81m_UniqueDirs(ESC[4;38;5;149mset):
276 """
277 Exclude previously-seen dirs from walk results,
278 avoiding infinite recursion.
279 Ref https://bugs.python.org/issue44497.
280 """
281
282 def __call__(self, walk_item):
283 """
284 Given an item from an os.walk result, determine
285 if the item represents a unique dir for this instance
286 and if not, prevent further traversal.
287 """
288 base, dirs, files = walk_item
289 stat = os.stat(base)
290 candidate = stat.st_dev, stat.st_ino
291 found = candidate in self
292 if found:
293 del dirs[:]
294 self.add(candidate)
295 return not found
296
297 @classmethod
298 def filter(cls, items):
299 return filter(cls(), items)
300
301
302 def findall(dir=os.curdir):
303 """
304 Find all files under 'dir' and return the list of full filenames.
305 Unless dir is '.', return full filenames with dir prepended.
306 """
307 files = _find_all_simple(dir)
308 if dir == os.curdir:
309 make_rel = functools.partial(os.path.relpath, start=dir)
310 files = map(make_rel, files)
311 return list(files)
312
313
314 def glob_to_re(pattern):
315 """Translate a shell-like glob pattern to a regular expression; return
316 a string containing the regex. Differs from 'fnmatch.translate()' in
317 that '*' does not match "special characters" (which are
318 platform-specific).
319 """
320 pattern_re = fnmatch.translate(pattern)
321
322 # '?' and '*' in the glob pattern become '.' and '.*' in the RE, which
323 # IMHO is wrong -- '?' and '*' aren't supposed to match slash in Unix,
324 # and by extension they shouldn't match such "special characters" under
325 # any OS. So change all non-escaped dots in the RE to match any
326 # character except the special characters (currently: just os.sep).
327 sep = os.sep
328 if os.sep == '\\':
329 # we're using a regex to manipulate a regex, so we need
330 # to escape the backslash twice
331 sep = r'\\\\'
332 escaped = r'\1[^%s]' % sep
333 pattern_re = re.sub(r'((?<!\\)(\\\\)*)\.', escaped, pattern_re)
334 return pattern_re
335
336
337 def translate_pattern(pattern, anchor=1, prefix=None, is_regex=0):
338 """Translate a shell-like wildcard pattern to a compiled regular
339 expression. Return the compiled regex. If 'is_regex' true,
340 then 'pattern' is directly compiled to a regex (if it's a string)
341 or just returned as-is (assumes it's a regex object).
342 """
343 if is_regex:
344 if isinstance(pattern, str):
345 return re.compile(pattern)
346 else:
347 return pattern
348
349 # ditch start and end characters
350 start, _, end = glob_to_re('_').partition('_')
351
352 if pattern:
353 pattern_re = glob_to_re(pattern)
354 assert pattern_re.startswith(start) and pattern_re.endswith(end)
355 else:
356 pattern_re = ''
357
358 if prefix is not None:
359 prefix_re = glob_to_re(prefix)
360 assert prefix_re.startswith(start) and prefix_re.endswith(end)
361 prefix_re = prefix_re[len(start) : len(prefix_re) - len(end)]
362 sep = os.sep
363 if os.sep == '\\':
364 sep = r'\\'
365 pattern_re = pattern_re[len(start) : len(pattern_re) - len(end)]
366 pattern_re = r'{}\A{}{}.*{}{}'.format(start, prefix_re, sep, pattern_re, end)
367 else: # no prefix -- respect anchor flag
368 if anchor:
369 pattern_re = r'{}\A{}'.format(start, pattern_re[len(start) :])
370
371 return re.compile(pattern_re)
Browse
Usage
Versions