1 """Core implementation of path-based import.
2
3 This module is NOT meant to be directly imported! It has been designed such
4 that it can be bootstrapped into Python as the implementation of import. As
5 such it requires the injection of specific modules and attributes in order to
6 work. One should use importlib as the public-facing version of this module.
7
8 """
9 # IMPORTANT: Whenever making changes to this module, be sure to run a top-level
10 # `make regen-importlib` followed by `make` in order to get the frozen version
11 # of the module updated. Not doing so will result in the Makefile to fail for
12 # all others who don't have a ./python around to freeze the module in the early
13 # stages of compilation.
14 #
15
16 # See importlib._setup() for what is injected into the global namespace.
17
18 # When editing this code be aware that code executed at import time CANNOT
19 # reference any injected objects! This includes not only global code but also
20 # anything specified at the class level.
21
22 # Module injected manually by _set_bootstrap_module()
23 _bootstrap = None
24
25 # Import builtin modules
26 import _imp
27 import _io
28 import sys
29 import _warnings
30 import marshal
31
32
33 _MS_WINDOWS = (sys.platform == 'win32')
34 if _MS_WINDOWS:
35 import nt as _os
36 import winreg
37 else:
38 import posix as _os
39
40
41 if _MS_WINDOWS:
42 path_separators = ['\\', '/']
43 else:
44 path_separators = ['/']
45 # Assumption made in _path_join()
46 assert all(len(sep) == 1 for sep in path_separators)
47 path_sep = path_separators[0]
48 path_sep_tuple = tuple(path_separators)
49 path_separators = ''.join(path_separators)
50 _pathseps_with_colon = {f':{s}' for s in path_separators}
51
52
53 # Bootstrap-related code ######################################################
54 _CASE_INSENSITIVE_PLATFORMS_STR_KEY = 'win',
55 _CASE_INSENSITIVE_PLATFORMS_BYTES_KEY = 'cygwin', 'darwin'
56 _CASE_INSENSITIVE_PLATFORMS = (_CASE_INSENSITIVE_PLATFORMS_BYTES_KEY
57 + _CASE_INSENSITIVE_PLATFORMS_STR_KEY)
58
59
60 def _make_relax_case():
61 if sys.platform.startswith(_CASE_INSENSITIVE_PLATFORMS):
62 if sys.platform.startswith(_CASE_INSENSITIVE_PLATFORMS_STR_KEY):
63 key = 'PYTHONCASEOK'
64 else:
65 key = b'PYTHONCASEOK'
66
67 def _relax_case():
68 """True if filenames must be checked case-insensitively and ignore environment flags are not set."""
69 return not sys.flags.ignore_environment and key in _os.environ
70 else:
71 def _relax_case():
72 """True if filenames must be checked case-insensitively."""
73 return False
74 return _relax_case
75
76 _relax_case = _make_relax_case()
77
78
79 def _pack_uint32(x):
80 """Convert a 32-bit integer to little-endian."""
81 return (int(x) & 0xFFFFFFFF).to_bytes(4, 'little')
82
83
84 def _unpack_uint32(data):
85 """Convert 4 bytes in little-endian to an integer."""
86 assert len(data) == 4
87 return int.from_bytes(data, 'little')
88
89 def _unpack_uint16(data):
90 """Convert 2 bytes in little-endian to an integer."""
91 assert len(data) == 2
92 return int.from_bytes(data, 'little')
93
94
95 if _MS_WINDOWS:
96 def _path_join(*path_parts):
97 """Replacement for os.path.join()."""
98 if not path_parts:
99 return ""
100 if len(path_parts) == 1:
101 return path_parts[0]
102 root = ""
103 path = []
104 for new_root, tail in map(_os._path_splitroot, path_parts):
105 if new_root.startswith(path_sep_tuple) or new_root.endswith(path_sep_tuple):
106 root = new_root.rstrip(path_separators) or root
107 path = [path_sep + tail]
108 elif new_root.endswith(':'):
109 if root.casefold() != new_root.casefold():
110 # Drive relative paths have to be resolved by the OS, so we reset the
111 # tail but do not add a path_sep prefix.
112 root = new_root
113 path = [tail]
114 else:
115 path.append(tail)
116 else:
117 root = new_root or root
118 path.append(tail)
119 path = [p.rstrip(path_separators) for p in path if p]
120 if len(path) == 1 and not path[0]:
121 # Avoid losing the root's trailing separator when joining with nothing
122 return root + path_sep
123 return root + path_sep.join(path)
124
125 else:
126 def _path_join(*path_parts):
127 """Replacement for os.path.join()."""
128 return path_sep.join([part.rstrip(path_separators)
129 for part in path_parts if part])
130
131
132 def _path_split(path):
133 """Replacement for os.path.split()."""
134 i = max(path.rfind(p) for p in path_separators)
135 if i < 0:
136 return '', path
137 return path[:i], path[i + 1:]
138
139
140 def _path_stat(path):
141 """Stat the path.
142
143 Made a separate function to make it easier to override in experiments
144 (e.g. cache stat results).
145
146 """
147 return _os.stat(path)
148
149
150 def _path_is_mode_type(path, mode):
151 """Test whether the path is the specified mode type."""
152 try:
153 stat_info = _path_stat(path)
154 except OSError:
155 return False
156 return (stat_info.st_mode & 0o170000) == mode
157
158
159 def _path_isfile(path):
160 """Replacement for os.path.isfile."""
161 return _path_is_mode_type(path, 0o100000)
162
163
164 def _path_isdir(path):
165 """Replacement for os.path.isdir."""
166 if not path:
167 path = _os.getcwd()
168 return _path_is_mode_type(path, 0o040000)
169
170
171 if _MS_WINDOWS:
172 def _path_isabs(path):
173 """Replacement for os.path.isabs."""
174 if not path:
175 return False
176 root = _os._path_splitroot(path)[0].replace('/', '\\')
177 return len(root) > 1 and (root.startswith('\\\\') or root.endswith('\\'))
178
179 else:
180 def _path_isabs(path):
181 """Replacement for os.path.isabs."""
182 return path.startswith(path_separators)
183
184
185 def _write_atomic(path, data, mode=0o666):
186 """Best-effort function to write data to a path atomically.
187 Be prepared to handle a FileExistsError if concurrent writing of the
188 temporary file is attempted."""
189 # id() is used to generate a pseudo-random filename.
190 path_tmp = '{}.{}'.format(path, id(path))
191 fd = _os.open(path_tmp,
192 _os.O_EXCL | _os.O_CREAT | _os.O_WRONLY, mode & 0o666)
193 try:
194 # We first write data to a temporary file, and then use os.replace() to
195 # perform an atomic rename.
196 with _io.FileIO(fd, 'wb') as file:
197 file.write(data)
198 _os.replace(path_tmp, path)
199 except OSError:
200 try:
201 _os.unlink(path_tmp)
202 except OSError:
203 pass
204 raise
205
206
207 _code_type = type(_write_atomic.__code__)
208
209
210 # Finder/loader utility code ###############################################
211
212 # Magic word to reject .pyc files generated by other Python versions.
213 # It should change for each incompatible change to the bytecode.
214 #
215 # The value of CR and LF is incorporated so if you ever read or write
216 # a .pyc file in text mode the magic number will be wrong; also, the
217 # Apple MPW compiler swaps their values, botching string constants.
218 #
219 # There were a variety of old schemes for setting the magic number.
220 # The current working scheme is to increment the previous value by
221 # 10.
222 #
223 # Starting with the adoption of PEP 3147 in Python 3.2, every bump in magic
224 # number also includes a new "magic tag", i.e. a human readable string used
225 # to represent the magic number in __pycache__ directories. When you change
226 # the magic number, you must also set a new unique magic tag. Generally this
227 # can be named after the Python major version of the magic number bump, but
228 # it can really be anything, as long as it's different than anything else
229 # that's come before. The tags are included in the following table, starting
230 # with Python 3.2a0.
231 #
232 # Known values:
233 # Python 1.5: 20121
234 # Python 1.5.1: 20121
235 # Python 1.5.2: 20121
236 # Python 1.6: 50428
237 # Python 2.0: 50823
238 # Python 2.0.1: 50823
239 # Python 2.1: 60202
240 # Python 2.1.1: 60202
241 # Python 2.1.2: 60202
242 # Python 2.2: 60717
243 # Python 2.3a0: 62011
244 # Python 2.3a0: 62021
245 # Python 2.3a0: 62011 (!)
246 # Python 2.4a0: 62041
247 # Python 2.4a3: 62051
248 # Python 2.4b1: 62061
249 # Python 2.5a0: 62071
250 # Python 2.5a0: 62081 (ast-branch)
251 # Python 2.5a0: 62091 (with)
252 # Python 2.5a0: 62092 (changed WITH_CLEANUP opcode)
253 # Python 2.5b3: 62101 (fix wrong code: for x, in ...)
254 # Python 2.5b3: 62111 (fix wrong code: x += yield)
255 # Python 2.5c1: 62121 (fix wrong lnotab with for loops and
256 # storing constants that should have been removed)
257 # Python 2.5c2: 62131 (fix wrong code: for x, in ... in listcomp/genexp)
258 # Python 2.6a0: 62151 (peephole optimizations and STORE_MAP opcode)
259 # Python 2.6a1: 62161 (WITH_CLEANUP optimization)
260 # Python 2.7a0: 62171 (optimize list comprehensions/change LIST_APPEND)
261 # Python 2.7a0: 62181 (optimize conditional branches:
262 # introduce POP_JUMP_IF_FALSE and POP_JUMP_IF_TRUE)
263 # Python 2.7a0 62191 (introduce SETUP_WITH)
264 # Python 2.7a0 62201 (introduce BUILD_SET)
265 # Python 2.7a0 62211 (introduce MAP_ADD and SET_ADD)
266 # Python 3000: 3000
267 # 3010 (removed UNARY_CONVERT)
268 # 3020 (added BUILD_SET)
269 # 3030 (added keyword-only parameters)
270 # 3040 (added signature annotations)
271 # 3050 (print becomes a function)
272 # 3060 (PEP 3115 metaclass syntax)
273 # 3061 (string literals become unicode)
274 # 3071 (PEP 3109 raise changes)
275 # 3081 (PEP 3137 make __file__ and __name__ unicode)
276 # 3091 (kill str8 interning)
277 # 3101 (merge from 2.6a0, see 62151)
278 # 3103 (__file__ points to source file)
279 # Python 3.0a4: 3111 (WITH_CLEANUP optimization).
280 # Python 3.0b1: 3131 (lexical exception stacking, including POP_EXCEPT
281 #3021)
282 # Python 3.1a1: 3141 (optimize list, set and dict comprehensions:
283 # change LIST_APPEND and SET_ADD, add MAP_ADD #2183)
284 # Python 3.1a1: 3151 (optimize conditional branches:
285 # introduce POP_JUMP_IF_FALSE and POP_JUMP_IF_TRUE
286 #4715)
287 # Python 3.2a1: 3160 (add SETUP_WITH #6101)
288 # tag: cpython-32
289 # Python 3.2a2: 3170 (add DUP_TOP_TWO, remove DUP_TOPX and ROT_FOUR #9225)
290 # tag: cpython-32
291 # Python 3.2a3 3180 (add DELETE_DEREF #4617)
292 # Python 3.3a1 3190 (__class__ super closure changed)
293 # Python 3.3a1 3200 (PEP 3155 __qualname__ added #13448)
294 # Python 3.3a1 3210 (added size modulo 2**32 to the pyc header #13645)
295 # Python 3.3a2 3220 (changed PEP 380 implementation #14230)
296 # Python 3.3a4 3230 (revert changes to implicit __class__ closure #14857)
297 # Python 3.4a1 3250 (evaluate positional default arguments before
298 # keyword-only defaults #16967)
299 # Python 3.4a1 3260 (add LOAD_CLASSDEREF; allow locals of class to override
300 # free vars #17853)
301 # Python 3.4a1 3270 (various tweaks to the __class__ closure #12370)
302 # Python 3.4a1 3280 (remove implicit class argument)
303 # Python 3.4a4 3290 (changes to __qualname__ computation #19301)
304 # Python 3.4a4 3300 (more changes to __qualname__ computation #19301)
305 # Python 3.4rc2 3310 (alter __qualname__ computation #20625)
306 # Python 3.5a1 3320 (PEP 465: Matrix multiplication operator #21176)
307 # Python 3.5b1 3330 (PEP 448: Additional Unpacking Generalizations #2292)
308 # Python 3.5b2 3340 (fix dictionary display evaluation order #11205)
309 # Python 3.5b3 3350 (add GET_YIELD_FROM_ITER opcode #24400)
310 # Python 3.5.2 3351 (fix BUILD_MAP_UNPACK_WITH_CALL opcode #27286)
311 # Python 3.6a0 3360 (add FORMAT_VALUE opcode #25483)
312 # Python 3.6a1 3361 (lineno delta of code.co_lnotab becomes signed #26107)
313 # Python 3.6a2 3370 (16 bit wordcode #26647)
314 # Python 3.6a2 3371 (add BUILD_CONST_KEY_MAP opcode #27140)
315 # Python 3.6a2 3372 (MAKE_FUNCTION simplification, remove MAKE_CLOSURE
316 # #27095)
317 # Python 3.6b1 3373 (add BUILD_STRING opcode #27078)
318 # Python 3.6b1 3375 (add SETUP_ANNOTATIONS and STORE_ANNOTATION opcodes
319 # #27985)
320 # Python 3.6b1 3376 (simplify CALL_FUNCTIONs & BUILD_MAP_UNPACK_WITH_CALL
321 #27213)
322 # Python 3.6b1 3377 (set __class__ cell from type.__new__ #23722)
323 # Python 3.6b2 3378 (add BUILD_TUPLE_UNPACK_WITH_CALL #28257)
324 # Python 3.6rc1 3379 (more thorough __class__ validation #23722)
325 # Python 3.7a1 3390 (add LOAD_METHOD and CALL_METHOD opcodes #26110)
326 # Python 3.7a2 3391 (update GET_AITER #31709)
327 # Python 3.7a4 3392 (PEP 552: Deterministic pycs #31650)
328 # Python 3.7b1 3393 (remove STORE_ANNOTATION opcode #32550)
329 # Python 3.7b5 3394 (restored docstring as the first stmt in the body;
330 # this might affected the first line number #32911)
331 # Python 3.8a1 3400 (move frame block handling to compiler #17611)
332 # Python 3.8a1 3401 (add END_ASYNC_FOR #33041)
333 # Python 3.8a1 3410 (PEP570 Python Positional-Only Parameters #36540)
334 # Python 3.8b2 3411 (Reverse evaluation order of key: value in dict
335 # comprehensions #35224)
336 # Python 3.8b2 3412 (Swap the position of positional args and positional
337 # only args in ast.arguments #37593)
338 # Python 3.8b4 3413 (Fix "break" and "continue" in "finally" #37830)
339 # Python 3.9a0 3420 (add LOAD_ASSERTION_ERROR #34880)
340 # Python 3.9a0 3421 (simplified bytecode for with blocks #32949)
341 # Python 3.9a0 3422 (remove BEGIN_FINALLY, END_FINALLY, CALL_FINALLY, POP_FINALLY bytecodes #33387)
342 # Python 3.9a2 3423 (add IS_OP, CONTAINS_OP and JUMP_IF_NOT_EXC_MATCH bytecodes #39156)
343 # Python 3.9a2 3424 (simplify bytecodes for *value unpacking)
344 # Python 3.9a2 3425 (simplify bytecodes for **value unpacking)
345 # Python 3.10a1 3430 (Make 'annotations' future by default)
346 # Python 3.10a1 3431 (New line number table format -- PEP 626)
347 # Python 3.10a2 3432 (Function annotation for MAKE_FUNCTION is changed from dict to tuple bpo-42202)
348 # Python 3.10a2 3433 (RERAISE restores f_lasti if oparg != 0)
349 # Python 3.10a6 3434 (PEP 634: Structural Pattern Matching)
350 # Python 3.10a7 3435 Use instruction offsets (as opposed to byte offsets).
351 # Python 3.10b1 3436 (Add GEN_START bytecode #43683)
352 # Python 3.10b1 3437 (Undo making 'annotations' future by default - We like to dance among core devs!)
353 # Python 3.10b1 3438 Safer line number table handling.
354 # Python 3.10b1 3439 (Add ROT_N)
355 # Python 3.11a1 3450 Use exception table for unwinding ("zero cost" exception handling)
356 # Python 3.11a1 3451 (Add CALL_METHOD_KW)
357 # Python 3.11a1 3452 (drop nlocals from marshaled code objects)
358 # Python 3.11a1 3453 (add co_fastlocalnames and co_fastlocalkinds)
359 # Python 3.11a1 3454 (compute cell offsets relative to locals bpo-43693)
360 # Python 3.11a1 3455 (add MAKE_CELL bpo-43693)
361 # Python 3.11a1 3456 (interleave cell args bpo-43693)
362 # Python 3.11a1 3457 (Change localsplus to a bytes object bpo-43693)
363 # Python 3.11a1 3458 (imported objects now don't use LOAD_METHOD/CALL_METHOD)
364 # Python 3.11a1 3459 (PEP 657: add end line numbers and column offsets for instructions)
365 # Python 3.11a1 3460 (Add co_qualname field to PyCodeObject bpo-44530)
366 # Python 3.11a1 3461 (JUMP_ABSOLUTE must jump backwards)
367 # Python 3.11a2 3462 (bpo-44511: remove COPY_DICT_WITHOUT_KEYS, change
368 # MATCH_CLASS and MATCH_KEYS, and add COPY)
369 # Python 3.11a3 3463 (bpo-45711: JUMP_IF_NOT_EXC_MATCH no longer pops the
370 # active exception)
371 # Python 3.11a3 3464 (bpo-45636: Merge numeric BINARY_*/INPLACE_* into
372 # BINARY_OP)
373 # Python 3.11a3 3465 (Add COPY_FREE_VARS opcode)
374 # Python 3.11a4 3466 (bpo-45292: PEP-654 except*)
375 # Python 3.11a4 3467 (Change CALL_xxx opcodes)
376 # Python 3.11a4 3468 (Add SEND opcode)
377 # Python 3.11a4 3469 (bpo-45711: remove type, traceback from exc_info)
378 # Python 3.11a4 3470 (bpo-46221: PREP_RERAISE_STAR no longer pushes lasti)
379 # Python 3.11a4 3471 (bpo-46202: remove pop POP_EXCEPT_AND_RERAISE)
380 # Python 3.11a4 3472 (bpo-46009: replace GEN_START with POP_TOP)
381 # Python 3.11a4 3473 (Add POP_JUMP_IF_NOT_NONE/POP_JUMP_IF_NONE opcodes)
382 # Python 3.11a4 3474 (Add RESUME opcode)
383 # Python 3.11a5 3475 (Add RETURN_GENERATOR opcode)
384 # Python 3.11a5 3476 (Add ASYNC_GEN_WRAP opcode)
385 # Python 3.11a5 3477 (Replace DUP_TOP/DUP_TOP_TWO with COPY and
386 # ROT_TWO/ROT_THREE/ROT_FOUR/ROT_N with SWAP)
387 # Python 3.11a5 3478 (New CALL opcodes)
388 # Python 3.11a5 3479 (Add PUSH_NULL opcode)
389 # Python 3.11a5 3480 (New CALL opcodes, second iteration)
390 # Python 3.11a5 3481 (Use inline cache for BINARY_OP)
391 # Python 3.11a5 3482 (Use inline caching for UNPACK_SEQUENCE and LOAD_GLOBAL)
392 # Python 3.11a5 3483 (Use inline caching for COMPARE_OP and BINARY_SUBSCR)
393 # Python 3.11a5 3484 (Use inline caching for LOAD_ATTR, LOAD_METHOD, and
394 # STORE_ATTR)
395 # Python 3.11a5 3485 (Add an oparg to GET_AWAITABLE)
396 # Python 3.11a6 3486 (Use inline caching for PRECALL and CALL)
397 # Python 3.11a6 3487 (Remove the adaptive "oparg counter" mechanism)
398 # Python 3.11a6 3488 (LOAD_GLOBAL can push additional NULL)
399 # Python 3.11a6 3489 (Add JUMP_BACKWARD, remove JUMP_ABSOLUTE)
400 # Python 3.11a6 3490 (remove JUMP_IF_NOT_EXC_MATCH, add CHECK_EXC_MATCH)
401 # Python 3.11a6 3491 (remove JUMP_IF_NOT_EG_MATCH, add CHECK_EG_MATCH,
402 # add JUMP_BACKWARD_NO_INTERRUPT, make JUMP_NO_INTERRUPT virtual)
403 # Python 3.11a7 3492 (make POP_JUMP_IF_NONE/NOT_NONE/TRUE/FALSE relative)
404 # Python 3.11a7 3493 (Make JUMP_IF_TRUE_OR_POP/JUMP_IF_FALSE_OR_POP relative)
405 # Python 3.11a7 3494 (New location info table)
406 # Python 3.11b4 3495 (Set line number of module's RESUME instr to 0 per PEP 626)
407 # Python 3.12 will start with magic number 3500
408
409
410 #
411 # MAGIC must change whenever the bytecode emitted by the compiler may no
412 # longer be understood by older implementations of the eval loop (usually
413 # due to the addition of new opcodes).
414 #
415 # Starting with Python 3.11, Python 3.n starts with magic number 2900+50n.
416 #
417 # Whenever MAGIC_NUMBER is changed, the ranges in the magic_values array
418 # in PC/launcher.c must also be updated.
419
420 MAGIC_NUMBER = (3495).to_bytes(2, 'little') + b'\r\n'
421
422 _RAW_MAGIC_NUMBER = int.from_bytes(MAGIC_NUMBER, 'little') # For import.c
423
424 _PYCACHE = '__pycache__'
425 _OPT = 'opt-'
426
427 SOURCE_SUFFIXES = ['.py']
428 if _MS_WINDOWS:
429 SOURCE_SUFFIXES.append('.pyw')
430
431 EXTENSION_SUFFIXES = _imp.extension_suffixes()
432
433 BYTECODE_SUFFIXES = ['.pyc']
434 # Deprecated.
435 DEBUG_BYTECODE_SUFFIXES = OPTIMIZED_BYTECODE_SUFFIXES = BYTECODE_SUFFIXES
436
437 def cache_from_source(path, debug_override=None, *, optimization=None):
438 """Given the path to a .py file, return the path to its .pyc file.
439
440 The .py file does not need to exist; this simply returns the path to the
441 .pyc file calculated as if the .py file were imported.
442
443 The 'optimization' parameter controls the presumed optimization level of
444 the bytecode file. If 'optimization' is not None, the string representation
445 of the argument is taken and verified to be alphanumeric (else ValueError
446 is raised).
447
448 The debug_override parameter is deprecated. If debug_override is not None,
449 a True value is the same as setting 'optimization' to the empty string
450 while a False value is equivalent to setting 'optimization' to '1'.
451
452 If sys.implementation.cache_tag is None then NotImplementedError is raised.
453
454 """
455 if debug_override is not None:
456 _warnings.warn('the debug_override parameter is deprecated; use '
457 "'optimization' instead", DeprecationWarning)
458 if optimization is not None:
459 message = 'debug_override or optimization must be set to None'
460 raise TypeError(message)
461 optimization = '' if debug_override else 1
462 path = _os.fspath(path)
463 head, tail = _path_split(path)
464 base, sep, rest = tail.rpartition('.')
465 tag = sys.implementation.cache_tag
466 if tag is None:
467 raise NotImplementedError('sys.implementation.cache_tag is None')
468 almost_filename = ''.join([(base if base else rest), sep, tag])
469 if optimization is None:
470 if sys.flags.optimize == 0:
471 optimization = ''
472 else:
473 optimization = sys.flags.optimize
474 optimization = str(optimization)
475 if optimization != '':
476 if not optimization.isalnum():
477 raise ValueError('{!r} is not alphanumeric'.format(optimization))
478 almost_filename = '{}.{}{}'.format(almost_filename, _OPT, optimization)
479 filename = almost_filename + BYTECODE_SUFFIXES[0]
480 if sys.pycache_prefix is not None:
481 # We need an absolute path to the py file to avoid the possibility of
482 # collisions within sys.pycache_prefix, if someone has two different
483 # `foo/bar.py` on their system and they import both of them using the
484 # same sys.pycache_prefix. Let's say sys.pycache_prefix is
485 # `C:\Bytecode`; the idea here is that if we get `Foo\Bar`, we first
486 # make it absolute (`C:\Somewhere\Foo\Bar`), then make it root-relative
487 # (`Somewhere\Foo\Bar`), so we end up placing the bytecode file in an
488 # unambiguous `C:\Bytecode\Somewhere\Foo\Bar\`.
489 if not _path_isabs(head):
490 head = _path_join(_os.getcwd(), head)
491
492 # Strip initial drive from a Windows path. We know we have an absolute
493 # path here, so the second part of the check rules out a POSIX path that
494 # happens to contain a colon at the second character.
495 if head[1] == ':' and head[0] not in path_separators:
496 head = head[2:]
497
498 # Strip initial path separator from `head` to complete the conversion
499 # back to a root-relative path before joining.
500 return _path_join(
501 sys.pycache_prefix,
502 head.lstrip(path_separators),
503 filename,
504 )
505 return _path_join(head, _PYCACHE, filename)
506
507
508 def source_from_cache(path):
509 """Given the path to a .pyc. file, return the path to its .py file.
510
511 The .pyc file does not need to exist; this simply returns the path to
512 the .py file calculated to correspond to the .pyc file. If path does
513 not conform to PEP 3147/488 format, ValueError will be raised. If
514 sys.implementation.cache_tag is None then NotImplementedError is raised.
515
516 """
517 if sys.implementation.cache_tag is None:
518 raise NotImplementedError('sys.implementation.cache_tag is None')
519 path = _os.fspath(path)
520 head, pycache_filename = _path_split(path)
521 found_in_pycache_prefix = False
522 if sys.pycache_prefix is not None:
523 stripped_path = sys.pycache_prefix.rstrip(path_separators)
524 if head.startswith(stripped_path + path_sep):
525 head = head[len(stripped_path):]
526 found_in_pycache_prefix = True
527 if not found_in_pycache_prefix:
528 head, pycache = _path_split(head)
529 if pycache != _PYCACHE:
530 raise ValueError(f'{_PYCACHE} not bottom-level directory in '
531 f'{path!r}')
532 dot_count = pycache_filename.count('.')
533 if dot_count not in {2, 3}:
534 raise ValueError(f'expected only 2 or 3 dots in {pycache_filename!r}')
535 elif dot_count == 3:
536 optimization = pycache_filename.rsplit('.', 2)[-2]
537 if not optimization.startswith(_OPT):
538 raise ValueError("optimization portion of filename does not start "
539 f"with {_OPT!r}")
540 opt_level = optimization[len(_OPT):]
541 if not opt_level.isalnum():
542 raise ValueError(f"optimization level {optimization!r} is not an "
543 "alphanumeric value")
544 base_filename = pycache_filename.partition('.')[0]
545 return _path_join(head, base_filename + SOURCE_SUFFIXES[0])
546
547
548 def _get_sourcefile(bytecode_path):
549 """Convert a bytecode file path to a source path (if possible).
550
551 This function exists purely for backwards-compatibility for
552 PyImport_ExecCodeModuleWithFilenames() in the C API.
553
554 """
555 if len(bytecode_path) == 0:
556 return None
557 rest, _, extension = bytecode_path.rpartition('.')
558 if not rest or extension.lower()[-3:-1] != 'py':
559 return bytecode_path
560 try:
561 source_path = source_from_cache(bytecode_path)
562 except (NotImplementedError, ValueError):
563 source_path = bytecode_path[:-1]
564 return source_path if _path_isfile(source_path) else bytecode_path
565
566
567 def _get_cached(filename):
568 if filename.endswith(tuple(SOURCE_SUFFIXES)):
569 try:
570 return cache_from_source(filename)
571 except NotImplementedError:
572 pass
573 elif filename.endswith(tuple(BYTECODE_SUFFIXES)):
574 return filename
575 else:
576 return None
577
578
579 def _calc_mode(path):
580 """Calculate the mode permissions for a bytecode file."""
581 try:
582 mode = _path_stat(path).st_mode
583 except OSError:
584 mode = 0o666
585 # We always ensure write access so we can update cached files
586 # later even when the source files are read-only on Windows (#6074)
587 mode |= 0o200
588 return mode
589
590
591 def _check_name(method):
592 """Decorator to verify that the module being requested matches the one the
593 loader can handle.
594
595 The first argument (self) must define _name which the second argument is
596 compared against. If the comparison fails then ImportError is raised.
597
598 """
599 def _check_name_wrapper(self, name=None, *args, **kwargs):
600 if name is None:
601 name = self.name
602 elif self.name != name:
603 raise ImportError('loader for %s cannot handle %s' %
604 (self.name, name), name=name)
605 return method(self, name, *args, **kwargs)
606
607 # FIXME: @_check_name is used to define class methods before the
608 # _bootstrap module is set by _set_bootstrap_module().
609 if _bootstrap is not None:
610 _wrap = _bootstrap._wrap
611 else:
612 def _wrap(new, old):
613 for replace in ['__module__', '__name__', '__qualname__', '__doc__']:
614 if hasattr(old, replace):
615 setattr(new, replace, getattr(old, replace))
616 new.__dict__.update(old.__dict__)
617
618 _wrap(_check_name_wrapper, method)
619 return _check_name_wrapper
620
621
622 def _find_module_shim(self, fullname):
623 """Try to find a loader for the specified module by delegating to
624 self.find_loader().
625
626 This method is deprecated in favor of finder.find_spec().
627
628 """
629 _warnings.warn("find_module() is deprecated and "
630 "slated for removal in Python 3.12; use find_spec() instead",
631 DeprecationWarning)
632 # Call find_loader(). If it returns a string (indicating this
633 # is a namespace package portion), generate a warning and
634 # return None.
635 loader, portions = self.find_loader(fullname)
636 if loader is None and len(portions):
637 msg = 'Not importing directory {}: missing __init__'
638 _warnings.warn(msg.format(portions[0]), ImportWarning)
639 return loader
640
641
642 def _classify_pyc(data, name, exc_details):
643 """Perform basic validity checking of a pyc header and return the flags field,
644 which determines how the pyc should be further validated against the source.
645
646 *data* is the contents of the pyc file. (Only the first 16 bytes are
647 required, though.)
648
649 *name* is the name of the module being imported. It is used for logging.
650
651 *exc_details* is a dictionary passed to ImportError if it raised for
652 improved debugging.
653
654 ImportError is raised when the magic number is incorrect or when the flags
655 field is invalid. EOFError is raised when the data is found to be truncated.
656
657 """
658 magic = data[:4]
659 if magic != MAGIC_NUMBER:
660 message = f'bad magic number in {name!r}: {magic!r}'
661 _bootstrap._verbose_message('{}', message)
662 raise ImportError(message, **exc_details)
663 if len(data) < 16:
664 message = f'reached EOF while reading pyc header of {name!r}'
665 _bootstrap._verbose_message('{}', message)
666 raise EOFError(message)
667 flags = _unpack_uint32(data[4:8])
668 # Only the first two flags are defined.
669 if flags & ~0b11:
670 message = f'invalid flags {flags!r} in {name!r}'
671 raise ImportError(message, **exc_details)
672 return flags
673
674
675 def _validate_timestamp_pyc(data, source_mtime, source_size, name,
676 exc_details):
677 """Validate a pyc against the source last-modified time.
678
679 *data* is the contents of the pyc file. (Only the first 16 bytes are
680 required.)
681
682 *source_mtime* is the last modified timestamp of the source file.
683
684 *source_size* is None or the size of the source file in bytes.
685
686 *name* is the name of the module being imported. It is used for logging.
687
688 *exc_details* is a dictionary passed to ImportError if it raised for
689 improved debugging.
690
691 An ImportError is raised if the bytecode is stale.
692
693 """
694 if _unpack_uint32(data[8:12]) != (source_mtime & 0xFFFFFFFF):
695 message = f'bytecode is stale for {name!r}'
696 _bootstrap._verbose_message('{}', message)
697 raise ImportError(message, **exc_details)
698 if (source_size is not None and
699 _unpack_uint32(data[12:16]) != (source_size & 0xFFFFFFFF)):
700 raise ImportError(f'bytecode is stale for {name!r}', **exc_details)
701
702
703 def _validate_hash_pyc(data, source_hash, name, exc_details):
704 """Validate a hash-based pyc by checking the real source hash against the one in
705 the pyc header.
706
707 *data* is the contents of the pyc file. (Only the first 16 bytes are
708 required.)
709
710 *source_hash* is the importlib.util.source_hash() of the source file.
711
712 *name* is the name of the module being imported. It is used for logging.
713
714 *exc_details* is a dictionary passed to ImportError if it raised for
715 improved debugging.
716
717 An ImportError is raised if the bytecode is stale.
718
719 """
720 if data[8:16] != source_hash:
721 raise ImportError(
722 f'hash in bytecode doesn\'t match hash of source {name!r}',
723 **exc_details,
724 )
725
726
727 def _compile_bytecode(data, name=None, bytecode_path=None, source_path=None):
728 """Compile bytecode as found in a pyc."""
729 code = marshal.loads(data)
730 if isinstance(code, _code_type):
731 _bootstrap._verbose_message('code object from {!r}', bytecode_path)
732 if source_path is not None:
733 _imp._fix_co_filename(code, source_path)
734 return code
735 else:
736 raise ImportError('Non-code object in {!r}'.format(bytecode_path),
737 name=name, path=bytecode_path)
738
739
740 def _code_to_timestamp_pyc(code, mtime=0, source_size=0):
741 "Produce the data for a timestamp-based pyc."
742 data = bytearray(MAGIC_NUMBER)
743 data.extend(_pack_uint32(0))
744 data.extend(_pack_uint32(mtime))
745 data.extend(_pack_uint32(source_size))
746 data.extend(marshal.dumps(code))
747 return data
748
749
750 def _code_to_hash_pyc(code, source_hash, checked=True):
751 "Produce the data for a hash-based pyc."
752 data = bytearray(MAGIC_NUMBER)
753 flags = 0b1 | checked << 1
754 data.extend(_pack_uint32(flags))
755 assert len(source_hash) == 8
756 data.extend(source_hash)
757 data.extend(marshal.dumps(code))
758 return data
759
760
761 def decode_source(source_bytes):
762 """Decode bytes representing source code and return the string.
763
764 Universal newline support is used in the decoding.
765 """
766 import tokenize # To avoid bootstrap issues.
767 source_bytes_readline = _io.BytesIO(source_bytes).readline
768 encoding = tokenize.detect_encoding(source_bytes_readline)
769 newline_decoder = _io.IncrementalNewlineDecoder(None, True)
770 return newline_decoder.decode(source_bytes.decode(encoding[0]))
771
772
773 # Module specifications #######################################################
774
775 _POPULATE = object()
776
777
778 def spec_from_file_location(name, location=None, *, loader=None,
779 submodule_search_locations=_POPULATE):
780 """Return a module spec based on a file location.
781
782 To indicate that the module is a package, set
783 submodule_search_locations to a list of directory paths. An
784 empty list is sufficient, though its not otherwise useful to the
785 import system.
786
787 The loader must take a spec as its only __init__() arg.
788
789 """
790 if location is None:
791 # The caller may simply want a partially populated location-
792 # oriented spec. So we set the location to a bogus value and
793 # fill in as much as we can.
794 location = '<unknown>'
795 if hasattr(loader, 'get_filename'):
796 # ExecutionLoader
797 try:
798 location = loader.get_filename(name)
799 except ImportError:
800 pass
801 else:
802 location = _os.fspath(location)
803 if not _path_isabs(location):
804 try:
805 location = _path_join(_os.getcwd(), location)
806 except OSError:
807 pass
808
809 # If the location is on the filesystem, but doesn't actually exist,
810 # we could return None here, indicating that the location is not
811 # valid. However, we don't have a good way of testing since an
812 # indirect location (e.g. a zip file or URL) will look like a
813 # non-existent file relative to the filesystem.
814
815 spec = _bootstrap.ModuleSpec(name, loader, origin=location)
816 spec._set_fileattr = True
817
818 # Pick a loader if one wasn't provided.
819 if loader is None:
820 for loader_class, suffixes in _get_supported_file_loaders():
821 if location.endswith(tuple(suffixes)):
822 loader = loader_class(name, location)
823 spec.loader = loader
824 break
825 else:
826 return None
827
828 # Set submodule_search_paths appropriately.
829 if submodule_search_locations is _POPULATE:
830 # Check the loader.
831 if hasattr(loader, 'is_package'):
832 try:
833 is_package = loader.is_package(name)
834 except ImportError:
835 pass
836 else:
837 if is_package:
838 spec.submodule_search_locations = []
839 else:
840 spec.submodule_search_locations = submodule_search_locations
841 if spec.submodule_search_locations == []:
842 if location:
843 dirname = _path_split(location)[0]
844 spec.submodule_search_locations.append(dirname)
845
846 return spec
847
848
849 # Loaders #####################################################################
850
851 class ESC[4;38;5;81mWindowsRegistryFinder:
852
853 """Meta path finder for modules declared in the Windows registry."""
854
855 REGISTRY_KEY = (
856 'Software\\Python\\PythonCore\\{sys_version}'
857 '\\Modules\\{fullname}')
858 REGISTRY_KEY_DEBUG = (
859 'Software\\Python\\PythonCore\\{sys_version}'
860 '\\Modules\\{fullname}\\Debug')
861 DEBUG_BUILD = (_MS_WINDOWS and '_d.pyd' in EXTENSION_SUFFIXES)
862
863 @staticmethod
864 def _open_registry(key):
865 try:
866 return winreg.OpenKey(winreg.HKEY_CURRENT_USER, key)
867 except OSError:
868 return winreg.OpenKey(winreg.HKEY_LOCAL_MACHINE, key)
869
870 @classmethod
871 def _search_registry(cls, fullname):
872 if cls.DEBUG_BUILD:
873 registry_key = cls.REGISTRY_KEY_DEBUG
874 else:
875 registry_key = cls.REGISTRY_KEY
876 key = registry_key.format(fullname=fullname,
877 sys_version='%d.%d' % sys.version_info[:2])
878 try:
879 with cls._open_registry(key) as hkey:
880 filepath = winreg.QueryValue(hkey, '')
881 except OSError:
882 return None
883 return filepath
884
885 @classmethod
886 def find_spec(cls, fullname, path=None, target=None):
887 filepath = cls._search_registry(fullname)
888 if filepath is None:
889 return None
890 try:
891 _path_stat(filepath)
892 except OSError:
893 return None
894 for loader, suffixes in _get_supported_file_loaders():
895 if filepath.endswith(tuple(suffixes)):
896 spec = _bootstrap.spec_from_loader(fullname,
897 loader(fullname, filepath),
898 origin=filepath)
899 return spec
900
901 @classmethod
902 def find_module(cls, fullname, path=None):
903 """Find module named in the registry.
904
905 This method is deprecated. Use find_spec() instead.
906
907 """
908 _warnings.warn("WindowsRegistryFinder.find_module() is deprecated and "
909 "slated for removal in Python 3.12; use find_spec() instead",
910 DeprecationWarning)
911 spec = cls.find_spec(fullname, path)
912 if spec is not None:
913 return spec.loader
914 else:
915 return None
916
917
918 class ESC[4;38;5;81m_LoaderBasics:
919
920 """Base class of common code needed by both SourceLoader and
921 SourcelessFileLoader."""
922
923 def is_package(self, fullname):
924 """Concrete implementation of InspectLoader.is_package by checking if
925 the path returned by get_filename has a filename of '__init__.py'."""
926 filename = _path_split(self.get_filename(fullname))[1]
927 filename_base = filename.rsplit('.', 1)[0]
928 tail_name = fullname.rpartition('.')[2]
929 return filename_base == '__init__' and tail_name != '__init__'
930
931 def create_module(self, spec):
932 """Use default semantics for module creation."""
933
934 def exec_module(self, module):
935 """Execute the module."""
936 code = self.get_code(module.__name__)
937 if code is None:
938 raise ImportError('cannot load module {!r} when get_code() '
939 'returns None'.format(module.__name__))
940 _bootstrap._call_with_frames_removed(exec, code, module.__dict__)
941
942 def load_module(self, fullname):
943 """This method is deprecated."""
944 # Warning implemented in _load_module_shim().
945 return _bootstrap._load_module_shim(self, fullname)
946
947
948 class ESC[4;38;5;81mSourceLoader(ESC[4;38;5;149m_LoaderBasics):
949
950 def path_mtime(self, path):
951 """Optional method that returns the modification time (an int) for the
952 specified path (a str).
953
954 Raises OSError when the path cannot be handled.
955 """
956 raise OSError
957
958 def path_stats(self, path):
959 """Optional method returning a metadata dict for the specified
960 path (a str).
961
962 Possible keys:
963 - 'mtime' (mandatory) is the numeric timestamp of last source
964 code modification;
965 - 'size' (optional) is the size in bytes of the source code.
966
967 Implementing this method allows the loader to read bytecode files.
968 Raises OSError when the path cannot be handled.
969 """
970 return {'mtime': self.path_mtime(path)}
971
972 def _cache_bytecode(self, source_path, cache_path, data):
973 """Optional method which writes data (bytes) to a file path (a str).
974
975 Implementing this method allows for the writing of bytecode files.
976
977 The source path is needed in order to correctly transfer permissions
978 """
979 # For backwards compatibility, we delegate to set_data()
980 return self.set_data(cache_path, data)
981
982 def set_data(self, path, data):
983 """Optional method which writes data (bytes) to a file path (a str).
984
985 Implementing this method allows for the writing of bytecode files.
986 """
987
988
989 def get_source(self, fullname):
990 """Concrete implementation of InspectLoader.get_source."""
991 path = self.get_filename(fullname)
992 try:
993 source_bytes = self.get_data(path)
994 except OSError as exc:
995 raise ImportError('source not available through get_data()',
996 name=fullname) from exc
997 return decode_source(source_bytes)
998
999 def source_to_code(self, data, path, *, _optimize=-1):
1000 """Return the code object compiled from source.
1001
1002 The 'data' argument can be any object type that compile() supports.
1003 """
1004 return _bootstrap._call_with_frames_removed(compile, data, path, 'exec',
1005 dont_inherit=True, optimize=_optimize)
1006
1007 def get_code(self, fullname):
1008 """Concrete implementation of InspectLoader.get_code.
1009
1010 Reading of bytecode requires path_stats to be implemented. To write
1011 bytecode, set_data must also be implemented.
1012
1013 """
1014 source_path = self.get_filename(fullname)
1015 source_mtime = None
1016 source_bytes = None
1017 source_hash = None
1018 hash_based = False
1019 check_source = True
1020 try:
1021 bytecode_path = cache_from_source(source_path)
1022 except NotImplementedError:
1023 bytecode_path = None
1024 else:
1025 try:
1026 st = self.path_stats(source_path)
1027 except OSError:
1028 pass
1029 else:
1030 source_mtime = int(st['mtime'])
1031 try:
1032 data = self.get_data(bytecode_path)
1033 except OSError:
1034 pass
1035 else:
1036 exc_details = {
1037 'name': fullname,
1038 'path': bytecode_path,
1039 }
1040 try:
1041 flags = _classify_pyc(data, fullname, exc_details)
1042 bytes_data = memoryview(data)[16:]
1043 hash_based = flags & 0b1 != 0
1044 if hash_based:
1045 check_source = flags & 0b10 != 0
1046 if (_imp.check_hash_based_pycs != 'never' and
1047 (check_source or
1048 _imp.check_hash_based_pycs == 'always')):
1049 source_bytes = self.get_data(source_path)
1050 source_hash = _imp.source_hash(
1051 _RAW_MAGIC_NUMBER,
1052 source_bytes,
1053 )
1054 _validate_hash_pyc(data, source_hash, fullname,
1055 exc_details)
1056 else:
1057 _validate_timestamp_pyc(
1058 data,
1059 source_mtime,
1060 st['size'],
1061 fullname,
1062 exc_details,
1063 )
1064 except (ImportError, EOFError):
1065 pass
1066 else:
1067 _bootstrap._verbose_message('{} matches {}', bytecode_path,
1068 source_path)
1069 return _compile_bytecode(bytes_data, name=fullname,
1070 bytecode_path=bytecode_path,
1071 source_path=source_path)
1072 if source_bytes is None:
1073 source_bytes = self.get_data(source_path)
1074 code_object = self.source_to_code(source_bytes, source_path)
1075 _bootstrap._verbose_message('code object from {}', source_path)
1076 if (not sys.dont_write_bytecode and bytecode_path is not None and
1077 source_mtime is not None):
1078 if hash_based:
1079 if source_hash is None:
1080 source_hash = _imp.source_hash(source_bytes)
1081 data = _code_to_hash_pyc(code_object, source_hash, check_source)
1082 else:
1083 data = _code_to_timestamp_pyc(code_object, source_mtime,
1084 len(source_bytes))
1085 try:
1086 self._cache_bytecode(source_path, bytecode_path, data)
1087 except NotImplementedError:
1088 pass
1089 return code_object
1090
1091
1092 class ESC[4;38;5;81mFileLoader:
1093
1094 """Base file loader class which implements the loader protocol methods that
1095 require file system usage."""
1096
1097 def __init__(self, fullname, path):
1098 """Cache the module name and the path to the file found by the
1099 finder."""
1100 self.name = fullname
1101 self.path = path
1102
1103 def __eq__(self, other):
1104 return (self.__class__ == other.__class__ and
1105 self.__dict__ == other.__dict__)
1106
1107 def __hash__(self):
1108 return hash(self.name) ^ hash(self.path)
1109
1110 @_check_name
1111 def load_module(self, fullname):
1112 """Load a module from a file.
1113
1114 This method is deprecated. Use exec_module() instead.
1115
1116 """
1117 # The only reason for this method is for the name check.
1118 # Issue #14857: Avoid the zero-argument form of super so the implementation
1119 # of that form can be updated without breaking the frozen module.
1120 return super(FileLoader, self).load_module(fullname)
1121
1122 @_check_name
1123 def get_filename(self, fullname):
1124 """Return the path to the source file as found by the finder."""
1125 return self.path
1126
1127 def get_data(self, path):
1128 """Return the data from path as raw bytes."""
1129 if isinstance(self, (SourceLoader, ExtensionFileLoader)):
1130 with _io.open_code(str(path)) as file:
1131 return file.read()
1132 else:
1133 with _io.FileIO(path, 'r') as file:
1134 return file.read()
1135
1136 @_check_name
1137 def get_resource_reader(self, module):
1138 from importlib.readers import FileReader
1139 return FileReader(self)
1140
1141
1142 class ESC[4;38;5;81mSourceFileLoader(ESC[4;38;5;149mFileLoader, ESC[4;38;5;149mSourceLoader):
1143
1144 """Concrete implementation of SourceLoader using the file system."""
1145
1146 def path_stats(self, path):
1147 """Return the metadata for the path."""
1148 st = _path_stat(path)
1149 return {'mtime': st.st_mtime, 'size': st.st_size}
1150
1151 def _cache_bytecode(self, source_path, bytecode_path, data):
1152 # Adapt between the two APIs
1153 mode = _calc_mode(source_path)
1154 return self.set_data(bytecode_path, data, _mode=mode)
1155
1156 def set_data(self, path, data, *, _mode=0o666):
1157 """Write bytes data to a file."""
1158 parent, filename = _path_split(path)
1159 path_parts = []
1160 # Figure out what directories are missing.
1161 while parent and not _path_isdir(parent):
1162 parent, part = _path_split(parent)
1163 path_parts.append(part)
1164 # Create needed directories.
1165 for part in reversed(path_parts):
1166 parent = _path_join(parent, part)
1167 try:
1168 _os.mkdir(parent)
1169 except FileExistsError:
1170 # Probably another Python process already created the dir.
1171 continue
1172 except OSError as exc:
1173 # Could be a permission error, read-only filesystem: just forget
1174 # about writing the data.
1175 _bootstrap._verbose_message('could not create {!r}: {!r}',
1176 parent, exc)
1177 return
1178 try:
1179 _write_atomic(path, data, _mode)
1180 _bootstrap._verbose_message('created {!r}', path)
1181 except OSError as exc:
1182 # Same as above: just don't write the bytecode.
1183 _bootstrap._verbose_message('could not create {!r}: {!r}', path,
1184 exc)
1185
1186
1187 class ESC[4;38;5;81mSourcelessFileLoader(ESC[4;38;5;149mFileLoader, ESC[4;38;5;149m_LoaderBasics):
1188
1189 """Loader which handles sourceless file imports."""
1190
1191 def get_code(self, fullname):
1192 path = self.get_filename(fullname)
1193 data = self.get_data(path)
1194 # Call _classify_pyc to do basic validation of the pyc but ignore the
1195 # result. There's no source to check against.
1196 exc_details = {
1197 'name': fullname,
1198 'path': path,
1199 }
1200 _classify_pyc(data, fullname, exc_details)
1201 return _compile_bytecode(
1202 memoryview(data)[16:],
1203 name=fullname,
1204 bytecode_path=path,
1205 )
1206
1207 def get_source(self, fullname):
1208 """Return None as there is no source code."""
1209 return None
1210
1211
1212 class ESC[4;38;5;81mExtensionFileLoader(ESC[4;38;5;149mFileLoader, ESC[4;38;5;149m_LoaderBasics):
1213
1214 """Loader for extension modules.
1215
1216 The constructor is designed to work with FileFinder.
1217
1218 """
1219
1220 def __init__(self, name, path):
1221 self.name = name
1222 self.path = path
1223
1224 def __eq__(self, other):
1225 return (self.__class__ == other.__class__ and
1226 self.__dict__ == other.__dict__)
1227
1228 def __hash__(self):
1229 return hash(self.name) ^ hash(self.path)
1230
1231 def create_module(self, spec):
1232 """Create an uninitialized extension module"""
1233 module = _bootstrap._call_with_frames_removed(
1234 _imp.create_dynamic, spec)
1235 _bootstrap._verbose_message('extension module {!r} loaded from {!r}',
1236 spec.name, self.path)
1237 return module
1238
1239 def exec_module(self, module):
1240 """Initialize an extension module"""
1241 _bootstrap._call_with_frames_removed(_imp.exec_dynamic, module)
1242 _bootstrap._verbose_message('extension module {!r} executed from {!r}',
1243 self.name, self.path)
1244
1245 def is_package(self, fullname):
1246 """Return True if the extension module is a package."""
1247 file_name = _path_split(self.path)[1]
1248 return any(file_name == '__init__' + suffix
1249 for suffix in EXTENSION_SUFFIXES)
1250
1251 def get_code(self, fullname):
1252 """Return None as an extension module cannot create a code object."""
1253 return None
1254
1255 def get_source(self, fullname):
1256 """Return None as extension modules have no source code."""
1257 return None
1258
1259 @_check_name
1260 def get_filename(self, fullname):
1261 """Return the path to the source file as found by the finder."""
1262 return self.path
1263
1264
1265 class ESC[4;38;5;81m_NamespacePath:
1266 """Represents a namespace package's path. It uses the module name
1267 to find its parent module, and from there it looks up the parent's
1268 __path__. When this changes, the module's own path is recomputed,
1269 using path_finder. For top-level modules, the parent module's path
1270 is sys.path."""
1271
1272 # When invalidate_caches() is called, this epoch is incremented
1273 # https://bugs.python.org/issue45703
1274 _epoch = 0
1275
1276 def __init__(self, name, path, path_finder):
1277 self._name = name
1278 self._path = path
1279 self._last_parent_path = tuple(self._get_parent_path())
1280 self._last_epoch = self._epoch
1281 self._path_finder = path_finder
1282
1283 def _find_parent_path_names(self):
1284 """Returns a tuple of (parent-module-name, parent-path-attr-name)"""
1285 parent, dot, me = self._name.rpartition('.')
1286 if dot == '':
1287 # This is a top-level module. sys.path contains the parent path.
1288 return 'sys', 'path'
1289 # Not a top-level module. parent-module.__path__ contains the
1290 # parent path.
1291 return parent, '__path__'
1292
1293 def _get_parent_path(self):
1294 parent_module_name, path_attr_name = self._find_parent_path_names()
1295 return getattr(sys.modules[parent_module_name], path_attr_name)
1296
1297 def _recalculate(self):
1298 # If the parent's path has changed, recalculate _path
1299 parent_path = tuple(self._get_parent_path()) # Make a copy
1300 if parent_path != self._last_parent_path or self._epoch != self._last_epoch:
1301 spec = self._path_finder(self._name, parent_path)
1302 # Note that no changes are made if a loader is returned, but we
1303 # do remember the new parent path
1304 if spec is not None and spec.loader is None:
1305 if spec.submodule_search_locations:
1306 self._path = spec.submodule_search_locations
1307 self._last_parent_path = parent_path # Save the copy
1308 self._last_epoch = self._epoch
1309 return self._path
1310
1311 def __iter__(self):
1312 return iter(self._recalculate())
1313
1314 def __getitem__(self, index):
1315 return self._recalculate()[index]
1316
1317 def __setitem__(self, index, path):
1318 self._path[index] = path
1319
1320 def __len__(self):
1321 return len(self._recalculate())
1322
1323 def __repr__(self):
1324 return '_NamespacePath({!r})'.format(self._path)
1325
1326 def __contains__(self, item):
1327 return item in self._recalculate()
1328
1329 def append(self, item):
1330 self._path.append(item)
1331
1332
1333 # This class is actually exposed publicly in a namespace package's __loader__
1334 # attribute, so it should be available through a non-private name.
1335 # https://bugs.python.org/issue35673
1336 class ESC[4;38;5;81mNamespaceLoader:
1337 def __init__(self, name, path, path_finder):
1338 self._path = _NamespacePath(name, path, path_finder)
1339
1340 @staticmethod
1341 def module_repr(module):
1342 """Return repr for the module.
1343
1344 The method is deprecated. The import machinery does the job itself.
1345
1346 """
1347 _warnings.warn("NamespaceLoader.module_repr() is deprecated and "
1348 "slated for removal in Python 3.12", DeprecationWarning)
1349 return '<module {!r} (namespace)>'.format(module.__name__)
1350
1351 def is_package(self, fullname):
1352 return True
1353
1354 def get_source(self, fullname):
1355 return ''
1356
1357 def get_code(self, fullname):
1358 return compile('', '<string>', 'exec', dont_inherit=True)
1359
1360 def create_module(self, spec):
1361 """Use default semantics for module creation."""
1362
1363 def exec_module(self, module):
1364 pass
1365
1366 def load_module(self, fullname):
1367 """Load a namespace module.
1368
1369 This method is deprecated. Use exec_module() instead.
1370
1371 """
1372 # The import system never calls this method.
1373 _bootstrap._verbose_message('namespace module loaded with path {!r}',
1374 self._path)
1375 # Warning implemented in _load_module_shim().
1376 return _bootstrap._load_module_shim(self, fullname)
1377
1378 def get_resource_reader(self, module):
1379 from importlib.readers import NamespaceReader
1380 return NamespaceReader(self._path)
1381
1382
1383 # We use this exclusively in module_from_spec() for backward-compatibility.
1384 _NamespaceLoader = NamespaceLoader
1385
1386
1387 # Finders #####################################################################
1388
1389 class ESC[4;38;5;81mPathFinder:
1390
1391 """Meta path finder for sys.path and package __path__ attributes."""
1392
1393 @staticmethod
1394 def invalidate_caches():
1395 """Call the invalidate_caches() method on all path entry finders
1396 stored in sys.path_importer_caches (where implemented)."""
1397 for name, finder in list(sys.path_importer_cache.items()):
1398 # Drop entry if finder name is a relative path. The current
1399 # working directory may have changed.
1400 if finder is None or not _path_isabs(name):
1401 del sys.path_importer_cache[name]
1402 elif hasattr(finder, 'invalidate_caches'):
1403 finder.invalidate_caches()
1404 # Also invalidate the caches of _NamespacePaths
1405 # https://bugs.python.org/issue45703
1406 _NamespacePath._epoch += 1
1407
1408 @staticmethod
1409 def _path_hooks(path):
1410 """Search sys.path_hooks for a finder for 'path'."""
1411 if sys.path_hooks is not None and not sys.path_hooks:
1412 _warnings.warn('sys.path_hooks is empty', ImportWarning)
1413 for hook in sys.path_hooks:
1414 try:
1415 return hook(path)
1416 except ImportError:
1417 continue
1418 else:
1419 return None
1420
1421 @classmethod
1422 def _path_importer_cache(cls, path):
1423 """Get the finder for the path entry from sys.path_importer_cache.
1424
1425 If the path entry is not in the cache, find the appropriate finder
1426 and cache it. If no finder is available, store None.
1427
1428 """
1429 if path == '':
1430 try:
1431 path = _os.getcwd()
1432 except FileNotFoundError:
1433 # Don't cache the failure as the cwd can easily change to
1434 # a valid directory later on.
1435 return None
1436 try:
1437 finder = sys.path_importer_cache[path]
1438 except KeyError:
1439 finder = cls._path_hooks(path)
1440 sys.path_importer_cache[path] = finder
1441 return finder
1442
1443 @classmethod
1444 def _legacy_get_spec(cls, fullname, finder):
1445 # This would be a good place for a DeprecationWarning if
1446 # we ended up going that route.
1447 if hasattr(finder, 'find_loader'):
1448 msg = (f"{_bootstrap._object_name(finder)}.find_spec() not found; "
1449 "falling back to find_loader()")
1450 _warnings.warn(msg, ImportWarning)
1451 loader, portions = finder.find_loader(fullname)
1452 else:
1453 msg = (f"{_bootstrap._object_name(finder)}.find_spec() not found; "
1454 "falling back to find_module()")
1455 _warnings.warn(msg, ImportWarning)
1456 loader = finder.find_module(fullname)
1457 portions = []
1458 if loader is not None:
1459 return _bootstrap.spec_from_loader(fullname, loader)
1460 spec = _bootstrap.ModuleSpec(fullname, None)
1461 spec.submodule_search_locations = portions
1462 return spec
1463
1464 @classmethod
1465 def _get_spec(cls, fullname, path, target=None):
1466 """Find the loader or namespace_path for this module/package name."""
1467 # If this ends up being a namespace package, namespace_path is
1468 # the list of paths that will become its __path__
1469 namespace_path = []
1470 for entry in path:
1471 if not isinstance(entry, str):
1472 continue
1473 finder = cls._path_importer_cache(entry)
1474 if finder is not None:
1475 if hasattr(finder, 'find_spec'):
1476 spec = finder.find_spec(fullname, target)
1477 else:
1478 spec = cls._legacy_get_spec(fullname, finder)
1479 if spec is None:
1480 continue
1481 if spec.loader is not None:
1482 return spec
1483 portions = spec.submodule_search_locations
1484 if portions is None:
1485 raise ImportError('spec missing loader')
1486 # This is possibly part of a namespace package.
1487 # Remember these path entries (if any) for when we
1488 # create a namespace package, and continue iterating
1489 # on path.
1490 namespace_path.extend(portions)
1491 else:
1492 spec = _bootstrap.ModuleSpec(fullname, None)
1493 spec.submodule_search_locations = namespace_path
1494 return spec
1495
1496 @classmethod
1497 def find_spec(cls, fullname, path=None, target=None):
1498 """Try to find a spec for 'fullname' on sys.path or 'path'.
1499
1500 The search is based on sys.path_hooks and sys.path_importer_cache.
1501 """
1502 if path is None:
1503 path = sys.path
1504 spec = cls._get_spec(fullname, path, target)
1505 if spec is None:
1506 return None
1507 elif spec.loader is None:
1508 namespace_path = spec.submodule_search_locations
1509 if namespace_path:
1510 # We found at least one namespace path. Return a spec which
1511 # can create the namespace package.
1512 spec.origin = None
1513 spec.submodule_search_locations = _NamespacePath(fullname, namespace_path, cls._get_spec)
1514 return spec
1515 else:
1516 return None
1517 else:
1518 return spec
1519
1520 @classmethod
1521 def find_module(cls, fullname, path=None):
1522 """find the module on sys.path or 'path' based on sys.path_hooks and
1523 sys.path_importer_cache.
1524
1525 This method is deprecated. Use find_spec() instead.
1526
1527 """
1528 _warnings.warn("PathFinder.find_module() is deprecated and "
1529 "slated for removal in Python 3.12; use find_spec() instead",
1530 DeprecationWarning)
1531 spec = cls.find_spec(fullname, path)
1532 if spec is None:
1533 return None
1534 return spec.loader
1535
1536 @staticmethod
1537 def find_distributions(*args, **kwargs):
1538 """
1539 Find distributions.
1540
1541 Return an iterable of all Distribution instances capable of
1542 loading the metadata for packages matching ``context.name``
1543 (or all names if ``None`` indicated) along the paths in the list
1544 of directories ``context.path``.
1545 """
1546 from importlib.metadata import MetadataPathFinder
1547 return MetadataPathFinder.find_distributions(*args, **kwargs)
1548
1549
1550 class ESC[4;38;5;81mFileFinder:
1551
1552 """File-based finder.
1553
1554 Interactions with the file system are cached for performance, being
1555 refreshed when the directory the finder is handling has been modified.
1556
1557 """
1558
1559 def __init__(self, path, *loader_details):
1560 """Initialize with the path to search on and a variable number of
1561 2-tuples containing the loader and the file suffixes the loader
1562 recognizes."""
1563 loaders = []
1564 for loader, suffixes in loader_details:
1565 loaders.extend((suffix, loader) for suffix in suffixes)
1566 self._loaders = loaders
1567 # Base (directory) path
1568 if not path or path == '.':
1569 self.path = _os.getcwd()
1570 elif not _path_isabs(path):
1571 self.path = _path_join(_os.getcwd(), path)
1572 else:
1573 self.path = path
1574 self._path_mtime = -1
1575 self._path_cache = set()
1576 self._relaxed_path_cache = set()
1577
1578 def invalidate_caches(self):
1579 """Invalidate the directory mtime."""
1580 self._path_mtime = -1
1581
1582 find_module = _find_module_shim
1583
1584 def find_loader(self, fullname):
1585 """Try to find a loader for the specified module, or the namespace
1586 package portions. Returns (loader, list-of-portions).
1587
1588 This method is deprecated. Use find_spec() instead.
1589
1590 """
1591 _warnings.warn("FileFinder.find_loader() is deprecated and "
1592 "slated for removal in Python 3.12; use find_spec() instead",
1593 DeprecationWarning)
1594 spec = self.find_spec(fullname)
1595 if spec is None:
1596 return None, []
1597 return spec.loader, spec.submodule_search_locations or []
1598
1599 def _get_spec(self, loader_class, fullname, path, smsl, target):
1600 loader = loader_class(fullname, path)
1601 return spec_from_file_location(fullname, path, loader=loader,
1602 submodule_search_locations=smsl)
1603
1604 def find_spec(self, fullname, target=None):
1605 """Try to find a spec for the specified module.
1606
1607 Returns the matching spec, or None if not found.
1608 """
1609 is_namespace = False
1610 tail_module = fullname.rpartition('.')[2]
1611 try:
1612 mtime = _path_stat(self.path or _os.getcwd()).st_mtime
1613 except OSError:
1614 mtime = -1
1615 if mtime != self._path_mtime:
1616 self._fill_cache()
1617 self._path_mtime = mtime
1618 # tail_module keeps the original casing, for __file__ and friends
1619 if _relax_case():
1620 cache = self._relaxed_path_cache
1621 cache_module = tail_module.lower()
1622 else:
1623 cache = self._path_cache
1624 cache_module = tail_module
1625 # Check if the module is the name of a directory (and thus a package).
1626 if cache_module in cache:
1627 base_path = _path_join(self.path, tail_module)
1628 for suffix, loader_class in self._loaders:
1629 init_filename = '__init__' + suffix
1630 full_path = _path_join(base_path, init_filename)
1631 if _path_isfile(full_path):
1632 return self._get_spec(loader_class, fullname, full_path, [base_path], target)
1633 else:
1634 # If a namespace package, return the path if we don't
1635 # find a module in the next section.
1636 is_namespace = _path_isdir(base_path)
1637 # Check for a file w/ a proper suffix exists.
1638 for suffix, loader_class in self._loaders:
1639 try:
1640 full_path = _path_join(self.path, tail_module + suffix)
1641 except ValueError:
1642 return None
1643 _bootstrap._verbose_message('trying {}', full_path, verbosity=2)
1644 if cache_module + suffix in cache:
1645 if _path_isfile(full_path):
1646 return self._get_spec(loader_class, fullname, full_path,
1647 None, target)
1648 if is_namespace:
1649 _bootstrap._verbose_message('possible namespace for {}', base_path)
1650 spec = _bootstrap.ModuleSpec(fullname, None)
1651 spec.submodule_search_locations = [base_path]
1652 return spec
1653 return None
1654
1655 def _fill_cache(self):
1656 """Fill the cache of potential modules and packages for this directory."""
1657 path = self.path
1658 try:
1659 contents = _os.listdir(path or _os.getcwd())
1660 except (FileNotFoundError, PermissionError, NotADirectoryError):
1661 # Directory has either been removed, turned into a file, or made
1662 # unreadable.
1663 contents = []
1664 # We store two cached versions, to handle runtime changes of the
1665 # PYTHONCASEOK environment variable.
1666 if not sys.platform.startswith('win'):
1667 self._path_cache = set(contents)
1668 else:
1669 # Windows users can import modules with case-insensitive file
1670 # suffixes (for legacy reasons). Make the suffix lowercase here
1671 # so it's done once instead of for every import. This is safe as
1672 # the specified suffixes to check against are always specified in a
1673 # case-sensitive manner.
1674 lower_suffix_contents = set()
1675 for item in contents:
1676 name, dot, suffix = item.partition('.')
1677 if dot:
1678 new_name = '{}.{}'.format(name, suffix.lower())
1679 else:
1680 new_name = name
1681 lower_suffix_contents.add(new_name)
1682 self._path_cache = lower_suffix_contents
1683 if sys.platform.startswith(_CASE_INSENSITIVE_PLATFORMS):
1684 self._relaxed_path_cache = {fn.lower() for fn in contents}
1685
1686 @classmethod
1687 def path_hook(cls, *loader_details):
1688 """A class method which returns a closure to use on sys.path_hook
1689 which will return an instance using the specified loaders and the path
1690 called on the closure.
1691
1692 If the path called on the closure is not a directory, ImportError is
1693 raised.
1694
1695 """
1696 def path_hook_for_FileFinder(path):
1697 """Path hook for importlib.machinery.FileFinder."""
1698 if not _path_isdir(path):
1699 raise ImportError('only directories are supported', path=path)
1700 return cls(path, *loader_details)
1701
1702 return path_hook_for_FileFinder
1703
1704 def __repr__(self):
1705 return 'FileFinder({!r})'.format(self.path)
1706
1707
1708 # Import setup ###############################################################
1709
1710 def _fix_up_module(ns, name, pathname, cpathname=None):
1711 # This function is used by PyImport_ExecCodeModuleObject().
1712 loader = ns.get('__loader__')
1713 spec = ns.get('__spec__')
1714 if not loader:
1715 if spec:
1716 loader = spec.loader
1717 elif pathname == cpathname:
1718 loader = SourcelessFileLoader(name, pathname)
1719 else:
1720 loader = SourceFileLoader(name, pathname)
1721 if not spec:
1722 spec = spec_from_file_location(name, pathname, loader=loader)
1723 try:
1724 ns['__spec__'] = spec
1725 ns['__loader__'] = loader
1726 ns['__file__'] = pathname
1727 ns['__cached__'] = cpathname
1728 except Exception:
1729 # Not important enough to report.
1730 pass
1731
1732
1733 def _get_supported_file_loaders():
1734 """Returns a list of file-based module loaders.
1735
1736 Each item is a tuple (loader, suffixes).
1737 """
1738 extensions = ExtensionFileLoader, _imp.extension_suffixes()
1739 source = SourceFileLoader, SOURCE_SUFFIXES
1740 bytecode = SourcelessFileLoader, BYTECODE_SUFFIXES
1741 return [extensions, source, bytecode]
1742
1743
1744 def _set_bootstrap_module(_bootstrap_module):
1745 global _bootstrap
1746 _bootstrap = _bootstrap_module
1747
1748
1749 def _install(_bootstrap_module):
1750 """Install the path-based import components."""
1751 _set_bootstrap_module(_bootstrap_module)
1752 supported_loaders = _get_supported_file_loaders()
1753 sys.path_hooks.extend([FileFinder.path_hook(*supported_loaders)])
1754 sys.meta_path.append(PathFinder)