1 """Core implementation of path-based import.
2
3 This module is NOT meant to be directly imported! It has been designed such
4 that it can be bootstrapped into Python as the implementation of import. As
5 such it requires the injection of specific modules and attributes in order to
6 work. One should use importlib as the public-facing version of this module.
7
8 """
9 # IMPORTANT: Whenever making changes to this module, be sure to run a top-level
10 # `make regen-importlib` followed by `make` in order to get the frozen version
11 # of the module updated. Not doing so will result in the Makefile to fail for
12 # all others who don't have a ./python around to freeze the module in the early
13 # stages of compilation.
14 #
15
16 # See importlib._setup() for what is injected into the global namespace.
17
18 # When editing this code be aware that code executed at import time CANNOT
19 # reference any injected objects! This includes not only global code but also
20 # anything specified at the class level.
21
22 # Module injected manually by _set_bootstrap_module()
23 _bootstrap = None
24
25 # Import builtin modules
26 import _imp
27 import _io
28 import sys
29 import _warnings
30 import marshal
31
32
33 _MS_WINDOWS = (sys.platform == 'win32')
34 if _MS_WINDOWS:
35 import nt as _os
36 import winreg
37 else:
38 import posix as _os
39
40
41 if _MS_WINDOWS:
42 path_separators = ['\\', '/']
43 else:
44 path_separators = ['/']
45 # Assumption made in _path_join()
46 assert all(len(sep) == 1 for sep in path_separators)
47 path_sep = path_separators[0]
48 path_sep_tuple = tuple(path_separators)
49 path_separators = ''.join(path_separators)
50 _pathseps_with_colon = {f':{s}' for s in path_separators}
51
52
53 # Bootstrap-related code ######################################################
54 _CASE_INSENSITIVE_PLATFORMS_STR_KEY = 'win',
55 _CASE_INSENSITIVE_PLATFORMS_BYTES_KEY = 'cygwin', 'darwin'
56 _CASE_INSENSITIVE_PLATFORMS = (_CASE_INSENSITIVE_PLATFORMS_BYTES_KEY
57 + _CASE_INSENSITIVE_PLATFORMS_STR_KEY)
58
59
60 def _make_relax_case():
61 if sys.platform.startswith(_CASE_INSENSITIVE_PLATFORMS):
62 if sys.platform.startswith(_CASE_INSENSITIVE_PLATFORMS_STR_KEY):
63 key = 'PYTHONCASEOK'
64 else:
65 key = b'PYTHONCASEOK'
66
67 def _relax_case():
68 """True if filenames must be checked case-insensitively and ignore environment flags are not set."""
69 return not sys.flags.ignore_environment and key in _os.environ
70 else:
71 def _relax_case():
72 """True if filenames must be checked case-insensitively."""
73 return False
74 return _relax_case
75
76 _relax_case = _make_relax_case()
77
78
79 def _pack_uint32(x):
80 """Convert a 32-bit integer to little-endian."""
81 return (int(x) & 0xFFFFFFFF).to_bytes(4, 'little')
82
83
84 def _unpack_uint32(data):
85 """Convert 4 bytes in little-endian to an integer."""
86 assert len(data) == 4
87 return int.from_bytes(data, 'little')
88
89 def _unpack_uint16(data):
90 """Convert 2 bytes in little-endian to an integer."""
91 assert len(data) == 2
92 return int.from_bytes(data, 'little')
93
94
95 if _MS_WINDOWS:
96 def _path_join(*path_parts):
97 """Replacement for os.path.join()."""
98 if not path_parts:
99 return ""
100 if len(path_parts) == 1:
101 return path_parts[0]
102 root = ""
103 path = []
104 for new_root, tail in map(_os._path_splitroot, path_parts):
105 if new_root.startswith(path_sep_tuple) or new_root.endswith(path_sep_tuple):
106 root = new_root.rstrip(path_separators) or root
107 path = [path_sep + tail]
108 elif new_root.endswith(':'):
109 if root.casefold() != new_root.casefold():
110 # Drive relative paths have to be resolved by the OS, so we reset the
111 # tail but do not add a path_sep prefix.
112 root = new_root
113 path = [tail]
114 else:
115 path.append(tail)
116 else:
117 root = new_root or root
118 path.append(tail)
119 path = [p.rstrip(path_separators) for p in path if p]
120 if len(path) == 1 and not path[0]:
121 # Avoid losing the root's trailing separator when joining with nothing
122 return root + path_sep
123 return root + path_sep.join(path)
124
125 else:
126 def _path_join(*path_parts):
127 """Replacement for os.path.join()."""
128 return path_sep.join([part.rstrip(path_separators)
129 for part in path_parts if part])
130
131
132 def _path_split(path):
133 """Replacement for os.path.split()."""
134 i = max(path.rfind(p) for p in path_separators)
135 if i < 0:
136 return '', path
137 return path[:i], path[i + 1:]
138
139
140 def _path_stat(path):
141 """Stat the path.
142
143 Made a separate function to make it easier to override in experiments
144 (e.g. cache stat results).
145
146 """
147 return _os.stat(path)
148
149
150 def _path_is_mode_type(path, mode):
151 """Test whether the path is the specified mode type."""
152 try:
153 stat_info = _path_stat(path)
154 except OSError:
155 return False
156 return (stat_info.st_mode & 0o170000) == mode
157
158
159 def _path_isfile(path):
160 """Replacement for os.path.isfile."""
161 return _path_is_mode_type(path, 0o100000)
162
163
164 def _path_isdir(path):
165 """Replacement for os.path.isdir."""
166 if not path:
167 path = _os.getcwd()
168 return _path_is_mode_type(path, 0o040000)
169
170
171 if _MS_WINDOWS:
172 def _path_isabs(path):
173 """Replacement for os.path.isabs."""
174 if not path:
175 return False
176 root = _os._path_splitroot(path)[0].replace('/', '\\')
177 return len(root) > 1 and (root.startswith('\\\\') or root.endswith('\\'))
178
179 else:
180 def _path_isabs(path):
181 """Replacement for os.path.isabs."""
182 return path.startswith(path_separators)
183
184
185 def _path_abspath(path):
186 """Replacement for os.path.abspath."""
187 if not _path_isabs(path):
188 for sep in path_separators:
189 path = path.removeprefix(f".{sep}")
190 return _path_join(_os.getcwd(), path)
191 else:
192 return path
193
194
195 def _write_atomic(path, data, mode=0o666):
196 """Best-effort function to write data to a path atomically.
197 Be prepared to handle a FileExistsError if concurrent writing of the
198 temporary file is attempted."""
199 # id() is used to generate a pseudo-random filename.
200 path_tmp = f'{path}.{id(path)}'
201 fd = _os.open(path_tmp,
202 _os.O_EXCL | _os.O_CREAT | _os.O_WRONLY, mode & 0o666)
203 try:
204 # We first write data to a temporary file, and then use os.replace() to
205 # perform an atomic rename.
206 with _io.FileIO(fd, 'wb') as file:
207 file.write(data)
208 _os.replace(path_tmp, path)
209 except OSError:
210 try:
211 _os.unlink(path_tmp)
212 except OSError:
213 pass
214 raise
215
216
217 _code_type = type(_write_atomic.__code__)
218
219
220 # Finder/loader utility code ###############################################
221
222 # Magic word to reject .pyc files generated by other Python versions.
223 # It should change for each incompatible change to the bytecode.
224 #
225 # The value of CR and LF is incorporated so if you ever read or write
226 # a .pyc file in text mode the magic number will be wrong; also, the
227 # Apple MPW compiler swaps their values, botching string constants.
228 #
229 # There were a variety of old schemes for setting the magic number.
230 # The current working scheme is to increment the previous value by
231 # 10.
232 #
233 # Starting with the adoption of PEP 3147 in Python 3.2, every bump in magic
234 # number also includes a new "magic tag", i.e. a human readable string used
235 # to represent the magic number in __pycache__ directories. When you change
236 # the magic number, you must also set a new unique magic tag. Generally this
237 # can be named after the Python major version of the magic number bump, but
238 # it can really be anything, as long as it's different than anything else
239 # that's come before. The tags are included in the following table, starting
240 # with Python 3.2a0.
241 #
242 # Known values:
243 # Python 1.5: 20121
244 # Python 1.5.1: 20121
245 # Python 1.5.2: 20121
246 # Python 1.6: 50428
247 # Python 2.0: 50823
248 # Python 2.0.1: 50823
249 # Python 2.1: 60202
250 # Python 2.1.1: 60202
251 # Python 2.1.2: 60202
252 # Python 2.2: 60717
253 # Python 2.3a0: 62011
254 # Python 2.3a0: 62021
255 # Python 2.3a0: 62011 (!)
256 # Python 2.4a0: 62041
257 # Python 2.4a3: 62051
258 # Python 2.4b1: 62061
259 # Python 2.5a0: 62071
260 # Python 2.5a0: 62081 (ast-branch)
261 # Python 2.5a0: 62091 (with)
262 # Python 2.5a0: 62092 (changed WITH_CLEANUP opcode)
263 # Python 2.5b3: 62101 (fix wrong code: for x, in ...)
264 # Python 2.5b3: 62111 (fix wrong code: x += yield)
265 # Python 2.5c1: 62121 (fix wrong lnotab with for loops and
266 # storing constants that should have been removed)
267 # Python 2.5c2: 62131 (fix wrong code: for x, in ... in listcomp/genexp)
268 # Python 2.6a0: 62151 (peephole optimizations and STORE_MAP opcode)
269 # Python 2.6a1: 62161 (WITH_CLEANUP optimization)
270 # Python 2.7a0: 62171 (optimize list comprehensions/change LIST_APPEND)
271 # Python 2.7a0: 62181 (optimize conditional branches:
272 # introduce POP_JUMP_IF_FALSE and POP_JUMP_IF_TRUE)
273 # Python 2.7a0 62191 (introduce SETUP_WITH)
274 # Python 2.7a0 62201 (introduce BUILD_SET)
275 # Python 2.7a0 62211 (introduce MAP_ADD and SET_ADD)
276 # Python 3000: 3000
277 # 3010 (removed UNARY_CONVERT)
278 # 3020 (added BUILD_SET)
279 # 3030 (added keyword-only parameters)
280 # 3040 (added signature annotations)
281 # 3050 (print becomes a function)
282 # 3060 (PEP 3115 metaclass syntax)
283 # 3061 (string literals become unicode)
284 # 3071 (PEP 3109 raise changes)
285 # 3081 (PEP 3137 make __file__ and __name__ unicode)
286 # 3091 (kill str8 interning)
287 # 3101 (merge from 2.6a0, see 62151)
288 # 3103 (__file__ points to source file)
289 # Python 3.0a4: 3111 (WITH_CLEANUP optimization).
290 # Python 3.0b1: 3131 (lexical exception stacking, including POP_EXCEPT
291 #3021)
292 # Python 3.1a1: 3141 (optimize list, set and dict comprehensions:
293 # change LIST_APPEND and SET_ADD, add MAP_ADD #2183)
294 # Python 3.1a1: 3151 (optimize conditional branches:
295 # introduce POP_JUMP_IF_FALSE and POP_JUMP_IF_TRUE
296 #4715)
297 # Python 3.2a1: 3160 (add SETUP_WITH #6101)
298 # tag: cpython-32
299 # Python 3.2a2: 3170 (add DUP_TOP_TWO, remove DUP_TOPX and ROT_FOUR #9225)
300 # tag: cpython-32
301 # Python 3.2a3 3180 (add DELETE_DEREF #4617)
302 # Python 3.3a1 3190 (__class__ super closure changed)
303 # Python 3.3a1 3200 (PEP 3155 __qualname__ added #13448)
304 # Python 3.3a1 3210 (added size modulo 2**32 to the pyc header #13645)
305 # Python 3.3a2 3220 (changed PEP 380 implementation #14230)
306 # Python 3.3a4 3230 (revert changes to implicit __class__ closure #14857)
307 # Python 3.4a1 3250 (evaluate positional default arguments before
308 # keyword-only defaults #16967)
309 # Python 3.4a1 3260 (add LOAD_CLASSDEREF; allow locals of class to override
310 # free vars #17853)
311 # Python 3.4a1 3270 (various tweaks to the __class__ closure #12370)
312 # Python 3.4a1 3280 (remove implicit class argument)
313 # Python 3.4a4 3290 (changes to __qualname__ computation #19301)
314 # Python 3.4a4 3300 (more changes to __qualname__ computation #19301)
315 # Python 3.4rc2 3310 (alter __qualname__ computation #20625)
316 # Python 3.5a1 3320 (PEP 465: Matrix multiplication operator #21176)
317 # Python 3.5b1 3330 (PEP 448: Additional Unpacking Generalizations #2292)
318 # Python 3.5b2 3340 (fix dictionary display evaluation order #11205)
319 # Python 3.5b3 3350 (add GET_YIELD_FROM_ITER opcode #24400)
320 # Python 3.5.2 3351 (fix BUILD_MAP_UNPACK_WITH_CALL opcode #27286)
321 # Python 3.6a0 3360 (add FORMAT_VALUE opcode #25483)
322 # Python 3.6a1 3361 (lineno delta of code.co_lnotab becomes signed #26107)
323 # Python 3.6a2 3370 (16 bit wordcode #26647)
324 # Python 3.6a2 3371 (add BUILD_CONST_KEY_MAP opcode #27140)
325 # Python 3.6a2 3372 (MAKE_FUNCTION simplification, remove MAKE_CLOSURE
326 # #27095)
327 # Python 3.6b1 3373 (add BUILD_STRING opcode #27078)
328 # Python 3.6b1 3375 (add SETUP_ANNOTATIONS and STORE_ANNOTATION opcodes
329 # #27985)
330 # Python 3.6b1 3376 (simplify CALL_FUNCTIONs & BUILD_MAP_UNPACK_WITH_CALL
331 #27213)
332 # Python 3.6b1 3377 (set __class__ cell from type.__new__ #23722)
333 # Python 3.6b2 3378 (add BUILD_TUPLE_UNPACK_WITH_CALL #28257)
334 # Python 3.6rc1 3379 (more thorough __class__ validation #23722)
335 # Python 3.7a1 3390 (add LOAD_METHOD and CALL_METHOD opcodes #26110)
336 # Python 3.7a2 3391 (update GET_AITER #31709)
337 # Python 3.7a4 3392 (PEP 552: Deterministic pycs #31650)
338 # Python 3.7b1 3393 (remove STORE_ANNOTATION opcode #32550)
339 # Python 3.7b5 3394 (restored docstring as the first stmt in the body;
340 # this might affected the first line number #32911)
341 # Python 3.8a1 3400 (move frame block handling to compiler #17611)
342 # Python 3.8a1 3401 (add END_ASYNC_FOR #33041)
343 # Python 3.8a1 3410 (PEP570 Python Positional-Only Parameters #36540)
344 # Python 3.8b2 3411 (Reverse evaluation order of key: value in dict
345 # comprehensions #35224)
346 # Python 3.8b2 3412 (Swap the position of positional args and positional
347 # only args in ast.arguments #37593)
348 # Python 3.8b4 3413 (Fix "break" and "continue" in "finally" #37830)
349 # Python 3.9a0 3420 (add LOAD_ASSERTION_ERROR #34880)
350 # Python 3.9a0 3421 (simplified bytecode for with blocks #32949)
351 # Python 3.9a0 3422 (remove BEGIN_FINALLY, END_FINALLY, CALL_FINALLY, POP_FINALLY bytecodes #33387)
352 # Python 3.9a2 3423 (add IS_OP, CONTAINS_OP and JUMP_IF_NOT_EXC_MATCH bytecodes #39156)
353 # Python 3.9a2 3424 (simplify bytecodes for *value unpacking)
354 # Python 3.9a2 3425 (simplify bytecodes for **value unpacking)
355 # Python 3.10a1 3430 (Make 'annotations' future by default)
356 # Python 3.10a1 3431 (New line number table format -- PEP 626)
357 # Python 3.10a2 3432 (Function annotation for MAKE_FUNCTION is changed from dict to tuple bpo-42202)
358 # Python 3.10a2 3433 (RERAISE restores f_lasti if oparg != 0)
359 # Python 3.10a6 3434 (PEP 634: Structural Pattern Matching)
360 # Python 3.10a7 3435 Use instruction offsets (as opposed to byte offsets).
361 # Python 3.10b1 3436 (Add GEN_START bytecode #43683)
362 # Python 3.10b1 3437 (Undo making 'annotations' future by default - We like to dance among core devs!)
363 # Python 3.10b1 3438 Safer line number table handling.
364 # Python 3.10b1 3439 (Add ROT_N)
365 # Python 3.11a1 3450 Use exception table for unwinding ("zero cost" exception handling)
366 # Python 3.11a1 3451 (Add CALL_METHOD_KW)
367 # Python 3.11a1 3452 (drop nlocals from marshaled code objects)
368 # Python 3.11a1 3453 (add co_fastlocalnames and co_fastlocalkinds)
369 # Python 3.11a1 3454 (compute cell offsets relative to locals bpo-43693)
370 # Python 3.11a1 3455 (add MAKE_CELL bpo-43693)
371 # Python 3.11a1 3456 (interleave cell args bpo-43693)
372 # Python 3.11a1 3457 (Change localsplus to a bytes object bpo-43693)
373 # Python 3.11a1 3458 (imported objects now don't use LOAD_METHOD/CALL_METHOD)
374 # Python 3.11a1 3459 (PEP 657: add end line numbers and column offsets for instructions)
375 # Python 3.11a1 3460 (Add co_qualname field to PyCodeObject bpo-44530)
376 # Python 3.11a1 3461 (JUMP_ABSOLUTE must jump backwards)
377 # Python 3.11a2 3462 (bpo-44511: remove COPY_DICT_WITHOUT_KEYS, change
378 # MATCH_CLASS and MATCH_KEYS, and add COPY)
379 # Python 3.11a3 3463 (bpo-45711: JUMP_IF_NOT_EXC_MATCH no longer pops the
380 # active exception)
381 # Python 3.11a3 3464 (bpo-45636: Merge numeric BINARY_*/INPLACE_* into
382 # BINARY_OP)
383 # Python 3.11a3 3465 (Add COPY_FREE_VARS opcode)
384 # Python 3.11a4 3466 (bpo-45292: PEP-654 except*)
385 # Python 3.11a4 3467 (Change CALL_xxx opcodes)
386 # Python 3.11a4 3468 (Add SEND opcode)
387 # Python 3.11a4 3469 (bpo-45711: remove type, traceback from exc_info)
388 # Python 3.11a4 3470 (bpo-46221: PREP_RERAISE_STAR no longer pushes lasti)
389 # Python 3.11a4 3471 (bpo-46202: remove pop POP_EXCEPT_AND_RERAISE)
390 # Python 3.11a4 3472 (bpo-46009: replace GEN_START with POP_TOP)
391 # Python 3.11a4 3473 (Add POP_JUMP_IF_NOT_NONE/POP_JUMP_IF_NONE opcodes)
392 # Python 3.11a4 3474 (Add RESUME opcode)
393 # Python 3.11a5 3475 (Add RETURN_GENERATOR opcode)
394 # Python 3.11a5 3476 (Add ASYNC_GEN_WRAP opcode)
395 # Python 3.11a5 3477 (Replace DUP_TOP/DUP_TOP_TWO with COPY and
396 # ROT_TWO/ROT_THREE/ROT_FOUR/ROT_N with SWAP)
397 # Python 3.11a5 3478 (New CALL opcodes)
398 # Python 3.11a5 3479 (Add PUSH_NULL opcode)
399 # Python 3.11a5 3480 (New CALL opcodes, second iteration)
400 # Python 3.11a5 3481 (Use inline cache for BINARY_OP)
401 # Python 3.11a5 3482 (Use inline caching for UNPACK_SEQUENCE and LOAD_GLOBAL)
402 # Python 3.11a5 3483 (Use inline caching for COMPARE_OP and BINARY_SUBSCR)
403 # Python 3.11a5 3484 (Use inline caching for LOAD_ATTR, LOAD_METHOD, and
404 # STORE_ATTR)
405 # Python 3.11a5 3485 (Add an oparg to GET_AWAITABLE)
406 # Python 3.11a6 3486 (Use inline caching for PRECALL and CALL)
407 # Python 3.11a6 3487 (Remove the adaptive "oparg counter" mechanism)
408 # Python 3.11a6 3488 (LOAD_GLOBAL can push additional NULL)
409 # Python 3.11a6 3489 (Add JUMP_BACKWARD, remove JUMP_ABSOLUTE)
410 # Python 3.11a6 3490 (remove JUMP_IF_NOT_EXC_MATCH, add CHECK_EXC_MATCH)
411 # Python 3.11a6 3491 (remove JUMP_IF_NOT_EG_MATCH, add CHECK_EG_MATCH,
412 # add JUMP_BACKWARD_NO_INTERRUPT, make JUMP_NO_INTERRUPT virtual)
413 # Python 3.11a7 3492 (make POP_JUMP_IF_NONE/NOT_NONE/TRUE/FALSE relative)
414 # Python 3.11a7 3493 (Make JUMP_IF_TRUE_OR_POP/JUMP_IF_FALSE_OR_POP relative)
415 # Python 3.11a7 3494 (New location info table)
416 # Python 3.12a1 3500 (Remove PRECALL opcode)
417 # Python 3.12a1 3501 (YIELD_VALUE oparg == stack_depth)
418 # Python 3.12a1 3502 (LOAD_FAST_CHECK, no NULL-check in LOAD_FAST)
419 # Python 3.12a1 3503 (Shrink LOAD_METHOD cache)
420 # Python 3.12a1 3504 (Merge LOAD_METHOD back into LOAD_ATTR)
421 # Python 3.12a1 3505 (Specialization/Cache for FOR_ITER)
422 # Python 3.12a1 3506 (Add BINARY_SLICE and STORE_SLICE instructions)
423 # Python 3.12a1 3507 (Set lineno of module's RESUME to 0)
424 # Python 3.12a1 3508 (Add CLEANUP_THROW)
425 # Python 3.12a1 3509 (Conditional jumps only jump forward)
426 # Python 3.12a2 3510 (FOR_ITER leaves iterator on the stack)
427 # Python 3.12a2 3511 (Add STOPITERATION_ERROR instruction)
428 # Python 3.12a2 3512 (Remove all unused consts from code objects)
429 # Python 3.12a4 3513 (Add CALL_INTRINSIC_1 instruction, removed STOPITERATION_ERROR, PRINT_EXPR, IMPORT_STAR)
430 # Python 3.12a4 3514 (Remove ASYNC_GEN_WRAP, LIST_TO_TUPLE, and UNARY_POSITIVE)
431 # Python 3.12a5 3515 (Embed jump mask in COMPARE_OP oparg)
432 # Python 3.12a5 3516 (Add COMPARE_AND_BRANCH instruction)
433 # Python 3.12a5 3517 (Change YIELD_VALUE oparg to exception block depth)
434 # Python 3.12a6 3518 (Add RETURN_CONST instruction)
435 # Python 3.12a6 3519 (Modify SEND instruction)
436 # Python 3.12a6 3520 (Remove PREP_RERAISE_STAR, add CALL_INTRINSIC_2)
437 # Python 3.12a7 3521 (Shrink the LOAD_GLOBAL caches)
438 # Python 3.12a7 3522 (Removed JUMP_IF_FALSE_OR_POP/JUMP_IF_TRUE_OR_POP)
439 # Python 3.12a7 3523 (Convert COMPARE_AND_BRANCH back to COMPARE_OP)
440 # Python 3.12a7 3524 (Shrink the BINARY_SUBSCR caches)
441 # Python 3.12b1 3525 (Shrink the CALL caches)
442 # Python 3.12b1 3526 (Add instrumentation support)
443 # Python 3.12b1 3527 (Add LOAD_SUPER_ATTR)
444 # Python 3.12b1 3528 (Add LOAD_SUPER_ATTR_METHOD specialization)
445 # Python 3.12b1 3529 (Inline list/dict/set comprehensions)
446 # Python 3.12b1 3530 (Shrink the LOAD_SUPER_ATTR caches)
447 # Python 3.12b1 3531 (Add PEP 695 changes)
448
449 # Python 3.13 will start with 3550
450
451 # Please don't copy-paste the same pre-release tag for new entries above!!!
452 # You should always use the *upcoming* tag. For example, if 3.12a6 came out
453 # a week ago, I should put "Python 3.12a7" next to my new magic number.
454
455 # MAGIC must change whenever the bytecode emitted by the compiler may no
456 # longer be understood by older implementations of the eval loop (usually
457 # due to the addition of new opcodes).
458 #
459 # Starting with Python 3.11, Python 3.n starts with magic number 2900+50n.
460 #
461 # Whenever MAGIC_NUMBER is changed, the ranges in the magic_values array
462 # in PC/launcher.c must also be updated.
463
464 MAGIC_NUMBER = (3531).to_bytes(2, 'little') + b'\r\n'
465
466 _RAW_MAGIC_NUMBER = int.from_bytes(MAGIC_NUMBER, 'little') # For import.c
467
468 _PYCACHE = '__pycache__'
469 _OPT = 'opt-'
470
471 SOURCE_SUFFIXES = ['.py']
472 if _MS_WINDOWS:
473 SOURCE_SUFFIXES.append('.pyw')
474
475 EXTENSION_SUFFIXES = _imp.extension_suffixes()
476
477 BYTECODE_SUFFIXES = ['.pyc']
478 # Deprecated.
479 DEBUG_BYTECODE_SUFFIXES = OPTIMIZED_BYTECODE_SUFFIXES = BYTECODE_SUFFIXES
480
481 def cache_from_source(path, debug_override=None, *, optimization=None):
482 """Given the path to a .py file, return the path to its .pyc file.
483
484 The .py file does not need to exist; this simply returns the path to the
485 .pyc file calculated as if the .py file were imported.
486
487 The 'optimization' parameter controls the presumed optimization level of
488 the bytecode file. If 'optimization' is not None, the string representation
489 of the argument is taken and verified to be alphanumeric (else ValueError
490 is raised).
491
492 The debug_override parameter is deprecated. If debug_override is not None,
493 a True value is the same as setting 'optimization' to the empty string
494 while a False value is equivalent to setting 'optimization' to '1'.
495
496 If sys.implementation.cache_tag is None then NotImplementedError is raised.
497
498 """
499 if debug_override is not None:
500 _warnings.warn('the debug_override parameter is deprecated; use '
501 "'optimization' instead", DeprecationWarning)
502 if optimization is not None:
503 message = 'debug_override or optimization must be set to None'
504 raise TypeError(message)
505 optimization = '' if debug_override else 1
506 path = _os.fspath(path)
507 head, tail = _path_split(path)
508 base, sep, rest = tail.rpartition('.')
509 tag = sys.implementation.cache_tag
510 if tag is None:
511 raise NotImplementedError('sys.implementation.cache_tag is None')
512 almost_filename = ''.join([(base if base else rest), sep, tag])
513 if optimization is None:
514 if sys.flags.optimize == 0:
515 optimization = ''
516 else:
517 optimization = sys.flags.optimize
518 optimization = str(optimization)
519 if optimization != '':
520 if not optimization.isalnum():
521 raise ValueError(f'{optimization!r} is not alphanumeric')
522 almost_filename = f'{almost_filename}.{_OPT}{optimization}'
523 filename = almost_filename + BYTECODE_SUFFIXES[0]
524 if sys.pycache_prefix is not None:
525 # We need an absolute path to the py file to avoid the possibility of
526 # collisions within sys.pycache_prefix, if someone has two different
527 # `foo/bar.py` on their system and they import both of them using the
528 # same sys.pycache_prefix. Let's say sys.pycache_prefix is
529 # `C:\Bytecode`; the idea here is that if we get `Foo\Bar`, we first
530 # make it absolute (`C:\Somewhere\Foo\Bar`), then make it root-relative
531 # (`Somewhere\Foo\Bar`), so we end up placing the bytecode file in an
532 # unambiguous `C:\Bytecode\Somewhere\Foo\Bar\`.
533 head = _path_abspath(head)
534
535 # Strip initial drive from a Windows path. We know we have an absolute
536 # path here, so the second part of the check rules out a POSIX path that
537 # happens to contain a colon at the second character.
538 if head[1] == ':' and head[0] not in path_separators:
539 head = head[2:]
540
541 # Strip initial path separator from `head` to complete the conversion
542 # back to a root-relative path before joining.
543 return _path_join(
544 sys.pycache_prefix,
545 head.lstrip(path_separators),
546 filename,
547 )
548 return _path_join(head, _PYCACHE, filename)
549
550
551 def source_from_cache(path):
552 """Given the path to a .pyc. file, return the path to its .py file.
553
554 The .pyc file does not need to exist; this simply returns the path to
555 the .py file calculated to correspond to the .pyc file. If path does
556 not conform to PEP 3147/488 format, ValueError will be raised. If
557 sys.implementation.cache_tag is None then NotImplementedError is raised.
558
559 """
560 if sys.implementation.cache_tag is None:
561 raise NotImplementedError('sys.implementation.cache_tag is None')
562 path = _os.fspath(path)
563 head, pycache_filename = _path_split(path)
564 found_in_pycache_prefix = False
565 if sys.pycache_prefix is not None:
566 stripped_path = sys.pycache_prefix.rstrip(path_separators)
567 if head.startswith(stripped_path + path_sep):
568 head = head[len(stripped_path):]
569 found_in_pycache_prefix = True
570 if not found_in_pycache_prefix:
571 head, pycache = _path_split(head)
572 if pycache != _PYCACHE:
573 raise ValueError(f'{_PYCACHE} not bottom-level directory in '
574 f'{path!r}')
575 dot_count = pycache_filename.count('.')
576 if dot_count not in {2, 3}:
577 raise ValueError(f'expected only 2 or 3 dots in {pycache_filename!r}')
578 elif dot_count == 3:
579 optimization = pycache_filename.rsplit('.', 2)[-2]
580 if not optimization.startswith(_OPT):
581 raise ValueError("optimization portion of filename does not start "
582 f"with {_OPT!r}")
583 opt_level = optimization[len(_OPT):]
584 if not opt_level.isalnum():
585 raise ValueError(f"optimization level {optimization!r} is not an "
586 "alphanumeric value")
587 base_filename = pycache_filename.partition('.')[0]
588 return _path_join(head, base_filename + SOURCE_SUFFIXES[0])
589
590
591 def _get_sourcefile(bytecode_path):
592 """Convert a bytecode file path to a source path (if possible).
593
594 This function exists purely for backwards-compatibility for
595 PyImport_ExecCodeModuleWithFilenames() in the C API.
596
597 """
598 if len(bytecode_path) == 0:
599 return None
600 rest, _, extension = bytecode_path.rpartition('.')
601 if not rest or extension.lower()[-3:-1] != 'py':
602 return bytecode_path
603 try:
604 source_path = source_from_cache(bytecode_path)
605 except (NotImplementedError, ValueError):
606 source_path = bytecode_path[:-1]
607 return source_path if _path_isfile(source_path) else bytecode_path
608
609
610 def _get_cached(filename):
611 if filename.endswith(tuple(SOURCE_SUFFIXES)):
612 try:
613 return cache_from_source(filename)
614 except NotImplementedError:
615 pass
616 elif filename.endswith(tuple(BYTECODE_SUFFIXES)):
617 return filename
618 else:
619 return None
620
621
622 def _calc_mode(path):
623 """Calculate the mode permissions for a bytecode file."""
624 try:
625 mode = _path_stat(path).st_mode
626 except OSError:
627 mode = 0o666
628 # We always ensure write access so we can update cached files
629 # later even when the source files are read-only on Windows (#6074)
630 mode |= 0o200
631 return mode
632
633
634 def _check_name(method):
635 """Decorator to verify that the module being requested matches the one the
636 loader can handle.
637
638 The first argument (self) must define _name which the second argument is
639 compared against. If the comparison fails then ImportError is raised.
640
641 """
642 def _check_name_wrapper(self, name=None, *args, **kwargs):
643 if name is None:
644 name = self.name
645 elif self.name != name:
646 raise ImportError('loader for %s cannot handle %s' %
647 (self.name, name), name=name)
648 return method(self, name, *args, **kwargs)
649
650 # FIXME: @_check_name is used to define class methods before the
651 # _bootstrap module is set by _set_bootstrap_module().
652 if _bootstrap is not None:
653 _wrap = _bootstrap._wrap
654 else:
655 def _wrap(new, old):
656 for replace in ['__module__', '__name__', '__qualname__', '__doc__']:
657 if hasattr(old, replace):
658 setattr(new, replace, getattr(old, replace))
659 new.__dict__.update(old.__dict__)
660
661 _wrap(_check_name_wrapper, method)
662 return _check_name_wrapper
663
664
665 def _classify_pyc(data, name, exc_details):
666 """Perform basic validity checking of a pyc header and return the flags field,
667 which determines how the pyc should be further validated against the source.
668
669 *data* is the contents of the pyc file. (Only the first 16 bytes are
670 required, though.)
671
672 *name* is the name of the module being imported. It is used for logging.
673
674 *exc_details* is a dictionary passed to ImportError if it raised for
675 improved debugging.
676
677 ImportError is raised when the magic number is incorrect or when the flags
678 field is invalid. EOFError is raised when the data is found to be truncated.
679
680 """
681 magic = data[:4]
682 if magic != MAGIC_NUMBER:
683 message = f'bad magic number in {name!r}: {magic!r}'
684 _bootstrap._verbose_message('{}', message)
685 raise ImportError(message, **exc_details)
686 if len(data) < 16:
687 message = f'reached EOF while reading pyc header of {name!r}'
688 _bootstrap._verbose_message('{}', message)
689 raise EOFError(message)
690 flags = _unpack_uint32(data[4:8])
691 # Only the first two flags are defined.
692 if flags & ~0b11:
693 message = f'invalid flags {flags!r} in {name!r}'
694 raise ImportError(message, **exc_details)
695 return flags
696
697
698 def _validate_timestamp_pyc(data, source_mtime, source_size, name,
699 exc_details):
700 """Validate a pyc against the source last-modified time.
701
702 *data* is the contents of the pyc file. (Only the first 16 bytes are
703 required.)
704
705 *source_mtime* is the last modified timestamp of the source file.
706
707 *source_size* is None or the size of the source file in bytes.
708
709 *name* is the name of the module being imported. It is used for logging.
710
711 *exc_details* is a dictionary passed to ImportError if it raised for
712 improved debugging.
713
714 An ImportError is raised if the bytecode is stale.
715
716 """
717 if _unpack_uint32(data[8:12]) != (source_mtime & 0xFFFFFFFF):
718 message = f'bytecode is stale for {name!r}'
719 _bootstrap._verbose_message('{}', message)
720 raise ImportError(message, **exc_details)
721 if (source_size is not None and
722 _unpack_uint32(data[12:16]) != (source_size & 0xFFFFFFFF)):
723 raise ImportError(f'bytecode is stale for {name!r}', **exc_details)
724
725
726 def _validate_hash_pyc(data, source_hash, name, exc_details):
727 """Validate a hash-based pyc by checking the real source hash against the one in
728 the pyc header.
729
730 *data* is the contents of the pyc file. (Only the first 16 bytes are
731 required.)
732
733 *source_hash* is the importlib.util.source_hash() of the source file.
734
735 *name* is the name of the module being imported. It is used for logging.
736
737 *exc_details* is a dictionary passed to ImportError if it raised for
738 improved debugging.
739
740 An ImportError is raised if the bytecode is stale.
741
742 """
743 if data[8:16] != source_hash:
744 raise ImportError(
745 f'hash in bytecode doesn\'t match hash of source {name!r}',
746 **exc_details,
747 )
748
749
750 def _compile_bytecode(data, name=None, bytecode_path=None, source_path=None):
751 """Compile bytecode as found in a pyc."""
752 code = marshal.loads(data)
753 if isinstance(code, _code_type):
754 _bootstrap._verbose_message('code object from {!r}', bytecode_path)
755 if source_path is not None:
756 _imp._fix_co_filename(code, source_path)
757 return code
758 else:
759 raise ImportError(f'Non-code object in {bytecode_path!r}',
760 name=name, path=bytecode_path)
761
762
763 def _code_to_timestamp_pyc(code, mtime=0, source_size=0):
764 "Produce the data for a timestamp-based pyc."
765 data = bytearray(MAGIC_NUMBER)
766 data.extend(_pack_uint32(0))
767 data.extend(_pack_uint32(mtime))
768 data.extend(_pack_uint32(source_size))
769 data.extend(marshal.dumps(code))
770 return data
771
772
773 def _code_to_hash_pyc(code, source_hash, checked=True):
774 "Produce the data for a hash-based pyc."
775 data = bytearray(MAGIC_NUMBER)
776 flags = 0b1 | checked << 1
777 data.extend(_pack_uint32(flags))
778 assert len(source_hash) == 8
779 data.extend(source_hash)
780 data.extend(marshal.dumps(code))
781 return data
782
783
784 def decode_source(source_bytes):
785 """Decode bytes representing source code and return the string.
786
787 Universal newline support is used in the decoding.
788 """
789 import tokenize # To avoid bootstrap issues.
790 source_bytes_readline = _io.BytesIO(source_bytes).readline
791 encoding = tokenize.detect_encoding(source_bytes_readline)
792 newline_decoder = _io.IncrementalNewlineDecoder(None, True)
793 return newline_decoder.decode(source_bytes.decode(encoding[0]))
794
795
796 # Module specifications #######################################################
797
798 _POPULATE = object()
799
800
801 def spec_from_file_location(name, location=None, *, loader=None,
802 submodule_search_locations=_POPULATE):
803 """Return a module spec based on a file location.
804
805 To indicate that the module is a package, set
806 submodule_search_locations to a list of directory paths. An
807 empty list is sufficient, though its not otherwise useful to the
808 import system.
809
810 The loader must take a spec as its only __init__() arg.
811
812 """
813 if location is None:
814 # The caller may simply want a partially populated location-
815 # oriented spec. So we set the location to a bogus value and
816 # fill in as much as we can.
817 location = '<unknown>'
818 if hasattr(loader, 'get_filename'):
819 # ExecutionLoader
820 try:
821 location = loader.get_filename(name)
822 except ImportError:
823 pass
824 else:
825 location = _os.fspath(location)
826 try:
827 location = _path_abspath(location)
828 except OSError:
829 pass
830
831 # If the location is on the filesystem, but doesn't actually exist,
832 # we could return None here, indicating that the location is not
833 # valid. However, we don't have a good way of testing since an
834 # indirect location (e.g. a zip file or URL) will look like a
835 # non-existent file relative to the filesystem.
836
837 spec = _bootstrap.ModuleSpec(name, loader, origin=location)
838 spec._set_fileattr = True
839
840 # Pick a loader if one wasn't provided.
841 if loader is None:
842 for loader_class, suffixes in _get_supported_file_loaders():
843 if location.endswith(tuple(suffixes)):
844 loader = loader_class(name, location)
845 spec.loader = loader
846 break
847 else:
848 return None
849
850 # Set submodule_search_paths appropriately.
851 if submodule_search_locations is _POPULATE:
852 # Check the loader.
853 if hasattr(loader, 'is_package'):
854 try:
855 is_package = loader.is_package(name)
856 except ImportError:
857 pass
858 else:
859 if is_package:
860 spec.submodule_search_locations = []
861 else:
862 spec.submodule_search_locations = submodule_search_locations
863 if spec.submodule_search_locations == []:
864 if location:
865 dirname = _path_split(location)[0]
866 spec.submodule_search_locations.append(dirname)
867
868 return spec
869
870
871 def _bless_my_loader(module_globals):
872 """Helper function for _warnings.c
873
874 See GH#97850 for details.
875 """
876 # 2022-10-06(warsaw): For now, this helper is only used in _warnings.c and
877 # that use case only has the module globals. This function could be
878 # extended to accept either that or a module object. However, in the
879 # latter case, it would be better to raise certain exceptions when looking
880 # at a module, which should have either a __loader__ or __spec__.loader.
881 # For backward compatibility, it is possible that we'll get an empty
882 # dictionary for the module globals, and that cannot raise an exception.
883 if not isinstance(module_globals, dict):
884 return None
885
886 missing = object()
887 loader = module_globals.get('__loader__', None)
888 spec = module_globals.get('__spec__', missing)
889
890 if loader is None:
891 if spec is missing:
892 # If working with a module:
893 # raise AttributeError('Module globals is missing a __spec__')
894 return None
895 elif spec is None:
896 raise ValueError('Module globals is missing a __spec__.loader')
897
898 spec_loader = getattr(spec, 'loader', missing)
899
900 if spec_loader in (missing, None):
901 if loader is None:
902 exc = AttributeError if spec_loader is missing else ValueError
903 raise exc('Module globals is missing a __spec__.loader')
904 _warnings.warn(
905 'Module globals is missing a __spec__.loader',
906 DeprecationWarning)
907 spec_loader = loader
908
909 assert spec_loader is not None
910 if loader is not None and loader != spec_loader:
911 _warnings.warn(
912 'Module globals; __loader__ != __spec__.loader',
913 DeprecationWarning)
914 return loader
915
916 return spec_loader
917
918
919 # Loaders #####################################################################
920
921 class ESC[4;38;5;81mWindowsRegistryFinder:
922
923 """Meta path finder for modules declared in the Windows registry."""
924
925 REGISTRY_KEY = (
926 'Software\\Python\\PythonCore\\{sys_version}'
927 '\\Modules\\{fullname}')
928 REGISTRY_KEY_DEBUG = (
929 'Software\\Python\\PythonCore\\{sys_version}'
930 '\\Modules\\{fullname}\\Debug')
931 DEBUG_BUILD = (_MS_WINDOWS and '_d.pyd' in EXTENSION_SUFFIXES)
932
933 @staticmethod
934 def _open_registry(key):
935 try:
936 return winreg.OpenKey(winreg.HKEY_CURRENT_USER, key)
937 except OSError:
938 return winreg.OpenKey(winreg.HKEY_LOCAL_MACHINE, key)
939
940 @classmethod
941 def _search_registry(cls, fullname):
942 if cls.DEBUG_BUILD:
943 registry_key = cls.REGISTRY_KEY_DEBUG
944 else:
945 registry_key = cls.REGISTRY_KEY
946 key = registry_key.format(fullname=fullname,
947 sys_version='%d.%d' % sys.version_info[:2])
948 try:
949 with cls._open_registry(key) as hkey:
950 filepath = winreg.QueryValue(hkey, '')
951 except OSError:
952 return None
953 return filepath
954
955 @classmethod
956 def find_spec(cls, fullname, path=None, target=None):
957 filepath = cls._search_registry(fullname)
958 if filepath is None:
959 return None
960 try:
961 _path_stat(filepath)
962 except OSError:
963 return None
964 for loader, suffixes in _get_supported_file_loaders():
965 if filepath.endswith(tuple(suffixes)):
966 spec = _bootstrap.spec_from_loader(fullname,
967 loader(fullname, filepath),
968 origin=filepath)
969 return spec
970
971
972 class ESC[4;38;5;81m_LoaderBasics:
973
974 """Base class of common code needed by both SourceLoader and
975 SourcelessFileLoader."""
976
977 def is_package(self, fullname):
978 """Concrete implementation of InspectLoader.is_package by checking if
979 the path returned by get_filename has a filename of '__init__.py'."""
980 filename = _path_split(self.get_filename(fullname))[1]
981 filename_base = filename.rsplit('.', 1)[0]
982 tail_name = fullname.rpartition('.')[2]
983 return filename_base == '__init__' and tail_name != '__init__'
984
985 def create_module(self, spec):
986 """Use default semantics for module creation."""
987
988 def exec_module(self, module):
989 """Execute the module."""
990 code = self.get_code(module.__name__)
991 if code is None:
992 raise ImportError(f'cannot load module {module.__name__!r} when '
993 'get_code() returns None')
994 _bootstrap._call_with_frames_removed(exec, code, module.__dict__)
995
996 def load_module(self, fullname):
997 """This method is deprecated."""
998 # Warning implemented in _load_module_shim().
999 return _bootstrap._load_module_shim(self, fullname)
1000
1001
1002 class ESC[4;38;5;81mSourceLoader(ESC[4;38;5;149m_LoaderBasics):
1003
1004 def path_mtime(self, path):
1005 """Optional method that returns the modification time (an int) for the
1006 specified path (a str).
1007
1008 Raises OSError when the path cannot be handled.
1009 """
1010 raise OSError
1011
1012 def path_stats(self, path):
1013 """Optional method returning a metadata dict for the specified
1014 path (a str).
1015
1016 Possible keys:
1017 - 'mtime' (mandatory) is the numeric timestamp of last source
1018 code modification;
1019 - 'size' (optional) is the size in bytes of the source code.
1020
1021 Implementing this method allows the loader to read bytecode files.
1022 Raises OSError when the path cannot be handled.
1023 """
1024 return {'mtime': self.path_mtime(path)}
1025
1026 def _cache_bytecode(self, source_path, cache_path, data):
1027 """Optional method which writes data (bytes) to a file path (a str).
1028
1029 Implementing this method allows for the writing of bytecode files.
1030
1031 The source path is needed in order to correctly transfer permissions
1032 """
1033 # For backwards compatibility, we delegate to set_data()
1034 return self.set_data(cache_path, data)
1035
1036 def set_data(self, path, data):
1037 """Optional method which writes data (bytes) to a file path (a str).
1038
1039 Implementing this method allows for the writing of bytecode files.
1040 """
1041
1042
1043 def get_source(self, fullname):
1044 """Concrete implementation of InspectLoader.get_source."""
1045 path = self.get_filename(fullname)
1046 try:
1047 source_bytes = self.get_data(path)
1048 except OSError as exc:
1049 raise ImportError('source not available through get_data()',
1050 name=fullname) from exc
1051 return decode_source(source_bytes)
1052
1053 def source_to_code(self, data, path, *, _optimize=-1):
1054 """Return the code object compiled from source.
1055
1056 The 'data' argument can be any object type that compile() supports.
1057 """
1058 return _bootstrap._call_with_frames_removed(compile, data, path, 'exec',
1059 dont_inherit=True, optimize=_optimize)
1060
1061 def get_code(self, fullname):
1062 """Concrete implementation of InspectLoader.get_code.
1063
1064 Reading of bytecode requires path_stats to be implemented. To write
1065 bytecode, set_data must also be implemented.
1066
1067 """
1068 source_path = self.get_filename(fullname)
1069 source_mtime = None
1070 source_bytes = None
1071 source_hash = None
1072 hash_based = False
1073 check_source = True
1074 try:
1075 bytecode_path = cache_from_source(source_path)
1076 except NotImplementedError:
1077 bytecode_path = None
1078 else:
1079 try:
1080 st = self.path_stats(source_path)
1081 except OSError:
1082 pass
1083 else:
1084 source_mtime = int(st['mtime'])
1085 try:
1086 data = self.get_data(bytecode_path)
1087 except OSError:
1088 pass
1089 else:
1090 exc_details = {
1091 'name': fullname,
1092 'path': bytecode_path,
1093 }
1094 try:
1095 flags = _classify_pyc(data, fullname, exc_details)
1096 bytes_data = memoryview(data)[16:]
1097 hash_based = flags & 0b1 != 0
1098 if hash_based:
1099 check_source = flags & 0b10 != 0
1100 if (_imp.check_hash_based_pycs != 'never' and
1101 (check_source or
1102 _imp.check_hash_based_pycs == 'always')):
1103 source_bytes = self.get_data(source_path)
1104 source_hash = _imp.source_hash(
1105 _RAW_MAGIC_NUMBER,
1106 source_bytes,
1107 )
1108 _validate_hash_pyc(data, source_hash, fullname,
1109 exc_details)
1110 else:
1111 _validate_timestamp_pyc(
1112 data,
1113 source_mtime,
1114 st['size'],
1115 fullname,
1116 exc_details,
1117 )
1118 except (ImportError, EOFError):
1119 pass
1120 else:
1121 _bootstrap._verbose_message('{} matches {}', bytecode_path,
1122 source_path)
1123 return _compile_bytecode(bytes_data, name=fullname,
1124 bytecode_path=bytecode_path,
1125 source_path=source_path)
1126 if source_bytes is None:
1127 source_bytes = self.get_data(source_path)
1128 code_object = self.source_to_code(source_bytes, source_path)
1129 _bootstrap._verbose_message('code object from {}', source_path)
1130 if (not sys.dont_write_bytecode and bytecode_path is not None and
1131 source_mtime is not None):
1132 if hash_based:
1133 if source_hash is None:
1134 source_hash = _imp.source_hash(_RAW_MAGIC_NUMBER,
1135 source_bytes)
1136 data = _code_to_hash_pyc(code_object, source_hash, check_source)
1137 else:
1138 data = _code_to_timestamp_pyc(code_object, source_mtime,
1139 len(source_bytes))
1140 try:
1141 self._cache_bytecode(source_path, bytecode_path, data)
1142 except NotImplementedError:
1143 pass
1144 return code_object
1145
1146
1147 class ESC[4;38;5;81mFileLoader:
1148
1149 """Base file loader class which implements the loader protocol methods that
1150 require file system usage."""
1151
1152 def __init__(self, fullname, path):
1153 """Cache the module name and the path to the file found by the
1154 finder."""
1155 self.name = fullname
1156 self.path = path
1157
1158 def __eq__(self, other):
1159 return (self.__class__ == other.__class__ and
1160 self.__dict__ == other.__dict__)
1161
1162 def __hash__(self):
1163 return hash(self.name) ^ hash(self.path)
1164
1165 @_check_name
1166 def load_module(self, fullname):
1167 """Load a module from a file.
1168
1169 This method is deprecated. Use exec_module() instead.
1170
1171 """
1172 # The only reason for this method is for the name check.
1173 # Issue #14857: Avoid the zero-argument form of super so the implementation
1174 # of that form can be updated without breaking the frozen module.
1175 return super(FileLoader, self).load_module(fullname)
1176
1177 @_check_name
1178 def get_filename(self, fullname):
1179 """Return the path to the source file as found by the finder."""
1180 return self.path
1181
1182 def get_data(self, path):
1183 """Return the data from path as raw bytes."""
1184 if isinstance(self, (SourceLoader, ExtensionFileLoader)):
1185 with _io.open_code(str(path)) as file:
1186 return file.read()
1187 else:
1188 with _io.FileIO(path, 'r') as file:
1189 return file.read()
1190
1191 @_check_name
1192 def get_resource_reader(self, module):
1193 from importlib.readers import FileReader
1194 return FileReader(self)
1195
1196
1197 class ESC[4;38;5;81mSourceFileLoader(ESC[4;38;5;149mFileLoader, ESC[4;38;5;149mSourceLoader):
1198
1199 """Concrete implementation of SourceLoader using the file system."""
1200
1201 def path_stats(self, path):
1202 """Return the metadata for the path."""
1203 st = _path_stat(path)
1204 return {'mtime': st.st_mtime, 'size': st.st_size}
1205
1206 def _cache_bytecode(self, source_path, bytecode_path, data):
1207 # Adapt between the two APIs
1208 mode = _calc_mode(source_path)
1209 return self.set_data(bytecode_path, data, _mode=mode)
1210
1211 def set_data(self, path, data, *, _mode=0o666):
1212 """Write bytes data to a file."""
1213 parent, filename = _path_split(path)
1214 path_parts = []
1215 # Figure out what directories are missing.
1216 while parent and not _path_isdir(parent):
1217 parent, part = _path_split(parent)
1218 path_parts.append(part)
1219 # Create needed directories.
1220 for part in reversed(path_parts):
1221 parent = _path_join(parent, part)
1222 try:
1223 _os.mkdir(parent)
1224 except FileExistsError:
1225 # Probably another Python process already created the dir.
1226 continue
1227 except OSError as exc:
1228 # Could be a permission error, read-only filesystem: just forget
1229 # about writing the data.
1230 _bootstrap._verbose_message('could not create {!r}: {!r}',
1231 parent, exc)
1232 return
1233 try:
1234 _write_atomic(path, data, _mode)
1235 _bootstrap._verbose_message('created {!r}', path)
1236 except OSError as exc:
1237 # Same as above: just don't write the bytecode.
1238 _bootstrap._verbose_message('could not create {!r}: {!r}', path,
1239 exc)
1240
1241
1242 class ESC[4;38;5;81mSourcelessFileLoader(ESC[4;38;5;149mFileLoader, ESC[4;38;5;149m_LoaderBasics):
1243
1244 """Loader which handles sourceless file imports."""
1245
1246 def get_code(self, fullname):
1247 path = self.get_filename(fullname)
1248 data = self.get_data(path)
1249 # Call _classify_pyc to do basic validation of the pyc but ignore the
1250 # result. There's no source to check against.
1251 exc_details = {
1252 'name': fullname,
1253 'path': path,
1254 }
1255 _classify_pyc(data, fullname, exc_details)
1256 return _compile_bytecode(
1257 memoryview(data)[16:],
1258 name=fullname,
1259 bytecode_path=path,
1260 )
1261
1262 def get_source(self, fullname):
1263 """Return None as there is no source code."""
1264 return None
1265
1266
1267 class ESC[4;38;5;81mExtensionFileLoader(ESC[4;38;5;149mFileLoader, ESC[4;38;5;149m_LoaderBasics):
1268
1269 """Loader for extension modules.
1270
1271 The constructor is designed to work with FileFinder.
1272
1273 """
1274
1275 def __init__(self, name, path):
1276 self.name = name
1277 self.path = path
1278
1279 def __eq__(self, other):
1280 return (self.__class__ == other.__class__ and
1281 self.__dict__ == other.__dict__)
1282
1283 def __hash__(self):
1284 return hash(self.name) ^ hash(self.path)
1285
1286 def create_module(self, spec):
1287 """Create an uninitialized extension module"""
1288 module = _bootstrap._call_with_frames_removed(
1289 _imp.create_dynamic, spec)
1290 _bootstrap._verbose_message('extension module {!r} loaded from {!r}',
1291 spec.name, self.path)
1292 return module
1293
1294 def exec_module(self, module):
1295 """Initialize an extension module"""
1296 _bootstrap._call_with_frames_removed(_imp.exec_dynamic, module)
1297 _bootstrap._verbose_message('extension module {!r} executed from {!r}',
1298 self.name, self.path)
1299
1300 def is_package(self, fullname):
1301 """Return True if the extension module is a package."""
1302 file_name = _path_split(self.path)[1]
1303 return any(file_name == '__init__' + suffix
1304 for suffix in EXTENSION_SUFFIXES)
1305
1306 def get_code(self, fullname):
1307 """Return None as an extension module cannot create a code object."""
1308 return None
1309
1310 def get_source(self, fullname):
1311 """Return None as extension modules have no source code."""
1312 return None
1313
1314 @_check_name
1315 def get_filename(self, fullname):
1316 """Return the path to the source file as found by the finder."""
1317 return self.path
1318
1319
1320 class ESC[4;38;5;81m_NamespacePath:
1321 """Represents a namespace package's path. It uses the module name
1322 to find its parent module, and from there it looks up the parent's
1323 __path__. When this changes, the module's own path is recomputed,
1324 using path_finder. For top-level modules, the parent module's path
1325 is sys.path."""
1326
1327 # When invalidate_caches() is called, this epoch is incremented
1328 # https://bugs.python.org/issue45703
1329 _epoch = 0
1330
1331 def __init__(self, name, path, path_finder):
1332 self._name = name
1333 self._path = path
1334 self._last_parent_path = tuple(self._get_parent_path())
1335 self._last_epoch = self._epoch
1336 self._path_finder = path_finder
1337
1338 def _find_parent_path_names(self):
1339 """Returns a tuple of (parent-module-name, parent-path-attr-name)"""
1340 parent, dot, me = self._name.rpartition('.')
1341 if dot == '':
1342 # This is a top-level module. sys.path contains the parent path.
1343 return 'sys', 'path'
1344 # Not a top-level module. parent-module.__path__ contains the
1345 # parent path.
1346 return parent, '__path__'
1347
1348 def _get_parent_path(self):
1349 parent_module_name, path_attr_name = self._find_parent_path_names()
1350 return getattr(sys.modules[parent_module_name], path_attr_name)
1351
1352 def _recalculate(self):
1353 # If the parent's path has changed, recalculate _path
1354 parent_path = tuple(self._get_parent_path()) # Make a copy
1355 if parent_path != self._last_parent_path or self._epoch != self._last_epoch:
1356 spec = self._path_finder(self._name, parent_path)
1357 # Note that no changes are made if a loader is returned, but we
1358 # do remember the new parent path
1359 if spec is not None and spec.loader is None:
1360 if spec.submodule_search_locations:
1361 self._path = spec.submodule_search_locations
1362 self._last_parent_path = parent_path # Save the copy
1363 self._last_epoch = self._epoch
1364 return self._path
1365
1366 def __iter__(self):
1367 return iter(self._recalculate())
1368
1369 def __getitem__(self, index):
1370 return self._recalculate()[index]
1371
1372 def __setitem__(self, index, path):
1373 self._path[index] = path
1374
1375 def __len__(self):
1376 return len(self._recalculate())
1377
1378 def __repr__(self):
1379 return f'_NamespacePath({self._path!r})'
1380
1381 def __contains__(self, item):
1382 return item in self._recalculate()
1383
1384 def append(self, item):
1385 self._path.append(item)
1386
1387
1388 # This class is actually exposed publicly in a namespace package's __loader__
1389 # attribute, so it should be available through a non-private name.
1390 # https://github.com/python/cpython/issues/92054
1391 class ESC[4;38;5;81mNamespaceLoader:
1392 def __init__(self, name, path, path_finder):
1393 self._path = _NamespacePath(name, path, path_finder)
1394
1395 def is_package(self, fullname):
1396 return True
1397
1398 def get_source(self, fullname):
1399 return ''
1400
1401 def get_code(self, fullname):
1402 return compile('', '<string>', 'exec', dont_inherit=True)
1403
1404 def create_module(self, spec):
1405 """Use default semantics for module creation."""
1406
1407 def exec_module(self, module):
1408 pass
1409
1410 def load_module(self, fullname):
1411 """Load a namespace module.
1412
1413 This method is deprecated. Use exec_module() instead.
1414
1415 """
1416 # The import system never calls this method.
1417 _bootstrap._verbose_message('namespace module loaded with path {!r}',
1418 self._path)
1419 # Warning implemented in _load_module_shim().
1420 return _bootstrap._load_module_shim(self, fullname)
1421
1422 def get_resource_reader(self, module):
1423 from importlib.readers import NamespaceReader
1424 return NamespaceReader(self._path)
1425
1426
1427 # We use this exclusively in module_from_spec() for backward-compatibility.
1428 _NamespaceLoader = NamespaceLoader
1429
1430
1431 # Finders #####################################################################
1432
1433 class ESC[4;38;5;81mPathFinder:
1434
1435 """Meta path finder for sys.path and package __path__ attributes."""
1436
1437 @staticmethod
1438 def invalidate_caches():
1439 """Call the invalidate_caches() method on all path entry finders
1440 stored in sys.path_importer_caches (where implemented)."""
1441 for name, finder in list(sys.path_importer_cache.items()):
1442 # Drop entry if finder name is a relative path. The current
1443 # working directory may have changed.
1444 if finder is None or not _path_isabs(name):
1445 del sys.path_importer_cache[name]
1446 elif hasattr(finder, 'invalidate_caches'):
1447 finder.invalidate_caches()
1448 # Also invalidate the caches of _NamespacePaths
1449 # https://bugs.python.org/issue45703
1450 _NamespacePath._epoch += 1
1451
1452 @staticmethod
1453 def _path_hooks(path):
1454 """Search sys.path_hooks for a finder for 'path'."""
1455 if sys.path_hooks is not None and not sys.path_hooks:
1456 _warnings.warn('sys.path_hooks is empty', ImportWarning)
1457 for hook in sys.path_hooks:
1458 try:
1459 return hook(path)
1460 except ImportError:
1461 continue
1462 else:
1463 return None
1464
1465 @classmethod
1466 def _path_importer_cache(cls, path):
1467 """Get the finder for the path entry from sys.path_importer_cache.
1468
1469 If the path entry is not in the cache, find the appropriate finder
1470 and cache it. If no finder is available, store None.
1471
1472 """
1473 if path == '':
1474 try:
1475 path = _os.getcwd()
1476 except FileNotFoundError:
1477 # Don't cache the failure as the cwd can easily change to
1478 # a valid directory later on.
1479 return None
1480 try:
1481 finder = sys.path_importer_cache[path]
1482 except KeyError:
1483 finder = cls._path_hooks(path)
1484 sys.path_importer_cache[path] = finder
1485 return finder
1486
1487 @classmethod
1488 def _get_spec(cls, fullname, path, target=None):
1489 """Find the loader or namespace_path for this module/package name."""
1490 # If this ends up being a namespace package, namespace_path is
1491 # the list of paths that will become its __path__
1492 namespace_path = []
1493 for entry in path:
1494 if not isinstance(entry, str):
1495 continue
1496 finder = cls._path_importer_cache(entry)
1497 if finder is not None:
1498 spec = finder.find_spec(fullname, target)
1499 if spec is None:
1500 continue
1501 if spec.loader is not None:
1502 return spec
1503 portions = spec.submodule_search_locations
1504 if portions is None:
1505 raise ImportError('spec missing loader')
1506 # This is possibly part of a namespace package.
1507 # Remember these path entries (if any) for when we
1508 # create a namespace package, and continue iterating
1509 # on path.
1510 namespace_path.extend(portions)
1511 else:
1512 spec = _bootstrap.ModuleSpec(fullname, None)
1513 spec.submodule_search_locations = namespace_path
1514 return spec
1515
1516 @classmethod
1517 def find_spec(cls, fullname, path=None, target=None):
1518 """Try to find a spec for 'fullname' on sys.path or 'path'.
1519
1520 The search is based on sys.path_hooks and sys.path_importer_cache.
1521 """
1522 if path is None:
1523 path = sys.path
1524 spec = cls._get_spec(fullname, path, target)
1525 if spec is None:
1526 return None
1527 elif spec.loader is None:
1528 namespace_path = spec.submodule_search_locations
1529 if namespace_path:
1530 # We found at least one namespace path. Return a spec which
1531 # can create the namespace package.
1532 spec.origin = None
1533 spec.submodule_search_locations = _NamespacePath(fullname, namespace_path, cls._get_spec)
1534 return spec
1535 else:
1536 return None
1537 else:
1538 return spec
1539
1540 @staticmethod
1541 def find_distributions(*args, **kwargs):
1542 """
1543 Find distributions.
1544
1545 Return an iterable of all Distribution instances capable of
1546 loading the metadata for packages matching ``context.name``
1547 (or all names if ``None`` indicated) along the paths in the list
1548 of directories ``context.path``.
1549 """
1550 from importlib.metadata import MetadataPathFinder
1551 return MetadataPathFinder.find_distributions(*args, **kwargs)
1552
1553
1554 class ESC[4;38;5;81mFileFinder:
1555
1556 """File-based finder.
1557
1558 Interactions with the file system are cached for performance, being
1559 refreshed when the directory the finder is handling has been modified.
1560
1561 """
1562
1563 def __init__(self, path, *loader_details):
1564 """Initialize with the path to search on and a variable number of
1565 2-tuples containing the loader and the file suffixes the loader
1566 recognizes."""
1567 loaders = []
1568 for loader, suffixes in loader_details:
1569 loaders.extend((suffix, loader) for suffix in suffixes)
1570 self._loaders = loaders
1571 # Base (directory) path
1572 if not path or path == '.':
1573 self.path = _os.getcwd()
1574 else:
1575 self.path = _path_abspath(path)
1576 self._path_mtime = -1
1577 self._path_cache = set()
1578 self._relaxed_path_cache = set()
1579
1580 def invalidate_caches(self):
1581 """Invalidate the directory mtime."""
1582 self._path_mtime = -1
1583
1584 def _get_spec(self, loader_class, fullname, path, smsl, target):
1585 loader = loader_class(fullname, path)
1586 return spec_from_file_location(fullname, path, loader=loader,
1587 submodule_search_locations=smsl)
1588
1589 def find_spec(self, fullname, target=None):
1590 """Try to find a spec for the specified module.
1591
1592 Returns the matching spec, or None if not found.
1593 """
1594 is_namespace = False
1595 tail_module = fullname.rpartition('.')[2]
1596 try:
1597 mtime = _path_stat(self.path or _os.getcwd()).st_mtime
1598 except OSError:
1599 mtime = -1
1600 if mtime != self._path_mtime:
1601 self._fill_cache()
1602 self._path_mtime = mtime
1603 # tail_module keeps the original casing, for __file__ and friends
1604 if _relax_case():
1605 cache = self._relaxed_path_cache
1606 cache_module = tail_module.lower()
1607 else:
1608 cache = self._path_cache
1609 cache_module = tail_module
1610 # Check if the module is the name of a directory (and thus a package).
1611 if cache_module in cache:
1612 base_path = _path_join(self.path, tail_module)
1613 for suffix, loader_class in self._loaders:
1614 init_filename = '__init__' + suffix
1615 full_path = _path_join(base_path, init_filename)
1616 if _path_isfile(full_path):
1617 return self._get_spec(loader_class, fullname, full_path, [base_path], target)
1618 else:
1619 # If a namespace package, return the path if we don't
1620 # find a module in the next section.
1621 is_namespace = _path_isdir(base_path)
1622 # Check for a file w/ a proper suffix exists.
1623 for suffix, loader_class in self._loaders:
1624 try:
1625 full_path = _path_join(self.path, tail_module + suffix)
1626 except ValueError:
1627 return None
1628 _bootstrap._verbose_message('trying {}', full_path, verbosity=2)
1629 if cache_module + suffix in cache:
1630 if _path_isfile(full_path):
1631 return self._get_spec(loader_class, fullname, full_path,
1632 None, target)
1633 if is_namespace:
1634 _bootstrap._verbose_message('possible namespace for {}', base_path)
1635 spec = _bootstrap.ModuleSpec(fullname, None)
1636 spec.submodule_search_locations = [base_path]
1637 return spec
1638 return None
1639
1640 def _fill_cache(self):
1641 """Fill the cache of potential modules and packages for this directory."""
1642 path = self.path
1643 try:
1644 contents = _os.listdir(path or _os.getcwd())
1645 except (FileNotFoundError, PermissionError, NotADirectoryError):
1646 # Directory has either been removed, turned into a file, or made
1647 # unreadable.
1648 contents = []
1649 # We store two cached versions, to handle runtime changes of the
1650 # PYTHONCASEOK environment variable.
1651 if not sys.platform.startswith('win'):
1652 self._path_cache = set(contents)
1653 else:
1654 # Windows users can import modules with case-insensitive file
1655 # suffixes (for legacy reasons). Make the suffix lowercase here
1656 # so it's done once instead of for every import. This is safe as
1657 # the specified suffixes to check against are always specified in a
1658 # case-sensitive manner.
1659 lower_suffix_contents = set()
1660 for item in contents:
1661 name, dot, suffix = item.partition('.')
1662 if dot:
1663 new_name = f'{name}.{suffix.lower()}'
1664 else:
1665 new_name = name
1666 lower_suffix_contents.add(new_name)
1667 self._path_cache = lower_suffix_contents
1668 if sys.platform.startswith(_CASE_INSENSITIVE_PLATFORMS):
1669 self._relaxed_path_cache = {fn.lower() for fn in contents}
1670
1671 @classmethod
1672 def path_hook(cls, *loader_details):
1673 """A class method which returns a closure to use on sys.path_hook
1674 which will return an instance using the specified loaders and the path
1675 called on the closure.
1676
1677 If the path called on the closure is not a directory, ImportError is
1678 raised.
1679
1680 """
1681 def path_hook_for_FileFinder(path):
1682 """Path hook for importlib.machinery.FileFinder."""
1683 if not _path_isdir(path):
1684 raise ImportError('only directories are supported', path=path)
1685 return cls(path, *loader_details)
1686
1687 return path_hook_for_FileFinder
1688
1689 def __repr__(self):
1690 return f'FileFinder({self.path!r})'
1691
1692
1693 # Import setup ###############################################################
1694
1695 def _fix_up_module(ns, name, pathname, cpathname=None):
1696 # This function is used by PyImport_ExecCodeModuleObject().
1697 loader = ns.get('__loader__')
1698 spec = ns.get('__spec__')
1699 if not loader:
1700 if spec:
1701 loader = spec.loader
1702 elif pathname == cpathname:
1703 loader = SourcelessFileLoader(name, pathname)
1704 else:
1705 loader = SourceFileLoader(name, pathname)
1706 if not spec:
1707 spec = spec_from_file_location(name, pathname, loader=loader)
1708 if cpathname:
1709 spec.cached = _path_abspath(cpathname)
1710 try:
1711 ns['__spec__'] = spec
1712 ns['__loader__'] = loader
1713 ns['__file__'] = pathname
1714 ns['__cached__'] = cpathname
1715 except Exception:
1716 # Not important enough to report.
1717 pass
1718
1719
1720 def _get_supported_file_loaders():
1721 """Returns a list of file-based module loaders.
1722
1723 Each item is a tuple (loader, suffixes).
1724 """
1725 extensions = ExtensionFileLoader, _imp.extension_suffixes()
1726 source = SourceFileLoader, SOURCE_SUFFIXES
1727 bytecode = SourcelessFileLoader, BYTECODE_SUFFIXES
1728 return [extensions, source, bytecode]
1729
1730
1731 def _set_bootstrap_module(_bootstrap_module):
1732 global _bootstrap
1733 _bootstrap = _bootstrap_module
1734
1735
1736 def _install(_bootstrap_module):
1737 """Install the path-based import components."""
1738 _set_bootstrap_module(_bootstrap_module)
1739 supported_loaders = _get_supported_file_loaders()
1740 sys.path_hooks.extend([FileFinder.path_hook(*supported_loaders)])
1741 sys.meta_path.append(PathFinder)