1 # Run the tests in Programs/_testembed.c (tests for the CPython embedding APIs)
2 from test import support
3 from test.support import import_helper, os_helper, MS_WINDOWS
4 import unittest
5
6 from collections import namedtuple
7 import contextlib
8 import json
9 import os
10 import os.path
11 import re
12 import shutil
13 import subprocess
14 import sys
15 import sysconfig
16 import tempfile
17 import textwrap
18
19 if not support.has_subprocess_support:
20 raise unittest.SkipTest("test module requires subprocess")
21
22 MACOS = (sys.platform == 'darwin')
23 Py_DEBUG = hasattr(sys, 'gettotalrefcount')
24 PYMEM_ALLOCATOR_NOT_SET = 0
25 PYMEM_ALLOCATOR_DEBUG = 2
26 PYMEM_ALLOCATOR_MALLOC = 3
27
28 # _PyCoreConfig_InitCompatConfig()
29 API_COMPAT = 1
30 # _PyCoreConfig_InitPythonConfig()
31 API_PYTHON = 2
32 # _PyCoreConfig_InitIsolatedConfig()
33 API_ISOLATED = 3
34
35 INIT_LOOPS = 4
36 MAX_HASH_SEED = 4294967295
37
38
39 # If we are running from a build dir, but the stdlib has been installed,
40 # some tests need to expect different results.
41 STDLIB_INSTALL = os.path.join(sys.prefix, sys.platlibdir,
42 f'python{sys.version_info.major}.{sys.version_info.minor}')
43 if not os.path.isfile(os.path.join(STDLIB_INSTALL, 'os.py')):
44 STDLIB_INSTALL = None
45
46 def debug_build(program):
47 program = os.path.basename(program)
48 name = os.path.splitext(program)[0]
49 return name.casefold().endswith("_d".casefold())
50
51
52 def remove_python_envvars():
53 env = dict(os.environ)
54 # Remove PYTHON* environment variables to get deterministic environment
55 for key in list(env):
56 if key.startswith('PYTHON'):
57 del env[key]
58 return env
59
60
61 class ESC[4;38;5;81mEmbeddingTestsMixin:
62 def setUp(self):
63 exename = "_testembed"
64 builddir = os.path.dirname(sys.executable)
65 if MS_WINDOWS:
66 ext = ("_d" if debug_build(sys.executable) else "") + ".exe"
67 exename += ext
68 exepath = builddir
69 else:
70 exepath = os.path.join(builddir, 'Programs')
71 self.test_exe = exe = os.path.join(exepath, exename)
72 if not os.path.exists(exe):
73 self.skipTest("%r doesn't exist" % exe)
74 # This is needed otherwise we get a fatal error:
75 # "Py_Initialize: Unable to get the locale encoding
76 # LookupError: no codec search functions registered: can't find encoding"
77 self.oldcwd = os.getcwd()
78 os.chdir(builddir)
79
80 def tearDown(self):
81 os.chdir(self.oldcwd)
82
83 def run_embedded_interpreter(self, *args, env=None,
84 timeout=None, returncode=0, input=None,
85 cwd=None):
86 """Runs a test in the embedded interpreter"""
87 cmd = [self.test_exe]
88 cmd.extend(args)
89 if env is not None and MS_WINDOWS:
90 # Windows requires at least the SYSTEMROOT environment variable to
91 # start Python.
92 env = env.copy()
93 env['SYSTEMROOT'] = os.environ['SYSTEMROOT']
94
95 p = subprocess.Popen(cmd,
96 stdout=subprocess.PIPE,
97 stderr=subprocess.PIPE,
98 universal_newlines=True,
99 env=env,
100 cwd=cwd)
101 try:
102 (out, err) = p.communicate(input=input, timeout=timeout)
103 except:
104 p.terminate()
105 p.wait()
106 raise
107 if p.returncode != returncode and support.verbose:
108 print(f"--- {cmd} failed ---")
109 print(f"stdout:\n{out}")
110 print(f"stderr:\n{err}")
111 print(f"------")
112
113 self.assertEqual(p.returncode, returncode,
114 "bad returncode %d, stderr is %r" %
115 (p.returncode, err))
116 return out, err
117
118 def run_repeated_init_and_subinterpreters(self):
119 out, err = self.run_embedded_interpreter("test_repeated_init_and_subinterpreters")
120 self.assertEqual(err, "")
121
122 # The output from _testembed looks like this:
123 # --- Pass 1 ---
124 # interp 0 <0x1cf9330>, thread state <0x1cf9700>: id(modules) = 139650431942728
125 # interp 1 <0x1d4f690>, thread state <0x1d35350>: id(modules) = 139650431165784
126 # interp 2 <0x1d5a690>, thread state <0x1d99ed0>: id(modules) = 139650413140368
127 # interp 3 <0x1d4f690>, thread state <0x1dc3340>: id(modules) = 139650412862200
128 # interp 0 <0x1cf9330>, thread state <0x1cf9700>: id(modules) = 139650431942728
129 # --- Pass 2 ---
130 # ...
131
132 interp_pat = (r"^interp (\d+) <(0x[\dA-F]+)>, "
133 r"thread state <(0x[\dA-F]+)>: "
134 r"id\(modules\) = ([\d]+)$")
135 Interp = namedtuple("Interp", "id interp tstate modules")
136
137 numloops = 1
138 current_run = []
139 for line in out.splitlines():
140 if line == "--- Pass {} ---".format(numloops):
141 self.assertEqual(len(current_run), 0)
142 if support.verbose > 1:
143 print(line)
144 numloops += 1
145 continue
146
147 self.assertLess(len(current_run), 5)
148 match = re.match(interp_pat, line)
149 if match is None:
150 self.assertRegex(line, interp_pat)
151
152 # Parse the line from the loop. The first line is the main
153 # interpreter and the 3 afterward are subinterpreters.
154 interp = Interp(*match.groups())
155 if support.verbose > 1:
156 print(interp)
157 self.assertTrue(interp.interp)
158 self.assertTrue(interp.tstate)
159 self.assertTrue(interp.modules)
160 current_run.append(interp)
161
162 # The last line in the loop should be the same as the first.
163 if len(current_run) == 5:
164 main = current_run[0]
165 self.assertEqual(interp, main)
166 yield current_run
167 current_run = []
168
169
170 class ESC[4;38;5;81mEmbeddingTests(ESC[4;38;5;149mEmbeddingTestsMixin, ESC[4;38;5;149munittestESC[4;38;5;149m.ESC[4;38;5;149mTestCase):
171 maxDiff = 100 * 50
172
173 def test_subinterps_main(self):
174 for run in self.run_repeated_init_and_subinterpreters():
175 main = run[0]
176
177 self.assertEqual(main.id, '0')
178
179 def test_subinterps_different_ids(self):
180 for run in self.run_repeated_init_and_subinterpreters():
181 main, *subs, _ = run
182
183 mainid = int(main.id)
184 for i, sub in enumerate(subs):
185 self.assertEqual(sub.id, str(mainid + i + 1))
186
187 def test_subinterps_distinct_state(self):
188 for run in self.run_repeated_init_and_subinterpreters():
189 main, *subs, _ = run
190
191 if '0x0' in main:
192 # XXX Fix on Windows (and other platforms): something
193 # is going on with the pointers in Programs/_testembed.c.
194 # interp.interp is 0x0 and interp.modules is the same
195 # between interpreters.
196 raise unittest.SkipTest('platform prints pointers as 0x0')
197
198 for sub in subs:
199 # A new subinterpreter may have the same
200 # PyInterpreterState pointer as a previous one if
201 # the earlier one has already been destroyed. So
202 # we compare with the main interpreter. The same
203 # applies to tstate.
204 self.assertNotEqual(sub.interp, main.interp)
205 self.assertNotEqual(sub.tstate, main.tstate)
206 self.assertNotEqual(sub.modules, main.modules)
207
208 def test_repeated_init_and_inittab(self):
209 out, err = self.run_embedded_interpreter("test_repeated_init_and_inittab")
210 self.assertEqual(err, "")
211
212 lines = [f"--- Pass {i} ---" for i in range(1, INIT_LOOPS+1)]
213 lines = "\n".join(lines) + "\n"
214 self.assertEqual(out, lines)
215
216 def test_forced_io_encoding(self):
217 # Checks forced configuration of embedded interpreter IO streams
218 env = dict(os.environ, PYTHONIOENCODING="utf-8:surrogateescape")
219 out, err = self.run_embedded_interpreter("test_forced_io_encoding", env=env)
220 if support.verbose > 1:
221 print()
222 print(out)
223 print(err)
224 expected_stream_encoding = "utf-8"
225 expected_errors = "surrogateescape"
226 expected_output = '\n'.join([
227 "--- Use defaults ---",
228 "Expected encoding: default",
229 "Expected errors: default",
230 "stdin: {in_encoding}:{errors}",
231 "stdout: {out_encoding}:{errors}",
232 "stderr: {out_encoding}:backslashreplace",
233 "--- Set errors only ---",
234 "Expected encoding: default",
235 "Expected errors: ignore",
236 "stdin: {in_encoding}:ignore",
237 "stdout: {out_encoding}:ignore",
238 "stderr: {out_encoding}:backslashreplace",
239 "--- Set encoding only ---",
240 "Expected encoding: iso8859-1",
241 "Expected errors: default",
242 "stdin: iso8859-1:{errors}",
243 "stdout: iso8859-1:{errors}",
244 "stderr: iso8859-1:backslashreplace",
245 "--- Set encoding and errors ---",
246 "Expected encoding: iso8859-1",
247 "Expected errors: replace",
248 "stdin: iso8859-1:replace",
249 "stdout: iso8859-1:replace",
250 "stderr: iso8859-1:backslashreplace"])
251 expected_output = expected_output.format(
252 in_encoding=expected_stream_encoding,
253 out_encoding=expected_stream_encoding,
254 errors=expected_errors)
255 # This is useful if we ever trip over odd platform behaviour
256 self.maxDiff = None
257 self.assertEqual(out.strip(), expected_output)
258
259 def test_pre_initialization_api(self):
260 """
261 Checks some key parts of the C-API that need to work before the runtime
262 is initialized (via Py_Initialize()).
263 """
264 env = dict(os.environ, PYTHONPATH=os.pathsep.join(sys.path))
265 out, err = self.run_embedded_interpreter("test_pre_initialization_api", env=env)
266 if MS_WINDOWS:
267 expected_path = self.test_exe
268 else:
269 expected_path = os.path.join(os.getcwd(), "spam")
270 expected_output = f"sys.executable: {expected_path}\n"
271 self.assertIn(expected_output, out)
272 self.assertEqual(err, '')
273
274 def test_pre_initialization_sys_options(self):
275 """
276 Checks that sys.warnoptions and sys._xoptions can be set before the
277 runtime is initialized (otherwise they won't be effective).
278 """
279 env = remove_python_envvars()
280 env['PYTHONPATH'] = os.pathsep.join(sys.path)
281 out, err = self.run_embedded_interpreter(
282 "test_pre_initialization_sys_options", env=env)
283 expected_output = (
284 "sys.warnoptions: ['once', 'module', 'default']\n"
285 "sys._xoptions: {'not_an_option': '1', 'also_not_an_option': '2'}\n"
286 "warnings.filters[:3]: ['default', 'module', 'once']\n"
287 )
288 self.assertIn(expected_output, out)
289 self.assertEqual(err, '')
290
291 def test_bpo20891(self):
292 """
293 bpo-20891: Calling PyGILState_Ensure in a non-Python thread must not
294 crash.
295 """
296 out, err = self.run_embedded_interpreter("test_bpo20891")
297 self.assertEqual(out, '')
298 self.assertEqual(err, '')
299
300 def test_initialize_twice(self):
301 """
302 bpo-33932: Calling Py_Initialize() twice should do nothing (and not
303 crash!).
304 """
305 out, err = self.run_embedded_interpreter("test_initialize_twice")
306 self.assertEqual(out, '')
307 self.assertEqual(err, '')
308
309 def test_initialize_pymain(self):
310 """
311 bpo-34008: Calling Py_Main() after Py_Initialize() must not fail.
312 """
313 out, err = self.run_embedded_interpreter("test_initialize_pymain")
314 self.assertEqual(out.rstrip(), "Py_Main() after Py_Initialize: sys.argv=['-c', 'arg2']")
315 self.assertEqual(err, '')
316
317 def test_run_main(self):
318 out, err = self.run_embedded_interpreter("test_run_main")
319 self.assertEqual(out.rstrip(), "Py_RunMain(): sys.argv=['-c', 'arg2']")
320 self.assertEqual(err, '')
321
322 def test_run_main_loop(self):
323 # bpo-40413: Calling Py_InitializeFromConfig()+Py_RunMain() multiple
324 # times must not crash.
325 nloop = 5
326 out, err = self.run_embedded_interpreter("test_run_main_loop")
327 self.assertEqual(out, "Py_RunMain(): sys.argv=['-c', 'arg2']\n" * nloop)
328 self.assertEqual(err, '')
329
330 def test_finalize_structseq(self):
331 # bpo-46417: Py_Finalize() clears structseq static types. Check that
332 # sys attributes using struct types still work when
333 # Py_Finalize()/Py_Initialize() is called multiple times.
334 # print() calls type->tp_repr(instance) and so checks that the types
335 # are still working properly.
336 script = support.findfile('_test_embed_structseq.py')
337 with open(script, encoding="utf-8") as fp:
338 code = fp.read()
339 out, err = self.run_embedded_interpreter("test_repeated_init_exec", code)
340 self.assertEqual(out, 'Tests passed\n' * INIT_LOOPS)
341
342 def test_simple_initialization_api(self):
343 # _testembed now uses Py_InitializeFromConfig by default
344 # This case specifically checks Py_Initialize(Ex) still works
345 out, err = self.run_embedded_interpreter("test_repeated_simple_init")
346 self.assertEqual(out, 'Finalized\n' * INIT_LOOPS)
347
348 def test_quickened_static_code_gets_unquickened_at_Py_FINALIZE(self):
349 # https://github.com/python/cpython/issues/92031
350
351 # Do these imports outside of the code string to avoid using
352 # importlib too much from within the code string, so that
353 # _handle_fromlist doesn't get quickened until we intend it to.
354 from dis import _all_opmap
355 resume = _all_opmap["RESUME"]
356 resume_quick = _all_opmap["RESUME_QUICK"]
357 from test.test_dis import QUICKENING_WARMUP_DELAY
358
359 code = textwrap.dedent(f"""\
360 import importlib._bootstrap
361 func = importlib._bootstrap._handle_fromlist
362 code = func.__code__
363
364 # Assert initially unquickened.
365 # Use sets to account for byte order.
366 if set(code._co_code_adaptive[:2]) != set([{resume}, 0]):
367 raise AssertionError()
368
369 for i in range({QUICKENING_WARMUP_DELAY}):
370 func(importlib._bootstrap, ["x"], lambda *args: None)
371
372 # Assert quickening worked
373 if set(code._co_code_adaptive[:2]) != set([{resume_quick}, 0]):
374 raise AssertionError()
375
376 print("Tests passed")
377 """)
378 run = self.run_embedded_interpreter
379 out, err = run("test_repeated_init_exec", code)
380 self.assertEqual(out, 'Tests passed\n' * INIT_LOOPS)
381
382 def test_ucnhash_capi_reset(self):
383 # bpo-47182: unicodeobject.c:ucnhash_capi was not reset on shutdown.
384 code = "print('\\N{digit nine}')"
385 out, err = self.run_embedded_interpreter("test_repeated_init_exec", code)
386 self.assertEqual(out, '9\n' * INIT_LOOPS)
387
388 class ESC[4;38;5;81mInitConfigTests(ESC[4;38;5;149mEmbeddingTestsMixin, ESC[4;38;5;149munittestESC[4;38;5;149m.ESC[4;38;5;149mTestCase):
389 maxDiff = 4096
390 UTF8_MODE_ERRORS = ('surrogatepass' if MS_WINDOWS else 'surrogateescape')
391
392 # Marker to read the default configuration: get_default_config()
393 GET_DEFAULT_CONFIG = object()
394
395 # Marker to ignore a configuration parameter
396 IGNORE_CONFIG = object()
397
398 PRE_CONFIG_COMPAT = {
399 '_config_init': API_COMPAT,
400 'allocator': PYMEM_ALLOCATOR_NOT_SET,
401 'parse_argv': 0,
402 'configure_locale': 1,
403 'coerce_c_locale': 0,
404 'coerce_c_locale_warn': 0,
405 'utf8_mode': 0,
406 }
407 if MS_WINDOWS:
408 PRE_CONFIG_COMPAT.update({
409 'legacy_windows_fs_encoding': 0,
410 })
411 PRE_CONFIG_PYTHON = dict(PRE_CONFIG_COMPAT,
412 _config_init=API_PYTHON,
413 parse_argv=1,
414 coerce_c_locale=GET_DEFAULT_CONFIG,
415 utf8_mode=GET_DEFAULT_CONFIG,
416 )
417 PRE_CONFIG_ISOLATED = dict(PRE_CONFIG_COMPAT,
418 _config_init=API_ISOLATED,
419 configure_locale=0,
420 isolated=1,
421 use_environment=0,
422 utf8_mode=0,
423 dev_mode=0,
424 coerce_c_locale=0,
425 )
426
427 COPY_PRE_CONFIG = [
428 'dev_mode',
429 'isolated',
430 'use_environment',
431 ]
432
433 CONFIG_COMPAT = {
434 '_config_init': API_COMPAT,
435 'isolated': 0,
436 'use_environment': 1,
437 'dev_mode': 0,
438
439 'install_signal_handlers': 1,
440 'use_hash_seed': 0,
441 'hash_seed': 0,
442 'faulthandler': 0,
443 'tracemalloc': 0,
444 'import_time': 0,
445 'code_debug_ranges': 1,
446 'show_ref_count': 0,
447 'dump_refs': 0,
448 'malloc_stats': 0,
449
450 'filesystem_encoding': GET_DEFAULT_CONFIG,
451 'filesystem_errors': GET_DEFAULT_CONFIG,
452
453 'pycache_prefix': None,
454 'program_name': GET_DEFAULT_CONFIG,
455 'parse_argv': 0,
456 'argv': [""],
457 'orig_argv': [],
458
459 'xoptions': [],
460 'warnoptions': [],
461
462 'pythonpath_env': None,
463 'home': None,
464 'executable': GET_DEFAULT_CONFIG,
465 'base_executable': GET_DEFAULT_CONFIG,
466
467 'prefix': GET_DEFAULT_CONFIG,
468 'base_prefix': GET_DEFAULT_CONFIG,
469 'exec_prefix': GET_DEFAULT_CONFIG,
470 'base_exec_prefix': GET_DEFAULT_CONFIG,
471 'module_search_paths': GET_DEFAULT_CONFIG,
472 'module_search_paths_set': 1,
473 'platlibdir': sys.platlibdir,
474 'stdlib_dir': GET_DEFAULT_CONFIG,
475
476 'site_import': 1,
477 'bytes_warning': 0,
478 'warn_default_encoding': 0,
479 'inspect': 0,
480 'interactive': 0,
481 'optimization_level': 0,
482 'parser_debug': 0,
483 'write_bytecode': 1,
484 'verbose': 0,
485 'quiet': 0,
486 'user_site_directory': 1,
487 'configure_c_stdio': 0,
488 'buffered_stdio': 1,
489
490 'stdio_encoding': GET_DEFAULT_CONFIG,
491 'stdio_errors': GET_DEFAULT_CONFIG,
492
493 'skip_source_first_line': 0,
494 'run_command': None,
495 'run_module': None,
496 'run_filename': None,
497
498 '_install_importlib': 1,
499 'check_hash_pycs_mode': 'default',
500 'pathconfig_warnings': 1,
501 '_init_main': 1,
502 '_isolated_interpreter': 0,
503 'use_frozen_modules': not Py_DEBUG,
504 'safe_path': 0,
505 '_is_python_build': IGNORE_CONFIG,
506 }
507 if MS_WINDOWS:
508 CONFIG_COMPAT.update({
509 'legacy_windows_stdio': 0,
510 })
511
512 CONFIG_PYTHON = dict(CONFIG_COMPAT,
513 _config_init=API_PYTHON,
514 configure_c_stdio=1,
515 parse_argv=2,
516 )
517 CONFIG_ISOLATED = dict(CONFIG_COMPAT,
518 _config_init=API_ISOLATED,
519 isolated=1,
520 use_environment=0,
521 user_site_directory=0,
522 safe_path=1,
523 dev_mode=0,
524 install_signal_handlers=0,
525 use_hash_seed=0,
526 faulthandler=0,
527 tracemalloc=0,
528 pathconfig_warnings=0,
529 )
530 if MS_WINDOWS:
531 CONFIG_ISOLATED['legacy_windows_stdio'] = 0
532
533 # global config
534 DEFAULT_GLOBAL_CONFIG = {
535 'Py_HasFileSystemDefaultEncoding': 0,
536 'Py_HashRandomizationFlag': 1,
537 '_Py_HasFileSystemDefaultEncodeErrors': 0,
538 }
539 COPY_GLOBAL_PRE_CONFIG = [
540 ('Py_UTF8Mode', 'utf8_mode'),
541 ]
542 COPY_GLOBAL_CONFIG = [
543 # Copy core config to global config for expected values
544 # True means that the core config value is inverted (0 => 1 and 1 => 0)
545 ('Py_BytesWarningFlag', 'bytes_warning'),
546 ('Py_DebugFlag', 'parser_debug'),
547 ('Py_DontWriteBytecodeFlag', 'write_bytecode', True),
548 ('Py_FileSystemDefaultEncodeErrors', 'filesystem_errors'),
549 ('Py_FileSystemDefaultEncoding', 'filesystem_encoding'),
550 ('Py_FrozenFlag', 'pathconfig_warnings', True),
551 ('Py_IgnoreEnvironmentFlag', 'use_environment', True),
552 ('Py_InspectFlag', 'inspect'),
553 ('Py_InteractiveFlag', 'interactive'),
554 ('Py_IsolatedFlag', 'isolated'),
555 ('Py_NoSiteFlag', 'site_import', True),
556 ('Py_NoUserSiteDirectory', 'user_site_directory', True),
557 ('Py_OptimizeFlag', 'optimization_level'),
558 ('Py_QuietFlag', 'quiet'),
559 ('Py_UnbufferedStdioFlag', 'buffered_stdio', True),
560 ('Py_VerboseFlag', 'verbose'),
561 ]
562 if MS_WINDOWS:
563 COPY_GLOBAL_PRE_CONFIG.extend((
564 ('Py_LegacyWindowsFSEncodingFlag', 'legacy_windows_fs_encoding'),
565 ))
566 COPY_GLOBAL_CONFIG.extend((
567 ('Py_LegacyWindowsStdioFlag', 'legacy_windows_stdio'),
568 ))
569
570 EXPECTED_CONFIG = None
571
572 @classmethod
573 def tearDownClass(cls):
574 # clear cache
575 cls.EXPECTED_CONFIG = None
576
577 def main_xoptions(self, xoptions_list):
578 xoptions = {}
579 for opt in xoptions_list:
580 if '=' in opt:
581 key, value = opt.split('=', 1)
582 xoptions[key] = value
583 else:
584 xoptions[opt] = True
585 return xoptions
586
587 def _get_expected_config_impl(self):
588 env = remove_python_envvars()
589 code = textwrap.dedent('''
590 import json
591 import sys
592 import _testinternalcapi
593
594 configs = _testinternalcapi.get_configs()
595
596 data = json.dumps(configs)
597 data = data.encode('utf-8')
598 sys.stdout.buffer.write(data)
599 sys.stdout.buffer.flush()
600 ''')
601
602 # Use -S to not import the site module: get the proper configuration
603 # when test_embed is run from a venv (bpo-35313)
604 args = [sys.executable, '-S', '-c', code]
605 proc = subprocess.run(args, env=env,
606 stdout=subprocess.PIPE,
607 stderr=subprocess.PIPE)
608 if proc.returncode:
609 raise Exception(f"failed to get the default config: "
610 f"stdout={proc.stdout!r} stderr={proc.stderr!r}")
611 stdout = proc.stdout.decode('utf-8')
612 # ignore stderr
613 try:
614 return json.loads(stdout)
615 except json.JSONDecodeError:
616 self.fail(f"fail to decode stdout: {stdout!r}")
617
618 def _get_expected_config(self):
619 cls = InitConfigTests
620 if cls.EXPECTED_CONFIG is None:
621 cls.EXPECTED_CONFIG = self._get_expected_config_impl()
622
623 # get a copy
624 configs = {}
625 for config_key, config_value in cls.EXPECTED_CONFIG.items():
626 config = {}
627 for key, value in config_value.items():
628 if isinstance(value, list):
629 value = value.copy()
630 config[key] = value
631 configs[config_key] = config
632 return configs
633
634 def get_expected_config(self, expected_preconfig, expected,
635 env, api, modify_path_cb=None):
636 configs = self._get_expected_config()
637
638 pre_config = configs['pre_config']
639 for key, value in expected_preconfig.items():
640 if value is self.GET_DEFAULT_CONFIG:
641 expected_preconfig[key] = pre_config[key]
642
643 if not expected_preconfig['configure_locale'] or api == API_COMPAT:
644 # there is no easy way to get the locale encoding before
645 # setlocale(LC_CTYPE, "") is called: don't test encodings
646 for key in ('filesystem_encoding', 'filesystem_errors',
647 'stdio_encoding', 'stdio_errors'):
648 expected[key] = self.IGNORE_CONFIG
649
650 if not expected_preconfig['configure_locale']:
651 # UTF-8 Mode depends on the locale. There is no easy way
652 # to guess if UTF-8 Mode will be enabled or not if the locale
653 # is not configured.
654 expected_preconfig['utf8_mode'] = self.IGNORE_CONFIG
655
656 if expected_preconfig['utf8_mode'] == 1:
657 if expected['filesystem_encoding'] is self.GET_DEFAULT_CONFIG:
658 expected['filesystem_encoding'] = 'utf-8'
659 if expected['filesystem_errors'] is self.GET_DEFAULT_CONFIG:
660 expected['filesystem_errors'] = self.UTF8_MODE_ERRORS
661 if expected['stdio_encoding'] is self.GET_DEFAULT_CONFIG:
662 expected['stdio_encoding'] = 'utf-8'
663 if expected['stdio_errors'] is self.GET_DEFAULT_CONFIG:
664 expected['stdio_errors'] = 'surrogateescape'
665
666 if MS_WINDOWS:
667 default_executable = self.test_exe
668 elif expected['program_name'] is not self.GET_DEFAULT_CONFIG:
669 default_executable = os.path.abspath(expected['program_name'])
670 else:
671 default_executable = os.path.join(os.getcwd(), '_testembed')
672 if expected['executable'] is self.GET_DEFAULT_CONFIG:
673 expected['executable'] = default_executable
674 if expected['base_executable'] is self.GET_DEFAULT_CONFIG:
675 expected['base_executable'] = default_executable
676 if expected['program_name'] is self.GET_DEFAULT_CONFIG:
677 expected['program_name'] = './_testembed'
678
679 config = configs['config']
680 for key, value in expected.items():
681 if value is self.GET_DEFAULT_CONFIG:
682 expected[key] = config[key]
683
684 if expected['module_search_paths'] is not self.IGNORE_CONFIG:
685 pythonpath_env = expected['pythonpath_env']
686 if pythonpath_env is not None:
687 paths = pythonpath_env.split(os.path.pathsep)
688 expected['module_search_paths'] = [*paths, *expected['module_search_paths']]
689 if modify_path_cb is not None:
690 expected['module_search_paths'] = expected['module_search_paths'].copy()
691 modify_path_cb(expected['module_search_paths'])
692
693 for key in self.COPY_PRE_CONFIG:
694 if key not in expected_preconfig:
695 expected_preconfig[key] = expected[key]
696
697 def check_pre_config(self, configs, expected):
698 pre_config = dict(configs['pre_config'])
699 for key, value in list(expected.items()):
700 if value is self.IGNORE_CONFIG:
701 pre_config.pop(key, None)
702 del expected[key]
703 self.assertEqual(pre_config, expected)
704
705 def check_config(self, configs, expected):
706 config = dict(configs['config'])
707 if MS_WINDOWS:
708 value = config.get(key := 'program_name')
709 if value and isinstance(value, str):
710 value = value[:len(value.lower().removesuffix('.exe'))]
711 if debug_build(sys.executable):
712 value = value[:len(value.lower().removesuffix('_d'))]
713 config[key] = value
714 for key, value in list(expected.items()):
715 if value is self.IGNORE_CONFIG:
716 config.pop(key, None)
717 del expected[key]
718 self.assertEqual(config, expected)
719
720 def check_global_config(self, configs):
721 pre_config = configs['pre_config']
722 config = configs['config']
723
724 expected = dict(self.DEFAULT_GLOBAL_CONFIG)
725 for item in self.COPY_GLOBAL_CONFIG:
726 if len(item) == 3:
727 global_key, core_key, opposite = item
728 expected[global_key] = 0 if config[core_key] else 1
729 else:
730 global_key, core_key = item
731 expected[global_key] = config[core_key]
732 for item in self.COPY_GLOBAL_PRE_CONFIG:
733 if len(item) == 3:
734 global_key, core_key, opposite = item
735 expected[global_key] = 0 if pre_config[core_key] else 1
736 else:
737 global_key, core_key = item
738 expected[global_key] = pre_config[core_key]
739
740 self.assertEqual(configs['global_config'], expected)
741
742 def check_all_configs(self, testname, expected_config=None,
743 expected_preconfig=None,
744 modify_path_cb=None,
745 stderr=None, *, api, preconfig_api=None,
746 env=None, ignore_stderr=False, cwd=None):
747 new_env = remove_python_envvars()
748 if env is not None:
749 new_env.update(env)
750 env = new_env
751
752 if preconfig_api is None:
753 preconfig_api = api
754 if preconfig_api == API_ISOLATED:
755 default_preconfig = self.PRE_CONFIG_ISOLATED
756 elif preconfig_api == API_PYTHON:
757 default_preconfig = self.PRE_CONFIG_PYTHON
758 else:
759 default_preconfig = self.PRE_CONFIG_COMPAT
760 if expected_preconfig is None:
761 expected_preconfig = {}
762 expected_preconfig = dict(default_preconfig, **expected_preconfig)
763
764 if expected_config is None:
765 expected_config = {}
766
767 if api == API_PYTHON:
768 default_config = self.CONFIG_PYTHON
769 elif api == API_ISOLATED:
770 default_config = self.CONFIG_ISOLATED
771 else:
772 default_config = self.CONFIG_COMPAT
773 expected_config = dict(default_config, **expected_config)
774
775 self.get_expected_config(expected_preconfig,
776 expected_config,
777 env,
778 api, modify_path_cb)
779
780 out, err = self.run_embedded_interpreter(testname,
781 env=env, cwd=cwd)
782 if stderr is None and not expected_config['verbose']:
783 stderr = ""
784 if stderr is not None and not ignore_stderr:
785 self.assertEqual(err.rstrip(), stderr)
786 try:
787 configs = json.loads(out)
788 except json.JSONDecodeError:
789 self.fail(f"fail to decode stdout: {out!r}")
790
791 self.check_pre_config(configs, expected_preconfig)
792 self.check_config(configs, expected_config)
793 self.check_global_config(configs)
794 return configs
795
796 def test_init_default_config(self):
797 self.check_all_configs("test_init_initialize_config", api=API_COMPAT)
798
799 def test_preinit_compat_config(self):
800 self.check_all_configs("test_preinit_compat_config", api=API_COMPAT)
801
802 def test_init_compat_config(self):
803 self.check_all_configs("test_init_compat_config", api=API_COMPAT)
804
805 def test_init_global_config(self):
806 preconfig = {
807 'utf8_mode': 1,
808 }
809 config = {
810 'program_name': './globalvar',
811 'site_import': 0,
812 'bytes_warning': 1,
813 'warnoptions': ['default::BytesWarning'],
814 'inspect': 1,
815 'interactive': 1,
816 'optimization_level': 2,
817 'write_bytecode': 0,
818 'verbose': 1,
819 'quiet': 1,
820 'buffered_stdio': 0,
821
822 'user_site_directory': 0,
823 'pathconfig_warnings': 0,
824 }
825 self.check_all_configs("test_init_global_config", config, preconfig,
826 api=API_COMPAT)
827
828 def test_init_from_config(self):
829 preconfig = {
830 'allocator': PYMEM_ALLOCATOR_MALLOC,
831 'utf8_mode': 1,
832 }
833 config = {
834 'install_signal_handlers': 0,
835 'use_hash_seed': 1,
836 'hash_seed': 123,
837 'tracemalloc': 2,
838 'import_time': 1,
839 'code_debug_ranges': 0,
840 'show_ref_count': 1,
841 'malloc_stats': 1,
842
843 'stdio_encoding': 'iso8859-1',
844 'stdio_errors': 'replace',
845
846 'pycache_prefix': 'conf_pycache_prefix',
847 'program_name': './conf_program_name',
848 'argv': ['-c', 'arg2'],
849 'orig_argv': ['python3',
850 '-W', 'cmdline_warnoption',
851 '-X', 'cmdline_xoption',
852 '-c', 'pass',
853 'arg2'],
854 'parse_argv': 2,
855 'xoptions': [
856 'config_xoption1=3',
857 'config_xoption2=',
858 'config_xoption3',
859 'cmdline_xoption',
860 ],
861 'warnoptions': [
862 'cmdline_warnoption',
863 'default::BytesWarning',
864 'config_warnoption',
865 ],
866 'run_command': 'pass\n',
867
868 'site_import': 0,
869 'bytes_warning': 1,
870 'inspect': 1,
871 'interactive': 1,
872 'optimization_level': 2,
873 'write_bytecode': 0,
874 'verbose': 1,
875 'quiet': 1,
876 'configure_c_stdio': 1,
877 'buffered_stdio': 0,
878 'user_site_directory': 0,
879 'faulthandler': 1,
880 'platlibdir': 'my_platlibdir',
881 'module_search_paths': self.IGNORE_CONFIG,
882 'safe_path': 1,
883
884 'check_hash_pycs_mode': 'always',
885 'pathconfig_warnings': 0,
886
887 '_isolated_interpreter': 1,
888 }
889 self.check_all_configs("test_init_from_config", config, preconfig,
890 api=API_COMPAT)
891
892 def test_init_compat_env(self):
893 preconfig = {
894 'allocator': PYMEM_ALLOCATOR_MALLOC,
895 }
896 config = {
897 'use_hash_seed': 1,
898 'hash_seed': 42,
899 'tracemalloc': 2,
900 'import_time': 1,
901 'code_debug_ranges': 0,
902 'malloc_stats': 1,
903 'inspect': 1,
904 'optimization_level': 2,
905 'pythonpath_env': '/my/path',
906 'pycache_prefix': 'env_pycache_prefix',
907 'write_bytecode': 0,
908 'verbose': 1,
909 'buffered_stdio': 0,
910 'stdio_encoding': 'iso8859-1',
911 'stdio_errors': 'replace',
912 'user_site_directory': 0,
913 'faulthandler': 1,
914 'warnoptions': ['EnvVar'],
915 'platlibdir': 'env_platlibdir',
916 'module_search_paths': self.IGNORE_CONFIG,
917 'safe_path': 1,
918 }
919 self.check_all_configs("test_init_compat_env", config, preconfig,
920 api=API_COMPAT)
921
922 def test_init_python_env(self):
923 preconfig = {
924 'allocator': PYMEM_ALLOCATOR_MALLOC,
925 'utf8_mode': 1,
926 }
927 config = {
928 'use_hash_seed': 1,
929 'hash_seed': 42,
930 'tracemalloc': 2,
931 'import_time': 1,
932 'code_debug_ranges': 0,
933 'malloc_stats': 1,
934 'inspect': 1,
935 'optimization_level': 2,
936 'pythonpath_env': '/my/path',
937 'pycache_prefix': 'env_pycache_prefix',
938 'write_bytecode': 0,
939 'verbose': 1,
940 'buffered_stdio': 0,
941 'stdio_encoding': 'iso8859-1',
942 'stdio_errors': 'replace',
943 'user_site_directory': 0,
944 'faulthandler': 1,
945 'warnoptions': ['EnvVar'],
946 'platlibdir': 'env_platlibdir',
947 'module_search_paths': self.IGNORE_CONFIG,
948 'safe_path': 1,
949 }
950 self.check_all_configs("test_init_python_env", config, preconfig,
951 api=API_PYTHON)
952
953 def test_init_env_dev_mode(self):
954 preconfig = dict(allocator=PYMEM_ALLOCATOR_DEBUG)
955 config = dict(dev_mode=1,
956 faulthandler=1,
957 warnoptions=['default'])
958 self.check_all_configs("test_init_env_dev_mode", config, preconfig,
959 api=API_COMPAT)
960
961 def test_init_env_dev_mode_alloc(self):
962 preconfig = dict(allocator=PYMEM_ALLOCATOR_MALLOC)
963 config = dict(dev_mode=1,
964 faulthandler=1,
965 warnoptions=['default'])
966 self.check_all_configs("test_init_env_dev_mode_alloc", config, preconfig,
967 api=API_COMPAT)
968
969 def test_init_dev_mode(self):
970 preconfig = {
971 'allocator': PYMEM_ALLOCATOR_DEBUG,
972 }
973 config = {
974 'faulthandler': 1,
975 'dev_mode': 1,
976 'warnoptions': ['default'],
977 }
978 self.check_all_configs("test_init_dev_mode", config, preconfig,
979 api=API_PYTHON)
980
981 def test_preinit_parse_argv(self):
982 # Pre-initialize implicitly using argv: make sure that -X dev
983 # is used to configure the allocation in preinitialization
984 preconfig = {
985 'allocator': PYMEM_ALLOCATOR_DEBUG,
986 }
987 config = {
988 'argv': ['script.py'],
989 'orig_argv': ['python3', '-X', 'dev', '-P', 'script.py'],
990 'run_filename': os.path.abspath('script.py'),
991 'dev_mode': 1,
992 'faulthandler': 1,
993 'warnoptions': ['default'],
994 'xoptions': ['dev'],
995 'safe_path': 1,
996 }
997 self.check_all_configs("test_preinit_parse_argv", config, preconfig,
998 api=API_PYTHON)
999
1000 def test_preinit_dont_parse_argv(self):
1001 # -X dev must be ignored by isolated preconfiguration
1002 preconfig = {
1003 'isolated': 0,
1004 }
1005 argv = ["python3",
1006 "-E", "-I", "-P",
1007 "-X", "dev",
1008 "-X", "utf8",
1009 "script.py"]
1010 config = {
1011 'argv': argv,
1012 'orig_argv': argv,
1013 'isolated': 0,
1014 }
1015 self.check_all_configs("test_preinit_dont_parse_argv", config, preconfig,
1016 api=API_ISOLATED)
1017
1018 def test_init_isolated_flag(self):
1019 config = {
1020 'isolated': 1,
1021 'safe_path': 1,
1022 'use_environment': 0,
1023 'user_site_directory': 0,
1024 }
1025 self.check_all_configs("test_init_isolated_flag", config, api=API_PYTHON)
1026
1027 def test_preinit_isolated1(self):
1028 # _PyPreConfig.isolated=1, _PyCoreConfig.isolated not set
1029 config = {
1030 'isolated': 1,
1031 'safe_path': 1,
1032 'use_environment': 0,
1033 'user_site_directory': 0,
1034 }
1035 self.check_all_configs("test_preinit_isolated1", config, api=API_COMPAT)
1036
1037 def test_preinit_isolated2(self):
1038 # _PyPreConfig.isolated=0, _PyCoreConfig.isolated=1
1039 config = {
1040 'isolated': 1,
1041 'safe_path': 1,
1042 'use_environment': 0,
1043 'user_site_directory': 0,
1044 }
1045 self.check_all_configs("test_preinit_isolated2", config, api=API_COMPAT)
1046
1047 def test_preinit_isolated_config(self):
1048 self.check_all_configs("test_preinit_isolated_config", api=API_ISOLATED)
1049
1050 def test_init_isolated_config(self):
1051 self.check_all_configs("test_init_isolated_config", api=API_ISOLATED)
1052
1053 def test_preinit_python_config(self):
1054 self.check_all_configs("test_preinit_python_config", api=API_PYTHON)
1055
1056 def test_init_python_config(self):
1057 self.check_all_configs("test_init_python_config", api=API_PYTHON)
1058
1059 def test_init_dont_configure_locale(self):
1060 # _PyPreConfig.configure_locale=0
1061 preconfig = {
1062 'configure_locale': 0,
1063 'coerce_c_locale': 0,
1064 }
1065 self.check_all_configs("test_init_dont_configure_locale", {}, preconfig,
1066 api=API_PYTHON)
1067
1068 @unittest.skip('as of 3.11 this test no longer works because '
1069 'path calculations do not occur on read')
1070 def test_init_read_set(self):
1071 config = {
1072 'program_name': './init_read_set',
1073 'executable': 'my_executable',
1074 'base_executable': 'my_executable',
1075 }
1076 def modify_path(path):
1077 path.insert(1, "test_path_insert1")
1078 path.append("test_path_append")
1079 self.check_all_configs("test_init_read_set", config,
1080 api=API_PYTHON,
1081 modify_path_cb=modify_path)
1082
1083 def test_init_sys_add(self):
1084 config = {
1085 'faulthandler': 1,
1086 'xoptions': [
1087 'config_xoption',
1088 'cmdline_xoption',
1089 'sysadd_xoption',
1090 'faulthandler',
1091 ],
1092 'warnoptions': [
1093 'ignore:::cmdline_warnoption',
1094 'ignore:::sysadd_warnoption',
1095 'ignore:::config_warnoption',
1096 ],
1097 'orig_argv': ['python3',
1098 '-W', 'ignore:::cmdline_warnoption',
1099 '-X', 'cmdline_xoption'],
1100 }
1101 self.check_all_configs("test_init_sys_add", config, api=API_PYTHON)
1102
1103 def test_init_run_main(self):
1104 code = ('import _testinternalcapi, json; '
1105 'print(json.dumps(_testinternalcapi.get_configs()))')
1106 config = {
1107 'argv': ['-c', 'arg2'],
1108 'orig_argv': ['python3', '-c', code, 'arg2'],
1109 'program_name': './python3',
1110 'run_command': code + '\n',
1111 'parse_argv': 2,
1112 }
1113 self.check_all_configs("test_init_run_main", config, api=API_PYTHON)
1114
1115 def test_init_main(self):
1116 code = ('import _testinternalcapi, json; '
1117 'print(json.dumps(_testinternalcapi.get_configs()))')
1118 config = {
1119 'argv': ['-c', 'arg2'],
1120 'orig_argv': ['python3',
1121 '-c', code,
1122 'arg2'],
1123 'program_name': './python3',
1124 'run_command': code + '\n',
1125 'parse_argv': 2,
1126 '_init_main': 0,
1127 }
1128 self.check_all_configs("test_init_main", config,
1129 api=API_PYTHON,
1130 stderr="Run Python code before _Py_InitializeMain")
1131
1132 def test_init_parse_argv(self):
1133 config = {
1134 'parse_argv': 2,
1135 'argv': ['-c', 'arg1', '-v', 'arg3'],
1136 'orig_argv': ['./argv0', '-E', '-c', 'pass', 'arg1', '-v', 'arg3'],
1137 'program_name': './argv0',
1138 'run_command': 'pass\n',
1139 'use_environment': 0,
1140 }
1141 self.check_all_configs("test_init_parse_argv", config, api=API_PYTHON)
1142
1143 def test_init_dont_parse_argv(self):
1144 pre_config = {
1145 'parse_argv': 0,
1146 }
1147 config = {
1148 'parse_argv': 0,
1149 'argv': ['./argv0', '-E', '-c', 'pass', 'arg1', '-v', 'arg3'],
1150 'orig_argv': ['./argv0', '-E', '-c', 'pass', 'arg1', '-v', 'arg3'],
1151 'program_name': './argv0',
1152 }
1153 self.check_all_configs("test_init_dont_parse_argv", config, pre_config,
1154 api=API_PYTHON)
1155
1156 def default_program_name(self, config):
1157 if MS_WINDOWS:
1158 program_name = 'python'
1159 executable = self.test_exe
1160 else:
1161 program_name = 'python3'
1162 if MACOS:
1163 executable = self.test_exe
1164 else:
1165 executable = shutil.which(program_name) or ''
1166 config.update({
1167 'program_name': program_name,
1168 'base_executable': executable,
1169 'executable': executable,
1170 })
1171
1172 def test_init_setpath(self):
1173 # Test Py_SetPath()
1174 config = self._get_expected_config()
1175 paths = config['config']['module_search_paths']
1176
1177 config = {
1178 'module_search_paths': paths,
1179 'prefix': '',
1180 'base_prefix': '',
1181 'exec_prefix': '',
1182 'base_exec_prefix': '',
1183 # The current getpath.c doesn't determine the stdlib dir
1184 # in this case.
1185 'stdlib_dir': '',
1186 }
1187 self.default_program_name(config)
1188 env = {'TESTPATH': os.path.pathsep.join(paths)}
1189
1190 self.check_all_configs("test_init_setpath", config,
1191 api=API_COMPAT, env=env,
1192 ignore_stderr=True)
1193
1194 def test_init_setpath_config(self):
1195 # Test Py_SetPath() with PyConfig
1196 config = self._get_expected_config()
1197 paths = config['config']['module_search_paths']
1198
1199 config = {
1200 # set by Py_SetPath()
1201 'module_search_paths': paths,
1202 'prefix': '',
1203 'base_prefix': '',
1204 'exec_prefix': '',
1205 'base_exec_prefix': '',
1206 # The current getpath.c doesn't determine the stdlib dir
1207 # in this case.
1208 'stdlib_dir': '',
1209 'use_frozen_modules': not Py_DEBUG,
1210 # overridden by PyConfig
1211 'program_name': 'conf_program_name',
1212 'base_executable': 'conf_executable',
1213 'executable': 'conf_executable',
1214 }
1215 env = {'TESTPATH': os.path.pathsep.join(paths)}
1216 self.check_all_configs("test_init_setpath_config", config,
1217 api=API_PYTHON, env=env, ignore_stderr=True)
1218
1219 def module_search_paths(self, prefix=None, exec_prefix=None):
1220 config = self._get_expected_config()
1221 if prefix is None:
1222 prefix = config['config']['prefix']
1223 if exec_prefix is None:
1224 exec_prefix = config['config']['prefix']
1225 if MS_WINDOWS:
1226 return config['config']['module_search_paths']
1227 else:
1228 ver = sys.version_info
1229 return [
1230 os.path.join(prefix, sys.platlibdir,
1231 f'python{ver.major}{ver.minor}.zip'),
1232 os.path.join(prefix, sys.platlibdir,
1233 f'python{ver.major}.{ver.minor}'),
1234 os.path.join(exec_prefix, sys.platlibdir,
1235 f'python{ver.major}.{ver.minor}', 'lib-dynload'),
1236 ]
1237
1238 @contextlib.contextmanager
1239 def tmpdir_with_python(self, subdir=None):
1240 # Temporary directory with a copy of the Python program
1241 with tempfile.TemporaryDirectory() as tmpdir:
1242 # bpo-38234: On macOS and FreeBSD, the temporary directory
1243 # can be symbolic link. For example, /tmp can be a symbolic link
1244 # to /var/tmp. Call realpath() to resolve all symbolic links.
1245 tmpdir = os.path.realpath(tmpdir)
1246 if subdir:
1247 tmpdir = os.path.normpath(os.path.join(tmpdir, subdir))
1248 os.makedirs(tmpdir)
1249
1250 if MS_WINDOWS:
1251 # Copy pythonXY.dll (or pythonXY_d.dll)
1252 import fnmatch
1253 exedir = os.path.dirname(self.test_exe)
1254 for f in os.listdir(exedir):
1255 if fnmatch.fnmatch(f, '*.dll'):
1256 shutil.copyfile(os.path.join(exedir, f), os.path.join(tmpdir, f))
1257
1258 # Copy Python program
1259 exec_copy = os.path.join(tmpdir, os.path.basename(self.test_exe))
1260 shutil.copyfile(self.test_exe, exec_copy)
1261 shutil.copystat(self.test_exe, exec_copy)
1262 self.test_exe = exec_copy
1263
1264 yield tmpdir
1265
1266 def test_init_setpythonhome(self):
1267 # Test Py_SetPythonHome(home) with PYTHONPATH env var
1268 config = self._get_expected_config()
1269 paths = config['config']['module_search_paths']
1270 paths_str = os.path.pathsep.join(paths)
1271
1272 for path in paths:
1273 if not os.path.isdir(path):
1274 continue
1275 if os.path.exists(os.path.join(path, 'os.py')):
1276 home = os.path.dirname(path)
1277 break
1278 else:
1279 self.fail(f"Unable to find home in {paths!r}")
1280
1281 prefix = exec_prefix = home
1282 if MS_WINDOWS:
1283 stdlib = os.path.join(home, "Lib")
1284 # Because we are specifying 'home', module search paths
1285 # are fairly static
1286 expected_paths = [paths[0], os.path.join(home, 'DLLs'), stdlib]
1287 else:
1288 version = f'{sys.version_info.major}.{sys.version_info.minor}'
1289 stdlib = os.path.join(home, sys.platlibdir, f'python{version}')
1290 expected_paths = self.module_search_paths(prefix=home, exec_prefix=home)
1291
1292 config = {
1293 'home': home,
1294 'module_search_paths': expected_paths,
1295 'prefix': prefix,
1296 'base_prefix': prefix,
1297 'exec_prefix': exec_prefix,
1298 'base_exec_prefix': exec_prefix,
1299 'pythonpath_env': paths_str,
1300 'stdlib_dir': stdlib,
1301 }
1302 self.default_program_name(config)
1303 env = {'TESTHOME': home, 'PYTHONPATH': paths_str}
1304 self.check_all_configs("test_init_setpythonhome", config,
1305 api=API_COMPAT, env=env)
1306
1307 def test_init_is_python_build_with_home(self):
1308 # Test _Py_path_config._is_python_build configuration (gh-91985)
1309 config = self._get_expected_config()
1310 paths = config['config']['module_search_paths']
1311 paths_str = os.path.pathsep.join(paths)
1312
1313 for path in paths:
1314 if not os.path.isdir(path):
1315 continue
1316 if os.path.exists(os.path.join(path, 'os.py')):
1317 home = os.path.dirname(path)
1318 break
1319 else:
1320 self.fail(f"Unable to find home in {paths!r}")
1321
1322 prefix = exec_prefix = home
1323 if MS_WINDOWS:
1324 stdlib = os.path.join(home, "Lib")
1325 # Because we are specifying 'home', module search paths
1326 # are fairly static
1327 expected_paths = [paths[0], os.path.join(home, 'DLLs'), stdlib]
1328 else:
1329 version = f'{sys.version_info.major}.{sys.version_info.minor}'
1330 stdlib = os.path.join(home, sys.platlibdir, f'python{version}')
1331 expected_paths = self.module_search_paths(prefix=home, exec_prefix=home)
1332
1333 config = {
1334 'home': home,
1335 'module_search_paths': expected_paths,
1336 'prefix': prefix,
1337 'base_prefix': prefix,
1338 'exec_prefix': exec_prefix,
1339 'base_exec_prefix': exec_prefix,
1340 'pythonpath_env': paths_str,
1341 'stdlib_dir': stdlib,
1342 }
1343 # The code above is taken from test_init_setpythonhome()
1344 env = {'TESTHOME': home, 'PYTHONPATH': paths_str}
1345
1346 env['NEGATIVE_ISPYTHONBUILD'] = '1'
1347 config['_is_python_build'] = 0
1348 self.check_all_configs("test_init_is_python_build", config,
1349 api=API_COMPAT, env=env)
1350
1351 env['NEGATIVE_ISPYTHONBUILD'] = '0'
1352 config['_is_python_build'] = 1
1353 exedir = os.path.dirname(sys.executable)
1354 with open(os.path.join(exedir, 'pybuilddir.txt'), encoding='utf8') as f:
1355 expected_paths[1 if MS_WINDOWS else 2] = os.path.normpath(
1356 os.path.join(exedir, f'{f.read()}\n$'.splitlines()[0]))
1357 if not MS_WINDOWS:
1358 # PREFIX (default) is set when running in build directory
1359 prefix = exec_prefix = sys.prefix
1360 # stdlib calculation (/Lib) is not yet supported
1361 expected_paths[0] = self.module_search_paths(prefix=prefix)[0]
1362 config.update(prefix=prefix, base_prefix=prefix,
1363 exec_prefix=exec_prefix, base_exec_prefix=exec_prefix)
1364 self.check_all_configs("test_init_is_python_build", config,
1365 api=API_COMPAT, env=env)
1366
1367 def copy_paths_by_env(self, config):
1368 all_configs = self._get_expected_config()
1369 paths = all_configs['config']['module_search_paths']
1370 paths_str = os.path.pathsep.join(paths)
1371 config['pythonpath_env'] = paths_str
1372 env = {'PYTHONPATH': paths_str}
1373 return env
1374
1375 @unittest.skipIf(MS_WINDOWS, 'See test_init_pybuilddir_win32')
1376 def test_init_pybuilddir(self):
1377 # Test path configuration with pybuilddir.txt configuration file
1378
1379 with self.tmpdir_with_python() as tmpdir:
1380 # pybuilddir.txt is a sub-directory relative to the current
1381 # directory (tmpdir)
1382 vpath = sysconfig.get_config_var("VPATH") or ''
1383 subdir = 'libdir'
1384 libdir = os.path.join(tmpdir, subdir)
1385 # The stdlib dir is dirname(executable) + VPATH + 'Lib'
1386 stdlibdir = os.path.normpath(os.path.join(tmpdir, vpath, 'Lib'))
1387 os.mkdir(libdir)
1388
1389 filename = os.path.join(tmpdir, 'pybuilddir.txt')
1390 with open(filename, "w", encoding="utf8") as fp:
1391 fp.write(subdir)
1392
1393 module_search_paths = self.module_search_paths()
1394 module_search_paths[-2] = stdlibdir
1395 module_search_paths[-1] = libdir
1396
1397 executable = self.test_exe
1398 config = {
1399 'base_exec_prefix': sysconfig.get_config_var("exec_prefix"),
1400 'base_prefix': sysconfig.get_config_var("prefix"),
1401 'base_executable': executable,
1402 'executable': executable,
1403 'module_search_paths': module_search_paths,
1404 'stdlib_dir': stdlibdir,
1405 }
1406 env = self.copy_paths_by_env(config)
1407 self.check_all_configs("test_init_compat_config", config,
1408 api=API_COMPAT, env=env,
1409 ignore_stderr=True, cwd=tmpdir)
1410
1411 @unittest.skipUnless(MS_WINDOWS, 'See test_init_pybuilddir')
1412 def test_init_pybuilddir_win32(self):
1413 # Test path configuration with pybuilddir.txt configuration file
1414
1415 vpath = sysconfig.get_config_var("VPATH")
1416 subdir = r'PCbuild\arch'
1417 if os.path.normpath(vpath).count(os.sep) == 2:
1418 subdir = os.path.join(subdir, 'instrumented')
1419
1420 with self.tmpdir_with_python(subdir) as tmpdir:
1421 # The prefix is dirname(executable) + VPATH
1422 prefix = os.path.normpath(os.path.join(tmpdir, vpath))
1423 # The stdlib dir is dirname(executable) + VPATH + 'Lib'
1424 stdlibdir = os.path.normpath(os.path.join(tmpdir, vpath, 'Lib'))
1425
1426 filename = os.path.join(tmpdir, 'pybuilddir.txt')
1427 with open(filename, "w", encoding="utf8") as fp:
1428 fp.write(tmpdir)
1429
1430 module_search_paths = self.module_search_paths()
1431 module_search_paths[-3] = os.path.join(tmpdir, os.path.basename(module_search_paths[-3]))
1432 module_search_paths[-2] = tmpdir
1433 module_search_paths[-1] = stdlibdir
1434
1435 executable = self.test_exe
1436 config = {
1437 'base_exec_prefix': prefix,
1438 'base_prefix': prefix,
1439 'base_executable': executable,
1440 'executable': executable,
1441 'prefix': prefix,
1442 'exec_prefix': prefix,
1443 'module_search_paths': module_search_paths,
1444 'stdlib_dir': stdlibdir,
1445 }
1446 env = self.copy_paths_by_env(config)
1447 self.check_all_configs("test_init_compat_config", config,
1448 api=API_COMPAT, env=env,
1449 ignore_stderr=False, cwd=tmpdir)
1450
1451 def test_init_pyvenv_cfg(self):
1452 # Test path configuration with pyvenv.cfg configuration file
1453
1454 with self.tmpdir_with_python() as tmpdir, \
1455 tempfile.TemporaryDirectory() as pyvenv_home:
1456 ver = sys.version_info
1457
1458 if not MS_WINDOWS:
1459 lib_dynload = os.path.join(pyvenv_home,
1460 sys.platlibdir,
1461 f'python{ver.major}.{ver.minor}',
1462 'lib-dynload')
1463 os.makedirs(lib_dynload)
1464 else:
1465 lib_folder = os.path.join(pyvenv_home, 'Lib')
1466 os.makedirs(lib_folder)
1467 # getpath.py uses Lib\os.py as the LANDMARK
1468 shutil.copyfile(
1469 os.path.join(support.STDLIB_DIR, 'os.py'),
1470 os.path.join(lib_folder, 'os.py'),
1471 )
1472
1473 filename = os.path.join(tmpdir, 'pyvenv.cfg')
1474 with open(filename, "w", encoding="utf8") as fp:
1475 print("home = %s" % pyvenv_home, file=fp)
1476 print("include-system-site-packages = false", file=fp)
1477
1478 paths = self.module_search_paths()
1479 if not MS_WINDOWS:
1480 paths[-1] = lib_dynload
1481 else:
1482 paths = [
1483 os.path.join(tmpdir, os.path.basename(paths[0])),
1484 pyvenv_home,
1485 os.path.join(pyvenv_home, "Lib"),
1486 ]
1487
1488 executable = self.test_exe
1489 base_executable = os.path.join(pyvenv_home, os.path.basename(executable))
1490 exec_prefix = pyvenv_home
1491 config = {
1492 'base_prefix': sysconfig.get_config_var("prefix"),
1493 'base_exec_prefix': exec_prefix,
1494 'exec_prefix': exec_prefix,
1495 'base_executable': base_executable,
1496 'executable': executable,
1497 'module_search_paths': paths,
1498 }
1499 if MS_WINDOWS:
1500 config['base_prefix'] = pyvenv_home
1501 config['prefix'] = pyvenv_home
1502 config['stdlib_dir'] = os.path.join(pyvenv_home, 'Lib')
1503 config['use_frozen_modules'] = int(not Py_DEBUG)
1504 else:
1505 # cannot reliably assume stdlib_dir here because it
1506 # depends too much on our build. But it ought to be found
1507 config['stdlib_dir'] = self.IGNORE_CONFIG
1508 config['use_frozen_modules'] = int(not Py_DEBUG)
1509
1510 env = self.copy_paths_by_env(config)
1511 self.check_all_configs("test_init_compat_config", config,
1512 api=API_COMPAT, env=env,
1513 ignore_stderr=True, cwd=tmpdir)
1514
1515 @unittest.skipUnless(MS_WINDOWS, 'specific to Windows')
1516 def test_getpath_abspath_win32(self):
1517 # Check _Py_abspath() is passed a backslashed path not to fall back to
1518 # GetFullPathNameW() on startup, which (re-)normalizes the path overly.
1519 # Currently, _Py_normpath() doesn't trim trailing dots and spaces.
1520 CASES = [
1521 ("C:/a. . .", "C:\\a. . ."),
1522 ("C:\\a. . .", "C:\\a. . ."),
1523 ("\\\\?\\C:////a////b. . .", "\\\\?\\C:\\a\\b. . ."),
1524 ("//a/b/c. . .", "\\\\a\\b\\c. . ."),
1525 ("\\\\a\\b\\c. . .", "\\\\a\\b\\c. . ."),
1526 ("a. . .", f"{os.getcwd()}\\a"), # relpath gets fully normalized
1527 ]
1528 out, err = self.run_embedded_interpreter(
1529 "test_init_initialize_config",
1530 env={**remove_python_envvars(),
1531 "PYTHONPATH": os.path.pathsep.join(c[0] for c in CASES)}
1532 )
1533 self.assertEqual(err, "")
1534 try:
1535 out = json.loads(out)
1536 except json.JSONDecodeError:
1537 self.fail(f"fail to decode stdout: {out!r}")
1538
1539 results = out['config']["module_search_paths"]
1540 for (_, expected), result in zip(CASES, results):
1541 self.assertEqual(result, expected)
1542
1543 def test_global_pathconfig(self):
1544 # Test C API functions getting the path configuration:
1545 #
1546 # - Py_GetExecPrefix()
1547 # - Py_GetPath()
1548 # - Py_GetPrefix()
1549 # - Py_GetProgramFullPath()
1550 # - Py_GetProgramName()
1551 # - Py_GetPythonHome()
1552 #
1553 # The global path configuration (_Py_path_config) must be a copy
1554 # of the path configuration of PyInterpreter.config (PyConfig).
1555 ctypes = import_helper.import_module('ctypes')
1556 _testinternalcapi = import_helper.import_module('_testinternalcapi')
1557
1558 def get_func(name):
1559 func = getattr(ctypes.pythonapi, name)
1560 func.argtypes = ()
1561 func.restype = ctypes.c_wchar_p
1562 return func
1563
1564 Py_GetPath = get_func('Py_GetPath')
1565 Py_GetPrefix = get_func('Py_GetPrefix')
1566 Py_GetExecPrefix = get_func('Py_GetExecPrefix')
1567 Py_GetProgramName = get_func('Py_GetProgramName')
1568 Py_GetProgramFullPath = get_func('Py_GetProgramFullPath')
1569 Py_GetPythonHome = get_func('Py_GetPythonHome')
1570
1571 config = _testinternalcapi.get_configs()['config']
1572
1573 self.assertEqual(Py_GetPath().split(os.path.pathsep),
1574 config['module_search_paths'])
1575 self.assertEqual(Py_GetPrefix(), config['prefix'])
1576 self.assertEqual(Py_GetExecPrefix(), config['exec_prefix'])
1577 self.assertEqual(Py_GetProgramName(), config['program_name'])
1578 self.assertEqual(Py_GetProgramFullPath(), config['executable'])
1579 self.assertEqual(Py_GetPythonHome(), config['home'])
1580
1581 def test_init_warnoptions(self):
1582 # lowest to highest priority
1583 warnoptions = [
1584 'ignore:::PyConfig_Insert0', # PyWideStringList_Insert(0)
1585 'default', # PyConfig.dev_mode=1
1586 'ignore:::env1', # PYTHONWARNINGS env var
1587 'ignore:::env2', # PYTHONWARNINGS env var
1588 'ignore:::cmdline1', # -W opt command line option
1589 'ignore:::cmdline2', # -W opt command line option
1590 'default::BytesWarning', # PyConfig.bytes_warnings=1
1591 'ignore:::PySys_AddWarnOption1', # PySys_AddWarnOption()
1592 'ignore:::PySys_AddWarnOption2', # PySys_AddWarnOption()
1593 'ignore:::PyConfig_BeforeRead', # PyConfig.warnoptions
1594 'ignore:::PyConfig_AfterRead'] # PyWideStringList_Append()
1595 preconfig = dict(allocator=PYMEM_ALLOCATOR_DEBUG)
1596 config = {
1597 'dev_mode': 1,
1598 'faulthandler': 1,
1599 'bytes_warning': 1,
1600 'warnoptions': warnoptions,
1601 'orig_argv': ['python3',
1602 '-Wignore:::cmdline1',
1603 '-Wignore:::cmdline2'],
1604 }
1605 self.check_all_configs("test_init_warnoptions", config, preconfig,
1606 api=API_PYTHON)
1607
1608 def test_init_set_config(self):
1609 config = {
1610 '_init_main': 0,
1611 'bytes_warning': 2,
1612 'warnoptions': ['error::BytesWarning'],
1613 }
1614 self.check_all_configs("test_init_set_config", config,
1615 api=API_ISOLATED)
1616
1617 def test_get_argc_argv(self):
1618 self.run_embedded_interpreter("test_get_argc_argv")
1619 # ignore output
1620
1621 def test_init_use_frozen_modules(self):
1622 tests = {
1623 ('=on', 1),
1624 ('=off', 0),
1625 ('=', 1),
1626 ('', 1),
1627 }
1628 for raw, expected in tests:
1629 optval = f'frozen_modules{raw}'
1630 config = {
1631 'parse_argv': 2,
1632 'argv': ['-c'],
1633 'orig_argv': ['./argv0', '-X', optval, '-c', 'pass'],
1634 'program_name': './argv0',
1635 'run_command': 'pass\n',
1636 'use_environment': 1,
1637 'xoptions': [optval],
1638 'use_frozen_modules': expected,
1639 }
1640 env = {'TESTFROZEN': raw[1:]} if raw else None
1641 with self.subTest(repr(raw)):
1642 self.check_all_configs("test_init_use_frozen_modules", config,
1643 api=API_PYTHON, env=env)
1644
1645
1646 class ESC[4;38;5;81mSetConfigTests(ESC[4;38;5;149munittestESC[4;38;5;149m.ESC[4;38;5;149mTestCase):
1647 def test_set_config(self):
1648 # bpo-42260: Test _PyInterpreterState_SetConfig()
1649 import_helper.import_module('_testcapi')
1650 cmd = [sys.executable, '-X', 'utf8', '-I', '-m', 'test._test_embed_set_config']
1651 proc = subprocess.run(cmd,
1652 stdout=subprocess.PIPE,
1653 stderr=subprocess.PIPE,
1654 encoding='utf-8', errors='backslashreplace')
1655 if proc.returncode and support.verbose:
1656 print(proc.stdout)
1657 print(proc.stderr)
1658 self.assertEqual(proc.returncode, 0,
1659 (proc.returncode, proc.stdout, proc.stderr))
1660
1661
1662 class ESC[4;38;5;81mAuditingTests(ESC[4;38;5;149mEmbeddingTestsMixin, ESC[4;38;5;149munittestESC[4;38;5;149m.ESC[4;38;5;149mTestCase):
1663 def test_open_code_hook(self):
1664 self.run_embedded_interpreter("test_open_code_hook")
1665
1666 def test_audit(self):
1667 self.run_embedded_interpreter("test_audit")
1668
1669 def test_audit_subinterpreter(self):
1670 self.run_embedded_interpreter("test_audit_subinterpreter")
1671
1672 def test_audit_run_command(self):
1673 self.run_embedded_interpreter("test_audit_run_command",
1674 timeout=support.SHORT_TIMEOUT,
1675 returncode=1)
1676
1677 def test_audit_run_file(self):
1678 self.run_embedded_interpreter("test_audit_run_file",
1679 timeout=support.SHORT_TIMEOUT,
1680 returncode=1)
1681
1682 def test_audit_run_interactivehook(self):
1683 startup = os.path.join(self.oldcwd, os_helper.TESTFN) + ".py"
1684 with open(startup, "w", encoding="utf-8") as f:
1685 print("import sys", file=f)
1686 print("sys.__interactivehook__ = lambda: None", file=f)
1687 try:
1688 env = {**remove_python_envvars(), "PYTHONSTARTUP": startup}
1689 self.run_embedded_interpreter("test_audit_run_interactivehook",
1690 timeout=support.SHORT_TIMEOUT,
1691 returncode=10, env=env)
1692 finally:
1693 os.unlink(startup)
1694
1695 def test_audit_run_startup(self):
1696 startup = os.path.join(self.oldcwd, os_helper.TESTFN) + ".py"
1697 with open(startup, "w", encoding="utf-8") as f:
1698 print("pass", file=f)
1699 try:
1700 env = {**remove_python_envvars(), "PYTHONSTARTUP": startup}
1701 self.run_embedded_interpreter("test_audit_run_startup",
1702 timeout=support.SHORT_TIMEOUT,
1703 returncode=10, env=env)
1704 finally:
1705 os.unlink(startup)
1706
1707 def test_audit_run_stdin(self):
1708 self.run_embedded_interpreter("test_audit_run_stdin",
1709 timeout=support.SHORT_TIMEOUT,
1710 returncode=1)
1711
1712 def test_get_incomplete_frame(self):
1713 self.run_embedded_interpreter("test_get_incomplete_frame")
1714
1715
1716 class ESC[4;38;5;81mMiscTests(ESC[4;38;5;149mEmbeddingTestsMixin, ESC[4;38;5;149munittestESC[4;38;5;149m.ESC[4;38;5;149mTestCase):
1717 def test_unicode_id_init(self):
1718 # bpo-42882: Test that _PyUnicode_FromId() works
1719 # when Python is initialized multiples times.
1720 self.run_embedded_interpreter("test_unicode_id_init")
1721
1722 # See bpo-44133
1723 @unittest.skipIf(os.name == 'nt',
1724 'Py_FrozenMain is not exported on Windows')
1725 def test_frozenmain(self):
1726 env = dict(os.environ)
1727 env['PYTHONUNBUFFERED'] = '1'
1728 out, err = self.run_embedded_interpreter("test_frozenmain", env=env)
1729 executable = os.path.realpath('./argv0')
1730 expected = textwrap.dedent(f"""
1731 Frozen Hello World
1732 sys.argv ['./argv0', '-E', 'arg1', 'arg2']
1733 config program_name: ./argv0
1734 config executable: {executable}
1735 config use_environment: 1
1736 config configure_c_stdio: 1
1737 config buffered_stdio: 0
1738 """).lstrip()
1739 self.assertEqual(out, expected)
1740
1741 @unittest.skipUnless(hasattr(sys, 'gettotalrefcount'),
1742 '-X showrefcount requires a Python debug build')
1743 def test_no_memleak(self):
1744 # bpo-1635741: Python must release all memory at exit
1745 tests = (
1746 ('off', 'pass'),
1747 ('on', 'pass'),
1748 ('off', 'import __hello__'),
1749 ('on', 'import __hello__'),
1750 )
1751 for flag, stmt in tests:
1752 xopt = f"frozen_modules={flag}"
1753 cmd = [sys.executable, "-I", "-X", "showrefcount", "-X", xopt, "-c", stmt]
1754 proc = subprocess.run(cmd,
1755 stdout=subprocess.PIPE,
1756 stderr=subprocess.STDOUT,
1757 text=True)
1758 self.assertEqual(proc.returncode, 0)
1759 out = proc.stdout.rstrip()
1760 match = re.match(r'^\[(-?\d+) refs, (-?\d+) blocks\]', out)
1761 if not match:
1762 self.fail(f"unexpected output: {out!a}")
1763 refs = int(match.group(1))
1764 blocks = int(match.group(2))
1765 with self.subTest(frozen_modules=flag, stmt=stmt):
1766 self.assertEqual(refs, 0, out)
1767 self.assertEqual(blocks, 0, out)
1768
1769
1770 class ESC[4;38;5;81mStdPrinterTests(ESC[4;38;5;149mEmbeddingTestsMixin, ESC[4;38;5;149munittestESC[4;38;5;149m.ESC[4;38;5;149mTestCase):
1771 # Test PyStdPrinter_Type which is used by _PySys_SetPreliminaryStderr():
1772 # "Set up a preliminary stderr printer until we have enough
1773 # infrastructure for the io module in place."
1774
1775 STDOUT_FD = 1
1776
1777 def create_printer(self, fd):
1778 ctypes = import_helper.import_module('ctypes')
1779 PyFile_NewStdPrinter = ctypes.pythonapi.PyFile_NewStdPrinter
1780 PyFile_NewStdPrinter.argtypes = (ctypes.c_int,)
1781 PyFile_NewStdPrinter.restype = ctypes.py_object
1782 return PyFile_NewStdPrinter(fd)
1783
1784 def test_write(self):
1785 message = "unicode:\xe9-\u20ac-\udc80!\n"
1786
1787 stdout_fd = self.STDOUT_FD
1788 stdout_fd_copy = os.dup(stdout_fd)
1789 self.addCleanup(os.close, stdout_fd_copy)
1790
1791 rfd, wfd = os.pipe()
1792 self.addCleanup(os.close, rfd)
1793 self.addCleanup(os.close, wfd)
1794 try:
1795 # PyFile_NewStdPrinter() only accepts fileno(stdout)
1796 # or fileno(stderr) file descriptor.
1797 os.dup2(wfd, stdout_fd)
1798
1799 printer = self.create_printer(stdout_fd)
1800 printer.write(message)
1801 finally:
1802 os.dup2(stdout_fd_copy, stdout_fd)
1803
1804 data = os.read(rfd, 100)
1805 self.assertEqual(data, message.encode('utf8', 'backslashreplace'))
1806
1807 def test_methods(self):
1808 fd = self.STDOUT_FD
1809 printer = self.create_printer(fd)
1810 self.assertEqual(printer.fileno(), fd)
1811 self.assertEqual(printer.isatty(), os.isatty(fd))
1812 printer.flush() # noop
1813 printer.close() # noop
1814
1815 def test_disallow_instantiation(self):
1816 fd = self.STDOUT_FD
1817 printer = self.create_printer(fd)
1818 support.check_disallow_instantiation(self, type(printer))
1819
1820
1821 if __name__ == "__main__":
1822 unittest.main()