1 #
2 # Code used to start processes when using the spawn or forkserver
3 # start methods.
4 #
5 # multiprocessing/spawn.py
6 #
7 # Copyright (c) 2006-2008, R Oudkerk
8 # Licensed to PSF under a Contributor Agreement.
9 #
10
11 import os
12 import sys
13 import runpy
14 import types
15
16 from . import get_start_method, set_start_method
17 from . import process
18 from .context import reduction
19 from . import util
20
21 __all__ = ['_main', 'freeze_support', 'set_executable', 'get_executable',
22 'get_preparation_data', 'get_command_line', 'import_main_path']
23
24 #
25 # _python_exe is the assumed path to the python executable.
26 # People embedding Python want to modify it.
27 #
28
29 if sys.platform != 'win32':
30 WINEXE = False
31 WINSERVICE = False
32 else:
33 WINEXE = getattr(sys, 'frozen', False)
34 WINSERVICE = sys.executable and sys.executable.lower().endswith("pythonservice.exe")
35
36 def set_executable(exe):
37 global _python_exe
38 if exe is None:
39 _python_exe = exe
40 elif sys.platform == 'win32':
41 _python_exe = os.fsdecode(exe)
42 else:
43 _python_exe = os.fsencode(exe)
44
45 def get_executable():
46 return _python_exe
47
48 if WINSERVICE:
49 set_executable(os.path.join(sys.exec_prefix, 'python.exe'))
50 else:
51 set_executable(sys.executable)
52
53 #
54 #
55 #
56
57 def is_forking(argv):
58 '''
59 Return whether commandline indicates we are forking
60 '''
61 if len(argv) >= 2 and argv[1] == '--multiprocessing-fork':
62 return True
63 else:
64 return False
65
66
67 def freeze_support():
68 '''
69 Run code for process object if this in not the main process
70 '''
71 if is_forking(sys.argv):
72 kwds = {}
73 for arg in sys.argv[2:]:
74 name, value = arg.split('=')
75 if value == 'None':
76 kwds[name] = None
77 else:
78 kwds[name] = int(value)
79 spawn_main(**kwds)
80 sys.exit()
81
82
83 def get_command_line(**kwds):
84 '''
85 Returns prefix of command line used for spawning a child process
86 '''
87 if getattr(sys, 'frozen', False):
88 return ([sys.executable, '--multiprocessing-fork'] +
89 ['%s=%r' % item for item in kwds.items()])
90 else:
91 prog = 'from multiprocessing.spawn import spawn_main; spawn_main(%s)'
92 prog %= ', '.join('%s=%r' % item for item in kwds.items())
93 opts = util._args_from_interpreter_flags()
94 exe = get_executable()
95 return [exe] + opts + ['-c', prog, '--multiprocessing-fork']
96
97
98 def spawn_main(pipe_handle, parent_pid=None, tracker_fd=None):
99 '''
100 Run code specified by data received over pipe
101 '''
102 assert is_forking(sys.argv), "Not forking"
103 if sys.platform == 'win32':
104 import msvcrt
105 import _winapi
106
107 if parent_pid is not None:
108 source_process = _winapi.OpenProcess(
109 _winapi.SYNCHRONIZE | _winapi.PROCESS_DUP_HANDLE,
110 False, parent_pid)
111 else:
112 source_process = None
113 new_handle = reduction.duplicate(pipe_handle,
114 source_process=source_process)
115 fd = msvcrt.open_osfhandle(new_handle, os.O_RDONLY)
116 parent_sentinel = source_process
117 else:
118 from . import resource_tracker
119 resource_tracker._resource_tracker._fd = tracker_fd
120 fd = pipe_handle
121 parent_sentinel = os.dup(pipe_handle)
122 exitcode = _main(fd, parent_sentinel)
123 sys.exit(exitcode)
124
125
126 def _main(fd, parent_sentinel):
127 with os.fdopen(fd, 'rb', closefd=True) as from_parent:
128 process.current_process()._inheriting = True
129 try:
130 preparation_data = reduction.pickle.load(from_parent)
131 prepare(preparation_data)
132 self = reduction.pickle.load(from_parent)
133 finally:
134 del process.current_process()._inheriting
135 return self._bootstrap(parent_sentinel)
136
137
138 def _check_not_importing_main():
139 if getattr(process.current_process(), '_inheriting', False):
140 raise RuntimeError('''
141 An attempt has been made to start a new process before the
142 current process has finished its bootstrapping phase.
143
144 This probably means that you are not using fork to start your
145 child processes and you have forgotten to use the proper idiom
146 in the main module:
147
148 if __name__ == '__main__':
149 freeze_support()
150 ...
151
152 The "freeze_support()" line can be omitted if the program
153 is not going to be frozen to produce an executable.
154
155 To fix this issue, refer to the "Safe importing of main module"
156 section in https://docs.python.org/3/library/multiprocessing.html
157 ''')
158
159
160 def get_preparation_data(name):
161 '''
162 Return info about parent needed by child to unpickle process object
163 '''
164 _check_not_importing_main()
165 d = dict(
166 log_to_stderr=util._log_to_stderr,
167 authkey=process.current_process().authkey,
168 )
169
170 if util._logger is not None:
171 d['log_level'] = util._logger.getEffectiveLevel()
172
173 sys_path=sys.path.copy()
174 try:
175 i = sys_path.index('')
176 except ValueError:
177 pass
178 else:
179 sys_path[i] = process.ORIGINAL_DIR
180
181 d.update(
182 name=name,
183 sys_path=sys_path,
184 sys_argv=sys.argv,
185 orig_dir=process.ORIGINAL_DIR,
186 dir=os.getcwd(),
187 start_method=get_start_method(),
188 )
189
190 # Figure out whether to initialise main in the subprocess as a module
191 # or through direct execution (or to leave it alone entirely)
192 main_module = sys.modules['__main__']
193 main_mod_name = getattr(main_module.__spec__, "name", None)
194 if main_mod_name is not None:
195 d['init_main_from_name'] = main_mod_name
196 elif sys.platform != 'win32' or (not WINEXE and not WINSERVICE):
197 main_path = getattr(main_module, '__file__', None)
198 if main_path is not None:
199 if (not os.path.isabs(main_path) and
200 process.ORIGINAL_DIR is not None):
201 main_path = os.path.join(process.ORIGINAL_DIR, main_path)
202 d['init_main_from_path'] = os.path.normpath(main_path)
203
204 return d
205
206 #
207 # Prepare current process
208 #
209
210 old_main_modules = []
211
212 def prepare(data):
213 '''
214 Try to get current process ready to unpickle process object
215 '''
216 if 'name' in data:
217 process.current_process().name = data['name']
218
219 if 'authkey' in data:
220 process.current_process().authkey = data['authkey']
221
222 if 'log_to_stderr' in data and data['log_to_stderr']:
223 util.log_to_stderr()
224
225 if 'log_level' in data:
226 util.get_logger().setLevel(data['log_level'])
227
228 if 'sys_path' in data:
229 sys.path = data['sys_path']
230
231 if 'sys_argv' in data:
232 sys.argv = data['sys_argv']
233
234 if 'dir' in data:
235 os.chdir(data['dir'])
236
237 if 'orig_dir' in data:
238 process.ORIGINAL_DIR = data['orig_dir']
239
240 if 'start_method' in data:
241 set_start_method(data['start_method'], force=True)
242
243 if 'init_main_from_name' in data:
244 _fixup_main_from_name(data['init_main_from_name'])
245 elif 'init_main_from_path' in data:
246 _fixup_main_from_path(data['init_main_from_path'])
247
248 # Multiprocessing module helpers to fix up the main module in
249 # spawned subprocesses
250 def _fixup_main_from_name(mod_name):
251 # __main__.py files for packages, directories, zip archives, etc, run
252 # their "main only" code unconditionally, so we don't even try to
253 # populate anything in __main__, nor do we make any changes to
254 # __main__ attributes
255 current_main = sys.modules['__main__']
256 if mod_name == "__main__" or mod_name.endswith(".__main__"):
257 return
258
259 # If this process was forked, __main__ may already be populated
260 if getattr(current_main.__spec__, "name", None) == mod_name:
261 return
262
263 # Otherwise, __main__ may contain some non-main code where we need to
264 # support unpickling it properly. We rerun it as __mp_main__ and make
265 # the normal __main__ an alias to that
266 old_main_modules.append(current_main)
267 main_module = types.ModuleType("__mp_main__")
268 main_content = runpy.run_module(mod_name,
269 run_name="__mp_main__",
270 alter_sys=True)
271 main_module.__dict__.update(main_content)
272 sys.modules['__main__'] = sys.modules['__mp_main__'] = main_module
273
274
275 def _fixup_main_from_path(main_path):
276 # If this process was forked, __main__ may already be populated
277 current_main = sys.modules['__main__']
278
279 # Unfortunately, the main ipython launch script historically had no
280 # "if __name__ == '__main__'" guard, so we work around that
281 # by treating it like a __main__.py file
282 # See https://github.com/ipython/ipython/issues/4698
283 main_name = os.path.splitext(os.path.basename(main_path))[0]
284 if main_name == 'ipython':
285 return
286
287 # Otherwise, if __file__ already has the setting we expect,
288 # there's nothing more to do
289 if getattr(current_main, '__file__', None) == main_path:
290 return
291
292 # If the parent process has sent a path through rather than a module
293 # name we assume it is an executable script that may contain
294 # non-main code that needs to be executed
295 old_main_modules.append(current_main)
296 main_module = types.ModuleType("__mp_main__")
297 main_content = runpy.run_path(main_path,
298 run_name="__mp_main__")
299 main_module.__dict__.update(main_content)
300 sys.modules['__main__'] = sys.modules['__mp_main__'] = main_module
301
302
303 def import_main_path(main_path):
304 '''
305 Set sys.modules['__main__'] to module at main_path
306 '''
307 _fixup_main_from_path(main_path)