1# 2# Code used to start processes when using the spawn or forkserver 3# start methods. 4# 5# multiprocessing/spawn.py 6# 7# Copyright (c) 2006-2008, R Oudkerk 8# Licensed to PSF under a Contributor Agreement. 9# 10 11import os 12import sys 13import runpy 14import types 15 16from . import get_start_method, set_start_method 17from . import process 18from .context import reduction 19from . import util 20 21__all__ = ['_main', 'freeze_support', 'set_executable', 'get_executable', 22 'get_preparation_data', 'get_command_line', 'import_main_path'] 23 24# 25# _python_exe is the assumed path to the python executable. 26# People embedding Python want to modify it. 27# 28 29if sys.platform != 'win32': 30 WINEXE = False 31 WINSERVICE = False 32else: 33 WINEXE = getattr(sys, 'frozen', False) 34 WINSERVICE = sys.executable and sys.executable.lower().endswith("pythonservice.exe") 35 36def set_executable(exe): 37 global _python_exe 38 if exe is None: 39 _python_exe = exe 40 elif sys.platform == 'win32': 41 _python_exe = os.fsdecode(exe) 42 else: 43 _python_exe = os.fsencode(exe) 44 45def get_executable(): 46 return _python_exe 47 48if WINSERVICE: 49 set_executable(os.path.join(sys.exec_prefix, 'python.exe')) 50else: 51 set_executable(sys.executable) 52 53# 54# 55# 56 57def is_forking(argv): 58 ''' 59 Return whether commandline indicates we are forking 60 ''' 61 if len(argv) >= 2 and argv[1] == '--multiprocessing-fork': 62 return True 63 else: 64 return False 65 66 67def freeze_support(): 68 ''' 69 Run code for process object if this in not the main process 70 ''' 71 if is_forking(sys.argv): 72 kwds = {} 73 for arg in sys.argv[2:]: 74 name, value = arg.split('=') 75 if value == 'None': 76 kwds[name] = None 77 else: 78 kwds[name] = int(value) 79 spawn_main(**kwds) 80 sys.exit() 81 82 83def get_command_line(**kwds): 84 ''' 85 Returns prefix of command line used for spawning a child process 86 ''' 87 if getattr(sys, 'frozen', False): 88 return ([sys.executable, '--multiprocessing-fork'] + 89 ['%s=%r' % item for item in kwds.items()]) 90 else: 91 prog = 'from multiprocessing.spawn import spawn_main; spawn_main(%s)' 92 prog %= ', '.join('%s=%r' % item for item in kwds.items()) 93 opts = util._args_from_interpreter_flags() 94 exe = get_executable() 95 return [exe] + opts + ['-c', prog, '--multiprocessing-fork'] 96 97 98def spawn_main(pipe_handle, parent_pid=None, tracker_fd=None): 99 ''' 100 Run code specified by data received over pipe 101 ''' 102 assert is_forking(sys.argv), "Not forking" 103 if sys.platform == 'win32': 104 import msvcrt 105 import _winapi 106 107 if parent_pid is not None: 108 source_process = _winapi.OpenProcess( 109 _winapi.SYNCHRONIZE | _winapi.PROCESS_DUP_HANDLE, 110 False, parent_pid) 111 else: 112 source_process = None 113 new_handle = reduction.duplicate(pipe_handle, 114 source_process=source_process) 115 fd = msvcrt.open_osfhandle(new_handle, os.O_RDONLY) 116 parent_sentinel = source_process 117 else: 118 from . import resource_tracker 119 resource_tracker._resource_tracker._fd = tracker_fd 120 fd = pipe_handle 121 parent_sentinel = os.dup(pipe_handle) 122 exitcode = _main(fd, parent_sentinel) 123 sys.exit(exitcode) 124 125 126def _main(fd, parent_sentinel): 127 with os.fdopen(fd, 'rb', closefd=True) as from_parent: 128 process.current_process()._inheriting = True 129 try: 130 preparation_data = reduction.pickle.load(from_parent) 131 prepare(preparation_data) 132 self = reduction.pickle.load(from_parent) 133 finally: 134 del process.current_process()._inheriting 135 return self._bootstrap(parent_sentinel) 136 137 138def _check_not_importing_main(): 139 if getattr(process.current_process(), '_inheriting', False): 140 raise RuntimeError(''' 141 An attempt has been made to start a new process before the 142 current process has finished its bootstrapping phase. 143 144 This probably means that you are not using fork to start your 145 child processes and you have forgotten to use the proper idiom 146 in the main module: 147 148 if __name__ == '__main__': 149 freeze_support() 150 ... 151 152 The "freeze_support()" line can be omitted if the program 153 is not going to be frozen to produce an executable. 154 155 To fix this issue, refer to the "Safe importing of main module" 156 section in https://docs.python.org/3/library/multiprocessing.html 157 ''') 158 159 160def get_preparation_data(name): 161 ''' 162 Return info about parent needed by child to unpickle process object 163 ''' 164 _check_not_importing_main() 165 d = dict( 166 log_to_stderr=util._log_to_stderr, 167 authkey=process.current_process().authkey, 168 ) 169 170 if util._logger is not None: 171 d['log_level'] = util._logger.getEffectiveLevel() 172 173 sys_path=sys.path.copy() 174 try: 175 i = sys_path.index('') 176 except ValueError: 177 pass 178 else: 179 sys_path[i] = process.ORIGINAL_DIR 180 181 d.update( 182 name=name, 183 sys_path=sys_path, 184 sys_argv=sys.argv, 185 orig_dir=process.ORIGINAL_DIR, 186 dir=os.getcwd(), 187 start_method=get_start_method(), 188 ) 189 190 # Figure out whether to initialise main in the subprocess as a module 191 # or through direct execution (or to leave it alone entirely) 192 main_module = sys.modules['__main__'] 193 main_mod_name = getattr(main_module.__spec__, "name", None) 194 if main_mod_name is not None: 195 d['init_main_from_name'] = main_mod_name 196 elif sys.platform != 'win32' or (not WINEXE and not WINSERVICE): 197 main_path = getattr(main_module, '__file__', None) 198 if main_path is not None: 199 if (not os.path.isabs(main_path) and 200 process.ORIGINAL_DIR is not None): 201 main_path = os.path.join(process.ORIGINAL_DIR, main_path) 202 d['init_main_from_path'] = os.path.normpath(main_path) 203 204 return d 205 206# 207# Prepare current process 208# 209 210old_main_modules = [] 211 212def prepare(data): 213 ''' 214 Try to get current process ready to unpickle process object 215 ''' 216 if 'name' in data: 217 process.current_process().name = data['name'] 218 219 if 'authkey' in data: 220 process.current_process().authkey = data['authkey'] 221 222 if 'log_to_stderr' in data and data['log_to_stderr']: 223 util.log_to_stderr() 224 225 if 'log_level' in data: 226 util.get_logger().setLevel(data['log_level']) 227 228 if 'sys_path' in data: 229 sys.path = data['sys_path'] 230 231 if 'sys_argv' in data: 232 sys.argv = data['sys_argv'] 233 234 if 'dir' in data: 235 os.chdir(data['dir']) 236 237 if 'orig_dir' in data: 238 process.ORIGINAL_DIR = data['orig_dir'] 239 240 if 'start_method' in data: 241 set_start_method(data['start_method'], force=True) 242 243 if 'init_main_from_name' in data: 244 _fixup_main_from_name(data['init_main_from_name']) 245 elif 'init_main_from_path' in data: 246 _fixup_main_from_path(data['init_main_from_path']) 247 248# Multiprocessing module helpers to fix up the main module in 249# spawned subprocesses 250def _fixup_main_from_name(mod_name): 251 # __main__.py files for packages, directories, zip archives, etc, run 252 # their "main only" code unconditionally, so we don't even try to 253 # populate anything in __main__, nor do we make any changes to 254 # __main__ attributes 255 current_main = sys.modules['__main__'] 256 if mod_name == "__main__" or mod_name.endswith(".__main__"): 257 return 258 259 # If this process was forked, __main__ may already be populated 260 if getattr(current_main.__spec__, "name", None) == mod_name: 261 return 262 263 # Otherwise, __main__ may contain some non-main code where we need to 264 # support unpickling it properly. We rerun it as __mp_main__ and make 265 # the normal __main__ an alias to that 266 old_main_modules.append(current_main) 267 main_module = types.ModuleType("__mp_main__") 268 main_content = runpy.run_module(mod_name, 269 run_name="__mp_main__", 270 alter_sys=True) 271 main_module.__dict__.update(main_content) 272 sys.modules['__main__'] = sys.modules['__mp_main__'] = main_module 273 274 275def _fixup_main_from_path(main_path): 276 # If this process was forked, __main__ may already be populated 277 current_main = sys.modules['__main__'] 278 279 # Unfortunately, the main ipython launch script historically had no 280 # "if __name__ == '__main__'" guard, so we work around that 281 # by treating it like a __main__.py file 282 # See https://github.com/ipython/ipython/issues/4698 283 main_name = os.path.splitext(os.path.basename(main_path))[0] 284 if main_name == 'ipython': 285 return 286 287 # Otherwise, if __file__ already has the setting we expect, 288 # there's nothing more to do 289 if getattr(current_main, '__file__', None) == main_path: 290 return 291 292 # If the parent process has sent a path through rather than a module 293 # name we assume it is an executable script that may contain 294 # non-main code that needs to be executed 295 old_main_modules.append(current_main) 296 main_module = types.ModuleType("__mp_main__") 297 main_content = runpy.run_path(main_path, 298 run_name="__mp_main__") 299 main_module.__dict__.update(main_content) 300 sys.modules['__main__'] = sys.modules['__mp_main__'] = main_module 301 302 303def import_main_path(main_path): 304 ''' 305 Set sys.modules['__main__'] to module at main_path 306 ''' 307 _fixup_main_from_path(main_path) 308