1# subprocess - Subprocesses with accessible I/O streams 2# 3# For more information about this module, see PEP 324. 4# 5# Copyright (c) 2003-2005 by Peter Astrand <astrand@lysator.liu.se> 6# 7# Licensed to PSF under a Contributor Agreement. 8# See http://www.python.org/2.4/license for licensing details. 9 10r"""Subprocesses with accessible I/O streams 11 12This module allows you to spawn processes, connect to their 13input/output/error pipes, and obtain their return codes. 14 15For a complete description of this module see the Python documentation. 16 17Main API 18======== 19run(...): Runs a command, waits for it to complete, then returns a 20 CompletedProcess instance. 21Popen(...): A class for flexibly executing a command in a new process 22 23Constants 24--------- 25DEVNULL: Special value that indicates that os.devnull should be used 26PIPE: Special value that indicates a pipe should be created 27STDOUT: Special value that indicates that stderr should go to stdout 28 29 30Older API 31========= 32call(...): Runs a command, waits for it to complete, then returns 33 the return code. 34check_call(...): Same as call() but raises CalledProcessError() 35 if return code is not 0 36check_output(...): Same as check_call() but returns the contents of 37 stdout instead of a return code 38getoutput(...): Runs a command in the shell, waits for it to complete, 39 then returns the output 40getstatusoutput(...): Runs a command in the shell, waits for it to complete, 41 then returns a (exitcode, output) tuple 42""" 43 44import builtins 45import errno 46import io 47import os 48import time 49import signal 50import sys 51import threading 52import warnings 53import contextlib 54from time import monotonic as _time 55import types 56 57try: 58 import pwd 59except ImportError: 60 pwd = None 61try: 62 import grp 63except ImportError: 64 grp = None 65 66__all__ = ["Popen", "PIPE", "STDOUT", "call", "check_call", "getstatusoutput", 67 "getoutput", "check_output", "run", "CalledProcessError", "DEVNULL", 68 "SubprocessError", "TimeoutExpired", "CompletedProcess"] 69 # NOTE: We intentionally exclude list2cmdline as it is 70 # considered an internal implementation detail. issue10838. 71 72try: 73 import msvcrt 74 import _winapi 75 _mswindows = True 76except ModuleNotFoundError: 77 _mswindows = False 78 import _posixsubprocess 79 import select 80 import selectors 81else: 82 from _winapi import (CREATE_NEW_CONSOLE, CREATE_NEW_PROCESS_GROUP, 83 STD_INPUT_HANDLE, STD_OUTPUT_HANDLE, 84 STD_ERROR_HANDLE, SW_HIDE, 85 STARTF_USESTDHANDLES, STARTF_USESHOWWINDOW, 86 ABOVE_NORMAL_PRIORITY_CLASS, BELOW_NORMAL_PRIORITY_CLASS, 87 HIGH_PRIORITY_CLASS, IDLE_PRIORITY_CLASS, 88 NORMAL_PRIORITY_CLASS, REALTIME_PRIORITY_CLASS, 89 CREATE_NO_WINDOW, DETACHED_PROCESS, 90 CREATE_DEFAULT_ERROR_MODE, CREATE_BREAKAWAY_FROM_JOB) 91 92 __all__.extend(["CREATE_NEW_CONSOLE", "CREATE_NEW_PROCESS_GROUP", 93 "STD_INPUT_HANDLE", "STD_OUTPUT_HANDLE", 94 "STD_ERROR_HANDLE", "SW_HIDE", 95 "STARTF_USESTDHANDLES", "STARTF_USESHOWWINDOW", 96 "STARTUPINFO", 97 "ABOVE_NORMAL_PRIORITY_CLASS", "BELOW_NORMAL_PRIORITY_CLASS", 98 "HIGH_PRIORITY_CLASS", "IDLE_PRIORITY_CLASS", 99 "NORMAL_PRIORITY_CLASS", "REALTIME_PRIORITY_CLASS", 100 "CREATE_NO_WINDOW", "DETACHED_PROCESS", 101 "CREATE_DEFAULT_ERROR_MODE", "CREATE_BREAKAWAY_FROM_JOB"]) 102 103 104# Exception classes used by this module. 105class SubprocessError(Exception): pass 106 107 108class CalledProcessError(SubprocessError): 109 """Raised when run() is called with check=True and the process 110 returns a non-zero exit status. 111 112 Attributes: 113 cmd, returncode, stdout, stderr, output 114 """ 115 def __init__(self, returncode, cmd, output=None, stderr=None): 116 self.returncode = returncode 117 self.cmd = cmd 118 self.output = output 119 self.stderr = stderr 120 121 def __str__(self): 122 if self.returncode and self.returncode < 0: 123 try: 124 return "Command '%s' died with %r." % ( 125 self.cmd, signal.Signals(-self.returncode)) 126 except ValueError: 127 return "Command '%s' died with unknown signal %d." % ( 128 self.cmd, -self.returncode) 129 else: 130 return "Command '%s' returned non-zero exit status %d." % ( 131 self.cmd, self.returncode) 132 133 @property 134 def stdout(self): 135 """Alias for output attribute, to match stderr""" 136 return self.output 137 138 @stdout.setter 139 def stdout(self, value): 140 # There's no obvious reason to set this, but allow it anyway so 141 # .stdout is a transparent alias for .output 142 self.output = value 143 144 145class TimeoutExpired(SubprocessError): 146 """This exception is raised when the timeout expires while waiting for a 147 child process. 148 149 Attributes: 150 cmd, output, stdout, stderr, timeout 151 """ 152 def __init__(self, cmd, timeout, output=None, stderr=None): 153 self.cmd = cmd 154 self.timeout = timeout 155 self.output = output 156 self.stderr = stderr 157 158 def __str__(self): 159 return ("Command '%s' timed out after %s seconds" % 160 (self.cmd, self.timeout)) 161 162 @property 163 def stdout(self): 164 return self.output 165 166 @stdout.setter 167 def stdout(self, value): 168 # There's no obvious reason to set this, but allow it anyway so 169 # .stdout is a transparent alias for .output 170 self.output = value 171 172 173if _mswindows: 174 class STARTUPINFO: 175 def __init__(self, *, dwFlags=0, hStdInput=None, hStdOutput=None, 176 hStdError=None, wShowWindow=0, lpAttributeList=None): 177 self.dwFlags = dwFlags 178 self.hStdInput = hStdInput 179 self.hStdOutput = hStdOutput 180 self.hStdError = hStdError 181 self.wShowWindow = wShowWindow 182 self.lpAttributeList = lpAttributeList or {"handle_list": []} 183 184 def copy(self): 185 attr_list = self.lpAttributeList.copy() 186 if 'handle_list' in attr_list: 187 attr_list['handle_list'] = list(attr_list['handle_list']) 188 189 return STARTUPINFO(dwFlags=self.dwFlags, 190 hStdInput=self.hStdInput, 191 hStdOutput=self.hStdOutput, 192 hStdError=self.hStdError, 193 wShowWindow=self.wShowWindow, 194 lpAttributeList=attr_list) 195 196 197 class Handle(int): 198 closed = False 199 200 def Close(self, CloseHandle=_winapi.CloseHandle): 201 if not self.closed: 202 self.closed = True 203 CloseHandle(self) 204 205 def Detach(self): 206 if not self.closed: 207 self.closed = True 208 return int(self) 209 raise ValueError("already closed") 210 211 def __repr__(self): 212 return "%s(%d)" % (self.__class__.__name__, int(self)) 213 214 __del__ = Close 215else: 216 # When select or poll has indicated that the file is writable, 217 # we can write up to _PIPE_BUF bytes without risk of blocking. 218 # POSIX defines PIPE_BUF as >= 512. 219 _PIPE_BUF = getattr(select, 'PIPE_BUF', 512) 220 221 # poll/select have the advantage of not requiring any extra file 222 # descriptor, contrarily to epoll/kqueue (also, they require a single 223 # syscall). 224 if hasattr(selectors, 'PollSelector'): 225 _PopenSelector = selectors.PollSelector 226 else: 227 _PopenSelector = selectors.SelectSelector 228 229 230if _mswindows: 231 # On Windows we just need to close `Popen._handle` when we no longer need 232 # it, so that the kernel can free it. `Popen._handle` gets closed 233 # implicitly when the `Popen` instance is finalized (see `Handle.__del__`, 234 # which is calling `CloseHandle` as requested in [1]), so there is nothing 235 # for `_cleanup` to do. 236 # 237 # [1] https://docs.microsoft.com/en-us/windows/desktop/ProcThread/ 238 # creating-processes 239 _active = None 240 241 def _cleanup(): 242 pass 243else: 244 # This lists holds Popen instances for which the underlying process had not 245 # exited at the time its __del__ method got called: those processes are 246 # wait()ed for synchronously from _cleanup() when a new Popen object is 247 # created, to avoid zombie processes. 248 _active = [] 249 250 def _cleanup(): 251 if _active is None: 252 return 253 for inst in _active[:]: 254 res = inst._internal_poll(_deadstate=sys.maxsize) 255 if res is not None: 256 try: 257 _active.remove(inst) 258 except ValueError: 259 # This can happen if two threads create a new Popen instance. 260 # It's harmless that it was already removed, so ignore. 261 pass 262 263PIPE = -1 264STDOUT = -2 265DEVNULL = -3 266 267 268# XXX This function is only used by multiprocessing and the test suite, 269# but it's here so that it can be imported when Python is compiled without 270# threads. 271 272def _optim_args_from_interpreter_flags(): 273 """Return a list of command-line arguments reproducing the current 274 optimization settings in sys.flags.""" 275 args = [] 276 value = sys.flags.optimize 277 if value > 0: 278 args.append('-' + 'O' * value) 279 return args 280 281 282def _args_from_interpreter_flags(): 283 """Return a list of command-line arguments reproducing the current 284 settings in sys.flags, sys.warnoptions and sys._xoptions.""" 285 flag_opt_map = { 286 'debug': 'd', 287 # 'inspect': 'i', 288 # 'interactive': 'i', 289 'dont_write_bytecode': 'B', 290 'no_site': 'S', 291 'verbose': 'v', 292 'bytes_warning': 'b', 293 'quiet': 'q', 294 # -O is handled in _optim_args_from_interpreter_flags() 295 } 296 args = _optim_args_from_interpreter_flags() 297 for flag, opt in flag_opt_map.items(): 298 v = getattr(sys.flags, flag) 299 if v > 0: 300 args.append('-' + opt * v) 301 302 if sys.flags.isolated: 303 args.append('-I') 304 else: 305 if sys.flags.ignore_environment: 306 args.append('-E') 307 if sys.flags.no_user_site: 308 args.append('-s') 309 310 # -W options 311 warnopts = sys.warnoptions[:] 312 bytes_warning = sys.flags.bytes_warning 313 xoptions = getattr(sys, '_xoptions', {}) 314 dev_mode = ('dev' in xoptions) 315 316 if bytes_warning > 1: 317 warnopts.remove("error::BytesWarning") 318 elif bytes_warning: 319 warnopts.remove("default::BytesWarning") 320 if dev_mode: 321 warnopts.remove('default') 322 for opt in warnopts: 323 args.append('-W' + opt) 324 325 # -X options 326 if dev_mode: 327 args.extend(('-X', 'dev')) 328 for opt in ('faulthandler', 'tracemalloc', 'importtime', 329 'showrefcount', 'utf8', 'oldparser'): 330 if opt in xoptions: 331 value = xoptions[opt] 332 if value is True: 333 arg = opt 334 else: 335 arg = '%s=%s' % (opt, value) 336 args.extend(('-X', arg)) 337 338 return args 339 340 341def call(*popenargs, timeout=None, **kwargs): 342 """Run command with arguments. Wait for command to complete or 343 timeout, then return the returncode attribute. 344 345 The arguments are the same as for the Popen constructor. Example: 346 347 retcode = call(["ls", "-l"]) 348 """ 349 with Popen(*popenargs, **kwargs) as p: 350 try: 351 return p.wait(timeout=timeout) 352 except: # Including KeyboardInterrupt, wait handled that. 353 p.kill() 354 # We don't call p.wait() again as p.__exit__ does that for us. 355 raise 356 357 358def check_call(*popenargs, **kwargs): 359 """Run command with arguments. Wait for command to complete. If 360 the exit code was zero then return, otherwise raise 361 CalledProcessError. The CalledProcessError object will have the 362 return code in the returncode attribute. 363 364 The arguments are the same as for the call function. Example: 365 366 check_call(["ls", "-l"]) 367 """ 368 retcode = call(*popenargs, **kwargs) 369 if retcode: 370 cmd = kwargs.get("args") 371 if cmd is None: 372 cmd = popenargs[0] 373 raise CalledProcessError(retcode, cmd) 374 return 0 375 376 377def check_output(*popenargs, timeout=None, **kwargs): 378 r"""Run command with arguments and return its output. 379 380 If the exit code was non-zero it raises a CalledProcessError. The 381 CalledProcessError object will have the return code in the returncode 382 attribute and output in the output attribute. 383 384 The arguments are the same as for the Popen constructor. Example: 385 386 >>> check_output(["ls", "-l", "/dev/null"]) 387 b'crw-rw-rw- 1 root root 1, 3 Oct 18 2007 /dev/null\n' 388 389 The stdout argument is not allowed as it is used internally. 390 To capture standard error in the result, use stderr=STDOUT. 391 392 >>> check_output(["/bin/sh", "-c", 393 ... "ls -l non_existent_file ; exit 0"], 394 ... stderr=STDOUT) 395 b'ls: non_existent_file: No such file or directory\n' 396 397 There is an additional optional argument, "input", allowing you to 398 pass a string to the subprocess's stdin. If you use this argument 399 you may not also use the Popen constructor's "stdin" argument, as 400 it too will be used internally. Example: 401 402 >>> check_output(["sed", "-e", "s/foo/bar/"], 403 ... input=b"when in the course of fooman events\n") 404 b'when in the course of barman events\n' 405 406 By default, all communication is in bytes, and therefore any "input" 407 should be bytes, and the return value will be bytes. If in text mode, 408 any "input" should be a string, and the return value will be a string 409 decoded according to locale encoding, or by "encoding" if set. Text mode 410 is triggered by setting any of text, encoding, errors or universal_newlines. 411 """ 412 if 'stdout' in kwargs: 413 raise ValueError('stdout argument not allowed, it will be overridden.') 414 415 if 'input' in kwargs and kwargs['input'] is None: 416 # Explicitly passing input=None was previously equivalent to passing an 417 # empty string. That is maintained here for backwards compatibility. 418 if kwargs.get('universal_newlines') or kwargs.get('text'): 419 empty = '' 420 else: 421 empty = b'' 422 kwargs['input'] = empty 423 424 return run(*popenargs, stdout=PIPE, timeout=timeout, check=True, 425 **kwargs).stdout 426 427 428class CompletedProcess(object): 429 """A process that has finished running. 430 431 This is returned by run(). 432 433 Attributes: 434 args: The list or str args passed to run(). 435 returncode: The exit code of the process, negative for signals. 436 stdout: The standard output (None if not captured). 437 stderr: The standard error (None if not captured). 438 """ 439 def __init__(self, args, returncode, stdout=None, stderr=None): 440 self.args = args 441 self.returncode = returncode 442 self.stdout = stdout 443 self.stderr = stderr 444 445 def __repr__(self): 446 args = ['args={!r}'.format(self.args), 447 'returncode={!r}'.format(self.returncode)] 448 if self.stdout is not None: 449 args.append('stdout={!r}'.format(self.stdout)) 450 if self.stderr is not None: 451 args.append('stderr={!r}'.format(self.stderr)) 452 return "{}({})".format(type(self).__name__, ', '.join(args)) 453 454 __class_getitem__ = classmethod(types.GenericAlias) 455 456 457 def check_returncode(self): 458 """Raise CalledProcessError if the exit code is non-zero.""" 459 if self.returncode: 460 raise CalledProcessError(self.returncode, self.args, self.stdout, 461 self.stderr) 462 463 464def run(*popenargs, 465 input=None, capture_output=False, timeout=None, check=False, **kwargs): 466 """Run command with arguments and return a CompletedProcess instance. 467 468 The returned instance will have attributes args, returncode, stdout and 469 stderr. By default, stdout and stderr are not captured, and those attributes 470 will be None. Pass stdout=PIPE and/or stderr=PIPE in order to capture them. 471 472 If check is True and the exit code was non-zero, it raises a 473 CalledProcessError. The CalledProcessError object will have the return code 474 in the returncode attribute, and output & stderr attributes if those streams 475 were captured. 476 477 If timeout is given, and the process takes too long, a TimeoutExpired 478 exception will be raised. 479 480 There is an optional argument "input", allowing you to 481 pass bytes or a string to the subprocess's stdin. If you use this argument 482 you may not also use the Popen constructor's "stdin" argument, as 483 it will be used internally. 484 485 By default, all communication is in bytes, and therefore any "input" should 486 be bytes, and the stdout and stderr will be bytes. If in text mode, any 487 "input" should be a string, and stdout and stderr will be strings decoded 488 according to locale encoding, or by "encoding" if set. Text mode is 489 triggered by setting any of text, encoding, errors or universal_newlines. 490 491 The other arguments are the same as for the Popen constructor. 492 """ 493 if input is not None: 494 if kwargs.get('stdin') is not None: 495 raise ValueError('stdin and input arguments may not both be used.') 496 kwargs['stdin'] = PIPE 497 498 if capture_output: 499 if kwargs.get('stdout') is not None or kwargs.get('stderr') is not None: 500 raise ValueError('stdout and stderr arguments may not be used ' 501 'with capture_output.') 502 kwargs['stdout'] = PIPE 503 kwargs['stderr'] = PIPE 504 505 with Popen(*popenargs, **kwargs) as process: 506 try: 507 stdout, stderr = process.communicate(input, timeout=timeout) 508 except TimeoutExpired as exc: 509 process.kill() 510 if _mswindows: 511 # Windows accumulates the output in a single blocking 512 # read() call run on child threads, with the timeout 513 # being done in a join() on those threads. communicate() 514 # _after_ kill() is required to collect that and add it 515 # to the exception. 516 exc.stdout, exc.stderr = process.communicate() 517 else: 518 # POSIX _communicate already populated the output so 519 # far into the TimeoutExpired exception. 520 process.wait() 521 raise 522 except: # Including KeyboardInterrupt, communicate handled that. 523 process.kill() 524 # We don't call process.wait() as .__exit__ does that for us. 525 raise 526 retcode = process.poll() 527 if check and retcode: 528 raise CalledProcessError(retcode, process.args, 529 output=stdout, stderr=stderr) 530 return CompletedProcess(process.args, retcode, stdout, stderr) 531 532 533def list2cmdline(seq): 534 """ 535 Translate a sequence of arguments into a command line 536 string, using the same rules as the MS C runtime: 537 538 1) Arguments are delimited by white space, which is either a 539 space or a tab. 540 541 2) A string surrounded by double quotation marks is 542 interpreted as a single argument, regardless of white space 543 contained within. A quoted string can be embedded in an 544 argument. 545 546 3) A double quotation mark preceded by a backslash is 547 interpreted as a literal double quotation mark. 548 549 4) Backslashes are interpreted literally, unless they 550 immediately precede a double quotation mark. 551 552 5) If backslashes immediately precede a double quotation mark, 553 every pair of backslashes is interpreted as a literal 554 backslash. If the number of backslashes is odd, the last 555 backslash escapes the next double quotation mark as 556 described in rule 3. 557 """ 558 559 # See 560 # http://msdn.microsoft.com/en-us/library/17w5ykft.aspx 561 # or search http://msdn.microsoft.com for 562 # "Parsing C++ Command-Line Arguments" 563 result = [] 564 needquote = False 565 for arg in map(os.fsdecode, seq): 566 bs_buf = [] 567 568 # Add a space to separate this argument from the others 569 if result: 570 result.append(' ') 571 572 needquote = (" " in arg) or ("\t" in arg) or not arg 573 if needquote: 574 result.append('"') 575 576 for c in arg: 577 if c == '\\': 578 # Don't know if we need to double yet. 579 bs_buf.append(c) 580 elif c == '"': 581 # Double backslashes. 582 result.append('\\' * len(bs_buf)*2) 583 bs_buf = [] 584 result.append('\\"') 585 else: 586 # Normal char 587 if bs_buf: 588 result.extend(bs_buf) 589 bs_buf = [] 590 result.append(c) 591 592 # Add remaining backslashes, if any. 593 if bs_buf: 594 result.extend(bs_buf) 595 596 if needquote: 597 result.extend(bs_buf) 598 result.append('"') 599 600 return ''.join(result) 601 602 603# Various tools for executing commands and looking at their output and status. 604# 605 606def getstatusoutput(cmd): 607 """Return (exitcode, output) of executing cmd in a shell. 608 609 Execute the string 'cmd' in a shell with 'check_output' and 610 return a 2-tuple (status, output). The locale encoding is used 611 to decode the output and process newlines. 612 613 A trailing newline is stripped from the output. 614 The exit status for the command can be interpreted 615 according to the rules for the function 'wait'. Example: 616 617 >>> import subprocess 618 >>> subprocess.getstatusoutput('ls /bin/ls') 619 (0, '/bin/ls') 620 >>> subprocess.getstatusoutput('cat /bin/junk') 621 (1, 'cat: /bin/junk: No such file or directory') 622 >>> subprocess.getstatusoutput('/bin/junk') 623 (127, 'sh: /bin/junk: not found') 624 >>> subprocess.getstatusoutput('/bin/kill $$') 625 (-15, '') 626 """ 627 try: 628 data = check_output(cmd, shell=True, text=True, stderr=STDOUT) 629 exitcode = 0 630 except CalledProcessError as ex: 631 data = ex.output 632 exitcode = ex.returncode 633 if data[-1:] == '\n': 634 data = data[:-1] 635 return exitcode, data 636 637def getoutput(cmd): 638 """Return output (stdout or stderr) of executing cmd in a shell. 639 640 Like getstatusoutput(), except the exit status is ignored and the return 641 value is a string containing the command's output. Example: 642 643 >>> import subprocess 644 >>> subprocess.getoutput('ls /bin/ls') 645 '/bin/ls' 646 """ 647 return getstatusoutput(cmd)[1] 648 649 650def _use_posix_spawn(): 651 """Check if posix_spawn() can be used for subprocess. 652 653 subprocess requires a posix_spawn() implementation that properly reports 654 errors to the parent process, & sets errno on the following failures: 655 656 * Process attribute actions failed. 657 * File actions failed. 658 * exec() failed. 659 660 Prefer an implementation which can use vfork() in some cases for best 661 performance. 662 """ 663 if _mswindows or not hasattr(os, 'posix_spawn'): 664 # os.posix_spawn() is not available 665 return False 666 667 if sys.platform == 'darwin': 668 # posix_spawn() is a syscall on macOS and properly reports errors 669 return True 670 671 # Check libc name and runtime libc version 672 try: 673 ver = os.confstr('CS_GNU_LIBC_VERSION') 674 # parse 'glibc 2.28' as ('glibc', (2, 28)) 675 parts = ver.split(maxsplit=1) 676 if len(parts) != 2: 677 # reject unknown format 678 raise ValueError 679 libc = parts[0] 680 version = tuple(map(int, parts[1].split('.'))) 681 682 if sys.platform == 'linux' and libc == 'glibc' and version >= (2, 24): 683 # glibc 2.24 has a new Linux posix_spawn implementation using vfork 684 # which properly reports errors to the parent process. 685 return True 686 # Note: Don't use the implementation in earlier glibc because it doesn't 687 # use vfork (even if glibc 2.26 added a pipe to properly report errors 688 # to the parent process). 689 except (AttributeError, ValueError, OSError): 690 # os.confstr() or CS_GNU_LIBC_VERSION value not available 691 pass 692 693 # By default, assume that posix_spawn() does not properly report errors. 694 return False 695 696 697_USE_POSIX_SPAWN = _use_posix_spawn() 698 699 700class Popen(object): 701 """ Execute a child program in a new process. 702 703 For a complete description of the arguments see the Python documentation. 704 705 Arguments: 706 args: A string, or a sequence of program arguments. 707 708 bufsize: supplied as the buffering argument to the open() function when 709 creating the stdin/stdout/stderr pipe file objects 710 711 executable: A replacement program to execute. 712 713 stdin, stdout and stderr: These specify the executed programs' standard 714 input, standard output and standard error file handles, respectively. 715 716 preexec_fn: (POSIX only) An object to be called in the child process 717 just before the child is executed. 718 719 close_fds: Controls closing or inheriting of file descriptors. 720 721 shell: If true, the command will be executed through the shell. 722 723 cwd: Sets the current directory before the child is executed. 724 725 env: Defines the environment variables for the new process. 726 727 text: If true, decode stdin, stdout and stderr using the given encoding 728 (if set) or the system default otherwise. 729 730 universal_newlines: Alias of text, provided for backwards compatibility. 731 732 startupinfo and creationflags (Windows only) 733 734 restore_signals (POSIX only) 735 736 start_new_session (POSIX only) 737 738 group (POSIX only) 739 740 extra_groups (POSIX only) 741 742 user (POSIX only) 743 744 umask (POSIX only) 745 746 pass_fds (POSIX only) 747 748 encoding and errors: Text mode encoding and error handling to use for 749 file objects stdin, stdout and stderr. 750 751 Attributes: 752 stdin, stdout, stderr, pid, returncode 753 """ 754 _child_created = False # Set here since __del__ checks it 755 756 def __init__(self, args, bufsize=-1, executable=None, 757 stdin=None, stdout=None, stderr=None, 758 preexec_fn=None, close_fds=True, 759 shell=False, cwd=None, env=None, universal_newlines=None, 760 startupinfo=None, creationflags=0, 761 restore_signals=True, start_new_session=False, 762 pass_fds=(), *, user=None, group=None, extra_groups=None, 763 encoding=None, errors=None, text=None, umask=-1): 764 """Create new Popen instance.""" 765 _cleanup() 766 # Held while anything is calling waitpid before returncode has been 767 # updated to prevent clobbering returncode if wait() or poll() are 768 # called from multiple threads at once. After acquiring the lock, 769 # code must re-check self.returncode to see if another thread just 770 # finished a waitpid() call. 771 self._waitpid_lock = threading.Lock() 772 773 self._input = None 774 self._communication_started = False 775 if bufsize is None: 776 bufsize = -1 # Restore default 777 if not isinstance(bufsize, int): 778 raise TypeError("bufsize must be an integer") 779 780 if _mswindows: 781 if preexec_fn is not None: 782 raise ValueError("preexec_fn is not supported on Windows " 783 "platforms") 784 else: 785 # POSIX 786 if pass_fds and not close_fds: 787 warnings.warn("pass_fds overriding close_fds.", RuntimeWarning) 788 close_fds = True 789 if startupinfo is not None: 790 raise ValueError("startupinfo is only supported on Windows " 791 "platforms") 792 if creationflags != 0: 793 raise ValueError("creationflags is only supported on Windows " 794 "platforms") 795 796 self.args = args 797 self.stdin = None 798 self.stdout = None 799 self.stderr = None 800 self.pid = None 801 self.returncode = None 802 self.encoding = encoding 803 self.errors = errors 804 805 # Validate the combinations of text and universal_newlines 806 if (text is not None and universal_newlines is not None 807 and bool(universal_newlines) != bool(text)): 808 raise SubprocessError('Cannot disambiguate when both text ' 809 'and universal_newlines are supplied but ' 810 'different. Pass one or the other.') 811 812 # Input and output objects. The general principle is like 813 # this: 814 # 815 # Parent Child 816 # ------ ----- 817 # p2cwrite ---stdin---> p2cread 818 # c2pread <--stdout--- c2pwrite 819 # errread <--stderr--- errwrite 820 # 821 # On POSIX, the child objects are file descriptors. On 822 # Windows, these are Windows file handles. The parent objects 823 # are file descriptors on both platforms. The parent objects 824 # are -1 when not using PIPEs. The child objects are -1 825 # when not redirecting. 826 827 (p2cread, p2cwrite, 828 c2pread, c2pwrite, 829 errread, errwrite) = self._get_handles(stdin, stdout, stderr) 830 831 # We wrap OS handles *before* launching the child, otherwise a 832 # quickly terminating child could make our fds unwrappable 833 # (see #8458). 834 835 if _mswindows: 836 if p2cwrite != -1: 837 p2cwrite = msvcrt.open_osfhandle(p2cwrite.Detach(), 0) 838 if c2pread != -1: 839 c2pread = msvcrt.open_osfhandle(c2pread.Detach(), 0) 840 if errread != -1: 841 errread = msvcrt.open_osfhandle(errread.Detach(), 0) 842 843 self.text_mode = encoding or errors or text or universal_newlines 844 845 # How long to resume waiting on a child after the first ^C. 846 # There is no right value for this. The purpose is to be polite 847 # yet remain good for interactive users trying to exit a tool. 848 self._sigint_wait_secs = 0.25 # 1/xkcd221.getRandomNumber() 849 850 self._closed_child_pipe_fds = False 851 852 if self.text_mode: 853 if bufsize == 1: 854 line_buffering = True 855 # Use the default buffer size for the underlying binary streams 856 # since they don't support line buffering. 857 bufsize = -1 858 else: 859 line_buffering = False 860 861 gid = None 862 if group is not None: 863 if not hasattr(os, 'setregid'): 864 raise ValueError("The 'group' parameter is not supported on the " 865 "current platform") 866 867 elif isinstance(group, str): 868 if grp is None: 869 raise ValueError("The group parameter cannot be a string " 870 "on systems without the grp module") 871 872 gid = grp.getgrnam(group).gr_gid 873 elif isinstance(group, int): 874 gid = group 875 else: 876 raise TypeError("Group must be a string or an integer, not {}" 877 .format(type(group))) 878 879 if gid < 0: 880 raise ValueError(f"Group ID cannot be negative, got {gid}") 881 882 gids = None 883 if extra_groups is not None: 884 if not hasattr(os, 'setgroups'): 885 raise ValueError("The 'extra_groups' parameter is not " 886 "supported on the current platform") 887 888 elif isinstance(extra_groups, str): 889 raise ValueError("Groups must be a list, not a string") 890 891 gids = [] 892 for extra_group in extra_groups: 893 if isinstance(extra_group, str): 894 if grp is None: 895 raise ValueError("Items in extra_groups cannot be " 896 "strings on systems without the " 897 "grp module") 898 899 gids.append(grp.getgrnam(extra_group).gr_gid) 900 elif isinstance(extra_group, int): 901 gids.append(extra_group) 902 else: 903 raise TypeError("Items in extra_groups must be a string " 904 "or integer, not {}" 905 .format(type(extra_group))) 906 907 # make sure that the gids are all positive here so we can do less 908 # checking in the C code 909 for gid_check in gids: 910 if gid_check < 0: 911 raise ValueError(f"Group ID cannot be negative, got {gid_check}") 912 913 uid = None 914 if user is not None: 915 if not hasattr(os, 'setreuid'): 916 raise ValueError("The 'user' parameter is not supported on " 917 "the current platform") 918 919 elif isinstance(user, str): 920 if pwd is None: 921 raise ValueError("The user parameter cannot be a string " 922 "on systems without the pwd module") 923 924 uid = pwd.getpwnam(user).pw_uid 925 elif isinstance(user, int): 926 uid = user 927 else: 928 raise TypeError("User must be a string or an integer") 929 930 if uid < 0: 931 raise ValueError(f"User ID cannot be negative, got {uid}") 932 933 try: 934 if p2cwrite != -1: 935 self.stdin = io.open(p2cwrite, 'wb', bufsize) 936 if self.text_mode: 937 self.stdin = io.TextIOWrapper(self.stdin, write_through=True, 938 line_buffering=line_buffering, 939 encoding=encoding, errors=errors) 940 if c2pread != -1: 941 self.stdout = io.open(c2pread, 'rb', bufsize) 942 if self.text_mode: 943 self.stdout = io.TextIOWrapper(self.stdout, 944 encoding=encoding, errors=errors) 945 if errread != -1: 946 self.stderr = io.open(errread, 'rb', bufsize) 947 if self.text_mode: 948 self.stderr = io.TextIOWrapper(self.stderr, 949 encoding=encoding, errors=errors) 950 951 self._execute_child(args, executable, preexec_fn, close_fds, 952 pass_fds, cwd, env, 953 startupinfo, creationflags, shell, 954 p2cread, p2cwrite, 955 c2pread, c2pwrite, 956 errread, errwrite, 957 restore_signals, 958 gid, gids, uid, umask, 959 start_new_session) 960 except: 961 # Cleanup if the child failed starting. 962 for f in filter(None, (self.stdin, self.stdout, self.stderr)): 963 try: 964 f.close() 965 except OSError: 966 pass # Ignore EBADF or other errors. 967 968 if not self._closed_child_pipe_fds: 969 to_close = [] 970 if stdin == PIPE: 971 to_close.append(p2cread) 972 if stdout == PIPE: 973 to_close.append(c2pwrite) 974 if stderr == PIPE: 975 to_close.append(errwrite) 976 if hasattr(self, '_devnull'): 977 to_close.append(self._devnull) 978 for fd in to_close: 979 try: 980 if _mswindows and isinstance(fd, Handle): 981 fd.Close() 982 else: 983 os.close(fd) 984 except OSError: 985 pass 986 987 raise 988 989 def __repr__(self): 990 obj_repr = ( 991 f"<{self.__class__.__name__}: " 992 f"returncode: {self.returncode} args: {list(self.args)!r}>" 993 ) 994 if len(obj_repr) > 80: 995 obj_repr = obj_repr[:76] + "...>" 996 return obj_repr 997 998 __class_getitem__ = classmethod(types.GenericAlias) 999 1000 @property 1001 def universal_newlines(self): 1002 # universal_newlines as retained as an alias of text_mode for API 1003 # compatibility. bpo-31756 1004 return self.text_mode 1005 1006 @universal_newlines.setter 1007 def universal_newlines(self, universal_newlines): 1008 self.text_mode = bool(universal_newlines) 1009 1010 def _translate_newlines(self, data, encoding, errors): 1011 data = data.decode(encoding, errors) 1012 return data.replace("\r\n", "\n").replace("\r", "\n") 1013 1014 def __enter__(self): 1015 return self 1016 1017 def __exit__(self, exc_type, value, traceback): 1018 if self.stdout: 1019 self.stdout.close() 1020 if self.stderr: 1021 self.stderr.close() 1022 try: # Flushing a BufferedWriter may raise an error 1023 if self.stdin: 1024 self.stdin.close() 1025 finally: 1026 if exc_type == KeyboardInterrupt: 1027 # https://bugs.python.org/issue25942 1028 # In the case of a KeyboardInterrupt we assume the SIGINT 1029 # was also already sent to our child processes. We can't 1030 # block indefinitely as that is not user friendly. 1031 # If we have not already waited a brief amount of time in 1032 # an interrupted .wait() or .communicate() call, do so here 1033 # for consistency. 1034 if self._sigint_wait_secs > 0: 1035 try: 1036 self._wait(timeout=self._sigint_wait_secs) 1037 except TimeoutExpired: 1038 pass 1039 self._sigint_wait_secs = 0 # Note that this has been done. 1040 return # resume the KeyboardInterrupt 1041 1042 # Wait for the process to terminate, to avoid zombies. 1043 self.wait() 1044 1045 def __del__(self, _maxsize=sys.maxsize, _warn=warnings.warn): 1046 if not self._child_created: 1047 # We didn't get to successfully create a child process. 1048 return 1049 if self.returncode is None: 1050 # Not reading subprocess exit status creates a zombie process which 1051 # is only destroyed at the parent python process exit 1052 _warn("subprocess %s is still running" % self.pid, 1053 ResourceWarning, source=self) 1054 # In case the child hasn't been waited on, check if it's done. 1055 self._internal_poll(_deadstate=_maxsize) 1056 if self.returncode is None and _active is not None: 1057 # Child is still running, keep us alive until we can wait on it. 1058 _active.append(self) 1059 1060 def _get_devnull(self): 1061 if not hasattr(self, '_devnull'): 1062 self._devnull = os.open(os.devnull, os.O_RDWR) 1063 return self._devnull 1064 1065 def _stdin_write(self, input): 1066 if input: 1067 try: 1068 self.stdin.write(input) 1069 except BrokenPipeError: 1070 pass # communicate() must ignore broken pipe errors. 1071 except OSError as exc: 1072 if exc.errno == errno.EINVAL: 1073 # bpo-19612, bpo-30418: On Windows, stdin.write() fails 1074 # with EINVAL if the child process exited or if the child 1075 # process is still running but closed the pipe. 1076 pass 1077 else: 1078 raise 1079 1080 try: 1081 self.stdin.close() 1082 except BrokenPipeError: 1083 pass # communicate() must ignore broken pipe errors. 1084 except OSError as exc: 1085 if exc.errno == errno.EINVAL: 1086 pass 1087 else: 1088 raise 1089 1090 def communicate(self, input=None, timeout=None): 1091 """Interact with process: Send data to stdin and close it. 1092 Read data from stdout and stderr, until end-of-file is 1093 reached. Wait for process to terminate. 1094 1095 The optional "input" argument should be data to be sent to the 1096 child process, or None, if no data should be sent to the child. 1097 communicate() returns a tuple (stdout, stderr). 1098 1099 By default, all communication is in bytes, and therefore any 1100 "input" should be bytes, and the (stdout, stderr) will be bytes. 1101 If in text mode (indicated by self.text_mode), any "input" should 1102 be a string, and (stdout, stderr) will be strings decoded 1103 according to locale encoding, or by "encoding" if set. Text mode 1104 is triggered by setting any of text, encoding, errors or 1105 universal_newlines. 1106 """ 1107 1108 if self._communication_started and input: 1109 raise ValueError("Cannot send input after starting communication") 1110 1111 # Optimization: If we are not worried about timeouts, we haven't 1112 # started communicating, and we have one or zero pipes, using select() 1113 # or threads is unnecessary. 1114 if (timeout is None and not self._communication_started and 1115 [self.stdin, self.stdout, self.stderr].count(None) >= 2): 1116 stdout = None 1117 stderr = None 1118 if self.stdin: 1119 self._stdin_write(input) 1120 elif self.stdout: 1121 stdout = self.stdout.read() 1122 self.stdout.close() 1123 elif self.stderr: 1124 stderr = self.stderr.read() 1125 self.stderr.close() 1126 self.wait() 1127 else: 1128 if timeout is not None: 1129 endtime = _time() + timeout 1130 else: 1131 endtime = None 1132 1133 try: 1134 stdout, stderr = self._communicate(input, endtime, timeout) 1135 except KeyboardInterrupt: 1136 # https://bugs.python.org/issue25942 1137 # See the detailed comment in .wait(). 1138 if timeout is not None: 1139 sigint_timeout = min(self._sigint_wait_secs, 1140 self._remaining_time(endtime)) 1141 else: 1142 sigint_timeout = self._sigint_wait_secs 1143 self._sigint_wait_secs = 0 # nothing else should wait. 1144 try: 1145 self._wait(timeout=sigint_timeout) 1146 except TimeoutExpired: 1147 pass 1148 raise # resume the KeyboardInterrupt 1149 1150 finally: 1151 self._communication_started = True 1152 1153 sts = self.wait(timeout=self._remaining_time(endtime)) 1154 1155 return (stdout, stderr) 1156 1157 1158 def poll(self): 1159 """Check if child process has terminated. Set and return returncode 1160 attribute.""" 1161 return self._internal_poll() 1162 1163 1164 def _remaining_time(self, endtime): 1165 """Convenience for _communicate when computing timeouts.""" 1166 if endtime is None: 1167 return None 1168 else: 1169 return endtime - _time() 1170 1171 1172 def _check_timeout(self, endtime, orig_timeout, stdout_seq, stderr_seq, 1173 skip_check_and_raise=False): 1174 """Convenience for checking if a timeout has expired.""" 1175 if endtime is None: 1176 return 1177 if skip_check_and_raise or _time() > endtime: 1178 raise TimeoutExpired( 1179 self.args, orig_timeout, 1180 output=b''.join(stdout_seq) if stdout_seq else None, 1181 stderr=b''.join(stderr_seq) if stderr_seq else None) 1182 1183 1184 def wait(self, timeout=None): 1185 """Wait for child process to terminate; returns self.returncode.""" 1186 if timeout is not None: 1187 endtime = _time() + timeout 1188 try: 1189 return self._wait(timeout=timeout) 1190 except KeyboardInterrupt: 1191 # https://bugs.python.org/issue25942 1192 # The first keyboard interrupt waits briefly for the child to 1193 # exit under the common assumption that it also received the ^C 1194 # generated SIGINT and will exit rapidly. 1195 if timeout is not None: 1196 sigint_timeout = min(self._sigint_wait_secs, 1197 self._remaining_time(endtime)) 1198 else: 1199 sigint_timeout = self._sigint_wait_secs 1200 self._sigint_wait_secs = 0 # nothing else should wait. 1201 try: 1202 self._wait(timeout=sigint_timeout) 1203 except TimeoutExpired: 1204 pass 1205 raise # resume the KeyboardInterrupt 1206 1207 def _close_pipe_fds(self, 1208 p2cread, p2cwrite, 1209 c2pread, c2pwrite, 1210 errread, errwrite): 1211 # self._devnull is not always defined. 1212 devnull_fd = getattr(self, '_devnull', None) 1213 1214 with contextlib.ExitStack() as stack: 1215 if _mswindows: 1216 if p2cread != -1: 1217 stack.callback(p2cread.Close) 1218 if c2pwrite != -1: 1219 stack.callback(c2pwrite.Close) 1220 if errwrite != -1: 1221 stack.callback(errwrite.Close) 1222 else: 1223 if p2cread != -1 and p2cwrite != -1 and p2cread != devnull_fd: 1224 stack.callback(os.close, p2cread) 1225 if c2pwrite != -1 and c2pread != -1 and c2pwrite != devnull_fd: 1226 stack.callback(os.close, c2pwrite) 1227 if errwrite != -1 and errread != -1 and errwrite != devnull_fd: 1228 stack.callback(os.close, errwrite) 1229 1230 if devnull_fd is not None: 1231 stack.callback(os.close, devnull_fd) 1232 1233 # Prevent a double close of these handles/fds from __init__ on error. 1234 self._closed_child_pipe_fds = True 1235 1236 if _mswindows: 1237 # 1238 # Windows methods 1239 # 1240 def _get_handles(self, stdin, stdout, stderr): 1241 """Construct and return tuple with IO objects: 1242 p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite 1243 """ 1244 if stdin is None and stdout is None and stderr is None: 1245 return (-1, -1, -1, -1, -1, -1) 1246 1247 p2cread, p2cwrite = -1, -1 1248 c2pread, c2pwrite = -1, -1 1249 errread, errwrite = -1, -1 1250 1251 if stdin is None: 1252 p2cread = _winapi.GetStdHandle(_winapi.STD_INPUT_HANDLE) 1253 if p2cread is None: 1254 p2cread, _ = _winapi.CreatePipe(None, 0) 1255 p2cread = Handle(p2cread) 1256 _winapi.CloseHandle(_) 1257 elif stdin == PIPE: 1258 p2cread, p2cwrite = _winapi.CreatePipe(None, 0) 1259 p2cread, p2cwrite = Handle(p2cread), Handle(p2cwrite) 1260 elif stdin == DEVNULL: 1261 p2cread = msvcrt.get_osfhandle(self._get_devnull()) 1262 elif isinstance(stdin, int): 1263 p2cread = msvcrt.get_osfhandle(stdin) 1264 else: 1265 # Assuming file-like object 1266 p2cread = msvcrt.get_osfhandle(stdin.fileno()) 1267 p2cread = self._make_inheritable(p2cread) 1268 1269 if stdout is None: 1270 c2pwrite = _winapi.GetStdHandle(_winapi.STD_OUTPUT_HANDLE) 1271 if c2pwrite is None: 1272 _, c2pwrite = _winapi.CreatePipe(None, 0) 1273 c2pwrite = Handle(c2pwrite) 1274 _winapi.CloseHandle(_) 1275 elif stdout == PIPE: 1276 c2pread, c2pwrite = _winapi.CreatePipe(None, 0) 1277 c2pread, c2pwrite = Handle(c2pread), Handle(c2pwrite) 1278 elif stdout == DEVNULL: 1279 c2pwrite = msvcrt.get_osfhandle(self._get_devnull()) 1280 elif isinstance(stdout, int): 1281 c2pwrite = msvcrt.get_osfhandle(stdout) 1282 else: 1283 # Assuming file-like object 1284 c2pwrite = msvcrt.get_osfhandle(stdout.fileno()) 1285 c2pwrite = self._make_inheritable(c2pwrite) 1286 1287 if stderr is None: 1288 errwrite = _winapi.GetStdHandle(_winapi.STD_ERROR_HANDLE) 1289 if errwrite is None: 1290 _, errwrite = _winapi.CreatePipe(None, 0) 1291 errwrite = Handle(errwrite) 1292 _winapi.CloseHandle(_) 1293 elif stderr == PIPE: 1294 errread, errwrite = _winapi.CreatePipe(None, 0) 1295 errread, errwrite = Handle(errread), Handle(errwrite) 1296 elif stderr == STDOUT: 1297 errwrite = c2pwrite 1298 elif stderr == DEVNULL: 1299 errwrite = msvcrt.get_osfhandle(self._get_devnull()) 1300 elif isinstance(stderr, int): 1301 errwrite = msvcrt.get_osfhandle(stderr) 1302 else: 1303 # Assuming file-like object 1304 errwrite = msvcrt.get_osfhandle(stderr.fileno()) 1305 errwrite = self._make_inheritable(errwrite) 1306 1307 return (p2cread, p2cwrite, 1308 c2pread, c2pwrite, 1309 errread, errwrite) 1310 1311 1312 def _make_inheritable(self, handle): 1313 """Return a duplicate of handle, which is inheritable""" 1314 h = _winapi.DuplicateHandle( 1315 _winapi.GetCurrentProcess(), handle, 1316 _winapi.GetCurrentProcess(), 0, 1, 1317 _winapi.DUPLICATE_SAME_ACCESS) 1318 return Handle(h) 1319 1320 1321 def _filter_handle_list(self, handle_list): 1322 """Filter out console handles that can't be used 1323 in lpAttributeList["handle_list"] and make sure the list 1324 isn't empty. This also removes duplicate handles.""" 1325 # An handle with it's lowest two bits set might be a special console 1326 # handle that if passed in lpAttributeList["handle_list"], will 1327 # cause it to fail. 1328 return list({handle for handle in handle_list 1329 if handle & 0x3 != 0x3 1330 or _winapi.GetFileType(handle) != 1331 _winapi.FILE_TYPE_CHAR}) 1332 1333 1334 def _execute_child(self, args, executable, preexec_fn, close_fds, 1335 pass_fds, cwd, env, 1336 startupinfo, creationflags, shell, 1337 p2cread, p2cwrite, 1338 c2pread, c2pwrite, 1339 errread, errwrite, 1340 unused_restore_signals, 1341 unused_gid, unused_gids, unused_uid, 1342 unused_umask, 1343 unused_start_new_session): 1344 """Execute program (MS Windows version)""" 1345 1346 assert not pass_fds, "pass_fds not supported on Windows." 1347 1348 if isinstance(args, str): 1349 pass 1350 elif isinstance(args, bytes): 1351 if shell: 1352 raise TypeError('bytes args is not allowed on Windows') 1353 args = list2cmdline([args]) 1354 elif isinstance(args, os.PathLike): 1355 if shell: 1356 raise TypeError('path-like args is not allowed when ' 1357 'shell is true') 1358 args = list2cmdline([args]) 1359 else: 1360 args = list2cmdline(args) 1361 1362 if executable is not None: 1363 executable = os.fsdecode(executable) 1364 1365 # Process startup details 1366 if startupinfo is None: 1367 startupinfo = STARTUPINFO() 1368 else: 1369 # bpo-34044: Copy STARTUPINFO since it is modified above, 1370 # so the caller can reuse it multiple times. 1371 startupinfo = startupinfo.copy() 1372 1373 use_std_handles = -1 not in (p2cread, c2pwrite, errwrite) 1374 if use_std_handles: 1375 startupinfo.dwFlags |= _winapi.STARTF_USESTDHANDLES 1376 startupinfo.hStdInput = p2cread 1377 startupinfo.hStdOutput = c2pwrite 1378 startupinfo.hStdError = errwrite 1379 1380 attribute_list = startupinfo.lpAttributeList 1381 have_handle_list = bool(attribute_list and 1382 "handle_list" in attribute_list and 1383 attribute_list["handle_list"]) 1384 1385 # If we were given an handle_list or need to create one 1386 if have_handle_list or (use_std_handles and close_fds): 1387 if attribute_list is None: 1388 attribute_list = startupinfo.lpAttributeList = {} 1389 handle_list = attribute_list["handle_list"] = \ 1390 list(attribute_list.get("handle_list", [])) 1391 1392 if use_std_handles: 1393 handle_list += [int(p2cread), int(c2pwrite), int(errwrite)] 1394 1395 handle_list[:] = self._filter_handle_list(handle_list) 1396 1397 if handle_list: 1398 if not close_fds: 1399 warnings.warn("startupinfo.lpAttributeList['handle_list'] " 1400 "overriding close_fds", RuntimeWarning) 1401 1402 # When using the handle_list we always request to inherit 1403 # handles but the only handles that will be inherited are 1404 # the ones in the handle_list 1405 close_fds = False 1406 1407 if shell: 1408 startupinfo.dwFlags |= _winapi.STARTF_USESHOWWINDOW 1409 startupinfo.wShowWindow = _winapi.SW_HIDE 1410 comspec = os.environ.get("COMSPEC", "cmd.exe") 1411 args = '{} /c "{}"'.format (comspec, args) 1412 1413 if cwd is not None: 1414 cwd = os.fsdecode(cwd) 1415 1416 sys.audit("subprocess.Popen", executable, args, cwd, env) 1417 1418 # Start the process 1419 try: 1420 hp, ht, pid, tid = _winapi.CreateProcess(executable, args, 1421 # no special security 1422 None, None, 1423 int(not close_fds), 1424 creationflags, 1425 env, 1426 cwd, 1427 startupinfo) 1428 finally: 1429 # Child is launched. Close the parent's copy of those pipe 1430 # handles that only the child should have open. You need 1431 # to make sure that no handles to the write end of the 1432 # output pipe are maintained in this process or else the 1433 # pipe will not close when the child process exits and the 1434 # ReadFile will hang. 1435 self._close_pipe_fds(p2cread, p2cwrite, 1436 c2pread, c2pwrite, 1437 errread, errwrite) 1438 1439 # Retain the process handle, but close the thread handle 1440 self._child_created = True 1441 self._handle = Handle(hp) 1442 self.pid = pid 1443 _winapi.CloseHandle(ht) 1444 1445 def _internal_poll(self, _deadstate=None, 1446 _WaitForSingleObject=_winapi.WaitForSingleObject, 1447 _WAIT_OBJECT_0=_winapi.WAIT_OBJECT_0, 1448 _GetExitCodeProcess=_winapi.GetExitCodeProcess): 1449 """Check if child process has terminated. Returns returncode 1450 attribute. 1451 1452 This method is called by __del__, so it can only refer to objects 1453 in its local scope. 1454 1455 """ 1456 if self.returncode is None: 1457 if _WaitForSingleObject(self._handle, 0) == _WAIT_OBJECT_0: 1458 self.returncode = _GetExitCodeProcess(self._handle) 1459 return self.returncode 1460 1461 1462 def _wait(self, timeout): 1463 """Internal implementation of wait() on Windows.""" 1464 if timeout is None: 1465 timeout_millis = _winapi.INFINITE 1466 else: 1467 timeout_millis = int(timeout * 1000) 1468 if self.returncode is None: 1469 # API note: Returns immediately if timeout_millis == 0. 1470 result = _winapi.WaitForSingleObject(self._handle, 1471 timeout_millis) 1472 if result == _winapi.WAIT_TIMEOUT: 1473 raise TimeoutExpired(self.args, timeout) 1474 self.returncode = _winapi.GetExitCodeProcess(self._handle) 1475 return self.returncode 1476 1477 1478 def _readerthread(self, fh, buffer): 1479 buffer.append(fh.read()) 1480 fh.close() 1481 1482 1483 def _communicate(self, input, endtime, orig_timeout): 1484 # Start reader threads feeding into a list hanging off of this 1485 # object, unless they've already been started. 1486 if self.stdout and not hasattr(self, "_stdout_buff"): 1487 self._stdout_buff = [] 1488 self.stdout_thread = \ 1489 threading.Thread(target=self._readerthread, 1490 args=(self.stdout, self._stdout_buff)) 1491 self.stdout_thread.daemon = True 1492 self.stdout_thread.start() 1493 if self.stderr and not hasattr(self, "_stderr_buff"): 1494 self._stderr_buff = [] 1495 self.stderr_thread = \ 1496 threading.Thread(target=self._readerthread, 1497 args=(self.stderr, self._stderr_buff)) 1498 self.stderr_thread.daemon = True 1499 self.stderr_thread.start() 1500 1501 if self.stdin: 1502 self._stdin_write(input) 1503 1504 # Wait for the reader threads, or time out. If we time out, the 1505 # threads remain reading and the fds left open in case the user 1506 # calls communicate again. 1507 if self.stdout is not None: 1508 self.stdout_thread.join(self._remaining_time(endtime)) 1509 if self.stdout_thread.is_alive(): 1510 raise TimeoutExpired(self.args, orig_timeout) 1511 if self.stderr is not None: 1512 self.stderr_thread.join(self._remaining_time(endtime)) 1513 if self.stderr_thread.is_alive(): 1514 raise TimeoutExpired(self.args, orig_timeout) 1515 1516 # Collect the output from and close both pipes, now that we know 1517 # both have been read successfully. 1518 stdout = None 1519 stderr = None 1520 if self.stdout: 1521 stdout = self._stdout_buff 1522 self.stdout.close() 1523 if self.stderr: 1524 stderr = self._stderr_buff 1525 self.stderr.close() 1526 1527 # All data exchanged. Translate lists into strings. 1528 if stdout is not None: 1529 stdout = stdout[0] 1530 if stderr is not None: 1531 stderr = stderr[0] 1532 1533 return (stdout, stderr) 1534 1535 def send_signal(self, sig): 1536 """Send a signal to the process.""" 1537 # Don't signal a process that we know has already died. 1538 if self.returncode is not None: 1539 return 1540 if sig == signal.SIGTERM: 1541 self.terminate() 1542 elif sig == signal.CTRL_C_EVENT: 1543 os.kill(self.pid, signal.CTRL_C_EVENT) 1544 elif sig == signal.CTRL_BREAK_EVENT: 1545 os.kill(self.pid, signal.CTRL_BREAK_EVENT) 1546 else: 1547 raise ValueError("Unsupported signal: {}".format(sig)) 1548 1549 def terminate(self): 1550 """Terminates the process.""" 1551 # Don't terminate a process that we know has already died. 1552 if self.returncode is not None: 1553 return 1554 try: 1555 _winapi.TerminateProcess(self._handle, 1) 1556 except PermissionError: 1557 # ERROR_ACCESS_DENIED (winerror 5) is received when the 1558 # process already died. 1559 rc = _winapi.GetExitCodeProcess(self._handle) 1560 if rc == _winapi.STILL_ACTIVE: 1561 raise 1562 self.returncode = rc 1563 1564 kill = terminate 1565 1566 else: 1567 # 1568 # POSIX methods 1569 # 1570 def _get_handles(self, stdin, stdout, stderr): 1571 """Construct and return tuple with IO objects: 1572 p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite 1573 """ 1574 p2cread, p2cwrite = -1, -1 1575 c2pread, c2pwrite = -1, -1 1576 errread, errwrite = -1, -1 1577 1578 if stdin is None: 1579 pass 1580 elif stdin == PIPE: 1581 p2cread, p2cwrite = os.pipe() 1582 elif stdin == DEVNULL: 1583 p2cread = self._get_devnull() 1584 elif isinstance(stdin, int): 1585 p2cread = stdin 1586 else: 1587 # Assuming file-like object 1588 p2cread = stdin.fileno() 1589 1590 if stdout is None: 1591 pass 1592 elif stdout == PIPE: 1593 c2pread, c2pwrite = os.pipe() 1594 elif stdout == DEVNULL: 1595 c2pwrite = self._get_devnull() 1596 elif isinstance(stdout, int): 1597 c2pwrite = stdout 1598 else: 1599 # Assuming file-like object 1600 c2pwrite = stdout.fileno() 1601 1602 if stderr is None: 1603 pass 1604 elif stderr == PIPE: 1605 errread, errwrite = os.pipe() 1606 elif stderr == STDOUT: 1607 if c2pwrite != -1: 1608 errwrite = c2pwrite 1609 else: # child's stdout is not set, use parent's stdout 1610 errwrite = sys.__stdout__.fileno() 1611 elif stderr == DEVNULL: 1612 errwrite = self._get_devnull() 1613 elif isinstance(stderr, int): 1614 errwrite = stderr 1615 else: 1616 # Assuming file-like object 1617 errwrite = stderr.fileno() 1618 1619 return (p2cread, p2cwrite, 1620 c2pread, c2pwrite, 1621 errread, errwrite) 1622 1623 1624 def _posix_spawn(self, args, executable, env, restore_signals, 1625 p2cread, p2cwrite, 1626 c2pread, c2pwrite, 1627 errread, errwrite): 1628 """Execute program using os.posix_spawn().""" 1629 if env is None: 1630 env = os.environ 1631 1632 kwargs = {} 1633 if restore_signals: 1634 # See _Py_RestoreSignals() in Python/pylifecycle.c 1635 sigset = [] 1636 for signame in ('SIGPIPE', 'SIGXFZ', 'SIGXFSZ'): 1637 signum = getattr(signal, signame, None) 1638 if signum is not None: 1639 sigset.append(signum) 1640 kwargs['setsigdef'] = sigset 1641 1642 file_actions = [] 1643 for fd in (p2cwrite, c2pread, errread): 1644 if fd != -1: 1645 file_actions.append((os.POSIX_SPAWN_CLOSE, fd)) 1646 for fd, fd2 in ( 1647 (p2cread, 0), 1648 (c2pwrite, 1), 1649 (errwrite, 2), 1650 ): 1651 if fd != -1: 1652 file_actions.append((os.POSIX_SPAWN_DUP2, fd, fd2)) 1653 if file_actions: 1654 kwargs['file_actions'] = file_actions 1655 1656 self.pid = os.posix_spawn(executable, args, env, **kwargs) 1657 self._child_created = True 1658 1659 self._close_pipe_fds(p2cread, p2cwrite, 1660 c2pread, c2pwrite, 1661 errread, errwrite) 1662 1663 def _execute_child(self, args, executable, preexec_fn, close_fds, 1664 pass_fds, cwd, env, 1665 startupinfo, creationflags, shell, 1666 p2cread, p2cwrite, 1667 c2pread, c2pwrite, 1668 errread, errwrite, 1669 restore_signals, 1670 gid, gids, uid, umask, 1671 start_new_session): 1672 """Execute program (POSIX version)""" 1673 1674 if isinstance(args, (str, bytes)): 1675 args = [args] 1676 elif isinstance(args, os.PathLike): 1677 if shell: 1678 raise TypeError('path-like args is not allowed when ' 1679 'shell is true') 1680 args = [args] 1681 else: 1682 args = list(args) 1683 1684 if shell: 1685 # On Android the default shell is at '/system/bin/sh'. 1686 unix_shell = ('/system/bin/sh' if 1687 hasattr(sys, 'getandroidapilevel') else '/bin/sh') 1688 args = [unix_shell, "-c"] + args 1689 if executable: 1690 args[0] = executable 1691 1692 if executable is None: 1693 executable = args[0] 1694 1695 sys.audit("subprocess.Popen", executable, args, cwd, env) 1696 1697 if (_USE_POSIX_SPAWN 1698 and os.path.dirname(executable) 1699 and preexec_fn is None 1700 and not close_fds 1701 and not pass_fds 1702 and cwd is None 1703 and (p2cread == -1 or p2cread > 2) 1704 and (c2pwrite == -1 or c2pwrite > 2) 1705 and (errwrite == -1 or errwrite > 2) 1706 and not start_new_session 1707 and gid is None 1708 and gids is None 1709 and uid is None 1710 and umask < 0): 1711 self._posix_spawn(args, executable, env, restore_signals, 1712 p2cread, p2cwrite, 1713 c2pread, c2pwrite, 1714 errread, errwrite) 1715 return 1716 1717 orig_executable = executable 1718 1719 # For transferring possible exec failure from child to parent. 1720 # Data format: "exception name:hex errno:description" 1721 # Pickle is not used; it is complex and involves memory allocation. 1722 errpipe_read, errpipe_write = os.pipe() 1723 # errpipe_write must not be in the standard io 0, 1, or 2 fd range. 1724 low_fds_to_close = [] 1725 while errpipe_write < 3: 1726 low_fds_to_close.append(errpipe_write) 1727 errpipe_write = os.dup(errpipe_write) 1728 for low_fd in low_fds_to_close: 1729 os.close(low_fd) 1730 try: 1731 try: 1732 # We must avoid complex work that could involve 1733 # malloc or free in the child process to avoid 1734 # potential deadlocks, thus we do all this here. 1735 # and pass it to fork_exec() 1736 1737 if env is not None: 1738 env_list = [] 1739 for k, v in env.items(): 1740 k = os.fsencode(k) 1741 if b'=' in k: 1742 raise ValueError("illegal environment variable name") 1743 env_list.append(k + b'=' + os.fsencode(v)) 1744 else: 1745 env_list = None # Use execv instead of execve. 1746 executable = os.fsencode(executable) 1747 if os.path.dirname(executable): 1748 executable_list = (executable,) 1749 else: 1750 # This matches the behavior of os._execvpe(). 1751 executable_list = tuple( 1752 os.path.join(os.fsencode(dir), executable) 1753 for dir in os.get_exec_path(env)) 1754 fds_to_keep = set(pass_fds) 1755 fds_to_keep.add(errpipe_write) 1756 self.pid = _posixsubprocess.fork_exec( 1757 args, executable_list, 1758 close_fds, tuple(sorted(map(int, fds_to_keep))), 1759 cwd, env_list, 1760 p2cread, p2cwrite, c2pread, c2pwrite, 1761 errread, errwrite, 1762 errpipe_read, errpipe_write, 1763 restore_signals, start_new_session, 1764 gid, gids, uid, umask, 1765 preexec_fn) 1766 self._child_created = True 1767 finally: 1768 # be sure the FD is closed no matter what 1769 os.close(errpipe_write) 1770 1771 self._close_pipe_fds(p2cread, p2cwrite, 1772 c2pread, c2pwrite, 1773 errread, errwrite) 1774 1775 # Wait for exec to fail or succeed; possibly raising an 1776 # exception (limited in size) 1777 errpipe_data = bytearray() 1778 while True: 1779 part = os.read(errpipe_read, 50000) 1780 errpipe_data += part 1781 if not part or len(errpipe_data) > 50000: 1782 break 1783 finally: 1784 # be sure the FD is closed no matter what 1785 os.close(errpipe_read) 1786 1787 if errpipe_data: 1788 try: 1789 pid, sts = os.waitpid(self.pid, 0) 1790 if pid == self.pid: 1791 self._handle_exitstatus(sts) 1792 else: 1793 self.returncode = sys.maxsize 1794 except ChildProcessError: 1795 pass 1796 1797 try: 1798 exception_name, hex_errno, err_msg = ( 1799 errpipe_data.split(b':', 2)) 1800 # The encoding here should match the encoding 1801 # written in by the subprocess implementations 1802 # like _posixsubprocess 1803 err_msg = err_msg.decode() 1804 except ValueError: 1805 exception_name = b'SubprocessError' 1806 hex_errno = b'0' 1807 err_msg = 'Bad exception data from child: {!r}'.format( 1808 bytes(errpipe_data)) 1809 child_exception_type = getattr( 1810 builtins, exception_name.decode('ascii'), 1811 SubprocessError) 1812 if issubclass(child_exception_type, OSError) and hex_errno: 1813 errno_num = int(hex_errno, 16) 1814 child_exec_never_called = (err_msg == "noexec") 1815 if child_exec_never_called: 1816 err_msg = "" 1817 # The error must be from chdir(cwd). 1818 err_filename = cwd 1819 else: 1820 err_filename = orig_executable 1821 if errno_num != 0: 1822 err_msg = os.strerror(errno_num) 1823 raise child_exception_type(errno_num, err_msg, err_filename) 1824 raise child_exception_type(err_msg) 1825 1826 1827 def _handle_exitstatus(self, sts, 1828 waitstatus_to_exitcode=os.waitstatus_to_exitcode, 1829 _WIFSTOPPED=os.WIFSTOPPED, 1830 _WSTOPSIG=os.WSTOPSIG): 1831 """All callers to this function MUST hold self._waitpid_lock.""" 1832 # This method is called (indirectly) by __del__, so it cannot 1833 # refer to anything outside of its local scope. 1834 if _WIFSTOPPED(sts): 1835 self.returncode = -_WSTOPSIG(sts) 1836 else: 1837 self.returncode = waitstatus_to_exitcode(sts) 1838 1839 def _internal_poll(self, _deadstate=None, _waitpid=os.waitpid, 1840 _WNOHANG=os.WNOHANG, _ECHILD=errno.ECHILD): 1841 """Check if child process has terminated. Returns returncode 1842 attribute. 1843 1844 This method is called by __del__, so it cannot reference anything 1845 outside of the local scope (nor can any methods it calls). 1846 1847 """ 1848 if self.returncode is None: 1849 if not self._waitpid_lock.acquire(False): 1850 # Something else is busy calling waitpid. Don't allow two 1851 # at once. We know nothing yet. 1852 return None 1853 try: 1854 if self.returncode is not None: 1855 return self.returncode # Another thread waited. 1856 pid, sts = _waitpid(self.pid, _WNOHANG) 1857 if pid == self.pid: 1858 self._handle_exitstatus(sts) 1859 except OSError as e: 1860 if _deadstate is not None: 1861 self.returncode = _deadstate 1862 elif e.errno == _ECHILD: 1863 # This happens if SIGCLD is set to be ignored or 1864 # waiting for child processes has otherwise been 1865 # disabled for our process. This child is dead, we 1866 # can't get the status. 1867 # http://bugs.python.org/issue15756 1868 self.returncode = 0 1869 finally: 1870 self._waitpid_lock.release() 1871 return self.returncode 1872 1873 1874 def _try_wait(self, wait_flags): 1875 """All callers to this function MUST hold self._waitpid_lock.""" 1876 try: 1877 (pid, sts) = os.waitpid(self.pid, wait_flags) 1878 except ChildProcessError: 1879 # This happens if SIGCLD is set to be ignored or waiting 1880 # for child processes has otherwise been disabled for our 1881 # process. This child is dead, we can't get the status. 1882 pid = self.pid 1883 sts = 0 1884 return (pid, sts) 1885 1886 1887 def _wait(self, timeout): 1888 """Internal implementation of wait() on POSIX.""" 1889 if self.returncode is not None: 1890 return self.returncode 1891 1892 if timeout is not None: 1893 endtime = _time() + timeout 1894 # Enter a busy loop if we have a timeout. This busy loop was 1895 # cribbed from Lib/threading.py in Thread.wait() at r71065. 1896 delay = 0.0005 # 500 us -> initial delay of 1 ms 1897 while True: 1898 if self._waitpid_lock.acquire(False): 1899 try: 1900 if self.returncode is not None: 1901 break # Another thread waited. 1902 (pid, sts) = self._try_wait(os.WNOHANG) 1903 assert pid == self.pid or pid == 0 1904 if pid == self.pid: 1905 self._handle_exitstatus(sts) 1906 break 1907 finally: 1908 self._waitpid_lock.release() 1909 remaining = self._remaining_time(endtime) 1910 if remaining <= 0: 1911 raise TimeoutExpired(self.args, timeout) 1912 delay = min(delay * 2, remaining, .05) 1913 time.sleep(delay) 1914 else: 1915 while self.returncode is None: 1916 with self._waitpid_lock: 1917 if self.returncode is not None: 1918 break # Another thread waited. 1919 (pid, sts) = self._try_wait(0) 1920 # Check the pid and loop as waitpid has been known to 1921 # return 0 even without WNOHANG in odd situations. 1922 # http://bugs.python.org/issue14396. 1923 if pid == self.pid: 1924 self._handle_exitstatus(sts) 1925 return self.returncode 1926 1927 1928 def _communicate(self, input, endtime, orig_timeout): 1929 if self.stdin and not self._communication_started: 1930 # Flush stdio buffer. This might block, if the user has 1931 # been writing to .stdin in an uncontrolled fashion. 1932 try: 1933 self.stdin.flush() 1934 except BrokenPipeError: 1935 pass # communicate() must ignore BrokenPipeError. 1936 if not input: 1937 try: 1938 self.stdin.close() 1939 except BrokenPipeError: 1940 pass # communicate() must ignore BrokenPipeError. 1941 1942 stdout = None 1943 stderr = None 1944 1945 # Only create this mapping if we haven't already. 1946 if not self._communication_started: 1947 self._fileobj2output = {} 1948 if self.stdout: 1949 self._fileobj2output[self.stdout] = [] 1950 if self.stderr: 1951 self._fileobj2output[self.stderr] = [] 1952 1953 if self.stdout: 1954 stdout = self._fileobj2output[self.stdout] 1955 if self.stderr: 1956 stderr = self._fileobj2output[self.stderr] 1957 1958 self._save_input(input) 1959 1960 if self._input: 1961 input_view = memoryview(self._input) 1962 1963 with _PopenSelector() as selector: 1964 if self.stdin and input: 1965 selector.register(self.stdin, selectors.EVENT_WRITE) 1966 if self.stdout and not self.stdout.closed: 1967 selector.register(self.stdout, selectors.EVENT_READ) 1968 if self.stderr and not self.stderr.closed: 1969 selector.register(self.stderr, selectors.EVENT_READ) 1970 1971 while selector.get_map(): 1972 timeout = self._remaining_time(endtime) 1973 if timeout is not None and timeout < 0: 1974 self._check_timeout(endtime, orig_timeout, 1975 stdout, stderr, 1976 skip_check_and_raise=True) 1977 raise RuntimeError( # Impossible :) 1978 '_check_timeout(..., skip_check_and_raise=True) ' 1979 'failed to raise TimeoutExpired.') 1980 1981 ready = selector.select(timeout) 1982 self._check_timeout(endtime, orig_timeout, stdout, stderr) 1983 1984 # XXX Rewrite these to use non-blocking I/O on the file 1985 # objects; they are no longer using C stdio! 1986 1987 for key, events in ready: 1988 if key.fileobj is self.stdin: 1989 chunk = input_view[self._input_offset : 1990 self._input_offset + _PIPE_BUF] 1991 try: 1992 self._input_offset += os.write(key.fd, chunk) 1993 except BrokenPipeError: 1994 selector.unregister(key.fileobj) 1995 key.fileobj.close() 1996 else: 1997 if self._input_offset >= len(self._input): 1998 selector.unregister(key.fileobj) 1999 key.fileobj.close() 2000 elif key.fileobj in (self.stdout, self.stderr): 2001 data = os.read(key.fd, 32768) 2002 if not data: 2003 selector.unregister(key.fileobj) 2004 key.fileobj.close() 2005 self._fileobj2output[key.fileobj].append(data) 2006 2007 self.wait(timeout=self._remaining_time(endtime)) 2008 2009 # All data exchanged. Translate lists into strings. 2010 if stdout is not None: 2011 stdout = b''.join(stdout) 2012 if stderr is not None: 2013 stderr = b''.join(stderr) 2014 2015 # Translate newlines, if requested. 2016 # This also turns bytes into strings. 2017 if self.text_mode: 2018 if stdout is not None: 2019 stdout = self._translate_newlines(stdout, 2020 self.stdout.encoding, 2021 self.stdout.errors) 2022 if stderr is not None: 2023 stderr = self._translate_newlines(stderr, 2024 self.stderr.encoding, 2025 self.stderr.errors) 2026 2027 return (stdout, stderr) 2028 2029 2030 def _save_input(self, input): 2031 # This method is called from the _communicate_with_*() methods 2032 # so that if we time out while communicating, we can continue 2033 # sending input if we retry. 2034 if self.stdin and self._input is None: 2035 self._input_offset = 0 2036 self._input = input 2037 if input is not None and self.text_mode: 2038 self._input = self._input.encode(self.stdin.encoding, 2039 self.stdin.errors) 2040 2041 2042 def send_signal(self, sig): 2043 """Send a signal to the process.""" 2044 # bpo-38630: Polling reduces the risk of sending a signal to the 2045 # wrong process if the process completed, the Popen.returncode 2046 # attribute is still None, and the pid has been reassigned 2047 # (recycled) to a new different process. This race condition can 2048 # happens in two cases. 2049 # 2050 # Case 1. Thread A calls Popen.poll(), thread B calls 2051 # Popen.send_signal(). In thread A, waitpid() succeed and returns 2052 # the exit status. Thread B calls kill() because poll() in thread A 2053 # did not set returncode yet. Calling poll() in thread B prevents 2054 # the race condition thanks to Popen._waitpid_lock. 2055 # 2056 # Case 2. waitpid(pid, 0) has been called directly, without 2057 # using Popen methods: returncode is still None is this case. 2058 # Calling Popen.poll() will set returncode to a default value, 2059 # since waitpid() fails with ProcessLookupError. 2060 self.poll() 2061 if self.returncode is not None: 2062 # Skip signalling a process that we know has already died. 2063 return 2064 2065 # The race condition can still happen if the race condition 2066 # described above happens between the returncode test 2067 # and the kill() call. 2068 try: 2069 os.kill(self.pid, sig) 2070 except ProcessLookupError: 2071 # Supress the race condition error; bpo-40550. 2072 pass 2073 2074 def terminate(self): 2075 """Terminate the process with SIGTERM 2076 """ 2077 self.send_signal(signal.SIGTERM) 2078 2079 def kill(self): 2080 """Kill the process with SIGKILL 2081 """ 2082 self.send_signal(signal.SIGKILL) 2083