1# subprocess - Subprocesses with accessible I/O streams 2# 3# For more information about this module, see PEP 324. 4# 5# Copyright (c) 2003-2005 by Peter Astrand <astrand@lysator.liu.se> 6# 7# Licensed to PSF under a Contributor Agreement. 8 9r"""Subprocesses with accessible I/O streams 10 11This module allows you to spawn processes, connect to their 12input/output/error pipes, and obtain their return codes. 13 14For a complete description of this module see the Python documentation. 15 16Main API 17======== 18run(...): Runs a command, waits for it to complete, then returns a 19 CompletedProcess instance. 20Popen(...): A class for flexibly executing a command in a new process 21 22Constants 23--------- 24DEVNULL: Special value that indicates that os.devnull should be used 25PIPE: Special value that indicates a pipe should be created 26STDOUT: Special value that indicates that stderr should go to stdout 27 28 29Older API 30========= 31call(...): Runs a command, waits for it to complete, then returns 32 the return code. 33check_call(...): Same as call() but raises CalledProcessError() 34 if return code is not 0 35check_output(...): Same as check_call() but returns the contents of 36 stdout instead of a return code 37getoutput(...): Runs a command in the shell, waits for it to complete, 38 then returns the output 39getstatusoutput(...): Runs a command in the shell, waits for it to complete, 40 then returns a (exitcode, output) tuple 41""" 42 43import builtins 44import errno 45import io 46import locale 47import os 48import time 49import signal 50import sys 51import threading 52import warnings 53import contextlib 54from time import monotonic as _time 55import types 56 57try: 58 import fcntl 59except ImportError: 60 fcntl = None 61 62 63__all__ = ["Popen", "PIPE", "STDOUT", "call", "check_call", "getstatusoutput", 64 "getoutput", "check_output", "run", "CalledProcessError", "DEVNULL", 65 "SubprocessError", "TimeoutExpired", "CompletedProcess"] 66 # NOTE: We intentionally exclude list2cmdline as it is 67 # considered an internal implementation detail. issue10838. 68 69# use presence of msvcrt to detect Windows-like platforms (see bpo-8110) 70try: 71 import msvcrt 72except ModuleNotFoundError: 73 _mswindows = False 74else: 75 _mswindows = True 76 77# some platforms do not support subprocesses 78_can_fork_exec = sys.platform not in {"emscripten", "wasi", "ios", "tvos", "watchos"} 79 80if _mswindows: 81 import _winapi 82 from _winapi import (CREATE_NEW_CONSOLE, CREATE_NEW_PROCESS_GROUP, 83 STD_INPUT_HANDLE, STD_OUTPUT_HANDLE, 84 STD_ERROR_HANDLE, SW_HIDE, 85 STARTF_USESTDHANDLES, STARTF_USESHOWWINDOW, 86 STARTF_FORCEONFEEDBACK, STARTF_FORCEOFFFEEDBACK, 87 ABOVE_NORMAL_PRIORITY_CLASS, BELOW_NORMAL_PRIORITY_CLASS, 88 HIGH_PRIORITY_CLASS, IDLE_PRIORITY_CLASS, 89 NORMAL_PRIORITY_CLASS, REALTIME_PRIORITY_CLASS, 90 CREATE_NO_WINDOW, DETACHED_PROCESS, 91 CREATE_DEFAULT_ERROR_MODE, CREATE_BREAKAWAY_FROM_JOB) 92 93 __all__.extend(["CREATE_NEW_CONSOLE", "CREATE_NEW_PROCESS_GROUP", 94 "STD_INPUT_HANDLE", "STD_OUTPUT_HANDLE", 95 "STD_ERROR_HANDLE", "SW_HIDE", 96 "STARTF_USESTDHANDLES", "STARTF_USESHOWWINDOW", 97 "STARTF_FORCEONFEEDBACK", "STARTF_FORCEOFFFEEDBACK", 98 "STARTUPINFO", 99 "ABOVE_NORMAL_PRIORITY_CLASS", "BELOW_NORMAL_PRIORITY_CLASS", 100 "HIGH_PRIORITY_CLASS", "IDLE_PRIORITY_CLASS", 101 "NORMAL_PRIORITY_CLASS", "REALTIME_PRIORITY_CLASS", 102 "CREATE_NO_WINDOW", "DETACHED_PROCESS", 103 "CREATE_DEFAULT_ERROR_MODE", "CREATE_BREAKAWAY_FROM_JOB"]) 104else: 105 if _can_fork_exec: 106 from _posixsubprocess import fork_exec as _fork_exec 107 # used in methods that are called by __del__ 108 class _del_safe: 109 waitpid = os.waitpid 110 waitstatus_to_exitcode = os.waitstatus_to_exitcode 111 WIFSTOPPED = os.WIFSTOPPED 112 WSTOPSIG = os.WSTOPSIG 113 WNOHANG = os.WNOHANG 114 ECHILD = errno.ECHILD 115 else: 116 class _del_safe: 117 waitpid = None 118 waitstatus_to_exitcode = None 119 WIFSTOPPED = None 120 WSTOPSIG = None 121 WNOHANG = None 122 ECHILD = errno.ECHILD 123 124 import select 125 import selectors 126 127 128# Exception classes used by this module. 129class SubprocessError(Exception): pass 130 131 132class CalledProcessError(SubprocessError): 133 """Raised when run() is called with check=True and the process 134 returns a non-zero exit status. 135 136 Attributes: 137 cmd, returncode, stdout, stderr, output 138 """ 139 def __init__(self, returncode, cmd, output=None, stderr=None): 140 self.returncode = returncode 141 self.cmd = cmd 142 self.output = output 143 self.stderr = stderr 144 145 def __str__(self): 146 if self.returncode and self.returncode < 0: 147 try: 148 return "Command '%s' died with %r." % ( 149 self.cmd, signal.Signals(-self.returncode)) 150 except ValueError: 151 return "Command '%s' died with unknown signal %d." % ( 152 self.cmd, -self.returncode) 153 else: 154 return "Command '%s' returned non-zero exit status %d." % ( 155 self.cmd, self.returncode) 156 157 @property 158 def stdout(self): 159 """Alias for output attribute, to match stderr""" 160 return self.output 161 162 @stdout.setter 163 def stdout(self, value): 164 # There's no obvious reason to set this, but allow it anyway so 165 # .stdout is a transparent alias for .output 166 self.output = value 167 168 169class TimeoutExpired(SubprocessError): 170 """This exception is raised when the timeout expires while waiting for a 171 child process. 172 173 Attributes: 174 cmd, output, stdout, stderr, timeout 175 """ 176 def __init__(self, cmd, timeout, output=None, stderr=None): 177 self.cmd = cmd 178 self.timeout = timeout 179 self.output = output 180 self.stderr = stderr 181 182 def __str__(self): 183 return ("Command '%s' timed out after %s seconds" % 184 (self.cmd, self.timeout)) 185 186 @property 187 def stdout(self): 188 return self.output 189 190 @stdout.setter 191 def stdout(self, value): 192 # There's no obvious reason to set this, but allow it anyway so 193 # .stdout is a transparent alias for .output 194 self.output = value 195 196 197if _mswindows: 198 class STARTUPINFO: 199 def __init__(self, *, dwFlags=0, hStdInput=None, hStdOutput=None, 200 hStdError=None, wShowWindow=0, lpAttributeList=None): 201 self.dwFlags = dwFlags 202 self.hStdInput = hStdInput 203 self.hStdOutput = hStdOutput 204 self.hStdError = hStdError 205 self.wShowWindow = wShowWindow 206 self.lpAttributeList = lpAttributeList or {"handle_list": []} 207 208 def copy(self): 209 attr_list = self.lpAttributeList.copy() 210 if 'handle_list' in attr_list: 211 attr_list['handle_list'] = list(attr_list['handle_list']) 212 213 return STARTUPINFO(dwFlags=self.dwFlags, 214 hStdInput=self.hStdInput, 215 hStdOutput=self.hStdOutput, 216 hStdError=self.hStdError, 217 wShowWindow=self.wShowWindow, 218 lpAttributeList=attr_list) 219 220 221 class Handle(int): 222 closed = False 223 224 def Close(self, CloseHandle=_winapi.CloseHandle): 225 if not self.closed: 226 self.closed = True 227 CloseHandle(self) 228 229 def Detach(self): 230 if not self.closed: 231 self.closed = True 232 return int(self) 233 raise ValueError("already closed") 234 235 def __repr__(self): 236 return "%s(%d)" % (self.__class__.__name__, int(self)) 237 238 __del__ = Close 239else: 240 # When select or poll has indicated that the file is writable, 241 # we can write up to _PIPE_BUF bytes without risk of blocking. 242 # POSIX defines PIPE_BUF as >= 512. 243 _PIPE_BUF = getattr(select, 'PIPE_BUF', 512) 244 245 # poll/select have the advantage of not requiring any extra file 246 # descriptor, contrarily to epoll/kqueue (also, they require a single 247 # syscall). 248 if hasattr(selectors, 'PollSelector'): 249 _PopenSelector = selectors.PollSelector 250 else: 251 _PopenSelector = selectors.SelectSelector 252 253 254if _mswindows: 255 # On Windows we just need to close `Popen._handle` when we no longer need 256 # it, so that the kernel can free it. `Popen._handle` gets closed 257 # implicitly when the `Popen` instance is finalized (see `Handle.__del__`, 258 # which is calling `CloseHandle` as requested in [1]), so there is nothing 259 # for `_cleanup` to do. 260 # 261 # [1] https://docs.microsoft.com/en-us/windows/desktop/ProcThread/ 262 # creating-processes 263 _active = None 264 265 def _cleanup(): 266 pass 267else: 268 # This lists holds Popen instances for which the underlying process had not 269 # exited at the time its __del__ method got called: those processes are 270 # wait()ed for synchronously from _cleanup() when a new Popen object is 271 # created, to avoid zombie processes. 272 _active = [] 273 274 def _cleanup(): 275 if _active is None: 276 return 277 for inst in _active[:]: 278 res = inst._internal_poll(_deadstate=sys.maxsize) 279 if res is not None: 280 try: 281 _active.remove(inst) 282 except ValueError: 283 # This can happen if two threads create a new Popen instance. 284 # It's harmless that it was already removed, so ignore. 285 pass 286 287PIPE = -1 288STDOUT = -2 289DEVNULL = -3 290 291 292# XXX This function is only used by multiprocessing and the test suite, 293# but it's here so that it can be imported when Python is compiled without 294# threads. 295 296def _optim_args_from_interpreter_flags(): 297 """Return a list of command-line arguments reproducing the current 298 optimization settings in sys.flags.""" 299 args = [] 300 value = sys.flags.optimize 301 if value > 0: 302 args.append('-' + 'O' * value) 303 return args 304 305 306def _args_from_interpreter_flags(): 307 """Return a list of command-line arguments reproducing the current 308 settings in sys.flags, sys.warnoptions and sys._xoptions.""" 309 flag_opt_map = { 310 'debug': 'd', 311 # 'inspect': 'i', 312 # 'interactive': 'i', 313 'dont_write_bytecode': 'B', 314 'no_site': 'S', 315 'verbose': 'v', 316 'bytes_warning': 'b', 317 'quiet': 'q', 318 # -O is handled in _optim_args_from_interpreter_flags() 319 } 320 args = _optim_args_from_interpreter_flags() 321 for flag, opt in flag_opt_map.items(): 322 v = getattr(sys.flags, flag) 323 if v > 0: 324 args.append('-' + opt * v) 325 326 if sys.flags.isolated: 327 args.append('-I') 328 else: 329 if sys.flags.ignore_environment: 330 args.append('-E') 331 if sys.flags.no_user_site: 332 args.append('-s') 333 if sys.flags.safe_path: 334 args.append('-P') 335 336 # -W options 337 warnopts = sys.warnoptions[:] 338 xoptions = getattr(sys, '_xoptions', {}) 339 bytes_warning = sys.flags.bytes_warning 340 dev_mode = sys.flags.dev_mode 341 342 if bytes_warning > 1: 343 warnopts.remove("error::BytesWarning") 344 elif bytes_warning: 345 warnopts.remove("default::BytesWarning") 346 if dev_mode: 347 warnopts.remove('default') 348 for opt in warnopts: 349 args.append('-W' + opt) 350 351 # -X options 352 if dev_mode: 353 args.extend(('-X', 'dev')) 354 for opt in ('faulthandler', 'tracemalloc', 'importtime', 355 'frozen_modules', 'showrefcount', 'utf8', 'gil'): 356 if opt in xoptions: 357 value = xoptions[opt] 358 if value is True: 359 arg = opt 360 else: 361 arg = '%s=%s' % (opt, value) 362 args.extend(('-X', arg)) 363 364 return args 365 366 367def _text_encoding(): 368 # Return default text encoding and emit EncodingWarning if 369 # sys.flags.warn_default_encoding is true. 370 if sys.flags.warn_default_encoding: 371 f = sys._getframe() 372 filename = f.f_code.co_filename 373 stacklevel = 2 374 while f := f.f_back: 375 if f.f_code.co_filename != filename: 376 break 377 stacklevel += 1 378 warnings.warn("'encoding' argument not specified.", 379 EncodingWarning, stacklevel) 380 381 if sys.flags.utf8_mode: 382 return "utf-8" 383 else: 384 return locale.getencoding() 385 386 387def call(*popenargs, timeout=None, **kwargs): 388 """Run command with arguments. Wait for command to complete or 389 timeout, then return the returncode attribute. 390 391 The arguments are the same as for the Popen constructor. Example: 392 393 retcode = call(["ls", "-l"]) 394 """ 395 with Popen(*popenargs, **kwargs) as p: 396 try: 397 return p.wait(timeout=timeout) 398 except: # Including KeyboardInterrupt, wait handled that. 399 p.kill() 400 # We don't call p.wait() again as p.__exit__ does that for us. 401 raise 402 403 404def check_call(*popenargs, **kwargs): 405 """Run command with arguments. Wait for command to complete. If 406 the exit code was zero then return, otherwise raise 407 CalledProcessError. The CalledProcessError object will have the 408 return code in the returncode attribute. 409 410 The arguments are the same as for the call function. Example: 411 412 check_call(["ls", "-l"]) 413 """ 414 retcode = call(*popenargs, **kwargs) 415 if retcode: 416 cmd = kwargs.get("args") 417 if cmd is None: 418 cmd = popenargs[0] 419 raise CalledProcessError(retcode, cmd) 420 return 0 421 422 423def check_output(*popenargs, timeout=None, **kwargs): 424 r"""Run command with arguments and return its output. 425 426 If the exit code was non-zero it raises a CalledProcessError. The 427 CalledProcessError object will have the return code in the returncode 428 attribute and output in the output attribute. 429 430 The arguments are the same as for the Popen constructor. Example: 431 432 >>> check_output(["ls", "-l", "/dev/null"]) 433 b'crw-rw-rw- 1 root root 1, 3 Oct 18 2007 /dev/null\n' 434 435 The stdout argument is not allowed as it is used internally. 436 To capture standard error in the result, use stderr=STDOUT. 437 438 >>> check_output(["/bin/sh", "-c", 439 ... "ls -l non_existent_file ; exit 0"], 440 ... stderr=STDOUT) 441 b'ls: non_existent_file: No such file or directory\n' 442 443 There is an additional optional argument, "input", allowing you to 444 pass a string to the subprocess's stdin. If you use this argument 445 you may not also use the Popen constructor's "stdin" argument, as 446 it too will be used internally. Example: 447 448 >>> check_output(["sed", "-e", "s/foo/bar/"], 449 ... input=b"when in the course of fooman events\n") 450 b'when in the course of barman events\n' 451 452 By default, all communication is in bytes, and therefore any "input" 453 should be bytes, and the return value will be bytes. If in text mode, 454 any "input" should be a string, and the return value will be a string 455 decoded according to locale encoding, or by "encoding" if set. Text mode 456 is triggered by setting any of text, encoding, errors or universal_newlines. 457 """ 458 for kw in ('stdout', 'check'): 459 if kw in kwargs: 460 raise ValueError(f'{kw} argument not allowed, it will be overridden.') 461 462 if 'input' in kwargs and kwargs['input'] is None: 463 # Explicitly passing input=None was previously equivalent to passing an 464 # empty string. That is maintained here for backwards compatibility. 465 if kwargs.get('universal_newlines') or kwargs.get('text') or kwargs.get('encoding') \ 466 or kwargs.get('errors'): 467 empty = '' 468 else: 469 empty = b'' 470 kwargs['input'] = empty 471 472 return run(*popenargs, stdout=PIPE, timeout=timeout, check=True, 473 **kwargs).stdout 474 475 476class CompletedProcess(object): 477 """A process that has finished running. 478 479 This is returned by run(). 480 481 Attributes: 482 args: The list or str args passed to run(). 483 returncode: The exit code of the process, negative for signals. 484 stdout: The standard output (None if not captured). 485 stderr: The standard error (None if not captured). 486 """ 487 def __init__(self, args, returncode, stdout=None, stderr=None): 488 self.args = args 489 self.returncode = returncode 490 self.stdout = stdout 491 self.stderr = stderr 492 493 def __repr__(self): 494 args = ['args={!r}'.format(self.args), 495 'returncode={!r}'.format(self.returncode)] 496 if self.stdout is not None: 497 args.append('stdout={!r}'.format(self.stdout)) 498 if self.stderr is not None: 499 args.append('stderr={!r}'.format(self.stderr)) 500 return "{}({})".format(type(self).__name__, ', '.join(args)) 501 502 __class_getitem__ = classmethod(types.GenericAlias) 503 504 505 def check_returncode(self): 506 """Raise CalledProcessError if the exit code is non-zero.""" 507 if self.returncode: 508 raise CalledProcessError(self.returncode, self.args, self.stdout, 509 self.stderr) 510 511 512def run(*popenargs, 513 input=None, capture_output=False, timeout=None, check=False, **kwargs): 514 """Run command with arguments and return a CompletedProcess instance. 515 516 The returned instance will have attributes args, returncode, stdout and 517 stderr. By default, stdout and stderr are not captured, and those attributes 518 will be None. Pass stdout=PIPE and/or stderr=PIPE in order to capture them, 519 or pass capture_output=True to capture both. 520 521 If check is True and the exit code was non-zero, it raises a 522 CalledProcessError. The CalledProcessError object will have the return code 523 in the returncode attribute, and output & stderr attributes if those streams 524 were captured. 525 526 If timeout is given, and the process takes too long, a TimeoutExpired 527 exception will be raised. 528 529 There is an optional argument "input", allowing you to 530 pass bytes or a string to the subprocess's stdin. If you use this argument 531 you may not also use the Popen constructor's "stdin" argument, as 532 it will be used internally. 533 534 By default, all communication is in bytes, and therefore any "input" should 535 be bytes, and the stdout and stderr will be bytes. If in text mode, any 536 "input" should be a string, and stdout and stderr will be strings decoded 537 according to locale encoding, or by "encoding" if set. Text mode is 538 triggered by setting any of text, encoding, errors or universal_newlines. 539 540 The other arguments are the same as for the Popen constructor. 541 """ 542 if input is not None: 543 if kwargs.get('stdin') is not None: 544 raise ValueError('stdin and input arguments may not both be used.') 545 kwargs['stdin'] = PIPE 546 547 if capture_output: 548 if kwargs.get('stdout') is not None or kwargs.get('stderr') is not None: 549 raise ValueError('stdout and stderr arguments may not be used ' 550 'with capture_output.') 551 kwargs['stdout'] = PIPE 552 kwargs['stderr'] = PIPE 553 554 with Popen(*popenargs, **kwargs) as process: 555 try: 556 stdout, stderr = process.communicate(input, timeout=timeout) 557 except TimeoutExpired as exc: 558 process.kill() 559 if _mswindows: 560 # Windows accumulates the output in a single blocking 561 # read() call run on child threads, with the timeout 562 # being done in a join() on those threads. communicate() 563 # _after_ kill() is required to collect that and add it 564 # to the exception. 565 exc.stdout, exc.stderr = process.communicate() 566 else: 567 # POSIX _communicate already populated the output so 568 # far into the TimeoutExpired exception. 569 process.wait() 570 raise 571 except: # Including KeyboardInterrupt, communicate handled that. 572 process.kill() 573 # We don't call process.wait() as .__exit__ does that for us. 574 raise 575 retcode = process.poll() 576 if check and retcode: 577 raise CalledProcessError(retcode, process.args, 578 output=stdout, stderr=stderr) 579 return CompletedProcess(process.args, retcode, stdout, stderr) 580 581 582def list2cmdline(seq): 583 """ 584 Translate a sequence of arguments into a command line 585 string, using the same rules as the MS C runtime: 586 587 1) Arguments are delimited by white space, which is either a 588 space or a tab. 589 590 2) A string surrounded by double quotation marks is 591 interpreted as a single argument, regardless of white space 592 contained within. A quoted string can be embedded in an 593 argument. 594 595 3) A double quotation mark preceded by a backslash is 596 interpreted as a literal double quotation mark. 597 598 4) Backslashes are interpreted literally, unless they 599 immediately precede a double quotation mark. 600 601 5) If backslashes immediately precede a double quotation mark, 602 every pair of backslashes is interpreted as a literal 603 backslash. If the number of backslashes is odd, the last 604 backslash escapes the next double quotation mark as 605 described in rule 3. 606 """ 607 608 # See 609 # http://msdn.microsoft.com/en-us/library/17w5ykft.aspx 610 # or search http://msdn.microsoft.com for 611 # "Parsing C++ Command-Line Arguments" 612 result = [] 613 needquote = False 614 for arg in map(os.fsdecode, seq): 615 bs_buf = [] 616 617 # Add a space to separate this argument from the others 618 if result: 619 result.append(' ') 620 621 needquote = (" " in arg) or ("\t" in arg) or not arg 622 if needquote: 623 result.append('"') 624 625 for c in arg: 626 if c == '\\': 627 # Don't know if we need to double yet. 628 bs_buf.append(c) 629 elif c == '"': 630 # Double backslashes. 631 result.append('\\' * len(bs_buf)*2) 632 bs_buf = [] 633 result.append('\\"') 634 else: 635 # Normal char 636 if bs_buf: 637 result.extend(bs_buf) 638 bs_buf = [] 639 result.append(c) 640 641 # Add remaining backslashes, if any. 642 if bs_buf: 643 result.extend(bs_buf) 644 645 if needquote: 646 result.extend(bs_buf) 647 result.append('"') 648 649 return ''.join(result) 650 651 652# Various tools for executing commands and looking at their output and status. 653# 654 655def getstatusoutput(cmd, *, encoding=None, errors=None): 656 """Return (exitcode, output) of executing cmd in a shell. 657 658 Execute the string 'cmd' in a shell with 'check_output' and 659 return a 2-tuple (status, output). The locale encoding is used 660 to decode the output and process newlines. 661 662 A trailing newline is stripped from the output. 663 The exit status for the command can be interpreted 664 according to the rules for the function 'wait'. Example: 665 666 >>> import subprocess 667 >>> subprocess.getstatusoutput('ls /bin/ls') 668 (0, '/bin/ls') 669 >>> subprocess.getstatusoutput('cat /bin/junk') 670 (1, 'cat: /bin/junk: No such file or directory') 671 >>> subprocess.getstatusoutput('/bin/junk') 672 (127, 'sh: /bin/junk: not found') 673 >>> subprocess.getstatusoutput('/bin/kill $$') 674 (-15, '') 675 """ 676 try: 677 data = check_output(cmd, shell=True, text=True, stderr=STDOUT, 678 encoding=encoding, errors=errors) 679 exitcode = 0 680 except CalledProcessError as ex: 681 data = ex.output 682 exitcode = ex.returncode 683 if data[-1:] == '\n': 684 data = data[:-1] 685 return exitcode, data 686 687def getoutput(cmd, *, encoding=None, errors=None): 688 """Return output (stdout or stderr) of executing cmd in a shell. 689 690 Like getstatusoutput(), except the exit status is ignored and the return 691 value is a string containing the command's output. Example: 692 693 >>> import subprocess 694 >>> subprocess.getoutput('ls /bin/ls') 695 '/bin/ls' 696 """ 697 return getstatusoutput(cmd, encoding=encoding, errors=errors)[1] 698 699 700 701def _use_posix_spawn(): 702 """Check if posix_spawn() can be used for subprocess. 703 704 subprocess requires a posix_spawn() implementation that properly reports 705 errors to the parent process, & sets errno on the following failures: 706 707 * Process attribute actions failed. 708 * File actions failed. 709 * exec() failed. 710 711 Prefer an implementation which can use vfork() in some cases for best 712 performance. 713 """ 714 if _mswindows or not hasattr(os, 'posix_spawn'): 715 # os.posix_spawn() is not available 716 return False 717 718 if sys.platform in ('darwin', 'sunos5'): 719 # posix_spawn() is a syscall on both macOS and Solaris, 720 # and properly reports errors 721 return True 722 723 # Check libc name and runtime libc version 724 try: 725 ver = os.confstr('CS_GNU_LIBC_VERSION') 726 # parse 'glibc 2.28' as ('glibc', (2, 28)) 727 parts = ver.split(maxsplit=1) 728 if len(parts) != 2: 729 # reject unknown format 730 raise ValueError 731 libc = parts[0] 732 version = tuple(map(int, parts[1].split('.'))) 733 734 if sys.platform == 'linux' and libc == 'glibc' and version >= (2, 24): 735 # glibc 2.24 has a new Linux posix_spawn implementation using vfork 736 # which properly reports errors to the parent process. 737 return True 738 # Note: Don't use the implementation in earlier glibc because it doesn't 739 # use vfork (even if glibc 2.26 added a pipe to properly report errors 740 # to the parent process). 741 except (AttributeError, ValueError, OSError): 742 # os.confstr() or CS_GNU_LIBC_VERSION value not available 743 pass 744 745 # By default, assume that posix_spawn() does not properly report errors. 746 return False 747 748 749# These are primarily fail-safe knobs for negatives. A True value does not 750# guarantee the given libc/syscall API will be used. 751_USE_POSIX_SPAWN = _use_posix_spawn() 752_USE_VFORK = True 753_HAVE_POSIX_SPAWN_CLOSEFROM = hasattr(os, 'POSIX_SPAWN_CLOSEFROM') 754 755 756class Popen: 757 """ Execute a child program in a new process. 758 759 For a complete description of the arguments see the Python documentation. 760 761 Arguments: 762 args: A string, or a sequence of program arguments. 763 764 bufsize: supplied as the buffering argument to the open() function when 765 creating the stdin/stdout/stderr pipe file objects 766 767 executable: A replacement program to execute. 768 769 stdin, stdout and stderr: These specify the executed programs' standard 770 input, standard output and standard error file handles, respectively. 771 772 preexec_fn: (POSIX only) An object to be called in the child process 773 just before the child is executed. 774 775 close_fds: Controls closing or inheriting of file descriptors. 776 777 shell: If true, the command will be executed through the shell. 778 779 cwd: Sets the current directory before the child is executed. 780 781 env: Defines the environment variables for the new process. 782 783 text: If true, decode stdin, stdout and stderr using the given encoding 784 (if set) or the system default otherwise. 785 786 universal_newlines: Alias of text, provided for backwards compatibility. 787 788 startupinfo and creationflags (Windows only) 789 790 restore_signals (POSIX only) 791 792 start_new_session (POSIX only) 793 794 process_group (POSIX only) 795 796 group (POSIX only) 797 798 extra_groups (POSIX only) 799 800 user (POSIX only) 801 802 umask (POSIX only) 803 804 pass_fds (POSIX only) 805 806 encoding and errors: Text mode encoding and error handling to use for 807 file objects stdin, stdout and stderr. 808 809 Attributes: 810 stdin, stdout, stderr, pid, returncode 811 """ 812 _child_created = False # Set here since __del__ checks it 813 814 def __init__(self, args, bufsize=-1, executable=None, 815 stdin=None, stdout=None, stderr=None, 816 preexec_fn=None, close_fds=True, 817 shell=False, cwd=None, env=None, universal_newlines=None, 818 startupinfo=None, creationflags=0, 819 restore_signals=True, start_new_session=False, 820 pass_fds=(), *, user=None, group=None, extra_groups=None, 821 encoding=None, errors=None, text=None, umask=-1, pipesize=-1, 822 process_group=None): 823 """Create new Popen instance.""" 824 if not _can_fork_exec: 825 raise OSError( 826 errno.ENOTSUP, f"{sys.platform} does not support processes." 827 ) 828 829 _cleanup() 830 # Held while anything is calling waitpid before returncode has been 831 # updated to prevent clobbering returncode if wait() or poll() are 832 # called from multiple threads at once. After acquiring the lock, 833 # code must re-check self.returncode to see if another thread just 834 # finished a waitpid() call. 835 self._waitpid_lock = threading.Lock() 836 837 self._input = None 838 self._communication_started = False 839 if bufsize is None: 840 bufsize = -1 # Restore default 841 if not isinstance(bufsize, int): 842 raise TypeError("bufsize must be an integer") 843 844 if stdout is STDOUT: 845 raise ValueError("STDOUT can only be used for stderr") 846 847 if pipesize is None: 848 pipesize = -1 # Restore default 849 if not isinstance(pipesize, int): 850 raise TypeError("pipesize must be an integer") 851 852 if _mswindows: 853 if preexec_fn is not None: 854 raise ValueError("preexec_fn is not supported on Windows " 855 "platforms") 856 else: 857 # POSIX 858 if pass_fds and not close_fds: 859 warnings.warn("pass_fds overriding close_fds.", RuntimeWarning) 860 close_fds = True 861 if startupinfo is not None: 862 raise ValueError("startupinfo is only supported on Windows " 863 "platforms") 864 if creationflags != 0: 865 raise ValueError("creationflags is only supported on Windows " 866 "platforms") 867 868 self.args = args 869 self.stdin = None 870 self.stdout = None 871 self.stderr = None 872 self.pid = None 873 self.returncode = None 874 self.encoding = encoding 875 self.errors = errors 876 self.pipesize = pipesize 877 878 # Validate the combinations of text and universal_newlines 879 if (text is not None and universal_newlines is not None 880 and bool(universal_newlines) != bool(text)): 881 raise SubprocessError('Cannot disambiguate when both text ' 882 'and universal_newlines are supplied but ' 883 'different. Pass one or the other.') 884 885 self.text_mode = encoding or errors or text or universal_newlines 886 if self.text_mode and encoding is None: 887 self.encoding = encoding = _text_encoding() 888 889 # How long to resume waiting on a child after the first ^C. 890 # There is no right value for this. The purpose is to be polite 891 # yet remain good for interactive users trying to exit a tool. 892 self._sigint_wait_secs = 0.25 # 1/xkcd221.getRandomNumber() 893 894 self._closed_child_pipe_fds = False 895 896 if self.text_mode: 897 if bufsize == 1: 898 line_buffering = True 899 # Use the default buffer size for the underlying binary streams 900 # since they don't support line buffering. 901 bufsize = -1 902 else: 903 line_buffering = False 904 905 if process_group is None: 906 process_group = -1 # The internal APIs are int-only 907 908 gid = None 909 if group is not None: 910 if not hasattr(os, 'setregid'): 911 raise ValueError("The 'group' parameter is not supported on the " 912 "current platform") 913 914 elif isinstance(group, str): 915 try: 916 import grp 917 except ImportError: 918 raise ValueError("The group parameter cannot be a string " 919 "on systems without the grp module") 920 921 gid = grp.getgrnam(group).gr_gid 922 elif isinstance(group, int): 923 gid = group 924 else: 925 raise TypeError("Group must be a string or an integer, not {}" 926 .format(type(group))) 927 928 if gid < 0: 929 raise ValueError(f"Group ID cannot be negative, got {gid}") 930 931 gids = None 932 if extra_groups is not None: 933 if not hasattr(os, 'setgroups'): 934 raise ValueError("The 'extra_groups' parameter is not " 935 "supported on the current platform") 936 937 elif isinstance(extra_groups, str): 938 raise ValueError("Groups must be a list, not a string") 939 940 gids = [] 941 for extra_group in extra_groups: 942 if isinstance(extra_group, str): 943 try: 944 import grp 945 except ImportError: 946 raise ValueError("Items in extra_groups cannot be " 947 "strings on systems without the " 948 "grp module") 949 950 gids.append(grp.getgrnam(extra_group).gr_gid) 951 elif isinstance(extra_group, int): 952 gids.append(extra_group) 953 else: 954 raise TypeError("Items in extra_groups must be a string " 955 "or integer, not {}" 956 .format(type(extra_group))) 957 958 # make sure that the gids are all positive here so we can do less 959 # checking in the C code 960 for gid_check in gids: 961 if gid_check < 0: 962 raise ValueError(f"Group ID cannot be negative, got {gid_check}") 963 964 uid = None 965 if user is not None: 966 if not hasattr(os, 'setreuid'): 967 raise ValueError("The 'user' parameter is not supported on " 968 "the current platform") 969 970 elif isinstance(user, str): 971 try: 972 import pwd 973 except ImportError: 974 raise ValueError("The user parameter cannot be a string " 975 "on systems without the pwd module") 976 uid = pwd.getpwnam(user).pw_uid 977 elif isinstance(user, int): 978 uid = user 979 else: 980 raise TypeError("User must be a string or an integer") 981 982 if uid < 0: 983 raise ValueError(f"User ID cannot be negative, got {uid}") 984 985 # Input and output objects. The general principle is like 986 # this: 987 # 988 # Parent Child 989 # ------ ----- 990 # p2cwrite ---stdin---> p2cread 991 # c2pread <--stdout--- c2pwrite 992 # errread <--stderr--- errwrite 993 # 994 # On POSIX, the child objects are file descriptors. On 995 # Windows, these are Windows file handles. The parent objects 996 # are file descriptors on both platforms. The parent objects 997 # are -1 when not using PIPEs. The child objects are -1 998 # when not redirecting. 999 1000 (p2cread, p2cwrite, 1001 c2pread, c2pwrite, 1002 errread, errwrite) = self._get_handles(stdin, stdout, stderr) 1003 1004 # From here on, raising exceptions may cause file descriptor leakage 1005 1006 # We wrap OS handles *before* launching the child, otherwise a 1007 # quickly terminating child could make our fds unwrappable 1008 # (see #8458). 1009 1010 if _mswindows: 1011 if p2cwrite != -1: 1012 p2cwrite = msvcrt.open_osfhandle(p2cwrite.Detach(), 0) 1013 if c2pread != -1: 1014 c2pread = msvcrt.open_osfhandle(c2pread.Detach(), 0) 1015 if errread != -1: 1016 errread = msvcrt.open_osfhandle(errread.Detach(), 0) 1017 1018 try: 1019 if p2cwrite != -1: 1020 self.stdin = io.open(p2cwrite, 'wb', bufsize) 1021 if self.text_mode: 1022 self.stdin = io.TextIOWrapper(self.stdin, write_through=True, 1023 line_buffering=line_buffering, 1024 encoding=encoding, errors=errors) 1025 if c2pread != -1: 1026 self.stdout = io.open(c2pread, 'rb', bufsize) 1027 if self.text_mode: 1028 self.stdout = io.TextIOWrapper(self.stdout, 1029 encoding=encoding, errors=errors) 1030 if errread != -1: 1031 self.stderr = io.open(errread, 'rb', bufsize) 1032 if self.text_mode: 1033 self.stderr = io.TextIOWrapper(self.stderr, 1034 encoding=encoding, errors=errors) 1035 1036 self._execute_child(args, executable, preexec_fn, close_fds, 1037 pass_fds, cwd, env, 1038 startupinfo, creationflags, shell, 1039 p2cread, p2cwrite, 1040 c2pread, c2pwrite, 1041 errread, errwrite, 1042 restore_signals, 1043 gid, gids, uid, umask, 1044 start_new_session, process_group) 1045 except: 1046 # Cleanup if the child failed starting. 1047 for f in filter(None, (self.stdin, self.stdout, self.stderr)): 1048 try: 1049 f.close() 1050 except OSError: 1051 pass # Ignore EBADF or other errors. 1052 1053 if not self._closed_child_pipe_fds: 1054 to_close = [] 1055 if stdin == PIPE: 1056 to_close.append(p2cread) 1057 if stdout == PIPE: 1058 to_close.append(c2pwrite) 1059 if stderr == PIPE: 1060 to_close.append(errwrite) 1061 if hasattr(self, '_devnull'): 1062 to_close.append(self._devnull) 1063 for fd in to_close: 1064 try: 1065 if _mswindows and isinstance(fd, Handle): 1066 fd.Close() 1067 else: 1068 os.close(fd) 1069 except OSError: 1070 pass 1071 1072 raise 1073 1074 def __repr__(self): 1075 obj_repr = ( 1076 f"<{self.__class__.__name__}: " 1077 f"returncode: {self.returncode} args: {self.args!r}>" 1078 ) 1079 if len(obj_repr) > 80: 1080 obj_repr = obj_repr[:76] + "...>" 1081 return obj_repr 1082 1083 __class_getitem__ = classmethod(types.GenericAlias) 1084 1085 @property 1086 def universal_newlines(self): 1087 # universal_newlines as retained as an alias of text_mode for API 1088 # compatibility. bpo-31756 1089 return self.text_mode 1090 1091 @universal_newlines.setter 1092 def universal_newlines(self, universal_newlines): 1093 self.text_mode = bool(universal_newlines) 1094 1095 def _translate_newlines(self, data, encoding, errors): 1096 data = data.decode(encoding, errors) 1097 return data.replace("\r\n", "\n").replace("\r", "\n") 1098 1099 def __enter__(self): 1100 return self 1101 1102 def __exit__(self, exc_type, value, traceback): 1103 if self.stdout: 1104 self.stdout.close() 1105 if self.stderr: 1106 self.stderr.close() 1107 try: # Flushing a BufferedWriter may raise an error 1108 if self.stdin: 1109 self.stdin.close() 1110 finally: 1111 if exc_type == KeyboardInterrupt: 1112 # https://bugs.python.org/issue25942 1113 # In the case of a KeyboardInterrupt we assume the SIGINT 1114 # was also already sent to our child processes. We can't 1115 # block indefinitely as that is not user friendly. 1116 # If we have not already waited a brief amount of time in 1117 # an interrupted .wait() or .communicate() call, do so here 1118 # for consistency. 1119 if self._sigint_wait_secs > 0: 1120 try: 1121 self._wait(timeout=self._sigint_wait_secs) 1122 except TimeoutExpired: 1123 pass 1124 self._sigint_wait_secs = 0 # Note that this has been done. 1125 return # resume the KeyboardInterrupt 1126 1127 # Wait for the process to terminate, to avoid zombies. 1128 self.wait() 1129 1130 def __del__(self, _maxsize=sys.maxsize, _warn=warnings.warn): 1131 if not self._child_created: 1132 # We didn't get to successfully create a child process. 1133 return 1134 if self.returncode is None: 1135 # Not reading subprocess exit status creates a zombie process which 1136 # is only destroyed at the parent python process exit 1137 _warn("subprocess %s is still running" % self.pid, 1138 ResourceWarning, source=self) 1139 # In case the child hasn't been waited on, check if it's done. 1140 self._internal_poll(_deadstate=_maxsize) 1141 if self.returncode is None and _active is not None: 1142 # Child is still running, keep us alive until we can wait on it. 1143 _active.append(self) 1144 1145 def _get_devnull(self): 1146 if not hasattr(self, '_devnull'): 1147 self._devnull = os.open(os.devnull, os.O_RDWR) 1148 return self._devnull 1149 1150 def _stdin_write(self, input): 1151 if input: 1152 try: 1153 self.stdin.write(input) 1154 except BrokenPipeError: 1155 pass # communicate() must ignore broken pipe errors. 1156 except OSError as exc: 1157 if exc.errno == errno.EINVAL: 1158 # bpo-19612, bpo-30418: On Windows, stdin.write() fails 1159 # with EINVAL if the child process exited or if the child 1160 # process is still running but closed the pipe. 1161 pass 1162 else: 1163 raise 1164 1165 try: 1166 self.stdin.close() 1167 except BrokenPipeError: 1168 pass # communicate() must ignore broken pipe errors. 1169 except OSError as exc: 1170 if exc.errno == errno.EINVAL: 1171 pass 1172 else: 1173 raise 1174 1175 def communicate(self, input=None, timeout=None): 1176 """Interact with process: Send data to stdin and close it. 1177 Read data from stdout and stderr, until end-of-file is 1178 reached. Wait for process to terminate. 1179 1180 The optional "input" argument should be data to be sent to the 1181 child process, or None, if no data should be sent to the child. 1182 communicate() returns a tuple (stdout, stderr). 1183 1184 By default, all communication is in bytes, and therefore any 1185 "input" should be bytes, and the (stdout, stderr) will be bytes. 1186 If in text mode (indicated by self.text_mode), any "input" should 1187 be a string, and (stdout, stderr) will be strings decoded 1188 according to locale encoding, or by "encoding" if set. Text mode 1189 is triggered by setting any of text, encoding, errors or 1190 universal_newlines. 1191 """ 1192 1193 if self._communication_started and input: 1194 raise ValueError("Cannot send input after starting communication") 1195 1196 # Optimization: If we are not worried about timeouts, we haven't 1197 # started communicating, and we have one or zero pipes, using select() 1198 # or threads is unnecessary. 1199 if (timeout is None and not self._communication_started and 1200 [self.stdin, self.stdout, self.stderr].count(None) >= 2): 1201 stdout = None 1202 stderr = None 1203 if self.stdin: 1204 self._stdin_write(input) 1205 elif self.stdout: 1206 stdout = self.stdout.read() 1207 self.stdout.close() 1208 elif self.stderr: 1209 stderr = self.stderr.read() 1210 self.stderr.close() 1211 self.wait() 1212 else: 1213 if timeout is not None: 1214 endtime = _time() + timeout 1215 else: 1216 endtime = None 1217 1218 try: 1219 stdout, stderr = self._communicate(input, endtime, timeout) 1220 except KeyboardInterrupt: 1221 # https://bugs.python.org/issue25942 1222 # See the detailed comment in .wait(). 1223 if timeout is not None: 1224 sigint_timeout = min(self._sigint_wait_secs, 1225 self._remaining_time(endtime)) 1226 else: 1227 sigint_timeout = self._sigint_wait_secs 1228 self._sigint_wait_secs = 0 # nothing else should wait. 1229 try: 1230 self._wait(timeout=sigint_timeout) 1231 except TimeoutExpired: 1232 pass 1233 raise # resume the KeyboardInterrupt 1234 1235 finally: 1236 self._communication_started = True 1237 1238 sts = self.wait(timeout=self._remaining_time(endtime)) 1239 1240 return (stdout, stderr) 1241 1242 1243 def poll(self): 1244 """Check if child process has terminated. Set and return returncode 1245 attribute.""" 1246 return self._internal_poll() 1247 1248 1249 def _remaining_time(self, endtime): 1250 """Convenience for _communicate when computing timeouts.""" 1251 if endtime is None: 1252 return None 1253 else: 1254 return endtime - _time() 1255 1256 1257 def _check_timeout(self, endtime, orig_timeout, stdout_seq, stderr_seq, 1258 skip_check_and_raise=False): 1259 """Convenience for checking if a timeout has expired.""" 1260 if endtime is None: 1261 return 1262 if skip_check_and_raise or _time() > endtime: 1263 raise TimeoutExpired( 1264 self.args, orig_timeout, 1265 output=b''.join(stdout_seq) if stdout_seq else None, 1266 stderr=b''.join(stderr_seq) if stderr_seq else None) 1267 1268 1269 def wait(self, timeout=None): 1270 """Wait for child process to terminate; returns self.returncode.""" 1271 if timeout is not None: 1272 endtime = _time() + timeout 1273 try: 1274 return self._wait(timeout=timeout) 1275 except KeyboardInterrupt: 1276 # https://bugs.python.org/issue25942 1277 # The first keyboard interrupt waits briefly for the child to 1278 # exit under the common assumption that it also received the ^C 1279 # generated SIGINT and will exit rapidly. 1280 if timeout is not None: 1281 sigint_timeout = min(self._sigint_wait_secs, 1282 self._remaining_time(endtime)) 1283 else: 1284 sigint_timeout = self._sigint_wait_secs 1285 self._sigint_wait_secs = 0 # nothing else should wait. 1286 try: 1287 self._wait(timeout=sigint_timeout) 1288 except TimeoutExpired: 1289 pass 1290 raise # resume the KeyboardInterrupt 1291 1292 def _close_pipe_fds(self, 1293 p2cread, p2cwrite, 1294 c2pread, c2pwrite, 1295 errread, errwrite): 1296 # self._devnull is not always defined. 1297 devnull_fd = getattr(self, '_devnull', None) 1298 1299 with contextlib.ExitStack() as stack: 1300 if _mswindows: 1301 if p2cread != -1: 1302 stack.callback(p2cread.Close) 1303 if c2pwrite != -1: 1304 stack.callback(c2pwrite.Close) 1305 if errwrite != -1: 1306 stack.callback(errwrite.Close) 1307 else: 1308 if p2cread != -1 and p2cwrite != -1 and p2cread != devnull_fd: 1309 stack.callback(os.close, p2cread) 1310 if c2pwrite != -1 and c2pread != -1 and c2pwrite != devnull_fd: 1311 stack.callback(os.close, c2pwrite) 1312 if errwrite != -1 and errread != -1 and errwrite != devnull_fd: 1313 stack.callback(os.close, errwrite) 1314 1315 if devnull_fd is not None: 1316 stack.callback(os.close, devnull_fd) 1317 1318 # Prevent a double close of these handles/fds from __init__ on error. 1319 self._closed_child_pipe_fds = True 1320 1321 @contextlib.contextmanager 1322 def _on_error_fd_closer(self): 1323 """Helper to ensure file descriptors opened in _get_handles are closed""" 1324 to_close = [] 1325 try: 1326 yield to_close 1327 except: 1328 if hasattr(self, '_devnull'): 1329 to_close.append(self._devnull) 1330 del self._devnull 1331 for fd in to_close: 1332 try: 1333 if _mswindows and isinstance(fd, Handle): 1334 fd.Close() 1335 else: 1336 os.close(fd) 1337 except OSError: 1338 pass 1339 raise 1340 1341 if _mswindows: 1342 # 1343 # Windows methods 1344 # 1345 def _get_handles(self, stdin, stdout, stderr): 1346 """Construct and return tuple with IO objects: 1347 p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite 1348 """ 1349 if stdin is None and stdout is None and stderr is None: 1350 return (-1, -1, -1, -1, -1, -1) 1351 1352 p2cread, p2cwrite = -1, -1 1353 c2pread, c2pwrite = -1, -1 1354 errread, errwrite = -1, -1 1355 1356 with self._on_error_fd_closer() as err_close_fds: 1357 if stdin is None: 1358 p2cread = _winapi.GetStdHandle(_winapi.STD_INPUT_HANDLE) 1359 if p2cread is None: 1360 p2cread, _ = _winapi.CreatePipe(None, 0) 1361 p2cread = Handle(p2cread) 1362 err_close_fds.append(p2cread) 1363 _winapi.CloseHandle(_) 1364 elif stdin == PIPE: 1365 p2cread, p2cwrite = _winapi.CreatePipe(None, 0) 1366 p2cread, p2cwrite = Handle(p2cread), Handle(p2cwrite) 1367 err_close_fds.extend((p2cread, p2cwrite)) 1368 elif stdin == DEVNULL: 1369 p2cread = msvcrt.get_osfhandle(self._get_devnull()) 1370 elif isinstance(stdin, int): 1371 p2cread = msvcrt.get_osfhandle(stdin) 1372 else: 1373 # Assuming file-like object 1374 p2cread = msvcrt.get_osfhandle(stdin.fileno()) 1375 p2cread = self._make_inheritable(p2cread) 1376 1377 if stdout is None: 1378 c2pwrite = _winapi.GetStdHandle(_winapi.STD_OUTPUT_HANDLE) 1379 if c2pwrite is None: 1380 _, c2pwrite = _winapi.CreatePipe(None, 0) 1381 c2pwrite = Handle(c2pwrite) 1382 err_close_fds.append(c2pwrite) 1383 _winapi.CloseHandle(_) 1384 elif stdout == PIPE: 1385 c2pread, c2pwrite = _winapi.CreatePipe(None, 0) 1386 c2pread, c2pwrite = Handle(c2pread), Handle(c2pwrite) 1387 err_close_fds.extend((c2pread, c2pwrite)) 1388 elif stdout == DEVNULL: 1389 c2pwrite = msvcrt.get_osfhandle(self._get_devnull()) 1390 elif isinstance(stdout, int): 1391 c2pwrite = msvcrt.get_osfhandle(stdout) 1392 else: 1393 # Assuming file-like object 1394 c2pwrite = msvcrt.get_osfhandle(stdout.fileno()) 1395 c2pwrite = self._make_inheritable(c2pwrite) 1396 1397 if stderr is None: 1398 errwrite = _winapi.GetStdHandle(_winapi.STD_ERROR_HANDLE) 1399 if errwrite is None: 1400 _, errwrite = _winapi.CreatePipe(None, 0) 1401 errwrite = Handle(errwrite) 1402 err_close_fds.append(errwrite) 1403 _winapi.CloseHandle(_) 1404 elif stderr == PIPE: 1405 errread, errwrite = _winapi.CreatePipe(None, 0) 1406 errread, errwrite = Handle(errread), Handle(errwrite) 1407 err_close_fds.extend((errread, errwrite)) 1408 elif stderr == STDOUT: 1409 errwrite = c2pwrite 1410 elif stderr == DEVNULL: 1411 errwrite = msvcrt.get_osfhandle(self._get_devnull()) 1412 elif isinstance(stderr, int): 1413 errwrite = msvcrt.get_osfhandle(stderr) 1414 else: 1415 # Assuming file-like object 1416 errwrite = msvcrt.get_osfhandle(stderr.fileno()) 1417 errwrite = self._make_inheritable(errwrite) 1418 1419 return (p2cread, p2cwrite, 1420 c2pread, c2pwrite, 1421 errread, errwrite) 1422 1423 1424 def _make_inheritable(self, handle): 1425 """Return a duplicate of handle, which is inheritable""" 1426 h = _winapi.DuplicateHandle( 1427 _winapi.GetCurrentProcess(), handle, 1428 _winapi.GetCurrentProcess(), 0, 1, 1429 _winapi.DUPLICATE_SAME_ACCESS) 1430 return Handle(h) 1431 1432 1433 def _filter_handle_list(self, handle_list): 1434 """Filter out console handles that can't be used 1435 in lpAttributeList["handle_list"] and make sure the list 1436 isn't empty. This also removes duplicate handles.""" 1437 # An handle with it's lowest two bits set might be a special console 1438 # handle that if passed in lpAttributeList["handle_list"], will 1439 # cause it to fail. 1440 return list({handle for handle in handle_list 1441 if handle & 0x3 != 0x3 1442 or _winapi.GetFileType(handle) != 1443 _winapi.FILE_TYPE_CHAR}) 1444 1445 1446 def _execute_child(self, args, executable, preexec_fn, close_fds, 1447 pass_fds, cwd, env, 1448 startupinfo, creationflags, shell, 1449 p2cread, p2cwrite, 1450 c2pread, c2pwrite, 1451 errread, errwrite, 1452 unused_restore_signals, 1453 unused_gid, unused_gids, unused_uid, 1454 unused_umask, 1455 unused_start_new_session, unused_process_group): 1456 """Execute program (MS Windows version)""" 1457 1458 assert not pass_fds, "pass_fds not supported on Windows." 1459 1460 if isinstance(args, str): 1461 pass 1462 elif isinstance(args, bytes): 1463 if shell: 1464 raise TypeError('bytes args is not allowed on Windows') 1465 args = list2cmdline([args]) 1466 elif isinstance(args, os.PathLike): 1467 if shell: 1468 raise TypeError('path-like args is not allowed when ' 1469 'shell is true') 1470 args = list2cmdline([args]) 1471 else: 1472 args = list2cmdline(args) 1473 1474 if executable is not None: 1475 executable = os.fsdecode(executable) 1476 1477 # Process startup details 1478 if startupinfo is None: 1479 startupinfo = STARTUPINFO() 1480 else: 1481 # bpo-34044: Copy STARTUPINFO since it is modified above, 1482 # so the caller can reuse it multiple times. 1483 startupinfo = startupinfo.copy() 1484 1485 use_std_handles = -1 not in (p2cread, c2pwrite, errwrite) 1486 if use_std_handles: 1487 startupinfo.dwFlags |= _winapi.STARTF_USESTDHANDLES 1488 startupinfo.hStdInput = p2cread 1489 startupinfo.hStdOutput = c2pwrite 1490 startupinfo.hStdError = errwrite 1491 1492 attribute_list = startupinfo.lpAttributeList 1493 have_handle_list = bool(attribute_list and 1494 "handle_list" in attribute_list and 1495 attribute_list["handle_list"]) 1496 1497 # If we were given an handle_list or need to create one 1498 if have_handle_list or (use_std_handles and close_fds): 1499 if attribute_list is None: 1500 attribute_list = startupinfo.lpAttributeList = {} 1501 handle_list = attribute_list["handle_list"] = \ 1502 list(attribute_list.get("handle_list", [])) 1503 1504 if use_std_handles: 1505 handle_list += [int(p2cread), int(c2pwrite), int(errwrite)] 1506 1507 handle_list[:] = self._filter_handle_list(handle_list) 1508 1509 if handle_list: 1510 if not close_fds: 1511 warnings.warn("startupinfo.lpAttributeList['handle_list'] " 1512 "overriding close_fds", RuntimeWarning) 1513 1514 # When using the handle_list we always request to inherit 1515 # handles but the only handles that will be inherited are 1516 # the ones in the handle_list 1517 close_fds = False 1518 1519 if shell: 1520 startupinfo.dwFlags |= _winapi.STARTF_USESHOWWINDOW 1521 startupinfo.wShowWindow = _winapi.SW_HIDE 1522 if not executable: 1523 # gh-101283: without a fully-qualified path, before Windows 1524 # checks the system directories, it first looks in the 1525 # application directory, and also the current directory if 1526 # NeedCurrentDirectoryForExePathW(ExeName) is true, so try 1527 # to avoid executing unqualified "cmd.exe". 1528 comspec = os.environ.get('ComSpec') 1529 if not comspec: 1530 system_root = os.environ.get('SystemRoot', '') 1531 comspec = os.path.join(system_root, 'System32', 'cmd.exe') 1532 if not os.path.isabs(comspec): 1533 raise FileNotFoundError('shell not found: neither %ComSpec% nor %SystemRoot% is set') 1534 if os.path.isabs(comspec): 1535 executable = comspec 1536 else: 1537 comspec = executable 1538 1539 args = '{} /c "{}"'.format (comspec, args) 1540 1541 if cwd is not None: 1542 cwd = os.fsdecode(cwd) 1543 1544 sys.audit("subprocess.Popen", executable, args, cwd, env) 1545 1546 # Start the process 1547 try: 1548 hp, ht, pid, tid = _winapi.CreateProcess(executable, args, 1549 # no special security 1550 None, None, 1551 int(not close_fds), 1552 creationflags, 1553 env, 1554 cwd, 1555 startupinfo) 1556 finally: 1557 # Child is launched. Close the parent's copy of those pipe 1558 # handles that only the child should have open. You need 1559 # to make sure that no handles to the write end of the 1560 # output pipe are maintained in this process or else the 1561 # pipe will not close when the child process exits and the 1562 # ReadFile will hang. 1563 self._close_pipe_fds(p2cread, p2cwrite, 1564 c2pread, c2pwrite, 1565 errread, errwrite) 1566 1567 # Retain the process handle, but close the thread handle 1568 self._child_created = True 1569 self._handle = Handle(hp) 1570 self.pid = pid 1571 _winapi.CloseHandle(ht) 1572 1573 def _internal_poll(self, _deadstate=None, 1574 _WaitForSingleObject=_winapi.WaitForSingleObject, 1575 _WAIT_OBJECT_0=_winapi.WAIT_OBJECT_0, 1576 _GetExitCodeProcess=_winapi.GetExitCodeProcess): 1577 """Check if child process has terminated. Returns returncode 1578 attribute. 1579 1580 This method is called by __del__, so it can only refer to objects 1581 in its local scope. 1582 1583 """ 1584 if self.returncode is None: 1585 if _WaitForSingleObject(self._handle, 0) == _WAIT_OBJECT_0: 1586 self.returncode = _GetExitCodeProcess(self._handle) 1587 return self.returncode 1588 1589 1590 def _wait(self, timeout): 1591 """Internal implementation of wait() on Windows.""" 1592 if timeout is None: 1593 timeout_millis = _winapi.INFINITE 1594 elif timeout <= 0: 1595 timeout_millis = 0 1596 else: 1597 timeout_millis = int(timeout * 1000) 1598 if self.returncode is None: 1599 # API note: Returns immediately if timeout_millis == 0. 1600 result = _winapi.WaitForSingleObject(self._handle, 1601 timeout_millis) 1602 if result == _winapi.WAIT_TIMEOUT: 1603 raise TimeoutExpired(self.args, timeout) 1604 self.returncode = _winapi.GetExitCodeProcess(self._handle) 1605 return self.returncode 1606 1607 1608 def _readerthread(self, fh, buffer): 1609 buffer.append(fh.read()) 1610 fh.close() 1611 1612 1613 def _communicate(self, input, endtime, orig_timeout): 1614 # Start reader threads feeding into a list hanging off of this 1615 # object, unless they've already been started. 1616 if self.stdout and not hasattr(self, "_stdout_buff"): 1617 self._stdout_buff = [] 1618 self.stdout_thread = \ 1619 threading.Thread(target=self._readerthread, 1620 args=(self.stdout, self._stdout_buff)) 1621 self.stdout_thread.daemon = True 1622 self.stdout_thread.start() 1623 if self.stderr and not hasattr(self, "_stderr_buff"): 1624 self._stderr_buff = [] 1625 self.stderr_thread = \ 1626 threading.Thread(target=self._readerthread, 1627 args=(self.stderr, self._stderr_buff)) 1628 self.stderr_thread.daemon = True 1629 self.stderr_thread.start() 1630 1631 if self.stdin: 1632 self._stdin_write(input) 1633 1634 # Wait for the reader threads, or time out. If we time out, the 1635 # threads remain reading and the fds left open in case the user 1636 # calls communicate again. 1637 if self.stdout is not None: 1638 self.stdout_thread.join(self._remaining_time(endtime)) 1639 if self.stdout_thread.is_alive(): 1640 raise TimeoutExpired(self.args, orig_timeout) 1641 if self.stderr is not None: 1642 self.stderr_thread.join(self._remaining_time(endtime)) 1643 if self.stderr_thread.is_alive(): 1644 raise TimeoutExpired(self.args, orig_timeout) 1645 1646 # Collect the output from and close both pipes, now that we know 1647 # both have been read successfully. 1648 stdout = None 1649 stderr = None 1650 if self.stdout: 1651 stdout = self._stdout_buff 1652 self.stdout.close() 1653 if self.stderr: 1654 stderr = self._stderr_buff 1655 self.stderr.close() 1656 1657 # All data exchanged. Translate lists into strings. 1658 stdout = stdout[0] if stdout else None 1659 stderr = stderr[0] if stderr else None 1660 1661 return (stdout, stderr) 1662 1663 def send_signal(self, sig): 1664 """Send a signal to the process.""" 1665 # Don't signal a process that we know has already died. 1666 if self.returncode is not None: 1667 return 1668 if sig == signal.SIGTERM: 1669 self.terminate() 1670 elif sig == signal.CTRL_C_EVENT: 1671 os.kill(self.pid, signal.CTRL_C_EVENT) 1672 elif sig == signal.CTRL_BREAK_EVENT: 1673 os.kill(self.pid, signal.CTRL_BREAK_EVENT) 1674 else: 1675 raise ValueError("Unsupported signal: {}".format(sig)) 1676 1677 def terminate(self): 1678 """Terminates the process.""" 1679 # Don't terminate a process that we know has already died. 1680 if self.returncode is not None: 1681 return 1682 try: 1683 _winapi.TerminateProcess(self._handle, 1) 1684 except PermissionError: 1685 # ERROR_ACCESS_DENIED (winerror 5) is received when the 1686 # process already died. 1687 rc = _winapi.GetExitCodeProcess(self._handle) 1688 if rc == _winapi.STILL_ACTIVE: 1689 raise 1690 self.returncode = rc 1691 1692 kill = terminate 1693 1694 else: 1695 # 1696 # POSIX methods 1697 # 1698 def _get_handles(self, stdin, stdout, stderr): 1699 """Construct and return tuple with IO objects: 1700 p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite 1701 """ 1702 p2cread, p2cwrite = -1, -1 1703 c2pread, c2pwrite = -1, -1 1704 errread, errwrite = -1, -1 1705 1706 with self._on_error_fd_closer() as err_close_fds: 1707 if stdin is None: 1708 pass 1709 elif stdin == PIPE: 1710 p2cread, p2cwrite = os.pipe() 1711 err_close_fds.extend((p2cread, p2cwrite)) 1712 if self.pipesize > 0 and hasattr(fcntl, "F_SETPIPE_SZ"): 1713 fcntl.fcntl(p2cwrite, fcntl.F_SETPIPE_SZ, self.pipesize) 1714 elif stdin == DEVNULL: 1715 p2cread = self._get_devnull() 1716 elif isinstance(stdin, int): 1717 p2cread = stdin 1718 else: 1719 # Assuming file-like object 1720 p2cread = stdin.fileno() 1721 1722 if stdout is None: 1723 pass 1724 elif stdout == PIPE: 1725 c2pread, c2pwrite = os.pipe() 1726 err_close_fds.extend((c2pread, c2pwrite)) 1727 if self.pipesize > 0 and hasattr(fcntl, "F_SETPIPE_SZ"): 1728 fcntl.fcntl(c2pwrite, fcntl.F_SETPIPE_SZ, self.pipesize) 1729 elif stdout == DEVNULL: 1730 c2pwrite = self._get_devnull() 1731 elif isinstance(stdout, int): 1732 c2pwrite = stdout 1733 else: 1734 # Assuming file-like object 1735 c2pwrite = stdout.fileno() 1736 1737 if stderr is None: 1738 pass 1739 elif stderr == PIPE: 1740 errread, errwrite = os.pipe() 1741 err_close_fds.extend((errread, errwrite)) 1742 if self.pipesize > 0 and hasattr(fcntl, "F_SETPIPE_SZ"): 1743 fcntl.fcntl(errwrite, fcntl.F_SETPIPE_SZ, self.pipesize) 1744 elif stderr == STDOUT: 1745 if c2pwrite != -1: 1746 errwrite = c2pwrite 1747 else: # child's stdout is not set, use parent's stdout 1748 errwrite = sys.__stdout__.fileno() 1749 elif stderr == DEVNULL: 1750 errwrite = self._get_devnull() 1751 elif isinstance(stderr, int): 1752 errwrite = stderr 1753 else: 1754 # Assuming file-like object 1755 errwrite = stderr.fileno() 1756 1757 return (p2cread, p2cwrite, 1758 c2pread, c2pwrite, 1759 errread, errwrite) 1760 1761 1762 def _posix_spawn(self, args, executable, env, restore_signals, close_fds, 1763 p2cread, p2cwrite, 1764 c2pread, c2pwrite, 1765 errread, errwrite): 1766 """Execute program using os.posix_spawn().""" 1767 kwargs = {} 1768 if restore_signals: 1769 # See _Py_RestoreSignals() in Python/pylifecycle.c 1770 sigset = [] 1771 for signame in ('SIGPIPE', 'SIGXFZ', 'SIGXFSZ'): 1772 signum = getattr(signal, signame, None) 1773 if signum is not None: 1774 sigset.append(signum) 1775 kwargs['setsigdef'] = sigset 1776 1777 file_actions = [] 1778 for fd in (p2cwrite, c2pread, errread): 1779 if fd != -1: 1780 file_actions.append((os.POSIX_SPAWN_CLOSE, fd)) 1781 for fd, fd2 in ( 1782 (p2cread, 0), 1783 (c2pwrite, 1), 1784 (errwrite, 2), 1785 ): 1786 if fd != -1: 1787 file_actions.append((os.POSIX_SPAWN_DUP2, fd, fd2)) 1788 1789 if close_fds: 1790 file_actions.append((os.POSIX_SPAWN_CLOSEFROM, 3)) 1791 1792 if file_actions: 1793 kwargs['file_actions'] = file_actions 1794 1795 self.pid = os.posix_spawn(executable, args, env, **kwargs) 1796 self._child_created = True 1797 1798 self._close_pipe_fds(p2cread, p2cwrite, 1799 c2pread, c2pwrite, 1800 errread, errwrite) 1801 1802 def _execute_child(self, args, executable, preexec_fn, close_fds, 1803 pass_fds, cwd, env, 1804 startupinfo, creationflags, shell, 1805 p2cread, p2cwrite, 1806 c2pread, c2pwrite, 1807 errread, errwrite, 1808 restore_signals, 1809 gid, gids, uid, umask, 1810 start_new_session, process_group): 1811 """Execute program (POSIX version)""" 1812 1813 if isinstance(args, (str, bytes)): 1814 args = [args] 1815 elif isinstance(args, os.PathLike): 1816 if shell: 1817 raise TypeError('path-like args is not allowed when ' 1818 'shell is true') 1819 args = [args] 1820 else: 1821 args = list(args) 1822 1823 if shell: 1824 # On Android the default shell is at '/system/bin/sh'. 1825 unix_shell = ('/system/bin/sh' if 1826 hasattr(sys, 'getandroidapilevel') else '/bin/sh') 1827 args = [unix_shell, "-c"] + args 1828 if executable: 1829 args[0] = executable 1830 1831 if executable is None: 1832 executable = args[0] 1833 1834 sys.audit("subprocess.Popen", executable, args, cwd, env) 1835 1836 if (_USE_POSIX_SPAWN 1837 and os.path.dirname(executable) 1838 and preexec_fn is None 1839 and (not close_fds or _HAVE_POSIX_SPAWN_CLOSEFROM) 1840 and not pass_fds 1841 and cwd is None 1842 and (p2cread == -1 or p2cread > 2) 1843 and (c2pwrite == -1 or c2pwrite > 2) 1844 and (errwrite == -1 or errwrite > 2) 1845 and not start_new_session 1846 and process_group == -1 1847 and gid is None 1848 and gids is None 1849 and uid is None 1850 and umask < 0): 1851 self._posix_spawn(args, executable, env, restore_signals, close_fds, 1852 p2cread, p2cwrite, 1853 c2pread, c2pwrite, 1854 errread, errwrite) 1855 return 1856 1857 orig_executable = executable 1858 1859 # For transferring possible exec failure from child to parent. 1860 # Data format: "exception name:hex errno:description" 1861 # Pickle is not used; it is complex and involves memory allocation. 1862 errpipe_read, errpipe_write = os.pipe() 1863 # errpipe_write must not be in the standard io 0, 1, or 2 fd range. 1864 low_fds_to_close = [] 1865 while errpipe_write < 3: 1866 low_fds_to_close.append(errpipe_write) 1867 errpipe_write = os.dup(errpipe_write) 1868 for low_fd in low_fds_to_close: 1869 os.close(low_fd) 1870 try: 1871 try: 1872 # We must avoid complex work that could involve 1873 # malloc or free in the child process to avoid 1874 # potential deadlocks, thus we do all this here. 1875 # and pass it to fork_exec() 1876 1877 if env is not None: 1878 env_list = [] 1879 for k, v in env.items(): 1880 k = os.fsencode(k) 1881 if b'=' in k: 1882 raise ValueError("illegal environment variable name") 1883 env_list.append(k + b'=' + os.fsencode(v)) 1884 else: 1885 env_list = None # Use execv instead of execve. 1886 executable = os.fsencode(executable) 1887 if os.path.dirname(executable): 1888 executable_list = (executable,) 1889 else: 1890 # This matches the behavior of os._execvpe(). 1891 executable_list = tuple( 1892 os.path.join(os.fsencode(dir), executable) 1893 for dir in os.get_exec_path(env)) 1894 fds_to_keep = set(pass_fds) 1895 fds_to_keep.add(errpipe_write) 1896 self.pid = _fork_exec( 1897 args, executable_list, 1898 close_fds, tuple(sorted(map(int, fds_to_keep))), 1899 cwd, env_list, 1900 p2cread, p2cwrite, c2pread, c2pwrite, 1901 errread, errwrite, 1902 errpipe_read, errpipe_write, 1903 restore_signals, start_new_session, 1904 process_group, gid, gids, uid, umask, 1905 preexec_fn, _USE_VFORK) 1906 self._child_created = True 1907 finally: 1908 # be sure the FD is closed no matter what 1909 os.close(errpipe_write) 1910 1911 self._close_pipe_fds(p2cread, p2cwrite, 1912 c2pread, c2pwrite, 1913 errread, errwrite) 1914 1915 # Wait for exec to fail or succeed; possibly raising an 1916 # exception (limited in size) 1917 errpipe_data = bytearray() 1918 while True: 1919 part = os.read(errpipe_read, 50000) 1920 errpipe_data += part 1921 if not part or len(errpipe_data) > 50000: 1922 break 1923 finally: 1924 # be sure the FD is closed no matter what 1925 os.close(errpipe_read) 1926 1927 if errpipe_data: 1928 try: 1929 pid, sts = os.waitpid(self.pid, 0) 1930 if pid == self.pid: 1931 self._handle_exitstatus(sts) 1932 else: 1933 self.returncode = sys.maxsize 1934 except ChildProcessError: 1935 pass 1936 1937 try: 1938 exception_name, hex_errno, err_msg = ( 1939 errpipe_data.split(b':', 2)) 1940 # The encoding here should match the encoding 1941 # written in by the subprocess implementations 1942 # like _posixsubprocess 1943 err_msg = err_msg.decode() 1944 except ValueError: 1945 exception_name = b'SubprocessError' 1946 hex_errno = b'0' 1947 err_msg = 'Bad exception data from child: {!r}'.format( 1948 bytes(errpipe_data)) 1949 child_exception_type = getattr( 1950 builtins, exception_name.decode('ascii'), 1951 SubprocessError) 1952 if issubclass(child_exception_type, OSError) and hex_errno: 1953 errno_num = int(hex_errno, 16) 1954 if err_msg == "noexec:chdir": 1955 err_msg = "" 1956 # The error must be from chdir(cwd). 1957 err_filename = cwd 1958 elif err_msg == "noexec": 1959 err_msg = "" 1960 err_filename = None 1961 else: 1962 err_filename = orig_executable 1963 if errno_num != 0: 1964 err_msg = os.strerror(errno_num) 1965 if err_filename is not None: 1966 raise child_exception_type(errno_num, err_msg, err_filename) 1967 else: 1968 raise child_exception_type(errno_num, err_msg) 1969 raise child_exception_type(err_msg) 1970 1971 1972 def _handle_exitstatus(self, sts, _del_safe=_del_safe): 1973 """All callers to this function MUST hold self._waitpid_lock.""" 1974 # This method is called (indirectly) by __del__, so it cannot 1975 # refer to anything outside of its local scope. 1976 if _del_safe.WIFSTOPPED(sts): 1977 self.returncode = -_del_safe.WSTOPSIG(sts) 1978 else: 1979 self.returncode = _del_safe.waitstatus_to_exitcode(sts) 1980 1981 def _internal_poll(self, _deadstate=None, _del_safe=_del_safe): 1982 """Check if child process has terminated. Returns returncode 1983 attribute. 1984 1985 This method is called by __del__, so it cannot reference anything 1986 outside of the local scope (nor can any methods it calls). 1987 1988 """ 1989 if self.returncode is None: 1990 if not self._waitpid_lock.acquire(False): 1991 # Something else is busy calling waitpid. Don't allow two 1992 # at once. We know nothing yet. 1993 return None 1994 try: 1995 if self.returncode is not None: 1996 return self.returncode # Another thread waited. 1997 pid, sts = _del_safe.waitpid(self.pid, _del_safe.WNOHANG) 1998 if pid == self.pid: 1999 self._handle_exitstatus(sts) 2000 except OSError as e: 2001 if _deadstate is not None: 2002 self.returncode = _deadstate 2003 elif e.errno == _del_safe.ECHILD: 2004 # This happens if SIGCLD is set to be ignored or 2005 # waiting for child processes has otherwise been 2006 # disabled for our process. This child is dead, we 2007 # can't get the status. 2008 # http://bugs.python.org/issue15756 2009 self.returncode = 0 2010 finally: 2011 self._waitpid_lock.release() 2012 return self.returncode 2013 2014 2015 def _try_wait(self, wait_flags): 2016 """All callers to this function MUST hold self._waitpid_lock.""" 2017 try: 2018 (pid, sts) = os.waitpid(self.pid, wait_flags) 2019 except ChildProcessError: 2020 # This happens if SIGCLD is set to be ignored or waiting 2021 # for child processes has otherwise been disabled for our 2022 # process. This child is dead, we can't get the status. 2023 pid = self.pid 2024 sts = 0 2025 return (pid, sts) 2026 2027 2028 def _wait(self, timeout): 2029 """Internal implementation of wait() on POSIX.""" 2030 if self.returncode is not None: 2031 return self.returncode 2032 2033 if timeout is not None: 2034 endtime = _time() + timeout 2035 # Enter a busy loop if we have a timeout. This busy loop was 2036 # cribbed from Lib/threading.py in Thread.wait() at r71065. 2037 delay = 0.0005 # 500 us -> initial delay of 1 ms 2038 while True: 2039 if self._waitpid_lock.acquire(False): 2040 try: 2041 if self.returncode is not None: 2042 break # Another thread waited. 2043 (pid, sts) = self._try_wait(os.WNOHANG) 2044 assert pid == self.pid or pid == 0 2045 if pid == self.pid: 2046 self._handle_exitstatus(sts) 2047 break 2048 finally: 2049 self._waitpid_lock.release() 2050 remaining = self._remaining_time(endtime) 2051 if remaining <= 0: 2052 raise TimeoutExpired(self.args, timeout) 2053 delay = min(delay * 2, remaining, .05) 2054 time.sleep(delay) 2055 else: 2056 while self.returncode is None: 2057 with self._waitpid_lock: 2058 if self.returncode is not None: 2059 break # Another thread waited. 2060 (pid, sts) = self._try_wait(0) 2061 # Check the pid and loop as waitpid has been known to 2062 # return 0 even without WNOHANG in odd situations. 2063 # http://bugs.python.org/issue14396. 2064 if pid == self.pid: 2065 self._handle_exitstatus(sts) 2066 return self.returncode 2067 2068 2069 def _communicate(self, input, endtime, orig_timeout): 2070 if self.stdin and not self._communication_started: 2071 # Flush stdio buffer. This might block, if the user has 2072 # been writing to .stdin in an uncontrolled fashion. 2073 try: 2074 self.stdin.flush() 2075 except BrokenPipeError: 2076 pass # communicate() must ignore BrokenPipeError. 2077 if not input: 2078 try: 2079 self.stdin.close() 2080 except BrokenPipeError: 2081 pass # communicate() must ignore BrokenPipeError. 2082 2083 stdout = None 2084 stderr = None 2085 2086 # Only create this mapping if we haven't already. 2087 if not self._communication_started: 2088 self._fileobj2output = {} 2089 if self.stdout: 2090 self._fileobj2output[self.stdout] = [] 2091 if self.stderr: 2092 self._fileobj2output[self.stderr] = [] 2093 2094 if self.stdout: 2095 stdout = self._fileobj2output[self.stdout] 2096 if self.stderr: 2097 stderr = self._fileobj2output[self.stderr] 2098 2099 self._save_input(input) 2100 2101 if self._input: 2102 input_view = memoryview(self._input) 2103 2104 with _PopenSelector() as selector: 2105 if self.stdin and input: 2106 selector.register(self.stdin, selectors.EVENT_WRITE) 2107 if self.stdout and not self.stdout.closed: 2108 selector.register(self.stdout, selectors.EVENT_READ) 2109 if self.stderr and not self.stderr.closed: 2110 selector.register(self.stderr, selectors.EVENT_READ) 2111 2112 while selector.get_map(): 2113 timeout = self._remaining_time(endtime) 2114 if timeout is not None and timeout < 0: 2115 self._check_timeout(endtime, orig_timeout, 2116 stdout, stderr, 2117 skip_check_and_raise=True) 2118 raise RuntimeError( # Impossible :) 2119 '_check_timeout(..., skip_check_and_raise=True) ' 2120 'failed to raise TimeoutExpired.') 2121 2122 ready = selector.select(timeout) 2123 self._check_timeout(endtime, orig_timeout, stdout, stderr) 2124 2125 # XXX Rewrite these to use non-blocking I/O on the file 2126 # objects; they are no longer using C stdio! 2127 2128 for key, events in ready: 2129 if key.fileobj is self.stdin: 2130 chunk = input_view[self._input_offset : 2131 self._input_offset + _PIPE_BUF] 2132 try: 2133 self._input_offset += os.write(key.fd, chunk) 2134 except BrokenPipeError: 2135 selector.unregister(key.fileobj) 2136 key.fileobj.close() 2137 else: 2138 if self._input_offset >= len(self._input): 2139 selector.unregister(key.fileobj) 2140 key.fileobj.close() 2141 elif key.fileobj in (self.stdout, self.stderr): 2142 data = os.read(key.fd, 32768) 2143 if not data: 2144 selector.unregister(key.fileobj) 2145 key.fileobj.close() 2146 self._fileobj2output[key.fileobj].append(data) 2147 2148 self.wait(timeout=self._remaining_time(endtime)) 2149 2150 # All data exchanged. Translate lists into strings. 2151 if stdout is not None: 2152 stdout = b''.join(stdout) 2153 if stderr is not None: 2154 stderr = b''.join(stderr) 2155 2156 # Translate newlines, if requested. 2157 # This also turns bytes into strings. 2158 if self.text_mode: 2159 if stdout is not None: 2160 stdout = self._translate_newlines(stdout, 2161 self.stdout.encoding, 2162 self.stdout.errors) 2163 if stderr is not None: 2164 stderr = self._translate_newlines(stderr, 2165 self.stderr.encoding, 2166 self.stderr.errors) 2167 2168 return (stdout, stderr) 2169 2170 2171 def _save_input(self, input): 2172 # This method is called from the _communicate_with_*() methods 2173 # so that if we time out while communicating, we can continue 2174 # sending input if we retry. 2175 if self.stdin and self._input is None: 2176 self._input_offset = 0 2177 self._input = input 2178 if input is not None and self.text_mode: 2179 self._input = self._input.encode(self.stdin.encoding, 2180 self.stdin.errors) 2181 2182 2183 def send_signal(self, sig): 2184 """Send a signal to the process.""" 2185 # bpo-38630: Polling reduces the risk of sending a signal to the 2186 # wrong process if the process completed, the Popen.returncode 2187 # attribute is still None, and the pid has been reassigned 2188 # (recycled) to a new different process. This race condition can 2189 # happens in two cases. 2190 # 2191 # Case 1. Thread A calls Popen.poll(), thread B calls 2192 # Popen.send_signal(). In thread A, waitpid() succeed and returns 2193 # the exit status. Thread B calls kill() because poll() in thread A 2194 # did not set returncode yet. Calling poll() in thread B prevents 2195 # the race condition thanks to Popen._waitpid_lock. 2196 # 2197 # Case 2. waitpid(pid, 0) has been called directly, without 2198 # using Popen methods: returncode is still None is this case. 2199 # Calling Popen.poll() will set returncode to a default value, 2200 # since waitpid() fails with ProcessLookupError. 2201 self.poll() 2202 if self.returncode is not None: 2203 # Skip signalling a process that we know has already died. 2204 return 2205 2206 # The race condition can still happen if the race condition 2207 # described above happens between the returncode test 2208 # and the kill() call. 2209 try: 2210 os.kill(self.pid, sig) 2211 except ProcessLookupError: 2212 # Suppress the race condition error; bpo-40550. 2213 pass 2214 2215 def terminate(self): 2216 """Terminate the process with SIGTERM 2217 """ 2218 self.send_signal(signal.SIGTERM) 2219 2220 def kill(self): 2221 """Kill the process with SIGKILL 2222 """ 2223 self.send_signal(signal.SIGKILL) 2224