1# subprocess - Subprocesses with accessible I/O streams 2# 3# For more information about this module, see PEP 324. 4# 5# Copyright (c) 2003-2005 by Peter Astrand <astrand@lysator.liu.se> 6# 7# Licensed to PSF under a Contributor Agreement. 8# See http://www.python.org/2.4/license for licensing details. 9 10r"""Subprocesses with accessible I/O streams 11 12This module allows you to spawn processes, connect to their 13input/output/error pipes, and obtain their return codes. 14 15For a complete description of this module see the Python documentation. 16 17Main API 18======== 19run(...): Runs a command, waits for it to complete, then returns a 20 CompletedProcess instance. 21Popen(...): A class for flexibly executing a command in a new process 22 23Constants 24--------- 25DEVNULL: Special value that indicates that os.devnull should be used 26PIPE: Special value that indicates a pipe should be created 27STDOUT: Special value that indicates that stderr should go to stdout 28 29 30Older API 31========= 32call(...): Runs a command, waits for it to complete, then returns 33 the return code. 34check_call(...): Same as call() but raises CalledProcessError() 35 if return code is not 0 36check_output(...): Same as check_call() but returns the contents of 37 stdout instead of a return code 38getoutput(...): Runs a command in the shell, waits for it to complete, 39 then returns the output 40getstatusoutput(...): Runs a command in the shell, waits for it to complete, 41 then returns a (exitcode, output) tuple 42""" 43 44import sys 45_mswindows = (sys.platform == "win32") 46 47import io 48import os 49import time 50import signal 51import builtins 52import warnings 53import errno 54from time import monotonic as _time 55 56# Exception classes used by this module. 57class SubprocessError(Exception): pass 58 59 60class CalledProcessError(SubprocessError): 61 """Raised when run() is called with check=True and the process 62 returns a non-zero exit status. 63 64 Attributes: 65 cmd, returncode, stdout, stderr, output 66 """ 67 def __init__(self, returncode, cmd, output=None, stderr=None): 68 self.returncode = returncode 69 self.cmd = cmd 70 self.output = output 71 self.stderr = stderr 72 73 def __str__(self): 74 if self.returncode and self.returncode < 0: 75 try: 76 return "Command '%s' died with %r." % ( 77 self.cmd, signal.Signals(-self.returncode)) 78 except ValueError: 79 return "Command '%s' died with unknown signal %d." % ( 80 self.cmd, -self.returncode) 81 else: 82 return "Command '%s' returned non-zero exit status %d." % ( 83 self.cmd, self.returncode) 84 85 @property 86 def stdout(self): 87 """Alias for output attribute, to match stderr""" 88 return self.output 89 90 @stdout.setter 91 def stdout(self, value): 92 # There's no obvious reason to set this, but allow it anyway so 93 # .stdout is a transparent alias for .output 94 self.output = value 95 96 97class TimeoutExpired(SubprocessError): 98 """This exception is raised when the timeout expires while waiting for a 99 child process. 100 101 Attributes: 102 cmd, output, stdout, stderr, timeout 103 """ 104 def __init__(self, cmd, timeout, output=None, stderr=None): 105 self.cmd = cmd 106 self.timeout = timeout 107 self.output = output 108 self.stderr = stderr 109 110 def __str__(self): 111 return ("Command '%s' timed out after %s seconds" % 112 (self.cmd, self.timeout)) 113 114 @property 115 def stdout(self): 116 return self.output 117 118 @stdout.setter 119 def stdout(self, value): 120 # There's no obvious reason to set this, but allow it anyway so 121 # .stdout is a transparent alias for .output 122 self.output = value 123 124 125if _mswindows: 126 import threading 127 import msvcrt 128 import _winapi 129 class STARTUPINFO: 130 def __init__(self, *, dwFlags=0, hStdInput=None, hStdOutput=None, 131 hStdError=None, wShowWindow=0, lpAttributeList=None): 132 self.dwFlags = dwFlags 133 self.hStdInput = hStdInput 134 self.hStdOutput = hStdOutput 135 self.hStdError = hStdError 136 self.wShowWindow = wShowWindow 137 self.lpAttributeList = lpAttributeList or {"handle_list": []} 138 139 def _copy(self): 140 attr_list = self.lpAttributeList.copy() 141 if 'handle_list' in attr_list: 142 attr_list['handle_list'] = list(attr_list['handle_list']) 143 144 return STARTUPINFO(dwFlags=self.dwFlags, 145 hStdInput=self.hStdInput, 146 hStdOutput=self.hStdOutput, 147 hStdError=self.hStdError, 148 wShowWindow=self.wShowWindow, 149 lpAttributeList=attr_list) 150 151else: 152 import _posixsubprocess 153 import select 154 import selectors 155 import threading 156 157 # When select or poll has indicated that the file is writable, 158 # we can write up to _PIPE_BUF bytes without risk of blocking. 159 # POSIX defines PIPE_BUF as >= 512. 160 _PIPE_BUF = getattr(select, 'PIPE_BUF', 512) 161 162 # poll/select have the advantage of not requiring any extra file 163 # descriptor, contrarily to epoll/kqueue (also, they require a single 164 # syscall). 165 if hasattr(selectors, 'PollSelector'): 166 _PopenSelector = selectors.PollSelector 167 else: 168 _PopenSelector = selectors.SelectSelector 169 170 171__all__ = ["Popen", "PIPE", "STDOUT", "call", "check_call", "getstatusoutput", 172 "getoutput", "check_output", "run", "CalledProcessError", "DEVNULL", 173 "SubprocessError", "TimeoutExpired", "CompletedProcess"] 174 # NOTE: We intentionally exclude list2cmdline as it is 175 # considered an internal implementation detail. issue10838. 176 177if _mswindows: 178 from _winapi import (CREATE_NEW_CONSOLE, CREATE_NEW_PROCESS_GROUP, 179 STD_INPUT_HANDLE, STD_OUTPUT_HANDLE, 180 STD_ERROR_HANDLE, SW_HIDE, 181 STARTF_USESTDHANDLES, STARTF_USESHOWWINDOW, 182 ABOVE_NORMAL_PRIORITY_CLASS, BELOW_NORMAL_PRIORITY_CLASS, 183 HIGH_PRIORITY_CLASS, IDLE_PRIORITY_CLASS, 184 NORMAL_PRIORITY_CLASS, REALTIME_PRIORITY_CLASS, 185 CREATE_NO_WINDOW, DETACHED_PROCESS, 186 CREATE_DEFAULT_ERROR_MODE, CREATE_BREAKAWAY_FROM_JOB) 187 188 __all__.extend(["CREATE_NEW_CONSOLE", "CREATE_NEW_PROCESS_GROUP", 189 "STD_INPUT_HANDLE", "STD_OUTPUT_HANDLE", 190 "STD_ERROR_HANDLE", "SW_HIDE", 191 "STARTF_USESTDHANDLES", "STARTF_USESHOWWINDOW", 192 "STARTUPINFO", 193 "ABOVE_NORMAL_PRIORITY_CLASS", "BELOW_NORMAL_PRIORITY_CLASS", 194 "HIGH_PRIORITY_CLASS", "IDLE_PRIORITY_CLASS", 195 "NORMAL_PRIORITY_CLASS", "REALTIME_PRIORITY_CLASS", 196 "CREATE_NO_WINDOW", "DETACHED_PROCESS", 197 "CREATE_DEFAULT_ERROR_MODE", "CREATE_BREAKAWAY_FROM_JOB"]) 198 199 class Handle(int): 200 closed = False 201 202 def Close(self, CloseHandle=_winapi.CloseHandle): 203 if not self.closed: 204 self.closed = True 205 CloseHandle(self) 206 207 def Detach(self): 208 if not self.closed: 209 self.closed = True 210 return int(self) 211 raise ValueError("already closed") 212 213 def __repr__(self): 214 return "%s(%d)" % (self.__class__.__name__, int(self)) 215 216 __del__ = Close 217 __str__ = __repr__ 218 219 220# This lists holds Popen instances for which the underlying process had not 221# exited at the time its __del__ method got called: those processes are wait()ed 222# for synchronously from _cleanup() when a new Popen object is created, to avoid 223# zombie processes. 224_active = [] 225 226def _cleanup(): 227 for inst in _active[:]: 228 res = inst._internal_poll(_deadstate=sys.maxsize) 229 if res is not None: 230 try: 231 _active.remove(inst) 232 except ValueError: 233 # This can happen if two threads create a new Popen instance. 234 # It's harmless that it was already removed, so ignore. 235 pass 236 237PIPE = -1 238STDOUT = -2 239DEVNULL = -3 240 241 242# XXX This function is only used by multiprocessing and the test suite, 243# but it's here so that it can be imported when Python is compiled without 244# threads. 245 246def _optim_args_from_interpreter_flags(): 247 """Return a list of command-line arguments reproducing the current 248 optimization settings in sys.flags.""" 249 args = [] 250 value = sys.flags.optimize 251 if value > 0: 252 args.append('-' + 'O' * value) 253 return args 254 255 256def _args_from_interpreter_flags(): 257 """Return a list of command-line arguments reproducing the current 258 settings in sys.flags, sys.warnoptions and sys._xoptions.""" 259 flag_opt_map = { 260 'debug': 'd', 261 # 'inspect': 'i', 262 # 'interactive': 'i', 263 'dont_write_bytecode': 'B', 264 'no_site': 'S', 265 'verbose': 'v', 266 'bytes_warning': 'b', 267 'quiet': 'q', 268 # -O is handled in _optim_args_from_interpreter_flags() 269 } 270 args = _optim_args_from_interpreter_flags() 271 for flag, opt in flag_opt_map.items(): 272 v = getattr(sys.flags, flag) 273 if v > 0: 274 args.append('-' + opt * v) 275 276 if sys.flags.isolated: 277 args.append('-I') 278 else: 279 if sys.flags.ignore_environment: 280 args.append('-E') 281 if sys.flags.no_user_site: 282 args.append('-s') 283 284 # -W options 285 warnopts = sys.warnoptions[:] 286 bytes_warning = sys.flags.bytes_warning 287 xoptions = getattr(sys, '_xoptions', {}) 288 dev_mode = ('dev' in xoptions) 289 290 if bytes_warning > 1: 291 warnopts.remove("error::BytesWarning") 292 elif bytes_warning: 293 warnopts.remove("default::BytesWarning") 294 if dev_mode: 295 warnopts.remove('default') 296 for opt in warnopts: 297 args.append('-W' + opt) 298 299 # -X options 300 if dev_mode: 301 args.extend(('-X', 'dev')) 302 for opt in ('faulthandler', 'tracemalloc', 'importtime', 303 'showalloccount', 'showrefcount', 'utf8'): 304 if opt in xoptions: 305 value = xoptions[opt] 306 if value is True: 307 arg = opt 308 else: 309 arg = '%s=%s' % (opt, value) 310 args.extend(('-X', arg)) 311 312 return args 313 314 315def call(*popenargs, timeout=None, **kwargs): 316 """Run command with arguments. Wait for command to complete or 317 timeout, then return the returncode attribute. 318 319 The arguments are the same as for the Popen constructor. Example: 320 321 retcode = call(["ls", "-l"]) 322 """ 323 with Popen(*popenargs, **kwargs) as p: 324 try: 325 return p.wait(timeout=timeout) 326 except: # Including KeyboardInterrupt, wait handled that. 327 p.kill() 328 # We don't call p.wait() again as p.__exit__ does that for us. 329 raise 330 331 332def check_call(*popenargs, **kwargs): 333 """Run command with arguments. Wait for command to complete. If 334 the exit code was zero then return, otherwise raise 335 CalledProcessError. The CalledProcessError object will have the 336 return code in the returncode attribute. 337 338 The arguments are the same as for the call function. Example: 339 340 check_call(["ls", "-l"]) 341 """ 342 retcode = call(*popenargs, **kwargs) 343 if retcode: 344 cmd = kwargs.get("args") 345 if cmd is None: 346 cmd = popenargs[0] 347 raise CalledProcessError(retcode, cmd) 348 return 0 349 350 351def check_output(*popenargs, timeout=None, **kwargs): 352 r"""Run command with arguments and return its output. 353 354 If the exit code was non-zero it raises a CalledProcessError. The 355 CalledProcessError object will have the return code in the returncode 356 attribute and output in the output attribute. 357 358 The arguments are the same as for the Popen constructor. Example: 359 360 >>> check_output(["ls", "-l", "/dev/null"]) 361 b'crw-rw-rw- 1 root root 1, 3 Oct 18 2007 /dev/null\n' 362 363 The stdout argument is not allowed as it is used internally. 364 To capture standard error in the result, use stderr=STDOUT. 365 366 >>> check_output(["/bin/sh", "-c", 367 ... "ls -l non_existent_file ; exit 0"], 368 ... stderr=STDOUT) 369 b'ls: non_existent_file: No such file or directory\n' 370 371 There is an additional optional argument, "input", allowing you to 372 pass a string to the subprocess's stdin. If you use this argument 373 you may not also use the Popen constructor's "stdin" argument, as 374 it too will be used internally. Example: 375 376 >>> check_output(["sed", "-e", "s/foo/bar/"], 377 ... input=b"when in the course of fooman events\n") 378 b'when in the course of barman events\n' 379 380 By default, all communication is in bytes, and therefore any "input" 381 should be bytes, and the return value wil be bytes. If in text mode, 382 any "input" should be a string, and the return value will be a string 383 decoded according to locale encoding, or by "encoding" if set. Text mode 384 is triggered by setting any of text, encoding, errors or universal_newlines. 385 """ 386 if 'stdout' in kwargs: 387 raise ValueError('stdout argument not allowed, it will be overridden.') 388 389 if 'input' in kwargs and kwargs['input'] is None: 390 # Explicitly passing input=None was previously equivalent to passing an 391 # empty string. That is maintained here for backwards compatibility. 392 kwargs['input'] = '' if kwargs.get('universal_newlines', False) else b'' 393 394 return run(*popenargs, stdout=PIPE, timeout=timeout, check=True, 395 **kwargs).stdout 396 397 398class CompletedProcess(object): 399 """A process that has finished running. 400 401 This is returned by run(). 402 403 Attributes: 404 args: The list or str args passed to run(). 405 returncode: The exit code of the process, negative for signals. 406 stdout: The standard output (None if not captured). 407 stderr: The standard error (None if not captured). 408 """ 409 def __init__(self, args, returncode, stdout=None, stderr=None): 410 self.args = args 411 self.returncode = returncode 412 self.stdout = stdout 413 self.stderr = stderr 414 415 def __repr__(self): 416 args = ['args={!r}'.format(self.args), 417 'returncode={!r}'.format(self.returncode)] 418 if self.stdout is not None: 419 args.append('stdout={!r}'.format(self.stdout)) 420 if self.stderr is not None: 421 args.append('stderr={!r}'.format(self.stderr)) 422 return "{}({})".format(type(self).__name__, ', '.join(args)) 423 424 def check_returncode(self): 425 """Raise CalledProcessError if the exit code is non-zero.""" 426 if self.returncode: 427 raise CalledProcessError(self.returncode, self.args, self.stdout, 428 self.stderr) 429 430 431def run(*popenargs, 432 input=None, capture_output=False, timeout=None, check=False, **kwargs): 433 """Run command with arguments and return a CompletedProcess instance. 434 435 The returned instance will have attributes args, returncode, stdout and 436 stderr. By default, stdout and stderr are not captured, and those attributes 437 will be None. Pass stdout=PIPE and/or stderr=PIPE in order to capture them. 438 439 If check is True and the exit code was non-zero, it raises a 440 CalledProcessError. The CalledProcessError object will have the return code 441 in the returncode attribute, and output & stderr attributes if those streams 442 were captured. 443 444 If timeout is given, and the process takes too long, a TimeoutExpired 445 exception will be raised. 446 447 There is an optional argument "input", allowing you to 448 pass bytes or a string to the subprocess's stdin. If you use this argument 449 you may not also use the Popen constructor's "stdin" argument, as 450 it will be used internally. 451 452 By default, all communication is in bytes, and therefore any "input" should 453 be bytes, and the stdout and stderr will be bytes. If in text mode, any 454 "input" should be a string, and stdout and stderr will be strings decoded 455 according to locale encoding, or by "encoding" if set. Text mode is 456 triggered by setting any of text, encoding, errors or universal_newlines. 457 458 The other arguments are the same as for the Popen constructor. 459 """ 460 if input is not None: 461 if 'stdin' in kwargs: 462 raise ValueError('stdin and input arguments may not both be used.') 463 kwargs['stdin'] = PIPE 464 465 if capture_output: 466 if ('stdout' in kwargs) or ('stderr' in kwargs): 467 raise ValueError('stdout and stderr arguments may not be used ' 468 'with capture_output.') 469 kwargs['stdout'] = PIPE 470 kwargs['stderr'] = PIPE 471 472 with Popen(*popenargs, **kwargs) as process: 473 try: 474 stdout, stderr = process.communicate(input, timeout=timeout) 475 except TimeoutExpired: 476 process.kill() 477 stdout, stderr = process.communicate() 478 raise TimeoutExpired(process.args, timeout, output=stdout, 479 stderr=stderr) 480 except: # Including KeyboardInterrupt, communicate handled that. 481 process.kill() 482 # We don't call process.wait() as .__exit__ does that for us. 483 raise 484 retcode = process.poll() 485 if check and retcode: 486 raise CalledProcessError(retcode, process.args, 487 output=stdout, stderr=stderr) 488 return CompletedProcess(process.args, retcode, stdout, stderr) 489 490 491def list2cmdline(seq): 492 """ 493 Translate a sequence of arguments into a command line 494 string, using the same rules as the MS C runtime: 495 496 1) Arguments are delimited by white space, which is either a 497 space or a tab. 498 499 2) A string surrounded by double quotation marks is 500 interpreted as a single argument, regardless of white space 501 contained within. A quoted string can be embedded in an 502 argument. 503 504 3) A double quotation mark preceded by a backslash is 505 interpreted as a literal double quotation mark. 506 507 4) Backslashes are interpreted literally, unless they 508 immediately precede a double quotation mark. 509 510 5) If backslashes immediately precede a double quotation mark, 511 every pair of backslashes is interpreted as a literal 512 backslash. If the number of backslashes is odd, the last 513 backslash escapes the next double quotation mark as 514 described in rule 3. 515 """ 516 517 # See 518 # http://msdn.microsoft.com/en-us/library/17w5ykft.aspx 519 # or search http://msdn.microsoft.com for 520 # "Parsing C++ Command-Line Arguments" 521 result = [] 522 needquote = False 523 for arg in seq: 524 bs_buf = [] 525 526 # Add a space to separate this argument from the others 527 if result: 528 result.append(' ') 529 530 needquote = (" " in arg) or ("\t" in arg) or not arg 531 if needquote: 532 result.append('"') 533 534 for c in arg: 535 if c == '\\': 536 # Don't know if we need to double yet. 537 bs_buf.append(c) 538 elif c == '"': 539 # Double backslashes. 540 result.append('\\' * len(bs_buf)*2) 541 bs_buf = [] 542 result.append('\\"') 543 else: 544 # Normal char 545 if bs_buf: 546 result.extend(bs_buf) 547 bs_buf = [] 548 result.append(c) 549 550 # Add remaining backslashes, if any. 551 if bs_buf: 552 result.extend(bs_buf) 553 554 if needquote: 555 result.extend(bs_buf) 556 result.append('"') 557 558 return ''.join(result) 559 560 561# Various tools for executing commands and looking at their output and status. 562# 563 564def getstatusoutput(cmd): 565 """Return (exitcode, output) of executing cmd in a shell. 566 567 Execute the string 'cmd' in a shell with 'check_output' and 568 return a 2-tuple (status, output). The locale encoding is used 569 to decode the output and process newlines. 570 571 A trailing newline is stripped from the output. 572 The exit status for the command can be interpreted 573 according to the rules for the function 'wait'. Example: 574 575 >>> import subprocess 576 >>> subprocess.getstatusoutput('ls /bin/ls') 577 (0, '/bin/ls') 578 >>> subprocess.getstatusoutput('cat /bin/junk') 579 (1, 'cat: /bin/junk: No such file or directory') 580 >>> subprocess.getstatusoutput('/bin/junk') 581 (127, 'sh: /bin/junk: not found') 582 >>> subprocess.getstatusoutput('/bin/kill $$') 583 (-15, '') 584 """ 585 try: 586 data = check_output(cmd, shell=True, text=True, stderr=STDOUT) 587 exitcode = 0 588 except CalledProcessError as ex: 589 data = ex.output 590 exitcode = ex.returncode 591 if data[-1:] == '\n': 592 data = data[:-1] 593 return exitcode, data 594 595def getoutput(cmd): 596 """Return output (stdout or stderr) of executing cmd in a shell. 597 598 Like getstatusoutput(), except the exit status is ignored and the return 599 value is a string containing the command's output. Example: 600 601 >>> import subprocess 602 >>> subprocess.getoutput('ls /bin/ls') 603 '/bin/ls' 604 """ 605 return getstatusoutput(cmd)[1] 606 607 608class Popen(object): 609 """ Execute a child program in a new process. 610 611 For a complete description of the arguments see the Python documentation. 612 613 Arguments: 614 args: A string, or a sequence of program arguments. 615 616 bufsize: supplied as the buffering argument to the open() function when 617 creating the stdin/stdout/stderr pipe file objects 618 619 executable: A replacement program to execute. 620 621 stdin, stdout and stderr: These specify the executed programs' standard 622 input, standard output and standard error file handles, respectively. 623 624 preexec_fn: (POSIX only) An object to be called in the child process 625 just before the child is executed. 626 627 close_fds: Controls closing or inheriting of file descriptors. 628 629 shell: If true, the command will be executed through the shell. 630 631 cwd: Sets the current directory before the child is executed. 632 633 env: Defines the environment variables for the new process. 634 635 text: If true, decode stdin, stdout and stderr using the given encoding 636 (if set) or the system default otherwise. 637 638 universal_newlines: Alias of text, provided for backwards compatibility. 639 640 startupinfo and creationflags (Windows only) 641 642 restore_signals (POSIX only) 643 644 start_new_session (POSIX only) 645 646 pass_fds (POSIX only) 647 648 encoding and errors: Text mode encoding and error handling to use for 649 file objects stdin, stdout and stderr. 650 651 Attributes: 652 stdin, stdout, stderr, pid, returncode 653 """ 654 _child_created = False # Set here since __del__ checks it 655 656 def __init__(self, args, bufsize=-1, executable=None, 657 stdin=None, stdout=None, stderr=None, 658 preexec_fn=None, close_fds=True, 659 shell=False, cwd=None, env=None, universal_newlines=None, 660 startupinfo=None, creationflags=0, 661 restore_signals=True, start_new_session=False, 662 pass_fds=(), *, encoding=None, errors=None, text=None): 663 """Create new Popen instance.""" 664 _cleanup() 665 # Held while anything is calling waitpid before returncode has been 666 # updated to prevent clobbering returncode if wait() or poll() are 667 # called from multiple threads at once. After acquiring the lock, 668 # code must re-check self.returncode to see if another thread just 669 # finished a waitpid() call. 670 self._waitpid_lock = threading.Lock() 671 672 self._input = None 673 self._communication_started = False 674 if bufsize is None: 675 bufsize = -1 # Restore default 676 if not isinstance(bufsize, int): 677 raise TypeError("bufsize must be an integer") 678 679 if _mswindows: 680 if preexec_fn is not None: 681 raise ValueError("preexec_fn is not supported on Windows " 682 "platforms") 683 else: 684 # POSIX 685 if pass_fds and not close_fds: 686 warnings.warn("pass_fds overriding close_fds.", RuntimeWarning) 687 close_fds = True 688 if startupinfo is not None: 689 raise ValueError("startupinfo is only supported on Windows " 690 "platforms") 691 if creationflags != 0: 692 raise ValueError("creationflags is only supported on Windows " 693 "platforms") 694 695 self.args = args 696 self.stdin = None 697 self.stdout = None 698 self.stderr = None 699 self.pid = None 700 self.returncode = None 701 self.encoding = encoding 702 self.errors = errors 703 704 # Validate the combinations of text and universal_newlines 705 if (text is not None and universal_newlines is not None 706 and bool(universal_newlines) != bool(text)): 707 raise SubprocessError('Cannot disambiguate when both text ' 708 'and universal_newlines are supplied but ' 709 'different. Pass one or the other.') 710 711 # Input and output objects. The general principle is like 712 # this: 713 # 714 # Parent Child 715 # ------ ----- 716 # p2cwrite ---stdin---> p2cread 717 # c2pread <--stdout--- c2pwrite 718 # errread <--stderr--- errwrite 719 # 720 # On POSIX, the child objects are file descriptors. On 721 # Windows, these are Windows file handles. The parent objects 722 # are file descriptors on both platforms. The parent objects 723 # are -1 when not using PIPEs. The child objects are -1 724 # when not redirecting. 725 726 (p2cread, p2cwrite, 727 c2pread, c2pwrite, 728 errread, errwrite) = self._get_handles(stdin, stdout, stderr) 729 730 # We wrap OS handles *before* launching the child, otherwise a 731 # quickly terminating child could make our fds unwrappable 732 # (see #8458). 733 734 if _mswindows: 735 if p2cwrite != -1: 736 p2cwrite = msvcrt.open_osfhandle(p2cwrite.Detach(), 0) 737 if c2pread != -1: 738 c2pread = msvcrt.open_osfhandle(c2pread.Detach(), 0) 739 if errread != -1: 740 errread = msvcrt.open_osfhandle(errread.Detach(), 0) 741 742 self.text_mode = encoding or errors or text or universal_newlines 743 744 # How long to resume waiting on a child after the first ^C. 745 # There is no right value for this. The purpose is to be polite 746 # yet remain good for interactive users trying to exit a tool. 747 self._sigint_wait_secs = 0.25 # 1/xkcd221.getRandomNumber() 748 749 self._closed_child_pipe_fds = False 750 751 try: 752 if p2cwrite != -1: 753 self.stdin = io.open(p2cwrite, 'wb', bufsize) 754 if self.text_mode: 755 self.stdin = io.TextIOWrapper(self.stdin, write_through=True, 756 line_buffering=(bufsize == 1), 757 encoding=encoding, errors=errors) 758 if c2pread != -1: 759 self.stdout = io.open(c2pread, 'rb', bufsize) 760 if self.text_mode: 761 self.stdout = io.TextIOWrapper(self.stdout, 762 encoding=encoding, errors=errors) 763 if errread != -1: 764 self.stderr = io.open(errread, 'rb', bufsize) 765 if self.text_mode: 766 self.stderr = io.TextIOWrapper(self.stderr, 767 encoding=encoding, errors=errors) 768 769 self._execute_child(args, executable, preexec_fn, close_fds, 770 pass_fds, cwd, env, 771 startupinfo, creationflags, shell, 772 p2cread, p2cwrite, 773 c2pread, c2pwrite, 774 errread, errwrite, 775 restore_signals, start_new_session) 776 except: 777 # Cleanup if the child failed starting. 778 for f in filter(None, (self.stdin, self.stdout, self.stderr)): 779 try: 780 f.close() 781 except OSError: 782 pass # Ignore EBADF or other errors. 783 784 if not self._closed_child_pipe_fds: 785 to_close = [] 786 if stdin == PIPE: 787 to_close.append(p2cread) 788 if stdout == PIPE: 789 to_close.append(c2pwrite) 790 if stderr == PIPE: 791 to_close.append(errwrite) 792 if hasattr(self, '_devnull'): 793 to_close.append(self._devnull) 794 for fd in to_close: 795 try: 796 if _mswindows and isinstance(fd, Handle): 797 fd.Close() 798 else: 799 os.close(fd) 800 except OSError: 801 pass 802 803 raise 804 805 @property 806 def universal_newlines(self): 807 # universal_newlines as retained as an alias of text_mode for API 808 # compatibility. bpo-31756 809 return self.text_mode 810 811 @universal_newlines.setter 812 def universal_newlines(self, universal_newlines): 813 self.text_mode = bool(universal_newlines) 814 815 def _translate_newlines(self, data, encoding, errors): 816 data = data.decode(encoding, errors) 817 return data.replace("\r\n", "\n").replace("\r", "\n") 818 819 def __enter__(self): 820 return self 821 822 def __exit__(self, exc_type, value, traceback): 823 if self.stdout: 824 self.stdout.close() 825 if self.stderr: 826 self.stderr.close() 827 try: # Flushing a BufferedWriter may raise an error 828 if self.stdin: 829 self.stdin.close() 830 finally: 831 if exc_type == KeyboardInterrupt: 832 # https://bugs.python.org/issue25942 833 # In the case of a KeyboardInterrupt we assume the SIGINT 834 # was also already sent to our child processes. We can't 835 # block indefinitely as that is not user friendly. 836 # If we have not already waited a brief amount of time in 837 # an interrupted .wait() or .communicate() call, do so here 838 # for consistency. 839 if self._sigint_wait_secs > 0: 840 try: 841 self._wait(timeout=self._sigint_wait_secs) 842 except TimeoutExpired: 843 pass 844 self._sigint_wait_secs = 0 # Note that this has been done. 845 return # resume the KeyboardInterrupt 846 847 # Wait for the process to terminate, to avoid zombies. 848 self.wait() 849 850 def __del__(self, _maxsize=sys.maxsize, _warn=warnings.warn): 851 if not self._child_created: 852 # We didn't get to successfully create a child process. 853 return 854 if self.returncode is None: 855 # Not reading subprocess exit status creates a zombie process which 856 # is only destroyed at the parent python process exit 857 _warn("subprocess %s is still running" % self.pid, 858 ResourceWarning, source=self) 859 # In case the child hasn't been waited on, check if it's done. 860 self._internal_poll(_deadstate=_maxsize) 861 if self.returncode is None and _active is not None: 862 # Child is still running, keep us alive until we can wait on it. 863 _active.append(self) 864 865 def _get_devnull(self): 866 if not hasattr(self, '_devnull'): 867 self._devnull = os.open(os.devnull, os.O_RDWR) 868 return self._devnull 869 870 def _stdin_write(self, input): 871 if input: 872 try: 873 self.stdin.write(input) 874 except BrokenPipeError: 875 pass # communicate() must ignore broken pipe errors. 876 except OSError as exc: 877 if exc.errno == errno.EINVAL: 878 # bpo-19612, bpo-30418: On Windows, stdin.write() fails 879 # with EINVAL if the child process exited or if the child 880 # process is still running but closed the pipe. 881 pass 882 else: 883 raise 884 885 try: 886 self.stdin.close() 887 except BrokenPipeError: 888 pass # communicate() must ignore broken pipe errors. 889 except OSError as exc: 890 if exc.errno == errno.EINVAL: 891 pass 892 else: 893 raise 894 895 def communicate(self, input=None, timeout=None): 896 """Interact with process: Send data to stdin and close it. 897 Read data from stdout and stderr, until end-of-file is 898 reached. Wait for process to terminate. 899 900 The optional "input" argument should be data to be sent to the 901 child process, or None, if no data should be sent to the child. 902 communicate() returns a tuple (stdout, stderr). 903 904 By default, all communication is in bytes, and therefore any 905 "input" should be bytes, and the (stdout, stderr) will be bytes. 906 If in text mode (indicated by self.text_mode), any "input" should 907 be a string, and (stdout, stderr) will be strings decoded 908 according to locale encoding, or by "encoding" if set. Text mode 909 is triggered by setting any of text, encoding, errors or 910 universal_newlines. 911 """ 912 913 if self._communication_started and input: 914 raise ValueError("Cannot send input after starting communication") 915 916 # Optimization: If we are not worried about timeouts, we haven't 917 # started communicating, and we have one or zero pipes, using select() 918 # or threads is unnecessary. 919 if (timeout is None and not self._communication_started and 920 [self.stdin, self.stdout, self.stderr].count(None) >= 2): 921 stdout = None 922 stderr = None 923 if self.stdin: 924 self._stdin_write(input) 925 elif self.stdout: 926 stdout = self.stdout.read() 927 self.stdout.close() 928 elif self.stderr: 929 stderr = self.stderr.read() 930 self.stderr.close() 931 self.wait() 932 else: 933 if timeout is not None: 934 endtime = _time() + timeout 935 else: 936 endtime = None 937 938 try: 939 stdout, stderr = self._communicate(input, endtime, timeout) 940 except KeyboardInterrupt: 941 # https://bugs.python.org/issue25942 942 # See the detailed comment in .wait(). 943 if timeout is not None: 944 sigint_timeout = min(self._sigint_wait_secs, 945 self._remaining_time(endtime)) 946 else: 947 sigint_timeout = self._sigint_wait_secs 948 self._sigint_wait_secs = 0 # nothing else should wait. 949 try: 950 self._wait(timeout=sigint_timeout) 951 except TimeoutExpired: 952 pass 953 raise # resume the KeyboardInterrupt 954 955 finally: 956 self._communication_started = True 957 958 sts = self.wait(timeout=self._remaining_time(endtime)) 959 960 return (stdout, stderr) 961 962 963 def poll(self): 964 """Check if child process has terminated. Set and return returncode 965 attribute.""" 966 return self._internal_poll() 967 968 969 def _remaining_time(self, endtime): 970 """Convenience for _communicate when computing timeouts.""" 971 if endtime is None: 972 return None 973 else: 974 return endtime - _time() 975 976 977 def _check_timeout(self, endtime, orig_timeout): 978 """Convenience for checking if a timeout has expired.""" 979 if endtime is None: 980 return 981 if _time() > endtime: 982 raise TimeoutExpired(self.args, orig_timeout) 983 984 985 def wait(self, timeout=None): 986 """Wait for child process to terminate; returns self.returncode.""" 987 if timeout is not None: 988 endtime = _time() + timeout 989 try: 990 return self._wait(timeout=timeout) 991 except KeyboardInterrupt: 992 # https://bugs.python.org/issue25942 993 # The first keyboard interrupt waits briefly for the child to 994 # exit under the common assumption that it also received the ^C 995 # generated SIGINT and will exit rapidly. 996 if timeout is not None: 997 sigint_timeout = min(self._sigint_wait_secs, 998 self._remaining_time(endtime)) 999 else: 1000 sigint_timeout = self._sigint_wait_secs 1001 self._sigint_wait_secs = 0 # nothing else should wait. 1002 try: 1003 self._wait(timeout=sigint_timeout) 1004 except TimeoutExpired: 1005 pass 1006 raise # resume the KeyboardInterrupt 1007 1008 1009 if _mswindows: 1010 # 1011 # Windows methods 1012 # 1013 def _get_handles(self, stdin, stdout, stderr): 1014 """Construct and return tuple with IO objects: 1015 p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite 1016 """ 1017 if stdin is None and stdout is None and stderr is None: 1018 return (-1, -1, -1, -1, -1, -1) 1019 1020 p2cread, p2cwrite = -1, -1 1021 c2pread, c2pwrite = -1, -1 1022 errread, errwrite = -1, -1 1023 1024 if stdin is None: 1025 p2cread = _winapi.GetStdHandle(_winapi.STD_INPUT_HANDLE) 1026 if p2cread is None: 1027 p2cread, _ = _winapi.CreatePipe(None, 0) 1028 p2cread = Handle(p2cread) 1029 _winapi.CloseHandle(_) 1030 elif stdin == PIPE: 1031 p2cread, p2cwrite = _winapi.CreatePipe(None, 0) 1032 p2cread, p2cwrite = Handle(p2cread), Handle(p2cwrite) 1033 elif stdin == DEVNULL: 1034 p2cread = msvcrt.get_osfhandle(self._get_devnull()) 1035 elif isinstance(stdin, int): 1036 p2cread = msvcrt.get_osfhandle(stdin) 1037 else: 1038 # Assuming file-like object 1039 p2cread = msvcrt.get_osfhandle(stdin.fileno()) 1040 p2cread = self._make_inheritable(p2cread) 1041 1042 if stdout is None: 1043 c2pwrite = _winapi.GetStdHandle(_winapi.STD_OUTPUT_HANDLE) 1044 if c2pwrite is None: 1045 _, c2pwrite = _winapi.CreatePipe(None, 0) 1046 c2pwrite = Handle(c2pwrite) 1047 _winapi.CloseHandle(_) 1048 elif stdout == PIPE: 1049 c2pread, c2pwrite = _winapi.CreatePipe(None, 0) 1050 c2pread, c2pwrite = Handle(c2pread), Handle(c2pwrite) 1051 elif stdout == DEVNULL: 1052 c2pwrite = msvcrt.get_osfhandle(self._get_devnull()) 1053 elif isinstance(stdout, int): 1054 c2pwrite = msvcrt.get_osfhandle(stdout) 1055 else: 1056 # Assuming file-like object 1057 c2pwrite = msvcrt.get_osfhandle(stdout.fileno()) 1058 c2pwrite = self._make_inheritable(c2pwrite) 1059 1060 if stderr is None: 1061 errwrite = _winapi.GetStdHandle(_winapi.STD_ERROR_HANDLE) 1062 if errwrite is None: 1063 _, errwrite = _winapi.CreatePipe(None, 0) 1064 errwrite = Handle(errwrite) 1065 _winapi.CloseHandle(_) 1066 elif stderr == PIPE: 1067 errread, errwrite = _winapi.CreatePipe(None, 0) 1068 errread, errwrite = Handle(errread), Handle(errwrite) 1069 elif stderr == STDOUT: 1070 errwrite = c2pwrite 1071 elif stderr == DEVNULL: 1072 errwrite = msvcrt.get_osfhandle(self._get_devnull()) 1073 elif isinstance(stderr, int): 1074 errwrite = msvcrt.get_osfhandle(stderr) 1075 else: 1076 # Assuming file-like object 1077 errwrite = msvcrt.get_osfhandle(stderr.fileno()) 1078 errwrite = self._make_inheritable(errwrite) 1079 1080 return (p2cread, p2cwrite, 1081 c2pread, c2pwrite, 1082 errread, errwrite) 1083 1084 1085 def _make_inheritable(self, handle): 1086 """Return a duplicate of handle, which is inheritable""" 1087 h = _winapi.DuplicateHandle( 1088 _winapi.GetCurrentProcess(), handle, 1089 _winapi.GetCurrentProcess(), 0, 1, 1090 _winapi.DUPLICATE_SAME_ACCESS) 1091 return Handle(h) 1092 1093 1094 def _filter_handle_list(self, handle_list): 1095 """Filter out console handles that can't be used 1096 in lpAttributeList["handle_list"] and make sure the list 1097 isn't empty. This also removes duplicate handles.""" 1098 # An handle with it's lowest two bits set might be a special console 1099 # handle that if passed in lpAttributeList["handle_list"], will 1100 # cause it to fail. 1101 return list({handle for handle in handle_list 1102 if handle & 0x3 != 0x3 1103 or _winapi.GetFileType(handle) != 1104 _winapi.FILE_TYPE_CHAR}) 1105 1106 1107 def _execute_child(self, args, executable, preexec_fn, close_fds, 1108 pass_fds, cwd, env, 1109 startupinfo, creationflags, shell, 1110 p2cread, p2cwrite, 1111 c2pread, c2pwrite, 1112 errread, errwrite, 1113 unused_restore_signals, unused_start_new_session): 1114 """Execute program (MS Windows version)""" 1115 1116 assert not pass_fds, "pass_fds not supported on Windows." 1117 1118 if not isinstance(args, str): 1119 args = list2cmdline(args) 1120 1121 # Process startup details 1122 if startupinfo is None: 1123 startupinfo = STARTUPINFO() 1124 else: 1125 # bpo-34044: Copy STARTUPINFO since it is modified above, 1126 # so the caller can reuse it multiple times. 1127 startupinfo = startupinfo._copy() 1128 1129 use_std_handles = -1 not in (p2cread, c2pwrite, errwrite) 1130 if use_std_handles: 1131 startupinfo.dwFlags |= _winapi.STARTF_USESTDHANDLES 1132 startupinfo.hStdInput = p2cread 1133 startupinfo.hStdOutput = c2pwrite 1134 startupinfo.hStdError = errwrite 1135 1136 attribute_list = startupinfo.lpAttributeList 1137 have_handle_list = bool(attribute_list and 1138 "handle_list" in attribute_list and 1139 attribute_list["handle_list"]) 1140 1141 # If we were given an handle_list or need to create one 1142 if have_handle_list or (use_std_handles and close_fds): 1143 if attribute_list is None: 1144 attribute_list = startupinfo.lpAttributeList = {} 1145 handle_list = attribute_list["handle_list"] = \ 1146 list(attribute_list.get("handle_list", [])) 1147 1148 if use_std_handles: 1149 handle_list += [int(p2cread), int(c2pwrite), int(errwrite)] 1150 1151 handle_list[:] = self._filter_handle_list(handle_list) 1152 1153 if handle_list: 1154 if not close_fds: 1155 warnings.warn("startupinfo.lpAttributeList['handle_list'] " 1156 "overriding close_fds", RuntimeWarning) 1157 1158 # When using the handle_list we always request to inherit 1159 # handles but the only handles that will be inherited are 1160 # the ones in the handle_list 1161 close_fds = False 1162 1163 if shell: 1164 startupinfo.dwFlags |= _winapi.STARTF_USESHOWWINDOW 1165 startupinfo.wShowWindow = _winapi.SW_HIDE 1166 comspec = os.environ.get("COMSPEC", "cmd.exe") 1167 args = '{} /c "{}"'.format (comspec, args) 1168 1169 # Start the process 1170 try: 1171 hp, ht, pid, tid = _winapi.CreateProcess(executable, args, 1172 # no special security 1173 None, None, 1174 int(not close_fds), 1175 creationflags, 1176 env, 1177 os.fspath(cwd) if cwd is not None else None, 1178 startupinfo) 1179 finally: 1180 # Child is launched. Close the parent's copy of those pipe 1181 # handles that only the child should have open. You need 1182 # to make sure that no handles to the write end of the 1183 # output pipe are maintained in this process or else the 1184 # pipe will not close when the child process exits and the 1185 # ReadFile will hang. 1186 if p2cread != -1: 1187 p2cread.Close() 1188 if c2pwrite != -1: 1189 c2pwrite.Close() 1190 if errwrite != -1: 1191 errwrite.Close() 1192 if hasattr(self, '_devnull'): 1193 os.close(self._devnull) 1194 # Prevent a double close of these handles/fds from __init__ 1195 # on error. 1196 self._closed_child_pipe_fds = True 1197 1198 # Retain the process handle, but close the thread handle 1199 self._child_created = True 1200 self._handle = Handle(hp) 1201 self.pid = pid 1202 _winapi.CloseHandle(ht) 1203 1204 def _internal_poll(self, _deadstate=None, 1205 _WaitForSingleObject=_winapi.WaitForSingleObject, 1206 _WAIT_OBJECT_0=_winapi.WAIT_OBJECT_0, 1207 _GetExitCodeProcess=_winapi.GetExitCodeProcess): 1208 """Check if child process has terminated. Returns returncode 1209 attribute. 1210 1211 This method is called by __del__, so it can only refer to objects 1212 in its local scope. 1213 1214 """ 1215 if self.returncode is None: 1216 if _WaitForSingleObject(self._handle, 0) == _WAIT_OBJECT_0: 1217 self.returncode = _GetExitCodeProcess(self._handle) 1218 return self.returncode 1219 1220 1221 def _wait(self, timeout): 1222 """Internal implementation of wait() on Windows.""" 1223 if timeout is None: 1224 timeout_millis = _winapi.INFINITE 1225 else: 1226 timeout_millis = int(timeout * 1000) 1227 if self.returncode is None: 1228 # API note: Returns immediately if timeout_millis == 0. 1229 result = _winapi.WaitForSingleObject(self._handle, 1230 timeout_millis) 1231 if result == _winapi.WAIT_TIMEOUT: 1232 raise TimeoutExpired(self.args, timeout) 1233 self.returncode = _winapi.GetExitCodeProcess(self._handle) 1234 return self.returncode 1235 1236 1237 def _readerthread(self, fh, buffer): 1238 buffer.append(fh.read()) 1239 fh.close() 1240 1241 1242 def _communicate(self, input, endtime, orig_timeout): 1243 # Start reader threads feeding into a list hanging off of this 1244 # object, unless they've already been started. 1245 if self.stdout and not hasattr(self, "_stdout_buff"): 1246 self._stdout_buff = [] 1247 self.stdout_thread = \ 1248 threading.Thread(target=self._readerthread, 1249 args=(self.stdout, self._stdout_buff)) 1250 self.stdout_thread.daemon = True 1251 self.stdout_thread.start() 1252 if self.stderr and not hasattr(self, "_stderr_buff"): 1253 self._stderr_buff = [] 1254 self.stderr_thread = \ 1255 threading.Thread(target=self._readerthread, 1256 args=(self.stderr, self._stderr_buff)) 1257 self.stderr_thread.daemon = True 1258 self.stderr_thread.start() 1259 1260 if self.stdin: 1261 self._stdin_write(input) 1262 1263 # Wait for the reader threads, or time out. If we time out, the 1264 # threads remain reading and the fds left open in case the user 1265 # calls communicate again. 1266 if self.stdout is not None: 1267 self.stdout_thread.join(self._remaining_time(endtime)) 1268 if self.stdout_thread.is_alive(): 1269 raise TimeoutExpired(self.args, orig_timeout) 1270 if self.stderr is not None: 1271 self.stderr_thread.join(self._remaining_time(endtime)) 1272 if self.stderr_thread.is_alive(): 1273 raise TimeoutExpired(self.args, orig_timeout) 1274 1275 # Collect the output from and close both pipes, now that we know 1276 # both have been read successfully. 1277 stdout = None 1278 stderr = None 1279 if self.stdout: 1280 stdout = self._stdout_buff 1281 self.stdout.close() 1282 if self.stderr: 1283 stderr = self._stderr_buff 1284 self.stderr.close() 1285 1286 # All data exchanged. Translate lists into strings. 1287 if stdout is not None: 1288 stdout = stdout[0] 1289 if stderr is not None: 1290 stderr = stderr[0] 1291 1292 return (stdout, stderr) 1293 1294 def send_signal(self, sig): 1295 """Send a signal to the process.""" 1296 # Don't signal a process that we know has already died. 1297 if self.returncode is not None: 1298 return 1299 if sig == signal.SIGTERM: 1300 self.terminate() 1301 elif sig == signal.CTRL_C_EVENT: 1302 os.kill(self.pid, signal.CTRL_C_EVENT) 1303 elif sig == signal.CTRL_BREAK_EVENT: 1304 os.kill(self.pid, signal.CTRL_BREAK_EVENT) 1305 else: 1306 raise ValueError("Unsupported signal: {}".format(sig)) 1307 1308 def terminate(self): 1309 """Terminates the process.""" 1310 # Don't terminate a process that we know has already died. 1311 if self.returncode is not None: 1312 return 1313 try: 1314 _winapi.TerminateProcess(self._handle, 1) 1315 except PermissionError: 1316 # ERROR_ACCESS_DENIED (winerror 5) is received when the 1317 # process already died. 1318 rc = _winapi.GetExitCodeProcess(self._handle) 1319 if rc == _winapi.STILL_ACTIVE: 1320 raise 1321 self.returncode = rc 1322 1323 kill = terminate 1324 1325 else: 1326 # 1327 # POSIX methods 1328 # 1329 def _get_handles(self, stdin, stdout, stderr): 1330 """Construct and return tuple with IO objects: 1331 p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite 1332 """ 1333 p2cread, p2cwrite = -1, -1 1334 c2pread, c2pwrite = -1, -1 1335 errread, errwrite = -1, -1 1336 1337 if stdin is None: 1338 pass 1339 elif stdin == PIPE: 1340 p2cread, p2cwrite = os.pipe() 1341 elif stdin == DEVNULL: 1342 p2cread = self._get_devnull() 1343 elif isinstance(stdin, int): 1344 p2cread = stdin 1345 else: 1346 # Assuming file-like object 1347 p2cread = stdin.fileno() 1348 1349 if stdout is None: 1350 pass 1351 elif stdout == PIPE: 1352 c2pread, c2pwrite = os.pipe() 1353 elif stdout == DEVNULL: 1354 c2pwrite = self._get_devnull() 1355 elif isinstance(stdout, int): 1356 c2pwrite = stdout 1357 else: 1358 # Assuming file-like object 1359 c2pwrite = stdout.fileno() 1360 1361 if stderr is None: 1362 pass 1363 elif stderr == PIPE: 1364 errread, errwrite = os.pipe() 1365 elif stderr == STDOUT: 1366 if c2pwrite != -1: 1367 errwrite = c2pwrite 1368 else: # child's stdout is not set, use parent's stdout 1369 errwrite = sys.__stdout__.fileno() 1370 elif stderr == DEVNULL: 1371 errwrite = self._get_devnull() 1372 elif isinstance(stderr, int): 1373 errwrite = stderr 1374 else: 1375 # Assuming file-like object 1376 errwrite = stderr.fileno() 1377 1378 return (p2cread, p2cwrite, 1379 c2pread, c2pwrite, 1380 errread, errwrite) 1381 1382 1383 def _execute_child(self, args, executable, preexec_fn, close_fds, 1384 pass_fds, cwd, env, 1385 startupinfo, creationflags, shell, 1386 p2cread, p2cwrite, 1387 c2pread, c2pwrite, 1388 errread, errwrite, 1389 restore_signals, start_new_session): 1390 """Execute program (POSIX version)""" 1391 1392 if isinstance(args, (str, bytes)): 1393 args = [args] 1394 else: 1395 args = list(args) 1396 1397 if shell: 1398 # On Android the default shell is at '/system/bin/sh'. 1399 unix_shell = ('/system/bin/sh' if 1400 hasattr(sys, 'getandroidapilevel') else '/bin/sh') 1401 args = [unix_shell, "-c"] + args 1402 if executable: 1403 args[0] = executable 1404 1405 if executable is None: 1406 executable = args[0] 1407 orig_executable = executable 1408 1409 # For transferring possible exec failure from child to parent. 1410 # Data format: "exception name:hex errno:description" 1411 # Pickle is not used; it is complex and involves memory allocation. 1412 errpipe_read, errpipe_write = os.pipe() 1413 # errpipe_write must not be in the standard io 0, 1, or 2 fd range. 1414 low_fds_to_close = [] 1415 while errpipe_write < 3: 1416 low_fds_to_close.append(errpipe_write) 1417 errpipe_write = os.dup(errpipe_write) 1418 for low_fd in low_fds_to_close: 1419 os.close(low_fd) 1420 try: 1421 try: 1422 # We must avoid complex work that could involve 1423 # malloc or free in the child process to avoid 1424 # potential deadlocks, thus we do all this here. 1425 # and pass it to fork_exec() 1426 1427 if env is not None: 1428 env_list = [] 1429 for k, v in env.items(): 1430 k = os.fsencode(k) 1431 if b'=' in k: 1432 raise ValueError("illegal environment variable name") 1433 env_list.append(k + b'=' + os.fsencode(v)) 1434 else: 1435 env_list = None # Use execv instead of execve. 1436 executable = os.fsencode(executable) 1437 if os.path.dirname(executable): 1438 executable_list = (executable,) 1439 else: 1440 # This matches the behavior of os._execvpe(). 1441 executable_list = tuple( 1442 os.path.join(os.fsencode(dir), executable) 1443 for dir in os.get_exec_path(env)) 1444 fds_to_keep = set(pass_fds) 1445 fds_to_keep.add(errpipe_write) 1446 self.pid = _posixsubprocess.fork_exec( 1447 args, executable_list, 1448 close_fds, tuple(sorted(map(int, fds_to_keep))), 1449 cwd, env_list, 1450 p2cread, p2cwrite, c2pread, c2pwrite, 1451 errread, errwrite, 1452 errpipe_read, errpipe_write, 1453 restore_signals, start_new_session, preexec_fn) 1454 self._child_created = True 1455 finally: 1456 # be sure the FD is closed no matter what 1457 os.close(errpipe_write) 1458 1459 # self._devnull is not always defined. 1460 devnull_fd = getattr(self, '_devnull', None) 1461 if p2cread != -1 and p2cwrite != -1 and p2cread != devnull_fd: 1462 os.close(p2cread) 1463 if c2pwrite != -1 and c2pread != -1 and c2pwrite != devnull_fd: 1464 os.close(c2pwrite) 1465 if errwrite != -1 and errread != -1 and errwrite != devnull_fd: 1466 os.close(errwrite) 1467 if devnull_fd is not None: 1468 os.close(devnull_fd) 1469 # Prevent a double close of these fds from __init__ on error. 1470 self._closed_child_pipe_fds = True 1471 1472 # Wait for exec to fail or succeed; possibly raising an 1473 # exception (limited in size) 1474 errpipe_data = bytearray() 1475 while True: 1476 part = os.read(errpipe_read, 50000) 1477 errpipe_data += part 1478 if not part or len(errpipe_data) > 50000: 1479 break 1480 finally: 1481 # be sure the FD is closed no matter what 1482 os.close(errpipe_read) 1483 1484 if errpipe_data: 1485 try: 1486 pid, sts = os.waitpid(self.pid, 0) 1487 if pid == self.pid: 1488 self._handle_exitstatus(sts) 1489 else: 1490 self.returncode = sys.maxsize 1491 except ChildProcessError: 1492 pass 1493 1494 try: 1495 exception_name, hex_errno, err_msg = ( 1496 errpipe_data.split(b':', 2)) 1497 # The encoding here should match the encoding 1498 # written in by the subprocess implementations 1499 # like _posixsubprocess 1500 err_msg = err_msg.decode() 1501 except ValueError: 1502 exception_name = b'SubprocessError' 1503 hex_errno = b'0' 1504 err_msg = 'Bad exception data from child: {!r}'.format( 1505 bytes(errpipe_data)) 1506 child_exception_type = getattr( 1507 builtins, exception_name.decode('ascii'), 1508 SubprocessError) 1509 if issubclass(child_exception_type, OSError) and hex_errno: 1510 errno_num = int(hex_errno, 16) 1511 child_exec_never_called = (err_msg == "noexec") 1512 if child_exec_never_called: 1513 err_msg = "" 1514 # The error must be from chdir(cwd). 1515 err_filename = cwd 1516 else: 1517 err_filename = orig_executable 1518 if errno_num != 0: 1519 err_msg = os.strerror(errno_num) 1520 if errno_num == errno.ENOENT: 1521 err_msg += ': ' + repr(err_filename) 1522 raise child_exception_type(errno_num, err_msg, err_filename) 1523 raise child_exception_type(err_msg) 1524 1525 1526 def _handle_exitstatus(self, sts, _WIFSIGNALED=os.WIFSIGNALED, 1527 _WTERMSIG=os.WTERMSIG, _WIFEXITED=os.WIFEXITED, 1528 _WEXITSTATUS=os.WEXITSTATUS, _WIFSTOPPED=os.WIFSTOPPED, 1529 _WSTOPSIG=os.WSTOPSIG): 1530 """All callers to this function MUST hold self._waitpid_lock.""" 1531 # This method is called (indirectly) by __del__, so it cannot 1532 # refer to anything outside of its local scope. 1533 if _WIFSIGNALED(sts): 1534 self.returncode = -_WTERMSIG(sts) 1535 elif _WIFEXITED(sts): 1536 self.returncode = _WEXITSTATUS(sts) 1537 elif _WIFSTOPPED(sts): 1538 self.returncode = -_WSTOPSIG(sts) 1539 else: 1540 # Should never happen 1541 raise SubprocessError("Unknown child exit status!") 1542 1543 1544 def _internal_poll(self, _deadstate=None, _waitpid=os.waitpid, 1545 _WNOHANG=os.WNOHANG, _ECHILD=errno.ECHILD): 1546 """Check if child process has terminated. Returns returncode 1547 attribute. 1548 1549 This method is called by __del__, so it cannot reference anything 1550 outside of the local scope (nor can any methods it calls). 1551 1552 """ 1553 if self.returncode is None: 1554 if not self._waitpid_lock.acquire(False): 1555 # Something else is busy calling waitpid. Don't allow two 1556 # at once. We know nothing yet. 1557 return None 1558 try: 1559 if self.returncode is not None: 1560 return self.returncode # Another thread waited. 1561 pid, sts = _waitpid(self.pid, _WNOHANG) 1562 if pid == self.pid: 1563 self._handle_exitstatus(sts) 1564 except OSError as e: 1565 if _deadstate is not None: 1566 self.returncode = _deadstate 1567 elif e.errno == _ECHILD: 1568 # This happens if SIGCLD is set to be ignored or 1569 # waiting for child processes has otherwise been 1570 # disabled for our process. This child is dead, we 1571 # can't get the status. 1572 # http://bugs.python.org/issue15756 1573 self.returncode = 0 1574 finally: 1575 self._waitpid_lock.release() 1576 return self.returncode 1577 1578 1579 def _try_wait(self, wait_flags): 1580 """All callers to this function MUST hold self._waitpid_lock.""" 1581 try: 1582 (pid, sts) = os.waitpid(self.pid, wait_flags) 1583 except ChildProcessError: 1584 # This happens if SIGCLD is set to be ignored or waiting 1585 # for child processes has otherwise been disabled for our 1586 # process. This child is dead, we can't get the status. 1587 pid = self.pid 1588 sts = 0 1589 return (pid, sts) 1590 1591 1592 def _wait(self, timeout): 1593 """Internal implementation of wait() on POSIX.""" 1594 if self.returncode is not None: 1595 return self.returncode 1596 1597 if timeout is not None: 1598 endtime = _time() + timeout 1599 # Enter a busy loop if we have a timeout. This busy loop was 1600 # cribbed from Lib/threading.py in Thread.wait() at r71065. 1601 delay = 0.0005 # 500 us -> initial delay of 1 ms 1602 while True: 1603 if self._waitpid_lock.acquire(False): 1604 try: 1605 if self.returncode is not None: 1606 break # Another thread waited. 1607 (pid, sts) = self._try_wait(os.WNOHANG) 1608 assert pid == self.pid or pid == 0 1609 if pid == self.pid: 1610 self._handle_exitstatus(sts) 1611 break 1612 finally: 1613 self._waitpid_lock.release() 1614 remaining = self._remaining_time(endtime) 1615 if remaining <= 0: 1616 raise TimeoutExpired(self.args, timeout) 1617 delay = min(delay * 2, remaining, .05) 1618 time.sleep(delay) 1619 else: 1620 while self.returncode is None: 1621 with self._waitpid_lock: 1622 if self.returncode is not None: 1623 break # Another thread waited. 1624 (pid, sts) = self._try_wait(0) 1625 # Check the pid and loop as waitpid has been known to 1626 # return 0 even without WNOHANG in odd situations. 1627 # http://bugs.python.org/issue14396. 1628 if pid == self.pid: 1629 self._handle_exitstatus(sts) 1630 return self.returncode 1631 1632 1633 def _communicate(self, input, endtime, orig_timeout): 1634 if self.stdin and not self._communication_started: 1635 # Flush stdio buffer. This might block, if the user has 1636 # been writing to .stdin in an uncontrolled fashion. 1637 try: 1638 self.stdin.flush() 1639 except BrokenPipeError: 1640 pass # communicate() must ignore BrokenPipeError. 1641 if not input: 1642 try: 1643 self.stdin.close() 1644 except BrokenPipeError: 1645 pass # communicate() must ignore BrokenPipeError. 1646 1647 stdout = None 1648 stderr = None 1649 1650 # Only create this mapping if we haven't already. 1651 if not self._communication_started: 1652 self._fileobj2output = {} 1653 if self.stdout: 1654 self._fileobj2output[self.stdout] = [] 1655 if self.stderr: 1656 self._fileobj2output[self.stderr] = [] 1657 1658 if self.stdout: 1659 stdout = self._fileobj2output[self.stdout] 1660 if self.stderr: 1661 stderr = self._fileobj2output[self.stderr] 1662 1663 self._save_input(input) 1664 1665 if self._input: 1666 input_view = memoryview(self._input) 1667 1668 with _PopenSelector() as selector: 1669 if self.stdin and input: 1670 selector.register(self.stdin, selectors.EVENT_WRITE) 1671 if self.stdout: 1672 selector.register(self.stdout, selectors.EVENT_READ) 1673 if self.stderr: 1674 selector.register(self.stderr, selectors.EVENT_READ) 1675 1676 while selector.get_map(): 1677 timeout = self._remaining_time(endtime) 1678 if timeout is not None and timeout < 0: 1679 raise TimeoutExpired(self.args, orig_timeout) 1680 1681 ready = selector.select(timeout) 1682 self._check_timeout(endtime, orig_timeout) 1683 1684 # XXX Rewrite these to use non-blocking I/O on the file 1685 # objects; they are no longer using C stdio! 1686 1687 for key, events in ready: 1688 if key.fileobj is self.stdin: 1689 chunk = input_view[self._input_offset : 1690 self._input_offset + _PIPE_BUF] 1691 try: 1692 self._input_offset += os.write(key.fd, chunk) 1693 except BrokenPipeError: 1694 selector.unregister(key.fileobj) 1695 key.fileobj.close() 1696 else: 1697 if self._input_offset >= len(self._input): 1698 selector.unregister(key.fileobj) 1699 key.fileobj.close() 1700 elif key.fileobj in (self.stdout, self.stderr): 1701 data = os.read(key.fd, 32768) 1702 if not data: 1703 selector.unregister(key.fileobj) 1704 key.fileobj.close() 1705 self._fileobj2output[key.fileobj].append(data) 1706 1707 self.wait(timeout=self._remaining_time(endtime)) 1708 1709 # All data exchanged. Translate lists into strings. 1710 if stdout is not None: 1711 stdout = b''.join(stdout) 1712 if stderr is not None: 1713 stderr = b''.join(stderr) 1714 1715 # Translate newlines, if requested. 1716 # This also turns bytes into strings. 1717 if self.text_mode: 1718 if stdout is not None: 1719 stdout = self._translate_newlines(stdout, 1720 self.stdout.encoding, 1721 self.stdout.errors) 1722 if stderr is not None: 1723 stderr = self._translate_newlines(stderr, 1724 self.stderr.encoding, 1725 self.stderr.errors) 1726 1727 return (stdout, stderr) 1728 1729 1730 def _save_input(self, input): 1731 # This method is called from the _communicate_with_*() methods 1732 # so that if we time out while communicating, we can continue 1733 # sending input if we retry. 1734 if self.stdin and self._input is None: 1735 self._input_offset = 0 1736 self._input = input 1737 if input is not None and self.text_mode: 1738 self._input = self._input.encode(self.stdin.encoding, 1739 self.stdin.errors) 1740 1741 1742 def send_signal(self, sig): 1743 """Send a signal to the process.""" 1744 # Skip signalling a process that we know has already died. 1745 if self.returncode is None: 1746 os.kill(self.pid, sig) 1747 1748 def terminate(self): 1749 """Terminate the process with SIGTERM 1750 """ 1751 self.send_signal(signal.SIGTERM) 1752 1753 def kill(self): 1754 """Kill the process with SIGKILL 1755 """ 1756 self.send_signal(signal.SIGKILL) 1757