1# Copyright (c) 2009, Google Inc. All rights reserved. 2# Copyright (c) 2009 Apple Inc. All rights reserved. 3# 4# Redistribution and use in source and binary forms, with or without 5# modification, are permitted provided that the following conditions are 6# met: 7# 8# * Redistributions of source code must retain the above copyright 9# notice, this list of conditions and the following disclaimer. 10# * Redistributions in binary form must reproduce the above 11# copyright notice, this list of conditions and the following disclaimer 12# in the documentation and/or other materials provided with the 13# distribution. 14# * Neither the name of Google Inc. nor the names of its 15# contributors may be used to endorse or promote products derived from 16# this software without specific prior written permission. 17# 18# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 19# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 20# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 21# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 22# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 23# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 24# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 25# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 26# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 27# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 28# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 30import errno 31import logging 32import multiprocessing 33import os 34import StringIO 35import signal 36import subprocess 37import sys 38import time 39 40from webkitpy.common.system.outputtee import Tee 41from webkitpy.common.system.filesystem import FileSystem 42 43 44_log = logging.getLogger(__name__) 45 46 47class ScriptError(Exception): 48 49 def __init__(self, 50 message=None, 51 script_args=None, 52 exit_code=None, 53 output=None, 54 cwd=None, 55 output_limit=500): 56 shortened_output = output 57 if output and output_limit and len(output) > output_limit: 58 shortened_output = "Last %s characters of output:\n%s" % (output_limit, output[-output_limit:]) 59 60 if not message: 61 message = 'Failed to run "%s"' % repr(script_args) 62 if exit_code: 63 message += " exit_code: %d" % exit_code 64 if cwd: 65 message += " cwd: %s" % cwd 66 67 if shortened_output: 68 message += "\n\noutput: %s" % shortened_output 69 70 Exception.__init__(self, message) 71 self.script_args = script_args # 'args' is already used by Exception 72 self.exit_code = exit_code 73 self.output = output 74 self.cwd = cwd 75 76 def message_with_output(self): 77 return unicode(self) 78 79 def command_name(self): 80 command_path = self.script_args 81 if type(command_path) is list: 82 command_path = command_path[0] 83 return os.path.basename(command_path) 84 85 86class Executive(object): 87 PIPE = subprocess.PIPE 88 STDOUT = subprocess.STDOUT 89 90 def _should_close_fds(self): 91 # We need to pass close_fds=True to work around Python bug #2320 92 # (otherwise we can hang when we kill DumpRenderTree when we are running 93 # multiple threads). See http://bugs.python.org/issue2320 . 94 # Note that close_fds isn't supported on Windows, but this bug only 95 # shows up on Mac and Linux. 96 return sys.platform not in ('win32', 'cygwin') 97 98 def _run_command_with_teed_output(self, args, teed_output, **kwargs): 99 child_process = self.popen(args, 100 stdout=self.PIPE, 101 stderr=self.STDOUT, 102 close_fds=self._should_close_fds(), 103 **kwargs) 104 105 # Use our own custom wait loop because Popen ignores a tee'd 106 # stderr/stdout. 107 # FIXME: This could be improved not to flatten output to stdout. 108 while True: 109 output_line = child_process.stdout.readline() 110 if output_line == "" and child_process.poll() != None: 111 # poll() is not threadsafe and can throw OSError due to: 112 # http://bugs.python.org/issue1731717 113 return child_process.poll() 114 # We assume that the child process wrote to us in utf-8, 115 # so no re-encoding is necessary before writing here. 116 teed_output.write(output_line) 117 118 # FIXME: Remove this deprecated method and move callers to run_command. 119 # FIXME: This method is a hack to allow running command which both 120 # capture their output and print out to stdin. Useful for things 121 # like "build-webkit" where we want to display to the user that we're building 122 # but still have the output to stuff into a log file. 123 def run_and_throw_if_fail(self, args, quiet=False, decode_output=True, **kwargs): 124 # Cache the child's output locally so it can be used for error reports. 125 child_out_file = StringIO.StringIO() 126 tee_stdout = sys.stdout 127 if quiet: 128 dev_null = open(os.devnull, "w") # FIXME: Does this need an encoding? 129 tee_stdout = dev_null 130 child_stdout = Tee(child_out_file, tee_stdout) 131 exit_code = self._run_command_with_teed_output(args, child_stdout, **kwargs) 132 if quiet: 133 dev_null.close() 134 135 child_output = child_out_file.getvalue() 136 child_out_file.close() 137 138 if decode_output: 139 child_output = child_output.decode(self._child_process_encoding()) 140 141 if exit_code: 142 raise ScriptError(script_args=args, 143 exit_code=exit_code, 144 output=child_output) 145 return child_output 146 147 def cpu_count(self): 148 return multiprocessing.cpu_count() 149 150 @staticmethod 151 def interpreter_for_script(script_path, fs=None): 152 fs = fs or FileSystem() 153 lines = fs.read_text_file(script_path).splitlines() 154 if not len(lines): 155 return None 156 first_line = lines[0] 157 if not first_line.startswith('#!'): 158 return None 159 if first_line.find('python') > -1: 160 return sys.executable 161 if first_line.find('perl') > -1: 162 return 'perl' 163 if first_line.find('ruby') > -1: 164 return 'ruby' 165 return None 166 167 @staticmethod 168 def shell_command_for_script(script_path, fs=None): 169 fs = fs or FileSystem() 170 # Win32 does not support shebang. We need to detect the interpreter ourself. 171 if sys.platform == 'win32': 172 interpreter = Executive.interpreter_for_script(script_path, fs) 173 if interpreter: 174 return [interpreter, script_path] 175 return [script_path] 176 177 def kill_process(self, pid): 178 """Attempts to kill the given pid. 179 Will fail silently if pid does not exist or insufficient permisssions.""" 180 if sys.platform == "win32": 181 # We only use taskkill.exe on windows (not cygwin) because subprocess.pid 182 # is a CYGWIN pid and taskkill.exe expects a windows pid. 183 # Thankfully os.kill on CYGWIN handles either pid type. 184 command = ["taskkill.exe", "/f", "/pid", pid] 185 # taskkill will exit 128 if the process is not found. We should log. 186 self.run_command(command, error_handler=self.ignore_error) 187 return 188 189 # According to http://docs.python.org/library/os.html 190 # os.kill isn't available on Windows. python 2.5.5 os.kill appears 191 # to work in cygwin, however it occasionally raises EAGAIN. 192 retries_left = 10 if sys.platform == "cygwin" else 1 193 while retries_left > 0: 194 try: 195 retries_left -= 1 196 os.kill(pid, signal.SIGKILL) 197 _ = os.waitpid(pid, os.WNOHANG) 198 except OSError, e: 199 if e.errno == errno.EAGAIN: 200 if retries_left <= 0: 201 _log.warn("Failed to kill pid %s. Too many EAGAIN errors." % pid) 202 continue 203 if e.errno == errno.ESRCH: # The process does not exist. 204 return 205 if e.errno == errno.EPIPE: # The process has exited already on cygwin 206 return 207 if e.errno == errno.ECHILD: 208 # Can't wait on a non-child process, but the kill worked. 209 return 210 if e.errno == errno.EACCES and sys.platform == 'cygwin': 211 # Cygwin python sometimes can't kill native processes. 212 return 213 raise 214 215 def _win32_check_running_pid(self, pid): 216 # importing ctypes at the top-level seems to cause weird crashes at 217 # exit under cygwin on apple's win port. Only win32 needs cygwin, so 218 # we import it here instead. See https://bugs.webkit.org/show_bug.cgi?id=91682 219 import ctypes 220 221 class PROCESSENTRY32(ctypes.Structure): 222 _fields_ = [("dwSize", ctypes.c_ulong), 223 ("cntUsage", ctypes.c_ulong), 224 ("th32ProcessID", ctypes.c_ulong), 225 ("th32DefaultHeapID", ctypes.POINTER(ctypes.c_ulong)), 226 ("th32ModuleID", ctypes.c_ulong), 227 ("cntThreads", ctypes.c_ulong), 228 ("th32ParentProcessID", ctypes.c_ulong), 229 ("pcPriClassBase", ctypes.c_ulong), 230 ("dwFlags", ctypes.c_ulong), 231 ("szExeFile", ctypes.c_char * 260)] 232 233 CreateToolhelp32Snapshot = ctypes.windll.kernel32.CreateToolhelp32Snapshot 234 Process32First = ctypes.windll.kernel32.Process32First 235 Process32Next = ctypes.windll.kernel32.Process32Next 236 CloseHandle = ctypes.windll.kernel32.CloseHandle 237 TH32CS_SNAPPROCESS = 0x00000002 # win32 magic number 238 hProcessSnap = CreateToolhelp32Snapshot(TH32CS_SNAPPROCESS, 0) 239 pe32 = PROCESSENTRY32() 240 pe32.dwSize = ctypes.sizeof(PROCESSENTRY32) 241 result = False 242 if not Process32First(hProcessSnap, ctypes.byref(pe32)): 243 _log.debug("Failed getting first process.") 244 CloseHandle(hProcessSnap) 245 return result 246 while True: 247 if pe32.th32ProcessID == pid: 248 result = True 249 break 250 if not Process32Next(hProcessSnap, ctypes.byref(pe32)): 251 break 252 CloseHandle(hProcessSnap) 253 return result 254 255 def check_running_pid(self, pid): 256 """Return True if pid is alive, otherwise return False.""" 257 if sys.platform == 'win32': 258 return self._win32_check_running_pid(pid) 259 260 try: 261 os.kill(pid, 0) 262 return True 263 except OSError: 264 return False 265 266 def running_pids(self, process_name_filter=None): 267 if not process_name_filter: 268 process_name_filter = lambda process_name: True 269 270 running_pids = [] 271 272 if sys.platform in ("win32", "cygwin"): 273 # FIXME: running_pids isn't implemented on Windows yet... 274 return [] 275 276 ps_process = self.popen(['ps', '-eo', 'pid,comm'], stdout=self.PIPE, stderr=self.PIPE) 277 stdout, _ = ps_process.communicate() 278 for line in stdout.splitlines(): 279 try: 280 # In some cases the line can contain one or more 281 # leading white-spaces, so strip it before split. 282 pid, process_name = line.strip().split(' ', 1) 283 if process_name_filter(process_name): 284 running_pids.append(int(pid)) 285 except ValueError, e: 286 pass 287 288 return sorted(running_pids) 289 290 def wait_newest(self, process_name_filter=None): 291 if not process_name_filter: 292 process_name_filter = lambda process_name: True 293 294 running_pids = self.running_pids(process_name_filter) 295 if not running_pids: 296 return 297 pid = running_pids[-1] 298 299 while self.check_running_pid(pid): 300 time.sleep(0.25) 301 302 def wait_limited(self, pid, limit_in_seconds=None, check_frequency_in_seconds=None): 303 seconds_left = limit_in_seconds or 10 304 sleep_length = check_frequency_in_seconds or 1 305 while seconds_left > 0 and self.check_running_pid(pid): 306 seconds_left -= sleep_length 307 time.sleep(sleep_length) 308 309 def _windows_image_name(self, process_name): 310 name, extension = os.path.splitext(process_name) 311 if not extension: 312 # taskkill expects processes to end in .exe 313 # If necessary we could add a flag to disable appending .exe. 314 process_name = "%s.exe" % name 315 return process_name 316 317 def interrupt(self, pid): 318 interrupt_signal = signal.SIGINT 319 # FIXME: The python docs seem to imply that platform == 'win32' may need to use signal.CTRL_C_EVENT 320 # http://docs.python.org/2/library/signal.html 321 try: 322 os.kill(pid, interrupt_signal) 323 except OSError: 324 # Silently ignore when the pid doesn't exist. 325 # It's impossible for callers to avoid race conditions with process shutdown. 326 pass 327 328 def kill_all(self, process_name): 329 """Attempts to kill processes matching process_name. 330 Will fail silently if no process are found.""" 331 if sys.platform in ("win32", "cygwin"): 332 image_name = self._windows_image_name(process_name) 333 command = ["taskkill.exe", "/f", "/im", image_name] 334 # taskkill will exit 128 if the process is not found. We should log. 335 self.run_command(command, error_handler=self.ignore_error) 336 return 337 338 # FIXME: This is inconsistent that kill_all uses TERM and kill_process 339 # uses KILL. Windows is always using /f (which seems like -KILL). 340 # We should pick one mode, or add support for switching between them. 341 # Note: Mac OS X 10.6 requires -SIGNALNAME before -u USER 342 command = ["killall", "-TERM", "-u", os.getenv("USER"), process_name] 343 # killall returns 1 if no process can be found and 2 on command error. 344 # FIXME: We should pass a custom error_handler to allow only exit_code 1. 345 # We should log in exit_code == 1 346 self.run_command(command, error_handler=self.ignore_error) 347 348 # Error handlers do not need to be static methods once all callers are 349 # updated to use an Executive object. 350 351 @staticmethod 352 def default_error_handler(error): 353 raise error 354 355 @staticmethod 356 def ignore_error(error): 357 pass 358 359 def _compute_stdin(self, input): 360 """Returns (stdin, string_to_communicate)""" 361 # FIXME: We should be returning /dev/null for stdin 362 # or closing stdin after process creation to prevent 363 # child processes from getting input from the user. 364 if not input: 365 return (None, None) 366 if hasattr(input, "read"): # Check if the input is a file. 367 return (input, None) # Assume the file is in the right encoding. 368 369 # Popen in Python 2.5 and before does not automatically encode unicode objects. 370 # http://bugs.python.org/issue5290 371 # See https://bugs.webkit.org/show_bug.cgi?id=37528 372 # for an example of a regresion caused by passing a unicode string directly. 373 # FIXME: We may need to encode differently on different platforms. 374 if isinstance(input, unicode): 375 input = input.encode(self._child_process_encoding()) 376 return (self.PIPE, input) 377 378 def command_for_printing(self, args): 379 """Returns a print-ready string representing command args. 380 The string should be copy/paste ready for execution in a shell.""" 381 args = self._stringify_args(args) 382 escaped_args = [] 383 for arg in args: 384 if isinstance(arg, unicode): 385 # Escape any non-ascii characters for easy copy/paste 386 arg = arg.encode("unicode_escape") 387 # FIXME: Do we need to fix quotes here? 388 escaped_args.append(arg) 389 return " ".join(escaped_args) 390 391 # FIXME: run_and_throw_if_fail should be merged into this method. 392 def run_command(self, 393 args, 394 cwd=None, 395 env=None, 396 input=None, 397 error_handler=None, 398 return_exit_code=False, 399 return_stderr=True, 400 decode_output=True, debug_logging=True): 401 """Popen wrapper for convenience and to work around python bugs.""" 402 assert(isinstance(args, list) or isinstance(args, tuple)) 403 start_time = time.time() 404 405 stdin, string_to_communicate = self._compute_stdin(input) 406 stderr = self.STDOUT if return_stderr else None 407 408 process = self.popen(args, 409 stdin=stdin, 410 stdout=self.PIPE, 411 stderr=stderr, 412 cwd=cwd, 413 env=env, 414 close_fds=self._should_close_fds()) 415 output = process.communicate(string_to_communicate)[0] 416 417 # run_command automatically decodes to unicode() unless explicitly told not to. 418 if decode_output: 419 output = output.decode(self._child_process_encoding()) 420 421 # wait() is not threadsafe and can throw OSError due to: 422 # http://bugs.python.org/issue1731717 423 exit_code = process.wait() 424 425 if debug_logging: 426 _log.debug('"%s" took %.2fs' % (self.command_for_printing(args), time.time() - start_time)) 427 428 if return_exit_code: 429 return exit_code 430 431 if exit_code: 432 script_error = ScriptError(script_args=args, 433 exit_code=exit_code, 434 output=output, 435 cwd=cwd) 436 (error_handler or self.default_error_handler)(script_error) 437 return output 438 439 def _child_process_encoding(self): 440 # Win32 Python 2.x uses CreateProcessA rather than CreateProcessW 441 # to launch subprocesses, so we have to encode arguments using the 442 # current code page. 443 if sys.platform == 'win32' and sys.version < '3': 444 return 'mbcs' 445 # All other platforms use UTF-8. 446 # FIXME: Using UTF-8 on Cygwin will confuse Windows-native commands 447 # which will expect arguments to be encoded using the current code 448 # page. 449 return 'utf-8' 450 451 def _should_encode_child_process_arguments(self): 452 # Cygwin's Python's os.execv doesn't support unicode command 453 # arguments, and neither does Cygwin's execv itself. 454 if sys.platform == 'cygwin': 455 return True 456 457 # Win32 Python 2.x uses CreateProcessA rather than CreateProcessW 458 # to launch subprocesses, so we have to encode arguments using the 459 # current code page. 460 if sys.platform == 'win32' and sys.version < '3': 461 return True 462 463 return False 464 465 def _encode_argument_if_needed(self, argument): 466 if not self._should_encode_child_process_arguments(): 467 return argument 468 return argument.encode(self._child_process_encoding()) 469 470 def _stringify_args(self, args): 471 # Popen will throw an exception if args are non-strings (like int()) 472 string_args = map(unicode, args) 473 # The Windows implementation of Popen cannot handle unicode strings. :( 474 return map(self._encode_argument_if_needed, string_args) 475 476 # The only required arugment to popen is named "args", the rest are optional keyword arguments. 477 def popen(self, args, **kwargs): 478 # FIXME: We should always be stringifying the args, but callers who pass shell=True 479 # expect that the exact bytes passed will get passed to the shell (even if they're wrongly encoded). 480 # shell=True is wrong for many other reasons, and we should remove this 481 # hack as soon as we can fix all callers to not use shell=True. 482 if kwargs.get('shell') == True: 483 string_args = args 484 else: 485 string_args = self._stringify_args(args) 486 return subprocess.Popen(string_args, **kwargs) 487 488 def call(self, args, **kwargs): 489 return subprocess.call(self._stringify_args(args), **kwargs) 490 491 def run_in_parallel(self, command_lines_and_cwds, processes=None): 492 """Runs a list of (cmd_line list, cwd string) tuples in parallel and returns a list of (retcode, stdout, stderr) tuples.""" 493 assert len(command_lines_and_cwds) 494 495 if sys.platform in ('cygwin', 'win32'): 496 return map(_run_command_thunk, command_lines_and_cwds) 497 pool = multiprocessing.Pool(processes=processes) 498 results = pool.map(_run_command_thunk, command_lines_and_cwds) 499 pool.close() 500 pool.join() 501 return results 502 503 504def _run_command_thunk(cmd_line_and_cwd): 505 # Note that this needs to be a bare module (and hence Picklable) method to work with multiprocessing.Pool. 506 (cmd_line, cwd) = cmd_line_and_cwd 507 proc = subprocess.Popen(cmd_line, cwd=cwd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 508 stdout, stderr = proc.communicate() 509 return (proc.returncode, stdout, stderr) 510