1# Copyright (c) 2009, Google Inc. All rights reserved. 2# Copyright (c) 2009 Apple Inc. All rights reserved. 3# 4# Redistribution and use in source and binary forms, with or without 5# modification, are permitted provided that the following conditions are 6# met: 7# 8# * Redistributions of source code must retain the above copyright 9# notice, this list of conditions and the following disclaimer. 10# * Redistributions in binary form must reproduce the above 11# copyright notice, this list of conditions and the following disclaimer 12# in the documentation and/or other materials provided with the 13# distribution. 14# * Neither the name of Google Inc. nor the names of its 15# contributors may be used to endorse or promote products derived from 16# this software without specific prior written permission. 17# 18# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 19# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 20# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 21# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 22# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 23# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 24# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 25# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 26# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 27# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 28# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 30try: 31 # This API exists only in Python 2.6 and higher. :( 32 import multiprocessing 33except ImportError: 34 multiprocessing = None 35 36import ctypes 37import errno 38import logging 39import os 40import platform 41import StringIO 42import signal 43import subprocess 44import sys 45import time 46 47from webkitpy.common.system.deprecated_logging import tee 48from webkitpy.common.system.filesystem import FileSystem 49from webkitpy.python24 import versioning 50 51 52_log = logging.getLogger("webkitpy.common.system") 53 54 55class ScriptError(Exception): 56 57 # This is a custom List.__str__ implementation to allow size limiting. 58 def _string_from_args(self, args, limit=100): 59 args_string = unicode(args) 60 # We could make this much fancier, but for now this is OK. 61 if len(args_string) > limit: 62 return args_string[:limit - 3] + "..." 63 return args_string 64 65 def __init__(self, 66 message=None, 67 script_args=None, 68 exit_code=None, 69 output=None, 70 cwd=None): 71 if not message: 72 message = 'Failed to run "%s"' % self._string_from_args(script_args) 73 if exit_code: 74 message += " exit_code: %d" % exit_code 75 if cwd: 76 message += " cwd: %s" % cwd 77 78 Exception.__init__(self, message) 79 self.script_args = script_args # 'args' is already used by Exception 80 self.exit_code = exit_code 81 self.output = output 82 self.cwd = cwd 83 84 def message_with_output(self, output_limit=500): 85 if self.output: 86 if output_limit and len(self.output) > output_limit: 87 return u"%s\n\nLast %s characters of output:\n%s" % \ 88 (self, output_limit, self.output[-output_limit:]) 89 return u"%s\n\n%s" % (self, self.output) 90 return unicode(self) 91 92 def command_name(self): 93 command_path = self.script_args 94 if type(command_path) is list: 95 command_path = command_path[0] 96 return os.path.basename(command_path) 97 98 99def run_command(*args, **kwargs): 100 # FIXME: This should not be a global static. 101 # New code should use Executive.run_command directly instead 102 return Executive().run_command(*args, **kwargs) 103 104 105class Executive(object): 106 107 def _should_close_fds(self): 108 # We need to pass close_fds=True to work around Python bug #2320 109 # (otherwise we can hang when we kill DumpRenderTree when we are running 110 # multiple threads). See http://bugs.python.org/issue2320 . 111 # Note that close_fds isn't supported on Windows, but this bug only 112 # shows up on Mac and Linux. 113 return sys.platform not in ('win32', 'cygwin') 114 115 def _run_command_with_teed_output(self, args, teed_output): 116 args = map(unicode, args) # Popen will throw an exception if args are non-strings (like int()) 117 args = map(self._encode_argument_if_needed, args) 118 119 child_process = subprocess.Popen(args, 120 stdout=subprocess.PIPE, 121 stderr=subprocess.STDOUT, 122 close_fds=self._should_close_fds()) 123 124 # Use our own custom wait loop because Popen ignores a tee'd 125 # stderr/stdout. 126 # FIXME: This could be improved not to flatten output to stdout. 127 while True: 128 output_line = child_process.stdout.readline() 129 if output_line == "" and child_process.poll() != None: 130 # poll() is not threadsafe and can throw OSError due to: 131 # http://bugs.python.org/issue1731717 132 return child_process.poll() 133 # We assume that the child process wrote to us in utf-8, 134 # so no re-encoding is necessary before writing here. 135 teed_output.write(output_line) 136 137 # FIXME: Remove this deprecated method and move callers to run_command. 138 # FIXME: This method is a hack to allow running command which both 139 # capture their output and print out to stdin. Useful for things 140 # like "build-webkit" where we want to display to the user that we're building 141 # but still have the output to stuff into a log file. 142 def run_and_throw_if_fail(self, args, quiet=False, decode_output=True): 143 # Cache the child's output locally so it can be used for error reports. 144 child_out_file = StringIO.StringIO() 145 tee_stdout = sys.stdout 146 if quiet: 147 dev_null = open(os.devnull, "w") # FIXME: Does this need an encoding? 148 tee_stdout = dev_null 149 child_stdout = tee(child_out_file, tee_stdout) 150 exit_code = self._run_command_with_teed_output(args, child_stdout) 151 if quiet: 152 dev_null.close() 153 154 child_output = child_out_file.getvalue() 155 child_out_file.close() 156 157 if decode_output: 158 child_output = child_output.decode(self._child_process_encoding()) 159 160 if exit_code: 161 raise ScriptError(script_args=args, 162 exit_code=exit_code, 163 output=child_output) 164 return child_output 165 166 def cpu_count(self): 167 if multiprocessing: 168 return multiprocessing.cpu_count() 169 # Darn. We don't have the multiprocessing package. 170 system_name = platform.system() 171 if system_name == "Darwin": 172 return int(self.run_command(["sysctl", "-n", "hw.ncpu"])) 173 elif system_name == "Windows": 174 return int(os.environ.get('NUMBER_OF_PROCESSORS', 1)) 175 elif system_name == "Linux": 176 num_cores = os.sysconf("SC_NPROCESSORS_ONLN") 177 if isinstance(num_cores, int) and num_cores > 0: 178 return num_cores 179 # This quantity is a lie but probably a reasonable guess for modern 180 # machines. 181 return 2 182 183 @staticmethod 184 def interpreter_for_script(script_path, fs=FileSystem()): 185 lines = fs.read_text_file(script_path).splitlines() 186 if not len(lines): 187 return None 188 first_line = lines[0] 189 if not first_line.startswith('#!'): 190 return None 191 if first_line.find('python') > -1: 192 return sys.executable 193 if first_line.find('perl') > -1: 194 return 'perl' 195 if first_line.find('ruby') > -1: 196 return 'ruby' 197 return None 198 199 def kill_process(self, pid): 200 """Attempts to kill the given pid. 201 Will fail silently if pid does not exist or insufficient permisssions.""" 202 if sys.platform == "win32": 203 # We only use taskkill.exe on windows (not cygwin) because subprocess.pid 204 # is a CYGWIN pid and taskkill.exe expects a windows pid. 205 # Thankfully os.kill on CYGWIN handles either pid type. 206 command = ["taskkill.exe", "/f", "/pid", pid] 207 # taskkill will exit 128 if the process is not found. We should log. 208 self.run_command(command, error_handler=self.ignore_error) 209 return 210 211 # According to http://docs.python.org/library/os.html 212 # os.kill isn't available on Windows. python 2.5.5 os.kill appears 213 # to work in cygwin, however it occasionally raises EAGAIN. 214 retries_left = 10 if sys.platform == "cygwin" else 1 215 while retries_left > 0: 216 try: 217 retries_left -= 1 218 os.kill(pid, signal.SIGKILL) 219 except OSError, e: 220 if e.errno == errno.EAGAIN: 221 if retries_left <= 0: 222 _log.warn("Failed to kill pid %s. Too many EAGAIN errors." % pid) 223 continue 224 if e.errno == errno.ESRCH: # The process does not exist. 225 _log.warn("Called kill_process with a non-existant pid %s" % pid) 226 return 227 raise 228 229 def _win32_check_running_pid(self, pid): 230 231 class PROCESSENTRY32(ctypes.Structure): 232 _fields_ = [("dwSize", ctypes.c_ulong), 233 ("cntUsage", ctypes.c_ulong), 234 ("th32ProcessID", ctypes.c_ulong), 235 ("th32DefaultHeapID", ctypes.c_ulong), 236 ("th32ModuleID", ctypes.c_ulong), 237 ("cntThreads", ctypes.c_ulong), 238 ("th32ParentProcessID", ctypes.c_ulong), 239 ("pcPriClassBase", ctypes.c_ulong), 240 ("dwFlags", ctypes.c_ulong), 241 ("szExeFile", ctypes.c_char * 260)] 242 243 CreateToolhelp32Snapshot = ctypes.windll.kernel32.CreateToolhelp32Snapshot 244 Process32First = ctypes.windll.kernel32.Process32First 245 Process32Next = ctypes.windll.kernel32.Process32Next 246 CloseHandle = ctypes.windll.kernel32.CloseHandle 247 TH32CS_SNAPPROCESS = 0x00000002 # win32 magic number 248 hProcessSnap = CreateToolhelp32Snapshot(TH32CS_SNAPPROCESS, 0) 249 pe32 = PROCESSENTRY32() 250 pe32.dwSize = ctypes.sizeof(PROCESSENTRY32) 251 result = False 252 if not Process32First(hProcessSnap, ctypes.byref(pe32)): 253 _log.debug("Failed getting first process.") 254 CloseHandle(hProcessSnap) 255 return result 256 while True: 257 if pe32.th32ProcessID == pid: 258 result = True 259 break 260 if not Process32Next(hProcessSnap, ctypes.byref(pe32)): 261 break 262 CloseHandle(hProcessSnap) 263 return result 264 265 def check_running_pid(self, pid): 266 """Return True if pid is alive, otherwise return False.""" 267 if sys.platform in ('darwin', 'linux2', 'cygwin'): 268 try: 269 os.kill(pid, 0) 270 return True 271 except OSError: 272 return False 273 elif sys.platform == 'win32': 274 return self._win32_check_running_pid(pid) 275 276 assert(False) 277 278 def _windows_image_name(self, process_name): 279 name, extension = os.path.splitext(process_name) 280 if not extension: 281 # taskkill expects processes to end in .exe 282 # If necessary we could add a flag to disable appending .exe. 283 process_name = "%s.exe" % name 284 return process_name 285 286 def kill_all(self, process_name): 287 """Attempts to kill processes matching process_name. 288 Will fail silently if no process are found.""" 289 if sys.platform in ("win32", "cygwin"): 290 image_name = self._windows_image_name(process_name) 291 command = ["taskkill.exe", "/f", "/im", image_name] 292 # taskkill will exit 128 if the process is not found. We should log. 293 self.run_command(command, error_handler=self.ignore_error) 294 return 295 296 # FIXME: This is inconsistent that kill_all uses TERM and kill_process 297 # uses KILL. Windows is always using /f (which seems like -KILL). 298 # We should pick one mode, or add support for switching between them. 299 # Note: Mac OS X 10.6 requires -SIGNALNAME before -u USER 300 command = ["killall", "-TERM", "-u", os.getenv("USER"), process_name] 301 # killall returns 1 if no process can be found and 2 on command error. 302 # FIXME: We should pass a custom error_handler to allow only exit_code 1. 303 # We should log in exit_code == 1 304 self.run_command(command, error_handler=self.ignore_error) 305 306 # Error handlers do not need to be static methods once all callers are 307 # updated to use an Executive object. 308 309 @staticmethod 310 def default_error_handler(error): 311 raise error 312 313 @staticmethod 314 def ignore_error(error): 315 pass 316 317 def _compute_stdin(self, input): 318 """Returns (stdin, string_to_communicate)""" 319 # FIXME: We should be returning /dev/null for stdin 320 # or closing stdin after process creation to prevent 321 # child processes from getting input from the user. 322 if not input: 323 return (None, None) 324 if hasattr(input, "read"): # Check if the input is a file. 325 return (input, None) # Assume the file is in the right encoding. 326 327 # Popen in Python 2.5 and before does not automatically encode unicode objects. 328 # http://bugs.python.org/issue5290 329 # See https://bugs.webkit.org/show_bug.cgi?id=37528 330 # for an example of a regresion caused by passing a unicode string directly. 331 # FIXME: We may need to encode differently on different platforms. 332 if isinstance(input, unicode): 333 input = input.encode(self._child_process_encoding()) 334 return (subprocess.PIPE, input) 335 336 def _command_for_printing(self, args): 337 """Returns a print-ready string representing command args. 338 The string should be copy/paste ready for execution in a shell.""" 339 escaped_args = [] 340 for arg in args: 341 if isinstance(arg, unicode): 342 # Escape any non-ascii characters for easy copy/paste 343 arg = arg.encode("unicode_escape") 344 # FIXME: Do we need to fix quotes here? 345 escaped_args.append(arg) 346 return " ".join(escaped_args) 347 348 # FIXME: run_and_throw_if_fail should be merged into this method. 349 def run_command(self, 350 args, 351 cwd=None, 352 input=None, 353 error_handler=None, 354 return_exit_code=False, 355 return_stderr=True, 356 decode_output=True): 357 """Popen wrapper for convenience and to work around python bugs.""" 358 assert(isinstance(args, list) or isinstance(args, tuple)) 359 start_time = time.time() 360 args = map(unicode, args) # Popen will throw an exception if args are non-strings (like int()) 361 args = map(self._encode_argument_if_needed, args) 362 363 stdin, string_to_communicate = self._compute_stdin(input) 364 stderr = subprocess.STDOUT if return_stderr else None 365 366 process = subprocess.Popen(args, 367 stdin=stdin, 368 stdout=subprocess.PIPE, 369 stderr=stderr, 370 cwd=cwd, 371 close_fds=self._should_close_fds()) 372 output = process.communicate(string_to_communicate)[0] 373 374 # run_command automatically decodes to unicode() unless explicitly told not to. 375 if decode_output: 376 output = output.decode(self._child_process_encoding()) 377 378 # wait() is not threadsafe and can throw OSError due to: 379 # http://bugs.python.org/issue1731717 380 exit_code = process.wait() 381 382 _log.debug('"%s" took %.2fs' % (self._command_for_printing(args), time.time() - start_time)) 383 384 if return_exit_code: 385 return exit_code 386 387 if exit_code: 388 script_error = ScriptError(script_args=args, 389 exit_code=exit_code, 390 output=output, 391 cwd=cwd) 392 (error_handler or self.default_error_handler)(script_error) 393 return output 394 395 def _child_process_encoding(self): 396 # Win32 Python 2.x uses CreateProcessA rather than CreateProcessW 397 # to launch subprocesses, so we have to encode arguments using the 398 # current code page. 399 if sys.platform == 'win32' and versioning.compare_version(sys, '3.0')[0] < 0: 400 return 'mbcs' 401 # All other platforms use UTF-8. 402 # FIXME: Using UTF-8 on Cygwin will confuse Windows-native commands 403 # which will expect arguments to be encoded using the current code 404 # page. 405 return 'utf-8' 406 407 def _should_encode_child_process_arguments(self): 408 # Cygwin's Python's os.execv doesn't support unicode command 409 # arguments, and neither does Cygwin's execv itself. 410 if sys.platform == 'cygwin': 411 return True 412 413 # Win32 Python 2.x uses CreateProcessA rather than CreateProcessW 414 # to launch subprocesses, so we have to encode arguments using the 415 # current code page. 416 if sys.platform == 'win32' and versioning.compare_version(sys, '3.0')[0] < 0: 417 return True 418 419 return False 420 421 def _encode_argument_if_needed(self, argument): 422 if not self._should_encode_child_process_arguments(): 423 return argument 424 return argument.encode(self._child_process_encoding()) 425