1# Copyright (C) 2012 Google, Inc. 2# Copyright (C) 2010 Chris Jerdonek (cjerdonek@webkit.org) 3# 4# Redistribution and use in source and binary forms, with or without 5# modification, are permitted provided that the following conditions 6# are met: 7# 1. Redistributions of source code must retain the above copyright 8# notice, this list of conditions and the following disclaimer. 9# 2. Redistributions in binary form must reproduce the above copyright 10# notice, this list of conditions and the following disclaimer in the 11# documentation and/or other materials provided with the distribution. 12# 13# THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' AND 14# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 15# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 16# DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS BE LIABLE FOR 17# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 18# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 19# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 20# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 21# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 22# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 23 24"""unit testing code for webkitpy.""" 25 26import logging 27import multiprocessing 28import optparse 29import os 30import StringIO 31import sys 32import time 33import traceback 34import unittest 35 36from webkitpy.common.webkit_finder import WebKitFinder 37from webkitpy.common.system.filesystem import FileSystem 38from webkitpy.common.system.executive import Executive 39from webkitpy.test.finder import Finder 40from webkitpy.test.printer import Printer 41from webkitpy.test.runner import Runner, unit_test_name 42 43_log = logging.getLogger(__name__) 44 45 46up = os.path.dirname 47webkit_root = up(up(up(up(up(os.path.abspath(__file__)))))) 48 49 50def main(): 51 filesystem = FileSystem() 52 wkf = WebKitFinder(filesystem) 53 tester = Tester(filesystem, wkf) 54 tester.add_tree(wkf.path_from_webkit_base('Tools', 'Scripts'), 'webkitpy') 55 56 tester.skip(('webkitpy.common.checkout.scm.scm_unittest',), 'are really, really, slow', 31818) 57 if sys.platform == 'win32': 58 tester.skip(('webkitpy.common.checkout', 'webkitpy.common.config', 'webkitpy.tool', 'webkitpy.w3c', 'webkitpy.layout_tests.layout_package.bot_test_expectations'), 'fail horribly on win32', 54526) 59 60 # This only needs to run on Unix, so don't worry about win32 for now. 61 appengine_sdk_path = '/usr/local/google_appengine' 62 if os.path.exists(appengine_sdk_path): 63 if not appengine_sdk_path in sys.path: 64 sys.path.append(appengine_sdk_path) 65 import dev_appserver 66 from google.appengine.dist import use_library 67 use_library('django', '1.2') 68 dev_appserver.fix_sys_path() 69 tester.add_tree(wkf.path_from_webkit_base('Tools', 'TestResultServer')) 70 else: 71 _log.info('Skipping TestResultServer tests; the Google AppEngine Python SDK is not installed.') 72 73 return not tester.run() 74 75 76class Tester(object): 77 def __init__(self, filesystem=None, webkit_finder=None): 78 self.filesystem = filesystem or FileSystem() 79 self.executive = Executive() 80 self.finder = Finder(self.filesystem) 81 self.printer = Printer(sys.stderr) 82 self.webkit_finder = webkit_finder or WebKitFinder(self.filesystem) 83 self._options = None 84 85 def add_tree(self, top_directory, starting_subdirectory=None): 86 self.finder.add_tree(top_directory, starting_subdirectory) 87 88 def skip(self, names, reason, bugid): 89 self.finder.skip(names, reason, bugid) 90 91 def _parse_args(self, argv): 92 parser = optparse.OptionParser(usage='usage: %prog [options] [args...]') 93 parser.add_option('-a', '--all', action='store_true', default=False, 94 help='run all the tests') 95 parser.add_option('-c', '--coverage', action='store_true', default=False, 96 help='generate code coverage info') 97 parser.add_option('-i', '--integration-tests', action='store_true', default=False, 98 help='run integration tests as well as unit tests'), 99 parser.add_option('-j', '--child-processes', action='store', type='int', default=(1 if sys.platform == 'win32' else multiprocessing.cpu_count()), 100 help='number of tests to run in parallel (default=%default)') 101 parser.add_option('-p', '--pass-through', action='store_true', default=False, 102 help='be debugger friendly by passing captured output through to the system') 103 parser.add_option('-q', '--quiet', action='store_true', default=False, 104 help='run quietly (errors, warnings, and progress only)') 105 parser.add_option('-t', '--timing', action='store_true', default=False, 106 help='display per-test execution time (implies --verbose)') 107 parser.add_option('-v', '--verbose', action='count', default=0, 108 help='verbose output (specify once for individual test results, twice for debug messages)') 109 110 parser.epilog = ('[args...] is an optional list of modules, test_classes, or individual tests. ' 111 'If no args are given, all the tests will be run.') 112 113 return parser.parse_args(argv) 114 115 def run(self): 116 argv = sys.argv[1:] 117 self._options, args = self._parse_args(argv) 118 119 # Make sure PYTHONPATH is set up properly. 120 sys.path = self.finder.additional_paths(sys.path) + sys.path 121 122 # FIXME: unittest2 needs to be in sys.path for its internal imports to work. 123 thirdparty_path = self.webkit_finder.path_from_webkit_base('Tools', 'Scripts', 'webkitpy', 'thirdparty') 124 if not thirdparty_path in sys.path: 125 sys.path.append(thirdparty_path) 126 127 self.printer.configure(self._options) 128 129 # Do this after configuring the printer, so that logging works properly. 130 if self._options.coverage: 131 argv = ['-j', '1'] + [arg for arg in argv if arg not in ('-c', '--coverage', '-j', '--child-processes')] 132 _log.warning('Checking code coverage, so running things serially') 133 return self._run_under_coverage(argv) 134 135 self.finder.clean_trees() 136 137 names = self.finder.find_names(args, self._options.all) 138 if not names: 139 _log.error('No tests to run') 140 return False 141 142 return self._run_tests(names) 143 144 def _run_under_coverage(self, argv): 145 # coverage doesn't run properly unless its parent dir is in PYTHONPATH. 146 # This means we need to add that dir to the environment. Also, the 147 # report output is best when the paths are relative to the Scripts dir. 148 dirname = self.filesystem.dirname 149 script_dir = dirname(dirname(dirname(__file__))) 150 thirdparty_dir = self.filesystem.join(script_dir, 'webkitpy', 'thirdparty') 151 152 env = os.environ.copy() 153 python_path = env.get('PYTHONPATH', '') 154 python_path = python_path + os.pathsep + thirdparty_dir 155 env['PYTHONPATH'] = python_path 156 157 prefix_cmd = [sys.executable, 'webkitpy/thirdparty/coverage'] 158 exit_code = self.executive.call(prefix_cmd + ['run', __file__] + argv, cwd=script_dir, env=env) 159 if not exit_code: 160 exit_code = self.executive.call(prefix_cmd + ['report', '--omit', 'webkitpy/thirdparty/*,/usr/*,/Library/*'], cwd=script_dir, env=env) 161 return (exit_code == 0) 162 163 def _run_tests(self, names): 164 self.printer.write_update("Checking imports ...") 165 if not self._check_imports(names): 166 return False 167 168 self.printer.write_update("Finding the individual test methods ...") 169 loader = _Loader() 170 parallel_tests, serial_tests = self._test_names(loader, names) 171 172 self.printer.write_update("Running the tests ...") 173 self.printer.num_tests = len(parallel_tests) + len(serial_tests) 174 start = time.time() 175 test_runner = Runner(self.printer, loader, self.webkit_finder) 176 test_runner.run(parallel_tests, self._options.child_processes) 177 test_runner.run(serial_tests, 1) 178 179 self.printer.print_result(time.time() - start) 180 181 return not self.printer.num_errors and not self.printer.num_failures 182 183 def _check_imports(self, names): 184 for name in names: 185 if self.finder.is_module(name): 186 # if we failed to load a name and it looks like a module, 187 # try importing it directly, because loadTestsFromName() 188 # produces lousy error messages for bad modules. 189 try: 190 __import__(name) 191 except ImportError: 192 _log.fatal('Failed to import %s:' % name) 193 self._log_exception() 194 return False 195 return True 196 197 def _test_names(self, loader, names): 198 parallel_test_method_prefixes = ['test_'] 199 serial_test_method_prefixes = ['serial_test_'] 200 if self._options.integration_tests: 201 parallel_test_method_prefixes.append('integration_test_') 202 serial_test_method_prefixes.append('serial_integration_test_') 203 204 parallel_tests = [] 205 loader.test_method_prefixes = parallel_test_method_prefixes 206 for name in names: 207 parallel_tests.extend(self._all_test_names(loader.loadTestsFromName(name, None))) 208 209 serial_tests = [] 210 loader.test_method_prefixes = serial_test_method_prefixes 211 for name in names: 212 serial_tests.extend(self._all_test_names(loader.loadTestsFromName(name, None))) 213 214 # loader.loadTestsFromName() will not verify that names begin with one of the test_method_prefixes 215 # if the names were explicitly provided (e.g., MainTest.test_basic), so this means that any individual 216 # tests will be included in both parallel_tests and serial_tests, and we need to de-dup them. 217 serial_tests = list(set(serial_tests).difference(set(parallel_tests))) 218 219 return (parallel_tests, serial_tests) 220 221 def _all_test_names(self, suite): 222 names = [] 223 if hasattr(suite, '_tests'): 224 for t in suite._tests: 225 names.extend(self._all_test_names(t)) 226 else: 227 names.append(unit_test_name(suite)) 228 return names 229 230 def _log_exception(self): 231 s = StringIO.StringIO() 232 traceback.print_exc(file=s) 233 for l in s.buflist: 234 _log.error(' ' + l.rstrip()) 235 236 237class _Loader(unittest.TestLoader): 238 test_method_prefixes = [] 239 240 def getTestCaseNames(self, testCaseClass): 241 def isTestMethod(attrname, testCaseClass=testCaseClass): 242 if not hasattr(getattr(testCaseClass, attrname), '__call__'): 243 return False 244 return (any(attrname.startswith(prefix) for prefix in self.test_method_prefixes)) 245 testFnNames = filter(isTestMethod, dir(testCaseClass)) 246 testFnNames.sort() 247 return testFnNames 248 249 250if __name__ == '__main__': 251 sys.exit(main()) 252