1# Copyright 2008 Google Inc. All Rights Reserved. 2# 3# Redistribution and use in source and binary forms, with or without 4# modification, are permitted provided that the following conditions are 5# met: 6# 7# * Redistributions of source code must retain the above copyright 8# notice, this list of conditions and the following disclaimer. 9# * Redistributions in binary form must reproduce the above 10# copyright notice, this list of conditions and the following disclaimer 11# in the documentation and/or other materials provided with the 12# distribution. 13# * Neither the name of Google Inc. nor the names of its 14# contributors may be used to endorse or promote products derived from 15# this software without specific prior written permission. 16# 17# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 18# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 19# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 20# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 21# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 22# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 23# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 24# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 25# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 29"""Provides facilities for running SCons-built Google Test/Mock tests.""" 30 31 32import optparse 33import os 34import re 35import sets 36import sys 37 38try: 39 # subrocess module is a preferable way to invoke subprocesses but it may 40 # not be available on MacOS X 10.4. 41 # Suppresses the 'Import not at the top of the file' lint complaint. 42 # pylint: disable-msg=C6204 43 import subprocess 44except ImportError: 45 subprocess = None 46 47HELP_MSG = """Runs the specified tests for %(proj)s. 48 49SYNOPSIS 50 run_tests.py [OPTION]... [BUILD_DIR]... [TEST]... 51 52DESCRIPTION 53 Runs the specified tests (either binary or Python), and prints a 54 summary of the results. BUILD_DIRS will be used to search for the 55 binaries. If no TESTs are specified, all binary tests found in 56 BUILD_DIRs and all Python tests found in the directory test/ (in the 57 %(proj)s root) are run. 58 59 TEST is a name of either a binary or a Python test. A binary test is 60 an executable file named *_test or *_unittest (with the .exe 61 extension on Windows) A Python test is a script named *_test.py or 62 *_unittest.py. 63 64OPTIONS 65 -h, --help 66 Print this help message. 67 -c CONFIGURATIONS 68 Specify build directories via build configurations. 69 CONFIGURATIONS is either a comma-separated list of build 70 configurations or 'all'. Each configuration is equivalent to 71 adding 'scons/build/<configuration>/%(proj)s/scons' to BUILD_DIRs. 72 Specifying -c=all is equivalent to providing all directories 73 listed in KNOWN BUILD DIRECTORIES section below. 74 -a 75 Equivalent to -c=all 76 -b 77 Equivalent to -c=all with the exception that the script will not 78 fail if some of the KNOWN BUILD DIRECTORIES do not exists; the 79 script will simply not run the tests there. 'b' stands for 80 'built directories'. 81 82RETURN VALUE 83 Returns 0 if all tests are successful; otherwise returns 1. 84 85EXAMPLES 86 run_tests.py 87 Runs all tests for the default build configuration. 88 run_tests.py -a 89 Runs all tests with binaries in KNOWN BUILD DIRECTORIES. 90 run_tests.py -b 91 Runs all tests in KNOWN BUILD DIRECTORIES that have been 92 built. 93 run_tests.py foo/ 94 Runs all tests in the foo/ directory and all Python tests in 95 the directory test. The Python tests are instructed to look 96 for binaries in foo/. 97 run_tests.py bar_test.exe test/baz_test.exe foo/ bar/ 98 Runs foo/bar_test.exe, bar/bar_test.exe, foo/baz_test.exe, and 99 bar/baz_test.exe. 100 run_tests.py foo bar test/foo_test.py 101 Runs test/foo_test.py twice instructing it to look for its 102 test binaries in the directories foo and bar, 103 correspondingly. 104 105KNOWN BUILD DIRECTORIES 106 run_tests.py knows about directories where the SCons build script 107 deposits its products. These are the directories where run_tests.py 108 will be looking for its binaries. Currently, %(proj)s's SConstruct file 109 defines them as follows (the default build directory is the first one 110 listed in each group): 111 On Windows: 112 <%(proj)s root>/scons/build/win-dbg8/%(proj)s/scons/ 113 <%(proj)s root>/scons/build/win-opt8/%(proj)s/scons/ 114 On Mac: 115 <%(proj)s root>/scons/build/mac-dbg/%(proj)s/scons/ 116 <%(proj)s root>/scons/build/mac-opt/%(proj)s/scons/ 117 On other platforms: 118 <%(proj)s root>/scons/build/dbg/%(proj)s/scons/ 119 <%(proj)s root>/scons/build/opt/%(proj)s/scons/""" 120 121IS_WINDOWS = os.name == 'nt' 122IS_MAC = os.name == 'posix' and os.uname()[0] == 'Darwin' 123IS_CYGWIN = os.name == 'posix' and 'CYGWIN' in os.uname()[0] 124 125# Definition of CONFIGS must match that of the build directory names in the 126# SConstruct script. The first list item is the default build configuration. 127if IS_WINDOWS: 128 CONFIGS = ('win-dbg8', 'win-opt8') 129elif IS_MAC: 130 CONFIGS = ('mac-dbg', 'mac-opt') 131else: 132 CONFIGS = ('dbg', 'opt') 133 134if IS_WINDOWS or IS_CYGWIN: 135 PYTHON_TEST_REGEX = re.compile(r'_(unit)?test\.py$', re.IGNORECASE) 136 BINARY_TEST_REGEX = re.compile(r'_(unit)?test(\.exe)?$', re.IGNORECASE) 137 BINARY_TEST_SEARCH_REGEX = re.compile(r'_(unit)?test\.exe$', re.IGNORECASE) 138else: 139 PYTHON_TEST_REGEX = re.compile(r'_(unit)?test\.py$') 140 BINARY_TEST_REGEX = re.compile(r'_(unit)?test$') 141 BINARY_TEST_SEARCH_REGEX = BINARY_TEST_REGEX 142 143 144def _GetGtestBuildDir(injected_os, script_dir, config): 145 """Calculates path to the Google Test SCons build directory.""" 146 147 return injected_os.path.normpath(injected_os.path.join(script_dir, 148 'scons/build', 149 config, 150 'gtest/scons')) 151 152 153def _GetConfigFromBuildDir(build_dir): 154 """Extracts the configuration name from the build directory.""" 155 156 # We don't want to depend on build_dir containing the correct path 157 # separators. 158 m = re.match(r'.*[\\/]([^\\/]+)[\\/][^\\/]+[\\/]scons[\\/]?$', build_dir) 159 if m: 160 return m.group(1) 161 else: 162 print >>sys.stderr, ('%s is an invalid build directory that does not ' 163 'correspond to any configuration.' % (build_dir,)) 164 return '' 165 166 167# All paths in this script are either absolute or relative to the current 168# working directory, unless otherwise specified. 169class TestRunner(object): 170 """Provides facilities for running Python and binary tests for Google Test.""" 171 172 def __init__(self, 173 script_dir, 174 build_dir_var_name='GTEST_BUILD_DIR', 175 injected_os=os, 176 injected_subprocess=subprocess, 177 injected_build_dir_finder=_GetGtestBuildDir): 178 """Initializes a TestRunner instance. 179 180 Args: 181 script_dir: File path to the calling script. 182 build_dir_var_name: Name of the env variable used to pass the 183 the build directory path to the invoked 184 tests. 185 injected_os: standard os module or a mock/stub for 186 testing. 187 injected_subprocess: standard subprocess module or a mock/stub 188 for testing 189 injected_build_dir_finder: function that determines the path to 190 the build directory. 191 """ 192 193 self.os = injected_os 194 self.subprocess = injected_subprocess 195 self.build_dir_finder = injected_build_dir_finder 196 self.build_dir_var_name = build_dir_var_name 197 self.script_dir = script_dir 198 199 def _GetBuildDirForConfig(self, config): 200 """Returns the build directory for a given configuration.""" 201 202 return self.build_dir_finder(self.os, self.script_dir, config) 203 204 def _Run(self, args): 205 """Runs the executable with given args (args[0] is the executable name). 206 207 Args: 208 args: Command line arguments for the process. 209 210 Returns: 211 Process's exit code if it exits normally, or -signal if the process is 212 killed by a signal. 213 """ 214 215 if self.subprocess: 216 return self.subprocess.Popen(args).wait() 217 else: 218 return self.os.spawnv(self.os.P_WAIT, args[0], args) 219 220 def _RunBinaryTest(self, test): 221 """Runs the binary test given its path. 222 223 Args: 224 test: Path to the test binary. 225 226 Returns: 227 Process's exit code if it exits normally, or -signal if the process is 228 killed by a signal. 229 """ 230 231 return self._Run([test]) 232 233 def _RunPythonTest(self, test, build_dir): 234 """Runs the Python test script with the specified build directory. 235 236 Args: 237 test: Path to the test's Python script. 238 build_dir: Path to the directory where the test binary is to be found. 239 240 Returns: 241 Process's exit code if it exits normally, or -signal if the process is 242 killed by a signal. 243 """ 244 245 old_build_dir = self.os.environ.get(self.build_dir_var_name) 246 247 try: 248 self.os.environ[self.build_dir_var_name] = build_dir 249 250 # If this script is run on a Windows machine that has no association 251 # between the .py extension and a python interpreter, simply passing 252 # the script name into subprocess.Popen/os.spawn will not work. 253 print 'Running %s . . .' % (test,) 254 return self._Run([sys.executable, test]) 255 256 finally: 257 if old_build_dir is None: 258 del self.os.environ[self.build_dir_var_name] 259 else: 260 self.os.environ[self.build_dir_var_name] = old_build_dir 261 262 def _FindFilesByRegex(self, directory, regex): 263 """Returns files in a directory whose names match a regular expression. 264 265 Args: 266 directory: Path to the directory to search for files. 267 regex: Regular expression to filter file names. 268 269 Returns: 270 The list of the paths to the files in the directory. 271 """ 272 273 return [self.os.path.join(directory, file_name) 274 for file_name in self.os.listdir(directory) 275 if re.search(regex, file_name)] 276 277 # TODO(vladl@google.com): Implement parsing of scons/SConscript to run all 278 # tests defined there when no tests are specified. 279 # TODO(vladl@google.com): Update the docstring after the code is changed to 280 # try to test all builds defined in scons/SConscript. 281 def GetTestsToRun(self, 282 args, 283 named_configurations, 284 built_configurations, 285 available_configurations=CONFIGS, 286 python_tests_to_skip=None): 287 """Determines what tests should be run. 288 289 Args: 290 args: The list of non-option arguments from the command line. 291 named_configurations: The list of configurations specified via -c or -a. 292 built_configurations: True if -b has been specified. 293 available_configurations: a list of configurations available on the 294 current platform, injectable for testing. 295 python_tests_to_skip: a collection of (configuration, python test name)s 296 that need to be skipped. 297 298 Returns: 299 A tuple with 2 elements: the list of Python tests to run and the list of 300 binary tests to run. 301 """ 302 303 if named_configurations == 'all': 304 named_configurations = ','.join(available_configurations) 305 306 normalized_args = [self.os.path.normpath(arg) for arg in args] 307 308 # A final list of build directories which will be searched for the test 309 # binaries. First, add directories specified directly on the command 310 # line. 311 build_dirs = filter(self.os.path.isdir, normalized_args) 312 313 # Adds build directories specified via their build configurations using 314 # the -c or -a options. 315 if named_configurations: 316 build_dirs += [self._GetBuildDirForConfig(config) 317 for config in named_configurations.split(',')] 318 319 # Adds KNOWN BUILD DIRECTORIES if -b is specified. 320 if built_configurations: 321 build_dirs += [self._GetBuildDirForConfig(config) 322 for config in available_configurations 323 if self.os.path.isdir(self._GetBuildDirForConfig(config))] 324 325 # If no directories were specified either via -a, -b, -c, or directly, use 326 # the default configuration. 327 elif not build_dirs: 328 build_dirs = [self._GetBuildDirForConfig(available_configurations[0])] 329 330 # Makes sure there are no duplications. 331 build_dirs = sets.Set(build_dirs) 332 333 errors_found = False 334 listed_python_tests = [] # All Python tests listed on the command line. 335 listed_binary_tests = [] # All binary tests listed on the command line. 336 337 test_dir = self.os.path.normpath(self.os.path.join(self.script_dir, 'test')) 338 339 # Sifts through non-directory arguments fishing for any Python or binary 340 # tests and detecting errors. 341 for argument in sets.Set(normalized_args) - build_dirs: 342 if re.search(PYTHON_TEST_REGEX, argument): 343 python_path = self.os.path.join(test_dir, 344 self.os.path.basename(argument)) 345 if self.os.path.isfile(python_path): 346 listed_python_tests.append(python_path) 347 else: 348 sys.stderr.write('Unable to find Python test %s' % argument) 349 errors_found = True 350 elif re.search(BINARY_TEST_REGEX, argument): 351 # This script also accepts binary test names prefixed with test/ for 352 # the convenience of typing them (can use path completions in the 353 # shell). Strips test/ prefix from the binary test names. 354 listed_binary_tests.append(self.os.path.basename(argument)) 355 else: 356 sys.stderr.write('%s is neither test nor build directory' % argument) 357 errors_found = True 358 359 if errors_found: 360 return None 361 362 user_has_listed_tests = listed_python_tests or listed_binary_tests 363 364 if user_has_listed_tests: 365 selected_python_tests = listed_python_tests 366 else: 367 selected_python_tests = self._FindFilesByRegex(test_dir, 368 PYTHON_TEST_REGEX) 369 370 # TODO(vladl@google.com): skip unbuilt Python tests when -b is specified. 371 python_test_pairs = [] 372 for directory in build_dirs: 373 for test in selected_python_tests: 374 config = _GetConfigFromBuildDir(directory) 375 file_name = os.path.basename(test) 376 if python_tests_to_skip and (config, file_name) in python_tests_to_skip: 377 print ('NOTE: %s is skipped for configuration %s, as it does not ' 378 'work there.' % (file_name, config)) 379 else: 380 python_test_pairs.append((directory, test)) 381 382 binary_test_pairs = [] 383 for directory in build_dirs: 384 if user_has_listed_tests: 385 binary_test_pairs.extend( 386 [(directory, self.os.path.join(directory, test)) 387 for test in listed_binary_tests]) 388 else: 389 tests = self._FindFilesByRegex(directory, BINARY_TEST_SEARCH_REGEX) 390 binary_test_pairs.extend([(directory, test) for test in tests]) 391 392 return (python_test_pairs, binary_test_pairs) 393 394 def RunTests(self, python_tests, binary_tests): 395 """Runs Python and binary tests and reports results to the standard output. 396 397 Args: 398 python_tests: List of Python tests to run in the form of tuples 399 (build directory, Python test script). 400 binary_tests: List of binary tests to run in the form of tuples 401 (build directory, binary file). 402 403 Returns: 404 The exit code the program should pass into sys.exit(). 405 """ 406 407 if python_tests or binary_tests: 408 results = [] 409 for directory, test in python_tests: 410 results.append((directory, 411 test, 412 self._RunPythonTest(test, directory) == 0)) 413 for directory, test in binary_tests: 414 results.append((directory, 415 self.os.path.basename(test), 416 self._RunBinaryTest(test) == 0)) 417 418 failed = [(directory, test) 419 for (directory, test, success) in results 420 if not success] 421 print 422 print '%d tests run.' % len(results) 423 if failed: 424 print 'The following %d tests failed:' % len(failed) 425 for (directory, test) in failed: 426 print '%s in %s' % (test, directory) 427 return 1 428 else: 429 print 'All tests passed!' 430 else: # No tests defined 431 print 'Nothing to test - no tests specified!' 432 433 return 0 434 435 436def ParseArgs(project_name, argv=None, help_callback=None): 437 """Parses the options run_tests.py uses.""" 438 439 # Suppresses lint warning on unused arguments. These arguments are 440 # required by optparse, even though they are unused. 441 # pylint: disable-msg=W0613 442 def PrintHelp(option, opt, value, parser): 443 print HELP_MSG % {'proj': project_name} 444 sys.exit(1) 445 446 parser = optparse.OptionParser() 447 parser.add_option('-c', 448 action='store', 449 dest='configurations', 450 default=None) 451 parser.add_option('-a', 452 action='store_const', 453 dest='configurations', 454 default=None, 455 const='all') 456 parser.add_option('-b', 457 action='store_const', 458 dest='built_configurations', 459 default=False, 460 const=True) 461 # Replaces the built-in help with ours. 462 parser.remove_option('-h') 463 parser.add_option('-h', '--help', 464 action='callback', 465 callback=help_callback or PrintHelp) 466 return parser.parse_args(argv) 467