1#!/usr/bin/env python2.7 2 3# Copyright 2015, VIXL authors 4# All rights reserved. 5# 6# Redistribution and use in source and binary forms, with or without 7# modification, are permitted provided that the following conditions are met: 8# 9# * Redistributions of source code must retain the above copyright notice, 10# this list of conditions and the following disclaimer. 11# * Redistributions in binary form must reproduce the above copyright notice, 12# this list of conditions and the following disclaimer in the documentation 13# and/or other materials provided with the distribution. 14# * Neither the name of ARM Limited nor the names of its contributors may be 15# used to endorse or promote products derived from this software without 16# specific prior written permission. 17# 18# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS CONTRIBUTORS "AS IS" AND 19# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 20# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 21# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE 22# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 23# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 24# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 25# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 26# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 29import argparse 30import fcntl 31import git 32import itertools 33import multiprocessing 34import os 35from os.path import join 36import platform 37import re 38import subprocess 39import sys 40import time 41 42import config 43import clang_format 44import lint 45import printer 46import test 47import threaded_tests 48import util 49 50 51dir_root = config.dir_root 52 53def Optionify(name): 54 return '--' + name 55 56 57# The options that can be tested are abstracted to provide an easy way to add 58# new ones. 59# Environment options influence the environment. They can be used for example to 60# set the compiler used. 61# Build options are options passed to scons, with a syntax like `scons opt=val` 62# Runtime options are options passed to the test program. 63# See the definition of `test_options` below. 64 65# 'all' is a special value for the options. If specified, all other values of 66# the option are tested. 67class TestOption(object): 68 type_environment = 'type_environment' 69 type_build = 'type_build' 70 type_run = 'type_run' 71 72 def __init__(self, option_type, name, help, 73 val_test_choices, val_test_default = None, 74 # If unset, the user can pass any value. 75 strict_choices = True, test_independently = False): 76 self.name = name 77 self.option_type = option_type 78 self.help = help 79 self.val_test_choices = val_test_choices 80 self.strict_choices = strict_choices 81 self.test_independently = test_independently 82 if val_test_default is not None: 83 self.val_test_default = val_test_default 84 else: 85 self.val_test_default = val_test_choices[0] 86 87 def ArgList(self, to_test): 88 res = [] 89 if to_test == 'all': 90 for value in self.val_test_choices: 91 if value != 'all': 92 res.append(self.GetOptionString(value)) 93 else: 94 for value in to_test: 95 res.append(self.GetOptionString(value)) 96 return res 97 98class EnvironmentOption(TestOption): 99 option_type = TestOption.type_environment 100 def __init__(self, name, environment_variable_name, help, 101 val_test_choices, val_test_default = None, 102 strict_choices = True): 103 super(EnvironmentOption, self).__init__(EnvironmentOption.option_type, 104 name, 105 help, 106 val_test_choices, 107 val_test_default, 108 strict_choices = strict_choices) 109 self.environment_variable_name = environment_variable_name 110 111 def GetOptionString(self, value): 112 return self.environment_variable_name + '=' + value 113 114 115class BuildOption(TestOption): 116 option_type = TestOption.type_build 117 def __init__(self, name, help, 118 val_test_choices, val_test_default = None, 119 strict_choices = True, test_independently = False): 120 super(BuildOption, self).__init__(BuildOption.option_type, 121 name, 122 help, 123 val_test_choices, 124 val_test_default, 125 strict_choices = strict_choices, 126 test_independently = test_independently) 127 def GetOptionString(self, value): 128 return self.name + '=' + value 129 130 131class RuntimeOption(TestOption): 132 option_type = TestOption.type_run 133 def __init__(self, name, help, 134 val_test_choices, val_test_default = None): 135 super(RuntimeOption, self).__init__(RuntimeOption.option_type, 136 name, 137 help, 138 val_test_choices, 139 val_test_default) 140 def GetOptionString(self, value): 141 if value == 'on': 142 return Optionify(self.name) 143 else: 144 return None 145 146 147 148environment_option_compiler = \ 149 EnvironmentOption('compiler', 'CXX', 'Test for the specified compilers.', 150 val_test_choices=['all'] + config.tested_compilers, 151 strict_choices = False) 152test_environment_options = [ 153 environment_option_compiler 154] 155 156build_option_mode = \ 157 BuildOption('mode', 'Test with the specified build modes.', 158 val_test_choices=['all'] + config.build_options_modes) 159build_option_standard = \ 160 BuildOption('std', 'Test with the specified C++ standard.', 161 val_test_choices=['all'] + config.tested_cpp_standards, 162 strict_choices = False) 163build_option_target = \ 164 BuildOption('target', 'Test with the specified isa enabled.', 165 val_test_choices=['all'] + config.build_options_target, 166 strict_choices = False, test_independently = True) 167build_option_negative_testing = \ 168 BuildOption('negative_testing', 'Test with negative testing enabled.', 169 val_test_choices=['all'] + config.build_options_negative_testing, 170 strict_choices = False, test_independently = True) 171test_build_options = [ 172 build_option_mode, 173 build_option_standard, 174 build_option_target, 175 build_option_negative_testing 176] 177 178runtime_option_debugger = \ 179 RuntimeOption('debugger', 180 '''Test with the specified configurations for the debugger. 181 Note that this is only tested if we are using the simulator.''', 182 val_test_choices=['all', 'on', 'off']) 183test_runtime_options = [ 184 runtime_option_debugger 185] 186 187test_options = \ 188 test_environment_options + test_build_options + test_runtime_options 189 190 191def BuildOptions(): 192 args = argparse.ArgumentParser( 193 description = 194 '''This tool runs all tests matching the specified filters for multiple 195 environment, build options, and runtime options configurations.''', 196 # Print default values. 197 formatter_class=argparse.ArgumentDefaultsHelpFormatter) 198 199 args.add_argument('filters', metavar='filter', nargs='*', 200 help='Run tests matching all of the (regexp) filters.') 201 202 # We automatically build the script options from the options to be tested. 203 test_arguments = args.add_argument_group( 204 'Test options', 205 'These options indicate what should be tested') 206 for option in test_options: 207 choices = option.val_test_choices if option.strict_choices else None 208 help = option.help 209 if not option.strict_choices: 210 help += ' Supported values: {' + ','.join(option.val_test_choices) + '}' 211 test_arguments.add_argument(Optionify(option.name), 212 nargs='+', 213 choices=choices, 214 default=option.val_test_default, 215 help=help, 216 action='store') 217 218 general_arguments = args.add_argument_group('General options') 219 general_arguments.add_argument('--fast', action='store_true', 220 help='''Skip the lint and clang-format tests, 221 and run only with one compiler, in one mode, 222 with one C++ standard, and with an appropriate 223 default for runtime options.''') 224 general_arguments.add_argument('--dry-run', action='store_true', 225 help='''Don't actually build or run anything, 226 but print the configurations that would be 227 tested.''') 228 general_arguments.add_argument( 229 '--jobs', '-j', metavar='N', type=int, nargs='?', 230 default=multiprocessing.cpu_count(), 231 const=multiprocessing.cpu_count(), 232 help='''Runs the tests using N jobs. If the option is set but no value is 233 provided, the script will use as many jobs as it thinks useful.''') 234 general_arguments.add_argument('--nobench', action='store_true', 235 help='Do not run benchmarks.') 236 general_arguments.add_argument('--nolint', action='store_true', 237 help='Do not run the linter.') 238 general_arguments.add_argument('--noclang-format', action='store_true', 239 help='Do not run clang-format.') 240 general_arguments.add_argument('--notest', action='store_true', 241 help='Do not run tests.') 242 general_arguments.add_argument('--fail-early', action='store_true', 243 help='Exit as soon as a test fails.') 244 sim_default = 'none' if platform.machine() == 'aarch64' else 'aarch64' 245 general_arguments.add_argument( 246 '--simulator', action='store', choices=['aarch64', 'none'], 247 default=sim_default, 248 help='Explicitly enable or disable the simulator.') 249 general_arguments.add_argument( 250 '--under_valgrind', action='store_true', 251 help='''Run the test-runner commands under Valgrind. 252 Note that a few tests are known to fail because of 253 issues in Valgrind''') 254 return args.parse_args() 255 256 257def RunCommand(command, environment_options = None): 258 # Create a copy of the environment. We do not want to pollute the environment 259 # of future commands run. 260 environment = os.environ 261 # Configure the environment. 262 # TODO: We currently pass the options as strings, so we need to parse them. We 263 # should instead pass them as a data structure and build the string option 264 # later. `environment_options` looks like `['CXX=compiler', 'OPT=val']`. 265 if environment_options: 266 for option in environment_options: 267 opt, val = option.split('=') 268 environment[opt] = val 269 270 printable_command = '' 271 if environment_options: 272 printable_command += ' '.join(environment_options) + ' ' 273 printable_command += ' '.join(command) 274 275 printable_command_orange = \ 276 printer.COLOUR_ORANGE + printable_command + printer.NO_COLOUR 277 printer.PrintOverwritableLine(printable_command_orange) 278 sys.stdout.flush() 279 280 # Start a process for the command. 281 # Interleave `stderr` and `stdout`. 282 p = subprocess.Popen(command, 283 stdout=subprocess.PIPE, 284 stderr=subprocess.STDOUT, 285 env=environment) 286 287 # We want to be able to display a continuously updated 'work indicator' while 288 # the process is running. Since the process can hang if the `stdout` pipe is 289 # full, we need to pull from it regularly. We cannot do so via the 290 # `readline()` function because it is blocking, and would thus cause the 291 # indicator to not be updated properly. So use file control mechanisms 292 # instead. 293 indicator = ' (still working: %d seconds elapsed)' 294 295 # Mark the process output as non-blocking. 296 flags = fcntl.fcntl(p.stdout, fcntl.F_GETFL) 297 fcntl.fcntl(p.stdout, fcntl.F_SETFL, flags | os.O_NONBLOCK) 298 299 t_start = time.time() 300 t_last_indication = t_start 301 process_output = '' 302 303 # Keep looping as long as the process is running. 304 while p.poll() is None: 305 # Avoid polling too often. 306 time.sleep(0.1) 307 # Update the progress indicator. 308 t_current = time.time() 309 if (t_current - t_start >= 2) and (t_current - t_last_indication >= 1): 310 printer.PrintOverwritableLine( 311 printable_command_orange + indicator % int(t_current - t_start)) 312 sys.stdout.flush() 313 t_last_indication = t_current 314 # Pull from the process output. 315 while True: 316 try: 317 line = os.read(p.stdout.fileno(), 1024) 318 except OSError: 319 line = '' 320 break 321 if line == '': break 322 process_output += line 323 324 # The process has exited. Don't forget to retrieve the rest of its output. 325 out, err = p.communicate() 326 rc = p.poll() 327 process_output += out 328 329 if rc == 0: 330 printer.Print(printer.COLOUR_GREEN + printable_command + printer.NO_COLOUR) 331 else: 332 printer.Print(printer.COLOUR_RED + printable_command + printer.NO_COLOUR) 333 printer.Print(process_output) 334 return rc 335 336 337def RunLinter(): 338 rc, default_tracked_files = lint.GetDefaultFilesToLint() 339 if rc: 340 return rc 341 return lint.RunLinter(map(lambda x: join(dir_root, x), default_tracked_files), 342 jobs = args.jobs, progress_prefix = 'cpp lint: ') 343 344 345def RunClangFormat(): 346 return clang_format.ClangFormatFiles(clang_format.GetCppSourceFilesToFormat(), 347 jobs = args.jobs, 348 progress_prefix = 'clang-format: ') 349 350 351 352def BuildAll(build_options, jobs): 353 scons_command = ["scons", "-C", dir_root, 'all', '-j', str(jobs)] 354 scons_command += list(build_options) 355 return RunCommand(scons_command, list(environment_options)) 356 357 358# Work out if the given options or args allow to run on the specified arch. 359# * arches is a list of ISA/architecture (a64, aarch32, etc) 360# * options are test.py's command line options if any. 361# * args are the arguments given to the build script. 362def CanRunOn(arches, options, args): 363 # First we check in the build specific options. 364 for option in options: 365 if 'target' in option: 366 # The option format is 'target=x,y,z'. 367 for target in (option.split('='))[1].split(','): 368 if target in arches: 369 return True 370 371 # There was a target build option but it didn't include the target arch. 372 return False 373 374 # No specific build option, check the script arguments. 375 # The meaning of 'all' will depend on the platform, e.g. 32-bit compilers 376 # cannot handle Aarch64 while 64-bit compiler can handle Aarch32. To avoid 377 # any issues no benchmarks are run for target='all'. 378 if args.target == 'all': return False 379 380 for target in args.target[0].split(','): 381 if target in arches: 382 return True 383 384 return False 385 386 387def CanRunAarch64(options, args): 388 return CanRunOn(['aarch64', 'a64'], options, args) 389 390 391def CanRunAarch32(options, args): 392 return CanRunOn(['aarch32', 'a32', 't32'], options, args) 393 394 395def RunBenchmarks(options, args): 396 rc = 0 397 if CanRunAarch32(options, args): 398 benchmark_names = util.ListCCFilesWithoutExt(config.dir_aarch32_benchmarks) 399 for bench in benchmark_names: 400 rc |= RunCommand( 401 [os.path.realpath( 402 join(config.dir_build_latest, 'benchmarks/aarch32', bench)), '10']) 403 if CanRunAarch64(options, args): 404 benchmark_names = util.ListCCFilesWithoutExt(config.dir_aarch64_benchmarks) 405 for bench in benchmark_names: 406 rc |= RunCommand( 407 [util.relrealpath( 408 join(config.dir_build_latest, 409 'benchmarks/aarch64', bench)), '10']) 410 return rc 411 412 413def PrintStatus(success): 414 printer.Print('\n$ ' + ' '.join(sys.argv)) 415 if success: 416 printer.Print('SUCCESS') 417 else: 418 printer.Print('FAILURE') 419 420 421 422if __name__ == '__main__': 423 util.require_program('scons') 424 rc = 0 425 426 args = BuildOptions() 427 428 def MaybeExitEarly(rc): 429 if args.fail_early and rc != 0: 430 PrintStatus(rc == 0) 431 sys.exit(rc) 432 433 if args.under_valgrind: 434 util.require_program('valgrind') 435 436 if args.fast: 437 def SetFast(option, specified, default): 438 option.val_test_choices = \ 439 [default if specified == 'all' else specified[0]] 440 # `g++` is very slow to compile a few aarch32 test files. 441 SetFast(environment_option_compiler, args.compiler, 'clang++') 442 SetFast(build_option_standard, args.std, 'c++98') 443 SetFast(build_option_mode, args.mode, 'debug') 444 SetFast(runtime_option_debugger, args.debugger, 'on') 445 446 if not args.nolint and not (args.fast or args.dry_run): 447 rc |= RunLinter() 448 MaybeExitEarly(rc) 449 450 if not args.noclang_format and not (args.fast or args.dry_run): 451 rc |= RunClangFormat() 452 MaybeExitEarly(rc) 453 454 # Don't try to test the debugger if we are not running with the simulator. 455 if not args.simulator: 456 test_runtime_options = \ 457 filter(lambda x: x.name != 'debugger', test_runtime_options) 458 459 # List all combinations of options that will be tested. 460 def ListCombinations(args, options): 461 opts_list = [ 462 opt.ArgList(args.__dict__[opt.name]) 463 for opt in options 464 if not opt.test_independently 465 ] 466 return list(itertools.product(*opts_list)) 467 # List combinations of options that should only be tested independently. 468 def ListIndependentCombinations(args, options, base): 469 n = [] 470 for opt in options: 471 if opt.test_independently: 472 for o in opt.ArgList(args.__dict__[opt.name]): 473 n.append(base + (o,)) 474 return n 475 # TODO: We should refine the configurations we test by default, instead of 476 # always testing all possible combinations. 477 test_env_combinations = ListCombinations(args, test_environment_options) 478 test_build_combinations = ListCombinations(args, test_build_options) 479 if not args.fast: 480 test_build_combinations.extend( 481 ListIndependentCombinations(args, 482 test_build_options, 483 test_build_combinations[0])) 484 test_runtime_combinations = ListCombinations(args, test_runtime_options) 485 486 for environment_options in test_env_combinations: 487 for build_options in test_build_combinations: 488 if (args.dry_run): 489 for runtime_options in test_runtime_combinations: 490 print(' '.join(filter(None, environment_options)) + ', ' + 491 ' '.join(filter(None, build_options)) + ', ' + 492 ' '.join(filter(None, runtime_options))) 493 continue 494 495 # Avoid going through the build stage if we are not using the build 496 # result. 497 if not (args.notest and args.nobench): 498 build_rc = BuildAll(build_options, args.jobs) 499 # Don't run the tests for this configuration if the build failed. 500 if build_rc != 0: 501 rc |= build_rc 502 MaybeExitEarly(rc) 503 continue 504 505 # Use the realpath of the test executable so that the commands printed 506 # can be copy-pasted and run. 507 test_executable = util.relrealpath( 508 join(config.dir_build_latest, 'test', 'test-runner')) 509 510 if not args.notest: 511 printer.Print(test_executable) 512 513 for runtime_options in test_runtime_combinations: 514 if not args.notest: 515 runtime_options = [x for x in runtime_options if x is not None] 516 prefix = ' ' + ' '.join(runtime_options) + ' ' 517 rc |= threaded_tests.RunTests(test_executable, 518 args.filters, 519 list(runtime_options), 520 args.under_valgrind, 521 jobs = args.jobs, prefix = prefix) 522 MaybeExitEarly(rc) 523 524 if not args.nobench: 525 rc |= RunBenchmarks(build_options, args) 526 MaybeExitEarly(rc) 527 528 if not args.dry_run: 529 PrintStatus(rc == 0) 530 531 sys.exit(rc) 532