#!/usr/bin/env python2.7 # Copyright 2015, VIXL authors # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: # # * Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright notice, # this list of conditions and the following disclaimer in the documentation # and/or other materials provided with the distribution. # * Neither the name of ARM Limited nor the names of its contributors may be # used to endorse or promote products derived from this software without # specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS CONTRIBUTORS "AS IS" AND # ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE # DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE # FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR # SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import argparse import fcntl import git import itertools import multiprocessing import os from os.path import join import platform import re import subprocess import sys import time import config import clang_format import lint import printer import test import threaded_tests import util dir_root = config.dir_root def Optionify(name): return '--' + name # The options that can be tested are abstracted to provide an easy way to add # new ones. # Environment options influence the environment. They can be used for example to # set the compiler used. # Build options are options passed to scons, with a syntax like `scons opt=val` # Runtime options are options passed to the test program. # See the definition of `test_options` below. # 'all' is a special value for the options. If specified, all other values of # the option are tested. class TestOption(object): type_environment = 'type_environment' type_build = 'type_build' type_run = 'type_run' def __init__(self, option_type, name, help, val_test_choices, val_test_default = None, # If unset, the user can pass any value. strict_choices = True, test_independently = False): self.name = name self.option_type = option_type self.help = help self.val_test_choices = val_test_choices self.strict_choices = strict_choices self.test_independently = test_independently if val_test_default is not None: self.val_test_default = val_test_default else: self.val_test_default = val_test_choices[0] def ArgList(self, to_test): res = [] if to_test == 'all': for value in self.val_test_choices: if value != 'all': res.append(self.GetOptionString(value)) else: for value in to_test: res.append(self.GetOptionString(value)) return res class EnvironmentOption(TestOption): option_type = TestOption.type_environment def __init__(self, name, environment_variable_name, help, val_test_choices, val_test_default = None, strict_choices = True): super(EnvironmentOption, self).__init__(EnvironmentOption.option_type, name, help, val_test_choices, val_test_default, strict_choices = strict_choices) self.environment_variable_name = environment_variable_name def GetOptionString(self, value): return self.environment_variable_name + '=' + value class BuildOption(TestOption): option_type = TestOption.type_build def __init__(self, name, help, val_test_choices, val_test_default = None, strict_choices = True, test_independently = False): super(BuildOption, self).__init__(BuildOption.option_type, name, help, val_test_choices, val_test_default, strict_choices = strict_choices, test_independently = test_independently) def GetOptionString(self, value): return self.name + '=' + value class RuntimeOption(TestOption): option_type = TestOption.type_run def __init__(self, name, help, val_test_choices, val_test_default = None): super(RuntimeOption, self).__init__(RuntimeOption.option_type, name, help, val_test_choices, val_test_default) def GetOptionString(self, value): if value == 'on': return Optionify(self.name) else: return None environment_option_compiler = \ EnvironmentOption('compiler', 'CXX', 'Test for the specified compilers.', val_test_choices=['all'] + config.tested_compilers, strict_choices = False) test_environment_options = [ environment_option_compiler ] build_option_mode = \ BuildOption('mode', 'Test with the specified build modes.', val_test_choices=['all'] + config.build_options_modes) build_option_standard = \ BuildOption('std', 'Test with the specified C++ standard.', val_test_choices=['all'] + config.tested_cpp_standards, strict_choices = False) build_option_target = \ BuildOption('target', 'Test with the specified isa enabled.', val_test_choices=['all'] + config.build_options_target, strict_choices = False, test_independently = True) build_option_negative_testing = \ BuildOption('negative_testing', 'Test with negative testing enabled.', val_test_choices=['all'] + config.build_options_negative_testing, strict_choices = False, test_independently = True) test_build_options = [ build_option_mode, build_option_standard, build_option_target, build_option_negative_testing ] runtime_option_debugger = \ RuntimeOption('debugger', '''Test with the specified configurations for the debugger. Note that this is only tested if we are using the simulator.''', val_test_choices=['all', 'on', 'off']) test_runtime_options = [ runtime_option_debugger ] test_options = \ test_environment_options + test_build_options + test_runtime_options def BuildOptions(): args = argparse.ArgumentParser( description = '''This tool runs all tests matching the specified filters for multiple environment, build options, and runtime options configurations.''', # Print default values. formatter_class=argparse.ArgumentDefaultsHelpFormatter) args.add_argument('filters', metavar='filter', nargs='*', help='Run tests matching all of the (regexp) filters.') # We automatically build the script options from the options to be tested. test_arguments = args.add_argument_group( 'Test options', 'These options indicate what should be tested') for option in test_options: choices = option.val_test_choices if option.strict_choices else None help = option.help if not option.strict_choices: help += ' Supported values: {' + ','.join(option.val_test_choices) + '}' test_arguments.add_argument(Optionify(option.name), nargs='+', choices=choices, default=option.val_test_default, help=help, action='store') general_arguments = args.add_argument_group('General options') general_arguments.add_argument('--fast', action='store_true', help='''Skip the lint and clang-format tests, and run only with one compiler, in one mode, with one C++ standard, and with an appropriate default for runtime options.''') general_arguments.add_argument('--dry-run', action='store_true', help='''Don't actually build or run anything, but print the configurations that would be tested.''') general_arguments.add_argument( '--jobs', '-j', metavar='N', type=int, nargs='?', default=multiprocessing.cpu_count(), const=multiprocessing.cpu_count(), help='''Runs the tests using N jobs. If the option is set but no value is provided, the script will use as many jobs as it thinks useful.''') general_arguments.add_argument('--nobench', action='store_true', help='Do not run benchmarks.') general_arguments.add_argument('--nolint', action='store_true', help='Do not run the linter.') general_arguments.add_argument('--noclang-format', action='store_true', help='Do not run clang-format.') general_arguments.add_argument('--notest', action='store_true', help='Do not run tests.') general_arguments.add_argument('--fail-early', action='store_true', help='Exit as soon as a test fails.') sim_default = 'none' if platform.machine() == 'aarch64' else 'aarch64' general_arguments.add_argument( '--simulator', action='store', choices=['aarch64', 'none'], default=sim_default, help='Explicitly enable or disable the simulator.') general_arguments.add_argument( '--under_valgrind', action='store_true', help='''Run the test-runner commands under Valgrind. Note that a few tests are known to fail because of issues in Valgrind''') return args.parse_args() def RunCommand(command, environment_options = None): # Create a copy of the environment. We do not want to pollute the environment # of future commands run. environment = os.environ # Configure the environment. # TODO: We currently pass the options as strings, so we need to parse them. We # should instead pass them as a data structure and build the string option # later. `environment_options` looks like `['CXX=compiler', 'OPT=val']`. if environment_options: for option in environment_options: opt, val = option.split('=') environment[opt] = val printable_command = '' if environment_options: printable_command += ' '.join(environment_options) + ' ' printable_command += ' '.join(command) printable_command_orange = \ printer.COLOUR_ORANGE + printable_command + printer.NO_COLOUR printer.PrintOverwritableLine(printable_command_orange) sys.stdout.flush() # Start a process for the command. # Interleave `stderr` and `stdout`. p = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, env=environment) # We want to be able to display a continuously updated 'work indicator' while # the process is running. Since the process can hang if the `stdout` pipe is # full, we need to pull from it regularly. We cannot do so via the # `readline()` function because it is blocking, and would thus cause the # indicator to not be updated properly. So use file control mechanisms # instead. indicator = ' (still working: %d seconds elapsed)' # Mark the process output as non-blocking. flags = fcntl.fcntl(p.stdout, fcntl.F_GETFL) fcntl.fcntl(p.stdout, fcntl.F_SETFL, flags | os.O_NONBLOCK) t_start = time.time() t_last_indication = t_start process_output = '' # Keep looping as long as the process is running. while p.poll() is None: # Avoid polling too often. time.sleep(0.1) # Update the progress indicator. t_current = time.time() if (t_current - t_start >= 2) and (t_current - t_last_indication >= 1): printer.PrintOverwritableLine( printable_command_orange + indicator % int(t_current - t_start)) sys.stdout.flush() t_last_indication = t_current # Pull from the process output. while True: try: line = os.read(p.stdout.fileno(), 1024) except OSError: line = '' break if line == '': break process_output += line # The process has exited. Don't forget to retrieve the rest of its output. out, err = p.communicate() rc = p.poll() process_output += out if rc == 0: printer.Print(printer.COLOUR_GREEN + printable_command + printer.NO_COLOUR) else: printer.Print(printer.COLOUR_RED + printable_command + printer.NO_COLOUR) printer.Print(process_output) return rc def RunLinter(): rc, default_tracked_files = lint.GetDefaultFilesToLint() if rc: return rc return lint.RunLinter(map(lambda x: join(dir_root, x), default_tracked_files), jobs = args.jobs, progress_prefix = 'cpp lint: ') def RunClangFormat(): return clang_format.ClangFormatFiles(clang_format.GetCppSourceFilesToFormat(), jobs = args.jobs, progress_prefix = 'clang-format: ') def BuildAll(build_options, jobs): scons_command = ["scons", "-C", dir_root, 'all', '-j', str(jobs)] scons_command += list(build_options) return RunCommand(scons_command, list(environment_options)) # Work out if the given options or args allow to run on the specified arch. # * arches is a list of ISA/architecture (a64, aarch32, etc) # * options are test.py's command line options if any. # * args are the arguments given to the build script. def CanRunOn(arches, options, args): # First we check in the build specific options. for option in options: if 'target' in option: # The option format is 'target=x,y,z'. for target in (option.split('='))[1].split(','): if target in arches: return True # There was a target build option but it didn't include the target arch. return False # No specific build option, check the script arguments. # The meaning of 'all' will depend on the platform, e.g. 32-bit compilers # cannot handle Aarch64 while 64-bit compiler can handle Aarch32. To avoid # any issues no benchmarks are run for target='all'. if args.target == 'all': return False for target in args.target[0].split(','): if target in arches: return True return False def CanRunAarch64(options, args): return CanRunOn(['aarch64', 'a64'], options, args) def CanRunAarch32(options, args): return CanRunOn(['aarch32', 'a32', 't32'], options, args) def RunBenchmarks(options, args): rc = 0 if CanRunAarch32(options, args): benchmark_names = util.ListCCFilesWithoutExt(config.dir_aarch32_benchmarks) for bench in benchmark_names: rc |= RunCommand( [os.path.realpath( join(config.dir_build_latest, 'benchmarks/aarch32', bench))]) if CanRunAarch64(options, args): benchmark_names = util.ListCCFilesWithoutExt(config.dir_aarch64_benchmarks) for bench in benchmark_names: rc |= RunCommand( [util.relrealpath( join(config.dir_build_latest, 'benchmarks/aarch64', bench))]) return rc def PrintStatus(success): printer.Print('\n$ ' + ' '.join(sys.argv)) if success: printer.Print('SUCCESS') else: printer.Print('FAILURE') if __name__ == '__main__': util.require_program('scons') rc = 0 args = BuildOptions() def MaybeExitEarly(rc): if args.fail_early and rc != 0: PrintStatus(rc == 0) sys.exit(rc) if args.under_valgrind: util.require_program('valgrind') if args.fast: def SetFast(option, specified, default): option.val_test_choices = \ [default if specified == 'all' else specified[0]] # `g++` is very slow to compile a few aarch32 test files. SetFast(environment_option_compiler, args.compiler, 'clang++') SetFast(build_option_standard, args.std, 'c++98') SetFast(build_option_mode, args.mode, 'debug') SetFast(runtime_option_debugger, args.debugger, 'on') if not args.nolint and not (args.fast or args.dry_run): rc |= RunLinter() MaybeExitEarly(rc) if not args.noclang_format and not (args.fast or args.dry_run): rc |= RunClangFormat() MaybeExitEarly(rc) # Don't try to test the debugger if we are not running with the simulator. if not args.simulator: test_runtime_options = \ filter(lambda x: x.name != 'debugger', test_runtime_options) # List all combinations of options that will be tested. def ListCombinations(args, options): opts_list = [ opt.ArgList(args.__dict__[opt.name]) for opt in options if not opt.test_independently ] return list(itertools.product(*opts_list)) # List combinations of options that should only be tested independently. def ListIndependentCombinations(args, options, base): n = [] for opt in options: if opt.test_independently: for o in opt.ArgList(args.__dict__[opt.name]): n.append(base + (o,)) return n # TODO: We should refine the configurations we test by default, instead of # always testing all possible combinations. test_env_combinations = ListCombinations(args, test_environment_options) test_build_combinations = ListCombinations(args, test_build_options) if not args.fast: test_build_combinations.extend( ListIndependentCombinations(args, test_build_options, test_build_combinations[0])) test_runtime_combinations = ListCombinations(args, test_runtime_options) for environment_options in test_env_combinations: for build_options in test_build_combinations: if (args.dry_run): for runtime_options in test_runtime_combinations: print(' '.join(filter(None, environment_options)) + ', ' + ' '.join(filter(None, build_options)) + ', ' + ' '.join(filter(None, runtime_options))) continue # Avoid going through the build stage if we are not using the build # result. if not (args.notest and args.nobench): build_rc = BuildAll(build_options, args.jobs) # Don't run the tests for this configuration if the build failed. if build_rc != 0: rc |= build_rc MaybeExitEarly(rc) continue # Use the realpath of the test executable so that the commands printed # can be copy-pasted and run. test_executable = util.relrealpath( join(config.dir_build_latest, 'test', 'test-runner')) if not args.notest: printer.Print(test_executable) for runtime_options in test_runtime_combinations: if not args.notest: runtime_options = [x for x in runtime_options if x is not None] prefix = ' ' + ' '.join(runtime_options) + ' ' rc |= threaded_tests.RunTests(test_executable, args.filters, list(runtime_options), args.under_valgrind, jobs = args.jobs, prefix = prefix) MaybeExitEarly(rc) if not args.nobench: rc |= RunBenchmarks(build_options, args) MaybeExitEarly(rc) if not args.dry_run: PrintStatus(rc == 0) sys.exit(rc)