1#!/usr/bin/env python2.7 2 3# Copyright 2014, ARM Limited 4# All rights reserved. 5# 6# Redistribution and use in source and binary forms, with or without 7# modification, are permitted provided that the following conditions are met: 8# 9# * Redistributions of source code must retain the above copyright notice, 10# this list of conditions and the following disclaimer. 11# * Redistributions in binary form must reproduce the above copyright notice, 12# this list of conditions and the following disclaimer in the documentation 13# and/or other materials provided with the distribution. 14# * Neither the name of ARM Limited nor the names of its contributors may be 15# used to endorse or promote products derived from this software without 16# specific prior written permission. 17# 18# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS CONTRIBUTORS "AS IS" AND 19# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 20# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 21# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE 22# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 23# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 24# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 25# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 26# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 29import os 30import sys 31import argparse 32import re 33import platform 34import subprocess 35import multiprocessing 36import time 37import util 38 39 40from printer import EnsureNewLine, Print, UpdateProgress 41 42 43def BuildOptions(): 44 result = argparse.ArgumentParser( 45 description = 46 '''This tool runs each test reported by $TEST --list (and filtered as 47 specified). A summary will be printed, and detailed test output will be 48 stored in log/$TEST.''', 49 # Print default values. 50 formatter_class=argparse.ArgumentDefaultsHelpFormatter) 51 result.add_argument('filters', metavar='filter', nargs='*', 52 help='Run tests matching all of the (regexp) filters.') 53 result.add_argument('--runner', action='store', required=True, 54 help='The test executable to run.') 55 result.add_argument('--coloured_trace', action='store_true', 56 help='''Pass --coloured_trace to the test runner. This 57 will put colour codes in the log files. The 58 coloured output can be viewed by "less -R", for 59 example.''') 60 result.add_argument('--debugger', action='store_true', 61 help='''Pass --debugger to test, so that the debugger is 62 used instead of the simulator. This has no effect 63 when running natively.''') 64 result.add_argument('--verbose', action='store_true', 65 help='Print verbose output.') 66 result.add_argument('--jobs', '-j', metavar='N', type=int, nargs='?', 67 default=1, const=multiprocessing.cpu_count(), 68 help='''Runs the tests using N jobs. If the option is set 69 but no value is provided, the script will use as many jobs 70 as it thinks useful.''') 71 sim_default = 'off' if platform.machine() == 'aarch64' else 'on' 72 result.add_argument('--simulator', action='store', choices=['on', 'off'], 73 default=sim_default, 74 help='Explicitly enable or disable the simulator.') 75 return result.parse_args() 76 77 78def VerbosePrint(verbose, string): 79 if verbose: 80 Print(string) 81 82 83# A class representing an individual test. 84class Test: 85 def __init__(self, name, runner, debugger, coloured_trace, verbose): 86 self.name = name 87 self.runner = runner 88 self.debugger = debugger 89 self.coloured_trace = coloured_trace 90 self.verbose = verbose 91 self.logpath = os.path.join('log', os.path.basename(self.runner)) 92 if self.debugger: 93 basename = name + '_debugger' 94 else: 95 basename = name 96 self.logout = os.path.join(self.logpath, basename + '.stdout') 97 self.logerr = os.path.join(self.logpath, basename + '.stderr') 98 if not os.path.exists(self.logpath): os.makedirs(self.logpath) 99 100 # Run the test. 101 # Use a thread to be able to control the test. 102 def Run(self): 103 command = \ 104 [self.runner, '--trace_sim', '--trace_reg', '--trace_write', self.name] 105 if self.coloured_trace: 106 command.append('--coloured_trace') 107 if self.debugger: 108 command.append('--debugger') 109 110 VerbosePrint(self.verbose, '==== Running ' + self.name + '... ====') 111 sys.stdout.flush() 112 113 process = subprocess.Popen(command, 114 stdout=subprocess.PIPE, 115 stderr=subprocess.PIPE) 116 # Get the output and return status of the test. 117 stdout, stderr = process.communicate() 118 retcode = process.poll() 119 120 # Write stdout and stderr to the log. 121 with open(self.logout, 'w') as f: f.write(stdout) 122 with open(self.logerr, 'w') as f: f.write(stderr) 123 124 if retcode == 0: 125 # Success. 126 # We normally only print the command on failure, but with --verbose we 127 # should also print it on success. 128 VerbosePrint(self.verbose, 'COMMAND: ' + ' '.join(command)) 129 VerbosePrint(self.verbose, 'LOG (stdout): ' + self.logout) 130 VerbosePrint(self.verbose, 'LOG (stderr): ' + self.logerr + '\n') 131 else: 132 # Failure. 133 Print('--- FAILED ' + self.name + ' ---') 134 Print('COMMAND: ' + ' '.join(command)) 135 Print('LOG (stdout): ' + self.logout) 136 Print('LOG (stderr): ' + self.logerr + '\n') 137 138 return retcode 139 140 141# Scan matching tests and return a test manifest. 142def ReadManifest(runner, filters = [], 143 debugger = False, coloured_trace = False, verbose = False): 144 status, output = util.getstatusoutput(runner + ' --list') 145 if status != 0: util.abort('Failed to list all tests') 146 147 names = output.split() 148 for f in filters: 149 names = filter(re.compile(f).search, names) 150 151 return map(lambda x: 152 Test(x, runner, debugger, coloured_trace, verbose), names) 153 154 155# Shared state for multiprocessing. Ideally the context should be passed with 156# arguments, but constraints from the multiprocessing module prevent us from 157# doing so: the shared variables (multiprocessing.Value) must be global, or no 158# work is started. So we abstract some additional state into global variables to 159# simplify the implementation. 160# Read-write variables for the workers. 161n_tests_passed = multiprocessing.Value('i', 0) 162n_tests_failed = multiprocessing.Value('i', 0) 163# Read-only for workers. 164n_tests = None 165start_time = None 166verbose_test_run = None 167test_suite_name = '' 168 169def RunTest(test): 170 UpdateProgress(start_time, n_tests_passed.value, n_tests_failed.value, 171 n_tests, verbose_test_run, test.name, test_suite_name) 172 # Run the test and update the statistics. 173 retcode = test.Run() 174 if retcode == 0: 175 with n_tests_passed.get_lock(): n_tests_passed.value += 1 176 else: 177 with n_tests_failed.get_lock(): n_tests_failed.value += 1 178 179 180# Run all tests in the manifest. 181# This function won't run in parallel due to constraints from the 182# multiprocessing module. 183__run_tests_lock__ = multiprocessing.Lock() 184def RunTests(manifest, jobs = 1, verbose = False, debugger = False, 185 progress_prefix = ''): 186 global n_tests 187 global start_time 188 global verbose_test_run 189 global test_suite_name 190 191 with __run_tests_lock__: 192 193 # Reset the counters. 194 n_tests_passed.value = 0 195 n_tests_failed.value = 0 196 197 verbose_test_run = verbose 198 test_suite_name = progress_prefix 199 200 n_tests = len(manifest) 201 202 if n_tests == 0: 203 Print('No tests to run.') 204 return 0 205 206 VerbosePrint(verbose, 'Running %d tests...' % (n_tests)) 207 start_time = time.time() 208 209 pool = multiprocessing.Pool(jobs) 210 # The '.get(9999999)' is workaround to allow killing the test script with 211 # ctrl+C from the shell. This bug is documented at 212 # http://bugs.python.org/issue8296. 213 work = pool.map_async(RunTest, manifest).get(9999999) 214 215 done_message = '== Done ==' 216 UpdateProgress(start_time, n_tests_passed.value, n_tests_failed.value, 217 n_tests, verbose, done_message, progress_prefix) 218 219 return n_tests_failed.value # 0 indicates success 220 221 222if __name__ == '__main__': 223 # $ROOT/tools/test.py 224 root_dir = os.path.dirname(os.path.dirname(os.path.abspath(sys.argv[0]))) 225 226 # Parse the arguments. 227 args = BuildOptions() 228 229 # Find a valid path to args.runner (in case it doesn't begin with './'). 230 args.runner = os.path.join('.', args.runner) 231 232 if not os.access(args.runner, os.X_OK): 233 print "'" + args.test + "' is not executable or does not exist." 234 sys.exit(1) 235 236 # List all matching tests. 237 manifest = ReadManifest(args.runner, args.filters, 238 args.debugger, args.coloured_trace, args.verbose) 239 240 # Run the tests. 241 status = RunTests(manifest, jobs=args.jobs, 242 verbose=args.verbose, debugger=args.debugger) 243 EnsureNewLine() 244 245 sys.exit(status) 246 247