• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/env python2.7
2
3# Copyright 2014, ARM Limited
4# All rights reserved.
5#
6# Redistribution and use in source and binary forms, with or without
7# modification, are permitted provided that the following conditions are met:
8#
9#   * Redistributions of source code must retain the above copyright notice,
10#     this list of conditions and the following disclaimer.
11#   * Redistributions in binary form must reproduce the above copyright notice,
12#     this list of conditions and the following disclaimer in the documentation
13#     and/or other materials provided with the distribution.
14#   * Neither the name of ARM Limited nor the names of its contributors may be
15#     used to endorse or promote products derived from this software without
16#     specific prior written permission.
17#
18# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS CONTRIBUTORS "AS IS" AND
19# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
20# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
21# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
22# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
23# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
24# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
25# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
26# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28
29import os
30import sys
31import argparse
32import re
33import platform
34import subprocess
35import multiprocessing
36import time
37import util
38
39
40from printer import EnsureNewLine, Print, UpdateProgress
41
42
43def BuildOptions():
44  result = argparse.ArgumentParser(
45      description =
46      '''This tool runs each test reported by $TEST --list (and filtered as
47         specified). A summary will be printed, and detailed test output will be
48         stored in log/$TEST.''',
49      # Print default values.
50      formatter_class=argparse.ArgumentDefaultsHelpFormatter)
51  result.add_argument('filters', metavar='filter', nargs='*',
52                      help='Run tests matching all of the (regexp) filters.')
53  result.add_argument('--runner', action='store', required=True,
54                      help='The test executable to run.')
55  result.add_argument('--coloured_trace', action='store_true',
56                      help='''Pass --coloured_trace to the test runner. This
57                              will put colour codes in the log files. The
58                              coloured output can be viewed by "less -R", for
59                              example.''')
60  result.add_argument('--debugger', action='store_true',
61                      help='''Pass --debugger to test, so that the debugger is
62                              used instead of the simulator. This has no effect
63                              when running natively.''')
64  result.add_argument('--verbose', action='store_true',
65                      help='Print verbose output.')
66  result.add_argument('--jobs', '-j', metavar='N', type=int, nargs='?',
67                      default=1, const=multiprocessing.cpu_count(),
68                      help='''Runs the tests using N jobs. If the option is set
69                      but no value is provided, the script will use as many jobs
70                      as it thinks useful.''')
71  sim_default = 'off' if platform.machine() == 'aarch64' else 'on'
72  result.add_argument('--simulator', action='store', choices=['on', 'off'],
73                      default=sim_default,
74                      help='Explicitly enable or disable the simulator.')
75  return result.parse_args()
76
77
78def VerbosePrint(verbose, string):
79  if verbose:
80    Print(string)
81
82
83# A class representing an individual test.
84class Test:
85  def __init__(self, name, runner, debugger, coloured_trace, verbose):
86    self.name = name
87    self.runner = runner
88    self.debugger = debugger
89    self.coloured_trace = coloured_trace
90    self.verbose = verbose
91    self.logpath = os.path.join('log', os.path.basename(self.runner))
92    if self.debugger:
93      basename = name + '_debugger'
94    else:
95      basename = name
96    self.logout = os.path.join(self.logpath, basename + '.stdout')
97    self.logerr = os.path.join(self.logpath, basename + '.stderr')
98    if not os.path.exists(self.logpath): os.makedirs(self.logpath)
99
100  # Run the test.
101  # Use a thread to be able to control the test.
102  def Run(self):
103    command = \
104        [self.runner, '--trace_sim', '--trace_reg', '--trace_write', self.name]
105    if self.coloured_trace:
106      command.append('--coloured_trace')
107    if self.debugger:
108      command.append('--debugger')
109
110    VerbosePrint(self.verbose, '==== Running ' + self.name + '... ====')
111    sys.stdout.flush()
112
113    process = subprocess.Popen(command,
114                               stdout=subprocess.PIPE,
115                               stderr=subprocess.PIPE)
116    # Get the output and return status of the test.
117    stdout, stderr = process.communicate()
118    retcode = process.poll()
119
120    # Write stdout and stderr to the log.
121    with open(self.logout, 'w') as f: f.write(stdout)
122    with open(self.logerr, 'w') as f: f.write(stderr)
123
124    if retcode == 0:
125      # Success.
126      # We normally only print the command on failure, but with --verbose we
127      # should also print it on success.
128      VerbosePrint(self.verbose, 'COMMAND: ' + ' '.join(command))
129      VerbosePrint(self.verbose, 'LOG (stdout): ' + self.logout)
130      VerbosePrint(self.verbose, 'LOG (stderr): ' + self.logerr + '\n')
131    else:
132      # Failure.
133      Print('--- FAILED ' + self.name + ' ---')
134      Print('COMMAND: ' + ' '.join(command))
135      Print('LOG (stdout): ' + self.logout)
136      Print('LOG (stderr): ' + self.logerr + '\n')
137
138    return retcode
139
140
141# Scan matching tests and return a test manifest.
142def ReadManifest(runner, filters = [],
143                 debugger = False, coloured_trace = False, verbose = False):
144  status, output = util.getstatusoutput(runner +  ' --list')
145  if status != 0: util.abort('Failed to list all tests')
146
147  names = output.split()
148  for f in filters:
149    names = filter(re.compile(f).search, names)
150
151  return map(lambda x:
152      Test(x, runner, debugger, coloured_trace, verbose), names)
153
154
155# Shared state for multiprocessing. Ideally the context should be passed with
156# arguments, but constraints from the multiprocessing module prevent us from
157# doing so: the shared variables (multiprocessing.Value) must be global, or no
158# work is started. So we abstract some additional state into global variables to
159# simplify the implementation.
160# Read-write variables for the workers.
161n_tests_passed = multiprocessing.Value('i', 0)
162n_tests_failed = multiprocessing.Value('i', 0)
163# Read-only for workers.
164n_tests = None
165start_time = None
166verbose_test_run = None
167test_suite_name = ''
168
169def RunTest(test):
170  UpdateProgress(start_time, n_tests_passed.value, n_tests_failed.value,
171                 n_tests, verbose_test_run, test.name, test_suite_name)
172  # Run the test and update the statistics.
173  retcode = test.Run()
174  if retcode == 0:
175    with n_tests_passed.get_lock(): n_tests_passed.value += 1
176  else:
177    with n_tests_failed.get_lock(): n_tests_failed.value += 1
178
179
180# Run all tests in the manifest.
181# This function won't run in parallel due to constraints from the
182# multiprocessing module.
183__run_tests_lock__ = multiprocessing.Lock()
184def RunTests(manifest, jobs = 1, verbose = False, debugger = False,
185             progress_prefix = ''):
186  global n_tests
187  global start_time
188  global verbose_test_run
189  global test_suite_name
190
191  with __run_tests_lock__:
192
193    # Reset the counters.
194    n_tests_passed.value = 0
195    n_tests_failed.value = 0
196
197    verbose_test_run = verbose
198    test_suite_name = progress_prefix
199
200    n_tests = len(manifest)
201
202    if n_tests == 0:
203      Print('No tests to run.')
204      return 0
205
206    VerbosePrint(verbose, 'Running %d tests...' % (n_tests))
207    start_time = time.time()
208
209    pool = multiprocessing.Pool(jobs)
210    # The '.get(9999999)' is workaround to allow killing the test script with
211    # ctrl+C from the shell. This bug is documented at
212    # http://bugs.python.org/issue8296.
213    work = pool.map_async(RunTest, manifest).get(9999999)
214
215    done_message = '== Done =='
216    UpdateProgress(start_time, n_tests_passed.value, n_tests_failed.value,
217                   n_tests, verbose, done_message, progress_prefix)
218
219    return n_tests_failed.value # 0 indicates success
220
221
222if __name__ == '__main__':
223  # $ROOT/tools/test.py
224  root_dir = os.path.dirname(os.path.dirname(os.path.abspath(sys.argv[0])))
225
226  # Parse the arguments.
227  args = BuildOptions()
228
229  # Find a valid path to args.runner (in case it doesn't begin with './').
230  args.runner = os.path.join('.', args.runner)
231
232  if not os.access(args.runner, os.X_OK):
233    print "'" + args.test + "' is not executable or does not exist."
234    sys.exit(1)
235
236  # List all matching tests.
237  manifest = ReadManifest(args.runner, args.filters,
238                          args.debugger, args.coloured_trace, args.verbose)
239
240  # Run the tests.
241  status = RunTests(manifest, jobs=args.jobs,
242                    verbose=args.verbose, debugger=args.debugger)
243  EnsureNewLine()
244
245  sys.exit(status)
246
247