• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/env python2.7
2
3# Copyright 2015, VIXL authors
4# All rights reserved.
5#
6# Redistribution and use in source and binary forms, with or without
7# modification, are permitted provided that the following conditions are met:
8#
9#   * Redistributions of source code must retain the above copyright notice,
10#     this list of conditions and the following disclaimer.
11#   * Redistributions in binary form must reproduce the above copyright notice,
12#     this list of conditions and the following disclaimer in the documentation
13#     and/or other materials provided with the distribution.
14#   * Neither the name of ARM Limited nor the names of its contributors may be
15#     used to endorse or promote products derived from this software without
16#     specific prior written permission.
17#
18# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS CONTRIBUTORS "AS IS" AND
19# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
20# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
21# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
22# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
23# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
24# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
25# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
26# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28
29import argparse
30import fcntl
31import git
32import itertools
33import multiprocessing
34import os
35from os.path import join
36import platform
37import subprocess
38import sys
39import time
40
41import config
42import clang_format
43import clang_tidy
44import lint
45import printer
46import test
47import test_runner
48import util
49
50
51dir_root = config.dir_root
52
53
54# Remove duplicates from a list
55def RemoveDuplicates(values):
56  # Convert the list into a set and back to list
57  # as sets guarantee items are unique.
58  return list(set(values))
59
60
61# Custom argparse.Action to automatically add and handle an 'all' option.
62# If no 'default' value is set, it will default to 'all.
63# If accepted options are set using 'choices' then only these values will be
64# allowed.
65# If they're set using 'soft_choices' then 'all' will default to these values,
66# but other values will also be accepted.
67class AllChoiceAction(argparse.Action):
68
69  # At least one option was set by the user.
70  WasSetByUser = False
71
72  def __init__(self, **kwargs):
73    if 'choices' in kwargs:
74      assert 'soft_choices' not in kwargs,\
75          "Can't have both 'choices' and 'soft_choices' options"
76      self.all_choices = list(kwargs['choices'])
77      kwargs['choices'].append('all')
78    else:
79      self.all_choices = kwargs['soft_choices']
80      kwargs['help'] += ' Supported values: {' + ','.join(
81          ['all'] + self.all_choices) + '}'
82      del kwargs['soft_choices']
83    if 'default' not in kwargs:
84      kwargs['default'] = self.all_choices
85    super(AllChoiceAction, self).__init__(**kwargs)
86
87  def __call__(self, parser, namespace, values, option_string=None):
88    AllChoiceAction.WasSetByUser = True
89    if 'all' in values:
90      # Substitute 'all' by the actual values.
91      values = self.all_choices + [value for value in values if value != 'all']
92
93    setattr(namespace, self.dest, RemoveDuplicates(values))
94
95
96def BuildOptions():
97  args = argparse.ArgumentParser(
98    description =
99    '''This tool runs all tests matching the specified filters for multiple
100    environment, build options, and runtime options configurations.''',
101    # Print default values.
102    formatter_class=argparse.ArgumentDefaultsHelpFormatter)
103
104  args.add_argument('filters', metavar='filter', nargs='*',
105                    help='Run tests matching all of the (regexp) filters.')
106
107  # We automatically build the script options from the options to be tested.
108  test_arguments = args.add_argument_group(
109    'Test options',
110    'These options indicate what should be tested')
111  test_arguments.add_argument(
112      '--negative_testing',
113      help='Tests with negative testing enabled.',
114      action='store_const',
115      const='on',
116      default='off')
117  test_arguments.add_argument(
118      '--compiler',
119      help='Test for the specified compilers.',
120      soft_choices=config.tested_compilers,
121      action=AllChoiceAction,
122      nargs="+")
123  test_arguments.add_argument(
124      '--mode',
125      help='Test with the specified build modes.',
126      choices=config.build_options_modes,
127      action=AllChoiceAction,
128      nargs="+")
129  test_arguments.add_argument(
130      '--std',
131      help='Test with the specified C++ standard.',
132      soft_choices=config.tested_cpp_standards,
133      action=AllChoiceAction,
134      nargs="+")
135  test_arguments.add_argument(
136      '--target',
137      help='Test with the specified isa enabled.',
138      soft_choices=config.build_options_target,
139      action=AllChoiceAction,
140      nargs="+")
141
142  general_arguments = args.add_argument_group('General options')
143  general_arguments.add_argument('--dry-run', action='store_true',
144                                 help='''Don't actually build or run anything,
145                                 but print the configurations that would be
146                                 tested.''')
147  general_arguments.add_argument('--verbose', action='store_true',
148                                 help='''Print extra information.''')
149  general_arguments.add_argument(
150    '--jobs', '-j', metavar='N', type=int, nargs='?',
151    default=multiprocessing.cpu_count(),
152    const=multiprocessing.cpu_count(),
153    help='''Runs the tests using N jobs. If the option is set but no value is
154    provided, the script will use as many jobs as it thinks useful.''')
155  general_arguments.add_argument('--clang-format',
156                                 default=clang_format.DEFAULT_CLANG_FORMAT,
157                                 help='Path to clang-format.')
158  general_arguments.add_argument('--clang-tidy',
159                                 default=clang_tidy.DEFAULT_CLANG_TIDY,
160                                 help='Path to clang-tidy.')
161  general_arguments.add_argument('--nobench', action='store_true',
162                                 help='Do not run benchmarks.')
163  general_arguments.add_argument('--nolint', action='store_true',
164                                 help='Do not run the linter.')
165  general_arguments.add_argument('--noclang-format', action='store_true',
166                                 help='Do not run clang-format.')
167  general_arguments.add_argument('--noclang-tidy', action='store_true',
168                                 help='Do not run clang-tidy.')
169  general_arguments.add_argument('--notest', action='store_true',
170                                 help='Do not run tests.')
171  general_arguments.add_argument('--fail-early', action='store_true',
172                                 help='Exit as soon as a test fails.')
173  general_arguments.add_argument(
174    '--under_valgrind', action='store_true',
175    help='''Run the test-runner commands under Valgrind.
176            Note that a few tests are known to fail because of
177            issues in Valgrind''')
178  return args.parse_args()
179
180
181def RunCommand(command, environment_options = None):
182  # Create a copy of the environment. We do not want to pollute the environment
183  # of future commands run.
184  environment = os.environ.copy()
185
186  printable_command = ''
187  if environment_options:
188    # Add the environment options to the environment:
189    environment.update(environment_options)
190    printable_command += ' ' + DictToString(environment_options) + ' '
191  printable_command += ' '.join(command)
192
193  printable_command_orange = \
194    printer.COLOUR_ORANGE + printable_command + printer.NO_COLOUR
195  printer.PrintOverwritableLine(printable_command_orange)
196  sys.stdout.flush()
197
198  # Start a process for the command.
199  # Interleave `stderr` and `stdout`.
200  p = subprocess.Popen(command,
201                       stdout=subprocess.PIPE,
202                       stderr=subprocess.STDOUT,
203                       env=environment)
204
205  # We want to be able to display a continuously updated 'work indicator' while
206  # the process is running. Since the process can hang if the `stdout` pipe is
207  # full, we need to pull from it regularly. We cannot do so via the
208  # `readline()` function because it is blocking, and would thus cause the
209  # indicator to not be updated properly. So use file control mechanisms
210  # instead.
211  indicator = ' (still working: %d seconds elapsed)'
212
213  # Mark the process output as non-blocking.
214  flags = fcntl.fcntl(p.stdout, fcntl.F_GETFL)
215  fcntl.fcntl(p.stdout, fcntl.F_SETFL, flags | os.O_NONBLOCK)
216
217  t_start = time.time()
218  t_current = t_start
219  t_last_indication = t_start
220  t_current = t_start
221  process_output = ''
222
223  # Keep looping as long as the process is running.
224  while p.poll() is None:
225    # Avoid polling too often.
226    time.sleep(0.1)
227    # Update the progress indicator.
228    t_current = time.time()
229    if (t_current - t_start >= 2) and (t_current - t_last_indication >= 1):
230      printer.PrintOverwritableLine(
231        printable_command_orange + indicator % int(t_current - t_start))
232      sys.stdout.flush()
233      t_last_indication = t_current
234    # Pull from the process output.
235    while True:
236      try:
237        line = os.read(p.stdout.fileno(), 1024)
238      except OSError:
239        line = ''
240        break
241      if line == '': break
242      process_output += line
243
244  # The process has exited. Don't forget to retrieve the rest of its output.
245  out, err = p.communicate()
246  rc = p.poll()
247  process_output += out
248
249  printable_command += ' (took %d seconds)' % int(t_current - t_start)
250  if rc == 0:
251    printer.Print(printer.COLOUR_GREEN + printable_command + printer.NO_COLOUR)
252  else:
253    printer.Print(printer.COLOUR_RED + printable_command + printer.NO_COLOUR)
254    printer.Print(process_output)
255  return rc
256
257
258def RunLinter(jobs):
259  return lint.RunLinter(map(lambda x: join(dir_root, x),
260                        util.get_source_files()),
261                        jobs = args.jobs, progress_prefix = 'cpp lint: ')
262
263
264def RunClangFormat(clang_path, jobs):
265  return clang_format.ClangFormatFiles(util.get_source_files(),
266                                       clang_path,
267                                       jobs = jobs,
268                                       progress_prefix = 'clang-format: ')
269
270def RunClangTidy(clang_path, jobs):
271  return clang_tidy.ClangTidyFiles(util.get_source_files(),
272                                   clang_path,
273                                   jobs = jobs,
274                                   progress_prefix = 'clang-tidy: ')
275
276def BuildAll(build_options, jobs, environment_options):
277  scons_command = ['scons', '-C', dir_root, 'all', '-j', str(jobs)]
278  if util.IsCommandAvailable('ccache'):
279    scons_command += ['compiler_wrapper=ccache']
280    # Fixes warnings for ccache 3.3.1 and lower:
281    environment_options = environment_options.copy()
282    environment_options["CCACHE_CPP2"] = 'yes'
283  scons_command += DictToString(build_options).split()
284  return RunCommand(scons_command, environment_options)
285
286
287def CanRunAarch64(options, args):
288  for target in options['target']:
289    if target in ['aarch64', 'a64']:
290      return True
291
292  return False
293
294
295def CanRunAarch32(options, args):
296  for target in options['target']:
297    if target in ['aarch32', 'a32', 't32']:
298      return True
299  return False
300
301
302def RunBenchmarks(options, args):
303  rc = 0
304  if CanRunAarch32(options, args):
305    benchmark_names = util.ListCCFilesWithoutExt(config.dir_aarch32_benchmarks)
306    for bench in benchmark_names:
307      rc |= RunCommand(
308        [os.path.realpath(
309          join(config.dir_build_latest, 'benchmarks/aarch32', bench)), '10'])
310  if CanRunAarch64(options, args):
311    benchmark_names = util.ListCCFilesWithoutExt(config.dir_aarch64_benchmarks)
312    for bench in benchmark_names:
313      rc |= RunCommand(
314        [util.relrealpath(
315            join(config.dir_build_latest,
316                'benchmarks/aarch64', bench)), '10'])
317  return rc
318
319
320
321# It is a precommit run if the user did not specify any of the
322# options that would affect the automatically generated combinations.
323def IsPrecommitRun(args):
324  return args.negative_testing == "off" and not AllChoiceAction.WasSetByUser
325
326# Generate a list of all the possible combinations of the passed list:
327# ListCombinations( a = [a0, a1], b = [b0, b1] ) will return
328# [ {a : a0, b : b0}, {a : a0, b : b1}, {a: a1, b : b0}, {a : a1, b : b1}]
329def ListCombinations(**kwargs):
330  # End of recursion: no options passed
331  if not kwargs:
332    return [{}]
333  option, values = kwargs.popitem()
334  configs = ListCombinations(**kwargs)
335  retval = []
336  if not isinstance(values, list):
337    values = [values]
338  for value in values:
339    for config in configs:
340      new_config = config.copy()
341      new_config[option] = value
342      retval.append(new_config)
343  return retval
344
345# Convert a dictionary into a space separated string
346# {a : a0, b : b0} --> "a=a0 b=b0"
347def DictToString(options):
348  return " ".join(
349      ["{}={}".format(option, value) for option, value in options.items()])
350
351
352if __name__ == '__main__':
353  util.require_program('scons')
354
355  args = BuildOptions()
356
357  rc = util.ReturnCode(args.fail_early, printer.Print)
358
359  if args.under_valgrind:
360    util.require_program('valgrind')
361
362  tests = test_runner.TestQueue()
363  if not args.nolint and not args.dry_run:
364    rc.Combine(RunLinter(args.jobs))
365
366  if not args.noclang_format and not args.dry_run:
367    rc.Combine(RunClangFormat(args.clang_format, args.jobs))
368
369  if not args.noclang_tidy and not args.dry_run:
370    rc.Combine(RunClangTidy(args.clang_tidy, args.jobs))
371
372  list_options = []
373  if IsPrecommitRun(args):
374    # Maximize the coverage for precommit testing.
375
376    # Debug builds with negative testing and all targets enabled.
377    list_options += ListCombinations(
378        compiler = args.compiler,
379        negative_testing = 'on',
380        std = args.std,
381        mode = 'debug',
382        target = 'a64,a32,t32')
383
384    # Release builds with all targets enabled.
385    list_options += ListCombinations(
386        compiler = args.compiler,
387        negative_testing = 'off',
388        std = args.std,
389        mode = 'release',
390        target = 'a64,a32,t32')
391
392    # Debug builds for individual targets.
393    list_options += ListCombinations(
394        compiler = args.compiler[0],
395        negative_testing = 'off',
396        std = args.std,
397        mode = 'debug',
398        target = ['a32', 't32', 'a64'])
399  else:
400    list_options = ListCombinations(
401        compiler = args.compiler,
402        negative_testing = args.negative_testing,
403        std = args.std,
404        mode = args.mode,
405        target = args.target)
406
407  for options in list_options:
408    if (args.dry_run):
409      print(DictToString(options))
410      continue
411    # Convert 'compiler' into an environment variable:
412    environment_options = {'CXX': options['compiler']}
413    del options['compiler']
414
415    # Avoid going through the build stage if we are not using the build
416    # result.
417    if not (args.notest and args.nobench):
418      build_rc = BuildAll(options, args.jobs, environment_options)
419      # Don't run the tests for this configuration if the build failed.
420      if build_rc != 0:
421        rc.Combine(build_rc)
422        continue
423
424      # Use the realpath of the test executable so that the commands printed
425      # can be copy-pasted and run.
426      test_executable = util.relrealpath(
427        join(config.dir_build_latest, 'test', 'test-runner'))
428
429      if not args.notest:
430        printer.Print(test_executable)
431        tests.AddTests(
432            test_executable,
433            args.filters,
434            list(),
435            args.under_valgrind)
436
437      if not args.nobench:
438        rc.Combine(RunBenchmarks(options, args))
439
440  rc.Combine(tests.Run(args.jobs, args.verbose))
441  if not args.dry_run:
442    rc.PrintStatus()
443
444  sys.exit(rc.Value)
445