• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/env python2.7
2
3# Copyright 2015, VIXL authors
4# All rights reserved.
5#
6# Redistribution and use in source and binary forms, with or without
7# modification, are permitted provided that the following conditions are met:
8#
9#   * Redistributions of source code must retain the above copyright notice,
10#     this list of conditions and the following disclaimer.
11#   * Redistributions in binary form must reproduce the above copyright notice,
12#     this list of conditions and the following disclaimer in the documentation
13#     and/or other materials provided with the distribution.
14#   * Neither the name of ARM Limited nor the names of its contributors may be
15#     used to endorse or promote products derived from this software without
16#     specific prior written permission.
17#
18# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS CONTRIBUTORS "AS IS" AND
19# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
20# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
21# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
22# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
23# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
24# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
25# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
26# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28
29import argparse
30import fcntl
31import git
32import itertools
33import multiprocessing
34import os
35from os.path import join
36import platform
37import subprocess
38import sys
39import time
40
41import config
42import clang_format
43import clang_tidy
44import lint
45import printer
46import test
47import test_runner
48import util
49
50
51dir_root = config.dir_root
52
53
54# Remove duplicates from a list
55def RemoveDuplicates(values):
56  # Convert the list into a set and back to list
57  # as sets guarantee items are unique.
58  return list(set(values))
59
60
61# Custom argparse.Action to automatically add and handle an 'all' option.
62# If no 'default' value is set, it will default to 'all.
63# If accepted options are set using 'choices' then only these values will be
64# allowed.
65# If they're set using 'soft_choices' then 'all' will default to these values,
66# but other values will also be accepted.
67class AllChoiceAction(argparse.Action):
68
69  # At least one option was set by the user.
70  WasSetByUser = False
71
72  def __init__(self, **kwargs):
73    if 'choices' in kwargs:
74      assert 'soft_choices' not in kwargs,\
75          "Can't have both 'choices' and 'soft_choices' options"
76      self.all_choices = list(kwargs['choices'])
77      kwargs['choices'].append('all')
78    else:
79      self.all_choices = kwargs['soft_choices']
80      kwargs['help'] += ' Supported values: {' + ','.join(
81          ['all'] + self.all_choices) + '}'
82      del kwargs['soft_choices']
83    if 'default' not in kwargs:
84      kwargs['default'] = self.all_choices
85    super(AllChoiceAction, self).__init__(**kwargs)
86
87  def __call__(self, parser, namespace, values, option_string=None):
88    AllChoiceAction.WasSetByUser = True
89    if 'all' in values:
90      # Substitute 'all' by the actual values.
91      values = self.all_choices + [value for value in values if value != 'all']
92
93    setattr(namespace, self.dest, RemoveDuplicates(values))
94
95
96def BuildOptions():
97  args = argparse.ArgumentParser(
98    description =
99    '''This tool runs all tests matching the specified filters for multiple
100    environment, build options, and runtime options configurations.''',
101    # Print default values.
102    formatter_class=argparse.ArgumentDefaultsHelpFormatter)
103
104  args.add_argument('filters', metavar='filter', nargs='*',
105                    help='Run tests matching all of the (regexp) filters.')
106
107  # We automatically build the script options from the options to be tested.
108  test_arguments = args.add_argument_group(
109    'Test options',
110    'These options indicate what should be tested')
111  test_arguments.add_argument(
112      '--negative_testing',
113      help='Tests with negative testing enabled.',
114      action='store_const',
115      const='on',
116      default='off')
117  test_arguments.add_argument(
118      '--compiler',
119      help='Test for the specified compilers.',
120      soft_choices=config.tested_compilers,
121      action=AllChoiceAction,
122      nargs="+")
123  test_arguments.add_argument(
124      '--mode',
125      help='Test with the specified build modes.',
126      choices=config.build_options_modes,
127      action=AllChoiceAction,
128      nargs="+")
129  test_arguments.add_argument(
130      '--std',
131      help='Test with the specified C++ standard.',
132      soft_choices=config.tested_cpp_standards,
133      action=AllChoiceAction,
134      nargs="+")
135  test_arguments.add_argument(
136      '--target',
137      help='Test with the specified isa enabled.',
138      soft_choices=config.build_options_target,
139      action=AllChoiceAction,
140      nargs="+")
141
142  general_arguments = args.add_argument_group('General options')
143  general_arguments.add_argument('--dry-run', action='store_true',
144                                 help='''Don't actually build or run anything,
145                                 but print the configurations that would be
146                                 tested.''')
147  general_arguments.add_argument('--verbose', action='store_true',
148                                 help='''Print extra information.''')
149  general_arguments.add_argument(
150    '--jobs', '-j', metavar='N', type=int, nargs='?',
151    default=multiprocessing.cpu_count(),
152    const=multiprocessing.cpu_count(),
153    help='''Runs the tests using N jobs. If the option is set but no value is
154    provided, the script will use as many jobs as it thinks useful.''')
155  general_arguments.add_argument('--clang-format',
156                                 default=clang_format.DEFAULT_CLANG_FORMAT,
157                                 help='Path to clang-format.')
158  general_arguments.add_argument('--clang-tidy',
159                                 default=clang_tidy.DEFAULT_CLANG_TIDY,
160                                 help='Path to clang-tidy.')
161  general_arguments.add_argument('--nobench', action='store_true',
162                                 help='Do not run benchmarks.')
163  general_arguments.add_argument('--nolint', action='store_true',
164                                 help='Do not run the linter.')
165  general_arguments.add_argument('--noclang-format', action='store_true',
166                                 help='Do not run clang-format.')
167  general_arguments.add_argument('--noclang-tidy', action='store_true',
168                                 help='Do not run clang-tidy.')
169  general_arguments.add_argument('--notest', action='store_true',
170                                 help='Do not run tests.')
171  general_arguments.add_argument('--nocheck-code-coverage', action='store_true',
172                                 help='Do not check code coverage results log.')
173  general_arguments.add_argument('--fail-early', action='store_true',
174                                 help='Exit as soon as a test fails.')
175  general_arguments.add_argument(
176    '--under_valgrind', action='store_true',
177    help='''Run the test-runner commands under Valgrind.
178            Note that a few tests are known to fail because of
179            issues in Valgrind''')
180  return args.parse_args()
181
182
183def RunCommand(command, environment_options = None):
184  # Create a copy of the environment. We do not want to pollute the environment
185  # of future commands run.
186  environment = os.environ.copy()
187
188  printable_command = ''
189  if environment_options:
190    # Add the environment options to the environment:
191    environment.update(environment_options)
192    printable_command += ' ' + DictToString(environment_options) + ' '
193  printable_command += ' '.join(command)
194
195  printable_command_orange = \
196    printer.COLOUR_ORANGE + printable_command + printer.NO_COLOUR
197  printer.PrintOverwritableLine(printable_command_orange)
198  sys.stdout.flush()
199
200  # Start a process for the command.
201  # Interleave `stderr` and `stdout`.
202  p = subprocess.Popen(command,
203                       stdout=subprocess.PIPE,
204                       stderr=subprocess.STDOUT,
205                       env=environment)
206
207  # We want to be able to display a continuously updated 'work indicator' while
208  # the process is running. Since the process can hang if the `stdout` pipe is
209  # full, we need to pull from it regularly. We cannot do so via the
210  # `readline()` function because it is blocking, and would thus cause the
211  # indicator to not be updated properly. So use file control mechanisms
212  # instead.
213  indicator = ' (still working: %d seconds elapsed)'
214
215  # Mark the process output as non-blocking.
216  flags = fcntl.fcntl(p.stdout, fcntl.F_GETFL)
217  fcntl.fcntl(p.stdout, fcntl.F_SETFL, flags | os.O_NONBLOCK)
218
219  t_start = time.time()
220  t_current = t_start
221  t_last_indication = t_start
222  t_current = t_start
223  process_output = ''
224
225  # Keep looping as long as the process is running.
226  while p.poll() is None:
227    # Avoid polling too often.
228    time.sleep(0.1)
229    # Update the progress indicator.
230    t_current = time.time()
231    if (t_current - t_start >= 2) and (t_current - t_last_indication >= 1):
232      printer.PrintOverwritableLine(
233        printable_command_orange + indicator % int(t_current - t_start))
234      sys.stdout.flush()
235      t_last_indication = t_current
236    # Pull from the process output.
237    while True:
238      try:
239        line = os.read(p.stdout.fileno(), 1024)
240      except OSError:
241        line = ''
242        break
243      if line == '': break
244      process_output += line
245
246  # The process has exited. Don't forget to retrieve the rest of its output.
247  out, err = p.communicate()
248  rc = p.poll()
249  process_output += out
250
251  printable_command += ' (took %d seconds)' % int(t_current - t_start)
252  if rc == 0:
253    printer.Print(printer.COLOUR_GREEN + printable_command + printer.NO_COLOUR)
254  else:
255    printer.Print(printer.COLOUR_RED + printable_command + printer.NO_COLOUR)
256    printer.Print(process_output)
257  return rc
258
259
260def RunLinter(jobs):
261  return lint.RunLinter(map(lambda x: join(dir_root, x),
262                        util.get_source_files()),
263                        jobs = args.jobs, progress_prefix = 'cpp lint: ')
264
265
266def RunClangFormat(clang_path, jobs):
267  return clang_format.ClangFormatFiles(util.get_source_files(),
268                                       clang_path,
269                                       jobs = jobs,
270                                       progress_prefix = 'clang-format: ')
271
272def RunClangTidy(clang_path, jobs):
273  return clang_tidy.ClangTidyFiles(util.get_source_files(),
274                                   clang_path,
275                                   jobs = jobs,
276                                   progress_prefix = 'clang-tidy: ')
277
278def CheckCodeCoverage():
279  command = ['tools/check_recent_coverage.sh']
280  return RunCommand(command)
281
282def BuildAll(build_options, jobs, environment_options):
283  scons_command = ['scons', '-C', dir_root, 'all', '-j', str(jobs)]
284  if util.IsCommandAvailable('ccache'):
285    scons_command += ['compiler_wrapper=ccache']
286    # Fixes warnings for ccache 3.3.1 and lower:
287    environment_options = environment_options.copy()
288    environment_options["CCACHE_CPP2"] = 'yes'
289  scons_command += DictToString(build_options).split()
290  return RunCommand(scons_command, environment_options)
291
292
293def CanRunAarch64(options, args):
294  for target in options['target']:
295    if target in ['aarch64', 'a64']:
296      return True
297
298  return False
299
300
301def CanRunAarch32(options, args):
302  for target in options['target']:
303    if target in ['aarch32', 'a32', 't32']:
304      return True
305  return False
306
307
308def RunBenchmarks(options, args):
309  rc = 0
310  if CanRunAarch32(options, args):
311    benchmark_names = util.ListCCFilesWithoutExt(config.dir_aarch32_benchmarks)
312    for bench in benchmark_names:
313      rc |= RunCommand(
314        [os.path.realpath(
315          join(config.dir_build_latest, 'benchmarks/aarch32', bench)), '10'])
316  if CanRunAarch64(options, args):
317    benchmark_names = util.ListCCFilesWithoutExt(config.dir_aarch64_benchmarks)
318    for bench in benchmark_names:
319      rc |= RunCommand(
320        [util.relrealpath(
321            join(config.dir_build_latest,
322                'benchmarks/aarch64', bench)), '10'])
323  return rc
324
325
326
327# It is a precommit run if the user did not specify any of the
328# options that would affect the automatically generated combinations.
329def IsPrecommitRun(args):
330  return args.negative_testing == "off" and not AllChoiceAction.WasSetByUser
331
332# Generate a list of all the possible combinations of the passed list:
333# ListCombinations( a = [a0, a1], b = [b0, b1] ) will return
334# [ {a : a0, b : b0}, {a : a0, b : b1}, {a: a1, b : b0}, {a : a1, b : b1}]
335def ListCombinations(**kwargs):
336  # End of recursion: no options passed
337  if not kwargs:
338    return [{}]
339  option, values = kwargs.popitem()
340  configs = ListCombinations(**kwargs)
341  retval = []
342  if not isinstance(values, list):
343    values = [values]
344  for value in values:
345    for config in configs:
346      new_config = config.copy()
347      new_config[option] = value
348      retval.append(new_config)
349  return retval
350
351# Convert a dictionary into a space separated string
352# {a : a0, b : b0} --> "a=a0 b=b0"
353def DictToString(options):
354  return " ".join(
355      ["{}={}".format(option, value) for option, value in options.items()])
356
357
358if __name__ == '__main__':
359  util.require_program('scons')
360
361  args = BuildOptions()
362
363  rc = util.ReturnCode(args.fail_early, printer.Print)
364
365  if args.under_valgrind:
366    util.require_program('valgrind')
367
368  if not args.nocheck_code_coverage:
369    rc.Combine(CheckCodeCoverage())
370
371  tests = test_runner.TestQueue()
372  if not args.nolint and not args.dry_run:
373    rc.Combine(RunLinter(args.jobs))
374
375  if not args.noclang_format and not args.dry_run:
376    rc.Combine(RunClangFormat(args.clang_format, args.jobs))
377
378  if not args.noclang_tidy and not args.dry_run:
379    rc.Combine(RunClangTidy(args.clang_tidy, args.jobs))
380
381  list_options = []
382  if IsPrecommitRun(args):
383    # Maximize the coverage for precommit testing.
384
385    # Debug builds with negative testing and all targets enabled.
386    list_options += ListCombinations(
387        compiler = args.compiler,
388        negative_testing = 'on',
389        std = args.std,
390        mode = 'debug',
391        target = 'a64,a32,t32')
392
393    # Release builds with all targets enabled.
394    list_options += ListCombinations(
395        compiler = args.compiler,
396        negative_testing = 'off',
397        std = args.std,
398        mode = 'release',
399        target = 'a64,a32,t32')
400
401    # Debug builds for individual targets.
402    list_options += ListCombinations(
403        compiler = args.compiler[0],
404        negative_testing = 'off',
405        std = args.std,
406        mode = 'debug',
407        target = ['a32', 't32', 'a64'])
408  else:
409    list_options = ListCombinations(
410        compiler = args.compiler,
411        negative_testing = args.negative_testing,
412        std = args.std,
413        mode = args.mode,
414        target = args.target)
415
416  for options in list_options:
417    if (args.dry_run):
418      print(DictToString(options))
419      continue
420    # Convert 'compiler' into an environment variable:
421    environment_options = {'CXX': options['compiler']}
422    del options['compiler']
423
424    # Avoid going through the build stage if we are not using the build
425    # result.
426    if not (args.notest and args.nobench):
427      build_rc = BuildAll(options, args.jobs, environment_options)
428      # Don't run the tests for this configuration if the build failed.
429      if build_rc != 0:
430        rc.Combine(build_rc)
431        continue
432
433      # Use the realpath of the test executable so that the commands printed
434      # can be copy-pasted and run.
435      test_executable = util.relrealpath(
436        join(config.dir_build_latest, 'test', 'test-runner'))
437
438      if not args.notest:
439        printer.Print(test_executable)
440        tests.AddTests(
441            test_executable,
442            args.filters,
443            list(),
444            args.under_valgrind)
445
446      if not args.nobench:
447        rc.Combine(RunBenchmarks(options, args))
448
449  rc.Combine(tests.Run(args.jobs, args.verbose))
450  if not args.dry_run:
451    rc.PrintStatus()
452
453  sys.exit(rc.Value)
454