• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/env python3
2#
3# Copyright 2017 the V8 project authors. All rights reserved.
4# Use of this source code is governed by a BSD-style license that can be
5# found in the LICENSE file.
6
7from functools import reduce
8
9import datetime
10import json
11import os
12import sys
13import tempfile
14
15# Adds testrunner to the path hence it has to be imported at the beggining.
16import testrunner.base_runner as base_runner
17
18from testrunner.local import utils
19from testrunner.local.variants import ALL_VARIANTS
20from testrunner.objects import predictable
21from testrunner.testproc.execution import ExecutionProc
22from testrunner.testproc.filter import StatusFileFilterProc, NameFilterProc
23from testrunner.testproc.loader import LoadProc
24from testrunner.testproc.seed import SeedProc
25from testrunner.testproc.sequence import SequenceProc
26from testrunner.testproc.variant import VariantProc
27
28
29VARIANTS = ['default']
30
31MORE_VARIANTS = [
32  'jitless',
33  'stress',
34  'stress_js_bg_compile_wasm_code_gc',
35  'stress_incremental_marking',
36]
37
38VARIANT_ALIASES = {
39  # The default for developer workstations.
40  'dev': VARIANTS,
41  # Additional variants, run on all bots.
42  'more': MORE_VARIANTS,
43  # Shortcut for the two above ('more' first - it has the longer running tests)
44  'exhaustive': MORE_VARIANTS + VARIANTS,
45  # Additional variants, run on a subset of bots.
46  'extra': ['nooptimization', 'future', 'no_wasm_traps',
47            'instruction_scheduling', 'always_sparkplug'],
48}
49
50# Extra flags passed to all tests using the standard test runner.
51EXTRA_DEFAULT_FLAGS = ['--testing-d8-test-runner']
52
53GC_STRESS_FLAGS = ['--gc-interval=500', '--stress-compaction',
54                   '--concurrent-recompilation-queue-length=64',
55                   '--concurrent-recompilation-delay=500',
56                   '--concurrent-recompilation',
57                   '--stress-flush-code', '--flush-bytecode',
58                   '--wasm-code-gc', '--stress-wasm-code-gc']
59
60RANDOM_GC_STRESS_FLAGS = ['--random-gc-interval=5000',
61                          '--stress-compaction-random']
62
63
64PREDICTABLE_WRAPPER = os.path.join(
65    base_runner.BASE_DIR, 'tools', 'predictable_wrapper.py')
66
67
68class StandardTestRunner(base_runner.BaseTestRunner):
69  def __init__(self, *args, **kwargs):
70    super(StandardTestRunner, self).__init__(*args, **kwargs)
71
72    self.sancov_dir = None
73    self._variants = None
74
75  @property
76  def framework_name(self):
77    return 'standard_runner'
78
79  def _get_default_suite_names(self):
80    return ['default']
81
82  def _add_parser_options(self, parser):
83    parser.add_option('--novfp3',
84                      help='Indicates that V8 was compiled without VFP3'
85                      ' support',
86                      default=False, action='store_true')
87
88    # Variants
89    parser.add_option('--no-variants', '--novariants',
90                      help='Deprecated. '
91                           'Equivalent to passing --variants=default',
92                      default=False, dest='no_variants', action='store_true')
93    parser.add_option('--variants',
94                      help='Comma-separated list of testing variants;'
95                      ' default: "%s"' % ','.join(VARIANTS))
96    parser.add_option('--exhaustive-variants',
97                      default=False, action='store_true',
98                      help='Deprecated. '
99                           'Equivalent to passing --variants=exhaustive')
100
101    # Filters
102    parser.add_option('--slow-tests', default='dontcare',
103                      help='Regard slow tests (run|skip|dontcare)')
104    parser.add_option('--pass-fail-tests', default='dontcare',
105                      help='Regard pass|fail tests (run|skip|dontcare)')
106    parser.add_option('--quickcheck', default=False, action='store_true',
107                      help=('Quick check mode (skip slow tests)'))
108
109    # Stress modes
110    parser.add_option('--gc-stress',
111                      help='Switch on GC stress mode',
112                      default=False, action='store_true')
113    parser.add_option('--random-gc-stress',
114                      help='Switch on random GC stress mode',
115                      default=False, action='store_true')
116    parser.add_option('--random-seed-stress-count', default=1, type='int',
117                      dest='random_seed_stress_count',
118                      help='Number of runs with different random seeds. Only '
119                           'with test processors: 0 means infinite '
120                           'generation.')
121
122    # Extra features.
123    parser.add_option('--max-heavy-tests', default=1, type='int',
124                      help='Maximum number of heavy tests run in parallel')
125    parser.add_option('--time', help='Print timing information after running',
126                      default=False, action='store_true')
127
128    # Noop
129    parser.add_option('--cfi-vptr',
130                      help='Run tests with UBSAN cfi_vptr option.',
131                      default=False, action='store_true')
132    parser.add_option('--no-sorting', '--nosorting',
133                      help='Don\'t sort tests according to duration of last'
134                      ' run.',
135                      default=False, dest='no_sorting', action='store_true')
136    parser.add_option('--no-presubmit', '--nopresubmit',
137                      help='Skip presubmit checks (deprecated)',
138                      default=False, dest='no_presubmit', action='store_true')
139
140    # Unimplemented for test processors
141    parser.add_option('--sancov-dir',
142                      help='Directory where to collect coverage data')
143    parser.add_option('--cat', help='Print the source of the tests',
144                      default=False, action='store_true')
145    parser.add_option('--flakiness-results',
146                      help='Path to a file for storing flakiness json.')
147    parser.add_option('--warn-unused', help='Report unused rules',
148                      default=False, action='store_true')
149    parser.add_option('--report', default=False, action='store_true',
150                      help='Print a summary of the tests to be run')
151
152  def _process_options(self, options):
153    if options.sancov_dir:
154      self.sancov_dir = options.sancov_dir
155      if not os.path.exists(self.sancov_dir):
156        print('sancov-dir %s doesn\'t exist' % self.sancov_dir)
157        raise base_runner.TestRunnerError()
158
159    if options.gc_stress:
160      options.extra_flags += GC_STRESS_FLAGS
161
162    if options.random_gc_stress:
163      options.extra_flags += RANDOM_GC_STRESS_FLAGS
164
165    if self.build_config.asan:
166      options.extra_flags.append('--invoke-weak-callbacks')
167
168    if options.novfp3:
169      options.extra_flags.append('--noenable-vfp3')
170
171    if options.no_variants:  # pragma: no cover
172      print ('Option --no-variants is deprecated. '
173             'Pass --variants=default instead.')
174      assert not options.variants
175      options.variants = 'default'
176
177    if options.exhaustive_variants:  # pragma: no cover
178      # TODO(machenbach): Switch infra to --variants=exhaustive after M65.
179      print ('Option --exhaustive-variants is deprecated. '
180             'Pass --variants=exhaustive instead.')
181      # This is used on many bots. It includes a larger set of default
182      # variants.
183      # Other options for manipulating variants still apply afterwards.
184      assert not options.variants
185      options.variants = 'exhaustive'
186
187    if options.quickcheck:
188      assert not options.variants
189      options.variants = 'stress,default'
190      options.slow_tests = 'skip'
191      options.pass_fail_tests = 'skip'
192
193    if self.build_config.predictable:
194      options.variants = 'default'
195      options.extra_flags.append('--predictable')
196      options.extra_flags.append('--verify-predictable')
197      options.extra_flags.append('--no-inline-new')
198      # Add predictable wrapper to command prefix.
199      options.command_prefix = (
200          [sys.executable, PREDICTABLE_WRAPPER] + options.command_prefix)
201
202    # TODO(machenbach): Figure out how to test a bigger subset of variants on
203    # msan.
204    if self.build_config.msan:
205      options.variants = 'default'
206
207    if options.variants == 'infra_staging':
208      options.variants = 'exhaustive'
209
210    self._variants = self._parse_variants(options.variants)
211
212    def CheckTestMode(name, option):  # pragma: no cover
213      if option not in ['run', 'skip', 'dontcare']:
214        print('Unknown %s mode %s' % (name, option))
215        raise base_runner.TestRunnerError()
216    CheckTestMode('slow test', options.slow_tests)
217    CheckTestMode('pass|fail test', options.pass_fail_tests)
218    if self.build_config.no_i18n:
219      base_runner.TEST_MAP['bot_default'].remove('intl')
220      base_runner.TEST_MAP['default'].remove('intl')
221      # TODO(machenbach): uncomment after infra side lands.
222      # base_runner.TEST_MAP['d8_default'].remove('intl')
223
224    if options.time and not options.json_test_results:
225      # We retrieve the slowest tests from the JSON output file, so create
226      # a temporary output file (which will automatically get deleted on exit)
227      # if the user didn't specify one.
228      self._temporary_json_output_file = tempfile.NamedTemporaryFile(
229          prefix="v8-test-runner-")
230      options.json_test_results = self._temporary_json_output_file.name
231
232  def _runner_flags(self):
233    return EXTRA_DEFAULT_FLAGS
234
235  def _parse_variants(self, aliases_str):
236    # Use developer defaults if no variant was specified.
237    aliases_str = aliases_str or 'dev'
238    aliases = aliases_str.split(',')
239    user_variants = set(reduce(
240        list.__add__, [VARIANT_ALIASES.get(a, [a]) for a in aliases]))
241
242    result = [v for v in ALL_VARIANTS if v in user_variants]
243    if len(result) == len(user_variants):
244      return result
245
246    for v in user_variants:
247      if v not in ALL_VARIANTS:
248        print('Unknown variant: %s' % v)
249        print('    Available variants: %s' % ALL_VARIANTS)
250        print('    Available variant aliases: %s' % VARIANT_ALIASES.keys());
251        raise base_runner.TestRunnerError()
252    assert False, 'Unreachable'
253
254  def _setup_env(self):
255    super(StandardTestRunner, self)._setup_env()
256
257    symbolizer_option = self._get_external_symbolizer_option()
258
259    if self.sancov_dir:
260      os.environ['ASAN_OPTIONS'] = ':'.join([
261        'coverage=1',
262        'coverage_dir=%s' % self.sancov_dir,
263        symbolizer_option,
264        'allow_user_segv_handler=1',
265      ])
266
267  def _get_statusfile_variables(self, options):
268    variables = (
269        super(StandardTestRunner, self)._get_statusfile_variables(options))
270
271    variables.update({
272      'gc_stress': options.gc_stress or options.random_gc_stress,
273      'gc_fuzzer': options.random_gc_stress,
274      'novfp3': options.novfp3,
275    })
276    return variables
277
278  def _create_sequence_proc(self, options):
279    """Create processor for sequencing heavy tests on swarming."""
280    return SequenceProc(options.max_heavy_tests) if options.swarming else None
281
282  def _do_execute(self, tests, args, options):
283    jobs = options.j
284
285    print('>>> Running with test processors')
286    loader = LoadProc(tests)
287    results = self._create_result_tracker(options)
288    indicators = self._create_progress_indicators(
289        tests.test_count_estimate, options)
290
291    outproc_factory = None
292    if self.build_config.predictable:
293      outproc_factory = predictable.get_outproc
294    execproc = ExecutionProc(jobs, outproc_factory)
295    sigproc = self._create_signal_proc()
296
297    procs = [
298      loader,
299      NameFilterProc(args) if args else None,
300      StatusFileFilterProc(options.slow_tests, options.pass_fail_tests),
301      VariantProc(self._variants),
302      StatusFileFilterProc(options.slow_tests, options.pass_fail_tests),
303      self._create_predictable_filter(),
304      self._create_shard_proc(options),
305      self._create_seed_proc(options),
306      self._create_sequence_proc(options),
307      sigproc,
308    ] + indicators + [
309      results,
310      self._create_timeout_proc(options),
311      self._create_rerun_proc(options),
312      execproc,
313    ]
314
315    self._prepare_procs(procs)
316
317    loader.load_initial_tests(initial_batch_size=options.j * 2)
318
319    # This starts up worker processes and blocks until all tests are
320    # processed.
321    execproc.run()
322
323    for indicator in indicators:
324      indicator.finished()
325
326    if tests.test_count_estimate:
327      percentage = float(results.total) / tests.test_count_estimate * 100
328    else:
329      percentage = 0
330
331    print (('>>> %d base tests produced %d (%d%s)'
332           ' non-filtered tests') % (
333        tests.test_count_estimate, results.total, percentage, '%'))
334
335    print('>>> %d tests ran' % (results.total - results.remaining))
336
337    exit_code = utils.EXIT_CODE_PASS
338    if results.failed:
339      exit_code = utils.EXIT_CODE_FAILURES
340    if not results.total:
341      exit_code = utils.EXIT_CODE_NO_TESTS
342
343    if options.time:
344      self._print_durations(options)
345
346    # Indicate if a SIGINT or SIGTERM happened.
347    return max(exit_code, sigproc.exit_code)
348
349  def _print_durations(self, options):
350
351    def format_duration(duration_in_seconds):
352      duration = datetime.timedelta(seconds=duration_in_seconds)
353      time = (datetime.datetime.min + duration).time()
354      return time.strftime('%M:%S:') + '%03i' % int(time.microsecond / 1000)
355
356    def _duration_results_text(test):
357      return [
358        'Test: %s' % test['name'],
359        'Flags: %s' % ' '.join(test['flags']),
360        'Command: %s' % test['command'],
361        'Duration: %s' % format_duration(test['duration']),
362      ]
363
364    assert os.path.exists(options.json_test_results)
365    with open(options.json_test_results, "r") as f:
366      output = json.load(f)
367    lines = []
368    for test in output['slowest_tests']:
369      suffix = ''
370      if test.get('marked_slow') is False:
371        suffix = ' *'
372      lines.append(
373          '%s %s%s' % (format_duration(test['duration']),
374                       test['name'], suffix))
375
376    # Slowest tests duration details.
377    lines.extend(['', 'Details:', ''])
378    for test in output['slowest_tests']:
379      lines.extend(_duration_results_text(test))
380    print("\n".join(lines))
381
382  def _create_predictable_filter(self):
383    if not self.build_config.predictable:
384      return None
385    return predictable.PredictableFilterProc()
386
387  def _create_seed_proc(self, options):
388    if options.random_seed_stress_count == 1:
389      return None
390    return SeedProc(options.random_seed_stress_count, options.random_seed,
391                    options.j * 4)
392
393
394if __name__ == '__main__':
395  sys.exit(StandardTestRunner().execute())
396