• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/env python
2#
3# Copyright 2017 the V8 project authors. All rights reserved.
4# Use of this source code is governed by a BSD-style license that can be
5# found in the LICENSE file.
6
7
8import os
9import re
10import sys
11
12# Adds testrunner to the path hence it has to be imported at the beggining.
13import base_runner
14
15from testrunner.local import utils
16from testrunner.local.variants import ALL_VARIANTS
17from testrunner.objects import predictable
18from testrunner.testproc.execution import ExecutionProc
19from testrunner.testproc.filter import StatusFileFilterProc, NameFilterProc
20from testrunner.testproc.loader import LoadProc
21from testrunner.testproc.progress import ResultsTracker, TestsCounter
22from testrunner.testproc.seed import SeedProc
23from testrunner.testproc.variant import VariantProc
24from testrunner.utils import random_utils
25
26
27ARCH_GUESS = utils.DefaultArch()
28
29VARIANTS = ["default"]
30
31MORE_VARIANTS = [
32  "nooptimization",
33  "stress",
34  "stress_background_compile",
35  "stress_incremental_marking",
36]
37
38VARIANT_ALIASES = {
39  # The default for developer workstations.
40  "dev": VARIANTS,
41  # Additional variants, run on all bots.
42  "more": MORE_VARIANTS,
43  # Shortcut for the two above ("more" first - it has the longer running tests).
44  "exhaustive": MORE_VARIANTS + VARIANTS,
45  # Additional variants, run on a subset of bots.
46  "extra": ["future", "no_liftoff", "no_wasm_traps", "trusted"],
47}
48
49GC_STRESS_FLAGS = ["--gc-interval=500", "--stress-compaction",
50                   "--concurrent-recompilation-queue-length=64",
51                   "--concurrent-recompilation-delay=500",
52                   "--concurrent-recompilation"]
53
54RANDOM_GC_STRESS_FLAGS = ["--random-gc-interval=5000",
55                          "--stress-compaction-random"]
56
57
58PREDICTABLE_WRAPPER = os.path.join(
59    base_runner.BASE_DIR, 'tools', 'predictable_wrapper.py')
60
61
62class StandardTestRunner(base_runner.BaseTestRunner):
63    def __init__(self, *args, **kwargs):
64        super(StandardTestRunner, self).__init__(*args, **kwargs)
65
66        self.sancov_dir = None
67        self._variants = None
68
69    def _get_default_suite_names(self):
70      return ['default']
71
72    def _add_parser_options(self, parser):
73      parser.add_option("--novfp3",
74                        help="Indicates that V8 was compiled without VFP3"
75                        " support",
76                        default=False, action="store_true")
77
78      # Variants
79      parser.add_option("--no-variants", "--novariants",
80                        help="Deprecated. "
81                             "Equivalent to passing --variants=default",
82                        default=False, dest="no_variants", action="store_true")
83      parser.add_option("--variants",
84                        help="Comma-separated list of testing variants;"
85                        " default: \"%s\"" % ",".join(VARIANTS))
86      parser.add_option("--exhaustive-variants",
87                        default=False, action="store_true",
88                        help="Deprecated. "
89                             "Equivalent to passing --variants=exhaustive")
90
91      # Filters
92      parser.add_option("--slow-tests", default="dontcare",
93                        help="Regard slow tests (run|skip|dontcare)")
94      parser.add_option("--pass-fail-tests", default="dontcare",
95                        help="Regard pass|fail tests (run|skip|dontcare)")
96      parser.add_option("--quickcheck", default=False, action="store_true",
97                        help=("Quick check mode (skip slow tests)"))
98      parser.add_option("--dont-skip-slow-simulator-tests",
99                        help="Don't skip more slow tests when using a"
100                        " simulator.",
101                        default=False, action="store_true",
102                        dest="dont_skip_simulator_slow_tests")
103
104      # Stress modes
105      parser.add_option("--gc-stress",
106                        help="Switch on GC stress mode",
107                        default=False, action="store_true")
108      parser.add_option("--random-gc-stress",
109                        help="Switch on random GC stress mode",
110                        default=False, action="store_true")
111      parser.add_option("--random-seed-stress-count", default=1, type="int",
112                        dest="random_seed_stress_count",
113                        help="Number of runs with different random seeds. Only "
114                             "with test processors: 0 means infinite "
115                             "generation.")
116
117      # Noop
118      parser.add_option("--cfi-vptr",
119                        help="Run tests with UBSAN cfi_vptr option.",
120                        default=False, action="store_true")
121      parser.add_option("--infra-staging", help="Use new test runner features",
122                        dest='infra_staging', default=None,
123                        action="store_true")
124      parser.add_option("--no-infra-staging",
125                        help="Opt out of new test runner features",
126                        dest='infra_staging', default=None,
127                        action="store_false")
128      parser.add_option("--no-sorting", "--nosorting",
129                        help="Don't sort tests according to duration of last"
130                        " run.",
131                        default=False, dest="no_sorting", action="store_true")
132      parser.add_option("--no-presubmit", "--nopresubmit",
133                        help='Skip presubmit checks (deprecated)',
134                        default=False, dest="no_presubmit", action="store_true")
135
136      # Unimplemented for test processors
137      parser.add_option("--sancov-dir",
138                        help="Directory where to collect coverage data")
139      parser.add_option("--cat", help="Print the source of the tests",
140                        default=False, action="store_true")
141      parser.add_option("--flakiness-results",
142                        help="Path to a file for storing flakiness json.")
143      parser.add_option("--time", help="Print timing information after running",
144                        default=False, action="store_true")
145      parser.add_option("--warn-unused", help="Report unused rules",
146                        default=False, action="store_true")
147      parser.add_option("--report", default=False, action="store_true",
148                        help="Print a summary of the tests to be run")
149
150
151    def _process_options(self, options):
152      if options.sancov_dir:
153        self.sancov_dir = options.sancov_dir
154        if not os.path.exists(self.sancov_dir):
155          print("sancov-dir %s doesn't exist" % self.sancov_dir)
156          raise base_runner.TestRunnerError()
157
158      if options.gc_stress:
159        options.extra_flags += GC_STRESS_FLAGS
160
161      if options.random_gc_stress:
162        options.extra_flags += RANDOM_GC_STRESS_FLAGS
163
164      if self.build_config.asan:
165        options.extra_flags.append("--invoke-weak-callbacks")
166        options.extra_flags.append("--omit-quit")
167
168      if self.build_config.no_snap:
169        # Speed up slow nosnap runs. Allocation verification is covered by
170        # running mksnapshot on other builders.
171        options.extra_flags.append("--no-turbo-verify-allocation")
172
173      if options.novfp3:
174        options.extra_flags.append("--noenable-vfp3")
175
176      if options.no_variants:  # pragma: no cover
177        print ("Option --no-variants is deprecated. "
178               "Pass --variants=default instead.")
179        assert not options.variants
180        options.variants = "default"
181
182      if options.exhaustive_variants:  # pragma: no cover
183        # TODO(machenbach): Switch infra to --variants=exhaustive after M65.
184        print ("Option --exhaustive-variants is deprecated. "
185               "Pass --variants=exhaustive instead.")
186        # This is used on many bots. It includes a larger set of default
187        # variants.
188        # Other options for manipulating variants still apply afterwards.
189        assert not options.variants
190        options.variants = "exhaustive"
191
192      if options.quickcheck:
193        assert not options.variants
194        options.variants = "stress,default"
195        options.slow_tests = "skip"
196        options.pass_fail_tests = "skip"
197
198      if self.build_config.predictable:
199        options.variants = "default"
200        options.extra_flags.append("--predictable")
201        options.extra_flags.append("--verify_predictable")
202        options.extra_flags.append("--no-inline-new")
203        # Add predictable wrapper to command prefix.
204        options.command_prefix = (
205            [sys.executable, PREDICTABLE_WRAPPER] + options.command_prefix)
206
207      # TODO(machenbach): Figure out how to test a bigger subset of variants on
208      # msan.
209      if self.build_config.msan:
210        options.variants = "default"
211
212      if options.variants == "infra_staging":
213        options.variants = "exhaustive"
214
215      self._variants = self._parse_variants(options.variants)
216
217      def CheckTestMode(name, option):  # pragma: no cover
218        if not option in ["run", "skip", "dontcare"]:
219          print "Unknown %s mode %s" % (name, option)
220          raise base_runner.TestRunnerError()
221      CheckTestMode("slow test", options.slow_tests)
222      CheckTestMode("pass|fail test", options.pass_fail_tests)
223      if self.build_config.no_i18n:
224        base_runner.TEST_MAP["bot_default"].remove("intl")
225        base_runner.TEST_MAP["default"].remove("intl")
226        # TODO(machenbach): uncomment after infra side lands.
227        # base_runner.TEST_MAP["d8_default"].remove("intl")
228
229    def _parse_variants(self, aliases_str):
230      # Use developer defaults if no variant was specified.
231      aliases_str = aliases_str or 'dev'
232      aliases = aliases_str.split(',')
233      user_variants = set(reduce(
234          list.__add__, [VARIANT_ALIASES.get(a, [a]) for a in aliases]))
235
236      result = [v for v in ALL_VARIANTS if v in user_variants]
237      if len(result) == len(user_variants):
238        return result
239
240      for v in user_variants:
241        if v not in ALL_VARIANTS:
242          print 'Unknown variant: %s' % v
243          raise base_runner.TestRunnerError()
244      assert False, 'Unreachable'
245
246    def _setup_env(self):
247      super(StandardTestRunner, self)._setup_env()
248
249      symbolizer_option = self._get_external_symbolizer_option()
250
251      if self.sancov_dir:
252        os.environ['ASAN_OPTIONS'] = ":".join([
253          'coverage=1',
254          'coverage_dir=%s' % self.sancov_dir,
255          symbolizer_option,
256          "allow_user_segv_handler=1",
257        ])
258
259    def _get_statusfile_variables(self, options):
260      variables = (
261          super(StandardTestRunner, self)._get_statusfile_variables(options))
262
263      simulator_run = (
264        not options.dont_skip_simulator_slow_tests and
265        self.build_config.arch in [
266          'arm64', 'arm', 'mipsel', 'mips', 'mips64', 'mips64el', 'ppc',
267          'ppc64', 's390', 's390x'] and
268        bool(ARCH_GUESS) and
269        self.build_config.arch != ARCH_GUESS)
270
271      variables.update({
272        'gc_stress': options.gc_stress or options.random_gc_stress,
273        'gc_fuzzer': options.random_gc_stress,
274        'novfp3': options.novfp3,
275        'simulator_run': simulator_run,
276      })
277      return variables
278
279    def _do_execute(self, tests, args, options):
280      jobs = options.j
281
282      print '>>> Running with test processors'
283      loader = LoadProc()
284      tests_counter = TestsCounter()
285      results = ResultsTracker()
286      indicators = self._create_progress_indicators(options)
287
288      outproc_factory = None
289      if self.build_config.predictable:
290        outproc_factory = predictable.get_outproc
291      execproc = ExecutionProc(jobs, outproc_factory)
292      sigproc = self._create_signal_proc()
293
294      procs = [
295        loader,
296        NameFilterProc(args) if args else None,
297        StatusFileFilterProc(options.slow_tests, options.pass_fail_tests),
298        self._create_shard_proc(options),
299        tests_counter,
300        VariantProc(self._variants),
301        StatusFileFilterProc(options.slow_tests, options.pass_fail_tests),
302        self._create_predictable_filter(),
303        self._create_seed_proc(options),
304        sigproc,
305      ] + indicators + [
306        results,
307        self._create_timeout_proc(options),
308        self._create_rerun_proc(options),
309        execproc,
310      ]
311
312      self._prepare_procs(procs)
313      tests.sort(key=lambda t: t.is_slow, reverse=True)
314
315      loader.load_tests(tests)
316
317      print '>>> Running %d base tests' % tests_counter.total
318      tests_counter.remove_from_chain()
319
320      # This starts up worker processes and blocks until all tests are
321      # processed.
322      execproc.run()
323
324      for indicator in indicators:
325        indicator.finished()
326
327      print '>>> %d tests ran' % (results.total - results.remaining)
328
329      exit_code = utils.EXIT_CODE_PASS
330      if results.failed:
331        exit_code = utils.EXIT_CODE_FAILURES
332      if not results.total:
333        exit_code = utils.EXIT_CODE_NO_TESTS
334
335      # Indicate if a SIGINT or SIGTERM happened.
336      exit_code = max(exit_code, sigproc.exit_code)
337
338      if exit_code == utils.EXIT_CODE_FAILURES and options.json_test_results:
339        print("Force exit code 0 after failures. Json test results file "
340              "generated with failure information.")
341        exit_code = utils.EXIT_CODE_PASS
342      return exit_code
343
344    def _create_predictable_filter(self):
345      if not self.build_config.predictable:
346        return None
347      return predictable.PredictableFilterProc()
348
349    def _create_seed_proc(self, options):
350      if options.random_seed_stress_count == 1:
351        return None
352      return SeedProc(options.random_seed_stress_count, options.random_seed,
353                      options.j * 4)
354
355
356if __name__ == '__main__':
357  sys.exit(StandardTestRunner().execute())
358