• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/env python3
2#
3# Copyright 2017 the V8 project authors. All rights reserved.
4# Use of this source code is governed by a BSD-style license that can be
5# found in the LICENSE file.
6
7import random
8import sys
9
10# Adds testrunner to the path hence it has to be imported at the beggining.
11from . import base_runner
12
13from testrunner.local import utils
14
15from testrunner.testproc import fuzzer
16from testrunner.testproc.base import TestProcProducer
17from testrunner.testproc.combiner import CombinerProc
18from testrunner.testproc.execution import ExecutionProc
19from testrunner.testproc.expectation import ExpectationProc
20from testrunner.testproc.filter import StatusFileFilterProc, NameFilterProc
21from testrunner.testproc.loader import LoadProc
22from testrunner.testproc.progress import ResultsTracker
23from testrunner.utils import random_utils
24
25
26DEFAULT_SUITES = ["mjsunit", "webkit", "benchmarks"]
27
28
29class NumFuzzer(base_runner.BaseTestRunner):
30  def __init__(self, *args, **kwargs):
31    super(NumFuzzer, self).__init__(*args, **kwargs)
32
33  @property
34  def framework_name(self):
35    return 'num_fuzzer'
36
37  def _add_parser_options(self, parser):
38    parser.add_option("--fuzzer-random-seed", default=0,
39                      help="Default seed for initializing fuzzer random "
40                      "generator")
41    parser.add_option("--tests-count", default=5, type="int",
42                      help="Number of tests to generate from each base test. "
43                           "Can be combined with --total-timeout-sec with "
44                           "value 0 to provide infinite number of subtests. "
45                           "When --combine-tests is set it indicates how many "
46                           "tests to create in total")
47
48    # Stress gc
49    parser.add_option("--stress-marking", default=0, type="int",
50                      help="probability [0-10] of adding --stress-marking "
51                           "flag to the test")
52    parser.add_option("--stress-scavenge", default=0, type="int",
53                      help="probability [0-10] of adding --stress-scavenge "
54                           "flag to the test")
55    parser.add_option("--stress-compaction", default=0, type="int",
56                      help="probability [0-10] of adding --stress-compaction "
57                           "flag to the test")
58    parser.add_option("--stress-gc", default=0, type="int",
59                      help="probability [0-10] of adding --random-gc-interval "
60                           "flag to the test")
61
62    # Stress stack size
63    parser.add_option("--stress-stack-size", default=0, type="int",
64                      help="probability [0-10] of adding --stack-size "
65                           "flag to the test")
66
67    # Stress tasks
68    parser.add_option("--stress-delay-tasks", default=0, type="int",
69                      help="probability [0-10] of adding --stress-delay-tasks "
70                           "flag to the test")
71    parser.add_option("--stress-thread-pool-size", default=0, type="int",
72                      help="probability [0-10] of adding --thread-pool-size "
73                           "flag to the test")
74
75    # Stress compiler
76    parser.add_option("--stress-deopt", default=0, type="int",
77                      help="probability [0-10] of adding --deopt-every-n-times "
78                           "flag to the test")
79    parser.add_option("--stress-deopt-min", default=1, type="int",
80                      help="extends --stress-deopt to have minimum interval "
81                           "between deopt points")
82    parser.add_option("--stress-interrupt-budget", default=0, type="int",
83                      help="probability [0-10] of adding the --interrupt-budget "
84                           "flag to the test")
85
86    # Combine multiple tests
87    parser.add_option("--combine-tests", default=False, action="store_true",
88                      help="Combine multiple tests as one and run with "
89                           "try-catch wrapper")
90    parser.add_option("--combine-max", default=100, type="int",
91                      help="Maximum number of tests to combine")
92    parser.add_option("--combine-min", default=2, type="int",
93                      help="Minimum number of tests to combine")
94
95    # Miscellaneous
96    parser.add_option("--variants", default='default',
97                      help="Comma-separated list of testing variants")
98
99    return parser
100
101
102  def _process_options(self, options):
103    if not options.fuzzer_random_seed:
104      options.fuzzer_random_seed = random_utils.random_seed()
105
106    if options.total_timeout_sec:
107      options.tests_count = 0
108
109    if options.combine_tests:
110      if options.combine_min > options.combine_max:
111        print(('min_group_size (%d) cannot be larger than max_group_size (%d)' %
112               options.min_group_size, options.max_group_size))
113        raise base_runner.TestRunnerError()
114
115    if options.variants != 'default':
116      print ('Only default testing variant is supported with numfuzz')
117      raise base_runner.TestRunnerError()
118
119    return True
120
121  def _get_default_suite_names(self):
122    return DEFAULT_SUITES
123
124  def _runner_flags(self):
125    """Extra default flags specific to the test runner implementation."""
126    return [
127      '--no-abort-on-contradictory-flags',
128      '--testing-d8-test-runner',
129      '--no-fail'
130    ]
131
132  def _get_statusfile_variables(self, options):
133    variables = (
134        super(NumFuzzer, self)._get_statusfile_variables(options))
135    variables.update({
136      'deopt_fuzzer': bool(options.stress_deopt),
137      'endurance_fuzzer': bool(options.combine_tests),
138      'gc_stress': bool(options.stress_gc),
139      'gc_fuzzer': bool(max([options.stress_marking,
140                             options.stress_scavenge,
141                             options.stress_compaction,
142                             options.stress_gc,
143                             options.stress_delay_tasks,
144                             options.stress_stack_size,
145                             options.stress_thread_pool_size])),
146    })
147    return variables
148
149  def _do_execute(self, tests, args, options):
150    loader = LoadProc(tests)
151    fuzzer_rng = random.Random(options.fuzzer_random_seed)
152
153    combiner = self._create_combiner(fuzzer_rng, options)
154    results = self._create_result_tracker(options)
155    execproc = ExecutionProc(options.j)
156    sigproc = self._create_signal_proc()
157    indicators = self._create_progress_indicators(
158      tests.test_count_estimate, options)
159    procs = [
160      loader,
161      NameFilterProc(args) if args else None,
162      StatusFileFilterProc(None, None),
163      # TODO(majeski): Improve sharding when combiner is present. Maybe select
164      # different random seeds for shards instead of splitting tests.
165      self._create_shard_proc(options),
166      ExpectationProc(),
167      combiner,
168      self._create_fuzzer(fuzzer_rng, options),
169      sigproc,
170    ] + indicators + [
171      results,
172      self._create_timeout_proc(options),
173      self._create_rerun_proc(options),
174      execproc,
175    ]
176    self._prepare_procs(procs)
177    loader.load_initial_tests(initial_batch_size=float('inf'))
178
179    # TODO(majeski): maybe some notification from loader would be better?
180    if combiner:
181      combiner.generate_initial_tests(options.j * 4)
182
183    # This starts up worker processes and blocks until all tests are
184    # processed.
185    execproc.run()
186
187    for indicator in indicators:
188      indicator.finished()
189
190    print('>>> %d tests ran' % results.total)
191    if results.failed:
192      return utils.EXIT_CODE_FAILURES
193
194    # Indicate if a SIGINT or SIGTERM happened.
195    return sigproc.exit_code
196
197  def _is_testsuite_supported(self, suite, options):
198    return not options.combine_tests or suite.test_combiner_available()
199
200  def _create_combiner(self, rng, options):
201    if not options.combine_tests:
202      return None
203    return CombinerProc(rng, options.combine_min, options.combine_max,
204                        options.tests_count)
205
206  def _create_fuzzer(self, rng, options):
207    return fuzzer.FuzzerProc(
208        rng,
209        self._tests_count(options),
210        self._create_fuzzer_configs(options),
211        self._disable_analysis(options),
212    )
213
214  def _tests_count(self, options):
215    if options.combine_tests:
216      return 1
217    return options.tests_count
218
219  def _disable_analysis(self, options):
220    """Disable analysis phase when options are used that don't support it."""
221    return options.combine_tests
222
223  def _create_fuzzer_configs(self, options):
224    fuzzers = []
225    def add(name, prob, *args):
226      if prob:
227        fuzzers.append(fuzzer.create_fuzzer_config(name, prob, *args))
228
229    add('compaction', options.stress_compaction)
230    add('marking', options.stress_marking)
231    add('scavenge', options.stress_scavenge)
232    add('gc_interval', options.stress_gc)
233    add('stack', options.stress_stack_size)
234    add('threads', options.stress_thread_pool_size)
235    add('delay', options.stress_delay_tasks)
236    add('deopt', options.stress_deopt, options.stress_deopt_min)
237    return fuzzers
238
239
240if __name__ == '__main__':
241  sys.exit(NumFuzzer().execute())
242