• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/env python
2# Copyright 2016 the V8 project authors. All rights reserved.
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5
6"""
7V8 correctness fuzzer launcher script.
8"""
9
10import argparse
11import hashlib
12import itertools
13import json
14import os
15import re
16import sys
17import traceback
18
19import v8_commands
20import v8_suppressions
21
22CONFIGS = dict(
23  default=['--validate-asm'],
24  fullcode=['--nocrankshaft', '--turbo-filter=~', '--validate-asm'],
25  ignition=['--ignition', '--turbo-filter=~', '--hydrogen-filter=~',
26            '--validate-asm', '--nocrankshaft'],
27  ignition_eager=['--ignition', '--turbo-filter=~', '--hydrogen-filter=~',
28                  '--validate-asm', '--nocrankshaft', '--no-lazy',
29                  '--no-lazy-inner-functions'],
30  ignition_staging=['--ignition-staging', '--validate-asm'],
31  ignition_turbo=['--ignition-staging', '--turbo', '--validate-asm'],
32  ignition_turbo_opt=['--ignition-staging', '--turbo', '--always-opt',
33                      '--validate-asm'],
34)
35
36# Timeout in seconds for one d8 run.
37TIMEOUT = 3
38
39# Return codes.
40RETURN_PASS = 0
41RETURN_FAIL = 2
42
43BASE_PATH = os.path.dirname(os.path.abspath(__file__))
44PREAMBLE = [
45  os.path.join(BASE_PATH, 'v8_mock.js'),
46  os.path.join(BASE_PATH, 'v8_suppressions.js'),
47]
48ARCH_MOCKS = os.path.join(BASE_PATH, 'v8_mock_archs.js')
49
50FLAGS = ['--abort_on_stack_overflow', '--expose-gc', '--allow-natives-syntax',
51         '--invoke-weak-callbacks', '--omit-quit', '--es-staging']
52
53SUPPORTED_ARCHS = ['ia32', 'x64', 'arm', 'arm64']
54
55# Output for suppressed failure case.
56FAILURE_HEADER_TEMPLATE = """#
57# V8 correctness failure
58# V8 correctness configs: %(configs)s
59# V8 correctness sources: %(source_key)s
60# V8 correctness suppression: %(suppression)s
61"""
62
63# Extended output for failure case. The 'CHECK' is for the minimizer.
64FAILURE_TEMPLATE = FAILURE_HEADER_TEMPLATE + """#
65# CHECK
66#
67# Compared %(first_config_label)s with %(second_config_label)s
68#
69# Flags of %(first_config_label)s:
70%(first_config_flags)s
71# Flags of %(second_config_label)s:
72%(second_config_flags)s
73#
74# Difference:
75%(difference)s
76#
77# Source file:
78%(source)s
79#
80### Start of configuration %(first_config_label)s:
81%(first_config_output)s
82### End of configuration %(first_config_label)s
83#
84### Start of configuration %(second_config_label)s:
85%(second_config_output)s
86### End of configuration %(second_config_label)s
87"""
88
89FUZZ_TEST_RE = re.compile(r'.*fuzz(-\d+\.js)')
90SOURCE_RE = re.compile(r'print\("v8-foozzie source: (.*)"\);')
91
92# The number of hex digits used from the hash of the original source file path.
93# Keep the number small to avoid duplicate explosion.
94ORIGINAL_SOURCE_HASH_LENGTH = 3
95
96# Placeholder string if no original source file could be determined.
97ORIGINAL_SOURCE_DEFAULT = 'none'
98
99
100def infer_arch(d8):
101  """Infer the V8 architecture from the build configuration next to the
102  executable.
103  """
104  with open(os.path.join(os.path.dirname(d8), 'v8_build_config.json')) as f:
105    arch = json.load(f)['v8_current_cpu']
106  return 'ia32' if arch == 'x86' else arch
107
108
109def parse_args():
110  parser = argparse.ArgumentParser()
111  parser.add_argument(
112    '--random-seed', type=int, required=True,
113    help='random seed passed to both runs')
114  parser.add_argument(
115      '--first-config', help='first configuration', default='ignition')
116  parser.add_argument(
117      '--second-config', help='second configuration', default='ignition_turbo')
118  parser.add_argument(
119      '--first-d8', default='d8',
120      help='optional path to first d8 executable, '
121           'default: bundled in the same directory as this script')
122  parser.add_argument(
123      '--second-d8',
124      help='optional path to second d8 executable, default: same as first')
125  parser.add_argument('testcase', help='path to test case')
126  options = parser.parse_args()
127
128  # Ensure we have a test case.
129  assert (os.path.exists(options.testcase) and
130          os.path.isfile(options.testcase)), (
131      'Test case %s doesn\'t exist' % options.testcase)
132
133  # Use first d8 as default for second d8.
134  options.second_d8 = options.second_d8 or options.first_d8
135
136  # Ensure absolute paths.
137  if not os.path.isabs(options.first_d8):
138    options.first_d8 = os.path.join(BASE_PATH, options.first_d8)
139  if not os.path.isabs(options.second_d8):
140    options.second_d8 = os.path.join(BASE_PATH, options.second_d8)
141
142  # Ensure executables exist.
143  assert os.path.exists(options.first_d8)
144  assert os.path.exists(options.second_d8)
145
146  # Infer architecture from build artifacts.
147  options.first_arch = infer_arch(options.first_d8)
148  options.second_arch = infer_arch(options.second_d8)
149
150  # Ensure we make a sane comparison.
151  assert (options.first_arch != options.second_arch or
152          options.first_config != options.second_config), (
153      'Need either arch or config difference.')
154  assert options.first_arch in SUPPORTED_ARCHS
155  assert options.second_arch in SUPPORTED_ARCHS
156  assert options.first_config in CONFIGS
157  assert options.second_config in CONFIGS
158
159  return options
160
161
162def get_meta_data(content):
163  """Extracts original-source-file paths from test case content."""
164  sources = []
165  for line in content.splitlines():
166    match = SOURCE_RE.match(line)
167    if match:
168      sources.append(match.group(1))
169  return {'sources': sources}
170
171
172def content_bailout(content, ignore_fun):
173  """Print failure state and return if ignore_fun matches content."""
174  bug = (ignore_fun(content) or '').strip()
175  if bug:
176    print FAILURE_HEADER_TEMPLATE % dict(
177        configs='', source_key='', suppression=bug)
178    return True
179  return False
180
181
182def pass_bailout(output, step_number):
183  """Print info and return if in timeout or crash pass states."""
184  if output.HasTimedOut():
185    # Dashed output, so that no other clusterfuzz tools can match the
186    # words timeout or crash.
187    print '# V8 correctness - T-I-M-E-O-U-T %d' % step_number
188    return True
189  if output.HasCrashed():
190    print '# V8 correctness - C-R-A-S-H %d' % step_number
191    return True
192  return False
193
194
195def fail_bailout(output, ignore_by_output_fun):
196  """Print failure state and return if ignore_by_output_fun matches output."""
197  bug = (ignore_by_output_fun(output.stdout) or '').strip()
198  if bug:
199    print FAILURE_HEADER_TEMPLATE % dict(
200        configs='', source_key='', suppression=bug)
201    return True
202  return False
203
204
205def main():
206  options = parse_args()
207
208  # Suppressions are architecture and configuration specific.
209  suppress = v8_suppressions.get_suppression(
210      options.first_arch, options.first_config,
211      options.second_arch, options.second_config,
212  )
213
214  # Static bailout based on test case content or metadata.
215  with open(options.testcase) as f:
216    content = f.read()
217  if content_bailout(get_meta_data(content), suppress.ignore_by_metadata):
218    return RETURN_FAIL
219  if content_bailout(content, suppress.ignore_by_content):
220    return RETURN_FAIL
221
222  # Set up runtime arguments.
223  common_flags = FLAGS + ['--random-seed', str(options.random_seed)]
224  first_config_flags = common_flags + CONFIGS[options.first_config]
225  second_config_flags = common_flags + CONFIGS[options.second_config]
226
227  def run_d8(d8, config_flags):
228    preamble = PREAMBLE[:]
229    if options.first_arch != options.second_arch:
230      preamble.append(ARCH_MOCKS)
231    args = [d8] + config_flags + preamble + [options.testcase]
232    print " ".join(args)
233    if d8.endswith('.py'):
234      # Wrap with python in tests.
235      args = [sys.executable] + args
236    return v8_commands.Execute(
237        args,
238        cwd=os.path.dirname(options.testcase),
239        timeout=TIMEOUT,
240    )
241
242  first_config_output = run_d8(options.first_d8, first_config_flags)
243
244  # Early bailout based on first run's output.
245  if pass_bailout(first_config_output, 1):
246    return RETURN_PASS
247
248  second_config_output = run_d8(options.second_d8, second_config_flags)
249
250  # Bailout based on second run's output.
251  if pass_bailout(second_config_output, 2):
252    return RETURN_PASS
253
254  difference, source = suppress.diff(
255      first_config_output.stdout, second_config_output.stdout)
256
257  if source:
258    source_key = hashlib.sha1(source).hexdigest()[:ORIGINAL_SOURCE_HASH_LENGTH]
259  else:
260    source = ORIGINAL_SOURCE_DEFAULT
261    source_key = ORIGINAL_SOURCE_DEFAULT
262
263  if difference:
264    # Only bail out due to suppressed output if there was a difference. If a
265    # suppression doesn't show up anymore in the statistics, we might want to
266    # remove it.
267    if fail_bailout(first_config_output, suppress.ignore_by_output1):
268      return RETURN_FAIL
269    if fail_bailout(second_config_output, suppress.ignore_by_output2):
270      return RETURN_FAIL
271
272    # The first three entries will be parsed by clusterfuzz. Format changes
273    # will require changes on the clusterfuzz side.
274    first_config_label = '%s,%s' % (options.first_arch, options.first_config)
275    second_config_label = '%s,%s' % (options.second_arch, options.second_config)
276    print (FAILURE_TEMPLATE % dict(
277        configs='%s:%s' % (first_config_label, second_config_label),
278        source_key=source_key,
279        suppression='', # We can't tie bugs to differences.
280        first_config_label=first_config_label,
281        second_config_label=second_config_label,
282        first_config_flags=' '.join(first_config_flags),
283        second_config_flags=' '.join(second_config_flags),
284        first_config_output=
285            first_config_output.stdout.decode('utf-8', 'replace'),
286        second_config_output=
287            second_config_output.stdout.decode('utf-8', 'replace'),
288        source=source,
289        difference=difference.decode('utf-8', 'replace'),
290    )).encode('utf-8', 'replace')
291    return RETURN_FAIL
292
293  # TODO(machenbach): Figure out if we could also return a bug in case there's
294  # no difference, but one of the line suppressions has matched - and without
295  # the match there would be a difference.
296
297  print '# V8 correctness - pass'
298  return RETURN_PASS
299
300
301if __name__ == "__main__":
302  try:
303    result = main()
304  except SystemExit:
305    # Make sure clusterfuzz reports internal errors and wrong usage.
306    # Use one label for all internal and usage errors.
307    print FAILURE_HEADER_TEMPLATE % dict(
308        configs='', source_key='', suppression='wrong_usage')
309    result = RETURN_FAIL
310  except MemoryError:
311    # Running out of memory happens occasionally but is not actionable.
312    print '# V8 correctness - pass'
313    result = RETURN_PASS
314  except Exception as e:
315    print FAILURE_HEADER_TEMPLATE % dict(
316        configs='', source_key='', suppression='internal_error')
317    print '# Internal error: %s' % e
318    traceback.print_exc(file=sys.stdout)
319    result = RETURN_FAIL
320
321  sys.exit(result)
322