• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/env python
2#
3# Copyright 2012 the V8 project authors. All rights reserved.
4# Redistribution and use in source and binary forms, with or without
5# modification, are permitted provided that the following conditions are
6# met:
7#
8#     * Redistributions of source code must retain the above copyright
9#       notice, this list of conditions and the following disclaimer.
10#     * Redistributions in binary form must reproduce the above
11#       copyright notice, this list of conditions and the following
12#       disclaimer in the documentation and/or other materials provided
13#       with the distribution.
14#     * Neither the name of Google Inc. nor the names of its
15#       contributors may be used to endorse or promote products derived
16#       from this software without specific prior written permission.
17#
18# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29
30
31import json
32import math
33import multiprocessing
34import optparse
35import os
36from os.path import join
37import random
38import shlex
39import subprocess
40import sys
41import time
42
43from testrunner.local import execution
44from testrunner.local import progress
45from testrunner.local import testsuite
46from testrunner.local import utils
47from testrunner.local import verbose
48from testrunner.objects import context
49
50
51# Base dir of the v8 checkout to be used as cwd.
52BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
53
54ARCH_GUESS = utils.DefaultArch()
55DEFAULT_TESTS = ["mjsunit", "webkit"]
56TIMEOUT_DEFAULT = 60
57TIMEOUT_SCALEFACTOR = {"debug"   : 4,
58                       "release" : 1 }
59
60MODE_FLAGS = {
61    "debug"   : ["--nohard-abort", "--nodead-code-elimination",
62                 "--nofold-constants", "--enable-slow-asserts",
63                 "--debug-code", "--verify-heap",
64                 "--noconcurrent-recompilation"],
65    "release" : ["--nohard-abort", "--nodead-code-elimination",
66                 "--nofold-constants", "--noconcurrent-recompilation"]}
67
68SUPPORTED_ARCHS = ["android_arm",
69                   "android_ia32",
70                   "arm",
71                   "ia32",
72                   "ppc",
73                   "ppc64",
74                   "s390",
75                   "s390x",
76                   "mipsel",
77                   "nacl_ia32",
78                   "nacl_x64",
79                   "x64"]
80# Double the timeout for these:
81SLOW_ARCHS = ["android_arm",
82              "android_ia32",
83              "arm",
84              "mipsel",
85              "nacl_ia32",
86              "nacl_x64"]
87MAX_DEOPT = 1000000000
88DISTRIBUTION_MODES = ["smooth", "random"]
89
90
91class RandomDistribution:
92  def __init__(self, seed=None):
93    seed = seed or random.randint(1, sys.maxint)
94    print "Using random distribution with seed %d" % seed
95    self._random = random.Random(seed)
96
97  def Distribute(self, n, m):
98    if n > m:
99      n = m
100    return self._random.sample(xrange(1, m + 1), n)
101
102
103class SmoothDistribution:
104  """Distribute n numbers into the interval [1:m].
105  F1: Factor of the first derivation of the distribution function.
106  F2: Factor of the second derivation of the distribution function.
107  With F1 and F2 set to 0, the distribution will be equal.
108  """
109  def __init__(self, factor1=2.0, factor2=0.2):
110    self._factor1 = factor1
111    self._factor2 = factor2
112
113  def Distribute(self, n, m):
114    if n > m:
115      n = m
116    if n <= 1:
117      return [ 1 ]
118
119    result = []
120    x = 0.0
121    dx = 1.0
122    ddx = self._factor1
123    dddx = self._factor2
124    for i in range(0, n):
125      result += [ x ]
126      x += dx
127      dx += ddx
128      ddx += dddx
129
130    # Project the distribution into the interval [0:M].
131    result = [ x * m / result[-1] for x in result ]
132
133    # Equalize by n. The closer n is to m, the more equal will be the
134    # distribution.
135    for (i, x) in enumerate(result):
136      # The value of x if it was equally distributed.
137      equal_x = i / float(n - 1) * float(m - 1) + 1
138
139      # Difference factor between actual and equal distribution.
140      diff = 1 - (x / equal_x)
141
142      # Equalize x dependent on the number of values to distribute.
143      result[i] = int(x + (i + 1) * diff)
144    return result
145
146
147def Distribution(options):
148  if options.distribution_mode == "random":
149    return RandomDistribution(options.seed)
150  if options.distribution_mode == "smooth":
151    return SmoothDistribution(options.distribution_factor1,
152                              options.distribution_factor2)
153
154
155def BuildOptions():
156  result = optparse.OptionParser()
157  result.add_option("--arch",
158                    help=("The architecture to run tests for, "
159                          "'auto' or 'native' for auto-detect"),
160                    default="ia32,x64,arm")
161  result.add_option("--arch-and-mode",
162                    help="Architecture and mode in the format 'arch.mode'",
163                    default=None)
164  result.add_option("--asan",
165                    help="Regard test expectations for ASAN",
166                    default=False, action="store_true")
167  result.add_option("--buildbot",
168                    help="Adapt to path structure used on buildbots",
169                    default=False, action="store_true")
170  result.add_option("--dcheck-always-on",
171                    help="Indicates that V8 was compiled with DCHECKs enabled",
172                    default=False, action="store_true")
173  result.add_option("--command-prefix",
174                    help="Prepended to each shell command used to run a test",
175                    default="")
176  result.add_option("--coverage", help=("Exponential test coverage "
177                    "(range 0.0, 1.0) -- 0.0: one test, 1.0 all tests (slow)"),
178                    default=0.4, type="float")
179  result.add_option("--coverage-lift", help=("Lifts test coverage for tests "
180                    "with a small number of deopt points (range 0, inf)"),
181                    default=20, type="int")
182  result.add_option("--download-data", help="Download missing test suite data",
183                    default=False, action="store_true")
184  result.add_option("--distribution-factor1", help=("Factor of the first "
185                    "derivation of the distribution function"), default=2.0,
186                    type="float")
187  result.add_option("--distribution-factor2", help=("Factor of the second "
188                    "derivation of the distribution function"), default=0.7,
189                    type="float")
190  result.add_option("--distribution-mode", help=("How to select deopt points "
191                    "for a given test (smooth|random)"),
192                    default="smooth")
193  result.add_option("--dump-results-file", help=("Dump maximum number of "
194                    "deopt points per test to a file"))
195  result.add_option("--extra-flags",
196                    help="Additional flags to pass to each test command",
197                    default="")
198  result.add_option("--isolates", help="Whether to test isolates",
199                    default=False, action="store_true")
200  result.add_option("-j", help="The number of parallel tasks to run",
201                    default=0, type="int")
202  result.add_option("-m", "--mode",
203                    help="The test modes in which to run (comma-separated)",
204                    default="release,debug")
205  result.add_option("--outdir", help="Base directory with compile output",
206                    default="out")
207  result.add_option("-p", "--progress",
208                    help=("The style of progress indicator"
209                          " (verbose, dots, color, mono)"),
210                    choices=progress.PROGRESS_INDICATORS.keys(),
211                    default="mono")
212  result.add_option("--shard-count",
213                    help="Split testsuites into this number of shards",
214                    default=1, type="int")
215  result.add_option("--shard-run",
216                    help="Run this shard from the split up tests.",
217                    default=1, type="int")
218  result.add_option("--shell-dir", help="Directory containing executables",
219                    default="")
220  result.add_option("--seed", help="The seed for the random distribution",
221                    type="int")
222  result.add_option("-t", "--timeout", help="Timeout in seconds",
223                    default= -1, type="int")
224  result.add_option("-v", "--verbose", help="Verbose output",
225                    default=False, action="store_true")
226  result.add_option("--random-seed", default=0, dest="random_seed",
227                    help="Default seed for initializing random generator")
228  return result
229
230
231def ProcessOptions(options):
232  global VARIANT_FLAGS
233
234  # Architecture and mode related stuff.
235  if options.arch_and_mode:
236    tokens = options.arch_and_mode.split(".")
237    options.arch = tokens[0]
238    options.mode = tokens[1]
239  options.mode = options.mode.split(",")
240  for mode in options.mode:
241    if not mode.lower() in ["debug", "release"]:
242      print "Unknown mode %s" % mode
243      return False
244  if options.arch in ["auto", "native"]:
245    options.arch = ARCH_GUESS
246  options.arch = options.arch.split(",")
247  for arch in options.arch:
248    if not arch in SUPPORTED_ARCHS:
249      print "Unknown architecture %s" % arch
250      return False
251
252  # Special processing of other options, sorted alphabetically.
253  options.command_prefix = shlex.split(options.command_prefix)
254  options.extra_flags = shlex.split(options.extra_flags)
255  if options.j == 0:
256    options.j = multiprocessing.cpu_count()
257  while options.random_seed == 0:
258    options.random_seed = random.SystemRandom().randint(-2147483648, 2147483647)
259  if not options.distribution_mode in DISTRIBUTION_MODES:
260    print "Unknown distribution mode %s" % options.distribution_mode
261    return False
262  if options.distribution_factor1 < 0.0:
263    print ("Distribution factor1 %s is out of range. Defaulting to 0.0"
264        % options.distribution_factor1)
265    options.distribution_factor1 = 0.0
266  if options.distribution_factor2 < 0.0:
267    print ("Distribution factor2 %s is out of range. Defaulting to 0.0"
268        % options.distribution_factor2)
269    options.distribution_factor2 = 0.0
270  if options.coverage < 0.0 or options.coverage > 1.0:
271    print ("Coverage %s is out of range. Defaulting to 0.4"
272        % options.coverage)
273    options.coverage = 0.4
274  if options.coverage_lift < 0:
275    print ("Coverage lift %s is out of range. Defaulting to 0"
276        % options.coverage_lift)
277    options.coverage_lift = 0
278  return True
279
280
281def ShardTests(tests, shard_count, shard_run):
282  if shard_count < 2:
283    return tests
284  if shard_run < 1 or shard_run > shard_count:
285    print "shard-run not a valid number, should be in [1:shard-count]"
286    print "defaulting back to running all tests"
287    return tests
288  count = 0
289  shard = []
290  for test in tests:
291    if count % shard_count == shard_run - 1:
292      shard.append(test)
293    count += 1
294  return shard
295
296
297def Main():
298  # Use the v8 root as cwd as some test cases use "load" with relative paths.
299  os.chdir(BASE_DIR)
300
301  parser = BuildOptions()
302  (options, args) = parser.parse_args()
303  if not ProcessOptions(options):
304    parser.print_help()
305    return 1
306
307  exit_code = 0
308
309  suite_paths = utils.GetSuitePaths(join(BASE_DIR, "test"))
310
311  if len(args) == 0:
312    suite_paths = [ s for s in suite_paths if s in DEFAULT_TESTS ]
313  else:
314    args_suites = set()
315    for arg in args:
316      suite = arg.split(os.path.sep)[0]
317      if not suite in args_suites:
318        args_suites.add(suite)
319    suite_paths = [ s for s in suite_paths if s in args_suites ]
320
321  suites = []
322  for root in suite_paths:
323    suite = testsuite.TestSuite.LoadTestSuite(
324        os.path.join(BASE_DIR, "test", root))
325    if suite:
326      suites.append(suite)
327
328  if options.download_data:
329    for s in suites:
330      s.DownloadData()
331
332  for mode in options.mode:
333    for arch in options.arch:
334      try:
335        code = Execute(arch, mode, args, options, suites, BASE_DIR)
336        exit_code = exit_code or code
337      except KeyboardInterrupt:
338        return 2
339  return exit_code
340
341
342def CalculateNTests(m, options):
343  """Calculates the number of tests from m deopt points with exponential
344  coverage.
345  The coverage is expected to be between 0.0 and 1.0.
346  The 'coverage lift' lifts the coverage for tests with smaller m values.
347  """
348  c = float(options.coverage)
349  l = float(options.coverage_lift)
350  return int(math.pow(m, (m * c + l) / (m + l)))
351
352
353def Execute(arch, mode, args, options, suites, workspace):
354  print(">>> Running tests for %s.%s" % (arch, mode))
355
356  dist = Distribution(options)
357
358  shell_dir = options.shell_dir
359  if not shell_dir:
360    if options.buildbot:
361      shell_dir = os.path.join(workspace, options.outdir, mode)
362      mode = mode.lower()
363    else:
364      shell_dir = os.path.join(workspace, options.outdir,
365                               "%s.%s" % (arch, mode))
366  shell_dir = os.path.relpath(shell_dir)
367
368  # Populate context object.
369  mode_flags = MODE_FLAGS[mode]
370  timeout = options.timeout
371  if timeout == -1:
372    # Simulators are slow, therefore allow a longer default timeout.
373    if arch in SLOW_ARCHS:
374      timeout = 2 * TIMEOUT_DEFAULT;
375    else:
376      timeout = TIMEOUT_DEFAULT;
377
378  timeout *= TIMEOUT_SCALEFACTOR[mode]
379  ctx = context.Context(arch, mode, shell_dir,
380                        mode_flags, options.verbose,
381                        timeout, options.isolates,
382                        options.command_prefix,
383                        options.extra_flags,
384                        False,  # Keep i18n on by default.
385                        options.random_seed,
386                        True,  # No sorting of test cases.
387                        0,  # Don't rerun failing tests.
388                        0,  # No use of a rerun-failing-tests maximum.
389                        False,  # No predictable mode.
390                        False,  # No no_harness mode.
391                        False,  # Don't use perf data.
392                        False)  # Coverage not supported.
393
394  # Find available test suites and read test cases from them.
395  variables = {
396    "arch": arch,
397    "asan": options.asan,
398    "deopt_fuzzer": True,
399    "gc_stress": False,
400    "gcov_coverage": False,
401    "ignition": False,
402    "ignition_turbofan": False,
403    "isolates": options.isolates,
404    "mode": mode,
405    "no_i18n": False,
406    "no_snap": False,
407    "simulator": utils.UseSimulator(arch),
408    "system": utils.GuessOS(),
409    "tsan": False,
410    "msan": False,
411    "dcheck_always_on": options.dcheck_always_on,
412    "novfp3": False,
413    "predictable": False,
414    "byteorder": sys.byteorder,
415  }
416  all_tests = []
417  num_tests = 0
418  test_id = 0
419
420  # Remember test case prototypes for the fuzzing phase.
421  test_backup = dict((s, []) for s in suites)
422
423  for s in suites:
424    s.ReadStatusFile(variables)
425    s.ReadTestCases(ctx)
426    if len(args) > 0:
427      s.FilterTestCasesByArgs(args)
428    all_tests += s.tests
429    s.FilterTestCasesByStatus(False)
430    test_backup[s] = s.tests
431    analysis_flags = ["--deopt-every-n-times", "%d" % MAX_DEOPT,
432                      "--print-deopt-stress"]
433    s.tests = [ t.CopyAddingFlags(t.variant, analysis_flags) for t in s.tests ]
434    num_tests += len(s.tests)
435    for t in s.tests:
436      t.id = test_id
437      test_id += 1
438
439  if num_tests == 0:
440    print "No tests to run."
441    return 0
442
443  print(">>> Collection phase")
444  progress_indicator = progress.PROGRESS_INDICATORS[options.progress]()
445  runner = execution.Runner(suites, progress_indicator, ctx)
446
447  exit_code = runner.Run(options.j)
448
449  print(">>> Analysis phase")
450  num_tests = 0
451  test_id = 0
452  for s in suites:
453    test_results = {}
454    for t in s.tests:
455      for line in t.output.stdout.splitlines():
456        if line.startswith("=== Stress deopt counter: "):
457          test_results[t.path] = MAX_DEOPT - int(line.split(" ")[-1])
458    for t in s.tests:
459      if t.path not in test_results:
460        print "Missing results for %s" % t.path
461    if options.dump_results_file:
462      results_dict = dict((t.path, n) for (t, n) in test_results.iteritems())
463      with file("%s.%d.txt" % (dump_results_file, time.time()), "w") as f:
464        f.write(json.dumps(results_dict))
465
466    # Reset tests and redistribute the prototypes from the collection phase.
467    s.tests = []
468    if options.verbose:
469      print "Test distributions:"
470    for t in test_backup[s]:
471      max_deopt = test_results.get(t.path, 0)
472      if max_deopt == 0:
473        continue
474      n_deopt = CalculateNTests(max_deopt, options)
475      distribution = dist.Distribute(n_deopt, max_deopt)
476      if options.verbose:
477        print "%s %s" % (t.path, distribution)
478      for i in distribution:
479        fuzzing_flags = ["--deopt-every-n-times", "%d" % i]
480        s.tests.append(t.CopyAddingFlags(t.variant, fuzzing_flags))
481    num_tests += len(s.tests)
482    for t in s.tests:
483      t.id = test_id
484      test_id += 1
485
486  if num_tests == 0:
487    print "No tests to run."
488    return 0
489
490  print(">>> Deopt fuzzing phase (%d test cases)" % num_tests)
491  progress_indicator = progress.PROGRESS_INDICATORS[options.progress]()
492  runner = execution.Runner(suites, progress_indicator, ctx)
493
494  code = runner.Run(options.j)
495  return exit_code or code
496
497
498if __name__ == "__main__":
499  sys.exit(Main())
500