• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/env python
2#
3# Copyright 2012 the V8 project authors. All rights reserved.
4# Redistribution and use in source and binary forms, with or without
5# modification, are permitted provided that the following conditions are
6# met:
7#
8#     * Redistributions of source code must retain the above copyright
9#       notice, this list of conditions and the following disclaimer.
10#     * Redistributions in binary form must reproduce the above
11#       copyright notice, this list of conditions and the following
12#       disclaimer in the documentation and/or other materials provided
13#       with the distribution.
14#     * Neither the name of Google Inc. nor the names of its
15#       contributors may be used to endorse or promote products derived
16#       from this software without specific prior written permission.
17#
18# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29
30
31import json
32import math
33import multiprocessing
34import optparse
35import os
36from os.path import join
37import random
38import shlex
39import subprocess
40import sys
41import time
42
43from testrunner.local import execution
44from testrunner.local import progress
45from testrunner.local import testsuite
46from testrunner.local import utils
47from testrunner.local import verbose
48from testrunner.objects import context
49
50
51# Base dir of the v8 checkout to be used as cwd.
52BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
53
54ARCH_GUESS = utils.DefaultArch()
55DEFAULT_TESTS = ["mjsunit", "webkit"]
56TIMEOUT_DEFAULT = 60
57TIMEOUT_SCALEFACTOR = {"debug"   : 4,
58                       "release" : 1 }
59
60MODE_FLAGS = {
61    "debug"   : ["--nohard-abort", "--nodead-code-elimination",
62                 "--nofold-constants", "--enable-slow-asserts",
63                 "--verify-heap", "--noconcurrent-recompilation"],
64    "release" : ["--nohard-abort", "--nodead-code-elimination",
65                 "--nofold-constants", "--noconcurrent-recompilation"]}
66
67SUPPORTED_ARCHS = ["android_arm",
68                   "android_ia32",
69                   "arm",
70                   "ia32",
71                   "ppc",
72                   "ppc64",
73                   "s390",
74                   "s390x",
75                   "mipsel",
76                   "x64"]
77# Double the timeout for these:
78SLOW_ARCHS = ["android_arm",
79              "android_ia32",
80              "arm",
81              "mipsel"]
82MAX_DEOPT = 1000000000
83DISTRIBUTION_MODES = ["smooth", "random"]
84
85
86class RandomDistribution:
87  def __init__(self, seed=None):
88    seed = seed or random.randint(1, sys.maxint)
89    print "Using random distribution with seed %d" % seed
90    self._random = random.Random(seed)
91
92  def Distribute(self, n, m):
93    if n > m:
94      n = m
95    return self._random.sample(xrange(1, m + 1), n)
96
97
98class SmoothDistribution:
99  """Distribute n numbers into the interval [1:m].
100  F1: Factor of the first derivation of the distribution function.
101  F2: Factor of the second derivation of the distribution function.
102  With F1 and F2 set to 0, the distribution will be equal.
103  """
104  def __init__(self, factor1=2.0, factor2=0.2):
105    self._factor1 = factor1
106    self._factor2 = factor2
107
108  def Distribute(self, n, m):
109    if n > m:
110      n = m
111    if n <= 1:
112      return [ 1 ]
113
114    result = []
115    x = 0.0
116    dx = 1.0
117    ddx = self._factor1
118    dddx = self._factor2
119    for i in range(0, n):
120      result += [ x ]
121      x += dx
122      dx += ddx
123      ddx += dddx
124
125    # Project the distribution into the interval [0:M].
126    result = [ x * m / result[-1] for x in result ]
127
128    # Equalize by n. The closer n is to m, the more equal will be the
129    # distribution.
130    for (i, x) in enumerate(result):
131      # The value of x if it was equally distributed.
132      equal_x = i / float(n - 1) * float(m - 1) + 1
133
134      # Difference factor between actual and equal distribution.
135      diff = 1 - (x / equal_x)
136
137      # Equalize x dependent on the number of values to distribute.
138      result[i] = int(x + (i + 1) * diff)
139    return result
140
141
142def Distribution(options):
143  if options.distribution_mode == "random":
144    return RandomDistribution(options.seed)
145  if options.distribution_mode == "smooth":
146    return SmoothDistribution(options.distribution_factor1,
147                              options.distribution_factor2)
148
149
150def BuildOptions():
151  result = optparse.OptionParser()
152  result.add_option("--arch",
153                    help=("The architecture to run tests for, "
154                          "'auto' or 'native' for auto-detect"),
155                    default="ia32,x64,arm")
156  result.add_option("--arch-and-mode",
157                    help="Architecture and mode in the format 'arch.mode'",
158                    default=None)
159  result.add_option("--asan",
160                    help="Regard test expectations for ASAN",
161                    default=False, action="store_true")
162  result.add_option("--buildbot",
163                    help="Adapt to path structure used on buildbots",
164                    default=False, action="store_true")
165  result.add_option("--dcheck-always-on",
166                    help="Indicates that V8 was compiled with DCHECKs enabled",
167                    default=False, action="store_true")
168  result.add_option("--command-prefix",
169                    help="Prepended to each shell command used to run a test",
170                    default="")
171  result.add_option("--coverage", help=("Exponential test coverage "
172                    "(range 0.0, 1.0) -- 0.0: one test, 1.0 all tests (slow)"),
173                    default=0.4, type="float")
174  result.add_option("--coverage-lift", help=("Lifts test coverage for tests "
175                    "with a small number of deopt points (range 0, inf)"),
176                    default=20, type="int")
177  result.add_option("--download-data", help="Download missing test suite data",
178                    default=False, action="store_true")
179  result.add_option("--distribution-factor1", help=("Factor of the first "
180                    "derivation of the distribution function"), default=2.0,
181                    type="float")
182  result.add_option("--distribution-factor2", help=("Factor of the second "
183                    "derivation of the distribution function"), default=0.7,
184                    type="float")
185  result.add_option("--distribution-mode", help=("How to select deopt points "
186                    "for a given test (smooth|random)"),
187                    default="smooth")
188  result.add_option("--dump-results-file", help=("Dump maximum number of "
189                    "deopt points per test to a file"))
190  result.add_option("--extra-flags",
191                    help="Additional flags to pass to each test command",
192                    default="")
193  result.add_option("--isolates", help="Whether to test isolates",
194                    default=False, action="store_true")
195  result.add_option("-j", help="The number of parallel tasks to run",
196                    default=0, type="int")
197  result.add_option("-m", "--mode",
198                    help="The test modes in which to run (comma-separated)",
199                    default="release,debug")
200  result.add_option("--outdir", help="Base directory with compile output",
201                    default="out")
202  result.add_option("-p", "--progress",
203                    help=("The style of progress indicator"
204                          " (verbose, dots, color, mono)"),
205                    choices=progress.PROGRESS_INDICATORS.keys(),
206                    default="mono")
207  result.add_option("--shard-count",
208                    help="Split testsuites into this number of shards",
209                    default=1, type="int")
210  result.add_option("--shard-run",
211                    help="Run this shard from the split up tests.",
212                    default=1, type="int")
213  result.add_option("--shell-dir", help="Directory containing executables",
214                    default="")
215  result.add_option("--seed", help="The seed for the random distribution",
216                    type="int")
217  result.add_option("-t", "--timeout", help="Timeout in seconds",
218                    default= -1, type="int")
219  result.add_option("-v", "--verbose", help="Verbose output",
220                    default=False, action="store_true")
221  result.add_option("--random-seed", default=0, dest="random_seed",
222                    help="Default seed for initializing random generator")
223  return result
224
225
226def ProcessOptions(options):
227  global VARIANT_FLAGS
228
229  # Architecture and mode related stuff.
230  if options.arch_and_mode:
231    tokens = options.arch_and_mode.split(".")
232    options.arch = tokens[0]
233    options.mode = tokens[1]
234  options.mode = options.mode.split(",")
235  for mode in options.mode:
236    if not mode.lower() in ["debug", "release"]:
237      print "Unknown mode %s" % mode
238      return False
239  if options.arch in ["auto", "native"]:
240    options.arch = ARCH_GUESS
241  options.arch = options.arch.split(",")
242  for arch in options.arch:
243    if not arch in SUPPORTED_ARCHS:
244      print "Unknown architecture %s" % arch
245      return False
246
247  # Special processing of other options, sorted alphabetically.
248  options.command_prefix = shlex.split(options.command_prefix)
249  options.extra_flags = shlex.split(options.extra_flags)
250  if options.j == 0:
251    options.j = multiprocessing.cpu_count()
252  while options.random_seed == 0:
253    options.random_seed = random.SystemRandom().randint(-2147483648, 2147483647)
254  if not options.distribution_mode in DISTRIBUTION_MODES:
255    print "Unknown distribution mode %s" % options.distribution_mode
256    return False
257  if options.distribution_factor1 < 0.0:
258    print ("Distribution factor1 %s is out of range. Defaulting to 0.0"
259        % options.distribution_factor1)
260    options.distribution_factor1 = 0.0
261  if options.distribution_factor2 < 0.0:
262    print ("Distribution factor2 %s is out of range. Defaulting to 0.0"
263        % options.distribution_factor2)
264    options.distribution_factor2 = 0.0
265  if options.coverage < 0.0 or options.coverage > 1.0:
266    print ("Coverage %s is out of range. Defaulting to 0.4"
267        % options.coverage)
268    options.coverage = 0.4
269  if options.coverage_lift < 0:
270    print ("Coverage lift %s is out of range. Defaulting to 0"
271        % options.coverage_lift)
272    options.coverage_lift = 0
273  return True
274
275
276def ShardTests(tests, shard_count, shard_run):
277  if shard_count < 2:
278    return tests
279  if shard_run < 1 or shard_run > shard_count:
280    print "shard-run not a valid number, should be in [1:shard-count]"
281    print "defaulting back to running all tests"
282    return tests
283  count = 0
284  shard = []
285  for test in tests:
286    if count % shard_count == shard_run - 1:
287      shard.append(test)
288    count += 1
289  return shard
290
291
292def Main():
293  # Use the v8 root as cwd as some test cases use "load" with relative paths.
294  os.chdir(BASE_DIR)
295
296  parser = BuildOptions()
297  (options, args) = parser.parse_args()
298  if not ProcessOptions(options):
299    parser.print_help()
300    return 1
301
302  exit_code = 0
303
304  suite_paths = utils.GetSuitePaths(join(BASE_DIR, "test"))
305
306  if len(args) == 0:
307    suite_paths = [ s for s in suite_paths if s in DEFAULT_TESTS ]
308  else:
309    args_suites = set()
310    for arg in args:
311      suite = arg.split(os.path.sep)[0]
312      if not suite in args_suites:
313        args_suites.add(suite)
314    suite_paths = [ s for s in suite_paths if s in args_suites ]
315
316  suites = []
317  for root in suite_paths:
318    suite = testsuite.TestSuite.LoadTestSuite(
319        os.path.join(BASE_DIR, "test", root))
320    if suite:
321      suites.append(suite)
322
323  if options.download_data:
324    for s in suites:
325      s.DownloadData()
326
327  for mode in options.mode:
328    for arch in options.arch:
329      try:
330        code = Execute(arch, mode, args, options, suites, BASE_DIR)
331        exit_code = exit_code or code
332      except KeyboardInterrupt:
333        return 2
334  return exit_code
335
336
337def CalculateNTests(m, options):
338  """Calculates the number of tests from m deopt points with exponential
339  coverage.
340  The coverage is expected to be between 0.0 and 1.0.
341  The 'coverage lift' lifts the coverage for tests with smaller m values.
342  """
343  c = float(options.coverage)
344  l = float(options.coverage_lift)
345  return int(math.pow(m, (m * c + l) / (m + l)))
346
347
348def Execute(arch, mode, args, options, suites, workspace):
349  print(">>> Running tests for %s.%s" % (arch, mode))
350
351  dist = Distribution(options)
352
353  shell_dir = options.shell_dir
354  if not shell_dir:
355    if options.buildbot:
356      shell_dir = os.path.join(workspace, options.outdir, mode)
357      mode = mode.lower()
358    else:
359      shell_dir = os.path.join(workspace, options.outdir,
360                               "%s.%s" % (arch, mode))
361  shell_dir = os.path.relpath(shell_dir)
362
363  # Populate context object.
364  mode_flags = MODE_FLAGS[mode]
365  timeout = options.timeout
366  if timeout == -1:
367    # Simulators are slow, therefore allow a longer default timeout.
368    if arch in SLOW_ARCHS:
369      timeout = 2 * TIMEOUT_DEFAULT;
370    else:
371      timeout = TIMEOUT_DEFAULT;
372
373  timeout *= TIMEOUT_SCALEFACTOR[mode]
374  ctx = context.Context(arch, mode, shell_dir,
375                        mode_flags, options.verbose,
376                        timeout, options.isolates,
377                        options.command_prefix,
378                        options.extra_flags,
379                        False,  # Keep i18n on by default.
380                        options.random_seed,
381                        True,  # No sorting of test cases.
382                        0,  # Don't rerun failing tests.
383                        0,  # No use of a rerun-failing-tests maximum.
384                        False,  # No predictable mode.
385                        False,  # No no_harness mode.
386                        False,  # Don't use perf data.
387                        False)  # Coverage not supported.
388
389  # Find available test suites and read test cases from them.
390  variables = {
391    "arch": arch,
392    "asan": options.asan,
393    "deopt_fuzzer": True,
394    "gc_stress": False,
395    "gcov_coverage": False,
396    "isolates": options.isolates,
397    "mode": mode,
398    "no_i18n": False,
399    "no_snap": False,
400    "simulator": utils.UseSimulator(arch),
401    "system": utils.GuessOS(),
402    "tsan": False,
403    "msan": False,
404    "dcheck_always_on": options.dcheck_always_on,
405    "novfp3": False,
406    "predictable": False,
407    "byteorder": sys.byteorder,
408  }
409  all_tests = []
410  num_tests = 0
411  test_id = 0
412
413  # Remember test case prototypes for the fuzzing phase.
414  test_backup = dict((s, []) for s in suites)
415
416  for s in suites:
417    s.ReadStatusFile(variables)
418    s.ReadTestCases(ctx)
419    if len(args) > 0:
420      s.FilterTestCasesByArgs(args)
421    all_tests += s.tests
422    s.FilterTestCasesByStatus(False)
423    test_backup[s] = s.tests
424    analysis_flags = ["--deopt-every-n-times", "%d" % MAX_DEOPT,
425                      "--print-deopt-stress"]
426    s.tests = [ t.CopyAddingFlags(t.variant, analysis_flags) for t in s.tests ]
427    num_tests += len(s.tests)
428    for t in s.tests:
429      t.id = test_id
430      test_id += 1
431
432  if num_tests == 0:
433    print "No tests to run."
434    return 0
435
436  print(">>> Collection phase")
437  progress_indicator = progress.PROGRESS_INDICATORS[options.progress]()
438  runner = execution.Runner(suites, progress_indicator, ctx)
439
440  exit_code = runner.Run(options.j)
441
442  print(">>> Analysis phase")
443  num_tests = 0
444  test_id = 0
445  for s in suites:
446    test_results = {}
447    for t in s.tests:
448      for line in t.output.stdout.splitlines():
449        if line.startswith("=== Stress deopt counter: "):
450          test_results[t.path] = MAX_DEOPT - int(line.split(" ")[-1])
451    for t in s.tests:
452      if t.path not in test_results:
453        print "Missing results for %s" % t.path
454    if options.dump_results_file:
455      results_dict = dict((t.path, n) for (t, n) in test_results.iteritems())
456      with file("%s.%d.txt" % (dump_results_file, time.time()), "w") as f:
457        f.write(json.dumps(results_dict))
458
459    # Reset tests and redistribute the prototypes from the collection phase.
460    s.tests = []
461    if options.verbose:
462      print "Test distributions:"
463    for t in test_backup[s]:
464      max_deopt = test_results.get(t.path, 0)
465      if max_deopt == 0:
466        continue
467      n_deopt = CalculateNTests(max_deopt, options)
468      distribution = dist.Distribute(n_deopt, max_deopt)
469      if options.verbose:
470        print "%s %s" % (t.path, distribution)
471      for i in distribution:
472        fuzzing_flags = ["--deopt-every-n-times", "%d" % i]
473        s.tests.append(t.CopyAddingFlags(t.variant, fuzzing_flags))
474    num_tests += len(s.tests)
475    for t in s.tests:
476      t.id = test_id
477      test_id += 1
478
479  if num_tests == 0:
480    print "No tests to run."
481    return 0
482
483  print(">>> Deopt fuzzing phase (%d test cases)" % num_tests)
484  progress_indicator = progress.PROGRESS_INDICATORS[options.progress]()
485  runner = execution.Runner(suites, progress_indicator, ctx)
486
487  code = runner.Run(options.j)
488  return exit_code or code
489
490
491if __name__ == "__main__":
492  sys.exit(Main())
493