• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/env python
2#
3# Copyright 2012 the V8 project authors. All rights reserved.
4# Redistribution and use in source and binary forms, with or without
5# modification, are permitted provided that the following conditions are
6# met:
7#
8#     * Redistributions of source code must retain the above copyright
9#       notice, this list of conditions and the following disclaimer.
10#     * Redistributions in binary form must reproduce the above
11#       copyright notice, this list of conditions and the following
12#       disclaimer in the documentation and/or other materials provided
13#       with the distribution.
14#     * Neither the name of Google Inc. nor the names of its
15#       contributors may be used to endorse or promote products derived
16#       from this software without specific prior written permission.
17#
18# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29
30
31import json
32import math
33import multiprocessing
34import optparse
35import os
36from os.path import join
37import random
38import shlex
39import subprocess
40import sys
41import time
42
43from testrunner.local import execution
44from testrunner.local import progress
45from testrunner.local import testsuite
46from testrunner.local import utils
47from testrunner.local import verbose
48from testrunner.objects import context
49
50
51ARCH_GUESS = utils.DefaultArch()
52DEFAULT_TESTS = ["mjsunit", "webkit"]
53TIMEOUT_DEFAULT = 60
54TIMEOUT_SCALEFACTOR = {"debug"   : 4,
55                       "release" : 1 }
56
57MODE_FLAGS = {
58    "debug"   : ["--nohard-abort", "--nodead-code-elimination",
59                 "--nofold-constants", "--enable-slow-asserts",
60                 "--debug-code", "--verify-heap",
61                 "--noconcurrent-recompilation"],
62    "release" : ["--nohard-abort", "--nodead-code-elimination",
63                 "--nofold-constants", "--noconcurrent-recompilation"]}
64
65SUPPORTED_ARCHS = ["android_arm",
66                   "android_ia32",
67                   "arm",
68                   "ia32",
69                   "ppc",
70                   "ppc64",
71                   "mipsel",
72                   "nacl_ia32",
73                   "nacl_x64",
74                   "x64"]
75# Double the timeout for these:
76SLOW_ARCHS = ["android_arm",
77              "android_ia32",
78              "arm",
79              "mipsel",
80              "nacl_ia32",
81              "nacl_x64"]
82MAX_DEOPT = 1000000000
83DISTRIBUTION_MODES = ["smooth", "random"]
84
85
86class RandomDistribution:
87  def __init__(self, seed=None):
88    seed = seed or random.randint(1, sys.maxint)
89    print "Using random distribution with seed %d" % seed
90    self._random = random.Random(seed)
91
92  def Distribute(self, n, m):
93    if n > m:
94      n = m
95    return self._random.sample(xrange(1, m + 1), n)
96
97
98class SmoothDistribution:
99  """Distribute n numbers into the interval [1:m].
100  F1: Factor of the first derivation of the distribution function.
101  F2: Factor of the second derivation of the distribution function.
102  With F1 and F2 set to 0, the distribution will be equal.
103  """
104  def __init__(self, factor1=2.0, factor2=0.2):
105    self._factor1 = factor1
106    self._factor2 = factor2
107
108  def Distribute(self, n, m):
109    if n > m:
110      n = m
111    if n <= 1:
112      return [ 1 ]
113
114    result = []
115    x = 0.0
116    dx = 1.0
117    ddx = self._factor1
118    dddx = self._factor2
119    for i in range(0, n):
120      result += [ x ]
121      x += dx
122      dx += ddx
123      ddx += dddx
124
125    # Project the distribution into the interval [0:M].
126    result = [ x * m / result[-1] for x in result ]
127
128    # Equalize by n. The closer n is to m, the more equal will be the
129    # distribution.
130    for (i, x) in enumerate(result):
131      # The value of x if it was equally distributed.
132      equal_x = i / float(n - 1) * float(m - 1) + 1
133
134      # Difference factor between actual and equal distribution.
135      diff = 1 - (x / equal_x)
136
137      # Equalize x dependent on the number of values to distribute.
138      result[i] = int(x + (i + 1) * diff)
139    return result
140
141
142def Distribution(options):
143  if options.distribution_mode == "random":
144    return RandomDistribution(options.seed)
145  if options.distribution_mode == "smooth":
146    return SmoothDistribution(options.distribution_factor1,
147                              options.distribution_factor2)
148
149
150def BuildOptions():
151  result = optparse.OptionParser()
152  result.add_option("--arch",
153                    help=("The architecture to run tests for, "
154                          "'auto' or 'native' for auto-detect"),
155                    default="ia32,x64,arm")
156  result.add_option("--arch-and-mode",
157                    help="Architecture and mode in the format 'arch.mode'",
158                    default=None)
159  result.add_option("--asan",
160                    help="Regard test expectations for ASAN",
161                    default=False, action="store_true")
162  result.add_option("--buildbot",
163                    help="Adapt to path structure used on buildbots",
164                    default=False, action="store_true")
165  result.add_option("--dcheck-always-on",
166                    help="Indicates that V8 was compiled with DCHECKs enabled",
167                    default=False, action="store_true")
168  result.add_option("--command-prefix",
169                    help="Prepended to each shell command used to run a test",
170                    default="")
171  result.add_option("--coverage", help=("Exponential test coverage "
172                    "(range 0.0, 1.0) -- 0.0: one test, 1.0 all tests (slow)"),
173                    default=0.4, type="float")
174  result.add_option("--coverage-lift", help=("Lifts test coverage for tests "
175                    "with a small number of deopt points (range 0, inf)"),
176                    default=20, type="int")
177  result.add_option("--download-data", help="Download missing test suite data",
178                    default=False, action="store_true")
179  result.add_option("--distribution-factor1", help=("Factor of the first "
180                    "derivation of the distribution function"), default=2.0,
181                    type="float")
182  result.add_option("--distribution-factor2", help=("Factor of the second "
183                    "derivation of the distribution function"), default=0.7,
184                    type="float")
185  result.add_option("--distribution-mode", help=("How to select deopt points "
186                    "for a given test (smooth|random)"),
187                    default="smooth")
188  result.add_option("--dump-results-file", help=("Dump maximum number of "
189                    "deopt points per test to a file"))
190  result.add_option("--extra-flags",
191                    help="Additional flags to pass to each test command",
192                    default="")
193  result.add_option("--isolates", help="Whether to test isolates",
194                    default=False, action="store_true")
195  result.add_option("-j", help="The number of parallel tasks to run",
196                    default=0, type="int")
197  result.add_option("-m", "--mode",
198                    help="The test modes in which to run (comma-separated)",
199                    default="release,debug")
200  result.add_option("--outdir", help="Base directory with compile output",
201                    default="out")
202  result.add_option("-p", "--progress",
203                    help=("The style of progress indicator"
204                          " (verbose, dots, color, mono)"),
205                    choices=progress.PROGRESS_INDICATORS.keys(),
206                    default="mono")
207  result.add_option("--shard-count",
208                    help="Split testsuites into this number of shards",
209                    default=1, type="int")
210  result.add_option("--shard-run",
211                    help="Run this shard from the split up tests.",
212                    default=1, type="int")
213  result.add_option("--shell-dir", help="Directory containing executables",
214                    default="")
215  result.add_option("--seed", help="The seed for the random distribution",
216                    type="int")
217  result.add_option("-t", "--timeout", help="Timeout in seconds",
218                    default= -1, type="int")
219  result.add_option("-v", "--verbose", help="Verbose output",
220                    default=False, action="store_true")
221  result.add_option("--random-seed", default=0, dest="random_seed",
222                    help="Default seed for initializing random generator")
223  return result
224
225
226def ProcessOptions(options):
227  global VARIANT_FLAGS
228
229  # Architecture and mode related stuff.
230  if options.arch_and_mode:
231    tokens = options.arch_and_mode.split(".")
232    options.arch = tokens[0]
233    options.mode = tokens[1]
234  options.mode = options.mode.split(",")
235  for mode in options.mode:
236    if not mode.lower() in ["debug", "release"]:
237      print "Unknown mode %s" % mode
238      return False
239  if options.arch in ["auto", "native"]:
240    options.arch = ARCH_GUESS
241  options.arch = options.arch.split(",")
242  for arch in options.arch:
243    if not arch in SUPPORTED_ARCHS:
244      print "Unknown architecture %s" % arch
245      return False
246
247  # Special processing of other options, sorted alphabetically.
248  options.command_prefix = shlex.split(options.command_prefix)
249  options.extra_flags = shlex.split(options.extra_flags)
250  if options.j == 0:
251    options.j = multiprocessing.cpu_count()
252  while options.random_seed == 0:
253    options.random_seed = random.SystemRandom().randint(-2147483648, 2147483647)
254  if not options.distribution_mode in DISTRIBUTION_MODES:
255    print "Unknown distribution mode %s" % options.distribution_mode
256    return False
257  if options.distribution_factor1 < 0.0:
258    print ("Distribution factor1 %s is out of range. Defaulting to 0.0"
259        % options.distribution_factor1)
260    options.distribution_factor1 = 0.0
261  if options.distribution_factor2 < 0.0:
262    print ("Distribution factor2 %s is out of range. Defaulting to 0.0"
263        % options.distribution_factor2)
264    options.distribution_factor2 = 0.0
265  if options.coverage < 0.0 or options.coverage > 1.0:
266    print ("Coverage %s is out of range. Defaulting to 0.4"
267        % options.coverage)
268    options.coverage = 0.4
269  if options.coverage_lift < 0:
270    print ("Coverage lift %s is out of range. Defaulting to 0"
271        % options.coverage_lift)
272    options.coverage_lift = 0
273  return True
274
275
276def ShardTests(tests, shard_count, shard_run):
277  if shard_count < 2:
278    return tests
279  if shard_run < 1 or shard_run > shard_count:
280    print "shard-run not a valid number, should be in [1:shard-count]"
281    print "defaulting back to running all tests"
282    return tests
283  count = 0
284  shard = []
285  for test in tests:
286    if count % shard_count == shard_run - 1:
287      shard.append(test)
288    count += 1
289  return shard
290
291
292def Main():
293  parser = BuildOptions()
294  (options, args) = parser.parse_args()
295  if not ProcessOptions(options):
296    parser.print_help()
297    return 1
298
299  exit_code = 0
300  workspace = os.path.abspath(join(os.path.dirname(sys.argv[0]), ".."))
301
302  suite_paths = utils.GetSuitePaths(join(workspace, "test"))
303
304  if len(args) == 0:
305    suite_paths = [ s for s in suite_paths if s in DEFAULT_TESTS ]
306  else:
307    args_suites = set()
308    for arg in args:
309      suite = arg.split(os.path.sep)[0]
310      if not suite in args_suites:
311        args_suites.add(suite)
312    suite_paths = [ s for s in suite_paths if s in args_suites ]
313
314  suites = []
315  for root in suite_paths:
316    suite = testsuite.TestSuite.LoadTestSuite(
317        os.path.join(workspace, "test", root))
318    if suite:
319      suite.SetupWorkingDirectory()
320      suites.append(suite)
321
322  if options.download_data:
323    for s in suites:
324      s.DownloadData()
325
326  for mode in options.mode:
327    for arch in options.arch:
328      try:
329        code = Execute(arch, mode, args, options, suites, workspace)
330        exit_code = exit_code or code
331      except KeyboardInterrupt:
332        return 2
333  return exit_code
334
335
336def CalculateNTests(m, options):
337  """Calculates the number of tests from m deopt points with exponential
338  coverage.
339  The coverage is expected to be between 0.0 and 1.0.
340  The 'coverage lift' lifts the coverage for tests with smaller m values.
341  """
342  c = float(options.coverage)
343  l = float(options.coverage_lift)
344  return int(math.pow(m, (m * c + l) / (m + l)))
345
346
347def Execute(arch, mode, args, options, suites, workspace):
348  print(">>> Running tests for %s.%s" % (arch, mode))
349
350  dist = Distribution(options)
351
352  shell_dir = options.shell_dir
353  if not shell_dir:
354    if options.buildbot:
355      shell_dir = os.path.join(workspace, options.outdir, mode)
356      mode = mode.lower()
357    else:
358      shell_dir = os.path.join(workspace, options.outdir,
359                               "%s.%s" % (arch, mode))
360  shell_dir = os.path.relpath(shell_dir)
361
362  # Populate context object.
363  mode_flags = MODE_FLAGS[mode]
364  timeout = options.timeout
365  if timeout == -1:
366    # Simulators are slow, therefore allow a longer default timeout.
367    if arch in SLOW_ARCHS:
368      timeout = 2 * TIMEOUT_DEFAULT;
369    else:
370      timeout = TIMEOUT_DEFAULT;
371
372  timeout *= TIMEOUT_SCALEFACTOR[mode]
373  ctx = context.Context(arch, mode, shell_dir,
374                        mode_flags, options.verbose,
375                        timeout, options.isolates,
376                        options.command_prefix,
377                        options.extra_flags,
378                        False,  # Keep i18n on by default.
379                        options.random_seed,
380                        True,  # No sorting of test cases.
381                        0,  # Don't rerun failing tests.
382                        0,  # No use of a rerun-failing-tests maximum.
383                        False,  # No predictable mode.
384                        False,  # No no_harness mode.
385                        False)   # Don't use perf data.
386
387  # Find available test suites and read test cases from them.
388  variables = {
389    "arch": arch,
390    "asan": options.asan,
391    "deopt_fuzzer": True,
392    "gc_stress": False,
393    "gcov_coverage": False,
394    "ignition": False,
395    "isolates": options.isolates,
396    "mode": mode,
397    "no_i18n": False,
398    "no_snap": False,
399    "simulator": utils.UseSimulator(arch),
400    "system": utils.GuessOS(),
401    "tsan": False,
402    "msan": False,
403    "dcheck_always_on": options.dcheck_always_on,
404    "novfp3": False,
405    "predictable": False,
406    "byteorder": sys.byteorder,
407  }
408  all_tests = []
409  num_tests = 0
410  test_id = 0
411
412  # Remember test case prototypes for the fuzzing phase.
413  test_backup = dict((s, []) for s in suites)
414
415  for s in suites:
416    s.ReadStatusFile(variables)
417    s.ReadTestCases(ctx)
418    if len(args) > 0:
419      s.FilterTestCasesByArgs(args)
420    all_tests += s.tests
421    s.FilterTestCasesByStatus(False)
422    test_backup[s] = s.tests
423    analysis_flags = ["--deopt-every-n-times", "%d" % MAX_DEOPT,
424                      "--print-deopt-stress"]
425    s.tests = [ t.CopyAddingFlags(t.variant, analysis_flags) for t in s.tests ]
426    num_tests += len(s.tests)
427    for t in s.tests:
428      t.id = test_id
429      test_id += 1
430
431  if num_tests == 0:
432    print "No tests to run."
433    return 0
434
435  print(">>> Collection phase")
436  progress_indicator = progress.PROGRESS_INDICATORS[options.progress]()
437  runner = execution.Runner(suites, progress_indicator, ctx)
438
439  exit_code = runner.Run(options.j)
440
441  print(">>> Analysis phase")
442  num_tests = 0
443  test_id = 0
444  for s in suites:
445    test_results = {}
446    for t in s.tests:
447      for line in t.output.stdout.splitlines():
448        if line.startswith("=== Stress deopt counter: "):
449          test_results[t.path] = MAX_DEOPT - int(line.split(" ")[-1])
450    for t in s.tests:
451      if t.path not in test_results:
452        print "Missing results for %s" % t.path
453    if options.dump_results_file:
454      results_dict = dict((t.path, n) for (t, n) in test_results.iteritems())
455      with file("%s.%d.txt" % (dump_results_file, time.time()), "w") as f:
456        f.write(json.dumps(results_dict))
457
458    # Reset tests and redistribute the prototypes from the collection phase.
459    s.tests = []
460    if options.verbose:
461      print "Test distributions:"
462    for t in test_backup[s]:
463      max_deopt = test_results.get(t.path, 0)
464      if max_deopt == 0:
465        continue
466      n_deopt = CalculateNTests(max_deopt, options)
467      distribution = dist.Distribute(n_deopt, max_deopt)
468      if options.verbose:
469        print "%s %s" % (t.path, distribution)
470      for i in distribution:
471        fuzzing_flags = ["--deopt-every-n-times", "%d" % i]
472        s.tests.append(t.CopyAddingFlags(t.variant, fuzzing_flags))
473    num_tests += len(s.tests)
474    for t in s.tests:
475      t.id = test_id
476      test_id += 1
477
478  if num_tests == 0:
479    print "No tests to run."
480    return 0
481
482  print(">>> Deopt fuzzing phase (%d test cases)" % num_tests)
483  progress_indicator = progress.PROGRESS_INDICATORS[options.progress]()
484  runner = execution.Runner(suites, progress_indicator, ctx)
485
486  code = runner.Run(options.j)
487  return exit_code or code
488
489
490if __name__ == "__main__":
491  sys.exit(Main())
492