• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/env python
2#
3# Copyright 2012 the V8 project authors. All rights reserved.
4# Redistribution and use in source and binary forms, with or without
5# modification, are permitted provided that the following conditions are
6# met:
7#
8#     * Redistributions of source code must retain the above copyright
9#       notice, this list of conditions and the following disclaimer.
10#     * Redistributions in binary form must reproduce the above
11#       copyright notice, this list of conditions and the following
12#       disclaimer in the documentation and/or other materials provided
13#       with the distribution.
14#     * Neither the name of Google Inc. nor the names of its
15#       contributors may be used to endorse or promote products derived
16#       from this software without specific prior written permission.
17#
18# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29
30
31import itertools
32import multiprocessing
33import optparse
34import os
35from os.path import join
36import shlex
37import subprocess
38import sys
39import time
40
41from testrunner.local import execution
42from testrunner.local import progress
43from testrunner.local import testsuite
44from testrunner.local import utils
45from testrunner.local import verbose
46from testrunner.network import network_execution
47from testrunner.objects import context
48
49
50ARCH_GUESS = utils.DefaultArch()
51DEFAULT_TESTS = ["mjsunit", "cctest", "message", "preparser"]
52TIMEOUT_DEFAULT = 60
53TIMEOUT_SCALEFACTOR = {"debug"   : 4,
54                       "release" : 1 }
55
56# Use this to run several variants of the tests.
57VARIANT_FLAGS = {
58    "default": [],
59    "stress": ["--stress-opt", "--always-opt"],
60    "nocrankshaft": ["--nocrankshaft"]}
61
62VARIANTS = ["default", "stress", "nocrankshaft"]
63
64MODE_FLAGS = {
65    "debug"   : ["--nobreak-on-abort", "--nodead-code-elimination",
66                 "--nofold-constants", "--enable-slow-asserts",
67                 "--debug-code", "--verify-heap"],
68    "release" : ["--nobreak-on-abort", "--nodead-code-elimination",
69                 "--nofold-constants"]}
70
71SUPPORTED_ARCHS = ["android_arm",
72                   "android_ia32",
73                   "arm",
74                   "ia32",
75                   "mipsel",
76                   "nacl_ia32",
77                   "nacl_x64",
78                   "x64"]
79# Double the timeout for these:
80SLOW_ARCHS = ["android_arm",
81              "android_ia32",
82              "arm",
83              "mipsel",
84              "nacl_ia32",
85              "nacl_x64"]
86
87
88def BuildOptions():
89  result = optparse.OptionParser()
90  result.add_option("--arch",
91                    help=("The architecture to run tests for, "
92                          "'auto' or 'native' for auto-detect"),
93                    default="ia32,x64,arm")
94  result.add_option("--arch-and-mode",
95                    help="Architecture and mode in the format 'arch.mode'",
96                    default=None)
97  result.add_option("--buildbot",
98                    help="Adapt to path structure used on buildbots",
99                    default=False, action="store_true")
100  result.add_option("--cat", help="Print the source of the tests",
101                    default=False, action="store_true")
102  result.add_option("--flaky-tests",
103                    help="Regard tests marked as flaky (run|skip|dontcare)",
104                    default="dontcare")
105  result.add_option("--slow-tests",
106                    help="Regard slow tests (run|skip|dontcare)",
107                    default="dontcare")
108  result.add_option("--pass-fail-tests",
109                    help="Regard pass|fail tests (run|skip|dontcare)",
110                    default="dontcare")
111  result.add_option("--command-prefix",
112                    help="Prepended to each shell command used to run a test",
113                    default="")
114  result.add_option("--download-data", help="Download missing test suite data",
115                    default=False, action="store_true")
116  result.add_option("--extra-flags",
117                    help="Additional flags to pass to each test command",
118                    default="")
119  result.add_option("--isolates", help="Whether to test isolates",
120                    default=False, action="store_true")
121  result.add_option("-j", help="The number of parallel tasks to run",
122                    default=0, type="int")
123  result.add_option("-m", "--mode",
124                    help="The test modes in which to run (comma-separated)",
125                    default="release,debug")
126  result.add_option("--no-i18n", "--noi18n",
127                    help="Skip internationalization tests",
128                    default=False, action="store_true")
129  result.add_option("--no-network", "--nonetwork",
130                    help="Don't distribute tests on the network",
131                    default=(utils.GuessOS() != "linux"),
132                    dest="no_network", action="store_true")
133  result.add_option("--no-presubmit", "--nopresubmit",
134                    help='Skip presubmit checks',
135                    default=False, dest="no_presubmit", action="store_true")
136  result.add_option("--no-stress", "--nostress",
137                    help="Don't run crankshaft --always-opt --stress-op test",
138                    default=False, dest="no_stress", action="store_true")
139  result.add_option("--no-variants", "--novariants",
140                    help="Don't run any testing variants",
141                    default=False, dest="no_variants", action="store_true")
142  result.add_option("--variants",
143                    help="Comma-separated list of testing variants")
144  result.add_option("--outdir", help="Base directory with compile output",
145                    default="out")
146  result.add_option("-p", "--progress",
147                    help=("The style of progress indicator"
148                          " (verbose, dots, color, mono)"),
149                    choices=progress.PROGRESS_INDICATORS.keys(), default="mono")
150  result.add_option("--quickcheck", default=False, action="store_true",
151                    help=("Quick check mode (skip slow/flaky tests)"))
152  result.add_option("--report", help="Print a summary of the tests to be run",
153                    default=False, action="store_true")
154  result.add_option("--shard-count",
155                    help="Split testsuites into this number of shards",
156                    default=1, type="int")
157  result.add_option("--shard-run",
158                    help="Run this shard from the split up tests.",
159                    default=1, type="int")
160  result.add_option("--shell", help="DEPRECATED! use --shell-dir", default="")
161  result.add_option("--shell-dir", help="Directory containing executables",
162                    default="")
163  result.add_option("--stress-only",
164                    help="Only run tests with --always-opt --stress-opt",
165                    default=False, action="store_true")
166  result.add_option("--time", help="Print timing information after running",
167                    default=False, action="store_true")
168  result.add_option("-t", "--timeout", help="Timeout in seconds",
169                    default= -1, type="int")
170  result.add_option("-v", "--verbose", help="Verbose output",
171                    default=False, action="store_true")
172  result.add_option("--valgrind", help="Run tests through valgrind",
173                    default=False, action="store_true")
174  result.add_option("--warn-unused", help="Report unused rules",
175                    default=False, action="store_true")
176  result.add_option("--junitout", help="File name of the JUnit output")
177  result.add_option("--junittestsuite",
178                    help="The testsuite name in the JUnit output file",
179                    default="v8tests")
180  return result
181
182
183def ProcessOptions(options):
184  global VARIANT_FLAGS
185  global VARIANTS
186
187  # Architecture and mode related stuff.
188  if options.arch_and_mode:
189    options.arch_and_mode = [arch_and_mode.split(".")
190        for arch_and_mode in options.arch_and_mode.split(",")]
191    options.arch = ",".join([tokens[0] for tokens in options.arch_and_mode])
192    options.mode = ",".join([tokens[1] for tokens in options.arch_and_mode])
193  options.mode = options.mode.split(",")
194  for mode in options.mode:
195    if not mode.lower() in ["debug", "release", "optdebug"]:
196      print "Unknown mode %s" % mode
197      return False
198  if options.arch in ["auto", "native"]:
199    options.arch = ARCH_GUESS
200  options.arch = options.arch.split(",")
201  for arch in options.arch:
202    if not arch in SUPPORTED_ARCHS:
203      print "Unknown architecture %s" % arch
204      return False
205
206  # Store the final configuration in arch_and_mode list. Don't overwrite
207  # predefined arch_and_mode since it is more expressive than arch and mode.
208  if not options.arch_and_mode:
209    options.arch_and_mode = itertools.product(options.arch, options.mode)
210
211  # Special processing of other options, sorted alphabetically.
212
213  if options.buildbot:
214    # Buildbots run presubmit tests as a separate step.
215    options.no_presubmit = True
216    options.no_network = True
217  if options.command_prefix:
218    print("Specifying --command-prefix disables network distribution, "
219          "running tests locally.")
220    options.no_network = True
221  options.command_prefix = shlex.split(options.command_prefix)
222  options.extra_flags = shlex.split(options.extra_flags)
223  if options.j == 0:
224    options.j = multiprocessing.cpu_count()
225
226  def excl(*args):
227    """Returns true if zero or one of multiple arguments are true."""
228    return reduce(lambda x, y: x + y, args) <= 1
229
230  if not excl(options.no_stress, options.stress_only, options.no_variants,
231              bool(options.variants), options.quickcheck):
232    print("Use only one of --no-stress, --stress-only, --no-variants, "
233          "--variants, or --quickcheck.")
234    return False
235  if options.no_stress:
236    VARIANTS = ["default", "nocrankshaft"]
237  if options.no_variants:
238    VARIANTS = ["default"]
239  if options.stress_only:
240    VARIANTS = ["stress"]
241  if options.variants:
242    VARIANTS = options.variants.split(",")
243    if not set(VARIANTS).issubset(VARIANT_FLAGS.keys()):
244      print "All variants must be in %s" % str(VARIANT_FLAGS.keys())
245      return False
246  if options.quickcheck:
247    VARIANTS = ["default", "stress"]
248    options.flaky_tests = "skip"
249    options.slow_tests = "skip"
250    options.pass_fail_tests = "skip"
251
252  if not options.shell_dir:
253    if options.shell:
254      print "Warning: --shell is deprecated, use --shell-dir instead."
255      options.shell_dir = os.path.dirname(options.shell)
256  if options.valgrind:
257    run_valgrind = os.path.join("tools", "run-valgrind.py")
258    # This is OK for distributed running, so we don't need to set no_network.
259    options.command_prefix = (["python", "-u", run_valgrind] +
260                              options.command_prefix)
261  def CheckTestMode(name, option):
262    if not option in ["run", "skip", "dontcare"]:
263      print "Unknown %s mode %s" % (name, option)
264      return False
265    return True
266  if not CheckTestMode("flaky test", options.flaky_tests):
267    return False
268  if not CheckTestMode("slow test", options.slow_tests):
269    return False
270  if not CheckTestMode("pass|fail test", options.pass_fail_tests):
271    return False
272  if not options.no_i18n:
273    DEFAULT_TESTS.append("intl")
274  return True
275
276
277def ShardTests(tests, shard_count, shard_run):
278  if shard_count < 2:
279    return tests
280  if shard_run < 1 or shard_run > shard_count:
281    print "shard-run not a valid number, should be in [1:shard-count]"
282    print "defaulting back to running all tests"
283    return tests
284  count = 0
285  shard = []
286  for test in tests:
287    if count % shard_count == shard_run - 1:
288      shard.append(test)
289    count += 1
290  return shard
291
292
293def Main():
294  parser = BuildOptions()
295  (options, args) = parser.parse_args()
296  if not ProcessOptions(options):
297    parser.print_help()
298    return 1
299
300  exit_code = 0
301  workspace = os.path.abspath(join(os.path.dirname(sys.argv[0]), ".."))
302  if not options.no_presubmit:
303    print ">>> running presubmit tests"
304    code = subprocess.call(
305        [sys.executable, join(workspace, "tools", "presubmit.py")])
306    exit_code = code
307
308  suite_paths = utils.GetSuitePaths(join(workspace, "test"))
309
310  if len(args) == 0:
311    suite_paths = [ s for s in DEFAULT_TESTS if s in suite_paths ]
312  else:
313    args_suites = set()
314    for arg in args:
315      suite = arg.split(os.path.sep)[0]
316      if not suite in args_suites:
317        args_suites.add(suite)
318    suite_paths = [ s for s in args_suites if s in suite_paths ]
319
320  suites = []
321  for root in suite_paths:
322    suite = testsuite.TestSuite.LoadTestSuite(
323        os.path.join(workspace, "test", root))
324    if suite:
325      suites.append(suite)
326
327  if options.download_data:
328    for s in suites:
329      s.DownloadData()
330
331  for (arch, mode) in options.arch_and_mode:
332    code = Execute(arch, mode, args, options, suites, workspace)
333    exit_code = exit_code or code
334  return exit_code
335
336
337def Execute(arch, mode, args, options, suites, workspace):
338  print(">>> Running tests for %s.%s" % (arch, mode))
339
340  shell_dir = options.shell_dir
341  if not shell_dir:
342    if options.buildbot:
343      shell_dir = os.path.join(workspace, options.outdir, mode)
344      mode = mode.lower()
345    else:
346      shell_dir = os.path.join(workspace, options.outdir,
347                               "%s.%s" % (arch, mode))
348  shell_dir = os.path.relpath(shell_dir)
349
350  if mode == "optdebug":
351    mode = "debug"  # "optdebug" is just an alias.
352
353  # Populate context object.
354  mode_flags = MODE_FLAGS[mode]
355  timeout = options.timeout
356  if timeout == -1:
357    # Simulators are slow, therefore allow a longer default timeout.
358    if arch in SLOW_ARCHS:
359      timeout = 2 * TIMEOUT_DEFAULT;
360    else:
361      timeout = TIMEOUT_DEFAULT;
362
363  timeout *= TIMEOUT_SCALEFACTOR[mode]
364  ctx = context.Context(arch, mode, shell_dir,
365                        mode_flags, options.verbose,
366                        timeout, options.isolates,
367                        options.command_prefix,
368                        options.extra_flags,
369                        options.no_i18n)
370
371  # Find available test suites and read test cases from them.
372  variables = {
373    "mode": mode,
374    "arch": arch,
375    "system": utils.GuessOS(),
376    "isolates": options.isolates,
377    "deopt_fuzzer": False,
378    "no_i18n": options.no_i18n,
379  }
380  all_tests = []
381  num_tests = 0
382  test_id = 0
383  for s in suites:
384    s.ReadStatusFile(variables)
385    s.ReadTestCases(ctx)
386    if len(args) > 0:
387      s.FilterTestCasesByArgs(args)
388    all_tests += s.tests
389    s.FilterTestCasesByStatus(options.warn_unused, options.flaky_tests,
390                              options.slow_tests, options.pass_fail_tests)
391    if options.cat:
392      verbose.PrintTestSource(s.tests)
393      continue
394    variant_flags = [VARIANT_FLAGS[var] for var in VARIANTS]
395    s.tests = [ t.CopyAddingFlags(v)
396                for t in s.tests
397                for v in s.VariantFlags(t, variant_flags) ]
398    s.tests = ShardTests(s.tests, options.shard_count, options.shard_run)
399    num_tests += len(s.tests)
400    for t in s.tests:
401      t.id = test_id
402      test_id += 1
403
404  if options.cat:
405    return 0  # We're done here.
406
407  if options.report:
408    verbose.PrintReport(all_tests)
409
410  if num_tests == 0:
411    print "No tests to run."
412    return 0
413
414  # Run the tests, either locally or distributed on the network.
415  try:
416    start_time = time.time()
417    progress_indicator = progress.PROGRESS_INDICATORS[options.progress]()
418    if options.junitout:
419      progress_indicator = progress.JUnitTestProgressIndicator(
420          progress_indicator, options.junitout, options.junittestsuite)
421
422    run_networked = not options.no_network
423    if not run_networked:
424      print("Network distribution disabled, running tests locally.")
425    elif utils.GuessOS() != "linux":
426      print("Network distribution is only supported on Linux, sorry!")
427      run_networked = False
428    peers = []
429    if run_networked:
430      peers = network_execution.GetPeers()
431      if not peers:
432        print("No connection to distribution server; running tests locally.")
433        run_networked = False
434      elif len(peers) == 1:
435        print("No other peers on the network; running tests locally.")
436        run_networked = False
437      elif num_tests <= 100:
438        print("Less than 100 tests, running them locally.")
439        run_networked = False
440
441    if run_networked:
442      runner = network_execution.NetworkedRunner(suites, progress_indicator,
443                                                 ctx, peers, workspace)
444    else:
445      runner = execution.Runner(suites, progress_indicator, ctx)
446
447    exit_code = runner.Run(options.j)
448    if runner.terminate:
449      return exit_code
450    overall_duration = time.time() - start_time
451  except KeyboardInterrupt:
452    return 1
453
454  if options.time:
455    verbose.PrintTestDurations(suites, overall_duration)
456  return exit_code
457
458
459if __name__ == "__main__":
460  sys.exit(Main())
461