• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/env python
2#
3# Copyright 2012 the V8 project authors. All rights reserved.
4# Redistribution and use in source and binary forms, with or without
5# modification, are permitted provided that the following conditions are
6# met:
7#
8#     * Redistributions of source code must retain the above copyright
9#       notice, this list of conditions and the following disclaimer.
10#     * Redistributions in binary form must reproduce the above
11#       copyright notice, this list of conditions and the following
12#       disclaimer in the documentation and/or other materials provided
13#       with the distribution.
14#     * Neither the name of Google Inc. nor the names of its
15#       contributors may be used to endorse or promote products derived
16#       from this software without specific prior written permission.
17#
18# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29
30
31from collections import OrderedDict
32import itertools
33import json
34import multiprocessing
35import optparse
36import os
37from os.path import getmtime, isdir, join
38import platform
39import random
40import shlex
41import subprocess
42import sys
43import time
44
45from testrunner.local import execution
46from testrunner.local import progress
47from testrunner.local import testsuite
48from testrunner.local.variants import ALL_VARIANTS
49from testrunner.local import utils
50from testrunner.local import verbose
51from testrunner.network import network_execution
52from testrunner.objects import context
53
54
55# Base dir of the v8 checkout to be used as cwd.
56BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
57
58DEFAULT_OUT_GN = "out.gn"
59
60ARCH_GUESS = utils.DefaultArch()
61
62# Map of test name synonyms to lists of test suites. Should be ordered by
63# expected runtimes (suites with slow test cases first). These groups are
64# invoked in seperate steps on the bots.
65TEST_MAP = {
66  # This needs to stay in sync with test/bot_default.isolate.
67  "bot_default": [
68    "debugger",
69    "mjsunit",
70    "cctest",
71    "inspector",
72    "webkit",
73    "fuzzer",
74    "message",
75    "preparser",
76    "intl",
77    "unittests",
78  ],
79  # This needs to stay in sync with test/default.isolate.
80  "default": [
81    "debugger",
82    "mjsunit",
83    "cctest",
84    "inspector",
85    "fuzzer",
86    "message",
87    "preparser",
88    "intl",
89    "unittests",
90  ],
91  # This needs to stay in sync with test/optimize_for_size.isolate.
92  "optimize_for_size": [
93    "debugger",
94    "mjsunit",
95    "cctest",
96    "inspector",
97    "webkit",
98    "intl",
99  ],
100  "unittests": [
101    "unittests",
102  ],
103}
104
105TIMEOUT_DEFAULT = 60
106
107# Variants ordered by expected runtime (slowest first).
108VARIANTS = ["ignition_staging", "default", "turbofan"]
109
110MORE_VARIANTS = [
111  "stress",
112  "turbofan_opt",
113  "ignition",
114  "asm_wasm",
115  "wasm_traps",
116]
117
118EXHAUSTIVE_VARIANTS = MORE_VARIANTS + VARIANTS
119
120VARIANT_ALIASES = {
121  # The default for developer workstations.
122  "dev": VARIANTS,
123  # Additional variants, run on all bots.
124  "more": MORE_VARIANTS,
125  # Additional variants, run on a subset of bots.
126  "extra": ["nocrankshaft"],
127}
128
129DEBUG_FLAGS = ["--nohard-abort", "--nodead-code-elimination",
130               "--nofold-constants", "--enable-slow-asserts",
131               "--verify-heap"]
132RELEASE_FLAGS = ["--nohard-abort", "--nodead-code-elimination",
133                 "--nofold-constants"]
134
135MODES = {
136  "debug": {
137    "flags": DEBUG_FLAGS,
138    "timeout_scalefactor": 4,
139    "status_mode": "debug",
140    "execution_mode": "debug",
141    "output_folder": "debug",
142  },
143  "optdebug": {
144    "flags": DEBUG_FLAGS,
145    "timeout_scalefactor": 4,
146    "status_mode": "debug",
147    "execution_mode": "debug",
148    "output_folder": "optdebug",
149  },
150  "release": {
151    "flags": RELEASE_FLAGS,
152    "timeout_scalefactor": 1,
153    "status_mode": "release",
154    "execution_mode": "release",
155    "output_folder": "release",
156  },
157  # Normal trybot release configuration. There, dchecks are always on which
158  # implies debug is set. Hence, the status file needs to assume debug-like
159  # behavior/timeouts.
160  "tryrelease": {
161    "flags": RELEASE_FLAGS,
162    "timeout_scalefactor": 1,
163    "status_mode": "debug",
164    "execution_mode": "release",
165    "output_folder": "release",
166  },
167  # This mode requires v8 to be compiled with dchecks and slow dchecks.
168  "slowrelease": {
169    "flags": RELEASE_FLAGS + ["--enable-slow-asserts"],
170    "timeout_scalefactor": 2,
171    "status_mode": "debug",
172    "execution_mode": "release",
173    "output_folder": "release",
174  },
175}
176
177GC_STRESS_FLAGS = ["--gc-interval=500", "--stress-compaction",
178                   "--concurrent-recompilation-queue-length=64",
179                   "--concurrent-recompilation-delay=500",
180                   "--concurrent-recompilation"]
181
182SUPPORTED_ARCHS = ["android_arm",
183                   "android_arm64",
184                   "android_ia32",
185                   "android_x64",
186                   "arm",
187                   "ia32",
188                   "x87",
189                   "mips",
190                   "mipsel",
191                   "mips64",
192                   "mips64el",
193                   "s390",
194                   "s390x",
195                   "ppc",
196                   "ppc64",
197                   "x64",
198                   "x32",
199                   "arm64"]
200# Double the timeout for these:
201SLOW_ARCHS = ["android_arm",
202              "android_arm64",
203              "android_ia32",
204              "android_x64",
205              "arm",
206              "mips",
207              "mipsel",
208              "mips64",
209              "mips64el",
210              "s390",
211              "s390x",
212              "x87",
213              "arm64"]
214
215
216def BuildOptions():
217  result = optparse.OptionParser()
218  result.usage = '%prog [options] [tests]'
219  result.description = """TESTS: %s""" % (TEST_MAP["default"])
220  result.add_option("--arch",
221                    help=("The architecture to run tests for, "
222                          "'auto' or 'native' for auto-detect: %s" % SUPPORTED_ARCHS),
223                    default="ia32,x64,arm")
224  result.add_option("--arch-and-mode",
225                    help="Architecture and mode in the format 'arch.mode'",
226                    default=None)
227  result.add_option("--asan",
228                    help="Regard test expectations for ASAN",
229                    default=False, action="store_true")
230  result.add_option("--sancov-dir",
231                    help="Directory where to collect coverage data")
232  result.add_option("--cfi-vptr",
233                    help="Run tests with UBSAN cfi_vptr option.",
234                    default=False, action="store_true")
235  result.add_option("--buildbot",
236                    help="Adapt to path structure used on buildbots",
237                    default=False, action="store_true")
238  result.add_option("--dcheck-always-on",
239                    help="Indicates that V8 was compiled with DCHECKs enabled",
240                    default=False, action="store_true")
241  result.add_option("--novfp3",
242                    help="Indicates that V8 was compiled without VFP3 support",
243                    default=False, action="store_true")
244  result.add_option("--cat", help="Print the source of the tests",
245                    default=False, action="store_true")
246  result.add_option("--slow-tests",
247                    help="Regard slow tests (run|skip|dontcare)",
248                    default="dontcare")
249  result.add_option("--pass-fail-tests",
250                    help="Regard pass|fail tests (run|skip|dontcare)",
251                    default="dontcare")
252  result.add_option("--gc-stress",
253                    help="Switch on GC stress mode",
254                    default=False, action="store_true")
255  result.add_option("--gcov-coverage",
256                    help="Uses executables instrumented for gcov coverage",
257                    default=False, action="store_true")
258  result.add_option("--command-prefix",
259                    help="Prepended to each shell command used to run a test",
260                    default="")
261  result.add_option("--download-data", help="Download missing test suite data",
262                    default=False, action="store_true")
263  result.add_option("--download-data-only",
264                    help="Deprecated",
265                    default=False, action="store_true")
266  result.add_option("--enable-inspector",
267                    help="Indicates a build with inspector support",
268                    default=False, action="store_true")
269  result.add_option("--extra-flags",
270                    help="Additional flags to pass to each test command",
271                    default="")
272  result.add_option("--isolates", help="Whether to test isolates",
273                    default=False, action="store_true")
274  result.add_option("-j", help="The number of parallel tasks to run",
275                    default=0, type="int")
276  result.add_option("-m", "--mode",
277                    help="The test modes in which to run (comma-separated,"
278                    " uppercase for ninja and buildbot builds): %s" % MODES.keys(),
279                    default="release,debug")
280  result.add_option("--no-harness", "--noharness",
281                    help="Run without test harness of a given suite",
282                    default=False, action="store_true")
283  result.add_option("--no-i18n", "--noi18n",
284                    help="Skip internationalization tests",
285                    default=False, action="store_true")
286  result.add_option("--no-network", "--nonetwork",
287                    help="Don't distribute tests on the network",
288                    default=(utils.GuessOS() != "linux"),
289                    dest="no_network", action="store_true")
290  result.add_option("--no-presubmit", "--nopresubmit",
291                    help='Skip presubmit checks (deprecated)',
292                    default=False, dest="no_presubmit", action="store_true")
293  result.add_option("--no-snap", "--nosnap",
294                    help='Test a build compiled without snapshot.',
295                    default=False, dest="no_snap", action="store_true")
296  result.add_option("--no-sorting", "--nosorting",
297                    help="Don't sort tests according to duration of last run.",
298                    default=False, dest="no_sorting", action="store_true")
299  result.add_option("--no-variants", "--novariants",
300                    help="Don't run any testing variants",
301                    default=False, dest="no_variants", action="store_true")
302  result.add_option("--variants",
303                    help="Comma-separated list of testing variants;"
304                    " default: \"%s\"" % ",".join(VARIANTS))
305  result.add_option("--exhaustive-variants",
306                    default=False, action="store_true",
307                    help="Use exhaustive set of default variants:"
308                    " \"%s\"" % ",".join(EXHAUSTIVE_VARIANTS))
309  result.add_option("--outdir", help="Base directory with compile output",
310                    default="out")
311  result.add_option("--gn", help="Scan out.gn for the last built configuration",
312                    default=False, action="store_true")
313  result.add_option("--predictable",
314                    help="Compare output of several reruns of each test",
315                    default=False, action="store_true")
316  result.add_option("-p", "--progress",
317                    help=("The style of progress indicator"
318                          " (verbose, dots, color, mono)"),
319                    choices=progress.PROGRESS_INDICATORS.keys(), default="mono")
320  result.add_option("--quickcheck", default=False, action="store_true",
321                    help=("Quick check mode (skip slow tests)"))
322  result.add_option("--report", help="Print a summary of the tests to be run",
323                    default=False, action="store_true")
324  result.add_option("--json-test-results",
325                    help="Path to a file for storing json results.")
326  result.add_option("--rerun-failures-count",
327                    help=("Number of times to rerun each failing test case. "
328                          "Very slow tests will be rerun only once."),
329                    default=0, type="int")
330  result.add_option("--rerun-failures-max",
331                    help="Maximum number of failing test cases to rerun.",
332                    default=100, type="int")
333  result.add_option("--shard-count",
334                    help="Split testsuites into this number of shards",
335                    default=1, type="int")
336  result.add_option("--shard-run",
337                    help="Run this shard from the split up tests.",
338                    default=1, type="int")
339  result.add_option("--shell", help="DEPRECATED! use --shell-dir", default="")
340  result.add_option("--shell-dir", help="Directory containing executables",
341                    default="")
342  result.add_option("--dont-skip-slow-simulator-tests",
343                    help="Don't skip more slow tests when using a simulator.",
344                    default=False, action="store_true",
345                    dest="dont_skip_simulator_slow_tests")
346  result.add_option("--swarming",
347                    help="Indicates running test driver on swarming.",
348                    default=False, action="store_true")
349  result.add_option("--time", help="Print timing information after running",
350                    default=False, action="store_true")
351  result.add_option("-t", "--timeout", help="Timeout in seconds",
352                    default=TIMEOUT_DEFAULT, type="int")
353  result.add_option("--tsan",
354                    help="Regard test expectations for TSAN",
355                    default=False, action="store_true")
356  result.add_option("-v", "--verbose", help="Verbose output",
357                    default=False, action="store_true")
358  result.add_option("--valgrind", help="Run tests through valgrind",
359                    default=False, action="store_true")
360  result.add_option("--warn-unused", help="Report unused rules",
361                    default=False, action="store_true")
362  result.add_option("--junitout", help="File name of the JUnit output")
363  result.add_option("--junittestsuite",
364                    help="The testsuite name in the JUnit output file",
365                    default="v8tests")
366  result.add_option("--random-seed", default=0, dest="random_seed", type="int",
367                    help="Default seed for initializing random generator")
368  result.add_option("--random-seed-stress-count", default=1, type="int",
369                    dest="random_seed_stress_count",
370                    help="Number of runs with different random seeds")
371  result.add_option("--msan",
372                    help="Regard test expectations for MSAN",
373                    default=False, action="store_true")
374  return result
375
376
377def RandomSeed():
378  seed = 0
379  while not seed:
380    seed = random.SystemRandom().randint(-2147483648, 2147483647)
381  return seed
382
383
384def BuildbotToV8Mode(config):
385  """Convert buildbot build configs to configs understood by the v8 runner.
386
387  V8 configs are always lower case and without the additional _x64 suffix for
388  64 bit builds on windows with ninja.
389  """
390  mode = config[:-4] if config.endswith('_x64') else config
391  return mode.lower()
392
393def SetupEnvironment(options):
394  """Setup additional environment variables."""
395
396  # Many tests assume an English interface.
397  os.environ['LANG'] = 'en_US.UTF-8'
398
399  symbolizer = 'external_symbolizer_path=%s' % (
400      os.path.join(
401          BASE_DIR, 'third_party', 'llvm-build', 'Release+Asserts', 'bin',
402          'llvm-symbolizer',
403      )
404  )
405
406  if options.asan:
407    asan_options = [symbolizer]
408    if not utils.GuessOS() == 'macos':
409      # LSAN is not available on mac.
410      asan_options.append('detect_leaks=1')
411      os.environ['LSAN_OPTIONS'] = ":".join([
412        'suppressions=%s' % os.path.join(
413            BASE_DIR, 'tools', 'memory', 'lsan', 'suppressions.txt'),
414      ])
415    os.environ['ASAN_OPTIONS'] = ":".join(asan_options)
416
417  if options.sancov_dir:
418    assert os.path.exists(options.sancov_dir)
419    os.environ['ASAN_OPTIONS'] = ":".join([
420      'coverage=1',
421      'coverage_dir=%s' % options.sancov_dir,
422      symbolizer,
423    ])
424
425  if options.cfi_vptr:
426    os.environ['UBSAN_OPTIONS'] = ":".join([
427      'print_stacktrace=1',
428      'print_summary=1',
429      'symbolize=1',
430      symbolizer,
431    ])
432
433  if options.msan:
434    os.environ['MSAN_OPTIONS'] = symbolizer
435
436  if options.tsan:
437    suppressions_file = os.path.join(
438        BASE_DIR, 'tools', 'sanitizers', 'tsan_suppressions.txt')
439    os.environ['TSAN_OPTIONS'] = " ".join([
440      symbolizer,
441      'suppressions=%s' % suppressions_file,
442      'exit_code=0',
443      'report_thread_leaks=0',
444      'history_size=7',
445      'report_destroy_locked=0',
446    ])
447
448def ProcessOptions(options):
449  global VARIANTS
450
451  # First try to auto-detect configurations based on the build if GN was
452  # used. This can't be overridden by cmd-line arguments.
453  options.auto_detect = False
454  if options.gn:
455    gn_out_dir = os.path.join(BASE_DIR, DEFAULT_OUT_GN)
456    latest_timestamp = -1
457    latest_config = None
458    for gn_config in os.listdir(gn_out_dir):
459      gn_config_dir = os.path.join(gn_out_dir, gn_config)
460      if not isdir(gn_config_dir):
461        continue
462      if os.path.getmtime(gn_config_dir) > latest_timestamp:
463        latest_timestamp = os.path.getmtime(gn_config_dir)
464        latest_config = gn_config
465    if latest_config:
466      print(">>> Latest GN build found is %s" % latest_config)
467      options.outdir = os.path.join(DEFAULT_OUT_GN, latest_config)
468
469  if options.buildbot:
470    build_config_path = os.path.join(
471        BASE_DIR, options.outdir, options.mode, "v8_build_config.json")
472  else:
473    build_config_path = os.path.join(
474        BASE_DIR, options.outdir, "v8_build_config.json")
475
476  if os.path.exists(build_config_path):
477    try:
478      with open(build_config_path) as f:
479        build_config = json.load(f)
480    except Exception:
481      print ("%s exists but contains invalid json. Is your build up-to-date?" %
482             build_config_path)
483      return False
484    options.auto_detect = True
485
486    # In auto-detect mode the outdir is always where we found the build config.
487    # This ensures that we'll also take the build products from there.
488    options.outdir = os.path.dirname(build_config_path)
489
490    options.arch_and_mode = None
491    options.arch = build_config["v8_target_cpu"]
492    if options.arch == 'x86':
493      # TODO(machenbach): Transform all to x86 eventually.
494      options.arch = 'ia32'
495    options.asan = build_config["is_asan"]
496    options.dcheck_always_on = build_config["dcheck_always_on"]
497    options.enable_inspector = build_config["v8_enable_inspector"]
498    options.mode = 'debug' if build_config["is_debug"] else 'release'
499    options.msan = build_config["is_msan"]
500    options.no_i18n = not build_config["v8_enable_i18n_support"]
501    options.no_snap = not build_config["v8_use_snapshot"]
502    options.tsan = build_config["is_tsan"]
503
504  # Architecture and mode related stuff.
505  if options.arch_and_mode:
506    options.arch_and_mode = [arch_and_mode.split(".")
507        for arch_and_mode in options.arch_and_mode.split(",")]
508    options.arch = ",".join([tokens[0] for tokens in options.arch_and_mode])
509    options.mode = ",".join([tokens[1] for tokens in options.arch_and_mode])
510  options.mode = options.mode.split(",")
511  for mode in options.mode:
512    if not BuildbotToV8Mode(mode) in MODES:
513      print "Unknown mode %s" % mode
514      return False
515  if options.arch in ["auto", "native"]:
516    options.arch = ARCH_GUESS
517  options.arch = options.arch.split(",")
518  for arch in options.arch:
519    if not arch in SUPPORTED_ARCHS:
520      print "Unknown architecture %s" % arch
521      return False
522
523  # Store the final configuration in arch_and_mode list. Don't overwrite
524  # predefined arch_and_mode since it is more expressive than arch and mode.
525  if not options.arch_and_mode:
526    options.arch_and_mode = itertools.product(options.arch, options.mode)
527
528  # Special processing of other options, sorted alphabetically.
529
530  if options.buildbot:
531    options.no_network = True
532  if options.command_prefix:
533    print("Specifying --command-prefix disables network distribution, "
534          "running tests locally.")
535    options.no_network = True
536  options.command_prefix = shlex.split(options.command_prefix)
537  options.extra_flags = shlex.split(options.extra_flags)
538
539  if options.gc_stress:
540    options.extra_flags += GC_STRESS_FLAGS
541
542  if options.asan:
543    options.extra_flags.append("--invoke-weak-callbacks")
544    options.extra_flags.append("--omit-quit")
545
546  if options.novfp3:
547    options.extra_flags.append("--noenable-vfp3")
548
549  if options.exhaustive_variants:
550    # This is used on many bots. It includes a larger set of default variants.
551    # Other options for manipulating variants still apply afterwards.
552    VARIANTS = EXHAUSTIVE_VARIANTS
553
554  # TODO(machenbach): Figure out how to test a bigger subset of variants on
555  # msan and tsan.
556  if options.msan:
557    VARIANTS = ["default"]
558
559  if options.tsan:
560    VARIANTS = ["default"]
561
562  if options.j == 0:
563    options.j = multiprocessing.cpu_count()
564
565  if options.random_seed_stress_count <= 1 and options.random_seed == 0:
566    options.random_seed = RandomSeed()
567
568  def excl(*args):
569    """Returns true if zero or one of multiple arguments are true."""
570    return reduce(lambda x, y: x + y, args) <= 1
571
572  if not excl(options.no_variants, bool(options.variants)):
573    print("Use only one of --no-variants or --variants.")
574    return False
575  if options.quickcheck:
576    VARIANTS = ["default", "stress"]
577    options.slow_tests = "skip"
578    options.pass_fail_tests = "skip"
579  if options.no_variants:
580    VARIANTS = ["default"]
581  if options.variants:
582    VARIANTS = options.variants.split(",")
583
584    # Resolve variant aliases.
585    VARIANTS = reduce(
586        list.__add__,
587        (VARIANT_ALIASES.get(v, [v]) for v in VARIANTS),
588        [],
589    )
590
591    if not set(VARIANTS).issubset(ALL_VARIANTS):
592      print "All variants must be in %s" % str(ALL_VARIANTS)
593      return False
594  if options.predictable:
595    VARIANTS = ["default"]
596    options.extra_flags.append("--predictable")
597    options.extra_flags.append("--verify_predictable")
598    options.extra_flags.append("--no-inline-new")
599
600  # Dedupe.
601  VARIANTS = list(set(VARIANTS))
602
603  if not options.shell_dir:
604    if options.shell:
605      print "Warning: --shell is deprecated, use --shell-dir instead."
606      options.shell_dir = os.path.dirname(options.shell)
607  if options.valgrind:
608    run_valgrind = os.path.join("tools", "run-valgrind.py")
609    # This is OK for distributed running, so we don't need to set no_network.
610    options.command_prefix = (["python", "-u", run_valgrind] +
611                              options.command_prefix)
612  def CheckTestMode(name, option):
613    if not option in ["run", "skip", "dontcare"]:
614      print "Unknown %s mode %s" % (name, option)
615      return False
616    return True
617  if not CheckTestMode("slow test", options.slow_tests):
618    return False
619  if not CheckTestMode("pass|fail test", options.pass_fail_tests):
620    return False
621  if options.no_i18n:
622    TEST_MAP["bot_default"].remove("intl")
623    TEST_MAP["default"].remove("intl")
624  if not options.enable_inspector:
625    TEST_MAP["default"].remove("inspector")
626    TEST_MAP["bot_default"].remove("inspector")
627    TEST_MAP["optimize_for_size"].remove("inspector")
628    TEST_MAP["default"].remove("debugger")
629    TEST_MAP["bot_default"].remove("debugger")
630    TEST_MAP["optimize_for_size"].remove("debugger")
631  return True
632
633
634def ShardTests(tests, options):
635  # Read gtest shard configuration from environment (e.g. set by swarming).
636  # If none is present, use values passed on the command line.
637  shard_count = int(os.environ.get('GTEST_TOTAL_SHARDS', options.shard_count))
638  shard_run = os.environ.get('GTEST_SHARD_INDEX')
639  if shard_run is not None:
640    # The v8 shard_run starts at 1, while GTEST_SHARD_INDEX starts at 0.
641    shard_run = int(shard_run) + 1
642  else:
643    shard_run = options.shard_run
644
645  if options.shard_count > 1:
646    # Log if a value was passed on the cmd line and it differs from the
647    # environment variables.
648    if options.shard_count != shard_count:
649      print("shard_count from cmd line differs from environment variable "
650            "GTEST_TOTAL_SHARDS")
651    if options.shard_run > 1 and options.shard_run != shard_run:
652      print("shard_run from cmd line differs from environment variable "
653            "GTEST_SHARD_INDEX")
654
655  if shard_count < 2:
656    return tests
657  if shard_run < 1 or shard_run > shard_count:
658    print "shard-run not a valid number, should be in [1:shard-count]"
659    print "defaulting back to running all tests"
660    return tests
661  count = 0
662  shard = []
663  for test in tests:
664    if count % shard_count == shard_run - 1:
665      shard.append(test)
666    count += 1
667  return shard
668
669
670def Main():
671  # Use the v8 root as cwd as some test cases use "load" with relative paths.
672  os.chdir(BASE_DIR)
673
674  parser = BuildOptions()
675  (options, args) = parser.parse_args()
676  if not ProcessOptions(options):
677    parser.print_help()
678    return 1
679  SetupEnvironment(options)
680
681  if options.swarming:
682    # Swarming doesn't print how isolated commands are called. Lets make this
683    # less cryptic by printing it ourselves.
684    print ' '.join(sys.argv)
685
686  exit_code = 0
687
688  suite_paths = utils.GetSuitePaths(join(BASE_DIR, "test"))
689
690  # Use default tests if no test configuration was provided at the cmd line.
691  if len(args) == 0:
692    args = ["default"]
693
694  # Expand arguments with grouped tests. The args should reflect the list of
695  # suites as otherwise filters would break.
696  def ExpandTestGroups(name):
697    if name in TEST_MAP:
698      return [suite for suite in TEST_MAP[name]]
699    else:
700      return [name]
701  args = reduce(lambda x, y: x + y,
702         [ExpandTestGroups(arg) for arg in args],
703         [])
704
705  args_suites = OrderedDict() # Used as set
706  for arg in args:
707    args_suites[arg.split('/')[0]] = True
708  suite_paths = [ s for s in args_suites if s in suite_paths ]
709
710  suites = []
711  for root in suite_paths:
712    suite = testsuite.TestSuite.LoadTestSuite(
713        os.path.join(BASE_DIR, "test", root))
714    if suite:
715      suites.append(suite)
716
717  if options.download_data or options.download_data_only:
718    for s in suites:
719      s.DownloadData()
720
721  if options.download_data_only:
722    return exit_code
723
724  for s in suites:
725    s.PrepareSources()
726
727  for (arch, mode) in options.arch_and_mode:
728    try:
729      code = Execute(arch, mode, args, options, suites)
730    except KeyboardInterrupt:
731      return 2
732    exit_code = exit_code or code
733  return exit_code
734
735
736def Execute(arch, mode, args, options, suites):
737  print(">>> Running tests for %s.%s" % (arch, mode))
738
739  shell_dir = options.shell_dir
740  if not shell_dir:
741    if options.auto_detect:
742      # If an output dir with a build was passed, test directly in that
743      # directory.
744      shell_dir = os.path.join(BASE_DIR, options.outdir)
745    elif options.buildbot:
746      # TODO(machenbach): Get rid of different output folder location on
747      # buildbot. Currently this is capitalized Release and Debug.
748      shell_dir = os.path.join(BASE_DIR, options.outdir, mode)
749      mode = BuildbotToV8Mode(mode)
750    else:
751      shell_dir = os.path.join(
752          BASE_DIR,
753          options.outdir,
754          "%s.%s" % (arch, MODES[mode]["output_folder"]),
755      )
756  if not os.path.exists(shell_dir):
757      raise Exception('Could not find shell_dir: "%s"' % shell_dir)
758
759  # Populate context object.
760  mode_flags = MODES[mode]["flags"]
761
762  # Simulators are slow, therefore allow a longer timeout.
763  if arch in SLOW_ARCHS:
764    options.timeout *= 2
765
766  options.timeout *= MODES[mode]["timeout_scalefactor"]
767
768  if options.predictable:
769    # Predictable mode is slower.
770    options.timeout *= 2
771
772  ctx = context.Context(arch, MODES[mode]["execution_mode"], shell_dir,
773                        mode_flags, options.verbose,
774                        options.timeout,
775                        options.isolates,
776                        options.command_prefix,
777                        options.extra_flags,
778                        options.no_i18n,
779                        options.random_seed,
780                        options.no_sorting,
781                        options.rerun_failures_count,
782                        options.rerun_failures_max,
783                        options.predictable,
784                        options.no_harness,
785                        use_perf_data=not options.swarming,
786                        sancov_dir=options.sancov_dir)
787
788  # TODO(all): Combine "simulator" and "simulator_run".
789  # TODO(machenbach): In GN we can derive simulator run from
790  # target_arch != v8_target_arch in the dumped build config.
791  simulator_run = not options.dont_skip_simulator_slow_tests and \
792      arch in ['arm64', 'arm', 'mipsel', 'mips', 'mips64', 'mips64el', \
793               'ppc', 'ppc64'] and \
794      ARCH_GUESS and arch != ARCH_GUESS
795  # Find available test suites and read test cases from them.
796  variables = {
797    "arch": arch,
798    "asan": options.asan,
799    "deopt_fuzzer": False,
800    "gc_stress": options.gc_stress,
801    "gcov_coverage": options.gcov_coverage,
802    "isolates": options.isolates,
803    "mode": MODES[mode]["status_mode"],
804    "no_i18n": options.no_i18n,
805    "no_snap": options.no_snap,
806    "simulator_run": simulator_run,
807    "simulator": utils.UseSimulator(arch),
808    "system": utils.GuessOS(),
809    "tsan": options.tsan,
810    "msan": options.msan,
811    "dcheck_always_on": options.dcheck_always_on,
812    "novfp3": options.novfp3,
813    "predictable": options.predictable,
814    "byteorder": sys.byteorder,
815  }
816  all_tests = []
817  num_tests = 0
818  for s in suites:
819    s.ReadStatusFile(variables)
820    s.ReadTestCases(ctx)
821    if len(args) > 0:
822      s.FilterTestCasesByArgs(args)
823    all_tests += s.tests
824
825    # First filtering by status applying the generic rules (independent of
826    # variants).
827    s.FilterTestCasesByStatus(options.warn_unused, options.slow_tests,
828                              options.pass_fail_tests)
829
830    if options.cat:
831      verbose.PrintTestSource(s.tests)
832      continue
833    variant_gen = s.CreateVariantGenerator(VARIANTS)
834    variant_tests = [ t.CopyAddingFlags(v, flags)
835                      for t in s.tests
836                      for v in variant_gen.FilterVariantsByTest(t)
837                      for flags in variant_gen.GetFlagSets(t, v) ]
838
839    if options.random_seed_stress_count > 1:
840      # Duplicate test for random seed stress mode.
841      def iter_seed_flags():
842        for i in range(0, options.random_seed_stress_count):
843          # Use given random seed for all runs (set by default in execution.py)
844          # or a new random seed if none is specified.
845          if options.random_seed:
846            yield []
847          else:
848            yield ["--random-seed=%d" % RandomSeed()]
849      s.tests = [
850        t.CopyAddingFlags(t.variant, flags)
851        for t in variant_tests
852        for flags in iter_seed_flags()
853      ]
854    else:
855      s.tests = variant_tests
856
857    # Second filtering by status applying the variant-dependent rules.
858    s.FilterTestCasesByStatus(options.warn_unused, options.slow_tests,
859                              options.pass_fail_tests, variants=True)
860
861    s.tests = ShardTests(s.tests, options)
862    num_tests += len(s.tests)
863
864  if options.cat:
865    return 0  # We're done here.
866
867  if options.report:
868    verbose.PrintReport(all_tests)
869
870  # Run the tests, either locally or distributed on the network.
871  start_time = time.time()
872  progress_indicator = progress.IndicatorNotifier()
873  progress_indicator.Register(progress.PROGRESS_INDICATORS[options.progress]())
874  if options.junitout:
875    progress_indicator.Register(progress.JUnitTestProgressIndicator(
876        options.junitout, options.junittestsuite))
877  if options.json_test_results:
878    progress_indicator.Register(progress.JsonTestProgressIndicator(
879        options.json_test_results, arch, MODES[mode]["execution_mode"],
880        ctx.random_seed))
881
882  run_networked = not options.no_network
883  if not run_networked:
884    if options.verbose:
885      print("Network distribution disabled, running tests locally.")
886  elif utils.GuessOS() != "linux":
887    print("Network distribution is only supported on Linux, sorry!")
888    run_networked = False
889  peers = []
890  if run_networked:
891    peers = network_execution.GetPeers()
892    if not peers:
893      print("No connection to distribution server; running tests locally.")
894      run_networked = False
895    elif len(peers) == 1:
896      print("No other peers on the network; running tests locally.")
897      run_networked = False
898    elif num_tests <= 100:
899      print("Less than 100 tests, running them locally.")
900      run_networked = False
901
902  if run_networked:
903    runner = network_execution.NetworkedRunner(suites, progress_indicator,
904                                               ctx, peers, BASE_DIR)
905  else:
906    runner = execution.Runner(suites, progress_indicator, ctx)
907
908  exit_code = runner.Run(options.j)
909  overall_duration = time.time() - start_time
910
911  if options.time:
912    verbose.PrintTestDurations(suites, overall_duration)
913
914  if num_tests == 0:
915    print("Warning: no tests were run!")
916
917  if exit_code == 1 and options.json_test_results:
918    print("Force exit code 0 after failures. Json test results file generated "
919          "with failure information.")
920    exit_code = 0
921
922  if options.sancov_dir:
923    # If tests ran with sanitizer coverage, merge coverage files in the end.
924    try:
925      print "Merging sancov files."
926      subprocess.check_call([
927        sys.executable,
928        join(BASE_DIR, "tools", "sanitizers", "sancov_merger.py"),
929        "--coverage-dir=%s" % options.sancov_dir])
930    except:
931      print >> sys.stderr, "Error: Merging sancov files failed."
932      exit_code = 1
933
934  return exit_code
935
936
937if __name__ == "__main__":
938  sys.exit(Main())
939