1#!/usr/bin/env python 2# 3# Copyright 2012 the V8 project authors. All rights reserved. 4# Redistribution and use in source and binary forms, with or without 5# modification, are permitted provided that the following conditions are 6# met: 7# 8# * Redistributions of source code must retain the above copyright 9# notice, this list of conditions and the following disclaimer. 10# * Redistributions in binary form must reproduce the above 11# copyright notice, this list of conditions and the following 12# disclaimer in the documentation and/or other materials provided 13# with the distribution. 14# * Neither the name of Google Inc. nor the names of its 15# contributors may be used to endorse or promote products derived 16# from this software without specific prior written permission. 17# 18# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 19# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 20# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 21# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 22# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 23# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 24# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 25# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 26# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 27# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 28# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 30 31from collections import OrderedDict 32import itertools 33import multiprocessing 34import optparse 35import os 36from os.path import join 37import platform 38import random 39import shlex 40import subprocess 41import sys 42import time 43 44from testrunner.local import execution 45from testrunner.local import progress 46from testrunner.local import testsuite 47from testrunner.local.testsuite import ALL_VARIANTS 48from testrunner.local import utils 49from testrunner.local import verbose 50from testrunner.network import network_execution 51from testrunner.objects import context 52 53 54# Base dir of the v8 checkout to be used as cwd. 55BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) 56 57ARCH_GUESS = utils.DefaultArch() 58 59# Map of test name synonyms to lists of test suites. Should be ordered by 60# expected runtimes (suites with slow test cases first). These groups are 61# invoked in seperate steps on the bots. 62TEST_MAP = { 63 "bot_default": [ 64 "mjsunit", 65 "cctest", 66 "webkit", 67 "message", 68 "preparser", 69 "intl", 70 "unittests", 71 ], 72 "default": [ 73 "mjsunit", 74 "cctest", 75 "message", 76 "preparser", 77 "intl", 78 "unittests", 79 ], 80 "ignition": [ 81 "mjsunit", 82 "cctest", 83 ], 84 "optimize_for_size": [ 85 "mjsunit", 86 "cctest", 87 "webkit", 88 "intl", 89 ], 90 "unittests": [ 91 "unittests", 92 ], 93} 94 95TIMEOUT_DEFAULT = 60 96 97VARIANTS = ["default", "stress", "turbofan"] 98 99EXHAUSTIVE_VARIANTS = VARIANTS + [ 100 "nocrankshaft", 101 "turbofan_opt", 102] 103 104DEBUG_FLAGS = ["--nohard-abort", "--nodead-code-elimination", 105 "--nofold-constants", "--enable-slow-asserts", 106 "--debug-code", "--verify-heap"] 107RELEASE_FLAGS = ["--nohard-abort", "--nodead-code-elimination", 108 "--nofold-constants"] 109 110MODES = { 111 "debug": { 112 "flags": DEBUG_FLAGS, 113 "timeout_scalefactor": 4, 114 "status_mode": "debug", 115 "execution_mode": "debug", 116 "output_folder": "debug", 117 }, 118 "optdebug": { 119 "flags": DEBUG_FLAGS, 120 "timeout_scalefactor": 4, 121 "status_mode": "debug", 122 "execution_mode": "debug", 123 "output_folder": "optdebug", 124 }, 125 "release": { 126 "flags": RELEASE_FLAGS, 127 "timeout_scalefactor": 1, 128 "status_mode": "release", 129 "execution_mode": "release", 130 "output_folder": "release", 131 }, 132 # Normal trybot release configuration. There, dchecks are always on which 133 # implies debug is set. Hence, the status file needs to assume debug-like 134 # behavior/timeouts. 135 "tryrelease": { 136 "flags": RELEASE_FLAGS, 137 "timeout_scalefactor": 1, 138 "status_mode": "debug", 139 "execution_mode": "release", 140 "output_folder": "release", 141 }, 142 # This mode requires v8 to be compiled with dchecks and slow dchecks. 143 "slowrelease": { 144 "flags": RELEASE_FLAGS + ["--enable-slow-asserts"], 145 "timeout_scalefactor": 2, 146 "status_mode": "debug", 147 "execution_mode": "release", 148 "output_folder": "release", 149 }, 150} 151 152GC_STRESS_FLAGS = ["--gc-interval=500", "--stress-compaction", 153 "--concurrent-recompilation-queue-length=64", 154 "--concurrent-recompilation-delay=500", 155 "--concurrent-recompilation"] 156 157SUPPORTED_ARCHS = ["android_arm", 158 "android_arm64", 159 "android_ia32", 160 "android_x64", 161 "arm", 162 "ia32", 163 "x87", 164 "mips", 165 "mipsel", 166 "mips64", 167 "mips64el", 168 "nacl_ia32", 169 "nacl_x64", 170 "ppc", 171 "ppc64", 172 "x64", 173 "x32", 174 "arm64"] 175# Double the timeout for these: 176SLOW_ARCHS = ["android_arm", 177 "android_arm64", 178 "android_ia32", 179 "android_x64", 180 "arm", 181 "mips", 182 "mipsel", 183 "mips64", 184 "mips64el", 185 "nacl_ia32", 186 "nacl_x64", 187 "x87", 188 "arm64"] 189 190 191def BuildOptions(): 192 result = optparse.OptionParser() 193 result.usage = '%prog [options] [tests]' 194 result.description = """TESTS: %s""" % (TEST_MAP["default"]) 195 result.add_option("--arch", 196 help=("The architecture to run tests for, " 197 "'auto' or 'native' for auto-detect: %s" % SUPPORTED_ARCHS), 198 default="ia32,x64,arm") 199 result.add_option("--arch-and-mode", 200 help="Architecture and mode in the format 'arch.mode'", 201 default=None) 202 result.add_option("--asan", 203 help="Regard test expectations for ASAN", 204 default=False, action="store_true") 205 result.add_option("--cfi-vptr", 206 help="Run tests with UBSAN cfi_vptr option.", 207 default=False, action="store_true") 208 result.add_option("--buildbot", 209 help="Adapt to path structure used on buildbots", 210 default=False, action="store_true") 211 result.add_option("--dcheck-always-on", 212 help="Indicates that V8 was compiled with DCHECKs enabled", 213 default=False, action="store_true") 214 result.add_option("--novfp3", 215 help="Indicates that V8 was compiled without VFP3 support", 216 default=False, action="store_true") 217 result.add_option("--cat", help="Print the source of the tests", 218 default=False, action="store_true") 219 result.add_option("--flaky-tests", 220 help="Regard tests marked as flaky (run|skip|dontcare)", 221 default="dontcare") 222 result.add_option("--slow-tests", 223 help="Regard slow tests (run|skip|dontcare)", 224 default="dontcare") 225 result.add_option("--pass-fail-tests", 226 help="Regard pass|fail tests (run|skip|dontcare)", 227 default="dontcare") 228 result.add_option("--gc-stress", 229 help="Switch on GC stress mode", 230 default=False, action="store_true") 231 result.add_option("--gcov-coverage", 232 help="Uses executables instrumented for gcov coverage", 233 default=False, action="store_true") 234 result.add_option("--command-prefix", 235 help="Prepended to each shell command used to run a test", 236 default="") 237 result.add_option("--download-data", help="Download missing test suite data", 238 default=False, action="store_true") 239 result.add_option("--download-data-only", 240 help="Download missing test suite data and exit", 241 default=False, action="store_true") 242 result.add_option("--extra-flags", 243 help="Additional flags to pass to each test command", 244 default="") 245 result.add_option("--ignition", help="Skip tests which don't run in ignition", 246 default=False, action="store_true") 247 result.add_option("--isolates", help="Whether to test isolates", 248 default=False, action="store_true") 249 result.add_option("-j", help="The number of parallel tasks to run", 250 default=0, type="int") 251 result.add_option("-m", "--mode", 252 help="The test modes in which to run (comma-separated," 253 " uppercase for ninja and buildbot builds): %s" % MODES.keys(), 254 default="release,debug") 255 result.add_option("--no-harness", "--noharness", 256 help="Run without test harness of a given suite", 257 default=False, action="store_true") 258 result.add_option("--no-i18n", "--noi18n", 259 help="Skip internationalization tests", 260 default=False, action="store_true") 261 result.add_option("--no-network", "--nonetwork", 262 help="Don't distribute tests on the network", 263 default=(utils.GuessOS() != "linux"), 264 dest="no_network", action="store_true") 265 result.add_option("--no-presubmit", "--nopresubmit", 266 help='Skip presubmit checks', 267 default=False, dest="no_presubmit", action="store_true") 268 result.add_option("--no-snap", "--nosnap", 269 help='Test a build compiled without snapshot.', 270 default=False, dest="no_snap", action="store_true") 271 result.add_option("--no-sorting", "--nosorting", 272 help="Don't sort tests according to duration of last run.", 273 default=False, dest="no_sorting", action="store_true") 274 result.add_option("--no-stress", "--nostress", 275 help="Don't run crankshaft --always-opt --stress-op test", 276 default=False, dest="no_stress", action="store_true") 277 result.add_option("--no-variants", "--novariants", 278 help="Don't run any testing variants", 279 default=False, dest="no_variants", action="store_true") 280 result.add_option("--variants", 281 help="Comma-separated list of testing variants: %s" % VARIANTS) 282 result.add_option("--exhaustive-variants", 283 default=False, action="store_true", 284 help="Use exhaustive set of default variants.") 285 result.add_option("--outdir", help="Base directory with compile output", 286 default="out") 287 result.add_option("--predictable", 288 help="Compare output of several reruns of each test", 289 default=False, action="store_true") 290 result.add_option("-p", "--progress", 291 help=("The style of progress indicator" 292 " (verbose, dots, color, mono)"), 293 choices=progress.PROGRESS_INDICATORS.keys(), default="mono") 294 result.add_option("--quickcheck", default=False, action="store_true", 295 help=("Quick check mode (skip slow/flaky tests)")) 296 result.add_option("--report", help="Print a summary of the tests to be run", 297 default=False, action="store_true") 298 result.add_option("--json-test-results", 299 help="Path to a file for storing json results.") 300 result.add_option("--rerun-failures-count", 301 help=("Number of times to rerun each failing test case. " 302 "Very slow tests will be rerun only once."), 303 default=0, type="int") 304 result.add_option("--rerun-failures-max", 305 help="Maximum number of failing test cases to rerun.", 306 default=100, type="int") 307 result.add_option("--shard-count", 308 help="Split testsuites into this number of shards", 309 default=1, type="int") 310 result.add_option("--shard-run", 311 help="Run this shard from the split up tests.", 312 default=1, type="int") 313 result.add_option("--shell", help="DEPRECATED! use --shell-dir", default="") 314 result.add_option("--shell-dir", help="Directory containing executables", 315 default="") 316 result.add_option("--dont-skip-slow-simulator-tests", 317 help="Don't skip more slow tests when using a simulator.", 318 default=False, action="store_true", 319 dest="dont_skip_simulator_slow_tests") 320 result.add_option("--stress-only", 321 help="Only run tests with --always-opt --stress-opt", 322 default=False, action="store_true") 323 result.add_option("--swarming", 324 help="Indicates running test driver on swarming.", 325 default=False, action="store_true") 326 result.add_option("--time", help="Print timing information after running", 327 default=False, action="store_true") 328 result.add_option("-t", "--timeout", help="Timeout in seconds", 329 default= -1, type="int") 330 result.add_option("--tsan", 331 help="Regard test expectations for TSAN", 332 default=False, action="store_true") 333 result.add_option("-v", "--verbose", help="Verbose output", 334 default=False, action="store_true") 335 result.add_option("--valgrind", help="Run tests through valgrind", 336 default=False, action="store_true") 337 result.add_option("--warn-unused", help="Report unused rules", 338 default=False, action="store_true") 339 result.add_option("--junitout", help="File name of the JUnit output") 340 result.add_option("--junittestsuite", 341 help="The testsuite name in the JUnit output file", 342 default="v8tests") 343 result.add_option("--random-seed", default=0, dest="random_seed", type="int", 344 help="Default seed for initializing random generator") 345 result.add_option("--random-seed-stress-count", default=1, type="int", 346 dest="random_seed_stress_count", 347 help="Number of runs with different random seeds") 348 result.add_option("--msan", 349 help="Regard test expectations for MSAN", 350 default=False, action="store_true") 351 return result 352 353 354def RandomSeed(): 355 seed = 0 356 while not seed: 357 seed = random.SystemRandom().randint(-2147483648, 2147483647) 358 return seed 359 360 361def BuildbotToV8Mode(config): 362 """Convert buildbot build configs to configs understood by the v8 runner. 363 364 V8 configs are always lower case and without the additional _x64 suffix for 365 64 bit builds on windows with ninja. 366 """ 367 mode = config[:-4] if config.endswith('_x64') else config 368 return mode.lower() 369 370def SetupEnvironment(options): 371 """Setup additional environment variables.""" 372 symbolizer = 'external_symbolizer_path=%s' % ( 373 os.path.join( 374 BASE_DIR, 'third_party', 'llvm-build', 'Release+Asserts', 'bin', 375 'llvm-symbolizer', 376 ) 377 ) 378 379 if options.asan: 380 os.environ['ASAN_OPTIONS'] = symbolizer 381 382 if options.cfi_vptr: 383 os.environ['UBSAN_OPTIONS'] = ":".join([ 384 'print_stacktrace=1', 385 'print_summary=1', 386 'symbolize=1', 387 symbolizer, 388 ]) 389 390 if options.msan: 391 os.environ['MSAN_OPTIONS'] = symbolizer 392 393 if options.tsan: 394 suppressions_file = os.path.join( 395 BASE_DIR, 'tools', 'sanitizers', 'tsan_suppressions.txt') 396 os.environ['TSAN_OPTIONS'] = " ".join([ 397 symbolizer, 398 'suppressions=%s' % suppressions_file, 399 'exit_code=0', 400 'report_thread_leaks=0', 401 'history_size=7', 402 'report_destroy_locked=0', 403 ]) 404 405def ProcessOptions(options): 406 global ALL_VARIANTS 407 global EXHAUSTIVE_VARIANTS 408 global VARIANTS 409 410 # Architecture and mode related stuff. 411 if options.arch_and_mode: 412 options.arch_and_mode = [arch_and_mode.split(".") 413 for arch_and_mode in options.arch_and_mode.split(",")] 414 options.arch = ",".join([tokens[0] for tokens in options.arch_and_mode]) 415 options.mode = ",".join([tokens[1] for tokens in options.arch_and_mode]) 416 options.mode = options.mode.split(",") 417 for mode in options.mode: 418 if not BuildbotToV8Mode(mode) in MODES: 419 print "Unknown mode %s" % mode 420 return False 421 if options.arch in ["auto", "native"]: 422 options.arch = ARCH_GUESS 423 options.arch = options.arch.split(",") 424 for arch in options.arch: 425 if not arch in SUPPORTED_ARCHS: 426 print "Unknown architecture %s" % arch 427 return False 428 429 # Store the final configuration in arch_and_mode list. Don't overwrite 430 # predefined arch_and_mode since it is more expressive than arch and mode. 431 if not options.arch_and_mode: 432 options.arch_and_mode = itertools.product(options.arch, options.mode) 433 434 # Special processing of other options, sorted alphabetically. 435 436 if options.buildbot: 437 # Buildbots run presubmit tests as a separate step. 438 options.no_presubmit = True 439 options.no_network = True 440 if options.download_data_only: 441 options.no_presubmit = True 442 if options.command_prefix: 443 print("Specifying --command-prefix disables network distribution, " 444 "running tests locally.") 445 options.no_network = True 446 options.command_prefix = shlex.split(options.command_prefix) 447 options.extra_flags = shlex.split(options.extra_flags) 448 449 if options.gc_stress: 450 options.extra_flags += GC_STRESS_FLAGS 451 452 if options.asan: 453 options.extra_flags.append("--invoke-weak-callbacks") 454 options.extra_flags.append("--omit-quit") 455 456 if options.novfp3: 457 options.extra_flags.append("--noenable-vfp3") 458 459 if options.exhaustive_variants: 460 # This is used on many bots. It includes a larger set of default variants. 461 # Other options for manipulating variants still apply afterwards. 462 VARIANTS = EXHAUSTIVE_VARIANTS 463 464 if options.msan: 465 VARIANTS = ["default"] 466 467 if options.tsan: 468 VARIANTS = ["default"] 469 470 if options.j == 0: 471 options.j = multiprocessing.cpu_count() 472 473 if options.random_seed_stress_count <= 1 and options.random_seed == 0: 474 options.random_seed = RandomSeed() 475 476 def excl(*args): 477 """Returns true if zero or one of multiple arguments are true.""" 478 return reduce(lambda x, y: x + y, args) <= 1 479 480 if not excl(options.no_stress, options.stress_only, options.no_variants, 481 bool(options.variants)): 482 print("Use only one of --no-stress, --stress-only, --no-variants, " 483 "or --variants.") 484 return False 485 if options.quickcheck: 486 VARIANTS = ["default", "stress"] 487 options.flaky_tests = "skip" 488 options.slow_tests = "skip" 489 options.pass_fail_tests = "skip" 490 if options.no_stress: 491 VARIANTS = ["default", "nocrankshaft"] 492 if options.no_variants: 493 VARIANTS = ["default"] 494 if options.stress_only: 495 VARIANTS = ["stress"] 496 if options.variants: 497 VARIANTS = options.variants.split(",") 498 if not set(VARIANTS).issubset(ALL_VARIANTS): 499 print "All variants must be in %s" % str(ALL_VARIANTS) 500 return False 501 if options.predictable: 502 VARIANTS = ["default"] 503 options.extra_flags.append("--predictable") 504 options.extra_flags.append("--verify_predictable") 505 options.extra_flags.append("--no-inline-new") 506 507 if not options.shell_dir: 508 if options.shell: 509 print "Warning: --shell is deprecated, use --shell-dir instead." 510 options.shell_dir = os.path.dirname(options.shell) 511 if options.valgrind: 512 run_valgrind = os.path.join("tools", "run-valgrind.py") 513 # This is OK for distributed running, so we don't need to set no_network. 514 options.command_prefix = (["python", "-u", run_valgrind] + 515 options.command_prefix) 516 def CheckTestMode(name, option): 517 if not option in ["run", "skip", "dontcare"]: 518 print "Unknown %s mode %s" % (name, option) 519 return False 520 return True 521 if not CheckTestMode("flaky test", options.flaky_tests): 522 return False 523 if not CheckTestMode("slow test", options.slow_tests): 524 return False 525 if not CheckTestMode("pass|fail test", options.pass_fail_tests): 526 return False 527 if options.no_i18n: 528 TEST_MAP["bot_default"].remove("intl") 529 TEST_MAP["default"].remove("intl") 530 return True 531 532 533def ShardTests(tests, options): 534 # Read gtest shard configuration from environment (e.g. set by swarming). 535 # If none is present, use values passed on the command line. 536 shard_count = int(os.environ.get('GTEST_TOTAL_SHARDS', options.shard_count)) 537 shard_run = os.environ.get('GTEST_SHARD_INDEX') 538 if shard_run is not None: 539 # The v8 shard_run starts at 1, while GTEST_SHARD_INDEX starts at 0. 540 shard_run = int(shard_run) + 1 541 else: 542 shard_run = options.shard_run 543 544 if options.shard_count > 1: 545 # Log if a value was passed on the cmd line and it differs from the 546 # environment variables. 547 if options.shard_count != shard_count: 548 print("shard_count from cmd line differs from environment variable " 549 "GTEST_TOTAL_SHARDS") 550 if options.shard_run > 1 and options.shard_run != shard_run: 551 print("shard_run from cmd line differs from environment variable " 552 "GTEST_SHARD_INDEX") 553 554 if shard_count < 2: 555 return tests 556 if shard_run < 1 or shard_run > shard_count: 557 print "shard-run not a valid number, should be in [1:shard-count]" 558 print "defaulting back to running all tests" 559 return tests 560 count = 0 561 shard = [] 562 for test in tests: 563 if count % shard_count == shard_run - 1: 564 shard.append(test) 565 count += 1 566 return shard 567 568 569def Main(): 570 # Use the v8 root as cwd as some test cases use "load" with relative paths. 571 os.chdir(BASE_DIR) 572 573 parser = BuildOptions() 574 (options, args) = parser.parse_args() 575 if not ProcessOptions(options): 576 parser.print_help() 577 return 1 578 SetupEnvironment(options) 579 580 exit_code = 0 581 if not options.no_presubmit: 582 print ">>> running presubmit tests" 583 exit_code = subprocess.call( 584 [sys.executable, join(BASE_DIR, "tools", "presubmit.py")]) 585 586 suite_paths = utils.GetSuitePaths(join(BASE_DIR, "test")) 587 588 # Use default tests if no test configuration was provided at the cmd line. 589 if len(args) == 0: 590 args = ["default"] 591 592 # Expand arguments with grouped tests. The args should reflect the list of 593 # suites as otherwise filters would break. 594 def ExpandTestGroups(name): 595 if name in TEST_MAP: 596 return [suite for suite in TEST_MAP[name]] 597 else: 598 return [name] 599 args = reduce(lambda x, y: x + y, 600 [ExpandTestGroups(arg) for arg in args], 601 []) 602 603 args_suites = OrderedDict() # Used as set 604 for arg in args: 605 args_suites[arg.split('/')[0]] = True 606 suite_paths = [ s for s in args_suites if s in suite_paths ] 607 608 suites = [] 609 for root in suite_paths: 610 suite = testsuite.TestSuite.LoadTestSuite( 611 os.path.join(BASE_DIR, "test", root)) 612 if suite: 613 suite.SetupWorkingDirectory() 614 suites.append(suite) 615 616 if options.download_data or options.download_data_only: 617 for s in suites: 618 s.DownloadData() 619 620 if options.download_data_only: 621 return exit_code 622 623 for (arch, mode) in options.arch_and_mode: 624 try: 625 code = Execute(arch, mode, args, options, suites) 626 except KeyboardInterrupt: 627 return 2 628 exit_code = exit_code or code 629 return exit_code 630 631 632def Execute(arch, mode, args, options, suites): 633 print(">>> Running tests for %s.%s" % (arch, mode)) 634 635 shell_dir = options.shell_dir 636 if not shell_dir: 637 if options.buildbot: 638 # TODO(machenbach): Get rid of different output folder location on 639 # buildbot. Currently this is capitalized Release and Debug. 640 shell_dir = os.path.join(BASE_DIR, options.outdir, mode) 641 mode = BuildbotToV8Mode(mode) 642 else: 643 shell_dir = os.path.join( 644 BASE_DIR, 645 options.outdir, 646 "%s.%s" % (arch, MODES[mode]["output_folder"]), 647 ) 648 if not os.path.exists(shell_dir): 649 raise Exception('Could not find shell_dir: "%s"' % shell_dir) 650 651 # Populate context object. 652 mode_flags = MODES[mode]["flags"] 653 timeout = options.timeout 654 if timeout == -1: 655 # Simulators are slow, therefore allow a longer default timeout. 656 if arch in SLOW_ARCHS: 657 timeout = 2 * TIMEOUT_DEFAULT; 658 else: 659 timeout = TIMEOUT_DEFAULT; 660 661 timeout *= MODES[mode]["timeout_scalefactor"] 662 663 if options.predictable: 664 # Predictable mode is slower. 665 timeout *= 2 666 667 # TODO(machenbach): Remove temporary verbose output on windows after 668 # debugging driver-hung-up on XP. 669 verbose_output = ( 670 options.verbose or 671 utils.IsWindows() and options.progress == "verbose" 672 ) 673 ctx = context.Context(arch, MODES[mode]["execution_mode"], shell_dir, 674 mode_flags, verbose_output, 675 timeout, options.isolates, 676 options.command_prefix, 677 options.extra_flags, 678 options.no_i18n, 679 options.random_seed, 680 options.no_sorting, 681 options.rerun_failures_count, 682 options.rerun_failures_max, 683 options.predictable, 684 options.no_harness, 685 use_perf_data=not options.swarming) 686 687 # TODO(all): Combine "simulator" and "simulator_run". 688 simulator_run = not options.dont_skip_simulator_slow_tests and \ 689 arch in ['arm64', 'arm', 'mipsel', 'mips', 'mips64', 'mips64el', \ 690 'ppc', 'ppc64'] and \ 691 ARCH_GUESS and arch != ARCH_GUESS 692 # Find available test suites and read test cases from them. 693 variables = { 694 "arch": arch, 695 "asan": options.asan, 696 "deopt_fuzzer": False, 697 "gc_stress": options.gc_stress, 698 "gcov_coverage": options.gcov_coverage, 699 "ignition": options.ignition, 700 "isolates": options.isolates, 701 "mode": MODES[mode]["status_mode"], 702 "no_i18n": options.no_i18n, 703 "no_snap": options.no_snap, 704 "simulator_run": simulator_run, 705 "simulator": utils.UseSimulator(arch), 706 "system": utils.GuessOS(), 707 "tsan": options.tsan, 708 "msan": options.msan, 709 "dcheck_always_on": options.dcheck_always_on, 710 "novfp3": options.novfp3, 711 "predictable": options.predictable, 712 "byteorder": sys.byteorder, 713 } 714 all_tests = [] 715 num_tests = 0 716 for s in suites: 717 s.ReadStatusFile(variables) 718 s.ReadTestCases(ctx) 719 if len(args) > 0: 720 s.FilterTestCasesByArgs(args) 721 all_tests += s.tests 722 s.FilterTestCasesByStatus(options.warn_unused, options.flaky_tests, 723 options.slow_tests, options.pass_fail_tests) 724 if options.cat: 725 verbose.PrintTestSource(s.tests) 726 continue 727 variant_gen = s.CreateVariantGenerator(VARIANTS) 728 variant_tests = [ t.CopyAddingFlags(v, flags) 729 for t in s.tests 730 for v in variant_gen.FilterVariantsByTest(t) 731 for flags in variant_gen.GetFlagSets(t, v) ] 732 733 if options.random_seed_stress_count > 1: 734 # Duplicate test for random seed stress mode. 735 def iter_seed_flags(): 736 for i in range(0, options.random_seed_stress_count): 737 # Use given random seed for all runs (set by default in execution.py) 738 # or a new random seed if none is specified. 739 if options.random_seed: 740 yield [] 741 else: 742 yield ["--random-seed=%d" % RandomSeed()] 743 s.tests = [ 744 t.CopyAddingFlags(t.variant, flags) 745 for t in variant_tests 746 for flags in iter_seed_flags() 747 ] 748 else: 749 s.tests = variant_tests 750 751 s.tests = ShardTests(s.tests, options) 752 num_tests += len(s.tests) 753 754 if options.cat: 755 return 0 # We're done here. 756 757 if options.report: 758 verbose.PrintReport(all_tests) 759 760 # Run the tests, either locally or distributed on the network. 761 start_time = time.time() 762 progress_indicator = progress.IndicatorNotifier() 763 progress_indicator.Register(progress.PROGRESS_INDICATORS[options.progress]()) 764 if options.junitout: 765 progress_indicator.Register(progress.JUnitTestProgressIndicator( 766 options.junitout, options.junittestsuite)) 767 if options.json_test_results: 768 progress_indicator.Register(progress.JsonTestProgressIndicator( 769 options.json_test_results, arch, MODES[mode]["execution_mode"], 770 ctx.random_seed)) 771 772 run_networked = not options.no_network 773 if not run_networked: 774 if verbose_output: 775 print("Network distribution disabled, running tests locally.") 776 elif utils.GuessOS() != "linux": 777 print("Network distribution is only supported on Linux, sorry!") 778 run_networked = False 779 peers = [] 780 if run_networked: 781 peers = network_execution.GetPeers() 782 if not peers: 783 print("No connection to distribution server; running tests locally.") 784 run_networked = False 785 elif len(peers) == 1: 786 print("No other peers on the network; running tests locally.") 787 run_networked = False 788 elif num_tests <= 100: 789 print("Less than 100 tests, running them locally.") 790 run_networked = False 791 792 if run_networked: 793 runner = network_execution.NetworkedRunner(suites, progress_indicator, 794 ctx, peers, BASE_DIR) 795 else: 796 runner = execution.Runner(suites, progress_indicator, ctx) 797 798 exit_code = runner.Run(options.j) 799 overall_duration = time.time() - start_time 800 801 if options.time: 802 verbose.PrintTestDurations(suites, overall_duration) 803 804 if num_tests == 0: 805 print("Warning: no tests were run!") 806 807 if exit_code == 1 and options.json_test_results: 808 print("Force exit code 0 after failures. Json test results file generated " 809 "with failure information.") 810 exit_code = 0 811 812 return exit_code 813 814 815if __name__ == "__main__": 816 sys.exit(Main()) 817