1#!/usr/bin/env python3 2# Copyright 2015 gRPC authors. 3# 4# Licensed under the Apache License, Version 2.0 (the "License"); 5# you may not use this file except in compliance with the License. 6# You may obtain a copy of the License at 7# 8# http://www.apache.org/licenses/LICENSE-2.0 9# 10# Unless required by applicable law or agreed to in writing, software 11# distributed under the License is distributed on an "AS IS" BASIS, 12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13# See the License for the specific language governing permissions and 14# limitations under the License. 15"""Run test matrix.""" 16 17from __future__ import print_function 18 19import argparse 20import multiprocessing 21import os 22import sys 23 24from python_utils.filter_pull_request_tests import filter_tests 25import python_utils.jobset as jobset 26import python_utils.report_utils as report_utils 27 28_ROOT = os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]), "../..")) 29os.chdir(_ROOT) 30 31_DEFAULT_RUNTESTS_TIMEOUT = 1 * 60 * 60 32 33# C/C++ tests can take long time 34_CPP_RUNTESTS_TIMEOUT = 6 * 60 * 60 35 36# Set timeout high for ObjC for Cocoapods to install pods 37_OBJC_RUNTESTS_TIMEOUT = 4 * 60 * 60 38 39# Set timeout high for Ruby for MacOS for slow xcodebuild 40_RUBY_RUNTESTS_TIMEOUT = 2 * 60 * 60 41 42# Number of jobs assigned to each run_tests.py instance 43_DEFAULT_INNER_JOBS = 2 44 45# Name of the top-level umbrella report that includes all the run_tests.py invocations 46# Note that the starting letter 't' matters so that the targets are listed AFTER 47# the per-test breakdown items that start with 'run_tests/' (it is more readable that way) 48_MATRIX_REPORT_NAME = "toplevel_run_tests_invocations" 49 50 51def _safe_report_name(name): 52 """Reports with '+' in target name won't show correctly in ResultStore""" 53 return name.replace("+", "p") 54 55 56def _report_filename(name): 57 """Generates report file name with directory structure that leads to better presentation by internal CI""" 58 # 'sponge_log.xml' suffix must be there for results to get recognized by kokoro. 59 return "%s/%s" % (_safe_report_name(name), "sponge_log.xml") 60 61 62def _matrix_job_logfilename(shortname_for_multi_target): 63 """Generate location for log file that will match the sponge_log.xml from the top-level matrix report.""" 64 # 'sponge_log.log' suffix must be there for log to get recognized as "target log" 65 # for the corresponding 'sponge_log.xml' report. 66 # the shortname_for_multi_target component must be set to match the sponge_log.xml location 67 # because the top-level render_junit_xml_report is called with multi_target=True 68 sponge_log_name = "%s/%s/%s" % ( 69 _MATRIX_REPORT_NAME, 70 shortname_for_multi_target, 71 "sponge_log.log", 72 ) 73 # env variable can be used to override the base location for the reports 74 # so we need to match that behavior here too 75 base_dir = os.getenv("GRPC_TEST_REPORT_BASE_DIR", None) 76 if base_dir: 77 sponge_log_name = os.path.join(base_dir, sponge_log_name) 78 return sponge_log_name 79 80 81def _docker_jobspec( 82 name, 83 runtests_args=[], 84 runtests_envs={}, 85 inner_jobs=_DEFAULT_INNER_JOBS, 86 timeout_seconds=None, 87): 88 """Run a single instance of run_tests.py in a docker container""" 89 if not timeout_seconds: 90 timeout_seconds = _DEFAULT_RUNTESTS_TIMEOUT 91 shortname = "run_tests_%s" % name 92 test_job = jobset.JobSpec( 93 cmdline=[ 94 "python3", 95 "tools/run_tests/run_tests.py", 96 "--use_docker", 97 "-t", 98 "-j", 99 str(inner_jobs), 100 "-x", 101 "run_tests/%s" % _report_filename(name), 102 "--report_suite_name", 103 "%s" % _safe_report_name(name), 104 ] 105 + runtests_args, 106 environ=runtests_envs, 107 shortname=shortname, 108 timeout_seconds=timeout_seconds, 109 logfilename=_matrix_job_logfilename(shortname), 110 ) 111 return test_job 112 113 114def _workspace_jobspec( 115 name, 116 runtests_args=[], 117 workspace_name=None, 118 runtests_envs={}, 119 inner_jobs=_DEFAULT_INNER_JOBS, 120 timeout_seconds=None, 121): 122 """Run a single instance of run_tests.py in a separate workspace""" 123 if not workspace_name: 124 workspace_name = "workspace_%s" % name 125 if not timeout_seconds: 126 timeout_seconds = _DEFAULT_RUNTESTS_TIMEOUT 127 shortname = "run_tests_%s" % name 128 env = {"WORKSPACE_NAME": workspace_name} 129 env.update(runtests_envs) 130 # if report base dir is set, we don't need to ".." to come out of the workspace dir 131 report_dir_prefix = ( 132 "" if os.getenv("GRPC_TEST_REPORT_BASE_DIR", None) else "../" 133 ) 134 test_job = jobset.JobSpec( 135 cmdline=[ 136 "bash", 137 "tools/run_tests/helper_scripts/run_tests_in_workspace.sh", 138 "-t", 139 "-j", 140 str(inner_jobs), 141 "-x", 142 "%srun_tests/%s" % (report_dir_prefix, _report_filename(name)), 143 "--report_suite_name", 144 "%s" % _safe_report_name(name), 145 ] 146 + runtests_args, 147 environ=env, 148 shortname=shortname, 149 timeout_seconds=timeout_seconds, 150 logfilename=_matrix_job_logfilename(shortname), 151 ) 152 return test_job 153 154 155def _generate_jobs( 156 languages, 157 configs, 158 platforms, 159 iomgr_platforms=["native"], 160 arch=None, 161 compiler=None, 162 labels=[], 163 extra_args=[], 164 extra_envs={}, 165 inner_jobs=_DEFAULT_INNER_JOBS, 166 timeout_seconds=None, 167): 168 result = [] 169 for language in languages: 170 for platform in platforms: 171 for iomgr_platform in iomgr_platforms: 172 for config in configs: 173 name = "%s_%s_%s_%s" % ( 174 language, 175 platform, 176 config, 177 iomgr_platform, 178 ) 179 runtests_args = [ 180 "-l", 181 language, 182 "-c", 183 config, 184 "--iomgr_platform", 185 iomgr_platform, 186 ] 187 if arch or compiler: 188 name += "_%s_%s" % (arch, compiler) 189 runtests_args += [ 190 "--arch", 191 arch, 192 "--compiler", 193 compiler, 194 ] 195 if "--build_only" in extra_args: 196 name += "_buildonly" 197 for extra_env in extra_envs: 198 name += "_%s_%s" % (extra_env, extra_envs[extra_env]) 199 200 runtests_args += extra_args 201 if platform == "linux": 202 job = _docker_jobspec( 203 name=name, 204 runtests_args=runtests_args, 205 runtests_envs=extra_envs, 206 inner_jobs=inner_jobs, 207 timeout_seconds=timeout_seconds, 208 ) 209 else: 210 job = _workspace_jobspec( 211 name=name, 212 runtests_args=runtests_args, 213 runtests_envs=extra_envs, 214 inner_jobs=inner_jobs, 215 timeout_seconds=timeout_seconds, 216 ) 217 218 job.labels = [ 219 platform, 220 config, 221 language, 222 iomgr_platform, 223 ] + labels 224 result.append(job) 225 return result 226 227 228def _create_test_jobs(extra_args=[], inner_jobs=_DEFAULT_INNER_JOBS): 229 test_jobs = [] 230 # sanity tests 231 test_jobs += _generate_jobs( 232 languages=["sanity", "clang-tidy"], 233 configs=["dbg"], 234 platforms=["linux"], 235 labels=["basictests"], 236 extra_args=extra_args + ["--report_multi_target"], 237 inner_jobs=inner_jobs, 238 ) 239 240 # supported on all platforms. 241 test_jobs += _generate_jobs( 242 languages=["c"], 243 configs=["dbg", "opt"], 244 platforms=["linux", "macos", "windows"], 245 labels=["basictests", "corelang"], 246 extra_args=extra_args, # don't use multi_target report because C has too many test cases 247 inner_jobs=inner_jobs, 248 timeout_seconds=_CPP_RUNTESTS_TIMEOUT, 249 ) 250 251 # C# tests (both on .NET desktop/mono and .NET core) 252 test_jobs += _generate_jobs( 253 languages=["csharp"], 254 configs=["dbg", "opt"], 255 platforms=["linux", "macos", "windows"], 256 labels=["basictests", "multilang"], 257 extra_args=extra_args + ["--report_multi_target"], 258 inner_jobs=inner_jobs, 259 ) 260 261 # ARM64 Linux C# tests 262 test_jobs += _generate_jobs( 263 languages=["csharp"], 264 configs=["dbg", "opt"], 265 platforms=["linux"], 266 arch="arm64", 267 compiler="default", 268 labels=["basictests_arm64"], 269 extra_args=extra_args + ["--report_multi_target"], 270 inner_jobs=inner_jobs, 271 ) 272 273 test_jobs += _generate_jobs( 274 languages=["python"], 275 configs=["opt"], 276 platforms=["linux", "macos", "windows"], 277 iomgr_platforms=["native"], 278 labels=["basictests", "multilang"], 279 extra_args=extra_args + ["--report_multi_target"], 280 inner_jobs=inner_jobs, 281 ) 282 283 # ARM64 Linux Python tests 284 test_jobs += _generate_jobs( 285 languages=["python"], 286 configs=["opt"], 287 platforms=["linux"], 288 arch="arm64", 289 compiler="default", 290 iomgr_platforms=["native"], 291 labels=["basictests_arm64"], 292 extra_args=extra_args + ["--report_multi_target"], 293 inner_jobs=inner_jobs, 294 ) 295 296 # supported on linux and mac. 297 test_jobs += _generate_jobs( 298 languages=["c++"], 299 configs=["dbg", "opt"], 300 platforms=["linux", "macos"], 301 labels=["basictests", "corelang"], 302 extra_args=extra_args, # don't use multi_target report because C++ has too many test cases 303 inner_jobs=inner_jobs, 304 timeout_seconds=_CPP_RUNTESTS_TIMEOUT, 305 ) 306 307 test_jobs += _generate_jobs( 308 languages=["ruby", "php8"], 309 configs=["dbg", "opt"], 310 platforms=["linux", "macos"], 311 labels=["basictests", "multilang"], 312 extra_args=extra_args + ["--report_multi_target"], 313 inner_jobs=inner_jobs, 314 timeout_seconds=_RUBY_RUNTESTS_TIMEOUT, 315 ) 316 317 # ARM64 Linux Ruby and PHP tests 318 test_jobs += _generate_jobs( 319 languages=["ruby", "php8"], 320 configs=["dbg", "opt"], 321 platforms=["linux"], 322 arch="arm64", 323 compiler="default", 324 labels=["basictests_arm64"], 325 extra_args=extra_args + ["--report_multi_target"], 326 inner_jobs=inner_jobs, 327 ) 328 329 # supported on mac only. 330 test_jobs += _generate_jobs( 331 languages=["objc"], 332 configs=["opt"], 333 platforms=["macos"], 334 labels=["basictests", "multilang"], 335 extra_args=extra_args + ["--report_multi_target"], 336 inner_jobs=inner_jobs, 337 timeout_seconds=_OBJC_RUNTESTS_TIMEOUT, 338 ) 339 340 return test_jobs 341 342 343def _create_portability_test_jobs( 344 extra_args=[], inner_jobs=_DEFAULT_INNER_JOBS 345): 346 test_jobs = [] 347 # portability C x86 348 test_jobs += _generate_jobs( 349 languages=["c"], 350 configs=["dbg"], 351 platforms=["linux"], 352 arch="x86", 353 compiler="default", 354 labels=["portability", "corelang"], 355 extra_args=extra_args, 356 inner_jobs=inner_jobs, 357 ) 358 359 # portability C and C++ on x64 360 for compiler in [ 361 "gcc8", 362 # TODO(b/283304471): Tests using OpenSSL's engine APIs were broken and removed 363 "gcc10.2_openssl102", 364 "gcc10.2_openssl111", 365 "gcc12_openssl309", 366 "gcc14", 367 "gcc_musl", 368 "clang7", 369 "clang19", 370 ]: 371 test_jobs += _generate_jobs( 372 languages=["c", "c++"], 373 configs=["dbg"], 374 platforms=["linux"], 375 arch="x64", 376 compiler=compiler, 377 labels=["portability", "corelang"] 378 + (["openssl"] if "openssl" in compiler else []), 379 extra_args=extra_args, 380 inner_jobs=inner_jobs, 381 timeout_seconds=_CPP_RUNTESTS_TIMEOUT, 382 ) 383 384 # portability C & C++ on Windows 64-bit 385 test_jobs += _generate_jobs( 386 languages=["c", "c++"], 387 configs=["dbg"], 388 platforms=["windows"], 389 arch="default", 390 compiler="cmake_ninja_vs2022", 391 labels=["portability", "corelang"], 392 extra_args=extra_args, 393 inner_jobs=inner_jobs, 394 timeout_seconds=_CPP_RUNTESTS_TIMEOUT, 395 ) 396 397 # portability C and C++ on Windows with the "Visual Studio 2022" cmake 398 # generator, i.e. not using Ninja (to verify that we can still build with msbuild) 399 # test_jobs += _generate_jobs( 400 # languages=["c", "c++"], 401 # configs=["dbg"], 402 # platforms=["windows"], 403 # arch="x64", 404 # compiler="cmake_vs2022", 405 # labels=["portability", "corelang"], 406 # extra_args=extra_args, 407 # inner_jobs=inner_jobs, 408 # timeout_seconds=_CPP_RUNTESTS_TIMEOUT, 409 # ) 410 411 # C and C++ with no-exceptions on Linux 412 test_jobs += _generate_jobs( 413 languages=["c", "c++"], 414 configs=["noexcept"], 415 platforms=["linux"], 416 labels=["portability", "corelang"], 417 extra_args=extra_args, 418 inner_jobs=inner_jobs, 419 timeout_seconds=_CPP_RUNTESTS_TIMEOUT, 420 ) 421 422 test_jobs += _generate_jobs( 423 languages=["python"], 424 configs=["dbg"], 425 platforms=["linux"], 426 arch="default", 427 compiler="python_alpine", 428 labels=["portability", "multilang"], 429 extra_args=extra_args + ["--report_multi_target"], 430 inner_jobs=inner_jobs, 431 ) 432 433 return test_jobs 434 435 436def _allowed_labels(): 437 """Returns a list of existing job labels.""" 438 all_labels = set() 439 for job in _create_test_jobs() + _create_portability_test_jobs(): 440 for label in job.labels: 441 all_labels.add(label) 442 return sorted(all_labels) 443 444 445def _runs_per_test_type(arg_str): 446 """Auxiliary function to parse the "runs_per_test" flag.""" 447 try: 448 n = int(arg_str) 449 if n <= 0: 450 raise ValueError 451 return n 452 except: 453 msg = "'{}' is not a positive integer".format(arg_str) 454 raise argparse.ArgumentTypeError(msg) 455 456 457if __name__ == "__main__": 458 argp = argparse.ArgumentParser( 459 description="Run a matrix of run_tests.py tests." 460 ) 461 argp.add_argument( 462 "-j", 463 "--jobs", 464 default=multiprocessing.cpu_count() / _DEFAULT_INNER_JOBS, 465 type=int, 466 help="Number of concurrent run_tests.py instances.", 467 ) 468 argp.add_argument( 469 "-f", 470 "--filter", 471 choices=_allowed_labels(), 472 nargs="+", 473 default=[], 474 help="Filter targets to run by label with AND semantics.", 475 ) 476 argp.add_argument( 477 "--exclude", 478 choices=_allowed_labels(), 479 nargs="+", 480 default=[], 481 help="Exclude targets with any of given labels.", 482 ) 483 argp.add_argument( 484 "--build_only", 485 default=False, 486 action="store_const", 487 const=True, 488 help="Pass --build_only flag to run_tests.py instances.", 489 ) 490 argp.add_argument( 491 "--force_default_poller", 492 default=False, 493 action="store_const", 494 const=True, 495 help="Pass --force_default_poller to run_tests.py instances.", 496 ) 497 argp.add_argument( 498 "--dry_run", 499 default=False, 500 action="store_const", 501 const=True, 502 help="Only print what would be run.", 503 ) 504 argp.add_argument( 505 "--filter_pr_tests", 506 default=False, 507 action="store_const", 508 const=True, 509 help="Filters out tests irrelevant to pull request changes.", 510 ) 511 argp.add_argument( 512 "--base_branch", 513 default="origin/master", 514 type=str, 515 help="Branch that pull request is requesting to merge into", 516 ) 517 argp.add_argument( 518 "--inner_jobs", 519 default=_DEFAULT_INNER_JOBS, 520 type=int, 521 help="Number of jobs in each run_tests.py instance", 522 ) 523 argp.add_argument( 524 "-n", 525 "--runs_per_test", 526 default=1, 527 type=_runs_per_test_type, 528 help="How many times to run each tests. >1 runs implies " 529 + "omitting passing test from the output & reports.", 530 ) 531 argp.add_argument( 532 "--max_time", 533 default=-1, 534 type=int, 535 help="Maximum amount of time to run tests for" 536 + "(other tests will be skipped)", 537 ) 538 argp.add_argument( 539 "--bq_result_table", 540 default="", 541 type=str, 542 nargs="?", 543 help="Upload test results to a specified BQ table.", 544 ) 545 argp.add_argument( 546 "--extra_args", 547 default="", 548 type=str, 549 nargs=argparse.REMAINDER, 550 help="Extra test args passed to each sub-script.", 551 ) 552 args = argp.parse_args() 553 554 extra_args = [] 555 if args.build_only: 556 extra_args.append("--build_only") 557 if args.force_default_poller: 558 extra_args.append("--force_default_poller") 559 if args.runs_per_test > 1: 560 extra_args.append("-n") 561 extra_args.append("%s" % args.runs_per_test) 562 extra_args.append("--quiet_success") 563 if args.max_time > 0: 564 extra_args.extend(("--max_time", "%d" % args.max_time)) 565 if args.bq_result_table: 566 extra_args.append("--bq_result_table") 567 extra_args.append("%s" % args.bq_result_table) 568 extra_args.append("--measure_cpu_costs") 569 if args.extra_args: 570 extra_args.extend(args.extra_args) 571 572 all_jobs = _create_test_jobs( 573 extra_args=extra_args, inner_jobs=args.inner_jobs 574 ) + _create_portability_test_jobs( 575 extra_args=extra_args, inner_jobs=args.inner_jobs 576 ) 577 578 jobs = [] 579 for job in all_jobs: 580 if not args.filter or all( 581 filter in job.labels for filter in args.filter 582 ): 583 if not any( 584 exclude_label in job.labels for exclude_label in args.exclude 585 ): 586 jobs.append(job) 587 588 if not jobs: 589 jobset.message( 590 "FAILED", "No test suites match given criteria.", do_newline=True 591 ) 592 sys.exit(1) 593 594 print("IMPORTANT: The changes you are testing need to be locally committed") 595 print("because only the committed changes in the current branch will be") 596 print("copied to the docker environment or into subworkspaces.") 597 598 skipped_jobs = [] 599 600 if args.filter_pr_tests: 601 print("Looking for irrelevant tests to skip...") 602 relevant_jobs = filter_tests(jobs, args.base_branch) 603 if len(relevant_jobs) == len(jobs): 604 print("No tests will be skipped.") 605 else: 606 print("These tests will be skipped:") 607 skipped_jobs = list(set(jobs) - set(relevant_jobs)) 608 # Sort by shortnames to make printing of skipped tests consistent 609 skipped_jobs.sort(key=lambda job: job.shortname) 610 for job in list(skipped_jobs): 611 print(" %s" % job.shortname) 612 jobs = relevant_jobs 613 614 print("Will run these tests:") 615 for job in jobs: 616 print(' %s: "%s"' % (job.shortname, " ".join(job.cmdline))) 617 print("") 618 619 if args.dry_run: 620 print("--dry_run was used, exiting") 621 sys.exit(1) 622 623 jobset.message("START", "Running test matrix.", do_newline=True) 624 num_failures, resultset = jobset.run( 625 jobs, newline_on_success=True, travis=True, maxjobs=args.jobs 626 ) 627 # Merge skipped tests into results to show skipped tests on report.xml 628 if skipped_jobs: 629 ignored_num_skipped_failures, skipped_results = jobset.run( 630 skipped_jobs, skip_jobs=True 631 ) 632 resultset.update(skipped_results) 633 report_utils.render_junit_xml_report( 634 resultset, 635 _report_filename(_MATRIX_REPORT_NAME), 636 suite_name=_MATRIX_REPORT_NAME, 637 multi_target=True, 638 ) 639 640 if num_failures == 0: 641 jobset.message( 642 "SUCCESS", 643 "All run_tests.py instances finished successfully.", 644 do_newline=True, 645 ) 646 else: 647 jobset.message( 648 "FAILED", 649 "Some run_tests.py instances have failed.", 650 do_newline=True, 651 ) 652 sys.exit(1) 653