1#!/usr/bin/env python3 2# Copyright 2015 gRPC authors. 3# 4# Licensed under the Apache License, Version 2.0 (the "License"); 5# you may not use this file except in compliance with the License. 6# You may obtain a copy of the License at 7# 8# http://www.apache.org/licenses/LICENSE-2.0 9# 10# Unless required by applicable law or agreed to in writing, software 11# distributed under the License is distributed on an "AS IS" BASIS, 12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13# See the License for the specific language governing permissions and 14# limitations under the License. 15"""Run tests in parallel.""" 16 17from __future__ import print_function 18 19import argparse 20import ast 21import collections 22import glob 23import itertools 24import json 25import logging 26import multiprocessing 27import os 28import os.path 29import platform 30import random 31import re 32import shlex 33import socket 34import subprocess 35import sys 36import tempfile 37import time 38import traceback 39import uuid 40 41import six 42from six.moves import urllib 43 44import python_utils.jobset as jobset 45import python_utils.report_utils as report_utils 46import python_utils.start_port_server as start_port_server 47import python_utils.watch_dirs as watch_dirs 48 49try: 50 from python_utils.upload_test_results import upload_results_to_bq 51except ImportError: 52 pass # It's ok to not import because this is only necessary to upload results to BQ. 53 54gcp_utils_dir = os.path.abspath( 55 os.path.join(os.path.dirname(__file__), "../gcp/utils") 56) 57sys.path.append(gcp_utils_dir) 58 59_ROOT = os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]), "../..")) 60os.chdir(_ROOT) 61 62_FORCE_ENVIRON_FOR_WRAPPERS = { 63 "GRPC_VERBOSITY": "DEBUG", 64} 65 66_POLLING_STRATEGIES = { 67 "linux": ["epoll1", "poll"], 68 "mac": ["poll"], 69} 70 71 72def platform_string(): 73 return jobset.platform_string() 74 75 76_DEFAULT_TIMEOUT_SECONDS = 5 * 60 77_PRE_BUILD_STEP_TIMEOUT_SECONDS = 10 * 60 78 79 80def run_shell_command(cmd, env=None, cwd=None): 81 try: 82 subprocess.check_output(cmd, shell=True, env=env, cwd=cwd) 83 except subprocess.CalledProcessError as e: 84 logging.exception( 85 "Error while running command '%s'. Exit status %d. Output:\n%s", 86 e.cmd, 87 e.returncode, 88 e.output, 89 ) 90 raise 91 92 93def max_parallel_tests_for_current_platform(): 94 # Too much test parallelization has only been seen to be a problem 95 # so far on windows. 96 if jobset.platform_string() == "windows": 97 return 64 98 return 1024 99 100 101def _print_debug_info_epilogue(dockerfile_dir=None): 102 """Use to print useful info for debug/repro just before exiting.""" 103 print("") 104 print("=== run_tests.py DEBUG INFO ===") 105 print('command: "%s"' % " ".join(sys.argv)) 106 if dockerfile_dir: 107 print("dockerfile: %s" % dockerfile_dir) 108 kokoro_job_name = os.getenv("KOKORO_JOB_NAME") 109 if kokoro_job_name: 110 print("kokoro job name: %s" % kokoro_job_name) 111 print("===============================") 112 113 114# SimpleConfig: just compile with CONFIG=config, and run the binary to test 115class Config(object): 116 def __init__( 117 self, 118 config, 119 environ=None, 120 timeout_multiplier=1, 121 tool_prefix=[], 122 iomgr_platform="native", 123 ): 124 if environ is None: 125 environ = {} 126 self.build_config = config 127 self.environ = environ 128 self.environ["CONFIG"] = config 129 self.tool_prefix = tool_prefix 130 self.timeout_multiplier = timeout_multiplier 131 self.iomgr_platform = iomgr_platform 132 133 def job_spec( 134 self, 135 cmdline, 136 timeout_seconds=_DEFAULT_TIMEOUT_SECONDS, 137 shortname=None, 138 environ={}, 139 cpu_cost=1.0, 140 flaky=False, 141 ): 142 """Construct a jobset.JobSpec for a test under this config 143 144 Args: 145 cmdline: a list of strings specifying the command line the test 146 would like to run 147 """ 148 actual_environ = self.environ.copy() 149 for k, v in environ.items(): 150 actual_environ[k] = v 151 if not flaky and shortname and shortname in flaky_tests: 152 flaky = True 153 if shortname in shortname_to_cpu: 154 cpu_cost = shortname_to_cpu[shortname] 155 return jobset.JobSpec( 156 cmdline=self.tool_prefix + cmdline, 157 shortname=shortname, 158 environ=actual_environ, 159 cpu_cost=cpu_cost, 160 timeout_seconds=( 161 self.timeout_multiplier * timeout_seconds 162 if timeout_seconds 163 else None 164 ), 165 flake_retries=4 if flaky or args.allow_flakes else 0, 166 timeout_retries=1 if flaky or args.allow_flakes else 0, 167 ) 168 169 170def get_c_tests(travis, test_lang): 171 out = [] 172 platforms_str = "ci_platforms" if travis else "platforms" 173 with open("tools/run_tests/generated/tests.json") as f: 174 js = json.load(f) 175 return [ 176 tgt 177 for tgt in js 178 if tgt["language"] == test_lang 179 and platform_string() in tgt[platforms_str] 180 and not (travis and tgt["flaky"]) 181 ] 182 183 184def _check_compiler(compiler, supported_compilers): 185 if compiler not in supported_compilers: 186 raise Exception( 187 "Compiler %s not supported (on this platform)." % compiler 188 ) 189 190 191def _check_arch(arch, supported_archs): 192 if arch not in supported_archs: 193 raise Exception("Architecture %s not supported." % arch) 194 195 196def _is_use_docker_child(): 197 """Returns True if running running as a --use_docker child.""" 198 return True if os.getenv("DOCKER_RUN_SCRIPT_COMMAND") else False 199 200 201_PythonConfigVars = collections.namedtuple( 202 "_ConfigVars", 203 [ 204 "shell", 205 "builder", 206 "builder_prefix_arguments", 207 "venv_relative_python", 208 "toolchain", 209 "runner", 210 ], 211) 212 213 214def _python_config_generator(name, major, minor, bits, config_vars): 215 build = ( 216 config_vars.shell 217 + config_vars.builder 218 + config_vars.builder_prefix_arguments 219 + [_python_pattern_function(major=major, minor=minor, bits=bits)] 220 + [name] 221 + config_vars.venv_relative_python 222 + config_vars.toolchain 223 ) 224 # run: [tools/run_tests/helper_scripts/run_python.sh py37/bin/python] 225 python_path = os.path.join(name, config_vars.venv_relative_python[0]) 226 run = config_vars.shell + config_vars.runner + [python_path] 227 return PythonConfig(name, build, run, python_path) 228 229 230def _pypy_config_generator(name, major, config_vars): 231 # Something like "py37/bin/python" 232 python_path = os.path.join(name, config_vars.venv_relative_python[0]) 233 return PythonConfig( 234 name, 235 config_vars.shell 236 + config_vars.builder 237 + config_vars.builder_prefix_arguments 238 + [_pypy_pattern_function(major=major)] 239 + [name] 240 + config_vars.venv_relative_python 241 + config_vars.toolchain, 242 config_vars.shell + config_vars.runner + [python_path], 243 python_path, 244 ) 245 246 247def _python_pattern_function(major, minor, bits): 248 # Bit-ness is handled by the test machine's environment 249 if os.name == "nt": 250 if bits == "64": 251 return "/c/Python{major}{minor}/python.exe".format( 252 major=major, minor=minor, bits=bits 253 ) 254 else: 255 return "/c/Python{major}{minor}_{bits}bits/python.exe".format( 256 major=major, minor=minor, bits=bits 257 ) 258 else: 259 return "python{major}.{minor}".format(major=major, minor=minor) 260 261 262def _pypy_pattern_function(major): 263 if major == "2": 264 return "pypy" 265 elif major == "3": 266 return "pypy3" 267 else: 268 raise ValueError("Unknown PyPy major version") 269 270 271class CLanguage(object): 272 def __init__(self, lang_suffix, test_lang): 273 self.lang_suffix = lang_suffix 274 self.platform = platform_string() 275 self.test_lang = test_lang 276 277 def configure(self, config, args): 278 self.config = config 279 self.args = args 280 if self.platform == "windows": 281 _check_compiler( 282 self.args.compiler, 283 [ 284 "default", 285 "cmake", 286 "cmake_ninja_vs2022", 287 "cmake_vs2022", 288 ], 289 ) 290 _check_arch(self.args.arch, ["default", "x64", "x86"]) 291 292 activate_vs_tools = "" 293 if ( 294 self.args.compiler == "cmake_ninja_vs2022" 295 or self.args.compiler == "cmake" 296 or self.args.compiler == "default" 297 ): 298 # cmake + ninja build is the default because it is faster and supports boringssl assembly optimizations 299 cmake_generator = "Ninja" 300 activate_vs_tools = "2022" 301 elif self.args.compiler == "cmake_vs2022": 302 cmake_generator = "Visual Studio 17 2022" 303 else: 304 print("should never reach here.") 305 sys.exit(1) 306 307 self._cmake_configure_extra_args = list( 308 self.args.cmake_configure_extra_args 309 ) + ["-DCMAKE_CXX_STANDARD=17"] 310 self._cmake_generator_windows = cmake_generator 311 # required to pass as cmake "-A" configuration for VS builds (but not for Ninja) 312 self._cmake_architecture_windows = ( 313 "x64" if self.args.arch == "x64" else "Win32" 314 ) 315 # when building with Ninja, the VS common tools need to be activated first 316 self._activate_vs_tools_windows = activate_vs_tools 317 # "x64_x86" means create 32bit binaries, but use 64bit toolkit to secure more memory for the build 318 self._vs_tools_architecture_windows = ( 319 "x64" if self.args.arch == "x64" else "x64_x86" 320 ) 321 322 else: 323 if self.platform == "linux": 324 # Allow all the known architectures. _check_arch_option has already checked that we're not doing 325 # something illegal when not running under docker. 326 _check_arch(self.args.arch, ["default", "x64", "x86", "arm64"]) 327 else: 328 _check_arch(self.args.arch, ["default"]) 329 330 ( 331 self._docker_distro, 332 self._cmake_configure_extra_args, 333 ) = self._compiler_options( 334 self.args.use_docker, 335 self.args.compiler, 336 self.args.cmake_configure_extra_args, 337 ) 338 339 def test_specs(self): 340 out = [] 341 binaries = get_c_tests(self.args.travis, self.test_lang) 342 for target in binaries: 343 if target.get("boringssl", False): 344 # cmake doesn't build boringssl tests 345 continue 346 auto_timeout_scaling = target.get("auto_timeout_scaling", True) 347 polling_strategies = ( 348 _POLLING_STRATEGIES.get(self.platform, ["all"]) 349 if target.get("uses_polling", True) 350 else ["none"] 351 ) 352 for polling_strategy in polling_strategies: 353 env = { 354 "GRPC_DEFAULT_SSL_ROOTS_FILE_PATH": _ROOT 355 + "/src/core/tsi/test_creds/ca.pem", 356 "GRPC_POLL_STRATEGY": polling_strategy, 357 "GRPC_VERBOSITY": "DEBUG", 358 } 359 resolver = os.environ.get("GRPC_DNS_RESOLVER", None) 360 if resolver: 361 env["GRPC_DNS_RESOLVER"] = resolver 362 shortname_ext = ( 363 "" 364 if polling_strategy == "all" 365 else " GRPC_POLL_STRATEGY=%s" % polling_strategy 366 ) 367 if polling_strategy in target.get("excluded_poll_engines", []): 368 continue 369 370 timeout_scaling = 1 371 if auto_timeout_scaling: 372 config = self.args.config 373 if ( 374 "asan" in config 375 or config == "msan" 376 or config == "tsan" 377 or config == "ubsan" 378 or config == "helgrind" 379 or config == "memcheck" 380 ): 381 # Scale overall test timeout if running under various sanitizers. 382 # scaling value is based on historical data analysis 383 timeout_scaling *= 3 384 385 if self.config.build_config in target["exclude_configs"]: 386 continue 387 if self.args.iomgr_platform in target.get("exclude_iomgrs", []): 388 continue 389 390 if self.platform == "windows": 391 if self._cmake_generator_windows == "Ninja": 392 binary = "cmake/build/%s.exe" % target["name"] 393 else: 394 binary = "cmake/build/%s/%s.exe" % ( 395 _MSBUILD_CONFIG[self.config.build_config], 396 target["name"], 397 ) 398 else: 399 binary = "cmake/build/%s" % target["name"] 400 401 cpu_cost = target["cpu_cost"] 402 if cpu_cost == "capacity": 403 cpu_cost = multiprocessing.cpu_count() 404 if os.path.isfile(binary): 405 list_test_command = None 406 filter_test_command = None 407 408 # these are the flag defined by gtest and benchmark framework to list 409 # and filter test runs. We use them to split each individual test 410 # into its own JobSpec, and thus into its own process. 411 if "benchmark" in target and target["benchmark"]: 412 with open(os.devnull, "w") as fnull: 413 tests = subprocess.check_output( 414 [binary, "--benchmark_list_tests"], stderr=fnull 415 ) 416 for line in tests.decode().split("\n"): 417 test = line.strip() 418 if not test: 419 continue 420 cmdline = [ 421 binary, 422 "--benchmark_filter=%s$" % test, 423 ] + target["args"] 424 out.append( 425 self.config.job_spec( 426 cmdline, 427 shortname="%s %s" 428 % (" ".join(cmdline), shortname_ext), 429 cpu_cost=cpu_cost, 430 timeout_seconds=target.get( 431 "timeout_seconds", 432 _DEFAULT_TIMEOUT_SECONDS, 433 ) 434 * timeout_scaling, 435 environ=env, 436 ) 437 ) 438 elif "gtest" in target and target["gtest"]: 439 # here we parse the output of --gtest_list_tests to build up a complete 440 # list of the tests contained in a binary for each test, we then 441 # add a job to run, filtering for just that test. 442 with open(os.devnull, "w") as fnull: 443 tests = subprocess.check_output( 444 [binary, "--gtest_list_tests"], stderr=fnull 445 ) 446 base = None 447 for line in tests.decode().split("\n"): 448 i = line.find("#") 449 if i >= 0: 450 line = line[:i] 451 if not line: 452 continue 453 if line[0] != " ": 454 base = line.strip() 455 else: 456 assert base is not None 457 assert line[1] == " " 458 test = base + line.strip() 459 cmdline = [ 460 binary, 461 "--gtest_filter=%s" % test, 462 ] + target["args"] 463 out.append( 464 self.config.job_spec( 465 cmdline, 466 shortname="%s %s" 467 % (" ".join(cmdline), shortname_ext), 468 cpu_cost=cpu_cost, 469 timeout_seconds=target.get( 470 "timeout_seconds", 471 _DEFAULT_TIMEOUT_SECONDS, 472 ) 473 * timeout_scaling, 474 environ=env, 475 ) 476 ) 477 else: 478 cmdline = [binary] + target["args"] 479 shortname = target.get( 480 "shortname", 481 " ".join(shlex.quote(arg) for arg in cmdline), 482 ) 483 shortname += shortname_ext 484 out.append( 485 self.config.job_spec( 486 cmdline, 487 shortname=shortname, 488 cpu_cost=cpu_cost, 489 flaky=target.get("flaky", False), 490 timeout_seconds=target.get( 491 "timeout_seconds", _DEFAULT_TIMEOUT_SECONDS 492 ) 493 * timeout_scaling, 494 environ=env, 495 ) 496 ) 497 elif self.args.regex == ".*" or self.platform == "windows": 498 print("\nWARNING: binary not found, skipping", binary) 499 return sorted(out) 500 501 def pre_build_steps(self): 502 return [] 503 504 def build_steps(self): 505 if self.platform == "windows": 506 return [ 507 [ 508 "tools\\run_tests\\helper_scripts\\build_cxx.bat", 509 "-DgRPC_BUILD_MSVC_MP_COUNT=%d" % self.args.jobs, 510 ] 511 + self._cmake_configure_extra_args 512 ] 513 else: 514 return [ 515 ["tools/run_tests/helper_scripts/build_cxx.sh"] 516 + self._cmake_configure_extra_args 517 ] 518 519 def build_steps_environ(self): 520 """Extra environment variables set for pre_build_steps and build_steps jobs.""" 521 environ = {"GRPC_RUN_TESTS_CXX_LANGUAGE_SUFFIX": self.lang_suffix} 522 if self.platform == "windows": 523 environ["GRPC_CMAKE_GENERATOR"] = self._cmake_generator_windows 524 environ[ 525 "GRPC_CMAKE_ARCHITECTURE" 526 ] = self._cmake_architecture_windows 527 environ[ 528 "GRPC_BUILD_ACTIVATE_VS_TOOLS" 529 ] = self._activate_vs_tools_windows 530 environ[ 531 "GRPC_BUILD_VS_TOOLS_ARCHITECTURE" 532 ] = self._vs_tools_architecture_windows 533 elif self.platform == "linux": 534 environ["GRPC_RUNTESTS_ARCHITECTURE"] = self.args.arch 535 return environ 536 537 def post_tests_steps(self): 538 if self.platform == "windows": 539 return [] 540 else: 541 return [["tools/run_tests/helper_scripts/post_tests_c.sh"]] 542 543 def _clang_cmake_configure_extra_args(self, version_suffix=""): 544 return [ 545 "-DCMAKE_C_COMPILER=clang%s" % version_suffix, 546 "-DCMAKE_CXX_COMPILER=clang++%s" % version_suffix, 547 ] 548 549 def _compiler_options( 550 self, use_docker, compiler, cmake_configure_extra_args 551 ): 552 """Returns docker distro and cmake configure args to use for given compiler.""" 553 if cmake_configure_extra_args: 554 # only allow specifying extra cmake args for "vanilla" compiler 555 _check_compiler(compiler, ["default", "cmake"]) 556 return ("nonexistent_docker_distro", cmake_configure_extra_args) 557 if not use_docker and not _is_use_docker_child(): 558 # if not running under docker, we cannot ensure the right compiler version will be used, 559 # so we only allow the non-specific choices. 560 _check_compiler(compiler, ["default", "cmake"]) 561 562 if compiler == "default" or compiler == "cmake": 563 return ("debian11", ["-DCMAKE_CXX_STANDARD=17"]) 564 elif compiler == "gcc8": 565 return ("gcc_8", ["-DCMAKE_CXX_STANDARD=17"]) 566 elif compiler == "gcc10.2": 567 return ("debian11", ["-DCMAKE_CXX_STANDARD=17"]) 568 elif compiler == "gcc10.2_openssl102": 569 return ( 570 "debian11_openssl102", 571 [ 572 "-DgRPC_SSL_PROVIDER=package", 573 "-DCMAKE_CXX_STANDARD=17", 574 ], 575 ) 576 elif compiler == "gcc10.2_openssl111": 577 return ( 578 "debian11_openssl111", 579 [ 580 "-DgRPC_SSL_PROVIDER=package", 581 "-DCMAKE_CXX_STANDARD=17", 582 ], 583 ) 584 elif compiler == "gcc12_openssl309": 585 return ( 586 "debian12_openssl309", 587 [ 588 "-DgRPC_SSL_PROVIDER=package", 589 "-DCMAKE_CXX_STANDARD=17", 590 ], 591 ) 592 elif compiler == "gcc14": 593 return ("gcc_14", ["-DCMAKE_CXX_STANDARD=20"]) 594 elif compiler == "gcc_musl": 595 return ("alpine", ["-DCMAKE_CXX_STANDARD=17"]) 596 elif compiler == "clang7": 597 return ( 598 "clang_7", 599 self._clang_cmake_configure_extra_args() 600 + "-DCMAKE_CXX_STANDARD=17", 601 ) 602 elif compiler == "clang19": 603 return ( 604 "clang_19", 605 self._clang_cmake_configure_extra_args() 606 + "-DCMAKE_CXX_STANDARD=17", 607 ) 608 else: 609 raise Exception("Compiler %s not supported." % compiler) 610 611 def dockerfile_dir(self): 612 return "tools/dockerfile/test/cxx_%s_%s" % ( 613 self._docker_distro, 614 _docker_arch_suffix(self.args.arch), 615 ) 616 617 def __str__(self): 618 return self.lang_suffix 619 620 621class Php8Language(object): 622 def configure(self, config, args): 623 self.config = config 624 self.args = args 625 _check_compiler(self.args.compiler, ["default"]) 626 627 def test_specs(self): 628 return [ 629 self.config.job_spec( 630 ["src/php/bin/run_tests.sh"], 631 environ=_FORCE_ENVIRON_FOR_WRAPPERS, 632 ) 633 ] 634 635 def pre_build_steps(self): 636 return [] 637 638 def build_steps(self): 639 return [["tools/run_tests/helper_scripts/build_php.sh"]] 640 641 def build_steps_environ(self): 642 """Extra environment variables set for pre_build_steps and build_steps jobs.""" 643 return {} 644 645 def post_tests_steps(self): 646 return [["tools/run_tests/helper_scripts/post_tests_php.sh"]] 647 648 def dockerfile_dir(self): 649 return "tools/dockerfile/test/php8_debian12_%s" % _docker_arch_suffix( 650 self.args.arch 651 ) 652 653 def __str__(self): 654 return "php8" 655 656 657class PythonConfig( 658 collections.namedtuple( 659 "PythonConfig", ["name", "build", "run", "python_path"] 660 ) 661): 662 """Tuple of commands (named s.t. 'what it says on the tin' applies)""" 663 664 665class PythonLanguage(object): 666 _TEST_SPECS_FILE = { 667 "native": ["src/python/grpcio_tests/tests/tests.json"], 668 "asyncio": ["src/python/grpcio_tests/tests_aio/tests.json"], 669 } 670 671 _TEST_COMMAND = { 672 "native": "test_lite", 673 "asyncio": "test_aio", 674 } 675 676 def configure(self, config, args): 677 self.config = config 678 self.args = args 679 self.pythons = self._get_pythons(self.args) 680 681 def test_specs(self): 682 # load list of known test suites 683 jobs = [] 684 685 # Run tests across all supported interpreters. 686 for python_config in self.pythons: 687 # Run non-io-manager-specific tests. 688 if os.name != "nt": 689 jobs.append( 690 self.config.job_spec( 691 [ 692 python_config.python_path, 693 "tools/distrib/python/xds_protos/generated_file_import_test.py", 694 ], 695 timeout_seconds=60, 696 environ=_FORCE_ENVIRON_FOR_WRAPPERS, 697 shortname=f"{python_config.name}.xds_protos", 698 ) 699 ) 700 701 # Run main test suite across all support IO managers. 702 for io_platform in self._TEST_SPECS_FILE: 703 test_cases = [] 704 for tests_json_file_name in self._TEST_SPECS_FILE[io_platform]: 705 with open(tests_json_file_name) as tests_json_file: 706 test_cases.extend(json.load(tests_json_file)) 707 708 environment = dict(_FORCE_ENVIRON_FOR_WRAPPERS) 709 # TODO(https://github.com/grpc/grpc/issues/21401) Fork handlers is not 710 # designed for non-native IO manager. It has a side-effect that 711 # overrides threading settings in C-Core. 712 if io_platform != "native": 713 environment["GRPC_ENABLE_FORK_SUPPORT"] = "0" 714 jobs.extend( 715 [ 716 self.config.job_spec( 717 python_config.run 718 + [self._TEST_COMMAND[io_platform]], 719 timeout_seconds=10 * 60, 720 environ=dict( 721 GRPC_PYTHON_TESTRUNNER_FILTER=str(test_case), 722 **environment, 723 ), 724 shortname=f"{python_config.name}.{io_platform}.{test_case}", 725 ) 726 for test_case in test_cases 727 ] 728 ) 729 return jobs 730 731 def pre_build_steps(self): 732 return [] 733 734 def build_steps(self): 735 return [config.build for config in self.pythons] 736 737 def build_steps_environ(self): 738 """Extra environment variables set for pre_build_steps and build_steps jobs.""" 739 return {} 740 741 def post_tests_steps(self): 742 if self.config.build_config != "gcov": 743 return [] 744 else: 745 return [["tools/run_tests/helper_scripts/post_tests_python.sh"]] 746 747 def dockerfile_dir(self): 748 return "tools/dockerfile/test/python_%s_%s" % ( 749 self._python_docker_distro_name(), 750 _docker_arch_suffix(self.args.arch), 751 ) 752 753 def _python_docker_distro_name(self): 754 """Choose the docker image to use based on python version.""" 755 if self.args.compiler == "python_alpine": 756 return "alpine" 757 else: 758 return "debian11_default" 759 760 def _get_pythons(self, args): 761 """Get python runtimes to test with, based on current platform, architecture, compiler etc.""" 762 if args.iomgr_platform != "native": 763 raise ValueError( 764 "Python builds no longer differentiate IO Manager platforms," 765 ' please use "native"' 766 ) 767 768 if args.arch == "x86": 769 bits = "32" 770 else: 771 bits = "64" 772 773 if os.name == "nt": 774 shell = ["bash"] 775 builder = [ 776 os.path.abspath( 777 "tools/run_tests/helper_scripts/build_python_msys2.sh" 778 ) 779 ] 780 builder_prefix_arguments = ["MINGW{}".format(bits)] 781 venv_relative_python = ["Scripts/python.exe"] 782 toolchain = ["mingw32"] 783 else: 784 shell = [] 785 builder = [ 786 os.path.abspath( 787 "tools/run_tests/helper_scripts/build_python.sh" 788 ) 789 ] 790 builder_prefix_arguments = [] 791 venv_relative_python = ["bin/python"] 792 toolchain = ["unix"] 793 794 runner = [ 795 os.path.abspath("tools/run_tests/helper_scripts/run_python.sh") 796 ] 797 798 config_vars = _PythonConfigVars( 799 shell, 800 builder, 801 builder_prefix_arguments, 802 venv_relative_python, 803 toolchain, 804 runner, 805 ) 806 807 # TODO: Supported version range should be defined by a single 808 # source of truth. 809 python38_config = _python_config_generator( 810 name="py38", 811 major="3", 812 minor="8", 813 bits=bits, 814 config_vars=config_vars, 815 ) 816 python39_config = _python_config_generator( 817 name="py39", 818 major="3", 819 minor="9", 820 bits=bits, 821 config_vars=config_vars, 822 ) 823 python310_config = _python_config_generator( 824 name="py310", 825 major="3", 826 minor="10", 827 bits=bits, 828 config_vars=config_vars, 829 ) 830 python311_config = _python_config_generator( 831 name="py311", 832 major="3", 833 minor="11", 834 bits=bits, 835 config_vars=config_vars, 836 ) 837 python312_config = _python_config_generator( 838 name="py312", 839 major="3", 840 minor="12", 841 bits=bits, 842 config_vars=config_vars, 843 ) 844 python313_config = _python_config_generator( 845 name="py313", 846 major="3", 847 minor="13", 848 bits=bits, 849 config_vars=config_vars, 850 ) 851 pypy27_config = _pypy_config_generator( 852 name="pypy", major="2", config_vars=config_vars 853 ) 854 pypy32_config = _pypy_config_generator( 855 name="pypy3", major="3", config_vars=config_vars 856 ) 857 858 if args.compiler == "default": 859 if os.name == "nt": 860 return (python38_config,) 861 elif os.uname()[0] == "Darwin": 862 # NOTE(rbellevi): Testing takes significantly longer on 863 # MacOS, so we restrict the number of interpreter versions 864 # tested. 865 return (python38_config,) 866 elif platform.machine() == "aarch64": 867 # Currently the python_debian11_default_arm64 docker image 868 # only has python3.9 installed (and that seems sufficient 869 # for arm64 testing) 870 return (python39_config,) 871 else: 872 # Default set tested on master. Test oldest and newest. 873 return ( 874 python38_config, 875 python313_config, 876 ) 877 elif args.compiler == "python3.8": 878 return (python38_config,) 879 elif args.compiler == "python3.9": 880 return (python39_config,) 881 elif args.compiler == "python3.10": 882 return (python310_config,) 883 elif args.compiler == "python3.11": 884 return (python311_config,) 885 elif args.compiler == "python3.12": 886 return (python312_config,) 887 elif args.compiler == "python3.13": 888 return (python313_config,) 889 elif args.compiler == "pypy": 890 return (pypy27_config,) 891 elif args.compiler == "pypy3": 892 return (pypy32_config,) 893 elif args.compiler == "python_alpine": 894 return (python310_config,) 895 elif args.compiler == "all_the_cpythons": 896 return ( 897 python38_config, 898 python39_config, 899 python310_config, 900 python311_config, 901 python312_config, 902 python313_config, 903 ) 904 else: 905 raise Exception("Compiler %s not supported." % args.compiler) 906 907 def __str__(self): 908 return "python" 909 910 911class RubyLanguage(object): 912 def configure(self, config, args): 913 self.config = config 914 self.args = args 915 _check_compiler(self.args.compiler, ["default"]) 916 917 def test_specs(self): 918 tests = [] 919 for test in [ 920 "src/ruby/spec/google_rpc_status_utils_spec.rb", 921 "src/ruby/spec/client_server_spec.rb", 922 "src/ruby/spec/errors_spec.rb", 923 "src/ruby/spec/pb/codegen/package_option_spec.rb", 924 "src/ruby/spec/pb/health/checker_spec.rb", 925 "src/ruby/spec/pb/duplicate/codegen_spec.rb", 926 "src/ruby/spec/server_spec.rb", 927 "src/ruby/spec/error_sanity_spec.rb", 928 "src/ruby/spec/channel_spec.rb", 929 "src/ruby/spec/user_agent_spec.rb", 930 "src/ruby/spec/call_credentials_spec.rb", 931 "src/ruby/spec/channel_credentials_spec.rb", 932 "src/ruby/spec/channel_connection_spec.rb", 933 "src/ruby/spec/compression_options_spec.rb", 934 "src/ruby/spec/time_consts_spec.rb", 935 "src/ruby/spec/server_credentials_spec.rb", 936 "src/ruby/spec/generic/server_interceptors_spec.rb", 937 "src/ruby/spec/generic/rpc_server_pool_spec.rb", 938 "src/ruby/spec/generic/client_stub_spec.rb", 939 "src/ruby/spec/generic/active_call_spec.rb", 940 "src/ruby/spec/generic/rpc_server_spec.rb", 941 "src/ruby/spec/generic/service_spec.rb", 942 "src/ruby/spec/generic/client_interceptors_spec.rb", 943 "src/ruby/spec/generic/rpc_desc_spec.rb", 944 "src/ruby/spec/generic/interceptor_registry_spec.rb", 945 "src/ruby/spec/debug_message_spec.rb", 946 "src/ruby/spec/logconfig_spec.rb", 947 "src/ruby/spec/call_spec.rb", 948 "src/ruby/spec/client_auth_spec.rb", 949 ]: 950 tests.append( 951 self.config.job_spec( 952 ["rspec", test], 953 shortname=test, 954 timeout_seconds=20 * 60, 955 environ=_FORCE_ENVIRON_FOR_WRAPPERS, 956 ) 957 ) 958 # TODO(apolcyn): re-enable the following tests after 959 # https://bugs.ruby-lang.org/issues/15499 is fixed: 960 # They previously worked on ruby 2.5 but needed to be disabled 961 # after dropping support for ruby 2.5: 962 # - src/ruby/end2end/channel_state_test.rb 963 # - src/ruby/end2end/sig_int_during_channel_watch_test.rb 964 # TODO(apolcyn): the following test is skipped because it sometimes 965 # hits "Bus Error" crashes while requiring the grpc/ruby C-extension. 966 # This crashes have been unreproducible outside of CI. Also see 967 # b/266212253. 968 # - src/ruby/end2end/grpc_class_init_test.rb 969 # - src/ruby/end2end/load_grpc_with_gc_stress_test.rb 970 for test in [ 971 "src/ruby/end2end/fork_test.rb", 972 "src/ruby/end2end/simple_fork_test.rb", 973 "src/ruby/end2end/prefork_without_using_grpc_test.rb", 974 "src/ruby/end2end/prefork_postfork_loop_test.rb", 975 "src/ruby/end2end/secure_fork_test.rb", 976 "src/ruby/end2end/bad_usage_fork_test.rb", 977 "src/ruby/end2end/sig_handling_test.rb", 978 "src/ruby/end2end/channel_closing_test.rb", 979 "src/ruby/end2end/killed_client_thread_test.rb", 980 "src/ruby/end2end/forking_client_test.rb", 981 "src/ruby/end2end/multiple_killed_watching_threads_test.rb", 982 "src/ruby/end2end/client_memory_usage_test.rb", 983 "src/ruby/end2end/package_with_underscore_test.rb", 984 "src/ruby/end2end/graceful_sig_handling_test.rb", 985 "src/ruby/end2end/graceful_sig_stop_test.rb", 986 "src/ruby/end2end/errors_load_before_grpc_lib_test.rb", 987 "src/ruby/end2end/logger_load_before_grpc_lib_test.rb", 988 "src/ruby/end2end/status_codes_load_before_grpc_lib_test.rb", 989 "src/ruby/end2end/call_credentials_timeout_test.rb", 990 "src/ruby/end2end/call_credentials_returning_bad_metadata_doesnt_kill_background_thread_test.rb", 991 ]: 992 if test in [ 993 "src/ruby/end2end/fork_test.rb", 994 "src/ruby/end2end/simple_fork_test.rb", 995 "src/ruby/end2end/secure_fork_test.rb", 996 "src/ruby/end2end/bad_usage_fork_test.rb", 997 "src/ruby/end2end/prefork_without_using_grpc_test.rb", 998 "src/ruby/end2end/prefork_postfork_loop_test.rb", 999 "src/ruby/end2end/fork_test_repro_35489.rb", 1000 ]: 1001 # Skip fork tests in general until https://github.com/grpc/grpc/issues/34442 1002 # is fixed. Otherwise we see too many flakes. 1003 # After that's fixed, we should continue to skip on mac 1004 # indefinitely, and on "dbg" builds until the Event Engine 1005 # migration completes. 1006 continue 1007 tests.append( 1008 self.config.job_spec( 1009 ["ruby", test], 1010 shortname=test, 1011 timeout_seconds=20 * 60, 1012 environ=_FORCE_ENVIRON_FOR_WRAPPERS, 1013 ) 1014 ) 1015 return tests 1016 1017 def pre_build_steps(self): 1018 return [["tools/run_tests/helper_scripts/pre_build_ruby.sh"]] 1019 1020 def build_steps(self): 1021 return [["tools/run_tests/helper_scripts/build_ruby.sh"]] 1022 1023 def build_steps_environ(self): 1024 """Extra environment variables set for pre_build_steps and build_steps jobs.""" 1025 return {} 1026 1027 def post_tests_steps(self): 1028 return [["tools/run_tests/helper_scripts/post_tests_ruby.sh"]] 1029 1030 def dockerfile_dir(self): 1031 return "tools/dockerfile/test/ruby_debian11_%s" % _docker_arch_suffix( 1032 self.args.arch 1033 ) 1034 1035 def __str__(self): 1036 return "ruby" 1037 1038 1039class CSharpLanguage(object): 1040 def __init__(self): 1041 self.platform = platform_string() 1042 1043 def configure(self, config, args): 1044 self.config = config 1045 self.args = args 1046 _check_compiler(self.args.compiler, ["default", "coreclr", "mono"]) 1047 if self.args.compiler == "default": 1048 # test both runtimes by default 1049 self.test_runtimes = ["coreclr", "mono"] 1050 else: 1051 # only test the specified runtime 1052 self.test_runtimes = [self.args.compiler] 1053 1054 if self.platform == "windows": 1055 _check_arch(self.args.arch, ["default"]) 1056 self._cmake_arch_option = "x64" 1057 else: 1058 self._docker_distro = "debian11" 1059 1060 def test_specs(self): 1061 with open("src/csharp/tests.json") as f: 1062 tests_by_assembly = json.load(f) 1063 1064 msbuild_config = _MSBUILD_CONFIG[self.config.build_config] 1065 nunit_args = ["--labels=All", "--noresult", "--workers=1"] 1066 1067 specs = [] 1068 for test_runtime in self.test_runtimes: 1069 if test_runtime == "coreclr": 1070 assembly_extension = ".dll" 1071 assembly_subdir = "bin/%s/netcoreapp3.1" % msbuild_config 1072 runtime_cmd = ["dotnet", "exec"] 1073 elif test_runtime == "mono": 1074 assembly_extension = ".exe" 1075 assembly_subdir = "bin/%s/net45" % msbuild_config 1076 if self.platform == "windows": 1077 runtime_cmd = [] 1078 elif self.platform == "mac": 1079 # mono before version 5.2 on MacOS defaults to 32bit runtime 1080 runtime_cmd = ["mono", "--arch=64"] 1081 else: 1082 runtime_cmd = ["mono"] 1083 else: 1084 raise Exception('Illegal runtime "%s" was specified.') 1085 1086 for assembly in six.iterkeys(tests_by_assembly): 1087 assembly_file = "src/csharp/%s/%s/%s%s" % ( 1088 assembly, 1089 assembly_subdir, 1090 assembly, 1091 assembly_extension, 1092 ) 1093 1094 # normally, run each test as a separate process 1095 for test in tests_by_assembly[assembly]: 1096 cmdline = ( 1097 runtime_cmd 1098 + [assembly_file, "--test=%s" % test] 1099 + nunit_args 1100 ) 1101 specs.append( 1102 self.config.job_spec( 1103 cmdline, 1104 shortname="csharp.%s.%s" % (test_runtime, test), 1105 environ=_FORCE_ENVIRON_FOR_WRAPPERS, 1106 ) 1107 ) 1108 return specs 1109 1110 def pre_build_steps(self): 1111 if self.platform == "windows": 1112 return [["tools\\run_tests\\helper_scripts\\pre_build_csharp.bat"]] 1113 else: 1114 return [["tools/run_tests/helper_scripts/pre_build_csharp.sh"]] 1115 1116 def build_steps(self): 1117 if self.platform == "windows": 1118 return [["tools\\run_tests\\helper_scripts\\build_csharp.bat"]] 1119 else: 1120 return [["tools/run_tests/helper_scripts/build_csharp.sh"]] 1121 1122 def build_steps_environ(self): 1123 """Extra environment variables set for pre_build_steps and build_steps jobs.""" 1124 if self.platform == "windows": 1125 return {"ARCHITECTURE": self._cmake_arch_option} 1126 else: 1127 return {} 1128 1129 def post_tests_steps(self): 1130 if self.platform == "windows": 1131 return [["tools\\run_tests\\helper_scripts\\post_tests_csharp.bat"]] 1132 else: 1133 return [["tools/run_tests/helper_scripts/post_tests_csharp.sh"]] 1134 1135 def dockerfile_dir(self): 1136 return "tools/dockerfile/test/csharp_%s_%s" % ( 1137 self._docker_distro, 1138 _docker_arch_suffix(self.args.arch), 1139 ) 1140 1141 def __str__(self): 1142 return "csharp" 1143 1144 1145class ObjCLanguage(object): 1146 def configure(self, config, args): 1147 self.config = config 1148 self.args = args 1149 _check_compiler(self.args.compiler, ["default"]) 1150 1151 def test_specs(self): 1152 out = [] 1153 out.append( 1154 self.config.job_spec( 1155 ["src/objective-c/tests/build_one_example.sh"], 1156 timeout_seconds=60 * 60, 1157 shortname="ios-buildtest-example-sample", 1158 cpu_cost=1e6, 1159 environ={ 1160 "SCHEME": "Sample", 1161 "EXAMPLE_PATH": "src/objective-c/examples/Sample", 1162 }, 1163 ) 1164 ) 1165 # TODO(jtattermusch): Create bazel target for the sample and remove the test task from here. 1166 out.append( 1167 self.config.job_spec( 1168 ["src/objective-c/tests/build_one_example.sh"], 1169 timeout_seconds=60 * 60, 1170 shortname="ios-buildtest-example-switftsample", 1171 cpu_cost=1e6, 1172 environ={ 1173 "SCHEME": "SwiftSample", 1174 "EXAMPLE_PATH": "src/objective-c/examples/SwiftSample", 1175 }, 1176 ) 1177 ) 1178 out.append( 1179 self.config.job_spec( 1180 ["src/objective-c/tests/build_one_example.sh"], 1181 timeout_seconds=60 * 60, 1182 shortname="ios-buildtest-example-switft-use-frameworks", 1183 cpu_cost=1e6, 1184 environ={ 1185 "SCHEME": "SwiftUseFrameworks", 1186 "EXAMPLE_PATH": "src/objective-c/examples/SwiftUseFrameworks", 1187 }, 1188 ) 1189 ) 1190 1191 # Disabled due to #20258 1192 # TODO (mxyan): Reenable this test when #20258 is resolved. 1193 # out.append( 1194 # self.config.job_spec( 1195 # ['src/objective-c/tests/build_one_example_bazel.sh'], 1196 # timeout_seconds=20 * 60, 1197 # shortname='ios-buildtest-example-watchOS-sample', 1198 # cpu_cost=1e6, 1199 # environ={ 1200 # 'SCHEME': 'watchOS-sample-WatchKit-App', 1201 # 'EXAMPLE_PATH': 'src/objective-c/examples/watchOS-sample', 1202 # 'FRAMEWORKS': 'NO' 1203 # })) 1204 1205 # TODO(jtattermusch): move the test out of the test/core/iomgr/CFStreamTests directory? 1206 # How does one add the cfstream dependency in bazel? 1207 # Disabled due to flakiness and being replaced with event engine 1208 # out.append( 1209 # self.config.job_spec( 1210 # ["test/core/iomgr/ios/CFStreamTests/build_and_run_tests.sh"], 1211 # timeout_seconds=60 * 60, 1212 # shortname="ios-test-cfstream-tests", 1213 # cpu_cost=1e6, 1214 # environ=_FORCE_ENVIRON_FOR_WRAPPERS, 1215 # ) 1216 # ) 1217 return sorted(out) 1218 1219 def pre_build_steps(self): 1220 return [] 1221 1222 def build_steps(self): 1223 return [] 1224 1225 def build_steps_environ(self): 1226 """Extra environment variables set for pre_build_steps and build_steps jobs.""" 1227 return {} 1228 1229 def post_tests_steps(self): 1230 return [] 1231 1232 def dockerfile_dir(self): 1233 return None 1234 1235 def __str__(self): 1236 return "objc" 1237 1238 1239class Sanity(object): 1240 def __init__(self, config_file): 1241 self.config_file = config_file 1242 1243 def configure(self, config, args): 1244 self.config = config 1245 self.args = args 1246 _check_compiler(self.args.compiler, ["default"]) 1247 1248 def test_specs(self): 1249 import yaml 1250 1251 with open("tools/run_tests/sanity/%s" % self.config_file, "r") as f: 1252 environ = {"TEST": "true"} 1253 if _is_use_docker_child(): 1254 environ["CLANG_FORMAT_SKIP_DOCKER"] = "true" 1255 environ["CLANG_TIDY_SKIP_DOCKER"] = "true" 1256 environ["IWYU_SKIP_DOCKER"] = "true" 1257 # sanity tests run tools/bazel wrapper concurrently 1258 # and that can result in a download/run race in the wrapper. 1259 # under docker we already have the right version of bazel 1260 # so we can just disable the wrapper. 1261 environ["DISABLE_BAZEL_WRAPPER"] = "true" 1262 return [ 1263 self.config.job_spec( 1264 cmd["script"].split(), 1265 timeout_seconds=45 * 60, 1266 environ=environ, 1267 cpu_cost=cmd.get("cpu_cost", 1), 1268 ) 1269 for cmd in yaml.safe_load(f) 1270 ] 1271 1272 def pre_build_steps(self): 1273 return [] 1274 1275 def build_steps(self): 1276 return [] 1277 1278 def build_steps_environ(self): 1279 """Extra environment variables set for pre_build_steps and build_steps jobs.""" 1280 return {} 1281 1282 def post_tests_steps(self): 1283 return [] 1284 1285 def dockerfile_dir(self): 1286 return "tools/dockerfile/test/sanity" 1287 1288 def __str__(self): 1289 return "sanity" 1290 1291 1292# different configurations we can run under 1293with open("tools/run_tests/generated/configs.json") as f: 1294 _CONFIGS = dict( 1295 (cfg["config"], Config(**cfg)) for cfg in ast.literal_eval(f.read()) 1296 ) 1297 1298_LANGUAGES = { 1299 "c++": CLanguage("cxx", "c++"), 1300 "c": CLanguage("c", "c"), 1301 "php8": Php8Language(), 1302 "python": PythonLanguage(), 1303 "ruby": RubyLanguage(), 1304 "csharp": CSharpLanguage(), 1305 "objc": ObjCLanguage(), 1306 "sanity": Sanity("sanity_tests.yaml"), 1307 "clang-tidy": Sanity("clang_tidy_tests.yaml"), 1308} 1309 1310_MSBUILD_CONFIG = { 1311 "dbg": "Debug", 1312 "opt": "Release", 1313 "gcov": "Debug", 1314} 1315 1316 1317def _build_step_environ(cfg, extra_env={}): 1318 """Environment variables set for each build step.""" 1319 environ = {"CONFIG": cfg, "GRPC_RUN_TESTS_JOBS": str(args.jobs)} 1320 msbuild_cfg = _MSBUILD_CONFIG.get(cfg) 1321 if msbuild_cfg: 1322 environ["MSBUILD_CONFIG"] = msbuild_cfg 1323 environ.update(extra_env) 1324 return environ 1325 1326 1327def _windows_arch_option(arch): 1328 """Returns msbuild cmdline option for selected architecture.""" 1329 if arch == "default" or arch == "x86": 1330 return "/p:Platform=Win32" 1331 elif arch == "x64": 1332 return "/p:Platform=x64" 1333 else: 1334 print("Architecture %s not supported." % arch) 1335 sys.exit(1) 1336 1337 1338def _check_arch_option(arch): 1339 """Checks that architecture option is valid.""" 1340 if platform_string() == "windows": 1341 _windows_arch_option(arch) 1342 elif platform_string() == "linux": 1343 # On linux, we need to be running under docker with the right architecture. 1344 runtime_machine = platform.machine() 1345 runtime_arch = platform.architecture()[0] 1346 if arch == "default": 1347 return 1348 elif ( 1349 runtime_machine == "x86_64" 1350 and runtime_arch == "64bit" 1351 and arch == "x64" 1352 ): 1353 return 1354 elif ( 1355 runtime_machine == "x86_64" 1356 and runtime_arch == "32bit" 1357 and arch == "x86" 1358 ): 1359 return 1360 elif ( 1361 runtime_machine == "aarch64" 1362 and runtime_arch == "64bit" 1363 and arch == "arm64" 1364 ): 1365 return 1366 else: 1367 print( 1368 "Architecture %s does not match current runtime architecture." 1369 % arch 1370 ) 1371 sys.exit(1) 1372 else: 1373 if args.arch != "default": 1374 print( 1375 "Architecture %s not supported on current platform." % args.arch 1376 ) 1377 sys.exit(1) 1378 1379 1380def _docker_arch_suffix(arch): 1381 """Returns suffix to dockerfile dir to use.""" 1382 if arch == "default" or arch == "x64": 1383 return "x64" 1384 elif arch == "x86": 1385 return "x86" 1386 elif arch == "arm64": 1387 return "arm64" 1388 else: 1389 print("Architecture %s not supported with current settings." % arch) 1390 sys.exit(1) 1391 1392 1393def runs_per_test_type(arg_str): 1394 """Auxiliary function to parse the "runs_per_test" flag. 1395 1396 Returns: 1397 A positive integer or 0, the latter indicating an infinite number of 1398 runs. 1399 1400 Raises: 1401 argparse.ArgumentTypeError: Upon invalid input. 1402 """ 1403 if arg_str == "inf": 1404 return 0 1405 try: 1406 n = int(arg_str) 1407 if n <= 0: 1408 raise ValueError 1409 return n 1410 except: 1411 msg = "'{}' is not a positive integer or 'inf'".format(arg_str) 1412 raise argparse.ArgumentTypeError(msg) 1413 1414 1415def percent_type(arg_str): 1416 pct = float(arg_str) 1417 if pct > 100 or pct < 0: 1418 raise argparse.ArgumentTypeError( 1419 "'%f' is not a valid percentage in the [0, 100] range" % pct 1420 ) 1421 return pct 1422 1423 1424# This is math.isclose in python >= 3.5 1425def isclose(a, b, rel_tol=1e-09, abs_tol=0.0): 1426 return abs(a - b) <= max(rel_tol * max(abs(a), abs(b)), abs_tol) 1427 1428 1429def _shut_down_legacy_server(legacy_server_port): 1430 """Shut down legacy version of port server.""" 1431 try: 1432 version = int( 1433 urllib.request.urlopen( 1434 "http://localhost:%d/version_number" % legacy_server_port, 1435 timeout=10, 1436 ).read() 1437 ) 1438 except: 1439 pass 1440 else: 1441 urllib.request.urlopen( 1442 "http://localhost:%d/quitquitquit" % legacy_server_port 1443 ).read() 1444 1445 1446def _calculate_num_runs_failures(list_of_results): 1447 """Calculate number of runs and failures for a particular test. 1448 1449 Args: 1450 list_of_results: (List) of JobResult object. 1451 Returns: 1452 A tuple of total number of runs and failures. 1453 """ 1454 num_runs = len(list_of_results) # By default, there is 1 run per JobResult. 1455 num_failures = 0 1456 for jobresult in list_of_results: 1457 if jobresult.retries > 0: 1458 num_runs += jobresult.retries 1459 if jobresult.num_failures > 0: 1460 num_failures += jobresult.num_failures 1461 return num_runs, num_failures 1462 1463 1464class BuildAndRunError(object): 1465 """Represents error type in _build_and_run.""" 1466 1467 BUILD = object() 1468 TEST = object() 1469 POST_TEST = object() 1470 1471 1472# returns a list of things that failed (or an empty list on success) 1473def _build_and_run( 1474 check_cancelled, newline_on_success, xml_report=None, build_only=False 1475): 1476 """Do one pass of building & running tests.""" 1477 # build latest sequentially 1478 num_failures, resultset = jobset.run( 1479 build_steps, 1480 maxjobs=1, 1481 stop_on_failure=True, 1482 newline_on_success=newline_on_success, 1483 travis=args.travis, 1484 ) 1485 if num_failures: 1486 return [BuildAndRunError.BUILD] 1487 1488 if build_only: 1489 if xml_report: 1490 report_utils.render_junit_xml_report( 1491 resultset, xml_report, suite_name=args.report_suite_name 1492 ) 1493 return [] 1494 1495 # start antagonists 1496 antagonists = [ 1497 subprocess.Popen(["tools/run_tests/python_utils/antagonist.py"]) 1498 for _ in range(0, args.antagonists) 1499 ] 1500 start_port_server.start_port_server() 1501 resultset = None 1502 num_test_failures = 0 1503 try: 1504 infinite_runs = runs_per_test == 0 1505 one_run = set( 1506 spec 1507 for language in languages 1508 for spec in language.test_specs() 1509 if ( 1510 re.search(args.regex, spec.shortname) 1511 and ( 1512 args.regex_exclude == "" 1513 or not re.search(args.regex_exclude, spec.shortname) 1514 ) 1515 ) 1516 ) 1517 # When running on travis, we want out test runs to be as similar as possible 1518 # for reproducibility purposes. 1519 if args.travis and args.max_time <= 0: 1520 massaged_one_run = sorted(one_run, key=lambda x: x.cpu_cost) 1521 else: 1522 # whereas otherwise, we want to shuffle things up to give all tests a 1523 # chance to run. 1524 massaged_one_run = list( 1525 one_run 1526 ) # random.sample needs an indexable seq. 1527 num_jobs = len(massaged_one_run) 1528 # for a random sample, get as many as indicated by the 'sample_percent' 1529 # argument. By default this arg is 100, resulting in a shuffle of all 1530 # jobs. 1531 sample_size = int(num_jobs * args.sample_percent / 100.0) 1532 massaged_one_run = random.sample(massaged_one_run, sample_size) 1533 if not isclose(args.sample_percent, 100.0): 1534 assert ( 1535 args.runs_per_test == 1 1536 ), "Can't do sampling (-p) over multiple runs (-n)." 1537 print( 1538 "Running %d tests out of %d (~%d%%)" 1539 % (sample_size, num_jobs, args.sample_percent) 1540 ) 1541 if infinite_runs: 1542 assert ( 1543 len(massaged_one_run) > 0 1544 ), "Must have at least one test for a -n inf run" 1545 runs_sequence = ( 1546 itertools.repeat(massaged_one_run) 1547 if infinite_runs 1548 else itertools.repeat(massaged_one_run, runs_per_test) 1549 ) 1550 all_runs = itertools.chain.from_iterable(runs_sequence) 1551 1552 if args.quiet_success: 1553 jobset.message( 1554 "START", 1555 "Running tests quietly, only failing tests will be reported", 1556 do_newline=True, 1557 ) 1558 num_test_failures, resultset = jobset.run( 1559 all_runs, 1560 check_cancelled, 1561 newline_on_success=newline_on_success, 1562 travis=args.travis, 1563 maxjobs=args.jobs, 1564 maxjobs_cpu_agnostic=max_parallel_tests_for_current_platform(), 1565 stop_on_failure=args.stop_on_failure, 1566 quiet_success=args.quiet_success, 1567 max_time=args.max_time, 1568 ) 1569 if resultset: 1570 for k, v in sorted(resultset.items()): 1571 num_runs, num_failures = _calculate_num_runs_failures(v) 1572 if num_failures > 0: 1573 if num_failures == num_runs: # what about infinite_runs??? 1574 jobset.message("FAILED", k, do_newline=True) 1575 else: 1576 jobset.message( 1577 "FLAKE", 1578 "%s [%d/%d runs flaked]" 1579 % (k, num_failures, num_runs), 1580 do_newline=True, 1581 ) 1582 finally: 1583 for antagonist in antagonists: 1584 antagonist.kill() 1585 if args.bq_result_table and resultset: 1586 upload_extra_fields = { 1587 "compiler": args.compiler, 1588 "config": args.config, 1589 "iomgr_platform": args.iomgr_platform, 1590 "language": args.language[ 1591 0 1592 ], # args.language is a list but will always have one element when uploading to BQ is enabled. 1593 "platform": platform_string(), 1594 } 1595 try: 1596 upload_results_to_bq( 1597 resultset, args.bq_result_table, upload_extra_fields 1598 ) 1599 except NameError as e: 1600 logging.warning( 1601 e 1602 ) # It's fine to ignore since this is not critical 1603 if xml_report and resultset: 1604 report_utils.render_junit_xml_report( 1605 resultset, 1606 xml_report, 1607 suite_name=args.report_suite_name, 1608 multi_target=args.report_multi_target, 1609 ) 1610 1611 number_failures, _ = jobset.run( 1612 post_tests_steps, 1613 maxjobs=1, 1614 stop_on_failure=False, 1615 newline_on_success=newline_on_success, 1616 travis=args.travis, 1617 ) 1618 1619 out = [] 1620 if number_failures: 1621 out.append(BuildAndRunError.POST_TEST) 1622 if num_test_failures: 1623 out.append(BuildAndRunError.TEST) 1624 1625 return out 1626 1627 1628# parse command line 1629argp = argparse.ArgumentParser(description="Run grpc tests.") 1630argp.add_argument( 1631 "-c", "--config", choices=sorted(_CONFIGS.keys()), default="opt" 1632) 1633argp.add_argument( 1634 "-n", 1635 "--runs_per_test", 1636 default=1, 1637 type=runs_per_test_type, 1638 help=( 1639 'A positive integer or "inf". If "inf", all tests will run in an ' 1640 'infinite loop. Especially useful in combination with "-f"' 1641 ), 1642) 1643argp.add_argument("-r", "--regex", default=".*", type=str) 1644argp.add_argument("--regex_exclude", default="", type=str) 1645argp.add_argument("-j", "--jobs", default=multiprocessing.cpu_count(), type=int) 1646argp.add_argument("-s", "--slowdown", default=1.0, type=float) 1647argp.add_argument( 1648 "-p", 1649 "--sample_percent", 1650 default=100.0, 1651 type=percent_type, 1652 help="Run a random sample with that percentage of tests", 1653) 1654argp.add_argument( 1655 "-t", 1656 "--travis", 1657 default=False, 1658 action="store_const", 1659 const=True, 1660 help=( 1661 "When set, indicates that the script is running on CI (= not locally)." 1662 ), 1663) 1664argp.add_argument( 1665 "--newline_on_success", default=False, action="store_const", const=True 1666) 1667argp.add_argument( 1668 "-l", 1669 "--language", 1670 choices=sorted(_LANGUAGES.keys()), 1671 nargs="+", 1672 required=True, 1673) 1674argp.add_argument( 1675 "-S", "--stop_on_failure", default=False, action="store_const", const=True 1676) 1677argp.add_argument( 1678 "--use_docker", 1679 default=False, 1680 action="store_const", 1681 const=True, 1682 help="Run all the tests under docker. That provides " 1683 + "additional isolation and prevents the need to install " 1684 + "language specific prerequisites. Only available on Linux.", 1685) 1686argp.add_argument( 1687 "--allow_flakes", 1688 default=False, 1689 action="store_const", 1690 const=True, 1691 help=( 1692 "Allow flaky tests to show as passing (re-runs failed tests up to five" 1693 " times)" 1694 ), 1695) 1696argp.add_argument( 1697 "--arch", 1698 choices=["default", "x86", "x64", "arm64"], 1699 default="default", 1700 help=( 1701 'Selects architecture to target. For some platforms "default" is the' 1702 " only supported choice." 1703 ), 1704) 1705argp.add_argument( 1706 "--compiler", 1707 choices=[ 1708 "default", 1709 "gcc8", 1710 "gcc10.2", 1711 "gcc10.2_openssl102", 1712 "gcc10.2_openssl111", 1713 "gcc12_openssl309", 1714 "gcc14", 1715 "gcc_musl", 1716 "clang7", 1717 "clang19", 1718 # TODO: Automatically populate from supported version 1719 "python3.7", 1720 "python3.8", 1721 "python3.9", 1722 "python3.10", 1723 "python3.11", 1724 "python3.12", 1725 "pypy", 1726 "pypy3", 1727 "python_alpine", 1728 "all_the_cpythons", 1729 "coreclr", 1730 "cmake", 1731 "cmake_ninja_vs2022", 1732 "cmake_vs2022", 1733 "mono", 1734 ], 1735 default="default", 1736 help=( 1737 "Selects compiler to use. Allowed values depend on the platform and" 1738 " language." 1739 ), 1740) 1741argp.add_argument( 1742 "--iomgr_platform", 1743 choices=["native", "gevent", "asyncio"], 1744 default="native", 1745 help="Selects iomgr platform to build on", 1746) 1747argp.add_argument( 1748 "--build_only", 1749 default=False, 1750 action="store_const", 1751 const=True, 1752 help="Perform all the build steps but don't run any tests.", 1753) 1754argp.add_argument( 1755 "--measure_cpu_costs", 1756 default=False, 1757 action="store_const", 1758 const=True, 1759 help="Measure the cpu costs of tests", 1760) 1761argp.add_argument("-a", "--antagonists", default=0, type=int) 1762argp.add_argument( 1763 "-x", 1764 "--xml_report", 1765 default=None, 1766 type=str, 1767 help="Generates a JUnit-compatible XML report", 1768) 1769argp.add_argument( 1770 "--report_suite_name", 1771 default="tests", 1772 type=str, 1773 help="Test suite name to use in generated JUnit XML report", 1774) 1775argp.add_argument( 1776 "--report_multi_target", 1777 default=False, 1778 const=True, 1779 action="store_const", 1780 help=( 1781 "Generate separate XML report for each test job (Looks better in UIs)." 1782 ), 1783) 1784argp.add_argument( 1785 "--quiet_success", 1786 default=False, 1787 action="store_const", 1788 const=True, 1789 help=( 1790 "Don't print anything when a test passes. Passing tests also will not" 1791 " be reported in XML report. " 1792 ) 1793 + "Useful when running many iterations of each test (argument -n).", 1794) 1795argp.add_argument( 1796 "--force_default_poller", 1797 default=False, 1798 action="store_const", 1799 const=True, 1800 help="Don't try to iterate over many polling strategies when they exist", 1801) 1802argp.add_argument( 1803 "--force_use_pollers", 1804 default=None, 1805 type=str, 1806 help=( 1807 "Only use the specified comma-delimited list of polling engines. " 1808 "Example: --force_use_pollers epoll1,poll " 1809 " (This flag has no effect if --force_default_poller flag is also used)" 1810 ), 1811) 1812argp.add_argument( 1813 "--max_time", default=-1, type=int, help="Maximum test runtime in seconds" 1814) 1815argp.add_argument( 1816 "--bq_result_table", 1817 default="", 1818 type=str, 1819 nargs="?", 1820 help="Upload test results to a specified BQ table.", 1821) 1822argp.add_argument( 1823 "--cmake_configure_extra_args", 1824 default=[], 1825 action="append", 1826 help="Extra arguments that will be passed to the cmake configure command. Only works for C/C++.", 1827) 1828args = argp.parse_args() 1829 1830flaky_tests = set() 1831shortname_to_cpu = {} 1832 1833if args.force_default_poller: 1834 _POLLING_STRATEGIES = {} 1835elif args.force_use_pollers: 1836 _POLLING_STRATEGIES[platform_string()] = args.force_use_pollers.split(",") 1837 1838jobset.measure_cpu_costs = args.measure_cpu_costs 1839 1840# grab config 1841run_config = _CONFIGS[args.config] 1842build_config = run_config.build_config 1843 1844languages = set(_LANGUAGES[l] for l in args.language) 1845for l in languages: 1846 l.configure(run_config, args) 1847 1848if len(languages) != 1: 1849 print("Building multiple languages simultaneously is not supported!") 1850 sys.exit(1) 1851 1852# If --use_docker was used, respawn the run_tests.py script under a docker container 1853# instead of continuing. 1854if args.use_docker: 1855 if not args.travis: 1856 print("Seen --use_docker flag, will run tests under docker.") 1857 print("") 1858 print( 1859 "IMPORTANT: The changes you are testing need to be locally" 1860 " committed" 1861 ) 1862 print( 1863 "because only the committed changes in the current branch will be" 1864 ) 1865 print("copied to the docker environment.") 1866 time.sleep(5) 1867 1868 dockerfile_dirs = set([l.dockerfile_dir() for l in languages]) 1869 if len(dockerfile_dirs) > 1: 1870 print( 1871 "Languages to be tested require running under different docker " 1872 "images." 1873 ) 1874 sys.exit(1) 1875 else: 1876 dockerfile_dir = next(iter(dockerfile_dirs)) 1877 1878 child_argv = [arg for arg in sys.argv if not arg == "--use_docker"] 1879 run_tests_cmd = "python3 tools/run_tests/run_tests.py %s" % " ".join( 1880 child_argv[1:] 1881 ) 1882 1883 env = os.environ.copy() 1884 env["DOCKERFILE_DIR"] = dockerfile_dir 1885 env["DOCKER_RUN_SCRIPT"] = "tools/run_tests/dockerize/docker_run.sh" 1886 env["DOCKER_RUN_SCRIPT_COMMAND"] = run_tests_cmd 1887 1888 retcode = subprocess.call( 1889 "tools/run_tests/dockerize/build_and_run_docker.sh", shell=True, env=env 1890 ) 1891 _print_debug_info_epilogue(dockerfile_dir=dockerfile_dir) 1892 sys.exit(retcode) 1893 1894_check_arch_option(args.arch) 1895 1896# collect pre-build steps (which get retried if they fail, e.g. to avoid 1897# flakes on downloading dependencies etc.) 1898build_steps = list( 1899 set( 1900 jobset.JobSpec( 1901 cmdline, 1902 environ=_build_step_environ( 1903 build_config, extra_env=l.build_steps_environ() 1904 ), 1905 timeout_seconds=_PRE_BUILD_STEP_TIMEOUT_SECONDS, 1906 flake_retries=2, 1907 ) 1908 for l in languages 1909 for cmdline in l.pre_build_steps() 1910 ) 1911) 1912 1913# collect build steps 1914build_steps.extend( 1915 set( 1916 jobset.JobSpec( 1917 cmdline, 1918 environ=_build_step_environ( 1919 build_config, extra_env=l.build_steps_environ() 1920 ), 1921 timeout_seconds=None, 1922 ) 1923 for l in languages 1924 for cmdline in l.build_steps() 1925 ) 1926) 1927 1928# collect post test steps 1929post_tests_steps = list( 1930 set( 1931 jobset.JobSpec( 1932 cmdline, 1933 environ=_build_step_environ( 1934 build_config, extra_env=l.build_steps_environ() 1935 ), 1936 ) 1937 for l in languages 1938 for cmdline in l.post_tests_steps() 1939 ) 1940) 1941runs_per_test = args.runs_per_test 1942 1943errors = _build_and_run( 1944 check_cancelled=lambda: False, 1945 newline_on_success=args.newline_on_success, 1946 xml_report=args.xml_report, 1947 build_only=args.build_only, 1948) 1949if not errors: 1950 jobset.message("SUCCESS", "All tests passed", do_newline=True) 1951else: 1952 jobset.message("FAILED", "Some tests failed", do_newline=True) 1953 1954if not _is_use_docker_child(): 1955 # if --use_docker was used, the outer invocation of run_tests.py will 1956 # print the debug info instead. 1957 _print_debug_info_epilogue() 1958 1959exit_code = 0 1960if BuildAndRunError.BUILD in errors: 1961 exit_code |= 1 1962if BuildAndRunError.TEST in errors: 1963 exit_code |= 2 1964if BuildAndRunError.POST_TEST in errors: 1965 exit_code |= 4 1966sys.exit(exit_code) 1967