1#!/usr/bin/env python3 2# Copyright 2015 gRPC authors. 3# 4# Licensed under the Apache License, Version 2.0 (the "License"); 5# you may not use this file except in compliance with the License. 6# You may obtain a copy of the License at 7# 8# http://www.apache.org/licenses/LICENSE-2.0 9# 10# Unless required by applicable law or agreed to in writing, software 11# distributed under the License is distributed on an "AS IS" BASIS, 12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13# See the License for the specific language governing permissions and 14# limitations under the License. 15"""Run interop (cross-language) tests in parallel.""" 16 17from __future__ import print_function 18 19import argparse 20import atexit 21import itertools 22import json 23import multiprocessing 24import os 25import re 26import subprocess 27import sys 28import tempfile 29import time 30import traceback 31import uuid 32 33import six 34 35import python_utils.dockerjob as dockerjob 36import python_utils.jobset as jobset 37import python_utils.report_utils as report_utils 38 39# It's ok to not import because this is only necessary to upload results to BQ. 40try: 41 from python_utils.upload_test_results import upload_interop_results_to_bq 42except ImportError as e: 43 print(e) 44 45# Docker doesn't clean up after itself, so we do it on exit. 46atexit.register(lambda: subprocess.call(["stty", "echo"])) 47 48ROOT = os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]), "../..")) 49os.chdir(ROOT) 50 51_DEFAULT_SERVER_PORT = 8080 52 53_SKIP_CLIENT_COMPRESSION = [ 54 "client_compressed_unary", 55 "client_compressed_streaming", 56] 57 58_SKIP_SERVER_COMPRESSION = [ 59 "server_compressed_unary", 60 "server_compressed_streaming", 61] 62 63_SKIP_COMPRESSION = _SKIP_CLIENT_COMPRESSION + _SKIP_SERVER_COMPRESSION 64 65_SKIP_ADVANCED = [ 66 "status_code_and_message", 67 "custom_metadata", 68 "unimplemented_method", 69 "unimplemented_service", 70] 71 72_SKIP_SPECIAL_STATUS_MESSAGE = ["special_status_message"] 73 74_ORCA_TEST_CASES = ["orca_per_rpc", "orca_oob"] 75 76_GOOGLE_DEFAULT_CREDS_TEST_CASE = "google_default_credentials" 77 78_SKIP_GOOGLE_DEFAULT_CREDS = [ 79 _GOOGLE_DEFAULT_CREDS_TEST_CASE, 80] 81 82_COMPUTE_ENGINE_CHANNEL_CREDS_TEST_CASE = "compute_engine_channel_credentials" 83 84_SKIP_COMPUTE_ENGINE_CHANNEL_CREDS = [ 85 _COMPUTE_ENGINE_CHANNEL_CREDS_TEST_CASE, 86] 87 88_TEST_TIMEOUT = 3 * 60 89 90# disable this test on core-based languages, 91# see https://github.com/grpc/grpc/issues/9779 92_SKIP_DATA_FRAME_PADDING = ["data_frame_padding"] 93 94# report suffix "sponge_log.xml" is important for reports to get picked up by internal CI 95_DOCKER_BUILD_XML_REPORT = "interop_docker_build/sponge_log.xml" 96_TESTS_XML_REPORT = "interop_test/sponge_log.xml" 97 98 99class CXXLanguage: 100 def __init__(self): 101 self.client_cwd = None 102 self.server_cwd = None 103 self.http2_cwd = None 104 self.safename = "cxx" 105 106 def client_cmd(self, args): 107 return ["cmake/build/interop_client"] + args 108 109 def client_cmd_http2interop(self, args): 110 return ["cmake/build/http2_client"] + args 111 112 def cloud_to_prod_env(self): 113 return {} 114 115 def server_cmd(self, args): 116 return ["cmake/build/interop_server"] + args 117 118 def global_env(self): 119 return {} 120 121 def unimplemented_test_cases(self): 122 return ( 123 _SKIP_DATA_FRAME_PADDING 124 + _SKIP_SPECIAL_STATUS_MESSAGE 125 + _SKIP_COMPUTE_ENGINE_CHANNEL_CREDS 126 ) 127 128 def unimplemented_test_cases_server(self): 129 return [] 130 131 def __str__(self): 132 return "c++" 133 134 135class AspNetCoreLanguage: 136 def __init__(self): 137 self.client_cwd = "../grpc-dotnet/output/InteropTestsClient" 138 self.server_cwd = "../grpc-dotnet/output/InteropTestsWebsite" 139 self.safename = str(self) 140 141 def cloud_to_prod_env(self): 142 return {} 143 144 def client_cmd(self, args): 145 return ["dotnet", "exec", "InteropTestsClient.dll"] + args 146 147 def server_cmd(self, args): 148 return ["dotnet", "exec", "InteropTestsWebsite.dll"] + args 149 150 def global_env(self): 151 return {} 152 153 def unimplemented_test_cases(self): 154 return ( 155 _SKIP_GOOGLE_DEFAULT_CREDS 156 + _SKIP_COMPUTE_ENGINE_CHANNEL_CREDS 157 + _ORCA_TEST_CASES 158 ) 159 160 def unimplemented_test_cases_server(self): 161 return _ORCA_TEST_CASES 162 163 def __str__(self): 164 return "aspnetcore" 165 166 167class DartLanguage: 168 def __init__(self): 169 self.client_cwd = "../grpc-dart/interop" 170 self.server_cwd = "../grpc-dart/interop" 171 self.http2_cwd = "../grpc-dart/interop" 172 self.safename = str(self) 173 174 def client_cmd(self, args): 175 return ["dart", "bin/client.dart"] + args 176 177 def cloud_to_prod_env(self): 178 return {} 179 180 def server_cmd(self, args): 181 return ["dart", "bin/server.dart"] + args 182 183 def global_env(self): 184 return {} 185 186 def unimplemented_test_cases(self): 187 return ( 188 _SKIP_COMPRESSION 189 + _SKIP_SPECIAL_STATUS_MESSAGE 190 + _SKIP_GOOGLE_DEFAULT_CREDS 191 + _SKIP_COMPUTE_ENGINE_CHANNEL_CREDS 192 + _ORCA_TEST_CASES 193 ) 194 195 def unimplemented_test_cases_server(self): 196 return ( 197 _SKIP_COMPRESSION + _SKIP_SPECIAL_STATUS_MESSAGE + _ORCA_TEST_CASES 198 ) 199 200 def __str__(self): 201 return "dart" 202 203 204class JavaLanguage: 205 def __init__(self): 206 self.client_cwd = "../grpc-java" 207 self.server_cwd = "../grpc-java" 208 self.http2_cwd = "../grpc-java" 209 self.safename = str(self) 210 211 def client_cmd(self, args): 212 return ["./run-test-client.sh"] + args 213 214 def client_cmd_http2interop(self, args): 215 return [ 216 "./interop-testing/build/install/grpc-interop-testing/bin/http2-client" 217 ] + args 218 219 def cloud_to_prod_env(self): 220 return {} 221 222 def server_cmd(self, args): 223 return ["./run-test-server.sh"] + args 224 225 def global_env(self): 226 return {} 227 228 def unimplemented_test_cases(self): 229 return [] 230 231 def unimplemented_test_cases_server(self): 232 # Does not support CompressedRequest feature. 233 # Only supports CompressedResponse feature for unary. 234 return _SKIP_CLIENT_COMPRESSION + ["server_compressed_streaming"] 235 236 def __str__(self): 237 return "java" 238 239 240class JavaOkHttpClient: 241 def __init__(self): 242 self.client_cwd = "../grpc-java" 243 self.safename = "java" 244 245 def client_cmd(self, args): 246 return ["./run-test-client.sh", "--use_okhttp=true"] + args 247 248 def cloud_to_prod_env(self): 249 return {} 250 251 def global_env(self): 252 return {} 253 254 def unimplemented_test_cases(self): 255 return _SKIP_DATA_FRAME_PADDING 256 257 def __str__(self): 258 return "javaokhttp" 259 260 261class GoLanguage: 262 def __init__(self): 263 # TODO: this relies on running inside docker 264 self.client_cwd = "/go/src/google.golang.org/grpc/interop/client" 265 self.server_cwd = "/go/src/google.golang.org/grpc/interop/server" 266 self.http2_cwd = "/go/src/google.golang.org/grpc/interop/http2" 267 self.safename = str(self) 268 269 def client_cmd(self, args): 270 return ["go", "run", "client.go"] + args 271 272 def client_cmd_http2interop(self, args): 273 return ["go", "run", "negative_http2_client.go"] + args 274 275 def cloud_to_prod_env(self): 276 return {} 277 278 def server_cmd(self, args): 279 return ["go", "run", "server.go"] + args 280 281 def global_env(self): 282 return {"GO111MODULE": "on"} 283 284 def unimplemented_test_cases(self): 285 return _SKIP_COMPRESSION 286 287 def unimplemented_test_cases_server(self): 288 return _SKIP_COMPRESSION 289 290 def __str__(self): 291 return "go" 292 293 294class Http2Server: 295 """Represents the HTTP/2 Interop Test server 296 297 This pretends to be a language in order to be built and run, but really it 298 isn't. 299 """ 300 301 def __init__(self): 302 self.server_cwd = None 303 self.safename = str(self) 304 305 def server_cmd(self, args): 306 return ["python test/http2_test/http2_test_server.py"] 307 308 def cloud_to_prod_env(self): 309 return {} 310 311 def global_env(self): 312 return {} 313 314 def unimplemented_test_cases(self): 315 return ( 316 _TEST_CASES 317 + _SKIP_DATA_FRAME_PADDING 318 + _SKIP_SPECIAL_STATUS_MESSAGE 319 + _SKIP_GOOGLE_DEFAULT_CREDS 320 + _SKIP_COMPUTE_ENGINE_CHANNEL_CREDS 321 ) 322 323 def unimplemented_test_cases_server(self): 324 return _TEST_CASES 325 326 def __str__(self): 327 return "http2" 328 329 330class Http2Client: 331 """Represents the HTTP/2 Interop Test 332 333 This pretends to be a language in order to be built and run, but really it 334 isn't. 335 """ 336 337 def __init__(self): 338 self.client_cwd = None 339 self.safename = str(self) 340 341 def client_cmd(self, args): 342 return ["tools/http2_interop/http2_interop.test", "-test.v"] + args 343 344 def cloud_to_prod_env(self): 345 return {} 346 347 def global_env(self): 348 return {} 349 350 def unimplemented_test_cases(self): 351 return ( 352 _TEST_CASES 353 + _SKIP_SPECIAL_STATUS_MESSAGE 354 + _SKIP_GOOGLE_DEFAULT_CREDS 355 + _SKIP_COMPUTE_ENGINE_CHANNEL_CREDS 356 ) 357 358 def unimplemented_test_cases_server(self): 359 return _TEST_CASES 360 361 def __str__(self): 362 return "http2" 363 364 365class NodeLanguage: 366 def __init__(self): 367 self.client_cwd = "../../../../home/appuser/grpc-node" 368 self.server_cwd = "../../../../home/appuser/grpc-node" 369 self.safename = str(self) 370 371 def client_cmd(self, args): 372 return [ 373 "packages/grpc-native-core/deps/grpc/tools/run_tests/interop/with_nvm.sh", 374 "node", 375 "--require", 376 "./test/fixtures/native_native", 377 "test/interop/interop_client.js", 378 ] + args 379 380 def cloud_to_prod_env(self): 381 return {} 382 383 def server_cmd(self, args): 384 return [ 385 "packages/grpc-native-core/deps/grpc/tools/run_tests/interop/with_nvm.sh", 386 "node", 387 "--require", 388 "./test/fixtures/native_native", 389 "test/interop/interop_server.js", 390 ] + args 391 392 def global_env(self): 393 return {} 394 395 def unimplemented_test_cases(self): 396 return ( 397 _SKIP_COMPRESSION 398 + _SKIP_DATA_FRAME_PADDING 399 + _SKIP_GOOGLE_DEFAULT_CREDS 400 + _SKIP_COMPUTE_ENGINE_CHANNEL_CREDS 401 + _ORCA_TEST_CASES 402 ) 403 404 def unimplemented_test_cases_server(self): 405 return _SKIP_COMPRESSION + _ORCA_TEST_CASES 406 407 def __str__(self): 408 return "node" 409 410 411class NodePureJSLanguage: 412 def __init__(self): 413 self.client_cwd = "../../../../home/appuser/grpc-node" 414 self.server_cwd = "../../../../home/appuser/grpc-node" 415 self.safename = str(self) 416 417 def client_cmd(self, args): 418 return [ 419 "packages/grpc-native-core/deps/grpc/tools/run_tests/interop/with_nvm.sh", 420 "node", 421 "--require", 422 "./test/fixtures/js_js", 423 "test/interop/interop_client.js", 424 ] + args 425 426 def cloud_to_prod_env(self): 427 return {} 428 429 def global_env(self): 430 return {} 431 432 def unimplemented_test_cases(self): 433 return ( 434 _SKIP_COMPRESSION 435 + _SKIP_DATA_FRAME_PADDING 436 + _SKIP_GOOGLE_DEFAULT_CREDS 437 + _SKIP_COMPUTE_ENGINE_CHANNEL_CREDS 438 + _ORCA_TEST_CASES 439 ) 440 441 def unimplemented_test_cases_server(self): 442 return _ORCA_TEST_CASES 443 444 def __str__(self): 445 return "nodepurejs" 446 447 448class PHP7Language: 449 def __init__(self): 450 self.client_cwd = None 451 self.server_cwd = None 452 self.safename = str(self) 453 454 def client_cmd(self, args): 455 return ["src/php/bin/interop_client.sh"] + args 456 457 def cloud_to_prod_env(self): 458 return {} 459 460 def server_cmd(self, args): 461 return ["src/php/bin/interop_server.sh"] + args 462 463 def global_env(self): 464 return {} 465 466 def unimplemented_test_cases(self): 467 return ( 468 _SKIP_SERVER_COMPRESSION 469 + _SKIP_DATA_FRAME_PADDING 470 + _SKIP_GOOGLE_DEFAULT_CREDS 471 + _SKIP_COMPUTE_ENGINE_CHANNEL_CREDS 472 + _ORCA_TEST_CASES 473 ) 474 475 def unimplemented_test_cases_server(self): 476 return _SKIP_COMPRESSION + _ORCA_TEST_CASES 477 478 def __str__(self): 479 return "php7" 480 481 482class ObjcLanguage: 483 def __init__(self): 484 self.client_cwd = "src/objective-c/tests" 485 self.safename = str(self) 486 487 def client_cmd(self, args): 488 # from args, extract the server port and craft xcodebuild command out of it 489 for arg in args: 490 port = re.search("--server_port=(\d+)", arg) 491 if port: 492 portnum = port.group(1) 493 cmdline = ( 494 "pod install && xcodebuild -workspace Tests.xcworkspace" 495 ' -scheme InteropTestsLocalSSL -destination name="iPhone 6"' 496 " HOST_PORT_LOCALSSL=localhost:%s test" % portnum 497 ) 498 return [cmdline] 499 500 def cloud_to_prod_env(self): 501 return {} 502 503 def global_env(self): 504 return {} 505 506 def unimplemented_test_cases(self): 507 # ObjC test runs all cases with the same command. It ignores the testcase 508 # cmdline argument. Here we return all but one test cases as unimplemented, 509 # and depend upon ObjC test's behavior that it runs all cases even when 510 # we tell it to run just one. 511 return ( 512 _TEST_CASES[1:] 513 + _SKIP_COMPRESSION 514 + _SKIP_DATA_FRAME_PADDING 515 + _SKIP_SPECIAL_STATUS_MESSAGE 516 + _SKIP_GOOGLE_DEFAULT_CREDS 517 + _SKIP_COMPUTE_ENGINE_CHANNEL_CREDS 518 + _ORCA_TEST_CASES 519 ) 520 521 def unimplemented_test_cases_server(self): 522 return _SKIP_COMPRESSION + _ORCA_TEST_CASES 523 524 def __str__(self): 525 return "objc" 526 527 528class RubyLanguage: 529 def __init__(self): 530 self.client_cwd = None 531 self.server_cwd = None 532 self.safename = str(self) 533 534 def client_cmd(self, args): 535 return [ 536 "tools/run_tests/interop/with_rvm.sh", 537 "ruby", 538 "src/ruby/pb/test/client.rb", 539 ] + args 540 541 def cloud_to_prod_env(self): 542 return {} 543 544 def server_cmd(self, args): 545 return [ 546 "tools/run_tests/interop/with_rvm.sh", 547 "ruby", 548 "src/ruby/pb/test/server.rb", 549 ] + args 550 551 def global_env(self): 552 return {} 553 554 def unimplemented_test_cases(self): 555 return ( 556 _SKIP_SERVER_COMPRESSION 557 + _SKIP_DATA_FRAME_PADDING 558 + _SKIP_GOOGLE_DEFAULT_CREDS 559 + _SKIP_COMPUTE_ENGINE_CHANNEL_CREDS 560 + _ORCA_TEST_CASES 561 ) 562 563 def unimplemented_test_cases_server(self): 564 return _SKIP_COMPRESSION + _ORCA_TEST_CASES 565 566 def __str__(self): 567 return "ruby" 568 569 570_PYTHON_BINARY = "py39/bin/python" 571 572 573class PythonLanguage: 574 def __init__(self): 575 self.client_cwd = None 576 self.server_cwd = None 577 self.http2_cwd = None 578 self.safename = str(self) 579 580 def client_cmd(self, args): 581 return [ 582 _PYTHON_BINARY, 583 "src/python/grpcio_tests/setup.py", 584 "run_interop", 585 "--client", 586 '--args="{}"'.format(" ".join(args)), 587 ] 588 589 def client_cmd_http2interop(self, args): 590 return [ 591 _PYTHON_BINARY, 592 "src/python/grpcio_tests/tests/http2/negative_http2_client.py", 593 ] + args 594 595 def cloud_to_prod_env(self): 596 return {} 597 598 def server_cmd(self, args): 599 return [ 600 _PYTHON_BINARY, 601 "src/python/grpcio_tests/setup.py", 602 "run_interop", 603 "--server", 604 '--args="{}"'.format(" ".join(args)), 605 ] 606 607 def global_env(self): 608 return { 609 "LD_LIBRARY_PATH": "{}/libs/opt".format(DOCKER_WORKDIR_ROOT), 610 "PYTHONPATH": "{}/src/python/gens".format(DOCKER_WORKDIR_ROOT), 611 } 612 613 def unimplemented_test_cases(self): 614 return ( 615 _SKIP_COMPRESSION 616 + _SKIP_DATA_FRAME_PADDING 617 + _SKIP_GOOGLE_DEFAULT_CREDS 618 + _SKIP_COMPUTE_ENGINE_CHANNEL_CREDS 619 + _ORCA_TEST_CASES 620 ) 621 622 def unimplemented_test_cases_server(self): 623 return _SKIP_COMPRESSION + _ORCA_TEST_CASES 624 625 def __str__(self): 626 return "python" 627 628 629class PythonAsyncIOLanguage: 630 def __init__(self): 631 self.client_cwd = None 632 self.server_cwd = None 633 self.http2_cwd = None 634 self.safename = str(self) 635 636 def client_cmd(self, args): 637 return [ 638 _PYTHON_BINARY, 639 "src/python/grpcio_tests/setup.py", 640 "run_interop", 641 "--use-asyncio", 642 "--client", 643 '--args="{}"'.format(" ".join(args)), 644 ] 645 646 def client_cmd_http2interop(self, args): 647 return [ 648 _PYTHON_BINARY, 649 "src/python/grpcio_tests/tests/http2/negative_http2_client.py", 650 ] + args 651 652 def cloud_to_prod_env(self): 653 return {} 654 655 def server_cmd(self, args): 656 return [ 657 _PYTHON_BINARY, 658 "src/python/grpcio_tests/setup.py", 659 "run_interop", 660 "--use-asyncio", 661 "--server", 662 '--args="{}"'.format(" ".join(args)), 663 ] 664 665 def global_env(self): 666 return { 667 "LD_LIBRARY_PATH": "{}/libs/opt".format(DOCKER_WORKDIR_ROOT), 668 "PYTHONPATH": "{}/src/python/gens".format(DOCKER_WORKDIR_ROOT), 669 } 670 671 def unimplemented_test_cases(self): 672 # TODO(https://github.com/grpc/grpc/issues/21707) 673 return ( 674 _SKIP_COMPRESSION 675 + _SKIP_DATA_FRAME_PADDING 676 + _AUTH_TEST_CASES 677 + ["timeout_on_sleeping_server"] 678 + _ORCA_TEST_CASES 679 ) 680 681 def unimplemented_test_cases_server(self): 682 # TODO(https://github.com/grpc/grpc/issues/21749) 683 return ( 684 _TEST_CASES 685 + _AUTH_TEST_CASES 686 + _HTTP2_TEST_CASES 687 + _HTTP2_SERVER_TEST_CASES 688 ) 689 690 def __str__(self): 691 return "pythonasyncio" 692 693 694_LANGUAGES = { 695 "c++": CXXLanguage(), 696 "aspnetcore": AspNetCoreLanguage(), 697 "dart": DartLanguage(), 698 "go": GoLanguage(), 699 "java": JavaLanguage(), 700 "javaokhttp": JavaOkHttpClient(), 701 "node": NodeLanguage(), 702 "nodepurejs": NodePureJSLanguage(), 703 "php7": PHP7Language(), 704 "objc": ObjcLanguage(), 705 "ruby": RubyLanguage(), 706 "python": PythonLanguage(), 707 "pythonasyncio": PythonAsyncIOLanguage(), 708} 709 710# languages supported as cloud_to_cloud servers 711_SERVERS = [ 712 "c++", 713 "node", 714 "aspnetcore", 715 "java", 716 "go", 717 "ruby", 718 "python", 719 "dart", 720 "pythonasyncio", 721 "php7", 722] 723 724_TEST_CASES = [ 725 "large_unary", 726 "empty_unary", 727 "ping_pong", 728 "empty_stream", 729 "client_streaming", 730 "server_streaming", 731 "cancel_after_begin", 732 "cancel_after_first_response", 733 "timeout_on_sleeping_server", 734 "custom_metadata", 735 "status_code_and_message", 736 "unimplemented_method", 737 "client_compressed_unary", 738 "server_compressed_unary", 739 "client_compressed_streaming", 740 "server_compressed_streaming", 741 "unimplemented_service", 742 "special_status_message", 743 "orca_per_rpc", 744 "orca_oob", 745] 746 747_AUTH_TEST_CASES = [ 748 "compute_engine_creds", 749 "jwt_token_creds", 750 "oauth2_auth_token", 751 "per_rpc_creds", 752 _GOOGLE_DEFAULT_CREDS_TEST_CASE, 753 _COMPUTE_ENGINE_CHANNEL_CREDS_TEST_CASE, 754] 755 756_HTTP2_TEST_CASES = ["tls", "framing"] 757 758_HTTP2_SERVER_TEST_CASES = [ 759 "rst_after_header", 760 "rst_after_data", 761 "rst_during_data", 762 "goaway", 763 "ping", 764 "max_streams", 765 "data_frame_padding", 766 "no_df_padding_sanity_test", 767] 768 769_GRPC_CLIENT_TEST_CASES_FOR_HTTP2_SERVER_TEST_CASES = { 770 "data_frame_padding": "large_unary", 771 "no_df_padding_sanity_test": "large_unary", 772} 773 774_HTTP2_SERVER_TEST_CASES_THAT_USE_GRPC_CLIENTS = list( 775 _GRPC_CLIENT_TEST_CASES_FOR_HTTP2_SERVER_TEST_CASES.keys() 776) 777 778_LANGUAGES_WITH_HTTP2_CLIENTS_FOR_HTTP2_SERVER_TEST_CASES = [ 779 "java", 780 "go", 781 "python", 782 "c++", 783] 784 785_LANGUAGES_FOR_ALTS_TEST_CASES = ["java", "go", "c++", "python"] 786 787_SERVERS_FOR_ALTS_TEST_CASES = ["java", "go", "c++", "python"] 788 789_TRANSPORT_SECURITY_OPTIONS = ["tls", "alts", "insecure"] 790 791_CUSTOM_CREDENTIALS_TYPE_OPTIONS = [ 792 "tls", 793 "google_default_credentials", 794 "compute_engine_channel_creds", 795] 796 797DOCKER_WORKDIR_ROOT = "/var/local/git/grpc" 798 799 800def docker_run_cmdline(cmdline, image, docker_args=[], cwd=None, environ=None): 801 """Wraps given cmdline array to create 'docker run' cmdline from it.""" 802 803 # don't use '-t' even when TTY is available, since that would break 804 # the testcases generated by tools/interop_matrix/create_testcases.sh 805 docker_cmdline = ["docker", "run", "-i", "--rm=true"] 806 807 # turn environ into -e docker args 808 if environ: 809 for k, v in list(environ.items()): 810 docker_cmdline += ["-e", "%s=%s" % (k, v)] 811 812 # set working directory 813 workdir = DOCKER_WORKDIR_ROOT 814 if cwd: 815 workdir = os.path.join(workdir, cwd) 816 docker_cmdline += ["-w", workdir] 817 818 docker_cmdline += docker_args + [image] + cmdline 819 return docker_cmdline 820 821 822def manual_cmdline(docker_cmdline, docker_image): 823 """Returns docker cmdline adjusted for manual invocation.""" 824 print_cmdline = [] 825 for item in docker_cmdline: 826 if item.startswith("--name="): 827 continue 828 if item == docker_image: 829 item = "$docker_image" 830 item = item.replace('"', '\\"') 831 # add quotes when necessary 832 if any(character.isspace() for character in item): 833 item = '"%s"' % item 834 print_cmdline.append(item) 835 return " ".join(print_cmdline) 836 837 838def write_cmdlog_maybe(cmdlog, filename): 839 """Returns docker cmdline adjusted for manual invocation.""" 840 if cmdlog: 841 with open(filename, "w") as logfile: 842 logfile.write("#!/bin/bash\n") 843 logfile.write("# DO NOT MODIFY\n") 844 logfile.write( 845 "# This file is generated by" 846 " run_interop_tests.py/create_testcases.sh\n" 847 ) 848 logfile.writelines("%s\n" % line for line in cmdlog) 849 print("Command log written to file %s" % filename) 850 851 852def bash_cmdline(cmdline): 853 """Creates bash -c cmdline from args list.""" 854 # Use login shell: 855 # * makes error messages clearer if executables are missing 856 return ["bash", "-c", " ".join(cmdline)] 857 858 859def compute_engine_creds_required(language, test_case): 860 """Returns True if given test requires access to compute engine creds.""" 861 language = str(language) 862 if test_case == "compute_engine_creds": 863 return True 864 if test_case == "oauth2_auth_token" and language == "c++": 865 # C++ oauth2 test uses GCE creds because C++ only supports JWT 866 return True 867 return False 868 869 870def auth_options( 871 language, 872 test_case, 873 google_default_creds_use_key_file, 874 service_account_key_file, 875 default_service_account, 876): 877 """Returns (cmdline, env) tuple with cloud_to_prod_auth test options.""" 878 879 language = str(language) 880 cmdargs = [] 881 env = {} 882 883 oauth_scope_arg = "--oauth_scope=https://www.googleapis.com/auth/xapi.zoo" 884 key_file_arg = "--service_account_key_file=%s" % service_account_key_file 885 default_account_arg = ( 886 "--default_service_account=%s" % default_service_account 887 ) 888 889 if test_case in ["jwt_token_creds", "per_rpc_creds", "oauth2_auth_token"]: 890 if language in [ 891 "aspnetcore", 892 "node", 893 "php7", 894 "python", 895 "ruby", 896 "nodepurejs", 897 ]: 898 env["GOOGLE_APPLICATION_CREDENTIALS"] = service_account_key_file 899 else: 900 cmdargs += [key_file_arg] 901 902 if test_case in ["per_rpc_creds", "oauth2_auth_token"]: 903 cmdargs += [oauth_scope_arg] 904 905 if test_case == "oauth2_auth_token" and language == "c++": 906 # C++ oauth2 test uses GCE creds and thus needs to know the default account 907 cmdargs += [default_account_arg] 908 909 if test_case == "compute_engine_creds": 910 cmdargs += [oauth_scope_arg, default_account_arg] 911 912 if test_case == _GOOGLE_DEFAULT_CREDS_TEST_CASE: 913 if google_default_creds_use_key_file: 914 env["GOOGLE_APPLICATION_CREDENTIALS"] = service_account_key_file 915 cmdargs += [default_account_arg] 916 917 if test_case == _COMPUTE_ENGINE_CHANNEL_CREDS_TEST_CASE: 918 cmdargs += [default_account_arg] 919 920 return (cmdargs, env) 921 922 923def _job_kill_handler(job): 924 if job._spec.container_name: 925 dockerjob.docker_kill(job._spec.container_name) 926 # When the job times out and we decide to kill it, 927 # we need to wait a before restarting the job 928 # to prevent "container name already in use" error. 929 # TODO(jtattermusch): figure out a cleaner way to this. 930 time.sleep(2) 931 932 933def cloud_to_prod_jobspec( 934 language, 935 test_case, 936 server_host_nickname, 937 server_host, 938 google_default_creds_use_key_file, 939 docker_image=None, 940 auth=False, 941 manual_cmd_log=None, 942 service_account_key_file=None, 943 default_service_account=None, 944 transport_security="tls", 945): 946 """Creates jobspec for cloud-to-prod interop test""" 947 container_name = None 948 cmdargs = [ 949 "--server_host=%s" % server_host, 950 "--server_port=443", 951 "--test_case=%s" % test_case, 952 ] 953 if transport_security == "tls": 954 transport_security_options = ["--use_tls=true"] 955 elif transport_security == "google_default_credentials" and str( 956 language 957 ) in ["c++", "go", "java", "javaokhttp"]: 958 transport_security_options = [ 959 "--custom_credentials_type=google_default_credentials" 960 ] 961 elif transport_security == "compute_engine_channel_creds" and str( 962 language 963 ) in ["go", "java", "javaokhttp"]: 964 transport_security_options = [ 965 "--custom_credentials_type=compute_engine_channel_creds" 966 ] 967 else: 968 print( 969 "Invalid transport security option %s in cloud_to_prod_jobspec." 970 " Lang: %s" % (str(language), transport_security) 971 ) 972 sys.exit(1) 973 cmdargs = cmdargs + transport_security_options 974 environ = dict(language.cloud_to_prod_env(), **language.global_env()) 975 if auth: 976 auth_cmdargs, auth_env = auth_options( 977 language, 978 test_case, 979 google_default_creds_use_key_file, 980 service_account_key_file, 981 default_service_account, 982 ) 983 cmdargs += auth_cmdargs 984 environ.update(auth_env) 985 cmdline = bash_cmdline(language.client_cmd(cmdargs)) 986 cwd = language.client_cwd 987 988 if docker_image: 989 container_name = dockerjob.random_name( 990 "interop_client_%s" % language.safename 991 ) 992 cmdline = docker_run_cmdline( 993 cmdline, 994 image=docker_image, 995 cwd=cwd, 996 environ=environ, 997 docker_args=["--net=host", "--name=%s" % container_name], 998 ) 999 if manual_cmd_log is not None: 1000 if manual_cmd_log == []: 1001 manual_cmd_log.append( 1002 'echo "Testing ${docker_image:=%s}"' % docker_image 1003 ) 1004 manual_cmd_log.append(manual_cmdline(cmdline, docker_image)) 1005 cwd = None 1006 environ = None 1007 1008 suite_name = "cloud_to_prod_auth" if auth else "cloud_to_prod" 1009 test_job = jobset.JobSpec( 1010 cmdline=cmdline, 1011 cwd=cwd, 1012 environ=environ, 1013 shortname="%s:%s:%s:%s:%s" 1014 % ( 1015 suite_name, 1016 language, 1017 server_host_nickname, 1018 test_case, 1019 transport_security, 1020 ), 1021 timeout_seconds=_TEST_TIMEOUT, 1022 flake_retries=4 if args.allow_flakes else 0, 1023 timeout_retries=2 if args.allow_flakes else 0, 1024 kill_handler=_job_kill_handler, 1025 ) 1026 if docker_image: 1027 test_job.container_name = container_name 1028 return test_job 1029 1030 1031def cloud_to_cloud_jobspec( 1032 language, 1033 test_case, 1034 server_name, 1035 server_host, 1036 server_port, 1037 docker_image=None, 1038 transport_security="tls", 1039 manual_cmd_log=None, 1040): 1041 """Creates jobspec for cloud-to-cloud interop test""" 1042 interop_only_options = [ 1043 "--server_host_override=foo.test.google.fr", 1044 "--use_test_ca=true", 1045 ] 1046 if transport_security == "tls": 1047 interop_only_options += ["--use_tls=true"] 1048 elif transport_security == "alts": 1049 interop_only_options += ["--use_tls=false", "--use_alts=true"] 1050 elif transport_security == "insecure": 1051 interop_only_options += ["--use_tls=false"] 1052 else: 1053 print( 1054 "Invalid transport security option %s in cloud_to_cloud_jobspec." 1055 % transport_security 1056 ) 1057 sys.exit(1) 1058 1059 client_test_case = test_case 1060 if test_case in _HTTP2_SERVER_TEST_CASES_THAT_USE_GRPC_CLIENTS: 1061 client_test_case = _GRPC_CLIENT_TEST_CASES_FOR_HTTP2_SERVER_TEST_CASES[ 1062 test_case 1063 ] 1064 if client_test_case in language.unimplemented_test_cases(): 1065 print( 1066 "asking client %s to run unimplemented test case %s" 1067 % (repr(language), client_test_case) 1068 ) 1069 sys.exit(1) 1070 1071 if test_case in _ORCA_TEST_CASES: 1072 interop_only_options += [ 1073 '--service_config_json=\'{"loadBalancingConfig":[{"test_backend_metrics_load_balancer":{}}]}\'' 1074 ] 1075 1076 common_options = [ 1077 "--test_case=%s" % client_test_case, 1078 "--server_host=%s" % server_host, 1079 "--server_port=%s" % server_port, 1080 ] 1081 1082 if test_case in _HTTP2_SERVER_TEST_CASES: 1083 if test_case in _HTTP2_SERVER_TEST_CASES_THAT_USE_GRPC_CLIENTS: 1084 client_options = interop_only_options + common_options 1085 cmdline = bash_cmdline(language.client_cmd(client_options)) 1086 cwd = language.client_cwd 1087 else: 1088 cmdline = bash_cmdline( 1089 language.client_cmd_http2interop(common_options) 1090 ) 1091 cwd = language.http2_cwd 1092 else: 1093 cmdline = bash_cmdline( 1094 language.client_cmd(common_options + interop_only_options) 1095 ) 1096 cwd = language.client_cwd 1097 1098 environ = language.global_env() 1099 if docker_image and language.safename != "objc": 1100 # we can't run client in docker for objc. 1101 container_name = dockerjob.random_name( 1102 "interop_client_%s" % language.safename 1103 ) 1104 cmdline = docker_run_cmdline( 1105 cmdline, 1106 image=docker_image, 1107 environ=environ, 1108 cwd=cwd, 1109 docker_args=["--net=host", "--name=%s" % container_name], 1110 ) 1111 if manual_cmd_log is not None: 1112 if manual_cmd_log == []: 1113 manual_cmd_log.append( 1114 'echo "Testing ${docker_image:=%s}"' % docker_image 1115 ) 1116 manual_cmd_log.append(manual_cmdline(cmdline, docker_image)) 1117 cwd = None 1118 1119 test_job = jobset.JobSpec( 1120 cmdline=cmdline, 1121 cwd=cwd, 1122 environ=environ, 1123 shortname="cloud_to_cloud:%s:%s_server:%s:%s" 1124 % (language, server_name, test_case, transport_security), 1125 timeout_seconds=_TEST_TIMEOUT, 1126 flake_retries=4 if args.allow_flakes else 0, 1127 timeout_retries=2 if args.allow_flakes else 0, 1128 kill_handler=_job_kill_handler, 1129 ) 1130 if docker_image: 1131 test_job.container_name = container_name 1132 return test_job 1133 1134 1135def server_jobspec( 1136 language, docker_image, transport_security="tls", manual_cmd_log=None 1137): 1138 """Create jobspec for running a server""" 1139 container_name = dockerjob.random_name( 1140 "interop_server_%s" % language.safename 1141 ) 1142 server_cmd = ["--port=%s" % _DEFAULT_SERVER_PORT] 1143 if transport_security == "tls": 1144 server_cmd += ["--use_tls=true"] 1145 elif transport_security == "alts": 1146 server_cmd += ["--use_tls=false", "--use_alts=true"] 1147 elif transport_security == "insecure": 1148 server_cmd += ["--use_tls=false"] 1149 else: 1150 print( 1151 "Invalid transport security option %s in server_jobspec." 1152 % transport_security 1153 ) 1154 sys.exit(1) 1155 cmdline = bash_cmdline(language.server_cmd(server_cmd)) 1156 environ = language.global_env() 1157 docker_args = ["--name=%s" % container_name] 1158 if language.safename == "http2": 1159 # we are running the http2 interop server. Open next N ports beginning 1160 # with the server port. These ports are used for http2 interop test 1161 # (one test case per port). 1162 docker_args += list( 1163 itertools.chain.from_iterable( 1164 ("-p", str(_DEFAULT_SERVER_PORT + i)) 1165 for i in range(len(_HTTP2_SERVER_TEST_CASES)) 1166 ) 1167 ) 1168 # Enable docker's healthcheck mechanism. 1169 # This runs a Python script inside the container every second. The script 1170 # pings the http2 server to verify it is ready. The 'health-retries' flag 1171 # specifies the number of consecutive failures before docker will report 1172 # the container's status as 'unhealthy'. Prior to the first 'health_retries' 1173 # failures or the first success, the status will be 'starting'. 'docker ps' 1174 # or 'docker inspect' can be used to see the health of the container on the 1175 # command line. 1176 docker_args += [ 1177 "--health-cmd=python test/http2_test/http2_server_health_check.py " 1178 "--server_host=%s --server_port=%d" 1179 % ("localhost", _DEFAULT_SERVER_PORT), 1180 "--health-interval=1s", 1181 "--health-retries=5", 1182 "--health-timeout=10s", 1183 ] 1184 1185 else: 1186 docker_args += ["-p", str(_DEFAULT_SERVER_PORT)] 1187 1188 docker_cmdline = docker_run_cmdline( 1189 cmdline, 1190 image=docker_image, 1191 cwd=language.server_cwd, 1192 environ=environ, 1193 docker_args=docker_args, 1194 ) 1195 if manual_cmd_log is not None: 1196 if manual_cmd_log == []: 1197 manual_cmd_log.append( 1198 'echo "Testing ${docker_image:=%s}"' % docker_image 1199 ) 1200 manual_cmd_log.append(manual_cmdline(docker_cmdline, docker_image)) 1201 server_job = jobset.JobSpec( 1202 cmdline=docker_cmdline, 1203 environ=environ, 1204 shortname="interop_server_%s" % language, 1205 timeout_seconds=30 * 60, 1206 ) 1207 server_job.container_name = container_name 1208 return server_job 1209 1210 1211def build_interop_image_jobspec(language, tag=None): 1212 """Creates jobspec for building interop docker image for a language""" 1213 if not tag: 1214 tag = "grpc_interop_%s:%s" % (language.safename, uuid.uuid4()) 1215 env = { 1216 "INTEROP_IMAGE": tag, 1217 "BASE_NAME": "grpc_interop_%s" % language.safename, 1218 } 1219 build_job = jobset.JobSpec( 1220 cmdline=["tools/run_tests/dockerize/build_interop_image.sh"], 1221 environ=env, 1222 shortname="build_docker_%s" % (language), 1223 timeout_seconds=45 * 60, 1224 ) 1225 build_job.tag = tag 1226 return build_job 1227 1228 1229def aggregate_http2_results(stdout): 1230 match = re.search(r'\{"cases[^\]]*\]\}', stdout) 1231 if not match: 1232 return None 1233 1234 results = json.loads(match.group(0)) 1235 skipped = 0 1236 passed = 0 1237 failed = 0 1238 failed_cases = [] 1239 for case in results["cases"]: 1240 if case.get("skipped", False): 1241 skipped += 1 1242 else: 1243 if case.get("passed", False): 1244 passed += 1 1245 else: 1246 failed += 1 1247 failed_cases.append(case.get("name", "NONAME")) 1248 return { 1249 "passed": passed, 1250 "failed": failed, 1251 "skipped": skipped, 1252 "failed_cases": ", ".join(failed_cases), 1253 "percent": 1.0 * passed / (passed + failed), 1254 } 1255 1256 1257# A dictionary of prod servers to test against. 1258# See go/grpc-interop-tests (internal-only) for details. 1259prod_servers = { 1260 "default": "grpc-test.sandbox.googleapis.com", 1261 "gateway_v4": "grpc-test4.sandbox.googleapis.com", 1262} 1263 1264argp = argparse.ArgumentParser(description="Run interop tests.") 1265argp.add_argument( 1266 "-l", 1267 "--language", 1268 choices=["all"] + sorted(_LANGUAGES), 1269 nargs="+", 1270 default=["all"], 1271 help="Clients to run. Objc client can be only run on OSX.", 1272) 1273argp.add_argument("-j", "--jobs", default=multiprocessing.cpu_count(), type=int) 1274argp.add_argument( 1275 "--cloud_to_prod", 1276 default=False, 1277 action="store_const", 1278 const=True, 1279 help="Run cloud_to_prod tests.", 1280) 1281argp.add_argument( 1282 "--cloud_to_prod_auth", 1283 default=False, 1284 action="store_const", 1285 const=True, 1286 help="Run cloud_to_prod_auth tests.", 1287) 1288argp.add_argument( 1289 "--google_default_creds_use_key_file", 1290 default=False, 1291 action="store_const", 1292 const=True, 1293 help=( 1294 "Whether or not we should use a key file for the " 1295 "google_default_credentials test case, e.g. by " 1296 "setting env var GOOGLE_APPLICATION_CREDENTIALS." 1297 ), 1298) 1299argp.add_argument( 1300 "--prod_servers", 1301 choices=list(prod_servers.keys()), 1302 default=["default"], 1303 nargs="+", 1304 help=( 1305 "The servers to run cloud_to_prod and cloud_to_prod_auth tests against." 1306 ), 1307) 1308argp.add_argument( 1309 "-s", 1310 "--server", 1311 choices=["all"] + sorted(_SERVERS), 1312 nargs="+", 1313 help="Run cloud_to_cloud servers in a separate docker " 1314 + "image. Servers can only be started automatically if " 1315 + "--use_docker option is enabled.", 1316 default=[], 1317) 1318argp.add_argument( 1319 "--override_server", 1320 action="append", 1321 type=lambda kv: kv.split("="), 1322 help=( 1323 "Use servername=HOST:PORT to explicitly specify a server. E.g." 1324 " csharp=localhost:50000" 1325 ), 1326 default=[], 1327) 1328# TODO(jtattermusch): the default service_account_key_file only works when --use_docker is used. 1329argp.add_argument( 1330 "--service_account_key_file", 1331 type=str, 1332 help="The service account key file to use for some auth interop tests.", 1333 default="/root/service_account/grpc-testing-ebe7c1ac7381.json", 1334) 1335argp.add_argument( 1336 "--default_service_account", 1337 type=str, 1338 help=( 1339 "Default GCE service account email to use for some auth interop tests." 1340 ), 1341 default="830293263384-compute@developer.gserviceaccount.com", 1342) 1343argp.add_argument( 1344 "-t", 1345 "--travis", 1346 default=False, 1347 action="store_const", 1348 const=True, 1349 help=( 1350 "When set, indicates that the script is running on CI (= not locally)." 1351 ), 1352) 1353argp.add_argument( 1354 "-v", "--verbose", default=False, action="store_const", const=True 1355) 1356argp.add_argument( 1357 "--use_docker", 1358 default=False, 1359 action="store_const", 1360 const=True, 1361 help="Run all the interop tests under docker. That provides " 1362 + "additional isolation and prevents the need to install " 1363 + "language specific prerequisites. Only available on Linux.", 1364) 1365argp.add_argument( 1366 "--allow_flakes", 1367 default=False, 1368 action="store_const", 1369 const=True, 1370 help=( 1371 "Allow flaky tests to show as passing (re-runs failed tests up to five" 1372 " times)" 1373 ), 1374) 1375argp.add_argument( 1376 "--manual_run", 1377 default=False, 1378 action="store_const", 1379 const=True, 1380 help="Prepare things for running interop tests manually. " 1381 + "Preserve docker images after building them and skip " 1382 "actually running the tests. Only print commands to run by " + "hand.", 1383) 1384argp.add_argument( 1385 "--http2_interop", 1386 default=False, 1387 action="store_const", 1388 const=True, 1389 help="Enable HTTP/2 client edge case testing. (Bad client, good server)", 1390) 1391argp.add_argument( 1392 "--http2_server_interop", 1393 default=False, 1394 action="store_const", 1395 const=True, 1396 help=( 1397 "Enable HTTP/2 server edge case testing. (Includes positive and" 1398 " negative tests" 1399 ), 1400) 1401argp.add_argument( 1402 "--transport_security", 1403 choices=_TRANSPORT_SECURITY_OPTIONS, 1404 default="tls", 1405 type=str, 1406 nargs="?", 1407 const=True, 1408 help="Which transport security mechanism to use.", 1409) 1410argp.add_argument( 1411 "--custom_credentials_type", 1412 choices=_CUSTOM_CREDENTIALS_TYPE_OPTIONS, 1413 default=_CUSTOM_CREDENTIALS_TYPE_OPTIONS, 1414 nargs="+", 1415 help=( 1416 "Credential types to test in the cloud_to_prod setup. Default is to" 1417 " test with all creds types possible." 1418 ), 1419) 1420argp.add_argument( 1421 "--skip_compute_engine_creds", 1422 default=False, 1423 action="store_const", 1424 const=True, 1425 help="Skip auth tests requiring access to compute engine credentials.", 1426) 1427argp.add_argument( 1428 "--bq_result_table", 1429 default="", 1430 type=str, 1431 nargs="?", 1432 help="Upload test results to a specified BQ table.", 1433) 1434args = argp.parse_args() 1435 1436servers = set( 1437 s 1438 for s in itertools.chain.from_iterable( 1439 _SERVERS if x == "all" else [x] for x in args.server 1440 ) 1441) 1442# ALTS servers are only available for certain languages. 1443if args.transport_security == "alts": 1444 servers = servers.intersection(_SERVERS_FOR_ALTS_TEST_CASES) 1445 1446if args.use_docker: 1447 if not args.travis: 1448 print("Seen --use_docker flag, will run interop tests under docker.") 1449 print("") 1450 print( 1451 "IMPORTANT: The changes you are testing need to be locally" 1452 " committed" 1453 ) 1454 print( 1455 "because only the committed changes in the current branch will be" 1456 ) 1457 print("copied to the docker environment.") 1458 time.sleep(5) 1459 1460if args.manual_run and not args.use_docker: 1461 print("--manual_run is only supported with --use_docker option enabled.") 1462 sys.exit(1) 1463 1464if not args.use_docker and servers: 1465 print( 1466 "Running interop servers is only supported with --use_docker option" 1467 " enabled." 1468 ) 1469 sys.exit(1) 1470 1471# we want to include everything but objc in 'all' 1472# because objc won't run on non-mac platforms 1473all_but_objc = set(six.iterkeys(_LANGUAGES)) - set(["objc"]) 1474languages = set( 1475 _LANGUAGES[l] 1476 for l in itertools.chain.from_iterable( 1477 all_but_objc if x == "all" else [x] for x in args.language 1478 ) 1479) 1480# ALTS interop clients are only available for certain languages. 1481if args.transport_security == "alts": 1482 alts_languages = set(_LANGUAGES[l] for l in _LANGUAGES_FOR_ALTS_TEST_CASES) 1483 languages = languages.intersection(alts_languages) 1484 1485languages_http2_clients_for_http2_server_interop = set() 1486if args.http2_server_interop: 1487 languages_http2_clients_for_http2_server_interop = set( 1488 _LANGUAGES[l] 1489 for l in _LANGUAGES_WITH_HTTP2_CLIENTS_FOR_HTTP2_SERVER_TEST_CASES 1490 if "all" in args.language or l in args.language 1491 ) 1492 1493http2Interop = Http2Client() if args.http2_interop else None 1494http2InteropServer = Http2Server() if args.http2_server_interop else None 1495 1496docker_images = {} 1497if args.use_docker: 1498 # languages for which to build docker images 1499 languages_to_build = set( 1500 _LANGUAGES[k] 1501 for k in set([str(l) for l in languages] + [s for s in servers]) 1502 ) 1503 languages_to_build = ( 1504 languages_to_build | languages_http2_clients_for_http2_server_interop 1505 ) 1506 1507 if args.http2_interop: 1508 languages_to_build.add(http2Interop) 1509 1510 if args.http2_server_interop: 1511 languages_to_build.add(http2InteropServer) 1512 1513 build_jobs = [] 1514 for l in languages_to_build: 1515 if str(l) == "objc": 1516 # we don't need to build a docker image for objc 1517 continue 1518 job = build_interop_image_jobspec(l) 1519 docker_images[str(l)] = job.tag 1520 build_jobs.append(job) 1521 1522 if build_jobs: 1523 jobset.message( 1524 "START", "Building interop docker images.", do_newline=True 1525 ) 1526 if args.verbose: 1527 print("Jobs to run: \n%s\n" % "\n".join(str(j) for j in build_jobs)) 1528 1529 num_failures, build_resultset = jobset.run( 1530 build_jobs, newline_on_success=True, maxjobs=args.jobs 1531 ) 1532 1533 report_utils.render_junit_xml_report( 1534 build_resultset, _DOCKER_BUILD_XML_REPORT 1535 ) 1536 1537 if num_failures == 0: 1538 jobset.message( 1539 "SUCCESS", 1540 "All docker images built successfully.", 1541 do_newline=True, 1542 ) 1543 else: 1544 jobset.message( 1545 "FAILED", 1546 "Failed to build interop docker images.", 1547 do_newline=True, 1548 ) 1549 for image in six.itervalues(docker_images): 1550 dockerjob.remove_image(image, skip_nonexistent=True) 1551 sys.exit(1) 1552 1553server_manual_cmd_log = [] if args.manual_run else None 1554client_manual_cmd_log = [] if args.manual_run else None 1555 1556# Start interop servers. 1557server_jobs = {} 1558server_addresses = {} 1559try: 1560 for s in servers: 1561 lang = str(s) 1562 spec = server_jobspec( 1563 _LANGUAGES[lang], 1564 docker_images.get(lang), 1565 args.transport_security, 1566 manual_cmd_log=server_manual_cmd_log, 1567 ) 1568 if not args.manual_run: 1569 job = dockerjob.DockerJob(spec) 1570 server_jobs[lang] = job 1571 server_addresses[lang] = ( 1572 "localhost", 1573 job.mapped_port(_DEFAULT_SERVER_PORT), 1574 ) 1575 else: 1576 # don't run the server, set server port to a placeholder value 1577 server_addresses[lang] = ("localhost", "${SERVER_PORT}") 1578 1579 http2_server_job = None 1580 if args.http2_server_interop: 1581 # launch a HTTP2 server emulator that creates edge cases 1582 lang = str(http2InteropServer) 1583 spec = server_jobspec( 1584 http2InteropServer, 1585 docker_images.get(lang), 1586 manual_cmd_log=server_manual_cmd_log, 1587 ) 1588 if not args.manual_run: 1589 http2_server_job = dockerjob.DockerJob(spec) 1590 server_jobs[lang] = http2_server_job 1591 else: 1592 # don't run the server, set server port to a placeholder value 1593 server_addresses[lang] = ("localhost", "${SERVER_PORT}") 1594 1595 jobs = [] 1596 if args.cloud_to_prod: 1597 if args.transport_security not in ["tls"]: 1598 print("TLS is always enabled for cloud_to_prod scenarios.") 1599 for server_host_nickname in args.prod_servers: 1600 for language in languages: 1601 for test_case in _TEST_CASES: 1602 if not test_case in language.unimplemented_test_cases(): 1603 if ( 1604 not test_case 1605 in _SKIP_ADVANCED 1606 + _SKIP_COMPRESSION 1607 + _SKIP_SPECIAL_STATUS_MESSAGE 1608 + _ORCA_TEST_CASES 1609 ): 1610 for ( 1611 transport_security 1612 ) in args.custom_credentials_type: 1613 # google_default_credentials not yet supported by all languages 1614 if ( 1615 transport_security 1616 == "google_default_credentials" 1617 and str(language) 1618 not in ["c++", "go", "java", "javaokhttp"] 1619 ): 1620 continue 1621 # compute_engine_channel_creds not yet supported by all languages 1622 if ( 1623 transport_security 1624 == "compute_engine_channel_creds" 1625 and str(language) 1626 not in ["go", "java", "javaokhttp"] 1627 ): 1628 continue 1629 test_job = cloud_to_prod_jobspec( 1630 language, 1631 test_case, 1632 server_host_nickname, 1633 prod_servers[server_host_nickname], 1634 google_default_creds_use_key_file=args.google_default_creds_use_key_file, 1635 docker_image=docker_images.get( 1636 str(language) 1637 ), 1638 manual_cmd_log=client_manual_cmd_log, 1639 service_account_key_file=args.service_account_key_file, 1640 default_service_account=args.default_service_account, 1641 transport_security=transport_security, 1642 ) 1643 jobs.append(test_job) 1644 if args.http2_interop: 1645 for test_case in _HTTP2_TEST_CASES: 1646 test_job = cloud_to_prod_jobspec( 1647 http2Interop, 1648 test_case, 1649 server_host_nickname, 1650 prod_servers[server_host_nickname], 1651 google_default_creds_use_key_file=args.google_default_creds_use_key_file, 1652 docker_image=docker_images.get(str(http2Interop)), 1653 manual_cmd_log=client_manual_cmd_log, 1654 service_account_key_file=args.service_account_key_file, 1655 default_service_account=args.default_service_account, 1656 transport_security=args.transport_security, 1657 ) 1658 jobs.append(test_job) 1659 1660 if args.cloud_to_prod_auth: 1661 if args.transport_security not in ["tls"]: 1662 print("TLS is always enabled for cloud_to_prod scenarios.") 1663 for server_host_nickname in args.prod_servers: 1664 for language in languages: 1665 for test_case in _AUTH_TEST_CASES: 1666 if ( 1667 not args.skip_compute_engine_creds 1668 or not compute_engine_creds_required( 1669 language, test_case 1670 ) 1671 ): 1672 if not test_case in language.unimplemented_test_cases(): 1673 if test_case == _GOOGLE_DEFAULT_CREDS_TEST_CASE: 1674 transport_security = ( 1675 "google_default_credentials" 1676 ) 1677 elif ( 1678 test_case 1679 == _COMPUTE_ENGINE_CHANNEL_CREDS_TEST_CASE 1680 ): 1681 transport_security = ( 1682 "compute_engine_channel_creds" 1683 ) 1684 else: 1685 transport_security = "tls" 1686 if ( 1687 transport_security 1688 not in args.custom_credentials_type 1689 ): 1690 continue 1691 test_job = cloud_to_prod_jobspec( 1692 language, 1693 test_case, 1694 server_host_nickname, 1695 prod_servers[server_host_nickname], 1696 google_default_creds_use_key_file=args.google_default_creds_use_key_file, 1697 docker_image=docker_images.get(str(language)), 1698 auth=True, 1699 manual_cmd_log=client_manual_cmd_log, 1700 service_account_key_file=args.service_account_key_file, 1701 default_service_account=args.default_service_account, 1702 transport_security=transport_security, 1703 ) 1704 jobs.append(test_job) 1705 for server in args.override_server: 1706 server_name = server[0] 1707 (server_host, server_port) = server[1].split(":") 1708 server_addresses[server_name] = (server_host, server_port) 1709 1710 for server_name, server_address in list(server_addresses.items()): 1711 (server_host, server_port) = server_address 1712 server_language = _LANGUAGES.get(server_name, None) 1713 skip_server = [] # test cases unimplemented by server 1714 if server_language: 1715 skip_server = server_language.unimplemented_test_cases_server() 1716 for language in languages: 1717 for test_case in _TEST_CASES: 1718 if not test_case in language.unimplemented_test_cases(): 1719 if not test_case in skip_server: 1720 test_job = cloud_to_cloud_jobspec( 1721 language, 1722 test_case, 1723 server_name, 1724 server_host, 1725 server_port, 1726 docker_image=docker_images.get(str(language)), 1727 transport_security=args.transport_security, 1728 manual_cmd_log=client_manual_cmd_log, 1729 ) 1730 jobs.append(test_job) 1731 1732 if args.http2_interop: 1733 for test_case in _HTTP2_TEST_CASES: 1734 if server_name == "go": 1735 # TODO(carl-mastrangelo): Reenable after https://github.com/grpc/grpc-go/issues/434 1736 continue 1737 test_job = cloud_to_cloud_jobspec( 1738 http2Interop, 1739 test_case, 1740 server_name, 1741 server_host, 1742 server_port, 1743 docker_image=docker_images.get(str(http2Interop)), 1744 transport_security=args.transport_security, 1745 manual_cmd_log=client_manual_cmd_log, 1746 ) 1747 jobs.append(test_job) 1748 1749 if args.http2_server_interop: 1750 if not args.manual_run: 1751 http2_server_job.wait_for_healthy(timeout_seconds=600) 1752 for language in languages_http2_clients_for_http2_server_interop: 1753 for test_case in set(_HTTP2_SERVER_TEST_CASES) - set( 1754 _HTTP2_SERVER_TEST_CASES_THAT_USE_GRPC_CLIENTS 1755 ): 1756 offset = sorted(_HTTP2_SERVER_TEST_CASES).index(test_case) 1757 server_port = _DEFAULT_SERVER_PORT + offset 1758 if not args.manual_run: 1759 server_port = http2_server_job.mapped_port(server_port) 1760 test_job = cloud_to_cloud_jobspec( 1761 language, 1762 test_case, 1763 str(http2InteropServer), 1764 "localhost", 1765 server_port, 1766 docker_image=docker_images.get(str(language)), 1767 manual_cmd_log=client_manual_cmd_log, 1768 ) 1769 jobs.append(test_job) 1770 for language in languages: 1771 # HTTP2_SERVER_TEST_CASES_THAT_USE_GRPC_CLIENTS is a subset of 1772 # HTTP_SERVER_TEST_CASES, in which clients use their gRPC interop clients rather 1773 # than specialized http2 clients, reusing existing test implementations. 1774 # For example, in the "data_frame_padding" test, use language's gRPC 1775 # interop clients and make them think that they're running "large_unary" 1776 # test case. This avoids implementing a new test case in each language. 1777 for test_case in _HTTP2_SERVER_TEST_CASES_THAT_USE_GRPC_CLIENTS: 1778 if test_case not in language.unimplemented_test_cases(): 1779 offset = sorted(_HTTP2_SERVER_TEST_CASES).index(test_case) 1780 server_port = _DEFAULT_SERVER_PORT + offset 1781 if not args.manual_run: 1782 server_port = http2_server_job.mapped_port(server_port) 1783 if args.transport_security != "insecure": 1784 print( 1785 "Creating grpc client to http2 server test case " 1786 "with insecure connection, even though " 1787 "args.transport_security is not insecure. Http2 " 1788 "test server only supports insecure connections." 1789 ) 1790 test_job = cloud_to_cloud_jobspec( 1791 language, 1792 test_case, 1793 str(http2InteropServer), 1794 "localhost", 1795 server_port, 1796 docker_image=docker_images.get(str(language)), 1797 transport_security="insecure", 1798 manual_cmd_log=client_manual_cmd_log, 1799 ) 1800 jobs.append(test_job) 1801 1802 if not jobs: 1803 print("No jobs to run.") 1804 for image in six.itervalues(docker_images): 1805 dockerjob.remove_image(image, skip_nonexistent=True) 1806 sys.exit(1) 1807 1808 if args.manual_run: 1809 print("All tests will skipped --manual_run option is active.") 1810 1811 if args.verbose: 1812 print("Jobs to run: \n%s\n" % "\n".join(str(job) for job in jobs)) 1813 1814 num_failures, resultset = jobset.run( 1815 jobs, 1816 newline_on_success=True, 1817 maxjobs=args.jobs, 1818 skip_jobs=args.manual_run, 1819 ) 1820 if args.bq_result_table and resultset: 1821 upload_interop_results_to_bq(resultset, args.bq_result_table) 1822 if num_failures: 1823 jobset.message("FAILED", "Some tests failed", do_newline=True) 1824 else: 1825 jobset.message("SUCCESS", "All tests passed", do_newline=True) 1826 1827 write_cmdlog_maybe(server_manual_cmd_log, "interop_server_cmds.sh") 1828 write_cmdlog_maybe(client_manual_cmd_log, "interop_client_cmds.sh") 1829 1830 report_utils.render_junit_xml_report(resultset, _TESTS_XML_REPORT) 1831 1832 for name, job in list(resultset.items()): 1833 if "http2" in name: 1834 job[0].http2results = aggregate_http2_results(job[0].message) 1835 1836 http2_server_test_cases = ( 1837 _HTTP2_SERVER_TEST_CASES if args.http2_server_interop else [] 1838 ) 1839 1840 if num_failures: 1841 sys.exit(1) 1842 else: 1843 sys.exit(0) 1844finally: 1845 # Check if servers are still running. 1846 for server, job in list(server_jobs.items()): 1847 if not job.is_running(): 1848 print('Server "%s" has exited prematurely.' % server) 1849 1850 dockerjob.finish_jobs([j for j in six.itervalues(server_jobs)]) 1851 1852 for image in six.itervalues(docker_images): 1853 if not args.manual_run: 1854 print("Removing docker image %s" % image) 1855 dockerjob.remove_image(image) 1856 else: 1857 print("Preserving docker image: %s" % image) 1858