1#!/usr/bin/env python3 2# Copyright 2015 gRPC authors. 3# 4# Licensed under the Apache License, Version 2.0 (the "License"); 5# you may not use this file except in compliance with the License. 6# You may obtain a copy of the License at 7# 8# http://www.apache.org/licenses/LICENSE-2.0 9# 10# Unless required by applicable law or agreed to in writing, software 11# distributed under the License is distributed on an "AS IS" BASIS, 12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13# See the License for the specific language governing permissions and 14# limitations under the License. 15"""Run interop (cross-language) tests in parallel.""" 16 17from __future__ import print_function 18 19import argparse 20import atexit 21import itertools 22import json 23import multiprocessing 24import os 25import re 26import subprocess 27import sys 28import tempfile 29import time 30import traceback 31import uuid 32 33import six 34 35import python_utils.dockerjob as dockerjob 36import python_utils.jobset as jobset 37import python_utils.report_utils as report_utils 38 39# It's ok to not import because this is only necessary to upload results to BQ. 40try: 41 from python_utils.upload_test_results import upload_interop_results_to_bq 42except ImportError as e: 43 print(e) 44 45# Docker doesn't clean up after itself, so we do it on exit. 46atexit.register(lambda: subprocess.call(['stty', 'echo'])) 47 48ROOT = os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]), '../..')) 49os.chdir(ROOT) 50 51_DEFAULT_SERVER_PORT = 8080 52 53_SKIP_CLIENT_COMPRESSION = [ 54 'client_compressed_unary', 'client_compressed_streaming' 55] 56 57_SKIP_SERVER_COMPRESSION = [ 58 'server_compressed_unary', 'server_compressed_streaming' 59] 60 61_SKIP_COMPRESSION = _SKIP_CLIENT_COMPRESSION + _SKIP_SERVER_COMPRESSION 62 63_SKIP_ADVANCED = [ 64 'status_code_and_message', 'custom_metadata', 'unimplemented_method', 65 'unimplemented_service' 66] 67 68_SKIP_SPECIAL_STATUS_MESSAGE = ['special_status_message'] 69 70_ORCA_TEST_CASES = ['orca_per_rpc', 'orca_oob'] 71 72_GOOGLE_DEFAULT_CREDS_TEST_CASE = 'google_default_credentials' 73 74_SKIP_GOOGLE_DEFAULT_CREDS = [ 75 _GOOGLE_DEFAULT_CREDS_TEST_CASE, 76] 77 78_COMPUTE_ENGINE_CHANNEL_CREDS_TEST_CASE = 'compute_engine_channel_credentials' 79 80_SKIP_COMPUTE_ENGINE_CHANNEL_CREDS = [ 81 _COMPUTE_ENGINE_CHANNEL_CREDS_TEST_CASE, 82] 83 84_TEST_TIMEOUT = 3 * 60 85 86# disable this test on core-based languages, 87# see https://github.com/grpc/grpc/issues/9779 88_SKIP_DATA_FRAME_PADDING = ['data_frame_padding'] 89 90# report suffix "sponge_log.xml" is important for reports to get picked up by internal CI 91_DOCKER_BUILD_XML_REPORT = 'interop_docker_build/sponge_log.xml' 92_TESTS_XML_REPORT = 'interop_test/sponge_log.xml' 93 94 95class CXXLanguage: 96 97 def __init__(self): 98 self.client_cwd = None 99 self.server_cwd = None 100 self.http2_cwd = None 101 self.safename = 'cxx' 102 103 def client_cmd(self, args): 104 return ['cmake/build/interop_client'] + args 105 106 def client_cmd_http2interop(self, args): 107 return ['cmake/build/http2_client'] + args 108 109 def cloud_to_prod_env(self): 110 return {} 111 112 def server_cmd(self, args): 113 return ['cmake/build/interop_server'] + args 114 115 def global_env(self): 116 return {} 117 118 def unimplemented_test_cases(self): 119 return _SKIP_DATA_FRAME_PADDING + \ 120 _SKIP_SPECIAL_STATUS_MESSAGE + \ 121 _SKIP_COMPUTE_ENGINE_CHANNEL_CREDS 122 123 def unimplemented_test_cases_server(self): 124 return [] 125 126 def __str__(self): 127 return 'c++' 128 129 130class AspNetCoreLanguage: 131 132 def __init__(self): 133 self.client_cwd = '../grpc-dotnet/output/InteropTestsClient' 134 self.server_cwd = '../grpc-dotnet/output/InteropTestsWebsite' 135 self.safename = str(self) 136 137 def cloud_to_prod_env(self): 138 return {} 139 140 def client_cmd(self, args): 141 return ['dotnet', 'exec', 'InteropTestsClient.dll'] + args 142 143 def server_cmd(self, args): 144 return ['dotnet', 'exec', 'InteropTestsWebsite.dll'] + args 145 146 def global_env(self): 147 return {} 148 149 def unimplemented_test_cases(self): 150 return _SKIP_GOOGLE_DEFAULT_CREDS + \ 151 _SKIP_COMPUTE_ENGINE_CHANNEL_CREDS + \ 152 _ORCA_TEST_CASES 153 154 def unimplemented_test_cases_server(self): 155 return _ORCA_TEST_CASES 156 157 def __str__(self): 158 return 'aspnetcore' 159 160 161class DartLanguage: 162 163 def __init__(self): 164 self.client_cwd = '../grpc-dart/interop' 165 self.server_cwd = '../grpc-dart/interop' 166 self.http2_cwd = '../grpc-dart/interop' 167 self.safename = str(self) 168 169 def client_cmd(self, args): 170 return ['dart', 'bin/client.dart'] + args 171 172 def cloud_to_prod_env(self): 173 return {} 174 175 def server_cmd(self, args): 176 return ['dart', 'bin/server.dart'] + args 177 178 def global_env(self): 179 return {} 180 181 def unimplemented_test_cases(self): 182 return _SKIP_COMPRESSION + \ 183 _SKIP_SPECIAL_STATUS_MESSAGE + \ 184 _SKIP_GOOGLE_DEFAULT_CREDS + \ 185 _SKIP_COMPUTE_ENGINE_CHANNEL_CREDS + \ 186 _ORCA_TEST_CASES 187 188 def unimplemented_test_cases_server(self): 189 return _SKIP_COMPRESSION + _SKIP_SPECIAL_STATUS_MESSAGE + _ORCA_TEST_CASES 190 191 def __str__(self): 192 return 'dart' 193 194 195class JavaLanguage: 196 197 def __init__(self): 198 self.client_cwd = '../grpc-java' 199 self.server_cwd = '../grpc-java' 200 self.http2_cwd = '../grpc-java' 201 self.safename = str(self) 202 203 def client_cmd(self, args): 204 return ['./run-test-client.sh'] + args 205 206 def client_cmd_http2interop(self, args): 207 return [ 208 './interop-testing/build/install/grpc-interop-testing/bin/http2-client' 209 ] + args 210 211 def cloud_to_prod_env(self): 212 return {} 213 214 def server_cmd(self, args): 215 return ['./run-test-server.sh'] + args 216 217 def global_env(self): 218 return {} 219 220 def unimplemented_test_cases(self): 221 return [] 222 223 def unimplemented_test_cases_server(self): 224 # Does not support CompressedRequest feature. 225 # Only supports CompressedResponse feature for unary. 226 return _SKIP_CLIENT_COMPRESSION + ['server_compressed_streaming'] 227 228 def __str__(self): 229 return 'java' 230 231 232class JavaOkHttpClient: 233 234 def __init__(self): 235 self.client_cwd = '../grpc-java' 236 self.safename = 'java' 237 238 def client_cmd(self, args): 239 return ['./run-test-client.sh', '--use_okhttp=true'] + args 240 241 def cloud_to_prod_env(self): 242 return {} 243 244 def global_env(self): 245 return {} 246 247 def unimplemented_test_cases(self): 248 return _SKIP_DATA_FRAME_PADDING 249 250 def __str__(self): 251 return 'javaokhttp' 252 253 254class GoLanguage: 255 256 def __init__(self): 257 # TODO: this relies on running inside docker 258 self.client_cwd = '/go/src/google.golang.org/grpc/interop/client' 259 self.server_cwd = '/go/src/google.golang.org/grpc/interop/server' 260 self.http2_cwd = '/go/src/google.golang.org/grpc/interop/http2' 261 self.safename = str(self) 262 263 def client_cmd(self, args): 264 return ['go', 'run', 'client.go'] + args 265 266 def client_cmd_http2interop(self, args): 267 return ['go', 'run', 'negative_http2_client.go'] + args 268 269 def cloud_to_prod_env(self): 270 return {} 271 272 def server_cmd(self, args): 273 return ['go', 'run', 'server.go'] + args 274 275 def global_env(self): 276 return {'GO111MODULE': 'on'} 277 278 def unimplemented_test_cases(self): 279 return _SKIP_COMPRESSION 280 281 def unimplemented_test_cases_server(self): 282 return _SKIP_COMPRESSION 283 284 def __str__(self): 285 return 'go' 286 287 288class Http2Server: 289 """Represents the HTTP/2 Interop Test server 290 291 This pretends to be a language in order to be built and run, but really it 292 isn't. 293 """ 294 295 def __init__(self): 296 self.server_cwd = None 297 self.safename = str(self) 298 299 def server_cmd(self, args): 300 return ['python test/http2_test/http2_test_server.py'] 301 302 def cloud_to_prod_env(self): 303 return {} 304 305 def global_env(self): 306 return {} 307 308 def unimplemented_test_cases(self): 309 return _TEST_CASES + \ 310 _SKIP_DATA_FRAME_PADDING + \ 311 _SKIP_SPECIAL_STATUS_MESSAGE + \ 312 _SKIP_GOOGLE_DEFAULT_CREDS + \ 313 _SKIP_COMPUTE_ENGINE_CHANNEL_CREDS 314 315 def unimplemented_test_cases_server(self): 316 return _TEST_CASES 317 318 def __str__(self): 319 return 'http2' 320 321 322class Http2Client: 323 """Represents the HTTP/2 Interop Test 324 325 This pretends to be a language in order to be built and run, but really it 326 isn't. 327 """ 328 329 def __init__(self): 330 self.client_cwd = None 331 self.safename = str(self) 332 333 def client_cmd(self, args): 334 return ['tools/http2_interop/http2_interop.test', '-test.v'] + args 335 336 def cloud_to_prod_env(self): 337 return {} 338 339 def global_env(self): 340 return {} 341 342 def unimplemented_test_cases(self): 343 return _TEST_CASES + \ 344 _SKIP_SPECIAL_STATUS_MESSAGE + \ 345 _SKIP_GOOGLE_DEFAULT_CREDS + \ 346 _SKIP_COMPUTE_ENGINE_CHANNEL_CREDS 347 348 def unimplemented_test_cases_server(self): 349 return _TEST_CASES 350 351 def __str__(self): 352 return 'http2' 353 354 355class NodeLanguage: 356 357 def __init__(self): 358 self.client_cwd = '../../../../home/appuser/grpc-node' 359 self.server_cwd = '../../../../home/appuser/grpc-node' 360 self.safename = str(self) 361 362 def client_cmd(self, args): 363 return [ 364 'packages/grpc-native-core/deps/grpc/tools/run_tests/interop/with_nvm.sh', 365 'node', '--require', './test/fixtures/native_native', 366 'test/interop/interop_client.js' 367 ] + args 368 369 def cloud_to_prod_env(self): 370 return {} 371 372 def server_cmd(self, args): 373 return [ 374 'packages/grpc-native-core/deps/grpc/tools/run_tests/interop/with_nvm.sh', 375 'node', '--require', './test/fixtures/native_native', 376 'test/interop/interop_server.js' 377 ] + args 378 379 def global_env(self): 380 return {} 381 382 def unimplemented_test_cases(self): 383 return _SKIP_COMPRESSION + \ 384 _SKIP_DATA_FRAME_PADDING + \ 385 _SKIP_GOOGLE_DEFAULT_CREDS + \ 386 _SKIP_COMPUTE_ENGINE_CHANNEL_CREDS + \ 387 _ORCA_TEST_CASES 388 389 def unimplemented_test_cases_server(self): 390 return _SKIP_COMPRESSION + \ 391 _ORCA_TEST_CASES 392 393 def __str__(self): 394 return 'node' 395 396 397class NodePureJSLanguage: 398 399 def __init__(self): 400 self.client_cwd = '../../../../home/appuser/grpc-node' 401 self.server_cwd = '../../../../home/appuser/grpc-node' 402 self.safename = str(self) 403 404 def client_cmd(self, args): 405 return [ 406 'packages/grpc-native-core/deps/grpc/tools/run_tests/interop/with_nvm.sh', 407 'node', '--require', './test/fixtures/js_js', 408 'test/interop/interop_client.js' 409 ] + args 410 411 def cloud_to_prod_env(self): 412 return {} 413 414 def global_env(self): 415 return {} 416 417 def unimplemented_test_cases(self): 418 return _SKIP_COMPRESSION + \ 419 _SKIP_DATA_FRAME_PADDING + \ 420 _SKIP_GOOGLE_DEFAULT_CREDS + \ 421 _SKIP_COMPUTE_ENGINE_CHANNEL_CREDS + \ 422 _ORCA_TEST_CASES 423 424 def unimplemented_test_cases_server(self): 425 return _ORCA_TEST_CASES 426 427 def __str__(self): 428 return 'nodepurejs' 429 430 431class PHP7Language: 432 433 def __init__(self): 434 self.client_cwd = None 435 self.server_cwd = None 436 self.safename = str(self) 437 438 def client_cmd(self, args): 439 return ['src/php/bin/interop_client.sh'] + args 440 441 def cloud_to_prod_env(self): 442 return {} 443 444 def server_cmd(self, args): 445 return ['src/php/bin/interop_server.sh'] + args 446 447 def global_env(self): 448 return {} 449 450 def unimplemented_test_cases(self): 451 return _SKIP_SERVER_COMPRESSION + \ 452 _SKIP_DATA_FRAME_PADDING + \ 453 _SKIP_GOOGLE_DEFAULT_CREDS + \ 454 _SKIP_COMPUTE_ENGINE_CHANNEL_CREDS + \ 455 _ORCA_TEST_CASES 456 457 def unimplemented_test_cases_server(self): 458 return _SKIP_COMPRESSION + \ 459 _ORCA_TEST_CASES 460 461 def __str__(self): 462 return 'php7' 463 464 465class ObjcLanguage: 466 467 def __init__(self): 468 self.client_cwd = 'src/objective-c/tests' 469 self.safename = str(self) 470 471 def client_cmd(self, args): 472 # from args, extract the server port and craft xcodebuild command out of it 473 for arg in args: 474 port = re.search('--server_port=(\d+)', arg) 475 if port: 476 portnum = port.group(1) 477 cmdline = 'pod install && xcodebuild -workspace Tests.xcworkspace -scheme InteropTestsLocalSSL -destination name="iPhone 6" HOST_PORT_LOCALSSL=localhost:%s test' % portnum 478 return [cmdline] 479 480 def cloud_to_prod_env(self): 481 return {} 482 483 def global_env(self): 484 return {} 485 486 def unimplemented_test_cases(self): 487 # ObjC test runs all cases with the same command. It ignores the testcase 488 # cmdline argument. Here we return all but one test cases as unimplemented, 489 # and depend upon ObjC test's behavior that it runs all cases even when 490 # we tell it to run just one. 491 return _TEST_CASES[1:] + \ 492 _SKIP_COMPRESSION + \ 493 _SKIP_DATA_FRAME_PADDING + \ 494 _SKIP_SPECIAL_STATUS_MESSAGE + \ 495 _SKIP_GOOGLE_DEFAULT_CREDS + \ 496 _SKIP_COMPUTE_ENGINE_CHANNEL_CREDS + \ 497 _ORCA_TEST_CASES 498 499 def unimplemented_test_cases_server(self): 500 return _SKIP_COMPRESSION + \ 501 _ORCA_TEST_CASES 502 503 def __str__(self): 504 return 'objc' 505 506 507class RubyLanguage: 508 509 def __init__(self): 510 self.client_cwd = None 511 self.server_cwd = None 512 self.safename = str(self) 513 514 def client_cmd(self, args): 515 return [ 516 'tools/run_tests/interop/with_rvm.sh', 'ruby', 517 'src/ruby/pb/test/client.rb' 518 ] + args 519 520 def cloud_to_prod_env(self): 521 return {} 522 523 def server_cmd(self, args): 524 return [ 525 'tools/run_tests/interop/with_rvm.sh', 'ruby', 526 'src/ruby/pb/test/server.rb' 527 ] + args 528 529 def global_env(self): 530 return {} 531 532 def unimplemented_test_cases(self): 533 return _SKIP_SERVER_COMPRESSION + \ 534 _SKIP_DATA_FRAME_PADDING + \ 535 _SKIP_SPECIAL_STATUS_MESSAGE + \ 536 _SKIP_GOOGLE_DEFAULT_CREDS + \ 537 _SKIP_COMPUTE_ENGINE_CHANNEL_CREDS + \ 538 _ORCA_TEST_CASES 539 540 def unimplemented_test_cases_server(self): 541 return _SKIP_COMPRESSION + \ 542 _ORCA_TEST_CASES 543 544 def __str__(self): 545 return 'ruby' 546 547 548_PYTHON_BINARY = 'py39/bin/python' 549 550 551class PythonLanguage: 552 553 def __init__(self): 554 self.client_cwd = None 555 self.server_cwd = None 556 self.http2_cwd = None 557 self.safename = str(self) 558 559 def client_cmd(self, args): 560 return [ 561 _PYTHON_BINARY, 'src/python/grpcio_tests/setup.py', 'run_interop', 562 '--client', '--args="{}"'.format(' '.join(args)) 563 ] 564 565 def client_cmd_http2interop(self, args): 566 return [ 567 _PYTHON_BINARY, 568 'src/python/grpcio_tests/tests/http2/negative_http2_client.py', 569 ] + args 570 571 def cloud_to_prod_env(self): 572 return {} 573 574 def server_cmd(self, args): 575 return [ 576 _PYTHON_BINARY, 'src/python/grpcio_tests/setup.py', 'run_interop', 577 '--server', '--args="{}"'.format(' '.join(args)) 578 ] 579 580 def global_env(self): 581 return { 582 'LD_LIBRARY_PATH': '{}/libs/opt'.format(DOCKER_WORKDIR_ROOT), 583 'PYTHONPATH': '{}/src/python/gens'.format(DOCKER_WORKDIR_ROOT) 584 } 585 586 def unimplemented_test_cases(self): 587 return _SKIP_COMPRESSION + \ 588 _SKIP_DATA_FRAME_PADDING + \ 589 _SKIP_GOOGLE_DEFAULT_CREDS + \ 590 _SKIP_COMPUTE_ENGINE_CHANNEL_CREDS + \ 591 _ORCA_TEST_CASES 592 593 def unimplemented_test_cases_server(self): 594 return _SKIP_COMPRESSION + \ 595 _ORCA_TEST_CASES 596 597 def __str__(self): 598 return 'python' 599 600 601class PythonAsyncIOLanguage: 602 603 def __init__(self): 604 self.client_cwd = None 605 self.server_cwd = None 606 self.http2_cwd = None 607 self.safename = str(self) 608 609 def client_cmd(self, args): 610 return [ 611 _PYTHON_BINARY, 'src/python/grpcio_tests/setup.py', 'run_interop', 612 '--use-asyncio', '--client', '--args="{}"'.format(' '.join(args)) 613 ] 614 615 def client_cmd_http2interop(self, args): 616 return [ 617 _PYTHON_BINARY, 618 'src/python/grpcio_tests/tests/http2/negative_http2_client.py', 619 ] + args 620 621 def cloud_to_prod_env(self): 622 return {} 623 624 def server_cmd(self, args): 625 return [ 626 _PYTHON_BINARY, 'src/python/grpcio_tests/setup.py', 'run_interop', 627 '--use-asyncio', '--server', '--args="{}"'.format(' '.join(args)) 628 ] 629 630 def global_env(self): 631 return { 632 'LD_LIBRARY_PATH': '{}/libs/opt'.format(DOCKER_WORKDIR_ROOT), 633 'PYTHONPATH': '{}/src/python/gens'.format(DOCKER_WORKDIR_ROOT) 634 } 635 636 def unimplemented_test_cases(self): 637 # TODO(https://github.com/grpc/grpc/issues/21707) 638 return _SKIP_COMPRESSION + \ 639 _SKIP_DATA_FRAME_PADDING + \ 640 _AUTH_TEST_CASES + \ 641 ['timeout_on_sleeping_server'] + \ 642 _ORCA_TEST_CASES 643 644 def unimplemented_test_cases_server(self): 645 # TODO(https://github.com/grpc/grpc/issues/21749) 646 return _TEST_CASES + \ 647 _AUTH_TEST_CASES + \ 648 _HTTP2_TEST_CASES + \ 649 _HTTP2_SERVER_TEST_CASES 650 651 def __str__(self): 652 return 'pythonasyncio' 653 654 655_LANGUAGES = { 656 'c++': CXXLanguage(), 657 'aspnetcore': AspNetCoreLanguage(), 658 'dart': DartLanguage(), 659 'go': GoLanguage(), 660 'java': JavaLanguage(), 661 'javaokhttp': JavaOkHttpClient(), 662 'node': NodeLanguage(), 663 'nodepurejs': NodePureJSLanguage(), 664 'php7': PHP7Language(), 665 'objc': ObjcLanguage(), 666 'ruby': RubyLanguage(), 667 'python': PythonLanguage(), 668 'pythonasyncio': PythonAsyncIOLanguage(), 669} 670 671# languages supported as cloud_to_cloud servers 672_SERVERS = [ 673 'c++', 'node', 'aspnetcore', 'java', 'go', 'ruby', 'python', 'dart', 674 'pythonasyncio', 'php7' 675] 676 677_TEST_CASES = [ 678 'large_unary', 'empty_unary', 'ping_pong', 'empty_stream', 679 'client_streaming', 'server_streaming', 'cancel_after_begin', 680 'cancel_after_first_response', 'timeout_on_sleeping_server', 681 'custom_metadata', 'status_code_and_message', 'unimplemented_method', 682 'client_compressed_unary', 'server_compressed_unary', 683 'client_compressed_streaming', 'server_compressed_streaming', 684 'unimplemented_service', 'special_status_message', 'orca_per_rpc', 685 'orca_oob' 686] 687 688_AUTH_TEST_CASES = [ 689 'compute_engine_creds', 690 'jwt_token_creds', 691 'oauth2_auth_token', 692 'per_rpc_creds', 693 _GOOGLE_DEFAULT_CREDS_TEST_CASE, 694 _COMPUTE_ENGINE_CHANNEL_CREDS_TEST_CASE, 695] 696 697_HTTP2_TEST_CASES = ['tls', 'framing'] 698 699_HTTP2_SERVER_TEST_CASES = [ 700 'rst_after_header', 'rst_after_data', 'rst_during_data', 'goaway', 'ping', 701 'max_streams', 'data_frame_padding', 'no_df_padding_sanity_test' 702] 703 704_GRPC_CLIENT_TEST_CASES_FOR_HTTP2_SERVER_TEST_CASES = { 705 'data_frame_padding': 'large_unary', 706 'no_df_padding_sanity_test': 'large_unary' 707} 708 709_HTTP2_SERVER_TEST_CASES_THAT_USE_GRPC_CLIENTS = list( 710 _GRPC_CLIENT_TEST_CASES_FOR_HTTP2_SERVER_TEST_CASES.keys()) 711 712_LANGUAGES_WITH_HTTP2_CLIENTS_FOR_HTTP2_SERVER_TEST_CASES = [ 713 'java', 'go', 'python', 'c++' 714] 715 716_LANGUAGES_FOR_ALTS_TEST_CASES = ['java', 'go', 'c++', 'python'] 717 718_SERVERS_FOR_ALTS_TEST_CASES = ['java', 'go', 'c++', 'python'] 719 720_TRANSPORT_SECURITY_OPTIONS = ['tls', 'alts', 'insecure'] 721 722_CUSTOM_CREDENTIALS_TYPE_OPTIONS = [ 723 'tls', 'google_default_credentials', 'compute_engine_channel_creds' 724] 725 726DOCKER_WORKDIR_ROOT = '/var/local/git/grpc' 727 728 729def docker_run_cmdline(cmdline, image, docker_args=[], cwd=None, environ=None): 730 """Wraps given cmdline array to create 'docker run' cmdline from it.""" 731 732 # don't use '-t' even when TTY is available, since that would break 733 # the testcases generated by tools/interop_matrix/create_testcases.sh 734 docker_cmdline = ['docker', 'run', '-i', '--rm=true'] 735 736 # turn environ into -e docker args 737 if environ: 738 for k, v in list(environ.items()): 739 docker_cmdline += ['-e', '%s=%s' % (k, v)] 740 741 # set working directory 742 workdir = DOCKER_WORKDIR_ROOT 743 if cwd: 744 workdir = os.path.join(workdir, cwd) 745 docker_cmdline += ['-w', workdir] 746 747 docker_cmdline += docker_args + [image] + cmdline 748 return docker_cmdline 749 750 751def manual_cmdline(docker_cmdline, docker_image): 752 """Returns docker cmdline adjusted for manual invocation.""" 753 print_cmdline = [] 754 for item in docker_cmdline: 755 if item.startswith('--name='): 756 continue 757 if item == docker_image: 758 item = "$docker_image" 759 item = item.replace('"', '\\"') 760 # add quotes when necessary 761 if any(character.isspace() for character in item): 762 item = "\"%s\"" % item 763 print_cmdline.append(item) 764 return ' '.join(print_cmdline) 765 766 767def write_cmdlog_maybe(cmdlog, filename): 768 """Returns docker cmdline adjusted for manual invocation.""" 769 if cmdlog: 770 with open(filename, 'w') as logfile: 771 logfile.write('#!/bin/bash\n') 772 logfile.write('# DO NOT MODIFY\n') 773 logfile.write( 774 '# This file is generated by run_interop_tests.py/create_testcases.sh\n' 775 ) 776 logfile.writelines("%s\n" % line for line in cmdlog) 777 print('Command log written to file %s' % filename) 778 779 780def bash_cmdline(cmdline): 781 """Creates bash -c cmdline from args list.""" 782 # Use login shell: 783 # * makes error messages clearer if executables are missing 784 return ['bash', '-c', ' '.join(cmdline)] 785 786 787def compute_engine_creds_required(language, test_case): 788 """Returns True if given test requires access to compute engine creds.""" 789 language = str(language) 790 if test_case == 'compute_engine_creds': 791 return True 792 if test_case == 'oauth2_auth_token' and language == 'c++': 793 # C++ oauth2 test uses GCE creds because C++ only supports JWT 794 return True 795 return False 796 797 798def auth_options(language, test_case, google_default_creds_use_key_file, 799 service_account_key_file, default_service_account): 800 """Returns (cmdline, env) tuple with cloud_to_prod_auth test options.""" 801 802 language = str(language) 803 cmdargs = [] 804 env = {} 805 806 oauth_scope_arg = '--oauth_scope=https://www.googleapis.com/auth/xapi.zoo' 807 key_file_arg = '--service_account_key_file=%s' % service_account_key_file 808 default_account_arg = '--default_service_account=%s' % default_service_account 809 810 if test_case in ['jwt_token_creds', 'per_rpc_creds', 'oauth2_auth_token']: 811 if language in [ 812 'aspnetcore', 'node', 'php7', 'python', 'ruby', 'nodepurejs' 813 ]: 814 env['GOOGLE_APPLICATION_CREDENTIALS'] = service_account_key_file 815 else: 816 cmdargs += [key_file_arg] 817 818 if test_case in ['per_rpc_creds', 'oauth2_auth_token']: 819 cmdargs += [oauth_scope_arg] 820 821 if test_case == 'oauth2_auth_token' and language == 'c++': 822 # C++ oauth2 test uses GCE creds and thus needs to know the default account 823 cmdargs += [default_account_arg] 824 825 if test_case == 'compute_engine_creds': 826 cmdargs += [oauth_scope_arg, default_account_arg] 827 828 if test_case == _GOOGLE_DEFAULT_CREDS_TEST_CASE: 829 if google_default_creds_use_key_file: 830 env['GOOGLE_APPLICATION_CREDENTIALS'] = service_account_key_file 831 cmdargs += [default_account_arg] 832 833 if test_case == _COMPUTE_ENGINE_CHANNEL_CREDS_TEST_CASE: 834 cmdargs += [default_account_arg] 835 836 return (cmdargs, env) 837 838 839def _job_kill_handler(job): 840 if job._spec.container_name: 841 dockerjob.docker_kill(job._spec.container_name) 842 # When the job times out and we decide to kill it, 843 # we need to wait a before restarting the job 844 # to prevent "container name already in use" error. 845 # TODO(jtattermusch): figure out a cleaner way to this. 846 time.sleep(2) 847 848 849def cloud_to_prod_jobspec(language, 850 test_case, 851 server_host_nickname, 852 server_host, 853 google_default_creds_use_key_file, 854 docker_image=None, 855 auth=False, 856 manual_cmd_log=None, 857 service_account_key_file=None, 858 default_service_account=None, 859 transport_security='tls'): 860 """Creates jobspec for cloud-to-prod interop test""" 861 container_name = None 862 cmdargs = [ 863 '--server_host=%s' % server_host, '--server_port=443', 864 '--test_case=%s' % test_case 865 ] 866 if transport_security == 'tls': 867 transport_security_options = ['--use_tls=true'] 868 elif transport_security == 'google_default_credentials' and str( 869 language) in ['c++', 'go', 'java', 'javaokhttp']: 870 transport_security_options = [ 871 '--custom_credentials_type=google_default_credentials' 872 ] 873 elif transport_security == 'compute_engine_channel_creds' and str( 874 language) in ['go', 'java', 'javaokhttp']: 875 transport_security_options = [ 876 '--custom_credentials_type=compute_engine_channel_creds' 877 ] 878 else: 879 print( 880 'Invalid transport security option %s in cloud_to_prod_jobspec. Lang: %s' 881 % (str(language), transport_security)) 882 sys.exit(1) 883 cmdargs = cmdargs + transport_security_options 884 environ = dict(language.cloud_to_prod_env(), **language.global_env()) 885 if auth: 886 auth_cmdargs, auth_env = auth_options( 887 language, test_case, google_default_creds_use_key_file, 888 service_account_key_file, default_service_account) 889 cmdargs += auth_cmdargs 890 environ.update(auth_env) 891 cmdline = bash_cmdline(language.client_cmd(cmdargs)) 892 cwd = language.client_cwd 893 894 if docker_image: 895 container_name = dockerjob.random_name('interop_client_%s' % 896 language.safename) 897 cmdline = docker_run_cmdline( 898 cmdline, 899 image=docker_image, 900 cwd=cwd, 901 environ=environ, 902 docker_args=['--net=host', 903 '--name=%s' % container_name]) 904 if manual_cmd_log is not None: 905 if manual_cmd_log == []: 906 manual_cmd_log.append('echo "Testing ${docker_image:=%s}"' % 907 docker_image) 908 manual_cmd_log.append(manual_cmdline(cmdline, docker_image)) 909 cwd = None 910 environ = None 911 912 suite_name = 'cloud_to_prod_auth' if auth else 'cloud_to_prod' 913 test_job = jobset.JobSpec(cmdline=cmdline, 914 cwd=cwd, 915 environ=environ, 916 shortname='%s:%s:%s:%s:%s' % 917 (suite_name, language, server_host_nickname, 918 test_case, transport_security), 919 timeout_seconds=_TEST_TIMEOUT, 920 flake_retries=4 if args.allow_flakes else 0, 921 timeout_retries=2 if args.allow_flakes else 0, 922 kill_handler=_job_kill_handler) 923 if docker_image: 924 test_job.container_name = container_name 925 return test_job 926 927 928def cloud_to_cloud_jobspec(language, 929 test_case, 930 server_name, 931 server_host, 932 server_port, 933 docker_image=None, 934 transport_security='tls', 935 manual_cmd_log=None): 936 """Creates jobspec for cloud-to-cloud interop test""" 937 interop_only_options = [ 938 '--server_host_override=foo.test.google.fr', 939 '--use_test_ca=true', 940 ] 941 if transport_security == 'tls': 942 interop_only_options += ['--use_tls=true'] 943 elif transport_security == 'alts': 944 interop_only_options += ['--use_tls=false', '--use_alts=true'] 945 elif transport_security == 'insecure': 946 interop_only_options += ['--use_tls=false'] 947 else: 948 print( 949 'Invalid transport security option %s in cloud_to_cloud_jobspec.' % 950 transport_security) 951 sys.exit(1) 952 953 client_test_case = test_case 954 if test_case in _HTTP2_SERVER_TEST_CASES_THAT_USE_GRPC_CLIENTS: 955 client_test_case = _GRPC_CLIENT_TEST_CASES_FOR_HTTP2_SERVER_TEST_CASES[ 956 test_case] 957 if client_test_case in language.unimplemented_test_cases(): 958 print('asking client %s to run unimplemented test case %s' % 959 (repr(language), client_test_case)) 960 sys.exit(1) 961 962 if test_case in _ORCA_TEST_CASES: 963 interop_only_options += [ 964 '--service_config_json=\'{"loadBalancingConfig":[{"test_backend_metrics_load_balancer":{}}]}\'' 965 ] 966 967 common_options = [ 968 '--test_case=%s' % client_test_case, 969 '--server_host=%s' % server_host, 970 '--server_port=%s' % server_port, 971 ] 972 973 if test_case in _HTTP2_SERVER_TEST_CASES: 974 if test_case in _HTTP2_SERVER_TEST_CASES_THAT_USE_GRPC_CLIENTS: 975 client_options = interop_only_options + common_options 976 cmdline = bash_cmdline(language.client_cmd(client_options)) 977 cwd = language.client_cwd 978 else: 979 cmdline = bash_cmdline( 980 language.client_cmd_http2interop(common_options)) 981 cwd = language.http2_cwd 982 else: 983 cmdline = bash_cmdline( 984 language.client_cmd(common_options + interop_only_options)) 985 cwd = language.client_cwd 986 987 environ = language.global_env() 988 if docker_image and language.safename != 'objc': 989 # we can't run client in docker for objc. 990 container_name = dockerjob.random_name('interop_client_%s' % 991 language.safename) 992 cmdline = docker_run_cmdline( 993 cmdline, 994 image=docker_image, 995 environ=environ, 996 cwd=cwd, 997 docker_args=['--net=host', 998 '--name=%s' % container_name]) 999 if manual_cmd_log is not None: 1000 if manual_cmd_log == []: 1001 manual_cmd_log.append('echo "Testing ${docker_image:=%s}"' % 1002 docker_image) 1003 manual_cmd_log.append(manual_cmdline(cmdline, docker_image)) 1004 cwd = None 1005 1006 test_job = jobset.JobSpec( 1007 cmdline=cmdline, 1008 cwd=cwd, 1009 environ=environ, 1010 shortname='cloud_to_cloud:%s:%s_server:%s:%s' % 1011 (language, server_name, test_case, transport_security), 1012 timeout_seconds=_TEST_TIMEOUT, 1013 flake_retries=4 if args.allow_flakes else 0, 1014 timeout_retries=2 if args.allow_flakes else 0, 1015 kill_handler=_job_kill_handler) 1016 if docker_image: 1017 test_job.container_name = container_name 1018 return test_job 1019 1020 1021def server_jobspec(language, 1022 docker_image, 1023 transport_security='tls', 1024 manual_cmd_log=None): 1025 """Create jobspec for running a server""" 1026 container_name = dockerjob.random_name('interop_server_%s' % 1027 language.safename) 1028 server_cmd = ['--port=%s' % _DEFAULT_SERVER_PORT] 1029 if transport_security == 'tls': 1030 server_cmd += ['--use_tls=true'] 1031 elif transport_security == 'alts': 1032 server_cmd += ['--use_tls=false', '--use_alts=true'] 1033 elif transport_security == 'insecure': 1034 server_cmd += ['--use_tls=false'] 1035 else: 1036 print('Invalid transport security option %s in server_jobspec.' % 1037 transport_security) 1038 sys.exit(1) 1039 cmdline = bash_cmdline(language.server_cmd(server_cmd)) 1040 environ = language.global_env() 1041 docker_args = ['--name=%s' % container_name] 1042 if language.safename == 'http2': 1043 # we are running the http2 interop server. Open next N ports beginning 1044 # with the server port. These ports are used for http2 interop test 1045 # (one test case per port). 1046 docker_args += list( 1047 itertools.chain.from_iterable( 1048 ('-p', str(_DEFAULT_SERVER_PORT + i)) 1049 for i in range(len(_HTTP2_SERVER_TEST_CASES)))) 1050 # Enable docker's healthcheck mechanism. 1051 # This runs a Python script inside the container every second. The script 1052 # pings the http2 server to verify it is ready. The 'health-retries' flag 1053 # specifies the number of consecutive failures before docker will report 1054 # the container's status as 'unhealthy'. Prior to the first 'health_retries' 1055 # failures or the first success, the status will be 'starting'. 'docker ps' 1056 # or 'docker inspect' can be used to see the health of the container on the 1057 # command line. 1058 docker_args += [ 1059 '--health-cmd=python test/http2_test/http2_server_health_check.py ' 1060 '--server_host=%s --server_port=%d' % 1061 ('localhost', _DEFAULT_SERVER_PORT), 1062 '--health-interval=1s', 1063 '--health-retries=5', 1064 '--health-timeout=10s', 1065 ] 1066 1067 else: 1068 docker_args += ['-p', str(_DEFAULT_SERVER_PORT)] 1069 1070 docker_cmdline = docker_run_cmdline(cmdline, 1071 image=docker_image, 1072 cwd=language.server_cwd, 1073 environ=environ, 1074 docker_args=docker_args) 1075 if manual_cmd_log is not None: 1076 if manual_cmd_log == []: 1077 manual_cmd_log.append('echo "Testing ${docker_image:=%s}"' % 1078 docker_image) 1079 manual_cmd_log.append(manual_cmdline(docker_cmdline, docker_image)) 1080 server_job = jobset.JobSpec(cmdline=docker_cmdline, 1081 environ=environ, 1082 shortname='interop_server_%s' % language, 1083 timeout_seconds=30 * 60) 1084 server_job.container_name = container_name 1085 return server_job 1086 1087 1088def build_interop_image_jobspec(language, tag=None): 1089 """Creates jobspec for building interop docker image for a language""" 1090 if not tag: 1091 tag = 'grpc_interop_%s:%s' % (language.safename, uuid.uuid4()) 1092 env = { 1093 'INTEROP_IMAGE': tag, 1094 'BASE_NAME': 'grpc_interop_%s' % language.safename 1095 } 1096 build_job = jobset.JobSpec( 1097 cmdline=['tools/run_tests/dockerize/build_interop_image.sh'], 1098 environ=env, 1099 shortname='build_docker_%s' % (language), 1100 timeout_seconds=30 * 60) 1101 build_job.tag = tag 1102 return build_job 1103 1104 1105def aggregate_http2_results(stdout): 1106 match = re.search(r'\{"cases[^\]]*\]\}', stdout) 1107 if not match: 1108 return None 1109 1110 results = json.loads(match.group(0)) 1111 skipped = 0 1112 passed = 0 1113 failed = 0 1114 failed_cases = [] 1115 for case in results['cases']: 1116 if case.get('skipped', False): 1117 skipped += 1 1118 else: 1119 if case.get('passed', False): 1120 passed += 1 1121 else: 1122 failed += 1 1123 failed_cases.append(case.get('name', "NONAME")) 1124 return { 1125 'passed': passed, 1126 'failed': failed, 1127 'skipped': skipped, 1128 'failed_cases': ', '.join(failed_cases), 1129 'percent': 1.0 * passed / (passed + failed) 1130 } 1131 1132 1133# A dictionary of prod servers to test against. 1134# See go/grpc-interop-tests (internal-only) for details. 1135prod_servers = { 1136 'default': 'grpc-test.sandbox.googleapis.com', 1137 'gateway_v4': 'grpc-test4.sandbox.googleapis.com', 1138} 1139 1140argp = argparse.ArgumentParser(description='Run interop tests.') 1141argp.add_argument('-l', 1142 '--language', 1143 choices=['all'] + sorted(_LANGUAGES), 1144 nargs='+', 1145 default=['all'], 1146 help='Clients to run. Objc client can be only run on OSX.') 1147argp.add_argument('-j', '--jobs', default=multiprocessing.cpu_count(), type=int) 1148argp.add_argument('--cloud_to_prod', 1149 default=False, 1150 action='store_const', 1151 const=True, 1152 help='Run cloud_to_prod tests.') 1153argp.add_argument('--cloud_to_prod_auth', 1154 default=False, 1155 action='store_const', 1156 const=True, 1157 help='Run cloud_to_prod_auth tests.') 1158argp.add_argument('--google_default_creds_use_key_file', 1159 default=False, 1160 action='store_const', 1161 const=True, 1162 help=('Whether or not we should use a key file for the ' 1163 'google_default_credentials test case, e.g. by ' 1164 'setting env var GOOGLE_APPLICATION_CREDENTIALS.')) 1165argp.add_argument('--prod_servers', 1166 choices=list(prod_servers.keys()), 1167 default=['default'], 1168 nargs='+', 1169 help=('The servers to run cloud_to_prod and ' 1170 'cloud_to_prod_auth tests against.')) 1171argp.add_argument('-s', 1172 '--server', 1173 choices=['all'] + sorted(_SERVERS), 1174 nargs='+', 1175 help='Run cloud_to_cloud servers in a separate docker ' + 1176 'image. Servers can only be started automatically if ' + 1177 '--use_docker option is enabled.', 1178 default=[]) 1179argp.add_argument( 1180 '--override_server', 1181 action='append', 1182 type=lambda kv: kv.split('='), 1183 help= 1184 'Use servername=HOST:PORT to explicitly specify a server. E.g. csharp=localhost:50000', 1185 default=[]) 1186# TODO(jtattermusch): the default service_account_key_file only works when --use_docker is used. 1187argp.add_argument( 1188 '--service_account_key_file', 1189 type=str, 1190 help='The service account key file to use for some auth interop tests.', 1191 default='/root/service_account/grpc-testing-ebe7c1ac7381.json') 1192argp.add_argument( 1193 '--default_service_account', 1194 type=str, 1195 help='Default GCE service account email to use for some auth interop tests.', 1196 default='830293263384-compute@developer.gserviceaccount.com') 1197argp.add_argument( 1198 '-t', 1199 '--travis', 1200 default=False, 1201 action='store_const', 1202 const=True, 1203 help='When set, indicates that the script is running on CI (= not locally).' 1204) 1205argp.add_argument('-v', 1206 '--verbose', 1207 default=False, 1208 action='store_const', 1209 const=True) 1210argp.add_argument( 1211 '--use_docker', 1212 default=False, 1213 action='store_const', 1214 const=True, 1215 help='Run all the interop tests under docker. That provides ' + 1216 'additional isolation and prevents the need to install ' + 1217 'language specific prerequisites. Only available on Linux.') 1218argp.add_argument( 1219 '--allow_flakes', 1220 default=False, 1221 action='store_const', 1222 const=True, 1223 help= 1224 'Allow flaky tests to show as passing (re-runs failed tests up to five times)' 1225) 1226argp.add_argument('--manual_run', 1227 default=False, 1228 action='store_const', 1229 const=True, 1230 help='Prepare things for running interop tests manually. ' + 1231 'Preserve docker images after building them and skip ' 1232 'actually running the tests. Only print commands to run by ' + 1233 'hand.') 1234argp.add_argument( 1235 '--http2_interop', 1236 default=False, 1237 action='store_const', 1238 const=True, 1239 help='Enable HTTP/2 client edge case testing. (Bad client, good server)') 1240argp.add_argument( 1241 '--http2_server_interop', 1242 default=False, 1243 action='store_const', 1244 const=True, 1245 help= 1246 'Enable HTTP/2 server edge case testing. (Includes positive and negative tests' 1247) 1248argp.add_argument('--transport_security', 1249 choices=_TRANSPORT_SECURITY_OPTIONS, 1250 default='tls', 1251 type=str, 1252 nargs='?', 1253 const=True, 1254 help='Which transport security mechanism to use.') 1255argp.add_argument( 1256 '--custom_credentials_type', 1257 choices=_CUSTOM_CREDENTIALS_TYPE_OPTIONS, 1258 default=_CUSTOM_CREDENTIALS_TYPE_OPTIONS, 1259 nargs='+', 1260 help= 1261 'Credential types to test in the cloud_to_prod setup. Default is to test with all creds types possible.' 1262) 1263argp.add_argument( 1264 '--skip_compute_engine_creds', 1265 default=False, 1266 action='store_const', 1267 const=True, 1268 help='Skip auth tests requiring access to compute engine credentials.') 1269argp.add_argument( 1270 '--internal_ci', 1271 default=False, 1272 action='store_const', 1273 const=True, 1274 help=( 1275 '(Deprecated, has no effect) Put reports into subdirectories to improve ' 1276 'presentation of results by Internal CI.')) 1277argp.add_argument('--bq_result_table', 1278 default='', 1279 type=str, 1280 nargs='?', 1281 help='Upload test results to a specified BQ table.') 1282args = argp.parse_args() 1283 1284servers = set(s for s in itertools.chain.from_iterable( 1285 _SERVERS if x == 'all' else [x] for x in args.server)) 1286# ALTS servers are only available for certain languages. 1287if args.transport_security == 'alts': 1288 servers = servers.intersection(_SERVERS_FOR_ALTS_TEST_CASES) 1289 1290if args.use_docker: 1291 if not args.travis: 1292 print('Seen --use_docker flag, will run interop tests under docker.') 1293 print('') 1294 print( 1295 'IMPORTANT: The changes you are testing need to be locally committed' 1296 ) 1297 print( 1298 'because only the committed changes in the current branch will be') 1299 print('copied to the docker environment.') 1300 time.sleep(5) 1301 1302if args.manual_run and not args.use_docker: 1303 print('--manual_run is only supported with --use_docker option enabled.') 1304 sys.exit(1) 1305 1306if not args.use_docker and servers: 1307 print( 1308 'Running interop servers is only supported with --use_docker option enabled.' 1309 ) 1310 sys.exit(1) 1311 1312# we want to include everything but objc in 'all' 1313# because objc won't run on non-mac platforms 1314all_but_objc = set(six.iterkeys(_LANGUAGES)) - set(['objc']) 1315languages = set(_LANGUAGES[l] for l in itertools.chain.from_iterable( 1316 all_but_objc if x == 'all' else [x] for x in args.language)) 1317# ALTS interop clients are only available for certain languages. 1318if args.transport_security == 'alts': 1319 alts_languages = set(_LANGUAGES[l] for l in _LANGUAGES_FOR_ALTS_TEST_CASES) 1320 languages = languages.intersection(alts_languages) 1321 1322languages_http2_clients_for_http2_server_interop = set() 1323if args.http2_server_interop: 1324 languages_http2_clients_for_http2_server_interop = set( 1325 _LANGUAGES[l] 1326 for l in _LANGUAGES_WITH_HTTP2_CLIENTS_FOR_HTTP2_SERVER_TEST_CASES 1327 if 'all' in args.language or l in args.language) 1328 1329http2Interop = Http2Client() if args.http2_interop else None 1330http2InteropServer = Http2Server() if args.http2_server_interop else None 1331 1332docker_images = {} 1333if args.use_docker: 1334 # languages for which to build docker images 1335 languages_to_build = set(_LANGUAGES[k] 1336 for k in set([str(l) for l in languages] + 1337 [s for s in servers])) 1338 languages_to_build = languages_to_build | languages_http2_clients_for_http2_server_interop 1339 1340 if args.http2_interop: 1341 languages_to_build.add(http2Interop) 1342 1343 if args.http2_server_interop: 1344 languages_to_build.add(http2InteropServer) 1345 1346 build_jobs = [] 1347 for l in languages_to_build: 1348 if str(l) == 'objc': 1349 # we don't need to build a docker image for objc 1350 continue 1351 job = build_interop_image_jobspec(l) 1352 docker_images[str(l)] = job.tag 1353 build_jobs.append(job) 1354 1355 if build_jobs: 1356 jobset.message('START', 1357 'Building interop docker images.', 1358 do_newline=True) 1359 if args.verbose: 1360 print('Jobs to run: \n%s\n' % '\n'.join(str(j) for j in build_jobs)) 1361 1362 num_failures, build_resultset = jobset.run(build_jobs, 1363 newline_on_success=True, 1364 maxjobs=args.jobs) 1365 1366 report_utils.render_junit_xml_report(build_resultset, 1367 _DOCKER_BUILD_XML_REPORT) 1368 1369 if num_failures == 0: 1370 jobset.message('SUCCESS', 1371 'All docker images built successfully.', 1372 do_newline=True) 1373 else: 1374 jobset.message('FAILED', 1375 'Failed to build interop docker images.', 1376 do_newline=True) 1377 for image in six.itervalues(docker_images): 1378 dockerjob.remove_image(image, skip_nonexistent=True) 1379 sys.exit(1) 1380 1381server_manual_cmd_log = [] if args.manual_run else None 1382client_manual_cmd_log = [] if args.manual_run else None 1383 1384# Start interop servers. 1385server_jobs = {} 1386server_addresses = {} 1387try: 1388 for s in servers: 1389 lang = str(s) 1390 spec = server_jobspec(_LANGUAGES[lang], 1391 docker_images.get(lang), 1392 args.transport_security, 1393 manual_cmd_log=server_manual_cmd_log) 1394 if not args.manual_run: 1395 job = dockerjob.DockerJob(spec) 1396 server_jobs[lang] = job 1397 server_addresses[lang] = ('localhost', 1398 job.mapped_port(_DEFAULT_SERVER_PORT)) 1399 else: 1400 # don't run the server, set server port to a placeholder value 1401 server_addresses[lang] = ('localhost', '${SERVER_PORT}') 1402 1403 http2_server_job = None 1404 if args.http2_server_interop: 1405 # launch a HTTP2 server emulator that creates edge cases 1406 lang = str(http2InteropServer) 1407 spec = server_jobspec(http2InteropServer, 1408 docker_images.get(lang), 1409 manual_cmd_log=server_manual_cmd_log) 1410 if not args.manual_run: 1411 http2_server_job = dockerjob.DockerJob(spec) 1412 server_jobs[lang] = http2_server_job 1413 else: 1414 # don't run the server, set server port to a placeholder value 1415 server_addresses[lang] = ('localhost', '${SERVER_PORT}') 1416 1417 jobs = [] 1418 if args.cloud_to_prod: 1419 if args.transport_security not in ['tls']: 1420 print('TLS is always enabled for cloud_to_prod scenarios.') 1421 for server_host_nickname in args.prod_servers: 1422 for language in languages: 1423 for test_case in _TEST_CASES: 1424 if not test_case in language.unimplemented_test_cases(): 1425 if not test_case in _SKIP_ADVANCED + _SKIP_COMPRESSION + _SKIP_SPECIAL_STATUS_MESSAGE + _ORCA_TEST_CASES: 1426 for transport_security in args.custom_credentials_type: 1427 # google_default_credentials not yet supported by all languages 1428 if transport_security == 'google_default_credentials' and str( 1429 language) not in [ 1430 'c++', 'go', 'java', 'javaokhttp' 1431 ]: 1432 continue 1433 # compute_engine_channel_creds not yet supported by all languages 1434 if transport_security == 'compute_engine_channel_creds' and str( 1435 language) not in [ 1436 'go', 'java', 'javaokhttp' 1437 ]: 1438 continue 1439 test_job = cloud_to_prod_jobspec( 1440 language, 1441 test_case, 1442 server_host_nickname, 1443 prod_servers[server_host_nickname], 1444 google_default_creds_use_key_file=args. 1445 google_default_creds_use_key_file, 1446 docker_image=docker_images.get( 1447 str(language)), 1448 manual_cmd_log=client_manual_cmd_log, 1449 service_account_key_file=args. 1450 service_account_key_file, 1451 default_service_account=args. 1452 default_service_account, 1453 transport_security=transport_security) 1454 jobs.append(test_job) 1455 if args.http2_interop: 1456 for test_case in _HTTP2_TEST_CASES: 1457 test_job = cloud_to_prod_jobspec( 1458 http2Interop, 1459 test_case, 1460 server_host_nickname, 1461 prod_servers[server_host_nickname], 1462 google_default_creds_use_key_file=args. 1463 google_default_creds_use_key_file, 1464 docker_image=docker_images.get(str(http2Interop)), 1465 manual_cmd_log=client_manual_cmd_log, 1466 service_account_key_file=args.service_account_key_file, 1467 default_service_account=args.default_service_account, 1468 transport_security=args.transport_security) 1469 jobs.append(test_job) 1470 1471 if args.cloud_to_prod_auth: 1472 if args.transport_security not in ['tls']: 1473 print('TLS is always enabled for cloud_to_prod scenarios.') 1474 for server_host_nickname in args.prod_servers: 1475 for language in languages: 1476 for test_case in _AUTH_TEST_CASES: 1477 if (not args.skip_compute_engine_creds or 1478 not compute_engine_creds_required( 1479 language, test_case)): 1480 if not test_case in language.unimplemented_test_cases(): 1481 if test_case == _GOOGLE_DEFAULT_CREDS_TEST_CASE: 1482 transport_security = 'google_default_credentials' 1483 elif test_case == _COMPUTE_ENGINE_CHANNEL_CREDS_TEST_CASE: 1484 transport_security = 'compute_engine_channel_creds' 1485 else: 1486 transport_security = 'tls' 1487 if transport_security not in args.custom_credentials_type: 1488 continue 1489 test_job = cloud_to_prod_jobspec( 1490 language, 1491 test_case, 1492 server_host_nickname, 1493 prod_servers[server_host_nickname], 1494 google_default_creds_use_key_file=args. 1495 google_default_creds_use_key_file, 1496 docker_image=docker_images.get(str(language)), 1497 auth=True, 1498 manual_cmd_log=client_manual_cmd_log, 1499 service_account_key_file=args. 1500 service_account_key_file, 1501 default_service_account=args. 1502 default_service_account, 1503 transport_security=transport_security) 1504 jobs.append(test_job) 1505 for server in args.override_server: 1506 server_name = server[0] 1507 (server_host, server_port) = server[1].split(':') 1508 server_addresses[server_name] = (server_host, server_port) 1509 1510 for server_name, server_address in list(server_addresses.items()): 1511 (server_host, server_port) = server_address 1512 server_language = _LANGUAGES.get(server_name, None) 1513 skip_server = [] # test cases unimplemented by server 1514 if server_language: 1515 skip_server = server_language.unimplemented_test_cases_server() 1516 for language in languages: 1517 for test_case in _TEST_CASES: 1518 if not test_case in language.unimplemented_test_cases(): 1519 if not test_case in skip_server: 1520 test_job = cloud_to_cloud_jobspec( 1521 language, 1522 test_case, 1523 server_name, 1524 server_host, 1525 server_port, 1526 docker_image=docker_images.get(str(language)), 1527 transport_security=args.transport_security, 1528 manual_cmd_log=client_manual_cmd_log) 1529 jobs.append(test_job) 1530 1531 if args.http2_interop: 1532 for test_case in _HTTP2_TEST_CASES: 1533 if server_name == "go": 1534 # TODO(carl-mastrangelo): Reenable after https://github.com/grpc/grpc-go/issues/434 1535 continue 1536 test_job = cloud_to_cloud_jobspec( 1537 http2Interop, 1538 test_case, 1539 server_name, 1540 server_host, 1541 server_port, 1542 docker_image=docker_images.get(str(http2Interop)), 1543 transport_security=args.transport_security, 1544 manual_cmd_log=client_manual_cmd_log) 1545 jobs.append(test_job) 1546 1547 if args.http2_server_interop: 1548 if not args.manual_run: 1549 http2_server_job.wait_for_healthy(timeout_seconds=600) 1550 for language in languages_http2_clients_for_http2_server_interop: 1551 for test_case in set(_HTTP2_SERVER_TEST_CASES) - set( 1552 _HTTP2_SERVER_TEST_CASES_THAT_USE_GRPC_CLIENTS): 1553 offset = sorted(_HTTP2_SERVER_TEST_CASES).index(test_case) 1554 server_port = _DEFAULT_SERVER_PORT + offset 1555 if not args.manual_run: 1556 server_port = http2_server_job.mapped_port(server_port) 1557 test_job = cloud_to_cloud_jobspec( 1558 language, 1559 test_case, 1560 str(http2InteropServer), 1561 'localhost', 1562 server_port, 1563 docker_image=docker_images.get(str(language)), 1564 manual_cmd_log=client_manual_cmd_log) 1565 jobs.append(test_job) 1566 for language in languages: 1567 # HTTP2_SERVER_TEST_CASES_THAT_USE_GRPC_CLIENTS is a subset of 1568 # HTTP_SERVER_TEST_CASES, in which clients use their gRPC interop clients rather 1569 # than specialized http2 clients, reusing existing test implementations. 1570 # For example, in the "data_frame_padding" test, use language's gRPC 1571 # interop clients and make them think that they're running "large_unary" 1572 # test case. This avoids implementing a new test case in each language. 1573 for test_case in _HTTP2_SERVER_TEST_CASES_THAT_USE_GRPC_CLIENTS: 1574 if test_case not in language.unimplemented_test_cases(): 1575 offset = sorted(_HTTP2_SERVER_TEST_CASES).index(test_case) 1576 server_port = _DEFAULT_SERVER_PORT + offset 1577 if not args.manual_run: 1578 server_port = http2_server_job.mapped_port(server_port) 1579 if args.transport_security != 'insecure': 1580 print( 1581 ('Creating grpc client to http2 server test case ' 1582 'with insecure connection, even though ' 1583 'args.transport_security is not insecure. Http2 ' 1584 'test server only supports insecure connections.')) 1585 test_job = cloud_to_cloud_jobspec( 1586 language, 1587 test_case, 1588 str(http2InteropServer), 1589 'localhost', 1590 server_port, 1591 docker_image=docker_images.get(str(language)), 1592 transport_security='insecure', 1593 manual_cmd_log=client_manual_cmd_log) 1594 jobs.append(test_job) 1595 1596 if not jobs: 1597 print('No jobs to run.') 1598 for image in six.itervalues(docker_images): 1599 dockerjob.remove_image(image, skip_nonexistent=True) 1600 sys.exit(1) 1601 1602 if args.manual_run: 1603 print('All tests will skipped --manual_run option is active.') 1604 1605 if args.verbose: 1606 print('Jobs to run: \n%s\n' % '\n'.join(str(job) for job in jobs)) 1607 1608 num_failures, resultset = jobset.run(jobs, 1609 newline_on_success=True, 1610 maxjobs=args.jobs, 1611 skip_jobs=args.manual_run) 1612 if args.bq_result_table and resultset: 1613 upload_interop_results_to_bq(resultset, args.bq_result_table) 1614 if num_failures: 1615 jobset.message('FAILED', 'Some tests failed', do_newline=True) 1616 else: 1617 jobset.message('SUCCESS', 'All tests passed', do_newline=True) 1618 1619 write_cmdlog_maybe(server_manual_cmd_log, 'interop_server_cmds.sh') 1620 write_cmdlog_maybe(client_manual_cmd_log, 'interop_client_cmds.sh') 1621 1622 report_utils.render_junit_xml_report(resultset, _TESTS_XML_REPORT) 1623 1624 for name, job in list(resultset.items()): 1625 if "http2" in name: 1626 job[0].http2results = aggregate_http2_results(job[0].message) 1627 1628 http2_server_test_cases = (_HTTP2_SERVER_TEST_CASES 1629 if args.http2_server_interop else []) 1630 1631 if num_failures: 1632 sys.exit(1) 1633 else: 1634 sys.exit(0) 1635finally: 1636 # Check if servers are still running. 1637 for server, job in list(server_jobs.items()): 1638 if not job.is_running(): 1639 print('Server "%s" has exited prematurely.' % server) 1640 1641 dockerjob.finish_jobs([j for j in six.itervalues(server_jobs)]) 1642 1643 for image in six.itervalues(docker_images): 1644 if not args.manual_run: 1645 print('Removing docker image %s' % image) 1646 dockerjob.remove_image(image) 1647 else: 1648 print('Preserving docker image: %s' % image) 1649