1#!/usr/bin/env python 2# Copyright 2015 gRPC authors. 3# 4# Licensed under the Apache License, Version 2.0 (the "License"); 5# you may not use this file except in compliance with the License. 6# You may obtain a copy of the License at 7# 8# http://www.apache.org/licenses/LICENSE-2.0 9# 10# Unless required by applicable law or agreed to in writing, software 11# distributed under the License is distributed on an "AS IS" BASIS, 12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13# See the License for the specific language governing permissions and 14# limitations under the License. 15"""Run interop (cross-language) tests in parallel.""" 16 17from __future__ import print_function 18 19import argparse 20import atexit 21import itertools 22import json 23import multiprocessing 24import os 25import re 26import subprocess 27import sys 28import tempfile 29import time 30import uuid 31import six 32import traceback 33 34import python_utils.dockerjob as dockerjob 35import python_utils.jobset as jobset 36import python_utils.report_utils as report_utils 37# It's ok to not import because this is only necessary to upload results to BQ. 38try: 39 from python_utils.upload_test_results import upload_interop_results_to_bq 40except ImportError as e: 41 print(e) 42 43# Docker doesn't clean up after itself, so we do it on exit. 44atexit.register(lambda: subprocess.call(['stty', 'echo'])) 45 46ROOT = os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]), '../..')) 47os.chdir(ROOT) 48 49_DEFAULT_SERVER_PORT = 8080 50 51_SKIP_CLIENT_COMPRESSION = [ 52 'client_compressed_unary', 'client_compressed_streaming' 53] 54 55_SKIP_SERVER_COMPRESSION = [ 56 'server_compressed_unary', 'server_compressed_streaming' 57] 58 59_SKIP_COMPRESSION = _SKIP_CLIENT_COMPRESSION + _SKIP_SERVER_COMPRESSION 60 61_SKIP_ADVANCED = [ 62 'status_code_and_message', 'custom_metadata', 'unimplemented_method', 63 'unimplemented_service' 64] 65 66_SKIP_SPECIAL_STATUS_MESSAGE = ['special_status_message'] 67 68_GOOGLE_DEFAULT_CREDS_TEST_CASE = 'google_default_credentials' 69 70_SKIP_GOOGLE_DEFAULT_CREDS = [ 71 _GOOGLE_DEFAULT_CREDS_TEST_CASE, 72] 73 74_COMPUTE_ENGINE_CHANNEL_CREDS_TEST_CASE = 'compute_engine_channel_credentials' 75 76_SKIP_COMPUTE_ENGINE_CHANNEL_CREDS = [ 77 _COMPUTE_ENGINE_CHANNEL_CREDS_TEST_CASE, 78] 79 80_TEST_TIMEOUT = 3 * 60 81 82# disable this test on core-based languages, 83# see https://github.com/grpc/grpc/issues/9779 84_SKIP_DATA_FRAME_PADDING = ['data_frame_padding'] 85 86# report suffix "sponge_log.xml" is important for reports to get picked up by internal CI 87_DOCKER_BUILD_XML_REPORT = 'interop_docker_build/sponge_log.xml' 88_TESTS_XML_REPORT = 'interop_test/sponge_log.xml' 89 90 91class CXXLanguage: 92 93 def __init__(self): 94 self.client_cwd = None 95 self.server_cwd = None 96 self.http2_cwd = None 97 self.safename = 'cxx' 98 99 def client_cmd(self, args): 100 return ['bins/opt/interop_client'] + args 101 102 def client_cmd_http2interop(self, args): 103 return ['bins/opt/http2_client'] + args 104 105 def cloud_to_prod_env(self): 106 return {} 107 108 def server_cmd(self, args): 109 return ['bins/opt/interop_server'] + args 110 111 def global_env(self): 112 return {} 113 114 def unimplemented_test_cases(self): 115 return _SKIP_DATA_FRAME_PADDING + \ 116 _SKIP_SPECIAL_STATUS_MESSAGE + \ 117 _SKIP_COMPUTE_ENGINE_CHANNEL_CREDS 118 119 def unimplemented_test_cases_server(self): 120 return [] 121 122 def __str__(self): 123 return 'c++' 124 125 126class CSharpLanguage: 127 128 def __init__(self): 129 self.client_cwd = 'src/csharp/Grpc.IntegrationTesting.Client/bin/Debug/net45' 130 self.server_cwd = 'src/csharp/Grpc.IntegrationTesting.Server/bin/Debug/net45' 131 self.safename = str(self) 132 133 def client_cmd(self, args): 134 return ['mono', 'Grpc.IntegrationTesting.Client.exe'] + args 135 136 def cloud_to_prod_env(self): 137 return {} 138 139 def server_cmd(self, args): 140 return ['mono', 'Grpc.IntegrationTesting.Server.exe'] + args 141 142 def global_env(self): 143 return {} 144 145 def unimplemented_test_cases(self): 146 return _SKIP_SERVER_COMPRESSION + \ 147 _SKIP_DATA_FRAME_PADDING + \ 148 _SKIP_GOOGLE_DEFAULT_CREDS + \ 149 _SKIP_COMPUTE_ENGINE_CHANNEL_CREDS 150 151 def unimplemented_test_cases_server(self): 152 return _SKIP_COMPRESSION 153 154 def __str__(self): 155 return 'csharp' 156 157 158class CSharpCoreCLRLanguage: 159 160 def __init__(self): 161 self.client_cwd = 'src/csharp/Grpc.IntegrationTesting.Client/bin/Debug/netcoreapp2.1' 162 self.server_cwd = 'src/csharp/Grpc.IntegrationTesting.Server/bin/Debug/netcoreapp2.1' 163 self.safename = str(self) 164 165 def client_cmd(self, args): 166 return ['dotnet', 'exec', 'Grpc.IntegrationTesting.Client.dll'] + args 167 168 def cloud_to_prod_env(self): 169 return {} 170 171 def server_cmd(self, args): 172 return ['dotnet', 'exec', 'Grpc.IntegrationTesting.Server.dll'] + args 173 174 def global_env(self): 175 return {} 176 177 def unimplemented_test_cases(self): 178 return _SKIP_SERVER_COMPRESSION + \ 179 _SKIP_DATA_FRAME_PADDING + \ 180 _SKIP_GOOGLE_DEFAULT_CREDS + \ 181 _SKIP_COMPUTE_ENGINE_CHANNEL_CREDS 182 183 def unimplemented_test_cases_server(self): 184 return _SKIP_COMPRESSION 185 186 def __str__(self): 187 return 'csharpcoreclr' 188 189 190class AspNetCoreLanguage: 191 192 def __init__(self): 193 self.client_cwd = '../grpc-dotnet/testassets/InteropTestsClient/bin/Debug/netcoreapp3.0' 194 self.server_cwd = '../grpc-dotnet/testassets/InteropTestsWebsite/bin/Debug/netcoreapp3.0' 195 self.safename = str(self) 196 197 def cloud_to_prod_env(self): 198 return {} 199 200 def client_cmd(self, args): 201 return ['dotnet', 'exec', 'InteropTestsClient.dll'] + args 202 203 def server_cmd(self, args): 204 return ['dotnet', 'exec', 'InteropTestsWebsite.dll'] + args 205 206 def global_env(self): 207 return {} 208 209 def unimplemented_test_cases(self): 210 return _SKIP_GOOGLE_DEFAULT_CREDS + \ 211 _SKIP_COMPUTE_ENGINE_CHANNEL_CREDS 212 213 def unimplemented_test_cases_server(self): 214 return [] 215 216 def __str__(self): 217 return 'aspnetcore' 218 219 220class DartLanguage: 221 222 def __init__(self): 223 self.client_cwd = '../grpc-dart/interop' 224 self.server_cwd = '../grpc-dart/interop' 225 self.http2_cwd = '../grpc-dart/interop' 226 self.safename = str(self) 227 228 def client_cmd(self, args): 229 return ['dart', 'bin/client.dart'] + args 230 231 def cloud_to_prod_env(self): 232 return {} 233 234 def server_cmd(self, args): 235 return ['dart', 'bin/server.dart'] + args 236 237 def global_env(self): 238 return {} 239 240 def unimplemented_test_cases(self): 241 return _SKIP_COMPRESSION + \ 242 _SKIP_SPECIAL_STATUS_MESSAGE + \ 243 _SKIP_GOOGLE_DEFAULT_CREDS + \ 244 _SKIP_COMPUTE_ENGINE_CHANNEL_CREDS 245 246 def unimplemented_test_cases_server(self): 247 return _SKIP_COMPRESSION + _SKIP_SPECIAL_STATUS_MESSAGE 248 249 def __str__(self): 250 return 'dart' 251 252 253class JavaLanguage: 254 255 def __init__(self): 256 self.client_cwd = '../grpc-java' 257 self.server_cwd = '../grpc-java' 258 self.http2_cwd = '../grpc-java' 259 self.safename = str(self) 260 261 def client_cmd(self, args): 262 return ['./run-test-client.sh'] + args 263 264 def client_cmd_http2interop(self, args): 265 return [ 266 './interop-testing/build/install/grpc-interop-testing/bin/http2-client' 267 ] + args 268 269 def cloud_to_prod_env(self): 270 return {} 271 272 def server_cmd(self, args): 273 return ['./run-test-server.sh'] + args 274 275 def global_env(self): 276 return {} 277 278 def unimplemented_test_cases(self): 279 return [] 280 281 def unimplemented_test_cases_server(self): 282 # Does not support CompressedRequest feature. 283 # Only supports CompressedResponse feature for unary. 284 return _SKIP_CLIENT_COMPRESSION + ['server_compressed_streaming'] 285 286 def __str__(self): 287 return 'java' 288 289 290class JavaOkHttpClient: 291 292 def __init__(self): 293 self.client_cwd = '../grpc-java' 294 self.safename = 'java' 295 296 def client_cmd(self, args): 297 return ['./run-test-client.sh', '--use_okhttp=true'] + args 298 299 def cloud_to_prod_env(self): 300 return {} 301 302 def global_env(self): 303 return {} 304 305 def unimplemented_test_cases(self): 306 return _SKIP_DATA_FRAME_PADDING 307 308 def __str__(self): 309 return 'javaokhttp' 310 311 312class GoLanguage: 313 314 def __init__(self): 315 # TODO: this relies on running inside docker 316 self.client_cwd = '/go/src/google.golang.org/grpc/interop/client' 317 self.server_cwd = '/go/src/google.golang.org/grpc/interop/server' 318 self.http2_cwd = '/go/src/google.golang.org/grpc/interop/http2' 319 self.safename = str(self) 320 321 def client_cmd(self, args): 322 return ['go', 'run', 'client.go'] + args 323 324 def client_cmd_http2interop(self, args): 325 return ['go', 'run', 'negative_http2_client.go'] + args 326 327 def cloud_to_prod_env(self): 328 return {} 329 330 def server_cmd(self, args): 331 return ['go', 'run', 'server.go'] + args 332 333 def global_env(self): 334 return {} 335 336 def unimplemented_test_cases(self): 337 return _SKIP_COMPRESSION 338 339 def unimplemented_test_cases_server(self): 340 return _SKIP_COMPRESSION 341 342 def __str__(self): 343 return 'go' 344 345 346class Http2Server: 347 """Represents the HTTP/2 Interop Test server 348 349 This pretends to be a language in order to be built and run, but really it 350 isn't. 351 """ 352 353 def __init__(self): 354 self.server_cwd = None 355 self.safename = str(self) 356 357 def server_cmd(self, args): 358 return ['python test/http2_test/http2_test_server.py'] 359 360 def cloud_to_prod_env(self): 361 return {} 362 363 def global_env(self): 364 return {} 365 366 def unimplemented_test_cases(self): 367 return _TEST_CASES + \ 368 _SKIP_DATA_FRAME_PADDING + \ 369 _SKIP_SPECIAL_STATUS_MESSAGE + \ 370 _SKIP_GOOGLE_DEFAULT_CREDS + \ 371 _SKIP_COMPUTE_ENGINE_CHANNEL_CREDS 372 373 def unimplemented_test_cases_server(self): 374 return _TEST_CASES 375 376 def __str__(self): 377 return 'http2' 378 379 380class Http2Client: 381 """Represents the HTTP/2 Interop Test 382 383 This pretends to be a language in order to be built and run, but really it 384 isn't. 385 """ 386 387 def __init__(self): 388 self.client_cwd = None 389 self.safename = str(self) 390 391 def client_cmd(self, args): 392 return ['tools/http2_interop/http2_interop.test', '-test.v'] + args 393 394 def cloud_to_prod_env(self): 395 return {} 396 397 def global_env(self): 398 return {} 399 400 def unimplemented_test_cases(self): 401 return _TEST_CASES + \ 402 _SKIP_SPECIAL_STATUS_MESSAGE + \ 403 _SKIP_GOOGLE_DEFAULT_CREDS + \ 404 _SKIP_COMPUTE_ENGINE_CHANNEL_CREDS 405 406 def unimplemented_test_cases_server(self): 407 return _TEST_CASES 408 409 def __str__(self): 410 return 'http2' 411 412 413class NodeLanguage: 414 415 def __init__(self): 416 self.client_cwd = '../grpc-node' 417 self.server_cwd = '../grpc-node' 418 self.safename = str(self) 419 420 def client_cmd(self, args): 421 return [ 422 'packages/grpc-native-core/deps/grpc/tools/run_tests/interop/with_nvm.sh', 423 'node', '--require', './test/fixtures/native_native', 424 'test/interop/interop_client.js' 425 ] + args 426 427 def cloud_to_prod_env(self): 428 return {} 429 430 def server_cmd(self, args): 431 return [ 432 'packages/grpc-native-core/deps/grpc/tools/run_tests/interop/with_nvm.sh', 433 'node', '--require', './test/fixtures/native_native', 434 'test/interop/interop_server.js' 435 ] + args 436 437 def global_env(self): 438 return {} 439 440 def unimplemented_test_cases(self): 441 return _SKIP_COMPRESSION + \ 442 _SKIP_DATA_FRAME_PADDING + \ 443 _SKIP_GOOGLE_DEFAULT_CREDS + \ 444 _SKIP_COMPUTE_ENGINE_CHANNEL_CREDS 445 446 def unimplemented_test_cases_server(self): 447 return _SKIP_COMPRESSION 448 449 def __str__(self): 450 return 'node' 451 452 453class NodePureJSLanguage: 454 455 def __init__(self): 456 self.client_cwd = '../grpc-node' 457 self.server_cwd = '../grpc-node' 458 self.safename = str(self) 459 460 def client_cmd(self, args): 461 return [ 462 'packages/grpc-native-core/deps/grpc/tools/run_tests/interop/with_nvm.sh', 463 'node', '--require', './test/fixtures/js_js', 464 'test/interop/interop_client.js' 465 ] + args 466 467 def cloud_to_prod_env(self): 468 return {} 469 470 def global_env(self): 471 return {} 472 473 def unimplemented_test_cases(self): 474 return _SKIP_COMPRESSION + \ 475 _SKIP_DATA_FRAME_PADDING + \ 476 _SKIP_GOOGLE_DEFAULT_CREDS + \ 477 _SKIP_COMPUTE_ENGINE_CHANNEL_CREDS 478 479 def unimplemented_test_cases_server(self): 480 return [] 481 482 def __str__(self): 483 return 'nodepurejs' 484 485 486class PHPLanguage: 487 488 def __init__(self): 489 self.client_cwd = None 490 self.safename = str(self) 491 492 def client_cmd(self, args): 493 return ['src/php/bin/interop_client.sh'] + args 494 495 def cloud_to_prod_env(self): 496 return {} 497 498 def global_env(self): 499 return {} 500 501 def unimplemented_test_cases(self): 502 return _SKIP_SERVER_COMPRESSION + \ 503 _SKIP_DATA_FRAME_PADDING + \ 504 _SKIP_SPECIAL_STATUS_MESSAGE + \ 505 _SKIP_GOOGLE_DEFAULT_CREDS + \ 506 _SKIP_COMPUTE_ENGINE_CHANNEL_CREDS 507 508 def unimplemented_test_cases_server(self): 509 return [] 510 511 def __str__(self): 512 return 'php' 513 514 515class PHP7Language: 516 517 def __init__(self): 518 self.client_cwd = None 519 self.safename = str(self) 520 521 def client_cmd(self, args): 522 return ['src/php/bin/interop_client.sh'] + args 523 524 def cloud_to_prod_env(self): 525 return {} 526 527 def global_env(self): 528 return {} 529 530 def unimplemented_test_cases(self): 531 return _SKIP_SERVER_COMPRESSION + \ 532 _SKIP_DATA_FRAME_PADDING + \ 533 _SKIP_SPECIAL_STATUS_MESSAGE + \ 534 _SKIP_GOOGLE_DEFAULT_CREDS + \ 535 _SKIP_COMPUTE_ENGINE_CHANNEL_CREDS 536 537 def unimplemented_test_cases_server(self): 538 return [] 539 540 def __str__(self): 541 return 'php7' 542 543 544class ObjcLanguage: 545 546 def __init__(self): 547 self.client_cwd = 'src/objective-c/tests' 548 self.safename = str(self) 549 550 def client_cmd(self, args): 551 # from args, extract the server port and craft xcodebuild command out of it 552 for arg in args: 553 port = re.search('--server_port=(\d+)', arg) 554 if port: 555 portnum = port.group(1) 556 cmdline = 'pod install && xcodebuild -workspace Tests.xcworkspace -scheme InteropTestsLocalSSL -destination name="iPhone 6" HOST_PORT_LOCALSSL=localhost:%s test' % portnum 557 return [cmdline] 558 559 def cloud_to_prod_env(self): 560 return {} 561 562 def global_env(self): 563 return {} 564 565 def unimplemented_test_cases(self): 566 # ObjC test runs all cases with the same command. It ignores the testcase 567 # cmdline argument. Here we return all but one test cases as unimplemented, 568 # and depend upon ObjC test's behavior that it runs all cases even when 569 # we tell it to run just one. 570 return _TEST_CASES[1:] + \ 571 _SKIP_COMPRESSION + \ 572 _SKIP_DATA_FRAME_PADDING + \ 573 _SKIP_SPECIAL_STATUS_MESSAGE + \ 574 _SKIP_GOOGLE_DEFAULT_CREDS + \ 575 _SKIP_COMPUTE_ENGINE_CHANNEL_CREDS 576 577 def unimplemented_test_cases_server(self): 578 return _SKIP_COMPRESSION 579 580 def __str__(self): 581 return 'objc' 582 583 584class RubyLanguage: 585 586 def __init__(self): 587 self.client_cwd = None 588 self.server_cwd = None 589 self.safename = str(self) 590 591 def client_cmd(self, args): 592 return [ 593 'tools/run_tests/interop/with_rvm.sh', 'ruby', 594 'src/ruby/pb/test/client.rb' 595 ] + args 596 597 def cloud_to_prod_env(self): 598 return {} 599 600 def server_cmd(self, args): 601 return [ 602 'tools/run_tests/interop/with_rvm.sh', 'ruby', 603 'src/ruby/pb/test/server.rb' 604 ] + args 605 606 def global_env(self): 607 return {} 608 609 def unimplemented_test_cases(self): 610 return _SKIP_SERVER_COMPRESSION + \ 611 _SKIP_DATA_FRAME_PADDING + \ 612 _SKIP_SPECIAL_STATUS_MESSAGE + \ 613 _SKIP_GOOGLE_DEFAULT_CREDS + \ 614 _SKIP_COMPUTE_ENGINE_CHANNEL_CREDS 615 616 def unimplemented_test_cases_server(self): 617 return _SKIP_COMPRESSION 618 619 def __str__(self): 620 return 'ruby' 621 622 623class PythonLanguage: 624 625 def __init__(self): 626 self.client_cwd = None 627 self.server_cwd = None 628 self.http2_cwd = None 629 self.safename = str(self) 630 631 def client_cmd(self, args): 632 return [ 633 'py37_native/bin/python', 'src/python/grpcio_tests/setup.py', 634 'run_interop', '--client', '--args="{}"'.format(' '.join(args)) 635 ] 636 637 def client_cmd_http2interop(self, args): 638 return [ 639 'py37_native/bin/python', 640 'src/python/grpcio_tests/tests/http2/negative_http2_client.py', 641 ] + args 642 643 def cloud_to_prod_env(self): 644 return {} 645 646 def server_cmd(self, args): 647 return [ 648 'py37_native/bin/python', 'src/python/grpcio_tests/setup.py', 649 'run_interop', '--server', '--args="{}"'.format(' '.join(args)) 650 ] 651 652 def global_env(self): 653 return { 654 'LD_LIBRARY_PATH': '{}/libs/opt'.format(DOCKER_WORKDIR_ROOT), 655 'PYTHONPATH': '{}/src/python/gens'.format(DOCKER_WORKDIR_ROOT) 656 } 657 658 def unimplemented_test_cases(self): 659 return _SKIP_COMPRESSION + \ 660 _SKIP_DATA_FRAME_PADDING + \ 661 _SKIP_GOOGLE_DEFAULT_CREDS + \ 662 _SKIP_COMPUTE_ENGINE_CHANNEL_CREDS 663 664 def unimplemented_test_cases_server(self): 665 return _SKIP_COMPRESSION 666 667 def __str__(self): 668 return 'python' 669 670 671class PythonAsyncIOLanguage: 672 673 def __init__(self): 674 self.client_cwd = None 675 self.server_cwd = None 676 self.http2_cwd = None 677 self.safename = str(self) 678 679 def client_cmd(self, args): 680 return [ 681 'py37_native/bin/python', 'src/python/grpcio_tests/setup.py', 682 'run_interop', '--use-asyncio', '--client', 683 '--args="{}"'.format(' '.join(args)) 684 ] 685 686 def client_cmd_http2interop(self, args): 687 return [ 688 'py37_native/bin/python', 689 'src/python/grpcio_tests/tests/http2/negative_http2_client.py', 690 ] + args 691 692 def cloud_to_prod_env(self): 693 return {} 694 695 def server_cmd(self, args): 696 return [ 697 'py37_native/bin/python', 'src/python/grpcio_tests/setup.py', 698 'run_interop', '--use-asyncio', '--server', 699 '--args="{}"'.format(' '.join(args)) 700 ] 701 702 def global_env(self): 703 return { 704 'LD_LIBRARY_PATH': '{}/libs/opt'.format(DOCKER_WORKDIR_ROOT), 705 'PYTHONPATH': '{}/src/python/gens'.format(DOCKER_WORKDIR_ROOT) 706 } 707 708 def unimplemented_test_cases(self): 709 # TODO(https://github.com/grpc/grpc/issues/21707) 710 return _SKIP_COMPRESSION + \ 711 _SKIP_DATA_FRAME_PADDING + \ 712 _AUTH_TEST_CASES + \ 713 ['timeout_on_sleeping_server'] 714 715 def unimplemented_test_cases_server(self): 716 # TODO(https://github.com/grpc/grpc/issues/21749) 717 return _TEST_CASES + \ 718 _AUTH_TEST_CASES + \ 719 _HTTP2_TEST_CASES + \ 720 _HTTP2_SERVER_TEST_CASES 721 722 def __str__(self): 723 return 'pythonasyncio' 724 725 726_LANGUAGES = { 727 'c++': CXXLanguage(), 728 'csharp': CSharpLanguage(), 729 'csharpcoreclr': CSharpCoreCLRLanguage(), 730 'aspnetcore': AspNetCoreLanguage(), 731 'dart': DartLanguage(), 732 'go': GoLanguage(), 733 'java': JavaLanguage(), 734 'javaokhttp': JavaOkHttpClient(), 735 'node': NodeLanguage(), 736 'nodepurejs': NodePureJSLanguage(), 737 'php': PHPLanguage(), 738 'php7': PHP7Language(), 739 'objc': ObjcLanguage(), 740 'ruby': RubyLanguage(), 741 'python': PythonLanguage(), 742 'pythonasyncio': PythonAsyncIOLanguage(), 743} 744 745# languages supported as cloud_to_cloud servers 746_SERVERS = [ 747 'c++', 'node', 'csharp', 'csharpcoreclr', 'aspnetcore', 'java', 'go', 748 'ruby', 'python', 'dart', 'pythonasyncio' 749] 750 751_TEST_CASES = [ 752 'large_unary', 'empty_unary', 'ping_pong', 'empty_stream', 753 'client_streaming', 'server_streaming', 'cancel_after_begin', 754 'cancel_after_first_response', 'timeout_on_sleeping_server', 755 'custom_metadata', 'status_code_and_message', 'unimplemented_method', 756 'client_compressed_unary', 'server_compressed_unary', 757 'client_compressed_streaming', 'server_compressed_streaming', 758 'unimplemented_service', 'special_status_message' 759] 760 761_AUTH_TEST_CASES = [ 762 'compute_engine_creds', 763 'jwt_token_creds', 764 'oauth2_auth_token', 765 'per_rpc_creds', 766 _GOOGLE_DEFAULT_CREDS_TEST_CASE, 767 _COMPUTE_ENGINE_CHANNEL_CREDS_TEST_CASE, 768] 769 770_HTTP2_TEST_CASES = ['tls', 'framing'] 771 772_HTTP2_SERVER_TEST_CASES = [ 773 'rst_after_header', 'rst_after_data', 'rst_during_data', 'goaway', 'ping', 774 'max_streams', 'data_frame_padding', 'no_df_padding_sanity_test' 775] 776 777_GRPC_CLIENT_TEST_CASES_FOR_HTTP2_SERVER_TEST_CASES = { 778 'data_frame_padding': 'large_unary', 779 'no_df_padding_sanity_test': 'large_unary' 780} 781 782_HTTP2_SERVER_TEST_CASES_THAT_USE_GRPC_CLIENTS = _GRPC_CLIENT_TEST_CASES_FOR_HTTP2_SERVER_TEST_CASES.keys( 783) 784 785_LANGUAGES_WITH_HTTP2_CLIENTS_FOR_HTTP2_SERVER_TEST_CASES = [ 786 'java', 'go', 'python', 'c++' 787] 788 789_LANGUAGES_FOR_ALTS_TEST_CASES = ['java', 'go', 'c++', 'python'] 790 791_SERVERS_FOR_ALTS_TEST_CASES = ['java', 'go', 'c++', 'python'] 792 793_TRANSPORT_SECURITY_OPTIONS = ['tls', 'alts', 'insecure'] 794 795_CUSTOM_CREDENTIALS_TYPE_OPTIONS = [ 796 'tls', 'google_default_credentials', 'compute_engine_channel_creds' 797] 798 799DOCKER_WORKDIR_ROOT = '/var/local/git/grpc' 800 801 802def docker_run_cmdline(cmdline, image, docker_args=[], cwd=None, environ=None): 803 """Wraps given cmdline array to create 'docker run' cmdline from it.""" 804 docker_cmdline = ['docker', 'run', '-i', '--rm=true'] 805 806 # turn environ into -e docker args 807 if environ: 808 for k, v in environ.items(): 809 docker_cmdline += ['-e', '%s=%s' % (k, v)] 810 811 # set working directory 812 workdir = DOCKER_WORKDIR_ROOT 813 if cwd: 814 workdir = os.path.join(workdir, cwd) 815 docker_cmdline += ['-w', workdir] 816 817 docker_cmdline += docker_args + [image] + cmdline 818 return docker_cmdline 819 820 821def manual_cmdline(docker_cmdline, docker_image): 822 """Returns docker cmdline adjusted for manual invocation.""" 823 print_cmdline = [] 824 for item in docker_cmdline: 825 if item.startswith('--name='): 826 continue 827 if item == docker_image: 828 item = "$docker_image" 829 item = item.replace('"', '\\"') 830 # add quotes when necessary 831 if any(character.isspace() for character in item): 832 item = "\"%s\"" % item 833 print_cmdline.append(item) 834 return ' '.join(print_cmdline) 835 836 837def write_cmdlog_maybe(cmdlog, filename): 838 """Returns docker cmdline adjusted for manual invocation.""" 839 if cmdlog: 840 with open(filename, 'w') as logfile: 841 logfile.write('#!/bin/bash\n') 842 logfile.write('# DO NOT MODIFY\n') 843 logfile.write( 844 '# This file is generated by run_interop_tests.py/create_testcases.sh\n' 845 ) 846 logfile.writelines("%s\n" % line for line in cmdlog) 847 print('Command log written to file %s' % filename) 848 849 850def bash_cmdline(cmdline): 851 """Creates bash -c cmdline from args list.""" 852 # Use login shell: 853 # * makes error messages clearer if executables are missing 854 return ['bash', '-c', ' '.join(cmdline)] 855 856 857def compute_engine_creds_required(language, test_case): 858 """Returns True if given test requires access to compute engine creds.""" 859 language = str(language) 860 if test_case == 'compute_engine_creds': 861 return True 862 if test_case == 'oauth2_auth_token' and language == 'c++': 863 # C++ oauth2 test uses GCE creds because C++ only supports JWT 864 return True 865 return False 866 867 868def auth_options(language, test_case, google_default_creds_use_key_file, 869 service_account_key_file, default_service_account): 870 """Returns (cmdline, env) tuple with cloud_to_prod_auth test options.""" 871 872 language = str(language) 873 cmdargs = [] 874 env = {} 875 876 oauth_scope_arg = '--oauth_scope=https://www.googleapis.com/auth/xapi.zoo' 877 key_file_arg = '--service_account_key_file=%s' % service_account_key_file 878 default_account_arg = '--default_service_account=%s' % default_service_account 879 880 if test_case in ['jwt_token_creds', 'per_rpc_creds', 'oauth2_auth_token']: 881 if language in [ 882 'csharp', 'csharpcoreclr', 'aspnetcore', 'node', 'php', 'php7', 883 'python', 'ruby', 'nodepurejs' 884 ]: 885 env['GOOGLE_APPLICATION_CREDENTIALS'] = service_account_key_file 886 else: 887 cmdargs += [key_file_arg] 888 889 if test_case in ['per_rpc_creds', 'oauth2_auth_token']: 890 cmdargs += [oauth_scope_arg] 891 892 if test_case == 'oauth2_auth_token' and language == 'c++': 893 # C++ oauth2 test uses GCE creds and thus needs to know the default account 894 cmdargs += [default_account_arg] 895 896 if test_case == 'compute_engine_creds': 897 cmdargs += [oauth_scope_arg, default_account_arg] 898 899 if test_case == _GOOGLE_DEFAULT_CREDS_TEST_CASE: 900 if google_default_creds_use_key_file: 901 env['GOOGLE_APPLICATION_CREDENTIALS'] = service_account_key_file 902 cmdargs += [default_account_arg] 903 904 if test_case == _COMPUTE_ENGINE_CHANNEL_CREDS_TEST_CASE: 905 cmdargs += [default_account_arg] 906 907 return (cmdargs, env) 908 909 910def _job_kill_handler(job): 911 if job._spec.container_name: 912 dockerjob.docker_kill(job._spec.container_name) 913 # When the job times out and we decide to kill it, 914 # we need to wait a before restarting the job 915 # to prevent "container name already in use" error. 916 # TODO(jtattermusch): figure out a cleaner way to this. 917 time.sleep(2) 918 919 920def cloud_to_prod_jobspec(language, 921 test_case, 922 server_host_nickname, 923 server_host, 924 google_default_creds_use_key_file, 925 docker_image=None, 926 auth=False, 927 manual_cmd_log=None, 928 service_account_key_file=None, 929 default_service_account=None, 930 transport_security='tls'): 931 """Creates jobspec for cloud-to-prod interop test""" 932 container_name = None 933 cmdargs = [ 934 '--server_host=%s' % server_host, '--server_port=443', 935 '--test_case=%s' % test_case 936 ] 937 if transport_security == 'tls': 938 transport_security_options = ['--use_tls=true'] 939 elif transport_security == 'google_default_credentials' and str( 940 language) in ['c++', 'go', 'java', 'javaokhttp']: 941 transport_security_options = [ 942 '--custom_credentials_type=google_default_credentials' 943 ] 944 elif transport_security == 'compute_engine_channel_creds' and str( 945 language) in ['go', 'java', 'javaokhttp']: 946 transport_security_options = [ 947 '--custom_credentials_type=compute_engine_channel_creds' 948 ] 949 else: 950 print( 951 'Invalid transport security option %s in cloud_to_prod_jobspec. Lang: %s' 952 % (str(language), transport_security)) 953 sys.exit(1) 954 cmdargs = cmdargs + transport_security_options 955 environ = dict(language.cloud_to_prod_env(), **language.global_env()) 956 if auth: 957 auth_cmdargs, auth_env = auth_options( 958 language, test_case, google_default_creds_use_key_file, 959 service_account_key_file, default_service_account) 960 cmdargs += auth_cmdargs 961 environ.update(auth_env) 962 cmdline = bash_cmdline(language.client_cmd(cmdargs)) 963 cwd = language.client_cwd 964 965 if docker_image: 966 container_name = dockerjob.random_name('interop_client_%s' % 967 language.safename) 968 cmdline = docker_run_cmdline( 969 cmdline, 970 image=docker_image, 971 cwd=cwd, 972 environ=environ, 973 docker_args=['--net=host', 974 '--name=%s' % container_name]) 975 if manual_cmd_log is not None: 976 if manual_cmd_log == []: 977 manual_cmd_log.append('echo "Testing ${docker_image:=%s}"' % 978 docker_image) 979 manual_cmd_log.append(manual_cmdline(cmdline, docker_image)) 980 cwd = None 981 environ = None 982 983 suite_name = 'cloud_to_prod_auth' if auth else 'cloud_to_prod' 984 test_job = jobset.JobSpec(cmdline=cmdline, 985 cwd=cwd, 986 environ=environ, 987 shortname='%s:%s:%s:%s:%s' % 988 (suite_name, language, server_host_nickname, 989 test_case, transport_security), 990 timeout_seconds=_TEST_TIMEOUT, 991 flake_retries=4 if args.allow_flakes else 0, 992 timeout_retries=2 if args.allow_flakes else 0, 993 kill_handler=_job_kill_handler) 994 if docker_image: 995 test_job.container_name = container_name 996 return test_job 997 998 999def cloud_to_cloud_jobspec(language, 1000 test_case, 1001 server_name, 1002 server_host, 1003 server_port, 1004 docker_image=None, 1005 transport_security='tls', 1006 manual_cmd_log=None): 1007 """Creates jobspec for cloud-to-cloud interop test""" 1008 interop_only_options = [ 1009 '--server_host_override=foo.test.google.fr', 1010 '--use_test_ca=true', 1011 ] 1012 if transport_security == 'tls': 1013 interop_only_options += ['--use_tls=true'] 1014 elif transport_security == 'alts': 1015 interop_only_options += ['--use_tls=false', '--use_alts=true'] 1016 elif transport_security == 'insecure': 1017 interop_only_options += ['--use_tls=false'] 1018 else: 1019 print( 1020 'Invalid transport security option %s in cloud_to_cloud_jobspec.' % 1021 transport_security) 1022 sys.exit(1) 1023 1024 client_test_case = test_case 1025 if test_case in _HTTP2_SERVER_TEST_CASES_THAT_USE_GRPC_CLIENTS: 1026 client_test_case = _GRPC_CLIENT_TEST_CASES_FOR_HTTP2_SERVER_TEST_CASES[ 1027 test_case] 1028 if client_test_case in language.unimplemented_test_cases(): 1029 print('asking client %s to run unimplemented test case %s' % 1030 (repr(language), client_test_case)) 1031 sys.exit(1) 1032 1033 common_options = [ 1034 '--test_case=%s' % client_test_case, 1035 '--server_host=%s' % server_host, 1036 '--server_port=%s' % server_port, 1037 ] 1038 1039 if test_case in _HTTP2_SERVER_TEST_CASES: 1040 if test_case in _HTTP2_SERVER_TEST_CASES_THAT_USE_GRPC_CLIENTS: 1041 client_options = interop_only_options + common_options 1042 cmdline = bash_cmdline(language.client_cmd(client_options)) 1043 cwd = language.client_cwd 1044 else: 1045 cmdline = bash_cmdline( 1046 language.client_cmd_http2interop(common_options)) 1047 cwd = language.http2_cwd 1048 else: 1049 cmdline = bash_cmdline( 1050 language.client_cmd(common_options + interop_only_options)) 1051 cwd = language.client_cwd 1052 1053 environ = language.global_env() 1054 if docker_image and language.safename != 'objc': 1055 # we can't run client in docker for objc. 1056 container_name = dockerjob.random_name('interop_client_%s' % 1057 language.safename) 1058 cmdline = docker_run_cmdline( 1059 cmdline, 1060 image=docker_image, 1061 environ=environ, 1062 cwd=cwd, 1063 docker_args=['--net=host', 1064 '--name=%s' % container_name]) 1065 if manual_cmd_log is not None: 1066 if manual_cmd_log == []: 1067 manual_cmd_log.append('echo "Testing ${docker_image:=%s}"' % 1068 docker_image) 1069 manual_cmd_log.append(manual_cmdline(cmdline, docker_image)) 1070 cwd = None 1071 1072 test_job = jobset.JobSpec( 1073 cmdline=cmdline, 1074 cwd=cwd, 1075 environ=environ, 1076 shortname='cloud_to_cloud:%s:%s_server:%s:%s' % 1077 (language, server_name, test_case, transport_security), 1078 timeout_seconds=_TEST_TIMEOUT, 1079 flake_retries=4 if args.allow_flakes else 0, 1080 timeout_retries=2 if args.allow_flakes else 0, 1081 kill_handler=_job_kill_handler) 1082 if docker_image: 1083 test_job.container_name = container_name 1084 return test_job 1085 1086 1087def server_jobspec(language, 1088 docker_image, 1089 transport_security='tls', 1090 manual_cmd_log=None): 1091 """Create jobspec for running a server""" 1092 container_name = dockerjob.random_name('interop_server_%s' % 1093 language.safename) 1094 server_cmd = ['--port=%s' % _DEFAULT_SERVER_PORT] 1095 if transport_security == 'tls': 1096 server_cmd += ['--use_tls=true'] 1097 elif transport_security == 'alts': 1098 server_cmd += ['--use_tls=false', '--use_alts=true'] 1099 elif transport_security == 'insecure': 1100 server_cmd += ['--use_tls=false'] 1101 else: 1102 print('Invalid transport security option %s in server_jobspec.' % 1103 transport_security) 1104 sys.exit(1) 1105 cmdline = bash_cmdline(language.server_cmd(server_cmd)) 1106 environ = language.global_env() 1107 docker_args = ['--name=%s' % container_name] 1108 if language.safename == 'http2': 1109 # we are running the http2 interop server. Open next N ports beginning 1110 # with the server port. These ports are used for http2 interop test 1111 # (one test case per port). 1112 docker_args += list( 1113 itertools.chain.from_iterable( 1114 ('-p', str(_DEFAULT_SERVER_PORT + i)) 1115 for i in range(len(_HTTP2_SERVER_TEST_CASES)))) 1116 # Enable docker's healthcheck mechanism. 1117 # This runs a Python script inside the container every second. The script 1118 # pings the http2 server to verify it is ready. The 'health-retries' flag 1119 # specifies the number of consecutive failures before docker will report 1120 # the container's status as 'unhealthy'. Prior to the first 'health_retries' 1121 # failures or the first success, the status will be 'starting'. 'docker ps' 1122 # or 'docker inspect' can be used to see the health of the container on the 1123 # command line. 1124 docker_args += [ 1125 '--health-cmd=python test/http2_test/http2_server_health_check.py ' 1126 '--server_host=%s --server_port=%d' % 1127 ('localhost', _DEFAULT_SERVER_PORT), 1128 '--health-interval=1s', 1129 '--health-retries=5', 1130 '--health-timeout=10s', 1131 ] 1132 1133 else: 1134 docker_args += ['-p', str(_DEFAULT_SERVER_PORT)] 1135 1136 docker_cmdline = docker_run_cmdline(cmdline, 1137 image=docker_image, 1138 cwd=language.server_cwd, 1139 environ=environ, 1140 docker_args=docker_args) 1141 if manual_cmd_log is not None: 1142 if manual_cmd_log == []: 1143 manual_cmd_log.append('echo "Testing ${docker_image:=%s}"' % 1144 docker_image) 1145 manual_cmd_log.append(manual_cmdline(docker_cmdline, docker_image)) 1146 server_job = jobset.JobSpec(cmdline=docker_cmdline, 1147 environ=environ, 1148 shortname='interop_server_%s' % language, 1149 timeout_seconds=30 * 60) 1150 server_job.container_name = container_name 1151 return server_job 1152 1153 1154def build_interop_image_jobspec(language, tag=None): 1155 """Creates jobspec for building interop docker image for a language""" 1156 if not tag: 1157 tag = 'grpc_interop_%s:%s' % (language.safename, uuid.uuid4()) 1158 env = { 1159 'INTEROP_IMAGE': tag, 1160 'BASE_NAME': 'grpc_interop_%s' % language.safename 1161 } 1162 if not args.travis: 1163 env['TTY_FLAG'] = '-t' 1164 # This env variable is used to get around the github rate limit 1165 # error when running the PHP `composer install` command 1166 host_file = '%s/.composer/auth.json' % os.environ['HOME'] 1167 if language.safename == 'php' and os.path.exists(host_file): 1168 env['BUILD_INTEROP_DOCKER_EXTRA_ARGS'] = \ 1169 '-v %s:/root/.composer/auth.json:ro' % host_file 1170 build_job = jobset.JobSpec( 1171 cmdline=['tools/run_tests/dockerize/build_interop_image.sh'], 1172 environ=env, 1173 shortname='build_docker_%s' % (language), 1174 timeout_seconds=30 * 60) 1175 build_job.tag = tag 1176 return build_job 1177 1178 1179def aggregate_http2_results(stdout): 1180 match = re.search(r'\{"cases[^\]]*\]\}', stdout) 1181 if not match: 1182 return None 1183 1184 results = json.loads(match.group(0)) 1185 skipped = 0 1186 passed = 0 1187 failed = 0 1188 failed_cases = [] 1189 for case in results['cases']: 1190 if case.get('skipped', False): 1191 skipped += 1 1192 else: 1193 if case.get('passed', False): 1194 passed += 1 1195 else: 1196 failed += 1 1197 failed_cases.append(case.get('name', "NONAME")) 1198 return { 1199 'passed': passed, 1200 'failed': failed, 1201 'skipped': skipped, 1202 'failed_cases': ', '.join(failed_cases), 1203 'percent': 1.0 * passed / (passed + failed) 1204 } 1205 1206 1207# A dictionary of prod servers to test against. 1208# See go/grpc-interop-tests (internal-only) for details. 1209prod_servers = { 1210 'default': 'grpc-test.sandbox.googleapis.com', 1211 'gateway_v4': 'grpc-test4.sandbox.googleapis.com', 1212} 1213 1214argp = argparse.ArgumentParser(description='Run interop tests.') 1215argp.add_argument('-l', 1216 '--language', 1217 choices=['all'] + sorted(_LANGUAGES), 1218 nargs='+', 1219 default=['all'], 1220 help='Clients to run. Objc client can be only run on OSX.') 1221argp.add_argument('-j', '--jobs', default=multiprocessing.cpu_count(), type=int) 1222argp.add_argument('--cloud_to_prod', 1223 default=False, 1224 action='store_const', 1225 const=True, 1226 help='Run cloud_to_prod tests.') 1227argp.add_argument('--cloud_to_prod_auth', 1228 default=False, 1229 action='store_const', 1230 const=True, 1231 help='Run cloud_to_prod_auth tests.') 1232argp.add_argument('--google_default_creds_use_key_file', 1233 default=False, 1234 action='store_const', 1235 const=True, 1236 help=('Whether or not we should use a key file for the ' 1237 'google_default_credentials test case, e.g. by ' 1238 'setting env var GOOGLE_APPLICATION_CREDENTIALS.')) 1239argp.add_argument('--prod_servers', 1240 choices=prod_servers.keys(), 1241 default=['default'], 1242 nargs='+', 1243 help=('The servers to run cloud_to_prod and ' 1244 'cloud_to_prod_auth tests against.')) 1245argp.add_argument('-s', 1246 '--server', 1247 choices=['all'] + sorted(_SERVERS), 1248 nargs='+', 1249 help='Run cloud_to_cloud servers in a separate docker ' + 1250 'image. Servers can only be started automatically if ' + 1251 '--use_docker option is enabled.', 1252 default=[]) 1253argp.add_argument( 1254 '--override_server', 1255 action='append', 1256 type=lambda kv: kv.split('='), 1257 help= 1258 'Use servername=HOST:PORT to explicitly specify a server. E.g. csharp=localhost:50000', 1259 default=[]) 1260# TODO(jtattermusch): the default service_account_key_file only works when --use_docker is used. 1261argp.add_argument( 1262 '--service_account_key_file', 1263 type=str, 1264 help='The service account key file to use for some auth interop tests.', 1265 default='/root/service_account/grpc-testing-ebe7c1ac7381.json') 1266argp.add_argument( 1267 '--default_service_account', 1268 type=str, 1269 help='Default GCE service account email to use for some auth interop tests.', 1270 default='830293263384-compute@developer.gserviceaccount.com') 1271argp.add_argument('-t', 1272 '--travis', 1273 default=False, 1274 action='store_const', 1275 const=True) 1276argp.add_argument('-v', 1277 '--verbose', 1278 default=False, 1279 action='store_const', 1280 const=True) 1281argp.add_argument( 1282 '--use_docker', 1283 default=False, 1284 action='store_const', 1285 const=True, 1286 help='Run all the interop tests under docker. That provides ' + 1287 'additional isolation and prevents the need to install ' + 1288 'language specific prerequisites. Only available on Linux.') 1289argp.add_argument( 1290 '--allow_flakes', 1291 default=False, 1292 action='store_const', 1293 const=True, 1294 help= 1295 'Allow flaky tests to show as passing (re-runs failed tests up to five times)' 1296) 1297argp.add_argument('--manual_run', 1298 default=False, 1299 action='store_const', 1300 const=True, 1301 help='Prepare things for running interop tests manually. ' + 1302 'Preserve docker images after building them and skip ' 1303 'actually running the tests. Only print commands to run by ' + 1304 'hand.') 1305argp.add_argument( 1306 '--http2_interop', 1307 default=False, 1308 action='store_const', 1309 const=True, 1310 help='Enable HTTP/2 client edge case testing. (Bad client, good server)') 1311argp.add_argument( 1312 '--http2_server_interop', 1313 default=False, 1314 action='store_const', 1315 const=True, 1316 help= 1317 'Enable HTTP/2 server edge case testing. (Includes positive and negative tests' 1318) 1319argp.add_argument('--transport_security', 1320 choices=_TRANSPORT_SECURITY_OPTIONS, 1321 default='tls', 1322 type=str, 1323 nargs='?', 1324 const=True, 1325 help='Which transport security mechanism to use.') 1326argp.add_argument( 1327 '--custom_credentials_type', 1328 choices=_CUSTOM_CREDENTIALS_TYPE_OPTIONS, 1329 default=_CUSTOM_CREDENTIALS_TYPE_OPTIONS, 1330 nargs='+', 1331 help= 1332 'Credential types to test in the cloud_to_prod setup. Default is to test with all creds types possible.' 1333) 1334argp.add_argument( 1335 '--skip_compute_engine_creds', 1336 default=False, 1337 action='store_const', 1338 const=True, 1339 help='Skip auth tests requiring access to compute engine credentials.') 1340argp.add_argument( 1341 '--internal_ci', 1342 default=False, 1343 action='store_const', 1344 const=True, 1345 help=( 1346 '(Deprecated, has no effect) Put reports into subdirectories to improve ' 1347 'presentation of results by Internal CI.')) 1348argp.add_argument('--bq_result_table', 1349 default='', 1350 type=str, 1351 nargs='?', 1352 help='Upload test results to a specified BQ table.') 1353args = argp.parse_args() 1354 1355servers = set(s for s in itertools.chain.from_iterable( 1356 _SERVERS if x == 'all' else [x] for x in args.server)) 1357# ALTS servers are only available for certain languages. 1358if args.transport_security == 'alts': 1359 servers = servers.intersection(_SERVERS_FOR_ALTS_TEST_CASES) 1360 1361if args.use_docker: 1362 if not args.travis: 1363 print('Seen --use_docker flag, will run interop tests under docker.') 1364 print('') 1365 print( 1366 'IMPORTANT: The changes you are testing need to be locally committed' 1367 ) 1368 print( 1369 'because only the committed changes in the current branch will be') 1370 print('copied to the docker environment.') 1371 time.sleep(5) 1372 1373if args.manual_run and not args.use_docker: 1374 print('--manual_run is only supported with --use_docker option enabled.') 1375 sys.exit(1) 1376 1377if not args.use_docker and servers: 1378 print( 1379 'Running interop servers is only supported with --use_docker option enabled.' 1380 ) 1381 sys.exit(1) 1382 1383# we want to include everything but objc in 'all' 1384# because objc won't run on non-mac platforms 1385all_but_objc = set(six.iterkeys(_LANGUAGES)) - set(['objc']) 1386languages = set(_LANGUAGES[l] for l in itertools.chain.from_iterable( 1387 all_but_objc if x == 'all' else [x] for x in args.language)) 1388# ALTS interop clients are only available for certain languages. 1389if args.transport_security == 'alts': 1390 alts_languages = set(_LANGUAGES[l] for l in _LANGUAGES_FOR_ALTS_TEST_CASES) 1391 languages = languages.intersection(alts_languages) 1392 1393languages_http2_clients_for_http2_server_interop = set() 1394if args.http2_server_interop: 1395 languages_http2_clients_for_http2_server_interop = set( 1396 _LANGUAGES[l] 1397 for l in _LANGUAGES_WITH_HTTP2_CLIENTS_FOR_HTTP2_SERVER_TEST_CASES 1398 if 'all' in args.language or l in args.language) 1399 1400http2Interop = Http2Client() if args.http2_interop else None 1401http2InteropServer = Http2Server() if args.http2_server_interop else None 1402 1403docker_images = {} 1404if args.use_docker: 1405 # languages for which to build docker images 1406 languages_to_build = set(_LANGUAGES[k] 1407 for k in set([str(l) for l in languages] + 1408 [s for s in servers])) 1409 languages_to_build = languages_to_build | languages_http2_clients_for_http2_server_interop 1410 1411 if args.http2_interop: 1412 languages_to_build.add(http2Interop) 1413 1414 if args.http2_server_interop: 1415 languages_to_build.add(http2InteropServer) 1416 1417 build_jobs = [] 1418 for l in languages_to_build: 1419 if str(l) == 'objc': 1420 # we don't need to build a docker image for objc 1421 continue 1422 job = build_interop_image_jobspec(l) 1423 docker_images[str(l)] = job.tag 1424 build_jobs.append(job) 1425 1426 if build_jobs: 1427 jobset.message('START', 1428 'Building interop docker images.', 1429 do_newline=True) 1430 if args.verbose: 1431 print('Jobs to run: \n%s\n' % '\n'.join(str(j) for j in build_jobs)) 1432 1433 num_failures, build_resultset = jobset.run(build_jobs, 1434 newline_on_success=True, 1435 maxjobs=args.jobs) 1436 1437 report_utils.render_junit_xml_report(build_resultset, 1438 _DOCKER_BUILD_XML_REPORT) 1439 1440 if num_failures == 0: 1441 jobset.message('SUCCESS', 1442 'All docker images built successfully.', 1443 do_newline=True) 1444 else: 1445 jobset.message('FAILED', 1446 'Failed to build interop docker images.', 1447 do_newline=True) 1448 for image in six.itervalues(docker_images): 1449 dockerjob.remove_image(image, skip_nonexistent=True) 1450 sys.exit(1) 1451 1452server_manual_cmd_log = [] if args.manual_run else None 1453client_manual_cmd_log = [] if args.manual_run else None 1454 1455# Start interop servers. 1456server_jobs = {} 1457server_addresses = {} 1458try: 1459 for s in servers: 1460 lang = str(s) 1461 spec = server_jobspec(_LANGUAGES[lang], 1462 docker_images.get(lang), 1463 args.transport_security, 1464 manual_cmd_log=server_manual_cmd_log) 1465 if not args.manual_run: 1466 job = dockerjob.DockerJob(spec) 1467 server_jobs[lang] = job 1468 server_addresses[lang] = ('localhost', 1469 job.mapped_port(_DEFAULT_SERVER_PORT)) 1470 else: 1471 # don't run the server, set server port to a placeholder value 1472 server_addresses[lang] = ('localhost', '${SERVER_PORT}') 1473 1474 http2_server_job = None 1475 if args.http2_server_interop: 1476 # launch a HTTP2 server emulator that creates edge cases 1477 lang = str(http2InteropServer) 1478 spec = server_jobspec(http2InteropServer, 1479 docker_images.get(lang), 1480 manual_cmd_log=server_manual_cmd_log) 1481 if not args.manual_run: 1482 http2_server_job = dockerjob.DockerJob(spec) 1483 server_jobs[lang] = http2_server_job 1484 else: 1485 # don't run the server, set server port to a placeholder value 1486 server_addresses[lang] = ('localhost', '${SERVER_PORT}') 1487 1488 jobs = [] 1489 if args.cloud_to_prod: 1490 if args.transport_security not in ['tls']: 1491 print('TLS is always enabled for cloud_to_prod scenarios.') 1492 for server_host_nickname in args.prod_servers: 1493 for language in languages: 1494 for test_case in _TEST_CASES: 1495 if not test_case in language.unimplemented_test_cases(): 1496 if not test_case in _SKIP_ADVANCED + _SKIP_COMPRESSION + _SKIP_SPECIAL_STATUS_MESSAGE: 1497 for transport_security in args.custom_credentials_type: 1498 # google_default_credentials not yet supported by all languages 1499 if transport_security == 'google_default_credentials' and str( 1500 language) not in [ 1501 'c++', 'go', 'java', 'javaokhttp' 1502 ]: 1503 continue 1504 # compute_engine_channel_creds not yet supported by all languages 1505 if transport_security == 'compute_engine_channel_creds' and str( 1506 language) not in [ 1507 'go', 'java', 'javaokhttp' 1508 ]: 1509 continue 1510 test_job = cloud_to_prod_jobspec( 1511 language, 1512 test_case, 1513 server_host_nickname, 1514 prod_servers[server_host_nickname], 1515 google_default_creds_use_key_file=args. 1516 google_default_creds_use_key_file, 1517 docker_image=docker_images.get( 1518 str(language)), 1519 manual_cmd_log=client_manual_cmd_log, 1520 service_account_key_file=args. 1521 service_account_key_file, 1522 default_service_account=args. 1523 default_service_account, 1524 transport_security=transport_security) 1525 jobs.append(test_job) 1526 if args.http2_interop: 1527 for test_case in _HTTP2_TEST_CASES: 1528 test_job = cloud_to_prod_jobspec( 1529 http2Interop, 1530 test_case, 1531 server_host_nickname, 1532 prod_servers[server_host_nickname], 1533 google_default_creds_use_key_file=args. 1534 google_default_creds_use_key_file, 1535 docker_image=docker_images.get(str(http2Interop)), 1536 manual_cmd_log=client_manual_cmd_log, 1537 service_account_key_file=args.service_account_key_file, 1538 default_service_account=args.default_service_account, 1539 transport_security=args.transport_security) 1540 jobs.append(test_job) 1541 1542 if args.cloud_to_prod_auth: 1543 if args.transport_security not in ['tls']: 1544 print('TLS is always enabled for cloud_to_prod scenarios.') 1545 for server_host_nickname in args.prod_servers: 1546 for language in languages: 1547 for test_case in _AUTH_TEST_CASES: 1548 if (not args.skip_compute_engine_creds or 1549 not compute_engine_creds_required( 1550 language, test_case)): 1551 if not test_case in language.unimplemented_test_cases(): 1552 if test_case == _GOOGLE_DEFAULT_CREDS_TEST_CASE: 1553 transport_security = 'google_default_credentials' 1554 elif test_case == _COMPUTE_ENGINE_CHANNEL_CREDS_TEST_CASE: 1555 transport_security = 'compute_engine_channel_creds' 1556 else: 1557 transport_security = 'tls' 1558 if transport_security not in args.custom_credentials_type: 1559 continue 1560 test_job = cloud_to_prod_jobspec( 1561 language, 1562 test_case, 1563 server_host_nickname, 1564 prod_servers[server_host_nickname], 1565 google_default_creds_use_key_file=args. 1566 google_default_creds_use_key_file, 1567 docker_image=docker_images.get(str(language)), 1568 auth=True, 1569 manual_cmd_log=client_manual_cmd_log, 1570 service_account_key_file=args. 1571 service_account_key_file, 1572 default_service_account=args. 1573 default_service_account, 1574 transport_security=transport_security) 1575 jobs.append(test_job) 1576 for server in args.override_server: 1577 server_name = server[0] 1578 (server_host, server_port) = server[1].split(':') 1579 server_addresses[server_name] = (server_host, server_port) 1580 1581 for server_name, server_address in server_addresses.items(): 1582 (server_host, server_port) = server_address 1583 server_language = _LANGUAGES.get(server_name, None) 1584 skip_server = [] # test cases unimplemented by server 1585 if server_language: 1586 skip_server = server_language.unimplemented_test_cases_server() 1587 for language in languages: 1588 for test_case in _TEST_CASES: 1589 if not test_case in language.unimplemented_test_cases(): 1590 if not test_case in skip_server: 1591 test_job = cloud_to_cloud_jobspec( 1592 language, 1593 test_case, 1594 server_name, 1595 server_host, 1596 server_port, 1597 docker_image=docker_images.get(str(language)), 1598 transport_security=args.transport_security, 1599 manual_cmd_log=client_manual_cmd_log) 1600 jobs.append(test_job) 1601 1602 if args.http2_interop: 1603 for test_case in _HTTP2_TEST_CASES: 1604 if server_name == "go": 1605 # TODO(carl-mastrangelo): Reenable after https://github.com/grpc/grpc-go/issues/434 1606 continue 1607 test_job = cloud_to_cloud_jobspec( 1608 http2Interop, 1609 test_case, 1610 server_name, 1611 server_host, 1612 server_port, 1613 docker_image=docker_images.get(str(http2Interop)), 1614 transport_security=args.transport_security, 1615 manual_cmd_log=client_manual_cmd_log) 1616 jobs.append(test_job) 1617 1618 if args.http2_server_interop: 1619 if not args.manual_run: 1620 http2_server_job.wait_for_healthy(timeout_seconds=600) 1621 for language in languages_http2_clients_for_http2_server_interop: 1622 for test_case in set(_HTTP2_SERVER_TEST_CASES) - set( 1623 _HTTP2_SERVER_TEST_CASES_THAT_USE_GRPC_CLIENTS): 1624 offset = sorted(_HTTP2_SERVER_TEST_CASES).index(test_case) 1625 server_port = _DEFAULT_SERVER_PORT + offset 1626 if not args.manual_run: 1627 server_port = http2_server_job.mapped_port(server_port) 1628 test_job = cloud_to_cloud_jobspec( 1629 language, 1630 test_case, 1631 str(http2InteropServer), 1632 'localhost', 1633 server_port, 1634 docker_image=docker_images.get(str(language)), 1635 manual_cmd_log=client_manual_cmd_log) 1636 jobs.append(test_job) 1637 for language in languages: 1638 # HTTP2_SERVER_TEST_CASES_THAT_USE_GRPC_CLIENTS is a subset of 1639 # HTTP_SERVER_TEST_CASES, in which clients use their gRPC interop clients rather 1640 # than specialized http2 clients, reusing existing test implementations. 1641 # For example, in the "data_frame_padding" test, use language's gRPC 1642 # interop clients and make them think that they're running "large_unary" 1643 # test case. This avoids implementing a new test case in each language. 1644 for test_case in _HTTP2_SERVER_TEST_CASES_THAT_USE_GRPC_CLIENTS: 1645 if test_case not in language.unimplemented_test_cases(): 1646 offset = sorted(_HTTP2_SERVER_TEST_CASES).index(test_case) 1647 server_port = _DEFAULT_SERVER_PORT + offset 1648 if not args.manual_run: 1649 server_port = http2_server_job.mapped_port(server_port) 1650 if args.transport_security != 'insecure': 1651 print( 1652 ('Creating grpc client to http2 server test case ' 1653 'with insecure connection, even though ' 1654 'args.transport_security is not insecure. Http2 ' 1655 'test server only supports insecure connections.')) 1656 test_job = cloud_to_cloud_jobspec( 1657 language, 1658 test_case, 1659 str(http2InteropServer), 1660 'localhost', 1661 server_port, 1662 docker_image=docker_images.get(str(language)), 1663 transport_security='insecure', 1664 manual_cmd_log=client_manual_cmd_log) 1665 jobs.append(test_job) 1666 1667 if not jobs: 1668 print('No jobs to run.') 1669 for image in six.itervalues(docker_images): 1670 dockerjob.remove_image(image, skip_nonexistent=True) 1671 sys.exit(1) 1672 1673 if args.manual_run: 1674 print('All tests will skipped --manual_run option is active.') 1675 1676 if args.verbose: 1677 print('Jobs to run: \n%s\n' % '\n'.join(str(job) for job in jobs)) 1678 1679 num_failures, resultset = jobset.run(jobs, 1680 newline_on_success=True, 1681 maxjobs=args.jobs, 1682 skip_jobs=args.manual_run) 1683 if args.bq_result_table and resultset: 1684 upload_interop_results_to_bq(resultset, args.bq_result_table) 1685 if num_failures: 1686 jobset.message('FAILED', 'Some tests failed', do_newline=True) 1687 else: 1688 jobset.message('SUCCESS', 'All tests passed', do_newline=True) 1689 1690 write_cmdlog_maybe(server_manual_cmd_log, 'interop_server_cmds.sh') 1691 write_cmdlog_maybe(client_manual_cmd_log, 'interop_client_cmds.sh') 1692 1693 report_utils.render_junit_xml_report(resultset, _TESTS_XML_REPORT) 1694 1695 for name, job in resultset.items(): 1696 if "http2" in name: 1697 job[0].http2results = aggregate_http2_results(job[0].message) 1698 1699 http2_server_test_cases = (_HTTP2_SERVER_TEST_CASES 1700 if args.http2_server_interop else []) 1701 1702 if num_failures: 1703 sys.exit(1) 1704 else: 1705 sys.exit(0) 1706finally: 1707 # Check if servers are still running. 1708 for server, job in server_jobs.items(): 1709 if not job.is_running(): 1710 print('Server "%s" has exited prematurely.' % server) 1711 1712 dockerjob.finish_jobs([j for j in six.itervalues(server_jobs)]) 1713 1714 for image in six.itervalues(docker_images): 1715 if not args.manual_run: 1716 print('Removing docker image %s' % image) 1717 dockerjob.remove_image(image) 1718 else: 1719 print('Preserving docker image: %s' % image) 1720