1#!/usr/bin/env python 2# Copyright 2015 gRPC authors. 3# 4# Licensed under the Apache License, Version 2.0 (the "License"); 5# you may not use this file except in compliance with the License. 6# You may obtain a copy of the License at 7# 8# http://www.apache.org/licenses/LICENSE-2.0 9# 10# Unless required by applicable law or agreed to in writing, software 11# distributed under the License is distributed on an "AS IS" BASIS, 12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13# See the License for the specific language governing permissions and 14# limitations under the License. 15"""Run interop (cross-language) tests in parallel.""" 16 17from __future__ import print_function 18 19import argparse 20import atexit 21import itertools 22import json 23import multiprocessing 24import os 25import re 26import subprocess 27import sys 28import tempfile 29import time 30import uuid 31import six 32import traceback 33 34import python_utils.dockerjob as dockerjob 35import python_utils.jobset as jobset 36import python_utils.report_utils as report_utils 37# It's ok to not import because this is only necessary to upload results to BQ. 38try: 39 from python_utils.upload_test_results import upload_interop_results_to_bq 40except ImportError as e: 41 print(e) 42 43# Docker doesn't clean up after itself, so we do it on exit. 44atexit.register(lambda: subprocess.call(['stty', 'echo'])) 45 46ROOT = os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]), '../..')) 47os.chdir(ROOT) 48 49_DEFAULT_SERVER_PORT = 8080 50 51_SKIP_CLIENT_COMPRESSION = [ 52 'client_compressed_unary', 'client_compressed_streaming' 53] 54 55_SKIP_SERVER_COMPRESSION = [ 56 'server_compressed_unary', 'server_compressed_streaming' 57] 58 59_SKIP_COMPRESSION = _SKIP_CLIENT_COMPRESSION + _SKIP_SERVER_COMPRESSION 60 61_SKIP_ADVANCED = [ 62 'status_code_and_message', 'custom_metadata', 'unimplemented_method', 63 'unimplemented_service' 64] 65 66_TEST_TIMEOUT = 3 * 60 67 68# disable this test on core-based languages, 69# see https://github.com/grpc/grpc/issues/9779 70_SKIP_DATA_FRAME_PADDING = ['data_frame_padding'] 71 72# report suffix is important for reports to get picked up by internal CI 73_INTERNAL_CL_XML_REPORT = 'sponge_log.xml' 74 75# report suffix is important for reports to get picked up by internal CI 76_XML_REPORT = 'report.xml' 77 78 79class CXXLanguage: 80 81 def __init__(self): 82 self.client_cwd = None 83 self.server_cwd = None 84 self.http2_cwd = None 85 self.safename = 'cxx' 86 87 def client_cmd(self, args): 88 return ['bins/opt/interop_client'] + args 89 90 def client_cmd_http2interop(self, args): 91 return ['bins/opt/http2_client'] + args 92 93 def cloud_to_prod_env(self): 94 return {} 95 96 def server_cmd(self, args): 97 return ['bins/opt/interop_server'] + args 98 99 def global_env(self): 100 return {} 101 102 def unimplemented_test_cases(self): 103 return _SKIP_DATA_FRAME_PADDING 104 105 def unimplemented_test_cases_server(self): 106 return [] 107 108 def __str__(self): 109 return 'c++' 110 111 112class CSharpLanguage: 113 114 def __init__(self): 115 self.client_cwd = 'src/csharp/Grpc.IntegrationTesting.Client/bin/Debug/net45' 116 self.server_cwd = 'src/csharp/Grpc.IntegrationTesting.Server/bin/Debug/net45' 117 self.safename = str(self) 118 119 def client_cmd(self, args): 120 return ['mono', 'Grpc.IntegrationTesting.Client.exe'] + args 121 122 def cloud_to_prod_env(self): 123 return {} 124 125 def server_cmd(self, args): 126 return ['mono', 'Grpc.IntegrationTesting.Server.exe'] + args 127 128 def global_env(self): 129 return {} 130 131 def unimplemented_test_cases(self): 132 return _SKIP_SERVER_COMPRESSION + _SKIP_DATA_FRAME_PADDING 133 134 def unimplemented_test_cases_server(self): 135 return _SKIP_COMPRESSION 136 137 def __str__(self): 138 return 'csharp' 139 140 141class CSharpCoreCLRLanguage: 142 143 def __init__(self): 144 self.client_cwd = 'src/csharp/Grpc.IntegrationTesting.Client/bin/Debug/netcoreapp1.0' 145 self.server_cwd = 'src/csharp/Grpc.IntegrationTesting.Server/bin/Debug/netcoreapp1.0' 146 self.safename = str(self) 147 148 def client_cmd(self, args): 149 return ['dotnet', 'exec', 'Grpc.IntegrationTesting.Client.dll'] + args 150 151 def cloud_to_prod_env(self): 152 return {} 153 154 def server_cmd(self, args): 155 return ['dotnet', 'exec', 'Grpc.IntegrationTesting.Server.dll'] + args 156 157 def global_env(self): 158 return {} 159 160 def unimplemented_test_cases(self): 161 return _SKIP_SERVER_COMPRESSION + _SKIP_DATA_FRAME_PADDING 162 163 def unimplemented_test_cases_server(self): 164 return _SKIP_COMPRESSION 165 166 def __str__(self): 167 return 'csharpcoreclr' 168 169 170class DartLanguage: 171 172 def __init__(self): 173 self.client_cwd = '../grpc-dart/interop' 174 self.server_cwd = '../grpc-dart/interop' 175 self.http2_cwd = '../grpc-dart/interop' 176 self.safename = str(self) 177 178 def client_cmd(self, args): 179 return ['dart', 'bin/client.dart'] + args 180 181 def cloud_to_prod_env(self): 182 return {} 183 184 def server_cmd(self, args): 185 return ['dart', 'bin/server.dart'] + args 186 187 def global_env(self): 188 return {} 189 190 def unimplemented_test_cases(self): 191 return _SKIP_COMPRESSION 192 193 def unimplemented_test_cases_server(self): 194 return _SKIP_COMPRESSION 195 196 def __str__(self): 197 return 'dart' 198 199 200class JavaLanguage: 201 202 def __init__(self): 203 self.client_cwd = '../grpc-java' 204 self.server_cwd = '../grpc-java' 205 self.http2_cwd = '../grpc-java' 206 self.safename = str(self) 207 208 def client_cmd(self, args): 209 return ['./run-test-client.sh'] + args 210 211 def client_cmd_http2interop(self, args): 212 return [ 213 './interop-testing/build/install/grpc-interop-testing/bin/http2-client' 214 ] + args 215 216 def cloud_to_prod_env(self): 217 return {} 218 219 def server_cmd(self, args): 220 return ['./run-test-server.sh'] + args 221 222 def global_env(self): 223 return {} 224 225 def unimplemented_test_cases(self): 226 return [] 227 228 def unimplemented_test_cases_server(self): 229 return _SKIP_COMPRESSION 230 231 def __str__(self): 232 return 'java' 233 234 235class JavaOkHttpClient: 236 237 def __init__(self): 238 self.client_cwd = '../grpc-java' 239 self.safename = 'java' 240 241 def client_cmd(self, args): 242 return ['./run-test-client.sh', '--use_okhttp=true'] + args 243 244 def cloud_to_prod_env(self): 245 return {} 246 247 def global_env(self): 248 return {} 249 250 def unimplemented_test_cases(self): 251 return _SKIP_DATA_FRAME_PADDING 252 253 def __str__(self): 254 return 'javaokhttp' 255 256 257class GoLanguage: 258 259 def __init__(self): 260 # TODO: this relies on running inside docker 261 self.client_cwd = '/go/src/google.golang.org/grpc/interop/client' 262 self.server_cwd = '/go/src/google.golang.org/grpc/interop/server' 263 self.http2_cwd = '/go/src/google.golang.org/grpc/interop/http2' 264 self.safename = str(self) 265 266 def client_cmd(self, args): 267 return ['go', 'run', 'client.go'] + args 268 269 def client_cmd_http2interop(self, args): 270 return ['go', 'run', 'negative_http2_client.go'] + args 271 272 def cloud_to_prod_env(self): 273 return {} 274 275 def server_cmd(self, args): 276 return ['go', 'run', 'server.go'] + args 277 278 def global_env(self): 279 return {} 280 281 def unimplemented_test_cases(self): 282 return _SKIP_COMPRESSION 283 284 def unimplemented_test_cases_server(self): 285 return _SKIP_COMPRESSION 286 287 def __str__(self): 288 return 'go' 289 290 291class Http2Server: 292 """Represents the HTTP/2 Interop Test server 293 294 This pretends to be a language in order to be built and run, but really it 295 isn't. 296 """ 297 298 def __init__(self): 299 self.server_cwd = None 300 self.safename = str(self) 301 302 def server_cmd(self, args): 303 return ['python test/http2_test/http2_test_server.py'] 304 305 def cloud_to_prod_env(self): 306 return {} 307 308 def global_env(self): 309 return {} 310 311 def unimplemented_test_cases(self): 312 return _TEST_CASES + _SKIP_DATA_FRAME_PADDING 313 314 def unimplemented_test_cases_server(self): 315 return _TEST_CASES 316 317 def __str__(self): 318 return 'http2' 319 320 321class Http2Client: 322 """Represents the HTTP/2 Interop Test 323 324 This pretends to be a language in order to be built and run, but really it 325 isn't. 326 """ 327 328 def __init__(self): 329 self.client_cwd = None 330 self.safename = str(self) 331 332 def client_cmd(self, args): 333 return ['tools/http2_interop/http2_interop.test', '-test.v'] + args 334 335 def cloud_to_prod_env(self): 336 return {} 337 338 def global_env(self): 339 return {} 340 341 def unimplemented_test_cases(self): 342 return _TEST_CASES 343 344 def unimplemented_test_cases_server(self): 345 return _TEST_CASES 346 347 def __str__(self): 348 return 'http2' 349 350 351class NodeLanguage: 352 353 def __init__(self): 354 self.client_cwd = '../grpc-node' 355 self.server_cwd = '../grpc-node' 356 self.safename = str(self) 357 358 def client_cmd(self, args): 359 return [ 360 'packages/grpc-native-core/deps/grpc/tools/run_tests/interop/with_nvm.sh', 361 'node', '--require', './test/fixtures/native_native', 362 'test/interop/interop_client.js' 363 ] + args 364 365 def cloud_to_prod_env(self): 366 return {} 367 368 def server_cmd(self, args): 369 return [ 370 'packages/grpc-native-core/deps/grpc/tools/run_tests/interop/with_nvm.sh', 371 'node', '--require', './test/fixtures/native_native', 372 'test/interop/interop_server.js' 373 ] + args 374 375 def global_env(self): 376 return {} 377 378 def unimplemented_test_cases(self): 379 return _SKIP_COMPRESSION + _SKIP_DATA_FRAME_PADDING 380 381 def unimplemented_test_cases_server(self): 382 return _SKIP_COMPRESSION 383 384 def __str__(self): 385 return 'node' 386 387 388class NodePureJSLanguage: 389 390 def __init__(self): 391 self.client_cwd = '../grpc-node' 392 self.server_cwd = '../grpc-node' 393 self.safename = str(self) 394 395 def client_cmd(self, args): 396 return [ 397 'packages/grpc-native-core/deps/grpc/tools/run_tests/interop/with_nvm.sh', 398 'node', '--require', './test/fixtures/js_js', 399 'test/interop/interop_client.js' 400 ] + args 401 402 def cloud_to_prod_env(self): 403 return {} 404 405 def global_env(self): 406 return {} 407 408 def unimplemented_test_cases(self): 409 return _SKIP_COMPRESSION + _SKIP_DATA_FRAME_PADDING 410 411 def unimplemented_test_cases_server(self): 412 return [] 413 414 def __str__(self): 415 return 'nodepurejs' 416 417 418class PHPLanguage: 419 420 def __init__(self): 421 self.client_cwd = None 422 self.safename = str(self) 423 424 def client_cmd(self, args): 425 return ['src/php/bin/interop_client.sh'] + args 426 427 def cloud_to_prod_env(self): 428 return {} 429 430 def global_env(self): 431 return {} 432 433 def unimplemented_test_cases(self): 434 return _SKIP_COMPRESSION + _SKIP_DATA_FRAME_PADDING 435 436 def unimplemented_test_cases_server(self): 437 return [] 438 439 def __str__(self): 440 return 'php' 441 442 443class PHP7Language: 444 445 def __init__(self): 446 self.client_cwd = None 447 self.safename = str(self) 448 449 def client_cmd(self, args): 450 return ['src/php/bin/interop_client.sh'] + args 451 452 def cloud_to_prod_env(self): 453 return {} 454 455 def global_env(self): 456 return {} 457 458 def unimplemented_test_cases(self): 459 return _SKIP_COMPRESSION + _SKIP_DATA_FRAME_PADDING 460 461 def unimplemented_test_cases_server(self): 462 return [] 463 464 def __str__(self): 465 return 'php7' 466 467 468class ObjcLanguage: 469 470 def __init__(self): 471 self.client_cwd = 'src/objective-c/tests' 472 self.safename = str(self) 473 474 def client_cmd(self, args): 475 # from args, extract the server port and craft xcodebuild command out of it 476 for arg in args: 477 port = re.search('--server_port=(\d+)', arg) 478 if port: 479 portnum = port.group(1) 480 cmdline = 'pod install && xcodebuild -workspace Tests.xcworkspace -scheme InteropTestsLocalSSL -destination name="iPhone 6" HOST_PORT_LOCALSSL=localhost:%s test' % portnum 481 return [cmdline] 482 483 def cloud_to_prod_env(self): 484 return {} 485 486 def global_env(self): 487 return {} 488 489 def unimplemented_test_cases(self): 490 # ObjC test runs all cases with the same command. It ignores the testcase 491 # cmdline argument. Here we return all but one test cases as unimplemented, 492 # and depend upon ObjC test's behavior that it runs all cases even when 493 # we tell it to run just one. 494 return _TEST_CASES[1:] + _SKIP_COMPRESSION + _SKIP_DATA_FRAME_PADDING 495 496 def unimplemented_test_cases_server(self): 497 return _SKIP_COMPRESSION 498 499 def __str__(self): 500 return 'objc' 501 502 503class RubyLanguage: 504 505 def __init__(self): 506 self.client_cwd = None 507 self.server_cwd = None 508 self.safename = str(self) 509 510 def client_cmd(self, args): 511 return [ 512 'tools/run_tests/interop/with_rvm.sh', 'ruby', 513 'src/ruby/pb/test/client.rb' 514 ] + args 515 516 def cloud_to_prod_env(self): 517 return {} 518 519 def server_cmd(self, args): 520 return [ 521 'tools/run_tests/interop/with_rvm.sh', 'ruby', 522 'src/ruby/pb/test/server.rb' 523 ] + args 524 525 def global_env(self): 526 return {} 527 528 def unimplemented_test_cases(self): 529 return _SKIP_SERVER_COMPRESSION + _SKIP_DATA_FRAME_PADDING 530 531 def unimplemented_test_cases_server(self): 532 return _SKIP_COMPRESSION 533 534 def __str__(self): 535 return 'ruby' 536 537 538class PythonLanguage: 539 540 def __init__(self): 541 self.client_cwd = None 542 self.server_cwd = None 543 self.http2_cwd = None 544 self.safename = str(self) 545 546 def client_cmd(self, args): 547 return [ 548 'py27_native/bin/python', 'src/python/grpcio_tests/setup.py', 549 'run_interop', '--client', '--args="{}"'.format(' '.join(args)) 550 ] 551 552 def client_cmd_http2interop(self, args): 553 return [ 554 'py27_native/bin/python', 555 'src/python/grpcio_tests/tests/http2/negative_http2_client.py', 556 ] + args 557 558 def cloud_to_prod_env(self): 559 return {} 560 561 def server_cmd(self, args): 562 return [ 563 'py27_native/bin/python', 'src/python/grpcio_tests/setup.py', 564 'run_interop', '--server', '--args="{}"'.format(' '.join(args)) 565 ] 566 567 def global_env(self): 568 return { 569 'LD_LIBRARY_PATH': '{}/libs/opt'.format(DOCKER_WORKDIR_ROOT), 570 'PYTHONPATH': '{}/src/python/gens'.format(DOCKER_WORKDIR_ROOT) 571 } 572 573 def unimplemented_test_cases(self): 574 return _SKIP_COMPRESSION + _SKIP_DATA_FRAME_PADDING 575 576 def unimplemented_test_cases_server(self): 577 return _SKIP_COMPRESSION 578 579 def __str__(self): 580 return 'python' 581 582 583_LANGUAGES = { 584 'c++': CXXLanguage(), 585 'csharp': CSharpLanguage(), 586 'csharpcoreclr': CSharpCoreCLRLanguage(), 587 'dart': DartLanguage(), 588 'go': GoLanguage(), 589 'java': JavaLanguage(), 590 'javaokhttp': JavaOkHttpClient(), 591 'node': NodeLanguage(), 592 'nodepurejs': NodePureJSLanguage(), 593 'php': PHPLanguage(), 594 'php7': PHP7Language(), 595 'objc': ObjcLanguage(), 596 'ruby': RubyLanguage(), 597 'python': PythonLanguage(), 598} 599 600# languages supported as cloud_to_cloud servers 601_SERVERS = [ 602 'c++', 'node', 'csharp', 'csharpcoreclr', 'java', 'go', 'ruby', 'python', 603 'dart' 604] 605 606_TEST_CASES = [ 607 'large_unary', 'empty_unary', 'ping_pong', 'empty_stream', 608 'client_streaming', 'server_streaming', 'cancel_after_begin', 609 'cancel_after_first_response', 'timeout_on_sleeping_server', 610 'custom_metadata', 'status_code_and_message', 'unimplemented_method', 611 'client_compressed_unary', 'server_compressed_unary', 612 'client_compressed_streaming', 'server_compressed_streaming', 613 'unimplemented_service' 614] 615 616_AUTH_TEST_CASES = [ 617 'compute_engine_creds', 'jwt_token_creds', 'oauth2_auth_token', 618 'per_rpc_creds' 619] 620 621_HTTP2_TEST_CASES = ['tls', 'framing'] 622 623_HTTP2_SERVER_TEST_CASES = [ 624 'rst_after_header', 'rst_after_data', 'rst_during_data', 'goaway', 'ping', 625 'max_streams', 'data_frame_padding', 'no_df_padding_sanity_test' 626] 627 628_GRPC_CLIENT_TEST_CASES_FOR_HTTP2_SERVER_TEST_CASES = { 629 'data_frame_padding': 'large_unary', 630 'no_df_padding_sanity_test': 'large_unary' 631} 632 633_HTTP2_SERVER_TEST_CASES_THAT_USE_GRPC_CLIENTS = _GRPC_CLIENT_TEST_CASES_FOR_HTTP2_SERVER_TEST_CASES.keys( 634) 635 636_LANGUAGES_WITH_HTTP2_CLIENTS_FOR_HTTP2_SERVER_TEST_CASES = [ 637 'java', 'go', 'python', 'c++' 638] 639 640_LANGUAGES_FOR_ALTS_TEST_CASES = ['java', 'go', 'c++'] 641 642_SERVERS_FOR_ALTS_TEST_CASES = ['java', 'go', 'c++'] 643 644_TRANSPORT_SECURITY_OPTIONS = [ 645 'tls', 'alts', 'google_default_credentials', 'insecure' 646] 647 648DOCKER_WORKDIR_ROOT = '/var/local/git/grpc' 649 650 651def docker_run_cmdline(cmdline, image, docker_args=[], cwd=None, environ=None): 652 """Wraps given cmdline array to create 'docker run' cmdline from it.""" 653 docker_cmdline = ['docker', 'run', '-i', '--rm=true'] 654 655 # turn environ into -e docker args 656 if environ: 657 for k, v in environ.items(): 658 docker_cmdline += ['-e', '%s=%s' % (k, v)] 659 660 # set working directory 661 workdir = DOCKER_WORKDIR_ROOT 662 if cwd: 663 workdir = os.path.join(workdir, cwd) 664 docker_cmdline += ['-w', workdir] 665 666 docker_cmdline += docker_args + [image] + cmdline 667 return docker_cmdline 668 669 670def manual_cmdline(docker_cmdline, docker_image): 671 """Returns docker cmdline adjusted for manual invocation.""" 672 print_cmdline = [] 673 for item in docker_cmdline: 674 if item.startswith('--name='): 675 continue 676 if item == docker_image: 677 item = "$docker_image" 678 item = item.replace('"', '\\"') 679 # add quotes when necessary 680 if any(character.isspace() for character in item): 681 item = "\"%s\"" % item 682 print_cmdline.append(item) 683 return ' '.join(print_cmdline) 684 685 686def write_cmdlog_maybe(cmdlog, filename): 687 """Returns docker cmdline adjusted for manual invocation.""" 688 if cmdlog: 689 with open(filename, 'w') as logfile: 690 logfile.write('#!/bin/bash\n') 691 logfile.writelines("%s\n" % line for line in cmdlog) 692 print('Command log written to file %s' % filename) 693 694 695def bash_cmdline(cmdline): 696 """Creates bash -c cmdline from args list.""" 697 # Use login shell: 698 # * makes error messages clearer if executables are missing 699 return ['bash', '-c', ' '.join(cmdline)] 700 701 702def compute_engine_creds_required(language, test_case): 703 """Returns True if given test requires access to compute engine creds.""" 704 language = str(language) 705 if test_case == 'compute_engine_creds': 706 return True 707 if test_case == 'oauth2_auth_token' and language == 'c++': 708 # C++ oauth2 test uses GCE creds because C++ only supports JWT 709 return True 710 return False 711 712 713def auth_options(language, test_case, service_account_key_file=None): 714 """Returns (cmdline, env) tuple with cloud_to_prod_auth test options.""" 715 716 language = str(language) 717 cmdargs = [] 718 env = {} 719 720 if not service_account_key_file: 721 # this file path only works inside docker 722 service_account_key_file = '/root/service_account/GrpcTesting-726eb1347f15.json' 723 oauth_scope_arg = '--oauth_scope=https://www.googleapis.com/auth/xapi.zoo' 724 key_file_arg = '--service_account_key_file=%s' % service_account_key_file 725 default_account_arg = '--default_service_account=830293263384-compute@developer.gserviceaccount.com' 726 727 # TODO: When using google_default_credentials outside of cloud-to-prod, the environment variable 728 # 'GOOGLE_APPLICATION_CREDENTIALS' needs to be set for the test case 729 # 'jwt_token_creds' to work. 730 if test_case in ['jwt_token_creds', 'per_rpc_creds', 'oauth2_auth_token']: 731 if language in [ 732 'csharp', 'csharpcoreclr', 'node', 'php', 'php7', 'python', 733 'ruby', 'nodepurejs' 734 ]: 735 env['GOOGLE_APPLICATION_CREDENTIALS'] = service_account_key_file 736 else: 737 cmdargs += [key_file_arg] 738 739 if test_case in ['per_rpc_creds', 'oauth2_auth_token']: 740 cmdargs += [oauth_scope_arg] 741 742 if test_case == 'oauth2_auth_token' and language == 'c++': 743 # C++ oauth2 test uses GCE creds and thus needs to know the default account 744 cmdargs += [default_account_arg] 745 746 if test_case == 'compute_engine_creds': 747 cmdargs += [oauth_scope_arg, default_account_arg] 748 749 return (cmdargs, env) 750 751 752def _job_kill_handler(job): 753 if job._spec.container_name: 754 dockerjob.docker_kill(job._spec.container_name) 755 # When the job times out and we decide to kill it, 756 # we need to wait a before restarting the job 757 # to prevent "container name already in use" error. 758 # TODO(jtattermusch): figure out a cleaner way to to this. 759 time.sleep(2) 760 761 762def cloud_to_prod_jobspec(language, 763 test_case, 764 server_host_nickname, 765 server_host, 766 docker_image=None, 767 auth=False, 768 manual_cmd_log=None, 769 service_account_key_file=None, 770 transport_security='tls'): 771 """Creates jobspec for cloud-to-prod interop test""" 772 container_name = None 773 cmdargs = [ 774 '--server_host=%s' % server_host, 775 '--server_host_override=%s' % server_host, '--server_port=443', 776 '--test_case=%s' % test_case 777 ] 778 if transport_security == 'tls': 779 transport_security_options = ['--use_tls=true'] 780 elif transport_security == 'google_default_credentials' and language == 'c++': 781 transport_security_options = [ 782 '--custom_credentials_type=google_default_credentials' 783 ] 784 else: 785 print('Invalid transport security option.') 786 sys.exit(1) 787 cmdargs = cmdargs + transport_security_options 788 environ = dict(language.cloud_to_prod_env(), **language.global_env()) 789 if auth: 790 auth_cmdargs, auth_env = auth_options(language, test_case, 791 service_account_key_file) 792 cmdargs += auth_cmdargs 793 environ.update(auth_env) 794 cmdline = bash_cmdline(language.client_cmd(cmdargs)) 795 cwd = language.client_cwd 796 797 if docker_image: 798 container_name = dockerjob.random_name( 799 'interop_client_%s' % language.safename) 800 cmdline = docker_run_cmdline( 801 cmdline, 802 image=docker_image, 803 cwd=cwd, 804 environ=environ, 805 docker_args=['--net=host', 806 '--name=%s' % container_name]) 807 if manual_cmd_log is not None: 808 if manual_cmd_log == []: 809 manual_cmd_log.append( 810 'echo "Testing ${docker_image:=%s}"' % docker_image) 811 manual_cmd_log.append(manual_cmdline(cmdline, docker_image)) 812 cwd = None 813 environ = None 814 815 suite_name = 'cloud_to_prod_auth' if auth else 'cloud_to_prod' 816 test_job = jobset.JobSpec( 817 cmdline=cmdline, 818 cwd=cwd, 819 environ=environ, 820 shortname='%s:%s:%s:%s' % (suite_name, language, server_host_nickname, 821 test_case), 822 timeout_seconds=_TEST_TIMEOUT, 823 flake_retries=4 if args.allow_flakes else 0, 824 timeout_retries=2 if args.allow_flakes else 0, 825 kill_handler=_job_kill_handler) 826 if docker_image: 827 test_job.container_name = container_name 828 return test_job 829 830 831def cloud_to_cloud_jobspec(language, 832 test_case, 833 server_name, 834 server_host, 835 server_port, 836 docker_image=None, 837 transport_security='tls', 838 manual_cmd_log=None): 839 """Creates jobspec for cloud-to-cloud interop test""" 840 interop_only_options = [ 841 '--server_host_override=foo.test.google.fr', 842 '--use_test_ca=true', 843 ] 844 if transport_security == 'tls': 845 interop_only_options += ['--use_tls=true'] 846 elif transport_security == 'alts': 847 interop_only_options += ['--use_tls=false', '--use_alts=true'] 848 elif transport_security == 'insecure': 849 interop_only_options += ['--use_tls=false'] 850 else: 851 print('Invalid transport security option.') 852 sys.exit(1) 853 854 client_test_case = test_case 855 if test_case in _HTTP2_SERVER_TEST_CASES_THAT_USE_GRPC_CLIENTS: 856 client_test_case = _GRPC_CLIENT_TEST_CASES_FOR_HTTP2_SERVER_TEST_CASES[ 857 test_case] 858 if client_test_case in language.unimplemented_test_cases(): 859 print('asking client %s to run unimplemented test case %s' % 860 (repr(language), client_test_case)) 861 sys.exit(1) 862 863 common_options = [ 864 '--test_case=%s' % client_test_case, 865 '--server_host=%s' % server_host, 866 '--server_port=%s' % server_port, 867 ] 868 869 if test_case in _HTTP2_SERVER_TEST_CASES: 870 if test_case in _HTTP2_SERVER_TEST_CASES_THAT_USE_GRPC_CLIENTS: 871 client_options = interop_only_options + common_options 872 cmdline = bash_cmdline(language.client_cmd(client_options)) 873 cwd = language.client_cwd 874 else: 875 cmdline = bash_cmdline( 876 language.client_cmd_http2interop(common_options)) 877 cwd = language.http2_cwd 878 else: 879 cmdline = bash_cmdline( 880 language.client_cmd(common_options + interop_only_options)) 881 cwd = language.client_cwd 882 883 environ = language.global_env() 884 if docker_image and language.safename != 'objc': 885 # we can't run client in docker for objc. 886 container_name = dockerjob.random_name( 887 'interop_client_%s' % language.safename) 888 cmdline = docker_run_cmdline( 889 cmdline, 890 image=docker_image, 891 environ=environ, 892 cwd=cwd, 893 docker_args=['--net=host', 894 '--name=%s' % container_name]) 895 if manual_cmd_log is not None: 896 if manual_cmd_log == []: 897 manual_cmd_log.append( 898 'echo "Testing ${docker_image:=%s}"' % docker_image) 899 manual_cmd_log.append(manual_cmdline(cmdline, docker_image)) 900 cwd = None 901 902 test_job = jobset.JobSpec( 903 cmdline=cmdline, 904 cwd=cwd, 905 environ=environ, 906 shortname='cloud_to_cloud:%s:%s_server:%s' % (language, server_name, 907 test_case), 908 timeout_seconds=_TEST_TIMEOUT, 909 flake_retries=4 if args.allow_flakes else 0, 910 timeout_retries=2 if args.allow_flakes else 0, 911 kill_handler=_job_kill_handler) 912 if docker_image: 913 test_job.container_name = container_name 914 return test_job 915 916 917def server_jobspec(language, 918 docker_image, 919 transport_security='tls', 920 manual_cmd_log=None): 921 """Create jobspec for running a server""" 922 container_name = dockerjob.random_name( 923 'interop_server_%s' % language.safename) 924 server_cmd = ['--port=%s' % _DEFAULT_SERVER_PORT] 925 if transport_security == 'tls': 926 server_cmd += ['--use_tls=true'] 927 elif transport_security == 'alts': 928 server_cmd += ['--use_tls=false', '--use_alts=true'] 929 elif transport_security == 'insecure': 930 server_cmd += ['--use_tls=false'] 931 else: 932 print('Invalid transport security option.') 933 sys.exit(1) 934 cmdline = bash_cmdline(language.server_cmd(server_cmd)) 935 environ = language.global_env() 936 docker_args = ['--name=%s' % container_name] 937 if language.safename == 'http2': 938 # we are running the http2 interop server. Open next N ports beginning 939 # with the server port. These ports are used for http2 interop test 940 # (one test case per port). 941 docker_args += list( 942 itertools.chain.from_iterable( 943 ('-p', str(_DEFAULT_SERVER_PORT + i)) 944 for i in range(len(_HTTP2_SERVER_TEST_CASES)))) 945 # Enable docker's healthcheck mechanism. 946 # This runs a Python script inside the container every second. The script 947 # pings the http2 server to verify it is ready. The 'health-retries' flag 948 # specifies the number of consecutive failures before docker will report 949 # the container's status as 'unhealthy'. Prior to the first 'health_retries' 950 # failures or the first success, the status will be 'starting'. 'docker ps' 951 # or 'docker inspect' can be used to see the health of the container on the 952 # command line. 953 docker_args += [ 954 '--health-cmd=python test/http2_test/http2_server_health_check.py ' 955 '--server_host=%s --server_port=%d' % ('localhost', 956 _DEFAULT_SERVER_PORT), 957 '--health-interval=1s', 958 '--health-retries=5', 959 '--health-timeout=10s', 960 ] 961 962 else: 963 docker_args += ['-p', str(_DEFAULT_SERVER_PORT)] 964 965 docker_cmdline = docker_run_cmdline( 966 cmdline, 967 image=docker_image, 968 cwd=language.server_cwd, 969 environ=environ, 970 docker_args=docker_args) 971 if manual_cmd_log is not None: 972 if manual_cmd_log == []: 973 manual_cmd_log.append( 974 'echo "Testing ${docker_image:=%s}"' % docker_image) 975 manual_cmd_log.append(manual_cmdline(docker_cmdline, docker_image)) 976 server_job = jobset.JobSpec( 977 cmdline=docker_cmdline, 978 environ=environ, 979 shortname='interop_server_%s' % language, 980 timeout_seconds=30 * 60) 981 server_job.container_name = container_name 982 return server_job 983 984 985def build_interop_image_jobspec(language, tag=None): 986 """Creates jobspec for building interop docker image for a language""" 987 if not tag: 988 tag = 'grpc_interop_%s:%s' % (language.safename, uuid.uuid4()) 989 env = { 990 'INTEROP_IMAGE': tag, 991 'BASE_NAME': 'grpc_interop_%s' % language.safename 992 } 993 if not args.travis: 994 env['TTY_FLAG'] = '-t' 995 # This env variable is used to get around the github rate limit 996 # error when running the PHP `composer install` command 997 host_file = '%s/.composer/auth.json' % os.environ['HOME'] 998 if language.safename == 'php' and os.path.exists(host_file): 999 env['BUILD_INTEROP_DOCKER_EXTRA_ARGS'] = \ 1000 '-v %s:/root/.composer/auth.json:ro' % host_file 1001 build_job = jobset.JobSpec( 1002 cmdline=['tools/run_tests/dockerize/build_interop_image.sh'], 1003 environ=env, 1004 shortname='build_docker_%s' % (language), 1005 timeout_seconds=30 * 60) 1006 build_job.tag = tag 1007 return build_job 1008 1009 1010def aggregate_http2_results(stdout): 1011 match = re.search(r'\{"cases[^\]]*\]\}', stdout) 1012 if not match: 1013 return None 1014 1015 results = json.loads(match.group(0)) 1016 skipped = 0 1017 passed = 0 1018 failed = 0 1019 failed_cases = [] 1020 for case in results['cases']: 1021 if case.get('skipped', False): 1022 skipped += 1 1023 else: 1024 if case.get('passed', False): 1025 passed += 1 1026 else: 1027 failed += 1 1028 failed_cases.append(case.get('name', "NONAME")) 1029 return { 1030 'passed': passed, 1031 'failed': failed, 1032 'skipped': skipped, 1033 'failed_cases': ', '.join(failed_cases), 1034 'percent': 1.0 * passed / (passed + failed) 1035 } 1036 1037 1038# A dictionary of prod servers to test. 1039prod_servers = { 1040 'default': 'grpc-test.sandbox.googleapis.com', 1041 'gateway_v4': 'grpc-test4.sandbox.googleapis.com', 1042} 1043 1044argp = argparse.ArgumentParser(description='Run interop tests.') 1045argp.add_argument( 1046 '-l', 1047 '--language', 1048 choices=['all'] + sorted(_LANGUAGES), 1049 nargs='+', 1050 default=['all'], 1051 help='Clients to run. Objc client can be only run on OSX.') 1052argp.add_argument('-j', '--jobs', default=multiprocessing.cpu_count(), type=int) 1053argp.add_argument( 1054 '--cloud_to_prod', 1055 default=False, 1056 action='store_const', 1057 const=True, 1058 help='Run cloud_to_prod tests.') 1059argp.add_argument( 1060 '--cloud_to_prod_auth', 1061 default=False, 1062 action='store_const', 1063 const=True, 1064 help='Run cloud_to_prod_auth tests.') 1065argp.add_argument( 1066 '--prod_servers', 1067 choices=prod_servers.keys(), 1068 default=['default'], 1069 nargs='+', 1070 help=('The servers to run cloud_to_prod and ' 1071 'cloud_to_prod_auth tests against.')) 1072argp.add_argument( 1073 '-s', 1074 '--server', 1075 choices=['all'] + sorted(_SERVERS), 1076 nargs='+', 1077 help='Run cloud_to_cloud servers in a separate docker ' + 1078 'image. Servers can only be started automatically if ' + 1079 '--use_docker option is enabled.', 1080 default=[]) 1081argp.add_argument( 1082 '--override_server', 1083 action='append', 1084 type=lambda kv: kv.split('='), 1085 help= 1086 'Use servername=HOST:PORT to explicitly specify a server. E.g. csharp=localhost:50000', 1087 default=[]) 1088argp.add_argument( 1089 '--service_account_key_file', 1090 type=str, 1091 help= 1092 'Override the default service account key file to use for auth interop tests.', 1093 default=None) 1094argp.add_argument( 1095 '-t', '--travis', default=False, action='store_const', const=True) 1096argp.add_argument( 1097 '-v', '--verbose', default=False, action='store_const', const=True) 1098argp.add_argument( 1099 '--use_docker', 1100 default=False, 1101 action='store_const', 1102 const=True, 1103 help='Run all the interop tests under docker. That provides ' + 1104 'additional isolation and prevents the need to install ' + 1105 'language specific prerequisites. Only available on Linux.') 1106argp.add_argument( 1107 '--allow_flakes', 1108 default=False, 1109 action='store_const', 1110 const=True, 1111 help= 1112 'Allow flaky tests to show as passing (re-runs failed tests up to five times)' 1113) 1114argp.add_argument( 1115 '--manual_run', 1116 default=False, 1117 action='store_const', 1118 const=True, 1119 help='Prepare things for running interop tests manually. ' + 1120 'Preserve docker images after building them and skip ' 1121 'actually running the tests. Only print commands to run by ' + 'hand.') 1122argp.add_argument( 1123 '--http2_interop', 1124 default=False, 1125 action='store_const', 1126 const=True, 1127 help='Enable HTTP/2 client edge case testing. (Bad client, good server)') 1128argp.add_argument( 1129 '--http2_server_interop', 1130 default=False, 1131 action='store_const', 1132 const=True, 1133 help= 1134 'Enable HTTP/2 server edge case testing. (Includes positive and negative tests' 1135) 1136argp.add_argument( 1137 '--transport_security', 1138 choices=_TRANSPORT_SECURITY_OPTIONS, 1139 default='tls', 1140 type=str, 1141 nargs='?', 1142 const=True, 1143 help='Which transport security mechanism to use.') 1144argp.add_argument( 1145 '--skip_compute_engine_creds', 1146 default=False, 1147 action='store_const', 1148 const=True, 1149 help='Skip auth tests requiring access to compute engine credentials.') 1150argp.add_argument( 1151 '--internal_ci', 1152 default=False, 1153 action='store_const', 1154 const=True, 1155 help=('Put reports into subdirectories to improve ' 1156 'presentation of results by Internal CI.')) 1157argp.add_argument( 1158 '--bq_result_table', 1159 default='', 1160 type=str, 1161 nargs='?', 1162 help='Upload test results to a specified BQ table.') 1163args = argp.parse_args() 1164 1165servers = set( 1166 s 1167 for s in itertools.chain.from_iterable( 1168 _SERVERS if x == 'all' else [x] for x in args.server)) 1169# ALTS servers are only available for certain languages. 1170if args.transport_security == 'alts': 1171 servers = servers.intersection(_SERVERS_FOR_ALTS_TEST_CASES) 1172 1173if args.use_docker: 1174 if not args.travis: 1175 print('Seen --use_docker flag, will run interop tests under docker.') 1176 print('') 1177 print( 1178 'IMPORTANT: The changes you are testing need to be locally committed' 1179 ) 1180 print( 1181 'because only the committed changes in the current branch will be') 1182 print('copied to the docker environment.') 1183 time.sleep(5) 1184 1185if args.manual_run and not args.use_docker: 1186 print('--manual_run is only supported with --use_docker option enabled.') 1187 sys.exit(1) 1188 1189if not args.use_docker and servers: 1190 print( 1191 'Running interop servers is only supported with --use_docker option enabled.' 1192 ) 1193 sys.exit(1) 1194 1195# we want to include everything but objc in 'all' 1196# because objc won't run on non-mac platforms 1197all_but_objc = set(six.iterkeys(_LANGUAGES)) - set(['objc']) 1198languages = set(_LANGUAGES[l] 1199 for l in itertools.chain.from_iterable( 1200 all_but_objc if x == 'all' else [x] for x in args.language)) 1201# ALTS interop clients are only available for certain languages. 1202if args.transport_security == 'alts': 1203 alts_languages = set(_LANGUAGES[l] for l in _LANGUAGES_FOR_ALTS_TEST_CASES) 1204 languages = languages.intersection(alts_languages) 1205 1206languages_http2_clients_for_http2_server_interop = set() 1207if args.http2_server_interop: 1208 languages_http2_clients_for_http2_server_interop = set( 1209 _LANGUAGES[l] 1210 for l in _LANGUAGES_WITH_HTTP2_CLIENTS_FOR_HTTP2_SERVER_TEST_CASES 1211 if 'all' in args.language or l in args.language) 1212 1213http2Interop = Http2Client() if args.http2_interop else None 1214http2InteropServer = Http2Server() if args.http2_server_interop else None 1215 1216docker_images = {} 1217if args.use_docker: 1218 # languages for which to build docker images 1219 languages_to_build = set( 1220 _LANGUAGES[k] 1221 for k in set([str(l) for l in languages] + [s for s in servers])) 1222 languages_to_build = languages_to_build | languages_http2_clients_for_http2_server_interop 1223 1224 if args.http2_interop: 1225 languages_to_build.add(http2Interop) 1226 1227 if args.http2_server_interop: 1228 languages_to_build.add(http2InteropServer) 1229 1230 build_jobs = [] 1231 for l in languages_to_build: 1232 if str(l) == 'objc': 1233 # we don't need to build a docker image for objc 1234 continue 1235 job = build_interop_image_jobspec(l) 1236 docker_images[str(l)] = job.tag 1237 build_jobs.append(job) 1238 1239 if build_jobs: 1240 jobset.message( 1241 'START', 'Building interop docker images.', do_newline=True) 1242 if args.verbose: 1243 print('Jobs to run: \n%s\n' % '\n'.join(str(j) for j in build_jobs)) 1244 1245 num_failures, _ = jobset.run( 1246 build_jobs, newline_on_success=True, maxjobs=args.jobs) 1247 if num_failures == 0: 1248 jobset.message( 1249 'SUCCESS', 1250 'All docker images built successfully.', 1251 do_newline=True) 1252 else: 1253 jobset.message( 1254 'FAILED', 1255 'Failed to build interop docker images.', 1256 do_newline=True) 1257 for image in six.itervalues(docker_images): 1258 dockerjob.remove_image(image, skip_nonexistent=True) 1259 sys.exit(1) 1260 1261server_manual_cmd_log = [] if args.manual_run else None 1262client_manual_cmd_log = [] if args.manual_run else None 1263 1264# Start interop servers. 1265server_jobs = {} 1266server_addresses = {} 1267try: 1268 for s in servers: 1269 lang = str(s) 1270 spec = server_jobspec( 1271 _LANGUAGES[lang], 1272 docker_images.get(lang), 1273 args.transport_security, 1274 manual_cmd_log=server_manual_cmd_log) 1275 if not args.manual_run: 1276 job = dockerjob.DockerJob(spec) 1277 server_jobs[lang] = job 1278 server_addresses[lang] = ('localhost', 1279 job.mapped_port(_DEFAULT_SERVER_PORT)) 1280 else: 1281 # don't run the server, set server port to a placeholder value 1282 server_addresses[lang] = ('localhost', '${SERVER_PORT}') 1283 1284 http2_server_job = None 1285 if args.http2_server_interop: 1286 # launch a HTTP2 server emulator that creates edge cases 1287 lang = str(http2InteropServer) 1288 spec = server_jobspec( 1289 http2InteropServer, 1290 docker_images.get(lang), 1291 manual_cmd_log=server_manual_cmd_log) 1292 if not args.manual_run: 1293 http2_server_job = dockerjob.DockerJob(spec) 1294 server_jobs[lang] = http2_server_job 1295 else: 1296 # don't run the server, set server port to a placeholder value 1297 server_addresses[lang] = ('localhost', '${SERVER_PORT}') 1298 1299 jobs = [] 1300 if args.cloud_to_prod: 1301 if args.transport_security not in ['tls', 'google_default_credentials']: 1302 print( 1303 'TLS or google default credential is always enabled for cloud_to_prod scenarios.' 1304 ) 1305 for server_host_nickname in args.prod_servers: 1306 for language in languages: 1307 for test_case in _TEST_CASES: 1308 if not test_case in language.unimplemented_test_cases(): 1309 if not test_case in _SKIP_ADVANCED + _SKIP_COMPRESSION: 1310 tls_test_job = cloud_to_prod_jobspec( 1311 language, 1312 test_case, 1313 server_host_nickname, 1314 prod_servers[server_host_nickname], 1315 docker_image=docker_images.get(str(language)), 1316 manual_cmd_log=client_manual_cmd_log, 1317 service_account_key_file=args. 1318 service_account_key_file, 1319 transport_security='tls') 1320 jobs.append(tls_test_job) 1321 if language == 'c++': 1322 google_default_creds_test_job = cloud_to_prod_jobspec( 1323 language, 1324 test_case, 1325 server_host_nickname, 1326 prod_servers[server_host_nickname], 1327 docker_image=docker_images.get( 1328 str(language)), 1329 manual_cmd_log=client_manual_cmd_log, 1330 service_account_key_file=args. 1331 service_account_key_file, 1332 transport_security= 1333 'google_default_credentials') 1334 jobs.append(google_default_creds_test_job) 1335 1336 if args.http2_interop: 1337 for test_case in _HTTP2_TEST_CASES: 1338 test_job = cloud_to_prod_jobspec( 1339 http2Interop, 1340 test_case, 1341 server_host_nickname, 1342 prod_servers[server_host_nickname], 1343 docker_image=docker_images.get(str(http2Interop)), 1344 manual_cmd_log=client_manual_cmd_log, 1345 service_account_key_file=args.service_account_key_file, 1346 transport_security=args.transport_security) 1347 jobs.append(test_job) 1348 1349 if args.cloud_to_prod_auth: 1350 if args.transport_security not in ['tls', 'google_default_credentials']: 1351 print( 1352 'TLS or google default credential is always enabled for cloud_to_prod scenarios.' 1353 ) 1354 for server_host_nickname in args.prod_servers: 1355 for language in languages: 1356 for test_case in _AUTH_TEST_CASES: 1357 if (not args.skip_compute_engine_creds or 1358 not compute_engine_creds_required( 1359 language, test_case)): 1360 if not test_case in language.unimplemented_test_cases(): 1361 tls_test_job = cloud_to_prod_jobspec( 1362 language, 1363 test_case, 1364 server_host_nickname, 1365 prod_servers[server_host_nickname], 1366 docker_image=docker_images.get(str(language)), 1367 auth=True, 1368 manual_cmd_log=client_manual_cmd_log, 1369 service_account_key_file=args. 1370 service_account_key_file, 1371 transport_security='tls') 1372 jobs.append(tls_test_job) 1373 if language == 'c++': 1374 google_default_creds_test_job = cloud_to_prod_jobspec( 1375 language, 1376 test_case, 1377 server_host_nickname, 1378 prod_servers[server_host_nickname], 1379 docker_image=docker_images.get( 1380 str(language)), 1381 manual_cmd_log=client_manual_cmd_log, 1382 service_account_key_file=args. 1383 service_account_key_file, 1384 transport_security= 1385 'google_default_credentials') 1386 jobs.append(google_default_creds_test_job) 1387 1388 for server in args.override_server: 1389 server_name = server[0] 1390 (server_host, server_port) = server[1].split(':') 1391 server_addresses[server_name] = (server_host, server_port) 1392 1393 for server_name, server_address in server_addresses.items(): 1394 (server_host, server_port) = server_address 1395 server_language = _LANGUAGES.get(server_name, None) 1396 skip_server = [] # test cases unimplemented by server 1397 if server_language: 1398 skip_server = server_language.unimplemented_test_cases_server() 1399 for language in languages: 1400 for test_case in _TEST_CASES: 1401 if not test_case in language.unimplemented_test_cases(): 1402 if not test_case in skip_server: 1403 test_job = cloud_to_cloud_jobspec( 1404 language, 1405 test_case, 1406 server_name, 1407 server_host, 1408 server_port, 1409 docker_image=docker_images.get(str(language)), 1410 transport_security=args.transport_security, 1411 manual_cmd_log=client_manual_cmd_log) 1412 jobs.append(test_job) 1413 1414 if args.http2_interop: 1415 for test_case in _HTTP2_TEST_CASES: 1416 if server_name == "go": 1417 # TODO(carl-mastrangelo): Reenable after https://github.com/grpc/grpc-go/issues/434 1418 continue 1419 test_job = cloud_to_cloud_jobspec( 1420 http2Interop, 1421 test_case, 1422 server_name, 1423 server_host, 1424 server_port, 1425 docker_image=docker_images.get(str(http2Interop)), 1426 transport_security=args.transport_security, 1427 manual_cmd_log=client_manual_cmd_log) 1428 jobs.append(test_job) 1429 1430 if args.http2_server_interop: 1431 if not args.manual_run: 1432 http2_server_job.wait_for_healthy(timeout_seconds=600) 1433 for language in languages_http2_clients_for_http2_server_interop: 1434 for test_case in set(_HTTP2_SERVER_TEST_CASES) - set( 1435 _HTTP2_SERVER_TEST_CASES_THAT_USE_GRPC_CLIENTS): 1436 offset = sorted(_HTTP2_SERVER_TEST_CASES).index(test_case) 1437 server_port = _DEFAULT_SERVER_PORT + offset 1438 if not args.manual_run: 1439 server_port = http2_server_job.mapped_port(server_port) 1440 test_job = cloud_to_cloud_jobspec( 1441 language, 1442 test_case, 1443 str(http2InteropServer), 1444 'localhost', 1445 server_port, 1446 docker_image=docker_images.get(str(language)), 1447 manual_cmd_log=client_manual_cmd_log) 1448 jobs.append(test_job) 1449 for language in languages: 1450 # HTTP2_SERVER_TEST_CASES_THAT_USE_GRPC_CLIENTS is a subset of 1451 # HTTP_SERVER_TEST_CASES, in which clients use their gRPC interop clients rather 1452 # than specialized http2 clients, reusing existing test implementations. 1453 # For example, in the "data_frame_padding" test, use language's gRPC 1454 # interop clients and make them think that theyre running "large_unary" 1455 # test case. This avoids implementing a new test case in each language. 1456 for test_case in _HTTP2_SERVER_TEST_CASES_THAT_USE_GRPC_CLIENTS: 1457 if test_case not in language.unimplemented_test_cases(): 1458 offset = sorted(_HTTP2_SERVER_TEST_CASES).index(test_case) 1459 server_port = _DEFAULT_SERVER_PORT + offset 1460 if not args.manual_run: 1461 server_port = http2_server_job.mapped_port(server_port) 1462 if args.transport_security != 'insecure': 1463 print( 1464 ('Creating grpc client to http2 server test case ' 1465 'with insecure connection, even though ' 1466 'args.transport_security is not insecure. Http2 ' 1467 'test server only supports insecure connections.')) 1468 test_job = cloud_to_cloud_jobspec( 1469 language, 1470 test_case, 1471 str(http2InteropServer), 1472 'localhost', 1473 server_port, 1474 docker_image=docker_images.get(str(language)), 1475 transport_security='insecure', 1476 manual_cmd_log=client_manual_cmd_log) 1477 jobs.append(test_job) 1478 1479 if not jobs: 1480 print('No jobs to run.') 1481 for image in six.itervalues(docker_images): 1482 dockerjob.remove_image(image, skip_nonexistent=True) 1483 sys.exit(1) 1484 1485 if args.manual_run: 1486 print('All tests will skipped --manual_run option is active.') 1487 1488 if args.verbose: 1489 print('Jobs to run: \n%s\n' % '\n'.join(str(job) for job in jobs)) 1490 1491 num_failures, resultset = jobset.run( 1492 jobs, 1493 newline_on_success=True, 1494 maxjobs=args.jobs, 1495 skip_jobs=args.manual_run) 1496 if args.bq_result_table and resultset: 1497 upload_interop_results_to_bq(resultset, args.bq_result_table) 1498 if num_failures: 1499 jobset.message('FAILED', 'Some tests failed', do_newline=True) 1500 else: 1501 jobset.message('SUCCESS', 'All tests passed', do_newline=True) 1502 1503 write_cmdlog_maybe(server_manual_cmd_log, 'interop_server_cmds.sh') 1504 write_cmdlog_maybe(client_manual_cmd_log, 'interop_client_cmds.sh') 1505 1506 xml_report_name = _XML_REPORT 1507 if args.internal_ci: 1508 xml_report_name = _INTERNAL_CL_XML_REPORT 1509 report_utils.render_junit_xml_report(resultset, xml_report_name) 1510 1511 for name, job in resultset.items(): 1512 if "http2" in name: 1513 job[0].http2results = aggregate_http2_results(job[0].message) 1514 1515 http2_server_test_cases = (_HTTP2_SERVER_TEST_CASES 1516 if args.http2_server_interop else []) 1517 1518 if num_failures: 1519 sys.exit(1) 1520 else: 1521 sys.exit(0) 1522finally: 1523 # Check if servers are still running. 1524 for server, job in server_jobs.items(): 1525 if not job.is_running(): 1526 print('Server "%s" has exited prematurely.' % server) 1527 1528 dockerjob.finish_jobs([j for j in six.itervalues(server_jobs)]) 1529 1530 for image in six.itervalues(docker_images): 1531 if not args.manual_run: 1532 print('Removing docker image %s' % image) 1533 dockerjob.remove_image(image) 1534 else: 1535 print('Preserving docker image: %s' % image) 1536