1#!/usr/bin/env python 2# Copyright 2015 gRPC authors. 3# 4# Licensed under the Apache License, Version 2.0 (the "License"); 5# you may not use this file except in compliance with the License. 6# You may obtain a copy of the License at 7# 8# http://www.apache.org/licenses/LICENSE-2.0 9# 10# Unless required by applicable law or agreed to in writing, software 11# distributed under the License is distributed on an "AS IS" BASIS, 12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13# See the License for the specific language governing permissions and 14# limitations under the License. 15"""Run interop (cross-language) tests in parallel.""" 16 17from __future__ import print_function 18 19import argparse 20import atexit 21import itertools 22import json 23import multiprocessing 24import os 25import re 26import subprocess 27import sys 28import tempfile 29import time 30import uuid 31import six 32import traceback 33 34import python_utils.dockerjob as dockerjob 35import python_utils.jobset as jobset 36import python_utils.report_utils as report_utils 37 38# Docker doesn't clean up after itself, so we do it on exit. 39atexit.register(lambda: subprocess.call(['stty', 'echo'])) 40 41ROOT = os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]), '../..')) 42os.chdir(ROOT) 43 44_FALLBACK_SERVER_PORT = 443 45_BALANCER_SERVER_PORT = 12000 46_BACKEND_SERVER_PORT = 8080 47 48_TEST_TIMEOUT = 30 49 50_FAKE_SERVERS_SAFENAME = 'fake_servers' 51 52# Use a name that's verified by the test certs 53_SERVICE_NAME = 'server.test.google.fr' 54 55 56class CXXLanguage: 57 58 def __init__(self): 59 self.client_cwd = '/var/local/git/grpc' 60 self.safename = 'cxx' 61 62 def client_cmd(self, args): 63 return ['bins/opt/interop_client'] + args 64 65 def global_env(self): 66 # 1) Set c-ares as the resolver, to 67 # enable grpclb. 68 # 2) Turn on verbose logging. 69 # 3) Set the ROOTS_PATH env variable 70 # to the test CA in order for 71 # GoogleDefaultCredentials to be 72 # able to use the test CA. 73 return { 74 'GRPC_DNS_RESOLVER': 75 'ares', 76 'GRPC_VERBOSITY': 77 'DEBUG', 78 'GRPC_TRACE': 79 'client_channel,glb', 80 'GRPC_DEFAULT_SSL_ROOTS_FILE_PATH': 81 '/var/local/git/grpc/src/core/tsi/test_creds/ca.pem', 82 } 83 84 def __str__(self): 85 return 'c++' 86 87 88class JavaLanguage: 89 90 def __init__(self): 91 self.client_cwd = '/var/local/git/grpc-java' 92 self.safename = str(self) 93 94 def client_cmd(self, args): 95 # Take necessary steps to import our test CA into 96 # the set of test CA's that the Java runtime of the 97 # docker container will pick up, so that 98 # Java GoogleDefaultCreds can use it. 99 pem_to_der_cmd = ('openssl x509 -outform der ' 100 '-in /external_mount/src/core/tsi/test_creds/ca.pem ' 101 '-out /tmp/test_ca.der') 102 keystore_import_cmd = ( 103 'keytool -import ' 104 '-keystore /usr/lib/jvm/java-8-oracle/jre/lib/security/cacerts ' 105 '-file /tmp/test_ca.der ' 106 '-deststorepass changeit ' 107 '-noprompt') 108 return [ 109 'bash', '-c', 110 ('{pem_to_der_cmd} && ' 111 '{keystore_import_cmd} && ' 112 './run-test-client.sh {java_client_args}').format( 113 pem_to_der_cmd=pem_to_der_cmd, 114 keystore_import_cmd=keystore_import_cmd, 115 java_client_args=' '.join(args)) 116 ] 117 118 def global_env(self): 119 # 1) Enable grpclb 120 # 2) Enable verbose logging 121 return { 122 'JAVA_OPTS': ( 123 '-Dio.grpc.internal.DnsNameResolverProvider.enable_grpclb=true ' 124 '-Djava.util.logging.config.file=/var/local/grpc_java_logging/logconf.txt' 125 ) 126 } 127 128 def __str__(self): 129 return 'java' 130 131 132class GoLanguage: 133 134 def __init__(self): 135 self.client_cwd = '/go/src/google.golang.org/grpc/interop/client' 136 self.safename = str(self) 137 138 def client_cmd(self, args): 139 # Copy the test CA file into the path that 140 # the Go runtime in the docker container will use, so 141 # that Go's GoogleDefaultCredentials can use it. 142 # See https://golang.org/src/crypto/x509/root_linux.go. 143 return [ 144 'bash', '-c', 145 ('cp /external_mount/src/core/tsi/test_creds/ca.pem ' 146 '/etc/ssl/certs/ca-certificates.crt && ' 147 '/go/bin/client {go_client_args}').format( 148 go_client_args=' '.join(args)) 149 ] 150 151 def global_env(self): 152 return { 153 'GRPC_GO_LOG_VERBOSITY_LEVEL': '3', 154 'GRPC_GO_LOG_SEVERITY_LEVEL': 'INFO' 155 } 156 157 def __str__(self): 158 return 'go' 159 160 161_LANGUAGES = { 162 'c++': CXXLanguage(), 163 'go': GoLanguage(), 164 'java': JavaLanguage(), 165} 166 167 168def docker_run_cmdline(cmdline, image, docker_args, cwd, environ=None): 169 """Wraps given cmdline array to create 'docker run' cmdline from it.""" 170 # turn environ into -e docker args 171 docker_cmdline = 'docker run -i --rm=true'.split() 172 if environ: 173 for k, v in environ.items(): 174 docker_cmdline += ['-e', '%s=%s' % (k, v)] 175 return docker_cmdline + ['-w', cwd] + docker_args + [image] + cmdline 176 177 178def _job_kill_handler(job): 179 assert job._spec.container_name 180 dockerjob.docker_kill(job._spec.container_name) 181 182 183def transport_security_to_args(transport_security): 184 args = [] 185 if transport_security == 'tls': 186 args += ['--use_tls=true'] 187 elif transport_security == 'alts': 188 args += ['--use_tls=false', '--use_alts=true'] 189 elif transport_security == 'insecure': 190 args += ['--use_tls=false'] 191 elif transport_security == 'google_default_credentials': 192 args += ['--custom_credentials_type=google_default_credentials'] 193 else: 194 print('Invalid transport security option.') 195 sys.exit(1) 196 return args 197 198 199def lb_client_interop_jobspec(language, 200 dns_server_ip, 201 docker_image, 202 transport_security='tls'): 203 """Runs a gRPC client under test in a docker container""" 204 interop_only_options = [ 205 '--server_host=%s' % _SERVICE_NAME, 206 '--server_port=%d' % _FALLBACK_SERVER_PORT 207 ] + transport_security_to_args(transport_security) 208 # Don't set the server host override in any client; 209 # Go and Java default to no override. 210 # We're using a DNS server so there's no need. 211 if language.safename == 'c++': 212 interop_only_options += ['--server_host_override=""'] 213 # Don't set --use_test_ca; we're configuring 214 # clients to use test CA's via alternate means. 215 interop_only_options += ['--use_test_ca=false'] 216 client_args = language.client_cmd(interop_only_options) 217 container_name = dockerjob.random_name('lb_interop_client_%s' % 218 language.safename) 219 docker_cmdline = docker_run_cmdline( 220 client_args, 221 environ=language.global_env(), 222 image=docker_image, 223 cwd=language.client_cwd, 224 docker_args=[ 225 '--dns=%s' % dns_server_ip, 226 '--net=host', 227 '--name=%s' % container_name, 228 '-v', 229 '{grpc_grpc_root_dir}:/external_mount:ro'.format( 230 grpc_grpc_root_dir=ROOT), 231 ]) 232 jobset.message('IDLE', 233 'docker_cmdline:\b|%s|' % ' '.join(docker_cmdline), 234 do_newline=True) 235 test_job = jobset.JobSpec(cmdline=docker_cmdline, 236 shortname=('lb_interop_client:%s' % language), 237 timeout_seconds=_TEST_TIMEOUT, 238 kill_handler=_job_kill_handler) 239 test_job.container_name = container_name 240 return test_job 241 242 243def fallback_server_jobspec(transport_security, shortname): 244 """Create jobspec for running a fallback server""" 245 cmdline = [ 246 'bin/server', 247 '--port=%d' % _FALLBACK_SERVER_PORT, 248 ] + transport_security_to_args(transport_security) 249 return grpc_server_in_docker_jobspec(server_cmdline=cmdline, 250 shortname=shortname) 251 252 253def backend_server_jobspec(transport_security, shortname): 254 """Create jobspec for running a backend server""" 255 cmdline = [ 256 'bin/server', 257 '--port=%d' % _BACKEND_SERVER_PORT, 258 ] + transport_security_to_args(transport_security) 259 return grpc_server_in_docker_jobspec(server_cmdline=cmdline, 260 shortname=shortname) 261 262 263def grpclb_jobspec(transport_security, short_stream, backend_addrs, shortname): 264 """Create jobspec for running a balancer server""" 265 cmdline = [ 266 'bin/fake_grpclb', 267 '--backend_addrs=%s' % ','.join(backend_addrs), 268 '--port=%d' % _BALANCER_SERVER_PORT, 269 '--short_stream=%s' % short_stream, 270 '--service_name=%s' % _SERVICE_NAME, 271 ] + transport_security_to_args(transport_security) 272 return grpc_server_in_docker_jobspec(server_cmdline=cmdline, 273 shortname=shortname) 274 275 276def grpc_server_in_docker_jobspec(server_cmdline, shortname): 277 container_name = dockerjob.random_name(shortname) 278 environ = { 279 'GRPC_GO_LOG_VERBOSITY_LEVEL': '3', 280 'GRPC_GO_LOG_SEVERITY_LEVEL': 'INFO ', 281 } 282 docker_cmdline = docker_run_cmdline( 283 server_cmdline, 284 cwd='/go', 285 image=docker_images.get(_FAKE_SERVERS_SAFENAME), 286 environ=environ, 287 docker_args=['--name=%s' % container_name]) 288 jobset.message('IDLE', 289 'docker_cmdline:\b|%s|' % ' '.join(docker_cmdline), 290 do_newline=True) 291 server_job = jobset.JobSpec(cmdline=docker_cmdline, 292 shortname=shortname, 293 timeout_seconds=30 * 60) 294 server_job.container_name = container_name 295 return server_job 296 297 298def dns_server_in_docker_jobspec(grpclb_ips, fallback_ips, shortname, 299 cause_no_error_no_data_for_balancer_a_record): 300 container_name = dockerjob.random_name(shortname) 301 run_dns_server_cmdline = [ 302 'python', 303 'test/cpp/naming/utils/run_dns_server_for_lb_interop_tests.py', 304 '--grpclb_ips=%s' % ','.join(grpclb_ips), 305 '--fallback_ips=%s' % ','.join(fallback_ips), 306 ] 307 if cause_no_error_no_data_for_balancer_a_record: 308 run_dns_server_cmdline.append( 309 '--cause_no_error_no_data_for_balancer_a_record') 310 docker_cmdline = docker_run_cmdline( 311 run_dns_server_cmdline, 312 cwd='/var/local/git/grpc', 313 image=docker_images.get(_FAKE_SERVERS_SAFENAME), 314 docker_args=['--name=%s' % container_name]) 315 jobset.message('IDLE', 316 'docker_cmdline:\b|%s|' % ' '.join(docker_cmdline), 317 do_newline=True) 318 server_job = jobset.JobSpec(cmdline=docker_cmdline, 319 shortname=shortname, 320 timeout_seconds=30 * 60) 321 server_job.container_name = container_name 322 return server_job 323 324 325def build_interop_image_jobspec(lang_safename, basename_prefix='grpc_interop'): 326 """Creates jobspec for building interop docker image for a language""" 327 tag = '%s_%s:%s' % (basename_prefix, lang_safename, uuid.uuid4()) 328 env = { 329 'INTEROP_IMAGE': tag, 330 'BASE_NAME': '%s_%s' % (basename_prefix, lang_safename), 331 } 332 build_job = jobset.JobSpec( 333 cmdline=['tools/run_tests/dockerize/build_interop_image.sh'], 334 environ=env, 335 shortname='build_docker_%s' % lang_safename, 336 timeout_seconds=30 * 60) 337 build_job.tag = tag 338 return build_job 339 340 341argp = argparse.ArgumentParser(description='Run interop tests.') 342argp.add_argument('-l', 343 '--language', 344 choices=['all'] + sorted(_LANGUAGES), 345 nargs='+', 346 default=['all'], 347 help='Clients to run.') 348argp.add_argument('-j', '--jobs', default=multiprocessing.cpu_count(), type=int) 349argp.add_argument('-s', 350 '--scenarios_file', 351 default=None, 352 type=str, 353 help='File containing test scenarios as JSON configs.') 354argp.add_argument( 355 '-n', 356 '--scenario_name', 357 default=None, 358 type=str, 359 help=( 360 'Useful for manual runs: specify the name of ' 361 'the scenario to run from scenarios_file. Run all scenarios if unset.')) 362argp.add_argument('--cxx_image_tag', 363 default=None, 364 type=str, 365 help=('Setting this skips the clients docker image ' 366 'build step and runs the client from the named ' 367 'image. Only supports running a one client language.')) 368argp.add_argument('--go_image_tag', 369 default=None, 370 type=str, 371 help=('Setting this skips the clients docker image build ' 372 'step and runs the client from the named image. Only ' 373 'supports running a one client language.')) 374argp.add_argument('--java_image_tag', 375 default=None, 376 type=str, 377 help=('Setting this skips the clients docker image build ' 378 'step and runs the client from the named image. Only ' 379 'supports running a one client language.')) 380argp.add_argument( 381 '--servers_image_tag', 382 default=None, 383 type=str, 384 help=('Setting this skips the fake servers docker image ' 385 'build step and runs the servers from the named image.')) 386argp.add_argument('--no_skips', 387 default=False, 388 type=bool, 389 nargs='?', 390 const=True, 391 help=('Useful for manual runs. Setting this overrides test ' 392 '"skips" configured in test scenarios.')) 393argp.add_argument('--verbose', 394 default=False, 395 type=bool, 396 nargs='?', 397 const=True, 398 help='Increase logging.') 399args = argp.parse_args() 400 401docker_images = {} 402 403build_jobs = [] 404if len(args.language) and args.language[0] == 'all': 405 languages = _LANGUAGES.keys() 406else: 407 languages = args.language 408for lang_name in languages: 409 l = _LANGUAGES[lang_name] 410 # First check if a pre-built image was supplied, and avoid 411 # rebuilding the particular docker image if so. 412 if lang_name == 'c++' and args.cxx_image_tag: 413 docker_images[str(l.safename)] = args.cxx_image_tag 414 elif lang_name == 'go' and args.go_image_tag: 415 docker_images[str(l.safename)] = args.go_image_tag 416 elif lang_name == 'java' and args.java_image_tag: 417 docker_images[str(l.safename)] = args.java_image_tag 418 else: 419 # Build the test client in docker and save the fully 420 # built image. 421 job = build_interop_image_jobspec(l.safename) 422 build_jobs.append(job) 423 docker_images[str(l.safename)] = job.tag 424 425# First check if a pre-built image was supplied. 426if args.servers_image_tag: 427 docker_images[_FAKE_SERVERS_SAFENAME] = args.servers_image_tag 428else: 429 # Build the test servers in docker and save the fully 430 # built image. 431 job = build_interop_image_jobspec(_FAKE_SERVERS_SAFENAME, 432 basename_prefix='lb_interop') 433 build_jobs.append(job) 434 docker_images[_FAKE_SERVERS_SAFENAME] = job.tag 435 436if build_jobs: 437 jobset.message('START', 'Building interop docker images.', do_newline=True) 438 print('Jobs to run: \n%s\n' % '\n'.join(str(j) for j in build_jobs)) 439 num_failures, _ = jobset.run(build_jobs, 440 newline_on_success=True, 441 maxjobs=args.jobs) 442 if num_failures == 0: 443 jobset.message('SUCCESS', 444 'All docker images built successfully.', 445 do_newline=True) 446 else: 447 jobset.message('FAILED', 448 'Failed to build interop docker images.', 449 do_newline=True) 450 sys.exit(1) 451 452 453def wait_until_dns_server_is_up(dns_server_ip): 454 """Probes the DNS server until it's running and safe for tests.""" 455 for i in range(0, 30): 456 print('Health check: attempt to connect to DNS server over TCP.') 457 tcp_connect_subprocess = subprocess.Popen([ 458 os.path.join(os.getcwd(), 'test/cpp/naming/utils/tcp_connect.py'), 459 '--server_host', dns_server_ip, '--server_port', 460 str(53), '--timeout', 461 str(1) 462 ]) 463 tcp_connect_subprocess.communicate() 464 if tcp_connect_subprocess.returncode == 0: 465 print(('Health check: attempt to make an A-record ' 466 'query to DNS server.')) 467 dns_resolver_subprocess = subprocess.Popen([ 468 os.path.join(os.getcwd(), 469 'test/cpp/naming/utils/dns_resolver.py'), 470 '--qname', 471 ('health-check-local-dns-server-is-alive.' 472 'resolver-tests.grpctestingexp'), '--server_host', 473 dns_server_ip, '--server_port', 474 str(53) 475 ], 476 stdout=subprocess.PIPE) 477 dns_resolver_stdout, _ = dns_resolver_subprocess.communicate() 478 if dns_resolver_subprocess.returncode == 0: 479 if '123.123.123.123' in dns_resolver_stdout: 480 print(('DNS server is up! ' 481 'Successfully reached it over UDP and TCP.')) 482 return 483 time.sleep(0.1) 484 raise Exception(('Failed to reach DNS server over TCP and/or UDP. ' 485 'Exitting without running tests.')) 486 487 488def shortname(shortname_prefix, shortname, index): 489 return '%s_%s_%d' % (shortname_prefix, shortname, index) 490 491 492def run_one_scenario(scenario_config): 493 jobset.message('START', 'Run scenario: %s' % scenario_config['name']) 494 server_jobs = {} 495 server_addresses = {} 496 suppress_server_logs = True 497 try: 498 backend_addrs = [] 499 fallback_ips = [] 500 grpclb_ips = [] 501 shortname_prefix = scenario_config['name'] 502 # Start backends 503 for i in xrange(len(scenario_config['backend_configs'])): 504 backend_config = scenario_config['backend_configs'][i] 505 backend_shortname = shortname(shortname_prefix, 'backend_server', i) 506 backend_spec = backend_server_jobspec( 507 backend_config['transport_sec'], backend_shortname) 508 backend_job = dockerjob.DockerJob(backend_spec) 509 server_jobs[backend_shortname] = backend_job 510 backend_addrs.append( 511 '%s:%d' % (backend_job.ip_address(), _BACKEND_SERVER_PORT)) 512 # Start fallbacks 513 for i in xrange(len(scenario_config['fallback_configs'])): 514 fallback_config = scenario_config['fallback_configs'][i] 515 fallback_shortname = shortname(shortname_prefix, 'fallback_server', 516 i) 517 fallback_spec = fallback_server_jobspec( 518 fallback_config['transport_sec'], fallback_shortname) 519 fallback_job = dockerjob.DockerJob(fallback_spec) 520 server_jobs[fallback_shortname] = fallback_job 521 fallback_ips.append(fallback_job.ip_address()) 522 # Start balancers 523 for i in xrange(len(scenario_config['balancer_configs'])): 524 balancer_config = scenario_config['balancer_configs'][i] 525 grpclb_shortname = shortname(shortname_prefix, 'grpclb_server', i) 526 grpclb_spec = grpclb_jobspec(balancer_config['transport_sec'], 527 balancer_config['short_stream'], 528 backend_addrs, grpclb_shortname) 529 grpclb_job = dockerjob.DockerJob(grpclb_spec) 530 server_jobs[grpclb_shortname] = grpclb_job 531 grpclb_ips.append(grpclb_job.ip_address()) 532 # Start DNS server 533 dns_server_shortname = shortname(shortname_prefix, 'dns_server', 0) 534 dns_server_spec = dns_server_in_docker_jobspec( 535 grpclb_ips, fallback_ips, dns_server_shortname, 536 scenario_config['cause_no_error_no_data_for_balancer_a_record']) 537 dns_server_job = dockerjob.DockerJob(dns_server_spec) 538 server_jobs[dns_server_shortname] = dns_server_job 539 # Get the IP address of the docker container running the DNS server. 540 # The DNS server is running on port 53 of that IP address. Note we will 541 # point the DNS resolvers of grpc clients under test to our controlled 542 # DNS server by effectively modifying the /etc/resolve.conf "nameserver" 543 # lists of their docker containers. 544 dns_server_ip = dns_server_job.ip_address() 545 wait_until_dns_server_is_up(dns_server_ip) 546 # Run clients 547 jobs = [] 548 for lang_name in languages: 549 # Skip languages that are known to not currently 550 # work for this test. 551 if not args.no_skips and lang_name in scenario_config.get( 552 'skip_langs', []): 553 jobset.message( 554 'IDLE', 'Skipping scenario: %s for language: %s\n' % 555 (scenario_config['name'], lang_name)) 556 continue 557 lang = _LANGUAGES[lang_name] 558 test_job = lb_client_interop_jobspec( 559 lang, 560 dns_server_ip, 561 docker_image=docker_images.get(lang.safename), 562 transport_security=scenario_config['transport_sec']) 563 jobs.append(test_job) 564 jobset.message( 565 'IDLE', 'Jobs to run: \n%s\n' % '\n'.join(str(job) for job in jobs)) 566 num_failures, resultset = jobset.run(jobs, 567 newline_on_success=True, 568 maxjobs=args.jobs) 569 report_utils.render_junit_xml_report(resultset, 'sponge_log.xml') 570 if num_failures: 571 suppress_server_logs = False 572 jobset.message('FAILED', 573 'Scenario: %s. Some tests failed' % 574 scenario_config['name'], 575 do_newline=True) 576 else: 577 jobset.message('SUCCESS', 578 'Scenario: %s. All tests passed' % 579 scenario_config['name'], 580 do_newline=True) 581 return num_failures 582 finally: 583 # Check if servers are still running. 584 for server, job in server_jobs.items(): 585 if not job.is_running(): 586 print('Server "%s" has exited prematurely.' % server) 587 suppress_failure = suppress_server_logs and not args.verbose 588 dockerjob.finish_jobs([j for j in six.itervalues(server_jobs)], 589 suppress_failure=suppress_failure) 590 591 592num_failures = 0 593with open(args.scenarios_file, 'r') as scenarios_input: 594 all_scenarios = json.loads(scenarios_input.read()) 595 for scenario in all_scenarios: 596 if args.scenario_name: 597 if args.scenario_name != scenario['name']: 598 jobset.message('IDLE', 599 'Skipping scenario: %s' % scenario['name']) 600 continue 601 num_failures += run_one_scenario(scenario) 602if num_failures == 0: 603 sys.exit(0) 604else: 605 sys.exit(1) 606