1#!/usr/bin/env python 2# Copyright 2015 gRPC authors. 3# 4# Licensed under the Apache License, Version 2.0 (the "License"); 5# you may not use this file except in compliance with the License. 6# You may obtain a copy of the License at 7# 8# http://www.apache.org/licenses/LICENSE-2.0 9# 10# Unless required by applicable law or agreed to in writing, software 11# distributed under the License is distributed on an "AS IS" BASIS, 12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13# See the License for the specific language governing permissions and 14# limitations under the License. 15"""Run test matrix.""" 16 17from __future__ import print_function 18 19import argparse 20import multiprocessing 21import os 22import sys 23 24import python_utils.jobset as jobset 25import python_utils.report_utils as report_utils 26from python_utils.filter_pull_request_tests import filter_tests 27 28_ROOT = os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]), '../..')) 29os.chdir(_ROOT) 30 31_DEFAULT_RUNTESTS_TIMEOUT = 1 * 60 * 60 32 33# Set the timeout high to allow enough time for sanitizers and pre-building 34# clang docker. 35_CPP_RUNTESTS_TIMEOUT = 4 * 60 * 60 36 37# C++ TSAN takes longer than other sanitizers 38_CPP_TSAN_RUNTESTS_TIMEOUT = 8 * 60 * 60 39 40# Set timeout high for ObjC for Cocoapods to install pods 41_OBJC_RUNTESTS_TIMEOUT = 90 * 60 42 43# Number of jobs assigned to each run_tests.py instance 44_DEFAULT_INNER_JOBS = 2 45 46# report suffix is important for reports to get picked up by internal CI 47_REPORT_SUFFIX = 'sponge_log.xml' 48 49 50def _safe_report_name(name): 51 """Reports with '+' in target name won't show correctly in ResultStore""" 52 return name.replace('+', 'p') 53 54 55def _report_filename(name): 56 """Generates report file name with directory structure that leads to better presentation by internal CI""" 57 return '%s/%s' % (_safe_report_name(name), _REPORT_SUFFIX) 58 59 60def _docker_jobspec(name, 61 runtests_args=[], 62 runtests_envs={}, 63 inner_jobs=_DEFAULT_INNER_JOBS, 64 timeout_seconds=None): 65 """Run a single instance of run_tests.py in a docker container""" 66 if not timeout_seconds: 67 timeout_seconds = _DEFAULT_RUNTESTS_TIMEOUT 68 test_job = jobset.JobSpec( 69 cmdline=[ 70 'python', 'tools/run_tests/run_tests.py', '--use_docker', '-t', 71 '-j', 72 str(inner_jobs), '-x', 73 _report_filename(name), '--report_suite_name', 74 '%s' % _safe_report_name(name) 75 ] + runtests_args, 76 environ=runtests_envs, 77 shortname='run_tests_%s' % name, 78 timeout_seconds=timeout_seconds) 79 return test_job 80 81 82def _workspace_jobspec(name, 83 runtests_args=[], 84 workspace_name=None, 85 runtests_envs={}, 86 inner_jobs=_DEFAULT_INNER_JOBS, 87 timeout_seconds=None): 88 """Run a single instance of run_tests.py in a separate workspace""" 89 if not workspace_name: 90 workspace_name = 'workspace_%s' % name 91 if not timeout_seconds: 92 timeout_seconds = _DEFAULT_RUNTESTS_TIMEOUT 93 env = {'WORKSPACE_NAME': workspace_name} 94 env.update(runtests_envs) 95 test_job = jobset.JobSpec( 96 cmdline=[ 97 'bash', 'tools/run_tests/helper_scripts/run_tests_in_workspace.sh', 98 '-t', '-j', 99 str(inner_jobs), '-x', 100 '../%s' % _report_filename(name), '--report_suite_name', 101 '%s' % _safe_report_name(name) 102 ] + runtests_args, 103 environ=env, 104 shortname='run_tests_%s' % name, 105 timeout_seconds=timeout_seconds) 106 return test_job 107 108 109def _generate_jobs(languages, 110 configs, 111 platforms, 112 iomgr_platforms=['native'], 113 arch=None, 114 compiler=None, 115 labels=[], 116 extra_args=[], 117 extra_envs={}, 118 inner_jobs=_DEFAULT_INNER_JOBS, 119 timeout_seconds=None): 120 result = [] 121 for language in languages: 122 for platform in platforms: 123 for iomgr_platform in iomgr_platforms: 124 for config in configs: 125 name = '%s_%s_%s_%s' % (language, platform, config, 126 iomgr_platform) 127 runtests_args = [ 128 '-l', language, '-c', config, '--iomgr_platform', 129 iomgr_platform 130 ] 131 if arch or compiler: 132 name += '_%s_%s' % (arch, compiler) 133 runtests_args += [ 134 '--arch', arch, '--compiler', compiler 135 ] 136 if '--build_only' in extra_args: 137 name += '_buildonly' 138 for extra_env in extra_envs: 139 name += '_%s_%s' % (extra_env, extra_envs[extra_env]) 140 141 runtests_args += extra_args 142 if platform == 'linux': 143 job = _docker_jobspec( 144 name=name, 145 runtests_args=runtests_args, 146 runtests_envs=extra_envs, 147 inner_jobs=inner_jobs, 148 timeout_seconds=timeout_seconds) 149 else: 150 job = _workspace_jobspec( 151 name=name, 152 runtests_args=runtests_args, 153 runtests_envs=extra_envs, 154 inner_jobs=inner_jobs, 155 timeout_seconds=timeout_seconds) 156 157 job.labels = [platform, config, language, iomgr_platform 158 ] + labels 159 result.append(job) 160 return result 161 162 163def _create_test_jobs(extra_args=[], inner_jobs=_DEFAULT_INNER_JOBS): 164 test_jobs = [] 165 # sanity tests 166 test_jobs += _generate_jobs( 167 languages=['sanity'], 168 configs=['dbg', 'opt'], 169 platforms=['linux'], 170 labels=['basictests'], 171 extra_args=extra_args, 172 inner_jobs=inner_jobs) 173 174 # supported on linux only 175 test_jobs += _generate_jobs( 176 languages=['php7'], 177 configs=['dbg', 'opt'], 178 platforms=['linux'], 179 labels=['basictests', 'multilang'], 180 extra_args=extra_args, 181 inner_jobs=inner_jobs) 182 183 # supported on all platforms. 184 test_jobs += _generate_jobs( 185 languages=['c'], 186 configs=['dbg', 'opt'], 187 platforms=['linux', 'macos', 'windows'], 188 labels=['basictests', 'corelang'], 189 extra_args=extra_args, 190 inner_jobs=inner_jobs, 191 timeout_seconds=_CPP_RUNTESTS_TIMEOUT) 192 193 test_jobs += _generate_jobs( 194 languages=['csharp'], 195 configs=['dbg', 'opt'], 196 platforms=['linux', 'macos', 'windows'], 197 labels=['basictests', 'multilang'], 198 extra_args=extra_args, 199 inner_jobs=inner_jobs) 200 201 test_jobs += _generate_jobs( 202 languages=['python'], 203 configs=['opt'], 204 platforms=['linux', 'macos', 'windows'], 205 iomgr_platforms=['native', 'gevent'], 206 labels=['basictests', 'multilang'], 207 extra_args=extra_args, 208 inner_jobs=inner_jobs) 209 210 # supported on linux and mac. 211 test_jobs += _generate_jobs( 212 languages=['c++'], 213 configs=['dbg', 'opt'], 214 platforms=['linux', 'macos'], 215 labels=['basictests', 'corelang'], 216 extra_args=extra_args, 217 inner_jobs=inner_jobs, 218 timeout_seconds=_CPP_RUNTESTS_TIMEOUT) 219 220 test_jobs += _generate_jobs( 221 languages=['grpc-node', 'ruby', 'php'], 222 configs=['dbg', 'opt'], 223 platforms=['linux', 'macos'], 224 labels=['basictests', 'multilang'], 225 extra_args=extra_args, 226 inner_jobs=inner_jobs) 227 228 # supported on mac only. 229 test_jobs += _generate_jobs( 230 languages=['objc'], 231 configs=['dbg', 'opt'], 232 platforms=['macos'], 233 labels=['basictests', 'multilang'], 234 extra_args=extra_args, 235 inner_jobs=inner_jobs, 236 timeout_seconds=_OBJC_RUNTESTS_TIMEOUT) 237 238 # sanitizers 239 test_jobs += _generate_jobs( 240 languages=['c'], 241 configs=['msan', 'asan', 'tsan', 'ubsan'], 242 platforms=['linux'], 243 arch='x64', 244 compiler='clang7.0', 245 labels=['sanitizers', 'corelang'], 246 extra_args=extra_args, 247 inner_jobs=inner_jobs, 248 timeout_seconds=_CPP_RUNTESTS_TIMEOUT) 249 test_jobs += _generate_jobs( 250 languages=['c++'], 251 configs=['asan'], 252 platforms=['linux'], 253 arch='x64', 254 compiler='clang7.0', 255 labels=['sanitizers', 'corelang'], 256 extra_args=extra_args, 257 inner_jobs=inner_jobs, 258 timeout_seconds=_CPP_RUNTESTS_TIMEOUT) 259 test_jobs += _generate_jobs( 260 languages=['c++'], 261 configs=['tsan'], 262 platforms=['linux'], 263 arch='x64', 264 compiler='clang7.0', 265 labels=['sanitizers', 'corelang'], 266 extra_args=extra_args, 267 inner_jobs=inner_jobs, 268 timeout_seconds=_CPP_TSAN_RUNTESTS_TIMEOUT) 269 270 return test_jobs 271 272 273def _create_portability_test_jobs(extra_args=[], 274 inner_jobs=_DEFAULT_INNER_JOBS): 275 test_jobs = [] 276 # portability C x86 277 test_jobs += _generate_jobs( 278 languages=['c'], 279 configs=['dbg'], 280 platforms=['linux'], 281 arch='x86', 282 compiler='default', 283 labels=['portability', 'corelang'], 284 extra_args=extra_args, 285 inner_jobs=inner_jobs) 286 287 # portability C and C++ on x64 288 for compiler in [ 289 'gcc4.8', 'gcc5.3', 'gcc7.2', 'gcc_musl', 'clang3.5', 'clang3.6', 290 'clang3.7', 'clang7.0' 291 ]: 292 test_jobs += _generate_jobs( 293 languages=['c', 'c++'], 294 configs=['dbg'], 295 platforms=['linux'], 296 arch='x64', 297 compiler=compiler, 298 labels=['portability', 'corelang'], 299 extra_args=extra_args, 300 inner_jobs=inner_jobs, 301 timeout_seconds=_CPP_RUNTESTS_TIMEOUT) 302 303 # portability C on Windows 64-bit (x86 is the default) 304 test_jobs += _generate_jobs( 305 languages=['c'], 306 configs=['dbg'], 307 platforms=['windows'], 308 arch='x64', 309 compiler='default', 310 labels=['portability', 'corelang'], 311 extra_args=extra_args, 312 inner_jobs=inner_jobs) 313 314 # portability C++ on Windows 315 # TODO(jtattermusch): some of the tests are failing, so we force --build_only 316 test_jobs += _generate_jobs( 317 languages=['c++'], 318 configs=['dbg'], 319 platforms=['windows'], 320 arch='default', 321 compiler='default', 322 labels=['portability', 'corelang'], 323 extra_args=extra_args + ['--build_only'], 324 inner_jobs=inner_jobs) 325 326 # portability C and C++ on Windows using VS2017 (build only) 327 # TODO(jtattermusch): some of the tests are failing, so we force --build_only 328 test_jobs += _generate_jobs( 329 languages=['c', 'c++'], 330 configs=['dbg'], 331 platforms=['windows'], 332 arch='x64', 333 compiler='cmake_vs2017', 334 labels=['portability', 'corelang'], 335 extra_args=extra_args + ['--build_only'], 336 inner_jobs=inner_jobs) 337 338 # C and C++ with the c-ares DNS resolver on Linux 339 test_jobs += _generate_jobs( 340 languages=['c', 'c++'], 341 configs=['dbg'], 342 platforms=['linux'], 343 labels=['portability', 'corelang'], 344 extra_args=extra_args, 345 extra_envs={'GRPC_DNS_RESOLVER': 'ares'}, 346 timeout_seconds=_CPP_RUNTESTS_TIMEOUT) 347 348 # C and C++ with no-exceptions on Linux 349 test_jobs += _generate_jobs( 350 languages=['c', 'c++'], 351 configs=['noexcept'], 352 platforms=['linux'], 353 labels=['portability', 'corelang'], 354 extra_args=extra_args, 355 timeout_seconds=_CPP_RUNTESTS_TIMEOUT) 356 357 # TODO(zyc): Turn on this test after adding c-ares support on windows. 358 # C with the c-ares DNS resolver on Windows 359 # test_jobs += _generate_jobs(languages=['c'], 360 # configs=['dbg'], platforms=['windows'], 361 # labels=['portability', 'corelang'], 362 # extra_args=extra_args, 363 # extra_envs={'GRPC_DNS_RESOLVER': 'ares'}) 364 365 # C and C++ build with cmake on Linux 366 # TODO(jtattermusch): some of the tests are failing, so we force --build_only 367 # to make sure it's buildable at least. 368 test_jobs += _generate_jobs( 369 languages=['c', 'c++'], 370 configs=['dbg'], 371 platforms=['linux'], 372 arch='default', 373 compiler='cmake', 374 labels=['portability', 'corelang'], 375 extra_args=extra_args + ['--build_only'], 376 inner_jobs=inner_jobs) 377 378 test_jobs += _generate_jobs( 379 languages=['python'], 380 configs=['dbg'], 381 platforms=['linux'], 382 arch='default', 383 compiler='python_alpine', 384 labels=['portability', 'multilang'], 385 extra_args=extra_args, 386 inner_jobs=inner_jobs) 387 388 test_jobs += _generate_jobs( 389 languages=['csharp'], 390 configs=['dbg'], 391 platforms=['linux'], 392 arch='default', 393 compiler='coreclr', 394 labels=['portability', 'multilang'], 395 extra_args=extra_args, 396 inner_jobs=inner_jobs) 397 398 test_jobs += _generate_jobs( 399 languages=['c'], 400 configs=['dbg'], 401 platforms=['linux'], 402 iomgr_platforms=['uv'], 403 labels=['portability', 'corelang'], 404 extra_args=extra_args, 405 inner_jobs=inner_jobs, 406 timeout_seconds=_CPP_RUNTESTS_TIMEOUT) 407 408 return test_jobs 409 410 411def _allowed_labels(): 412 """Returns a list of existing job labels.""" 413 all_labels = set() 414 for job in _create_test_jobs() + _create_portability_test_jobs(): 415 for label in job.labels: 416 all_labels.add(label) 417 return sorted(all_labels) 418 419 420def _runs_per_test_type(arg_str): 421 """Auxiliary function to parse the "runs_per_test" flag.""" 422 try: 423 n = int(arg_str) 424 if n <= 0: raise ValueError 425 return n 426 except: 427 msg = '\'{}\' is not a positive integer'.format(arg_str) 428 raise argparse.ArgumentTypeError(msg) 429 430 431if __name__ == "__main__": 432 argp = argparse.ArgumentParser( 433 description='Run a matrix of run_tests.py tests.') 434 argp.add_argument( 435 '-j', 436 '--jobs', 437 default=multiprocessing.cpu_count() / _DEFAULT_INNER_JOBS, 438 type=int, 439 help='Number of concurrent run_tests.py instances.') 440 argp.add_argument( 441 '-f', 442 '--filter', 443 choices=_allowed_labels(), 444 nargs='+', 445 default=[], 446 help='Filter targets to run by label with AND semantics.') 447 argp.add_argument( 448 '--exclude', 449 choices=_allowed_labels(), 450 nargs='+', 451 default=[], 452 help='Exclude targets with any of given labels.') 453 argp.add_argument( 454 '--build_only', 455 default=False, 456 action='store_const', 457 const=True, 458 help='Pass --build_only flag to run_tests.py instances.') 459 argp.add_argument( 460 '--force_default_poller', 461 default=False, 462 action='store_const', 463 const=True, 464 help='Pass --force_default_poller to run_tests.py instances.') 465 argp.add_argument( 466 '--dry_run', 467 default=False, 468 action='store_const', 469 const=True, 470 help='Only print what would be run.') 471 argp.add_argument( 472 '--filter_pr_tests', 473 default=False, 474 action='store_const', 475 const=True, 476 help='Filters out tests irrelevant to pull request changes.') 477 argp.add_argument( 478 '--base_branch', 479 default='origin/master', 480 type=str, 481 help='Branch that pull request is requesting to merge into') 482 argp.add_argument( 483 '--inner_jobs', 484 default=_DEFAULT_INNER_JOBS, 485 type=int, 486 help='Number of jobs in each run_tests.py instance') 487 argp.add_argument( 488 '-n', 489 '--runs_per_test', 490 default=1, 491 type=_runs_per_test_type, 492 help='How many times to run each tests. >1 runs implies ' + 493 'omitting passing test from the output & reports.') 494 argp.add_argument( 495 '--max_time', 496 default=-1, 497 type=int, 498 help='Maximum amount of time to run tests for' + 499 '(other tests will be skipped)') 500 argp.add_argument( 501 '--internal_ci', 502 default=False, 503 action='store_const', 504 const=True, 505 help= 506 '(Deprecated, has no effect) Put reports into subdirectories to improve presentation of ' 507 'results by Kokoro.') 508 argp.add_argument( 509 '--bq_result_table', 510 default='', 511 type=str, 512 nargs='?', 513 help='Upload test results to a specified BQ table.') 514 args = argp.parse_args() 515 516 extra_args = [] 517 if args.build_only: 518 extra_args.append('--build_only') 519 if args.force_default_poller: 520 extra_args.append('--force_default_poller') 521 if args.runs_per_test > 1: 522 extra_args.append('-n') 523 extra_args.append('%s' % args.runs_per_test) 524 extra_args.append('--quiet_success') 525 if args.max_time > 0: 526 extra_args.extend(('--max_time', '%d' % args.max_time)) 527 if args.bq_result_table: 528 extra_args.append('--bq_result_table') 529 extra_args.append('%s' % args.bq_result_table) 530 extra_args.append('--measure_cpu_costs') 531 532 all_jobs = _create_test_jobs(extra_args=extra_args, inner_jobs=args.inner_jobs) + \ 533 _create_portability_test_jobs(extra_args=extra_args, inner_jobs=args.inner_jobs) 534 535 jobs = [] 536 for job in all_jobs: 537 if not args.filter or all( 538 filter in job.labels for filter in args.filter): 539 if not any(exclude_label in job.labels 540 for exclude_label in args.exclude): 541 jobs.append(job) 542 543 if not jobs: 544 jobset.message( 545 'FAILED', 'No test suites match given criteria.', do_newline=True) 546 sys.exit(1) 547 548 print('IMPORTANT: The changes you are testing need to be locally committed') 549 print('because only the committed changes in the current branch will be') 550 print('copied to the docker environment or into subworkspaces.') 551 552 skipped_jobs = [] 553 554 if args.filter_pr_tests: 555 print('Looking for irrelevant tests to skip...') 556 relevant_jobs = filter_tests(jobs, args.base_branch) 557 if len(relevant_jobs) == len(jobs): 558 print('No tests will be skipped.') 559 else: 560 print('These tests will be skipped:') 561 skipped_jobs = list(set(jobs) - set(relevant_jobs)) 562 # Sort by shortnames to make printing of skipped tests consistent 563 skipped_jobs.sort(key=lambda job: job.shortname) 564 for job in list(skipped_jobs): 565 print(' %s' % job.shortname) 566 jobs = relevant_jobs 567 568 print('Will run these tests:') 569 for job in jobs: 570 if args.dry_run: 571 print(' %s: "%s"' % (job.shortname, ' '.join(job.cmdline))) 572 else: 573 print(' %s' % job.shortname) 574 print 575 576 if args.dry_run: 577 print('--dry_run was used, exiting') 578 sys.exit(1) 579 580 jobset.message('START', 'Running test matrix.', do_newline=True) 581 num_failures, resultset = jobset.run( 582 jobs, newline_on_success=True, travis=True, maxjobs=args.jobs) 583 # Merge skipped tests into results to show skipped tests on report.xml 584 if skipped_jobs: 585 ignored_num_skipped_failures, skipped_results = jobset.run( 586 skipped_jobs, skip_jobs=True) 587 resultset.update(skipped_results) 588 report_utils.render_junit_xml_report( 589 resultset, 590 _report_filename('aggregate_tests'), 591 suite_name='aggregate_tests') 592 593 if num_failures == 0: 594 jobset.message( 595 'SUCCESS', 596 'All run_tests.py instance finished successfully.', 597 do_newline=True) 598 else: 599 jobset.message( 600 'FAILED', 601 'Some run_tests.py instance have failed.', 602 do_newline=True) 603 sys.exit(1) 604