1#!/usr/bin/env python 2# Copyright 2015 gRPC authors. 3# 4# Licensed under the Apache License, Version 2.0 (the "License"); 5# you may not use this file except in compliance with the License. 6# You may obtain a copy of the License at 7# 8# http://www.apache.org/licenses/LICENSE-2.0 9# 10# Unless required by applicable law or agreed to in writing, software 11# distributed under the License is distributed on an "AS IS" BASIS, 12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13# See the License for the specific language governing permissions and 14# limitations under the License. 15"""Run test matrix.""" 16 17from __future__ import print_function 18 19import argparse 20import multiprocessing 21import os 22import sys 23 24import python_utils.jobset as jobset 25import python_utils.report_utils as report_utils 26from python_utils.filter_pull_request_tests import filter_tests 27 28_ROOT = os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]), '../..')) 29os.chdir(_ROOT) 30 31_DEFAULT_RUNTESTS_TIMEOUT = 1 * 60 * 60 32 33# C/C++ tests can take long time 34_CPP_RUNTESTS_TIMEOUT = 4 * 60 * 60 35 36# Set timeout high for ObjC for Cocoapods to install pods 37_OBJC_RUNTESTS_TIMEOUT = 90 * 60 38 39# Number of jobs assigned to each run_tests.py instance 40_DEFAULT_INNER_JOBS = 2 41 42# Name of the top-level umbrella report that includes all the run_tests.py invocations 43# Note that the starting letter 't' matters so that the targets are listed AFTER 44# the per-test breakdown items that start with 'run_tests/' (it is more readable that way) 45_MATRIX_REPORT_NAME = 'toplevel_run_tests_invocations' 46 47 48def _safe_report_name(name): 49 """Reports with '+' in target name won't show correctly in ResultStore""" 50 return name.replace('+', 'p') 51 52 53def _report_filename(name): 54 """Generates report file name with directory structure that leads to better presentation by internal CI""" 55 # 'sponge_log.xml' suffix must be there for results to get recognized by kokoro. 56 return '%s/%s' % (_safe_report_name(name), 'sponge_log.xml') 57 58 59def _matrix_job_logfilename(shortname_for_multi_target): 60 """Generate location for log file that will match the sponge_log.xml from the top-level matrix report.""" 61 # 'sponge_log.log' suffix must be there for log to get recognized as "target log" 62 # for the corresponding 'sponge_log.xml' report. 63 # the shortname_for_multi_target component must be set to match the sponge_log.xml location 64 # because the top-level render_junit_xml_report is called with multi_target=True 65 return '%s/%s/%s' % (_MATRIX_REPORT_NAME, shortname_for_multi_target, 66 'sponge_log.log') 67 68 69def _docker_jobspec(name, 70 runtests_args=[], 71 runtests_envs={}, 72 inner_jobs=_DEFAULT_INNER_JOBS, 73 timeout_seconds=None): 74 """Run a single instance of run_tests.py in a docker container""" 75 if not timeout_seconds: 76 timeout_seconds = _DEFAULT_RUNTESTS_TIMEOUT 77 shortname = 'run_tests_%s' % name 78 test_job = jobset.JobSpec(cmdline=[ 79 'python', 'tools/run_tests/run_tests.py', '--use_docker', '-t', '-j', 80 str(inner_jobs), '-x', 81 'run_tests/%s' % _report_filename(name), '--report_suite_name', 82 '%s' % _safe_report_name(name) 83 ] + runtests_args, 84 environ=runtests_envs, 85 shortname=shortname, 86 timeout_seconds=timeout_seconds, 87 logfilename=_matrix_job_logfilename(shortname)) 88 return test_job 89 90 91def _workspace_jobspec(name, 92 runtests_args=[], 93 workspace_name=None, 94 runtests_envs={}, 95 inner_jobs=_DEFAULT_INNER_JOBS, 96 timeout_seconds=None): 97 """Run a single instance of run_tests.py in a separate workspace""" 98 if not workspace_name: 99 workspace_name = 'workspace_%s' % name 100 if not timeout_seconds: 101 timeout_seconds = _DEFAULT_RUNTESTS_TIMEOUT 102 shortname = 'run_tests_%s' % name 103 env = {'WORKSPACE_NAME': workspace_name} 104 env.update(runtests_envs) 105 test_job = jobset.JobSpec(cmdline=[ 106 'bash', 'tools/run_tests/helper_scripts/run_tests_in_workspace.sh', 107 '-t', '-j', 108 str(inner_jobs), '-x', 109 '../run_tests/%s' % _report_filename(name), '--report_suite_name', 110 '%s' % _safe_report_name(name) 111 ] + runtests_args, 112 environ=env, 113 shortname=shortname, 114 timeout_seconds=timeout_seconds, 115 logfilename=_matrix_job_logfilename(shortname)) 116 return test_job 117 118 119def _generate_jobs(languages, 120 configs, 121 platforms, 122 iomgr_platforms=['native'], 123 arch=None, 124 compiler=None, 125 labels=[], 126 extra_args=[], 127 extra_envs={}, 128 inner_jobs=_DEFAULT_INNER_JOBS, 129 timeout_seconds=None): 130 result = [] 131 for language in languages: 132 for platform in platforms: 133 for iomgr_platform in iomgr_platforms: 134 for config in configs: 135 name = '%s_%s_%s_%s' % (language, platform, config, 136 iomgr_platform) 137 runtests_args = [ 138 '-l', language, '-c', config, '--iomgr_platform', 139 iomgr_platform 140 ] 141 if arch or compiler: 142 name += '_%s_%s' % (arch, compiler) 143 runtests_args += [ 144 '--arch', arch, '--compiler', compiler 145 ] 146 if '--build_only' in extra_args: 147 name += '_buildonly' 148 for extra_env in extra_envs: 149 name += '_%s_%s' % (extra_env, extra_envs[extra_env]) 150 151 runtests_args += extra_args 152 if platform == 'linux': 153 job = _docker_jobspec(name=name, 154 runtests_args=runtests_args, 155 runtests_envs=extra_envs, 156 inner_jobs=inner_jobs, 157 timeout_seconds=timeout_seconds) 158 else: 159 job = _workspace_jobspec( 160 name=name, 161 runtests_args=runtests_args, 162 runtests_envs=extra_envs, 163 inner_jobs=inner_jobs, 164 timeout_seconds=timeout_seconds) 165 166 job.labels = [platform, config, language, iomgr_platform 167 ] + labels 168 result.append(job) 169 return result 170 171 172def _create_test_jobs(extra_args=[], inner_jobs=_DEFAULT_INNER_JOBS): 173 test_jobs = [] 174 # sanity tests 175 test_jobs += _generate_jobs(languages=['sanity'], 176 configs=['dbg'], 177 platforms=['linux'], 178 labels=['basictests'], 179 extra_args=extra_args + 180 ['--report_multi_target'], 181 inner_jobs=inner_jobs) 182 183 # supported on linux only 184 test_jobs += _generate_jobs(languages=['php7'], 185 configs=['dbg', 'opt'], 186 platforms=['linux'], 187 labels=['basictests', 'multilang'], 188 extra_args=extra_args + 189 ['--report_multi_target'], 190 inner_jobs=inner_jobs) 191 192 # supported on all platforms. 193 test_jobs += _generate_jobs( 194 languages=['c'], 195 configs=['dbg', 'opt'], 196 platforms=['linux', 'macos', 'windows'], 197 labels=['basictests', 'corelang'], 198 extra_args= 199 extra_args, # don't use multi_target report because C has too many test cases 200 inner_jobs=inner_jobs, 201 timeout_seconds=_CPP_RUNTESTS_TIMEOUT) 202 203 # C# tests on .NET desktop/mono 204 test_jobs += _generate_jobs(languages=['csharp'], 205 configs=['dbg', 'opt'], 206 platforms=['linux', 'macos', 'windows'], 207 labels=['basictests', 'multilang'], 208 extra_args=extra_args + 209 ['--report_multi_target'], 210 inner_jobs=inner_jobs) 211 # C# tests on .NET core 212 test_jobs += _generate_jobs(languages=['csharp'], 213 configs=['dbg', 'opt'], 214 platforms=['linux', 'macos', 'windows'], 215 arch='default', 216 compiler='coreclr', 217 labels=['basictests', 'multilang'], 218 extra_args=extra_args + 219 ['--report_multi_target'], 220 inner_jobs=inner_jobs) 221 222 test_jobs += _generate_jobs(languages=['python'], 223 configs=['opt'], 224 platforms=['linux', 'macos', 'windows'], 225 iomgr_platforms=['native', 'gevent', 'asyncio'], 226 labels=['basictests', 'multilang'], 227 extra_args=extra_args + 228 ['--report_multi_target'], 229 inner_jobs=inner_jobs) 230 231 # supported on linux and mac. 232 test_jobs += _generate_jobs( 233 languages=['c++'], 234 configs=['dbg', 'opt'], 235 platforms=['linux', 'macos'], 236 labels=['basictests', 'corelang'], 237 extra_args= 238 extra_args, # don't use multi_target report because C++ has too many test cases 239 inner_jobs=inner_jobs, 240 timeout_seconds=_CPP_RUNTESTS_TIMEOUT) 241 242 test_jobs += _generate_jobs(languages=['grpc-node', 'ruby', 'php'], 243 configs=['dbg', 'opt'], 244 platforms=['linux', 'macos'], 245 labels=['basictests', 'multilang'], 246 extra_args=extra_args + 247 ['--report_multi_target'], 248 inner_jobs=inner_jobs) 249 250 # supported on mac only. 251 test_jobs += _generate_jobs(languages=['objc'], 252 configs=['opt'], 253 platforms=['macos'], 254 labels=['basictests', 'multilang'], 255 extra_args=extra_args + 256 ['--report_multi_target'], 257 inner_jobs=inner_jobs, 258 timeout_seconds=_OBJC_RUNTESTS_TIMEOUT) 259 260 return test_jobs 261 262 263def _create_portability_test_jobs(extra_args=[], 264 inner_jobs=_DEFAULT_INNER_JOBS): 265 test_jobs = [] 266 # portability C x86 267 test_jobs += _generate_jobs(languages=['c'], 268 configs=['dbg'], 269 platforms=['linux'], 270 arch='x86', 271 compiler='default', 272 labels=['portability', 'corelang'], 273 extra_args=extra_args, 274 inner_jobs=inner_jobs) 275 276 # portability C and C++ on x64 277 for compiler in [ 278 'gcc4.9', 'gcc5.3', 'gcc7.4', 'gcc8.3', 'gcc_musl', 'clang3.5', 279 'clang3.6', 'clang3.7', 'clang7.0' 280 ]: 281 test_jobs += _generate_jobs(languages=['c', 'c++'], 282 configs=['dbg'], 283 platforms=['linux'], 284 arch='x64', 285 compiler=compiler, 286 labels=['portability', 'corelang'], 287 extra_args=extra_args, 288 inner_jobs=inner_jobs, 289 timeout_seconds=_CPP_RUNTESTS_TIMEOUT) 290 291 # portability C on Windows 64-bit (x86 is the default) 292 test_jobs += _generate_jobs(languages=['c'], 293 configs=['dbg'], 294 platforms=['windows'], 295 arch='x64', 296 compiler='default', 297 labels=['portability', 'corelang'], 298 extra_args=extra_args, 299 inner_jobs=inner_jobs) 300 301 # portability C++ on Windows 302 # TODO(jtattermusch): some of the tests are failing, so we force --build_only 303 test_jobs += _generate_jobs(languages=['c++'], 304 configs=['dbg'], 305 platforms=['windows'], 306 arch='default', 307 compiler='default', 308 labels=['portability', 'corelang'], 309 extra_args=extra_args + ['--build_only'], 310 inner_jobs=inner_jobs, 311 timeout_seconds=_CPP_RUNTESTS_TIMEOUT) 312 313 # portability C and C++ on Windows using VS2017 (build only) 314 # TODO(jtattermusch): some of the tests are failing, so we force --build_only 315 test_jobs += _generate_jobs(languages=['c', 'c++'], 316 configs=['dbg'], 317 platforms=['windows'], 318 arch='x64', 319 compiler='cmake_vs2017', 320 labels=['portability', 'corelang'], 321 extra_args=extra_args + ['--build_only'], 322 inner_jobs=inner_jobs, 323 timeout_seconds=_CPP_RUNTESTS_TIMEOUT) 324 325 # C and C++ with the c-ares DNS resolver on Linux 326 test_jobs += _generate_jobs(languages=['c', 'c++'], 327 configs=['dbg'], 328 platforms=['linux'], 329 labels=['portability', 'corelang'], 330 extra_args=extra_args, 331 extra_envs={'GRPC_DNS_RESOLVER': 'ares'}, 332 timeout_seconds=_CPP_RUNTESTS_TIMEOUT) 333 334 # C and C++ with no-exceptions on Linux 335 test_jobs += _generate_jobs(languages=['c', 'c++'], 336 configs=['noexcept'], 337 platforms=['linux'], 338 labels=['portability', 'corelang'], 339 extra_args=extra_args, 340 timeout_seconds=_CPP_RUNTESTS_TIMEOUT) 341 342 # TODO(zyc): Turn on this test after adding c-ares support on windows. 343 # C with the c-ares DNS resolver on Windows 344 # test_jobs += _generate_jobs(languages=['c'], 345 # configs=['dbg'], platforms=['windows'], 346 # labels=['portability', 'corelang'], 347 # extra_args=extra_args, 348 # extra_envs={'GRPC_DNS_RESOLVER': 'ares'}) 349 350 # C and C++ build with cmake on Linux 351 # TODO(jtattermusch): some of the tests are failing, so we force --build_only 352 # to make sure it's buildable at least. 353 test_jobs += _generate_jobs(languages=['c', 'c++'], 354 configs=['dbg'], 355 platforms=['linux'], 356 arch='default', 357 compiler='cmake', 358 labels=['portability', 'corelang'], 359 extra_args=extra_args + ['--build_only'], 360 inner_jobs=inner_jobs) 361 362 test_jobs += _generate_jobs(languages=['python'], 363 configs=['dbg'], 364 platforms=['linux'], 365 arch='default', 366 compiler='python_alpine', 367 labels=['portability', 'multilang'], 368 extra_args=extra_args + 369 ['--report_multi_target'], 370 inner_jobs=inner_jobs) 371 372 # TODO(jtattermusch): a large portion of the libuv tests is failing, 373 # which can end up killing the kokoro job due to gigabytes of error logs 374 # generated. Remove the --build_only flag 375 # once https://github.com/grpc/grpc/issues/17556 is fixed. 376 test_jobs += _generate_jobs(languages=['c'], 377 configs=['dbg'], 378 platforms=['linux'], 379 iomgr_platforms=['uv'], 380 labels=['portability', 'corelang'], 381 extra_args=extra_args + ['--build_only'], 382 inner_jobs=inner_jobs, 383 timeout_seconds=_CPP_RUNTESTS_TIMEOUT) 384 385 return test_jobs 386 387 388def _allowed_labels(): 389 """Returns a list of existing job labels.""" 390 all_labels = set() 391 for job in _create_test_jobs() + _create_portability_test_jobs(): 392 for label in job.labels: 393 all_labels.add(label) 394 return sorted(all_labels) 395 396 397def _runs_per_test_type(arg_str): 398 """Auxiliary function to parse the "runs_per_test" flag.""" 399 try: 400 n = int(arg_str) 401 if n <= 0: raise ValueError 402 return n 403 except: 404 msg = '\'{}\' is not a positive integer'.format(arg_str) 405 raise argparse.ArgumentTypeError(msg) 406 407 408if __name__ == "__main__": 409 argp = argparse.ArgumentParser( 410 description='Run a matrix of run_tests.py tests.') 411 argp.add_argument('-j', 412 '--jobs', 413 default=multiprocessing.cpu_count() / _DEFAULT_INNER_JOBS, 414 type=int, 415 help='Number of concurrent run_tests.py instances.') 416 argp.add_argument('-f', 417 '--filter', 418 choices=_allowed_labels(), 419 nargs='+', 420 default=[], 421 help='Filter targets to run by label with AND semantics.') 422 argp.add_argument('--exclude', 423 choices=_allowed_labels(), 424 nargs='+', 425 default=[], 426 help='Exclude targets with any of given labels.') 427 argp.add_argument('--build_only', 428 default=False, 429 action='store_const', 430 const=True, 431 help='Pass --build_only flag to run_tests.py instances.') 432 argp.add_argument( 433 '--force_default_poller', 434 default=False, 435 action='store_const', 436 const=True, 437 help='Pass --force_default_poller to run_tests.py instances.') 438 argp.add_argument('--dry_run', 439 default=False, 440 action='store_const', 441 const=True, 442 help='Only print what would be run.') 443 argp.add_argument( 444 '--filter_pr_tests', 445 default=False, 446 action='store_const', 447 const=True, 448 help='Filters out tests irrelevant to pull request changes.') 449 argp.add_argument( 450 '--base_branch', 451 default='origin/master', 452 type=str, 453 help='Branch that pull request is requesting to merge into') 454 argp.add_argument('--inner_jobs', 455 default=_DEFAULT_INNER_JOBS, 456 type=int, 457 help='Number of jobs in each run_tests.py instance') 458 argp.add_argument( 459 '-n', 460 '--runs_per_test', 461 default=1, 462 type=_runs_per_test_type, 463 help='How many times to run each tests. >1 runs implies ' + 464 'omitting passing test from the output & reports.') 465 argp.add_argument('--max_time', 466 default=-1, 467 type=int, 468 help='Maximum amount of time to run tests for' + 469 '(other tests will be skipped)') 470 argp.add_argument( 471 '--internal_ci', 472 default=False, 473 action='store_const', 474 const=True, 475 help= 476 '(Deprecated, has no effect) Put reports into subdirectories to improve presentation of ' 477 'results by Kokoro.') 478 argp.add_argument('--bq_result_table', 479 default='', 480 type=str, 481 nargs='?', 482 help='Upload test results to a specified BQ table.') 483 argp.add_argument('--extra_args', 484 default='', 485 type=str, 486 nargs=argparse.REMAINDER, 487 help='Extra test args passed to each sub-script.') 488 args = argp.parse_args() 489 490 extra_args = [] 491 if args.build_only: 492 extra_args.append('--build_only') 493 if args.force_default_poller: 494 extra_args.append('--force_default_poller') 495 if args.runs_per_test > 1: 496 extra_args.append('-n') 497 extra_args.append('%s' % args.runs_per_test) 498 extra_args.append('--quiet_success') 499 if args.max_time > 0: 500 extra_args.extend(('--max_time', '%d' % args.max_time)) 501 if args.bq_result_table: 502 extra_args.append('--bq_result_table') 503 extra_args.append('%s' % args.bq_result_table) 504 extra_args.append('--measure_cpu_costs') 505 if args.extra_args: 506 extra_args.extend(args.extra_args) 507 508 all_jobs = _create_test_jobs(extra_args=extra_args, inner_jobs=args.inner_jobs) + \ 509 _create_portability_test_jobs(extra_args=extra_args, inner_jobs=args.inner_jobs) 510 511 jobs = [] 512 for job in all_jobs: 513 if not args.filter or all( 514 filter in job.labels for filter in args.filter): 515 if not any(exclude_label in job.labels 516 for exclude_label in args.exclude): 517 jobs.append(job) 518 519 if not jobs: 520 jobset.message('FAILED', 521 'No test suites match given criteria.', 522 do_newline=True) 523 sys.exit(1) 524 525 print('IMPORTANT: The changes you are testing need to be locally committed') 526 print('because only the committed changes in the current branch will be') 527 print('copied to the docker environment or into subworkspaces.') 528 529 skipped_jobs = [] 530 531 if args.filter_pr_tests: 532 print('Looking for irrelevant tests to skip...') 533 relevant_jobs = filter_tests(jobs, args.base_branch) 534 if len(relevant_jobs) == len(jobs): 535 print('No tests will be skipped.') 536 else: 537 print('These tests will be skipped:') 538 skipped_jobs = list(set(jobs) - set(relevant_jobs)) 539 # Sort by shortnames to make printing of skipped tests consistent 540 skipped_jobs.sort(key=lambda job: job.shortname) 541 for job in list(skipped_jobs): 542 print(' %s' % job.shortname) 543 jobs = relevant_jobs 544 545 print('Will run these tests:') 546 for job in jobs: 547 print(' %s: "%s"' % (job.shortname, ' '.join(job.cmdline))) 548 print('') 549 550 if args.dry_run: 551 print('--dry_run was used, exiting') 552 sys.exit(1) 553 554 jobset.message('START', 'Running test matrix.', do_newline=True) 555 num_failures, resultset = jobset.run(jobs, 556 newline_on_success=True, 557 travis=True, 558 maxjobs=args.jobs) 559 # Merge skipped tests into results to show skipped tests on report.xml 560 if skipped_jobs: 561 ignored_num_skipped_failures, skipped_results = jobset.run( 562 skipped_jobs, skip_jobs=True) 563 resultset.update(skipped_results) 564 report_utils.render_junit_xml_report(resultset, 565 _report_filename(_MATRIX_REPORT_NAME), 566 suite_name=_MATRIX_REPORT_NAME, 567 multi_target=True) 568 569 if num_failures == 0: 570 jobset.message('SUCCESS', 571 'All run_tests.py instances finished successfully.', 572 do_newline=True) 573 else: 574 jobset.message('FAILED', 575 'Some run_tests.py instances have failed.', 576 do_newline=True) 577 sys.exit(1) 578