1#!/usr/bin/env python 2# Copyright 2015 gRPC authors. 3# 4# Licensed under the Apache License, Version 2.0 (the "License"); 5# you may not use this file except in compliance with the License. 6# You may obtain a copy of the License at 7# 8# http://www.apache.org/licenses/LICENSE-2.0 9# 10# Unless required by applicable law or agreed to in writing, software 11# distributed under the License is distributed on an "AS IS" BASIS, 12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13# See the License for the specific language governing permissions and 14# limitations under the License. 15"""Run test matrix.""" 16 17from __future__ import print_function 18 19import argparse 20import multiprocessing 21import os 22import sys 23 24import python_utils.jobset as jobset 25import python_utils.report_utils as report_utils 26from python_utils.filter_pull_request_tests import filter_tests 27 28_ROOT = os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]), '../..')) 29os.chdir(_ROOT) 30 31_DEFAULT_RUNTESTS_TIMEOUT = 1 * 60 * 60 32 33# C/C++ tests can take long time 34_CPP_RUNTESTS_TIMEOUT = 4 * 60 * 60 35 36# Set timeout high for ObjC for Cocoapods to install pods 37_OBJC_RUNTESTS_TIMEOUT = 90 * 60 38 39# Number of jobs assigned to each run_tests.py instance 40_DEFAULT_INNER_JOBS = 2 41 42# Name of the top-level umbrella report that includes all the run_tests.py invocations 43# Note that the starting letter 't' matters so that the targets are listed AFTER 44# the per-test breakdown items that start with 'run_tests/' (it is more readable that way) 45_MATRIX_REPORT_NAME = 'toplevel_run_tests_invocations' 46 47 48def _safe_report_name(name): 49 """Reports with '+' in target name won't show correctly in ResultStore""" 50 return name.replace('+', 'p') 51 52 53def _report_filename(name): 54 """Generates report file name with directory structure that leads to better presentation by internal CI""" 55 # 'sponge_log.xml' suffix must be there for results to get recognized by kokoro. 56 return '%s/%s' % (_safe_report_name(name), 'sponge_log.xml') 57 58 59def _matrix_job_logfilename(shortname_for_multi_target): 60 """Generate location for log file that will match the sponge_log.xml from the top-level matrix report.""" 61 # 'sponge_log.log' suffix must be there for log to get recognized as "target log" 62 # for the corresponding 'sponge_log.xml' report. 63 # the shortname_for_multi_target component must be set to match the sponge_log.xml location 64 # because the top-level render_junit_xml_report is called with multi_target=True 65 return '%s/%s/%s' % (_MATRIX_REPORT_NAME, shortname_for_multi_target, 66 'sponge_log.log') 67 68 69def _docker_jobspec(name, 70 runtests_args=[], 71 runtests_envs={}, 72 inner_jobs=_DEFAULT_INNER_JOBS, 73 timeout_seconds=None): 74 """Run a single instance of run_tests.py in a docker container""" 75 if not timeout_seconds: 76 timeout_seconds = _DEFAULT_RUNTESTS_TIMEOUT 77 shortname = 'run_tests_%s' % name 78 test_job = jobset.JobSpec(cmdline=[ 79 'python', 'tools/run_tests/run_tests.py', '--use_docker', '-t', '-j', 80 str(inner_jobs), '-x', 81 'run_tests/%s' % _report_filename(name), '--report_suite_name', 82 '%s' % _safe_report_name(name) 83 ] + runtests_args, 84 environ=runtests_envs, 85 shortname=shortname, 86 timeout_seconds=timeout_seconds, 87 logfilename=_matrix_job_logfilename(shortname)) 88 return test_job 89 90 91def _workspace_jobspec(name, 92 runtests_args=[], 93 workspace_name=None, 94 runtests_envs={}, 95 inner_jobs=_DEFAULT_INNER_JOBS, 96 timeout_seconds=None): 97 """Run a single instance of run_tests.py in a separate workspace""" 98 if not workspace_name: 99 workspace_name = 'workspace_%s' % name 100 if not timeout_seconds: 101 timeout_seconds = _DEFAULT_RUNTESTS_TIMEOUT 102 shortname = 'run_tests_%s' % name 103 env = {'WORKSPACE_NAME': workspace_name} 104 env.update(runtests_envs) 105 test_job = jobset.JobSpec(cmdline=[ 106 'bash', 'tools/run_tests/helper_scripts/run_tests_in_workspace.sh', 107 '-t', '-j', 108 str(inner_jobs), '-x', 109 '../run_tests/%s' % _report_filename(name), '--report_suite_name', 110 '%s' % _safe_report_name(name) 111 ] + runtests_args, 112 environ=env, 113 shortname=shortname, 114 timeout_seconds=timeout_seconds, 115 logfilename=_matrix_job_logfilename(shortname)) 116 return test_job 117 118 119def _generate_jobs(languages, 120 configs, 121 platforms, 122 iomgr_platforms=['native'], 123 arch=None, 124 compiler=None, 125 labels=[], 126 extra_args=[], 127 extra_envs={}, 128 inner_jobs=_DEFAULT_INNER_JOBS, 129 timeout_seconds=None): 130 result = [] 131 for language in languages: 132 for platform in platforms: 133 for iomgr_platform in iomgr_platforms: 134 for config in configs: 135 name = '%s_%s_%s_%s' % (language, platform, config, 136 iomgr_platform) 137 runtests_args = [ 138 '-l', language, '-c', config, '--iomgr_platform', 139 iomgr_platform 140 ] 141 if arch or compiler: 142 name += '_%s_%s' % (arch, compiler) 143 runtests_args += [ 144 '--arch', arch, '--compiler', compiler 145 ] 146 if '--build_only' in extra_args: 147 name += '_buildonly' 148 for extra_env in extra_envs: 149 name += '_%s_%s' % (extra_env, extra_envs[extra_env]) 150 151 runtests_args += extra_args 152 if platform == 'linux': 153 job = _docker_jobspec(name=name, 154 runtests_args=runtests_args, 155 runtests_envs=extra_envs, 156 inner_jobs=inner_jobs, 157 timeout_seconds=timeout_seconds) 158 else: 159 job = _workspace_jobspec( 160 name=name, 161 runtests_args=runtests_args, 162 runtests_envs=extra_envs, 163 inner_jobs=inner_jobs, 164 timeout_seconds=timeout_seconds) 165 166 job.labels = [platform, config, language, iomgr_platform 167 ] + labels 168 result.append(job) 169 return result 170 171 172def _create_test_jobs(extra_args=[], inner_jobs=_DEFAULT_INNER_JOBS): 173 test_jobs = [] 174 # sanity tests 175 test_jobs += _generate_jobs(languages=['sanity'], 176 configs=['dbg'], 177 platforms=['linux'], 178 labels=['basictests'], 179 extra_args=extra_args + 180 ['--report_multi_target'], 181 inner_jobs=inner_jobs) 182 183 # supported on all platforms. 184 test_jobs += _generate_jobs( 185 languages=['c'], 186 configs=['dbg', 'opt'], 187 platforms=['linux', 'macos', 'windows'], 188 labels=['basictests', 'corelang'], 189 extra_args= 190 extra_args, # don't use multi_target report because C has too many test cases 191 inner_jobs=inner_jobs, 192 timeout_seconds=_CPP_RUNTESTS_TIMEOUT) 193 194 # C# tests on .NET desktop/mono 195 test_jobs += _generate_jobs(languages=['csharp'], 196 configs=['dbg', 'opt'], 197 platforms=['linux', 'macos', 'windows'], 198 labels=['basictests', 'multilang'], 199 extra_args=extra_args + 200 ['--report_multi_target'], 201 inner_jobs=inner_jobs) 202 # C# tests on .NET core 203 test_jobs += _generate_jobs(languages=['csharp'], 204 configs=['dbg', 'opt'], 205 platforms=['linux', 'macos', 'windows'], 206 arch='default', 207 compiler='coreclr', 208 labels=['basictests', 'multilang'], 209 extra_args=extra_args + 210 ['--report_multi_target'], 211 inner_jobs=inner_jobs) 212 213 test_jobs += _generate_jobs(languages=['python'], 214 configs=['opt'], 215 platforms=['linux', 'macos', 'windows'], 216 iomgr_platforms=['native', 'gevent', 'asyncio'], 217 labels=['basictests', 'multilang'], 218 extra_args=extra_args + 219 ['--report_multi_target'], 220 inner_jobs=inner_jobs) 221 222 # supported on linux and mac. 223 test_jobs += _generate_jobs( 224 languages=['c++'], 225 configs=['dbg', 'opt'], 226 platforms=['linux', 'macos'], 227 labels=['basictests', 'corelang'], 228 extra_args= 229 extra_args, # don't use multi_target report because C++ has too many test cases 230 inner_jobs=inner_jobs, 231 timeout_seconds=_CPP_RUNTESTS_TIMEOUT) 232 233 test_jobs += _generate_jobs(languages=['grpc-node', 'ruby', 'php7'], 234 configs=['dbg', 'opt'], 235 platforms=['linux', 'macos'], 236 labels=['basictests', 'multilang'], 237 extra_args=extra_args + 238 ['--report_multi_target'], 239 inner_jobs=inner_jobs) 240 241 # supported on mac only. 242 test_jobs += _generate_jobs(languages=['objc'], 243 configs=['opt'], 244 platforms=['macos'], 245 labels=['basictests', 'multilang'], 246 extra_args=extra_args + 247 ['--report_multi_target'], 248 inner_jobs=inner_jobs, 249 timeout_seconds=_OBJC_RUNTESTS_TIMEOUT) 250 251 return test_jobs 252 253 254def _create_portability_test_jobs(extra_args=[], 255 inner_jobs=_DEFAULT_INNER_JOBS): 256 test_jobs = [] 257 # portability C x86 258 test_jobs += _generate_jobs(languages=['c'], 259 configs=['dbg'], 260 platforms=['linux'], 261 arch='x86', 262 compiler='default', 263 labels=['portability', 'corelang'], 264 extra_args=extra_args, 265 inner_jobs=inner_jobs) 266 267 # portability C and C++ on x64 268 for compiler in [ 269 'gcc4.9', 'gcc5.3', 'gcc7.4', 'gcc8.3', 'gcc_musl', 'clang4.0', 270 'clang5.0' 271 ]: 272 test_jobs += _generate_jobs(languages=['c', 'c++'], 273 configs=['dbg'], 274 platforms=['linux'], 275 arch='x64', 276 compiler=compiler, 277 labels=['portability', 'corelang'], 278 extra_args=extra_args, 279 inner_jobs=inner_jobs, 280 timeout_seconds=_CPP_RUNTESTS_TIMEOUT) 281 282 # portability C on Windows 64-bit (x86 is the default) 283 test_jobs += _generate_jobs(languages=['c'], 284 configs=['dbg'], 285 platforms=['windows'], 286 arch='x64', 287 compiler='default', 288 labels=['portability', 'corelang'], 289 extra_args=extra_args, 290 inner_jobs=inner_jobs) 291 292 # portability C++ on Windows 293 # TODO(jtattermusch): some of the tests are failing, so we force --build_only 294 test_jobs += _generate_jobs(languages=['c++'], 295 configs=['dbg'], 296 platforms=['windows'], 297 arch='default', 298 compiler='default', 299 labels=['portability', 'corelang'], 300 extra_args=extra_args + ['--build_only'], 301 inner_jobs=inner_jobs, 302 timeout_seconds=_CPP_RUNTESTS_TIMEOUT) 303 304 # portability C and C++ on Windows using VS2017 (build only) 305 # TODO(jtattermusch): some of the tests are failing, so we force --build_only 306 test_jobs += _generate_jobs(languages=['c', 'c++'], 307 configs=['dbg'], 308 platforms=['windows'], 309 arch='x64', 310 compiler='cmake_vs2017', 311 labels=['portability', 'corelang'], 312 extra_args=extra_args + ['--build_only'], 313 inner_jobs=inner_jobs, 314 timeout_seconds=_CPP_RUNTESTS_TIMEOUT) 315 316 # C and C++ with the c-ares DNS resolver on Linux 317 test_jobs += _generate_jobs(languages=['c', 'c++'], 318 configs=['dbg'], 319 platforms=['linux'], 320 labels=['portability', 'corelang'], 321 extra_args=extra_args, 322 extra_envs={'GRPC_DNS_RESOLVER': 'ares'}, 323 timeout_seconds=_CPP_RUNTESTS_TIMEOUT) 324 325 # C and C++ with no-exceptions on Linux 326 test_jobs += _generate_jobs(languages=['c', 'c++'], 327 configs=['noexcept'], 328 platforms=['linux'], 329 labels=['portability', 'corelang'], 330 extra_args=extra_args, 331 timeout_seconds=_CPP_RUNTESTS_TIMEOUT) 332 333 test_jobs += _generate_jobs(languages=['python'], 334 configs=['dbg'], 335 platforms=['linux'], 336 arch='default', 337 compiler='python_alpine', 338 labels=['portability', 'multilang'], 339 extra_args=extra_args + 340 ['--report_multi_target'], 341 inner_jobs=inner_jobs) 342 343 # TODO(jtattermusch): a large portion of the libuv tests is failing, 344 # which can end up killing the kokoro job due to gigabytes of error logs 345 # generated. Remove the --build_only flag 346 # once https://github.com/grpc/grpc/issues/17556 is fixed. 347 test_jobs += _generate_jobs(languages=['c'], 348 configs=['dbg'], 349 platforms=['linux'], 350 iomgr_platforms=['uv'], 351 labels=['portability', 'corelang'], 352 extra_args=extra_args + ['--build_only'], 353 inner_jobs=inner_jobs, 354 timeout_seconds=_CPP_RUNTESTS_TIMEOUT) 355 356 return test_jobs 357 358 359def _allowed_labels(): 360 """Returns a list of existing job labels.""" 361 all_labels = set() 362 for job in _create_test_jobs() + _create_portability_test_jobs(): 363 for label in job.labels: 364 all_labels.add(label) 365 return sorted(all_labels) 366 367 368def _runs_per_test_type(arg_str): 369 """Auxiliary function to parse the "runs_per_test" flag.""" 370 try: 371 n = int(arg_str) 372 if n <= 0: 373 raise ValueError 374 return n 375 except: 376 msg = '\'{}\' is not a positive integer'.format(arg_str) 377 raise argparse.ArgumentTypeError(msg) 378 379 380if __name__ == "__main__": 381 argp = argparse.ArgumentParser( 382 description='Run a matrix of run_tests.py tests.') 383 argp.add_argument('-j', 384 '--jobs', 385 default=multiprocessing.cpu_count() / _DEFAULT_INNER_JOBS, 386 type=int, 387 help='Number of concurrent run_tests.py instances.') 388 argp.add_argument('-f', 389 '--filter', 390 choices=_allowed_labels(), 391 nargs='+', 392 default=[], 393 help='Filter targets to run by label with AND semantics.') 394 argp.add_argument('--exclude', 395 choices=_allowed_labels(), 396 nargs='+', 397 default=[], 398 help='Exclude targets with any of given labels.') 399 argp.add_argument('--build_only', 400 default=False, 401 action='store_const', 402 const=True, 403 help='Pass --build_only flag to run_tests.py instances.') 404 argp.add_argument( 405 '--force_default_poller', 406 default=False, 407 action='store_const', 408 const=True, 409 help='Pass --force_default_poller to run_tests.py instances.') 410 argp.add_argument('--dry_run', 411 default=False, 412 action='store_const', 413 const=True, 414 help='Only print what would be run.') 415 argp.add_argument( 416 '--filter_pr_tests', 417 default=False, 418 action='store_const', 419 const=True, 420 help='Filters out tests irrelevant to pull request changes.') 421 argp.add_argument( 422 '--base_branch', 423 default='origin/master', 424 type=str, 425 help='Branch that pull request is requesting to merge into') 426 argp.add_argument('--inner_jobs', 427 default=_DEFAULT_INNER_JOBS, 428 type=int, 429 help='Number of jobs in each run_tests.py instance') 430 argp.add_argument( 431 '-n', 432 '--runs_per_test', 433 default=1, 434 type=_runs_per_test_type, 435 help='How many times to run each tests. >1 runs implies ' + 436 'omitting passing test from the output & reports.') 437 argp.add_argument('--max_time', 438 default=-1, 439 type=int, 440 help='Maximum amount of time to run tests for' + 441 '(other tests will be skipped)') 442 argp.add_argument( 443 '--internal_ci', 444 default=False, 445 action='store_const', 446 const=True, 447 help= 448 '(Deprecated, has no effect) Put reports into subdirectories to improve presentation of ' 449 'results by Kokoro.') 450 argp.add_argument('--bq_result_table', 451 default='', 452 type=str, 453 nargs='?', 454 help='Upload test results to a specified BQ table.') 455 argp.add_argument('--extra_args', 456 default='', 457 type=str, 458 nargs=argparse.REMAINDER, 459 help='Extra test args passed to each sub-script.') 460 args = argp.parse_args() 461 462 extra_args = [] 463 if args.build_only: 464 extra_args.append('--build_only') 465 if args.force_default_poller: 466 extra_args.append('--force_default_poller') 467 if args.runs_per_test > 1: 468 extra_args.append('-n') 469 extra_args.append('%s' % args.runs_per_test) 470 extra_args.append('--quiet_success') 471 if args.max_time > 0: 472 extra_args.extend(('--max_time', '%d' % args.max_time)) 473 if args.bq_result_table: 474 extra_args.append('--bq_result_table') 475 extra_args.append('%s' % args.bq_result_table) 476 extra_args.append('--measure_cpu_costs') 477 if args.extra_args: 478 extra_args.extend(args.extra_args) 479 480 all_jobs = _create_test_jobs(extra_args=extra_args, inner_jobs=args.inner_jobs) + \ 481 _create_portability_test_jobs(extra_args=extra_args, inner_jobs=args.inner_jobs) 482 483 jobs = [] 484 for job in all_jobs: 485 if not args.filter or all( 486 filter in job.labels for filter in args.filter): 487 if not any(exclude_label in job.labels 488 for exclude_label in args.exclude): 489 jobs.append(job) 490 491 if not jobs: 492 jobset.message('FAILED', 493 'No test suites match given criteria.', 494 do_newline=True) 495 sys.exit(1) 496 497 print('IMPORTANT: The changes you are testing need to be locally committed') 498 print('because only the committed changes in the current branch will be') 499 print('copied to the docker environment or into subworkspaces.') 500 501 skipped_jobs = [] 502 503 if args.filter_pr_tests: 504 print('Looking for irrelevant tests to skip...') 505 relevant_jobs = filter_tests(jobs, args.base_branch) 506 if len(relevant_jobs) == len(jobs): 507 print('No tests will be skipped.') 508 else: 509 print('These tests will be skipped:') 510 skipped_jobs = list(set(jobs) - set(relevant_jobs)) 511 # Sort by shortnames to make printing of skipped tests consistent 512 skipped_jobs.sort(key=lambda job: job.shortname) 513 for job in list(skipped_jobs): 514 print(' %s' % job.shortname) 515 jobs = relevant_jobs 516 517 print('Will run these tests:') 518 for job in jobs: 519 print(' %s: "%s"' % (job.shortname, ' '.join(job.cmdline))) 520 print('') 521 522 if args.dry_run: 523 print('--dry_run was used, exiting') 524 sys.exit(1) 525 526 jobset.message('START', 'Running test matrix.', do_newline=True) 527 num_failures, resultset = jobset.run(jobs, 528 newline_on_success=True, 529 travis=True, 530 maxjobs=args.jobs) 531 # Merge skipped tests into results to show skipped tests on report.xml 532 if skipped_jobs: 533 ignored_num_skipped_failures, skipped_results = jobset.run( 534 skipped_jobs, skip_jobs=True) 535 resultset.update(skipped_results) 536 report_utils.render_junit_xml_report(resultset, 537 _report_filename(_MATRIX_REPORT_NAME), 538 suite_name=_MATRIX_REPORT_NAME, 539 multi_target=True) 540 541 if num_failures == 0: 542 jobset.message('SUCCESS', 543 'All run_tests.py instances finished successfully.', 544 do_newline=True) 545 else: 546 jobset.message('FAILED', 547 'Some run_tests.py instances have failed.', 548 do_newline=True) 549 sys.exit(1) 550