1# Copyright 2016 gRPC authors. 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); 4# you may not use this file except in compliance with the License. 5# You may obtain a copy of the License at 6# 7# http://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12# See the License for the specific language governing permissions and 13# limitations under the License. 14 15# performance scenario configuration for various languages 16 17import math 18 19WARMUP_SECONDS = 5 20JAVA_WARMUP_SECONDS = 15 # Java needs more warmup time for JIT to kick in. 21BENCHMARK_SECONDS = 30 22 23SMOKETEST = 'smoketest' 24SCALABLE = 'scalable' 25INPROC = 'inproc' 26SWEEP = 'sweep' 27DEFAULT_CATEGORIES = [SCALABLE, SMOKETEST] 28 29SECURE_SECARGS = { 30 'use_test_ca': True, 31 'server_host_override': 'foo.test.google.fr' 32} 33 34HISTOGRAM_PARAMS = { 35 'resolution': 0.01, 36 'max_possible': 60e9, 37} 38 39# target number of RPCs outstanding on across all client channels in 40# non-ping-pong tests (since we can only specify per-channel numbers, the 41# actual target will be slightly higher) 42OUTSTANDING_REQUESTS = {'async': 6400, 'async-limited': 800, 'sync': 1000} 43 44# wide is the number of client channels in multi-channel tests (1 otherwise) 45WIDE = 64 46 47 48def _get_secargs(is_secure): 49 if is_secure: 50 return SECURE_SECARGS 51 else: 52 return None 53 54 55def remove_nonproto_fields(scenario): 56 """Remove special-purpose that contains some extra info about the scenario 57 but don't belong to the ScenarioConfig protobuf message""" 58 scenario.pop('CATEGORIES', None) 59 scenario.pop('CLIENT_LANGUAGE', None) 60 scenario.pop('SERVER_LANGUAGE', None) 61 scenario.pop('EXCLUDED_POLL_ENGINES', None) 62 return scenario 63 64 65def geometric_progression(start, stop, step): 66 n = start 67 while n < stop: 68 yield int(round(n)) 69 n *= step 70 71 72def _payload_type(use_generic_payload, req_size, resp_size): 73 r = {} 74 sizes = { 75 'req_size': req_size, 76 'resp_size': resp_size, 77 } 78 if use_generic_payload: 79 r['bytebuf_params'] = sizes 80 else: 81 r['simple_params'] = sizes 82 return r 83 84 85def _load_params(offered_load): 86 r = {} 87 if offered_load is None: 88 r['closed_loop'] = {} 89 else: 90 load = {} 91 load['offered_load'] = offered_load 92 r['poisson'] = load 93 return r 94 95 96def _add_channel_arg(config, key, value): 97 if 'channel_args' in config: 98 channel_args = config['channel_args'] 99 else: 100 channel_args = [] 101 config['channel_args'] = channel_args 102 arg = {'name': key} 103 if isinstance(value, int): 104 arg['int_value'] = value 105 else: 106 arg['str_value'] = value 107 channel_args.append(arg) 108 109 110def _ping_pong_scenario(name, 111 rpc_type, 112 client_type, 113 server_type, 114 secure=True, 115 use_generic_payload=False, 116 req_size=0, 117 resp_size=0, 118 unconstrained_client=None, 119 client_language=None, 120 server_language=None, 121 async_server_threads=0, 122 server_threads_per_cq=0, 123 client_threads_per_cq=0, 124 warmup_seconds=WARMUP_SECONDS, 125 categories=DEFAULT_CATEGORIES, 126 channels=None, 127 outstanding=None, 128 num_clients=None, 129 resource_quota_size=None, 130 messages_per_stream=None, 131 excluded_poll_engines=[], 132 minimal_stack=False, 133 offered_load=None): 134 """Creates a basic ping pong scenario.""" 135 scenario = { 136 'name': name, 137 'num_servers': 1, 138 'num_clients': 1, 139 'client_config': { 140 'client_type': client_type, 141 'security_params': _get_secargs(secure), 142 'outstanding_rpcs_per_channel': 1, 143 'client_channels': 1, 144 'async_client_threads': 1, 145 'threads_per_cq': client_threads_per_cq, 146 'rpc_type': rpc_type, 147 'histogram_params': HISTOGRAM_PARAMS, 148 'channel_args': [], 149 }, 150 'server_config': { 151 'server_type': server_type, 152 'security_params': _get_secargs(secure), 153 'async_server_threads': async_server_threads, 154 'threads_per_cq': server_threads_per_cq, 155 'channel_args': [], 156 }, 157 'warmup_seconds': warmup_seconds, 158 'benchmark_seconds': BENCHMARK_SECONDS 159 } 160 if resource_quota_size: 161 scenario['server_config']['resource_quota_size'] = resource_quota_size 162 if use_generic_payload: 163 if server_type != 'ASYNC_GENERIC_SERVER': 164 raise Exception('Use ASYNC_GENERIC_SERVER for generic payload.') 165 scenario['server_config']['payload_config'] = _payload_type( 166 use_generic_payload, req_size, resp_size) 167 168 scenario['client_config']['payload_config'] = _payload_type( 169 use_generic_payload, req_size, resp_size) 170 171 # Optimization target of 'throughput' does not work well with epoll1 polling 172 # engine. Use the default value of 'blend' 173 optimization_target = 'throughput' 174 175 if unconstrained_client: 176 outstanding_calls = outstanding if outstanding is not None else OUTSTANDING_REQUESTS[ 177 unconstrained_client] 178 # clamp buffer usage to something reasonable (16 gig for now) 179 MAX_MEMORY_USE = 16 * 1024 * 1024 * 1024 180 if outstanding_calls * max(req_size, resp_size) > MAX_MEMORY_USE: 181 outstanding_calls = max(1, 182 MAX_MEMORY_USE / max(req_size, resp_size)) 183 wide = channels if channels is not None else WIDE 184 deep = int(math.ceil(1.0 * outstanding_calls / wide)) 185 186 scenario[ 187 'num_clients'] = num_clients if num_clients is not None else 0 # use as many clients as available. 188 scenario['client_config']['outstanding_rpcs_per_channel'] = deep 189 scenario['client_config']['client_channels'] = wide 190 scenario['client_config']['async_client_threads'] = 0 191 if offered_load is not None: 192 optimization_target = 'latency' 193 else: 194 scenario['client_config']['outstanding_rpcs_per_channel'] = 1 195 scenario['client_config']['client_channels'] = 1 196 scenario['client_config']['async_client_threads'] = 1 197 optimization_target = 'latency' 198 199 scenario['client_config']['load_params'] = _load_params(offered_load) 200 201 optimization_channel_arg = { 202 'name': 'grpc.optimization_target', 203 'str_value': optimization_target 204 } 205 scenario['client_config']['channel_args'].append(optimization_channel_arg) 206 scenario['server_config']['channel_args'].append(optimization_channel_arg) 207 208 if minimal_stack: 209 _add_channel_arg(scenario['client_config'], 'grpc.minimal_stack', 1) 210 _add_channel_arg(scenario['server_config'], 'grpc.minimal_stack', 1) 211 212 if messages_per_stream: 213 scenario['client_config']['messages_per_stream'] = messages_per_stream 214 if client_language: 215 # the CLIENT_LANGUAGE field is recognized by run_performance_tests.py 216 scenario['CLIENT_LANGUAGE'] = client_language 217 if server_language: 218 # the SERVER_LANGUAGE field is recognized by run_performance_tests.py 219 scenario['SERVER_LANGUAGE'] = server_language 220 if categories: 221 scenario['CATEGORIES'] = categories 222 if len(excluded_poll_engines): 223 # The polling engines for which this scenario is excluded 224 scenario['EXCLUDED_POLL_ENGINES'] = excluded_poll_engines 225 return scenario 226 227 228class CXXLanguage: 229 230 def __init__(self): 231 self.safename = 'cxx' 232 233 def worker_cmdline(self): 234 return ['bins/opt/qps_worker'] 235 236 def worker_port_offset(self): 237 return 0 238 239 def scenarios(self): 240 # TODO(ctiller): add 70% load latency test 241 yield _ping_pong_scenario( 242 'cpp_protobuf_async_unary_1channel_100rpcs_1MB', 243 rpc_type='UNARY', 244 client_type='ASYNC_CLIENT', 245 server_type='ASYNC_SERVER', 246 req_size=1024 * 1024, 247 resp_size=1024 * 1024, 248 unconstrained_client='async', 249 outstanding=100, 250 channels=1, 251 num_clients=1, 252 secure=False, 253 categories=[SMOKETEST] + [INPROC] + [SCALABLE]) 254 255 yield _ping_pong_scenario( 256 'cpp_protobuf_async_streaming_from_client_1channel_1MB', 257 rpc_type='STREAMING_FROM_CLIENT', 258 client_type='ASYNC_CLIENT', 259 server_type='ASYNC_SERVER', 260 req_size=1024 * 1024, 261 resp_size=1024 * 1024, 262 unconstrained_client='async', 263 outstanding=1, 264 channels=1, 265 num_clients=1, 266 secure=False, 267 categories=[SMOKETEST] + [INPROC] + [SCALABLE]) 268 269 yield _ping_pong_scenario( 270 'cpp_protobuf_async_unary_75Kqps_600channel_60Krpcs_300Breq_50Bresp', 271 rpc_type='UNARY', 272 client_type='ASYNC_CLIENT', 273 server_type='ASYNC_SERVER', 274 req_size=300, 275 resp_size=50, 276 unconstrained_client='async', 277 outstanding=30000, 278 channels=300, 279 offered_load=37500, 280 secure=False, 281 async_server_threads=16, 282 server_threads_per_cq=1, 283 categories=[SMOKETEST] + [SCALABLE]) 284 285 for secure in [True, False]: 286 secstr = 'secure' if secure else 'insecure' 287 smoketest_categories = ([SMOKETEST] 288 if secure else [INPROC]) + [SCALABLE] 289 290 yield _ping_pong_scenario( 291 'cpp_generic_async_streaming_ping_pong_%s' % secstr, 292 rpc_type='STREAMING', 293 client_type='ASYNC_CLIENT', 294 server_type='ASYNC_GENERIC_SERVER', 295 use_generic_payload=True, 296 async_server_threads=1, 297 secure=secure, 298 categories=smoketest_categories) 299 300 yield _ping_pong_scenario( 301 'cpp_generic_async_streaming_qps_unconstrained_%s' % secstr, 302 rpc_type='STREAMING', 303 client_type='ASYNC_CLIENT', 304 server_type='ASYNC_GENERIC_SERVER', 305 unconstrained_client='async', 306 use_generic_payload=True, 307 secure=secure, 308 minimal_stack=not secure, 309 categories=smoketest_categories + [SCALABLE]) 310 311 for mps in geometric_progression(1, 20, 10): 312 yield _ping_pong_scenario( 313 'cpp_generic_async_streaming_qps_unconstrained_%smps_%s' % 314 (mps, secstr), 315 rpc_type='STREAMING', 316 client_type='ASYNC_CLIENT', 317 server_type='ASYNC_GENERIC_SERVER', 318 unconstrained_client='async', 319 use_generic_payload=True, 320 secure=secure, 321 messages_per_stream=mps, 322 minimal_stack=not secure, 323 categories=smoketest_categories + [SCALABLE]) 324 325 for mps in geometric_progression(1, 200, math.sqrt(10)): 326 yield _ping_pong_scenario( 327 'cpp_generic_async_streaming_qps_unconstrained_%smps_%s' % 328 (mps, secstr), 329 rpc_type='STREAMING', 330 client_type='ASYNC_CLIENT', 331 server_type='ASYNC_GENERIC_SERVER', 332 unconstrained_client='async', 333 use_generic_payload=True, 334 secure=secure, 335 messages_per_stream=mps, 336 minimal_stack=not secure, 337 categories=[SWEEP]) 338 339 yield _ping_pong_scenario( 340 'cpp_generic_async_streaming_qps_1channel_1MBmsg_%s' % secstr, 341 rpc_type='STREAMING', 342 req_size=1024 * 1024, 343 resp_size=1024 * 1024, 344 client_type='ASYNC_CLIENT', 345 server_type='ASYNC_GENERIC_SERVER', 346 unconstrained_client='async', 347 use_generic_payload=True, 348 secure=secure, 349 minimal_stack=not secure, 350 categories=smoketest_categories + [SCALABLE], 351 channels=1, 352 outstanding=100) 353 354 yield _ping_pong_scenario( 355 'cpp_generic_async_streaming_qps_unconstrained_64KBmsg_%s' % 356 secstr, 357 rpc_type='STREAMING', 358 req_size=64 * 1024, 359 resp_size=64 * 1024, 360 client_type='ASYNC_CLIENT', 361 server_type='ASYNC_GENERIC_SERVER', 362 unconstrained_client='async', 363 use_generic_payload=True, 364 secure=secure, 365 minimal_stack=not secure, 366 categories=smoketest_categories + [SCALABLE]) 367 368 yield _ping_pong_scenario( 369 'cpp_generic_async_streaming_qps_unconstrained_1cq_%s' % secstr, 370 rpc_type='STREAMING', 371 client_type='ASYNC_CLIENT', 372 server_type='ASYNC_GENERIC_SERVER', 373 unconstrained_client='async-limited', 374 use_generic_payload=True, 375 secure=secure, 376 client_threads_per_cq=1000000, 377 server_threads_per_cq=1000000, 378 categories=smoketest_categories + [SCALABLE]) 379 380 yield _ping_pong_scenario( 381 'cpp_generic_async_streaming_qps_unconstrained_2waysharedcq_%s' 382 % secstr, 383 rpc_type='STREAMING', 384 client_type='ASYNC_CLIENT', 385 server_type='ASYNC_GENERIC_SERVER', 386 unconstrained_client='async', 387 use_generic_payload=True, 388 secure=secure, 389 client_threads_per_cq=2, 390 server_threads_per_cq=2, 391 categories=smoketest_categories + [SCALABLE]) 392 393 yield _ping_pong_scenario( 394 'cpp_protobuf_async_streaming_qps_unconstrained_1cq_%s' % 395 secstr, 396 rpc_type='STREAMING', 397 client_type='ASYNC_CLIENT', 398 server_type='ASYNC_SERVER', 399 unconstrained_client='async-limited', 400 secure=secure, 401 client_threads_per_cq=1000000, 402 server_threads_per_cq=1000000, 403 categories=smoketest_categories + [SCALABLE]) 404 405 yield _ping_pong_scenario( 406 'cpp_protobuf_async_streaming_qps_unconstrained_2waysharedcq_%s' 407 % secstr, 408 rpc_type='STREAMING', 409 client_type='ASYNC_CLIENT', 410 server_type='ASYNC_SERVER', 411 unconstrained_client='async', 412 secure=secure, 413 client_threads_per_cq=2, 414 server_threads_per_cq=2, 415 categories=smoketest_categories + [SCALABLE]) 416 417 yield _ping_pong_scenario( 418 'cpp_protobuf_async_unary_qps_unconstrained_1cq_%s' % secstr, 419 rpc_type='UNARY', 420 client_type='ASYNC_CLIENT', 421 server_type='ASYNC_SERVER', 422 unconstrained_client='async-limited', 423 secure=secure, 424 client_threads_per_cq=1000000, 425 server_threads_per_cq=1000000, 426 categories=smoketest_categories + [SCALABLE]) 427 428 yield _ping_pong_scenario( 429 'cpp_protobuf_async_unary_qps_unconstrained_2waysharedcq_%s' % 430 secstr, 431 rpc_type='UNARY', 432 client_type='ASYNC_CLIENT', 433 server_type='ASYNC_SERVER', 434 unconstrained_client='async', 435 secure=secure, 436 client_threads_per_cq=2, 437 server_threads_per_cq=2, 438 categories=smoketest_categories + [SCALABLE]) 439 440 yield _ping_pong_scenario( 441 'cpp_generic_async_streaming_qps_one_server_core_%s' % secstr, 442 rpc_type='STREAMING', 443 client_type='ASYNC_CLIENT', 444 server_type='ASYNC_GENERIC_SERVER', 445 unconstrained_client='async-limited', 446 use_generic_payload=True, 447 async_server_threads=1, 448 minimal_stack=not secure, 449 secure=secure) 450 451 yield _ping_pong_scenario( 452 'cpp_protobuf_async_client_sync_server_unary_qps_unconstrained_%s' 453 % (secstr), 454 rpc_type='UNARY', 455 client_type='ASYNC_CLIENT', 456 server_type='SYNC_SERVER', 457 unconstrained_client='async', 458 secure=secure, 459 minimal_stack=not secure, 460 categories=smoketest_categories + [SCALABLE], 461 excluded_poll_engines=['poll-cv']) 462 463 yield _ping_pong_scenario( 464 'cpp_protobuf_async_client_unary_1channel_64wide_128Breq_8MBresp_%s' 465 % (secstr), 466 rpc_type='UNARY', 467 client_type='ASYNC_CLIENT', 468 server_type='ASYNC_SERVER', 469 channels=1, 470 outstanding=64, 471 req_size=128, 472 resp_size=8 * 1024 * 1024, 473 secure=secure, 474 minimal_stack=not secure, 475 categories=smoketest_categories + [SCALABLE]) 476 477 yield _ping_pong_scenario( 478 'cpp_protobuf_async_client_sync_server_streaming_qps_unconstrained_%s' 479 % secstr, 480 rpc_type='STREAMING', 481 client_type='ASYNC_CLIENT', 482 server_type='SYNC_SERVER', 483 unconstrained_client='async', 484 secure=secure, 485 minimal_stack=not secure, 486 categories=smoketest_categories + [SCALABLE], 487 excluded_poll_engines=['poll-cv']) 488 489 yield _ping_pong_scenario( 490 'cpp_protobuf_async_unary_ping_pong_%s_1MB' % secstr, 491 rpc_type='UNARY', 492 client_type='ASYNC_CLIENT', 493 server_type='ASYNC_SERVER', 494 req_size=1024 * 1024, 495 resp_size=1024 * 1024, 496 secure=secure, 497 minimal_stack=not secure, 498 categories=smoketest_categories + [SCALABLE]) 499 500 for rpc_type in [ 501 'unary', 'streaming', 'streaming_from_client', 502 'streaming_from_server' 503 ]: 504 for synchronicity in ['sync', 'async']: 505 yield _ping_pong_scenario( 506 'cpp_protobuf_%s_%s_ping_pong_%s' % (synchronicity, 507 rpc_type, secstr), 508 rpc_type=rpc_type.upper(), 509 client_type='%s_CLIENT' % synchronicity.upper(), 510 server_type='%s_SERVER' % synchronicity.upper(), 511 async_server_threads=1, 512 minimal_stack=not secure, 513 secure=secure) 514 515 for size in geometric_progression(1, 1024 * 1024 * 1024 + 1, 516 8): 517 yield _ping_pong_scenario( 518 'cpp_protobuf_%s_%s_qps_unconstrained_%s_%db' % 519 (synchronicity, rpc_type, secstr, size), 520 rpc_type=rpc_type.upper(), 521 req_size=size, 522 resp_size=size, 523 client_type='%s_CLIENT' % synchronicity.upper(), 524 server_type='%s_SERVER' % synchronicity.upper(), 525 unconstrained_client=synchronicity, 526 secure=secure, 527 minimal_stack=not secure, 528 categories=[SWEEP]) 529 530 yield _ping_pong_scenario( 531 'cpp_protobuf_%s_%s_qps_unconstrained_%s' % 532 (synchronicity, rpc_type, secstr), 533 rpc_type=rpc_type.upper(), 534 client_type='%s_CLIENT' % synchronicity.upper(), 535 server_type='%s_SERVER' % synchronicity.upper(), 536 unconstrained_client=synchronicity, 537 secure=secure, 538 minimal_stack=not secure, 539 server_threads_per_cq=3, 540 client_threads_per_cq=3, 541 categories=smoketest_categories + [SCALABLE]) 542 543 # TODO(vjpai): Re-enable this test. It has a lot of timeouts 544 # and hasn't yet been conclusively identified as a test failure 545 # or race in the library 546 # yield _ping_pong_scenario( 547 # 'cpp_protobuf_%s_%s_qps_unconstrained_%s_500kib_resource_quota' % (synchronicity, rpc_type, secstr), 548 # rpc_type=rpc_type.upper(), 549 # client_type='%s_CLIENT' % synchronicity.upper(), 550 # server_type='%s_SERVER' % synchronicity.upper(), 551 # unconstrained_client=synchronicity, 552 # secure=secure, 553 # categories=smoketest_categories+[SCALABLE], 554 # resource_quota_size=500*1024) 555 556 if rpc_type == 'streaming': 557 for mps in geometric_progression(1, 20, 10): 558 yield _ping_pong_scenario( 559 'cpp_protobuf_%s_%s_qps_unconstrained_%smps_%s' 560 % (synchronicity, rpc_type, mps, secstr), 561 rpc_type=rpc_type.upper(), 562 client_type='%s_CLIENT' % synchronicity.upper(), 563 server_type='%s_SERVER' % synchronicity.upper(), 564 unconstrained_client=synchronicity, 565 secure=secure, 566 messages_per_stream=mps, 567 minimal_stack=not secure, 568 categories=smoketest_categories + [SCALABLE]) 569 570 for mps in geometric_progression(1, 200, math.sqrt(10)): 571 yield _ping_pong_scenario( 572 'cpp_protobuf_%s_%s_qps_unconstrained_%smps_%s' 573 % (synchronicity, rpc_type, mps, secstr), 574 rpc_type=rpc_type.upper(), 575 client_type='%s_CLIENT' % synchronicity.upper(), 576 server_type='%s_SERVER' % synchronicity.upper(), 577 unconstrained_client=synchronicity, 578 secure=secure, 579 messages_per_stream=mps, 580 minimal_stack=not secure, 581 categories=[SWEEP]) 582 583 for channels in geometric_progression( 584 1, 20000, math.sqrt(10)): 585 for outstanding in geometric_progression( 586 1, 200000, math.sqrt(10)): 587 if synchronicity == 'sync' and outstanding > 1200: 588 continue 589 if outstanding < channels: continue 590 yield _ping_pong_scenario( 591 'cpp_protobuf_%s_%s_qps_unconstrained_%s_%d_channels_%d_outstanding' 592 % (synchronicity, rpc_type, secstr, channels, 593 outstanding), 594 rpc_type=rpc_type.upper(), 595 client_type='%s_CLIENT' % synchronicity.upper(), 596 server_type='%s_SERVER' % synchronicity.upper(), 597 unconstrained_client=synchronicity, 598 secure=secure, 599 minimal_stack=not secure, 600 categories=[SWEEP], 601 channels=channels, 602 outstanding=outstanding) 603 604 def __str__(self): 605 return 'c++' 606 607 608class CSharpLanguage: 609 610 def __init__(self): 611 self.safename = str(self) 612 613 def worker_cmdline(self): 614 return ['tools/run_tests/performance/run_worker_csharp.sh'] 615 616 def worker_port_offset(self): 617 return 100 618 619 def scenarios(self): 620 yield _ping_pong_scenario( 621 'csharp_generic_async_streaming_ping_pong', 622 rpc_type='STREAMING', 623 client_type='ASYNC_CLIENT', 624 server_type='ASYNC_GENERIC_SERVER', 625 use_generic_payload=True, 626 categories=[SMOKETEST, SCALABLE]) 627 628 yield _ping_pong_scenario( 629 'csharp_generic_async_streaming_ping_pong_insecure_1MB', 630 rpc_type='STREAMING', 631 client_type='ASYNC_CLIENT', 632 server_type='ASYNC_GENERIC_SERVER', 633 req_size=1024 * 1024, 634 resp_size=1024 * 1024, 635 use_generic_payload=True, 636 secure=False, 637 categories=[SMOKETEST, SCALABLE]) 638 639 yield _ping_pong_scenario( 640 'csharp_generic_async_streaming_qps_unconstrained_insecure', 641 rpc_type='STREAMING', 642 client_type='ASYNC_CLIENT', 643 server_type='ASYNC_GENERIC_SERVER', 644 unconstrained_client='async', 645 use_generic_payload=True, 646 secure=False, 647 categories=[SMOKETEST, SCALABLE]) 648 649 yield _ping_pong_scenario( 650 'csharp_protobuf_async_streaming_ping_pong', 651 rpc_type='STREAMING', 652 client_type='ASYNC_CLIENT', 653 server_type='ASYNC_SERVER') 654 655 yield _ping_pong_scenario( 656 'csharp_protobuf_async_unary_ping_pong', 657 rpc_type='UNARY', 658 client_type='ASYNC_CLIENT', 659 server_type='ASYNC_SERVER', 660 categories=[SMOKETEST, SCALABLE]) 661 662 yield _ping_pong_scenario( 663 'csharp_protobuf_sync_to_async_unary_ping_pong', 664 rpc_type='UNARY', 665 client_type='SYNC_CLIENT', 666 server_type='ASYNC_SERVER') 667 668 yield _ping_pong_scenario( 669 'csharp_protobuf_async_unary_qps_unconstrained', 670 rpc_type='UNARY', 671 client_type='ASYNC_CLIENT', 672 server_type='ASYNC_SERVER', 673 unconstrained_client='async', 674 categories=[SMOKETEST, SCALABLE]) 675 676 yield _ping_pong_scenario( 677 'csharp_protobuf_async_streaming_qps_unconstrained', 678 rpc_type='STREAMING', 679 client_type='ASYNC_CLIENT', 680 server_type='ASYNC_SERVER', 681 unconstrained_client='async', 682 categories=[SCALABLE]) 683 684 yield _ping_pong_scenario( 685 'csharp_to_cpp_protobuf_sync_unary_ping_pong', 686 rpc_type='UNARY', 687 client_type='SYNC_CLIENT', 688 server_type='SYNC_SERVER', 689 server_language='c++', 690 async_server_threads=1, 691 categories=[SMOKETEST, SCALABLE]) 692 693 yield _ping_pong_scenario( 694 'csharp_to_cpp_protobuf_async_streaming_ping_pong', 695 rpc_type='STREAMING', 696 client_type='ASYNC_CLIENT', 697 server_type='ASYNC_SERVER', 698 server_language='c++', 699 async_server_threads=1) 700 701 yield _ping_pong_scenario( 702 'csharp_to_cpp_protobuf_async_unary_qps_unconstrained', 703 rpc_type='UNARY', 704 client_type='ASYNC_CLIENT', 705 server_type='ASYNC_SERVER', 706 unconstrained_client='async', 707 server_language='c++', 708 categories=[SCALABLE]) 709 710 yield _ping_pong_scenario( 711 'csharp_to_cpp_protobuf_sync_to_async_unary_qps_unconstrained', 712 rpc_type='UNARY', 713 client_type='SYNC_CLIENT', 714 server_type='ASYNC_SERVER', 715 unconstrained_client='sync', 716 server_language='c++', 717 categories=[SCALABLE]) 718 719 yield _ping_pong_scenario( 720 'cpp_to_csharp_protobuf_async_unary_qps_unconstrained', 721 rpc_type='UNARY', 722 client_type='ASYNC_CLIENT', 723 server_type='ASYNC_SERVER', 724 unconstrained_client='async', 725 client_language='c++', 726 categories=[SCALABLE]) 727 728 yield _ping_pong_scenario( 729 'csharp_protobuf_async_unary_ping_pong_1MB', 730 rpc_type='UNARY', 731 client_type='ASYNC_CLIENT', 732 server_type='ASYNC_SERVER', 733 req_size=1024 * 1024, 734 resp_size=1024 * 1024, 735 categories=[SMOKETEST, SCALABLE]) 736 737 def __str__(self): 738 return 'csharp' 739 740 741class PythonLanguage: 742 743 def __init__(self): 744 self.safename = 'python' 745 746 def worker_cmdline(self): 747 return ['tools/run_tests/performance/run_worker_python.sh'] 748 749 def worker_port_offset(self): 750 return 500 751 752 def scenarios(self): 753 yield _ping_pong_scenario( 754 'python_generic_sync_streaming_ping_pong', 755 rpc_type='STREAMING', 756 client_type='SYNC_CLIENT', 757 server_type='ASYNC_GENERIC_SERVER', 758 use_generic_payload=True, 759 categories=[SMOKETEST, SCALABLE]) 760 761 yield _ping_pong_scenario( 762 'python_protobuf_sync_streaming_ping_pong', 763 rpc_type='STREAMING', 764 client_type='SYNC_CLIENT', 765 server_type='ASYNC_SERVER') 766 767 yield _ping_pong_scenario( 768 'python_protobuf_async_unary_ping_pong', 769 rpc_type='UNARY', 770 client_type='ASYNC_CLIENT', 771 server_type='ASYNC_SERVER') 772 773 yield _ping_pong_scenario( 774 'python_protobuf_sync_unary_ping_pong', 775 rpc_type='UNARY', 776 client_type='SYNC_CLIENT', 777 server_type='ASYNC_SERVER', 778 categories=[SMOKETEST, SCALABLE]) 779 780 yield _ping_pong_scenario( 781 'python_protobuf_sync_unary_qps_unconstrained', 782 rpc_type='UNARY', 783 client_type='SYNC_CLIENT', 784 server_type='ASYNC_SERVER', 785 unconstrained_client='sync') 786 787 yield _ping_pong_scenario( 788 'python_protobuf_sync_streaming_qps_unconstrained', 789 rpc_type='STREAMING', 790 client_type='SYNC_CLIENT', 791 server_type='ASYNC_SERVER', 792 unconstrained_client='sync') 793 794 yield _ping_pong_scenario( 795 'python_to_cpp_protobuf_sync_unary_ping_pong', 796 rpc_type='UNARY', 797 client_type='SYNC_CLIENT', 798 server_type='ASYNC_SERVER', 799 server_language='c++', 800 async_server_threads=1, 801 categories=[SMOKETEST, SCALABLE]) 802 803 yield _ping_pong_scenario( 804 'python_to_cpp_protobuf_sync_streaming_ping_pong', 805 rpc_type='STREAMING', 806 client_type='SYNC_CLIENT', 807 server_type='ASYNC_SERVER', 808 server_language='c++', 809 async_server_threads=1) 810 811 yield _ping_pong_scenario( 812 'python_protobuf_sync_unary_ping_pong_1MB', 813 rpc_type='UNARY', 814 client_type='SYNC_CLIENT', 815 server_type='ASYNC_SERVER', 816 req_size=1024 * 1024, 817 resp_size=1024 * 1024, 818 categories=[SMOKETEST, SCALABLE]) 819 820 def __str__(self): 821 return 'python' 822 823 824class RubyLanguage: 825 826 def __init__(self): 827 pass 828 self.safename = str(self) 829 830 def worker_cmdline(self): 831 return ['tools/run_tests/performance/run_worker_ruby.sh'] 832 833 def worker_port_offset(self): 834 return 300 835 836 def scenarios(self): 837 yield _ping_pong_scenario( 838 'ruby_protobuf_sync_streaming_ping_pong', 839 rpc_type='STREAMING', 840 client_type='SYNC_CLIENT', 841 server_type='SYNC_SERVER', 842 categories=[SMOKETEST, SCALABLE]) 843 844 yield _ping_pong_scenario( 845 'ruby_protobuf_unary_ping_pong', 846 rpc_type='UNARY', 847 client_type='SYNC_CLIENT', 848 server_type='SYNC_SERVER', 849 categories=[SMOKETEST, SCALABLE]) 850 851 yield _ping_pong_scenario( 852 'ruby_protobuf_sync_unary_qps_unconstrained', 853 rpc_type='UNARY', 854 client_type='SYNC_CLIENT', 855 server_type='SYNC_SERVER', 856 unconstrained_client='sync') 857 858 yield _ping_pong_scenario( 859 'ruby_protobuf_sync_streaming_qps_unconstrained', 860 rpc_type='STREAMING', 861 client_type='SYNC_CLIENT', 862 server_type='SYNC_SERVER', 863 unconstrained_client='sync') 864 865 yield _ping_pong_scenario( 866 'ruby_to_cpp_protobuf_sync_unary_ping_pong', 867 rpc_type='UNARY', 868 client_type='SYNC_CLIENT', 869 server_type='SYNC_SERVER', 870 server_language='c++', 871 async_server_threads=1) 872 873 yield _ping_pong_scenario( 874 'ruby_to_cpp_protobuf_sync_streaming_ping_pong', 875 rpc_type='STREAMING', 876 client_type='SYNC_CLIENT', 877 server_type='SYNC_SERVER', 878 server_language='c++', 879 async_server_threads=1) 880 881 yield _ping_pong_scenario( 882 'ruby_protobuf_unary_ping_pong_1MB', 883 rpc_type='UNARY', 884 client_type='SYNC_CLIENT', 885 server_type='SYNC_SERVER', 886 req_size=1024 * 1024, 887 resp_size=1024 * 1024, 888 categories=[SMOKETEST, SCALABLE]) 889 890 def __str__(self): 891 return 'ruby' 892 893 894class Php7Language: 895 896 def __init__(self, php7_protobuf_c=False): 897 pass 898 self.php7_protobuf_c = php7_protobuf_c 899 self.safename = str(self) 900 901 def worker_cmdline(self): 902 if self.php7_protobuf_c: 903 return [ 904 'tools/run_tests/performance/run_worker_php.sh', 905 '--use_protobuf_c_extension' 906 ] 907 return ['tools/run_tests/performance/run_worker_php.sh'] 908 909 def worker_port_offset(self): 910 if self.php7_protobuf_c: 911 return 900 912 return 800 913 914 def scenarios(self): 915 php7_extension_mode = 'php7_protobuf_php_extension' 916 if self.php7_protobuf_c: 917 php7_extension_mode = 'php7_protobuf_c_extension' 918 919 yield _ping_pong_scenario( 920 '%s_to_cpp_protobuf_sync_unary_ping_pong' % php7_extension_mode, 921 rpc_type='UNARY', 922 client_type='SYNC_CLIENT', 923 server_type='SYNC_SERVER', 924 server_language='c++', 925 async_server_threads=1) 926 927 yield _ping_pong_scenario( 928 '%s_to_cpp_protobuf_sync_streaming_ping_pong' % php7_extension_mode, 929 rpc_type='STREAMING', 930 client_type='SYNC_CLIENT', 931 server_type='SYNC_SERVER', 932 server_language='c++', 933 async_server_threads=1) 934 935 # TODO(ddyihai): Investigate why when async_server_threads=1/CPU usage 340%, the QPS performs 936 # better than async_server_threads=0/CPU usage 490%. 937 yield _ping_pong_scenario( 938 '%s_to_cpp_protobuf_sync_unary_qps_unconstrained' % 939 php7_extension_mode, 940 rpc_type='UNARY', 941 client_type='SYNC_CLIENT', 942 server_type='ASYNC_SERVER', 943 server_language='c++', 944 outstanding=1, 945 async_server_threads=1, 946 unconstrained_client='sync') 947 948 yield _ping_pong_scenario( 949 '%s_to_cpp_protobuf_sync_streaming_qps_unconstrained' % 950 php7_extension_mode, 951 rpc_type='STREAMING', 952 client_type='SYNC_CLIENT', 953 server_type='ASYNC_SERVER', 954 server_language='c++', 955 outstanding=1, 956 async_server_threads=1, 957 unconstrained_client='sync') 958 959 def __str__(self): 960 if self.php7_protobuf_c: 961 return 'php7_protobuf_c' 962 return 'php7' 963 964 965class JavaLanguage: 966 967 def __init__(self): 968 pass 969 self.safename = str(self) 970 971 def worker_cmdline(self): 972 return ['tools/run_tests/performance/run_worker_java.sh'] 973 974 def worker_port_offset(self): 975 return 400 976 977 def scenarios(self): 978 for secure in [True, False]: 979 secstr = 'secure' if secure else 'insecure' 980 smoketest_categories = ([SMOKETEST] if secure else []) + [SCALABLE] 981 982 yield _ping_pong_scenario( 983 'java_generic_async_streaming_ping_pong_%s' % secstr, 984 rpc_type='STREAMING', 985 client_type='ASYNC_CLIENT', 986 server_type='ASYNC_GENERIC_SERVER', 987 use_generic_payload=True, 988 async_server_threads=1, 989 secure=secure, 990 warmup_seconds=JAVA_WARMUP_SECONDS, 991 categories=smoketest_categories) 992 993 yield _ping_pong_scenario( 994 'java_protobuf_async_streaming_ping_pong_%s' % secstr, 995 rpc_type='STREAMING', 996 client_type='ASYNC_CLIENT', 997 server_type='ASYNC_SERVER', 998 async_server_threads=1, 999 secure=secure, 1000 warmup_seconds=JAVA_WARMUP_SECONDS) 1001 1002 yield _ping_pong_scenario( 1003 'java_protobuf_async_unary_ping_pong_%s' % secstr, 1004 rpc_type='UNARY', 1005 client_type='ASYNC_CLIENT', 1006 server_type='ASYNC_SERVER', 1007 async_server_threads=1, 1008 secure=secure, 1009 warmup_seconds=JAVA_WARMUP_SECONDS, 1010 categories=smoketest_categories) 1011 1012 yield _ping_pong_scenario( 1013 'java_protobuf_unary_ping_pong_%s' % secstr, 1014 rpc_type='UNARY', 1015 client_type='SYNC_CLIENT', 1016 server_type='SYNC_SERVER', 1017 async_server_threads=1, 1018 secure=secure, 1019 warmup_seconds=JAVA_WARMUP_SECONDS) 1020 1021 yield _ping_pong_scenario( 1022 'java_protobuf_async_unary_qps_unconstrained_%s' % secstr, 1023 rpc_type='UNARY', 1024 client_type='ASYNC_CLIENT', 1025 server_type='ASYNC_SERVER', 1026 unconstrained_client='async', 1027 secure=secure, 1028 warmup_seconds=JAVA_WARMUP_SECONDS, 1029 categories=smoketest_categories + [SCALABLE]) 1030 1031 yield _ping_pong_scenario( 1032 'java_protobuf_async_streaming_qps_unconstrained_%s' % secstr, 1033 rpc_type='STREAMING', 1034 client_type='ASYNC_CLIENT', 1035 server_type='ASYNC_SERVER', 1036 unconstrained_client='async', 1037 secure=secure, 1038 warmup_seconds=JAVA_WARMUP_SECONDS, 1039 categories=[SCALABLE]) 1040 1041 yield _ping_pong_scenario( 1042 'java_generic_async_streaming_qps_unconstrained_%s' % secstr, 1043 rpc_type='STREAMING', 1044 client_type='ASYNC_CLIENT', 1045 server_type='ASYNC_GENERIC_SERVER', 1046 unconstrained_client='async', 1047 use_generic_payload=True, 1048 secure=secure, 1049 warmup_seconds=JAVA_WARMUP_SECONDS, 1050 categories=[SCALABLE]) 1051 1052 yield _ping_pong_scenario( 1053 'java_generic_async_streaming_qps_one_server_core_%s' % secstr, 1054 rpc_type='STREAMING', 1055 client_type='ASYNC_CLIENT', 1056 server_type='ASYNC_GENERIC_SERVER', 1057 unconstrained_client='async-limited', 1058 use_generic_payload=True, 1059 async_server_threads=1, 1060 secure=secure, 1061 warmup_seconds=JAVA_WARMUP_SECONDS) 1062 1063 # TODO(jtattermusch): add scenarios java vs C++ 1064 1065 def __str__(self): 1066 return 'java' 1067 1068 1069class GoLanguage: 1070 1071 def __init__(self): 1072 pass 1073 self.safename = str(self) 1074 1075 def worker_cmdline(self): 1076 return ['tools/run_tests/performance/run_worker_go.sh'] 1077 1078 def worker_port_offset(self): 1079 return 600 1080 1081 def scenarios(self): 1082 for secure in [True, False]: 1083 secstr = 'secure' if secure else 'insecure' 1084 smoketest_categories = ([SMOKETEST] if secure else []) + [SCALABLE] 1085 1086 # ASYNC_GENERIC_SERVER for Go actually uses a sync streaming server, 1087 # but that's mostly because of lack of better name of the enum value. 1088 yield _ping_pong_scenario( 1089 'go_generic_sync_streaming_ping_pong_%s' % secstr, 1090 rpc_type='STREAMING', 1091 client_type='SYNC_CLIENT', 1092 server_type='ASYNC_GENERIC_SERVER', 1093 use_generic_payload=True, 1094 async_server_threads=1, 1095 secure=secure, 1096 categories=smoketest_categories) 1097 1098 yield _ping_pong_scenario( 1099 'go_protobuf_sync_streaming_ping_pong_%s' % secstr, 1100 rpc_type='STREAMING', 1101 client_type='SYNC_CLIENT', 1102 server_type='SYNC_SERVER', 1103 async_server_threads=1, 1104 secure=secure) 1105 1106 yield _ping_pong_scenario( 1107 'go_protobuf_sync_unary_ping_pong_%s' % secstr, 1108 rpc_type='UNARY', 1109 client_type='SYNC_CLIENT', 1110 server_type='SYNC_SERVER', 1111 async_server_threads=1, 1112 secure=secure, 1113 categories=smoketest_categories) 1114 1115 # unconstrained_client='async' is intended (client uses goroutines) 1116 yield _ping_pong_scenario( 1117 'go_protobuf_sync_unary_qps_unconstrained_%s' % secstr, 1118 rpc_type='UNARY', 1119 client_type='SYNC_CLIENT', 1120 server_type='SYNC_SERVER', 1121 unconstrained_client='async', 1122 secure=secure, 1123 categories=smoketest_categories + [SCALABLE]) 1124 1125 # unconstrained_client='async' is intended (client uses goroutines) 1126 yield _ping_pong_scenario( 1127 'go_protobuf_sync_streaming_qps_unconstrained_%s' % secstr, 1128 rpc_type='STREAMING', 1129 client_type='SYNC_CLIENT', 1130 server_type='SYNC_SERVER', 1131 unconstrained_client='async', 1132 secure=secure, 1133 categories=[SCALABLE]) 1134 1135 # unconstrained_client='async' is intended (client uses goroutines) 1136 # ASYNC_GENERIC_SERVER for Go actually uses a sync streaming server, 1137 # but that's mostly because of lack of better name of the enum value. 1138 yield _ping_pong_scenario( 1139 'go_generic_sync_streaming_qps_unconstrained_%s' % secstr, 1140 rpc_type='STREAMING', 1141 client_type='SYNC_CLIENT', 1142 server_type='ASYNC_GENERIC_SERVER', 1143 unconstrained_client='async', 1144 use_generic_payload=True, 1145 secure=secure, 1146 categories=[SCALABLE]) 1147 1148 # TODO(jtattermusch): add scenarios go vs C++ 1149 1150 def __str__(self): 1151 return 'go' 1152 1153 1154class NodeLanguage: 1155 1156 def __init__(self, node_purejs=False): 1157 pass 1158 self.node_purejs = node_purejs 1159 self.safename = str(self) 1160 1161 def worker_cmdline(self): 1162 fixture = 'native_js' if self.node_purejs else 'native_native' 1163 return [ 1164 'tools/run_tests/performance/run_worker_node.sh', fixture, 1165 '--benchmark_impl=grpc' 1166 ] 1167 1168 def worker_port_offset(self): 1169 if self.node_purejs: 1170 return 1100 1171 return 1000 1172 1173 def scenarios(self): 1174 node_implementation = 'node_purejs' if self.node_purejs else 'node' 1175 for secure in [True, False]: 1176 secstr = 'secure' if secure else 'insecure' 1177 smoketest_categories = ([SMOKETEST] if secure else []) + [SCALABLE] 1178 1179 yield _ping_pong_scenario( 1180 '%s_to_node_generic_async_streaming_ping_pong_%s' % 1181 (node_implementation, secstr), 1182 rpc_type='STREAMING', 1183 client_type='ASYNC_CLIENT', 1184 server_type='ASYNC_GENERIC_SERVER', 1185 server_language='node', 1186 use_generic_payload=True, 1187 async_server_threads=1, 1188 secure=secure, 1189 categories=smoketest_categories) 1190 1191 yield _ping_pong_scenario( 1192 '%s_to_node_protobuf_async_streaming_ping_pong_%s' % 1193 (node_implementation, secstr), 1194 rpc_type='STREAMING', 1195 client_type='ASYNC_CLIENT', 1196 server_type='ASYNC_SERVER', 1197 server_language='node', 1198 async_server_threads=1, 1199 secure=secure) 1200 1201 yield _ping_pong_scenario( 1202 '%s_to_node_protobuf_async_unary_ping_pong_%s' % 1203 (node_implementation, secstr), 1204 rpc_type='UNARY', 1205 client_type='ASYNC_CLIENT', 1206 server_type='ASYNC_SERVER', 1207 server_language='node', 1208 async_server_threads=1, 1209 secure=secure, 1210 categories=smoketest_categories) 1211 1212 yield _ping_pong_scenario( 1213 '%s_to_node_protobuf_async_unary_qps_unconstrained_%s' % 1214 (node_implementation, secstr), 1215 rpc_type='UNARY', 1216 client_type='ASYNC_CLIENT', 1217 server_type='ASYNC_SERVER', 1218 server_language='node', 1219 unconstrained_client='async', 1220 secure=secure, 1221 categories=smoketest_categories + [SCALABLE]) 1222 1223 yield _ping_pong_scenario( 1224 '%s_to_node_protobuf_async_streaming_qps_unconstrained_%s' % 1225 (node_implementation, secstr), 1226 rpc_type='STREAMING', 1227 client_type='ASYNC_CLIENT', 1228 server_type='ASYNC_SERVER', 1229 server_language='node', 1230 unconstrained_client='async', 1231 secure=secure, 1232 categories=[SCALABLE]) 1233 1234 yield _ping_pong_scenario( 1235 '%s_to_node_generic_async_streaming_qps_unconstrained_%s' % 1236 (node_implementation, secstr), 1237 rpc_type='STREAMING', 1238 client_type='ASYNC_CLIENT', 1239 server_type='ASYNC_GENERIC_SERVER', 1240 server_language='node', 1241 unconstrained_client='async', 1242 use_generic_payload=True, 1243 secure=secure, 1244 categories=[SCALABLE]) 1245 1246 # TODO(murgatroid99): add scenarios node vs C++ 1247 1248 def __str__(self): 1249 if self.node_purejs: 1250 return 'node_purejs' 1251 return 'node' 1252 1253 1254LANGUAGES = { 1255 'c++': CXXLanguage(), 1256 'csharp': CSharpLanguage(), 1257 'ruby': RubyLanguage(), 1258 'php7': Php7Language(), 1259 'php7_protobuf_c': Php7Language(php7_protobuf_c=True), 1260 'java': JavaLanguage(), 1261 'python': PythonLanguage(), 1262 'go': GoLanguage(), 1263 'node': NodeLanguage(), 1264 'node_purejs': NodeLanguage(node_purejs=True) 1265} 1266