1#!/usr/bin/env python2.7 2 3# Copyright 2015 gRPC authors. 4# 5# Licensed under the Apache License, Version 2.0 (the "License"); 6# you may not use this file except in compliance with the License. 7# You may obtain a copy of the License at 8# 9# http://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an "AS IS" BASIS, 13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14# See the License for the specific language governing permissions and 15# limitations under the License. 16 17from __future__ import print_function 18import json 19import pipes 20import shutil 21import sys 22import os 23import yaml 24 25run_tests_root = os.path.abspath( 26 os.path.join(os.path.dirname(sys.argv[0]), '../../../tools/run_tests')) 27sys.path.append(run_tests_root) 28 29import performance.scenario_config as scenario_config 30 31configs_from_yaml = yaml.load( 32 open( 33 os.path.join(os.path.dirname(sys.argv[0]), 34 '../../../build_handwritten.yaml')))['configs'].keys() 35 36 37def mutate_scenario(scenario_json, is_tsan): 38 # tweak parameters to get fast test times 39 scenario_json = dict(scenario_json) 40 scenario_json['warmup_seconds'] = 0 41 scenario_json['benchmark_seconds'] = 1 42 outstanding_rpcs_divisor = 1 43 if is_tsan and ( 44 scenario_json['client_config']['client_type'] == 'SYNC_CLIENT' or 45 scenario_json['server_config']['server_type'] == 'SYNC_SERVER'): 46 outstanding_rpcs_divisor = 10 47 scenario_json['client_config']['outstanding_rpcs_per_channel'] = max( 48 1, 49 int(scenario_json['client_config']['outstanding_rpcs_per_channel'] / 50 outstanding_rpcs_divisor)) 51 return scenario_json 52 53 54def _scenario_json_string(scenario_json, is_tsan): 55 scenarios_json = { 56 'scenarios': [ 57 scenario_config.remove_nonproto_fields( 58 mutate_scenario(scenario_json, is_tsan)) 59 ] 60 } 61 return json.dumps(scenarios_json) 62 63 64def threads_required(scenario_json, where, is_tsan): 65 scenario_json = mutate_scenario(scenario_json, is_tsan) 66 if scenario_json['%s_config' % where]['%s_type' % 67 where] == 'ASYNC_%s' % where.upper(): 68 return scenario_json['%s_config' % where].get( 69 'async_%s_threads' % where, 0) 70 return scenario_json['client_config'][ 71 'outstanding_rpcs_per_channel'] * scenario_json['client_config'][ 72 'client_channels'] 73 74 75def guess_cpu(scenario_json, is_tsan): 76 client = threads_required(scenario_json, 'client', is_tsan) 77 server = threads_required(scenario_json, 'server', is_tsan) 78 # make an arbitrary guess if set to auto-detect 79 # about the size of the jenkins instances we have for unit tests 80 if client == 0 or server == 0: return 'capacity' 81 return (scenario_json['num_clients'] * client + 82 scenario_json['num_servers'] * server) 83 84 85def maybe_exclude_gcov(scenario_json): 86 if scenario_json['client_config']['client_channels'] > 100: 87 return ['gcov'] 88 return [] 89 90 91# Originally, this method was used to generate qps test cases for build.yaml, 92# but since the test cases are now extracted from bazel BUILD file, 93# this is not used for generating run_tests.py test cases anymore. 94# Nevertheless, the output is still used by json_run_localhost_scenario_gen.py 95# and qps_json_driver_scenario_gen.py to generate the scenario list for bazel. 96# TODO(jtattermusch): cleanup this file, so that it only generates data needed 97# by bazel. 98def generate_yaml(): 99 return { 100 'tests': 101 [{ 102 'name': 103 'json_run_localhost', 104 'shortname': 105 'json_run_localhost:%s' % scenario_json['name'], 106 'args': [ 107 '--scenarios_json', 108 _scenario_json_string(scenario_json, False) 109 ], 110 'ci_platforms': ['linux'], 111 'platforms': ['linux'], 112 'flaky': 113 False, 114 'language': 115 'c++', 116 'boringssl': 117 True, 118 'defaults': 119 'boringssl', 120 'cpu_cost': 121 guess_cpu(scenario_json, False), 122 'exclude_configs': ['tsan', 'asan'] + 123 maybe_exclude_gcov(scenario_json), 124 'timeout_seconds': 125 2 * 60, 126 'excluded_poll_engines': 127 scenario_json.get('EXCLUDED_POLL_ENGINES', []), 128 'auto_timeout_scaling': 129 False 130 } 131 for scenario_json in scenario_config.CXXLanguage().scenarios() 132 if 'scalable' in scenario_json.get('CATEGORIES', [])] + 133 [{ 134 'name': 135 'qps_json_driver', 136 'shortname': 137 'qps_json_driver:inproc_%s' % scenario_json['name'], 138 'args': [ 139 '--run_inproc', '--scenarios_json', 140 _scenario_json_string(scenario_json, False) 141 ], 142 'ci_platforms': ['linux'], 143 'platforms': ['linux'], 144 'flaky': 145 False, 146 'language': 147 'c++', 148 'boringssl': 149 True, 150 'defaults': 151 'boringssl', 152 'cpu_cost': 153 guess_cpu(scenario_json, False), 154 'exclude_configs': ['tsan', 'asan'], 155 'timeout_seconds': 156 6 * 60, 157 'excluded_poll_engines': 158 scenario_json.get('EXCLUDED_POLL_ENGINES', []) 159 } 160 for scenario_json in scenario_config.CXXLanguage().scenarios() 161 if 'inproc' in scenario_json.get('CATEGORIES', [])] + 162 [{ 163 'name': 164 'json_run_localhost', 165 'shortname': 166 'json_run_localhost:%s_low_thread_count' % 167 scenario_json['name'], 168 'args': [ 169 '--scenarios_json', 170 _scenario_json_string(scenario_json, True) 171 ], 172 'ci_platforms': ['linux'], 173 'platforms': ['linux'], 174 'flaky': 175 False, 176 'language': 177 'c++', 178 'boringssl': 179 True, 180 'defaults': 181 'boringssl', 182 'cpu_cost': 183 guess_cpu(scenario_json, True), 184 'exclude_configs': 185 sorted(c 186 for c in configs_from_yaml 187 if c not in ('tsan', 'asan')), 188 'timeout_seconds': 189 10 * 60, 190 'excluded_poll_engines': 191 scenario_json.get('EXCLUDED_POLL_ENGINES', []), 192 'auto_timeout_scaling': 193 False 194 } 195 for scenario_json in scenario_config.CXXLanguage().scenarios() 196 if 'scalable' in scenario_json.get('CATEGORIES', [])] 197 } 198