• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/env python2.7
2
3# Copyright 2015 gRPC authors.
4#
5# Licensed under the Apache License, Version 2.0 (the "License");
6# you may not use this file except in compliance with the License.
7# You may obtain a copy of the License at
8#
9#     http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16
17from __future__ import print_function
18import json
19import pipes
20import shutil
21import sys
22import os
23import yaml
24
25run_tests_root = os.path.abspath(
26    os.path.join(os.path.dirname(sys.argv[0]), '../../../tools/run_tests'))
27sys.path.append(run_tests_root)
28
29import performance.scenario_config as scenario_config
30
31configs_from_yaml = yaml.load(
32    open(
33        os.path.join(os.path.dirname(sys.argv[0]),
34                     '../../../build_handwritten.yaml')))['configs'].keys()
35
36
37def mutate_scenario(scenario_json, is_tsan):
38    # tweak parameters to get fast test times
39    scenario_json = dict(scenario_json)
40    scenario_json['warmup_seconds'] = 0
41    scenario_json['benchmark_seconds'] = 1
42    outstanding_rpcs_divisor = 1
43    if is_tsan and (
44            scenario_json['client_config']['client_type'] == 'SYNC_CLIENT' or
45            scenario_json['server_config']['server_type'] == 'SYNC_SERVER'):
46        outstanding_rpcs_divisor = 10
47    scenario_json['client_config']['outstanding_rpcs_per_channel'] = max(
48        1,
49        int(scenario_json['client_config']['outstanding_rpcs_per_channel'] /
50            outstanding_rpcs_divisor))
51    return scenario_json
52
53
54def _scenario_json_string(scenario_json, is_tsan):
55    scenarios_json = {
56        'scenarios': [
57            scenario_config.remove_nonproto_fields(
58                mutate_scenario(scenario_json, is_tsan))
59        ]
60    }
61    return json.dumps(scenarios_json)
62
63
64def threads_required(scenario_json, where, is_tsan):
65    scenario_json = mutate_scenario(scenario_json, is_tsan)
66    if scenario_json['%s_config' % where]['%s_type' %
67                                          where] == 'ASYNC_%s' % where.upper():
68        return scenario_json['%s_config' % where].get(
69            'async_%s_threads' % where, 0)
70    return scenario_json['client_config'][
71        'outstanding_rpcs_per_channel'] * scenario_json['client_config'][
72            'client_channels']
73
74
75def guess_cpu(scenario_json, is_tsan):
76    client = threads_required(scenario_json, 'client', is_tsan)
77    server = threads_required(scenario_json, 'server', is_tsan)
78    # make an arbitrary guess if set to auto-detect
79    # about the size of the jenkins instances we have for unit tests
80    if client == 0 or server == 0: return 'capacity'
81    return (scenario_json['num_clients'] * client +
82            scenario_json['num_servers'] * server)
83
84
85def maybe_exclude_gcov(scenario_json):
86    if scenario_json['client_config']['client_channels'] > 100:
87        return ['gcov']
88    return []
89
90
91# Originally, this method was used to generate qps test cases for build.yaml,
92# but since the test cases are now extracted from bazel BUILD file,
93# this is not used for generating run_tests.py test cases anymore.
94# Nevertheless, the output is still used by json_run_localhost_scenario_gen.py
95# and qps_json_driver_scenario_gen.py to generate the scenario list for bazel.
96# TODO(jtattermusch): cleanup this file, so that it only generates data needed
97# by bazel.
98def generate_yaml():
99    return {
100        'tests':
101            [{
102                'name':
103                    'json_run_localhost',
104                'shortname':
105                    'json_run_localhost:%s' % scenario_json['name'],
106                'args': [
107                    '--scenarios_json',
108                    _scenario_json_string(scenario_json, False)
109                ],
110                'ci_platforms': ['linux'],
111                'platforms': ['linux'],
112                'flaky':
113                    False,
114                'language':
115                    'c++',
116                'boringssl':
117                    True,
118                'defaults':
119                    'boringssl',
120                'cpu_cost':
121                    guess_cpu(scenario_json, False),
122                'exclude_configs': ['tsan', 'asan'] +
123                                   maybe_exclude_gcov(scenario_json),
124                'timeout_seconds':
125                    2 * 60,
126                'excluded_poll_engines':
127                    scenario_json.get('EXCLUDED_POLL_ENGINES', []),
128                'auto_timeout_scaling':
129                    False
130            }
131             for scenario_json in scenario_config.CXXLanguage().scenarios()
132             if 'scalable' in scenario_json.get('CATEGORIES', [])] +
133            [{
134                'name':
135                    'qps_json_driver',
136                'shortname':
137                    'qps_json_driver:inproc_%s' % scenario_json['name'],
138                'args': [
139                    '--run_inproc', '--scenarios_json',
140                    _scenario_json_string(scenario_json, False)
141                ],
142                'ci_platforms': ['linux'],
143                'platforms': ['linux'],
144                'flaky':
145                    False,
146                'language':
147                    'c++',
148                'boringssl':
149                    True,
150                'defaults':
151                    'boringssl',
152                'cpu_cost':
153                    guess_cpu(scenario_json, False),
154                'exclude_configs': ['tsan', 'asan'],
155                'timeout_seconds':
156                    6 * 60,
157                'excluded_poll_engines':
158                    scenario_json.get('EXCLUDED_POLL_ENGINES', [])
159            }
160             for scenario_json in scenario_config.CXXLanguage().scenarios()
161             if 'inproc' in scenario_json.get('CATEGORIES', [])] +
162            [{
163                'name':
164                    'json_run_localhost',
165                'shortname':
166                    'json_run_localhost:%s_low_thread_count' %
167                    scenario_json['name'],
168                'args': [
169                    '--scenarios_json',
170                    _scenario_json_string(scenario_json, True)
171                ],
172                'ci_platforms': ['linux'],
173                'platforms': ['linux'],
174                'flaky':
175                    False,
176                'language':
177                    'c++',
178                'boringssl':
179                    True,
180                'defaults':
181                    'boringssl',
182                'cpu_cost':
183                    guess_cpu(scenario_json, True),
184                'exclude_configs':
185                    sorted(c
186                           for c in configs_from_yaml
187                           if c not in ('tsan', 'asan')),
188                'timeout_seconds':
189                    10 * 60,
190                'excluded_poll_engines':
191                    scenario_json.get('EXCLUDED_POLL_ENGINES', []),
192                'auto_timeout_scaling':
193                    False
194            }
195             for scenario_json in scenario_config.CXXLanguage().scenarios()
196             if 'scalable' in scenario_json.get('CATEGORIES', [])]
197    }
198