1# Copyright 2017 gRPC authors. 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); 4# you may not use this file except in compliance with the License. 5# You may obtain a copy of the License at 6# 7# http://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12# See the License for the specific language governing permissions and 13# limitations under the License. 14 15# Utilities for manipulating JSON data that represents microbenchmark results. 16 17import os 18 19# template arguments and dynamic arguments of individual benchmark types 20# Example benchmark name: "BM_UnaryPingPong<TCP, NoOpMutator, NoOpMutator>/0/0" 21_BM_SPECS = { 22 'BM_UnaryPingPong': { 23 'tpl': ['fixture', 'client_mutator', 'server_mutator'], 24 'dyn': ['request_size', 'response_size'], 25 }, 26 'BM_PumpStreamClientToServer': { 27 'tpl': ['fixture'], 28 'dyn': ['request_size'], 29 }, 30 'BM_PumpStreamServerToClient': { 31 'tpl': ['fixture'], 32 'dyn': ['request_size'], 33 }, 34 'BM_StreamingPingPong': { 35 'tpl': ['fixture', 'client_mutator', 'server_mutator'], 36 'dyn': ['request_size', 'request_count'], 37 }, 38 'BM_StreamingPingPongMsgs': { 39 'tpl': ['fixture', 'client_mutator', 'server_mutator'], 40 'dyn': ['request_size'], 41 }, 42 'BM_PumpStreamServerToClient_Trickle': { 43 'tpl': [], 44 'dyn': ['request_size', 'bandwidth_kilobits'], 45 }, 46 'BM_PumpUnbalancedUnary_Trickle': { 47 'tpl': [], 48 'dyn': ['cli_req_size', 'svr_req_size', 'bandwidth_kilobits'], 49 }, 50 'BM_ErrorStringOnNewError': { 51 'tpl': ['fixture'], 52 'dyn': [], 53 }, 54 'BM_ErrorStringRepeatedly': { 55 'tpl': ['fixture'], 56 'dyn': [], 57 }, 58 'BM_ErrorGetStatus': { 59 'tpl': ['fixture'], 60 'dyn': [], 61 }, 62 'BM_ErrorGetStatusCode': { 63 'tpl': ['fixture'], 64 'dyn': [], 65 }, 66 'BM_ErrorHttpError': { 67 'tpl': ['fixture'], 68 'dyn': [], 69 }, 70 'BM_HasClearGrpcStatus': { 71 'tpl': ['fixture'], 72 'dyn': [], 73 }, 74 'BM_IsolatedFilter': { 75 'tpl': ['fixture', 'client_mutator'], 76 'dyn': [], 77 }, 78 'BM_HpackEncoderEncodeHeader': { 79 'tpl': ['fixture'], 80 'dyn': ['end_of_stream', 'request_size'], 81 }, 82 'BM_HpackParserParseHeader': { 83 'tpl': ['fixture', 'on_header'], 84 'dyn': [], 85 }, 86 'BM_CallCreateDestroy': { 87 'tpl': ['fixture'], 88 'dyn': [], 89 }, 90 'BM_Zalloc': { 91 'tpl': [], 92 'dyn': ['request_size'], 93 }, 94 'BM_PollEmptyPollset_SpeedOfLight': { 95 'tpl': [], 96 'dyn': ['request_size', 'request_count'], 97 }, 98 'BM_StreamCreateSendInitialMetadataDestroy': { 99 'tpl': ['fixture'], 100 'dyn': [], 101 }, 102 'BM_TransportStreamSend': { 103 'tpl': [], 104 'dyn': ['request_size'], 105 }, 106 'BM_TransportStreamRecv': { 107 'tpl': [], 108 'dyn': ['request_size'], 109 }, 110 'BM_StreamingPingPongWithCoalescingApi': { 111 'tpl': ['fixture', 'client_mutator', 'server_mutator'], 112 'dyn': ['request_size', 'request_count', 'end_of_stream'], 113 }, 114 'BM_Base16SomeStuff': { 115 'tpl': [], 116 'dyn': ['request_size'], 117 } 118} 119 120 121def numericalize(s): 122 """Convert abbreviations like '100M' or '10k' to a number.""" 123 if not s: return '' 124 if s[-1] == 'k': 125 return float(s[:-1]) * 1024 126 if s[-1] == 'M': 127 return float(s[:-1]) * 1024 * 1024 128 if 0 <= (ord(s[-1]) - ord('0')) <= 9: 129 return float(s) 130 assert 'not a number: %s' % s 131 132 133def parse_name(name): 134 cpp_name = name 135 if '<' not in name and '/' not in name and name not in _BM_SPECS: 136 return {'name': name, 'cpp_name': name} 137 rest = name 138 out = {} 139 tpl_args = [] 140 dyn_args = [] 141 if '<' in rest: 142 tpl_bit = rest[rest.find('<') + 1:rest.rfind('>')] 143 arg = '' 144 nesting = 0 145 for c in tpl_bit: 146 if c == '<': 147 nesting += 1 148 arg += c 149 elif c == '>': 150 nesting -= 1 151 arg += c 152 elif c == ',': 153 if nesting == 0: 154 tpl_args.append(arg.strip()) 155 arg = '' 156 else: 157 arg += c 158 else: 159 arg += c 160 tpl_args.append(arg.strip()) 161 rest = rest[:rest.find('<')] + rest[rest.rfind('>') + 1:] 162 if '/' in rest: 163 s = rest.split('/') 164 rest = s[0] 165 dyn_args = s[1:] 166 name = rest 167 assert name in _BM_SPECS, '_BM_SPECS needs to be expanded for %s' % name 168 assert len(dyn_args) == len(_BM_SPECS[name]['dyn']) 169 assert len(tpl_args) == len(_BM_SPECS[name]['tpl']) 170 out['name'] = name 171 out['cpp_name'] = cpp_name 172 out.update( 173 dict((k, numericalize(v)) 174 for k, v in zip(_BM_SPECS[name]['dyn'], dyn_args))) 175 out.update(dict(zip(_BM_SPECS[name]['tpl'], tpl_args))) 176 return out 177 178 179def expand_json(js, js2=None): 180 if not js and not js2: raise StopIteration() 181 if not js: js = js2 182 for bm in js['benchmarks']: 183 if bm['name'].endswith('_stddev') or bm['name'].endswith('_mean'): 184 continue 185 context = js['context'] 186 if 'label' in bm: 187 labels_list = [ 188 s.split(':') 189 for s in bm['label'].strip().split(' ') 190 if len(s) and s[0] != '#' 191 ] 192 for el in labels_list: 193 el[0] = el[0].replace('/iter', '_per_iteration') 194 labels = dict(labels_list) 195 else: 196 labels = {} 197 row = { 198 'jenkins_build': os.environ.get('BUILD_NUMBER', ''), 199 'jenkins_job': os.environ.get('JOB_NAME', ''), 200 } 201 row.update(context) 202 row.update(bm) 203 row.update(parse_name(row['name'])) 204 row.update(labels) 205 if js2: 206 for bm2 in js2['benchmarks']: 207 if bm['name'] == bm2['name'] and 'already_used' not in bm2: 208 row['cpu_time'] = bm2['cpu_time'] 209 row['real_time'] = bm2['real_time'] 210 row['iterations'] = bm2['iterations'] 211 bm2['already_used'] = True 212 break 213 yield row 214