1#!/usr/bin/env python2.7 2# 3# Copyright 2017 gRPC authors. 4# 5# Licensed under the Apache License, Version 2.0 (the "License"); 6# you may not use this file except in compliance with the License. 7# You may obtain a copy of the License at 8# 9# http://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an "AS IS" BASIS, 13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14# See the License for the specific language governing permissions and 15# limitations under the License. 16""" Computes the diff between two bm runs and outputs significant results """ 17 18import bm_constants 19import bm_speedup 20 21import sys 22import os 23 24sys.path.append(os.path.join(os.path.dirname(sys.argv[0]), '..')) 25import bm_json 26 27import json 28import tabulate 29import argparse 30import collections 31import subprocess 32 33verbose = False 34 35 36def _median(ary): 37 assert (len(ary)) 38 ary = sorted(ary) 39 n = len(ary) 40 if n % 2 == 0: 41 return (ary[(n - 1) / 2] + ary[(n - 1) / 2 + 1]) / 2.0 42 else: 43 return ary[n / 2] 44 45 46def _args(): 47 argp = argparse.ArgumentParser( 48 description='Perform diff on microbenchmarks') 49 argp.add_argument('-t', 50 '--track', 51 choices=sorted(bm_constants._INTERESTING), 52 nargs='+', 53 default=sorted(bm_constants._INTERESTING), 54 help='Which metrics to track') 55 argp.add_argument('-b', 56 '--benchmarks', 57 nargs='+', 58 choices=bm_constants._AVAILABLE_BENCHMARK_TESTS, 59 default=bm_constants._AVAILABLE_BENCHMARK_TESTS, 60 help='Which benchmarks to run') 61 argp.add_argument( 62 '-l', 63 '--loops', 64 type=int, 65 default=20, 66 help= 67 'Number of times to loops the benchmarks. Must match what was passed to bm_run.py' 68 ) 69 argp.add_argument('-r', 70 '--regex', 71 type=str, 72 default="", 73 help='Regex to filter benchmarks run') 74 argp.add_argument('--counters', dest='counters', action='store_true') 75 argp.add_argument('--no-counters', dest='counters', action='store_false') 76 argp.set_defaults(counters=True) 77 argp.add_argument('-n', '--new', type=str, help='New benchmark name') 78 argp.add_argument('-o', '--old', type=str, help='Old benchmark name') 79 argp.add_argument('-v', 80 '--verbose', 81 type=bool, 82 help='Print details of before/after') 83 args = argp.parse_args() 84 global verbose 85 if args.verbose: verbose = True 86 assert args.new 87 assert args.old 88 return args 89 90 91def _maybe_print(str): 92 if verbose: print str 93 94 95class Benchmark: 96 97 def __init__(self): 98 self.samples = { 99 True: collections.defaultdict(list), 100 False: collections.defaultdict(list) 101 } 102 self.final = {} 103 104 def add_sample(self, track, data, new): 105 for f in track: 106 if f in data: 107 self.samples[new][f].append(float(data[f])) 108 109 def process(self, track, new_name, old_name): 110 for f in sorted(track): 111 new = self.samples[True][f] 112 old = self.samples[False][f] 113 if not new or not old: continue 114 mdn_diff = abs(_median(new) - _median(old)) 115 _maybe_print('%s: %s=%r %s=%r mdn_diff=%r' % 116 (f, new_name, new, old_name, old, mdn_diff)) 117 s = bm_speedup.speedup(new, old, 1e-5) 118 if abs(s) > 3: 119 if mdn_diff > 0.5 or 'trickle' in f: 120 self.final[f] = '%+d%%' % s 121 return self.final.keys() 122 123 def skip(self): 124 return not self.final 125 126 def row(self, flds): 127 return [self.final[f] if f in self.final else '' for f in flds] 128 129 130def _read_json(filename, badjson_files, nonexistant_files): 131 stripped = ".".join(filename.split(".")[:-2]) 132 try: 133 with open(filename) as f: 134 r = f.read() 135 return json.loads(r) 136 except IOError, e: 137 if stripped in nonexistant_files: 138 nonexistant_files[stripped] += 1 139 else: 140 nonexistant_files[stripped] = 1 141 return None 142 except ValueError, e: 143 print r 144 if stripped in badjson_files: 145 badjson_files[stripped] += 1 146 else: 147 badjson_files[stripped] = 1 148 return None 149 150 151def fmt_dict(d): 152 return ''.join([" " + k + ": " + str(d[k]) + "\n" for k in d]) 153 154 155def diff(bms, loops, regex, track, old, new, counters): 156 benchmarks = collections.defaultdict(Benchmark) 157 158 badjson_files = {} 159 nonexistant_files = {} 160 for bm in bms: 161 for loop in range(0, loops): 162 for line in subprocess.check_output([ 163 'bm_diff_%s/opt/%s' % (old, bm), '--benchmark_list_tests', 164 '--benchmark_filter=%s' % regex 165 ]).splitlines(): 166 stripped_line = line.strip().replace("/", "_").replace( 167 "<", "_").replace(">", "_").replace(", ", "_") 168 js_new_opt = _read_json( 169 '%s.%s.opt.%s.%d.json' % (bm, stripped_line, new, loop), 170 badjson_files, nonexistant_files) 171 js_old_opt = _read_json( 172 '%s.%s.opt.%s.%d.json' % (bm, stripped_line, old, loop), 173 badjson_files, nonexistant_files) 174 if counters: 175 js_new_ctr = _read_json( 176 '%s.%s.counters.%s.%d.json' % 177 (bm, stripped_line, new, loop), badjson_files, 178 nonexistant_files) 179 js_old_ctr = _read_json( 180 '%s.%s.counters.%s.%d.json' % 181 (bm, stripped_line, old, loop), badjson_files, 182 nonexistant_files) 183 else: 184 js_new_ctr = None 185 js_old_ctr = None 186 187 for row in bm_json.expand_json(js_new_ctr, js_new_opt): 188 name = row['cpp_name'] 189 if name.endswith('_mean') or name.endswith('_stddev'): 190 continue 191 benchmarks[name].add_sample(track, row, True) 192 for row in bm_json.expand_json(js_old_ctr, js_old_opt): 193 name = row['cpp_name'] 194 if name.endswith('_mean') or name.endswith('_stddev'): 195 continue 196 benchmarks[name].add_sample(track, row, False) 197 198 really_interesting = set() 199 for name, bm in benchmarks.items(): 200 _maybe_print(name) 201 really_interesting.update(bm.process(track, new, old)) 202 fields = [f for f in track if f in really_interesting] 203 204 headers = ['Benchmark'] + fields 205 rows = [] 206 for name in sorted(benchmarks.keys()): 207 if benchmarks[name].skip(): continue 208 rows.append([name] + benchmarks[name].row(fields)) 209 note = None 210 if len(badjson_files): 211 note = 'Corrupt JSON data (indicates timeout or crash): \n%s' % fmt_dict( 212 badjson_files) 213 if len(nonexistant_files): 214 if note: 215 note += '\n\nMissing files (indicates new benchmark): \n%s' % fmt_dict( 216 nonexistant_files) 217 else: 218 note = '\n\nMissing files (indicates new benchmark): \n%s' % fmt_dict( 219 nonexistant_files) 220 if rows: 221 return tabulate.tabulate(rows, headers=headers, floatfmt='+.2f'), note 222 else: 223 return None, note 224 225 226if __name__ == '__main__': 227 args = _args() 228 diff, note = diff(args.benchmarks, args.loops, args.regex, args.track, 229 args.old, args.new, args.counters) 230 print('%s\n%s' % (note, diff if diff else "No performance differences")) 231