1#!/usr/bin/env python2.7 2# 3# Copyright 2017 gRPC authors. 4# 5# Licensed under the Apache License, Version 2.0 (the "License"); 6# you may not use this file except in compliance with the License. 7# You may obtain a copy of the License at 8# 9# http://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an "AS IS" BASIS, 13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14# See the License for the specific language governing permissions and 15# limitations under the License. 16""" Computes the diff between two bm runs and outputs significant results """ 17 18import bm_constants 19import bm_speedup 20 21import sys 22import os 23 24sys.path.append(os.path.join(os.path.dirname(sys.argv[0]), '..')) 25import bm_json 26 27import json 28import tabulate 29import argparse 30import collections 31import subprocess 32 33verbose = False 34 35 36def _median(ary): 37 assert (len(ary)) 38 ary = sorted(ary) 39 n = len(ary) 40 if n % 2 == 0: 41 return (ary[(n - 1) / 2] + ary[(n - 1) / 2 + 1]) / 2.0 42 else: 43 return ary[n / 2] 44 45 46def _args(): 47 argp = argparse.ArgumentParser( 48 description='Perform diff on microbenchmarks') 49 argp.add_argument( 50 '-t', 51 '--track', 52 choices=sorted(bm_constants._INTERESTING), 53 nargs='+', 54 default=sorted(bm_constants._INTERESTING), 55 help='Which metrics to track') 56 argp.add_argument( 57 '-b', 58 '--benchmarks', 59 nargs='+', 60 choices=bm_constants._AVAILABLE_BENCHMARK_TESTS, 61 default=bm_constants._AVAILABLE_BENCHMARK_TESTS, 62 help='Which benchmarks to run') 63 argp.add_argument( 64 '-l', 65 '--loops', 66 type=int, 67 default=20, 68 help= 69 'Number of times to loops the benchmarks. Must match what was passed to bm_run.py' 70 ) 71 argp.add_argument( 72 '-r', 73 '--regex', 74 type=str, 75 default="", 76 help='Regex to filter benchmarks run') 77 argp.add_argument('--counters', dest='counters', action='store_true') 78 argp.add_argument('--no-counters', dest='counters', action='store_false') 79 argp.set_defaults(counters=True) 80 argp.add_argument('-n', '--new', type=str, help='New benchmark name') 81 argp.add_argument('-o', '--old', type=str, help='Old benchmark name') 82 argp.add_argument( 83 '-v', '--verbose', type=bool, help='Print details of before/after') 84 args = argp.parse_args() 85 global verbose 86 if args.verbose: verbose = True 87 assert args.new 88 assert args.old 89 return args 90 91 92def _maybe_print(str): 93 if verbose: print str 94 95 96class Benchmark: 97 98 def __init__(self): 99 self.samples = { 100 True: collections.defaultdict(list), 101 False: collections.defaultdict(list) 102 } 103 self.final = {} 104 105 def add_sample(self, track, data, new): 106 for f in track: 107 if f in data: 108 self.samples[new][f].append(float(data[f])) 109 110 def process(self, track, new_name, old_name): 111 for f in sorted(track): 112 new = self.samples[True][f] 113 old = self.samples[False][f] 114 if not new or not old: continue 115 mdn_diff = abs(_median(new) - _median(old)) 116 _maybe_print('%s: %s=%r %s=%r mdn_diff=%r' % 117 (f, new_name, new, old_name, old, mdn_diff)) 118 s = bm_speedup.speedup(new, old, 1e-5) 119 if abs(s) > 3: 120 if mdn_diff > 0.5 or 'trickle' in f: 121 self.final[f] = '%+d%%' % s 122 return self.final.keys() 123 124 def skip(self): 125 return not self.final 126 127 def row(self, flds): 128 return [self.final[f] if f in self.final else '' for f in flds] 129 130 131def _read_json(filename, badjson_files, nonexistant_files): 132 stripped = ".".join(filename.split(".")[:-2]) 133 try: 134 with open(filename) as f: 135 r = f.read() 136 return json.loads(r) 137 except IOError, e: 138 if stripped in nonexistant_files: 139 nonexistant_files[stripped] += 1 140 else: 141 nonexistant_files[stripped] = 1 142 return None 143 except ValueError, e: 144 print r 145 if stripped in badjson_files: 146 badjson_files[stripped] += 1 147 else: 148 badjson_files[stripped] = 1 149 return None 150 151 152def fmt_dict(d): 153 return ''.join([" " + k + ": " + str(d[k]) + "\n" for k in d]) 154 155 156def diff(bms, loops, regex, track, old, new, counters): 157 benchmarks = collections.defaultdict(Benchmark) 158 159 badjson_files = {} 160 nonexistant_files = {} 161 for bm in bms: 162 for loop in range(0, loops): 163 for line in subprocess.check_output([ 164 'bm_diff_%s/opt/%s' % (old, bm), '--benchmark_list_tests', 165 '--benchmark_filter=%s' % regex 166 ]).splitlines(): 167 stripped_line = line.strip().replace("/", "_").replace( 168 "<", "_").replace(">", "_").replace(", ", "_") 169 js_new_opt = _read_json('%s.%s.opt.%s.%d.json' % 170 (bm, stripped_line, new, loop), 171 badjson_files, nonexistant_files) 172 js_old_opt = _read_json('%s.%s.opt.%s.%d.json' % 173 (bm, stripped_line, old, loop), 174 badjson_files, nonexistant_files) 175 if counters: 176 js_new_ctr = _read_json('%s.%s.counters.%s.%d.json' % 177 (bm, stripped_line, new, loop), 178 badjson_files, nonexistant_files) 179 js_old_ctr = _read_json('%s.%s.counters.%s.%d.json' % 180 (bm, stripped_line, old, loop), 181 badjson_files, nonexistant_files) 182 else: 183 js_new_ctr = None 184 js_old_ctr = None 185 186 for row in bm_json.expand_json(js_new_ctr, js_new_opt): 187 name = row['cpp_name'] 188 if name.endswith('_mean') or name.endswith('_stddev'): 189 continue 190 benchmarks[name].add_sample(track, row, True) 191 for row in bm_json.expand_json(js_old_ctr, js_old_opt): 192 name = row['cpp_name'] 193 if name.endswith('_mean') or name.endswith('_stddev'): 194 continue 195 benchmarks[name].add_sample(track, row, False) 196 197 really_interesting = set() 198 for name, bm in benchmarks.items(): 199 _maybe_print(name) 200 really_interesting.update(bm.process(track, new, old)) 201 fields = [f for f in track if f in really_interesting] 202 203 headers = ['Benchmark'] + fields 204 rows = [] 205 for name in sorted(benchmarks.keys()): 206 if benchmarks[name].skip(): continue 207 rows.append([name] + benchmarks[name].row(fields)) 208 note = None 209 if len(badjson_files): 210 note = 'Corrupt JSON data (indicates timeout or crash): \n%s' % fmt_dict( 211 badjson_files) 212 if len(nonexistant_files): 213 if note: 214 note += '\n\nMissing files (indicates new benchmark): \n%s' % fmt_dict( 215 nonexistant_files) 216 else: 217 note = '\n\nMissing files (indicates new benchmark): \n%s' % fmt_dict( 218 nonexistant_files) 219 if rows: 220 return tabulate.tabulate(rows, headers=headers, floatfmt='+.2f'), note 221 else: 222 return None, note 223 224 225if __name__ == '__main__': 226 args = _args() 227 diff, note = diff(args.benchmarks, args.loops, args.regex, args.track, 228 args.old, args.new, args.counters) 229 print('%s\n%s' % (note, diff if diff else "No performance differences")) 230