1 #!/usr/bin/python2 2 # 3 # Copyright 2010 Google Inc. All Rights Reserved. 4 """Script to compare ChromeOS benchmarks 5 6 Inputs: 7 <perflab-output directory 1 - baseline> 8 <perflab-output directory 2 - results> 9 --csv - comma separated results 10 11 This script doesn't really know much about benchmarks. It simply looks for 12 similarly names directories and a results.txt file, and compares 13 the results and presents it, along with a geometric mean. 14 15 """ 16 17 from __future__ import print_function 18 19 __author__ = 'bjanakiraman@google.com (Bhaskar Janakiraman)' 20 21 import glob 22 import math 23 import argparse 24 import re 25 import sys 26 27 from cros_utils import command_executer 28 29 BENCHDIRS = ('%s/default/default/*/gcc-4.4.3-glibc-2.11.1-grte-k8-opt/ref/*' 30 '/results.txt') 31 32 # Common initializations 33 cmd_executer = command_executer.GetCommandExecuter() 34 35 36 def Usage(parser, message): 37 print('ERROR: %s' % message) 38 parser.print_help() 39 sys.exit(0) 40 41 42 def GetStats(in_file): 43 """Return stats from file""" 44 f = open(in_file, 'r') 45 pairs = [] 46 for l in f: 47 line = l.strip() 48 # Look for match lines like the following: 49 # METRIC isolated TotalTime_ms (down, scalar) trial_run_0: ['1524.4'] 50 # METRIC isolated isolated_walltime (down, scalar) trial_run_0: \ 51 # ['167.407445192'] 52 m = re.match(r"METRIC\s+isolated\s+(\S+).*\['(\d+(?:\.\d+)?)'\]", line) 53 if not m: 54 continue 55 metric = m.group(1) 56 if re.match(r'isolated_walltime', metric): 57 continue 58 59 value = float(m.group(2)) 60 pairs.append((metric, value)) 61 62 return dict(pairs) 63 64 65 def PrintDash(n): 66 tmpstr = '' 67 for _ in range(n): 68 tmpstr += '-' 69 print(tmpstr) 70 71 72 def PrintHeaderCSV(hdr): 73 tmpstr = '' 74 for i in range(len(hdr)): 75 if tmpstr != '': 76 tmpstr += ',' 77 tmpstr += hdr[i] 78 print(tmpstr) 79 80 81 def PrintHeader(hdr): 82 tot_len = len(hdr) 83 PrintDash(tot_len * 15) 84 85 tmpstr = '' 86 for i in range(len(hdr)): 87 tmpstr += '%15.15s' % hdr[i] 88 89 print(tmpstr) 90 PrintDash(tot_len * 15) 91 92 93 def Main(argv): 94 """Compare Benchmarks.""" 95 # Common initializations 96 97 parser = argparse.ArgumentParser() 98 parser.add_argument('-c', 99 '--csv', 100 dest='csv_output', 101 action='store_true', 102 default=False, 103 help='Output in csv form.') 104 parser.add_argument('args', nargs='+', help='positional arguments: ' 105 '<baseline-output-dir> <results-output-dir>') 106 107 options = parser.parse_args(argv[1:]) 108 109 # validate args 110 if len(options.args) != 2: 111 Usage(parser, 'Needs <baseline output dir> <results output dir>') 112 113 base_dir = options.args[0] 114 res_dir = options.args[1] 115 116 # find res benchmarks that have results 117 resbenches_glob = BENCHDIRS % res_dir 118 resbenches = glob.glob(resbenches_glob) 119 120 basebenches_glob = BENCHDIRS % base_dir 121 basebenches = glob.glob(basebenches_glob) 122 123 to_compare = [] 124 for resbench in resbenches: 125 tmp = resbench.replace(res_dir, base_dir, 1) 126 if tmp in basebenches: 127 to_compare.append((resbench, tmp)) 128 129 for (resfile, basefile) in to_compare: 130 stats = GetStats(resfile) 131 basestats = GetStats(basefile) 132 # Print a header 133 # benchname (remove results.txt), basetime, restime, %speed-up 134 hdr = [] 135 benchname = re.split('/', resfile)[-2:-1][0] 136 benchname = benchname.replace('chromeos__', '', 1) 137 hdr.append(benchname) 138 hdr.append('basetime') 139 hdr.append('restime') 140 hdr.append('%speed up') 141 if options.csv_output: 142 PrintHeaderCSV(hdr) 143 else: 144 PrintHeader(hdr) 145 146 # For geomean computations 147 prod = 1.0 148 count = 0 149 for key in stats.keys(): 150 if key in basestats.keys(): 151 # ignore very small values. 152 if stats[key] < 0.01: 153 continue 154 count = count + 1 155 prod = prod * (stats[key] / basestats[key]) 156 speedup = (basestats[key] - stats[key]) / basestats[key] 157 speedup = speedup * 100.0 158 if options.csv_output: 159 print('%s,%f,%f,%f' % (key, basestats[key], stats[key], speedup)) 160 else: 161 print('%15.15s%15.2f%15.2f%14.2f%%' % (key, basestats[key], 162 stats[key], speedup)) 163 164 prod = math.exp(1.0 / count * math.log(prod)) 165 prod = (1.0 - prod) * 100 166 if options.csv_output: 167 print('%s,,,%f' % ('Geomean', prod)) 168 else: 169 print('%15.15s%15.15s%15.15s%14.2f%%' % ('Geomean', '', '', prod)) 170 print('') 171 return 0 172 173 174 if __name__ == '__main__': 175 retval = Main(sys.argv) 176 sys.exit(retval) 177