1#!/usr/bin/env python 2# Copyright 2017 the V8 project authors. All rights reserved. 3# Use of this source code is governed by a BSD-style license that can be 4# found in the LICENSE file. 5''' 6python %prog 7 8Compare perf trybot JSON files and output the results into a pleasing HTML page. 9Examples: 10 %prog -t "ia32 results" Result,../result.json Master,/path-to/master.json -o results.html 11 %prog -t "x64 results" ../result.json master.json -o results.html 12''' 13 14from collections import OrderedDict 15import json 16import math 17from argparse import ArgumentParser 18import os 19import shutil 20import sys 21import tempfile 22 23PERCENT_CONSIDERED_SIGNIFICANT = 0.5 24PROBABILITY_CONSIDERED_SIGNIFICANT = 0.02 25PROBABILITY_CONSIDERED_MEANINGLESS = 0.05 26 27class Statistics: 28 @staticmethod 29 def Mean(values): 30 return float(sum(values)) / len(values) 31 32 @staticmethod 33 def Variance(values, average): 34 return map(lambda x: (x - average) ** 2, values) 35 36 @staticmethod 37 def StandardDeviation(values, average): 38 return math.sqrt(Statistics.Mean(Statistics.Variance(values, average))) 39 40 @staticmethod 41 def ComputeZ(baseline_avg, baseline_sigma, mean, n): 42 if baseline_sigma == 0: 43 return 1000.0; 44 return abs((mean - baseline_avg) / (baseline_sigma / math.sqrt(n))) 45 46 # Values from http://www.fourmilab.ch/rpkp/experiments/analysis/zCalc.html 47 @staticmethod 48 def ComputeProbability(z): 49 if z > 2.575829: # p 0.005: two sided < 0.01 50 return 0 51 if z > 2.326348: # p 0.010 52 return 0.01 53 if z > 2.170091: # p 0.015 54 return 0.02 55 if z > 2.053749: # p 0.020 56 return 0.03 57 if z > 1.959964: # p 0.025: two sided < 0.05 58 return 0.04 59 if z > 1.880793: # p 0.030 60 return 0.05 61 if z > 1.811910: # p 0.035 62 return 0.06 63 if z > 1.750686: # p 0.040 64 return 0.07 65 if z > 1.695397: # p 0.045 66 return 0.08 67 if z > 1.644853: # p 0.050: two sided < 0.10 68 return 0.09 69 if z > 1.281551: # p 0.100: two sided < 0.20 70 return 0.10 71 return 0.20 # two sided p >= 0.20 72 73 74class ResultsDiff: 75 def __init__(self, significant, notable, percentage_string): 76 self.significant_ = significant 77 self.notable_ = notable 78 self.percentage_string_ = percentage_string 79 80 def percentage_string(self): 81 return self.percentage_string_; 82 83 def isSignificant(self): 84 return self.significant_ 85 86 def isNotablyPositive(self): 87 return self.notable_ > 0 88 89 def isNotablyNegative(self): 90 return self.notable_ < 0 91 92 93class BenchmarkResult: 94 def __init__(self, units, count, result, sigma): 95 self.units_ = units 96 self.count_ = float(count) 97 self.result_ = float(result) 98 self.sigma_ = float(sigma) 99 100 def Compare(self, other): 101 if self.units_ != other.units_: 102 print ("Incompatible units: %s and %s" % (self.units_, other.units_)) 103 sys.exit(1) 104 105 significant = False 106 notable = 0 107 percentage_string = "" 108 # compute notability and significance. 109 if self.units_ == "score": 110 compare_num = 100*self.result_/other.result_ - 100 111 else: 112 compare_num = 100*other.result_/self.result_ - 100 113 if abs(compare_num) > 0.1: 114 percentage_string = "%3.1f" % (compare_num) 115 z = Statistics.ComputeZ(other.result_, other.sigma_, 116 self.result_, self.count_) 117 p = Statistics.ComputeProbability(z) 118 if p < PROBABILITY_CONSIDERED_SIGNIFICANT: 119 significant = True 120 if compare_num >= PERCENT_CONSIDERED_SIGNIFICANT: 121 notable = 1 122 elif compare_num <= -PERCENT_CONSIDERED_SIGNIFICANT: 123 notable = -1 124 return ResultsDiff(significant, notable, percentage_string) 125 126 def result(self): 127 return self.result_ 128 129 def sigma(self): 130 return self.sigma_ 131 132 133class Benchmark: 134 def __init__(self, name): 135 self.name_ = name 136 self.runs_ = {} 137 138 def name(self): 139 return self.name_ 140 141 def getResult(self, run_name): 142 return self.runs_.get(run_name) 143 144 def appendResult(self, run_name, trace): 145 values = map(float, trace['results']) 146 count = len(values) 147 mean = Statistics.Mean(values) 148 stddev = float(trace.get('stddev') or 149 Statistics.StandardDeviation(values, mean)) 150 units = trace["units"] 151 # print run_name, units, count, mean, stddev 152 self.runs_[run_name] = BenchmarkResult(units, count, mean, stddev) 153 154 155class BenchmarkSuite: 156 def __init__(self, name): 157 self.name_ = name 158 self.benchmarks_ = {} 159 160 def SortedTestKeys(self): 161 keys = self.benchmarks_.keys() 162 keys.sort() 163 t = "Total" 164 if t in keys: 165 keys.remove(t) 166 keys.append(t) 167 return keys 168 169 def name(self): 170 return self.name_ 171 172 def getBenchmark(self, benchmark_name): 173 benchmark_object = self.benchmarks_.get(benchmark_name) 174 if benchmark_object == None: 175 benchmark_object = Benchmark(benchmark_name) 176 self.benchmarks_[benchmark_name] = benchmark_object 177 return benchmark_object 178 179 180class ResultTableRenderer: 181 def __init__(self, output_file): 182 self.benchmarks_ = [] 183 self.print_output_ = [] 184 self.output_file_ = output_file 185 186 def Print(self, str_data): 187 self.print_output_.append(str_data) 188 189 def FlushOutput(self): 190 string_data = "\n".join(self.print_output_) 191 print_output = [] 192 if self.output_file_: 193 # create a file 194 with open(self.output_file_, "w") as text_file: 195 text_file.write(string_data) 196 else: 197 print(string_data) 198 199 def bold(self, data): 200 return "<b>%s</b>" % data 201 202 def red(self, data): 203 return "<font color=\"red\">%s</font>" % data 204 205 206 def green(self, data): 207 return "<font color=\"green\">%s</font>" % data 208 209 def PrintHeader(self): 210 data = """<html> 211<head> 212<title>Output</title> 213<style type="text/css"> 214/* 215Style inspired by Andy Ferra's gist at https://gist.github.com/andyferra/2554919 216*/ 217body { 218 font-family: Helvetica, arial, sans-serif; 219 font-size: 14px; 220 line-height: 1.6; 221 padding-top: 10px; 222 padding-bottom: 10px; 223 background-color: white; 224 padding: 30px; 225} 226h1, h2, h3, h4, h5, h6 { 227 margin: 20px 0 10px; 228 padding: 0; 229 font-weight: bold; 230 -webkit-font-smoothing: antialiased; 231 cursor: text; 232 position: relative; 233} 234h1 { 235 font-size: 28px; 236 color: black; 237} 238 239h2 { 240 font-size: 24px; 241 border-bottom: 1px solid #cccccc; 242 color: black; 243} 244 245h3 { 246 font-size: 18px; 247} 248 249h4 { 250 font-size: 16px; 251} 252 253h5 { 254 font-size: 14px; 255} 256 257h6 { 258 color: #777777; 259 font-size: 14px; 260} 261 262p, blockquote, ul, ol, dl, li, table, pre { 263 margin: 15px 0; 264} 265 266li p.first { 267 display: inline-block; 268} 269 270ul, ol { 271 padding-left: 30px; 272} 273 274ul :first-child, ol :first-child { 275 margin-top: 0; 276} 277 278ul :last-child, ol :last-child { 279 margin-bottom: 0; 280} 281 282table { 283 padding: 0; 284} 285 286table tr { 287 border-top: 1px solid #cccccc; 288 background-color: white; 289 margin: 0; 290 padding: 0; 291} 292 293table tr:nth-child(2n) { 294 background-color: #f8f8f8; 295} 296 297table tr th { 298 font-weight: bold; 299 border: 1px solid #cccccc; 300 text-align: left; 301 margin: 0; 302 padding: 6px 13px; 303} 304table tr td { 305 border: 1px solid #cccccc; 306 text-align: right; 307 margin: 0; 308 padding: 6px 13px; 309} 310table tr td.name-column { 311 text-align: left; 312} 313table tr th :first-child, table tr td :first-child { 314 margin-top: 0; 315} 316table tr th :last-child, table tr td :last-child { 317 margin-bottom: 0; 318} 319</style> 320</head> 321<body> 322""" 323 self.Print(data) 324 325 def StartSuite(self, suite_name, run_names): 326 self.Print("<h2>") 327 self.Print("<a name=\"%s\">%s</a> <a href=\"#top\">(top)</a>" % 328 (suite_name, suite_name)) 329 self.Print("</h2>"); 330 self.Print("<table class=\"benchmark\">") 331 self.Print("<thead>") 332 self.Print(" <th>Test</th>") 333 main_run = None 334 for run_name in run_names: 335 self.Print(" <th>%s</th>" % run_name) 336 if main_run == None: 337 main_run = run_name 338 else: 339 self.Print(" <th>%</th>") 340 self.Print("</thead>") 341 self.Print("<tbody>") 342 343 344 def FinishSuite(self): 345 self.Print("</tbody>") 346 self.Print("</table>") 347 348 349 def StartBenchmark(self, benchmark_name): 350 self.Print(" <tr>") 351 self.Print(" <td class=\"name-column\">%s</td>" % benchmark_name) 352 353 def FinishBenchmark(self): 354 self.Print(" </tr>") 355 356 357 def PrintResult(self, run): 358 if run == None: 359 self.PrintEmptyCell() 360 return 361 self.Print(" <td>%3.1f</td>" % run.result()) 362 363 364 def PrintComparison(self, run, main_run): 365 if run == None or main_run == None: 366 self.PrintEmptyCell() 367 return 368 diff = run.Compare(main_run) 369 res = diff.percentage_string() 370 if diff.isSignificant(): 371 res = self.bold(res) 372 if diff.isNotablyPositive(): 373 res = self.green(res) 374 elif diff.isNotablyNegative(): 375 res = self.red(res) 376 self.Print(" <td>%s</td>" % res) 377 378 379 def PrintEmptyCell(self): 380 self.Print(" <td></td>") 381 382 383 def StartTOC(self, title): 384 self.Print("<h1>%s</h1>" % title) 385 self.Print("<ul>") 386 387 def FinishTOC(self): 388 self.Print("</ul>") 389 390 def PrintBenchmarkLink(self, benchmark): 391 self.Print("<li><a href=\"#" + benchmark + "\">" + benchmark + "</a></li>") 392 393 def PrintFooter(self): 394 data = """</body> 395</html> 396""" 397 self.Print(data) 398 399 400def Render(args): 401 benchmark_suites = {} 402 run_names = OrderedDict() 403 404 for json_file_list in args.json_file_list: 405 run_name = json_file_list[0] 406 if run_name.endswith(".json"): 407 # The first item in the list is also a file name 408 run_name = os.path.splitext(run_name)[0] 409 filenames = json_file_list 410 else: 411 filenames = json_file_list[1:] 412 413 for filename in filenames: 414 print ("Processing result set \"%s\", file: %s" % (run_name, filename)) 415 with open(filename) as json_data: 416 data = json.load(json_data) 417 418 run_names[run_name] = 0 419 420 for error in data["errors"]: 421 print "Error:", error 422 423 for trace in data["traces"]: 424 suite_name = trace["graphs"][0] 425 benchmark_name = "/".join(trace["graphs"][1:]) 426 427 benchmark_suite_object = benchmark_suites.get(suite_name) 428 if benchmark_suite_object == None: 429 benchmark_suite_object = BenchmarkSuite(suite_name) 430 benchmark_suites[suite_name] = benchmark_suite_object 431 432 benchmark_object = benchmark_suite_object.getBenchmark(benchmark_name) 433 benchmark_object.appendResult(run_name, trace); 434 435 436 renderer = ResultTableRenderer(args.output) 437 renderer.PrintHeader() 438 439 title = args.title or "Benchmark results" 440 renderer.StartTOC(title) 441 for suite_name, benchmark_suite_object in sorted(benchmark_suites.iteritems()): 442 renderer.PrintBenchmarkLink(suite_name) 443 renderer.FinishTOC() 444 445 for suite_name, benchmark_suite_object in sorted(benchmark_suites.iteritems()): 446 renderer.StartSuite(suite_name, run_names) 447 for benchmark_name in benchmark_suite_object.SortedTestKeys(): 448 benchmark_object = benchmark_suite_object.getBenchmark(benchmark_name) 449 # print suite_name, benchmark_object.name() 450 451 renderer.StartBenchmark(benchmark_name) 452 main_run = None 453 main_result = None 454 for run_name in run_names: 455 result = benchmark_object.getResult(run_name) 456 renderer.PrintResult(result) 457 if main_run == None: 458 main_run = run_name 459 main_result = result 460 else: 461 renderer.PrintComparison(result, main_result) 462 renderer.FinishBenchmark() 463 renderer.FinishSuite() 464 465 renderer.PrintFooter() 466 renderer.FlushOutput() 467 468def CommaSeparatedList(arg): 469 return [x for x in arg.split(',')] 470 471if __name__ == '__main__': 472 parser = ArgumentParser(description="Compare perf trybot JSON files and " + 473 "output the results into a pleasing HTML page.") 474 parser.add_argument("-t", "--title", dest="title", 475 help="Optional title of the web page") 476 parser.add_argument("-o", "--output", dest="output", 477 help="Write html output to this file rather than stdout") 478 parser.add_argument("json_file_list", nargs="+", type=CommaSeparatedList, 479 help="[column name,]./path-to/result.json - a comma-separated" + 480 " list of optional column name and paths to json files") 481 482 args = parser.parse_args() 483 Render(args) 484