• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/env python
2# Copyright 2017 the V8 project authors. All rights reserved.
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5'''
6python %prog
7
8Compare perf trybot JSON files and output the results into a pleasing HTML page.
9Examples:
10  %prog -t "ia32 results" Result,../result.json Master,/path-to/master.json -o results.html
11  %prog -t "x64 results" ../result.json master.json -o results.html
12'''
13
14from collections import OrderedDict
15import json
16import math
17from argparse import ArgumentParser
18import os
19import shutil
20import sys
21import tempfile
22
23PERCENT_CONSIDERED_SIGNIFICANT = 0.5
24PROBABILITY_CONSIDERED_SIGNIFICANT = 0.02
25PROBABILITY_CONSIDERED_MEANINGLESS = 0.05
26
27class Statistics:
28  @staticmethod
29  def Mean(values):
30    return float(sum(values)) / len(values)
31
32  @staticmethod
33  def Variance(values, average):
34    return map(lambda x: (x - average) ** 2, values)
35
36  @staticmethod
37  def StandardDeviation(values, average):
38    return math.sqrt(Statistics.Mean(Statistics.Variance(values, average)))
39
40  @staticmethod
41  def ComputeZ(baseline_avg, baseline_sigma, mean, n):
42    if baseline_sigma == 0:
43      return 1000.0;
44    return abs((mean - baseline_avg) / (baseline_sigma / math.sqrt(n)))
45
46  # Values from http://www.fourmilab.ch/rpkp/experiments/analysis/zCalc.html
47  @staticmethod
48  def ComputeProbability(z):
49    if z > 2.575829: # p 0.005: two sided < 0.01
50      return 0
51    if z > 2.326348: # p 0.010
52      return 0.01
53    if z > 2.170091: # p 0.015
54      return 0.02
55    if z > 2.053749: # p 0.020
56      return 0.03
57    if z > 1.959964: # p 0.025: two sided < 0.05
58      return 0.04
59    if z > 1.880793: # p 0.030
60      return 0.05
61    if z > 1.811910: # p 0.035
62      return 0.06
63    if z > 1.750686: # p 0.040
64      return 0.07
65    if z > 1.695397: # p 0.045
66      return 0.08
67    if z > 1.644853: # p 0.050: two sided < 0.10
68      return 0.09
69    if z > 1.281551: # p 0.100: two sided < 0.20
70      return 0.10
71    return 0.20 # two sided p >= 0.20
72
73
74class ResultsDiff:
75  def __init__(self, significant, notable, percentage_string):
76    self.significant_ = significant
77    self.notable_ = notable
78    self.percentage_string_ = percentage_string
79
80  def percentage_string(self):
81    return self.percentage_string_;
82
83  def isSignificant(self):
84    return self.significant_
85
86  def isNotablyPositive(self):
87    return self.notable_ > 0
88
89  def isNotablyNegative(self):
90    return self.notable_ < 0
91
92
93class BenchmarkResult:
94  def __init__(self, units, count, result, sigma):
95    self.units_ = units
96    self.count_ = float(count)
97    self.result_ = float(result)
98    self.sigma_ = float(sigma)
99
100  def Compare(self, other):
101    if self.units_ != other.units_:
102      print ("Incompatible units: %s and %s" % (self.units_, other.units_))
103      sys.exit(1)
104
105    significant = False
106    notable = 0
107    percentage_string = ""
108    # compute notability and significance.
109    if self.units_ == "score":
110      compare_num = 100*self.result_/other.result_ - 100
111    else:
112      compare_num = 100*other.result_/self.result_ - 100
113    if abs(compare_num) > 0.1:
114      percentage_string = "%3.1f" % (compare_num)
115      z = Statistics.ComputeZ(other.result_, other.sigma_,
116                              self.result_, self.count_)
117      p = Statistics.ComputeProbability(z)
118      if p < PROBABILITY_CONSIDERED_SIGNIFICANT:
119        significant = True
120      if compare_num >= PERCENT_CONSIDERED_SIGNIFICANT:
121        notable = 1
122      elif compare_num <= -PERCENT_CONSIDERED_SIGNIFICANT:
123        notable = -1
124    return ResultsDiff(significant, notable, percentage_string)
125
126  def result(self):
127    return self.result_
128
129  def sigma(self):
130    return self.sigma_
131
132
133class Benchmark:
134  def __init__(self, name):
135    self.name_ = name
136    self.runs_ = {}
137
138  def name(self):
139    return self.name_
140
141  def getResult(self, run_name):
142    return self.runs_.get(run_name)
143
144  def appendResult(self, run_name, trace):
145    values = map(float, trace['results'])
146    count = len(values)
147    mean = Statistics.Mean(values)
148    stddev = float(trace.get('stddev') or
149                   Statistics.StandardDeviation(values, mean))
150    units = trace["units"]
151    # print run_name, units, count, mean, stddev
152    self.runs_[run_name] = BenchmarkResult(units, count, mean, stddev)
153
154
155class BenchmarkSuite:
156  def __init__(self, name):
157    self.name_ = name
158    self.benchmarks_ = {}
159
160  def SortedTestKeys(self):
161    keys = self.benchmarks_.keys()
162    keys.sort()
163    t = "Total"
164    if t in keys:
165      keys.remove(t)
166      keys.append(t)
167    return keys
168
169  def name(self):
170    return self.name_
171
172  def getBenchmark(self, benchmark_name):
173    benchmark_object = self.benchmarks_.get(benchmark_name)
174    if benchmark_object == None:
175      benchmark_object = Benchmark(benchmark_name)
176      self.benchmarks_[benchmark_name] = benchmark_object
177    return benchmark_object
178
179
180class ResultTableRenderer:
181  def __init__(self, output_file):
182    self.benchmarks_ = []
183    self.print_output_ = []
184    self.output_file_ = output_file
185
186  def Print(self, str_data):
187    self.print_output_.append(str_data)
188
189  def FlushOutput(self):
190    string_data = "\n".join(self.print_output_)
191    print_output = []
192    if self.output_file_:
193      # create a file
194      with open(self.output_file_, "w") as text_file:
195        text_file.write(string_data)
196    else:
197      print(string_data)
198
199  def bold(self, data):
200    return "<b>%s</b>" % data
201
202  def red(self, data):
203    return "<font color=\"red\">%s</font>" % data
204
205
206  def green(self, data):
207    return "<font color=\"green\">%s</font>" % data
208
209  def PrintHeader(self):
210    data = """<html>
211<head>
212<title>Output</title>
213<style type="text/css">
214/*
215Style inspired by Andy Ferra's gist at https://gist.github.com/andyferra/2554919
216*/
217body {
218  font-family: Helvetica, arial, sans-serif;
219  font-size: 14px;
220  line-height: 1.6;
221  padding-top: 10px;
222  padding-bottom: 10px;
223  background-color: white;
224  padding: 30px;
225}
226h1, h2, h3, h4, h5, h6 {
227  margin: 20px 0 10px;
228  padding: 0;
229  font-weight: bold;
230  -webkit-font-smoothing: antialiased;
231  cursor: text;
232  position: relative;
233}
234h1 {
235  font-size: 28px;
236  color: black;
237}
238
239h2 {
240  font-size: 24px;
241  border-bottom: 1px solid #cccccc;
242  color: black;
243}
244
245h3 {
246  font-size: 18px;
247}
248
249h4 {
250  font-size: 16px;
251}
252
253h5 {
254  font-size: 14px;
255}
256
257h6 {
258  color: #777777;
259  font-size: 14px;
260}
261
262p, blockquote, ul, ol, dl, li, table, pre {
263  margin: 15px 0;
264}
265
266li p.first {
267  display: inline-block;
268}
269
270ul, ol {
271  padding-left: 30px;
272}
273
274ul :first-child, ol :first-child {
275  margin-top: 0;
276}
277
278ul :last-child, ol :last-child {
279  margin-bottom: 0;
280}
281
282table {
283  padding: 0;
284}
285
286table tr {
287  border-top: 1px solid #cccccc;
288  background-color: white;
289  margin: 0;
290  padding: 0;
291}
292
293table tr:nth-child(2n) {
294  background-color: #f8f8f8;
295}
296
297table tr th {
298  font-weight: bold;
299  border: 1px solid #cccccc;
300  text-align: left;
301  margin: 0;
302  padding: 6px 13px;
303}
304table tr td {
305  border: 1px solid #cccccc;
306  text-align: right;
307  margin: 0;
308  padding: 6px 13px;
309}
310table tr td.name-column {
311  text-align: left;
312}
313table tr th :first-child, table tr td :first-child {
314  margin-top: 0;
315}
316table tr th :last-child, table tr td :last-child {
317  margin-bottom: 0;
318}
319</style>
320</head>
321<body>
322"""
323    self.Print(data)
324
325  def StartSuite(self, suite_name, run_names):
326    self.Print("<h2>")
327    self.Print("<a name=\"%s\">%s</a> <a href=\"#top\">(top)</a>" %
328               (suite_name, suite_name))
329    self.Print("</h2>");
330    self.Print("<table class=\"benchmark\">")
331    self.Print("<thead>")
332    self.Print("  <th>Test</th>")
333    main_run = None
334    for run_name in run_names:
335      self.Print("  <th>%s</th>" % run_name)
336      if main_run == None:
337        main_run = run_name
338      else:
339        self.Print("  <th>%</th>")
340    self.Print("</thead>")
341    self.Print("<tbody>")
342
343
344  def FinishSuite(self):
345    self.Print("</tbody>")
346    self.Print("</table>")
347
348
349  def StartBenchmark(self, benchmark_name):
350    self.Print("  <tr>")
351    self.Print("    <td class=\"name-column\">%s</td>" % benchmark_name)
352
353  def FinishBenchmark(self):
354    self.Print("  </tr>")
355
356
357  def PrintResult(self, run):
358    if run == None:
359      self.PrintEmptyCell()
360      return
361    self.Print("    <td>%3.1f</td>" % run.result())
362
363
364  def PrintComparison(self, run, main_run):
365    if run == None or main_run == None:
366      self.PrintEmptyCell()
367      return
368    diff = run.Compare(main_run)
369    res = diff.percentage_string()
370    if diff.isSignificant():
371      res = self.bold(res)
372    if diff.isNotablyPositive():
373      res = self.green(res)
374    elif diff.isNotablyNegative():
375      res = self.red(res)
376    self.Print("    <td>%s</td>" % res)
377
378
379  def PrintEmptyCell(self):
380    self.Print("    <td></td>")
381
382
383  def StartTOC(self, title):
384    self.Print("<h1>%s</h1>" % title)
385    self.Print("<ul>")
386
387  def FinishTOC(self):
388    self.Print("</ul>")
389
390  def PrintBenchmarkLink(self, benchmark):
391    self.Print("<li><a href=\"#" + benchmark + "\">" + benchmark + "</a></li>")
392
393  def PrintFooter(self):
394    data = """</body>
395</html>
396"""
397    self.Print(data)
398
399
400def Render(args):
401  benchmark_suites = {}
402  run_names = OrderedDict()
403
404  for json_file_list in args.json_file_list:
405    run_name = json_file_list[0]
406    if run_name.endswith(".json"):
407      # The first item in the list is also a file name
408      run_name = os.path.splitext(run_name)[0]
409      filenames = json_file_list
410    else:
411      filenames = json_file_list[1:]
412
413    for filename in filenames:
414      print ("Processing result set \"%s\", file: %s" % (run_name, filename))
415      with open(filename) as json_data:
416        data = json.load(json_data)
417
418      run_names[run_name] = 0
419
420      for error in data["errors"]:
421        print "Error:", error
422
423      for trace in data["traces"]:
424        suite_name = trace["graphs"][0]
425        benchmark_name = "/".join(trace["graphs"][1:])
426
427        benchmark_suite_object = benchmark_suites.get(suite_name)
428        if benchmark_suite_object == None:
429          benchmark_suite_object = BenchmarkSuite(suite_name)
430          benchmark_suites[suite_name] = benchmark_suite_object
431
432        benchmark_object = benchmark_suite_object.getBenchmark(benchmark_name)
433        benchmark_object.appendResult(run_name, trace);
434
435
436  renderer = ResultTableRenderer(args.output)
437  renderer.PrintHeader()
438
439  title = args.title or "Benchmark results"
440  renderer.StartTOC(title)
441  for suite_name, benchmark_suite_object in sorted(benchmark_suites.iteritems()):
442    renderer.PrintBenchmarkLink(suite_name)
443  renderer.FinishTOC()
444
445  for suite_name, benchmark_suite_object in sorted(benchmark_suites.iteritems()):
446    renderer.StartSuite(suite_name, run_names)
447    for benchmark_name in benchmark_suite_object.SortedTestKeys():
448      benchmark_object = benchmark_suite_object.getBenchmark(benchmark_name)
449      # print suite_name, benchmark_object.name()
450
451      renderer.StartBenchmark(benchmark_name)
452      main_run = None
453      main_result = None
454      for run_name in run_names:
455        result = benchmark_object.getResult(run_name)
456        renderer.PrintResult(result)
457        if main_run == None:
458          main_run = run_name
459          main_result = result
460        else:
461          renderer.PrintComparison(result, main_result)
462      renderer.FinishBenchmark()
463    renderer.FinishSuite()
464
465  renderer.PrintFooter()
466  renderer.FlushOutput()
467
468def CommaSeparatedList(arg):
469  return [x for x in arg.split(',')]
470
471if __name__ == '__main__':
472  parser = ArgumentParser(description="Compare perf trybot JSON files and " +
473                          "output the results into a pleasing HTML page.")
474  parser.add_argument("-t", "--title", dest="title",
475                      help="Optional title of the web page")
476  parser.add_argument("-o", "--output", dest="output",
477                      help="Write html output to this file rather than stdout")
478  parser.add_argument("json_file_list", nargs="+", type=CommaSeparatedList,
479                      help="[column name,]./path-to/result.json - a comma-separated" +
480                      " list of optional column name and paths to json files")
481
482  args = parser.parse_args()
483  Render(args)
484