• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/env python
2# Copyright 2017 the V8 project authors. All rights reserved.
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5'''
6python %prog
7
8Compare perf trybot JSON files and output the results into a pleasing HTML page.
9Examples:
10  %prog -t "ia32 results" Result,../result.json Master,/path-to/master.json -o results.html
11  %prog -t "x64 results" ../result.json master.json -o results.html
12'''
13
14# for py2/py3 compatibility
15from __future__ import print_function
16
17from collections import OrderedDict
18import json
19import math
20from argparse import ArgumentParser
21import os
22import shutil
23import sys
24import tempfile
25
26PERCENT_CONSIDERED_SIGNIFICANT = 0.5
27PROBABILITY_CONSIDERED_SIGNIFICANT = 0.02
28PROBABILITY_CONSIDERED_MEANINGLESS = 0.05
29
30class Statistics:
31  @staticmethod
32  def Mean(values):
33    return float(sum(values)) / len(values)
34
35  @staticmethod
36  def Variance(values, average):
37    return map(lambda x: (x - average) ** 2, values)
38
39  @staticmethod
40  def StandardDeviation(values, average):
41    return math.sqrt(Statistics.Mean(Statistics.Variance(values, average)))
42
43  @staticmethod
44  def ComputeZ(baseline_avg, baseline_sigma, mean, n):
45    if baseline_sigma == 0:
46      return 1000.0;
47    return abs((mean - baseline_avg) / (baseline_sigma / math.sqrt(n)))
48
49  # Values from http://www.fourmilab.ch/rpkp/experiments/analysis/zCalc.html
50  @staticmethod
51  def ComputeProbability(z):
52    if z > 2.575829: # p 0.005: two sided < 0.01
53      return 0
54    if z > 2.326348: # p 0.010
55      return 0.01
56    if z > 2.170091: # p 0.015
57      return 0.02
58    if z > 2.053749: # p 0.020
59      return 0.03
60    if z > 1.959964: # p 0.025: two sided < 0.05
61      return 0.04
62    if z > 1.880793: # p 0.030
63      return 0.05
64    if z > 1.811910: # p 0.035
65      return 0.06
66    if z > 1.750686: # p 0.040
67      return 0.07
68    if z > 1.695397: # p 0.045
69      return 0.08
70    if z > 1.644853: # p 0.050: two sided < 0.10
71      return 0.09
72    if z > 1.281551: # p 0.100: two sided < 0.20
73      return 0.10
74    return 0.20 # two sided p >= 0.20
75
76
77class ResultsDiff:
78  def __init__(self, significant, notable, percentage_string):
79    self.significant_ = significant
80    self.notable_ = notable
81    self.percentage_string_ = percentage_string
82
83  def percentage_string(self):
84    return self.percentage_string_;
85
86  def isSignificant(self):
87    return self.significant_
88
89  def isNotablyPositive(self):
90    return self.notable_ > 0
91
92  def isNotablyNegative(self):
93    return self.notable_ < 0
94
95
96class BenchmarkResult:
97  def __init__(self, units, count, result, sigma):
98    self.units_ = units
99    self.count_ = float(count)
100    self.result_ = float(result)
101    self.sigma_ = float(sigma)
102
103  def Compare(self, other):
104    if self.units_ != other.units_:
105      print ("Incompatible units: %s and %s" % (self.units_, other.units_))
106      sys.exit(1)
107
108    significant = False
109    notable = 0
110    percentage_string = ""
111    # compute notability and significance.
112    if self.units_ == "score":
113      compare_num = 100*self.result_/other.result_ - 100
114    else:
115      compare_num = 100*other.result_/self.result_ - 100
116    if abs(compare_num) > 0.1:
117      percentage_string = "%3.1f" % (compare_num)
118      z = Statistics.ComputeZ(other.result_, other.sigma_,
119                              self.result_, self.count_)
120      p = Statistics.ComputeProbability(z)
121      if p < PROBABILITY_CONSIDERED_SIGNIFICANT:
122        significant = True
123      if compare_num >= PERCENT_CONSIDERED_SIGNIFICANT:
124        notable = 1
125      elif compare_num <= -PERCENT_CONSIDERED_SIGNIFICANT:
126        notable = -1
127    return ResultsDiff(significant, notable, percentage_string)
128
129  def result(self):
130    return self.result_
131
132  def sigma(self):
133    return self.sigma_
134
135
136class Benchmark:
137  def __init__(self, name):
138    self.name_ = name
139    self.runs_ = {}
140
141  def name(self):
142    return self.name_
143
144  def getResult(self, run_name):
145    return self.runs_.get(run_name)
146
147  def appendResult(self, run_name, trace):
148    values = map(float, trace['results'])
149    count = len(values)
150    mean = Statistics.Mean(values)
151    stddev = float(trace.get('stddev') or
152                   Statistics.StandardDeviation(values, mean))
153    units = trace["units"]
154    # print run_name, units, count, mean, stddev
155    self.runs_[run_name] = BenchmarkResult(units, count, mean, stddev)
156
157
158class BenchmarkSuite:
159  def __init__(self, name):
160    self.name_ = name
161    self.benchmarks_ = {}
162
163  def SortedTestKeys(self):
164    keys = self.benchmarks_.keys()
165    keys.sort()
166    t = "Total"
167    if t in keys:
168      keys.remove(t)
169      keys.append(t)
170    return keys
171
172  def name(self):
173    return self.name_
174
175  def getBenchmark(self, benchmark_name):
176    benchmark_object = self.benchmarks_.get(benchmark_name)
177    if benchmark_object == None:
178      benchmark_object = Benchmark(benchmark_name)
179      self.benchmarks_[benchmark_name] = benchmark_object
180    return benchmark_object
181
182
183class ResultTableRenderer:
184  def __init__(self, output_file):
185    self.benchmarks_ = []
186    self.print_output_ = []
187    self.output_file_ = output_file
188
189  def Print(self, str_data):
190    self.print_output_.append(str_data)
191
192  def FlushOutput(self):
193    string_data = "\n".join(self.print_output_)
194    print_output = []
195    if self.output_file_:
196      # create a file
197      with open(self.output_file_, "w") as text_file:
198        text_file.write(string_data)
199    else:
200      print(string_data)
201
202  def bold(self, data):
203    return "<b>%s</b>" % data
204
205  def red(self, data):
206    return "<font color=\"red\">%s</font>" % data
207
208
209  def green(self, data):
210    return "<font color=\"green\">%s</font>" % data
211
212  def PrintHeader(self):
213    data = """<html>
214<head>
215<title>Output</title>
216<style type="text/css">
217/*
218Style inspired by Andy Ferra's gist at https://gist.github.com/andyferra/2554919
219*/
220body {
221  font-family: Helvetica, arial, sans-serif;
222  font-size: 14px;
223  line-height: 1.6;
224  padding-top: 10px;
225  padding-bottom: 10px;
226  background-color: white;
227  padding: 30px;
228}
229h1, h2, h3, h4, h5, h6 {
230  margin: 20px 0 10px;
231  padding: 0;
232  font-weight: bold;
233  -webkit-font-smoothing: antialiased;
234  cursor: text;
235  position: relative;
236}
237h1 {
238  font-size: 28px;
239  color: black;
240}
241
242h2 {
243  font-size: 24px;
244  border-bottom: 1px solid #cccccc;
245  color: black;
246}
247
248h3 {
249  font-size: 18px;
250}
251
252h4 {
253  font-size: 16px;
254}
255
256h5 {
257  font-size: 14px;
258}
259
260h6 {
261  color: #777777;
262  font-size: 14px;
263}
264
265p, blockquote, ul, ol, dl, li, table, pre {
266  margin: 15px 0;
267}
268
269li p.first {
270  display: inline-block;
271}
272
273ul, ol {
274  padding-left: 30px;
275}
276
277ul :first-child, ol :first-child {
278  margin-top: 0;
279}
280
281ul :last-child, ol :last-child {
282  margin-bottom: 0;
283}
284
285table {
286  padding: 0;
287}
288
289table tr {
290  border-top: 1px solid #cccccc;
291  background-color: white;
292  margin: 0;
293  padding: 0;
294}
295
296table tr:nth-child(2n) {
297  background-color: #f8f8f8;
298}
299
300table tr th {
301  font-weight: bold;
302  border: 1px solid #cccccc;
303  text-align: left;
304  margin: 0;
305  padding: 6px 13px;
306}
307table tr td {
308  border: 1px solid #cccccc;
309  text-align: right;
310  margin: 0;
311  padding: 6px 13px;
312}
313table tr td.name-column {
314  text-align: left;
315}
316table tr th :first-child, table tr td :first-child {
317  margin-top: 0;
318}
319table tr th :last-child, table tr td :last-child {
320  margin-bottom: 0;
321}
322</style>
323</head>
324<body>
325"""
326    self.Print(data)
327
328  def StartSuite(self, suite_name, run_names):
329    self.Print("<h2>")
330    self.Print("<a name=\"%s\">%s</a> <a href=\"#top\">(top)</a>" %
331               (suite_name, suite_name))
332    self.Print("</h2>");
333    self.Print("<table class=\"benchmark\">")
334    self.Print("<thead>")
335    self.Print("  <th>Test</th>")
336    main_run = None
337    for run_name in run_names:
338      self.Print("  <th>%s</th>" % run_name)
339      if main_run == None:
340        main_run = run_name
341      else:
342        self.Print("  <th>%</th>")
343    self.Print("</thead>")
344    self.Print("<tbody>")
345
346
347  def FinishSuite(self):
348    self.Print("</tbody>")
349    self.Print("</table>")
350
351
352  def StartBenchmark(self, benchmark_name):
353    self.Print("  <tr>")
354    self.Print("    <td class=\"name-column\">%s</td>" % benchmark_name)
355
356  def FinishBenchmark(self):
357    self.Print("  </tr>")
358
359
360  def PrintResult(self, run):
361    if run == None:
362      self.PrintEmptyCell()
363      return
364    self.Print("    <td>%3.1f</td>" % run.result())
365
366
367  def PrintComparison(self, run, main_run):
368    if run == None or main_run == None:
369      self.PrintEmptyCell()
370      return
371    diff = run.Compare(main_run)
372    res = diff.percentage_string()
373    if diff.isSignificant():
374      res = self.bold(res)
375    if diff.isNotablyPositive():
376      res = self.green(res)
377    elif diff.isNotablyNegative():
378      res = self.red(res)
379    self.Print("    <td>%s</td>" % res)
380
381
382  def PrintEmptyCell(self):
383    self.Print("    <td></td>")
384
385
386  def StartTOC(self, title):
387    self.Print("<h1>%s</h1>" % title)
388    self.Print("<ul>")
389
390  def FinishTOC(self):
391    self.Print("</ul>")
392
393  def PrintBenchmarkLink(self, benchmark):
394    self.Print("<li><a href=\"#" + benchmark + "\">" + benchmark + "</a></li>")
395
396  def PrintFooter(self):
397    data = """</body>
398</html>
399"""
400    self.Print(data)
401
402
403def Render(args):
404  benchmark_suites = {}
405  run_names = OrderedDict()
406
407  for json_file_list in args.json_file_list:
408    run_name = json_file_list[0]
409    if run_name.endswith(".json"):
410      # The first item in the list is also a file name
411      run_name = os.path.splitext(run_name)[0]
412      filenames = json_file_list
413    else:
414      filenames = json_file_list[1:]
415
416    for filename in filenames:
417      print ("Processing result set \"%s\", file: %s" % (run_name, filename))
418      with open(filename) as json_data:
419        data = json.load(json_data)
420
421      run_names[run_name] = 0
422
423      for error in data["errors"]:
424        print("Error:", error)
425
426      for trace in data["traces"]:
427        suite_name = trace["graphs"][0]
428        benchmark_name = "/".join(trace["graphs"][1:])
429
430        benchmark_suite_object = benchmark_suites.get(suite_name)
431        if benchmark_suite_object == None:
432          benchmark_suite_object = BenchmarkSuite(suite_name)
433          benchmark_suites[suite_name] = benchmark_suite_object
434
435        benchmark_object = benchmark_suite_object.getBenchmark(benchmark_name)
436        benchmark_object.appendResult(run_name, trace);
437
438
439  renderer = ResultTableRenderer(args.output)
440  renderer.PrintHeader()
441
442  title = args.title or "Benchmark results"
443  renderer.StartTOC(title)
444  for suite_name, benchmark_suite_object in sorted(benchmark_suites.iteritems()):
445    renderer.PrintBenchmarkLink(suite_name)
446  renderer.FinishTOC()
447
448  for suite_name, benchmark_suite_object in sorted(benchmark_suites.iteritems()):
449    renderer.StartSuite(suite_name, run_names)
450    for benchmark_name in benchmark_suite_object.SortedTestKeys():
451      benchmark_object = benchmark_suite_object.getBenchmark(benchmark_name)
452      # print suite_name, benchmark_object.name()
453
454      renderer.StartBenchmark(benchmark_name)
455      main_run = None
456      main_result = None
457      for run_name in run_names:
458        result = benchmark_object.getResult(run_name)
459        renderer.PrintResult(result)
460        if main_run == None:
461          main_run = run_name
462          main_result = result
463        else:
464          renderer.PrintComparison(result, main_result)
465      renderer.FinishBenchmark()
466    renderer.FinishSuite()
467
468  renderer.PrintFooter()
469  renderer.FlushOutput()
470
471def CommaSeparatedList(arg):
472  return [x for x in arg.split(',')]
473
474if __name__ == '__main__':
475  parser = ArgumentParser(description="Compare perf trybot JSON files and " +
476                          "output the results into a pleasing HTML page.")
477  parser.add_argument("-t", "--title", dest="title",
478                      help="Optional title of the web page")
479  parser.add_argument("-o", "--output", dest="output",
480                      help="Write html output to this file rather than stdout")
481  parser.add_argument("json_file_list", nargs="+", type=CommaSeparatedList,
482                      help="[column name,]./path-to/result.json - a comma-separated" +
483                      " list of optional column name and paths to json files")
484
485  args = parser.parse_args()
486  Render(args)
487