• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/env python
2# Copyright (c) 2012 The Chromium Authors. All rights reserved.
3# Use of this source code is governed by a BSD-style license that can be found
4# in the LICENSE file.
5
6""" Analyze recent bench data from graphs, and output suggested ranges.
7
8This script reads and parses Skia benchmark values from the xhtml files
9generated by bench_graph_svg.py, and outputs an html file containing suggested
10bench ranges to use in bench_expectations.txt, with analytical plots.
11"""
12
13__author__ = 'bensong@google.com (Ben Chen)'
14
15import getopt
16import math
17import re
18import sys
19import urllib
20from datetime import datetime
21
22
23# Constants for calculating suggested bench ranges.
24WINDOW = 5  # Moving average sliding window size.
25# We use moving average as expected bench value, and calculate average Variance
26# of bench from the moving average. Set range to be [X_UB * Variance above
27# moving average, X_LB * Variance below moving average] of latest revision.
28X_UB = 4.0
29X_LB = 5.0
30
31# List of platforms.
32PLATFORMS = ['GalaxyNexus_4-1_Float_Release',
33             'Mac_Float_NoDebug_32',
34             'Mac_Float_NoDebug_64',
35             'MacMiniLion_Float_NoDebug_32',
36             'MacMiniLion_Float_NoDebug_64',
37             'Nexus7_4-1_Float_Release',
38             'Shuttle_Ubuntu12_ATI5770_Float_Release_64',
39             'Shuttle_Win7_Intel_Float_Release_32',
40             'Shuttle_Win7_Intel_Float_Release_64',
41             'Xoom_4-1_Float_Release'
42            ]
43
44# List of bench representation algorithms. Flag "-a" is chosen from the list.
45ALGS = ['25th', 'avg', 'med', 'min']
46
47# Regular expressions for parsing bench/revision values.
48HEIGHT_RE = 'height (\d+\.\d+) corresponds to bench value (\d+\.\d+).-->'
49REV_RE = '<rect id="(\d+)" x="(\d+\.\d+)" y="'  # Revision corresponding x.
50LINE_RE = '<polyline id="(.*)".*points="(.*)"/>'  # Bench value lines.
51
52# Bench graph url pattern.
53INPUT_URL_TEMPLATE = ('http://chromium-skia-gm.commondatastorage.googleapis.com'
54                      '/graph-Skia_%s-2.xhtml')
55
56# Output HTML elements and templates.
57HTML_HEAD = ('<html><head><title>Skia Bench Expected Ranges</title>'
58             '<script type="text/javascript" src="https://skia.googlecode.com/'
59             'svn/buildbot/dygraph-combined.js"></script></head><body>Please '
60             'adjust values as appropriate and update benches to monitor in '
61             'bench/bench_expectations.txt.<br><br>')
62HTML_SUFFIX = '</body></html>'
63GRAPH_PREFIX = ('<br>%s<br><div id="%s" style="width:400px;height:200px"></div>'
64                '<script type="text/javascript">g%s=new Dygraph('
65                'document.getElementById("%s"),"rev,bench,alert\\n')
66GRAPH_SUFFIX = ('",{customBars: true,"alert":{strokeWidth:0.0,drawPoints:true,'
67                'pointSize:4,highlightCircleSize:6}});</script>')
68
69
70def Usage():
71  """Prints flag usage information."""
72  print '-a <representation-algorithm>: defaults to "25th".'
73  print '  If set, must be one of the list element in ALGS defined above.'
74  print '-b <bench-prefix>: prefix of matching bench names to analyze.'
75  print '  Only include benchmarks whose names start with this string.'
76  print '  Cannot be empty, because there are too many benches overall.'
77  print '-o <file>: html output filename. Output to STDOUT if not set.'
78  print '-p <platform-prefix>: prefix of platform names to analyze.'
79  print '  PLATFORMS has list of matching candidates. Matches all if not set.'
80
81def GetBenchValues(page, bench_prefix):
82  """Returns a dict of matching bench values from the given xhtml page.
83  Args:
84    page: substring used to construct the specific bench graph URL to fetch.
85    bench_prefix: only benches starting with this string will be included.
86
87  Returns:
88    a dict mapping benchmark name and revision combinations to bench values.
89  """
90  height = None
91  max_bench = None
92  height_scale = None
93  revisions = []
94  x_axes = []  # For calculating corresponding revisions.
95  val_dic = {}  # dict[bench_name][revision] -> bench_value
96
97  lines = urllib.urlopen(INPUT_URL_TEMPLATE % page).readlines()
98  for line in lines:
99    height_match = re.search(HEIGHT_RE, line)
100    if height_match:
101      height = float(height_match.group(1))
102      max_bench = float(height_match.group(2))
103      height_scale = max_bench / height
104
105    rev_match = re.search(REV_RE, line)
106    if rev_match:
107      revisions.append(int(rev_match.group(1)))
108      x_axes.append(float(rev_match.group(2)))
109
110    line_match = re.search(LINE_RE, line)
111    if not line_match:
112      continue
113    bench = line_match.group(1)
114    bench = bench[:bench.find('_{')]
115    if not bench.startswith(bench_prefix):
116      continue
117    if bench not in val_dic:
118      val_dic[bench] = {}
119
120    vals = line_match.group(2).strip().split(' ')
121    if len(vals) < WINDOW:  # Too few bench data points; skip.
122      continue
123    for val in vals:
124      x, y = [float(i) for i in val.split(',')]
125      for i in range(len(x_axes)):
126        if x <= x_axes[i]:  # Found corresponding bench revision.
127          break
128      val_dic[bench][revisions[i]] = float(
129          '%.3f' % ((height - y) * height_scale))
130
131  return val_dic
132
133def CreateBenchOutput(page, bench, val_dic):
134  """Returns output for the given page and bench data in dict.
135  Args:
136    page: substring of bench graph webpage, to indicate the bench platform.
137    bench: name of the benchmark to process.
138    val_dic: dict[bench_name][revision] -> bench_value.
139
140  Returns:
141    string of html/javascript as part of the whole script output for the bench.
142  """
143  revs = val_dic[bench].keys()
144  revs.sort()
145  # Uses moving average to calculate expected bench variance, then sets
146  # expectations and ranges accordingly.
147  variances = []
148  moving_avgs = []
149  points = []
150  for rev in revs:
151    points.append(val_dic[bench][rev])
152    if len(points) >= WINDOW:
153      moving_avgs.append(sum(points[-WINDOW:]) / WINDOW)
154      variances.append(abs(points[-1] - moving_avgs[-1]))
155    else:  # For the first WINDOW-1 points, cannot calculate moving average.
156      moving_avgs.append(points[-1])  # Uses actual value as estimates.
157      variances.append(0)
158  if len(variances) >= WINDOW:
159    for i in range(WINDOW - 1):
160      # Backfills estimated variances for the first WINDOW-1 points.
161      variances[i] = variances[WINDOW - 1]
162
163  avg_var = sum(variances) / len(variances)
164  for val in variances:  # Removes outlier variances. Only does one iter.
165    if val > min(X_LB, X_UB) * avg_var:
166      variances.remove(val)
167  avg_var = sum(variances) / len(variances)
168
169  graph_id = '%s_%s' % (bench, page.replace('-', '_'))
170  expectations = '%s,%s,%.2f,%.2f,%.2f' % (bench, page, moving_avgs[-1],
171                                           moving_avgs[-1] - X_LB * avg_var,
172                                           moving_avgs[-1] + X_UB * avg_var)
173  out = GRAPH_PREFIX % (expectations, graph_id, graph_id, graph_id)
174  for i in range(len(revs)):
175    out += '%s,%.2f;%.2f;%.2f,' % (revs[i], moving_avgs[i] - X_LB * avg_var,
176                                   points[i], moving_avgs[i] + X_UB * avg_var)
177    if (points[i] > moving_avgs[i] + X_UB * avg_var or
178        points[i] < moving_avgs[i] - X_LB * avg_var):  # Mark as alert point.
179      out += '%.2f;%.2f;%.2f\\n' % (points[i], points[i], points[i])
180    else:
181      out += 'NaN;NaN;NaN\\n'
182
183  return out
184
185def main():
186  """Parses flags and outputs analysis results."""
187  try:
188    opts, _ = getopt.getopt(sys.argv[1:], 'a:b:o:p:')
189  except getopt.GetoptError, err:
190    Usage()
191    sys.exit(2)
192
193  alg = '25th'
194  bench_prefix = None
195  out_file = None
196  platform_prefix = ''
197  for option, value in opts:
198    if option == '-a':
199      if value not in ALGS:
200        raise Exception('Invalid flag -a (%s): must be set to one of %s.' %
201                        (value, str(ALGS)))
202      alg = value
203    elif option == '-b':
204      bench_prefix = value
205    elif option == '-o':
206      out_file = value
207    elif option == '-p':
208      platform_prefix = value
209    else:
210      Usage()
211      raise Exception('Error handling flags.')
212
213  if not bench_prefix:
214    raise Exception('Must provide nonempty Flag -b (bench name prefix).')
215
216  pages = []
217  for platform in PLATFORMS:
218    if not platform.startswith(platform_prefix):
219      continue
220    pages.append('%s-%s' % (platform, alg))
221
222  if not pages:  # No matching platform found.
223    raise Exception('Flag -p (platform prefix: %s) does not match any of %s.' %
224                    (platform_prefix, str(PLATFORMS)))
225
226  body = ''
227  # Iterates through bench graph xhtml pages for oututting matching benches.
228  for page in pages:
229    bench_value_dict = GetBenchValues(page, bench_prefix)
230    for bench in bench_value_dict:
231      body += CreateBenchOutput(page, bench, bench_value_dict) + GRAPH_SUFFIX
232
233  if not body:
234    raise Exception('No bench outputs. Most likely there are no matching bench'
235                    ' prefix (%s) in Flags -b for platforms %s.\nPlease also '
236                    'check if the bench graph URLs are valid at %s.' % (
237                        bench_prefix, str(PLATFORMS), INPUT_URL_TEMPLATE))
238  if out_file:
239    f = open(out_file, 'w+')
240    f.write(HTML_HEAD + body + HTML_SUFFIX)
241    f.close()
242  else:
243    print HTML_HEAD + body + HTML_SUFFIX
244
245
246if '__main__' == __name__:
247  main()
248