• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/env python
2# Copyright (c) 2011 The Chromium Authors. All rights reserved.
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5
6"""Dromaeo benchmark automation script.
7
8Script runs dromaeo tests in browsers specified by --browser switch and saves
9results to a spreadsheet on docs.google.com.
10
11Prerequisites:
121. Install Google Data APIs Python Client Library from
13   http://code.google.com/p/gdata-python-client.
142. Checkout Dromaeo benchmark from
15   http://src.chromium.org/svn/trunk/src/chrome/test/data/dromaeo and provide
16   local path to it in --dromaeo_home switch.
173. Create a spreadsheet at http://docs.google.com and specify its name in
18   --spreadsheet switch
19
20Benchmark results are presented in the following format:
21browser | date time
22test 1 name|m11|...|m1n|test 1 average mean| |e11|...|e1n|test 1 average error
23test 2 name|m21|...|m2n|test 2 average mean| |e21|...|e2n|test 2 average error
24...
25
26Here mij is mean run/s in individual dromaeo test i during benchmark run j,
27eij is error in individual dromaeo test i during benchmark run j.
28
29Example usage:
30dromaeo_benchmark_runner.py -b "E:\chromium\src\chrome\Release\chrome.exe"
31    -b "C:\Program Files (x86)\Safari\safari.exe"
32    -b "C:\Program Files (x86)\Opera 10.50 pre-alpha\opera.exe" -n 1
33    -d "E:\chromium\src\chrome\test\data\dromaeo" -f dom -e example@gmail.com
34
35"""
36
37import getpass
38import json
39import os
40import re
41import subprocess
42import time
43import urlparse
44from optparse import OptionParser
45from BaseHTTPServer import HTTPServer
46import SimpleHTTPServer
47import gdata.spreadsheet.service
48
49max_spreadsheet_columns = 20
50test_props = ['mean', 'error']
51
52
53def ParseArguments():
54  parser = OptionParser()
55  parser.add_option("-b", "--browser",
56                    action="append", dest="browsers",
57                    help="list of browsers to test")
58  parser.add_option("-n", "--run_count", dest="run_count", type="int",
59                    default=5, help="number of runs")
60  parser.add_option("-d", "--dromaeo_home", dest="dromaeo_home",
61                    help="directory with your dromaeo files")
62  parser.add_option("-p", "--port", dest="port", type="int",
63                    default=8080, help="http server port")
64  parser.add_option("-f", "--filter", dest="filter",
65                    default="dom", help="dromaeo suite filter")
66  parser.add_option("-e", "--email", dest="email",
67                    help="your google docs account")
68  parser.add_option("-s", "--spreadsheet", dest="spreadsheet_title",
69                    default="dromaeo",
70                    help="your google docs spreadsheet name")
71
72  options = parser.parse_args()[0]
73
74  if not options.dromaeo_home:
75    raise Exception('please specify dromaeo_home')
76
77  return options
78
79
80def KillProcessByName(process_name):
81  process = subprocess.Popen('wmic process get processid, executablepath',
82                             stdout=subprocess.PIPE)
83  stdout = str(process.communicate()[0])
84  match = re.search(re.escape(process_name) + '\s+(\d+)', stdout)
85  if match:
86    pid = match.group(1)
87    subprocess.call('taskkill /pid %s' % pid)
88
89
90class SpreadsheetWriter(object):
91  "Utility class for storing benchmarking results in Google spreadsheets."
92
93  def __init__(self, email, spreadsheet_title):
94    '''Login to google docs and search for spreadsheet'''
95
96    self.token_file = os.path.expanduser("~/.dromaeo_bot_auth_token")
97    self.gd_client = gdata.spreadsheet.service.SpreadsheetsService()
98
99    authenticated = False
100    if os.path.exists(self.token_file):
101      token = ''
102      try:
103        file = open(self.token_file, 'r')
104        token = file.read()
105        file.close()
106        self.gd_client.SetClientLoginToken(token)
107        self.gd_client.GetSpreadsheetsFeed()
108        authenticated = True
109      except (IOError, gdata.service.RequestError):
110        pass
111    if not authenticated:
112      self.gd_client.email = email
113      self.gd_client.password = getpass.getpass('Password for %s: ' % email)
114      self.gd_client.source = 'python robot for dromaeo'
115      self.gd_client.ProgrammaticLogin()
116      token = self.gd_client.GetClientLoginToken()
117      try:
118        file = open(self.token_file, 'w')
119        file.write(token)
120        file.close()
121      except (IOError):
122        pass
123      os.chmod(self.token_file, 0600)
124
125    # Search for the spreadsheet with title = spreadsheet_title.
126    spreadsheet_feed = self.gd_client.GetSpreadsheetsFeed()
127    for spreadsheet in spreadsheet_feed.entry:
128      if spreadsheet.title.text == spreadsheet_title:
129        self.spreadsheet_key = spreadsheet.id.text.rsplit('/', 1)[1]
130    if not self.spreadsheet_key:
131      raise Exception('Spreadsheet %s not found' % spreadsheet_title)
132
133    # Get the key of the first worksheet in spreadsheet.
134    worksheet_feed = self.gd_client.GetWorksheetsFeed(self.spreadsheet_key)
135    self.worksheet_key = worksheet_feed.entry[0].id.text.rsplit('/', 1)[1]
136
137  def _InsertRow(self, row):
138    row = dict([('c' + str(i), row[i]) for i in xrange(len(row))])
139    self.gd_client.InsertRow(row, self.spreadsheet_key, self.worksheet_key)
140
141  def _InsertBlankRow(self):
142    self._InsertRow('-' * self.columns_count)
143
144  def PrepareSpreadsheet(self, run_count):
145    """Update cells in worksheet topmost row with service information.
146
147    Calculate column count corresponding to run_count and create worksheet
148    column titles [c0, c1, ...] in the topmost row to speed up spreadsheet
149    updates (it allows to insert a whole row with a single request)
150    """
151
152    # Calculate the number of columns we need to present all test results.
153    self.columns_count = (run_count + 2) * len(test_props)
154    if self.columns_count > max_spreadsheet_columns:
155      # Google spreadsheet has just max_spreadsheet_columns columns.
156      max_run_count = max_spreadsheet_columns / len(test_props) - 2
157      raise Exception('maximum run count is %i' % max_run_count)
158    # Create worksheet column titles [c0, c1, ..., cn].
159    for i in xrange(self.columns_count):
160      self.gd_client.UpdateCell(1, i + 1, 'c' + str(i), self.spreadsheet_key,
161                                self.worksheet_key)
162
163  def WriteColumnTitles(self, run_count):
164    "Create titles for test results (mean 1, mean 2, ..., average mean, ...)"
165    row = []
166    for prop in test_props:
167      row.append('')
168      for i in xrange(run_count):
169        row.append('%s %i' % (prop, i + 1))
170      row.append('average ' + prop)
171    self._InsertRow(row)
172
173  def WriteBrowserBenchmarkTitle(self, browser_name):
174    "Create browser benchmark title (browser name, date time)"
175    self._InsertBlankRow()
176    self._InsertRow([browser_name, time.strftime('%d.%m.%Y %H:%M:%S')])
177
178  def WriteBrowserBenchmarkResults(self, test_name, test_data):
179    "Insert a row with single test results"
180    row = []
181    for prop in test_props:
182      if not row:
183        row.append(test_name)
184      else:
185        row.append('')
186      row.extend([str(x) for x in test_data[prop]])
187      row.append(str(sum(test_data[prop]) / len(test_data[prop])))
188    self._InsertRow(row)
189
190
191class DromaeoHandler(SimpleHTTPServer.SimpleHTTPRequestHandler):
192
193  def do_POST(self):
194    self.send_response(200)
195    self.end_headers()
196    self.wfile.write("<HTML>POST OK.<BR><BR>");
197    length = int(self.headers.getheader('content-length'))
198    parameters = urlparse.parse_qs(self.rfile.read(length))
199    self.server.got_post = True
200    self.server.post_data = parameters['data']
201
202
203class BenchmarkResults(object):
204  "Storage class for dromaeo benchmark results"
205
206  def __init__(self):
207    self.data = {}
208
209  def ProcessBrowserPostData(self, data):
210    "Convert dromaeo test results in internal format"
211    tests = json.loads(data[0])
212    for test in tests:
213      test_name = test['name']
214      if test_name not in self.data:
215        # Test is encountered for the first time.
216        self.data[test_name] = dict([(prop, []) for prop in test_props])
217      # Append current run results.
218      for prop in test_props:
219        value = -1
220        if prop in test: value = test[prop] # workaround for Opera 10.5
221        self.data[test_name][prop].append(value)
222
223
224def main():
225  options = ParseArguments()
226
227  # Start sever with dromaeo.
228  os.chdir(options.dromaeo_home)
229  server = HTTPServer(('', options.port), DromaeoHandler)
230
231  # Open and prepare spreadsheet on google docs.
232  spreadsheet_writer = SpreadsheetWriter(options.email,
233                                         options.spreadsheet_title)
234  spreadsheet_writer.PrepareSpreadsheet(options.run_count)
235  spreadsheet_writer.WriteColumnTitles(options.run_count)
236
237  for browser in options.browsers:
238    browser_name = os.path.splitext(os.path.basename(browser))[0]
239    spreadsheet_writer.WriteBrowserBenchmarkTitle(browser_name)
240    benchmark_results = BenchmarkResults()
241    for run_number in xrange(options.run_count):
242      print '%s run %i' % (browser_name, run_number + 1)
243      # Run browser.
244      test_page = 'http://localhost:%i/index.html?%s&automated&post_json' % (
245        options.port, options.filter)
246      browser_process = subprocess.Popen('%s "%s"' % (browser, test_page))
247      server.got_post = False
248      server.post_data = None
249      # Wait until POST request from browser.
250      while not server.got_post:
251        server.handle_request()
252      benchmark_results.ProcessBrowserPostData(server.post_data)
253      # Kill browser.
254      KillProcessByName(browser)
255      browser_process.wait()
256
257    # Insert test results into spreadsheet.
258    for (test_name, test_data) in benchmark_results.data.iteritems():
259      spreadsheet_writer.WriteBrowserBenchmarkResults(test_name, test_data)
260
261  server.socket.close()
262  return 0
263
264
265if __name__ == '__main__':
266  sys.exit(main())
267