• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/env python
2# Copyright (c) 2012 The Chromium Authors. All rights reserved.
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5
6"""Automate the setup process of Chrome Endure environment.
7
8Usage:
9  python endure_setup.py [option]
10
11We use <ENDURE_DIR> to refer to the root directory in which Chrome Endure
12is set up. By default, <ENDURE_DIR> is the current working directory.
13
14First, run:
15  >python endure_setup.py
16This command will automatically setup Chrome Endure in <ENDURE_DIR>.
17
18Next, run your first endure test by:
19  >TEST_LENGTH=30 LOCAL_PERF_DIR="<ENDURE_DIR>/chrome_graph" \\
20   python <ENDURE_DIR>/src/chrome/test/functional/perf_endure.py \\
21   perf_endure.ChromeEndureGmailTest.testGmailComposeDiscard \\
22The above commands runs a Chrome Endure test for 30 seconds and saves
23the results to <ENDURE_DIR>/chrome_graph.
24
25Last, to view the graphs, run another script endure_server.py
26within <ENDURE_DIR> to start a local HTTP server that serves
27the graph directory, see endure_server.py for details.
28
29Use python endure_setup.py --help for more options.
30
31This script depends on the following modules
32(which will be downloaded automatically):
33  depot_tools
34  src/chrome/test/pyautolib/fetch_prebuilt_pyauto.py
35
36Supported platforms: Linux and Linux_x64.
37"""
38
39import logging
40import optparse
41import os
42import platform
43import shutil
44import subprocess
45import sys
46import urllib
47import urllib2
48import zipfile
49
50URLS = {'depot_tools': ('http://src.chromium.org'
51                        '/chrome/trunk/tools/depot_tools'),
52        'pyauto': ('https://src.chromium.org/'
53                   'chrome/trunk/src/chrome/test/functional.DEPS'),
54        'binary': ('http://commondatastorage.googleapis.com/'
55                   'chromium-browser-continuous/{os_type}/{revision}'),
56       }
57
58
59class SetupError(Exception):
60  """Catch errors in setting up Chrome Endure."""
61  pass
62
63
64class HelpFormatter(optparse.IndentedHelpFormatter):
65  """Format the help message of this script."""
66
67  def format_description(self, description):
68    """Override to keep the original format of the description."""
69    return description + '\n' if description else ''
70
71
72def Main(argv):
73  """Fetch Chrome Endure.
74
75  Usage:
76    python endure_setup.py [options]
77
78  Examples:
79    >python endure_setup.py
80      Fetch the latest version of Chrome Endure to the current
81      working directory.
82
83    >python endure_setup.py --endure-dir=/home/user/endure_dir
84      Fetch the latest version of Chrome Endure to /home/user/endure_dir.
85  """
86  parser = optparse.OptionParser(
87      formatter=HelpFormatter(), description=Main.__doc__)
88  parser.add_option(
89      '-d', '--endure-dir', type='string', default=os.getcwd(),
90      help='Directory in which to setup or update. ' \
91           'Default value is the current working directory.')
92  # TODO(fdeng): remove this option once the Chrome Endure
93  # graphing code is checked into chrome tree.
94  parser.add_option(
95      '-g', '--graph-zip-url', type='string', default=None,
96      help='URL to a zip file containing the chrome graphs.')
97  os_type = GetCurrentOSType()
98  if not os_type.startswith('Linux'):
99    raise SetupError('Only support Linux or Linux_x64, %s found'
100                     % os_type)
101  options, _ = parser.parse_args(argv)
102  endure_dir = os.path.abspath(options.endure_dir)
103  depot_dir = os.path.join(endure_dir, 'depot_tools')
104  gclient = os.path.join(depot_dir, 'gclient')
105  fetch_py = os.path.join(endure_dir, 'src', 'chrome',
106                          'test', 'pyautolib',
107                          'fetch_prebuilt_pyauto.py')
108  binary_dir = os.path.join(endure_dir, 'src', 'out', 'Release')
109  graph_zip_url = options.graph_zip_url
110  graph_dir = os.path.join(endure_dir, 'chrome_graph')
111
112  if not os.path.isdir(endure_dir):
113    os.makedirs(endure_dir)
114
115  logging.info('Fetching depot tools...')
116  FetchDepot(depot_dir)
117  logging.info('Fetching PyAuto (python code)...')
118  FetchPyAuto(gclient, endure_dir)
119  logging.info('Fetching binaries(chrome, pyautolib, chrome driver)...')
120  FetchBinaries(fetch_py, binary_dir, os_type)
121  # TODO(fdeng): remove this after it is checked into the chrome tree.
122  logging.info('Fetching chrome graphing files...')
123  FetchGraph(graph_zip_url, graph_dir)
124  return 0
125
126
127def FetchDepot(depot_dir):
128  """Fetch depot_tools.
129
130  Args:
131    depot_dir: The directory where depot_tools will be checked out.
132
133  Raises:
134    SetupError: If fail.
135  """
136  if subprocess.call(['svn', 'co', URLS['depot_tools'], depot_dir]) != 0:
137    raise SetupError('Error found when checking out depot_tools.')
138  if not CheckDepot(depot_dir):
139    raise SetupError('Could not get depot_tools.')
140
141
142def CheckDepot(depot_dir):
143  """Check that some expected depot_tools files exist.
144
145  Args:
146    depot_dir: The directory where depot_tools are checked out.
147
148  Returns:
149    True if check passes otherwise False.
150  """
151  gclient = os.path.join(depot_dir, 'gclient')
152  gclient_py = os.path.join(depot_dir, 'gclient.py')
153  files = [gclient, gclient_py]
154  for f in files:
155    if not os.path.exists(f):
156      return False
157  try:
158    subprocess.call([gclient, '--version'])
159  except OSError:
160    return False
161  return True
162
163
164def FetchPyAuto(gclient, endure_dir):
165  """Use gclient to fetch python code.
166
167  Args:
168    gclient: The path to the gclient executable.
169    endure_dir: Directory where Chrome Endure and
170                its dependencies will be checked out.
171
172  Raises:
173    SetupError: if fails.
174  """
175  cur_dir = os.getcwd()
176  os.chdir(endure_dir)
177  config_cmd = [gclient, 'config', URLS['pyauto']]
178  if subprocess.call(config_cmd) != 0:
179    raise SetupError('Running "%s" failed.' % ' '.join(config_cmd))
180  sync_cmd = [gclient, 'sync']
181  if subprocess.call(sync_cmd) != 0:
182    raise SetupError('Running "%s" failed.' % ' '.join(sync_cmd))
183  CheckPyAuto(endure_dir)
184  logging.info('Sync PyAuto python code done.')
185  os.chdir(cur_dir)
186
187
188def CheckPyAuto(endure_dir):
189  """Sanity check for Chrome Endure code.
190
191  Args:
192    endure_dir: Directory of Chrome Endure and its dependencies.
193
194  Raises:
195    SetupError: If fails.
196  """
197  fetch_py = os.path.join(endure_dir, 'src', 'chrome',
198                          'test', 'pyautolib',
199                          'fetch_prebuilt_pyauto.py')
200  pyauto_py = os.path.join(endure_dir, 'src',
201                           'chrome', 'test',
202                           'pyautolib', 'pyauto.py')
203  files = [fetch_py, pyauto_py]
204  for f in files:
205    if not os.path.exists(f):
206      raise SetupError('Checking %s failed.' % f)
207
208
209def FetchBinaries(fetch_py, binary_dir, os_type):
210  """Get the prebuilt binaries from continuous build archive.
211
212  Args:
213    fetch_py: Path to the script which fetches pre-built binaries.
214    binary_dir: Directory of the pre-built binaries.
215    os_type: 'Mac', 'Win', 'Linux', 'Linux_x64'.
216
217  Raises:
218    SetupError: If fails.
219  """
220  revision = GetLatestRevision(os_type)
221  logging.info('Cleaning %s', binary_dir)
222  if os.path.exists(binary_dir):
223    shutil.rmtree(binary_dir)
224  logging.info('Downloading binaries...')
225  cmd = [fetch_py, '-d', binary_dir,
226         URLS['binary'].format(
227             os_type=os_type, revision=revision)]
228  if subprocess.call(cmd) == 0 and os.path.exists(binary_dir):
229    logging.info('Binaries at revision %s', revision)
230  else:
231    raise SetupError('Running "%s" failed.' % ' '.join(cmd))
232
233
234def FetchGraph(graph_zip_url, graph_dir):
235  """Fetch graph code.
236
237  Args:
238    graph_zip_url: The url to a zip file containing the chrome graphs.
239    graph_dir: Directory of the chrome graphs.
240
241  Raises:
242    SetupError: if unable to retrive the zip file.
243  """
244  # TODO(fdeng): remove this function once chrome graph
245  # is checked into chrome tree.
246  if not graph_zip_url:
247    logging.info(
248        'Skip fetching chrome graphs' +
249        ' since --graph-zip-url is not set.')
250    return
251  graph_zip = urllib.urlretrieve(graph_zip_url)[0]
252  if graph_zip is None or not os.path.exists(graph_zip):
253    raise SetupError('Unable to retrieve %s' % graph_zip_url)
254  if not os.path.exists(graph_dir):
255    os.mkdir(graph_dir)
256  UnzipFilenameToDir(graph_zip, graph_dir)
257  logging.info('Graph code is downloaded to %s', graph_dir)
258
259
260def GetCurrentOSType():
261  """Get a string representation for the current OS.
262
263  Returns:
264    'Mac', 'Win', 'Linux', or 'Linux_64'.
265
266  Raises:
267    RuntimeError: if OS can't be identified.
268  """
269  if sys.platform == 'darwin':
270    os_type = 'Mac'
271  if sys.platform == 'win32':
272    os_type = 'Win'
273  if sys.platform.startswith('linux'):
274    os_type = 'Linux'
275    if platform.architecture()[0] == '64bit':
276      os_type += '_x64'
277  else:
278    raise RuntimeError('Unknown platform')
279  return os_type
280
281
282def GetLatestRevision(os_type):
283  """Figure out the latest revision number of the prebuilt binary archive.
284
285  Args:
286    os_type: 'Mac', 'Win', 'Linux', or 'Linux_64'.
287
288  Returns:
289    A string of latest revision number.
290
291  Raises:
292    SetupError: If unable to get the latest revision number.
293  """
294  last_change_url = ('http://commondatastorage.googleapis.com/'
295                     'chromium-browser-continuous/%s/LAST_CHANGE' % os_type)
296  response = urllib2.urlopen(last_change_url)
297  last_change = response.read()
298  if not last_change:
299    raise SetupError('Unable to get the latest revision number from %s' %
300                     last_change_url)
301  return last_change
302
303
304def UnzipFilenameToDir(filename, directory):
305  """Unzip |filename| to directory |directory|.
306
307  This works with as low as python2.4 (used on win).
308  (Code is adapted from fetch_prebuilt_pyauto.py)
309  """
310  # TODO(fdeng): remove this function as soon as the Chrome Endure
311  # graphing code is checked into the chrome tree.
312  zf = zipfile.ZipFile(filename)
313  pushd = os.getcwd()
314  if not os.path.isdir(directory):
315    os.mkdir(directory)
316  os.chdir(directory)
317  # Extract files.
318  for info in zf.infolist():
319    name = info.filename
320    if name.endswith('/'):  # dir
321      if not os.path.isdir(name):
322        os.makedirs(name)
323    else:  # file
324      directory = os.path.dirname(name)
325      if directory and not os.path.isdir(directory):
326        os.makedirs(directory)
327      out = open(name, 'wb')
328      out.write(zf.read(name))
329      out.close()
330    # Set permissions. Permission info in external_attr is shifted 16 bits.
331    os.chmod(name, info.external_attr >> 16L)
332  os.chdir(pushd)
333
334
335if '__main__' == __name__:
336  logging.basicConfig(format='[%(levelname)s] %(message)s', level=logging.DEBUG)
337  sys.exit(Main(sys.argv[1:]))
338