• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/env python
2# Copyright 2015 The Chromium Authors. All rights reserved.
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5
6import argparse
7import json
8import logging
9import os
10import re
11import shutil
12import stat
13import subprocess
14import sys
15import tempfile
16import time
17import urllib2
18import zipfile
19
20from hooks import install
21
22from catapult_base import xvfb
23
24# URL on omahaproxy.appspot.com which lists the current version for the os
25# and channel.
26VERSION_LOOKUP_URL = 'https://omahaproxy.appspot.com/all?os=%s&channel=%s'
27
28# URL on omahaproxy.appspot.com which looks up base positions from versions.
29BASE_POS_LOOKUP_URL = 'http://omahaproxy.appspot.com/revision.json?version=%s'
30
31# URL on cloud storage which looks up the chromium download url from base pos.
32CLOUD_STORAGE_LOOKUP_URL = ('https://www.googleapis.com/storage/v1/b/'
33                            'chromium-browser-snapshots/o?delimiter=/&prefix='
34                            '%s/%s&fields=items(kind,mediaLink,metadata,name,'
35                            'size,updated),kind,prefixes,nextPageToken')
36
37# URL on cloud storage to download chromium at a base pos from.
38CLOUD_STORAGE_DOWNLOAD_URL = ('https://www.googleapis.com/download/storage/v1/b'
39                              '/chromium-browser-snapshots/o/%s%%2F%s%%2F'
40                              'chrome-%s.zip?alt=media')
41
42# URL in cloud storage to download Chrome zip from.
43CLOUDSTORAGE_URL = ('https://commondatastorage.googleapis.com/chrome-unsigned'
44                    '/desktop-W15K3Y/%s/%s/chrome-%s.zip')
45
46# Default port to run on if not auto-assigning from OS
47DEFAULT_PORT = '8111'
48
49# Mapping of sys.platform -> platform-specific names and paths.
50PLATFORM_MAPPING = {
51    'linux2': {
52        'omaha': 'linux',
53        'prefix': 'Linux_x64',
54        'zip_prefix': 'linux',
55        'chromepath': 'chrome-linux/chrome'
56    },
57    'win32': {
58        'omaha': 'win',
59        'prefix': 'Win',
60        'zip_prefix': 'win32',
61        'chromepath': 'chrome-win32\\chrome.exe',
62    },
63    'darwin': {
64        'omaha': 'mac',
65        'prefix': 'Mac',
66        'zip_prefix': 'mac',
67        'chromepath': ('chrome-mac/Chromium.app/Contents/MacOS/Chromium'),
68        'version_path': 'chrome-mac/Chromium.app/Contents/Versions/',
69        'additional_paths': [
70            ('chrome-mac/Chromium.app/Contents/Versions/%VERSION%/'
71             'Chromium Helper.app/Contents/MacOS/Chromium Helper'),
72        ],
73    },
74}
75
76
77def IsDepotToolsPath(path):
78  return os.path.isfile(os.path.join(path, 'gclient'))
79
80
81def FindDepotTools():
82  # Check if depot_tools is already in PYTHONPATH
83  for path in sys.path:
84    if path.rstrip(os.sep).endswith('depot_tools') and IsDepotToolsPath(path):
85      return path
86
87  # Check if depot_tools is in the path
88  for path in os.environ['PATH'].split(os.pathsep):
89    if IsDepotToolsPath(path):
90      return path.rstrip(os.sep)
91
92  return None
93
94
95def DownloadChromium(channel):
96  """
97  Gets the version of Chrome current for the given channel from omahaproxy, then
98  follows instructions for downloading a prebuilt version of chromium from the
99  commit at the branch cut for that version. This downloads a chromium binary
100  which does not have any commits merged onto the branch. It is close to the
101  released Chrome, but not exact. Downloading the released Chrome is not
102  supported.
103  https://www.chromium.org/getting-involved/download-chromium
104  """
105  # Get the version for the current channel from omahaproxy
106  platform_data = PLATFORM_MAPPING[sys.platform]
107  omaha_platform = platform_data['omaha']
108  version_lookup_url = VERSION_LOOKUP_URL % (omaha_platform, channel)
109  print 'Getting version from %s' % version_lookup_url
110  response = urllib2.urlopen(version_lookup_url, timeout=120)
111  version = response.readlines()[1].split(',')[2]
112
113  # Get the base position for that version from omahaproxy
114  base_pos_lookup_url = BASE_POS_LOOKUP_URL % version
115  print 'Getting base_pos from %s' % base_pos_lookup_url
116  response = urllib2.urlopen(base_pos_lookup_url, timeout=120)
117  base_pos = json.load(response)['chromium_base_position']
118
119  # Find the build from that base position in cloud storage. If it's not found,
120  # decrement base position until one is found.
121  cloud_storage_lookup_url = CLOUD_STORAGE_LOOKUP_URL % (
122      platform_data['prefix'], base_pos)
123  download_url = None
124  while not download_url:
125    print 'Getting download url from %s' % cloud_storage_lookup_url
126    response = urllib2.urlopen(cloud_storage_lookup_url, timeout=120)
127    prefixes = json.load(response).get('prefixes')
128    if prefixes:
129      download_url = CLOUD_STORAGE_DOWNLOAD_URL % (
130          platform_data['prefix'], base_pos, platform_data['zip_prefix'])
131      break
132    base_pos = int(base_pos) - 1
133    cloud_storage_lookup_url = CLOUD_STORAGE_LOOKUP_URL % (
134        platform_data['prefix'], base_pos)
135
136  print 'Approximating Chrome %s with chromium from base position %s.' % (
137      version, base_pos)
138  print 'Downloading from %s' % download_url
139
140  tmpdir = tempfile.mkdtemp()
141  zip_path = os.path.join(tmpdir, 'chrome.zip')
142  with open(zip_path, 'wb') as local_file:
143    local_file.write(urllib2.urlopen(download_url, timeout=600).read())
144  zf = zipfile.ZipFile(zip_path)
145  zf.extractall(path=tmpdir)
146  return tmpdir, version, download_url
147
148
149def GetLocalChromePath(path_from_command_line):
150  if path_from_command_line:
151    return path_from_command_line
152
153  if sys.platform == 'darwin':  # Mac
154    chrome_path = (
155        '/Applications/Google Chrome.app/Contents/MacOS/Google Chrome')
156    if os.path.isfile(chrome_path):
157      return chrome_path
158  elif sys.platform.startswith('linux'):
159    found = False
160    try:
161      with open(os.devnull, 'w') as devnull:
162        found = subprocess.call(['google-chrome', '--version'],
163                                stdout=devnull, stderr=devnull) == 0
164    except OSError:
165      pass
166    if found:
167      return 'google-chrome'
168  elif sys.platform == 'win32':
169    search_paths = [os.getenv('PROGRAMFILES(X86)'),
170                    os.getenv('PROGRAMFILES'),
171                    os.getenv('LOCALAPPDATA')]
172    chrome_path = os.path.join('Google', 'Chrome', 'Application', 'chrome.exe')
173    for search_path in search_paths:
174      test_path = os.path.join(search_path, chrome_path)
175      if os.path.isfile(test_path):
176        return test_path
177  return None
178
179
180def Main(argv):
181  try:
182    parser = argparse.ArgumentParser(
183        description='Run dev_server tests for a project.')
184    parser.add_argument('--chrome_path', type=str,
185                        help='Path to Chrome browser binary.')
186    parser.add_argument('--no-use-local-chrome',
187                        dest='use_local_chrome', action='store_false')
188    parser.add_argument(
189        '--no-install-hooks', dest='install_hooks', action='store_false')
190    parser.add_argument('--tests', type=str,
191                        help='Set of tests to run (tracing or perf_insights)')
192    parser.add_argument('--channel', type=str, default='stable',
193                        help='Chrome channel to run (stable or canary)')
194    parser.add_argument('--presentation-json', type=str,
195                        help='Recipe presentation-json output file path')
196    parser.set_defaults(install_hooks=True)
197    parser.set_defaults(use_local_chrome=True)
198    args = parser.parse_args(argv[1:])
199
200    if args.install_hooks:
201      install.InstallHooks()
202
203    platform_data = PLATFORM_MAPPING[sys.platform]
204    user_data_dir = tempfile.mkdtemp()
205    tmpdir = None
206    xvfb_process = None
207
208    server_path = os.path.join(os.path.dirname(
209        os.path.abspath(__file__)), os.pardir, 'bin', 'run_dev_server')
210    # TODO(anniesullie): Make OS selection of port work on Windows. See #1235.
211    if sys.platform == 'win32':
212      port = DEFAULT_PORT
213    else:
214      port = '0'
215    server_command = [server_path, '--no-install-hooks', '--port', port]
216    if sys.platform.startswith('win'):
217      server_command = ['python.exe'] + server_command
218    print "Starting dev_server..."
219    server_process = subprocess.Popen(
220        server_command, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
221        bufsize=1)
222    time.sleep(1)
223    if sys.platform != 'win32':
224      output = server_process.stderr.readline()
225      port = re.search(
226          r'Now running on http://127.0.0.1:([\d]+)', output).group(1)
227
228    chrome_info = None
229    if args.use_local_chrome:
230      chrome_path = GetLocalChromePath(args.chrome_path)
231      if not chrome_path:
232        logging.error('Could not find path to chrome.')
233        sys.exit(1)
234      chrome_info = 'with command `%s`' % chrome_path
235    else:
236      channel = args.channel
237      if sys.platform == 'linux2' and channel == 'canary':
238        channel = 'dev'
239      assert channel in ['stable', 'beta', 'dev', 'canary']
240
241
242      tmpdir, version, download_url = DownloadChromium(channel)
243      if xvfb.ShouldStartXvfb():
244        xvfb_process = xvfb.StartXvfb()
245      chrome_path = os.path.join(
246          tmpdir, platform_data['chromepath'])
247      os.chmod(chrome_path, os.stat(chrome_path).st_mode | stat.S_IEXEC)
248      # On Mac, we need to update a file with the version in the path, and
249      # the version we downloaded could be slightly different than what we
250      # requested. Update it.
251      if platform_data.get('version_path'):
252        contents = os.listdir(
253            os.path.join(tmpdir, platform_data['version_path']))
254        for path in contents:
255          if re.match(r'\d+\.\d+\.\d+\.\d+', path):
256            version = path
257      if platform_data.get('additional_paths'):
258        for path in platform_data.get('additional_paths'):
259          path = path.replace('%VERSION%', version)
260          path = os.path.join(tmpdir, path)
261          if os.path.exists(path):
262            os.chmod(path, os.stat(path).st_mode | stat.S_IEXEC)
263      chrome_info = version
264    chrome_command = [
265        chrome_path,
266        '--user-data-dir=%s' % user_data_dir,
267        '--no-sandbox',
268        '--no-experiments',
269        '--no-first-run',
270        '--noerrdialogs',
271        '--window-size=1280,1024',
272        ('http://localhost:%s/%s/tests.html?' % (port, args.tests)) +
273        'headless=true&testTypeToRun=all',
274    ]
275    print "Starting Chrome %s..." % chrome_info
276    chrome_process = subprocess.Popen(
277        chrome_command, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
278    print "Waiting for tests to finish..."
279    server_out, server_err = server_process.communicate()
280    print "Killing Chrome..."
281    if sys.platform == 'win32':
282      # Use taskkill on Windows to make sure Chrome and all subprocesses are
283      # killed.
284      subprocess.call(['taskkill', '/F', '/T', '/PID', str(chrome_process.pid)])
285    else:
286      chrome_process.kill()
287    if server_process.returncode != 0:
288      logging.error('Tests failed!')
289      logging.error('Server stderr:')
290      logging.error(server_err)
291      logging.error('Server stdout:')
292      logging.error(server_out)
293    else:
294      print server_out
295    if args.presentation_json:
296      with open(args.presentation_json, 'w') as recipe_out:
297        # Add a link to the buildbot status for the step saying which version
298        # of Chrome the test ran on. The actual linking feature is not used,
299        # but there isn't a way to just add text.
300        link_name = 'Chrome Version %s' % version
301        presentation_info = {'links': {link_name: download_url}}
302        json.dump(presentation_info, recipe_out)
303  finally:
304    # Wait for Chrome to be killed before deleting temp Chrome dir. Only have
305    # this timing issue on Windows.
306    if sys.platform == 'win32':
307      time.sleep(5)
308    if tmpdir:
309      try:
310        shutil.rmtree(tmpdir)
311        shutil.rmtree(user_data_dir)
312      except OSError as e:
313        logging.error('Error cleaning up temp dirs %s and %s: %s',
314                      tmpdir, user_data_dir, e)
315    if xvfb_process:
316      xvfb_process.kill()
317
318  sys.exit(server_process.returncode)
319