• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/env python
2# Copyright (c) 2012 The Chromium Authors. All rights reserved.
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5
6"""This script is used to download prebuilt clang binaries."""
7
8import os
9import shutil
10import subprocess
11import stat
12import sys
13import tarfile
14import tempfile
15import time
16import urllib2
17
18
19# CLANG_REVISION and CLANG_SUB_REVISION determine the build of clang
20# to use. These should be synced with tools/clang/scripts/update.py in
21# Chromium.
22CLANG_REVISION = '321529'
23CLANG_SUB_REVISION=2
24
25PACKAGE_VERSION = "%s-%s" % (CLANG_REVISION, CLANG_SUB_REVISION)
26
27# Path constants. (All of these should be absolute paths.)
28THIS_DIR = os.path.abspath(os.path.dirname(__file__))
29LLVM_BUILD_DIR = os.path.join(THIS_DIR, 'llvm-build')
30STAMP_FILE = os.path.join(LLVM_BUILD_DIR, 'cr_build_revision')
31
32# URL for pre-built binaries.
33CDS_URL = os.environ.get('CDS_CLANG_BUCKET_OVERRIDE',
34    'https://commondatastorage.googleapis.com/chromium-browser-clang')
35
36# Bump after VC updates.
37DIA_DLL = {
38  '2013': 'msdia120.dll',
39  '2015': 'msdia140.dll',
40  '2017': 'msdia140.dll',
41}
42
43
44def DownloadUrl(url, output_file):
45  """Download url into output_file."""
46  CHUNK_SIZE = 4096
47  TOTAL_DOTS = 10
48  num_retries = 3
49  retry_wait_s = 5  # Doubled at each retry.
50
51  while True:
52    try:
53      sys.stdout.write('Downloading %s ' % url)
54      sys.stdout.flush()
55      response = urllib2.urlopen(url)
56      total_size = int(response.info().getheader('Content-Length').strip())
57      bytes_done = 0
58      dots_printed = 0
59      while True:
60        chunk = response.read(CHUNK_SIZE)
61        if not chunk:
62          break
63        output_file.write(chunk)
64        bytes_done += len(chunk)
65        num_dots = TOTAL_DOTS * bytes_done / total_size
66        sys.stdout.write('.' * (num_dots - dots_printed))
67        sys.stdout.flush()
68        dots_printed = num_dots
69      if bytes_done != total_size:
70        raise urllib2.URLError("only got %d of %d bytes" %
71                               (bytes_done, total_size))
72      print ' Done.'
73      return
74    except urllib2.URLError as e:
75      sys.stdout.write('\n')
76      print e
77      if num_retries == 0 or isinstance(e, urllib2.HTTPError) and e.code == 404:
78        raise e
79      num_retries -= 1
80      print 'Retrying in %d s ...' % retry_wait_s
81      time.sleep(retry_wait_s)
82      retry_wait_s *= 2
83
84
85def EnsureDirExists(path):
86  if not os.path.exists(path):
87    print "Creating directory %s" % path
88    os.makedirs(path)
89
90
91def DownloadAndUnpack(url, output_dir):
92  with tempfile.TemporaryFile() as f:
93    DownloadUrl(url, f)
94    f.seek(0)
95    EnsureDirExists(output_dir)
96    tarfile.open(mode='r:gz', fileobj=f).extractall(path=output_dir)
97
98
99def ReadStampFile(path=STAMP_FILE):
100  """Return the contents of the stamp file, or '' if it doesn't exist."""
101  try:
102    with open(path, 'r') as f:
103      return f.read().rstrip()
104  except IOError:
105    return ''
106
107
108def WriteStampFile(s, path=STAMP_FILE):
109  """Write s to the stamp file."""
110  EnsureDirExists(os.path.dirname(path))
111  with open(path, 'w') as f:
112    f.write(s)
113    f.write('\n')
114
115
116def RmTree(dir):
117  """Delete dir."""
118  def ChmodAndRetry(func, path, _):
119    # Subversion can leave read-only files around.
120    if not os.access(path, os.W_OK):
121      os.chmod(path, stat.S_IWUSR)
122      return func(path)
123    raise
124
125  shutil.rmtree(dir, onerror=ChmodAndRetry)
126
127
128def CopyFile(src, dst):
129  """Copy a file from src to dst."""
130  print "Copying %s to %s" % (src, dst)
131  shutil.copy(src, dst)
132
133
134vs_version = None
135def GetVSVersion():
136  global vs_version
137  if vs_version:
138    return vs_version
139
140  # Try using the toolchain in depot_tools.
141  # This sets environment variables used by SelectVisualStudioVersion below.
142  sys.path.append(THIS_DIR)
143  import vs_toolchain
144  vs_toolchain.SetEnvironmentAndGetRuntimeDllDirs()
145
146  # Use gyp to find the MSVS installation, either in depot_tools as per above,
147  # or a system-wide installation otherwise.
148  sys.path.append(os.path.join(THIS_DIR, 'gyp', 'pylib'))
149  import gyp.MSVSVersion
150  vs_version = gyp.MSVSVersion.SelectVisualStudioVersion(
151      vs_toolchain.GetVisualStudioVersion())
152  return vs_version
153
154
155def CopyDiaDllTo(target_dir):
156  # This script always wants to use the 64-bit msdia*.dll.
157  dia_path = os.path.join(GetVSVersion().Path(), 'DIA SDK', 'bin', 'amd64')
158  dia_dll = os.path.join(dia_path, DIA_DLL[GetVSVersion().ShortName()])
159  CopyFile(dia_dll, target_dir)
160
161
162def UpdateClang():
163  cds_file = "clang-%s.tgz" %  PACKAGE_VERSION
164  if sys.platform == 'win32' or sys.platform == 'cygwin':
165    cds_full_url = CDS_URL + '/Win/' + cds_file
166  elif sys.platform.startswith('linux'):
167    cds_full_url = CDS_URL + '/Linux_x64/' + cds_file
168  else:
169    return 0
170
171  print 'Updating Clang to %s...' % PACKAGE_VERSION
172
173  if ReadStampFile() == PACKAGE_VERSION:
174    print 'Clang is already up to date.'
175    return 0
176
177  # Reset the stamp file in case the build is unsuccessful.
178  WriteStampFile('')
179
180  print 'Downloading prebuilt clang'
181  if os.path.exists(LLVM_BUILD_DIR):
182    RmTree(LLVM_BUILD_DIR)
183  try:
184    DownloadAndUnpack(cds_full_url, LLVM_BUILD_DIR)
185    print 'clang %s unpacked' % PACKAGE_VERSION
186    if sys.platform == 'win32':
187      CopyDiaDllTo(os.path.join(LLVM_BUILD_DIR, 'bin'))
188    WriteStampFile(PACKAGE_VERSION)
189    return 0
190  except urllib2.URLError:
191    print 'Failed to download prebuilt clang %s' % cds_file
192    print 'Exiting.'
193    return 1
194
195
196def main():
197  # Don't buffer stdout, so that print statements are immediately flushed.
198  sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 0)
199  return UpdateClang()
200
201
202if __name__ == '__main__':
203  sys.exit(main())
204