• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/env python
2# Copyright (c) 2015 The Chromium Authors. All rights reserved.
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5
6"""This script will check out llvm and clang, and then package the results up
7to a tgz file."""
8
9import argparse
10import fnmatch
11import itertools
12import os
13import shutil
14import subprocess
15import sys
16import tarfile
17
18# Path constants.
19THIS_DIR = os.path.dirname(__file__)
20CHROMIUM_DIR = os.path.abspath(os.path.join(THIS_DIR, '..', '..', '..'))
21THIRD_PARTY_DIR = os.path.join(THIS_DIR, '..', '..', '..', 'third_party')
22LLVM_DIR = os.path.join(THIRD_PARTY_DIR, 'llvm')
23LLVM_BOOTSTRAP_DIR = os.path.join(THIRD_PARTY_DIR, 'llvm-bootstrap')
24LLVM_BOOTSTRAP_INSTALL_DIR = os.path.join(THIRD_PARTY_DIR,
25                                          'llvm-bootstrap-install')
26LLVM_BUILD_DIR = os.path.join(THIRD_PARTY_DIR, 'llvm-build')
27LLVM_RELEASE_DIR = os.path.join(LLVM_BUILD_DIR, 'Release+Asserts')
28LLVM_LTO_GOLD_PLUGIN_DIR = os.path.join(THIRD_PARTY_DIR, 'llvm-lto-gold-plugin')
29STAMP_FILE = os.path.join(LLVM_BUILD_DIR, 'cr_build_revision')
30
31
32def Tee(output, logfile):
33  logfile.write(output)
34  print output,
35
36
37def TeeCmd(cmd, logfile, fail_hard=True):
38  """Runs cmd and writes the output to both stdout and logfile."""
39  # Reading from PIPE can deadlock if one buffer is full but we wait on a
40  # different one.  To work around this, pipe the subprocess's stderr to
41  # its stdout buffer and don't give it a stdin.
42  # shell=True is required in cmd.exe since depot_tools has an svn.bat, and
43  # bat files only work with shell=True set.
44  proc = subprocess.Popen(cmd, bufsize=1, shell=sys.platform == 'win32',
45                          stdin=open(os.devnull), stdout=subprocess.PIPE,
46                          stderr=subprocess.STDOUT)
47  for line in iter(proc.stdout.readline,''):
48    Tee(line, logfile)
49    if proc.poll() is not None:
50      break
51  exit_code = proc.wait()
52  if exit_code != 0 and fail_hard:
53    print 'Failed:', cmd
54    sys.exit(1)
55
56
57def PrintTarProgress(tarinfo):
58  print 'Adding', tarinfo.name
59  return tarinfo
60
61
62def GetExpectedStamp():
63  rev_cmd = [sys.executable, os.path.join(THIS_DIR, 'update.py'),
64             '--print-revision']
65  return subprocess.check_output(rev_cmd).rstrip()
66
67
68def GetGsutilPath():
69  if not 'find_depot_tools' in sys.modules:
70    sys.path.insert(0, os.path.join(CHROMIUM_DIR, 'build'))
71    global find_depot_tools
72    import find_depot_tools
73  depot_path = find_depot_tools.add_depot_tools_to_path()
74  if depot_path is None:
75    print ('depot_tools are not found in PATH. '
76           'Follow the instructions in this document '
77           'http://dev.chromium.org/developers/how-tos/install-depot-tools'
78           ' to install depot_tools and then try again.')
79    sys.exit(1)
80  gsutil_path = os.path.join(depot_path, 'gsutil.py')
81  return gsutil_path
82
83
84def RunGsutil(args):
85  return subprocess.call([sys.executable, GetGsutilPath()] + args)
86
87
88def GsutilArchiveExists(archive_name, platform):
89  gsutil_args = ['-q', 'stat',
90                 'gs://chromium-browser-clang/%s/%s.tgz' %
91                 (platform, archive_name)]
92  return RunGsutil(gsutil_args) == 0
93
94
95def MaybeUpload(args, archive_name, platform):
96  # We don't want to rewrite the file, if it already exists on the server,
97  # so -n option to gsutil is used. It will warn, if the upload was aborted.
98  gsutil_args = ['cp', '-n', '-a', 'public-read',
99                  '%s.tgz' % archive_name,
100                  'gs://chromium-browser-clang/%s/%s.tgz' %
101                 (platform, archive_name)]
102  if args.upload:
103    print 'Uploading %s to Google Cloud Storage...' % archive_name
104    exit_code = RunGsutil(gsutil_args)
105    if exit_code != 0:
106      print "gsutil failed, exit_code: %s" % exit_code
107      os.exit(exit_code)
108  else:
109    print 'To upload, run:'
110    print ('gsutil %s' % ' '.join(gsutil_args))
111
112
113def main():
114  parser = argparse.ArgumentParser(description='build and package clang')
115  parser.add_argument('--upload', action='store_true',
116                      help='Upload the target archive to Google Cloud Storage.')
117  args = parser.parse_args()
118
119  # Check that the script is not going to upload a toolchain built from HEAD.
120  use_head_revision = 'LLVM_FORCE_HEAD_REVISION' in os.environ
121  if args.upload and use_head_revision:
122    print ("--upload and LLVM_FORCE_HEAD_REVISION could not be used "
123           "at the same time.")
124    return 1
125
126  expected_stamp = GetExpectedStamp()
127  pdir = 'clang-' + expected_stamp
128  golddir = 'llvmgold-' + expected_stamp
129  print pdir
130
131  if sys.platform == 'darwin':
132    platform = 'Mac'
133  elif sys.platform == 'win32':
134    platform = 'Win'
135  else:
136    platform = 'Linux_x64'
137
138  # Check if Google Cloud Storage already has the artifacts we want to build.
139  if (args.upload and GsutilArchiveExists(pdir, platform) and
140      not sys.platform.startswith('linux') or
141      GsutilArchiveExists(golddir, platform)):
142    print ('Desired toolchain revision %s is already available '
143           'in Google Cloud Storage:') % expected_stamp
144    print 'gs://chromium-browser-clang/%s/%s.tgz' % (platform, pdir)
145    if sys.platform.startswith('linux'):
146      print 'gs://chromium-browser-clang/%s/%s.tgz' % (platform, golddir)
147    return 0
148
149  with open('buildlog.txt', 'w') as log:
150    Tee('Diff in llvm:\n', log)
151    TeeCmd(['svn', 'stat', LLVM_DIR], log, fail_hard=False)
152    TeeCmd(['svn', 'diff', LLVM_DIR], log, fail_hard=False)
153    Tee('Diff in llvm/tools/clang:\n', log)
154    TeeCmd(['svn', 'stat', os.path.join(LLVM_DIR, 'tools', 'clang')],
155           log, fail_hard=False)
156    TeeCmd(['svn', 'diff', os.path.join(LLVM_DIR, 'tools', 'clang')],
157           log, fail_hard=False)
158    # TODO(thakis): compiler-rt is in projects/compiler-rt on Windows but
159    # llvm/compiler-rt elsewhere. So this diff call is currently only right on
160    # Windows.
161    Tee('Diff in llvm/compiler-rt:\n', log)
162    TeeCmd(['svn', 'stat', os.path.join(LLVM_DIR, 'projects', 'compiler-rt')],
163           log, fail_hard=False)
164    TeeCmd(['svn', 'diff', os.path.join(LLVM_DIR, 'projects', 'compiler-rt')],
165           log, fail_hard=False)
166    Tee('Diff in llvm/projects/libcxx:\n', log)
167    TeeCmd(['svn', 'stat', os.path.join(LLVM_DIR, 'projects', 'libcxx')],
168           log, fail_hard=False)
169    TeeCmd(['svn', 'diff', os.path.join(LLVM_DIR, 'projects', 'libcxx')],
170           log, fail_hard=False)
171
172    Tee('Starting build\n', log)
173
174    # Do a clobber build.
175    shutil.rmtree(LLVM_BOOTSTRAP_DIR, ignore_errors=True)
176    shutil.rmtree(LLVM_BOOTSTRAP_INSTALL_DIR, ignore_errors=True)
177    shutil.rmtree(LLVM_BUILD_DIR, ignore_errors=True)
178
179    opt_flags = []
180    if sys.platform.startswith('linux'):
181      opt_flags += ['--lto-gold-plugin']
182    build_cmd = [sys.executable, os.path.join(THIS_DIR, 'update.py'),
183                 '--bootstrap', '--force-local-build',
184                 '--run-tests'] + opt_flags
185    TeeCmd(build_cmd, log)
186
187  stamp = open(STAMP_FILE).read().rstrip()
188  if stamp != expected_stamp:
189    print 'Actual stamp (%s) != expected stamp (%s).' % (stamp, expected_stamp)
190    return 1
191
192  shutil.rmtree(pdir, ignore_errors=True)
193
194  # Copy a whitelist of files to the directory we're going to tar up.
195  # This supports the same patterns that the fnmatch module understands.
196  exe_ext = '.exe' if sys.platform == 'win32' else ''
197  want = ['bin/llvm-symbolizer' + exe_ext,
198          'bin/sancov' + exe_ext,
199          'lib/clang/*/asan_blacklist.txt',
200          'lib/clang/*/cfi_blacklist.txt',
201          # Copy built-in headers (lib/clang/3.x.y/include).
202          'lib/clang/*/include/*',
203          ]
204  if sys.platform == 'win32':
205    want.append('bin/clang-cl.exe')
206    want.append('bin/lld-link.exe')
207  else:
208    so_ext = 'dylib' if sys.platform == 'darwin' else 'so'
209    want.extend(['bin/clang',
210                 'lib/libFindBadConstructs.' + so_ext,
211                 'lib/libBlinkGCPlugin.' + so_ext,
212                 ])
213  if sys.platform == 'darwin':
214    want.extend([# Copy only the OSX and iossim (ASan and profile) runtime
215                 # libraries:
216                 'lib/clang/*/lib/darwin/*asan_osx*',
217                 'lib/clang/*/lib/darwin/*asan_iossim*',
218                 'lib/clang/*/lib/darwin/*profile_osx*',
219                 'lib/clang/*/lib/darwin/*profile_iossim*',
220                 ])
221  elif sys.platform.startswith('linux'):
222    # Copy the libstdc++.so.6 we linked Clang against so it can run.
223    want.append('lib/libstdc++.so.6')
224    # Copy only
225    # lib/clang/*/lib/linux/libclang_rt.{[atm]san,san,ubsan,profile}-*.a ,
226    # but not dfsan.
227    want.extend(['lib/clang/*/lib/linux/*[atm]san*',
228                 'lib/clang/*/lib/linux/*ubsan*',
229                 'lib/clang/*/lib/linux/*libclang_rt.san*',
230                 'lib/clang/*/lib/linux/*profile*',
231                 'lib/clang/*/msan_blacklist.txt',
232                 ])
233  elif sys.platform == 'win32':
234    want.extend(['lib/clang/*/lib/windows/clang_rt.asan*.dll',
235                 'lib/clang/*/lib/windows/clang_rt.asan*.lib',
236                 'lib/clang/*/include_sanitizer/*',
237                 ])
238
239  for root, dirs, files in os.walk(LLVM_RELEASE_DIR):
240    # root: third_party/llvm-build/Release+Asserts/lib/..., rel_root: lib/...
241    rel_root = root[len(LLVM_RELEASE_DIR)+1:]
242    rel_files = [os.path.join(rel_root, f) for f in files]
243    wanted_files = list(set(itertools.chain.from_iterable(
244        fnmatch.filter(rel_files, p) for p in want)))
245    if wanted_files:
246      # Guaranteed to not yet exist at this point:
247      os.makedirs(os.path.join(pdir, rel_root))
248    for f in wanted_files:
249      src = os.path.join(LLVM_RELEASE_DIR, f)
250      dest = os.path.join(pdir, f)
251      shutil.copy(src, dest)
252      # Strip libraries.
253      if sys.platform == 'darwin' and f.endswith('.dylib'):
254        subprocess.call(['strip', '-x', dest])
255      elif (sys.platform.startswith('linux') and
256            os.path.splitext(f)[1] in ['.so', '.a']):
257        subprocess.call(['strip', '-g', dest])
258
259  # Set up symlinks.
260  if sys.platform != 'win32':
261    os.symlink('clang', os.path.join(pdir, 'bin', 'clang++'))
262    os.symlink('clang', os.path.join(pdir, 'bin', 'clang-cl'))
263
264  # Copy libc++ headers.
265  if sys.platform == 'darwin':
266    shutil.copytree(os.path.join(LLVM_BOOTSTRAP_INSTALL_DIR, 'include', 'c++'),
267                    os.path.join(pdir, 'include', 'c++'))
268
269  # Copy buildlog over.
270  shutil.copy('buildlog.txt', pdir)
271
272  # Create archive.
273  tar_entries = ['bin', 'lib', 'buildlog.txt']
274  if sys.platform == 'darwin':
275    tar_entries += ['include']
276  with tarfile.open(pdir + '.tgz', 'w:gz') as tar:
277    for entry in tar_entries:
278      tar.add(os.path.join(pdir, entry), arcname=entry, filter=PrintTarProgress)
279
280  MaybeUpload(args, pdir, platform)
281
282  # Zip up gold plugin on Linux.
283  if sys.platform.startswith('linux'):
284    shutil.rmtree(golddir, ignore_errors=True)
285    os.makedirs(os.path.join(golddir, 'lib'))
286    shutil.copy(os.path.join(LLVM_LTO_GOLD_PLUGIN_DIR, 'lib', 'LLVMgold.so'),
287                os.path.join(golddir, 'lib'))
288    with tarfile.open(golddir + '.tgz', 'w:gz') as tar:
289      tar.add(os.path.join(golddir, 'lib'), arcname='lib',
290              filter=PrintTarProgress)
291    MaybeUpload(args, golddir, platform)
292
293  # Zip up llvm-objdump for sanitizer coverage.
294  objdumpdir = 'llvmobjdump-' + stamp
295  shutil.rmtree(objdumpdir, ignore_errors=True)
296  os.makedirs(os.path.join(objdumpdir, 'bin'))
297  shutil.copy(os.path.join(LLVM_RELEASE_DIR, 'bin', 'llvm-objdump' + exe_ext),
298              os.path.join(objdumpdir, 'bin'))
299  with tarfile.open(objdumpdir + '.tgz', 'w:gz') as tar:
300    tar.add(os.path.join(objdumpdir, 'bin'), arcname='bin',
301            filter=PrintTarProgress)
302  MaybeUpload(args, objdumpdir, platform)
303
304  # FIXME: Warn if the file already exists on the server.
305
306
307if __name__ == '__main__':
308  sys.exit(main())
309