• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/env python
2# Copyright (c) 2012 The Chromium Authors. All rights reserved.
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5
6"""This script is used to download prebuilt clang binaries.
7
8It is also used by package.py to build the prebuilt clang binaries."""
9
10import argparse
11import distutils.spawn
12import glob
13import os
14import pipes
15import re
16import shutil
17import subprocess
18import stat
19import sys
20import tarfile
21import tempfile
22import time
23import urllib2
24import zipfile
25
26
27# Do NOT CHANGE this if you don't know what you're doing -- see
28# https://chromium.googlesource.com/chromium/src/+/master/docs/updating_clang.md
29# Reverting problematic clang rolls is safe, though.
30CLANG_REVISION = '289944'
31
32use_head_revision = 'LLVM_FORCE_HEAD_REVISION' in os.environ
33if use_head_revision:
34  CLANG_REVISION = 'HEAD'
35
36# This is incremented when pushing a new build of Clang at the same revision.
37CLANG_SUB_REVISION=2
38
39PACKAGE_VERSION = "%s-%s" % (CLANG_REVISION, CLANG_SUB_REVISION)
40
41# Path constants. (All of these should be absolute paths.)
42THIS_DIR = os.path.abspath(os.path.dirname(__file__))
43CHROMIUM_DIR = os.path.abspath(os.path.join(THIS_DIR, '..', '..', '..'))
44THIRD_PARTY_DIR = os.path.join(CHROMIUM_DIR, 'third_party')
45LLVM_DIR = os.path.join(THIRD_PARTY_DIR, 'llvm')
46LLVM_BOOTSTRAP_DIR = os.path.join(THIRD_PARTY_DIR, 'llvm-bootstrap')
47LLVM_BOOTSTRAP_INSTALL_DIR = os.path.join(THIRD_PARTY_DIR,
48                                          'llvm-bootstrap-install')
49LLVM_LTO_GOLD_PLUGIN_DIR = os.path.join(THIRD_PARTY_DIR, 'llvm-lto-gold-plugin')
50CHROME_TOOLS_SHIM_DIR = os.path.join(LLVM_DIR, 'tools', 'chrometools')
51LLVM_BUILD_DIR = os.path.join(CHROMIUM_DIR, 'third_party', 'llvm-build',
52                              'Release+Asserts')
53COMPILER_RT_BUILD_DIR = os.path.join(LLVM_BUILD_DIR, 'compiler-rt')
54CLANG_DIR = os.path.join(LLVM_DIR, 'tools', 'clang')
55LLD_DIR = os.path.join(LLVM_DIR, 'tools', 'lld')
56# compiler-rt is built as part of the regular LLVM build on Windows to get
57# the 64-bit runtime, and out-of-tree elsewhere.
58# TODO(thakis): Try to unify this.
59if sys.platform == 'win32':
60  COMPILER_RT_DIR = os.path.join(LLVM_DIR, 'projects', 'compiler-rt')
61else:
62  COMPILER_RT_DIR = os.path.join(LLVM_DIR, 'compiler-rt')
63LIBCXX_DIR = os.path.join(LLVM_DIR, 'projects', 'libcxx')
64LIBCXXABI_DIR = os.path.join(LLVM_DIR, 'projects', 'libcxxabi')
65LLVM_BUILD_TOOLS_DIR = os.path.abspath(
66    os.path.join(LLVM_DIR, '..', 'llvm-build-tools'))
67STAMP_FILE = os.path.normpath(
68    os.path.join(LLVM_DIR, '..', 'llvm-build', 'cr_build_revision'))
69BINUTILS_DIR = os.path.join(THIRD_PARTY_DIR, 'binutils')
70BINUTILS_BIN_DIR = os.path.join(BINUTILS_DIR, BINUTILS_DIR,
71                                'Linux_x64', 'Release', 'bin')
72BFD_PLUGINS_DIR = os.path.join(BINUTILS_DIR, 'Linux_x64', 'Release',
73                               'lib', 'bfd-plugins')
74VERSION = '4.0.0'
75ANDROID_NDK_DIR = os.path.join(
76    CHROMIUM_DIR, 'third_party', 'android_tools', 'ndk')
77
78# URL for pre-built binaries.
79CDS_URL = os.environ.get('CDS_CLANG_BUCKET_OVERRIDE',
80    'https://commondatastorage.googleapis.com/chromium-browser-clang')
81
82LLVM_REPO_URL='https://llvm.org/svn/llvm-project'
83if 'LLVM_REPO_URL' in os.environ:
84  LLVM_REPO_URL = os.environ['LLVM_REPO_URL']
85
86# Bump after VC updates.
87DIA_DLL = {
88  '2013': 'msdia120.dll',
89  '2015': 'msdia140.dll',
90}
91
92
93def DownloadUrl(url, output_file):
94  """Download url into output_file."""
95  CHUNK_SIZE = 4096
96  TOTAL_DOTS = 10
97  num_retries = 3
98  retry_wait_s = 5  # Doubled at each retry.
99
100  while True:
101    try:
102      sys.stdout.write('Downloading %s ' % url)
103      sys.stdout.flush()
104      response = urllib2.urlopen(url)
105      total_size = int(response.info().getheader('Content-Length').strip())
106      bytes_done = 0
107      dots_printed = 0
108      while True:
109        chunk = response.read(CHUNK_SIZE)
110        if not chunk:
111          break
112        output_file.write(chunk)
113        bytes_done += len(chunk)
114        num_dots = TOTAL_DOTS * bytes_done / total_size
115        sys.stdout.write('.' * (num_dots - dots_printed))
116        sys.stdout.flush()
117        dots_printed = num_dots
118      if bytes_done != total_size:
119        raise urllib2.URLError("only got %d of %d bytes" %
120                               (bytes_done, total_size))
121      print ' Done.'
122      return
123    except urllib2.URLError as e:
124      sys.stdout.write('\n')
125      print e
126      if num_retries == 0 or isinstance(e, urllib2.HTTPError) and e.code == 404:
127        raise e
128      num_retries -= 1
129      print 'Retrying in %d s ...' % retry_wait_s
130      time.sleep(retry_wait_s)
131      retry_wait_s *= 2
132
133
134def EnsureDirExists(path):
135  if not os.path.exists(path):
136    print "Creating directory %s" % path
137    os.makedirs(path)
138
139
140def DownloadAndUnpack(url, output_dir):
141  with tempfile.TemporaryFile() as f:
142    DownloadUrl(url, f)
143    f.seek(0)
144    EnsureDirExists(output_dir)
145    if url.endswith('.zip'):
146      zipfile.ZipFile(f).extractall(path=output_dir)
147    else:
148      tarfile.open(mode='r:gz', fileobj=f).extractall(path=output_dir)
149
150
151def ReadStampFile(path=STAMP_FILE):
152  """Return the contents of the stamp file, or '' if it doesn't exist."""
153  try:
154    with open(path, 'r') as f:
155      return f.read().rstrip()
156  except IOError:
157    return ''
158
159
160def WriteStampFile(s, path=STAMP_FILE):
161  """Write s to the stamp file."""
162  EnsureDirExists(os.path.dirname(path))
163  with open(path, 'w') as f:
164    f.write(s)
165    f.write('\n')
166
167
168def GetSvnRevision(svn_repo):
169  """Returns current revision of the svn repo at svn_repo."""
170  svn_info = subprocess.check_output('svn info ' + svn_repo, shell=True)
171  m = re.search(r'Revision: (\d+)', svn_info)
172  return m.group(1)
173
174
175def RmTree(dir):
176  """Delete dir."""
177  def ChmodAndRetry(func, path, _):
178    # Subversion can leave read-only files around.
179    if not os.access(path, os.W_OK):
180      os.chmod(path, stat.S_IWUSR)
181      return func(path)
182    raise
183
184  shutil.rmtree(dir, onerror=ChmodAndRetry)
185
186
187def RmCmakeCache(dir):
188  """Delete CMake cache related files from dir."""
189  for dirpath, dirs, files in os.walk(dir):
190    if 'CMakeCache.txt' in files:
191      os.remove(os.path.join(dirpath, 'CMakeCache.txt'))
192    if 'CMakeFiles' in dirs:
193      RmTree(os.path.join(dirpath, 'CMakeFiles'))
194
195
196def RunCommand(command, msvc_arch=None, env=None, fail_hard=True):
197  """Run command and return success (True) or failure; or if fail_hard is
198     True, exit on failure.  If msvc_arch is set, runs the command in a
199     shell with the msvc tools for that architecture."""
200
201  if msvc_arch and sys.platform == 'win32':
202    command = GetVSVersion().SetupScript(msvc_arch) + ['&&'] + command
203
204  # https://docs.python.org/2/library/subprocess.html:
205  # "On Unix with shell=True [...] if args is a sequence, the first item
206  # specifies the command string, and any additional items will be treated as
207  # additional arguments to the shell itself.  That is to say, Popen does the
208  # equivalent of:
209  #   Popen(['/bin/sh', '-c', args[0], args[1], ...])"
210  #
211  # We want to pass additional arguments to command[0], not to the shell,
212  # so manually join everything into a single string.
213  # Annoyingly, for "svn co url c:\path", pipes.quote() thinks that it should
214  # quote c:\path but svn can't handle quoted paths on Windows.  Since on
215  # Windows follow-on args are passed to args[0] instead of the shell, don't
216  # do the single-string transformation there.
217  if sys.platform != 'win32':
218    command = ' '.join([pipes.quote(c) for c in command])
219  print 'Running', command
220  if subprocess.call(command, env=env, shell=True) == 0:
221    return True
222  print 'Failed.'
223  if fail_hard:
224    sys.exit(1)
225  return False
226
227
228def CopyFile(src, dst):
229  """Copy a file from src to dst."""
230  print "Copying %s to %s" % (src, dst)
231  shutil.copy(src, dst)
232
233
234def CopyDirectoryContents(src, dst, filename_filter=None):
235  """Copy the files from directory src to dst
236  with an optional filename filter."""
237  dst = os.path.realpath(dst)  # realpath() in case dst ends in /..
238  EnsureDirExists(dst)
239  for root, _, files in os.walk(src):
240    for f in files:
241      if filename_filter and not re.match(filename_filter, f):
242        continue
243      CopyFile(os.path.join(root, f), dst)
244
245
246def Checkout(name, url, dir):
247  """Checkout the SVN module at url into dir. Use name for the log message."""
248  print "Checking out %s r%s into '%s'" % (name, CLANG_REVISION, dir)
249
250  command = ['svn', 'checkout', '--force', url + '@' + CLANG_REVISION, dir]
251  if RunCommand(command, fail_hard=False):
252    return
253
254  if os.path.isdir(dir):
255    print "Removing %s." % (dir)
256    RmTree(dir)
257
258  print "Retrying."
259  RunCommand(command)
260
261
262def DeleteChromeToolsShim():
263  OLD_SHIM_DIR = os.path.join(LLVM_DIR, 'tools', 'zzz-chrometools')
264  shutil.rmtree(OLD_SHIM_DIR, ignore_errors=True)
265  shutil.rmtree(CHROME_TOOLS_SHIM_DIR, ignore_errors=True)
266
267
268def CreateChromeToolsShim():
269  """Hooks the Chrome tools into the LLVM build.
270
271  Several Chrome tools have dependencies on LLVM/Clang libraries. The LLVM build
272  detects implicit tools in the tools subdirectory, so this helper install a
273  shim CMakeLists.txt that forwards to the real directory for the Chrome tools.
274
275  Note that the shim directory name intentionally has no - or _. The implicit
276  tool detection logic munges them in a weird way."""
277  assert not any(i in os.path.basename(CHROME_TOOLS_SHIM_DIR) for i in '-_')
278  os.mkdir(CHROME_TOOLS_SHIM_DIR)
279  with file(os.path.join(CHROME_TOOLS_SHIM_DIR, 'CMakeLists.txt'), 'w') as f:
280    f.write('# Automatically generated by tools/clang/scripts/update.py. ' +
281            'Do not edit.\n')
282    f.write('# Since tools/clang is located in another directory, use the \n')
283    f.write('# two arg version to specify where build artifacts go. CMake\n')
284    f.write('# disallows reuse of the same binary dir for multiple source\n')
285    f.write('# dirs, so the build artifacts need to go into a subdirectory.\n')
286    f.write('# dirs, so the build artifacts need to go into a subdirectory.\n')
287    f.write('if (CHROMIUM_TOOLS_SRC)\n')
288    f.write('  add_subdirectory(${CHROMIUM_TOOLS_SRC} ' +
289              '${CMAKE_CURRENT_BINARY_DIR}/a)\n')
290    f.write('endif (CHROMIUM_TOOLS_SRC)\n')
291
292
293def DownloadHostGcc(args):
294  """Downloads gcc 4.8.5 and makes sure args.gcc_toolchain is set."""
295  if not sys.platform.startswith('linux') or args.gcc_toolchain:
296    return
297  # Unconditionally download a prebuilt gcc to guarantee the included libstdc++
298  # works on Ubuntu Precise.
299  gcc_dir = os.path.join(LLVM_BUILD_TOOLS_DIR, 'gcc485precise')
300  if not os.path.exists(gcc_dir):
301    print 'Downloading pre-built GCC 4.8.5...'
302    DownloadAndUnpack(
303        CDS_URL + '/tools/gcc485precise.tgz', LLVM_BUILD_TOOLS_DIR)
304  args.gcc_toolchain = gcc_dir
305
306
307def AddSvnToPathOnWin():
308  """Download svn.exe and add it to PATH."""
309  if sys.platform != 'win32':
310    return
311  svn_ver = 'svn-1.6.6-win'
312  svn_dir = os.path.join(LLVM_BUILD_TOOLS_DIR, svn_ver)
313  if not os.path.exists(svn_dir):
314    DownloadAndUnpack(CDS_URL + '/tools/%s.zip' % svn_ver, LLVM_BUILD_TOOLS_DIR)
315  os.environ['PATH'] = svn_dir + os.pathsep + os.environ.get('PATH', '')
316
317
318def AddCMakeToPath():
319  """Download CMake and add it to PATH."""
320  if sys.platform == 'win32':
321    zip_name = 'cmake-3.4.3-win32-x86.zip'
322    cmake_dir = os.path.join(LLVM_BUILD_TOOLS_DIR,
323                             'cmake-3.4.3-win32-x86', 'bin')
324  else:
325    suffix = 'Darwin' if sys.platform == 'darwin' else 'Linux'
326    zip_name = 'cmake343_%s.tgz' % suffix
327    cmake_dir = os.path.join(LLVM_BUILD_TOOLS_DIR, 'cmake343', 'bin')
328  if not os.path.exists(cmake_dir):
329    DownloadAndUnpack(CDS_URL + '/tools/' + zip_name, LLVM_BUILD_TOOLS_DIR)
330  os.environ['PATH'] = cmake_dir + os.pathsep + os.environ.get('PATH', '')
331
332
333def AddGnuWinToPath():
334  """Download some GNU win tools and add them to PATH."""
335  if sys.platform != 'win32':
336    return
337
338  gnuwin_dir = os.path.join(LLVM_BUILD_TOOLS_DIR, 'gnuwin')
339  GNUWIN_VERSION = '5'
340  GNUWIN_STAMP = os.path.join(gnuwin_dir, 'stamp')
341  if ReadStampFile(GNUWIN_STAMP) == GNUWIN_VERSION:
342    print 'GNU Win tools already up to date.'
343  else:
344    zip_name = 'gnuwin-%s.zip' % GNUWIN_VERSION
345    DownloadAndUnpack(CDS_URL + '/tools/' + zip_name, LLVM_BUILD_TOOLS_DIR)
346    WriteStampFile(GNUWIN_VERSION, GNUWIN_STAMP)
347
348  os.environ['PATH'] = gnuwin_dir + os.pathsep + os.environ.get('PATH', '')
349
350
351vs_version = None
352def GetVSVersion():
353  global vs_version
354  if vs_version:
355    return vs_version
356
357  # Try using the toolchain in depot_tools.
358  # This sets environment variables used by SelectVisualStudioVersion below.
359  sys.path.append(os.path.join(CHROMIUM_DIR, 'build'))
360  import vs_toolchain
361  vs_toolchain.SetEnvironmentAndGetRuntimeDllDirs()
362
363  # Use gyp to find the MSVS installation, either in depot_tools as per above,
364  # or a system-wide installation otherwise.
365  sys.path.append(os.path.join(CHROMIUM_DIR, 'tools', 'gyp', 'pylib'))
366  import gyp.MSVSVersion
367  vs_version = gyp.MSVSVersion.SelectVisualStudioVersion(
368      vs_toolchain.GetVisualStudioVersion())
369  return vs_version
370
371
372def CopyDiaDllTo(target_dir):
373  # This script always wants to use the 64-bit msdia*.dll.
374  dia_path = os.path.join(GetVSVersion().Path(), 'DIA SDK', 'bin', 'amd64')
375  dia_dll = os.path.join(dia_path, DIA_DLL[GetVSVersion().ShortName()])
376  CopyFile(dia_dll, target_dir)
377
378
379def VeryifyVersionOfBuiltClangMatchesVERSION():
380  """Checks that `clang --version` outputs VERSION.  If this fails, VERSION
381  in this file is out-of-date and needs to be updated (possibly in an
382  `if use_head_revision:` block in main() first)."""
383  clang = os.path.join(LLVM_BUILD_DIR, 'bin', 'clang')
384  if sys.platform == 'win32':
385    # TODO: Parse `clang-cl /?` output for built clang's version and check that
386    # to check the binary we're actually shipping? But clang-cl.exe is just
387    # a copy of clang.exe, so this does check the same thing.
388    clang += '.exe'
389  version_out = subprocess.check_output([clang, '--version'])
390  version_out = re.match(r'clang version ([0-9.]+)', version_out).group(1)
391  if version_out != VERSION:
392    print ('unexpected clang version %s (not %s), update VERSION in update.py'
393           % (version_out, VERSION))
394    sys.exit(1)
395
396
397def UpdateClang(args):
398  print 'Updating Clang to %s...' % PACKAGE_VERSION
399
400  need_gold_plugin = 'LLVM_DOWNLOAD_GOLD_PLUGIN' in os.environ or (
401      sys.platform.startswith('linux') and
402      'buildtype=Official' in os.environ.get('GYP_DEFINES', ''))
403
404  if ReadStampFile() == PACKAGE_VERSION and not args.force_local_build:
405    print 'Clang is already up to date.'
406    if not need_gold_plugin or os.path.exists(
407        os.path.join(LLVM_BUILD_DIR, "lib/LLVMgold.so")):
408      return 0
409
410  # Reset the stamp file in case the build is unsuccessful.
411  WriteStampFile('')
412
413  if not args.force_local_build:
414    cds_file = "clang-%s.tgz" %  PACKAGE_VERSION
415    if sys.platform == 'win32' or sys.platform == 'cygwin':
416      cds_full_url = CDS_URL + '/Win/' + cds_file
417    elif sys.platform == 'darwin':
418      cds_full_url = CDS_URL + '/Mac/' + cds_file
419    else:
420      assert sys.platform.startswith('linux')
421      cds_full_url = CDS_URL + '/Linux_x64/' + cds_file
422
423    print 'Downloading prebuilt clang'
424    if os.path.exists(LLVM_BUILD_DIR):
425      RmTree(LLVM_BUILD_DIR)
426    try:
427      DownloadAndUnpack(cds_full_url, LLVM_BUILD_DIR)
428      print 'clang %s unpacked' % PACKAGE_VERSION
429      if sys.platform == 'win32':
430        CopyDiaDllTo(os.path.join(LLVM_BUILD_DIR, 'bin'))
431      # Download the gold plugin if requested to by an environment variable.
432      # This is used by the CFI ClusterFuzz bot, and it's required for official
433      # builds on linux.
434      if need_gold_plugin:
435        RunCommand(['python', CHROMIUM_DIR+'/build/download_gold_plugin.py'])
436      WriteStampFile(PACKAGE_VERSION)
437      return 0
438    except urllib2.URLError:
439      print 'Failed to download prebuilt clang %s' % cds_file
440      print 'Use --force-local-build if you want to build locally.'
441      print 'Exiting.'
442      return 1
443
444  if args.with_android and not os.path.exists(ANDROID_NDK_DIR):
445    print 'Android NDK not found at ' + ANDROID_NDK_DIR
446    print 'The Android NDK is needed to build a Clang whose -fsanitize=address'
447    print 'works on Android. See '
448    print 'https://www.chromium.org/developers/how-tos/android-build-instructions'
449    print 'for how to install the NDK, or pass --without-android.'
450    return 1
451
452  DownloadHostGcc(args)
453  AddSvnToPathOnWin()
454  AddCMakeToPath()
455  AddGnuWinToPath()
456
457  DeleteChromeToolsShim()
458
459  Checkout('LLVM', LLVM_REPO_URL + '/llvm/trunk', LLVM_DIR)
460  Checkout('Clang', LLVM_REPO_URL + '/cfe/trunk', CLANG_DIR)
461  if sys.platform == 'win32' or use_head_revision:
462    Checkout('LLD', LLVM_REPO_URL + '/lld/trunk', LLD_DIR)
463  elif os.path.exists(LLD_DIR):
464    # In case someone sends a tryjob that temporary adds lld to the checkout,
465    # make sure it's not around on future builds.
466    RmTree(LLD_DIR)
467  Checkout('compiler-rt', LLVM_REPO_URL + '/compiler-rt/trunk', COMPILER_RT_DIR)
468  if sys.platform == 'darwin':
469    # clang needs a libc++ checkout, else -stdlib=libc++ won't find includes
470    # (i.e. this is needed for bootstrap builds).
471    Checkout('libcxx', LLVM_REPO_URL + '/libcxx/trunk', LIBCXX_DIR)
472    # We used to check out libcxxabi on OS X; we no longer need that.
473    if os.path.exists(LIBCXXABI_DIR):
474      RmTree(LIBCXXABI_DIR)
475
476  cc, cxx = None, None
477  libstdcpp = None
478  if args.gcc_toolchain:  # This option is only used on Linux.
479    # Use the specified gcc installation for building.
480    cc = os.path.join(args.gcc_toolchain, 'bin', 'gcc')
481    cxx = os.path.join(args.gcc_toolchain, 'bin', 'g++')
482
483    if not os.access(cc, os.X_OK):
484      print 'Invalid --gcc-toolchain: "%s"' % args.gcc_toolchain
485      print '"%s" does not appear to be valid.' % cc
486      return 1
487
488    # Set LD_LIBRARY_PATH to make auxiliary targets (tablegen, bootstrap
489    # compiler, etc.) find the .so.
490    libstdcpp = subprocess.check_output(
491        [cxx, '-print-file-name=libstdc++.so.6']).rstrip()
492    os.environ['LD_LIBRARY_PATH'] = os.path.dirname(libstdcpp)
493
494  cflags = []
495  cxxflags = []
496  ldflags = []
497
498  base_cmake_args = ['-GNinja',
499                     '-DCMAKE_BUILD_TYPE=Release',
500                     '-DLLVM_ENABLE_ASSERTIONS=ON',
501                     '-DLLVM_ENABLE_THREADS=OFF',
502                     # Statically link MSVCRT to avoid DLL dependencies.
503                     '-DLLVM_USE_CRT_RELEASE=MT',
504                     ]
505
506  binutils_incdir = ''
507  if sys.platform.startswith('linux'):
508    binutils_incdir = os.path.join(BINUTILS_DIR, 'Linux_x64/Release/include')
509
510  if args.bootstrap:
511    print 'Building bootstrap compiler'
512    EnsureDirExists(LLVM_BOOTSTRAP_DIR)
513    os.chdir(LLVM_BOOTSTRAP_DIR)
514    bootstrap_args = base_cmake_args + [
515        '-DLLVM_BINUTILS_INCDIR=' + binutils_incdir,
516        '-DLLVM_TARGETS_TO_BUILD=host',
517        '-DCMAKE_INSTALL_PREFIX=' + LLVM_BOOTSTRAP_INSTALL_DIR,
518        '-DCMAKE_C_FLAGS=' + ' '.join(cflags),
519        '-DCMAKE_CXX_FLAGS=' + ' '.join(cxxflags),
520        ]
521    if cc is not None:  bootstrap_args.append('-DCMAKE_C_COMPILER=' + cc)
522    if cxx is not None: bootstrap_args.append('-DCMAKE_CXX_COMPILER=' + cxx)
523    RmCmakeCache('.')
524    RunCommand(['cmake'] + bootstrap_args + [LLVM_DIR], msvc_arch='x64')
525    RunCommand(['ninja'], msvc_arch='x64')
526    if args.run_tests:
527      if sys.platform == 'win32':
528        CopyDiaDllTo(os.path.join(LLVM_BOOTSTRAP_DIR, 'bin'))
529      RunCommand(['ninja', 'check-all'], msvc_arch='x64')
530    RunCommand(['ninja', 'install'], msvc_arch='x64')
531    if args.gcc_toolchain:
532      # Copy that gcc's stdlibc++.so.6 to the build dir, so the bootstrap
533      # compiler can start.
534      CopyFile(libstdcpp, os.path.join(LLVM_BOOTSTRAP_INSTALL_DIR, 'lib'))
535
536    if sys.platform == 'win32':
537      cc = os.path.join(LLVM_BOOTSTRAP_INSTALL_DIR, 'bin', 'clang-cl.exe')
538      cxx = os.path.join(LLVM_BOOTSTRAP_INSTALL_DIR, 'bin', 'clang-cl.exe')
539      # CMake has a hard time with backslashes in compiler paths:
540      # https://stackoverflow.com/questions/13050827
541      cc = cc.replace('\\', '/')
542      cxx = cxx.replace('\\', '/')
543    else:
544      cc = os.path.join(LLVM_BOOTSTRAP_INSTALL_DIR, 'bin', 'clang')
545      cxx = os.path.join(LLVM_BOOTSTRAP_INSTALL_DIR, 'bin', 'clang++')
546
547    if args.gcc_toolchain:
548      # Tell the bootstrap compiler to use a specific gcc prefix to search
549      # for standard library headers and shared object files.
550      cflags = ['--gcc-toolchain=' + args.gcc_toolchain]
551      cxxflags = ['--gcc-toolchain=' + args.gcc_toolchain]
552    print 'Building final compiler'
553
554  # Build LLVM gold plugin with LTO. That speeds up the linker by ~10%.
555  # We only use LTO for Linux now.
556  if args.bootstrap and args.lto_gold_plugin:
557    print 'Building LTO LLVM Gold plugin'
558    if os.path.exists(LLVM_LTO_GOLD_PLUGIN_DIR):
559      RmTree(LLVM_LTO_GOLD_PLUGIN_DIR)
560    EnsureDirExists(LLVM_LTO_GOLD_PLUGIN_DIR)
561    os.chdir(LLVM_LTO_GOLD_PLUGIN_DIR)
562
563    # Create a symlink to LLVMgold.so build in the previous step so that ar
564    # and ranlib could find it while linking LLVMgold.so with LTO.
565    EnsureDirExists(BFD_PLUGINS_DIR)
566    RunCommand(['ln', '-sf',
567                os.path.join(LLVM_BOOTSTRAP_INSTALL_DIR, 'lib', 'LLVMgold.so'),
568                os.path.join(BFD_PLUGINS_DIR, 'LLVMgold.so')])
569
570    lto_cflags = ['-flto']
571    lto_ldflags = ['-fuse-ld=gold']
572    if args.gcc_toolchain:
573      # Tell the bootstrap compiler to use a specific gcc prefix to search
574      # for standard library headers and shared object files.
575      lto_cflags += ['--gcc-toolchain=' + args.gcc_toolchain]
576    lto_cmake_args = base_cmake_args + [
577        '-DLLVM_BINUTILS_INCDIR=' + binutils_incdir,
578        '-DCMAKE_C_COMPILER=' + cc,
579        '-DCMAKE_CXX_COMPILER=' + cxx,
580        '-DCMAKE_C_FLAGS=' + ' '.join(lto_cflags),
581        '-DCMAKE_CXX_FLAGS=' + ' '.join(lto_cflags),
582        '-DCMAKE_EXE_LINKER_FLAGS=' + ' '.join(lto_ldflags),
583        '-DCMAKE_SHARED_LINKER_FLAGS=' + ' '.join(lto_ldflags),
584        '-DCMAKE_MODULE_LINKER_FLAGS=' + ' '.join(lto_ldflags)]
585
586    # We need to use the proper binutils which support LLVM Gold plugin.
587    lto_env = os.environ.copy()
588    lto_env['PATH'] = BINUTILS_BIN_DIR + os.pathsep + lto_env.get('PATH', '')
589
590    RmCmakeCache('.')
591    RunCommand(['cmake'] + lto_cmake_args + [LLVM_DIR], env=lto_env)
592    RunCommand(['ninja', 'LLVMgold'], env=lto_env)
593
594
595  # LLVM uses C++11 starting in llvm 3.5. On Linux, this means libstdc++4.7+ is
596  # needed, on OS X it requires libc++. clang only automatically links to libc++
597  # when targeting OS X 10.9+, so add stdlib=libc++ explicitly so clang can run
598  # on OS X versions as old as 10.7.
599  deployment_target = ''
600
601  if sys.platform == 'darwin' and args.bootstrap:
602    # When building on 10.9, /usr/include usually doesn't exist, and while
603    # Xcode's clang automatically sets a sysroot, self-built clangs don't.
604    cflags = ['-isysroot', subprocess.check_output(
605        ['xcrun', '--show-sdk-path']).rstrip()]
606    cxxflags = ['-stdlib=libc++'] + cflags
607    ldflags += ['-stdlib=libc++']
608    deployment_target = '10.7'
609    # Running libc++ tests takes a long time. Since it was only needed for
610    # the install step above, don't build it as part of the main build.
611    # This makes running package.py over 10% faster (30 min instead of 34 min)
612    RmTree(LIBCXX_DIR)
613
614  # Build clang.
615
616  # If building at head, define a macro that plugins can use for #ifdefing
617  # out code that builds at head, but not at CLANG_REVISION or vice versa.
618  if use_head_revision:
619    cflags += ['-DLLVM_FORCE_HEAD_REVISION']
620    cxxflags += ['-DLLVM_FORCE_HEAD_REVISION']
621
622  CreateChromeToolsShim()
623
624  deployment_env = None
625  if deployment_target:
626    deployment_env = os.environ.copy()
627    deployment_env['MACOSX_DEPLOYMENT_TARGET'] = deployment_target
628
629  cmake_args = []
630  # TODO(thakis): Unconditionally append this to base_cmake_args instead once
631  # compiler-rt can build with clang-cl on Windows (http://llvm.org/PR23698)
632  cc_args = base_cmake_args if sys.platform != 'win32' else cmake_args
633  if cc is not None:  cc_args.append('-DCMAKE_C_COMPILER=' + cc)
634  if cxx is not None: cc_args.append('-DCMAKE_CXX_COMPILER=' + cxx)
635  chrome_tools = list(set(['plugins', 'blink_gc_plugin'] + args.extra_tools))
636  cmake_args += base_cmake_args + [
637      '-DLLVM_BINUTILS_INCDIR=' + binutils_incdir,
638      '-DCMAKE_C_FLAGS=' + ' '.join(cflags),
639      '-DCMAKE_CXX_FLAGS=' + ' '.join(cxxflags),
640      '-DCMAKE_EXE_LINKER_FLAGS=' + ' '.join(ldflags),
641      '-DCMAKE_SHARED_LINKER_FLAGS=' + ' '.join(ldflags),
642      '-DCMAKE_MODULE_LINKER_FLAGS=' + ' '.join(ldflags),
643      '-DCMAKE_INSTALL_PREFIX=' + LLVM_BUILD_DIR,
644      # TODO(thakis): Remove this once official builds pass -Wl,--build-id
645      # explicitly, https://crbug.com/622775
646      '-DENABLE_LINKER_BUILD_ID=ON',
647      '-DCHROMIUM_TOOLS_SRC=%s' % os.path.join(CHROMIUM_DIR, 'tools', 'clang'),
648      '-DCHROMIUM_TOOLS=%s' % ';'.join(chrome_tools)]
649
650  EnsureDirExists(LLVM_BUILD_DIR)
651  os.chdir(LLVM_BUILD_DIR)
652  RmCmakeCache('.')
653  RunCommand(['cmake'] + cmake_args + [LLVM_DIR],
654             msvc_arch='x64', env=deployment_env)
655
656  if args.gcc_toolchain:
657    # Copy in the right stdlibc++.so.6 so clang can start.
658    if not os.path.exists(os.path.join(LLVM_BUILD_DIR, 'lib')):
659      os.mkdir(os.path.join(LLVM_BUILD_DIR, 'lib'))
660    libstdcpp = subprocess.check_output(
661        [cxx] + cxxflags + ['-print-file-name=libstdc++.so.6']).rstrip()
662    CopyFile(libstdcpp, os.path.join(LLVM_BUILD_DIR, 'lib'))
663
664  RunCommand(['ninja'], msvc_arch='x64')
665
666  if chrome_tools:
667    # If any Chromium tools were built, install those now.
668    RunCommand(['ninja', 'cr-install'], msvc_arch='x64')
669
670  if sys.platform == 'darwin':
671    # See http://crbug.com/256342
672    RunCommand(['strip', '-x', os.path.join(LLVM_BUILD_DIR, 'bin', 'clang')])
673  elif sys.platform.startswith('linux'):
674    RunCommand(['strip', os.path.join(LLVM_BUILD_DIR, 'bin', 'clang')])
675
676  VeryifyVersionOfBuiltClangMatchesVERSION()
677
678  # Do an out-of-tree build of compiler-rt.
679  # On Windows, this is used to get the 32-bit ASan run-time.
680  # TODO(hans): Remove once the regular build above produces this.
681  # On Mac and Linux, this is used to get the regular 64-bit run-time.
682  # Do a clobbered build due to cmake changes.
683  if os.path.isdir(COMPILER_RT_BUILD_DIR):
684    RmTree(COMPILER_RT_BUILD_DIR)
685  os.makedirs(COMPILER_RT_BUILD_DIR)
686  os.chdir(COMPILER_RT_BUILD_DIR)
687  # TODO(thakis): Add this once compiler-rt can build with clang-cl (see
688  # above).
689  #if args.bootstrap and sys.platform == 'win32':
690    # The bootstrap compiler produces 64-bit binaries by default.
691    #cflags += ['-m32']
692    #cxxflags += ['-m32']
693  compiler_rt_args = base_cmake_args + [
694      '-DCMAKE_C_FLAGS=' + ' '.join(cflags),
695      '-DCMAKE_CXX_FLAGS=' + ' '.join(cxxflags)]
696  if sys.platform == 'darwin':
697    compiler_rt_args += ['-DCOMPILER_RT_ENABLE_IOS=ON']
698  if sys.platform != 'win32':
699    compiler_rt_args += ['-DLLVM_CONFIG_PATH=' +
700                         os.path.join(LLVM_BUILD_DIR, 'bin', 'llvm-config'),
701                        '-DSANITIZER_MIN_OSX_VERSION="10.7"']
702  # compiler-rt is part of the llvm checkout on Windows but a stand-alone
703  # directory elsewhere, see the TODO above COMPILER_RT_DIR.
704  RmCmakeCache('.')
705  RunCommand(['cmake'] + compiler_rt_args +
706             [LLVM_DIR if sys.platform == 'win32' else COMPILER_RT_DIR],
707             msvc_arch='x86', env=deployment_env)
708  RunCommand(['ninja', 'compiler-rt'], msvc_arch='x86')
709
710  # Copy select output to the main tree.
711  # TODO(hans): Make this (and the .gypi and .isolate files) version number
712  # independent.
713  if sys.platform == 'win32':
714    platform = 'windows'
715  elif sys.platform == 'darwin':
716    platform = 'darwin'
717  else:
718    assert sys.platform.startswith('linux')
719    platform = 'linux'
720  asan_rt_lib_src_dir = os.path.join(COMPILER_RT_BUILD_DIR, 'lib', platform)
721  if sys.platform == 'win32':
722    # TODO(thakis): This too is due to compiler-rt being part of the checkout
723    # on Windows, see TODO above COMPILER_RT_DIR.
724    asan_rt_lib_src_dir = os.path.join(COMPILER_RT_BUILD_DIR, 'lib', 'clang',
725                                       VERSION, 'lib', platform)
726  asan_rt_lib_dst_dir = os.path.join(LLVM_BUILD_DIR, 'lib', 'clang',
727                                     VERSION, 'lib', platform)
728  # Blacklists:
729  CopyDirectoryContents(os.path.join(asan_rt_lib_src_dir, '..', '..'),
730                        os.path.join(asan_rt_lib_dst_dir, '..', '..'),
731                        r'^.*blacklist\.txt$')
732  # Headers:
733  if sys.platform != 'win32':
734    CopyDirectoryContents(
735        os.path.join(COMPILER_RT_BUILD_DIR, 'include/sanitizer'),
736        os.path.join(LLVM_BUILD_DIR, 'lib/clang', VERSION, 'include/sanitizer'))
737  # Static and dynamic libraries:
738  CopyDirectoryContents(asan_rt_lib_src_dir, asan_rt_lib_dst_dir)
739  if sys.platform == 'darwin':
740    for dylib in glob.glob(os.path.join(asan_rt_lib_dst_dir, '*.dylib')):
741      # Fix LC_ID_DYLIB for the ASan dynamic libraries to be relative to
742      # @executable_path.
743      # TODO(glider): this is transitional. We'll need to fix the dylib
744      # name either in our build system, or in Clang. See also
745      # http://crbug.com/344836.
746      subprocess.call(['install_name_tool', '-id',
747                       '@executable_path/' + os.path.basename(dylib), dylib])
748
749
750  if sys.platform == 'win32':
751    # Make an extra copy of the sanitizer headers, to be put on the include path
752    # of the fallback compiler.
753    sanitizer_include_dir = os.path.join(LLVM_BUILD_DIR, 'lib', 'clang',
754                                         VERSION, 'include', 'sanitizer')
755    aux_sanitizer_include_dir = os.path.join(LLVM_BUILD_DIR, 'lib', 'clang',
756                                             VERSION, 'include_sanitizer',
757                                             'sanitizer')
758    EnsureDirExists(aux_sanitizer_include_dir)
759    for _, _, files in os.walk(sanitizer_include_dir):
760      for f in files:
761        CopyFile(os.path.join(sanitizer_include_dir, f),
762                 aux_sanitizer_include_dir)
763
764  if args.with_android:
765    make_toolchain = os.path.join(
766        ANDROID_NDK_DIR, 'build', 'tools', 'make_standalone_toolchain.py')
767    for target_arch in ['aarch64', 'arm', 'i686']:
768      # Make standalone Android toolchain for target_arch.
769      toolchain_dir = os.path.join(
770          LLVM_BUILD_DIR, 'android-toolchain-' + target_arch)
771      RunCommand([
772          make_toolchain,
773          '--api=' + ('21' if target_arch == 'aarch64' else '19'),
774          '--force',
775          '--install-dir=%s' % toolchain_dir,
776          '--stl=stlport',
777          '--arch=' + {
778              'aarch64': 'arm64',
779              'arm': 'arm',
780              'i686': 'x86',
781          }[target_arch]])
782      # Android NDK r9d copies a broken unwind.h into the toolchain, see
783      # http://crbug.com/357890
784      for f in glob.glob(os.path.join(toolchain_dir, 'include/c++/*/unwind.h')):
785        os.remove(f)
786
787      # Build ASan runtime for Android in a separate build tree.
788      build_dir = os.path.join(LLVM_BUILD_DIR, 'android-' + target_arch)
789      if not os.path.exists(build_dir):
790        os.mkdir(os.path.join(build_dir))
791      os.chdir(build_dir)
792      cflags = ['--target=%s-linux-androideabi' % target_arch,
793                '--sysroot=%s/sysroot' % toolchain_dir,
794                '-B%s' % toolchain_dir]
795      android_args = base_cmake_args + [
796        '-DCMAKE_C_COMPILER=' + os.path.join(LLVM_BUILD_DIR, 'bin/clang'),
797        '-DCMAKE_CXX_COMPILER=' + os.path.join(LLVM_BUILD_DIR, 'bin/clang++'),
798        '-DLLVM_CONFIG_PATH=' + os.path.join(LLVM_BUILD_DIR, 'bin/llvm-config'),
799        '-DCMAKE_C_FLAGS=' + ' '.join(cflags),
800        '-DCMAKE_CXX_FLAGS=' + ' '.join(cflags),
801        '-DANDROID=1']
802      RmCmakeCache('.')
803      RunCommand(['cmake'] + android_args + [COMPILER_RT_DIR])
804      RunCommand(['ninja', 'libclang_rt.asan-%s-android.so' % target_arch])
805
806      # And copy it into the main build tree.
807      runtime = 'libclang_rt.asan-%s-android.so' % target_arch
808      for root, _, files in os.walk(build_dir):
809        if runtime in files:
810          shutil.copy(os.path.join(root, runtime), asan_rt_lib_dst_dir)
811
812  # Run tests.
813  if args.run_tests or use_head_revision:
814    os.chdir(LLVM_BUILD_DIR)
815    RunCommand(['ninja', 'cr-check-all'], msvc_arch='x64')
816  if args.run_tests:
817    if sys.platform == 'win32':
818      CopyDiaDllTo(os.path.join(LLVM_BUILD_DIR, 'bin'))
819    os.chdir(LLVM_BUILD_DIR)
820    RunCommand(['ninja', 'check-all'], msvc_arch='x64')
821
822  WriteStampFile(PACKAGE_VERSION)
823  print 'Clang update was successful.'
824  return 0
825
826
827def main():
828  parser = argparse.ArgumentParser(description='Build Clang.')
829  parser.add_argument('--bootstrap', action='store_true',
830                      help='first build clang with CC, then with itself.')
831  parser.add_argument('--if-needed', action='store_true',
832                      help="run only if the script thinks clang is needed")
833  parser.add_argument('--force-local-build', action='store_true',
834                      help="don't try to download prebuild binaries")
835  parser.add_argument('--gcc-toolchain', help='set the version for which gcc '
836                      'version be used for building; --gcc-toolchain=/opt/foo '
837                      'picks /opt/foo/bin/gcc')
838  parser.add_argument('--lto-gold-plugin', action='store_true',
839                      help='build LLVM Gold plugin with LTO')
840  parser.add_argument('--llvm-force-head-revision', action='store_true',
841                      help=('use the revision in the repo when printing '
842                            'the revision'))
843  parser.add_argument('--print-revision', action='store_true',
844                      help='print current clang revision and exit.')
845  parser.add_argument('--print-clang-version', action='store_true',
846                      help='print current clang version (e.g. x.y.z) and exit.')
847  parser.add_argument('--run-tests', action='store_true',
848                      help='run tests after building; only for local builds')
849  parser.add_argument('--extra-tools', nargs='*', default=[],
850                      help='select additional chrome tools to build')
851  parser.add_argument('--without-android', action='store_false',
852                      help='don\'t build Android ASan runtime (linux only)',
853                      dest='with_android',
854                      default=sys.platform.startswith('linux'))
855  args = parser.parse_args()
856
857  if args.lto_gold_plugin and not args.bootstrap:
858    print '--lto-gold-plugin requires --bootstrap'
859    return 1
860  if args.lto_gold_plugin and not sys.platform.startswith('linux'):
861    print '--lto-gold-plugin is only effective on Linux. Ignoring the option.'
862    args.lto_gold_plugin = False
863
864  if args.if_needed:
865    is_clang_required = False
866    # clang is always used on Mac and Linux.
867    if sys.platform == 'darwin' or sys.platform.startswith('linux'):
868      is_clang_required = True
869    # clang requested via $GYP_DEFINES.
870    if re.search(r'\b(clang|asan|lsan|msan|tsan)=1',
871                 os.environ.get('GYP_DEFINES', '')):
872      is_clang_required = True
873    # clang previously downloaded, keep it up to date.
874    # If you don't want this, delete third_party/llvm-build on your machine.
875    if os.path.isdir(LLVM_BUILD_DIR):
876      is_clang_required = True
877    if not is_clang_required:
878      return 0
879    if re.search(r'\b(make_clang_dir)=', os.environ.get('GYP_DEFINES', '')):
880      print 'Skipping Clang update (make_clang_dir= was set in GYP_DEFINES).'
881      return 0
882
883  if use_head_revision:
884    # TODO(hans): Trunk was updated; remove after the next roll.
885    global VERSION
886    VERSION = '5.0.0'
887
888  global CLANG_REVISION, PACKAGE_VERSION
889  if args.print_revision:
890    if use_head_revision or args.llvm_force_head_revision:
891      print GetSvnRevision(LLVM_DIR)
892    else:
893      print PACKAGE_VERSION
894    return 0
895
896  if args.print_clang_version:
897    sys.stdout.write(VERSION)
898    return 0
899
900  # Don't buffer stdout, so that print statements are immediately flushed.
901  # Do this only after --print-revision has been handled, else we'll get
902  # an error message when this script is run from gn for some reason.
903  sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 0)
904
905  if use_head_revision:
906    # Use a real revision number rather than HEAD to make sure that the stamp
907    # file logic works.
908    CLANG_REVISION = GetSvnRevision(LLVM_REPO_URL)
909    PACKAGE_VERSION = CLANG_REVISION + '-0'
910
911    args.force_local_build = True
912    if 'OS=android' not in os.environ.get('GYP_DEFINES', ''):
913      # Only build the Android ASan rt on ToT bots when targetting Android.
914      args.with_android = False
915
916  return UpdateClang(args)
917
918
919if __name__ == '__main__':
920  sys.exit(main())
921