• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/env python
2# Copyright (c) 2012 The Chromium Authors. All rights reserved.
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5
6"""This script is used to download prebuilt clang binaries.
7
8It is also used by package.py to build the prebuilt clang binaries."""
9
10import argparse
11import distutils.spawn
12import glob
13import os
14import pipes
15import re
16import shutil
17import subprocess
18import stat
19import sys
20import tarfile
21import tempfile
22import time
23import urllib2
24import zipfile
25
26
27# Do NOT CHANGE this if you don't know what you're doing -- see
28# https://chromium.googlesource.com/chromium/src/+/master/docs/updating_clang.md
29# Reverting problematic clang rolls is safe, though.
30CLANG_REVISION = '338452'
31
32use_head_revision = bool(os.environ.get('LLVM_FORCE_HEAD_REVISION', '0')
33                         in ('1', 'YES'))
34if use_head_revision:
35  CLANG_REVISION = 'HEAD'
36
37# This is incremented when pushing a new build of Clang at the same revision.
38CLANG_SUB_REVISION=1
39
40PACKAGE_VERSION = "%s-%s" % (CLANG_REVISION, CLANG_SUB_REVISION)
41
42# Path constants. (All of these should be absolute paths.)
43THIS_DIR = os.path.abspath(os.path.dirname(__file__))
44CHROMIUM_DIR = os.path.abspath(os.path.join(THIS_DIR, '..', '..', '..'))
45GCLIENT_CONFIG = os.path.join(os.path.dirname(CHROMIUM_DIR), '.gclient')
46THIRD_PARTY_DIR = os.path.join(CHROMIUM_DIR, 'third_party')
47LLVM_DIR = os.path.join(THIRD_PARTY_DIR, 'llvm')
48LLVM_BOOTSTRAP_DIR = os.path.join(THIRD_PARTY_DIR, 'llvm-bootstrap')
49LLVM_BOOTSTRAP_INSTALL_DIR = os.path.join(THIRD_PARTY_DIR,
50                                          'llvm-bootstrap-install')
51CHROME_TOOLS_SHIM_DIR = os.path.join(LLVM_DIR, 'tools', 'chrometools')
52LLVM_BUILD_DIR = os.path.join(CHROMIUM_DIR, 'third_party', 'llvm-build',
53                              'Release+Asserts')
54THREADS_ENABLED_BUILD_DIR = os.path.join(LLVM_BUILD_DIR, 'threads_enabled')
55COMPILER_RT_BUILD_DIR = os.path.join(LLVM_BUILD_DIR, 'compiler-rt')
56CLANG_DIR = os.path.join(LLVM_DIR, 'tools', 'clang')
57LLD_DIR = os.path.join(LLVM_DIR, 'tools', 'lld')
58# compiler-rt is built as part of the regular LLVM build on Windows to get
59# the 64-bit runtime, and out-of-tree elsewhere.
60# TODO(thakis): Try to unify this.
61if sys.platform == 'win32':
62  COMPILER_RT_DIR = os.path.join(LLVM_DIR, 'projects', 'compiler-rt')
63else:
64  COMPILER_RT_DIR = os.path.join(LLVM_DIR, 'compiler-rt')
65LIBCXX_DIR = os.path.join(LLVM_DIR, 'projects', 'libcxx')
66LIBCXXABI_DIR = os.path.join(LLVM_DIR, 'projects', 'libcxxabi')
67LLVM_BUILD_TOOLS_DIR = os.path.abspath(
68    os.path.join(LLVM_DIR, '..', 'llvm-build-tools'))
69STAMP_FILE = os.path.normpath(
70    os.path.join(LLVM_DIR, '..', 'llvm-build', 'cr_build_revision'))
71VERSION = '7.0.0'
72ANDROID_NDK_DIR = os.path.join(
73    CHROMIUM_DIR, 'third_party', 'android_ndk')
74
75# URL for pre-built binaries.
76CDS_URL = os.environ.get('CDS_CLANG_BUCKET_OVERRIDE',
77    'https://commondatastorage.googleapis.com/chromium-browser-clang')
78
79LLVM_REPO_URL='https://llvm.org/svn/llvm-project'
80if 'LLVM_REPO_URL' in os.environ:
81  LLVM_REPO_URL = os.environ['LLVM_REPO_URL']
82
83
84
85def DownloadUrl(url, output_file):
86  """Download url into output_file."""
87  CHUNK_SIZE = 4096
88  TOTAL_DOTS = 10
89  num_retries = 3
90  retry_wait_s = 5  # Doubled at each retry.
91
92  while True:
93    try:
94      sys.stdout.write('Downloading %s ' % url)
95      sys.stdout.flush()
96      response = urllib2.urlopen(url)
97      total_size = int(response.info().getheader('Content-Length').strip())
98      bytes_done = 0
99      dots_printed = 0
100      while True:
101        chunk = response.read(CHUNK_SIZE)
102        if not chunk:
103          break
104        output_file.write(chunk)
105        bytes_done += len(chunk)
106        num_dots = TOTAL_DOTS * bytes_done / total_size
107        sys.stdout.write('.' * (num_dots - dots_printed))
108        sys.stdout.flush()
109        dots_printed = num_dots
110      if bytes_done != total_size:
111        raise urllib2.URLError("only got %d of %d bytes" %
112                               (bytes_done, total_size))
113      print ' Done.'
114      return
115    except urllib2.URLError as e:
116      sys.stdout.write('\n')
117      print e
118      if num_retries == 0 or isinstance(e, urllib2.HTTPError) and e.code == 404:
119        raise e
120      num_retries -= 1
121      print 'Retrying in %d s ...' % retry_wait_s
122      time.sleep(retry_wait_s)
123      retry_wait_s *= 2
124
125
126def EnsureDirExists(path):
127  if not os.path.exists(path):
128    os.makedirs(path)
129
130
131def DownloadAndUnpack(url, output_dir, path_prefix=None):
132  """Download an archive from url and extract into output_dir. If path_prefix is
133     not None, only extract files whose paths within the archive start with
134     path_prefix."""
135  with tempfile.TemporaryFile() as f:
136    DownloadUrl(url, f)
137    f.seek(0)
138    EnsureDirExists(output_dir)
139    if url.endswith('.zip'):
140      assert path_prefix is None
141      zipfile.ZipFile(f).extractall(path=output_dir)
142    else:
143      t = tarfile.open(mode='r:gz', fileobj=f)
144      members = None
145      if path_prefix is not None:
146        members = [m for m in t.getmembers() if m.name.startswith(path_prefix)]
147      t.extractall(path=output_dir, members=members)
148
149
150def ReadStampFile(path=STAMP_FILE):
151  """Return the contents of the stamp file, or '' if it doesn't exist."""
152  try:
153    with open(path, 'r') as f:
154      return f.read().rstrip()
155  except IOError:
156    return ''
157
158
159def WriteStampFile(s, path=STAMP_FILE):
160  """Write s to the stamp file."""
161  EnsureDirExists(os.path.dirname(path))
162  with open(path, 'w') as f:
163    f.write(s)
164    f.write('\n')
165
166
167def GetSvnRevision(svn_repo):
168  """Returns current revision of the svn repo at svn_repo."""
169  svn_info = subprocess.check_output('svn info ' + svn_repo, shell=True)
170  m = re.search(r'Revision: (\d+)', svn_info)
171  return m.group(1)
172
173
174def RmTree(dir):
175  """Delete dir."""
176  def ChmodAndRetry(func, path, _):
177    # Subversion can leave read-only files around.
178    if not os.access(path, os.W_OK):
179      os.chmod(path, stat.S_IWUSR)
180      return func(path)
181    raise
182
183  shutil.rmtree(dir, onerror=ChmodAndRetry)
184
185
186def RmCmakeCache(dir):
187  """Delete CMake cache related files from dir."""
188  for dirpath, dirs, files in os.walk(dir):
189    if 'CMakeCache.txt' in files:
190      os.remove(os.path.join(dirpath, 'CMakeCache.txt'))
191    if 'CMakeFiles' in dirs:
192      RmTree(os.path.join(dirpath, 'CMakeFiles'))
193
194
195def RunCommand(command, msvc_arch=None, env=None, fail_hard=True):
196  """Run command and return success (True) or failure; or if fail_hard is
197     True, exit on failure.  If msvc_arch is set, runs the command in a
198     shell with the msvc tools for that architecture."""
199
200  if msvc_arch and sys.platform == 'win32':
201    command = [os.path.join(GetWinSDKDir(), 'bin', 'SetEnv.cmd'),
202               "/" + msvc_arch, '&&'] + command
203
204  # https://docs.python.org/2/library/subprocess.html:
205  # "On Unix with shell=True [...] if args is a sequence, the first item
206  # specifies the command string, and any additional items will be treated as
207  # additional arguments to the shell itself.  That is to say, Popen does the
208  # equivalent of:
209  #   Popen(['/bin/sh', '-c', args[0], args[1], ...])"
210  #
211  # We want to pass additional arguments to command[0], not to the shell,
212  # so manually join everything into a single string.
213  # Annoyingly, for "svn co url c:\path", pipes.quote() thinks that it should
214  # quote c:\path but svn can't handle quoted paths on Windows.  Since on
215  # Windows follow-on args are passed to args[0] instead of the shell, don't
216  # do the single-string transformation there.
217  if sys.platform != 'win32':
218    command = ' '.join([pipes.quote(c) for c in command])
219  print 'Running', command
220  if subprocess.call(command, env=env, shell=True) == 0:
221    return True
222  print 'Failed.'
223  if fail_hard:
224    sys.exit(1)
225  return False
226
227
228def CopyFile(src, dst):
229  """Copy a file from src to dst."""
230  print "Copying %s to %s" % (src, dst)
231  shutil.copy(src, dst)
232
233
234def CopyDirectoryContents(src, dst):
235  """Copy the files from directory src to dst."""
236  dst = os.path.realpath(dst)  # realpath() in case dst ends in /..
237  EnsureDirExists(dst)
238  for f in os.listdir(src):
239    CopyFile(os.path.join(src, f), dst)
240
241
242def Checkout(name, url, dir):
243  """Checkout the SVN module at url into dir. Use name for the log message."""
244  print "Checking out %s r%s into '%s'" % (name, CLANG_REVISION, dir)
245
246  command = ['svn', 'checkout', '--force', url + '@' + CLANG_REVISION, dir]
247  if RunCommand(command, fail_hard=False):
248    return
249
250  if os.path.isdir(dir):
251    print "Removing %s." % (dir)
252    RmTree(dir)
253
254  print "Retrying."
255  RunCommand(command)
256
257
258def CheckoutRepos(args):
259  if args.skip_checkout:
260    return
261
262  Checkout('LLVM', LLVM_REPO_URL + '/llvm/trunk', LLVM_DIR)
263  Checkout('Clang', LLVM_REPO_URL + '/cfe/trunk', CLANG_DIR)
264  if True:
265    Checkout('LLD', LLVM_REPO_URL + '/lld/trunk', LLD_DIR)
266  elif os.path.exists(LLD_DIR):
267    # In case someone sends a tryjob that temporary adds lld to the checkout,
268    # make sure it's not around on future builds.
269    RmTree(LLD_DIR)
270  Checkout('compiler-rt', LLVM_REPO_URL + '/compiler-rt/trunk', COMPILER_RT_DIR)
271  if sys.platform == 'darwin':
272    # clang needs a libc++ checkout, else -stdlib=libc++ won't find includes
273    # (i.e. this is needed for bootstrap builds).
274    Checkout('libcxx', LLVM_REPO_URL + '/libcxx/trunk', LIBCXX_DIR)
275    # We used to check out libcxxabi on OS X; we no longer need that.
276    if os.path.exists(LIBCXXABI_DIR):
277      RmTree(LIBCXXABI_DIR)
278
279
280def DeleteChromeToolsShim():
281  OLD_SHIM_DIR = os.path.join(LLVM_DIR, 'tools', 'zzz-chrometools')
282  shutil.rmtree(OLD_SHIM_DIR, ignore_errors=True)
283  shutil.rmtree(CHROME_TOOLS_SHIM_DIR, ignore_errors=True)
284
285
286def CreateChromeToolsShim():
287  """Hooks the Chrome tools into the LLVM build.
288
289  Several Chrome tools have dependencies on LLVM/Clang libraries. The LLVM build
290  detects implicit tools in the tools subdirectory, so this helper install a
291  shim CMakeLists.txt that forwards to the real directory for the Chrome tools.
292
293  Note that the shim directory name intentionally has no - or _. The implicit
294  tool detection logic munges them in a weird way."""
295  assert not any(i in os.path.basename(CHROME_TOOLS_SHIM_DIR) for i in '-_')
296  os.mkdir(CHROME_TOOLS_SHIM_DIR)
297  with file(os.path.join(CHROME_TOOLS_SHIM_DIR, 'CMakeLists.txt'), 'w') as f:
298    f.write('# Automatically generated by tools/clang/scripts/update.py. ' +
299            'Do not edit.\n')
300    f.write('# Since tools/clang is located in another directory, use the \n')
301    f.write('# two arg version to specify where build artifacts go. CMake\n')
302    f.write('# disallows reuse of the same binary dir for multiple source\n')
303    f.write('# dirs, so the build artifacts need to go into a subdirectory.\n')
304    f.write('if (CHROMIUM_TOOLS_SRC)\n')
305    f.write('  add_subdirectory(${CHROMIUM_TOOLS_SRC} ' +
306              '${CMAKE_CURRENT_BINARY_DIR}/a)\n')
307    f.write('endif (CHROMIUM_TOOLS_SRC)\n')
308
309
310def AddSvnToPathOnWin():
311  """Download svn.exe and add it to PATH."""
312  if sys.platform != 'win32':
313    return
314  svn_ver = 'svn-1.6.6-win'
315  svn_dir = os.path.join(LLVM_BUILD_TOOLS_DIR, svn_ver)
316  if not os.path.exists(svn_dir):
317    DownloadAndUnpack(CDS_URL + '/tools/%s.zip' % svn_ver, LLVM_BUILD_TOOLS_DIR)
318  os.environ['PATH'] = svn_dir + os.pathsep + os.environ.get('PATH', '')
319
320
321def AddCMakeToPath(args):
322  """Download CMake and add it to PATH."""
323  if args.use_system_cmake:
324    return
325
326  if sys.platform == 'win32':
327    zip_name = 'cmake-3.12.1-win32-x86.zip'
328    dir_name = ['cmake-3.12.1-win32-x86', 'bin']
329  elif sys.platform == 'darwin':
330    zip_name = 'cmake-3.12.1-Darwin-x86_64.tar.gz'
331    dir_name = ['cmake-3.12.1-Darwin-x86_64', 'CMake.app', 'Contents', 'bin']
332  else:
333    zip_name = 'cmake-3.12.1-Linux-x86_64.tar.gz'
334    dir_name = ['cmake-3.12.1-Linux-x86_64', 'bin']
335
336  cmake_dir = os.path.join(LLVM_BUILD_TOOLS_DIR, *dir_name)
337  if not os.path.exists(cmake_dir):
338    DownloadAndUnpack(CDS_URL + '/tools/' + zip_name, LLVM_BUILD_TOOLS_DIR)
339  os.environ['PATH'] = cmake_dir + os.pathsep + os.environ.get('PATH', '')
340
341
342def AddGnuWinToPath():
343  """Download some GNU win tools and add them to PATH."""
344  if sys.platform != 'win32':
345    return
346
347  gnuwin_dir = os.path.join(LLVM_BUILD_TOOLS_DIR, 'gnuwin')
348  GNUWIN_VERSION = '8'
349  GNUWIN_STAMP = os.path.join(gnuwin_dir, 'stamp')
350  if ReadStampFile(GNUWIN_STAMP) == GNUWIN_VERSION:
351    print 'GNU Win tools already up to date.'
352  else:
353    zip_name = 'gnuwin-%s.zip' % GNUWIN_VERSION
354    DownloadAndUnpack(CDS_URL + '/tools/' + zip_name, LLVM_BUILD_TOOLS_DIR)
355    WriteStampFile(GNUWIN_VERSION, GNUWIN_STAMP)
356
357  os.environ['PATH'] = gnuwin_dir + os.pathsep + os.environ.get('PATH', '')
358
359
360win_sdk_dir = None
361dia_dll = None
362def GetWinSDKDir():
363  """Get the location of the current SDK. Sets dia_dll as a side-effect."""
364  global win_sdk_dir
365  global dia_dll
366  if win_sdk_dir:
367    return win_sdk_dir
368
369  # Bump after VC updates.
370  DIA_DLL = {
371    '2013': 'msdia120.dll',
372    '2015': 'msdia140.dll',
373    '2017': 'msdia140.dll',
374  }
375
376  # Don't let vs_toolchain overwrite our environment.
377  environ_bak = os.environ
378
379  sys.path.append(os.path.join(CHROMIUM_DIR, 'build'))
380  import vs_toolchain
381  win_sdk_dir = vs_toolchain.SetEnvironmentAndGetSDKDir()
382  msvs_version = vs_toolchain.GetVisualStudioVersion()
383
384  if bool(int(os.environ.get('DEPOT_TOOLS_WIN_TOOLCHAIN', '1'))):
385    dia_path = os.path.join(win_sdk_dir, '..', 'DIA SDK', 'bin', 'amd64')
386  else:
387    if 'GYP_MSVS_OVERRIDE_PATH' not in os.environ:
388      vs_path = vs_toolchain.DetectVisualStudioPath()
389    else:
390      vs_path = os.environ['GYP_MSVS_OVERRIDE_PATH']
391    dia_path = os.path.join(vs_path, 'DIA SDK', 'bin', 'amd64')
392
393  dia_dll = os.path.join(dia_path, DIA_DLL[msvs_version])
394
395  os.environ = environ_bak
396  return win_sdk_dir
397
398
399def CopyDiaDllTo(target_dir):
400  # This script always wants to use the 64-bit msdia*.dll.
401  GetWinSDKDir()
402  CopyFile(dia_dll, target_dir)
403
404
405def VeryifyVersionOfBuiltClangMatchesVERSION():
406  """Checks that `clang --version` outputs VERSION.  If this fails, VERSION
407  in this file is out-of-date and needs to be updated (possibly in an
408  `if use_head_revision:` block in main() first)."""
409  clang = os.path.join(LLVM_BUILD_DIR, 'bin', 'clang')
410  if sys.platform == 'win32':
411    # TODO: Parse `clang-cl /?` output for built clang's version and check that
412    # to check the binary we're actually shipping? But clang-cl.exe is just
413    # a copy of clang.exe, so this does check the same thing.
414    clang += '.exe'
415  version_out = subprocess.check_output([clang, '--version'])
416  version_out = re.match(r'clang version ([0-9.]+)', version_out).group(1)
417  if version_out != VERSION:
418    print ('unexpected clang version %s (not %s), update VERSION in update.py'
419           % (version_out, VERSION))
420    sys.exit(1)
421
422
423def GetPlatformUrlPrefix(platform):
424  if platform == 'win32' or platform == 'cygwin':
425    return CDS_URL + '/Win/'
426  if platform == 'darwin':
427    return CDS_URL + '/Mac/'
428  assert platform.startswith('linux')
429  return CDS_URL + '/Linux_x64/'
430
431
432def DownloadAndUnpackClangPackage(platform, runtimes_only=False):
433  cds_file = "clang-%s.tgz" %  PACKAGE_VERSION
434  cds_full_url = GetPlatformUrlPrefix(platform) + cds_file
435  try:
436    path_prefix = None
437    if runtimes_only:
438      path_prefix = 'lib/clang/' + VERSION + '/lib/'
439    DownloadAndUnpack(cds_full_url, LLVM_BUILD_DIR, path_prefix)
440  except urllib2.URLError:
441    print 'Failed to download prebuilt clang %s' % cds_file
442    print 'Use --force-local-build if you want to build locally.'
443    print 'Exiting.'
444    sys.exit(1)
445
446
447def UpdateClang(args):
448  # Read target_os from .gclient so we know which non-native runtimes we need.
449  # TODO(pcc): See if we can download just the runtimes instead of the entire
450  # clang package, and do that from DEPS instead of here.
451  target_os = []
452  try:
453    env = {}
454    execfile(GCLIENT_CONFIG, env, env)
455    target_os = env.get('target_os', target_os)
456  except:
457    pass
458
459  expected_stamp = ','.join([PACKAGE_VERSION] + target_os)
460  if ReadStampFile() == expected_stamp and not args.force_local_build:
461    return 0
462
463  # Reset the stamp file in case the build is unsuccessful.
464  WriteStampFile('')
465
466  if not args.force_local_build:
467    if os.path.exists(LLVM_BUILD_DIR):
468      RmTree(LLVM_BUILD_DIR)
469
470    DownloadAndUnpackClangPackage(sys.platform)
471    if 'win' in target_os:
472      DownloadAndUnpackClangPackage('win32', runtimes_only=True)
473    if sys.platform == 'win32':
474      CopyDiaDllTo(os.path.join(LLVM_BUILD_DIR, 'bin'))
475    WriteStampFile(expected_stamp)
476    return 0
477
478  if args.with_android and not os.path.exists(ANDROID_NDK_DIR):
479    print 'Android NDK not found at ' + ANDROID_NDK_DIR
480    print 'The Android NDK is needed to build a Clang whose -fsanitize=address'
481    print 'works on Android. See '
482    print 'https://www.chromium.org/developers/how-tos/android-build-instructions'
483    print 'for how to install the NDK, or pass --without-android.'
484    return 1
485
486  print 'Locally building Clang %s...' % PACKAGE_VERSION
487
488  if use_head_revision:
489    # TODO(hans): Trunk version was updated; remove after the next roll.
490    # Remove the old lib dir.
491    old_lib_dir = os.path.join(LLVM_BUILD_DIR, 'lib', 'clang', '7.0.0')
492    if (os.path.isdir(old_lib_dir)):
493      print 'Removing old lib dir: %s' % old_lib_dir
494      RmTree(old_lib_dir)
495
496  AddCMakeToPath(args)
497  AddGnuWinToPath()
498
499  DeleteChromeToolsShim()
500
501  CheckoutRepos(args)
502
503  if args.skip_build:
504    return
505
506  cc, cxx = None, None
507  libstdcpp = None
508
509  cflags = []
510  cxxflags = []
511  ldflags = []
512
513  base_cmake_args = ['-GNinja',
514                     '-DCMAKE_BUILD_TYPE=Release',
515                     '-DLLVM_ENABLE_ASSERTIONS=ON',
516                     '-DLLVM_ENABLE_TERMINFO=OFF',
517                     # Statically link MSVCRT to avoid DLL dependencies.
518                     '-DLLVM_USE_CRT_RELEASE=MT',
519                     ]
520
521  if sys.platform != 'win32':
522    # libxml2 is required by the Win manifest merging tool used in cross-builds.
523    base_cmake_args.append('-DLLVM_ENABLE_LIBXML2=FORCE_ON')
524
525  if args.bootstrap:
526    print 'Building bootstrap compiler'
527    EnsureDirExists(LLVM_BOOTSTRAP_DIR)
528    os.chdir(LLVM_BOOTSTRAP_DIR)
529    bootstrap_args = base_cmake_args + [
530        '-DLLVM_TARGETS_TO_BUILD=X86;ARM;AArch64',
531        '-DCMAKE_INSTALL_PREFIX=' + LLVM_BOOTSTRAP_INSTALL_DIR,
532        '-DCMAKE_C_FLAGS=' + ' '.join(cflags),
533        '-DCMAKE_CXX_FLAGS=' + ' '.join(cxxflags),
534        ]
535    if cc is not None:  bootstrap_args.append('-DCMAKE_C_COMPILER=' + cc)
536    if cxx is not None: bootstrap_args.append('-DCMAKE_CXX_COMPILER=' + cxx)
537    RmCmakeCache('.')
538    RunCommand(['cmake'] + bootstrap_args + [LLVM_DIR], msvc_arch='x64')
539    RunCommand(['ninja'], msvc_arch='x64')
540    if args.run_tests:
541      if sys.platform == 'win32':
542        CopyDiaDllTo(os.path.join(LLVM_BOOTSTRAP_DIR, 'bin'))
543      RunCommand(['ninja', 'check-all'], msvc_arch='x64')
544    RunCommand(['ninja', 'install'], msvc_arch='x64')
545
546    if sys.platform == 'win32':
547      cc = os.path.join(LLVM_BOOTSTRAP_INSTALL_DIR, 'bin', 'clang-cl.exe')
548      cxx = os.path.join(LLVM_BOOTSTRAP_INSTALL_DIR, 'bin', 'clang-cl.exe')
549      # CMake has a hard time with backslashes in compiler paths:
550      # https://stackoverflow.com/questions/13050827
551      cc = cc.replace('\\', '/')
552      cxx = cxx.replace('\\', '/')
553    else:
554      cc = os.path.join(LLVM_BOOTSTRAP_INSTALL_DIR, 'bin', 'clang')
555      cxx = os.path.join(LLVM_BOOTSTRAP_INSTALL_DIR, 'bin', 'clang++')
556
557    print 'Building final compiler'
558
559  # LLVM uses C++11 starting in llvm 3.5. On Linux, this means libstdc++4.7+ is
560  # needed, on OS X it requires libc++. clang only automatically links to libc++
561  # when targeting OS X 10.9+, so add stdlib=libc++ explicitly so clang can run
562  # on OS X versions as old as 10.7.
563  deployment_target = ''
564
565  if sys.platform == 'darwin' and args.bootstrap:
566    # When building on 10.9, /usr/include usually doesn't exist, and while
567    # Xcode's clang automatically sets a sysroot, self-built clangs don't.
568    cflags = ['-isysroot', subprocess.check_output(
569        ['xcrun', '--show-sdk-path']).rstrip()]
570    cxxflags = ['-stdlib=libc++'] + cflags
571    ldflags += ['-stdlib=libc++']
572    deployment_target = '10.7'
573    # Running libc++ tests takes a long time. Since it was only needed for
574    # the install step above, don't build it as part of the main build.
575    # This makes running package.py over 10% faster (30 min instead of 34 min)
576    RmTree(LIBCXX_DIR)
577
578
579  # If building at head, define a macro that plugins can use for #ifdefing
580  # out code that builds at head, but not at CLANG_REVISION or vice versa.
581  if use_head_revision:
582    cflags += ['-DLLVM_FORCE_HEAD_REVISION']
583    cxxflags += ['-DLLVM_FORCE_HEAD_REVISION']
584
585  # Build PDBs for archival on Windows.  Don't use RelWithDebInfo since it
586  # has different optimization defaults than Release.
587  # Also disable stack cookies (/GS-) for performance.
588  if sys.platform == 'win32':
589    cflags += ['/Zi', '/GS-']
590    cxxflags += ['/Zi', '/GS-']
591    ldflags += ['/DEBUG', '/OPT:REF', '/OPT:ICF']
592
593  deployment_env = None
594  if deployment_target:
595    deployment_env = os.environ.copy()
596    deployment_env['MACOSX_DEPLOYMENT_TARGET'] = deployment_target
597
598  # Build lld and code coverage tools. This is done separately from the rest of
599  # the build because these tools require threading support.
600  tools_with_threading = [ 'lld', 'llvm-cov', 'llvm-profdata' ]
601  print 'Building the following tools with threading support: %s' % (
602        str(tools_with_threading))
603
604  if os.path.exists(THREADS_ENABLED_BUILD_DIR):
605    RmTree(THREADS_ENABLED_BUILD_DIR)
606  EnsureDirExists(THREADS_ENABLED_BUILD_DIR)
607  os.chdir(THREADS_ENABLED_BUILD_DIR)
608
609  threads_enabled_cmake_args = base_cmake_args + [
610      '-DCMAKE_C_FLAGS=' + ' '.join(cflags),
611      '-DCMAKE_CXX_FLAGS=' + ' '.join(cxxflags),
612      '-DCMAKE_EXE_LINKER_FLAGS=' + ' '.join(ldflags),
613      '-DCMAKE_SHARED_LINKER_FLAGS=' + ' '.join(ldflags),
614      '-DCMAKE_MODULE_LINKER_FLAGS=' + ' '.join(ldflags)]
615  if cc is not None:
616    threads_enabled_cmake_args.append('-DCMAKE_C_COMPILER=' + cc)
617  if cxx is not None:
618    threads_enabled_cmake_args.append('-DCMAKE_CXX_COMPILER=' + cxx)
619
620  if args.lto_lld:
621    # Build lld with LTO. That speeds up the linker by ~10%.
622    # We only use LTO for Linux now.
623    #
624    # The linker expects all archive members to have symbol tables, so the
625    # archiver needs to be able to create symbol tables for bitcode files.
626    # GNU ar and ranlib don't understand bitcode files, but llvm-ar and
627    # llvm-ranlib do, so use them.
628    ar = os.path.join(LLVM_BOOTSTRAP_INSTALL_DIR, 'bin', 'llvm-ar')
629    ranlib = os.path.join(LLVM_BOOTSTRAP_INSTALL_DIR, 'bin', 'llvm-ranlib')
630    threads_enabled_cmake_args += [
631        '-DCMAKE_AR=' + ar,
632        '-DCMAKE_RANLIB=' + ranlib,
633        '-DLLVM_ENABLE_LTO=thin',
634        '-DLLVM_USE_LINKER=lld']
635
636  RmCmakeCache('.')
637  RunCommand(['cmake'] + threads_enabled_cmake_args + [LLVM_DIR],
638             msvc_arch='x64', env=deployment_env)
639  RunCommand(['ninja'] + tools_with_threading, msvc_arch='x64')
640
641  # Build clang and other tools.
642  CreateChromeToolsShim()
643
644  cmake_args = []
645  # TODO(thakis): Unconditionally append this to base_cmake_args instead once
646  # compiler-rt can build with clang-cl on Windows (http://llvm.org/PR23698)
647  cc_args = base_cmake_args if sys.platform != 'win32' else cmake_args
648  if cc is not None:  cc_args.append('-DCMAKE_C_COMPILER=' + cc)
649  if cxx is not None: cc_args.append('-DCMAKE_CXX_COMPILER=' + cxx)
650  default_tools = ['plugins', 'blink_gc_plugin', 'translation_unit']
651  chrome_tools = list(set(default_tools + args.extra_tools))
652  cmake_args += base_cmake_args + [
653      '-DLLVM_ENABLE_THREADS=OFF',
654      '-DCMAKE_C_FLAGS=' + ' '.join(cflags),
655      '-DCMAKE_CXX_FLAGS=' + ' '.join(cxxflags),
656      '-DCMAKE_EXE_LINKER_FLAGS=' + ' '.join(ldflags),
657      '-DCMAKE_SHARED_LINKER_FLAGS=' + ' '.join(ldflags),
658      '-DCMAKE_MODULE_LINKER_FLAGS=' + ' '.join(ldflags),
659      '-DCMAKE_INSTALL_PREFIX=' + LLVM_BUILD_DIR,
660      '-DCHROMIUM_TOOLS_SRC=%s' % os.path.join(CHROMIUM_DIR, 'tools', 'clang'),
661      '-DCHROMIUM_TOOLS=%s' % ';'.join(chrome_tools)]
662
663  EnsureDirExists(LLVM_BUILD_DIR)
664  os.chdir(LLVM_BUILD_DIR)
665  RmCmakeCache('.')
666  RunCommand(['cmake'] + cmake_args + [LLVM_DIR],
667             msvc_arch='x64', env=deployment_env)
668  RunCommand(['ninja'], msvc_arch='x64')
669
670  # Copy in the threaded versions of lld and other tools.
671  if sys.platform == 'win32':
672    CopyFile(os.path.join(THREADS_ENABLED_BUILD_DIR, 'bin', 'lld-link.exe'),
673             os.path.join(LLVM_BUILD_DIR, 'bin'))
674    CopyFile(os.path.join(THREADS_ENABLED_BUILD_DIR, 'bin', 'lld.pdb'),
675             os.path.join(LLVM_BUILD_DIR, 'bin'))
676  else:
677    for tool in tools_with_threading:
678      CopyFile(os.path.join(THREADS_ENABLED_BUILD_DIR, 'bin', tool),
679               os.path.join(LLVM_BUILD_DIR, 'bin'))
680
681  if chrome_tools:
682    # If any Chromium tools were built, install those now.
683    RunCommand(['ninja', 'cr-install'], msvc_arch='x64')
684
685  VeryifyVersionOfBuiltClangMatchesVERSION()
686
687  # Do an out-of-tree build of compiler-rt.
688  # On Windows, this is used to get the 32-bit ASan run-time.
689  # TODO(hans): Remove once the regular build above produces this.
690  # On Mac and Linux, this is used to get the regular 64-bit run-time.
691  # Do a clobbered build due to cmake changes.
692  if os.path.isdir(COMPILER_RT_BUILD_DIR):
693    RmTree(COMPILER_RT_BUILD_DIR)
694  os.makedirs(COMPILER_RT_BUILD_DIR)
695  os.chdir(COMPILER_RT_BUILD_DIR)
696  # TODO(thakis): Add this once compiler-rt can build with clang-cl (see
697  # above).
698  #if args.bootstrap and sys.platform == 'win32':
699    # The bootstrap compiler produces 64-bit binaries by default.
700    #cflags += ['-m32']
701    #cxxflags += ['-m32']
702  compiler_rt_args = base_cmake_args + [
703      '-DLLVM_ENABLE_THREADS=OFF',
704      '-DCMAKE_C_FLAGS=' + ' '.join(cflags),
705      '-DCMAKE_CXX_FLAGS=' + ' '.join(cxxflags)]
706  if sys.platform == 'darwin':
707    compiler_rt_args += ['-DCOMPILER_RT_ENABLE_IOS=ON']
708  if sys.platform != 'win32':
709    compiler_rt_args += ['-DLLVM_CONFIG_PATH=' +
710                         os.path.join(LLVM_BUILD_DIR, 'bin', 'llvm-config'),
711                        '-DSANITIZER_MIN_OSX_VERSION="10.7"']
712  # compiler-rt is part of the llvm checkout on Windows but a stand-alone
713  # directory elsewhere, see the TODO above COMPILER_RT_DIR.
714  RmCmakeCache('.')
715  RunCommand(['cmake'] + compiler_rt_args +
716             [LLVM_DIR if sys.platform == 'win32' else COMPILER_RT_DIR],
717             msvc_arch='x86', env=deployment_env)
718  RunCommand(['ninja', 'compiler-rt'], msvc_arch='x86')
719  if sys.platform != 'win32':
720    RunCommand(['ninja', 'fuzzer'])
721
722  # Copy select output to the main tree.
723  # TODO(hans): Make this (and the .gypi and .isolate files) version number
724  # independent.
725  if sys.platform == 'win32':
726    platform = 'windows'
727  elif sys.platform == 'darwin':
728    platform = 'darwin'
729  else:
730    assert sys.platform.startswith('linux')
731    platform = 'linux'
732  rt_lib_src_dir = os.path.join(COMPILER_RT_BUILD_DIR, 'lib', platform)
733  if sys.platform == 'win32':
734    # TODO(thakis): This too is due to compiler-rt being part of the checkout
735    # on Windows, see TODO above COMPILER_RT_DIR.
736    rt_lib_src_dir = os.path.join(COMPILER_RT_BUILD_DIR, 'lib', 'clang',
737                                  VERSION, 'lib', platform)
738  rt_lib_dst_dir = os.path.join(LLVM_BUILD_DIR, 'lib', 'clang', VERSION, 'lib',
739                                platform)
740  # Blacklists:
741  CopyDirectoryContents(os.path.join(rt_lib_src_dir, '..', '..', 'share'),
742                        os.path.join(rt_lib_dst_dir, '..', '..', 'share'))
743  # Headers:
744  if sys.platform != 'win32':
745    CopyDirectoryContents(
746        os.path.join(COMPILER_RT_BUILD_DIR, 'include/sanitizer'),
747        os.path.join(LLVM_BUILD_DIR, 'lib/clang', VERSION, 'include/sanitizer'))
748  # Static and dynamic libraries:
749  CopyDirectoryContents(rt_lib_src_dir, rt_lib_dst_dir)
750  if sys.platform == 'darwin':
751    for dylib in glob.glob(os.path.join(rt_lib_dst_dir, '*.dylib')):
752      # Fix LC_ID_DYLIB for the ASan dynamic libraries to be relative to
753      # @executable_path.
754      # TODO(glider): this is transitional. We'll need to fix the dylib
755      # name either in our build system, or in Clang. See also
756      # http://crbug.com/344836.
757      subprocess.call(['install_name_tool', '-id',
758                       '@executable_path/' + os.path.basename(dylib), dylib])
759
760  if args.with_android:
761    make_toolchain = os.path.join(
762        ANDROID_NDK_DIR, 'build', 'tools', 'make_standalone_toolchain.py')
763    for target_arch in ['aarch64', 'arm', 'i686']:
764      # Make standalone Android toolchain for target_arch.
765      toolchain_dir = os.path.join(
766          LLVM_BUILD_DIR, 'android-toolchain-' + target_arch)
767      api_level = '21' if target_arch == 'aarch64' else '19'
768      RunCommand([
769          make_toolchain,
770          '--api=' + api_level,
771          '--force',
772          '--install-dir=%s' % toolchain_dir,
773          '--stl=libc++',
774          '--arch=' + {
775              'aarch64': 'arm64',
776              'arm': 'arm',
777              'i686': 'x86',
778          }[target_arch]])
779
780      # Build compiler-rt runtimes needed for Android in a separate build tree.
781      build_dir = os.path.join(LLVM_BUILD_DIR, 'android-' + target_arch)
782      if not os.path.exists(build_dir):
783        os.mkdir(os.path.join(build_dir))
784      os.chdir(build_dir)
785      target_triple = target_arch
786      abi_libs = 'c++abi'
787      if target_arch == 'arm':
788        target_triple = 'armv7'
789        abi_libs += ';unwind'
790      target_triple += '-linux-android' + api_level
791      cflags = ['--target=%s' % target_triple,
792                '--sysroot=%s/sysroot' % toolchain_dir,
793                '-B%s' % toolchain_dir]
794      android_args = base_cmake_args + [
795        '-DLLVM_ENABLE_THREADS=OFF',
796        '-DCMAKE_C_COMPILER=' + os.path.join(LLVM_BUILD_DIR, 'bin/clang'),
797        '-DCMAKE_CXX_COMPILER=' + os.path.join(LLVM_BUILD_DIR, 'bin/clang++'),
798        '-DLLVM_CONFIG_PATH=' + os.path.join(LLVM_BUILD_DIR, 'bin/llvm-config'),
799        '-DCMAKE_C_FLAGS=' + ' '.join(cflags),
800        '-DCMAKE_CXX_FLAGS=' + ' '.join(cflags),
801        '-DSANITIZER_CXX_ABI=none',
802        '-DSANITIZER_CXX_ABI_LIBRARY=' + abi_libs,
803        '-DCMAKE_SHARED_LINKER_FLAGS=-Wl,-u__cxa_demangle',
804        '-DANDROID=1']
805      RmCmakeCache('.')
806      RunCommand(['cmake'] + android_args + [COMPILER_RT_DIR])
807      RunCommand(['ninja', 'asan', 'ubsan', 'profile'])
808
809      # And copy them into the main build tree.
810      asan_lib_path_format = 'lib/linux/libclang_rt.asan-{0}-android.so'
811      libs_want = [
812          asan_lib_path_format,
813          'lib/linux/libclang_rt.ubsan_standalone-{0}-android.so',
814          'lib/linux/libclang_rt.profile-{0}-android.a',
815      ]
816      for arch in ['aarch64', 'arm']:
817        for p in libs_want:
818          lib_path = os.path.join(build_dir, p.format(arch))
819          if os.path.exists(lib_path):
820            shutil.copy(lib_path, rt_lib_dst_dir)
821
822      # We also use ASan i686 build for fuzzing.
823      lib_path = os.path.join(build_dir, asan_lib_path_format.format('i686'))
824      if os.path.exists(lib_path):
825        shutil.copy(lib_path, rt_lib_dst_dir)
826
827  # Run tests.
828  if args.run_tests or use_head_revision:
829    os.chdir(LLVM_BUILD_DIR)
830    RunCommand(['ninja', 'cr-check-all'], msvc_arch='x64')
831  if args.run_tests:
832    if sys.platform == 'win32':
833      CopyDiaDllTo(os.path.join(LLVM_BUILD_DIR, 'bin'))
834    os.chdir(LLVM_BUILD_DIR)
835    RunCommand(['ninja', 'check-all'], msvc_arch='x64')
836
837  WriteStampFile(PACKAGE_VERSION)
838  print 'Clang update was successful.'
839  return 0
840
841
842def main():
843  parser = argparse.ArgumentParser(description='Build Clang.')
844  parser.add_argument('--bootstrap', action='store_true',
845                      help='first build clang with CC, then with itself.')
846  # TODO(phajdan.jr): remove --if-needed after fixing callers. It's no-op.
847  parser.add_argument('--if-needed', action='store_true',
848                      help="run only if the script thinks clang is needed")
849  parser.add_argument('--force-local-build', action='store_true',
850                      help="don't try to download prebuild binaries")
851  parser.add_argument('--gcc-toolchain', help='set the version for which gcc '
852                      'version be used for building; --gcc-toolchain=/opt/foo '
853                      'picks /opt/foo/bin/gcc')
854  parser.add_argument('--lto-lld', action='store_true',
855                      help='build lld with LTO')
856  parser.add_argument('--llvm-force-head-revision', action='store_true',
857                      help=('use the revision in the repo when printing '
858                            'the revision'))
859  parser.add_argument('--print-revision', action='store_true',
860                      help='print current clang revision and exit.')
861  parser.add_argument('--print-clang-version', action='store_true',
862                      help='print current clang version (e.g. x.y.z) and exit.')
863  parser.add_argument('--run-tests', action='store_true',
864                      help='run tests after building; only for local builds')
865  parser.add_argument('--skip-build', action='store_true',
866                      help='do not build anything')
867  parser.add_argument('--skip-checkout', action='store_true',
868                      help='do not create or update any checkouts')
869  parser.add_argument('--extra-tools', nargs='*', default=[],
870                      help='select additional chrome tools to build')
871  parser.add_argument('--use-system-cmake', action='store_true',
872                      help='use the cmake from PATH instead of downloading '
873                      'and using prebuilt cmake binaries')
874  parser.add_argument('--verify-version',
875                      help='verify that clang has the passed-in version')
876  parser.add_argument('--without-android', action='store_false',
877                      help='don\'t build Android ASan runtime (linux only)',
878                      dest='with_android',
879                      default=sys.platform.startswith('linux'))
880  args = parser.parse_args()
881
882  if args.lto_lld and not args.bootstrap:
883    print '--lto-lld requires --bootstrap'
884    return 1
885  if args.lto_lld and not sys.platform.startswith('linux'):
886    print '--lto-lld is only effective on Linux. Ignoring the option.'
887    args.lto_lld = False
888
889  # Get svn if we're going to use it to check the revision or do a local build.
890  if (use_head_revision or args.llvm_force_head_revision or
891      args.force_local_build):
892    AddSvnToPathOnWin()
893
894  if use_head_revision:
895    # TODO(hans): Trunk version was updated; remove after the next roll.
896    global VERSION
897    VERSION = '8.0.0'
898
899  if args.verify_version and args.verify_version != VERSION:
900    print 'VERSION is %s but --verify-version argument was %s, exiting.' % (
901        VERSION, args.verify_version)
902    print 'clang_version in build/toolchain/toolchain.gni is likely outdated.'
903    return 1
904
905  global CLANG_REVISION, PACKAGE_VERSION
906  if args.print_revision:
907    if use_head_revision or args.llvm_force_head_revision:
908      print GetSvnRevision(LLVM_DIR)
909    else:
910      print PACKAGE_VERSION
911    return 0
912
913  if args.print_clang_version:
914    sys.stdout.write(VERSION)
915    return 0
916
917  # Don't buffer stdout, so that print statements are immediately flushed.
918  # Do this only after --print-revision has been handled, else we'll get
919  # an error message when this script is run from gn for some reason.
920  sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 0)
921
922  if use_head_revision:
923    # Use a real revision number rather than HEAD to make sure that the stamp
924    # file logic works.
925    CLANG_REVISION = GetSvnRevision(LLVM_REPO_URL)
926    PACKAGE_VERSION = CLANG_REVISION + '-0'
927
928    args.force_local_build = True
929    if 'OS=android' not in os.environ.get('GYP_DEFINES', ''):
930      # Only build the Android ASan rt on ToT bots when targetting Android.
931      args.with_android = False
932
933  return UpdateClang(args)
934
935
936if __name__ == '__main__':
937  sys.exit(main())
938