• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/env python
2# Copyright (C) 2017 The Android Open Source Project
3#
4# Licensed under the Apache License, Version 2.0 (the "License");
5# you may not use this file except in compliance with the License.
6# You may obtain a copy of the License at
7#
8#      http://www.apache.org/licenses/LICENSE-2.0
9#
10# Unless required by applicable law or agreed to in writing, software
11# distributed under the License is distributed on an "AS IS" BASIS,
12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13# See the License for the specific language governing permissions and
14# limitations under the License.
15
16import argparse
17import hashlib
18import logging
19import os
20import shutil
21import subprocess
22import sys
23import urllib
24import zipfile
25
26from collections import namedtuple
27
28# When adding a new git dependency here please also add a corresponding entry in
29# .travis.yml under the "cache:" section.
30
31# The format for the deps below is the following:
32# (target_folder, source_url, sha1, target_platform)
33# |source_url| can be either a git repo or a http url.
34# If a git repo, |sha1| is the committish that will be checked out.
35# If a http url, |sha1| is the shasum of the original file.
36# If the url is a .zip or .tgz file it will be automatically deflated under
37# |target_folder|, taking care of stripping the root folder if it's a single
38# root (to avoid ending up with buildtools/protobuf/protobuf-1.2.3/... and have
39# instead just buildtools/protobuf).
40# |target_platform| is either 'darwin', 'linux2' or 'all' and applies the dep
41# only on the given platform (ask python why linux2 and not just linux).
42
43# Dependencies required to build code on the host or when targeting desktop OS.
44BUILD_DEPS_HOST = [
45  # GN
46  ('buildtools/mac/gn',
47   'https://storage.googleapis.com/chromium-gn/c2c934d4dda1f470a6511b1015dda9a9fb1ce50b',
48   'c2c934d4dda1f470a6511b1015dda9a9fb1ce50b',
49   'darwin'
50  ),
51  ('buildtools/linux64/gn',
52   'https://storage.googleapis.com/chromium-gn/b53fa13e950948c6f9a062189b76b34a9610281f',
53   'b53fa13e950948c6f9a062189b76b34a9610281f',
54   'linux2'
55  ),
56
57  # clang-format
58  ('buildtools/mac/clang-format',
59   'https://storage.googleapis.com/chromium-clang-format/0679b295e2ce2fce7919d1e8d003e497475f24a3',
60   '0679b295e2ce2fce7919d1e8d003e497475f24a3',
61   'darwin'
62  ),
63  ('buildtools/linux64/clang-format',
64   'https://storage.googleapis.com/chromium-clang-format/5349d1954e17f6ccafb6e6663b0f13cdb2bb33c8',
65   '5349d1954e17f6ccafb6e6663b0f13cdb2bb33c8',
66   'linux2'
67  ),
68  # Keep the SHA1 in sync with |clang_format_rev| in chromium //buildtools/DEPS.
69  ('buildtools/clang_format/script',
70   'https://chromium.googlesource.com/chromium/llvm-project/cfe/tools/clang-format.git',
71   '0653eee0c81ea04715c635dd0885e8096ff6ba6d',
72   'all'
73  ),
74
75  # Ninja
76  ('buildtools/mac/ninja',
77   'https://storage.googleapis.com/fuchsia-build/fuchsia/ninja/mac/a1db595e824c50cf565fbf0af2437fd91b7babf4',
78   'a1db595e824c50cf565fbf0af2437fd91b7babf4',
79   'darwin'
80  ),
81  ('buildtools/linux64/ninja',
82   'https://storage.googleapis.com/fuchsia-build/fuchsia/ninja/linux64/d35b36c84a09f7e38b25947cafada10e8bf835bc',
83   'd35b36c84a09f7e38b25947cafada10e8bf835bc',
84   'linux2'
85  ),
86
87  # Keep in sync with Android's //external/googletest/README.version.
88  ('buildtools/googletest.zip',
89   'https://github.com/google/googletest/archive/ff07a5de0e81580547f1685e101194ed1a4fcd56.zip',
90   'c7edec7d7e6db1fc37a20710de9c4d89e3a3893b',
91   'all'
92  ),
93
94  # Keep in sync with Android's //external/protobuf/README.version.
95  ('buildtools/protobuf.zip',
96   'https://github.com/google/protobuf/releases/download/v3.0.0-beta-3/protobuf-cpp-3.0.0-beta-3.zip',
97   '3caec60aa9d8eefc8c3c3201b6b8ca19935edb89',
98   'all'
99  ),
100
101  # libc++, libc++abi and libunwind for Linux where we need to rebuild the C++
102  # lib from sources. Keep the SHA1s in sync with Chrome's src/buildtools/DEPS.
103  ('buildtools/libcxx',
104   'https://chromium.googlesource.com/chromium/llvm-project/libcxx.git',
105   '3a07dd740be63878167a0ea19fe81869954badd7',
106   'all'
107  ),
108  ('buildtools/libcxxabi',
109   'https://chromium.googlesource.com/chromium/llvm-project/libcxxabi.git',
110   '4072e8fd76febee37f60aeda76d6d9f5e3791daa',
111   'all'
112  ),
113  ('buildtools/libunwind',
114   'https://chromium.googlesource.com/external/llvm.org/libunwind.git',
115   '41f982e5887185b904a456e20dfcd58e6be6cc19',
116   'all'
117  ),
118
119  # Keep the revision in sync with Chrome's CLANG_REVISION in
120  # tools/clang/scripts/update.py.
121  ('buildtools/clang.tgz',
122   'https://commondatastorage.googleapis.com/chromium-browser-clang/Linux_x64/clang-321529-1.tgz',
123   '7ca04034ac7e4a956b0084a8bc604bacab94af26',
124   'linux2'
125  ),
126
127  # Benchmarking tool.
128  ('buildtools/benchmark.zip',
129   'https://github.com/google/benchmark/archive/v1.3.0.zip',
130   'f387e0df37d54bfd5be239e8d0d3ea2e2c3e34f4',
131   'all'
132  ),
133
134  # Libbacktrace, for stacktraces in Linux/Android debug builds.
135  ('buildtools/libbacktrace.zip',
136   'https://github.com/ianlancetaylor/libbacktrace/archive/177940370e4a6b2509e92a0aaa9749184e64af43.zip',
137   'b723fe9d671d1ab54df1297f6afbf2893a41c3ea',
138   'all'
139  ),
140]
141
142# Dependencies required to build Android code.
143# URLs and SHA1s taken from:
144# - https://dl.google.com/android/repository/repository-11.xml
145# - https://dl.google.com/android/repository/sys-img/android/sys-img.xml
146BUILD_DEPS_ANDROID = [
147  # Android NDK
148  ('buildtools/ndk.zip',
149   'https://dl.google.com/android/repository/android-ndk-r15c-darwin-x86_64.zip',
150   'ea4b5d76475db84745aa8828000d009625fc1f98',
151   'darwin'
152  ),
153  ('buildtools/ndk.zip',
154   'https://dl.google.com/android/repository/android-ndk-r15c-linux-x86_64.zip',
155   '0bf02d4e8b85fd770fd7b9b2cdec57f9441f27a2',
156   'linux2'
157  ),
158]
159
160# Dependencies required to run Android tests.
161TEST_DEPS_ANDROID = [
162  # Android emulator images.
163  ('buildtools/aosp-arm.zip',
164   'https://storage.googleapis.com/perfetto/aosp-02022018-arm.zip',
165   'a480d5e7d3ca888b0a58fe15ce76b1791537429a',
166   'all'
167  ),
168
169  # platform-tools.zip contains adb binaries.
170  ('buildtools/android_sdk/platform-tools.zip',
171   'https://dl.google.com/android/repository/platform-tools_r26.0.0-darwin.zip',
172   'e75b6137dc444f777eb02f44a6d9819b3aabff82',
173   'darwin'
174  ),
175  ('buildtools/android_sdk/platform-tools.zip',
176   'https://dl.google.com/android/repository/platform-tools_r26.0.0-linux.zip',
177   '00de8a6631405b617c10f68cd11ff2e1cd528e23',
178   'linux2'
179  ),
180
181  # Android emulator binaries.
182  ('buildtools/emulator',
183   'https://android.googlesource.com/platform/prebuilts/android-emulator.git',
184   '4b260028dc27bc92c39bee9129cb2ba839970956',
185   'all'
186  ),
187]
188
189ROOT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
190
191
192def ReadFile(path):
193  if not os.path.exists(path):
194    return None
195  with open(path) as f:
196      return f.read().strip()
197
198
199def MkdirRecursive(path):
200  # Works with both relative and absolute paths
201  cwd = '/' if path.startswith('/') else ROOT_DIR
202  for part in path.split('/'):
203    cwd = os.path.join(cwd, part)
204    if not os.path.exists(cwd):
205      os.makedirs(cwd)
206    else:
207      assert(os.path.isdir(cwd))
208
209
210def HashLocalFile(path):
211  if not os.path.exists(path):
212    return None
213  with open(path, 'rb') as f:
214    return hashlib.sha1(f.read()).hexdigest()
215
216
217def ExtractZipfilePreservePermissions(zf, info, path):
218  zf.extract(info.filename, path=path)
219  target_path = os.path.join(path, info.filename)
220  min_acls = 0o755 if info.filename.endswith('/') else 0o644
221  os.chmod(target_path, (info.external_attr >> 16L) | min_acls)
222
223
224def IsGitRepoCheckoutOutAtRevision(path, revision):
225  return ReadFile(os.path.join(path, '.git', 'HEAD')) == revision
226
227
228def CheckoutGitRepo(path, git_url, revision):
229  if IsGitRepoCheckoutOutAtRevision(path, revision):
230    return
231  if os.path.exists(path):
232    shutil.rmtree(path)
233  MkdirRecursive(path)
234  logging.info('Fetching %s @ %s into %s', git_url, revision, path)
235  subprocess.check_call(['git', 'init', path], cwd=path)
236  subprocess.check_call(
237    ['git', 'fetch', '--depth', '1', git_url, revision], cwd=path)
238  subprocess.check_call(['git', 'checkout', revision, '--quiet'], cwd=path)
239  assert(IsGitRepoCheckoutOutAtRevision(path, revision))
240
241
242def Main():
243  parser = argparse.ArgumentParser()
244  parser.add_argument('--no-android', action='store_true')
245  args = parser.parse_args()
246  deps = BUILD_DEPS_HOST
247  if not args.no_android:
248    deps += BUILD_DEPS_ANDROID + TEST_DEPS_ANDROID
249  for rel_path, url, expected_sha1, platform in deps:
250    if (platform != 'all' and platform != sys.platform):
251      continue
252    local_path = os.path.join(ROOT_DIR, rel_path)
253    if url.endswith('.git'):
254      CheckoutGitRepo(local_path, url, expected_sha1)
255      continue
256    is_zip = local_path.endswith('.zip') or local_path.endswith('.tgz')
257    zip_target_dir = local_path[:-4] if is_zip else None
258    zip_dir_stamp = os.path.join(zip_target_dir, '.stamp') if is_zip else None
259
260    if ((not is_zip and HashLocalFile(local_path) == expected_sha1) or
261        (is_zip and ReadFile(zip_dir_stamp) == expected_sha1)):
262      continue
263    MkdirRecursive(os.path.dirname(rel_path))
264    if HashLocalFile(local_path) != expected_sha1:
265      download_path = local_path + '.tmp'
266      logging.info('Downloading %s from %s', local_path, url)
267      urllib.urlretrieve(url, download_path)
268      os.chmod(download_path, 0o755)
269      if (HashLocalFile(download_path) != expected_sha1):
270        os.remove(download_path)
271        logging.fatal('SHA1 mismatch for %s', download_path)
272        return 1
273      os.rename(download_path, local_path)
274    assert(HashLocalFile(local_path) == expected_sha1)
275
276    if is_zip:
277      logging.info('Extracting %s into %s' % (local_path, zip_target_dir))
278      assert(os.path.commonprefix((ROOT_DIR, zip_target_dir)) == ROOT_DIR)
279      if os.path.exists(zip_target_dir):
280        logging.info('Deleting stale dir %s' % zip_target_dir)
281        shutil.rmtree(zip_target_dir)
282
283      # Decompress the archive.
284      if local_path.endswith('.tgz'):
285        MkdirRecursive(zip_target_dir)
286        subprocess.check_call(['tar', '-zxf', local_path], cwd=zip_target_dir)
287      elif local_path.endswith('.zip'):
288        with zipfile.ZipFile(local_path, 'r') as zf:
289          for info in zf.infolist():
290            ExtractZipfilePreservePermissions(zf, info, zip_target_dir)
291
292      # If the zip contains one root folder, rebase one level up moving all
293      # its sub files and folders inside |target_dir|.
294      subdir = os.listdir(zip_target_dir)
295      if len(subdir) == 1:
296        subdir = os.path.join(zip_target_dir, subdir[0])
297        if os.path.isdir(subdir):
298          for subf in os.listdir(subdir):
299            shutil.move(os.path.join(subdir,subf), zip_target_dir)
300          os.rmdir(subdir)
301
302      # Create stamp and remove the archive.
303      with open(zip_dir_stamp, 'w') as stamp_file:
304        stamp_file.write(expected_sha1)
305      os.remove(local_path)
306
307
308if __name__ == '__main__':
309  logging.basicConfig(level=logging.INFO)
310  sys.exit(Main())
311