• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/env python3
2# Copyright (C) 2017 The Android Open Source Project
3#
4# Licensed under the Apache License, Version 2.0 (the "License");
5# you may not use this file except in compliance with the License.
6# You may obtain a copy of the License at
7#
8#      http://www.apache.org/licenses/LICENSE-2.0
9#
10# Unless required by applicable law or agreed to in writing, software
11# distributed under the License is distributed on an "AS IS" BASIS,
12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13# See the License for the specific language governing permissions and
14# limitations under the License.
15
16import argparse
17import hashlib
18import logging
19import os
20import shutil
21import subprocess
22import stat
23import sys
24import tempfile
25import zipfile
26
27from collections import namedtuple
28from platform import system, machine
29
30# The format for the deps below is the following:
31# (target_folder, source_url, sha1, target_os, target_arch)
32# |source_url| can be either a git repo or a http url.
33# If a git repo, |checksum| is the SHA1 committish that will be checked out.
34# If a http url, |checksum| is the SHA256 of the downloaded file.
35# If the url is a .zip or .tgz file it will be automatically deflated under
36# |target_folder|, taking care of stripping the root folder if it's a single
37# root (to avoid ending up with buildtools/protobuf/protobuf-1.2.3/... and have
38# instead just buildtools/protobuf).
39# |target_os| is either 'darwin', 'linux', 'windows' or 'all'
40# |target_arch| is either 'x64', 'arm64' or 'all'
41# in both cases the dep only applies on matching platforms
42# |target_arch| can be 'all' when 'target_os' is not 'all' for example in the
43# case of MacOS universal binaries.
44Dependency = namedtuple(
45    'Dependency',
46    ['target_folder', 'source_url', 'checksum', 'target_os', 'target_arch'])
47
48# This is to remove old directories when build tools get {re,}moved. This is to
49# avoid accidentally referring to stale dir in custom user scripts.
50CLEANUP_OLD_DIRS = [
51    'buildtools/nodejs',  # Moved to buildtools/{mac,linux64}/nodejs
52    'buildtools/emsdk',  # Moved to buildtools/{mac,linux64}/emsdk
53    'buildtools/test_data',  # Moved to test/data by r.android.com/1539381 .
54    'buildtools/d8',  # Removed by r.android.com/1424334 .
55]
56
57# Dependencies required to build code on the host or when targeting desktop OS.
58BUILD_DEPS_TOOLCHAIN_HOST = [
59    # GN. From https://chrome-infra-packages.appspot.com/dl/gn/gn/.
60    # git_revision:0725d7827575b239594fbc8fd5192873a1d62f44 .
61    Dependency(
62        'buildtools/mac/gn',
63        'https://storage.googleapis.com/perfetto/gn-mac-1968-0725d782',
64        '9ced623a664560bba38bbadb9b91158ca4186358c847e17ab7d982b351373c2e',
65        'darwin', 'x64'),
66    Dependency(
67        'buildtools/mac/gn',
68        'https://storage.googleapis.com/perfetto/gn-mac-arm64-1968-0725d782',
69        'd22336b5210b4dad5e36e8c28ce81187f491822cf4d8fd0a257b30d6bee3fd3f',
70        'darwin', 'arm64'),
71    Dependency(
72        'buildtools/linux64/gn',
73        'https://storage.googleapis.com/perfetto/gn-linux64-1968-0725d782',
74        'f706aaa0676e3e22f5fc9ca482295d7caee8535d1869f99efa2358177b64f5cd',
75        'linux', 'x64'),
76    Dependency(
77        'buildtools/win/gn.exe',
78        'https://storage.googleapis.com/perfetto/gn-win-1968-0725d782',
79        '001f777f023c7a6959c778fb3a6b6cfc63f6baef953410ecdeaec350fb12285b',
80        'windows', 'x64'),
81
82    # clang-format
83    # From https://chromium.googlesource.com/chromium/src/buildtools/+/refs/heads/master/mac/clang-format.sha1
84    Dependency(
85        'buildtools/mac/clang-format',
86        'https://storage.googleapis.com/chromium-clang-format/62bde1baa7196ad9df969fc1f06b66360b1a927b',
87        '6df686a937443cbe6efc013467a7ba5f98d3f187eb7765bb7abc6ce47626cf66',
88        'darwin', 'all'),
89    # From https://chromium.googlesource.com/chromium/src/buildtools/+/refs/heads/master/linux64/clang-format.sha1
90    Dependency(
91        'buildtools/linux64/clang-format',
92        'https://storage.googleapis.com/chromium-clang-format/1baf0089e895c989a311b6a38ed94d0e8be4c0a7',
93        'd02a97a87e8c28898033aaf5986967b24dc47ebd5b376e1cd93e5009f22cd75e',
94        'linux', 'x64'),
95    # From https://chromium.googlesource.com/chromium/src/buildtools/+/refs/heads/master/win/clang-format.exe.sha1
96    Dependency(
97        'buildtools/win/clang-format.exe',
98        'https://storage.googleapis.com/chromium-clang-format/d4afd4eba27022f5f6d518133aebde57281677c9',
99        '2ba1b4d3ade90ea80316890b598ab5fc16777572be26afec6ce23117da121b80',
100        'windows', 'x64'),
101
102    # Keep the SHA1 in sync with |clang_format_rev| in chromium //buildtools/DEPS.
103    Dependency(
104        'buildtools/clang_format/script',
105        'https://chromium.googlesource.com/chromium/llvm-project/cfe/tools/clang-format.git',
106        '96636aa0e9f047f17447f2d45a094d0b59ed7917', 'all', 'all'),
107
108    # Ninja
109    Dependency(
110        'buildtools/mac/ninja',
111        'https://storage.googleapis.com/perfetto/ninja-mac-x64_and_arm64-182',
112        '36e8b7aaa06911e1334feb664dd731a1cd69a15eb916a231a3d10ff65fca2c73',
113        'darwin', 'all'),
114    Dependency(
115        'buildtools/linux64/ninja',
116        'https://storage.googleapis.com/perfetto/ninja-linux64-182',
117        '54ac6a01362190aaabf4cf276f9c8982cdf11b225438940fdde3339be0f2ecdc',
118        'linux', 'x64'),
119    Dependency(
120        'buildtools/win/ninja.exe',
121        'https://storage.googleapis.com/perfetto/ninja-win-182',
122        '09ced0fcd1a4dec7d1b798a2cf9ce5d20e5d2fbc2337343827f192ce47d0f491',
123        'windows', 'x64'),
124
125    # Keep the revision in sync with Chrome's PACKAGE_VERSION in
126    # tools/clang/scripts/update.py.
127    Dependency(
128        'buildtools/linux64/clang.tgz',
129        'https://commondatastorage.googleapis.com/chromium-browser-clang/Linux_x64/clang-llvmorg-14-init-3191-g0e03450a-1.tgz',
130        'dd7479d43ce61401e057a5dee8b7e32bc2bd0d0e15d4f46c6858daf9170c9978',
131        'linux', 'x64'),
132    Dependency(
133        'buildtools/win/clang.tgz',
134        'https://commondatastorage.googleapis.com/chromium-browser-clang/Win/clang-llvmorg-14-init-3191-g0e03450a-1.tgz',
135        '4292d191742e7120200c63224f02e1f2f2a7be8b57c0f18edf6ca0955bdd43df',
136        'windows', 'x64'),
137]
138
139BUILD_DEPS_HOST = [
140    # Keep in sync with Android's //external/googletest/README.version.
141    Dependency(
142        'buildtools/googletest',
143        'https://android.googlesource.com/platform/external/googletest.git',
144        '3f05f651ae3621db58468153e32016bc1397800b', 'all', 'all'),
145
146    # Keep in sync with Chromium's //third_party/protobuf.
147    Dependency(
148        'buildtools/protobuf',
149        'https://chromium.googlesource.com/external/github.com/google/protobuf.git',
150        '6a59a2ad1f61d9696092f79b6d74368b4d7970a3',  # refs/tags/v3.9.0
151        'all',
152        'all'),
153
154    # libc++, libc++abi and libunwind for Linux where we need to rebuild the C++
155    # lib from sources. Keep the SHA1s in sync with Chrome's src/buildtools/DEPS.
156    Dependency(
157        'buildtools/libcxx',
158        'https://chromium.googlesource.com/external/github.com/llvm/llvm-project/libcxx.git',
159        'd9040c75cfea5928c804ab7c235fed06a63f743a', 'all', 'all'),
160    Dependency(
161        'buildtools/libcxxabi',
162        'https://chromium.googlesource.com/external/github.com/llvm/llvm-project/libcxxabi.git',
163        '196ba1aaa8ac285d94f4ea8d9836390a45360533', 'all', 'all'),
164    Dependency(
165        'buildtools/libunwind',
166        'https://chromium.googlesource.com/external/github.com/llvm/llvm-project/libunwind.git',
167        'd999d54f4bca789543a2eb6c995af2d9b5a1f3ed', 'all', 'all'),
168
169    # Keep in sync with chromium DEPS.
170    Dependency(
171        'buildtools/libfuzzer',
172        'https://chromium.googlesource.com/chromium/llvm-project/compiler-rt/lib/fuzzer.git',
173        'debe7d2d1982e540fbd6bd78604bf001753f9e74', 'linux', 'all'),
174
175    # Benchmarking tool.
176    Dependency(
177        'buildtools/benchmark',
178        'https://chromium.googlesource.com/external/github.com/google/benchmark.git',
179        '090faecb454fbd6e6e17a75ef8146acb037118d4', 'all', 'all'),
180
181    # Libbacktrace, for stacktraces in Linux/Android debug builds.
182    # From https://github.com/ianlancetaylor/libbacktrace/archive/177940370e4a6b2509e92a0aaa9749184e64af43.zip
183    Dependency(
184        'buildtools/libbacktrace.zip',
185        'https://storage.googleapis.com/perfetto/libbacktrace-177940370e4a6b2509e92a0aaa9749184e64af43.zip',
186        '21ac9a4209f7aeef766c482be53a7fa365063c031c7077e2070b491202983b31',
187        'all', 'all'),
188
189    # Sqlite for the trace processing library.
190    # This is the amalgamated source whose compiled output is meant to be faster.
191    # We still pull the full source for the extensions (which are not available
192    # in the amalgamation).
193    # If updating the version, also update bazel/deps.bzl.
194    Dependency(
195        'buildtools/sqlite.zip',
196        'https://storage.googleapis.com/perfetto/sqlite-amalgamation-3350400.zip',
197        'f3bf0df69f5de0675196f4644e05d07dbc698d674dc563a12eff17d5b215cdf5',
198        'all', 'all'),
199    Dependency(
200        'buildtools/sqlite_src',
201        'https://chromium.googlesource.com/external/github.com/sqlite/sqlite.git',
202        'ee3686eb50c0e3dbb087c9a0976f7e37e1b014ae',  # refs/tags/version-3.32.3.
203        'all',
204        'all'),
205
206    # JsonCpp for legacy json import. Used only by the trace processor in
207    # standalone builds.
208    # If updating the version, also update bazel/deps.bzl.
209    Dependency(
210        'buildtools/jsoncpp',
211        'https://chromium.googlesource.com/external/github.com/open-source-parsers/jsoncpp.git',
212        '6aba23f4a8628d599a9ef7fa4811c4ff6e4070e2',  # refs/tags/1.9.3.
213        'all',
214        'all'),
215
216    # Archive with only the demangling sources from llvm-project.
217    # See tools/repackage_llvm_demangler.sh on how to update this.
218    # File suffix is the git reference to the commit at which we rearchived the
219    # sources, as hosted on https://llvm.googlesource.com/llvm-project.
220    # If updating the version, also update bazel/deps.bzl.
221    Dependency(
222        'buildtools/llvm-project.tgz',
223        'https://storage.googleapis.com/perfetto/llvm-project-3b4c59c156919902c785ce3cbae0eee2ee53064d.tgz',
224        'f4a52e7f36edd7cacc844d5ae0e5f60b6f57c5afc40683e99f295886c9ce8ff4',
225        'all', 'all'),
226
227    # These dependencies are for libunwindstack, which is used by src/profiling.
228    Dependency('buildtools/android-core',
229               'https://android.googlesource.com/platform/system/core.git',
230               '9e6cef7f07d8c11b3ea820938aeb7ff2e9dbaa52', 'all', 'all'),
231    Dependency(
232        'buildtools/android-unwinding',
233        'https://android.googlesource.com/platform/system/unwinding.git',
234        '215644709bd7bbda4c05a9db4d55e96be61bcf64', 'all', 'all'),
235    Dependency('buildtools/android-logging',
236               'https://android.googlesource.com/platform/system/logging.git',
237               '7b36b566c9113fc703d68f76e8f40c0c2432481c', 'all', 'all'),
238    Dependency('buildtools/android-libbase',
239               'https://android.googlesource.com/platform/system/libbase.git',
240               '78f1c2f83e625bdf66d55b48bdb3a301c20d2fb3', 'all', 'all'),
241    Dependency(
242        'buildtools/android-libprocinfo',
243        'https://android.googlesource.com/platform/system/libprocinfo.git',
244        'fd214c13ededecae97a3b15b5fccc8925a749a84', 'all', 'all'),
245    Dependency('buildtools/lzma',
246               'https://android.googlesource.com/platform/external/lzma.git',
247               '7851dce6f4ca17f5caa1c93a4e0a45686b1d56c3', 'all', 'all'),
248    Dependency('buildtools/bionic',
249               'https://android.googlesource.com/platform/bionic.git',
250               '332065d57e734b65f56474d136d22d767e36cbcd', 'all', 'all'),
251
252    # Zlib used both in the tracing binaries, as well as the trace processor and
253    # assorted tools.
254    # If updating the version, also update bazel/deps.bzl.
255    Dependency('buildtools/zlib',
256               'https://android.googlesource.com/platform/external/zlib.git',
257               '5c85a2da4c13eda07f69d81a1579a5afddd35f59', 'all', 'all'),
258
259    # Linenoise, used only by trace_processor in standalone builds.
260    # If updating the version, also update bazel/deps.bzl.
261    Dependency('buildtools/linenoise',
262               'https://fuchsia.googlesource.com/third_party/linenoise.git',
263               'c894b9e59f02203dbe4e2be657572cf88c4230c3', 'all', 'all'),
264
265    # Bloaty, used to investigate binary size
266    Dependency(
267        'buildtools/bloaty.zip',
268        'https://storage.googleapis.com/perfetto/bloaty-1.1-b3b829de35babc2fe831b9488ad2e50bca939412-mac.zip',
269        '2d301bd72a20e3f42888c9274ceb4dca76c103608053572322412c2c65ab8cb8',
270        'darwin', 'all'),
271]
272
273# Dependencies required to build Android code.
274# URLs and SHA1s taken from:
275# - https://dl.google.com/android/repository/repository-11.xml
276# - https://dl.google.com/android/repository/sys-img/android/sys-img.xml
277BUILD_DEPS_ANDROID = [
278    # Android NDK
279    Dependency(
280        'buildtools/ndk.zip',
281        'https://dl.google.com/android/repository/android-ndk-r21e-darwin-x86_64.zip',
282        '437278103a3db12632c05b1be5c41bbb8522791a67e415cc54411a65366f499d',
283        'darwin', 'all'),
284    Dependency(
285        'buildtools/ndk.zip',
286        'https://dl.google.com/android/repository/android-ndk-r21e-linux-x86_64.zip',
287        'ad7ce5467e18d40050dc51b8e7affc3e635c85bd8c59be62de32352328ed467e',
288        'linux', 'x64'),
289]
290
291# Dependencies required to run Android tests.
292TEST_DEPS_ANDROID = [
293    # Android emulator images.
294    Dependency(
295        'buildtools/aosp-arm.zip',
296        'https://storage.googleapis.com/perfetto/aosp-02022018-arm.zip',
297        'f5c7a3a22ad7aa0bd14ba467e8697e1e917d306699bd25622aa4419a413b9b67',
298        'all', 'all'),
299
300    # platform-tools.zip contains adb binaries.
301    Dependency(
302        'buildtools/android_sdk/platform-tools.zip',
303        'https://dl.google.com/android/repository/platform-tools_r26.0.0-darwin.zip',
304        '98d392cbd21ca20d643c7e1605760cc49075611e317c534096b5564053f4ac8e',
305        'darwin', 'all'),
306    Dependency(
307        'buildtools/android_sdk/platform-tools.zip',
308        'https://dl.google.com/android/repository/platform-tools_r26.0.0-linux.zip',
309        '90208207521d85abf0d46e3374aa4e04b7aff74e4f355c792ac334de7a77e50b',
310        'linux', 'x64'),
311
312    # Android emulator binaries.
313    Dependency(
314        'buildtools/emulator',
315        'https://android.googlesource.com/platform/prebuilts/android-emulator.git',
316        '4b260028dc27bc92c39bee9129cb2ba839970956', 'all', 'x64'),
317]
318
319# This variable is updated by tools/roll-catapult-trace-viewer.
320CATAPULT_SHA256 = 'b30108e05268ce6c65bb4126b65f6bfac165d17f5c1fd285046e7e6fd76c209f'
321
322TYPEFACES_SHA256 = 'f5f78f8f4395db65cdf5fdc1bf51da65f2161e6d61a305091d6ea54d3094a1f0'
323
324UI_DEPS = [
325    Dependency(
326        'buildtools/mac/nodejs.tgz',
327        'https://storage.googleapis.com/chromium-nodejs/14.15.4/17ba7216e09de1bffb9dc80b7ec617a1cee40330',
328        'b81a466347d2ae34b1370b6681ba173e9fb082338170a41624b37be7a2052b7e',
329        'darwin', 'all'),
330    Dependency(
331        'buildtools/linux64/nodejs.tgz',
332        'https://storage.googleapis.com/chromium-nodejs/14.15.4/b2e40ddbac04d05baafbb007f203c6663c9d4ca9',
333        '5aa88f1e2bf036950790277f3431634f64044ec78362f3e4f0dc8da28d61e9a4',
334        'linux', 'x64'),
335    Dependency(
336        'buildtools/mac/emsdk.tgz',
337        'https://storage.googleapis.com/perfetto/emscripten-2.0.12-mac.tgz',
338        'aa125f8c8ff8a386d43e18c8ea0c98c875cc19160a899403e8967a5478f96f31',
339        'darwin', 'all'),
340    Dependency(
341        'buildtools/linux64/emsdk.tgz',
342        'https://storage.googleapis.com/perfetto/emscripten-2.0.12-linux.tgz',
343        'bfff9fb0326363c12e19b542f27a5f12cedbfc310f30621dc497c9af51d2d2e3',
344        'linux', 'x64'),
345    Dependency(
346        'buildtools/catapult_trace_viewer.tgz',
347        'https://storage.googleapis.com/perfetto/catapult_trace_viewer-%s.tar.gz'
348        % CATAPULT_SHA256, CATAPULT_SHA256, 'all', 'all'),
349    Dependency(
350        'buildtools/typefaces.tgz',
351        'https://storage.googleapis.com/perfetto/typefaces-%s.tar.gz' %
352        TYPEFACES_SHA256, TYPEFACES_SHA256, 'all', 'all')
353]
354
355# Sysroots required to cross-compile Linux targets (linux-arm{,64}).
356# These are taken from Chromium's build/linux/sysroot_scripts/sysroots.json.
357BUILD_DEPS_LINUX_CROSS_SYSROOTS = [
358    Dependency(
359        'buildtools/debian_sid_arm-sysroot.tgz',
360        'https://commondatastorage.googleapis.com/chrome-linux-sysroot/toolchain/11d6f690ca49e8ba01a1d8c5346cedad2cf308fd/debian_sid_arm_sysroot.tar.xz',
361        'ff192fe073d140d836c9ca1e68f7200d558bb9aa6c5c8f4f76f794f82890f99a',
362        'linux', 'all'),
363    Dependency(
364        'buildtools/debian_sid_arm64-sysroot.tgz',
365        'https://commondatastorage.googleapis.com/chrome-linux-sysroot/toolchain/2befe8ce3e88be6080e4fb7e6d412278ea6a7625/debian_sid_arm64_sysroot.tar.xz',
366        'e4389eab2fe363f3fbdfa4d3ce9d94457d78fd2c0e62171a7534867623eadc90',
367        'linux', 'all'),
368]
369
370ALL_DEPS = (
371    BUILD_DEPS_HOST + BUILD_DEPS_ANDROID + BUILD_DEPS_LINUX_CROSS_SYSROOTS +
372    TEST_DEPS_ANDROID + UI_DEPS)
373
374ROOT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
375UI_DIR = os.path.join(ROOT_DIR, 'ui')
376TOOLS_DIR = os.path.join(ROOT_DIR, 'tools')
377NODE_MODULES_STATUS_FILE = os.path.join(UI_DIR, 'node_modules', '.last_install')
378TEST_DATA_SCRIPT = os.path.join(TOOLS_DIR, 'test_data')
379
380
381def DownloadURL(url, out_file):
382  subprocess.check_call(['curl', '-L', '-#', '-o', out_file, url])
383
384
385def GetArch():
386  arch = machine()
387  if arch == 'arm64':
388    return 'arm64'
389  else:
390    # Assume everything else is x64 matching previous behaviour.
391    return 'x64'
392
393
394def ReadFile(path):
395  if not os.path.exists(path):
396    return None
397  with open(path) as f:
398    return f.read().strip()
399
400
401def MkdirRecursive(path):
402  # Works with both relative and absolute paths
403  cwd = '/' if path.startswith('/') else ROOT_DIR
404  for part in path.split('/'):
405    cwd = os.path.join(cwd, part)
406    if not os.path.exists(cwd):
407      os.makedirs(cwd)
408    else:
409      assert (os.path.isdir(cwd))
410
411
412def HashLocalFile(path):
413  if not os.path.exists(path):
414    return None
415  with open(path, 'rb') as f:
416    return hashlib.sha256(f.read()).hexdigest()
417
418
419def ExtractZipfilePreservePermissions(zf, info, path):
420  zf.extract(info.filename, path=path)
421  target_path = os.path.join(path, info.filename)
422  min_acls = 0o755 if info.filename.endswith('/') else 0o644
423  os.chmod(target_path, (info.external_attr >> 16) | min_acls)
424
425
426def IsGitRepoCheckoutOutAtRevision(path, revision):
427  return ReadFile(os.path.join(path, '.git', 'HEAD')) == revision
428
429
430def RmtreeIfExists(path):
431  # Git creates read-only files on windows, which cause failures with rmtree.
432  # This seems the socially accepted way to deal with it.
433  # See https://bugs.python.org/issue19643 .
434  def del_read_only_for_windows(_action, name, _exc):
435    os.chmod(name, stat.S_IWRITE)
436    os.remove(name)
437
438  if not os.path.exists(path):
439    return
440  buildtools_path = os.path.abspath(os.path.join(ROOT_DIR, 'buildtools'))
441  test_path = os.path.abspath(os.path.join(ROOT_DIR, 'test', 'data'))
442  if (not os.path.abspath(path).startswith(buildtools_path) and
443      not os.path.abspath(path).startswith(test_path)):
444    # Safety check to prevent that some merge confilct ends up doing some
445    # rm -rf / or similar.
446    logging.fatal('Cannot remove %s: outside of buildtools and test/data', path)
447    sys.exit(1)
448  logging.info('Removing %s' % path)
449  shutil.rmtree(path, onerror=del_read_only_for_windows)
450
451
452def CheckoutGitRepo(path, git_url, revision, check_only):
453  if IsGitRepoCheckoutOutAtRevision(path, revision):
454    return False
455  if check_only:
456    return True
457  path = path.replace('/', os.sep)
458  RmtreeIfExists(path)
459  MkdirRecursive(path)
460  logging.info('Fetching %s @ %s into %s', git_url, revision, path)
461  subprocess.check_call(['git', 'init', path], cwd=path)
462  subprocess.check_call(
463      ['git', 'fetch', '--quiet', '--depth', '1', git_url, revision], cwd=path)
464  subprocess.check_call(['git', 'checkout', revision, '--quiet'], cwd=path)
465  assert (IsGitRepoCheckoutOutAtRevision(path, revision))
466  return True
467
468
469def InstallNodeModules(force_clean=False):
470  if force_clean:
471    node_modules = os.path.join(UI_DIR, 'node_modules')
472    logging.info('Clearing %s', node_modules)
473    subprocess.check_call(['git', 'clean', '-qxffd', node_modules],
474                          cwd=ROOT_DIR)
475  logging.info("Running `npm ci` in {0}".format(UI_DIR))
476  # `npm ci` is like `npm install` but respects package-lock.json.
477  subprocess.check_call([os.path.join(TOOLS_DIR, 'npm'), 'ci'], cwd=UI_DIR)
478  # pbjs has the bad habit of installing extra packages on its first run. Run
479  # it here, so we avoid fetches while building.
480  node_bin = os.path.join(TOOLS_DIR, 'node')
481  pbjs = [node_bin, 'node_modules/.bin/pbjs', '/dev/null', '-o', '/dev/null']
482  subprocess.call(pbjs, cwd=UI_DIR)
483  with open(NODE_MODULES_STATUS_FILE, 'w') as f:
484    f.write(HashLocalFile(os.path.join(UI_DIR, 'package-lock.json')))
485
486
487def CheckNodeModules():
488  """Returns True if the modules are up-to-date.
489
490  There doesn't seem to be an easy way to check node modules versions. Instead
491  just check if package-lock.json changed since the last `npm install` call.
492  """
493  if not os.path.exists(NODE_MODULES_STATUS_FILE):
494    return False
495  with open(NODE_MODULES_STATUS_FILE, 'r') as f:
496    actual = f.read()
497  expected = HashLocalFile(os.path.join(UI_DIR, 'package-lock.json'))
498  return expected == actual
499
500
501def CheckHashes():
502  for dep in ALL_DEPS:
503    if dep.source_url.endswith('.git'):
504      continue
505    logging.info('Downloading %s for %s-%s', dep.source_url, dep.target_os,
506                 dep.target_arch)
507    with tempfile.NamedTemporaryFile(delete=False) as f:
508      f.close()
509      DownloadURL(dep.source_url, f.name)
510      actual_checksum = HashLocalFile(f.name)
511      os.unlink(f.name)
512      if (actual_checksum != dep.checksum):
513        logging.fatal('SHA-256 mismatch for {} expected {} was {}'.format(
514            dep.source_url, dep.checksum, actual_checksum))
515
516
517def Main():
518  parser = argparse.ArgumentParser()
519  parser.add_argument(
520      '--android',
521      action='store_true',
522      help='NDK and emulator images target_os="android"')
523  parser.add_argument(
524      '--linux-arm',
525      action='store_true',
526      help='Debian sysroots for target_os="linux" target_cpu="arm|arm64"')
527  parser.add_argument(
528      '--ui',
529      action='store_true',
530      help='Node and NPM packages to Build the Web-based UI via ./ui/build')
531  parser.add_argument('--check-only')
532  parser.add_argument('--filter', default='')
533  parser.add_argument('--verify', help='Check all URLs', action='store_true')
534  parser.add_argument(
535      '--no-toolchain', help='Do not download toolchain', action='store_true')
536  args = parser.parse_args()
537  if args.verify:
538    CheckHashes()
539    return 0
540
541  target_os = system().lower()
542  if args.ui and target_os == 'windows':
543    print('Building the UI on Windows is unsupported')
544    return 1
545
546  deps = BUILD_DEPS_HOST
547  if not args.no_toolchain:
548    deps += BUILD_DEPS_TOOLCHAIN_HOST
549  if args.android:
550    deps += BUILD_DEPS_ANDROID + TEST_DEPS_ANDROID
551  if args.linux_arm:
552    deps += BUILD_DEPS_LINUX_CROSS_SYSROOTS
553  if args.ui:
554    deps += UI_DEPS
555  deps_updated = False
556  nodejs_updated = False
557
558  for old_dir in CLEANUP_OLD_DIRS:
559    RmtreeIfExists(os.path.join(ROOT_DIR, old_dir))
560
561  for dep in deps:
562    target_arch = GetArch()
563    matches_os = dep.target_os == 'all' or target_os == dep.target_os
564    matches_arch = dep.target_arch == 'all' or target_arch == dep.target_arch
565    if not matches_os or not matches_arch:
566      continue
567    if args.filter and args.filter not in dep.target_folder:
568      continue
569    local_path = os.path.join(ROOT_DIR, dep.target_folder)
570    if dep.source_url.endswith('.git'):
571      deps_updated |= CheckoutGitRepo(local_path, dep.source_url, dep.checksum,
572                                      args.check_only)
573      continue
574    is_zip = local_path.endswith('.zip') or local_path.endswith('.tgz')
575    zip_target_dir = local_path[:-4] if is_zip else None
576    zip_dir_stamp = os.path.join(zip_target_dir, '.stamp') if is_zip else None
577
578    if ((not is_zip and HashLocalFile(local_path) == dep.checksum) or
579        (is_zip and ReadFile(zip_dir_stamp) == dep.checksum)):
580      continue
581    deps_updated = True
582    if args.check_only:
583      continue
584    MkdirRecursive(os.path.dirname(dep.target_folder))
585    if HashLocalFile(local_path) != dep.checksum:
586      download_path = local_path + '.tmp'
587      logging.info('Downloading %s from %s', local_path, dep.source_url)
588      DownloadURL(dep.source_url, download_path)
589      os.chmod(download_path, 0o755)
590      actual_checksum = HashLocalFile(download_path)
591      if (actual_checksum != dep.checksum):
592        os.remove(download_path)
593        logging.fatal('SHA-256 mismatch for {} expected {} was {}'.format(
594            download_path, dep.checksum, actual_checksum))
595        return 1
596      shutil.move(download_path, local_path)
597      if 'nodejs' in dep.target_folder:
598        nodejs_updated = True
599
600    assert (HashLocalFile(local_path) == dep.checksum)
601
602    if is_zip:
603      logging.info('Extracting %s into %s' % (local_path, zip_target_dir))
604      assert (os.path.commonprefix((ROOT_DIR, zip_target_dir)) == ROOT_DIR)
605      RmtreeIfExists(zip_target_dir)
606
607      # Decompress the archive.
608      if local_path.endswith('.tgz'):
609        MkdirRecursive(zip_target_dir)
610        subprocess.check_call(['tar', '-xf', local_path], cwd=zip_target_dir)
611      elif local_path.endswith('.zip'):
612        with zipfile.ZipFile(local_path, 'r') as zf:
613          for info in zf.infolist():
614            ExtractZipfilePreservePermissions(zf, info, zip_target_dir)
615
616      # If the zip contains one root folder, rebase one level up moving all
617      # its sub files and folders inside |target_dir|.
618      subdir = os.listdir(zip_target_dir)
619      if len(subdir) == 1:
620        subdir = os.path.join(zip_target_dir, subdir[0])
621        if os.path.isdir(subdir):
622          for subf in os.listdir(subdir):
623            shutil.move(os.path.join(subdir, subf), zip_target_dir)
624          os.rmdir(subdir)
625
626      # Create stamp and remove the archive.
627      with open(zip_dir_stamp, 'w') as stamp_file:
628        stamp_file.write(dep.checksum)
629      os.remove(local_path)
630
631  if args.ui:
632    # Needs to happen after nodejs is installed above.
633    if args.check_only:
634      deps_updated |= not CheckNodeModules()
635    else:
636      InstallNodeModules(force_clean=nodejs_updated)
637
638  cur_python_interpreter = sys.executable
639  test_data_synced = 0 == subprocess.call([
640      cur_python_interpreter, TEST_DATA_SCRIPT, 'status', '--quiet',
641      '--ignore-new'
642  ])
643  if args.check_only:
644    if not deps_updated and test_data_synced:
645      with open(args.check_only, 'w') as f:
646        f.write('OK')  # The content is irrelevant, just keep GN happy.
647      return 0
648    argz = ' '.join(
649        [x for x in sys.argv[1:] if not x.startswith('--check-only')])
650    print('\033[91mBuild deps are stale. ' +
651          'Please run tools/install-build-deps %s\033[0m' % argz)
652    if not test_data_synced:
653      print('//test/data/ is out of sync. `tools/test_data status` for details')
654    return 1
655
656  if not test_data_synced:
657    cmd = [cur_python_interpreter, TEST_DATA_SCRIPT, 'download', '--overwrite']
658    if not sys.stdout.isatty():
659      cmd += ['--verbose']  # For CI bots
660    subprocess.check_call(cmd)
661
662  if deps_updated:
663    # Stale binary files may be compiled against old sysroot headers that aren't
664    # tracked by gn.
665    logging.warning('Remember to run "gn clean <output_directory>" ' +
666                    'to avoid stale binary files.')
667
668
669if __name__ == '__main__':
670  logging.basicConfig(level=logging.INFO)
671  sys.exit(Main())
672