• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/env python3
2# Copyright (C) 2017 The Android Open Source Project
3#
4# Licensed under the Apache License, Version 2.0 (the "License");
5# you may not use this file except in compliance with the License.
6# You may obtain a copy of the License at
7#
8#      http://www.apache.org/licenses/LICENSE-2.0
9#
10# Unless required by applicable law or agreed to in writing, software
11# distributed under the License is distributed on an "AS IS" BASIS,
12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13# See the License for the specific language governing permissions and
14# limitations under the License.
15
16import argparse
17import dataclasses as dc
18import hashlib
19import logging
20import os
21import shutil
22import subprocess
23import stat
24import sys
25import tempfile
26import time
27import zipfile
28
29from collections import namedtuple
30from platform import system, machine
31
32
33# The format for the deps below is the following:
34# (target_folder, source_url, sha1, target_os, target_arch)
35# |source_url| can be either a git repo or a http url.
36# If a git repo, |checksum| is the SHA1 committish that will be checked out.
37# If a http url, |checksum| is the SHA256 of the downloaded file.
38# If the url is a .zip or .tgz file it will be automatically deflated under
39# |target_folder|, taking care of stripping the root folder if it's a single
40# root (to avoid ending up with buildtools/protobuf/protobuf-1.2.3/... and have
41# instead just buildtools/protobuf).
42# |target_os| is either 'darwin', 'linux', 'windows' or 'all'
43# |target_arch| is either 'x64', 'arm64' or 'all'
44# in both cases the dep only applies on matching platforms
45# |target_arch| can be 'all' when 'target_os' is not 'all' for example in the
46# case of MacOS universal binaries.
47@dc.dataclass
48class Dependency:
49  target_folder: str
50  source_url: str
51  checksum: str
52  target_os: str
53  target_arch: str
54  submodules: bool = False
55
56
57# This is to remove old directories when build tools get {re,}moved. This is to
58# avoid accidentally referring to stale dir in custom user scripts.
59CLEANUP_OLD_DIRS = [
60    'buildtools/nodejs',  # Moved to buildtools/{mac,linux64}/nodejs
61    'buildtools/emsdk',  # Moved to buildtools/{mac,linux64}/emsdk
62    'buildtools/test_data',  # Moved to test/data by r.android.com/1539381 .
63    'buildtools/d8',  # Removed by r.android.com/1424334 .
64
65    # Build toools moved to third_party/ by r.android.com/2327602 .
66    'buildtools/mac/clang-format',
67    'buildtools/mac/gn',
68    'buildtools/mac/ninja',
69    'buildtools/linux64/clang-format',
70    'buildtools/linux64/gn',
71    'buildtools/linux64/ninja',
72    'buildtools/win/clang-format.exe',
73    'buildtools/win/gn.exe',
74    'buildtools/win/ninja.exe',
75]
76
77# Dependencies required to build code on the host or when targeting desktop OS.
78BUILD_DEPS_TOOLCHAIN_HOST = [
79    # GN. From https://chrome-infra-packages.appspot.com/dl/gn/gn/.
80    # git_revision:0725d7827575b239594fbc8fd5192873a1d62f44 .
81    Dependency(
82        'third_party/gn/gn',
83        'https://storage.googleapis.com/perfetto/gn-mac-1968-0725d782',
84        '9ced623a664560bba38bbadb9b91158ca4186358c847e17ab7d982b351373c2e',
85        'darwin', 'x64'),
86    Dependency(
87        'third_party/gn/gn',
88        'https://storage.googleapis.com/perfetto/gn-mac-arm64-1968-0725d782',
89        'd22336b5210b4dad5e36e8c28ce81187f491822cf4d8fd0a257b30d6bee3fd3f',
90        'darwin', 'arm64'),
91    Dependency(
92        'third_party/gn/gn',
93        'https://storage.googleapis.com/perfetto/gn-linux64-1968-0725d782',
94        'f706aaa0676e3e22f5fc9ca482295d7caee8535d1869f99efa2358177b64f5cd',
95        'linux', 'x64'),
96    Dependency(
97        'third_party/gn/gn.exe',
98        'https://storage.googleapis.com/perfetto/gn-win-1968-0725d782',
99        '001f777f023c7a6959c778fb3a6b6cfc63f6baef953410ecdeaec350fb12285b',
100        'windows', 'x64'),
101
102    # clang-format
103    # From https://chromium.googlesource.com/chromium/src/buildtools/+/refs/heads/master/mac/clang-format.sha1
104    Dependency(
105        'third_party/clang-format/clang-format',
106        'https://storage.googleapis.com/chromium-clang-format/62bde1baa7196ad9df969fc1f06b66360b1a927b',
107        '6df686a937443cbe6efc013467a7ba5f98d3f187eb7765bb7abc6ce47626cf66',
108        'darwin', 'all'),
109    # From https://chromium.googlesource.com/chromium/src/buildtools/+/refs/heads/master/linux64/clang-format.sha1
110    Dependency(
111        'third_party/clang-format/clang-format',
112        'https://storage.googleapis.com/chromium-clang-format/1baf0089e895c989a311b6a38ed94d0e8be4c0a7',
113        'd02a97a87e8c28898033aaf5986967b24dc47ebd5b376e1cd93e5009f22cd75e',
114        'linux', 'x64'),
115    # From https://chromium.googlesource.com/chromium/src/buildtools/+/refs/heads/master/win/clang-format.exe.sha1
116    Dependency(
117        'third_party/clang-format/clang-format.exe',
118        'https://storage.googleapis.com/chromium-clang-format/d4afd4eba27022f5f6d518133aebde57281677c9',
119        '2ba1b4d3ade90ea80316890b598ab5fc16777572be26afec6ce23117da121b80',
120        'windows', 'x64'),
121
122    # Keep the SHA1 in sync with |clang_format_rev| in chromium //buildtools/DEPS.
123    Dependency(
124        'buildtools/clang_format/script',
125        'https://chromium.googlesource.com/chromium/llvm-project/cfe/tools/clang-format.git',
126        '96636aa0e9f047f17447f2d45a094d0b59ed7917', 'all', 'all'),
127
128    # Ninja
129    Dependency(
130        'third_party/ninja/ninja',
131        'https://storage.googleapis.com/perfetto/ninja-mac-x64_and_arm64-182',
132        '36e8b7aaa06911e1334feb664dd731a1cd69a15eb916a231a3d10ff65fca2c73',
133        'darwin', 'all'),
134    Dependency(
135        'third_party/ninja/ninja',
136        'https://storage.googleapis.com/perfetto/ninja-linux64-182',
137        '54ac6a01362190aaabf4cf276f9c8982cdf11b225438940fdde3339be0f2ecdc',
138        'linux', 'x64'),
139    Dependency(
140        'third_party/ninja/ninja.exe',
141        'https://storage.googleapis.com/perfetto/ninja-win-182',
142        '09ced0fcd1a4dec7d1b798a2cf9ce5d20e5d2fbc2337343827f192ce47d0f491',
143        'windows', 'x64'),
144
145    # Keep the revision in sync with Chrome's PACKAGE_VERSION in
146    # tools/clang/scripts/update.py.
147    Dependency(
148        'buildtools/linux64/clang.tgz',
149        'https://commondatastorage.googleapis.com/chromium-browser-clang/Linux_x64/clang-llvmorg-16-init-8697-g60809cd2-1.tgz',
150        '5ae35f85e0d32136795c6b223bf64263d46678dd4a24fea4e9039e58a32670de',
151        'linux', 'x64'),
152    Dependency(
153        'buildtools/win/clang.tgz',
154        'https://commondatastorage.googleapis.com/chromium-browser-clang/Win/clang-llvmorg-16-init-8697-g60809cd2-1.tgz',
155        '086faec822acba5b9c0308c6a8be34424031027d757efa2b81805aed18ffc521',
156        'windows', 'x64'),
157]
158
159BUILD_DEPS_HOST = [
160    # Keep in sync with Android's //external/googletest/METADATA.
161    Dependency(
162        'buildtools/googletest',
163        'https://android.googlesource.com/platform/external/googletest.git',
164        '609281088cfefc76f9d0ce82e1ff6c30cc3591e5', 'all', 'all'),
165
166    # Keep in sync with Chromium's //third_party/protobuf.
167    Dependency(
168        'buildtools/protobuf',
169        'https://chromium.googlesource.com/external/github.com/protocolbuffers/protobuf.git',
170        'fe271ab76f2ad2b2b28c10443865d2af21e27e0e',  # refs/tags/v3.20.3
171        'all',
172        'all'),
173
174    # libc++, libc++abi and libunwind for Linux where we need to rebuild the C++
175    # lib from sources. Keep the SHA1s in sync with Chrome's src/buildtools/DEPS.
176    Dependency(
177        'buildtools/libcxx',
178        'https://chromium.googlesource.com/external/github.com/llvm/llvm-project/libcxx.git',
179        'f8571eaba606bde2eb8cd34b30104ca33e7c207e', 'all', 'all'),
180    Dependency(
181        'buildtools/libcxxabi',
182        'https://chromium.googlesource.com/external/github.com/llvm/llvm-project/libcxxabi.git',
183        '8dd405113a4f3694e910b79785dd7fb7535a888a', 'all', 'all'),
184    Dependency(
185        'buildtools/libunwind',
186        'https://chromium.googlesource.com/external/github.com/llvm/llvm-project/libunwind.git',
187        'aabcd8753678f1536e15eb6385a948470debdae4', 'all', 'all'),
188
189    # Keep in sync with chromium DEPS.
190    Dependency(
191        'buildtools/libfuzzer',
192        'https://chromium.googlesource.com/chromium/llvm-project/compiler-rt/lib/fuzzer.git',
193        'debe7d2d1982e540fbd6bd78604bf001753f9e74', 'linux', 'all'),
194
195    # Benchmarking tool.
196    Dependency(
197        'buildtools/benchmark',
198        'https://chromium.googlesource.com/external/github.com/google/benchmark.git',
199        'e991355c02b93fe17713efe04cbc2e278e00fdbd', 'all', 'all'),
200
201    # Libbacktrace, for stacktraces in Linux/Android debug builds.
202    # From https://github.com/ianlancetaylor/libbacktrace/archive/177940370e4a6b2509e92a0aaa9749184e64af43.zip
203    Dependency(
204        'buildtools/libbacktrace.zip',
205        'https://storage.googleapis.com/perfetto/libbacktrace-177940370e4a6b2509e92a0aaa9749184e64af43.zip',
206        '21ac9a4209f7aeef766c482be53a7fa365063c031c7077e2070b491202983b31',
207        'all', 'all'),
208
209    # Sqlite for the trace processing library.
210    # This is the amalgamated source whose compiled output is meant to be faster.
211    # We still pull the full source for the extensions (which are not available
212    # in the amalgamation).
213    # If updating the version, also update bazel/deps.bzl.
214    Dependency(
215        'buildtools/sqlite.zip',
216        'https://storage.googleapis.com/perfetto/sqlite-amalgamation-3390200.zip',
217        '87775784f8b22d0d0f1d7811870d39feaa7896319c7c20b849a4181c5a50609b',
218        'all', 'all'),
219    Dependency(
220        'buildtools/sqlite_src',
221        'https://chromium.googlesource.com/external/github.com/sqlite/sqlite.git',
222        '202b2a7b54ea2dd13a8a5adfd75523abe4dcf17f',  # refs/tags/version-3.39.2.
223        'all',
224        'all'),
225
226    # JsonCpp for legacy json import. Used only by the trace processor in
227    # standalone builds.
228    # If updating the version, also update bazel/deps.bzl.
229    Dependency(
230        'buildtools/jsoncpp',
231        'https://chromium.googlesource.com/external/github.com/open-source-parsers/jsoncpp.git',
232        '6aba23f4a8628d599a9ef7fa4811c4ff6e4070e2',  # refs/tags/1.9.3.
233        'all',
234        'all'),
235
236    # Archive with only the demangling sources from llvm-project.
237    # See tools/repackage_llvm_demangler.sh on how to update this.
238    # File suffix is the git reference to the commit at which we rearchived the
239    # sources, as hosted on https://llvm.googlesource.com/llvm-project.
240    # If updating the version, also update bazel/deps.bzl.
241    Dependency(
242        'buildtools/llvm-project.tgz',
243        'https://storage.googleapis.com/perfetto/llvm-project-3b4c59c156919902c785ce3cbae0eee2ee53064d.tgz',
244        'f4a52e7f36edd7cacc844d5ae0e5f60b6f57c5afc40683e99f295886c9ce8ff4',
245        'all', 'all'),
246
247    # These dependencies are for libunwindstack, which is used by src/profiling.
248    Dependency('buildtools/android-core',
249               'https://android.googlesource.com/platform/system/core.git',
250               '9e6cef7f07d8c11b3ea820938aeb7ff2e9dbaa52', 'all', 'all'),
251    Dependency(
252        'buildtools/android-unwinding',
253        'https://android.googlesource.com/platform/system/unwinding.git',
254        '11cf5564fbff659b5cdba9c3e977d4fc51bab4b3', 'all', 'all'),
255    Dependency('buildtools/android-logging',
256               'https://android.googlesource.com/platform/system/logging.git',
257               '7b36b566c9113fc703d68f76e8f40c0c2432481c', 'all', 'all'),
258    Dependency('buildtools/android-libbase',
259               'https://android.googlesource.com/platform/system/libbase.git',
260               '78f1c2f83e625bdf66d55b48bdb3a301c20d2fb3', 'all', 'all'),
261    Dependency(
262        'buildtools/android-libprocinfo',
263        'https://android.googlesource.com/platform/system/libprocinfo.git',
264        'fd214c13ededecae97a3b15b5fccc8925a749a84', 'all', 'all'),
265    Dependency('buildtools/lzma',
266               'https://android.googlesource.com/platform/external/lzma.git',
267               '7851dce6f4ca17f5caa1c93a4e0a45686b1d56c3', 'all', 'all'),
268    Dependency('buildtools/bionic',
269               'https://android.googlesource.com/platform/bionic.git',
270               '4b0e16bc72a82a63c699977376a7d6eadca1b206', 'all', 'all'),
271
272    # Zlib used both in the tracing binaries, as well as the trace processor and
273    # assorted tools.
274    # If updating the version, also update bazel/deps.bzl.
275    Dependency('buildtools/zlib',
276               'https://android.googlesource.com/platform/external/zlib.git',
277               '6d3f6aa0f87c9791ca7724c279ef61384f331dfd', 'all', 'all'),
278
279    # Linenoise, used only by trace_processor in standalone builds.
280    # If updating the version, also update bazel/deps.bzl.
281    Dependency('buildtools/linenoise',
282               'https://fuchsia.googlesource.com/third_party/linenoise.git',
283               'c894b9e59f02203dbe4e2be657572cf88c4230c3', 'all', 'all'),
284
285    # Bloaty, used to investigate binary size
286    Dependency(
287        'buildtools/bloaty.zip',
288        'https://storage.googleapis.com/perfetto/bloaty-1.1-b3b829de35babc2fe831b9488ad2e50bca939412-mac.zip',
289        '2d301bd72a20e3f42888c9274ceb4dca76c103608053572322412c2c65ab8cb8',
290        'darwin', 'all'),
291]
292
293# Dependencies required to build Android code.
294# URLs and SHA1s taken from:
295# - https://dl.google.com/android/repository/repository-11.xml
296# - https://dl.google.com/android/repository/sys-img/android/sys-img.xml
297BUILD_DEPS_ANDROID = [
298    # Android NDK
299    Dependency(
300        'buildtools/ndk.zip',
301        'https://dl.google.com/android/repository/android-ndk-r21e-darwin-x86_64.zip',
302        '437278103a3db12632c05b1be5c41bbb8522791a67e415cc54411a65366f499d',
303        'darwin', 'all'),
304    Dependency(
305        'buildtools/ndk.zip',
306        'https://dl.google.com/android/repository/android-ndk-r21e-linux-x86_64.zip',
307        'ad7ce5467e18d40050dc51b8e7affc3e635c85bd8c59be62de32352328ed467e',
308        'linux', 'x64'),
309]
310
311# Dependencies required to run Android tests.
312TEST_DEPS_ANDROID = [
313    # Android emulator images.
314    Dependency(
315        'buildtools/aosp-arm.zip',
316        'https://storage.googleapis.com/perfetto/aosp-02022018-arm.zip',
317        'f5c7a3a22ad7aa0bd14ba467e8697e1e917d306699bd25622aa4419a413b9b67',
318        'all', 'all'),
319
320    # platform-tools.zip contains adb binaries.
321    Dependency(
322        'buildtools/android_sdk/platform-tools.zip',
323        'https://dl.google.com/android/repository/platform-tools_r26.0.0-darwin.zip',
324        '98d392cbd21ca20d643c7e1605760cc49075611e317c534096b5564053f4ac8e',
325        'darwin', 'all'),
326    Dependency(
327        'buildtools/android_sdk/platform-tools.zip',
328        'https://dl.google.com/android/repository/platform-tools_r26.0.0-linux.zip',
329        '90208207521d85abf0d46e3374aa4e04b7aff74e4f355c792ac334de7a77e50b',
330        'linux', 'x64'),
331
332    # Android emulator binaries.
333    Dependency(
334        'buildtools/emulator',
335        'https://android.googlesource.com/platform/prebuilts/android-emulator.git',
336        '4b260028dc27bc92c39bee9129cb2ba839970956', 'all', 'x64'),
337]
338
339# This variable is updated by tools/roll-catapult-trace-viewer.
340CATAPULT_SHA256 = 'b30108e05268ce6c65bb4126b65f6bfac165d17f5c1fd285046e7e6fd76c209f'
341
342TYPEFACES_SHA256 = '1065172aaf0e9c22bc4f206ed9fdf5f1b4355d233fb21f9f26a89cd66c941940'
343
344UI_DEPS = [
345    Dependency(
346        'buildtools/mac/nodejs.tgz',
347        'https://storage.googleapis.com/chromium-nodejs/16.13.0/31859fc1fa0994a95f44f09c367d6ff63607cfde',
348        'd9cf1f36b16e08ce52cfbdef427c6596eed2bd2f3608a79112d25b4ec8f75753',
349        'darwin', 'arm64'),
350    Dependency(
351        'buildtools/mac/nodejs.tgz',
352        'https://storage.googleapis.com/chromium-nodejs/16.13.0/16dfd094763b71988933a31735f9dea966f9abd6',
353        '065ced7e93f0e26276cb74d9688706674323865c4a8fc89e4adfa6868d4ebb4d',
354        'darwin', 'x64'),
355    Dependency(
356        'buildtools/linux64/nodejs.tgz',
357        'https://storage.googleapis.com/chromium-nodejs/16.13.0/ab9544e24e752d3d17f335fb7b2055062e582d11',
358        '3b5ca150a55f3aadfa18f3a10a4495aaf9653614ba3e460647170fef5287ec4f',
359        'linux', 'x64'),
360    Dependency(
361        'buildtools/mac/emsdk.tgz',
362        'https://storage.googleapis.com/perfetto/emscripten-2.0.12-mac.tgz',
363        'aa125f8c8ff8a386d43e18c8ea0c98c875cc19160a899403e8967a5478f96f31',
364        'darwin', 'all'),
365    Dependency(
366        'buildtools/linux64/emsdk.tgz',
367        'https://storage.googleapis.com/perfetto/emscripten-2.0.12-linux.tgz',
368        'bfff9fb0326363c12e19b542f27a5f12cedbfc310f30621dc497c9af51d2d2e3',
369        'linux', 'x64'),
370    Dependency(
371        'buildtools/catapult_trace_viewer.tgz',
372        'https://storage.googleapis.com/perfetto/catapult_trace_viewer-%s.tar.gz'
373        % CATAPULT_SHA256, CATAPULT_SHA256, 'all', 'all'),
374    Dependency(
375        'buildtools/typefaces.tgz',
376        'https://storage.googleapis.com/perfetto/typefaces-%s.tar.gz' %
377        TYPEFACES_SHA256, TYPEFACES_SHA256, 'all', 'all')
378]
379
380# Dependencies to build gRPC.
381GRPC_DEPS = [
382    Dependency(
383        'buildtools/grpc/src',
384        'https://chromium.googlesource.com/external/github.com/grpc/grpc.git',
385        '6943c1841f57cac4666b165aea4f618fe73b3ff1', 'all', 'all', True),
386]
387
388# Sysroots required to cross-compile Linux targets (linux-arm{,64}).
389# These are taken from Chromium's build/linux/sysroot_scripts/sysroots.json.
390BUILD_DEPS_LINUX_CROSS_SYSROOTS = [
391    Dependency(
392        'buildtools/debian_sid_arm-sysroot.tgz',
393        'https://commondatastorage.googleapis.com/chrome-linux-sysroot/toolchain/11d6f690ca49e8ba01a1d8c5346cedad2cf308fd/debian_sid_arm_sysroot.tar.xz',
394        'ff192fe073d140d836c9ca1e68f7200d558bb9aa6c5c8f4f76f794f82890f99a',
395        'linux', 'all'),
396    Dependency(
397        'buildtools/debian_sid_arm64-sysroot.tgz',
398        'https://commondatastorage.googleapis.com/chrome-linux-sysroot/toolchain/2befe8ce3e88be6080e4fb7e6d412278ea6a7625/debian_sid_arm64_sysroot.tar.xz',
399        'e4389eab2fe363f3fbdfa4d3ce9d94457d78fd2c0e62171a7534867623eadc90',
400        'linux', 'all'),
401]
402
403ALL_DEPS = (
404    BUILD_DEPS_HOST + BUILD_DEPS_ANDROID + BUILD_DEPS_LINUX_CROSS_SYSROOTS +
405    TEST_DEPS_ANDROID + UI_DEPS)
406
407ROOT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
408UI_DIR = os.path.join(ROOT_DIR, 'ui')
409TOOLS_DIR = os.path.join(ROOT_DIR, 'tools')
410NODE_MODULES_STATUS_FILE = os.path.join(UI_DIR, 'node_modules', '.last_install')
411TEST_DATA_SCRIPT = os.path.join(TOOLS_DIR, 'test_data')
412
413
414def CheckCallRetry(*args, **kwargs):
415  """ Like subprocess.check_call, with retries up to 5 times. """
416  MAX_ATTEMPTS = 5
417  for attempt in range(1, MAX_ATTEMPTS + 1):
418    try:
419      return subprocess.check_call(*args, **kwargs)
420    except subprocess.CalledProcessError as error:
421      if attempt == MAX_ATTEMPTS:
422        raise error
423      else:
424        logging.error(error)
425        time.sleep(attempt * 3)
426
427
428def DownloadURL(url, out_file):
429  CheckCallRetry(['curl', '-L', '-#', '-o', out_file, url])
430
431
432def GetArch():
433  arch = machine()
434  if arch == 'arm64':
435    return 'arm64'
436  else:
437    # Assume everything else is x64 matching previous behaviour.
438    return 'x64'
439
440
441def ReadFile(path):
442  if not os.path.exists(path):
443    return None
444  with open(path) as f:
445    return f.read().strip()
446
447
448def MkdirRecursive(path):
449  # Works with both relative and absolute paths
450  cwd = '/' if path.startswith('/') else ROOT_DIR
451  for part in path.split('/'):
452    cwd = os.path.join(cwd, part)
453    if not os.path.exists(cwd):
454      os.makedirs(cwd)
455    else:
456      assert (os.path.isdir(cwd))
457
458
459def HashLocalFile(path):
460  if not os.path.exists(path):
461    return None
462  with open(path, 'rb') as f:
463    return hashlib.sha256(f.read()).hexdigest()
464
465
466def ExtractZipfilePreservePermissions(zf, info, path):
467  zf.extract(info.filename, path=path)
468  target_path = os.path.join(path, info.filename)
469  min_acls = 0o755 if info.filename.endswith('/') else 0o644
470  os.chmod(target_path, (info.external_attr >> 16) | min_acls)
471
472
473def IsGitRepoCheckoutOutAtRevision(path, revision):
474  return ReadFile(os.path.join(path, '.git', 'HEAD')) == revision
475
476
477def RmtreeIfExists(path):
478  # Git creates read-only files on windows, which cause failures with rmtree.
479  # This seems the socially accepted way to deal with it.
480  # See https://bugs.python.org/issue19643 .
481  def del_read_only_for_windows(_action, name, _exc):
482    os.chmod(name, stat.S_IWRITE)
483    os.remove(name)
484
485  if not os.path.exists(path):
486    return
487  third_party_path = os.path.abspath(os.path.join(ROOT_DIR, 'third_party'))
488  buildtools_path = os.path.abspath(os.path.join(ROOT_DIR, 'buildtools'))
489  test_path = os.path.abspath(os.path.join(ROOT_DIR, 'test', 'data'))
490  if (not os.path.abspath(path).startswith(buildtools_path) and
491      not os.path.abspath(path).startswith(test_path) and
492      not os.path.abspath(path).startswith(third_party_path)):
493    # Safety check to prevent that some merge confilct ends up doing some
494    # rm -rf / or similar.
495    logging.fatal(
496        'Cannot remove %s: outside of {buildtools, test/data, third_party}',
497        path)
498    sys.exit(1)
499  logging.info('Removing %s' % path)
500  if os.path.isdir(path):
501    shutil.rmtree(path, onerror=del_read_only_for_windows)
502  else:
503    os.remove(path)
504
505
506def CheckoutGitRepo(path, git_url, revision, check_only):
507  if IsGitRepoCheckoutOutAtRevision(path, revision):
508    return False
509  if check_only:
510    return True
511  path = path.replace('/', os.sep)
512  RmtreeIfExists(path)
513  MkdirRecursive(path)
514  logging.info('Fetching %s @ %s into %s', git_url, revision, path)
515  subprocess.check_call(['git', 'init', path], cwd=path)
516  CheckCallRetry(['git', 'fetch', '--quiet', '--depth', '1', git_url, revision],
517                 cwd=path)
518  subprocess.check_call(['git', 'checkout', revision, '--quiet'], cwd=path)
519  CheckCallRetry(
520      ['git', 'submodule', 'update', '--init', '--recursive', '--quiet'],
521      cwd=path)
522  assert (IsGitRepoCheckoutOutAtRevision(path, revision))
523  return True
524
525
526def InstallNodeModules(force_clean=False):
527  if force_clean:
528    node_modules = os.path.join(UI_DIR, 'node_modules')
529    logging.info('Clearing %s', node_modules)
530    subprocess.check_call(['git', 'clean', '-qxffd', node_modules],
531                          cwd=ROOT_DIR)
532  logging.info("Running `npm ci` in {0}".format(UI_DIR))
533  # `npm ci` is like `npm install` but respects package-lock.json.
534  subprocess.check_call([os.path.join(TOOLS_DIR, 'npm'), 'ci'], cwd=UI_DIR)
535  # pbjs has the bad habit of installing extra packages on its first run. Run
536  # it here, so we avoid fetches while building.
537  node_bin = os.path.join(TOOLS_DIR, 'node')
538  pbjs = [node_bin, 'node_modules/.bin/pbjs', '/dev/null', '-o', '/dev/null']
539  subprocess.call(pbjs, cwd=UI_DIR)
540  with open(NODE_MODULES_STATUS_FILE, 'w') as f:
541    f.write(HashLocalFile(os.path.join(UI_DIR, 'package-lock.json')))
542
543
544def CheckNodeModules():
545  """Returns True if the modules are up-to-date.
546
547  There doesn't seem to be an easy way to check node modules versions. Instead
548  just check if package-lock.json changed since the last `npm install` call.
549  """
550  if not os.path.exists(NODE_MODULES_STATUS_FILE):
551    return False
552  with open(NODE_MODULES_STATUS_FILE, 'r') as f:
553    actual = f.read()
554  expected = HashLocalFile(os.path.join(UI_DIR, 'package-lock.json'))
555  return expected == actual
556
557
558def CheckHashes():
559  for dep in ALL_DEPS:
560    if dep.source_url.endswith('.git'):
561      continue
562    logging.info('Downloading %s for %s-%s', dep.source_url, dep.target_os,
563                 dep.target_arch)
564    with tempfile.NamedTemporaryFile(delete=False) as f:
565      f.close()
566      DownloadURL(dep.source_url, f.name)
567      actual_checksum = HashLocalFile(f.name)
568      os.unlink(f.name)
569      if (actual_checksum != dep.checksum):
570        logging.fatal('SHA-256 mismatch for {} expected {} was {}'.format(
571            dep.source_url, dep.checksum, actual_checksum))
572
573
574def CheckDepotToolsIsRecent():
575  gn_py_path = shutil.which('gn.py')
576  if gn_py_path is None:
577    return True  # depot_tools doesn't seem to be installed in the PATH.
578  dt_dir = os.path.abspath(os.path.dirname(gn_py_path))
579  cmd = ['git', '-C', dt_dir, 'merge-base', '--is-ancestor', 'a0cf4321', 'HEAD']
580  git_ret = subprocess.call(cmd, stderr=subprocess.DEVNULL)
581  if git_ret == 0:
582    return True
583  print('\033[91mYour depot_tools revision is too old. Please run:\033[0m')
584  print('git -C %s fetch origin && git -C %s checkout -B main -t origin/main' %
585        (dt_dir, dt_dir))
586  return False
587
588
589def Main():
590  parser = argparse.ArgumentParser()
591  parser.add_argument(
592      '--android',
593      action='store_true',
594      help='NDK and emulator images target_os="android"')
595  parser.add_argument(
596      '--linux-arm',
597      action='store_true',
598      help='Debian sysroots for target_os="linux" target_cpu="arm|arm64"')
599  parser.add_argument(
600      '--ui',
601      action='store_true',
602      help='Node and NPM packages to Build the Web-based UI via ./ui/build')
603  parser.add_argument(
604      '--grpc', action='store_true', help='Packages to build gRPC')
605  parser.add_argument('--check-only')
606  parser.add_argument('--filter', action='append')
607  parser.add_argument('--verify', help='Check all URLs', action='store_true')
608  parser.add_argument(
609      '--no-toolchain', help='Do not download toolchain', action='store_true')
610  args = parser.parse_args()
611  if args.verify:
612    CheckHashes()
613    return 0
614
615  target_os = system().lower()
616  if args.ui and target_os == 'windows':
617    print('Building the UI on Windows is unsupported')
618    return 1
619
620  if not CheckDepotToolsIsRecent():
621    return 1
622
623  deps = BUILD_DEPS_HOST
624  if not args.no_toolchain:
625    deps += BUILD_DEPS_TOOLCHAIN_HOST
626  if args.android:
627    deps += BUILD_DEPS_ANDROID + TEST_DEPS_ANDROID
628  if args.linux_arm:
629    deps += BUILD_DEPS_LINUX_CROSS_SYSROOTS
630  if args.ui:
631    deps += UI_DEPS
632  if args.grpc:
633    deps += GRPC_DEPS
634  deps_updated = False
635  nodejs_updated = False
636
637  for old_dir in CLEANUP_OLD_DIRS:
638    RmtreeIfExists(os.path.join(ROOT_DIR, old_dir))
639
640  for dep in deps:
641    target_arch = GetArch()
642    matches_os = dep.target_os == 'all' or target_os == dep.target_os
643    matches_arch = dep.target_arch == 'all' or target_arch == dep.target_arch
644    if not matches_os or not matches_arch:
645      continue
646    if args.filter and not any(f in dep.target_folder for f in args.filter):
647      continue
648    local_path = os.path.join(ROOT_DIR, dep.target_folder)
649    if dep.source_url.endswith('.git'):
650      deps_updated |= CheckoutGitRepo(local_path, dep.source_url, dep.checksum,
651                                      args.check_only)
652      continue
653    is_zip = local_path.endswith('.zip') or local_path.endswith('.tgz')
654    zip_target_dir = local_path[:-4] if is_zip else None
655    zip_dir_stamp = os.path.join(zip_target_dir, '.stamp') if is_zip else None
656
657    if ((not is_zip and HashLocalFile(local_path) == dep.checksum) or
658        (is_zip and ReadFile(zip_dir_stamp) == dep.checksum)):
659      continue
660    deps_updated = True
661    if args.check_only:
662      continue
663    MkdirRecursive(os.path.dirname(dep.target_folder))
664    if HashLocalFile(local_path) != dep.checksum:
665      download_path = local_path + '.tmp'
666      logging.info('Downloading %s from %s', local_path, dep.source_url)
667      DownloadURL(dep.source_url, download_path)
668      os.chmod(download_path, 0o755)
669      actual_checksum = HashLocalFile(download_path)
670      if (actual_checksum != dep.checksum):
671        os.remove(download_path)
672        logging.fatal('SHA-256 mismatch for {} expected {} was {}'.format(
673            download_path, dep.checksum, actual_checksum))
674        return 1
675      shutil.move(download_path, local_path)
676      if 'nodejs' in dep.target_folder:
677        nodejs_updated = True
678
679    assert (HashLocalFile(local_path) == dep.checksum)
680
681    if is_zip:
682      logging.info('Extracting %s into %s' % (local_path, zip_target_dir))
683      assert (os.path.commonprefix((ROOT_DIR, zip_target_dir)) == ROOT_DIR)
684      RmtreeIfExists(zip_target_dir)
685
686      # Decompress the archive.
687      if local_path.endswith('.tgz'):
688        MkdirRecursive(zip_target_dir)
689        subprocess.check_call(['tar', '-oxf', local_path], cwd=zip_target_dir)
690      elif local_path.endswith('.zip'):
691        with zipfile.ZipFile(local_path, 'r') as zf:
692          for info in zf.infolist():
693            ExtractZipfilePreservePermissions(zf, info, zip_target_dir)
694
695      # If the zip contains one root folder, rebase one level up moving all
696      # its sub files and folders inside |target_dir|.
697      subdir = os.listdir(zip_target_dir)
698      if len(subdir) == 1:
699        subdir = os.path.join(zip_target_dir, subdir[0])
700        if os.path.isdir(subdir):
701          for subf in os.listdir(subdir):
702            shutil.move(os.path.join(subdir, subf), zip_target_dir)
703          os.rmdir(subdir)
704
705      # Create stamp and remove the archive.
706      with open(zip_dir_stamp, 'w') as stamp_file:
707        stamp_file.write(dep.checksum)
708      os.remove(local_path)
709
710  if args.ui:
711    # Needs to happen after nodejs is installed above.
712    if args.check_only:
713      deps_updated |= not CheckNodeModules()
714    else:
715      InstallNodeModules(force_clean=nodejs_updated)
716
717  cur_python_interpreter = sys.executable
718  test_data_synced = 0 == subprocess.call([
719      cur_python_interpreter, TEST_DATA_SCRIPT, 'status', '--quiet',
720      '--ignore-new'
721  ])
722  if args.check_only:
723    if not deps_updated and test_data_synced:
724      with open(args.check_only, 'w') as f:
725        f.write('OK')  # The content is irrelevant, just keep GN happy.
726      return 0
727    argz = ' '.join(
728        [x for x in sys.argv[1:] if not x.startswith('--check-only')])
729    print('\033[91mBuild deps are stale. ' +
730          'Please run tools/install-build-deps %s\033[0m' % argz)
731    if not test_data_synced:
732      print('//test/data/ is out of sync. `tools/test_data status` for details')
733    return 1
734
735  if not test_data_synced:
736    cmd = [cur_python_interpreter, TEST_DATA_SCRIPT, 'download', '--overwrite']
737    if not sys.stdout.isatty():
738      cmd += ['--verbose']  # For CI bots
739    subprocess.check_call(cmd)
740
741  if deps_updated:
742    # Stale binary files may be compiled against old sysroot headers that aren't
743    # tracked by gn.
744    logging.warning('Remember to run "gn clean <output_directory>" ' +
745                    'to avoid stale binary files.')
746
747
748if __name__ == '__main__':
749  logging.basicConfig(level=logging.INFO)
750  sys.exit(Main())
751