• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/env python3
2# Copyright (C) 2017 The Android Open Source Project
3#
4# Licensed under the Apache License, Version 2.0 (the "License");
5# you may not use this file except in compliance with the License.
6# You may obtain a copy of the License at
7#
8#      http://www.apache.org/licenses/LICENSE-2.0
9#
10# Unless required by applicable law or agreed to in writing, software
11# distributed under the License is distributed on an "AS IS" BASIS,
12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13# See the License for the specific language governing permissions and
14# limitations under the License.
15
16import argparse
17import dataclasses as dc
18import hashlib
19import logging
20import os
21import shutil
22import subprocess
23import stat
24import sys
25import tempfile
26import time
27import zipfile
28import bz2
29
30from collections import namedtuple
31from platform import system, machine
32
33
34# The format for the deps below is the following:
35# (target_folder, source_url, sha1, target_os, target_arch)
36# |source_url| can be either a git repo or a http url.
37# If a git repo, |checksum| is the SHA1 committish that will be checked out.
38# If a http url, |checksum| is the SHA256 of the downloaded file.
39# If the url is a .zip, .tgz, or .tbz2 file it will be automatically deflated under
40# |target_folder|, taking care of stripping the root folder if it's a single
41# root (to avoid ending up with buildtools/protobuf/protobuf-1.2.3/... and have
42# instead just buildtools/protobuf).
43# |target_os| is either 'darwin', 'linux', 'windows' or 'all'
44# |target_arch| is either 'x64', 'arm64' or 'all'
45# in both cases the dep only applies on matching platforms
46# |target_arch| can be 'all' when 'target_os' is not 'all' for example in the
47# case of MacOS universal binaries.
48@dc.dataclass
49class Dependency:
50  target_folder: str
51  source_url: str
52  checksum: str
53  target_os: str
54  target_arch: str
55  submodules: bool = False
56
57
58# This is to remove old directories when build tools get {re,}moved. This is to
59# avoid accidentally referring to stale dir in custom user scripts.
60CLEANUP_OLD_DIRS = [
61    'buildtools/nodejs',  # Moved to buildtools/{mac,linux64}/nodejs
62    'buildtools/emsdk',  # Moved to buildtools/{mac,linux64}/emsdk
63    'buildtools/test_data',  # Moved to test/data by r.android.com/1539381 .
64    'buildtools/d8',  # Removed by r.android.com/1424334 .
65
66    # Build tools moved to third_party/ by r.android.com/2327602 .
67    'buildtools/mac/clang-format',
68    'buildtools/mac/gn',
69    'buildtools/mac/ninja',
70    'buildtools/linux64/clang-format',
71    'buildtools/linux64/gn',
72    'buildtools/linux64/ninja',
73    'buildtools/win/clang-format.exe',
74    'buildtools/win/gn.exe',
75    'buildtools/win/ninja.exe',
76]
77
78# Dependencies required to build code on the host or when targeting desktop OS.
79BUILD_DEPS_TOOLCHAIN_HOST = [
80    # GN. From https://chrome-infra-packages.appspot.com/dl/gn/gn/.
81    # git_revision:0725d7827575b239594fbc8fd5192873a1d62f44 .
82    Dependency(
83        'third_party/gn/gn',
84        'https://storage.googleapis.com/perfetto/gn-mac-1968-0725d782',
85        '9ced623a664560bba38bbadb9b91158ca4186358c847e17ab7d982b351373c2e',
86        'darwin', 'x64'),
87    Dependency(
88        'third_party/gn/gn',
89        'https://storage.googleapis.com/perfetto/gn-mac-arm64-1968-0725d782',
90        'd22336b5210b4dad5e36e8c28ce81187f491822cf4d8fd0a257b30d6bee3fd3f',
91        'darwin', 'arm64'),
92    Dependency(
93        'third_party/gn/gn',
94        'https://storage.googleapis.com/perfetto/gn-linux64-1968-0725d782',
95        'f706aaa0676e3e22f5fc9ca482295d7caee8535d1869f99efa2358177b64f5cd',
96        'linux', 'x64'),
97    Dependency(
98        'third_party/gn/gn',
99        'https://storage.googleapis.com/perfetto/gn-linux-arm64-1968-0725d782',
100        'c2a372cd4f911028d8bc351fbf24835c9b1194fcc92beadf6c5a2b3addae973c',
101        'linux', 'arm64'),
102    Dependency(
103        'third_party/gn/gn.exe',
104        'https://storage.googleapis.com/perfetto/gn-win-1968-0725d782',
105        '001f777f023c7a6959c778fb3a6b6cfc63f6baef953410ecdeaec350fb12285b',
106        'windows', 'x64'),
107
108    # clang-format
109    # From
110    # https://chromium.googlesource.com/chromium/src/buildtools/+/refs/heads/master/mac/clang-format.arm64.sha1
111    Dependency(
112        'third_party/clang-format/clang-format',
113        'https://storage.googleapis.com/chromium-clang-format/5553d7a3d912b7d49381ad68c9a56740601a57a0',
114        'e077990b2ea6807f6abc71b4cf1e487719d7e40574baddd2b51187fdcc8db803',
115        'darwin', 'arm64'),
116    # From
117    # https://chromium.googlesource.com/chromium/src/buildtools/+/refs/heads/master/mac/clang-format.x64.sha1
118    Dependency(
119        'third_party/clang-format/clang-format',
120        'https://storage.googleapis.com/chromium-clang-format/87d69aeff220c916b85b5d6d162fa5668aa9d64f',
121        '71179a8788a009cfd589636d50f0eb9f95f58b0cfda4351430bed7c0a48c080b',
122        'darwin', 'x64'),
123    # From https://chromium.googlesource.com/chromium/src/buildtools/+/refs/heads/master/linux64/clang-format.sha1
124    Dependency(
125        'third_party/clang-format/clang-format',
126        'https://storage.googleapis.com/chromium-clang-format/1facab3101fc6da6b9467543f27f0622b966bc19',
127        '5e459118d8ac825452e9e1f2717e4de5a36399dc6cc6aec7ec483ad27a0c927e',
128        'linux', 'x64'),
129    # From https://chromium.googlesource.com/chromium/src/buildtools/+/refs/heads/master/win/clang-format.exe.sha1
130    Dependency(
131        'third_party/clang-format/clang-format.exe',
132        'https://storage.googleapis.com/chromium-clang-format/2e569921b9884daf157021d6ae9e21991cd6cf81',
133        '2227376ada89ea832995b832222b722a27c4d5d8d59e9c4d7842877f99a1e30d',
134        'windows', 'x64'),
135
136    # Keep the SHA1 in sync with |clang_format_rev| in chromium //buildtools/DEPS.
137    Dependency(
138        'buildtools/clang_format/script',
139        'https://chromium.googlesource.com/external/github.com/llvm/llvm-project/clang/tools/clang-format.git',
140        'f97059df7f8b205064625cdb5f97b56668a125ef', 'all', 'all'),
141
142    # Ninja
143    Dependency(
144        'third_party/ninja/ninja',
145        'https://storage.googleapis.com/perfetto/ninja-mac-x64_and_arm64-182',
146        '36e8b7aaa06911e1334feb664dd731a1cd69a15eb916a231a3d10ff65fca2c73',
147        'darwin', 'all'),
148    Dependency(
149        'third_party/ninja/ninja',
150        'https://storage.googleapis.com/perfetto/ninja-linux64-182',
151        '54ac6a01362190aaabf4cf276f9c8982cdf11b225438940fdde3339be0f2ecdc',
152        'linux', 'x64'),
153    Dependency(
154        'third_party/ninja/ninja.exe',
155        'https://storage.googleapis.com/perfetto/ninja-win-182',
156        '09ced0fcd1a4dec7d1b798a2cf9ce5d20e5d2fbc2337343827f192ce47d0f491',
157        'windows', 'x64'),
158    Dependency(
159        'third_party/ninja/ninja',
160        'https://storage.googleapis.com/perfetto/ninja-linux-arm64-1111',
161        '05031a734ec4310a51b2cfe9f0096b26fce25ab4ff19e5b51abe6371de066cc5',
162        'linux', 'arm64'),
163
164    # Keep the revision in sync with Chrome's PACKAGE_VERSION in
165    # tools/clang/scripts/update.py.
166    Dependency(
167        'buildtools/linux64/clang.tgz',
168        'https://commondatastorage.googleapis.com/chromium-browser-clang/Linux_x64/clang-llvmorg-19-init-2941-ga0b3dbaf-22.tgz',
169        '6741cc1083f935795330b6e04617ac891a7b5d2b5647b664c5b0fccc354adb43',
170        'linux', 'x64'),
171    Dependency(
172        'buildtools/win/clang.tgz',
173        'https://commondatastorage.googleapis.com/chromium-browser-clang/Win/clang-llvmorg-19-init-2941-ga0b3dbaf-22.tgz',
174        'f627080ed53d4c156f089323e04fa3690c8bb459110b62cd1952b0e1f0755987',
175        'windows', 'x64'),
176]
177
178BUILD_DEPS_HOST = [
179    # Keep in sync with Android's //external/googletest/METADATA.
180    Dependency(
181        'buildtools/googletest',
182        'https://android.googlesource.com/platform/external/googletest.git',
183        '609281088cfefc76f9d0ce82e1ff6c30cc3591e5', 'all', 'all'),
184
185    # Keep in sync with Chromium's //third_party/protobuf.
186    Dependency(
187        'buildtools/protobuf',
188        # If you revert the below version back to an earlier version of
189        # protobuf, make sure to revert the changes to
190        # //gn/standalone/protoc.py as well.
191        #
192        # This comment can be removed with protobuf is next upreved.
193        'https://chromium.googlesource.com/external/github.com/protocolbuffers/protobuf.git',
194        'f0dc78d7e6e331b8c6bb2d5283e06aa26883ca7c',  # refs/tags/v21.12
195        'all',
196        'all'),
197
198    # libc++, libc++abi and libunwind for Linux where we need to rebuild the C++
199    # lib from sources. Keep the SHA1s in sync with Chrome's src/buildtools/DEPS.
200    Dependency(
201        'buildtools/libcxx',
202        'https://chromium.googlesource.com/external/github.com/llvm/llvm-project/libcxx.git',
203        '852bc6746f45add53fec19f3a29280e69e358d44', 'all', 'all'),
204    Dependency(
205        'buildtools/libcxxabi',
206        'https://chromium.googlesource.com/external/github.com/llvm/llvm-project/libcxxabi.git',
207        'a37a3aa431f132b02a58656f13984d51098330a2', 'all', 'all'),
208    Dependency(
209        'buildtools/libunwind',
210        'https://chromium.googlesource.com/external/github.com/llvm/llvm-project/libunwind.git',
211        '419b03c0b8f20d6da9ddcb0d661a94a97cdd7dad', 'all', 'all'),
212
213    # Keep in sync with chromium DEPS.
214    Dependency(
215        'buildtools/libfuzzer',
216        'https://chromium.googlesource.com/chromium/llvm-project/compiler-rt/lib/fuzzer.git',
217        'debe7d2d1982e540fbd6bd78604bf001753f9e74', 'linux', 'all'),
218
219    # Benchmarking tool.
220    Dependency(
221        'buildtools/benchmark',
222        'https://chromium.googlesource.com/external/github.com/google/benchmark.git',
223        'e991355c02b93fe17713efe04cbc2e278e00fdbd', 'all', 'all'),
224
225    # Libbacktrace, for stacktraces in Linux/Android debug builds.
226    # From https://github.com/ianlancetaylor/libbacktrace/archive/177940370e4a6b2509e92a0aaa9749184e64af43.zip
227    Dependency(
228        'buildtools/libbacktrace.zip',
229        'https://storage.googleapis.com/perfetto/libbacktrace-14818b7783eeb9a56c3f0fca78cefd3143f8c5f6.zip',
230        '0d09295938155aa84d9a6049f63df8cd2def3a28302b3550ea3ead9100b3d086',
231        'all', 'all'),
232
233    # Sqlite for the trace processing library.
234    # This is the amalgamated source whose compiled output is meant to be faster.
235    # We still pull the full source for the extensions (which are not available
236    # in the amalgamation).
237    # If updating the version, also update bazel/deps.bzl.
238    Dependency(
239        'buildtools/sqlite.zip',
240        'https://storage.googleapis.com/perfetto/sqlite-amalgamation-3440200.zip',
241        '833be89b53b3be8b40a2e3d5fedb635080e3edb204957244f3d6987c2bb2345f',
242        'all', 'all'),
243    Dependency(
244        'buildtools/sqlite_src',
245        'https://chromium.googlesource.com/external/github.com/sqlite/sqlite.git',
246        'c8f9803dc32bfee78a9ca2b1abbe39499729219b',  # refs/tags/version-3.44.2.
247        'all',
248        'all'),
249
250    # JsonCpp for legacy json import. Used only by the trace processor in
251    # standalone builds.
252    # If updating the version, also update bazel/deps.bzl.
253    Dependency(
254        'buildtools/jsoncpp',
255        'https://chromium.googlesource.com/external/github.com/open-source-parsers/jsoncpp.git',
256        '6aba23f4a8628d599a9ef7fa4811c4ff6e4070e2',  # refs/tags/1.9.3.
257        'all',
258        'all'),
259
260    # Libexpat for Instruments XML import.
261    # If updating the version, also update bazel/deps.bzl.
262    Dependency(
263        'buildtools/expat/src',
264        'https://chromium.googlesource.com/external/github.com/libexpat/libexpat.git',
265        'fa75b96546c069d17b8f80d91e0f4ef0cde3790d',  # refs/tags/upstream/R_2_6_2.
266        'all',
267        'all'),
268
269    # Archive with only the demangling sources from llvm-project.
270    # See tools/repackage_llvm_demangler.sh on how to update this.
271    # File suffix is the git reference to the commit at which we rearchived the
272    # sources, as hosted on https://llvm.googlesource.com/llvm-project.
273    # If updating the version, also update bazel/deps.bzl.
274    Dependency(
275        'buildtools/llvm-project.tgz',
276        'https://storage.googleapis.com/perfetto/llvm-project-617a15a9eac96088ae5e9134248d8236e34b91b1.tgz',
277        '7e2541446a27f2a09a84520da7bc93cd71749ba0f17318f2d5291fbf45b97956',
278        'all', 'all'),
279
280    # These dependencies are for libunwindstack, which is used by src/profiling.
281    Dependency('buildtools/android-core',
282               'https://android.googlesource.com/platform/system/core.git',
283               '9e6cef7f07d8c11b3ea820938aeb7ff2e9dbaa52', 'all', 'all'),
284    Dependency(
285        'buildtools/android-unwinding',
286        'https://android.googlesource.com/platform/system/unwinding.git',
287        '4b59ea8471e89d01300481a92de3230b79b6d7c7', 'all', 'all'),
288    Dependency('buildtools/android-logging',
289               'https://android.googlesource.com/platform/system/logging.git',
290               '7b36b566c9113fc703d68f76e8f40c0c2432481c', 'all', 'all'),
291    Dependency('buildtools/android-libbase',
292               'https://android.googlesource.com/platform/system/libbase.git',
293               '78f1c2f83e625bdf66d55b48bdb3a301c20d2fb3', 'all', 'all'),
294    Dependency(
295        'buildtools/android-libprocinfo',
296        'https://android.googlesource.com/platform/system/libprocinfo.git',
297        'fd214c13ededecae97a3b15b5fccc8925a749a84', 'all', 'all'),
298    Dependency('buildtools/lzma',
299               'https://android.googlesource.com/platform/external/lzma.git',
300               '7851dce6f4ca17f5caa1c93a4e0a45686b1d56c3', 'all', 'all'),
301    Dependency('buildtools/zstd',
302               'https://android.googlesource.com/platform/external/zstd.git',
303               '77211fcc5e08c781734a386402ada93d0d18d093', 'all', 'all'),
304    Dependency('buildtools/bionic',
305               'https://android.googlesource.com/platform/bionic.git',
306               'a0d0355105cb9d4a4b5384897448676133d7b8e2', 'all', 'all'),
307
308    # Zlib used both in the tracing binaries, as well as the trace processor and
309    # assorted tools.
310    # If updating the version, also update bazel/deps.bzl.
311    Dependency('buildtools/zlib',
312               'https://android.googlesource.com/platform/external/zlib.git',
313               '6d3f6aa0f87c9791ca7724c279ef61384f331dfd', 'all', 'all'),
314
315    # Linenoise, used only by trace_processor in standalone builds.
316    # If updating the version, also update bazel/deps.bzl.
317    Dependency('buildtools/linenoise',
318               'https://fuchsia.googlesource.com/third_party/linenoise.git',
319               'c894b9e59f02203dbe4e2be657572cf88c4230c3', 'all', 'all'),
320
321    # Bloaty, used to investigate binary size
322    Dependency(
323        'buildtools/bloaty.zip',
324        'https://storage.googleapis.com/perfetto/bloaty-1.1-b3b829de35babc2fe831b9488ad2e50bca939412-mac.zip',
325        '2d301bd72a20e3f42888c9274ceb4dca76c103608053572322412c2c65ab8cb8',
326        'darwin', 'all'),
327
328    Dependency('buildtools/open_csd',
329            'https://android.googlesource.com/platform/external/OpenCSD.git',
330            '0ce01e934f95efb6a216a6efa35af1245151c779', 'all', 'all'),
331
332    # sqlglot for SQL formatting.
333    Dependency(
334        'buildtools/sqlglot',
335        'https://github.com/tobymao/sqlglot.git',
336        'dd1cdb0b91ac597a9cb1f1f517a616c264f5b654', 'all', 'all'),
337]
338
339# Dependencies required to build code on the host using Bazel build system.
340# Only macOS and Linux.
341BUILD_DEPS_BAZEL = [
342    Dependency(
343        'buildtools/mac/bazel',
344        'https://github.com/bazelbuild/bazel/releases/download/7.4.1/bazel-7.4.1-darwin-x86_64',
345        '52dd34c17cc97b3aa5bdfe3d45c4e3938226f23dd0bfb47beedd625a953f1f05',
346        'darwin', 'x64'),
347    Dependency(
348        'buildtools/mac/bazel',
349        'https://github.com/bazelbuild/bazel/releases/download/7.4.1/bazel-7.4.1-darwin-arm64',
350        '02b117b97d0921ae4d4f4e11d27e2c0930381df416e373435d5d0419c6a26f24',
351        'darwin', 'arm64'),
352    Dependency(
353        'buildtools/linux64/bazel',
354        'https://github.com/bazelbuild/bazel/releases/download/7.4.1/bazel-7.4.1-linux-x86_64',
355        'c97f02133adce63f0c28678ac1f21d65fa8255c80429b588aeeba8a1fac6202b',
356        'linux', 'x64'),
357    Dependency(
358        'buildtools/linux64/bazel',
359        'https://github.com/bazelbuild/bazel/releases/download/7.4.1/bazel-7.4.1-linux-arm64',
360        'd7aedc8565ed47b6231badb80b09f034e389c5f2b1c2ac2c55406f7c661d8b88',
361        'linux', 'arm64'),
362]
363
364# Dependencies required to build Android code.
365# URLs and SHA1s taken from:
366# - https://dl.google.com/android/repository/repository-11.xml
367# - https://dl.google.com/android/repository/sys-img/android/sys-img.xml
368# - https://dl.google.com/android/repository/repository2-2.xml
369BUILD_DEPS_ANDROID = [
370    # Android NDK
371    Dependency(
372        'buildtools/ndk.zip',
373        'https://dl.google.com/android/repository/android-ndk-r26c-darwin.zip',
374        '312756dfcbdbf389d35d651e17ca98683bd36cb83cc7bf7ad51cac5c06bd064b',
375        'darwin', 'all'),
376    Dependency(
377        'buildtools/ndk.zip',
378        'https://dl.google.com/android/repository/android-ndk-r26c-linux.zip',
379        '6d6e659834d28bb24ba7ae66148ad05115ebbad7dabed1af9b3265674774fcf6',
380        'linux', 'x64'),
381    # Android Java SDK.
382    Dependency(
383        'buildtools/android_sdk/platforms/android-35.zip',
384        'https://dl.google.com/android/repository/platform-35_r02.zip',
385        '0988cacad01b38a18a47bac14a0695f246bc76c1b06c0eeb8eb0dc825ab0c8e0',
386        'all', 'all'),
387    # Android build tools.
388    Dependency(
389        'buildtools/android_sdk/build-tools/35.0.1.zip',
390        'https://dl.google.com/android/repository/build-tools_r35.0.1_linux.zip',
391        '5993499f3229a021b89f87088c57242aeefaa62316bf3d69da7de40bfd5350f1',
392        'linux', 'all'),
393    Dependency(
394        'buildtools/android_sdk/build-tools/35.0.1.zip',
395        'https://dl.google.com/android/repository/build-tools_r35.0.1_macosx.zip',
396        'c01e4b763da96ae5ef67e8bdf2abc94fb6cb3e73a42209581feb6a7019a51b9c',
397        'darwin', 'all'),
398]
399
400# Dependencies required to run Android tests.
401TEST_DEPS_ANDROID = [
402    # Android emulator images.
403    Dependency(
404        'buildtools/aosp-arm.zip',
405        'https://storage.googleapis.com/perfetto/aosp-02022018-arm.zip',
406        'f5c7a3a22ad7aa0bd14ba467e8697e1e917d306699bd25622aa4419a413b9b67',
407        'all', 'all'),
408
409    # platform-tools.zip contains adb binaries.
410    Dependency(
411        'buildtools/android_sdk/platform-tools.zip',
412        'https://dl.google.com/android/repository/platform-tools_r26.0.0-darwin.zip',
413        '98d392cbd21ca20d643c7e1605760cc49075611e317c534096b5564053f4ac8e',
414        'darwin', 'all'),
415    Dependency(
416        'buildtools/android_sdk/platform-tools.zip',
417        'https://dl.google.com/android/repository/platform-tools_r26.0.0-linux.zip',
418        '90208207521d85abf0d46e3374aa4e04b7aff74e4f355c792ac334de7a77e50b',
419        'linux', 'x64'),
420
421    # Android emulator binaries.
422    Dependency(
423        'buildtools/emulator',
424        'https://android.googlesource.com/platform/prebuilts/android-emulator.git',
425        '4b260028dc27bc92c39bee9129cb2ba839970956', 'all', 'x64'),
426]
427
428# This variable is updated by tools/roll-catapult-trace-viewer.
429CATAPULT_SHA256 = 'b30108e05268ce6c65bb4126b65f6bfac165d17f5c1fd285046e7e6fd76c209f'
430
431TYPEFACES_SHA256 = '1065172aaf0e9c22bc4f206ed9fdf5f1b4355d233fb21f9f26a89cd66c941940'
432
433UI_DEPS = [
434    Dependency(
435        'buildtools/mac/nodejs.tgz',
436        'https://storage.googleapis.com/chromium-nodejs/20.11.0/5b5681e12a21cda986410f69e03e6220a21dd4d2',
437        'cecb99fbb369a9090dddc27e228b66335cd72555b44fa8839ef78e56c51682c5',
438        'darwin', 'arm64'),
439    Dependency(
440        'buildtools/mac/nodejs.tgz',
441        'https://storage.googleapis.com/chromium-nodejs/20.11.0/e3c0fd53caae857309815f3f8de7c2dce49d7bca',
442        '20affacca2480c368b75a1d91ec1a2720604b325207ef0cf39cfef3c235dad19',
443        'darwin', 'x64'),
444    Dependency(
445        'buildtools/linux64/nodejs.tgz',
446        'https://storage.googleapis.com/chromium-nodejs/20.11.0/f9a337cfa0e2b92d3e5c671c26b454bd8e99769e',
447        '0ba9cc91698c1f833a1fdc1fe0cb392d825ad484c71b0d84388ac80bfd3d6079',
448        'linux', 'x64'),
449    Dependency(
450        'buildtools/mac/emsdk.tgz',
451        'https://storage.googleapis.com/perfetto/emscripten-2.0.12-mac.tgz',
452        'aa125f8c8ff8a386d43e18c8ea0c98c875cc19160a899403e8967a5478f96f31',
453        'darwin', 'all'),
454    Dependency(
455        'buildtools/linux64/emsdk.tgz',
456        'https://storage.googleapis.com/perfetto/emscripten-2.0.12-linux.tgz',
457        'bfff9fb0326363c12e19b542f27a5f12cedbfc310f30621dc497c9af51d2d2e3',
458        'linux', 'x64'),
459    Dependency(
460        'buildtools/catapult_trace_viewer.tgz',
461        'https://storage.googleapis.com/perfetto/catapult_trace_viewer-%s.tar.gz'
462        % CATAPULT_SHA256, CATAPULT_SHA256, 'all', 'all'),
463    Dependency(
464        'buildtools/typefaces.tgz',
465        'https://storage.googleapis.com/perfetto/typefaces-%s.tar.gz' %
466        TYPEFACES_SHA256, TYPEFACES_SHA256, 'all', 'all'),
467
468    Dependency(
469        'third_party/pnpm/pnpm',
470        'https://storage.googleapis.com/perfetto/pnpm-linux-arm64-8.6.3',
471        'ac76e9ab6a770479f93c1a2bf978d72636dbcb02608554378cf30075a78a22ac',
472        'linux', 'arm64'),
473    Dependency(
474        'third_party/pnpm/pnpm',
475        'https://storage.googleapis.com/perfetto/pnpm-linux-x64-8.6.3',
476        '5a58ccd78d44faac138d901976a7a8917c0f2a2f83743cfdd895fcd0bb6aa135',
477        'linux', 'x64'),
478    Dependency(
479        'third_party/pnpm/pnpm',
480        'https://storage.googleapis.com/perfetto/pnpm-macos-arm64-8.6.3',
481        'f527713d3183e30cfbfd7fd6403ceed730831c53649e50c979961eab3b2cf866',
482        'darwin', 'arm64'),
483    Dependency(
484        'third_party/pnpm/pnpm',
485        'https://storage.googleapis.com/perfetto/pnpm-macos-x64-8.6.3',
486        '6b425f7f0342341e9ee9427a9a2be2c89936c4a04efe6125f7af667eb02b10c1',
487        'darwin', 'x64'),
488]
489
490# Dependencies to build gRPC.
491BIGTRACE_DEPS = [
492    Dependency(
493        'buildtools/grpc/src',
494        'https://chromium.googlesource.com/external/github.com/grpc/grpc.git',
495        '4795c5e69b25e8c767b498bea784da0ef8c96fd5', 'all', 'all', True),
496    Dependency(
497      'buildtools/cpp-httplib',
498      'https://github.com/yhirose/cpp-httplib.git',
499      '6c3e8482f7b4e3b307bb42afbb85fd8771da86b8',
500      'all', 'all', True
501    )
502]
503
504# Sysroots required to cross-compile Linux targets (linux-arm{,64}).
505# These are taken from Chromium's build/linux/sysroot_scripts/sysroots.json.
506BUILD_DEPS_LINUX_CROSS_SYSROOTS = [
507    Dependency(
508        'buildtools/debian_sid_arm-sysroot.tgz',
509        'https://commondatastorage.googleapis.com/chrome-linux-sysroot/toolchain/11d6f690ca49e8ba01a1d8c5346cedad2cf308fd/debian_sid_arm_sysroot.tar.xz',
510        'ff192fe073d140d836c9ca1e68f7200d558bb9aa6c5c8f4f76f794f82890f99a',
511        'linux', 'all'),
512    Dependency(
513        'buildtools/debian_sid_arm64-sysroot.tgz',
514        'https://commondatastorage.googleapis.com/chrome-linux-sysroot/toolchain/2befe8ce3e88be6080e4fb7e6d412278ea6a7625/debian_sid_arm64_sysroot.tar.xz',
515        'e4389eab2fe363f3fbdfa4d3ce9d94457d78fd2c0e62171a7534867623eadc90',
516        'linux', 'all'),
517]
518
519ALL_DEPS = (
520    BUILD_DEPS_HOST + BUILD_DEPS_BAZEL + BUILD_DEPS_ANDROID +
521    BUILD_DEPS_LINUX_CROSS_SYSROOTS + TEST_DEPS_ANDROID + UI_DEPS)
522
523ROOT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
524UI_DIR = os.path.join(ROOT_DIR, 'ui')
525TOOLS_DIR = os.path.join(ROOT_DIR, 'tools')
526NODE_MODULES_STATUS_FILE = os.path.join(UI_DIR, 'node_modules', '.last_install')
527TEST_DATA_SCRIPT = os.path.join(TOOLS_DIR, 'test_data')
528
529
530def CheckCallRetry(*args, **kwargs):
531  """ Like subprocess.check_call, with retries up to 5 times. """
532  MAX_ATTEMPTS = 5
533  for attempt in range(1, MAX_ATTEMPTS + 1):
534    try:
535      return subprocess.check_call(*args, **kwargs)
536    except subprocess.CalledProcessError as error:
537      if attempt == MAX_ATTEMPTS:
538        raise error
539      else:
540        logging.error(error)
541        time.sleep(attempt * 3)
542
543
544def DownloadURL(url, out_file):
545  CheckCallRetry(['curl', '-L', '-#', '-o', out_file, url])
546
547
548def GetArch():
549  arch = machine()
550  if arch == 'arm64':
551    return 'arm64'
552  elif arch == 'aarch64':
553    return 'arm64'
554  else:
555    # Assume everything else is x64 matching previous behaviour.
556    return 'x64'
557
558
559def ReadFile(path):
560  if not os.path.exists(path):
561    return None
562  with open(path) as f:
563    return f.read().strip()
564
565
566def MkdirRecursive(path):
567  # Works with both relative and absolute paths
568  cwd = '/' if path.startswith('/') else ROOT_DIR
569  for part in path.split('/'):
570    cwd = os.path.join(cwd, part)
571    if not os.path.exists(cwd):
572      os.makedirs(cwd)
573    else:
574      assert (os.path.isdir(cwd))
575
576
577def HashLocalFile(path):
578  if not os.path.exists(path):
579    return None
580  with open(path, 'rb') as f:
581    return hashlib.sha256(f.read()).hexdigest()
582
583
584def ExtractZipfilePreservePermissions(zf, info, path):
585  target_path = os.path.join(path, info.filename)
586  mode = info.external_attr >> 16
587  S_IFLNK  = 0o120000  # symbolic link
588  if (mode & S_IFLNK) == S_IFLNK:
589    dst = zf.read(info).decode()
590    os.symlink(dst, target_path)
591    return
592  zf.extract(info.filename, path=path)
593  min_acls = 0o755 if info.filename.endswith('/') else 0o644
594  os.chmod(target_path, mode | min_acls)
595
596
597def IsGitRepoCheckoutOutAtRevision(path, revision):
598  return ReadFile(os.path.join(path, '.git', 'HEAD')) == revision
599
600
601def RmtreeIfExists(path):
602  # Git creates read-only files on windows, which cause failures with rmtree.
603  # This seems the socially accepted way to deal with it.
604  # See https://bugs.python.org/issue19643 .
605  def del_read_only_for_windows(_action, name, _exc):
606    os.chmod(name, stat.S_IWRITE)
607    os.remove(name)
608
609  if not os.path.exists(path):
610    return
611  third_party_path = os.path.abspath(os.path.join(ROOT_DIR, 'third_party'))
612  buildtools_path = os.path.abspath(os.path.join(ROOT_DIR, 'buildtools'))
613  test_path = os.path.abspath(os.path.join(ROOT_DIR, 'test', 'data'))
614  if (not os.path.abspath(path).startswith(buildtools_path) and
615      not os.path.abspath(path).startswith(test_path) and
616      not os.path.abspath(path).startswith(third_party_path)):
617    # Safety check to prevent that some merge confilct ends up doing some
618    # rm -rf / or similar.
619    logging.fatal(
620        'Cannot remove %s: outside of {buildtools, test/data, third_party}',
621        path)
622    sys.exit(1)
623  logging.info('Removing %s' % path)
624  if os.path.isdir(path):
625    shutil.rmtree(path, onerror=del_read_only_for_windows)
626  else:
627    os.remove(path)
628
629
630def CheckoutGitRepo(path, git_url, revision, check_only):
631  if IsGitRepoCheckoutOutAtRevision(path, revision):
632    return False
633  if check_only:
634    return True
635  path = path.replace('/', os.sep)
636  RmtreeIfExists(path)
637  MkdirRecursive(path)
638  logging.info('Fetching %s @ %s into %s', git_url, revision, path)
639  subprocess.check_call(['git', 'init', path], cwd=path)
640  CheckCallRetry(['git', 'fetch', '--quiet', '--depth', '1', git_url, revision],
641                 cwd=path)
642  subprocess.check_call(['git', 'checkout', revision, '--quiet'], cwd=path)
643  CheckCallRetry(
644      ['git', 'submodule', 'update', '--init', '--recursive', '--quiet'],
645      cwd=path)
646  assert (IsGitRepoCheckoutOutAtRevision(path, revision))
647  return True
648
649
650def InstallNodeModules(force_clean=False):
651  if force_clean:
652    node_modules = os.path.join(UI_DIR, 'node_modules')
653    logging.info('Clearing %s', node_modules)
654    subprocess.check_call(['git', 'clean', '-qxffd', node_modules],
655                          cwd=ROOT_DIR)
656  logging.info("Running `pnpm install --shamefully-hoist --frozen-lockfile` in {0}".format(UI_DIR))
657
658  # Some node modules have postinstall scripts (already bad) but worse
659  # sometimes they are in the form: "postinstall: 'node ./scripts/foo'"
660  # so here we need to ensure that our hermetic node is available in
661  # PATH.
662  env = os.environ.copy()
663  env['PATH'] = TOOLS_DIR + ':' + env['PATH']
664
665  subprocess.check_call([
666    os.path.join(TOOLS_DIR, 'pnpm'),
667    'install',
668    '--shamefully-hoist',
669    '--frozen-lockfile'],
670    cwd=UI_DIR,
671    env=env)
672  # pbjs has the bad habit of installing extra packages on its first
673  # run. Run it here, so we avoid fetches while building.
674  pbjs = ['node_modules/.bin/pbjs', '/dev/null', '-o', '/dev/null']
675  subprocess.call(pbjs, cwd=UI_DIR, env=env)
676  with open(NODE_MODULES_STATUS_FILE, 'w') as f:
677    f.write(HashLocalFile(os.path.join(UI_DIR, 'pnpm-lock.yaml')))
678
679
680def CheckNodeModules():
681  """Returns True if the modules are up-to-date.
682
683  There doesn't seem to be an easy way to check node modules versions. Instead
684  just check if pnpm-lock.json changed since the last `pnpm install` call.
685  """
686  if not os.path.exists(NODE_MODULES_STATUS_FILE):
687    return False
688  with open(NODE_MODULES_STATUS_FILE, 'r') as f:
689    actual = f.read()
690  expected = HashLocalFile(os.path.join(UI_DIR, 'pnpm-lock.yaml'))
691  return expected == actual
692
693
694def CheckHashes():
695  for dep in ALL_DEPS:
696    if dep.source_url.endswith('.git'):
697      continue
698    logging.info('Downloading %s for %s-%s', dep.source_url, dep.target_os,
699                 dep.target_arch)
700    with tempfile.NamedTemporaryFile(delete=False) as f:
701      f.close()
702      DownloadURL(dep.source_url, f.name)
703      actual_checksum = HashLocalFile(f.name)
704      os.unlink(f.name)
705      if (actual_checksum != dep.checksum):
706        logging.fatal('SHA-256 mismatch for {} expected {} was {}'.format(
707            dep.source_url, dep.checksum, actual_checksum))
708
709
710def CheckDepotToolsIsRecent():
711  gn_py_path = shutil.which('gn.py')
712  if gn_py_path is None:
713    return True  # depot_tools doesn't seem to be installed in the PATH.
714  dt_dir = os.path.abspath(os.path.dirname(gn_py_path))
715  cmd = ['git', '-C', dt_dir, 'merge-base', '--is-ancestor', 'a0cf4321', 'HEAD']
716  git_ret = subprocess.call(cmd, stderr=subprocess.DEVNULL)
717  if git_ret == 0:
718    return True
719  print('\033[91mYour depot_tools revision is too old. Please run:\033[0m')
720  print('git -C %s fetch origin && git -C %s checkout -B main -t origin/main' %
721        (dt_dir, dt_dir))
722  return False
723
724
725def Main():
726  parser = argparse.ArgumentParser()
727  parser.add_argument(
728      '--bazel',
729      action='store_true',
730      help='Bazel build tool executable to build the project using Bazel')
731  parser.add_argument(
732      '--android',
733      action='store_true',
734      help='NDK and emulator images target_os="android"')
735  parser.add_argument(
736      '--linux-arm',
737      action='store_true',
738      help='Debian sysroots for target_os="linux" target_cpu="arm|arm64"')
739  parser.add_argument(
740      '--ui',
741      action='store_true',
742      help='Node and NPM packages to Build the Web-based UI via ./ui/build')
743  parser.add_argument(
744      '--grpc', action='store_true', help='Packages to build gRPC')
745  parser.add_argument('--check-only')
746  parser.add_argument('--filter', action='append')
747  parser.add_argument('--verify', help='Check all URLs', action='store_true')
748  parser.add_argument(
749      '--no-toolchain', help='Do not download toolchain', action='store_true')
750  parser.add_argument(
751      '--build-os',
752      default=system().lower(),
753      choices=['windows', 'darwin', 'linux'],
754      help='Override the autodetected build operating system')
755  parser.add_argument(
756      '--build-arch',
757      default=GetArch(),
758      choices=['arm64', 'x64'],
759      help='Override the autodetected build CPU architecture')
760  args = parser.parse_args()
761  if args.verify:
762    CheckHashes()
763    return 0
764
765  target_os = args.build_os
766  if args.ui and target_os == 'windows':
767    print('Building the UI on Windows is unsupported')
768    return 1
769
770  if not CheckDepotToolsIsRecent():
771    return 1
772
773  deps = BUILD_DEPS_HOST
774  if not args.no_toolchain:
775    deps += BUILD_DEPS_TOOLCHAIN_HOST
776  if args.bazel:
777    # We build Android Java SDK using Bazel, so we need Android build and test
778    # deps to do build ":all" targets and run tests.
779    deps += BUILD_DEPS_BAZEL + BUILD_DEPS_ANDROID + TEST_DEPS_ANDROID
780  if args.android:
781    deps += BUILD_DEPS_ANDROID + TEST_DEPS_ANDROID
782  if args.linux_arm:
783    deps += BUILD_DEPS_LINUX_CROSS_SYSROOTS
784  if args.ui:
785    deps += UI_DEPS
786  # TODO(b/360084012) Change the arg name to bigtrace
787  if args.grpc:
788    deps += BIGTRACE_DEPS
789  deps_updated = False
790  nodejs_updated = False
791
792  for old_dir in CLEANUP_OLD_DIRS:
793    RmtreeIfExists(os.path.join(ROOT_DIR, old_dir))
794
795  for dep in deps:
796    target_arch = args.build_arch
797    matches_os = dep.target_os == 'all' or target_os == dep.target_os
798    matches_arch = dep.target_arch == 'all' or target_arch == dep.target_arch
799    if not matches_os or not matches_arch:
800      continue
801    if args.filter and not any(f in dep.target_folder for f in args.filter):
802      continue
803    local_path = os.path.join(ROOT_DIR, dep.target_folder)
804    if dep.source_url.endswith('.git'):
805      deps_updated |= CheckoutGitRepo(local_path, dep.source_url, dep.checksum,
806                                      args.check_only)
807      continue
808    is_compressed = any([local_path.endswith(ext) for ext in ['.zip', '.tgz', '.tbz2']])
809    compressed_target_dir = os.path.splitext(local_path)[0] if is_compressed else None
810    compressed_dir_stamp = os.path.join(compressed_target_dir, '.stamp') if is_compressed else None
811
812    if ((not is_compressed and HashLocalFile(local_path) == dep.checksum) or
813        (is_compressed and ReadFile(compressed_dir_stamp) == dep.checksum)):
814      continue
815    deps_updated = True
816    if args.check_only:
817      continue
818    MkdirRecursive(os.path.dirname(dep.target_folder))
819    if HashLocalFile(local_path) != dep.checksum:
820      download_path = local_path + '.tmp'
821      logging.info('Downloading %s from %s', local_path, dep.source_url)
822      DownloadURL(dep.source_url, download_path)
823      os.chmod(download_path, 0o755)
824      actual_checksum = HashLocalFile(download_path)
825      if (actual_checksum != dep.checksum):
826        os.remove(download_path)
827        logging.fatal('SHA-256 mismatch for {} expected {} was {}'.format(
828            download_path, dep.checksum, actual_checksum))
829        return 1
830      shutil.move(download_path, local_path)
831      if 'nodejs' in dep.target_folder:
832        nodejs_updated = True
833
834    assert (HashLocalFile(local_path) == dep.checksum)
835
836    if is_compressed:
837      logging.info('Extracting %s into %s' % (local_path, compressed_target_dir))
838      assert (os.path.commonprefix((ROOT_DIR, compressed_target_dir)) == ROOT_DIR)
839      RmtreeIfExists(compressed_target_dir)
840
841      # Decompress the archive.
842      if local_path.endswith('.tgz'):
843        MkdirRecursive(compressed_target_dir)
844        subprocess.check_call(['tar', '-oxf', local_path], cwd=compressed_target_dir)
845      elif local_path.endswith('.zip'):
846        with zipfile.ZipFile(local_path, 'r') as zf:
847          for info in zf.infolist():
848            ExtractZipfilePreservePermissions(zf, info, compressed_target_dir)
849      elif local_path.endswith('.tbz2'):
850        tar_path = '{}.tar.tmp'.format(local_path)
851        with open(tar_path, 'w') as f:
852          with bz2.open(local_path, 'r') as bf:
853            f.write(bf.read())
854        MkdirRecursive(compressed_target_dir)
855        subprocess.check_call(['tar', '-oxf', tar_path], cwd=compressed_target_dir)
856
857      # If the zip contains one root folder, rebase one level up moving all
858      # its sub files and folders inside |target_dir|.
859      subdir = os.listdir(compressed_target_dir)
860      if len(subdir) == 1:
861        subdir = os.path.join(compressed_target_dir, subdir[0])
862        if os.path.isdir(subdir):
863          for subf in os.listdir(subdir):
864            shutil.move(os.path.join(subdir, subf), compressed_target_dir)
865          os.rmdir(subdir)
866
867      # Create stamp and remove the archive.
868      with open(compressed_dir_stamp, 'w') as stamp_file:
869        stamp_file.write(dep.checksum)
870      os.remove(local_path)
871
872  if args.ui:
873    # Needs to happen after nodejs is installed above.
874    if args.check_only:
875      deps_updated |= not CheckNodeModules()
876    else:
877      InstallNodeModules(force_clean=nodejs_updated)
878
879  cur_python_interpreter = sys.executable
880  test_data_synced = 0 == subprocess.call([
881      cur_python_interpreter, TEST_DATA_SCRIPT, 'status', '--quiet',
882      '--ignore-new'
883  ])
884  if args.check_only:
885    if not deps_updated and test_data_synced:
886      with open(args.check_only, 'w') as f:
887        f.write('OK')  # The content is irrelevant, just keep GN happy.
888      return 0
889    argz = ' '.join(
890        [x for x in sys.argv[1:] if not x.startswith('--check-only')])
891    print('\033[91mBuild deps are stale. ' +
892          'Please run tools/install-build-deps %s\033[0m' % argz)
893    if not test_data_synced:
894      print('//test/data/ is out of sync. `tools/test_data status` for details')
895    return 1
896
897  if not test_data_synced:
898    cmd = [cur_python_interpreter, TEST_DATA_SCRIPT, 'download', '--overwrite']
899    if not sys.stdout.isatty():
900      cmd += ['--verbose']  # For CI bots
901    subprocess.check_call(cmd)
902
903  if deps_updated:
904    # Stale binary files may be compiled against old sysroot headers that aren't
905    # tracked by gn.
906    logging.warning('Remember to run "gn clean <output_directory>" ' +
907                    'to avoid stale binary files.')
908
909
910if __name__ == '__main__':
911  logging.basicConfig(level=logging.INFO)
912  sys.exit(Main())
913