• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/env python3
2# Copyright (C) 2017 The Android Open Source Project
3#
4# Licensed under the Apache License, Version 2.0 (the "License");
5# you may not use this file except in compliance with the License.
6# You may obtain a copy of the License at
7#
8#      http://www.apache.org/licenses/LICENSE-2.0
9#
10# Unless required by applicable law or agreed to in writing, software
11# distributed under the License is distributed on an "AS IS" BASIS,
12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13# See the License for the specific language governing permissions and
14# limitations under the License.
15
16import argparse
17import dataclasses as dc
18import hashlib
19import logging
20import os
21import shutil
22import subprocess
23import stat
24import sys
25import tempfile
26import time
27import zipfile
28import bz2
29
30from collections import namedtuple
31from platform import system, machine
32
33
34# The format for the deps below is the following:
35# (target_folder, source_url, sha1, target_os, target_arch)
36# |source_url| can be either a git repo or a http url.
37# If a git repo, |checksum| is the SHA1 committish that will be checked out.
38# If a http url, |checksum| is the SHA256 of the downloaded file.
39# If the url is a .zip, .tgz, or .tbz2 file it will be automatically deflated under
40# |target_folder|, taking care of stripping the root folder if it's a single
41# root (to avoid ending up with buildtools/protobuf/protobuf-1.2.3/... and have
42# instead just buildtools/protobuf).
43# |target_os| is either 'darwin', 'linux', 'windows' or 'all'
44# |target_arch| is either 'x64', 'arm64' or 'all'
45# in both cases the dep only applies on matching platforms
46# |target_arch| can be 'all' when 'target_os' is not 'all' for example in the
47# case of MacOS universal binaries.
48@dc.dataclass
49class Dependency:
50  target_folder: str
51  source_url: str
52  checksum: str
53  target_os: str
54  target_arch: str
55  submodules: bool = False
56
57
58# This is to remove old directories when build tools get {re,}moved. This is to
59# avoid accidentally referring to stale dir in custom user scripts.
60CLEANUP_OLD_DIRS = [
61    'buildtools/nodejs',  # Moved to buildtools/{mac,linux64}/nodejs
62    'buildtools/emsdk',  # Moved to buildtools/{mac,linux64}/emsdk
63    'buildtools/test_data',  # Moved to test/data by r.android.com/1539381 .
64    'buildtools/d8',  # Removed by r.android.com/1424334 .
65
66    # Build tools moved to third_party/ by r.android.com/2327602 .
67    'buildtools/mac/clang-format',
68    'buildtools/mac/gn',
69    'buildtools/mac/ninja',
70    'buildtools/linux64/clang-format',
71    'buildtools/linux64/gn',
72    'buildtools/linux64/ninja',
73    'buildtools/win/clang-format.exe',
74    'buildtools/win/gn.exe',
75    'buildtools/win/ninja.exe',
76]
77
78# Dependencies required to build code on the host or when targeting desktop OS.
79BUILD_DEPS_TOOLCHAIN_HOST = [
80    # GN. From https://chrome-infra-packages.appspot.com/dl/gn/gn/.
81    # git_revision:0725d7827575b239594fbc8fd5192873a1d62f44 .
82    Dependency(
83        'third_party/gn/gn',
84        'https://storage.googleapis.com/perfetto/gn-mac-1968-0725d782',
85        '9ced623a664560bba38bbadb9b91158ca4186358c847e17ab7d982b351373c2e',
86        'darwin', 'x64'),
87    Dependency(
88        'third_party/gn/gn',
89        'https://storage.googleapis.com/perfetto/gn-mac-arm64-1968-0725d782',
90        'd22336b5210b4dad5e36e8c28ce81187f491822cf4d8fd0a257b30d6bee3fd3f',
91        'darwin', 'arm64'),
92    Dependency(
93        'third_party/gn/gn',
94        'https://storage.googleapis.com/perfetto/gn-linux64-1968-0725d782',
95        'f706aaa0676e3e22f5fc9ca482295d7caee8535d1869f99efa2358177b64f5cd',
96        'linux', 'x64'),
97    Dependency(
98        'third_party/gn/gn',
99        'https://storage.googleapis.com/perfetto/gn-linux-arm64-1968-0725d782',
100        'c2a372cd4f911028d8bc351fbf24835c9b1194fcc92beadf6c5a2b3addae973c',
101        'linux', 'arm64'),
102    Dependency(
103        'third_party/gn/gn.exe',
104        'https://storage.googleapis.com/perfetto/gn-win-1968-0725d782',
105        '001f777f023c7a6959c778fb3a6b6cfc63f6baef953410ecdeaec350fb12285b',
106        'windows', 'x64'),
107
108    # clang-format
109    # From
110    # https://chromium.googlesource.com/chromium/src/buildtools/+/refs/heads/master/mac/clang-format.arm64.sha1
111    Dependency(
112        'third_party/clang-format/clang-format',
113        'https://storage.googleapis.com/chromium-clang-format/5553d7a3d912b7d49381ad68c9a56740601a57a0',
114        'e077990b2ea6807f6abc71b4cf1e487719d7e40574baddd2b51187fdcc8db803',
115        'darwin', 'arm64'),
116    # From
117    # https://chromium.googlesource.com/chromium/src/buildtools/+/refs/heads/master/mac/clang-format.x64.sha1
118    Dependency(
119        'third_party/clang-format/clang-format',
120        'https://storage.googleapis.com/chromium-clang-format/87d69aeff220c916b85b5d6d162fa5668aa9d64f',
121        '71179a8788a009cfd589636d50f0eb9f95f58b0cfda4351430bed7c0a48c080b',
122        'darwin', 'x64'),
123    # From https://chromium.googlesource.com/chromium/src/buildtools/+/refs/heads/master/linux64/clang-format.sha1
124    Dependency(
125        'third_party/clang-format/clang-format',
126        'https://storage.googleapis.com/chromium-clang-format/1facab3101fc6da6b9467543f27f0622b966bc19',
127        '5e459118d8ac825452e9e1f2717e4de5a36399dc6cc6aec7ec483ad27a0c927e',
128        'linux', 'x64'),
129    # From https://chromium.googlesource.com/chromium/src/buildtools/+/refs/heads/master/win/clang-format.exe.sha1
130    Dependency(
131        'third_party/clang-format/clang-format.exe',
132        'https://storage.googleapis.com/chromium-clang-format/2e569921b9884daf157021d6ae9e21991cd6cf81',
133        '2227376ada89ea832995b832222b722a27c4d5d8d59e9c4d7842877f99a1e30d',
134        'windows', 'x64'),
135
136    # Keep the SHA1 in sync with |clang_format_rev| in chromium //buildtools/DEPS.
137    Dependency(
138        'buildtools/clang_format/script',
139        'https://chromium.googlesource.com/external/github.com/llvm/llvm-project/clang/tools/clang-format.git',
140        'f97059df7f8b205064625cdb5f97b56668a125ef', 'all', 'all'),
141
142    # Ninja
143    Dependency(
144        'third_party/ninja/ninja',
145        'https://storage.googleapis.com/perfetto/ninja-mac-x64_and_arm64-182',
146        '36e8b7aaa06911e1334feb664dd731a1cd69a15eb916a231a3d10ff65fca2c73',
147        'darwin', 'all'),
148    Dependency(
149        'third_party/ninja/ninja',
150        'https://storage.googleapis.com/perfetto/ninja-linux64-182',
151        '54ac6a01362190aaabf4cf276f9c8982cdf11b225438940fdde3339be0f2ecdc',
152        'linux', 'x64'),
153    Dependency(
154        'third_party/ninja/ninja.exe',
155        'https://storage.googleapis.com/perfetto/ninja-win-182',
156        '09ced0fcd1a4dec7d1b798a2cf9ce5d20e5d2fbc2337343827f192ce47d0f491',
157        'windows', 'x64'),
158    Dependency(
159        'third_party/ninja/ninja',
160        'https://storage.googleapis.com/perfetto/ninja-linux-arm64-1111',
161        '05031a734ec4310a51b2cfe9f0096b26fce25ab4ff19e5b51abe6371de066cc5',
162        'linux', 'arm64'),
163
164    # Keep the revision in sync with Chrome's PACKAGE_VERSION in
165    # tools/clang/scripts/update.py.
166    Dependency(
167        'buildtools/linux64/clang.tgz',
168        'https://commondatastorage.googleapis.com/chromium-browser-clang/Linux_x64/clang-llvmorg-19-init-2941-ga0b3dbaf-22.tgz',
169        '6741cc1083f935795330b6e04617ac891a7b5d2b5647b664c5b0fccc354adb43',
170        'linux', 'x64'),
171    Dependency(
172        'buildtools/win/clang.tgz',
173        'https://commondatastorage.googleapis.com/chromium-browser-clang/Win/clang-llvmorg-19-init-2941-ga0b3dbaf-22.tgz',
174        'f627080ed53d4c156f089323e04fa3690c8bb459110b62cd1952b0e1f0755987',
175        'windows', 'x64'),
176]
177
178BUILD_DEPS_HOST = [
179    # Keep in sync with Android's //external/googletest/METADATA.
180    Dependency(
181        'buildtools/googletest',
182        'https://android.googlesource.com/platform/external/googletest.git',
183        '609281088cfefc76f9d0ce82e1ff6c30cc3591e5', 'all', 'all'),
184
185    # Keep in sync with Chromium's //third_party/protobuf.
186    Dependency(
187        'buildtools/protobuf',
188        # If you revert the below version back to an earlier version of
189        # protobuf, make sure to revert the changes to
190        # //gn/standalone/protoc.py as well.
191        #
192        # This comment can be removed with protobuf is next upreved.
193        'https://chromium.googlesource.com/external/github.com/protocolbuffers/protobuf.git',
194        'f0dc78d7e6e331b8c6bb2d5283e06aa26883ca7c',  # refs/tags/v21.12
195        'all',
196        'all'),
197
198    # libc++, libc++abi and libunwind for Linux where we need to rebuild the C++
199    # lib from sources. Keep the SHA1s in sync with Chrome's src/buildtools/DEPS.
200    Dependency(
201        'buildtools/libcxx',
202        'https://chromium.googlesource.com/external/github.com/llvm/llvm-project/libcxx.git',
203        '852bc6746f45add53fec19f3a29280e69e358d44', 'all', 'all'),
204    Dependency(
205        'buildtools/libcxxabi',
206        'https://chromium.googlesource.com/external/github.com/llvm/llvm-project/libcxxabi.git',
207        'a37a3aa431f132b02a58656f13984d51098330a2', 'all', 'all'),
208    Dependency(
209        'buildtools/libunwind',
210        'https://chromium.googlesource.com/external/github.com/llvm/llvm-project/libunwind.git',
211        '419b03c0b8f20d6da9ddcb0d661a94a97cdd7dad', 'all', 'all'),
212
213    # Keep in sync with chromium DEPS.
214    Dependency(
215        'buildtools/libfuzzer',
216        'https://chromium.googlesource.com/chromium/llvm-project/compiler-rt/lib/fuzzer.git',
217        'debe7d2d1982e540fbd6bd78604bf001753f9e74', 'linux', 'all'),
218
219    # Benchmarking tool.
220    Dependency(
221        'buildtools/benchmark',
222        'https://chromium.googlesource.com/external/github.com/google/benchmark.git',
223        'e991355c02b93fe17713efe04cbc2e278e00fdbd', 'all', 'all'),
224
225    # Libbacktrace, for stacktraces in Linux/Android debug builds.
226    # From https://github.com/ianlancetaylor/libbacktrace/archive/177940370e4a6b2509e92a0aaa9749184e64af43.zip
227    Dependency(
228        'buildtools/libbacktrace.zip',
229        'https://storage.googleapis.com/perfetto/libbacktrace-14818b7783eeb9a56c3f0fca78cefd3143f8c5f6.zip',
230        '0d09295938155aa84d9a6049f63df8cd2def3a28302b3550ea3ead9100b3d086',
231        'all', 'all'),
232
233    # Sqlite for the trace processing library.
234    # This is the amalgamated source whose compiled output is meant to be faster.
235    # We still pull the full source for the extensions (which are not available
236    # in the amalgamation).
237    # If updating the version, also update bazel/deps.bzl.
238    Dependency(
239        'buildtools/sqlite.zip',
240        'https://storage.googleapis.com/perfetto/sqlite-amalgamation-3440200.zip',
241        '833be89b53b3be8b40a2e3d5fedb635080e3edb204957244f3d6987c2bb2345f',
242        'all', 'all'),
243    Dependency(
244        'buildtools/sqlite_src',
245        'https://chromium.googlesource.com/external/github.com/sqlite/sqlite.git',
246        'c8f9803dc32bfee78a9ca2b1abbe39499729219b',  # refs/tags/version-3.44.2.
247        'all',
248        'all'),
249
250    # JsonCpp for legacy json import. Used only by the trace processor in
251    # standalone builds.
252    # If updating the version, also update bazel/deps.bzl.
253    Dependency(
254        'buildtools/jsoncpp',
255        'https://chromium.googlesource.com/external/github.com/open-source-parsers/jsoncpp.git',
256        '6aba23f4a8628d599a9ef7fa4811c4ff6e4070e2',  # refs/tags/1.9.3.
257        'all',
258        'all'),
259
260    # Archive with only the demangling sources from llvm-project.
261    # See tools/repackage_llvm_demangler.sh on how to update this.
262    # File suffix is the git reference to the commit at which we rearchived the
263    # sources, as hosted on https://llvm.googlesource.com/llvm-project.
264    # If updating the version, also update bazel/deps.bzl.
265    Dependency(
266        'buildtools/llvm-project.tgz',
267        'https://storage.googleapis.com/perfetto/llvm-project-617a15a9eac96088ae5e9134248d8236e34b91b1.tgz',
268        '7e2541446a27f2a09a84520da7bc93cd71749ba0f17318f2d5291fbf45b97956',
269        'all', 'all'),
270
271    # These dependencies are for libunwindstack, which is used by src/profiling.
272    Dependency('buildtools/android-core',
273               'https://android.googlesource.com/platform/system/core.git',
274               '9e6cef7f07d8c11b3ea820938aeb7ff2e9dbaa52', 'all', 'all'),
275    Dependency(
276        'buildtools/android-unwinding',
277        'https://android.googlesource.com/platform/system/unwinding.git',
278        '4b59ea8471e89d01300481a92de3230b79b6d7c7', 'all', 'all'),
279    Dependency('buildtools/android-logging',
280               'https://android.googlesource.com/platform/system/logging.git',
281               '7b36b566c9113fc703d68f76e8f40c0c2432481c', 'all', 'all'),
282    Dependency('buildtools/android-libbase',
283               'https://android.googlesource.com/platform/system/libbase.git',
284               '78f1c2f83e625bdf66d55b48bdb3a301c20d2fb3', 'all', 'all'),
285    Dependency(
286        'buildtools/android-libprocinfo',
287        'https://android.googlesource.com/platform/system/libprocinfo.git',
288        'fd214c13ededecae97a3b15b5fccc8925a749a84', 'all', 'all'),
289    Dependency('buildtools/lzma',
290               'https://android.googlesource.com/platform/external/lzma.git',
291               '7851dce6f4ca17f5caa1c93a4e0a45686b1d56c3', 'all', 'all'),
292    Dependency('buildtools/zstd',
293               'https://android.googlesource.com/platform/external/zstd.git',
294               '77211fcc5e08c781734a386402ada93d0d18d093', 'all', 'all'),
295    Dependency('buildtools/bionic',
296               'https://android.googlesource.com/platform/bionic.git',
297               'a0d0355105cb9d4a4b5384897448676133d7b8e2', 'all', 'all'),
298
299    # Zlib used both in the tracing binaries, as well as the trace processor and
300    # assorted tools.
301    # If updating the version, also update bazel/deps.bzl.
302    Dependency('buildtools/zlib',
303               'https://android.googlesource.com/platform/external/zlib.git',
304               '6d3f6aa0f87c9791ca7724c279ef61384f331dfd', 'all', 'all'),
305
306    # Linenoise, used only by trace_processor in standalone builds.
307    # If updating the version, also update bazel/deps.bzl.
308    Dependency('buildtools/linenoise',
309               'https://fuchsia.googlesource.com/third_party/linenoise.git',
310               'c894b9e59f02203dbe4e2be657572cf88c4230c3', 'all', 'all'),
311
312    # Bloaty, used to investigate binary size
313    Dependency(
314        'buildtools/bloaty.zip',
315        'https://storage.googleapis.com/perfetto/bloaty-1.1-b3b829de35babc2fe831b9488ad2e50bca939412-mac.zip',
316        '2d301bd72a20e3f42888c9274ceb4dca76c103608053572322412c2c65ab8cb8',
317        'darwin', 'all'),
318]
319
320# Dependencies required to build Android code.
321# URLs and SHA1s taken from:
322# - https://dl.google.com/android/repository/repository-11.xml
323# - https://dl.google.com/android/repository/sys-img/android/sys-img.xml
324BUILD_DEPS_ANDROID = [
325    # Android NDK
326    Dependency(
327        'buildtools/ndk.zip',
328        'https://dl.google.com/android/repository/android-ndk-r26c-darwin.zip',
329        '312756dfcbdbf389d35d651e17ca98683bd36cb83cc7bf7ad51cac5c06bd064b',
330        'darwin', 'all'),
331    Dependency(
332        'buildtools/ndk.zip',
333        'https://dl.google.com/android/repository/android-ndk-r26c-linux.zip',
334        '6d6e659834d28bb24ba7ae66148ad05115ebbad7dabed1af9b3265674774fcf6',
335        'linux', 'x64'),
336]
337
338# Dependencies required to run Android tests.
339TEST_DEPS_ANDROID = [
340    # Android emulator images.
341    Dependency(
342        'buildtools/aosp-arm.zip',
343        'https://storage.googleapis.com/perfetto/aosp-02022018-arm.zip',
344        'f5c7a3a22ad7aa0bd14ba467e8697e1e917d306699bd25622aa4419a413b9b67',
345        'all', 'all'),
346
347    # platform-tools.zip contains adb binaries.
348    Dependency(
349        'buildtools/android_sdk/platform-tools.zip',
350        'https://dl.google.com/android/repository/platform-tools_r26.0.0-darwin.zip',
351        '98d392cbd21ca20d643c7e1605760cc49075611e317c534096b5564053f4ac8e',
352        'darwin', 'all'),
353    Dependency(
354        'buildtools/android_sdk/platform-tools.zip',
355        'https://dl.google.com/android/repository/platform-tools_r26.0.0-linux.zip',
356        '90208207521d85abf0d46e3374aa4e04b7aff74e4f355c792ac334de7a77e50b',
357        'linux', 'x64'),
358
359    # Android emulator binaries.
360    Dependency(
361        'buildtools/emulator',
362        'https://android.googlesource.com/platform/prebuilts/android-emulator.git',
363        '4b260028dc27bc92c39bee9129cb2ba839970956', 'all', 'x64'),
364]
365
366# This variable is updated by tools/roll-catapult-trace-viewer.
367CATAPULT_SHA256 = 'b30108e05268ce6c65bb4126b65f6bfac165d17f5c1fd285046e7e6fd76c209f'
368
369TYPEFACES_SHA256 = '1065172aaf0e9c22bc4f206ed9fdf5f1b4355d233fb21f9f26a89cd66c941940'
370
371UI_DEPS = [
372    Dependency(
373        'buildtools/mac/nodejs.tgz',
374        'https://storage.googleapis.com/chromium-nodejs/16.13.0/31859fc1fa0994a95f44f09c367d6ff63607cfde',
375        'd9cf1f36b16e08ce52cfbdef427c6596eed2bd2f3608a79112d25b4ec8f75753',
376        'darwin', 'arm64'),
377    Dependency(
378        'buildtools/mac/nodejs.tgz',
379        'https://storage.googleapis.com/chromium-nodejs/16.13.0/16dfd094763b71988933a31735f9dea966f9abd6',
380        '065ced7e93f0e26276cb74d9688706674323865c4a8fc89e4adfa6868d4ebb4d',
381        'darwin', 'x64'),
382    Dependency(
383        'buildtools/linux64/nodejs.tgz',
384        'https://storage.googleapis.com/chromium-nodejs/16.13.0/ab9544e24e752d3d17f335fb7b2055062e582d11',
385        '3b5ca150a55f3aadfa18f3a10a4495aaf9653614ba3e460647170fef5287ec4f',
386        'linux', 'x64'),
387    Dependency(
388        'buildtools/mac/emsdk.tgz',
389        'https://storage.googleapis.com/perfetto/emscripten-2.0.12-mac.tgz',
390        'aa125f8c8ff8a386d43e18c8ea0c98c875cc19160a899403e8967a5478f96f31',
391        'darwin', 'all'),
392    Dependency(
393        'buildtools/linux64/emsdk.tgz',
394        'https://storage.googleapis.com/perfetto/emscripten-2.0.12-linux.tgz',
395        'bfff9fb0326363c12e19b542f27a5f12cedbfc310f30621dc497c9af51d2d2e3',
396        'linux', 'x64'),
397    Dependency(
398        'buildtools/catapult_trace_viewer.tgz',
399        'https://storage.googleapis.com/perfetto/catapult_trace_viewer-%s.tar.gz'
400        % CATAPULT_SHA256, CATAPULT_SHA256, 'all', 'all'),
401    Dependency(
402        'buildtools/typefaces.tgz',
403        'https://storage.googleapis.com/perfetto/typefaces-%s.tar.gz' %
404        TYPEFACES_SHA256, TYPEFACES_SHA256, 'all', 'all'),
405
406    Dependency(
407        'third_party/pnpm/pnpm',
408        'https://storage.googleapis.com/perfetto/pnpm-linux-arm64-8.6.3',
409        'ac76e9ab6a770479f93c1a2bf978d72636dbcb02608554378cf30075a78a22ac',
410        'linux', 'arm64'),
411    Dependency(
412        'third_party/pnpm/pnpm',
413        'https://storage.googleapis.com/perfetto/pnpm-linux-x64-8.6.3',
414        '5a58ccd78d44faac138d901976a7a8917c0f2a2f83743cfdd895fcd0bb6aa135',
415        'linux', 'x64'),
416    Dependency(
417        'third_party/pnpm/pnpm',
418        'https://storage.googleapis.com/perfetto/pnpm-macos-arm64-8.6.3',
419        'f527713d3183e30cfbfd7fd6403ceed730831c53649e50c979961eab3b2cf866',
420        'darwin', 'arm64'),
421    Dependency(
422        'third_party/pnpm/pnpm',
423        'https://storage.googleapis.com/perfetto/pnpm-macos-x64-8.6.3',
424        '6b425f7f0342341e9ee9427a9a2be2c89936c4a04efe6125f7af667eb02b10c1',
425        'darwin', 'x64'),
426]
427
428# Dependencies to build gRPC.
429GRPC_DEPS = [
430    Dependency(
431        'buildtools/grpc/src',
432        'https://chromium.googlesource.com/external/github.com/grpc/grpc.git',
433        '4795c5e69b25e8c767b498bea784da0ef8c96fd5', 'all', 'all', True),
434]
435
436# Sysroots required to cross-compile Linux targets (linux-arm{,64}).
437# These are taken from Chromium's build/linux/sysroot_scripts/sysroots.json.
438BUILD_DEPS_LINUX_CROSS_SYSROOTS = [
439    Dependency(
440        'buildtools/debian_sid_arm-sysroot.tgz',
441        'https://commondatastorage.googleapis.com/chrome-linux-sysroot/toolchain/11d6f690ca49e8ba01a1d8c5346cedad2cf308fd/debian_sid_arm_sysroot.tar.xz',
442        'ff192fe073d140d836c9ca1e68f7200d558bb9aa6c5c8f4f76f794f82890f99a',
443        'linux', 'all'),
444    Dependency(
445        'buildtools/debian_sid_arm64-sysroot.tgz',
446        'https://commondatastorage.googleapis.com/chrome-linux-sysroot/toolchain/2befe8ce3e88be6080e4fb7e6d412278ea6a7625/debian_sid_arm64_sysroot.tar.xz',
447        'e4389eab2fe363f3fbdfa4d3ce9d94457d78fd2c0e62171a7534867623eadc90',
448        'linux', 'all'),
449]
450
451ALL_DEPS = (
452    BUILD_DEPS_HOST + BUILD_DEPS_ANDROID + BUILD_DEPS_LINUX_CROSS_SYSROOTS +
453    TEST_DEPS_ANDROID + UI_DEPS)
454
455ROOT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
456UI_DIR = os.path.join(ROOT_DIR, 'ui')
457TOOLS_DIR = os.path.join(ROOT_DIR, 'tools')
458NODE_MODULES_STATUS_FILE = os.path.join(UI_DIR, 'node_modules', '.last_install')
459TEST_DATA_SCRIPT = os.path.join(TOOLS_DIR, 'test_data')
460
461
462def CheckCallRetry(*args, **kwargs):
463  """ Like subprocess.check_call, with retries up to 5 times. """
464  MAX_ATTEMPTS = 5
465  for attempt in range(1, MAX_ATTEMPTS + 1):
466    try:
467      return subprocess.check_call(*args, **kwargs)
468    except subprocess.CalledProcessError as error:
469      if attempt == MAX_ATTEMPTS:
470        raise error
471      else:
472        logging.error(error)
473        time.sleep(attempt * 3)
474
475
476def DownloadURL(url, out_file):
477  CheckCallRetry(['curl', '-L', '-#', '-o', out_file, url])
478
479
480def GetArch():
481  arch = machine()
482  if arch == 'arm64':
483    return 'arm64'
484  elif arch == 'aarch64':
485    return 'arm64'
486  else:
487    # Assume everything else is x64 matching previous behaviour.
488    return 'x64'
489
490
491def ReadFile(path):
492  if not os.path.exists(path):
493    return None
494  with open(path) as f:
495    return f.read().strip()
496
497
498def MkdirRecursive(path):
499  # Works with both relative and absolute paths
500  cwd = '/' if path.startswith('/') else ROOT_DIR
501  for part in path.split('/'):
502    cwd = os.path.join(cwd, part)
503    if not os.path.exists(cwd):
504      os.makedirs(cwd)
505    else:
506      assert (os.path.isdir(cwd))
507
508
509def HashLocalFile(path):
510  if not os.path.exists(path):
511    return None
512  with open(path, 'rb') as f:
513    return hashlib.sha256(f.read()).hexdigest()
514
515
516def ExtractZipfilePreservePermissions(zf, info, path):
517  target_path = os.path.join(path, info.filename)
518  mode = info.external_attr >> 16
519  S_IFLNK  = 0o120000  # symbolic link
520  if (mode & S_IFLNK) == S_IFLNK:
521    dst = zf.read(info).decode()
522    os.symlink(dst, target_path)
523    return
524  zf.extract(info.filename, path=path)
525  min_acls = 0o755 if info.filename.endswith('/') else 0o644
526  os.chmod(target_path, mode | min_acls)
527
528
529def IsGitRepoCheckoutOutAtRevision(path, revision):
530  return ReadFile(os.path.join(path, '.git', 'HEAD')) == revision
531
532
533def RmtreeIfExists(path):
534  # Git creates read-only files on windows, which cause failures with rmtree.
535  # This seems the socially accepted way to deal with it.
536  # See https://bugs.python.org/issue19643 .
537  def del_read_only_for_windows(_action, name, _exc):
538    os.chmod(name, stat.S_IWRITE)
539    os.remove(name)
540
541  if not os.path.exists(path):
542    return
543  third_party_path = os.path.abspath(os.path.join(ROOT_DIR, 'third_party'))
544  buildtools_path = os.path.abspath(os.path.join(ROOT_DIR, 'buildtools'))
545  test_path = os.path.abspath(os.path.join(ROOT_DIR, 'test', 'data'))
546  if (not os.path.abspath(path).startswith(buildtools_path) and
547      not os.path.abspath(path).startswith(test_path) and
548      not os.path.abspath(path).startswith(third_party_path)):
549    # Safety check to prevent that some merge confilct ends up doing some
550    # rm -rf / or similar.
551    logging.fatal(
552        'Cannot remove %s: outside of {buildtools, test/data, third_party}',
553        path)
554    sys.exit(1)
555  logging.info('Removing %s' % path)
556  if os.path.isdir(path):
557    shutil.rmtree(path, onerror=del_read_only_for_windows)
558  else:
559    os.remove(path)
560
561
562def CheckoutGitRepo(path, git_url, revision, check_only):
563  if IsGitRepoCheckoutOutAtRevision(path, revision):
564    return False
565  if check_only:
566    return True
567  path = path.replace('/', os.sep)
568  RmtreeIfExists(path)
569  MkdirRecursive(path)
570  logging.info('Fetching %s @ %s into %s', git_url, revision, path)
571  subprocess.check_call(['git', 'init', path], cwd=path)
572  CheckCallRetry(['git', 'fetch', '--quiet', '--depth', '1', git_url, revision],
573                 cwd=path)
574  subprocess.check_call(['git', 'checkout', revision, '--quiet'], cwd=path)
575  CheckCallRetry(
576      ['git', 'submodule', 'update', '--init', '--recursive', '--quiet'],
577      cwd=path)
578  assert (IsGitRepoCheckoutOutAtRevision(path, revision))
579  return True
580
581
582def InstallNodeModules(force_clean=False):
583  if force_clean:
584    node_modules = os.path.join(UI_DIR, 'node_modules')
585    logging.info('Clearing %s', node_modules)
586    subprocess.check_call(['git', 'clean', '-qxffd', node_modules],
587                          cwd=ROOT_DIR)
588  logging.info("Running `pnpm install --shamefully-hoist --frozen-lockfile` in {0}".format(UI_DIR))
589
590  # Some node modules have postinstall scripts (already bad) but worse
591  # sometimes they are in the form: "postinstall: 'node ./scripts/foo'"
592  # so here we need to ensure that our hermetic node is available in
593  # PATH.
594  env = os.environ.copy()
595  env['PATH'] = TOOLS_DIR + ':' + env['PATH']
596
597  subprocess.check_call([
598    os.path.join(TOOLS_DIR, 'pnpm'),
599    'install',
600    '--shamefully-hoist',
601    '--frozen-lockfile'],
602    cwd=UI_DIR,
603    env=env)
604  # pbjs has the bad habit of installing extra packages on its first
605  # run. Run it here, so we avoid fetches while building.
606  pbjs = ['node_modules/.bin/pbjs', '/dev/null', '-o', '/dev/null']
607  subprocess.call(pbjs, cwd=UI_DIR, env=env)
608  with open(NODE_MODULES_STATUS_FILE, 'w') as f:
609    f.write(HashLocalFile(os.path.join(UI_DIR, 'pnpm-lock.yaml')))
610
611
612def CheckNodeModules():
613  """Returns True if the modules are up-to-date.
614
615  There doesn't seem to be an easy way to check node modules versions. Instead
616  just check if pnpm-lock.json changed since the last `pnpm install` call.
617  """
618  if not os.path.exists(NODE_MODULES_STATUS_FILE):
619    return False
620  with open(NODE_MODULES_STATUS_FILE, 'r') as f:
621    actual = f.read()
622  expected = HashLocalFile(os.path.join(UI_DIR, 'pnpm-lock.yaml'))
623  return expected == actual
624
625
626def CheckHashes():
627  for dep in ALL_DEPS:
628    if dep.source_url.endswith('.git'):
629      continue
630    logging.info('Downloading %s for %s-%s', dep.source_url, dep.target_os,
631                 dep.target_arch)
632    with tempfile.NamedTemporaryFile(delete=False) as f:
633      f.close()
634      DownloadURL(dep.source_url, f.name)
635      actual_checksum = HashLocalFile(f.name)
636      os.unlink(f.name)
637      if (actual_checksum != dep.checksum):
638        logging.fatal('SHA-256 mismatch for {} expected {} was {}'.format(
639            dep.source_url, dep.checksum, actual_checksum))
640
641
642def CheckDepotToolsIsRecent():
643  gn_py_path = shutil.which('gn.py')
644  if gn_py_path is None:
645    return True  # depot_tools doesn't seem to be installed in the PATH.
646  dt_dir = os.path.abspath(os.path.dirname(gn_py_path))
647  cmd = ['git', '-C', dt_dir, 'merge-base', '--is-ancestor', 'a0cf4321', 'HEAD']
648  git_ret = subprocess.call(cmd, stderr=subprocess.DEVNULL)
649  if git_ret == 0:
650    return True
651  print('\033[91mYour depot_tools revision is too old. Please run:\033[0m')
652  print('git -C %s fetch origin && git -C %s checkout -B main -t origin/main' %
653        (dt_dir, dt_dir))
654  return False
655
656
657def Main():
658  parser = argparse.ArgumentParser()
659  parser.add_argument(
660      '--android',
661      action='store_true',
662      help='NDK and emulator images target_os="android"')
663  parser.add_argument(
664      '--linux-arm',
665      action='store_true',
666      help='Debian sysroots for target_os="linux" target_cpu="arm|arm64"')
667  parser.add_argument(
668      '--ui',
669      action='store_true',
670      help='Node and NPM packages to Build the Web-based UI via ./ui/build')
671  parser.add_argument(
672      '--grpc', action='store_true', help='Packages to build gRPC')
673  parser.add_argument('--check-only')
674  parser.add_argument('--filter', action='append')
675  parser.add_argument('--verify', help='Check all URLs', action='store_true')
676  parser.add_argument(
677      '--no-toolchain', help='Do not download toolchain', action='store_true')
678  parser.add_argument(
679      '--build-os',
680      default=system().lower(),
681      choices=['windows', 'darwin', 'linux'],
682      help='Override the autodetected build operating system')
683  parser.add_argument(
684      '--build-arch',
685      default=GetArch(),
686      choices=['arm64', 'x64'],
687      help='Override the autodetected build CPU architecture')
688  args = parser.parse_args()
689  if args.verify:
690    CheckHashes()
691    return 0
692
693  target_os = args.build_os
694  if args.ui and target_os == 'windows':
695    print('Building the UI on Windows is unsupported')
696    return 1
697
698  if not CheckDepotToolsIsRecent():
699    return 1
700
701  deps = BUILD_DEPS_HOST
702  if not args.no_toolchain:
703    deps += BUILD_DEPS_TOOLCHAIN_HOST
704  if args.android:
705    deps += BUILD_DEPS_ANDROID + TEST_DEPS_ANDROID
706  if args.linux_arm:
707    deps += BUILD_DEPS_LINUX_CROSS_SYSROOTS
708  if args.ui:
709    deps += UI_DEPS
710  if args.grpc:
711    deps += GRPC_DEPS
712  deps_updated = False
713  nodejs_updated = False
714
715  for old_dir in CLEANUP_OLD_DIRS:
716    RmtreeIfExists(os.path.join(ROOT_DIR, old_dir))
717
718  for dep in deps:
719    target_arch = args.build_arch
720    matches_os = dep.target_os == 'all' or target_os == dep.target_os
721    matches_arch = dep.target_arch == 'all' or target_arch == dep.target_arch
722    if not matches_os or not matches_arch:
723      continue
724    if args.filter and not any(f in dep.target_folder for f in args.filter):
725      continue
726    local_path = os.path.join(ROOT_DIR, dep.target_folder)
727    if dep.source_url.endswith('.git'):
728      deps_updated |= CheckoutGitRepo(local_path, dep.source_url, dep.checksum,
729                                      args.check_only)
730      continue
731    is_compressed = any([local_path.endswith(ext) for ext in ['.zip', '.tgz', '.tbz2']])
732    compressed_target_dir = os.path.splitext(local_path)[0] if is_compressed else None
733    compressed_dir_stamp = os.path.join(compressed_target_dir, '.stamp') if is_compressed else None
734
735    if ((not is_compressed and HashLocalFile(local_path) == dep.checksum) or
736        (is_compressed and ReadFile(compressed_dir_stamp) == dep.checksum)):
737      continue
738    deps_updated = True
739    if args.check_only:
740      continue
741    MkdirRecursive(os.path.dirname(dep.target_folder))
742    if HashLocalFile(local_path) != dep.checksum:
743      download_path = local_path + '.tmp'
744      logging.info('Downloading %s from %s', local_path, dep.source_url)
745      DownloadURL(dep.source_url, download_path)
746      os.chmod(download_path, 0o755)
747      actual_checksum = HashLocalFile(download_path)
748      if (actual_checksum != dep.checksum):
749        os.remove(download_path)
750        logging.fatal('SHA-256 mismatch for {} expected {} was {}'.format(
751            download_path, dep.checksum, actual_checksum))
752        return 1
753      shutil.move(download_path, local_path)
754      if 'nodejs' in dep.target_folder:
755        nodejs_updated = True
756
757    assert (HashLocalFile(local_path) == dep.checksum)
758
759    if is_compressed:
760      logging.info('Extracting %s into %s' % (local_path, compressed_target_dir))
761      assert (os.path.commonprefix((ROOT_DIR, compressed_target_dir)) == ROOT_DIR)
762      RmtreeIfExists(compressed_target_dir)
763
764      # Decompress the archive.
765      if local_path.endswith('.tgz'):
766        MkdirRecursive(compressed_target_dir)
767        subprocess.check_call(['tar', '-oxf', local_path], cwd=compressed_target_dir)
768      elif local_path.endswith('.zip'):
769        with zipfile.ZipFile(local_path, 'r') as zf:
770          for info in zf.infolist():
771            ExtractZipfilePreservePermissions(zf, info, compressed_target_dir)
772      elif local_path.endswith('.tbz2'):
773        tar_path = '{}.tar.tmp'.format(local_path)
774        with open(tar_path, 'w') as f:
775          with bz2.open(local_path, 'r') as bf:
776            f.write(bf.read())
777        MkdirRecursive(compressed_target_dir)
778        subprocess.check_call(['tar', '-oxf', tar_path], cwd=compressed_target_dir)
779
780      # If the zip contains one root folder, rebase one level up moving all
781      # its sub files and folders inside |target_dir|.
782      subdir = os.listdir(compressed_target_dir)
783      if len(subdir) == 1:
784        subdir = os.path.join(compressed_target_dir, subdir[0])
785        if os.path.isdir(subdir):
786          for subf in os.listdir(subdir):
787            shutil.move(os.path.join(subdir, subf), compressed_target_dir)
788          os.rmdir(subdir)
789
790      # Create stamp and remove the archive.
791      with open(compressed_dir_stamp, 'w') as stamp_file:
792        stamp_file.write(dep.checksum)
793      os.remove(local_path)
794
795  if args.ui:
796    # Needs to happen after nodejs is installed above.
797    if args.check_only:
798      deps_updated |= not CheckNodeModules()
799    else:
800      InstallNodeModules(force_clean=nodejs_updated)
801
802  cur_python_interpreter = sys.executable
803  test_data_synced = 0 == subprocess.call([
804      cur_python_interpreter, TEST_DATA_SCRIPT, 'status', '--quiet',
805      '--ignore-new'
806  ])
807  if args.check_only:
808    if not deps_updated and test_data_synced:
809      with open(args.check_only, 'w') as f:
810        f.write('OK')  # The content is irrelevant, just keep GN happy.
811      return 0
812    argz = ' '.join(
813        [x for x in sys.argv[1:] if not x.startswith('--check-only')])
814    print('\033[91mBuild deps are stale. ' +
815          'Please run tools/install-build-deps %s\033[0m' % argz)
816    if not test_data_synced:
817      print('//test/data/ is out of sync. `tools/test_data status` for details')
818    return 1
819
820  if not test_data_synced:
821    cmd = [cur_python_interpreter, TEST_DATA_SCRIPT, 'download', '--overwrite']
822    if not sys.stdout.isatty():
823      cmd += ['--verbose']  # For CI bots
824    subprocess.check_call(cmd)
825
826  if deps_updated:
827    # Stale binary files may be compiled against old sysroot headers that aren't
828    # tracked by gn.
829    logging.warning('Remember to run "gn clean <output_directory>" ' +
830                    'to avoid stale binary files.')
831
832
833if __name__ == '__main__':
834  logging.basicConfig(level=logging.INFO)
835  sys.exit(Main())
836