1#!/usr/bin/env python 2# Copyright 2020 The Pigweed Authors 3# 4# Licensed under the Apache License, Version 2.0 (the "License"); you may not 5# use this file except in compliance with the License. You may obtain a copy of 6# the License at 7# 8# https://www.apache.org/licenses/LICENSE-2.0 9# 10# Unless required by applicable law or agreed to in writing, software 11# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 12# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 13# License for the specific language governing permissions and limitations under 14# the License. 15"""Installs or updates prebuilt tools. 16 17Must be tested with Python 2 and Python 3. 18 19The stdout of this script is meant to be executed by the invoking shell. 20""" 21 22import collections 23import hashlib 24import json 25import os 26import platform as platform_module 27import re 28import subprocess 29import sys 30 31 32def _stderr(*args): 33 return print(*args, file=sys.stderr) 34 35 36def check_auth(cipd, package_files, cipd_service_account, spin): 37 """Check have access to CIPD pigweed directory.""" 38 cmd = [cipd] 39 extra_args = [] 40 if cipd_service_account: 41 extra_args.extend(['-service-account-json', cipd_service_account]) 42 43 paths = [] 44 for package_file in package_files: 45 with open(package_file, 'r') as ins: 46 # This is an expensive RPC, so only check the first few entries 47 # in each file. 48 for i, entry in enumerate(json.load(ins).get('packages', ())): 49 if i >= 3: 50 break 51 parts = entry['path'].split('/') 52 while '${' in parts[-1]: 53 parts.pop(-1) 54 paths.append('/'.join(parts)) 55 56 username = None 57 try: 58 output = subprocess.check_output( 59 cmd + ['auth-info'] + extra_args, stderr=subprocess.STDOUT 60 ).decode() 61 logged_in = True 62 63 match = re.search(r'Logged in as (\S*)\.', output) 64 if match: 65 username = match.group(1) 66 67 except subprocess.CalledProcessError: 68 logged_in = False 69 70 def _check_all_paths(): 71 inaccessible_paths = [] 72 73 for path in paths: 74 # Not catching CalledProcessError because 'cipd ls' seems to never 75 # return an error code unless it can't reach the CIPD server. 76 output = subprocess.check_output( 77 cmd + ['ls', '-h'] + extra_args + [path], 78 stderr=subprocess.STDOUT, 79 ).decode() 80 if 'No matching packages' not in output: 81 continue 82 83 # 'cipd ls' only lists sub-packages but ignores any packages at the 84 # given path. 'cipd instances' will give versions of that package. 85 # 'cipd instances' does use an error code if there's no such package 86 # or that package is inaccessible. 87 try: 88 subprocess.check_output( 89 cmd + ['instances'] + extra_args + [path], 90 stderr=subprocess.STDOUT, 91 ) 92 except subprocess.CalledProcessError: 93 inaccessible_paths.append(path) 94 95 return inaccessible_paths 96 97 inaccessible_paths = _check_all_paths() 98 99 if inaccessible_paths and not logged_in: 100 with spin.pause(): 101 _stderr() 102 _stderr( 103 'Not logged in to CIPD and no anonymous access to the ' 104 'following CIPD paths:' 105 ) 106 for path in inaccessible_paths: 107 _stderr(' {}'.format(path)) 108 _stderr() 109 _stderr('Attempting CIPD login') 110 try: 111 # Note that with -service-account-json, auth-login is a no-op. 112 subprocess.check_call(cmd + ['auth-login'] + extra_args) 113 except subprocess.CalledProcessError: 114 _stderr('CIPD login failed') 115 return False 116 117 inaccessible_paths = _check_all_paths() 118 119 if inaccessible_paths: 120 _stderr('=' * 60) 121 username_part = '' 122 if username: 123 username_part = '({}) '.format(username) 124 _stderr( 125 'Your account {}does not have access to the following ' 126 'paths'.format(username_part) 127 ) 128 _stderr('(or they do not exist)') 129 for path in inaccessible_paths: 130 _stderr(' {}'.format(path)) 131 _stderr('=' * 60) 132 return False 133 134 return True 135 136 137def platform(rosetta=False): 138 """Return the CIPD platform string of the current system.""" 139 osname = { 140 'darwin': 'mac', 141 'linux': 'linux', 142 'windows': 'windows', 143 }[platform_module.system().lower()] 144 145 if platform_module.machine().startswith(('aarch64', 'armv8')): 146 arch = 'arm64' 147 elif platform_module.machine() == 'x86_64': 148 arch = 'amd64' 149 elif platform_module.machine() == 'i686': 150 arch = 'i386' 151 else: 152 arch = platform_module.machine() 153 154 platform_arch = '{}-{}'.format(osname, arch).lower() 155 156 # Support `mac-arm64` through Rosetta until `mac-arm64` binaries are ready 157 if platform_arch == 'mac-arm64' and rosetta: 158 return 'mac-amd64' 159 160 return platform_arch 161 162 163def all_package_files(env_vars, package_files): 164 """Recursively retrieve all package files.""" 165 166 to_process = [] 167 for pkg_file in package_files: 168 args = [] 169 if env_vars: 170 args.append(env_vars.get('PW_PROJECT_ROOT')) 171 args.append(pkg_file) 172 173 # The signature here is os.path.join(a, *p). Pylint doesn't like when 174 # we call os.path.join(*args), but is happy if we instead call 175 # os.path.join(args[0], *args[1:]). Disabling the option on this line 176 # seems to be a less confusing choice. 177 path = os.path.join(*args) # pylint: disable=no-value-for-parameter 178 179 to_process.append(path) 180 181 processed_files = [] 182 183 def flatten_package_files(package_files): 184 """Flatten nested package files.""" 185 for package_file in package_files: 186 yield package_file 187 processed_files.append(package_file) 188 189 with open(package_file, 'r') as ins: 190 entries = json.load(ins).get('included_files', ()) 191 entries = [ 192 os.path.join(os.path.dirname(package_file), entry) 193 for entry in entries 194 ] 195 entries = [ 196 entry for entry in entries if entry not in processed_files 197 ] 198 199 if entries: 200 for entry in flatten_package_files(entries): 201 yield entry 202 203 return list(flatten_package_files(to_process)) 204 205 206def update_subdir(package, package_file): 207 """Updates subdir in package and saves original.""" 208 name = package_file_name(package_file) 209 if 'subdir' in package: 210 package['original_subdir'] = package['subdir'] 211 package['subdir'] = '/'.join([name, package['subdir']]) 212 else: 213 package['subdir'] = name 214 215 216def all_packages(package_files): 217 packages = [] 218 for package_file in package_files: 219 with open(package_file, 'r') as ins: 220 file_packages = json.load(ins).get('packages', ()) 221 for package in file_packages: 222 update_subdir(package, package_file) 223 packages.extend(file_packages) 224 return packages 225 226 227def deduplicate_packages(packages): 228 deduped = collections.OrderedDict() 229 for package in packages: 230 # Use the package + the subdir as the key 231 pkg_key = package['path'] 232 pkg_key += package.get('original_subdir', '') 233 234 if pkg_key in deduped: 235 # Delete the old package 236 del deduped[pkg_key] 237 238 # Insert the new package at the end 239 deduped[pkg_key] = package 240 return list(deduped.values()) 241 242 243def write_ensure_file( 244 package_files, ensure_file, platform 245): # pylint: disable=redefined-outer-name 246 logdir = os.path.dirname(ensure_file) 247 packages = all_packages(package_files) 248 with open(os.path.join(logdir, 'all-packages.json'), 'w') as outs: 249 json.dump(packages, outs, indent=4) 250 deduped_packages = deduplicate_packages(packages) 251 with open(os.path.join(logdir, 'deduped-packages.json'), 'w') as outs: 252 json.dump(deduped_packages, outs, indent=4) 253 254 with open(ensure_file, 'w') as outs: 255 outs.write( 256 '$VerifiedPlatform linux-amd64\n' 257 '$VerifiedPlatform mac-amd64\n' 258 '$ParanoidMode CheckPresence\n' 259 ) 260 261 for pkg in deduped_packages: 262 # If this is a new-style package manifest platform handling must 263 # be done here instead of by the cipd executable. 264 if 'platforms' in pkg and platform not in pkg['platforms']: 265 continue 266 267 outs.write('@Subdir {}\n'.format(pkg.get('subdir', ''))) 268 outs.write('{} {}\n'.format(pkg['path'], ' '.join(pkg['tags']))) 269 270 271def package_file_name(package_file): 272 return os.path.basename(os.path.splitext(package_file)[0]) 273 274 275def package_installation_path(root_install_dir, package_file): 276 """Returns the package installation path. 277 278 Args: 279 root_install_dir: The CIPD installation directory. 280 package_file: The path to the .json package definition file. 281 """ 282 return os.path.join( 283 root_install_dir, 'packages', package_file_name(package_file) 284 ) 285 286 287def update( # pylint: disable=too-many-locals 288 cipd, 289 package_files, 290 root_install_dir, 291 cache_dir, 292 rosetta=False, 293 env_vars=None, 294 spin=None, 295 trust_hash=False, 296): 297 """Grab the tools listed in ensure_files.""" 298 299 package_files = all_package_files(env_vars, package_files) 300 301 # TODO(mohrr) use os.makedirs(..., exist_ok=True). 302 if not os.path.isdir(root_install_dir): 303 os.makedirs(root_install_dir) 304 305 # This file is read by 'pw doctor' which needs to know which package files 306 # were used in the environment. 307 package_files_file = os.path.join( 308 root_install_dir, '_all_package_files.json' 309 ) 310 with open(package_files_file, 'w') as outs: 311 json.dump(package_files, outs, indent=2) 312 313 if env_vars: 314 env_vars.prepend('PATH', root_install_dir) 315 env_vars.set('PW_CIPD_INSTALL_DIR', root_install_dir) 316 if cache_dir: 317 env_vars.set('CIPD_CACHE_DIR', cache_dir) 318 319 pw_root = None 320 321 if env_vars: 322 pw_root = env_vars.get('PW_ROOT', None) 323 if not pw_root: 324 pw_root = os.environ['PW_ROOT'] 325 326 plat = platform(rosetta) 327 328 ensure_file = os.path.join(root_install_dir, 'packages.ensure') 329 write_ensure_file(package_files, ensure_file, plat) 330 331 install_dir = os.path.join(root_install_dir, 'packages') 332 333 cmd = [ 334 cipd, 335 'ensure', 336 '-ensure-file', 337 ensure_file, 338 '-root', 339 install_dir, 340 '-log-level', 341 'debug', 342 '-json-output', 343 os.path.join(root_install_dir, 'packages.json'), 344 '-max-threads', 345 '0', # 0 means use CPU count. 346 ] 347 348 if cache_dir: 349 cmd.extend(('-cache-dir', cache_dir)) 350 351 cipd_service_account = None 352 if env_vars: 353 cipd_service_account = env_vars.get('PW_CIPD_SERVICE_ACCOUNT_JSON') 354 if not cipd_service_account: 355 cipd_service_account = os.environ.get('PW_CIPD_SERVICE_ACCOUNT_JSON') 356 if cipd_service_account: 357 cmd.extend(['-service-account-json', cipd_service_account]) 358 359 hasher = hashlib.sha256() 360 encoded = '\0'.join(cmd) 361 if hasattr(encoded, 'encode'): 362 encoded = encoded.encode() 363 hasher.update(encoded) 364 with open(ensure_file, 'rb') as ins: 365 hasher.update(ins.read()) 366 digest = hasher.hexdigest() 367 368 with open(os.path.join(root_install_dir, 'hash.log'), 'w') as hashlog: 369 print('calculated digest:', digest, file=hashlog) 370 371 hash_file = os.path.join(root_install_dir, 'packages.sha256') 372 print('hash file path:', hash_file, file=hashlog) 373 print('exists:', os.path.isfile(hash_file), file=hashlog) 374 print('trust_hash:', trust_hash, file=hashlog) 375 if trust_hash and os.path.isfile(hash_file): 376 with open(hash_file, 'r') as ins: 377 digest_file = ins.read().strip() 378 print('contents:', digest_file, file=hashlog) 379 print('equal:', digest == digest_file, file=hashlog) 380 if digest == digest_file: 381 return True 382 383 if not check_auth(cipd, package_files, cipd_service_account, spin): 384 return False 385 386 log = os.path.join(root_install_dir, 'packages.log') 387 try: 388 with open(log, 'w') as outs: 389 print(*cmd, file=outs) 390 subprocess.check_call(cmd, stdout=outs, stderr=subprocess.STDOUT) 391 except subprocess.CalledProcessError: 392 with open(log, 'r') as ins: 393 sys.stderr.write(repr(cmd)) 394 sys.stderr.write(ins.read()) 395 raise 396 397 with open(hash_file, 'w') as outs: 398 print(digest, file=outs) 399 400 # Set environment variables so tools can later find things under, for 401 # example, 'share'. 402 if env_vars: 403 for package_file in reversed(package_files): 404 name = package_file_name(package_file) 405 file_install_dir = os.path.join(install_dir, name) 406 407 # The MinGW package isn't always structured correctly, and might 408 # live nested in a `mingw64` subdirectory. 409 maybe_mingw = os.path.join(file_install_dir, 'mingw64', 'bin') 410 if os.name == 'nt' and os.path.isdir(maybe_mingw): 411 env_vars.prepend('PATH', maybe_mingw) 412 413 # If this package file has no packages and just includes one other 414 # file, there won't be any contents of the folder for this package. 415 # In that case, point the variable that would point to this folder 416 # to the folder of the included file. 417 with open(package_file) as ins: 418 contents = json.load(ins) 419 entries = contents.get('included_files', ()) 420 file_packages = contents.get('packages', ()) 421 if not file_packages and len(entries) == 1: 422 file_install_dir = os.path.join( 423 install_dir, 424 package_file_name(os.path.basename(entries[0])), 425 ) 426 427 # Some executables get installed at top-level and some get 428 # installed under 'bin'. A small number of old packages prefix the 429 # entire tree with the platform (e.g., chromium/third_party/tcl). 430 for bin_dir in ( 431 file_install_dir, 432 os.path.join(file_install_dir, 'bin'), 433 os.path.join(file_install_dir, plat, 'bin'), 434 ): 435 if os.path.isdir(bin_dir): 436 env_vars.prepend('PATH', bin_dir) 437 env_vars.set( 438 'PW_{}_CIPD_INSTALL_DIR'.format(name.upper().replace('-', '_')), 439 file_install_dir, 440 ) 441 442 return True 443