1#!/usr/bin/env python 2# Copyright 2020 The Pigweed Authors 3# 4# Licensed under the Apache License, Version 2.0 (the "License"); you may not 5# use this file except in compliance with the License. You may obtain a copy of 6# the License at 7# 8# https://www.apache.org/licenses/LICENSE-2.0 9# 10# Unless required by applicable law or agreed to in writing, software 11# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 12# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 13# License for the specific language governing permissions and limitations under 14# the License. 15"""Installs or updates prebuilt tools. 16 17Must be tested with Python 2 and Python 3. 18 19The stdout of this script is meant to be executed by the invoking shell. 20""" 21 22from __future__ import print_function 23 24import collections 25import hashlib 26import json 27import os 28import platform as platform_module 29import re 30import subprocess 31import sys 32 33 34def check_auth(cipd, package_files, cipd_service_account, spin): 35 """Check have access to CIPD pigweed directory.""" 36 cmd = [cipd] 37 extra_args = [] 38 if cipd_service_account: 39 extra_args.extend(['-service-account-json', cipd_service_account]) 40 41 paths = [] 42 for package_file in package_files: 43 with open(package_file, 'r') as ins: 44 # This is an expensive RPC, so only check the first few entries 45 # in each file. 46 for i, entry in enumerate(json.load(ins).get('packages', ())): 47 if i >= 3: 48 break 49 parts = entry['path'].split('/') 50 while '${' in parts[-1]: 51 parts.pop(-1) 52 paths.append('/'.join(parts)) 53 54 username = None 55 try: 56 output = subprocess.check_output( 57 cmd + ['auth-info'] + extra_args, stderr=subprocess.STDOUT 58 ).decode() 59 logged_in = True 60 61 match = re.search(r'Logged in as (\S*)\.', output) 62 if match: 63 username = match.group(1) 64 65 except subprocess.CalledProcessError: 66 logged_in = False 67 68 def _check_all_paths(): 69 inaccessible_paths = [] 70 71 for path in paths: 72 # Not catching CalledProcessError because 'cipd ls' seems to never 73 # return an error code unless it can't reach the CIPD server. 74 output = subprocess.check_output( 75 cmd + ['ls', path] + extra_args, stderr=subprocess.STDOUT 76 ).decode() 77 if 'No matching packages' not in output: 78 continue 79 80 # 'cipd ls' only lists sub-packages but ignores any packages at the 81 # given path. 'cipd instances' will give versions of that package. 82 # 'cipd instances' does use an error code if there's no such package 83 # or that package is inaccessible. 84 try: 85 subprocess.check_output( 86 cmd + ['instances', path] + extra_args, 87 stderr=subprocess.STDOUT, 88 ) 89 except subprocess.CalledProcessError: 90 inaccessible_paths.append(path) 91 92 return inaccessible_paths 93 94 inaccessible_paths = _check_all_paths() 95 96 if inaccessible_paths and not logged_in: 97 with spin.pause(): 98 stderr = lambda *args: print(*args, file=sys.stderr) 99 stderr() 100 stderr( 101 'Not logged in to CIPD and no anonymous access to the ' 102 'following CIPD paths:' 103 ) 104 for path in inaccessible_paths: 105 stderr(' {}'.format(path)) 106 stderr() 107 stderr('Attempting CIPD login') 108 try: 109 # Note that with -service-account-json, auth-login is a no-op. 110 subprocess.check_call(cmd + ['auth-login'] + extra_args) 111 except subprocess.CalledProcessError: 112 stderr('CIPD login failed') 113 return False 114 115 inaccessible_paths = _check_all_paths() 116 117 if inaccessible_paths: 118 stderr = lambda *args: print(*args, file=sys.stderr) 119 stderr('=' * 60) 120 username_part = '' 121 if username: 122 username_part = '({}) '.format(username) 123 stderr( 124 'Your account {}does not have access to the following ' 125 'paths'.format(username_part) 126 ) 127 stderr('(or they do not exist)') 128 for path in inaccessible_paths: 129 stderr(' {}'.format(path)) 130 stderr('=' * 60) 131 return False 132 133 return True 134 135 136def platform(rosetta=False): 137 """Return the CIPD platform string of the current system.""" 138 osname = { 139 'darwin': 'mac', 140 'linux': 'linux', 141 'windows': 'windows', 142 }[platform_module.system().lower()] 143 144 if platform_module.machine().startswith(('aarch64', 'armv8')): 145 arch = 'arm64' 146 elif platform_module.machine() == 'x86_64': 147 arch = 'amd64' 148 elif platform_module.machine() == 'i686': 149 arch = 'i386' 150 else: 151 arch = platform_module.machine() 152 153 platform_arch = '{}-{}'.format(osname, arch).lower() 154 155 # Support `mac-arm64` through Rosetta until `mac-arm64` binaries are ready 156 if platform_arch == 'mac-arm64' and rosetta: 157 return 'mac-amd64' 158 159 return platform_arch 160 161 162def all_package_files(env_vars, package_files): 163 """Recursively retrieve all package files.""" 164 165 to_process = [] 166 for pkg_file in package_files: 167 args = [] 168 if env_vars: 169 args.append(env_vars.get('PW_PROJECT_ROOT')) 170 args.append(pkg_file) 171 172 # The signature here is os.path.join(a, *p). Pylint doesn't like when 173 # we call os.path.join(*args), but is happy if we instead call 174 # os.path.join(args[0], *args[1:]). Disabling the option on this line 175 # seems to be a less confusing choice. 176 path = os.path.join(*args) # pylint: disable=no-value-for-parameter 177 178 to_process.append(path) 179 180 processed_files = [] 181 182 def flatten_package_files(package_files): 183 """Flatten nested package files.""" 184 for package_file in package_files: 185 yield package_file 186 processed_files.append(package_file) 187 188 with open(package_file, 'r') as ins: 189 entries = json.load(ins).get('included_files', ()) 190 entries = [ 191 os.path.join(os.path.dirname(package_file), entry) 192 for entry in entries 193 ] 194 entries = [ 195 entry for entry in entries if entry not in processed_files 196 ] 197 198 if entries: 199 for entry in flatten_package_files(entries): 200 yield entry 201 202 return list(flatten_package_files(to_process)) 203 204 205def update_subdir(package, package_file): 206 """Updates subdir in package and saves original.""" 207 name = package_file_name(package_file) 208 if 'subdir' in package: 209 package['original_subdir'] = package['subdir'] 210 package['subdir'] = '/'.join([name, package['subdir']]) 211 else: 212 package['subdir'] = name 213 214 215def all_packages(package_files): 216 packages = [] 217 for package_file in package_files: 218 with open(package_file, 'r') as ins: 219 file_packages = json.load(ins).get('packages', ()) 220 for package in file_packages: 221 update_subdir(package, package_file) 222 packages.extend(file_packages) 223 return packages 224 225 226def deduplicate_packages(packages): 227 deduped = collections.OrderedDict() 228 for package in packages: 229 # Use the package + the subdir as the key 230 pkg_key = package['path'] 231 pkg_key += package.get('original_subdir', '') 232 233 if pkg_key in deduped: 234 # Delete the old package 235 del deduped[pkg_key] 236 237 # Insert the new package at the end 238 deduped[pkg_key] = package 239 return list(deduped.values()) 240 241 242def write_ensure_file( 243 package_files, ensure_file, platform 244): # pylint: disable=redefined-outer-name 245 logdir = os.path.dirname(ensure_file) 246 packages = all_packages(package_files) 247 with open(os.path.join(logdir, 'all-packages.json'), 'w') as outs: 248 json.dump(packages, outs, indent=4) 249 deduped_packages = deduplicate_packages(packages) 250 with open(os.path.join(logdir, 'deduped-packages.json'), 'w') as outs: 251 json.dump(deduped_packages, outs, indent=4) 252 253 with open(ensure_file, 'w') as outs: 254 outs.write( 255 '$VerifiedPlatform linux-amd64\n' 256 '$VerifiedPlatform mac-amd64\n' 257 '$ParanoidMode CheckPresence\n' 258 ) 259 260 for pkg in deduped_packages: 261 # If this is a new-style package manifest platform handling must 262 # be done here instead of by the cipd executable. 263 if 'platforms' in pkg and platform not in pkg['platforms']: 264 continue 265 266 outs.write('@Subdir {}\n'.format(pkg.get('subdir', ''))) 267 outs.write('{} {}\n'.format(pkg['path'], ' '.join(pkg['tags']))) 268 269 270def package_file_name(package_file): 271 return os.path.basename(os.path.splitext(package_file)[0]) 272 273 274def package_installation_path(root_install_dir, package_file): 275 """Returns the package installation path. 276 277 Args: 278 root_install_dir: The CIPD installation directory. 279 package_file: The path to the .json package definition file. 280 """ 281 return os.path.join( 282 root_install_dir, 'packages', package_file_name(package_file) 283 ) 284 285 286def update( # pylint: disable=too-many-locals 287 cipd, 288 package_files, 289 root_install_dir, 290 cache_dir, 291 rosetta=False, 292 env_vars=None, 293 spin=None, 294 trust_hash=False, 295): 296 """Grab the tools listed in ensure_files.""" 297 298 package_files = all_package_files(env_vars, package_files) 299 300 # TODO(mohrr) use os.makedirs(..., exist_ok=True). 301 if not os.path.isdir(root_install_dir): 302 os.makedirs(root_install_dir) 303 304 # This file is read by 'pw doctor' which needs to know which package files 305 # were used in the environment. 306 package_files_file = os.path.join( 307 root_install_dir, '_all_package_files.json' 308 ) 309 with open(package_files_file, 'w') as outs: 310 json.dump(package_files, outs, indent=2) 311 312 if env_vars: 313 env_vars.prepend('PATH', root_install_dir) 314 env_vars.set('PW_CIPD_INSTALL_DIR', root_install_dir) 315 if cache_dir: 316 env_vars.set('CIPD_CACHE_DIR', cache_dir) 317 318 pw_root = None 319 320 if env_vars: 321 pw_root = env_vars.get('PW_ROOT', None) 322 if not pw_root: 323 pw_root = os.environ['PW_ROOT'] 324 325 plat = platform(rosetta) 326 327 ensure_file = os.path.join(root_install_dir, 'packages.ensure') 328 write_ensure_file(package_files, ensure_file, plat) 329 330 install_dir = os.path.join(root_install_dir, 'packages') 331 332 cmd = [ 333 cipd, 334 'ensure', 335 '-ensure-file', 336 ensure_file, 337 '-root', 338 install_dir, 339 '-log-level', 340 'debug', 341 '-json-output', 342 os.path.join(root_install_dir, 'packages.json'), 343 '-max-threads', 344 '0', # 0 means use CPU count. 345 ] 346 347 if cache_dir: 348 cmd.extend(('-cache-dir', cache_dir)) 349 350 cipd_service_account = None 351 if env_vars: 352 cipd_service_account = env_vars.get('PW_CIPD_SERVICE_ACCOUNT_JSON') 353 if not cipd_service_account: 354 cipd_service_account = os.environ.get('PW_CIPD_SERVICE_ACCOUNT_JSON') 355 if cipd_service_account: 356 cmd.extend(['-service-account-json', cipd_service_account]) 357 358 hasher = hashlib.sha256() 359 encoded = '\0'.join(cmd) 360 if hasattr(encoded, 'encode'): 361 encoded = encoded.encode() 362 hasher.update(encoded) 363 with open(ensure_file, 'rb') as ins: 364 hasher.update(ins.read()) 365 digest = hasher.hexdigest() 366 367 with open(os.path.join(root_install_dir, 'hash.log'), 'w') as hashlog: 368 print('calculated digest:', digest, file=hashlog) 369 370 hash_file = os.path.join(root_install_dir, 'packages.sha256') 371 print('hash file path:', hash_file, file=hashlog) 372 print('exists:', os.path.isfile(hash_file), file=hashlog) 373 print('trust_hash:', trust_hash, file=hashlog) 374 if trust_hash and os.path.isfile(hash_file): 375 with open(hash_file, 'r') as ins: 376 digest_file = ins.read().strip() 377 print('contents:', digest_file, file=hashlog) 378 print('equal:', digest == digest_file, file=hashlog) 379 if digest == digest_file: 380 return True 381 382 if not check_auth(cipd, package_files, cipd_service_account, spin): 383 return False 384 385 log = os.path.join(root_install_dir, 'packages.log') 386 try: 387 with open(log, 'w') as outs: 388 print(*cmd, file=outs) 389 subprocess.check_call(cmd, stdout=outs, stderr=subprocess.STDOUT) 390 except subprocess.CalledProcessError: 391 with open(log, 'r') as ins: 392 sys.stderr.write(ins.read()) 393 raise 394 395 with open(hash_file, 'w') as outs: 396 print(digest, file=outs) 397 398 # Set environment variables so tools can later find things under, for 399 # example, 'share'. 400 if env_vars: 401 for package_file in reversed(package_files): 402 name = package_file_name(package_file) 403 file_install_dir = os.path.join(install_dir, name) 404 405 # The MinGW package isn't always structured correctly, and might 406 # live nested in a `mingw64` subdirectory. 407 maybe_mingw = os.path.join(file_install_dir, 'mingw64', 'bin') 408 if os.name == 'nt' and os.path.isdir(maybe_mingw): 409 env_vars.prepend('PATH', maybe_mingw) 410 411 # If this package file has no packages and just includes one other 412 # file, there won't be any contents of the folder for this package. 413 # In that case, point the variable that would point to this folder 414 # to the folder of the included file. 415 with open(package_file) as ins: 416 contents = json.load(ins) 417 entries = contents.get('included_files', ()) 418 file_packages = contents.get('packages', ()) 419 if not file_packages and len(entries) == 1: 420 file_install_dir = os.path.join( 421 install_dir, 422 package_file_name(os.path.basename(entries[0])), 423 ) 424 425 # Some executables get installed at top-level and some get 426 # installed under 'bin'. A small number of old packages prefix the 427 # entire tree with the platform (e.g., chromium/third_party/tcl). 428 for bin_dir in ( 429 file_install_dir, 430 os.path.join(file_install_dir, 'bin'), 431 os.path.join(file_install_dir, plat, 'bin'), 432 ): 433 if os.path.isdir(bin_dir): 434 env_vars.prepend('PATH', bin_dir) 435 env_vars.set( 436 'PW_{}_CIPD_INSTALL_DIR'.format(name.upper().replace('-', '_')), 437 file_install_dir, 438 ) 439 440 return True 441