1#!/usr/bin/env python 2 3# Copyright 2020 The Pigweed Authors 4# 5# Licensed under the Apache License, Version 2.0 (the "License"); you may not 6# use this file except in compliance with the License. You may obtain a copy of 7# the License at 8# 9# https://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 13# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 14# License for the specific language governing permissions and limitations under 15# the License. 16"""Environment setup script for Pigweed. 17 18This script installs everything and writes out a file for the user's shell 19to source. 20 21For now, this is valid Python 2 and Python 3. Once we switch to running this 22with PyOxidizer it can be upgraded to recent Python 3. 23""" 24 25from __future__ import print_function 26 27import argparse 28import copy 29import glob 30import inspect 31import json 32import os 33import shutil 34import subprocess 35import sys 36import time 37 38# If we're running oxidized, filesystem-centric import hacks won't work. In that 39# case, jump straight to the imports and assume oxidation brought in the deps. 40if not getattr(sys, 'oxidized', False): 41 old_sys_path = copy.deepcopy(sys.path) 42 filename = None 43 if hasattr(sys.modules[__name__], '__file__'): 44 filename = __file__ 45 else: 46 # Try introspection in environments where __file__ is not populated. 47 frame = inspect.currentframe() 48 if frame is not None: 49 filename = inspect.getfile(frame) 50 # If none of our strategies worked, we're in a strange runtime environment. 51 # The imports are almost certainly going to fail. 52 if filename is None: 53 raise RuntimeError( 54 'Unable to locate pw_env_setup module; cannot continue.\n' 55 '\n' 56 'Try updating to one of the standard Python implemetations:\n' 57 ' https://www.python.org/downloads/' 58 ) 59 sys.path = [ 60 os.path.abspath(os.path.join(filename, os.path.pardir, os.path.pardir)) 61 ] 62 import pw_env_setup # pylint: disable=unused-import 63 64 sys.path = old_sys_path 65 66# pylint: disable=wrong-import-position 67from pw_env_setup.cipd_setup import update as cipd_update 68from pw_env_setup.cipd_setup import wrapper as cipd_wrapper 69from pw_env_setup.colors import Color, enable_colors 70from pw_env_setup import environment 71from pw_env_setup import spinner 72from pw_env_setup import virtualenv_setup 73from pw_env_setup import windows_env_start 74 75 76def _which( 77 executable, pathsep=os.pathsep, use_pathext=None, case_sensitive=None 78): 79 if use_pathext is None: 80 use_pathext = os.name == 'nt' 81 if case_sensitive is None: 82 case_sensitive = os.name != 'nt' and sys.platform != 'darwin' 83 84 if not case_sensitive: 85 executable = executable.lower() 86 87 exts = None 88 if use_pathext: 89 exts = frozenset(os.environ['PATHEXT'].split(pathsep)) 90 if not case_sensitive: 91 exts = frozenset(x.lower() for x in exts) 92 if not exts: 93 raise ValueError('empty PATHEXT') 94 95 paths = os.environ['PATH'].split(pathsep) 96 for path in paths: 97 try: 98 entries = frozenset(os.listdir(path)) 99 if not case_sensitive: 100 entries = frozenset(x.lower() for x in entries) 101 except OSError: 102 continue 103 104 if exts: 105 for ext in exts: 106 if executable + ext in entries: 107 return os.path.join(path, executable + ext) 108 else: 109 if executable in entries: 110 return os.path.join(path, executable) 111 112 return None 113 114 115class _Result: 116 class Status: 117 DONE = 'done' 118 SKIPPED = 'skipped' 119 FAILED = 'failed' 120 121 def __init__(self, status, *messages): 122 self._status = status 123 self._messages = list(messages) 124 125 def ok(self): 126 return self._status in {_Result.Status.DONE, _Result.Status.SKIPPED} 127 128 def status_str(self, duration=None): 129 if not duration: 130 return self._status 131 132 duration_parts = [] 133 if duration > 60: 134 minutes = int(duration // 60) 135 duration %= 60 136 duration_parts.append('{}m'.format(minutes)) 137 duration_parts.append('{:.1f}s'.format(duration)) 138 return '{} ({})'.format(self._status, ''.join(duration_parts)) 139 140 def messages(self): 141 return self._messages 142 143 144class ConfigError(Exception): 145 pass 146 147 148def result_func(glob_warnings=()): 149 def result(status, *args): 150 return _Result(status, *([str(x) for x in glob_warnings] + list(args))) 151 152 return result 153 154 155class ConfigFileError(Exception): 156 pass 157 158 159class MissingSubmodulesError(Exception): 160 pass 161 162 163def _assert_sequence(value): 164 assert isinstance(value, (list, tuple)) 165 return value 166 167 168# TODO(mohrr) remove disable=useless-object-inheritance once in Python 3. 169# pylint: disable=useless-object-inheritance 170# pylint: disable=too-many-instance-attributes 171# pylint: disable=too-many-arguments 172class EnvSetup(object): 173 """Run environment setup for Pigweed.""" 174 175 def __init__( 176 self, 177 pw_root, 178 cipd_cache_dir, 179 shell_file, 180 quiet, 181 install_dir, 182 strict, 183 virtualenv_gn_out_dir, 184 json_file, 185 project_root, 186 config_file, 187 use_existing_cipd, 188 check_submodules, 189 use_pinned_pip_packages, 190 cipd_only, 191 trust_cipd_hash, 192 additional_cipd_file, 193 disable_rosetta, 194 ): 195 self._env = environment.Environment() 196 self._project_root = project_root 197 self._pw_root = pw_root 198 self._setup_root = os.path.join( 199 pw_root, 'pw_env_setup', 'py', 'pw_env_setup' 200 ) 201 self._cipd_cache_dir = cipd_cache_dir 202 self._shell_file = shell_file 203 self._env._shell_file = shell_file 204 self._is_windows = os.name == 'nt' 205 self._quiet = quiet 206 self._install_dir = install_dir 207 self._virtualenv_root = os.path.join(self._install_dir, 'pigweed-venv') 208 self._strict = strict 209 self._cipd_only = cipd_only 210 self._trust_cipd_hash = trust_cipd_hash 211 self._additional_cipd_file = additional_cipd_file 212 self._disable_rosetta = disable_rosetta 213 214 if os.path.isfile(shell_file): 215 os.unlink(shell_file) 216 217 if isinstance(self._pw_root, bytes) and bytes != str: 218 self._pw_root = self._pw_root.decode() 219 220 self._cipd_package_file = [] 221 self._project_actions = [] 222 self._virtualenv_requirements = [] 223 self._virtualenv_constraints = [] 224 self._virtualenv_gn_targets = [] 225 self._virtualenv_gn_args = [] 226 self._virtualenv_pip_install_disable_cache = False 227 self._virtualenv_pip_install_find_links = [] 228 self._virtualenv_pip_install_offline = False 229 self._virtualenv_pip_install_require_hashes = False 230 self._use_pinned_pip_packages = use_pinned_pip_packages 231 self._optional_submodules = [] 232 self._required_submodules = [] 233 self._virtualenv_system_packages = False 234 self._pw_packages = [] 235 self._root_variable = None 236 237 self._check_submodules = check_submodules 238 239 self._json_file = json_file 240 self._gni_file = None 241 242 self._config_file_name = config_file 243 self._env.set( 244 '_PW_ENVIRONMENT_CONFIG_FILE', os.path.abspath(config_file) 245 ) 246 if config_file: 247 self._parse_config_file(config_file) 248 249 self._check_submodule_presence() 250 251 self._use_existing_cipd = use_existing_cipd 252 self._virtualenv_gn_out_dir = virtualenv_gn_out_dir 253 254 if self._root_variable: 255 self._env.set(self._root_variable, project_root, deactivate=False) 256 self._env.set('PW_PROJECT_ROOT', project_root, deactivate=False) 257 self._env.set('PW_ROOT', pw_root, deactivate=False) 258 self._env.set('_PW_ACTUAL_ENVIRONMENT_ROOT', install_dir) 259 self._env.set('VIRTUAL_ENV', self._virtualenv_root) 260 self._env.add_replacement('_PW_ACTUAL_ENVIRONMENT_ROOT', install_dir) 261 self._env.add_replacement('PW_ROOT', pw_root) 262 263 def _process_globs(self, globs): 264 unique_globs = [] 265 for pat in globs: 266 if pat and pat not in unique_globs: 267 unique_globs.append(pat) 268 269 files = [] 270 warnings = [] 271 for pat in unique_globs: 272 if pat: 273 matches = glob.glob(pat) 274 if not matches: 275 warning = 'pattern "{}" matched 0 files'.format(pat) 276 warnings.append('warning: {}'.format(warning)) 277 if self._strict: 278 raise ConfigError(warning) 279 280 files.extend(matches) 281 282 if globs and not files: 283 warnings.append('warning: matched 0 total files') 284 if self._strict: 285 raise ConfigError('matched 0 total files') 286 287 return files, warnings 288 289 def _parse_config_file(self, config_file): 290 # This should use pw_env_setup.config_file instead. 291 with open(config_file, 'r') as ins: 292 config = json.load(ins) 293 294 # While transitioning, allow environment config to be at the top of 295 # the JSON file or at '.["pw"]["pw_env_setup"]'. 296 config = config.get('pw', config) 297 config = config.get('pw_env_setup', config) 298 299 self._root_variable = config.pop('root_variable', None) 300 301 # This variable is not used by env setup since we already have it. 302 # However, other tools may use it, so we double-check that it's correct. 303 pigweed_root = os.path.join( 304 self._project_root, 305 config.pop('relative_pigweed_root', self._pw_root), 306 ) 307 if os.path.abspath(self._pw_root) != os.path.abspath(pigweed_root): 308 raise ValueError( 309 'expected Pigweed root {!r} in config but found {!r}'.format( 310 os.path.relpath(self._pw_root, self._project_root), 311 os.path.relpath(pigweed_root, self._project_root), 312 ) 313 ) 314 315 rosetta = config.pop('rosetta', 'allow') 316 if rosetta not in ('never', 'allow', 'force'): 317 raise ValueError(rosetta) 318 self._rosetta = rosetta in ('allow', 'force') 319 if self._disable_rosetta: 320 self._rosetta = False 321 self._env.set('_PW_ROSETTA', str(int(self._rosetta))) 322 323 if 'json_file' in config: 324 self._json_file = config.pop('json_file') 325 326 self._gni_file = config.pop('gni_file', None) 327 328 self._optional_submodules.extend( 329 _assert_sequence(config.pop('optional_submodules', ())) 330 ) 331 self._required_submodules.extend( 332 _assert_sequence(config.pop('required_submodules', ())) 333 ) 334 335 if self._optional_submodules and self._required_submodules: 336 raise ValueError( 337 '{} contains both "optional_submodules" and ' 338 '"required_submodules", but these options are mutually ' 339 'exclusive'.format(self._config_file_name) 340 ) 341 342 self._cipd_package_file.extend( 343 os.path.join(self._project_root, x) 344 for x in _assert_sequence(config.pop('cipd_package_files', ())) 345 ) 346 self._cipd_package_file.extend( 347 os.path.join(self._project_root, x) 348 for x in self._additional_cipd_file or () 349 ) 350 351 for action in config.pop('project_actions', {}): 352 # We can add a 'phase' option in the future if we end up needing to 353 # support project actions at more than one point in the setup flow. 354 self._project_actions.append( 355 (action['import_path'], action['module_name']) 356 ) 357 358 for pkg in _assert_sequence(config.pop('pw_packages', ())): 359 self._pw_packages.append(pkg) 360 361 virtualenv = config.pop('virtualenv', {}) 362 363 if virtualenv.get('gn_root'): 364 root = os.path.join(self._project_root, virtualenv.pop('gn_root')) 365 else: 366 root = self._project_root 367 368 for target in _assert_sequence(virtualenv.pop('gn_targets', ())): 369 self._virtualenv_gn_targets.append( 370 virtualenv_setup.GnTarget('{}#{}'.format(root, target)) 371 ) 372 373 self._virtualenv_gn_args = _assert_sequence( 374 virtualenv.pop('gn_args', ()) 375 ) 376 377 self._virtualenv_system_packages = virtualenv.pop( 378 'system_packages', False 379 ) 380 381 for req_txt in _assert_sequence(virtualenv.pop('requirements', ())): 382 self._virtualenv_requirements.append( 383 os.path.join(self._project_root, req_txt) 384 ) 385 386 for constraint_txt in _assert_sequence( 387 virtualenv.pop('constraints', ()) 388 ): 389 self._virtualenv_constraints.append( 390 os.path.join(self._project_root, constraint_txt) 391 ) 392 393 for pip_cache_dir in _assert_sequence( 394 virtualenv.pop('pip_install_find_links', ()) 395 ): 396 self._virtualenv_pip_install_find_links.append(pip_cache_dir) 397 398 self._virtualenv_pip_install_disable_cache = virtualenv.pop( 399 'pip_install_disable_cache', False 400 ) 401 self._virtualenv_pip_install_offline = virtualenv.pop( 402 'pip_install_offline', False 403 ) 404 self._virtualenv_pip_install_require_hashes = virtualenv.pop( 405 'pip_install_require_hashes', False 406 ) 407 408 if virtualenv: 409 raise ConfigFileError( 410 'unrecognized option in {}: "virtualenv.{}"'.format( 411 self._config_file_name, next(iter(virtualenv)) 412 ) 413 ) 414 415 if config: 416 raise ConfigFileError( 417 'unrecognized option in {}: "{}"'.format( 418 self._config_file_name, next(iter(config)) 419 ) 420 ) 421 422 def _check_submodule_presence(self): 423 uninitialized = set() 424 425 # Don't check submodule presence if using the Android Repo Tool. 426 if os.path.isdir(os.path.join(self._project_root, '.repo')): 427 return 428 429 if not self._check_submodules: 430 return 431 432 cmd = ['git', 'submodule', 'status', '--recursive'] 433 434 for line in subprocess.check_output( 435 cmd, cwd=self._project_root 436 ).splitlines(): 437 if isinstance(line, bytes): 438 line = line.decode() 439 # Anything but an initial '-' means the submodule is initialized. 440 if not line.startswith('-'): 441 continue 442 uninitialized.add(line.split()[1]) 443 444 missing = uninitialized - set(self._optional_submodules) 445 if self._required_submodules: 446 missing = set(self._required_submodules) & uninitialized 447 448 if missing: 449 print( 450 'Not all submodules are initialized. Please run the ' 451 'following commands.', 452 file=sys.stderr, 453 ) 454 print('', file=sys.stderr) 455 456 for miss in sorted(missing): 457 print( 458 ' git submodule update --init {}'.format(miss), 459 file=sys.stderr, 460 ) 461 print('', file=sys.stderr) 462 463 if self._required_submodules: 464 print( 465 'If these submodules are not required, remove them from ' 466 'the "required_submodules"', 467 file=sys.stderr, 468 ) 469 470 else: 471 print( 472 'If these submodules are not required, add them to the ' 473 '"optional_submodules"', 474 file=sys.stderr, 475 ) 476 477 print('list in the environment config JSON file:', file=sys.stderr) 478 print(' {}'.format(self._config_file_name), file=sys.stderr) 479 print('', file=sys.stderr) 480 481 raise MissingSubmodulesError(', '.join(sorted(missing))) 482 483 def _write_gni_file(self): 484 if self._cipd_only: 485 return 486 487 gni_file = os.path.join( 488 self._project_root, 'build_overrides', 'pigweed_environment.gni' 489 ) 490 if self._gni_file: 491 gni_file = os.path.join(self._project_root, self._gni_file) 492 493 with open(gni_file, 'w') as outs: 494 self._env.gni(outs, self._project_root, gni_file) 495 shutil.copy(gni_file, os.path.join(self._install_dir, 'logs')) 496 497 def _log(self, *args, **kwargs): 498 # Not using logging module because it's awkward to flush a log handler. 499 if self._quiet: 500 return 501 flush = kwargs.pop('flush', False) 502 print(*args, **kwargs) 503 if flush: 504 sys.stdout.flush() 505 506 def setup(self): 507 """Runs each of the env_setup steps.""" 508 509 if os.name == 'nt': 510 windows_env_start.print_banner(bootstrap=True, no_shell_file=False) 511 else: 512 enable_colors() 513 514 steps = [ 515 ('CIPD package manager', self.cipd), 516 ('Project actions', self.project_actions), 517 ('Python environment', self.virtualenv), 518 ('pw packages', self.pw_package), 519 ('Host tools', self.host_tools), 520 ] 521 522 if self._is_windows: 523 steps.append(("Windows scripts", self.win_scripts)) 524 525 if self._cipd_only: 526 steps = [('CIPD package manager', self.cipd)] 527 528 self._log( 529 Color.bold( 530 'Downloading and installing packages into local ' 531 'source directory:\n' 532 ) 533 ) 534 535 max_name_len = max(len(name) for name, _ in steps) 536 537 self._env.comment( 538 ''' 539This file is automatically generated. DO NOT EDIT! 540For details, see $PW_ROOT/pw_env_setup/py/pw_env_setup/env_setup.py and 541$PW_ROOT/pw_env_setup/py/pw_env_setup/environment.py. 542'''.strip() 543 ) 544 545 if not self._is_windows: 546 self._env.comment( 547 ''' 548For help debugging errors in this script, uncomment the next line. 549set -x 550Then use `set +x` to go back to normal. 551'''.strip() 552 ) 553 554 self._env.echo( 555 Color.bold( 556 'Activating environment (setting environment variables):' 557 ) 558 ) 559 self._env.echo('') 560 561 for name, step in steps: 562 self._log( 563 ' Setting up {name:.<{width}}...'.format( 564 name=name, width=max_name_len 565 ), 566 end='', 567 flush=True, 568 ) 569 self._env.echo( 570 ' Setting environment variables for ' 571 '{name:.<{width}}...'.format(name=name, width=max_name_len), 572 newline=False, 573 ) 574 575 start = time.time() 576 spin = spinner.Spinner(self._quiet) 577 with spin(): 578 result = step(spin) 579 stop = time.time() 580 581 self._log(result.status_str(stop - start)) 582 583 self._env.echo(result.status_str()) 584 for message in result.messages(): 585 sys.stderr.write('{}\n'.format(message)) 586 self._env.echo(message) 587 588 if not result.ok(): 589 return -1 590 591 # Log the environment state at the end of each step for debugging. 592 log_dir = os.path.join(self._install_dir, 'logs') 593 if not os.path.isdir(log_dir): 594 os.makedirs(log_dir) 595 actions_json = os.path.join( 596 log_dir, 'post-{}.json'.format(name.replace(' ', '_')) 597 ) 598 with open(actions_json, 'w') as outs: 599 self._env.json(outs) 600 601 # This file needs to be written after the CIPD step and before the 602 # Python virtualenv step. It also needs to be rewritten after the 603 # Python virtualenv step, so it's easiest to just write it after 604 # every step. 605 self._write_gni_file() 606 607 # Only write stuff for GitHub Actions once, at the end. 608 if 'GITHUB_ACTIONS' in os.environ: 609 self._env.github(self._install_dir) 610 611 self._log('') 612 self._env.echo('') 613 614 self._env.finalize() 615 616 self._env.echo(Color.bold('Checking the environment:')) 617 self._env.echo() 618 619 self._env.doctor() 620 self._env.echo() 621 622 self._env.echo( 623 Color.bold('Environment looks good, you are ready to go!') 624 ) 625 self._env.echo() 626 627 # Don't write new files if all we did was update CIPD packages. 628 if self._cipd_only: 629 return 0 630 631 with open(self._shell_file, 'w') as outs: 632 self._env.write(outs, shell_file=self._shell_file) 633 634 deactivate = os.path.join( 635 self._install_dir, 636 'deactivate{}'.format(os.path.splitext(self._shell_file)[1]), 637 ) 638 with open(deactivate, 'w') as outs: 639 self._env.write_deactivate(outs, shell_file=deactivate) 640 641 config = { 642 # Skipping sysname and nodename in os.uname(). nodename could change 643 # based on the current network. sysname won't change, but is 644 # redundant because it's contained in release or version, and 645 # skipping it here simplifies logic. 646 'uname': ' '.join(getattr(os, 'uname', lambda: ())()[2:]), 647 'os': os.name, 648 } 649 650 with open(os.path.join(self._install_dir, 'config.json'), 'w') as outs: 651 outs.write( 652 json.dumps(config, indent=4, separators=(',', ': ')) + '\n' 653 ) 654 655 json_file = self._json_file or os.path.join( 656 self._install_dir, 'actions.json' 657 ) 658 with open(json_file, 'w') as outs: 659 self._env.json(outs) 660 661 return 0 662 663 def cipd(self, spin): 664 """Set up cipd and install cipd packages.""" 665 666 install_dir = os.path.join(self._install_dir, 'cipd') 667 668 # There's no way to get to the UnsupportedPlatform exception if this 669 # flag is set, but this flag should only be set in LUCI builds which 670 # will always have CIPD. 671 if self._use_existing_cipd: 672 cipd_client = 'cipd' 673 674 else: 675 try: 676 cipd_client = cipd_wrapper.init( 677 install_dir, 678 silent=True, 679 rosetta=self._rosetta, 680 ) 681 except cipd_wrapper.UnsupportedPlatform as exc: 682 return result_func((' {!r}'.format(exc),))( 683 _Result.Status.SKIPPED, 684 ' abandoning CIPD setup', 685 ) 686 687 package_files, glob_warnings = self._process_globs( 688 self._cipd_package_file 689 ) 690 result = result_func(glob_warnings) 691 692 if not package_files: 693 return result(_Result.Status.SKIPPED) 694 695 if not cipd_update.update( 696 cipd=cipd_client, 697 root_install_dir=install_dir, 698 package_files=package_files, 699 cache_dir=self._cipd_cache_dir, 700 env_vars=self._env, 701 rosetta=self._rosetta, 702 spin=spin, 703 trust_hash=self._trust_cipd_hash, 704 ): 705 return result(_Result.Status.FAILED) 706 707 return result(_Result.Status.DONE) 708 709 def project_actions(self, unused_spin): 710 """Perform project install actions. 711 712 This is effectively a limited plugin system for performing 713 project-specific actions (e.g. fetching tools) after CIPD but before 714 virtualenv setup. 715 """ 716 result = result_func() 717 718 if not self._project_actions: 719 return result(_Result.Status.SKIPPED) 720 721 if sys.version_info[0] < 3: 722 raise ValueError( 723 'Project Actions require Python 3 or higher. ' 724 'The current python version is %s' % sys.version_info 725 ) 726 727 # Once Keir okays removing 2.7 support for env_setup, move this import 728 # to the main list of imports at the top of the file. 729 import importlib # pylint: disable=import-outside-toplevel 730 731 for import_path, module_name in self._project_actions: 732 sys.path.append(import_path) 733 mod = importlib.import_module(module_name) 734 mod.run_action(env=self._env) 735 736 return result(_Result.Status.DONE) 737 738 def virtualenv(self, unused_spin): 739 """Setup virtualenv.""" 740 741 requirements, req_glob_warnings = self._process_globs( 742 self._virtualenv_requirements 743 ) 744 745 constraints, constraint_glob_warnings = self._process_globs( 746 self._virtualenv_constraints 747 ) 748 749 result = result_func(req_glob_warnings + constraint_glob_warnings) 750 751 orig_python3 = _which('python3') 752 with self._env(): 753 new_python3 = _which('python3') 754 755 # There is an issue with the virtualenv module on Windows where it 756 # expects sys.executable to be called "python.exe" or it fails to 757 # properly execute. If we installed Python 3 in the CIPD step we need 758 # to address this. Detect if we did so and if so create a copy of 759 # python3.exe called python.exe so that virtualenv works. 760 if orig_python3 != new_python3 and self._is_windows: 761 python3_copy = os.path.join( 762 os.path.dirname(new_python3), 'python.exe' 763 ) 764 if not os.path.exists(python3_copy): 765 shutil.copyfile(new_python3, python3_copy) 766 new_python3 = python3_copy 767 768 if not requirements and not self._virtualenv_gn_targets: 769 return result(_Result.Status.SKIPPED) 770 771 if not virtualenv_setup.install( 772 project_root=self._project_root, 773 venv_path=self._virtualenv_root, 774 requirements=requirements, 775 constraints=constraints, 776 pip_install_find_links=self._virtualenv_pip_install_find_links, 777 pip_install_offline=self._virtualenv_pip_install_offline, 778 pip_install_require_hashes=( 779 self._virtualenv_pip_install_require_hashes 780 ), 781 pip_install_disable_cache=( 782 self._virtualenv_pip_install_disable_cache 783 ), 784 gn_args=self._virtualenv_gn_args, 785 gn_targets=self._virtualenv_gn_targets, 786 gn_out_dir=self._virtualenv_gn_out_dir, 787 python=new_python3, 788 env=self._env, 789 system_packages=self._virtualenv_system_packages, 790 use_pinned_pip_packages=self._use_pinned_pip_packages, 791 ): 792 return result(_Result.Status.FAILED) 793 794 return result(_Result.Status.DONE) 795 796 def pw_package(self, unused_spin): 797 """Install "default" pw packages.""" 798 799 result = result_func() 800 801 pkg_dir = os.path.join(self._install_dir, 'packages') 802 self._env.set('PW_PACKAGE_ROOT', pkg_dir) 803 804 if not os.path.isdir(pkg_dir): 805 os.makedirs(pkg_dir) 806 807 if not self._pw_packages: 808 return result(_Result.Status.SKIPPED) 809 810 for pkg in self._pw_packages: 811 print('installing {}'.format(pkg)) 812 cmd = ['pw', 'package', 'install', pkg] 813 814 log = os.path.join(pkg_dir, '{}.log'.format(pkg)) 815 try: 816 with open(log, 'w') as outs, self._env(): 817 print(*cmd, file=outs) 818 subprocess.check_call( 819 cmd, 820 cwd=self._project_root, 821 stdout=outs, 822 stderr=subprocess.STDOUT, 823 ) 824 except subprocess.CalledProcessError: 825 with open(log, 'r') as ins: 826 sys.stderr.write(ins.read()) 827 raise 828 829 return result(_Result.Status.DONE) 830 831 def host_tools(self, unused_spin): 832 # The host tools are grabbed from CIPD, at least initially. If the 833 # user has a current host build, that build will be used instead. 834 # TODO(mohrr) find a way to do stuff like this for all projects. 835 host_dir = os.path.join(self._pw_root, 'out', 'host') 836 self._env.prepend('PATH', os.path.join(host_dir, 'host_tools')) 837 return _Result(_Result.Status.DONE) 838 839 def win_scripts(self, unused_spin): 840 # These scripts act as a compatibility layer for windows. 841 env_setup_dir = os.path.join(self._pw_root, 'pw_env_setup') 842 self._env.prepend( 843 'PATH', os.path.join(env_setup_dir, 'windows_scripts') 844 ) 845 return _Result(_Result.Status.DONE) 846 847 848def parse(argv=None): 849 """Parse command-line arguments.""" 850 parser = argparse.ArgumentParser(prog="python -m pw_env_setup.env_setup") 851 852 pw_root = os.environ.get('PW_ROOT', None) 853 if not pw_root: 854 try: 855 with open(os.devnull, 'w') as outs: 856 pw_root = subprocess.check_output( 857 ['git', 'rev-parse', '--show-toplevel'], stderr=outs 858 ).strip() 859 except subprocess.CalledProcessError: 860 pw_root = None 861 862 parser.add_argument( 863 '--pw-root', 864 default=pw_root, 865 required=not pw_root, 866 ) 867 868 project_root = os.environ.get('PW_PROJECT_ROOT', None) or pw_root 869 870 parser.add_argument( 871 '--project-root', 872 default=project_root, 873 required=not project_root, 874 ) 875 876 default_cipd_cache_dir = os.environ.get( 877 'CIPD_CACHE_DIR', os.path.expanduser('~/.cipd-cache-dir') 878 ) 879 if 'PW_NO_CIPD_CACHE_DIR' in os.environ: 880 default_cipd_cache_dir = None 881 882 parser.add_argument('--cipd-cache-dir', default=default_cipd_cache_dir) 883 884 parser.add_argument( 885 '--no-cipd-cache-dir', 886 action='store_const', 887 const=None, 888 dest='cipd_cache_dir', 889 ) 890 891 parser.add_argument( 892 '--trust-cipd-hash', 893 action='store_true', 894 help='Only run the cipd executable if the ensure file or command-line ' 895 'has changed. Defaults to false since files could have been deleted ' 896 'from the installation directory and cipd would add them back.', 897 ) 898 899 parser.add_argument( 900 '--shell-file', 901 help='Where to write the file for shells to source.', 902 required=True, 903 ) 904 905 parser.add_argument( 906 '--quiet', 907 help='Reduce output.', 908 action='store_true', 909 default='PW_ENVSETUP_QUIET' in os.environ, 910 ) 911 912 parser.add_argument( 913 '--install-dir', 914 help='Location to install environment.', 915 required=True, 916 ) 917 918 parser.add_argument( 919 '--config-file', 920 help='Path to pigweed.json file.', 921 default=os.path.join(project_root, 'pigweed.json'), 922 ) 923 924 parser.add_argument( 925 '--additional-cipd-file', 926 help=( 927 'Path to additional CIPD files, in addition to those referenced by ' 928 'the --config-file file.' 929 ), 930 action='append', 931 ) 932 933 parser.add_argument( 934 '--virtualenv-gn-out-dir', 935 help=( 936 'Output directory to use when building and installing Python ' 937 'packages with GN; defaults to a unique path in the environment ' 938 'directory.' 939 ), 940 ) 941 942 parser.add_argument('--json-file', help=argparse.SUPPRESS, default=None) 943 944 parser.add_argument( 945 '--use-existing-cipd', 946 help='Use cipd executable from the environment instead of fetching it.', 947 action='store_true', 948 ) 949 950 parser.add_argument( 951 '--strict', 952 help='Fail if there are any warnings.', 953 action='store_true', 954 ) 955 956 parser.add_argument( 957 '--unpin-pip-packages', 958 dest='use_pinned_pip_packages', 959 help='Do not use pins of pip packages.', 960 action='store_false', 961 ) 962 963 parser.add_argument( 964 '--cipd-only', 965 help='Skip non-CIPD steps.', 966 action='store_true', 967 ) 968 969 parser.add_argument( 970 '--skip-submodule-check', 971 help='Skip checking for submodule presence.', 972 dest='check_submodules', 973 action='store_false', 974 ) 975 976 parser.add_argument( 977 '--disable-rosetta', 978 help=( 979 "Disable Rosetta on ARM Macs, regardless of what's in " 980 'pigweed.json.' 981 ), 982 action='store_true', 983 ) 984 985 args = parser.parse_args(argv) 986 987 return args 988 989 990def main(): 991 try: 992 return EnvSetup(**vars(parse())).setup() 993 except subprocess.CalledProcessError as err: 994 print() 995 print(err.output) 996 raise 997 998 999if __name__ == '__main__': 1000 sys.exit(main()) 1001