• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/env python
2
3# Copyright 2020 The Pigweed Authors
4#
5# Licensed under the Apache License, Version 2.0 (the "License"); you may not
6# use this file except in compliance with the License. You may obtain a copy of
7# the License at
8#
9#     https://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
13# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
14# License for the specific language governing permissions and limitations under
15# the License.
16"""Environment setup script for Pigweed.
17
18This script installs everything and writes out a file for the user's shell
19to source.
20
21For now, this is valid Python 2 and Python 3. Once we switch to running this
22with PyOxidizer it can be upgraded to recent Python 3.
23"""
24
25from __future__ import print_function
26
27import argparse
28import copy
29import glob
30import inspect
31import json
32import os
33import shutil
34import subprocess
35import sys
36import time
37
38# TODO(pwbug/67): Remove import hacks once the oxidized prebuilt binaries are
39# proven stable for first-time bootstrapping. For now, continue to support
40# running directly from source without assuming a functioning Python
41# environment when running for the first time.
42
43# If we're running oxidized, filesystem-centric import hacks won't work. In that
44# case, jump straight to the imports and assume oxidation brought in the deps.
45if not getattr(sys, 'oxidized', False):
46    old_sys_path = copy.deepcopy(sys.path)
47    filename = None
48    if hasattr(sys.modules[__name__], '__file__'):
49        filename = __file__
50    else:
51        # Try introspection in environments where __file__ is not populated.
52        frame = inspect.currentframe()
53        if frame is not None:
54            filename = inspect.getfile(frame)
55    # If none of our strategies worked, we're in a strange runtime environment.
56    # The imports are almost certainly going to fail.
57    if filename is None:
58        raise RuntimeError(
59            'Unable to locate pw_env_setup module; cannot continue.\n'
60            '\n'
61            'Try updating to one of the standard Python implemetations:\n'
62            '  https://www.python.org/downloads/')
63    sys.path = [
64        os.path.abspath(os.path.join(filename, os.path.pardir, os.path.pardir))
65    ]
66    import pw_env_setup  # pylint: disable=unused-import
67    sys.path = old_sys_path
68
69# pylint: disable=wrong-import-position
70from pw_env_setup.cipd_setup import update as cipd_update
71from pw_env_setup.cipd_setup import wrapper as cipd_wrapper
72from pw_env_setup.colors import Color, enable_colors
73from pw_env_setup import environment
74from pw_env_setup import spinner
75from pw_env_setup import virtualenv_setup
76from pw_env_setup import windows_env_start
77
78
79# TODO(pwbug/67, pwbug/68) switch to shutil.which().
80def _which(executable,
81           pathsep=os.pathsep,
82           use_pathext=None,
83           case_sensitive=None):
84    if use_pathext is None:
85        use_pathext = (os.name == 'nt')
86    if case_sensitive is None:
87        case_sensitive = (os.name != 'nt' and sys.platform != 'darwin')
88
89    if not case_sensitive:
90        executable = executable.lower()
91
92    exts = None
93    if use_pathext:
94        exts = frozenset(os.environ['PATHEXT'].split(pathsep))
95        if not case_sensitive:
96            exts = frozenset(x.lower() for x in exts)
97        if not exts:
98            raise ValueError('empty PATHEXT')
99
100    paths = os.environ['PATH'].split(pathsep)
101    for path in paths:
102        try:
103            entries = frozenset(os.listdir(path))
104            if not case_sensitive:
105                entries = frozenset(x.lower() for x in entries)
106        except OSError:
107            continue
108
109        if exts:
110            for ext in exts:
111                if executable + ext in entries:
112                    return os.path.join(path, executable + ext)
113        else:
114            if executable in entries:
115                return os.path.join(path, executable)
116
117    return None
118
119
120class _Result:
121    class Status:
122        DONE = 'done'
123        SKIPPED = 'skipped'
124        FAILED = 'failed'
125
126    def __init__(self, status, *messages):
127        self._status = status
128        self._messages = list(messages)
129
130    def ok(self):
131        return self._status in {_Result.Status.DONE, _Result.Status.SKIPPED}
132
133    def status_str(self, duration=None):
134        if not duration:
135            return self._status
136
137        duration_parts = []
138        if duration > 60:
139            minutes = int(duration // 60)
140            duration %= 60
141            duration_parts.append('{}m'.format(minutes))
142        duration_parts.append('{:.1f}s'.format(duration))
143        return '{} ({})'.format(self._status, ''.join(duration_parts))
144
145    def messages(self):
146        return self._messages
147
148
149class ConfigError(Exception):
150    pass
151
152
153def result_func(glob_warnings=()):
154    def result(status, *args):
155        return _Result(status, *([str(x) for x in glob_warnings] + list(args)))
156
157    return result
158
159
160class ConfigFileError(Exception):
161    pass
162
163
164class MissingSubmodulesError(Exception):
165    pass
166
167
168# TODO(mohrr) remove disable=useless-object-inheritance once in Python 3.
169# pylint: disable=useless-object-inheritance
170# pylint: disable=too-many-instance-attributes
171# pylint: disable=too-many-arguments
172class EnvSetup(object):
173    """Run environment setup for Pigweed."""
174    def __init__(self, pw_root, cipd_cache_dir, shell_file, quiet, install_dir,
175                 virtualenv_root, strict, virtualenv_gn_out_dir, json_file,
176                 project_root, config_file, use_existing_cipd,
177                 use_pinned_pip_packages, cipd_only, trust_cipd_hash):
178        self._env = environment.Environment()
179        self._project_root = project_root
180        self._pw_root = pw_root
181        self._setup_root = os.path.join(pw_root, 'pw_env_setup', 'py',
182                                        'pw_env_setup')
183        self._cipd_cache_dir = cipd_cache_dir
184        self._shell_file = shell_file
185        self._is_windows = os.name == 'nt'
186        self._quiet = quiet
187        self._install_dir = install_dir
188        self._virtualenv_root = (virtualenv_root
189                                 or os.path.join(install_dir, 'pigweed-venv'))
190        self._strict = strict
191        self._cipd_only = cipd_only
192        self._trust_cipd_hash = trust_cipd_hash
193
194        if os.path.isfile(shell_file):
195            os.unlink(shell_file)
196
197        if isinstance(self._pw_root, bytes) and bytes != str:
198            self._pw_root = self._pw_root.decode()
199
200        self._cipd_package_file = []
201        self._virtualenv_requirements = []
202        self._virtualenv_gn_targets = []
203        self._virtualenv_gn_args = []
204        self._use_pinned_pip_packages = use_pinned_pip_packages
205        self._optional_submodules = []
206        self._required_submodules = []
207        self._virtualenv_system_packages = False
208        self._pw_packages = []
209        self._root_variable = None
210
211        self._json_file = json_file
212        self._gni_file = None
213
214        self._config_file_name = getattr(config_file, 'name', 'config file')
215        self._env.set('_PW_ENVIRONMENT_CONFIG_FILE', self._config_file_name)
216        if config_file:
217            self._parse_config_file(config_file)
218
219        self._check_submodules()
220
221        self._use_existing_cipd = use_existing_cipd
222        self._virtualenv_gn_out_dir = virtualenv_gn_out_dir
223
224        if self._root_variable:
225            self._env.set(self._root_variable, project_root, deactivate=False)
226        self._env.set('PW_PROJECT_ROOT', project_root, deactivate=False)
227        self._env.set('PW_ROOT', pw_root, deactivate=False)
228        self._env.set('_PW_ACTUAL_ENVIRONMENT_ROOT', install_dir)
229        self._env.add_replacement('_PW_ACTUAL_ENVIRONMENT_ROOT', install_dir)
230        self._env.add_replacement('PW_ROOT', pw_root)
231
232    def _process_globs(self, globs):
233        unique_globs = []
234        for pat in globs:
235            if pat and pat not in unique_globs:
236                unique_globs.append(pat)
237
238        files = []
239        warnings = []
240        for pat in unique_globs:
241            if pat:
242                matches = glob.glob(pat)
243                if not matches:
244                    warning = 'pattern "{}" matched 0 files'.format(pat)
245                    warnings.append('warning: {}'.format(warning))
246                    if self._strict:
247                        raise ConfigError(warning)
248
249                files.extend(matches)
250
251        if globs and not files:
252            warnings.append('warning: matched 0 total files')
253            if self._strict:
254                raise ConfigError('matched 0 total files')
255
256        return files, warnings
257
258    def _parse_config_file(self, config_file):
259        config = json.load(config_file)
260
261        self._root_variable = config.pop('root_variable', None)
262
263        if 'json_file' in config:
264            self._json_file = config.pop('json_file')
265
266        self._gni_file = config.pop('gni_file', None)
267
268        self._optional_submodules.extend(config.pop('optional_submodules', ()))
269        self._required_submodules.extend(config.pop('required_submodules', ()))
270
271        if self._optional_submodules and self._required_submodules:
272            raise ValueError(
273                '{} contains both "optional_submodules" and '
274                '"required_submodules", but these options are mutually '
275                'exclusive'.format(self._config_file_name))
276
277        self._cipd_package_file.extend(
278            os.path.join(self._project_root, x)
279            for x in config.pop('cipd_package_files', ()))
280
281        for pkg in config.pop('pw_packages', ()):
282            self._pw_packages.append(pkg)
283
284        virtualenv = config.pop('virtualenv', {})
285
286        if virtualenv.get('gn_root'):
287            root = os.path.join(self._project_root, virtualenv.pop('gn_root'))
288        else:
289            root = self._project_root
290
291        for target in virtualenv.pop('gn_targets', ()):
292            self._virtualenv_gn_targets.append(
293                virtualenv_setup.GnTarget('{}#{}'.format(root, target)))
294
295        self._virtualenv_gn_args = virtualenv.pop('gn_args', ())
296
297        self._virtualenv_system_packages = virtualenv.pop(
298            'system_packages', False)
299
300        if virtualenv:
301            raise ConfigFileError(
302                'unrecognized option in {}: "virtualenv.{}"'.format(
303                    self._config_file_name, next(iter(virtualenv))))
304
305        if config:
306            raise ConfigFileError('unrecognized option in {}: "{}"'.format(
307                self._config_file_name, next(iter(config))))
308
309    def _check_submodules(self):
310        unitialized = set()
311
312        # Don't check submodule presence if using the Android Repo Tool.
313        if os.path.isdir(os.path.join(self._project_root, '.repo')):
314            return
315
316        cmd = ['git', 'submodule', 'status', '--recursive']
317
318        for line in subprocess.check_output(
319                cmd, cwd=self._project_root).splitlines():
320            if isinstance(line, bytes):
321                line = line.decode()
322            # Anything but an initial '-' means the submodule is initialized.
323            if not line.startswith('-'):
324                continue
325            unitialized.add(line.split()[1])
326
327        missing = unitialized - set(self._optional_submodules)
328        if self._required_submodules:
329            missing = set(self._required_submodules) & unitialized
330
331        if missing:
332            print(
333                'Not all submodules are initialized. Please run the '
334                'following commands.',
335                file=sys.stderr)
336            print('', file=sys.stderr)
337
338            for miss in missing:
339                print('    git submodule update --init {}'.format(miss),
340                      file=sys.stderr)
341            print('', file=sys.stderr)
342
343            if self._required_submodules:
344                print(
345                    'If these submodules are not required, remove them from '
346                    'the "required_submodules"',
347                    file=sys.stderr)
348
349            else:
350                print(
351                    'If these submodules are not required, add them to the '
352                    '"optional_submodules"',
353                    file=sys.stderr)
354
355            print('list in the environment config JSON file:', file=sys.stderr)
356            print('    {}'.format(self._config_file_name), file=sys.stderr)
357            print('', file=sys.stderr)
358
359            raise MissingSubmodulesError(', '.join(sorted(missing)))
360
361    def _write_gni_file(self):
362        gni_file = os.path.join(self._project_root, 'build_overrides',
363                                'pigweed_environment.gni')
364        if self._gni_file:
365            gni_file = os.path.join(self._project_root, self._gni_file)
366
367        with open(gni_file, 'w') as outs:
368            self._env.gni(outs, self._project_root)
369
370    def _log(self, *args, **kwargs):
371        # Not using logging module because it's awkward to flush a log handler.
372        if self._quiet:
373            return
374        flush = kwargs.pop('flush', False)
375        print(*args, **kwargs)
376        if flush:
377            sys.stdout.flush()
378
379    def setup(self):
380        """Runs each of the env_setup steps."""
381
382        if os.name == 'nt':
383            windows_env_start.print_banner(bootstrap=True, no_shell_file=False)
384        else:
385            enable_colors()
386
387        steps = [
388            ('CIPD package manager', self.cipd),
389            ('Python environment', self.virtualenv),
390            ('pw packages', self.pw_package),
391            ('Host tools', self.host_tools),
392        ]
393
394        if self._is_windows:
395            steps.append(("Windows scripts", self.win_scripts))
396
397        if self._cipd_only:
398            steps = [('CIPD package manager', self.cipd)]
399
400        self._log(
401            Color.bold('Downloading and installing packages into local '
402                       'source directory:\n'))
403
404        max_name_len = max(len(name) for name, _ in steps)
405
406        self._env.comment('''
407This file is automatically generated. DO NOT EDIT!
408For details, see $PW_ROOT/pw_env_setup/py/pw_env_setup/env_setup.py and
409$PW_ROOT/pw_env_setup/py/pw_env_setup/environment.py.
410'''.strip())
411
412        if not self._is_windows:
413            self._env.comment('''
414For help debugging errors in this script, uncomment the next line.
415set -x
416Then use `set +x` to go back to normal.
417'''.strip())
418
419        self._env.echo(
420            Color.bold(
421                'Activating environment (setting environment variables):'))
422        self._env.echo('')
423
424        for name, step in steps:
425            self._log('  Setting up {name:.<{width}}...'.format(
426                name=name, width=max_name_len),
427                      end='',
428                      flush=True)
429            self._env.echo(
430                '  Setting environment variables for {name:.<{width}}...'.
431                format(name=name, width=max_name_len),
432                newline=False,
433            )
434
435            start = time.time()
436            spin = spinner.Spinner(self._quiet)
437            with spin():
438                result = step(spin)
439            stop = time.time()
440
441            self._log(result.status_str(stop - start))
442
443            self._env.echo(result.status_str())
444            for message in result.messages():
445                sys.stderr.write('{}\n'.format(message))
446                self._env.echo(message)
447
448            if not result.ok():
449                return -1
450
451            # Log the environment state at the end of each step for debugging.
452            log_dir = os.path.join(self._install_dir, 'logs')
453            if not os.path.isdir(log_dir):
454                os.makedirs(log_dir)
455            actions_json = os.path.join(
456                log_dir, 'post-{}.json'.format(name.replace(' ', '_')))
457            with open(actions_json, 'w') as outs:
458                self._env.json(outs)
459
460            # This file needs to be written after the CIPD step and before the
461            # Python virtualenv step. It also needs to be rewritten after the
462            # Python virtualenv step, so it's easiest to just write it after
463            # every step.
464            self._write_gni_file()
465
466        self._log('')
467        self._env.echo('')
468
469        self._env.finalize()
470
471        self._env.echo(Color.bold('Checking the environment:'))
472        self._env.echo()
473
474        self._env.doctor()
475        self._env.echo()
476
477        self._env.echo(
478            Color.bold('Environment looks good, you are ready to go!'))
479        self._env.echo()
480
481        # Don't write new files if all we did was update CIPD packages.
482        if self._cipd_only:
483            return 0
484
485        with open(self._shell_file, 'w') as outs:
486            self._env.write(outs)
487
488        deactivate = os.path.join(
489            self._install_dir,
490            'deactivate{}'.format(os.path.splitext(self._shell_file)[1]))
491        with open(deactivate, 'w') as outs:
492            self._env.write_deactivate(outs)
493
494        config = {
495            # Skipping sysname and nodename in os.uname(). nodename could change
496            # based on the current network. sysname won't change, but is
497            # redundant because it's contained in release or version, and
498            # skipping it here simplifies logic.
499            'uname': ' '.join(getattr(os, 'uname', lambda: ())()[2:]),
500            'os': os.name,
501        }
502
503        with open(os.path.join(self._install_dir, 'config.json'), 'w') as outs:
504            outs.write(
505                json.dumps(config, indent=4, separators=(',', ': ')) + '\n')
506
507        json_file = (self._json_file
508                     or os.path.join(self._install_dir, 'actions.json'))
509        with open(json_file, 'w') as outs:
510            self._env.json(outs)
511
512        return 0
513
514    def cipd(self, spin):
515        """Set up cipd and install cipd packages."""
516
517        install_dir = os.path.join(self._install_dir, 'cipd')
518
519        # There's no way to get to the UnsupportedPlatform exception if this
520        # flag is set, but this flag should only be set in LUCI builds which
521        # will always have CIPD.
522        if self._use_existing_cipd:
523            cipd_client = 'cipd'
524
525        else:
526            try:
527                cipd_client = cipd_wrapper.init(install_dir, silent=True)
528            except cipd_wrapper.UnsupportedPlatform as exc:
529                return result_func(('    {!r}'.format(exc), ))(
530                    _Result.Status.SKIPPED,
531                    '    abandoning CIPD setup',
532                )
533
534        package_files, glob_warnings = self._process_globs(
535            self._cipd_package_file)
536        result = result_func(glob_warnings)
537
538        if not package_files:
539            return result(_Result.Status.SKIPPED)
540
541        if not cipd_update.update(cipd=cipd_client,
542                                  root_install_dir=install_dir,
543                                  package_files=package_files,
544                                  cache_dir=self._cipd_cache_dir,
545                                  env_vars=self._env,
546                                  spin=spin,
547                                  trust_hash=self._trust_cipd_hash):
548            return result(_Result.Status.FAILED)
549
550        return result(_Result.Status.DONE)
551
552    def virtualenv(self, unused_spin):
553        """Setup virtualenv."""
554
555        requirements, req_glob_warnings = self._process_globs(
556            self._virtualenv_requirements)
557        result = result_func(req_glob_warnings)
558
559        orig_python3 = _which('python3')
560        with self._env():
561            new_python3 = _which('python3')
562
563        # There is an issue with the virtualenv module on Windows where it
564        # expects sys.executable to be called "python.exe" or it fails to
565        # properly execute. If we installed Python 3 in the CIPD step we need
566        # to address this. Detect if we did so and if so create a copy of
567        # python3.exe called python.exe so that virtualenv works.
568        if orig_python3 != new_python3 and self._is_windows:
569            python3_copy = os.path.join(os.path.dirname(new_python3),
570                                        'python.exe')
571            if not os.path.exists(python3_copy):
572                shutil.copyfile(new_python3, python3_copy)
573            new_python3 = python3_copy
574
575        if not requirements and not self._virtualenv_gn_targets:
576            return result(_Result.Status.SKIPPED)
577
578        if not virtualenv_setup.install(
579                project_root=self._project_root,
580                venv_path=self._virtualenv_root,
581                requirements=requirements,
582                gn_args=self._virtualenv_gn_args,
583                gn_targets=self._virtualenv_gn_targets,
584                gn_out_dir=self._virtualenv_gn_out_dir,
585                python=new_python3,
586                env=self._env,
587                system_packages=self._virtualenv_system_packages,
588                use_pinned_pip_packages=self._use_pinned_pip_packages,
589        ):
590            return result(_Result.Status.FAILED)
591
592        return result(_Result.Status.DONE)
593
594    def pw_package(self, unused_spin):
595        """Install "default" pw packages."""
596
597        result = result_func()
598
599        if not self._pw_packages:
600            return result(_Result.Status.SKIPPED)
601
602        logdir = os.path.join(self._install_dir, 'packages')
603        if not os.path.isdir(logdir):
604            os.makedirs(logdir)
605
606        for pkg in self._pw_packages:
607            print('installing {}'.format(pkg))
608            cmd = ['pw', 'package', 'install', pkg]
609
610            log = os.path.join(logdir, '{}.log'.format(pkg))
611            try:
612                with open(log, 'w') as outs, self._env():
613                    print(*cmd, file=outs)
614                    subprocess.check_call(cmd,
615                                          cwd=self._project_root,
616                                          stdout=outs,
617                                          stderr=subprocess.STDOUT)
618            except subprocess.CalledProcessError:
619                with open(log, 'r') as ins:
620                    sys.stderr.write(ins.read())
621                    raise
622
623        return result(_Result.Status.DONE)
624
625    def host_tools(self, unused_spin):
626        # The host tools are grabbed from CIPD, at least initially. If the
627        # user has a current host build, that build will be used instead.
628        # TODO(mohrr) find a way to do stuff like this for all projects.
629        host_dir = os.path.join(self._pw_root, 'out', 'host')
630        self._env.prepend('PATH', os.path.join(host_dir, 'host_tools'))
631        return _Result(_Result.Status.DONE)
632
633    def win_scripts(self, unused_spin):
634        # These scripts act as a compatibility layer for windows.
635        env_setup_dir = os.path.join(self._pw_root, 'pw_env_setup')
636        self._env.prepend('PATH', os.path.join(env_setup_dir,
637                                               'windows_scripts'))
638        return _Result(_Result.Status.DONE)
639
640
641def parse(argv=None):
642    """Parse command-line arguments."""
643    parser = argparse.ArgumentParser()
644
645    pw_root = os.environ.get('PW_ROOT', None)
646    if not pw_root:
647        try:
648            with open(os.devnull, 'w') as outs:
649                pw_root = subprocess.check_output(
650                    ['git', 'rev-parse', '--show-toplevel'],
651                    stderr=outs).strip()
652        except subprocess.CalledProcessError:
653            pw_root = None
654
655    parser.add_argument(
656        '--pw-root',
657        default=pw_root,
658        required=not pw_root,
659    )
660
661    project_root = os.environ.get('PW_PROJECT_ROOT', None) or pw_root
662
663    parser.add_argument(
664        '--project-root',
665        default=project_root,
666        required=not project_root,
667    )
668
669    parser.add_argument(
670        '--cipd-cache-dir',
671        default=os.environ.get('CIPD_CACHE_DIR',
672                               os.path.expanduser('~/.cipd-cache-dir')),
673    )
674
675    parser.add_argument(
676        '--trust-cipd-hash',
677        action='store_true',
678        help='Only run the cipd executable if the ensure file or command-line '
679        'has changed. Defaults to false since files could have been deleted '
680        'from the installation directory and cipd would add them back.',
681    )
682
683    parser.add_argument(
684        '--shell-file',
685        help='Where to write the file for shells to source.',
686        required=True,
687    )
688
689    parser.add_argument(
690        '--quiet',
691        help='Reduce output.',
692        action='store_true',
693        default='PW_ENVSETUP_QUIET' in os.environ,
694    )
695
696    parser.add_argument(
697        '--install-dir',
698        help='Location to install environment.',
699        required=True,
700    )
701
702    parser.add_argument(
703        '--config-file',
704        help='JSON file describing CIPD and virtualenv requirements.',
705        type=argparse.FileType('r'),
706        required=True,
707    )
708
709    parser.add_argument(
710        '--virtualenv-gn-out-dir',
711        help=('Output directory to use when building and installing Python '
712              'packages with GN; defaults to a unique path in the environment '
713              'directory.'))
714
715    parser.add_argument(
716        '--virtualenv-root',
717        help=('Root of virtualenv directory. Default: '
718              '<install_dir>/pigweed-venv'),
719        default=None,
720    )
721
722    parser.add_argument('--json-file', help=argparse.SUPPRESS, default=None)
723
724    parser.add_argument(
725        '--use-existing-cipd',
726        help='Use cipd executable from the environment instead of fetching it.',
727        action='store_true',
728    )
729
730    parser.add_argument(
731        '--strict',
732        help='Fail if there are any warnings.',
733        action='store_true',
734    )
735
736    parser.add_argument(
737        '--unpin-pip-packages',
738        dest='use_pinned_pip_packages',
739        help='Do not use pins of pip packages.',
740        action='store_false',
741    )
742
743    parser.add_argument(
744        '--cipd-only',
745        help='Skip non-CIPD steps.',
746        action='store_true',
747    )
748
749    args = parser.parse_args(argv)
750
751    return args
752
753
754def main():
755    try:
756        return EnvSetup(**vars(parse())).setup()
757    except subprocess.CalledProcessError as err:
758        print()
759        print(err.output)
760        raise
761
762
763if __name__ == '__main__':
764    sys.exit(main())
765