• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/env python3
2# Copyright 2019 The Pigweed Authors
3#
4# Licensed under the Apache License, Version 2.0 (the "License"); you may not
5# use this file except in compliance with the License. You may obtain a copy of
6# the License at
7#
8#     https://www.apache.org/licenses/LICENSE-2.0
9#
10# Unless required by applicable law or agreed to in writing, software
11# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
12# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
13# License for the specific language governing permissions and limitations under
14# the License.
15"""Checks if the environment is set up correctly for Pigweed."""
16
17import argparse
18from concurrent import futures
19import logging
20import json
21import os
22import pathlib
23import shutil
24import subprocess
25import sys
26import tempfile
27from typing import Callable, Iterable, List, Set
28
29import pw_cli.pw_command_plugins
30import pw_env_setup.cipd_setup.update as cipd_update
31
32
33def call_stdout(*args, **kwargs):
34    kwargs.update(stdout=subprocess.PIPE)
35    proc = subprocess.run(*args, **kwargs)
36    return proc.stdout.decode('utf-8')
37
38
39class _Fatal(Exception):
40    pass
41
42
43class Doctor:
44    def __init__(self, *, log: logging.Logger = None, strict: bool = False):
45        self.strict = strict
46        self.log = log or logging.getLogger(__name__)
47        self.failures: Set[str] = set()
48
49    def run(self, checks: Iterable[Callable]):
50        with futures.ThreadPoolExecutor() as executor:
51            futures.wait([
52                executor.submit(self._run_check, c, executor) for c in checks
53            ])
54
55    def _run_check(self, check, executor):
56        ctx = DoctorContext(self, check.__name__, executor)
57        try:
58            self.log.debug('Running check %s', ctx.check)
59            check(ctx)
60            ctx.wait()
61        except _Fatal:
62            pass
63        except:  # pylint: disable=bare-except
64            self.failures.add(ctx.check)
65            self.log.exception('%s failed with an unexpected exception',
66                               check.__name__)
67
68        self.log.debug('Completed check %s', ctx.check)
69
70
71class DoctorContext:
72    """The context object provided to each context function."""
73    def __init__(self, doctor: Doctor, check: str, executor: futures.Executor):
74        self._doctor = doctor
75        self.check = check
76        self._executor = executor
77        self._futures: List[futures.Future] = []
78
79    def submit(self, function, *args, **kwargs):
80        """Starts running the provided function in parallel."""
81        self._futures.append(
82            self._executor.submit(self._run_job, function, *args, **kwargs))
83
84    def wait(self):
85        """Waits for all parallel tasks started with submit() to complete."""
86        futures.wait(self._futures)
87        self._futures.clear()
88
89    def _run_job(self, function, *args, **kwargs):
90        try:
91            function(*args, **kwargs)
92        except _Fatal:
93            pass
94        except:  # pylint: disable=bare-except
95            self._doctor.failures.add(self.check)
96            self._doctor.log.exception(
97                '%s failed with an unexpected exception', self.check)
98
99    def fatal(self, fmt, *args, **kwargs):
100        """Same as error() but terminates the check early."""
101        self.error(fmt, *args, **kwargs)
102        raise _Fatal()
103
104    def error(self, fmt, *args, **kwargs):
105        self._doctor.log.error(fmt, *args, **kwargs)
106        self._doctor.failures.add(self.check)
107
108    def warning(self, fmt, *args, **kwargs):
109        if self._doctor.strict:
110            self.error(fmt, *args, **kwargs)
111        else:
112            self._doctor.log.warning(fmt, *args, **kwargs)
113
114    def info(self, fmt, *args, **kwargs):
115        self._doctor.log.info(fmt, *args, **kwargs)
116
117    def debug(self, fmt, *args, **kwargs):
118        self._doctor.log.debug(fmt, *args, **kwargs)
119
120
121def register_into(dest):
122    def decorate(func):
123        dest.append(func)
124        return func
125
126    return decorate
127
128
129CHECKS: List[Callable] = []
130
131
132@register_into(CHECKS)
133def pw_plugins(ctx: DoctorContext):
134    if pw_cli.pw_command_plugins.errors():
135        ctx.error('Not all pw plugins loaded successfully')
136
137
138@register_into(CHECKS)
139def env_os(ctx: DoctorContext):
140    """Check that the environment matches this machine."""
141    if '_PW_ACTUAL_ENVIRONMENT_ROOT' not in os.environ:
142        return
143    env_root = pathlib.Path(os.environ['_PW_ACTUAL_ENVIRONMENT_ROOT'])
144    config = env_root / 'config.json'
145    if not config.is_file():
146        return
147
148    with open(config, 'r') as ins:
149        data = json.load(ins)
150    if data['os'] != os.name:
151        ctx.error('Current OS (%s) does not match bootstrapped OS (%s)',
152                  os.name, data['os'])
153
154    # Skipping sysname and nodename in os.uname(). nodename could change
155    # based on the current network. sysname won't change, but is
156    # redundant because it's contained in release or version, and
157    # skipping it here simplifies logic.
158    uname = ' '.join(getattr(os, 'uname', lambda: ())()[2:])
159    if data['uname'] != uname:
160        ctx.warning(
161            'Current uname (%s) does not match Bootstrap uname (%s), '
162            'you may need to rerun bootstrap on this system', uname,
163            data['uname'])
164
165
166@register_into(CHECKS)
167def pw_root(ctx: DoctorContext):
168    """Check that environment variable PW_ROOT is set and makes sense."""
169    try:
170        root = pathlib.Path(os.environ['PW_ROOT']).resolve()
171    except KeyError:
172        ctx.fatal('PW_ROOT not set')
173
174    git_root = pathlib.Path(
175        call_stdout(['git', 'rev-parse', '--show-toplevel'], cwd=root).strip())
176    git_root = git_root.resolve()
177    if root != git_root:
178        ctx.error('PW_ROOT (%s) != `git rev-parse --show-toplevel` (%s)', root,
179                  git_root)
180
181
182@register_into(CHECKS)
183def git_hook(ctx: DoctorContext):
184    """Check that presubmit git hook is installed."""
185    if not os.environ.get('PW_ENABLE_PRESUBMIT_HOOK_WARNING'):
186        return
187
188    try:
189        root = pathlib.Path(os.environ['PW_ROOT'])
190    except KeyError:
191        return  # This case is handled elsewhere.
192
193    hook = root / '.git' / 'hooks' / 'pre-push'
194    if not os.path.isfile(hook):
195        ctx.info('Presubmit hook not installed, please run '
196                 "'pw presubmit --install' before pushing changes.")
197
198
199@register_into(CHECKS)
200def python_version(ctx: DoctorContext):
201    """Check the Python version is correct."""
202    actual = sys.version_info
203    expected = (3, 8)
204    latest = (3, 9)
205    if (actual[0:2] < expected or actual[0] != expected[0]
206            or actual[0:2] > latest):
207        # If we get the wrong version but it still came from CIPD print a
208        # warning but give it a pass.
209        if 'chromium' in sys.version:
210            ctx.warning('Python %d.%d.x expected, got Python %d.%d.%d',
211                        *expected, *actual[0:3])
212        else:
213            ctx.error('Python %d.%d.x required, got Python %d.%d.%d',
214                      *expected, *actual[0:3])
215
216
217@register_into(CHECKS)
218def virtualenv(ctx: DoctorContext):
219    """Check that we're in the correct virtualenv."""
220    try:
221        venv_path = pathlib.Path(os.environ['VIRTUAL_ENV']).resolve()
222    except KeyError:
223        ctx.error('VIRTUAL_ENV not set')
224        return
225
226    # When running in LUCI we might not have gone through the normal environment
227    # setup process, so we need to skip the rest of this step.
228    if 'LUCI_CONTEXT' in os.environ:
229        return
230
231    var = 'PW_ROOT'
232    if '_PW_ACTUAL_ENVIRONMENT_ROOT' in os.environ:
233        var = '_PW_ACTUAL_ENVIRONMENT_ROOT'
234    root = pathlib.Path(os.environ[var]).resolve()
235
236    if root not in venv_path.parents:
237        ctx.error('VIRTUAL_ENV (%s) not inside %s (%s)', venv_path, var, root)
238        ctx.error('\n'.join(os.environ.keys()))
239
240
241@register_into(CHECKS)
242def cipd(ctx: DoctorContext):
243    """Check cipd is set up correctly and in use."""
244    if os.environ.get('PW_DOCTOR_SKIP_CIPD_CHECKS'):
245        return
246
247    cipd_path = 'pigweed'
248
249    cipd_exe = shutil.which('cipd')
250    if not cipd_exe:
251        ctx.fatal('cipd not in PATH (%s)', os.environ['PATH'])
252
253    temp = tempfile.NamedTemporaryFile(prefix='cipd', delete=False)
254    subprocess.run(['cipd', 'acl-check', '-json-output', temp.name, cipd_path],
255                   stdout=subprocess.PIPE)
256    if not json.load(temp)['result']:
257        ctx.fatal(
258            "can't access %s CIPD directory, have you run "
259            "'cipd auth-login'?", cipd_path)
260
261    commands_expected_from_cipd = [
262        'arm-none-eabi-gcc',
263        'gn',
264        'ninja',
265        'protoc',
266    ]
267
268    # TODO(mohrr) get these tools in CIPD for Windows.
269    if os.name == 'posix':
270        commands_expected_from_cipd += [
271            'bloaty',
272            'clang++',
273        ]
274
275    for command in commands_expected_from_cipd:
276        path = shutil.which(command)
277        if path is None:
278            ctx.error('could not find %s in PATH (%s)', command,
279                      os.environ['PATH'])
280        elif 'cipd' not in path:
281            ctx.warning('not using %s from cipd, got %s (path is %s)', command,
282                        path, os.environ['PATH'])
283
284
285@register_into(CHECKS)
286def cipd_versions(ctx: DoctorContext):
287    """Check cipd tool versions are current."""
288
289    if os.environ.get('PW_DOCTOR_SKIP_CIPD_CHECKS'):
290        return
291
292    if 'PW_CIPD_INSTALL_DIR' not in os.environ:
293        ctx.error('PW_CIPD_INSTALL_DIR not set')
294    cipd_dir = pathlib.Path(os.environ['PW_CIPD_INSTALL_DIR'])
295
296    with open(cipd_dir / '_all_package_files.json', 'r') as ins:
297        json_paths = [pathlib.Path(x) for x in json.load(ins)]
298
299    platform = cipd_update.platform()
300
301    def check_cipd(package, install_path):
302        if platform not in package['platforms']:
303            ctx.debug("skipping %s because it doesn't apply to %s",
304                      package['path'], platform)
305            return
306
307        tags_without_refs = [x for x in package['tags'] if ':' in x]
308        if not tags_without_refs:
309            ctx.debug('skipping %s because it tracks a ref, not a tag (%s)',
310                      package['path'], ', '.join(package['tags']))
311            return
312
313        ctx.debug('checking version of %s', package['path'])
314
315        name = [
316            part for part in package['path'].split('/') if '{' not in part
317        ][-1]
318
319        # If the exact path is specified in the JSON file use it, and require it
320        # exist.
321        if 'version_file' in package:
322            path = install_path / package['version_file']
323            if not path.is_file():
324                ctx.error(f'no version file for {name} at {path}')
325                return
326
327        # Otherwise, follow a heuristic to find the file but don't require the
328        # file to exist.
329        else:
330            path = install_path / '.versions' / f'{name}.cipd_version'
331            if not path.is_file():
332                ctx.debug(f'no version file for {name} at {path}')
333                return
334
335        with path.open() as ins:
336            installed = json.load(ins)
337        ctx.debug(f'found version file for {name} at {path}')
338
339        describe = (
340            'cipd',
341            'describe',
342            installed['package_name'],
343            '-version',
344            installed['instance_id'],
345        )
346        ctx.debug('%s', ' '.join(describe))
347        output_raw = subprocess.check_output(describe).decode()
348        ctx.debug('output: %r', output_raw)
349        output = output_raw.split()
350
351        for tag in package['tags']:
352            if tag not in output:
353                ctx.error(
354                    'CIPD package %s in %s is out of date, please rerun '
355                    'bootstrap', installed['package_name'], install_path)
356
357            else:
358                ctx.debug('CIPD package %s in %s is current',
359                          installed['package_name'], install_path)
360
361    for json_path in json_paths:
362        ctx.debug(f'Checking packages in {json_path}')
363        install_path = pathlib.Path(
364            cipd_update.package_installation_path(cipd_dir, json_path))
365        for package in json.loads(json_path.read_text()).get('packages', ()):
366            ctx.submit(check_cipd, package, install_path)
367
368
369@register_into(CHECKS)
370def symlinks(ctx: DoctorContext):
371    """Check that the platform supports symlinks."""
372
373    try:
374        root = pathlib.Path(os.environ['PW_ROOT']).resolve()
375    except KeyError:
376        return  # This case is handled elsewhere.
377
378    with tempfile.TemporaryDirectory() as tmpdir:
379        dest = pathlib.Path(tmpdir).resolve() / 'symlink'
380        try:
381            os.symlink(root, dest)
382            failure = False
383        except OSError:
384            # TODO(pwbug/500) Find out what errno is set when symlinks aren't
385            # supported by the OS.
386            failure = True
387
388        if not os.path.islink(dest) or failure:
389            ctx.warning(
390                'Symlinks are not supported or current user does not have '
391                'permission to use them. This may cause build issues. If on '
392                'Windows, turn on Development Mode to enable symlink support.')
393
394
395def run_doctor(strict=False, checks=None):
396    """Run all the Check subclasses defined in this file."""
397
398    if checks is None:
399        checks = tuple(CHECKS)
400
401    doctor = Doctor(strict=strict)
402    doctor.log.debug('Doctor running %d checks...', len(checks))
403
404    doctor.run(checks)
405
406    if doctor.failures:
407        doctor.log.info('Failed checks: %s', ', '.join(doctor.failures))
408        doctor.log.info(
409            "Your environment setup has completed, but something isn't right "
410            'and some things may not work correctly. You may continue with '
411            'development, but please seek support at '
412            'https://bugs.pigweed.dev/ or by reaching out to your team.')
413    else:
414        doctor.log.info('Environment passes all checks!')
415    return len(doctor.failures)
416
417
418def main() -> int:
419    """Check that the environment is set up correctly for Pigweed."""
420    parser = argparse.ArgumentParser(description=__doc__)
421    parser.add_argument(
422        '--strict',
423        action='store_true',
424        help='Run additional checks.',
425    )
426
427    return run_doctor(**vars(parser.parse_args()))
428
429
430if __name__ == '__main__':
431    # By default, display log messages like a simple print statement.
432    logging.basicConfig(format='%(message)s', level=logging.INFO)
433    sys.exit(main())
434