• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1# Copyright 2020 The Pigweed Authors
2#
3# Licensed under the Apache License, Version 2.0 (the "License"); you may not
4# use this file except in compliance with the License. You may obtain a copy of
5# the License at
6#
7#     https://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
11# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
12# License for the specific language governing permissions and limitations under
13# the License.
14"""Sets up a Python 3 virtualenv for Pigweed."""
15
16from __future__ import print_function
17
18import contextlib
19import datetime
20import glob
21import os
22import platform
23import re
24import shutil
25import subprocess
26import sys
27import tempfile
28
29# Grabbing datetime string once so it will always be the same for all GnTarget
30# objects.
31_DATETIME_STRING = datetime.datetime.now().strftime('%Y%m%d-%H%M%S')
32
33
34class GnTarget(object):  # pylint: disable=useless-object-inheritance
35    def __init__(self, val):
36        self.directory, self.target = val.split('#', 1)
37        self.name = '-'.join(
38            (re.sub(r'\W+', '_', self.target).strip('_'), _DATETIME_STRING))
39
40
41def git_stdout(*args, **kwargs):
42    """Run git, passing args as git params and kwargs to subprocess."""
43    return subprocess.check_output(['git'] + list(args), **kwargs).strip()
44
45
46def git_repo_root(path='./'):
47    """Find git repository root."""
48    try:
49        return git_stdout('-C', path, 'rev-parse', '--show-toplevel')
50    except subprocess.CalledProcessError:
51        return None
52
53
54class GitRepoNotFound(Exception):
55    """Git repository not found."""
56
57
58def _installed_packages(venv_python):
59    cmd = (venv_python, '-m', 'pip', 'list', '--disable-pip-version-check')
60    output = subprocess.check_output(cmd).splitlines()
61    return set(x.split()[0].lower() for x in output[2:])
62
63
64def _required_packages(requirements):
65    packages = set()
66
67    for req in requirements:
68        with open(req, 'r') as ins:
69            for line in ins:
70                line = line.strip()
71                if not line or line.startswith('#'):
72                    continue
73                packages.add(line.split('=')[0])
74
75    return packages
76
77
78# TODO(pwbug/135) Move to common utility module.
79def _check_call(args, **kwargs):
80    stdout = kwargs.get('stdout', sys.stdout)
81
82    with tempfile.TemporaryFile(mode='w+') as temp:
83        try:
84            kwargs['stdout'] = temp
85            kwargs['stderr'] = subprocess.STDOUT
86            print(args, kwargs, file=temp)
87            subprocess.check_call(args, **kwargs)
88        except subprocess.CalledProcessError:
89            temp.seek(0)
90            stdout.write(temp.read())
91            raise
92
93
94def _find_files_by_name(roots, name, allow_nesting=False):
95    matches = []
96    for root in roots:
97        for dirpart, dirs, files in os.walk(root):
98            if name in files:
99                matches.append(os.path.join(dirpart, name))
100                # If this directory is a match don't recurse inside it looking
101                # for more matches.
102                if not allow_nesting:
103                    dirs[:] = []
104
105            # Filter directories starting with . to avoid searching unnecessary
106            # paths and finding files that should be hidden.
107            dirs[:] = [d for d in dirs if not d.startswith('.')]
108    return matches
109
110
111def _check_venv(python, version, venv_path, pyvenv_cfg):
112    # TODO(pwbug/400) Re-enable this check on Windows.
113    if platform.system().lower() == 'windows':
114        return
115
116    # Check if the python location and version used for the existing virtualenv
117    # is the same as the python we're using. If it doesn't match, we need to
118    # delete the existing virtualenv and start again.
119    if os.path.exists(pyvenv_cfg):
120        pyvenv_values = {}
121        with open(pyvenv_cfg, 'r') as ins:
122            for line in ins:
123                key, value = line.strip().split(' = ', 1)
124                pyvenv_values[key] = value
125        pydir = os.path.dirname(python)
126        home = pyvenv_values.get('home')
127        if pydir != home and not pydir.startswith(venv_path):
128            shutil.rmtree(venv_path)
129        elif pyvenv_values.get('version') not in version:
130            shutil.rmtree(venv_path)
131
132
133def install(  # pylint: disable=too-many-arguments
134    project_root,
135    venv_path,
136    full_envsetup=True,
137    requirements=(),
138    gn_args=(),
139    gn_targets=(),
140    gn_out_dir=None,
141    python=sys.executable,
142    env=None,
143    system_packages=False,
144    use_pinned_pip_packages=True,
145):
146    """Creates a venv and installs all packages in this Git repo."""
147
148    version = subprocess.check_output(
149        (python, '--version'), stderr=subprocess.STDOUT).strip().decode()
150    # We expect Python 3.8, but if it came from CIPD let it pass anyway.
151    if ('3.8' not in version and '3.9' not in version
152            and 'chromium' not in version):
153        print('=' * 60, file=sys.stderr)
154        print('Unexpected Python version:', version, file=sys.stderr)
155        print('=' * 60, file=sys.stderr)
156        return False
157
158    # The bin/ directory is called Scripts/ on Windows. Don't ask.
159    venv_bin = os.path.join(venv_path, 'Scripts' if os.name == 'nt' else 'bin')
160
161    if env:
162        env.set('VIRTUAL_ENV', venv_path)
163        env.prepend('PATH', venv_bin)
164        env.clear('PYTHONHOME')
165        env.clear('__PYVENV_LAUNCHER__')
166    else:
167        env = contextlib.nullcontext()
168
169    # Delete activation scripts. Typically they're created read-only and venv
170    # will complain when trying to write over them fails.
171    if os.path.isdir(venv_bin):
172        for entry in os.listdir(venv_bin):
173            if entry.lower().startswith('activate'):
174                os.unlink(os.path.join(venv_bin, entry))
175
176    pyvenv_cfg = os.path.join(venv_path, 'pyvenv.cfg')
177
178    _check_venv(python, version, venv_path, pyvenv_cfg)
179
180    if full_envsetup or not os.path.exists(pyvenv_cfg):
181        # On Mac sometimes the CIPD Python has __PYVENV_LAUNCHER__ set to
182        # point to the system Python, which causes CIPD Python to create
183        # virtualenvs that reference the system Python instead of the CIPD
184        # Python. Clearing __PYVENV_LAUNCHER__ fixes that. See also pwbug/59.
185        envcopy = os.environ.copy()
186        if '__PYVENV_LAUNCHER__' in envcopy:
187            del envcopy['__PYVENV_LAUNCHER__']
188
189        # TODO(spang): Pass --upgrade-deps and remove pip & setuptools
190        # upgrade below. This can only be done once the minimum python
191        # version is at least 3.9.
192        cmd = [python, '-m', 'venv', '--upgrade']
193        cmd += ['--system-site-packages'] if system_packages else []
194        cmd += [venv_path]
195        _check_call(cmd, env=envcopy)
196
197    venv_python = os.path.join(venv_bin, 'python')
198
199    pw_root = os.environ.get('PW_ROOT')
200    if not pw_root and env:
201        pw_root = env.PW_ROOT
202    if not pw_root:
203        pw_root = git_repo_root()
204    if not pw_root:
205        raise GitRepoNotFound()
206
207    # Sometimes we get an error saying "Egg-link ... does not match
208    # installed location". This gets around that. The egg-link files
209    # all come from 'pw'-prefixed packages we installed with --editable.
210    # Source: https://stackoverflow.com/a/48972085
211    for egg_link in glob.glob(
212            os.path.join(venv_path, 'lib/python*/site-packages/*.egg-link')):
213        os.unlink(egg_link)
214
215    def pip_install(*args):
216        with env():
217            cmd = [venv_python, '-m', 'pip', 'install'] + list(args)
218            return _check_call(cmd)
219
220    pip_install(
221        '--log',
222        os.path.join(venv_path, 'pip-upgrade.log'),
223        '--upgrade',
224        'pip',
225        'setuptools',
226        # Include wheel so pip installs can be done without build
227        # isolation.
228        'wheel')
229
230    if requirements:
231        requirement_args = tuple('--requirement={}'.format(req)
232                                 for req in requirements)
233        pip_install('--log', os.path.join(venv_path, 'pip-requirements.log'),
234                    *requirement_args)
235
236    def install_packages(gn_target):
237        if gn_out_dir is None:
238            build_dir = os.path.join(venv_path, 'gn-install-dir')
239        else:
240            build_dir = gn_out_dir
241
242        env_log = 'env-{}.log'.format(gn_target.name)
243        env_log_path = os.path.join(venv_path, env_log)
244        with open(env_log_path, 'w') as outs:
245            for key, value in sorted(os.environ.items()):
246                if key.upper().endswith('PATH'):
247                    print(key, '=', file=outs)
248                    # pylint: disable=invalid-name
249                    for v in value.split(os.pathsep):
250                        print('   ', v, file=outs)
251                    # pylint: enable=invalid-name
252                else:
253                    print(key, '=', value, file=outs)
254
255        gn_log = 'gn-gen-{}.log'.format(gn_target.name)
256        gn_log_path = os.path.join(venv_path, gn_log)
257        try:
258            with open(gn_log_path, 'w') as outs:
259                gn_cmd = ['gn', 'gen', build_dir]
260
261                args = list(gn_args)
262                if not use_pinned_pip_packages:
263                    args.append('pw_build_PIP_CONSTRAINTS=[]')
264
265                args.append('dir_pigweed="{}"'.format(pw_root))
266                gn_cmd.append('--args={}'.format(' '.join(args)))
267
268                print(gn_cmd, file=outs)
269                subprocess.check_call(gn_cmd,
270                                      cwd=os.path.join(project_root,
271                                                       gn_target.directory),
272                                      stdout=outs,
273                                      stderr=outs)
274        except subprocess.CalledProcessError as err:
275            with open(gn_log_path, 'r') as ins:
276                raise subprocess.CalledProcessError(err.returncode, err.cmd,
277                                                    ins.read())
278
279        ninja_log = 'ninja-{}.log'.format(gn_target.name)
280        ninja_log_path = os.path.join(venv_path, ninja_log)
281        try:
282            with open(ninja_log_path, 'w') as outs:
283                ninja_cmd = ['ninja', '-C', build_dir, '-v']
284                ninja_cmd.append(gn_target.target)
285                print(ninja_cmd, file=outs)
286                subprocess.check_call(ninja_cmd, stdout=outs, stderr=outs)
287        except subprocess.CalledProcessError as err:
288            with open(ninja_log_path, 'r') as ins:
289                raise subprocess.CalledProcessError(err.returncode, err.cmd,
290                                                    ins.read())
291
292        with open(os.path.join(venv_path, 'pip-list.log'), 'w') as outs:
293            subprocess.check_call(
294                [venv_python, '-m', 'pip', 'list'],
295                stdout=outs,
296            )
297
298    if gn_targets:
299        with env():
300            for gn_target in gn_targets:
301                install_packages(gn_target)
302
303    return True
304