1# Copyright 2020 The Pigweed Authors 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); you may not 4# use this file except in compliance with the License. You may obtain a copy of 5# the License at 6# 7# https://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12# License for the specific language governing permissions and limitations under 13# the License. 14"""Sets up a Python 3 virtualenv for Pigweed.""" 15 16from __future__ import print_function 17 18import contextlib 19import datetime 20import glob 21import os 22import platform 23import re 24import shutil 25import subprocess 26import sys 27import stat 28import tempfile 29 30# Grabbing datetime string once so it will always be the same for all GnTarget 31# objects. 32_DATETIME_STRING = datetime.datetime.now().strftime('%Y%m%d-%H%M%S') 33 34 35class GnTarget(object): # pylint: disable=useless-object-inheritance 36 def __init__(self, val): 37 self.directory, self.target = val.split('#', 1) 38 self.name = '-'.join( 39 (re.sub(r'\W+', '_', self.target).strip('_'), _DATETIME_STRING) 40 ) 41 42 43def git_stdout(*args, **kwargs): 44 """Run git, passing args as git params and kwargs to subprocess.""" 45 return subprocess.check_output(['git'] + list(args), **kwargs).strip() 46 47 48def git_repo_root(path='./'): 49 """Find git repository root.""" 50 try: 51 return git_stdout('-C', path, 'rev-parse', '--show-toplevel') 52 except subprocess.CalledProcessError: 53 return None 54 55 56class GitRepoNotFound(Exception): 57 """Git repository not found.""" 58 59 60def _installed_packages(venv_python): 61 cmd = (venv_python, '-m', 'pip', 'list', '--disable-pip-version-check') 62 output = subprocess.check_output(cmd).splitlines() 63 return set(x.split()[0].lower() for x in output[2:]) 64 65 66def _required_packages(requirements): 67 packages = set() 68 69 for req in requirements: 70 with open(req, 'r') as ins: 71 for line in ins: 72 line = line.strip() 73 if not line or line.startswith('#'): 74 continue 75 packages.add(line.split('=')[0]) 76 77 return packages 78 79 80def _check_call(args, **kwargs): 81 stdout = kwargs.get('stdout', sys.stdout) 82 83 with tempfile.TemporaryFile(mode='w+') as temp: 84 try: 85 kwargs['stdout'] = temp 86 kwargs['stderr'] = subprocess.STDOUT 87 print(args, kwargs, file=temp) 88 subprocess.check_call(args, **kwargs) 89 except subprocess.CalledProcessError: 90 temp.seek(0) 91 stdout.write(temp.read()) 92 raise 93 94 95def _find_files_by_name(roots, name, allow_nesting=False): 96 matches = [] 97 for root in roots: 98 for dirpart, dirs, files in os.walk(root): 99 if name in files: 100 matches.append(os.path.join(dirpart, name)) 101 # If this directory is a match don't recurse inside it looking 102 # for more matches. 103 if not allow_nesting: 104 dirs[:] = [] 105 106 # Filter directories starting with . to avoid searching unnecessary 107 # paths and finding files that should be hidden. 108 dirs[:] = [d for d in dirs if not d.startswith('.')] 109 return matches 110 111 112def _check_venv(python, version, venv_path, pyvenv_cfg): 113 if platform.system().lower() == 'windows': 114 return 115 116 # Check if the python location and version used for the existing virtualenv 117 # is the same as the python we're using. If it doesn't match, we need to 118 # delete the existing virtualenv and start again. 119 if os.path.exists(pyvenv_cfg): 120 pyvenv_values = {} 121 with open(pyvenv_cfg, 'r') as ins: 122 for line in ins: 123 key, value = line.strip().split(' = ', 1) 124 pyvenv_values[key] = value 125 pydir = os.path.dirname(python) 126 home = pyvenv_values.get('home') 127 if pydir != home and not pydir.startswith(venv_path): 128 shutil.rmtree(venv_path) 129 elif pyvenv_values.get('version') not in version: 130 shutil.rmtree(venv_path) 131 132 133def _check_python_install_permissions(python): 134 # These pickle files are not included on windows. 135 # The path on windows is environment/cipd/packages/python/bin/Lib/lib2to3/ 136 if platform.system().lower() == 'windows': 137 return 138 139 # Make any existing lib2to3 pickle files read+write. This is needed for 140 # importing yapf. 141 lib2to3_path = os.path.join( 142 os.path.dirname(os.path.dirname(python)), 'lib', 'python3.9', 'lib2to3' 143 ) 144 pickle_file_paths = [] 145 if os.path.isdir(lib2to3_path): 146 pickle_file_paths.extend( 147 file_path 148 for file_path in os.listdir(lib2to3_path) 149 if '.pickle' in file_path 150 ) 151 try: 152 for pickle_file in pickle_file_paths: 153 pickle_full_path = os.path.join(lib2to3_path, pickle_file) 154 os.chmod( 155 pickle_full_path, stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP 156 ) 157 except PermissionError: 158 pass 159 160 161def _flatten(*items): 162 """Yields items from a series of items and nested iterables.""" 163 164 for item in items: 165 if isinstance(item, (list, tuple)): 166 for i in _flatten(*item): 167 yield i 168 else: 169 yield item 170 171 172def install( # pylint: disable=too-many-arguments,too-many-locals 173 project_root, 174 venv_path, 175 full_envsetup=True, 176 requirements=None, 177 constraints=None, 178 gn_args=(), 179 gn_targets=(), 180 gn_out_dir=None, 181 python=sys.executable, 182 env=None, 183 system_packages=False, 184 use_pinned_pip_packages=True, 185): 186 """Creates a venv and installs all packages in this Git repo.""" 187 188 version = ( 189 subprocess.check_output((python, '--version'), stderr=subprocess.STDOUT) 190 .strip() 191 .decode() 192 ) 193 if ' 3.' not in version: 194 print('=' * 60, file=sys.stderr) 195 print('Unexpected Python version:', version, file=sys.stderr) 196 print('=' * 60, file=sys.stderr) 197 return False 198 199 # The bin/ directory is called Scripts/ on Windows. Don't ask. 200 venv_bin = os.path.join(venv_path, 'Scripts' if os.name == 'nt' else 'bin') 201 202 if env: 203 env.set('VIRTUAL_ENV', venv_path) 204 env.prepend('PATH', venv_bin) 205 env.clear('PYTHONHOME') 206 env.clear('__PYVENV_LAUNCHER__') 207 else: 208 env = contextlib.nullcontext() 209 210 # Delete activation scripts. Typically they're created read-only and venv 211 # will complain when trying to write over them fails. 212 if os.path.isdir(venv_bin): 213 for entry in os.listdir(venv_bin): 214 if entry.lower().startswith('activate'): 215 os.unlink(os.path.join(venv_bin, entry)) 216 217 pyvenv_cfg = os.path.join(venv_path, 'pyvenv.cfg') 218 219 _check_python_install_permissions(python) 220 _check_venv(python, version, venv_path, pyvenv_cfg) 221 222 if full_envsetup or not os.path.exists(pyvenv_cfg): 223 # On Mac sometimes the CIPD Python has __PYVENV_LAUNCHER__ set to 224 # point to the system Python, which causes CIPD Python to create 225 # virtualenvs that reference the system Python instead of the CIPD 226 # Python. Clearing __PYVENV_LAUNCHER__ fixes that. See also pwbug/59. 227 envcopy = os.environ.copy() 228 if '__PYVENV_LAUNCHER__' in envcopy: 229 del envcopy['__PYVENV_LAUNCHER__'] 230 231 # TODO(spang): Pass --upgrade-deps and remove pip & setuptools 232 # upgrade below. This can only be done once the minimum python 233 # version is at least 3.9. 234 cmd = [python, '-m', 'venv', '--upgrade'] 235 cmd += ['--system-site-packages'] if system_packages else [] 236 cmd += [venv_path] 237 _check_call(cmd, env=envcopy) 238 239 venv_python = os.path.join(venv_bin, 'python') 240 241 pw_root = os.environ.get('PW_ROOT') 242 if not pw_root and env: 243 pw_root = env.PW_ROOT 244 if not pw_root: 245 pw_root = git_repo_root() 246 if not pw_root: 247 raise GitRepoNotFound() 248 249 # Sometimes we get an error saying "Egg-link ... does not match 250 # installed location". This gets around that. The egg-link files 251 # all come from 'pw'-prefixed packages we installed with --editable. 252 # Source: https://stackoverflow.com/a/48972085 253 for egg_link in glob.glob( 254 os.path.join(venv_path, 'lib/python*/site-packages/*.egg-link') 255 ): 256 os.unlink(egg_link) 257 258 def pip_install(*args): 259 args = list(_flatten(args)) 260 with env(): 261 cmd = [venv_python, '-m', 'pip', 'install'] + args 262 return _check_call(cmd) 263 264 constraint_args = [] 265 if constraints: 266 constraint_args.extend( 267 '--constraint={}'.format(constraint) for constraint in constraints 268 ) 269 270 pip_install( 271 '--log', 272 os.path.join(venv_path, 'pip-upgrade.log'), 273 '--upgrade', 274 'pip', 275 'setuptools', 276 'toml', # Needed for pyproject.toml package installs. 277 # Include wheel so pip installs can be done without build 278 # isolation. 279 'wheel', 280 constraint_args, 281 ) 282 283 # TODO(tonymd): Remove this when projects have defined requirements. 284 if (not requirements) and constraints: 285 requirements = constraints 286 287 if requirements: 288 requirement_args = [] 289 # Note: --no-build-isolation should be avoided for installing 3rd party 290 # Python packages that use C/C++ extension modules. 291 # https://setuptools.pypa.io/en/latest/userguide/ext_modules.html 292 requirement_args.extend( 293 '--requirement={}'.format(req) for req in requirements 294 ) 295 combined_requirement_args = requirement_args + constraint_args 296 pip_install( 297 '--log', 298 os.path.join(venv_path, 'pip-requirements.log'), 299 combined_requirement_args, 300 ) 301 302 def install_packages(gn_target): 303 if gn_out_dir is None: 304 build_dir = os.path.join(venv_path, 'gn-install-dir') 305 else: 306 build_dir = gn_out_dir 307 308 env_log = 'env-{}.log'.format(gn_target.name) 309 env_log_path = os.path.join(venv_path, env_log) 310 with open(env_log_path, 'w') as outs: 311 for key, value in sorted(os.environ.items()): 312 if key.upper().endswith('PATH'): 313 print(key, '=', file=outs) 314 # pylint: disable=invalid-name 315 for v in value.split(os.pathsep): 316 print(' ', v, file=outs) 317 # pylint: enable=invalid-name 318 else: 319 print(key, '=', value, file=outs) 320 321 gn_log = 'gn-gen-{}.log'.format(gn_target.name) 322 gn_log_path = os.path.join(venv_path, gn_log) 323 try: 324 with open(gn_log_path, 'w') as outs: 325 gn_cmd = ['gn', 'gen', build_dir] 326 327 args = list(gn_args) 328 if not use_pinned_pip_packages: 329 args.append('pw_build_PIP_CONSTRAINTS=[]') 330 331 args.append('dir_pigweed="{}"'.format(pw_root)) 332 gn_cmd.append('--args={}'.format(' '.join(args))) 333 334 print(gn_cmd, file=outs) 335 subprocess.check_call( 336 gn_cmd, 337 cwd=os.path.join(project_root, gn_target.directory), 338 stdout=outs, 339 stderr=outs, 340 ) 341 except subprocess.CalledProcessError as err: 342 with open(gn_log_path, 'r') as ins: 343 raise subprocess.CalledProcessError( 344 err.returncode, err.cmd, ins.read() 345 ) 346 347 ninja_log = 'ninja-{}.log'.format(gn_target.name) 348 ninja_log_path = os.path.join(venv_path, ninja_log) 349 try: 350 with open(ninja_log_path, 'w') as outs: 351 ninja_cmd = ['ninja', '-C', build_dir, '-v'] 352 ninja_cmd.append(gn_target.target) 353 print(ninja_cmd, file=outs) 354 subprocess.check_call(ninja_cmd, stdout=outs, stderr=outs) 355 except subprocess.CalledProcessError as err: 356 with open(ninja_log_path, 'r') as ins: 357 raise subprocess.CalledProcessError( 358 err.returncode, err.cmd, ins.read() 359 ) 360 361 with open(os.path.join(venv_path, 'pip-list.log'), 'w') as outs: 362 subprocess.check_call( 363 [venv_python, '-m', 'pip', 'list'], 364 stdout=outs, 365 ) 366 367 if gn_targets: 368 with env(): 369 for gn_target in gn_targets: 370 install_packages(gn_target) 371 372 return True 373