1# Copyright 2020 The Pigweed Authors 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); you may not 4# use this file except in compliance with the License. You may obtain a copy of 5# the License at 6# 7# https://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12# License for the specific language governing permissions and limitations under 13# the License. 14"""Sets up a Python 3 virtualenv for Pigweed.""" 15 16from __future__ import print_function 17 18import contextlib 19import datetime 20import glob 21import os 22import platform 23import re 24import shutil 25import subprocess 26import sys 27import stat 28import tempfile 29 30# Grabbing datetime string once so it will always be the same for all GnTarget 31# objects. 32_DATETIME_STRING = datetime.datetime.now().strftime('%Y%m%d-%H%M%S') 33 34 35def _is_windows() -> bool: 36 return platform.system().lower() == 'windows' 37 38 39class GnTarget(object): # pylint: disable=useless-object-inheritance 40 def __init__(self, val): 41 self.directory, self.target = val.split('#', 1) 42 self.name = '-'.join( 43 (re.sub(r'\W+', '_', self.target).strip('_'), _DATETIME_STRING) 44 ) 45 46 47def git_stdout(*args, **kwargs): 48 """Run git, passing args as git params and kwargs to subprocess.""" 49 return subprocess.check_output(['git'] + list(args), **kwargs).strip() 50 51 52def git_repo_root(path='./'): 53 """Find git repository root.""" 54 try: 55 return git_stdout('-C', path, 'rev-parse', '--show-toplevel') 56 except subprocess.CalledProcessError: 57 return None 58 59 60class GitRepoNotFound(Exception): 61 """Git repository not found.""" 62 63 64def _installed_packages(venv_python): 65 cmd = (venv_python, '-m', 'pip', '--disable-pip-version-check', 'list') 66 output = subprocess.check_output(cmd).splitlines() 67 return set(x.split()[0].lower() for x in output[2:]) 68 69 70def _required_packages(requirements): 71 packages = set() 72 73 for req in requirements: 74 with open(req, 'r') as ins: 75 for line in ins: 76 line = line.strip() 77 if not line or line.startswith('#'): 78 continue 79 packages.add(line.split('=')[0]) 80 81 return packages 82 83 84def _check_call(args, **kwargs): 85 stdout = kwargs.get('stdout', sys.stdout) 86 87 with tempfile.TemporaryFile(mode='w+') as temp: 88 try: 89 kwargs['stdout'] = temp 90 kwargs['stderr'] = subprocess.STDOUT 91 print(args, kwargs, file=temp) 92 subprocess.check_call(args, **kwargs) 93 except subprocess.CalledProcessError: 94 temp.seek(0) 95 stdout.write(temp.read()) 96 raise 97 98 99def _find_files_by_name(roots, name, allow_nesting=False): 100 matches = [] 101 for root in roots: 102 for dirpart, dirs, files in os.walk(root): 103 if name in files: 104 matches.append(os.path.join(dirpart, name)) 105 # If this directory is a match don't recurse inside it looking 106 # for more matches. 107 if not allow_nesting: 108 dirs[:] = [] 109 110 # Filter directories starting with . to avoid searching unnecessary 111 # paths and finding files that should be hidden. 112 dirs[:] = [d for d in dirs if not d.startswith('.')] 113 return matches 114 115 116def _check_venv(python, version, venv_path, pyvenv_cfg): 117 if _is_windows(): 118 return 119 120 # Check if the python location and version used for the existing virtualenv 121 # is the same as the python we're using. If it doesn't match, we need to 122 # delete the existing virtualenv and start again. 123 if os.path.exists(pyvenv_cfg): 124 pyvenv_values = {} 125 with open(pyvenv_cfg, 'r') as ins: 126 for line in ins: 127 key, value = line.strip().split(' = ', 1) 128 pyvenv_values[key] = value 129 pydir = os.path.dirname(python) 130 home = pyvenv_values.get('home') 131 if pydir != home and not pydir.startswith(venv_path): 132 shutil.rmtree(venv_path) 133 elif pyvenv_values.get('version') not in '.'.join(map(str, version)): 134 shutil.rmtree(venv_path) 135 136 137def _check_python_install_permissions(python): 138 # These pickle files are not included on windows. 139 # The path on windows is environment/cipd/packages/python/bin/Lib/lib2to3/ 140 if _is_windows(): 141 return 142 143 # Make any existing lib2to3 pickle files read+write. This is needed for 144 # importing yapf. 145 lib2to3_path = os.path.join( 146 os.path.dirname(os.path.dirname(python)), 'lib', 'python3.9', 'lib2to3' 147 ) 148 pickle_file_paths = [] 149 if os.path.isdir(lib2to3_path): 150 pickle_file_paths.extend( 151 file_path 152 for file_path in os.listdir(lib2to3_path) 153 if '.pickle' in file_path 154 ) 155 try: 156 for pickle_file in pickle_file_paths: 157 pickle_full_path = os.path.join(lib2to3_path, pickle_file) 158 os.chmod( 159 pickle_full_path, stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP 160 ) 161 except PermissionError: 162 pass 163 164 165def _flatten(*items): 166 """Yields items from a series of items and nested iterables.""" 167 168 for item in items: 169 if isinstance(item, (list, tuple)): 170 for i in _flatten(*item): 171 yield i 172 else: 173 yield item 174 175 176def _python_version(python_path: str): 177 """Returns the version (major, minor, rev) of the `python_path` binary.""" 178 # Prints values like "3.10.0" 179 command = ( 180 python_path, 181 '-c', 182 'import sys; print(".".join(map(str, sys.version_info[:3])))', 183 ) 184 version_str = ( 185 subprocess.check_output(command, stderr=subprocess.STDOUT) 186 .strip() 187 .decode() 188 ) 189 return tuple(map(int, version_str.split('.'))) 190 191 192def install( # pylint: disable=too-many-arguments,too-many-locals,too-many-statements 193 project_root, 194 venv_path, 195 full_envsetup=True, 196 requirements=None, 197 constraints=None, 198 pip_install_disable_cache=None, 199 pip_install_find_links=None, 200 pip_install_offline=None, 201 pip_install_require_hashes=None, 202 gn_args=(), 203 gn_targets=(), 204 gn_out_dir=None, 205 python=sys.executable, 206 env=None, 207 system_packages=False, 208 use_pinned_pip_packages=True, 209): 210 """Creates a venv and installs all packages in this Git repo.""" 211 212 version = _python_version(python) 213 if version[0] != 3: 214 print('=' * 60, file=sys.stderr) 215 print('Unexpected Python version:', version, file=sys.stderr) 216 print('=' * 60, file=sys.stderr) 217 return False 218 219 # The bin/ directory is called Scripts/ on Windows. Don't ask. 220 venv_bin = os.path.join(venv_path, 'Scripts' if os.name == 'nt' else 'bin') 221 222 if env: 223 env.set('VIRTUAL_ENV', venv_path) 224 env.prepend('PATH', venv_bin) 225 env.clear('PYTHONHOME') 226 env.clear('__PYVENV_LAUNCHER__') 227 else: 228 env = contextlib.nullcontext() 229 230 # Virtual environments may contain read-only files (notably activate 231 # scripts). `venv` calls below will fail if they are not writeable. 232 if os.path.isdir(venv_path): 233 for root, _dirs, files in os.walk(venv_path): 234 for file in files: 235 path = os.path.join(root, file) 236 mode = os.lstat(path).st_mode 237 if not (stat.S_ISLNK(mode) or (mode & stat.S_IWRITE)): 238 os.chmod(path, mode | stat.S_IWRITE) 239 240 pyvenv_cfg = os.path.join(venv_path, 'pyvenv.cfg') 241 242 _check_python_install_permissions(python) 243 _check_venv(python, version, venv_path, pyvenv_cfg) 244 245 if full_envsetup or not os.path.exists(pyvenv_cfg): 246 # On Mac sometimes the CIPD Python has __PYVENV_LAUNCHER__ set to 247 # point to the system Python, which causes CIPD Python to create 248 # virtualenvs that reference the system Python instead of the CIPD 249 # Python. Clearing __PYVENV_LAUNCHER__ fixes that. See also pwbug/59. 250 envcopy = os.environ.copy() 251 if '__PYVENV_LAUNCHER__' in envcopy: 252 del envcopy['__PYVENV_LAUNCHER__'] 253 254 # TODO(spang): Pass --upgrade-deps and remove pip & setuptools 255 # upgrade below. This can only be done once the minimum python 256 # version is at least 3.9. 257 cmd = [python, '-m', 'venv'] 258 259 # Windows requires strange wizardry, and must follow symlinks 260 # starting with 3.11. 261 # 262 # Without this, windows fails bootstrap trying to copy 263 # "environment\cipd\packages\python\bin\venvlauncher.exe" 264 # 265 # This file doesn't exist in Python 3.11 on Windows and may be a bug 266 # in venv. Pigweed already uses symlinks on Windows for the GN build, 267 # so adding this option is not an issue. 268 # 269 # Further excitement is had when trying to update a virtual environment 270 # that is created using symlinks under Windows. `venv` will fail with 271 # and error that the source and destination are the same file. To work 272 # around this, we run `venv` in `--clear` mode under Windows. 273 if _is_windows() and version >= (3, 11): 274 cmd += ['--clear', '--symlinks'] 275 else: 276 cmd += ['--upgrade'] 277 278 cmd += ['--system-site-packages'] if system_packages else [] 279 cmd += [venv_path] 280 _check_call(cmd, env=envcopy) 281 282 venv_python = os.path.join(venv_bin, 'python') 283 284 pw_root = os.environ.get('PW_ROOT') 285 if not pw_root and env: 286 pw_root = env.PW_ROOT 287 if not pw_root: 288 pw_root = git_repo_root() 289 if not pw_root: 290 raise GitRepoNotFound() 291 292 # Sometimes we get an error saying "Egg-link ... does not match 293 # installed location". This gets around that. The egg-link files 294 # all come from 'pw'-prefixed packages we installed with --editable. 295 # Source: https://stackoverflow.com/a/48972085 296 for egg_link in glob.glob( 297 os.path.join(venv_path, 'lib/python*/site-packages/*.egg-link') 298 ): 299 os.unlink(egg_link) 300 301 pip_install_args = [] 302 if pip_install_find_links: 303 for package_dir in pip_install_find_links: 304 pip_install_args.append('--find-links') 305 with env(): 306 pip_install_args.append(os.path.expandvars(package_dir)) 307 if pip_install_require_hashes: 308 pip_install_args.append('--require-hashes') 309 if pip_install_offline: 310 pip_install_args.append('--no-index') 311 if pip_install_disable_cache: 312 pip_install_args.append('--no-cache-dir') 313 314 def pip_install(*args): 315 args = list(_flatten(args)) 316 with env(): 317 cmd = ( 318 [ 319 venv_python, 320 '-m', 321 'pip', 322 '--disable-pip-version-check', 323 'install', 324 ] 325 + pip_install_args 326 + args 327 ) 328 return _check_call(cmd) 329 330 constraint_args = [] 331 if constraints: 332 constraint_args.extend( 333 '--constraint={}'.format(constraint) for constraint in constraints 334 ) 335 336 pip_install( 337 '--log', 338 os.path.join(venv_path, 'pip-upgrade.log'), 339 '--upgrade', 340 'pip', 341 'setuptools', 342 'toml', # Needed for pyproject.toml package installs. 343 # Include wheel so pip installs can be done without build 344 # isolation. 345 'wheel', 346 'pip-tools', 347 constraint_args, 348 ) 349 350 # TODO(tonymd): Remove this when projects have defined requirements. 351 if (not requirements) and constraints: 352 requirements = constraints 353 354 if requirements: 355 requirement_args = [] 356 # Note: --no-build-isolation should be avoided for installing 3rd party 357 # Python packages that use C/C++ extension modules. 358 # https://setuptools.pypa.io/en/latest/userguide/ext_modules.html 359 requirement_args.extend( 360 '--requirement={}'.format(req) for req in requirements 361 ) 362 combined_requirement_args = requirement_args + constraint_args 363 pip_install( 364 '--log', 365 os.path.join(venv_path, 'pip-requirements.log'), 366 combined_requirement_args, 367 ) 368 369 def install_packages(gn_target): 370 if gn_out_dir is None: 371 build_dir = os.path.join(venv_path, 'gn') 372 else: 373 build_dir = gn_out_dir 374 375 env_log = 'env-{}.log'.format(gn_target.name) 376 env_log_path = os.path.join(venv_path, env_log) 377 with open(env_log_path, 'w') as outs: 378 for key, value in sorted(os.environ.items()): 379 if key.upper().endswith('PATH'): 380 print(key, '=', file=outs) 381 # pylint: disable=invalid-name 382 for v in value.split(os.pathsep): 383 print(' ', v, file=outs) 384 # pylint: enable=invalid-name 385 else: 386 print(key, '=', value, file=outs) 387 388 gn_log = 'gn-gen-{}.log'.format(gn_target.name) 389 gn_log_path = os.path.join(venv_path, gn_log) 390 try: 391 with open(gn_log_path, 'w') as outs: 392 gn_cmd = ['gn', 'gen', build_dir] 393 394 args = list(gn_args) 395 if not use_pinned_pip_packages: 396 args.append('pw_build_PIP_CONSTRAINTS=[]') 397 398 args.append('dir_pigweed="{}"'.format(pw_root)) 399 gn_cmd.append('--args={}'.format(' '.join(args))) 400 401 print(gn_cmd, file=outs) 402 subprocess.check_call( 403 gn_cmd, 404 cwd=os.path.join(project_root, gn_target.directory), 405 stdout=outs, 406 stderr=outs, 407 ) 408 except subprocess.CalledProcessError as err: 409 with open(gn_log_path, 'r') as ins: 410 raise subprocess.CalledProcessError( 411 err.returncode, err.cmd, ins.read() 412 ) 413 414 ninja_log = 'ninja-{}.log'.format(gn_target.name) 415 ninja_log_path = os.path.join(venv_path, ninja_log) 416 try: 417 with open(ninja_log_path, 'w') as outs: 418 ninja_cmd = ['ninja', '-C', build_dir, '-v'] 419 ninja_cmd.append(gn_target.target) 420 print(ninja_cmd, file=outs) 421 subprocess.check_call(ninja_cmd, stdout=outs, stderr=outs) 422 except subprocess.CalledProcessError as err: 423 with open(ninja_log_path, 'r') as ins: 424 raise subprocess.CalledProcessError( 425 err.returncode, err.cmd, ins.read() 426 ) 427 428 with open(os.path.join(venv_path, 'pip-list.log'), 'w') as outs: 429 subprocess.check_call( 430 [ 431 venv_python, 432 '-m', 433 'pip', 434 '--disable-pip-version-check', 435 'list', 436 ], 437 stdout=outs, 438 ) 439 440 if gn_targets: 441 with env(): 442 for gn_target in gn_targets: 443 install_packages(gn_target) 444 445 return True 446