1""" 2Virtual environment (venv) package for Python. Based on PEP 405. 3 4Copyright (C) 2011-2014 Vinay Sajip. 5Licensed to the PSF under a contributor agreement. 6""" 7import logging 8import os 9import shutil 10import subprocess 11import sys 12import sysconfig 13import types 14import shlex 15 16 17CORE_VENV_DEPS = ('pip', 'setuptools') 18logger = logging.getLogger(__name__) 19 20 21class EnvBuilder: 22 """ 23 This class exists to allow virtual environment creation to be 24 customized. The constructor parameters determine the builder's 25 behaviour when called upon to create a virtual environment. 26 27 By default, the builder makes the system (global) site-packages dir 28 *un*available to the created environment. 29 30 If invoked using the Python -m option, the default is to use copying 31 on Windows platforms but symlinks elsewhere. If instantiated some 32 other way, the default is to *not* use symlinks. 33 34 :param system_site_packages: If True, the system (global) site-packages 35 dir is available to created environments. 36 :param clear: If True, delete the contents of the environment directory if 37 it already exists, before environment creation. 38 :param symlinks: If True, attempt to symlink rather than copy files into 39 virtual environment. 40 :param upgrade: If True, upgrade an existing virtual environment. 41 :param with_pip: If True, ensure pip is installed in the virtual 42 environment 43 :param prompt: Alternative terminal prefix for the environment. 44 :param upgrade_deps: Update the base venv modules to the latest on PyPI 45 """ 46 47 def __init__(self, system_site_packages=False, clear=False, 48 symlinks=False, upgrade=False, with_pip=False, prompt=None, 49 upgrade_deps=False): 50 self.system_site_packages = system_site_packages 51 self.clear = clear 52 self.symlinks = symlinks 53 self.upgrade = upgrade 54 self.with_pip = with_pip 55 self.orig_prompt = prompt 56 if prompt == '.': # see bpo-38901 57 prompt = os.path.basename(os.getcwd()) 58 self.prompt = prompt 59 self.upgrade_deps = upgrade_deps 60 61 def create(self, env_dir): 62 """ 63 Create a virtual environment in a directory. 64 65 :param env_dir: The target directory to create an environment in. 66 67 """ 68 env_dir = os.path.abspath(env_dir) 69 context = self.ensure_directories(env_dir) 70 # See issue 24875. We need system_site_packages to be False 71 # until after pip is installed. 72 true_system_site_packages = self.system_site_packages 73 self.system_site_packages = False 74 self.create_configuration(context) 75 self.setup_python(context) 76 if self.with_pip: 77 self._setup_pip(context) 78 if not self.upgrade: 79 self.setup_scripts(context) 80 self.post_setup(context) 81 if true_system_site_packages: 82 # We had set it to False before, now 83 # restore it and rewrite the configuration 84 self.system_site_packages = True 85 self.create_configuration(context) 86 if self.upgrade_deps: 87 self.upgrade_dependencies(context) 88 89 def clear_directory(self, path): 90 for fn in os.listdir(path): 91 fn = os.path.join(path, fn) 92 if os.path.islink(fn) or os.path.isfile(fn): 93 os.remove(fn) 94 elif os.path.isdir(fn): 95 shutil.rmtree(fn) 96 97 def _venv_path(self, env_dir, name): 98 vars = { 99 'base': env_dir, 100 'platbase': env_dir, 101 'installed_base': env_dir, 102 'installed_platbase': env_dir, 103 } 104 return sysconfig.get_path(name, scheme='venv', vars=vars) 105 106 def ensure_directories(self, env_dir): 107 """ 108 Create the directories for the environment. 109 110 Returns a context object which holds paths in the environment, 111 for use by subsequent logic. 112 """ 113 114 def create_if_needed(d): 115 if not os.path.exists(d): 116 os.makedirs(d) 117 elif os.path.islink(d) or os.path.isfile(d): 118 raise ValueError('Unable to create directory %r' % d) 119 120 if os.pathsep in os.fspath(env_dir): 121 raise ValueError(f'Refusing to create a venv in {env_dir} because ' 122 f'it contains the PATH separator {os.pathsep}.') 123 if os.path.exists(env_dir) and self.clear: 124 self.clear_directory(env_dir) 125 context = types.SimpleNamespace() 126 context.env_dir = env_dir 127 context.env_name = os.path.split(env_dir)[1] 128 prompt = self.prompt if self.prompt is not None else context.env_name 129 context.prompt = '(%s) ' % prompt 130 create_if_needed(env_dir) 131 executable = sys._base_executable 132 if not executable: # see gh-96861 133 raise ValueError('Unable to determine path to the running ' 134 'Python interpreter. Provide an explicit path or ' 135 'check that your PATH environment variable is ' 136 'correctly set.') 137 dirname, exename = os.path.split(os.path.abspath(executable)) 138 context.executable = executable 139 context.python_dir = dirname 140 context.python_exe = exename 141 binpath = self._venv_path(env_dir, 'scripts') 142 incpath = self._venv_path(env_dir, 'include') 143 libpath = self._venv_path(env_dir, 'purelib') 144 145 context.inc_path = incpath 146 create_if_needed(incpath) 147 create_if_needed(libpath) 148 # Issue 21197: create lib64 as a symlink to lib on 64-bit non-OS X POSIX 149 if ((sys.maxsize > 2**32) and (os.name == 'posix') and 150 (sys.platform != 'darwin')): 151 link_path = os.path.join(env_dir, 'lib64') 152 if not os.path.exists(link_path): # Issue #21643 153 os.symlink('lib', link_path) 154 context.bin_path = binpath 155 context.bin_name = os.path.relpath(binpath, env_dir) 156 context.env_exe = os.path.join(binpath, exename) 157 create_if_needed(binpath) 158 # Assign and update the command to use when launching the newly created 159 # environment, in case it isn't simply the executable script (e.g. bpo-45337) 160 context.env_exec_cmd = context.env_exe 161 if sys.platform == 'win32': 162 # bpo-45337: Fix up env_exec_cmd to account for file system redirections. 163 # Some redirects only apply to CreateFile and not CreateProcess 164 real_env_exe = os.path.realpath(context.env_exe) 165 if os.path.normcase(real_env_exe) != os.path.normcase(context.env_exe): 166 logger.warning('Actual environment location may have moved due to ' 167 'redirects, links or junctions.\n' 168 ' Requested location: "%s"\n' 169 ' Actual location: "%s"', 170 context.env_exe, real_env_exe) 171 context.env_exec_cmd = real_env_exe 172 return context 173 174 def create_configuration(self, context): 175 """ 176 Create a configuration file indicating where the environment's Python 177 was copied from, and whether the system site-packages should be made 178 available in the environment. 179 180 :param context: The information for the environment creation request 181 being processed. 182 """ 183 context.cfg_path = path = os.path.join(context.env_dir, 'pyvenv.cfg') 184 with open(path, 'w', encoding='utf-8') as f: 185 f.write('home = %s\n' % context.python_dir) 186 if self.system_site_packages: 187 incl = 'true' 188 else: 189 incl = 'false' 190 f.write('include-system-site-packages = %s\n' % incl) 191 f.write('version = %d.%d.%d\n' % sys.version_info[:3]) 192 if self.prompt is not None: 193 f.write(f'prompt = {self.prompt!r}\n') 194 f.write('executable = %s\n' % os.path.realpath(sys.executable)) 195 args = [] 196 nt = os.name == 'nt' 197 if nt and self.symlinks: 198 args.append('--symlinks') 199 if not nt and not self.symlinks: 200 args.append('--copies') 201 if not self.with_pip: 202 args.append('--without-pip') 203 if self.system_site_packages: 204 args.append('--system-site-packages') 205 if self.clear: 206 args.append('--clear') 207 if self.upgrade: 208 args.append('--upgrade') 209 if self.upgrade_deps: 210 args.append('--upgrade-deps') 211 if self.orig_prompt is not None: 212 args.append(f'--prompt="{self.orig_prompt}"') 213 214 args.append(context.env_dir) 215 args = ' '.join(args) 216 f.write(f'command = {sys.executable} -m venv {args}\n') 217 218 if os.name != 'nt': 219 def symlink_or_copy(self, src, dst, relative_symlinks_ok=False): 220 """ 221 Try symlinking a file, and if that fails, fall back to copying. 222 """ 223 force_copy = not self.symlinks 224 if not force_copy: 225 try: 226 if not os.path.islink(dst): # can't link to itself! 227 if relative_symlinks_ok: 228 assert os.path.dirname(src) == os.path.dirname(dst) 229 os.symlink(os.path.basename(src), dst) 230 else: 231 os.symlink(src, dst) 232 except Exception: # may need to use a more specific exception 233 logger.warning('Unable to symlink %r to %r', src, dst) 234 force_copy = True 235 if force_copy: 236 shutil.copyfile(src, dst) 237 else: 238 def symlink_or_copy(self, src, dst, relative_symlinks_ok=False): 239 """ 240 Try symlinking a file, and if that fails, fall back to copying. 241 """ 242 bad_src = os.path.lexists(src) and not os.path.exists(src) 243 if self.symlinks and not bad_src and not os.path.islink(dst): 244 try: 245 if relative_symlinks_ok: 246 assert os.path.dirname(src) == os.path.dirname(dst) 247 os.symlink(os.path.basename(src), dst) 248 else: 249 os.symlink(src, dst) 250 return 251 except Exception: # may need to use a more specific exception 252 logger.warning('Unable to symlink %r to %r', src, dst) 253 254 # On Windows, we rewrite symlinks to our base python.exe into 255 # copies of venvlauncher.exe 256 basename, ext = os.path.splitext(os.path.basename(src)) 257 srcfn = os.path.join(os.path.dirname(__file__), 258 "scripts", 259 "nt", 260 basename + ext) 261 # Builds or venv's from builds need to remap source file 262 # locations, as we do not put them into Lib/venv/scripts 263 if sysconfig.is_python_build() or not os.path.isfile(srcfn): 264 if basename.endswith('_d'): 265 ext = '_d' + ext 266 basename = basename[:-2] 267 if basename == 'python': 268 basename = 'venvlauncher' 269 elif basename == 'pythonw': 270 basename = 'venvwlauncher' 271 src = os.path.join(os.path.dirname(src), basename + ext) 272 else: 273 src = srcfn 274 if not os.path.exists(src): 275 if not bad_src: 276 logger.warning('Unable to copy %r', src) 277 return 278 279 shutil.copyfile(src, dst) 280 281 def setup_python(self, context): 282 """ 283 Set up a Python executable in the environment. 284 285 :param context: The information for the environment creation request 286 being processed. 287 """ 288 binpath = context.bin_path 289 path = context.env_exe 290 copier = self.symlink_or_copy 291 dirname = context.python_dir 292 if os.name != 'nt': 293 copier(context.executable, path) 294 if not os.path.islink(path): 295 os.chmod(path, 0o755) 296 for suffix in ('python', 'python3', f'python3.{sys.version_info[1]}'): 297 path = os.path.join(binpath, suffix) 298 if not os.path.exists(path): 299 # Issue 18807: make copies if 300 # symlinks are not wanted 301 copier(context.env_exe, path, relative_symlinks_ok=True) 302 if not os.path.islink(path): 303 os.chmod(path, 0o755) 304 else: 305 if self.symlinks: 306 # For symlinking, we need a complete copy of the root directory 307 # If symlinks fail, you'll get unnecessary copies of files, but 308 # we assume that if you've opted into symlinks on Windows then 309 # you know what you're doing. 310 suffixes = [ 311 f for f in os.listdir(dirname) if 312 os.path.normcase(os.path.splitext(f)[1]) in ('.exe', '.dll') 313 ] 314 if sysconfig.is_python_build(): 315 suffixes = [ 316 f for f in suffixes if 317 os.path.normcase(f).startswith(('python', 'vcruntime')) 318 ] 319 else: 320 suffixes = {'python.exe', 'python_d.exe', 'pythonw.exe', 'pythonw_d.exe'} 321 base_exe = os.path.basename(context.env_exe) 322 suffixes.add(base_exe) 323 324 for suffix in suffixes: 325 src = os.path.join(dirname, suffix) 326 if os.path.lexists(src): 327 copier(src, os.path.join(binpath, suffix)) 328 329 if sysconfig.is_python_build(): 330 # copy init.tcl 331 for root, dirs, files in os.walk(context.python_dir): 332 if 'init.tcl' in files: 333 tcldir = os.path.basename(root) 334 tcldir = os.path.join(context.env_dir, 'Lib', tcldir) 335 if not os.path.exists(tcldir): 336 os.makedirs(tcldir) 337 src = os.path.join(root, 'init.tcl') 338 dst = os.path.join(tcldir, 'init.tcl') 339 shutil.copyfile(src, dst) 340 break 341 342 def _call_new_python(self, context, *py_args, **kwargs): 343 """Executes the newly created Python using safe-ish options""" 344 # gh-98251: We do not want to just use '-I' because that masks 345 # legitimate user preferences (such as not writing bytecode). All we 346 # really need is to ensure that the path variables do not overrule 347 # normal venv handling. 348 args = [context.env_exec_cmd, *py_args] 349 kwargs['env'] = env = os.environ.copy() 350 env['VIRTUAL_ENV'] = context.env_dir 351 env.pop('PYTHONHOME', None) 352 env.pop('PYTHONPATH', None) 353 kwargs['cwd'] = context.env_dir 354 kwargs['executable'] = context.env_exec_cmd 355 subprocess.check_output(args, **kwargs) 356 357 def _setup_pip(self, context): 358 """Installs or upgrades pip in a virtual environment""" 359 self._call_new_python(context, '-m', 'ensurepip', '--upgrade', 360 '--default-pip', stderr=subprocess.STDOUT) 361 362 def setup_scripts(self, context): 363 """ 364 Set up scripts into the created environment from a directory. 365 366 This method installs the default scripts into the environment 367 being created. You can prevent the default installation by overriding 368 this method if you really need to, or if you need to specify 369 a different location for the scripts to install. By default, the 370 'scripts' directory in the venv package is used as the source of 371 scripts to install. 372 """ 373 path = os.path.abspath(os.path.dirname(__file__)) 374 path = os.path.join(path, 'scripts') 375 self.install_scripts(context, path) 376 377 def post_setup(self, context): 378 """ 379 Hook for post-setup modification of the venv. Subclasses may install 380 additional packages or scripts here, add activation shell scripts, etc. 381 382 :param context: The information for the environment creation request 383 being processed. 384 """ 385 pass 386 387 def replace_variables(self, text, context): 388 """ 389 Replace variable placeholders in script text with context-specific 390 variables. 391 392 Return the text passed in , but with variables replaced. 393 394 :param text: The text in which to replace placeholder variables. 395 :param context: The information for the environment creation request 396 being processed. 397 """ 398 replacements = { 399 '__VENV_DIR__': context.env_dir, 400 '__VENV_NAME__': context.env_name, 401 '__VENV_PROMPT__': context.prompt, 402 '__VENV_BIN_NAME__': context.bin_name, 403 '__VENV_PYTHON__': context.env_exe, 404 } 405 406 def quote_ps1(s): 407 """ 408 This should satisfy PowerShell quoting rules [1], unless the quoted 409 string is passed directly to Windows native commands [2]. 410 [1]: https://learn.microsoft.com/en-us/powershell/module/microsoft.powershell.core/about/about_quoting_rules 411 [2]: https://learn.microsoft.com/en-us/powershell/module/microsoft.powershell.core/about/about_parsing#passing-arguments-that-contain-quote-characters 412 """ 413 s = s.replace("'", "''") 414 return f"'{s}'" 415 416 def quote_bat(s): 417 return s 418 419 # gh-124651: need to quote the template strings properly 420 quote = shlex.quote 421 script_path = context.script_path 422 if script_path.endswith('.ps1'): 423 quote = quote_ps1 424 elif script_path.endswith('.bat'): 425 quote = quote_bat 426 else: 427 # fallbacks to POSIX shell compliant quote 428 quote = shlex.quote 429 430 replacements = {key: quote(s) for key, s in replacements.items()} 431 for key, quoted in replacements.items(): 432 text = text.replace(key, quoted) 433 return text 434 435 def install_scripts(self, context, path): 436 """ 437 Install scripts into the created environment from a directory. 438 439 :param context: The information for the environment creation request 440 being processed. 441 :param path: Absolute pathname of a directory containing script. 442 Scripts in the 'common' subdirectory of this directory, 443 and those in the directory named for the platform 444 being run on, are installed in the created environment. 445 Placeholder variables are replaced with environment- 446 specific values. 447 """ 448 binpath = context.bin_path 449 plen = len(path) 450 for root, dirs, files in os.walk(path): 451 if root == path: # at top-level, remove irrelevant dirs 452 for d in dirs[:]: 453 if d not in ('common', os.name): 454 dirs.remove(d) 455 continue # ignore files in top level 456 for f in files: 457 if (os.name == 'nt' and f.startswith('python') 458 and f.endswith(('.exe', '.pdb'))): 459 continue 460 srcfile = os.path.join(root, f) 461 suffix = root[plen:].split(os.sep)[2:] 462 if not suffix: 463 dstdir = binpath 464 else: 465 dstdir = os.path.join(binpath, *suffix) 466 if not os.path.exists(dstdir): 467 os.makedirs(dstdir) 468 dstfile = os.path.join(dstdir, f) 469 with open(srcfile, 'rb') as f: 470 data = f.read() 471 if not srcfile.endswith(('.exe', '.pdb')): 472 context.script_path = srcfile 473 try: 474 data = data.decode('utf-8') 475 data = self.replace_variables(data, context) 476 data = data.encode('utf-8') 477 except UnicodeError as e: 478 data = None 479 logger.warning('unable to copy script %r, ' 480 'may be binary: %s', srcfile, e) 481 if data is not None: 482 with open(dstfile, 'wb') as f: 483 f.write(data) 484 shutil.copymode(srcfile, dstfile) 485 486 def upgrade_dependencies(self, context): 487 logger.debug( 488 f'Upgrading {CORE_VENV_DEPS} packages in {context.bin_path}' 489 ) 490 self._call_new_python(context, '-m', 'pip', 'install', '--upgrade', 491 *CORE_VENV_DEPS) 492 493 494def create(env_dir, system_site_packages=False, clear=False, 495 symlinks=False, with_pip=False, prompt=None, upgrade_deps=False): 496 """Create a virtual environment in a directory.""" 497 builder = EnvBuilder(system_site_packages=system_site_packages, 498 clear=clear, symlinks=symlinks, with_pip=with_pip, 499 prompt=prompt, upgrade_deps=upgrade_deps) 500 builder.create(env_dir) 501 502def main(args=None): 503 compatible = True 504 if sys.version_info < (3, 3): 505 compatible = False 506 elif not hasattr(sys, 'base_prefix'): 507 compatible = False 508 if not compatible: 509 raise ValueError('This script is only for use with Python >= 3.3') 510 else: 511 import argparse 512 513 parser = argparse.ArgumentParser(prog=__name__, 514 description='Creates virtual Python ' 515 'environments in one or ' 516 'more target ' 517 'directories.', 518 epilog='Once an environment has been ' 519 'created, you may wish to ' 520 'activate it, e.g. by ' 521 'sourcing an activate script ' 522 'in its bin directory.') 523 parser.add_argument('dirs', metavar='ENV_DIR', nargs='+', 524 help='A directory to create the environment in.') 525 parser.add_argument('--system-site-packages', default=False, 526 action='store_true', dest='system_site', 527 help='Give the virtual environment access to the ' 528 'system site-packages dir.') 529 if os.name == 'nt': 530 use_symlinks = False 531 else: 532 use_symlinks = True 533 group = parser.add_mutually_exclusive_group() 534 group.add_argument('--symlinks', default=use_symlinks, 535 action='store_true', dest='symlinks', 536 help='Try to use symlinks rather than copies, ' 537 'when symlinks are not the default for ' 538 'the platform.') 539 group.add_argument('--copies', default=not use_symlinks, 540 action='store_false', dest='symlinks', 541 help='Try to use copies rather than symlinks, ' 542 'even when symlinks are the default for ' 543 'the platform.') 544 parser.add_argument('--clear', default=False, action='store_true', 545 dest='clear', help='Delete the contents of the ' 546 'environment directory if it ' 547 'already exists, before ' 548 'environment creation.') 549 parser.add_argument('--upgrade', default=False, action='store_true', 550 dest='upgrade', help='Upgrade the environment ' 551 'directory to use this version ' 552 'of Python, assuming Python ' 553 'has been upgraded in-place.') 554 parser.add_argument('--without-pip', dest='with_pip', 555 default=True, action='store_false', 556 help='Skips installing or upgrading pip in the ' 557 'virtual environment (pip is bootstrapped ' 558 'by default)') 559 parser.add_argument('--prompt', 560 help='Provides an alternative prompt prefix for ' 561 'this environment.') 562 parser.add_argument('--upgrade-deps', default=False, action='store_true', 563 dest='upgrade_deps', 564 help='Upgrade core dependencies: {} to the latest ' 565 'version in PyPI'.format( 566 ' '.join(CORE_VENV_DEPS))) 567 options = parser.parse_args(args) 568 if options.upgrade and options.clear: 569 raise ValueError('you cannot supply --upgrade and --clear together.') 570 builder = EnvBuilder(system_site_packages=options.system_site, 571 clear=options.clear, 572 symlinks=options.symlinks, 573 upgrade=options.upgrade, 574 with_pip=options.with_pip, 575 prompt=options.prompt, 576 upgrade_deps=options.upgrade_deps) 577 for d in options.dirs: 578 builder.create(d) 579 580if __name__ == '__main__': 581 rc = 1 582 try: 583 main() 584 rc = 0 585 except Exception as e: 586 print('Error: %s' % e, file=sys.stderr) 587 sys.exit(rc) 588