1""" 2Virtual environment (venv) package for Python. Based on PEP 405. 3 4Copyright (C) 2011-2014 Vinay Sajip. 5Licensed to the PSF under a contributor agreement. 6""" 7import logging 8import os 9import shutil 10import subprocess 11import sys 12import sysconfig 13import types 14 15 16CORE_VENV_DEPS = ('pip', 'setuptools') 17logger = logging.getLogger(__name__) 18 19 20class EnvBuilder: 21 """ 22 This class exists to allow virtual environment creation to be 23 customized. The constructor parameters determine the builder's 24 behaviour when called upon to create a virtual environment. 25 26 By default, the builder makes the system (global) site-packages dir 27 *un*available to the created environment. 28 29 If invoked using the Python -m option, the default is to use copying 30 on Windows platforms but symlinks elsewhere. If instantiated some 31 other way, the default is to *not* use symlinks. 32 33 :param system_site_packages: If True, the system (global) site-packages 34 dir is available to created environments. 35 :param clear: If True, delete the contents of the environment directory if 36 it already exists, before environment creation. 37 :param symlinks: If True, attempt to symlink rather than copy files into 38 virtual environment. 39 :param upgrade: If True, upgrade an existing virtual environment. 40 :param with_pip: If True, ensure pip is installed in the virtual 41 environment 42 :param prompt: Alternative terminal prefix for the environment. 43 :param upgrade_deps: Update the base venv modules to the latest on PyPI 44 """ 45 46 def __init__(self, system_site_packages=False, clear=False, 47 symlinks=False, upgrade=False, with_pip=False, prompt=None, 48 upgrade_deps=False): 49 self.system_site_packages = system_site_packages 50 self.clear = clear 51 self.symlinks = symlinks 52 self.upgrade = upgrade 53 self.with_pip = with_pip 54 if prompt == '.': # see bpo-38901 55 prompt = os.path.basename(os.getcwd()) 56 self.prompt = prompt 57 self.upgrade_deps = upgrade_deps 58 59 def create(self, env_dir): 60 """ 61 Create a virtual environment in a directory. 62 63 :param env_dir: The target directory to create an environment in. 64 65 """ 66 env_dir = os.path.abspath(env_dir) 67 context = self.ensure_directories(env_dir) 68 # See issue 24875. We need system_site_packages to be False 69 # until after pip is installed. 70 true_system_site_packages = self.system_site_packages 71 self.system_site_packages = False 72 self.create_configuration(context) 73 self.setup_python(context) 74 if self.with_pip: 75 self._setup_pip(context) 76 if not self.upgrade: 77 self.setup_scripts(context) 78 self.post_setup(context) 79 if true_system_site_packages: 80 # We had set it to False before, now 81 # restore it and rewrite the configuration 82 self.system_site_packages = True 83 self.create_configuration(context) 84 if self.upgrade_deps: 85 self.upgrade_dependencies(context) 86 87 def clear_directory(self, path): 88 for fn in os.listdir(path): 89 fn = os.path.join(path, fn) 90 if os.path.islink(fn) or os.path.isfile(fn): 91 os.remove(fn) 92 elif os.path.isdir(fn): 93 shutil.rmtree(fn) 94 95 def ensure_directories(self, env_dir): 96 """ 97 Create the directories for the environment. 98 99 Returns a context object which holds paths in the environment, 100 for use by subsequent logic. 101 """ 102 103 def create_if_needed(d): 104 if not os.path.exists(d): 105 os.makedirs(d) 106 elif os.path.islink(d) or os.path.isfile(d): 107 raise ValueError('Unable to create directory %r' % d) 108 109 if os.path.exists(env_dir) and self.clear: 110 self.clear_directory(env_dir) 111 context = types.SimpleNamespace() 112 context.env_dir = env_dir 113 context.env_name = os.path.split(env_dir)[1] 114 prompt = self.prompt if self.prompt is not None else context.env_name 115 context.prompt = '(%s) ' % prompt 116 create_if_needed(env_dir) 117 executable = sys._base_executable 118 dirname, exename = os.path.split(os.path.abspath(executable)) 119 context.executable = executable 120 context.python_dir = dirname 121 context.python_exe = exename 122 if sys.platform == 'win32': 123 binname = 'Scripts' 124 incpath = 'Include' 125 libpath = os.path.join(env_dir, 'Lib', 'site-packages') 126 else: 127 binname = 'bin' 128 incpath = 'include' 129 libpath = os.path.join(env_dir, 'lib', 130 'python%d.%d' % sys.version_info[:2], 131 'site-packages') 132 context.inc_path = path = os.path.join(env_dir, incpath) 133 create_if_needed(path) 134 create_if_needed(libpath) 135 # Issue 21197: create lib64 as a symlink to lib on 64-bit non-OS X POSIX 136 if ((sys.maxsize > 2**32) and (os.name == 'posix') and 137 (sys.platform != 'darwin')): 138 link_path = os.path.join(env_dir, 'lib64') 139 if not os.path.exists(link_path): # Issue #21643 140 os.symlink('lib', link_path) 141 context.bin_path = binpath = os.path.join(env_dir, binname) 142 context.bin_name = binname 143 context.env_exe = os.path.join(binpath, exename) 144 create_if_needed(binpath) 145 # Assign and update the command to use when launching the newly created 146 # environment, in case it isn't simply the executable script (e.g. bpo-45337) 147 context.env_exec_cmd = context.env_exe 148 if sys.platform == 'win32': 149 # bpo-45337: Fix up env_exec_cmd to account for file system redirections. 150 # Some redirects only apply to CreateFile and not CreateProcess 151 real_env_exe = os.path.realpath(context.env_exe) 152 if os.path.normcase(real_env_exe) != os.path.normcase(context.env_exe): 153 logger.warning('Actual environment location may have moved due to ' 154 'redirects, links or junctions.\n' 155 ' Requested location: "%s"\n' 156 ' Actual location: "%s"', 157 context.env_exe, real_env_exe) 158 context.env_exec_cmd = real_env_exe 159 return context 160 161 def create_configuration(self, context): 162 """ 163 Create a configuration file indicating where the environment's Python 164 was copied from, and whether the system site-packages should be made 165 available in the environment. 166 167 :param context: The information for the environment creation request 168 being processed. 169 """ 170 context.cfg_path = path = os.path.join(context.env_dir, 'pyvenv.cfg') 171 with open(path, 'w', encoding='utf-8') as f: 172 f.write('home = %s\n' % context.python_dir) 173 if self.system_site_packages: 174 incl = 'true' 175 else: 176 incl = 'false' 177 f.write('include-system-site-packages = %s\n' % incl) 178 f.write('version = %d.%d.%d\n' % sys.version_info[:3]) 179 if self.prompt is not None: 180 f.write(f'prompt = {self.prompt!r}\n') 181 182 if os.name != 'nt': 183 def symlink_or_copy(self, src, dst, relative_symlinks_ok=False): 184 """ 185 Try symlinking a file, and if that fails, fall back to copying. 186 """ 187 force_copy = not self.symlinks 188 if not force_copy: 189 try: 190 if not os.path.islink(dst): # can't link to itself! 191 if relative_symlinks_ok: 192 assert os.path.dirname(src) == os.path.dirname(dst) 193 os.symlink(os.path.basename(src), dst) 194 else: 195 os.symlink(src, dst) 196 except Exception: # may need to use a more specific exception 197 logger.warning('Unable to symlink %r to %r', src, dst) 198 force_copy = True 199 if force_copy: 200 shutil.copyfile(src, dst) 201 else: 202 def symlink_or_copy(self, src, dst, relative_symlinks_ok=False): 203 """ 204 Try symlinking a file, and if that fails, fall back to copying. 205 """ 206 bad_src = os.path.lexists(src) and not os.path.exists(src) 207 if self.symlinks and not bad_src and not os.path.islink(dst): 208 try: 209 if relative_symlinks_ok: 210 assert os.path.dirname(src) == os.path.dirname(dst) 211 os.symlink(os.path.basename(src), dst) 212 else: 213 os.symlink(src, dst) 214 return 215 except Exception: # may need to use a more specific exception 216 logger.warning('Unable to symlink %r to %r', src, dst) 217 218 # On Windows, we rewrite symlinks to our base python.exe into 219 # copies of venvlauncher.exe 220 basename, ext = os.path.splitext(os.path.basename(src)) 221 srcfn = os.path.join(os.path.dirname(__file__), 222 "scripts", 223 "nt", 224 basename + ext) 225 # Builds or venv's from builds need to remap source file 226 # locations, as we do not put them into Lib/venv/scripts 227 if sysconfig.is_python_build(True) or not os.path.isfile(srcfn): 228 if basename.endswith('_d'): 229 ext = '_d' + ext 230 basename = basename[:-2] 231 if basename == 'python': 232 basename = 'venvlauncher' 233 elif basename == 'pythonw': 234 basename = 'venvwlauncher' 235 src = os.path.join(os.path.dirname(src), basename + ext) 236 else: 237 src = srcfn 238 if not os.path.exists(src): 239 if not bad_src: 240 logger.warning('Unable to copy %r', src) 241 return 242 243 shutil.copyfile(src, dst) 244 245 def setup_python(self, context): 246 """ 247 Set up a Python executable in the environment. 248 249 :param context: The information for the environment creation request 250 being processed. 251 """ 252 binpath = context.bin_path 253 path = context.env_exe 254 copier = self.symlink_or_copy 255 dirname = context.python_dir 256 if os.name != 'nt': 257 copier(context.executable, path) 258 if not os.path.islink(path): 259 os.chmod(path, 0o755) 260 for suffix in ('python', 'python3', f'python3.{sys.version_info[1]}'): 261 path = os.path.join(binpath, suffix) 262 if not os.path.exists(path): 263 # Issue 18807: make copies if 264 # symlinks are not wanted 265 copier(context.env_exe, path, relative_symlinks_ok=True) 266 if not os.path.islink(path): 267 os.chmod(path, 0o755) 268 else: 269 if self.symlinks: 270 # For symlinking, we need a complete copy of the root directory 271 # If symlinks fail, you'll get unnecessary copies of files, but 272 # we assume that if you've opted into symlinks on Windows then 273 # you know what you're doing. 274 suffixes = [ 275 f for f in os.listdir(dirname) if 276 os.path.normcase(os.path.splitext(f)[1]) in ('.exe', '.dll') 277 ] 278 if sysconfig.is_python_build(True): 279 suffixes = [ 280 f for f in suffixes if 281 os.path.normcase(f).startswith(('python', 'vcruntime')) 282 ] 283 else: 284 suffixes = {'python.exe', 'python_d.exe', 'pythonw.exe', 'pythonw_d.exe'} 285 base_exe = os.path.basename(context.env_exe) 286 suffixes.add(base_exe) 287 288 for suffix in suffixes: 289 src = os.path.join(dirname, suffix) 290 if os.path.lexists(src): 291 copier(src, os.path.join(binpath, suffix)) 292 293 if sysconfig.is_python_build(True): 294 # copy init.tcl 295 for root, dirs, files in os.walk(context.python_dir): 296 if 'init.tcl' in files: 297 tcldir = os.path.basename(root) 298 tcldir = os.path.join(context.env_dir, 'Lib', tcldir) 299 if not os.path.exists(tcldir): 300 os.makedirs(tcldir) 301 src = os.path.join(root, 'init.tcl') 302 dst = os.path.join(tcldir, 'init.tcl') 303 shutil.copyfile(src, dst) 304 break 305 306 def _setup_pip(self, context): 307 """Installs or upgrades pip in a virtual environment""" 308 # We run ensurepip in isolated mode to avoid side effects from 309 # environment vars, the current directory and anything else 310 # intended for the global Python environment 311 cmd = [context.env_exec_cmd, '-Im', 'ensurepip', '--upgrade', 312 '--default-pip'] 313 subprocess.check_output(cmd, stderr=subprocess.STDOUT) 314 315 def setup_scripts(self, context): 316 """ 317 Set up scripts into the created environment from a directory. 318 319 This method installs the default scripts into the environment 320 being created. You can prevent the default installation by overriding 321 this method if you really need to, or if you need to specify 322 a different location for the scripts to install. By default, the 323 'scripts' directory in the venv package is used as the source of 324 scripts to install. 325 """ 326 path = os.path.abspath(os.path.dirname(__file__)) 327 path = os.path.join(path, 'scripts') 328 self.install_scripts(context, path) 329 330 def post_setup(self, context): 331 """ 332 Hook for post-setup modification of the venv. Subclasses may install 333 additional packages or scripts here, add activation shell scripts, etc. 334 335 :param context: The information for the environment creation request 336 being processed. 337 """ 338 pass 339 340 def replace_variables(self, text, context): 341 """ 342 Replace variable placeholders in script text with context-specific 343 variables. 344 345 Return the text passed in , but with variables replaced. 346 347 :param text: The text in which to replace placeholder variables. 348 :param context: The information for the environment creation request 349 being processed. 350 """ 351 text = text.replace('__VENV_DIR__', context.env_dir) 352 text = text.replace('__VENV_NAME__', context.env_name) 353 text = text.replace('__VENV_PROMPT__', context.prompt) 354 text = text.replace('__VENV_BIN_NAME__', context.bin_name) 355 text = text.replace('__VENV_PYTHON__', context.env_exe) 356 return text 357 358 def install_scripts(self, context, path): 359 """ 360 Install scripts into the created environment from a directory. 361 362 :param context: The information for the environment creation request 363 being processed. 364 :param path: Absolute pathname of a directory containing script. 365 Scripts in the 'common' subdirectory of this directory, 366 and those in the directory named for the platform 367 being run on, are installed in the created environment. 368 Placeholder variables are replaced with environment- 369 specific values. 370 """ 371 binpath = context.bin_path 372 plen = len(path) 373 for root, dirs, files in os.walk(path): 374 if root == path: # at top-level, remove irrelevant dirs 375 for d in dirs[:]: 376 if d not in ('common', os.name): 377 dirs.remove(d) 378 continue # ignore files in top level 379 for f in files: 380 if (os.name == 'nt' and f.startswith('python') 381 and f.endswith(('.exe', '.pdb'))): 382 continue 383 srcfile = os.path.join(root, f) 384 suffix = root[plen:].split(os.sep)[2:] 385 if not suffix: 386 dstdir = binpath 387 else: 388 dstdir = os.path.join(binpath, *suffix) 389 if not os.path.exists(dstdir): 390 os.makedirs(dstdir) 391 dstfile = os.path.join(dstdir, f) 392 with open(srcfile, 'rb') as f: 393 data = f.read() 394 if not srcfile.endswith(('.exe', '.pdb')): 395 try: 396 data = data.decode('utf-8') 397 data = self.replace_variables(data, context) 398 data = data.encode('utf-8') 399 except UnicodeError as e: 400 data = None 401 logger.warning('unable to copy script %r, ' 402 'may be binary: %s', srcfile, e) 403 if data is not None: 404 with open(dstfile, 'wb') as f: 405 f.write(data) 406 shutil.copymode(srcfile, dstfile) 407 408 def upgrade_dependencies(self, context): 409 logger.debug( 410 f'Upgrading {CORE_VENV_DEPS} packages in {context.bin_path}' 411 ) 412 cmd = [context.env_exec_cmd, '-m', 'pip', 'install', '--upgrade'] 413 cmd.extend(CORE_VENV_DEPS) 414 subprocess.check_call(cmd) 415 416 417def create(env_dir, system_site_packages=False, clear=False, 418 symlinks=False, with_pip=False, prompt=None, upgrade_deps=False): 419 """Create a virtual environment in a directory.""" 420 builder = EnvBuilder(system_site_packages=system_site_packages, 421 clear=clear, symlinks=symlinks, with_pip=with_pip, 422 prompt=prompt, upgrade_deps=upgrade_deps) 423 builder.create(env_dir) 424 425def main(args=None): 426 compatible = True 427 if sys.version_info < (3, 3): 428 compatible = False 429 elif not hasattr(sys, 'base_prefix'): 430 compatible = False 431 if not compatible: 432 raise ValueError('This script is only for use with Python >= 3.3') 433 else: 434 import argparse 435 436 parser = argparse.ArgumentParser(prog=__name__, 437 description='Creates virtual Python ' 438 'environments in one or ' 439 'more target ' 440 'directories.', 441 epilog='Once an environment has been ' 442 'created, you may wish to ' 443 'activate it, e.g. by ' 444 'sourcing an activate script ' 445 'in its bin directory.') 446 parser.add_argument('dirs', metavar='ENV_DIR', nargs='+', 447 help='A directory to create the environment in.') 448 parser.add_argument('--system-site-packages', default=False, 449 action='store_true', dest='system_site', 450 help='Give the virtual environment access to the ' 451 'system site-packages dir.') 452 if os.name == 'nt': 453 use_symlinks = False 454 else: 455 use_symlinks = True 456 group = parser.add_mutually_exclusive_group() 457 group.add_argument('--symlinks', default=use_symlinks, 458 action='store_true', dest='symlinks', 459 help='Try to use symlinks rather than copies, ' 460 'when symlinks are not the default for ' 461 'the platform.') 462 group.add_argument('--copies', default=not use_symlinks, 463 action='store_false', dest='symlinks', 464 help='Try to use copies rather than symlinks, ' 465 'even when symlinks are the default for ' 466 'the platform.') 467 parser.add_argument('--clear', default=False, action='store_true', 468 dest='clear', help='Delete the contents of the ' 469 'environment directory if it ' 470 'already exists, before ' 471 'environment creation.') 472 parser.add_argument('--upgrade', default=False, action='store_true', 473 dest='upgrade', help='Upgrade the environment ' 474 'directory to use this version ' 475 'of Python, assuming Python ' 476 'has been upgraded in-place.') 477 parser.add_argument('--without-pip', dest='with_pip', 478 default=True, action='store_false', 479 help='Skips installing or upgrading pip in the ' 480 'virtual environment (pip is bootstrapped ' 481 'by default)') 482 parser.add_argument('--prompt', 483 help='Provides an alternative prompt prefix for ' 484 'this environment.') 485 parser.add_argument('--upgrade-deps', default=False, action='store_true', 486 dest='upgrade_deps', 487 help='Upgrade core dependencies: {} to the latest ' 488 'version in PyPI'.format( 489 ' '.join(CORE_VENV_DEPS))) 490 options = parser.parse_args(args) 491 if options.upgrade and options.clear: 492 raise ValueError('you cannot supply --upgrade and --clear together.') 493 builder = EnvBuilder(system_site_packages=options.system_site, 494 clear=options.clear, 495 symlinks=options.symlinks, 496 upgrade=options.upgrade, 497 with_pip=options.with_pip, 498 prompt=options.prompt, 499 upgrade_deps=options.upgrade_deps) 500 for d in options.dirs: 501 builder.create(d) 502 503if __name__ == '__main__': 504 rc = 1 505 try: 506 main() 507 rc = 0 508 except Exception as e: 509 print('Error: %s' % e, file=sys.stderr) 510 sys.exit(rc) 511