• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1"""
2Test harness for the venv module.
3
4Copyright (C) 2011-2012 Vinay Sajip.
5Licensed to the PSF under a contributor agreement.
6"""
7
8import ensurepip
9import os
10import os.path
11import re
12import shutil
13import struct
14import subprocess
15import sys
16import tempfile
17from test.support import (captured_stdout, captured_stderr, requires_zlib,
18                          skip_if_broken_multiprocessing_synchronize)
19from test.support.os_helper import (can_symlink, EnvironmentVarGuard, rmtree)
20import unittest
21import venv
22from unittest.mock import patch
23
24try:
25    import ctypes
26except ImportError:
27    ctypes = None
28
29# Platforms that set sys._base_executable can create venvs from within
30# another venv, so no need to skip tests that require venv.create().
31requireVenvCreate = unittest.skipUnless(
32    sys.prefix == sys.base_prefix
33    or sys._base_executable != sys.executable,
34    'cannot run venv.create from within a venv on this platform')
35
36def check_output(cmd, encoding=None):
37    p = subprocess.Popen(cmd,
38        stdout=subprocess.PIPE,
39        stderr=subprocess.PIPE,
40        encoding=encoding)
41    out, err = p.communicate()
42    if p.returncode:
43        raise subprocess.CalledProcessError(
44            p.returncode, cmd, out, err)
45    return out, err
46
47class BaseTest(unittest.TestCase):
48    """Base class for venv tests."""
49    maxDiff = 80 * 50
50
51    def setUp(self):
52        self.env_dir = os.path.realpath(tempfile.mkdtemp())
53        if os.name == 'nt':
54            self.bindir = 'Scripts'
55            self.lib = ('Lib',)
56            self.include = 'Include'
57        else:
58            self.bindir = 'bin'
59            self.lib = ('lib', 'python%d.%d' % sys.version_info[:2])
60            self.include = 'include'
61        executable = sys._base_executable
62        self.exe = os.path.split(executable)[-1]
63        if (sys.platform == 'win32'
64            and os.path.lexists(executable)
65            and not os.path.exists(executable)):
66            self.cannot_link_exe = True
67        else:
68            self.cannot_link_exe = False
69
70    def tearDown(self):
71        rmtree(self.env_dir)
72
73    def run_with_capture(self, func, *args, **kwargs):
74        with captured_stdout() as output:
75            with captured_stderr() as error:
76                func(*args, **kwargs)
77        return output.getvalue(), error.getvalue()
78
79    def get_env_file(self, *args):
80        return os.path.join(self.env_dir, *args)
81
82    def get_text_file_contents(self, *args, encoding='utf-8'):
83        with open(self.get_env_file(*args), 'r', encoding=encoding) as f:
84            result = f.read()
85        return result
86
87class BasicTest(BaseTest):
88    """Test venv module functionality."""
89
90    def isdir(self, *args):
91        fn = self.get_env_file(*args)
92        self.assertTrue(os.path.isdir(fn))
93
94    def test_defaults(self):
95        """
96        Test the create function with default arguments.
97        """
98        rmtree(self.env_dir)
99        self.run_with_capture(venv.create, self.env_dir)
100        self.isdir(self.bindir)
101        self.isdir(self.include)
102        self.isdir(*self.lib)
103        # Issue 21197
104        p = self.get_env_file('lib64')
105        conditions = ((struct.calcsize('P') == 8) and (os.name == 'posix') and
106                      (sys.platform != 'darwin'))
107        if conditions:
108            self.assertTrue(os.path.islink(p))
109        else:
110            self.assertFalse(os.path.exists(p))
111        data = self.get_text_file_contents('pyvenv.cfg')
112        executable = sys._base_executable
113        path = os.path.dirname(executable)
114        self.assertIn('home = %s' % path, data)
115        fn = self.get_env_file(self.bindir, self.exe)
116        if not os.path.exists(fn):  # diagnostics for Windows buildbot failures
117            bd = self.get_env_file(self.bindir)
118            print('Contents of %r:' % bd)
119            print('    %r' % os.listdir(bd))
120        self.assertTrue(os.path.exists(fn), 'File %r should exist.' % fn)
121
122    def test_prompt(self):
123        env_name = os.path.split(self.env_dir)[1]
124
125        rmtree(self.env_dir)
126        builder = venv.EnvBuilder()
127        self.run_with_capture(builder.create, self.env_dir)
128        context = builder.ensure_directories(self.env_dir)
129        data = self.get_text_file_contents('pyvenv.cfg')
130        self.assertEqual(context.prompt, '(%s) ' % env_name)
131        self.assertNotIn("prompt = ", data)
132
133        rmtree(self.env_dir)
134        builder = venv.EnvBuilder(prompt='My prompt')
135        self.run_with_capture(builder.create, self.env_dir)
136        context = builder.ensure_directories(self.env_dir)
137        data = self.get_text_file_contents('pyvenv.cfg')
138        self.assertEqual(context.prompt, '(My prompt) ')
139        self.assertIn("prompt = 'My prompt'\n", data)
140
141        rmtree(self.env_dir)
142        builder = venv.EnvBuilder(prompt='.')
143        cwd = os.path.basename(os.getcwd())
144        self.run_with_capture(builder.create, self.env_dir)
145        context = builder.ensure_directories(self.env_dir)
146        data = self.get_text_file_contents('pyvenv.cfg')
147        self.assertEqual(context.prompt, '(%s) ' % cwd)
148        self.assertIn("prompt = '%s'\n" % cwd, data)
149
150    def test_upgrade_dependencies(self):
151        builder = venv.EnvBuilder()
152        bin_path = 'Scripts' if sys.platform == 'win32' else 'bin'
153        python_exe = os.path.split(sys.executable)[1]
154        with tempfile.TemporaryDirectory() as fake_env_dir:
155            expect_exe = os.path.normcase(
156                os.path.join(fake_env_dir, bin_path, python_exe)
157            )
158            if sys.platform == 'win32':
159                expect_exe = os.path.normcase(os.path.realpath(expect_exe))
160
161            def pip_cmd_checker(cmd):
162                cmd[0] = os.path.normcase(cmd[0])
163                self.assertEqual(
164                    cmd,
165                    [
166                        expect_exe,
167                        '-m',
168                        'pip',
169                        'install',
170                        '--upgrade',
171                        'pip',
172                        'setuptools'
173                    ]
174                )
175
176            fake_context = builder.ensure_directories(fake_env_dir)
177            with patch('venv.subprocess.check_call', pip_cmd_checker):
178                builder.upgrade_dependencies(fake_context)
179
180    @requireVenvCreate
181    def test_prefixes(self):
182        """
183        Test that the prefix values are as expected.
184        """
185        # check a venv's prefixes
186        rmtree(self.env_dir)
187        self.run_with_capture(venv.create, self.env_dir)
188        envpy = os.path.join(self.env_dir, self.bindir, self.exe)
189        cmd = [envpy, '-c', None]
190        for prefix, expected in (
191            ('prefix', self.env_dir),
192            ('exec_prefix', self.env_dir),
193            ('base_prefix', sys.base_prefix),
194            ('base_exec_prefix', sys.base_exec_prefix)):
195            cmd[2] = 'import sys; print(sys.%s)' % prefix
196            out, err = check_output(cmd)
197            self.assertEqual(out.strip(), expected.encode())
198
199    if sys.platform == 'win32':
200        ENV_SUBDIRS = (
201            ('Scripts',),
202            ('Include',),
203            ('Lib',),
204            ('Lib', 'site-packages'),
205        )
206    else:
207        ENV_SUBDIRS = (
208            ('bin',),
209            ('include',),
210            ('lib',),
211            ('lib', 'python%d.%d' % sys.version_info[:2]),
212            ('lib', 'python%d.%d' % sys.version_info[:2], 'site-packages'),
213        )
214
215    def create_contents(self, paths, filename):
216        """
217        Create some files in the environment which are unrelated
218        to the virtual environment.
219        """
220        for subdirs in paths:
221            d = os.path.join(self.env_dir, *subdirs)
222            os.mkdir(d)
223            fn = os.path.join(d, filename)
224            with open(fn, 'wb') as f:
225                f.write(b'Still here?')
226
227    def test_overwrite_existing(self):
228        """
229        Test creating environment in an existing directory.
230        """
231        self.create_contents(self.ENV_SUBDIRS, 'foo')
232        venv.create(self.env_dir)
233        for subdirs in self.ENV_SUBDIRS:
234            fn = os.path.join(self.env_dir, *(subdirs + ('foo',)))
235            self.assertTrue(os.path.exists(fn))
236            with open(fn, 'rb') as f:
237                self.assertEqual(f.read(), b'Still here?')
238
239        builder = venv.EnvBuilder(clear=True)
240        builder.create(self.env_dir)
241        for subdirs in self.ENV_SUBDIRS:
242            fn = os.path.join(self.env_dir, *(subdirs + ('foo',)))
243            self.assertFalse(os.path.exists(fn))
244
245    def clear_directory(self, path):
246        for fn in os.listdir(path):
247            fn = os.path.join(path, fn)
248            if os.path.islink(fn) or os.path.isfile(fn):
249                os.remove(fn)
250            elif os.path.isdir(fn):
251                rmtree(fn)
252
253    def test_unoverwritable_fails(self):
254        #create a file clashing with directories in the env dir
255        for paths in self.ENV_SUBDIRS[:3]:
256            fn = os.path.join(self.env_dir, *paths)
257            with open(fn, 'wb') as f:
258                f.write(b'')
259            self.assertRaises((ValueError, OSError), venv.create, self.env_dir)
260            self.clear_directory(self.env_dir)
261
262    def test_upgrade(self):
263        """
264        Test upgrading an existing environment directory.
265        """
266        # See Issue #21643: the loop needs to run twice to ensure
267        # that everything works on the upgrade (the first run just creates
268        # the venv).
269        for upgrade in (False, True):
270            builder = venv.EnvBuilder(upgrade=upgrade)
271            self.run_with_capture(builder.create, self.env_dir)
272            self.isdir(self.bindir)
273            self.isdir(self.include)
274            self.isdir(*self.lib)
275            fn = self.get_env_file(self.bindir, self.exe)
276            if not os.path.exists(fn):
277                # diagnostics for Windows buildbot failures
278                bd = self.get_env_file(self.bindir)
279                print('Contents of %r:' % bd)
280                print('    %r' % os.listdir(bd))
281            self.assertTrue(os.path.exists(fn), 'File %r should exist.' % fn)
282
283    def test_isolation(self):
284        """
285        Test isolation from system site-packages
286        """
287        for ssp, s in ((True, 'true'), (False, 'false')):
288            builder = venv.EnvBuilder(clear=True, system_site_packages=ssp)
289            builder.create(self.env_dir)
290            data = self.get_text_file_contents('pyvenv.cfg')
291            self.assertIn('include-system-site-packages = %s\n' % s, data)
292
293    @unittest.skipUnless(can_symlink(), 'Needs symlinks')
294    def test_symlinking(self):
295        """
296        Test symlinking works as expected
297        """
298        for usl in (False, True):
299            builder = venv.EnvBuilder(clear=True, symlinks=usl)
300            builder.create(self.env_dir)
301            fn = self.get_env_file(self.bindir, self.exe)
302            # Don't test when False, because e.g. 'python' is always
303            # symlinked to 'python3.3' in the env, even when symlinking in
304            # general isn't wanted.
305            if usl:
306                if self.cannot_link_exe:
307                    # Symlinking is skipped when our executable is already a
308                    # special app symlink
309                    self.assertFalse(os.path.islink(fn))
310                else:
311                    self.assertTrue(os.path.islink(fn))
312
313    # If a venv is created from a source build and that venv is used to
314    # run the test, the pyvenv.cfg in the venv created in the test will
315    # point to the venv being used to run the test, and we lose the link
316    # to the source build - so Python can't initialise properly.
317    @requireVenvCreate
318    def test_executable(self):
319        """
320        Test that the sys.executable value is as expected.
321        """
322        rmtree(self.env_dir)
323        self.run_with_capture(venv.create, self.env_dir)
324        envpy = os.path.join(os.path.realpath(self.env_dir),
325                             self.bindir, self.exe)
326        out, err = check_output([envpy, '-c',
327            'import sys; print(sys.executable)'])
328        self.assertEqual(out.strip(), envpy.encode())
329
330    @unittest.skipUnless(can_symlink(), 'Needs symlinks')
331    def test_executable_symlinks(self):
332        """
333        Test that the sys.executable value is as expected.
334        """
335        rmtree(self.env_dir)
336        builder = venv.EnvBuilder(clear=True, symlinks=True)
337        builder.create(self.env_dir)
338        envpy = os.path.join(os.path.realpath(self.env_dir),
339                             self.bindir, self.exe)
340        out, err = check_output([envpy, '-c',
341            'import sys; print(sys.executable)'])
342        self.assertEqual(out.strip(), envpy.encode())
343
344    @unittest.skipUnless(os.name == 'nt', 'only relevant on Windows')
345    def test_unicode_in_batch_file(self):
346        """
347        Test handling of Unicode paths
348        """
349        rmtree(self.env_dir)
350        env_dir = os.path.join(os.path.realpath(self.env_dir), 'ϼўТλФЙ')
351        builder = venv.EnvBuilder(clear=True)
352        builder.create(env_dir)
353        activate = os.path.join(env_dir, self.bindir, 'activate.bat')
354        envpy = os.path.join(env_dir, self.bindir, self.exe)
355        out, err = check_output(
356            [activate, '&', self.exe, '-c', 'print(0)'],
357            encoding='oem',
358        )
359        self.assertEqual(out.strip(), '0')
360
361    @requireVenvCreate
362    def test_multiprocessing(self):
363        """
364        Test that the multiprocessing is able to spawn.
365        """
366        # bpo-36342: Instantiation of a Pool object imports the
367        # multiprocessing.synchronize module. Skip the test if this module
368        # cannot be imported.
369        skip_if_broken_multiprocessing_synchronize()
370
371        rmtree(self.env_dir)
372        self.run_with_capture(venv.create, self.env_dir)
373        envpy = os.path.join(os.path.realpath(self.env_dir),
374                             self.bindir, self.exe)
375        out, err = check_output([envpy, '-c',
376            'from multiprocessing import Pool; '
377            'pool = Pool(1); '
378            'print(pool.apply_async("Python".lower).get(3)); '
379            'pool.terminate()'])
380        self.assertEqual(out.strip(), "python".encode())
381
382    @unittest.skipIf(os.name == 'nt', 'not relevant on Windows')
383    def test_deactivate_with_strict_bash_opts(self):
384        bash = shutil.which("bash")
385        if bash is None:
386            self.skipTest("bash required for this test")
387        rmtree(self.env_dir)
388        builder = venv.EnvBuilder(clear=True)
389        builder.create(self.env_dir)
390        activate = os.path.join(self.env_dir, self.bindir, "activate")
391        test_script = os.path.join(self.env_dir, "test_strict.sh")
392        with open(test_script, "w") as f:
393            f.write("set -euo pipefail\n"
394                    f"source {activate}\n"
395                    "deactivate\n")
396        out, err = check_output([bash, test_script])
397        self.assertEqual(out, "".encode())
398        self.assertEqual(err, "".encode())
399
400
401    @unittest.skipUnless(sys.platform == 'darwin', 'only relevant on macOS')
402    def test_macos_env(self):
403        rmtree(self.env_dir)
404        builder = venv.EnvBuilder()
405        builder.create(self.env_dir)
406
407        envpy = os.path.join(os.path.realpath(self.env_dir),
408                             self.bindir, self.exe)
409        out, err = check_output([envpy, '-c',
410            'import os; print("__PYVENV_LAUNCHER__" in os.environ)'])
411        self.assertEqual(out.strip(), 'False'.encode())
412
413@requireVenvCreate
414class EnsurePipTest(BaseTest):
415    """Test venv module installation of pip."""
416    def assert_pip_not_installed(self):
417        envpy = os.path.join(os.path.realpath(self.env_dir),
418                             self.bindir, self.exe)
419        out, err = check_output([envpy, '-c',
420            'try:\n import pip\nexcept ImportError:\n print("OK")'])
421        # We force everything to text, so unittest gives the detailed diff
422        # if we get unexpected results
423        err = err.decode("latin-1") # Force to text, prevent decoding errors
424        self.assertEqual(err, "")
425        out = out.decode("latin-1") # Force to text, prevent decoding errors
426        self.assertEqual(out.strip(), "OK")
427
428
429    def test_no_pip_by_default(self):
430        rmtree(self.env_dir)
431        self.run_with_capture(venv.create, self.env_dir)
432        self.assert_pip_not_installed()
433
434    def test_explicit_no_pip(self):
435        rmtree(self.env_dir)
436        self.run_with_capture(venv.create, self.env_dir, with_pip=False)
437        self.assert_pip_not_installed()
438
439    def test_devnull(self):
440        # Fix for issue #20053 uses os.devnull to force a config file to
441        # appear empty. However http://bugs.python.org/issue20541 means
442        # that doesn't currently work properly on Windows. Once that is
443        # fixed, the "win_location" part of test_with_pip should be restored
444        with open(os.devnull, "rb") as f:
445            self.assertEqual(f.read(), b"")
446
447        self.assertTrue(os.path.exists(os.devnull))
448
449    def do_test_with_pip(self, system_site_packages):
450        rmtree(self.env_dir)
451        with EnvironmentVarGuard() as envvars:
452            # pip's cross-version compatibility may trigger deprecation
453            # warnings in current versions of Python. Ensure related
454            # environment settings don't cause venv to fail.
455            envvars["PYTHONWARNINGS"] = "ignore"
456            # ensurepip is different enough from a normal pip invocation
457            # that we want to ensure it ignores the normal pip environment
458            # variable settings. We set PIP_NO_INSTALL here specifically
459            # to check that ensurepip (and hence venv) ignores it.
460            # See http://bugs.python.org/issue19734
461            envvars["PIP_NO_INSTALL"] = "1"
462            # Also check that we ignore the pip configuration file
463            # See http://bugs.python.org/issue20053
464            with tempfile.TemporaryDirectory() as home_dir:
465                envvars["HOME"] = home_dir
466                bad_config = "[global]\nno-install=1"
467                # Write to both config file names on all platforms to reduce
468                # cross-platform variation in test code behaviour
469                win_location = ("pip", "pip.ini")
470                posix_location = (".pip", "pip.conf")
471                # Skips win_location due to http://bugs.python.org/issue20541
472                for dirname, fname in (posix_location,):
473                    dirpath = os.path.join(home_dir, dirname)
474                    os.mkdir(dirpath)
475                    fpath = os.path.join(dirpath, fname)
476                    with open(fpath, 'w') as f:
477                        f.write(bad_config)
478
479                # Actually run the create command with all that unhelpful
480                # config in place to ensure we ignore it
481                try:
482                    self.run_with_capture(venv.create, self.env_dir,
483                                          system_site_packages=system_site_packages,
484                                          with_pip=True)
485                except subprocess.CalledProcessError as exc:
486                    # The output this produces can be a little hard to read,
487                    # but at least it has all the details
488                    details = exc.output.decode(errors="replace")
489                    msg = "{}\n\n**Subprocess Output**\n{}"
490                    self.fail(msg.format(exc, details))
491        # Ensure pip is available in the virtual environment
492        envpy = os.path.join(os.path.realpath(self.env_dir), self.bindir, self.exe)
493        # Ignore DeprecationWarning since pip code is not part of Python
494        out, err = check_output([envpy, '-W', 'ignore::DeprecationWarning',
495               '-W', 'ignore::ImportWarning', '-I',
496               '-m', 'pip', '--version'])
497        # We force everything to text, so unittest gives the detailed diff
498        # if we get unexpected results
499        err = err.decode("latin-1") # Force to text, prevent decoding errors
500        self.assertEqual(err, "")
501        out = out.decode("latin-1") # Force to text, prevent decoding errors
502        expected_version = "pip {}".format(ensurepip.version())
503        self.assertEqual(out[:len(expected_version)], expected_version)
504        env_dir = os.fsencode(self.env_dir).decode("latin-1")
505        self.assertIn(env_dir, out)
506
507        # http://bugs.python.org/issue19728
508        # Check the private uninstall command provided for the Windows
509        # installers works (at least in a virtual environment)
510        with EnvironmentVarGuard() as envvars:
511            # It seems ensurepip._uninstall calls subprocesses which do not
512            # inherit the interpreter settings.
513            envvars["PYTHONWARNINGS"] = "ignore"
514            out, err = check_output([envpy,
515                '-W', 'ignore::DeprecationWarning',
516                '-W', 'ignore::ImportWarning', '-I',
517                '-m', 'ensurepip._uninstall'])
518        # We force everything to text, so unittest gives the detailed diff
519        # if we get unexpected results
520        err = err.decode("latin-1") # Force to text, prevent decoding errors
521        # Ignore the warning:
522        #   "The directory '$HOME/.cache/pip/http' or its parent directory
523        #    is not owned by the current user and the cache has been disabled.
524        #    Please check the permissions and owner of that directory. If
525        #    executing pip with sudo, you may want sudo's -H flag."
526        # where $HOME is replaced by the HOME environment variable.
527        err = re.sub("^(WARNING: )?The directory .* or its parent directory "
528                     "is not owned or is not writable by the current user.*$", "",
529                     err, flags=re.MULTILINE)
530        self.assertEqual(err.rstrip(), "")
531        # Being fairly specific regarding the expected behaviour for the
532        # initial bundling phase in Python 3.4. If the output changes in
533        # future pip versions, this test can likely be relaxed further.
534        out = out.decode("latin-1") # Force to text, prevent decoding errors
535        self.assertIn("Successfully uninstalled pip", out)
536        self.assertIn("Successfully uninstalled setuptools", out)
537        # Check pip is now gone from the virtual environment. This only
538        # applies in the system_site_packages=False case, because in the
539        # other case, pip may still be available in the system site-packages
540        if not system_site_packages:
541            self.assert_pip_not_installed()
542
543    # Issue #26610: pip/pep425tags.py requires ctypes
544    @unittest.skipUnless(ctypes, 'pip requires ctypes')
545    @requires_zlib()
546    def test_with_pip(self):
547        self.do_test_with_pip(False)
548        self.do_test_with_pip(True)
549
550if __name__ == "__main__":
551    unittest.main()
552