• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1"""
2Test harness for the venv module.
3
4Copyright (C) 2011-2012 Vinay Sajip.
5Licensed to the PSF under a contributor agreement.
6"""
7
8import ensurepip
9import os
10import os.path
11import re
12import shutil
13import struct
14import subprocess
15import sys
16import tempfile
17from test.support import (captured_stdout, captured_stderr, requires_zlib,
18                          can_symlink, EnvironmentVarGuard, rmtree,
19                          import_module)
20import threading
21import unittest
22import venv
23
24try:
25    import ctypes
26except ImportError:
27    ctypes = None
28
29# Platforms that set sys._base_executable can create venvs from within
30# another venv, so no need to skip tests that require venv.create().
31requireVenvCreate = unittest.skipUnless(
32    sys.prefix == sys.base_prefix
33    or sys._base_executable != sys.executable,
34    'cannot run venv.create from within a venv on this platform')
35
36def check_output(cmd, encoding=None):
37    p = subprocess.Popen(cmd,
38        stdout=subprocess.PIPE,
39        stderr=subprocess.PIPE,
40        encoding=encoding)
41    out, err = p.communicate()
42    if p.returncode:
43        raise subprocess.CalledProcessError(
44            p.returncode, cmd, out, err)
45    return out, err
46
47class BaseTest(unittest.TestCase):
48    """Base class for venv tests."""
49    maxDiff = 80 * 50
50
51    def setUp(self):
52        self.env_dir = os.path.realpath(tempfile.mkdtemp())
53        if os.name == 'nt':
54            self.bindir = 'Scripts'
55            self.lib = ('Lib',)
56            self.include = 'Include'
57        else:
58            self.bindir = 'bin'
59            self.lib = ('lib', 'python%d.%d' % sys.version_info[:2])
60            self.include = 'include'
61        executable = sys._base_executable
62        self.exe = os.path.split(executable)[-1]
63        if (sys.platform == 'win32'
64            and os.path.lexists(executable)
65            and not os.path.exists(executable)):
66            self.cannot_link_exe = True
67        else:
68            self.cannot_link_exe = False
69
70    def tearDown(self):
71        rmtree(self.env_dir)
72
73    def run_with_capture(self, func, *args, **kwargs):
74        with captured_stdout() as output:
75            with captured_stderr() as error:
76                func(*args, **kwargs)
77        return output.getvalue(), error.getvalue()
78
79    def get_env_file(self, *args):
80        return os.path.join(self.env_dir, *args)
81
82    def get_text_file_contents(self, *args):
83        with open(self.get_env_file(*args), 'r') as f:
84            result = f.read()
85        return result
86
87class BasicTest(BaseTest):
88    """Test venv module functionality."""
89
90    def isdir(self, *args):
91        fn = self.get_env_file(*args)
92        self.assertTrue(os.path.isdir(fn))
93
94    def test_defaults(self):
95        """
96        Test the create function with default arguments.
97        """
98        rmtree(self.env_dir)
99        self.run_with_capture(venv.create, self.env_dir)
100        self.isdir(self.bindir)
101        self.isdir(self.include)
102        self.isdir(*self.lib)
103        # Issue 21197
104        p = self.get_env_file('lib64')
105        conditions = ((struct.calcsize('P') == 8) and (os.name == 'posix') and
106                      (sys.platform != 'darwin'))
107        if conditions:
108            self.assertTrue(os.path.islink(p))
109        else:
110            self.assertFalse(os.path.exists(p))
111        data = self.get_text_file_contents('pyvenv.cfg')
112        executable = sys._base_executable
113        path = os.path.dirname(executable)
114        self.assertIn('home = %s' % path, data)
115        fn = self.get_env_file(self.bindir, self.exe)
116        if not os.path.exists(fn):  # diagnostics for Windows buildbot failures
117            bd = self.get_env_file(self.bindir)
118            print('Contents of %r:' % bd)
119            print('    %r' % os.listdir(bd))
120        self.assertTrue(os.path.exists(fn), 'File %r should exist.' % fn)
121
122    def test_prompt(self):
123        env_name = os.path.split(self.env_dir)[1]
124
125        rmtree(self.env_dir)
126        builder = venv.EnvBuilder()
127        self.run_with_capture(builder.create, self.env_dir)
128        context = builder.ensure_directories(self.env_dir)
129        data = self.get_text_file_contents('pyvenv.cfg')
130        self.assertEqual(context.prompt, '(%s) ' % env_name)
131        self.assertNotIn("prompt = ", data)
132
133        rmtree(self.env_dir)
134        builder = venv.EnvBuilder(prompt='My prompt')
135        self.run_with_capture(builder.create, self.env_dir)
136        context = builder.ensure_directories(self.env_dir)
137        data = self.get_text_file_contents('pyvenv.cfg')
138        self.assertEqual(context.prompt, '(My prompt) ')
139        self.assertIn("prompt = 'My prompt'\n", data)
140
141    @requireVenvCreate
142    def test_prefixes(self):
143        """
144        Test that the prefix values are as expected.
145        """
146        # check a venv's prefixes
147        rmtree(self.env_dir)
148        self.run_with_capture(venv.create, self.env_dir)
149        envpy = os.path.join(self.env_dir, self.bindir, self.exe)
150        cmd = [envpy, '-c', None]
151        for prefix, expected in (
152            ('prefix', self.env_dir),
153            ('exec_prefix', self.env_dir),
154            ('base_prefix', sys.base_prefix),
155            ('base_exec_prefix', sys.base_exec_prefix)):
156            cmd[2] = 'import sys; print(sys.%s)' % prefix
157            out, err = check_output(cmd)
158            self.assertEqual(out.strip(), expected.encode())
159
160    if sys.platform == 'win32':
161        ENV_SUBDIRS = (
162            ('Scripts',),
163            ('Include',),
164            ('Lib',),
165            ('Lib', 'site-packages'),
166        )
167    else:
168        ENV_SUBDIRS = (
169            ('bin',),
170            ('include',),
171            ('lib',),
172            ('lib', 'python%d.%d' % sys.version_info[:2]),
173            ('lib', 'python%d.%d' % sys.version_info[:2], 'site-packages'),
174        )
175
176    def create_contents(self, paths, filename):
177        """
178        Create some files in the environment which are unrelated
179        to the virtual environment.
180        """
181        for subdirs in paths:
182            d = os.path.join(self.env_dir, *subdirs)
183            os.mkdir(d)
184            fn = os.path.join(d, filename)
185            with open(fn, 'wb') as f:
186                f.write(b'Still here?')
187
188    def test_overwrite_existing(self):
189        """
190        Test creating environment in an existing directory.
191        """
192        self.create_contents(self.ENV_SUBDIRS, 'foo')
193        venv.create(self.env_dir)
194        for subdirs in self.ENV_SUBDIRS:
195            fn = os.path.join(self.env_dir, *(subdirs + ('foo',)))
196            self.assertTrue(os.path.exists(fn))
197            with open(fn, 'rb') as f:
198                self.assertEqual(f.read(), b'Still here?')
199
200        builder = venv.EnvBuilder(clear=True)
201        builder.create(self.env_dir)
202        for subdirs in self.ENV_SUBDIRS:
203            fn = os.path.join(self.env_dir, *(subdirs + ('foo',)))
204            self.assertFalse(os.path.exists(fn))
205
206    def clear_directory(self, path):
207        for fn in os.listdir(path):
208            fn = os.path.join(path, fn)
209            if os.path.islink(fn) or os.path.isfile(fn):
210                os.remove(fn)
211            elif os.path.isdir(fn):
212                rmtree(fn)
213
214    def test_unoverwritable_fails(self):
215        #create a file clashing with directories in the env dir
216        for paths in self.ENV_SUBDIRS[:3]:
217            fn = os.path.join(self.env_dir, *paths)
218            with open(fn, 'wb') as f:
219                f.write(b'')
220            self.assertRaises((ValueError, OSError), venv.create, self.env_dir)
221            self.clear_directory(self.env_dir)
222
223    def test_upgrade(self):
224        """
225        Test upgrading an existing environment directory.
226        """
227        # See Issue #21643: the loop needs to run twice to ensure
228        # that everything works on the upgrade (the first run just creates
229        # the venv).
230        for upgrade in (False, True):
231            builder = venv.EnvBuilder(upgrade=upgrade)
232            self.run_with_capture(builder.create, self.env_dir)
233            self.isdir(self.bindir)
234            self.isdir(self.include)
235            self.isdir(*self.lib)
236            fn = self.get_env_file(self.bindir, self.exe)
237            if not os.path.exists(fn):
238                # diagnostics for Windows buildbot failures
239                bd = self.get_env_file(self.bindir)
240                print('Contents of %r:' % bd)
241                print('    %r' % os.listdir(bd))
242            self.assertTrue(os.path.exists(fn), 'File %r should exist.' % fn)
243
244    def test_isolation(self):
245        """
246        Test isolation from system site-packages
247        """
248        for ssp, s in ((True, 'true'), (False, 'false')):
249            builder = venv.EnvBuilder(clear=True, system_site_packages=ssp)
250            builder.create(self.env_dir)
251            data = self.get_text_file_contents('pyvenv.cfg')
252            self.assertIn('include-system-site-packages = %s\n' % s, data)
253
254    @unittest.skipUnless(can_symlink(), 'Needs symlinks')
255    def test_symlinking(self):
256        """
257        Test symlinking works as expected
258        """
259        for usl in (False, True):
260            builder = venv.EnvBuilder(clear=True, symlinks=usl)
261            builder.create(self.env_dir)
262            fn = self.get_env_file(self.bindir, self.exe)
263            # Don't test when False, because e.g. 'python' is always
264            # symlinked to 'python3.3' in the env, even when symlinking in
265            # general isn't wanted.
266            if usl:
267                if self.cannot_link_exe:
268                    # Symlinking is skipped when our executable is already a
269                    # special app symlink
270                    self.assertFalse(os.path.islink(fn))
271                else:
272                    self.assertTrue(os.path.islink(fn))
273
274    # If a venv is created from a source build and that venv is used to
275    # run the test, the pyvenv.cfg in the venv created in the test will
276    # point to the venv being used to run the test, and we lose the link
277    # to the source build - so Python can't initialise properly.
278    @requireVenvCreate
279    def test_executable(self):
280        """
281        Test that the sys.executable value is as expected.
282        """
283        rmtree(self.env_dir)
284        self.run_with_capture(venv.create, self.env_dir)
285        envpy = os.path.join(os.path.realpath(self.env_dir),
286                             self.bindir, self.exe)
287        out, err = check_output([envpy, '-c',
288            'import sys; print(sys.executable)'])
289        self.assertEqual(out.strip(), envpy.encode())
290
291    @unittest.skipUnless(can_symlink(), 'Needs symlinks')
292    def test_executable_symlinks(self):
293        """
294        Test that the sys.executable value is as expected.
295        """
296        rmtree(self.env_dir)
297        builder = venv.EnvBuilder(clear=True, symlinks=True)
298        builder.create(self.env_dir)
299        envpy = os.path.join(os.path.realpath(self.env_dir),
300                             self.bindir, self.exe)
301        out, err = check_output([envpy, '-c',
302            'import sys; print(sys.executable)'])
303        self.assertEqual(out.strip(), envpy.encode())
304
305    @unittest.skipUnless(os.name == 'nt', 'only relevant on Windows')
306    def test_unicode_in_batch_file(self):
307        """
308        Test handling of Unicode paths
309        """
310        rmtree(self.env_dir)
311        env_dir = os.path.join(os.path.realpath(self.env_dir), 'ϼўТλФЙ')
312        builder = venv.EnvBuilder(clear=True)
313        builder.create(env_dir)
314        activate = os.path.join(env_dir, self.bindir, 'activate.bat')
315        envpy = os.path.join(env_dir, self.bindir, self.exe)
316        out, err = check_output(
317            [activate, '&', self.exe, '-c', 'print(0)'],
318            encoding='oem',
319        )
320        self.assertEqual(out.strip(), '0')
321
322    @requireVenvCreate
323    def test_multiprocessing(self):
324        """
325        Test that the multiprocessing is able to spawn.
326        """
327        # Issue bpo-36342: Instanciation of a Pool object imports the
328        # multiprocessing.synchronize module. Skip the test if this module
329        # cannot be imported.
330        import_module('multiprocessing.synchronize')
331        rmtree(self.env_dir)
332        self.run_with_capture(venv.create, self.env_dir)
333        envpy = os.path.join(os.path.realpath(self.env_dir),
334                             self.bindir, self.exe)
335        out, err = check_output([envpy, '-c',
336            'from multiprocessing import Pool; '
337            'pool = Pool(1); '
338            'print(pool.apply_async("Python".lower).get(3)); '
339            'pool.terminate()'])
340        self.assertEqual(out.strip(), "python".encode())
341
342    @unittest.skipIf(os.name == 'nt', 'not relevant on Windows')
343    def test_deactivate_with_strict_bash_opts(self):
344        bash = shutil.which("bash")
345        if bash is None:
346            self.skipTest("bash required for this test")
347        rmtree(self.env_dir)
348        builder = venv.EnvBuilder(clear=True)
349        builder.create(self.env_dir)
350        activate = os.path.join(self.env_dir, self.bindir, "activate")
351        test_script = os.path.join(self.env_dir, "test_strict.sh")
352        with open(test_script, "w") as f:
353            f.write("set -euo pipefail\n"
354                    f"source {activate}\n"
355                    "deactivate\n")
356        out, err = check_output([bash, test_script])
357        self.assertEqual(out, "".encode())
358        self.assertEqual(err, "".encode())
359
360
361@requireVenvCreate
362class EnsurePipTest(BaseTest):
363    """Test venv module installation of pip."""
364    def assert_pip_not_installed(self):
365        envpy = os.path.join(os.path.realpath(self.env_dir),
366                             self.bindir, self.exe)
367        out, err = check_output([envpy, '-c',
368            'try:\n import pip\nexcept ImportError:\n print("OK")'])
369        # We force everything to text, so unittest gives the detailed diff
370        # if we get unexpected results
371        err = err.decode("latin-1") # Force to text, prevent decoding errors
372        self.assertEqual(err, "")
373        out = out.decode("latin-1") # Force to text, prevent decoding errors
374        self.assertEqual(out.strip(), "OK")
375
376
377    def test_no_pip_by_default(self):
378        rmtree(self.env_dir)
379        self.run_with_capture(venv.create, self.env_dir)
380        self.assert_pip_not_installed()
381
382    def test_explicit_no_pip(self):
383        rmtree(self.env_dir)
384        self.run_with_capture(venv.create, self.env_dir, with_pip=False)
385        self.assert_pip_not_installed()
386
387    def test_devnull(self):
388        # Fix for issue #20053 uses os.devnull to force a config file to
389        # appear empty. However http://bugs.python.org/issue20541 means
390        # that doesn't currently work properly on Windows. Once that is
391        # fixed, the "win_location" part of test_with_pip should be restored
392        with open(os.devnull, "rb") as f:
393            self.assertEqual(f.read(), b"")
394
395        self.assertTrue(os.path.exists(os.devnull))
396
397    def do_test_with_pip(self, system_site_packages):
398        rmtree(self.env_dir)
399        with EnvironmentVarGuard() as envvars:
400            # pip's cross-version compatibility may trigger deprecation
401            # warnings in current versions of Python. Ensure related
402            # environment settings don't cause venv to fail.
403            envvars["PYTHONWARNINGS"] = "e"
404            # ensurepip is different enough from a normal pip invocation
405            # that we want to ensure it ignores the normal pip environment
406            # variable settings. We set PIP_NO_INSTALL here specifically
407            # to check that ensurepip (and hence venv) ignores it.
408            # See http://bugs.python.org/issue19734
409            envvars["PIP_NO_INSTALL"] = "1"
410            # Also check that we ignore the pip configuration file
411            # See http://bugs.python.org/issue20053
412            with tempfile.TemporaryDirectory() as home_dir:
413                envvars["HOME"] = home_dir
414                bad_config = "[global]\nno-install=1"
415                # Write to both config file names on all platforms to reduce
416                # cross-platform variation in test code behaviour
417                win_location = ("pip", "pip.ini")
418                posix_location = (".pip", "pip.conf")
419                # Skips win_location due to http://bugs.python.org/issue20541
420                for dirname, fname in (posix_location,):
421                    dirpath = os.path.join(home_dir, dirname)
422                    os.mkdir(dirpath)
423                    fpath = os.path.join(dirpath, fname)
424                    with open(fpath, 'w') as f:
425                        f.write(bad_config)
426
427                # Actually run the create command with all that unhelpful
428                # config in place to ensure we ignore it
429                try:
430                    self.run_with_capture(venv.create, self.env_dir,
431                                          system_site_packages=system_site_packages,
432                                          with_pip=True)
433                except subprocess.CalledProcessError as exc:
434                    # The output this produces can be a little hard to read,
435                    # but at least it has all the details
436                    details = exc.output.decode(errors="replace")
437                    msg = "{}\n\n**Subprocess Output**\n{}"
438                    self.fail(msg.format(exc, details))
439        # Ensure pip is available in the virtual environment
440        envpy = os.path.join(os.path.realpath(self.env_dir), self.bindir, self.exe)
441        # Ignore DeprecationWarning since pip code is not part of Python
442        out, err = check_output([envpy, '-W', 'ignore::DeprecationWarning', '-I',
443               '-m', 'pip', '--version'])
444        # We force everything to text, so unittest gives the detailed diff
445        # if we get unexpected results
446        err = err.decode("latin-1") # Force to text, prevent decoding errors
447        self.assertEqual(err, "")
448        out = out.decode("latin-1") # Force to text, prevent decoding errors
449        expected_version = "pip {}".format(ensurepip.version())
450        self.assertEqual(out[:len(expected_version)], expected_version)
451        env_dir = os.fsencode(self.env_dir).decode("latin-1")
452        self.assertIn(env_dir, out)
453
454        # http://bugs.python.org/issue19728
455        # Check the private uninstall command provided for the Windows
456        # installers works (at least in a virtual environment)
457        with EnvironmentVarGuard() as envvars:
458            out, err = check_output([envpy,
459                '-W', 'ignore::DeprecationWarning', '-I',
460                '-m', 'ensurepip._uninstall'])
461        # We force everything to text, so unittest gives the detailed diff
462        # if we get unexpected results
463        err = err.decode("latin-1") # Force to text, prevent decoding errors
464        # Ignore the warning:
465        #   "The directory '$HOME/.cache/pip/http' or its parent directory
466        #    is not owned by the current user and the cache has been disabled.
467        #    Please check the permissions and owner of that directory. If
468        #    executing pip with sudo, you may want sudo's -H flag."
469        # where $HOME is replaced by the HOME environment variable.
470        err = re.sub("^(WARNING: )?The directory .* or its parent directory "
471                     "is not owned by the current user .*$", "",
472                     err, flags=re.MULTILINE)
473        self.assertEqual(err.rstrip(), "")
474        # Being fairly specific regarding the expected behaviour for the
475        # initial bundling phase in Python 3.4. If the output changes in
476        # future pip versions, this test can likely be relaxed further.
477        out = out.decode("latin-1") # Force to text, prevent decoding errors
478        self.assertIn("Successfully uninstalled pip", out)
479        self.assertIn("Successfully uninstalled setuptools", out)
480        # Check pip is now gone from the virtual environment. This only
481        # applies in the system_site_packages=False case, because in the
482        # other case, pip may still be available in the system site-packages
483        if not system_site_packages:
484            self.assert_pip_not_installed()
485
486    # Issue #26610: pip/pep425tags.py requires ctypes
487    @unittest.skipUnless(ctypes, 'pip requires ctypes')
488    @requires_zlib
489    def test_with_pip(self):
490        self.do_test_with_pip(False)
491        self.do_test_with_pip(True)
492
493if __name__ == "__main__":
494    unittest.main()
495