1# Copyright 2014 Altera Corporation. All Rights Reserved. 2# Copyright 2015-2017 John McGehee 3# 4# Licensed under the Apache License, Version 2.0 (the "License"); 5# you may not use this file except in compliance with the License. 6# You may obtain a copy of the License at 7# 8# http://www.apache.org/licenses/LICENSE-2.0 9# 10# Unless required by applicable law or agreed to in writing, software 11# distributed under the License is distributed on an "AS IS" BASIS, 12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13# See the License for the specific language governing permissions and 14# limitations under the License. 15 16"""This module provides a base class derived from `unittest.TestClass` 17for unit tests using the :py:class:`pyfakefs` module. 18 19`fake_filesystem_unittest.TestCase` searches `sys.modules` for modules 20that import the `os`, `io`, `path` `shutil`, and `pathlib` modules. 21 22The `setUpPyfakefs()` method binds these modules to the corresponding fake 23modules from `pyfakefs`. Further, the `open()` built-in is bound to a fake 24`open()`. 25 26It is expected that `setUpPyfakefs()` be invoked at the beginning of the 27derived class' `setUp()` method. There is no need to add anything to the 28derived class' `tearDown()` method. 29 30During the test, everything uses the fake file system and modules. This means 31that even in your test fixture, familiar functions like `open()` and 32`os.makedirs()` manipulate the fake file system. 33 34Existing unit tests that use the real file system can be retrofitted to use 35pyfakefs by simply changing their base class from `:py:class`unittest.TestCase` 36to `:py:class`pyfakefs.fake_filesystem_unittest.TestCase`. 37""" 38 39import _io # type:ignore[import] 40import doctest 41import functools 42import genericpath 43import glob 44import inspect 45import io 46import linecache 47import os 48import shutil 49import sys 50import tempfile 51import tokenize 52import unittest 53import warnings 54from importlib import reload 55from importlib.abc import Loader, MetaPathFinder 56from importlib.machinery import ModuleSpec 57from importlib.util import spec_from_file_location, module_from_spec 58from types import ModuleType, TracebackType, FunctionType 59from typing import ( 60 Any, 61 Callable, 62 Dict, 63 List, 64 Set, 65 Tuple, 66 Optional, 67 Union, 68 Type, 69 Iterator, 70 cast, 71 ItemsView, 72 Sequence, 73) 74from unittest import TestSuite 75 76from pyfakefs import fake_filesystem, fake_io, fake_os, fake_open, fake_path, fake_file 77from pyfakefs import fake_filesystem_shutil 78from pyfakefs import fake_legacy_modules 79from pyfakefs import fake_pathlib 80from pyfakefs import mox3_stubout 81from pyfakefs.fake_filesystem import ( 82 set_uid, 83 set_gid, 84 reset_ids, 85 PatchMode, 86 FakeFilesystem, 87) 88from pyfakefs.fake_os import use_original_os 89from pyfakefs.helpers import IS_PYPY 90from pyfakefs.legacy_packages import pathlib2, scandir 91from pyfakefs.mox3_stubout import StubOutForTesting 92 93OS_MODULE = "nt" if sys.platform == "win32" else "posix" 94PATH_MODULE = "ntpath" if sys.platform == "win32" else "posixpath" 95 96 97class TempfilePatcher: 98 """Handles tempfile patching for Posix systems.""" 99 100 def __init__(self): 101 self.tempfile_cleanup = None 102 103 def start_patching(self): 104 if self.tempfile_cleanup is not None: 105 return 106 if sys.version_info >= (3, 12): 107 108 def cleanup(self_, windows=(os.name == "nt"), unlink=None): 109 self.tempfile_cleanup(self_, windows, unlink or os.unlink) 110 111 self.tempfile_cleanup = tempfile._TemporaryFileCloser.cleanup # type: ignore[module-attr] 112 tempfile._TemporaryFileCloser.cleanup = cleanup # type: ignore[module-attr] 113 elif sys.platform != "win32": 114 115 def close(self_, unlink=None): 116 self.tempfile_cleanup(self_, unlink or os.unlink) 117 118 self.tempfile_cleanup = tempfile._TemporaryFileCloser.close # type: ignore[module-attr] 119 tempfile._TemporaryFileCloser.close = close # type: ignore[module-attr] 120 121 def stop_patching(self): 122 if self.tempfile_cleanup is None: 123 return 124 if sys.version_info < (3, 12): 125 tempfile._TemporaryFileCloser.close = self.tempfile_cleanup # type: ignore[module-attr] 126 else: 127 tempfile._TemporaryFileCloser.cleanup = self.tempfile_cleanup # type: ignore[module-attr] 128 self.tempfile_cleanup = None 129 # reset the cached tempdir in tempfile 130 tempfile.tempdir = None 131 132 133def patchfs( 134 _func: Optional[Callable] = None, 135 *, 136 additional_skip_names: Optional[List[Union[str, ModuleType]]] = None, 137 modules_to_reload: Optional[List[ModuleType]] = None, 138 modules_to_patch: Optional[Dict[str, ModuleType]] = None, 139 allow_root_user: bool = True, 140 use_known_patches: bool = True, 141 patch_open_code: PatchMode = PatchMode.OFF, 142 patch_default_args: bool = False, 143 use_cache: bool = True, 144 use_dynamic_patch: bool = True, 145) -> Callable: 146 """Convenience decorator to use patcher with additional parameters in a 147 test function. 148 149 Usage:: 150 151 @patchfs 152 def test_my_function(fake_fs): 153 fake_fs.create_file('foo') 154 155 @patchfs(allow_root_user=False) 156 def test_with_patcher_args(fs): 157 os.makedirs('foo/bar') 158 """ 159 160 def wrap_patchfs(f: Callable) -> Callable: 161 @functools.wraps(f) 162 def wrapped(*args, **kwargs): 163 with Patcher( 164 additional_skip_names=additional_skip_names, 165 modules_to_reload=modules_to_reload, 166 modules_to_patch=modules_to_patch, 167 allow_root_user=allow_root_user, 168 use_known_patches=use_known_patches, 169 patch_open_code=patch_open_code, 170 patch_default_args=patch_default_args, 171 use_cache=use_cache, 172 use_dynamic_patch=use_dynamic_patch, 173 ) as p: 174 args = list(args) 175 args.append(p.fs) 176 return f(*args, **kwargs) 177 178 return wrapped 179 180 if _func: 181 if not callable(_func): 182 raise TypeError( 183 "Decorator argument is not a function.\n" 184 "Did you mean `@patchfs(additional_skip_names=...)`?" 185 ) 186 if hasattr(_func, "patchings"): 187 _func.nr_patches = len(_func.patchings) # type: ignore 188 return wrap_patchfs(_func) 189 190 return wrap_patchfs 191 192 193DOCTEST_PATCHER = None 194 195 196def load_doctests( 197 loader: Any, 198 tests: TestSuite, 199 ignore: Any, 200 module: ModuleType, 201 additional_skip_names: Optional[List[Union[str, ModuleType]]] = None, 202 modules_to_reload: Optional[List[ModuleType]] = None, 203 modules_to_patch: Optional[Dict[str, ModuleType]] = None, 204 allow_root_user: bool = True, 205 use_known_patches: bool = True, 206 patch_open_code: PatchMode = PatchMode.OFF, 207 patch_default_args: bool = False, 208 use_dynamic_patch: bool = True, 209) -> TestSuite: # pylint:disable=unused-argument 210 """Load the doctest tests for the specified module into unittest. 211 Args: 212 loader, tests, ignore : arguments passed in from `load_tests()` 213 module: module under test 214 remaining args: see :py:class:`TestCase` for an explanation 215 216 File `example_test.py` in the pyfakefs release provides a usage example. 217 """ 218 has_patcher = Patcher.DOC_PATCHER is not None 219 if not has_patcher: 220 Patcher.DOC_PATCHER = Patcher( 221 additional_skip_names=additional_skip_names, 222 modules_to_reload=modules_to_reload, 223 modules_to_patch=modules_to_patch, 224 allow_root_user=allow_root_user, 225 use_known_patches=use_known_patches, 226 patch_open_code=patch_open_code, 227 patch_default_args=patch_default_args, 228 use_dynamic_patch=use_dynamic_patch, 229 is_doc_test=True, 230 ) 231 assert Patcher.DOC_PATCHER is not None 232 globs = Patcher.DOC_PATCHER.replace_globs(vars(module)) 233 tests.addTests( 234 doctest.DocTestSuite( 235 module, 236 globs=globs, 237 setUp=Patcher.DOC_PATCHER.setUp, 238 tearDown=Patcher.DOC_PATCHER.tearDown, 239 ) 240 ) 241 return tests 242 243 244class TestCaseMixin: 245 """Test case mixin that automatically replaces file-system related 246 modules by fake implementations. 247 248 Attributes: 249 additional_skip_names: names of modules where no module 250 replacement shall be performed, in addition to the names in 251 :py:attr:`fake_filesystem_unittest.Patcher.SKIPNAMES`. 252 Instead of the module names, the modules themselves may be used. 253 modules_to_reload: A list of modules that need to be reloaded 254 to be patched dynamically; may be needed if the module 255 imports file system modules under an alias 256 257 .. caution:: Reloading modules may have unwanted side effects. 258 modules_to_patch: A dictionary of fake modules mapped to the 259 fully qualified patched module names. Can be used to add patching 260 of modules not provided by `pyfakefs`. 261 262 If you specify some of these attributes here, and you have DocTests, 263 consider also specifying the same arguments to :py:func:`load_doctests`. 264 265 Example usage in derived test classes:: 266 267 from unittest import TestCase 268 from fake_filesystem_unittest import TestCaseMixin 269 270 class MyTestCase(TestCase, TestCaseMixin): 271 def __init__(self, methodName='runTest'): 272 super(MyTestCase, self).__init__( 273 methodName=methodName, 274 additional_skip_names=['posixpath']) 275 276 import sut 277 278 class AnotherTestCase(TestCase, TestCaseMixin): 279 def __init__(self, methodName='runTest'): 280 super(MyTestCase, self).__init__( 281 methodName=methodName, modules_to_reload=[sut]) 282 """ 283 284 additional_skip_names: Optional[List[Union[str, ModuleType]]] = None 285 modules_to_reload: Optional[List[ModuleType]] = None 286 modules_to_patch: Optional[Dict[str, ModuleType]] = None 287 288 @property 289 def patcher(self): 290 if hasattr(self, "_patcher"): 291 return self._patcher or Patcher.PATCHER 292 return Patcher.PATCHER 293 294 @property 295 def fs(self) -> FakeFilesystem: 296 return cast(FakeFilesystem, self.patcher.fs) 297 298 def setUpPyfakefs( 299 self, 300 additional_skip_names: Optional[List[Union[str, ModuleType]]] = None, 301 modules_to_reload: Optional[List[ModuleType]] = None, 302 modules_to_patch: Optional[Dict[str, ModuleType]] = None, 303 allow_root_user: bool = True, 304 use_known_patches: bool = True, 305 patch_open_code: PatchMode = PatchMode.OFF, 306 patch_default_args: bool = False, 307 use_cache: bool = True, 308 use_dynamic_patch: bool = True, 309 ) -> None: 310 """Bind the file-related modules to the :py:class:`pyfakefs` fake file 311 system instead of the real file system. Also bind the fake `open()` 312 function. 313 314 Invoke this at the beginning of the `setUp()` method in your unit test 315 class. 316 For the arguments, see the `TestCaseMixin` attribute description. 317 If any of the arguments is not None, it overwrites the settings for 318 the current test case. Settings the arguments here may be a more 319 convenient way to adapt the setting than overwriting `__init__()`. 320 """ 321 # if the class has already a patcher setup, we use this one 322 if Patcher.PATCHER is not None: 323 return 324 325 if additional_skip_names is None: 326 additional_skip_names = self.additional_skip_names 327 if modules_to_reload is None: 328 modules_to_reload = self.modules_to_reload 329 if modules_to_patch is None: 330 modules_to_patch = self.modules_to_patch 331 self._patcher = Patcher( 332 additional_skip_names=additional_skip_names, 333 modules_to_reload=modules_to_reload, 334 modules_to_patch=modules_to_patch, 335 allow_root_user=allow_root_user, 336 use_known_patches=use_known_patches, 337 patch_open_code=patch_open_code, 338 patch_default_args=patch_default_args, 339 use_cache=use_cache, 340 use_dynamic_patch=use_dynamic_patch, 341 ) 342 343 self._patcher.setUp() 344 cast(TestCase, self).addCleanup(self._patcher.tearDown) 345 346 @classmethod 347 def setUpClassPyfakefs( 348 cls, 349 additional_skip_names: Optional[List[Union[str, ModuleType]]] = None, 350 modules_to_reload: Optional[List[ModuleType]] = None, 351 modules_to_patch: Optional[Dict[str, ModuleType]] = None, 352 allow_root_user: bool = True, 353 use_known_patches: bool = True, 354 patch_open_code: PatchMode = PatchMode.OFF, 355 patch_default_args: bool = False, 356 use_cache: bool = True, 357 use_dynamic_patch: bool = True, 358 ) -> None: 359 """Similar to :py:func:`setUpPyfakefs`, but as a class method that 360 can be used in `setUpClass` instead of in `setUp`. 361 The fake filesystem will live in all test methods in the test class 362 and can be used in the usual way. 363 Note that using both :py:func:`setUpClassPyfakefs` and 364 :py:func:`setUpPyfakefs` in the same class will not work correctly. 365 366 .. note:: This method is only available from Python 3.8 onwards. 367 .. note:: If using `pytest` as testrunner, you need at least pytest 6.2 368 for this method to work. 369 """ 370 if sys.version_info < (3, 8): 371 raise NotImplementedError( 372 "setUpClassPyfakefs is only available in " 373 "Python versions starting from 3.8" 374 ) 375 376 # if the class has already a patcher setup, we use this one 377 if Patcher.PATCHER is not None: 378 return 379 380 if additional_skip_names is None: 381 additional_skip_names = cls.additional_skip_names 382 if modules_to_reload is None: 383 modules_to_reload = cls.modules_to_reload 384 if modules_to_patch is None: 385 modules_to_patch = cls.modules_to_patch 386 Patcher.PATCHER = Patcher( 387 additional_skip_names=additional_skip_names, 388 modules_to_reload=modules_to_reload, 389 modules_to_patch=modules_to_patch, 390 allow_root_user=allow_root_user, 391 use_known_patches=use_known_patches, 392 patch_open_code=patch_open_code, 393 patch_default_args=patch_default_args, 394 use_cache=use_cache, 395 use_dynamic_patch=use_dynamic_patch, 396 ) 397 398 Patcher.PATCHER.setUp() 399 cast(TestCase, cls).addClassCleanup(Patcher.PATCHER.tearDown) 400 401 @classmethod 402 def fake_fs(cls): 403 """Convenience class method for accessing the fake filesystem. 404 For use inside `setUpClass`, after :py:func:`setUpClassPyfakefs` 405 has been called. 406 """ 407 if Patcher.PATCHER: 408 return Patcher.PATCHER.fs 409 return None 410 411 def pause(self) -> None: 412 """Pause the patching of the file system modules until `resume` is 413 called. After that call, all file system calls are executed in the 414 real file system. 415 Calling pause() twice is silently ignored. 416 417 """ 418 self.patcher.pause() 419 420 def resume(self) -> None: 421 """Resume the patching of the file system modules if `pause` has 422 been called before. After that call, all file system calls are 423 executed in the fake file system. 424 Does nothing if patching is not paused. 425 """ 426 self.patcher.resume() 427 428 429class TestCase(unittest.TestCase, TestCaseMixin): 430 """Test case class that automatically replaces file-system related 431 modules by fake implementations. Inherits :py:class:`TestCaseMixin`. 432 433 The arguments are explained in :py:class:`TestCaseMixin`. 434 """ 435 436 def __init__( 437 self, 438 methodName: str = "runTest", 439 additional_skip_names: Optional[List[Union[str, ModuleType]]] = None, 440 modules_to_reload: Optional[List[ModuleType]] = None, 441 modules_to_patch: Optional[Dict[str, ModuleType]] = None, 442 ): 443 """Creates the test class instance and the patcher used to stub out 444 file system related modules. 445 446 Args: 447 methodName: The name of the test method (same as in 448 unittest.TestCase) 449 """ 450 super().__init__(methodName) 451 452 self.additional_skip_names = additional_skip_names 453 self.modules_to_reload = modules_to_reload 454 self.modules_to_patch = modules_to_patch 455 456 def tearDownPyfakefs(self) -> None: 457 """This method is deprecated and exists only for backward 458 compatibility. It does nothing. 459 """ 460 461 462class Patcher: 463 """ 464 Instantiate a stub creator to bind and un-bind the file-related modules to 465 the :py:mod:`pyfakefs` fake modules. 466 467 The arguments are explained in :py:class:`TestCaseMixin`. 468 469 :py:class:`Patcher` is used in :py:class:`TestCaseMixin`. 470 :py:class:`Patcher` also works as a context manager for other tests:: 471 472 with Patcher(): 473 doStuff() 474 """ 475 476 """Stub nothing that is imported within these modules. 477 `sys` is included to prevent `sys.path` from being stubbed with the fake 478 `os.path`. 479 The `linecache` module is used to read the test file in case of test 480 failure to get traceback information before test tear down. 481 In order to make sure that reading the test file is not faked, 482 we skip faking the module. 483 We also have to set back the cached open function in tokenize. 484 """ 485 SKIPMODULES = { 486 None, 487 fake_filesystem, 488 fake_filesystem_shutil, 489 fake_os, 490 fake_io, 491 fake_open, 492 fake_path, 493 fake_file, 494 sys, 495 linecache, 496 tokenize, 497 os, 498 io, 499 _io, 500 genericpath, 501 os.path, 502 } 503 if sys.platform == "win32": 504 import nt # type:ignore[import] 505 import ntpath 506 507 SKIPMODULES.add(nt) 508 SKIPMODULES.add(ntpath) 509 else: 510 import posix 511 import posixpath 512 import fcntl 513 514 SKIPMODULES.add(posix) 515 SKIPMODULES.add(posixpath) 516 SKIPMODULES.add(fcntl) 517 518 # a list of modules detected at run-time 519 # each tool defines one or more module name prefixes for modules to be skipped 520 RUNTIME_SKIPMODULES = { 521 "pydevd": ["_pydevd_", "pydevd", "_pydev_"], # Python debugger (PyCharm/VSCode) 522 "_jb_runner_tools": ["_jb_"], # JetBrains tools 523 } 524 525 # caches all modules that do not have file system modules or function 526 # to speed up _find_modules 527 CACHED_MODULES: Set[ModuleType] = set() 528 FS_MODULES: Dict[str, Set[Tuple[ModuleType, str]]] = {} 529 FS_FUNCTIONS: Dict[Tuple[str, str, str], Set[ModuleType]] = {} 530 FS_DEFARGS: List[Tuple[FunctionType, int, Callable[..., Any]]] = [] 531 SKIPPED_FS_MODULES: Dict[str, Set[Tuple[ModuleType, str]]] = {} 532 533 assert None in SKIPMODULES, "sys.modules contains 'None' values; must skip them." 534 535 IS_WINDOWS = sys.platform in ("win32", "cygwin") 536 537 SKIPNAMES: Set[str] = set() 538 539 # hold values from last call - if changed, the cache has to be invalidated 540 PATCHED_MODULE_NAMES: Set[str] = set() 541 ADDITIONAL_SKIP_NAMES: Set[str] = set() 542 PATCH_DEFAULT_ARGS = False 543 PATCHER: Optional["Patcher"] = None 544 DOC_PATCHER: Optional["Patcher"] = None 545 REF_COUNT = 0 546 DOC_REF_COUNT = 0 547 548 def __new__(cls, *args, **kwargs): 549 if kwargs.get("is_doc_test", False): 550 if cls.DOC_PATCHER is None: 551 cls.DOC_PATCHER = super().__new__(cls) 552 return cls.DOC_PATCHER 553 if cls.PATCHER is None: 554 cls.PATCHER = super().__new__(cls) 555 return cls.PATCHER 556 557 def __init__( 558 self, 559 additional_skip_names: Optional[List[Union[str, ModuleType]]] = None, 560 modules_to_reload: Optional[List[ModuleType]] = None, 561 modules_to_patch: Optional[Dict[str, ModuleType]] = None, 562 allow_root_user: bool = True, 563 use_known_patches: bool = True, 564 patch_open_code: PatchMode = PatchMode.OFF, 565 patch_default_args: bool = False, 566 use_cache: bool = True, 567 use_dynamic_patch: bool = True, 568 is_doc_test: bool = False, 569 ) -> None: 570 """ 571 Args: 572 additional_skip_names: names of modules where no module 573 replacement shall be performed, in addition to the names in 574 :py:attr:`fake_filesystem_unittest.Patcher.SKIPNAMES`. 575 Instead of the module names, the modules themselves 576 may be used. 577 modules_to_reload: A list of modules that need to be reloaded 578 to be patched dynamically; may be needed if the module 579 imports file system modules under an alias 580 581 .. caution:: Reloading modules may have unwanted side effects. 582 modules_to_patch: A dictionary of fake modules mapped to the 583 fully qualified patched module names. Can be used to add 584 patching of modules not provided by `pyfakefs`. 585 allow_root_user: If True (default), if the test is run as root 586 user, the user in the fake file system is also considered a 587 root user, otherwise it is always considered a regular user. 588 use_known_patches: If True (the default), some patches for commonly 589 used packages are applied which make them usable with pyfakefs. 590 patch_open_code: If True, `io.open_code` is patched. The default 591 is not to patch it, as it mostly is used to load compiled 592 modules that are not in the fake file system. 593 patch_default_args: If True, default arguments are checked for 594 file system functions, which are patched. This check is 595 expansive, so it is off by default. 596 use_cache: If True (default), patched and non-patched modules are 597 cached between tests for performance reasons. As this is a new 598 feature, this argument allows to turn it off in case it 599 causes any problems. 600 use_dynamic_patch: If `True`, dynamic patching after setup is used 601 (for example for modules loaded locally inside of functions). 602 Can be switched off if it causes unwanted side effects. 603 """ 604 self.is_doc_test = is_doc_test 605 if is_doc_test: 606 if self.DOC_REF_COUNT > 0: 607 return 608 elif self.REF_COUNT > 0: 609 return 610 if not allow_root_user: 611 # set non-root IDs even if the real user is root 612 set_uid(1) 613 set_gid(1) 614 615 self.skip_names = self.SKIPNAMES.copy() 616 # save the original open function for use in pytest plugin 617 self.original_open = open 618 self.patch_open_code = patch_open_code 619 self.linecache_updatecache = None 620 self.linecache_checkcache = None 621 self.tempfile_patcher = TempfilePatcher() 622 623 if additional_skip_names is not None: 624 skip_names = [ 625 cast(ModuleType, m).__name__ if inspect.ismodule(m) else cast(str, m) 626 for m in additional_skip_names 627 ] 628 self.skip_names.update(skip_names) 629 630 self._fake_module_classes: Dict[str, Any] = {} 631 self._unfaked_module_classes: Dict[str, Any] = {} 632 self._class_modules: Dict[str, List[str]] = {} 633 self._init_fake_module_classes() 634 635 # reload tempfile under posix to patch default argument 636 self.modules_to_reload: List[ModuleType] = [] 637 if modules_to_reload is not None: 638 self.modules_to_reload.extend(modules_to_reload) 639 self.patch_default_args = patch_default_args 640 self.use_cache = use_cache 641 self.use_dynamic_patch = use_dynamic_patch 642 self.cleanup_handlers: Dict[str, Callable[[str], bool]] = {} 643 644 if use_known_patches: 645 from pyfakefs.patched_packages import ( 646 get_modules_to_patch, 647 get_classes_to_patch, 648 get_fake_module_classes, 649 get_cleanup_handlers, 650 ) 651 652 modules_to_patch = modules_to_patch or {} 653 modules_to_patch.update(get_modules_to_patch()) 654 self._class_modules.update(get_classes_to_patch()) 655 self._fake_module_classes.update(get_fake_module_classes()) 656 self.cleanup_handlers.update(get_cleanup_handlers()) 657 658 if modules_to_patch is not None: 659 for name, fake_module in modules_to_patch.items(): 660 self._fake_module_classes[name] = fake_module 661 patched_module_names = set(modules_to_patch) 662 else: 663 patched_module_names = set() 664 clear_cache = not use_cache 665 if use_cache: 666 if patched_module_names != self.PATCHED_MODULE_NAMES: 667 self.__class__.PATCHED_MODULE_NAMES = patched_module_names 668 clear_cache = True 669 if self.skip_names != self.ADDITIONAL_SKIP_NAMES: 670 self.__class__.ADDITIONAL_SKIP_NAMES = self.skip_names 671 clear_cache = True 672 if patch_default_args != self.PATCH_DEFAULT_ARGS: 673 self.__class__.PATCH_DEFAULT_ARGS = patch_default_args 674 clear_cache = True 675 676 if clear_cache: 677 self.clear_cache() 678 self._fake_module_functions: Dict[str, Dict] = {} 679 self._init_fake_module_functions() 680 681 # Attributes set by _refresh() 682 self._stubs: Optional[StubOutForTesting] = None 683 self.fs: Optional[FakeFilesystem] = None 684 self.fake_modules: Dict[str, Any] = {} 685 self.unfaked_modules: Dict[str, Any] = {} 686 687 # _isStale is set by tearDown(), reset by _refresh() 688 self._isStale = True 689 self._dyn_patcher: Optional[DynamicPatcher] = None 690 self._patching = False 691 self._paused = False 692 693 def checkcache(self, filename=None): 694 """Calls the original linecache.checkcache making sure no fake OS calls 695 are used.""" 696 with use_original_os(): 697 return self.linecache_checkcache(filename) 698 699 def updatecache(self, filename, module_globals=None): 700 """Calls the original linecache.updatecache making sure no fake OS calls 701 are used.""" 702 with use_original_os(): 703 # workaround for updatecache problem with pytest under Windows, see #1096 704 if not filename.endswith(r"pytest.exe\__main__.py"): 705 return self.linecache_updatecache(filename, module_globals) 706 return [] 707 708 @classmethod 709 def clear_fs_cache(cls) -> None: 710 """Clear the module cache.""" 711 cls.CACHED_MODULES = set() 712 cls.FS_MODULES = {} 713 cls.FS_FUNCTIONS = {} 714 cls.FS_DEFARGS = [] 715 cls.SKIPPED_FS_MODULES = {} 716 717 def clear_cache(self) -> None: 718 """Clear the module cache (convenience instance method).""" 719 self.__class__.clear_fs_cache() 720 721 def register_cleanup_handler(self, name: str, handler: Callable[[str], bool]): 722 """Register a handler for cleaning up a module after it had been loaded by 723 the dynamic patcher. This allows to handle modules that cannot be reloaded 724 without unwanted side effects. 725 726 Args: 727 name: The fully qualified module name. 728 handler: A callable that may do any module cleanup, or do nothing 729 and return `True` in case reloading shall be prevented. 730 731 Returns: 732 `True` if no further cleanup/reload shall occur after the handler is 733 executed, `False` if the cleanup/reload shall still happen. 734 """ 735 self.cleanup_handlers[name] = handler 736 737 def _init_fake_module_classes(self) -> None: 738 # IMPORTANT TESTING NOTE: Whenever you add a new module below, test 739 # it by adding an attribute in fixtures/module_with_attributes.py 740 # and a test in fake_filesystem_unittest_test.py, class 741 # TestAttributesWithFakeModuleNames. 742 self._fake_module_classes = { 743 "os": fake_os.FakeOsModule, 744 "shutil": fake_filesystem_shutil.FakeShutilModule, 745 "io": fake_io.FakeIoModule, 746 "pathlib": fake_pathlib.FakePathlibModule, 747 } 748 if sys.version_info >= (3, 13): 749 # for Python 3.13, we need both pathlib (path with __init__.py) and 750 # pathlib._local (has the actual implementation); 751 # depending on how pathlib is imported, either may be used 752 self._fake_module_classes["pathlib._local"] = fake_pathlib.FakePathlibModule 753 if IS_PYPY or sys.version_info >= (3, 12): 754 # in PyPy and later cpython versions, the module is referenced as _io 755 self._fake_module_classes["_io"] = fake_io.FakeIoModule2 756 if sys.platform == "win32": 757 self._fake_module_classes["nt"] = fake_path.FakeNtModule 758 else: 759 self._fake_module_classes["fcntl"] = fake_filesystem.FakeFcntlModule 760 761 # class modules maps class names against a list of modules they can 762 # be contained in - this allows for alternative modules like 763 # `pathlib` and `pathlib2` 764 self._class_modules["Path"] = ["pathlib"] 765 if sys.version_info >= (3, 13): 766 self._class_modules["Path"].append("pathlib._local") 767 self._unfaked_module_classes["pathlib"] = fake_pathlib.RealPathlibModule 768 if sys.version_info >= (3, 13): 769 self._unfaked_module_classes["pathlib._local"] = ( 770 fake_pathlib.RealPathlibModule 771 ) 772 if pathlib2: 773 self._fake_module_classes["pathlib2"] = ( 774 fake_legacy_modules.FakePathlib2Module 775 ) 776 self._class_modules["Path"].append("pathlib2") 777 self._unfaked_module_classes["pathlib2"] = fake_pathlib.RealPathlibModule 778 if scandir: 779 self._fake_module_classes["scandir"] = fake_legacy_modules.FakeScanDirModule 780 self._fake_module_classes["Path"] = fake_pathlib.FakePathlibPathModule 781 self._unfaked_module_classes["Path"] = fake_pathlib.RealPathlibPathModule 782 783 def _init_fake_module_functions(self) -> None: 784 # handle patching function imported separately like 785 # `from os import stat` 786 # each patched function name has to be looked up separately 787 for mod_name, fake_module in self._fake_module_classes.items(): 788 if hasattr(fake_module, "dir"): 789 module_dir = fake_module.dir 790 if inspect.isfunction(module_dir): 791 for fct_name in fake_module.dir(): 792 module_attr = (getattr(fake_module, fct_name), mod_name) 793 self._fake_module_functions.setdefault(fct_name, {})[ 794 mod_name 795 ] = module_attr 796 if mod_name == "os": 797 self._fake_module_functions.setdefault(fct_name, {})[ 798 OS_MODULE 799 ] = module_attr 800 801 # special handling for functions in os.path 802 fake_module = fake_filesystem.FakePathModule 803 for fct_name in fake_module.dir(): 804 module_attr = (getattr(fake_module, fct_name), PATH_MODULE) 805 self._fake_module_functions.setdefault(fct_name, {})["genericpath"] = ( 806 module_attr 807 ) 808 self._fake_module_functions.setdefault(fct_name, {})[PATH_MODULE] = ( 809 module_attr 810 ) 811 812 def __enter__(self) -> "Patcher": 813 """Context manager for usage outside of 814 fake_filesystem_unittest.TestCase. 815 Ensure that all patched modules are removed in case of an 816 unhandled exception. 817 """ 818 self.setUp() 819 return self 820 821 def __exit__( 822 self, 823 exc_type: Optional[Type[BaseException]], 824 exc_val: Optional[BaseException], 825 exc_tb: Optional[TracebackType], 826 ) -> None: 827 self.tearDown() 828 829 def _is_fs_module( 830 self, mod: ModuleType, name: str, module_names: List[str] 831 ) -> bool: 832 try: 833 return ( 834 inspect.ismodule(mod) 835 and mod.__name__ in module_names 836 or inspect.isclass(mod) 837 and mod.__module__ in self._class_modules.get(name, []) 838 ) 839 except Exception: 840 # handle cases where the module has no __name__ or __module__ 841 # attribute - see #460, and any other exception triggered 842 # by inspect functions 843 return False 844 845 def _is_fs_function(self, fct: FunctionType) -> bool: 846 try: 847 return ( 848 (inspect.isfunction(fct) or inspect.isbuiltin(fct)) 849 and fct.__name__ in self._fake_module_functions 850 and fct.__module__ in self._fake_module_functions[fct.__name__] 851 ) 852 except Exception: 853 # handle cases where the function has no __name__ or __module__ 854 # attribute, or any other exception in inspect functions 855 return False 856 857 def _def_values( 858 self, item: FunctionType 859 ) -> Iterator[Tuple[FunctionType, int, Any]]: 860 """Find default arguments that are file-system functions to be 861 patched in top-level functions and members of top-level classes.""" 862 # check for module-level functions 863 try: 864 if item.__defaults__ and inspect.isfunction(item): 865 for i, d in enumerate(item.__defaults__): 866 if self._is_fs_function(d): 867 yield item, i, d 868 except Exception: 869 pass 870 try: 871 if inspect.isclass(item): 872 # check for methods in class 873 # (nested classes are ignored for now) 874 # inspect.getmembers is very expansive! 875 for m in inspect.getmembers(item, predicate=inspect.isfunction): 876 f = cast(FunctionType, m[1]) 877 if f.__defaults__: 878 for i, d in enumerate(f.__defaults__): 879 if self._is_fs_function(d): 880 yield f, i, d 881 except Exception: 882 # Ignore any exception, examples: 883 # ImportError: No module named '_gdbm' 884 # _DontDoThat() (see #523) 885 pass 886 887 def _find_def_values(self, module_items: ItemsView[str, FunctionType]) -> None: 888 for _, fct in module_items: 889 for f, i, d in self._def_values(fct): 890 self.__class__.FS_DEFARGS.append((f, i, d)) 891 892 def _find_modules(self) -> None: 893 """Find and cache all modules that import file system modules. 894 Later, `setUp()` will stub these with the fake file system 895 modules. 896 """ 897 module_names = list(self._fake_module_classes.keys()) + [PATH_MODULE] 898 for name, module in list(sys.modules.items()): 899 try: 900 if ( 901 self.use_cache 902 and module in self.CACHED_MODULES 903 or not inspect.ismodule(module) 904 ): 905 continue 906 except Exception: 907 # workaround for some py (part of pytest) versions 908 # where py.error has no __name__ attribute 909 # see https://github.com/pytest-dev/py/issues/73 910 # and any other exception triggered by inspect.ismodule 911 if self.use_cache: 912 try: 913 self.__class__.CACHED_MODULES.add(module) 914 except TypeError: 915 # unhashable module - don't cache it 916 pass 917 continue 918 skipped = module in self.SKIPMODULES or any( 919 [sn.startswith(module.__name__) for sn in self.skip_names] 920 ) 921 module_items = module.__dict__.copy().items() 922 923 modules = { 924 name: mod 925 for name, mod in module_items 926 if self._is_fs_module(mod, name, module_names) 927 } 928 929 if skipped: 930 for name, mod in modules.items(): 931 self.__class__.SKIPPED_FS_MODULES.setdefault(name, set()).add( 932 (module, mod.__name__) 933 ) 934 else: 935 for name, mod in modules.items(): 936 self.__class__.FS_MODULES.setdefault(name, set()).add( 937 (module, mod.__name__) 938 ) 939 functions = { 940 name: fct for name, fct in module_items if self._is_fs_function(fct) 941 } 942 943 for name, fct in functions.items(): 944 self.__class__.FS_FUNCTIONS.setdefault( 945 (name, fct.__name__, fct.__module__), set() 946 ).add(module) 947 948 # find default arguments that are file system functions 949 if self.patch_default_args: 950 self._find_def_values(module_items) 951 952 if self.use_cache: 953 self.__class__.CACHED_MODULES.add(module) 954 955 def _refresh(self) -> None: 956 """Renew the fake file system and set the _isStale flag to `False`.""" 957 if self._stubs is not None: 958 self._stubs.smart_unset_all() 959 self._stubs = mox3_stubout.StubOutForTesting() 960 961 self.fs = fake_filesystem.FakeFilesystem(patcher=self, create_temp_dir=True) 962 self.fs.patch_open_code = self.patch_open_code 963 for name in self._fake_module_classes: 964 self.fake_modules[name] = self._fake_module_classes[name](self.fs) 965 if hasattr(self.fake_modules[name], "skip_names"): 966 self.fake_modules[name].skip_names = self.skip_names 967 self.fake_modules[PATH_MODULE] = self.fake_modules["os"].path 968 for name in self._unfaked_module_classes: 969 self.unfaked_modules[name] = self._unfaked_module_classes[name]() 970 971 self._isStale = False 972 973 def setUp(self, doctester: Any = None) -> None: 974 """Bind the file-related modules to the :py:mod:`pyfakefs` fake 975 modules real ones. Also bind the fake `file()` and `open()` functions. 976 """ 977 if self.is_doc_test: 978 self.__class__.DOC_REF_COUNT += 1 979 if self.__class__.DOC_REF_COUNT > 1: 980 return 981 else: 982 self.__class__.REF_COUNT += 1 983 if self.__class__.REF_COUNT > 1: 984 return 985 self.has_fcopy_file = ( 986 sys.platform == "darwin" 987 and hasattr(shutil, "_HAS_FCOPYFILE") 988 and shutil._HAS_FCOPYFILE 989 ) 990 if self.has_fcopy_file: 991 shutil._HAS_FCOPYFILE = False # type: ignore[attr-defined] 992 993 # do not use the fd functions, as they may not be available in the target OS 994 if hasattr(shutil, "_use_fd_functions"): 995 shutil._use_fd_functions = False # type: ignore[module-attr] 996 # in Python 3.14, _rmtree_impl is set at load time based on _use_fd_functions 997 # the safe version cannot be used at the moment as it used asserts of type 998 # 'assert func is os.rmtree', which do not work with the fake versions 999 if hasattr(shutil, "_rmtree_impl"): 1000 shutil._rmtree_impl = shutil._rmtree_unsafe # type: ignore[attr-defined] 1001 1002 with warnings.catch_warnings(): 1003 # ignore warnings, see #542 and #614 1004 warnings.filterwarnings("ignore") 1005 self._find_modules() 1006 1007 self._refresh() 1008 1009 if doctester is not None: 1010 doctester.globs = self.replace_globs(doctester.globs) 1011 1012 self.start_patching() 1013 linecache.open = self.original_open # type: ignore[attr-defined] 1014 tokenize._builtin_open = self.original_open # type: ignore 1015 1016 def start_patching(self) -> None: 1017 if not self._patching: 1018 self._patching = True 1019 self._paused = False 1020 1021 if sys.version_info >= (3, 12): 1022 # in linecache, 'os' is now imported locally, which involves the 1023 # dynamic patcher, therefore we patch the affected functions 1024 self.linecache_updatecache = linecache.updatecache 1025 linecache.updatecache = self.updatecache 1026 self.linecache_checkcache = linecache.checkcache 1027 linecache.checkcache = self.checkcache 1028 1029 self.tempfile_patcher.start_patching() 1030 1031 self.patch_modules() 1032 self.patch_functions() 1033 self.patch_defaults() 1034 self._set_glob_os_functions() 1035 1036 self._dyn_patcher = DynamicPatcher(self) 1037 sys.meta_path.insert(0, self._dyn_patcher) 1038 for module in self.modules_to_reload: 1039 if sys.modules.get(module.__name__) is module: 1040 reload(module) 1041 if not self.use_dynamic_patch: 1042 self._dyn_patcher.cleanup() 1043 sys.meta_path.pop(0) 1044 1045 def _set_glob_os_functions(self): 1046 # make sure the os functions cached in glob are patched 1047 if sys.version_info >= (3, 13): 1048 globber = glob._StringGlobber # type: ignore[module-attr] 1049 globber.lstat = staticmethod(os.lstat) 1050 globber.scandir = staticmethod(os.scandir) 1051 1052 def patch_functions(self) -> None: 1053 assert self._stubs is not None 1054 for (name, ft_name, ft_mod), modules in self.FS_FUNCTIONS.items(): 1055 method, mod_name = self._fake_module_functions[ft_name][ft_mod] 1056 fake_module = self.fake_modules[mod_name] 1057 attr = method.__get__( 1058 fake_module, fake_module.__class__ 1059 ) # pytype: disable=attribute-error 1060 for module in modules: 1061 self._stubs.smart_set(module, name, attr) 1062 1063 def patch_modules(self) -> None: 1064 skip_prefix_list = [] 1065 for rt_skip_module, prefixes in self.RUNTIME_SKIPMODULES.items(): 1066 if rt_skip_module in sys.modules: 1067 skip_prefix_list.extend(prefixes) 1068 skip_prefixes = tuple(skip_prefix_list) 1069 1070 assert self._stubs is not None 1071 for name, modules in self.FS_MODULES.items(): 1072 for module, attr in modules: 1073 try: 1074 if not skip_prefixes or not module.__name__.startswith( 1075 skip_prefixes 1076 ): 1077 self._stubs.smart_set(module, name, self.fake_modules[attr]) 1078 elif attr in self.unfaked_modules: 1079 self._stubs.smart_set(module, name, self.unfaked_modules[attr]) 1080 except Exception: 1081 # handle the rare case that a module has no __name__ 1082 pass 1083 1084 for name, modules in self.SKIPPED_FS_MODULES.items(): 1085 for module, attr in modules: 1086 if attr in self.unfaked_modules: 1087 self._stubs.smart_set(module, name, self.unfaked_modules[attr]) 1088 1089 def patch_defaults(self) -> None: 1090 for fct, idx, ft in self.FS_DEFARGS: 1091 method, mod_name = self._fake_module_functions[ft.__name__][ft.__module__] 1092 fake_module = self.fake_modules[mod_name] 1093 attr = method.__get__( 1094 fake_module, fake_module.__class__ 1095 ) # pytype: disable=attribute-error 1096 new_defaults = [] 1097 assert fct.__defaults__ is not None 1098 for i, d in enumerate(fct.__defaults__): 1099 if i == idx: 1100 new_defaults.append(attr) 1101 else: 1102 new_defaults.append(d) 1103 fct.__defaults__ = tuple(new_defaults) 1104 1105 def replace_globs(self, globs_: Dict[str, Any]) -> Dict[str, Any]: 1106 globs = globs_.copy() 1107 if self._isStale: 1108 self._refresh() 1109 for name in self._fake_module_classes: 1110 if name in globs: 1111 globs[name] = self._fake_module_classes[name](self.fs) 1112 return globs 1113 1114 def tearDown(self, doctester: Any = None): 1115 """Clear the fake filesystem bindings created by `setUp()`.""" 1116 if self.is_doc_test: 1117 self.__class__.DOC_REF_COUNT -= 1 1118 if self.__class__.DOC_REF_COUNT > 0: 1119 return 1120 else: 1121 self.__class__.REF_COUNT -= 1 1122 if self.__class__.REF_COUNT > 0: 1123 return 1124 self.stop_patching() 1125 if self.has_fcopy_file: 1126 shutil._HAS_FCOPYFILE = True # type: ignore[attr-defined] 1127 1128 reset_ids() 1129 if self.is_doc_test: 1130 self.__class__.DOC_PATCHER = None 1131 else: 1132 self.__class__.PATCHER = None 1133 1134 def stop_patching(self, temporary=False) -> None: 1135 if self._patching: 1136 self._isStale = True 1137 self._patching = False 1138 self._paused = temporary 1139 if self._stubs: 1140 self._stubs.smart_unset_all() 1141 self.unset_defaults() 1142 if self.use_dynamic_patch and self._dyn_patcher: 1143 self._dyn_patcher.cleanup() 1144 sys.meta_path.pop(0) 1145 self.tempfile_patcher.stop_patching() 1146 if self.linecache_updatecache is not None: 1147 linecache.updatecache = self.linecache_updatecache 1148 linecache.checkcache = self.linecache_checkcache 1149 self._set_glob_os_functions() 1150 1151 @property 1152 def is_patching(self): 1153 return self._patching 1154 1155 def unset_defaults(self) -> None: 1156 for fct, idx, ft in self.FS_DEFARGS: 1157 new_defaults = [] 1158 for i, d in enumerate(cast(Tuple, fct.__defaults__)): 1159 if i == idx: 1160 new_defaults.append(ft) 1161 else: 1162 new_defaults.append(d) 1163 fct.__defaults__ = tuple(new_defaults) 1164 1165 def pause(self) -> None: 1166 """Pause the patching of the file system modules until `resume` is 1167 called. After that call, all file system calls are executed in the 1168 real file system. 1169 Calling pause() twice is silently ignored. 1170 1171 """ 1172 self.stop_patching(temporary=True) 1173 1174 def resume(self) -> None: 1175 """Resume the patching of the file system modules if `pause` has 1176 been called before. After that call, all file system calls are 1177 executed in the fake file system. 1178 Does nothing if patching is not paused. 1179 """ 1180 if self._paused: 1181 self.start_patching() 1182 1183 1184class Pause: 1185 """Simple context manager that allows to pause/resume patching the 1186 filesystem. Patching is paused in the context manager, and resumed after 1187 going out of its scope. 1188 """ 1189 1190 def __init__(self, caller: Union[Patcher, TestCaseMixin, FakeFilesystem]): 1191 """Initializes the context manager with the fake filesystem. 1192 1193 Args: 1194 caller: either the FakeFilesystem instance, the Patcher instance 1195 or the pyfakefs test case. 1196 """ 1197 if isinstance(caller, (Patcher, TestCaseMixin)): 1198 assert caller.fs is not None 1199 self._fs: FakeFilesystem = caller.fs 1200 elif isinstance(caller, FakeFilesystem): 1201 self._fs = caller 1202 else: 1203 raise ValueError( 1204 "Invalid argument - should be of type " 1205 '"fake_filesystem_unittest.Patcher", ' 1206 '"fake_filesystem_unittest.TestCase" ' 1207 'or "fake_filesystem.FakeFilesystem"' 1208 ) 1209 1210 def __enter__(self) -> FakeFilesystem: 1211 self._fs.pause() 1212 return self._fs 1213 1214 def __exit__(self, *args: Any) -> None: 1215 self._fs.resume() 1216 1217 1218class DynamicPatcher(MetaPathFinder, Loader): 1219 """A file loader that replaces file system related modules by their 1220 fake implementation if they are loaded after calling `setUpPyfakefs()`. 1221 Implements the protocol needed for import hooks. 1222 """ 1223 1224 def __init__(self, patcher: Patcher) -> None: 1225 self._patcher = patcher 1226 self.sysmodules = {} 1227 self.modules = self._patcher.fake_modules 1228 self._loaded_module_names: Set[str] = set() 1229 self.cleanup_handlers = patcher.cleanup_handlers 1230 1231 # remove all modules that have to be patched from `sys.modules`, 1232 # otherwise the find_... methods will not be called 1233 for name in self.modules: 1234 if self.needs_patch(name) and name in sys.modules: 1235 self.sysmodules[name] = sys.modules[name] 1236 del sys.modules[name] 1237 1238 for name, module in self.modules.items(): 1239 sys.modules[name] = module 1240 1241 def cleanup(self) -> None: 1242 for module_name in self.sysmodules: 1243 sys.modules[module_name] = self.sysmodules[module_name] 1244 for module in self._patcher.modules_to_reload: 1245 if module.__name__ in sys.modules: 1246 reload(module) 1247 reloaded_module_names = [ 1248 module.__name__ for module in self._patcher.modules_to_reload 1249 ] 1250 # Delete all modules loaded during the test, ensuring that 1251 # they are reloaded after the test. 1252 for name in self._loaded_module_names: 1253 if name in sys.modules and name not in reloaded_module_names: 1254 if name in self.cleanup_handlers and self.cleanup_handlers[name](name): 1255 continue 1256 del sys.modules[name] 1257 1258 def needs_patch(self, name: str) -> bool: 1259 """Checks if the module with the given name shall be replaced.""" 1260 if name not in self.modules: 1261 self._loaded_module_names.add(name) 1262 return False 1263 if name in sys.modules and type(sys.modules[name]) is self.modules[name]: 1264 return False 1265 return True 1266 1267 def fake_module_path(self, name: str) -> str: 1268 """Checks if the module with the given name is a module existing in the fake 1269 filesystem and returns its path in this case. 1270 """ 1271 fs = self._patcher.fs 1272 # we assume that the module name is the absolute module path 1273 if fs is not None: 1274 base_path = name.replace(".", fs.path_separator) 1275 for path in sys.path: 1276 module_path = fs.joinpaths(path, base_path) 1277 py_module_path = module_path + ".py" 1278 if fs.exists(py_module_path): 1279 return fs.absnormpath(py_module_path) 1280 init_path = fs.joinpaths(module_path, "__init__.py") 1281 if fs.exists(init_path): 1282 return fs.absnormpath(init_path) 1283 return "" 1284 1285 def find_spec( 1286 self, 1287 fullname: str, 1288 path: Optional[Sequence[Union[bytes, str]]], 1289 target: Optional[ModuleType] = None, 1290 ) -> Optional[ModuleSpec]: 1291 """Module finder.""" 1292 if self.needs_patch(fullname): 1293 return ModuleSpec(fullname, self) 1294 if self._patcher.patch_open_code != PatchMode.OFF: 1295 # handle modules created in the fake filesystem 1296 module_path = self.fake_module_path(fullname) 1297 if module_path: 1298 spec = spec_from_file_location(fullname, module_path) 1299 if spec: 1300 module = module_from_spec(spec) 1301 sys.modules[fullname] = module 1302 return ModuleSpec(fullname, self) 1303 return None 1304 1305 def load_module(self, fullname: str) -> ModuleType: 1306 """Replaces the module by its fake implementation.""" 1307 sys.modules[fullname] = self.modules[fullname] 1308 return self.modules[fullname] 1309