1# Copyright 2014 Altera Corporation. All Rights Reserved. 2# Copyright 2015-2017 John McGehee 3# 4# Licensed under the Apache License, Version 2.0 (the "License"); 5# you may not use this file except in compliance with the License. 6# You may obtain a copy of the License at 7# 8# http://www.apache.org/licenses/LICENSE-2.0 9# 10# Unless required by applicable law or agreed to in writing, software 11# distributed under the License is distributed on an "AS IS" BASIS, 12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13# See the License for the specific language governing permissions and 14# limitations under the License. 15 16"""This module provides a base class derived from `unittest.TestClass` 17for unit tests using the :py:class:`pyfakefs` module. 18 19`fake_filesystem_unittest.TestCase` searches `sys.modules` for modules 20that import the `os`, `io`, `path` `shutil`, and `pathlib` modules. 21 22The `setUpPyfakefs()` method binds these modules to the corresponding fake 23modules from `pyfakefs`. Further, the `open()` built-in is bound to a fake 24`open()`. 25 26It is expected that `setUpPyfakefs()` be invoked at the beginning of the 27derived class' `setUp()` method. There is no need to add anything to the 28derived class' `tearDown()` method. 29 30During the test, everything uses the fake file system and modules. This means 31that even in your test fixture, familiar functions like `open()` and 32`os.makedirs()` manipulate the fake file system. 33 34Existing unit tests that use the real file system can be retrofitted to use 35pyfakefs by simply changing their base class from `:py:class`unittest.TestCase` 36to `:py:class`pyfakefs.fake_filesystem_unittest.TestCase`. 37""" 38import doctest 39import functools 40import inspect 41import linecache 42import shutil 43import sys 44import tempfile 45import tokenize 46from importlib.abc import Loader, MetaPathFinder 47from types import ModuleType, TracebackType, FunctionType 48from typing import ( 49 Any, Callable, Dict, List, Set, Tuple, Optional, Union, 50 AnyStr, Type, Iterator, cast, ItemsView, Sequence 51) 52import unittest 53import warnings 54from unittest import TestSuite 55 56from pyfakefs.deprecator import Deprecator 57from pyfakefs.fake_filesystem import ( 58 set_uid, set_gid, reset_ids, PatchMode, FakeFile, FakeFilesystem 59) 60from pyfakefs.helpers import IS_PYPY 61from pyfakefs.mox3_stubout import StubOutForTesting 62 63try: 64 from importlib.machinery import ModuleSpec 65except ImportError: 66 ModuleSpec = object # type: ignore[assignment, misc] 67 68from importlib import reload 69 70from pyfakefs import fake_filesystem 71from pyfakefs import fake_filesystem_shutil 72from pyfakefs import fake_pathlib 73from pyfakefs import mox3_stubout 74from pyfakefs.extra_packages import pathlib2, use_scandir 75 76if use_scandir: 77 from pyfakefs import fake_scandir 78 79OS_MODULE = 'nt' if sys.platform == 'win32' else 'posix' 80PATH_MODULE = 'ntpath' if sys.platform == 'win32' else 'posixpath' 81 82 83def patchfs(_func: Callable = None, *, 84 additional_skip_names: Optional[ 85 List[Union[str, ModuleType]]] = None, 86 modules_to_reload: Optional[List[ModuleType]] = None, 87 modules_to_patch: Optional[Dict[str, ModuleType]] = None, 88 allow_root_user: bool = True, 89 use_known_patches: bool = True, 90 patch_open_code: PatchMode = PatchMode.OFF, 91 patch_default_args: bool = False, 92 use_cache: bool = True) -> Callable: 93 """Convenience decorator to use patcher with additional parameters in a 94 test function. 95 96 Usage:: 97 98 @patchfs 99 def test_my_function(fake_fs): 100 fake_fs.create_file('foo') 101 102 @patchfs(allow_root_user=False) 103 def test_with_patcher_args(fs): 104 os.makedirs('foo/bar') 105 """ 106 107 def wrap_patchfs(f: Callable) -> Callable: 108 @functools.wraps(f) 109 def wrapped(*args, **kwargs): 110 with Patcher( 111 additional_skip_names=additional_skip_names, 112 modules_to_reload=modules_to_reload, 113 modules_to_patch=modules_to_patch, 114 allow_root_user=allow_root_user, 115 use_known_patches=use_known_patches, 116 patch_open_code=patch_open_code, 117 patch_default_args=patch_default_args, 118 use_cache=use_cache) as p: 119 args = list(args) 120 args.append(p.fs) 121 return f(*args, **kwargs) 122 123 return wrapped 124 125 if _func: 126 if not callable(_func): 127 raise TypeError( 128 "Decorator argument is not a function.\n" 129 "Did you mean `@patchfs(additional_skip_names=...)`?" 130 ) 131 if hasattr(_func, 'patchings'): 132 _func.nr_patches = len(_func.patchings) # type: ignore 133 return wrap_patchfs(_func) 134 135 return wrap_patchfs 136 137 138def load_doctests( 139 loader: Any, tests: TestSuite, ignore: Any, module: ModuleType, 140 additional_skip_names: Optional[ 141 List[Union[str, ModuleType]]] = None, 142 modules_to_reload: Optional[List[ModuleType]] = None, 143 modules_to_patch: Optional[Dict[str, ModuleType]] = None, 144 allow_root_user: bool = True, 145 use_known_patches: bool = True, 146 patch_open_code: PatchMode = PatchMode.OFF, 147 patch_default_args: bool = False 148) -> TestSuite: # pylint:disable=unused-argument 149 """Load the doctest tests for the specified module into unittest. 150 Args: 151 loader, tests, ignore : arguments passed in from `load_tests()` 152 module: module under test 153 remaining args: see :py:class:`TestCase` for an explanation 154 155 File `example_test.py` in the pyfakefs release provides a usage example. 156 """ 157 _patcher = Patcher(additional_skip_names=additional_skip_names, 158 modules_to_reload=modules_to_reload, 159 modules_to_patch=modules_to_patch, 160 allow_root_user=allow_root_user, 161 use_known_patches=use_known_patches, 162 patch_open_code=patch_open_code, 163 patch_default_args=patch_default_args) 164 globs = _patcher.replace_globs(vars(module)) 165 tests.addTests(doctest.DocTestSuite(module, 166 globs=globs, 167 setUp=_patcher.setUp, 168 tearDown=_patcher.tearDown)) 169 return tests 170 171 172class TestCaseMixin: 173 """Test case mixin that automatically replaces file-system related 174 modules by fake implementations. 175 176 Attributes: 177 additional_skip_names: names of modules inside of which no module 178 replacement shall be performed, in addition to the names in 179 :py:attr:`fake_filesystem_unittest.Patcher.SKIPNAMES`. 180 Instead of the module names, the modules themselves may be used. 181 modules_to_reload: A list of modules that need to be reloaded 182 to be patched dynamically; may be needed if the module 183 imports file system modules under an alias 184 185 .. caution:: Reloading modules may have unwanted side effects. 186 modules_to_patch: A dictionary of fake modules mapped to the 187 fully qualified patched module names. Can be used to add patching 188 of modules not provided by `pyfakefs`. 189 190 If you specify some of these attributes here and you have DocTests, 191 consider also specifying the same arguments to :py:func:`load_doctests`. 192 193 Example usage in derived test classes:: 194 195 from unittest import TestCase 196 from fake_filesystem_unittest import TestCaseMixin 197 198 class MyTestCase(TestCase, TestCaseMixin): 199 def __init__(self, methodName='runTest'): 200 super(MyTestCase, self).__init__( 201 methodName=methodName, 202 additional_skip_names=['posixpath']) 203 204 import sut 205 206 class AnotherTestCase(TestCase, TestCaseMixin): 207 def __init__(self, methodName='runTest'): 208 super(MyTestCase, self).__init__( 209 methodName=methodName, modules_to_reload=[sut]) 210 """ 211 212 additional_skip_names: Optional[List[Union[str, ModuleType]]] = None 213 modules_to_reload: Optional[List[ModuleType]] = None 214 modules_to_patch: Optional[Dict[str, ModuleType]] = None 215 216 @property 217 def fs(self) -> FakeFilesystem: 218 return cast(FakeFilesystem, self._stubber.fs) 219 220 def setUpPyfakefs(self, 221 additional_skip_names: Optional[ 222 List[Union[str, ModuleType]]] = None, 223 modules_to_reload: Optional[List[ModuleType]] = None, 224 modules_to_patch: Optional[Dict[str, ModuleType]] = None, 225 allow_root_user: bool = True, 226 use_known_patches: bool = True, 227 patch_open_code: PatchMode = PatchMode.OFF, 228 patch_default_args: bool = False, 229 use_cache: bool = True) -> None: 230 """Bind the file-related modules to the :py:class:`pyfakefs` fake file 231 system instead of the real file system. Also bind the fake `open()` 232 function. 233 234 Invoke this at the beginning of the `setUp()` method in your unit test 235 class. 236 For the arguments, see the `TestCaseMixin` attribute description. 237 If any of the arguments is not None, it overwrites the settings for 238 the current test case. Settings the arguments here may be a more 239 convenient way to adapt the setting than overwriting `__init__()`. 240 """ 241 if additional_skip_names is None: 242 additional_skip_names = self.additional_skip_names 243 if modules_to_reload is None: 244 modules_to_reload = self.modules_to_reload 245 if modules_to_patch is None: 246 modules_to_patch = self.modules_to_patch 247 self._stubber = Patcher( 248 additional_skip_names=additional_skip_names, 249 modules_to_reload=modules_to_reload, 250 modules_to_patch=modules_to_patch, 251 allow_root_user=allow_root_user, 252 use_known_patches=use_known_patches, 253 patch_open_code=patch_open_code, 254 patch_default_args=patch_default_args, 255 use_cache=use_cache 256 ) 257 258 self._stubber.setUp() 259 cast(TestCase, self).addCleanup(self._stubber.tearDown) 260 261 def pause(self) -> None: 262 """Pause the patching of the file system modules until `resume` is 263 called. After that call, all file system calls are executed in the 264 real file system. 265 Calling pause() twice is silently ignored. 266 267 """ 268 self._stubber.pause() 269 270 def resume(self) -> None: 271 """Resume the patching of the file system modules if `pause` has 272 been called before. After that call, all file system calls are 273 executed in the fake file system. 274 Does nothing if patching is not paused. 275 """ 276 self._stubber.resume() 277 278 279class TestCase(unittest.TestCase, TestCaseMixin): 280 """Test case class that automatically replaces file-system related 281 modules by fake implementations. Inherits :py:class:`TestCaseMixin`. 282 283 The arguments are explained in :py:class:`TestCaseMixin`. 284 """ 285 286 def __init__(self, methodName: str = 'runTest', 287 additional_skip_names: Optional[ 288 List[Union[str, ModuleType]]] = None, 289 modules_to_reload: Optional[List[ModuleType]] = None, 290 modules_to_patch: Optional[Dict[str, ModuleType]] = None): 291 """Creates the test class instance and the patcher used to stub out 292 file system related modules. 293 294 Args: 295 methodName: The name of the test method (same as in 296 unittest.TestCase) 297 """ 298 super().__init__(methodName) 299 300 self.additional_skip_names = additional_skip_names 301 self.modules_to_reload = modules_to_reload 302 self.modules_to_patch = modules_to_patch 303 304 @Deprecator('add_real_file') 305 def copyRealFile(self, real_file_path: AnyStr, 306 fake_file_path: Optional[AnyStr] = None, 307 create_missing_dirs: bool = True) -> FakeFile: 308 """Add the file `real_file_path` in the real file system to the same 309 path in the fake file system. 310 311 **This method is deprecated** in favor of 312 :py:meth:`FakeFilesystem..add_real_file`. 313 `copyRealFile()` is retained with limited functionality for backward 314 compatibility only. 315 316 Args: 317 real_file_path: Path to the file in both the real and fake 318 file systems 319 fake_file_path: Deprecated. Use the default, which is 320 `real_file_path`. 321 If a value other than `real_file_path` is specified, a `ValueError` 322 exception will be raised. 323 create_missing_dirs: Deprecated. Use the default, which creates 324 missing directories in the fake file system. If `False` is 325 specified, a `ValueError` exception is raised. 326 327 Returns: 328 The newly created FakeFile object. 329 330 Raises: 331 OSError: If the file already exists in the fake file system. 332 ValueError: If deprecated argument values are specified. 333 334 See: 335 :py:meth:`FakeFileSystem.add_real_file` 336 """ 337 if fake_file_path is not None and real_file_path != fake_file_path: 338 raise ValueError("CopyRealFile() is deprecated and no longer " 339 "supports different real and fake file paths") 340 if not create_missing_dirs: 341 raise ValueError("CopyRealFile() is deprecated and no longer " 342 "supports NOT creating missing directories") 343 assert self._stubber.fs is not None 344 return self._stubber.fs.add_real_file(real_file_path, read_only=False) 345 346 def tearDownPyfakefs(self) -> None: 347 """This method is deprecated and exists only for backward 348 compatibility. It does nothing. 349 """ 350 pass 351 352 353class Patcher: 354 """ 355 Instantiate a stub creator to bind and un-bind the file-related modules to 356 the :py:mod:`pyfakefs` fake modules. 357 358 The arguments are explained in :py:class:`TestCaseMixin`. 359 360 :py:class:`Patcher` is used in :py:class:`TestCaseMixin`. 361 :py:class:`Patcher` also works as a context manager for other tests:: 362 363 with Patcher(): 364 doStuff() 365 """ 366 '''Stub nothing that is imported within these modules. 367 `sys` is included to prevent `sys.path` from being stubbed with the fake 368 `os.path`. 369 The `pytest` and `py` modules are used by pytest and have to access the 370 real file system. 371 The `linecache` module is used to read the test file in case of test 372 failure to get traceback information before test tear down. 373 In order to make sure that reading the test file is not faked, 374 we skip faking the module. 375 We also have to set back the cached open function in tokenize. 376 ''' 377 SKIPMODULES = { 378 None, fake_filesystem, fake_filesystem_shutil, 379 sys, linecache, tokenize 380 } 381 # caches all modules that do not have file system modules or function 382 # to speed up _find_modules 383 CACHED_MODULES: Set[ModuleType] = set() 384 FS_MODULES: Dict[str, Set[Tuple[ModuleType, str]]] = {} 385 FS_FUNCTIONS: Dict[Tuple[str, str, str], Set[ModuleType]] = {} 386 FS_DEFARGS: List[Tuple[FunctionType, int, Callable[..., Any]]] = [] 387 SKIPPED_FS_MODULES: Dict[str, Set[Tuple[ModuleType, str]]] = {} 388 389 assert None in SKIPMODULES, ("sys.modules contains 'None' values;" 390 " must skip them.") 391 392 IS_WINDOWS = sys.platform in ('win32', 'cygwin') 393 394 SKIPNAMES = {'os', 'path', 'io', 'genericpath', 'fcntl', 395 OS_MODULE, PATH_MODULE} 396 397 # hold values from last call - if changed, the cache has to be invalidated 398 PATCHED_MODULE_NAMES: Set[str] = set() 399 ADDITIONAL_SKIP_NAMES: Set[str] = set() 400 PATCH_DEFAULT_ARGS = False 401 402 def __init__(self, additional_skip_names: Optional[ 403 List[Union[str, ModuleType]]] = None, 404 modules_to_reload: Optional[List[ModuleType]] = None, 405 modules_to_patch: Optional[Dict[str, ModuleType]] = None, 406 allow_root_user: bool = True, 407 use_known_patches: bool = True, 408 patch_open_code: PatchMode = PatchMode.OFF, 409 patch_default_args: bool = False, 410 use_cache: bool = True) -> None: 411 """ 412 Args: 413 additional_skip_names: names of modules inside of which no module 414 replacement shall be performed, in addition to the names in 415 :py:attr:`fake_filesystem_unittest.Patcher.SKIPNAMES`. 416 Instead of the module names, the modules themselves 417 may be used. 418 modules_to_reload: A list of modules that need to be reloaded 419 to be patched dynamically; may be needed if the module 420 imports file system modules under an alias 421 422 .. caution:: Reloading modules may have unwanted side effects. 423 modules_to_patch: A dictionary of fake modules mapped to the 424 fully qualified patched module names. Can be used to add 425 patching of modules not provided by `pyfakefs`. 426 allow_root_user: If True (default), if the test is run as root 427 user, the user in the fake file system is also considered a 428 root user, otherwise it is always considered a regular user. 429 use_known_patches: If True (the default), some patches for commonly 430 used packages are applied which make them usable with pyfakefs. 431 patch_open_code: If True, `io.open_code` is patched. The default 432 is not to patch it, as it mostly is used to load compiled 433 modules that are not in the fake file system. 434 patch_default_args: If True, default arguments are checked for 435 file system functions, which are patched. This check is 436 expansive, so it is off by default. 437 use_cache: If True (default), patched and non-patched modules are 438 cached between tests for performance reasons. As this is a new 439 feature, this argument allows to turn it off in case it 440 causes any problems. 441 """ 442 443 if not allow_root_user: 444 # set non-root IDs even if the real user is root 445 set_uid(1) 446 set_gid(1) 447 448 self._skip_names = self.SKIPNAMES.copy() 449 # save the original open function for use in pytest plugin 450 self.original_open = open 451 self.patch_open_code = patch_open_code 452 453 if additional_skip_names is not None: 454 skip_names = [ 455 cast(ModuleType, m).__name__ if inspect.ismodule(m) 456 else cast(str, m) for m in additional_skip_names 457 ] 458 self._skip_names.update(skip_names) 459 460 self._fake_module_classes: Dict[str, Any] = {} 461 self._unfaked_module_classes: Dict[str, Any] = {} 462 self._class_modules: Dict[str, List[str]] = {} 463 self._init_fake_module_classes() 464 465 # reload tempfile under posix to patch default argument 466 self.modules_to_reload: List[ModuleType] = ( 467 [] if sys.platform == 'win32' else [tempfile] 468 ) 469 if modules_to_reload is not None: 470 self.modules_to_reload.extend(modules_to_reload) 471 self.patch_default_args = patch_default_args 472 self.use_cache = use_cache 473 474 if use_known_patches: 475 from pyfakefs.patched_packages import ( 476 get_modules_to_patch, get_classes_to_patch, 477 get_fake_module_classes 478 ) 479 480 modules_to_patch = modules_to_patch or {} 481 modules_to_patch.update(get_modules_to_patch()) 482 self._class_modules.update(get_classes_to_patch()) 483 self._fake_module_classes.update(get_fake_module_classes()) 484 485 if modules_to_patch is not None: 486 for name, fake_module in modules_to_patch.items(): 487 self._fake_module_classes[name] = fake_module 488 patched_module_names = set(modules_to_patch) 489 else: 490 patched_module_names = set() 491 clear_cache = not use_cache 492 if use_cache: 493 if patched_module_names != self.PATCHED_MODULE_NAMES: 494 self.__class__.PATCHED_MODULE_NAMES = patched_module_names 495 clear_cache = True 496 if self._skip_names != self.ADDITIONAL_SKIP_NAMES: 497 self.__class__.ADDITIONAL_SKIP_NAMES = self._skip_names 498 clear_cache = True 499 if patch_default_args != self.PATCH_DEFAULT_ARGS: 500 self.__class__.PATCH_DEFAULT_ARGS = patch_default_args 501 clear_cache = True 502 503 if clear_cache: 504 self.clear_cache() 505 self._fake_module_functions: Dict[str, Dict] = {} 506 self._init_fake_module_functions() 507 508 # Attributes set by _refresh() 509 self._stubs: Optional[StubOutForTesting] = None 510 self.fs: Optional[FakeFilesystem] = None 511 self.fake_modules: Dict[str, Any] = {} 512 self.unfaked_modules: Dict[str, Any] = {} 513 514 # _isStale is set by tearDown(), reset by _refresh() 515 self._isStale = True 516 self._dyn_patcher: Optional[DynamicPatcher] = None 517 self._patching = False 518 519 def clear_cache(self) -> None: 520 """Clear the module cache.""" 521 self.__class__.CACHED_MODULES = set() 522 self.__class__.FS_MODULES = {} 523 self.__class__.FS_FUNCTIONS = {} 524 self.__class__.FS_DEFARGS = [] 525 self.__class__.SKIPPED_FS_MODULES = {} 526 527 def _init_fake_module_classes(self) -> None: 528 # IMPORTANT TESTING NOTE: Whenever you add a new module below, test 529 # it by adding an attribute in fixtures/module_with_attributes.py 530 # and a test in fake_filesystem_unittest_test.py, class 531 # TestAttributesWithFakeModuleNames. 532 self._fake_module_classes = { 533 'os': fake_filesystem.FakeOsModule, 534 'shutil': fake_filesystem_shutil.FakeShutilModule, 535 'io': fake_filesystem.FakeIoModule, 536 'pathlib': fake_pathlib.FakePathlibModule 537 } 538 if IS_PYPY: 539 # in PyPy io.open, the module is referenced as _io 540 self._fake_module_classes['_io'] = fake_filesystem.FakeIoModule 541 if sys.platform != 'win32': 542 self._fake_module_classes[ 543 'fcntl'] = fake_filesystem.FakeFcntlModule 544 545 # class modules maps class names against a list of modules they can 546 # be contained in - this allows for alternative modules like 547 # `pathlib` and `pathlib2` 548 self._class_modules['Path'] = ['pathlib'] 549 self._unfaked_module_classes[ 550 'pathlib'] = fake_pathlib.RealPathlibModule 551 if pathlib2: 552 self._fake_module_classes[ 553 'pathlib2'] = fake_pathlib.FakePathlibModule 554 self._class_modules['Path'].append('pathlib2') 555 self._unfaked_module_classes[ 556 'pathlib2'] = fake_pathlib.RealPathlibModule 557 self._fake_module_classes[ 558 'Path'] = fake_pathlib.FakePathlibPathModule 559 self._unfaked_module_classes[ 560 'Path'] = fake_pathlib.RealPathlibPathModule 561 if use_scandir: 562 self._fake_module_classes[ 563 'scandir'] = fake_scandir.FakeScanDirModule 564 565 def _init_fake_module_functions(self) -> None: 566 # handle patching function imported separately like 567 # `from os import stat` 568 # each patched function name has to be looked up separately 569 for mod_name, fake_module in self._fake_module_classes.items(): 570 if (hasattr(fake_module, 'dir') and 571 inspect.isfunction(fake_module.dir)): 572 for fct_name in fake_module.dir(): 573 module_attr = (getattr(fake_module, fct_name), mod_name) 574 self._fake_module_functions.setdefault( 575 fct_name, {})[mod_name] = module_attr 576 if mod_name == 'os': 577 self._fake_module_functions.setdefault( 578 fct_name, {})[OS_MODULE] = module_attr 579 580 # special handling for functions in os.path 581 fake_module = fake_filesystem.FakePathModule 582 for fct_name in fake_module.dir(): 583 module_attr = (getattr(fake_module, fct_name), PATH_MODULE) 584 self._fake_module_functions.setdefault( 585 fct_name, {})['genericpath'] = module_attr 586 self._fake_module_functions.setdefault( 587 fct_name, {})[PATH_MODULE] = module_attr 588 589 def __enter__(self) -> 'Patcher': 590 """Context manager for usage outside of 591 fake_filesystem_unittest.TestCase. 592 Ensure that all patched modules are removed in case of an 593 unhandled exception. 594 """ 595 self.setUp() 596 return self 597 598 def __exit__(self, 599 exc_type: Optional[Type[BaseException]], 600 exc_val: Optional[BaseException], 601 exc_tb: Optional[TracebackType]) -> None: 602 self.tearDown() 603 604 def _is_fs_module(self, mod: ModuleType, 605 name: str, 606 module_names: List[str]) -> bool: 607 try: 608 # check for __name__ first and ignore the AttributeException 609 # if it does not exist - avoids calling expansive ismodule 610 if mod.__name__ in module_names and inspect.ismodule(mod): 611 return True 612 except Exception: 613 pass 614 try: 615 if (name in self._class_modules and 616 mod.__module__ in self._class_modules[name]): 617 return inspect.isclass(mod) 618 except Exception: 619 # handle AttributeError and any other exception possibly triggered 620 # by side effects of inspect methods 621 pass 622 return False 623 624 def _is_fs_function(self, fct: FunctionType) -> bool: 625 try: 626 # check for __name__ first and ignore the AttributeException 627 # if it does not exist - avoids calling expansive inspect 628 # methods in most cases 629 return (fct.__name__ in self._fake_module_functions and 630 fct.__module__ in self._fake_module_functions[ 631 fct.__name__] and 632 (inspect.isfunction(fct) or inspect.isbuiltin(fct))) 633 except Exception: 634 # handle AttributeError and any other exception possibly triggered 635 # by side effects of inspect methods 636 return False 637 638 def _def_values( 639 self, 640 item: FunctionType) -> Iterator[Tuple[FunctionType, int, Any]]: 641 """Find default arguments that are file-system functions to be 642 patched in top-level functions and members of top-level classes.""" 643 # check for module-level functions 644 try: 645 if item.__defaults__ and inspect.isfunction(item): 646 for i, d in enumerate(item.__defaults__): 647 if self._is_fs_function(d): 648 yield item, i, d 649 except Exception: 650 pass 651 try: 652 if inspect.isclass(item): 653 # check for methods in class 654 # (nested classes are ignored for now) 655 # inspect.getmembers is very expansive! 656 for m in inspect.getmembers(item, 657 predicate=inspect.isfunction): 658 f = cast(FunctionType, m[1]) 659 if f.__defaults__: 660 for i, d in enumerate(f.__defaults__): 661 if self._is_fs_function(d): 662 yield f, i, d 663 except Exception: 664 # Ignore any exception, examples: 665 # ImportError: No module named '_gdbm' 666 # _DontDoThat() (see #523) 667 pass 668 669 def _find_def_values( 670 self, module_items: ItemsView[str, FunctionType]) -> None: 671 for _, fct in module_items: 672 for f, i, d in self._def_values(fct): 673 self.__class__.FS_DEFARGS.append((f, i, d)) 674 675 def _find_modules(self) -> None: 676 """Find and cache all modules that import file system modules. 677 Later, `setUp()` will stub these with the fake file system 678 modules. 679 """ 680 module_names = list(self._fake_module_classes.keys()) + [PATH_MODULE] 681 for name, module in list(sys.modules.items()): 682 try: 683 if (self.use_cache and module in self.CACHED_MODULES or 684 not inspect.ismodule(module)): 685 continue 686 except Exception: 687 # workaround for some py (part of pytest) versions 688 # where py.error has no __name__ attribute 689 # see https://github.com/pytest-dev/py/issues/73 690 # and any other exception triggered by inspect.ismodule 691 if self.use_cache: 692 self.__class__.CACHED_MODULES.add(module) 693 continue 694 skipped = (module in self.SKIPMODULES or 695 any([sn.startswith(module.__name__) 696 for sn in self._skip_names])) 697 module_items = module.__dict__.copy().items() 698 699 modules = {name: mod for name, mod in module_items 700 if self._is_fs_module(mod, name, module_names)} 701 702 if skipped: 703 for name, mod in modules.items(): 704 self.__class__.SKIPPED_FS_MODULES.setdefault( 705 name, set()).add((module, mod.__name__)) 706 else: 707 for name, mod in modules.items(): 708 self.__class__.FS_MODULES.setdefault(name, set()).add( 709 (module, mod.__name__)) 710 functions = {name: fct for name, fct in 711 module_items 712 if self._is_fs_function(fct)} 713 714 for name, fct in functions.items(): 715 self.__class__.FS_FUNCTIONS.setdefault( 716 (name, fct.__name__, fct.__module__), 717 set()).add(module) 718 719 # find default arguments that are file system functions 720 if self.patch_default_args: 721 self._find_def_values(module_items) 722 723 if self.use_cache: 724 self.__class__.CACHED_MODULES.add(module) 725 726 def _refresh(self) -> None: 727 """Renew the fake file system and set the _isStale flag to `False`.""" 728 if self._stubs is not None: 729 self._stubs.smart_unset_all() 730 self._stubs = mox3_stubout.StubOutForTesting() 731 732 self.fs = fake_filesystem.FakeFilesystem(patcher=self) 733 self.fs.patch_open_code = self.patch_open_code 734 for name in self._fake_module_classes: 735 self.fake_modules[name] = self._fake_module_classes[name](self.fs) 736 if hasattr(self.fake_modules[name], 'skip_names'): 737 self.fake_modules[name].skip_names = self._skip_names 738 self.fake_modules[PATH_MODULE] = self.fake_modules['os'].path 739 for name in self._unfaked_module_classes: 740 self.unfaked_modules[name] = self._unfaked_module_classes[name]() 741 742 self._isStale = False 743 744 def setUp(self, doctester: Any = None) -> None: 745 """Bind the file-related modules to the :py:mod:`pyfakefs` fake 746 modules real ones. Also bind the fake `file()` and `open()` functions. 747 """ 748 self.has_fcopy_file = (sys.platform == 'darwin' and 749 hasattr(shutil, '_HAS_FCOPYFILE') and 750 shutil._HAS_FCOPYFILE) 751 if self.has_fcopy_file: 752 shutil._HAS_FCOPYFILE = False # type: ignore[attr-defined] 753 754 temp_dir = tempfile.gettempdir() 755 with warnings.catch_warnings(): 756 # ignore warnings, see #542 and #614 757 warnings.filterwarnings( 758 'ignore' 759 ) 760 self._find_modules() 761 762 self._refresh() 763 764 if doctester is not None: 765 doctester.globs = self.replace_globs(doctester.globs) 766 767 self.start_patching() 768 linecache.open = self.original_open # type: ignore[attr-defined] 769 tokenize._builtin_open = self.original_open # type: ignore 770 771 # the temp directory is assumed to exist at least in `tempfile1`, 772 # so we create it here for convenience 773 assert self.fs is not None 774 self.fs.create_dir(temp_dir) 775 776 def start_patching(self) -> None: 777 if not self._patching: 778 self._patching = True 779 780 self.patch_modules() 781 self.patch_functions() 782 self.patch_defaults() 783 784 self._dyn_patcher = DynamicPatcher(self) 785 sys.meta_path.insert(0, self._dyn_patcher) 786 for module in self.modules_to_reload: 787 if sys.modules.get(module.__name__) is module: 788 reload(module) 789 790 def patch_functions(self) -> None: 791 assert self._stubs is not None 792 for (name, ft_name, ft_mod), modules in self.FS_FUNCTIONS.items(): 793 method, mod_name = self._fake_module_functions[ft_name][ft_mod] 794 fake_module = self.fake_modules[mod_name] 795 attr = method.__get__(fake_module, fake_module.__class__) 796 for module in modules: 797 self._stubs.smart_set(module, name, attr) 798 799 def patch_modules(self) -> None: 800 assert self._stubs is not None 801 for name, modules in self.FS_MODULES.items(): 802 for module, attr in modules: 803 self._stubs.smart_set( 804 module, name, self.fake_modules[attr]) 805 for name, modules in self.SKIPPED_FS_MODULES.items(): 806 for module, attr in modules: 807 if attr in self.unfaked_modules: 808 self._stubs.smart_set( 809 module, name, self.unfaked_modules[attr]) 810 811 def patch_defaults(self) -> None: 812 for (fct, idx, ft) in self.FS_DEFARGS: 813 method, mod_name = self._fake_module_functions[ 814 ft.__name__][ft.__module__] 815 fake_module = self.fake_modules[mod_name] 816 attr = method.__get__(fake_module, fake_module.__class__) 817 new_defaults = [] 818 assert fct.__defaults__ is not None 819 for i, d in enumerate(fct.__defaults__): 820 if i == idx: 821 new_defaults.append(attr) 822 else: 823 new_defaults.append(d) 824 fct.__defaults__ = tuple(new_defaults) 825 826 def replace_globs(self, globs_: Dict[str, Any]) -> Dict[str, Any]: 827 globs = globs_.copy() 828 if self._isStale: 829 self._refresh() 830 for name in self._fake_module_classes: 831 if name in globs: 832 globs[name] = self._fake_module_classes[name](self.fs) 833 return globs 834 835 def tearDown(self, doctester: Any = None): 836 """Clear the fake filesystem bindings created by `setUp()`.""" 837 self.stop_patching() 838 if self.has_fcopy_file: 839 shutil._HAS_FCOPYFILE = True # type: ignore[attr-defined] 840 841 reset_ids() 842 843 def stop_patching(self) -> None: 844 if self._patching: 845 self._isStale = True 846 self._patching = False 847 if self._stubs: 848 self._stubs.smart_unset_all() 849 self.unset_defaults() 850 if self._dyn_patcher: 851 self._dyn_patcher.cleanup() 852 sys.meta_path.pop(0) 853 854 def unset_defaults(self) -> None: 855 for (fct, idx, ft) in self.FS_DEFARGS: 856 new_defaults = [] 857 for i, d in enumerate(cast(Tuple, fct.__defaults__)): 858 if i == idx: 859 new_defaults.append(ft) 860 else: 861 new_defaults.append(d) 862 fct.__defaults__ = tuple(new_defaults) 863 864 def pause(self) -> None: 865 """Pause the patching of the file system modules until `resume` is 866 called. After that call, all file system calls are executed in the 867 real file system. 868 Calling pause() twice is silently ignored. 869 870 """ 871 self.stop_patching() 872 873 def resume(self) -> None: 874 """Resume the patching of the file system modules if `pause` has 875 been called before. After that call, all file system calls are 876 executed in the fake file system. 877 Does nothing if patching is not paused. 878 """ 879 self.start_patching() 880 881 882class Pause: 883 """Simple context manager that allows to pause/resume patching the 884 filesystem. Patching is paused in the context manager, and resumed after 885 going out of it's scope. 886 """ 887 888 def __init__(self, caller: Union[Patcher, TestCaseMixin, FakeFilesystem]): 889 """Initializes the context manager with the fake filesystem. 890 891 Args: 892 caller: either the FakeFilesystem instance, the Patcher instance 893 or the pyfakefs test case. 894 """ 895 if isinstance(caller, (Patcher, TestCaseMixin)): 896 assert caller.fs is not None 897 self._fs: FakeFilesystem = caller.fs 898 elif isinstance(caller, FakeFilesystem): 899 self._fs = caller 900 else: 901 raise ValueError('Invalid argument - should be of type ' 902 '"fake_filesystem_unittest.Patcher", ' 903 '"fake_filesystem_unittest.TestCase" ' 904 'or "fake_filesystem.FakeFilesystem"') 905 906 def __enter__(self) -> FakeFilesystem: 907 self._fs.pause() 908 return self._fs 909 910 def __exit__(self, *args: Any) -> None: 911 self._fs.resume() 912 913 914class DynamicPatcher(MetaPathFinder, Loader): 915 """A file loader that replaces file system related modules by their 916 fake implementation if they are loaded after calling `setUpPyfakefs()`. 917 Implements the protocol needed for import hooks. 918 """ 919 920 def __init__(self, patcher: Patcher) -> None: 921 self._patcher = patcher 922 self.sysmodules = {} 923 self.modules = self._patcher.fake_modules 924 self._loaded_module_names: Set[str] = set() 925 926 # remove all modules that have to be patched from `sys.modules`, 927 # otherwise the find_... methods will not be called 928 for name in self.modules: 929 if self.needs_patch(name) and name in sys.modules: 930 self.sysmodules[name] = sys.modules[name] 931 del sys.modules[name] 932 933 for name, module in self.modules.items(): 934 sys.modules[name] = module 935 936 def cleanup(self) -> None: 937 for module_name in self.sysmodules: 938 sys.modules[module_name] = self.sysmodules[module_name] 939 for module in self._patcher.modules_to_reload: 940 if module.__name__ in sys.modules: 941 reload(module) 942 reloaded_module_names = [module.__name__ 943 for module in self._patcher.modules_to_reload] 944 # Dereference all modules loaded during the test so they will reload on 945 # the next use, ensuring that no faked modules are referenced after the 946 # test. 947 for name in self._loaded_module_names: 948 if name in sys.modules and name not in reloaded_module_names: 949 del sys.modules[name] 950 951 def needs_patch(self, name: str) -> bool: 952 """Check if the module with the given name shall be replaced.""" 953 if name not in self.modules: 954 self._loaded_module_names.add(name) 955 return False 956 if (name in sys.modules and 957 type(sys.modules[name]) == self.modules[name]): 958 return False 959 return True 960 961 def find_spec(self, fullname: str, 962 path: Optional[Sequence[Union[bytes, str]]], 963 target: Optional[ModuleType] = None) -> Optional[ModuleSpec]: 964 """Module finder.""" 965 if self.needs_patch(fullname): 966 return ModuleSpec(fullname, self) 967 return None 968 969 def load_module(self, fullname: str) -> ModuleType: 970 """Replaces the module by its fake implementation.""" 971 sys.modules[fullname] = self.modules[fullname] 972 return self.modules[fullname] 973