1"""Utility code for constructing importers, etc.""" 2from ._abc import Loader 3from ._bootstrap import module_from_spec 4from ._bootstrap import _resolve_name 5from ._bootstrap import spec_from_loader 6from ._bootstrap import _find_spec 7from ._bootstrap_external import MAGIC_NUMBER 8from ._bootstrap_external import _RAW_MAGIC_NUMBER 9from ._bootstrap_external import cache_from_source 10from ._bootstrap_external import decode_source 11from ._bootstrap_external import source_from_cache 12from ._bootstrap_external import spec_from_file_location 13 14import _imp 15import sys 16import types 17 18 19def source_hash(source_bytes): 20 "Return the hash of *source_bytes* as used in hash-based pyc files." 21 return _imp.source_hash(_RAW_MAGIC_NUMBER, source_bytes) 22 23 24def resolve_name(name, package): 25 """Resolve a relative module name to an absolute one.""" 26 if not name.startswith('.'): 27 return name 28 elif not package: 29 raise ImportError(f'no package specified for {repr(name)} ' 30 '(required for relative module names)') 31 level = 0 32 for character in name: 33 if character != '.': 34 break 35 level += 1 36 return _resolve_name(name[level:], package, level) 37 38 39def _find_spec_from_path(name, path=None): 40 """Return the spec for the specified module. 41 42 First, sys.modules is checked to see if the module was already imported. If 43 so, then sys.modules[name].__spec__ is returned. If that happens to be 44 set to None, then ValueError is raised. If the module is not in 45 sys.modules, then sys.meta_path is searched for a suitable spec with the 46 value of 'path' given to the finders. None is returned if no spec could 47 be found. 48 49 Dotted names do not have their parent packages implicitly imported. You will 50 most likely need to explicitly import all parent packages in the proper 51 order for a submodule to get the correct spec. 52 53 """ 54 if name not in sys.modules: 55 return _find_spec(name, path) 56 else: 57 module = sys.modules[name] 58 if module is None: 59 return None 60 try: 61 spec = module.__spec__ 62 except AttributeError: 63 raise ValueError(f'{name}.__spec__ is not set') from None 64 else: 65 if spec is None: 66 raise ValueError(f'{name}.__spec__ is None') 67 return spec 68 69 70def find_spec(name, package=None): 71 """Return the spec for the specified module. 72 73 First, sys.modules is checked to see if the module was already imported. If 74 so, then sys.modules[name].__spec__ is returned. If that happens to be 75 set to None, then ValueError is raised. If the module is not in 76 sys.modules, then sys.meta_path is searched for a suitable spec with the 77 value of 'path' given to the finders. None is returned if no spec could 78 be found. 79 80 If the name is for submodule (contains a dot), the parent module is 81 automatically imported. 82 83 The name and package arguments work the same as importlib.import_module(). 84 In other words, relative module names (with leading dots) work. 85 86 """ 87 fullname = resolve_name(name, package) if name.startswith('.') else name 88 if fullname not in sys.modules: 89 parent_name = fullname.rpartition('.')[0] 90 if parent_name: 91 parent = __import__(parent_name, fromlist=['__path__']) 92 try: 93 parent_path = parent.__path__ 94 except AttributeError as e: 95 raise ModuleNotFoundError( 96 f"__path__ attribute not found on {parent_name!r} " 97 f"while trying to find {fullname!r}", name=fullname) from e 98 else: 99 parent_path = None 100 return _find_spec(fullname, parent_path) 101 else: 102 module = sys.modules[fullname] 103 if module is None: 104 return None 105 try: 106 spec = module.__spec__ 107 except AttributeError: 108 raise ValueError(f'{name}.__spec__ is not set') from None 109 else: 110 if spec is None: 111 raise ValueError(f'{name}.__spec__ is None') 112 return spec 113 114 115# Normally we would use contextlib.contextmanager. However, this module 116# is imported by runpy, which means we want to avoid any unnecessary 117# dependencies. Thus we use a class. 118 119class _incompatible_extension_module_restrictions: 120 """A context manager that can temporarily skip the compatibility check. 121 122 NOTE: This function is meant to accommodate an unusual case; one 123 which is likely to eventually go away. There's is a pretty good 124 chance this is not what you were looking for. 125 126 WARNING: Using this function to disable the check can lead to 127 unexpected behavior and even crashes. It should only be used during 128 extension module development. 129 130 If "disable_check" is True then the compatibility check will not 131 happen while the context manager is active. Otherwise the check 132 *will* happen. 133 134 Normally, extensions that do not support multiple interpreters 135 may not be imported in a subinterpreter. That implies modules 136 that do not implement multi-phase init or that explicitly of out. 137 138 Likewise for modules import in a subinterpreter with its own GIL 139 when the extension does not support a per-interpreter GIL. This 140 implies the module does not have a Py_mod_multiple_interpreters slot 141 set to Py_MOD_PER_INTERPRETER_GIL_SUPPORTED. 142 143 In both cases, this context manager may be used to temporarily 144 disable the check for compatible extension modules. 145 146 You can get the same effect as this function by implementing the 147 basic interface of multi-phase init (PEP 489) and lying about 148 support for multiple interpreters (or per-interpreter GIL). 149 """ 150 151 def __init__(self, *, disable_check): 152 self.disable_check = bool(disable_check) 153 154 def __enter__(self): 155 self.old = _imp._override_multi_interp_extensions_check(self.override) 156 return self 157 158 def __exit__(self, *args): 159 old = self.old 160 del self.old 161 _imp._override_multi_interp_extensions_check(old) 162 163 @property 164 def override(self): 165 return -1 if self.disable_check else 1 166 167 168class _LazyModule(types.ModuleType): 169 170 """A subclass of the module type which triggers loading upon attribute access.""" 171 172 def __getattribute__(self, attr): 173 """Trigger the load of the module and return the attribute.""" 174 __spec__ = object.__getattribute__(self, '__spec__') 175 loader_state = __spec__.loader_state 176 with loader_state['lock']: 177 # Only the first thread to get the lock should trigger the load 178 # and reset the module's class. The rest can now getattr(). 179 if object.__getattribute__(self, '__class__') is _LazyModule: 180 __class__ = loader_state['__class__'] 181 182 # Reentrant calls from the same thread must be allowed to proceed without 183 # triggering the load again. 184 # exec_module() and self-referential imports are the primary ways this can 185 # happen, but in any case we must return something to avoid deadlock. 186 if loader_state['is_loading']: 187 return __class__.__getattribute__(self, attr) 188 loader_state['is_loading'] = True 189 190 __dict__ = __class__.__getattribute__(self, '__dict__') 191 192 # All module metadata must be gathered from __spec__ in order to avoid 193 # using mutated values. 194 # Get the original name to make sure no object substitution occurred 195 # in sys.modules. 196 original_name = __spec__.name 197 # Figure out exactly what attributes were mutated between the creation 198 # of the module and now. 199 attrs_then = loader_state['__dict__'] 200 attrs_now = __dict__ 201 attrs_updated = {} 202 for key, value in attrs_now.items(): 203 # Code that set an attribute may have kept a reference to the 204 # assigned object, making identity more important than equality. 205 if key not in attrs_then: 206 attrs_updated[key] = value 207 elif id(attrs_now[key]) != id(attrs_then[key]): 208 attrs_updated[key] = value 209 __spec__.loader.exec_module(self) 210 # If exec_module() was used directly there is no guarantee the module 211 # object was put into sys.modules. 212 if original_name in sys.modules: 213 if id(self) != id(sys.modules[original_name]): 214 raise ValueError(f"module object for {original_name!r} " 215 "substituted in sys.modules during a lazy " 216 "load") 217 # Update after loading since that's what would happen in an eager 218 # loading situation. 219 __dict__.update(attrs_updated) 220 # Finally, stop triggering this method, if the module did not 221 # already update its own __class__. 222 if isinstance(self, _LazyModule): 223 object.__setattr__(self, '__class__', __class__) 224 225 return getattr(self, attr) 226 227 def __delattr__(self, attr): 228 """Trigger the load and then perform the deletion.""" 229 # To trigger the load and raise an exception if the attribute 230 # doesn't exist. 231 self.__getattribute__(attr) 232 delattr(self, attr) 233 234 235class LazyLoader(Loader): 236 237 """A loader that creates a module which defers loading until attribute access.""" 238 239 @staticmethod 240 def __check_eager_loader(loader): 241 if not hasattr(loader, 'exec_module'): 242 raise TypeError('loader must define exec_module()') 243 244 @classmethod 245 def factory(cls, loader): 246 """Construct a callable which returns the eager loader made lazy.""" 247 cls.__check_eager_loader(loader) 248 return lambda *args, **kwargs: cls(loader(*args, **kwargs)) 249 250 def __init__(self, loader): 251 self.__check_eager_loader(loader) 252 self.loader = loader 253 254 def create_module(self, spec): 255 return self.loader.create_module(spec) 256 257 def exec_module(self, module): 258 """Make the module load lazily.""" 259 # Threading is only needed for lazy loading, and importlib.util can 260 # be pulled in at interpreter startup, so defer until needed. 261 import threading 262 module.__spec__.loader = self.loader 263 module.__loader__ = self.loader 264 # Don't need to worry about deep-copying as trying to set an attribute 265 # on an object would have triggered the load, 266 # e.g. ``module.__spec__.loader = None`` would trigger a load from 267 # trying to access module.__spec__. 268 loader_state = {} 269 loader_state['__dict__'] = module.__dict__.copy() 270 loader_state['__class__'] = module.__class__ 271 loader_state['lock'] = threading.RLock() 272 loader_state['is_loading'] = False 273 module.__spec__.loader_state = loader_state 274 module.__class__ = _LazyModule 275