1from test.support import run_unittest, unload, check_warnings, CleanImport 2import unittest 3import sys 4import importlib 5from importlib.util import spec_from_file_location 6import pkgutil 7import os 8import os.path 9import tempfile 10import shutil 11import zipfile 12 13# Note: pkgutil.walk_packages is currently tested in test_runpy. This is 14# a hack to get a major issue resolved for 3.3b2. Longer term, it should 15# be moved back here, perhaps by factoring out the helper code for 16# creating interesting package layouts to a separate module. 17# Issue #15348 declares this is indeed a dodgy hack ;) 18 19class PkgutilTests(unittest.TestCase): 20 21 def setUp(self): 22 self.dirname = tempfile.mkdtemp() 23 self.addCleanup(shutil.rmtree, self.dirname) 24 sys.path.insert(0, self.dirname) 25 26 def tearDown(self): 27 del sys.path[0] 28 29 def test_getdata_filesys(self): 30 pkg = 'test_getdata_filesys' 31 32 # Include a LF and a CRLF, to test that binary data is read back 33 RESOURCE_DATA = b'Hello, world!\nSecond line\r\nThird line' 34 35 # Make a package with some resources 36 package_dir = os.path.join(self.dirname, pkg) 37 os.mkdir(package_dir) 38 # Empty init.py 39 f = open(os.path.join(package_dir, '__init__.py'), "wb") 40 f.close() 41 # Resource files, res.txt, sub/res.txt 42 f = open(os.path.join(package_dir, 'res.txt'), "wb") 43 f.write(RESOURCE_DATA) 44 f.close() 45 os.mkdir(os.path.join(package_dir, 'sub')) 46 f = open(os.path.join(package_dir, 'sub', 'res.txt'), "wb") 47 f.write(RESOURCE_DATA) 48 f.close() 49 50 # Check we can read the resources 51 res1 = pkgutil.get_data(pkg, 'res.txt') 52 self.assertEqual(res1, RESOURCE_DATA) 53 res2 = pkgutil.get_data(pkg, 'sub/res.txt') 54 self.assertEqual(res2, RESOURCE_DATA) 55 56 del sys.modules[pkg] 57 58 def test_getdata_zipfile(self): 59 zip = 'test_getdata_zipfile.zip' 60 pkg = 'test_getdata_zipfile' 61 62 # Include a LF and a CRLF, to test that binary data is read back 63 RESOURCE_DATA = b'Hello, world!\nSecond line\r\nThird line' 64 65 # Make a package with some resources 66 zip_file = os.path.join(self.dirname, zip) 67 z = zipfile.ZipFile(zip_file, 'w') 68 69 # Empty init.py 70 z.writestr(pkg + '/__init__.py', "") 71 # Resource files, res.txt, sub/res.txt 72 z.writestr(pkg + '/res.txt', RESOURCE_DATA) 73 z.writestr(pkg + '/sub/res.txt', RESOURCE_DATA) 74 z.close() 75 76 # Check we can read the resources 77 sys.path.insert(0, zip_file) 78 res1 = pkgutil.get_data(pkg, 'res.txt') 79 self.assertEqual(res1, RESOURCE_DATA) 80 res2 = pkgutil.get_data(pkg, 'sub/res.txt') 81 self.assertEqual(res2, RESOURCE_DATA) 82 83 names = [] 84 for moduleinfo in pkgutil.iter_modules([zip_file]): 85 self.assertIsInstance(moduleinfo, pkgutil.ModuleInfo) 86 names.append(moduleinfo.name) 87 self.assertEqual(names, ['test_getdata_zipfile']) 88 89 del sys.path[0] 90 91 del sys.modules[pkg] 92 93 def test_unreadable_dir_on_syspath(self): 94 # issue7367 - walk_packages failed if unreadable dir on sys.path 95 package_name = "unreadable_package" 96 d = os.path.join(self.dirname, package_name) 97 # this does not appear to create an unreadable dir on Windows 98 # but the test should not fail anyway 99 os.mkdir(d, 0) 100 self.addCleanup(os.rmdir, d) 101 for t in pkgutil.walk_packages(path=[self.dirname]): 102 self.fail("unexpected package found") 103 104 def test_walkpackages_filesys(self): 105 pkg1 = 'test_walkpackages_filesys' 106 pkg1_dir = os.path.join(self.dirname, pkg1) 107 os.mkdir(pkg1_dir) 108 f = open(os.path.join(pkg1_dir, '__init__.py'), "wb") 109 f.close() 110 os.mkdir(os.path.join(pkg1_dir, 'sub')) 111 f = open(os.path.join(pkg1_dir, 'sub', '__init__.py'), "wb") 112 f.close() 113 f = open(os.path.join(pkg1_dir, 'sub', 'mod.py'), "wb") 114 f.close() 115 116 # Now, to juice it up, let's add the opposite packages, too. 117 pkg2 = 'sub' 118 pkg2_dir = os.path.join(self.dirname, pkg2) 119 os.mkdir(pkg2_dir) 120 f = open(os.path.join(pkg2_dir, '__init__.py'), "wb") 121 f.close() 122 os.mkdir(os.path.join(pkg2_dir, 'test_walkpackages_filesys')) 123 f = open(os.path.join(pkg2_dir, 'test_walkpackages_filesys', '__init__.py'), "wb") 124 f.close() 125 f = open(os.path.join(pkg2_dir, 'test_walkpackages_filesys', 'mod.py'), "wb") 126 f.close() 127 128 expected = [ 129 'sub', 130 'sub.test_walkpackages_filesys', 131 'sub.test_walkpackages_filesys.mod', 132 'test_walkpackages_filesys', 133 'test_walkpackages_filesys.sub', 134 'test_walkpackages_filesys.sub.mod', 135 ] 136 actual= [e[1] for e in pkgutil.walk_packages([self.dirname])] 137 self.assertEqual(actual, expected) 138 139 for pkg in expected: 140 if pkg.endswith('mod'): 141 continue 142 del sys.modules[pkg] 143 144 def test_walkpackages_zipfile(self): 145 """Tests the same as test_walkpackages_filesys, only with a zip file.""" 146 147 zip = 'test_walkpackages_zipfile.zip' 148 pkg1 = 'test_walkpackages_zipfile' 149 pkg2 = 'sub' 150 151 zip_file = os.path.join(self.dirname, zip) 152 z = zipfile.ZipFile(zip_file, 'w') 153 z.writestr(pkg2 + '/__init__.py', "") 154 z.writestr(pkg2 + '/' + pkg1 + '/__init__.py', "") 155 z.writestr(pkg2 + '/' + pkg1 + '/mod.py', "") 156 z.writestr(pkg1 + '/__init__.py', "") 157 z.writestr(pkg1 + '/' + pkg2 + '/__init__.py', "") 158 z.writestr(pkg1 + '/' + pkg2 + '/mod.py', "") 159 z.close() 160 161 sys.path.insert(0, zip_file) 162 expected = [ 163 'sub', 164 'sub.test_walkpackages_zipfile', 165 'sub.test_walkpackages_zipfile.mod', 166 'test_walkpackages_zipfile', 167 'test_walkpackages_zipfile.sub', 168 'test_walkpackages_zipfile.sub.mod', 169 ] 170 actual= [e[1] for e in pkgutil.walk_packages([zip_file])] 171 self.assertEqual(actual, expected) 172 del sys.path[0] 173 174 for pkg in expected: 175 if pkg.endswith('mod'): 176 continue 177 del sys.modules[pkg] 178 179 def test_walk_packages_raises_on_string_or_bytes_input(self): 180 181 str_input = 'test_dir' 182 with self.assertRaises((TypeError, ValueError)): 183 list(pkgutil.walk_packages(str_input)) 184 185 bytes_input = b'test_dir' 186 with self.assertRaises((TypeError, ValueError)): 187 list(pkgutil.walk_packages(bytes_input)) 188 189 190class PkgutilPEP302Tests(unittest.TestCase): 191 192 class MyTestLoader(object): 193 def create_module(self, spec): 194 return None 195 196 def exec_module(self, mod): 197 # Count how many times the module is reloaded 198 mod.__dict__['loads'] = mod.__dict__.get('loads', 0) + 1 199 200 def get_data(self, path): 201 return "Hello, world!" 202 203 class MyTestImporter(object): 204 def find_spec(self, fullname, path=None, target=None): 205 loader = PkgutilPEP302Tests.MyTestLoader() 206 return spec_from_file_location(fullname, 207 '<%s>' % loader.__class__.__name__, 208 loader=loader, 209 submodule_search_locations=[]) 210 211 def setUp(self): 212 sys.meta_path.insert(0, self.MyTestImporter()) 213 214 def tearDown(self): 215 del sys.meta_path[0] 216 217 def test_getdata_pep302(self): 218 # Use a dummy finder/loader 219 self.assertEqual(pkgutil.get_data('foo', 'dummy'), "Hello, world!") 220 del sys.modules['foo'] 221 222 def test_alreadyloaded(self): 223 # Ensure that get_data works without reloading - the "loads" module 224 # variable in the example loader should count how many times a reload 225 # occurs. 226 import foo 227 self.assertEqual(foo.loads, 1) 228 self.assertEqual(pkgutil.get_data('foo', 'dummy'), "Hello, world!") 229 self.assertEqual(foo.loads, 1) 230 del sys.modules['foo'] 231 232 233# These tests, especially the setup and cleanup, are hideous. They 234# need to be cleaned up once issue 14715 is addressed. 235class ExtendPathTests(unittest.TestCase): 236 def create_init(self, pkgname): 237 dirname = tempfile.mkdtemp() 238 sys.path.insert(0, dirname) 239 240 pkgdir = os.path.join(dirname, pkgname) 241 os.mkdir(pkgdir) 242 with open(os.path.join(pkgdir, '__init__.py'), 'w') as fl: 243 fl.write('from pkgutil import extend_path\n__path__ = extend_path(__path__, __name__)\n') 244 245 return dirname 246 247 def create_submodule(self, dirname, pkgname, submodule_name, value): 248 module_name = os.path.join(dirname, pkgname, submodule_name + '.py') 249 with open(module_name, 'w') as fl: 250 print('value={}'.format(value), file=fl) 251 252 def test_simple(self): 253 pkgname = 'foo' 254 dirname_0 = self.create_init(pkgname) 255 dirname_1 = self.create_init(pkgname) 256 self.create_submodule(dirname_0, pkgname, 'bar', 0) 257 self.create_submodule(dirname_1, pkgname, 'baz', 1) 258 import foo.bar 259 import foo.baz 260 # Ensure we read the expected values 261 self.assertEqual(foo.bar.value, 0) 262 self.assertEqual(foo.baz.value, 1) 263 264 # Ensure the path is set up correctly 265 self.assertEqual(sorted(foo.__path__), 266 sorted([os.path.join(dirname_0, pkgname), 267 os.path.join(dirname_1, pkgname)])) 268 269 # Cleanup 270 shutil.rmtree(dirname_0) 271 shutil.rmtree(dirname_1) 272 del sys.path[0] 273 del sys.path[0] 274 del sys.modules['foo'] 275 del sys.modules['foo.bar'] 276 del sys.modules['foo.baz'] 277 278 279 # Another awful testing hack to be cleaned up once the test_runpy 280 # helpers are factored out to a common location 281 def test_iter_importers(self): 282 iter_importers = pkgutil.iter_importers 283 get_importer = pkgutil.get_importer 284 285 pkgname = 'spam' 286 modname = 'eggs' 287 dirname = self.create_init(pkgname) 288 pathitem = os.path.join(dirname, pkgname) 289 fullname = '{}.{}'.format(pkgname, modname) 290 sys.modules.pop(fullname, None) 291 sys.modules.pop(pkgname, None) 292 try: 293 self.create_submodule(dirname, pkgname, modname, 0) 294 295 importlib.import_module(fullname) 296 297 importers = list(iter_importers(fullname)) 298 expected_importer = get_importer(pathitem) 299 for finder in importers: 300 spec = pkgutil._get_spec(finder, fullname) 301 loader = spec.loader 302 try: 303 loader = loader.loader 304 except AttributeError: 305 # For now we still allow raw loaders from 306 # find_module(). 307 pass 308 self.assertIsInstance(finder, importlib.machinery.FileFinder) 309 self.assertEqual(finder, expected_importer) 310 self.assertIsInstance(loader, 311 importlib.machinery.SourceFileLoader) 312 self.assertIsNone(pkgutil._get_spec(finder, pkgname)) 313 314 with self.assertRaises(ImportError): 315 list(iter_importers('invalid.module')) 316 317 with self.assertRaises(ImportError): 318 list(iter_importers('.spam')) 319 finally: 320 shutil.rmtree(dirname) 321 del sys.path[0] 322 try: 323 del sys.modules['spam'] 324 del sys.modules['spam.eggs'] 325 except KeyError: 326 pass 327 328 329 def test_mixed_namespace(self): 330 pkgname = 'foo' 331 dirname_0 = self.create_init(pkgname) 332 dirname_1 = self.create_init(pkgname) 333 self.create_submodule(dirname_0, pkgname, 'bar', 0) 334 # Turn this into a PEP 420 namespace package 335 os.unlink(os.path.join(dirname_0, pkgname, '__init__.py')) 336 self.create_submodule(dirname_1, pkgname, 'baz', 1) 337 import foo.bar 338 import foo.baz 339 # Ensure we read the expected values 340 self.assertEqual(foo.bar.value, 0) 341 self.assertEqual(foo.baz.value, 1) 342 343 # Ensure the path is set up correctly 344 self.assertEqual(sorted(foo.__path__), 345 sorted([os.path.join(dirname_0, pkgname), 346 os.path.join(dirname_1, pkgname)])) 347 348 # Cleanup 349 shutil.rmtree(dirname_0) 350 shutil.rmtree(dirname_1) 351 del sys.path[0] 352 del sys.path[0] 353 del sys.modules['foo'] 354 del sys.modules['foo.bar'] 355 del sys.modules['foo.baz'] 356 357 # XXX: test .pkg files 358 359 360class NestedNamespacePackageTest(unittest.TestCase): 361 362 def setUp(self): 363 self.basedir = tempfile.mkdtemp() 364 self.old_path = sys.path[:] 365 366 def tearDown(self): 367 sys.path[:] = self.old_path 368 shutil.rmtree(self.basedir) 369 370 def create_module(self, name, contents): 371 base, final = name.rsplit('.', 1) 372 base_path = os.path.join(self.basedir, base.replace('.', os.path.sep)) 373 os.makedirs(base_path, exist_ok=True) 374 with open(os.path.join(base_path, final + ".py"), 'w') as f: 375 f.write(contents) 376 377 def test_nested(self): 378 pkgutil_boilerplate = ( 379 'import pkgutil; ' 380 '__path__ = pkgutil.extend_path(__path__, __name__)') 381 self.create_module('a.pkg.__init__', pkgutil_boilerplate) 382 self.create_module('b.pkg.__init__', pkgutil_boilerplate) 383 self.create_module('a.pkg.subpkg.__init__', pkgutil_boilerplate) 384 self.create_module('b.pkg.subpkg.__init__', pkgutil_boilerplate) 385 self.create_module('a.pkg.subpkg.c', 'c = 1') 386 self.create_module('b.pkg.subpkg.d', 'd = 2') 387 sys.path.insert(0, os.path.join(self.basedir, 'a')) 388 sys.path.insert(0, os.path.join(self.basedir, 'b')) 389 import pkg 390 self.addCleanup(unload, 'pkg') 391 self.assertEqual(len(pkg.__path__), 2) 392 import pkg.subpkg 393 self.addCleanup(unload, 'pkg.subpkg') 394 self.assertEqual(len(pkg.subpkg.__path__), 2) 395 from pkg.subpkg.c import c 396 from pkg.subpkg.d import d 397 self.assertEqual(c, 1) 398 self.assertEqual(d, 2) 399 400 401class ImportlibMigrationTests(unittest.TestCase): 402 # With full PEP 302 support in the standard import machinery, the 403 # PEP 302 emulation in this module is in the process of being 404 # deprecated in favour of importlib proper 405 406 def check_deprecated(self): 407 return check_warnings( 408 ("This emulation is deprecated, use 'importlib' instead", 409 DeprecationWarning)) 410 411 def test_importer_deprecated(self): 412 with self.check_deprecated(): 413 pkgutil.ImpImporter("") 414 415 def test_loader_deprecated(self): 416 with self.check_deprecated(): 417 pkgutil.ImpLoader("", "", "", "") 418 419 def test_get_loader_avoids_emulation(self): 420 with check_warnings() as w: 421 self.assertIsNotNone(pkgutil.get_loader("sys")) 422 self.assertIsNotNone(pkgutil.get_loader("os")) 423 self.assertIsNotNone(pkgutil.get_loader("test.support")) 424 self.assertEqual(len(w.warnings), 0) 425 426 @unittest.skipIf(__name__ == '__main__', 'not compatible with __main__') 427 def test_get_loader_handles_missing_loader_attribute(self): 428 global __loader__ 429 this_loader = __loader__ 430 del __loader__ 431 try: 432 with check_warnings() as w: 433 self.assertIsNotNone(pkgutil.get_loader(__name__)) 434 self.assertEqual(len(w.warnings), 0) 435 finally: 436 __loader__ = this_loader 437 438 def test_get_loader_handles_missing_spec_attribute(self): 439 name = 'spam' 440 mod = type(sys)(name) 441 del mod.__spec__ 442 with CleanImport(name): 443 sys.modules[name] = mod 444 loader = pkgutil.get_loader(name) 445 self.assertIsNone(loader) 446 447 def test_get_loader_handles_spec_attribute_none(self): 448 name = 'spam' 449 mod = type(sys)(name) 450 mod.__spec__ = None 451 with CleanImport(name): 452 sys.modules[name] = mod 453 loader = pkgutil.get_loader(name) 454 self.assertIsNone(loader) 455 456 def test_get_loader_None_in_sys_modules(self): 457 name = 'totally bogus' 458 sys.modules[name] = None 459 try: 460 loader = pkgutil.get_loader(name) 461 finally: 462 del sys.modules[name] 463 self.assertIsNone(loader) 464 465 def test_find_loader_missing_module(self): 466 name = 'totally bogus' 467 loader = pkgutil.find_loader(name) 468 self.assertIsNone(loader) 469 470 def test_find_loader_avoids_emulation(self): 471 with check_warnings() as w: 472 self.assertIsNotNone(pkgutil.find_loader("sys")) 473 self.assertIsNotNone(pkgutil.find_loader("os")) 474 self.assertIsNotNone(pkgutil.find_loader("test.support")) 475 self.assertEqual(len(w.warnings), 0) 476 477 def test_get_importer_avoids_emulation(self): 478 # We use an illegal path so *none* of the path hooks should fire 479 with check_warnings() as w: 480 self.assertIsNone(pkgutil.get_importer("*??")) 481 self.assertEqual(len(w.warnings), 0) 482 483 def test_iter_importers_avoids_emulation(self): 484 with check_warnings() as w: 485 for importer in pkgutil.iter_importers(): pass 486 self.assertEqual(len(w.warnings), 0) 487 488 489def test_main(): 490 run_unittest(PkgutilTests, PkgutilPEP302Tests, ExtendPathTests, 491 NestedNamespacePackageTest, ImportlibMigrationTests) 492 # this is necessary if test is run repeated (like when finding leaks) 493 import zipimport 494 import importlib 495 zipimport._zip_directory_cache.clear() 496 importlib.invalidate_caches() 497 498 499if __name__ == '__main__': 500 test_main() 501