1import os 2import sys 3from configparser import ConfigParser 4from itertools import product 5 6from setuptools.command.sdist import sdist 7from setuptools.dist import Distribution 8from setuptools.discovery import find_package_path, find_parent_package 9from setuptools.errors import PackageDiscoveryError 10 11import setuptools # noqa -- force distutils.core to be patched 12import distutils.core 13 14import pytest 15import jaraco.path 16from path import Path as _Path 17 18from .contexts import quiet 19from .integration.helpers import get_sdist_members, get_wheel_members, run 20from .textwrap import DALS 21 22 23class TestFindParentPackage: 24 def test_single_package(self, tmp_path): 25 # find_parent_package should find a non-namespace parent package 26 (tmp_path / "src/namespace/pkg/nested").mkdir(exist_ok=True, parents=True) 27 (tmp_path / "src/namespace/pkg/nested/__init__.py").touch() 28 (tmp_path / "src/namespace/pkg/__init__.py").touch() 29 packages = ["namespace", "namespace.pkg", "namespace.pkg.nested"] 30 assert find_parent_package(packages, {"": "src"}, tmp_path) == "namespace.pkg" 31 32 def test_multiple_toplevel(self, tmp_path): 33 # find_parent_package should return null if the given list of packages does not 34 # have a single parent package 35 multiple = ["pkg", "pkg1", "pkg2"] 36 for name in multiple: 37 (tmp_path / f"src/{name}").mkdir(exist_ok=True, parents=True) 38 (tmp_path / f"src/{name}/__init__.py").touch() 39 assert find_parent_package(multiple, {"": "src"}, tmp_path) is None 40 41 42class TestDiscoverPackagesAndPyModules: 43 """Make sure discovered values for ``packages`` and ``py_modules`` work 44 similarly to explicit configuration for the simple scenarios. 45 """ 46 OPTIONS = { 47 # Different options according to the circumstance being tested 48 "explicit-src": { 49 "package_dir": {"": "src"}, 50 "packages": ["pkg"] 51 }, 52 "variation-lib": { 53 "package_dir": {"": "lib"}, # variation of the source-layout 54 }, 55 "explicit-flat": { 56 "packages": ["pkg"] 57 }, 58 "explicit-single_module": { 59 "py_modules": ["pkg"] 60 }, 61 "explicit-namespace": { 62 "packages": ["ns", "ns.pkg"] 63 }, 64 "automatic-src": {}, 65 "automatic-flat": {}, 66 "automatic-single_module": {}, 67 "automatic-namespace": {} 68 } 69 FILES = { 70 "src": ["src/pkg/__init__.py", "src/pkg/main.py"], 71 "lib": ["lib/pkg/__init__.py", "lib/pkg/main.py"], 72 "flat": ["pkg/__init__.py", "pkg/main.py"], 73 "single_module": ["pkg.py"], 74 "namespace": ["ns/pkg/__init__.py"] 75 } 76 77 def _get_info(self, circumstance): 78 _, _, layout = circumstance.partition("-") 79 files = self.FILES[layout] 80 options = self.OPTIONS[circumstance] 81 return files, options 82 83 @pytest.mark.parametrize("circumstance", OPTIONS.keys()) 84 def test_sdist_filelist(self, tmp_path, circumstance): 85 files, options = self._get_info(circumstance) 86 _populate_project_dir(tmp_path, files, options) 87 88 _, cmd = _run_sdist_programatically(tmp_path, options) 89 90 manifest = [f.replace(os.sep, "/") for f in cmd.filelist.files] 91 for file in files: 92 assert any(f.endswith(file) for f in manifest) 93 94 @pytest.mark.parametrize("circumstance", OPTIONS.keys()) 95 def test_project(self, tmp_path, circumstance): 96 files, options = self._get_info(circumstance) 97 _populate_project_dir(tmp_path, files, options) 98 99 # Simulate a pre-existing `build` directory 100 (tmp_path / "build").mkdir() 101 (tmp_path / "build/lib").mkdir() 102 (tmp_path / "build/bdist.linux-x86_64").mkdir() 103 (tmp_path / "build/bdist.linux-x86_64/file.py").touch() 104 (tmp_path / "build/lib/__init__.py").touch() 105 (tmp_path / "build/lib/file.py").touch() 106 (tmp_path / "dist").mkdir() 107 (tmp_path / "dist/file.py").touch() 108 109 _run_build(tmp_path) 110 111 sdist_files = get_sdist_members(next(tmp_path.glob("dist/*.tar.gz"))) 112 print("~~~~~ sdist_members ~~~~~") 113 print('\n'.join(sdist_files)) 114 assert sdist_files >= set(files) 115 116 wheel_files = get_wheel_members(next(tmp_path.glob("dist/*.whl"))) 117 print("~~~~~ wheel_members ~~~~~") 118 print('\n'.join(wheel_files)) 119 orig_files = {f.replace("src/", "").replace("lib/", "") for f in files} 120 assert wheel_files >= orig_files 121 122 # Make sure build files are not included by mistake 123 for file in wheel_files: 124 assert "build" not in files 125 assert "dist" not in files 126 127 PURPOSEFULLY_EMPY = { 128 "setup.cfg": DALS( 129 """ 130 [metadata] 131 name = myproj 132 version = 0.0.0 133 134 [options] 135 {param} = 136 """ 137 ), 138 "setup.py": DALS( 139 """ 140 __import__('setuptools').setup( 141 name="myproj", 142 version="0.0.0", 143 {param}=[] 144 ) 145 """ 146 ), 147 "pyproject.toml": DALS( 148 """ 149 [build-system] 150 requires = [] 151 build-backend = 'setuptools.build_meta' 152 153 [project] 154 name = "myproj" 155 version = "0.0.0" 156 157 [tool.setuptools] 158 {param} = [] 159 """ 160 ), 161 "template-pyproject.toml": DALS( 162 """ 163 [build-system] 164 requires = [] 165 build-backend = 'setuptools.build_meta' 166 """ 167 ) 168 } 169 170 @pytest.mark.parametrize( 171 "config_file, param, circumstance", 172 product( 173 ["setup.cfg", "setup.py", "pyproject.toml"], 174 ["packages", "py_modules"], 175 FILES.keys() 176 ) 177 ) 178 def test_purposefully_empty(self, tmp_path, config_file, param, circumstance): 179 files = self.FILES[circumstance] + ["mod.py", "other.py", "src/pkg/__init__.py"] 180 _populate_project_dir(tmp_path, files, {}) 181 182 if config_file == "pyproject.toml": 183 template_param = param.replace("_", "-") 184 else: 185 # Make sure build works with or without setup.cfg 186 pyproject = self.PURPOSEFULLY_EMPY["template-pyproject.toml"] 187 (tmp_path / "pyproject.toml").write_text(pyproject) 188 template_param = param 189 190 config = self.PURPOSEFULLY_EMPY[config_file].format(param=template_param) 191 (tmp_path / config_file).write_text(config) 192 193 dist = _get_dist(tmp_path, {}) 194 # When either parameter package or py_modules is an empty list, 195 # then there should be no discovery 196 assert getattr(dist, param) == [] 197 other = {"py_modules": "packages", "packages": "py_modules"}[param] 198 assert getattr(dist, other) is None 199 200 @pytest.mark.parametrize( 201 "extra_files, pkgs", 202 [ 203 (["venv/bin/simulate_venv"], {"pkg"}), 204 (["pkg-stubs/__init__.pyi"], {"pkg", "pkg-stubs"}), 205 (["other-stubs/__init__.pyi"], {"pkg", "other-stubs"}), 206 ( 207 # Type stubs can also be namespaced 208 ["namespace-stubs/pkg/__init__.pyi"], 209 {"pkg", "namespace-stubs", "namespace-stubs.pkg"}, 210 ), 211 ( 212 # Just the top-level package can have `-stubs`, ignore nested ones 213 ["namespace-stubs/pkg-stubs/__init__.pyi"], 214 {"pkg", "namespace-stubs"} 215 ), 216 (["_hidden/file.py"], {"pkg"}), 217 (["news/finalize.py"], {"pkg"}), 218 ] 219 ) 220 def test_flat_layout_with_extra_files(self, tmp_path, extra_files, pkgs): 221 files = self.FILES["flat"] + extra_files 222 _populate_project_dir(tmp_path, files, {}) 223 dist = _get_dist(tmp_path, {}) 224 assert set(dist.packages) == pkgs 225 226 @pytest.mark.parametrize( 227 "extra_files", 228 [ 229 ["other/__init__.py"], 230 ["other/finalize.py"], 231 ] 232 ) 233 def test_flat_layout_with_dangerous_extra_files(self, tmp_path, extra_files): 234 files = self.FILES["flat"] + extra_files 235 _populate_project_dir(tmp_path, files, {}) 236 with pytest.raises(PackageDiscoveryError, match="multiple (packages|modules)"): 237 _get_dist(tmp_path, {}) 238 239 def test_flat_layout_with_single_module(self, tmp_path): 240 files = self.FILES["single_module"] + ["invalid-module-name.py"] 241 _populate_project_dir(tmp_path, files, {}) 242 dist = _get_dist(tmp_path, {}) 243 assert set(dist.py_modules) == {"pkg"} 244 245 def test_flat_layout_with_multiple_modules(self, tmp_path): 246 files = self.FILES["single_module"] + ["valid_module_name.py"] 247 _populate_project_dir(tmp_path, files, {}) 248 with pytest.raises(PackageDiscoveryError, match="multiple (packages|modules)"): 249 _get_dist(tmp_path, {}) 250 251 252class TestNoConfig: 253 DEFAULT_VERSION = "0.0.0" # Default version given by setuptools 254 255 EXAMPLES = { 256 "pkg1": ["src/pkg1.py"], 257 "pkg2": ["src/pkg2/__init__.py"], 258 "pkg3": ["src/pkg3/__init__.py", "src/pkg3-stubs/__init__.py"], 259 "pkg4": ["pkg4/__init__.py", "pkg4-stubs/__init__.py"], 260 "ns.nested.pkg1": ["src/ns/nested/pkg1/__init__.py"], 261 "ns.nested.pkg2": ["ns/nested/pkg2/__init__.py"], 262 } 263 264 @pytest.mark.parametrize("example", EXAMPLES.keys()) 265 def test_discover_name(self, tmp_path, example): 266 _populate_project_dir(tmp_path, self.EXAMPLES[example], {}) 267 dist = _get_dist(tmp_path, {}) 268 assert dist.get_name() == example 269 270 def test_build_with_discovered_name(self, tmp_path): 271 files = ["src/ns/nested/pkg/__init__.py"] 272 _populate_project_dir(tmp_path, files, {}) 273 _run_build(tmp_path, "--sdist") 274 # Expected distribution file 275 dist_file = tmp_path / f"dist/ns.nested.pkg-{self.DEFAULT_VERSION}.tar.gz" 276 assert dist_file.is_file() 277 278 279class TestWithAttrDirective: 280 @pytest.mark.parametrize( 281 "folder, opts", 282 [ 283 ("src", {}), 284 ("lib", {"packages": "find:", "packages.find": {"where": "lib"}}), 285 ] 286 ) 287 def test_setupcfg_metadata(self, tmp_path, folder, opts): 288 files = [f"{folder}/pkg/__init__.py", "setup.cfg"] 289 _populate_project_dir(tmp_path, files, opts) 290 (tmp_path / folder / "pkg/__init__.py").write_text("version = 42") 291 (tmp_path / "setup.cfg").write_text( 292 "[metadata]\nversion = attr: pkg.version\n" 293 + (tmp_path / "setup.cfg").read_text() 294 ) 295 296 dist = _get_dist(tmp_path, {}) 297 assert dist.get_name() == "pkg" 298 assert dist.get_version() == "42" 299 assert dist.package_dir 300 package_path = find_package_path("pkg", dist.package_dir, tmp_path) 301 assert os.path.exists(package_path) 302 assert folder in _Path(package_path).parts() 303 304 _run_build(tmp_path, "--sdist") 305 dist_file = tmp_path / "dist/pkg-42.tar.gz" 306 assert dist_file.is_file() 307 308 def test_pyproject_metadata(self, tmp_path): 309 _populate_project_dir(tmp_path, ["src/pkg/__init__.py"], {}) 310 (tmp_path / "src/pkg/__init__.py").write_text("version = 42") 311 (tmp_path / "pyproject.toml").write_text( 312 "[project]\nname = 'pkg'\ndynamic = ['version']\n" 313 "[tool.setuptools.dynamic]\nversion = {attr = 'pkg.version'}\n" 314 ) 315 dist = _get_dist(tmp_path, {}) 316 assert dist.get_version() == "42" 317 assert dist.package_dir == {"": "src"} 318 319 320class TestWithCExtension: 321 def _simulate_package_with_extension(self, tmp_path): 322 # This example is based on: https://github.com/nucleic/kiwi/tree/1.4.0 323 files = [ 324 "benchmarks/file.py", 325 "docs/Makefile", 326 "docs/requirements.txt", 327 "docs/source/conf.py", 328 "proj/header.h", 329 "proj/file.py", 330 "py/proj.cpp", 331 "py/other.cpp", 332 "py/file.py", 333 "py/py.typed", 334 "py/tests/test_proj.py", 335 "README.rst", 336 ] 337 _populate_project_dir(tmp_path, files, {}) 338 339 setup_script = """ 340 from setuptools import Extension, setup 341 342 ext_modules = [ 343 Extension( 344 "proj", 345 ["py/proj.cpp", "py/other.cpp"], 346 include_dirs=["."], 347 language="c++", 348 ), 349 ] 350 setup(ext_modules=ext_modules) 351 """ 352 (tmp_path / "setup.py").write_text(DALS(setup_script)) 353 354 def test_skip_discovery_with_setupcfg_metadata(self, tmp_path): 355 """Ensure that auto-discovery is not triggered when the project is based on 356 C-extensions only, for backward compatibility. 357 """ 358 self._simulate_package_with_extension(tmp_path) 359 360 pyproject = """ 361 [build-system] 362 requires = [] 363 build-backend = 'setuptools.build_meta' 364 """ 365 (tmp_path / "pyproject.toml").write_text(DALS(pyproject)) 366 367 setupcfg = """ 368 [metadata] 369 name = proj 370 version = 42 371 """ 372 (tmp_path / "setup.cfg").write_text(DALS(setupcfg)) 373 374 dist = _get_dist(tmp_path, {}) 375 assert dist.get_name() == "proj" 376 assert dist.get_version() == "42" 377 assert dist.py_modules is None 378 assert dist.packages is None 379 assert len(dist.ext_modules) == 1 380 assert dist.ext_modules[0].name == "proj" 381 382 def test_dont_skip_discovery_with_pyproject_metadata(self, tmp_path): 383 """When opting-in to pyproject.toml metadata, auto-discovery will be active if 384 the package lists C-extensions, but does not configure py-modules or packages. 385 386 This way we ensure users with complex package layouts that would lead to the 387 discovery of multiple top-level modules/packages see errors and are forced to 388 explicitly set ``packages`` or ``py-modules``. 389 """ 390 self._simulate_package_with_extension(tmp_path) 391 392 pyproject = """ 393 [project] 394 name = 'proj' 395 version = '42' 396 """ 397 (tmp_path / "pyproject.toml").write_text(DALS(pyproject)) 398 with pytest.raises(PackageDiscoveryError, match="multiple (packages|modules)"): 399 _get_dist(tmp_path, {}) 400 401 402class TestWithPackageData: 403 def _simulate_package_with_data_files(self, tmp_path, src_root): 404 files = [ 405 f"{src_root}/proj/__init__.py", 406 f"{src_root}/proj/file1.txt", 407 f"{src_root}/proj/nested/file2.txt", 408 ] 409 _populate_project_dir(tmp_path, files, {}) 410 411 manifest = """ 412 global-include *.py *.txt 413 """ 414 (tmp_path / "MANIFEST.in").write_text(DALS(manifest)) 415 416 EXAMPLE_SETUPCFG = """ 417 [metadata] 418 name = proj 419 version = 42 420 421 [options] 422 include_package_data = True 423 """ 424 EXAMPLE_PYPROJECT = """ 425 [project] 426 name = "proj" 427 version = "42" 428 """ 429 430 PYPROJECT_PACKAGE_DIR = """ 431 [tool.setuptools] 432 package-dir = {"" = "src"} 433 """ 434 435 @pytest.mark.parametrize( 436 "src_root, files", 437 [ 438 (".", {"setup.cfg": DALS(EXAMPLE_SETUPCFG)}), 439 (".", {"pyproject.toml": DALS(EXAMPLE_PYPROJECT)}), 440 ("src", {"setup.cfg": DALS(EXAMPLE_SETUPCFG)}), 441 ("src", {"pyproject.toml": DALS(EXAMPLE_PYPROJECT)}), 442 ( 443 "src", 444 { 445 "setup.cfg": DALS(EXAMPLE_SETUPCFG) + DALS( 446 """ 447 packages = find: 448 package_dir = 449 =src 450 451 [options.packages.find] 452 where = src 453 """ 454 ) 455 } 456 ), 457 ( 458 "src", 459 { 460 "pyproject.toml": DALS(EXAMPLE_PYPROJECT) + DALS( 461 """ 462 [tool.setuptools] 463 package-dir = {"" = "src"} 464 """ 465 ) 466 }, 467 ), 468 ] 469 ) 470 def test_include_package_data(self, tmp_path, src_root, files): 471 """ 472 Make sure auto-discovery does not affect package include_package_data. 473 See issue #3196. 474 """ 475 jaraco.path.build(files, prefix=str(tmp_path)) 476 self._simulate_package_with_data_files(tmp_path, src_root) 477 478 expected = { 479 os.path.normpath(f"{src_root}/proj/file1.txt").replace(os.sep, "/"), 480 os.path.normpath(f"{src_root}/proj/nested/file2.txt").replace(os.sep, "/"), 481 } 482 483 _run_build(tmp_path) 484 485 sdist_files = get_sdist_members(next(tmp_path.glob("dist/*.tar.gz"))) 486 print("~~~~~ sdist_members ~~~~~") 487 print('\n'.join(sdist_files)) 488 assert sdist_files >= expected 489 490 wheel_files = get_wheel_members(next(tmp_path.glob("dist/*.whl"))) 491 print("~~~~~ wheel_members ~~~~~") 492 print('\n'.join(wheel_files)) 493 orig_files = {f.replace("src/", "").replace("lib/", "") for f in expected} 494 assert wheel_files >= orig_files 495 496 497def test_compatible_with_numpy_configuration(tmp_path): 498 files = [ 499 "dir1/__init__.py", 500 "dir2/__init__.py", 501 "file.py", 502 ] 503 _populate_project_dir(tmp_path, files, {}) 504 dist = Distribution({}) 505 dist.configuration = object() 506 dist.set_defaults() 507 assert dist.py_modules is None 508 assert dist.packages is None 509 510 511def _populate_project_dir(root, files, options): 512 # NOTE: Currently pypa/build will refuse to build the project if no 513 # `pyproject.toml` or `setup.py` is found. So it is impossible to do 514 # completely "config-less" projects. 515 (root / "setup.py").write_text("import setuptools\nsetuptools.setup()") 516 (root / "README.md").write_text("# Example Package") 517 (root / "LICENSE").write_text("Copyright (c) 2018") 518 _write_setupcfg(root, options) 519 paths = (root / f for f in files) 520 for path in paths: 521 path.parent.mkdir(exist_ok=True, parents=True) 522 path.touch() 523 524 525def _write_setupcfg(root, options): 526 if not options: 527 print("~~~~~ **NO** setup.cfg ~~~~~") 528 return 529 setupcfg = ConfigParser() 530 setupcfg.add_section("options") 531 for key, value in options.items(): 532 if key == "packages.find": 533 setupcfg.add_section(f"options.{key}") 534 setupcfg[f"options.{key}"].update(value) 535 elif isinstance(value, list): 536 setupcfg["options"][key] = ", ".join(value) 537 elif isinstance(value, dict): 538 str_value = "\n".join(f"\t{k} = {v}" for k, v in value.items()) 539 setupcfg["options"][key] = "\n" + str_value 540 else: 541 setupcfg["options"][key] = str(value) 542 with open(root / "setup.cfg", "w") as f: 543 setupcfg.write(f) 544 print("~~~~~ setup.cfg ~~~~~") 545 print((root / "setup.cfg").read_text()) 546 547 548def _run_build(path, *flags): 549 cmd = [sys.executable, "-m", "build", "--no-isolation", *flags, str(path)] 550 return run(cmd, env={'DISTUTILS_DEBUG': ''}) 551 552 553def _get_dist(dist_path, attrs): 554 root = "/".join(os.path.split(dist_path)) # POSIX-style 555 556 script = dist_path / 'setup.py' 557 if script.exists(): 558 with _Path(dist_path): 559 dist = distutils.core.run_setup("setup.py", {}, stop_after="init") 560 else: 561 dist = Distribution(attrs) 562 563 dist.src_root = root 564 dist.script_name = "setup.py" 565 with _Path(dist_path): 566 dist.parse_config_files() 567 568 dist.set_defaults() 569 return dist 570 571 572def _run_sdist_programatically(dist_path, attrs): 573 dist = _get_dist(dist_path, attrs) 574 cmd = sdist(dist) 575 cmd.ensure_finalized() 576 assert cmd.distribution.packages or cmd.distribution.py_modules 577 578 with quiet(), _Path(dist_path): 579 cmd.run() 580 581 return dist, cmd 582