1"""Load setuptools configuration from ``setup.cfg`` files""" 2import os 3 4import warnings 5import functools 6from collections import defaultdict 7from functools import partial 8from functools import wraps 9from typing import (TYPE_CHECKING, Callable, Any, Dict, Generic, Iterable, List, 10 Optional, Tuple, TypeVar, Union) 11 12from distutils.errors import DistutilsOptionError, DistutilsFileError 13from setuptools.extern.packaging.version import Version, InvalidVersion 14from setuptools.extern.packaging.specifiers import SpecifierSet 15 16from . import expand 17 18if TYPE_CHECKING: 19 from setuptools.dist import Distribution # noqa 20 from distutils.dist import DistributionMetadata # noqa 21 22_Path = Union[str, os.PathLike] 23SingleCommandOptions = Dict["str", Tuple["str", Any]] 24"""Dict that associate the name of the options of a particular command to a 25tuple. The first element of the tuple indicates the origin of the option value 26(e.g. the name of the configuration file where it was read from), 27while the second element of the tuple is the option value itself 28""" 29AllCommandOptions = Dict["str", SingleCommandOptions] # cmd name => its options 30Target = TypeVar("Target", bound=Union["Distribution", "DistributionMetadata"]) 31 32 33def read_configuration( 34 filepath: _Path, 35 find_others=False, 36 ignore_option_errors=False 37) -> dict: 38 """Read given configuration file and returns options from it as a dict. 39 40 :param str|unicode filepath: Path to configuration file 41 to get options from. 42 43 :param bool find_others: Whether to search for other configuration files 44 which could be on in various places. 45 46 :param bool ignore_option_errors: Whether to silently ignore 47 options, values of which could not be resolved (e.g. due to exceptions 48 in directives such as file:, attr:, etc.). 49 If False exceptions are propagated as expected. 50 51 :rtype: dict 52 """ 53 from setuptools.dist import Distribution 54 55 dist = Distribution() 56 filenames = dist.find_config_files() if find_others else [] 57 handlers = _apply(dist, filepath, filenames, ignore_option_errors) 58 return configuration_to_dict(handlers) 59 60 61def apply_configuration(dist: "Distribution", filepath: _Path) -> "Distribution": 62 """Apply the configuration from a ``setup.cfg`` file into an existing 63 distribution object. 64 """ 65 _apply(dist, filepath) 66 dist._finalize_requires() 67 return dist 68 69 70def _apply( 71 dist: "Distribution", filepath: _Path, 72 other_files: Iterable[_Path] = (), 73 ignore_option_errors: bool = False 74) -> Tuple["ConfigHandler", ...]: 75 """Read configuration from ``filepath`` and applies to the ``dist`` object.""" 76 from setuptools.dist import _Distribution 77 78 filepath = os.path.abspath(filepath) 79 80 if not os.path.isfile(filepath): 81 raise DistutilsFileError('Configuration file %s does not exist.' % filepath) 82 83 current_directory = os.getcwd() 84 os.chdir(os.path.dirname(filepath)) 85 filenames = [*other_files, filepath] 86 87 try: 88 _Distribution.parse_config_files(dist, filenames=filenames) 89 handlers = parse_configuration( 90 dist, dist.command_options, ignore_option_errors=ignore_option_errors 91 ) 92 dist._finalize_license_files() 93 finally: 94 os.chdir(current_directory) 95 96 return handlers 97 98 99def _get_option(target_obj: Target, key: str): 100 """ 101 Given a target object and option key, get that option from 102 the target object, either through a get_{key} method or 103 from an attribute directly. 104 """ 105 getter_name = 'get_{key}'.format(**locals()) 106 by_attribute = functools.partial(getattr, target_obj, key) 107 getter = getattr(target_obj, getter_name, by_attribute) 108 return getter() 109 110 111def configuration_to_dict(handlers: Tuple["ConfigHandler", ...]) -> dict: 112 """Returns configuration data gathered by given handlers as a dict. 113 114 :param list[ConfigHandler] handlers: Handlers list, 115 usually from parse_configuration() 116 117 :rtype: dict 118 """ 119 config_dict: dict = defaultdict(dict) 120 121 for handler in handlers: 122 for option in handler.set_options: 123 value = _get_option(handler.target_obj, option) 124 config_dict[handler.section_prefix][option] = value 125 126 return config_dict 127 128 129def parse_configuration( 130 distribution: "Distribution", 131 command_options: AllCommandOptions, 132 ignore_option_errors=False 133) -> Tuple["ConfigMetadataHandler", "ConfigOptionsHandler"]: 134 """Performs additional parsing of configuration options 135 for a distribution. 136 137 Returns a list of used option handlers. 138 139 :param Distribution distribution: 140 :param dict command_options: 141 :param bool ignore_option_errors: Whether to silently ignore 142 options, values of which could not be resolved (e.g. due to exceptions 143 in directives such as file:, attr:, etc.). 144 If False exceptions are propagated as expected. 145 :rtype: list 146 """ 147 with expand.EnsurePackagesDiscovered(distribution) as ensure_discovered: 148 options = ConfigOptionsHandler( 149 distribution, 150 command_options, 151 ignore_option_errors, 152 ensure_discovered, 153 ) 154 155 options.parse() 156 if not distribution.package_dir: 157 distribution.package_dir = options.package_dir # Filled by `find_packages` 158 159 meta = ConfigMetadataHandler( 160 distribution.metadata, 161 command_options, 162 ignore_option_errors, 163 ensure_discovered, 164 distribution.package_dir, 165 distribution.src_root, 166 ) 167 meta.parse() 168 169 return meta, options 170 171 172class ConfigHandler(Generic[Target]): 173 """Handles metadata supplied in configuration files.""" 174 175 section_prefix: str 176 """Prefix for config sections handled by this handler. 177 Must be provided by class heirs. 178 179 """ 180 181 aliases: Dict[str, str] = {} 182 """Options aliases. 183 For compatibility with various packages. E.g.: d2to1 and pbr. 184 Note: `-` in keys is replaced with `_` by config parser. 185 186 """ 187 188 def __init__( 189 self, 190 target_obj: Target, 191 options: AllCommandOptions, 192 ignore_option_errors, 193 ensure_discovered: expand.EnsurePackagesDiscovered, 194 ): 195 sections: AllCommandOptions = {} 196 197 section_prefix = self.section_prefix 198 for section_name, section_options in options.items(): 199 if not section_name.startswith(section_prefix): 200 continue 201 202 section_name = section_name.replace(section_prefix, '').strip('.') 203 sections[section_name] = section_options 204 205 self.ignore_option_errors = ignore_option_errors 206 self.target_obj = target_obj 207 self.sections = sections 208 self.set_options: List[str] = [] 209 self.ensure_discovered = ensure_discovered 210 211 @property 212 def parsers(self): 213 """Metadata item name to parser function mapping.""" 214 raise NotImplementedError( 215 '%s must provide .parsers property' % self.__class__.__name__ 216 ) 217 218 def __setitem__(self, option_name, value): 219 unknown = tuple() 220 target_obj = self.target_obj 221 222 # Translate alias into real name. 223 option_name = self.aliases.get(option_name, option_name) 224 225 current_value = getattr(target_obj, option_name, unknown) 226 227 if current_value is unknown: 228 raise KeyError(option_name) 229 230 if current_value: 231 # Already inhabited. Skipping. 232 return 233 234 skip_option = False 235 parser = self.parsers.get(option_name) 236 if parser: 237 try: 238 value = parser(value) 239 240 except Exception: 241 skip_option = True 242 if not self.ignore_option_errors: 243 raise 244 245 if skip_option: 246 return 247 248 setter = getattr(target_obj, 'set_%s' % option_name, None) 249 if setter is None: 250 setattr(target_obj, option_name, value) 251 else: 252 setter(value) 253 254 self.set_options.append(option_name) 255 256 @classmethod 257 def _parse_list(cls, value, separator=','): 258 """Represents value as a list. 259 260 Value is split either by separator (defaults to comma) or by lines. 261 262 :param value: 263 :param separator: List items separator character. 264 :rtype: list 265 """ 266 if isinstance(value, list): # _get_parser_compound case 267 return value 268 269 if '\n' in value: 270 value = value.splitlines() 271 else: 272 value = value.split(separator) 273 274 return [chunk.strip() for chunk in value if chunk.strip()] 275 276 @classmethod 277 def _parse_dict(cls, value): 278 """Represents value as a dict. 279 280 :param value: 281 :rtype: dict 282 """ 283 separator = '=' 284 result = {} 285 for line in cls._parse_list(value): 286 key, sep, val = line.partition(separator) 287 if sep != separator: 288 raise DistutilsOptionError( 289 'Unable to parse option value to dict: %s' % value 290 ) 291 result[key.strip()] = val.strip() 292 293 return result 294 295 @classmethod 296 def _parse_bool(cls, value): 297 """Represents value as boolean. 298 299 :param value: 300 :rtype: bool 301 """ 302 value = value.lower() 303 return value in ('1', 'true', 'yes') 304 305 @classmethod 306 def _exclude_files_parser(cls, key): 307 """Returns a parser function to make sure field inputs 308 are not files. 309 310 Parses a value after getting the key so error messages are 311 more informative. 312 313 :param key: 314 :rtype: callable 315 """ 316 317 def parser(value): 318 exclude_directive = 'file:' 319 if value.startswith(exclude_directive): 320 raise ValueError( 321 'Only strings are accepted for the {0} field, ' 322 'files are not accepted'.format(key) 323 ) 324 return value 325 326 return parser 327 328 @classmethod 329 def _parse_file(cls, value, root_dir: _Path): 330 """Represents value as a string, allowing including text 331 from nearest files using `file:` directive. 332 333 Directive is sandboxed and won't reach anything outside 334 directory with setup.py. 335 336 Examples: 337 file: README.rst, CHANGELOG.md, src/file.txt 338 339 :param str value: 340 :rtype: str 341 """ 342 include_directive = 'file:' 343 344 if not isinstance(value, str): 345 return value 346 347 if not value.startswith(include_directive): 348 return value 349 350 spec = value[len(include_directive) :] 351 filepaths = (path.strip() for path in spec.split(',')) 352 return expand.read_files(filepaths, root_dir) 353 354 def _parse_attr(self, value, package_dir, root_dir: _Path): 355 """Represents value as a module attribute. 356 357 Examples: 358 attr: package.attr 359 attr: package.module.attr 360 361 :param str value: 362 :rtype: str 363 """ 364 attr_directive = 'attr:' 365 if not value.startswith(attr_directive): 366 return value 367 368 attr_desc = value.replace(attr_directive, '') 369 370 # Make sure package_dir is populated correctly, so `attr:` directives can work 371 package_dir.update(self.ensure_discovered.package_dir) 372 return expand.read_attr(attr_desc, package_dir, root_dir) 373 374 @classmethod 375 def _get_parser_compound(cls, *parse_methods): 376 """Returns parser function to represents value as a list. 377 378 Parses a value applying given methods one after another. 379 380 :param parse_methods: 381 :rtype: callable 382 """ 383 384 def parse(value): 385 parsed = value 386 387 for method in parse_methods: 388 parsed = method(parsed) 389 390 return parsed 391 392 return parse 393 394 @classmethod 395 def _parse_section_to_dict(cls, section_options, values_parser=None): 396 """Parses section options into a dictionary. 397 398 Optionally applies a given parser to values. 399 400 :param dict section_options: 401 :param callable values_parser: 402 :rtype: dict 403 """ 404 value = {} 405 values_parser = values_parser or (lambda val: val) 406 for key, (_, val) in section_options.items(): 407 value[key] = values_parser(val) 408 return value 409 410 def parse_section(self, section_options): 411 """Parses configuration file section. 412 413 :param dict section_options: 414 """ 415 for (name, (_, value)) in section_options.items(): 416 try: 417 self[name] = value 418 419 except KeyError: 420 pass # Keep silent for a new option may appear anytime. 421 422 def parse(self): 423 """Parses configuration file items from one 424 or more related sections. 425 426 """ 427 for section_name, section_options in self.sections.items(): 428 429 method_postfix = '' 430 if section_name: # [section.option] variant 431 method_postfix = '_%s' % section_name 432 433 section_parser_method: Optional[Callable] = getattr( 434 self, 435 # Dots in section names are translated into dunderscores. 436 ('parse_section%s' % method_postfix).replace('.', '__'), 437 None, 438 ) 439 440 if section_parser_method is None: 441 raise DistutilsOptionError( 442 'Unsupported distribution option section: [%s.%s]' 443 % (self.section_prefix, section_name) 444 ) 445 446 section_parser_method(section_options) 447 448 def _deprecated_config_handler(self, func, msg, warning_class): 449 """this function will wrap around parameters that are deprecated 450 451 :param msg: deprecation message 452 :param warning_class: class of warning exception to be raised 453 :param func: function to be wrapped around 454 """ 455 456 @wraps(func) 457 def config_handler(*args, **kwargs): 458 warnings.warn(msg, warning_class) 459 return func(*args, **kwargs) 460 461 return config_handler 462 463 464class ConfigMetadataHandler(ConfigHandler["DistributionMetadata"]): 465 466 section_prefix = 'metadata' 467 468 aliases = { 469 'home_page': 'url', 470 'summary': 'description', 471 'classifier': 'classifiers', 472 'platform': 'platforms', 473 } 474 475 strict_mode = False 476 """We need to keep it loose, to be partially compatible with 477 `pbr` and `d2to1` packages which also uses `metadata` section. 478 479 """ 480 481 def __init__( 482 self, 483 target_obj: "DistributionMetadata", 484 options: AllCommandOptions, 485 ignore_option_errors: bool, 486 ensure_discovered: expand.EnsurePackagesDiscovered, 487 package_dir: Optional[dict] = None, 488 root_dir: _Path = os.curdir 489 ): 490 super().__init__(target_obj, options, ignore_option_errors, ensure_discovered) 491 self.package_dir = package_dir 492 self.root_dir = root_dir 493 494 @property 495 def parsers(self): 496 """Metadata item name to parser function mapping.""" 497 parse_list = self._parse_list 498 parse_file = partial(self._parse_file, root_dir=self.root_dir) 499 parse_dict = self._parse_dict 500 exclude_files_parser = self._exclude_files_parser 501 502 return { 503 'platforms': parse_list, 504 'keywords': parse_list, 505 'provides': parse_list, 506 'requires': self._deprecated_config_handler( 507 parse_list, 508 "The requires parameter is deprecated, please use " 509 "install_requires for runtime dependencies.", 510 DeprecationWarning, 511 ), 512 'obsoletes': parse_list, 513 'classifiers': self._get_parser_compound(parse_file, parse_list), 514 'license': exclude_files_parser('license'), 515 'license_file': self._deprecated_config_handler( 516 exclude_files_parser('license_file'), 517 "The license_file parameter is deprecated, " 518 "use license_files instead.", 519 DeprecationWarning, 520 ), 521 'license_files': parse_list, 522 'description': parse_file, 523 'long_description': parse_file, 524 'version': self._parse_version, 525 'project_urls': parse_dict, 526 } 527 528 def _parse_version(self, value): 529 """Parses `version` option value. 530 531 :param value: 532 :rtype: str 533 534 """ 535 version = self._parse_file(value, self.root_dir) 536 537 if version != value: 538 version = version.strip() 539 # Be strict about versions loaded from file because it's easy to 540 # accidentally include newlines and other unintended content 541 try: 542 Version(version) 543 except InvalidVersion: 544 tmpl = ( 545 'Version loaded from {value} does not ' 546 'comply with PEP 440: {version}' 547 ) 548 raise DistutilsOptionError(tmpl.format(**locals())) 549 550 return version 551 552 return expand.version(self._parse_attr(value, self.package_dir, self.root_dir)) 553 554 555class ConfigOptionsHandler(ConfigHandler["Distribution"]): 556 557 section_prefix = 'options' 558 559 def __init__( 560 self, 561 target_obj: "Distribution", 562 options: AllCommandOptions, 563 ignore_option_errors: bool, 564 ensure_discovered: expand.EnsurePackagesDiscovered, 565 ): 566 super().__init__(target_obj, options, ignore_option_errors, ensure_discovered) 567 self.root_dir = target_obj.src_root 568 self.package_dir: Dict[str, str] = {} # To be filled by `find_packages` 569 570 @property 571 def parsers(self): 572 """Metadata item name to parser function mapping.""" 573 parse_list = self._parse_list 574 parse_list_semicolon = partial(self._parse_list, separator=';') 575 parse_bool = self._parse_bool 576 parse_dict = self._parse_dict 577 parse_cmdclass = self._parse_cmdclass 578 parse_file = partial(self._parse_file, root_dir=self.root_dir) 579 580 return { 581 'zip_safe': parse_bool, 582 'include_package_data': parse_bool, 583 'package_dir': parse_dict, 584 'scripts': parse_list, 585 'eager_resources': parse_list, 586 'dependency_links': parse_list, 587 'namespace_packages': parse_list, 588 'install_requires': parse_list_semicolon, 589 'setup_requires': parse_list_semicolon, 590 'tests_require': parse_list_semicolon, 591 'packages': self._parse_packages, 592 'entry_points': parse_file, 593 'py_modules': parse_list, 594 'python_requires': SpecifierSet, 595 'cmdclass': parse_cmdclass, 596 } 597 598 def _parse_cmdclass(self, value): 599 package_dir = self.ensure_discovered.package_dir 600 return expand.cmdclass(self._parse_dict(value), package_dir, self.root_dir) 601 602 def _parse_packages(self, value): 603 """Parses `packages` option value. 604 605 :param value: 606 :rtype: list 607 """ 608 find_directives = ['find:', 'find_namespace:'] 609 trimmed_value = value.strip() 610 611 if trimmed_value not in find_directives: 612 return self._parse_list(value) 613 614 # Read function arguments from a dedicated section. 615 find_kwargs = self.parse_section_packages__find( 616 self.sections.get('packages.find', {}) 617 ) 618 619 find_kwargs.update( 620 namespaces=(trimmed_value == find_directives[1]), 621 root_dir=self.root_dir, 622 fill_package_dir=self.package_dir, 623 ) 624 625 return expand.find_packages(**find_kwargs) 626 627 def parse_section_packages__find(self, section_options): 628 """Parses `packages.find` configuration file section. 629 630 To be used in conjunction with _parse_packages(). 631 632 :param dict section_options: 633 """ 634 section_data = self._parse_section_to_dict(section_options, self._parse_list) 635 636 valid_keys = ['where', 'include', 'exclude'] 637 638 find_kwargs = dict( 639 [(k, v) for k, v in section_data.items() if k in valid_keys and v] 640 ) 641 642 where = find_kwargs.get('where') 643 if where is not None: 644 find_kwargs['where'] = where[0] # cast list to single val 645 646 return find_kwargs 647 648 def parse_section_entry_points(self, section_options): 649 """Parses `entry_points` configuration file section. 650 651 :param dict section_options: 652 """ 653 parsed = self._parse_section_to_dict(section_options, self._parse_list) 654 self['entry_points'] = parsed 655 656 def _parse_package_data(self, section_options): 657 package_data = self._parse_section_to_dict(section_options, self._parse_list) 658 return expand.canonic_package_data(package_data) 659 660 def parse_section_package_data(self, section_options): 661 """Parses `package_data` configuration file section. 662 663 :param dict section_options: 664 """ 665 self['package_data'] = self._parse_package_data(section_options) 666 667 def parse_section_exclude_package_data(self, section_options): 668 """Parses `exclude_package_data` configuration file section. 669 670 :param dict section_options: 671 """ 672 self['exclude_package_data'] = self._parse_package_data(section_options) 673 674 def parse_section_extras_require(self, section_options): 675 """Parses `extras_require` configuration file section. 676 677 :param dict section_options: 678 """ 679 parse_list = partial(self._parse_list, separator=';') 680 self['extras_require'] = self._parse_section_to_dict( 681 section_options, parse_list 682 ) 683 684 def parse_section_data_files(self, section_options): 685 """Parses `data_files` configuration file section. 686 687 :param dict section_options: 688 """ 689 parsed = self._parse_section_to_dict(section_options, self._parse_list) 690 self['data_files'] = expand.canonic_data_files(parsed, self.root_dir) 691