• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1"""Load setuptools configuration from ``setup.cfg`` files"""
2import os
3
4import warnings
5import functools
6from collections import defaultdict
7from functools import partial
8from functools import wraps
9from typing import (TYPE_CHECKING, Callable, Any, Dict, Generic, Iterable, List,
10                    Optional, Tuple, TypeVar, Union)
11
12from distutils.errors import DistutilsOptionError, DistutilsFileError
13from setuptools.extern.packaging.version import Version, InvalidVersion
14from setuptools.extern.packaging.specifiers import SpecifierSet
15
16from . import expand
17
18if TYPE_CHECKING:
19    from setuptools.dist import Distribution  # noqa
20    from distutils.dist import DistributionMetadata  # noqa
21
22_Path = Union[str, os.PathLike]
23SingleCommandOptions = Dict["str", Tuple["str", Any]]
24"""Dict that associate the name of the options of a particular command to a
25tuple. The first element of the tuple indicates the origin of the option value
26(e.g. the name of the configuration file where it was read from),
27while the second element of the tuple is the option value itself
28"""
29AllCommandOptions = Dict["str", SingleCommandOptions]  # cmd name => its options
30Target = TypeVar("Target", bound=Union["Distribution", "DistributionMetadata"])
31
32
33def read_configuration(
34    filepath: _Path,
35    find_others=False,
36    ignore_option_errors=False
37) -> dict:
38    """Read given configuration file and returns options from it as a dict.
39
40    :param str|unicode filepath: Path to configuration file
41        to get options from.
42
43    :param bool find_others: Whether to search for other configuration files
44        which could be on in various places.
45
46    :param bool ignore_option_errors: Whether to silently ignore
47        options, values of which could not be resolved (e.g. due to exceptions
48        in directives such as file:, attr:, etc.).
49        If False exceptions are propagated as expected.
50
51    :rtype: dict
52    """
53    from setuptools.dist import Distribution
54
55    dist = Distribution()
56    filenames = dist.find_config_files() if find_others else []
57    handlers = _apply(dist, filepath, filenames, ignore_option_errors)
58    return configuration_to_dict(handlers)
59
60
61def apply_configuration(dist: "Distribution", filepath: _Path) -> "Distribution":
62    """Apply the configuration from a ``setup.cfg`` file into an existing
63    distribution object.
64    """
65    _apply(dist, filepath)
66    dist._finalize_requires()
67    return dist
68
69
70def _apply(
71    dist: "Distribution", filepath: _Path,
72    other_files: Iterable[_Path] = (),
73    ignore_option_errors: bool = False
74) -> Tuple["ConfigHandler", ...]:
75    """Read configuration from ``filepath`` and applies to the ``dist`` object."""
76    from setuptools.dist import _Distribution
77
78    filepath = os.path.abspath(filepath)
79
80    if not os.path.isfile(filepath):
81        raise DistutilsFileError('Configuration file %s does not exist.' % filepath)
82
83    current_directory = os.getcwd()
84    os.chdir(os.path.dirname(filepath))
85    filenames = [*other_files, filepath]
86
87    try:
88        _Distribution.parse_config_files(dist, filenames=filenames)
89        handlers = parse_configuration(
90            dist, dist.command_options, ignore_option_errors=ignore_option_errors
91        )
92        dist._finalize_license_files()
93    finally:
94        os.chdir(current_directory)
95
96    return handlers
97
98
99def _get_option(target_obj: Target, key: str):
100    """
101    Given a target object and option key, get that option from
102    the target object, either through a get_{key} method or
103    from an attribute directly.
104    """
105    getter_name = 'get_{key}'.format(**locals())
106    by_attribute = functools.partial(getattr, target_obj, key)
107    getter = getattr(target_obj, getter_name, by_attribute)
108    return getter()
109
110
111def configuration_to_dict(handlers: Tuple["ConfigHandler", ...]) -> dict:
112    """Returns configuration data gathered by given handlers as a dict.
113
114    :param list[ConfigHandler] handlers: Handlers list,
115        usually from parse_configuration()
116
117    :rtype: dict
118    """
119    config_dict: dict = defaultdict(dict)
120
121    for handler in handlers:
122        for option in handler.set_options:
123            value = _get_option(handler.target_obj, option)
124            config_dict[handler.section_prefix][option] = value
125
126    return config_dict
127
128
129def parse_configuration(
130    distribution: "Distribution",
131    command_options: AllCommandOptions,
132    ignore_option_errors=False
133) -> Tuple["ConfigMetadataHandler", "ConfigOptionsHandler"]:
134    """Performs additional parsing of configuration options
135    for a distribution.
136
137    Returns a list of used option handlers.
138
139    :param Distribution distribution:
140    :param dict command_options:
141    :param bool ignore_option_errors: Whether to silently ignore
142        options, values of which could not be resolved (e.g. due to exceptions
143        in directives such as file:, attr:, etc.).
144        If False exceptions are propagated as expected.
145    :rtype: list
146    """
147    with expand.EnsurePackagesDiscovered(distribution) as ensure_discovered:
148        options = ConfigOptionsHandler(
149            distribution,
150            command_options,
151            ignore_option_errors,
152            ensure_discovered,
153        )
154
155        options.parse()
156        if not distribution.package_dir:
157            distribution.package_dir = options.package_dir  # Filled by `find_packages`
158
159        meta = ConfigMetadataHandler(
160            distribution.metadata,
161            command_options,
162            ignore_option_errors,
163            ensure_discovered,
164            distribution.package_dir,
165            distribution.src_root,
166        )
167        meta.parse()
168
169    return meta, options
170
171
172class ConfigHandler(Generic[Target]):
173    """Handles metadata supplied in configuration files."""
174
175    section_prefix: str
176    """Prefix for config sections handled by this handler.
177    Must be provided by class heirs.
178
179    """
180
181    aliases: Dict[str, str] = {}
182    """Options aliases.
183    For compatibility with various packages. E.g.: d2to1 and pbr.
184    Note: `-` in keys is replaced with `_` by config parser.
185
186    """
187
188    def __init__(
189        self,
190        target_obj: Target,
191        options: AllCommandOptions,
192        ignore_option_errors,
193        ensure_discovered: expand.EnsurePackagesDiscovered,
194    ):
195        sections: AllCommandOptions = {}
196
197        section_prefix = self.section_prefix
198        for section_name, section_options in options.items():
199            if not section_name.startswith(section_prefix):
200                continue
201
202            section_name = section_name.replace(section_prefix, '').strip('.')
203            sections[section_name] = section_options
204
205        self.ignore_option_errors = ignore_option_errors
206        self.target_obj = target_obj
207        self.sections = sections
208        self.set_options: List[str] = []
209        self.ensure_discovered = ensure_discovered
210
211    @property
212    def parsers(self):
213        """Metadata item name to parser function mapping."""
214        raise NotImplementedError(
215            '%s must provide .parsers property' % self.__class__.__name__
216        )
217
218    def __setitem__(self, option_name, value):
219        unknown = tuple()
220        target_obj = self.target_obj
221
222        # Translate alias into real name.
223        option_name = self.aliases.get(option_name, option_name)
224
225        current_value = getattr(target_obj, option_name, unknown)
226
227        if current_value is unknown:
228            raise KeyError(option_name)
229
230        if current_value:
231            # Already inhabited. Skipping.
232            return
233
234        skip_option = False
235        parser = self.parsers.get(option_name)
236        if parser:
237            try:
238                value = parser(value)
239
240            except Exception:
241                skip_option = True
242                if not self.ignore_option_errors:
243                    raise
244
245        if skip_option:
246            return
247
248        setter = getattr(target_obj, 'set_%s' % option_name, None)
249        if setter is None:
250            setattr(target_obj, option_name, value)
251        else:
252            setter(value)
253
254        self.set_options.append(option_name)
255
256    @classmethod
257    def _parse_list(cls, value, separator=','):
258        """Represents value as a list.
259
260        Value is split either by separator (defaults to comma) or by lines.
261
262        :param value:
263        :param separator: List items separator character.
264        :rtype: list
265        """
266        if isinstance(value, list):  # _get_parser_compound case
267            return value
268
269        if '\n' in value:
270            value = value.splitlines()
271        else:
272            value = value.split(separator)
273
274        return [chunk.strip() for chunk in value if chunk.strip()]
275
276    @classmethod
277    def _parse_dict(cls, value):
278        """Represents value as a dict.
279
280        :param value:
281        :rtype: dict
282        """
283        separator = '='
284        result = {}
285        for line in cls._parse_list(value):
286            key, sep, val = line.partition(separator)
287            if sep != separator:
288                raise DistutilsOptionError(
289                    'Unable to parse option value to dict: %s' % value
290                )
291            result[key.strip()] = val.strip()
292
293        return result
294
295    @classmethod
296    def _parse_bool(cls, value):
297        """Represents value as boolean.
298
299        :param value:
300        :rtype: bool
301        """
302        value = value.lower()
303        return value in ('1', 'true', 'yes')
304
305    @classmethod
306    def _exclude_files_parser(cls, key):
307        """Returns a parser function to make sure field inputs
308        are not files.
309
310        Parses a value after getting the key so error messages are
311        more informative.
312
313        :param key:
314        :rtype: callable
315        """
316
317        def parser(value):
318            exclude_directive = 'file:'
319            if value.startswith(exclude_directive):
320                raise ValueError(
321                    'Only strings are accepted for the {0} field, '
322                    'files are not accepted'.format(key)
323                )
324            return value
325
326        return parser
327
328    @classmethod
329    def _parse_file(cls, value, root_dir: _Path):
330        """Represents value as a string, allowing including text
331        from nearest files using `file:` directive.
332
333        Directive is sandboxed and won't reach anything outside
334        directory with setup.py.
335
336        Examples:
337            file: README.rst, CHANGELOG.md, src/file.txt
338
339        :param str value:
340        :rtype: str
341        """
342        include_directive = 'file:'
343
344        if not isinstance(value, str):
345            return value
346
347        if not value.startswith(include_directive):
348            return value
349
350        spec = value[len(include_directive) :]
351        filepaths = (path.strip() for path in spec.split(','))
352        return expand.read_files(filepaths, root_dir)
353
354    def _parse_attr(self, value, package_dir, root_dir: _Path):
355        """Represents value as a module attribute.
356
357        Examples:
358            attr: package.attr
359            attr: package.module.attr
360
361        :param str value:
362        :rtype: str
363        """
364        attr_directive = 'attr:'
365        if not value.startswith(attr_directive):
366            return value
367
368        attr_desc = value.replace(attr_directive, '')
369
370        # Make sure package_dir is populated correctly, so `attr:` directives can work
371        package_dir.update(self.ensure_discovered.package_dir)
372        return expand.read_attr(attr_desc, package_dir, root_dir)
373
374    @classmethod
375    def _get_parser_compound(cls, *parse_methods):
376        """Returns parser function to represents value as a list.
377
378        Parses a value applying given methods one after another.
379
380        :param parse_methods:
381        :rtype: callable
382        """
383
384        def parse(value):
385            parsed = value
386
387            for method in parse_methods:
388                parsed = method(parsed)
389
390            return parsed
391
392        return parse
393
394    @classmethod
395    def _parse_section_to_dict(cls, section_options, values_parser=None):
396        """Parses section options into a dictionary.
397
398        Optionally applies a given parser to values.
399
400        :param dict section_options:
401        :param callable values_parser:
402        :rtype: dict
403        """
404        value = {}
405        values_parser = values_parser or (lambda val: val)
406        for key, (_, val) in section_options.items():
407            value[key] = values_parser(val)
408        return value
409
410    def parse_section(self, section_options):
411        """Parses configuration file section.
412
413        :param dict section_options:
414        """
415        for (name, (_, value)) in section_options.items():
416            try:
417                self[name] = value
418
419            except KeyError:
420                pass  # Keep silent for a new option may appear anytime.
421
422    def parse(self):
423        """Parses configuration file items from one
424        or more related sections.
425
426        """
427        for section_name, section_options in self.sections.items():
428
429            method_postfix = ''
430            if section_name:  # [section.option] variant
431                method_postfix = '_%s' % section_name
432
433            section_parser_method: Optional[Callable] = getattr(
434                self,
435                # Dots in section names are translated into dunderscores.
436                ('parse_section%s' % method_postfix).replace('.', '__'),
437                None,
438            )
439
440            if section_parser_method is None:
441                raise DistutilsOptionError(
442                    'Unsupported distribution option section: [%s.%s]'
443                    % (self.section_prefix, section_name)
444                )
445
446            section_parser_method(section_options)
447
448    def _deprecated_config_handler(self, func, msg, warning_class):
449        """this function will wrap around parameters that are deprecated
450
451        :param msg: deprecation message
452        :param warning_class: class of warning exception to be raised
453        :param func: function to be wrapped around
454        """
455
456        @wraps(func)
457        def config_handler(*args, **kwargs):
458            warnings.warn(msg, warning_class)
459            return func(*args, **kwargs)
460
461        return config_handler
462
463
464class ConfigMetadataHandler(ConfigHandler["DistributionMetadata"]):
465
466    section_prefix = 'metadata'
467
468    aliases = {
469        'home_page': 'url',
470        'summary': 'description',
471        'classifier': 'classifiers',
472        'platform': 'platforms',
473    }
474
475    strict_mode = False
476    """We need to keep it loose, to be partially compatible with
477    `pbr` and `d2to1` packages which also uses `metadata` section.
478
479    """
480
481    def __init__(
482        self,
483        target_obj: "DistributionMetadata",
484        options: AllCommandOptions,
485        ignore_option_errors: bool,
486        ensure_discovered: expand.EnsurePackagesDiscovered,
487        package_dir: Optional[dict] = None,
488        root_dir: _Path = os.curdir
489    ):
490        super().__init__(target_obj, options, ignore_option_errors, ensure_discovered)
491        self.package_dir = package_dir
492        self.root_dir = root_dir
493
494    @property
495    def parsers(self):
496        """Metadata item name to parser function mapping."""
497        parse_list = self._parse_list
498        parse_file = partial(self._parse_file, root_dir=self.root_dir)
499        parse_dict = self._parse_dict
500        exclude_files_parser = self._exclude_files_parser
501
502        return {
503            'platforms': parse_list,
504            'keywords': parse_list,
505            'provides': parse_list,
506            'requires': self._deprecated_config_handler(
507                parse_list,
508                "The requires parameter is deprecated, please use "
509                "install_requires for runtime dependencies.",
510                DeprecationWarning,
511            ),
512            'obsoletes': parse_list,
513            'classifiers': self._get_parser_compound(parse_file, parse_list),
514            'license': exclude_files_parser('license'),
515            'license_file': self._deprecated_config_handler(
516                exclude_files_parser('license_file'),
517                "The license_file parameter is deprecated, "
518                "use license_files instead.",
519                DeprecationWarning,
520            ),
521            'license_files': parse_list,
522            'description': parse_file,
523            'long_description': parse_file,
524            'version': self._parse_version,
525            'project_urls': parse_dict,
526        }
527
528    def _parse_version(self, value):
529        """Parses `version` option value.
530
531        :param value:
532        :rtype: str
533
534        """
535        version = self._parse_file(value, self.root_dir)
536
537        if version != value:
538            version = version.strip()
539            # Be strict about versions loaded from file because it's easy to
540            # accidentally include newlines and other unintended content
541            try:
542                Version(version)
543            except InvalidVersion:
544                tmpl = (
545                    'Version loaded from {value} does not '
546                    'comply with PEP 440: {version}'
547                )
548                raise DistutilsOptionError(tmpl.format(**locals()))
549
550            return version
551
552        return expand.version(self._parse_attr(value, self.package_dir, self.root_dir))
553
554
555class ConfigOptionsHandler(ConfigHandler["Distribution"]):
556
557    section_prefix = 'options'
558
559    def __init__(
560        self,
561        target_obj: "Distribution",
562        options: AllCommandOptions,
563        ignore_option_errors: bool,
564        ensure_discovered: expand.EnsurePackagesDiscovered,
565    ):
566        super().__init__(target_obj, options, ignore_option_errors, ensure_discovered)
567        self.root_dir = target_obj.src_root
568        self.package_dir: Dict[str, str] = {}  # To be filled by `find_packages`
569
570    @property
571    def parsers(self):
572        """Metadata item name to parser function mapping."""
573        parse_list = self._parse_list
574        parse_list_semicolon = partial(self._parse_list, separator=';')
575        parse_bool = self._parse_bool
576        parse_dict = self._parse_dict
577        parse_cmdclass = self._parse_cmdclass
578        parse_file = partial(self._parse_file, root_dir=self.root_dir)
579
580        return {
581            'zip_safe': parse_bool,
582            'include_package_data': parse_bool,
583            'package_dir': parse_dict,
584            'scripts': parse_list,
585            'eager_resources': parse_list,
586            'dependency_links': parse_list,
587            'namespace_packages': parse_list,
588            'install_requires': parse_list_semicolon,
589            'setup_requires': parse_list_semicolon,
590            'tests_require': parse_list_semicolon,
591            'packages': self._parse_packages,
592            'entry_points': parse_file,
593            'py_modules': parse_list,
594            'python_requires': SpecifierSet,
595            'cmdclass': parse_cmdclass,
596        }
597
598    def _parse_cmdclass(self, value):
599        package_dir = self.ensure_discovered.package_dir
600        return expand.cmdclass(self._parse_dict(value), package_dir, self.root_dir)
601
602    def _parse_packages(self, value):
603        """Parses `packages` option value.
604
605        :param value:
606        :rtype: list
607        """
608        find_directives = ['find:', 'find_namespace:']
609        trimmed_value = value.strip()
610
611        if trimmed_value not in find_directives:
612            return self._parse_list(value)
613
614        # Read function arguments from a dedicated section.
615        find_kwargs = self.parse_section_packages__find(
616            self.sections.get('packages.find', {})
617        )
618
619        find_kwargs.update(
620            namespaces=(trimmed_value == find_directives[1]),
621            root_dir=self.root_dir,
622            fill_package_dir=self.package_dir,
623        )
624
625        return expand.find_packages(**find_kwargs)
626
627    def parse_section_packages__find(self, section_options):
628        """Parses `packages.find` configuration file section.
629
630        To be used in conjunction with _parse_packages().
631
632        :param dict section_options:
633        """
634        section_data = self._parse_section_to_dict(section_options, self._parse_list)
635
636        valid_keys = ['where', 'include', 'exclude']
637
638        find_kwargs = dict(
639            [(k, v) for k, v in section_data.items() if k in valid_keys and v]
640        )
641
642        where = find_kwargs.get('where')
643        if where is not None:
644            find_kwargs['where'] = where[0]  # cast list to single val
645
646        return find_kwargs
647
648    def parse_section_entry_points(self, section_options):
649        """Parses `entry_points` configuration file section.
650
651        :param dict section_options:
652        """
653        parsed = self._parse_section_to_dict(section_options, self._parse_list)
654        self['entry_points'] = parsed
655
656    def _parse_package_data(self, section_options):
657        package_data = self._parse_section_to_dict(section_options, self._parse_list)
658        return expand.canonic_package_data(package_data)
659
660    def parse_section_package_data(self, section_options):
661        """Parses `package_data` configuration file section.
662
663        :param dict section_options:
664        """
665        self['package_data'] = self._parse_package_data(section_options)
666
667    def parse_section_exclude_package_data(self, section_options):
668        """Parses `exclude_package_data` configuration file section.
669
670        :param dict section_options:
671        """
672        self['exclude_package_data'] = self._parse_package_data(section_options)
673
674    def parse_section_extras_require(self, section_options):
675        """Parses `extras_require` configuration file section.
676
677        :param dict section_options:
678        """
679        parse_list = partial(self._parse_list, separator=';')
680        self['extras_require'] = self._parse_section_to_dict(
681            section_options, parse_list
682        )
683
684    def parse_section_data_files(self, section_options):
685        """Parses `data_files` configuration file section.
686
687        :param dict section_options:
688        """
689        parsed = self._parse_section_to_dict(section_options, self._parse_list)
690        self['data_files'] = expand.canonic_data_files(parsed, self.root_dir)
691