1from __future__ import absolute_import, unicode_literals 2import io 3import os 4import sys 5from collections import defaultdict 6from functools import partial 7from importlib import import_module 8 9from distutils.errors import DistutilsOptionError, DistutilsFileError 10from setuptools.extern.six import string_types 11 12 13def read_configuration( 14 filepath, find_others=False, ignore_option_errors=False): 15 """Read given configuration file and returns options from it as a dict. 16 17 :param str|unicode filepath: Path to configuration file 18 to get options from. 19 20 :param bool find_others: Whether to search for other configuration files 21 which could be on in various places. 22 23 :param bool ignore_option_errors: Whether to silently ignore 24 options, values of which could not be resolved (e.g. due to exceptions 25 in directives such as file:, attr:, etc.). 26 If False exceptions are propagated as expected. 27 28 :rtype: dict 29 """ 30 from setuptools.dist import Distribution, _Distribution 31 32 filepath = os.path.abspath(filepath) 33 34 if not os.path.isfile(filepath): 35 raise DistutilsFileError( 36 'Configuration file %s does not exist.' % filepath) 37 38 current_directory = os.getcwd() 39 os.chdir(os.path.dirname(filepath)) 40 41 try: 42 dist = Distribution() 43 44 filenames = dist.find_config_files() if find_others else [] 45 if filepath not in filenames: 46 filenames.append(filepath) 47 48 _Distribution.parse_config_files(dist, filenames=filenames) 49 50 handlers = parse_configuration( 51 dist, dist.command_options, 52 ignore_option_errors=ignore_option_errors) 53 54 finally: 55 os.chdir(current_directory) 56 57 return configuration_to_dict(handlers) 58 59 60def configuration_to_dict(handlers): 61 """Returns configuration data gathered by given handlers as a dict. 62 63 :param list[ConfigHandler] handlers: Handlers list, 64 usually from parse_configuration() 65 66 :rtype: dict 67 """ 68 config_dict = defaultdict(dict) 69 70 for handler in handlers: 71 72 obj_alias = handler.section_prefix 73 target_obj = handler.target_obj 74 75 for option in handler.set_options: 76 getter = getattr(target_obj, 'get_%s' % option, None) 77 78 if getter is None: 79 value = getattr(target_obj, option) 80 81 else: 82 value = getter() 83 84 config_dict[obj_alias][option] = value 85 86 return config_dict 87 88 89def parse_configuration( 90 distribution, command_options, ignore_option_errors=False): 91 """Performs additional parsing of configuration options 92 for a distribution. 93 94 Returns a list of used option handlers. 95 96 :param Distribution distribution: 97 :param dict command_options: 98 :param bool ignore_option_errors: Whether to silently ignore 99 options, values of which could not be resolved (e.g. due to exceptions 100 in directives such as file:, attr:, etc.). 101 If False exceptions are propagated as expected. 102 :rtype: list 103 """ 104 meta = ConfigMetadataHandler( 105 distribution.metadata, command_options, ignore_option_errors) 106 meta.parse() 107 108 options = ConfigOptionsHandler( 109 distribution, command_options, ignore_option_errors) 110 options.parse() 111 112 return meta, options 113 114 115class ConfigHandler(object): 116 """Handles metadata supplied in configuration files.""" 117 118 section_prefix = None 119 """Prefix for config sections handled by this handler. 120 Must be provided by class heirs. 121 122 """ 123 124 aliases = {} 125 """Options aliases. 126 For compatibility with various packages. E.g.: d2to1 and pbr. 127 Note: `-` in keys is replaced with `_` by config parser. 128 129 """ 130 131 def __init__(self, target_obj, options, ignore_option_errors=False): 132 sections = {} 133 134 section_prefix = self.section_prefix 135 for section_name, section_options in options.items(): 136 if not section_name.startswith(section_prefix): 137 continue 138 139 section_name = section_name.replace(section_prefix, '').strip('.') 140 sections[section_name] = section_options 141 142 self.ignore_option_errors = ignore_option_errors 143 self.target_obj = target_obj 144 self.sections = sections 145 self.set_options = [] 146 147 @property 148 def parsers(self): 149 """Metadata item name to parser function mapping.""" 150 raise NotImplementedError( 151 '%s must provide .parsers property' % self.__class__.__name__) 152 153 def __setitem__(self, option_name, value): 154 unknown = tuple() 155 target_obj = self.target_obj 156 157 # Translate alias into real name. 158 option_name = self.aliases.get(option_name, option_name) 159 160 current_value = getattr(target_obj, option_name, unknown) 161 162 if current_value is unknown: 163 raise KeyError(option_name) 164 165 if current_value: 166 # Already inhabited. Skipping. 167 return 168 169 skip_option = False 170 parser = self.parsers.get(option_name) 171 if parser: 172 try: 173 value = parser(value) 174 175 except Exception: 176 skip_option = True 177 if not self.ignore_option_errors: 178 raise 179 180 if skip_option: 181 return 182 183 setter = getattr(target_obj, 'set_%s' % option_name, None) 184 if setter is None: 185 setattr(target_obj, option_name, value) 186 else: 187 setter(value) 188 189 self.set_options.append(option_name) 190 191 @classmethod 192 def _parse_list(cls, value, separator=','): 193 """Represents value as a list. 194 195 Value is split either by separator (defaults to comma) or by lines. 196 197 :param value: 198 :param separator: List items separator character. 199 :rtype: list 200 """ 201 if isinstance(value, list): # _get_parser_compound case 202 return value 203 204 if '\n' in value: 205 value = value.splitlines() 206 else: 207 value = value.split(separator) 208 209 return [chunk.strip() for chunk in value if chunk.strip()] 210 211 @classmethod 212 def _parse_dict(cls, value): 213 """Represents value as a dict. 214 215 :param value: 216 :rtype: dict 217 """ 218 separator = '=' 219 result = {} 220 for line in cls._parse_list(value): 221 key, sep, val = line.partition(separator) 222 if sep != separator: 223 raise DistutilsOptionError( 224 'Unable to parse option value to dict: %s' % value) 225 result[key.strip()] = val.strip() 226 227 return result 228 229 @classmethod 230 def _parse_bool(cls, value): 231 """Represents value as boolean. 232 233 :param value: 234 :rtype: bool 235 """ 236 value = value.lower() 237 return value in ('1', 'true', 'yes') 238 239 @classmethod 240 def _parse_file(cls, value): 241 """Represents value as a string, allowing including text 242 from nearest files using `file:` directive. 243 244 Directive is sandboxed and won't reach anything outside 245 directory with setup.py. 246 247 Examples: 248 file: LICENSE 249 file: README.rst, CHANGELOG.md, src/file.txt 250 251 :param str value: 252 :rtype: str 253 """ 254 include_directive = 'file:' 255 256 if not isinstance(value, string_types): 257 return value 258 259 if not value.startswith(include_directive): 260 return value 261 262 spec = value[len(include_directive):] 263 filepaths = (os.path.abspath(path.strip()) for path in spec.split(',')) 264 return '\n'.join( 265 cls._read_file(path) 266 for path in filepaths 267 if (cls._assert_local(path) or True) 268 and os.path.isfile(path) 269 ) 270 271 @staticmethod 272 def _assert_local(filepath): 273 if not filepath.startswith(os.getcwd()): 274 raise DistutilsOptionError( 275 '`file:` directive can not access %s' % filepath) 276 277 @staticmethod 278 def _read_file(filepath): 279 with io.open(filepath, encoding='utf-8') as f: 280 return f.read() 281 282 @classmethod 283 def _parse_attr(cls, value): 284 """Represents value as a module attribute. 285 286 Examples: 287 attr: package.attr 288 attr: package.module.attr 289 290 :param str value: 291 :rtype: str 292 """ 293 attr_directive = 'attr:' 294 if not value.startswith(attr_directive): 295 return value 296 297 attrs_path = value.replace(attr_directive, '').strip().split('.') 298 attr_name = attrs_path.pop() 299 300 module_name = '.'.join(attrs_path) 301 module_name = module_name or '__init__' 302 303 sys.path.insert(0, os.getcwd()) 304 try: 305 module = import_module(module_name) 306 value = getattr(module, attr_name) 307 308 finally: 309 sys.path = sys.path[1:] 310 311 return value 312 313 @classmethod 314 def _get_parser_compound(cls, *parse_methods): 315 """Returns parser function to represents value as a list. 316 317 Parses a value applying given methods one after another. 318 319 :param parse_methods: 320 :rtype: callable 321 """ 322 def parse(value): 323 parsed = value 324 325 for method in parse_methods: 326 parsed = method(parsed) 327 328 return parsed 329 330 return parse 331 332 @classmethod 333 def _parse_section_to_dict(cls, section_options, values_parser=None): 334 """Parses section options into a dictionary. 335 336 Optionally applies a given parser to values. 337 338 :param dict section_options: 339 :param callable values_parser: 340 :rtype: dict 341 """ 342 value = {} 343 values_parser = values_parser or (lambda val: val) 344 for key, (_, val) in section_options.items(): 345 value[key] = values_parser(val) 346 return value 347 348 def parse_section(self, section_options): 349 """Parses configuration file section. 350 351 :param dict section_options: 352 """ 353 for (name, (_, value)) in section_options.items(): 354 try: 355 self[name] = value 356 357 except KeyError: 358 pass # Keep silent for a new option may appear anytime. 359 360 def parse(self): 361 """Parses configuration file items from one 362 or more related sections. 363 364 """ 365 for section_name, section_options in self.sections.items(): 366 367 method_postfix = '' 368 if section_name: # [section.option] variant 369 method_postfix = '_%s' % section_name 370 371 section_parser_method = getattr( 372 self, 373 # Dots in section names are tranlsated into dunderscores. 374 ('parse_section%s' % method_postfix).replace('.', '__'), 375 None) 376 377 if section_parser_method is None: 378 raise DistutilsOptionError( 379 'Unsupported distribution option section: [%s.%s]' % ( 380 self.section_prefix, section_name)) 381 382 section_parser_method(section_options) 383 384 385class ConfigMetadataHandler(ConfigHandler): 386 387 section_prefix = 'metadata' 388 389 aliases = { 390 'home_page': 'url', 391 'summary': 'description', 392 'classifier': 'classifiers', 393 'platform': 'platforms', 394 } 395 396 strict_mode = False 397 """We need to keep it loose, to be partially compatible with 398 `pbr` and `d2to1` packages which also uses `metadata` section. 399 400 """ 401 402 @property 403 def parsers(self): 404 """Metadata item name to parser function mapping.""" 405 parse_list = self._parse_list 406 parse_file = self._parse_file 407 parse_dict = self._parse_dict 408 409 return { 410 'platforms': parse_list, 411 'keywords': parse_list, 412 'provides': parse_list, 413 'requires': parse_list, 414 'obsoletes': parse_list, 415 'classifiers': self._get_parser_compound(parse_file, parse_list), 416 'license': parse_file, 417 'description': parse_file, 418 'long_description': parse_file, 419 'version': self._parse_version, 420 'project_urls': parse_dict, 421 } 422 423 def _parse_version(self, value): 424 """Parses `version` option value. 425 426 :param value: 427 :rtype: str 428 429 """ 430 version = self._parse_attr(value) 431 432 if callable(version): 433 version = version() 434 435 if not isinstance(version, string_types): 436 if hasattr(version, '__iter__'): 437 version = '.'.join(map(str, version)) 438 else: 439 version = '%s' % version 440 441 return version 442 443 444class ConfigOptionsHandler(ConfigHandler): 445 446 section_prefix = 'options' 447 448 @property 449 def parsers(self): 450 """Metadata item name to parser function mapping.""" 451 parse_list = self._parse_list 452 parse_list_semicolon = partial(self._parse_list, separator=';') 453 parse_bool = self._parse_bool 454 parse_dict = self._parse_dict 455 456 return { 457 'zip_safe': parse_bool, 458 'use_2to3': parse_bool, 459 'include_package_data': parse_bool, 460 'package_dir': parse_dict, 461 'use_2to3_fixers': parse_list, 462 'use_2to3_exclude_fixers': parse_list, 463 'convert_2to3_doctests': parse_list, 464 'scripts': parse_list, 465 'eager_resources': parse_list, 466 'dependency_links': parse_list, 467 'namespace_packages': parse_list, 468 'install_requires': parse_list_semicolon, 469 'setup_requires': parse_list_semicolon, 470 'tests_require': parse_list_semicolon, 471 'packages': self._parse_packages, 472 'entry_points': self._parse_file, 473 'py_modules': parse_list, 474 } 475 476 def _parse_packages(self, value): 477 """Parses `packages` option value. 478 479 :param value: 480 :rtype: list 481 """ 482 find_directive = 'find:' 483 484 if not value.startswith(find_directive): 485 return self._parse_list(value) 486 487 # Read function arguments from a dedicated section. 488 find_kwargs = self.parse_section_packages__find( 489 self.sections.get('packages.find', {})) 490 491 from setuptools import find_packages 492 493 return find_packages(**find_kwargs) 494 495 def parse_section_packages__find(self, section_options): 496 """Parses `packages.find` configuration file section. 497 498 To be used in conjunction with _parse_packages(). 499 500 :param dict section_options: 501 """ 502 section_data = self._parse_section_to_dict( 503 section_options, self._parse_list) 504 505 valid_keys = ['where', 'include', 'exclude'] 506 507 find_kwargs = dict( 508 [(k, v) for k, v in section_data.items() if k in valid_keys and v]) 509 510 where = find_kwargs.get('where') 511 if where is not None: 512 find_kwargs['where'] = where[0] # cast list to single val 513 514 return find_kwargs 515 516 def parse_section_entry_points(self, section_options): 517 """Parses `entry_points` configuration file section. 518 519 :param dict section_options: 520 """ 521 parsed = self._parse_section_to_dict(section_options, self._parse_list) 522 self['entry_points'] = parsed 523 524 def _parse_package_data(self, section_options): 525 parsed = self._parse_section_to_dict(section_options, self._parse_list) 526 527 root = parsed.get('*') 528 if root: 529 parsed[''] = root 530 del parsed['*'] 531 532 return parsed 533 534 def parse_section_package_data(self, section_options): 535 """Parses `package_data` configuration file section. 536 537 :param dict section_options: 538 """ 539 self['package_data'] = self._parse_package_data(section_options) 540 541 def parse_section_exclude_package_data(self, section_options): 542 """Parses `exclude_package_data` configuration file section. 543 544 :param dict section_options: 545 """ 546 self['exclude_package_data'] = self._parse_package_data( 547 section_options) 548 549 def parse_section_extras_require(self, section_options): 550 """Parses `extras_require` configuration file section. 551 552 :param dict section_options: 553 """ 554 parse_list = partial(self._parse_list, separator=';') 555 self['extras_require'] = self._parse_section_to_dict( 556 section_options, parse_list) 557