1# Copyright 2021 The Pigweed Authors 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); you may not 4# use this file except in compliance with the License. You may obtain a copy of 5# the License at 6# 7# https://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12# License for the specific language governing permissions and limitations under 13# the License. 14"""Dataclass for a Python package.""" 15 16import configparser 17from contextlib import contextmanager 18import copy 19from dataclasses import dataclass, asdict 20import io 21import json 22import os 23from pathlib import Path 24import pprint 25import re 26import shutil 27from typing import Any, Dict, List, Optional, Iterable 28 29_pretty_format = pprint.PrettyPrinter(indent=1, width=120).pformat 30 31# List of known environment markers supported by pip. 32# https://peps.python.org/pep-0508/#environment-markers 33_PY_REQUIRE_ENVIRONMENT_MARKER_NAMES = [ 34 'os_name', 35 'sys_platform', 36 'platform_machine', 37 'platform_python_implementation', 38 'platform_release', 39 'platform_system', 40 'platform_version', 41 'python_version', 42 'python_full_version', 43 'implementation_name', 44 'implementation_version', 45 'extra', 46] 47 48 49@contextmanager 50def change_working_dir(directory: Path): 51 original_dir = Path.cwd() 52 try: 53 os.chdir(directory) 54 yield directory 55 finally: 56 os.chdir(original_dir) 57 58 59class UnknownPythonPackageName(Exception): 60 """Exception thrown when a Python package_name cannot be determined.""" 61 62 63class MissingSetupSources(Exception): 64 """Exception thrown when a Python package is missing setup source files. 65 66 For example: setup.cfg and pyproject.toml.i 67 """ 68 69 70def _sanitize_install_requires(metadata_dict: dict) -> dict: 71 """Convert install_requires lists into strings joined with line breaks.""" 72 try: 73 install_requires = metadata_dict['options']['install_requires'] 74 if isinstance(install_requires, list): 75 metadata_dict['options']['install_requires'] = '\n'.join( 76 install_requires 77 ) 78 except KeyError: 79 pass 80 return metadata_dict 81 82 83@dataclass 84class PythonPackage: 85 """Class to hold a single Python package's metadata.""" 86 87 sources: List[Path] 88 setup_sources: List[Path] 89 tests: List[Path] 90 inputs: List[Path] 91 gn_target_name: str = '' 92 generate_setup: Optional[Dict] = None 93 config: Optional[configparser.ConfigParser] = None 94 95 @staticmethod 96 def from_dict(**kwargs) -> 'PythonPackage': 97 """Build a PythonPackage instance from a dictionary.""" 98 transformed_kwargs = copy.copy(kwargs) 99 100 # Transform string filenames to Paths 101 for attribute in ['sources', 'tests', 'inputs', 'setup_sources']: 102 transformed_kwargs[attribute] = [Path(s) for s in kwargs[attribute]] 103 104 return PythonPackage(**transformed_kwargs) 105 106 def __post_init__(self): 107 # Read the setup.cfg file if possible 108 if not self.config: 109 self.config = self._load_config() 110 111 @property 112 def setup_dir(self) -> Optional[Path]: 113 if not self.setup_sources: 114 return None 115 # Assuming all setup_source files live in the same parent directory. 116 return self.setup_sources[0].parent 117 118 @property 119 def setup_py(self) -> Path: 120 setup_py = [ 121 setup_file 122 for setup_file in self.setup_sources 123 if str(setup_file).endswith('setup.py') 124 ] 125 # setup.py will not exist for GN generated Python packages 126 assert len(setup_py) == 1 127 return setup_py[0] 128 129 @property 130 def setup_cfg(self) -> Optional[Path]: 131 setup_cfg = [ 132 setup_file 133 for setup_file in self.setup_sources 134 if str(setup_file).endswith('setup.cfg') 135 ] 136 if len(setup_cfg) < 1: 137 return None 138 return setup_cfg[0] 139 140 def as_dict(self) -> Dict[Any, Any]: 141 """Return a dict representation of this class.""" 142 self_dict = asdict(self) 143 if self.config: 144 # Expand self.config into text. 145 setup_cfg_text = io.StringIO() 146 self.config.write(setup_cfg_text) 147 self_dict['config'] = setup_cfg_text.getvalue() 148 return self_dict 149 150 @property 151 def package_name(self) -> str: 152 unknown_package_message = ( 153 'Cannot determine the package_name for the Python ' 154 f'library/package: {self.gn_target_name}\n\n' 155 'This could be due to a missing python dependency in GN for:\n' 156 f'{self.gn_target_name}\n\n' 157 ) 158 159 if self.config: 160 try: 161 name = self.config['metadata']['name'] 162 except KeyError: 163 raise UnknownPythonPackageName( 164 unknown_package_message + _pretty_format(self.as_dict()) 165 ) 166 return name 167 top_level_source_dir = self.top_level_source_dir 168 if top_level_source_dir: 169 return top_level_source_dir.name 170 171 actual_gn_target_name = self.gn_target_name.split(':') 172 if len(actual_gn_target_name) < 2: 173 raise UnknownPythonPackageName(unknown_package_message) 174 175 return actual_gn_target_name[-1] 176 177 @property 178 def package_dir(self) -> Path: 179 if self.setup_cfg and self.setup_cfg.is_file(): 180 return self.setup_cfg.parent / self.package_name 181 root_source_dir = self.top_level_source_dir 182 if root_source_dir: 183 return root_source_dir 184 return self.sources[0].parent 185 186 @property 187 def top_level_source_dir(self) -> Optional[Path]: 188 source_dir_paths = sorted( 189 set((len(sfile.parts), sfile.parent) for sfile in self.sources), 190 key=lambda s: s[1], 191 ) 192 if not source_dir_paths: 193 return None 194 195 top_level_source_dir = source_dir_paths[0][1] 196 if not top_level_source_dir.is_dir(): 197 return None 198 199 return top_level_source_dir 200 201 def _load_config(self) -> Optional[configparser.ConfigParser]: 202 config = configparser.ConfigParser() 203 204 # Check for a setup.cfg and load that config. 205 if self.setup_cfg: 206 if self.setup_cfg.is_file(): 207 with self.setup_cfg.open() as config_file: 208 config.read_file(config_file) 209 return config 210 if self.setup_cfg.with_suffix('.json').is_file(): 211 return self._load_setup_json_config() 212 213 # Fallback on the generate_setup scope from GN 214 if self.generate_setup: 215 config.read_dict(_sanitize_install_requires(self.generate_setup)) 216 return config 217 return None 218 219 def _load_setup_json_config(self) -> configparser.ConfigParser: 220 assert self.setup_cfg 221 setup_json = self.setup_cfg.with_suffix('.json') 222 config = configparser.ConfigParser() 223 with setup_json.open() as json_fp: 224 json_dict = _sanitize_install_requires(json.load(json_fp)) 225 226 config.read_dict(json_dict) 227 return config 228 229 def copy_sources_to(self, destination: Path) -> None: 230 """Copy this PythonPackage source files to another path.""" 231 new_destination = destination / self.package_dir.name 232 new_destination.mkdir(parents=True, exist_ok=True) 233 shutil.copytree(self.package_dir, new_destination, dirs_exist_ok=True) 234 235 def install_requires_entries(self) -> List[str]: 236 """Convert the install_requires entry into a list of strings.""" 237 this_requires: List[str] = [] 238 # If there's no setup.cfg, do nothing. 239 if not self.config: 240 return this_requires 241 242 # Requires are delimited by newlines or semicolons. 243 # Split existing list on either one. 244 for req in re.split( 245 r' *[\n;] *', self.config['options']['install_requires'] 246 ): 247 # Skip empty lines. 248 if not req: 249 continue 250 # Get the name part part of the dep, ignoring any spaces or 251 # other characters. 252 req_name_match = re.match(r'^(?P<name_part>[A-Za-z0-9_-]+)', req) 253 if not req_name_match: 254 continue 255 req_name = req_name_match.groupdict().get('name_part', '') 256 # Check if this is an environment marker. 257 if req_name in _PY_REQUIRE_ENVIRONMENT_MARKER_NAMES: 258 # Append this req as an environment marker for the previous 259 # requirement. 260 this_requires[-1] += f';{req}' 261 continue 262 # Normal pip requirement, save to this_requires. 263 this_requires.append(req) 264 return this_requires 265 266 267def load_packages( 268 input_list_files: Iterable[Path], ignore_missing=False 269) -> List[PythonPackage]: 270 """Load Python package metadata and configs.""" 271 272 packages = [] 273 for input_path in input_list_files: 274 if ignore_missing and not input_path.is_file(): 275 continue 276 with input_path.open() as input_file: 277 # Each line contains the path to a json file. 278 for json_file in input_file.readlines(): 279 # Load the json as a dict. 280 json_file_path = Path(json_file.strip()).resolve() 281 with json_file_path.open() as json_fp: 282 json_dict = json.load(json_fp) 283 284 packages.append(PythonPackage.from_dict(**json_dict)) 285 return packages 286