1# Copyright 2023 The Pigweed Authors 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); you may not 4# use this file except in compliance with the License. You may obtain a copy of 5# the License at 6# 7# https://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12# License for the specific language governing permissions and limitations under 13# the License. 14"""Generates a BUILD.gn file from `cc_library` rules in Bazel workspace.""" 15 16import argparse 17import json 18import os 19import subprocess 20from collections import defaultdict 21from pathlib import Path 22from string import Template 23from typing import IO, Iterable, Iterator, Set 24 25from pw_build.bazel_query import BazelWorkspace 26from pw_build.gn_config import consolidate_configs, GnConfig 27from pw_build.gn_target import GnTarget 28from pw_build.gn_utils import GnLabel, GnPath 29from pw_build.gn_writer import gn_format, GnFile, GnWriter, MalformedGnError 30 31_DOCS_RST_TEMPLATE = Template( 32 ''' 33.. _module-pw_third_party_$repo: 34 35$name_section 36$name 37$name_section 38The ``$$dir_pw_third_party/$repo/`` module provides build files to allow 39optionally including upstream $name. 40 41---------------$name_subsection 42Using upstream $name 43---------------$name_subsection 44If you want to use $name, you must do the following: 45 46Submodule 47========= 48Add $name to your workspace with the following command. 49 50.. code-block:: sh 51 52 git submodule add $url \\ 53 third_party/$repo/src 54 55GN 56== 57* Set the GN var ``dir_pw_third_party_$repo`` to the location of the 58 $name source. 59 60 If you used the command above, this will be 61 ``//third_party/$repo/src`` 62 63 This can be set in your args.gn or .gn file like: 64 ``dir_pw_third_party_$repo_var = "//third_party/$repo/src"`` 65 66Updating 67======== 68The GN build files are generated from the third-party Bazel build files using 69$$dir_pw_build/py/pw_build/$script. 70 71The script uses data taken from ``$$dir_pw_third_party/$repo/repo.json``. 72The schema of ``repo.json`` is described in :ref:`module-pw_build-third-party`. 73 74The script should be re-run whenever the submodule is updated or the JSON file 75is modified. Specify the location of the Bazel repository can be specified using 76the ``-w`` option, e.g. 77 78.. code-block:: sh 79 80 python pw_build/py/pw_build/$script \\ 81 -w third_party/$repo/src 82 83.. DO NOT EDIT BELOW THIS LINE. Generated section. 84'''.lstrip() 85) 86 87_GIT_SHORT_REV_LEN = 8 88 89 90class GnGenerator: 91 """Maintains state while GN files are generated from Bazel files. 92 93 Attributes: 94 packages: The set of packages/sub-directories, each of which will have a 95 BUILD.gn. 96 configs: A mapping of package names to common GN configs for that 97 package. 98 targets: A mapping of package names to that Gn targets in that package. 99 """ 100 101 def __init__(self) -> None: 102 self.packages: Set[str] = set() 103 self._workspace: BazelWorkspace 104 self._base_label: GnLabel 105 self._base_path: GnPath 106 self._repo: str 107 self._repo_var: str 108 self._repos: dict[str, Set[str]] = defaultdict(set) 109 self._no_gn_check: list[GnLabel] = [] 110 self.configs: dict[str, list[GnConfig]] = defaultdict(list) 111 self.targets: dict[str, list[GnTarget]] = defaultdict(list) 112 113 self.packages.add('') 114 self.configs[''] = [] 115 self.targets[''] = [] 116 117 def set_repo(self, repo: str) -> None: 118 """Sets the repository related variables. 119 120 This does not need to be called unless `load_workspace` is not being 121 called (as in some unit tests). 122 123 Args: 124 repo: The repository name. 125 """ 126 self._repo = repo 127 self._repo_var = repo.replace('-', '_') 128 self._base_label = GnLabel(f'$dir_pw_third_party/{repo}') 129 self._base_path = GnPath(f'$dir_pw_third_party_{self._repo_var}') 130 131 def exclude_from_gn_check(self, **kwargs) -> None: 132 """Mark a target as being excluding from `gn check`. 133 134 This should be called before loading or adding targets. 135 136 Args: 137 kwargs: Same as `GnLabel`. 138 """ 139 self._no_gn_check.append(GnLabel(self._base_label, **kwargs)) 140 141 def load_workspace(self, workspace_path: Path) -> str: 142 """Loads a Bazel workspace. 143 144 Args: 145 workspace_path: The path to the Bazel workspace. 146 """ 147 self._workspace = BazelWorkspace(workspace_path) 148 self.set_repo(self._workspace.repo) 149 return self._repo 150 151 def load_targets(self, kind: str, allow_testonly: bool) -> None: 152 """Analyzes a Bazel workspace and loads target info from it. 153 154 Target info will only be added for `kind` rules. Additionally, 155 libraries marked "testonly" may be included if `allow_testonly` is 156 present and true in the repo.json file. 157 158 Args: 159 kind: The Bazel rule kind to parse. 160 allow_testonly: If true, include testonly targets as well. 161 """ 162 for rule in self._workspace.get_rules(kind): 163 self.add_target(allow_testonly, bazel=rule) 164 165 def add_target(self, allow_testonly: bool = False, **kwargs) -> None: 166 """Adds a target using this object's base label and source root. 167 168 Args: 169 allow_testonly: If true, include testonly targets as well. 170 171 Keyword Args: 172 Same as `GnTarget`. 173 """ 174 target = GnTarget(self._base_label.dir(), self._base_path, **kwargs) 175 target.check_includes = target.label() not in self._no_gn_check 176 if allow_testonly or not target.testonly: 177 package = target.package() 178 self.packages.add(package) 179 self._repos[package].update(target.repos()) 180 self.targets[package].append(target) 181 182 def add_configs(self, package: str, *configs: GnConfig) -> None: 183 """Adds configs for the given package. 184 185 This is mostly used for testing and debugging. 186 187 Args: 188 package: The package to add configs for. 189 190 Variable Args: 191 configs: The configs to add. 192 """ 193 self.configs[package].extend(configs) 194 195 def generate_configs( 196 self, configs_to_add: Iterable[str], configs_to_remove: Iterable[str] 197 ) -> bool: 198 """Extracts the most common flags into common configs. 199 200 Args: 201 configs_to_add: Additional configs to add to every target. 202 configs_to_remove: Default configs to remove from every target. 203 """ 204 added = [GnLabel(label) for label in configs_to_add] 205 removed = [GnLabel(label) for label in configs_to_remove] 206 all_configs = [] 207 for targets in self.targets.values(): 208 for target in targets: 209 for label in added: 210 target.add_config(label) 211 for label in removed: 212 target.remove_config(label) 213 all_configs.append(target.config) 214 215 consolidated = list(consolidate_configs(self._base_label, *all_configs)) 216 217 def count_packages(config: GnConfig) -> int: 218 return sum( 219 [ 220 any(config.within(target.config) for target in targets) 221 for _package, targets in self.targets.items() 222 ] 223 ) 224 225 common = list( 226 filter(lambda config: count_packages(config) > 1, consolidated) 227 ) 228 229 self.configs[''] = sorted(common) 230 for targets in self.targets.values(): 231 for target in targets: 232 for config in target.config.deduplicate(*common): 233 if not config.label: 234 raise MalformedGnError('config is missing label') 235 target.add_config(config.label, public=config.public()) 236 237 for package, targets in self.targets.items(): 238 configs = [target.config for target in targets] 239 common = list( 240 consolidate_configs( 241 self._base_label.joinlabel(package), 242 *configs, 243 extract_public=True, 244 ) 245 ) 246 self.configs[package].extend(common) 247 self.configs[package].sort() 248 for target in targets: 249 for config in target.config.deduplicate(*common): 250 if not config.label: 251 raise MalformedGnError('config is missing label') 252 target.add_config(config.label, public=config.public()) 253 return True 254 255 def write_repo_gni(self, repo_gni: GnWriter, name: str) -> None: 256 """Write the top-level GN import that declares build arguments. 257 258 Args: 259 repo_gni: The output writer object. 260 name: The third party module_name. 261 """ 262 repo_gni.write_target_start('declare_args') 263 repo_gni.write_comment( 264 f'If compiling tests with {name}, this variable is set to the path ' 265 f'to the {name} installation. When set, a pw_source_set for the ' 266 f'{name} library is created at "$dir_pw_third_party/{self._repo}".' 267 ) 268 repo_gni.write(f'dir_pw_third_party_{self._repo_var} = ""') 269 repo_gni.write_end() 270 271 def write_build_gn(self, package: str, build_gn: GnWriter) -> None: 272 """Write the target info for a package to a BUILD.gn file. 273 274 Args: 275 package: The name of the package to write a BUILD.gn for. 276 build_gn: The output writer object. 277 no_gn_check: List of targets with `check_includes = false`. 278 """ 279 build_gn.write_imports(['//build_overrides/pigweed.gni']) 280 build_gn.write_blank() 281 imports = set() 282 imports.add('$dir_pw_build/target_types.gni') 283 284 if not package: 285 imports.add('$dir_pw_docgen/docs.gni') 286 for repo in self._repos[package]: 287 repo = build_gn.repos[repo] 288 imports.add('$dir_pw_build/error.gni') 289 imports.add(f'$dir_pw_third_party/{repo}/{repo}.gni') 290 if self._repo: 291 imports.add(f'$dir_pw_third_party/{self._repo}/{self._repo}.gni') 292 build_gn.write_imports(sorted(list(imports))) 293 294 if not package: 295 build_gn.write_if(f'dir_pw_third_party_{self._repo_var} != ""') 296 297 for config in sorted(self.configs[package], reverse=True): 298 build_gn.write_config(config) 299 300 targets = self.targets[package] 301 targets.sort(key=lambda target: target.name()) 302 for target in targets: 303 build_gn.write_target(target) 304 305 if not package: 306 build_gn.write_end() 307 build_gn.write_target_start('pw_doc_group', 'docs') 308 build_gn.write_list('sources', ['docs.rst']) 309 build_gn.write_end() 310 311 def write_docs_rst(self, docs_rst: IO, name: str) -> None: 312 """Writes new, top-level documentation for the repo. 313 314 Args: 315 docs_rst: The output object. 316 name: The third party module_name. 317 """ 318 contents = _DOCS_RST_TEMPLATE.substitute( 319 script=os.path.basename(__file__), 320 name=name, 321 name_section='=' * len(name), 322 name_subsection='-' * len(name), 323 repo=self._repo, 324 repo_var=self._repo_var, 325 url=self._workspace.url(), 326 ) 327 docs_rst.write('\n'.join(self.update_version(contents.split('\n')))) 328 329 def update_version(self, lines: Iterable[str]) -> Iterator[str]: 330 """Replaces the "Version" part of docs.rst with the latest revision. 331 332 This will truncate everything after the "generated section" comment and 333 add the comment and version information. If the file does not have the 334 comment, the comment and information will appended to the end of the 335 file. 336 337 Args: 338 lines: Iterator of lines. 339 """ 340 comment = '.. DO NOT EDIT BELOW THIS LINE. Generated section.' 341 url = self._workspace.url().rstrip('.git') 342 revision = self._workspace.revision() 343 short = revision[:_GIT_SHORT_REV_LEN] 344 for line in lines: 345 line = line.rstrip() 346 if line == comment: 347 break 348 yield line 349 yield comment 350 yield '' 351 yield 'Version' 352 yield '=======' 353 yield f'The update script was last run for revision `{short}`_.' 354 yield '' 355 yield f'.. _{short}: {url}/tree/{revision}' 356 yield '' 357 358 def update_third_party_docs(self, contents: str) -> str: 359 """Adds a dep on the generated docs to a "third_party_docs" group.""" 360 lines = contents.split('\n') 361 new_deps = f'deps = ["$dir_pigweed/third_party/{self._repo}:docs",' 362 for i in range(len(lines) - 1): 363 if lines[i] == 'group("third_party_docs") {': 364 lines[i + 1] = new_deps 365 return '\n'.join(lines) 366 raise ValueError('"third_party_docs" target not found') 367 368 def write_extra(self, extra: IO, label: str) -> None: 369 """Runs a Bazel target to generate an extra file.""" 370 self._workspace.run(label, output=extra) 371 372 373def write_owners(owners: IO) -> None: 374 """Write an OWNERS file, but only if it does not already exist. 375 376 Args: 377 owners: The output object. 378 """ 379 try: 380 result = subprocess.run( 381 ['git', 'config', '--get', 'user.email'], 382 check=True, 383 capture_output=True, 384 ) 385 owners.write(result.stdout.decode('utf-8')) 386 except subprocess.CalledProcessError: 387 # Couldn't get owner from git config. 388 pass 389 390 391def _parse_args() -> list[Path]: 392 """Parse arguments.""" 393 parser = argparse.ArgumentParser(description=__doc__) 394 parser.add_argument( 395 'workspace', 396 type=Path, 397 nargs='+', 398 help=('Path to Bazel workspace to be queried.'), 399 ) 400 args = parser.parse_args() 401 return args.workspace 402 403 404def _generate_gn(workspace_path: Path) -> None: 405 """Creates GN, doc, and OWNERS files for a third-party repository. 406 407 Args: 408 workspace_path: Path to the Bazel workspace. 409 """ 410 pw_root = os.getenv('PW_ROOT') 411 if not pw_root: 412 raise RuntimeError('PW_ROOT is not set') 413 414 generator = GnGenerator() 415 repo = generator.load_workspace(workspace_path) 416 output = Path(pw_root, 'third_party', repo) 417 418 with open(output.joinpath('repo.json')) as file: 419 repo_json = json.load(file) 420 421 for exclusion in repo_json.get('no_gn_check', []): 422 generator.exclude_from_gn_check(bazel=exclusion) 423 generator.load_targets('cc_library', repo_json.get('allow_testonly', False)) 424 generator.generate_configs( 425 repo_json.get('add', []), repo_json.get('remove', []) 426 ) 427 428 name = repo_json['name'] 429 with GnFile(Path(output, f'{repo}.gni')) as repo_gni: 430 generator.write_repo_gni(repo_gni, name) 431 432 for package in generator.packages: 433 with GnFile(Path(output, package, 'BUILD.gn'), package) as build_gn: 434 build_gn.repos = repo_json.get('repos', {}) 435 build_gn.aliases = repo_json.get('aliases', {}) 436 generator.write_build_gn(package, build_gn) 437 438 created_docs_rst = False 439 try: 440 with open(Path(output, 'docs.rst'), 'x') as docs_rst: 441 generator.write_docs_rst(docs_rst, name) 442 created_docs_rst = True 443 except OSError: 444 pass # docs.rst file already exists. 445 446 if created_docs_rst: 447 # Add the doc group to //docs:third_party_docs 448 docs_build_gn_path = Path(pw_root, 'docs', 'BUILD.gn') 449 with open(docs_build_gn_path, 'r') as docs_build_gn: 450 contents = docs_build_gn.read() 451 with open(docs_build_gn_path, 'w') as docs_build_gn: 452 docs_build_gn.write(generator.update_third_party_docs(contents)) 453 gn_format(docs_build_gn_path) 454 455 else: 456 # Replace the version section of the existing docs.rst. 457 with open(Path(output, 'docs.rst'), 'r') as docs_rst: 458 contents = '\n'.join(generator.update_version(docs_rst)) 459 with open(Path(output, 'docs.rst'), 'w') as docs_rst: 460 docs_rst.write(contents) 461 462 try: 463 with open(Path(output, 'OWNERS'), 'x') as owners: 464 write_owners(owners) 465 except OSError: 466 pass # OWNERS file already exists. 467 468 for filename, label in repo_json.get('extra_files', {}).items(): 469 with open(Path(output, filename), 'w') as extra: 470 generator.write_extra(extra, label) 471 472 473if __name__ == '__main__': 474 for workspace in _parse_args(): 475 _generate_gn(workspace) 476