1# Copyright 2022 The Pigweed Authors 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); you may not 4# use this file except in compliance with the License. You may obtain a copy of 5# the License at 6# 7# https://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12# License for the specific language governing permissions and limitations under 13# the License. 14"""Manages the list of Pigweed modules. 15 16Used by modules.gni to generate: 17 18- a build arg for each module, 19- a list of module paths (pw_modules), 20- a list of module tests (pw_module_tests), and 21- a list of module docs (pw_module_docs). 22""" 23 24import argparse 25import difflib 26import enum 27import io 28import os 29from pathlib import Path 30import sys 31import subprocess 32from typing import Iterator, Sequence 33 34_COPYRIGHT_NOTICE = '''\ 35# Copyright 2022 The Pigweed Authors 36# 37# Licensed under the Apache License, Version 2.0 (the "License"); you may not 38# use this file except in compliance with the License. You may obtain a copy of 39# the License at 40# 41# https://www.apache.org/licenses/LICENSE-2.0 42# 43# Unless required by applicable law or agreed to in writing, software 44# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 45# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 46# License for the specific language governing permissions and limitations under 47# the License.''' 48 49_WARNING = '\033[31m\033[1mWARNING:\033[0m ' # Red WARNING: prefix 50_ERROR = '\033[41m\033[37m\033[1mERROR:\033[0m ' # Red background ERROR: prefix 51 52_MISSING_MODULES_WARNING = ( 53 _WARNING 54 + '''\ 55The PIGWEED_MODULES list is missing the following modules: 56{modules} 57 58If the listed modules are Pigweed modules, add them to PIGWEED_MODULES. 59 60If the listed modules are not actual Pigweed modules, remove any stray pw_* 61directories in the Pigweed repository (git clean -fd). 62''' 63) 64 65_OUT_OF_DATE_WARNING = ( 66 _ERROR 67 + '''\ 68The generated Pigweed modules list .gni file is out of date! 69 70Regenerate the modules lists and commit it to fix this: 71 72 ninja -C {out_dir} update_modules 73 74 git add {file} 75''' 76) 77 78_FORMAT_FAILED_WARNING = ( 79 _ERROR 80 + '''\ 81Failed to generate a valid .gni from PIGWEED_MODULES! 82 83This may be a Pigweed bug; please report this to the Pigweed team. 84''' 85) 86 87_DO_NOT_SET = 'DO NOT SET THIS BUILD ARGUMENT!' 88 89 90def _module_list_warnings(root: Path, modules: Sequence[str]) -> Iterator[str]: 91 missing = _missing_modules(root, modules) 92 if missing: 93 yield _MISSING_MODULES_WARNING.format( 94 modules=''.join(f'\n - {module}' for module in missing) 95 ) 96 97 if any(modules[i] > modules[i + 1] for i in range(len(modules) - 1)): 98 yield _WARNING + 'The PIGWEED_MODULES list is not sorted!' 99 yield '' 100 yield 'Apply the following diff to fix the order:' 101 yield '' 102 yield from difflib.unified_diff( 103 modules, 104 sorted(modules), 105 lineterm='', 106 n=1, 107 fromfile='PIGWEED_MODULES', 108 tofile='PIGWEED_MODULES', 109 ) 110 111 yield '' 112 113 114def _generate_modules_gni( 115 prefix: Path, modules: Sequence[str] 116) -> Iterator[str]: 117 """Generates a .gni file with variables and lists for Pigweed modules.""" 118 script_path = Path(__file__).resolve() 119 script = script_path.relative_to(script_path.parent.parent).as_posix() 120 121 module_paths: Sequence[Path] = list(Path(module) for module in modules) 122 123 yield _COPYRIGHT_NOTICE 124 yield '' 125 yield '# Build args and lists for all modules in Pigweed.' 126 yield '#' 127 yield f'# DO NOT EDIT! Generated by {script}.' 128 yield '#' 129 yield '# To add modules here, list them in PIGWEED_MODULES and build the' 130 yield '# update_modules target and commit the updated version of this file:' 131 yield '#' 132 yield '# ninja -C out update_modules' 133 yield '#' 134 yield '# DO NOT IMPORT THIS FILE DIRECTLY!' 135 yield '#' 136 yield '# Import it through //build_overrides/pigweed.gni instead.' 137 yield '' 138 yield '# Declare a build arg for each module.' 139 yield 'declare_args() {' 140 141 for module in module_paths: 142 final_path = (prefix / module).as_posix() 143 yield f'dir_{module.name} = get_path_info("{final_path}", "abspath")' 144 145 yield '}' 146 yield '' 147 yield '# Declare these as GN args in case this is imported in args.gni.' 148 yield '# Use a separate block so variables in the prior block can be used.' 149 yield 'declare_args() {' 150 yield f'# A list with paths to all Pigweed module. {_DO_NOT_SET}' 151 yield 'pw_modules = [' 152 153 for module in module_paths: 154 yield f'dir_{module.name},' 155 156 yield ']' 157 yield '' 158 159 yield f'# A list with all Pigweed module test groups. {_DO_NOT_SET}' 160 yield 'pw_module_tests = [' 161 162 for module in module_paths: 163 yield f'"$dir_{module.name}:tests",' 164 165 yield ']' 166 yield '' 167 yield f'# A list with all Pigweed modules docs groups. {_DO_NOT_SET}' 168 yield 'pw_module_docs = [' 169 170 for module in module_paths: 171 yield f'"$dir_{module.name}:docs",' 172 173 yield ']' 174 yield '' 175 yield '}' 176 177 178def _missing_modules(root: Path, modules: Sequence[str]) -> Sequence[str]: 179 return sorted( 180 frozenset( 181 str(p.relative_to(root)) for p in root.glob('pw_*') if p.is_dir() 182 ) 183 - frozenset(modules) 184 ) 185 186 187class Mode(enum.Enum): 188 WARN = 0 # Warn if anything is out of date 189 CHECK = 1 # Fail if anything is out of date 190 UPDATE = 2 # Update the generated modules lists 191 192 193def _parse_args() -> dict: 194 parser = argparse.ArgumentParser( 195 description=__doc__, 196 formatter_class=argparse.RawDescriptionHelpFormatter, 197 ) 198 parser.add_argument('root', type=Path, help='Root build dir') 199 parser.add_argument('modules_list', type=Path, help='Input modules list') 200 parser.add_argument('modules_gni_file', type=Path, help='Output .gni file') 201 parser.add_argument( 202 '--mode', type=Mode.__getitem__, choices=Mode, required=True 203 ) 204 parser.add_argument( 205 '--stamp', 206 type=Path, 207 help='Stamp file for operations that should only run once (warn)', 208 ) 209 return vars(parser.parse_args()) 210 211 212def main( 213 root: Path, 214 modules_list: Path, 215 modules_gni_file: Path, 216 mode: Mode, 217 stamp: Path | None = None, 218) -> int: 219 """Manages the list of Pigweed modules.""" 220 prefix = Path(os.path.relpath(root, modules_gni_file.parent)) 221 modules = modules_list.read_text().splitlines() 222 223 # Detect any problems with the modules list. 224 warnings = list(_module_list_warnings(root, modules)) 225 errors = [] 226 227 modules.sort() # Sort in case the modules list in case it wasn't sorted. 228 229 # Check if the contents of the .gni file are out of date. 230 if mode in (Mode.WARN, Mode.CHECK): 231 text = io.StringIO() 232 for line in _generate_modules_gni(prefix, modules): 233 print(line, file=text) 234 235 process = subprocess.run( 236 ['gn', 'format', '--stdin'], 237 input=text.getvalue().encode('utf-8'), 238 stdout=subprocess.PIPE, 239 ) 240 if process.returncode != 0: 241 errors.append(_FORMAT_FAILED_WARNING) 242 243 # Make a diff of required changes 244 modules_gni_relpath = os.path.relpath(modules_gni_file, root) 245 diff = list( 246 difflib.unified_diff( 247 modules_gni_file.read_text().splitlines(), 248 process.stdout.decode('utf-8', errors='replace').splitlines(), 249 fromfile=os.path.join('a', modules_gni_relpath), 250 tofile=os.path.join('b', modules_gni_relpath), 251 lineterm='', 252 n=1, 253 ) 254 ) 255 # If any differences were found, print the error and the diff. 256 if diff: 257 errors.append( 258 _OUT_OF_DATE_WARNING.format( 259 out_dir=os.path.relpath(os.curdir, root), 260 file=os.path.relpath(modules_gni_file, root), 261 ) 262 ) 263 errors.append('Expected Diff:\n') 264 errors.append('\n'.join(diff)) 265 errors.append('\n') 266 267 elif mode is Mode.UPDATE: # Update the modules .gni file 268 with modules_gni_file.open('w', encoding='utf-8') as file: 269 for line in _generate_modules_gni(prefix, modules): 270 print(line, file=file) 271 272 process = subprocess.run( 273 ['gn', 'format', modules_gni_file], stdout=subprocess.DEVNULL 274 ) 275 if process.returncode != 0: 276 errors.append(_FORMAT_FAILED_WARNING) 277 278 # If there are errors, display them and abort. 279 if warnings or errors: 280 for line in warnings + errors: 281 print(line, file=sys.stderr) 282 283 # Delete the stamp so this always reruns. Deleting is necessary since 284 # some of the checks do not depend on input files. 285 if stamp and stamp.exists(): 286 stamp.unlink() 287 288 if mode is Mode.WARN: 289 return 0 290 291 if mode is Mode.CHECK: 292 return 1 293 294 return 1 if errors else 0 # Allow warnings but not errors when updating 295 296 if stamp: 297 stamp.touch() 298 299 return 0 300 301 302if __name__ == '__main__': 303 sys.exit(main(**_parse_args())) 304