• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1# Copyright 2022 The Pigweed Authors
2#
3# Licensed under the Apache License, Version 2.0 (the "License"); you may not
4# use this file except in compliance with the License. You may obtain a copy of
5# the License at
6#
7#     https://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
11# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
12# License for the specific language governing permissions and limitations under
13# the License.
14"""Manages the list of Pigweed modules.
15
16Used by modules.gni to generate:
17
18- a build arg for each module,
19- a list of module paths (pw_modules),
20- a list of module tests (pw_module_tests), and
21- a list of module docs (pw_module_docs).
22"""
23
24import argparse
25import difflib
26import enum
27import io
28import os
29from pathlib import Path
30import sys
31import subprocess
32from typing import Iterator, Optional, Sequence
33
34_COPYRIGHT_NOTICE = '''\
35# Copyright 2022 The Pigweed Authors
36#
37# Licensed under the Apache License, Version 2.0 (the "License"); you may not
38# use this file except in compliance with the License. You may obtain a copy of
39# the License at
40#
41#     https://www.apache.org/licenses/LICENSE-2.0
42#
43# Unless required by applicable law or agreed to in writing, software
44# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
45# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
46# License for the specific language governing permissions and limitations under
47# the License.'''
48
49_WARNING = '\033[31m\033[1mWARNING:\033[0m '  # Red WARNING: prefix
50_ERROR = '\033[41m\033[37m\033[1mERROR:\033[0m '  # Red background ERROR: prefix
51
52_MISSING_MODULES_WARNING = (
53    _WARNING
54    + '''\
55The PIGWEED_MODULES list is missing the following modules:
56{modules}
57
58If the listed modules are Pigweed modules, add them to PIGWEED_MODULES.
59
60If the listed modules are not actual Pigweed modules, remove any stray pw_*
61directories in the Pigweed repository (git clean -fd).
62'''
63)
64
65_OUT_OF_DATE_WARNING = (
66    _ERROR
67    + '''\
68The generated Pigweed modules list .gni file is out of date!
69
70Regenerate the modules lists and commit it to fix this:
71
72  ninja -C {out_dir} update_modules
73
74  git add {file}
75'''
76)
77
78_FORMAT_FAILED_WARNING = (
79    _ERROR
80    + '''\
81Failed to generate a valid .gni from PIGWEED_MODULES!
82
83This may be a Pigweed bug; please report this to the Pigweed team.
84'''
85)
86
87_DO_NOT_SET = 'DO NOT SET THIS BUILD ARGUMENT!'
88
89
90def _module_list_warnings(root: Path, modules: Sequence[str]) -> Iterator[str]:
91    missing = _missing_modules(root, modules)
92    if missing:
93        yield _MISSING_MODULES_WARNING.format(
94            modules=''.join(f'\n  - {module}' for module in missing)
95        )
96
97    if any(modules[i] > modules[i + 1] for i in range(len(modules) - 1)):
98        yield _WARNING + 'The PIGWEED_MODULES list is not sorted!'
99        yield ''
100        yield 'Apply the following diff to fix the order:'
101        yield ''
102        yield from difflib.unified_diff(
103            modules,
104            sorted(modules),
105            lineterm='',
106            n=1,
107            fromfile='PIGWEED_MODULES',
108            tofile='PIGWEED_MODULES',
109        )
110
111        yield ''
112
113
114def _generate_modules_gni(
115    prefix: Path, modules: Sequence[str]
116) -> Iterator[str]:
117    """Generates a .gni file with variables and lists for Pigweed modules."""
118    script_path = Path(__file__).resolve()
119    script = script_path.relative_to(script_path.parent.parent).as_posix()
120
121    yield _COPYRIGHT_NOTICE
122    yield ''
123    yield '# Build args and lists for all modules in Pigweed.'
124    yield '#'
125    yield f'# DO NOT EDIT! Generated by {script}.'
126    yield '#'
127    yield '# To add modules here, list them in PIGWEED_MODULES and build the'
128    yield '# update_modules target and commit the updated version of this file:'
129    yield '#'
130    yield '#   ninja -C out update_modules'
131    yield '#'
132    yield '# DO NOT IMPORT THIS FILE DIRECTLY!'
133    yield '#'
134    yield '# Import it through //build_overrides/pigweed.gni instead.'
135    yield ''
136    yield '# Declare a build arg for each module.'
137    yield 'declare_args() {'
138
139    for module in modules:
140        module_path = prefix.joinpath(module).as_posix()
141        yield f'dir_{module} = get_path_info("{module_path}", "abspath")'
142
143    yield '}'
144    yield ''
145    yield '# Declare these as GN args in case this is imported in args.gni.'
146    yield '# Use a separate block so variables in the prior block can be used.'
147    yield 'declare_args() {'
148    yield f'# A list with paths to all Pigweed module. {_DO_NOT_SET}'
149    yield 'pw_modules = ['
150
151    for module in modules:
152        yield f'dir_{module},'
153
154    yield ']'
155    yield ''
156
157    yield f'# A list with all Pigweed module test groups. {_DO_NOT_SET}'
158    yield 'pw_module_tests = ['
159
160    for module in modules:
161        yield f'"$dir_{module}:tests",'
162
163    yield ']'
164    yield ''
165    yield f'# A list with all Pigweed modules docs groups. {_DO_NOT_SET}'
166    yield 'pw_module_docs = ['
167
168    for module in modules:
169        yield f'"$dir_{module}:docs",'
170
171    yield ']'
172    yield ''
173    yield '}'
174
175
176def _missing_modules(root: Path, modules: Sequence[str]) -> Sequence[str]:
177    return sorted(
178        frozenset(
179            str(p.relative_to(root)) for p in root.glob('pw_*') if p.is_dir()
180        )
181        - frozenset(modules)
182    )
183
184
185class Mode(enum.Enum):
186    WARN = 0  # Warn if anything is out of date
187    CHECK = 1  # Fail if anything is out of date
188    UPDATE = 2  # Update the generated modules lists
189
190
191def _parse_args() -> dict:
192    parser = argparse.ArgumentParser(
193        description=__doc__,
194        formatter_class=argparse.RawDescriptionHelpFormatter,
195    )
196    parser.add_argument('root', type=Path, help='Root build dir')
197    parser.add_argument('modules_list', type=Path, help='Input modules list')
198    parser.add_argument('modules_gni_file', type=Path, help='Output .gni file')
199    parser.add_argument(
200        '--mode', type=Mode.__getitem__, choices=Mode, required=True
201    )
202    parser.add_argument(
203        '--stamp',
204        type=Path,
205        help='Stamp file for operations that should only run once (warn)',
206    )
207    return vars(parser.parse_args())
208
209
210def main(
211    root: Path,
212    modules_list: Path,
213    modules_gni_file: Path,
214    mode: Mode,
215    stamp: Optional[Path] = None,
216) -> int:
217    """Manages the list of Pigweed modules."""
218    prefix = Path(os.path.relpath(root, modules_gni_file.parent))
219    modules = modules_list.read_text().splitlines()
220
221    # Detect any problems with the modules list.
222    warnings = list(_module_list_warnings(root, modules))
223    errors = []
224
225    modules.sort()  # Sort in case the modules list in case it wasn't sorted.
226
227    # Check if the contents of the .gni file are out of date.
228    if mode in (Mode.WARN, Mode.CHECK):
229        text = io.StringIO()
230        for line in _generate_modules_gni(prefix, modules):
231            print(line, file=text)
232
233        process = subprocess.run(
234            ['gn', 'format', '--stdin'],
235            input=text.getvalue().encode('utf-8'),
236            stdout=subprocess.PIPE,
237        )
238        if process.returncode != 0:
239            errors.append(_FORMAT_FAILED_WARNING)
240
241        # Make a diff of required changes
242        modules_gni_relpath = os.path.relpath(modules_gni_file, root)
243        diff = list(
244            difflib.unified_diff(
245                modules_gni_file.read_text().splitlines(),
246                process.stdout.decode('utf-8', errors='replace').splitlines(),
247                fromfile=os.path.join('a', modules_gni_relpath),
248                tofile=os.path.join('b', modules_gni_relpath),
249                lineterm='',
250                n=1,
251            )
252        )
253        # If any differences were found, print the error and the diff.
254        if diff:
255            errors.append(
256                _OUT_OF_DATE_WARNING.format(
257                    out_dir=os.path.relpath(os.curdir, root),
258                    file=os.path.relpath(modules_gni_file, root),
259                )
260            )
261            errors.append('Expected Diff:\n')
262            errors.append('\n'.join(diff))
263            errors.append('\n')
264
265    elif mode is Mode.UPDATE:  # Update the modules .gni file
266        with modules_gni_file.open('w', encoding='utf-8') as file:
267            for line in _generate_modules_gni(prefix, modules):
268                print(line, file=file)
269
270        process = subprocess.run(
271            ['gn', 'format', modules_gni_file], stdout=subprocess.DEVNULL
272        )
273        if process.returncode != 0:
274            errors.append(_FORMAT_FAILED_WARNING)
275
276    # If there are errors, display them and abort.
277    if warnings or errors:
278        for line in warnings + errors:
279            print(line, file=sys.stderr)
280
281        # Delete the stamp so this always reruns. Deleting is necessary since
282        # some of the checks do not depend on input files.
283        if stamp and stamp.exists():
284            stamp.unlink()
285
286        if mode is Mode.WARN:
287            return 0
288
289        if mode is Mode.CHECK:
290            return 1
291
292        return 1 if errors else 0  # Allow warnings but not errors when updating
293
294    if stamp:
295        stamp.touch()
296
297    return 0
298
299
300if __name__ == '__main__':
301    sys.exit(main(**_parse_args()))
302