• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1# Copyright 2020 The Pigweed Authors
2#
3# Licensed under the Apache License, Version 2.0 (the "License"); you may not
4# use this file except in compliance with the License. You may obtain a copy of
5# the License at
6#
7#     https://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
11# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
12# License for the specific language governing permissions and limitations under
13# the License.
14"""Functions for building code during presubmit checks."""
15
16import collections
17import contextlib
18import itertools
19import json
20import logging
21import os
22from pathlib import Path
23import re
24import subprocess
25from shutil import which
26from typing import (Collection, Container, Dict, Iterable, List, Mapping, Set,
27                    Tuple, Union)
28
29from pw_package import package_manager
30from pw_presubmit import (
31    call,
32    Check,
33    filter_paths,
34    format_code,
35    log_run,
36    plural,
37    PresubmitContext,
38    PresubmitFailure,
39    tools,
40)
41
42_LOG = logging.getLogger(__name__)
43
44
45def bazel(ctx: PresubmitContext, cmd: str, *args: str) -> None:
46    """Invokes Bazel with some common flags set.
47
48    Intended for use with bazel build and test. May not work with others.
49    """
50    call('bazel',
51         cmd,
52         '--verbose_failures',
53         '--verbose_explanations',
54         '--worker_verbose',
55         f'--symlink_prefix={ctx.output_dir / ".bazel-"}',
56         *args,
57         cwd=ctx.root,
58         env=env_with_clang_vars())
59
60
61def install_package(root: Path, name: str) -> None:
62    """Install package with given name in given path."""
63    mgr = package_manager.PackageManager(root)
64
65    if not mgr.list():
66        raise PresubmitFailure(
67            'no packages configured, please import your pw_package '
68            'configuration module')
69
70    if not mgr.status(name):
71        mgr.install(name)
72
73
74def gn_args(**kwargs) -> str:
75    """Builds a string to use for the --args argument to gn gen.
76
77    Currently supports bool, int, and str values. In the case of str values,
78    quotation marks will be added automatically, unless the string already
79    contains one or more double quotation marks, or starts with a { or [
80    character, in which case it will be passed through as-is.
81    """
82    transformed_args = []
83    for arg, val in kwargs.items():
84        if isinstance(val, bool):
85            transformed_args.append(f'{arg}={str(val).lower()}')
86            continue
87        if (isinstance(val, str) and '"' not in val and not val.startswith("{")
88                and not val.startswith("[")):
89            transformed_args.append(f'{arg}="{val}"')
90            continue
91        # Fall-back case handles integers as well as strings that already
92        # contain double quotation marks, or look like scopes or lists.
93        transformed_args.append(f'{arg}={val}')
94    # Use ccache if available for faster repeat presubmit runs.
95    if which('ccache'):
96        transformed_args.append('pw_command_launcher="ccache"')
97
98    return '--args=' + ' '.join(transformed_args)
99
100
101def gn_gen(gn_source_dir: Path,
102           gn_output_dir: Path,
103           *args: str,
104           gn_check: bool = True,
105           gn_fail_on_unused: bool = True,
106           export_compile_commands: Union[bool, str] = True,
107           **gn_arguments) -> None:
108    """Runs gn gen in the specified directory with optional GN args."""
109    args_option = gn_args(**gn_arguments)
110
111    # Delete args.gn to ensure this is a clean build.
112    args_gn = gn_output_dir / 'args.gn'
113    if args_gn.is_file():
114        args_gn.unlink()
115
116    export_commands_arg = ''
117    if export_compile_commands:
118        export_commands_arg = '--export-compile-commands'
119        if isinstance(export_compile_commands, str):
120            export_commands_arg += f'={export_compile_commands}'
121
122    call('gn',
123         'gen',
124         gn_output_dir,
125         '--color=always',
126         *(['--fail-on-unused-args'] if gn_fail_on_unused else []),
127         *([export_commands_arg] if export_commands_arg else []),
128         *args,
129         args_option,
130         cwd=gn_source_dir)
131
132    if gn_check:
133        call('gn',
134             'check',
135             gn_output_dir,
136             '--check-generated',
137             '--check-system',
138             cwd=gn_source_dir)
139
140
141def ninja(directory: Path,
142          *args,
143          save_compdb=True,
144          save_graph=True,
145          **kwargs) -> None:
146    """Runs ninja in the specified directory."""
147    if save_compdb:
148        proc = subprocess.run(
149            ['ninja', '-C', directory, '-t', 'compdb', *args],
150            capture_output=True,
151            **kwargs)
152        (directory / 'ninja.compdb').write_bytes(proc.stdout)
153
154    if save_graph:
155        proc = subprocess.run(['ninja', '-C', directory, '-t', 'graph', *args],
156                              capture_output=True,
157                              **kwargs)
158        (directory / 'ninja.graph').write_bytes(proc.stdout)
159
160    call('ninja', '-C', directory, *args, **kwargs)
161    (directory / '.ninja_log').rename(directory / 'ninja.log')
162
163
164def get_gn_args(directory: Path) -> List[Dict[str, Dict[str, str]]]:
165    """Dumps GN variables to JSON."""
166    proc = subprocess.run(['gn', 'args', directory, '--list', '--json'],
167                          stdout=subprocess.PIPE)
168    return json.loads(proc.stdout)
169
170
171def cmake(source_dir: Path,
172          output_dir: Path,
173          *args: str,
174          env: Mapping['str', 'str'] = None) -> None:
175    """Runs CMake for Ninja on the given source and output directories."""
176    call('cmake',
177         '-B',
178         output_dir,
179         '-S',
180         source_dir,
181         '-G',
182         'Ninja',
183         *args,
184         env=env)
185
186
187def env_with_clang_vars() -> Mapping[str, str]:
188    """Returns the environment variables with CC, CXX, etc. set for clang."""
189    env = os.environ.copy()
190    env['CC'] = env['LD'] = env['AS'] = 'clang'
191    env['CXX'] = 'clang++'
192    return env
193
194
195def _get_paths_from_command(source_dir: Path, *args, **kwargs) -> Set[Path]:
196    """Runs a command and reads Bazel or GN //-style paths from it."""
197    process = log_run(args, capture_output=True, cwd=source_dir, **kwargs)
198
199    if process.returncode:
200        _LOG.error('Build invocation failed with return code %d!',
201                   process.returncode)
202        _LOG.error('[COMMAND] %s\n%s\n%s', *tools.format_command(args, kwargs),
203                   process.stderr.decode())
204        raise PresubmitFailure
205
206    files = set()
207
208    for line in process.stdout.splitlines():
209        path = line.strip().lstrip(b'/').replace(b':', b'/').decode()
210        path = source_dir.joinpath(path)
211        if path.is_file():
212            files.add(path)
213
214    return files
215
216
217# Finds string literals with '.' in them.
218_MAYBE_A_PATH = re.compile(
219    r'"'  # Starting double quote.
220    # Start capture group 1 - the whole filename:
221    #   File basename, a single period, file extension.
222    r'([^\n" ]+\.[^\n" ]+)'
223    # Non-capturing group 2 (optional).
224    r'(?: > [^\n"]+)?'  # pw_zip style string "input_file.txt > output_file.txt"
225    r'"'  # Ending double quote.
226)
227
228
229def _search_files_for_paths(build_files: Iterable[Path]) -> Iterable[Path]:
230    for build_file in build_files:
231        directory = build_file.parent
232
233        for string in _MAYBE_A_PATH.finditer(build_file.read_text()):
234            path = directory / string.group(1)
235            if path.is_file():
236                yield path
237
238
239def _read_compile_commands(compile_commands: Path) -> dict:
240    with compile_commands.open('rb') as fd:
241        return json.load(fd)
242
243
244def compiled_files(compile_commands: Path) -> Iterable[Path]:
245    for command in _read_compile_commands(compile_commands):
246        file = Path(command['file'])
247        if file.is_absolute():
248            yield file
249        else:
250            yield file.joinpath(command['directory']).resolve()
251
252
253def check_compile_commands_for_files(
254    compile_commands: Union[Path, Iterable[Path]],
255    files: Iterable[Path],
256    extensions: Collection[str] = format_code.CPP_SOURCE_EXTS,
257) -> List[Path]:
258    """Checks for paths in one or more compile_commands.json files.
259
260    Only checks C and C++ source files by default.
261    """
262    if isinstance(compile_commands, Path):
263        compile_commands = [compile_commands]
264
265    compiled = frozenset(
266        itertools.chain.from_iterable(
267            compiled_files(cmds) for cmds in compile_commands))
268    return [f for f in files if f not in compiled and f.suffix in extensions]
269
270
271def check_builds_for_files(
272        bazel_extensions_to_check: Container[str],
273        gn_extensions_to_check: Container[str],
274        files: Iterable[Path],
275        bazel_dirs: Iterable[Path] = (),
276        gn_dirs: Iterable[Tuple[Path, Path]] = (),
277        gn_build_files: Iterable[Path] = (),
278) -> Dict[str, List[Path]]:
279    """Checks that source files are in the GN and Bazel builds.
280
281    Args:
282        bazel_extensions_to_check: which file suffixes to look for in Bazel
283        gn_extensions_to_check: which file suffixes to look for in GN
284        files: the files that should be checked
285        bazel_dirs: directories in which to run bazel query
286        gn_dirs: (source_dir, output_dir) tuples with which to run gn desc
287        gn_build_files: paths to BUILD.gn files to directly search for paths
288
289    Returns:
290        a dictionary mapping build system ('Bazel' or 'GN' to a list of missing
291        files; will be empty if there were no missing files
292    """
293
294    # Collect all paths in the Bazel builds.
295    bazel_builds: Set[Path] = set()
296    for directory in bazel_dirs:
297        bazel_builds.update(
298            _get_paths_from_command(directory, 'bazel', 'query',
299                                    'kind("source file", //...:*)'))
300
301    # Collect all paths in GN builds.
302    gn_builds: Set[Path] = set()
303
304    for source_dir, output_dir in gn_dirs:
305        gn_builds.update(
306            _get_paths_from_command(source_dir, 'gn', 'desc', output_dir, '*'))
307
308    gn_builds.update(_search_files_for_paths(gn_build_files))
309
310    missing: Dict[str, List[Path]] = collections.defaultdict(list)
311
312    if bazel_dirs:
313        for path in (p for p in files
314                     if p.suffix in bazel_extensions_to_check):
315            if path not in bazel_builds:
316                # TODO(pwbug/176) Replace this workaround for fuzzers.
317                if 'fuzz' not in str(path):
318                    missing['Bazel'].append(path)
319
320    if gn_dirs or gn_build_files:
321        for path in (p for p in files if p.suffix in gn_extensions_to_check):
322            if path not in gn_builds:
323                missing['GN'].append(path)
324
325    for builder, paths in missing.items():
326        _LOG.warning('%s missing from the %s build:\n%s',
327                     plural(paths, 'file', are=True), builder,
328                     '\n'.join(str(x) for x in paths))
329
330    return missing
331
332
333@contextlib.contextmanager
334def test_server(executable: str, output_dir: Path):
335    """Context manager that runs a test server executable.
336
337    Args:
338        executable: name of the test server executable
339        output_dir: path to the output directory (for logs)
340    """
341
342    with open(output_dir / 'test_server.log', 'w') as outs:
343        try:
344            proc = subprocess.Popen(
345                [executable, '--verbose'],
346                stdout=outs,
347                stderr=subprocess.STDOUT,
348            )
349
350            yield
351
352        finally:
353            proc.terminate()
354
355
356@filter_paths(endswith=('.bzl', '.bazel'))
357def bazel_lint(ctx: PresubmitContext):
358    """Runs buildifier with lint on Bazel files.
359
360    Should be run after bazel_format since that will give more useful output
361    for formatting-only issues.
362    """
363
364    failure = False
365    for path in ctx.paths:
366        try:
367            call('buildifier', '--lint=warn', '--mode=check', path)
368        except PresubmitFailure:
369            failure = True
370
371    if failure:
372        raise PresubmitFailure
373
374
375@Check
376def gn_gen_check(ctx: PresubmitContext):
377    """Runs gn gen --check to enforce correct header dependencies."""
378    pw_project_root = Path(os.environ['PW_PROJECT_ROOT'])
379    gn_gen(pw_project_root, ctx.output_dir, gn_check=True)
380