• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1# Copyright 2020 The Pigweed Authors
2#
3# Licensed under the Apache License, Version 2.0 (the "License"); you may not
4# use this file except in compliance with the License. You may obtain a copy of
5# the License at
6#
7#     https://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
11# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
12# License for the specific language governing permissions and limitations under
13# the License.
14"""Functions for building code during presubmit checks."""
15
16import collections
17import itertools
18import json
19import logging
20import os
21from pathlib import Path
22import re
23from typing import (Collection, Container, Dict, Iterable, List, Mapping, Set,
24                    Tuple, Union)
25
26from pw_package import package_manager
27from pw_presubmit import call, log_run, plural, PresubmitFailure, tools
28
29_LOG = logging.getLogger(__name__)
30
31
32def install_package(root: Path, name: str) -> None:
33    """Install package with given name in given path."""
34    mgr = package_manager.PackageManager(root)
35
36    if not mgr.list():
37        raise PresubmitFailure(
38            'no packages configured, please import your pw_package '
39            'configuration module')
40
41    if not mgr.status(name):
42        mgr.install(name)
43
44
45def gn_args(**kwargs) -> str:
46    """Builds a string to use for the --args argument to gn gen.
47
48    Currently supports bool, int, and str values. In the case of str values,
49    quotation marks will be added automatically, unless the string already
50    contains one or more double quotation marks, or starts with a { or [
51    character, in which case it will be passed through as-is.
52    """
53    transformed_args = []
54    for arg, val in kwargs.items():
55        if isinstance(val, bool):
56            transformed_args.append(f'{arg}={str(val).lower()}')
57            continue
58        if (isinstance(val, str) and '"' not in val and not val.startswith("{")
59                and not val.startswith("[")):
60            transformed_args.append(f'{arg}="{val}"')
61            continue
62        # Fall-back case handles integers as well as strings that already
63        # contain double quotation marks, or look like scopes or lists.
64        transformed_args.append(f'{arg}={val}')
65    return '--args=' + ' '.join(transformed_args)
66
67
68def gn_gen(gn_source_dir: Path,
69           gn_output_dir: Path,
70           *args: str,
71           gn_check: bool = True,
72           gn_fail_on_unused: bool = True,
73           **gn_arguments) -> None:
74    """Runs gn gen in the specified directory with optional GN args."""
75    args_option = (gn_args(**gn_arguments), ) if gn_arguments else ()
76
77    # Delete args.gn to ensure this is a clean build.
78    args_gn = gn_output_dir / 'args.gn'
79    if args_gn.is_file():
80        args_gn.unlink()
81
82    call('gn',
83         'gen',
84         gn_output_dir,
85         '--color=always',
86         *(['--check'] if gn_check else []),
87         *(['--fail-on-unused-args'] if gn_fail_on_unused else []),
88         *args,
89         *args_option,
90         cwd=gn_source_dir)
91
92
93def ninja(directory: Path, *args, **kwargs) -> None:
94    """Runs ninja in the specified directory."""
95    call('ninja', '-C', directory, *args, **kwargs)
96
97
98def cmake(source_dir: Path,
99          output_dir: Path,
100          *args: str,
101          env: Mapping['str', 'str'] = None) -> None:
102    """Runs CMake for Ninja on the given source and output directories."""
103    call('cmake',
104         '-B',
105         output_dir,
106         '-S',
107         source_dir,
108         '-G',
109         'Ninja',
110         *args,
111         env=env)
112
113
114def env_with_clang_vars() -> Mapping[str, str]:
115    """Returns the environment variables with CC, CXX, etc. set for clang."""
116    env = os.environ.copy()
117    env['CC'] = env['LD'] = env['AS'] = 'clang'
118    env['CXX'] = 'clang++'
119    return env
120
121
122def _get_paths_from_command(source_dir: Path, *args, **kwargs) -> Set[Path]:
123    """Runs a command and reads Bazel or GN //-style paths from it."""
124    process = log_run(args, capture_output=True, cwd=source_dir, **kwargs)
125
126    if process.returncode:
127        _LOG.error('Build invocation failed with return code %d!',
128                   process.returncode)
129        _LOG.error('[COMMAND] %s\n%s\n%s', *tools.format_command(args, kwargs),
130                   process.stderr.decode())
131        raise PresubmitFailure
132
133    files = set()
134
135    for line in process.stdout.splitlines():
136        path = line.strip().lstrip(b'/').replace(b':', b'/').decode()
137        path = source_dir.joinpath(path)
138        if path.is_file():
139            files.add(path)
140
141    return files
142
143
144# Finds string literals with '.' in them.
145_MAYBE_A_PATH = re.compile(r'"([^\n"]+\.[^\n"]+)"')
146
147
148def _search_files_for_paths(build_files: Iterable[Path]) -> Iterable[Path]:
149    for build_file in build_files:
150        directory = build_file.parent
151
152        for string in _MAYBE_A_PATH.finditer(build_file.read_text()):
153            path = directory / string.group(1)
154            if path.is_file():
155                yield path
156
157
158def _read_compile_commands(compile_commands: Path) -> dict:
159    with compile_commands.open('rb') as fd:
160        return json.load(fd)
161
162
163def compiled_files(compile_commands: Path) -> Iterable[Path]:
164    for command in _read_compile_commands(compile_commands):
165        file = Path(command['file'])
166        if file.is_absolute():
167            yield file
168        else:
169            yield file.joinpath(command['directory']).resolve()
170
171
172def check_compile_commands_for_files(
173        compile_commands: Union[Path, Iterable[Path]],
174        files: Iterable[Path],
175        extensions: Collection[str] = ('.c', '.cc', '.cpp'),
176) -> List[Path]:
177    """Checks for paths in one or more compile_commands.json files.
178
179    Only checks C and C++ source files by default.
180    """
181    if isinstance(compile_commands, Path):
182        compile_commands = [compile_commands]
183
184    compiled = frozenset(
185        itertools.chain.from_iterable(
186            compiled_files(cmds) for cmds in compile_commands))
187    return [f for f in files if f not in compiled and f.suffix in extensions]
188
189
190def check_builds_for_files(
191        bazel_extensions_to_check: Container[str],
192        gn_extensions_to_check: Container[str],
193        files: Iterable[Path],
194        bazel_dirs: Iterable[Path] = (),
195        gn_dirs: Iterable[Tuple[Path, Path]] = (),
196        gn_build_files: Iterable[Path] = (),
197) -> Dict[str, List[Path]]:
198    """Checks that source files are in the GN and Bazel builds.
199
200    Args:
201        bazel_extensions_to_check: which file suffixes to look for in Bazel
202        gn_extensions_to_check: which file suffixes to look for in GN
203        files: the files that should be checked
204        bazel_dirs: directories in which to run bazel query
205        gn_dirs: (source_dir, output_dir) tuples with which to run gn desc
206        gn_build_files: paths to BUILD.gn files to directly search for paths
207
208    Returns:
209        a dictionary mapping build system ('Bazel' or 'GN' to a list of missing
210        files; will be empty if there were no missing files
211    """
212
213    # Collect all paths in the Bazel builds.
214    bazel_builds: Set[Path] = set()
215    for directory in bazel_dirs:
216        bazel_builds.update(
217            _get_paths_from_command(directory, 'bazel', 'query',
218                                    'kind("source file", //...:*)'))
219
220    # Collect all paths in GN builds.
221    gn_builds: Set[Path] = set()
222
223    for source_dir, output_dir in gn_dirs:
224        gn_builds.update(
225            _get_paths_from_command(source_dir, 'gn', 'desc', output_dir, '*'))
226
227    gn_builds.update(_search_files_for_paths(gn_build_files))
228
229    missing: Dict[str, List[Path]] = collections.defaultdict(list)
230
231    if bazel_dirs:
232        for path in (p for p in files
233                     if p.suffix in bazel_extensions_to_check):
234            if path not in bazel_builds:
235                # TODO(pwbug/176) Replace this workaround for fuzzers.
236                if 'fuzz' not in str(path):
237                    missing['Bazel'].append(path)
238
239    if gn_dirs or gn_build_files:
240        for path in (p for p in files if p.suffix in gn_extensions_to_check):
241            if path not in gn_builds:
242                missing['GN'].append(path)
243
244    for builder, paths in missing.items():
245        _LOG.warning('%s missing from the %s build:\n%s',
246                     plural(paths, 'file', are=True), builder,
247                     '\n'.join(str(x) for x in paths))
248
249    return missing
250