• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1# Copyright 2022 The Pigweed Authors
2#
3# Licensed under the Apache License, Version 2.0 (the "License"); you may not
4# use this file except in compliance with the License. You may obtain a copy of
5# the License at
6#
7#     https://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
11# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
12# License for the specific language governing permissions and limitations under
13# the License.
14"""Executes a compilation failure test."""
15
16import argparse
17import logging
18from pathlib import Path
19import re
20import shlex
21import string
22import sys
23import subprocess
24
25import pw_cli.log
26
27from pw_compilation_testing.generator import Compiler, Expectation, TestCase
28
29_LOG = logging.getLogger(__package__)
30
31_RULE_REGEX = re.compile('^rule (?:cxx|.*_cxx)$')
32_NINJA_VARIABLE = re.compile('^([a-zA-Z0-9_]+) = ?')
33
34
35# TODO(hepler): Could do this step just once and output the results.
36def find_cc_rule(toolchain_ninja_file: Path) -> str | None:
37    """Searches the toolchain.ninja file for the cc rule."""
38    cmd_prefix = '  command = '
39
40    found_rule = False
41
42    with toolchain_ninja_file.open() as fd:
43        for line in fd:
44            if found_rule:
45                if line.startswith(cmd_prefix):
46                    cmd = line[len(cmd_prefix) :].strip()
47                    if cmd.startswith('ccache '):
48                        cmd = cmd[len('ccache ') :]
49                    return cmd
50
51                if not line.startswith('  '):
52                    break
53            elif _RULE_REGEX.match(line):
54                found_rule = True
55
56    return None
57
58
59def _parse_ninja_variables(target_ninja_file: Path) -> dict[str, str]:
60    variables: dict[str, str] = {}
61
62    with target_ninja_file.open() as fd:
63        for line in fd:
64            match = _NINJA_VARIABLE.match(line)
65            if match:
66                variables[match.group(1)] = line[match.end() :].strip()
67
68    return variables
69
70
71_EXPECTED_GN_VARS = (
72    'asmflags',
73    'cflags',
74    'cflags_c',
75    'cflags_cc',
76    'cflags_objc',
77    'cflags_objcc',
78    'defines',
79    'include_dirs',
80)
81
82_TEST_MACRO = 'PW_NC_TEST_EXECUTE_CASE_'
83# Regular expression to find and remove ANSI escape sequences, based on
84# https://stackoverflow.com/a/14693789.
85_ANSI_ESCAPE_SEQUENCES = re.compile(r'\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])')
86
87
88class _TestFailure(Exception):
89    pass
90
91
92def _red(message: str) -> str:
93    return f'\033[31m\033[1m{message}\033[0m'
94
95
96_TITLE_1 = '     NEGATIVE     '
97_TITLE_2 = ' COMPILATION TEST '
98
99_BOX_TOP = f'┏{"━" * len(_TITLE_1)}┓'
100_BOX_MID_1 = f'┃{_red(_TITLE_1)}┃ \033[1m{{test_name}}\033[0m'
101_BOX_MID_2 = f'┃{_red(_TITLE_2)}┃ \033[0m{{source}}:{{line}}\033[0m'
102_BOX_BOT = f'┻{"━" * (len(_TITLE_1))}┻{"━" * (77 - len(_TITLE_1))}┓'
103_FOOTER = '\n' + '━' * 79 + '┛'
104
105
106def _start_failure(test: TestCase, command: str) -> None:
107    print(_BOX_TOP, file=sys.stderr)
108    print(_BOX_MID_1.format(test_name=test.name()), file=sys.stderr)
109    print(
110        _BOX_MID_2.format(source=test.source, line=test.line), file=sys.stderr
111    )
112    print(_BOX_BOT, file=sys.stderr)
113    print(file=sys.stderr)
114
115    _LOG.debug('Compilation command:\n%s', command)
116
117
118def _check_results(
119    test: TestCase, command: str, process: subprocess.CompletedProcess
120) -> None:
121    stderr = process.stderr.decode(errors='replace')
122
123    if process.returncode == 0:
124        _start_failure(test, command)
125        _LOG.error('Compilation succeeded, but it should have failed!')
126        _LOG.error('Update the test code so that is fails to compile.')
127        _LOG.error('Compilation command:\n%s', command)
128        raise _TestFailure
129
130    compiler_str = command.split(' ', 1)[0]
131    compiler = Compiler.from_command(compiler_str)
132
133    _LOG.debug('%s is %s', compiler_str, compiler)
134    expectations: list[Expectation] = [
135        e for e in test.expectations if compiler.matches(e.compiler)
136    ]
137
138    _LOG.debug(
139        '%s: Checking compilation from %s (%s) for %d of %d patterns:',
140        test.name(),
141        compiler_str,
142        compiler,
143        len(expectations),
144        len(test.expectations),
145    )
146    for expectation in expectations:
147        _LOG.debug('    %s', expectation.pattern.pattern)
148
149    if not expectations:
150        _start_failure(test, command)
151        _LOG.error(
152            'Compilation with %s failed, but no PW_NC_EXPECT() patterns '
153            'that apply to %s were provided',
154            compiler_str,
155            compiler_str,
156        )
157
158        _LOG.error('Compilation output:\n%s', stderr)
159        _LOG.error('')
160        _LOG.error(
161            'Add at least one PW_NC_EXPECT("<regex>") or '
162            'PW_NC_EXPECT_%s("<regex>") expectation to %s',
163            compiler.name,
164            test.case,
165        )
166        raise _TestFailure
167
168    no_color = _ANSI_ESCAPE_SEQUENCES.sub('', stderr)
169
170    failed = [e for e in expectations if not e.pattern.search(no_color)]
171    if failed:
172        _start_failure(test, command)
173        _LOG.error(
174            'Compilation with %s failed, but the output did not '
175            'match the expected patterns.',
176            compiler_str,
177        )
178        _LOG.error(
179            '%d of %d expected patterns did not match:',
180            len(failed),
181            len(expectations),
182        )
183        _LOG.error('')
184        for expectation in expectations:
185            _LOG.error(
186                '  %s %s:%d: %s',
187                '❌' if expectation in failed else '✅',
188                test.source.name,
189                expectation.line,
190                expectation.pattern.pattern,
191            )
192        _LOG.error('')
193
194        _LOG.error('Compilation output:\n%s', stderr)
195        _LOG.error('')
196        _LOG.error(
197            'Update the test so that compilation fails with the '
198            'expected output'
199        )
200        raise _TestFailure
201
202
203def _should_skip_test(base_command: str) -> bool:
204    # Attempt to run the preprocessor while setting the PW_NC_TEST macro to an
205    # illegal statement (defined() with no identifier). If the preprocessor
206    # passes, the test was skipped.
207    preprocessor_cmd = f'{base_command}{shlex.quote("defined()")} -E'
208    process = subprocess.run(
209        preprocessor_cmd,
210        shell=True,
211        stdout=subprocess.DEVNULL,
212        stderr=subprocess.DEVNULL,
213    )
214    _LOG.debug(
215        'Preprocessor command to check if test is enabled returned %d:\n%s',
216        process.returncode,
217        preprocessor_cmd,
218    )
219    return process.returncode == 0
220
221
222def _execute_test(
223    test: TestCase,
224    command: str,
225    variables: dict[str, str],
226    all_tests: list[str],
227) -> None:
228    variables['in'] = str(test.source)
229
230    base_command = ' '.join(
231        [
232            string.Template(command).substitute(variables),
233            '-DPW_NEGATIVE_COMPILATION_TESTS_ENABLED',
234            # Define macros to disable all tests except this one.
235            *(f'-D{_TEST_MACRO}{t}=0' for t in all_tests if t != test.case),
236            f'-D{_TEST_MACRO}{test.case}=',
237        ]
238    )
239
240    if _should_skip_test(base_command):
241        _LOG.info(
242            "Skipping test %s since it is excluded by the preprocessor",
243            test.name(),
244        )
245        return
246
247    compile_command = base_command + '1'  # set macro to 1 to enable the test
248    process = subprocess.run(compile_command, shell=True, capture_output=True)
249    _check_results(test, compile_command, process)
250
251
252def _main(
253    test: TestCase, toolchain_ninja: Path, target_ninja: Path, all_tests: Path
254) -> int:
255    """Compiles a compile fail test and returns 1 if compilation succeeds."""
256    command = find_cc_rule(toolchain_ninja)
257
258    if command is None:
259        _LOG.critical(
260            'Failed to find C++ compilation command in %s', toolchain_ninja
261        )
262        return 2
263
264    variables = {key: '' for key in _EXPECTED_GN_VARS}
265    variables.update(_parse_ninja_variables(target_ninja))
266
267    variables['out'] = str(
268        target_ninja.parent / f'{target_ninja.stem}.compile_fail_test.out'
269    )
270
271    try:
272        _execute_test(
273            test, command, variables, all_tests.read_text().splitlines()
274        )
275    except _TestFailure:
276        print(_FOOTER, file=sys.stderr)
277        return 1
278
279    return 0
280
281
282def _parse_args() -> dict:
283    """Parses command-line arguments."""
284
285    parser = argparse.ArgumentParser(
286        description='Emits an error when a facade has a null backend'
287    )
288    parser.add_argument(
289        '--toolchain-ninja',
290        type=Path,
291        required=True,
292        help='Ninja file with the compilation command for the toolchain',
293    )
294    parser.add_argument(
295        '--target-ninja',
296        type=Path,
297        required=True,
298        help='Ninja file with the compilation commands to the test target',
299    )
300    parser.add_argument(
301        '--test-data',
302        dest='test',
303        required=True,
304        type=TestCase.deserialize,
305        help='Serialized TestCase object',
306    )
307    parser.add_argument('--all-tests', type=Path, help='List of all tests')
308    return vars(parser.parse_args())
309
310
311if __name__ == '__main__':
312    pw_cli.log.install(level=logging.INFO)
313    sys.exit(_main(**_parse_args()))
314