• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1# Copyright 2022 The Pigweed Authors
2#
3# Licensed under the Apache License, Version 2.0 (the "License"); you may not
4# use this file except in compliance with the License. You may obtain a copy of
5# the License at
6#
7#     https://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
11# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
12# License for the specific language governing permissions and limitations under
13# the License.
14"""Executes a compilation failure test."""
15
16import argparse
17import logging
18from pathlib import Path
19import re
20import string
21import sys
22import subprocess
23from typing import Dict, List, Optional
24
25import pw_cli.log
26
27from pw_compilation_testing.generator import Compiler, Expectation, TestCase
28
29_LOG = logging.getLogger(__package__)
30
31_RULE_REGEX = re.compile('^rule (?:cxx|.*_cxx)$')
32_NINJA_VARIABLE = re.compile('^([a-zA-Z0-9_]+) = ?')
33
34
35# TODO(hepler): Could do this step just once and output the results.
36def find_cc_rule(toolchain_ninja_file: Path) -> Optional[str]:
37    """Searches the toolchain.ninja file for the cc rule."""
38    cmd_prefix = '  command = '
39
40    found_rule = False
41
42    with toolchain_ninja_file.open() as fd:
43        for line in fd:
44            if found_rule:
45                if line.startswith(cmd_prefix):
46                    cmd = line[len(cmd_prefix) :].strip()
47                    if cmd.startswith('ccache '):
48                        cmd = cmd[len('ccache ') :]
49                    return cmd
50
51                if not line.startswith('  '):
52                    break
53            elif _RULE_REGEX.match(line):
54                found_rule = True
55
56    return None
57
58
59def _parse_ninja_variables(target_ninja_file: Path) -> Dict[str, str]:
60    variables: Dict[str, str] = {}
61
62    with target_ninja_file.open() as fd:
63        for line in fd:
64            match = _NINJA_VARIABLE.match(line)
65            if match:
66                variables[match.group(1)] = line[match.end() :].strip()
67
68    return variables
69
70
71_EXPECTED_GN_VARS = (
72    'asmflags',
73    'cflags',
74    'cflags_c',
75    'cflags_cc',
76    'cflags_objc',
77    'cflags_objcc',
78    'defines',
79    'include_dirs',
80)
81
82_ENABLE_TEST_MACRO = '-DPW_NC_TEST_EXECUTE_CASE_'
83# Regular expression to find and remove ANSI escape sequences, based on
84# https://stackoverflow.com/a/14693789.
85_ANSI_ESCAPE_SEQUENCES = re.compile(r'\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])')
86
87
88class _TestFailure(Exception):
89    pass
90
91
92def _red(message: str) -> str:
93    return f'\033[31m\033[1m{message}\033[0m'
94
95
96_TITLE_1 = '     NEGATIVE     '
97_TITLE_2 = ' COMPILATION TEST '
98
99_BOX_TOP = f'┏{"━" * len(_TITLE_1)}┓'
100_BOX_MID_1 = f'┃{_red(_TITLE_1)}┃ \033[1m{{test_name}}\033[0m'
101_BOX_MID_2 = f'┃{_red(_TITLE_2)}┃ \033[0m{{source}}:{{line}}\033[0m'
102_BOX_BOT = f'┻{"━" * (len(_TITLE_1))}┻{"━" * (77 - len(_TITLE_1))}┓'
103_FOOTER = '\n' + '━' * 79 + '┛'
104
105
106def _start_failure(test: TestCase, command: str) -> None:
107    print(_BOX_TOP, file=sys.stderr)
108    print(_BOX_MID_1.format(test_name=test.name()), file=sys.stderr)
109    print(
110        _BOX_MID_2.format(source=test.source, line=test.line), file=sys.stderr
111    )
112    print(_BOX_BOT, file=sys.stderr)
113    print(file=sys.stderr)
114
115    _LOG.debug('Compilation command:\n%s', command)
116
117
118def _check_results(
119    test: TestCase, command: str, process: subprocess.CompletedProcess
120) -> None:
121    stderr = process.stderr.decode(errors='replace')
122
123    if process.returncode == 0:
124        _start_failure(test, command)
125        _LOG.error('Compilation succeeded, but it should have failed!')
126        _LOG.error('Update the test code so that is fails to compile.')
127        raise _TestFailure
128
129    compiler_str = command.split(' ', 1)[0]
130    compiler = Compiler.from_command(compiler_str)
131
132    _LOG.debug('%s is %s', compiler_str, compiler)
133    expectations: List[Expectation] = [
134        e for e in test.expectations if compiler.matches(e.compiler)
135    ]
136
137    _LOG.debug(
138        '%s: Checking compilation from %s (%s) for %d of %d patterns:',
139        test.name(),
140        compiler_str,
141        compiler,
142        len(expectations),
143        len(test.expectations),
144    )
145    for expectation in expectations:
146        _LOG.debug('    %s', expectation.pattern.pattern)
147
148    if not expectations:
149        _start_failure(test, command)
150        _LOG.error(
151            'Compilation with %s failed, but no PW_NC_EXPECT() patterns '
152            'that apply to %s were provided',
153            compiler_str,
154            compiler_str,
155        )
156
157        _LOG.error('Compilation output:\n%s', stderr)
158        _LOG.error('')
159        _LOG.error(
160            'Add at least one PW_NC_EXPECT("<regex>") or '
161            'PW_NC_EXPECT_%s("<regex>") expectation to %s',
162            compiler.name,
163            test.case,
164        )
165        raise _TestFailure
166
167    no_color = _ANSI_ESCAPE_SEQUENCES.sub('', stderr)
168
169    failed = [e for e in expectations if not e.pattern.search(no_color)]
170    if failed:
171        _start_failure(test, command)
172        _LOG.error(
173            'Compilation with %s failed, but the output did not '
174            'match the expected patterns.',
175            compiler_str,
176        )
177        _LOG.error(
178            '%d of %d expected patterns did not match:',
179            len(failed),
180            len(expectations),
181        )
182        _LOG.error('')
183        for expectation in expectations:
184            _LOG.error(
185                '  %s %s:%d: %s',
186                '❌' if expectation in failed else '✅',
187                test.source.name,
188                expectation.line,
189                expectation.pattern.pattern,
190            )
191        _LOG.error('')
192
193        _LOG.error('Compilation output:\n%s', stderr)
194        _LOG.error('')
195        _LOG.error(
196            'Update the test so that compilation fails with the '
197            'expected output'
198        )
199        raise _TestFailure
200
201
202def _execute_test(
203    test: TestCase,
204    command: str,
205    variables: Dict[str, str],
206    all_tests: List[str],
207) -> None:
208    variables['in'] = str(test.source)
209
210    command = string.Template(command).substitute(variables)
211    command = ' '.join(
212        [
213            command,
214            '-DPW_NEGATIVE_COMPILATION_TESTS_ENABLED',
215            # Define macros to disable all tests except this one.
216            *(
217                f'{_ENABLE_TEST_MACRO}{t}={1 if test.case == t else 0}'
218                for t in all_tests
219            ),
220        ]
221    )
222    process = subprocess.run(command, shell=True, capture_output=True)
223
224    _check_results(test, command, process)
225
226
227def _main(
228    test: TestCase, toolchain_ninja: Path, target_ninja: Path, all_tests: Path
229) -> int:
230    """Compiles a compile fail test and returns 1 if compilation succeeds."""
231    command = find_cc_rule(toolchain_ninja)
232
233    if command is None:
234        _LOG.critical(
235            'Failed to find C++ compilation command in %s', toolchain_ninja
236        )
237        return 2
238
239    variables = {key: '' for key in _EXPECTED_GN_VARS}
240    variables.update(_parse_ninja_variables(target_ninja))
241
242    variables['out'] = str(
243        target_ninja.parent / f'{target_ninja.stem}.compile_fail_test.out'
244    )
245
246    try:
247        _execute_test(
248            test, command, variables, all_tests.read_text().splitlines()
249        )
250    except _TestFailure:
251        print(_FOOTER, file=sys.stderr)
252        return 1
253
254    return 0
255
256
257def _parse_args() -> dict:
258    """Parses command-line arguments."""
259
260    parser = argparse.ArgumentParser(
261        description='Emits an error when a facade has a null backend'
262    )
263    parser.add_argument(
264        '--toolchain-ninja',
265        type=Path,
266        required=True,
267        help='Ninja file with the compilation command for the toolchain',
268    )
269    parser.add_argument(
270        '--target-ninja',
271        type=Path,
272        required=True,
273        help='Ninja file with the compilation commands to the test target',
274    )
275    parser.add_argument(
276        '--test-data',
277        dest='test',
278        required=True,
279        type=TestCase.deserialize,
280        help='Serialized TestCase object',
281    )
282    parser.add_argument('--all-tests', type=Path, help='List of all tests')
283    return vars(parser.parse_args())
284
285
286if __name__ == '__main__':
287    pw_cli.log.install(level=logging.INFO)
288    sys.exit(_main(**_parse_args()))
289