# Copyright 2022 The Pigweed Authors # # Licensed under the Apache License, Version 2.0 (the "License"); you may not # use this file except in compliance with the License. You may obtain a copy of # the License at # # https://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations under # the License. """Executes a compilation failure test.""" import argparse import logging from pathlib import Path import re import string import sys import subprocess from typing import Dict, List, Optional import pw_cli.log from pw_compilation_testing.generator import Compiler, Expectation, TestCase _LOG = logging.getLogger(__package__) _RULE_REGEX = re.compile('^rule (?:cxx|.*_cxx)$') _NINJA_VARIABLE = re.compile('^([a-zA-Z0-9_]+) = ?') # TODO(hepler): Could do this step just once and output the results. def find_cc_rule(toolchain_ninja_file: Path) -> Optional[str]: """Searches the toolchain.ninja file for the cc rule.""" cmd_prefix = ' command = ' found_rule = False with toolchain_ninja_file.open() as fd: for line in fd: if found_rule: if line.startswith(cmd_prefix): cmd = line[len(cmd_prefix) :].strip() if cmd.startswith('ccache '): cmd = cmd[len('ccache ') :] return cmd if not line.startswith(' '): break elif _RULE_REGEX.match(line): found_rule = True return None def _parse_ninja_variables(target_ninja_file: Path) -> Dict[str, str]: variables: Dict[str, str] = {} with target_ninja_file.open() as fd: for line in fd: match = _NINJA_VARIABLE.match(line) if match: variables[match.group(1)] = line[match.end() :].strip() return variables _EXPECTED_GN_VARS = ( 'asmflags', 'cflags', 'cflags_c', 'cflags_cc', 'cflags_objc', 'cflags_objcc', 'defines', 'include_dirs', ) _ENABLE_TEST_MACRO = '-DPW_NC_TEST_EXECUTE_CASE_' # Regular expression to find and remove ANSI escape sequences, based on # https://stackoverflow.com/a/14693789. _ANSI_ESCAPE_SEQUENCES = re.compile(r'\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])') class _TestFailure(Exception): pass def _red(message: str) -> str: return f'\033[31m\033[1m{message}\033[0m' _TITLE_1 = ' NEGATIVE ' _TITLE_2 = ' COMPILATION TEST ' _BOX_TOP = f'┏{"━" * len(_TITLE_1)}┓' _BOX_MID_1 = f'┃{_red(_TITLE_1)}┃ \033[1m{{test_name}}\033[0m' _BOX_MID_2 = f'┃{_red(_TITLE_2)}┃ \033[0m{{source}}:{{line}}\033[0m' _BOX_BOT = f'┻{"━" * (len(_TITLE_1))}┻{"━" * (77 - len(_TITLE_1))}┓' _FOOTER = '\n' + '━' * 79 + '┛' def _start_failure(test: TestCase, command: str) -> None: print(_BOX_TOP, file=sys.stderr) print(_BOX_MID_1.format(test_name=test.name()), file=sys.stderr) print( _BOX_MID_2.format(source=test.source, line=test.line), file=sys.stderr ) print(_BOX_BOT, file=sys.stderr) print(file=sys.stderr) _LOG.debug('Compilation command:\n%s', command) def _check_results( test: TestCase, command: str, process: subprocess.CompletedProcess ) -> None: stderr = process.stderr.decode(errors='replace') if process.returncode == 0: _start_failure(test, command) _LOG.error('Compilation succeeded, but it should have failed!') _LOG.error('Update the test code so that is fails to compile.') raise _TestFailure compiler_str = command.split(' ', 1)[0] compiler = Compiler.from_command(compiler_str) _LOG.debug('%s is %s', compiler_str, compiler) expectations: List[Expectation] = [ e for e in test.expectations if compiler.matches(e.compiler) ] _LOG.debug( '%s: Checking compilation from %s (%s) for %d of %d patterns:', test.name(), compiler_str, compiler, len(expectations), len(test.expectations), ) for expectation in expectations: _LOG.debug(' %s', expectation.pattern.pattern) if not expectations: _start_failure(test, command) _LOG.error( 'Compilation with %s failed, but no PW_NC_EXPECT() patterns ' 'that apply to %s were provided', compiler_str, compiler_str, ) _LOG.error('Compilation output:\n%s', stderr) _LOG.error('') _LOG.error( 'Add at least one PW_NC_EXPECT("") or ' 'PW_NC_EXPECT_%s("") expectation to %s', compiler.name, test.case, ) raise _TestFailure no_color = _ANSI_ESCAPE_SEQUENCES.sub('', stderr) failed = [e for e in expectations if not e.pattern.search(no_color)] if failed: _start_failure(test, command) _LOG.error( 'Compilation with %s failed, but the output did not ' 'match the expected patterns.', compiler_str, ) _LOG.error( '%d of %d expected patterns did not match:', len(failed), len(expectations), ) _LOG.error('') for expectation in expectations: _LOG.error( ' %s %s:%d: %s', '❌' if expectation in failed else '✅', test.source.name, expectation.line, expectation.pattern.pattern, ) _LOG.error('') _LOG.error('Compilation output:\n%s', stderr) _LOG.error('') _LOG.error( 'Update the test so that compilation fails with the ' 'expected output' ) raise _TestFailure def _execute_test( test: TestCase, command: str, variables: Dict[str, str], all_tests: List[str], ) -> None: variables['in'] = str(test.source) command = string.Template(command).substitute(variables) command = ' '.join( [ command, '-DPW_NEGATIVE_COMPILATION_TESTS_ENABLED', # Define macros to disable all tests except this one. *( f'{_ENABLE_TEST_MACRO}{t}={1 if test.case == t else 0}' for t in all_tests ), ] ) process = subprocess.run(command, shell=True, capture_output=True) _check_results(test, command, process) def _main( test: TestCase, toolchain_ninja: Path, target_ninja: Path, all_tests: Path ) -> int: """Compiles a compile fail test and returns 1 if compilation succeeds.""" command = find_cc_rule(toolchain_ninja) if command is None: _LOG.critical( 'Failed to find C++ compilation command in %s', toolchain_ninja ) return 2 variables = {key: '' for key in _EXPECTED_GN_VARS} variables.update(_parse_ninja_variables(target_ninja)) variables['out'] = str( target_ninja.parent / f'{target_ninja.stem}.compile_fail_test.out' ) try: _execute_test( test, command, variables, all_tests.read_text().splitlines() ) except _TestFailure: print(_FOOTER, file=sys.stderr) return 1 return 0 def _parse_args() -> dict: """Parses command-line arguments.""" parser = argparse.ArgumentParser( description='Emits an error when a facade has a null backend' ) parser.add_argument( '--toolchain-ninja', type=Path, required=True, help='Ninja file with the compilation command for the toolchain', ) parser.add_argument( '--target-ninja', type=Path, required=True, help='Ninja file with the compilation commands to the test target', ) parser.add_argument( '--test-data', dest='test', required=True, type=TestCase.deserialize, help='Serialized TestCase object', ) parser.add_argument('--all-tests', type=Path, help='List of all tests') return vars(parser.parse_args()) if __name__ == '__main__': pw_cli.log.install(level=logging.INFO) sys.exit(_main(**_parse_args()))