1#!/usr/bin/env python3 2# Copyright 2016 Google Inc. All Rights Reserved. 3# 4# Licensed under the Apache License, Version 2.0 (the "License"); 5# you may not use this file except in compliance with the License. 6# You may obtain a copy of the License at 7# 8# http://www.apache.org/licenses/LICENSE-2.0 9# 10# Unless required by applicable law or agreed to in writing, software 11# distributed under the License is distributed on an "AS-IS" BASIS, 12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13# See the License for the specific language governing permissions and 14# limitations under the License. 15 16import os 17import tempfile 18import unittest 19import textwrap 20import re 21import sys 22 23import itertools 24 25import subprocess 26 27import pytest 28 29from fruit_test_config import * 30 31run_under_valgrind = RUN_TESTS_UNDER_VALGRIND.lower() not in ('false', 'off', 'no', '0', '') 32 33def pretty_print_command(command): 34 return ' '.join('"' + x + '"' for x in command) 35 36class CommandFailedException(Exception): 37 def __init__(self, command, stdout, stderr, error_code): 38 self.command = command 39 self.stdout = stdout 40 self.stderr = stderr 41 self.error_code = error_code 42 43 def __str__(self): 44 return textwrap.dedent('''\ 45 Ran command: {command} 46 Exit code {error_code} 47 Stdout: 48 {stdout} 49 50 Stderr: 51 {stderr} 52 ''').format(command=pretty_print_command(self.command), error_code=self.error_code, stdout=self.stdout, stderr=self.stderr) 53 54def run_command(executable, args=[], modify_env=lambda env: env): 55 command = [executable] + args 56 modified_env = modify_env(os.environ) 57 print('Executing command:', pretty_print_command(command)) 58 try: 59 p = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True, env=modified_env) 60 (stdout, stderr) = p.communicate() 61 except Exception as e: 62 raise Exception("While executing: %s" % command) 63 if p.returncode != 0: 64 raise CommandFailedException(command, stdout, stderr, p.returncode) 65 print('Execution successful.') 66 print('stdout:') 67 print(stdout) 68 print('') 69 print('stderr:') 70 print(stderr) 71 print('') 72 return (stdout, stderr) 73 74def run_compiled_executable(executable): 75 if run_under_valgrind: 76 args = VALGRIND_FLAGS.split() + [executable] 77 run_command('valgrind', args = args, modify_env = modify_env_for_compiled_executables) 78 else: 79 run_command(executable, modify_env = modify_env_for_compiled_executables) 80 81class CompilationFailedException(Exception): 82 def __init__(self, command, error_message): 83 self.command = command 84 self.error_message = error_message 85 86 def __str__(self): 87 return textwrap.dedent('''\ 88 Ran command: {command} 89 Error message: 90 {error_message} 91 ''').format(command=pretty_print_command(self.command), error_message=self.error_message) 92 93class PosixCompiler: 94 def __init__(self): 95 self.executable = CXX 96 self.name = CXX_COMPILER_NAME 97 98 def compile_discarding_output(self, source, include_dirs, args=[]): 99 try: 100 args = args + ['-c', source, '-o', os.path.devnull] 101 self._compile(include_dirs, args=args) 102 except CommandFailedException as e: 103 raise CompilationFailedException(e.command, e.stderr) 104 105 def compile_and_link(self, source, include_dirs, output_file_name, args=[]): 106 self._compile( 107 include_dirs, 108 args = ( 109 [source] 110 + ADDITIONAL_LINKER_FLAGS.split() 111 + args 112 + ['-o', output_file_name] 113 )) 114 115 def _compile(self, include_dirs, args): 116 include_flags = ['-I%s' % include_dir for include_dir in include_dirs] 117 args = ( 118 FRUIT_COMPILE_FLAGS.split() 119 + include_flags 120 + ['-g0', '-Werror'] 121 + args 122 ) 123 run_command(self.executable, args) 124 125 def get_disable_deprecation_warning_flags(self): 126 return ['-Wno-deprecated-declarations'] 127 128 def get_disable_all_warnings_flags(self): 129 return ['-Wno-error'] 130 131class MsvcCompiler: 132 def __init__(self): 133 self.executable = CXX 134 self.name = CXX_COMPILER_NAME 135 136 def compile_discarding_output(self, source, include_dirs, args=[]): 137 try: 138 args = args + ['/c', source] 139 self._compile(include_dirs, args = args) 140 except CommandFailedException as e: 141 # Note that we use stdout here, unlike above. MSVC reports compilation warnings and errors on stdout. 142 raise CompilationFailedException(e.command, e.stdout) 143 144 def compile_and_link(self, source, include_dirs, output_file_name, args=[]): 145 self._compile( 146 include_dirs, 147 args = ( 148 [source] 149 + ADDITIONAL_LINKER_FLAGS.split() 150 + args 151 + ['/Fe' + output_file_name] 152 )) 153 154 def _compile(self, include_dirs, args): 155 include_flags = ['-I%s' % include_dir for include_dir in include_dirs] 156 args = ( 157 FRUIT_COMPILE_FLAGS.split() 158 + include_flags 159 + ['/WX'] 160 + args 161 ) 162 run_command(self.executable, args) 163 164 def get_disable_deprecation_warning_flags(self): 165 return ['/wd4996'] 166 167 def get_disable_all_warnings_flags(self): 168 return ['/WX:NO'] 169 170if CXX_COMPILER_NAME == 'MSVC': 171 compiler = MsvcCompiler() 172 if PATH_TO_COMPILED_FRUIT_LIB.endswith('.dll'): 173 path_to_fruit_lib = PATH_TO_COMPILED_FRUIT_LIB[:-4] + '.lib' 174 else: 175 path_to_fruit_lib = PATH_TO_COMPILED_FRUIT_LIB 176 fruit_tests_linker_flags = [path_to_fruit_lib] 177 fruit_error_message_extraction_regex = 'error C2338: (.*)' 178else: 179 compiler = PosixCompiler() 180 fruit_tests_linker_flags = [ 181 '-lfruit', 182 '-L' + PATH_TO_COMPILED_FRUIT, 183 '-Wl,-rpath,' + PATH_TO_COMPILED_FRUIT, 184 ] 185 fruit_error_message_extraction_regex = 'static.assert(.*)' 186 187fruit_tests_include_dirs = ADDITIONAL_INCLUDE_DIRS.splitlines() + [ 188 PATH_TO_FRUIT_TEST_HEADERS, 189 PATH_TO_FRUIT_STATIC_HEADERS, 190 PATH_TO_FRUIT_GENERATED_HEADERS, 191] 192 193_assert_helper = unittest.TestCase() 194 195def modify_env_for_compiled_executables(env): 196 env = env.copy() 197 path_to_fruit_lib_dir = os.path.dirname(PATH_TO_COMPILED_FRUIT_LIB) 198 print('PATH_TO_COMPILED_FRUIT_LIB:', PATH_TO_COMPILED_FRUIT_LIB) 199 print('Adding directory to PATH:', path_to_fruit_lib_dir) 200 env["PATH"] += os.pathsep + path_to_fruit_lib_dir 201 return env 202 203def _create_temporary_file(file_content, file_name_suffix=''): 204 file_descriptor, file_name = tempfile.mkstemp(text=True, suffix=file_name_suffix) 205 file = os.fdopen(file_descriptor, mode='w') 206 file.write(file_content) 207 file.close() 208 return file_name 209 210def _cap_to_lines(s, n): 211 lines = s.splitlines() 212 if len(lines) <= n: 213 return s 214 else: 215 return '\n'.join(lines[0:n] + ['...']) 216 217def _replace_using_test_params(s, test_params): 218 for var_name, value in test_params.items(): 219 if isinstance(value, str): 220 s = re.sub(r'\b%s\b' % var_name, value, s) 221 return s 222 223def _construct_final_source_code(setup_source_code, source_code, test_params): 224 setup_source_code = textwrap.dedent(setup_source_code) 225 source_code = textwrap.dedent(source_code) 226 source_code = _replace_using_test_params(source_code, test_params) 227 return setup_source_code + source_code 228 229def try_remove_temporary_file(filename): 230 try: 231 os.remove(filename) 232 except: 233 # When running Fruit tests on Windows using Appveyor, the remove command fails for temporary files sometimes. 234 # This shouldn't cause the tests to fail, so we ignore the exception and go ahead. 235 pass 236 237def normalize_error_message_lines(lines): 238 # Different compilers output a different number of spaces when pretty-printing types. 239 # When using libc++, sometimes std::foo identifiers are reported as std::__1::foo. 240 return [line.replace(' ', '').replace('std::__1::', 'std::') for line in lines] 241 242def expect_compile_error_helper( 243 check_error_fun, 244 setup_source_code, 245 source_code, 246 test_params={}, 247 ignore_deprecation_warnings=False, 248 ignore_warnings=False): 249 source_code = _construct_final_source_code(setup_source_code, source_code, test_params) 250 251 source_file_name = _create_temporary_file(source_code, file_name_suffix='.cpp') 252 253 try: 254 args = [] 255 if ignore_deprecation_warnings: 256 args += compiler.get_disable_deprecation_warning_flags() 257 if ignore_warnings: 258 args += compiler.get_disable_all_warnings_flags() 259 if ENABLE_COVERAGE: 260 # When collecting coverage these arguments are enabled by default; however we must disable them in tests 261 # expected to fail at compile-time because GCC would otherwise fail with an error like: 262 # /tmp/tmp4m22cey7.cpp:1:0: error: cannot open /dev/null.gcno 263 args += ['-fno-profile-arcs', '-fno-test-coverage'] 264 compiler.compile_discarding_output( 265 source=source_file_name, 266 include_dirs=fruit_tests_include_dirs, 267 args=args) 268 raise Exception('The test should have failed to compile, but it compiled successfully') 269 except CompilationFailedException as e1: 270 e = e1 271 272 error_message = e.error_message 273 error_message_lines = error_message.splitlines() 274 error_message_lines = error_message.splitlines() 275 error_message_head = _cap_to_lines(error_message, 40) 276 277 check_error_fun(e, error_message_lines, error_message_head) 278 279 try_remove_temporary_file(source_file_name) 280 281def apply_any_error_context_replacements(error_string, following_lines): 282 if CXX_COMPILER_NAME == 'MSVC': 283 # MSVC errors are of the form: 284 # 285 # C:\Path\To\header\foo.h(59): note: see reference to class template instantiation 'fruit::impl::NoBindingFoundError<fruit::Annotated<Annotation,U>>' being compiled 286 # with 287 # [ 288 # Annotation=Annotation1, 289 # U=std::function<std::unique_ptr<ScalerImpl,std::default_delete<ScalerImpl>> (double)> 290 # ] 291 # 292 # So we need to parse the following few lines and use them to replace the placeholder types in the Fruit error type. 293 replacement_lines = [] 294 if len(following_lines) >= 4 and following_lines[0].strip() == 'with': 295 assert following_lines[1].strip() == '[', 'Line was: ' + following_lines[1] 296 for line in itertools.islice(following_lines, 2, None): 297 line = line.strip() 298 if line == ']': 299 break 300 if line.endswith(','): 301 line = line[:-1] 302 replacement_lines.append(line) 303 304 for replacement_line in replacement_lines: 305 match = re.search('([A-Za-z0-9_-]*)=(.*)', replacement_line) 306 if not match: 307 raise Exception('Failed to parse replacement line: %s' % replacement_line) 308 (type_variable, type_expression) = match.groups() 309 error_string = re.sub(r'\b' + type_variable + r'\b', type_expression, error_string) 310 return error_string 311 312def expect_generic_compile_error(expected_error_regex, setup_source_code, source_code, test_params={}): 313 """ 314 Tests that the given source produces the expected error during compilation. 315 316 :param expected_fruit_error_regex: A regex used to match the Fruit error type, 317 e.g. 'NoBindingFoundForAbstractClassError<ScalerImpl>'. 318 Any identifiers contained in the regex will be replaced using test_params (where a replacement is defined). 319 :param expected_fruit_error_desc_regex: A regex used to match the Fruit error description, 320 e.g. 'No explicit binding was found for C, and C is an abstract class'. 321 :param setup_source_code: The first part of the source code. This is dedented separately from source_code and it's 322 *not* subject to test_params, unlike source_code. 323 :param source_code: The second part of the source code. Any identifiers will be replaced using test_params 324 (where a replacement is defined). This will be dedented. 325 :param test_params: A dict containing the definition of some identifiers. Each identifier in 326 expected_fruit_error_regex and source_code will be replaced (textually) with its definition (if a definition 327 was provided). 328 """ 329 330 expected_error_regex = _replace_using_test_params(expected_error_regex, test_params) 331 expected_error_regex = expected_error_regex.replace(' ', '') 332 333 def check_error(e, error_message_lines, error_message_head): 334 error_message_lines_with_replacements = [ 335 apply_any_error_context_replacements(line, error_message_lines[line_number + 1:]) 336 for line_number, line in enumerate(error_message_lines)] 337 338 normalized_error_message_lines = normalize_error_message_lines(error_message_lines_with_replacements) 339 340 for line in normalized_error_message_lines: 341 if re.search(expected_error_regex, line): 342 return 343 raise Exception(textwrap.dedent('''\ 344 Expected error {expected_error} but the compiler output did not contain that. 345 Compiler command line: {compiler_command} 346 Error message was: 347 {error_message} 348 ''').format(expected_error = expected_error_regex, compiler_command=e.command, error_message = error_message_head)) 349 350 expect_compile_error_helper(check_error, setup_source_code, source_code, test_params) 351 352def expect_compile_error( 353 expected_fruit_error_regex, 354 expected_fruit_error_desc_regex, 355 setup_source_code, 356 source_code, 357 test_params={}, 358 ignore_deprecation_warnings=False, 359 ignore_warnings=False, 360 disable_error_line_number_check=False): 361 """ 362 Tests that the given source produces the expected error during compilation. 363 364 :param expected_fruit_error_regex: A regex used to match the Fruit error type, 365 e.g. 'NoBindingFoundForAbstractClassError<ScalerImpl>'. 366 Any identifiers contained in the regex will be replaced using test_params (where a replacement is defined). 367 :param expected_fruit_error_desc_regex: A regex used to match the Fruit error description, 368 e.g. 'No explicit binding was found for C, and C is an abstract class'. 369 :param setup_source_code: The first part of the source code. This is dedented separately from source_code and it's 370 *not* subject to test_params, unlike source_code. 371 :param source_code: The second part of the source code. Any identifiers will be replaced using test_params 372 (where a replacement is defined). This will be dedented. 373 :param test_params: A dict containing the definition of some identifiers. Each identifier in 374 expected_fruit_error_regex and source_code will be replaced (textually) with its definition (if a definition 375 was provided). 376 :param ignore_deprecation_warnings: A boolean. If True, deprecation warnings will be ignored. 377 :param ignore_warnings: A boolean. If True, all warnings will be ignored. 378 :param disable_error_line_number_check: A boolean. If True, the test will not fail if there are other diagnostic 379 lines before the expected error. 380 """ 381 if '\n' in expected_fruit_error_regex: 382 raise Exception('expected_fruit_error_regex should not contain newlines') 383 if '\n' in expected_fruit_error_desc_regex: 384 raise Exception('expected_fruit_error_desc_regex should not contain newlines') 385 386 expected_fruit_error_regex = _replace_using_test_params(expected_fruit_error_regex, test_params) 387 expected_fruit_error_regex = expected_fruit_error_regex.replace(' ', '') 388 389 def check_error(e, error_message_lines, error_message_head): 390 normalized_error_message_lines = normalize_error_message_lines(error_message_lines) 391 392 for line_number, line in enumerate(normalized_error_message_lines): 393 match = re.search('fruit::impl::(.*Error<.*>)', line) 394 if match: 395 actual_fruit_error_line_number = line_number 396 actual_fruit_error = match.groups()[0] 397 actual_fruit_error = apply_any_error_context_replacements(actual_fruit_error, normalized_error_message_lines[line_number + 1:]) 398 break 399 else: 400 raise Exception(textwrap.dedent('''\ 401 Expected error {expected_error} but the compiler output did not contain user-facing Fruit errors. 402 Compiler command line: {compiler_command} 403 Error message was: 404 {error_message} 405 ''').format(expected_error = expected_fruit_error_regex, compiler_command = e.command, error_message = error_message_head)) 406 407 for line_number, line in enumerate(error_message_lines): 408 match = re.search(fruit_error_message_extraction_regex, line) 409 if match: 410 actual_static_assert_error_line_number = line_number 411 actual_static_assert_error = match.groups()[0] 412 break 413 else: 414 raise Exception(textwrap.dedent('''\ 415 Expected error {expected_error} but the compiler output did not contain static_assert errors. 416 Compiler command line: {compiler_command} 417 Error message was: 418 {error_message} 419 ''').format(expected_error = expected_fruit_error_regex, compiler_command=e.command, error_message = error_message_head)) 420 421 try: 422 regex_search_result = re.search(expected_fruit_error_regex, actual_fruit_error) 423 except Exception as e: 424 raise Exception('re.search() failed for regex \'%s\'' % expected_fruit_error_regex) from e 425 if not regex_search_result: 426 raise Exception(textwrap.dedent('''\ 427 The compilation failed as expected, but with a different error type. 428 Expected Fruit error type: {expected_fruit_error_regex} 429 Error type was: {actual_fruit_error} 430 Expected static assert error: {expected_fruit_error_desc_regex} 431 Static assert was: {actual_static_assert_error} 432 Error message was: 433 {error_message} 434 '''.format( 435 expected_fruit_error_regex = expected_fruit_error_regex, 436 actual_fruit_error = actual_fruit_error, 437 expected_fruit_error_desc_regex = expected_fruit_error_desc_regex, 438 actual_static_assert_error = actual_static_assert_error, 439 error_message = error_message_head))) 440 try: 441 regex_search_result = re.search(expected_fruit_error_desc_regex, actual_static_assert_error) 442 except Exception as e: 443 raise Exception('re.search() failed for regex \'%s\'' % expected_fruit_error_desc_regex) from e 444 if not regex_search_result: 445 raise Exception(textwrap.dedent('''\ 446 The compilation failed as expected, but with a different error message. 447 Expected Fruit error type: {expected_fruit_error_regex} 448 Error type was: {actual_fruit_error} 449 Expected static assert error: {expected_fruit_error_desc_regex} 450 Static assert was: {actual_static_assert_error} 451 Error message: 452 {error_message} 453 '''.format( 454 expected_fruit_error_regex = expected_fruit_error_regex, 455 actual_fruit_error = actual_fruit_error, 456 expected_fruit_error_desc_regex = expected_fruit_error_desc_regex, 457 actual_static_assert_error = actual_static_assert_error, 458 error_message = error_message_head))) 459 460 # 6 is just a constant that works for both g++ (<=6.0.0 at least) and clang++ (<=4.0.0 at least). 461 # It might need to be changed. 462 if not disable_error_line_number_check and (actual_fruit_error_line_number > 6 or actual_static_assert_error_line_number > 6): 463 raise Exception(textwrap.dedent('''\ 464 The compilation failed with the expected message, but the error message contained too many lines before the relevant ones. 465 The error type was reported on line {actual_fruit_error_line_number} of the message (should be <=6). 466 The static assert was reported on line {actual_static_assert_error_line_number} of the message (should be <=6). 467 Error message: 468 {error_message} 469 '''.format( 470 actual_fruit_error_line_number = actual_fruit_error_line_number, 471 actual_static_assert_error_line_number = actual_static_assert_error_line_number, 472 error_message = error_message_head))) 473 474 for line in error_message_lines[:max(actual_fruit_error_line_number, actual_static_assert_error_line_number)]: 475 if re.search('fruit::impl::meta', line): 476 raise Exception( 477 'The compilation failed with the expected message, but the error message contained some metaprogramming types in the output (besides Error). Error message:\n%s' + error_message_head) 478 479 expect_compile_error_helper(check_error, setup_source_code, source_code, test_params, ignore_deprecation_warnings, ignore_warnings) 480 481 482def expect_runtime_error( 483 expected_error_regex, 484 setup_source_code, 485 source_code, 486 test_params={}, 487 ignore_deprecation_warnings=False): 488 """ 489 Tests that the given source (compiles successfully and) produces the expected error at runtime. 490 491 :param expected_error_regex: A regex used to match the content of stderr. 492 Any identifiers contained in the regex will be replaced using test_params (where a replacement is defined). 493 :param setup_source_code: The first part of the source code. This is dedented separately from source_code and it's 494 *not* subject to test_params, unlike source_code. 495 :param source_code: The second part of the source code. Any identifiers will be replaced using test_params 496 (where a replacement is defined). This will be dedented. 497 :param test_params: A dict containing the definition of some identifiers. Each identifier in 498 expected_error_regex and source_code will be replaced (textually) with its definition (if a definition 499 was provided). 500 """ 501 expected_error_regex = _replace_using_test_params(expected_error_regex, test_params) 502 source_code = _construct_final_source_code(setup_source_code, source_code, test_params) 503 504 source_file_name = _create_temporary_file(source_code, file_name_suffix='.cpp') 505 executable_suffix = {'posix': '', 'nt': '.exe'}[os.name] 506 output_file_name = _create_temporary_file('', executable_suffix) 507 508 args = fruit_tests_linker_flags.copy() 509 if ignore_deprecation_warnings: 510 args += compiler.get_disable_deprecation_warning_flags() 511 compiler.compile_and_link( 512 source=source_file_name, 513 include_dirs=fruit_tests_include_dirs, 514 output_file_name=output_file_name, 515 args=args) 516 517 try: 518 run_compiled_executable(output_file_name) 519 raise Exception('The test should have failed at runtime, but it ran successfully') 520 except CommandFailedException as e1: 521 e = e1 522 523 stderr = e.stderr 524 stderr_head = _cap_to_lines(stderr, 40) 525 526 if '\n' in expected_error_regex: 527 regex_flags = re.MULTILINE 528 else: 529 regex_flags = 0 530 531 try: 532 regex_search_result = re.search(expected_error_regex, stderr, flags=regex_flags) 533 except Exception as e: 534 raise Exception('re.search() failed for regex \'%s\'' % expected_error_regex) from e 535 if not regex_search_result: 536 raise Exception(textwrap.dedent('''\ 537 The test failed as expected, but with a different message. 538 Expected: {expected_error_regex} 539 Was: 540 {stderr} 541 '''.format(expected_error_regex = expected_error_regex, stderr = stderr_head))) 542 543 # Note that we don't delete the temporary files if the test failed. This is intentional, keeping them around helps debugging the failure. 544 if not ENABLE_COVERAGE: 545 try_remove_temporary_file(source_file_name) 546 try_remove_temporary_file(output_file_name) 547 548 549def expect_success(setup_source_code, source_code, test_params={}, ignore_deprecation_warnings=False): 550 """ 551 Tests that the given source compiles and runs successfully. 552 553 :param setup_source_code: The first part of the source code. This is dedented separately from source_code and it's 554 *not* subject to test_params, unlike source_code. 555 :param source_code: The second part of the source code. Any identifiers will be replaced using test_params 556 (where a replacement is defined). This will be dedented. 557 :param test_params: A dict containing the definition of some identifiers. Each identifier in 558 source_code will be replaced (textually) with its definition (if a definition was provided). 559 """ 560 source_code = _construct_final_source_code(setup_source_code, source_code, test_params) 561 562 if 'main(' not in source_code: 563 source_code += textwrap.dedent(''' 564 int main() { 565 } 566 ''') 567 568 source_file_name = _create_temporary_file(source_code, file_name_suffix='.cpp') 569 executable_suffix = {'posix': '', 'nt': '.exe'}[os.name] 570 output_file_name = _create_temporary_file('', executable_suffix) 571 572 args = fruit_tests_linker_flags.copy() 573 if ignore_deprecation_warnings: 574 args += compiler.get_disable_deprecation_warning_flags() 575 compiler.compile_and_link( 576 source=source_file_name, 577 include_dirs=fruit_tests_include_dirs, 578 output_file_name=output_file_name, 579 args=args) 580 581 run_compiled_executable(output_file_name) 582 583 # Note that we don't delete the temporary files if the test failed. This is intentional, keeping them around helps debugging the failure. 584 if not ENABLE_COVERAGE: 585 try_remove_temporary_file(source_file_name) 586 try_remove_temporary_file(output_file_name) 587 588 589# Note: this is not the main function of this file, it's meant to be used as main function from test_*.py files. 590def main(file): 591 code = pytest.main(args = sys.argv + [os.path.realpath(file)]) 592 exit(code) 593