1#!/usr/bin/env python3 2# Copyright 2016 The PDFium Authors 3# Use of this source code is governed by a BSD-style license that can be 4# found in the LICENSE file. 5 6import argparse 7from dataclasses import dataclass, field 8from datetime import timedelta 9from io import BytesIO 10import multiprocessing 11import os 12import re 13import shutil 14import subprocess 15import sys 16import time 17 18import common 19import pdfium_root 20import pngdiffer 21from skia_gold import skia_gold 22import suppressor 23 24pdfium_root.add_source_directory_to_import_path(os.path.join('build', 'util')) 25from lib.results import result_sink, result_types 26 27 28# Arbitrary timestamp, expressed in seconds since the epoch, used to make sure 29# that tests that depend on the current time are stable. Happens to be the 30# timestamp of the first commit to repo, 2014/5/9 17:48:50. 31TEST_SEED_TIME = '1399672130' 32 33# List of test types that should run text tests instead of pixel tests. 34TEXT_TESTS = ['javascript'] 35 36# Timeout (in seconds) for individual test commands. 37# TODO(crbug.com/pdfium/1967): array_buffer.in is slow under MSan, so need a 38# very generous 5 minute timeout for now. 39TEST_TIMEOUT = timedelta(minutes=5).total_seconds() 40 41 42class TestRunner: 43 44 def __init__(self, dirname): 45 # Currently the only used directories are corpus, javascript, and pixel, 46 # which all correspond directly to the type for the test being run. In the 47 # future if there are tests that don't have this clean correspondence, then 48 # an argument for the type will need to be added. 49 self.per_process_config = _PerProcessConfig( 50 test_dir=dirname, test_type=dirname) 51 52 @property 53 def options(self): 54 return self.per_process_config.options 55 56 def IsSkiaGoldEnabled(self): 57 return (self.options.run_skia_gold and 58 self.per_process_config.test_type not in TEXT_TESTS) 59 60 def IsExecutionSuppressed(self, input_path): 61 return self.per_process_state.test_suppressor.IsExecutionSuppressed( 62 input_path) 63 64 def IsResultSuppressed(self, input_filename): 65 return self.per_process_state.test_suppressor.IsResultSuppressed( 66 input_filename) 67 68 def HandleResult(self, test_case, test_result): 69 input_filename = os.path.basename(test_case.input_path) 70 71 test_result.status = self._SuppressStatus(input_filename, 72 test_result.status) 73 if test_result.status == result_types.UNKNOWN: 74 self.result_suppressed_cases.append(input_filename) 75 self.surprises.append(test_case.input_path) 76 elif test_result.status == result_types.SKIP: 77 self.result_suppressed_cases.append(input_filename) 78 elif not test_result.IsPass(): 79 self.failures.append(test_case.input_path) 80 81 for artifact in test_result.image_artifacts: 82 if artifact.skia_gold_status == result_types.PASS: 83 if self.IsResultSuppressed(artifact.image_path): 84 self.skia_gold_unexpected_successes.append(artifact.GetSkiaGoldId()) 85 else: 86 self.skia_gold_successes.append(artifact.GetSkiaGoldId()) 87 elif artifact.skia_gold_status == result_types.FAIL: 88 self.skia_gold_failures.append(artifact.GetSkiaGoldId()) 89 90 # Log test result. 91 print(f'{test_result.status}: {test_result.test_id}') 92 if not test_result.IsPass(): 93 if test_result.reason: 94 print(f'Failure reason: {test_result.reason}') 95 if test_result.log: 96 print(f'Test output:\n{test_result.log}') 97 for artifact in test_result.image_artifacts: 98 if artifact.skia_gold_status == result_types.FAIL: 99 print(f'Failed Skia Gold: {artifact.image_path}') 100 if artifact.image_diff: 101 print(f'Failed image diff: {artifact.image_diff.reason}') 102 103 # Report test result to ResultDB. 104 if self.resultdb: 105 only_artifacts = None 106 only_failure_reason = test_result.reason 107 if len(test_result.image_artifacts) == 1: 108 only = test_result.image_artifacts[0] 109 only_artifacts = only.GetDiffArtifacts() 110 if only.GetDiffReason(): 111 only_failure_reason += f': {only.GetDiffReason()}' 112 self.resultdb.Post( 113 test_id=test_result.test_id, 114 status=test_result.status, 115 duration=test_result.duration_milliseconds, 116 test_log=test_result.log, 117 test_file=None, 118 artifacts=only_artifacts, 119 failure_reason=only_failure_reason) 120 121 # Milo only supports a single diff per test, so if we have multiple pages, 122 # report each page as its own "test." 123 if len(test_result.image_artifacts) > 1: 124 for page, artifact in enumerate(test_result.image_artifacts): 125 self.resultdb.Post( 126 test_id=f'{test_result.test_id}/{page}', 127 status=self._SuppressArtifactStatus(test_result, 128 artifact.GetDiffStatus()), 129 duration=None, 130 test_log=None, 131 test_file=None, 132 artifacts=artifact.GetDiffArtifacts(), 133 failure_reason=artifact.GetDiffReason()) 134 135 def _SuppressStatus(self, input_filename, status): 136 if not self.IsResultSuppressed(input_filename): 137 return status 138 139 if status == result_types.PASS: 140 # There isn't an actual status for succeeded-but-ignored, so use the 141 # "abort" status to differentiate this from failed-but-ignored. 142 # 143 # Note that this appears as a preliminary failure in Gerrit. 144 return result_types.UNKNOWN 145 146 # There isn't an actual status for failed-but-ignored, so use the "skip" 147 # status to differentiate this from succeeded-but-ignored. 148 return result_types.SKIP 149 150 def _SuppressArtifactStatus(self, test_result, status): 151 if status != result_types.FAIL: 152 return status 153 154 if test_result.status != result_types.SKIP: 155 return status 156 157 return result_types.SKIP 158 159 def Run(self): 160 # Running a test defines a number of attributes on the fly. 161 # pylint: disable=attribute-defined-outside-init 162 163 relative_test_dir = self.per_process_config.test_dir 164 if relative_test_dir != 'corpus': 165 relative_test_dir = os.path.join('resources', relative_test_dir) 166 167 parser = argparse.ArgumentParser() 168 169 parser.add_argument( 170 '--build-dir', 171 default=os.path.join('out', 'Debug'), 172 help='relative path from the base source directory') 173 174 parser.add_argument( 175 '-j', 176 default=multiprocessing.cpu_count(), 177 dest='num_workers', 178 type=int, 179 help='run NUM_WORKERS jobs in parallel') 180 181 parser.add_argument( 182 '--disable-javascript', 183 action='store_true', 184 help='Prevents JavaScript from executing in PDF files.') 185 186 parser.add_argument( 187 '--disable-xfa', 188 action='store_true', 189 help='Prevents processing XFA forms.') 190 191 parser.add_argument( 192 '--render-oneshot', 193 action='store_true', 194 help='Sets whether to use the oneshot renderer.') 195 196 parser.add_argument( 197 '--run-skia-gold', 198 action='store_true', 199 default=False, 200 help='When flag is on, skia gold tests will be run.') 201 202 parser.add_argument( 203 '--regenerate_expected', 204 action='store_true', 205 help='Regenerates expected images. For each failing image diff, this ' 206 'will regenerate the most specific expected image file that exists. ' 207 'This also will suggest removals of unnecessary expected image files ' 208 'by renaming them with an additional ".bak" extension, although these ' 209 'removals should be reviewed manually. Use "git clean" to quickly deal ' 210 'with any ".bak" files.') 211 212 parser.add_argument( 213 '--reverse-byte-order', 214 action='store_true', 215 help='Run image-based tests using --reverse-byte-order.') 216 217 parser.add_argument( 218 '--ignore_errors', 219 action='store_true', 220 help='Prevents the return value from being non-zero ' 221 'when image comparison fails.') 222 223 parser.add_argument( 224 '--use-renderer', 225 choices=('agg', 'gdi', 'skia'), 226 help='Forces the renderer to use.') 227 228 parser.add_argument( 229 'inputted_file_paths', 230 nargs='*', 231 help='Path to test files to run, relative to ' 232 f'testing/{relative_test_dir}. If omitted, runs all test files under ' 233 f'testing/{relative_test_dir}.', 234 metavar='relative/test/path') 235 236 skia_gold.add_skia_gold_args(parser) 237 238 self.per_process_config.options = parser.parse_args() 239 240 finder = self.per_process_config.NewFinder() 241 pdfium_test_path = self.per_process_config.GetPdfiumTestPath(finder) 242 if not os.path.exists(pdfium_test_path): 243 print(f"FAILURE: Can't find test executable '{pdfium_test_path}'") 244 print('Use --build-dir to specify its location.') 245 return 1 246 247 error_message = self.per_process_config.InitializeFeatures(pdfium_test_path) 248 if error_message: 249 print('FAILURE:', error_message) 250 return 1 251 252 self.per_process_state = _PerProcessState(self.per_process_config) 253 shutil.rmtree(self.per_process_state.working_dir, ignore_errors=True) 254 os.makedirs(self.per_process_state.working_dir) 255 256 error_message = self.per_process_state.image_differ.CheckMissingTools( 257 self.options.regenerate_expected) 258 if error_message: 259 print('FAILURE:', error_message) 260 return 1 261 262 self.resultdb = result_sink.TryInitClient() 263 if self.resultdb: 264 print('Detected ResultSink environment') 265 266 # Collect test cases. 267 walk_from_dir = finder.TestingDir(relative_test_dir) 268 269 self.test_cases = TestCaseManager() 270 self.execution_suppressed_cases = [] 271 input_file_re = re.compile('^.+[.](in|pdf)$') 272 if self.options.inputted_file_paths: 273 for file_name in self.options.inputted_file_paths: 274 input_path = os.path.join(walk_from_dir, file_name) 275 if not os.path.isfile(input_path): 276 print(f"Can't find test file '{file_name}'") 277 return 1 278 279 self.test_cases.NewTestCase(input_path) 280 else: 281 for file_dir, _, filename_list in os.walk(walk_from_dir): 282 for input_filename in filename_list: 283 if input_file_re.match(input_filename): 284 input_path = os.path.join(file_dir, input_filename) 285 if self.IsExecutionSuppressed(input_path): 286 self.execution_suppressed_cases.append(input_path) 287 continue 288 if not os.path.isfile(input_path): 289 continue 290 291 self.test_cases.NewTestCase(input_path) 292 293 # Execute test cases. 294 self.failures = [] 295 self.surprises = [] 296 self.skia_gold_successes = [] 297 self.skia_gold_unexpected_successes = [] 298 self.skia_gold_failures = [] 299 self.result_suppressed_cases = [] 300 301 if self.IsSkiaGoldEnabled(): 302 assert self.options.gold_output_dir 303 # Clear out and create top level gold output directory before starting 304 skia_gold.clear_gold_output_dir(self.options.gold_output_dir) 305 306 with multiprocessing.Pool( 307 processes=self.options.num_workers, 308 initializer=_InitializePerProcessState, 309 initargs=[self.per_process_config]) as pool: 310 if self.per_process_config.test_type in TEXT_TESTS: 311 test_function = _RunTextTest 312 else: 313 test_function = _RunPixelTest 314 for result in pool.imap(test_function, self.test_cases): 315 self.HandleResult(self.test_cases.GetTestCase(result.test_id), result) 316 317 # Report test results. 318 if self.surprises: 319 self.surprises.sort() 320 print('\nUnexpected Successes:') 321 for surprise in self.surprises: 322 print(surprise) 323 324 if self.failures: 325 self.failures.sort() 326 print('\nSummary of Failures:') 327 for failure in self.failures: 328 print(failure) 329 330 if self.skia_gold_unexpected_successes: 331 self.skia_gold_unexpected_successes.sort() 332 print('\nUnexpected Skia Gold Successes:') 333 for surprise in self.skia_gold_unexpected_successes: 334 print(surprise) 335 336 if self.skia_gold_failures: 337 self.skia_gold_failures.sort() 338 print('\nSummary of Skia Gold Failures:') 339 for failure in self.skia_gold_failures: 340 print(failure) 341 342 self._PrintSummary() 343 344 if self.failures: 345 if not self.options.ignore_errors: 346 return 1 347 348 return 0 349 350 def _PrintSummary(self): 351 number_test_cases = len(self.test_cases) 352 number_failures = len(self.failures) 353 number_suppressed = len(self.result_suppressed_cases) 354 number_successes = number_test_cases - number_failures - number_suppressed 355 number_surprises = len(self.surprises) 356 print('\nTest cases executed:', number_test_cases) 357 print(' Successes:', number_successes) 358 print(' Suppressed:', number_suppressed) 359 print(' Surprises:', number_surprises) 360 print(' Failures:', number_failures) 361 if self.IsSkiaGoldEnabled(): 362 number_gold_failures = len(self.skia_gold_failures) 363 number_gold_successes = len(self.skia_gold_successes) 364 number_gold_surprises = len(self.skia_gold_unexpected_successes) 365 number_total_gold_tests = sum( 366 [number_gold_failures, number_gold_successes, number_gold_surprises]) 367 print('\nSkia Gold Test cases executed:', number_total_gold_tests) 368 print(' Skia Gold Successes:', number_gold_successes) 369 print(' Skia Gold Surprises:', number_gold_surprises) 370 print(' Skia Gold Failures:', number_gold_failures) 371 skia_tester = self.per_process_state.GetSkiaGoldTester() 372 if self.skia_gold_failures and skia_tester.IsTryjobRun(): 373 cl_triage_link = skia_tester.GetCLTriageLink() 374 print(' Triage link for CL:', cl_triage_link) 375 skia_tester.WriteCLTriageLink(cl_triage_link) 376 print() 377 print('Test cases not executed:', len(self.execution_suppressed_cases)) 378 379 def SetDeleteOutputOnSuccess(self, new_value): 380 """Set whether to delete generated output if the test passes.""" 381 self.per_process_config.delete_output_on_success = new_value 382 383 def SetEnforceExpectedImages(self, new_value): 384 """Set whether to enforce that each test case provide an expected image.""" 385 self.per_process_config.enforce_expected_images = new_value 386 387 388def _RunTextTest(test_case): 389 """Runs a text test case.""" 390 test_case_runner = _TestCaseRunner(test_case) 391 with test_case_runner: 392 test_case_runner.test_result = test_case_runner.GenerateAndTest( 393 test_case_runner.TestText) 394 return test_case_runner.test_result 395 396 397def _RunPixelTest(test_case): 398 """Runs a pixel test case.""" 399 test_case_runner = _TestCaseRunner(test_case) 400 with test_case_runner: 401 test_case_runner.test_result = test_case_runner.GenerateAndTest( 402 test_case_runner.TestPixel) 403 return test_case_runner.test_result 404 405 406# `_PerProcessState` singleton. This is initialized when creating the 407# `multiprocessing.Pool()`. `TestRunner.Run()` creates its own separate 408# instance of `_PerProcessState` as well. 409_per_process_state = None 410 411 412def _InitializePerProcessState(config): 413 """Initializes the `_per_process_state` singleton.""" 414 global _per_process_state 415 assert not _per_process_state 416 _per_process_state = _PerProcessState(config) 417 418 419@dataclass 420class _PerProcessConfig: 421 """Configuration for initializing `_PerProcessState`. 422 423 Attributes: 424 test_dir: The name of the test directory. 425 test_type: The test type. 426 delete_output_on_success: Whether to delete output on success. 427 enforce_expected_images: Whether to enforce expected images. 428 options: The dictionary of command line options. 429 features: The set of features supported by `pdfium_test`. 430 rendering_option: The renderer to use (agg, gdi, or skia). 431 """ 432 test_dir: str 433 test_type: str 434 delete_output_on_success: bool = False 435 enforce_expected_images: bool = False 436 options: dict = None 437 features: set = None 438 default_renderer: str = None 439 rendering_option: str = None 440 441 def NewFinder(self): 442 return common.DirectoryFinder(self.options.build_dir) 443 444 def GetPdfiumTestPath(self, finder): 445 return finder.ExecutablePath('pdfium_test') 446 447 def InitializeFeatures(self, pdfium_test_path): 448 output = subprocess.check_output([pdfium_test_path, '--show-config'], 449 timeout=TEST_TIMEOUT) 450 self.features = set(output.decode('utf-8').strip().split(',')) 451 452 if 'SKIA' in self.features: 453 self.default_renderer = 'skia' 454 else: 455 self.default_renderer = 'agg' 456 self.rendering_option = self.default_renderer 457 458 if self.options.use_renderer == 'agg': 459 self.rendering_option = 'agg' 460 elif self.options.use_renderer == 'gdi': 461 if 'GDI' not in self.features: 462 return 'pdfium_test missing GDI renderer support' 463 self.rendering_option = 'gdi' 464 elif self.options.use_renderer == 'skia': 465 if 'SKIA' not in self.features: 466 return 'pdfium_test missing Skia renderer support' 467 self.rendering_option = 'skia' 468 469 return None 470 471 472class _PerProcessState: 473 """State defined per process.""" 474 475 def __init__(self, config): 476 self.test_dir = config.test_dir 477 self.test_type = config.test_type 478 self.delete_output_on_success = config.delete_output_on_success 479 self.enforce_expected_images = config.enforce_expected_images 480 self.options = config.options 481 self.features = config.features 482 483 finder = config.NewFinder() 484 self.pdfium_test_path = config.GetPdfiumTestPath(finder) 485 self.fixup_path = finder.ScriptPath('fixup_pdf_template.py') 486 self.text_diff_path = finder.ScriptPath('text_diff.py') 487 self.font_dir = os.path.join(finder.TestingDir(), 'resources', 'fonts') 488 self.third_party_font_dir = finder.ThirdPartyFontsDir() 489 490 self.source_dir = finder.TestingDir() 491 self.working_dir = finder.WorkingDir(os.path.join('testing', self.test_dir)) 492 493 self.test_suppressor = suppressor.Suppressor( 494 finder, self.features, self.options.disable_javascript, 495 self.options.disable_xfa, config.rendering_option) 496 self.image_differ = pngdiffer.PNGDiffer(finder, 497 self.options.reverse_byte_order, 498 config.rendering_option, 499 config.default_renderer) 500 501 self.process_name = multiprocessing.current_process().name 502 self.skia_tester = None 503 504 def __getstate__(self): 505 raise RuntimeError('Cannot pickle per-process state') 506 507 def GetSkiaGoldTester(self): 508 """Gets the `SkiaGoldTester` singleton for this worker.""" 509 if not self.skia_tester: 510 self.skia_tester = skia_gold.SkiaGoldTester( 511 source_type=self.test_type, 512 skia_gold_args=self.options, 513 process_name=self.process_name) 514 return self.skia_tester 515 516 517class _TestCaseRunner: 518 """Runner for a single test case.""" 519 520 def __init__(self, test_case): 521 self.test_case = test_case 522 self.test_result = None 523 self.duration_start = 0 524 525 self.source_dir, self.input_filename = os.path.split( 526 self.test_case.input_path) 527 self.pdf_path = os.path.join(self.working_dir, f'{self.test_id}.pdf') 528 self.actual_images = None 529 530 def __enter__(self): 531 self.duration_start = time.perf_counter_ns() 532 return self 533 534 def __exit__(self, exc_type, exc_value, traceback): 535 if not self.test_result: 536 self.test_result = self.test_case.NewResult( 537 result_types.UNKNOWN, reason='No test result recorded') 538 duration = time.perf_counter_ns() - self.duration_start 539 self.test_result.duration_milliseconds = duration * 1e-6 540 541 @property 542 def options(self): 543 return _per_process_state.options 544 545 @property 546 def test_id(self): 547 return self.test_case.test_id 548 549 @property 550 def working_dir(self): 551 return _per_process_state.working_dir 552 553 def IsResultSuppressed(self): 554 return _per_process_state.test_suppressor.IsResultSuppressed( 555 self.input_filename) 556 557 def IsImageDiffSuppressed(self): 558 return _per_process_state.test_suppressor.IsImageDiffSuppressed( 559 self.input_filename) 560 561 def GetImageMatchingAlgorithm(self): 562 return _per_process_state.test_suppressor.GetImageMatchingAlgorithm( 563 self.input_filename) 564 565 def RunCommand(self, command, stdout=None): 566 """Runs a test command. 567 568 Args: 569 command: The list of command arguments. 570 stdout: Optional `file`-like object to send standard output. 571 572 Returns: 573 The test result. 574 """ 575 576 # Standard output and error are directed to the test log. If `stdout` was 577 # provided, redirect standard output to it instead. 578 if stdout: 579 assert stdout != subprocess.PIPE 580 try: 581 stdout.fileno() 582 except OSError: 583 # `stdout` doesn't have a file descriptor, so it can't be passed to 584 # `subprocess.run()` directly. 585 original_stdout = stdout 586 stdout = subprocess.PIPE 587 stderr = subprocess.PIPE 588 else: 589 stdout = subprocess.PIPE 590 stderr = subprocess.STDOUT 591 592 test_result = self.test_case.NewResult(result_types.PASS) 593 try: 594 run_result = subprocess.run( 595 command, 596 stdout=stdout, 597 stderr=stderr, 598 timeout=TEST_TIMEOUT, 599 check=False) 600 if run_result.returncode != 0: 601 test_result.status = result_types.FAIL 602 test_result.reason = 'Command {} exited with code {}'.format( 603 run_result.args, run_result.returncode) 604 except subprocess.TimeoutExpired as timeout_expired: 605 run_result = timeout_expired 606 test_result.status = result_types.TIMEOUT 607 test_result.reason = 'Command {} timed out'.format(run_result.cmd) 608 609 if stdout == subprocess.PIPE and stderr == subprocess.PIPE: 610 # Copy captured standard output, if any, to the original `stdout`. 611 if run_result.stdout: 612 original_stdout.write(run_result.stdout) 613 614 if not test_result.IsPass(): 615 # On failure, report captured output to the test log. 616 if stderr == subprocess.STDOUT: 617 test_result.log = run_result.stdout 618 else: 619 test_result.log = run_result.stderr 620 test_result.log = test_result.log.decode(errors='backslashreplace') 621 return test_result 622 623 def GenerateAndTest(self, test_function): 624 """Generate test input and run pdfium_test.""" 625 test_result = self.Generate() 626 if not test_result.IsPass(): 627 return test_result 628 629 return test_function() 630 631 def _RegenerateIfNeeded(self): 632 if not self.options.regenerate_expected: 633 return 634 if self.IsResultSuppressed() or self.IsImageDiffSuppressed(): 635 return 636 _per_process_state.image_differ.Regenerate( 637 self.input_filename, 638 self.source_dir, 639 self.working_dir, 640 image_matching_algorithm=self.GetImageMatchingAlgorithm()) 641 642 def Generate(self): 643 input_event_path = os.path.join(self.source_dir, f'{self.test_id}.evt') 644 if os.path.exists(input_event_path): 645 output_event_path = f'{os.path.splitext(self.pdf_path)[0]}.evt' 646 shutil.copyfile(input_event_path, output_event_path) 647 648 template_path = os.path.join(self.source_dir, f'{self.test_id}.in') 649 if not os.path.exists(template_path): 650 if os.path.exists(self.test_case.input_path): 651 shutil.copyfile(self.test_case.input_path, self.pdf_path) 652 return self.test_case.NewResult(result_types.PASS) 653 654 return self.RunCommand([ 655 sys.executable, _per_process_state.fixup_path, 656 f'--output-dir={self.working_dir}', template_path 657 ]) 658 659 def TestText(self): 660 txt_path = os.path.join(self.working_dir, f'{self.test_id}.txt') 661 with open(txt_path, 'w') as outfile: 662 cmd_to_run = [ 663 _per_process_state.pdfium_test_path, '--send-events', 664 f'--time={TEST_SEED_TIME}' 665 ] 666 667 if self.options.disable_javascript: 668 cmd_to_run.append('--disable-javascript') 669 670 if self.options.disable_xfa: 671 cmd_to_run.append('--disable-xfa') 672 673 cmd_to_run.append(self.pdf_path) 674 test_result = self.RunCommand(cmd_to_run, stdout=outfile) 675 if not test_result.IsPass(): 676 return test_result 677 678 # If the expected file does not exist, the output is expected to be empty. 679 expected_txt_path = os.path.join(self.source_dir, 680 f'{self.test_id}_expected.txt') 681 if not os.path.exists(expected_txt_path): 682 return self._VerifyEmptyText(txt_path) 683 684 # If JavaScript is disabled, the output should be empty. 685 # However, if the test is suppressed and JavaScript is disabled, do not 686 # verify that the text is empty so the suppressed test does not surprise. 687 if self.options.disable_javascript and not self.IsResultSuppressed(): 688 return self._VerifyEmptyText(txt_path) 689 690 return self.RunCommand([ 691 sys.executable, _per_process_state.text_diff_path, expected_txt_path, 692 txt_path 693 ]) 694 695 def _VerifyEmptyText(self, txt_path): 696 with open(txt_path, 'rb') as txt_file: 697 txt_data = txt_file.read() 698 699 if txt_data: 700 return self.test_case.NewResult( 701 result_types.FAIL, 702 log=txt_data.decode(errors='backslashreplace'), 703 reason=f'{txt_path} should be empty') 704 705 return self.test_case.NewResult(result_types.PASS) 706 707 # TODO(crbug.com/pdfium/1656): Remove when ready to fully switch over to 708 # Skia Gold 709 def TestPixel(self): 710 # Remove any existing generated images from previous runs. 711 self.actual_images = _per_process_state.image_differ.GetActualFiles( 712 self.input_filename, self.source_dir, self.working_dir) 713 self._CleanupPixelTest() 714 715 # Generate images. 716 cmd_to_run = [ 717 _per_process_state.pdfium_test_path, '--send-events', '--png', '--md5', 718 f'--time={TEST_SEED_TIME}' 719 ] 720 721 if 'use_ahem' in self.source_dir: 722 font_path = os.path.join(_per_process_state.font_dir, 'ahem') 723 cmd_to_run.append(f'--font-dir={font_path}') 724 elif 'use_symbolneu' in self.source_dir: 725 font_path = os.path.join(_per_process_state.font_dir, 'symbolneu') 726 cmd_to_run.append(f'--font-dir={font_path}') 727 else: 728 cmd_to_run.append(f'--font-dir={_per_process_state.third_party_font_dir}') 729 cmd_to_run.append('--croscore-font-names') 730 731 if self.options.disable_javascript: 732 cmd_to_run.append('--disable-javascript') 733 734 if self.options.disable_xfa: 735 cmd_to_run.append('--disable-xfa') 736 737 if self.options.render_oneshot: 738 cmd_to_run.append('--render-oneshot') 739 740 if self.options.reverse_byte_order: 741 cmd_to_run.append('--reverse-byte-order') 742 743 if self.options.use_renderer: 744 cmd_to_run.append(f'--use-renderer={self.options.use_renderer}') 745 746 cmd_to_run.append(self.pdf_path) 747 748 with BytesIO() as command_output: 749 test_result = self.RunCommand(cmd_to_run, stdout=command_output) 750 if not test_result.IsPass(): 751 return test_result 752 753 test_result.image_artifacts = [] 754 for line in command_output.getvalue().splitlines(): 755 # Expect this format: MD5:<path to image file>:<hexadecimal MD5 hash> 756 line = bytes.decode(line).strip() 757 if line.startswith('MD5:'): 758 image_path, md5_hash = line[4:].rsplit(':', 1) 759 test_result.image_artifacts.append( 760 self._NewImageArtifact( 761 image_path=image_path.strip(), md5_hash=md5_hash.strip())) 762 763 if self.actual_images: 764 image_diffs = _per_process_state.image_differ.ComputeDifferences( 765 self.input_filename, 766 self.source_dir, 767 self.working_dir, 768 image_matching_algorithm=self.GetImageMatchingAlgorithm()) 769 if image_diffs: 770 test_result.status = result_types.FAIL 771 test_result.reason = 'Images differ' 772 773 # Merge image diffs into test result. 774 diff_map = {} 775 diff_log = [] 776 for diff in image_diffs: 777 diff_map[diff.actual_path] = diff 778 diff_log.append(f'{os.path.basename(diff.actual_path)} vs. ') 779 if diff.expected_path: 780 diff_log.append(f'{os.path.basename(diff.expected_path)}\n') 781 else: 782 diff_log.append('missing expected file\n') 783 784 for artifact in test_result.image_artifacts: 785 artifact.image_diff = diff_map.get(artifact.image_path) 786 test_result.log = ''.join(diff_log) 787 788 elif _per_process_state.enforce_expected_images: 789 if not self.IsImageDiffSuppressed(): 790 test_result.status = result_types.FAIL 791 test_result.reason = 'Missing expected images' 792 793 if not test_result.IsPass(): 794 self._RegenerateIfNeeded() 795 return test_result 796 797 if _per_process_state.delete_output_on_success: 798 self._CleanupPixelTest() 799 return test_result 800 801 def _NewImageArtifact(self, *, image_path, md5_hash): 802 artifact = ImageArtifact(image_path=image_path, md5_hash=md5_hash) 803 804 if self.options.run_skia_gold: 805 if _per_process_state.GetSkiaGoldTester().UploadTestResultToSkiaGold( 806 artifact.GetSkiaGoldId(), artifact.image_path): 807 artifact.skia_gold_status = result_types.PASS 808 else: 809 artifact.skia_gold_status = result_types.FAIL 810 811 return artifact 812 813 def _CleanupPixelTest(self): 814 for image_file in self.actual_images: 815 if os.path.exists(image_file): 816 os.remove(image_file) 817 818 819@dataclass 820class TestCase: 821 """Description of a test case to run. 822 823 Attributes: 824 test_id: A unique identifier for the test. 825 input_path: The absolute path to the test file. 826 """ 827 test_id: str 828 input_path: str 829 830 def NewResult(self, status, **kwargs): 831 """Derives a new test result corresponding to this test case.""" 832 return TestResult(test_id=self.test_id, status=status, **kwargs) 833 834 835@dataclass 836class TestResult: 837 """Results from running a test case. 838 839 Attributes: 840 test_id: The corresponding test case ID. 841 status: The overall `result_types` status. 842 duration_milliseconds: Test time in milliseconds. 843 log: Optional log of the test's output. 844 image_artfacts: Optional list of image artifacts. 845 reason: Optional reason why the test failed. 846 """ 847 test_id: str 848 status: str 849 duration_milliseconds: float = None 850 log: str = None 851 image_artifacts: list = field(default_factory=list) 852 reason: str = None 853 854 def IsPass(self): 855 """Whether the test passed.""" 856 return self.status == result_types.PASS 857 858 859@dataclass 860class ImageArtifact: 861 """Image artifact for a test result. 862 863 Attributes: 864 image_path: The absolute path to the image file. 865 md5_hash: The MD5 hash of the pixel buffer. 866 skia_gold_status: Optional Skia Gold status. 867 image_diff: Optional image diff. 868 """ 869 image_path: str 870 md5_hash: str 871 skia_gold_status: str = None 872 image_diff: pngdiffer.ImageDiff = None 873 874 def GetSkiaGoldId(self): 875 # The output filename without image extension becomes the test ID. For 876 # example, "/path/to/.../testing/corpus/example_005.pdf.0.png" becomes 877 # "example_005.pdf.0". 878 return _GetTestId(os.path.basename(self.image_path)) 879 880 def GetDiffStatus(self): 881 return result_types.FAIL if self.image_diff else result_types.PASS 882 883 def GetDiffReason(self): 884 return self.image_diff.reason if self.image_diff else None 885 886 def GetDiffArtifacts(self): 887 if not self.image_diff: 888 return None 889 if not self.image_diff.expected_path or not self.image_diff.diff_path: 890 return None 891 return { 892 'actual_image': 893 _GetArtifactFromFilePath(self.image_path), 894 'expected_image': 895 _GetArtifactFromFilePath(self.image_diff.expected_path), 896 'image_diff': 897 _GetArtifactFromFilePath(self.image_diff.diff_path) 898 } 899 900 901class TestCaseManager: 902 """Manages a collection of test cases.""" 903 904 def __init__(self): 905 self.test_cases = {} 906 907 def __len__(self): 908 return len(self.test_cases) 909 910 def __iter__(self): 911 return iter(self.test_cases.values()) 912 913 def NewTestCase(self, input_path, **kwargs): 914 """Creates and registers a new test case.""" 915 input_basename = os.path.basename(input_path) 916 test_id = _GetTestId(input_basename) 917 if test_id in self.test_cases: 918 raise ValueError( 919 f'Test ID "{test_id}" derived from "{input_basename}" must be unique') 920 921 test_case = TestCase(test_id=test_id, input_path=input_path, **kwargs) 922 self.test_cases[test_id] = test_case 923 return test_case 924 925 def GetTestCase(self, test_id): 926 """Looks up a test case previously registered by `NewTestCase()`.""" 927 return self.test_cases[test_id] 928 929 930def _GetTestId(input_basename): 931 """Constructs a test ID by stripping the last extension from the basename.""" 932 return os.path.splitext(input_basename)[0] 933 934 935def _GetArtifactFromFilePath(file_path): 936 """Constructs a ResultSink artifact from a file path.""" 937 return {'filePath': file_path} 938
