1#!/usr/bin/env python 2# Copyright 2016 The PDFium Authors. All rights reserved. 3# Use of this source code is governed by a BSD-style license that can be 4# found in the LICENSE file. 5 6import cStringIO 7import functools 8import multiprocessing 9import optparse 10import os 11import re 12import shutil 13import subprocess 14import sys 15 16import common 17import gold 18import pngdiffer 19import suppressor 20 21class KeyboardInterruptError(Exception): pass 22 23# Nomenclature: 24# x_root - "x" 25# x_filename - "x.ext" 26# x_path - "path/to/a/b/c/x.ext" 27# c_dir - "path/to/a/b/c" 28 29def TestOneFileParallel(this, test_case): 30 """Wrapper to call GenerateAndTest() and redirect output to stdout.""" 31 try: 32 input_filename, source_dir = test_case 33 result = this.GenerateAndTest(input_filename, source_dir); 34 return (result, input_filename, source_dir) 35 except KeyboardInterrupt: 36 raise KeyboardInterruptError() 37 38 39class TestRunner: 40 def __init__(self, dirname): 41 self.test_dir = dirname 42 43 # GenerateAndTest returns a tuple <success, outputfiles> where 44 # success is a boolean indicating whether the tests passed comparison 45 # tests and outputfiles is a list tuples: 46 # (path_to_image, md5_hash_of_pixelbuffer) 47 def GenerateAndTest(self, input_filename, source_dir): 48 input_root, _ = os.path.splitext(input_filename) 49 expected_txt_path = os.path.join(source_dir, input_root + '_expected.txt') 50 51 pdf_path = os.path.join(self.working_dir, input_root + '.pdf') 52 53 # Remove any existing generated images from previous runs. 54 actual_images = self.image_differ.GetActualFiles(input_filename, source_dir, 55 self.working_dir) 56 for image in actual_images: 57 if os.path.exists(image): 58 os.remove(image) 59 60 sys.stdout.flush() 61 62 raised_exception = self.Generate(source_dir, input_filename, input_root, 63 pdf_path) 64 65 if raised_exception != None: 66 print "FAILURE: " + input_filename + "; " + str(raised_exception) 67 return False, [] 68 69 results = [] 70 if os.path.exists(expected_txt_path): 71 raised_exception = self.TestText(input_root, expected_txt_path, pdf_path) 72 else: 73 raised_exception, results = self.TestPixel(input_root, pdf_path) 74 75 if raised_exception != None: 76 print "FAILURE: " + input_filename + "; " + str(raised_exception) 77 return False, results 78 79 if len(actual_images): 80 if self.image_differ.HasDifferences(input_filename, source_dir, 81 self.working_dir): 82 return False, results 83 return True, results 84 85 def Generate(self, source_dir, input_filename, input_root, pdf_path): 86 original_path = os.path.join(source_dir, input_filename) 87 input_path = os.path.join(source_dir, input_root + '.in') 88 89 input_event_path = os.path.join(source_dir, input_root + ".evt") 90 if os.path.exists(input_event_path): 91 output_event_path = os.path.splitext(pdf_path)[0] + ".evt" 92 shutil.copyfile(input_event_path, output_event_path) 93 94 if not os.path.exists(input_path): 95 if os.path.exists(original_path): 96 shutil.copyfile(original_path, pdf_path) 97 return None 98 99 sys.stdout.flush() 100 101 return common.RunCommand( 102 [sys.executable, self.fixup_path, '--output-dir=' + self.working_dir, 103 input_path]) 104 105 106 def TestText(self, input_root, expected_txt_path, pdf_path): 107 txt_path = os.path.join(self.working_dir, input_root + '.txt') 108 109 with open(txt_path, 'w') as outfile: 110 cmd_to_run = [self.pdfium_test_path, pdf_path] 111 subprocess.check_call(cmd_to_run, stdout=outfile) 112 113 cmd = [sys.executable, self.text_diff_path, expected_txt_path, txt_path] 114 return common.RunCommand(cmd) 115 116 117 def TestPixel(self, input_root, pdf_path): 118 cmd_to_run = [self.pdfium_test_path, '--send-events', '--png'] 119 if self.gold_results: 120 cmd_to_run.append('--md5') 121 cmd_to_run.append(pdf_path) 122 return common.RunCommandExtractHashedFiles(cmd_to_run) 123 124 def HandleResult(self, input_filename, input_path, result): 125 success, image_paths = result 126 if self.gold_results: 127 if image_paths: 128 for img_path, md5_hash in image_paths: 129 # the output filename (without extension becomes the test name) 130 test_name = os.path.splitext(os.path.split(img_path)[1])[0] 131 self.gold_results.AddTestResult(test_name, md5_hash, img_path) 132 133 if self.test_suppressor.IsResultSuppressed(input_filename): 134 if success: 135 self.surprises.append(input_path) 136 else: 137 if not success: 138 self.failures.append(input_path) 139 140 141 def Run(self): 142 parser = optparse.OptionParser() 143 144 parser.add_option('--build-dir', default=os.path.join('out', 'Debug'), 145 help='relative path from the base source directory') 146 147 parser.add_option('-j', default=multiprocessing.cpu_count(), 148 dest='num_workers', type='int', 149 help='run NUM_WORKERS jobs in parallel') 150 151 parser.add_option('--gold_properties', default='', dest="gold_properties", 152 help='Key value pairs that are written to the top level of the JSON file that is ingested by Gold.') 153 154 parser.add_option('--gold_key', default='', dest="gold_key", 155 help='Key value pairs that are added to the "key" field of the JSON file that is ingested by Gold.') 156 157 parser.add_option('--gold_output_dir', default='', dest="gold_output_dir", 158 help='Path of where to write the JSON output to be uploaded to Gold.') 159 160 parser.add_option('--gold_ignore_hashes', default='', dest="gold_ignore_hashes", 161 help='Path to a file with MD5 hashes we wish to ignore.') 162 163 parser.add_option('--ignore_errors', action="store_true", dest="ignore_errors", 164 help='Prevents the return value from being non-zero when image comparison fails.') 165 166 options, args = parser.parse_args() 167 168 finder = common.DirectoryFinder(options.build_dir) 169 self.fixup_path = finder.ScriptPath('fixup_pdf_template.py') 170 self.text_diff_path = finder.ScriptPath('text_diff.py') 171 172 self.source_dir = finder.TestingDir() 173 if self.test_dir != 'corpus': 174 test_dir = finder.TestingDir(os.path.join('resources', self.test_dir)) 175 else: 176 test_dir = finder.TestingDir(self.test_dir) 177 178 self.pdfium_test_path = finder.ExecutablePath('pdfium_test') 179 if not os.path.exists(self.pdfium_test_path): 180 print "FAILURE: Can't find test executable '%s'" % self.pdfium_test_path 181 print "Use --build-dir to specify its location." 182 return 1 183 184 self.working_dir = finder.WorkingDir(os.path.join('testing', self.test_dir)) 185 if not os.path.exists(self.working_dir): 186 os.makedirs(self.working_dir) 187 188 self.feature_string = subprocess.check_output([self.pdfium_test_path, 189 '--show-config']) 190 self.test_suppressor = suppressor.Suppressor(finder, self.feature_string) 191 self.image_differ = pngdiffer.PNGDiffer(finder) 192 193 walk_from_dir = finder.TestingDir(test_dir); 194 195 test_cases = [] 196 input_file_re = re.compile('^[a-zA-Z0-9_.]+[.](in|pdf)$') 197 if len(args): 198 for file_name in args: 199 file_name.replace(".pdf", ".in") 200 input_path = os.path.join(walk_from_dir, file_name) 201 if not os.path.isfile(input_path): 202 print "Can't find test file '%s'" % file_name 203 return 1 204 205 test_cases.append((os.path.basename(input_path), 206 os.path.dirname(input_path))) 207 else: 208 for file_dir, _, filename_list in os.walk(walk_from_dir): 209 for input_filename in filename_list: 210 if input_file_re.match(input_filename): 211 input_path = os.path.join(file_dir, input_filename) 212 if not self.test_suppressor.IsExecutionSuppressed(input_path): 213 if os.path.isfile(input_path): 214 test_cases.append((input_filename, file_dir)) 215 216 self.failures = [] 217 self.surprises = [] 218 219 # Collect Gold results if an output directory was named. 220 self.gold_results = None 221 if options.gold_output_dir: 222 self.gold_results = gold.GoldResults("pdfium", 223 options.gold_output_dir, 224 options.gold_properties, 225 options.gold_key, 226 options.gold_ignore_hashes) 227 228 if options.num_workers > 1 and len(test_cases) > 1: 229 try: 230 pool = multiprocessing.Pool(options.num_workers) 231 worker_func = functools.partial(TestOneFileParallel, self) 232 233 worker_results = pool.imap(worker_func, test_cases) 234 for worker_result in worker_results: 235 result, input_filename, source_dir = worker_result 236 input_path = os.path.join(source_dir, input_filename) 237 238 self.HandleResult(input_filename, input_path, result) 239 240 except KeyboardInterrupt: 241 pool.terminate() 242 finally: 243 pool.close() 244 pool.join() 245 else: 246 for test_case in test_cases: 247 input_filename, input_file_dir = test_case 248 result = self.GenerateAndTest(input_filename, input_file_dir) 249 self.HandleResult(input_filename, 250 os.path.join(input_file_dir, input_filename), result) 251 252 if self.gold_results: 253 self.gold_results.WriteResults() 254 255 if self.surprises: 256 self.surprises.sort() 257 print '\n\nUnexpected Successes:' 258 for surprise in self.surprises: 259 print surprise; 260 261 if self.failures: 262 self.failures.sort() 263 print '\n\nSummary of Failures:' 264 for failure in self.failures: 265 print failure 266 267 if not options.ignore_errors: 268 return 1 269 return 0 270