• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/env python
2# Copyright 2016 The PDFium Authors. All rights reserved.
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5
6import cStringIO
7import functools
8import multiprocessing
9import optparse
10import os
11import re
12import shutil
13import subprocess
14import sys
15
16import common
17import gold
18import pngdiffer
19import suppressor
20
21class KeyboardInterruptError(Exception): pass
22
23# Nomenclature:
24#   x_root - "x"
25#   x_filename - "x.ext"
26#   x_path - "path/to/a/b/c/x.ext"
27#   c_dir - "path/to/a/b/c"
28
29def TestOneFileParallel(this, test_case):
30  """Wrapper to call GenerateAndTest() and redirect output to stdout."""
31  try:
32    input_filename, source_dir = test_case
33    result = this.GenerateAndTest(input_filename, source_dir);
34    return (result, input_filename, source_dir)
35  except KeyboardInterrupt:
36    raise KeyboardInterruptError()
37
38
39class TestRunner:
40  def __init__(self, dirname):
41    self.test_dir = dirname
42    self.enforce_expected_images = False
43    self.oneshot_renderer = False
44
45  # GenerateAndTest returns a tuple <success, outputfiles> where
46  # success is a boolean indicating whether the tests passed comparison
47  # tests and outputfiles is a list tuples:
48  #          (path_to_image, md5_hash_of_pixelbuffer)
49  def GenerateAndTest(self, input_filename, source_dir):
50    input_root, _ = os.path.splitext(input_filename)
51    expected_txt_path = os.path.join(source_dir, input_root + '_expected.txt')
52
53    pdf_path = os.path.join(self.working_dir, input_root + '.pdf')
54
55    # Remove any existing generated images from previous runs.
56    actual_images = self.image_differ.GetActualFiles(input_filename, source_dir,
57                                                     self.working_dir)
58    for image in actual_images:
59      if os.path.exists(image):
60        os.remove(image)
61
62    sys.stdout.flush()
63
64    raised_exception = self.Generate(source_dir, input_filename, input_root,
65                                     pdf_path)
66
67    if raised_exception is not None:
68      print 'FAILURE: %s; %s' % (input_filename, raised_exception)
69      return False, []
70
71    results = []
72    if os.path.exists(expected_txt_path):
73      raised_exception = self.TestText(input_root, expected_txt_path, pdf_path)
74    else:
75      raised_exception, results = self.TestPixel(input_root, pdf_path)
76
77    if raised_exception is not None:
78      print 'FAILURE: %s; %s' % (input_filename, raised_exception)
79      return False, results
80
81    if actual_images:
82      if self.image_differ.HasDifferences(input_filename, source_dir,
83                                          self.working_dir):
84        if (self.options.regenerate_expected
85            and not self.test_suppressor.IsResultSuppressed(input_filename)
86            and not self.test_suppressor.IsImageDiffSuppressed(input_filename)):
87          platform_only = (self.options.regenerate_expected == 'platform')
88          self.image_differ.Regenerate(input_filename, source_dir,
89                                       self.working_dir, platform_only)
90        return False, results
91    else:
92      if (self.enforce_expected_images
93          and not self.test_suppressor.IsImageDiffSuppressed(input_filename)):
94        print 'FAILURE: %s; Missing expected images' % input_filename
95        return False, results
96
97    return True, results
98
99  def Generate(self, source_dir, input_filename, input_root, pdf_path):
100    original_path = os.path.join(source_dir, input_filename)
101    input_path = os.path.join(source_dir, input_root + '.in')
102
103    input_event_path = os.path.join(source_dir, input_root + '.evt')
104    if os.path.exists(input_event_path):
105      output_event_path = os.path.splitext(pdf_path)[0] + '.evt'
106      shutil.copyfile(input_event_path, output_event_path)
107
108    if not os.path.exists(input_path):
109      if os.path.exists(original_path):
110        shutil.copyfile(original_path, pdf_path)
111      return None
112
113    sys.stdout.flush()
114
115    return common.RunCommand(
116        [sys.executable, self.fixup_path, '--output-dir=' + self.working_dir,
117            input_path])
118
119  def TestText(self, input_root, expected_txt_path, pdf_path):
120    txt_path = os.path.join(self.working_dir, input_root + '.txt')
121
122    with open(txt_path, 'w') as outfile:
123      cmd_to_run = [self.pdfium_test_path, '--send-events', pdf_path]
124      subprocess.check_call(cmd_to_run, stdout=outfile)
125
126    cmd = [sys.executable, self.text_diff_path, expected_txt_path, txt_path]
127    return common.RunCommand(cmd)
128
129  def TestPixel(self, input_root, pdf_path):
130    cmd_to_run = [self.pdfium_test_path, '--send-events', '--png', '--md5']
131    if self.oneshot_renderer:
132      cmd_to_run.append('--render-oneshot')
133    cmd_to_run.append(pdf_path)
134    return common.RunCommandExtractHashedFiles(cmd_to_run)
135
136  def HandleResult(self, input_filename, input_path, result):
137    success, image_paths = result
138
139    if image_paths:
140      for img_path, md5_hash in image_paths:
141        # The output filename without image extension becomes the test name.
142        # For example, "/path/to/.../testing/corpus/example_005.pdf.0.png"
143        # becomes "example_005.pdf.0".
144        test_name = os.path.splitext(os.path.split(img_path)[1])[0]
145
146        if not self.test_suppressor.IsResultSuppressed(input_filename):
147          matched = self.gold_baseline.MatchLocalResult(test_name, md5_hash)
148          if matched == gold.GoldBaseline.MISMATCH:
149            print 'Skia Gold hash mismatch for test case: %s' % test_name
150          elif matched ==  gold.GoldBaseline.NO_BASELINE:
151            print 'No Skia Gold baseline found for test case: %s' % test_name
152
153        if self.gold_results:
154          self.gold_results.AddTestResult(test_name, md5_hash, img_path)
155
156    if self.test_suppressor.IsResultSuppressed(input_filename):
157      self.result_suppressed_cases.append(input_filename)
158      if success:
159        self.surprises.append(input_path)
160    else:
161      if not success:
162        self.failures.append(input_path)
163
164  def Run(self):
165    parser = optparse.OptionParser()
166
167    parser.add_option('--build-dir', default=os.path.join('out', 'Debug'),
168                      help='relative path from the base source directory')
169
170    parser.add_option('-j', default=multiprocessing.cpu_count(),
171                      dest='num_workers', type='int',
172                      help='run NUM_WORKERS jobs in parallel')
173
174    parser.add_option('--gold_properties', default='', dest="gold_properties",
175                      help='Key value pairs that are written to the top level '
176                           'of the JSON file that is ingested by Gold.')
177
178    parser.add_option('--gold_key', default='', dest="gold_key",
179                      help='Key value pairs that are added to the "key" field '
180                           'of the JSON file that is ingested by Gold.')
181
182    parser.add_option('--gold_output_dir', default='', dest="gold_output_dir",
183                      help='Path of where to write the JSON output to be '
184                           'uploaded to Gold.')
185
186    parser.add_option('--gold_ignore_hashes', default='',
187                      dest="gold_ignore_hashes",
188                      help='Path to a file with MD5 hashes we wish to ignore.')
189
190    parser.add_option('--regenerate_expected', default='',
191                      dest="regenerate_expected",
192                      help='Regenerates expected images. Valid values are '
193                           '"all" to regenerate all expected pngs, and '
194                           '"platform" to regenerate only platform-specific '
195                           'expected pngs.')
196
197    parser.add_option('--ignore_errors', action="store_true",
198                      dest="ignore_errors",
199                      help='Prevents the return value from being non-zero '
200                           'when image comparison fails.')
201
202    self.options, self.args = parser.parse_args()
203
204    if (self.options.regenerate_expected
205        and self.options.regenerate_expected not in ['all', 'platform']) :
206      print 'FAILURE: --regenerate_expected must be "all" or "platform"'
207      return 1
208
209    finder = common.DirectoryFinder(self.options.build_dir)
210    self.fixup_path = finder.ScriptPath('fixup_pdf_template.py')
211    self.text_diff_path = finder.ScriptPath('text_diff.py')
212
213    self.source_dir = finder.TestingDir()
214    if self.test_dir != 'corpus':
215      test_dir = finder.TestingDir(os.path.join('resources', self.test_dir))
216    else:
217      test_dir = finder.TestingDir(self.test_dir)
218
219    self.pdfium_test_path = finder.ExecutablePath('pdfium_test')
220    if not os.path.exists(self.pdfium_test_path):
221      print "FAILURE: Can't find test executable '%s'" % self.pdfium_test_path
222      print 'Use --build-dir to specify its location.'
223      return 1
224
225    self.working_dir = finder.WorkingDir(os.path.join('testing', self.test_dir))
226    if not os.path.exists(self.working_dir):
227      os.makedirs(self.working_dir)
228
229    self.feature_string = subprocess.check_output([self.pdfium_test_path,
230                                                   '--show-config'])
231    self.test_suppressor = suppressor.Suppressor(finder, self.feature_string)
232    self.image_differ = pngdiffer.PNGDiffer(finder)
233
234    self.gold_baseline = gold.GoldBaseline(self.options.gold_properties)
235
236    walk_from_dir = finder.TestingDir(test_dir);
237
238    self.test_cases = []
239    self.execution_suppressed_cases = []
240    input_file_re = re.compile('^.+[.](in|pdf)$')
241    if self.args:
242      for file_name in self.args:
243        file_name.replace('.pdf', '.in')
244        input_path = os.path.join(walk_from_dir, file_name)
245        if not os.path.isfile(input_path):
246          print "Can't find test file '%s'" % file_name
247          return 1
248
249        self.test_cases.append((os.path.basename(input_path),
250                           os.path.dirname(input_path)))
251    else:
252      for file_dir, _, filename_list in os.walk(walk_from_dir):
253        for input_filename in filename_list:
254          if input_file_re.match(input_filename):
255            input_path = os.path.join(file_dir, input_filename)
256            if self.test_suppressor.IsExecutionSuppressed(input_path):
257              self.execution_suppressed_cases.append(input_path)
258            else:
259              if os.path.isfile(input_path):
260                self.test_cases.append((input_filename, file_dir))
261
262    self.failures = []
263    self.surprises = []
264    self.result_suppressed_cases = []
265
266    # Collect Gold results if an output directory was named.
267    self.gold_results = None
268    if self.options.gold_output_dir:
269      self.gold_results = gold.GoldResults('pdfium',
270                                           self.options.gold_output_dir,
271                                           self.options.gold_properties,
272                                           self.options.gold_key,
273                                           self.options.gold_ignore_hashes)
274
275    if self.options.num_workers > 1 and len(self.test_cases) > 1:
276      try:
277        pool = multiprocessing.Pool(self.options.num_workers)
278        worker_func = functools.partial(TestOneFileParallel, self)
279
280        worker_results = pool.imap(worker_func, self.test_cases)
281        for worker_result in worker_results:
282          result, input_filename, source_dir = worker_result
283          input_path = os.path.join(source_dir, input_filename)
284
285          self.HandleResult(input_filename, input_path, result)
286
287      except KeyboardInterrupt:
288        pool.terminate()
289      finally:
290        pool.close()
291        pool.join()
292    else:
293      for test_case in self.test_cases:
294        input_filename, input_file_dir = test_case
295        result = self.GenerateAndTest(input_filename, input_file_dir)
296        self.HandleResult(input_filename,
297                          os.path.join(input_file_dir, input_filename), result)
298
299    if self.gold_results:
300      self.gold_results.WriteResults()
301
302    if self.surprises:
303      self.surprises.sort()
304      print '\n\nUnexpected Successes:'
305      for surprise in self.surprises:
306        print surprise
307
308    if self.failures:
309      self.failures.sort()
310      print '\n\nSummary of Failures:'
311      for failure in self.failures:
312        print failure
313
314    self._PrintSummary()
315
316    if self.failures:
317      if not self.options.ignore_errors:
318        return 1
319
320    return 0
321
322  def _PrintSummary(self):
323    number_test_cases = len(self.test_cases)
324    number_failures = len(self.failures)
325    number_suppressed = len(self.result_suppressed_cases)
326    number_successes = number_test_cases - number_failures - number_suppressed
327    number_surprises = len(self.surprises)
328    print
329    print 'Test cases executed: %d' % number_test_cases
330    print '  Successes: %d' % number_successes
331    print '  Suppressed: %d' % number_suppressed
332    print '    Surprises: %d' % number_surprises
333    print '  Failures: %d' % number_failures
334    print
335    print 'Test cases not executed: %d' % len(self.execution_suppressed_cases)
336
337  def SetEnforceExpectedImages(self, new_value):
338    """Set whether to enforce that each test case provide an expected image."""
339    self.enforce_expected_images = new_value
340
341  def SetOneShotRenderer(self, new_value):
342    """Set whether to use the oneshot renderer. """
343    self.oneshot_renderer = new_value
344