• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/python
2
3'''
4Copyright 2012 Google Inc.
5
6Use of this source code is governed by a BSD-style license that can be
7found in the LICENSE file.
8'''
9
10'''
11Rebaselines the given GM tests, on all bots and all configurations.
12'''
13
14# System-level imports
15import argparse
16import json
17import os
18import re
19import subprocess
20import sys
21import urllib2
22
23# Imports from within Skia
24#
25# We need to add the 'gm' directory, so that we can import gm_json.py within
26# that directory.  That script allows us to parse the actual-results.json file
27# written out by the GM tool.
28# Make sure that the 'gm' dir is in the PYTHONPATH, but add it at the *end*
29# so any dirs that are already in the PYTHONPATH will be preferred.
30#
31# This assumes that the 'gm' directory has been checked out as a sibling of
32# the 'tools' directory containing this script, which will be the case if
33# 'trunk' was checked out as a single unit.
34GM_DIRECTORY = os.path.realpath(
35    os.path.join(os.path.dirname(os.path.dirname(__file__)), 'gm'))
36if GM_DIRECTORY not in sys.path:
37  sys.path.append(GM_DIRECTORY)
38import gm_json
39
40# TODO(epoger): In the long run, we want to build this list automatically,
41# but for now we hard-code it until we can properly address
42# https://code.google.com/p/skia/issues/detail?id=1544
43# ('live query of builder list makes rebaseline.py slow to start up')
44TEST_BUILDERS = [
45    'Test-Android-GalaxyNexus-SGX540-Arm7-Debug',
46    'Test-Android-GalaxyNexus-SGX540-Arm7-Release',
47    'Test-Android-IntelRhb-SGX544-x86-Debug',
48    'Test-Android-IntelRhb-SGX544-x86-Release',
49    'Test-Android-Nexus10-MaliT604-Arm7-Debug',
50    'Test-Android-Nexus10-MaliT604-Arm7-Release',
51    'Test-Android-Nexus4-Adreno320-Arm7-Debug',
52    'Test-Android-Nexus4-Adreno320-Arm7-Release',
53    'Test-Android-Nexus7-Tegra3-Arm7-Debug',
54    'Test-Android-Nexus7-Tegra3-Arm7-Release',
55    'Test-Android-NexusS-SGX540-Arm7-Debug',
56    'Test-Android-NexusS-SGX540-Arm7-Release',
57    'Test-Android-Xoom-Tegra2-Arm7-Debug',
58    'Test-Android-Xoom-Tegra2-Arm7-Release',
59    'Test-ChromeOS-Alex-GMA3150-x86-Debug',
60    'Test-ChromeOS-Alex-GMA3150-x86-Release',
61    'Test-ChromeOS-Daisy-MaliT604-Arm7-Debug',
62    'Test-ChromeOS-Daisy-MaliT604-Arm7-Release',
63    'Test-ChromeOS-Link-HD4000-x86_64-Debug',
64    'Test-ChromeOS-Link-HD4000-x86_64-Release',
65    'Test-Mac10.6-MacMini4.1-GeForce320M-x86-Debug',
66    'Test-Mac10.6-MacMini4.1-GeForce320M-x86-Release',
67    'Test-Mac10.6-MacMini4.1-GeForce320M-x86_64-Debug',
68    'Test-Mac10.6-MacMini4.1-GeForce320M-x86_64-Release',
69    'Test-Mac10.7-MacMini4.1-GeForce320M-x86-Debug',
70    'Test-Mac10.7-MacMini4.1-GeForce320M-x86-Release',
71    'Test-Mac10.7-MacMini4.1-GeForce320M-x86_64-Debug',
72    'Test-Mac10.7-MacMini4.1-GeForce320M-x86_64-Release',
73    'Test-Mac10.8-MacMini4.1-GeForce320M-x86-Debug',
74    'Test-Mac10.8-MacMini4.1-GeForce320M-x86-Release',
75    'Test-Mac10.8-MacMini4.1-GeForce320M-x86_64-Debug',
76    'Test-Mac10.8-MacMini4.1-GeForce320M-x86_64-Release',
77    'Test-Ubuntu12-ShuttleA-ATI5770-x86-Debug',
78    'Test-Ubuntu12-ShuttleA-ATI5770-x86-Release',
79    'Test-Ubuntu12-ShuttleA-ATI5770-x86_64-Debug',
80    'Test-Ubuntu12-ShuttleA-ATI5770-x86_64-Release',
81    'Test-Ubuntu12-ShuttleA-HD2000-x86_64-Release-Valgrind',
82    'Test-Ubuntu12-ShuttleA-NoGPU-x86_64-Debug',
83    'Test-Ubuntu13-ShuttleA-HD2000-x86_64-Debug-ASAN',
84    'Test-Win7-ShuttleA-HD2000-x86-Debug',
85    'Test-Win7-ShuttleA-HD2000-x86-Debug-ANGLE',
86    'Test-Win7-ShuttleA-HD2000-x86-Debug-DirectWrite',
87    'Test-Win7-ShuttleA-HD2000-x86-Release',
88    'Test-Win7-ShuttleA-HD2000-x86-Release-ANGLE',
89    'Test-Win7-ShuttleA-HD2000-x86-Release-DirectWrite',
90    'Test-Win7-ShuttleA-HD2000-x86_64-Debug',
91    'Test-Win7-ShuttleA-HD2000-x86_64-Release',
92]
93
94# TODO: Get this from builder_name_schema in buildbot.
95TRYBOT_SUFFIX = '-Trybot'
96
97
98class _InternalException(Exception):
99  pass
100
101class ExceptionHandler(object):
102  """ Object that handles exceptions, either raising them immediately or
103  collecting them to display later on."""
104
105  # params:
106  def __init__(self, keep_going_on_failure=False):
107    """
108    params:
109      keep_going_on_failure: if False, report failures and quit right away;
110                             if True, collect failures until
111                             ReportAllFailures() is called
112    """
113    self._keep_going_on_failure = keep_going_on_failure
114    self._failures_encountered = []
115
116  def RaiseExceptionOrContinue(self):
117    """ We have encountered an exception; either collect the info and keep
118    going, or exit the program right away."""
119    # Get traceback information about the most recently raised exception.
120    exc_info = sys.exc_info()
121
122    if self._keep_going_on_failure:
123      print >> sys.stderr, ('WARNING: swallowing exception %s' %
124                            repr(exc_info[1]))
125      self._failures_encountered.append(exc_info)
126    else:
127      print >> sys.stderr, (
128          '\nHalting at first exception.\n' +
129          'Please file a bug to epoger@google.com at ' +
130          'https://code.google.com/p/skia/issues/entry, containing the ' +
131          'command you ran and the following stack trace.\n\n' +
132          'Afterwards, you can re-run with the --keep-going-on-failure ' +
133          'option set.\n')
134      raise exc_info[1], None, exc_info[2]
135
136  def ReportAllFailures(self):
137    if self._failures_encountered:
138      print >> sys.stderr, ('Encountered %d failures (see above).' %
139                            len(self._failures_encountered))
140      sys.exit(1)
141
142
143# Object that rebaselines a JSON expectations file (not individual image files).
144class JsonRebaseliner(object):
145
146  # params:
147  #  expectations_root: root directory of all expectations JSON files
148  #  expectations_input_filename: filename (under expectations_root) of JSON
149  #                               expectations file to read; typically
150  #                               "expected-results.json"
151  #  expectations_output_filename: filename (under expectations_root) to
152  #                                which updated expectations should be
153  #                                written; typically the same as
154  #                                expectations_input_filename, to overwrite
155  #                                the old content
156  #  actuals_base_url: base URL from which to read actual-result JSON files
157  #  actuals_filename: filename (under actuals_base_url) from which to read a
158  #                    summary of results; typically "actual-results.json"
159  #  exception_handler: reference to rebaseline.ExceptionHandler object
160  #  tests: list of tests to rebaseline, or None if we should rebaseline
161  #         whatever files the JSON results summary file tells us to
162  #  configs: which configs to run for each test, or None if we should
163  #           rebaseline whatever configs the JSON results summary file tells
164  #           us to
165  #  add_new: if True, add expectations for tests which don't have any yet
166  #  add_ignored: if True, add expectations for tests for which failures are
167  #               currently ignored
168  #  bugs: optional list of bug numbers which pertain to these expectations
169  #  notes: free-form text notes to add to all updated expectations
170  #  mark_unreviewed: if True, mark these expectations as NOT having been
171  #                   reviewed by a human; otherwise, leave that field blank.
172  #                   Currently, there is no way to make this script mark
173  #                   expectations as reviewed-by-human=True.
174  #                   TODO(epoger): Add that capability to a review tool.
175  #  mark_ignore_failure: if True, mark failures of a given test as being
176  #                       ignored.
177  #  from_trybot: if True, read actual-result JSON files generated from a
178  #               trybot run rather than a waterfall run.
179  def __init__(self, expectations_root, expectations_input_filename,
180               expectations_output_filename, actuals_base_url,
181               actuals_filename, exception_handler,
182               tests=None, configs=None, add_new=False, add_ignored=False,
183               bugs=None, notes=None, mark_unreviewed=None,
184               mark_ignore_failure=False, from_trybot=False):
185    self._expectations_root = expectations_root
186    self._expectations_input_filename = expectations_input_filename
187    self._expectations_output_filename = expectations_output_filename
188    self._tests = tests
189    self._configs = configs
190    self._actuals_base_url = actuals_base_url
191    self._actuals_filename = actuals_filename
192    self._exception_handler = exception_handler
193    self._add_new = add_new
194    self._add_ignored = add_ignored
195    self._bugs = bugs
196    self._notes = notes
197    self._mark_unreviewed = mark_unreviewed
198    self._mark_ignore_failure = mark_ignore_failure;
199    if self._tests or self._configs:
200      self._image_filename_re = re.compile(gm_json.IMAGE_FILENAME_PATTERN)
201    else:
202      self._image_filename_re = None
203    self._using_svn = os.path.isdir(os.path.join(expectations_root, '.svn'))
204    self._from_trybot = from_trybot
205
206  # Executes subprocess.call(cmd).
207  # Raises an Exception if the command fails.
208  def _Call(self, cmd):
209    if subprocess.call(cmd) != 0:
210      raise _InternalException('error running command: ' + ' '.join(cmd))
211
212  # Returns the full contents of filepath, as a single string.
213  # If filepath looks like a URL, try to read it that way instead of as
214  # a path on local storage.
215  #
216  # Raises _InternalException if there is a problem.
217  def _GetFileContents(self, filepath):
218    if filepath.startswith('http:') or filepath.startswith('https:'):
219      try:
220        return urllib2.urlopen(filepath).read()
221      except urllib2.HTTPError as e:
222        raise _InternalException('unable to read URL %s: %s' % (
223            filepath, e))
224    else:
225      return open(filepath, 'r').read()
226
227  # Returns a dictionary of actual results from actual-results.json file.
228  #
229  # The dictionary returned has this format:
230  # {
231  #  u'imageblur_565.png': [u'bitmap-64bitMD5', 3359963596899141322],
232  #  u'imageblur_8888.png': [u'bitmap-64bitMD5', 4217923806027861152],
233  #  u'shadertext3_8888.png': [u'bitmap-64bitMD5', 3713708307125704716]
234  # }
235  #
236  # If the JSON actual result summary file cannot be loaded, logs a warning
237  # message and returns None.
238  # If the JSON actual result summary file can be loaded, but we have
239  # trouble parsing it, raises an Exception.
240  #
241  # params:
242  #  json_url: URL pointing to a JSON actual result summary file
243  #  sections: a list of section names to include in the results, e.g.
244  #            [gm_json.JSONKEY_ACTUALRESULTS_FAILED,
245  #             gm_json.JSONKEY_ACTUALRESULTS_NOCOMPARISON] ;
246  #            if None, then include ALL sections.
247  def _GetActualResults(self, json_url, sections=None):
248    try:
249      json_contents = self._GetFileContents(json_url)
250    except _InternalException:
251      print >> sys.stderr, (
252          'could not read json_url %s ; skipping this platform.' %
253          json_url)
254      return None
255    json_dict = gm_json.LoadFromString(json_contents)
256    results_to_return = {}
257    actual_results = json_dict[gm_json.JSONKEY_ACTUALRESULTS]
258    if not sections:
259      sections = actual_results.keys()
260    for section in sections:
261      section_results = actual_results[section]
262      if section_results:
263        results_to_return.update(section_results)
264    return results_to_return
265
266  # Rebaseline all tests/types we specified in the constructor,
267  # within this builder's subdirectory in expectations/gm .
268  #
269  # params:
270  #  builder : e.g. 'Test-Win7-ShuttleA-HD2000-x86-Release'
271  def RebaselineSubdir(self, builder):
272    # Read in the actual result summary, and extract all the tests whose
273    # results we need to update.
274    results_builder = str(builder)
275    if self._from_trybot:
276      results_builder = results_builder + TRYBOT_SUFFIX
277    actuals_url = '/'.join([self._actuals_base_url, results_builder,
278                            self._actuals_filename])
279    # Only update results for tests that are currently failing.
280    # We don't want to rewrite results for tests that are already succeeding,
281    # because we don't want to add annotation fields (such as
282    # JSONKEY_EXPECTEDRESULTS_BUGS) except for tests whose expectations we
283    # are actually modifying.
284    sections = [gm_json.JSONKEY_ACTUALRESULTS_FAILED]
285    if self._add_new:
286      sections.append(gm_json.JSONKEY_ACTUALRESULTS_NOCOMPARISON)
287    if self._add_ignored:
288      sections.append(gm_json.JSONKEY_ACTUALRESULTS_FAILUREIGNORED)
289    results_to_update = self._GetActualResults(json_url=actuals_url,
290                                               sections=sections)
291
292    # Read in current expectations.
293    expectations_input_filepath = os.path.join(
294        self._expectations_root, builder, self._expectations_input_filename)
295    expectations_dict = gm_json.LoadFromFile(expectations_input_filepath)
296    expected_results = expectations_dict.get(gm_json.JSONKEY_EXPECTEDRESULTS)
297    if not expected_results:
298      expected_results = {}
299      expectations_dict[gm_json.JSONKEY_EXPECTEDRESULTS] = expected_results
300
301    # Update the expectations in memory, skipping any tests/configs that
302    # the caller asked to exclude.
303    skipped_images = []
304    if results_to_update:
305      for (image_name, image_results) in results_to_update.iteritems():
306        if self._image_filename_re:
307          (test, config) = self._image_filename_re.match(image_name).groups()
308          if self._tests:
309            if test not in self._tests:
310              skipped_images.append(image_name)
311              continue
312          if self._configs:
313            if config not in self._configs:
314              skipped_images.append(image_name)
315              continue
316        if not expected_results.get(image_name):
317          expected_results[image_name] = {}
318        expected_results[image_name]\
319                        [gm_json.JSONKEY_EXPECTEDRESULTS_ALLOWEDDIGESTS]\
320                        = [image_results]
321        if self._mark_unreviewed:
322          expected_results[image_name]\
323                          [gm_json.JSONKEY_EXPECTEDRESULTS_REVIEWED]\
324                          = False
325        if self._mark_ignore_failure:
326          expected_results[image_name]\
327                          [gm_json.JSONKEY_EXPECTEDRESULTS_IGNOREFAILURE]\
328                          = True
329        if self._bugs:
330          expected_results[image_name]\
331                          [gm_json.JSONKEY_EXPECTEDRESULTS_BUGS]\
332                          = self._bugs
333        if self._notes:
334          expected_results[image_name]\
335                          [gm_json.JSONKEY_EXPECTEDRESULTS_NOTES]\
336                          = self._notes
337
338    # Write out updated expectations.
339    expectations_output_filepath = os.path.join(
340        self._expectations_root, builder, self._expectations_output_filename)
341    gm_json.WriteToFile(expectations_dict, expectations_output_filepath)
342
343    # Mark the JSON file as plaintext, so text-style diffs can be applied.
344    # Fixes https://code.google.com/p/skia/issues/detail?id=1442
345    if self._using_svn:
346      self._Call(['svn', 'propset', '--quiet', 'svn:mime-type',
347                  'text/x-json', expectations_output_filepath])
348
349# main...
350
351parser = argparse.ArgumentParser(
352    formatter_class=argparse.RawDescriptionHelpFormatter,
353    epilog='Here is the full set of builders we know about:' +
354           '\n    '.join([''] + sorted(TEST_BUILDERS)))
355parser.add_argument('--actuals-base-url',
356                    help=('base URL from which to read files containing JSON '
357                          'summaries of actual GM results; defaults to '
358                          '%(default)s. To get a specific revision (useful for '
359                          'trybots) replace "svn" with "svn-history/r123". '
360                          'If SKIMAGE is True, defaults to ' +
361                          gm_json.SKIMAGE_ACTUALS_BASE_URL),
362                    default='http://skia-autogen.googlecode.com/svn/gm-actual')
363parser.add_argument('--actuals-filename',
364                    help=('filename (within builder-specific subdirectories '
365                          'of ACTUALS_BASE_URL) to read a summary of results '
366                          'from; defaults to %(default)s'),
367                    default='actual-results.json')
368parser.add_argument('--add-new', action='store_true',
369                    help=('in addition to the standard behavior of '
370                          'updating expectations for failing tests, add '
371                          'expectations for tests which don\'t have '
372                          'expectations yet.'))
373parser.add_argument('--add-ignored', action='store_true',
374                    help=('in addition to the standard behavior of '
375                          'updating expectations for failing tests, add '
376                          'expectations for tests for which failures are '
377                          'currently ignored.'))
378parser.add_argument('--bugs', metavar='BUG', type=int, nargs='+',
379                    help=('Skia bug numbers (under '
380                          'https://code.google.com/p/skia/issues/list ) which '
381                          'pertain to this set of rebaselines.'))
382parser.add_argument('--builders', metavar='BUILDER', nargs='+',
383                    help=('which platforms to rebaseline; '
384                          'if unspecified, rebaseline all known platforms '
385                          '(see below for a list)'))
386# TODO(epoger): Add test that exercises --configs argument.
387parser.add_argument('--configs', metavar='CONFIG', nargs='+',
388                    help=('which configurations to rebaseline, e.g. '
389                          '"--configs 565 8888", as a filter over the full set '
390                          'of results in ACTUALS_FILENAME; if unspecified, '
391                          'rebaseline *all* configs that are available.'))
392parser.add_argument('--expectations-filename',
393                    help=('filename (under EXPECTATIONS_ROOT) to read '
394                          'current expectations from, and to write new '
395                          'expectations into (unless a separate '
396                          'EXPECTATIONS_FILENAME_OUTPUT has been specified); '
397                          'defaults to %(default)s'),
398                    default='expected-results.json')
399parser.add_argument('--expectations-filename-output',
400                    help=('filename (under EXPECTATIONS_ROOT) to write '
401                          'updated expectations into; by default, overwrites '
402                          'the input file (EXPECTATIONS_FILENAME)'),
403                    default='')
404parser.add_argument('--expectations-root',
405                    help=('root of expectations directory to update-- should '
406                          'contain one or more builder subdirectories. '
407                          'Defaults to %(default)s. If SKIMAGE is set, '
408                          ' defaults to ' + gm_json.SKIMAGE_EXPECTATIONS_ROOT),
409                    default=os.path.join('expectations', 'gm'))
410parser.add_argument('--keep-going-on-failure', action='store_true',
411                    help=('instead of halting at the first error encountered, '
412                          'keep going and rebaseline as many tests as '
413                          'possible, and then report the full set of errors '
414                          'at the end'))
415parser.add_argument('--notes',
416                    help=('free-form text notes to add to all updated '
417                          'expectations'))
418# TODO(epoger): Add test that exercises --tests argument.
419parser.add_argument('--tests', metavar='TEST', nargs='+',
420                    help=('which tests to rebaseline, e.g. '
421                          '"--tests aaclip bigmatrix", as a filter over the '
422                          'full set of results in ACTUALS_FILENAME; if '
423                          'unspecified, rebaseline *all* tests that are '
424                          'available.'))
425parser.add_argument('--unreviewed', action='store_true',
426                    help=('mark all expectations modified by this run as '
427                          '"%s": False' %
428                          gm_json.JSONKEY_EXPECTEDRESULTS_REVIEWED))
429parser.add_argument('--ignore-failure', action='store_true',
430                    help=('mark all expectations modified by this run as '
431                          '"%s": True' %
432                          gm_json.JSONKEY_ACTUALRESULTS_FAILUREIGNORED))
433parser.add_argument('--from-trybot', action='store_true',
434                    help=('pull the actual-results.json file from the '
435                          'corresponding trybot, rather than the main builder'))
436parser.add_argument('--skimage', action='store_true',
437                    help=('Rebaseline skimage results instead of gm. Defaults '
438                          'to False. If True, TESTS and CONFIGS are ignored, '
439                          'and ACTUALS_BASE_URL and EXPECTATIONS_ROOT are set '
440                          'to alternate defaults, specific to skimage.'))
441args = parser.parse_args()
442exception_handler = ExceptionHandler(
443    keep_going_on_failure=args.keep_going_on_failure)
444if args.builders:
445  builders = args.builders
446  missing_json_is_fatal = True
447else:
448  builders = sorted(TEST_BUILDERS)
449  missing_json_is_fatal = False
450if args.skimage:
451  # Use a different default if --skimage is specified.
452  if args.actuals_base_url == parser.get_default('actuals_base_url'):
453    args.actuals_base_url = gm_json.SKIMAGE_ACTUALS_BASE_URL
454  if args.expectations_root == parser.get_default('expectations_root'):
455    args.expectations_root = gm_json.SKIMAGE_EXPECTATIONS_ROOT
456for builder in builders:
457  if not builder in TEST_BUILDERS:
458    raise Exception(('unrecognized builder "%s"; ' +
459                     'should be one of %s') % (
460                         builder, TEST_BUILDERS))
461
462  expectations_json_file = os.path.join(args.expectations_root, builder,
463                                        args.expectations_filename)
464  if os.path.isfile(expectations_json_file):
465    rebaseliner = JsonRebaseliner(
466        expectations_root=args.expectations_root,
467        expectations_input_filename=args.expectations_filename,
468        expectations_output_filename=(args.expectations_filename_output or
469                                      args.expectations_filename),
470        tests=args.tests, configs=args.configs,
471        actuals_base_url=args.actuals_base_url,
472        actuals_filename=args.actuals_filename,
473        exception_handler=exception_handler,
474        add_new=args.add_new, add_ignored=args.add_ignored,
475        bugs=args.bugs, notes=args.notes,
476        mark_unreviewed=args.unreviewed,
477        mark_ignore_failure=args.ignore_failure,
478        from_trybot=args.from_trybot)
479    try:
480      rebaseliner.RebaselineSubdir(builder=builder)
481    except:
482      exception_handler.RaiseExceptionOrContinue()
483  else:
484    try:
485      raise _InternalException('expectations_json_file %s not found' %
486                               expectations_json_file)
487    except:
488      exception_handler.RaiseExceptionOrContinue()
489
490exception_handler.ReportAllFailures()
491