1#!/usr/bin/python 2 3''' 4Copyright 2012 Google Inc. 5 6Use of this source code is governed by a BSD-style license that can be 7found in the LICENSE file. 8''' 9 10''' 11Rebaselines the given GM tests, on all bots and all configurations. 12''' 13 14# System-level imports 15import argparse 16import json 17import os 18import re 19import subprocess 20import sys 21import urllib2 22 23# Imports from within Skia 24# 25# We need to add the 'gm' directory, so that we can import gm_json.py within 26# that directory. That script allows us to parse the actual-results.json file 27# written out by the GM tool. 28# Make sure that the 'gm' dir is in the PYTHONPATH, but add it at the *end* 29# so any dirs that are already in the PYTHONPATH will be preferred. 30# 31# This assumes that the 'gm' directory has been checked out as a sibling of 32# the 'tools' directory containing this script, which will be the case if 33# 'trunk' was checked out as a single unit. 34GM_DIRECTORY = os.path.realpath( 35 os.path.join(os.path.dirname(os.path.dirname(__file__)), 'gm')) 36if GM_DIRECTORY not in sys.path: 37 sys.path.append(GM_DIRECTORY) 38import gm_json 39 40# TODO(epoger): In the long run, we want to build this list automatically, 41# but for now we hard-code it until we can properly address 42# https://code.google.com/p/skia/issues/detail?id=1544 43# ('live query of builder list makes rebaseline.py slow to start up') 44TEST_BUILDERS = [ 45 'Test-Android-GalaxyNexus-SGX540-Arm7-Debug', 46 'Test-Android-GalaxyNexus-SGX540-Arm7-Release', 47 'Test-Android-IntelRhb-SGX544-x86-Debug', 48 'Test-Android-IntelRhb-SGX544-x86-Release', 49 'Test-Android-Nexus10-MaliT604-Arm7-Debug', 50 'Test-Android-Nexus10-MaliT604-Arm7-Release', 51 'Test-Android-Nexus4-Adreno320-Arm7-Debug', 52 'Test-Android-Nexus4-Adreno320-Arm7-Release', 53 'Test-Android-Nexus7-Tegra3-Arm7-Debug', 54 'Test-Android-Nexus7-Tegra3-Arm7-Release', 55 'Test-Android-NexusS-SGX540-Arm7-Debug', 56 'Test-Android-NexusS-SGX540-Arm7-Release', 57 'Test-Android-Xoom-Tegra2-Arm7-Debug', 58 'Test-Android-Xoom-Tegra2-Arm7-Release', 59 'Test-ChromeOS-Alex-GMA3150-x86-Debug', 60 'Test-ChromeOS-Alex-GMA3150-x86-Release', 61 'Test-ChromeOS-Daisy-MaliT604-Arm7-Debug', 62 'Test-ChromeOS-Daisy-MaliT604-Arm7-Release', 63 'Test-ChromeOS-Link-HD4000-x86_64-Debug', 64 'Test-ChromeOS-Link-HD4000-x86_64-Release', 65 'Test-Mac10.6-MacMini4.1-GeForce320M-x86-Debug', 66 'Test-Mac10.6-MacMini4.1-GeForce320M-x86-Release', 67 'Test-Mac10.6-MacMini4.1-GeForce320M-x86_64-Debug', 68 'Test-Mac10.6-MacMini4.1-GeForce320M-x86_64-Release', 69 'Test-Mac10.7-MacMini4.1-GeForce320M-x86-Debug', 70 'Test-Mac10.7-MacMini4.1-GeForce320M-x86-Release', 71 'Test-Mac10.7-MacMini4.1-GeForce320M-x86_64-Debug', 72 'Test-Mac10.7-MacMini4.1-GeForce320M-x86_64-Release', 73 'Test-Mac10.8-MacMini4.1-GeForce320M-x86-Debug', 74 'Test-Mac10.8-MacMini4.1-GeForce320M-x86-Release', 75 'Test-Mac10.8-MacMini4.1-GeForce320M-x86_64-Debug', 76 'Test-Mac10.8-MacMini4.1-GeForce320M-x86_64-Release', 77 'Test-Ubuntu12-ShuttleA-ATI5770-x86-Debug', 78 'Test-Ubuntu12-ShuttleA-ATI5770-x86-Release', 79 'Test-Ubuntu12-ShuttleA-ATI5770-x86_64-Debug', 80 'Test-Ubuntu12-ShuttleA-ATI5770-x86_64-Release', 81 'Test-Ubuntu12-ShuttleA-HD2000-x86_64-Release-Valgrind', 82 'Test-Ubuntu12-ShuttleA-NoGPU-x86_64-Debug', 83 'Test-Ubuntu13-ShuttleA-HD2000-x86_64-Debug-ASAN', 84 'Test-Win7-ShuttleA-HD2000-x86-Debug', 85 'Test-Win7-ShuttleA-HD2000-x86-Debug-ANGLE', 86 'Test-Win7-ShuttleA-HD2000-x86-Debug-DirectWrite', 87 'Test-Win7-ShuttleA-HD2000-x86-Release', 88 'Test-Win7-ShuttleA-HD2000-x86-Release-ANGLE', 89 'Test-Win7-ShuttleA-HD2000-x86-Release-DirectWrite', 90 'Test-Win7-ShuttleA-HD2000-x86_64-Debug', 91 'Test-Win7-ShuttleA-HD2000-x86_64-Release', 92] 93 94# TODO: Get this from builder_name_schema in buildbot. 95TRYBOT_SUFFIX = '-Trybot' 96 97 98class _InternalException(Exception): 99 pass 100 101class ExceptionHandler(object): 102 """ Object that handles exceptions, either raising them immediately or 103 collecting them to display later on.""" 104 105 # params: 106 def __init__(self, keep_going_on_failure=False): 107 """ 108 params: 109 keep_going_on_failure: if False, report failures and quit right away; 110 if True, collect failures until 111 ReportAllFailures() is called 112 """ 113 self._keep_going_on_failure = keep_going_on_failure 114 self._failures_encountered = [] 115 116 def RaiseExceptionOrContinue(self): 117 """ We have encountered an exception; either collect the info and keep 118 going, or exit the program right away.""" 119 # Get traceback information about the most recently raised exception. 120 exc_info = sys.exc_info() 121 122 if self._keep_going_on_failure: 123 print >> sys.stderr, ('WARNING: swallowing exception %s' % 124 repr(exc_info[1])) 125 self._failures_encountered.append(exc_info) 126 else: 127 print >> sys.stderr, ( 128 '\nHalting at first exception.\n' + 129 'Please file a bug to epoger@google.com at ' + 130 'https://code.google.com/p/skia/issues/entry, containing the ' + 131 'command you ran and the following stack trace.\n\n' + 132 'Afterwards, you can re-run with the --keep-going-on-failure ' + 133 'option set.\n') 134 raise exc_info[1], None, exc_info[2] 135 136 def ReportAllFailures(self): 137 if self._failures_encountered: 138 print >> sys.stderr, ('Encountered %d failures (see above).' % 139 len(self._failures_encountered)) 140 sys.exit(1) 141 142 143# Object that rebaselines a JSON expectations file (not individual image files). 144class JsonRebaseliner(object): 145 146 # params: 147 # expectations_root: root directory of all expectations JSON files 148 # expectations_input_filename: filename (under expectations_root) of JSON 149 # expectations file to read; typically 150 # "expected-results.json" 151 # expectations_output_filename: filename (under expectations_root) to 152 # which updated expectations should be 153 # written; typically the same as 154 # expectations_input_filename, to overwrite 155 # the old content 156 # actuals_base_url: base URL from which to read actual-result JSON files 157 # actuals_filename: filename (under actuals_base_url) from which to read a 158 # summary of results; typically "actual-results.json" 159 # exception_handler: reference to rebaseline.ExceptionHandler object 160 # tests: list of tests to rebaseline, or None if we should rebaseline 161 # whatever files the JSON results summary file tells us to 162 # configs: which configs to run for each test, or None if we should 163 # rebaseline whatever configs the JSON results summary file tells 164 # us to 165 # add_new: if True, add expectations for tests which don't have any yet 166 # add_ignored: if True, add expectations for tests for which failures are 167 # currently ignored 168 # bugs: optional list of bug numbers which pertain to these expectations 169 # notes: free-form text notes to add to all updated expectations 170 # mark_unreviewed: if True, mark these expectations as NOT having been 171 # reviewed by a human; otherwise, leave that field blank. 172 # Currently, there is no way to make this script mark 173 # expectations as reviewed-by-human=True. 174 # TODO(epoger): Add that capability to a review tool. 175 # mark_ignore_failure: if True, mark failures of a given test as being 176 # ignored. 177 # from_trybot: if True, read actual-result JSON files generated from a 178 # trybot run rather than a waterfall run. 179 def __init__(self, expectations_root, expectations_input_filename, 180 expectations_output_filename, actuals_base_url, 181 actuals_filename, exception_handler, 182 tests=None, configs=None, add_new=False, add_ignored=False, 183 bugs=None, notes=None, mark_unreviewed=None, 184 mark_ignore_failure=False, from_trybot=False): 185 self._expectations_root = expectations_root 186 self._expectations_input_filename = expectations_input_filename 187 self._expectations_output_filename = expectations_output_filename 188 self._tests = tests 189 self._configs = configs 190 self._actuals_base_url = actuals_base_url 191 self._actuals_filename = actuals_filename 192 self._exception_handler = exception_handler 193 self._add_new = add_new 194 self._add_ignored = add_ignored 195 self._bugs = bugs 196 self._notes = notes 197 self._mark_unreviewed = mark_unreviewed 198 self._mark_ignore_failure = mark_ignore_failure; 199 if self._tests or self._configs: 200 self._image_filename_re = re.compile(gm_json.IMAGE_FILENAME_PATTERN) 201 else: 202 self._image_filename_re = None 203 self._using_svn = os.path.isdir(os.path.join(expectations_root, '.svn')) 204 self._from_trybot = from_trybot 205 206 # Executes subprocess.call(cmd). 207 # Raises an Exception if the command fails. 208 def _Call(self, cmd): 209 if subprocess.call(cmd) != 0: 210 raise _InternalException('error running command: ' + ' '.join(cmd)) 211 212 # Returns the full contents of filepath, as a single string. 213 # If filepath looks like a URL, try to read it that way instead of as 214 # a path on local storage. 215 # 216 # Raises _InternalException if there is a problem. 217 def _GetFileContents(self, filepath): 218 if filepath.startswith('http:') or filepath.startswith('https:'): 219 try: 220 return urllib2.urlopen(filepath).read() 221 except urllib2.HTTPError as e: 222 raise _InternalException('unable to read URL %s: %s' % ( 223 filepath, e)) 224 else: 225 return open(filepath, 'r').read() 226 227 # Returns a dictionary of actual results from actual-results.json file. 228 # 229 # The dictionary returned has this format: 230 # { 231 # u'imageblur_565.png': [u'bitmap-64bitMD5', 3359963596899141322], 232 # u'imageblur_8888.png': [u'bitmap-64bitMD5', 4217923806027861152], 233 # u'shadertext3_8888.png': [u'bitmap-64bitMD5', 3713708307125704716] 234 # } 235 # 236 # If the JSON actual result summary file cannot be loaded, logs a warning 237 # message and returns None. 238 # If the JSON actual result summary file can be loaded, but we have 239 # trouble parsing it, raises an Exception. 240 # 241 # params: 242 # json_url: URL pointing to a JSON actual result summary file 243 # sections: a list of section names to include in the results, e.g. 244 # [gm_json.JSONKEY_ACTUALRESULTS_FAILED, 245 # gm_json.JSONKEY_ACTUALRESULTS_NOCOMPARISON] ; 246 # if None, then include ALL sections. 247 def _GetActualResults(self, json_url, sections=None): 248 try: 249 json_contents = self._GetFileContents(json_url) 250 except _InternalException: 251 print >> sys.stderr, ( 252 'could not read json_url %s ; skipping this platform.' % 253 json_url) 254 return None 255 json_dict = gm_json.LoadFromString(json_contents) 256 results_to_return = {} 257 actual_results = json_dict[gm_json.JSONKEY_ACTUALRESULTS] 258 if not sections: 259 sections = actual_results.keys() 260 for section in sections: 261 section_results = actual_results[section] 262 if section_results: 263 results_to_return.update(section_results) 264 return results_to_return 265 266 # Rebaseline all tests/types we specified in the constructor, 267 # within this builder's subdirectory in expectations/gm . 268 # 269 # params: 270 # builder : e.g. 'Test-Win7-ShuttleA-HD2000-x86-Release' 271 def RebaselineSubdir(self, builder): 272 # Read in the actual result summary, and extract all the tests whose 273 # results we need to update. 274 results_builder = str(builder) 275 if self._from_trybot: 276 results_builder = results_builder + TRYBOT_SUFFIX 277 actuals_url = '/'.join([self._actuals_base_url, results_builder, 278 self._actuals_filename]) 279 # Only update results for tests that are currently failing. 280 # We don't want to rewrite results for tests that are already succeeding, 281 # because we don't want to add annotation fields (such as 282 # JSONKEY_EXPECTEDRESULTS_BUGS) except for tests whose expectations we 283 # are actually modifying. 284 sections = [gm_json.JSONKEY_ACTUALRESULTS_FAILED] 285 if self._add_new: 286 sections.append(gm_json.JSONKEY_ACTUALRESULTS_NOCOMPARISON) 287 if self._add_ignored: 288 sections.append(gm_json.JSONKEY_ACTUALRESULTS_FAILUREIGNORED) 289 results_to_update = self._GetActualResults(json_url=actuals_url, 290 sections=sections) 291 292 # Read in current expectations. 293 expectations_input_filepath = os.path.join( 294 self._expectations_root, builder, self._expectations_input_filename) 295 expectations_dict = gm_json.LoadFromFile(expectations_input_filepath) 296 expected_results = expectations_dict.get(gm_json.JSONKEY_EXPECTEDRESULTS) 297 if not expected_results: 298 expected_results = {} 299 expectations_dict[gm_json.JSONKEY_EXPECTEDRESULTS] = expected_results 300 301 # Update the expectations in memory, skipping any tests/configs that 302 # the caller asked to exclude. 303 skipped_images = [] 304 if results_to_update: 305 for (image_name, image_results) in results_to_update.iteritems(): 306 if self._image_filename_re: 307 (test, config) = self._image_filename_re.match(image_name).groups() 308 if self._tests: 309 if test not in self._tests: 310 skipped_images.append(image_name) 311 continue 312 if self._configs: 313 if config not in self._configs: 314 skipped_images.append(image_name) 315 continue 316 if not expected_results.get(image_name): 317 expected_results[image_name] = {} 318 expected_results[image_name]\ 319 [gm_json.JSONKEY_EXPECTEDRESULTS_ALLOWEDDIGESTS]\ 320 = [image_results] 321 if self._mark_unreviewed: 322 expected_results[image_name]\ 323 [gm_json.JSONKEY_EXPECTEDRESULTS_REVIEWED]\ 324 = False 325 if self._mark_ignore_failure: 326 expected_results[image_name]\ 327 [gm_json.JSONKEY_EXPECTEDRESULTS_IGNOREFAILURE]\ 328 = True 329 if self._bugs: 330 expected_results[image_name]\ 331 [gm_json.JSONKEY_EXPECTEDRESULTS_BUGS]\ 332 = self._bugs 333 if self._notes: 334 expected_results[image_name]\ 335 [gm_json.JSONKEY_EXPECTEDRESULTS_NOTES]\ 336 = self._notes 337 338 # Write out updated expectations. 339 expectations_output_filepath = os.path.join( 340 self._expectations_root, builder, self._expectations_output_filename) 341 gm_json.WriteToFile(expectations_dict, expectations_output_filepath) 342 343 # Mark the JSON file as plaintext, so text-style diffs can be applied. 344 # Fixes https://code.google.com/p/skia/issues/detail?id=1442 345 if self._using_svn: 346 self._Call(['svn', 'propset', '--quiet', 'svn:mime-type', 347 'text/x-json', expectations_output_filepath]) 348 349# main... 350 351parser = argparse.ArgumentParser( 352 formatter_class=argparse.RawDescriptionHelpFormatter, 353 epilog='Here is the full set of builders we know about:' + 354 '\n '.join([''] + sorted(TEST_BUILDERS))) 355parser.add_argument('--actuals-base-url', 356 help=('base URL from which to read files containing JSON ' 357 'summaries of actual GM results; defaults to ' 358 '%(default)s. To get a specific revision (useful for ' 359 'trybots) replace "svn" with "svn-history/r123". ' 360 'If SKIMAGE is True, defaults to ' + 361 gm_json.SKIMAGE_ACTUALS_BASE_URL), 362 default='http://skia-autogen.googlecode.com/svn/gm-actual') 363parser.add_argument('--actuals-filename', 364 help=('filename (within builder-specific subdirectories ' 365 'of ACTUALS_BASE_URL) to read a summary of results ' 366 'from; defaults to %(default)s'), 367 default='actual-results.json') 368parser.add_argument('--add-new', action='store_true', 369 help=('in addition to the standard behavior of ' 370 'updating expectations for failing tests, add ' 371 'expectations for tests which don\'t have ' 372 'expectations yet.')) 373parser.add_argument('--add-ignored', action='store_true', 374 help=('in addition to the standard behavior of ' 375 'updating expectations for failing tests, add ' 376 'expectations for tests for which failures are ' 377 'currently ignored.')) 378parser.add_argument('--bugs', metavar='BUG', type=int, nargs='+', 379 help=('Skia bug numbers (under ' 380 'https://code.google.com/p/skia/issues/list ) which ' 381 'pertain to this set of rebaselines.')) 382parser.add_argument('--builders', metavar='BUILDER', nargs='+', 383 help=('which platforms to rebaseline; ' 384 'if unspecified, rebaseline all known platforms ' 385 '(see below for a list)')) 386# TODO(epoger): Add test that exercises --configs argument. 387parser.add_argument('--configs', metavar='CONFIG', nargs='+', 388 help=('which configurations to rebaseline, e.g. ' 389 '"--configs 565 8888", as a filter over the full set ' 390 'of results in ACTUALS_FILENAME; if unspecified, ' 391 'rebaseline *all* configs that are available.')) 392parser.add_argument('--expectations-filename', 393 help=('filename (under EXPECTATIONS_ROOT) to read ' 394 'current expectations from, and to write new ' 395 'expectations into (unless a separate ' 396 'EXPECTATIONS_FILENAME_OUTPUT has been specified); ' 397 'defaults to %(default)s'), 398 default='expected-results.json') 399parser.add_argument('--expectations-filename-output', 400 help=('filename (under EXPECTATIONS_ROOT) to write ' 401 'updated expectations into; by default, overwrites ' 402 'the input file (EXPECTATIONS_FILENAME)'), 403 default='') 404parser.add_argument('--expectations-root', 405 help=('root of expectations directory to update-- should ' 406 'contain one or more builder subdirectories. ' 407 'Defaults to %(default)s. If SKIMAGE is set, ' 408 ' defaults to ' + gm_json.SKIMAGE_EXPECTATIONS_ROOT), 409 default=os.path.join('expectations', 'gm')) 410parser.add_argument('--keep-going-on-failure', action='store_true', 411 help=('instead of halting at the first error encountered, ' 412 'keep going and rebaseline as many tests as ' 413 'possible, and then report the full set of errors ' 414 'at the end')) 415parser.add_argument('--notes', 416 help=('free-form text notes to add to all updated ' 417 'expectations')) 418# TODO(epoger): Add test that exercises --tests argument. 419parser.add_argument('--tests', metavar='TEST', nargs='+', 420 help=('which tests to rebaseline, e.g. ' 421 '"--tests aaclip bigmatrix", as a filter over the ' 422 'full set of results in ACTUALS_FILENAME; if ' 423 'unspecified, rebaseline *all* tests that are ' 424 'available.')) 425parser.add_argument('--unreviewed', action='store_true', 426 help=('mark all expectations modified by this run as ' 427 '"%s": False' % 428 gm_json.JSONKEY_EXPECTEDRESULTS_REVIEWED)) 429parser.add_argument('--ignore-failure', action='store_true', 430 help=('mark all expectations modified by this run as ' 431 '"%s": True' % 432 gm_json.JSONKEY_ACTUALRESULTS_FAILUREIGNORED)) 433parser.add_argument('--from-trybot', action='store_true', 434 help=('pull the actual-results.json file from the ' 435 'corresponding trybot, rather than the main builder')) 436parser.add_argument('--skimage', action='store_true', 437 help=('Rebaseline skimage results instead of gm. Defaults ' 438 'to False. If True, TESTS and CONFIGS are ignored, ' 439 'and ACTUALS_BASE_URL and EXPECTATIONS_ROOT are set ' 440 'to alternate defaults, specific to skimage.')) 441args = parser.parse_args() 442exception_handler = ExceptionHandler( 443 keep_going_on_failure=args.keep_going_on_failure) 444if args.builders: 445 builders = args.builders 446 missing_json_is_fatal = True 447else: 448 builders = sorted(TEST_BUILDERS) 449 missing_json_is_fatal = False 450if args.skimage: 451 # Use a different default if --skimage is specified. 452 if args.actuals_base_url == parser.get_default('actuals_base_url'): 453 args.actuals_base_url = gm_json.SKIMAGE_ACTUALS_BASE_URL 454 if args.expectations_root == parser.get_default('expectations_root'): 455 args.expectations_root = gm_json.SKIMAGE_EXPECTATIONS_ROOT 456for builder in builders: 457 if not builder in TEST_BUILDERS: 458 raise Exception(('unrecognized builder "%s"; ' + 459 'should be one of %s') % ( 460 builder, TEST_BUILDERS)) 461 462 expectations_json_file = os.path.join(args.expectations_root, builder, 463 args.expectations_filename) 464 if os.path.isfile(expectations_json_file): 465 rebaseliner = JsonRebaseliner( 466 expectations_root=args.expectations_root, 467 expectations_input_filename=args.expectations_filename, 468 expectations_output_filename=(args.expectations_filename_output or 469 args.expectations_filename), 470 tests=args.tests, configs=args.configs, 471 actuals_base_url=args.actuals_base_url, 472 actuals_filename=args.actuals_filename, 473 exception_handler=exception_handler, 474 add_new=args.add_new, add_ignored=args.add_ignored, 475 bugs=args.bugs, notes=args.notes, 476 mark_unreviewed=args.unreviewed, 477 mark_ignore_failure=args.ignore_failure, 478 from_trybot=args.from_trybot) 479 try: 480 rebaseliner.RebaselineSubdir(builder=builder) 481 except: 482 exception_handler.RaiseExceptionOrContinue() 483 else: 484 try: 485 raise _InternalException('expectations_json_file %s not found' % 486 expectations_json_file) 487 except: 488 exception_handler.RaiseExceptionOrContinue() 489 490exception_handler.ReportAllFailures() 491