1# Copyright (c) 2010 Google Inc. All rights reserved. 2# 3# Redistribution and use in source and binary forms, with or without 4# modification, are permitted provided that the following conditions are 5# met: 6# 7# * Redistributions of source code must retain the above copyright 8# notice, this list of conditions and the following disclaimer. 9# * Redistributions in binary form must reproduce the above 10# copyright notice, this list of conditions and the following disclaimer 11# in the documentation and/or other materials provided with the 12# distribution. 13# * Neither the name of Google Inc. nor the names of its 14# contributors may be used to endorse or promote products derived from 15# this software without specific prior written permission. 16# 17# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 18# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 19# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 20# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 21# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 22# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 23# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 24# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 25# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 29"""Starts a local HTTP server which displays layout test failures (given a test 30results directory), provides comparisons of expected and actual results (both 31images and text) and allows one-click rebaselining of tests.""" 32from __future__ import with_statement 33 34import codecs 35import datetime 36import fnmatch 37import mimetypes 38import os 39import os.path 40import shutil 41import threading 42import time 43import urlparse 44import BaseHTTPServer 45 46from optparse import make_option 47from wsgiref.handlers import format_date_time 48 49from webkitpy.common import system 50from webkitpy.layout_tests.layout_package import json_results_generator 51from webkitpy.layout_tests.port import factory 52from webkitpy.layout_tests.port.webkit import WebKitPort 53from webkitpy.tool.multicommandtool import AbstractDeclarativeCommand 54from webkitpy.thirdparty import simplejson 55 56STATE_NEEDS_REBASELINE = 'needs_rebaseline' 57STATE_REBASELINE_FAILED = 'rebaseline_failed' 58STATE_REBASELINE_SUCCEEDED = 'rebaseline_succeeded' 59 60class RebaselineHTTPServer(BaseHTTPServer.HTTPServer): 61 def __init__(self, httpd_port, test_config, results_json, platforms_json): 62 BaseHTTPServer.HTTPServer.__init__(self, ("", httpd_port), RebaselineHTTPRequestHandler) 63 self.test_config = test_config 64 self.results_json = results_json 65 self.platforms_json = platforms_json 66 67 68class RebaselineHTTPRequestHandler(BaseHTTPServer.BaseHTTPRequestHandler): 69 STATIC_FILE_NAMES = frozenset([ 70 "index.html", 71 "loupe.js", 72 "main.js", 73 "main.css", 74 "queue.js", 75 "util.js", 76 ]) 77 78 STATIC_FILE_DIRECTORY = os.path.join( 79 os.path.dirname(__file__), "data", "rebaselineserver") 80 81 def do_GET(self): 82 self._handle_request() 83 84 def do_POST(self): 85 self._handle_request() 86 87 def _handle_request(self): 88 # Parse input. 89 if "?" in self.path: 90 path, query_string = self.path.split("?", 1) 91 self.query = urlparse.parse_qs(query_string) 92 else: 93 path = self.path 94 self.query = {} 95 function_or_file_name = path[1:] or "index.html" 96 97 # See if a static file matches. 98 if function_or_file_name in RebaselineHTTPRequestHandler.STATIC_FILE_NAMES: 99 self._serve_static_file(function_or_file_name) 100 return 101 102 # See if a class method matches. 103 function_name = function_or_file_name.replace(".", "_") 104 if not hasattr(self, function_name): 105 self.send_error(404, "Unknown function %s" % function_name) 106 return 107 if function_name[0] == "_": 108 self.send_error( 109 401, "Not allowed to invoke private or protected methods") 110 return 111 function = getattr(self, function_name) 112 function() 113 114 def _serve_static_file(self, static_path): 115 self._serve_file(os.path.join( 116 RebaselineHTTPRequestHandler.STATIC_FILE_DIRECTORY, static_path)) 117 118 def rebaseline(self): 119 test = self.query['test'][0] 120 baseline_target = self.query['baseline-target'][0] 121 baseline_move_to = self.query['baseline-move-to'][0] 122 test_json = self.server.results_json['tests'][test] 123 124 if test_json['state'] != STATE_NEEDS_REBASELINE: 125 self.send_error(400, "Test %s is in unexpected state: %s" % 126 (test, test_json["state"])) 127 return 128 129 log = [] 130 success = _rebaseline_test( 131 test, 132 baseline_target, 133 baseline_move_to, 134 self.server.test_config, 135 log=lambda l: log.append(l)) 136 137 if success: 138 test_json['state'] = STATE_REBASELINE_SUCCEEDED 139 self.send_response(200) 140 else: 141 test_json['state'] = STATE_REBASELINE_FAILED 142 self.send_response(500) 143 144 self.send_header('Content-type', 'text/plain') 145 self.end_headers() 146 self.wfile.write('\n'.join(log)) 147 148 def quitquitquit(self): 149 self.send_response(200) 150 self.send_header("Content-type", "text/plain") 151 self.end_headers() 152 self.wfile.write("Quit.\n") 153 154 # Shutdown has to happen on another thread from the server's thread, 155 # otherwise there's a deadlock 156 threading.Thread(target=lambda: self.server.shutdown()).start() 157 158 def test_result(self): 159 test_name, _ = os.path.splitext(self.query['test'][0]) 160 mode = self.query['mode'][0] 161 if mode == 'expected-image': 162 file_name = test_name + '-expected.png' 163 elif mode == 'actual-image': 164 file_name = test_name + '-actual.png' 165 if mode == 'expected-checksum': 166 file_name = test_name + '-expected.checksum' 167 elif mode == 'actual-checksum': 168 file_name = test_name + '-actual.checksum' 169 elif mode == 'diff-image': 170 file_name = test_name + '-diff.png' 171 if mode == 'expected-text': 172 file_name = test_name + '-expected.txt' 173 elif mode == 'actual-text': 174 file_name = test_name + '-actual.txt' 175 elif mode == 'diff-text': 176 file_name = test_name + '-diff.txt' 177 elif mode == 'diff-text-pretty': 178 file_name = test_name + '-pretty-diff.html' 179 180 file_path = os.path.join(self.server.test_config.results_directory, file_name) 181 182 # Let results be cached for 60 seconds, so that they can be pre-fetched 183 # by the UI 184 self._serve_file(file_path, cacheable_seconds=60) 185 186 def results_json(self): 187 self._serve_json(self.server.results_json) 188 189 def platforms_json(self): 190 self._serve_json(self.server.platforms_json) 191 192 def _serve_json(self, json): 193 self.send_response(200) 194 self.send_header('Content-type', 'application/json') 195 self.end_headers() 196 simplejson.dump(json, self.wfile) 197 198 def _serve_file(self, file_path, cacheable_seconds=0): 199 if not os.path.exists(file_path): 200 self.send_error(404, "File not found") 201 return 202 with codecs.open(file_path, "rb") as static_file: 203 self.send_response(200) 204 self.send_header("Content-Length", os.path.getsize(file_path)) 205 mime_type, encoding = mimetypes.guess_type(file_path) 206 if mime_type: 207 self.send_header("Content-type", mime_type) 208 209 if cacheable_seconds: 210 expires_time = (datetime.datetime.now() + 211 datetime.timedelta(0, cacheable_seconds)) 212 expires_formatted = format_date_time( 213 time.mktime(expires_time.timetuple())) 214 self.send_header("Expires", expires_formatted) 215 self.end_headers() 216 217 shutil.copyfileobj(static_file, self.wfile) 218 219 220class TestConfig(object): 221 def __init__(self, test_port, layout_tests_directory, results_directory, platforms, filesystem, scm): 222 self.test_port = test_port 223 self.layout_tests_directory = layout_tests_directory 224 self.results_directory = results_directory 225 self.platforms = platforms 226 self.filesystem = filesystem 227 self.scm = scm 228 229 230def _get_actual_result_files(test_file, test_config): 231 test_name, _ = os.path.splitext(test_file) 232 test_directory = os.path.dirname(test_file) 233 234 test_results_directory = test_config.filesystem.join( 235 test_config.results_directory, test_directory) 236 actual_pattern = os.path.basename(test_name) + '-actual.*' 237 actual_files = [] 238 for filename in test_config.filesystem.listdir(test_results_directory): 239 if fnmatch.fnmatch(filename, actual_pattern): 240 actual_files.append(filename) 241 actual_files.sort() 242 return tuple(actual_files) 243 244 245def _rebaseline_test(test_file, baseline_target, baseline_move_to, test_config, log): 246 test_name, _ = os.path.splitext(test_file) 247 test_directory = os.path.dirname(test_name) 248 249 log('Rebaselining %s...' % test_name) 250 251 actual_result_files = _get_actual_result_files(test_file, test_config) 252 filesystem = test_config.filesystem 253 scm = test_config.scm 254 layout_tests_directory = test_config.layout_tests_directory 255 results_directory = test_config.results_directory 256 target_expectations_directory = filesystem.join( 257 layout_tests_directory, 'platform', baseline_target, test_directory) 258 test_results_directory = test_config.filesystem.join( 259 test_config.results_directory, test_directory) 260 261 # If requested, move current baselines out 262 current_baselines = _get_test_baselines(test_file, test_config) 263 if baseline_target in current_baselines and baseline_move_to != 'none': 264 log(' Moving current %s baselines to %s' % 265 (baseline_target, baseline_move_to)) 266 267 # See which ones we need to move (only those that are about to be 268 # updated), and make sure we're not clobbering any files in the 269 # destination. 270 current_extensions = set(current_baselines[baseline_target].keys()) 271 actual_result_extensions = [ 272 os.path.splitext(f)[1] for f in actual_result_files] 273 extensions_to_move = current_extensions.intersection( 274 actual_result_extensions) 275 276 if extensions_to_move.intersection( 277 current_baselines.get(baseline_move_to, {}).keys()): 278 log(' Already had baselines in %s, could not move existing ' 279 '%s ones' % (baseline_move_to, baseline_target)) 280 return False 281 282 # Do the actual move. 283 if extensions_to_move: 284 if not _move_test_baselines( 285 test_file, 286 list(extensions_to_move), 287 baseline_target, 288 baseline_move_to, 289 test_config, 290 log): 291 return False 292 else: 293 log(' No current baselines to move') 294 295 log(' Updating baselines for %s' % baseline_target) 296 filesystem.maybe_make_directory(target_expectations_directory) 297 for source_file in actual_result_files: 298 source_path = filesystem.join(test_results_directory, source_file) 299 destination_file = source_file.replace('-actual', '-expected') 300 destination_path = filesystem.join( 301 target_expectations_directory, destination_file) 302 filesystem.copyfile(source_path, destination_path) 303 exit_code = scm.add(destination_path, return_exit_code=True) 304 if exit_code: 305 log(' Could not update %s in SCM, exit code %d' % 306 (destination_file, exit_code)) 307 return False 308 else: 309 log(' Updated %s' % destination_file) 310 311 return True 312 313 314def _move_test_baselines(test_file, extensions_to_move, source_platform, destination_platform, test_config, log): 315 test_file_name = os.path.splitext(os.path.basename(test_file))[0] 316 test_directory = os.path.dirname(test_file) 317 filesystem = test_config.filesystem 318 319 # Want predictable output order for unit tests. 320 extensions_to_move.sort() 321 322 source_directory = os.path.join( 323 test_config.layout_tests_directory, 324 'platform', 325 source_platform, 326 test_directory) 327 destination_directory = os.path.join( 328 test_config.layout_tests_directory, 329 'platform', 330 destination_platform, 331 test_directory) 332 filesystem.maybe_make_directory(destination_directory) 333 334 for extension in extensions_to_move: 335 file_name = test_file_name + '-expected' + extension 336 source_path = filesystem.join(source_directory, file_name) 337 destination_path = filesystem.join(destination_directory, file_name) 338 filesystem.copyfile(source_path, destination_path) 339 exit_code = test_config.scm.add(destination_path, return_exit_code=True) 340 if exit_code: 341 log(' Could not update %s in SCM, exit code %d' % 342 (file_name, exit_code)) 343 return False 344 else: 345 log(' Moved %s' % file_name) 346 347 return True 348 349def _get_test_baselines(test_file, test_config): 350 class AllPlatformsPort(WebKitPort): 351 def __init__(self): 352 WebKitPort.__init__(self, filesystem=test_config.filesystem) 353 self._platforms_by_directory = dict( 354 [(self._webkit_baseline_path(p), p) for p in test_config.platforms]) 355 356 def baseline_search_path(self): 357 return self._platforms_by_directory.keys() 358 359 def platform_from_directory(self, directory): 360 return self._platforms_by_directory[directory] 361 362 test_path = test_config.filesystem.join( 363 test_config.layout_tests_directory, test_file) 364 365 all_platforms_port = AllPlatformsPort() 366 367 all_test_baselines = {} 368 for baseline_extension in ('.txt', '.checksum', '.png'): 369 test_baselines = test_config.test_port.expected_baselines( 370 test_path, baseline_extension) 371 baselines = all_platforms_port.expected_baselines( 372 test_path, baseline_extension, all_baselines=True) 373 for platform_directory, expected_filename in baselines: 374 if not platform_directory: 375 continue 376 if platform_directory == test_config.layout_tests_directory: 377 platform = 'base' 378 else: 379 platform = all_platforms_port.platform_from_directory( 380 platform_directory) 381 platform_baselines = all_test_baselines.setdefault(platform, {}) 382 was_used_for_test = ( 383 platform_directory, expected_filename) in test_baselines 384 platform_baselines[baseline_extension] = was_used_for_test 385 386 return all_test_baselines 387 388 389class RebaselineServer(AbstractDeclarativeCommand): 390 name = "rebaseline-server" 391 help_text = __doc__ 392 argument_names = "/path/to/results/directory" 393 394 def __init__(self): 395 options = [ 396 make_option("--httpd-port", action="store", type="int", default=8127, help="Port to use for the the rebaseline HTTP server"), 397 ] 398 AbstractDeclarativeCommand.__init__(self, options=options) 399 400 def execute(self, options, args, tool): 401 results_directory = args[0] 402 filesystem = system.filesystem.FileSystem() 403 scm = self._tool.scm() 404 405 if options.dry_run: 406 407 def no_op_copyfile(src, dest): 408 pass 409 410 def no_op_add(path, return_exit_code=False): 411 if return_exit_code: 412 return 0 413 414 filesystem.copyfile = no_op_copyfile 415 scm.add = no_op_add 416 417 print 'Parsing unexpected_results.json...' 418 results_json_path = filesystem.join(results_directory, 'unexpected_results.json') 419 results_json = json_results_generator.load_json(filesystem, results_json_path) 420 421 port = factory.get() 422 layout_tests_directory = port.layout_tests_dir() 423 platforms = filesystem.listdir( 424 filesystem.join(layout_tests_directory, 'platform')) 425 test_config = TestConfig( 426 port, 427 layout_tests_directory, 428 results_directory, 429 platforms, 430 filesystem, 431 scm) 432 433 print 'Gathering current baselines...' 434 for test_file, test_json in results_json['tests'].items(): 435 test_json['state'] = STATE_NEEDS_REBASELINE 436 test_path = filesystem.join(layout_tests_directory, test_file) 437 test_json['baselines'] = _get_test_baselines(test_file, test_config) 438 439 server_url = "http://localhost:%d/" % options.httpd_port 440 print "Starting server at %s" % server_url 441 print ("Use the 'Exit' link in the UI, %squitquitquit " 442 "or Ctrl-C to stop") % server_url 443 444 threading.Timer( 445 .1, lambda: self._tool.user.open_url(server_url)).start() 446 447 httpd = RebaselineHTTPServer( 448 httpd_port=options.httpd_port, 449 test_config=test_config, 450 results_json=results_json, 451 platforms_json={ 452 'platforms': platforms, 453 'defaultPlatform': port.name(), 454 }) 455 httpd.serve_forever() 456