1#!/usr/bin/env python 2# Copyright (C) 2010 Google Inc. All rights reserved. 3# 4# Redistribution and use in source and binary forms, with or without 5# modification, are permitted provided that the following conditions are 6# met: 7# 8# * Redistributions of source code must retain the above copyright 9# notice, this list of conditions and the following disclaimer. 10# * Redistributions in binary form must reproduce the above 11# copyright notice, this list of conditions and the following disclaimer 12# in the documentation and/or other materials provided with the 13# distribution. 14# * Neither the name of Google Inc. nor the names of its 15# contributors may be used to endorse or promote products derived from 16# this software without specific prior written permission. 17# 18# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 19# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 20# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 21# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 22# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 23# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 24# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 25# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 26# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 27# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 28# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 30"""Rebaselining tool that automatically produces baselines for all platforms. 31 32The script does the following for each platform specified: 33 1. Compile a list of tests that need rebaselining. 34 2. Download test result archive from buildbot for the platform. 35 3. Extract baselines from the archive file for all identified files. 36 4. Add new baselines to SVN repository. 37 5. For each test that has been rebaselined, remove this platform option from 38 the test in test_expectation.txt. If no other platforms remain after 39 removal, delete the rebaselined test from the file. 40 41At the end, the script generates a html that compares old and new baselines. 42""" 43 44import logging 45import optparse 46import os 47import re 48import shutil 49import subprocess 50import sys 51import tempfile 52import time 53import urllib 54import webbrowser 55import zipfile 56 57from layout_package import path_utils 58from layout_package import test_expectations 59from test_types import image_diff 60from test_types import text_diff 61 62# Repository type constants. 63REPO_SVN, REPO_UNKNOWN = range(2) 64 65BASELINE_SUFFIXES = ['.txt', '.png', '.checksum'] 66REBASELINE_PLATFORM_ORDER = ['mac', 'win', 'win-xp', 'win-vista', 'linux'] 67ARCHIVE_DIR_NAME_DICT = {'win': 'webkit-rel', 68 'win-vista': 'webkit-dbg-vista', 69 'win-xp': 'webkit-rel', 70 'mac': 'webkit-rel-mac5', 71 'linux': 'webkit-rel-linux', 72 'win-canary': 'webkit-rel-webkit-org', 73 'win-vista-canary': 'webkit-dbg-vista', 74 'win-xp-canary': 'webkit-rel-webkit-org', 75 'mac-canary': 'webkit-rel-mac-webkit-org', 76 'linux-canary': 'webkit-rel-linux-webkit-org'} 77 78 79# FIXME: Should be rolled into webkitpy.Executive 80def run_shell_with_return_code(command, print_output=False): 81 """Executes a command and returns the output and process return code. 82 83 Args: 84 command: program and arguments. 85 print_output: if true, print the command results to standard output. 86 87 Returns: 88 command output, return code 89 """ 90 91 # Use a shell for subcommands on Windows to get a PATH search. 92 use_shell = sys.platform.startswith('win') 93 p = subprocess.Popen(command, stdout=subprocess.PIPE, 94 stderr=subprocess.STDOUT, shell=use_shell) 95 if print_output: 96 output_array = [] 97 while True: 98 line = p.stdout.readline() 99 if not line: 100 break 101 if print_output: 102 print line.strip('\n') 103 output_array.append(line) 104 output = ''.join(output_array) 105 else: 106 output = p.stdout.read() 107 p.wait() 108 p.stdout.close() 109 110 return output, p.returncode 111 112 113# FIXME: Should be rolled into webkitpy.Executive 114def run_shell(command, print_output=False): 115 """Executes a command and returns the output. 116 117 Args: 118 command: program and arguments. 119 print_output: if true, print the command results to standard output. 120 121 Returns: 122 command output 123 """ 124 125 output, return_code = run_shell_with_return_code(command, print_output) 126 return output 127 128 129def log_dashed_string(text, platform, logging_level=logging.INFO): 130 """Log text message with dashes on both sides.""" 131 132 msg = text 133 if platform: 134 msg += ': ' + platform 135 if len(msg) < 78: 136 dashes = '-' * ((78 - len(msg)) / 2) 137 msg = '%s %s %s' % (dashes, msg, dashes) 138 139 if logging_level == logging.ERROR: 140 logging.error(msg) 141 elif logging_level == logging.WARNING: 142 logging.warn(msg) 143 else: 144 logging.info(msg) 145 146 147def setup_html_directory(html_directory): 148 """Setup the directory to store html results. 149 150 All html related files are stored in the "rebaseline_html" subdirectory. 151 152 Args: 153 html_directory: parent directory that stores the rebaselining results. 154 If None, a temp directory is created. 155 156 Returns: 157 the directory that stores the html related rebaselining results. 158 """ 159 160 if not html_directory: 161 html_directory = tempfile.mkdtemp() 162 elif not os.path.exists(html_directory): 163 os.mkdir(html_directory) 164 165 html_directory = os.path.join(html_directory, 'rebaseline_html') 166 logging.info('Html directory: "%s"', html_directory) 167 168 if os.path.exists(html_directory): 169 shutil.rmtree(html_directory, True) 170 logging.info('Deleted file at html directory: "%s"', html_directory) 171 172 if not os.path.exists(html_directory): 173 os.mkdir(html_directory) 174 return html_directory 175 176 177def get_result_file_fullpath(html_directory, baseline_filename, platform, 178 result_type): 179 """Get full path of the baseline result file. 180 181 Args: 182 html_directory: directory that stores the html related files. 183 baseline_filename: name of the baseline file. 184 platform: win, linux or mac 185 result_type: type of the baseline result: '.txt', '.png'. 186 187 Returns: 188 Full path of the baseline file for rebaselining result comparison. 189 """ 190 191 base, ext = os.path.splitext(baseline_filename) 192 result_filename = '%s-%s-%s%s' % (base, platform, result_type, ext) 193 fullpath = os.path.join(html_directory, result_filename) 194 logging.debug(' Result file full path: "%s".', fullpath) 195 return fullpath 196 197 198class Rebaseliner(object): 199 """Class to produce new baselines for a given platform.""" 200 201 REVISION_REGEX = r'<a href=\"(\d+)/\">' 202 203 def __init__(self, platform, options): 204 self._file_dir = path_utils.path_from_base('webkit', 'tools', 205 'layout_tests') 206 self._platform = platform 207 self._options = options 208 self._rebaselining_tests = [] 209 self._rebaselined_tests = [] 210 211 # Create tests and expectations helper which is used to: 212 # -. compile list of tests that need rebaselining. 213 # -. update the tests in test_expectations file after rebaseline 214 # is done. 215 self._test_expectations = \ 216 test_expectations.TestExpectations(None, 217 self._file_dir, 218 platform, 219 False, 220 False) 221 222 self._repo_type = self._get_repo_type() 223 224 def run(self, backup): 225 """Run rebaseline process.""" 226 227 log_dashed_string('Compiling rebaselining tests', self._platform) 228 if not self._compile_rebaselining_tests(): 229 return True 230 231 log_dashed_string('Downloading archive', self._platform) 232 archive_file = self._download_buildbot_archive() 233 logging.info('') 234 if not archive_file: 235 logging.error('No archive found.') 236 return False 237 238 log_dashed_string('Extracting and adding new baselines', 239 self._platform) 240 if not self._extract_and_add_new_baselines(archive_file): 241 return False 242 243 log_dashed_string('Updating rebaselined tests in file', 244 self._platform) 245 self._update_rebaselined_tests_in_file(backup) 246 logging.info('') 247 248 if len(self._rebaselining_tests) != len(self._rebaselined_tests): 249 logging.warning('NOT ALL TESTS THAT NEED REBASELINING HAVE BEEN ' 250 'REBASELINED.') 251 logging.warning(' Total tests needing rebaselining: %d', 252 len(self._rebaselining_tests)) 253 logging.warning(' Total tests rebaselined: %d', 254 len(self._rebaselined_tests)) 255 return False 256 257 logging.warning('All tests needing rebaselining were successfully ' 258 'rebaselined.') 259 260 return True 261 262 def get_rebaselining_tests(self): 263 return self._rebaselining_tests 264 265 def _get_repo_type(self): 266 """Get the repository type that client is using.""" 267 output, return_code = run_shell_with_return_code(['svn', 'info'], 268 False) 269 if return_code == 0: 270 return REPO_SVN 271 272 return REPO_UNKNOWN 273 274 def _compile_rebaselining_tests(self): 275 """Compile list of tests that need rebaselining for the platform. 276 277 Returns: 278 List of tests that need rebaselining or 279 None if there is no such test. 280 """ 281 282 self._rebaselining_tests = \ 283 self._test_expectations.get_rebaselining_failures() 284 if not self._rebaselining_tests: 285 logging.warn('No tests found that need rebaselining.') 286 return None 287 288 logging.info('Total number of tests needing rebaselining ' 289 'for "%s": "%d"', self._platform, 290 len(self._rebaselining_tests)) 291 292 test_no = 1 293 for test in self._rebaselining_tests: 294 logging.info(' %d: %s', test_no, test) 295 test_no += 1 296 297 return self._rebaselining_tests 298 299 def _get_latest_revision(self, url): 300 """Get the latest layout test revision number from buildbot. 301 302 Args: 303 url: Url to retrieve layout test revision numbers. 304 305 Returns: 306 latest revision or 307 None on failure. 308 """ 309 310 logging.debug('Url to retrieve revision: "%s"', url) 311 312 f = urllib.urlopen(url) 313 content = f.read() 314 f.close() 315 316 revisions = re.findall(self.REVISION_REGEX, content) 317 if not revisions: 318 logging.error('Failed to find revision, content: "%s"', content) 319 return None 320 321 revisions.sort(key=int) 322 logging.info('Latest revision: "%s"', revisions[len(revisions) - 1]) 323 return revisions[len(revisions) - 1] 324 325 def _get_archive_dir_name(self, platform, webkit_canary): 326 """Get name of the layout test archive directory. 327 328 Returns: 329 Directory name or 330 None on failure 331 """ 332 333 if webkit_canary: 334 platform += '-canary' 335 336 if platform in ARCHIVE_DIR_NAME_DICT: 337 return ARCHIVE_DIR_NAME_DICT[platform] 338 else: 339 logging.error('Cannot find platform key %s in archive ' 340 'directory name dictionary', platform) 341 return None 342 343 def _get_archive_url(self): 344 """Generate the url to download latest layout test archive. 345 346 Returns: 347 Url to download archive or 348 None on failure 349 """ 350 351 dir_name = self._get_archive_dir_name(self._platform, 352 self._options.webkit_canary) 353 if not dir_name: 354 return None 355 356 logging.debug('Buildbot platform dir name: "%s"', dir_name) 357 358 url_base = '%s/%s/' % (self._options.archive_url, dir_name) 359 latest_revision = self._get_latest_revision(url_base) 360 if latest_revision is None or latest_revision <= 0: 361 return None 362 363 archive_url = ('%s%s/layout-test-results.zip' % (url_base, 364 latest_revision)) 365 logging.info('Archive url: "%s"', archive_url) 366 return archive_url 367 368 def _download_buildbot_archive(self): 369 """Download layout test archive file from buildbot. 370 371 Returns: 372 True if download succeeded or 373 False otherwise. 374 """ 375 376 url = self._get_archive_url() 377 if url is None: 378 return None 379 380 fn = urllib.urlretrieve(url)[0] 381 logging.info('Archive downloaded and saved to file: "%s"', fn) 382 return fn 383 384 def _extract_and_add_new_baselines(self, archive_file): 385 """Extract new baselines from archive and add them to SVN repository. 386 387 Args: 388 archive_file: full path to the archive file. 389 390 Returns: 391 List of tests that have been rebaselined or 392 None on failure. 393 """ 394 395 zip_file = zipfile.ZipFile(archive_file, 'r') 396 zip_namelist = zip_file.namelist() 397 398 logging.debug('zip file namelist:') 399 for name in zip_namelist: 400 logging.debug(' ' + name) 401 402 platform = path_utils.platform_name(self._platform) 403 logging.debug('Platform dir: "%s"', platform) 404 405 test_no = 1 406 self._rebaselined_tests = [] 407 for test in self._rebaselining_tests: 408 logging.info('Test %d: %s', test_no, test) 409 410 found = False 411 svn_error = False 412 test_basename = os.path.splitext(test)[0] 413 for suffix in BASELINE_SUFFIXES: 414 archive_test_name = ('layout-test-results/%s-actual%s' % 415 (test_basename, suffix)) 416 logging.debug(' Archive test file name: "%s"', 417 archive_test_name) 418 if not archive_test_name in zip_namelist: 419 logging.info(' %s file not in archive.', suffix) 420 continue 421 422 found = True 423 logging.info(' %s file found in archive.', suffix) 424 425 # Extract new baseline from archive and save it to a temp file. 426 data = zip_file.read(archive_test_name) 427 temp_fd, temp_name = tempfile.mkstemp(suffix) 428 f = os.fdopen(temp_fd, 'wb') 429 f.write(data) 430 f.close() 431 432 expected_filename = '%s-expected%s' % (test_basename, suffix) 433 expected_fullpath = os.path.join( 434 path_utils.chromium_baseline_path(platform), 435 expected_filename) 436 expected_fullpath = os.path.normpath(expected_fullpath) 437 logging.debug(' Expected file full path: "%s"', 438 expected_fullpath) 439 440 # TODO(victorw): for now, the rebaselining tool checks whether 441 # or not THIS baseline is duplicate and should be skipped. 442 # We could improve the tool to check all baselines in upper 443 # and lower 444 # levels and remove all duplicated baselines. 445 if self._is_dup_baseline(temp_name, 446 expected_fullpath, 447 test, 448 suffix, 449 self._platform): 450 os.remove(temp_name) 451 self._delete_baseline(expected_fullpath) 452 continue 453 454 # Create the new baseline directory if it doesn't already 455 # exist. 456 path_utils.maybe_make_directory( 457 os.path.dirname(expected_fullpath)) 458 459 shutil.move(temp_name, expected_fullpath) 460 461 if not self._svn_add(expected_fullpath): 462 svn_error = True 463 elif suffix != '.checksum': 464 self._create_html_baseline_files(expected_fullpath) 465 466 if not found: 467 logging.warn(' No new baselines found in archive.') 468 else: 469 if svn_error: 470 logging.warn(' Failed to add baselines to SVN.') 471 else: 472 logging.info(' Rebaseline succeeded.') 473 self._rebaselined_tests.append(test) 474 475 test_no += 1 476 477 zip_file.close() 478 os.remove(archive_file) 479 480 return self._rebaselined_tests 481 482 def _is_dup_baseline(self, new_baseline, baseline_path, test, suffix, 483 platform): 484 """Check whether a baseline is duplicate and can fallback to same 485 baseline for another platform. For example, if a test has same 486 baseline on linux and windows, then we only store windows 487 baseline and linux baseline will fallback to the windows version. 488 489 Args: 490 expected_filename: baseline expectation file name. 491 test: test name. 492 suffix: file suffix of the expected results, including dot; 493 e.g. '.txt' or '.png'. 494 platform: baseline platform 'mac', 'win' or 'linux'. 495 496 Returns: 497 True if the baseline is unnecessary. 498 False otherwise. 499 """ 500 test_filepath = os.path.join(path_utils.layout_tests_dir(), test) 501 all_baselines = path_utils.expected_baselines(test_filepath, 502 suffix, platform, True) 503 for (fallback_dir, fallback_file) in all_baselines: 504 if fallback_dir and fallback_file: 505 fallback_fullpath = os.path.normpath( 506 os.path.join(fallback_dir, fallback_file)) 507 if fallback_fullpath.lower() != baseline_path.lower(): 508 if not self._diff_baselines(new_baseline, 509 fallback_fullpath): 510 logging.info(' Found same baseline at %s', 511 fallback_fullpath) 512 return True 513 else: 514 return False 515 516 return False 517 518 def _diff_baselines(self, file1, file2): 519 """Check whether two baselines are different. 520 521 Args: 522 file1, file2: full paths of the baselines to compare. 523 524 Returns: 525 True if two files are different or have different extensions. 526 False otherwise. 527 """ 528 529 ext1 = os.path.splitext(file1)[1].upper() 530 ext2 = os.path.splitext(file2)[1].upper() 531 if ext1 != ext2: 532 logging.warn('Files to compare have different ext. ' 533 'File1: %s; File2: %s', file1, file2) 534 return True 535 536 if ext1 == '.PNG': 537 return image_diff.ImageDiff(self._platform, '').diff_files(file1, 538 file2) 539 else: 540 return text_diff.TestTextDiff(self._platform, '').diff_files(file1, 541 file2) 542 543 def _delete_baseline(self, filename): 544 """Remove the file from repository and delete it from disk. 545 546 Args: 547 filename: full path of the file to delete. 548 """ 549 550 if not filename or not os.path.isfile(filename): 551 return 552 553 if self._repo_type == REPO_SVN: 554 parent_dir, basename = os.path.split(filename) 555 original_dir = os.getcwd() 556 os.chdir(parent_dir) 557 run_shell(['svn', 'delete', '--force', basename], False) 558 os.chdir(original_dir) 559 else: 560 os.remove(filename) 561 562 def _update_rebaselined_tests_in_file(self, backup): 563 """Update the rebaselined tests in test expectations file. 564 565 Args: 566 backup: if True, backup the original test expectations file. 567 568 Returns: 569 no 570 """ 571 572 if self._rebaselined_tests: 573 self._test_expectations.remove_platform_from_file( 574 self._rebaselined_tests, self._platform, backup) 575 else: 576 logging.info('No test was rebaselined so nothing to remove.') 577 578 def _svn_add(self, filename): 579 """Add the file to SVN repository. 580 581 Args: 582 filename: full path of the file to add. 583 584 Returns: 585 True if the file already exists in SVN or is sucessfully added 586 to SVN. 587 False otherwise. 588 """ 589 590 if not filename: 591 return False 592 593 parent_dir, basename = os.path.split(filename) 594 if self._repo_type != REPO_SVN or parent_dir == filename: 595 logging.info("No svn checkout found, skip svn add.") 596 return True 597 598 original_dir = os.getcwd() 599 os.chdir(parent_dir) 600 status_output = run_shell(['svn', 'status', basename], False) 601 os.chdir(original_dir) 602 output = status_output.upper() 603 if output.startswith('A') or output.startswith('M'): 604 logging.info(' File already added to SVN: "%s"', filename) 605 return True 606 607 if output.find('IS NOT A WORKING COPY') >= 0: 608 logging.info(' File is not a working copy, add its parent: "%s"', 609 parent_dir) 610 return self._svn_add(parent_dir) 611 612 os.chdir(parent_dir) 613 add_output = run_shell(['svn', 'add', basename], True) 614 os.chdir(original_dir) 615 output = add_output.upper().rstrip() 616 if output.startswith('A') and output.find(basename.upper()) >= 0: 617 logging.info(' Added new file: "%s"', filename) 618 self._svn_prop_set(filename) 619 return True 620 621 if (not status_output) and (add_output.upper().find( 622 'ALREADY UNDER VERSION CONTROL') >= 0): 623 logging.info(' File already under SVN and has no change: "%s"', 624 filename) 625 return True 626 627 logging.warn(' Failed to add file to SVN: "%s"', filename) 628 logging.warn(' Svn status output: "%s"', status_output) 629 logging.warn(' Svn add output: "%s"', add_output) 630 return False 631 632 def _svn_prop_set(self, filename): 633 """Set the baseline property 634 635 Args: 636 filename: full path of the file to add. 637 638 Returns: 639 True if the file already exists in SVN or is sucessfully added 640 to SVN. 641 False otherwise. 642 """ 643 ext = os.path.splitext(filename)[1].upper() 644 if ext != '.TXT' and ext != '.PNG' and ext != '.CHECKSUM': 645 return 646 647 parent_dir, basename = os.path.split(filename) 648 original_dir = os.getcwd() 649 os.chdir(parent_dir) 650 if ext == '.PNG': 651 cmd = ['svn', 'pset', 'svn:mime-type', 'image/png', basename] 652 else: 653 cmd = ['svn', 'pset', 'svn:eol-style', 'LF', basename] 654 655 logging.debug(' Set svn prop: %s', ' '.join(cmd)) 656 run_shell(cmd, False) 657 os.chdir(original_dir) 658 659 def _create_html_baseline_files(self, baseline_fullpath): 660 """Create baseline files (old, new and diff) in html directory. 661 662 The files are used to compare the rebaselining results. 663 664 Args: 665 baseline_fullpath: full path of the expected baseline file. 666 """ 667 668 if not baseline_fullpath or not os.path.exists(baseline_fullpath): 669 return 670 671 # Copy the new baseline to html directory for result comparison. 672 baseline_filename = os.path.basename(baseline_fullpath) 673 new_file = get_result_file_fullpath(self._options.html_directory, 674 baseline_filename, self._platform, 675 'new') 676 shutil.copyfile(baseline_fullpath, new_file) 677 logging.info(' Html: copied new baseline file from "%s" to "%s".', 678 baseline_fullpath, new_file) 679 680 # Get the old baseline from SVN and save to the html directory. 681 output = run_shell(['svn', 'cat', '-r', 'BASE', baseline_fullpath]) 682 if (not output) or (output.upper().rstrip().endswith( 683 'NO SUCH FILE OR DIRECTORY')): 684 logging.info(' No base file: "%s"', baseline_fullpath) 685 return 686 base_file = get_result_file_fullpath(self._options.html_directory, 687 baseline_filename, self._platform, 688 'old') 689 f = open(base_file, 'wb') 690 f.write(output) 691 f.close() 692 logging.info(' Html: created old baseline file: "%s".', 693 base_file) 694 695 # Get the diff between old and new baselines and save to the html dir. 696 if baseline_filename.upper().endswith('.TXT'): 697 # If the user specified a custom diff command in their svn config 698 # file, then it'll be used when we do svn diff, which we don't want 699 # to happen since we want the unified diff. Using --diff-cmd=diff 700 # doesn't always work, since they can have another diff executable 701 # in their path that gives different line endings. So we use a 702 # bogus temp directory as the config directory, which gets 703 # around these problems. 704 if sys.platform.startswith("win"): 705 parent_dir = tempfile.gettempdir() 706 else: 707 parent_dir = sys.path[0] # tempdir is not secure. 708 bogus_dir = os.path.join(parent_dir, "temp_svn_config") 709 logging.debug(' Html: temp config dir: "%s".', bogus_dir) 710 if not os.path.exists(bogus_dir): 711 os.mkdir(bogus_dir) 712 delete_bogus_dir = True 713 else: 714 delete_bogus_dir = False 715 716 output = run_shell(["svn", "diff", "--config-dir", bogus_dir, 717 baseline_fullpath]) 718 if output: 719 diff_file = get_result_file_fullpath( 720 self._options.html_directory, baseline_filename, 721 self._platform, 'diff') 722 f = open(diff_file, 'wb') 723 f.write(output) 724 f.close() 725 logging.info(' Html: created baseline diff file: "%s".', 726 diff_file) 727 728 if delete_bogus_dir: 729 shutil.rmtree(bogus_dir, True) 730 logging.debug(' Html: removed temp config dir: "%s".', 731 bogus_dir) 732 733 734class HtmlGenerator(object): 735 """Class to generate rebaselining result comparison html.""" 736 737 HTML_REBASELINE = ('<html>' 738 '<head>' 739 '<style>' 740 'body {font-family: sans-serif;}' 741 '.mainTable {background: #666666;}' 742 '.mainTable td , .mainTable th {background: white;}' 743 '.detail {margin-left: 10px; margin-top: 3px;}' 744 '</style>' 745 '<title>Rebaselining Result Comparison (%(time)s)' 746 '</title>' 747 '</head>' 748 '<body>' 749 '<h2>Rebaselining Result Comparison (%(time)s)</h2>' 750 '%(body)s' 751 '</body>' 752 '</html>') 753 HTML_NO_REBASELINING_TESTS = ( 754 '<p>No tests found that need rebaselining.</p>') 755 HTML_TABLE_TEST = ('<table class="mainTable" cellspacing=1 cellpadding=5>' 756 '%s</table><br>') 757 HTML_TR_TEST = ('<tr>' 758 '<th style="background-color: #CDECDE; border-bottom: ' 759 '1px solid black; font-size: 18pt; font-weight: bold" ' 760 'colspan="5">' 761 '<a href="%s">%s</a>' 762 '</th>' 763 '</tr>') 764 HTML_TEST_DETAIL = ('<div class="detail">' 765 '<tr>' 766 '<th width="100">Baseline</th>' 767 '<th width="100">Platform</th>' 768 '<th width="200">Old</th>' 769 '<th width="200">New</th>' 770 '<th width="150">Difference</th>' 771 '</tr>' 772 '%s' 773 '</div>') 774 HTML_TD_NOLINK = '<td align=center><a>%s</a></td>' 775 HTML_TD_LINK = '<td align=center><a href="%(uri)s">%(name)s</a></td>' 776 HTML_TD_LINK_IMG = ('<td><a href="%(uri)s">' 777 '<img style="width: 200" src="%(uri)s" /></a></td>') 778 HTML_TR = '<tr>%s</tr>' 779 780 def __init__(self, options, platforms, rebaselining_tests): 781 self._html_directory = options.html_directory 782 self._platforms = platforms 783 self._rebaselining_tests = rebaselining_tests 784 self._html_file = os.path.join(options.html_directory, 785 'rebaseline.html') 786 787 def generate_html(self): 788 """Generate html file for rebaselining result comparison.""" 789 790 logging.info('Generating html file') 791 792 html_body = '' 793 if not self._rebaselining_tests: 794 html_body += self.HTML_NO_REBASELINING_TESTS 795 else: 796 tests = list(self._rebaselining_tests) 797 tests.sort() 798 799 test_no = 1 800 for test in tests: 801 logging.info('Test %d: %s', test_no, test) 802 html_body += self._generate_html_for_one_test(test) 803 804 html = self.HTML_REBASELINE % ({'time': time.asctime(), 805 'body': html_body}) 806 logging.debug(html) 807 808 f = open(self._html_file, 'w') 809 f.write(html) 810 f.close() 811 812 logging.info('Baseline comparison html generated at "%s"', 813 self._html_file) 814 815 def show_html(self): 816 """Launch the rebaselining html in brwoser.""" 817 818 logging.info('Launching html: "%s"', self._html_file) 819 820 html_uri = path_utils.filename_to_uri(self._html_file) 821 webbrowser.open(html_uri, 1) 822 823 logging.info('Html launched.') 824 825 def _generate_baseline_links(self, test_basename, suffix, platform): 826 """Generate links for baseline results (old, new and diff). 827 828 Args: 829 test_basename: base filename of the test 830 suffix: baseline file suffixes: '.txt', '.png' 831 platform: win, linux or mac 832 833 Returns: 834 html links for showing baseline results (old, new and diff) 835 """ 836 837 baseline_filename = '%s-expected%s' % (test_basename, suffix) 838 logging.debug(' baseline filename: "%s"', baseline_filename) 839 840 new_file = get_result_file_fullpath(self._html_directory, 841 baseline_filename, platform, 'new') 842 logging.info(' New baseline file: "%s"', new_file) 843 if not os.path.exists(new_file): 844 logging.info(' No new baseline file: "%s"', new_file) 845 return '' 846 847 old_file = get_result_file_fullpath(self._html_directory, 848 baseline_filename, platform, 'old') 849 logging.info(' Old baseline file: "%s"', old_file) 850 if suffix == '.png': 851 html_td_link = self.HTML_TD_LINK_IMG 852 else: 853 html_td_link = self.HTML_TD_LINK 854 855 links = '' 856 if os.path.exists(old_file): 857 links += html_td_link % { 858 'uri': path_utils.filename_to_uri(old_file), 859 'name': baseline_filename} 860 else: 861 logging.info(' No old baseline file: "%s"', old_file) 862 links += self.HTML_TD_NOLINK % '' 863 864 links += html_td_link % {'uri': path_utils.filename_to_uri(new_file), 865 'name': baseline_filename} 866 867 diff_file = get_result_file_fullpath(self._html_directory, 868 baseline_filename, platform, 869 'diff') 870 logging.info(' Baseline diff file: "%s"', diff_file) 871 if os.path.exists(diff_file): 872 links += html_td_link % {'uri': path_utils.filename_to_uri( 873 diff_file), 'name': 'Diff'} 874 else: 875 logging.info(' No baseline diff file: "%s"', diff_file) 876 links += self.HTML_TD_NOLINK % '' 877 878 return links 879 880 def _generate_html_for_one_test(self, test): 881 """Generate html for one rebaselining test. 882 883 Args: 884 test: layout test name 885 886 Returns: 887 html that compares baseline results for the test. 888 """ 889 890 test_basename = os.path.basename(os.path.splitext(test)[0]) 891 logging.info(' basename: "%s"', test_basename) 892 rows = [] 893 for suffix in BASELINE_SUFFIXES: 894 if suffix == '.checksum': 895 continue 896 897 logging.info(' Checking %s files', suffix) 898 for platform in self._platforms: 899 links = self._generate_baseline_links(test_basename, suffix, 900 platform) 901 if links: 902 row = self.HTML_TD_NOLINK % self._get_baseline_result_type( 903 suffix) 904 row += self.HTML_TD_NOLINK % platform 905 row += links 906 logging.debug(' html row: %s', row) 907 908 rows.append(self.HTML_TR % row) 909 910 if rows: 911 test_path = os.path.join(path_utils.layout_tests_dir(), test) 912 html = self.HTML_TR_TEST % (path_utils.filename_to_uri(test_path), 913 test) 914 html += self.HTML_TEST_DETAIL % ' '.join(rows) 915 916 logging.debug(' html for test: %s', html) 917 return self.HTML_TABLE_TEST % html 918 919 return '' 920 921 def _get_baseline_result_type(self, suffix): 922 """Name of the baseline result type.""" 923 924 if suffix == '.png': 925 return 'Pixel' 926 elif suffix == '.txt': 927 return 'Render Tree' 928 else: 929 return 'Other' 930 931 932def main(): 933 """Main function to produce new baselines.""" 934 935 option_parser = optparse.OptionParser() 936 option_parser.add_option('-v', '--verbose', 937 action='store_true', 938 default=False, 939 help='include debug-level logging.') 940 941 option_parser.add_option('-p', '--platforms', 942 default='mac,win,win-xp,win-vista,linux', 943 help=('Comma delimited list of platforms ' 944 'that need rebaselining.')) 945 946 option_parser.add_option('-u', '--archive_url', 947 default=('http://build.chromium.org/buildbot/' 948 'layout_test_results'), 949 help=('Url to find the layout test result archive' 950 ' file.')) 951 952 option_parser.add_option('-w', '--webkit_canary', 953 action='store_true', 954 default=False, 955 help=('If True, pull baselines from webkit.org ' 956 'canary bot.')) 957 958 option_parser.add_option('-b', '--backup', 959 action='store_true', 960 default=False, 961 help=('Whether or not to backup the original test' 962 ' expectations file after rebaseline.')) 963 964 option_parser.add_option('-d', '--html_directory', 965 default='', 966 help=('The directory that stores the results for' 967 ' rebaselining comparison.')) 968 969 options = option_parser.parse_args()[0] 970 971 # Set up our logging format. 972 log_level = logging.INFO 973 if options.verbose: 974 log_level = logging.DEBUG 975 logging.basicConfig(level=log_level, 976 format=('%(asctime)s %(filename)s:%(lineno)-3d ' 977 '%(levelname)s %(message)s'), 978 datefmt='%y%m%d %H:%M:%S') 979 980 # Verify 'platforms' option is valid 981 if not options.platforms: 982 logging.error('Invalid "platforms" option. --platforms must be ' 983 'specified in order to rebaseline.') 984 sys.exit(1) 985 platforms = [p.strip().lower() for p in options.platforms.split(',')] 986 for platform in platforms: 987 if not platform in REBASELINE_PLATFORM_ORDER: 988 logging.error('Invalid platform: "%s"' % (platform)) 989 sys.exit(1) 990 991 # Adjust the platform order so rebaseline tool is running at the order of 992 # 'mac', 'win' and 'linux'. This is in same order with layout test baseline 993 # search paths. It simplifies how the rebaseline tool detects duplicate 994 # baselines. Check _IsDupBaseline method for details. 995 rebaseline_platforms = [] 996 for platform in REBASELINE_PLATFORM_ORDER: 997 if platform in platforms: 998 rebaseline_platforms.append(platform) 999 1000 options.html_directory = setup_html_directory(options.html_directory) 1001 1002 rebaselining_tests = set() 1003 backup = options.backup 1004 for platform in rebaseline_platforms: 1005 rebaseliner = Rebaseliner(platform, options) 1006 1007 logging.info('') 1008 log_dashed_string('Rebaseline started', platform) 1009 if rebaseliner.run(backup): 1010 # Only need to backup one original copy of test expectation file. 1011 backup = False 1012 log_dashed_string('Rebaseline done', platform) 1013 else: 1014 log_dashed_string('Rebaseline failed', platform, logging.ERROR) 1015 1016 rebaselining_tests |= set(rebaseliner.get_rebaselining_tests()) 1017 1018 logging.info('') 1019 log_dashed_string('Rebaselining result comparison started', None) 1020 html_generator = HtmlGenerator(options, 1021 rebaseline_platforms, 1022 rebaselining_tests) 1023 html_generator.generate_html() 1024 html_generator.show_html() 1025 log_dashed_string('Rebaselining result comparison done', None) 1026 1027 sys.exit(0) 1028 1029if '__main__' == __name__: 1030 main() 1031