1#!/usr/bin/env python 2# Copyright (C) 2010 Google Inc. All rights reserved. 3# 4# Redistribution and use in source and binary forms, with or without 5# modification, are permitted provided that the following conditions are 6# met: 7# 8# * Redistributions of source code must retain the above copyright 9# notice, this list of conditions and the following disclaimer. 10# * Redistributions in binary form must reproduce the above 11# copyright notice, this list of conditions and the following disclaimer 12# in the documentation and/or other materials provided with the 13# distribution. 14# * Neither the name of Google Inc. nor the names of its 15# contributors may be used to endorse or promote products derived from 16# this software without specific prior written permission. 17# 18# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 19# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 20# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 21# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 22# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 23# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 24# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 25# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 26# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 27# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 28# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 30"""Defines the interface TestTypeBase which other test types inherit from. 31 32Also defines the TestArguments "struct" to pass them additional arguments. 33""" 34 35import cgi 36import errno 37import logging 38import os.path 39 40 41class TestArguments(object): 42 """Struct-like wrapper for additional arguments needed by 43 specific tests.""" 44 # Whether to save new baseline results. 45 new_baseline = False 46 47 # Path to the actual PNG file generated by pixel tests 48 png_path = None 49 50 # Value of checksum generated by pixel tests. 51 hash = None 52 53 # Whether to use wdiff to generate by-word diffs. 54 wdiff = False 55 56 # Whether to report the locations of the expected result files used. 57 show_sources = False 58 59# Python bug workaround. See the wdiff code in WriteOutputFiles for an 60# explanation. 61_wdiff_available = True 62 63 64class TestTypeBase(object): 65 66 # Filename pieces when writing failures to the test results directory. 67 FILENAME_SUFFIX_ACTUAL = "-actual" 68 FILENAME_SUFFIX_EXPECTED = "-expected" 69 FILENAME_SUFFIX_DIFF = "-diff" 70 FILENAME_SUFFIX_WDIFF = "-wdiff.html" 71 FILENAME_SUFFIX_COMPARE = "-diff.png" 72 73 def __init__(self, port, platform, root_output_dir): 74 """Initialize a TestTypeBase object. 75 76 Args: 77 platform: the platform (e.g., 'chromium-mac-leopard') 78 identifying the platform-specific results to be used. 79 root_output_dir: The unix style path to the output dir. 80 """ 81 self._root_output_dir = root_output_dir 82 self._port = port 83 self._platform = platform 84 85 def _make_output_directory(self, filename): 86 """Creates the output directory (if needed) for a given test 87 filename.""" 88 output_filename = os.path.join(self._root_output_dir, 89 self._port.relative_test_filename(filename)) 90 self._port.maybe_make_directory(os.path.split(output_filename)[0]) 91 92 def _save_baseline_data(self, filename, data, modifier): 93 """Saves a new baseline file into the platform directory. 94 95 The file will be named simply "<test>-expected<modifier>", suitable for 96 use as the expected results in a later run. 97 98 Args: 99 filename: path to the test file 100 data: result to be saved as the new baseline 101 modifier: type of the result file, e.g. ".txt" or ".png" 102 """ 103 relative_dir = os.path.dirname( 104 self._port.relative_test_filename(filename)) 105 output_dir = os.path.join( 106 self._port.chromium_baseline_path(self._platform), relative_dir) 107 output_file = os.path.basename(os.path.splitext(filename)[0] + 108 self.FILENAME_SUFFIX_EXPECTED + modifier) 109 110 self._port.maybe_make_directory(output_dir) 111 output_path = os.path.join(output_dir, output_file) 112 logging.debug('writing new baseline to "%s"' % (output_path)) 113 open(output_path, "wb").write(data) 114 115 def output_filename(self, filename, modifier): 116 """Returns a filename inside the output dir that contains modifier. 117 118 For example, if filename is c:/.../fast/dom/foo.html and modifier is 119 "-expected.txt", the return value is 120 c:/cygwin/tmp/layout-test-results/fast/dom/foo-expected.txt 121 122 Args: 123 filename: absolute filename to test file 124 modifier: a string to replace the extension of filename with 125 126 Return: 127 The absolute windows path to the output filename 128 """ 129 output_filename = os.path.join(self._root_output_dir, 130 self._port.relative_test_filename(filename)) 131 return os.path.splitext(output_filename)[0] + modifier 132 133 def compare_output(self, port, filename, output, test_args, target): 134 """Method that compares the output from the test with the 135 expected value. 136 137 This is an abstract method to be implemented by all sub classes. 138 139 Args: 140 filename: absolute filename to test file 141 output: a string containing the output of the test 142 test_args: a TestArguments object holding optional additional 143 arguments 144 target: Debug or Release 145 146 Return: 147 a list of TestFailure objects, empty if the test passes 148 """ 149 raise NotImplemented 150 151 def write_output_files(self, port, filename, test_type, file_type, 152 output, expected, diff=True, wdiff=False): 153 """Writes the test output, the expected output and optionally the diff 154 between the two to files in the results directory. 155 156 The full output filename of the actual, for example, will be 157 <filename><test_type>-actual<file_type> 158 For instance, 159 my_test-simp-actual.txt 160 161 Args: 162 filename: The test filename 163 test_type: A string describing the test type, e.g. "simp" 164 file_type: A string describing the test output file type, e.g. ".txt" 165 output: A string containing the test output 166 expected: A string containing the expected test output 167 diff: if True, write a file containing the diffs too. This should be 168 False for results that are not text 169 wdiff: if True, write an HTML file containing word-by-word diffs 170 """ 171 self._make_output_directory(filename) 172 actual_filename = self.output_filename(filename, 173 test_type + self.FILENAME_SUFFIX_ACTUAL + file_type) 174 expected_filename = self.output_filename(filename, 175 test_type + self.FILENAME_SUFFIX_EXPECTED + file_type) 176 if output: 177 open(actual_filename, "wb").write(output) 178 if expected: 179 open(expected_filename, "wb").write(expected) 180 181 if not output or not expected: 182 return 183 184 if diff: 185 diff = port.diff_text(expected, output, expected_filename, 186 actual_filename) 187 diff_filename = self.output_filename(filename, 188 test_type + self.FILENAME_SUFFIX_DIFF + file_type) 189 open(diff_filename, "wb").write(diff) 190 191 if wdiff: 192 # Shell out to wdiff to get colored inline diffs. 193 wdiff = port.wdiff_text(expected_filename, actual_filename) 194 filename = self.output_filename(filename, test_type + 195 self.FILENAME_SUFFIX_WDIFF) 196 out = open(filename, 'wb').write(wdiff) 197