• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/env python
2# Copyright (C) 2010 Google Inc. All rights reserved.
3#
4# Redistribution and use in source and binary forms, with or without
5# modification, are permitted provided that the following conditions are
6# met:
7#
8#     * Redistributions of source code must retain the above copyright
9# notice, this list of conditions and the following disclaimer.
10#     * Redistributions in binary form must reproduce the above
11# copyright notice, this list of conditions and the following disclaimer
12# in the documentation and/or other materials provided with the
13# distribution.
14#     * Neither the name of Google Inc. nor the names of its
15# contributors may be used to endorse or promote products derived from
16# this software without specific prior written permission.
17#
18# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29
30"""Classes for failures that occur during tests."""
31
32import os
33import test_expectations
34
35
36def determine_result_type(failure_list):
37    """Takes a set of test_failures and returns which result type best fits
38    the list of failures. "Best fits" means we use the worst type of failure.
39
40    Returns:
41      one of the test_expectations result types - PASS, TEXT, CRASH, etc."""
42
43    if not failure_list or len(failure_list) == 0:
44        return test_expectations.PASS
45
46    failure_types = [type(f) for f in failure_list]
47    if FailureCrash in failure_types:
48        return test_expectations.CRASH
49    elif FailureTimeout in failure_types:
50        return test_expectations.TIMEOUT
51    elif (FailureMissingResult in failure_types or
52          FailureMissingImage in failure_types or
53          FailureMissingImageHash in failure_types):
54        return test_expectations.MISSING
55    else:
56        is_text_failure = FailureTextMismatch in failure_types
57        is_image_failure = (FailureImageHashIncorrect in failure_types or
58                            FailureImageHashMismatch in failure_types)
59        if is_text_failure and is_image_failure:
60            return test_expectations.IMAGE_PLUS_TEXT
61        elif is_text_failure:
62            return test_expectations.TEXT
63        elif is_image_failure:
64            return test_expectations.IMAGE
65        else:
66            raise ValueError("unclassifiable set of failures: "
67                             + str(failure_types))
68
69
70class TestFailure(object):
71    """Abstract base class that defines the failure interface."""
72
73    @staticmethod
74    def message():
75        """Returns a string describing the failure in more detail."""
76        raise NotImplemented
77
78    def result_html_output(self, filename):
79        """Returns an HTML string to be included on the results.html page."""
80        raise NotImplemented
81
82    def should_kill_test_shell(self):
83        """Returns True if we should kill the test shell before the next
84        test."""
85        return False
86
87    def relative_output_filename(self, filename, modifier):
88        """Returns a relative filename inside the output dir that contains
89        modifier.
90
91        For example, if filename is fast\dom\foo.html and modifier is
92        "-expected.txt", the return value is fast\dom\foo-expected.txt
93
94        Args:
95          filename: relative filename to test file
96          modifier: a string to replace the extension of filename with
97
98        Return:
99          The relative windows path to the output filename
100        """
101        return os.path.splitext(filename)[0] + modifier
102
103
104class FailureWithType(TestFailure):
105    """Base class that produces standard HTML output based on the test type.
106
107    Subclasses may commonly choose to override the ResultHtmlOutput, but still
108    use the standard OutputLinks.
109    """
110
111    def __init__(self, test_type):
112        TestFailure.__init__(self)
113        # TODO(ojan): This class no longer needs to know the test_type.
114        self._test_type = test_type
115
116    # Filename suffixes used by ResultHtmlOutput.
117    OUT_FILENAMES = []
118
119    def output_links(self, filename, out_names):
120        """Returns a string holding all applicable output file links.
121
122        Args:
123          filename: the test filename, used to construct the result file names
124          out_names: list of filename suffixes for the files. If three or more
125              suffixes are in the list, they should be [actual, expected, diff,
126              wdiff]. Two suffixes should be [actual, expected], and a
127              single item is the [actual] filename suffix.
128              If out_names is empty, returns the empty string.
129        """
130        links = ['']
131        uris = [self.relative_output_filename(filename, fn) for
132                fn in out_names]
133        if len(uris) > 1:
134            links.append("<a href='%s'>expected</a>" % uris[1])
135        if len(uris) > 0:
136            links.append("<a href='%s'>actual</a>" % uris[0])
137        if len(uris) > 2:
138            links.append("<a href='%s'>diff</a>" % uris[2])
139        if len(uris) > 3:
140            links.append("<a href='%s'>wdiff</a>" % uris[3])
141        return ' '.join(links)
142
143    def result_html_output(self, filename):
144        return self.message() + self.output_links(filename, self.OUT_FILENAMES)
145
146
147class FailureTimeout(TestFailure):
148    """Test timed out.  We also want to restart the test shell if this
149    happens."""
150
151    @staticmethod
152    def message():
153        return "Test timed out"
154
155    def result_html_output(self, filename):
156        return "<strong>%s</strong>" % self.message()
157
158    def should_kill_test_shell(self):
159        return True
160
161
162class FailureCrash(TestFailure):
163    """Test shell crashed."""
164
165    @staticmethod
166    def message():
167        return "Test shell crashed"
168
169    def result_html_output(self, filename):
170        # TODO(tc): create a link to the minidump file
171        stack = self.relative_output_filename(filename, "-stack.txt")
172        return "<strong>%s</strong> <a href=%s>stack</a>" % (self.message(),
173                                                             stack)
174
175    def should_kill_test_shell(self):
176        return True
177
178
179class FailureMissingResult(FailureWithType):
180    """Expected result was missing."""
181    OUT_FILENAMES = ["-actual.txt"]
182
183    @staticmethod
184    def message():
185        return "No expected results found"
186
187    def result_html_output(self, filename):
188        return ("<strong>%s</strong>" % self.message() +
189                self.output_links(filename, self.OUT_FILENAMES))
190
191
192class FailureTextMismatch(FailureWithType):
193    """Text diff output failed."""
194    # Filename suffixes used by ResultHtmlOutput.
195    OUT_FILENAMES = ["-actual.txt", "-expected.txt", "-diff.txt"]
196    OUT_FILENAMES_WDIFF = ["-actual.txt", "-expected.txt", "-diff.txt",
197                           "-wdiff.html"]
198
199    def __init__(self, test_type, has_wdiff):
200        FailureWithType.__init__(self, test_type)
201        if has_wdiff:
202            self.OUT_FILENAMES = self.OUT_FILENAMES_WDIFF
203
204    @staticmethod
205    def message():
206        return "Text diff mismatch"
207
208
209class FailureMissingImageHash(FailureWithType):
210    """Actual result hash was missing."""
211    # Chrome doesn't know to display a .checksum file as text, so don't bother
212    # putting in a link to the actual result.
213    OUT_FILENAMES = []
214
215    @staticmethod
216    def message():
217        return "No expected image hash found"
218
219    def result_html_output(self, filename):
220        return "<strong>%s</strong>" % self.message()
221
222
223class FailureMissingImage(FailureWithType):
224    """Actual result image was missing."""
225    OUT_FILENAMES = ["-actual.png"]
226
227    @staticmethod
228    def message():
229        return "No expected image found"
230
231    def result_html_output(self, filename):
232        return ("<strong>%s</strong>" % self.message() +
233                self.output_links(filename, self.OUT_FILENAMES))
234
235
236class FailureImageHashMismatch(FailureWithType):
237    """Image hashes didn't match."""
238    OUT_FILENAMES = ["-actual.png", "-expected.png", "-diff.png"]
239
240    @staticmethod
241    def message():
242        # We call this a simple image mismatch to avoid confusion, since
243        # we link to the PNGs rather than the checksums.
244        return "Image mismatch"
245
246
247class FailureFuzzyFailure(FailureWithType):
248    """Image hashes didn't match."""
249    OUT_FILENAMES = ["-actual.png", "-expected.png"]
250
251    @staticmethod
252    def message():
253        return "Fuzzy image match also failed"
254
255
256class FailureImageHashIncorrect(FailureWithType):
257    """Actual result hash is incorrect."""
258    # Chrome doesn't know to display a .checksum file as text, so don't bother
259    # putting in a link to the actual result.
260    OUT_FILENAMES = []
261
262    @staticmethod
263    def message():
264        return "Images match, expected image hash incorrect. "
265
266    def result_html_output(self, filename):
267        return "<strong>%s</strong>" % self.message()
268