• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/env python
2# Copyright (C) 2010 Google Inc. All rights reserved.
3#
4# Redistribution and use in source and binary forms, with or without
5# modification, are permitted provided that the following conditions are
6# met:
7#
8#     * Redistributions of source code must retain the above copyright
9# notice, this list of conditions and the following disclaimer.
10#     * Redistributions in binary form must reproduce the above
11# copyright notice, this list of conditions and the following disclaimer
12# in the documentation and/or other materials provided with the
13# distribution.
14#     * Neither the name of Google Inc. nor the names of its
15# contributors may be used to endorse or promote products derived from
16# this software without specific prior written permission.
17#
18# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29
30"""Classes for failures that occur during tests."""
31
32import test_expectations
33
34import cPickle
35
36
37# FIXME: This is backwards.  Each TestFailure subclass should know what
38# test_expectation type it corresponds too.  Then this method just
39# collects them all from the failure list and returns the worst one.
40def determine_result_type(failure_list):
41    """Takes a set of test_failures and returns which result type best fits
42    the list of failures. "Best fits" means we use the worst type of failure.
43
44    Returns:
45      one of the test_expectations result types - PASS, TEXT, CRASH, etc."""
46
47    if not failure_list or len(failure_list) == 0:
48        return test_expectations.PASS
49
50    failure_types = [type(f) for f in failure_list]
51    if FailureCrash in failure_types:
52        return test_expectations.CRASH
53    elif FailureTimeout in failure_types:
54        return test_expectations.TIMEOUT
55    elif (FailureMissingResult in failure_types or
56          FailureMissingImage in failure_types or
57          FailureMissingImageHash in failure_types or
58          FailureMissingAudio in failure_types):
59        return test_expectations.MISSING
60    else:
61        is_text_failure = FailureTextMismatch in failure_types
62        is_image_failure = (FailureImageHashIncorrect in failure_types or
63                            FailureImageHashMismatch in failure_types)
64        is_reftest_failure = (FailureReftestMismatch in failure_types or
65                              FailureReftestMismatchDidNotOccur in failure_types)
66        is_audio_failure = (FailureAudioMismatch in failure_types)
67        if is_text_failure and is_image_failure:
68            return test_expectations.IMAGE_PLUS_TEXT
69        elif is_text_failure:
70            return test_expectations.TEXT
71        elif is_image_failure or is_reftest_failure:
72            return test_expectations.IMAGE
73        elif is_audio_failure:
74            return test_expectations.AUDIO
75        else:
76            raise ValueError("unclassifiable set of failures: "
77                             + str(failure_types))
78
79
80class TestFailure(object):
81    """Abstract base class that defines the failure interface."""
82
83    @staticmethod
84    def loads(s):
85        """Creates a TestFailure object from the specified string."""
86        return cPickle.loads(s)
87
88    @staticmethod
89    def message():
90        """Returns a string describing the failure in more detail."""
91        raise NotImplementedError
92
93    def __eq__(self, other):
94        return self.__class__.__name__ == other.__class__.__name__
95
96    def __ne__(self, other):
97        return self.__class__.__name__ != other.__class__.__name__
98
99    def __hash__(self):
100        return hash(self.__class__.__name__)
101
102    def dumps(self):
103        """Returns the string/JSON representation of a TestFailure."""
104        return cPickle.dumps(self)
105
106    def should_kill_dump_render_tree(self):
107        """Returns True if we should kill DumpRenderTree before the next
108        test."""
109        return False
110
111
112class FailureTimeout(TestFailure):
113    """Test timed out.  We also want to restart DumpRenderTree if this
114    happens."""
115    def __init__(self, is_reftest=False):
116        self.is_reftest = is_reftest
117
118    @staticmethod
119    def message():
120        return "Test timed out"
121
122    def should_kill_dump_render_tree(self):
123        return True
124
125
126class FailureCrash(TestFailure):
127    """DumpRenderTree crashed."""
128    def __init__(self, is_reftest=False):
129        self.is_reftest = is_reftest
130
131    @staticmethod
132    def message():
133        return "DumpRenderTree crashed"
134
135    def should_kill_dump_render_tree(self):
136        return True
137
138
139class FailureMissingResult(TestFailure):
140    """Expected result was missing."""
141
142    @staticmethod
143    def message():
144        return "No expected results found"
145
146
147class FailureTextMismatch(TestFailure):
148    """Text diff output failed."""
149
150    @staticmethod
151    def message():
152        return "Text diff mismatch"
153
154
155class FailureMissingImageHash(TestFailure):
156    """Actual result hash was missing."""
157    # Chrome doesn't know to display a .checksum file as text, so don't bother
158    # putting in a link to the actual result.
159
160    @staticmethod
161    def message():
162        return "No expected image hash found"
163
164
165class FailureMissingImage(TestFailure):
166    """Actual result image was missing."""
167
168    @staticmethod
169    def message():
170        return "No expected image found"
171
172
173class FailureImageHashMismatch(TestFailure):
174    """Image hashes didn't match."""
175
176    @staticmethod
177    def message():
178        # We call this a simple image mismatch to avoid confusion, since
179        # we link to the PNGs rather than the checksums.
180        return "Image mismatch"
181
182
183class FailureImageHashIncorrect(TestFailure):
184    """Actual result hash is incorrect."""
185    # Chrome doesn't know to display a .checksum file as text, so don't bother
186    # putting in a link to the actual result.
187
188    @staticmethod
189    def message():
190        return "Images match, expected image hash incorrect. "
191
192
193class FailureReftestMismatch(TestFailure):
194    """The result didn't match the reference rendering."""
195
196    @staticmethod
197    def message():
198        return "Mismatch with reference"
199
200
201class FailureReftestMismatchDidNotOccur(TestFailure):
202    """Unexpected match between the result and the reference rendering."""
203
204    @staticmethod
205    def message():
206        return "Mismatch with the reference did not occur"
207
208
209class FailureMissingAudio(TestFailure):
210    """Actual result image was missing."""
211
212    @staticmethod
213    def message():
214        return "No expected audio found"
215
216
217class FailureAudioMismatch(TestFailure):
218    """Audio files didn't match."""
219
220    @staticmethod
221    def message():
222        return "Audio mismatch"
223
224
225# Convenient collection of all failure classes for anything that might
226# need to enumerate over them all.
227ALL_FAILURE_CLASSES = (FailureTimeout, FailureCrash, FailureMissingResult,
228                       FailureTextMismatch, FailureMissingImageHash,
229                       FailureMissingImage, FailureImageHashMismatch,
230                       FailureImageHashIncorrect, FailureReftestMismatch,
231                       FailureReftestMismatchDidNotOccur)
232