• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1# Copyright (C) 2010 Google Inc. All rights reserved.
2#
3# Redistribution and use in source and binary forms, with or without
4# modification, are permitted provided that the following conditions are
5# met:
6#
7#     * Redistributions of source code must retain the above copyright
8# notice, this list of conditions and the following disclaimer.
9#     * Redistributions in binary form must reproduce the above
10# copyright notice, this list of conditions and the following disclaimer
11# in the documentation and/or other materials provided with the
12# distribution.
13#     * Neither the name of Google Inc. nor the names of its
14# contributors may be used to endorse or promote products derived from
15# this software without specific prior written permission.
16#
17# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
18# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
19# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
20# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
21# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
22# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
23# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28
29import cPickle
30
31from webkitpy.layout_tests.models import test_expectations
32
33
34def is_reftest_failure(failure_list):
35    failure_types = [type(f) for f in failure_list]
36    return set((FailureReftestMismatch, FailureReftestMismatchDidNotOccur, FailureReftestNoImagesGenerated)).intersection(failure_types)
37
38# FIXME: This is backwards.  Each TestFailure subclass should know what
39# test_expectation type it corresponds too.  Then this method just
40# collects them all from the failure list and returns the worst one.
41def determine_result_type(failure_list):
42    """Takes a set of test_failures and returns which result type best fits
43    the list of failures. "Best fits" means we use the worst type of failure.
44
45    Returns:
46      one of the test_expectations result types - PASS, FAIL, CRASH, etc."""
47
48    if not failure_list or len(failure_list) == 0:
49        return test_expectations.PASS
50
51    failure_types = [type(f) for f in failure_list]
52    if FailureCrash in failure_types:
53        return test_expectations.CRASH
54    elif FailureLeak in failure_types:
55        return test_expectations.LEAK
56    elif FailureTimeout in failure_types:
57        return test_expectations.TIMEOUT
58    elif FailureEarlyExit in failure_types:
59        return test_expectations.SKIP
60    elif (FailureMissingResult in failure_types or
61          FailureMissingImage in failure_types or
62          FailureMissingImageHash in failure_types or
63          FailureMissingAudio in failure_types):
64        return test_expectations.MISSING
65    else:
66        is_text_failure = (FailureTextMismatch in failure_types or
67                           FailureTestHarnessAssertion in failure_types)
68        is_image_failure = (FailureImageHashIncorrect in failure_types or
69                            FailureImageHashMismatch in failure_types)
70        is_audio_failure = (FailureAudioMismatch in failure_types)
71        if is_text_failure and is_image_failure:
72            return test_expectations.IMAGE_PLUS_TEXT
73        elif is_text_failure:
74            return test_expectations.TEXT
75        elif is_image_failure or is_reftest_failure(failure_list):
76            return test_expectations.IMAGE
77        elif is_audio_failure:
78            return test_expectations.AUDIO
79        else:
80            raise ValueError("unclassifiable set of failures: "
81                             + str(failure_types))
82
83
84class TestFailure(object):
85    """Abstract base class that defines the failure interface."""
86
87    @staticmethod
88    def loads(s):
89        """Creates a TestFailure object from the specified string."""
90        return cPickle.loads(s)
91
92    def message(self):
93        """Returns a string describing the failure in more detail."""
94        raise NotImplementedError
95
96    def __eq__(self, other):
97        return self.__class__.__name__ == other.__class__.__name__
98
99    def __ne__(self, other):
100        return self.__class__.__name__ != other.__class__.__name__
101
102    def __hash__(self):
103        return hash(self.__class__.__name__)
104
105    def dumps(self):
106        """Returns the string/JSON representation of a TestFailure."""
107        return cPickle.dumps(self)
108
109    def driver_needs_restart(self):
110        """Returns True if we should kill the driver before the next test."""
111        return False
112
113
114class FailureTimeout(TestFailure):
115    def __init__(self, is_reftest=False):
116        super(FailureTimeout, self).__init__()
117        self.is_reftest = is_reftest
118
119    def message(self):
120        return "test timed out"
121
122    def driver_needs_restart(self):
123        return True
124
125
126class FailureCrash(TestFailure):
127    def __init__(self, is_reftest=False, process_name='content_shell', pid=None, has_log=False):
128        super(FailureCrash, self).__init__()
129        self.process_name = process_name
130        self.pid = pid
131        self.is_reftest = is_reftest
132        self.has_log = has_log
133
134    def message(self):
135        if self.pid:
136            return "%s crashed [pid=%d]" % (self.process_name, self.pid)
137        return self.process_name + " crashed"
138
139    def driver_needs_restart(self):
140        return True
141
142
143class FailureLeak(TestFailure):
144    def __init__(self, is_reftest=False, log=''):
145        super(FailureLeak, self).__init__()
146        self.is_reftest = is_reftest
147        self.log = log
148
149    def message(self):
150        return "leak detected: %s" % (self.log)
151
152
153class FailureMissingResult(TestFailure):
154    def message(self):
155        return "-expected.txt was missing"
156
157
158class FailureTestHarnessAssertion(TestFailure):
159    def message(self):
160        return "asserts failed"
161
162
163class FailureTextMismatch(TestFailure):
164    def message(self):
165        return "text diff"
166
167
168class FailureMissingImageHash(TestFailure):
169    def message(self):
170        return "-expected.png was missing an embedded checksum"
171
172
173class FailureMissingImage(TestFailure):
174    def message(self):
175        return "-expected.png was missing"
176
177
178class FailureImageHashMismatch(TestFailure):
179    def message(self):
180        return "image diff"
181
182
183class FailureImageHashIncorrect(TestFailure):
184    def message(self):
185        return "-expected.png embedded checksum is incorrect"
186
187
188class FailureReftestMismatch(TestFailure):
189    def __init__(self, reference_filename=None):
190        super(FailureReftestMismatch, self).__init__()
191        self.reference_filename = reference_filename
192
193    def message(self):
194        return "reference mismatch"
195
196
197class FailureReftestMismatchDidNotOccur(TestFailure):
198    def __init__(self, reference_filename=None):
199        super(FailureReftestMismatchDidNotOccur, self).__init__()
200        self.reference_filename = reference_filename
201
202    def message(self):
203        return "reference mismatch didn't happen"
204
205
206class FailureReftestNoImagesGenerated(TestFailure):
207    def __init__(self, reference_filename=None):
208        super(FailureReftestNoImagesGenerated, self).__init__()
209        self.reference_filename = reference_filename
210
211    def message(self):
212        return "reference didn't generate pixel results."
213
214
215class FailureMissingAudio(TestFailure):
216    def message(self):
217        return "expected audio result was missing"
218
219
220class FailureAudioMismatch(TestFailure):
221    def message(self):
222        return "audio mismatch"
223
224
225class FailureEarlyExit(TestFailure):
226    def message(self):
227        return "skipped due to early exit"
228
229
230# Convenient collection of all failure classes for anything that might
231# need to enumerate over them all.
232ALL_FAILURE_CLASSES = (FailureTimeout, FailureCrash, FailureMissingResult,
233                       FailureTestHarnessAssertion,
234                       FailureTextMismatch, FailureMissingImageHash,
235                       FailureMissingImage, FailureImageHashMismatch,
236                       FailureImageHashIncorrect, FailureReftestMismatch,
237                       FailureReftestMismatchDidNotOccur, FailureReftestNoImagesGenerated,
238                       FailureMissingAudio, FailureAudioMismatch,
239                       FailureEarlyExit)
240