• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1import os
2from xml.sax.saxutils import escape
3from json import JSONEncoder
4
5# Test result codes.
6
7class ResultCode(object):
8    """Test result codes."""
9
10    # We override __new__ and __getnewargs__ to ensure that pickling still
11    # provides unique ResultCode objects in any particular instance.
12    _instances = {}
13    def __new__(cls, name, isFailure):
14        res = cls._instances.get(name)
15        if res is None:
16            cls._instances[name] = res = super(ResultCode, cls).__new__(cls)
17        return res
18    def __getnewargs__(self):
19        return (self.name, self.isFailure)
20
21    def __init__(self, name, isFailure):
22        self.name = name
23        self.isFailure = isFailure
24
25    def __repr__(self):
26        return '%s%r' % (self.__class__.__name__,
27                         (self.name, self.isFailure))
28
29PASS        = ResultCode('PASS', False)
30FLAKYPASS   = ResultCode('FLAKYPASS', False)
31XFAIL       = ResultCode('XFAIL', False)
32FAIL        = ResultCode('FAIL', True)
33XPASS       = ResultCode('XPASS', True)
34UNRESOLVED  = ResultCode('UNRESOLVED', True)
35UNSUPPORTED = ResultCode('UNSUPPORTED', False)
36
37# Test metric values.
38
39class MetricValue(object):
40    def format(self):
41        """
42        format() -> str
43
44        Convert this metric to a string suitable for displaying as part of the
45        console output.
46        """
47        raise RuntimeError("abstract method")
48
49    def todata(self):
50        """
51        todata() -> json-serializable data
52
53        Convert this metric to content suitable for serializing in the JSON test
54        output.
55        """
56        raise RuntimeError("abstract method")
57
58class IntMetricValue(MetricValue):
59    def __init__(self, value):
60        self.value = value
61
62    def format(self):
63        return str(self.value)
64
65    def todata(self):
66        return self.value
67
68class RealMetricValue(MetricValue):
69    def __init__(self, value):
70        self.value = value
71
72    def format(self):
73        return '%.4f' % self.value
74
75    def todata(self):
76        return self.value
77
78class JSONMetricValue(MetricValue):
79    """
80        JSONMetricValue is used for types that are representable in the output
81        but that are otherwise uninterpreted.
82    """
83    def __init__(self, value):
84        # Ensure the value is a serializable by trying to encode it.
85        # WARNING: The value may change before it is encoded again, and may
86        #          not be encodable after the change.
87        try:
88            e = JSONEncoder()
89            e.encode(value)
90        except TypeError:
91            raise
92        self.value = value
93
94    def format(self):
95        e = JSONEncoder(indent=2, sort_keys=True)
96        return e.encode(self.value)
97
98    def todata(self):
99        return self.value
100
101def toMetricValue(value):
102    if isinstance(value, MetricValue):
103        return value
104    elif isinstance(value, int) or isinstance(value, long):
105        return IntMetricValue(value)
106    elif isinstance(value, float):
107        return RealMetricValue(value)
108    else:
109        # Try to create a JSONMetricValue and let the constructor throw
110        # if value is not a valid type.
111        return JSONMetricValue(value)
112
113
114# Test results.
115
116class Result(object):
117    """Wrapper for the results of executing an individual test."""
118
119    def __init__(self, code, output='', elapsed=None):
120        # The result code.
121        self.code = code
122        # The test output.
123        self.output = output
124        # The wall timing to execute the test, if timing.
125        self.elapsed = elapsed
126        # The metrics reported by this test.
127        self.metrics = {}
128
129    def addMetric(self, name, value):
130        """
131        addMetric(name, value)
132
133        Attach a test metric to the test result, with the given name and list of
134        values. It is an error to attempt to attach the metrics with the same
135        name multiple times.
136
137        Each value must be an instance of a MetricValue subclass.
138        """
139        if name in self.metrics:
140            raise ValueError("result already includes metrics for %r" % (
141                    name,))
142        if not isinstance(value, MetricValue):
143            raise TypeError("unexpected metric value: %r" % (value,))
144        self.metrics[name] = value
145
146# Test classes.
147
148class TestSuite:
149    """TestSuite - Information on a group of tests.
150
151    A test suite groups together a set of logically related tests.
152    """
153
154    def __init__(self, name, source_root, exec_root, config):
155        self.name = name
156        self.source_root = source_root
157        self.exec_root = exec_root
158        # The test suite configuration.
159        self.config = config
160
161    def getSourcePath(self, components):
162        return os.path.join(self.source_root, *components)
163
164    def getExecPath(self, components):
165        return os.path.join(self.exec_root, *components)
166
167class Test:
168    """Test - Information on a single test instance."""
169
170    def __init__(self, suite, path_in_suite, config, file_path = None):
171        self.suite = suite
172        self.path_in_suite = path_in_suite
173        self.config = config
174        self.file_path = file_path
175        # A list of conditions under which this test is expected to fail. These
176        # can optionally be provided by test format handlers, and will be
177        # honored when the test result is supplied.
178        self.xfails = []
179        # The test result, once complete.
180        self.result = None
181
182    def setResult(self, result):
183        if self.result is not None:
184            raise ArgumentError("test result already set")
185        if not isinstance(result, Result):
186            raise ArgumentError("unexpected result type")
187
188        self.result = result
189
190        # Apply the XFAIL handling to resolve the result exit code.
191        if self.isExpectedToFail():
192            if self.result.code == PASS:
193                self.result.code = XPASS
194            elif self.result.code == FAIL:
195                self.result.code = XFAIL
196
197    def getFullName(self):
198        return self.suite.config.name + ' :: ' + '/'.join(self.path_in_suite)
199
200    def getFilePath(self):
201        if self.file_path:
202            return self.file_path
203        return self.getSourcePath()
204
205    def getSourcePath(self):
206        return self.suite.getSourcePath(self.path_in_suite)
207
208    def getExecPath(self):
209        return self.suite.getExecPath(self.path_in_suite)
210
211    def isExpectedToFail(self):
212        """
213        isExpectedToFail() -> bool
214
215        Check whether this test is expected to fail in the current
216        configuration. This check relies on the test xfails property which by
217        some test formats may not be computed until the test has first been
218        executed.
219        """
220
221        # Check if any of the xfails match an available feature or the target.
222        for item in self.xfails:
223            # If this is the wildcard, it always fails.
224            if item == '*':
225                return True
226
227            # If this is an exact match for one of the features, it fails.
228            if item in self.config.available_features:
229                return True
230
231            # If this is a part of the target triple, it fails.
232            if item in self.suite.config.target_triple:
233                return True
234
235        return False
236
237
238    def getJUnitXML(self):
239        test_name = self.path_in_suite[-1]
240        test_path = self.path_in_suite[:-1]
241        safe_test_path = [x.replace(".","_") for x in test_path]
242        safe_name = self.suite.name.replace(".","-")
243
244        if safe_test_path:
245            class_name = safe_name + "." + "/".join(safe_test_path)
246        else:
247            class_name = safe_name + "." + safe_name
248
249        xml = "<testcase classname='" + class_name + "' name='" + \
250            test_name + "'"
251        xml += " time='%.2f'" % (self.result.elapsed,)
252        if self.result.code.isFailure:
253            xml += ">\n\t<failure >\n" + escape(self.result.output)
254            xml += "\n\t</failure>\n</testcase>"
255        else:
256            xml += "/>"
257        return xml
258