• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1'''Test runner and result class for the regression test suite.
2
3'''
4
5import functools
6import io
7import sys
8import time
9import traceback
10import unittest
11from test import support
12from test.libregrtest.utils import sanitize_xml
13
14class RegressionTestResult(unittest.TextTestResult):
15    USE_XML = False
16
17    def __init__(self, stream, descriptions, verbosity):
18        super().__init__(stream=stream, descriptions=descriptions,
19                         verbosity=2 if verbosity else 0)
20        self.buffer = True
21        if self.USE_XML:
22            from xml.etree import ElementTree as ET
23            from datetime import datetime, UTC
24            self.__ET = ET
25            self.__suite = ET.Element('testsuite')
26            self.__suite.set('start',
27                             datetime.now(UTC)
28                                     .replace(tzinfo=None)
29                                     .isoformat(' '))
30            self.__e = None
31        self.__start_time = None
32
33    @classmethod
34    def __getId(cls, test):
35        try:
36            test_id = test.id
37        except AttributeError:
38            return str(test)
39        try:
40            return test_id()
41        except TypeError:
42            return str(test_id)
43        return repr(test)
44
45    def startTest(self, test):
46        super().startTest(test)
47        if self.USE_XML:
48            self.__e = e = self.__ET.SubElement(self.__suite, 'testcase')
49        self.__start_time = time.perf_counter()
50
51    def _add_result(self, test, capture=False, **args):
52        if not self.USE_XML:
53            return
54        e = self.__e
55        self.__e = None
56        if e is None:
57            return
58        ET = self.__ET
59
60        e.set('name', args.pop('name', self.__getId(test)))
61        e.set('status', args.pop('status', 'run'))
62        e.set('result', args.pop('result', 'completed'))
63        if self.__start_time:
64            e.set('time', f'{time.perf_counter() - self.__start_time:0.6f}')
65
66        if capture:
67            if self._stdout_buffer is not None:
68                stdout = self._stdout_buffer.getvalue().rstrip()
69                ET.SubElement(e, 'system-out').text = sanitize_xml(stdout)
70            if self._stderr_buffer is not None:
71                stderr = self._stderr_buffer.getvalue().rstrip()
72                ET.SubElement(e, 'system-err').text = sanitize_xml(stderr)
73
74        for k, v in args.items():
75            if not k or not v:
76                continue
77
78            e2 = ET.SubElement(e, k)
79            if hasattr(v, 'items'):
80                for k2, v2 in v.items():
81                    if k2:
82                        e2.set(k2, sanitize_xml(str(v2)))
83                    else:
84                        e2.text = sanitize_xml(str(v2))
85            else:
86                e2.text = sanitize_xml(str(v))
87
88    @classmethod
89    def __makeErrorDict(cls, err_type, err_value, err_tb):
90        if isinstance(err_type, type):
91            if err_type.__module__ == 'builtins':
92                typename = err_type.__name__
93            else:
94                typename = f'{err_type.__module__}.{err_type.__name__}'
95        else:
96            typename = repr(err_type)
97
98        msg = traceback.format_exception(err_type, err_value, None)
99        tb = traceback.format_exception(err_type, err_value, err_tb)
100
101        return {
102            'type': typename,
103            'message': ''.join(msg),
104            '': ''.join(tb),
105        }
106
107    def addError(self, test, err):
108        self._add_result(test, True, error=self.__makeErrorDict(*err))
109        super().addError(test, err)
110
111    def addExpectedFailure(self, test, err):
112        self._add_result(test, True, output=self.__makeErrorDict(*err))
113        super().addExpectedFailure(test, err)
114
115    def addFailure(self, test, err):
116        self._add_result(test, True, failure=self.__makeErrorDict(*err))
117        super().addFailure(test, err)
118        if support.failfast:
119            self.stop()
120
121    def addSkip(self, test, reason):
122        self._add_result(test, skipped=reason)
123        super().addSkip(test, reason)
124
125    def addSuccess(self, test):
126        self._add_result(test)
127        super().addSuccess(test)
128
129    def addUnexpectedSuccess(self, test):
130        self._add_result(test, outcome='UNEXPECTED_SUCCESS')
131        super().addUnexpectedSuccess(test)
132
133    def get_xml_element(self):
134        if not self.USE_XML:
135            raise ValueError("USE_XML is false")
136        e = self.__suite
137        e.set('tests', str(self.testsRun))
138        e.set('errors', str(len(self.errors)))
139        e.set('failures', str(len(self.failures)))
140        return e
141
142class QuietRegressionTestRunner:
143    def __init__(self, stream, buffer=False):
144        self.result = RegressionTestResult(stream, None, 0)
145        self.result.buffer = buffer
146
147    def run(self, test):
148        test(self.result)
149        return self.result
150
151def get_test_runner_class(verbosity, buffer=False):
152    if verbosity:
153        return functools.partial(unittest.TextTestRunner,
154                                 resultclass=RegressionTestResult,
155                                 buffer=buffer,
156                                 verbosity=verbosity)
157    return functools.partial(QuietRegressionTestRunner, buffer=buffer)
158
159def get_test_runner(stream, verbosity, capture_output=False):
160    return get_test_runner_class(verbosity, capture_output)(stream)
161
162if __name__ == '__main__':
163    import xml.etree.ElementTree as ET
164    RegressionTestResult.USE_XML = True
165
166    class TestTests(unittest.TestCase):
167        def test_pass(self):
168            pass
169
170        def test_pass_slow(self):
171            time.sleep(1.0)
172
173        def test_fail(self):
174            print('stdout', file=sys.stdout)
175            print('stderr', file=sys.stderr)
176            self.fail('failure message')
177
178        def test_error(self):
179            print('stdout', file=sys.stdout)
180            print('stderr', file=sys.stderr)
181            raise RuntimeError('error message')
182
183    suite = unittest.TestSuite()
184    suite.addTest(unittest.TestLoader().loadTestsFromTestCase(TestTests))
185    stream = io.StringIO()
186    runner_cls = get_test_runner_class(sum(a == '-v' for a in sys.argv))
187    runner = runner_cls(sys.stdout)
188    result = runner.run(suite)
189    print('Output:', stream.getvalue())
190    print('XML: ', end='')
191    for s in ET.tostringlist(result.get_xml_element()):
192        print(s.decode(), end='')
193    print()
194