• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1import io
2import sys
3import textwrap
4
5from test import support
6
7import traceback
8import unittest
9
10
11class MockTraceback(object):
12    class TracebackException:
13        def __init__(self, *args, **kwargs):
14            self.capture_locals = kwargs.get('capture_locals', False)
15        def format(self):
16            result = ['A traceback']
17            if self.capture_locals:
18                result.append('locals')
19            return result
20
21def restore_traceback():
22    unittest.result.traceback = traceback
23
24
25class Test_TestResult(unittest.TestCase):
26    # Note: there are not separate tests for TestResult.wasSuccessful(),
27    # TestResult.errors, TestResult.failures, TestResult.testsRun or
28    # TestResult.shouldStop because these only have meaning in terms of
29    # other TestResult methods.
30    #
31    # Accordingly, tests for the aforenamed attributes are incorporated
32    # in with the tests for the defining methods.
33    ################################################################
34
35    def test_init(self):
36        result = unittest.TestResult()
37
38        self.assertTrue(result.wasSuccessful())
39        self.assertEqual(len(result.errors), 0)
40        self.assertEqual(len(result.failures), 0)
41        self.assertEqual(result.testsRun, 0)
42        self.assertEqual(result.shouldStop, False)
43        self.assertIsNone(result._stdout_buffer)
44        self.assertIsNone(result._stderr_buffer)
45
46    # "This method can be called to signal that the set of tests being
47    # run should be aborted by setting the TestResult's shouldStop
48    # attribute to True."
49    def test_stop(self):
50        result = unittest.TestResult()
51
52        result.stop()
53
54        self.assertEqual(result.shouldStop, True)
55
56    # "Called when the test case test is about to be run. The default
57    # implementation simply increments the instance's testsRun counter."
58    def test_startTest(self):
59        class Foo(unittest.TestCase):
60            def test_1(self):
61                pass
62
63        test = Foo('test_1')
64
65        result = unittest.TestResult()
66
67        result.startTest(test)
68
69        self.assertTrue(result.wasSuccessful())
70        self.assertEqual(len(result.errors), 0)
71        self.assertEqual(len(result.failures), 0)
72        self.assertEqual(result.testsRun, 1)
73        self.assertEqual(result.shouldStop, False)
74
75        result.stopTest(test)
76
77    # "Called after the test case test has been executed, regardless of
78    # the outcome. The default implementation does nothing."
79    def test_stopTest(self):
80        class Foo(unittest.TestCase):
81            def test_1(self):
82                pass
83
84        test = Foo('test_1')
85
86        result = unittest.TestResult()
87
88        result.startTest(test)
89
90        self.assertTrue(result.wasSuccessful())
91        self.assertEqual(len(result.errors), 0)
92        self.assertEqual(len(result.failures), 0)
93        self.assertEqual(result.testsRun, 1)
94        self.assertEqual(result.shouldStop, False)
95
96        result.stopTest(test)
97
98        # Same tests as above; make sure nothing has changed
99        self.assertTrue(result.wasSuccessful())
100        self.assertEqual(len(result.errors), 0)
101        self.assertEqual(len(result.failures), 0)
102        self.assertEqual(result.testsRun, 1)
103        self.assertEqual(result.shouldStop, False)
104
105    # "Called before and after tests are run. The default implementation does nothing."
106    def test_startTestRun_stopTestRun(self):
107        result = unittest.TestResult()
108        result.startTestRun()
109        result.stopTestRun()
110
111    # "addSuccess(test)"
112    # ...
113    # "Called when the test case test succeeds"
114    # ...
115    # "wasSuccessful() - Returns True if all tests run so far have passed,
116    # otherwise returns False"
117    # ...
118    # "testsRun - The total number of tests run so far."
119    # ...
120    # "errors - A list containing 2-tuples of TestCase instances and
121    # formatted tracebacks. Each tuple represents a test which raised an
122    # unexpected exception. Contains formatted
123    # tracebacks instead of sys.exc_info() results."
124    # ...
125    # "failures - A list containing 2-tuples of TestCase instances and
126    # formatted tracebacks. Each tuple represents a test where a failure was
127    # explicitly signalled using the TestCase.fail*() or TestCase.assert*()
128    # methods. Contains formatted tracebacks instead
129    # of sys.exc_info() results."
130    def test_addSuccess(self):
131        class Foo(unittest.TestCase):
132            def test_1(self):
133                pass
134
135        test = Foo('test_1')
136
137        result = unittest.TestResult()
138
139        result.startTest(test)
140        result.addSuccess(test)
141        result.stopTest(test)
142
143        self.assertTrue(result.wasSuccessful())
144        self.assertEqual(len(result.errors), 0)
145        self.assertEqual(len(result.failures), 0)
146        self.assertEqual(result.testsRun, 1)
147        self.assertEqual(result.shouldStop, False)
148
149    # "addFailure(test, err)"
150    # ...
151    # "Called when the test case test signals a failure. err is a tuple of
152    # the form returned by sys.exc_info(): (type, value, traceback)"
153    # ...
154    # "wasSuccessful() - Returns True if all tests run so far have passed,
155    # otherwise returns False"
156    # ...
157    # "testsRun - The total number of tests run so far."
158    # ...
159    # "errors - A list containing 2-tuples of TestCase instances and
160    # formatted tracebacks. Each tuple represents a test which raised an
161    # unexpected exception. Contains formatted
162    # tracebacks instead of sys.exc_info() results."
163    # ...
164    # "failures - A list containing 2-tuples of TestCase instances and
165    # formatted tracebacks. Each tuple represents a test where a failure was
166    # explicitly signalled using the TestCase.fail*() or TestCase.assert*()
167    # methods. Contains formatted tracebacks instead
168    # of sys.exc_info() results."
169    def test_addFailure(self):
170        class Foo(unittest.TestCase):
171            def test_1(self):
172                pass
173
174        test = Foo('test_1')
175        try:
176            test.fail("foo")
177        except:
178            exc_info_tuple = sys.exc_info()
179
180        result = unittest.TestResult()
181
182        result.startTest(test)
183        result.addFailure(test, exc_info_tuple)
184        result.stopTest(test)
185
186        self.assertFalse(result.wasSuccessful())
187        self.assertEqual(len(result.errors), 0)
188        self.assertEqual(len(result.failures), 1)
189        self.assertEqual(result.testsRun, 1)
190        self.assertEqual(result.shouldStop, False)
191
192        test_case, formatted_exc = result.failures[0]
193        self.assertIs(test_case, test)
194        self.assertIsInstance(formatted_exc, str)
195
196    # "addError(test, err)"
197    # ...
198    # "Called when the test case test raises an unexpected exception err
199    # is a tuple of the form returned by sys.exc_info():
200    # (type, value, traceback)"
201    # ...
202    # "wasSuccessful() - Returns True if all tests run so far have passed,
203    # otherwise returns False"
204    # ...
205    # "testsRun - The total number of tests run so far."
206    # ...
207    # "errors - A list containing 2-tuples of TestCase instances and
208    # formatted tracebacks. Each tuple represents a test which raised an
209    # unexpected exception. Contains formatted
210    # tracebacks instead of sys.exc_info() results."
211    # ...
212    # "failures - A list containing 2-tuples of TestCase instances and
213    # formatted tracebacks. Each tuple represents a test where a failure was
214    # explicitly signalled using the TestCase.fail*() or TestCase.assert*()
215    # methods. Contains formatted tracebacks instead
216    # of sys.exc_info() results."
217    def test_addError(self):
218        class Foo(unittest.TestCase):
219            def test_1(self):
220                pass
221
222        test = Foo('test_1')
223        try:
224            raise TypeError()
225        except:
226            exc_info_tuple = sys.exc_info()
227
228        result = unittest.TestResult()
229
230        result.startTest(test)
231        result.addError(test, exc_info_tuple)
232        result.stopTest(test)
233
234        self.assertFalse(result.wasSuccessful())
235        self.assertEqual(len(result.errors), 1)
236        self.assertEqual(len(result.failures), 0)
237        self.assertEqual(result.testsRun, 1)
238        self.assertEqual(result.shouldStop, False)
239
240        test_case, formatted_exc = result.errors[0]
241        self.assertIs(test_case, test)
242        self.assertIsInstance(formatted_exc, str)
243
244    def test_addError_locals(self):
245        class Foo(unittest.TestCase):
246            def test_1(self):
247                1/0
248
249        test = Foo('test_1')
250        result = unittest.TestResult()
251        result.tb_locals = True
252
253        unittest.result.traceback = MockTraceback
254        self.addCleanup(restore_traceback)
255        result.startTestRun()
256        test.run(result)
257        result.stopTestRun()
258
259        self.assertEqual(len(result.errors), 1)
260        test_case, formatted_exc = result.errors[0]
261        self.assertEqual('A tracebacklocals', formatted_exc)
262
263    def test_addSubTest(self):
264        class Foo(unittest.TestCase):
265            def test_1(self):
266                nonlocal subtest
267                with self.subTest(foo=1):
268                    subtest = self._subtest
269                    try:
270                        1/0
271                    except ZeroDivisionError:
272                        exc_info_tuple = sys.exc_info()
273                    # Register an error by hand (to check the API)
274                    result.addSubTest(test, subtest, exc_info_tuple)
275                    # Now trigger a failure
276                    self.fail("some recognizable failure")
277
278        subtest = None
279        test = Foo('test_1')
280        result = unittest.TestResult()
281
282        test.run(result)
283
284        self.assertFalse(result.wasSuccessful())
285        self.assertEqual(len(result.errors), 1)
286        self.assertEqual(len(result.failures), 1)
287        self.assertEqual(result.testsRun, 1)
288        self.assertEqual(result.shouldStop, False)
289
290        test_case, formatted_exc = result.errors[0]
291        self.assertIs(test_case, subtest)
292        self.assertIn("ZeroDivisionError", formatted_exc)
293        test_case, formatted_exc = result.failures[0]
294        self.assertIs(test_case, subtest)
295        self.assertIn("some recognizable failure", formatted_exc)
296
297    def testGetDescriptionWithoutDocstring(self):
298        result = unittest.TextTestResult(None, True, 1)
299        self.assertEqual(
300                result.getDescription(self),
301                'testGetDescriptionWithoutDocstring (' + __name__ +
302                '.Test_TestResult)')
303
304    def testGetSubTestDescriptionWithoutDocstring(self):
305        with self.subTest(foo=1, bar=2):
306            result = unittest.TextTestResult(None, True, 1)
307            self.assertEqual(
308                    result.getDescription(self._subtest),
309                    'testGetSubTestDescriptionWithoutDocstring (' + __name__ +
310                    '.Test_TestResult) (bar=2, foo=1)')
311        with self.subTest('some message'):
312            result = unittest.TextTestResult(None, True, 1)
313            self.assertEqual(
314                    result.getDescription(self._subtest),
315                    'testGetSubTestDescriptionWithoutDocstring (' + __name__ +
316                    '.Test_TestResult) [some message]')
317
318    def testGetSubTestDescriptionWithoutDocstringAndParams(self):
319        with self.subTest():
320            result = unittest.TextTestResult(None, True, 1)
321            self.assertEqual(
322                    result.getDescription(self._subtest),
323                    'testGetSubTestDescriptionWithoutDocstringAndParams '
324                    '(' + __name__ + '.Test_TestResult) (<subtest>)')
325
326    def testGetSubTestDescriptionForFalsyValues(self):
327        expected = 'testGetSubTestDescriptionForFalsyValues (%s.Test_TestResult) [%s]'
328        result = unittest.TextTestResult(None, True, 1)
329        for arg in [0, None, []]:
330            with self.subTest(arg):
331                self.assertEqual(
332                    result.getDescription(self._subtest),
333                    expected % (__name__, arg)
334                )
335
336    def testGetNestedSubTestDescriptionWithoutDocstring(self):
337        with self.subTest(foo=1):
338            with self.subTest(bar=2):
339                result = unittest.TextTestResult(None, True, 1)
340                self.assertEqual(
341                        result.getDescription(self._subtest),
342                        'testGetNestedSubTestDescriptionWithoutDocstring '
343                        '(' + __name__ + '.Test_TestResult) (bar=2, foo=1)')
344
345    @unittest.skipIf(sys.flags.optimize >= 2,
346                     "Docstrings are omitted with -O2 and above")
347    def testGetDescriptionWithOneLineDocstring(self):
348        """Tests getDescription() for a method with a docstring."""
349        result = unittest.TextTestResult(None, True, 1)
350        self.assertEqual(
351                result.getDescription(self),
352               ('testGetDescriptionWithOneLineDocstring '
353                '(' + __name__ + '.Test_TestResult)\n'
354                'Tests getDescription() for a method with a docstring.'))
355
356    @unittest.skipIf(sys.flags.optimize >= 2,
357                     "Docstrings are omitted with -O2 and above")
358    def testGetSubTestDescriptionWithOneLineDocstring(self):
359        """Tests getDescription() for a method with a docstring."""
360        result = unittest.TextTestResult(None, True, 1)
361        with self.subTest(foo=1, bar=2):
362            self.assertEqual(
363                result.getDescription(self._subtest),
364               ('testGetSubTestDescriptionWithOneLineDocstring '
365                '(' + __name__ + '.Test_TestResult) (bar=2, foo=1)\n'
366                'Tests getDescription() for a method with a docstring.'))
367
368    @unittest.skipIf(sys.flags.optimize >= 2,
369                     "Docstrings are omitted with -O2 and above")
370    def testGetDescriptionWithMultiLineDocstring(self):
371        """Tests getDescription() for a method with a longer docstring.
372        The second line of the docstring.
373        """
374        result = unittest.TextTestResult(None, True, 1)
375        self.assertEqual(
376                result.getDescription(self),
377               ('testGetDescriptionWithMultiLineDocstring '
378                '(' + __name__ + '.Test_TestResult)\n'
379                'Tests getDescription() for a method with a longer '
380                'docstring.'))
381
382    @unittest.skipIf(sys.flags.optimize >= 2,
383                     "Docstrings are omitted with -O2 and above")
384    def testGetSubTestDescriptionWithMultiLineDocstring(self):
385        """Tests getDescription() for a method with a longer docstring.
386        The second line of the docstring.
387        """
388        result = unittest.TextTestResult(None, True, 1)
389        with self.subTest(foo=1, bar=2):
390            self.assertEqual(
391                result.getDescription(self._subtest),
392               ('testGetSubTestDescriptionWithMultiLineDocstring '
393                '(' + __name__ + '.Test_TestResult) (bar=2, foo=1)\n'
394                'Tests getDescription() for a method with a longer '
395                'docstring.'))
396
397    def testStackFrameTrimming(self):
398        class Frame(object):
399            class tb_frame(object):
400                f_globals = {}
401        result = unittest.TestResult()
402        self.assertFalse(result._is_relevant_tb_level(Frame))
403
404        Frame.tb_frame.f_globals['__unittest'] = True
405        self.assertTrue(result._is_relevant_tb_level(Frame))
406
407    def testFailFast(self):
408        result = unittest.TestResult()
409        result._exc_info_to_string = lambda *_: ''
410        result.failfast = True
411        result.addError(None, None)
412        self.assertTrue(result.shouldStop)
413
414        result = unittest.TestResult()
415        result._exc_info_to_string = lambda *_: ''
416        result.failfast = True
417        result.addFailure(None, None)
418        self.assertTrue(result.shouldStop)
419
420        result = unittest.TestResult()
421        result._exc_info_to_string = lambda *_: ''
422        result.failfast = True
423        result.addUnexpectedSuccess(None)
424        self.assertTrue(result.shouldStop)
425
426    def testFailFastSetByRunner(self):
427        runner = unittest.TextTestRunner(stream=io.StringIO(), failfast=True)
428        def test(result):
429            self.assertTrue(result.failfast)
430        result = runner.run(test)
431
432
433classDict = dict(unittest.TestResult.__dict__)
434for m in ('addSkip', 'addExpectedFailure', 'addUnexpectedSuccess',
435           '__init__'):
436    del classDict[m]
437
438def __init__(self, stream=None, descriptions=None, verbosity=None):
439    self.failures = []
440    self.errors = []
441    self.testsRun = 0
442    self.shouldStop = False
443    self.buffer = False
444    self.tb_locals = False
445
446classDict['__init__'] = __init__
447OldResult = type('OldResult', (object,), classDict)
448
449class Test_OldTestResult(unittest.TestCase):
450
451    def assertOldResultWarning(self, test, failures):
452        with support.check_warnings(("TestResult has no add.+ method,",
453                                     RuntimeWarning)):
454            result = OldResult()
455            test.run(result)
456            self.assertEqual(len(result.failures), failures)
457
458    def testOldTestResult(self):
459        class Test(unittest.TestCase):
460            def testSkip(self):
461                self.skipTest('foobar')
462            @unittest.expectedFailure
463            def testExpectedFail(self):
464                raise TypeError
465            @unittest.expectedFailure
466            def testUnexpectedSuccess(self):
467                pass
468
469        for test_name, should_pass in (('testSkip', True),
470                                       ('testExpectedFail', True),
471                                       ('testUnexpectedSuccess', False)):
472            test = Test(test_name)
473            self.assertOldResultWarning(test, int(not should_pass))
474
475    def testOldTestTesultSetup(self):
476        class Test(unittest.TestCase):
477            def setUp(self):
478                self.skipTest('no reason')
479            def testFoo(self):
480                pass
481        self.assertOldResultWarning(Test('testFoo'), 0)
482
483    def testOldTestResultClass(self):
484        @unittest.skip('no reason')
485        class Test(unittest.TestCase):
486            def testFoo(self):
487                pass
488        self.assertOldResultWarning(Test('testFoo'), 0)
489
490    def testOldResultWithRunner(self):
491        class Test(unittest.TestCase):
492            def testFoo(self):
493                pass
494        runner = unittest.TextTestRunner(resultclass=OldResult,
495                                          stream=io.StringIO())
496        # This will raise an exception if TextTestRunner can't handle old
497        # test result objects
498        runner.run(Test('testFoo'))
499
500
501class TestOutputBuffering(unittest.TestCase):
502
503    def setUp(self):
504        self._real_out = sys.stdout
505        self._real_err = sys.stderr
506
507    def tearDown(self):
508        sys.stdout = self._real_out
509        sys.stderr = self._real_err
510
511    def testBufferOutputOff(self):
512        real_out = self._real_out
513        real_err = self._real_err
514
515        result = unittest.TestResult()
516        self.assertFalse(result.buffer)
517
518        self.assertIs(real_out, sys.stdout)
519        self.assertIs(real_err, sys.stderr)
520
521        result.startTest(self)
522
523        self.assertIs(real_out, sys.stdout)
524        self.assertIs(real_err, sys.stderr)
525
526    def testBufferOutputStartTestAddSuccess(self):
527        real_out = self._real_out
528        real_err = self._real_err
529
530        result = unittest.TestResult()
531        self.assertFalse(result.buffer)
532
533        result.buffer = True
534
535        self.assertIs(real_out, sys.stdout)
536        self.assertIs(real_err, sys.stderr)
537
538        result.startTest(self)
539
540        self.assertIsNot(real_out, sys.stdout)
541        self.assertIsNot(real_err, sys.stderr)
542        self.assertIsInstance(sys.stdout, io.StringIO)
543        self.assertIsInstance(sys.stderr, io.StringIO)
544        self.assertIsNot(sys.stdout, sys.stderr)
545
546        out_stream = sys.stdout
547        err_stream = sys.stderr
548
549        result._original_stdout = io.StringIO()
550        result._original_stderr = io.StringIO()
551
552        print('foo')
553        print('bar', file=sys.stderr)
554
555        self.assertEqual(out_stream.getvalue(), 'foo\n')
556        self.assertEqual(err_stream.getvalue(), 'bar\n')
557
558        self.assertEqual(result._original_stdout.getvalue(), '')
559        self.assertEqual(result._original_stderr.getvalue(), '')
560
561        result.addSuccess(self)
562        result.stopTest(self)
563
564        self.assertIs(sys.stdout, result._original_stdout)
565        self.assertIs(sys.stderr, result._original_stderr)
566
567        self.assertEqual(result._original_stdout.getvalue(), '')
568        self.assertEqual(result._original_stderr.getvalue(), '')
569
570        self.assertEqual(out_stream.getvalue(), '')
571        self.assertEqual(err_stream.getvalue(), '')
572
573
574    def getStartedResult(self):
575        result = unittest.TestResult()
576        result.buffer = True
577        result.startTest(self)
578        return result
579
580    def testBufferOutputAddErrorOrFailure(self):
581        unittest.result.traceback = MockTraceback
582        self.addCleanup(restore_traceback)
583
584        for message_attr, add_attr, include_error in [
585            ('errors', 'addError', True),
586            ('failures', 'addFailure', False),
587            ('errors', 'addError', True),
588            ('failures', 'addFailure', False)
589        ]:
590            result = self.getStartedResult()
591            buffered_out = sys.stdout
592            buffered_err = sys.stderr
593            result._original_stdout = io.StringIO()
594            result._original_stderr = io.StringIO()
595
596            print('foo', file=sys.stdout)
597            if include_error:
598                print('bar', file=sys.stderr)
599
600
601            addFunction = getattr(result, add_attr)
602            addFunction(self, (None, None, None))
603            result.stopTest(self)
604
605            result_list = getattr(result, message_attr)
606            self.assertEqual(len(result_list), 1)
607
608            test, message = result_list[0]
609            expectedOutMessage = textwrap.dedent("""
610                Stdout:
611                foo
612            """)
613            expectedErrMessage = ''
614            if include_error:
615                expectedErrMessage = textwrap.dedent("""
616                Stderr:
617                bar
618            """)
619
620            expectedFullMessage = 'A traceback%s%s' % (expectedOutMessage, expectedErrMessage)
621
622            self.assertIs(test, self)
623            self.assertEqual(result._original_stdout.getvalue(), expectedOutMessage)
624            self.assertEqual(result._original_stderr.getvalue(), expectedErrMessage)
625            self.assertMultiLineEqual(message, expectedFullMessage)
626
627    def testBufferSetupClass(self):
628        result = unittest.TestResult()
629        result.buffer = True
630
631        class Foo(unittest.TestCase):
632            @classmethod
633            def setUpClass(cls):
634                1/0
635            def test_foo(self):
636                pass
637        suite = unittest.TestSuite([Foo('test_foo')])
638        suite(result)
639        self.assertEqual(len(result.errors), 1)
640
641    def testBufferTearDownClass(self):
642        result = unittest.TestResult()
643        result.buffer = True
644
645        class Foo(unittest.TestCase):
646            @classmethod
647            def tearDownClass(cls):
648                1/0
649            def test_foo(self):
650                pass
651        suite = unittest.TestSuite([Foo('test_foo')])
652        suite(result)
653        self.assertEqual(len(result.errors), 1)
654
655    def testBufferSetUpModule(self):
656        result = unittest.TestResult()
657        result.buffer = True
658
659        class Foo(unittest.TestCase):
660            def test_foo(self):
661                pass
662        class Module(object):
663            @staticmethod
664            def setUpModule():
665                1/0
666
667        Foo.__module__ = 'Module'
668        sys.modules['Module'] = Module
669        self.addCleanup(sys.modules.pop, 'Module')
670        suite = unittest.TestSuite([Foo('test_foo')])
671        suite(result)
672        self.assertEqual(len(result.errors), 1)
673
674    def testBufferTearDownModule(self):
675        result = unittest.TestResult()
676        result.buffer = True
677
678        class Foo(unittest.TestCase):
679            def test_foo(self):
680                pass
681        class Module(object):
682            @staticmethod
683            def tearDownModule():
684                1/0
685
686        Foo.__module__ = 'Module'
687        sys.modules['Module'] = Module
688        self.addCleanup(sys.modules.pop, 'Module')
689        suite = unittest.TestSuite([Foo('test_foo')])
690        suite(result)
691        self.assertEqual(len(result.errors), 1)
692
693
694if __name__ == '__main__':
695    unittest.main()
696