• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/env python3
2#
3#   Licensed under the Apache License, Version 2.0 (the "License");
4#   you may not use this file except in compliance with the License.
5#   You may obtain a copy of the License at
6#
7#       http://www.apache.org/licenses/LICENSE-2.0
8#
9#   Unless required by applicable law or agreed to in writing, software
10#   distributed under the License is distributed on an "AS IS" BASIS,
11#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12#   See the License for the specific language governing permissions and
13#   limitations under the License.
14
15import mock
16import shutil
17import tempfile
18import unittest
19import mock
20
21from mobly import config_parser as mobly_config_parser
22
23from acts import base_test
24from acts import signals
25from acts import test_decorators
26from acts import test_runner
27from acts.controllers.sl4a_lib import rpc_client
28from acts.libs.testtracker.testtracker_results_writer import KEY_EFFORT_NAME
29
30TEST_TRACKER_UUID = '12345'
31UUID_KEY = 'uuid'
32
33
34def return_true():
35    return True
36
37
38def return_false():
39    return False
40
41
42def raise_pass():
43    raise signals.TestPass('')
44
45
46def raise_failure():
47    raise signals.TestFailure('')
48
49
50def raise_sl4a():
51    raise rpc_client.Sl4aException('')
52
53
54def raise_generic():
55    raise Exception('')
56
57
58class TestDecoratorUnitTests(unittest.TestCase):
59    def _verify_test_info(self, func):
60        try:
61            test_decorators.test_info(uuid=TEST_TRACKER_UUID)(func)()
62            self.fail('Expected decorator to raise exception.')
63        except signals.TestSignal as e:
64            self.assertTrue(hasattr(e, 'extras'),
65                            'Expected thrown exception to have extras.')
66            self.assertTrue(UUID_KEY in e.extras,
67                            'Expected extras to have %s.' % UUID_KEY)
68            self.assertEqual(e.extras[UUID_KEY], TEST_TRACKER_UUID)
69
70    def test_test_info_on_return_true(self):
71        self._verify_test_info(return_true)
72
73    def test_test_info_on_return_false(self):
74        self._verify_test_info(return_false)
75
76    def test_test_info_on_raise_pass(self):
77        self._verify_test_info(raise_pass)
78
79    def test_test_info_on_raise_failure(self):
80        self._verify_test_info(raise_failure)
81
82    def test_test_info_on_raise_sl4a(self):
83        self._verify_test_info(raise_sl4a)
84
85    def test_test_info_on_raise_generic(self):
86        self._verify_test_info(raise_generic)
87
88    def test_test_tracker_info_on_raise_generic_is_chained(self):
89        @test_decorators.test_tracker_info(uuid='SOME_UID')
90        def raise_generic():
91            raise ValueError('I am a ValueError')
92
93        with self.assertRaises(signals.TestError) as context:
94            raise_generic()
95
96        self.assertIsInstance(context.exception.__cause__, ValueError)
97
98    def test_tti_returns_existing_test_pass(self):
99        @test_decorators.test_info(uuid='SOME_UID')
100        def test_raises_test_pass():
101            raise signals.TestPass('Expected Message')
102
103        with self.assertRaises(signals.TestPass) as context:
104            test_raises_test_pass()
105
106        self.assertEqual(context.exception.details, 'Expected Message')
107
108    def test_tti_returns_existing_test_failure(self):
109        @test_decorators.test_info(uuid='SOME_UID')
110        def test_raises_test_failure():
111            raise signals.TestFailure('Expected Message')
112
113        with self.assertRaises(signals.TestFailure) as context:
114            test_raises_test_failure()
115
116        self.assertEqual(context.exception.details, 'Expected Message')
117
118    def test_tti_returns_test_error_on_non_signal_error(self):
119        expected_error = ValueError('Some Message')
120
121        @test_decorators.test_info(uuid='SOME_UID')
122        def test_raises_non_signal():
123            raise expected_error
124
125        with self.assertRaises(signals.TestError) as context:
126            test_raises_non_signal()
127
128        self.assertEqual(context.exception.details, expected_error)
129
130    def test_tti_returns_test_pass_if_no_return_value_specified(self):
131        @test_decorators.test_info(uuid='SOME_UID')
132        def test_returns_nothing():
133            pass
134
135        with self.assertRaises(signals.TestPass):
136            test_returns_nothing()
137
138    def test_tti_returns_test_fail_if_return_value_is_truthy(self):
139        """This is heavily frowned upon. Use signals.TestPass instead!"""
140
141        @test_decorators.test_info(uuid='SOME_UID')
142        def test_returns_truthy():
143            return True
144
145        with self.assertRaises(signals.TestPass):
146            test_returns_truthy()
147
148    def test_tti_returns_test_fail_if_return_value_is_falsy(self):
149        """This is heavily frowned upon. Use signals.TestFailure instead!"""
150
151        @test_decorators.test_info(uuid='SOME_UID')
152        def test_returns_falsy_but_not_none():
153            return False
154
155        with self.assertRaises(signals.TestFailure):
156            test_returns_falsy_but_not_none()
157
158    def test_function_name(self):
159        """Test test_func.__name__ returns its original unwrapped name"""
160        @test_decorators.test_info(uuid='SOME_UID')
161        def test_func():
162            pass
163
164        self.assertEqual(test_func.__name__, "test_func")
165
166    def test_function_doc(self):
167        """Test test_func.__doc__ returns its original unwrapped doc string"""
168        @test_decorators.test_info(uuid='SOME_UID')
169        def test_func():
170            """DOC_STRING"""
171            pass
172
173        self.assertEqual(test_func.__doc__, "DOC_STRING")
174
175    def test_function_module(self):
176        """Test test_func.__module__ returns its original unwrapped module"""
177        @test_decorators.test_info(uuid='SOME_UID')
178        def test_func():
179            pass
180
181        self.assertEqual(test_func.__module__, self.__module__)
182
183
184class MockTest(base_test.BaseTestClass):
185    TEST_CASE_LIST = 'test_run_mock_test'
186    TEST_LOGIC_ATTR = 'test_logic'
187
188    @test_decorators.test_info(uuid=TEST_TRACKER_UUID)
189    def test_run_mock_test(self):
190        getattr(MockTest, MockTest.TEST_LOGIC_ATTR, None)()
191
192
193class FakeTest(base_test.BaseTestClass):
194    """A fake test class used to test TestTrackerInfoDecoratorFunc."""
195
196    def __init__(self):
197        self.user_params = {
198            'testtracker_properties': 'tt_prop_a=param_a,'
199                                      'tt_prop_b=param_b',
200            'param_a': 'prop_a_value',
201            'param_b': 'prop_b_value',
202        }
203        self.log_path = '/dummy/path'
204        self.begin_time = 1000000
205
206    @test_decorators.test_tracker_info(uuid='SOME_UID')
207    def test_case(self):
208        pass
209
210
211class TestTrackerInfoDecoratorFuncTests(unittest.TestCase):
212
213    @mock.patch('acts.test_decorators.TestTrackerResultsWriter')
214    def test_write_to_testtracker_no_effort_name_no_ad(self, mock_writer):
215        """Should not set the TT Effort name."""
216        with self.assertRaises(signals.TestPass):
217            FakeTest().test_case()
218
219        testtracker_properties = mock_writer.call_args[0][1]
220        self.assertEqual(testtracker_properties,
221                         {'tt_prop_a': 'prop_a_value',
222                          'tt_prop_b': 'prop_b_value'})
223
224    @mock.patch('acts.test_decorators.TestTrackerResultsWriter')
225    def test_write_to_testtracker_no_effort_name_has_ad(self, mock_writer):
226        """Should set the TT Effort Name using the AndroidDevice."""
227        fake_test = FakeTest()
228        fake_test.android_devices = [mock.Mock()]
229        fake_test.android_devices[0].build_info = {'build_id': 'EFFORT_NAME'}
230        with self.assertRaises(signals.TestPass):
231            fake_test.test_case()
232
233        testtracker_properties = mock_writer.call_args[0][1]
234        self.assertEqual(testtracker_properties,
235                         {'tt_prop_a': 'prop_a_value',
236                          'tt_prop_b': 'prop_b_value',
237                          KEY_EFFORT_NAME: 'EFFORT_NAME'})
238
239    @mock.patch('acts.test_decorators.TestTrackerResultsWriter')
240    def test_write_to_testtracker_has_effort_name_has_ad(self, mock_writer):
241        """Should set the TT Effort Name using the AndroidDevice."""
242        fake_test = FakeTest()
243        fake_test.android_devices = [mock.Mock()]
244        fake_test.android_devices[0].build_info = {'build_id': 'BAD_EFFORT'}
245        fake_test.user_params['testtracker_properties'] += (
246                    ',' + KEY_EFFORT_NAME + '=param_effort_name')
247        fake_test.user_params['param_effort_name'] = 'GOOD_EFFORT_NAME'
248
249        with self.assertRaises(signals.TestPass):
250            fake_test.test_case()
251
252        testtracker_properties = mock_writer.call_args[0][1]
253        self.assertEqual(testtracker_properties,
254                         {'tt_prop_a': 'prop_a_value',
255                          'tt_prop_b': 'prop_b_value',
256                          KEY_EFFORT_NAME: 'GOOD_EFFORT_NAME'})
257
258    @mock.patch('acts.test_decorators.TestTrackerResultsWriter')
259    def test_no_testtracker_properties_provided(self, mock_writer):
260        fake_test = FakeTest()
261        del fake_test.user_params['testtracker_properties']
262
263        with self.assertRaises(signals.TestPass):
264            fake_test.test_case()
265
266        self.assertFalse(mock_writer.called)
267
268    @mock.patch('acts.test_decorators.TestTrackerResultsWriter')
269    def test_needs_base_class_set_to_write_to_testtracker(self, mock_writer):
270        class DoesntHaveBaseClass(object):
271            @test_decorators.test_tracker_info(uuid='SOME_UID')
272            def test_case(self):
273                pass
274
275        with self.assertRaises(signals.TestPass):
276            DoesntHaveBaseClass().test_case()
277
278        self.assertFalse(mock_writer.called)
279
280    @mock.patch('acts.test_decorators.TestTrackerResultsWriter')
281    def test_does_not_write_when_decorated_args_are_missing(self, mock_writer):
282        @test_decorators.test_tracker_info(uuid='SOME_UID')
283        def some_decorated_func():
284            pass
285
286        with self.assertRaises(signals.TestPass):
287            some_decorated_func()
288
289        self.assertFalse(mock_writer.called)
290
291
292
293class TestDecoratorIntegrationTests(unittest.TestCase):
294    @classmethod
295    def setUpClass(cls):
296        cls.tmp_dir = tempfile.mkdtemp()
297        cls.MOCK_CONFIG = mobly_config_parser.TestRunConfig()
298        cls.MOCK_CONFIG.testbed_name = 'SampleTestBed'
299        cls.MOCK_CONFIG.log_path = cls.tmp_dir
300
301        cls.MOCK_TEST_RUN_LIST = [(MockTest.__name__,
302                                   [MockTest.TEST_CASE_LIST])]
303
304    @classmethod
305    def tearDownClass(cls):
306        shutil.rmtree(cls.tmp_dir)
307
308    def _run_with_test_logic(self, func):
309        if hasattr(MockTest, MockTest.TEST_LOGIC_ATTR):
310            delattr(MockTest, MockTest.TEST_LOGIC_ATTR)
311        setattr(MockTest, MockTest.TEST_LOGIC_ATTR, func)
312        self.test_runner = test_runner.TestRunner(self.MOCK_CONFIG,
313                                                  self.MOCK_TEST_RUN_LIST)
314        self.test_runner.run(MockTest)
315
316    def _validate_results_has_extra(self, result, extra_key, extra_value):
317        results = self.test_runner.results
318        self.assertGreaterEqual(len(results.executed), 1,
319                                'Expected at least one executed test.')
320        record = results.executed[0]
321        self.assertIsNotNone(record.extras,
322                             'Expected the test record to have extras.')
323        self.assertEqual(record.extras[extra_key], extra_value)
324
325    def test_mock_test_with_raise_pass(self):
326        self._run_with_test_logic(raise_pass)
327        self._validate_results_has_extra(self.test_runner.results, UUID_KEY,
328                                         TEST_TRACKER_UUID)
329
330    def test_mock_test_with_raise_generic(self):
331        self._run_with_test_logic(raise_generic)
332        self._validate_results_has_extra(self.test_runner.results, UUID_KEY,
333                                         TEST_TRACKER_UUID)
334
335
336class RepeatedTestTests(unittest.TestCase):
337    def test_all_error_types_count_toward_failures(self):
338        def result_selector(results, _):
339            self.assertIsInstance(results[0], AssertionError)
340            self.assertIsInstance(results[1], signals.TestFailure)
341            self.assertIsInstance(results[2], signals.TestError)
342            self.assertIsInstance(results[3], IndexError)
343            raise signals.TestPass('Expected failures occurred')
344
345        @test_decorators.repeated_test(1, 3, result_selector)
346        def test_case(_, attempt_number):
347            if attempt_number == 1:
348                raise AssertionError()
349            elif attempt_number == 2:
350                raise signals.TestFailure('Failed')
351            elif attempt_number == 3:
352                raise signals.TestError('Error')
353            else:
354                # Note that any Exception that does not fall into another bucket
355                # is also considered a failure
356                raise IndexError('Bad index')
357
358        with self.assertRaises(signals.TestPass):
359            test_case(mock.Mock())
360
361    def test_passes_stop_repeating_the_test_case(self):
362        def result_selector(results, _):
363            self.assertEqual(len(results), 3)
364            for result in results:
365                self.assertIsInstance(result, signals.TestPass)
366            raise signals.TestPass('Expected passes occurred')
367
368        @test_decorators.repeated_test(3, 0, result_selector)
369        def test_case(*_):
370            raise signals.TestPass('Passed')
371
372        with self.assertRaises(signals.TestPass):
373            test_case(mock.Mock())
374
375    def test_abort_signals_are_uncaught(self):
376        @test_decorators.repeated_test(3, 0)
377        def test_case(*_):
378            raise signals.TestAbortClass('Abort All')
379
380        with self.assertRaises(signals.TestAbortClass):
381            test_case(mock.Mock())
382
383    def test_keyboard_interrupt_is_uncaught(self):
384        @test_decorators.repeated_test(3, 0)
385        def test_case(*_):
386            raise KeyboardInterrupt()
387
388        with self.assertRaises(KeyboardInterrupt):
389            test_case(mock.Mock())
390
391    def test_teardown_and_setup_are_called_between_test_cases(self):
392        mock_test_class = mock.Mock()
393        @test_decorators.repeated_test(1, 1)
394        def test_case(*_):
395            raise signals.TestFailure('Failed')
396
397        with self.assertRaises(signals.TestFailure):
398            test_case(mock_test_class)
399
400        self.assertTrue(mock_test_class.setup_test.called)
401        self.assertTrue(mock_test_class.teardown_test.called)
402
403    def test_result_selector_returned_value_gets_raised(self):
404        def result_selector(*_):
405            return signals.TestPass('Expect this to be raised.')
406
407        @test_decorators.repeated_test(3, 0, result_selector=result_selector)
408        def test_case(*_):
409            raise signals.TestFailure('Result selector ignores this.')
410
411        with self.assertRaises(signals.TestPass):
412            test_case(mock.Mock())
413
414
415if __name__ == '__main__':
416    unittest.main()
417