1#!/usr/bin/env python 2# 3# Copyright 2018, The Android Open Source Project 4# 5# Licensed under the Apache License, Version 2.0 (the "License"); 6# you may not use this file except in compliance with the License. 7# You may obtain a copy of the License at 8# 9# http://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an "AS IS" BASIS, 13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14# See the License for the specific language governing permissions and 15# limitations under the License. 16 17"""Unittests for result_reporter.""" 18 19import unittest 20import mock 21 22import result_reporter 23from test_runners import test_runner_base 24 25RESULT_PASSED_TEST = test_runner_base.TestResult( 26 runner_name='someTestRunner', 27 group_name='someTestModule', 28 test_name='someClassName#sostName', 29 status=test_runner_base.PASSED_STATUS, 30 details=None, 31 test_count=1, 32 test_time='(10ms)', 33 runner_total=None, 34 group_total=2, 35 perf_info={} 36) 37 38RESULT_PASSED_TEST_MODULE_2 = test_runner_base.TestResult( 39 runner_name='someTestRunner', 40 group_name='someTestModule2', 41 test_name='someClassName#sostName', 42 status=test_runner_base.PASSED_STATUS, 43 details=None, 44 test_count=1, 45 test_time='(10ms)', 46 runner_total=None, 47 group_total=2, 48 perf_info={} 49) 50 51RESULT_PASSED_TEST_RUNNER_2_NO_MODULE = test_runner_base.TestResult( 52 runner_name='someTestRunner2', 53 group_name=None, 54 test_name='someClassName#sostName', 55 status=test_runner_base.PASSED_STATUS, 56 details=None, 57 test_count=1, 58 test_time='(10ms)', 59 runner_total=None, 60 group_total=2, 61 perf_info={} 62) 63 64RESULT_FAILED_TEST = test_runner_base.TestResult( 65 runner_name='someTestRunner', 66 group_name='someTestModule', 67 test_name='someClassName2#sestName2', 68 status=test_runner_base.FAILED_STATUS, 69 details='someTrace', 70 test_count=1, 71 test_time='', 72 runner_total=None, 73 group_total=2, 74 perf_info={} 75) 76 77RESULT_RUN_FAILURE = test_runner_base.TestResult( 78 runner_name='someTestRunner', 79 group_name='someTestModule', 80 test_name='someClassName#sostName', 81 status=test_runner_base.ERROR_STATUS, 82 details='someRunFailureReason', 83 test_count=1, 84 test_time='', 85 runner_total=None, 86 group_total=2, 87 perf_info={} 88) 89 90RESULT_INVOCATION_FAILURE = test_runner_base.TestResult( 91 runner_name='someTestRunner', 92 group_name=None, 93 test_name=None, 94 status=test_runner_base.ERROR_STATUS, 95 details='someInvocationFailureReason', 96 test_count=1, 97 test_time='', 98 runner_total=None, 99 group_total=None, 100 perf_info={} 101) 102 103RESULT_IGNORED_TEST = test_runner_base.TestResult( 104 runner_name='someTestRunner', 105 group_name='someTestModule', 106 test_name='someClassName#sostName', 107 status=test_runner_base.IGNORED_STATUS, 108 details=None, 109 test_count=1, 110 test_time='(10ms)', 111 runner_total=None, 112 group_total=2, 113 perf_info={} 114) 115 116RESULT_ASSUMPTION_FAILED_TEST = test_runner_base.TestResult( 117 runner_name='someTestRunner', 118 group_name='someTestModule', 119 test_name='someClassName#sostName', 120 status=test_runner_base.ASSUMPTION_FAILED, 121 details=None, 122 test_count=1, 123 test_time='(10ms)', 124 runner_total=None, 125 group_total=2, 126 perf_info={} 127) 128 129#pylint: disable=protected-access 130#pylint: disable=invalid-name 131class ResultReporterUnittests(unittest.TestCase): 132 """Unit tests for result_reporter.py""" 133 134 def setUp(self): 135 self.rr = result_reporter.ResultReporter() 136 137 def tearDown(self): 138 mock.patch.stopall() 139 140 @mock.patch.object(result_reporter.ResultReporter, '_print_group_title') 141 @mock.patch.object(result_reporter.ResultReporter, '_update_stats') 142 @mock.patch.object(result_reporter.ResultReporter, '_print_result') 143 def test_process_test_result(self, mock_print, mock_update, mock_title): 144 """Test process_test_result method.""" 145 # Passed Test 146 self.assertTrue('someTestRunner' not in self.rr.runners) 147 self.rr.process_test_result(RESULT_PASSED_TEST) 148 self.assertTrue('someTestRunner' in self.rr.runners) 149 group = self.rr.runners['someTestRunner'].get('someTestModule') 150 self.assertIsNotNone(group) 151 mock_title.assert_called_with(RESULT_PASSED_TEST) 152 mock_update.assert_called_with(RESULT_PASSED_TEST, group) 153 mock_print.assert_called_with(RESULT_PASSED_TEST) 154 # Failed Test 155 mock_title.reset_mock() 156 self.rr.process_test_result(RESULT_FAILED_TEST) 157 mock_title.assert_not_called() 158 mock_update.assert_called_with(RESULT_FAILED_TEST, group) 159 mock_print.assert_called_with(RESULT_FAILED_TEST) 160 # Test with new Group 161 mock_title.reset_mock() 162 self.rr.process_test_result(RESULT_PASSED_TEST_MODULE_2) 163 self.assertTrue('someTestModule2' in self.rr.runners['someTestRunner']) 164 mock_title.assert_called_with(RESULT_PASSED_TEST_MODULE_2) 165 # Test with new Runner 166 mock_title.reset_mock() 167 self.rr.process_test_result(RESULT_PASSED_TEST_RUNNER_2_NO_MODULE) 168 self.assertTrue('someTestRunner2' in self.rr.runners) 169 mock_title.assert_called_with(RESULT_PASSED_TEST_RUNNER_2_NO_MODULE) 170 171 def test_register_unsupported_runner(self): 172 """Test register_unsupported_runner method.""" 173 self.rr.register_unsupported_runner('NotSupported') 174 runner = self.rr.runners['NotSupported'] 175 self.assertIsNotNone(runner) 176 self.assertEquals(runner, result_reporter.UNSUPPORTED_FLAG) 177 178 def test_update_stats_passed(self): 179 """Test _update_stats method.""" 180 # Passed Test 181 group = result_reporter.RunStat() 182 self.rr._update_stats(RESULT_PASSED_TEST, group) 183 self.assertEquals(self.rr.run_stats.passed, 1) 184 self.assertEquals(self.rr.run_stats.failed, 0) 185 self.assertEquals(self.rr.run_stats.run_errors, False) 186 self.assertEquals(self.rr.failed_tests, []) 187 self.assertEquals(group.passed, 1) 188 self.assertEquals(group.failed, 0) 189 self.assertEquals(group.ignored, 0) 190 self.assertEquals(group.run_errors, False) 191 # Passed Test New Group 192 group2 = result_reporter.RunStat() 193 self.rr._update_stats(RESULT_PASSED_TEST_MODULE_2, group2) 194 self.assertEquals(self.rr.run_stats.passed, 2) 195 self.assertEquals(self.rr.run_stats.failed, 0) 196 self.assertEquals(self.rr.run_stats.run_errors, False) 197 self.assertEquals(self.rr.failed_tests, []) 198 self.assertEquals(group2.passed, 1) 199 self.assertEquals(group2.failed, 0) 200 self.assertEquals(group.ignored, 0) 201 self.assertEquals(group2.run_errors, False) 202 203 def test_update_stats_failed(self): 204 """Test _update_stats method.""" 205 # Passed Test 206 group = result_reporter.RunStat() 207 self.rr._update_stats(RESULT_PASSED_TEST, group) 208 # Passed Test New Group 209 group2 = result_reporter.RunStat() 210 self.rr._update_stats(RESULT_PASSED_TEST_MODULE_2, group2) 211 # Failed Test Old Group 212 self.rr._update_stats(RESULT_FAILED_TEST, group) 213 self.assertEquals(self.rr.run_stats.passed, 2) 214 self.assertEquals(self.rr.run_stats.failed, 1) 215 self.assertEquals(self.rr.run_stats.run_errors, False) 216 self.assertEquals(self.rr.failed_tests, [RESULT_FAILED_TEST.test_name]) 217 self.assertEquals(group.passed, 1) 218 self.assertEquals(group.failed, 1) 219 self.assertEquals(group.ignored, 0) 220 self.assertEquals(group.total, 2) 221 self.assertEquals(group2.total, 1) 222 self.assertEquals(group.run_errors, False) 223 # Test Run Failure 224 self.rr._update_stats(RESULT_RUN_FAILURE, group) 225 self.assertEquals(self.rr.run_stats.passed, 2) 226 self.assertEquals(self.rr.run_stats.failed, 1) 227 self.assertEquals(self.rr.run_stats.run_errors, True) 228 self.assertEquals(self.rr.failed_tests, [RESULT_FAILED_TEST.test_name]) 229 self.assertEquals(group.passed, 1) 230 self.assertEquals(group.failed, 1) 231 self.assertEquals(group.ignored, 0) 232 self.assertEquals(group.run_errors, True) 233 self.assertEquals(group2.run_errors, False) 234 # Invocation Failure 235 self.rr._update_stats(RESULT_INVOCATION_FAILURE, group) 236 self.assertEquals(self.rr.run_stats.passed, 2) 237 self.assertEquals(self.rr.run_stats.failed, 1) 238 self.assertEquals(self.rr.run_stats.run_errors, True) 239 self.assertEquals(self.rr.failed_tests, [RESULT_FAILED_TEST.test_name]) 240 self.assertEquals(group.passed, 1) 241 self.assertEquals(group.failed, 1) 242 self.assertEquals(group.ignored, 0) 243 self.assertEquals(group.run_errors, True) 244 245 def test_update_stats_ignored_and_assumption_failure(self): 246 """Test _update_stats method.""" 247 # Passed Test 248 group = result_reporter.RunStat() 249 self.rr._update_stats(RESULT_PASSED_TEST, group) 250 # Passed Test New Group 251 group2 = result_reporter.RunStat() 252 self.rr._update_stats(RESULT_PASSED_TEST_MODULE_2, group2) 253 # Failed Test Old Group 254 self.rr._update_stats(RESULT_FAILED_TEST, group) 255 # Test Run Failure 256 self.rr._update_stats(RESULT_RUN_FAILURE, group) 257 # Invocation Failure 258 self.rr._update_stats(RESULT_INVOCATION_FAILURE, group) 259 # Ignored Test 260 self.rr._update_stats(RESULT_IGNORED_TEST, group) 261 self.assertEquals(self.rr.run_stats.passed, 2) 262 self.assertEquals(self.rr.run_stats.failed, 1) 263 self.assertEquals(self.rr.run_stats.run_errors, True) 264 self.assertEquals(self.rr.failed_tests, [RESULT_FAILED_TEST.test_name]) 265 self.assertEquals(group.passed, 1) 266 self.assertEquals(group.failed, 1) 267 self.assertEquals(group.ignored, 1) 268 self.assertEquals(group.run_errors, True) 269 # 2nd Ignored Test 270 self.rr._update_stats(RESULT_IGNORED_TEST, group) 271 self.assertEquals(self.rr.run_stats.passed, 2) 272 self.assertEquals(self.rr.run_stats.failed, 1) 273 self.assertEquals(self.rr.run_stats.run_errors, True) 274 self.assertEquals(self.rr.failed_tests, [RESULT_FAILED_TEST.test_name]) 275 self.assertEquals(group.passed, 1) 276 self.assertEquals(group.failed, 1) 277 self.assertEquals(group.ignored, 2) 278 self.assertEquals(group.run_errors, True) 279 280 # Assumption_Failure test 281 self.rr._update_stats(RESULT_ASSUMPTION_FAILED_TEST, group) 282 self.assertEquals(group.assumption_failed, 1) 283 # 2nd Assumption_Failure test 284 self.rr._update_stats(RESULT_ASSUMPTION_FAILED_TEST, group) 285 self.assertEquals(group.assumption_failed, 2) 286 287 def test_print_summary_ret_val(self): 288 """Test print_summary method's return value.""" 289 # PASS Case 290 self.rr.process_test_result(RESULT_PASSED_TEST) 291 self.assertEquals(0, self.rr.print_summary()) 292 # PASS Case + Fail Case 293 self.rr.process_test_result(RESULT_FAILED_TEST) 294 self.assertNotEqual(0, self.rr.print_summary()) 295 # PASS Case + Fail Case + PASS Case 296 self.rr.process_test_result(RESULT_PASSED_TEST_MODULE_2) 297 self.assertNotEqual(0, self.rr.print_summary()) 298 299 def test_print_summary_ret_val_err_stat(self): 300 """Test print_summary method's return value.""" 301 # PASS Case 302 self.rr.process_test_result(RESULT_PASSED_TEST) 303 self.assertEquals(0, self.rr.print_summary()) 304 # PASS Case + Fail Case 305 self.rr.process_test_result(RESULT_RUN_FAILURE) 306 self.assertNotEqual(0, self.rr.print_summary()) 307 # PASS Case + Fail Case + PASS Case 308 self.rr.process_test_result(RESULT_PASSED_TEST_MODULE_2) 309 self.assertNotEqual(0, self.rr.print_summary()) 310 311if __name__ == '__main__': 312 unittest.main() 313