1#!/usr/bin/env python 2# 3# Copyright 2018, The Android Open Source Project 4# 5# Licensed under the Apache License, Version 2.0 (the "License"); 6# you may not use this file except in compliance with the License. 7# You may obtain a copy of the License at 8# 9# http://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an "AS IS" BASIS, 13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14# See the License for the specific language governing permissions and 15# limitations under the License. 16 17"""Unittests for result_reporter.""" 18 19import sys 20import unittest 21import mock 22 23import result_reporter 24from test_runners import test_runner_base 25 26if sys.version_info[0] == 2: 27 from StringIO import StringIO 28else: 29 from io import StringIO 30 31RESULT_PASSED_TEST = test_runner_base.TestResult( 32 runner_name='someTestRunner', 33 group_name='someTestModule', 34 test_name='someClassName#sostName', 35 status=test_runner_base.PASSED_STATUS, 36 details=None, 37 test_count=1, 38 test_time='(10ms)', 39 runner_total=None, 40 group_total=2, 41 additional_info={}, 42 test_run_name='com.android.UnitTests' 43) 44 45RESULT_PASSED_TEST_MODULE_2 = test_runner_base.TestResult( 46 runner_name='someTestRunner', 47 group_name='someTestModule2', 48 test_name='someClassName#sostName', 49 status=test_runner_base.PASSED_STATUS, 50 details=None, 51 test_count=1, 52 test_time='(10ms)', 53 runner_total=None, 54 group_total=2, 55 additional_info={}, 56 test_run_name='com.android.UnitTests' 57) 58 59RESULT_PASSED_TEST_RUNNER_2_NO_MODULE = test_runner_base.TestResult( 60 runner_name='someTestRunner2', 61 group_name=None, 62 test_name='someClassName#sostName', 63 status=test_runner_base.PASSED_STATUS, 64 details=None, 65 test_count=1, 66 test_time='(10ms)', 67 runner_total=None, 68 group_total=2, 69 additional_info={}, 70 test_run_name='com.android.UnitTests' 71) 72 73RESULT_FAILED_TEST = test_runner_base.TestResult( 74 runner_name='someTestRunner', 75 group_name='someTestModule', 76 test_name='someClassName2#sestName2', 77 status=test_runner_base.FAILED_STATUS, 78 details='someTrace', 79 test_count=1, 80 test_time='', 81 runner_total=None, 82 group_total=2, 83 additional_info={}, 84 test_run_name='com.android.UnitTests' 85) 86 87RESULT_RUN_FAILURE = test_runner_base.TestResult( 88 runner_name='someTestRunner', 89 group_name='someTestModule', 90 test_name='someClassName#sostName', 91 status=test_runner_base.ERROR_STATUS, 92 details='someRunFailureReason', 93 test_count=1, 94 test_time='', 95 runner_total=None, 96 group_total=2, 97 additional_info={}, 98 test_run_name='com.android.UnitTests' 99) 100 101RESULT_INVOCATION_FAILURE = test_runner_base.TestResult( 102 runner_name='someTestRunner', 103 group_name=None, 104 test_name=None, 105 status=test_runner_base.ERROR_STATUS, 106 details='someInvocationFailureReason', 107 test_count=1, 108 test_time='', 109 runner_total=None, 110 group_total=None, 111 additional_info={}, 112 test_run_name='com.android.UnitTests' 113) 114 115RESULT_IGNORED_TEST = test_runner_base.TestResult( 116 runner_name='someTestRunner', 117 group_name='someTestModule', 118 test_name='someClassName#sostName', 119 status=test_runner_base.IGNORED_STATUS, 120 details=None, 121 test_count=1, 122 test_time='(10ms)', 123 runner_total=None, 124 group_total=2, 125 additional_info={}, 126 test_run_name='com.android.UnitTests' 127) 128 129RESULT_ASSUMPTION_FAILED_TEST = test_runner_base.TestResult( 130 runner_name='someTestRunner', 131 group_name='someTestModule', 132 test_name='someClassName#sostName', 133 status=test_runner_base.ASSUMPTION_FAILED, 134 details=None, 135 test_count=1, 136 test_time='(10ms)', 137 runner_total=None, 138 group_total=2, 139 additional_info={}, 140 test_run_name='com.android.UnitTests' 141) 142 143ADDITIONAL_INFO_PERF01_TEST01 = {u'repetition_index': u'0', 144 u'cpu_time': u'10001.10001', 145 u'name': u'perfName01', 146 u'repetitions': u'0', u'run_type': u'iteration', 147 u'label': u'2123', u'threads': u'1', 148 u'time_unit': u'ns', u'iterations': u'1001', 149 u'run_name': u'perfName01', 150 u'real_time': u'11001.11001'} 151 152RESULT_PERF01_TEST01 = test_runner_base.TestResult( 153 runner_name='someTestRunner', 154 group_name='someTestModule', 155 test_name='somePerfClass01#perfName01', 156 status=test_runner_base.PASSED_STATUS, 157 details=None, 158 test_count=1, 159 test_time='(10ms)', 160 runner_total=None, 161 group_total=2, 162 additional_info=ADDITIONAL_INFO_PERF01_TEST01, 163 test_run_name='com.android.UnitTests' 164) 165 166RESULT_PERF01_TEST02 = test_runner_base.TestResult( 167 runner_name='someTestRunner', 168 group_name='someTestModule', 169 test_name='somePerfClass01#perfName02', 170 status=test_runner_base.PASSED_STATUS, 171 details=None, 172 test_count=1, 173 test_time='(10ms)', 174 runner_total=None, 175 group_total=2, 176 additional_info={u'repetition_index': u'0', u'cpu_time': u'10002.10002', 177 u'name': u'perfName02', 178 u'repetitions': u'0', u'run_type': u'iteration', 179 u'label': u'2123', u'threads': u'1', 180 u'time_unit': u'ns', u'iterations': u'1002', 181 u'run_name': u'perfName02', 182 u'real_time': u'11002.11002'}, 183 test_run_name='com.android.UnitTests' 184) 185 186RESULT_PERF01_TEST03_NO_CPU_TIME = test_runner_base.TestResult( 187 runner_name='someTestRunner', 188 group_name='someTestModule', 189 test_name='somePerfClass01#perfName03', 190 status=test_runner_base.PASSED_STATUS, 191 details=None, 192 test_count=1, 193 test_time='(10ms)', 194 runner_total=None, 195 group_total=2, 196 additional_info={u'repetition_index': u'0', 197 u'name': u'perfName03', 198 u'repetitions': u'0', u'run_type': u'iteration', 199 u'label': u'2123', u'threads': u'1', 200 u'time_unit': u'ns', u'iterations': u'1003', 201 u'run_name': u'perfName03', 202 u'real_time': u'11003.11003'}, 203 test_run_name='com.android.UnitTests' 204) 205 206RESULT_PERF02_TEST01 = test_runner_base.TestResult( 207 runner_name='someTestRunner', 208 group_name='someTestModule', 209 test_name='somePerfClass02#perfName11', 210 status=test_runner_base.PASSED_STATUS, 211 details=None, 212 test_count=1, 213 test_time='(10ms)', 214 runner_total=None, 215 group_total=2, 216 additional_info={u'repetition_index': u'0', u'cpu_time': u'20001.20001', 217 u'name': u'perfName11', 218 u'repetitions': u'0', u'run_type': u'iteration', 219 u'label': u'2123', u'threads': u'1', 220 u'time_unit': u'ns', u'iterations': u'2001', 221 u'run_name': u'perfName11', 222 u'real_time': u'210001.21001'}, 223 test_run_name='com.android.UnitTests' 224) 225 226#pylint: disable=protected-access 227#pylint: disable=invalid-name 228class ResultReporterUnittests(unittest.TestCase): 229 """Unit tests for result_reporter.py""" 230 231 def setUp(self): 232 self.rr = result_reporter.ResultReporter() 233 234 def tearDown(self): 235 mock.patch.stopall() 236 237 @mock.patch.object(result_reporter.ResultReporter, '_print_group_title') 238 @mock.patch.object(result_reporter.ResultReporter, '_update_stats') 239 @mock.patch.object(result_reporter.ResultReporter, '_print_result') 240 def test_process_test_result(self, mock_print, mock_update, mock_title): 241 """Test process_test_result method.""" 242 # Passed Test 243 self.assertTrue('someTestRunner' not in self.rr.runners) 244 self.rr.process_test_result(RESULT_PASSED_TEST) 245 self.assertTrue('someTestRunner' in self.rr.runners) 246 group = self.rr.runners['someTestRunner'].get('someTestModule') 247 self.assertIsNotNone(group) 248 mock_title.assert_called_with(RESULT_PASSED_TEST) 249 mock_update.assert_called_with(RESULT_PASSED_TEST, group) 250 mock_print.assert_called_with(RESULT_PASSED_TEST) 251 # Failed Test 252 mock_title.reset_mock() 253 self.rr.process_test_result(RESULT_FAILED_TEST) 254 mock_title.assert_not_called() 255 mock_update.assert_called_with(RESULT_FAILED_TEST, group) 256 mock_print.assert_called_with(RESULT_FAILED_TEST) 257 # Test with new Group 258 mock_title.reset_mock() 259 self.rr.process_test_result(RESULT_PASSED_TEST_MODULE_2) 260 self.assertTrue('someTestModule2' in self.rr.runners['someTestRunner']) 261 mock_title.assert_called_with(RESULT_PASSED_TEST_MODULE_2) 262 # Test with new Runner 263 mock_title.reset_mock() 264 self.rr.process_test_result(RESULT_PASSED_TEST_RUNNER_2_NO_MODULE) 265 self.assertTrue('someTestRunner2' in self.rr.runners) 266 mock_title.assert_called_with(RESULT_PASSED_TEST_RUNNER_2_NO_MODULE) 267 268 def test_print_result_run_name(self): 269 """Test print run name function in print_result method.""" 270 try: 271 rr = result_reporter.ResultReporter() 272 capture_output = StringIO() 273 sys.stdout = capture_output 274 run_name = 'com.android.UnitTests' 275 rr._print_result(test_runner_base.TestResult( 276 runner_name='runner_name', 277 group_name='someTestModule', 278 test_name='someClassName#someTestName', 279 status=test_runner_base.FAILED_STATUS, 280 details='someTrace', 281 test_count=2, 282 test_time='(2h44m36.402s)', 283 runner_total=None, 284 group_total=2, 285 additional_info={}, 286 test_run_name=run_name 287 )) 288 # Make sure run name in the first line. 289 capture_output_str = capture_output.getvalue().strip() 290 self.assertTrue(run_name in capture_output_str.split('\n')[0]) 291 run_name2 = 'com.android.UnitTests2' 292 capture_output = StringIO() 293 sys.stdout = capture_output 294 rr._print_result(test_runner_base.TestResult( 295 runner_name='runner_name', 296 group_name='someTestModule', 297 test_name='someClassName#someTestName', 298 status=test_runner_base.FAILED_STATUS, 299 details='someTrace', 300 test_count=2, 301 test_time='(2h43m36.402s)', 302 runner_total=None, 303 group_total=2, 304 additional_info={}, 305 test_run_name=run_name2 306 )) 307 # Make sure run name in the first line. 308 capture_output_str = capture_output.getvalue().strip() 309 self.assertTrue(run_name2 in capture_output_str.split('\n')[0]) 310 finally: 311 sys.stdout = sys.__stdout__ 312 313 def test_register_unsupported_runner(self): 314 """Test register_unsupported_runner method.""" 315 self.rr.register_unsupported_runner('NotSupported') 316 runner = self.rr.runners['NotSupported'] 317 self.assertIsNotNone(runner) 318 self.assertEquals(runner, result_reporter.UNSUPPORTED_FLAG) 319 320 def test_update_stats_passed(self): 321 """Test _update_stats method.""" 322 # Passed Test 323 group = result_reporter.RunStat() 324 self.rr._update_stats(RESULT_PASSED_TEST, group) 325 self.assertEquals(self.rr.run_stats.passed, 1) 326 self.assertEquals(self.rr.run_stats.failed, 0) 327 self.assertEquals(self.rr.run_stats.run_errors, False) 328 self.assertEquals(self.rr.failed_tests, []) 329 self.assertEquals(group.passed, 1) 330 self.assertEquals(group.failed, 0) 331 self.assertEquals(group.ignored, 0) 332 self.assertEquals(group.run_errors, False) 333 # Passed Test New Group 334 group2 = result_reporter.RunStat() 335 self.rr._update_stats(RESULT_PASSED_TEST_MODULE_2, group2) 336 self.assertEquals(self.rr.run_stats.passed, 2) 337 self.assertEquals(self.rr.run_stats.failed, 0) 338 self.assertEquals(self.rr.run_stats.run_errors, False) 339 self.assertEquals(self.rr.failed_tests, []) 340 self.assertEquals(group2.passed, 1) 341 self.assertEquals(group2.failed, 0) 342 self.assertEquals(group.ignored, 0) 343 self.assertEquals(group2.run_errors, False) 344 345 def test_update_stats_failed(self): 346 """Test _update_stats method.""" 347 # Passed Test 348 group = result_reporter.RunStat() 349 self.rr._update_stats(RESULT_PASSED_TEST, group) 350 # Passed Test New Group 351 group2 = result_reporter.RunStat() 352 self.rr._update_stats(RESULT_PASSED_TEST_MODULE_2, group2) 353 # Failed Test Old Group 354 self.rr._update_stats(RESULT_FAILED_TEST, group) 355 self.assertEquals(self.rr.run_stats.passed, 2) 356 self.assertEquals(self.rr.run_stats.failed, 1) 357 self.assertEquals(self.rr.run_stats.run_errors, False) 358 self.assertEquals(self.rr.failed_tests, [RESULT_FAILED_TEST.test_name]) 359 self.assertEquals(group.passed, 1) 360 self.assertEquals(group.failed, 1) 361 self.assertEquals(group.ignored, 0) 362 self.assertEquals(group.total, 2) 363 self.assertEquals(group2.total, 1) 364 self.assertEquals(group.run_errors, False) 365 # Test Run Failure 366 self.rr._update_stats(RESULT_RUN_FAILURE, group) 367 self.assertEquals(self.rr.run_stats.passed, 2) 368 self.assertEquals(self.rr.run_stats.failed, 1) 369 self.assertEquals(self.rr.run_stats.run_errors, True) 370 self.assertEquals(self.rr.failed_tests, [RESULT_FAILED_TEST.test_name]) 371 self.assertEquals(group.passed, 1) 372 self.assertEquals(group.failed, 1) 373 self.assertEquals(group.ignored, 0) 374 self.assertEquals(group.run_errors, True) 375 self.assertEquals(group2.run_errors, False) 376 # Invocation Failure 377 self.rr._update_stats(RESULT_INVOCATION_FAILURE, group) 378 self.assertEquals(self.rr.run_stats.passed, 2) 379 self.assertEquals(self.rr.run_stats.failed, 1) 380 self.assertEquals(self.rr.run_stats.run_errors, True) 381 self.assertEquals(self.rr.failed_tests, [RESULT_FAILED_TEST.test_name]) 382 self.assertEquals(group.passed, 1) 383 self.assertEquals(group.failed, 1) 384 self.assertEquals(group.ignored, 0) 385 self.assertEquals(group.run_errors, True) 386 387 def test_update_stats_ignored_and_assumption_failure(self): 388 """Test _update_stats method.""" 389 # Passed Test 390 group = result_reporter.RunStat() 391 self.rr._update_stats(RESULT_PASSED_TEST, group) 392 # Passed Test New Group 393 group2 = result_reporter.RunStat() 394 self.rr._update_stats(RESULT_PASSED_TEST_MODULE_2, group2) 395 # Failed Test Old Group 396 self.rr._update_stats(RESULT_FAILED_TEST, group) 397 # Test Run Failure 398 self.rr._update_stats(RESULT_RUN_FAILURE, group) 399 # Invocation Failure 400 self.rr._update_stats(RESULT_INVOCATION_FAILURE, group) 401 # Ignored Test 402 self.rr._update_stats(RESULT_IGNORED_TEST, group) 403 self.assertEquals(self.rr.run_stats.passed, 2) 404 self.assertEquals(self.rr.run_stats.failed, 1) 405 self.assertEquals(self.rr.run_stats.run_errors, True) 406 self.assertEquals(self.rr.failed_tests, [RESULT_FAILED_TEST.test_name]) 407 self.assertEquals(group.passed, 1) 408 self.assertEquals(group.failed, 1) 409 self.assertEquals(group.ignored, 1) 410 self.assertEquals(group.run_errors, True) 411 # 2nd Ignored Test 412 self.rr._update_stats(RESULT_IGNORED_TEST, group) 413 self.assertEquals(self.rr.run_stats.passed, 2) 414 self.assertEquals(self.rr.run_stats.failed, 1) 415 self.assertEquals(self.rr.run_stats.run_errors, True) 416 self.assertEquals(self.rr.failed_tests, [RESULT_FAILED_TEST.test_name]) 417 self.assertEquals(group.passed, 1) 418 self.assertEquals(group.failed, 1) 419 self.assertEquals(group.ignored, 2) 420 self.assertEquals(group.run_errors, True) 421 422 # Assumption_Failure test 423 self.rr._update_stats(RESULT_ASSUMPTION_FAILED_TEST, group) 424 self.assertEquals(group.assumption_failed, 1) 425 # 2nd Assumption_Failure test 426 self.rr._update_stats(RESULT_ASSUMPTION_FAILED_TEST, group) 427 self.assertEquals(group.assumption_failed, 2) 428 429 def test_print_summary_ret_val(self): 430 """Test print_summary method's return value.""" 431 # PASS Case 432 self.rr.process_test_result(RESULT_PASSED_TEST) 433 self.assertEquals(0, self.rr.print_summary()) 434 # PASS Case + Fail Case 435 self.rr.process_test_result(RESULT_FAILED_TEST) 436 self.assertNotEqual(0, self.rr.print_summary()) 437 # PASS Case + Fail Case + PASS Case 438 self.rr.process_test_result(RESULT_PASSED_TEST_MODULE_2) 439 self.assertNotEqual(0, self.rr.print_summary()) 440 441 def test_print_summary_ret_val_err_stat(self): 442 """Test print_summary method's return value.""" 443 # PASS Case 444 self.rr.process_test_result(RESULT_PASSED_TEST) 445 self.assertEquals(0, self.rr.print_summary()) 446 # PASS Case + Fail Case 447 self.rr.process_test_result(RESULT_RUN_FAILURE) 448 self.assertNotEqual(0, self.rr.print_summary()) 449 # PASS Case + Fail Case + PASS Case 450 self.rr.process_test_result(RESULT_PASSED_TEST_MODULE_2) 451 self.assertNotEqual(0, self.rr.print_summary()) 452 453 def test_update_perf_info(self): 454 """Test update_perf_info method.""" 455 group = result_reporter.RunStat() 456 # 1. Test PerfInfo after RESULT_PERF01_TEST01 457 # _update_stats() will call _update_perf_info() 458 self.rr._update_stats(RESULT_PERF01_TEST01, group) 459 correct_perf_info = [] 460 # trim the time form 10001.10001 to 10001 461 trim_perf01_test01 = {u'repetition_index': u'0', u'cpu_time': u'10001', 462 u'name': u'perfName01', 463 u'repetitions': u'0', u'run_type': u'iteration', 464 u'label': u'2123', u'threads': u'1', 465 u'time_unit': u'ns', u'iterations': u'1001', 466 u'run_name': u'perfName01', 467 u'real_time': u'11001', 468 'test_name': 'somePerfClass01#perfName01'} 469 correct_perf_info.append(trim_perf01_test01) 470 self.assertEquals(self.rr.run_stats.perf_info.perf_info, 471 correct_perf_info) 472 # 2. Test PerfInfo after RESULT_PERF01_TEST01 473 self.rr._update_stats(RESULT_PERF01_TEST02, group) 474 trim_perf01_test02 = {u'repetition_index': u'0', u'cpu_time': u'10002', 475 u'name': u'perfName02', 476 u'repetitions': u'0', u'run_type': u'iteration', 477 u'label': u'2123', u'threads': u'1', 478 u'time_unit': u'ns', u'iterations': u'1002', 479 u'run_name': u'perfName02', 480 u'real_time': u'11002', 481 'test_name': 'somePerfClass01#perfName02'} 482 correct_perf_info.append(trim_perf01_test02) 483 self.assertEquals(self.rr.run_stats.perf_info.perf_info, 484 correct_perf_info) 485 # 3. Test PerfInfo after RESULT_PERF02_TEST01 486 self.rr._update_stats(RESULT_PERF02_TEST01, group) 487 trim_perf02_test01 = {u'repetition_index': u'0', u'cpu_time': u'20001', 488 u'name': u'perfName11', 489 u'repetitions': u'0', u'run_type': u'iteration', 490 u'label': u'2123', u'threads': u'1', 491 u'time_unit': u'ns', u'iterations': u'2001', 492 u'run_name': u'perfName11', 493 u'real_time': u'210001', 494 'test_name': 'somePerfClass02#perfName11'} 495 correct_perf_info.append(trim_perf02_test01) 496 self.assertEquals(self.rr.run_stats.perf_info.perf_info, 497 correct_perf_info) 498 # 4. Test PerfInfo after RESULT_PERF01_TEST03_NO_CPU_TIME 499 self.rr._update_stats(RESULT_PERF01_TEST03_NO_CPU_TIME, group) 500 # Nothing added since RESULT_PERF01_TEST03_NO_CPU_TIME lack of cpu_time 501 self.assertEquals(self.rr.run_stats.perf_info.perf_info, 502 correct_perf_info) 503 504 def test_classify_perf_info(self): 505 """Test _classify_perf_info method.""" 506 group = result_reporter.RunStat() 507 self.rr._update_stats(RESULT_PERF01_TEST01, group) 508 self.rr._update_stats(RESULT_PERF01_TEST02, group) 509 self.rr._update_stats(RESULT_PERF02_TEST01, group) 510 # trim the time form 10001.10001 to 10001 511 trim_perf01_test01 = {u'repetition_index': u'0', u'cpu_time': u'10001', 512 u'name': u'perfName01', 513 u'repetitions': u'0', u'run_type': u'iteration', 514 u'label': u'2123', u'threads': u'1', 515 u'time_unit': u'ns', u'iterations': u'1001', 516 u'run_name': u'perfName01', 517 u'real_time': u'11001', 518 'test_name': 'somePerfClass01#perfName01'} 519 trim_perf01_test02 = {u'repetition_index': u'0', u'cpu_time': u'10002', 520 u'name': u'perfName02', 521 u'repetitions': u'0', u'run_type': u'iteration', 522 u'label': u'2123', u'threads': u'1', 523 u'time_unit': u'ns', u'iterations': u'1002', 524 u'run_name': u'perfName02', 525 u'real_time': u'11002', 526 'test_name': 'somePerfClass01#perfName02'} 527 trim_perf02_test01 = {u'repetition_index': u'0', u'cpu_time': u'20001', 528 u'name': u'perfName11', 529 u'repetitions': u'0', u'run_type': u'iteration', 530 u'label': u'2123', u'threads': u'1', 531 u'time_unit': u'ns', u'iterations': u'2001', 532 u'run_name': u'perfName11', 533 u'real_time': u'210001', 534 'test_name': 'somePerfClass02#perfName11'} 535 correct_classify_perf_info = {"somePerfClass01":[trim_perf01_test01, 536 trim_perf01_test02], 537 "somePerfClass02":[trim_perf02_test01]} 538 classify_perf_info, max_len = self.rr.run_stats.perf_info._classify_perf_info() 539 correct_max_len = {'real_time': 6, 'cpu_time': 5, 'name': 10, 540 'iterations': 9, 'time_unit': 2} 541 self.assertEquals(max_len, correct_max_len) 542 self.assertEquals(classify_perf_info, correct_classify_perf_info) 543 544 545if __name__ == '__main__': 546 unittest.main() 547