1#!/usr/bin/env python3 2# 3# Copyright 2018, The Android Open Source Project 4# 5# Licensed under the Apache License, Version 2.0 (the "License"); 6# you may not use this file except in compliance with the License. 7# You may obtain a copy of the License at 8# 9# http://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an "AS IS" BASIS, 13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14# See the License for the specific language governing permissions and 15# limitations under the License. 16 17"""Unittests for result_reporter.""" 18 19# pylint: disable=line-too-long 20 21import sys 22import unittest 23 24from io import StringIO 25from unittest import mock 26 27import result_reporter 28 29from test_runners import test_runner_base 30 31 32RESULT_PASSED_TEST = test_runner_base.TestResult( 33 runner_name='someTestRunner', 34 group_name='someTestModule', 35 test_name='someClassName#sostName', 36 status=test_runner_base.PASSED_STATUS, 37 details=None, 38 test_count=1, 39 test_time='(10ms)', 40 runner_total=None, 41 group_total=2, 42 additional_info={}, 43 test_run_name='com.android.UnitTests' 44) 45 46RESULT_PASSED_TEST_MODULE_2 = test_runner_base.TestResult( 47 runner_name='someTestRunner', 48 group_name='someTestModule2', 49 test_name='someClassName#sostName', 50 status=test_runner_base.PASSED_STATUS, 51 details=None, 52 test_count=1, 53 test_time='(10ms)', 54 runner_total=None, 55 group_total=2, 56 additional_info={}, 57 test_run_name='com.android.UnitTests' 58) 59 60RESULT_PASSED_TEST_RUNNER_2_NO_MODULE = test_runner_base.TestResult( 61 runner_name='someTestRunner2', 62 group_name=None, 63 test_name='someClassName#sostName', 64 status=test_runner_base.PASSED_STATUS, 65 details=None, 66 test_count=1, 67 test_time='(10ms)', 68 runner_total=None, 69 group_total=2, 70 additional_info={}, 71 test_run_name='com.android.UnitTests' 72) 73 74RESULT_FAILED_TEST = test_runner_base.TestResult( 75 runner_name='someTestRunner', 76 group_name='someTestModule', 77 test_name='someClassName2#sestName2', 78 status=test_runner_base.FAILED_STATUS, 79 details='someTrace', 80 test_count=1, 81 test_time='', 82 runner_total=None, 83 group_total=2, 84 additional_info={}, 85 test_run_name='com.android.UnitTests' 86) 87 88RESULT_RUN_FAILURE = test_runner_base.TestResult( 89 runner_name='someTestRunner', 90 group_name='someTestModule', 91 test_name='someClassName#sostName', 92 status=test_runner_base.ERROR_STATUS, 93 details='someRunFailureReason', 94 test_count=1, 95 test_time='', 96 runner_total=None, 97 group_total=2, 98 additional_info={}, 99 test_run_name='com.android.UnitTests' 100) 101 102RESULT_INVOCATION_FAILURE = test_runner_base.TestResult( 103 runner_name='someTestRunner', 104 group_name=None, 105 test_name=None, 106 status=test_runner_base.ERROR_STATUS, 107 details='someInvocationFailureReason', 108 test_count=1, 109 test_time='', 110 runner_total=None, 111 group_total=None, 112 additional_info={}, 113 test_run_name='com.android.UnitTests' 114) 115 116RESULT_IGNORED_TEST = test_runner_base.TestResult( 117 runner_name='someTestRunner', 118 group_name='someTestModule', 119 test_name='someClassName#sostName', 120 status=test_runner_base.IGNORED_STATUS, 121 details=None, 122 test_count=1, 123 test_time='(10ms)', 124 runner_total=None, 125 group_total=2, 126 additional_info={}, 127 test_run_name='com.android.UnitTests' 128) 129 130RESULT_ASSUMPTION_FAILED_TEST = test_runner_base.TestResult( 131 runner_name='someTestRunner', 132 group_name='someTestModule', 133 test_name='someClassName#sostName', 134 status=test_runner_base.ASSUMPTION_FAILED, 135 details=None, 136 test_count=1, 137 test_time='(10ms)', 138 runner_total=None, 139 group_total=2, 140 additional_info={}, 141 test_run_name='com.android.UnitTests' 142) 143 144ADDITIONAL_INFO_PERF01_TEST01 = {u'repetition_index': u'0', 145 u'cpu_time': u'10001.10001', 146 u'name': u'perfName01', 147 u'repetitions': u'0', u'run_type': u'iteration', 148 u'label': u'2123', u'threads': u'1', 149 u'time_unit': u'ns', u'iterations': u'1001', 150 u'run_name': u'perfName01', 151 u'real_time': u'11001.11001'} 152 153RESULT_PERF01_TEST01 = test_runner_base.TestResult( 154 runner_name='someTestRunner', 155 group_name='someTestModule', 156 test_name='somePerfClass01#perfName01', 157 status=test_runner_base.PASSED_STATUS, 158 details=None, 159 test_count=1, 160 test_time='(10ms)', 161 runner_total=None, 162 group_total=2, 163 additional_info=ADDITIONAL_INFO_PERF01_TEST01, 164 test_run_name='com.android.UnitTests' 165) 166 167RESULT_PERF01_TEST02 = test_runner_base.TestResult( 168 runner_name='someTestRunner', 169 group_name='someTestModule', 170 test_name='somePerfClass01#perfName02', 171 status=test_runner_base.PASSED_STATUS, 172 details=None, 173 test_count=1, 174 test_time='(10ms)', 175 runner_total=None, 176 group_total=2, 177 additional_info={u'repetition_index': u'0', u'cpu_time': u'10002.10002', 178 u'name': u'perfName02', 179 u'repetitions': u'0', u'run_type': u'iteration', 180 u'label': u'2123', u'threads': u'1', 181 u'time_unit': u'ns', u'iterations': u'1002', 182 u'run_name': u'perfName02', 183 u'real_time': u'11002.11002'}, 184 test_run_name='com.android.UnitTests' 185) 186 187RESULT_PERF01_TEST03_NO_CPU_TIME = test_runner_base.TestResult( 188 runner_name='someTestRunner', 189 group_name='someTestModule', 190 test_name='somePerfClass01#perfName03', 191 status=test_runner_base.PASSED_STATUS, 192 details=None, 193 test_count=1, 194 test_time='(10ms)', 195 runner_total=None, 196 group_total=2, 197 additional_info={u'repetition_index': u'0', 198 u'name': u'perfName03', 199 u'repetitions': u'0', u'run_type': u'iteration', 200 u'label': u'2123', u'threads': u'1', 201 u'time_unit': u'ns', u'iterations': u'1003', 202 u'run_name': u'perfName03', 203 u'real_time': u'11003.11003'}, 204 test_run_name='com.android.UnitTests' 205) 206 207RESULT_PERF02_TEST01 = test_runner_base.TestResult( 208 runner_name='someTestRunner', 209 group_name='someTestModule', 210 test_name='somePerfClass02#perfName11', 211 status=test_runner_base.PASSED_STATUS, 212 details=None, 213 test_count=1, 214 test_time='(10ms)', 215 runner_total=None, 216 group_total=2, 217 additional_info={u'repetition_index': u'0', u'cpu_time': u'20001.20001', 218 u'name': u'perfName11', 219 u'repetitions': u'0', u'run_type': u'iteration', 220 u'label': u'2123', u'threads': u'1', 221 u'time_unit': u'ns', u'iterations': u'2001', 222 u'run_name': u'perfName11', 223 u'real_time': u'210001.21001'}, 224 test_run_name='com.android.UnitTests' 225) 226 227#pylint: disable=protected-access 228#pylint: disable=invalid-name 229class ResultReporterUnittests(unittest.TestCase): 230 """Unit tests for result_reporter.py""" 231 232 def setUp(self): 233 self.rr = result_reporter.ResultReporter() 234 235 def tearDown(self): 236 mock.patch.stopall() 237 238 @mock.patch.object(result_reporter.ResultReporter, '_print_group_title') 239 @mock.patch.object(result_reporter.ResultReporter, '_update_stats') 240 @mock.patch.object(result_reporter.ResultReporter, '_print_result') 241 def test_process_test_result(self, mock_print, mock_update, mock_title): 242 """Test process_test_result method.""" 243 # Passed Test 244 self.assertTrue('someTestRunner' not in self.rr.runners) 245 self.rr.process_test_result(RESULT_PASSED_TEST) 246 self.assertTrue('someTestRunner' in self.rr.runners) 247 group = self.rr.runners['someTestRunner'].get('someTestModule') 248 self.assertIsNotNone(group) 249 mock_title.assert_called_with(RESULT_PASSED_TEST) 250 mock_update.assert_called_with(RESULT_PASSED_TEST, group) 251 mock_print.assert_called_with(RESULT_PASSED_TEST) 252 # Failed Test 253 mock_title.reset_mock() 254 self.rr.process_test_result(RESULT_FAILED_TEST) 255 mock_title.assert_not_called() 256 mock_update.assert_called_with(RESULT_FAILED_TEST, group) 257 mock_print.assert_called_with(RESULT_FAILED_TEST) 258 # Test with new Group 259 mock_title.reset_mock() 260 self.rr.process_test_result(RESULT_PASSED_TEST_MODULE_2) 261 self.assertTrue('someTestModule2' in self.rr.runners['someTestRunner']) 262 mock_title.assert_called_with(RESULT_PASSED_TEST_MODULE_2) 263 # Test with new Runner 264 mock_title.reset_mock() 265 self.rr.process_test_result(RESULT_PASSED_TEST_RUNNER_2_NO_MODULE) 266 self.assertTrue('someTestRunner2' in self.rr.runners) 267 mock_title.assert_called_with(RESULT_PASSED_TEST_RUNNER_2_NO_MODULE) 268 269 def test_print_result_run_name(self): 270 """Test print run name function in print_result method.""" 271 try: 272 rr = result_reporter.ResultReporter() 273 capture_output = StringIO() 274 sys.stdout = capture_output 275 run_name = 'com.android.UnitTests' 276 rr._print_result(test_runner_base.TestResult( 277 runner_name='runner_name', 278 group_name='someTestModule', 279 test_name='someClassName#someTestName', 280 status=test_runner_base.FAILED_STATUS, 281 details='someTrace', 282 test_count=2, 283 test_time='(2h44m36.402s)', 284 runner_total=None, 285 group_total=2, 286 additional_info={}, 287 test_run_name=run_name 288 )) 289 # Make sure run name in the first line. 290 capture_output_str = capture_output.getvalue().strip() 291 self.assertTrue(run_name in capture_output_str.split('\n')[0]) 292 run_name2 = 'com.android.UnitTests2' 293 capture_output = StringIO() 294 sys.stdout = capture_output 295 rr._print_result(test_runner_base.TestResult( 296 runner_name='runner_name', 297 group_name='someTestModule', 298 test_name='someClassName#someTestName', 299 status=test_runner_base.FAILED_STATUS, 300 details='someTrace', 301 test_count=2, 302 test_time='(2h43m36.402s)', 303 runner_total=None, 304 group_total=2, 305 additional_info={}, 306 test_run_name=run_name2 307 )) 308 # Make sure run name in the first line. 309 capture_output_str = capture_output.getvalue().strip() 310 self.assertTrue(run_name2 in capture_output_str.split('\n')[0]) 311 finally: 312 sys.stdout = sys.__stdout__ 313 314 def test_register_unsupported_runner(self): 315 """Test register_unsupported_runner method.""" 316 self.rr.register_unsupported_runner('NotSupported') 317 runner = self.rr.runners['NotSupported'] 318 self.assertIsNotNone(runner) 319 self.assertEqual(runner, result_reporter.UNSUPPORTED_FLAG) 320 321 def test_update_stats_passed(self): 322 """Test _update_stats method.""" 323 # Passed Test 324 group = result_reporter.RunStat() 325 self.rr._update_stats(RESULT_PASSED_TEST, group) 326 self.assertEqual(self.rr.run_stats.passed, 1) 327 self.assertEqual(self.rr.run_stats.failed, 0) 328 self.assertEqual(self.rr.run_stats.run_errors, False) 329 self.assertEqual(self.rr.failed_tests, []) 330 self.assertEqual(group.passed, 1) 331 self.assertEqual(group.failed, 0) 332 self.assertEqual(group.ignored, 0) 333 self.assertEqual(group.run_errors, False) 334 # Passed Test New Group 335 group2 = result_reporter.RunStat() 336 self.rr._update_stats(RESULT_PASSED_TEST_MODULE_2, group2) 337 self.assertEqual(self.rr.run_stats.passed, 2) 338 self.assertEqual(self.rr.run_stats.failed, 0) 339 self.assertEqual(self.rr.run_stats.run_errors, False) 340 self.assertEqual(self.rr.failed_tests, []) 341 self.assertEqual(group2.passed, 1) 342 self.assertEqual(group2.failed, 0) 343 self.assertEqual(group.ignored, 0) 344 self.assertEqual(group2.run_errors, False) 345 346 def test_update_stats_failed(self): 347 """Test _update_stats method.""" 348 # Passed Test 349 group = result_reporter.RunStat() 350 self.rr._update_stats(RESULT_PASSED_TEST, group) 351 # Passed Test New Group 352 group2 = result_reporter.RunStat() 353 self.rr._update_stats(RESULT_PASSED_TEST_MODULE_2, group2) 354 # Failed Test Old Group 355 self.rr._update_stats(RESULT_FAILED_TEST, group) 356 self.assertEqual(self.rr.run_stats.passed, 2) 357 self.assertEqual(self.rr.run_stats.failed, 1) 358 self.assertEqual(self.rr.run_stats.run_errors, False) 359 self.assertEqual(self.rr.failed_tests, [RESULT_FAILED_TEST.test_name]) 360 self.assertEqual(group.passed, 1) 361 self.assertEqual(group.failed, 1) 362 self.assertEqual(group.ignored, 0) 363 self.assertEqual(group.total, 2) 364 self.assertEqual(group2.total, 1) 365 self.assertEqual(group.run_errors, False) 366 # Test Run Failure 367 self.rr._update_stats(RESULT_RUN_FAILURE, group) 368 self.assertEqual(self.rr.run_stats.passed, 2) 369 self.assertEqual(self.rr.run_stats.failed, 1) 370 self.assertEqual(self.rr.run_stats.run_errors, True) 371 self.assertEqual(self.rr.failed_tests, [RESULT_FAILED_TEST.test_name]) 372 self.assertEqual(group.passed, 1) 373 self.assertEqual(group.failed, 1) 374 self.assertEqual(group.ignored, 0) 375 self.assertEqual(group.run_errors, True) 376 self.assertEqual(group2.run_errors, False) 377 # Invocation Failure 378 self.rr._update_stats(RESULT_INVOCATION_FAILURE, group) 379 self.assertEqual(self.rr.run_stats.passed, 2) 380 self.assertEqual(self.rr.run_stats.failed, 1) 381 self.assertEqual(self.rr.run_stats.run_errors, True) 382 self.assertEqual(self.rr.failed_tests, [RESULT_FAILED_TEST.test_name]) 383 self.assertEqual(group.passed, 1) 384 self.assertEqual(group.failed, 1) 385 self.assertEqual(group.ignored, 0) 386 self.assertEqual(group.run_errors, True) 387 388 def test_update_stats_ignored_and_assumption_failure(self): 389 """Test _update_stats method.""" 390 # Passed Test 391 group = result_reporter.RunStat() 392 self.rr._update_stats(RESULT_PASSED_TEST, group) 393 # Passed Test New Group 394 group2 = result_reporter.RunStat() 395 self.rr._update_stats(RESULT_PASSED_TEST_MODULE_2, group2) 396 # Failed Test Old Group 397 self.rr._update_stats(RESULT_FAILED_TEST, group) 398 # Test Run Failure 399 self.rr._update_stats(RESULT_RUN_FAILURE, group) 400 # Invocation Failure 401 self.rr._update_stats(RESULT_INVOCATION_FAILURE, group) 402 # Ignored Test 403 self.rr._update_stats(RESULT_IGNORED_TEST, group) 404 self.assertEqual(self.rr.run_stats.passed, 2) 405 self.assertEqual(self.rr.run_stats.failed, 1) 406 self.assertEqual(self.rr.run_stats.run_errors, True) 407 self.assertEqual(self.rr.failed_tests, [RESULT_FAILED_TEST.test_name]) 408 self.assertEqual(group.passed, 1) 409 self.assertEqual(group.failed, 1) 410 self.assertEqual(group.ignored, 1) 411 self.assertEqual(group.run_errors, True) 412 # 2nd Ignored Test 413 self.rr._update_stats(RESULT_IGNORED_TEST, group) 414 self.assertEqual(self.rr.run_stats.passed, 2) 415 self.assertEqual(self.rr.run_stats.failed, 1) 416 self.assertEqual(self.rr.run_stats.run_errors, True) 417 self.assertEqual(self.rr.failed_tests, [RESULT_FAILED_TEST.test_name]) 418 self.assertEqual(group.passed, 1) 419 self.assertEqual(group.failed, 1) 420 self.assertEqual(group.ignored, 2) 421 self.assertEqual(group.run_errors, True) 422 423 # Assumption_Failure test 424 self.rr._update_stats(RESULT_ASSUMPTION_FAILED_TEST, group) 425 self.assertEqual(group.assumption_failed, 1) 426 # 2nd Assumption_Failure test 427 self.rr._update_stats(RESULT_ASSUMPTION_FAILED_TEST, group) 428 self.assertEqual(group.assumption_failed, 2) 429 430 def test_print_summary_ret_val(self): 431 """Test print_summary method's return value.""" 432 # PASS Case 433 self.rr.process_test_result(RESULT_PASSED_TEST) 434 self.assertEqual(0, self.rr.print_summary()) 435 # PASS Case + Fail Case 436 self.rr.process_test_result(RESULT_FAILED_TEST) 437 self.assertNotEqual(0, self.rr.print_summary()) 438 # PASS Case + Fail Case + PASS Case 439 self.rr.process_test_result(RESULT_PASSED_TEST_MODULE_2) 440 self.assertNotEqual(0, self.rr.print_summary()) 441 442 def test_print_summary_ret_val_err_stat(self): 443 """Test print_summary method's return value.""" 444 # PASS Case 445 self.rr.process_test_result(RESULT_PASSED_TEST) 446 self.assertEqual(0, self.rr.print_summary()) 447 # PASS Case + Fail Case 448 self.rr.process_test_result(RESULT_RUN_FAILURE) 449 self.assertNotEqual(0, self.rr.print_summary()) 450 # PASS Case + Fail Case + PASS Case 451 self.rr.process_test_result(RESULT_PASSED_TEST_MODULE_2) 452 self.assertNotEqual(0, self.rr.print_summary()) 453 454 def test_update_perf_info(self): 455 """Test update_perf_info method.""" 456 group = result_reporter.RunStat() 457 # 1. Test PerfInfo after RESULT_PERF01_TEST01 458 # _update_stats() will call _update_perf_info() 459 self.rr._update_stats(RESULT_PERF01_TEST01, group) 460 correct_perf_info = [] 461 # trim the time form 10001.10001 to 10001 462 trim_perf01_test01 = {u'repetition_index': u'0', u'cpu_time': u'10001', 463 u'name': u'perfName01', 464 u'repetitions': u'0', u'run_type': u'iteration', 465 u'label': u'2123', u'threads': u'1', 466 u'time_unit': u'ns', u'iterations': u'1001', 467 u'run_name': u'perfName01', 468 u'real_time': u'11001', 469 'test_name': 'somePerfClass01#perfName01'} 470 correct_perf_info.append(trim_perf01_test01) 471 self.assertEqual(self.rr.run_stats.perf_info.perf_info, 472 correct_perf_info) 473 # 2. Test PerfInfo after RESULT_PERF01_TEST01 474 self.rr._update_stats(RESULT_PERF01_TEST02, group) 475 trim_perf01_test02 = {u'repetition_index': u'0', u'cpu_time': u'10002', 476 u'name': u'perfName02', 477 u'repetitions': u'0', u'run_type': u'iteration', 478 u'label': u'2123', u'threads': u'1', 479 u'time_unit': u'ns', u'iterations': u'1002', 480 u'run_name': u'perfName02', 481 u'real_time': u'11002', 482 'test_name': 'somePerfClass01#perfName02'} 483 correct_perf_info.append(trim_perf01_test02) 484 self.assertEqual(self.rr.run_stats.perf_info.perf_info, 485 correct_perf_info) 486 # 3. Test PerfInfo after RESULT_PERF02_TEST01 487 self.rr._update_stats(RESULT_PERF02_TEST01, group) 488 trim_perf02_test01 = {u'repetition_index': u'0', u'cpu_time': u'20001', 489 u'name': u'perfName11', 490 u'repetitions': u'0', u'run_type': u'iteration', 491 u'label': u'2123', u'threads': u'1', 492 u'time_unit': u'ns', u'iterations': u'2001', 493 u'run_name': u'perfName11', 494 u'real_time': u'210001', 495 'test_name': 'somePerfClass02#perfName11'} 496 correct_perf_info.append(trim_perf02_test01) 497 self.assertEqual(self.rr.run_stats.perf_info.perf_info, 498 correct_perf_info) 499 # 4. Test PerfInfo after RESULT_PERF01_TEST03_NO_CPU_TIME 500 self.rr._update_stats(RESULT_PERF01_TEST03_NO_CPU_TIME, group) 501 # Nothing added since RESULT_PERF01_TEST03_NO_CPU_TIME lack of cpu_time 502 self.assertEqual(self.rr.run_stats.perf_info.perf_info, 503 correct_perf_info) 504 505 def test_classify_perf_info(self): 506 """Test _classify_perf_info method.""" 507 group = result_reporter.RunStat() 508 self.rr._update_stats(RESULT_PERF01_TEST01, group) 509 self.rr._update_stats(RESULT_PERF01_TEST02, group) 510 self.rr._update_stats(RESULT_PERF02_TEST01, group) 511 # trim the time form 10001.10001 to 10001 512 trim_perf01_test01 = {u'repetition_index': u'0', u'cpu_time': u'10001', 513 u'name': u'perfName01', 514 u'repetitions': u'0', u'run_type': u'iteration', 515 u'label': u'2123', u'threads': u'1', 516 u'time_unit': u'ns', u'iterations': u'1001', 517 u'run_name': u'perfName01', 518 u'real_time': u'11001', 519 'test_name': 'somePerfClass01#perfName01'} 520 trim_perf01_test02 = {u'repetition_index': u'0', u'cpu_time': u'10002', 521 u'name': u'perfName02', 522 u'repetitions': u'0', u'run_type': u'iteration', 523 u'label': u'2123', u'threads': u'1', 524 u'time_unit': u'ns', u'iterations': u'1002', 525 u'run_name': u'perfName02', 526 u'real_time': u'11002', 527 'test_name': 'somePerfClass01#perfName02'} 528 trim_perf02_test01 = {u'repetition_index': u'0', u'cpu_time': u'20001', 529 u'name': u'perfName11', 530 u'repetitions': u'0', u'run_type': u'iteration', 531 u'label': u'2123', u'threads': u'1', 532 u'time_unit': u'ns', u'iterations': u'2001', 533 u'run_name': u'perfName11', 534 u'real_time': u'210001', 535 'test_name': 'somePerfClass02#perfName11'} 536 correct_classify_perf_info = {"somePerfClass01":[trim_perf01_test01, 537 trim_perf01_test02], 538 "somePerfClass02":[trim_perf02_test01]} 539 classify_perf_info, max_len = self.rr.run_stats.perf_info._classify_perf_info() 540 correct_max_len = {'real_time': 6, 'cpu_time': 5, 'name': 10, 541 'iterations': 9, 'time_unit': 2} 542 self.assertEqual(max_len, correct_max_len) 543 self.assertEqual(classify_perf_info, correct_classify_perf_info) 544 545 546if __name__ == '__main__': 547 unittest.main() 548