1#!/usr/bin/env python3 2# 3# Copyright 2018, The Android Open Source Project 4# 5# Licensed under the Apache License, Version 2.0 (the "License"); 6# you may not use this file except in compliance with the License. 7# You may obtain a copy of the License at 8# 9# http://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an "AS IS" BASIS, 13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14# See the License for the specific language governing permissions and 15# limitations under the License. 16 17"""Unittests for result_reporter.""" 18 19 20from io import StringIO 21import sys 22import unittest 23from unittest import mock 24 25from atest import atest_configs 26from atest import result_reporter 27from atest.test_runners import test_runner_base 28 29 30RESULT_PASSED_TEST = test_runner_base.TestResult( 31 runner_name='someTestRunner', 32 group_name='someTestModule', 33 test_name='someClassName#sostName', 34 status=test_runner_base.PASSED_STATUS, 35 details=None, 36 test_count=1, 37 test_time='(10ms)', 38 runner_total=None, 39 group_total=2, 40 additional_info={}, 41 test_run_name='com.android.UnitTests', 42) 43 44RESULT_PASSED_TEST_MODULE_2 = test_runner_base.TestResult( 45 runner_name='someTestRunner', 46 group_name='someTestModule2', 47 test_name='someClassName#sostName', 48 status=test_runner_base.PASSED_STATUS, 49 details=None, 50 test_count=1, 51 test_time='(10ms)', 52 runner_total=None, 53 group_total=2, 54 additional_info={}, 55 test_run_name='com.android.UnitTests', 56) 57 58RESULT_PASSED_TEST_RUNNER_2_NO_MODULE = test_runner_base.TestResult( 59 runner_name='someTestRunner2', 60 group_name=None, 61 test_name='someClassName#sostName', 62 status=test_runner_base.PASSED_STATUS, 63 details=None, 64 test_count=1, 65 test_time='(10ms)', 66 runner_total=None, 67 group_total=2, 68 additional_info={}, 69 test_run_name='com.android.UnitTests', 70) 71 72RESULT_FAILED_TEST = test_runner_base.TestResult( 73 runner_name='someTestRunner', 74 group_name='someTestModule', 75 test_name='someClassName2#sestName2', 76 status=test_runner_base.FAILED_STATUS, 77 details='someTrace', 78 test_count=1, 79 test_time='', 80 runner_total=None, 81 group_total=2, 82 additional_info={}, 83 test_run_name='com.android.UnitTests', 84) 85 86RESULT_RUN_FAILURE = test_runner_base.TestResult( 87 runner_name='someTestRunner', 88 group_name='someTestModule', 89 test_name='someClassName#sostName', 90 status=test_runner_base.ERROR_STATUS, 91 details='someRunFailureReason', 92 test_count=1, 93 test_time='', 94 runner_total=None, 95 group_total=2, 96 additional_info={}, 97 test_run_name='com.android.UnitTests', 98) 99 100RESULT_INVOCATION_FAILURE = test_runner_base.TestResult( 101 runner_name='someTestRunner', 102 group_name=None, 103 test_name=None, 104 status=test_runner_base.ERROR_STATUS, 105 details='someInvocationFailureReason', 106 test_count=1, 107 test_time='', 108 runner_total=None, 109 group_total=None, 110 additional_info={}, 111 test_run_name='com.android.UnitTests', 112) 113 114RESULT_IGNORED_TEST = test_runner_base.TestResult( 115 runner_name='someTestRunner', 116 group_name='someTestModule', 117 test_name='someClassName#sostName', 118 status=test_runner_base.IGNORED_STATUS, 119 details=None, 120 test_count=1, 121 test_time='(10ms)', 122 runner_total=None, 123 group_total=2, 124 additional_info={}, 125 test_run_name='com.android.UnitTests', 126) 127 128RESULT_ASSUMPTION_FAILED_TEST = test_runner_base.TestResult( 129 runner_name='someTestRunner', 130 group_name='someTestModule', 131 test_name='someClassName#sostName', 132 status=test_runner_base.ASSUMPTION_FAILED, 133 details=None, 134 test_count=1, 135 test_time='(10ms)', 136 runner_total=None, 137 group_total=2, 138 additional_info={}, 139 test_run_name='com.android.UnitTests', 140) 141 142ADDITIONAL_INFO_PERF01_TEST01 = { 143 'repetition_index': '0', 144 'cpu_time': '10001.10001', 145 'name': 'perfName01', 146 'repetitions': '0', 147 'run_type': 'iteration', 148 'label': '2123', 149 'threads': '1', 150 'time_unit': 'ns', 151 'iterations': '1001', 152 'run_name': 'perfName01', 153 'real_time': '11001.11001', 154} 155 156RESULT_PERF01_TEST01 = test_runner_base.TestResult( 157 runner_name='someTestRunner', 158 group_name='someTestModule', 159 test_name='somePerfClass01#perfName01', 160 status=test_runner_base.PASSED_STATUS, 161 details=None, 162 test_count=1, 163 test_time='(10ms)', 164 runner_total=None, 165 group_total=2, 166 additional_info=ADDITIONAL_INFO_PERF01_TEST01, 167 test_run_name='com.android.UnitTests', 168) 169 170RESULT_PERF01_TEST02 = test_runner_base.TestResult( 171 runner_name='someTestRunner', 172 group_name='someTestModule', 173 test_name='somePerfClass01#perfName02', 174 status=test_runner_base.PASSED_STATUS, 175 details=None, 176 test_count=1, 177 test_time='(10ms)', 178 runner_total=None, 179 group_total=2, 180 additional_info={ 181 'repetition_index': '0', 182 'cpu_time': '10002.10002', 183 'name': 'perfName02', 184 'repetitions': '0', 185 'run_type': 'iteration', 186 'label': '2123', 187 'threads': '1', 188 'time_unit': 'ns', 189 'iterations': '1002', 190 'run_name': 'perfName02', 191 'real_time': '11002.11002', 192 }, 193 test_run_name='com.android.UnitTests', 194) 195 196RESULT_PERF01_TEST03_NO_CPU_TIME = test_runner_base.TestResult( 197 runner_name='someTestRunner', 198 group_name='someTestModule', 199 test_name='somePerfClass01#perfName03', 200 status=test_runner_base.PASSED_STATUS, 201 details=None, 202 test_count=1, 203 test_time='(10ms)', 204 runner_total=None, 205 group_total=2, 206 additional_info={ 207 'repetition_index': '0', 208 'name': 'perfName03', 209 'repetitions': '0', 210 'run_type': 'iteration', 211 'label': '2123', 212 'threads': '1', 213 'time_unit': 'ns', 214 'iterations': '1003', 215 'run_name': 'perfName03', 216 'real_time': '11003.11003', 217 }, 218 test_run_name='com.android.UnitTests', 219) 220 221RESULT_PERF02_TEST01 = test_runner_base.TestResult( 222 runner_name='someTestRunner', 223 group_name='someTestModule', 224 test_name='somePerfClass02#perfName11', 225 status=test_runner_base.PASSED_STATUS, 226 details=None, 227 test_count=1, 228 test_time='(10ms)', 229 runner_total=None, 230 group_total=2, 231 additional_info={ 232 'repetition_index': '0', 233 'cpu_time': '20001.20001', 234 'name': 'perfName11', 235 'repetitions': '0', 236 'run_type': 'iteration', 237 'label': '2123', 238 'threads': '1', 239 'time_unit': 'ns', 240 'iterations': '2001', 241 'run_name': 'perfName11', 242 'real_time': '21001.21001', 243 }, 244 test_run_name='com.android.UnitTests', 245) 246 247 248# pylint: disable=protected-access 249# pylint: disable=invalid-name 250class ResultReporterUnittests(unittest.TestCase): 251 """Unit tests for result_reporter.py""" 252 253 def setUp(self): 254 self.rr = result_reporter.ResultReporter() 255 256 def tearDown(self): 257 mock.patch.stopall() 258 259 @mock.patch.object(result_reporter.ResultReporter, '_print_group_title') 260 @mock.patch.object(result_reporter.ResultReporter, '_update_stats') 261 @mock.patch.object(result_reporter.ResultReporter, '_print_result') 262 def test_process_test_result(self, mock_print, mock_update, mock_title): 263 """Test process_test_result method.""" 264 # Passed Test 265 self.assertTrue('someTestRunner' not in self.rr.runners) 266 self.rr.process_test_result(RESULT_PASSED_TEST) 267 self.assertTrue('someTestRunner' in self.rr.runners) 268 group = self.rr.runners['someTestRunner'].get('someTestModule') 269 self.assertIsNotNone(group) 270 mock_title.assert_called_with(RESULT_PASSED_TEST) 271 mock_update.assert_called_with(RESULT_PASSED_TEST, group) 272 mock_print.assert_called_with(RESULT_PASSED_TEST) 273 # Failed Test 274 mock_title.reset_mock() 275 self.rr.process_test_result(RESULT_FAILED_TEST) 276 mock_title.assert_not_called() 277 mock_update.assert_called_with(RESULT_FAILED_TEST, group) 278 mock_print.assert_called_with(RESULT_FAILED_TEST) 279 # Test with new Group 280 mock_title.reset_mock() 281 self.rr.process_test_result(RESULT_PASSED_TEST_MODULE_2) 282 self.assertTrue('someTestModule2' in self.rr.runners['someTestRunner']) 283 mock_title.assert_called_with(RESULT_PASSED_TEST_MODULE_2) 284 # Test with new Runner 285 mock_title.reset_mock() 286 self.rr.process_test_result(RESULT_PASSED_TEST_RUNNER_2_NO_MODULE) 287 self.assertTrue('someTestRunner2' in self.rr.runners) 288 mock_title.assert_called_with(RESULT_PASSED_TEST_RUNNER_2_NO_MODULE) 289 290 def test_print_result_run_name(self): 291 """Test print run name function in print_result method.""" 292 try: 293 rr = result_reporter.ResultReporter() 294 capture_output = StringIO() 295 sys.stdout = capture_output 296 run_name = 'com.android.UnitTests' 297 rr._print_result( 298 test_runner_base.TestResult( 299 runner_name='runner_name', 300 group_name='someTestModule', 301 test_name='someClassName#someTestName', 302 status=test_runner_base.FAILED_STATUS, 303 details='someTrace', 304 test_count=2, 305 test_time='(2h44m36.402s)', 306 runner_total=None, 307 group_total=2, 308 additional_info={}, 309 test_run_name=run_name, 310 ) 311 ) 312 # Make sure run name in the first line. 313 capture_output_str = capture_output.getvalue().strip() 314 self.assertTrue(run_name in capture_output_str.split('\n')[0]) 315 run_name2 = 'com.android.UnitTests2' 316 capture_output = StringIO() 317 sys.stdout = capture_output 318 rr._print_result( 319 test_runner_base.TestResult( 320 runner_name='runner_name', 321 group_name='someTestModule', 322 test_name='someClassName#someTestName', 323 status=test_runner_base.FAILED_STATUS, 324 details='someTrace', 325 test_count=2, 326 test_time='(2h43m36.402s)', 327 runner_total=None, 328 group_total=2, 329 additional_info={}, 330 test_run_name=run_name2, 331 ) 332 ) 333 # Make sure run name in the first line. 334 capture_output_str = capture_output.getvalue().strip() 335 self.assertTrue(run_name2 in capture_output_str.split('\n')[0]) 336 finally: 337 sys.stdout = sys.__stdout__ 338 339 def test_register_unsupported_runner(self): 340 """Test register_unsupported_runner method.""" 341 self.rr.register_unsupported_runner('NotSupported') 342 runner = self.rr.runners['NotSupported'] 343 self.assertIsNotNone(runner) 344 self.assertEqual(runner, result_reporter.UNSUPPORTED_FLAG) 345 346 def test_update_stats_passed(self): 347 """Test _update_stats method.""" 348 # Passed Test 349 group = result_reporter.RunStat() 350 self.rr._update_stats(RESULT_PASSED_TEST, group) 351 self.assertEqual(self.rr.run_stats.passed, 1) 352 self.assertEqual(self.rr.run_stats.failed, 0) 353 self.assertEqual(self.rr.run_stats.run_errors, False) 354 self.assertEqual(self.rr.failed_tests, []) 355 self.assertEqual(group.passed, 1) 356 self.assertEqual(group.failed, 0) 357 self.assertEqual(group.ignored, 0) 358 self.assertEqual(group.run_errors, False) 359 # Passed Test New Group 360 group2 = result_reporter.RunStat() 361 self.rr._update_stats(RESULT_PASSED_TEST_MODULE_2, group2) 362 self.assertEqual(self.rr.run_stats.passed, 2) 363 self.assertEqual(self.rr.run_stats.failed, 0) 364 self.assertEqual(self.rr.run_stats.run_errors, False) 365 self.assertEqual(self.rr.failed_tests, []) 366 self.assertEqual(group2.passed, 1) 367 self.assertEqual(group2.failed, 0) 368 self.assertEqual(group.ignored, 0) 369 self.assertEqual(group2.run_errors, False) 370 371 def test_update_stats_failed(self): 372 """Test _update_stats method.""" 373 # Passed Test 374 group = result_reporter.RunStat() 375 self.rr._update_stats(RESULT_PASSED_TEST, group) 376 # Passed Test New Group 377 group2 = result_reporter.RunStat() 378 self.rr._update_stats(RESULT_PASSED_TEST_MODULE_2, group2) 379 # Failed Test Old Group 380 self.rr._update_stats(RESULT_FAILED_TEST, group) 381 self.assertEqual(self.rr.run_stats.passed, 2) 382 self.assertEqual(self.rr.run_stats.failed, 1) 383 self.assertEqual(self.rr.run_stats.run_errors, False) 384 self.assertEqual(self.rr.failed_tests, [RESULT_FAILED_TEST.test_name]) 385 self.assertEqual(group.passed, 1) 386 self.assertEqual(group.failed, 1) 387 self.assertEqual(group.ignored, 0) 388 self.assertEqual(group.total, 2) 389 self.assertEqual(group2.total, 1) 390 self.assertEqual(group.run_errors, False) 391 # Test Run Failure 392 self.rr._update_stats(RESULT_RUN_FAILURE, group) 393 self.assertEqual(self.rr.run_stats.passed, 2) 394 self.assertEqual(self.rr.run_stats.failed, 1) 395 self.assertEqual(self.rr.run_stats.run_errors, True) 396 self.assertEqual(self.rr.failed_tests, [RESULT_FAILED_TEST.test_name]) 397 self.assertEqual(group.passed, 1) 398 self.assertEqual(group.failed, 1) 399 self.assertEqual(group.ignored, 0) 400 self.assertEqual(group.run_errors, True) 401 self.assertEqual(group2.run_errors, False) 402 # Invocation Failure 403 self.rr._update_stats(RESULT_INVOCATION_FAILURE, group) 404 self.assertEqual(self.rr.run_stats.passed, 2) 405 self.assertEqual(self.rr.run_stats.failed, 1) 406 self.assertEqual(self.rr.run_stats.run_errors, True) 407 self.assertEqual(self.rr.failed_tests, [RESULT_FAILED_TEST.test_name]) 408 self.assertEqual(group.passed, 1) 409 self.assertEqual(group.failed, 1) 410 self.assertEqual(group.ignored, 0) 411 self.assertEqual(group.run_errors, True) 412 413 def test_update_stats_ignored_and_assumption_failure(self): 414 """Test _update_stats method.""" 415 # Passed Test 416 group = result_reporter.RunStat() 417 self.rr._update_stats(RESULT_PASSED_TEST, group) 418 # Passed Test New Group 419 group2 = result_reporter.RunStat() 420 self.rr._update_stats(RESULT_PASSED_TEST_MODULE_2, group2) 421 # Failed Test Old Group 422 self.rr._update_stats(RESULT_FAILED_TEST, group) 423 # Test Run Failure 424 self.rr._update_stats(RESULT_RUN_FAILURE, group) 425 # Invocation Failure 426 self.rr._update_stats(RESULT_INVOCATION_FAILURE, group) 427 # Ignored Test 428 self.rr._update_stats(RESULT_IGNORED_TEST, group) 429 self.assertEqual(self.rr.run_stats.passed, 2) 430 self.assertEqual(self.rr.run_stats.failed, 1) 431 self.assertEqual(self.rr.run_stats.run_errors, True) 432 self.assertEqual(self.rr.failed_tests, [RESULT_FAILED_TEST.test_name]) 433 self.assertEqual(group.passed, 1) 434 self.assertEqual(group.failed, 1) 435 self.assertEqual(group.ignored, 1) 436 self.assertEqual(group.run_errors, True) 437 # 2nd Ignored Test 438 self.rr._update_stats(RESULT_IGNORED_TEST, group) 439 self.assertEqual(self.rr.run_stats.passed, 2) 440 self.assertEqual(self.rr.run_stats.failed, 1) 441 self.assertEqual(self.rr.run_stats.run_errors, True) 442 self.assertEqual(self.rr.failed_tests, [RESULT_FAILED_TEST.test_name]) 443 self.assertEqual(group.passed, 1) 444 self.assertEqual(group.failed, 1) 445 self.assertEqual(group.ignored, 2) 446 self.assertEqual(group.run_errors, True) 447 448 # Assumption_Failure test 449 self.rr._update_stats(RESULT_ASSUMPTION_FAILED_TEST, group) 450 self.assertEqual(group.assumption_failed, 1) 451 # 2nd Assumption_Failure test 452 self.rr._update_stats(RESULT_ASSUMPTION_FAILED_TEST, group) 453 self.assertEqual(group.assumption_failed, 2) 454 455 def test_print_summary_ret_val(self): 456 """Test print_summary method's return value.""" 457 atest_configs.GLOBAL_ARGS = mock.Mock() 458 atest_configs.GLOBAL_ARGS.aggregate_metric_filter = None 459 460 # PASS Case 461 self.rr.process_test_result(RESULT_PASSED_TEST) 462 self.assertEqual(0, self.rr.print_summary()) 463 # PASS Case + Fail Case 464 self.rr.process_test_result(RESULT_FAILED_TEST) 465 self.assertNotEqual(0, self.rr.print_summary()) 466 # PASS Case + Fail Case + PASS Case 467 self.rr.process_test_result(RESULT_PASSED_TEST_MODULE_2) 468 self.assertNotEqual(0, self.rr.print_summary()) 469 470 def test_print_summary_ret_val_err_stat(self): 471 """Test print_summary method's return value.""" 472 atest_configs.GLOBAL_ARGS = mock.Mock() 473 atest_configs.GLOBAL_ARGS.aggregate_metric_filter = None 474 475 # PASS Case 476 self.rr.process_test_result(RESULT_PASSED_TEST) 477 self.assertEqual(0, self.rr.print_summary()) 478 # PASS Case + Fail Case 479 self.rr.process_test_result(RESULT_RUN_FAILURE) 480 self.assertNotEqual(0, self.rr.print_summary()) 481 # PASS Case + Fail Case + PASS Case 482 self.rr.process_test_result(RESULT_PASSED_TEST_MODULE_2) 483 self.assertNotEqual(0, self.rr.print_summary()) 484 485 def test_update_perf_info(self): 486 """Test update_perf_info method.""" 487 group = result_reporter.RunStat() 488 # 1. Test PerfInfo after RESULT_PERF01_TEST01 489 # _update_stats() will call _update_perf_info() 490 self.rr._update_stats(RESULT_PERF01_TEST01, group) 491 correct_perf_info = [] 492 trim_perf01_test01 = { 493 'repetition_index': '0', 494 'cpu_time': '10001.10001', 495 'name': 'perfName01', 496 'repetitions': '0', 497 'run_type': 'iteration', 498 'label': '2123', 499 'threads': '1', 500 'time_unit': 'ns', 501 'iterations': '1001', 502 'run_name': 'perfName01', 503 'real_time': '11001.11001', 504 'test_name': 'somePerfClass01#perfName01', 505 } 506 correct_perf_info.append(trim_perf01_test01) 507 self.assertEqual(self.rr.run_stats.perf_info.perf_info, correct_perf_info) 508 # 2. Test PerfInfo after RESULT_PERF01_TEST01 509 self.rr._update_stats(RESULT_PERF01_TEST02, group) 510 trim_perf01_test02 = { 511 'repetition_index': '0', 512 'cpu_time': '10002.10002', 513 'name': 'perfName02', 514 'repetitions': '0', 515 'run_type': 'iteration', 516 'label': '2123', 517 'threads': '1', 518 'time_unit': 'ns', 519 'iterations': '1002', 520 'run_name': 'perfName02', 521 'real_time': '11002.11002', 522 'test_name': 'somePerfClass01#perfName02', 523 } 524 correct_perf_info.append(trim_perf01_test02) 525 self.assertEqual(self.rr.run_stats.perf_info.perf_info, correct_perf_info) 526 # 3. Test PerfInfo after RESULT_PERF02_TEST01 527 self.rr._update_stats(RESULT_PERF02_TEST01, group) 528 trim_perf02_test01 = { 529 'repetition_index': '0', 530 'cpu_time': '20001.20001', 531 'name': 'perfName11', 532 'repetitions': '0', 533 'run_type': 'iteration', 534 'label': '2123', 535 'threads': '1', 536 'time_unit': 'ns', 537 'iterations': '2001', 538 'run_name': 'perfName11', 539 'real_time': '21001.21001', 540 'test_name': 'somePerfClass02#perfName11', 541 } 542 correct_perf_info.append(trim_perf02_test01) 543 self.assertEqual(self.rr.run_stats.perf_info.perf_info, correct_perf_info) 544 # 4. Test PerfInfo after RESULT_PERF01_TEST03_NO_CPU_TIME 545 self.rr._update_stats(RESULT_PERF01_TEST03_NO_CPU_TIME, group) 546 # Nothing added since RESULT_PERF01_TEST03_NO_CPU_TIME lack of cpu_time 547 self.assertEqual(self.rr.run_stats.perf_info.perf_info, correct_perf_info) 548 549 def test_classify_perf_info(self): 550 """Test _classify_perf_info method.""" 551 group = result_reporter.RunStat() 552 self.rr._update_stats(RESULT_PERF01_TEST01, group) 553 self.rr._update_stats(RESULT_PERF01_TEST02, group) 554 self.rr._update_stats(RESULT_PERF02_TEST01, group) 555 # trim the time form 10001.10001 to 10001 556 trim_perf01_test01 = { 557 'repetition_index': '0', 558 'cpu_time': '10001.10001', 559 'name': 'perfName01', 560 'repetitions': '0', 561 'run_type': 'iteration', 562 'label': '2123', 563 'threads': '1', 564 'time_unit': 'ns', 565 'iterations': '1001', 566 'run_name': 'perfName01', 567 'real_time': '11001.11001', 568 'test_name': 'somePerfClass01#perfName01', 569 } 570 trim_perf01_test02 = { 571 'repetition_index': '0', 572 'cpu_time': '10002.10002', 573 'name': 'perfName02', 574 'repetitions': '0', 575 'run_type': 'iteration', 576 'label': '2123', 577 'threads': '1', 578 'time_unit': 'ns', 579 'iterations': '1002', 580 'run_name': 'perfName02', 581 'real_time': '11002.11002', 582 'test_name': 'somePerfClass01#perfName02', 583 } 584 trim_perf02_test01 = { 585 'repetition_index': '0', 586 'cpu_time': '20001.20001', 587 'name': 'perfName11', 588 'repetitions': '0', 589 'run_type': 'iteration', 590 'label': '2123', 591 'threads': '1', 592 'time_unit': 'ns', 593 'iterations': '2001', 594 'run_name': 'perfName11', 595 'real_time': '21001.21001', 596 'test_name': 'somePerfClass02#perfName11', 597 } 598 correct_classify_perf_info = { 599 'somePerfClass01': [trim_perf01_test01, trim_perf01_test02], 600 'somePerfClass02': [trim_perf02_test01], 601 } 602 classify_perf_info, max_len = ( 603 self.rr.run_stats.perf_info._classify_perf_info() 604 ) 605 correct_max_len = { 606 'real_time': 11, 607 'cpu_time': 11, 608 'name': 10, 609 'iterations': 9, 610 'time_unit': 2, 611 } 612 self.assertEqual(max_len, correct_max_len) 613 self.assertEqual(classify_perf_info, correct_classify_perf_info) 614 615 616if __name__ == '__main__': 617 unittest.main() 618