1#!/usr/bin/env python3 2# 3# Copyright 2018, The Android Open Source Project 4# 5# Licensed under the Apache License, Version 2.0 (the "License"); 6# you may not use this file except in compliance with the License. 7# You may obtain a copy of the License at 8# 9# http://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an "AS IS" BASIS, 13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14# See the License for the specific language governing permissions and 15# limitations under the License. 16 17"""Unittests for result_reporter.""" 18 19# pylint: disable=line-too-long 20 21import sys 22import unittest 23 24from io import StringIO 25from unittest import mock 26 27import atest_configs 28import result_reporter 29 30from test_runners import test_runner_base 31 32 33RESULT_PASSED_TEST = test_runner_base.TestResult( 34 runner_name='someTestRunner', 35 group_name='someTestModule', 36 test_name='someClassName#sostName', 37 status=test_runner_base.PASSED_STATUS, 38 details=None, 39 test_count=1, 40 test_time='(10ms)', 41 runner_total=None, 42 group_total=2, 43 additional_info={}, 44 test_run_name='com.android.UnitTests' 45) 46 47RESULT_PASSED_TEST_MODULE_2 = test_runner_base.TestResult( 48 runner_name='someTestRunner', 49 group_name='someTestModule2', 50 test_name='someClassName#sostName', 51 status=test_runner_base.PASSED_STATUS, 52 details=None, 53 test_count=1, 54 test_time='(10ms)', 55 runner_total=None, 56 group_total=2, 57 additional_info={}, 58 test_run_name='com.android.UnitTests' 59) 60 61RESULT_PASSED_TEST_RUNNER_2_NO_MODULE = test_runner_base.TestResult( 62 runner_name='someTestRunner2', 63 group_name=None, 64 test_name='someClassName#sostName', 65 status=test_runner_base.PASSED_STATUS, 66 details=None, 67 test_count=1, 68 test_time='(10ms)', 69 runner_total=None, 70 group_total=2, 71 additional_info={}, 72 test_run_name='com.android.UnitTests' 73) 74 75RESULT_FAILED_TEST = test_runner_base.TestResult( 76 runner_name='someTestRunner', 77 group_name='someTestModule', 78 test_name='someClassName2#sestName2', 79 status=test_runner_base.FAILED_STATUS, 80 details='someTrace', 81 test_count=1, 82 test_time='', 83 runner_total=None, 84 group_total=2, 85 additional_info={}, 86 test_run_name='com.android.UnitTests' 87) 88 89RESULT_RUN_FAILURE = test_runner_base.TestResult( 90 runner_name='someTestRunner', 91 group_name='someTestModule', 92 test_name='someClassName#sostName', 93 status=test_runner_base.ERROR_STATUS, 94 details='someRunFailureReason', 95 test_count=1, 96 test_time='', 97 runner_total=None, 98 group_total=2, 99 additional_info={}, 100 test_run_name='com.android.UnitTests' 101) 102 103RESULT_INVOCATION_FAILURE = test_runner_base.TestResult( 104 runner_name='someTestRunner', 105 group_name=None, 106 test_name=None, 107 status=test_runner_base.ERROR_STATUS, 108 details='someInvocationFailureReason', 109 test_count=1, 110 test_time='', 111 runner_total=None, 112 group_total=None, 113 additional_info={}, 114 test_run_name='com.android.UnitTests' 115) 116 117RESULT_IGNORED_TEST = test_runner_base.TestResult( 118 runner_name='someTestRunner', 119 group_name='someTestModule', 120 test_name='someClassName#sostName', 121 status=test_runner_base.IGNORED_STATUS, 122 details=None, 123 test_count=1, 124 test_time='(10ms)', 125 runner_total=None, 126 group_total=2, 127 additional_info={}, 128 test_run_name='com.android.UnitTests' 129) 130 131RESULT_ASSUMPTION_FAILED_TEST = test_runner_base.TestResult( 132 runner_name='someTestRunner', 133 group_name='someTestModule', 134 test_name='someClassName#sostName', 135 status=test_runner_base.ASSUMPTION_FAILED, 136 details=None, 137 test_count=1, 138 test_time='(10ms)', 139 runner_total=None, 140 group_total=2, 141 additional_info={}, 142 test_run_name='com.android.UnitTests' 143) 144 145ADDITIONAL_INFO_PERF01_TEST01 = {u'repetition_index': u'0', 146 u'cpu_time': u'10001.10001', 147 u'name': u'perfName01', 148 u'repetitions': u'0', u'run_type': u'iteration', 149 u'label': u'2123', u'threads': u'1', 150 u'time_unit': u'ns', u'iterations': u'1001', 151 u'run_name': u'perfName01', 152 u'real_time': u'11001.11001'} 153 154RESULT_PERF01_TEST01 = test_runner_base.TestResult( 155 runner_name='someTestRunner', 156 group_name='someTestModule', 157 test_name='somePerfClass01#perfName01', 158 status=test_runner_base.PASSED_STATUS, 159 details=None, 160 test_count=1, 161 test_time='(10ms)', 162 runner_total=None, 163 group_total=2, 164 additional_info=ADDITIONAL_INFO_PERF01_TEST01, 165 test_run_name='com.android.UnitTests' 166) 167 168RESULT_PERF01_TEST02 = test_runner_base.TestResult( 169 runner_name='someTestRunner', 170 group_name='someTestModule', 171 test_name='somePerfClass01#perfName02', 172 status=test_runner_base.PASSED_STATUS, 173 details=None, 174 test_count=1, 175 test_time='(10ms)', 176 runner_total=None, 177 group_total=2, 178 additional_info={u'repetition_index': u'0', u'cpu_time': u'10002.10002', 179 u'name': u'perfName02', 180 u'repetitions': u'0', u'run_type': u'iteration', 181 u'label': u'2123', u'threads': u'1', 182 u'time_unit': u'ns', u'iterations': u'1002', 183 u'run_name': u'perfName02', 184 u'real_time': u'11002.11002'}, 185 test_run_name='com.android.UnitTests' 186) 187 188RESULT_PERF01_TEST03_NO_CPU_TIME = test_runner_base.TestResult( 189 runner_name='someTestRunner', 190 group_name='someTestModule', 191 test_name='somePerfClass01#perfName03', 192 status=test_runner_base.PASSED_STATUS, 193 details=None, 194 test_count=1, 195 test_time='(10ms)', 196 runner_total=None, 197 group_total=2, 198 additional_info={u'repetition_index': u'0', 199 u'name': u'perfName03', 200 u'repetitions': u'0', u'run_type': u'iteration', 201 u'label': u'2123', u'threads': u'1', 202 u'time_unit': u'ns', u'iterations': u'1003', 203 u'run_name': u'perfName03', 204 u'real_time': u'11003.11003'}, 205 test_run_name='com.android.UnitTests' 206) 207 208RESULT_PERF02_TEST01 = test_runner_base.TestResult( 209 runner_name='someTestRunner', 210 group_name='someTestModule', 211 test_name='somePerfClass02#perfName11', 212 status=test_runner_base.PASSED_STATUS, 213 details=None, 214 test_count=1, 215 test_time='(10ms)', 216 runner_total=None, 217 group_total=2, 218 additional_info={u'repetition_index': u'0', u'cpu_time': u'20001.20001', 219 u'name': u'perfName11', 220 u'repetitions': u'0', u'run_type': u'iteration', 221 u'label': u'2123', u'threads': u'1', 222 u'time_unit': u'ns', u'iterations': u'2001', 223 u'run_name': u'perfName11', 224 u'real_time': u'210001.21001'}, 225 test_run_name='com.android.UnitTests' 226) 227 228#pylint: disable=protected-access 229#pylint: disable=invalid-name 230class ResultReporterUnittests(unittest.TestCase): 231 """Unit tests for result_reporter.py""" 232 233 def setUp(self): 234 self.rr = result_reporter.ResultReporter() 235 236 def tearDown(self): 237 mock.patch.stopall() 238 239 @mock.patch.object(result_reporter.ResultReporter, '_print_group_title') 240 @mock.patch.object(result_reporter.ResultReporter, '_update_stats') 241 @mock.patch.object(result_reporter.ResultReporter, '_print_result') 242 def test_process_test_result(self, mock_print, mock_update, mock_title): 243 """Test process_test_result method.""" 244 # Passed Test 245 self.assertTrue('someTestRunner' not in self.rr.runners) 246 self.rr.process_test_result(RESULT_PASSED_TEST) 247 self.assertTrue('someTestRunner' in self.rr.runners) 248 group = self.rr.runners['someTestRunner'].get('someTestModule') 249 self.assertIsNotNone(group) 250 mock_title.assert_called_with(RESULT_PASSED_TEST) 251 mock_update.assert_called_with(RESULT_PASSED_TEST, group) 252 mock_print.assert_called_with(RESULT_PASSED_TEST) 253 # Failed Test 254 mock_title.reset_mock() 255 self.rr.process_test_result(RESULT_FAILED_TEST) 256 mock_title.assert_not_called() 257 mock_update.assert_called_with(RESULT_FAILED_TEST, group) 258 mock_print.assert_called_with(RESULT_FAILED_TEST) 259 # Test with new Group 260 mock_title.reset_mock() 261 self.rr.process_test_result(RESULT_PASSED_TEST_MODULE_2) 262 self.assertTrue('someTestModule2' in self.rr.runners['someTestRunner']) 263 mock_title.assert_called_with(RESULT_PASSED_TEST_MODULE_2) 264 # Test with new Runner 265 mock_title.reset_mock() 266 self.rr.process_test_result(RESULT_PASSED_TEST_RUNNER_2_NO_MODULE) 267 self.assertTrue('someTestRunner2' in self.rr.runners) 268 mock_title.assert_called_with(RESULT_PASSED_TEST_RUNNER_2_NO_MODULE) 269 270 def test_print_result_run_name(self): 271 """Test print run name function in print_result method.""" 272 try: 273 rr = result_reporter.ResultReporter() 274 capture_output = StringIO() 275 sys.stdout = capture_output 276 run_name = 'com.android.UnitTests' 277 rr._print_result(test_runner_base.TestResult( 278 runner_name='runner_name', 279 group_name='someTestModule', 280 test_name='someClassName#someTestName', 281 status=test_runner_base.FAILED_STATUS, 282 details='someTrace', 283 test_count=2, 284 test_time='(2h44m36.402s)', 285 runner_total=None, 286 group_total=2, 287 additional_info={}, 288 test_run_name=run_name 289 )) 290 # Make sure run name in the first line. 291 capture_output_str = capture_output.getvalue().strip() 292 self.assertTrue(run_name in capture_output_str.split('\n')[0]) 293 run_name2 = 'com.android.UnitTests2' 294 capture_output = StringIO() 295 sys.stdout = capture_output 296 rr._print_result(test_runner_base.TestResult( 297 runner_name='runner_name', 298 group_name='someTestModule', 299 test_name='someClassName#someTestName', 300 status=test_runner_base.FAILED_STATUS, 301 details='someTrace', 302 test_count=2, 303 test_time='(2h43m36.402s)', 304 runner_total=None, 305 group_total=2, 306 additional_info={}, 307 test_run_name=run_name2 308 )) 309 # Make sure run name in the first line. 310 capture_output_str = capture_output.getvalue().strip() 311 self.assertTrue(run_name2 in capture_output_str.split('\n')[0]) 312 finally: 313 sys.stdout = sys.__stdout__ 314 315 def test_register_unsupported_runner(self): 316 """Test register_unsupported_runner method.""" 317 self.rr.register_unsupported_runner('NotSupported') 318 runner = self.rr.runners['NotSupported'] 319 self.assertIsNotNone(runner) 320 self.assertEqual(runner, result_reporter.UNSUPPORTED_FLAG) 321 322 def test_update_stats_passed(self): 323 """Test _update_stats method.""" 324 # Passed Test 325 group = result_reporter.RunStat() 326 self.rr._update_stats(RESULT_PASSED_TEST, group) 327 self.assertEqual(self.rr.run_stats.passed, 1) 328 self.assertEqual(self.rr.run_stats.failed, 0) 329 self.assertEqual(self.rr.run_stats.run_errors, False) 330 self.assertEqual(self.rr.failed_tests, []) 331 self.assertEqual(group.passed, 1) 332 self.assertEqual(group.failed, 0) 333 self.assertEqual(group.ignored, 0) 334 self.assertEqual(group.run_errors, False) 335 # Passed Test New Group 336 group2 = result_reporter.RunStat() 337 self.rr._update_stats(RESULT_PASSED_TEST_MODULE_2, group2) 338 self.assertEqual(self.rr.run_stats.passed, 2) 339 self.assertEqual(self.rr.run_stats.failed, 0) 340 self.assertEqual(self.rr.run_stats.run_errors, False) 341 self.assertEqual(self.rr.failed_tests, []) 342 self.assertEqual(group2.passed, 1) 343 self.assertEqual(group2.failed, 0) 344 self.assertEqual(group.ignored, 0) 345 self.assertEqual(group2.run_errors, False) 346 347 def test_update_stats_failed(self): 348 """Test _update_stats method.""" 349 # Passed Test 350 group = result_reporter.RunStat() 351 self.rr._update_stats(RESULT_PASSED_TEST, group) 352 # Passed Test New Group 353 group2 = result_reporter.RunStat() 354 self.rr._update_stats(RESULT_PASSED_TEST_MODULE_2, group2) 355 # Failed Test Old Group 356 self.rr._update_stats(RESULT_FAILED_TEST, group) 357 self.assertEqual(self.rr.run_stats.passed, 2) 358 self.assertEqual(self.rr.run_stats.failed, 1) 359 self.assertEqual(self.rr.run_stats.run_errors, False) 360 self.assertEqual(self.rr.failed_tests, [RESULT_FAILED_TEST.test_name]) 361 self.assertEqual(group.passed, 1) 362 self.assertEqual(group.failed, 1) 363 self.assertEqual(group.ignored, 0) 364 self.assertEqual(group.total, 2) 365 self.assertEqual(group2.total, 1) 366 self.assertEqual(group.run_errors, False) 367 # Test Run Failure 368 self.rr._update_stats(RESULT_RUN_FAILURE, group) 369 self.assertEqual(self.rr.run_stats.passed, 2) 370 self.assertEqual(self.rr.run_stats.failed, 1) 371 self.assertEqual(self.rr.run_stats.run_errors, True) 372 self.assertEqual(self.rr.failed_tests, [RESULT_FAILED_TEST.test_name]) 373 self.assertEqual(group.passed, 1) 374 self.assertEqual(group.failed, 1) 375 self.assertEqual(group.ignored, 0) 376 self.assertEqual(group.run_errors, True) 377 self.assertEqual(group2.run_errors, False) 378 # Invocation Failure 379 self.rr._update_stats(RESULT_INVOCATION_FAILURE, group) 380 self.assertEqual(self.rr.run_stats.passed, 2) 381 self.assertEqual(self.rr.run_stats.failed, 1) 382 self.assertEqual(self.rr.run_stats.run_errors, True) 383 self.assertEqual(self.rr.failed_tests, [RESULT_FAILED_TEST.test_name]) 384 self.assertEqual(group.passed, 1) 385 self.assertEqual(group.failed, 1) 386 self.assertEqual(group.ignored, 0) 387 self.assertEqual(group.run_errors, True) 388 389 def test_update_stats_ignored_and_assumption_failure(self): 390 """Test _update_stats method.""" 391 # Passed Test 392 group = result_reporter.RunStat() 393 self.rr._update_stats(RESULT_PASSED_TEST, group) 394 # Passed Test New Group 395 group2 = result_reporter.RunStat() 396 self.rr._update_stats(RESULT_PASSED_TEST_MODULE_2, group2) 397 # Failed Test Old Group 398 self.rr._update_stats(RESULT_FAILED_TEST, group) 399 # Test Run Failure 400 self.rr._update_stats(RESULT_RUN_FAILURE, group) 401 # Invocation Failure 402 self.rr._update_stats(RESULT_INVOCATION_FAILURE, group) 403 # Ignored Test 404 self.rr._update_stats(RESULT_IGNORED_TEST, group) 405 self.assertEqual(self.rr.run_stats.passed, 2) 406 self.assertEqual(self.rr.run_stats.failed, 1) 407 self.assertEqual(self.rr.run_stats.run_errors, True) 408 self.assertEqual(self.rr.failed_tests, [RESULT_FAILED_TEST.test_name]) 409 self.assertEqual(group.passed, 1) 410 self.assertEqual(group.failed, 1) 411 self.assertEqual(group.ignored, 1) 412 self.assertEqual(group.run_errors, True) 413 # 2nd Ignored Test 414 self.rr._update_stats(RESULT_IGNORED_TEST, group) 415 self.assertEqual(self.rr.run_stats.passed, 2) 416 self.assertEqual(self.rr.run_stats.failed, 1) 417 self.assertEqual(self.rr.run_stats.run_errors, True) 418 self.assertEqual(self.rr.failed_tests, [RESULT_FAILED_TEST.test_name]) 419 self.assertEqual(group.passed, 1) 420 self.assertEqual(group.failed, 1) 421 self.assertEqual(group.ignored, 2) 422 self.assertEqual(group.run_errors, True) 423 424 # Assumption_Failure test 425 self.rr._update_stats(RESULT_ASSUMPTION_FAILED_TEST, group) 426 self.assertEqual(group.assumption_failed, 1) 427 # 2nd Assumption_Failure test 428 self.rr._update_stats(RESULT_ASSUMPTION_FAILED_TEST, group) 429 self.assertEqual(group.assumption_failed, 2) 430 431 def test_print_summary_ret_val(self): 432 """Test print_summary method's return value.""" 433 atest_configs.GLOBAL_ARGS = mock.Mock() 434 atest_configs.GLOBAL_ARGS.aggregate_metric_filter = None 435 436 # PASS Case 437 self.rr.process_test_result(RESULT_PASSED_TEST) 438 self.assertEqual(0, self.rr.print_summary()) 439 # PASS Case + Fail Case 440 self.rr.process_test_result(RESULT_FAILED_TEST) 441 self.assertNotEqual(0, self.rr.print_summary()) 442 # PASS Case + Fail Case + PASS Case 443 self.rr.process_test_result(RESULT_PASSED_TEST_MODULE_2) 444 self.assertNotEqual(0, self.rr.print_summary()) 445 446 def test_print_summary_ret_val_err_stat(self): 447 """Test print_summary method's return value.""" 448 atest_configs.GLOBAL_ARGS = mock.Mock() 449 atest_configs.GLOBAL_ARGS.aggregate_metric_filter = None 450 451 # PASS Case 452 self.rr.process_test_result(RESULT_PASSED_TEST) 453 self.assertEqual(0, self.rr.print_summary()) 454 # PASS Case + Fail Case 455 self.rr.process_test_result(RESULT_RUN_FAILURE) 456 self.assertNotEqual(0, self.rr.print_summary()) 457 # PASS Case + Fail Case + PASS Case 458 self.rr.process_test_result(RESULT_PASSED_TEST_MODULE_2) 459 self.assertNotEqual(0, self.rr.print_summary()) 460 461 def test_update_perf_info(self): 462 """Test update_perf_info method.""" 463 group = result_reporter.RunStat() 464 # 1. Test PerfInfo after RESULT_PERF01_TEST01 465 # _update_stats() will call _update_perf_info() 466 self.rr._update_stats(RESULT_PERF01_TEST01, group) 467 correct_perf_info = [] 468 # trim the time form 10001.10001 to 10001 469 trim_perf01_test01 = {u'repetition_index': u'0', u'cpu_time': u'10001', 470 u'name': u'perfName01', 471 u'repetitions': u'0', u'run_type': u'iteration', 472 u'label': u'2123', u'threads': u'1', 473 u'time_unit': u'ns', u'iterations': u'1001', 474 u'run_name': u'perfName01', 475 u'real_time': u'11001', 476 'test_name': 'somePerfClass01#perfName01'} 477 correct_perf_info.append(trim_perf01_test01) 478 self.assertEqual(self.rr.run_stats.perf_info.perf_info, 479 correct_perf_info) 480 # 2. Test PerfInfo after RESULT_PERF01_TEST01 481 self.rr._update_stats(RESULT_PERF01_TEST02, group) 482 trim_perf01_test02 = {u'repetition_index': u'0', u'cpu_time': u'10002', 483 u'name': u'perfName02', 484 u'repetitions': u'0', u'run_type': u'iteration', 485 u'label': u'2123', u'threads': u'1', 486 u'time_unit': u'ns', u'iterations': u'1002', 487 u'run_name': u'perfName02', 488 u'real_time': u'11002', 489 'test_name': 'somePerfClass01#perfName02'} 490 correct_perf_info.append(trim_perf01_test02) 491 self.assertEqual(self.rr.run_stats.perf_info.perf_info, 492 correct_perf_info) 493 # 3. Test PerfInfo after RESULT_PERF02_TEST01 494 self.rr._update_stats(RESULT_PERF02_TEST01, group) 495 trim_perf02_test01 = {u'repetition_index': u'0', u'cpu_time': u'20001', 496 u'name': u'perfName11', 497 u'repetitions': u'0', u'run_type': u'iteration', 498 u'label': u'2123', u'threads': u'1', 499 u'time_unit': u'ns', u'iterations': u'2001', 500 u'run_name': u'perfName11', 501 u'real_time': u'210001', 502 'test_name': 'somePerfClass02#perfName11'} 503 correct_perf_info.append(trim_perf02_test01) 504 self.assertEqual(self.rr.run_stats.perf_info.perf_info, 505 correct_perf_info) 506 # 4. Test PerfInfo after RESULT_PERF01_TEST03_NO_CPU_TIME 507 self.rr._update_stats(RESULT_PERF01_TEST03_NO_CPU_TIME, group) 508 # Nothing added since RESULT_PERF01_TEST03_NO_CPU_TIME lack of cpu_time 509 self.assertEqual(self.rr.run_stats.perf_info.perf_info, 510 correct_perf_info) 511 512 def test_classify_perf_info(self): 513 """Test _classify_perf_info method.""" 514 group = result_reporter.RunStat() 515 self.rr._update_stats(RESULT_PERF01_TEST01, group) 516 self.rr._update_stats(RESULT_PERF01_TEST02, group) 517 self.rr._update_stats(RESULT_PERF02_TEST01, group) 518 # trim the time form 10001.10001 to 10001 519 trim_perf01_test01 = {u'repetition_index': u'0', u'cpu_time': u'10001', 520 u'name': u'perfName01', 521 u'repetitions': u'0', u'run_type': u'iteration', 522 u'label': u'2123', u'threads': u'1', 523 u'time_unit': u'ns', u'iterations': u'1001', 524 u'run_name': u'perfName01', 525 u'real_time': u'11001', 526 'test_name': 'somePerfClass01#perfName01'} 527 trim_perf01_test02 = {u'repetition_index': u'0', u'cpu_time': u'10002', 528 u'name': u'perfName02', 529 u'repetitions': u'0', u'run_type': u'iteration', 530 u'label': u'2123', u'threads': u'1', 531 u'time_unit': u'ns', u'iterations': u'1002', 532 u'run_name': u'perfName02', 533 u'real_time': u'11002', 534 'test_name': 'somePerfClass01#perfName02'} 535 trim_perf02_test01 = {u'repetition_index': u'0', u'cpu_time': u'20001', 536 u'name': u'perfName11', 537 u'repetitions': u'0', u'run_type': u'iteration', 538 u'label': u'2123', u'threads': u'1', 539 u'time_unit': u'ns', u'iterations': u'2001', 540 u'run_name': u'perfName11', 541 u'real_time': u'210001', 542 'test_name': 'somePerfClass02#perfName11'} 543 correct_classify_perf_info = {"somePerfClass01":[trim_perf01_test01, 544 trim_perf01_test02], 545 "somePerfClass02":[trim_perf02_test01]} 546 classify_perf_info, max_len = self.rr.run_stats.perf_info._classify_perf_info() 547 correct_max_len = {'real_time': 6, 'cpu_time': 5, 'name': 10, 548 'iterations': 9, 'time_unit': 2} 549 self.assertEqual(max_len, correct_max_len) 550 self.assertEqual(classify_perf_info, correct_classify_perf_info) 551 552 553if __name__ == '__main__': 554 unittest.main() 555