1#!/usr/bin/env python3 2# 3# Copyright 2016 - The Android Open Source Project 4# 5# Licensed under the Apache License, Version 2.0 (the "License"); 6# you may not use this file except in compliance with the License. 7# You may obtain a copy of the License at 8# 9# http://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an "AS IS" BASIS, 13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14# See the License for the specific language governing permissions and 15# limitations under the License. 16 17import shutil 18import tempfile 19import unittest 20 21import mock 22import mock_controller 23 24from acts import asserts 25from acts import base_test 26from acts import error 27from acts import signals 28 29MSG_EXPECTED_EXCEPTION = 'This is an expected exception.' 30MSG_EXPECTED_TEST_FAILURE = 'This is an expected test failure.' 31MSG_UNEXPECTED_EXCEPTION = 'Unexpected exception!' 32 33MOCK_EXTRA = {'key': 'value', 'answer_to_everything': 42} 34 35 36def never_call(): 37 raise Exception(MSG_UNEXPECTED_EXCEPTION) 38 39 40class SomeError(Exception): 41 """A custom exception class used for tests in this module.""" 42 43 44class ActsBaseClassTest(unittest.TestCase): 45 def setUp(self): 46 self.tmp_dir = tempfile.mkdtemp() 47 self.tb_key = 'testbed_configs' 48 self.mock_test_cls_configs = { 49 self.tb_key: { 50 'name': 'SampleTestBed', 51 }, 52 'reporter': mock.MagicMock(), 53 'log': mock.MagicMock(), 54 'log_path': self.tmp_dir, 55 'summary_writer': mock.MagicMock(), 56 'cli_args': None, 57 'user_params': { 58 'some_param': 'hahaha' 59 } 60 } 61 self.mock_test_name = 'test_something' 62 63 def tearDown(self): 64 shutil.rmtree(self.tmp_dir) 65 66 def test_current_test_case_name(self): 67 class MockBaseTest(base_test.BaseTestClass): 68 def test_func(self): 69 asserts.assert_true( 70 self.current_test_name == 'test_func', 71 'Got unexpected test name %s.' % self.current_test_name) 72 73 bt_cls = MockBaseTest(self.mock_test_cls_configs) 74 bt_cls.run(test_names=['test_func']) 75 actual_record = bt_cls.results.passed[0] 76 self.assertEqual(actual_record.test_name, 'test_func') 77 self.assertIsNone(actual_record.details) 78 self.assertIsNone(actual_record.extras) 79 80 def test_self_tests_list(self): 81 class MockBaseTest(base_test.BaseTestClass): 82 def __init__(self, controllers): 83 super(MockBaseTest, self).__init__(controllers) 84 self.tests = ('test_something',) 85 86 def test_something(self): 87 pass 88 89 def test_never(self): 90 # This should not execute it's not on default test list. 91 never_call() 92 93 bt_cls = MockBaseTest(self.mock_test_cls_configs) 94 bt_cls.run() 95 actual_record = bt_cls.results.passed[0] 96 self.assertEqual(actual_record.test_name, 'test_something') 97 98 def test_self_tests_list_fail_by_convention(self): 99 class MockBaseTest(base_test.BaseTestClass): 100 def __init__(self, controllers): 101 super(MockBaseTest, self).__init__(controllers) 102 self.tests = ('not_a_test_something',) 103 104 def not_a_test_something(self): 105 pass 106 107 def test_never(self): 108 # This should not execute it's not on default test list. 109 never_call() 110 111 bt_cls = MockBaseTest(self.mock_test_cls_configs) 112 expected_msg = ('Test case name not_a_test_something does not follow ' 113 'naming convention test_\*, abort.') 114 with self.assertRaisesRegex(base_test.Error, expected_msg): 115 bt_cls.run() 116 117 def test_cli_test_selection_match_self_tests_list(self): 118 class MockBaseTest(base_test.BaseTestClass): 119 def __init__(self, controllers): 120 super(MockBaseTest, self).__init__(controllers) 121 self.tests = ('test_star1', 'test_star2', 'test_question_mark', 122 'test_char_seq', 'test_no_match') 123 124 def test_star1(self): 125 pass 126 127 def test_star2(self): 128 pass 129 130 def test_question_mark(self): 131 pass 132 133 def test_char_seq(self): 134 pass 135 136 def test_no_match(self): 137 # This should not execute because it does not match any regex 138 # in the cmd line input. 139 never_call() 140 141 bt_cls = MockBaseTest(self.mock_test_cls_configs) 142 test_names = ['test_st*r1', 'test_*2', 'test_?uestion_mark', 143 'test_c[fghi]ar_seq'] 144 bt_cls.run(test_names=test_names) 145 passed_names = [p.test_name for p in bt_cls.results.passed] 146 self.assertEqual(len(passed_names), len(test_names)) 147 for test in ['test_star1', 'test_star2', 'test_question_mark', 148 'test_char_seq']: 149 self.assertIn(test, passed_names) 150 151 def test_default_execution_of_all_tests(self): 152 class MockBaseTest(base_test.BaseTestClass): 153 def test_something(self): 154 pass 155 156 def not_a_test(self): 157 # This should not execute its name doesn't follow test case 158 # naming convention. 159 never_call() 160 161 bt_cls = MockBaseTest(self.mock_test_cls_configs) 162 bt_cls.run() 163 actual_record = bt_cls.results.passed[0] 164 self.assertEqual(actual_record.test_name, 'test_something') 165 166 def test_missing_requested_test_func(self): 167 class MockBaseTest(base_test.BaseTestClass): 168 def __init__(self, controllers): 169 super(MockBaseTest, self).__init__(controllers) 170 self.tests = ('test_something',) 171 172 bt_cls = MockBaseTest(self.mock_test_cls_configs) 173 bt_cls.run() 174 self.assertFalse(bt_cls.results.executed) 175 self.assertTrue(bt_cls.results.skipped) 176 177 def test_setup_class_fail_by_exception(self): 178 call_check = mock.MagicMock() 179 180 class MockBaseTest(base_test.BaseTestClass): 181 def setup_class(self): 182 raise Exception(MSG_EXPECTED_EXCEPTION) 183 184 def test_something(self): 185 # This should not execute because setup_class failed. 186 never_call() 187 188 def on_skip(self, test_name, begin_time): 189 call_check('haha') 190 191 bt_cls = MockBaseTest(self.mock_test_cls_configs) 192 bt_cls.run() 193 actual_record = bt_cls.results.error[0] 194 self.assertEqual(actual_record.test_name, 'test_something') 195 expected_summary = { 196 'Error': 1, 'Executed': 1, 197 'Failed': 0, 'Passed': 0, 'Requested': 1, 'Skipped': 0 198 } 199 self.assertEqual(bt_cls.results.summary_dict(), expected_summary) 200 call_check.assert_called_once_with('haha') 201 202 def test_setup_test_fail_by_exception(self): 203 class MockBaseTest(base_test.BaseTestClass): 204 def setup_test(self): 205 raise Exception(MSG_EXPECTED_EXCEPTION) 206 207 def test_something(self): 208 # This should not execute because setup_test failed. 209 never_call() 210 211 bt_cls = MockBaseTest(self.mock_test_cls_configs) 212 bt_cls.run(test_names=['test_something']) 213 actual_record = bt_cls.results.error[0] 214 self.assertEqual(actual_record.test_name, self.mock_test_name) 215 self.assertEqual(actual_record.details, MSG_EXPECTED_EXCEPTION) 216 self.assertIsNone(actual_record.extras) 217 expected_summary = { 218 'Error': 1, 'Executed': 1, 219 'Failed': 0, 'Passed': 0, 'Requested': 1, 'Skipped': 0 220 } 221 self.assertEqual(bt_cls.results.summary_dict(), expected_summary) 222 223 def test_setup_test_fail_by_test_signal(self): 224 class MockBaseTest(base_test.BaseTestClass): 225 def setup_test(self): 226 raise signals.TestFailure(MSG_EXPECTED_EXCEPTION) 227 228 def test_something(self): 229 # This should not execute because setup_test failed. 230 never_call() 231 232 bt_cls = MockBaseTest(self.mock_test_cls_configs) 233 bt_cls.run(test_names=['test_something']) 234 actual_record = bt_cls.results.failed[0] 235 self.assertEqual(actual_record.test_name, self.mock_test_name) 236 self.assertEqual(actual_record.details, MSG_EXPECTED_EXCEPTION) 237 self.assertIsNone(actual_record.extras) 238 expected_summary = { 239 'Error': 0, 'Executed': 1, 240 'Failed': 1, 'Passed': 0, 'Requested': 1, 'Skipped': 0 241 } 242 self.assertEqual(bt_cls.results.summary_dict(), expected_summary) 243 244 def test_setup_test_fail_by_return_False(self): 245 class MockBaseTest(base_test.BaseTestClass): 246 def setup_test(self): 247 return False 248 249 def test_something(self): 250 # This should not execute because setup_test failed. 251 never_call() 252 253 bt_cls = MockBaseTest(self.mock_test_cls_configs) 254 bt_cls.run(test_names=['test_something']) 255 actual_record = bt_cls.results.failed[0] 256 expected_msg = 'Setup for %s failed.' % self.mock_test_name 257 self.assertEqual(actual_record.test_name, self.mock_test_name) 258 self.assertEqual(actual_record.details, expected_msg) 259 self.assertIsNone(actual_record.extras) 260 expected_summary = { 261 'Error': 0, 'Executed': 1, 262 'Failed': 1, 'Passed': 0, 'Requested': 1, 'Skipped': 0 263 } 264 self.assertEqual(bt_cls.results.summary_dict(), expected_summary) 265 266 def test_run_fail_by_ActsError_(self): 267 class MockBaseTest(base_test.BaseTestClass): 268 def __init__(self, controllers): 269 super(MockBaseTest, self).__init__(controllers) 270 271 def test_something(self): 272 raise error.ActsError() 273 274 bt_cls = MockBaseTest(self.mock_test_cls_configs) 275 bt_cls.run(test_names=['test_something']) 276 expected_summary = { 277 'Error': 1, 'Executed': 1, 278 'Failed': 0, 'Passed': 0, 'Requested': 1, 'Skipped': 0 279 } 280 self.assertEqual(bt_cls.results.summary_dict(), expected_summary) 281 282 def test_teardown_test_assert_fail(self): 283 class MockBaseTest(base_test.BaseTestClass): 284 def teardown_test(self): 285 asserts.assert_true(False, MSG_EXPECTED_EXCEPTION) 286 287 def test_something(self): 288 pass 289 290 bt_cls = MockBaseTest(self.mock_test_cls_configs) 291 bt_cls.run() 292 actual_record = bt_cls.results.error[0] 293 self.assertEqual(actual_record.test_name, self.mock_test_name) 294 self.assertEqual(actual_record.details, MSG_EXPECTED_EXCEPTION) 295 self.assertIsNone(actual_record.extras) 296 expected_summary = { 297 'Error': 1, 'Executed': 1, 298 'Failed': 0, 'Passed': 0, 'Requested': 1, 'Skipped': 0 299 } 300 self.assertEqual(bt_cls.results.summary_dict(), expected_summary) 301 302 def test_teardown_test_raise_exception(self): 303 class MockBaseTest(base_test.BaseTestClass): 304 def teardown_test(self): 305 raise Exception(MSG_EXPECTED_EXCEPTION) 306 307 def test_something(self): 308 pass 309 310 bt_cls = MockBaseTest(self.mock_test_cls_configs) 311 bt_cls.run() 312 actual_record = bt_cls.results.error[0] 313 self.assertEqual(actual_record.test_name, self.mock_test_name) 314 self.assertEqual(actual_record.details, MSG_EXPECTED_EXCEPTION) 315 self.assertIsNone(actual_record.extras) 316 expected_summary = { 317 'Error': 1, 'Executed': 1, 318 'Failed': 0, 'Passed': 0, 'Requested': 1, 'Skipped': 0 319 } 320 self.assertEqual(bt_cls.results.summary_dict(), expected_summary) 321 322 def test_teardown_test_executed_if_test_pass(self): 323 my_mock = mock.MagicMock() 324 325 class MockBaseTest(base_test.BaseTestClass): 326 def teardown_test(self): 327 my_mock('teardown_test') 328 329 def test_something(self): 330 pass 331 332 bt_cls = MockBaseTest(self.mock_test_cls_configs) 333 bt_cls.run() 334 actual_record = bt_cls.results.passed[0] 335 my_mock.assert_called_once_with('teardown_test') 336 self.assertEqual(actual_record.test_name, self.mock_test_name) 337 self.assertIsNone(actual_record.details) 338 self.assertIsNone(actual_record.extras) 339 expected_summary = { 340 'Error': 0, 'Executed': 1, 341 'Failed': 0, 'Passed': 1, 'Requested': 1, 'Skipped': 0 342 } 343 self.assertEqual(bt_cls.results.summary_dict(), expected_summary) 344 345 def test_teardown_test_executed_if_setup_test_fails(self): 346 my_mock = mock.MagicMock() 347 348 class MockBaseTest(base_test.BaseTestClass): 349 def setup_test(self): 350 raise Exception(MSG_EXPECTED_EXCEPTION) 351 352 def teardown_test(self): 353 my_mock('teardown_test') 354 355 def test_something(self): 356 pass 357 358 bt_cls = MockBaseTest(self.mock_test_cls_configs) 359 bt_cls.run() 360 actual_record = bt_cls.results.error[0] 361 my_mock.assert_called_once_with('teardown_test') 362 self.assertEqual(actual_record.test_name, self.mock_test_name) 363 self.assertEqual(actual_record.details, MSG_EXPECTED_EXCEPTION) 364 self.assertIsNone(actual_record.extras) 365 expected_summary = { 366 'Error': 1, 'Executed': 1, 367 'Failed': 0, 'Passed': 0, 'Requested': 1, 'Skipped': 0 368 } 369 self.assertEqual(bt_cls.results.summary_dict(), expected_summary) 370 371 def test_teardown_test_executed_if_test_fails(self): 372 my_mock = mock.MagicMock() 373 374 class MockBaseTest(base_test.BaseTestClass): 375 def teardown_test(self): 376 my_mock('teardown_test') 377 378 def test_something(self): 379 raise Exception(MSG_EXPECTED_EXCEPTION) 380 381 bt_cls = MockBaseTest(self.mock_test_cls_configs) 382 bt_cls.run() 383 actual_record = bt_cls.results.error[0] 384 my_mock.assert_called_once_with('teardown_test') 385 self.assertEqual(actual_record.test_name, self.mock_test_name) 386 self.assertEqual(actual_record.details, MSG_EXPECTED_EXCEPTION) 387 self.assertIsNone(actual_record.extras) 388 expected_summary = { 389 'Error': 1, 'Executed': 1, 390 'Failed': 0, 'Passed': 0, 'Requested': 1, 'Skipped': 0 391 } 392 self.assertEqual(bt_cls.results.summary_dict(), expected_summary) 393 394 def test_on_exception_executed_if_teardown_test_fails(self): 395 my_mock = mock.MagicMock() 396 397 class MockBaseTest(base_test.BaseTestClass): 398 def on_exception(self, test_name, begin_time): 399 my_mock('on_exception') 400 401 def teardown_test(self): 402 raise Exception(MSG_EXPECTED_EXCEPTION) 403 404 def test_something(self): 405 pass 406 407 bt_cls = MockBaseTest(self.mock_test_cls_configs) 408 bt_cls.run() 409 my_mock.assert_called_once_with('on_exception') 410 actual_record = bt_cls.results.error[0] 411 self.assertEqual(actual_record.test_name, self.mock_test_name) 412 self.assertEqual(actual_record.details, MSG_EXPECTED_EXCEPTION) 413 self.assertIsNone(actual_record.extras) 414 expected_summary = { 415 'Error': 1, 'Executed': 1, 416 'Failed': 0, 'Passed': 0, 'Requested': 1, 'Skipped': 0 417 } 418 self.assertEqual(bt_cls.results.summary_dict(), expected_summary) 419 420 def test_on_fail_executed_if_test_fails(self): 421 my_mock = mock.MagicMock() 422 423 class MockBaseTest(base_test.BaseTestClass): 424 def on_fail(self, test_name, begin_time): 425 my_mock('on_fail') 426 427 def test_something(self): 428 asserts.assert_true(False, MSG_EXPECTED_EXCEPTION) 429 430 bt_cls = MockBaseTest(self.mock_test_cls_configs) 431 bt_cls.run() 432 my_mock.assert_called_once_with('on_fail') 433 actual_record = bt_cls.results.failed[0] 434 self.assertEqual(actual_record.test_name, self.mock_test_name) 435 self.assertEqual(actual_record.details, MSG_EXPECTED_EXCEPTION) 436 self.assertIsNone(actual_record.extras) 437 expected_summary = { 438 'Error': 0, 'Executed': 1, 439 'Failed': 1, 'Passed': 0, 'Requested': 1, 'Skipped': 0 440 } 441 self.assertEqual(bt_cls.results.summary_dict(), expected_summary) 442 443 def test_on_fail_executed_if_test_setup_fails_by_exception(self): 444 my_mock = mock.MagicMock() 445 446 class MockBaseTest(base_test.BaseTestClass): 447 def setup_test(self): 448 raise Exception(MSG_EXPECTED_EXCEPTION) 449 450 def on_fail(self, test_name, begin_time): 451 my_mock('on_fail') 452 453 def test_something(self): 454 pass 455 456 bt_cls = MockBaseTest(self.mock_test_cls_configs) 457 bt_cls.run() 458 my_mock.assert_called_once_with('on_fail') 459 actual_record = bt_cls.results.error[0] 460 self.assertEqual(actual_record.test_name, self.mock_test_name) 461 self.assertEqual(actual_record.details, MSG_EXPECTED_EXCEPTION) 462 self.assertIsNone(actual_record.extras) 463 expected_summary = { 464 'Error': 1, 'Executed': 1, 465 'Failed': 0, 'Passed': 0, 'Requested': 1, 'Skipped': 0 466 } 467 self.assertEqual(bt_cls.results.summary_dict(), expected_summary) 468 469 def test_on_fail_executed_if_test_setup_fails_by_return_False(self): 470 my_mock = mock.MagicMock() 471 472 class MockBaseTest(base_test.BaseTestClass): 473 def setup_test(self): 474 return False 475 476 def on_fail(self, test_name, begin_time): 477 my_mock('on_fail') 478 479 def test_something(self): 480 pass 481 482 bt_cls = MockBaseTest(self.mock_test_cls_configs) 483 bt_cls.run() 484 my_mock.assert_called_once_with('on_fail') 485 actual_record = bt_cls.results.failed[0] 486 self.assertEqual(actual_record.test_name, self.mock_test_name) 487 self.assertEqual(actual_record.details, 488 'Setup for test_something failed.') 489 self.assertIsNone(actual_record.extras) 490 expected_summary = { 491 'Error': 0, 'Executed': 1, 492 'Failed': 1, 'Passed': 0, 'Requested': 1, 'Skipped': 0 493 } 494 self.assertEqual(bt_cls.results.summary_dict(), expected_summary) 495 496 def test_failure_to_call_procedure_function_is_recorded(self): 497 class MockBaseTest(base_test.BaseTestClass): 498 # Wrong method signature; will raise exception 499 def on_pass(self): 500 pass 501 502 def test_something(self): 503 asserts.explicit_pass(MSG_EXPECTED_EXCEPTION) 504 505 bt_cls = MockBaseTest(self.mock_test_cls_configs) 506 bt_cls.run() 507 actual_record = bt_cls.results.error[0] 508 self.assertIn('_on_pass', actual_record.extra_errors) 509 self.assertEqual(actual_record.test_name, self.mock_test_name) 510 self.assertEqual(actual_record.details, MSG_EXPECTED_EXCEPTION) 511 self.assertIsNone(actual_record.extras) 512 expected_summary = { 513 'Error': 1, 'Executed': 1, 514 'Failed': 0, 'Passed': 0, 'Requested': 1, 'Skipped': 0 515 } 516 self.assertEqual(bt_cls.results.summary_dict(), expected_summary) 517 518 def test_failure_in_procedure_functions_is_recorded(self): 519 expected_msg = 'Something failed in on_pass.' 520 521 class MockBaseTest(base_test.BaseTestClass): 522 def on_pass(self, test_name, begin_time): 523 raise Exception(expected_msg) 524 525 def test_something(self): 526 asserts.explicit_pass(MSG_EXPECTED_EXCEPTION) 527 528 bt_cls = MockBaseTest(self.mock_test_cls_configs) 529 bt_cls.run() 530 actual_record = bt_cls.results.error[0] 531 self.assertEqual(actual_record.test_name, self.mock_test_name) 532 self.assertEqual(actual_record.details, MSG_EXPECTED_EXCEPTION) 533 self.assertIsNone(actual_record.extras) 534 expected_summary = { 535 'Error': 1, 'Executed': 1, 536 'Failed': 0, 'Passed': 0, 'Requested': 1, 'Skipped': 0 537 } 538 self.assertEqual(bt_cls.results.summary_dict(), expected_summary) 539 540 def test_both_teardown_and_test_body_raise_exceptions(self): 541 class MockBaseTest(base_test.BaseTestClass): 542 def teardown_test(self): 543 asserts.assert_true(False, MSG_EXPECTED_EXCEPTION) 544 545 def test_something(self): 546 raise Exception('Test Body Exception.') 547 548 bt_cls = MockBaseTest(self.mock_test_cls_configs) 549 bt_cls.run() 550 actual_record = bt_cls.results.error[0] 551 self.assertEqual(actual_record.test_name, self.mock_test_name) 552 self.assertEqual(actual_record.details, 'Test Body Exception.') 553 self.assertIsNone(actual_record.extras) 554 self.assertEqual(actual_record.extra_errors['teardown_test'].details, 555 'This is an expected exception.') 556 expected_summary = { 557 'Error': 1, 'Executed': 1, 558 'Failed': 0, 'Passed': 0, 'Requested': 1, 'Skipped': 0 559 } 560 self.assertEqual(bt_cls.results.summary_dict(), expected_summary) 561 562 def test_explicit_pass_but_teardown_test_raises_an_exception(self): 563 """Test record result should be marked as UNKNOWN as opposed to PASS. 564 """ 565 566 class MockBaseTest(base_test.BaseTestClass): 567 def teardown_test(self): 568 asserts.assert_true(False, MSG_EXPECTED_EXCEPTION) 569 570 def test_something(self): 571 asserts.explicit_pass('Test Passed!') 572 573 bt_cls = MockBaseTest(self.mock_test_cls_configs) 574 bt_cls.run() 575 actual_record = bt_cls.results.error[0] 576 self.assertEqual(actual_record.test_name, self.mock_test_name) 577 self.assertEqual(actual_record.details, 'Test Passed!') 578 self.assertIsNone(actual_record.extras) 579 self.assertEqual(actual_record.extra_errors['teardown_test'].details, 580 'This is an expected exception.') 581 expected_summary = { 582 'Error': 1, 'Executed': 1, 583 'Failed': 0, 'Passed': 0, 'Requested': 1, 'Skipped': 0 584 } 585 self.assertEqual(bt_cls.results.summary_dict(), expected_summary) 586 587 def test_on_pass_raise_exception(self): 588 class MockBaseTest(base_test.BaseTestClass): 589 def on_pass(self, test_name, begin_time): 590 raise Exception(MSG_EXPECTED_EXCEPTION) 591 592 def test_something(self): 593 asserts.explicit_pass( 594 MSG_EXPECTED_EXCEPTION, extras=MOCK_EXTRA) 595 596 bt_cls = MockBaseTest(self.mock_test_cls_configs) 597 bt_cls.run() 598 actual_record = bt_cls.results.error[0] 599 self.assertEqual(actual_record.test_name, self.mock_test_name) 600 self.assertEqual(actual_record.details, MSG_EXPECTED_EXCEPTION) 601 self.assertEqual(actual_record.extras, MOCK_EXTRA) 602 expected_summary = { 603 'Error': 1, 'Executed': 1, 604 'Failed': 0, 'Passed': 0, 'Requested': 1, 'Skipped': 0 605 } 606 self.assertEqual(bt_cls.results.summary_dict(), expected_summary) 607 608 def test_on_fail_raise_exception(self): 609 class MockBaseTest(base_test.BaseTestClass): 610 def on_fail(self, test_name, begin_time): 611 raise Exception(MSG_EXPECTED_EXCEPTION) 612 613 def test_something(self): 614 asserts.fail(MSG_EXPECTED_EXCEPTION, extras=MOCK_EXTRA) 615 616 bt_cls = MockBaseTest(self.mock_test_cls_configs) 617 bt_cls.run() 618 actual_record = bt_cls.results.failed[0] 619 self.assertEqual(bt_cls.results.error, []) 620 self.assertEqual(actual_record.test_name, self.mock_test_name) 621 self.assertEqual(actual_record.details, MSG_EXPECTED_EXCEPTION) 622 self.assertEqual(actual_record.extras, MOCK_EXTRA) 623 expected_summary = { 624 'Error': 0, 'Executed': 1, 625 'Failed': 1, 'Passed': 0, 'Requested': 1, 'Skipped': 0 626 } 627 self.assertEqual(bt_cls.results.summary_dict(), expected_summary) 628 629 def test_abort_class(self): 630 class MockBaseTest(base_test.BaseTestClass): 631 def test_1(self): 632 pass 633 634 def test_2(self): 635 asserts.abort_class(MSG_EXPECTED_EXCEPTION) 636 never_call() 637 638 def test_3(self): 639 never_call() 640 641 bt_cls = MockBaseTest(self.mock_test_cls_configs) 642 bt_cls.run(test_names=['test_1', 'test_2', 'test_3']) 643 self.assertEqual(bt_cls.results.passed[0].test_name, 'test_1') 644 self.assertEqual(bt_cls.results.failed[0].details, 645 MSG_EXPECTED_EXCEPTION) 646 expected_summary = { 647 'Error': 0, 'Executed': 2, 648 'Failed': 1, 'Passed': 1, 'Requested': 3, 'Skipped': 0 649 } 650 self.assertEqual(bt_cls.results.summary_dict(), expected_summary) 651 652 def test_uncaught_exception(self): 653 class MockBaseTest(base_test.BaseTestClass): 654 def test_func(self): 655 raise Exception(MSG_EXPECTED_EXCEPTION) 656 never_call() 657 658 bt_cls = MockBaseTest(self.mock_test_cls_configs) 659 bt_cls.run(test_names=['test_func']) 660 actual_record = bt_cls.results.error[0] 661 self.assertEqual(actual_record.test_name, 'test_func') 662 self.assertEqual(actual_record.details, MSG_EXPECTED_EXCEPTION) 663 self.assertIsNone(actual_record.extras) 664 665 def test_fail(self): 666 class MockBaseTest(base_test.BaseTestClass): 667 def test_func(self): 668 asserts.fail(MSG_EXPECTED_EXCEPTION, extras=MOCK_EXTRA) 669 never_call() 670 671 bt_cls = MockBaseTest(self.mock_test_cls_configs) 672 bt_cls.run(test_names=['test_func']) 673 actual_record = bt_cls.results.failed[0] 674 self.assertEqual(actual_record.test_name, 'test_func') 675 self.assertEqual(actual_record.details, MSG_EXPECTED_EXCEPTION) 676 self.assertEqual(actual_record.extras, MOCK_EXTRA) 677 678 def test_assert_true(self): 679 class MockBaseTest(base_test.BaseTestClass): 680 def test_func(self): 681 asserts.assert_true( 682 False, MSG_EXPECTED_EXCEPTION, extras=MOCK_EXTRA) 683 never_call() 684 685 bt_cls = MockBaseTest(self.mock_test_cls_configs) 686 bt_cls.run(test_names=['test_func']) 687 actual_record = bt_cls.results.failed[0] 688 self.assertEqual(actual_record.test_name, 'test_func') 689 self.assertEqual(actual_record.details, MSG_EXPECTED_EXCEPTION) 690 self.assertEqual(actual_record.extras, MOCK_EXTRA) 691 692 def test_assert_equal_pass(self): 693 class MockBaseTest(base_test.BaseTestClass): 694 def test_func(self): 695 asserts.assert_equal(1, 1, extras=MOCK_EXTRA) 696 697 bt_cls = MockBaseTest(self.mock_test_cls_configs) 698 bt_cls.run() 699 actual_record = bt_cls.results.passed[0] 700 self.assertEqual(actual_record.test_name, 'test_func') 701 self.assertIsNone(actual_record.details) 702 self.assertIsNone(actual_record.extras) 703 704 def test_assert_equal_fail(self): 705 class MockBaseTest(base_test.BaseTestClass): 706 def test_func(self): 707 asserts.assert_equal(1, 2, extras=MOCK_EXTRA) 708 709 bt_cls = MockBaseTest(self.mock_test_cls_configs) 710 bt_cls.run() 711 actual_record = bt_cls.results.failed[0] 712 self.assertEqual(actual_record.test_name, 'test_func') 713 self.assertEqual(actual_record.details, '1 != 2') 714 self.assertEqual(actual_record.extras, MOCK_EXTRA) 715 716 def test_assert_equal_fail_with_msg(self): 717 class MockBaseTest(base_test.BaseTestClass): 718 def test_func(self): 719 asserts.assert_equal( 720 1, 2, msg=MSG_EXPECTED_EXCEPTION, extras=MOCK_EXTRA) 721 722 bt_cls = MockBaseTest(self.mock_test_cls_configs) 723 bt_cls.run() 724 actual_record = bt_cls.results.failed[0] 725 self.assertEqual(actual_record.test_name, 'test_func') 726 expected_msg = '1 != 2 ' + MSG_EXPECTED_EXCEPTION 727 self.assertEqual(actual_record.details, expected_msg) 728 self.assertEqual(actual_record.extras, MOCK_EXTRA) 729 730 def test_assert_raises_pass(self): 731 class MockBaseTest(base_test.BaseTestClass): 732 def test_func(self): 733 with asserts.assert_raises(SomeError, extras=MOCK_EXTRA): 734 raise SomeError(MSG_EXPECTED_EXCEPTION) 735 736 bt_cls = MockBaseTest(self.mock_test_cls_configs) 737 bt_cls.run() 738 actual_record = bt_cls.results.passed[0] 739 self.assertEqual(actual_record.test_name, 'test_func') 740 self.assertIsNone(actual_record.details) 741 self.assertIsNone(actual_record.extras) 742 743 def test_assert_raises_regex_pass(self): 744 class MockBaseTest(base_test.BaseTestClass): 745 def test_func(self): 746 with asserts.assert_raises_regex( 747 SomeError, 748 expected_regex=MSG_EXPECTED_EXCEPTION, 749 extras=MOCK_EXTRA): 750 raise SomeError(MSG_EXPECTED_EXCEPTION) 751 752 bt_cls = MockBaseTest(self.mock_test_cls_configs) 753 bt_cls.run() 754 actual_record = bt_cls.results.passed[0] 755 self.assertEqual(actual_record.test_name, 'test_func') 756 self.assertIsNone(actual_record.details) 757 self.assertIsNone(actual_record.extras) 758 759 def test_assert_raises_fail_with_noop(self): 760 class MockBaseTest(base_test.BaseTestClass): 761 def test_func(self): 762 with asserts.assert_raises_regex( 763 SomeError, 764 expected_regex=MSG_EXPECTED_EXCEPTION, 765 extras=MOCK_EXTRA): 766 pass 767 768 bt_cls = MockBaseTest(self.mock_test_cls_configs) 769 bt_cls.run() 770 actual_record = bt_cls.results.failed[0] 771 self.assertEqual(actual_record.test_name, 'test_func') 772 self.assertEqual(actual_record.details, 'SomeError not raised') 773 self.assertEqual(actual_record.extras, MOCK_EXTRA) 774 775 def test_assert_raises_fail_with_wrong_regex(self): 776 wrong_msg = 'ha' 777 778 class MockBaseTest(base_test.BaseTestClass): 779 def test_func(self): 780 with asserts.assert_raises_regex( 781 SomeError, 782 expected_regex=MSG_EXPECTED_EXCEPTION, 783 extras=MOCK_EXTRA): 784 raise SomeError(wrong_msg) 785 786 bt_cls = MockBaseTest(self.mock_test_cls_configs) 787 bt_cls.run() 788 actual_record = bt_cls.results.failed[0] 789 self.assertEqual(actual_record.test_name, 'test_func') 790 expected_details = ('"This is an expected exception." does not match ' 791 '"%s"') % wrong_msg 792 self.assertEqual(actual_record.details, expected_details) 793 self.assertEqual(actual_record.extras, MOCK_EXTRA) 794 795 def test_assert_raises_fail_with_wrong_error(self): 796 class MockBaseTest(base_test.BaseTestClass): 797 def test_func(self): 798 with asserts.assert_raises_regex( 799 SomeError, 800 expected_regex=MSG_EXPECTED_EXCEPTION, 801 extras=MOCK_EXTRA): 802 raise AttributeError(MSG_UNEXPECTED_EXCEPTION) 803 804 bt_cls = MockBaseTest(self.mock_test_cls_configs) 805 bt_cls.run() 806 actual_record = bt_cls.results.error[0] 807 self.assertEqual(actual_record.test_name, 'test_func') 808 self.assertEqual(actual_record.details, MSG_UNEXPECTED_EXCEPTION) 809 self.assertIsNone(actual_record.extras) 810 811 def test_explicit_pass(self): 812 class MockBaseTest(base_test.BaseTestClass): 813 def test_func(self): 814 asserts.explicit_pass( 815 MSG_EXPECTED_EXCEPTION, extras=MOCK_EXTRA) 816 never_call() 817 818 bt_cls = MockBaseTest(self.mock_test_cls_configs) 819 bt_cls.run(test_names=['test_func']) 820 actual_record = bt_cls.results.passed[0] 821 self.assertEqual(actual_record.test_name, 'test_func') 822 self.assertEqual(actual_record.details, MSG_EXPECTED_EXCEPTION) 823 self.assertEqual(actual_record.extras, MOCK_EXTRA) 824 825 def test_implicit_pass(self): 826 class MockBaseTest(base_test.BaseTestClass): 827 def test_func(self): 828 pass 829 830 bt_cls = MockBaseTest(self.mock_test_cls_configs) 831 bt_cls.run(test_names=['test_func']) 832 actual_record = bt_cls.results.passed[0] 833 self.assertEqual(actual_record.test_name, 'test_func') 834 self.assertIsNone(actual_record.details) 835 self.assertIsNone(actual_record.extras) 836 837 def test_skip(self): 838 class MockBaseTest(base_test.BaseTestClass): 839 def test_func(self): 840 asserts.skip(MSG_EXPECTED_EXCEPTION, extras=MOCK_EXTRA) 841 never_call() 842 843 bt_cls = MockBaseTest(self.mock_test_cls_configs) 844 bt_cls.run(test_names=['test_func']) 845 actual_record = bt_cls.results.skipped[0] 846 self.assertEqual(actual_record.test_name, 'test_func') 847 self.assertEqual(actual_record.details, MSG_EXPECTED_EXCEPTION) 848 self.assertEqual(actual_record.extras, MOCK_EXTRA) 849 850 def test_skip_if(self): 851 class MockBaseTest(base_test.BaseTestClass): 852 def test_func(self): 853 asserts.skip_if(False, MSG_UNEXPECTED_EXCEPTION) 854 asserts.skip_if( 855 True, MSG_EXPECTED_EXCEPTION, extras=MOCK_EXTRA) 856 never_call() 857 858 bt_cls = MockBaseTest(self.mock_test_cls_configs) 859 bt_cls.run(test_names=['test_func']) 860 actual_record = bt_cls.results.skipped[0] 861 self.assertEqual(actual_record.test_name, 'test_func') 862 self.assertEqual(actual_record.details, MSG_EXPECTED_EXCEPTION) 863 self.assertEqual(actual_record.extras, MOCK_EXTRA) 864 865 def test_unpack_userparams_required(self): 866 """Missing a required param should raise an error.""" 867 required = ['some_param'] 868 bc = base_test.BaseTestClass(self.mock_test_cls_configs) 869 bc.unpack_userparams(required) 870 expected_value = self.mock_test_cls_configs['user_params'][ 871 'some_param'] 872 self.assertEqual(bc.some_param, expected_value) 873 874 def test_unpack_userparams_required_missing(self): 875 """Missing a required param should raise an error.""" 876 required = ['something'] 877 bc = base_test.BaseTestClass(self.mock_test_cls_configs) 878 expected_msg = ('Missing required user param "%s" in test ' 879 'configuration.') % required[0] 880 with self.assertRaises(base_test.Error, msg=expected_msg): 881 bc.unpack_userparams(required) 882 883 def test_unpack_userparams_optional(self): 884 """If an optional param is specified, the value should be what's in the 885 config. 886 """ 887 opt = ['some_param'] 888 bc = base_test.BaseTestClass(self.mock_test_cls_configs) 889 bc.unpack_userparams(opt_param_names=opt) 890 expected_value = self.mock_test_cls_configs['user_params'][ 891 'some_param'] 892 self.assertEqual(bc.some_param, expected_value) 893 894 def test_unpack_userparams_optional_with_default(self): 895 """If an optional param is specified with a default value, and the 896 param is not in the config, the value should be the default value. 897 """ 898 bc = base_test.BaseTestClass(self.mock_test_cls_configs) 899 bc.unpack_userparams(optional_thing='whatever') 900 self.assertEqual(bc.optional_thing, 'whatever') 901 902 def test_unpack_userparams_default_overwrite_by_optional_param_list(self): 903 """If an optional param is specified in kwargs, and the param is in the 904 config, the value should be the one in the config. 905 """ 906 bc = base_test.BaseTestClass(self.mock_test_cls_configs) 907 bc.unpack_userparams(some_param='whatever') 908 expected_value = self.mock_test_cls_configs['user_params'][ 909 'some_param'] 910 self.assertEqual(bc.some_param, expected_value) 911 912 def test_unpack_userparams_default_overwrite_by_required_param_list(self): 913 """If an optional param is specified in kwargs, the param is in the 914 required param list, and the param is not specified in the config, the 915 param's alue should be the default value and there should be no error 916 thrown. 917 """ 918 bc = base_test.BaseTestClass(self.mock_test_cls_configs) 919 bc.unpack_userparams( 920 req_param_names=['a_kwarg_param'], a_kwarg_param='whatever') 921 self.assertEqual(bc.a_kwarg_param, 'whatever') 922 923 def test_unpack_userparams_optional_missing(self): 924 """Missing an optional param should not raise an error.""" 925 opt = ['something'] 926 bc = base_test.BaseTestClass(self.mock_test_cls_configs) 927 bc.unpack_userparams(opt_param_names=opt) 928 929 def test_unpack_userparams_basic(self): 930 """Required and optional params are unpacked properly.""" 931 required = ['something'] 932 optional = ['something_else'] 933 configs = dict(self.mock_test_cls_configs) 934 configs['user_params']['something'] = 42 935 configs['user_params']['something_else'] = 53 936 bc = base_test.BaseTestClass(configs) 937 bc.unpack_userparams( 938 req_param_names=required, opt_param_names=optional) 939 self.assertEqual(bc.something, 42) 940 self.assertEqual(bc.something_else, 53) 941 942 def test_unpack_userparams_default_overwrite(self): 943 default_arg_val = 'haha' 944 actual_arg_val = 'wawa' 945 arg_name = 'arg1' 946 configs = dict(self.mock_test_cls_configs) 947 configs['user_params'][arg_name] = actual_arg_val 948 bc = base_test.BaseTestClass(configs) 949 bc.unpack_userparams(opt_param_names=[arg_name], arg1=default_arg_val) 950 self.assertEqual(bc.arg1, actual_arg_val) 951 952 def test_unpack_userparams_default_None(self): 953 bc = base_test.BaseTestClass(self.mock_test_cls_configs) 954 bc.unpack_userparams(arg1='haha') 955 self.assertEqual(bc.arg1, 'haha') 956 957 def test_register_controller_no_config(self): 958 base_cls = base_test.BaseTestClass(self.mock_test_cls_configs) 959 with self.assertRaisesRegexp(signals.ControllerError, 960 'No corresponding config found for'): 961 base_cls.register_controller(mock_controller) 962 963 def test_register_optional_controller_no_config(self): 964 base_cls = base_test.BaseTestClass(self.mock_test_cls_configs) 965 self.assertIsNone( 966 base_cls.register_controller(mock_controller, required=False)) 967 968 def test_register_controller_third_party_dup_register(self): 969 """Verifies correctness of registration, internal tally of controllers 970 objects, and the right error happen when a controller module is 971 registered twice. 972 """ 973 mock_test_config = dict(self.mock_test_cls_configs) 974 mock_ctrlr_config_name = mock_controller.ACTS_CONTROLLER_CONFIG_NAME 975 mock_test_config[self.tb_key][mock_ctrlr_config_name] = [ 976 'magic1', 'magic2' 977 ] 978 base_cls = base_test.BaseTestClass(self.mock_test_cls_configs) 979 base_cls.register_controller(mock_controller) 980 registered_name = 'mock_controller' 981 controller_objects = base_cls._controller_manager._controller_objects 982 self.assertTrue(registered_name in controller_objects) 983 mock_ctrlrs = controller_objects[registered_name] 984 self.assertEqual(mock_ctrlrs[0].magic, 'magic1') 985 self.assertEqual(mock_ctrlrs[1].magic, 'magic2') 986 expected_msg = 'Controller module .* has already been registered.' 987 with self.assertRaisesRegexp(signals.ControllerError, expected_msg): 988 base_cls.register_controller(mock_controller) 989 990 def test_register_optional_controller_third_party_dup_register(self): 991 """Verifies correctness of registration, internal tally of controllers 992 objects, and the right error happen when an optional controller module 993 is registered twice. 994 """ 995 mock_test_config = dict(self.mock_test_cls_configs) 996 mock_ctrlr_config_name = mock_controller.ACTS_CONTROLLER_CONFIG_NAME 997 mock_test_config[self.tb_key][mock_ctrlr_config_name] = [ 998 'magic1', 'magic2' 999 ] 1000 base_cls = base_test.BaseTestClass(self.mock_test_cls_configs) 1001 base_cls.register_controller(mock_controller, required=False) 1002 expected_msg = 'Controller module .* has already been registered.' 1003 with self.assertRaisesRegexp(signals.ControllerError, expected_msg): 1004 base_cls.register_controller(mock_controller, required=False) 1005 1006 def test_register_controller_builtin_dup_register(self): 1007 """Same as test_register_controller_third_party_dup_register, except 1008 this is for a builtin controller module. 1009 """ 1010 mock_test_config = dict(self.mock_test_cls_configs) 1011 mock_ctrlr_config_name = mock_controller.ACTS_CONTROLLER_CONFIG_NAME 1012 mock_ref_name = 'haha' 1013 setattr(mock_controller, 'ACTS_CONTROLLER_REFERENCE_NAME', 1014 mock_ref_name) 1015 try: 1016 mock_ctrlr_ref_name = mock_controller.ACTS_CONTROLLER_REFERENCE_NAME 1017 mock_test_config[self.tb_key][mock_ctrlr_config_name] = [ 1018 'magic1', 'magic2' 1019 ] 1020 base_cls = base_test.BaseTestClass(self.mock_test_cls_configs) 1021 base_cls.register_controller(mock_controller, builtin=True) 1022 self.assertTrue(hasattr(base_cls, mock_ref_name)) 1023 self.assertTrue(mock_controller.__name__ in 1024 base_cls._controller_manager._controller_objects) 1025 mock_ctrlrs = getattr(base_cls, mock_ctrlr_ref_name) 1026 self.assertEqual(mock_ctrlrs[0].magic, 'magic1') 1027 self.assertEqual(mock_ctrlrs[1].magic, 'magic2') 1028 expected_msg = 'Controller module .* has already been registered.' 1029 with self.assertRaisesRegexp(signals.ControllerError, 1030 expected_msg): 1031 base_cls.register_controller(mock_controller, builtin=True) 1032 finally: 1033 delattr(mock_controller, 'ACTS_CONTROLLER_REFERENCE_NAME') 1034 1035 def test_register_controller_no_get_info(self): 1036 mock_test_config = dict(self.mock_test_cls_configs) 1037 mock_ctrlr_config_name = mock_controller.ACTS_CONTROLLER_CONFIG_NAME 1038 mock_ref_name = 'haha' 1039 get_info = getattr(mock_controller, 'get_info') 1040 delattr(mock_controller, 'get_info') 1041 try: 1042 mock_test_config[self.tb_key][mock_ctrlr_config_name] = [ 1043 'magic1', 'magic2' 1044 ] 1045 base_cls = base_test.BaseTestClass(self.mock_test_cls_configs) 1046 base_cls.register_controller(mock_controller) 1047 self.assertEqual(base_cls.results.controller_info, {}) 1048 finally: 1049 setattr(mock_controller, 'get_info', get_info) 1050 1051 def test_register_controller_return_value(self): 1052 mock_test_config = dict(self.mock_test_cls_configs) 1053 mock_ctrlr_config_name = mock_controller.ACTS_CONTROLLER_CONFIG_NAME 1054 mock_test_config[self.tb_key][mock_ctrlr_config_name] = [ 1055 'magic1', 'magic2' 1056 ] 1057 base_cls = base_test.BaseTestClass(self.mock_test_cls_configs) 1058 magic_devices = base_cls.register_controller(mock_controller) 1059 self.assertEqual(magic_devices[0].magic, 'magic1') 1060 self.assertEqual(magic_devices[1].magic, 'magic2') 1061 1062 1063if __name__ == '__main__': 1064 unittest.main() 1065