1# 2# Copyright (C) 2016 The Android Open Source Project 3# 4# Licensed under the Apache License, Version 2.0 (the "License"); 5# you may not use this file except in compliance with the License. 6# You may obtain a copy of the License at 7# 8# http://www.apache.org/licenses/LICENSE-2.0 9# 10# Unless required by applicable law or agreed to in writing, software 11# distributed under the License is distributed on an "AS IS" BASIS, 12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13# See the License for the specific language governing permissions and 14# limitations under the License. 15# 16 17import logging 18import os 19 20from vts.proto import VtsReportMessage_pb2 as ReportMsg 21from vts.runners.host import asserts 22from vts.runners.host import errors 23from vts.runners.host import keys 24from vts.runners.host import logger 25from vts.runners.host import records 26from vts.runners.host import signals 27from vts.runners.host import utils 28from vts.runners.host import const 29from vts.utils.python.common import list_utils 30from vts.utils.python.coverage import coverage_utils 31from vts.utils.python.profiling import profiling_utils 32from vts.utils.python.reporting import log_uploading_utils 33from vts.utils.python.systrace import systrace_utils 34from vts.utils.python.web import feature_utils 35from vts.utils.python.web import web_utils 36 37# Macro strings for test result reporting 38TEST_CASE_TOKEN = "[Test Case]" 39RESULT_LINE_TEMPLATE = TEST_CASE_TOKEN + " %s %s" 40STR_TEST = "test" 41STR_GENERATE = "generate" 42 43 44class BaseTestClass(object): 45 """Base class for all test classes to inherit from. 46 47 This class gets all the controller objects from test_runner and executes 48 the test cases requested within itself. 49 50 Most attributes of this class are set at runtime based on the configuration 51 provided. 52 53 Attributes: 54 tests: A list of strings, each representing a test case name. 55 TAG: A string used to refer to a test class. Default is the test class 56 name. 57 results: A records.TestResult object for aggregating test results from 58 the execution of test cases. 59 currentTestName: A string that's the name of the test case currently 60 being executed. If no test is executing, this should 61 be None. 62 include_filer: A list of string, each representing a test case name to 63 include. 64 exclude_filer: A list of string, each representing a test case name to 65 exclude. Has no effect if include_filer is not empty. 66 abi_name: String, name of abi in use 67 abi_bitness: String, bitness of abi in use 68 web: WebFeature, object storing web feature util for test run 69 coverage: CoverageFeature, object storing coverage feature util for test run 70 profiling: ProfilingFeature, object storing profiling feature util for test run 71 _skip_all_testcases: A boolean, can be set by a subclass in 72 setUpClass() to skip all test cases. 73 """ 74 75 TAG = None 76 77 def __init__(self, configs): 78 self.tests = [] 79 if not self.TAG: 80 self.TAG = self.__class__.__name__ 81 # Set all the controller objects and params. 82 for name, value in configs.items(): 83 setattr(self, name, value) 84 self.results = records.TestResult() 85 self.currentTestName = None 86 87 # Setup test filters (optional) 88 if keys.ConfigKeys.KEY_TEST_SUITE in self.user_params: 89 test_suite = self.user_params[keys.ConfigKeys.KEY_TEST_SUITE] 90 filters = [keys.ConfigKeys.KEY_INCLUDE_FILTER, 91 keys.ConfigKeys.KEY_EXCLUDE_FILTER] 92 for filter in filters: 93 if filter in test_suite: 94 setattr(self, filter, test_suite[filter]) 95 96 # TODO: get abi information differently for multi-device support. 97 # Set other optional parameters 98 opt_param_names = [keys.ConfigKeys.IKEY_ABI_NAME, 99 keys.ConfigKeys.IKEY_ABI_BITNESS, 100 keys.ConfigKeys.IKEY_SKIP_ON_32BIT_ABI, 101 keys.ConfigKeys.IKEY_SKIP_ON_64BIT_ABI, 102 keys.ConfigKeys.IKEY_RUN_32BIT_ON_64BIT_ABI] 103 self.getUserParams(opt_param_names=opt_param_names) 104 self.web = web_utils.WebFeature(self.user_params) 105 self.coverage = coverage_utils.CoverageFeature( 106 self.user_params, web=self.web) 107 self.profiling = profiling_utils.ProfilingFeature( 108 self.user_params, web=self.web) 109 self.systrace = systrace_utils.SystraceFeature( 110 self.user_params, web=self.web) 111 self.log_uploading = log_uploading_utils.LogUploadingFeature( 112 self.user_params, web=self.web) 113 self._skip_all_testcases = False 114 115 def __enter__(self): 116 return self 117 118 def __exit__(self, *args): 119 self._exec_func(self.cleanUp) 120 121 def getUserParams(self, req_param_names=[], opt_param_names=[], **kwargs): 122 """Unpacks user defined parameters in test config into individual 123 variables. 124 125 Instead of accessing the user param with self.user_params["xxx"], the 126 variable can be directly accessed with self.xxx. 127 128 A missing required param will raise an exception. If an optional param 129 is missing, an INFO line will be logged. 130 131 Args: 132 req_param_names: A list of names of the required user params. 133 opt_param_names: A list of names of the optional user params. 134 **kwargs: Arguments that provide default values. 135 e.g. getUserParams(required_list, opt_list, arg_a="hello") 136 self.arg_a will be "hello" unless it is specified again in 137 required_list or opt_list. 138 139 Raises: 140 BaseTestError is raised if a required user params is missing from 141 test config. 142 """ 143 for k, v in kwargs.items(): 144 setattr(self, k, v) 145 for name in req_param_names: 146 if name not in self.user_params: 147 raise errors.BaseTestError(("Missing required user param '%s' " 148 "in test configuration.") % name) 149 setattr(self, name, self.user_params[name]) 150 for name in opt_param_names: 151 if name not in self.user_params: 152 logging.info(("Missing optional user param '%s' in " 153 "configuration, continue."), name) 154 else: 155 setattr(self, name, self.user_params[name]) 156 157 def getUserParam(self, 158 param_name, 159 error_if_not_found=False, 160 log_warning_and_continue_if_not_found=False, 161 default_value=None): 162 """Get the value of a single user parameter. 163 164 This method returns the value of specified user parameter. 165 Note: this method will not automatically set attribute using the parameter name and value. 166 167 Args: 168 param_name: string or list of string, denoting user parameter names. If provided 169 a single string, self.user_params["<param_name>"] will be accessed. 170 If provided multiple strings, 171 self.user_params["<param_name1>"]["<param_name2>"]["<param_name3>"]... 172 will be accessed. 173 error_if_not_found: bool, whether to raise error if parameter not exists. Default: 174 False 175 log_warning_and_continue_if_not_found: bool, log a warning message if parameter value 176 not found. 177 default_value: object, default value to return if not found. If error_if_not_found is 178 True, this parameter has no effect. Default: None 179 180 Returns: 181 object, value of the specified parameter name chain if exists; 182 <default_value> if not exists. 183 """ 184 if not param_name: 185 if error_if_not_found: 186 raise errors.BaseTestError("empty param_name provided") 187 logging.error("empty param_name") 188 return default_value 189 190 if not isinstance(param_name, list): 191 param_name = [param_name] 192 193 curr_obj = self.user_params 194 for param in param_name: 195 if param not in curr_obj: 196 msg = "Missing user param '%s' in test configuration." % param_name 197 if error_if_not_found: 198 raise errors.BaseTestError(msg) 199 elif log_warning_and_continue_if_not_found: 200 logging.warn(msg) 201 return default_value 202 curr_obj = curr_obj[param] 203 204 return curr_obj 205 206 def _setUpClass(self): 207 """Proxy function to guarantee the base implementation of setUpClass 208 is called. 209 """ 210 return self.setUpClass() 211 212 def setUpClass(self): 213 """Setup function that will be called before executing any test case in 214 the test class. 215 216 To signal setup failure, return False or raise an exception. If 217 exceptions were raised, the stack trace would appear in log, but the 218 exceptions would not propagate to upper levels. 219 220 Implementation is optional. 221 """ 222 pass 223 224 def _tearDownClass(self): 225 """Proxy function to guarantee the base implementation of tearDownClass 226 is called. 227 """ 228 ret = self.tearDownClass() 229 if self.log_uploading.enabled: 230 self.log_uploading.UploadLogs() 231 if self.web.enabled: 232 self.web.Upload(self.results.requested, self.results.executed) 233 return ret 234 235 def tearDownClass(self): 236 """Teardown function that will be called after all the selected test 237 cases in the test class have been executed. 238 239 Implementation is optional. 240 """ 241 pass 242 243 def _testEntry(self, test_name): 244 """Internal function to be called upon entry of a test case.""" 245 self.currentTestName = test_name 246 if self.web.enabled: 247 self.web.AddTestReport(test_name) 248 249 def _setUp(self, test_name): 250 """Proxy function to guarantee the base implementation of setUp is 251 called. 252 """ 253 if self.systrace.enabled: 254 self.systrace.StartSystrace() 255 return self.setUp() 256 257 def setUp(self): 258 """Setup function that will be called every time before executing each 259 test case in the test class. 260 261 To signal setup failure, return False or raise an exception. If 262 exceptions were raised, the stack trace would appear in log, but the 263 exceptions would not propagate to upper levels. 264 265 Implementation is optional. 266 """ 267 268 def _testExit(self, test_name): 269 """Internal function to be called upon exit of a test.""" 270 self.currentTestName = None 271 272 def _tearDown(self, test_name): 273 """Proxy function to guarantee the base implementation of tearDown 274 is called. 275 """ 276 if self.systrace.enabled: 277 self.systrace.ProcessAndUploadSystrace(test_name) 278 self.tearDown() 279 280 def tearDown(self): 281 """Teardown function that will be called every time a test case has 282 been executed. 283 284 Implementation is optional. 285 """ 286 287 def _onFail(self, record): 288 """Proxy function to guarantee the base implementation of onFail is 289 called. 290 291 Args: 292 record: The records.TestResultRecord object for the failed test 293 case. 294 """ 295 test_name = record.test_name 296 logging.error(record.details) 297 begin_time = logger.epochToLogLineTimestamp(record.begin_time) 298 logging.info(RESULT_LINE_TEMPLATE, test_name, record.result) 299 if self.web.enabled: 300 self.web.SetTestResult(ReportMsg.TEST_CASE_RESULT_FAIL) 301 self.onFail(test_name, begin_time) 302 303 def onFail(self, test_name, begin_time): 304 """A function that is executed upon a test case failure. 305 306 User implementation is optional. 307 308 Args: 309 test_name: Name of the test that triggered this function. 310 begin_time: Logline format timestamp taken when the test started. 311 """ 312 313 def _onPass(self, record): 314 """Proxy function to guarantee the base implementation of onPass is 315 called. 316 317 Args: 318 record: The records.TestResultRecord object for the passed test 319 case. 320 """ 321 test_name = record.test_name 322 begin_time = logger.epochToLogLineTimestamp(record.begin_time) 323 msg = record.details 324 if msg: 325 logging.info(msg) 326 logging.info(RESULT_LINE_TEMPLATE, test_name, record.result) 327 if self.web.enabled: 328 self.web.SetTestResult(ReportMsg.TEST_CASE_RESULT_PASS) 329 self.onPass(test_name, begin_time) 330 331 def onPass(self, test_name, begin_time): 332 """A function that is executed upon a test case passing. 333 334 Implementation is optional. 335 336 Args: 337 test_name: Name of the test that triggered this function. 338 begin_time: Logline format timestamp taken when the test started. 339 """ 340 341 def _onSkip(self, record): 342 """Proxy function to guarantee the base implementation of onSkip is 343 called. 344 345 Args: 346 record: The records.TestResultRecord object for the skipped test 347 case. 348 """ 349 test_name = record.test_name 350 begin_time = logger.epochToLogLineTimestamp(record.begin_time) 351 logging.info(RESULT_LINE_TEMPLATE, test_name, record.result) 352 logging.info("Reason to skip: %s", record.details) 353 if self.web.enabled: 354 self.web.SetTestResult(ReportMsg.TEST_CASE_RESULT_SKIP) 355 self.onSkip(test_name, begin_time) 356 357 def onSkip(self, test_name, begin_time): 358 """A function that is executed upon a test case being skipped. 359 360 Implementation is optional. 361 362 Args: 363 test_name: Name of the test that triggered this function. 364 begin_time: Logline format timestamp taken when the test started. 365 """ 366 367 def _onSilent(self, record): 368 """Proxy function to guarantee the base implementation of onSilent is 369 called. 370 371 Args: 372 record: The records.TestResultRecord object for the skipped test 373 case. 374 """ 375 test_name = record.test_name 376 begin_time = logger.epochToLogLineTimestamp(record.begin_time) 377 if self.web.enabled: 378 self.web.SetTestResult(None) 379 self.onSilent(test_name, begin_time) 380 381 def onSilent(self, test_name, begin_time): 382 """A function that is executed upon a test case being marked as silent. 383 384 Implementation is optional. 385 386 Args: 387 test_name: Name of the test that triggered this function. 388 begin_time: Logline format timestamp taken when the test started. 389 """ 390 391 def _onException(self, record): 392 """Proxy function to guarantee the base implementation of onException 393 is called. 394 395 Args: 396 record: The records.TestResultRecord object for the failed test 397 case. 398 """ 399 test_name = record.test_name 400 logging.exception(record.details) 401 begin_time = logger.epochToLogLineTimestamp(record.begin_time) 402 if self.web.enabled: 403 self.web.SetTestResult(ReportMsg.TEST_CASE_RESULT_EXCEPTION) 404 self.onException(test_name, begin_time) 405 406 def onException(self, test_name, begin_time): 407 """A function that is executed upon an unhandled exception from a test 408 case. 409 410 Implementation is optional. 411 412 Args: 413 test_name: Name of the test that triggered this function. 414 begin_time: Logline format timestamp taken when the test started. 415 """ 416 417 def _exec_procedure_func(self, func, tr_record): 418 """Executes a procedure function like onPass, onFail etc. 419 420 This function will alternate the 'Result' of the test's record if 421 exceptions happened when executing the procedure function. 422 423 This will let signals.TestAbortAll through so abortAll works in all 424 procedure functions. 425 426 Args: 427 func: The procedure function to be executed. 428 tr_record: The TestResultRecord object associated with the test 429 case executed. 430 """ 431 try: 432 func(tr_record) 433 except signals.TestAbortAll: 434 raise 435 except Exception as e: 436 logging.exception("Exception happened when executing %s for %s.", 437 func.__name__, self.currentTestName) 438 tr_record.addError(func.__name__, e) 439 440 def filterOneTest(self, test_name): 441 """Check test filters for a test name. 442 443 The first layer of filter is user defined test filters: 444 if a include filter is not empty, only tests in include filter will 445 be executed regardless whether they are also in exclude filter. Else 446 if include filter is empty, only tests not in exclude filter will be 447 executed. 448 449 The second layer of filter is checking _skip_all_testcases flag: 450 the subclass may set _skip_all_testcases to True in its implementation 451 of setUpClass. If the flag is set, this method raises signals.TestSkip. 452 453 The third layer of filter is checking abi bitness: 454 if a test has a suffix indicating the intended architecture bitness, 455 and the current abi bitness information is available, non matching tests 456 will be skipped. By our convention, this function will look for bitness in suffix 457 formated as "32bit", "32Bit", "32BIT", or 64 bit equivalents. 458 459 This method assumes const.SUFFIX_32BIT and const.SUFFIX_64BIT are in lower cases. 460 461 Args: 462 test_name: string, name of a test 463 464 Raises: 465 signals.TestSilent if a test should not be executed 466 signals.TestSkip if a test should be logged but not be executed 467 """ 468 if (hasattr(self, keys.ConfigKeys.KEY_INCLUDE_FILTER) and 469 getattr(self, keys.ConfigKeys.KEY_INCLUDE_FILTER)): 470 if test_name not in getattr(self, 471 keys.ConfigKeys.KEY_INCLUDE_FILTER): 472 logging.info("Test case '%s' not in include filter." % 473 test_name) 474 raise signals.TestSilent( 475 "Test case '%s' not in include filter." % test_name) 476 elif (hasattr(self, keys.ConfigKeys.KEY_EXCLUDE_FILTER) and 477 test_name in getattr(self, keys.ConfigKeys.KEY_EXCLUDE_FILTER)): 478 logging.info("Test case '%s' in exclude filter." % test_name) 479 raise signals.TestSilent("Test case '%s' in exclude filter." % 480 test_name) 481 482 if self._skip_all_testcases: 483 raise signals.TestSkip("All test cases skipped.") 484 485 if hasattr(self, keys.ConfigKeys.IKEY_ABI_BITNESS): 486 bitness = getattr(self, keys.ConfigKeys.IKEY_ABI_BITNESS) 487 run_32bit_on_64bit_abi = getattr( 488 self, keys.ConfigKeys.IKEY_RUN_32BIT_ON_64BIT_ABI, False) 489 490 skip_on_32bit_abi = getattr( 491 self, keys.ConfigKeys.IKEY_SKIP_ON_32BIT_ABI, False) 492 skip_on_64bit_abi = getattr( 493 self, keys.ConfigKeys.IKEY_SKIP_ON_64BIT_ABI, False) 494 495 asserts.skipIf( 496 ((skip_on_32bit_abi is True) and bitness == "32") or ( 497 (skip_on_64bit_abi is True) and bitness == "64") or 498 (test_name.lower().endswith(const.SUFFIX_32BIT) and 499 bitness != "32") or ( 500 test_name.lower().endswith(const.SUFFIX_64BIT) and 501 bitness != "64" and not run_32bit_on_64bit_abi), 502 "Test case '{}' excluded as ABI bitness is {}.".format( 503 test_name, bitness)) 504 505 def execOneTest(self, test_name, test_func, args, **kwargs): 506 """Executes one test case and update test results. 507 508 Executes one test case, create a records.TestResultRecord object with 509 the execution information, and add the record to the test class's test 510 results. 511 512 Args: 513 test_name: Name of the test. 514 test_func: The test function. 515 args: A tuple of params. 516 kwargs: Extra kwargs. 517 """ 518 is_silenced = False 519 tr_record = records.TestResultRecord(test_name, self.TAG) 520 tr_record.testBegin() 521 logging.info("%s %s", TEST_CASE_TOKEN, test_name) 522 verdict = None 523 try: 524 ret = self._testEntry(test_name) 525 asserts.assertTrue(ret is not False, 526 "Setup test entry for %s failed." % test_name) 527 self.filterOneTest(test_name) 528 try: 529 ret = self._setUp(test_name) 530 asserts.assertTrue(ret is not False, 531 "Setup for %s failed." % test_name) 532 533 if args or kwargs: 534 verdict = test_func(*args, **kwargs) 535 else: 536 verdict = test_func() 537 finally: 538 self._tearDown(test_name) 539 except (signals.TestFailure, AssertionError) as e: 540 tr_record.testFail(e) 541 self._exec_procedure_func(self._onFail, tr_record) 542 except signals.TestSkip as e: 543 # Test skipped. 544 tr_record.testSkip(e) 545 self._exec_procedure_func(self._onSkip, tr_record) 546 except (signals.TestAbortClass, signals.TestAbortAll) as e: 547 # Abort signals, pass along. 548 tr_record.testFail(e) 549 raise e 550 except signals.TestPass as e: 551 # Explicit test pass. 552 tr_record.testPass(e) 553 self._exec_procedure_func(self._onPass, tr_record) 554 except signals.TestSilent as e: 555 # Suppress test reporting. 556 is_silenced = True 557 self._exec_procedure_func(self._onSilent, tr_record) 558 self.results.requested.remove(test_name) 559 except Exception as e: 560 # Exception happened during test. 561 logging.exception(e) 562 tr_record.testError(e) 563 self._exec_procedure_func(self._onException, tr_record) 564 self._exec_procedure_func(self._onFail, tr_record) 565 else: 566 # Keep supporting return False for now. 567 # TODO(angli): Deprecate return False support. 568 if verdict or (verdict is None): 569 # Test passed. 570 tr_record.testPass() 571 self._exec_procedure_func(self._onPass, tr_record) 572 return 573 # Test failed because it didn't return True. 574 # This should be removed eventually. 575 tr_record.testFail() 576 self._exec_procedure_func(self._onFail, tr_record) 577 finally: 578 if not is_silenced: 579 self.results.addRecord(tr_record) 580 self._testExit(test_name) 581 582 def runGeneratedTests(self, 583 test_func, 584 settings, 585 args=None, 586 kwargs=None, 587 tag="", 588 name_func=None): 589 """Runs generated test cases. 590 591 Generated test cases are not written down as functions, but as a list 592 of parameter sets. This way we reduce code repetition and improve 593 test case scalability. 594 595 Args: 596 test_func: The common logic shared by all these generated test 597 cases. This function should take at least one argument, 598 which is a parameter set. 599 settings: A list of strings representing parameter sets. These are 600 usually json strings that get loaded in the test_func. 601 args: Iterable of additional position args to be passed to 602 test_func. 603 kwargs: Dict of additional keyword args to be passed to test_func 604 tag: Name of this group of generated test cases. Ignored if 605 name_func is provided and operates properly. 606 name_func: A function that takes a test setting and generates a 607 proper test name. The test name should be shorter than 608 utils.MAX_FILENAME_LEN. Names over the limit will be 609 truncated. 610 611 Returns: 612 A list of settings that did not pass. 613 """ 614 args = args or () 615 kwargs = kwargs or {} 616 failed_settings = [] 617 for s in settings: 618 test_name = "{} {}".format(tag, s) 619 if name_func: 620 try: 621 test_name = name_func(s, *args, **kwargs) 622 except: 623 logging.exception(("Failed to get test name from " 624 "test_func. Fall back to default %s"), 625 test_name) 626 self.results.requested.append(test_name) 627 if len(test_name) > utils.MAX_FILENAME_LEN: 628 test_name = test_name[:utils.MAX_FILENAME_LEN] 629 previous_success_cnt = len(self.results.passed) 630 self.execOneTest(test_name, test_func, (s, ) + args, **kwargs) 631 if len(self.results.passed) - previous_success_cnt != 1: 632 failed_settings.append(s) 633 return failed_settings 634 635 def _exec_func(self, func, *args): 636 """Executes a function with exception safeguard. 637 638 This will let signals.TestAbortAll through so abortAll works in all 639 procedure functions. 640 641 Args: 642 func: Function to be executed. 643 args: Arguments to be passed to the function. 644 645 Returns: 646 Whatever the function returns, or False if unhandled exception 647 occured. 648 """ 649 try: 650 return func(*args) 651 except signals.TestAbortAll: 652 raise 653 except: 654 logging.exception("Exception happened when executing %s in %s.", 655 func.__name__, self.TAG) 656 return False 657 658 def _get_all_test_names(self): 659 """Finds all the function names that match the test case naming 660 convention in this class. 661 662 Returns: 663 A list of strings, each is a test case name. 664 """ 665 test_names = [] 666 for name in dir(self): 667 if name.startswith(STR_TEST) or name.startswith(STR_GENERATE): 668 attr_func = getattr(self, name) 669 if hasattr(attr_func, "__call__"): 670 test_names.append(name) 671 return test_names 672 673 def _get_test_funcs(self, test_names): 674 """Obtain the actual functions of test cases based on test names. 675 676 Args: 677 test_names: A list of strings, each string is a test case name. 678 679 Returns: 680 A list of tuples of (string, function). String is the test case 681 name, function is the actual test case function. 682 683 Raises: 684 errors.USERError is raised if the test name does not follow 685 naming convention "test_*". This can only be caused by user input 686 here. 687 """ 688 test_funcs = [] 689 for test_name in test_names: 690 if not hasattr(self, test_name): 691 logging.warning("%s does not have test case %s.", self.TAG, 692 test_name) 693 elif (test_name.startswith(STR_TEST) or 694 test_name.startswith(STR_GENERATE)): 695 test_funcs.append((test_name, getattr(self, test_name))) 696 else: 697 msg = ("Test case name %s does not follow naming convention " 698 "test*, abort.") % test_name 699 raise errors.USERError(msg) 700 701 return test_funcs 702 703 def run(self, test_names=None): 704 """Runs test cases within a test class by the order they appear in the 705 execution list. 706 707 One of these test cases lists will be executed, shown here in priority 708 order: 709 1. The test_names list, which is passed from cmd line. Invalid names 710 are guarded by cmd line arg parsing. 711 2. The self.tests list defined in test class. Invalid names are 712 ignored. 713 3. All function that matches test case naming convention in the test 714 class. 715 716 Args: 717 test_names: A list of string that are test case names requested in 718 cmd line. 719 720 Returns: 721 The test results object of this class. 722 """ 723 logging.info("==========> %s <==========", self.TAG) 724 # Devise the actual test cases to run in the test class. 725 if not test_names: 726 if self.tests: 727 # Specified by run list in class. 728 test_names = list(self.tests) 729 else: 730 # No test case specified by user, execute all in the test class 731 test_names = self._get_all_test_names() 732 self.results.requested = [test_name for test_name in test_names 733 if test_name.startswith(STR_TEST)] 734 tests = self._get_test_funcs(test_names) 735 # Setup for the class. 736 try: 737 if self._setUpClass() is False: 738 raise signals.TestFailure("Failed to setup %s." % self.TAG) 739 except Exception as e: 740 logging.exception("Failed to setup %s.", self.TAG) 741 self.results.failClass(self.TAG, e) 742 self._exec_func(self._tearDownClass) 743 return self.results 744 # Run tests in order. 745 try: 746 for test_name, test_func in tests: 747 if test_name.startswith(STR_GENERATE): 748 logging.info( 749 "Executing generated test trigger function '%s'", 750 test_name) 751 test_func() 752 logging.info("Finished '%s'", test_name) 753 else: 754 self.execOneTest(test_name, test_func, None) 755 if self._skip_all_testcases and not self.results.executed: 756 self.results.skipClass(self.TAG, 757 "All test cases skipped; unable to find any test case.") 758 return self.results 759 except signals.TestAbortClass: 760 logging.info("Received TestAbortClass signal") 761 return self.results 762 except signals.TestAbortAll as e: 763 logging.info("Received TestAbortAll signal") 764 # Piggy-back test results on this exception object so we don't lose 765 # results from this test class. 766 setattr(e, "results", self.results) 767 raise e 768 except Exception as e: 769 # Exception happened during test. 770 logging.exception(e) 771 raise e 772 finally: 773 self._exec_func(self._tearDownClass) 774 logging.info("Summary for test class %s: %s", self.TAG, 775 self.results.summary()) 776 777 def cleanUp(self): 778 """A function that is executed upon completion of all tests cases 779 selected in the test class. 780 781 This function should clean up objects initialized in the constructor by 782 user. 783 """ 784