1# 2# Copyright (C) 2016 The Android Open Source Project 3# 4# Licensed under the Apache License, Version 2.0 (the "License"); 5# you may not use this file except in compliance with the License. 6# You may obtain a copy of the License at 7# 8# http://www.apache.org/licenses/LICENSE-2.0 9# 10# Unless required by applicable law or agreed to in writing, software 11# distributed under the License is distributed on an "AS IS" BASIS, 12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13# See the License for the specific language governing permissions and 14# limitations under the License. 15# 16 17import logging 18import os 19import re 20 21from vts.proto import VtsReportMessage_pb2 as ReportMsg 22from vts.runners.host import asserts 23from vts.runners.host import const 24from vts.runners.host import errors 25from vts.runners.host import keys 26from vts.runners.host import logger 27from vts.runners.host import records 28from vts.runners.host import signals 29from vts.runners.host import utils 30from vts.utils.python.controllers import android_device 31from vts.utils.python.common import filter_utils 32from vts.utils.python.common import list_utils 33from vts.utils.python.coverage import coverage_utils 34from vts.utils.python.profiling import profiling_utils 35from vts.utils.python.reporting import log_uploading_utils 36from vts.utils.python.systrace import systrace_utils 37from vts.utils.python.web import feature_utils 38from vts.utils.python.web import web_utils 39 40# Macro strings for test result reporting 41TEST_CASE_TOKEN = "[Test Case]" 42RESULT_LINE_TEMPLATE = TEST_CASE_TOKEN + " %s %s" 43STR_TEST = "test" 44STR_GENERATE = "generate" 45_REPORT_MESSAGE_FILE_NAME = "report_proto.msg" 46_BUG_REPORT_FILE_PREFIX = "bugreport" 47_BUG_REPORT_FILE_EXTENSION = ".zip" 48_ANDROID_DEVICES = '_android_devices' 49 50 51class BaseTestClass(object): 52 """Base class for all test classes to inherit from. 53 54 This class gets all the controller objects from test_runner and executes 55 the test cases requested within itself. 56 57 Most attributes of this class are set at runtime based on the configuration 58 provided. 59 60 Attributes: 61 android_devices: A list of AndroidDevice object, representing android 62 devices. 63 tests: A list of strings, each representing a test case name. 64 TAG: A string used to refer to a test class. Default is the test class 65 name. 66 results: A records.TestResult object for aggregating test results from 67 the execution of test cases. 68 _current_record: A records.TestResultRecord object for the test case 69 currently being executed. If no test is running, this 70 should be None. 71 include_filer: A list of string, each representing a test case name to 72 include. 73 exclude_filer: A list of string, each representing a test case name to 74 exclude. Has no effect if include_filer is not empty. 75 abi_name: String, name of abi in use 76 abi_bitness: String, bitness of abi in use 77 web: WebFeature, object storing web feature util for test run 78 coverage: CoverageFeature, object storing coverage feature util for test run 79 profiling: ProfilingFeature, object storing profiling feature util for test run 80 _skip_all_testcases: A boolean, can be set by a subclass in 81 setUpClass() to skip all test cases. 82 _bug_report_on_failure: bool, whether to catch bug report at the end 83 of failed test cases. 84 test_filter: Filter object to filter test names. 85 """ 86 TAG = None 87 88 def __init__(self, configs): 89 self.tests = [] 90 if not self.TAG: 91 self.TAG = self.__class__.__name__ 92 # Set all the controller objects and params. 93 for name, value in configs.items(): 94 setattr(self, name, value) 95 self.results = records.TestResult() 96 self._current_record = None 97 98 # Setup test filters 99 self.include_filter = self.getUserParam( 100 [ 101 keys.ConfigKeys.KEY_TEST_SUITE, 102 keys.ConfigKeys.KEY_INCLUDE_FILTER 103 ], 104 default_value=[]) 105 self.exclude_filter = self.getUserParam( 106 [ 107 keys.ConfigKeys.KEY_TEST_SUITE, 108 keys.ConfigKeys.KEY_EXCLUDE_FILTER 109 ], 110 default_value=[]) 111 112 # TODO(yuexima): remove include_filter and exclude_filter from class attributes 113 # after confirming all modules no longer have reference to them 114 self.include_filter = list_utils.ExpandItemDelimiters( 115 list_utils.ItemsToStr(self.include_filter), ',') 116 self.exclude_filter = list_utils.ExpandItemDelimiters( 117 list_utils.ItemsToStr(self.exclude_filter), ',') 118 exclude_over_include = self.getUserParam( 119 keys.ConfigKeys.KEY_EXCLUDE_OVER_INCLUDE, default_value=None) 120 self.test_module_name = self.getUserParam(keys.ConfigKeys.KEY_TESTBED_NAME, 121 default_value=None) 122 self.test_filter = filter_utils.Filter( 123 self.include_filter, 124 self.exclude_filter, 125 enable_regex=True, 126 exclude_over_include=exclude_over_include, 127 enable_negative_pattern=True, 128 enable_module_name_prefix_matching=True, 129 module_name=self.test_module_name) 130 self.test_filter.ExpandBitness() 131 logging.info('Test filter: %s' % self.test_filter) 132 133 # TODO: get abi information differently for multi-device support. 134 # Set other optional parameters 135 self.abi_name = self.getUserParam( 136 keys.ConfigKeys.IKEY_ABI_NAME, default_value=None) 137 self.abi_bitness = self.getUserParam( 138 keys.ConfigKeys.IKEY_ABI_BITNESS, default_value=None) 139 self.skip_on_32bit_abi = self.getUserParam( 140 keys.ConfigKeys.IKEY_SKIP_ON_32BIT_ABI, default_value=False) 141 self.skip_on_64bit_abi = self.getUserParam( 142 keys.ConfigKeys.IKEY_SKIP_ON_64BIT_ABI, default_value=False) 143 self.run_32bit_on_64bit_abi = self.getUserParam( 144 keys.ConfigKeys.IKEY_RUN_32BIT_ON_64BIT_ABI, default_value=False) 145 self.web = web_utils.WebFeature(self.user_params) 146 self.coverage = coverage_utils.CoverageFeature( 147 self.user_params, web=self.web) 148 self.profiling = profiling_utils.ProfilingFeature( 149 self.user_params, web=self.web) 150 self.systrace = systrace_utils.SystraceFeature( 151 self.user_params, web=self.web) 152 self.log_uploading = log_uploading_utils.LogUploadingFeature( 153 self.user_params, web=self.web) 154 self._skip_all_testcases = False 155 self._bug_report_on_failure = self.getUserParam( 156 keys.ConfigKeys.IKEY_BUG_REPORT_ON_FAILURE, default_value=False) 157 158 @property 159 def android_devices(self): 160 """Returns a list of AndroidDevice objects""" 161 if not hasattr(self, _ANDROID_DEVICES): 162 setattr(self, _ANDROID_DEVICES, 163 self.registerController(android_device)) 164 return getattr(self, _ANDROID_DEVICES) 165 166 @android_devices.setter 167 def android_devices(self, devices): 168 """Set the list of AndroidDevice objects""" 169 setattr(self, _ANDROID_DEVICES, devices) 170 171 def __enter__(self): 172 return self 173 174 def __exit__(self, *args): 175 self._exec_func(self.cleanUp) 176 177 def getUserParams(self, req_param_names=[], opt_param_names=[], **kwargs): 178 """Unpacks user defined parameters in test config into individual 179 variables. 180 181 Instead of accessing the user param with self.user_params["xxx"], the 182 variable can be directly accessed with self.xxx. 183 184 A missing required param will raise an exception. If an optional param 185 is missing, an INFO line will be logged. 186 187 Args: 188 req_param_names: A list of names of the required user params. 189 opt_param_names: A list of names of the optional user params. 190 **kwargs: Arguments that provide default values. 191 e.g. getUserParams(required_list, opt_list, arg_a="hello") 192 self.arg_a will be "hello" unless it is specified again in 193 required_list or opt_list. 194 195 Raises: 196 BaseTestError is raised if a required user params is missing from 197 test config. 198 """ 199 for k, v in kwargs.items(): 200 setattr(self, k, v) 201 for name in req_param_names: 202 if name not in self.user_params: 203 raise errors.BaseTestError(("Missing required user param '%s' " 204 "in test configuration.") % name) 205 setattr(self, name, self.user_params[name]) 206 for name in opt_param_names: 207 if name not in self.user_params: 208 logging.info(("Missing optional user param '%s' in " 209 "configuration, continue."), name) 210 else: 211 setattr(self, name, self.user_params[name]) 212 213 def getUserParam(self, 214 param_name, 215 error_if_not_found=False, 216 log_warning_and_continue_if_not_found=False, 217 default_value=None, 218 to_str=False): 219 """Get the value of a single user parameter. 220 221 This method returns the value of specified user parameter. 222 Note: this method will not automatically set attribute using the parameter name and value. 223 224 Args: 225 param_name: string or list of string, denoting user parameter names. If provided 226 a single string, self.user_params["<param_name>"] will be accessed. 227 If provided multiple strings, 228 self.user_params["<param_name1>"]["<param_name2>"]["<param_name3>"]... 229 will be accessed. 230 error_if_not_found: bool, whether to raise error if parameter not exists. Default: 231 False 232 log_warning_and_continue_if_not_found: bool, log a warning message if parameter value 233 not found. 234 default_value: object, default value to return if not found. If error_if_not_found is 235 True, this parameter has no effect. Default: None 236 to_str: boolean, whether to convert the result object to string if not None. 237 Note, strings passing in from java json config are usually unicode. 238 239 Returns: 240 object, value of the specified parameter name chain if exists; 241 <default_value> if not exists. 242 """ 243 244 def ToStr(return_value): 245 """Check to_str option and convert to string if not None""" 246 if to_str and return_value is not None: 247 return str(return_value) 248 return return_value 249 250 if not param_name: 251 if error_if_not_found: 252 raise errors.BaseTestError("empty param_name provided") 253 logging.error("empty param_name") 254 return ToStr(default_value) 255 256 if not isinstance(param_name, list): 257 param_name = [param_name] 258 259 curr_obj = self.user_params 260 for param in param_name: 261 if param not in curr_obj: 262 msg = "Missing user param '%s' in test configuration." % param_name 263 if error_if_not_found: 264 raise errors.BaseTestError(msg) 265 elif log_warning_and_continue_if_not_found: 266 logging.warn(msg) 267 return ToStr(default_value) 268 curr_obj = curr_obj[param] 269 270 return ToStr(curr_obj) 271 272 def _setUpClass(self): 273 """Proxy function to guarantee the base implementation of setUpClass 274 is called. 275 """ 276 return self.setUpClass() 277 278 def setUpClass(self): 279 """Setup function that will be called before executing any test case in 280 the test class. 281 282 To signal setup failure, return False or raise an exception. If 283 exceptions were raised, the stack trace would appear in log, but the 284 exceptions would not propagate to upper levels. 285 286 Implementation is optional. 287 """ 288 pass 289 290 def _tearDownClass(self): 291 """Proxy function to guarantee the base implementation of tearDownClass 292 is called. 293 """ 294 ret = self.tearDownClass() 295 if self.log_uploading.enabled: 296 self.log_uploading.UploadLogs() 297 if self.web.enabled: 298 message_b = self.web.GenerateReportMessage(self.results.requested, 299 self.results.executed) 300 else: 301 message_b = '' 302 303 report_proto_path = os.path.join(logging.log_path, 304 _REPORT_MESSAGE_FILE_NAME) 305 306 if message_b: 307 logging.info('Result proto message path: %s', report_proto_path) 308 309 with open(report_proto_path, "wb") as f: 310 f.write(message_b) 311 312 return ret 313 314 def tearDownClass(self): 315 """Teardown function that will be called after all the selected test 316 cases in the test class have been executed. 317 318 Implementation is optional. 319 """ 320 pass 321 322 def _testEntry(self, test_record): 323 """Internal function to be called upon entry of a test case. 324 325 Args: 326 test_record: The TestResultRecord object for the test case going to 327 be executed. 328 """ 329 self._current_record = test_record 330 if self.web.enabled: 331 self.web.AddTestReport(test_record.test_name) 332 333 def _setUp(self, test_name): 334 """Proxy function to guarantee the base implementation of setUp is 335 called. 336 """ 337 if self.systrace.enabled: 338 self.systrace.StartSystrace() 339 return self.setUp() 340 341 def setUp(self): 342 """Setup function that will be called every time before executing each 343 test case in the test class. 344 345 To signal setup failure, return False or raise an exception. If 346 exceptions were raised, the stack trace would appear in log, but the 347 exceptions would not propagate to upper levels. 348 349 Implementation is optional. 350 """ 351 352 def _testExit(self): 353 """Internal function to be called upon exit of a test.""" 354 self._current_record = None 355 356 def _tearDown(self, test_name): 357 """Proxy function to guarantee the base implementation of tearDown 358 is called. 359 """ 360 if self.systrace.enabled: 361 self.systrace.ProcessAndUploadSystrace(test_name) 362 self.tearDown() 363 364 def tearDown(self): 365 """Teardown function that will be called every time a test case has 366 been executed. 367 368 Implementation is optional. 369 """ 370 371 def _onFail(self): 372 """Proxy function to guarantee the base implementation of onFail is 373 called. 374 """ 375 record = self._current_record 376 logging.error(record.details) 377 begin_time = logger.epochToLogLineTimestamp(record.begin_time) 378 logging.info(RESULT_LINE_TEMPLATE, record.test_name, record.result) 379 if self.web.enabled: 380 self.web.SetTestResult(ReportMsg.TEST_CASE_RESULT_FAIL) 381 self.onFail(record.test_name, begin_time) 382 if self._bug_report_on_failure: 383 self.CatchBugReport('%s-%s' % (self.TAG, record.test_name)) 384 385 def onFail(self, test_name, begin_time): 386 """A function that is executed upon a test case failure. 387 388 User implementation is optional. 389 390 Args: 391 test_name: Name of the test that triggered this function. 392 begin_time: Logline format timestamp taken when the test started. 393 """ 394 395 def _onPass(self): 396 """Proxy function to guarantee the base implementation of onPass is 397 called. 398 """ 399 record = self._current_record 400 test_name = record.test_name 401 begin_time = logger.epochToLogLineTimestamp(record.begin_time) 402 msg = record.details 403 if msg: 404 logging.info(msg) 405 logging.info(RESULT_LINE_TEMPLATE, test_name, record.result) 406 if self.web.enabled: 407 self.web.SetTestResult(ReportMsg.TEST_CASE_RESULT_PASS) 408 self.onPass(test_name, begin_time) 409 410 def onPass(self, test_name, begin_time): 411 """A function that is executed upon a test case passing. 412 413 Implementation is optional. 414 415 Args: 416 test_name: Name of the test that triggered this function. 417 begin_time: Logline format timestamp taken when the test started. 418 """ 419 420 def _onSkip(self): 421 """Proxy function to guarantee the base implementation of onSkip is 422 called. 423 """ 424 record = self._current_record 425 test_name = record.test_name 426 begin_time = logger.epochToLogLineTimestamp(record.begin_time) 427 logging.info(RESULT_LINE_TEMPLATE, test_name, record.result) 428 logging.info("Reason to skip: %s", record.details) 429 if self.web.enabled: 430 self.web.SetTestResult(ReportMsg.TEST_CASE_RESULT_SKIP) 431 self.onSkip(test_name, begin_time) 432 433 def onSkip(self, test_name, begin_time): 434 """A function that is executed upon a test case being skipped. 435 436 Implementation is optional. 437 438 Args: 439 test_name: Name of the test that triggered this function. 440 begin_time: Logline format timestamp taken when the test started. 441 """ 442 443 def _onSilent(self): 444 """Proxy function to guarantee the base implementation of onSilent is 445 called. 446 """ 447 record = self._current_record 448 test_name = record.test_name 449 begin_time = logger.epochToLogLineTimestamp(record.begin_time) 450 if self.web.enabled: 451 self.web.SetTestResult(None) 452 self.onSilent(test_name, begin_time) 453 454 def onSilent(self, test_name, begin_time): 455 """A function that is executed upon a test case being marked as silent. 456 457 Implementation is optional. 458 459 Args: 460 test_name: Name of the test that triggered this function. 461 begin_time: Logline format timestamp taken when the test started. 462 """ 463 464 def _onException(self): 465 """Proxy function to guarantee the base implementation of onException 466 is called. 467 """ 468 record = self._current_record 469 test_name = record.test_name 470 logging.exception(record.details) 471 begin_time = logger.epochToLogLineTimestamp(record.begin_time) 472 if self.web.enabled: 473 self.web.SetTestResult(ReportMsg.TEST_CASE_RESULT_EXCEPTION) 474 self.onException(test_name, begin_time) 475 if self._bug_report_on_failure: 476 self.CatchBugReport('%s-%s' % (self.TAG, record.test_name)) 477 478 def onException(self, test_name, begin_time): 479 """A function that is executed upon an unhandled exception from a test 480 case. 481 482 Implementation is optional. 483 484 Args: 485 test_name: Name of the test that triggered this function. 486 begin_time: Logline format timestamp taken when the test started. 487 """ 488 489 def _exec_procedure_func(self, func): 490 """Executes a procedure function like onPass, onFail etc. 491 492 This function will alternate the 'Result' of the test's record if 493 exceptions happened when executing the procedure function. 494 495 This will let signals.TestAbortAll through so abortAll works in all 496 procedure functions. 497 498 Args: 499 func: The procedure function to be executed. 500 """ 501 record = self._current_record 502 if record is None: 503 logging.error("Cannot execute %s. No record for current test.", 504 func.__name__) 505 return 506 try: 507 func() 508 except signals.TestAbortAll: 509 raise 510 except Exception as e: 511 logging.exception("Exception happened when executing %s for %s.", 512 func.__name__, record.test_name) 513 record.addError(func.__name__, e) 514 515 def addTableToResult(self, name, rows): 516 """Adds a table to current test record. 517 518 A subclass can call this method to add a table to _current_record when 519 running test cases. 520 521 Args: 522 name: String, the table name. 523 rows: A 2-dimensional list which contains the data. 524 """ 525 self._current_record.addTable(name, rows) 526 527 def filterOneTest(self, test_name, test_filter=None): 528 """Check test filters for a test name. 529 530 The first layer of filter is user defined test filters: 531 if a include filter is not empty, only tests in include filter will 532 be executed regardless whether they are also in exclude filter. Else 533 if include filter is empty, only tests not in exclude filter will be 534 executed. 535 536 The second layer of filter is checking _skip_all_testcases flag: 537 the subclass may set _skip_all_testcases to True in its implementation 538 of setUpClass. If the flag is set, this method raises signals.TestSkip. 539 540 The third layer of filter is checking abi bitness: 541 if a test has a suffix indicating the intended architecture bitness, 542 and the current abi bitness information is available, non matching tests 543 will be skipped. By our convention, this function will look for bitness in suffix 544 formated as "32bit", "32Bit", "32BIT", or 64 bit equivalents. 545 546 This method assumes const.SUFFIX_32BIT and const.SUFFIX_64BIT are in lower cases. 547 548 Args: 549 test_name: string, name of a test 550 test_filter: Filter object, test filter 551 552 Raises: 553 signals.TestSilent if a test should not be executed 554 signals.TestSkip if a test should be logged but not be executed 555 """ 556 self._filterOneTestThroughTestFilter(test_name, test_filter) 557 self._filterOneTestThroughAbiBitness(test_name) 558 559 def _filterOneTestThroughTestFilter(self, test_name, test_filter=None): 560 """Check test filter for the given test name. 561 562 Args: 563 test_name: string, name of a test 564 565 Raises: 566 signals.TestSilent if a test should not be executed 567 signals.TestSkip if a test should be logged but not be executed 568 """ 569 if not test_filter: 570 test_filter = self.test_filter 571 572 if not test_filter.Filter(test_name): 573 raise signals.TestSilent("Test case '%s' did not pass filters.") 574 575 if self._skip_all_testcases: 576 raise signals.TestSkip("All test cases skipped.") 577 578 def _filterOneTestThroughAbiBitness(self, test_name): 579 """Check test filter for the given test name. 580 581 Args: 582 test_name: string, name of a test 583 584 Raises: 585 signals.TestSilent if a test should not be executed 586 """ 587 asserts.skipIf( 588 self.abi_bitness and 589 ((self.skip_on_32bit_abi is True) and self.abi_bitness == "32") or 590 ((self.skip_on_64bit_abi is True) and self.abi_bitness == "64") or 591 (test_name.lower().endswith(const.SUFFIX_32BIT) and 592 self.abi_bitness != "32") or 593 (test_name.lower().endswith(const.SUFFIX_64BIT) and 594 self.abi_bitness != "64" and not self.run_32bit_on_64bit_abi), 595 "Test case '{}' excluded as ABI bitness is {}.".format( 596 test_name, self.abi_bitness)) 597 598 def execOneTest(self, test_name, test_func, args, **kwargs): 599 """Executes one test case and update test results. 600 601 Executes one test case, create a records.TestResultRecord object with 602 the execution information, and add the record to the test class's test 603 results. 604 605 Args: 606 test_name: Name of the test. 607 test_func: The test function. 608 args: A tuple of params. 609 kwargs: Extra kwargs. 610 """ 611 is_silenced = False 612 tr_record = records.TestResultRecord(test_name, self.TAG) 613 tr_record.testBegin() 614 logging.info("%s %s", TEST_CASE_TOKEN, test_name) 615 verdict = None 616 finished = False 617 try: 618 ret = self._testEntry(tr_record) 619 asserts.assertTrue(ret is not False, 620 "Setup test entry for %s failed." % test_name) 621 self.filterOneTest(test_name) 622 try: 623 ret = self._setUp(test_name) 624 asserts.assertTrue(ret is not False, 625 "Setup for %s failed." % test_name) 626 627 if args or kwargs: 628 verdict = test_func(*args, **kwargs) 629 else: 630 verdict = test_func() 631 finished = True 632 finally: 633 self._tearDown(test_name) 634 except (signals.TestFailure, AssertionError) as e: 635 tr_record.testFail(e) 636 self._exec_procedure_func(self._onFail) 637 finished = True 638 except signals.TestSkip as e: 639 # Test skipped. 640 tr_record.testSkip(e) 641 self._exec_procedure_func(self._onSkip) 642 finished = True 643 except (signals.TestAbortClass, signals.TestAbortAll) as e: 644 # Abort signals, pass along. 645 tr_record.testFail(e) 646 finished = True 647 raise e 648 except signals.TestPass as e: 649 # Explicit test pass. 650 tr_record.testPass(e) 651 self._exec_procedure_func(self._onPass) 652 finished = True 653 except signals.TestSilent as e: 654 # Suppress test reporting. 655 is_silenced = True 656 self._exec_procedure_func(self._onSilent) 657 self.results.removeRecord(tr_record) 658 finished = True 659 except Exception as e: 660 # Exception happened during test. 661 logging.exception(e) 662 tr_record.testError(e) 663 self._exec_procedure_func(self._onException) 664 self._exec_procedure_func(self._onFail) 665 finished = True 666 else: 667 # Keep supporting return False for now. 668 # TODO(angli): Deprecate return False support. 669 if verdict or (verdict is None): 670 # Test passed. 671 tr_record.testPass() 672 self._exec_procedure_func(self._onPass) 673 return 674 # Test failed because it didn't return True. 675 # This should be removed eventually. 676 tr_record.testFail() 677 self._exec_procedure_func(self._onFail) 678 finished = True 679 finally: 680 if not finished: 681 for device in self.android_devices: 682 device.shell.enabled = False 683 684 logging.error('Test timed out.') 685 tr_record.testError() 686 self._exec_procedure_func(self._onException) 687 self._exec_procedure_func(self._onFail) 688 689 if not is_silenced: 690 self.results.addRecord(tr_record) 691 self._testExit() 692 693 def runGeneratedTests(self, 694 test_func, 695 settings, 696 args=None, 697 kwargs=None, 698 tag="", 699 name_func=None): 700 """Runs generated test cases. 701 702 Generated test cases are not written down as functions, but as a list 703 of parameter sets. This way we reduce code repetition and improve 704 test case scalability. 705 706 Args: 707 test_func: The common logic shared by all these generated test 708 cases. This function should take at least one argument, 709 which is a parameter set. 710 settings: A list of strings representing parameter sets. These are 711 usually json strings that get loaded in the test_func. 712 args: Iterable of additional position args to be passed to 713 test_func. 714 kwargs: Dict of additional keyword args to be passed to test_func 715 tag: Name of this group of generated test cases. Ignored if 716 name_func is provided and operates properly. 717 name_func: A function that takes a test setting and generates a 718 proper test name. The test name should be shorter than 719 utils.MAX_FILENAME_LEN. Names over the limit will be 720 truncated. 721 722 Returns: 723 A list of settings that did not pass. 724 """ 725 args = args or () 726 kwargs = kwargs or {} 727 failed_settings = [] 728 for s in settings: 729 test_name = "{} {}".format(tag, s) 730 if name_func: 731 try: 732 test_name = name_func(s, *args, **kwargs) 733 except: 734 logging.exception(("Failed to get test name from " 735 "test_func. Fall back to default %s"), 736 test_name) 737 738 tr_record = records.TestResultRecord(test_name, self.TAG) 739 self.results.requested.append(tr_record) 740 if len(test_name) > utils.MAX_FILENAME_LEN: 741 test_name = test_name[:utils.MAX_FILENAME_LEN] 742 previous_success_cnt = len(self.results.passed) 743 self.execOneTest(test_name, test_func, (s, ) + args, **kwargs) 744 if len(self.results.passed) - previous_success_cnt != 1: 745 failed_settings.append(s) 746 return failed_settings 747 748 def _exec_func(self, func, *args): 749 """Executes a function with exception safeguard. 750 751 This will let signals.TestAbortAll through so abortAll works in all 752 procedure functions. 753 754 Args: 755 func: Function to be executed. 756 args: Arguments to be passed to the function. 757 758 Returns: 759 Whatever the function returns, or False if non-caught exception 760 occurred. 761 """ 762 try: 763 return func(*args) 764 except signals.TestAbortAll: 765 raise 766 except: 767 logging.exception("Exception happened when executing %s in %s.", 768 func.__name__, self.TAG) 769 return False 770 771 def _get_all_test_names(self): 772 """Finds all the function names that match the test case naming 773 convention in this class. 774 775 Returns: 776 A list of strings, each is a test case name. 777 """ 778 test_names = [] 779 for name in dir(self): 780 if name.startswith(STR_TEST) or name.startswith(STR_GENERATE): 781 attr_func = getattr(self, name) 782 if hasattr(attr_func, "__call__"): 783 test_names.append(name) 784 return test_names 785 786 def _get_test_funcs(self, test_names): 787 """Obtain the actual functions of test cases based on test names. 788 789 Args: 790 test_names: A list of strings, each string is a test case name. 791 792 Returns: 793 A list of tuples of (string, function). String is the test case 794 name, function is the actual test case function. 795 796 Raises: 797 errors.USERError is raised if the test name does not follow 798 naming convention "test_*". This can only be caused by user input 799 here. 800 """ 801 test_funcs = [] 802 for test_name in test_names: 803 if not hasattr(self, test_name): 804 logging.warning("%s does not have test case %s.", self.TAG, 805 test_name) 806 elif (test_name.startswith(STR_TEST) or 807 test_name.startswith(STR_GENERATE)): 808 test_funcs.append((test_name, getattr(self, test_name))) 809 else: 810 msg = ("Test case name %s does not follow naming convention " 811 "test*, abort.") % test_name 812 raise errors.USERError(msg) 813 814 return test_funcs 815 816 def run(self, test_names=None): 817 """Runs test cases within a test class by the order they appear in the 818 execution list. 819 820 One of these test cases lists will be executed, shown here in priority 821 order: 822 1. The test_names list, which is passed from cmd line. Invalid names 823 are guarded by cmd line arg parsing. 824 2. The self.tests list defined in test class. Invalid names are 825 ignored. 826 3. All function that matches test case naming convention in the test 827 class. 828 829 Args: 830 test_names: A list of string that are test case names requested in 831 cmd line. 832 833 Returns: 834 The test results object of this class. 835 """ 836 logging.info("==========> %s <==========", self.TAG) 837 # Devise the actual test cases to run in the test class. 838 if not test_names: 839 if self.tests: 840 # Specified by run list in class. 841 test_names = list(self.tests) 842 else: 843 # No test case specified by user, execute all in the test class 844 test_names = self._get_all_test_names() 845 self.results.requested = [ 846 records.TestResultRecord(test_name, self.TAG) 847 for test_name in test_names if test_name.startswith(STR_TEST) 848 ] 849 tests = self._get_test_funcs(test_names) 850 # Setup for the class. 851 try: 852 if self._setUpClass() is False: 853 raise signals.TestFailure("Failed to setup %s." % self.TAG) 854 except Exception as e: 855 logging.exception("Failed to setup %s.", self.TAG) 856 self.results.failClass(self.TAG, e) 857 self._exec_func(self._tearDownClass) 858 return self.results 859 # Run tests in order. 860 try: 861 for test_name, test_func in tests: 862 if test_name.startswith(STR_GENERATE): 863 logging.info( 864 "Executing generated test trigger function '%s'", 865 test_name) 866 test_func() 867 logging.info("Finished '%s'", test_name) 868 else: 869 self.execOneTest(test_name, test_func, None) 870 if self._skip_all_testcases and not self.results.executed: 871 self.results.skipClass( 872 self.TAG, 873 "All test cases skipped; unable to find any test case.") 874 return self.results 875 except signals.TestAbortClass: 876 logging.info("Received TestAbortClass signal") 877 return self.results 878 except signals.TestAbortAll as e: 879 logging.info("Received TestAbortAll signal") 880 # Piggy-back test results on this exception object so we don't lose 881 # results from this test class. 882 setattr(e, "results", self.results) 883 raise e 884 except Exception as e: 885 # Exception happened during test. 886 logging.exception(e) 887 raise e 888 finally: 889 self._exec_func(self._tearDownClass) 890 if self.web.enabled: 891 name, timestamp = self.web.GetTestModuleKeys() 892 self.results.setTestModuleKeys(name, timestamp) 893 logging.info("Summary for test class %s: %s", self.TAG, 894 self.results.summary()) 895 896 def cleanUp(self): 897 """A function that is executed upon completion of all tests cases 898 selected in the test class. 899 900 This function should clean up objects initialized in the constructor by 901 user. 902 """ 903 904 def CatchBugReport(self, prefix=''): 905 """Get device bugreport through adb command. 906 907 Args: 908 prefix: string, file name prefix. Usually in format of 909 <test_module>-<test_case> 910 """ 911 if prefix: 912 prefix = re.sub('[^\w\-_\. ]', '_', prefix) + '_' 913 914 for i in range(len(self.android_devices)): 915 device = self.android_devices[i] 916 bug_report_file_name = prefix + _BUG_REPORT_FILE_PREFIX + str( 917 i) + _BUG_REPORT_FILE_EXTENSION 918 bug_report_file_path = os.path.join(logging.log_path, 919 bug_report_file_name) 920 921 logging.info('Catching bugreport %s' % bug_report_file_path) 922 device.adb.bugreport(bug_report_file_path) 923