1#!/usr/bin/env python3 2# 3# Copyright 2016 - The Android Open Source Project 4# 5# Licensed under the Apache License, Version 2.0 (the "License"); 6# you may not use this file except in compliance with the License. 7# You may obtain a copy of the License at 8# 9# http://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an "AS IS" BASIS, 13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14# See the License for the specific language governing permissions and 15# limitations under the License. 16 17import fnmatch 18import importlib 19import logging 20import os 21import traceback 22from concurrent.futures import ThreadPoolExecutor 23 24from acts import asserts 25from acts import keys 26from acts import logger 27from acts import records 28from acts import signals 29from acts import error 30from acts import tracelogger 31from acts import utils 32from acts.event import event_bus 33from acts.event import subscription_bundle 34from acts.event.decorators import subscribe_static 35from acts.event.event import TestCaseBeginEvent 36from acts.event.event import TestCaseEndEvent 37from acts.event.event import TestClassBeginEvent 38from acts.event.event import TestClassEndEvent 39from acts.event.subscription_bundle import SubscriptionBundle 40 41from mobly import controller_manager 42from mobly.records import ExceptionRecord 43 44# Macro strings for test result reporting 45TEST_CASE_TOKEN = "[Test Case]" 46RESULT_LINE_TEMPLATE = TEST_CASE_TOKEN + " %s %s" 47 48 49@subscribe_static(TestCaseBeginEvent) 50def _logcat_log_test_begin(event): 51 """Ensures that logcat is running. Write a logcat line indicating test case 52 begin.""" 53 test_instance = event.test_class 54 try: 55 for ad in getattr(test_instance, 'android_devices', []): 56 if not ad.is_adb_logcat_on: 57 ad.start_adb_logcat() 58 # Write test start token to adb log if android device is attached. 59 if not ad.skip_sl4a: 60 ad.droid.logV("%s BEGIN %s" % (TEST_CASE_TOKEN, 61 event.test_case_name)) 62 63 except error.ActsError as e: 64 test_instance.results.error.append( 65 ExceptionRecord( 66 e, 'Logcat for test begin: %s' % event.test_case_name)) 67 test_instance.log.error('BaseTest setup_test error: %s' % e.message) 68 69 except Exception as e: 70 test_instance.log.warning( 71 'Unable to send BEGIN log command to all devices.') 72 test_instance.log.warning('Error: %s' % e) 73 74 75@subscribe_static(TestCaseEndEvent) 76def _logcat_log_test_end(event): 77 """Write a logcat line indicating test case end.""" 78 test_instance = event.test_class 79 try: 80 # Write test end token to adb log if android device is attached. 81 for ad in getattr(test_instance, 'android_devices', []): 82 if not ad.skip_sl4a: 83 ad.droid.logV("%s END %s" % (TEST_CASE_TOKEN, 84 event.test_case_name)) 85 86 except error.ActsError as e: 87 test_instance.results.error.append( 88 ExceptionRecord( 89 e, 'Logcat for test end: %s' % event.test_case_name)) 90 test_instance.log.error('BaseTest teardown_test error: %s' % e.message) 91 92 except Exception as e: 93 test_instance.log.warning( 94 'Unable to send END log command to all devices.') 95 test_instance.log.warning('Error: %s' % e) 96 97 98event_bus.register_subscription(_logcat_log_test_begin.subscription) 99event_bus.register_subscription(_logcat_log_test_end.subscription) 100 101 102class Error(Exception): 103 """Raised for exceptions that occured in BaseTestClass.""" 104 105 106class BaseTestClass(object): 107 """Base class for all test classes to inherit from. 108 109 This class gets all the controller objects from test_runner and executes 110 the test cases requested within itself. 111 112 Most attributes of this class are set at runtime based on the configuration 113 provided. 114 115 Attributes: 116 tests: A list of strings, each representing a test case name. 117 TAG: A string used to refer to a test class. Default is the test class 118 name. 119 log: A logger object used for logging. 120 results: A records.TestResult object for aggregating test results from 121 the execution of test cases. 122 consecutive_failures: Tracks the number of consecutive test case 123 failures within this class. 124 consecutive_failure_limit: Number of consecutive test failures to allow 125 before blocking remaining tests in the same 126 test class. 127 size_limit_reached: True if the size of the log directory has reached 128 its limit. 129 current_test_name: A string that's the name of the test case currently 130 being executed. If no test is executing, this should 131 be None. 132 """ 133 134 TAG = None 135 136 def __init__(self, configs): 137 self.class_subscriptions = SubscriptionBundle() 138 self.class_subscriptions.register() 139 self.all_subscriptions = [self.class_subscriptions] 140 141 self.tests = [] 142 if not self.TAG: 143 self.TAG = self.__class__.__name__ 144 # Set all the controller objects and params. 145 self.user_params = {} 146 self.testbed_configs = {} 147 for name, value in configs.items(): 148 setattr(self, name, value) 149 self.results = records.TestResult() 150 self.current_test_name = None 151 self.log = tracelogger.TraceLogger(self.log) 152 self.consecutive_failures = 0 153 self.consecutive_failure_limit = self.user_params.get( 154 'consecutive_failure_limit', -1) 155 self.size_limit_reached = False 156 157 # Initialize a controller manager (Mobly) 158 self._controller_manager = controller_manager.ControllerManager( 159 class_name=self.__class__.__name__, 160 controller_configs=self.testbed_configs) 161 162 # Import and register the built-in controller modules specified 163 # in testbed config. 164 for module in self._import_builtin_controllers(): 165 self.register_controller(module, builtin=True) 166 if hasattr(self, 'android_devices'): 167 for ad in self.android_devices: 168 if ad.droid: 169 utils.set_location_service(ad, False) 170 utils.sync_device_time(ad) 171 self.testbed_name = '' 172 173 def __enter__(self): 174 return self 175 176 def __exit__(self, *args): 177 self._exec_func(self.clean_up) 178 179 def unpack_userparams(self, 180 req_param_names=[], 181 opt_param_names=[], 182 **kwargs): 183 """An optional function that unpacks user defined parameters into 184 individual variables. 185 186 After unpacking, the params can be directly accessed with self.xxx. 187 188 If a required param is not provided, an exception is raised. If an 189 optional param is not provided, a warning line will be logged. 190 191 To provide a param, add it in the config file or pass it in as a kwarg. 192 If a param appears in both the config file and kwarg, the value in the 193 config file is used. 194 195 User params from the config file can also be directly accessed in 196 self.user_params. 197 198 Args: 199 req_param_names: A list of names of the required user params. 200 opt_param_names: A list of names of the optional user params. 201 **kwargs: Arguments that provide default values. 202 e.g. unpack_userparams(required_list, opt_list, arg_a="hello") 203 self.arg_a will be "hello" unless it is specified again in 204 required_list or opt_list. 205 206 Raises: 207 Error is raised if a required user params is not provided. 208 """ 209 for k, v in kwargs.items(): 210 if k in self.user_params: 211 v = self.user_params[k] 212 setattr(self, k, v) 213 for name in req_param_names: 214 if hasattr(self, name): 215 continue 216 if name not in self.user_params: 217 raise Error(("Missing required user param '%s' in test " 218 "configuration.") % name) 219 setattr(self, name, self.user_params[name]) 220 for name in opt_param_names: 221 if hasattr(self, name): 222 continue 223 if name in self.user_params: 224 setattr(self, name, self.user_params[name]) 225 else: 226 self.log.warning(("Missing optional user param '%s' in " 227 "configuration, continue."), name) 228 229 def _import_builtin_controllers(self): 230 """Import built-in controller modules. 231 232 Go through the testbed configs, find any built-in controller configs 233 and import the corresponding controller module from acts.controllers 234 package. 235 236 Returns: 237 A list of controller modules. 238 """ 239 builtin_controllers = [] 240 for ctrl_name in keys.Config.builtin_controller_names.value: 241 if ctrl_name in self.testbed_configs: 242 module_name = keys.get_module_name(ctrl_name) 243 module = importlib.import_module( 244 "acts.controllers.%s" % module_name) 245 builtin_controllers.append(module) 246 return builtin_controllers 247 248 @staticmethod 249 def get_module_reference_name(a_module): 250 """Returns the module's reference name. 251 252 This is largely for backwards compatibility with log parsing. If the 253 module defines ACTS_CONTROLLER_REFERENCE_NAME, it will return that 254 value, or the module's submodule name. 255 256 Args: 257 a_module: Any module. Ideally, a controller module. 258 Returns: 259 A string corresponding to the module's name. 260 """ 261 if hasattr(a_module, 'ACTS_CONTROLLER_REFERENCE_NAME'): 262 return a_module.ACTS_CONTROLLER_REFERENCE_NAME 263 else: 264 return a_module.__name__.split('.')[-1] 265 266 def register_controller(self, 267 controller_module, 268 required=True, 269 builtin=False): 270 """Registers an ACTS controller module for a test class. Invokes Mobly's 271 implementation of register_controller. 272 273 An ACTS controller module is a Python lib that can be used to control 274 a device, service, or equipment. To be ACTS compatible, a controller 275 module needs to have the following members: 276 277 def create(configs): 278 [Required] Creates controller objects from configurations. 279 Args: 280 configs: A list of serialized data like string/dict. Each 281 element of the list is a configuration for a 282 controller object. 283 Returns: 284 A list of objects. 285 286 def destroy(objects): 287 [Required] Destroys controller objects created by the create 288 function. Each controller object shall be properly cleaned up 289 and all the resources held should be released, e.g. memory 290 allocation, sockets, file handlers etc. 291 Args: 292 A list of controller objects created by the create function. 293 294 def get_info(objects): 295 [Optional] Gets info from the controller objects used in a test 296 run. The info will be included in test_result_summary.json under 297 the key "ControllerInfo". Such information could include unique 298 ID, version, or anything that could be useful for describing the 299 test bed and debugging. 300 Args: 301 objects: A list of controller objects created by the create 302 function. 303 Returns: 304 A list of json serializable objects, each represents the 305 info of a controller object. The order of the info object 306 should follow that of the input objects. 307 def get_post_job_info(controller_list): 308 [Optional] Returns information about the controller after the 309 test has run. This info is sent to test_run_summary.json's 310 "Extras" key. 311 Args: 312 The list of controller objects created by the module 313 Returns: 314 A (name, data) tuple. 315 Registering a controller module declares a test class's dependency the 316 controller. If the module config exists and the module matches the 317 controller interface, controller objects will be instantiated with 318 corresponding configs. The module should be imported first. 319 320 Args: 321 controller_module: A module that follows the controller module 322 interface. 323 required: A bool. If True, failing to register the specified 324 controller module raises exceptions. If False, returns None upon 325 failures. 326 builtin: Specifies that the module is a builtin controller module in 327 ACTS. If true, adds itself to test attributes. 328 Returns: 329 A list of controller objects instantiated from controller_module, or 330 None. 331 332 Raises: 333 When required is True, ControllerError is raised if no corresponding 334 config can be found. 335 Regardless of the value of "required", ControllerError is raised if 336 the controller module has already been registered or any other error 337 occurred in the registration process. 338 """ 339 module_ref_name = self.get_module_reference_name(controller_module) 340 341 # Substitute Mobly controller's module config name with the ACTS one 342 module_config_name = controller_module.ACTS_CONTROLLER_CONFIG_NAME 343 controller_module.MOBLY_CONTROLLER_CONFIG_NAME = module_config_name 344 345 # Get controller objects from Mobly's register_controller 346 controllers = self._controller_manager.register_controller( 347 controller_module, required=required) 348 if not controllers: 349 return None 350 351 # Collect controller information and write to test result. 352 # Implementation of "get_info" is optional for a controller module. 353 if hasattr(controller_module, "get_info"): 354 controller_info = controller_module.get_info(controllers) 355 self.log.info("Controller %s: %s", module_config_name, 356 controller_info) 357 self.results.add_controller_info(module_config_name, 358 controller_info) 359 else: 360 self.log.warning("No controller info obtained for %s", 361 module_config_name) 362 self._record_controller_info() 363 364 if builtin: 365 setattr(self, module_ref_name, controllers) 366 return controllers 367 368 def unregister_controllers(self): 369 """Destroy controller objects and clear internal registry. Invokes 370 Mobly's controller manager's unregister_controllers. 371 372 This will be called upon test class teardown. 373 """ 374 controller_modules = self._controller_manager._controller_modules 375 controller_objects = self._controller_manager._controller_objects 376 # Record post job info for the controller 377 for name, controller_module in controller_modules.items(): 378 if hasattr(controller_module, 'get_post_job_info'): 379 self.log.debug('Getting post job info for %s', name) 380 try: 381 name, value = controller_module.get_post_job_info( 382 controller_objects[name]) 383 self.results.set_extra_data(name, value) 384 self.summary_writer.dump( 385 {name: value}, records.TestSummaryEntryType.USER_DATA) 386 except: 387 self.log.error("Fail to get post job info for %s", name) 388 self._controller_manager.unregister_controllers() 389 390 def _record_controller_info(self): 391 """Collect controller information and write to summary file.""" 392 try: 393 manager = self._controller_manager 394 for record in manager.get_controller_info_records(): 395 self.summary_writer.dump( 396 record.to_dict(), 397 records.TestSummaryEntryType.CONTROLLER_INFO) 398 except Exception: 399 self.log.exception('Unable to write controller info records to' 400 ' summary file') 401 402 def _setup_class(self): 403 """Proxy function to guarantee the base implementation of setup_class 404 is called. 405 """ 406 event_bus.post(TestClassBeginEvent(self)) 407 return self.setup_class() 408 409 def setup_class(self): 410 """Setup function that will be called before executing any test case in 411 the test class. 412 413 To signal setup failure, return False or raise an exception. If 414 exceptions were raised, the stack trace would appear in log, but the 415 exceptions would not propagate to upper levels. 416 417 Implementation is optional. 418 """ 419 420 def _teardown_class(self): 421 """Proxy function to guarantee the base implementation of teardown_class 422 is called. 423 """ 424 self.teardown_class() 425 self.unregister_controllers() 426 event_bus.post(TestClassEndEvent(self, self.results)) 427 428 def teardown_class(self): 429 """Teardown function that will be called after all the selected test 430 cases in the test class have been executed. 431 432 Implementation is optional. 433 """ 434 435 def _setup_test(self, test_name): 436 """Proxy function to guarantee the base implementation of setup_test is 437 called. 438 """ 439 self.current_test_name = test_name 440 441 # Skip the test if the consecutive test case failure limit is reached. 442 if self.consecutive_failures == self.consecutive_failure_limit: 443 raise signals.TestError('Consecutive test failure') 444 445 return self.setup_test() 446 447 def setup_test(self): 448 """Setup function that will be called every time before executing each 449 test case in the test class. 450 451 To signal setup failure, return False or raise an exception. If 452 exceptions were raised, the stack trace would appear in log, but the 453 exceptions would not propagate to upper levels. 454 455 Implementation is optional. 456 """ 457 return True 458 459 def _teardown_test(self, test_name): 460 """Proxy function to guarantee the base implementation of teardown_test 461 is called. 462 """ 463 self.log.debug('Tearing down test %s' % test_name) 464 465 try: 466 self.teardown_test() 467 finally: 468 self.current_test_name = None 469 470 def teardown_test(self): 471 """Teardown function that will be called every time a test case has 472 been executed. 473 474 Implementation is optional. 475 """ 476 477 def _on_fail(self, record): 478 """Proxy function to guarantee the base implementation of on_fail is 479 called. 480 481 Args: 482 record: The records.TestResultRecord object for the failed test 483 case. 484 """ 485 self.consecutive_failures += 1 486 if record.details: 487 self.log.error(record.details) 488 self.log.info(RESULT_LINE_TEMPLATE, record.test_name, record.result) 489 self.on_fail(record.test_name, record.begin_time) 490 491 def on_fail(self, test_name, begin_time): 492 """A function that is executed upon a test case failure. 493 494 User implementation is optional. 495 496 Args: 497 test_name: Name of the test that triggered this function. 498 begin_time: Logline format timestamp taken when the test started. 499 """ 500 501 def _on_pass(self, record): 502 """Proxy function to guarantee the base implementation of on_pass is 503 called. 504 505 Args: 506 record: The records.TestResultRecord object for the passed test 507 case. 508 """ 509 self.consecutive_failures = 0 510 msg = record.details 511 if msg: 512 self.log.info(msg) 513 self.log.info(RESULT_LINE_TEMPLATE, record.test_name, record.result) 514 self.on_pass(record.test_name, record.begin_time) 515 516 def on_pass(self, test_name, begin_time): 517 """A function that is executed upon a test case passing. 518 519 Implementation is optional. 520 521 Args: 522 test_name: Name of the test that triggered this function. 523 begin_time: Logline format timestamp taken when the test started. 524 """ 525 526 def _on_skip(self, record): 527 """Proxy function to guarantee the base implementation of on_skip is 528 called. 529 530 Args: 531 record: The records.TestResultRecord object for the skipped test 532 case. 533 """ 534 self.log.info(RESULT_LINE_TEMPLATE, record.test_name, record.result) 535 self.log.info("Reason to skip: %s", record.details) 536 self.on_skip(record.test_name, record.begin_time) 537 538 def on_skip(self, test_name, begin_time): 539 """A function that is executed upon a test case being skipped. 540 541 Implementation is optional. 542 543 Args: 544 test_name: Name of the test that triggered this function. 545 begin_time: Logline format timestamp taken when the test started. 546 """ 547 548 def _on_exception(self, record): 549 """Proxy function to guarantee the base implementation of on_exception 550 is called. 551 552 Args: 553 record: The records.TestResultRecord object for the failed test 554 case. 555 """ 556 self.log.exception(record.details) 557 self.on_exception(record.test_name, record.begin_time) 558 559 def on_exception(self, test_name, begin_time): 560 """A function that is executed upon an unhandled exception from a test 561 case. 562 563 Implementation is optional. 564 565 Args: 566 test_name: Name of the test that triggered this function. 567 begin_time: Logline format timestamp taken when the test started. 568 """ 569 570 def _exec_procedure_func(self, func, tr_record): 571 """Executes a procedure function like on_pass, on_fail etc. 572 573 This function will alternate the 'Result' of the test's record if 574 exceptions happened when executing the procedure function. 575 576 This will let signals.TestAbortAll through so abort_all works in all 577 procedure functions. 578 579 Args: 580 func: The procedure function to be executed. 581 tr_record: The TestResultRecord object associated with the test 582 case executed. 583 """ 584 try: 585 func(tr_record) 586 except signals.TestAbortAll: 587 raise 588 except Exception as e: 589 self.log.exception("Exception happened when executing %s for %s.", 590 func.__name__, self.current_test_name) 591 tr_record.add_error(func.__name__, e) 592 593 def exec_one_testcase(self, test_name, test_func, args, **kwargs): 594 """Executes one test case and update test results. 595 596 Executes one test case, create a records.TestResultRecord object with 597 the execution information, and add the record to the test class's test 598 results. 599 600 Args: 601 test_name: Name of the test. 602 test_func: The test function. 603 args: A tuple of params. 604 kwargs: Extra kwargs. 605 """ 606 class_name = self.__class__.__name__ 607 tr_record = records.TestResultRecord(test_name, class_name) 608 tr_record.test_begin() 609 self.begin_time = int(tr_record.begin_time) 610 self.log_begin_time = tr_record.log_begin_time 611 self.test_name = tr_record.test_name 612 event_bus.post(TestCaseBeginEvent(self, self.test_name)) 613 self.log.info("%s %s", TEST_CASE_TOKEN, test_name) 614 615 # Enable test retry if specified in the ACTS config 616 retry_tests = self.user_params.get('retry_tests', []) 617 full_test_name = '%s.%s' % (class_name, self.test_name) 618 if any(name in retry_tests for name in [class_name, full_test_name]): 619 test_func = self.get_func_with_retry(test_func) 620 621 verdict = None 622 test_signal = None 623 try: 624 try: 625 ret = self._setup_test(self.test_name) 626 asserts.assert_true(ret is not False, 627 "Setup for %s failed." % test_name) 628 if args or kwargs: 629 verdict = test_func(*args, **kwargs) 630 else: 631 verdict = test_func() 632 633 finally: 634 try: 635 self._teardown_test(self.test_name) 636 except signals.TestAbortAll: 637 raise 638 except Exception as e: 639 self.log.error(traceback.format_exc()) 640 tr_record.add_error("teardown_test", e) 641 self._exec_procedure_func(self._on_exception, tr_record) 642 except (signals.TestFailure, AssertionError) as e: 643 test_signal = e 644 if self.user_params.get( 645 keys.Config.key_test_failure_tracebacks.value, False): 646 self.log.exception(e) 647 tr_record.test_fail(e) 648 self._exec_procedure_func(self._on_fail, tr_record) 649 except signals.TestSkip as e: 650 # Test skipped. 651 test_signal = e 652 tr_record.test_skip(e) 653 self._exec_procedure_func(self._on_skip, tr_record) 654 except (signals.TestAbortClass, signals.TestAbortAll) as e: 655 # Abort signals, pass along. 656 test_signal = e 657 tr_record.test_fail(e) 658 self._exec_procedure_func(self._on_fail, tr_record) 659 raise e 660 except signals.TestPass as e: 661 # Explicit test pass. 662 test_signal = e 663 tr_record.test_pass(e) 664 self._exec_procedure_func(self._on_pass, tr_record) 665 except error.ActsError as e: 666 test_signal = e 667 tr_record.test_error(e) 668 self.log.error( 669 'BaseTest execute_one_test_case error: %s' % e.message) 670 except Exception as e: 671 test_signal = e 672 self.log.error(traceback.format_exc()) 673 # Exception happened during test. 674 tr_record.test_error(e) 675 self._exec_procedure_func(self._on_exception, tr_record) 676 self._exec_procedure_func(self._on_fail, tr_record) 677 else: 678 if verdict or (verdict is None): 679 # Test passed. 680 tr_record.test_pass() 681 self._exec_procedure_func(self._on_pass, tr_record) 682 return 683 tr_record.test_fail() 684 self._exec_procedure_func(self._on_fail, tr_record) 685 finally: 686 self.results.add_record(tr_record) 687 self.summary_writer.dump( 688 tr_record.to_dict(), records.TestSummaryEntryType.RECORD) 689 self.current_test_name = None 690 event_bus.post(TestCaseEndEvent(self, self.test_name, test_signal)) 691 692 def get_func_with_retry(self, func, attempts=2): 693 """Returns a wrapped test method that re-runs after failure. Return test 694 result upon success. If attempt limit reached, collect all failure 695 messages and raise a TestFailure signal. 696 697 Params: 698 func: The test method 699 attempts: Number of attempts to run test 700 701 Returns: result of the test method 702 """ 703 def wrapper(*args, **kwargs): 704 error_msgs = [] 705 extras = {} 706 retry = False 707 for i in range(attempts): 708 try: 709 if retry: 710 self.teardown_test() 711 self.setup_test() 712 return func(*args, **kwargs) 713 except signals.TestFailure as e: 714 retry = True 715 msg = 'Failure on attempt %d: %s' % (i+1, e.details) 716 self.log.warning(msg) 717 error_msgs.append(msg) 718 if e.extras: 719 extras['Attempt %d' % (i+1)] = e.extras 720 raise signals.TestFailure('\n'.join(error_msgs), extras) 721 722 return wrapper 723 724 def run_generated_testcases(self, 725 test_func, 726 settings, 727 args=None, 728 kwargs=None, 729 tag="", 730 name_func=None, 731 format_args=False): 732 """Runs generated test cases. 733 734 Generated test cases are not written down as functions, but as a list 735 of parameter sets. This way we reduce code repetition and improve 736 test case scalability. 737 738 Args: 739 test_func: The common logic shared by all these generated test 740 cases. This function should take at least one argument, 741 which is a parameter set. 742 settings: A list of strings representing parameter sets. These are 743 usually json strings that get loaded in the test_func. 744 args: Iterable of additional position args to be passed to 745 test_func. 746 kwargs: Dict of additional keyword args to be passed to test_func 747 tag: Name of this group of generated test cases. Ignored if 748 name_func is provided and operates properly. 749 name_func: A function that takes a test setting and generates a 750 proper test name. The test name should be shorter than 751 utils.MAX_FILENAME_LEN. Names over the limit will be 752 truncated. 753 format_args: If True, args will be appended as the first argument 754 in the args list passed to test_func. 755 756 Returns: 757 A list of settings that did not pass. 758 """ 759 args = args or () 760 kwargs = kwargs or {} 761 failed_settings = [] 762 763 for setting in settings: 764 test_name = "{} {}".format(tag, setting) 765 766 if name_func: 767 try: 768 test_name = name_func(setting, *args, **kwargs) 769 except: 770 self.log.exception(("Failed to get test name from " 771 "test_func. Fall back to default %s"), 772 test_name) 773 774 self.results.requested.append(test_name) 775 776 if len(test_name) > utils.MAX_FILENAME_LEN: 777 test_name = test_name[:utils.MAX_FILENAME_LEN] 778 779 previous_success_cnt = len(self.results.passed) 780 781 if format_args: 782 self.exec_one_testcase(test_name, test_func, 783 args + (setting,), **kwargs) 784 else: 785 self.exec_one_testcase(test_name, test_func, 786 (setting,) + args, **kwargs) 787 788 if len(self.results.passed) - previous_success_cnt != 1: 789 failed_settings.append(setting) 790 791 return failed_settings 792 793 def _exec_func(self, func, *args): 794 """Executes a function with exception safeguard. 795 796 This will let signals.TestAbortAll through so abort_all works in all 797 procedure functions. 798 799 Args: 800 func: Function to be executed. 801 args: Arguments to be passed to the function. 802 803 Returns: 804 Whatever the function returns, or False if unhandled exception 805 occured. 806 """ 807 try: 808 return func(*args) 809 except signals.TestAbortAll: 810 raise 811 except: 812 self.log.exception("Exception happened when executing %s in %s.", 813 func.__name__, self.TAG) 814 return False 815 816 def _get_all_test_names(self): 817 """Finds all the function names that match the test case naming 818 convention in this class. 819 820 Returns: 821 A list of strings, each is a test case name. 822 """ 823 test_names = [] 824 for name in dir(self): 825 if name.startswith("test_"): 826 test_names.append(name) 827 return test_names 828 829 def _get_test_funcs(self, test_names): 830 """Obtain the actual functions of test cases based on test names. 831 832 Args: 833 test_names: A list of strings, each string is a test case name. 834 835 Returns: 836 A list of tuples of (string, function). String is the test case 837 name, function is the actual test case function. 838 839 Raises: 840 Error is raised if the test name does not follow 841 naming convention "test_*". This can only be caused by user input 842 here. 843 """ 844 test_funcs = [] 845 for test_name in test_names: 846 test_funcs.append(self._get_test_func(test_name)) 847 848 return test_funcs 849 850 def _get_test_func(self, test_name): 851 """Obtain the actual function of test cases based on the test name. 852 853 Args: 854 test_name: String, The name of the test. 855 856 Returns: 857 A tuples of (string, function). String is the test case 858 name, function is the actual test case function. 859 860 Raises: 861 Error is raised if the test name does not follow 862 naming convention "test_*". This can only be caused by user input 863 here. 864 """ 865 if not test_name.startswith("test_"): 866 raise Error(("Test case name %s does not follow naming " 867 "convention test_*, abort.") % test_name) 868 try: 869 return test_name, getattr(self, test_name) 870 except: 871 872 def test_skip_func(*args, **kwargs): 873 raise signals.TestSkip("Test %s does not exist" % test_name) 874 875 self.log.info("Test case %s not found in %s.", test_name, self.TAG) 876 return test_name, test_skip_func 877 878 def _block_all_test_cases(self, tests, reason='Failed class setup'): 879 """ 880 Block all passed in test cases. 881 Args: 882 tests: The tests to block. 883 reason: Message describing the reason that the tests are blocked. 884 Default is 'Failed class setup' 885 """ 886 for test_name, test_func in tests: 887 signal = signals.TestError(reason) 888 record = records.TestResultRecord(test_name, self.TAG) 889 record.test_begin() 890 if hasattr(test_func, 'gather'): 891 signal.extras = test_func.gather() 892 record.test_error(signal) 893 self.results.add_record(record) 894 self.summary_writer.dump( 895 record.to_dict(), records.TestSummaryEntryType.RECORD) 896 self._on_skip(record) 897 898 def run(self, test_names=None, test_case_iterations=1): 899 """Runs test cases within a test class by the order they appear in the 900 execution list. 901 902 One of these test cases lists will be executed, shown here in priority 903 order: 904 1. The test_names list, which is passed from cmd line. 905 2. The self.tests list defined in test class. Invalid names are 906 ignored. 907 3. All function that matches test case naming convention in the test 908 class. 909 910 Args: 911 test_names: A list of string that are test case names/patterns 912 requested in cmd line. 913 914 Returns: 915 The test results object of this class. 916 """ 917 self.register_test_class_event_subscriptions() 918 self.log.info("==========> %s <==========", self.TAG) 919 # Devise the actual test cases to run in the test class. 920 if self.tests: 921 # Specified by run list in class. 922 valid_tests = list(self.tests) 923 else: 924 # No test case specified by user, execute all in the test class 925 valid_tests = self._get_all_test_names() 926 if test_names: 927 # Match test cases with any of the user-specified patterns 928 matches = [] 929 for test_name in test_names: 930 for valid_test in valid_tests: 931 if (fnmatch.fnmatch(valid_test, test_name) 932 and valid_test not in matches): 933 matches.append(valid_test) 934 else: 935 matches = valid_tests 936 self.results.requested = matches 937 self.summary_writer.dump(self.results.requested_test_names_dict(), 938 records.TestSummaryEntryType.TEST_NAME_LIST) 939 tests = self._get_test_funcs(matches) 940 941 # Setup for the class. 942 setup_fail = False 943 try: 944 if self._setup_class() is False: 945 self.log.error("Failed to setup %s.", self.TAG) 946 self._block_all_test_cases(tests) 947 setup_fail = True 948 except signals.TestAbortClass: 949 try: 950 self._exec_func(self._teardown_class) 951 except Exception as e: 952 self.log.warning(e) 953 setup_fail = True 954 except Exception as e: 955 self.log.exception("Failed to setup %s.", self.TAG) 956 self._block_all_test_cases(tests) 957 self._exec_func(self._teardown_class) 958 setup_fail = True 959 if setup_fail: 960 self.log.info("Summary for test class %s: %s", self.TAG, 961 self.results.summary_str()) 962 return self.results 963 964 # Run tests in order. 965 try: 966 for test_name, test_func in tests: 967 for _ in range(test_case_iterations): 968 self.exec_one_testcase(test_name, test_func, self.cli_args) 969 return self.results 970 except signals.TestAbortClass: 971 return self.results 972 except signals.TestAbortAll as e: 973 # Piggy-back test results on this exception object so we don't lose 974 # results from this test class. 975 setattr(e, "results", self.results) 976 raise e 977 finally: 978 self._exec_func(self._teardown_class) 979 self.log.info("Summary for test class %s: %s", self.TAG, 980 self.results.summary_str()) 981 982 def clean_up(self): 983 """A function that is executed upon completion of all tests cases 984 selected in the test class. 985 986 This function should clean up objects initialized in the constructor by 987 user. 988 """ 989 990 def _ad_take_bugreport(self, ad, test_name, begin_time): 991 for i in range(3): 992 try: 993 ad.take_bug_report(test_name, begin_time) 994 return True 995 except Exception as e: 996 ad.log.error("bugreport attempt %s error: %s", i + 1, e) 997 998 def _ad_take_extra_logs(self, ad, test_name, begin_time): 999 result = True 1000 if getattr(ad, "qxdm_log", False): 1001 # Gather qxdm log modified 3 minutes earlier than test start time 1002 if begin_time: 1003 qxdm_begin_time = begin_time - 1000 * 60 * 3 1004 else: 1005 qxdm_begin_time = None 1006 try: 1007 ad.get_qxdm_logs(test_name, qxdm_begin_time) 1008 except Exception as e: 1009 ad.log.error("Failed to get QXDM log for %s with error %s", 1010 test_name, e) 1011 result = False 1012 1013 try: 1014 ad.check_crash_report(test_name, begin_time, log_crash_report=True) 1015 except Exception as e: 1016 ad.log.error("Failed to check crash report for %s with error %s", 1017 test_name, e) 1018 result = False 1019 return result 1020 1021 def _skip_bug_report(self): 1022 """A function to check whether we should skip creating a bug report. 1023 1024 Returns: True if bug report is to be skipped. 1025 """ 1026 if "no_bug_report_on_fail" in self.user_params: 1027 return True 1028 1029 # If the current test class or test case is found in the set of 1030 # problematic tests, we skip bugreport and other failure artifact 1031 # creation. 1032 class_name = self.__class__.__name__ 1033 quiet_tests = self.user_params.get('quiet_tests', []) 1034 if class_name in quiet_tests: 1035 self.log.info( 1036 "Skipping bug report, as directed for this test class.") 1037 return True 1038 full_test_name = '%s.%s' % (class_name, self.test_name) 1039 if full_test_name in quiet_tests: 1040 self.log.info( 1041 "Skipping bug report, as directed for this test case.") 1042 return True 1043 1044 # Once we hit a certain log path size, it's not going to get smaller. 1045 # We cache the result so we don't have to keep doing directory walks. 1046 if self.size_limit_reached: 1047 return True 1048 try: 1049 max_log_size = int( 1050 self.user_params.get("soft_output_size_limit") or "invalid") 1051 log_path = getattr(logging, "log_path", None) 1052 if log_path: 1053 curr_log_size = utils.get_directory_size(log_path) 1054 if curr_log_size > max_log_size: 1055 self.log.info( 1056 "Skipping bug report, as we've reached the size limit." 1057 ) 1058 self.size_limit_reached = True 1059 return True 1060 except ValueError: 1061 pass 1062 return False 1063 1064 def _take_bug_report(self, test_name, begin_time): 1065 if self._skip_bug_report(): 1066 return 1067 1068 executor = ThreadPoolExecutor(max_workers=10) 1069 for ad in getattr(self, 'android_devices', []): 1070 executor.submit(self._ad_take_bugreport, ad, test_name, begin_time) 1071 executor.submit(self._ad_take_extra_logs, ad, test_name, 1072 begin_time) 1073 executor.shutdown() 1074 1075 def _reboot_device(self, ad): 1076 ad.log.info("Rebooting device.") 1077 ad = ad.reboot() 1078 1079 def _cleanup_logger_sessions(self): 1080 for (mylogger, session) in self.logger_sessions: 1081 self.log.info("Resetting a diagnostic session %s, %s", mylogger, 1082 session) 1083 mylogger.reset() 1084 self.logger_sessions = [] 1085 1086 def _pull_diag_logs(self, test_name, begin_time): 1087 for (mylogger, session) in self.logger_sessions: 1088 self.log.info("Pulling diagnostic session %s", mylogger) 1089 mylogger.stop(session) 1090 diag_path = os.path.join( 1091 self.log_path, logger.epoch_to_log_line_timestamp(begin_time)) 1092 utils.create_dir(diag_path) 1093 mylogger.pull(session, diag_path) 1094 1095 def register_test_class_event_subscriptions(self): 1096 self.class_subscriptions = subscription_bundle.create_from_instance( 1097 self) 1098 self.class_subscriptions.register() 1099 1100 def unregister_test_class_event_subscriptions(self): 1101 for package in self.all_subscriptions: 1102 package.unregister() 1103