1#!/usr/bin/env python3 2# 3# Copyright 2016 - The Android Open Source Project 4# 5# Licensed under the Apache License, Version 2.0 (the "License"); 6# you may not use this file except in compliance with the License. 7# You may obtain a copy of the License at 8# 9# http://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an "AS IS" BASIS, 13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14# See the License for the specific language governing permissions and 15# limitations under the License. 16 17import fnmatch 18import functools 19import importlib 20import logging 21import os 22import traceback 23from concurrent.futures import ThreadPoolExecutor 24 25from acts import asserts 26from acts import error 27from acts import keys 28from acts import logger 29from acts import records 30from acts import signals 31from acts import tracelogger 32from acts import utils 33from acts.event import event_bus 34from acts.event import subscription_bundle 35from acts.event.decorators import subscribe_static 36from acts.event.event import TestCaseBeginEvent 37from acts.event.event import TestCaseEndEvent 38from acts.event.event import TestClassBeginEvent 39from acts.event.event import TestClassEndEvent 40from acts.event.subscription_bundle import SubscriptionBundle 41 42from mobly.base_test import BaseTestClass as MoblyBaseTest 43from mobly.records import ExceptionRecord 44 45# Macro strings for test result reporting 46TEST_CASE_TOKEN = "[Test Case]" 47RESULT_LINE_TEMPLATE = TEST_CASE_TOKEN + " %s %s" 48 49 50@subscribe_static(TestCaseBeginEvent) 51def _logcat_log_test_begin(event): 52 """Ensures that logcat is running. Write a logcat line indicating test case 53 begin.""" 54 test_instance = event.test_class 55 try: 56 for ad in getattr(test_instance, 'android_devices', []): 57 if not ad.is_adb_logcat_on: 58 ad.start_adb_logcat() 59 # Write test start token to adb log if android device is attached. 60 if not ad.skip_sl4a and ad.droid: 61 ad.droid.logV("%s BEGIN %s" % 62 (TEST_CASE_TOKEN, event.test_case_name)) 63 64 except error.ActsError as e: 65 test_instance.results.error.append( 66 ExceptionRecord(e, 'Logcat for test begin: %s' % 67 event.test_case_name)) 68 test_instance.log.error('BaseTest setup_test error: %s' % e.details) 69 except Exception as e: 70 test_instance.log.warning( 71 'Unable to send BEGIN log command to all devices.') 72 test_instance.log.warning('Error: %s' % e) 73 74 75@subscribe_static(TestCaseEndEvent) 76def _logcat_log_test_end(event): 77 """Write a logcat line indicating test case end.""" 78 test_instance = event.test_class 79 try: 80 # Write test end token to adb log if android device is attached. 81 for ad in getattr(test_instance, 'android_devices', []): 82 if not ad.skip_sl4a and ad.droid: 83 ad.droid.logV("%s END %s" % 84 (TEST_CASE_TOKEN, event.test_case_name)) 85 86 except error.ActsError as e: 87 test_instance.results.error.append( 88 ExceptionRecord(e, 89 'Logcat for test end: %s' % event.test_case_name)) 90 test_instance.log.error('BaseTest teardown_test error: %s' % e.details) 91 except Exception as e: 92 test_instance.log.warning( 93 'Unable to send END log command to all devices.') 94 test_instance.log.warning('Error: %s' % e) 95 96 97@subscribe_static(TestCaseBeginEvent) 98def _syslog_log_test_begin(event): 99 """This adds a BEGIN log message with the test name to the syslog of any 100 Fuchsia device""" 101 test_instance = event.test_class 102 try: 103 for fd in getattr(test_instance, 'fuchsia_devices', []): 104 if hasattr(fd, '_sl4f'): 105 fd.sl4f.logging_lib.logI( 106 "%s BEGIN %s" % (TEST_CASE_TOKEN, event.test_case_name)) 107 108 except Exception as e: 109 test_instance.log.warning( 110 'Unable to send BEGIN log command to all devices.') 111 test_instance.log.warning('Error: %s' % e) 112 113 114@subscribe_static(TestCaseEndEvent) 115def _syslog_log_test_end(event): 116 """This adds a END log message with the test name to the syslog of any 117 Fuchsia device""" 118 test_instance = event.test_class 119 try: 120 for fd in getattr(test_instance, 'fuchsia_devices', []): 121 if hasattr(fd, '_sl4f'): 122 fd.sl4f.logging_lib.logI( 123 "%s END %s" % (TEST_CASE_TOKEN, event.test_case_name)) 124 125 except Exception as e: 126 test_instance.log.warning( 127 'Unable to send END log command to all devices.') 128 test_instance.log.warning('Error: %s' % e) 129 130 131event_bus.register_subscription(_logcat_log_test_begin.subscription) 132event_bus.register_subscription(_logcat_log_test_end.subscription) 133event_bus.register_subscription(_syslog_log_test_begin.subscription) 134event_bus.register_subscription(_syslog_log_test_end.subscription) 135 136 137class Error(Exception): 138 """Raised for exceptions that occured in BaseTestClass.""" 139 140 141class BaseTestClass(MoblyBaseTest): 142 """Base class for all test classes to inherit from. Inherits some 143 functionality from Mobly's base test class. 144 145 This class gets all the controller objects from test_runner and executes 146 the test cases requested within itself. 147 148 Most attributes of this class are set at runtime based on the configuration 149 provided. 150 151 Attributes: 152 tests: A list of strings, each representing a test case name. 153 TAG: A string used to refer to a test class. Default is the test class 154 name. 155 log: A logger object used for logging. 156 results: A records.TestResult object for aggregating test results from 157 the execution of test cases. 158 controller_configs: A dict of controller configs provided by the user 159 via the testbed config. 160 consecutive_failures: Tracks the number of consecutive test case 161 failures within this class. 162 consecutive_failure_limit: Number of consecutive test failures to allow 163 before blocking remaining tests in the same 164 test class. 165 size_limit_reached: True if the size of the log directory has reached 166 its limit. 167 current_test_name: A string that's the name of the test case currently 168 being executed. If no test is executing, this should 169 be None. 170 """ 171 172 TAG = None 173 174 def __init__(self, configs): 175 """Initializes a BaseTestClass given a TestRunConfig, which provides 176 all of the config information for this test class. 177 178 Args: 179 configs: A config_parser.TestRunConfig object. 180 """ 181 super().__init__(configs) 182 183 self.__handle_file_user_params() 184 185 self.class_subscriptions = SubscriptionBundle() 186 self.class_subscriptions.register() 187 self.all_subscriptions = [self.class_subscriptions] 188 189 self.current_test_name = None 190 self.log = tracelogger.TraceLogger(logging.getLogger()) 191 # TODO: remove after converging log path definitions with mobly 192 self.log_path = configs.log_path 193 194 self.consecutive_failures = 0 195 self.consecutive_failure_limit = self.user_params.get( 196 'consecutive_failure_limit', -1) 197 self.size_limit_reached = False 198 self.retryable_exceptions = signals.TestFailure 199 200 def _import_builtin_controllers(self): 201 """Import built-in controller modules. 202 203 Go through the testbed configs, find any built-in controller configs 204 and import the corresponding controller module from acts.controllers 205 package. 206 207 Returns: 208 A list of controller modules. 209 """ 210 builtin_controllers = [] 211 for ctrl_name in keys.Config.builtin_controller_names.value: 212 if ctrl_name in self.controller_configs: 213 module_name = keys.get_module_name(ctrl_name) 214 module = importlib.import_module("acts.controllers.%s" % 215 module_name) 216 builtin_controllers.append(module) 217 return builtin_controllers 218 219 def __handle_file_user_params(self): 220 """For backwards compatibility, moves all contents of the "files" dict 221 into the root level of user_params. 222 223 This allows existing tests to run with the new Mobly-style format 224 without needing to make changes. 225 """ 226 for key, value in self.user_params.items(): 227 if key.endswith('files') and isinstance(value, dict): 228 new_user_params = dict(value) 229 new_user_params.update(self.user_params) 230 self.user_params = new_user_params 231 break 232 233 @staticmethod 234 def get_module_reference_name(a_module): 235 """Returns the module's reference name. 236 237 This is largely for backwards compatibility with log parsing. If the 238 module defines ACTS_CONTROLLER_REFERENCE_NAME, it will return that 239 value, or the module's submodule name. 240 241 Args: 242 a_module: Any module. Ideally, a controller module. 243 Returns: 244 A string corresponding to the module's name. 245 """ 246 if hasattr(a_module, 'ACTS_CONTROLLER_REFERENCE_NAME'): 247 return a_module.ACTS_CONTROLLER_REFERENCE_NAME 248 else: 249 return a_module.__name__.split('.')[-1] 250 251 def register_controller(self, 252 controller_module, 253 required=True, 254 builtin=False): 255 """Registers an ACTS controller module for a test class. Invokes Mobly's 256 implementation of register_controller. 257 258 An ACTS controller module is a Python lib that can be used to control 259 a device, service, or equipment. To be ACTS compatible, a controller 260 module needs to have the following members: 261 262 def create(configs): 263 [Required] Creates controller objects from configurations. 264 Args: 265 configs: A list of serialized data like string/dict. Each 266 element of the list is a configuration for a 267 controller object. 268 Returns: 269 A list of objects. 270 271 def destroy(objects): 272 [Required] Destroys controller objects created by the create 273 function. Each controller object shall be properly cleaned up 274 and all the resources held should be released, e.g. memory 275 allocation, sockets, file handlers etc. 276 Args: 277 A list of controller objects created by the create function. 278 279 def get_info(objects): 280 [Optional] Gets info from the controller objects used in a test 281 run. The info will be included in test_result_summary.json under 282 the key "ControllerInfo". Such information could include unique 283 ID, version, or anything that could be useful for describing the 284 test bed and debugging. 285 Args: 286 objects: A list of controller objects created by the create 287 function. 288 Returns: 289 A list of json serializable objects, each represents the 290 info of a controller object. The order of the info object 291 should follow that of the input objects. 292 Registering a controller module declares a test class's dependency the 293 controller. If the module config exists and the module matches the 294 controller interface, controller objects will be instantiated with 295 corresponding configs. The module should be imported first. 296 297 Args: 298 controller_module: A module that follows the controller module 299 interface. 300 required: A bool. If True, failing to register the specified 301 controller module raises exceptions. If False, returns None upon 302 failures. 303 builtin: Specifies that the module is a builtin controller module in 304 ACTS. If true, adds itself to test attributes. 305 Returns: 306 A list of controller objects instantiated from controller_module, or 307 None. 308 309 Raises: 310 When required is True, ControllerError is raised if no corresponding 311 config can be found. 312 Regardless of the value of "required", ControllerError is raised if 313 the controller module has already been registered or any other error 314 occurred in the registration process. 315 """ 316 module_ref_name = self.get_module_reference_name(controller_module) 317 module_config_name = controller_module.MOBLY_CONTROLLER_CONFIG_NAME 318 319 # Get controller objects from Mobly's register_controller 320 controllers = self._controller_manager.register_controller( 321 controller_module, required=required) 322 if not controllers: 323 return None 324 325 # Log controller information 326 # Implementation of "get_info" is optional for a controller module. 327 if hasattr(controller_module, "get_info"): 328 controller_info = controller_module.get_info(controllers) 329 self.log.info("Controller %s: %s", module_config_name, 330 controller_info) 331 332 if builtin: 333 setattr(self, module_ref_name, controllers) 334 return controllers 335 336 def _setup_class(self): 337 """Proxy function to guarantee the base implementation of setup_class 338 is called. 339 """ 340 event_bus.post(TestClassBeginEvent(self)) 341 # Import and register the built-in controller modules specified 342 # in testbed config. 343 for module in self._import_builtin_controllers(): 344 self.register_controller(module, builtin=True) 345 return self.setup_class() 346 347 def _teardown_class(self): 348 """Proxy function to guarantee the base implementation of teardown_class 349 is called. 350 """ 351 super()._teardown_class() 352 event_bus.post(TestClassEndEvent(self, self.results)) 353 354 def _setup_test(self, test_name): 355 """Proxy function to guarantee the base implementation of setup_test is 356 called. 357 """ 358 self.current_test_name = test_name 359 360 # Skip the test if the consecutive test case failure limit is reached. 361 if self.consecutive_failures == self.consecutive_failure_limit: 362 raise signals.TestError('Consecutive test failure') 363 364 return self.setup_test() 365 366 def setup_test(self): 367 """Setup function that will be called every time before executing each 368 test case in the test class. 369 370 To signal setup failure, return False or raise an exception. If 371 exceptions were raised, the stack trace would appear in log, but the 372 exceptions would not propagate to upper levels. 373 374 Implementation is optional. 375 """ 376 return True 377 378 def _teardown_test(self, test_name): 379 """Proxy function to guarantee the base implementation of teardown_test 380 is called. 381 """ 382 self.log.debug('Tearing down test %s' % test_name) 383 self.teardown_test() 384 385 def _on_fail(self, record): 386 """Proxy function to guarantee the base implementation of on_fail is 387 called. 388 389 Args: 390 record: The records.TestResultRecord object for the failed test 391 case. 392 """ 393 self.consecutive_failures += 1 394 if record.details: 395 self.log.error(record.details) 396 self.log.info(RESULT_LINE_TEMPLATE, record.test_name, record.result) 397 self.on_fail(record.test_name, record.begin_time) 398 399 def on_fail(self, test_name, begin_time): 400 """A function that is executed upon a test case failure. 401 402 User implementation is optional. 403 404 Args: 405 test_name: Name of the test that triggered this function. 406 begin_time: Logline format timestamp taken when the test started. 407 """ 408 409 def _on_pass(self, record): 410 """Proxy function to guarantee the base implementation of on_pass is 411 called. 412 413 Args: 414 record: The records.TestResultRecord object for the passed test 415 case. 416 """ 417 self.consecutive_failures = 0 418 msg = record.details 419 if msg: 420 self.log.info(msg) 421 self.log.info(RESULT_LINE_TEMPLATE, record.test_name, record.result) 422 self.on_pass(record.test_name, record.begin_time) 423 424 def on_pass(self, test_name, begin_time): 425 """A function that is executed upon a test case passing. 426 427 Implementation is optional. 428 429 Args: 430 test_name: Name of the test that triggered this function. 431 begin_time: Logline format timestamp taken when the test started. 432 """ 433 434 def _on_skip(self, record): 435 """Proxy function to guarantee the base implementation of on_skip is 436 called. 437 438 Args: 439 record: The records.TestResultRecord object for the skipped test 440 case. 441 """ 442 self.log.info(RESULT_LINE_TEMPLATE, record.test_name, record.result) 443 self.log.info("Reason to skip: %s", record.details) 444 self.on_skip(record.test_name, record.begin_time) 445 446 def on_skip(self, test_name, begin_time): 447 """A function that is executed upon a test case being skipped. 448 449 Implementation is optional. 450 451 Args: 452 test_name: Name of the test that triggered this function. 453 begin_time: Logline format timestamp taken when the test started. 454 """ 455 456 def _on_exception(self, record): 457 """Proxy function to guarantee the base implementation of on_exception 458 is called. 459 460 Args: 461 record: The records.TestResultRecord object for the failed test 462 case. 463 """ 464 self.log.exception(record.details) 465 self.on_exception(record.test_name, record.begin_time) 466 467 def on_exception(self, test_name, begin_time): 468 """A function that is executed upon an unhandled exception from a test 469 case. 470 471 Implementation is optional. 472 473 Args: 474 test_name: Name of the test that triggered this function. 475 begin_time: Logline format timestamp taken when the test started. 476 """ 477 478 def on_retry(self): 479 """Function to run before retrying a test through get_func_with_retry. 480 481 This function runs when a test is automatically retried. The function 482 can be used to modify internal test parameters, for example, to retry 483 a test with slightly different input variables. 484 """ 485 486 def _exec_procedure_func(self, func, tr_record): 487 """Executes a procedure function like on_pass, on_fail etc. 488 489 This function will alternate the 'Result' of the test's record if 490 exceptions happened when executing the procedure function. 491 492 This will let signals.TestAbortAll through so abort_all works in all 493 procedure functions. 494 495 Args: 496 func: The procedure function to be executed. 497 tr_record: The TestResultRecord object associated with the test 498 case executed. 499 """ 500 try: 501 func(tr_record) 502 except signals.TestAbortAll: 503 raise 504 except Exception as e: 505 self.log.exception("Exception happened when executing %s for %s.", 506 func.__name__, self.current_test_name) 507 tr_record.add_error(func.__name__, e) 508 509 def exec_one_testcase(self, test_name, test_func): 510 """Executes one test case and update test results. 511 512 Executes one test case, create a records.TestResultRecord object with 513 the execution information, and add the record to the test class's test 514 results. 515 516 Args: 517 test_name: Name of the test. 518 test_func: The test function. 519 """ 520 class_name = self.__class__.__name__ 521 tr_record = records.TestResultRecord(test_name, class_name) 522 tr_record.test_begin() 523 self.begin_time = int(tr_record.begin_time) 524 self.log_begin_time = tr_record.log_begin_time 525 self.test_name = tr_record.test_name 526 event_bus.post(TestCaseBeginEvent(self, self.test_name)) 527 self.log.info("%s %s", TEST_CASE_TOKEN, test_name) 528 529 # Enable test retry if specified in the ACTS config 530 retry_tests = self.user_params.get('retry_tests', []) 531 full_test_name = '%s.%s' % (class_name, self.test_name) 532 if any(name in retry_tests for name in [class_name, full_test_name]): 533 test_func = self.get_func_with_retry(test_func) 534 535 verdict = None 536 test_signal = None 537 try: 538 try: 539 ret = self._setup_test(self.test_name) 540 asserts.assert_true(ret is not False, 541 "Setup for %s failed." % test_name) 542 verdict = test_func() 543 finally: 544 try: 545 self._teardown_test(self.test_name) 546 except signals.TestAbortAll: 547 raise 548 except Exception as e: 549 self.log.error(traceback.format_exc()) 550 tr_record.add_error("teardown_test", e) 551 except (signals.TestFailure, AssertionError) as e: 552 test_signal = e 553 if self.user_params.get( 554 keys.Config.key_test_failure_tracebacks.value, False): 555 self.log.exception(e) 556 tr_record.test_fail(e) 557 except signals.TestSkip as e: 558 # Test skipped. 559 test_signal = e 560 tr_record.test_skip(e) 561 except (signals.TestAbortClass, signals.TestAbortAll) as e: 562 # Abort signals, pass along. 563 test_signal = e 564 tr_record.test_fail(e) 565 raise e 566 except signals.TestPass as e: 567 # Explicit test pass. 568 test_signal = e 569 tr_record.test_pass(e) 570 except Exception as e: 571 test_signal = e 572 self.log.error(traceback.format_exc()) 573 # Exception happened during test. 574 tr_record.test_error(e) 575 else: 576 if verdict or (verdict is None): 577 # Test passed. 578 tr_record.test_pass() 579 return 580 tr_record.test_fail() 581 finally: 582 tr_record.update_record() 583 try: 584 # Execute post-test procedures 585 result = tr_record.result 586 if result == records.TestResultEnums.TEST_RESULT_PASS: 587 self._exec_procedure_func(self._on_pass, tr_record) 588 elif result == records.TestResultEnums.TEST_RESULT_FAIL: 589 self._exec_procedure_func(self._on_fail, tr_record) 590 elif result == records.TestResultEnums.TEST_RESULT_SKIP: 591 self._exec_procedure_func(self._on_skip, tr_record) 592 elif result == records.TestResultEnums.TEST_RESULT_ERROR: 593 self._exec_procedure_func(self._on_exception, tr_record) 594 self._exec_procedure_func(self._on_fail, tr_record) 595 finally: 596 self.results.add_record(tr_record) 597 self.summary_writer.dump(tr_record.to_dict(), 598 records.TestSummaryEntryType.RECORD) 599 self.current_test_name = None 600 event_bus.post( 601 TestCaseEndEvent(self, self.test_name, test_signal)) 602 603 def get_func_with_retry(self, func, attempts=2): 604 """Returns a wrapped test method that re-runs after failure. Return test 605 result upon success. If attempt limit reached, collect all failure 606 messages and raise a TestFailure signal. 607 608 Params: 609 func: The test method 610 attempts: Number of attempts to run test 611 612 Returns: result of the test method 613 """ 614 exceptions = self.retryable_exceptions 615 616 def wrapper(*args, **kwargs): 617 error_msgs = [] 618 extras = {} 619 retry = False 620 for i in range(attempts): 621 try: 622 if retry: 623 self.teardown_test() 624 self.setup_test() 625 self.on_retry() 626 return func(*args, **kwargs) 627 except exceptions as e: 628 retry = True 629 msg = 'Failure on attempt %d: %s' % (i + 1, e.details) 630 self.log.warning(msg) 631 error_msgs.append(msg) 632 if e.extras: 633 extras['Attempt %d' % (i + 1)] = e.extras 634 raise signals.TestFailure('\n'.join(error_msgs), extras) 635 636 return wrapper 637 638 def run_generated_testcases(self, 639 test_func, 640 settings, 641 args=None, 642 kwargs=None, 643 tag="", 644 name_func=None, 645 format_args=False): 646 """Deprecated. Please use pre_run and generate_tests. 647 648 Generated test cases are not written down as functions, but as a list 649 of parameter sets. This way we reduce code repetition and improve 650 test case scalability. 651 652 Args: 653 test_func: The common logic shared by all these generated test 654 cases. This function should take at least one argument, 655 which is a parameter set. 656 settings: A list of strings representing parameter sets. These are 657 usually json strings that get loaded in the test_func. 658 args: Iterable of additional position args to be passed to 659 test_func. 660 kwargs: Dict of additional keyword args to be passed to test_func 661 tag: Name of this group of generated test cases. Ignored if 662 name_func is provided and operates properly. 663 name_func: A function that takes a test setting and generates a 664 proper test name. The test name should be shorter than 665 utils.MAX_FILENAME_LEN. Names over the limit will be 666 truncated. 667 format_args: If True, args will be appended as the first argument 668 in the args list passed to test_func. 669 670 Returns: 671 A list of settings that did not pass. 672 """ 673 args = args or () 674 kwargs = kwargs or {} 675 failed_settings = [] 676 677 for setting in settings: 678 test_name = "{} {}".format(tag, setting) 679 680 if name_func: 681 try: 682 test_name = name_func(setting, *args, **kwargs) 683 except: 684 self.log.exception(("Failed to get test name from " 685 "test_func. Fall back to default %s"), 686 test_name) 687 688 self.results.requested.append(test_name) 689 690 if len(test_name) > utils.MAX_FILENAME_LEN: 691 test_name = test_name[:utils.MAX_FILENAME_LEN] 692 693 previous_success_cnt = len(self.results.passed) 694 695 if format_args: 696 self.exec_one_testcase( 697 test_name, 698 functools.partial(test_func, *(args + (setting, )), 699 **kwargs)) 700 else: 701 self.exec_one_testcase( 702 test_name, 703 functools.partial(test_func, *((setting, ) + args), 704 **kwargs)) 705 706 if len(self.results.passed) - previous_success_cnt != 1: 707 failed_settings.append(setting) 708 709 return failed_settings 710 711 def _exec_func(self, func, *args): 712 """Executes a function with exception safeguard. 713 714 This will let signals.TestAbortAll through so abort_all works in all 715 procedure functions. 716 717 Args: 718 func: Function to be executed. 719 args: Arguments to be passed to the function. 720 721 Returns: 722 Whatever the function returns, or False if unhandled exception 723 occured. 724 """ 725 try: 726 return func(*args) 727 except signals.TestAbortAll: 728 raise 729 except: 730 self.log.exception("Exception happened when executing %s in %s.", 731 func.__name__, self.TAG) 732 return False 733 734 def _block_all_test_cases(self, tests, reason='Failed class setup'): 735 """ 736 Block all passed in test cases. 737 Args: 738 tests: The tests to block. 739 reason: Message describing the reason that the tests are blocked. 740 Default is 'Failed class setup' 741 """ 742 for test_name, test_func in tests: 743 signal = signals.TestError(reason) 744 record = records.TestResultRecord(test_name, self.TAG) 745 record.test_begin() 746 if hasattr(test_func, 'gather'): 747 signal.extras = test_func.gather() 748 record.test_error(signal) 749 self.results.add_record(record) 750 self.summary_writer.dump(record.to_dict(), 751 records.TestSummaryEntryType.RECORD) 752 self._on_skip(record) 753 754 def run(self, test_names=None): 755 """Runs test cases within a test class by the order they appear in the 756 execution list. 757 758 One of these test cases lists will be executed, shown here in priority 759 order: 760 1. The test_names list, which is passed from cmd line. 761 2. The self.tests list defined in test class. Invalid names are 762 ignored. 763 3. All function that matches test case naming convention in the test 764 class. 765 766 Args: 767 test_names: A list of string that are test case names/patterns 768 requested in cmd line. 769 770 Returns: 771 The test results object of this class. 772 """ 773 # Executes pre-setup procedures, like generating test methods. 774 if not self._pre_run(): 775 return self.results 776 777 self.register_test_class_event_subscriptions() 778 self.log.info("==========> %s <==========", self.TAG) 779 # Devise the actual test cases to run in the test class. 780 if self.tests: 781 # Specified by run list in class. 782 valid_tests = list(self.tests) 783 else: 784 # No test case specified by user, gather the run list automatically. 785 valid_tests = self.get_existing_test_names() 786 if test_names: 787 # Match test cases with any of the user-specified patterns 788 matches = [] 789 for test_name in test_names: 790 for valid_test in valid_tests: 791 if (fnmatch.fnmatch(valid_test, test_name) 792 and valid_test not in matches): 793 matches.append(valid_test) 794 else: 795 matches = valid_tests 796 self.results.requested = matches 797 self.summary_writer.dump(self.results.requested_test_names_dict(), 798 records.TestSummaryEntryType.TEST_NAME_LIST) 799 tests = self._get_test_methods(matches) 800 801 # Setup for the class. 802 setup_fail = False 803 try: 804 if self._setup_class() is False: 805 self.log.error("Failed to setup %s.", self.TAG) 806 self._block_all_test_cases(tests) 807 setup_fail = True 808 except signals.TestAbortClass: 809 self.log.exception('Test class %s aborted' % self.TAG) 810 setup_fail = True 811 except Exception as e: 812 self.log.exception("Failed to setup %s.", self.TAG) 813 self._block_all_test_cases(tests) 814 setup_fail = True 815 if setup_fail: 816 self._exec_func(self._teardown_class) 817 self.log.info("Summary for test class %s: %s", self.TAG, 818 self.results.summary_str()) 819 return self.results 820 821 # Run tests in order. 822 test_case_iterations = self.user_params.get( 823 keys.Config.key_test_case_iterations.value, 1) 824 if any([ 825 substr in self.__class__.__name__ 826 for substr in ['Preflight', 'Postflight'] 827 ]): 828 test_case_iterations = 1 829 try: 830 for test_name, test_func in tests: 831 for _ in range(test_case_iterations): 832 self.exec_one_testcase(test_name, test_func) 833 return self.results 834 except signals.TestAbortClass: 835 self.log.exception('Test class %s aborted' % self.TAG) 836 return self.results 837 except signals.TestAbortAll as e: 838 # Piggy-back test results on this exception object so we don't lose 839 # results from this test class. 840 setattr(e, "results", self.results) 841 raise e 842 finally: 843 self._exec_func(self._teardown_class) 844 self.log.info("Summary for test class %s: %s", self.TAG, 845 self.results.summary_str()) 846 847 def _ad_take_bugreport(self, ad, test_name, begin_time): 848 for i in range(3): 849 try: 850 ad.take_bug_report(test_name, begin_time) 851 return True 852 except Exception as e: 853 ad.log.error("bugreport attempt %s error: %s", i + 1, e) 854 855 def _ad_take_extra_logs(self, ad, test_name, begin_time): 856 result = True 857 if getattr(ad, "qxdm_log", False): 858 # Gather qxdm log modified 3 minutes earlier than test start time 859 if begin_time: 860 qxdm_begin_time = begin_time - 1000 * 60 * 3 861 else: 862 qxdm_begin_time = None 863 try: 864 ad.get_qxdm_logs(test_name, qxdm_begin_time) 865 except Exception as e: 866 ad.log.error("Failed to get QXDM log for %s with error %s", 867 test_name, e) 868 result = False 869 870 try: 871 ad.check_crash_report(test_name, begin_time, log_crash_report=True) 872 except Exception as e: 873 ad.log.error("Failed to check crash report for %s with error %s", 874 test_name, e) 875 result = False 876 return result 877 878 def _skip_bug_report(self, test_name): 879 """A function to check whether we should skip creating a bug report. 880 881 Args: 882 test_name: The test case name 883 884 Returns: True if bug report is to be skipped. 885 """ 886 if "no_bug_report_on_fail" in self.user_params: 887 return True 888 889 # If the current test class or test case is found in the set of 890 # problematic tests, we skip bugreport and other failure artifact 891 # creation. 892 class_name = self.__class__.__name__ 893 quiet_tests = self.user_params.get('quiet_tests', []) 894 if class_name in quiet_tests: 895 self.log.info( 896 "Skipping bug report, as directed for this test class.") 897 return True 898 full_test_name = '%s.%s' % (class_name, test_name) 899 if full_test_name in quiet_tests: 900 self.log.info( 901 "Skipping bug report, as directed for this test case.") 902 return True 903 904 # Once we hit a certain log path size, it's not going to get smaller. 905 # We cache the result so we don't have to keep doing directory walks. 906 if self.size_limit_reached: 907 return True 908 try: 909 max_log_size = int( 910 self.user_params.get("soft_output_size_limit") or "invalid") 911 log_path = getattr(logging, "log_path", None) 912 if log_path: 913 curr_log_size = utils.get_directory_size(log_path) 914 if curr_log_size > max_log_size: 915 self.log.info( 916 "Skipping bug report, as we've reached the size limit." 917 ) 918 self.size_limit_reached = True 919 return True 920 except ValueError: 921 pass 922 return False 923 924 def _take_bug_report(self, test_name, begin_time): 925 if self._skip_bug_report(test_name): 926 return 927 928 executor = ThreadPoolExecutor(max_workers=10) 929 for ad in getattr(self, 'android_devices', []): 930 executor.submit(self._ad_take_bugreport, ad, test_name, begin_time) 931 executor.submit(self._ad_take_extra_logs, ad, test_name, 932 begin_time) 933 executor.shutdown() 934 935 def _reboot_device(self, ad): 936 ad.log.info("Rebooting device.") 937 ad = ad.reboot() 938 939 def _cleanup_logger_sessions(self): 940 for (mylogger, session) in self.logger_sessions: 941 self.log.info("Resetting a diagnostic session %s, %s", mylogger, 942 session) 943 mylogger.reset() 944 self.logger_sessions = [] 945 946 def _pull_diag_logs(self, test_name, begin_time): 947 for (mylogger, session) in self.logger_sessions: 948 self.log.info("Pulling diagnostic session %s", mylogger) 949 mylogger.stop(session) 950 diag_path = os.path.join( 951 self.log_path, logger.epoch_to_log_line_timestamp(begin_time)) 952 os.makedirs(diag_path, exist_ok=True) 953 mylogger.pull(session, diag_path) 954 955 def register_test_class_event_subscriptions(self): 956 self.class_subscriptions = subscription_bundle.create_from_instance( 957 self) 958 self.class_subscriptions.register() 959 960 def unregister_test_class_event_subscriptions(self): 961 for package in self.all_subscriptions: 962 package.unregister() 963