1#!/usr/bin/env python3 2# 3# Copyright 2016 - The Android Open Source Project 4# 5# Licensed under the Apache License, Version 2.0 (the "License"); 6# you may not use this file except in compliance with the License. 7# You may obtain a copy of the License at 8# 9# http://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an "AS IS" BASIS, 13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14# See the License for the specific language governing permissions and 15# limitations under the License. 16 17import fnmatch 18import functools 19import importlib 20import logging 21import os 22import traceback 23from concurrent.futures import ThreadPoolExecutor 24 25from acts import asserts 26from acts import error 27from acts import keys 28from acts import logger 29from acts import records 30from acts import signals 31from acts import tracelogger 32from acts import utils 33from acts.event import event_bus 34from acts.event import subscription_bundle 35from acts.event.decorators import subscribe_static 36from acts.event.event import TestCaseBeginEvent 37from acts.event.event import TestCaseEndEvent 38from acts.event.event import TestClassBeginEvent 39from acts.event.event import TestClassEndEvent 40from acts.event.subscription_bundle import SubscriptionBundle 41 42from mobly.base_test import BaseTestClass as MoblyBaseTest 43from mobly.records import ExceptionRecord 44 45# Macro strings for test result reporting 46TEST_CASE_TOKEN = "[Test Case]" 47RESULT_LINE_TEMPLATE = TEST_CASE_TOKEN + " %s %s" 48 49 50@subscribe_static(TestCaseBeginEvent) 51def _logcat_log_test_begin(event): 52 """Ensures that logcat is running. Write a logcat line indicating test case 53 begin.""" 54 test_instance = event.test_class 55 try: 56 for ad in getattr(test_instance, 'android_devices', []): 57 if not ad.is_adb_logcat_on: 58 ad.start_adb_logcat() 59 # Write test start token to adb log if android device is attached. 60 if not ad.skip_sl4a: 61 ad.droid.logV("%s BEGIN %s" % 62 (TEST_CASE_TOKEN, event.test_case_name)) 63 64 except error.ActsError as e: 65 test_instance.results.error.append( 66 ExceptionRecord(e, 'Logcat for test begin: %s' % 67 event.test_case_name)) 68 test_instance.log.error('BaseTest setup_test error: %s' % e.message) 69 70 except Exception as e: 71 test_instance.log.warning( 72 'Unable to send BEGIN log command to all devices.') 73 test_instance.log.warning('Error: %s' % e) 74 75 76@subscribe_static(TestCaseEndEvent) 77def _logcat_log_test_end(event): 78 """Write a logcat line indicating test case end.""" 79 test_instance = event.test_class 80 try: 81 # Write test end token to adb log if android device is attached. 82 for ad in getattr(test_instance, 'android_devices', []): 83 if not ad.skip_sl4a: 84 ad.droid.logV("%s END %s" % 85 (TEST_CASE_TOKEN, event.test_case_name)) 86 87 except error.ActsError as e: 88 test_instance.results.error.append( 89 ExceptionRecord(e, 90 'Logcat for test end: %s' % event.test_case_name)) 91 test_instance.log.error('BaseTest teardown_test error: %s' % e.message) 92 93 except Exception as e: 94 test_instance.log.warning( 95 'Unable to send END log command to all devices.') 96 test_instance.log.warning('Error: %s' % e) 97 98 99@subscribe_static(TestCaseBeginEvent) 100def _syslog_log_test_begin(event): 101 """This adds a BEGIN log message with the test name to the syslog of any 102 Fuchsia device""" 103 test_instance = event.test_class 104 try: 105 for fd in getattr(test_instance, 'fuchsia_devices', []): 106 if not fd.skip_sl4f: 107 fd.logging_lib.logI("%s BEGIN %s" % 108 (TEST_CASE_TOKEN, event.test_case_name)) 109 110 except Exception as e: 111 test_instance.log.warning( 112 'Unable to send BEGIN log command to all devices.') 113 test_instance.log.warning('Error: %s' % e) 114 115 116@subscribe_static(TestCaseEndEvent) 117def _syslog_log_test_end(event): 118 """This adds a END log message with the test name to the syslog of any 119 Fuchsia device""" 120 test_instance = event.test_class 121 try: 122 for fd in getattr(test_instance, 'fuchsia_devices', []): 123 if not fd.skip_sl4f: 124 fd.logging_lib.logI("%s END %s" % 125 (TEST_CASE_TOKEN, event.test_case_name)) 126 127 except Exception as e: 128 test_instance.log.warning( 129 'Unable to send END log command to all devices.') 130 test_instance.log.warning('Error: %s' % e) 131 132 133event_bus.register_subscription(_logcat_log_test_begin.subscription) 134event_bus.register_subscription(_logcat_log_test_end.subscription) 135event_bus.register_subscription(_syslog_log_test_begin.subscription) 136event_bus.register_subscription(_syslog_log_test_end.subscription) 137 138 139class Error(Exception): 140 """Raised for exceptions that occured in BaseTestClass.""" 141 142 143class BaseTestClass(MoblyBaseTest): 144 """Base class for all test classes to inherit from. Inherits some 145 functionality from Mobly's base test class. 146 147 This class gets all the controller objects from test_runner and executes 148 the test cases requested within itself. 149 150 Most attributes of this class are set at runtime based on the configuration 151 provided. 152 153 Attributes: 154 tests: A list of strings, each representing a test case name. 155 TAG: A string used to refer to a test class. Default is the test class 156 name. 157 log: A logger object used for logging. 158 results: A records.TestResult object for aggregating test results from 159 the execution of test cases. 160 controller_configs: A dict of controller configs provided by the user 161 via the testbed config. 162 consecutive_failures: Tracks the number of consecutive test case 163 failures within this class. 164 consecutive_failure_limit: Number of consecutive test failures to allow 165 before blocking remaining tests in the same 166 test class. 167 size_limit_reached: True if the size of the log directory has reached 168 its limit. 169 current_test_name: A string that's the name of the test case currently 170 being executed. If no test is executing, this should 171 be None. 172 """ 173 174 TAG = None 175 176 def __init__(self, configs): 177 """Initializes a BaseTestClass given a TestRunConfig, which provides 178 all of the config information for this test class. 179 180 Args: 181 configs: A config_parser.TestRunConfig object. 182 """ 183 super().__init__(configs) 184 185 self.class_subscriptions = SubscriptionBundle() 186 self.class_subscriptions.register() 187 self.all_subscriptions = [self.class_subscriptions] 188 189 self.current_test_name = None 190 self.log = tracelogger.TraceLogger(logging.getLogger()) 191 # TODO: remove after converging log path definitions with mobly 192 self.log_path = configs.log_path 193 194 self.consecutive_failures = 0 195 self.consecutive_failure_limit = self.user_params.get( 196 'consecutive_failure_limit', -1) 197 self.size_limit_reached = False 198 self.retryable_exceptions = signals.TestFailure 199 200 def _import_builtin_controllers(self): 201 """Import built-in controller modules. 202 203 Go through the testbed configs, find any built-in controller configs 204 and import the corresponding controller module from acts.controllers 205 package. 206 207 Returns: 208 A list of controller modules. 209 """ 210 builtin_controllers = [] 211 for ctrl_name in keys.Config.builtin_controller_names.value: 212 if ctrl_name in self.controller_configs: 213 module_name = keys.get_module_name(ctrl_name) 214 module = importlib.import_module("acts.controllers.%s" % 215 module_name) 216 builtin_controllers.append(module) 217 return builtin_controllers 218 219 @staticmethod 220 def get_module_reference_name(a_module): 221 """Returns the module's reference name. 222 223 This is largely for backwards compatibility with log parsing. If the 224 module defines ACTS_CONTROLLER_REFERENCE_NAME, it will return that 225 value, or the module's submodule name. 226 227 Args: 228 a_module: Any module. Ideally, a controller module. 229 Returns: 230 A string corresponding to the module's name. 231 """ 232 if hasattr(a_module, 'ACTS_CONTROLLER_REFERENCE_NAME'): 233 return a_module.ACTS_CONTROLLER_REFERENCE_NAME 234 else: 235 return a_module.__name__.split('.')[-1] 236 237 def register_controller(self, 238 controller_module, 239 required=True, 240 builtin=False): 241 """Registers an ACTS controller module for a test class. Invokes Mobly's 242 implementation of register_controller. 243 244 An ACTS controller module is a Python lib that can be used to control 245 a device, service, or equipment. To be ACTS compatible, a controller 246 module needs to have the following members: 247 248 def create(configs): 249 [Required] Creates controller objects from configurations. 250 Args: 251 configs: A list of serialized data like string/dict. Each 252 element of the list is a configuration for a 253 controller object. 254 Returns: 255 A list of objects. 256 257 def destroy(objects): 258 [Required] Destroys controller objects created by the create 259 function. Each controller object shall be properly cleaned up 260 and all the resources held should be released, e.g. memory 261 allocation, sockets, file handlers etc. 262 Args: 263 A list of controller objects created by the create function. 264 265 def get_info(objects): 266 [Optional] Gets info from the controller objects used in a test 267 run. The info will be included in test_result_summary.json under 268 the key "ControllerInfo". Such information could include unique 269 ID, version, or anything that could be useful for describing the 270 test bed and debugging. 271 Args: 272 objects: A list of controller objects created by the create 273 function. 274 Returns: 275 A list of json serializable objects, each represents the 276 info of a controller object. The order of the info object 277 should follow that of the input objects. 278 Registering a controller module declares a test class's dependency the 279 controller. If the module config exists and the module matches the 280 controller interface, controller objects will be instantiated with 281 corresponding configs. The module should be imported first. 282 283 Args: 284 controller_module: A module that follows the controller module 285 interface. 286 required: A bool. If True, failing to register the specified 287 controller module raises exceptions. If False, returns None upon 288 failures. 289 builtin: Specifies that the module is a builtin controller module in 290 ACTS. If true, adds itself to test attributes. 291 Returns: 292 A list of controller objects instantiated from controller_module, or 293 None. 294 295 Raises: 296 When required is True, ControllerError is raised if no corresponding 297 config can be found. 298 Regardless of the value of "required", ControllerError is raised if 299 the controller module has already been registered or any other error 300 occurred in the registration process. 301 """ 302 module_ref_name = self.get_module_reference_name(controller_module) 303 304 # Substitute Mobly controller's module config name with the ACTS one 305 module_config_name = controller_module.ACTS_CONTROLLER_CONFIG_NAME 306 controller_module.MOBLY_CONTROLLER_CONFIG_NAME = module_config_name 307 308 # Get controller objects from Mobly's register_controller 309 controllers = self._controller_manager.register_controller( 310 controller_module, required=required) 311 if not controllers: 312 return None 313 314 # Log controller information 315 # Implementation of "get_info" is optional for a controller module. 316 if hasattr(controller_module, "get_info"): 317 controller_info = controller_module.get_info(controllers) 318 self.log.info("Controller %s: %s", module_config_name, 319 controller_info) 320 else: 321 self.log.warning("No controller info obtained for %s", 322 module_config_name) 323 324 if builtin: 325 setattr(self, module_ref_name, controllers) 326 return controllers 327 328 def _setup_class(self): 329 """Proxy function to guarantee the base implementation of setup_class 330 is called. 331 """ 332 event_bus.post(TestClassBeginEvent(self)) 333 # Import and register the built-in controller modules specified 334 # in testbed config. 335 for module in self._import_builtin_controllers(): 336 self.register_controller(module, builtin=True) 337 return self.setup_class() 338 339 def _teardown_class(self): 340 """Proxy function to guarantee the base implementation of teardown_class 341 is called. 342 """ 343 super()._teardown_class() 344 event_bus.post(TestClassEndEvent(self, self.results)) 345 346 def _setup_test(self, test_name): 347 """Proxy function to guarantee the base implementation of setup_test is 348 called. 349 """ 350 self.current_test_name = test_name 351 352 # Skip the test if the consecutive test case failure limit is reached. 353 if self.consecutive_failures == self.consecutive_failure_limit: 354 raise signals.TestError('Consecutive test failure') 355 356 return self.setup_test() 357 358 def setup_test(self): 359 """Setup function that will be called every time before executing each 360 test case in the test class. 361 362 To signal setup failure, return False or raise an exception. If 363 exceptions were raised, the stack trace would appear in log, but the 364 exceptions would not propagate to upper levels. 365 366 Implementation is optional. 367 """ 368 return True 369 370 def _teardown_test(self, test_name): 371 """Proxy function to guarantee the base implementation of teardown_test 372 is called. 373 """ 374 self.log.debug('Tearing down test %s' % test_name) 375 self.teardown_test() 376 377 def _on_fail(self, record): 378 """Proxy function to guarantee the base implementation of on_fail is 379 called. 380 381 Args: 382 record: The records.TestResultRecord object for the failed test 383 case. 384 """ 385 self.consecutive_failures += 1 386 if record.details: 387 self.log.error(record.details) 388 self.log.info(RESULT_LINE_TEMPLATE, record.test_name, record.result) 389 self.on_fail(record.test_name, record.begin_time) 390 391 def on_fail(self, test_name, begin_time): 392 """A function that is executed upon a test case failure. 393 394 User implementation is optional. 395 396 Args: 397 test_name: Name of the test that triggered this function. 398 begin_time: Logline format timestamp taken when the test started. 399 """ 400 def _on_pass(self, record): 401 """Proxy function to guarantee the base implementation of on_pass is 402 called. 403 404 Args: 405 record: The records.TestResultRecord object for the passed test 406 case. 407 """ 408 self.consecutive_failures = 0 409 msg = record.details 410 if msg: 411 self.log.info(msg) 412 self.log.info(RESULT_LINE_TEMPLATE, record.test_name, record.result) 413 self.on_pass(record.test_name, record.begin_time) 414 415 def on_pass(self, test_name, begin_time): 416 """A function that is executed upon a test case passing. 417 418 Implementation is optional. 419 420 Args: 421 test_name: Name of the test that triggered this function. 422 begin_time: Logline format timestamp taken when the test started. 423 """ 424 def _on_skip(self, record): 425 """Proxy function to guarantee the base implementation of on_skip is 426 called. 427 428 Args: 429 record: The records.TestResultRecord object for the skipped test 430 case. 431 """ 432 self.log.info(RESULT_LINE_TEMPLATE, record.test_name, record.result) 433 self.log.info("Reason to skip: %s", record.details) 434 self.on_skip(record.test_name, record.begin_time) 435 436 def on_skip(self, test_name, begin_time): 437 """A function that is executed upon a test case being skipped. 438 439 Implementation is optional. 440 441 Args: 442 test_name: Name of the test that triggered this function. 443 begin_time: Logline format timestamp taken when the test started. 444 """ 445 def _on_exception(self, record): 446 """Proxy function to guarantee the base implementation of on_exception 447 is called. 448 449 Args: 450 record: The records.TestResultRecord object for the failed test 451 case. 452 """ 453 self.log.exception(record.details) 454 self.on_exception(record.test_name, record.begin_time) 455 456 def on_exception(self, test_name, begin_time): 457 """A function that is executed upon an unhandled exception from a test 458 case. 459 460 Implementation is optional. 461 462 Args: 463 test_name: Name of the test that triggered this function. 464 begin_time: Logline format timestamp taken when the test started. 465 """ 466 def on_retry(self): 467 """Function to run before retrying a test through get_func_with_retry. 468 469 This function runs when a test is automatically retried. The function 470 can be used to modify internal test parameters, for example, to retry 471 a test with slightly different input variables. 472 """ 473 def _exec_procedure_func(self, func, tr_record): 474 """Executes a procedure function like on_pass, on_fail etc. 475 476 This function will alternate the 'Result' of the test's record if 477 exceptions happened when executing the procedure function. 478 479 This will let signals.TestAbortAll through so abort_all works in all 480 procedure functions. 481 482 Args: 483 func: The procedure function to be executed. 484 tr_record: The TestResultRecord object associated with the test 485 case executed. 486 """ 487 try: 488 func(tr_record) 489 except signals.TestAbortAll: 490 raise 491 except Exception as e: 492 self.log.exception("Exception happened when executing %s for %s.", 493 func.__name__, self.current_test_name) 494 tr_record.add_error(func.__name__, e) 495 496 def exec_one_testcase(self, test_name, test_func): 497 """Executes one test case and update test results. 498 499 Executes one test case, create a records.TestResultRecord object with 500 the execution information, and add the record to the test class's test 501 results. 502 503 Args: 504 test_name: Name of the test. 505 test_func: The test function. 506 """ 507 class_name = self.__class__.__name__ 508 tr_record = records.TestResultRecord(test_name, class_name) 509 tr_record.test_begin() 510 self.begin_time = int(tr_record.begin_time) 511 self.log_begin_time = tr_record.log_begin_time 512 self.test_name = tr_record.test_name 513 event_bus.post(TestCaseBeginEvent(self, self.test_name)) 514 self.log.info("%s %s", TEST_CASE_TOKEN, test_name) 515 516 # Enable test retry if specified in the ACTS config 517 retry_tests = self.user_params.get('retry_tests', []) 518 full_test_name = '%s.%s' % (class_name, self.test_name) 519 if any(name in retry_tests for name in [class_name, full_test_name]): 520 test_func = self.get_func_with_retry(test_func) 521 522 verdict = None 523 test_signal = None 524 try: 525 try: 526 ret = self._setup_test(self.test_name) 527 asserts.assert_true(ret is not False, 528 "Setup for %s failed." % test_name) 529 verdict = test_func() 530 finally: 531 try: 532 self._teardown_test(self.test_name) 533 except signals.TestAbortAll: 534 raise 535 except Exception as e: 536 self.log.error(traceback.format_exc()) 537 tr_record.add_error("teardown_test", e) 538 self._exec_procedure_func(self._on_exception, tr_record) 539 except (signals.TestFailure, AssertionError) as e: 540 test_signal = e 541 if self.user_params.get( 542 keys.Config.key_test_failure_tracebacks.value, False): 543 self.log.exception(e) 544 tr_record.test_fail(e) 545 self._exec_procedure_func(self._on_fail, tr_record) 546 except signals.TestSkip as e: 547 # Test skipped. 548 test_signal = e 549 tr_record.test_skip(e) 550 self._exec_procedure_func(self._on_skip, tr_record) 551 except (signals.TestAbortClass, signals.TestAbortAll) as e: 552 # Abort signals, pass along. 553 test_signal = e 554 tr_record.test_fail(e) 555 self._exec_procedure_func(self._on_fail, tr_record) 556 raise e 557 except signals.TestPass as e: 558 # Explicit test pass. 559 test_signal = e 560 tr_record.test_pass(e) 561 self._exec_procedure_func(self._on_pass, tr_record) 562 except error.ActsError as e: 563 test_signal = e 564 tr_record.test_error(e) 565 self.log.error('BaseTest execute_one_test_case error: %s' % 566 e.message) 567 except Exception as e: 568 test_signal = e 569 self.log.error(traceback.format_exc()) 570 # Exception happened during test. 571 tr_record.test_error(e) 572 self._exec_procedure_func(self._on_exception, tr_record) 573 self._exec_procedure_func(self._on_fail, tr_record) 574 else: 575 if verdict or (verdict is None): 576 # Test passed. 577 tr_record.test_pass() 578 self._exec_procedure_func(self._on_pass, tr_record) 579 return 580 tr_record.test_fail() 581 self._exec_procedure_func(self._on_fail, tr_record) 582 finally: 583 self.results.add_record(tr_record) 584 self.summary_writer.dump(tr_record.to_dict(), 585 records.TestSummaryEntryType.RECORD) 586 self.current_test_name = None 587 event_bus.post(TestCaseEndEvent(self, self.test_name, test_signal)) 588 589 def get_func_with_retry(self, func, attempts=2): 590 """Returns a wrapped test method that re-runs after failure. Return test 591 result upon success. If attempt limit reached, collect all failure 592 messages and raise a TestFailure signal. 593 594 Params: 595 func: The test method 596 attempts: Number of attempts to run test 597 598 Returns: result of the test method 599 """ 600 exceptions = self.retryable_exceptions 601 602 def wrapper(*args, **kwargs): 603 error_msgs = [] 604 extras = {} 605 retry = False 606 for i in range(attempts): 607 try: 608 if retry: 609 self.teardown_test() 610 self.setup_test() 611 self.on_retry() 612 return func(*args, **kwargs) 613 except exceptions as e: 614 retry = True 615 msg = 'Failure on attempt %d: %s' % (i + 1, e.details) 616 self.log.warning(msg) 617 error_msgs.append(msg) 618 if e.extras: 619 extras['Attempt %d' % (i + 1)] = e.extras 620 raise signals.TestFailure('\n'.join(error_msgs), extras) 621 622 return wrapper 623 624 def run_generated_testcases(self, 625 test_func, 626 settings, 627 args=None, 628 kwargs=None, 629 tag="", 630 name_func=None, 631 format_args=False): 632 """Runs generated test cases. 633 634 Generated test cases are not written down as functions, but as a list 635 of parameter sets. This way we reduce code repetition and improve 636 test case scalability. 637 638 Args: 639 test_func: The common logic shared by all these generated test 640 cases. This function should take at least one argument, 641 which is a parameter set. 642 settings: A list of strings representing parameter sets. These are 643 usually json strings that get loaded in the test_func. 644 args: Iterable of additional position args to be passed to 645 test_func. 646 kwargs: Dict of additional keyword args to be passed to test_func 647 tag: Name of this group of generated test cases. Ignored if 648 name_func is provided and operates properly. 649 name_func: A function that takes a test setting and generates a 650 proper test name. The test name should be shorter than 651 utils.MAX_FILENAME_LEN. Names over the limit will be 652 truncated. 653 format_args: If True, args will be appended as the first argument 654 in the args list passed to test_func. 655 656 Returns: 657 A list of settings that did not pass. 658 """ 659 args = args or () 660 kwargs = kwargs or {} 661 failed_settings = [] 662 663 for setting in settings: 664 test_name = "{} {}".format(tag, setting) 665 666 if name_func: 667 try: 668 test_name = name_func(setting, *args, **kwargs) 669 except: 670 self.log.exception(("Failed to get test name from " 671 "test_func. Fall back to default %s"), 672 test_name) 673 674 self.results.requested.append(test_name) 675 676 if len(test_name) > utils.MAX_FILENAME_LEN: 677 test_name = test_name[:utils.MAX_FILENAME_LEN] 678 679 previous_success_cnt = len(self.results.passed) 680 681 if format_args: 682 self.exec_one_testcase( 683 test_name, 684 functools.partial(test_func, *(args + (setting, )), 685 **kwargs)) 686 else: 687 self.exec_one_testcase( 688 test_name, 689 functools.partial(test_func, *((setting, ) + args), 690 **kwargs)) 691 692 if len(self.results.passed) - previous_success_cnt != 1: 693 failed_settings.append(setting) 694 695 return failed_settings 696 697 def _exec_func(self, func, *args): 698 """Executes a function with exception safeguard. 699 700 This will let signals.TestAbortAll through so abort_all works in all 701 procedure functions. 702 703 Args: 704 func: Function to be executed. 705 args: Arguments to be passed to the function. 706 707 Returns: 708 Whatever the function returns, or False if unhandled exception 709 occured. 710 """ 711 try: 712 return func(*args) 713 except signals.TestAbortAll: 714 raise 715 except: 716 self.log.exception("Exception happened when executing %s in %s.", 717 func.__name__, self.TAG) 718 return False 719 720 def _get_test_funcs(self, test_names): 721 """Obtain the actual functions of test cases based on test names. 722 723 Args: 724 test_names: A list of strings, each string is a test case name. 725 726 Returns: 727 A list of tuples of (string, function). String is the test case 728 name, function is the actual test case function. 729 730 Raises: 731 Error is raised if the test name does not follow 732 naming convention "test_*". This can only be caused by user input 733 here. 734 """ 735 test_funcs = [] 736 for test_name in test_names: 737 test_funcs.append(self._get_test_func(test_name)) 738 739 return test_funcs 740 741 def _get_test_func(self, test_name): 742 """Obtain the actual function of test cases based on the test name. 743 744 Args: 745 test_name: String, The name of the test. 746 747 Returns: 748 A tuples of (string, function). String is the test case 749 name, function is the actual test case function. 750 751 Raises: 752 Error is raised if the test name does not follow 753 naming convention "test_*". This can only be caused by user input 754 here. 755 """ 756 if not test_name.startswith("test_"): 757 raise Error(("Test case name %s does not follow naming " 758 "convention test_*, abort.") % test_name) 759 try: 760 return test_name, getattr(self, test_name) 761 except: 762 763 def test_skip_func(*args, **kwargs): 764 raise signals.TestSkip("Test %s does not exist" % test_name) 765 766 self.log.info("Test case %s not found in %s.", test_name, self.TAG) 767 return test_name, test_skip_func 768 769 def _block_all_test_cases(self, tests, reason='Failed class setup'): 770 """ 771 Block all passed in test cases. 772 Args: 773 tests: The tests to block. 774 reason: Message describing the reason that the tests are blocked. 775 Default is 'Failed class setup' 776 """ 777 for test_name, test_func in tests: 778 signal = signals.TestError(reason) 779 record = records.TestResultRecord(test_name, self.TAG) 780 record.test_begin() 781 if hasattr(test_func, 'gather'): 782 signal.extras = test_func.gather() 783 record.test_error(signal) 784 self.results.add_record(record) 785 self.summary_writer.dump(record.to_dict(), 786 records.TestSummaryEntryType.RECORD) 787 self._on_skip(record) 788 789 def run(self, test_names=None): 790 """Runs test cases within a test class by the order they appear in the 791 execution list. 792 793 One of these test cases lists will be executed, shown here in priority 794 order: 795 1. The test_names list, which is passed from cmd line. 796 2. The self.tests list defined in test class. Invalid names are 797 ignored. 798 3. All function that matches test case naming convention in the test 799 class. 800 801 Args: 802 test_names: A list of string that are test case names/patterns 803 requested in cmd line. 804 805 Returns: 806 The test results object of this class. 807 """ 808 self.register_test_class_event_subscriptions() 809 self.log.info("==========> %s <==========", self.TAG) 810 # Devise the actual test cases to run in the test class. 811 if self.tests: 812 # Specified by run list in class. 813 valid_tests = list(self.tests) 814 else: 815 # No test case specified by user, execute all in the test class 816 valid_tests = self.get_existing_test_names() 817 if test_names: 818 # Match test cases with any of the user-specified patterns 819 matches = [] 820 for test_name in test_names: 821 for valid_test in valid_tests: 822 if (fnmatch.fnmatch(valid_test, test_name) 823 and valid_test not in matches): 824 matches.append(valid_test) 825 else: 826 matches = valid_tests 827 self.results.requested = matches 828 self.summary_writer.dump(self.results.requested_test_names_dict(), 829 records.TestSummaryEntryType.TEST_NAME_LIST) 830 tests = self._get_test_funcs(matches) 831 832 # Setup for the class. 833 setup_fail = False 834 try: 835 if self._setup_class() is False: 836 self.log.error("Failed to setup %s.", self.TAG) 837 self._block_all_test_cases(tests) 838 setup_fail = True 839 except signals.TestAbortClass: 840 self.log.exception('Test class %s aborted' % self.TAG) 841 setup_fail = True 842 except Exception as e: 843 self.log.exception("Failed to setup %s.", self.TAG) 844 self._block_all_test_cases(tests) 845 setup_fail = True 846 if setup_fail: 847 self._exec_func(self._teardown_class) 848 self.log.info("Summary for test class %s: %s", self.TAG, 849 self.results.summary_str()) 850 return self.results 851 852 # Run tests in order. 853 test_case_iterations = self.user_params.get( 854 keys.Config.key_test_case_iterations.value, 1) 855 if any([substr in self.__class__.__name__ for substr in 856 ['Preflight', 'Postflight']]): 857 test_case_iterations = 1 858 try: 859 for test_name, test_func in tests: 860 for _ in range(test_case_iterations): 861 self.exec_one_testcase(test_name, test_func) 862 return self.results 863 except signals.TestAbortClass: 864 self.log.exception('Test class %s aborted' % self.TAG) 865 return self.results 866 except signals.TestAbortAll as e: 867 # Piggy-back test results on this exception object so we don't lose 868 # results from this test class. 869 setattr(e, "results", self.results) 870 raise e 871 finally: 872 self._exec_func(self._teardown_class) 873 self.log.info("Summary for test class %s: %s", self.TAG, 874 self.results.summary_str()) 875 876 def _ad_take_bugreport(self, ad, test_name, begin_time): 877 for i in range(3): 878 try: 879 ad.take_bug_report(test_name, begin_time) 880 return True 881 except Exception as e: 882 ad.log.error("bugreport attempt %s error: %s", i + 1, e) 883 884 def _ad_take_extra_logs(self, ad, test_name, begin_time): 885 result = True 886 if getattr(ad, "qxdm_log", False): 887 # Gather qxdm log modified 3 minutes earlier than test start time 888 if begin_time: 889 qxdm_begin_time = begin_time - 1000 * 60 * 3 890 else: 891 qxdm_begin_time = None 892 try: 893 ad.get_qxdm_logs(test_name, qxdm_begin_time) 894 except Exception as e: 895 ad.log.error("Failed to get QXDM log for %s with error %s", 896 test_name, e) 897 result = False 898 899 try: 900 ad.check_crash_report(test_name, begin_time, log_crash_report=True) 901 except Exception as e: 902 ad.log.error("Failed to check crash report for %s with error %s", 903 test_name, e) 904 result = False 905 return result 906 907 def _skip_bug_report(self, test_name): 908 """A function to check whether we should skip creating a bug report. 909 910 Args: 911 test_name: The test case name 912 913 Returns: True if bug report is to be skipped. 914 """ 915 if "no_bug_report_on_fail" in self.user_params: 916 return True 917 918 # If the current test class or test case is found in the set of 919 # problematic tests, we skip bugreport and other failure artifact 920 # creation. 921 class_name = self.__class__.__name__ 922 quiet_tests = self.user_params.get('quiet_tests', []) 923 if class_name in quiet_tests: 924 self.log.info( 925 "Skipping bug report, as directed for this test class.") 926 return True 927 full_test_name = '%s.%s' % (class_name, test_name) 928 if full_test_name in quiet_tests: 929 self.log.info( 930 "Skipping bug report, as directed for this test case.") 931 return True 932 933 # Once we hit a certain log path size, it's not going to get smaller. 934 # We cache the result so we don't have to keep doing directory walks. 935 if self.size_limit_reached: 936 return True 937 try: 938 max_log_size = int( 939 self.user_params.get("soft_output_size_limit") or "invalid") 940 log_path = getattr(logging, "log_path", None) 941 if log_path: 942 curr_log_size = utils.get_directory_size(log_path) 943 if curr_log_size > max_log_size: 944 self.log.info( 945 "Skipping bug report, as we've reached the size limit." 946 ) 947 self.size_limit_reached = True 948 return True 949 except ValueError: 950 pass 951 return False 952 953 def _take_bug_report(self, test_name, begin_time): 954 if self._skip_bug_report(test_name): 955 return 956 957 executor = ThreadPoolExecutor(max_workers=10) 958 for ad in getattr(self, 'android_devices', []): 959 executor.submit(self._ad_take_bugreport, ad, test_name, begin_time) 960 executor.submit(self._ad_take_extra_logs, ad, test_name, 961 begin_time) 962 executor.shutdown() 963 964 def _reboot_device(self, ad): 965 ad.log.info("Rebooting device.") 966 ad = ad.reboot() 967 968 def _cleanup_logger_sessions(self): 969 for (mylogger, session) in self.logger_sessions: 970 self.log.info("Resetting a diagnostic session %s, %s", mylogger, 971 session) 972 mylogger.reset() 973 self.logger_sessions = [] 974 975 def _pull_diag_logs(self, test_name, begin_time): 976 for (mylogger, session) in self.logger_sessions: 977 self.log.info("Pulling diagnostic session %s", mylogger) 978 mylogger.stop(session) 979 diag_path = os.path.join( 980 self.log_path, logger.epoch_to_log_line_timestamp(begin_time)) 981 os.makedirs(diag_path, exist_ok=True) 982 mylogger.pull(session, diag_path) 983 984 def register_test_class_event_subscriptions(self): 985 self.class_subscriptions = subscription_bundle.create_from_instance( 986 self) 987 self.class_subscriptions.register() 988 989 def unregister_test_class_event_subscriptions(self): 990 for package in self.all_subscriptions: 991 package.unregister() 992