1#!/usr/bin/env python3.4 2# 3# Copyright 2016 - The Android Open Source Project 4# 5# Licensed under the Apache License, Version 2.0 (the "License"); 6# you may not use this file except in compliance with the License. 7# You may obtain a copy of the License at 8# 9# http://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an "AS IS" BASIS, 13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14# See the License for the specific language governing permissions and 15# limitations under the License. 16 17__author__ = "angli@google.com" 18 19from future import standard_library 20standard_library.install_aliases() 21 22import copy 23import importlib 24import inspect 25import os 26import pkgutil 27import sys 28 29from acts import keys 30from acts import logger 31from acts import records 32from acts import signals 33from acts import utils 34 35 36class USERError(Exception): 37 """Raised when a problem is caused by user mistake, e.g. wrong command, 38 misformatted config, test info, wrong test paths etc. 39 """ 40 41class TestRunner(object): 42 """The class that instantiates test classes, executes test cases, and 43 report results. 44 45 Attributes: 46 self.test_run_info: A dictionary containing the information needed by 47 test classes for this test run, including params, 48 controllers, and other objects. All of these will 49 be passed to test classes. 50 self.test_configs: A dictionary that is the original test configuration 51 passed in by user. 52 self.id: A string that is the unique identifier of this test run. 53 self.log_path: A string representing the path of the dir under which 54 all logs from this test run should be written. 55 self.log: The logger object used throughout this test run. 56 self.controller_registry: A dictionary that holds the controller 57 objects used in a test run. 58 self.controller_destructors: A dictionary that holds the controller 59 distructors. Keys are controllers' names. 60 self.test_classes: A dictionary where we can look up the test classes 61 by name to instantiate. 62 self.run_list: A list of tuples specifying what tests to run. 63 self.results: The test result object used to record the results of 64 this test run. 65 self.running: A boolean signifies whether this test run is ongoing or 66 not. 67 """ 68 def __init__(self, test_configs, run_list): 69 self.test_run_info = {} 70 self.test_configs = test_configs 71 self.testbed_configs = self.test_configs[keys.Config.key_testbed.value] 72 self.testbed_name = self.testbed_configs[keys.Config.key_testbed_name.value] 73 start_time = logger.get_log_file_timestamp() 74 self.id = "{}@{}".format(self.testbed_name, start_time) 75 # log_path should be set before parsing configs. 76 l_path = os.path.join(self.test_configs[keys.Config.key_log_path.value], 77 self.testbed_name, 78 start_time) 79 self.log_path = os.path.abspath(l_path) 80 self.log = logger.get_test_logger(self.log_path, 81 self.id, 82 self.testbed_name) 83 self.controller_registry = {} 84 self.controller_destructors = {} 85 self.run_list = run_list 86 self.results = records.TestResult() 87 self.running = False 88 89 def import_test_modules(self, test_paths): 90 """Imports test classes from test scripts. 91 92 1. Locate all .py files under test paths. 93 2. Import the .py files as modules. 94 3. Find the module members that are test classes. 95 4. Categorize the test classes by name. 96 97 Args: 98 test_paths: A list of directory paths where the test files reside. 99 100 Returns: 101 A dictionary where keys are test class name strings, values are 102 actual test classes that can be instantiated. 103 """ 104 def is_testfile_name(name, ext): 105 if ext == ".py": 106 if name.endswith("Test") or name.endswith("_test"): 107 return True 108 return False 109 file_list = utils.find_files(test_paths, is_testfile_name) 110 test_classes = {} 111 for path, name, _ in file_list: 112 sys.path.append(path) 113 try: 114 module = importlib.import_module(name) 115 except: 116 for test_cls_name, _ in self.run_list: 117 alt_name = name.replace('_', '').lower() 118 alt_cls_name = test_cls_name.lower() 119 # Only block if a test class on the run list causes an 120 # import error. We need to check against both naming 121 # conventions: AaaBbb and aaa_bbb. 122 if name == test_cls_name or alt_name == alt_cls_name: 123 msg = ("Encountered error importing test class %s, " 124 "abort.") % test_cls_name 125 # This exception is logged here to help with debugging 126 # under py2, because "raise X from Y" syntax is only 127 # supported under py3. 128 self.log.exception(msg) 129 raise USERError(msg) 130 continue 131 for member_name in dir(module): 132 if not member_name.startswith("__"): 133 if member_name.endswith("Test"): 134 test_class = getattr(module, member_name) 135 if inspect.isclass(test_class): 136 test_classes[member_name] = test_class 137 return test_classes 138 139 @staticmethod 140 def verify_controller_module(module): 141 """Verifies a module object follows the required interface for 142 controllers. 143 144 Args: 145 module: An object that is a controller module. This is usually 146 imported with import statements or loaded by importlib. 147 148 Raises: 149 ControllerError is raised if the module does not match the ACTS 150 controller interface, or one of the required members is null. 151 """ 152 required_attributes = ("create", 153 "destroy", 154 "ACTS_CONTROLLER_CONFIG_NAME") 155 for attr in required_attributes: 156 if not hasattr(module, attr): 157 raise signals.ControllerError(("Module %s missing required " 158 "controller module attribute %s.") % (module.__name__, 159 attr)) 160 if not getattr(module, attr): 161 raise signals.ControllerError(("Controller interface %s in %s " 162 "cannot be null.") % (attr, module.__name__)) 163 164 def register_controller(self, module, required=True): 165 """Registers a controller module for a test run. 166 167 This declares a controller dependency of this test class. If the target 168 module exists and matches the controller interface, the controller 169 module will be instantiated with corresponding configs in the test 170 config file. The module should be imported first. 171 172 Params: 173 module: A module that follows the controller module interface. 174 required: A bool. If True, failing to register the specified 175 controller module raises exceptions. If False, returns 176 None upon failures. 177 178 Returns: 179 A list of controller objects instantiated from controller_module, or 180 None. 181 182 Raises: 183 When required is True, ControllerError is raised if no corresponding 184 config can be found. 185 Regardless of the value of "required", ControllerError is raised if 186 the controller module has already been registered or any other error 187 occurred in the registration process. 188 """ 189 TestRunner.verify_controller_module(module) 190 try: 191 # If this is a builtin controller module, use the default ref name. 192 module_ref_name = module.ACTS_CONTROLLER_REFERENCE_NAME 193 builtin = True 194 except AttributeError: 195 # Or use the module's name 196 builtin = False 197 module_ref_name = module.__name__.split('.')[-1] 198 if module_ref_name in self.controller_registry: 199 raise signals.ControllerError(("Controller module %s has already " 200 "been registered. It can not be " 201 "registered again." 202 ) % module_ref_name) 203 # Create controller objects. 204 create = module.create 205 module_config_name = module.ACTS_CONTROLLER_CONFIG_NAME 206 if module_config_name not in self.testbed_configs: 207 if required: 208 raise signals.ControllerError( 209 "No corresponding config found for %s" % 210 module_config_name) 211 self.log.warning( 212 "No corresponding config found for optional controller %s", 213 module_config_name) 214 return None 215 try: 216 # Make a deep copy of the config to pass to the controller module, 217 # in case the controller module modifies the config internally. 218 original_config = self.testbed_configs[module_config_name] 219 controller_config = copy.deepcopy(original_config) 220 objects = create(controller_config, self.log) 221 except: 222 self.log.exception(("Failed to initialize objects for controller " 223 "%s, abort!"), module_config_name) 224 raise 225 if not isinstance(objects, list): 226 raise ControllerError(("Controller module %s did not return a list" 227 " of objects, abort.") % module_ref_name) 228 self.controller_registry[module_ref_name] = objects 229 # TODO(angli): After all tests move to register_controller, stop 230 # tracking controller objs in test_run_info. 231 if builtin: 232 self.test_run_info[module_ref_name] = objects 233 self.log.debug("Found %d objects for controller %s", len(objects), 234 module_config_name) 235 destroy_func = module.destroy 236 self.controller_destructors[module_ref_name] = destroy_func 237 return objects 238 239 def unregister_controllers(self): 240 """Destroy controller objects and clear internal registry. 241 242 This will be called at the end of each TestRunner.run call. 243 """ 244 for name, destroy in self.controller_destructors.items(): 245 try: 246 self.log.debug("Destroying %s.", name) 247 destroy(self.controller_registry[name]) 248 except: 249 self.log.exception("Exception occurred destroying %s.", name) 250 self.controller_registry = {} 251 self.controller_destructors = {} 252 253 def parse_config(self, test_configs): 254 """Parses the test configuration and unpacks objects and parameters 255 into a dictionary to be passed to test classes. 256 257 Args: 258 test_configs: A json object representing the test configurations. 259 """ 260 self.test_run_info[keys.Config.ikey_testbed_name.value] = self.testbed_name 261 # Instantiate builtin controllers 262 for ctrl_name in keys.Config.builtin_controller_names.value: 263 if ctrl_name in self.testbed_configs: 264 module_name = keys.get_module_name(ctrl_name) 265 module = importlib.import_module("acts.controllers.%s" % 266 module_name) 267 self.register_controller(module) 268 # Unpack other params. 269 self.test_run_info["register_controller"] = self.register_controller 270 self.test_run_info[keys.Config.ikey_logpath.value] = self.log_path 271 self.test_run_info[keys.Config.ikey_logger.value] = self.log 272 cli_args = test_configs[keys.Config.ikey_cli_args.value] 273 self.test_run_info[keys.Config.ikey_cli_args.value] = cli_args 274 user_param_pairs = [] 275 for item in test_configs.items(): 276 if item[0] not in keys.Config.reserved_keys.value: 277 user_param_pairs.append(item) 278 self.test_run_info[keys.Config.ikey_user_param.value] = copy.deepcopy( 279 dict(user_param_pairs)) 280 281 def set_test_util_logs(self, module=None): 282 """Sets the log object to each test util module. 283 284 This recursively include all modules under acts.test_utils and sets the 285 main test logger to each module. 286 287 Args: 288 module: A module under acts.test_utils. 289 """ 290 # Initial condition of recursion. 291 if not module: 292 module = importlib.import_module("acts.test_utils") 293 # Somehow pkgutil.walk_packages is not working for me. 294 # Using iter_modules for now. 295 pkg_iter = pkgutil.iter_modules(module.__path__, module.__name__ + '.') 296 for _, module_name, ispkg in pkg_iter: 297 m = importlib.import_module(module_name) 298 if ispkg: 299 self.set_test_util_logs(module=m) 300 else: 301 self.log.debug("Setting logger to test util module %s", 302 module_name) 303 setattr(m, "log", self.log) 304 305 def run_test_class(self, test_cls_name, test_cases=None): 306 """Instantiates and executes a test class. 307 308 If test_cases is None, the test cases listed by self.tests will be 309 executed instead. If self.tests is empty as well, no test case in this 310 test class will be executed. 311 312 Args: 313 test_cls_name: Name of the test class to execute. 314 test_cases: List of test case names to execute within the class. 315 316 Returns: 317 A tuple, with the number of cases passed at index 0, and the total 318 number of test cases at index 1. 319 """ 320 try: 321 test_cls = self.test_classes[test_cls_name] 322 except KeyError: 323 raise USERError(("Unable to locate class %s in any of the test " 324 "paths specified.") % test_cls_name) 325 326 with test_cls(self.test_run_info) as test_cls_instance: 327 try: 328 cls_result = test_cls_instance.run(test_cases) 329 self.results += cls_result 330 except signals.TestAbortAll as e: 331 self.results += e.results 332 raise e 333 334 def run(self): 335 """Executes test cases. 336 337 This will instantiate controller and test classes, and execute test 338 classes. This can be called multiple times to repeatly execute the 339 requested test cases. 340 341 A call to TestRunner.stop should eventually happen to conclude the life 342 cycle of a TestRunner. 343 """ 344 if not self.running: 345 self.running = True 346 # Initialize controller objects and pack appropriate objects/params 347 # to be passed to test class. 348 self.parse_config(self.test_configs) 349 t_configs = self.test_configs[keys.Config.key_test_paths.value] 350 self.test_classes = self.import_test_modules(t_configs) 351 self.log.debug("Executing run list %s.", self.run_list) 352 try: 353 for test_cls_name, test_case_names in self.run_list: 354 if not self.running: 355 break 356 if test_case_names: 357 self.log.debug("Executing test cases %s in test class %s.", 358 test_case_names, 359 test_cls_name) 360 else: 361 self.log.debug("Executing test class %s", test_cls_name) 362 try: 363 self.run_test_class(test_cls_name, test_case_names) 364 except signals.TestAbortAll as e: 365 self.log.warning(("Abort all subsequent test classes. Reason: " 366 "%s"), e) 367 raise 368 finally: 369 self.unregister_controllers() 370 371 def stop(self): 372 """Releases resources from test run. Should always be called after 373 TestRunner.run finishes. 374 375 This function concludes a test run and writes out a test report. 376 """ 377 if self.running: 378 msg = "\nSummary for test run %s: %s\n" % (self.id, 379 self.results.summary_str()) 380 self._write_results_json_str() 381 self.log.info(msg.strip()) 382 logger.kill_test_logger(self.log) 383 self.running = False 384 385 def _write_results_json_str(self): 386 """Writes out a json file with the test result info for easy parsing. 387 388 TODO(angli): This should be replaced by standard log record mechanism. 389 """ 390 path = os.path.join(self.log_path, "test_run_summary.json") 391 with open(path, 'w') as f: 392 f.write(self.results.json_str()) 393 394if __name__ == "__main__": 395 pass 396