1# 2# Copyright (C) 2017 The Android Open Source Project 3# 4# Licensed under the Apache License, Version 2.0 (the "License"); 5# you may not use this file except in compliance with the License. 6# You may obtain a copy of the License at 7# 8# http://www.apache.org/licenses/LICENSE-2.0 9# 10# Unless required by applicable law or agreed to in writing, software 11# distributed under the License is distributed on an "AS IS" BASIS, 12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13# See the License for the specific language governing permissions and 14# limitations under the License. 15# 16 17import importlib 18import json 19import logging 20import os 21import sys 22import time 23import yaml 24 25from vts.runners.host import asserts 26from vts.runners.host import base_test 27from vts.runners.host import config_parser 28from vts.runners.host import keys 29from vts.runners.host import records 30from vts.runners.host import test_runner 31from vts.utils.python.io import capture_printout 32from vts.utils.python.io import file_util 33 34from mobly import test_runner as mobly_test_runner 35 36 37LIST_TEST_OUTPUT_START = '==========> ' 38LIST_TEST_OUTPUT_END = ' <==========' 39# Temp directory inside python log path. The name is required to be 40# the set value for tradefed to skip reading contents as logs. 41TEMP_DIR_NAME = 'temp' 42CONFIG_FILE_NAME = 'test_config.yaml' 43MOBLY_RESULT_JSON_FILE_NAME = 'test_run_summary.json' 44MOBLY_RESULT_YAML_FILE_NAME = 'test_summary.yaml' 45 46 47MOBLY_CONFIG_TEXT = '''TestBeds: 48 - Name: {module_name} 49 Controllers: 50 AndroidDevice: 51 - serial: {serial1} 52 - serial: {serial2} 53 54MoblyParams: 55 LogPath: {log_path} 56''' 57 58#TODO(yuexima): 59# 1. make DEVICES_REQUIRED configurable 60# 2. add include filter function 61DEVICES_REQUIRED = 2 62 63RESULT_KEY_TYPE = 'Type' 64RESULT_TYPE_SUMMARY = 'Summary' 65RESULT_TYPE_RECORD = 'Record' 66RESULT_TYPE_TEST_NAME_LIST = 'TestNameList' 67RESULT_TYPE_CONTROLLER_INFO = 'ControllerInfo' 68 69 70class MoblyTest(base_test.BaseTestClass): 71 '''Template class for running mobly test cases. 72 73 Attributes: 74 mobly_dir: string, mobly test temp directory for mobly runner 75 mobly_config_file_path: string, mobly test config file path 76 result_handlers: dict, map of result type and handler functions 77 ''' 78 def setUpClass(self): 79 asserts.assertEqual( 80 len(self.android_devices), DEVICES_REQUIRED, 81 'Exactly %s devices are required for this test.' % DEVICES_REQUIRED 82 ) 83 84 for ad in self.android_devices: 85 logging.debug('Android device serial: %s' % ad.serial) 86 87 logging.debug('Test cases: %s' % self.ListTestCases()) 88 89 self.mobly_dir = os.path.join(logging.log_path, TEMP_DIR_NAME, 90 'mobly', str(time.time())) 91 92 file_util.Makedirs(self.mobly_dir) 93 94 logging.debug('mobly log path: %s' % self.mobly_dir) 95 96 self.result_handlers = { 97 RESULT_TYPE_SUMMARY: self.HandleSimplePrint, 98 RESULT_TYPE_RECORD: self.HandleRecord, 99 RESULT_TYPE_TEST_NAME_LIST: self.HandleSimplePrint, 100 RESULT_TYPE_CONTROLLER_INFO: self.HandleSimplePrint, 101 } 102 103 def tearDownClass(self): 104 ''' Clear the mobly directory.''' 105 file_util.Rmdirs(self.mobly_dir, ignore_errors=True) 106 107 def PrepareConfigFile(self): 108 '''Prepare mobly config file for running test.''' 109 self.mobly_config_file_path = os.path.join(self.mobly_dir, 110 CONFIG_FILE_NAME) 111 config_text = MOBLY_CONFIG_TEXT.format( 112 module_name=self.test_module_name, 113 serial1=self.android_devices[0].serial, 114 serial2=self.android_devices[1].serial, 115 log_path=self.mobly_dir 116 ) 117 with open(self.mobly_config_file_path, 'w') as f: 118 f.write(config_text) 119 120 def ListTestCases(self): 121 '''List test cases. 122 123 Returns: 124 List of string, test names. 125 ''' 126 classes = mobly_test_runner._find_test_class() 127 128 with capture_printout.CaptureStdout() as output: 129 mobly_test_runner._print_test_names(classes) 130 131 test_names = [] 132 133 for line in output: 134 if (not line.startswith(LIST_TEST_OUTPUT_START) 135 and line.endswith(LIST_TEST_OUTPUT_END)): 136 test_names.append(line) 137 tr_record = records.TestResultRecord(line, self.test_module_name) 138 self.results.requested.append(tr_record) 139 140 return test_names 141 142 def RunMoblyModule(self): 143 '''Execute mobly test module.''' 144 # Because mobly and vts uses a similar runner, both will modify 145 # log_path from python logging. The following step is to preserve 146 # log path after mobly test finishes. 147 148 # An alternative way is to start a new python process through shell 149 # command. In that case, test print out needs to be piped. 150 # This will also help avoid log overlapping 151 152 logger = logging.getLogger() 153 logger_path = logger.log_path 154 logging_path = logging.log_path 155 156 try: 157 mobly_test_runner.main(argv=['-c', self.mobly_config_file_path]) 158 finally: 159 logger.log_path = logger_path 160 logging.log_path = logging_path 161 162 def GetMoblyResults(self): 163 '''Get mobly module run results and put in vts results.''' 164 file_handlers = ( 165 (MOBLY_RESULT_YAML_FILE_NAME, self.ParseYamlResults), 166 (MOBLY_RESULT_JSON_FILE_NAME, self.ParseJsonResults), 167 ) 168 169 for pair in file_handlers: 170 file_path = file_util.FindFile(self.mobly_dir, pair[0]) 171 172 if file_path: 173 logging.debug('Mobly test yaml result path: %s', file_path) 174 pair[1](file_path) 175 return 176 177 asserts.fail('Mobly test result file not found.') 178 179 def generateAllTests(self): 180 '''Run the mobly test module and parse results.''' 181 #TODO(yuexima): report test names 182 183 self.PrepareConfigFile() 184 self.RunMoblyModule() 185 #TODO(yuexima): check whether DEBUG logs from mobly run are included 186 self.GetMoblyResults() 187 188 def ParseJsonResults(self, result_path): 189 '''Parse mobly test json result. 190 191 Args: 192 result_path: string, result json file path. 193 ''' 194 with open(path, 'r') as f: 195 mobly_summary = json.load(f) 196 197 mobly_results = mobly_summary['Results'] 198 for result in mobly_results: 199 logging.debug('Adding result for %s' % result[records.TestResultEnums.RECORD_NAME]) 200 record = records.TestResultRecord(result[records.TestResultEnums.RECORD_NAME]) 201 record.test_class = result[records.TestResultEnums.RECORD_CLASS] 202 record.begin_time = result[records.TestResultEnums.RECORD_BEGIN_TIME] 203 record.end_time = result[records.TestResultEnums.RECORD_END_TIME] 204 record.result = result[records.TestResultEnums.RECORD_RESULT] 205 record.uid = result[records.TestResultEnums.RECORD_UID] 206 record.extras = result[records.TestResultEnums.RECORD_EXTRAS] 207 record.details = result[records.TestResultEnums.RECORD_DETAILS] 208 record.extra_errors = result[records.TestResultEnums.RECORD_EXTRA_ERRORS] 209 210 self.results.addRecord(record) 211 212 def ParseYamlResults(self, result_path): 213 '''Parse mobly test yaml result. 214 215 Args: 216 result_path: string, result yaml file path. 217 ''' 218 with open(result_path, 'r') as stream: 219 try: 220 docs = yaml.load_all(stream) 221 for doc in docs: 222 type = doc.get(RESULT_KEY_TYPE) 223 if type is None: 224 logging.warn( 225 'Mobly result document type unrecognized: %s', doc) 226 continue 227 228 logging.debug('Parsing result type: %s', type) 229 230 handler = self.result_handlers.get(type) 231 if handler is None: 232 logging.debug('Unknown result type: %s', type) 233 handler = self.HandleSimplePrint 234 235 handler(doc) 236 except yaml.YAMLError as exc: 237 print(exc) 238 239 def HandleRecord(self, doc): 240 '''Handle record result document type. 241 242 Args: 243 doc: dict, result document item 244 ''' 245 logging.debug('Adding result for %s' % doc.get(records.TestResultEnums.RECORD_NAME)) 246 record = records.TestResultRecord(doc.get(records.TestResultEnums.RECORD_NAME)) 247 record.test_class = doc.get(records.TestResultEnums.RECORD_CLASS) 248 record.begin_time = doc.get(records.TestResultEnums.RECORD_BEGIN_TIME) 249 record.end_time = doc.get(records.TestResultEnums.RECORD_END_TIME) 250 record.result = doc.get(records.TestResultEnums.RECORD_RESULT) 251 record.uid = doc.get(records.TestResultEnums.RECORD_UID) 252 record.extras = doc.get(records.TestResultEnums.RECORD_EXTRAS) 253 record.details = doc.get(records.TestResultEnums.RECORD_DETAILS) 254 record.extra_errors = doc.get(records.TestResultEnums.RECORD_EXTRA_ERRORS) 255 256 # 'Stacktrace' in yaml result is ignored. 'Stacktrace' is a more 257 # detailed version of record.details when exception is emitted. 258 259 self.results.addRecord(record) 260 261 def HandleSimplePrint(self, doc): 262 '''Simply print result document to log. 263 264 Args: 265 doc: dict, result document item 266 ''' 267 for k, v in doc.items(): 268 logging.debug(str(k) + ": " + str(v)) 269 270def GetTestModuleNames(): 271 '''Returns a list of mobly test module specified in test configuration.''' 272 configs = config_parser.load_test_config_file(sys.argv[1]) 273 reduce_func = lambda x, y: x + y.get(keys.ConfigKeys.MOBLY_TEST_MODULE, []) 274 return reduce(reduce_func, configs, []) 275 276def ImportTestModules(): 277 '''Dynamically import mobly test modules.''' 278 for module_name in GetTestModuleNames(): 279 module, cls = module_name.rsplit('.', 1) 280 sys.modules['__main__'].__dict__[cls] = getattr( 281 importlib.import_module(module), cls) 282 283if __name__ == "__main__": 284 ImportTestModules() 285 test_runner.main() 286