1# 2# Copyright (C) 2016 The Android Open Source Project 3# 4# Licensed under the Apache License, Version 2.0 (the "License"); 5# you may not use this file except in compliance with the License. 6# You may obtain a copy of the License at 7# 8# http://www.apache.org/licenses/LICENSE-2.0 9# 10# Unless required by applicable law or agreed to in writing, software 11# distributed under the License is distributed on an "AS IS" BASIS, 12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13# See the License for the specific language governing permissions and 14# limitations under the License. 15# 16 17import logging 18import os 19import time 20 21from vts.runners.host import asserts 22from vts.runners.host import base_test 23from vts.runners.host import const 24from vts.runners.host import keys 25from vts.runners.host import test_runner 26from vts.utils.python.controllers import android_device 27from vts.utils.python.common import list_utils 28from vts.utils.python.os import path_utils 29from vts.utils.python.precondition import precondition_utils 30from vts.utils.python.web import feature_utils 31 32from vts.testcases.template.binary_test import binary_test_case 33 34 35class BinaryTest(base_test.BaseTestClass): 36 '''Base class to run binary tests on target. 37 38 Attributes: 39 _dut: AndroidDevice, the device under test as config 40 shell: ShellMirrorObject, shell mirror 41 testcases: list of BinaryTestCase objects, list of test cases to run 42 tags: all the tags that appeared in binary list 43 DEVICE_TMP_DIR: string, temp location for storing binary 44 TAG_DELIMITER: string, separator used to separate tag and path 45 SYSPROP_VTS_NATIVE_SERVER: string, the name of a system property which 46 tells whether to stop properly configured 47 native servers where properly configured 48 means a server's init.rc is configured to 49 stop when that property's value is 1. 50 ''' 51 SYSPROP_VTS_NATIVE_SERVER = "vts.native_server.on" 52 53 DEVICE_TMP_DIR = '/data/local/tmp' 54 TAG_DELIMITER = '::' 55 PUSH_DELIMITER = '->' 56 DEFAULT_TAG_32 = '_32bit' 57 DEFAULT_TAG_64 = '_64bit' 58 DEFAULT_LD_LIBRARY_PATH_32 = '/data/local/tmp/32/' 59 DEFAULT_LD_LIBRARY_PATH_64 = '/data/local/tmp/64/' 60 DEFAULT_PROFILING_LIBRARY_PATH_32 = '/data/local/tmp/32/' 61 DEFAULT_PROFILING_LIBRARY_PATH_64 = '/data/local/tmp/64/' 62 63 def setUpClass(self): 64 '''Prepare class, push binaries, set permission, create test cases.''' 65 required_params = [keys.ConfigKeys.IKEY_DATA_FILE_PATH, ] 66 opt_params = [ 67 keys.ConfigKeys.IKEY_BINARY_TEST_SOURCE, 68 keys.ConfigKeys.IKEY_BINARY_TEST_WORKING_DIRECTORY, 69 keys.ConfigKeys.IKEY_BINARY_TEST_ENVP, 70 keys.ConfigKeys.IKEY_BINARY_TEST_ARGS, 71 keys.ConfigKeys.IKEY_BINARY_TEST_LD_LIBRARY_PATH, 72 keys.ConfigKeys.IKEY_BINARY_TEST_PROFILING_LIBRARY_PATH, 73 keys.ConfigKeys.IKEY_BINARY_TEST_DISABLE_FRAMEWORK, 74 keys.ConfigKeys.IKEY_BINARY_TEST_STOP_NATIVE_SERVERS, 75 keys.ConfigKeys.IKEY_NATIVE_SERVER_PROCESS_NAME, 76 ] 77 self.getUserParams( 78 req_param_names=required_params, opt_param_names=opt_params) 79 80 # test-module-name is required in binary tests. 81 self.getUserParam( 82 keys.ConfigKeys.KEY_TESTBED_NAME, error_if_not_found=True) 83 84 logging.info("%s: %s", keys.ConfigKeys.IKEY_DATA_FILE_PATH, 85 self.data_file_path) 86 87 self.binary_test_source = self.getUserParam( 88 keys.ConfigKeys.IKEY_BINARY_TEST_SOURCE, default_value=[]) 89 90 self.working_directory = {} 91 if hasattr(self, keys.ConfigKeys.IKEY_BINARY_TEST_WORKING_DIRECTORY): 92 self.binary_test_working_directory = map( 93 str, self.binary_test_working_directory) 94 for token in self.binary_test_working_directory: 95 tag = '' 96 path = token 97 if self.TAG_DELIMITER in token: 98 tag, path = token.split(self.TAG_DELIMITER) 99 self.working_directory[tag] = path 100 101 self.envp = {} 102 if hasattr(self, keys.ConfigKeys.IKEY_BINARY_TEST_ENVP): 103 self.binary_test_envp = map(str, self.binary_test_envp) 104 for token in self.binary_test_envp: 105 tag = '' 106 path = token 107 if self.TAG_DELIMITER in token: 108 tag, path = token.split(self.TAG_DELIMITER) 109 if tag in self.envp: 110 self.envp[tag] += ' %s' % path 111 else: 112 self.envp[tag] = path 113 114 self.args = {} 115 if hasattr(self, keys.ConfigKeys.IKEY_BINARY_TEST_ARGS): 116 self.binary_test_args = map(str, self.binary_test_args) 117 for token in self.binary_test_args: 118 tag = '' 119 arg = token 120 if self.TAG_DELIMITER in token: 121 tag, arg = token.split(self.TAG_DELIMITER) 122 if tag in self.args: 123 self.args[tag] += ' %s' % arg 124 else: 125 self.args[tag] = arg 126 127 self.ld_library_path = { 128 self.DEFAULT_TAG_32: self.DEFAULT_LD_LIBRARY_PATH_32, 129 self.DEFAULT_TAG_64: self.DEFAULT_LD_LIBRARY_PATH_64, 130 } 131 if hasattr(self, keys.ConfigKeys.IKEY_BINARY_TEST_LD_LIBRARY_PATH): 132 self.binary_test_ld_library_path = map( 133 str, self.binary_test_ld_library_path) 134 for token in self.binary_test_ld_library_path: 135 tag = '' 136 path = token 137 if self.TAG_DELIMITER in token: 138 tag, path = token.split(self.TAG_DELIMITER) 139 if tag in self.ld_library_path: 140 self.ld_library_path[tag] = '{}:{}'.format( 141 path, self.ld_library_path[tag]) 142 else: 143 self.ld_library_path[tag] = path 144 145 self.profiling_library_path = { 146 self.DEFAULT_TAG_32: self.DEFAULT_PROFILING_LIBRARY_PATH_32, 147 self.DEFAULT_TAG_64: self.DEFAULT_PROFILING_LIBRARY_PATH_64, 148 } 149 if hasattr(self, 150 keys.ConfigKeys.IKEY_BINARY_TEST_PROFILING_LIBRARY_PATH): 151 self.binary_test_profiling_library_path = map( 152 str, self.binary_test_profiling_library_path) 153 for token in self.binary_test_profiling_library_path: 154 tag = '' 155 path = token 156 if self.TAG_DELIMITER in token: 157 tag, path = token.split(self.TAG_DELIMITER) 158 self.profiling_library_path[tag] = path 159 160 if not hasattr(self, "_dut"): 161 self._dut = self.registerController(android_device)[0] 162 163 self._dut.shell.InvokeTerminal("one", int(self.abi_bitness)) 164 self.shell = self._dut.shell.one 165 166 if self.coverage.enabled: 167 self.coverage.LoadArtifacts() 168 self.coverage.InitializeDeviceCoverage(self._dut) 169 170 # TODO: only set permissive mode for userdebug and eng build. 171 self.shell.Execute("setenforce 0") # SELinux permissive mode 172 173 if not precondition_utils.CanRunHidlHalTest(self, self._dut, 174 self._dut.shell.one): 175 self._skip_all_testcases = True 176 177 self.testcases = [] 178 self.tags = set() 179 self.CreateTestCases() 180 cmd = list( 181 set('chmod 755 %s' % test_case.path 182 for test_case in self.testcases)) 183 cmd_results = self.shell.Execute(cmd) 184 if any(cmd_results[const.EXIT_CODE]): 185 logging.error('Failed to set permission to some of the binaries:\n' 186 '%s\n%s', cmd, cmd_results) 187 188 self.include_filter = self.ExpandListItemTags(self.include_filter) 189 self.exclude_filter = self.ExpandListItemTags(self.exclude_filter) 190 191 stop_requested = False 192 193 if getattr(self, keys.ConfigKeys.IKEY_BINARY_TEST_DISABLE_FRAMEWORK, 194 False): 195 # Stop Android runtime to reduce interference. 196 self._dut.stop() 197 stop_requested = True 198 199 if getattr(self, keys.ConfigKeys.IKEY_BINARY_TEST_STOP_NATIVE_SERVERS, 200 False): 201 # Stops all (properly configured) native servers. 202 results = self._dut.setProp(self.SYSPROP_VTS_NATIVE_SERVER, "1") 203 stop_requested = True 204 205 if stop_requested: 206 native_server_process_names = getattr( 207 self, keys.ConfigKeys.IKEY_NATIVE_SERVER_PROCESS_NAME, []) 208 if native_server_process_names: 209 for native_server_process_name in native_server_process_names: 210 while True: 211 cmd_result = self.shell.Execute("ps -A") 212 if cmd_result[const.EXIT_CODE][0] != 0: 213 logging.error("ps command failed (exit code: %s", 214 cmd_result[const.EXIT_CODE][0]) 215 break 216 if (native_server_process_name not in 217 cmd_result[const.STDOUT][0]): 218 logging.info("Process %s not running", 219 native_server_process_name) 220 break 221 logging.info("Checking process %s", 222 native_server_process_name) 223 time.sleep(1) 224 225 def CreateTestCases(self): 226 '''Push files to device and create test case objects.''' 227 source_list = list(map(self.ParseTestSource, self.binary_test_source)) 228 source_list = filter(bool, source_list) 229 logging.info('Parsed test sources: %s', source_list) 230 231 # Push source files first 232 for src, dst, tag in source_list: 233 if src: 234 if os.path.isdir(src): 235 src = os.path.join(src, '.') 236 logging.info('Pushing from %s to %s.', src, dst) 237 self._dut.adb.push('{src} {dst}'.format(src=src, dst=dst)) 238 self.shell.Execute('ls %s' % dst) 239 240 # Then create test cases 241 for src, dst, tag in source_list: 242 if tag is not None: 243 # tag not being None means to create a test case 244 self.tags.add(tag) 245 logging.info('Creating test case from %s with tag %s', dst, 246 tag) 247 testcase = self.CreateTestCase(dst, tag) 248 if not testcase: 249 continue 250 251 if type(testcase) is list: 252 self.testcases.extend(testcase) 253 else: 254 self.testcases.append(testcase) 255 256 def PutTag(self, name, tag): 257 '''Put tag on name and return the resulting string. 258 259 Args: 260 name: string, a test name 261 tag: string 262 263 Returns: 264 String, the result string after putting tag on the name 265 ''' 266 return '{}{}'.format(name, tag) 267 268 def ExpandListItemTags(self, input_list): 269 '''Expand list items with tags. 270 271 Since binary test allows a tag to be added in front of the binary 272 path, test names are generated with tags attached. This function is 273 used to expand the filters correspondingly. If a filter contains 274 a tag, only test name with that tag will be included in output. 275 Otherwise, all known tags will be paired to the test name in output 276 list. 277 278 Args: 279 input_list: list of string, the list to expand 280 281 Returns: 282 A list of string 283 ''' 284 result = [] 285 for item in input_list: 286 if self.TAG_DELIMITER in item: 287 tag, name = item.split(self.TAG_DELIMITER) 288 result.append(self.PutTag(name, tag)) 289 for tag in self.tags: 290 result.append(self.PutTag(item, tag)) 291 return result 292 293 def tearDownClass(self): 294 '''Perform clean-up tasks''' 295 if getattr(self, keys.ConfigKeys.IKEY_BINARY_TEST_STOP_NATIVE_SERVERS, 296 False): 297 # Restarts all (properly configured) native servers. 298 results = self._dut.setProp(self.SYSPROP_VTS_NATIVE_SERVER, "0") 299 300 # Restart Android runtime. 301 if getattr(self, keys.ConfigKeys.IKEY_BINARY_TEST_DISABLE_FRAMEWORK, 302 False): 303 self._dut.start() 304 305 # Retrieve coverage if applicable 306 if self.coverage.enabled: 307 self.coverage.SetCoverageData(dut=self._dut, isGlobal=True) 308 309 # Clean up the pushed binaries 310 logging.info('Start class cleaning up jobs.') 311 # Delete pushed files 312 313 sources = [self.ParseTestSource(src) 314 for src in self.binary_test_source] 315 sources = set(filter(bool, sources)) 316 paths = [dst for src, dst, tag in sources if src and dst] 317 cmd = ['rm -rf %s' % dst for dst in paths] 318 cmd_results = self.shell.Execute(cmd) 319 if not cmd_results or any(cmd_results[const.EXIT_CODE]): 320 logging.warning('Failed to clean up test class: %s', cmd_results) 321 322 # Delete empty directories in working directories 323 dir_set = set(path_utils.TargetDirName(dst) for dst in paths) 324 dir_set.add(self.ParseTestSource('')[1]) 325 dirs = list(dir_set) 326 dirs.sort(lambda x, y: cmp(len(y), len(x))) 327 cmd = ['rmdir %s' % d for d in dirs] 328 cmd_results = self.shell.Execute(cmd) 329 if not cmd_results or any(cmd_results[const.EXIT_CODE]): 330 logging.warning('Failed to remove: %s', cmd_results) 331 332 if self.profiling.enabled: 333 self.profiling.ProcessAndUploadTraceData() 334 335 logging.info('Finished class cleaning up jobs.') 336 337 def ParseTestSource(self, source): 338 '''Convert host side binary path to device side path. 339 340 Args: 341 source: string, binary test source string 342 343 Returns: 344 A tuple of (string, string, string), representing (host side 345 absolute path, device side absolute path, tag). Returned tag 346 will be None if the test source is for pushing file to working 347 directory only. If source file is specified for adb push but does not 348 exist on host, None will be returned. 349 ''' 350 tag = '' 351 path = source 352 if self.TAG_DELIMITER in source: 353 tag, path = source.split(self.TAG_DELIMITER) 354 355 src = path 356 dst = None 357 if self.PUSH_DELIMITER in path: 358 src, dst = path.split(self.PUSH_DELIMITER) 359 360 if src: 361 src = os.path.join(self.data_file_path, src) 362 if not os.path.exists(src): 363 logging.warning('binary source file is specified ' 364 'but does not exist on host: %s', src) 365 return None 366 367 push_only = dst is not None and dst == '' 368 369 if not dst: 370 if tag in self.working_directory: 371 dst = path_utils.JoinTargetPath(self.working_directory[tag], 372 os.path.basename(src)) 373 else: 374 dst = path_utils.JoinTargetPath( 375 self.DEVICE_TMP_DIR, 'binary_test_temp_%s' % 376 self.__class__.__name__, tag, os.path.basename(src)) 377 378 if push_only: 379 tag = None 380 381 return str(src), str(dst), tag 382 383 def CreateTestCase(self, path, tag=''): 384 '''Create a list of TestCase objects from a binary path. 385 386 Args: 387 path: string, absolute path of a binary on device 388 tag: string, a tag that will be appended to the end of test name 389 390 Returns: 391 A list of BinaryTestCase objects 392 ''' 393 working_directory = self.working_directory[ 394 tag] if tag in self.working_directory else None 395 envp = self.envp[tag] if tag in self.envp else '' 396 args = self.args[tag] if tag in self.args else '' 397 ld_library_path = self.ld_library_path[ 398 tag] if tag in self.ld_library_path else None 399 profiling_library_path = self.profiling_library_path[ 400 tag] if tag in self.profiling_library_path else None 401 402 return binary_test_case.BinaryTestCase( 403 '', 404 path_utils.TargetBaseName(path), 405 path, 406 tag, 407 self.PutTag, 408 working_directory, 409 ld_library_path, 410 profiling_library_path, 411 envp=envp, 412 args=args) 413 414 def VerifyTestResult(self, test_case, command_results): 415 '''Parse command result. 416 417 Args: 418 command_results: dict of lists, shell command result 419 ''' 420 asserts.assertTrue(command_results, 'Empty command response.') 421 asserts.assertFalse( 422 any(command_results[const.EXIT_CODE]), 423 'Test {} failed with the following results: {}'.format( 424 test_case, command_results)) 425 426 def RunTestCase(self, test_case): 427 '''Runs a test_case. 428 429 Args: 430 test_case: BinaryTestCase object 431 ''' 432 if self.profiling.enabled: 433 self.profiling.EnableVTSProfiling(self.shell, 434 test_case.profiling_library_path) 435 436 cmd = test_case.GetRunCommand() 437 logging.info("Executing binary test command: %s", cmd) 438 command_results = self.shell.Execute(cmd) 439 440 self.VerifyTestResult(test_case, command_results) 441 442 if self.profiling.enabled: 443 self.profiling.ProcessTraceDataForTestCase(self._dut) 444 self.profiling.DisableVTSProfiling(self.shell) 445 446 def generateAllTests(self): 447 '''Runs all binary tests.''' 448 self.runGeneratedTests( 449 test_func=self.RunTestCase, settings=self.testcases, name_func=str) 450 451 452if __name__ == "__main__": 453 test_runner.main() 454