1#!/usr/bin/python2.4 2# 3# Copyright 2008, The Android Open Source Project 4# 5# Licensed under the Apache License, Version 2.0 (the "License"); 6# you may not use this file except in compliance with the License. 7# You may obtain a copy of the License at 8# 9# http://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an "AS IS" BASIS, 13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14# See the License for the specific language governing permissions and 15# limitations under the License. 16 17"""Command line utility for running Android tests 18 19runtest helps automate the instructions for building and running tests 20- It builds the corresponding test package for the code you want to test 21- It pushes the test package to your device or emulator 22- It launches InstrumentationTestRunner (or similar) to run the tests you 23specify. 24 25runtest supports running tests whose attributes have been pre-defined in 26_TEST_FILE_NAME files, (runtest <testname>), or by specifying the file 27system path to the test to run (runtest --path <path>). 28 29Do runtest --help to see full list of options. 30""" 31 32# Python imports 33import glob 34import optparse 35import os 36from sets import Set 37import sys 38import time 39 40# local imports 41import adb_interface 42import android_build 43import coverage 44import errors 45import logger 46import run_command 47from test_defs import test_defs 48from test_defs import test_walker 49 50 51class TestRunner(object): 52 """Command line utility class for running pre-defined Android test(s).""" 53 54 _TEST_FILE_NAME = "test_defs.xml" 55 56 # file path to android core platform tests, relative to android build root 57 # TODO move these test data files to another directory 58 _CORE_TEST_PATH = os.path.join("development", "testrunner", 59 _TEST_FILE_NAME) 60 61 # vendor glob file path patterns to tests, relative to android 62 # build root 63 _VENDOR_TEST_PATH = os.path.join("vendor", "*", "tests", "testinfo", 64 _TEST_FILE_NAME) 65 66 _RUNTEST_USAGE = ( 67 "usage: runtest.py [options] short-test-name[s]\n\n" 68 "The runtest script works in two ways. You can query it " 69 "for a list of tests, or you can launch one or more tests.") 70 71 # default value for make -jX 72 _DEFAULT_JOBS = 16 73 74 _DALVIK_VERIFIER_OFF_PROP = "dalvik.vm.dexopt-flags = v=n" 75 76 def __init__(self): 77 # disable logging of timestamp 78 self._root_path = android_build.GetTop() 79 logger.SetTimestampLogging(False) 80 self._adb = None 81 self._known_tests = None 82 self._options = None 83 self._test_args = None 84 self._tests_to_run = None 85 86 def _ProcessOptions(self): 87 """Processes command-line options.""" 88 # TODO error messages on once-only or mutually-exclusive options. 89 user_test_default = os.path.join(os.environ.get("HOME"), ".android", 90 self._TEST_FILE_NAME) 91 92 parser = optparse.OptionParser(usage=self._RUNTEST_USAGE) 93 94 parser.add_option("-l", "--list-tests", dest="only_list_tests", 95 default=False, action="store_true", 96 help="To view the list of tests") 97 parser.add_option("-b", "--skip-build", dest="skip_build", default=False, 98 action="store_true", help="Skip build - just launch") 99 parser.add_option("-j", "--jobs", dest="make_jobs", 100 metavar="X", default=self._DEFAULT_JOBS, 101 help="Number of make jobs to use when building") 102 parser.add_option("-n", "--skip_execute", dest="preview", default=False, 103 action="store_true", 104 help="Do not execute, just preview commands") 105 parser.add_option("-r", "--raw-mode", dest="raw_mode", default=False, 106 action="store_true", 107 help="Raw mode (for output to other tools)") 108 parser.add_option("-a", "--suite-assign", dest="suite_assign_mode", 109 default=False, action="store_true", 110 help="Suite assignment (for details & usage see " 111 "InstrumentationTestRunner)") 112 parser.add_option("-v", "--verbose", dest="verbose", default=False, 113 action="store_true", 114 help="Increase verbosity of %s" % sys.argv[0]) 115 parser.add_option("-w", "--wait-for-debugger", dest="wait_for_debugger", 116 default=False, action="store_true", 117 help="Wait for debugger before launching tests") 118 parser.add_option("-c", "--test-class", dest="test_class", 119 help="Restrict test to a specific class") 120 parser.add_option("-m", "--test-method", dest="test_method", 121 help="Restrict test to a specific method") 122 parser.add_option("-p", "--test-package", dest="test_package", 123 help="Restrict test to a specific java package") 124 parser.add_option("-z", "--size", dest="test_size", 125 help="Restrict test to a specific test size") 126 parser.add_option("--annotation", dest="test_annotation", 127 help="Include only those tests tagged with a specific" 128 " annotation") 129 parser.add_option("--not-annotation", dest="test_not_annotation", 130 help="Exclude any tests tagged with a specific" 131 " annotation") 132 parser.add_option("-u", "--user-tests-file", dest="user_tests_file", 133 metavar="FILE", default=user_test_default, 134 help="Alternate source of user test definitions") 135 parser.add_option("-o", "--coverage", dest="coverage", 136 default=False, action="store_true", 137 help="Generate code coverage metrics for test(s)") 138 parser.add_option("-x", "--path", dest="test_path", 139 help="Run test(s) at given file system path") 140 parser.add_option("-t", "--all-tests", dest="all_tests", 141 default=False, action="store_true", 142 help="Run all defined tests") 143 parser.add_option("--continuous", dest="continuous_tests", 144 default=False, action="store_true", 145 help="Run all tests defined as part of the continuous " 146 "test set") 147 parser.add_option("--timeout", dest="timeout", 148 default=300, help="Set a timeout limit (in sec) for " 149 "running native tests on a device (default: 300 secs)") 150 parser.add_option("--suite", dest="suite", 151 help="Run all tests defined as part of the " 152 "the given test suite") 153 group = optparse.OptionGroup( 154 parser, "Targets", "Use these options to direct tests to a specific " 155 "Android target") 156 group.add_option("-e", "--emulator", dest="emulator", default=False, 157 action="store_true", help="use emulator") 158 group.add_option("-d", "--device", dest="device", default=False, 159 action="store_true", help="use device") 160 group.add_option("-s", "--serial", dest="serial", 161 help="use specific serial") 162 parser.add_option_group(group) 163 self._options, self._test_args = parser.parse_args() 164 165 if (not self._options.only_list_tests 166 and not self._options.all_tests 167 and not self._options.continuous_tests 168 and not self._options.suite 169 and not self._options.test_path 170 and len(self._test_args) < 1): 171 parser.print_help() 172 logger.SilentLog("at least one test name must be specified") 173 raise errors.AbortError 174 175 self._adb = adb_interface.AdbInterface() 176 if self._options.emulator: 177 self._adb.SetEmulatorTarget() 178 elif self._options.device: 179 self._adb.SetDeviceTarget() 180 elif self._options.serial is not None: 181 self._adb.SetTargetSerial(self._options.serial) 182 183 if self._options.verbose: 184 logger.SetVerbose(True) 185 186 self._known_tests = self._ReadTests() 187 188 self._options.host_lib_path = android_build.GetHostLibraryPath() 189 self._options.test_data_path = android_build.GetTestAppPath() 190 191 def _ReadTests(self): 192 """Parses the set of test definition data. 193 194 Returns: 195 A TestDefinitions object that contains the set of parsed tests. 196 Raises: 197 AbortError: If a fatal error occurred when parsing the tests. 198 """ 199 try: 200 known_tests = test_defs.TestDefinitions() 201 # only read tests when not in path mode 202 if not self._options.test_path: 203 core_test_path = os.path.join(self._root_path, self._CORE_TEST_PATH) 204 if os.path.isfile(core_test_path): 205 known_tests.Parse(core_test_path) 206 # read all <android root>/vendor/*/tests/testinfo/test_defs.xml paths 207 vendor_tests_pattern = os.path.join(self._root_path, 208 self._VENDOR_TEST_PATH) 209 test_file_paths = glob.glob(vendor_tests_pattern) 210 for test_file_path in test_file_paths: 211 known_tests.Parse(test_file_path) 212 if os.path.isfile(self._options.user_tests_file): 213 known_tests.Parse(self._options.user_tests_file) 214 return known_tests 215 except errors.ParseError: 216 raise errors.AbortError 217 218 def _DumpTests(self): 219 """Prints out set of defined tests.""" 220 print "The following tests are currently defined:\n" 221 print "%-25s %-40s %s" % ("name", "build path", "description") 222 print "-" * 80 223 for test in self._known_tests: 224 print "%-25s %-40s %s" % (test.GetName(), test.GetBuildPath(), 225 test.GetDescription()) 226 print "\nSee %s for more information" % self._TEST_FILE_NAME 227 228 def _DoBuild(self): 229 logger.SilentLog("Building tests...") 230 231 tests = self._GetTestsToRun() 232 # turn off dalvik verifier if necessary 233 self._TurnOffVerifier(tests) 234 self._DoFullBuild(tests) 235 236 target_set = [] 237 238 extra_args_set = [] 239 for test_suite in tests: 240 self._AddBuildTarget(test_suite, target_set, extra_args_set) 241 242 if not self._options.preview: 243 self._adb.EnableAdbRoot() 244 else: 245 logger.Log("adb root") 246 rebuild_libcore = False 247 if target_set: 248 if self._options.coverage: 249 coverage.EnableCoverageBuild() 250 # hack to remove core library intermediates 251 # hack is needed because: 252 # 1. EMMA_INSTRUMENT changes what source files to include in libcore 253 # but it does not trigger a rebuild 254 # 2. there's no target (like "clear-intermediates") to remove the files 255 # decently 256 rebuild_libcore = not coverage.TestDeviceCoverageSupport(self._adb) 257 if rebuild_libcore: 258 cmd = "rm -rf %s" % os.path.join( 259 self._root_path, 260 "out/target/common/obj/JAVA_LIBRARIES/core_intermediates/") 261 logger.Log(cmd) 262 run_command.RunCommand(cmd, return_output=False) 263 264 target_build_string = " ".join(target_set) 265 extra_args_string = " ".join(extra_args_set) 266 267 # mmm cannot be used from python, so perform a similar operation using 268 # ONE_SHOT_MAKEFILE 269 cmd = 'ONE_SHOT_MAKEFILE="%s" make -j%s -C "%s" all_modules %s' % ( 270 target_build_string, self._options.make_jobs, self._root_path, 271 extra_args_string) 272 logger.Log(cmd) 273 274 if self._options.preview: 275 # in preview mode, just display to the user what command would have been 276 # run 277 logger.Log("adb sync") 278 else: 279 # set timeout for build to 10 minutes, since libcore may need to 280 # be rebuilt 281 run_command.RunCommand(cmd, return_output=False, timeout_time=600) 282 logger.Log("Syncing to device...") 283 self._adb.Sync(runtime_restart=rebuild_libcore) 284 285 def _DoFullBuild(self, tests): 286 """If necessary, run a full 'make' command for the tests that need it.""" 287 extra_args_set = Set() 288 289 # hack to build cts dependencies 290 # TODO: remove this when cts dependencies are removed 291 if self._IsCtsTests(tests): 292 # need to use make since these fail building with ONE_SHOT_MAKEFILE 293 extra_args_set.add('CtsTestStubs') 294 extra_args_set.add('android.core.tests.runner') 295 for test in tests: 296 if test.IsFullMake(): 297 if test.GetExtraBuildArgs(): 298 # extra args contains the args to pass to 'make' 299 extra_args_set.add(test.GetExtraBuildArgs()) 300 else: 301 logger.Log("Warning: test %s needs a full build but does not specify" 302 " extra_build_args" % test.GetName()) 303 304 # check if there is actually any tests that required a full build 305 if extra_args_set: 306 cmd = ('make -j%s %s' % (self._options.make_jobs, 307 ' '.join(list(extra_args_set)))) 308 logger.Log(cmd) 309 if not self._options.preview: 310 old_dir = os.getcwd() 311 os.chdir(self._root_path) 312 run_command.RunCommand(cmd, return_output=False) 313 os.chdir(old_dir) 314 315 def _AddBuildTarget(self, test_suite, target_set, extra_args_set): 316 if not test_suite.IsFullMake(): 317 build_dir = test_suite.GetBuildPath() 318 if self._AddBuildTargetPath(build_dir, target_set): 319 extra_args_set.append(test_suite.GetExtraBuildArgs()) 320 for path in test_suite.GetBuildDependencies(self._options): 321 self._AddBuildTargetPath(path, target_set) 322 323 def _AddBuildTargetPath(self, build_dir, target_set): 324 if build_dir is not None: 325 build_file_path = os.path.join(build_dir, "Android.mk") 326 if os.path.isfile(os.path.join(self._root_path, build_file_path)): 327 target_set.append(build_file_path) 328 return True 329 else: 330 logger.Log("%s has no Android.mk, skipping" % build_dir) 331 return False 332 333 def _GetTestsToRun(self): 334 """Get a list of TestSuite objects to run, based on command line args.""" 335 if self._tests_to_run: 336 return self._tests_to_run 337 338 self._tests_to_run = [] 339 if self._options.all_tests: 340 self._tests_to_run = self._known_tests.GetTests() 341 elif self._options.continuous_tests: 342 self._tests_to_run = self._known_tests.GetContinuousTests() 343 elif self._options.suite: 344 self._tests_to_run = \ 345 self._known_tests.GetTestsInSuite(self._options.suite) 346 elif self._options.test_path: 347 walker = test_walker.TestWalker() 348 self._tests_to_run = walker.FindTests(self._options.test_path) 349 350 for name in self._test_args: 351 test = self._known_tests.GetTest(name) 352 if test is None: 353 logger.Log("Error: Could not find test %s" % name) 354 self._DumpTests() 355 raise errors.AbortError 356 self._tests_to_run.append(test) 357 return self._tests_to_run 358 359 def _IsCtsTests(self, test_list): 360 """Check if any cts tests are included in given list of tests to run.""" 361 for test in test_list: 362 if test.GetSuite() == 'cts': 363 return True 364 return False 365 366 def _TurnOffVerifier(self, test_list): 367 """Turn off the dalvik verifier if needed by given tests. 368 369 If one or more tests needs dalvik verifier off, and it is not already off, 370 turns off verifier and reboots device to allow change to take effect. 371 """ 372 # hack to check if these are framework/base tests. If so, turn off verifier 373 # to allow framework tests to access package-private framework api 374 framework_test = False 375 for test in test_list: 376 if os.path.commonprefix([test.GetBuildPath(), "frameworks/base"]): 377 framework_test = True 378 if framework_test: 379 # check if verifier is off already - to avoid the reboot if not 380 # necessary 381 output = self._adb.SendShellCommand("cat /data/local.prop") 382 if not self._DALVIK_VERIFIER_OFF_PROP in output: 383 if self._options.preview: 384 logger.Log("adb shell \"echo %s >> /data/local.prop\"" 385 % self._DALVIK_VERIFIER_OFF_PROP) 386 logger.Log("adb shell chmod 644 /data/local.prop") 387 logger.Log("adb reboot") 388 logger.Log("adb wait-for-device") 389 else: 390 logger.Log("Turning off dalvik verifier and rebooting") 391 self._adb.SendShellCommand("\"echo %s >> /data/local.prop\"" 392 % self._DALVIK_VERIFIER_OFF_PROP) 393 394 self._ChmodReboot() 395 elif not self._options.preview: 396 # check the permissions on the file 397 permout = self._adb.SendShellCommand("ls -l /data/local.prop") 398 if not "-rw-r--r--" in permout: 399 logger.Log("Fixing permissions on /data/local.prop and rebooting") 400 self._ChmodReboot() 401 402 def _ChmodReboot(self): 403 """Perform a chmod of /data/local.prop and reboot. 404 """ 405 self._adb.SendShellCommand("chmod 644 /data/local.prop") 406 self._adb.SendCommand("reboot") 407 # wait for device to go offline 408 time.sleep(10) 409 self._adb.SendCommand("wait-for-device", timeout_time=60, 410 retry_count=3) 411 self._adb.EnableAdbRoot() 412 413 414 def RunTests(self): 415 """Main entry method - executes the tests according to command line args.""" 416 try: 417 run_command.SetAbortOnError() 418 self._ProcessOptions() 419 if self._options.only_list_tests: 420 self._DumpTests() 421 return 422 423 if not self._options.skip_build: 424 self._DoBuild() 425 426 for test_suite in self._GetTestsToRun(): 427 try: 428 test_suite.Run(self._options, self._adb) 429 except errors.WaitForResponseTimedOutError: 430 logger.Log("Timed out waiting for response") 431 432 except KeyboardInterrupt: 433 logger.Log("Exiting...") 434 except errors.AbortError, error: 435 logger.Log(error.msg) 436 logger.SilentLog("Exiting due to AbortError...") 437 except errors.WaitForResponseTimedOutError: 438 logger.Log("Timed out waiting for response") 439 440 441def RunTests(): 442 runner = TestRunner() 443 runner.RunTests() 444 445if __name__ == "__main__": 446 RunTests() 447