1#!/usr/bin/env python 2# 3# Copyright 2008, The Android Open Source Project 4# 5# Licensed under the Apache License, Version 2.0 (the "License"); 6# you may not use this file except in compliance with the License. 7# You may obtain a copy of the License at 8# 9# http://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an "AS IS" BASIS, 13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14# See the License for the specific language governing permissions and 15# limitations under the License. 16 17"""Command line utility for running Android tests 18 19runtest helps automate the instructions for building and running tests 20- It builds the corresponding test package for the code you want to test 21- It pushes the test package to your device or emulator 22- It launches InstrumentationTestRunner (or similar) to run the tests you 23specify. 24 25runtest supports running tests whose attributes have been pre-defined in 26_TEST_FILE_NAME files, (runtest <testname>), or by specifying the file 27system path to the test to run (runtest --path <path>). 28 29Do runtest --help to see full list of options. 30""" 31 32# Python imports 33import glob 34import optparse 35import os 36import re 37from sets import Set 38import sys 39import time 40 41# local imports 42import adb_interface 43import android_build 44from coverage import coverage 45import errors 46import logger 47import make_tree 48import run_command 49from test_defs import test_defs 50from test_defs import test_walker 51 52 53class TestRunner(object): 54 """Command line utility class for running pre-defined Android test(s).""" 55 56 _TEST_FILE_NAME = "test_defs.xml" 57 58 # file path to android core platform tests, relative to android build root 59 # TODO move these test data files to another directory 60 _CORE_TEST_PATH = os.path.join("development", "testrunner", 61 _TEST_FILE_NAME) 62 63 # vendor glob file path patterns to tests, relative to android 64 # build root 65 _VENDOR_TEST_PATH = os.path.join("vendor", "*", "tests", "testinfo", 66 _TEST_FILE_NAME) 67 68 _RUNTEST_USAGE = ( 69 "usage: runtest.py [options] short-test-name[s]\n\n" 70 "The runtest script works in two ways. You can query it " 71 "for a list of tests, or you can launch one or more tests.") 72 73 # default value for make -jX 74 _DEFAULT_JOBS = 16 75 76 _DALVIK_VERIFIER_PROP = "dalvik.vm.dexopt-flags" 77 _DALVIK_VERIFIER_OFF_VALUE = "v=n" 78 _DALVIK_VERIFIER_OFF_PROP = "%s = %s" %(_DALVIK_VERIFIER_PROP, _DALVIK_VERIFIER_OFF_VALUE) 79 80 # regular expression to match path to artifacts to install in make output 81 _RE_MAKE_INSTALL = re.compile(r'INSTALL-PATH:\s([^\s]+)\s(.*)$') 82 83 def __init__(self): 84 # disable logging of timestamp 85 self._root_path = android_build.GetTop() 86 out_base_name = os.path.basename(android_build.GetOutDir()) 87 # regular expression to find remote device path from a file path relative 88 # to build root 89 pattern = r'' + out_base_name + r'\/target\/product\/\w+\/(.+)$' 90 self._re_make_install_path = re.compile(pattern) 91 logger.SetTimestampLogging(False) 92 self._adb = None 93 self._known_tests = None 94 self._options = None 95 self._test_args = None 96 self._tests_to_run = None 97 98 def _ProcessOptions(self): 99 """Processes command-line options.""" 100 # TODO error messages on once-only or mutually-exclusive options. 101 user_test_default = os.path.join(os.environ.get("HOME"), ".android", 102 self._TEST_FILE_NAME) 103 104 parser = optparse.OptionParser(usage=self._RUNTEST_USAGE) 105 106 parser.add_option("-l", "--list-tests", dest="only_list_tests", 107 default=False, action="store_true", 108 help="To view the list of tests") 109 parser.add_option("-b", "--skip-build", dest="skip_build", default=False, 110 action="store_true", help="Skip build - just launch") 111 parser.add_option("-j", "--jobs", dest="make_jobs", 112 metavar="X", default=self._DEFAULT_JOBS, 113 help="Number of make jobs to use when building") 114 parser.add_option("-n", "--skip_execute", dest="preview", default=False, 115 action="store_true", 116 help="Do not execute, just preview commands") 117 parser.add_option("-i", "--build-install-only", dest="build_install_only", default=False, 118 action="store_true", 119 help="Do not execute, build tests and install to device only") 120 parser.add_option("-r", "--raw-mode", dest="raw_mode", default=False, 121 action="store_true", 122 help="Raw mode (for output to other tools)") 123 parser.add_option("-a", "--suite-assign", dest="suite_assign_mode", 124 default=False, action="store_true", 125 help="Suite assignment (for details & usage see " 126 "InstrumentationTestRunner)") 127 parser.add_option("-v", "--verbose", dest="verbose", default=False, 128 action="store_true", 129 help="Increase verbosity of %s" % sys.argv[0]) 130 parser.add_option("-w", "--wait-for-debugger", dest="wait_for_debugger", 131 default=False, action="store_true", 132 help="Wait for debugger before launching tests") 133 parser.add_option("-c", "--test-class", dest="test_class", 134 help="Restrict test to a specific class") 135 parser.add_option("-m", "--test-method", dest="test_method", 136 help="Restrict test to a specific method") 137 parser.add_option("-p", "--test-package", dest="test_package", 138 help="Restrict test to a specific java package") 139 parser.add_option("-z", "--size", dest="test_size", 140 help="Restrict test to a specific test size") 141 parser.add_option("--annotation", dest="test_annotation", 142 help="Include only those tests tagged with a specific" 143 " annotation") 144 parser.add_option("--not-annotation", dest="test_not_annotation", 145 help="Exclude any tests tagged with a specific" 146 " annotation") 147 parser.add_option("-u", "--user-tests-file", dest="user_tests_file", 148 metavar="FILE", default=user_test_default, 149 help="Alternate source of user test definitions") 150 parser.add_option("-o", "--coverage", dest="coverage", 151 default=False, action="store_true", 152 help="Generate code coverage metrics for test(s)") 153 parser.add_option("--coverage-target", dest="coverage_target_path", 154 default=None, 155 help="Path to app to collect code coverage target data for.") 156 parser.add_option("-k", "--skip-permissions", dest="skip_permissions", 157 default=False, action="store_true", 158 help="Do not grant runtime permissions during test package" 159 " installation.") 160 parser.add_option("-x", "--path", dest="test_path", 161 help="Run test(s) at given file system path") 162 parser.add_option("-t", "--all-tests", dest="all_tests", 163 default=False, action="store_true", 164 help="Run all defined tests") 165 parser.add_option("--continuous", dest="continuous_tests", 166 default=False, action="store_true", 167 help="Run all tests defined as part of the continuous " 168 "test set") 169 parser.add_option("--timeout", dest="timeout", 170 default=300, help="Set a timeout limit (in sec) for " 171 "running native tests on a device (default: 300 secs)") 172 parser.add_option("--suite", dest="suite", 173 help="Run all tests defined as part of the " 174 "the given test suite") 175 group = optparse.OptionGroup( 176 parser, "Targets", "Use these options to direct tests to a specific " 177 "Android target") 178 group.add_option("-e", "--emulator", dest="emulator", default=False, 179 action="store_true", help="use emulator") 180 group.add_option("-d", "--device", dest="device", default=False, 181 action="store_true", help="use device") 182 group.add_option("-s", "--serial", dest="serial", 183 help="use specific serial") 184 parser.add_option_group(group) 185 self._options, self._test_args = parser.parse_args() 186 187 if (not self._options.only_list_tests 188 and not self._options.all_tests 189 and not self._options.continuous_tests 190 and not self._options.suite 191 and not self._options.test_path 192 and len(self._test_args) < 1): 193 parser.print_help() 194 logger.SilentLog("at least one test name must be specified") 195 raise errors.AbortError 196 197 self._adb = adb_interface.AdbInterface() 198 if self._options.emulator: 199 self._adb.SetEmulatorTarget() 200 elif self._options.device: 201 self._adb.SetDeviceTarget() 202 elif self._options.serial is not None: 203 self._adb.SetTargetSerial(self._options.serial) 204 if self._options.verbose: 205 logger.SetVerbose(True) 206 207 if self._options.coverage_target_path: 208 self._options.coverage = True 209 210 self._known_tests = self._ReadTests() 211 212 self._options.host_lib_path = android_build.GetHostLibraryPath() 213 self._options.test_data_path = android_build.GetTestAppPath() 214 215 def _ReadTests(self): 216 """Parses the set of test definition data. 217 218 Returns: 219 A TestDefinitions object that contains the set of parsed tests. 220 Raises: 221 AbortError: If a fatal error occurred when parsing the tests. 222 """ 223 try: 224 known_tests = test_defs.TestDefinitions() 225 # only read tests when not in path mode 226 if not self._options.test_path: 227 core_test_path = os.path.join(self._root_path, self._CORE_TEST_PATH) 228 if os.path.isfile(core_test_path): 229 known_tests.Parse(core_test_path) 230 # read all <android root>/vendor/*/tests/testinfo/test_defs.xml paths 231 vendor_tests_pattern = os.path.join(self._root_path, 232 self._VENDOR_TEST_PATH) 233 test_file_paths = glob.glob(vendor_tests_pattern) 234 for test_file_path in test_file_paths: 235 known_tests.Parse(test_file_path) 236 if os.path.isfile(self._options.user_tests_file): 237 known_tests.Parse(self._options.user_tests_file) 238 return known_tests 239 except errors.ParseError: 240 raise errors.AbortError 241 242 def _DumpTests(self): 243 """Prints out set of defined tests.""" 244 print "The following tests are currently defined:\n" 245 print "%-25s %-40s %s" % ("name", "build path", "description") 246 print "-" * 80 247 for test in self._known_tests: 248 print "%-25s %-40s %s" % (test.GetName(), test.GetBuildPath(), 249 test.GetDescription()) 250 print "\nSee %s for more information" % self._TEST_FILE_NAME 251 252 def _DoBuild(self): 253 logger.SilentLog("Building tests...") 254 tests = self._GetTestsToRun() 255 256 # Build and install tests that do not get granted permissions 257 self._DoPermissionAwareBuild(tests, False) 258 259 # Build and install tests that require granted permissions 260 self._DoPermissionAwareBuild(tests, True) 261 262 def _DoPermissionAwareBuild(self, tests, test_requires_permissions): 263 # turn off dalvik verifier if necessary 264 # TODO: skip turning off verifier for now, since it puts device in bad 265 # state b/14088982 266 #self._TurnOffVerifier(tests) 267 self._DoFullBuild(tests, test_requires_permissions) 268 269 target_tree = make_tree.MakeTree() 270 271 extra_args_set = [] 272 for test_suite in tests: 273 if test_suite.IsGrantedPermissions() == test_requires_permissions: 274 self._AddBuildTarget(test_suite, target_tree, extra_args_set) 275 276 if not self._options.preview: 277 self._adb.EnableAdbRoot() 278 else: 279 logger.Log("adb root") 280 281 if not target_tree.IsEmpty(): 282 if self._options.coverage: 283 coverage.EnableCoverageBuild() 284 target_tree.AddPath("external/emma") 285 286 target_list = target_tree.GetPrunedMakeList() 287 target_dir_list = [re.sub(r'Android[.]mk$', r'', i) for i in target_list] 288 target_build_string = " ".join(target_list) 289 target_dir_build_string = " ".join(target_dir_list) 290 extra_args_string = " ".join(extra_args_set) 291 292 # mmm cannot be used from python, so perform a similar operation using 293 # ONE_SHOT_MAKEFILE 294 cmd = 'ONE_SHOT_MAKEFILE="%s" make -j%s -C "%s" GET-INSTALL-PATH all_modules %s' % ( 295 target_build_string, self._options.make_jobs, self._root_path, 296 extra_args_string) 297 # mmma equivalent, used when regular mmm fails 298 alt_cmd = 'make -j%s -C "%s" -f build/core/main.mk %s all_modules BUILD_MODULES_IN_PATHS="%s"' % ( 299 self._options.make_jobs, self._root_path, extra_args_string, target_dir_build_string) 300 301 logger.Log(cmd) 302 if not self._options.preview: 303 run_command.SetAbortOnError() 304 try: 305 output = run_command.RunCommand(cmd, return_output=True, timeout_time=600) 306 ## Chances are this failed because it didn't build the dependencies 307 except errors.AbortError: 308 logger.Log("make failed. Trying to rebuild all dependencies.") 309 logger.Log("mmma -j%s %s" %(self._options.make_jobs, target_dir_build_string)) 310 # Try again with mma equivalent, which will build the dependencies 311 run_command.RunCommand(alt_cmd, return_output=False, timeout_time=600) 312 # Run mmm again to get the install paths only 313 output = run_command.RunCommand(cmd, return_output=True, timeout_time=600) 314 run_command.SetAbortOnError(False) 315 logger.SilentLog(output) 316 self._DoInstall(output, test_requires_permissions) 317 318 def _DoInstall(self, make_output, test_requires_permissions): 319 """Install artifacts from build onto device. 320 321 Looks for 'install:' text from make output to find artifacts to install. 322 323 Files with the .apk extension get 'adb install'ed, all other files 324 get 'adb push'ed onto the device. 325 326 Args: 327 make_output: stdout from make command 328 """ 329 for line in make_output.split("\n"): 330 m = self._RE_MAKE_INSTALL.match(line) 331 if m: 332 # strip the 'INSTALL: <name>' from the left hand side 333 # the remaining string is a space-separated list of build-generated files 334 install_paths = m.group(2) 335 for install_path in re.split(r'\s+', install_paths): 336 if install_path.endswith(".apk"): 337 abs_install_path = os.path.join(self._root_path, install_path) 338 extra_flags = "" 339 if test_requires_permissions and not self._options.skip_permissions: 340 extra_flags = "-g" 341 logger.Log("adb install -r %s %s" % (extra_flags, abs_install_path)) 342 logger.Log(self._adb.Install(abs_install_path, extra_flags)) 343 else: 344 self._PushInstallFileToDevice(install_path) 345 346 def _PushInstallFileToDevice(self, install_path): 347 m = self._re_make_install_path.match(install_path) 348 if m: 349 remote_path = m.group(1) 350 remote_dir = os.path.dirname(remote_path) 351 logger.Log("adb shell mkdir -p %s" % remote_dir) 352 self._adb.SendShellCommand("mkdir -p %s" % remote_dir) 353 abs_install_path = os.path.join(self._root_path, install_path) 354 logger.Log("adb push %s %s" % (abs_install_path, remote_path)) 355 self._adb.Push(abs_install_path, remote_path) 356 else: 357 logger.Log("Error: Failed to recognize path of file to install %s" % install_path) 358 359 def _DoFullBuild(self, tests, test_requires_permissions): 360 """If necessary, run a full 'make' command for the tests that need it.""" 361 extra_args_set = Set() 362 363 for test in tests: 364 if test.IsFullMake() and test.IsGrantedPermissions() == test_requires_permissions: 365 if test.GetExtraBuildArgs(): 366 # extra args contains the args to pass to 'make' 367 extra_args_set.add(test.GetExtraBuildArgs()) 368 else: 369 logger.Log("Warning: test %s needs a full build but does not specify" 370 " extra_build_args" % test.GetName()) 371 372 # check if there is actually any tests that required a full build 373 if extra_args_set: 374 cmd = ('make -j%s %s' % (self._options.make_jobs, 375 ' '.join(list(extra_args_set)))) 376 logger.Log(cmd) 377 if not self._options.preview: 378 old_dir = os.getcwd() 379 os.chdir(self._root_path) 380 output = run_command.RunCommand(cmd, return_output=True) 381 logger.SilentLog(output) 382 os.chdir(old_dir) 383 self._DoInstall(output, test_requires_permissions) 384 385 def _AddBuildTarget(self, test_suite, target_tree, extra_args_set): 386 if not test_suite.IsFullMake(): 387 build_dir = test_suite.GetBuildPath() 388 if self._AddBuildTargetPath(build_dir, target_tree): 389 extra_args_set.append(test_suite.GetExtraBuildArgs()) 390 for path in test_suite.GetBuildDependencies(self._options): 391 self._AddBuildTargetPath(path, target_tree) 392 393 def _AddBuildTargetPath(self, build_dir, target_tree): 394 if build_dir is not None: 395 target_tree.AddPath(build_dir) 396 return True 397 return False 398 399 def _GetTestsToRun(self): 400 """Get a list of TestSuite objects to run, based on command line args.""" 401 if self._tests_to_run: 402 return self._tests_to_run 403 404 self._tests_to_run = [] 405 if self._options.all_tests: 406 self._tests_to_run = self._known_tests.GetTests() 407 elif self._options.continuous_tests: 408 self._tests_to_run = self._known_tests.GetContinuousTests() 409 elif self._options.suite: 410 self._tests_to_run = \ 411 self._known_tests.GetTestsInSuite(self._options.suite) 412 elif self._options.test_path: 413 walker = test_walker.TestWalker() 414 self._tests_to_run = walker.FindTests(self._options.test_path) 415 416 for name in self._test_args: 417 test = self._known_tests.GetTest(name) 418 if test is None: 419 logger.Log("Error: Could not find test %s" % name) 420 self._DumpTests() 421 raise errors.AbortError 422 self._tests_to_run.append(test) 423 return self._tests_to_run 424 425 def _IsCtsTests(self, test_list): 426 """Check if any cts tests are included in given list of tests to run.""" 427 for test in test_list: 428 if test.GetSuite() == 'cts': 429 return True 430 return False 431 432 def _TurnOffVerifier(self, test_list): 433 """Turn off the dalvik verifier if needed by given tests. 434 435 If one or more tests needs dalvik verifier off, and it is not already off, 436 turns off verifier and reboots device to allow change to take effect. 437 """ 438 # hack to check if these are frameworks/base tests. If so, turn off verifier 439 # to allow framework tests to access private/protected/package-private framework api 440 framework_test = False 441 for test in test_list: 442 if os.path.commonprefix([test.GetBuildPath(), "frameworks/base"]): 443 framework_test = True 444 if framework_test: 445 # check if verifier is off already - to avoid the reboot if not 446 # necessary 447 output = self._adb.SendShellCommand("cat /data/local.prop") 448 if not self._DALVIK_VERIFIER_OFF_PROP in output: 449 450 # Read the existing dalvik verifier flags. 451 old_prop_value = self._adb.SendShellCommand("getprop %s" \ 452 %(self._DALVIK_VERIFIER_PROP)) 453 old_prop_value = old_prop_value.strip() if old_prop_value else "" 454 455 # Append our verifier flags to existing flags 456 new_prop_value = "%s %s" %(self._DALVIK_VERIFIER_OFF_VALUE, old_prop_value) 457 458 # Update property now, as /data/local.prop is not read until reboot 459 logger.Log("adb shell setprop %s '%s'" \ 460 %(self._DALVIK_VERIFIER_PROP, new_prop_value)) 461 if not self._options.preview: 462 self._adb.SendShellCommand("setprop %s '%s'" \ 463 %(self._DALVIK_VERIFIER_PROP, new_prop_value)) 464 465 # Write prop to /data/local.prop 466 # Every time device is booted, it will pick up this value 467 new_prop_assignment = "%s = %s" %(self._DALVIK_VERIFIER_PROP, new_prop_value) 468 if self._options.preview: 469 logger.Log("adb shell \"echo %s >> /data/local.prop\"" 470 % new_prop_assignment) 471 logger.Log("adb shell chmod 644 /data/local.prop") 472 else: 473 logger.Log("Turning off dalvik verifier and rebooting") 474 self._adb.SendShellCommand("\"echo %s >> /data/local.prop\"" 475 % new_prop_assignment) 476 477 # Reset runtime so that dalvik picks up new verifier flags from prop 478 self._ChmodRuntimeReset() 479 elif not self._options.preview: 480 # check the permissions on the file 481 permout = self._adb.SendShellCommand("ls -l /data/local.prop") 482 if not "-rw-r--r--" in permout: 483 logger.Log("Fixing permissions on /data/local.prop and rebooting") 484 self._ChmodRuntimeReset() 485 486 def _ChmodRuntimeReset(self): 487 """Perform a chmod of /data/local.prop and reset the runtime. 488 """ 489 logger.Log("adb shell chmod 644 /data/local.prop ## u+w,a+r") 490 if not self._options.preview: 491 self._adb.SendShellCommand("chmod 644 /data/local.prop") 492 493 self._adb.RuntimeReset(preview_only=self._options.preview) 494 495 if not self._options.preview: 496 self._adb.EnableAdbRoot() 497 498 499 def RunTests(self): 500 """Main entry method - executes the tests according to command line args.""" 501 try: 502 run_command.SetAbortOnError() 503 self._ProcessOptions() 504 if self._options.only_list_tests: 505 self._DumpTests() 506 return 507 508 if not self._options.skip_build: 509 self._DoBuild() 510 511 if self._options.build_install_only: 512 logger.Log("Skipping test execution (due to --build-install-only flag)") 513 return 514 515 for test_suite in self._GetTestsToRun(): 516 try: 517 test_suite.Run(self._options, self._adb) 518 except errors.WaitForResponseTimedOutError: 519 logger.Log("Timed out waiting for response") 520 521 except KeyboardInterrupt: 522 logger.Log("Exiting...") 523 except errors.AbortError, error: 524 logger.Log(error.msg) 525 logger.SilentLog("Exiting due to AbortError...") 526 except errors.WaitForResponseTimedOutError: 527 logger.Log("Timed out waiting for response") 528 529 530def RunTests(): 531 runner = TestRunner() 532 runner.RunTests() 533 534if __name__ == "__main__": 535 RunTests() 536