1#!/usr/bin/env python 2# 3# Copyright 2008, The Android Open Source Project 4# 5# Licensed under the Apache License, Version 2.0 (the "License"); 6# you may not use this file except in compliance with the License. 7# You may obtain a copy of the License at 8# 9# http://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an "AS IS" BASIS, 13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14# See the License for the specific language governing permissions and 15# limitations under the License. 16 17"""Command line utility for running Android tests 18 19runtest helps automate the instructions for building and running tests 20- It builds the corresponding test package for the code you want to test 21- It pushes the test package to your device or emulator 22- It launches InstrumentationTestRunner (or similar) to run the tests you 23specify. 24 25runtest supports running tests whose attributes have been pre-defined in 26_TEST_FILE_NAME files, (runtest <testname>), or by specifying the file 27system path to the test to run (runtest --path <path>). 28 29Do runtest --help to see full list of options. 30""" 31 32# Python imports 33import glob 34import optparse 35import os 36import re 37from sets import Set 38import sys 39import time 40 41# local imports 42import adb_interface 43import android_build 44from coverage import coverage 45import errors 46import logger 47import make_tree 48import run_command 49from test_defs import test_defs 50from test_defs import test_walker 51 52 53class TestRunner(object): 54 """Command line utility class for running pre-defined Android test(s).""" 55 56 _TEST_FILE_NAME = "test_defs.xml" 57 58 # file path to android core platform tests, relative to android build root 59 # TODO move these test data files to another directory 60 _CORE_TEST_PATH = os.path.join("development", "testrunner", 61 _TEST_FILE_NAME) 62 63 # vendor glob file path patterns to tests, relative to android 64 # build root 65 _VENDOR_TEST_PATH = os.path.join("vendor", "*", "tests", "testinfo", 66 _TEST_FILE_NAME) 67 68 _RUNTEST_USAGE = ( 69 "usage: runtest.py [options] short-test-name[s]\n\n" 70 "The runtest script works in two ways. You can query it " 71 "for a list of tests, or you can launch one or more tests.") 72 73 # default value for make -jX 74 _DEFAULT_JOBS = 16 75 76 _DALVIK_VERIFIER_PROP = "dalvik.vm.dexopt-flags" 77 _DALVIK_VERIFIER_OFF_VALUE = "v=n" 78 _DALVIK_VERIFIER_OFF_PROP = "%s = %s" %(_DALVIK_VERIFIER_PROP, _DALVIK_VERIFIER_OFF_VALUE) 79 80 # regular expression to match path to artifacts to install in make output 81 _RE_MAKE_INSTALL = re.compile(r'INSTALL-PATH:\s([^\s]+)\s(.*)$') 82 83 def __init__(self): 84 # disable logging of timestamp 85 self._root_path = android_build.GetTop() 86 out_base_name = os.path.basename(android_build.GetOutDir()) 87 # regular expression to find remote device path from a file path relative 88 # to build root 89 pattern = r'' + out_base_name + r'\/target\/product\/\w+\/(.+)$' 90 self._re_make_install_path = re.compile(pattern) 91 logger.SetTimestampLogging(False) 92 self._adb = None 93 self._known_tests = None 94 self._options = None 95 self._test_args = None 96 self._tests_to_run = None 97 98 def _ProcessOptions(self): 99 """Processes command-line options.""" 100 # TODO error messages on once-only or mutually-exclusive options. 101 user_test_default = os.path.join(os.environ.get("HOME"), ".android", 102 self._TEST_FILE_NAME) 103 104 parser = optparse.OptionParser(usage=self._RUNTEST_USAGE) 105 106 parser.add_option("-l", "--list-tests", dest="only_list_tests", 107 default=False, action="store_true", 108 help="To view the list of tests") 109 parser.add_option("-b", "--skip-build", dest="skip_build", default=False, 110 action="store_true", help="Skip build - just launch") 111 parser.add_option("-j", "--jobs", dest="make_jobs", 112 metavar="X", default=self._DEFAULT_JOBS, 113 help="Number of make jobs to use when building") 114 parser.add_option("-n", "--skip_execute", dest="preview", default=False, 115 action="store_true", 116 help="Do not execute, just preview commands") 117 parser.add_option("-i", "--build-install-only", dest="build_install_only", default=False, 118 action="store_true", 119 help="Do not execute, build tests and install to device only") 120 parser.add_option("-r", "--raw-mode", dest="raw_mode", default=False, 121 action="store_true", 122 help="Raw mode (for output to other tools)") 123 parser.add_option("-a", "--suite-assign", dest="suite_assign_mode", 124 default=False, action="store_true", 125 help="Suite assignment (for details & usage see " 126 "InstrumentationTestRunner)") 127 parser.add_option("-v", "--verbose", dest="verbose", default=False, 128 action="store_true", 129 help="Increase verbosity of %s" % sys.argv[0]) 130 parser.add_option("-w", "--wait-for-debugger", dest="wait_for_debugger", 131 default=False, action="store_true", 132 help="Wait for debugger before launching tests") 133 parser.add_option("-c", "--test-class", dest="test_class", 134 help="Restrict test to a specific class") 135 parser.add_option("-m", "--test-method", dest="test_method", 136 help="Restrict test to a specific method") 137 parser.add_option("-p", "--test-package", dest="test_package", 138 help="Restrict test to a specific java package") 139 parser.add_option("-z", "--size", dest="test_size", 140 help="Restrict test to a specific test size") 141 parser.add_option("--annotation", dest="test_annotation", 142 help="Include only those tests tagged with a specific" 143 " annotation") 144 parser.add_option("--not-annotation", dest="test_not_annotation", 145 help="Exclude any tests tagged with a specific" 146 " annotation") 147 parser.add_option("-u", "--user-tests-file", dest="user_tests_file", 148 metavar="FILE", default=user_test_default, 149 help="Alternate source of user test definitions") 150 parser.add_option("-o", "--coverage", dest="coverage", 151 default=False, action="store_true", 152 help="Generate code coverage metrics for test(s)") 153 parser.add_option("--coverage-target", dest="coverage_target_path", 154 default=None, 155 help="Path to app to collect code coverage target data for.") 156 parser.add_option("-k", "--skip-permissions", dest="skip_permissions", 157 default=False, action="store_true", 158 help="Do not grant runtime permissions during test package" 159 " installation.") 160 parser.add_option("-x", "--path", dest="test_path", 161 help="Run test(s) at given file system path") 162 parser.add_option("-t", "--all-tests", dest="all_tests", 163 default=False, action="store_true", 164 help="Run all defined tests") 165 parser.add_option("--continuous", dest="continuous_tests", 166 default=False, action="store_true", 167 help="Run all tests defined as part of the continuous " 168 "test set") 169 parser.add_option("--timeout", dest="timeout", 170 default=300, help="Set a timeout limit (in sec) for " 171 "running native tests on a device (default: 300 secs)") 172 parser.add_option("--suite", dest="suite", 173 help="Run all tests defined as part of the " 174 "the given test suite") 175 parser.add_option("--user", dest="user", 176 help="The user that test apks are installing to." 177 " This is the integer user id, e.g. 0 or 10." 178 " If no user is specified, apk will be installed with" 179 " adb's default behavior, which is currently all users.") 180 parser.add_option("--install-filter", dest="filter_re", 181 help="Regular expression which generated apks have to" 182 " match to be installed to target device. Default is None" 183 " and will install all packages built. This is" 184 " useful when the test path has a lot of apks but you" 185 " only care about one.") 186 parser.add_option("--no-hidden-api-checks", dest="no_hidden_api_checks", 187 default=False, action="store_true", 188 help="Disable hidden API checks in instrumentation" 189 " tests.") 190 group = optparse.OptionGroup( 191 parser, "Targets", "Use these options to direct tests to a specific " 192 "Android target") 193 group.add_option("-e", "--emulator", dest="emulator", default=False, 194 action="store_true", help="use emulator") 195 group.add_option("-d", "--device", dest="device", default=False, 196 action="store_true", help="use device") 197 group.add_option("-s", "--serial", dest="serial", 198 help="use specific serial") 199 parser.add_option_group(group) 200 self._options, self._test_args = parser.parse_args() 201 202 if (not self._options.only_list_tests 203 and not self._options.all_tests 204 and not self._options.continuous_tests 205 and not self._options.suite 206 and not self._options.test_path 207 and len(self._test_args) < 1): 208 parser.print_help() 209 logger.SilentLog("at least one test name must be specified") 210 raise errors.AbortError 211 212 self._adb = adb_interface.AdbInterface() 213 if self._options.emulator: 214 self._adb.SetEmulatorTarget() 215 elif self._options.device: 216 self._adb.SetDeviceTarget() 217 elif self._options.serial is not None: 218 self._adb.SetTargetSerial(self._options.serial) 219 if self._options.verbose: 220 logger.SetVerbose(True) 221 222 if self._options.coverage_target_path: 223 self._options.coverage = True 224 225 self._known_tests = self._ReadTests() 226 227 self._options.host_lib_path = android_build.GetHostLibraryPath() 228 self._options.test_data_path = android_build.GetTestAppPath() 229 230 def _ReadTests(self): 231 """Parses the set of test definition data. 232 233 Returns: 234 A TestDefinitions object that contains the set of parsed tests. 235 Raises: 236 AbortError: If a fatal error occurred when parsing the tests. 237 """ 238 try: 239 known_tests = test_defs.TestDefinitions() 240 # only read tests when not in path mode 241 if not self._options.test_path: 242 core_test_path = os.path.join(self._root_path, self._CORE_TEST_PATH) 243 if os.path.isfile(core_test_path): 244 known_tests.Parse(core_test_path) 245 # read all <android root>/vendor/*/tests/testinfo/test_defs.xml paths 246 vendor_tests_pattern = os.path.join(self._root_path, 247 self._VENDOR_TEST_PATH) 248 test_file_paths = glob.glob(vendor_tests_pattern) 249 for test_file_path in test_file_paths: 250 known_tests.Parse(test_file_path) 251 if os.path.isfile(self._options.user_tests_file): 252 known_tests.Parse(self._options.user_tests_file) 253 return known_tests 254 except errors.ParseError: 255 raise errors.AbortError 256 257 def _DumpTests(self): 258 """Prints out set of defined tests.""" 259 print "The following tests are currently defined:\n" 260 print "%-25s %-40s %s" % ("name", "build path", "description") 261 print "-" * 80 262 for test in self._known_tests: 263 print "%-25s %-40s %s" % (test.GetName(), test.GetBuildPath(), 264 test.GetDescription()) 265 print "\nSee %s for more information" % self._TEST_FILE_NAME 266 267 def _DoBuild(self): 268 logger.SilentLog("Building tests...") 269 tests = self._GetTestsToRun() 270 271 # Build and install tests that do not get granted permissions 272 self._DoPermissionAwareBuild(tests, False) 273 274 # Build and install tests that require granted permissions 275 self._DoPermissionAwareBuild(tests, True) 276 277 def _DoPermissionAwareBuild(self, tests, test_requires_permissions): 278 # turn off dalvik verifier if necessary 279 # TODO: skip turning off verifier for now, since it puts device in bad 280 # state b/14088982 281 #self._TurnOffVerifier(tests) 282 self._DoFullBuild(tests, test_requires_permissions) 283 284 target_tree = make_tree.MakeTree() 285 286 extra_args_set = [] 287 for test_suite in tests: 288 if test_suite.IsGrantedPermissions() == test_requires_permissions: 289 self._AddBuildTarget(test_suite, target_tree, extra_args_set) 290 291 if not self._options.preview: 292 self._adb.EnableAdbRoot() 293 else: 294 logger.Log("adb root") 295 296 if not target_tree.IsEmpty(): 297 if self._options.coverage: 298 coverage.EnableCoverageBuild() 299 target_tree.AddPath("external/emma") 300 301 target_list = target_tree.GetPrunedMakeList() 302 target_dir_list = [re.sub(r'Android[.]mk$', r'', i) for i in target_list] 303 target_build_string = " ".join(target_list) 304 target_dir_build_string = " ".join(target_dir_list) 305 extra_args_string = " ".join(extra_args_set) 306 307 install_path_goals = [] 308 mmma_goals = [] 309 for d in target_dir_list: 310 if d.startswith("./"): 311 d = d[2:] 312 if d.endswith("/"): 313 d = d[:-1] 314 install_path_goals.append("GET-INSTALL-PATH-IN-" + d.replace("/","-")) 315 mmma_goals.append("MODULES-IN-" + d.replace("/","-")) 316 # mmm cannot be used from python, so perform a similar operation using 317 # ONE_SHOT_MAKEFILE 318 cmd = 'ONE_SHOT_MAKEFILE="%s" make -j%s -C "%s" %s %s %s' % ( 319 target_build_string, self._options.make_jobs, self._root_path, 320 " ".join(install_path_goals), " ".join(mmma_goals), extra_args_string) 321 # mmma cannot be used from python, so perform a similar operation 322 alt_cmd = 'make -j%s -C "%s" -f build/core/main.mk %s %s' % ( 323 self._options.make_jobs, self._root_path, extra_args_string, " ".join(mmma_goals)) 324 325 logger.Log(cmd) 326 if not self._options.preview: 327 run_command.SetAbortOnError() 328 try: 329 output = run_command.RunCommand(cmd, return_output=True, timeout_time=600) 330 ## Chances are this failed because it didn't build the dependencies 331 except errors.AbortError: 332 logger.Log("make failed. Trying to rebuild all dependencies.") 333 logger.Log("mmma -j%s %s" %(self._options.make_jobs, target_dir_build_string)) 334 # Try again with mma equivalent, which will build the dependencies 335 run_command.RunCommand(alt_cmd, return_output=False, timeout_time=600) 336 # Run mmm again to get the install paths only 337 output = run_command.RunCommand(cmd, return_output=True, timeout_time=600) 338 run_command.SetAbortOnError(False) 339 logger.SilentLog(output) 340 filter_re = re.compile(self._options.filter_re) if self._options.filter_re else None 341 342 self._DoInstall(output, test_requires_permissions, filter_re=filter_re) 343 344 def _DoInstall(self, make_output, test_requires_permissions, filter_re=None): 345 """Install artifacts from build onto device. 346 347 Looks for 'install:' text from make output to find artifacts to install. 348 349 Files with the .apk extension get 'adb install'ed, all other files 350 get 'adb push'ed onto the device. 351 352 Args: 353 make_output: stdout from make command 354 """ 355 for line in make_output.split("\n"): 356 m = self._RE_MAKE_INSTALL.match(line) 357 if m: 358 # strip the 'INSTALL: <name>' from the left hand side 359 # the remaining string is a space-separated list of build-generated files 360 install_paths = m.group(2) 361 for install_path in re.split(r'\s+', install_paths): 362 if filter_re and not filter_re.match(install_path): 363 continue 364 if install_path.endswith(".apk"): 365 abs_install_path = os.path.join(self._root_path, install_path) 366 extra_flags = "" 367 if test_requires_permissions and not self._options.skip_permissions: 368 extra_flags = "-g" 369 if self._options.user: 370 extra_flags += " --user " + self._options.user 371 logger.Log("adb install -r %s %s" % (extra_flags, abs_install_path)) 372 logger.Log(self._adb.Install(abs_install_path, extra_flags)) 373 else: 374 self._PushInstallFileToDevice(install_path) 375 376 def _PushInstallFileToDevice(self, install_path): 377 m = self._re_make_install_path.match(install_path) 378 if m: 379 remote_path = m.group(1) 380 remote_dir = os.path.dirname(remote_path) 381 logger.Log("adb shell mkdir -p %s" % remote_dir) 382 self._adb.SendShellCommand("mkdir -p %s" % remote_dir) 383 abs_install_path = os.path.join(self._root_path, install_path) 384 logger.Log("adb push %s %s" % (abs_install_path, remote_path)) 385 self._adb.Push(abs_install_path, remote_path) 386 else: 387 logger.Log("Error: Failed to recognize path of file to install %s" % install_path) 388 389 def _DoFullBuild(self, tests, test_requires_permissions): 390 """If necessary, run a full 'make' command for the tests that need it.""" 391 extra_args_set = Set() 392 393 for test in tests: 394 if test.IsFullMake() and test.IsGrantedPermissions() == test_requires_permissions: 395 if test.GetExtraBuildArgs(): 396 # extra args contains the args to pass to 'make' 397 extra_args_set.add(test.GetExtraBuildArgs()) 398 else: 399 logger.Log("Warning: test %s needs a full build but does not specify" 400 " extra_build_args" % test.GetName()) 401 402 # check if there is actually any tests that required a full build 403 if extra_args_set: 404 cmd = ('make -j%s %s' % (self._options.make_jobs, 405 ' '.join(list(extra_args_set)))) 406 logger.Log(cmd) 407 if not self._options.preview: 408 old_dir = os.getcwd() 409 os.chdir(self._root_path) 410 output = run_command.RunCommand(cmd, return_output=True) 411 logger.SilentLog(output) 412 os.chdir(old_dir) 413 self._DoInstall(output, test_requires_permissions) 414 415 def _AddBuildTarget(self, test_suite, target_tree, extra_args_set): 416 if not test_suite.IsFullMake(): 417 build_dir = test_suite.GetBuildPath() 418 if self._AddBuildTargetPath(build_dir, target_tree): 419 extra_args_set.append(test_suite.GetExtraBuildArgs()) 420 for path in test_suite.GetBuildDependencies(self._options): 421 self._AddBuildTargetPath(path, target_tree) 422 423 def _AddBuildTargetPath(self, build_dir, target_tree): 424 if build_dir is not None: 425 target_tree.AddPath(build_dir) 426 return True 427 return False 428 429 def _GetTestsToRun(self): 430 """Get a list of TestSuite objects to run, based on command line args.""" 431 if self._tests_to_run: 432 return self._tests_to_run 433 434 self._tests_to_run = [] 435 if self._options.all_tests: 436 self._tests_to_run = self._known_tests.GetTests() 437 elif self._options.continuous_tests: 438 self._tests_to_run = self._known_tests.GetContinuousTests() 439 elif self._options.suite: 440 self._tests_to_run = \ 441 self._known_tests.GetTestsInSuite(self._options.suite) 442 elif self._options.test_path: 443 walker = test_walker.TestWalker() 444 self._tests_to_run = walker.FindTests(self._options.test_path) 445 446 for name in self._test_args: 447 test = self._known_tests.GetTest(name) 448 if test is None: 449 logger.Log("Error: Could not find test %s" % name) 450 self._DumpTests() 451 raise errors.AbortError 452 self._tests_to_run.append(test) 453 return self._tests_to_run 454 455 def _TurnOffVerifier(self, test_list): 456 """Turn off the dalvik verifier if needed by given tests. 457 458 If one or more tests needs dalvik verifier off, and it is not already off, 459 turns off verifier and reboots device to allow change to take effect. 460 """ 461 # hack to check if these are frameworks/base tests. If so, turn off verifier 462 # to allow framework tests to access private/protected/package-private framework api 463 framework_test = False 464 for test in test_list: 465 if os.path.commonprefix([test.GetBuildPath(), "frameworks/base"]): 466 framework_test = True 467 if framework_test: 468 # check if verifier is off already - to avoid the reboot if not 469 # necessary 470 output = self._adb.SendShellCommand("cat /data/local.prop") 471 if not self._DALVIK_VERIFIER_OFF_PROP in output: 472 473 # Read the existing dalvik verifier flags. 474 old_prop_value = self._adb.SendShellCommand("getprop %s" \ 475 %(self._DALVIK_VERIFIER_PROP)) 476 old_prop_value = old_prop_value.strip() if old_prop_value else "" 477 478 # Append our verifier flags to existing flags 479 new_prop_value = "%s %s" %(self._DALVIK_VERIFIER_OFF_VALUE, old_prop_value) 480 481 # Update property now, as /data/local.prop is not read until reboot 482 logger.Log("adb shell setprop %s '%s'" \ 483 %(self._DALVIK_VERIFIER_PROP, new_prop_value)) 484 if not self._options.preview: 485 self._adb.SendShellCommand("setprop %s '%s'" \ 486 %(self._DALVIK_VERIFIER_PROP, new_prop_value)) 487 488 # Write prop to /data/local.prop 489 # Every time device is booted, it will pick up this value 490 new_prop_assignment = "%s = %s" %(self._DALVIK_VERIFIER_PROP, new_prop_value) 491 if self._options.preview: 492 logger.Log("adb shell \"echo %s >> /data/local.prop\"" 493 % new_prop_assignment) 494 logger.Log("adb shell chmod 644 /data/local.prop") 495 else: 496 logger.Log("Turning off dalvik verifier and rebooting") 497 self._adb.SendShellCommand("\"echo %s >> /data/local.prop\"" 498 % new_prop_assignment) 499 500 # Reset runtime so that dalvik picks up new verifier flags from prop 501 self._ChmodRuntimeReset() 502 elif not self._options.preview: 503 # check the permissions on the file 504 permout = self._adb.SendShellCommand("ls -l /data/local.prop") 505 if not "-rw-r--r--" in permout: 506 logger.Log("Fixing permissions on /data/local.prop and rebooting") 507 self._ChmodRuntimeReset() 508 509 def _ChmodRuntimeReset(self): 510 """Perform a chmod of /data/local.prop and reset the runtime. 511 """ 512 logger.Log("adb shell chmod 644 /data/local.prop ## u+w,a+r") 513 if not self._options.preview: 514 self._adb.SendShellCommand("chmod 644 /data/local.prop") 515 516 self._adb.RuntimeReset(preview_only=self._options.preview) 517 518 if not self._options.preview: 519 self._adb.EnableAdbRoot() 520 521 522 def RunTests(self): 523 """Main entry method - executes the tests according to command line args.""" 524 try: 525 run_command.SetAbortOnError() 526 self._ProcessOptions() 527 if self._options.only_list_tests: 528 self._DumpTests() 529 return 530 531 if not self._options.skip_build: 532 self._DoBuild() 533 534 if self._options.build_install_only: 535 logger.Log("Skipping test execution (due to --build-install-only flag)") 536 return 537 538 for test_suite in self._GetTestsToRun(): 539 try: 540 test_suite.Run(self._options, self._adb) 541 except errors.WaitForResponseTimedOutError: 542 logger.Log("Timed out waiting for response") 543 544 except KeyboardInterrupt: 545 logger.Log("Exiting...") 546 except errors.AbortError, error: 547 logger.Log(error.msg) 548 logger.SilentLog("Exiting due to AbortError...") 549 except errors.WaitForResponseTimedOutError: 550 logger.Log("Timed out waiting for response") 551 552 553def RunTests(): 554 runner = TestRunner() 555 runner.RunTests() 556 557if __name__ == "__main__": 558 RunTests() 559