1#!/usr/bin/env python 2# 3# Copyright 2008, The Android Open Source Project 4# 5# Licensed under the Apache License, Version 2.0 (the "License"); 6# you may not use this file except in compliance with the License. 7# You may obtain a copy of the License at 8# 9# http://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an "AS IS" BASIS, 13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14# See the License for the specific language governing permissions and 15# limitations under the License. 16 17"""Command line utility for running Android tests 18 19runtest helps automate the instructions for building and running tests 20- It builds the corresponding test package for the code you want to test 21- It pushes the test package to your device or emulator 22- It launches InstrumentationTestRunner (or similar) to run the tests you 23specify. 24 25runtest supports running tests whose attributes have been pre-defined in 26_TEST_FILE_NAME files, (runtest <testname>), or by specifying the file 27system path to the test to run (runtest --path <path>). 28 29Do runtest --help to see full list of options. 30""" 31 32# Python imports 33import glob 34import optparse 35import os 36import re 37from sets import Set 38import sys 39import time 40 41# local imports 42import adb_interface 43import android_build 44from coverage import coverage 45import errors 46import logger 47import make_tree 48import run_command 49from test_defs import test_defs 50from test_defs import test_walker 51 52 53class TestRunner(object): 54 """Command line utility class for running pre-defined Android test(s).""" 55 56 _TEST_FILE_NAME = "test_defs.xml" 57 58 # file path to android core platform tests, relative to android build root 59 # TODO move these test data files to another directory 60 _CORE_TEST_PATH = os.path.join("development", "testrunner", 61 _TEST_FILE_NAME) 62 63 # vendor glob file path patterns to tests, relative to android 64 # build root 65 _VENDOR_TEST_PATH = os.path.join("vendor", "*", "tests", "testinfo", 66 _TEST_FILE_NAME) 67 68 _RUNTEST_USAGE = ( 69 "usage: runtest.py [options] short-test-name[s]\n\n" 70 "The runtest script works in two ways. You can query it " 71 "for a list of tests, or you can launch one or more tests.") 72 73 # default value for make -jX 74 _DEFAULT_JOBS = 16 75 76 _DALVIK_VERIFIER_PROP = "dalvik.vm.dexopt-flags" 77 _DALVIK_VERIFIER_OFF_VALUE = "v=n" 78 _DALVIK_VERIFIER_OFF_PROP = "%s = %s" %(_DALVIK_VERIFIER_PROP, _DALVIK_VERIFIER_OFF_VALUE) 79 80 # regular expression to match path to artifacts to install in make output 81 _RE_MAKE_INSTALL = re.compile(r'INSTALL-PATH:\s([^\s]+)\s(.*)$') 82 83 def __init__(self): 84 # disable logging of timestamp 85 self._root_path = android_build.GetTop() 86 out_base_name = os.path.basename(android_build.GetOutDir()) 87 # regular expression to find remote device path from a file path relative 88 # to build root 89 pattern = r'' + out_base_name + r'\/target\/product\/\w+\/(.+)$' 90 self._re_make_install_path = re.compile(pattern) 91 logger.SetTimestampLogging(False) 92 self._adb = None 93 self._known_tests = None 94 self._options = None 95 self._test_args = None 96 self._tests_to_run = None 97 98 def _ProcessOptions(self): 99 """Processes command-line options.""" 100 # TODO error messages on once-only or mutually-exclusive options. 101 user_test_default = os.path.join(os.environ.get("HOME"), ".android", 102 self._TEST_FILE_NAME) 103 104 parser = optparse.OptionParser(usage=self._RUNTEST_USAGE) 105 106 parser.add_option("-l", "--list-tests", dest="only_list_tests", 107 default=False, action="store_true", 108 help="To view the list of tests") 109 parser.add_option("-b", "--skip-build", dest="skip_build", default=False, 110 action="store_true", help="Skip build - just launch") 111 parser.add_option("-j", "--jobs", dest="make_jobs", 112 metavar="X", default=self._DEFAULT_JOBS, 113 help="Number of make jobs to use when building") 114 parser.add_option("-n", "--skip_execute", dest="preview", default=False, 115 action="store_true", 116 help="Do not execute, just preview commands") 117 parser.add_option("-i", "--build-install-only", dest="build_install_only", default=False, 118 action="store_true", 119 help="Do not execute, build tests and install to device only") 120 parser.add_option("-r", "--raw-mode", dest="raw_mode", default=False, 121 action="store_true", 122 help="Raw mode (for output to other tools)") 123 parser.add_option("-a", "--suite-assign", dest="suite_assign_mode", 124 default=False, action="store_true", 125 help="Suite assignment (for details & usage see " 126 "InstrumentationTestRunner)") 127 parser.add_option("-v", "--verbose", dest="verbose", default=False, 128 action="store_true", 129 help="Increase verbosity of %s" % sys.argv[0]) 130 parser.add_option("-w", "--wait-for-debugger", dest="wait_for_debugger", 131 default=False, action="store_true", 132 help="Wait for debugger before launching tests") 133 parser.add_option("-c", "--test-class", dest="test_class", 134 help="Restrict test to a specific class") 135 parser.add_option("-m", "--test-method", dest="test_method", 136 help="Restrict test to a specific method") 137 parser.add_option("-p", "--test-package", dest="test_package", 138 help="Restrict test to a specific java package") 139 parser.add_option("-z", "--size", dest="test_size", 140 help="Restrict test to a specific test size") 141 parser.add_option("--annotation", dest="test_annotation", 142 help="Include only those tests tagged with a specific" 143 " annotation") 144 parser.add_option("--not-annotation", dest="test_not_annotation", 145 help="Exclude any tests tagged with a specific" 146 " annotation") 147 parser.add_option("-u", "--user-tests-file", dest="user_tests_file", 148 metavar="FILE", default=user_test_default, 149 help="Alternate source of user test definitions") 150 parser.add_option("-o", "--coverage", dest="coverage", 151 default=False, action="store_true", 152 help="Generate code coverage metrics for test(s)") 153 parser.add_option("--coverage-target", dest="coverage_target_path", 154 default=None, 155 help="Path to app to collect code coverage target data for.") 156 parser.add_option("-x", "--path", dest="test_path", 157 help="Run test(s) at given file system path") 158 parser.add_option("-t", "--all-tests", dest="all_tests", 159 default=False, action="store_true", 160 help="Run all defined tests") 161 parser.add_option("--continuous", dest="continuous_tests", 162 default=False, action="store_true", 163 help="Run all tests defined as part of the continuous " 164 "test set") 165 parser.add_option("--timeout", dest="timeout", 166 default=300, help="Set a timeout limit (in sec) for " 167 "running native tests on a device (default: 300 secs)") 168 parser.add_option("--suite", dest="suite", 169 help="Run all tests defined as part of the " 170 "the given test suite") 171 group = optparse.OptionGroup( 172 parser, "Targets", "Use these options to direct tests to a specific " 173 "Android target") 174 group.add_option("-e", "--emulator", dest="emulator", default=False, 175 action="store_true", help="use emulator") 176 group.add_option("-d", "--device", dest="device", default=False, 177 action="store_true", help="use device") 178 group.add_option("-s", "--serial", dest="serial", 179 help="use specific serial") 180 parser.add_option_group(group) 181 self._options, self._test_args = parser.parse_args() 182 183 if (not self._options.only_list_tests 184 and not self._options.all_tests 185 and not self._options.continuous_tests 186 and not self._options.suite 187 and not self._options.test_path 188 and len(self._test_args) < 1): 189 parser.print_help() 190 logger.SilentLog("at least one test name must be specified") 191 raise errors.AbortError 192 193 self._adb = adb_interface.AdbInterface() 194 if self._options.emulator: 195 self._adb.SetEmulatorTarget() 196 elif self._options.device: 197 self._adb.SetDeviceTarget() 198 elif self._options.serial is not None: 199 self._adb.SetTargetSerial(self._options.serial) 200 if self._options.verbose: 201 logger.SetVerbose(True) 202 203 if self._options.coverage_target_path: 204 self._options.coverage = True 205 206 self._known_tests = self._ReadTests() 207 208 self._options.host_lib_path = android_build.GetHostLibraryPath() 209 self._options.test_data_path = android_build.GetTestAppPath() 210 211 def _ReadTests(self): 212 """Parses the set of test definition data. 213 214 Returns: 215 A TestDefinitions object that contains the set of parsed tests. 216 Raises: 217 AbortError: If a fatal error occurred when parsing the tests. 218 """ 219 try: 220 known_tests = test_defs.TestDefinitions() 221 # only read tests when not in path mode 222 if not self._options.test_path: 223 core_test_path = os.path.join(self._root_path, self._CORE_TEST_PATH) 224 if os.path.isfile(core_test_path): 225 known_tests.Parse(core_test_path) 226 # read all <android root>/vendor/*/tests/testinfo/test_defs.xml paths 227 vendor_tests_pattern = os.path.join(self._root_path, 228 self._VENDOR_TEST_PATH) 229 test_file_paths = glob.glob(vendor_tests_pattern) 230 for test_file_path in test_file_paths: 231 known_tests.Parse(test_file_path) 232 if os.path.isfile(self._options.user_tests_file): 233 known_tests.Parse(self._options.user_tests_file) 234 return known_tests 235 except errors.ParseError: 236 raise errors.AbortError 237 238 def _DumpTests(self): 239 """Prints out set of defined tests.""" 240 print "The following tests are currently defined:\n" 241 print "%-25s %-40s %s" % ("name", "build path", "description") 242 print "-" * 80 243 for test in self._known_tests: 244 print "%-25s %-40s %s" % (test.GetName(), test.GetBuildPath(), 245 test.GetDescription()) 246 print "\nSee %s for more information" % self._TEST_FILE_NAME 247 248 def _DoBuild(self): 249 logger.SilentLog("Building tests...") 250 251 tests = self._GetTestsToRun() 252 # turn off dalvik verifier if necessary 253 # TODO: skip turning off verifier for now, since it puts device in bad 254 # state b/14088982 255 #self._TurnOffVerifier(tests) 256 self._DoFullBuild(tests) 257 258 target_tree = make_tree.MakeTree() 259 260 extra_args_set = [] 261 for test_suite in tests: 262 self._AddBuildTarget(test_suite, target_tree, extra_args_set) 263 264 if not self._options.preview: 265 self._adb.EnableAdbRoot() 266 else: 267 logger.Log("adb root") 268 269 if not target_tree.IsEmpty(): 270 if self._options.coverage: 271 coverage.EnableCoverageBuild() 272 target_tree.AddPath("external/emma") 273 274 target_list = target_tree.GetPrunedMakeList() 275 target_dir_list = [re.sub(r'Android[.]mk$', r'', i) for i in target_list] 276 target_build_string = " ".join(target_list) 277 target_dir_build_string = " ".join(target_dir_list) 278 extra_args_string = " ".join(extra_args_set) 279 280 # mmm cannot be used from python, so perform a similar operation using 281 # ONE_SHOT_MAKEFILE 282 cmd = 'ONE_SHOT_MAKEFILE="%s" make -j%s -C "%s" GET-INSTALL-PATH all_modules %s' % ( 283 target_build_string, self._options.make_jobs, self._root_path, 284 extra_args_string) 285 # mmma equivalent, used when regular mmm fails 286 alt_cmd = 'make -j%s -C "%s" -f build/core/main.mk %s all_modules BUILD_MODULES_IN_PATHS="%s"' % ( 287 self._options.make_jobs, self._root_path, extra_args_string, target_dir_build_string) 288 289 logger.Log(cmd) 290 if not self._options.preview: 291 run_command.SetAbortOnError() 292 try: 293 output = run_command.RunCommand(cmd, return_output=True, timeout_time=600) 294 ## Chances are this failed because it didn't build the dependencies 295 except errors.AbortError: 296 logger.Log("make failed. Trying to rebuild all dependencies.") 297 logger.Log("mmma -j%s %s" %(self._options.make_jobs, target_dir_build_string)) 298 # Try again with mma equivalent, which will build the dependencies 299 run_command.RunCommand(alt_cmd, return_output=False, timeout_time=600) 300 # Run mmm again to get the install paths only 301 output = run_command.RunCommand(cmd, return_output=True, timeout_time=600) 302 run_command.SetAbortOnError(False) 303 logger.SilentLog(output) 304 self._DoInstall(output) 305 306 def _DoInstall(self, make_output): 307 """Install artifacts from build onto device. 308 309 Looks for 'install:' text from make output to find artifacts to install. 310 311 Files with the .apk extension get 'adb install'ed, all other files 312 get 'adb push'ed onto the device. 313 314 Args: 315 make_output: stdout from make command 316 """ 317 for line in make_output.split("\n"): 318 m = self._RE_MAKE_INSTALL.match(line) 319 if m: 320 # strip the 'INSTALL: <name>' from the left hand side 321 # the remaining string is a space-separated list of build-generated files 322 install_paths = m.group(2) 323 for install_path in re.split(r'\s+', install_paths): 324 if install_path.endswith(".apk"): 325 abs_install_path = os.path.join(self._root_path, install_path) 326 logger.Log("adb install -r %s" % abs_install_path) 327 logger.Log(self._adb.Install(abs_install_path)) 328 else: 329 self._PushInstallFileToDevice(install_path) 330 331 def _PushInstallFileToDevice(self, install_path): 332 m = self._re_make_install_path.match(install_path) 333 if m: 334 remote_path = m.group(1) 335 remote_dir = os.path.dirname(remote_path) 336 logger.Log("adb shell mkdir -p %s" % remote_dir) 337 self._adb.SendShellCommand("mkdir -p %s" % remote_dir) 338 abs_install_path = os.path.join(self._root_path, install_path) 339 logger.Log("adb push %s %s" % (abs_install_path, remote_path)) 340 self._adb.Push(abs_install_path, remote_path) 341 else: 342 logger.Log("Error: Failed to recognize path of file to install %s" % install_path) 343 344 def _DoFullBuild(self, tests): 345 """If necessary, run a full 'make' command for the tests that need it.""" 346 extra_args_set = Set() 347 348 for test in tests: 349 if test.IsFullMake(): 350 if test.GetExtraBuildArgs(): 351 # extra args contains the args to pass to 'make' 352 extra_args_set.add(test.GetExtraBuildArgs()) 353 else: 354 logger.Log("Warning: test %s needs a full build but does not specify" 355 " extra_build_args" % test.GetName()) 356 357 # check if there is actually any tests that required a full build 358 if extra_args_set: 359 cmd = ('make -j%s %s' % (self._options.make_jobs, 360 ' '.join(list(extra_args_set)))) 361 logger.Log(cmd) 362 if not self._options.preview: 363 old_dir = os.getcwd() 364 os.chdir(self._root_path) 365 output = run_command.RunCommand(cmd, return_output=True) 366 logger.SilentLog(output) 367 os.chdir(old_dir) 368 self._DoInstall(output) 369 370 def _AddBuildTarget(self, test_suite, target_tree, extra_args_set): 371 if not test_suite.IsFullMake(): 372 build_dir = test_suite.GetBuildPath() 373 if self._AddBuildTargetPath(build_dir, target_tree): 374 extra_args_set.append(test_suite.GetExtraBuildArgs()) 375 for path in test_suite.GetBuildDependencies(self._options): 376 self._AddBuildTargetPath(path, target_tree) 377 378 def _AddBuildTargetPath(self, build_dir, target_tree): 379 if build_dir is not None: 380 target_tree.AddPath(build_dir) 381 return True 382 return False 383 384 def _GetTestsToRun(self): 385 """Get a list of TestSuite objects to run, based on command line args.""" 386 if self._tests_to_run: 387 return self._tests_to_run 388 389 self._tests_to_run = [] 390 if self._options.all_tests: 391 self._tests_to_run = self._known_tests.GetTests() 392 elif self._options.continuous_tests: 393 self._tests_to_run = self._known_tests.GetContinuousTests() 394 elif self._options.suite: 395 self._tests_to_run = \ 396 self._known_tests.GetTestsInSuite(self._options.suite) 397 elif self._options.test_path: 398 walker = test_walker.TestWalker() 399 self._tests_to_run = walker.FindTests(self._options.test_path) 400 401 for name in self._test_args: 402 test = self._known_tests.GetTest(name) 403 if test is None: 404 logger.Log("Error: Could not find test %s" % name) 405 self._DumpTests() 406 raise errors.AbortError 407 self._tests_to_run.append(test) 408 return self._tests_to_run 409 410 def _IsCtsTests(self, test_list): 411 """Check if any cts tests are included in given list of tests to run.""" 412 for test in test_list: 413 if test.GetSuite() == 'cts': 414 return True 415 return False 416 417 def _TurnOffVerifier(self, test_list): 418 """Turn off the dalvik verifier if needed by given tests. 419 420 If one or more tests needs dalvik verifier off, and it is not already off, 421 turns off verifier and reboots device to allow change to take effect. 422 """ 423 # hack to check if these are frameworks/base tests. If so, turn off verifier 424 # to allow framework tests to access private/protected/package-private framework api 425 framework_test = False 426 for test in test_list: 427 if os.path.commonprefix([test.GetBuildPath(), "frameworks/base"]): 428 framework_test = True 429 if framework_test: 430 # check if verifier is off already - to avoid the reboot if not 431 # necessary 432 output = self._adb.SendShellCommand("cat /data/local.prop") 433 if not self._DALVIK_VERIFIER_OFF_PROP in output: 434 435 # Read the existing dalvik verifier flags. 436 old_prop_value = self._adb.SendShellCommand("getprop %s" \ 437 %(self._DALVIK_VERIFIER_PROP)) 438 old_prop_value = old_prop_value.strip() if old_prop_value else "" 439 440 # Append our verifier flags to existing flags 441 new_prop_value = "%s %s" %(self._DALVIK_VERIFIER_OFF_VALUE, old_prop_value) 442 443 # Update property now, as /data/local.prop is not read until reboot 444 logger.Log("adb shell setprop %s '%s'" \ 445 %(self._DALVIK_VERIFIER_PROP, new_prop_value)) 446 if not self._options.preview: 447 self._adb.SendShellCommand("setprop %s '%s'" \ 448 %(self._DALVIK_VERIFIER_PROP, new_prop_value)) 449 450 # Write prop to /data/local.prop 451 # Every time device is booted, it will pick up this value 452 new_prop_assignment = "%s = %s" %(self._DALVIK_VERIFIER_PROP, new_prop_value) 453 if self._options.preview: 454 logger.Log("adb shell \"echo %s >> /data/local.prop\"" 455 % new_prop_assignment) 456 logger.Log("adb shell chmod 644 /data/local.prop") 457 else: 458 logger.Log("Turning off dalvik verifier and rebooting") 459 self._adb.SendShellCommand("\"echo %s >> /data/local.prop\"" 460 % new_prop_assignment) 461 462 # Reset runtime so that dalvik picks up new verifier flags from prop 463 self._ChmodRuntimeReset() 464 elif not self._options.preview: 465 # check the permissions on the file 466 permout = self._adb.SendShellCommand("ls -l /data/local.prop") 467 if not "-rw-r--r--" in permout: 468 logger.Log("Fixing permissions on /data/local.prop and rebooting") 469 self._ChmodRuntimeReset() 470 471 def _ChmodRuntimeReset(self): 472 """Perform a chmod of /data/local.prop and reset the runtime. 473 """ 474 logger.Log("adb shell chmod 644 /data/local.prop ## u+w,a+r") 475 if not self._options.preview: 476 self._adb.SendShellCommand("chmod 644 /data/local.prop") 477 478 self._adb.RuntimeReset(preview_only=self._options.preview) 479 480 if not self._options.preview: 481 self._adb.EnableAdbRoot() 482 483 484 def RunTests(self): 485 """Main entry method - executes the tests according to command line args.""" 486 try: 487 run_command.SetAbortOnError() 488 self._ProcessOptions() 489 if self._options.only_list_tests: 490 self._DumpTests() 491 return 492 493 if not self._options.skip_build: 494 self._DoBuild() 495 496 if self._options.build_install_only: 497 logger.Log("Skipping test execution (due to --build-install-only flag)") 498 return 499 500 for test_suite in self._GetTestsToRun(): 501 try: 502 test_suite.Run(self._options, self._adb) 503 except errors.WaitForResponseTimedOutError: 504 logger.Log("Timed out waiting for response") 505 506 except KeyboardInterrupt: 507 logger.Log("Exiting...") 508 except errors.AbortError, error: 509 logger.Log(error.msg) 510 logger.SilentLog("Exiting due to AbortError...") 511 except errors.WaitForResponseTimedOutError: 512 logger.Log("Timed out waiting for response") 513 514 515def RunTests(): 516 runner = TestRunner() 517 runner.RunTests() 518 519if __name__ == "__main__": 520 RunTests() 521