• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/python2.4
2#
3# Copyright 2008, The Android Open Source Project
4#
5# Licensed under the Apache License, Version 2.0 (the "License");
6# you may not use this file except in compliance with the License.
7# You may obtain a copy of the License at
8#
9#     http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16
17"""Command line utility for running Android tests
18
19runtest helps automate the instructions for building and running tests
20- It builds the corresponding test package for the code you want to test
21- It pushes the test package to your device or emulator
22- It launches InstrumentationTestRunner (or similar) to run the tests you
23specify.
24
25runtest supports running tests whose attributes have been pre-defined in
26_TEST_FILE_NAME files, (runtest <testname>), or by specifying the file
27system path to the test to run (runtest --path <path>).
28
29Do runtest --help to see full list of options.
30"""
31
32# Python imports
33import glob
34import optparse
35import os
36from sets import Set
37import sys
38
39# local imports
40import adb_interface
41import android_build
42import coverage
43import errors
44import logger
45import run_command
46from test_defs import test_defs
47from test_defs import test_walker
48
49
50class TestRunner(object):
51  """Command line utility class for running pre-defined Android test(s)."""
52
53  _TEST_FILE_NAME = "test_defs.xml"
54
55  # file path to android core platform tests, relative to android build root
56  # TODO move these test data files to another directory
57  _CORE_TEST_PATH = os.path.join("development", "testrunner",
58                                 _TEST_FILE_NAME)
59
60  # vendor glob file path patterns to tests, relative to android
61  # build root
62  _VENDOR_TEST_PATH = os.path.join("vendor", "*", "tests", "testinfo",
63                                   _TEST_FILE_NAME)
64
65  _RUNTEST_USAGE = (
66      "usage: runtest.py [options] short-test-name[s]\n\n"
67      "The runtest script works in two ways.  You can query it "
68      "for a list of tests, or you can launch one or more tests.")
69
70  # default value for make -jX
71  _DEFAULT_JOBS = 4
72
73  def __init__(self):
74    # disable logging of timestamp
75    self._root_path = android_build.GetTop()
76    logger.SetTimestampLogging(False)
77    self._adb = None
78    self._known_tests = None
79    self._options = None
80    self._test_args = None
81    self._tests_to_run = None
82
83  def _ProcessOptions(self):
84    """Processes command-line options."""
85    # TODO error messages on once-only or mutually-exclusive options.
86    user_test_default = os.path.join(os.environ.get("HOME"), ".android",
87                                     self._TEST_FILE_NAME)
88
89    parser = optparse.OptionParser(usage=self._RUNTEST_USAGE)
90
91    parser.add_option("-l", "--list-tests", dest="only_list_tests",
92                      default=False, action="store_true",
93                      help="To view the list of tests")
94    parser.add_option("-b", "--skip-build", dest="skip_build", default=False,
95                      action="store_true", help="Skip build - just launch")
96    parser.add_option("-j", "--jobs", dest="make_jobs",
97                      metavar="X", default=self._DEFAULT_JOBS,
98                      help="Number of make jobs to use when building")
99    parser.add_option("-n", "--skip_execute", dest="preview", default=False,
100                      action="store_true",
101                      help="Do not execute, just preview commands")
102    parser.add_option("-r", "--raw-mode", dest="raw_mode", default=False,
103                      action="store_true",
104                      help="Raw mode (for output to other tools)")
105    parser.add_option("-a", "--suite-assign", dest="suite_assign_mode",
106                      default=False, action="store_true",
107                      help="Suite assignment (for details & usage see "
108                      "InstrumentationTestRunner)")
109    parser.add_option("-v", "--verbose", dest="verbose", default=False,
110                      action="store_true",
111                      help="Increase verbosity of %s" % sys.argv[0])
112    parser.add_option("-w", "--wait-for-debugger", dest="wait_for_debugger",
113                      default=False, action="store_true",
114                      help="Wait for debugger before launching tests")
115    parser.add_option("-c", "--test-class", dest="test_class",
116                      help="Restrict test to a specific class")
117    parser.add_option("-m", "--test-method", dest="test_method",
118                      help="Restrict test to a specific method")
119    parser.add_option("-p", "--test-package", dest="test_package",
120                      help="Restrict test to a specific java package")
121    parser.add_option("-z", "--size", dest="test_size",
122                      help="Restrict test to a specific test size")
123    parser.add_option("-u", "--user-tests-file", dest="user_tests_file",
124                      metavar="FILE", default=user_test_default,
125                      help="Alternate source of user test definitions")
126    parser.add_option("-o", "--coverage", dest="coverage",
127                      default=False, action="store_true",
128                      help="Generate code coverage metrics for test(s)")
129    parser.add_option("-x", "--path", dest="test_path",
130                      help="Run test(s) at given file system path")
131    parser.add_option("-t", "--all-tests", dest="all_tests",
132                      default=False, action="store_true",
133                      help="Run all defined tests")
134    parser.add_option("--continuous", dest="continuous_tests",
135                      default=False, action="store_true",
136                      help="Run all tests defined as part of the continuous "
137                      "test set")
138    parser.add_option("--timeout", dest="timeout",
139                      default=300, help="Set a timeout limit (in sec) for "
140                      "running native tests on a device (default: 300 secs)")
141    parser.add_option("--cts", dest="cts_tests",
142                      default=False, action="store_true",
143                      help="Run all tests defined as part of the "
144                      "compatibility test suite")
145    group = optparse.OptionGroup(
146        parser, "Targets", "Use these options to direct tests to a specific "
147        "Android target")
148    group.add_option("-e", "--emulator", dest="emulator", default=False,
149                     action="store_true", help="use emulator")
150    group.add_option("-d", "--device", dest="device", default=False,
151                     action="store_true", help="use device")
152    group.add_option("-s", "--serial", dest="serial",
153                     help="use specific serial")
154    parser.add_option_group(group)
155
156    self._options, self._test_args = parser.parse_args()
157
158    if (not self._options.only_list_tests
159        and not self._options.all_tests
160        and not self._options.continuous_tests
161        and not self._options.cts_tests
162        and not self._options.test_path
163        and len(self._test_args) < 1):
164      parser.print_help()
165      logger.SilentLog("at least one test name must be specified")
166      raise errors.AbortError
167
168    self._adb = adb_interface.AdbInterface()
169    if self._options.emulator:
170      self._adb.SetEmulatorTarget()
171    elif self._options.device:
172      self._adb.SetDeviceTarget()
173    elif self._options.serial is not None:
174      self._adb.SetTargetSerial(self._options.serial)
175
176    if self._options.verbose:
177      logger.SetVerbose(True)
178
179    self._known_tests = self._ReadTests()
180
181    self._options.host_lib_path = android_build.GetHostLibraryPath()
182    self._options.test_data_path = android_build.GetTestAppPath()
183
184  def _ReadTests(self):
185    """Parses the set of test definition data.
186
187    Returns:
188      A TestDefinitions object that contains the set of parsed tests.
189    Raises:
190      AbortError: If a fatal error occurred when parsing the tests.
191    """
192    core_test_path = os.path.join(self._root_path, self._CORE_TEST_PATH)
193    try:
194      known_tests = test_defs.TestDefinitions()
195      known_tests.Parse(core_test_path)
196      # read all <android root>/vendor/*/tests/testinfo/test_defs.xml paths
197      vendor_tests_pattern = os.path.join(self._root_path,
198                                          self._VENDOR_TEST_PATH)
199      test_file_paths = glob.glob(vendor_tests_pattern)
200      for test_file_path in test_file_paths:
201        known_tests.Parse(test_file_path)
202      if os.path.isfile(self._options.user_tests_file):
203        known_tests.Parse(self._options.user_tests_file)
204      return known_tests
205    except errors.ParseError:
206      raise errors.AbortError
207
208  def _DumpTests(self):
209    """Prints out set of defined tests."""
210    print "The following tests are currently defined:\n"
211    print "%-25s %-40s %s" % ("name", "build path", "description")
212    print "-" * 80
213    for test in self._known_tests:
214      print "%-25s %-40s %s" % (test.GetName(), test.GetBuildPath(),
215                                test.GetDescription())
216    print "\nSee %s for more information" % self._TEST_FILE_NAME
217
218  def _DoBuild(self):
219    logger.SilentLog("Building tests...")
220    target_set = Set()
221    extra_args_set = Set()
222    tests = self._GetTestsToRun()
223    for test_suite in tests:
224      self._AddBuildTarget(test_suite, target_set, extra_args_set)
225
226    if target_set:
227      if self._options.coverage:
228        coverage.EnableCoverageBuild()
229
230      # hack to build cts dependencies
231      # TODO: remove this when build dependency support added to runtest or
232      # cts dependencies are removed
233      if self._IsCtsTests(tests):
234        # need to use make since these fail building with ONE_SHOT_MAKEFILE
235        cmd = ('make -j%s CtsTestStubs android.core.tests.runner' %
236               self._options.make_jobs)
237        logger.Log(cmd)
238        if not self._options.preview:
239          old_dir = os.getcwd()
240          os.chdir(self._root_path)
241          run_command.RunCommand(cmd, return_output=False)
242          os.chdir(old_dir)
243      target_build_string = " ".join(list(target_set))
244      extra_args_string = " ".join(list(extra_args_set))
245      # mmm cannot be used from python, so perform a similar operation using
246      # ONE_SHOT_MAKEFILE
247      cmd = 'ONE_SHOT_MAKEFILE="%s" make -j%s -C "%s" files %s' % (
248          target_build_string, self._options.make_jobs, self._root_path,
249          extra_args_string)
250      logger.Log(cmd)
251
252      if self._options.preview:
253        # in preview mode, just display to the user what command would have been
254        # run
255        logger.Log("adb sync")
256      else:
257        run_command.RunCommand(cmd, return_output=False)
258        logger.Log("Syncing to device...")
259        self._adb.Sync()
260
261  def _AddBuildTarget(self, test_suite, target_set, extra_args_set):
262    build_dir = test_suite.GetBuildPath()
263    if self._AddBuildTargetPath(build_dir, target_set):
264      extra_args_set.add(test_suite.GetExtraBuildArgs())
265    for path in test_suite.GetBuildDependencies(self._options):
266      self._AddBuildTargetPath(path, target_set)
267
268  def _AddBuildTargetPath(self, build_dir, target_set):
269    if build_dir is not None:
270      build_file_path = os.path.join(build_dir, "Android.mk")
271      if os.path.isfile(os.path.join(self._root_path, build_file_path)):
272        target_set.add(build_file_path)
273        return True
274      else:
275        logger.Log("%s has no Android.mk, skipping" % build_dir)
276    return False
277
278  def _GetTestsToRun(self):
279    """Get a list of TestSuite objects to run, based on command line args."""
280    if self._tests_to_run:
281      return self._tests_to_run
282
283    self._tests_to_run = []
284    if self._options.all_tests:
285      self._tests_to_run = self._known_tests.GetTests()
286    elif self._options.continuous_tests:
287      self._tests_to_run = self._known_tests.GetContinuousTests()
288    elif self._options.cts_tests:
289      self._tests_to_run = self._known_tests.GetCtsTests()
290    elif self._options.test_path:
291      walker = test_walker.TestWalker()
292      self._tests_to_run = walker.FindTests(self._options.test_path)
293
294    for name in self._test_args:
295      test = self._known_tests.GetTest(name)
296      if test is None:
297        logger.Log("Error: Could not find test %s" % name)
298        self._DumpTests()
299        raise errors.AbortError
300      self._tests_to_run.append(test)
301    return self._tests_to_run
302
303  def _IsCtsTests(self, test_list):
304    """Check if any cts tests are included in given list of tests to run."""
305    for test in test_list:
306      if test.IsCts():
307        return True
308    return False
309
310  def RunTests(self):
311    """Main entry method - executes the tests according to command line args."""
312    try:
313      run_command.SetAbortOnError()
314      self._ProcessOptions()
315      if self._options.only_list_tests:
316        self._DumpTests()
317        return
318
319      if not self._options.skip_build:
320        self._DoBuild()
321
322      for test_suite in self._GetTestsToRun():
323        test_suite.Run(self._options, self._adb)
324
325    except KeyboardInterrupt:
326      logger.Log("Exiting...")
327    except errors.AbortError, error:
328      logger.Log(error.msg)
329      logger.SilentLog("Exiting due to AbortError...")
330    except errors.WaitForResponseTimedOutError:
331      logger.Log("Timed out waiting for response")
332
333
334def RunTests():
335  runner = TestRunner()
336  runner.RunTests()
337
338if __name__ == "__main__":
339  RunTests()
340