• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1# Copyright 2013 The Chromium Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5"""Generates test runner factory and tests for GTests."""
6
7import fnmatch
8import glob
9import logging
10import os
11import shutil
12import sys
13
14from pylib import android_commands
15from pylib import cmd_helper
16from pylib import constants
17from pylib import ports
18
19import test_package_apk
20import test_package_exe
21import test_runner
22
23sys.path.insert(0,
24                os.path.join(constants.DIR_SOURCE_ROOT, 'build', 'util', 'lib',
25                             'common'))
26import unittest_util
27
28
29_ISOLATE_FILE_PATHS = {
30    'base_unittests': 'base/base_unittests.isolate',
31    'breakpad_unittests': 'breakpad/breakpad_unittests.isolate',
32    'cc_perftests': 'cc/cc_perftests.isolate',
33    'components_unittests': 'components/components_unittests.isolate',
34    'content_browsertests': 'content/content_browsertests.isolate',
35    'content_unittests': 'content/content_unittests.isolate',
36    'media_perftests': 'media/media_perftests.isolate',
37    'media_unittests': 'media/media_unittests.isolate',
38    'net_unittests': 'net/net_unittests.isolate',
39    'ui_unittests': 'ui/ui_unittests.isolate',
40    'unit_tests': 'chrome/unit_tests.isolate',
41    'webkit_unit_tests':
42      'third_party/WebKit/Source/web/WebKitUnitTests.isolate',
43}
44
45# Paths relative to third_party/webrtc/ (kept separate for readability).
46_WEBRTC_ISOLATE_FILE_PATHS = {
47    'audio_decoder_unittests':
48      'modules/audio_coding/neteq4/audio_decoder_unittests.isolate',
49    'common_audio_unittests': 'common_audio/common_audio_unittests.isolate',
50    'common_video_unittests': 'common_video/common_video_unittests.isolate',
51    'metrics_unittests': 'test/metrics_unittests.isolate',
52    'modules_tests': 'modules/modules_tests.isolate',
53    'modules_unittests': 'modules/modules_unittests.isolate',
54    'neteq_unittests': 'modules/audio_coding/neteq/neteq_unittests.isolate',
55    'system_wrappers_unittests':
56      'system_wrappers/source/system_wrappers_unittests.isolate',
57    'test_support_unittests': 'test/test_support_unittests.isolate',
58    'tools_unittests': 'tools/tools_unittests.isolate',
59    'video_engine_core_unittests':
60      'video_engine/video_engine_core_unittests.isolate',
61    'voice_engine_unittests': 'voice_engine/voice_engine_unittests.isolate',
62}
63
64# Append the WebRTC tests with the full path from Chromium's src/ root.
65for test, isolate_path in _WEBRTC_ISOLATE_FILE_PATHS.items():
66  _ISOLATE_FILE_PATHS[test] = 'third_party/webrtc/%s' % isolate_path
67
68# Used for filtering large data deps at a finer grain than what's allowed in
69# isolate files since pushing deps to devices is expensive.
70# Wildcards are allowed.
71_DEPS_EXCLUSION_LIST = [
72    'chrome/test/data/extensions/api_test',
73    'chrome/test/data/extensions/secure_shell',
74    'chrome/test/data/firefox*',
75    'chrome/test/data/gpu',
76    'chrome/test/data/image_decoding',
77    'chrome/test/data/import',
78    'chrome/test/data/page_cycler',
79    'chrome/test/data/perf',
80    'chrome/test/data/pyauto_private',
81    'chrome/test/data/safari_import',
82    'chrome/test/data/scroll',
83    'chrome/test/data/third_party',
84    'third_party/hunspell_dictionaries/*.dic',
85    # crbug.com/258690
86    'webkit/data/bmp_decoder',
87    'webkit/data/ico_decoder',
88]
89
90_ISOLATE_SCRIPT = os.path.join(
91    constants.DIR_SOURCE_ROOT, 'tools', 'swarming_client', 'isolate.py')
92
93
94def _GenerateDepsDirUsingIsolate(suite_name):
95  """Generate the dependency dir for the test suite using isolate.
96
97  Args:
98    suite_name: Name of the test suite (e.g. base_unittests).
99  """
100  if os.path.isdir(constants.ISOLATE_DEPS_DIR):
101    shutil.rmtree(constants.ISOLATE_DEPS_DIR)
102
103  isolate_rel_path = _ISOLATE_FILE_PATHS.get(suite_name)
104  if not isolate_rel_path:
105    logging.info('Did not find an isolate file for the test suite.')
106    return
107
108  isolate_abs_path = os.path.join(constants.DIR_SOURCE_ROOT, isolate_rel_path)
109  isolated_abs_path = os.path.join(
110      constants.GetOutDirectory(), '%s.isolated' % suite_name)
111  assert os.path.exists(isolate_abs_path)
112  isolate_cmd = [
113      'python', _ISOLATE_SCRIPT,
114      'remap',
115      '--isolate', isolate_abs_path,
116      '--isolated', isolated_abs_path,
117      '--path-variable', 'PRODUCT_DIR', constants.GetOutDirectory(),
118      '--config-variable', 'OS', 'android',
119      '--outdir', constants.ISOLATE_DEPS_DIR,
120  ]
121  assert not cmd_helper.RunCmd(isolate_cmd)
122
123  # We're relying on the fact that timestamps are preserved
124  # by the remap command (hardlinked). Otherwise, all the data
125  # will be pushed to the device once we move to using time diff
126  # instead of md5sum. Perform a sanity check here.
127  for root, _, filenames in os.walk(constants.ISOLATE_DEPS_DIR):
128    if filenames:
129      linked_file = os.path.join(root, filenames[0])
130      orig_file = os.path.join(
131          constants.DIR_SOURCE_ROOT,
132          os.path.relpath(linked_file, constants.ISOLATE_DEPS_DIR))
133      if os.stat(linked_file).st_ino == os.stat(orig_file).st_ino:
134        break
135      else:
136        raise Exception('isolate remap command did not use hardlinks.')
137
138  # Delete excluded files as defined by _DEPS_EXCLUSION_LIST.
139  old_cwd = os.getcwd()
140  try:
141    os.chdir(constants.ISOLATE_DEPS_DIR)
142    excluded_paths = [x for y in _DEPS_EXCLUSION_LIST for x in glob.glob(y)]
143    if excluded_paths:
144      logging.info('Excluding the following from dependency list: %s',
145                   excluded_paths)
146    for p in excluded_paths:
147      if os.path.isdir(p):
148        shutil.rmtree(p)
149      else:
150        os.remove(p)
151  finally:
152    os.chdir(old_cwd)
153
154  # On Android, all pak files need to be in the top-level 'paks' directory.
155  paks_dir = os.path.join(constants.ISOLATE_DEPS_DIR, 'paks')
156  os.mkdir(paks_dir)
157  for root, _, filenames in os.walk(os.path.join(constants.ISOLATE_DEPS_DIR,
158                                                 'out')):
159    for filename in fnmatch.filter(filenames, '*.pak'):
160      shutil.move(os.path.join(root, filename), paks_dir)
161
162  # Move everything in PRODUCT_DIR to top level.
163  deps_product_dir = os.path.join(constants.ISOLATE_DEPS_DIR, 'out',
164      constants.GetBuildType())
165  if os.path.isdir(deps_product_dir):
166    for p in os.listdir(deps_product_dir):
167      shutil.move(os.path.join(deps_product_dir, p), constants.ISOLATE_DEPS_DIR)
168    os.rmdir(deps_product_dir)
169    os.rmdir(os.path.join(constants.ISOLATE_DEPS_DIR, 'out'))
170
171
172def _GetDisabledTestsFilterFromFile(suite_name):
173  """Returns a gtest filter based on the *_disabled file.
174
175  Args:
176    suite_name: Name of the test suite (e.g. base_unittests).
177
178  Returns:
179    A gtest filter which excludes disabled tests.
180    Example: '*-StackTrace.*:StringPrintfTest.StringPrintfMisc'
181  """
182  filter_file_path = os.path.join(
183      os.path.abspath(os.path.dirname(__file__)),
184      'filter', '%s_disabled' % suite_name)
185
186  if not filter_file_path or not os.path.exists(filter_file_path):
187    logging.info('No filter file found at %s', filter_file_path)
188    return '*'
189
190  filters = [x for x in [x.strip() for x in file(filter_file_path).readlines()]
191             if x and x[0] != '#']
192  disabled_filter = '*-%s' % ':'.join(filters)
193  logging.info('Applying filter "%s" obtained from %s',
194               disabled_filter, filter_file_path)
195  return disabled_filter
196
197
198def _GetTestsFromDevice(runner_factory, devices):
199  """Get a list of tests from a device.
200
201  Args:
202    runner_factory: Callable that takes device and shard_index and returns
203        a TestRunner.
204    devices: A list of device ids.
205
206  Returns:
207    All the tests in the test suite.
208  """
209  for device in devices:
210    try:
211      logging.info('Obtaining tests from %s', device)
212      return runner_factory(device, 0).GetAllTests()
213    except (android_commands.errors.WaitForResponseTimedOutError,
214            android_commands.errors.DeviceUnresponsiveError), e:
215      logging.warning('Failed obtaining test list from %s with exception: %s',
216                      device, e)
217  raise Exception('Failed to obtain test list from devices.')
218
219
220def _FilterTestsUsingPrefixes(all_tests, pre=False, manual=False):
221  """Removes tests with disabled prefixes.
222
223  Args:
224    all_tests: List of tests to filter.
225    pre: If True, include tests with PRE_ prefix.
226    manual: If True, include tests with MANUAL_ prefix.
227
228  Returns:
229    List of tests remaining.
230  """
231  filtered_tests = []
232  filter_prefixes = ['DISABLED_', 'FLAKY_', 'FAILS_']
233
234  if not pre:
235    filter_prefixes.append('PRE_')
236
237  if not manual:
238    filter_prefixes.append('MANUAL_')
239
240  for t in all_tests:
241    test_case, test = t.split('.', 1)
242    if not any([test_case.startswith(prefix) or test.startswith(prefix) for
243                prefix in filter_prefixes]):
244      filtered_tests.append(t)
245  return filtered_tests
246
247
248def _FilterDisabledTests(tests, suite_name, has_gtest_filter):
249  """Removes disabled tests from |tests|.
250
251  Applies the following filters in order:
252    1. Remove tests with disabled prefixes.
253    2. Remove tests specified in the *_disabled files in the 'filter' dir
254
255  Args:
256    tests: List of tests.
257    suite_name: Name of the test suite (e.g. base_unittests).
258    has_gtest_filter: Whether a gtest_filter is provided.
259
260  Returns:
261    List of tests remaining.
262  """
263  tests = _FilterTestsUsingPrefixes(
264      tests, has_gtest_filter, has_gtest_filter)
265  tests = unittest_util.FilterTestNames(
266      tests, _GetDisabledTestsFilterFromFile(suite_name))
267
268  return tests
269
270
271def Setup(test_options, devices):
272  """Create the test runner factory and tests.
273
274  Args:
275    test_options: A GTestOptions object.
276    devices: A list of attached devices.
277
278  Returns:
279    A tuple of (TestRunnerFactory, tests).
280  """
281  test_package = test_package_apk.TestPackageApk(test_options.suite_name)
282  if not os.path.exists(test_package.suite_path):
283    test_package = test_package_exe.TestPackageExecutable(
284        test_options.suite_name)
285    if not os.path.exists(test_package.suite_path):
286      raise Exception(
287          'Did not find %s target. Ensure it has been built.'
288          % test_options.suite_name)
289  logging.warning('Found target %s', test_package.suite_path)
290
291  _GenerateDepsDirUsingIsolate(test_options.suite_name)
292
293  # Constructs a new TestRunner with the current options.
294  def TestRunnerFactory(device, shard_index):
295    return test_runner.TestRunner(
296        test_options,
297        device,
298        test_package)
299
300  tests = _GetTestsFromDevice(TestRunnerFactory, devices)
301  if test_options.run_disabled:
302    test_options = test_options._replace(
303        test_arguments=('%s --gtest_also_run_disabled_tests' %
304                        test_options.test_arguments))
305  else:
306    tests = _FilterDisabledTests(tests, test_options.suite_name,
307                                 bool(test_options.gtest_filter))
308  if test_options.gtest_filter:
309    tests = unittest_util.FilterTestNames(tests, test_options.gtest_filter)
310
311  # Coalesce unit tests into a single test per device
312  if test_options.suite_name != 'content_browsertests':
313    num_devices = len(devices)
314    tests = [':'.join(tests[i::num_devices]) for i in xrange(num_devices)]
315    tests = [t for t in tests if t]
316
317  return (TestRunnerFactory, tests)
318