1# Copyright 2014 The Chromium Authors. All rights reserved. 2# Use of this source code is governed by a BSD-style license that can be 3# found in the LICENSE file. 4 5import logging 6import os 7import re 8import tempfile 9 10from devil.android import apk_helper 11from pylib import constants 12from pylib.constants import host_paths 13from pylib.base import base_test_result 14from pylib.base import test_instance 15from pylib.utils import isolator 16 17with host_paths.SysPath(host_paths.BUILD_COMMON_PATH): 18 import unittest_util # pylint: disable=import-error 19 20 21BROWSER_TEST_SUITES = [ 22 'components_browsertests', 23 'content_browsertests', 24] 25 26RUN_IN_SUB_THREAD_TEST_SUITES = ['net_unittests'] 27 28 29_DEFAULT_ISOLATE_FILE_PATHS = { 30 'base_unittests': 'base/base_unittests.isolate', 31 'blink_heap_unittests': 32 'third_party/WebKit/Source/platform/heap/BlinkHeapUnitTests.isolate', 33 'blink_platform_unittests': 34 'third_party/WebKit/Source/platform/blink_platform_unittests.isolate', 35 'cc_perftests': 'cc/cc_perftests.isolate', 36 'components_browsertests': 'components/components_browsertests.isolate', 37 'components_unittests': 'components/components_unittests.isolate', 38 'content_browsertests': 'content/content_browsertests.isolate', 39 'content_unittests': 'content/content_unittests.isolate', 40 'media_perftests': 'media/media_perftests.isolate', 41 'media_unittests': 'media/media_unittests.isolate', 42 'midi_unittests': 'media/midi/midi_unittests.isolate', 43 'net_unittests': 'net/net_unittests.isolate', 44 'sql_unittests': 'sql/sql_unittests.isolate', 45 'sync_unit_tests': 'sync/sync_unit_tests.isolate', 46 'ui_base_unittests': 'ui/base/ui_base_tests.isolate', 47 'unit_tests': 'chrome/unit_tests.isolate', 48 'webkit_unit_tests': 49 'third_party/WebKit/Source/web/WebKitUnitTests.isolate', 50} 51 52 53# Used for filtering large data deps at a finer grain than what's allowed in 54# isolate files since pushing deps to devices is expensive. 55# Wildcards are allowed. 56_DEPS_EXCLUSION_LIST = [ 57 'chrome/test/data/extensions/api_test', 58 'chrome/test/data/extensions/secure_shell', 59 'chrome/test/data/firefox*', 60 'chrome/test/data/gpu', 61 'chrome/test/data/image_decoding', 62 'chrome/test/data/import', 63 'chrome/test/data/page_cycler', 64 'chrome/test/data/perf', 65 'chrome/test/data/pyauto_private', 66 'chrome/test/data/safari_import', 67 'chrome/test/data/scroll', 68 'chrome/test/data/third_party', 69 'third_party/hunspell_dictionaries/*.dic', 70 # crbug.com/258690 71 'webkit/data/bmp_decoder', 72 'webkit/data/ico_decoder', 73] 74 75 76_EXTRA_NATIVE_TEST_ACTIVITY = ( 77 'org.chromium.native_test.NativeTestInstrumentationTestRunner.' 78 'NativeTestActivity') 79_EXTRA_RUN_IN_SUB_THREAD = ( 80 'org.chromium.native_test.NativeTestActivity.RunInSubThread') 81EXTRA_SHARD_NANO_TIMEOUT = ( 82 'org.chromium.native_test.NativeTestInstrumentationTestRunner.' 83 'ShardNanoTimeout') 84_EXTRA_SHARD_SIZE_LIMIT = ( 85 'org.chromium.native_test.NativeTestInstrumentationTestRunner.' 86 'ShardSizeLimit') 87 88# TODO(jbudorick): Remove these once we're no longer parsing stdout to generate 89# results. 90_RE_TEST_STATUS = re.compile( 91 r'\[ +((?:RUN)|(?:FAILED)|(?:OK)|(?:CRASHED)) +\]' 92 r' ?([^ ]+)?(?: \((\d+) ms\))?$') 93_RE_TEST_RUN_STATUS = re.compile( 94 r'\[ +(PASSED|RUNNER_FAILED|CRASHED) \] ?[^ ]+') 95# Crash detection constants. 96_RE_TEST_ERROR = re.compile(r'FAILURES!!! Tests run: \d+,' 97 r' Failures: \d+, Errors: 1') 98_RE_TEST_CURRENTLY_RUNNING = re.compile(r'\[ERROR:.*?\]' 99 r' Currently running: (.*)') 100 101# TODO(jbudorick): Make this a class method of GtestTestInstance once 102# test_package_apk and test_package_exe are gone. 103def ParseGTestListTests(raw_list): 104 """Parses a raw test list as provided by --gtest_list_tests. 105 106 Args: 107 raw_list: The raw test listing with the following format: 108 109 IPCChannelTest. 110 SendMessageInChannelConnected 111 IPCSyncChannelTest. 112 Simple 113 DISABLED_SendWithTimeoutMixedOKAndTimeout 114 115 Returns: 116 A list of all tests. For the above raw listing: 117 118 [IPCChannelTest.SendMessageInChannelConnected, IPCSyncChannelTest.Simple, 119 IPCSyncChannelTest.DISABLED_SendWithTimeoutMixedOKAndTimeout] 120 """ 121 ret = [] 122 current = '' 123 for test in raw_list: 124 if not test: 125 continue 126 if test[0] != ' ': 127 test_case = test.split()[0] 128 if test_case.endswith('.'): 129 current = test_case 130 elif not 'YOU HAVE' in test: 131 test_name = test.split()[0] 132 ret += [current + test_name] 133 return ret 134 135 136class GtestTestInstance(test_instance.TestInstance): 137 138 def __init__(self, args, isolate_delegate, error_func): 139 super(GtestTestInstance, self).__init__() 140 # TODO(jbudorick): Support multiple test suites. 141 if len(args.suite_name) > 1: 142 raise ValueError('Platform mode currently supports only 1 gtest suite') 143 self._extract_test_list_from_filter = args.extract_test_list_from_filter 144 self._shard_timeout = args.shard_timeout 145 self._suite = args.suite_name[0] 146 self._exe_dist_dir = None 147 148 # GYP: 149 if args.executable_dist_dir: 150 self._exe_dist_dir = os.path.abspath(args.executable_dist_dir) 151 else: 152 # TODO(agrieve): Remove auto-detection once recipes pass flag explicitly. 153 exe_dist_dir = os.path.join(constants.GetOutDirectory(), 154 '%s__dist' % self._suite) 155 156 if os.path.exists(exe_dist_dir): 157 self._exe_dist_dir = exe_dist_dir 158 159 incremental_part = '' 160 if args.test_apk_incremental_install_script: 161 incremental_part = '_incremental' 162 163 apk_path = os.path.join( 164 constants.GetOutDirectory(), '%s_apk' % self._suite, 165 '%s-debug%s.apk' % (self._suite, incremental_part)) 166 self._test_apk_incremental_install_script = ( 167 args.test_apk_incremental_install_script) 168 if not os.path.exists(apk_path): 169 self._apk_helper = None 170 else: 171 self._apk_helper = apk_helper.ApkHelper(apk_path) 172 self._extras = { 173 _EXTRA_NATIVE_TEST_ACTIVITY: self._apk_helper.GetActivityName(), 174 } 175 if self._suite in RUN_IN_SUB_THREAD_TEST_SUITES: 176 self._extras[_EXTRA_RUN_IN_SUB_THREAD] = 1 177 if self._suite in BROWSER_TEST_SUITES: 178 self._extras[_EXTRA_SHARD_SIZE_LIMIT] = 1 179 self._extras[EXTRA_SHARD_NANO_TIMEOUT] = int(1e9 * self._shard_timeout) 180 self._shard_timeout = 900 181 182 if not self._apk_helper and not self._exe_dist_dir: 183 error_func('Could not find apk or executable for %s' % self._suite) 184 185 self._data_deps = [] 186 if args.test_filter: 187 self._gtest_filter = args.test_filter 188 elif args.test_filter_file: 189 with open(args.test_filter_file, 'r') as f: 190 self._gtest_filter = ':'.join(l.strip() for l in f) 191 else: 192 self._gtest_filter = None 193 194 if not args.isolate_file_path: 195 default_isolate_file_path = _DEFAULT_ISOLATE_FILE_PATHS.get(self._suite) 196 if default_isolate_file_path: 197 args.isolate_file_path = os.path.join( 198 host_paths.DIR_SOURCE_ROOT, default_isolate_file_path) 199 200 if (args.isolate_file_path and 201 not isolator.IsIsolateEmpty(args.isolate_file_path)): 202 self._isolate_abs_path = os.path.abspath(args.isolate_file_path) 203 self._isolate_delegate = isolate_delegate 204 self._isolated_abs_path = os.path.join( 205 constants.GetOutDirectory(), '%s.isolated' % self._suite) 206 else: 207 logging.warning('No isolate file provided. No data deps will be pushed.') 208 self._isolate_delegate = None 209 210 if args.app_data_files: 211 self._app_data_files = args.app_data_files 212 if args.app_data_file_dir: 213 self._app_data_file_dir = args.app_data_file_dir 214 else: 215 self._app_data_file_dir = tempfile.mkdtemp() 216 logging.critical('Saving app files to %s', self._app_data_file_dir) 217 else: 218 self._app_data_files = None 219 self._app_data_file_dir = None 220 221 self._test_arguments = args.test_arguments 222 223 @property 224 def activity(self): 225 return self._apk_helper and self._apk_helper.GetActivityName() 226 227 @property 228 def apk(self): 229 return self._apk_helper and self._apk_helper.path 230 231 @property 232 def apk_helper(self): 233 return self._apk_helper 234 235 @property 236 def app_file_dir(self): 237 return self._app_data_file_dir 238 239 @property 240 def app_files(self): 241 return self._app_data_files 242 243 @property 244 def exe_dist_dir(self): 245 return self._exe_dist_dir 246 247 @property 248 def extras(self): 249 return self._extras 250 251 @property 252 def gtest_filter(self): 253 return self._gtest_filter 254 255 @property 256 def package(self): 257 return self._apk_helper and self._apk_helper.GetPackageName() 258 259 @property 260 def permissions(self): 261 return self._apk_helper and self._apk_helper.GetPermissions() 262 263 @property 264 def runner(self): 265 return self._apk_helper and self._apk_helper.GetInstrumentationName() 266 267 @property 268 def shard_timeout(self): 269 return self._shard_timeout 270 271 @property 272 def suite(self): 273 return self._suite 274 275 @property 276 def test_apk_incremental_install_script(self): 277 return self._test_apk_incremental_install_script 278 279 @property 280 def test_arguments(self): 281 return self._test_arguments 282 283 @property 284 def extract_test_list_from_filter(self): 285 return self._extract_test_list_from_filter 286 287 #override 288 def TestType(self): 289 return 'gtest' 290 291 #override 292 def SetUp(self): 293 """Map data dependencies via isolate.""" 294 if self._isolate_delegate: 295 self._isolate_delegate.Remap( 296 self._isolate_abs_path, self._isolated_abs_path) 297 self._isolate_delegate.PurgeExcluded(_DEPS_EXCLUSION_LIST) 298 self._isolate_delegate.MoveOutputDeps() 299 dest_dir = None 300 self._data_deps.extend([ 301 (self._isolate_delegate.isolate_deps_dir, dest_dir)]) 302 303 304 def GetDataDependencies(self): 305 """Returns the test suite's data dependencies. 306 307 Returns: 308 A list of (host_path, device_path) tuples to push. If device_path is 309 None, the client is responsible for determining where to push the file. 310 """ 311 return self._data_deps 312 313 def FilterTests(self, test_list, disabled_prefixes=None): 314 """Filters |test_list| based on prefixes and, if present, a filter string. 315 316 Args: 317 test_list: The list of tests to filter. 318 disabled_prefixes: A list of test prefixes to filter. Defaults to 319 DISABLED_, FLAKY_, FAILS_, PRE_, and MANUAL_ 320 Returns: 321 A filtered list of tests to run. 322 """ 323 gtest_filter_strings = [ 324 self._GenerateDisabledFilterString(disabled_prefixes)] 325 if self._gtest_filter: 326 gtest_filter_strings.append(self._gtest_filter) 327 328 filtered_test_list = test_list 329 for gtest_filter_string in gtest_filter_strings: 330 logging.debug('Filtering tests using: %s', gtest_filter_string) 331 filtered_test_list = unittest_util.FilterTestNames( 332 filtered_test_list, gtest_filter_string) 333 return filtered_test_list 334 335 def _GenerateDisabledFilterString(self, disabled_prefixes): 336 disabled_filter_items = [] 337 338 if disabled_prefixes is None: 339 disabled_prefixes = ['DISABLED_', 'FLAKY_', 'FAILS_', 'PRE_', 'MANUAL_'] 340 disabled_filter_items += ['%s*' % dp for dp in disabled_prefixes] 341 disabled_filter_items += ['*.%s*' % dp for dp in disabled_prefixes] 342 343 disabled_tests_file_path = os.path.join( 344 host_paths.DIR_SOURCE_ROOT, 'build', 'android', 'pylib', 'gtest', 345 'filter', '%s_disabled' % self._suite) 346 if disabled_tests_file_path and os.path.exists(disabled_tests_file_path): 347 with open(disabled_tests_file_path) as disabled_tests_file: 348 disabled_filter_items += [ 349 '%s' % l for l in (line.strip() for line in disabled_tests_file) 350 if l and not l.startswith('#')] 351 352 return '*-%s' % ':'.join(disabled_filter_items) 353 354 # pylint: disable=no-self-use 355 def ParseGTestOutput(self, output): 356 """Parses raw gtest output and returns a list of results. 357 358 Args: 359 output: A list of output lines. 360 Returns: 361 A list of base_test_result.BaseTestResults. 362 """ 363 log = [] 364 result_type = None 365 results = [] 366 test_name = None 367 for l in output: 368 logging.info(l) 369 matcher = _RE_TEST_STATUS.match(l) 370 if matcher: 371 # Be aware that test name and status might not appear on same line. 372 test_name = matcher.group(2) if matcher.group(2) else test_name 373 duration = int(matcher.group(3)) if matcher.group(3) else 0 374 if matcher.group(1) == 'RUN': 375 log = [] 376 elif matcher.group(1) == 'OK': 377 result_type = base_test_result.ResultType.PASS 378 elif matcher.group(1) == 'FAILED': 379 result_type = base_test_result.ResultType.FAIL 380 elif matcher.group(1) == 'CRASHED': 381 result_type = base_test_result.ResultType.CRASH 382 383 # Needs another matcher here to match crashes, like those of DCHECK. 384 matcher = _RE_TEST_CURRENTLY_RUNNING.match(l) 385 if matcher: 386 test_name = matcher.group(1) 387 result_type = base_test_result.ResultType.CRASH 388 duration = 0 # Don't know. 389 390 if log is not None: 391 log.append(l) 392 393 if result_type: 394 results.append(base_test_result.BaseTestResult( 395 test_name, result_type, duration, 396 log=('\n'.join(log) if log else ''))) 397 log = None 398 result_type = None 399 400 return results 401 402 #override 403 def TearDown(self): 404 """Clear the mappings created by SetUp.""" 405 if self._isolate_delegate: 406 self._isolate_delegate.Clear() 407 408