# Copyright 2014 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. import json import logging import os import re import tempfile import threading import xml.etree.ElementTree import six from devil.android import apk_helper from pylib import constants from pylib.constants import host_paths from pylib.base import base_test_result from pylib.base import test_instance from pylib.symbols import stack_symbolizer from pylib.utils import test_filter with host_paths.SysPath(host_paths.BUILD_COMMON_PATH): import unittest_util # pylint: disable=import-error BROWSER_TEST_SUITES = [ 'android_browsertests', 'android_sync_integration_tests', 'components_browsertests', 'content_browsertests', 'weblayer_browsertests', ] # The max number of tests to run on a shard during the test run. MAX_SHARDS = 256 RUN_IN_SUB_THREAD_TEST_SUITES = [ # Multiprocess tests should be run outside of the main thread. 'base_unittests', # file_locking_unittest.cc uses a child process. 'gwp_asan_unittests', 'ipc_perftests', 'ipc_tests', 'mojo_perftests', 'mojo_unittests', 'net_unittests' ] # Used for filtering large data deps at a finer grain than what's allowed in # isolate files since pushing deps to devices is expensive. # Wildcards are allowed. _DEPS_EXCLUSION_LIST = [ 'chrome/test/data/extensions/api_test', 'chrome/test/data/extensions/secure_shell', 'chrome/test/data/firefox*', 'chrome/test/data/gpu', 'chrome/test/data/image_decoding', 'chrome/test/data/import', 'chrome/test/data/page_cycler', 'chrome/test/data/perf', 'chrome/test/data/pyauto_private', 'chrome/test/data/safari_import', 'chrome/test/data/scroll', 'chrome/test/data/third_party', 'third_party/hunspell_dictionaries/*.dic', # crbug.com/258690 'webkit/data/bmp_decoder', 'webkit/data/ico_decoder', ] _EXTRA_NATIVE_TEST_ACTIVITY = ( 'org.chromium.native_test.NativeTestInstrumentationTestRunner.' 'NativeTestActivity') _EXTRA_RUN_IN_SUB_THREAD = ( 'org.chromium.native_test.NativeTest.RunInSubThread') EXTRA_SHARD_NANO_TIMEOUT = ( 'org.chromium.native_test.NativeTestInstrumentationTestRunner.' 'ShardNanoTimeout') _EXTRA_SHARD_SIZE_LIMIT = ( 'org.chromium.native_test.NativeTestInstrumentationTestRunner.' 'ShardSizeLimit') # TODO(jbudorick): Remove these once we're no longer parsing stdout to generate # results. _RE_TEST_STATUS = re.compile( # Test state. r'\[ +((?:RUN)|(?:FAILED)|(?:OK)|(?:CRASHED)|(?:SKIPPED)) +\] ?' # Test name. r'([^ ]+)?' # Optional parameters. r'(?:, where' # Type parameter r'(?: TypeParam = [^()]*(?: and)?)?' # Value parameter r'(?: GetParam\(\) = [^()]*)?' # End of optional parameters. ')?' # Optional test execution time. r'(?: \((\d+) ms\))?$') # Crash detection constants. _RE_TEST_ERROR = re.compile(r'FAILURES!!! Tests run: \d+,' r' Failures: \d+, Errors: 1') _RE_TEST_CURRENTLY_RUNNING = re.compile( r'\[ERROR:.*?\] Currently running: (.*)') _RE_TEST_DCHECK_FATAL = re.compile(r'\[.*:FATAL:.*\] (.*)') _RE_DISABLED = re.compile(r'DISABLED_') _RE_FLAKY = re.compile(r'FLAKY_') # Detect stack line in stdout. _STACK_LINE_RE = re.compile(r'\s*#\d+') def ParseGTestListTests(raw_list): """Parses a raw test list as provided by --gtest_list_tests. Args: raw_list: The raw test listing with the following format: IPCChannelTest. SendMessageInChannelConnected IPCSyncChannelTest. Simple DISABLED_SendWithTimeoutMixedOKAndTimeout Returns: A list of all tests. For the above raw listing: [IPCChannelTest.SendMessageInChannelConnected, IPCSyncChannelTest.Simple, IPCSyncChannelTest.DISABLED_SendWithTimeoutMixedOKAndTimeout] """ ret = [] current = '' for test in raw_list: if not test: continue if not test.startswith(' '): test_case = test.split()[0] if test_case.endswith('.'): current = test_case else: test = test.strip() if test and not 'YOU HAVE' in test: test_name = test.split()[0] ret += [current + test_name] return ret def ParseGTestOutput(output, symbolizer, device_abi): """Parses raw gtest output and returns a list of results. Args: output: A list of output lines. symbolizer: The symbolizer used to symbolize stack. device_abi: Device abi that is needed for symbolization. Returns: A list of base_test_result.BaseTestResults. """ duration = 0 fallback_result_type = None log = [] stack = [] result_type = None results = [] test_name = None def symbolize_stack_and_merge_with_log(): log_string = '\n'.join(log or []) if not stack: stack_string = '' else: stack_string = '\n'.join( symbolizer.ExtractAndResolveNativeStackTraces( stack, device_abi)) return '%s\n%s' % (log_string, stack_string) def handle_possibly_unknown_test(): if test_name is not None: results.append( base_test_result.BaseTestResult( TestNameWithoutDisabledPrefix(test_name), # If we get here, that means we started a test, but it did not # produce a definitive test status output, so assume it crashed. # crbug/1191716 fallback_result_type or base_test_result.ResultType.CRASH, duration, log=symbolize_stack_and_merge_with_log())) for l in output: matcher = _RE_TEST_STATUS.match(l) if matcher: if matcher.group(1) == 'RUN': handle_possibly_unknown_test() duration = 0 fallback_result_type = None log = [] stack = [] result_type = None elif matcher.group(1) == 'OK': result_type = base_test_result.ResultType.PASS elif matcher.group(1) == 'SKIPPED': result_type = base_test_result.ResultType.SKIP elif matcher.group(1) == 'FAILED': result_type = base_test_result.ResultType.FAIL elif matcher.group(1) == 'CRASHED': fallback_result_type = base_test_result.ResultType.CRASH # Be aware that test name and status might not appear on same line. test_name = matcher.group(2) if matcher.group(2) else test_name duration = int(matcher.group(3)) if matcher.group(3) else 0 else: # Can possibly add more matchers, such as different results from DCHECK. currently_running_matcher = _RE_TEST_CURRENTLY_RUNNING.match(l) dcheck_matcher = _RE_TEST_DCHECK_FATAL.match(l) if currently_running_matcher: test_name = currently_running_matcher.group(1) result_type = base_test_result.ResultType.CRASH duration = None # Don't know. Not using 0 as this is unknown vs 0. elif dcheck_matcher: result_type = base_test_result.ResultType.CRASH duration = None # Don't know. Not using 0 as this is unknown vs 0. if log is not None: if not matcher and _STACK_LINE_RE.match(l): stack.append(l) else: log.append(l) if result_type and test_name: # Don't bother symbolizing output if the test passed. if result_type == base_test_result.ResultType.PASS: stack = [] results.append(base_test_result.BaseTestResult( TestNameWithoutDisabledPrefix(test_name), result_type, duration, log=symbolize_stack_and_merge_with_log())) test_name = None handle_possibly_unknown_test() return results def ParseGTestXML(xml_content): """Parse gtest XML result.""" results = [] if not xml_content: return results html = six.moves.html_parser.HTMLParser() testsuites = xml.etree.ElementTree.fromstring(xml_content) for testsuite in testsuites: suite_name = testsuite.attrib['name'] for testcase in testsuite: case_name = testcase.attrib['name'] result_type = base_test_result.ResultType.PASS log = [] for failure in testcase: result_type = base_test_result.ResultType.FAIL log.append(html.unescape(failure.attrib['message'])) results.append(base_test_result.BaseTestResult( '%s.%s' % (suite_name, TestNameWithoutDisabledPrefix(case_name)), result_type, int(float(testcase.attrib['time']) * 1000), log=('\n'.join(log) if log else ''))) return results def ParseGTestJSON(json_content): """Parse results in the JSON Test Results format.""" results = [] if not json_content: return results json_data = json.loads(json_content) openstack = list(json_data['tests'].items()) while openstack: name, value = openstack.pop() if 'expected' in value and 'actual' in value: if value['actual'] == 'PASS': result_type = base_test_result.ResultType.PASS elif value['actual'] == 'SKIP': result_type = base_test_result.ResultType.SKIP elif value['actual'] == 'CRASH': result_type = base_test_result.ResultType.CRASH elif value['actual'] == 'TIMEOUT': result_type = base_test_result.ResultType.TIMEOUT else: result_type = base_test_result.ResultType.FAIL results.append(base_test_result.BaseTestResult(name, result_type)) else: openstack += [("%s.%s" % (name, k), v) for k, v in six.iteritems(value)] return results def TestNameWithoutDisabledPrefix(test_name): """Modify the test name without disabled prefix if prefix 'DISABLED_' or 'FLAKY_' presents. Args: test_name: The name of a test. Returns: A test name without prefix 'DISABLED_' or 'FLAKY_'. """ disabled_prefixes = [_RE_DISABLED, _RE_FLAKY] for dp in disabled_prefixes: test_name = dp.sub('', test_name) return test_name class GtestTestInstance(test_instance.TestInstance): def __init__(self, args, data_deps_delegate, error_func): super().__init__() # TODO(jbudorick): Support multiple test suites. if len(args.suite_name) > 1: raise ValueError('Platform mode currently supports only 1 gtest suite') self._coverage_dir = args.coverage_dir self._exe_dist_dir = None self._external_shard_index = args.test_launcher_shard_index self._extract_test_list_from_filter = args.extract_test_list_from_filter self._filter_tests_lock = threading.Lock() self._gs_test_artifacts_bucket = args.gs_test_artifacts_bucket self._isolated_script_test_output = args.isolated_script_test_output self._isolated_script_test_perf_output = ( args.isolated_script_test_perf_output) self._render_test_output_dir = args.render_test_output_dir self._shard_timeout = args.shard_timeout self._store_tombstones = args.store_tombstones self._suite = args.suite_name[0] self._symbolizer = stack_symbolizer.Symbolizer(None) self._total_external_shards = args.test_launcher_total_shards self._wait_for_java_debugger = args.wait_for_java_debugger self._use_existing_test_data = args.use_existing_test_data # GYP: if args.executable_dist_dir: self._exe_dist_dir = os.path.abspath(args.executable_dist_dir) else: # TODO(agrieve): Remove auto-detection once recipes pass flag explicitly. exe_dist_dir = os.path.join(constants.GetOutDirectory(), '%s__dist' % self._suite) if os.path.exists(exe_dist_dir): self._exe_dist_dir = exe_dist_dir incremental_part = '' if args.test_apk_incremental_install_json: incremental_part = '_incremental' self._test_launcher_batch_limit = MAX_SHARDS if (args.test_launcher_batch_limit and 0 < args.test_launcher_batch_limit < MAX_SHARDS): self._test_launcher_batch_limit = args.test_launcher_batch_limit apk_path = os.path.join( constants.GetOutDirectory(), '%s_apk' % self._suite, '%s-debug%s.apk' % (self._suite, incremental_part)) self._test_apk_incremental_install_json = ( args.test_apk_incremental_install_json) if not os.path.exists(apk_path): self._apk_helper = None else: self._apk_helper = apk_helper.ApkHelper(apk_path) self._extras = { _EXTRA_NATIVE_TEST_ACTIVITY: self._apk_helper.GetActivityName(), } if self._suite in RUN_IN_SUB_THREAD_TEST_SUITES: self._extras[_EXTRA_RUN_IN_SUB_THREAD] = 1 if self._suite in BROWSER_TEST_SUITES: self._extras[_EXTRA_SHARD_SIZE_LIMIT] = 1 self._extras[EXTRA_SHARD_NANO_TIMEOUT] = int(1e9 * self._shard_timeout) self._shard_timeout = 10 * self._shard_timeout if args.wait_for_java_debugger: self._extras[EXTRA_SHARD_NANO_TIMEOUT] = int(1e15) # Forever if not self._apk_helper and not self._exe_dist_dir: error_func('Could not find apk or executable for %s' % self._suite) self._data_deps = [] self._gtest_filters = test_filter.InitializeFiltersFromArgs(args) self._run_disabled = args.run_disabled self._data_deps_delegate = data_deps_delegate self._runtime_deps_path = args.runtime_deps_path if not self._runtime_deps_path: logging.warning('No data dependencies will be pushed.') if args.app_data_files: self._app_data_files = args.app_data_files if args.app_data_file_dir: self._app_data_file_dir = args.app_data_file_dir else: self._app_data_file_dir = tempfile.mkdtemp() logging.critical('Saving app files to %s', self._app_data_file_dir) else: self._app_data_files = None self._app_data_file_dir = None self._flags = None self._initializeCommandLineFlags(args) # TODO(jbudorick): Remove this once it's deployed. self._enable_xml_result_parsing = args.enable_xml_result_parsing def _initializeCommandLineFlags(self, args): self._flags = [] if args.command_line_flags: self._flags.extend(args.command_line_flags) if args.device_flags_file: with open(args.device_flags_file) as f: stripped_lines = (l.strip() for l in f) self._flags.extend(flag for flag in stripped_lines if flag) if args.run_disabled: self._flags.append('--gtest_also_run_disabled_tests') @property def activity(self): return self._apk_helper and self._apk_helper.GetActivityName() @property def apk(self): return self._apk_helper and self._apk_helper.path @property def apk_helper(self): return self._apk_helper @property def app_file_dir(self): return self._app_data_file_dir @property def app_files(self): return self._app_data_files @property def coverage_dir(self): return self._coverage_dir @property def enable_xml_result_parsing(self): return self._enable_xml_result_parsing @property def exe_dist_dir(self): return self._exe_dist_dir @property def external_shard_index(self): return self._external_shard_index @property def extract_test_list_from_filter(self): return self._extract_test_list_from_filter @property def extras(self): return self._extras @property def flags(self): return self._flags @property def gs_test_artifacts_bucket(self): return self._gs_test_artifacts_bucket @property def gtest_filters(self): return self._gtest_filters @property def isolated_script_test_output(self): return self._isolated_script_test_output @property def isolated_script_test_perf_output(self): return self._isolated_script_test_perf_output @property def render_test_output_dir(self): return self._render_test_output_dir @property def package(self): return self._apk_helper and self._apk_helper.GetPackageName() @property def permissions(self): return self._apk_helper and self._apk_helper.GetPermissions() @property def runner(self): return self._apk_helper and self._apk_helper.GetInstrumentationName() @property def shard_timeout(self): return self._shard_timeout @property def store_tombstones(self): return self._store_tombstones @property def suite(self): return self._suite @property def symbolizer(self): return self._symbolizer @property def test_apk_incremental_install_json(self): return self._test_apk_incremental_install_json @property def test_launcher_batch_limit(self): return self._test_launcher_batch_limit @property def total_external_shards(self): return self._total_external_shards @property def wait_for_java_debugger(self): return self._wait_for_java_debugger @property def use_existing_test_data(self): return self._use_existing_test_data #override def TestType(self): return 'gtest' #override def GetPreferredAbis(self): if not self._apk_helper: return None return self._apk_helper.GetAbis() #override def SetUp(self): """Map data dependencies via isolate.""" self._data_deps.extend( self._data_deps_delegate(self._runtime_deps_path)) def GetDataDependencies(self): """Returns the test suite's data dependencies. Returns: A list of (host_path, device_path) tuples to push. If device_path is None, the client is responsible for determining where to push the file. """ return self._data_deps def FilterTests(self, test_list, disabled_prefixes=None): """Filters |test_list| based on prefixes and, if present, a filter string. Args: test_list: The list of tests to filter. disabled_prefixes: A list of test prefixes to filter. Defaults to DISABLED_, FLAKY_, FAILS_, PRE_, and MANUAL_ Returns: A filtered list of tests to run. """ gtest_filter_strings = [ self._GenerateDisabledFilterString(disabled_prefixes)] if self._gtest_filters: gtest_filter_strings.extend(self._gtest_filters) filtered_test_list = test_list # This lock is required because on older versions of Python # |unittest_util.FilterTestNames| use of |fnmatch| is not threadsafe. with self._filter_tests_lock: for gtest_filter_string in gtest_filter_strings: logging.debug('Filtering tests using: %s', gtest_filter_string) filtered_test_list = unittest_util.FilterTestNames( filtered_test_list, gtest_filter_string) if self._run_disabled and self._gtest_filters: out_filtered_test_list = list(set(test_list)-set(filtered_test_list)) for test in out_filtered_test_list: test_name_no_disabled = TestNameWithoutDisabledPrefix(test) if test_name_no_disabled == test: continue if all( unittest_util.FilterTestNames([test_name_no_disabled], gtest_filter) for gtest_filter in self._gtest_filters): filtered_test_list.append(test) return filtered_test_list def _GenerateDisabledFilterString(self, disabled_prefixes): disabled_filter_items = [] if disabled_prefixes is None: disabled_prefixes = ['FAILS_', 'PRE_'] if '--run-manual' not in self._flags: disabled_prefixes += ['MANUAL_'] if not self._run_disabled: disabled_prefixes += ['DISABLED_', 'FLAKY_'] disabled_filter_items += ['%s*' % dp for dp in disabled_prefixes] disabled_filter_items += ['*.%s*' % dp for dp in disabled_prefixes] disabled_tests_file_path = os.path.join( host_paths.DIR_SOURCE_ROOT, 'build', 'android', 'pylib', 'gtest', 'filter', '%s_disabled' % self._suite) if disabled_tests_file_path and os.path.exists(disabled_tests_file_path): with open(disabled_tests_file_path) as disabled_tests_file: disabled_filter_items += [ '%s' % l for l in (line.strip() for line in disabled_tests_file) if l and not l.startswith('#')] return '*-%s' % ':'.join(disabled_filter_items) #override def TearDown(self): """Do nothing."""