• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1# Copyright 2014 The Chromium Authors
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5
6
7import html.parser
8import json
9import logging
10import os
11import re
12import tempfile
13import threading
14import xml.etree.ElementTree
15
16from devil.android import apk_helper
17from pylib import constants
18from pylib.constants import host_paths
19from pylib.base import base_test_result
20from pylib.base import test_instance
21from pylib.symbols import stack_symbolizer
22from pylib.utils import test_filter
23
24
25with host_paths.SysPath(host_paths.BUILD_COMMON_PATH):
26  import unittest_util # pylint: disable=import-error
27
28
29BROWSER_TEST_SUITES = [
30    'android_browsertests',
31    'android_sync_integration_tests',
32    'components_browsertests',
33    'content_browsertests',
34    'weblayer_browsertests',
35]
36
37# The max number of tests to run on a shard during the test run.
38MAX_SHARDS = 256
39
40RUN_IN_SUB_THREAD_TEST_SUITES = [
41    # Multiprocess tests should be run outside of the main thread.
42    'base_unittests',  # file_locking_unittest.cc uses a child process.
43    'gwp_asan_unittests',
44    'ipc_perftests',
45    'ipc_tests',
46    'mojo_perftests',
47    'mojo_unittests',
48    'net_unittests'
49]
50
51
52# Used for filtering large data deps at a finer grain than what's allowed in
53# isolate files since pushing deps to devices is expensive.
54# Wildcards are allowed.
55_DEPS_EXCLUSION_LIST = [
56    'chrome/test/data/extensions/api_test',
57    'chrome/test/data/extensions/secure_shell',
58    'chrome/test/data/firefox*',
59    'chrome/test/data/gpu',
60    'chrome/test/data/image_decoding',
61    'chrome/test/data/import',
62    'chrome/test/data/page_cycler',
63    'chrome/test/data/perf',
64    'chrome/test/data/pyauto_private',
65    'chrome/test/data/safari_import',
66    'chrome/test/data/scroll',
67    'chrome/test/data/third_party',
68    'third_party/hunspell_dictionaries/*.dic',
69    # crbug.com/258690
70    'webkit/data/bmp_decoder',
71    'webkit/data/ico_decoder',
72]
73
74
75_EXTRA_NATIVE_TEST_ACTIVITY = (
76    'org.chromium.native_test.NativeTestInstrumentationTestRunner.'
77        'NativeTestActivity')
78_EXTRA_RUN_IN_SUB_THREAD = (
79    'org.chromium.native_test.NativeTest.RunInSubThread')
80EXTRA_SHARD_NANO_TIMEOUT = (
81    'org.chromium.native_test.NativeTestInstrumentationTestRunner.'
82        'ShardNanoTimeout')
83_EXTRA_SHARD_SIZE_LIMIT = (
84    'org.chromium.native_test.NativeTestInstrumentationTestRunner.'
85        'ShardSizeLimit')
86
87# TODO(jbudorick): Remove these once we're no longer parsing stdout to generate
88# results.
89_RE_TEST_STATUS = re.compile(
90    # Test state.
91    r'\[ +((?:RUN)|(?:FAILED)|(?:OK)|(?:CRASHED)|(?:SKIPPED)) +\] ?'
92    # Test name.
93    r'([^ ]+)?'
94    # Optional parameters.
95    r'(?:, where'
96    #   Type parameter
97    r'(?: TypeParam = [^()]*(?: and)?)?'
98    #   Value parameter
99    r'(?: GetParam\(\) = [^()]*)?'
100    # End of optional parameters.
101    ')?'
102    # Optional test execution time.
103    r'(?: \((\d+) ms\))?$')
104# Crash detection constants.
105_RE_TEST_ERROR = re.compile(r'FAILURES!!! Tests run: \d+,'
106                                    r' Failures: \d+, Errors: 1')
107_RE_TEST_CURRENTLY_RUNNING = re.compile(
108    r'\[.*ERROR:.*?\] Currently running: (.*)')
109_RE_TEST_DCHECK_FATAL = re.compile(r'\[.*:FATAL:.*\] (.*)')
110_RE_DISABLED = re.compile(r'DISABLED_')
111_RE_FLAKY = re.compile(r'FLAKY_')
112
113# Regex that matches the printout when there are test failures.
114# matches "[  FAILED  ] 1 test, listed below:"
115_RE_ANY_TESTS_FAILED = re.compile(r'\[ +FAILED +\].*listed below')
116
117# Detect stack line in stdout.
118_STACK_LINE_RE = re.compile(r'\s*#\d+')
119
120def ParseGTestListTests(raw_list):
121  """Parses a raw test list as provided by --gtest_list_tests.
122
123  Args:
124    raw_list: The raw test listing with the following format:
125
126    IPCChannelTest.
127      SendMessageInChannelConnected
128    IPCSyncChannelTest.
129      Simple
130      DISABLED_SendWithTimeoutMixedOKAndTimeout
131
132  Returns:
133    A list of all tests. For the above raw listing:
134
135    [IPCChannelTest.SendMessageInChannelConnected, IPCSyncChannelTest.Simple,
136     IPCSyncChannelTest.DISABLED_SendWithTimeoutMixedOKAndTimeout]
137  """
138  ret = []
139  current = ''
140  for test in raw_list:
141    if not test:
142      continue
143    if not test.startswith(' '):
144      test_case = test.split()[0]
145      if test_case.endswith('.'):
146        current = test_case
147    else:
148      test = test.strip()
149      if test and not 'YOU HAVE' in test:
150        test_name = test.split()[0]
151        ret += [current + test_name]
152  return ret
153
154
155def ParseGTestOutput(output, symbolizer, device_abi):
156  """Parses raw gtest output and returns a list of results.
157
158  Args:
159    output: A list of output lines.
160    symbolizer: The symbolizer used to symbolize stack.
161    device_abi: Device abi that is needed for symbolization.
162  Returns:
163    A list of base_test_result.BaseTestResults.
164  """
165  duration = 0
166  fallback_result_type = None
167  log = []
168  stack = []
169  result_type = None
170  results = []
171  test_name = None
172
173  def symbolize_stack_and_merge_with_log():
174    log_string = '\n'.join(log or [])
175    if not stack:
176      stack_string = ''
177    else:
178      stack_string = '\n'.join(
179          symbolizer.ExtractAndResolveNativeStackTraces(
180              stack, device_abi))
181    return '%s\n%s' % (log_string, stack_string)
182
183  def handle_possibly_unknown_test():
184    if test_name is not None:
185      results.append(
186          base_test_result.BaseTestResult(
187              TestNameWithoutDisabledPrefix(test_name),
188              # If we get here, that means we started a test, but it did not
189              # produce a definitive test status output, so assume it crashed.
190              # crbug/1191716
191              fallback_result_type or base_test_result.ResultType.CRASH,
192              duration,
193              log=symbolize_stack_and_merge_with_log()))
194
195  for l in output:
196    matcher = _RE_TEST_STATUS.match(l)
197    if matcher:
198      if matcher.group(1) == 'RUN':
199        handle_possibly_unknown_test()
200        duration = 0
201        fallback_result_type = None
202        log = []
203        stack = []
204        result_type = None
205      elif matcher.group(1) == 'OK':
206        result_type = base_test_result.ResultType.PASS
207      elif matcher.group(1) == 'SKIPPED':
208        result_type = base_test_result.ResultType.SKIP
209      elif matcher.group(1) == 'FAILED':
210        result_type = base_test_result.ResultType.FAIL
211      elif matcher.group(1) == 'CRASHED':
212        fallback_result_type = base_test_result.ResultType.CRASH
213      # Be aware that test name and status might not appear on same line.
214      test_name = matcher.group(2) if matcher.group(2) else test_name
215      duration = int(matcher.group(3)) if matcher.group(3) else 0
216
217    else:
218      # Can possibly add more matchers, such as different results from DCHECK.
219      currently_running_matcher = _RE_TEST_CURRENTLY_RUNNING.match(l)
220      dcheck_matcher = _RE_TEST_DCHECK_FATAL.match(l)
221
222      if currently_running_matcher:
223        test_name = currently_running_matcher.group(1)
224        result_type = base_test_result.ResultType.CRASH
225        duration = None  # Don't know. Not using 0 as this is unknown vs 0.
226      elif dcheck_matcher:
227        result_type = base_test_result.ResultType.CRASH
228        duration = None  # Don't know.  Not using 0 as this is unknown vs 0.
229
230    if log is not None:
231      if not matcher and _STACK_LINE_RE.match(l):
232        stack.append(l)
233      else:
234        log.append(l)
235
236    if _RE_ANY_TESTS_FAILED.match(l):
237      break
238
239    if result_type and test_name:
240      # Don't bother symbolizing output if the test passed.
241      if result_type == base_test_result.ResultType.PASS:
242        stack = []
243      results.append(base_test_result.BaseTestResult(
244          TestNameWithoutDisabledPrefix(test_name), result_type, duration,
245          log=symbolize_stack_and_merge_with_log()))
246      test_name = None
247
248  else:
249    # Executing this after tests have finished with a failure causes a
250    # duplicate test entry to be added to results. crbug/1380825
251    handle_possibly_unknown_test()
252
253  return results
254
255
256def ParseGTestXML(xml_content):
257  """Parse gtest XML result."""
258  results = []
259  if not xml_content:
260    return results
261
262  html_parser = html.parser.HTMLParser()
263
264  testsuites = xml.etree.ElementTree.fromstring(xml_content)
265  for testsuite in testsuites:
266    suite_name = testsuite.attrib['name']
267    for testcase in testsuite:
268      case_name = testcase.attrib['name']
269      result_type = base_test_result.ResultType.PASS
270      log = []
271      for failure in testcase:
272        result_type = base_test_result.ResultType.FAIL
273        log.append(html_parser.unescape(failure.attrib['message']))
274
275      results.append(base_test_result.BaseTestResult(
276          '%s.%s' % (suite_name, TestNameWithoutDisabledPrefix(case_name)),
277          result_type,
278          int(float(testcase.attrib['time']) * 1000),
279          log=('\n'.join(log) if log else '')))
280
281  return results
282
283
284def ParseGTestJSON(json_content):
285  """Parse results in the JSON Test Results format."""
286  results = []
287  if not json_content:
288    return results
289
290  json_data = json.loads(json_content)
291
292  openstack = list(json_data['tests'].items())
293
294  while openstack:
295    name, value = openstack.pop()
296
297    if 'expected' in value and 'actual' in value:
298      if value['actual'] == 'PASS':
299        result_type = base_test_result.ResultType.PASS
300      elif value['actual'] == 'SKIP':
301        result_type = base_test_result.ResultType.SKIP
302      elif value['actual'] == 'CRASH':
303        result_type = base_test_result.ResultType.CRASH
304      elif value['actual'] == 'TIMEOUT':
305        result_type = base_test_result.ResultType.TIMEOUT
306      else:
307        result_type = base_test_result.ResultType.FAIL
308      results.append(base_test_result.BaseTestResult(name, result_type))
309    else:
310      openstack += [("%s.%s" % (name, k), v) for k, v in value.items()]
311
312  return results
313
314
315def TestNameWithoutDisabledPrefix(test_name):
316  """Modify the test name without disabled prefix if prefix 'DISABLED_' or
317  'FLAKY_' presents.
318
319  Args:
320    test_name: The name of a test.
321  Returns:
322    A test name without prefix 'DISABLED_' or 'FLAKY_'.
323  """
324  disabled_prefixes = [_RE_DISABLED, _RE_FLAKY]
325  for dp in disabled_prefixes:
326    test_name = dp.sub('', test_name)
327  return test_name
328
329class GtestTestInstance(test_instance.TestInstance):
330
331  def __init__(self, args, data_deps_delegate, error_func):
332    super().__init__()
333    # TODO(jbudorick): Support multiple test suites.
334    if len(args.suite_name) > 1:
335      raise ValueError('Platform mode currently supports only 1 gtest suite')
336    self._additional_apks = []
337    self._coverage_dir = args.coverage_dir
338    self._exe_dist_dir = None
339    self._external_shard_index = args.test_launcher_shard_index
340    self._extract_test_list_from_filter = args.extract_test_list_from_filter
341    self._filter_tests_lock = threading.Lock()
342    self._gs_test_artifacts_bucket = args.gs_test_artifacts_bucket
343    self._isolated_script_test_output = args.isolated_script_test_output
344    self._isolated_script_test_perf_output = (
345        args.isolated_script_test_perf_output)
346    self._render_test_output_dir = args.render_test_output_dir
347    self._shard_timeout = args.shard_timeout
348    self._store_tombstones = args.store_tombstones
349    self._suite = args.suite_name[0]
350    self._symbolizer = stack_symbolizer.Symbolizer(None)
351    self._total_external_shards = args.test_launcher_total_shards
352    self._wait_for_java_debugger = args.wait_for_java_debugger
353    self._use_existing_test_data = args.use_existing_test_data
354
355    # GYP:
356    if args.executable_dist_dir:
357      self._exe_dist_dir = os.path.abspath(args.executable_dist_dir)
358    else:
359      # TODO(agrieve): Remove auto-detection once recipes pass flag explicitly.
360      exe_dist_dir = os.path.join(constants.GetOutDirectory(),
361                                  '%s__dist' % self._suite)
362
363      if os.path.exists(exe_dist_dir):
364        self._exe_dist_dir = exe_dist_dir
365
366    incremental_part = ''
367    if args.test_apk_incremental_install_json:
368      incremental_part = '_incremental'
369
370    self._test_launcher_batch_limit = MAX_SHARDS
371    if (args.test_launcher_batch_limit
372        and 0 < args.test_launcher_batch_limit < MAX_SHARDS):
373      self._test_launcher_batch_limit = args.test_launcher_batch_limit
374
375    apk_path = os.path.join(
376        constants.GetOutDirectory(), '%s_apk' % self._suite,
377        '%s-debug%s.apk' % (self._suite, incremental_part))
378    self._test_apk_incremental_install_json = (
379        args.test_apk_incremental_install_json)
380    if not os.path.exists(apk_path):
381      self._apk_helper = None
382    else:
383      self._apk_helper = apk_helper.ApkHelper(apk_path)
384      self._extras = {
385          _EXTRA_NATIVE_TEST_ACTIVITY: self._apk_helper.GetActivityName(),
386      }
387      if args.timeout_scale and args.timeout_scale != 1:
388        self._extras[_EXTRA_RUN_IN_SUB_THREAD] = 1
389
390      if self._suite in RUN_IN_SUB_THREAD_TEST_SUITES:
391        self._extras[_EXTRA_RUN_IN_SUB_THREAD] = 1
392      if self._suite in BROWSER_TEST_SUITES:
393        self._extras[_EXTRA_SHARD_SIZE_LIMIT] = 1
394        self._extras[EXTRA_SHARD_NANO_TIMEOUT] = int(1e9 * self._shard_timeout)
395        self._shard_timeout = 10 * self._shard_timeout
396      if args.wait_for_java_debugger:
397        self._extras[EXTRA_SHARD_NANO_TIMEOUT] = int(1e15)  # Forever
398
399    if not self._apk_helper and not self._exe_dist_dir:
400      error_func('Could not find apk or executable for %s' % self._suite)
401
402    for x in args.additional_apks:
403      if not os.path.exists(x):
404        error_func('Could not find additional APK: %s' % x)
405
406      apk = apk_helper.ToHelper(x)
407      self._additional_apks.append(apk)
408
409    self._data_deps = []
410    self._gtest_filters = test_filter.InitializeFiltersFromArgs(args)
411    self._run_disabled = args.run_disabled
412    self._run_pre_tests = args.run_pre_tests
413
414    self._data_deps_delegate = data_deps_delegate
415    self._runtime_deps_path = args.runtime_deps_path
416    if not self._runtime_deps_path:
417      logging.warning('No data dependencies will be pushed.')
418
419    if args.app_data_files:
420      self._app_data_files = args.app_data_files
421      if args.app_data_file_dir:
422        self._app_data_file_dir = args.app_data_file_dir
423      else:
424        self._app_data_file_dir = tempfile.mkdtemp()
425        logging.critical('Saving app files to %s', self._app_data_file_dir)
426    else:
427      self._app_data_files = None
428      self._app_data_file_dir = None
429
430    self._flags = None
431    self._initializeCommandLineFlags(args)
432
433    # TODO(jbudorick): Remove this once it's deployed.
434    self._enable_xml_result_parsing = args.enable_xml_result_parsing
435
436  def _initializeCommandLineFlags(self, args):
437    self._flags = []
438    if args.command_line_flags:
439      self._flags.extend(args.command_line_flags)
440    if args.device_flags_file:
441      with open(args.device_flags_file) as f:
442        stripped_lines = (l.strip() for l in f)
443        self._flags.extend(flag for flag in stripped_lines if flag)
444    if args.run_disabled:
445      self._flags.append('--gtest_also_run_disabled_tests')
446
447  @property
448  def activity(self):
449    return self._apk_helper and self._apk_helper.GetActivityName()
450
451  @property
452  def additional_apks(self):
453    return self._additional_apks
454
455  @property
456  def apk(self):
457    return self._apk_helper and self._apk_helper.path
458
459  @property
460  def apk_helper(self):
461    return self._apk_helper
462
463  @property
464  def app_file_dir(self):
465    return self._app_data_file_dir
466
467  @property
468  def app_files(self):
469    return self._app_data_files
470
471  @property
472  def coverage_dir(self):
473    return self._coverage_dir
474
475  @property
476  def enable_xml_result_parsing(self):
477    return self._enable_xml_result_parsing
478
479  @property
480  def exe_dist_dir(self):
481    return self._exe_dist_dir
482
483  @property
484  def external_shard_index(self):
485    return self._external_shard_index
486
487  @property
488  def extract_test_list_from_filter(self):
489    return self._extract_test_list_from_filter
490
491  @property
492  def extras(self):
493    return self._extras
494
495  @property
496  def flags(self):
497    return self._flags
498
499  @property
500  def gs_test_artifacts_bucket(self):
501    return self._gs_test_artifacts_bucket
502
503  @property
504  def gtest_filters(self):
505    return self._gtest_filters
506
507  @property
508  def isolated_script_test_output(self):
509    return self._isolated_script_test_output
510
511  @property
512  def isolated_script_test_perf_output(self):
513    return self._isolated_script_test_perf_output
514
515  @property
516  def render_test_output_dir(self):
517    return self._render_test_output_dir
518
519  @property
520  def package(self):
521    return self._apk_helper and self._apk_helper.GetPackageName()
522
523  @property
524  def permissions(self):
525    return self._apk_helper and self._apk_helper.GetPermissions()
526
527  @property
528  def runner(self):
529    return self._apk_helper and self._apk_helper.GetInstrumentationName()
530
531  @property
532  def shard_timeout(self):
533    return self._shard_timeout
534
535  @property
536  def store_tombstones(self):
537    return self._store_tombstones
538
539  @property
540  def suite(self):
541    return self._suite
542
543  @property
544  def symbolizer(self):
545    return self._symbolizer
546
547  @property
548  def test_apk_incremental_install_json(self):
549    return self._test_apk_incremental_install_json
550
551  @property
552  def test_launcher_batch_limit(self):
553    return self._test_launcher_batch_limit
554
555  @property
556  def total_external_shards(self):
557    return self._total_external_shards
558
559  @property
560  def wait_for_java_debugger(self):
561    return self._wait_for_java_debugger
562
563  @property
564  def use_existing_test_data(self):
565    return self._use_existing_test_data
566
567  @property
568  def run_pre_tests(self):
569    return self._run_pre_tests
570
571  #override
572  def TestType(self):
573    return 'gtest'
574
575  #override
576  def GetPreferredAbis(self):
577    if not self._apk_helper:
578      return None
579    return self._apk_helper.GetAbis()
580
581  #override
582  def SetUp(self):
583    """Map data dependencies via isolate."""
584    self._data_deps.extend(
585        self._data_deps_delegate(self._runtime_deps_path))
586
587  def GetDataDependencies(self):
588    """Returns the test suite's data dependencies.
589
590    Returns:
591      A list of (host_path, device_path) tuples to push. If device_path is
592      None, the client is responsible for determining where to push the file.
593    """
594    return self._data_deps
595
596  def FilterTests(self, test_list, disabled_prefixes=None):
597    """Filters |test_list| based on prefixes and, if present, a filter string.
598
599    Args:
600      test_list: The list of tests to filter.
601      disabled_prefixes: A list of test prefixes to filter. Defaults to
602        DISABLED_, FLAKY_, FAILS_, PRE_, and MANUAL_
603    Returns:
604      A filtered list of tests to run.
605    """
606    gtest_filter_strings = [
607        self._GenerateDisabledFilterString(disabled_prefixes)]
608    if self._gtest_filters:
609      gtest_filter_strings.extend(self._gtest_filters)
610
611    filtered_test_list = test_list
612    # This lock is required because on older versions of Python
613    # |unittest_util.FilterTestNames| use of |fnmatch| is not threadsafe.
614    with self._filter_tests_lock:
615      for gtest_filter_string in gtest_filter_strings:
616        logging.debug('Filtering tests using: %s', gtest_filter_string)
617        filtered_test_list = unittest_util.FilterTestNames(
618            filtered_test_list, gtest_filter_string)
619
620      if self._run_disabled and self._gtest_filters:
621        out_filtered_test_list = list(set(test_list)-set(filtered_test_list))
622        for test in out_filtered_test_list:
623          test_name_no_disabled = TestNameWithoutDisabledPrefix(test)
624          if test_name_no_disabled == test:
625            continue
626          if all(
627              unittest_util.FilterTestNames([test_name_no_disabled],
628                                            gtest_filter)
629              for gtest_filter in self._gtest_filters):
630            filtered_test_list.append(test)
631    return filtered_test_list
632
633  def _GenerateDisabledFilterString(self, disabled_prefixes):
634    disabled_filter_items = []
635
636    if disabled_prefixes is None:
637      disabled_prefixes = ['FAILS_']
638      if '--run-manual' not in self._flags:
639        disabled_prefixes += ['MANUAL_']
640      if not self._run_disabled:
641        disabled_prefixes += ['DISABLED_', 'FLAKY_']
642      if not self._run_pre_tests:
643        disabled_prefixes += ['PRE_']
644
645    disabled_filter_items += ['%s*' % dp for dp in disabled_prefixes]
646    disabled_filter_items += ['*.%s*' % dp for dp in disabled_prefixes]
647
648    disabled_tests_file_path = os.path.join(
649        host_paths.DIR_SOURCE_ROOT, 'build', 'android', 'pylib', 'gtest',
650        'filter', '%s_disabled' % self._suite)
651    if disabled_tests_file_path and os.path.exists(disabled_tests_file_path):
652      with open(disabled_tests_file_path) as disabled_tests_file:
653        disabled_filter_items += [
654            '%s' % l for l in (line.strip() for line in disabled_tests_file)
655            if l and not l.startswith('#')]
656
657    return '*-%s' % ':'.join(disabled_filter_items)
658
659  #override
660  def TearDown(self):
661    """Do nothing."""
662