• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1# Copyright 2014 The Chromium Authors
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5
6
7import json
8import logging
9import os
10import re
11import tempfile
12import threading
13import xml.etree.ElementTree
14
15import six
16from devil.android import apk_helper
17from pylib import constants
18from pylib.constants import host_paths
19from pylib.base import base_test_result
20from pylib.base import test_instance
21from pylib.symbols import stack_symbolizer
22from pylib.utils import test_filter
23
24
25with host_paths.SysPath(host_paths.BUILD_COMMON_PATH):
26  import unittest_util # pylint: disable=import-error
27
28
29BROWSER_TEST_SUITES = [
30    'android_browsertests',
31    'android_sync_integration_tests',
32    'components_browsertests',
33    'content_browsertests',
34    'weblayer_browsertests',
35]
36
37# The max number of tests to run on a shard during the test run.
38MAX_SHARDS = 256
39
40RUN_IN_SUB_THREAD_TEST_SUITES = [
41    # Multiprocess tests should be run outside of the main thread.
42    'base_unittests',  # file_locking_unittest.cc uses a child process.
43    'gwp_asan_unittests',
44    'ipc_perftests',
45    'ipc_tests',
46    'mojo_perftests',
47    'mojo_unittests',
48    'net_unittests'
49]
50
51
52# Used for filtering large data deps at a finer grain than what's allowed in
53# isolate files since pushing deps to devices is expensive.
54# Wildcards are allowed.
55_DEPS_EXCLUSION_LIST = [
56    'chrome/test/data/extensions/api_test',
57    'chrome/test/data/extensions/secure_shell',
58    'chrome/test/data/firefox*',
59    'chrome/test/data/gpu',
60    'chrome/test/data/image_decoding',
61    'chrome/test/data/import',
62    'chrome/test/data/page_cycler',
63    'chrome/test/data/perf',
64    'chrome/test/data/pyauto_private',
65    'chrome/test/data/safari_import',
66    'chrome/test/data/scroll',
67    'chrome/test/data/third_party',
68    'third_party/hunspell_dictionaries/*.dic',
69    # crbug.com/258690
70    'webkit/data/bmp_decoder',
71    'webkit/data/ico_decoder',
72]
73
74
75_EXTRA_NATIVE_TEST_ACTIVITY = (
76    'org.chromium.native_test.NativeTestInstrumentationTestRunner.'
77        'NativeTestActivity')
78_EXTRA_RUN_IN_SUB_THREAD = (
79    'org.chromium.native_test.NativeTest.RunInSubThread')
80EXTRA_SHARD_NANO_TIMEOUT = (
81    'org.chromium.native_test.NativeTestInstrumentationTestRunner.'
82        'ShardNanoTimeout')
83_EXTRA_SHARD_SIZE_LIMIT = (
84    'org.chromium.native_test.NativeTestInstrumentationTestRunner.'
85        'ShardSizeLimit')
86
87# TODO(jbudorick): Remove these once we're no longer parsing stdout to generate
88# results.
89_RE_TEST_STATUS = re.compile(
90    # Test state.
91    r'\[ +((?:RUN)|(?:FAILED)|(?:OK)|(?:CRASHED)|(?:SKIPPED)) +\] ?'
92    # Test name.
93    r'([^ ]+)?'
94    # Optional parameters.
95    r'(?:, where'
96    #   Type parameter
97    r'(?: TypeParam = [^()]*(?: and)?)?'
98    #   Value parameter
99    r'(?: GetParam\(\) = [^()]*)?'
100    # End of optional parameters.
101    ')?'
102    # Optional test execution time.
103    r'(?: \((\d+) ms\))?$')
104# Crash detection constants.
105_RE_TEST_ERROR = re.compile(r'FAILURES!!! Tests run: \d+,'
106                                    r' Failures: \d+, Errors: 1')
107_RE_TEST_CURRENTLY_RUNNING = re.compile(
108    r'\[ERROR:.*?\] Currently running: (.*)')
109_RE_TEST_DCHECK_FATAL = re.compile(r'\[.*:FATAL:.*\] (.*)')
110_RE_DISABLED = re.compile(r'DISABLED_')
111_RE_FLAKY = re.compile(r'FLAKY_')
112
113# Detect stack line in stdout.
114_STACK_LINE_RE = re.compile(r'\s*#\d+')
115
116def ParseGTestListTests(raw_list):
117  """Parses a raw test list as provided by --gtest_list_tests.
118
119  Args:
120    raw_list: The raw test listing with the following format:
121
122    IPCChannelTest.
123      SendMessageInChannelConnected
124    IPCSyncChannelTest.
125      Simple
126      DISABLED_SendWithTimeoutMixedOKAndTimeout
127
128  Returns:
129    A list of all tests. For the above raw listing:
130
131    [IPCChannelTest.SendMessageInChannelConnected, IPCSyncChannelTest.Simple,
132     IPCSyncChannelTest.DISABLED_SendWithTimeoutMixedOKAndTimeout]
133  """
134  ret = []
135  current = ''
136  for test in raw_list:
137    if not test:
138      continue
139    if not test.startswith(' '):
140      test_case = test.split()[0]
141      if test_case.endswith('.'):
142        current = test_case
143    else:
144      test = test.strip()
145      if test and not 'YOU HAVE' in test:
146        test_name = test.split()[0]
147        ret += [current + test_name]
148  return ret
149
150
151def ParseGTestOutput(output, symbolizer, device_abi):
152  """Parses raw gtest output and returns a list of results.
153
154  Args:
155    output: A list of output lines.
156    symbolizer: The symbolizer used to symbolize stack.
157    device_abi: Device abi that is needed for symbolization.
158  Returns:
159    A list of base_test_result.BaseTestResults.
160  """
161  duration = 0
162  fallback_result_type = None
163  log = []
164  stack = []
165  result_type = None
166  results = []
167  test_name = None
168
169  def symbolize_stack_and_merge_with_log():
170    log_string = '\n'.join(log or [])
171    if not stack:
172      stack_string = ''
173    else:
174      stack_string = '\n'.join(
175          symbolizer.ExtractAndResolveNativeStackTraces(
176              stack, device_abi))
177    return '%s\n%s' % (log_string, stack_string)
178
179  def handle_possibly_unknown_test():
180    if test_name is not None:
181      results.append(
182          base_test_result.BaseTestResult(
183              TestNameWithoutDisabledPrefix(test_name),
184              # If we get here, that means we started a test, but it did not
185              # produce a definitive test status output, so assume it crashed.
186              # crbug/1191716
187              fallback_result_type or base_test_result.ResultType.CRASH,
188              duration,
189              log=symbolize_stack_and_merge_with_log()))
190
191  for l in output:
192    matcher = _RE_TEST_STATUS.match(l)
193    if matcher:
194      if matcher.group(1) == 'RUN':
195        handle_possibly_unknown_test()
196        duration = 0
197        fallback_result_type = None
198        log = []
199        stack = []
200        result_type = None
201      elif matcher.group(1) == 'OK':
202        result_type = base_test_result.ResultType.PASS
203      elif matcher.group(1) == 'SKIPPED':
204        result_type = base_test_result.ResultType.SKIP
205      elif matcher.group(1) == 'FAILED':
206        result_type = base_test_result.ResultType.FAIL
207      elif matcher.group(1) == 'CRASHED':
208        fallback_result_type = base_test_result.ResultType.CRASH
209      # Be aware that test name and status might not appear on same line.
210      test_name = matcher.group(2) if matcher.group(2) else test_name
211      duration = int(matcher.group(3)) if matcher.group(3) else 0
212
213    else:
214      # Can possibly add more matchers, such as different results from DCHECK.
215      currently_running_matcher = _RE_TEST_CURRENTLY_RUNNING.match(l)
216      dcheck_matcher = _RE_TEST_DCHECK_FATAL.match(l)
217
218      if currently_running_matcher:
219        test_name = currently_running_matcher.group(1)
220        result_type = base_test_result.ResultType.CRASH
221        duration = None  # Don't know. Not using 0 as this is unknown vs 0.
222      elif dcheck_matcher:
223        result_type = base_test_result.ResultType.CRASH
224        duration = None  # Don't know.  Not using 0 as this is unknown vs 0.
225
226    if log is not None:
227      if not matcher and _STACK_LINE_RE.match(l):
228        stack.append(l)
229      else:
230        log.append(l)
231
232    if result_type and test_name:
233      # Don't bother symbolizing output if the test passed.
234      if result_type == base_test_result.ResultType.PASS:
235        stack = []
236      results.append(base_test_result.BaseTestResult(
237          TestNameWithoutDisabledPrefix(test_name), result_type, duration,
238          log=symbolize_stack_and_merge_with_log()))
239      test_name = None
240
241  handle_possibly_unknown_test()
242
243  return results
244
245
246def ParseGTestXML(xml_content):
247  """Parse gtest XML result."""
248  results = []
249  if not xml_content:
250    return results
251
252  html = six.moves.html_parser.HTMLParser()
253
254  testsuites = xml.etree.ElementTree.fromstring(xml_content)
255  for testsuite in testsuites:
256    suite_name = testsuite.attrib['name']
257    for testcase in testsuite:
258      case_name = testcase.attrib['name']
259      result_type = base_test_result.ResultType.PASS
260      log = []
261      for failure in testcase:
262        result_type = base_test_result.ResultType.FAIL
263        log.append(html.unescape(failure.attrib['message']))
264
265      results.append(base_test_result.BaseTestResult(
266          '%s.%s' % (suite_name, TestNameWithoutDisabledPrefix(case_name)),
267          result_type,
268          int(float(testcase.attrib['time']) * 1000),
269          log=('\n'.join(log) if log else '')))
270
271  return results
272
273
274def ParseGTestJSON(json_content):
275  """Parse results in the JSON Test Results format."""
276  results = []
277  if not json_content:
278    return results
279
280  json_data = json.loads(json_content)
281
282  openstack = list(json_data['tests'].items())
283
284  while openstack:
285    name, value = openstack.pop()
286
287    if 'expected' in value and 'actual' in value:
288      if value['actual'] == 'PASS':
289        result_type = base_test_result.ResultType.PASS
290      elif value['actual'] == 'SKIP':
291        result_type = base_test_result.ResultType.SKIP
292      elif value['actual'] == 'CRASH':
293        result_type = base_test_result.ResultType.CRASH
294      elif value['actual'] == 'TIMEOUT':
295        result_type = base_test_result.ResultType.TIMEOUT
296      else:
297        result_type = base_test_result.ResultType.FAIL
298      results.append(base_test_result.BaseTestResult(name, result_type))
299    else:
300      openstack += [("%s.%s" % (name, k), v) for k, v in six.iteritems(value)]
301
302  return results
303
304
305def TestNameWithoutDisabledPrefix(test_name):
306  """Modify the test name without disabled prefix if prefix 'DISABLED_' or
307  'FLAKY_' presents.
308
309  Args:
310    test_name: The name of a test.
311  Returns:
312    A test name without prefix 'DISABLED_' or 'FLAKY_'.
313  """
314  disabled_prefixes = [_RE_DISABLED, _RE_FLAKY]
315  for dp in disabled_prefixes:
316    test_name = dp.sub('', test_name)
317  return test_name
318
319class GtestTestInstance(test_instance.TestInstance):
320
321  def __init__(self, args, data_deps_delegate, error_func):
322    super().__init__()
323    # TODO(jbudorick): Support multiple test suites.
324    if len(args.suite_name) > 1:
325      raise ValueError('Platform mode currently supports only 1 gtest suite')
326    self._coverage_dir = args.coverage_dir
327    self._exe_dist_dir = None
328    self._external_shard_index = args.test_launcher_shard_index
329    self._extract_test_list_from_filter = args.extract_test_list_from_filter
330    self._filter_tests_lock = threading.Lock()
331    self._gs_test_artifacts_bucket = args.gs_test_artifacts_bucket
332    self._isolated_script_test_output = args.isolated_script_test_output
333    self._isolated_script_test_perf_output = (
334        args.isolated_script_test_perf_output)
335    self._render_test_output_dir = args.render_test_output_dir
336    self._shard_timeout = args.shard_timeout
337    self._store_tombstones = args.store_tombstones
338    self._suite = args.suite_name[0]
339    self._symbolizer = stack_symbolizer.Symbolizer(None)
340    self._total_external_shards = args.test_launcher_total_shards
341    self._wait_for_java_debugger = args.wait_for_java_debugger
342    self._use_existing_test_data = args.use_existing_test_data
343
344    # GYP:
345    if args.executable_dist_dir:
346      self._exe_dist_dir = os.path.abspath(args.executable_dist_dir)
347    else:
348      # TODO(agrieve): Remove auto-detection once recipes pass flag explicitly.
349      exe_dist_dir = os.path.join(constants.GetOutDirectory(),
350                                  '%s__dist' % self._suite)
351
352      if os.path.exists(exe_dist_dir):
353        self._exe_dist_dir = exe_dist_dir
354
355    incremental_part = ''
356    if args.test_apk_incremental_install_json:
357      incremental_part = '_incremental'
358
359    self._test_launcher_batch_limit = MAX_SHARDS
360    if (args.test_launcher_batch_limit
361        and 0 < args.test_launcher_batch_limit < MAX_SHARDS):
362      self._test_launcher_batch_limit = args.test_launcher_batch_limit
363
364    apk_path = os.path.join(
365        constants.GetOutDirectory(), '%s_apk' % self._suite,
366        '%s-debug%s.apk' % (self._suite, incremental_part))
367    self._test_apk_incremental_install_json = (
368        args.test_apk_incremental_install_json)
369    if not os.path.exists(apk_path):
370      self._apk_helper = None
371    else:
372      self._apk_helper = apk_helper.ApkHelper(apk_path)
373      self._extras = {
374          _EXTRA_NATIVE_TEST_ACTIVITY: self._apk_helper.GetActivityName(),
375      }
376      if self._suite in RUN_IN_SUB_THREAD_TEST_SUITES:
377        self._extras[_EXTRA_RUN_IN_SUB_THREAD] = 1
378      if self._suite in BROWSER_TEST_SUITES:
379        self._extras[_EXTRA_SHARD_SIZE_LIMIT] = 1
380        self._extras[EXTRA_SHARD_NANO_TIMEOUT] = int(1e9 * self._shard_timeout)
381        self._shard_timeout = 10 * self._shard_timeout
382      if args.wait_for_java_debugger:
383        self._extras[EXTRA_SHARD_NANO_TIMEOUT] = int(1e15)  # Forever
384
385    if not self._apk_helper and not self._exe_dist_dir:
386      error_func('Could not find apk or executable for %s' % self._suite)
387
388    self._data_deps = []
389    self._gtest_filters = test_filter.InitializeFiltersFromArgs(args)
390    self._run_disabled = args.run_disabled
391
392    self._data_deps_delegate = data_deps_delegate
393    self._runtime_deps_path = args.runtime_deps_path
394    if not self._runtime_deps_path:
395      logging.warning('No data dependencies will be pushed.')
396
397    if args.app_data_files:
398      self._app_data_files = args.app_data_files
399      if args.app_data_file_dir:
400        self._app_data_file_dir = args.app_data_file_dir
401      else:
402        self._app_data_file_dir = tempfile.mkdtemp()
403        logging.critical('Saving app files to %s', self._app_data_file_dir)
404    else:
405      self._app_data_files = None
406      self._app_data_file_dir = None
407
408    self._flags = None
409    self._initializeCommandLineFlags(args)
410
411    # TODO(jbudorick): Remove this once it's deployed.
412    self._enable_xml_result_parsing = args.enable_xml_result_parsing
413
414  def _initializeCommandLineFlags(self, args):
415    self._flags = []
416    if args.command_line_flags:
417      self._flags.extend(args.command_line_flags)
418    if args.device_flags_file:
419      with open(args.device_flags_file) as f:
420        stripped_lines = (l.strip() for l in f)
421        self._flags.extend(flag for flag in stripped_lines if flag)
422    if args.run_disabled:
423      self._flags.append('--gtest_also_run_disabled_tests')
424
425  @property
426  def activity(self):
427    return self._apk_helper and self._apk_helper.GetActivityName()
428
429  @property
430  def apk(self):
431    return self._apk_helper and self._apk_helper.path
432
433  @property
434  def apk_helper(self):
435    return self._apk_helper
436
437  @property
438  def app_file_dir(self):
439    return self._app_data_file_dir
440
441  @property
442  def app_files(self):
443    return self._app_data_files
444
445  @property
446  def coverage_dir(self):
447    return self._coverage_dir
448
449  @property
450  def enable_xml_result_parsing(self):
451    return self._enable_xml_result_parsing
452
453  @property
454  def exe_dist_dir(self):
455    return self._exe_dist_dir
456
457  @property
458  def external_shard_index(self):
459    return self._external_shard_index
460
461  @property
462  def extract_test_list_from_filter(self):
463    return self._extract_test_list_from_filter
464
465  @property
466  def extras(self):
467    return self._extras
468
469  @property
470  def flags(self):
471    return self._flags
472
473  @property
474  def gs_test_artifacts_bucket(self):
475    return self._gs_test_artifacts_bucket
476
477  @property
478  def gtest_filters(self):
479    return self._gtest_filters
480
481  @property
482  def isolated_script_test_output(self):
483    return self._isolated_script_test_output
484
485  @property
486  def isolated_script_test_perf_output(self):
487    return self._isolated_script_test_perf_output
488
489  @property
490  def render_test_output_dir(self):
491    return self._render_test_output_dir
492
493  @property
494  def package(self):
495    return self._apk_helper and self._apk_helper.GetPackageName()
496
497  @property
498  def permissions(self):
499    return self._apk_helper and self._apk_helper.GetPermissions()
500
501  @property
502  def runner(self):
503    return self._apk_helper and self._apk_helper.GetInstrumentationName()
504
505  @property
506  def shard_timeout(self):
507    return self._shard_timeout
508
509  @property
510  def store_tombstones(self):
511    return self._store_tombstones
512
513  @property
514  def suite(self):
515    return self._suite
516
517  @property
518  def symbolizer(self):
519    return self._symbolizer
520
521  @property
522  def test_apk_incremental_install_json(self):
523    return self._test_apk_incremental_install_json
524
525  @property
526  def test_launcher_batch_limit(self):
527    return self._test_launcher_batch_limit
528
529  @property
530  def total_external_shards(self):
531    return self._total_external_shards
532
533  @property
534  def wait_for_java_debugger(self):
535    return self._wait_for_java_debugger
536
537  @property
538  def use_existing_test_data(self):
539    return self._use_existing_test_data
540
541  #override
542  def TestType(self):
543    return 'gtest'
544
545  #override
546  def GetPreferredAbis(self):
547    if not self._apk_helper:
548      return None
549    return self._apk_helper.GetAbis()
550
551  #override
552  def SetUp(self):
553    """Map data dependencies via isolate."""
554    self._data_deps.extend(
555        self._data_deps_delegate(self._runtime_deps_path))
556
557  def GetDataDependencies(self):
558    """Returns the test suite's data dependencies.
559
560    Returns:
561      A list of (host_path, device_path) tuples to push. If device_path is
562      None, the client is responsible for determining where to push the file.
563    """
564    return self._data_deps
565
566  def FilterTests(self, test_list, disabled_prefixes=None):
567    """Filters |test_list| based on prefixes and, if present, a filter string.
568
569    Args:
570      test_list: The list of tests to filter.
571      disabled_prefixes: A list of test prefixes to filter. Defaults to
572        DISABLED_, FLAKY_, FAILS_, PRE_, and MANUAL_
573    Returns:
574      A filtered list of tests to run.
575    """
576    gtest_filter_strings = [
577        self._GenerateDisabledFilterString(disabled_prefixes)]
578    if self._gtest_filters:
579      gtest_filter_strings.extend(self._gtest_filters)
580
581    filtered_test_list = test_list
582    # This lock is required because on older versions of Python
583    # |unittest_util.FilterTestNames| use of |fnmatch| is not threadsafe.
584    with self._filter_tests_lock:
585      for gtest_filter_string in gtest_filter_strings:
586        logging.debug('Filtering tests using: %s', gtest_filter_string)
587        filtered_test_list = unittest_util.FilterTestNames(
588            filtered_test_list, gtest_filter_string)
589
590      if self._run_disabled and self._gtest_filters:
591        out_filtered_test_list = list(set(test_list)-set(filtered_test_list))
592        for test in out_filtered_test_list:
593          test_name_no_disabled = TestNameWithoutDisabledPrefix(test)
594          if test_name_no_disabled == test:
595            continue
596          if all(
597              unittest_util.FilterTestNames([test_name_no_disabled],
598                                            gtest_filter)
599              for gtest_filter in self._gtest_filters):
600            filtered_test_list.append(test)
601    return filtered_test_list
602
603  def _GenerateDisabledFilterString(self, disabled_prefixes):
604    disabled_filter_items = []
605
606    if disabled_prefixes is None:
607      disabled_prefixes = ['FAILS_', 'PRE_']
608      if '--run-manual' not in self._flags:
609        disabled_prefixes += ['MANUAL_']
610      if not self._run_disabled:
611        disabled_prefixes += ['DISABLED_', 'FLAKY_']
612
613    disabled_filter_items += ['%s*' % dp for dp in disabled_prefixes]
614    disabled_filter_items += ['*.%s*' % dp for dp in disabled_prefixes]
615
616    disabled_tests_file_path = os.path.join(
617        host_paths.DIR_SOURCE_ROOT, 'build', 'android', 'pylib', 'gtest',
618        'filter', '%s_disabled' % self._suite)
619    if disabled_tests_file_path and os.path.exists(disabled_tests_file_path):
620      with open(disabled_tests_file_path) as disabled_tests_file:
621        disabled_filter_items += [
622            '%s' % l for l in (line.strip() for line in disabled_tests_file)
623            if l and not l.startswith('#')]
624
625    return '*-%s' % ':'.join(disabled_filter_items)
626
627  #override
628  def TearDown(self):
629    """Do nothing."""
630