• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/env vpython3
2#
3# Copyright 2013 The Chromium Authors
4# Use of this source code is governed by a BSD-style license that can be
5# found in the LICENSE file.
6
7"""Runs all types of tests from one unified interface."""
8
9from __future__ import absolute_import
10import argparse
11import collections
12import contextlib
13import io
14import itertools
15import logging
16import os
17import re
18import shlex
19import shutil
20import signal
21import sys
22import tempfile
23import threading
24import traceback
25import unittest
26
27# Import _strptime before threaded code. datetime.datetime.strptime is
28# threadsafe except for the initial import of the _strptime module.
29# See http://crbug.com/724524 and https://bugs.python.org/issue7980.
30import _strptime  # pylint: disable=unused-import
31
32# pylint: disable=ungrouped-imports
33from pylib.constants import host_paths
34
35if host_paths.DEVIL_PATH not in sys.path:
36  sys.path.append(host_paths.DEVIL_PATH)
37
38from devil import base_error
39from devil.utils import reraiser_thread
40from devil.utils import run_tests_helper
41
42from pylib import constants
43from pylib.base import base_test_result
44from pylib.base import environment_factory
45from pylib.base import output_manager
46from pylib.base import output_manager_factory
47from pylib.base import test_instance_factory
48from pylib.base import test_run_factory
49from pylib.results import json_results
50from pylib.results import report_results
51from pylib.results.presentation import test_results_presentation
52from pylib.utils import local_utils
53from pylib.utils import logdog_helper
54from pylib.utils import logging_utils
55from pylib.utils import test_filter
56
57from py_utils import contextlib_ext
58
59from lib.results import result_sink  # pylint: disable=import-error
60
61_DEVIL_STATIC_CONFIG_FILE = os.path.abspath(os.path.join(
62    host_paths.DIR_SOURCE_ROOT, 'build', 'android', 'devil_config.json'))
63
64_RERUN_FAILED_TESTS_FILE = 'rerun_failed_tests.filter'
65
66
67def _RealPath(arg):
68  if arg.startswith('//'):
69    arg = os.path.abspath(os.path.join(host_paths.DIR_SOURCE_ROOT,
70                                       arg[2:].replace('/', os.sep)))
71  return os.path.realpath(arg)
72
73
74def AddTestLauncherOptions(parser):
75  """Adds arguments mirroring //base/test/launcher.
76
77  Args:
78    parser: The parser to which arguments should be added.
79  Returns:
80    The given parser.
81  """
82  parser.add_argument(
83      '--test-launcher-retry-limit',
84      '--test_launcher_retry_limit',
85      '--num_retries', '--num-retries',
86      '--isolated-script-test-launcher-retry-limit',
87      dest='num_retries', type=int, default=2,
88      help='Number of retries for a test before '
89           'giving up (default: %(default)s).')
90  parser.add_argument(
91      '--test-launcher-summary-output',
92      '--json-results-file',
93      dest='json_results_file', type=os.path.realpath,
94      help='If set, will dump results in JSON form to the specified file. '
95           'Note that this will also trigger saving per-test logcats to '
96           'logdog.')
97  parser.add_argument(
98      '--test-launcher-shard-index',
99      type=int, default=os.environ.get('GTEST_SHARD_INDEX', 0),
100      help='Index of the external shard to run.')
101  parser.add_argument(
102      '--test-launcher-total-shards',
103      type=int, default=os.environ.get('GTEST_TOTAL_SHARDS', 1),
104      help='Total number of external shards.')
105
106  test_filter.AddFilterOptions(parser)
107
108  return parser
109
110
111def AddCommandLineOptions(parser):
112  """Adds arguments to support passing command-line flags to the device."""
113  parser.add_argument(
114      '--device-flags-file',
115      type=os.path.realpath,
116      help='The relative filepath to a file containing '
117           'command-line flags to set on the device')
118  parser.add_argument(
119      '--use-apk-under-test-flags-file',
120      action='store_true',
121      help='Wether to use the flags file for the apk under test. If set, '
122           "the filename will be looked up in the APK's PackageInfo.")
123  parser.add_argument('--variations-test-seed-path',
124                      type=os.path.relpath,
125                      default=None,
126                      help='Path to variations seed file.')
127  parser.add_argument('--webview-variations-test-seed-path',
128                      type=os.path.relpath,
129                      default=None,
130                      help='Path to variations seed file for WebView.')
131
132  parser.set_defaults(allow_unknown=True)
133  parser.set_defaults(command_line_flags=None)
134
135
136def AddTracingOptions(parser):
137  # TODO(shenghuazhang): Move this into AddCommonOptions once it's supported
138  # for all test types.
139  parser.add_argument(
140      '--trace-output',
141      metavar='FILENAME', type=os.path.realpath,
142      help='Path to save test_runner trace json output to.')
143
144  parser.add_argument(
145      '--trace-all',
146      action='store_true',
147      help='Whether to trace all function calls.')
148
149
150def AddCommonOptions(parser):
151  """Adds all common options to |parser|."""
152
153  default_build_type = os.environ.get('BUILDTYPE', 'Debug')
154
155  debug_or_release_group = parser.add_mutually_exclusive_group()
156  debug_or_release_group.add_argument(
157      '--debug',
158      action='store_const', const='Debug', dest='build_type',
159      default=default_build_type,
160      help='If set, run test suites under out/Debug. '
161           'Default is env var BUILDTYPE or Debug.')
162  debug_or_release_group.add_argument(
163      '--release',
164      action='store_const', const='Release', dest='build_type',
165      help='If set, run test suites under out/Release. '
166           'Default is env var BUILDTYPE or Debug.')
167
168  parser.add_argument(
169      '--break-on-failure', '--break_on_failure',
170      dest='break_on_failure', action='store_true',
171      help='Whether to break on failure.')
172
173  # TODO(jbudorick): Remove this once everything has switched to platform
174  # mode.
175  parser.add_argument(
176      '--enable-platform-mode',
177      action='store_true',
178      help='Run the test scripts in platform mode, which '
179           'conceptually separates the test runner from the '
180           '"device" (local or remote, real or emulated) on '
181           'which the tests are running. [experimental]')
182
183  parser.add_argument(
184      '-e', '--environment',
185      default='local', choices=constants.VALID_ENVIRONMENTS,
186      help='Test environment to run in (default: %(default)s).')
187
188  parser.add_argument(
189      '--local-output',
190      action='store_true',
191      help='Whether to archive test output locally and generate '
192           'a local results detail page.')
193
194  parser.add_argument('--list-tests',
195                      action='store_true',
196                      help='List available tests and exit.')
197
198  parser.add_argument('--wrapper-script-args',
199                      help='A string of args that were passed to the wrapper '
200                      'script. This should probably not be edited by a '
201                      'user as it is passed by the wrapper itself.')
202
203  class FastLocalDevAction(argparse.Action):
204    def __call__(self, parser, namespace, values, option_string=None):
205      namespace.enable_concurrent_adb = True
206      namespace.enable_device_cache = True
207      namespace.extract_test_list_from_filter = True
208      namespace.local_output = True
209      namespace.num_retries = 0
210      namespace.skip_clear_data = True
211      namespace.use_persistent_shell = True
212
213  parser.add_argument(
214      '--fast-local-dev',
215      type=bool,
216      nargs=0,
217      action=FastLocalDevAction,
218      help='Alias for: --num-retries=0 --enable-device-cache '
219      '--enable-concurrent-adb --skip-clear-data '
220      '--extract-test-list-from-filter --use-persistent-shell --local-output')
221
222  # TODO(jbudorick): Remove this once downstream bots have switched to
223  # api.test_results.
224  parser.add_argument(
225      '--flakiness-dashboard-server',
226      dest='flakiness_dashboard_server',
227      help=argparse.SUPPRESS)
228  parser.add_argument(
229      '--gs-results-bucket',
230      help='Google Storage bucket to upload results to.')
231
232  parser.add_argument(
233      '--output-directory',
234      dest='output_directory', type=os.path.realpath,
235      help='Path to the directory in which build files are'
236           ' located (must include build type). This will take'
237           ' precedence over --debug and --release')
238  parser.add_argument(
239      '-v', '--verbose',
240      dest='verbose_count', default=0, action='count',
241      help='Verbose level (multiple times for more)')
242
243  parser.add_argument(
244      '--repeat', '--gtest_repeat', '--gtest-repeat',
245      '--isolated-script-test-repeat',
246      dest='repeat', type=int, default=0,
247      help='Number of times to repeat the specified set of tests.')
248
249  # Not useful for junit tests.
250  parser.add_argument(
251      '--use-persistent-shell',
252      action='store_true',
253      help='Uses a persistent shell connection for the adb connection.')
254
255  parser.add_argument('--disable-test-server',
256                      action='store_true',
257                      help='Disables SpawnedTestServer which doesn'
258                      't work with remote adb. '
259                      'WARNING: Will break tests which require the server.')
260
261  # This is currently only implemented for gtests and instrumentation tests.
262  parser.add_argument(
263      '--gtest_also_run_disabled_tests', '--gtest-also-run-disabled-tests',
264      '--isolated-script-test-also-run-disabled-tests',
265      dest='run_disabled', action='store_true',
266      help='Also run disabled tests if applicable.')
267
268  # These are currently only implemented for gtests.
269  parser.add_argument('--isolated-script-test-output',
270                      help='If present, store test results on this path.')
271  parser.add_argument('--isolated-script-test-perf-output',
272                      help='If present, store chartjson results on this path.')
273  parser.add_argument('--timeout-scale',
274                      type=float,
275                      help='Factor by which timeouts should be scaled.')
276
277  AddTestLauncherOptions(parser)
278
279
280def ProcessCommonOptions(args):
281  """Processes and handles all common options."""
282  run_tests_helper.SetLogLevel(args.verbose_count, add_handler=False)
283  if args.verbose_count > 0:
284    handler = logging_utils.ColorStreamHandler()
285  else:
286    handler = logging.StreamHandler(sys.stdout)
287  handler.setFormatter(run_tests_helper.CustomFormatter())
288  logging.getLogger().addHandler(handler)
289
290  constants.SetBuildType(args.build_type)
291  if args.output_directory:
292    constants.SetOutputDirectory(args.output_directory)
293
294
295def AddDeviceOptions(parser):
296  """Adds device options to |parser|."""
297
298  parser = parser.add_argument_group('device arguments')
299
300  parser.add_argument(
301      '--adb-path',
302      type=os.path.realpath,
303      help='Specify the absolute path of the adb binary that '
304           'should be used.')
305  parser.add_argument(
306      '--use-local-devil-tools',
307      action='store_true',
308      help='Use locally built versions of tools used by devil_chromium.')
309  parser.add_argument('--denylist-file',
310                      type=os.path.realpath,
311                      help='Device denylist file.')
312  parser.add_argument(
313      '-d', '--device', nargs='+',
314      dest='test_devices',
315      help='Target device(s) for the test suite to run on.')
316  parser.add_argument(
317      '--enable-concurrent-adb',
318      action='store_true',
319      help='Run multiple adb commands at the same time, even '
320           'for the same device.')
321  parser.add_argument(
322      '--enable-device-cache',
323      action='store_true',
324      help='Cache device state to disk between runs')
325  parser.add_argument(
326      '--skip-clear-data',
327      action='store_true',
328      help='Do not wipe app data between tests. Use this to '
329           'speed up local development and never on bots '
330                     '(increases flakiness)')
331  parser.add_argument(
332      '--recover-devices',
333      action='store_true',
334      help='Attempt to recover devices prior to the final retry. Warning: '
335      'this will cause all devices to reboot.')
336
337  parser.add_argument(
338      '--upload-logcats-file',
339      action='store_true',
340      dest='upload_logcats_file',
341      help='Whether to upload logcat file to logdog.')
342
343  logcat_output_group = parser.add_mutually_exclusive_group()
344  logcat_output_group.add_argument(
345      '--logcat-output-dir', type=os.path.realpath,
346      help='If set, will dump logcats recorded during test run to directory. '
347           'File names will be the device ids with timestamps.')
348  logcat_output_group.add_argument(
349      '--logcat-output-file', type=os.path.realpath,
350      help='If set, will merge logcats recorded during test run and dump them '
351           'to the specified file.')
352
353  parser.add_argument(
354      '--force-main-user',
355      action='store_true',
356      help='Force the applicable adb commands to run with "--user" param set '
357      'to the id of the main user on device. Only use when the main user is a '
358      'secondary user, e.g. Android Automotive OS.')
359
360
361def AddEmulatorOptions(parser):
362  """Adds emulator-specific options to |parser|."""
363  parser = parser.add_argument_group('emulator arguments')
364
365  parser.add_argument(
366      '--avd-config',
367      type=os.path.realpath,
368      help='Path to the avd config textpb. '
369      '(See //tools/android/avd/proto/ for message definition'
370      ' and existing textpb files.)')
371  parser.add_argument(
372      '--emulator-count',
373      type=int,
374      default=1,
375      help='Number of emulators to use.')
376  parser.add_argument(
377      '--emulator-window',
378      action='store_true',
379      default=False,
380      help='Enable graphical window display on the emulator.')
381  parser.add_argument(
382      '--emulator-debug-tags',
383      help='Comma-separated list of debug tags. This can be used to enable or '
384      'disable debug messages from specific parts of the emulator, e.g. '
385      'init,snapshot. See "emulator -help-debug-tags" '
386      'for a full list of tags.')
387  parser.add_argument(
388      '--emulator-enable-network',
389      action='store_true',
390      help='Enable the network (WiFi and mobile data) on the emulator.')
391
392
393def AddGTestOptions(parser):
394  """Adds gtest options to |parser|."""
395
396  parser = parser.add_argument_group('gtest arguments')
397
398  parser.add_argument(
399      '--additional-apk',
400      action='append', dest='additional_apks', default=[],
401      type=_RealPath,
402      help='Additional apk that must be installed on '
403           'the device when the tests are run.')
404  parser.add_argument(
405      '--app-data-file',
406      action='append', dest='app_data_files',
407      help='A file path relative to the app data directory '
408           'that should be saved to the host.')
409  parser.add_argument(
410      '--app-data-file-dir',
411      help='Host directory to which app data files will be'
412           ' saved. Used with --app-data-file.')
413  parser.add_argument(
414      '--enable-xml-result-parsing',
415      action='store_true', help=argparse.SUPPRESS)
416  parser.add_argument(
417      '--executable-dist-dir',
418      type=os.path.realpath,
419      help="Path to executable's dist directory for native"
420           " (non-apk) tests.")
421  parser.add_argument(
422      '--extract-test-list-from-filter',
423      action='store_true',
424      help='When a test filter is specified, and the list of '
425           'tests can be determined from it, skip querying the '
426           'device for the list of all tests. Speeds up local '
427           'development, but is not safe to use on bots ('
428           'http://crbug.com/549214')
429  parser.add_argument(
430      '--gs-test-artifacts-bucket',
431      help=('If present, test artifacts will be uploaded to this Google '
432            'Storage bucket.'))
433  parser.add_argument(
434      '--render-test-output-dir',
435      help='If present, store rendering artifacts in this path.')
436  parser.add_argument(
437      '--runtime-deps-path',
438      dest='runtime_deps_path', type=os.path.realpath,
439      help='Runtime data dependency file from GN.')
440  parser.add_argument(
441      '-t', '--shard-timeout',
442      dest='shard_timeout', type=int, default=120,
443      help='Timeout to wait for each test (default: %(default)s).')
444  parser.add_argument(
445      '--store-tombstones',
446      dest='store_tombstones', action='store_true',
447      help='Add tombstones in results if crash.')
448  parser.add_argument(
449      '-s', '--suite',
450      dest='suite_name', nargs='+', metavar='SUITE_NAME', required=True,
451      help='Executable name of the test suite to run.')
452  parser.add_argument(
453      '--test-apk-incremental-install-json',
454      type=os.path.realpath,
455      help='Path to install json for the test apk.')
456  parser.add_argument('--test-launcher-batch-limit',
457                      dest='test_launcher_batch_limit',
458                      type=int,
459                      help='The max number of tests to run in a shard. '
460                      'Ignores non-positive ints and those greater than '
461                      'MAX_SHARDS')
462  parser.add_argument(
463      '-w', '--wait-for-java-debugger', action='store_true',
464      help='Wait for java debugger to attach before running any application '
465           'code. Also disables test timeouts and sets retries=0.')
466  parser.add_argument(
467      '--coverage-dir',
468      type=os.path.realpath,
469      help='Directory in which to place all generated coverage files.')
470  parser.add_argument(
471      '--use-existing-test-data',
472      action='store_true',
473      help='Do not push new files to the device, instead using existing APK '
474      'and test data. Only use when running the same test for multiple '
475      'iterations.')
476  # This is currently only implemented for gtests tests.
477  parser.add_argument('--gtest_also_run_pre_tests',
478                      '--gtest-also-run-pre-tests',
479                      dest='run_pre_tests',
480                      action='store_true',
481                      help='Also run PRE_ tests if applicable.')
482
483
484def AddInstrumentationTestOptions(parser):
485  """Adds Instrumentation test options to |parser|."""
486
487  parser = parser.add_argument_group('instrumentation arguments')
488
489  parser.add_argument('--additional-apex',
490                      action='append',
491                      dest='additional_apexs',
492                      default=[],
493                      type=_RealPath,
494                      help='Additional apex that must be installed on '
495                      'the device when the tests are run')
496  parser.add_argument(
497      '--additional-apk',
498      action='append', dest='additional_apks', default=[],
499      type=_RealPath,
500      help='Additional apk that must be installed on '
501           'the device when the tests are run')
502  parser.add_argument('--forced-queryable-additional-apk',
503                      action='append',
504                      dest='forced_queryable_additional_apks',
505                      default=[],
506                      type=_RealPath,
507                      help='Configures an additional-apk to be forced '
508                      'to be queryable by other APKs.')
509  parser.add_argument('--instant-additional-apk',
510                      action='append',
511                      dest='instant_additional_apks',
512                      default=[],
513                      type=_RealPath,
514                      help='Configures an additional-apk to be an instant APK')
515  parser.add_argument(
516      '-A', '--annotation',
517      dest='annotation_str',
518      help='Comma-separated list of annotations. Run only tests with any of '
519           'the given annotations. An annotation can be either a key or a '
520           'key-values pair. A test that has no annotation is considered '
521           '"SmallTest".')
522  # TODO(jbudorick): Remove support for name-style APK specification once
523  # bots are no longer doing it.
524  parser.add_argument(
525      '--apk-under-test',
526      help='Path or name of the apk under test.')
527  parser.add_argument(
528      '--store-data-dependencies-in-temp',
529      action='store_true',
530      help='Store data dependencies in /data/local/tmp/chromium_tests_root')
531  parser.add_argument(
532      '--module',
533      action='append',
534      dest='modules',
535      help='Specify Android App Bundle modules to install in addition to the '
536      'base module.')
537  parser.add_argument(
538      '--fake-module',
539      action='append',
540      dest='fake_modules',
541      help='Specify Android App Bundle modules to fake install in addition to '
542      'the real modules.')
543  parser.add_argument(
544      '--additional-locale',
545      action='append',
546      dest='additional_locales',
547      help='Specify locales in addition to the device locale to install splits '
548      'for when --apk-under-test is an Android App Bundle.')
549  parser.add_argument(
550      '--coverage-dir',
551      type=os.path.realpath,
552      help='Directory in which to place all generated '
553      'Jacoco coverage files.')
554  parser.add_argument(
555      '--disable-dalvik-asserts',
556      dest='set_asserts', action='store_false', default=True,
557      help='Removes the dalvik.vm.enableassertions property')
558  parser.add_argument(
559      '--proguard-mapping-path',
560      help='.mapping file to use to Deobfuscate java stack traces in test '
561      'output and logcat.')
562  parser.add_argument(
563      '-E', '--exclude-annotation',
564      dest='exclude_annotation_str',
565      help='Comma-separated list of annotations. Exclude tests with these '
566           'annotations.')
567  parser.add_argument(
568      '--enable-breakpad-dump',
569      action='store_true',
570      help='Stores any breakpad dumps till the end of the test.')
571  parser.add_argument(
572      '--replace-system-package',
573      type=_RealPath,
574      default=None,
575      help='Use this apk to temporarily replace a system package with the same '
576      'package name.')
577  parser.add_argument(
578      '--remove-system-package',
579      default=[],
580      action='append',
581      dest='system_packages_to_remove',
582      help='Specifies a system package to remove before testing if it exists '
583      'on the system. WARNING: THIS WILL PERMANENTLY REMOVE THE SYSTEM APP. '
584      'Unlike --replace-system-package, the app will not be restored after '
585      'tests are finished.')
586  parser.add_argument(
587      '--use-voice-interaction-service',
588      help='This can be used to update the voice interaction service to be a '
589      'custom one. This is useful for mocking assistants. eg: '
590      'android.assist.service/.MainInteractionService')
591  parser.add_argument(
592      '--use-webview-provider',
593      type=_RealPath, default=None,
594      help='Use this apk as the webview provider during test. '
595           'The original provider will be restored if possible, '
596           "on Nougat the provider can't be determined and so "
597           'the system will choose the default provider.')
598  parser.add_argument(
599      '--webview-command-line-arg',
600      default=[],
601      action='append',
602      help="Specifies command line arguments to add to WebView's flag file")
603  parser.add_argument(
604      '--webview-process-mode',
605      choices=['single', 'multiple'],
606      help='Run WebView instrumentation tests only in the specified process '
607      'mode. If not set, both single and multiple process modes will execute.')
608  parser.add_argument(
609      '--run-setup-command',
610      default=[],
611      action='append',
612      dest='run_setup_commands',
613      help='This can be used to run a custom shell command on the device as a '
614      'setup step')
615  parser.add_argument(
616      '--run-teardown-command',
617      default=[],
618      action='append',
619      dest='run_teardown_commands',
620      help='This can be used to run a custom shell command on the device as a '
621      'teardown step')
622  parser.add_argument(
623      '--runtime-deps-path',
624      dest='runtime_deps_path', type=os.path.realpath,
625      help='Runtime data dependency file from GN.')
626  parser.add_argument(
627      '--screenshot-directory',
628      dest='screenshot_dir', type=os.path.realpath,
629      help='Capture screenshots of test failures')
630  parser.add_argument(
631      '--store-tombstones',
632      action='store_true', dest='store_tombstones',
633      help='Add tombstones in results if crash.')
634  parser.add_argument(
635      '--strict-mode',
636      dest='strict_mode', default='testing',
637      help='StrictMode command-line flag set on the device, '
638           'death/testing to kill the process, off to stop '
639           'checking, flash to flash only. (default: %(default)s)')
640  parser.add_argument(
641      '--test-apk',
642      required=True,
643      help='Path or name of the apk containing the tests.')
644  parser.add_argument(
645      '--test-apk-as-instant',
646      action='store_true',
647      help='Install the test apk as an instant app. '
648      'Instant apps run in a more restrictive execution environment.')
649  parser.add_argument(
650      '--test-launcher-batch-limit',
651      dest='test_launcher_batch_limit',
652      type=int,
653      help=('Not actually used for instrumentation tests, but can be used as '
654            'a proxy for determining if the current run is a retry without '
655            'patch.'))
656  parser.add_argument(
657      '--is-unit-test',
658      action='store_true',
659      help=('Specify the test suite as composed of unit tests, blocking '
660            'certain operations.'))
661  parser.add_argument(
662      '-w', '--wait-for-java-debugger', action='store_true',
663      help='Wait for java debugger to attach before running any application '
664           'code. Also disables test timeouts and sets retries=0.')
665
666  # WPR record mode.
667  parser.add_argument('--wpr-enable-record',
668                      action='store_true',
669                      default=False,
670                      help='If true, WPR server runs in record mode.'
671                      'otherwise, runs in replay mode.')
672
673  parser.add_argument(
674      '--approve-app-links',
675      help='Force enables Digital Asset Link verification for the provided '
676      'package and domain, example usage: --approve-app-links '
677      'com.android.package:www.example.com')
678
679  # These arguments are suppressed from the help text because they should
680  # only ever be specified by an intermediate script.
681  parser.add_argument(
682      '--apk-under-test-incremental-install-json',
683      help=argparse.SUPPRESS)
684  parser.add_argument(
685      '--test-apk-incremental-install-json',
686      type=os.path.realpath,
687      help=argparse.SUPPRESS)
688
689
690def AddSkiaGoldTestOptions(parser):
691  """Adds Skia Gold test options to |parser|."""
692  parser = parser.add_argument_group("Skia Gold arguments")
693  parser.add_argument(
694      '--code-review-system',
695      help='A non-default code review system to pass to pass to Gold, if '
696      'applicable')
697  parser.add_argument(
698      '--continuous-integration-system',
699      help='A non-default continuous integration system to pass to Gold, if '
700      'applicable')
701  parser.add_argument(
702      '--git-revision', help='The git commit currently being tested.')
703  parser.add_argument(
704      '--gerrit-issue',
705      help='The Gerrit issue this test is being run on, if applicable.')
706  parser.add_argument(
707      '--gerrit-patchset',
708      help='The Gerrit patchset this test is being run on, if applicable.')
709  parser.add_argument(
710      '--buildbucket-id',
711      help='The Buildbucket build ID that this test was triggered from, if '
712      'applicable.')
713  local_group = parser.add_mutually_exclusive_group()
714  local_group.add_argument(
715      '--local-pixel-tests',
716      action='store_true',
717      default=None,
718      help='Specifies to run the Skia Gold pixel tests in local mode. When run '
719      'in local mode, uploading to Gold is disabled and traditional '
720      'generated/golden/diff images are output instead of triage links. '
721      'Running in local mode also implies --no-luci-auth. If both this '
722      'and --no-local-pixel-tests are left unset, the test harness will '
723      'attempt to detect whether it is running on a workstation or not '
724      'and set the options accordingly.')
725  local_group.add_argument(
726      '--no-local-pixel-tests',
727      action='store_false',
728      dest='local_pixel_tests',
729      help='Specifies to run the Skia Gold pixel tests in non-local (bot) '
730      'mode. When run in this mode, data is actually uploaded to Gold and '
731      'triage links are generated. If both this and --local-pixel-tests '
732      'are left unset, the test harness will attempt to detect whether '
733      'it is running on a workstation or not and set the options '
734      'accordingly.')
735  parser.add_argument(
736      '--no-luci-auth',
737      action='store_true',
738      default=False,
739      help="Don't use the serve account provided by LUCI for authentication "
740      'with Skia Gold, instead relying on gsutil to be pre-authenticated. '
741      'Meant for testing locally instead of on the bots.')
742  parser.add_argument(
743      '--bypass-skia-gold-functionality',
744      action='store_true',
745      default=False,
746      help='Bypass all interaction with Skia Gold, effectively disabling the '
747      'image comparison portion of any tests that use Gold. Only meant to be '
748      'used in case a Gold outage occurs and cannot be fixed quickly.')
749
750
751def AddHostsideTestOptions(parser):
752  """Adds hostside test options to |parser|."""
753
754  parser = parser.add_argument_group('hostside arguments')
755
756  parser.add_argument(
757      '-s', '--test-suite', required=True,
758      help='Hostside test suite to run.')
759  parser.add_argument(
760      '--test-apk-as-instant',
761      action='store_true',
762      help='Install the test apk as an instant app. '
763      'Instant apps run in a more restrictive execution environment.')
764  parser.add_argument(
765      '--additional-apk',
766      action='append',
767      dest='additional_apks',
768      default=[],
769      type=_RealPath,
770      help='Additional apk that must be installed on '
771           'the device when the tests are run')
772  parser.add_argument(
773      '--use-webview-provider',
774      type=_RealPath, default=None,
775      help='Use this apk as the webview provider during test. '
776           'The original provider will be restored if possible, '
777           "on Nougat the provider can't be determined and so "
778           'the system will choose the default provider.')
779  parser.add_argument(
780      '--tradefed-executable',
781      type=_RealPath, default=None,
782      help='Location of the cts-tradefed script')
783  parser.add_argument(
784      '--tradefed-aapt-path',
785      type=_RealPath, default=None,
786      help='Location of the directory containing aapt binary')
787  parser.add_argument(
788      '--tradefed-adb-path',
789      type=_RealPath, default=None,
790      help='Location of the directory containing adb binary')
791  # The below arguments are not used, but allow us to pass the same arguments
792  # from run_cts.py regardless of type of run (instrumentation/hostside)
793  parser.add_argument(
794      '--apk-under-test',
795      help=argparse.SUPPRESS)
796  parser.add_argument(
797      '--use-apk-under-test-flags-file',
798      action='store_true',
799      help=argparse.SUPPRESS)
800  parser.add_argument(
801      '-E', '--exclude-annotation',
802      dest='exclude_annotation_str',
803      help=argparse.SUPPRESS)
804
805
806def AddJUnitTestOptions(parser):
807  """Adds junit test options to |parser|."""
808
809  parser = parser.add_argument_group('junit arguments')
810
811  parser.add_argument(
812      '--coverage-on-the-fly',
813      action='store_true',
814      help='Generate coverage data by Jacoco on-the-fly instrumentation.')
815  parser.add_argument(
816      '--coverage-dir', type=os.path.realpath,
817      help='Directory to store coverage info.')
818  parser.add_argument(
819      '--package-filter',
820      help='Filters tests by package.')
821  parser.add_argument(
822      '--runner-filter',
823      help='Filters tests by runner class. Must be fully qualified.')
824  parser.add_argument('--json-config',
825                      help='Runs only tests listed in this config.')
826  parser.add_argument(
827      '--shards',
828      type=int,
829      help='Number of shards to run junit tests in parallel on. Only 1 shard '
830      'is supported when test-filter is specified. Values less than 1 will '
831      'use auto select.')
832  parser.add_argument('--shard-filter',
833                      help='Comma separated list of shard indices to run.')
834  parser.add_argument(
835      '-s', '--test-suite', required=True,
836      help='JUnit test suite to run.')
837  debug_group = parser.add_mutually_exclusive_group()
838  debug_group.add_argument(
839      '-w', '--wait-for-java-debugger', action='store_const', const='8701',
840      dest='debug_socket', help='Alias for --debug-socket=8701')
841  debug_group.add_argument(
842      '--debug-socket',
843      help='Wait for java debugger to attach at specified socket address '
844           'before running any application code. Also disables test timeouts '
845           'and sets retries=0.')
846
847  # These arguments are for Android Robolectric tests.
848  parser.add_argument(
849      '--robolectric-runtime-deps-dir',
850      help='Path to runtime deps for Robolectric.')
851  parser.add_argument('--native-libs-dir',
852                      help='Path to search for native libraries.')
853  parser.add_argument(
854      '--resource-apk',
855      required=True,
856      help='Path to .ap_ containing binary resources for Robolectric.')
857  parser.add_argument('--shadows-allowlist',
858                      help='Path to Allowlist file for Shadows.')
859
860
861def AddLinkerTestOptions(parser):
862
863  parser = parser.add_argument_group('linker arguments')
864
865  parser.add_argument(
866      '--test-apk',
867      type=os.path.realpath,
868      help='Path to the linker test APK.')
869
870
871def AddMonkeyTestOptions(parser):
872  """Adds monkey test options to |parser|."""
873
874  parser = parser.add_argument_group('monkey arguments')
875
876  parser.add_argument('--browser',
877                      required=True,
878                      choices=list(constants.PACKAGE_INFO.keys()),
879                      metavar='BROWSER',
880                      help='Browser under test.')
881  parser.add_argument(
882      '--category',
883      nargs='*', dest='categories', default=[],
884      help='A list of allowed categories. Monkey will only visit activities '
885           'that are listed with one of the specified categories.')
886  parser.add_argument(
887      '--event-count',
888      default=10000, type=int,
889      help='Number of events to generate (default: %(default)s).')
890  parser.add_argument(
891      '--seed',
892      type=int,
893      help='Seed value for pseudo-random generator. Same seed value generates '
894           'the same sequence of events. Seed is randomized by default.')
895  parser.add_argument(
896      '--throttle',
897      default=100, type=int,
898      help='Delay between events (ms) (default: %(default)s). ')
899
900
901def AddPythonTestOptions(parser):
902
903  parser = parser.add_argument_group('python arguments')
904
905  parser.add_argument('-s',
906                      '--suite',
907                      dest='suite_name',
908                      metavar='SUITE_NAME',
909                      choices=list(constants.PYTHON_UNIT_TEST_SUITES.keys()),
910                      help='Name of the test suite to run.')
911
912
913def _CreateClassToFileNameDict(test_apk):
914  """Creates a dict mapping classes to file names from size-info apk."""
915  constants.CheckOutputDirectory()
916  test_apk_size_info = os.path.join(constants.GetOutDirectory(), 'size-info',
917                                    os.path.basename(test_apk) + '.jar.info')
918
919  class_to_file_dict = {}
920  # Some tests such as webview_cts_tests use a separately downloaded apk to run
921  # tests. This means the apk may not have been built by the system and hence
922  # no size info file exists.
923  if not os.path.exists(test_apk_size_info):
924    logging.debug('Apk size file not found. %s', test_apk_size_info)
925    return class_to_file_dict
926
927  with open(test_apk_size_info, 'r') as f:
928    for line in f:
929      file_class, file_name = line.rstrip().split(',', 1)
930      # Only want files that are not prebuilt.
931      if file_name.startswith('../../'):
932        class_to_file_dict[file_class] = str(
933            file_name.replace('../../', '//', 1))
934
935  return class_to_file_dict
936
937
938def _RunPythonTests(args):
939  """Subcommand of RunTestsCommand which runs python unit tests."""
940  suite_vars = constants.PYTHON_UNIT_TEST_SUITES[args.suite_name]
941  suite_path = suite_vars['path']
942  suite_test_modules = suite_vars['test_modules']
943
944  sys.path = [suite_path] + sys.path
945  try:
946    suite = unittest.TestSuite()
947    suite.addTests(unittest.defaultTestLoader.loadTestsFromName(m)
948                   for m in suite_test_modules)
949    runner = unittest.TextTestRunner(verbosity=1+args.verbose_count)
950    return 0 if runner.run(suite).wasSuccessful() else 1
951  finally:
952    sys.path = sys.path[1:]
953
954
955_DEFAULT_PLATFORM_MODE_TESTS = [
956    'gtest', 'hostside', 'instrumentation', 'junit', 'linker', 'monkey'
957]
958
959
960def RunTestsCommand(args, result_sink_client=None):
961  """Checks test type and dispatches to the appropriate function.
962
963  Args:
964    args: argparse.Namespace object.
965    result_sink_client: A ResultSinkClient object.
966
967  Returns:
968    Integer indicated exit code.
969
970  Raises:
971    Exception: Unknown command name passed in, or an exception from an
972        individual test runner.
973  """
974  command = args.command
975
976  ProcessCommonOptions(args)
977  logging.info('command: %s', ' '.join(sys.argv))
978  if args.enable_platform_mode or command in _DEFAULT_PLATFORM_MODE_TESTS:
979    return RunTestsInPlatformMode(args, result_sink_client)
980
981  if command == 'python':
982    return _RunPythonTests(args)
983  raise Exception('Unknown test type.')
984
985
986def _SinkTestResult(test_result, test_file_name, result_sink_client):
987  """Upload test result to result_sink.
988
989  Args:
990    test_result: A BaseTestResult object
991    test_file_name: A string representing the file location of the test
992    result_sink_client: A ResultSinkClient object
993
994  Returns:
995    N/A
996  """
997  # Some tests put in non utf-8 char as part of the test
998  # which breaks uploads, so need to decode and re-encode.
999  log_decoded = test_result.GetLog()
1000  if isinstance(log_decoded, bytes):
1001    log_decoded = log_decoded.decode('utf-8', 'replace')
1002  html_artifact = ''
1003  https_artifacts = []
1004  for link_name, link_url in sorted(test_result.GetLinks().items()):
1005    if link_url.startswith('https:'):
1006      https_artifacts.append('<li><a target="_blank" href=%s>%s</a></li>' %
1007                             (link_url, link_name))
1008    else:
1009      logging.info('Skipping non-https link %r (%s) for test %s.', link_name,
1010                   link_url, test_result.GetName())
1011  if https_artifacts:
1012    html_artifact += '<ul>%s</ul>' % '\n'.join(https_artifacts)
1013  result_sink_client.Post(test_result.GetNameForResultSink(),
1014                          test_result.GetType(),
1015                          test_result.GetDuration(),
1016                          log_decoded,
1017                          test_file_name,
1018                          variant=test_result.GetVariantForResultSink(),
1019                          failure_reason=test_result.GetFailureReason(),
1020                          html_artifact=html_artifact)
1021
1022
1023_SUPPORTED_IN_PLATFORM_MODE = [
1024  # TODO(jbudorick): Add support for more test types.
1025  'gtest',
1026  'hostside',
1027  'instrumentation',
1028  'junit',
1029  'linker',
1030  'monkey',
1031]
1032
1033
1034def RunTestsInPlatformMode(args, result_sink_client=None):
1035
1036  def infra_error(message):
1037    logging.fatal(message)
1038    sys.exit(constants.INFRA_EXIT_CODE)
1039
1040  if args.command not in _SUPPORTED_IN_PLATFORM_MODE:
1041    infra_error('%s is not yet supported in platform mode' % args.command)
1042
1043  ### Set up sigterm handler.
1044
1045  contexts_to_notify_on_sigterm = []
1046  def unexpected_sigterm(_signum, _frame):
1047    msg = [
1048      'Received SIGTERM. Shutting down.',
1049    ]
1050    for live_thread in threading.enumerate():
1051      # pylint: disable=protected-access
1052      thread_stack = ''.join(traceback.format_stack(
1053          sys._current_frames()[live_thread.ident]))
1054      msg.extend([
1055        'Thread "%s" (ident: %s) is currently running:' % (
1056            live_thread.name, live_thread.ident),
1057        thread_stack])
1058
1059    for context in contexts_to_notify_on_sigterm:
1060      context.ReceivedSigterm()
1061
1062    infra_error('\n'.join(msg))
1063
1064  signal.signal(signal.SIGTERM, unexpected_sigterm)
1065
1066  ### Set up results handling.
1067  # TODO(jbudorick): Rewrite results handling.
1068
1069  # all_raw_results is a list of lists of
1070  # base_test_result.TestRunResults objects. Each instance of
1071  # TestRunResults contains all test results produced by a single try,
1072  # while each list of TestRunResults contains all tries in a single
1073  # iteration.
1074  all_raw_results = []
1075
1076  # all_iteration_results is a list of base_test_result.TestRunResults
1077  # objects. Each instance of TestRunResults contains the last test
1078  # result for each test run in that iteration.
1079  all_iteration_results = []
1080
1081  global_results_tags = set()
1082
1083  json_file = tempfile.NamedTemporaryFile(delete=False)
1084  json_file.close()
1085
1086  @contextlib.contextmanager
1087  def json_finalizer():
1088    try:
1089      yield
1090    finally:
1091      if args.json_results_file and os.path.exists(json_file.name):
1092        shutil.move(json_file.name, args.json_results_file)
1093      elif args.isolated_script_test_output and os.path.exists(json_file.name):
1094        shutil.move(json_file.name, args.isolated_script_test_output)
1095      else:
1096        os.remove(json_file.name)
1097
1098  @contextlib.contextmanager
1099  def json_writer():
1100    try:
1101      yield
1102    except Exception:
1103      global_results_tags.add('UNRELIABLE_RESULTS')
1104      raise
1105    finally:
1106      if args.isolated_script_test_output:
1107        interrupted = 'UNRELIABLE_RESULTS' in global_results_tags
1108        json_results.GenerateJsonTestResultFormatFile(all_raw_results,
1109                                                      interrupted,
1110                                                      json_file.name,
1111                                                      indent=2)
1112      else:
1113        json_results.GenerateJsonResultsFile(
1114            all_raw_results,
1115            json_file.name,
1116            global_tags=list(global_results_tags),
1117            indent=2)
1118
1119      test_class_to_file_name_dict = {}
1120      # Test Location is only supported for instrumentation tests as it
1121      # requires the size-info file.
1122      if test_instance.TestType() == 'instrumentation':
1123        test_class_to_file_name_dict = _CreateClassToFileNameDict(args.test_apk)
1124
1125      if result_sink_client:
1126        for run in all_raw_results:
1127          for results in run:
1128            for r in results.GetAll():
1129              # Matches chrome.page_info.PageInfoViewTest#testChromePage
1130              match = re.search(r'^(.+\..+)#', r.GetName())
1131              test_file_name = test_class_to_file_name_dict.get(
1132                  match.group(1)) if match else None
1133              _SinkTestResult(r, test_file_name, result_sink_client)
1134
1135  @contextlib.contextmanager
1136  def upload_logcats_file():
1137    try:
1138      yield
1139    finally:
1140      if not args.logcat_output_file:
1141        logging.critical('Cannot upload logcat file: no file specified.')
1142      elif not os.path.exists(args.logcat_output_file):
1143        logging.critical("Cannot upload logcat file: file doesn't exist.")
1144      else:
1145        with open(args.logcat_output_file) as src:
1146          dst = logdog_helper.open_text('unified_logcats')
1147          if dst:
1148            shutil.copyfileobj(src, dst)
1149            dst.close()
1150            logging.critical(
1151                'Logcat: %s', logdog_helper.get_viewer_url('unified_logcats'))
1152
1153
1154  logcats_uploader = contextlib_ext.Optional(
1155      upload_logcats_file(),
1156      'upload_logcats_file' in args and args.upload_logcats_file)
1157
1158  save_detailed_results = (args.local_output or not local_utils.IsOnSwarming()
1159                           ) and not args.isolated_script_test_output
1160
1161  ### Set up test objects.
1162
1163  out_manager = output_manager_factory.CreateOutputManager(args)
1164  env = environment_factory.CreateEnvironment(
1165      args, out_manager, infra_error)
1166  test_instance = test_instance_factory.CreateTestInstance(args, infra_error)
1167  test_run = test_run_factory.CreateTestRun(env, test_instance, infra_error)
1168
1169  contexts_to_notify_on_sigterm.append(env)
1170  contexts_to_notify_on_sigterm.append(test_run)
1171
1172  if args.list_tests:
1173    try:
1174      with out_manager, env, test_instance, test_run:
1175        test_names = test_run.GetTestsForListing()
1176      print('There are {} tests:'.format(len(test_names)))
1177      for n in test_names:
1178        print(n)
1179      return 0
1180    except NotImplementedError:
1181      sys.stderr.write('Test does not support --list-tests (type={}).\n'.format(
1182          args.command))
1183      return 1
1184
1185  ### Run.
1186  with out_manager, json_finalizer():
1187    # |raw_logs_fh| is only used by Robolectric tests.
1188    raw_logs_fh = io.StringIO() if save_detailed_results else None
1189
1190    with json_writer(), logcats_uploader, env, test_instance, test_run:
1191
1192      repetitions = (range(args.repeat +
1193                           1) if args.repeat >= 0 else itertools.count())
1194      result_counts = collections.defaultdict(
1195          lambda: collections.defaultdict(int))
1196      iteration_count = 0
1197      for _ in repetitions:
1198        # raw_results will be populated with base_test_result.TestRunResults by
1199        # test_run.RunTests(). It is immediately added to all_raw_results so
1200        # that in the event of an exception, all_raw_results will already have
1201        # the up-to-date results and those can be written to disk.
1202        raw_results = []
1203        all_raw_results.append(raw_results)
1204
1205        test_run.RunTests(raw_results, raw_logs_fh=raw_logs_fh)
1206        if not raw_results:
1207          all_raw_results.pop()
1208          continue
1209
1210        iteration_results = base_test_result.TestRunResults()
1211        for r in reversed(raw_results):
1212          iteration_results.AddTestRunResults(r)
1213        all_iteration_results.append(iteration_results)
1214        iteration_count += 1
1215
1216        for r in iteration_results.GetAll():
1217          result_counts[r.GetName()][r.GetType()] += 1
1218
1219        report_results.LogFull(
1220            results=iteration_results,
1221            test_type=test_instance.TestType(),
1222            test_package=test_run.TestPackage(),
1223            annotation=getattr(args, 'annotations', None),
1224            flakiness_server=getattr(args, 'flakiness_dashboard_server',
1225                                     None))
1226
1227        failed_tests = (iteration_results.GetNotPass() -
1228                        iteration_results.GetSkip())
1229        if failed_tests:
1230          _LogRerunStatement(failed_tests, args.wrapper_script_args)
1231
1232        if args.break_on_failure and not iteration_results.DidRunPass():
1233          break
1234
1235      if iteration_count > 1:
1236        # display summary results
1237        # only display results for a test if at least one test did not pass
1238        all_pass = 0
1239        tot_tests = 0
1240        for test_name in result_counts:
1241          tot_tests += 1
1242          if any(result_counts[test_name][x] for x in (
1243              base_test_result.ResultType.FAIL,
1244              base_test_result.ResultType.CRASH,
1245              base_test_result.ResultType.TIMEOUT,
1246              base_test_result.ResultType.UNKNOWN)):
1247            logging.critical(
1248                '%s: %s',
1249                test_name,
1250                ', '.join('%s %s' % (str(result_counts[test_name][i]), i)
1251                          for i in base_test_result.ResultType.GetTypes()))
1252          else:
1253            all_pass += 1
1254
1255        logging.critical('%s of %s tests passed in all %s runs',
1256                         str(all_pass),
1257                         str(tot_tests),
1258                         str(iteration_count))
1259
1260    if save_detailed_results:
1261      assert raw_logs_fh
1262      raw_logs_fh.seek(0)
1263      raw_logs = raw_logs_fh.read()
1264      if raw_logs:
1265        with out_manager.ArchivedTempfile(
1266            'raw_logs.txt', 'raw_logs',
1267            output_manager.Datatype.TEXT) as raw_logs_file:
1268          raw_logs_file.write(raw_logs)
1269        logging.critical('RAW LOGS: %s', raw_logs_file.Link())
1270
1271      with out_manager.ArchivedTempfile(
1272          'test_results_presentation.html',
1273          'test_results_presentation',
1274          output_manager.Datatype.HTML) as results_detail_file:
1275        result_html_string, _, _ = test_results_presentation.result_details(
1276            json_path=json_file.name,
1277            test_name=args.command,
1278            cs_base_url='http://cs.chromium.org',
1279            local_output=True)
1280        results_detail_file.write(result_html_string)
1281        results_detail_file.flush()
1282      logging.critical('TEST RESULTS: %s', results_detail_file.Link())
1283
1284      ui_screenshots = test_results_presentation.ui_screenshot_set(
1285          json_file.name)
1286      if ui_screenshots:
1287        with out_manager.ArchivedTempfile(
1288            'ui_screenshots.json',
1289            'ui_capture',
1290            output_manager.Datatype.JSON) as ui_screenshot_file:
1291          ui_screenshot_file.write(ui_screenshots)
1292        logging.critical('UI Screenshots: %s', ui_screenshot_file.Link())
1293
1294  return (0 if all(r.DidRunPass() for r in all_iteration_results)
1295          else constants.ERROR_EXIT_CODE)
1296
1297
1298def _LogRerunStatement(failed_tests, wrapper_arg_str):
1299  """Logs a message that can rerun the failed tests.
1300
1301  Logs a copy/pasteable message that filters tests so just the failing tests
1302  are run.
1303
1304  Args:
1305    failed_tests: A set of test results that did not pass.
1306    wrapper_arg_str: A string of args that were passed to the called wrapper
1307        script.
1308  """
1309  rerun_arg_list = []
1310  try:
1311    constants.CheckOutputDirectory()
1312  # constants.CheckOutputDirectory throws bare exceptions.
1313  except:  # pylint: disable=bare-except
1314    logging.exception('Output directory not found. Unable to generate failing '
1315                      'test filter file.')
1316    return
1317
1318  output_directory = constants.GetOutDirectory()
1319  if not os.path.exists(output_directory):
1320    logging.error('Output directory not found. Unable to generate failing '
1321                  'test filter file.')
1322    return
1323
1324  test_filter_file = os.path.join(os.path.relpath(output_directory),
1325                                  _RERUN_FAILED_TESTS_FILE)
1326  arg_list = shlex.split(wrapper_arg_str) if wrapper_arg_str else sys.argv
1327  index = 0
1328  while index < len(arg_list):
1329    arg = arg_list[index]
1330    # Skip adding the filter=<file> and/or the filter arg as we're replacing
1331    # it with the new filter arg.
1332    # This covers --test-filter=, --test-launcher-filter-file=, --gtest-filter=,
1333    # --test-filter *Foobar.baz, -f *foobar, --package-filter <package>,
1334    # --runner-filter <runner>.
1335    if 'filter' in arg or arg == '-f':
1336      index += 1 if '=' in arg else 2
1337      continue
1338
1339    rerun_arg_list.append(arg)
1340    index += 1
1341
1342  failed_test_list = [str(t) for t in failed_tests]
1343  with open(test_filter_file, 'w') as fp:
1344    for t in failed_test_list:
1345      # Test result names can have # in them that don't match when applied as
1346      # a test name filter.
1347      fp.write('%s\n' % t.replace('#', '.'))
1348
1349  rerun_arg_list.append('--test-launcher-filter-file=%s' % test_filter_file)
1350  msg = """
1351    %d Test(s) failed.
1352    Rerun failed tests with copy and pastable command:
1353        %s
1354    """
1355  logging.critical(msg, len(failed_tests), shlex.join(rerun_arg_list))
1356
1357
1358def DumpThreadStacks(_signal, _frame):
1359  for thread in threading.enumerate():
1360    reraiser_thread.LogThreadStack(thread)
1361
1362
1363def main():
1364  signal.signal(signal.SIGUSR1, DumpThreadStacks)
1365
1366  parser = argparse.ArgumentParser()
1367  command_parsers = parser.add_subparsers(
1368      title='test types', dest='command')
1369
1370  subp = command_parsers.add_parser(
1371      'gtest',
1372      help='googletest-based C++ tests')
1373  AddCommonOptions(subp)
1374  AddDeviceOptions(subp)
1375  AddEmulatorOptions(subp)
1376  AddGTestOptions(subp)
1377  AddTracingOptions(subp)
1378  AddCommandLineOptions(subp)
1379
1380  subp = command_parsers.add_parser(
1381      'hostside',
1382      help='Webview CTS host-side tests')
1383  AddCommonOptions(subp)
1384  AddDeviceOptions(subp)
1385  AddEmulatorOptions(subp)
1386  AddHostsideTestOptions(subp)
1387
1388  subp = command_parsers.add_parser(
1389      'instrumentation',
1390      help='InstrumentationTestCase-based Java tests')
1391  AddCommonOptions(subp)
1392  AddDeviceOptions(subp)
1393  AddEmulatorOptions(subp)
1394  AddInstrumentationTestOptions(subp)
1395  AddSkiaGoldTestOptions(subp)
1396  AddTracingOptions(subp)
1397  AddCommandLineOptions(subp)
1398
1399  subp = command_parsers.add_parser(
1400      'junit',
1401      help='JUnit4-based Java tests')
1402  AddCommonOptions(subp)
1403  AddJUnitTestOptions(subp)
1404
1405  subp = command_parsers.add_parser(
1406      'linker',
1407      help='linker tests')
1408  AddCommonOptions(subp)
1409  AddDeviceOptions(subp)
1410  AddEmulatorOptions(subp)
1411  AddLinkerTestOptions(subp)
1412
1413  subp = command_parsers.add_parser(
1414      'monkey',
1415      help="tests based on Android's monkey command")
1416  AddCommonOptions(subp)
1417  AddDeviceOptions(subp)
1418  AddEmulatorOptions(subp)
1419  AddMonkeyTestOptions(subp)
1420
1421  subp = command_parsers.add_parser(
1422      'python',
1423      help='python tests based on unittest.TestCase')
1424  AddCommonOptions(subp)
1425  AddPythonTestOptions(subp)
1426
1427  args, unknown_args = parser.parse_known_args()
1428
1429  if unknown_args:
1430    if getattr(args, 'allow_unknown', None):
1431      args.command_line_flags = unknown_args
1432    else:
1433      parser.error('unrecognized arguments: %s' % ' '.join(unknown_args))
1434
1435  # --enable-concurrent-adb does not handle device reboots gracefully.
1436  if getattr(args, 'enable_concurrent_adb', None):
1437    if getattr(args, 'replace_system_package', None):
1438      logging.warning(
1439          'Ignoring --enable-concurrent-adb due to --replace-system-package')
1440      args.enable_concurrent_adb = False
1441    elif getattr(args, 'system_packages_to_remove', None):
1442      logging.warning(
1443          'Ignoring --enable-concurrent-adb due to --remove-system-package')
1444      args.enable_concurrent_adb = False
1445    elif getattr(args, 'use_webview_provider', None):
1446      logging.warning(
1447          'Ignoring --enable-concurrent-adb due to --use-webview-provider')
1448      args.enable_concurrent_adb = False
1449
1450  if (getattr(args, 'coverage_on_the_fly', False)
1451      and not getattr(args, 'coverage_dir', '')):
1452    parser.error('--coverage-on-the-fly requires --coverage-dir')
1453
1454  if (getattr(args, 'debug_socket', None)
1455      or getattr(args, 'wait_for_java_debugger', None)):
1456    args.num_retries = 0
1457
1458  # Result-sink may not exist in the environment if rdb stream is not enabled.
1459  result_sink_client = result_sink.TryInitClient()
1460
1461  try:
1462    return RunTestsCommand(args, result_sink_client)
1463  except base_error.BaseError as e:
1464    logging.exception('Error occurred.')
1465    if e.is_infra_error:
1466      return constants.INFRA_EXIT_CODE
1467    return constants.ERROR_EXIT_CODE
1468  except Exception:  # pylint: disable=W0703
1469    logging.exception('Unrecognized error occurred.')
1470    return constants.ERROR_EXIT_CODE
1471
1472
1473if __name__ == '__main__':
1474  exit_code = main()
1475  if exit_code == constants.INFRA_EXIT_CODE:
1476    # This exit code is returned in case of missing, unreachable,
1477    # or otherwise not fit for purpose test devices.
1478    # When this happens, the graceful cleanup triggered by sys.exit()
1479    # hangs indefinitely (on swarming - until it hits 20min timeout).
1480    # Skip cleanup (other than flushing output streams) and exit forcefully
1481    # to avoid the hang.
1482    sys.stdout.flush()
1483    sys.stderr.flush()
1484    os._exit(exit_code)  # pylint: disable=protected-access
1485  else:
1486    sys.exit(exit_code)
1487