1# Copyright 2014 The Chromium Authors 2# Use of this source code is governed by a BSD-style license that can be 3# found in the LICENSE file. 4 5 6import contextlib 7import collections 8import fnmatch 9import itertools 10import logging 11import math 12import os 13import posixpath 14import subprocess 15import shutil 16import time 17 18from six.moves import range # pylint: disable=redefined-builtin 19from devil import base_error 20from devil.android import crash_handler 21from devil.android import device_errors 22from devil.android import device_temp_file 23from devil.android import logcat_monitor 24from devil.android import ports 25from devil.android.sdk import version_codes 26from devil.utils import reraiser_thread 27from incremental_install import installer 28from pylib import constants 29from pylib.base import base_test_result 30from pylib.gtest import gtest_test_instance 31from pylib.local import local_test_server_spawner 32from pylib.local.device import local_device_environment 33from pylib.local.device import local_device_test_run 34from pylib.utils import google_storage_helper 35from pylib.utils import logdog_helper 36from py_trace_event import trace_event 37from py_utils import contextlib_ext 38from py_utils import tempfile_ext 39import tombstones 40 41_MAX_INLINE_FLAGS_LENGTH = 50 # Arbitrarily chosen. 42_EXTRA_COMMAND_LINE_FILE = ( 43 'org.chromium.native_test.NativeTest.CommandLineFile') 44_EXTRA_COMMAND_LINE_FLAGS = ( 45 'org.chromium.native_test.NativeTest.CommandLineFlags') 46_EXTRA_COVERAGE_DEVICE_FILE = ( 47 'org.chromium.native_test.NativeTest.CoverageDeviceFile') 48_EXTRA_STDOUT_FILE = ( 49 'org.chromium.native_test.NativeTestInstrumentationTestRunner' 50 '.StdoutFile') 51_EXTRA_TEST = ( 52 'org.chromium.native_test.NativeTestInstrumentationTestRunner' 53 '.Test') 54_EXTRA_TEST_LIST = ( 55 'org.chromium.native_test.NativeTestInstrumentationTestRunner' 56 '.TestList') 57 58# Used to identify the prefix in gtests. 59_GTEST_PRETEST_PREFIX = 'PRE_' 60 61_SECONDS_TO_NANOS = int(1e9) 62 63# Tests that use SpawnedTestServer must run the LocalTestServerSpawner on the 64# host machine. 65# TODO(jbudorick): Move this up to the test instance if the net test server is 66# handled outside of the APK for the remote_device environment. 67_SUITE_REQUIRES_TEST_SERVER_SPAWNER = [ 68 'components_browsertests', 'content_unittests', 'content_browsertests', 69 'net_unittests', 'services_unittests', 'unit_tests' 70] 71 72# These are use for code coverage. 73_LLVM_PROFDATA_PATH = os.path.join(constants.DIR_SOURCE_ROOT, 'third_party', 74 'llvm-build', 'Release+Asserts', 'bin', 75 'llvm-profdata') 76# Name of the file extension for profraw data files. 77_PROFRAW_FILE_EXTENSION = 'profraw' 78# Name of the file where profraw data files are merged. 79_MERGE_PROFDATA_FILE_NAME = 'coverage_merged.' + _PROFRAW_FILE_EXTENSION 80 81# No-op context manager. If we used Python 3, we could change this to 82# contextlib.ExitStack() 83class _NullContextManager: 84 def __enter__(self): 85 pass 86 def __exit__(self, *args): 87 pass 88 89 90def _GenerateSequentialFileNames(filename): 91 """Infinite generator of names: 'name.ext', 'name_1.ext', 'name_2.ext', ...""" 92 yield filename 93 base, ext = os.path.splitext(filename) 94 for i in itertools.count(1): 95 yield '%s_%d%s' % (base, i, ext) 96 97 98def _ExtractTestsFromFilters(gtest_filters): 99 """Returns the list of tests specified by the given filters. 100 101 Returns: 102 None if the device should be queried for the test list instead. 103 """ 104 # - means exclude filter. 105 for gtest_filter in gtest_filters: 106 if '-' in gtest_filter: 107 return None 108 # Empty means all tests 109 if not any(gtest_filters): 110 return None 111 112 if len(gtest_filters) == 1: 113 patterns = gtest_filters[0].split(':') 114 # For a single pattern, allow it even if it has a wildcard so long as the 115 # wildcard comes at the end and there is at least one . to prove the scope 116 # is not too large. 117 # This heuristic is not necessarily faster, but normally is. 118 if len(patterns) == 1 and patterns[0].endswith('*'): 119 no_suffix = patterns[0].rstrip('*') 120 if '*' not in no_suffix and '.' in no_suffix: 121 return patterns 122 123 all_patterns = set(gtest_filters[0].split(':')) 124 for gtest_filter in gtest_filters: 125 patterns = gtest_filter.split(':') 126 for pattern in patterns: 127 if '*' in pattern: 128 return None 129 all_patterns = all_patterns.intersection(set(patterns)) 130 return list(all_patterns) 131 132 133def _GetDeviceTimeoutMultiplier(): 134 # Emulated devices typically run 20-150x slower than real-time. 135 # Give a way to control this through the DEVICE_TIMEOUT_MULTIPLIER 136 # environment variable. 137 multiplier = os.getenv("DEVICE_TIMEOUT_MULTIPLIER") 138 if multiplier: 139 return int(multiplier) 140 return 1 141 142 143def _MergeCoverageFiles(coverage_dir, profdata_dir): 144 """Merge coverage data files. 145 146 Each instrumentation activity generates a separate profraw data file. This 147 merges all profraw files in profdata_dir into a single file in 148 coverage_dir. This happens after each test, rather than waiting until after 149 all tests are ran to reduce the memory footprint used by all the profraw 150 files. 151 152 Args: 153 coverage_dir: The path to the coverage directory. 154 profdata_dir: The directory where the profraw data file(s) are located. 155 156 Return: 157 None 158 """ 159 # profdata_dir may not exist if pulling coverage files failed. 160 if not os.path.exists(profdata_dir): 161 logging.debug('Profraw directory does not exist.') 162 return 163 164 merge_file = os.path.join(coverage_dir, _MERGE_PROFDATA_FILE_NAME) 165 profraw_files = [ 166 os.path.join(profdata_dir, f) for f in os.listdir(profdata_dir) 167 if f.endswith(_PROFRAW_FILE_EXTENSION) 168 ] 169 170 try: 171 logging.debug('Merging target profraw files into merged profraw file.') 172 subprocess_cmd = [ 173 _LLVM_PROFDATA_PATH, 174 'merge', 175 '-o', 176 merge_file, 177 '-sparse=true', 178 ] 179 # Grow the merge file by merging it with itself and the new files. 180 if os.path.exists(merge_file): 181 subprocess_cmd.append(merge_file) 182 subprocess_cmd.extend(profraw_files) 183 output = subprocess.check_output(subprocess_cmd) 184 logging.debug('Merge output: %s', output) 185 except subprocess.CalledProcessError: 186 # Don't raise error as that will kill the test run. When code coverage 187 # generates a report, that will raise the error in the report generation. 188 logging.error( 189 'Failed to merge target profdata files to create merged profraw file.') 190 191 # Free up memory space on bot as all data is in the merge file. 192 for f in profraw_files: 193 os.remove(f) 194 195 196def _PullCoverageFiles(device, device_coverage_dir, output_dir): 197 """Pulls coverage files on device to host directory. 198 199 Args: 200 device: The working device. 201 device_coverage_dir: The directory to store coverage data on device. 202 output_dir: The output directory on host. 203 """ 204 try: 205 if not os.path.exists(output_dir): 206 os.makedirs(output_dir) 207 device.PullFile(device_coverage_dir, output_dir) 208 if not os.listdir(os.path.join(output_dir, 'profraw')): 209 logging.warning('No coverage data was generated for this run') 210 except (OSError, base_error.BaseError) as e: 211 logging.warning('Failed to handle coverage data after tests: %s', e) 212 finally: 213 device.RemovePath(device_coverage_dir, force=True, recursive=True) 214 215 216def _GetDeviceCoverageDir(device): 217 """Gets the directory to generate coverage data on device. 218 219 Args: 220 device: The working device. 221 222 Returns: 223 The directory path on the device. 224 """ 225 return posixpath.join(device.GetExternalStoragePath(), 'chrome', 'test', 226 'coverage', 'profraw') 227 228 229def _GetLLVMProfilePath(device_coverage_dir, suite, coverage_index): 230 """Gets 'LLVM_PROFILE_FILE' environment variable path. 231 232 Dumping data to ONLY 1 file may cause warning and data overwrite in 233 browsertests, so that pattern "%2m" is used to expand to 2 raw profiles 234 at runtime. 235 236 Args: 237 device_coverage_dir: The directory to generate data on device. 238 suite: Test suite name. 239 coverage_index: The incremental index for this test suite. 240 241 Returns: 242 The path pattern for environment variable 'LLVM_PROFILE_FILE'. 243 """ 244 return posixpath.join(device_coverage_dir, 245 '_'.join([suite, 246 str(coverage_index), '%2m.profraw'])) 247 248 249class _ApkDelegate: 250 def __init__(self, test_instance, tool): 251 self._activity = test_instance.activity 252 self._apk_helper = test_instance.apk_helper 253 self._test_apk_incremental_install_json = ( 254 test_instance.test_apk_incremental_install_json) 255 self._package = test_instance.package 256 self._runner = test_instance.runner 257 self._permissions = test_instance.permissions 258 self._suite = test_instance.suite 259 self._component = '%s/%s' % (self._package, self._runner) 260 self._extras = test_instance.extras 261 self._wait_for_java_debugger = test_instance.wait_for_java_debugger 262 self._tool = tool 263 self._coverage_dir = test_instance.coverage_dir 264 self._coverage_index = 0 265 self._use_existing_test_data = test_instance.use_existing_test_data 266 267 def GetTestDataRoot(self, device): 268 # pylint: disable=no-self-use 269 return posixpath.join(device.GetExternalStoragePath(), 270 'chromium_tests_root') 271 272 def Install(self, device): 273 if self._use_existing_test_data: 274 return 275 if self._test_apk_incremental_install_json: 276 installer.Install(device, self._test_apk_incremental_install_json, 277 apk=self._apk_helper, permissions=self._permissions) 278 else: 279 device.Install( 280 self._apk_helper, 281 allow_downgrade=True, 282 reinstall=True, 283 permissions=self._permissions) 284 285 def ResultsDirectory(self, device): # pylint: disable=no-self-use 286 return device.GetExternalStoragePath() 287 288 def Run(self, test, device, flags=None, **kwargs): 289 extras = dict(self._extras) 290 device_api = device.build_version_sdk 291 292 if self._coverage_dir and device_api >= version_codes.LOLLIPOP: 293 device_coverage_dir = _GetDeviceCoverageDir(device) 294 extras[_EXTRA_COVERAGE_DEVICE_FILE] = _GetLLVMProfilePath( 295 device_coverage_dir, self._suite, self._coverage_index) 296 self._coverage_index += 1 297 298 if ('timeout' in kwargs 299 and gtest_test_instance.EXTRA_SHARD_NANO_TIMEOUT not in extras): 300 # Make sure the instrumentation doesn't kill the test before the 301 # scripts do. The provided timeout value is in seconds, but the 302 # instrumentation deals with nanoseconds because that's how Android 303 # handles time. 304 extras[gtest_test_instance.EXTRA_SHARD_NANO_TIMEOUT] = int( 305 kwargs['timeout'] * _SECONDS_TO_NANOS) 306 307 command_line_file = _NullContextManager() 308 if flags: 309 if len(flags) > _MAX_INLINE_FLAGS_LENGTH: 310 command_line_file = device_temp_file.DeviceTempFile(device.adb) 311 device.WriteFile(command_line_file.name, '_ %s' % flags) 312 extras[_EXTRA_COMMAND_LINE_FILE] = command_line_file.name 313 else: 314 extras[_EXTRA_COMMAND_LINE_FLAGS] = flags 315 316 test_list_file = _NullContextManager() 317 if test: 318 if len(test) > 1: 319 test_list_file = device_temp_file.DeviceTempFile(device.adb) 320 device.WriteFile(test_list_file.name, '\n'.join(test)) 321 extras[_EXTRA_TEST_LIST] = test_list_file.name 322 else: 323 extras[_EXTRA_TEST] = test[0] 324 325 # We need to use GetAppWritablePath here instead of GetExternalStoragePath 326 # since we will not have yet applied legacy storage permission workarounds 327 # on R+. 328 stdout_file = device_temp_file.DeviceTempFile( 329 device.adb, dir=device.GetAppWritablePath(), suffix='.gtest_out') 330 extras[_EXTRA_STDOUT_FILE] = stdout_file.name 331 332 if self._wait_for_java_debugger: 333 cmd = ['am', 'set-debug-app', '-w', self._package] 334 device.RunShellCommand(cmd, check_return=True) 335 logging.warning('*' * 80) 336 logging.warning('Waiting for debugger to attach to process: %s', 337 self._package) 338 logging.warning('*' * 80) 339 340 with command_line_file, test_list_file, stdout_file: 341 try: 342 device.StartInstrumentation( 343 self._component, extras=extras, raw=False, **kwargs) 344 except device_errors.CommandFailedError: 345 logging.exception('gtest shard failed.') 346 except device_errors.CommandTimeoutError: 347 logging.exception('gtest shard timed out.') 348 except device_errors.DeviceUnreachableError: 349 logging.exception('gtest shard device unreachable.') 350 except Exception: 351 device.ForceStop(self._package) 352 raise 353 finally: 354 if self._coverage_dir and device_api >= version_codes.LOLLIPOP: 355 if not os.path.isdir(self._coverage_dir): 356 os.makedirs(self._coverage_dir) 357 # TODO(crbug.com/1179004) Use _MergeCoverageFiles when llvm-profdata 358 # not found is fixed. 359 _PullCoverageFiles( 360 device, device_coverage_dir, 361 os.path.join(self._coverage_dir, str(self._coverage_index))) 362 363 return device.ReadFile(stdout_file.name).splitlines() 364 365 def PullAppFiles(self, device, files, directory): 366 device_dir = device.GetApplicationDataDirectory(self._package) 367 host_dir = os.path.join(directory, str(device)) 368 for f in files: 369 device_file = posixpath.join(device_dir, f) 370 host_file = os.path.join(host_dir, *f.split(posixpath.sep)) 371 for host_file in _GenerateSequentialFileNames(host_file): 372 if not os.path.exists(host_file): 373 break 374 device.PullFile(device_file, host_file) 375 376 def Clear(self, device): 377 device.ClearApplicationState(self._package, permissions=self._permissions) 378 379 380class _ExeDelegate: 381 382 def __init__(self, tr, test_instance, tool): 383 self._host_dist_dir = test_instance.exe_dist_dir 384 self._exe_file_name = os.path.basename( 385 test_instance.exe_dist_dir)[:-len('__dist')] 386 self._device_dist_dir = posixpath.join( 387 constants.TEST_EXECUTABLE_DIR, 388 os.path.basename(test_instance.exe_dist_dir)) 389 self._test_run = tr 390 self._tool = tool 391 self._suite = test_instance.suite 392 self._coverage_dir = test_instance.coverage_dir 393 self._coverage_index = 0 394 395 def GetTestDataRoot(self, device): 396 # pylint: disable=no-self-use 397 # pylint: disable=unused-argument 398 return posixpath.join(constants.TEST_EXECUTABLE_DIR, 'chromium_tests_root') 399 400 def Install(self, device): 401 # TODO(jbudorick): Look into merging this with normal data deps pushing if 402 # executables become supported on nonlocal environments. 403 device.PushChangedFiles([(self._host_dist_dir, self._device_dist_dir)], 404 delete_device_stale=True) 405 406 def ResultsDirectory(self, device): 407 # pylint: disable=no-self-use 408 # pylint: disable=unused-argument 409 return constants.TEST_EXECUTABLE_DIR 410 411 def Run(self, test, device, flags=None, **kwargs): 412 tool = self._test_run.GetTool(device).GetTestWrapper() 413 if tool: 414 cmd = [tool] 415 else: 416 cmd = [] 417 cmd.append(posixpath.join(self._device_dist_dir, self._exe_file_name)) 418 419 if test: 420 cmd.append('--gtest_filter=%s' % ':'.join(test)) 421 if flags: 422 # TODO(agrieve): This won't work if multiple flags are passed. 423 cmd.append(flags) 424 cwd = constants.TEST_EXECUTABLE_DIR 425 426 env = { 427 'LD_LIBRARY_PATH': self._device_dist_dir 428 } 429 430 if self._coverage_dir: 431 device_coverage_dir = _GetDeviceCoverageDir(device) 432 env['LLVM_PROFILE_FILE'] = _GetLLVMProfilePath( 433 device_coverage_dir, self._suite, self._coverage_index) 434 self._coverage_index += 1 435 436 if self._tool != 'asan': 437 env['UBSAN_OPTIONS'] = constants.UBSAN_OPTIONS 438 439 try: 440 gcov_strip_depth = os.environ['NATIVE_COVERAGE_DEPTH_STRIP'] 441 external = device.GetExternalStoragePath() 442 env['GCOV_PREFIX'] = '%s/gcov' % external 443 env['GCOV_PREFIX_STRIP'] = gcov_strip_depth 444 except (device_errors.CommandFailedError, KeyError): 445 pass 446 447 # Executable tests return a nonzero exit code on test failure, which is 448 # fine from the test runner's perspective; thus check_return=False. 449 output = device.RunShellCommand( 450 cmd, cwd=cwd, env=env, check_return=False, large_output=True, **kwargs) 451 452 if self._coverage_dir: 453 _PullCoverageFiles( 454 device, device_coverage_dir, 455 os.path.join(self._coverage_dir, str(self._coverage_index))) 456 457 return output 458 459 def PullAppFiles(self, device, files, directory): 460 pass 461 462 def Clear(self, device): 463 device.KillAll(self._exe_file_name, 464 blocking=True, 465 timeout=30 * _GetDeviceTimeoutMultiplier(), 466 quiet=True) 467 468 469class LocalDeviceGtestRun(local_device_test_run.LocalDeviceTestRun): 470 471 def __init__(self, env, test_instance): 472 assert isinstance(env, local_device_environment.LocalDeviceEnvironment) 473 assert isinstance(test_instance, gtest_test_instance.GtestTestInstance) 474 super().__init__(env, test_instance) 475 476 if self._test_instance.apk_helper: 477 self._installed_packages = [ 478 self._test_instance.apk_helper.GetPackageName() 479 ] 480 481 if self._test_instance.apk: 482 self._delegate = _ApkDelegate(self._test_instance, env.tool) 483 elif self._test_instance.exe_dist_dir: 484 self._delegate = _ExeDelegate(self, self._test_instance, self._env.tool) 485 if self._test_instance.isolated_script_test_perf_output: 486 self._test_perf_output_filenames = _GenerateSequentialFileNames( 487 self._test_instance.isolated_script_test_perf_output) 488 else: 489 self._test_perf_output_filenames = itertools.repeat(None) 490 self._crashes = set() 491 self._servers = collections.defaultdict(list) 492 493 #override 494 def TestPackage(self): 495 return self._test_instance.suite 496 497 #override 498 def SetUp(self): 499 @local_device_environment.handle_shard_failures_with( 500 on_failure=self._env.DenylistDevice) 501 @trace_event.traced 502 def individual_device_set_up(device, host_device_tuples): 503 def install_apk(dev): 504 # Install test APK. 505 self._delegate.Install(dev) 506 507 def push_test_data(dev): 508 if self._test_instance.use_existing_test_data: 509 return 510 # Push data dependencies. 511 device_root = self._delegate.GetTestDataRoot(dev) 512 host_device_tuples_substituted = [ 513 (h, local_device_test_run.SubstituteDeviceRoot(d, device_root)) 514 for h, d in host_device_tuples] 515 local_device_environment.place_nomedia_on_device(dev, device_root) 516 dev.PushChangedFiles( 517 host_device_tuples_substituted, 518 delete_device_stale=True, 519 # Some gtest suites, e.g. unit_tests, have data dependencies that 520 # can take longer than the default timeout to push. See 521 # crbug.com/791632 for context. 522 timeout=600 * math.ceil(_GetDeviceTimeoutMultiplier() / 10)) 523 if not host_device_tuples: 524 dev.RemovePath(device_root, force=True, recursive=True, rename=True) 525 dev.RunShellCommand(['mkdir', '-p', device_root], check_return=True) 526 527 def init_tool_and_start_servers(dev): 528 tool = self.GetTool(dev) 529 tool.CopyFiles(dev) 530 tool.SetupEnvironment() 531 532 if self._env.disable_test_server: 533 logging.warning('Not starting test server. Some tests may fail.') 534 return 535 536 try: 537 # See https://crbug.com/1030827. 538 # This is a hack that may break in the future. We're relying on the 539 # fact that adb doesn't use ipv6 for it's server, and so doesn't 540 # listen on ipv6, but ssh remote forwarding does. 5037 is the port 541 # number adb uses for its server. 542 if b"[::1]:5037" in subprocess.check_output( 543 "ss -o state listening 'sport = 5037'", shell=True): 544 logging.error( 545 'Test Server cannot be started with a remote-forwarded adb ' 546 'server. Continuing anyways, but some tests may fail.') 547 return 548 except subprocess.CalledProcessError: 549 pass 550 551 self._servers[str(dev)] = [] 552 if self.TestPackage() in _SUITE_REQUIRES_TEST_SERVER_SPAWNER: 553 self._servers[str(dev)].append( 554 local_test_server_spawner.LocalTestServerSpawner( 555 ports.AllocateTestServerPort(), dev, tool)) 556 557 for s in self._servers[str(dev)]: 558 s.SetUp() 559 560 def bind_crash_handler(step, dev): 561 return lambda: crash_handler.RetryOnSystemCrash(step, dev) 562 563 steps = [ 564 bind_crash_handler(s, device) 565 for s in (install_apk, push_test_data, init_tool_and_start_servers)] 566 if self._env.concurrent_adb: 567 reraiser_thread.RunAsync(steps) 568 else: 569 for step in steps: 570 step() 571 572 self._env.parallel_devices.pMap( 573 individual_device_set_up, 574 self._test_instance.GetDataDependencies()) 575 576 #override 577 def _ShouldShardTestsForDevices(self): 578 """Shard tests across several devices. 579 580 Returns: 581 True if tests should be sharded across several devices, 582 False otherwise. 583 """ 584 return True 585 586 #override 587 def _CreateShardsForDevices(self, tests): 588 """Create shards of tests to run on devices. 589 590 Args: 591 tests: List containing tests or test batches. 592 593 Returns: 594 List of test batches. 595 """ 596 # _crashes are tests that might crash and make the tests in the same shard 597 # following the crashed testcase not run. 598 # Thus we need to create separate shards for each crashed testcase, 599 # so that other tests can be run. 600 device_count = len(self._env.devices) 601 shards = [] 602 603 # Add shards with only one suspect testcase. 604 shards += [[crash] for crash in self._crashes if crash in tests] 605 606 # Delete suspect testcase from tests. 607 tests = [test for test in tests if not test in self._crashes] 608 609 # Sort tests by hash. 610 # TODO(crbug.com/1257820): Add sorting logic back to _PartitionTests. 611 tests = self._SortTests(tests) 612 613 max_shard_size = self._test_instance.test_launcher_batch_limit 614 615 shards.extend(self._PartitionTests(tests, device_count, max_shard_size)) 616 return shards 617 618 #override 619 def _GetTests(self): 620 if self._test_instance.extract_test_list_from_filter: 621 # When the exact list of tests to run is given via command-line (e.g. when 622 # locally iterating on a specific test), skip querying the device (which 623 # takes ~3 seconds). 624 tests = _ExtractTestsFromFilters(self._test_instance.gtest_filters) 625 if tests: 626 return tests 627 628 # Even when there's only one device, it still makes sense to retrieve the 629 # test list so that tests can be split up and run in batches rather than all 630 # at once (since test output is not streamed). 631 @local_device_environment.handle_shard_failures_with( 632 on_failure=self._env.DenylistDevice) 633 def list_tests(dev): 634 timeout = 30 * _GetDeviceTimeoutMultiplier() 635 retries = 1 636 if self._test_instance.wait_for_java_debugger: 637 timeout = None 638 639 flags = [ 640 f for f in self._test_instance.flags if f not in [ 641 '--wait-for-debugger', '--wait-for-java-debugger', 642 '--gtest_also_run_disabled_tests' 643 ] 644 ] 645 flags.append('--gtest_list_tests') 646 647 # TODO(crbug.com/726880): Remove retries when no longer necessary. 648 for i in range(0, retries+1): 649 logging.info('flags:') 650 for f in flags: 651 logging.info(' %s', f) 652 653 with self._ArchiveLogcat(dev, 'list_tests'): 654 raw_test_list = crash_handler.RetryOnSystemCrash( 655 lambda d: self._delegate.Run( 656 None, d, flags=' '.join(flags), timeout=timeout), 657 device=dev) 658 659 tests = gtest_test_instance.ParseGTestListTests(raw_test_list) 660 if not tests: 661 logging.info('No tests found. Output:') 662 for l in raw_test_list: 663 logging.info(' %s', l) 664 if i < retries: 665 logging.info('Retrying...') 666 else: 667 break 668 return tests 669 670 # Query all devices in case one fails. 671 test_lists = self._env.parallel_devices.pMap(list_tests).pGet(None) 672 673 # If all devices failed to list tests, raise an exception. 674 # Check that tl is not None and is not empty. 675 if all(not tl for tl in test_lists): 676 raise device_errors.CommandFailedError( 677 'Failed to list tests on any device') 678 tests = list(sorted(set().union(*[set(tl) for tl in test_lists if tl]))) 679 tests = self._test_instance.FilterTests(tests) 680 tests = self._ApplyExternalSharding( 681 tests, self._test_instance.external_shard_index, 682 self._test_instance.total_external_shards) 683 return tests 684 685 #override 686 def _GroupTests(self, tests): 687 pre_tests = dict() 688 other_tests = [] 689 for test in tests: 690 test_name_start = max(test.find('.') + 1, 0) 691 test_name = test[test_name_start:] 692 if test_name_start == 0 or not test_name.startswith( 693 _GTEST_PRETEST_PREFIX): 694 other_tests.append(test) 695 else: 696 test_suite = test[:test_name_start - 1] 697 trim_test = test 698 trim_tests = [test] 699 700 while test_name.startswith(_GTEST_PRETEST_PREFIX): 701 test_name = test_name[len(_GTEST_PRETEST_PREFIX):] 702 trim_test = '%s.%s' % (test_suite, test_name) 703 trim_tests.append(trim_test) 704 705 if not trim_test in pre_tests or len( 706 pre_tests[trim_test]) < len(trim_tests): 707 pre_tests[trim_test] = trim_tests 708 709 all_tests = [] 710 for other_test in other_tests: 711 if not other_test in pre_tests: 712 all_tests.append(other_test) 713 714 # TODO(crbug.com/1257820): Add logic to support grouping tests. 715 # Once grouping logic is added, switch to 'append' from 'extend'. 716 for _, test_list in pre_tests.items(): 717 all_tests.extend(test_list) 718 719 return all_tests 720 721 def _UploadTestArtifacts(self, device, test_artifacts_dir): 722 # TODO(jbudorick): Reconcile this with the output manager once 723 # https://codereview.chromium.org/2933993002/ lands. 724 if test_artifacts_dir: 725 with tempfile_ext.NamedTemporaryDirectory() as test_artifacts_host_dir: 726 device.PullFile(test_artifacts_dir.name, test_artifacts_host_dir) 727 with tempfile_ext.NamedTemporaryDirectory() as temp_zip_dir: 728 zip_base_name = os.path.join(temp_zip_dir, 'test_artifacts') 729 test_artifacts_zip = shutil.make_archive( 730 zip_base_name, 'zip', test_artifacts_host_dir) 731 link = google_storage_helper.upload( 732 google_storage_helper.unique_name( 733 'test_artifacts', device=device), 734 test_artifacts_zip, 735 bucket='%s/test_artifacts' % ( 736 self._test_instance.gs_test_artifacts_bucket)) 737 logging.info('Uploading test artifacts to %s.', link) 738 return link 739 return None 740 741 def _PullRenderTestOutput(self, device, render_test_output_device_dir): 742 # We pull the render tests into a temp directory then copy them over 743 # individually. Otherwise we end up with a temporary directory name 744 # in the host output directory. 745 with tempfile_ext.NamedTemporaryDirectory() as tmp_host_dir: 746 try: 747 device.PullFile(render_test_output_device_dir, tmp_host_dir) 748 except device_errors.CommandFailedError: 749 logging.exception('Failed to pull render test output dir %s', 750 render_test_output_device_dir) 751 temp_host_dir = os.path.join( 752 tmp_host_dir, os.path.basename(render_test_output_device_dir)) 753 for output_file in os.listdir(temp_host_dir): 754 src_path = os.path.join(temp_host_dir, output_file) 755 dst_path = os.path.join(self._test_instance.render_test_output_dir, 756 output_file) 757 shutil.move(src_path, dst_path) 758 759 @contextlib.contextmanager 760 def _ArchiveLogcat(self, device, test): 761 if isinstance(test, str): 762 desc = test 763 else: 764 desc = hash(tuple(test)) 765 766 stream_name = 'logcat_%s_shard%s_%s_%s' % ( 767 desc, self._test_instance.external_shard_index, 768 time.strftime('%Y%m%dT%H%M%S-UTC', time.gmtime()), device.serial) 769 770 logcat_file = None 771 logmon = None 772 try: 773 with self._env.output_manager.ArchivedTempfile(stream_name, 774 'logcat') as logcat_file: 775 with logcat_monitor.LogcatMonitor( 776 device.adb, 777 filter_specs=local_device_environment.LOGCAT_FILTERS, 778 output_file=logcat_file.name, 779 check_error=False) as logmon: 780 with contextlib_ext.Optional(trace_event.trace(str(test)), 781 self._env.trace_output): 782 yield logcat_file 783 finally: 784 if logmon: 785 logmon.Close() 786 if logcat_file and logcat_file.Link(): 787 logging.critical('Logcat saved to %s', logcat_file.Link()) 788 789 #override 790 def _RunTest(self, device, test): 791 # Run the test. 792 timeout = (self._test_instance.shard_timeout * 793 self.GetTool(device).GetTimeoutScale() * 794 _GetDeviceTimeoutMultiplier()) 795 if self._test_instance.wait_for_java_debugger: 796 timeout = None 797 if self._test_instance.store_tombstones: 798 tombstones.ClearAllTombstones(device) 799 test_perf_output_filename = next(self._test_perf_output_filenames) 800 801 if self._test_instance.isolated_script_test_output: 802 suffix = '.json' 803 else: 804 suffix = '.xml' 805 806 with device_temp_file.DeviceTempFile( 807 adb=device.adb, 808 dir=self._delegate.ResultsDirectory(device), 809 suffix=suffix) as device_tmp_results_file: 810 with contextlib_ext.Optional( 811 device_temp_file.NamedDeviceTemporaryDirectory( 812 adb=device.adb, dir='/sdcard/'), 813 self._test_instance.gs_test_artifacts_bucket) as test_artifacts_dir: 814 with (contextlib_ext.Optional( 815 device_temp_file.DeviceTempFile( 816 adb=device.adb, dir=self._delegate.ResultsDirectory(device)), 817 test_perf_output_filename)) as isolated_script_test_perf_output: 818 with contextlib_ext.Optional( 819 device_temp_file.NamedDeviceTemporaryDirectory(adb=device.adb, 820 dir='/sdcard/'), 821 self._test_instance.render_test_output_dir 822 ) as render_test_output_dir: 823 824 flags = list(self._test_instance.flags) 825 if self._test_instance.enable_xml_result_parsing: 826 flags.append('--gtest_output=xml:%s' % 827 device_tmp_results_file.name) 828 829 if self._test_instance.gs_test_artifacts_bucket: 830 flags.append('--test_artifacts_dir=%s' % test_artifacts_dir.name) 831 832 if self._test_instance.isolated_script_test_output: 833 flags.append('--isolated-script-test-output=%s' % 834 device_tmp_results_file.name) 835 836 if test_perf_output_filename: 837 flags.append('--isolated_script_test_perf_output=%s' % 838 isolated_script_test_perf_output.name) 839 840 if self._test_instance.render_test_output_dir: 841 flags.append('--render-test-output-dir=%s' % 842 render_test_output_dir.name) 843 844 logging.info('flags:') 845 for f in flags: 846 logging.info(' %s', f) 847 848 with self._ArchiveLogcat(device, test) as logcat_file: 849 output = self._delegate.Run(test, 850 device, 851 flags=' '.join(flags), 852 timeout=timeout, 853 retries=0) 854 855 if self._test_instance.enable_xml_result_parsing: 856 try: 857 gtest_xml = device.ReadFile(device_tmp_results_file.name) 858 except device_errors.CommandFailedError: 859 logging.exception('Failed to pull gtest results XML file %s', 860 device_tmp_results_file.name) 861 gtest_xml = None 862 863 if self._test_instance.isolated_script_test_output: 864 try: 865 gtest_json = device.ReadFile(device_tmp_results_file.name) 866 except device_errors.CommandFailedError: 867 logging.exception('Failed to pull gtest results JSON file %s', 868 device_tmp_results_file.name) 869 gtest_json = None 870 871 if test_perf_output_filename: 872 try: 873 device.PullFile(isolated_script_test_perf_output.name, 874 test_perf_output_filename) 875 except device_errors.CommandFailedError: 876 logging.exception('Failed to pull chartjson results %s', 877 isolated_script_test_perf_output.name) 878 879 test_artifacts_url = self._UploadTestArtifacts( 880 device, test_artifacts_dir) 881 882 if render_test_output_dir: 883 self._PullRenderTestOutput(device, render_test_output_dir.name) 884 885 for s in self._servers[str(device)]: 886 s.Reset() 887 if self._test_instance.app_files: 888 self._delegate.PullAppFiles(device, self._test_instance.app_files, 889 self._test_instance.app_file_dir) 890 if not self._env.skip_clear_data: 891 self._delegate.Clear(device) 892 893 for l in output: 894 logging.info(l) 895 896 # Parse the output. 897 # TODO(jbudorick): Transition test scripts away from parsing stdout. 898 if self._test_instance.enable_xml_result_parsing: 899 results = gtest_test_instance.ParseGTestXML(gtest_xml) 900 elif self._test_instance.isolated_script_test_output: 901 results = gtest_test_instance.ParseGTestJSON(gtest_json) 902 else: 903 results = gtest_test_instance.ParseGTestOutput( 904 output, self._test_instance.symbolizer, device.product_cpu_abi) 905 906 tombstones_url = None 907 for r in results: 908 if logcat_file: 909 r.SetLink('logcat', logcat_file.Link()) 910 911 if self._test_instance.gs_test_artifacts_bucket: 912 r.SetLink('test_artifacts', test_artifacts_url) 913 914 if r.GetType() == base_test_result.ResultType.CRASH: 915 self._crashes.add(r.GetName()) 916 if self._test_instance.store_tombstones: 917 if not tombstones_url: 918 resolved_tombstones = tombstones.ResolveTombstones( 919 device, 920 resolve_all_tombstones=True, 921 include_stack_symbols=False, 922 wipe_tombstones=True) 923 stream_name = 'tombstones_%s_%s' % ( 924 time.strftime('%Y%m%dT%H%M%S', time.localtime()), 925 device.serial) 926 tombstones_url = logdog_helper.text( 927 stream_name, '\n'.join(resolved_tombstones)) 928 r.SetLink('tombstones', tombstones_url) 929 930 tests_stripped_disabled_prefix = set() 931 for t in test: 932 tests_stripped_disabled_prefix.add( 933 gtest_test_instance.TestNameWithoutDisabledPrefix(t)) 934 not_run_tests = tests_stripped_disabled_prefix.difference( 935 set(r.GetName() for r in results)) 936 937 if self._test_instance.extract_test_list_from_filter: 938 # A test string might end with a * in this mode, and so may not match any 939 # r.GetName() for the set difference. It's possible a filter like foo.* 940 # can match two tests, ie foo.baz and foo.foo. 941 # When running it's possible Foo.baz is ran, foo.foo is not, but the test 942 # list foo.* will not be reran as at least one result matched it. 943 not_run_tests = { 944 t 945 for t in not_run_tests 946 if not any(fnmatch.fnmatch(r.GetName(), t) for r in results) 947 } 948 949 return results, list(not_run_tests) if results else None 950 951 #override 952 def TearDown(self): 953 # By default, teardown will invoke ADB. When receiving SIGTERM due to a 954 # timeout, there's a high probability that ADB is non-responsive. In these 955 # cases, sending an ADB command will potentially take a long time to time 956 # out. Before this happens, the process will be hard-killed for not 957 # responding to SIGTERM fast enough. 958 if self._received_sigterm: 959 return 960 961 @local_device_environment.handle_shard_failures 962 @trace_event.traced 963 def individual_device_tear_down(dev): 964 for s in self._servers.get(str(dev), []): 965 s.TearDown() 966 967 tool = self.GetTool(dev) 968 tool.CleanUpEnvironment() 969 970 self._env.parallel_devices.pMap(individual_device_tear_down) 971