1#!/usr/bin/env vpython3 2# Copyright 2022 The Chromium Authors 3# Use of this source code is governed by a BSD-style license that can be 4# found in the LICENSE file. 5"""Implements commands for standalone CFv2 test executables.""" 6 7import argparse 8import logging 9import os 10import shutil 11import subprocess 12import sys 13 14from typing import List, Optional 15 16from common import get_component_uri, get_host_arch, \ 17 register_common_args, register_device_args, \ 18 register_log_args 19from compatible_utils import map_filter_file_to_package_file 20from ffx_integration import FfxTestRunner, run_symbolizer 21from test_runner import TestRunner 22 23DEFAULT_TEST_SERVER_CONCURRENCY = 4 24 25 26def _copy_custom_output_file(test_runner: FfxTestRunner, file: str, 27 dest: str) -> None: 28 """Copy custom test output file from the device to the host.""" 29 30 artifact_dir = test_runner.get_custom_artifact_directory() 31 if not artifact_dir: 32 logging.error( 33 'Failed to parse custom artifact directory from test summary ' 34 'output files. Not copying %s from the device', file) 35 return 36 shutil.copy(os.path.join(artifact_dir, file), dest) 37 38 39def _copy_coverage_files(test_runner: FfxTestRunner, dest: str) -> None: 40 """Copy debug data file from the device to the host if it exists.""" 41 42 coverage_dir = test_runner.get_debug_data_directory() 43 if not coverage_dir: 44 logging.info( 45 'Failed to parse coverage data directory from test summary ' 46 'output files. Not copying coverage files from the device.') 47 return 48 shutil.copytree(coverage_dir, dest, dirs_exist_ok=True) 49 50 51class ExecutableTestRunner(TestRunner): 52 """Test runner for running standalone test executables.""" 53 54 def __init__( # pylint: disable=too-many-arguments 55 self, out_dir: str, test_args: List[str], test_name: str, 56 target_id: Optional[str], code_coverage_dir: str, 57 logs_dir: Optional[str], package_deps: List[str], 58 test_realm: Optional[str]) -> None: 59 super().__init__(out_dir, test_args, [test_name], target_id, 60 package_deps) 61 if not self._test_args: 62 self._test_args = [] 63 self._test_name = test_name 64 self._code_coverage_dir = code_coverage_dir 65 self._custom_artifact_directory = None 66 self._isolated_script_test_output = None 67 self._isolated_script_test_perf_output = None 68 self._logs_dir = logs_dir 69 self._test_launcher_summary_output = None 70 self._test_server = None 71 self._test_realm = test_realm 72 73 def _get_args(self) -> List[str]: 74 parser = argparse.ArgumentParser() 75 parser.add_argument( 76 '--isolated-script-test-output', 77 help='If present, store test results on this path.') 78 parser.add_argument('--isolated-script-test-perf-output', 79 help='If present, store chartjson results on this ' 80 'path.') 81 parser.add_argument( 82 '--test-launcher-shard-index', 83 type=int, 84 default=os.environ.get('GTEST_SHARD_INDEX'), 85 help='Index of this instance amongst swarming shards.') 86 parser.add_argument( 87 '--test-launcher-summary-output', 88 help='Where the test launcher will output its json.') 89 parser.add_argument( 90 '--test-launcher-total-shards', 91 type=int, 92 default=os.environ.get('GTEST_TOTAL_SHARDS'), 93 help='Total number of swarming shards of this suite.') 94 parser.add_argument( 95 '--test-launcher-filter-file', 96 help='Filter file(s) passed to target test process. Use ";" to ' 97 'separate multiple filter files.') 98 parser.add_argument('--test-launcher-jobs', 99 type=int, 100 help='Sets the number of parallel test jobs.') 101 parser.add_argument('--enable-test-server', 102 action='store_true', 103 default=False, 104 help='Enable Chrome test server spawner.') 105 parser.add_argument('--test-arg', 106 dest='test_args', 107 action='append', 108 help='Legacy flag to pass in arguments for ' 109 'the test process. These arguments can now be ' 110 'passed in without a preceding "--" flag.') 111 args, child_args = parser.parse_known_args(self._test_args) 112 if args.isolated_script_test_output: 113 self._isolated_script_test_output = args.isolated_script_test_output 114 child_args.append( 115 '--isolated-script-test-output=/custom_artifacts/%s' % 116 os.path.basename(self._isolated_script_test_output)) 117 if args.isolated_script_test_perf_output: 118 self._isolated_script_test_perf_output = \ 119 args.isolated_script_test_perf_output 120 child_args.append( 121 '--isolated-script-test-perf-output=/custom_artifacts/%s' % 122 os.path.basename(self._isolated_script_test_perf_output)) 123 if args.test_launcher_shard_index is not None: 124 child_args.append('--test-launcher-shard-index=%d' % 125 args.test_launcher_shard_index) 126 if args.test_launcher_total_shards is not None: 127 child_args.append('--test-launcher-total-shards=%d' % 128 args.test_launcher_total_shards) 129 if args.test_launcher_summary_output: 130 self._test_launcher_summary_output = \ 131 args.test_launcher_summary_output 132 child_args.append( 133 '--test-launcher-summary-output=/custom_artifacts/%s' % 134 os.path.basename(self._test_launcher_summary_output)) 135 if args.test_launcher_filter_file: 136 test_launcher_filter_files = map( 137 map_filter_file_to_package_file, 138 args.test_launcher_filter_file.split(';')) 139 child_args.append('--test-launcher-filter-file=' + 140 ';'.join(test_launcher_filter_files)) 141 if args.test_launcher_jobs is not None: 142 test_concurrency = args.test_launcher_jobs 143 else: 144 test_concurrency = DEFAULT_TEST_SERVER_CONCURRENCY 145 if args.enable_test_server: 146 # Repos other than chromium may not have chrome_test_server_spawner, 147 # and they may not run server at all, so only import the test_server 148 # when it's really necessary. 149 150 # pylint: disable=import-outside-toplevel 151 from test_server import setup_test_server 152 # pylint: enable=import-outside-toplevel 153 self._test_server, spawner_url_base = setup_test_server( 154 self._target_id, test_concurrency) 155 child_args.append('--remote-test-server-spawner-url-base=%s' % 156 spawner_url_base) 157 if get_host_arch() == 'x64': 158 # TODO(crbug.com/40202294) Remove once Vulkan is enabled by 159 # default. 160 child_args.append('--use-vulkan=native') 161 else: 162 # TODO(crbug.com/42050042, crbug.com/42050537) Remove swiftshader 163 # once the vulkan is enabled by default. 164 child_args.extend( 165 ['--use-vulkan=swiftshader', '--ozone-platform=headless']) 166 if args.test_args: 167 child_args.extend(args.test_args) 168 return child_args 169 170 def _postprocess(self, test_runner: FfxTestRunner) -> None: 171 if self._test_server: 172 self._test_server.Stop() 173 if self._test_launcher_summary_output: 174 _copy_custom_output_file( 175 test_runner, 176 os.path.basename(self._test_launcher_summary_output), 177 self._test_launcher_summary_output) 178 if self._isolated_script_test_output: 179 _copy_custom_output_file( 180 test_runner, 181 os.path.basename(self._isolated_script_test_output), 182 self._isolated_script_test_output) 183 if self._isolated_script_test_perf_output: 184 _copy_custom_output_file( 185 test_runner, 186 os.path.basename(self._isolated_script_test_perf_output), 187 self._isolated_script_test_perf_output) 188 if self._code_coverage_dir: 189 _copy_coverage_files(test_runner, 190 os.path.basename(self._code_coverage_dir)) 191 192 def run_test(self) -> subprocess.Popen: 193 test_args = self._get_args() 194 with FfxTestRunner(self._logs_dir) as test_runner: 195 test_proc = test_runner.run_test( 196 get_component_uri(self._test_name), test_args, self._target_id, 197 self._test_realm) 198 199 symbol_paths = [] 200 for pkg_path in self.package_deps.values(): 201 symbol_paths.append( 202 os.path.join(os.path.dirname(pkg_path), 'ids.txt')) 203 # Symbolize output from test process and print to terminal. 204 symbolizer_proc = run_symbolizer(symbol_paths, test_proc.stdout, 205 sys.stdout) 206 symbolizer_proc.communicate() 207 208 if test_proc.wait() == 0: 209 logging.info('Process exited normally with status code 0.') 210 else: 211 # The test runner returns an error status code if *any* 212 # tests fail, so we should proceed anyway. 213 logging.warning('Process exited with status code %d.', 214 test_proc.returncode) 215 self._postprocess(test_runner) 216 return test_proc 217 218 219def create_executable_test_runner(runner_args: argparse.Namespace, 220 test_args: List[str]): 221 """Helper for creating an ExecutableTestRunner.""" 222 223 return ExecutableTestRunner(runner_args.out_dir, test_args, 224 runner_args.test_type, runner_args.target_id, 225 runner_args.code_coverage_dir, 226 runner_args.logs_dir, runner_args.package_deps, 227 runner_args.test_realm) 228 229 230def register_executable_test_args(parser: argparse.ArgumentParser) -> None: 231 """Register common arguments for ExecutableTestRunner.""" 232 233 test_args = parser.add_argument_group('test', 'arguments for test running') 234 test_args.add_argument('--code-coverage-dir', 235 default=None, 236 help='Directory to place code coverage ' 237 'information. Only relevant when the target was ' 238 'built with |fuchsia_code_coverage| set to true.') 239 test_args.add_argument('--test-name', 240 dest='test_type', 241 help='Name of the test package (e.g. ' 242 'unit_tests).') 243 test_args.add_argument( 244 '--test-realm', 245 default=None, 246 help='The realm to run the test in. This field is optional and takes ' 247 'the form: /path/to/realm:test_collection. See ' 248 'https://fuchsia.dev/go/components/non-hermetic-tests') 249 test_args.add_argument('--package-deps', 250 action='append', 251 help='A list of the full path of the dependencies ' 252 'to retrieve the symbol ids. Keeping it empty to ' 253 'automatically generates from package_metadata.') 254 255 256def main(): 257 """Stand-alone function for running executable tests.""" 258 259 parser = argparse.ArgumentParser() 260 register_common_args(parser) 261 register_device_args(parser) 262 register_log_args(parser) 263 register_executable_test_args(parser) 264 runner_args, test_args = parser.parse_known_args() 265 runner = create_executable_test_runner(runner_args, test_args) 266 return runner.run_test().returncode 267 268 269if __name__ == '__main__': 270 sys.exit(main()) 271