1# Copyright 2017, The Android Open Source Project 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); 4# you may not use this file except in compliance with the License. 5# You may obtain a copy of the License at 6# 7# http://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12# See the License for the specific language governing permissions and 13# limitations under the License. 14 15"""Atest Tradefed test runner class.""" 16 17# pylint: disable=line-too-long 18# pylint: disable=too-many-lines 19 20from __future__ import print_function 21 22import json 23import logging 24import os 25import re 26import select 27import shutil 28import socket 29 30from functools import partial 31from pathlib import Path 32from typing import Any, List, Tuple 33 34import atest_configs 35import atest_error 36import atest_utils 37import constants 38import module_info 39import result_reporter 40 41from atest_enum import DetectType, ExitCode 42from logstorage import atest_gcp_utils 43from logstorage import logstorage_utils 44from metrics import metrics 45from test_finders import test_finder_utils 46from test_finders import test_info 47from test_runners import test_runner_base as trb 48from .event_handler import EventHandler 49 50POLL_FREQ_SECS = 10 51SOCKET_HOST = '127.0.0.1' 52SOCKET_QUEUE_MAX = 1 53SOCKET_BUFFER = 4096 54SELECT_TIMEOUT = 0.5 55 56# Socket Events of form FIRST_EVENT {JSON_DATA}\nSECOND_EVENT {JSON_DATA} 57# EVENT_RE has groups for the name and the data. "." does not match \n. 58EVENT_RE = re.compile(r'\n*(?P<event_name>[A-Z_]+) (?P<json_data>{.*})(?=\n|.)*') 59 60# Remove aapt from build dependency, use prebuilt version instead. 61EXEC_DEPENDENCIES = ('adb', 'fastboot') 62 63LOG_FOLDER_NAME = 'log' 64 65_INTEGRATION_FINDERS = frozenset(['', 'INTEGRATION', 'INTEGRATION_FILE_PATH']) 66 67# AAPT binary name 68_AAPT = 'aapt' 69 70# The exist code mapping of tradefed. 71_TF_EXIT_CODE = [ 72 'NO_ERROR', 73 'CONFIG_EXCEPTION', 74 'NO_BUILD', 75 'DEVICE_UNRESPONSIVE', 76 'DEVICE_UNAVAILABLE', 77 'FATAL_HOST_ERROR', 78 'THROWABLE_EXCEPTION', 79 'NO_DEVICE_ALLOCATED', 80 'WRONG_JAVA_VERSION'] 81 82 83class TradeFedExitError(Exception): 84 """Raised when TradeFed exists before test run has finished.""" 85 def __init__(self, exit_code): 86 super().__init__() 87 self.exit_code = exit_code 88 89 def __str__(self): 90 tf_error_reason = self._get_exit_reason(self.exit_code) 91 return (f'TradeFed subprocess exited early with exit code=' 92 f'{self.exit_code}({tf_error_reason}).') 93 94 def _get_exit_reason(self, exit_code): 95 if 0 < exit_code < len(_TF_EXIT_CODE): 96 return atest_utils.colorize(_TF_EXIT_CODE[exit_code], constants.RED) 97 return 'Unknown exit status' 98 99class AtestTradefedTestRunner(trb.TestRunnerBase): 100 """TradeFed Test Runner class.""" 101 NAME = 'AtestTradefedTestRunner' 102 EXECUTABLE = 'atest_tradefed.sh' 103 _TF_TEMPLATE = 'template/atest_local_min' 104 # Use --no-enable-granular-attempts to control reporter replay behavior. 105 # TODO(b/142630648): Enable option enable-granular-attempts 106 # in sharding mode. 107 _LOG_ARGS = ('--logcat-on-failure --{log_root_option_name}={log_path} ' 108 '{log_ext_option} ' 109 '--no-enable-granular-attempts ' 110 '--proto-output-file={proto_path}') 111 _RUN_CMD = ('{env} {exe} {template} ' 112 '--template:map test=atest ' 113 '--template:map log_saver={log_saver} ' 114 '{tf_customize_template} {log_args} {args}') 115 _BUILD_REQ = {'tradefed-core'} 116 _RERUN_OPTION_GROUP = [constants.ITERATIONS, 117 constants.RERUN_UNTIL_FAILURE, 118 constants.RETRY_ANY_FAILURE] 119 120 def __init__(self, results_dir: str, 121 mod_info: module_info.ModuleInfo=None, **kwargs): 122 """Init stuff for base class.""" 123 super().__init__(results_dir, **kwargs) 124 self.module_info = mod_info 125 self.log_path = os.path.join(results_dir, LOG_FOLDER_NAME) 126 if not os.path.exists(self.log_path): 127 os.makedirs(self.log_path) 128 log_args = {'log_root_option_name': constants.LOG_ROOT_OPTION_NAME, 129 'log_ext_option': constants.LOG_SAVER_EXT_OPTION, 130 'log_path': self.log_path, 131 'proto_path': os.path.join(self.results_dir, constants.ATEST_TEST_RECORD_PROTO)} 132 self.run_cmd_dict = {'env': self._get_ld_library_path(), 133 'exe': self.EXECUTABLE, 134 'template': self._TF_TEMPLATE, 135 'log_saver': constants.ATEST_TF_LOG_SAVER, 136 'tf_customize_template': '', 137 'args': '', 138 'log_args': self._LOG_ARGS.format(**log_args)} 139 self.is_verbose = logging.getLogger().isEnabledFor(logging.DEBUG) 140 self.root_dir = os.environ.get(constants.ANDROID_BUILD_TOP) 141 142 def _get_ld_library_path(self): 143 """Get the extra environment setup string for running TF. 144 145 Returns: 146 Strings for the environment passed to TF. Currently only 147 LD_LIBRARY_PATH for TF to load the correct local shared libraries. 148 """ 149 out_dir = os.environ.get(constants.ANDROID_HOST_OUT, '') 150 # From b/188179058, if a 64bit tests, it will break the tests due to the 151 # elf format is not 64bit for the lib path. But for b/160741384, it is 152 # ok to load lib path first. Change the lib_dirs sequence to lib64 first 153 # due to ATest by default only testing the main abi and even a 32bit 154 # only target the lib64 folder is actually not exist. 155 lib_dirs = ['lib64', 'lib'] 156 path = '' 157 for lib in lib_dirs: 158 lib_dir = os.path.join(out_dir, lib) 159 path = path + lib_dir + ':' 160 return 'LD_LIBRARY_PATH=%s' % path 161 162 def _try_set_gts_authentication_key(self): 163 """Set GTS authentication key if it is available or exists. 164 165 Strategy: 166 Get APE_API_KEY from os.environ: 167 - If APE_API_KEY is already set by user -> do nothing. 168 Get the APE_API_KEY from constants: 169 - If the key file exists -> set to env var. 170 If APE_API_KEY isn't set and the key file doesn't exist: 171 - Warn user some GTS tests may fail without authentication. 172 """ 173 if os.environ.get('APE_API_KEY'): 174 logging.debug('APE_API_KEY is set by developer.') 175 return 176 ape_api_key = constants.GTS_GOOGLE_SERVICE_ACCOUNT 177 key_path = os.path.join(self.root_dir, ape_api_key) 178 if ape_api_key and os.path.exists(key_path): 179 logging.debug('Set APE_API_KEY: %s', ape_api_key) 180 os.environ['APE_API_KEY'] = key_path 181 else: 182 logging.debug('APE_API_KEY not set, some GTS tests may fail' 183 ' without authentication.') 184 185 def run_tests(self, test_infos, extra_args, reporter): 186 """Run the list of test_infos. See base class for more. 187 188 Args: 189 test_infos: A list of TestInfos. 190 extra_args: Dict of extra args to add to test run. 191 reporter: An instance of result_report.ResultReporter. 192 193 Returns: 194 0 if tests succeed, non-zero otherwise. 195 """ 196 reporter.log_path = self.log_path 197 reporter.rerun_options = self._extract_rerun_options(extra_args) 198 # Set google service key if it's available or found before 199 # running tests. 200 self._try_set_gts_authentication_key() 201 result = 0 202 creds, inv = atest_gcp_utils.do_upload_flow(extra_args) 203 try: 204 verify_key = atest_utils.get_verify_key([test_infos[0].test_name], 205 extra_args) 206 if extra_args.get(constants.VERIFY_ENV_VARIABLE, False): 207 # check environment variables. 208 atest_utils.handle_test_env_var( 209 verify_key, result_path=constants.VERIFY_ENV_PATH) 210 return 0 211 # Change CWD to repo root to ensure TF can find prebuilt SDKs 212 # for some path-sensitive tests like robolectric. 213 os.chdir(os.path.abspath(os.getenv(constants.ANDROID_BUILD_TOP))) 214 if os.getenv(trb.OLD_OUTPUT_ENV_VAR): 215 result = self.run_tests_raw(test_infos, extra_args, reporter) 216 result = self.run_tests_pretty(test_infos, extra_args, reporter) 217 except atest_error.DryRunVerificationError as e: 218 atest_utils.colorful_print(str(e), constants.RED) 219 return ExitCode.VERIFY_FAILURE 220 finally: 221 if inv: 222 try: 223 logging.disable(logging.INFO) 224 # Always set invocation status to completed due to the ATest 225 # handle whole process by its own. 226 inv['schedulerState'] = 'completed' 227 logstorage_utils.BuildClient(creds).update_invocation(inv) 228 reporter.test_result_link = (constants.RESULT_LINK 229 % inv['invocationId']) 230 finally: 231 logging.disable(logging.NOTSET) 232 return result 233 234 def run_tests_raw(self, test_infos, extra_args, reporter): 235 """Run the list of test_infos. See base class for more. 236 237 Args: 238 test_infos: A list of TestInfos. 239 extra_args: Dict of extra args to add to test run. 240 reporter: An instance of result_report.ResultReporter. 241 242 Returns: 243 0 if tests succeed, non-zero otherwise. 244 """ 245 iterations = self._generate_iterations(extra_args) 246 reporter.register_unsupported_runner(self.NAME) 247 248 ret_code = ExitCode.SUCCESS 249 for _ in range(iterations): 250 run_cmds = self.generate_run_commands(test_infos, extra_args) 251 subproc = self.run(run_cmds[0], output_to_stdout=True, 252 env_vars=self.generate_env_vars(extra_args)) 253 ret_code |= self.wait_for_subprocess(subproc) 254 return ret_code 255 256 def run_tests_pretty(self, test_infos, extra_args, reporter): 257 """Run the list of test_infos. See base class for more. 258 259 Args: 260 test_infos: A list of TestInfos. 261 extra_args: Dict of extra args to add to test run. 262 reporter: An instance of result_report.ResultReporter. 263 264 Returns: 265 0 if tests succeed, non-zero otherwise. 266 """ 267 iterations = self._generate_iterations(extra_args) 268 ret_code = ExitCode.SUCCESS 269 for _ in range(iterations): 270 server = self._start_socket_server() 271 run_cmds = self.generate_run_commands(test_infos, extra_args, 272 server.getsockname()[1]) 273 subproc = self.run(run_cmds[0], output_to_stdout=self.is_verbose, 274 env_vars=self.generate_env_vars(extra_args)) 275 self.handle_subprocess(subproc, partial(self._start_monitor, 276 server, 277 subproc, 278 reporter, 279 extra_args)) 280 server.close() 281 ret_code |= self.wait_for_subprocess(subproc) 282 return ret_code 283 284 # pylint: disable=too-many-branches 285 # pylint: disable=too-many-locals 286 def _start_monitor(self, server, tf_subproc, reporter, extra_args): 287 """Polling and process event. 288 289 Args: 290 server: Socket server object. 291 tf_subproc: The tradefed subprocess to poll. 292 reporter: Result_Reporter object. 293 extra_args: Dict of extra args to add to test run. 294 """ 295 inputs = [server] 296 event_handlers = {} 297 data_map = {} 298 inv_socket = None 299 while inputs: 300 try: 301 readable, _, _ = select.select(inputs, [], [], SELECT_TIMEOUT) 302 for socket_object in readable: 303 if socket_object is server: 304 conn, addr = socket_object.accept() 305 logging.debug('Accepted connection from %s', addr) 306 conn.setblocking(False) 307 inputs.append(conn) 308 data_map[conn] = '' 309 # The First connection should be invocation 310 # level reporter. 311 if not inv_socket: 312 inv_socket = conn 313 else: 314 # Count invocation level reporter events 315 # without showing real-time information. 316 if inv_socket == socket_object: 317 reporter.silent = True 318 event_handler = event_handlers.setdefault( 319 socket_object, EventHandler(reporter, 320 self.NAME)) 321 else: 322 event_handler = event_handlers.setdefault( 323 socket_object, EventHandler( 324 result_reporter.ResultReporter( 325 collect_only=extra_args.get( 326 constants.COLLECT_TESTS_ONLY), 327 flakes_info=extra_args.get( 328 constants.FLAKES_INFO)), 329 330 self.NAME)) 331 recv_data = self._process_connection(data_map, 332 socket_object, 333 event_handler) 334 if not recv_data: 335 inputs.remove(socket_object) 336 socket_object.close() 337 finally: 338 # Subprocess ended and all socket clients were closed. 339 if tf_subproc.poll() is not None and len(inputs) == 1: 340 inputs.pop().close() 341 if not reporter.all_test_results: 342 atest_utils.colorful_print( 343 r'No test to run. Please check: ' 344 r'{} for detail.'.format(reporter.log_path), 345 constants.RED, highlight=True) 346 if not data_map: 347 metrics.LocalDetectEvent( 348 detect_type=DetectType.TF_EXIT_CODE, 349 result=tf_subproc.returncode) 350 raise TradeFedExitError(tf_subproc.returncode) 351 self._handle_log_associations(event_handlers) 352 353 def _process_connection(self, data_map, conn, event_handler): 354 """Process a socket connection betwen TF and ATest. 355 356 Expect data of form EVENT_NAME {JSON_DATA}. Multiple events will be 357 \n deliminated. Need to buffer data in case data exceeds socket 358 buffer. 359 E.q. 360 TEST_RUN_STARTED {runName":"hello_world_test","runAttempt":0}\n 361 TEST_STARTED {"start_time":2172917, "testName":"PrintHelloWorld"}\n 362 Args: 363 data_map: The data map of all connections. 364 conn: Socket connection. 365 event_handler: EventHandler object. 366 367 Returns: 368 True if conn.recv() has data , False otherwise. 369 """ 370 # Set connection into blocking mode. 371 conn.settimeout(None) 372 data = conn.recv(SOCKET_BUFFER) 373 if isinstance(data, bytes): 374 data = data.decode() 375 logging.debug('received: %s', data) 376 if data: 377 data_map[conn] += data 378 while True: 379 match = EVENT_RE.match(data_map[conn]) 380 if not match: 381 break 382 try: 383 event_data = json.loads(match.group('json_data')) 384 except ValueError: 385 logging.debug('Json incomplete, wait for more data') 386 break 387 event_name = match.group('event_name') 388 event_handler.process_event(event_name, event_data) 389 data_map[conn] = data_map[conn][match.end():] 390 return bool(data) 391 392 def _start_socket_server(self): 393 """Start a TCP server.""" 394 server = socket.socket(socket.AF_INET, socket.SOCK_STREAM) 395 server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) 396 # Port 0 lets the OS pick an open port between 1024 and 65535. 397 server.bind((SOCKET_HOST, 0)) 398 server.listen(SOCKET_QUEUE_MAX) 399 server.settimeout(POLL_FREQ_SECS) 400 logging.debug('Socket server started on port %s', 401 server.getsockname()[1]) 402 return server 403 404 def generate_env_vars(self, extra_args): 405 """Convert extra args into env vars.""" 406 env_vars = os.environ.copy() 407 if constants.TF_GLOBAL_CONFIG: 408 env_vars["TF_GLOBAL_CONFIG"] = constants.TF_GLOBAL_CONFIG 409 debug_port = extra_args.get(constants.TF_DEBUG, '') 410 if debug_port: 411 env_vars['TF_DEBUG'] = 'true' 412 env_vars['TF_DEBUG_PORT'] = str(debug_port) 413 filtered_paths = [] 414 for path in str(env_vars['PYTHONPATH']).split(':'): 415 # TODO (b/166216843) Remove the hacky PYTHON path workaround. 416 if (str(path).startswith('/tmp/Soong.python_') and 417 str(path).find('googleapiclient') > 0): 418 continue 419 filtered_paths.append(path) 420 env_vars['PYTHONPATH'] = ':'.join(filtered_paths) 421 422 # Use prebuilt aapt if there's no aapt under android system path which 423 # is aligned with build system. 424 # https://android.googlesource.com/platform/build/+/master/core/config.mk#529 425 if self._is_missing_exec(_AAPT): 426 prebuilt_aapt = Path.joinpath( 427 atest_utils.get_prebuilt_sdk_tools_dir(), _AAPT) 428 if os.path.exists(prebuilt_aapt): 429 env_vars['PATH'] = (str(prebuilt_aapt.parent) + ':' 430 + env_vars['PATH']) 431 return env_vars 432 433 # pylint: disable=unnecessary-pass 434 # Please keep above disable flag to ensure host_env_check is overriden. 435 def host_env_check(self): 436 """Check that host env has everything we need. 437 438 We actually can assume the host env is fine because we have the same 439 requirements that atest has. Update this to check for android env vars 440 if that changes. 441 """ 442 pass 443 444 @staticmethod 445 def _is_missing_exec(executable): 446 """Check if system build executable is available. 447 448 Args: 449 executable: Executable we are checking for. 450 451 Returns: 452 True if executable is missing, False otherwise. 453 """ 454 output = shutil.which(executable) 455 if not output: 456 return True 457 # TODO: Check if there is a clever way to determine if system adb is 458 # good enough. 459 root_dir = os.environ.get(constants.ANDROID_BUILD_TOP, '') 460 return os.path.commonprefix([output, root_dir]) != root_dir 461 462 def get_test_runner_build_reqs(self): 463 """Return the build requirements. 464 465 Returns: 466 Set of build targets. 467 """ 468 build_req = self._BUILD_REQ 469 # Use different base build requirements if google-tf is around. 470 if self.module_info.is_module(constants.GTF_MODULE): 471 build_req = {constants.GTF_TARGET} 472 # Always add ATest's own TF target. 473 build_req.add(constants.ATEST_TF_MODULE) 474 # Add adb if we can't find it. 475 for executable in EXEC_DEPENDENCIES: 476 if self._is_missing_exec(executable): 477 if self.module_info.is_module(executable): 478 build_req.add(executable) 479 return build_req 480 481 def _parse_extra_args(self, test_infos, extra_args): 482 """Convert the extra args into something tf can understand. 483 484 Args: 485 extra_args: Dict of args 486 487 Returns: 488 Tuple of args to append and args not supported. 489 """ 490 args_to_append, args_not_supported = extra_args_to_tf_args( 491 self.module_info, test_infos, extra_args) 492 493 # Set exclude instant app annotation for non-instant mode run. 494 if (constants.INSTANT not in extra_args and 495 self._has_instant_app_config(test_infos, self.module_info)): 496 args_to_append.append(constants.TF_TEST_ARG) 497 args_to_append.append( 498 '{tf_class}:{option_name}:{option_value}'.format( 499 tf_class=constants.TF_AND_JUNIT_CLASS, 500 option_name=constants.TF_EXCLUDE_ANNOTATE, 501 option_value=constants.INSTANT_MODE_ANNOTATE)) 502 # Force append --enable-parameterized-modules if args_to_append has 503 # --module-parameter in args_to_append 504 if constants.TF_MODULE_PARAMETER in args_to_append: 505 if constants.TF_ENABLE_PARAMETERIZED_MODULES not in args_to_append: 506 args_to_append.append(constants.TF_ENABLE_PARAMETERIZED_MODULES) 507 # If all the test config has config with auto enable parameter, force 508 # exclude those default parameters(ex: instant_app, secondary_user) 509 if self._is_all_tests_parameter_auto_enabled(test_infos): 510 if constants.TF_ENABLE_PARAMETERIZED_MODULES not in args_to_append: 511 args_to_append.append(constants.TF_ENABLE_PARAMETERIZED_MODULES) 512 for exclude_parameter in constants.DEFAULT_EXCLUDE_PARAS: 513 args_to_append.append('--exclude-module-parameters') 514 args_to_append.append(exclude_parameter) 515 return args_to_append, args_not_supported 516 517 def _generate_metrics_folder(self, extra_args): 518 """Generate metrics folder.""" 519 metrics_folder = '' 520 if extra_args.get(constants.PRE_PATCH_ITERATIONS): 521 metrics_folder = os.path.join(self.results_dir, 'baseline-metrics') 522 elif extra_args.get(constants.POST_PATCH_ITERATIONS): 523 metrics_folder = os.path.join(self.results_dir, 'new-metrics') 524 return metrics_folder 525 526 def _generate_iterations(self, extra_args): 527 """Generate iterations.""" 528 iterations = 1 529 if extra_args.get(constants.PRE_PATCH_ITERATIONS): 530 iterations = extra_args.pop(constants.PRE_PATCH_ITERATIONS) 531 elif extra_args.get(constants.POST_PATCH_ITERATIONS): 532 iterations = extra_args.pop(constants.POST_PATCH_ITERATIONS) 533 return iterations 534 535 def generate_run_commands(self, test_infos, extra_args, port=None): 536 """Generate a single run command from TestInfos. 537 538 Args: 539 test_infos: A set of TestInfo instances. 540 extra_args: A Dict of extra args to append. 541 port: Optional. An int of the port number to send events to. If 542 None, then subprocess reporter in TF won't try to connect. 543 544 Returns: 545 A list that contains the string of atest tradefed run command. 546 Only one command is returned. 547 """ 548 args = self._create_test_args(test_infos) 549 metrics_folder = self._generate_metrics_folder(extra_args) 550 551 # Create a copy of args as more args could be added to the list. 552 test_args = list(args) 553 if port: 554 test_args.extend(['--subprocess-report-port', str(port)]) 555 if metrics_folder: 556 test_args.extend(['--metrics-folder', metrics_folder]) 557 logging.info('Saved metrics in: %s', metrics_folder) 558 if extra_args.get(constants.INVOCATION_ID, None): 559 test_args.append('--invocation-data invocation_id=%s' 560 % extra_args[constants.INVOCATION_ID]) 561 if extra_args.get(constants.WORKUNIT_ID, None): 562 test_args.append('--invocation-data work_unit_id=%s' 563 % extra_args[constants.WORKUNIT_ID]) 564 if extra_args.get(constants.LOCAL_BUILD_ID, None): 565 # TODO: (b/207584685) Replace with TF local build solutions. 566 test_args.append('--use-stub-build true') 567 test_args.append('--stub-build-id %s' 568 % extra_args[constants.LOCAL_BUILD_ID]) 569 test_args.append('--stub-build-target %s' 570 % extra_args[constants.BUILD_TARGET]) 571 if extra_args.get(constants.ENABLE_DEVICE_PREPARER, False): 572 test_args.append('--template:map preparers=%s' 573 % constants.DEVICE_SETUP_PREPARER) 574 for info in test_infos: 575 if constants.TEST_WITH_MAINLINE_MODULES_RE.match(info.test_name): 576 test_args.append(constants.TF_ENABLE_MAINLINE_PARAMETERIZED_MODULES) 577 break 578 # For detailed logs, set TF options log-level/log-level-display as 579 # 'VERBOSE' by default. 580 log_level = 'VERBOSE' 581 test_args.extend(['--log-level-display', log_level]) 582 test_args.extend(['--log-level', log_level]) 583 # Set no-early-device-release by default to speed up TF teardown time. 584 if not constants.TF_EARLY_DEVICE_RELEASE in extra_args: 585 test_args.extend(['--no-early-device-release']) 586 587 args_to_add, args_not_supported = self._parse_extra_args(test_infos, extra_args) 588 589 # If multiple devices in test config, automatically append 590 # --replicate-parent-setup and --multi-device-count 591 device_count = atest_configs.GLOBAL_ARGS.device_count_config 592 if device_count and device_count > 1: 593 args_to_add.append('--replicate-parent-setup') 594 args_to_add.append('--multi-device-count') 595 args_to_add.append(str(device_count)) 596 597 # TODO(b/122889707) Remove this after finding the root cause. 598 env_serial = os.environ.get(constants.ANDROID_SERIAL) 599 # Use the env variable ANDROID_SERIAL if it's set by user but only when 600 # the target tests are not deviceless tests. 601 if env_serial and '--serial' not in args_to_add and '-n' not in args_to_add: 602 args_to_add.append("--serial") 603 args_to_add.append(env_serial) 604 605 test_args.extend(args_to_add) 606 if args_not_supported: 607 logging.info('%s does not support the following args %s', 608 self.EXECUTABLE, args_not_supported) 609 610 # Only need to check one TestInfo to determine if the tests are 611 # configured in TEST_MAPPING. 612 for_test_mapping = test_infos and test_infos[0].from_test_mapping 613 test_args.extend(atest_utils.get_result_server_args(for_test_mapping)) 614 self.run_cmd_dict['args'] = ' '.join(test_args) 615 self.run_cmd_dict['tf_customize_template'] = ( 616 self._extract_customize_tf_templates(extra_args, test_infos)) 617 618 # Copy symbols if there are tests belong to native test. 619 self._handle_native_tests(test_infos) 620 return [self._RUN_CMD.format(**self.run_cmd_dict)] 621 622 def _flatten_test_infos(self, test_infos): 623 """Sort and group test_infos by module_name and sort and group filters 624 by class name. 625 626 Example of three test_infos in a set: 627 Module1, {(classA, {})} 628 Module1, {(classB, {Method1})} 629 Module1, {(classB, {Method2}} 630 Becomes a set with one element: 631 Module1, {(ClassA, {}), (ClassB, {Method1, Method2})} 632 Where: 633 Each line is a test_info namedtuple 634 {} = Frozenset 635 () = TestFilter namedtuple 636 637 Args: 638 test_infos: A set of TestInfo namedtuples. 639 640 Returns: 641 A set of TestInfos flattened. 642 """ 643 results = set() 644 key = lambda x: x.test_name 645 for module, group in atest_utils.sort_and_group(test_infos, key): 646 # module is a string, group is a generator of grouped TestInfos. 647 # Module Test, so flatten test_infos: 648 no_filters = False 649 filters = set() 650 test_runner = None 651 test_finder = None 652 build_targets = set() 653 data = {} 654 module_args = [] 655 for test_info_i in group: 656 data.update(test_info_i.data) 657 # Extend data with constants.TI_MODULE_ARG instead of 658 # overwriting. 659 module_args.extend(test_info_i.data.get( 660 constants.TI_MODULE_ARG, [])) 661 test_runner = test_info_i.test_runner 662 test_finder = test_info_i.test_finder 663 build_targets |= test_info_i.build_targets 664 test_filters = test_info_i.data.get(constants.TI_FILTER) 665 if not test_filters or no_filters: 666 # test_info wants whole module run, so hardcode no filters. 667 no_filters = True 668 filters = set() 669 continue 670 filters |= test_filters 671 if module_args: 672 data[constants.TI_MODULE_ARG] = module_args 673 data[constants.TI_FILTER] = self._flatten_test_filters(filters) 674 results.add( 675 test_info.TestInfo(test_name=module, 676 test_runner=test_runner, 677 test_finder=test_finder, 678 build_targets=build_targets, 679 data=data)) 680 return results 681 682 @staticmethod 683 def _flatten_test_filters(filters): 684 """Sort and group test_filters by class_name. 685 686 Example of three test_filters in a frozenset: 687 classA, {} 688 classB, {Method1} 689 classB, {Method2} 690 Becomes a frozenset with these elements: 691 classA, {} 692 classB, {Method1, Method2} 693 Where: 694 Each line is a TestFilter namedtuple 695 {} = Frozenset 696 697 Args: 698 filters: A frozenset of test_filters. 699 700 Returns: 701 A frozenset of test_filters flattened. 702 """ 703 results = set() 704 key = lambda x: x.class_name 705 for class_name, group in atest_utils.sort_and_group(filters, key): 706 # class_name is a string, group is a generator of TestFilters 707 assert class_name is not None 708 methods = set() 709 for test_filter in group: 710 if not test_filter.methods: 711 # Whole class should be run 712 methods = set() 713 break 714 methods |= test_filter.methods 715 results.add(test_info.TestFilter(class_name, frozenset(methods))) 716 return frozenset(results) 717 718 def _is_all_tests_parameter_auto_enabled(self, test_infos): 719 """Check if all the test infos are parameter auto enabled. 720 721 Args: 722 test_infos: A set of TestInfo instances. 723 724 Returns: True if all tests are parameter auto enabled, False otherwise. 725 """ 726 for info in test_infos: 727 if not self._is_parameter_auto_enabled_cfg(info, self.module_info): 728 return False 729 return True 730 731 def _create_test_args(self, test_infos): 732 """Compile TF command line args based on the given test infos. 733 734 Args: 735 test_infos: A set of TestInfo instances. 736 737 Returns: A list of TF arguments to run the tests. 738 """ 739 args = [] 740 if not test_infos: 741 return [] 742 743 test_infos = self._flatten_test_infos(test_infos) 744 has_integration_test = False 745 746 # Because current --include-filter arg will not working if ATest pass 747 # both --module and --include-filter to TF, only test by --module will 748 # be run. Make a check first, only use --module if all tests are all 749 # parameter auto enabled. 750 use_module_arg = self._is_all_tests_parameter_auto_enabled(test_infos) 751 752 for info in test_infos: 753 # Integration test exists in TF's jar, so it must have the option 754 # if it's integration finder. 755 if info.test_finder in _INTEGRATION_FINDERS: 756 has_integration_test = True 757 # For non-paramertize test module, use --include-filter, but for 758 # tests which have auto enable paramertize config use --module 759 # instead. 760 if (use_module_arg 761 and self._is_parameter_auto_enabled_cfg( 762 info, self.module_info)): 763 args.extend([constants.TF_MODULE_FILTER, info.test_name]) 764 else: 765 args.extend([constants.TF_INCLUDE_FILTER, info.test_name]) 766 for option in info.data.get(constants.TI_MODULE_ARG, []): 767 if constants.TF_INCLUDE_FILTER_OPTION == option[0]: 768 suite_filter = ( 769 constants.TF_SUITE_FILTER_ARG_VALUE_FMT.format( 770 test_name=info.test_name, option_value=option[1])) 771 args.extend([constants.TF_INCLUDE_FILTER, suite_filter]) 772 elif constants.TF_EXCLUDE_FILTER_OPTION == option[0]: 773 suite_filter = ( 774 constants.TF_SUITE_FILTER_ARG_VALUE_FMT.format( 775 test_name=info.test_name, option_value=option[1])) 776 args.extend([constants.TF_EXCLUDE_FILTER, suite_filter]) 777 else: 778 module_arg = ( 779 constants.TF_MODULE_ARG_VALUE_FMT.format( 780 test_name=info.test_name, option_name=option[0], 781 option_value=option[1])) 782 args.extend([constants.TF_MODULE_ARG, module_arg]) 783 784 # Add ATest include filter 785 args.extend(get_include_filter(test_infos)) 786 787 # TODO (b/141090547) Pass the config path to TF to load configs. 788 # Compile option in TF if finder is not INTEGRATION or not set. 789 if not has_integration_test: 790 args.append(constants.TF_SKIP_LOADING_CONFIG_JAR) 791 return args 792 793 def _extract_rerun_options(self, extra_args): 794 """Extract rerun options to a string for output. 795 796 Args: 797 extra_args: Dict of extra args for test runners to use. 798 799 Returns: A string of rerun options. 800 """ 801 extracted_options = ['{} {}'.format(arg, extra_args[arg]) 802 for arg in extra_args 803 if arg in self._RERUN_OPTION_GROUP] 804 return ' '.join(extracted_options) 805 806 def _extract_customize_tf_templates(self, extra_args, test_infos): 807 """Extract tradefed template options to a string for output. 808 809 Args: 810 extra_args: Dict of extra args for test runners to use. 811 test_infos: A set of TestInfo instances. 812 813 Returns: A string of tradefed template options. 814 """ 815 tf_templates = extra_args.get(constants.TF_TEMPLATE, []) 816 for info in test_infos: 817 if info.aggregate_metrics_result: 818 template_key = 'metric_post_processor' 819 template_value = ( 820 'google/template/postprocessors/metric-file-aggregate') 821 tf_templates.append(f'{template_key}={template_value}') 822 return ' '.join(['--template:map %s' % x for x in tf_templates]) 823 824 def _handle_log_associations(self, event_handlers): 825 """Handle TF's log associations information data. 826 827 log_association dict: 828 {'loggedFile': '/tmp/serial-util11375755456514097276.ser', 829 'dataName': 'device_logcat_setup_127.0.0.1:58331', 830 'time': 1602038599.856113}, 831 832 Args: 833 event_handlers: Dict of {socket_object:EventHandler}. 834 835 """ 836 log_associations = [] 837 for _, event_handler in event_handlers.items(): 838 if event_handler.log_associations: 839 log_associations += event_handler.log_associations 840 device_test_end_log_time = '' 841 device_teardown_log_time = '' 842 for log_association in log_associations: 843 if 'device_logcat_test' in log_association.get('dataName', ''): 844 device_test_end_log_time = log_association.get('time') 845 if 'device_logcat_teardown' in log_association.get('dataName', ''): 846 device_teardown_log_time = log_association.get('time') 847 if device_test_end_log_time and device_teardown_log_time: 848 teardowntime = (float(device_teardown_log_time) - 849 float(device_test_end_log_time)) 850 logging.debug('TF logcat teardown time=%s seconds.', teardowntime) 851 metrics.LocalDetectEvent( 852 detect_type=DetectType.TF_TEARDOWN_LOGCAT, 853 result=int(teardowntime)) 854 855 @staticmethod 856 def _has_instant_app_config(test_infos, mod_info): 857 """Check if one of the input tests defined instant app mode in config. 858 859 Args: 860 test_infos: A set of TestInfo instances. 861 mod_info: ModuleInfo object. 862 863 Returns: True if one of the tests set up instant app mode. 864 """ 865 for tinfo in test_infos: 866 test_config, _ = test_finder_utils.get_test_config_and_srcs( 867 tinfo, mod_info) 868 if test_config: 869 parameters = atest_utils.get_config_parameter(test_config) 870 if constants.TF_PARA_INSTANT_APP in parameters: 871 return True 872 return False 873 874 @staticmethod 875 def _is_parameter_auto_enabled_cfg(tinfo, mod_info): 876 """Check if input tests contains auto enable support parameters. 877 878 Args: 879 test_infos: A set of TestInfo instances. 880 mod_info: ModuleInfo object. 881 882 Returns: True if input test has parameter setting which is not in the 883 exclude list. 884 """ 885 test_config, _ = test_finder_utils.get_test_config_and_srcs( 886 tinfo, mod_info) 887 if test_config: 888 parameters = atest_utils.get_config_parameter(test_config) 889 if (parameters - constants.DEFAULT_EXCLUDE_PARAS 890 - constants.DEFAULT_EXCLUDE_NOT_PARAS): 891 return True 892 return False 893 894 def _handle_native_tests(self, test_infos): 895 """Handling some extra tasks for running native tests from tradefed. 896 897 Args: 898 test_infos: A set of TestInfo instances. 899 """ 900 for tinfo in test_infos: 901 test_config, _ = test_finder_utils.get_test_config_and_srcs( 902 tinfo, self.module_info) 903 if test_config: 904 module_name, device_path = atest_utils.get_config_gtest_args( 905 test_config) 906 if module_name and device_path: 907 atest_utils.copy_native_symbols(module_name, device_path) 908 909 910def generate_annotation_filter_args( 911 arg_value: Any, mod_info: module_info.ModuleInfo, 912 test_infos: List[test_info.TestInfo]) -> List[str]: 913 """Generate TF annotation filter arguments. 914 915 Args: 916 arg_value: Argument value for annotation filter. 917 mod_info: ModuleInfo object. 918 test_infos: A set of TestInfo instances. 919 920 Returns: 921 List of TF annotation filter arguments. 922 """ 923 annotation_filter_args = [] 924 for info in test_infos: 925 test_name = info.test_name 926 for keyword in arg_value: 927 annotation = atest_utils.get_full_annotation_class_name( 928 mod_info.get_module_info(test_name), keyword) 929 if annotation: 930 module_arg = (constants.TF_MODULE_ARG_VALUE_FMT.format( 931 test_name=test_name, 932 option_name=constants.INCLUDE_ANNOTATION, 933 option_value=annotation)) 934 annotation_filter_args.extend([constants.TF_MODULE_ARG, module_arg]) 935 logging.error( 936 atest_utils.colorize( 937 f'Cannot find similar annotation: {keyword}', 938 constants.RED)) 939 return annotation_filter_args 940 941 942def extra_args_to_tf_args(mod_info: module_info.ModuleInfo, 943 test_infos: List[test_info.TestInfo], 944 extra_args: trb.ARGS) -> Tuple[trb.ARGS, trb.ARGS]: 945 """Convert the extra args into atest_tf_test_runner supported args. 946 947 Args: 948 mod_info: ModuleInfo object. 949 test_infos: A set of TestInfo instances. 950 extra_args: Dict of args 951 952 Returns: 953 Tuple of ARGS that atest_tf supported and not supported. 954 """ 955 supported_args = [] 956 unsupported_args = [] 957 958 def constant_list(*value): 959 return lambda *_: value 960 961 # pylint: disable=unused-argument 962 def print_message(message): 963 def inner(*args): 964 print(message) 965 return [] 966 return inner 967 968 # Mapping supported TF arguments to the processing function. 969 supported_tf_args = dict({ 970 constants.WAIT_FOR_DEBUGGER: 971 constant_list('--wait-for-debugger'), 972 constants.DISABLE_INSTALL: 973 constant_list('--disable-target-preparers'), 974 constants.SERIAL: 975 lambda arg_value, *_: 976 [j for d in arg_value for j in ('--serial', d)], 977 constants.SHARDING: 978 lambda arg_value, *_: ['--shard-count', 979 str(arg_value)], 980 constants.DISABLE_TEARDOWN: 981 constant_list('--disable-teardown'), 982 constants.HOST: 983 constant_list('-n', '--prioritize-host-config', 984 '--skip-host-arch-check'), 985 constants.CUSTOM_ARGS: 986 # custom args value is a list. 987 lambda arg_value, *_: arg_value, 988 constants.ALL_ABI: 989 constant_list('--all-abi'), 990 constants.INSTANT: 991 constant_list(constants.TF_ENABLE_PARAMETERIZED_MODULES, 992 constants.TF_MODULE_PARAMETER, 'instant_app'), 993 constants.USER_TYPE: 994 lambda arg_value, *_: [ 995 constants.TF_ENABLE_PARAMETERIZED_MODULES, 996 '--enable-optional-parameterization', 997 constants.TF_MODULE_PARAMETER, 998 str(arg_value) 999 ], 1000 constants.ITERATIONS: 1001 lambda arg_value, *_: [ 1002 '--retry-strategy', constants.ITERATIONS, 1003 '--max-testcase-run-count', str(arg_value) 1004 ], 1005 constants.RERUN_UNTIL_FAILURE: 1006 lambda arg_value, *_: [ 1007 '--retry-strategy', constants.RERUN_UNTIL_FAILURE, 1008 '--max-testcase-run-count', str(arg_value) 1009 ], 1010 constants.RETRY_ANY_FAILURE: 1011 lambda arg_value, *_: [ 1012 '--retry-strategy', constants.RETRY_ANY_FAILURE, 1013 '--max-testcase-run-count', str(arg_value) 1014 ], 1015 constants.COLLECT_TESTS_ONLY: 1016 constant_list('--collect-tests-only'), 1017 constants.NO_ENABLE_ROOT: 1018 constant_list('--no-enable-root'), 1019 constants.TF_DEBUG: 1020 print_message("Please attach process to your IDE..."), 1021 constants.ANNOTATION_FILTER: 1022 generate_annotation_filter_args, 1023 constants.TEST_FILTER: 1024 lambda arg_value, *_: [ 1025 '--test-arg', 1026 'com.android.tradefed.testtype.AndroidJUnitTest:' 1027 f'include-filter:{arg_value}', 1028 '--test-arg', 1029 'com.android.tradefed.testtype.GTest:native-test-flag:' 1030 f'--gtest_filter={arg_value}' 1031 ], 1032 constants.TEST_TIMEOUT: 1033 lambda arg_value, *_: [ 1034 '--test-arg', 1035 'com.android.tradefed.testtype.AndroidJUnitTest:' 1036 f'shell-timeout:{arg_value}', 1037 '--test-arg', 1038 'com.android.tradefed.testtype.AndroidJUnitTest:' 1039 f'test-timeout:{arg_value}', 1040 '--test-arg', 1041 'com.android.tradefed.testtype.HostGTest:' 1042 f'native-test-timeout:{arg_value}', 1043 '--test-arg', 1044 'com.android.tradefed.testtype.GTest:' 1045 f'native-test-timeout:{arg_value}', 1046 ] 1047 }) 1048 1049 for arg in extra_args: 1050 if arg in supported_tf_args: 1051 tf_args = supported_tf_args[arg](extra_args[arg], mod_info, 1052 test_infos) 1053 if tf_args: 1054 supported_args.extend(tf_args) 1055 continue 1056 1057 if arg in (constants.TF_TEMPLATE, 1058 constants.TF_EARLY_DEVICE_RELEASE, 1059 constants.INVOCATION_ID, 1060 constants.WORKUNIT_ID, 1061 constants.REQUEST_UPLOAD_RESULT, 1062 constants.LOCAL_BUILD_ID, 1063 constants.BUILD_TARGET, 1064 constants.ENABLE_DEVICE_PREPARER, 1065 constants.DRY_RUN, 1066 constants.VERIFY_ENV_VARIABLE, 1067 constants.FLAKES_INFO, 1068 constants.DISABLE_UPLOAD_RESULT): 1069 continue 1070 unsupported_args.append(arg) 1071 return supported_args, unsupported_args 1072 1073def get_include_filter(test_infos: List[test_info.TestInfo]) -> List[str]: 1074 """Generate a list of tradefed filter argument from TestInfos. 1075 1076 The tradefed argument format should be: 1077 atest-include-filter <module-name>:<include-filter-value> 1078 """ 1079 tf_args = [] 1080 for info in test_infos: 1081 filters = set() 1082 for test_info_filter in info.data.get(constants.TI_FILTER, []): 1083 filters.update(test_info_filter.to_set_of_tf_strings()) 1084 for test_filter in filters: 1085 filter_arg = constants.TF_ATEST_INCLUDE_FILTER_VALUE_FMT.format( 1086 test_name=info.test_name, test_filter=test_filter) 1087 tf_args.extend([constants.TF_ATEST_INCLUDE_FILTER, filter_arg]) 1088 return tf_args 1089