1# Copyright 2018, The Android Open Source Project 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); 4# you may not use this file except in compliance with the License. 5# You may obtain a copy of the License at 6# 7# http://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12# See the License for the specific language governing permissions and 13# limitations under the License. 14 15""" 16Robolectric test runner class. 17 18This test runner will be short lived, once robolectric support v2 is in, then 19robolectric tests will be invoked through AtestTFTestRunner. 20""" 21 22# pylint: disable=line-too-long 23 24import json 25import logging 26import os 27import re 28import tempfile 29import time 30 31from functools import partial 32from pathlib import Path 33from typing import List 34 35from atest import atest_utils 36from atest import constants 37 38from atest.atest_enum import ExitCode 39from atest.test_finders import test_info 40from atest.test_runners import test_runner_base 41from atest.test_runners.event_handler import EventHandler 42 43POLL_FREQ_SECS = 0.1 44# A pattern to match event like below 45#TEST_FAILED {'className':'SomeClass', 'testName':'SomeTestName', 46# 'trace':'{"trace":"AssertionError: <true> is equal to <false>\n 47# at FailureStrategy.fail(FailureStrategy.java:24)\n 48# at FailureStrategy.fail(FailureStrategy.java:20)\n"}\n\n 49EVENT_RE = re.compile(r'^(?P<event_name>[A-Z_]+) (?P<json_data>{(.\r*|\n)*})(?:\n|$)') 50 51 52class RobolectricTestRunner(test_runner_base.TestRunnerBase): 53 """Robolectric Test Runner class.""" 54 NAME = 'RobolectricTestRunner' 55 # We don't actually use EXECUTABLE because we're going to use 56 # atest_utils.build to kick off the test but if we don't set it, the base 57 # class will raise an exception. 58 EXECUTABLE = 'make' 59 60 # pylint: disable=useless-super-delegation 61 def __init__(self, results_dir, **kwargs): 62 """Init stuff for robolectric runner class.""" 63 super().__init__(results_dir, **kwargs) 64 # TODO: Rollback when found a solution to b/183335046. 65 if not os.getenv(test_runner_base.OLD_OUTPUT_ENV_VAR): 66 self.is_verbose = True 67 else: 68 self.is_verbose = logging.getLogger().isEnabledFor(logging.DEBUG) 69 70 def run_tests(self, test_infos, extra_args, reporter): 71 """Run the list of test_infos. See base class for more. 72 73 Args: 74 test_infos: A list of TestInfos. 75 extra_args: Dict of extra args to add to test run. 76 reporter: An instance of result_report.ResultReporter. 77 78 Returns: 79 0 if tests succeed, non-zero otherwise. 80 """ 81 # TODO: Rollback when found a solution to b/183335046. 82 if os.getenv(test_runner_base.OLD_OUTPUT_ENV_VAR): 83 return self.run_tests_pretty(test_infos, extra_args, reporter) 84 return self.run_tests_raw(test_infos, extra_args, reporter) 85 86 def run_tests_raw(self, test_infos, extra_args, reporter): 87 """Run the list of test_infos with raw output. 88 89 Args: 90 test_infos: List of TestInfo. 91 extra_args: Dict of extra args to add to test run. 92 reporter: A ResultReporter Instance. 93 94 Returns: 95 0 if tests succeed, non-zero otherwise. 96 """ 97 reporter.register_unsupported_runner(self.NAME) 98 ret_code = ExitCode.SUCCESS 99 for test_info in test_infos: 100 full_env_vars = self._get_full_build_environ(test_info, 101 extra_args) 102 run_cmd = self.generate_run_commands([test_info], extra_args)[0] 103 subproc = self.run(run_cmd, 104 output_to_stdout=self.is_verbose, 105 env_vars=full_env_vars) 106 ret_code |= self.wait_for_subprocess(subproc) 107 if not ret_code: 108 ret_code = self._check_robo_tests_result(test_infos) 109 return ret_code 110 111 def run_tests_pretty(self, test_infos, extra_args, reporter): 112 """Run the list of test_infos with pretty output mode. 113 114 Args: 115 test_infos: List of TestInfo. 116 extra_args: Dict of extra args to add to test run. 117 reporter: A ResultReporter Instance. 118 119 Returns: 120 0 if tests succeed, non-zero otherwise. 121 """ 122 ret_code = ExitCode.SUCCESS 123 for test_info in test_infos: 124 # Create a temp communication file. 125 with tempfile.NamedTemporaryFile(dir=self.results_dir) as event_file: 126 # Prepare build environment parameter. 127 full_env_vars = self._get_full_build_environ(test_info, 128 extra_args, 129 event_file) 130 run_cmd = self.generate_run_commands([test_info], extra_args)[0] 131 subproc = self.run(run_cmd, 132 output_to_stdout=self.is_verbose, 133 env_vars=full_env_vars) 134 event_handler = EventHandler(reporter, self.NAME) 135 # Start polling. 136 self.handle_subprocess(subproc, 137 partial(self._exec_with_robo_polling, 138 event_file, 139 subproc, 140 event_handler)) 141 ret_code |= self.wait_for_subprocess(subproc) 142 if not ret_code: 143 ret_code = self._check_robo_tests_result(test_infos) 144 return ret_code 145 146 def _get_full_build_environ(self, test_info=None, extra_args=None, 147 event_file=None): 148 """Helper to get full build environment. 149 150 Args: 151 test_info: TestInfo object. 152 extra_args: Dict of extra args to add to test run. 153 event_file: A file-like object that can be used as a temporary 154 storage area. 155 """ 156 full_env_vars = os.environ.copy() 157 env_vars = self.generate_env_vars(test_info, 158 extra_args, 159 event_file) 160 full_env_vars.update(env_vars) 161 return full_env_vars 162 163 def _exec_with_robo_polling(self, communication_file, robo_proc, 164 event_handler): 165 """Polling data from communication file 166 167 Polling data from communication file. Exit when communication file 168 is empty and subprocess ended. 169 170 Args: 171 communication_file: A monitored communication file. 172 robo_proc: The build process. 173 event_handler: A file-like object storing the events of robolectric tests. 174 """ 175 buf = '' 176 while True: 177 # Make sure that ATest gets content from current position. 178 communication_file.seek(0, 1) 179 data = communication_file.read() 180 if isinstance(data, bytes): 181 data = data.decode() 182 buf += data 183 reg = re.compile(r'(.|\n)*}\n\n') 184 if not reg.match(buf) or data == '': 185 if robo_proc.poll() is not None: 186 logging.debug('Build process exited early') 187 return 188 time.sleep(POLL_FREQ_SECS) 189 else: 190 # Read all new data and handle it at one time. 191 for event in re.split(r'\n\n', buf): 192 match = EVENT_RE.match(event) 193 if match: 194 try: 195 event_data = json.loads(match.group('json_data'), 196 strict=False) 197 except ValueError: 198 # Parse event fail, continue to parse next one. 199 logging.debug('"%s" is not valid json format.', 200 match.group('json_data')) 201 continue 202 event_name = match.group('event_name') 203 event_handler.process_event(event_name, event_data) 204 buf = '' 205 206 @staticmethod 207 def generate_env_vars(test_info, extra_args, event_file=None): 208 """Turn the args into env vars. 209 210 Robolectric tests specify args through env vars, so look for class 211 filters and debug args to apply to the env. 212 213 Args: 214 test_info: TestInfo class that holds the class filter info. 215 extra_args: Dict of extra args to apply for test run. 216 event_file: A file-like object storing the events of robolectric 217 tests. 218 219 Returns: 220 Dict of env vars to pass into invocation. 221 """ 222 env_var = {} 223 for arg in extra_args: 224 if constants.WAIT_FOR_DEBUGGER == arg: 225 env_var['DEBUG_ROBOLECTRIC'] = 'true' 226 continue 227 filters = test_info.data.get(constants.TI_FILTER) 228 if filters: 229 robo_filter = next(iter(filters)) 230 env_var['ROBOTEST_FILTER'] = robo_filter.class_name 231 if robo_filter.methods: 232 logging.debug('method filtering not supported for robolectric ' 233 'tests yet.') 234 if event_file: 235 env_var['EVENT_FILE_ROBOLECTRIC'] = event_file.name 236 return env_var 237 238 # pylint: disable=unnecessary-pass 239 # Please keep above disable flag to ensure host_env_check is overriden. 240 def host_env_check(self): 241 """Check that host env has everything we need. 242 243 We actually can assume the host env is fine because we have the same 244 requirements that atest has. Update this to check for android env vars 245 if that changes. 246 """ 247 pass 248 249 def get_test_runner_build_reqs(self, test_infos: List[test_info.TestInfo]): 250 """Return the build requirements. 251 252 Args: 253 test_infos: List of TestInfo. 254 255 Returns: 256 Set of build targets. 257 """ 258 build_targets = set() 259 build_targets |= test_runner_base.gather_build_targets(test_infos) 260 return build_targets 261 262 # pylint: disable=unused-argument 263 def generate_run_commands(self, test_infos, extra_args, port=None): 264 """Generate a list of run commands from TestInfos. 265 266 Args: 267 test_infos: A set of TestInfo instances. 268 extra_args: A Dict of extra args to append. 269 port: Optional. An int of the port number to send events to. 270 Subprocess reporter in TF won't try to connect if it's None. 271 272 Returns: 273 A list of run commands to run the tests. 274 """ 275 run_cmds = [] 276 for test_info in test_infos: 277 robo_command = atest_utils.get_build_cmd() + [str(test_info.test_name)] 278 run_cmd = ' '.join(x for x in robo_command) 279 if constants.DRY_RUN in extra_args: 280 run_cmd = run_cmd.replace( 281 os.environ.get(constants.ANDROID_BUILD_TOP) + os.sep, '') 282 run_cmds.append(run_cmd) 283 return run_cmds 284 285 @staticmethod 286 def _check_robo_tests_result(test_infos): 287 """Check the result of test_infos with raw output. 288 289 Args: 290 test_infos: List of TestInfo. 291 292 Returns: 293 0 if tests succeed, non-zero otherwise. 294 """ 295 for test_info in test_infos: 296 result_output = Path( 297 os.getenv(constants.ANDROID_PRODUCT_OUT, '')).joinpath( 298 f'obj/ROBOLECTRIC/{test_info.test_name}' 299 f'_intermediates/output.out') 300 if result_output.exists(): 301 with result_output.open() as f: 302 for line in f.readlines(): 303 if str(line).find('FAILURES!!!') >= 0: 304 logging.debug('%s is failed from %s', 305 test_info.test_name, result_output) 306 return ExitCode.TEST_FAILURE 307 return ExitCode.SUCCESS 308