1#!/usr/bin/env python 2# 3# Copyright 2018, The Android Open Source Project 4# 5# Licensed under the Apache License, Version 2.0 (the "License"); 6# you may not use this file except in compliance with the License. 7# You may obtain a copy of the License at 8# 9# http://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an "AS IS" BASIS, 13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14# See the License for the specific language governing permissions and 15# limitations under the License. 16 17"""Unittests for atest_tf_test_runner.""" 18 19import os 20import sys 21import tempfile 22import unittest 23import json 24import socket 25import mock 26 27# pylint: disable=import-error 28import constants 29import unittest_constants as uc 30import unittest_utils 31import atest_tf_test_runner as atf_tr 32import event_handler 33from test_finders import test_info 34 35if sys.version_info[0] == 2: 36 from StringIO import StringIO 37else: 38 from io import StringIO 39 40#pylint: disable=protected-access 41#pylint: disable=invalid-name 42TEST_INFO_DIR = '/tmp/atest_run_1510085893_pi_Nbi' 43METRICS_DIR = '%s/baseline-metrics' % TEST_INFO_DIR 44METRICS_DIR_ARG = '--metrics-folder %s ' % METRICS_DIR 45RUN_CMD_ARGS = '{metrics}--log-level WARN{serial}' 46RUN_CMD = atf_tr.AtestTradefedTestRunner._RUN_CMD.format( 47 exe=atf_tr.AtestTradefedTestRunner.EXECUTABLE, 48 template=atf_tr.AtestTradefedTestRunner._TF_TEMPLATE, 49 args=RUN_CMD_ARGS) 50FULL_CLASS2_NAME = 'android.jank.cts.ui.SomeOtherClass' 51CLASS2_FILTER = test_info.TestFilter(FULL_CLASS2_NAME, frozenset()) 52METHOD2_FILTER = test_info.TestFilter(uc.FULL_CLASS_NAME, frozenset([uc.METHOD2_NAME])) 53MODULE_ARG1 = [(constants.TF_INCLUDE_FILTER_OPTION, "A"), 54 (constants.TF_INCLUDE_FILTER_OPTION, "B")] 55MODULE_ARG2 = [] 56CLASS2_METHOD_FILTER = test_info.TestFilter(FULL_CLASS2_NAME, 57 frozenset([uc.METHOD_NAME, uc.METHOD2_NAME])) 58MODULE2_INFO = test_info.TestInfo(uc.MODULE2_NAME, 59 atf_tr.AtestTradefedTestRunner.NAME, 60 set(), 61 data={constants.TI_REL_CONFIG: uc.CONFIG2_FILE, 62 constants.TI_FILTER: frozenset()}) 63CLASS1_BUILD_TARGETS = {'class_1_build_target'} 64CLASS1_INFO = test_info.TestInfo(uc.MODULE_NAME, 65 atf_tr.AtestTradefedTestRunner.NAME, 66 CLASS1_BUILD_TARGETS, 67 data={constants.TI_REL_CONFIG: uc.CONFIG_FILE, 68 constants.TI_FILTER: frozenset([uc.CLASS_FILTER])}) 69CLASS2_BUILD_TARGETS = {'class_2_build_target'} 70CLASS2_INFO = test_info.TestInfo(uc.MODULE_NAME, 71 atf_tr.AtestTradefedTestRunner.NAME, 72 CLASS2_BUILD_TARGETS, 73 data={constants.TI_REL_CONFIG: uc.CONFIG_FILE, 74 constants.TI_FILTER: frozenset([CLASS2_FILTER])}) 75CLASS3_BUILD_TARGETS = {'class_3_build_target'} 76CLASS3_INFO = test_info.TestInfo(uc.MODULE_NAME, 77 atf_tr.AtestTradefedTestRunner.NAME, 78 CLASS3_BUILD_TARGETS, 79 data={constants.TI_REL_CONFIG: uc.CONFIG_FILE, 80 constants.TI_FILTER: frozenset(), 81 constants.TI_MODULE_ARG: MODULE_ARG1}) 82CLASS4_BUILD_TARGETS = {'class_4_build_target'} 83CLASS4_INFO = test_info.TestInfo(uc.MODULE_NAME, 84 atf_tr.AtestTradefedTestRunner.NAME, 85 CLASS4_BUILD_TARGETS, 86 data={constants.TI_REL_CONFIG: uc.CONFIG_FILE, 87 constants.TI_FILTER: frozenset(), 88 constants.TI_MODULE_ARG: MODULE_ARG2}) 89CLASS1_CLASS2_MODULE_INFO = test_info.TestInfo( 90 uc.MODULE_NAME, 91 atf_tr.AtestTradefedTestRunner.NAME, 92 uc.MODULE_BUILD_TARGETS | CLASS1_BUILD_TARGETS | CLASS2_BUILD_TARGETS, 93 uc.MODULE_DATA) 94FLAT_CLASS_INFO = test_info.TestInfo( 95 uc.MODULE_NAME, 96 atf_tr.AtestTradefedTestRunner.NAME, 97 CLASS1_BUILD_TARGETS | CLASS2_BUILD_TARGETS, 98 data={constants.TI_REL_CONFIG: uc.CONFIG_FILE, 99 constants.TI_FILTER: frozenset([uc.CLASS_FILTER, CLASS2_FILTER])}) 100FLAT2_CLASS_INFO = test_info.TestInfo( 101 uc.MODULE_NAME, 102 atf_tr.AtestTradefedTestRunner.NAME, 103 CLASS3_BUILD_TARGETS | CLASS4_BUILD_TARGETS, 104 data={constants.TI_REL_CONFIG: uc.CONFIG_FILE, 105 constants.TI_FILTER: frozenset(), 106 constants.TI_MODULE_ARG: MODULE_ARG1 + MODULE_ARG2}) 107GTF_INT_CONFIG = os.path.join(uc.GTF_INT_DIR, uc.GTF_INT_NAME + '.xml') 108CLASS2_METHOD_INFO = test_info.TestInfo( 109 uc.MODULE_NAME, 110 atf_tr.AtestTradefedTestRunner.NAME, 111 set(), 112 data={constants.TI_REL_CONFIG: uc.CONFIG_FILE, 113 constants.TI_FILTER: 114 frozenset([test_info.TestFilter( 115 FULL_CLASS2_NAME, frozenset([uc.METHOD_NAME, uc.METHOD2_NAME]))])}) 116METHOD_AND_CLASS2_METHOD = test_info.TestInfo( 117 uc.MODULE_NAME, 118 atf_tr.AtestTradefedTestRunner.NAME, 119 uc.MODULE_BUILD_TARGETS, 120 data={constants.TI_REL_CONFIG: uc.CONFIG_FILE, 121 constants.TI_FILTER: frozenset([uc.METHOD_FILTER, CLASS2_METHOD_FILTER])}) 122METHOD_METHOD2_AND_CLASS2_METHOD = test_info.TestInfo( 123 uc.MODULE_NAME, 124 atf_tr.AtestTradefedTestRunner.NAME, 125 uc.MODULE_BUILD_TARGETS, 126 data={constants.TI_REL_CONFIG: uc.CONFIG_FILE, 127 constants.TI_FILTER: frozenset([uc.FLAT_METHOD_FILTER, CLASS2_METHOD_FILTER])}) 128METHOD2_INFO = test_info.TestInfo( 129 uc.MODULE_NAME, 130 atf_tr.AtestTradefedTestRunner.NAME, 131 set(), 132 data={constants.TI_REL_CONFIG: uc.CONFIG_FILE, 133 constants.TI_FILTER: frozenset([METHOD2_FILTER])}) 134 135EVENTS_NORMAL = [ 136 ('TEST_MODULE_STARTED', { 137 'moduleContextFileName':'serial-util1146216{974}2772610436.ser', 138 'moduleName':'someTestModule'}), 139 ('TEST_RUN_STARTED', {'testCount': 2}), 140 ('TEST_STARTED', {'start_time':52, 'className':'someClassName', 141 'testName':'someTestName'}), 142 ('TEST_ENDED', {'end_time':1048, 'className':'someClassName', 143 'testName':'someTestName'}), 144 ('TEST_STARTED', {'start_time':48, 'className':'someClassName2', 145 'testName':'someTestName2'}), 146 ('TEST_FAILED', {'className':'someClassName2', 'testName':'someTestName2', 147 'trace': 'someTrace'}), 148 ('TEST_ENDED', {'end_time':9876450, 'className':'someClassName2', 149 'testName':'someTestName2'}), 150 ('TEST_RUN_ENDED', {}), 151 ('TEST_MODULE_ENDED', {'foo': 'bar'}), 152] 153 154class AtestTradefedTestRunnerUnittests(unittest.TestCase): 155 """Unit tests for atest_tf_test_runner.py""" 156 157 def setUp(self): 158 self.tr = atf_tr.AtestTradefedTestRunner(results_dir=TEST_INFO_DIR) 159 160 def tearDown(self): 161 mock.patch.stopall() 162 163 @mock.patch.object(atf_tr.AtestTradefedTestRunner, 164 '_start_socket_server') 165 @mock.patch.object(atf_tr.AtestTradefedTestRunner, 166 'run') 167 @mock.patch.object(atf_tr.AtestTradefedTestRunner, 168 '_exec_with_tf_polling') 169 @mock.patch.object(atf_tr.AtestTradefedTestRunner, 170 '_create_test_args', return_value=['some_args']) 171 @mock.patch.object(atf_tr.AtestTradefedTestRunner, 172 'generate_run_commands', return_value='some_cmd') 173 @mock.patch.object(atf_tr.AtestTradefedTestRunner, 174 '_process_connection', return_value=None) 175 @mock.patch('os.killpg', return_value=None) 176 @mock.patch('os.getpgid', return_value=None) 177 @mock.patch('signal.signal', return_value=None) 178 def test_run_tests_pretty(self, _signal, _pgid, _killpg, _process, 179 _run_cmd, _test_args, mock_exec_w_poll, 180 mock_run, mock_start_socket_server): 181 """Test _run_tests_pretty method.""" 182 mock_subproc = mock.Mock() 183 mock_run.return_value = mock_subproc 184 mock_subproc.returncode = 0 185 mock_server = mock.Mock() 186 mock_server.getsockname.return_value = ('', '') 187 mock_start_socket_server.return_value = mock_server 188 mock_reporter = mock.Mock() 189 190 # Test no early TF exit 191 mock_exec_w_poll.return_value = ('some_conn', 'some_addr') 192 self.tr.run_tests_pretty([MODULE2_INFO], {}, mock_reporter) 193 194 # Test early TF exit 195 tmp_file = tempfile.NamedTemporaryFile() 196 with open(tmp_file.name, 'w') as f: 197 f.write("tf msg") 198 self.tr.test_log_file = tmp_file 199 mock_exec_w_poll.side_effect = atf_tr.TradeFedExitError() 200 capture_output = StringIO() 201 sys.stdout = capture_output 202 self.assertRaises(atf_tr.TradeFedExitError, self.tr.run_tests_pretty, 203 [MODULE2_INFO], {}, mock_reporter) 204 sys.stdout = sys.__stdout__ 205 self.assertTrue('tf msg' in capture_output.getvalue()) 206 207 def test_exec_with_tf_polling(self): 208 """Test _exec_with_tf_polling method.""" 209 mock_socket_func = mock.Mock() 210 mock_socket_func.side_effect = [socket.timeout, socket.timeout, 211 socket.timeout] 212 mock_tf_subproc = mock.Mock() 213 exit_code = 7 214 mock_tf_subproc.poll.side_effect = [None, None, exit_code] 215 mock_tf_subproc.returncode.returns = exit_code 216 # First call should raise, because TF exits before socket_func returns 217 self.assertRaises(atf_tr.TradeFedExitError, 218 self.tr._exec_with_tf_polling, 219 mock_socket_func, mock_tf_subproc) 220 # Second call succeeds because socket_func returns before TF exits 221 mock_socket_func.side_effect = [socket.timeout, 'some_return_value'] 222 mock_tf_subproc.poll.side_effect = [None] 223 self.tr._exec_with_tf_polling(mock_socket_func, mock_tf_subproc) 224 225 def test_start_socket_server(self): 226 """Test start_socket_server method.""" 227 server = self.tr._start_socket_server() 228 host, port = server.getsockname() 229 self.assertEquals(host, atf_tr.SOCKET_HOST) 230 self.assertLessEqual(port, 65535) 231 self.assertGreaterEqual(port, 1024) 232 server.close() 233 234 @mock.patch('os.path.exists') 235 @mock.patch.dict('os.environ', {'APE_API_KEY':'/tmp/123.json'}) 236 def test_try_set_gts_authentication_key_is_set_by_user(self, mock_exist): 237 """Test try_set_authentication_key_is_set_by_user method.""" 238 # Test key is set by user. 239 self.tr._try_set_gts_authentication_key() 240 mock_exist.assert_not_called() 241 242 @mock.patch('constants.GTS_GOOGLE_SERVICE_ACCOUNT') 243 @mock.patch('os.path.exists') 244 def test_try_set_gts_authentication_key_not_set(self, mock_exist, mock_key): 245 """Test try_set_authentication_key_not_set method.""" 246 # Test key neither exists nor set by user. 247 mock_exist.return_value = False 248 mock_key.return_value = '' 249 self.tr._try_set_gts_authentication_key() 250 self.assertEquals(os.environ.get('APE_API_KEY'), None) 251 252 @mock.patch.object(event_handler.EventHandler, 'process_event') 253 def test_process_connection(self, mock_pe): 254 """Test _process_connection method.""" 255 mock_socket = mock.Mock() 256 socket_data = ['%s %s' % (name, json.dumps(data)) 257 for name, data in EVENTS_NORMAL] 258 socket_data.append('') 259 mock_socket.recv.side_effect = socket_data 260 self.tr._process_connection(mock_socket, mock_pe) 261 262 calls = [mock.call.process_event(name, data) for name, data in EVENTS_NORMAL] 263 mock_pe.assert_has_calls(calls) 264 265 @mock.patch.object(event_handler.EventHandler, 'process_event') 266 def test_process_connection_multiple_lines_in_single_recv(self, mock_pe): 267 """Test _process_connection when recv reads multiple lines in one go.""" 268 mock_socket = mock.Mock() 269 squashed_events = '\n'.join(['%s %s' % (name, json.dumps(data)) 270 for name, data in EVENTS_NORMAL]) 271 socket_data = [squashed_events, ''] 272 mock_socket.recv.side_effect = socket_data 273 self.tr._process_connection(mock_socket, mock_pe) 274 calls = [mock.call.process_event(name, data) for name, data in EVENTS_NORMAL] 275 mock_pe.assert_has_calls(calls) 276 277 @mock.patch.object(event_handler.EventHandler, 'process_event') 278 def test_process_connection_with_buffering(self, mock_pe): 279 """Test _process_connection when events overflow socket buffer size""" 280 mock_socket = mock.Mock() 281 module_events = [EVENTS_NORMAL[0], EVENTS_NORMAL[-1]] 282 socket_events = ['%s %s' % (name, json.dumps(data)) 283 for name, data in module_events] 284 # test try-block code by breaking apart first event after first } 285 index = socket_events[0].index('}') + 1 286 socket_data = [socket_events[0][:index], socket_events[0][index:]] 287 # test non-try block buffering with second event 288 socket_data.extend([socket_events[1][:-4], socket_events[1][-4:], '']) 289 mock_socket.recv.side_effect = socket_data 290 self.tr._process_connection(mock_socket, mock_pe) 291 calls = [mock.call.process_event(name, data) for name, data in module_events] 292 mock_pe.assert_has_calls(calls) 293 294 @mock.patch('os.environ.get', return_value=None) 295 @mock.patch.object(atf_tr.AtestTradefedTestRunner, '_generate_metrics_folder') 296 @mock.patch('atest_utils.get_result_server_args') 297 def test_generate_run_commands_without_serial_env(self, mock_resultargs, mock_mertrics, _): 298 """Test generate_run_command method.""" 299 # Basic Run Cmd 300 mock_resultargs.return_value = [] 301 mock_mertrics.return_value = '' 302 unittest_utils.assert_strict_equal( 303 self, 304 self.tr.generate_run_commands([], {}), 305 [RUN_CMD.format(metrics='', serial='')]) 306 mock_mertrics.return_value = METRICS_DIR 307 unittest_utils.assert_strict_equal( 308 self, 309 self.tr.generate_run_commands([], {}), 310 [RUN_CMD.format(metrics=METRICS_DIR_ARG, serial='')]) 311 # Run cmd with result server args. 312 result_arg = '--result_arg' 313 mock_resultargs.return_value = [result_arg] 314 mock_mertrics.return_value = '' 315 unittest_utils.assert_strict_equal( 316 self, 317 self.tr.generate_run_commands([], {}), 318 [RUN_CMD.format(metrics='', serial='') + ' ' + result_arg]) 319 320 @mock.patch('os.environ.get') 321 @mock.patch.object(atf_tr.AtestTradefedTestRunner, '_generate_metrics_folder') 322 @mock.patch('atest_utils.get_result_server_args') 323 def test_generate_run_commands_with_serial_env(self, mock_resultargs, mock_mertrics, mock_env): 324 """Test generate_run_command method.""" 325 # Basic Run Cmd 326 env_device_serial = 'env-device-0' 327 mock_resultargs.return_value = [] 328 mock_mertrics.return_value = '' 329 mock_env.return_value = env_device_serial 330 env_serial_arg = ' --serial %s' % env_device_serial 331 # Serial env be set and without --serial arg. 332 unittest_utils.assert_strict_equal( 333 self, 334 self.tr.generate_run_commands([], {}), 335 [RUN_CMD.format(metrics='', serial=env_serial_arg)]) 336 # Serial env be set but with --serial arg. 337 arg_device_serial = 'arg-device-0' 338 arg_serial_arg = ' --serial %s' % arg_device_serial 339 unittest_utils.assert_strict_equal( 340 self, 341 self.tr.generate_run_commands([], {constants.SERIAL:arg_device_serial}), 342 [RUN_CMD.format(metrics='', serial=arg_serial_arg)]) 343 # Serial env be set but with -n arg 344 unittest_utils.assert_strict_equal( 345 self, 346 self.tr.generate_run_commands([], {constants.HOST}), 347 [RUN_CMD.format(metrics='', serial='') + 348 ' -n --prioritize-host-config --skip-host-arch-check']) 349 350 351 def test_flatten_test_filters(self): 352 """Test _flatten_test_filters method.""" 353 # No Flattening 354 filters = self.tr._flatten_test_filters({uc.CLASS_FILTER}) 355 unittest_utils.assert_strict_equal(self, frozenset([uc.CLASS_FILTER]), 356 filters) 357 filters = self.tr._flatten_test_filters({CLASS2_FILTER}) 358 unittest_utils.assert_strict_equal( 359 self, frozenset([CLASS2_FILTER]), filters) 360 filters = self.tr._flatten_test_filters({uc.METHOD_FILTER}) 361 unittest_utils.assert_strict_equal( 362 self, frozenset([uc.METHOD_FILTER]), filters) 363 filters = self.tr._flatten_test_filters({uc.METHOD_FILTER, 364 CLASS2_METHOD_FILTER}) 365 unittest_utils.assert_strict_equal( 366 self, frozenset([uc.METHOD_FILTER, CLASS2_METHOD_FILTER]), filters) 367 # Flattening 368 filters = self.tr._flatten_test_filters({uc.METHOD_FILTER, 369 METHOD2_FILTER}) 370 unittest_utils.assert_strict_equal( 371 self, filters, frozenset([uc.FLAT_METHOD_FILTER])) 372 filters = self.tr._flatten_test_filters({uc.METHOD_FILTER, 373 METHOD2_FILTER, 374 CLASS2_METHOD_FILTER,}) 375 unittest_utils.assert_strict_equal( 376 self, filters, frozenset([uc.FLAT_METHOD_FILTER, 377 CLASS2_METHOD_FILTER])) 378 379 def test_flatten_test_infos(self): 380 """Test _flatten_test_infos method.""" 381 # No Flattening 382 test_infos = self.tr._flatten_test_infos({uc.MODULE_INFO}) 383 unittest_utils.assert_equal_testinfo_sets(self, test_infos, 384 {uc.MODULE_INFO}) 385 386 test_infos = self.tr._flatten_test_infos([uc.MODULE_INFO, MODULE2_INFO]) 387 unittest_utils.assert_equal_testinfo_sets( 388 self, test_infos, {uc.MODULE_INFO, MODULE2_INFO}) 389 390 test_infos = self.tr._flatten_test_infos({CLASS1_INFO}) 391 unittest_utils.assert_equal_testinfo_sets(self, test_infos, 392 {CLASS1_INFO}) 393 394 test_infos = self.tr._flatten_test_infos({uc.INT_INFO}) 395 unittest_utils.assert_equal_testinfo_sets(self, test_infos, 396 {uc.INT_INFO}) 397 398 test_infos = self.tr._flatten_test_infos({uc.METHOD_INFO}) 399 unittest_utils.assert_equal_testinfo_sets(self, test_infos, 400 {uc.METHOD_INFO}) 401 402 # Flattening 403 test_infos = self.tr._flatten_test_infos({CLASS1_INFO, CLASS2_INFO}) 404 unittest_utils.assert_equal_testinfo_sets(self, test_infos, 405 {FLAT_CLASS_INFO}) 406 407 test_infos = self.tr._flatten_test_infos({CLASS1_INFO, uc.INT_INFO, 408 CLASS2_INFO}) 409 unittest_utils.assert_equal_testinfo_sets(self, test_infos, 410 {uc.INT_INFO, 411 FLAT_CLASS_INFO}) 412 413 test_infos = self.tr._flatten_test_infos({CLASS1_INFO, uc.MODULE_INFO, 414 CLASS2_INFO}) 415 unittest_utils.assert_equal_testinfo_sets(self, test_infos, 416 {CLASS1_CLASS2_MODULE_INFO}) 417 418 test_infos = self.tr._flatten_test_infos({MODULE2_INFO, uc.INT_INFO, 419 CLASS1_INFO, CLASS2_INFO, 420 uc.GTF_INT_INFO}) 421 unittest_utils.assert_equal_testinfo_sets(self, test_infos, 422 {uc.INT_INFO, uc.GTF_INT_INFO, 423 FLAT_CLASS_INFO, 424 MODULE2_INFO}) 425 426 test_infos = self.tr._flatten_test_infos({uc.METHOD_INFO, 427 CLASS2_METHOD_INFO}) 428 unittest_utils.assert_equal_testinfo_sets(self, test_infos, 429 {METHOD_AND_CLASS2_METHOD}) 430 431 test_infos = self.tr._flatten_test_infos({uc.METHOD_INFO, METHOD2_INFO, 432 CLASS2_METHOD_INFO}) 433 unittest_utils.assert_equal_testinfo_sets( 434 self, test_infos, {METHOD_METHOD2_AND_CLASS2_METHOD}) 435 test_infos = self.tr._flatten_test_infos({uc.METHOD_INFO, METHOD2_INFO, 436 CLASS2_METHOD_INFO, 437 MODULE2_INFO, 438 uc.INT_INFO}) 439 unittest_utils.assert_equal_testinfo_sets( 440 self, test_infos, {uc.INT_INFO, MODULE2_INFO, 441 METHOD_METHOD2_AND_CLASS2_METHOD}) 442 443 test_infos = self.tr._flatten_test_infos({CLASS3_INFO, CLASS4_INFO}) 444 unittest_utils.assert_equal_testinfo_sets(self, test_infos, 445 {FLAT2_CLASS_INFO}) 446 447 448if __name__ == '__main__': 449 unittest.main() 450