1#!/usr/bin/env python3 2# 3# Copyright 2018, The Android Open Source Project 4# 5# Licensed under the Apache License, Version 2.0 (the "License"); 6# you may not use this file except in compliance with the License. 7# You may obtain a copy of the License at 8# 9# http://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an "AS IS" BASIS, 13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14# See the License for the specific language governing permissions and 15# limitations under the License. 16 17"""Unittests for atest_tf_test_runner.""" 18 19# pylint: disable=line-too-long 20 21import os 22import sys 23import tempfile 24import unittest 25import json 26 27from io import StringIO 28from unittest import mock 29 30import atest_utils 31import constants 32import unittest_constants as uc 33import unittest_utils 34 35from logstorage import atest_gcp_utils 36from test_finders import test_finder_utils 37from test_finders import test_info 38from test_runners import event_handler 39from test_runners import atest_tf_test_runner as atf_tr 40 41#pylint: disable=protected-access 42#pylint: disable=invalid-name 43METRICS_DIR = '%s/baseline-metrics' % uc.TEST_INFO_DIR 44METRICS_DIR_ARG = '--metrics-folder %s ' % METRICS_DIR 45# TODO(147567606): Replace {serial} with {extra_args} for general extra 46# arguments testing. 47RUN_CMD_ARGS = ('{metrics}--log-level-display VERBOSE --log-level VERBOSE' 48 '{device_early_release}{serial}') 49LOG_ARGS = atf_tr.AtestTradefedTestRunner._LOG_ARGS.format( 50 log_path=os.path.join(uc.TEST_INFO_DIR, atf_tr.LOG_FOLDER_NAME), 51 proto_path=os.path.join(uc.TEST_INFO_DIR, constants.ATEST_TEST_RECORD_PROTO)) 52RUN_ENV_STR = 'tf_env_var=test' 53RUN_CMD = atf_tr.AtestTradefedTestRunner._RUN_CMD.format( 54 env=RUN_ENV_STR, 55 exe=atf_tr.AtestTradefedTestRunner.EXECUTABLE, 56 template=atf_tr.AtestTradefedTestRunner._TF_TEMPLATE, 57 tf_customize_template='{tf_customize_template}', 58 args=RUN_CMD_ARGS, 59 log_args=LOG_ARGS) 60FULL_CLASS2_NAME = 'android.jank.cts.ui.SomeOtherClass' 61CLASS2_FILTER = test_info.TestFilter(FULL_CLASS2_NAME, frozenset()) 62METHOD2_FILTER = test_info.TestFilter(uc.FULL_CLASS_NAME, frozenset([uc.METHOD2_NAME])) 63MODULE_ARG1 = [(constants.TF_INCLUDE_FILTER_OPTION, "A"), 64 (constants.TF_INCLUDE_FILTER_OPTION, "B")] 65MODULE_ARG2 = [] 66CLASS2_METHOD_FILTER = test_info.TestFilter(FULL_CLASS2_NAME, 67 frozenset([uc.METHOD_NAME, uc.METHOD2_NAME])) 68MODULE2_INFO = test_info.TestInfo(uc.MODULE2_NAME, 69 atf_tr.AtestTradefedTestRunner.NAME, 70 set(), 71 data={constants.TI_REL_CONFIG: uc.CONFIG2_FILE, 72 constants.TI_FILTER: frozenset()}) 73CLASS1_BUILD_TARGETS = {'class_1_build_target'} 74CLASS1_INFO = test_info.TestInfo(uc.MODULE_NAME, 75 atf_tr.AtestTradefedTestRunner.NAME, 76 CLASS1_BUILD_TARGETS, 77 data={constants.TI_REL_CONFIG: uc.CONFIG_FILE, 78 constants.TI_FILTER: frozenset([uc.CLASS_FILTER])}) 79CLASS2_BUILD_TARGETS = {'class_2_build_target'} 80CLASS2_INFO = test_info.TestInfo(uc.MODULE_NAME, 81 atf_tr.AtestTradefedTestRunner.NAME, 82 CLASS2_BUILD_TARGETS, 83 data={constants.TI_REL_CONFIG: uc.CONFIG_FILE, 84 constants.TI_FILTER: frozenset([CLASS2_FILTER])}) 85CLASS3_BUILD_TARGETS = {'class_3_build_target'} 86CLASS3_INFO = test_info.TestInfo(uc.MODULE_NAME, 87 atf_tr.AtestTradefedTestRunner.NAME, 88 CLASS3_BUILD_TARGETS, 89 data={constants.TI_REL_CONFIG: uc.CONFIG_FILE, 90 constants.TI_FILTER: frozenset(), 91 constants.TI_MODULE_ARG: MODULE_ARG1}) 92CLASS4_BUILD_TARGETS = {'class_4_build_target'} 93CLASS4_INFO = test_info.TestInfo(uc.MODULE_NAME, 94 atf_tr.AtestTradefedTestRunner.NAME, 95 CLASS4_BUILD_TARGETS, 96 data={constants.TI_REL_CONFIG: uc.CONFIG_FILE, 97 constants.TI_FILTER: frozenset(), 98 constants.TI_MODULE_ARG: MODULE_ARG2}) 99CLASS1_CLASS2_MODULE_INFO = test_info.TestInfo( 100 uc.MODULE_NAME, 101 atf_tr.AtestTradefedTestRunner.NAME, 102 uc.MODULE_BUILD_TARGETS | CLASS1_BUILD_TARGETS | CLASS2_BUILD_TARGETS, 103 uc.MODULE_DATA) 104FLAT_CLASS_INFO = test_info.TestInfo( 105 uc.MODULE_NAME, 106 atf_tr.AtestTradefedTestRunner.NAME, 107 CLASS1_BUILD_TARGETS | CLASS2_BUILD_TARGETS, 108 data={constants.TI_REL_CONFIG: uc.CONFIG_FILE, 109 constants.TI_FILTER: frozenset([uc.CLASS_FILTER, CLASS2_FILTER])}) 110FLAT2_CLASS_INFO = test_info.TestInfo( 111 uc.MODULE_NAME, 112 atf_tr.AtestTradefedTestRunner.NAME, 113 CLASS3_BUILD_TARGETS | CLASS4_BUILD_TARGETS, 114 data={constants.TI_REL_CONFIG: uc.CONFIG_FILE, 115 constants.TI_FILTER: frozenset(), 116 constants.TI_MODULE_ARG: MODULE_ARG1 + MODULE_ARG2}) 117GTF_INT_CONFIG = os.path.join(uc.GTF_INT_DIR, uc.GTF_INT_NAME + '.xml') 118CLASS2_METHOD_INFO = test_info.TestInfo( 119 uc.MODULE_NAME, 120 atf_tr.AtestTradefedTestRunner.NAME, 121 set(), 122 data={constants.TI_REL_CONFIG: uc.CONFIG_FILE, 123 constants.TI_FILTER: 124 frozenset([test_info.TestFilter( 125 FULL_CLASS2_NAME, frozenset([uc.METHOD_NAME, uc.METHOD2_NAME]))])}) 126METHOD_AND_CLASS2_METHOD = test_info.TestInfo( 127 uc.MODULE_NAME, 128 atf_tr.AtestTradefedTestRunner.NAME, 129 uc.MODULE_BUILD_TARGETS, 130 data={constants.TI_REL_CONFIG: uc.CONFIG_FILE, 131 constants.TI_FILTER: frozenset([uc.METHOD_FILTER, CLASS2_METHOD_FILTER])}) 132METHOD_METHOD2_AND_CLASS2_METHOD = test_info.TestInfo( 133 uc.MODULE_NAME, 134 atf_tr.AtestTradefedTestRunner.NAME, 135 uc.MODULE_BUILD_TARGETS, 136 data={constants.TI_REL_CONFIG: uc.CONFIG_FILE, 137 constants.TI_FILTER: frozenset([uc.FLAT_METHOD_FILTER, CLASS2_METHOD_FILTER])}) 138METHOD2_INFO = test_info.TestInfo( 139 uc.MODULE_NAME, 140 atf_tr.AtestTradefedTestRunner.NAME, 141 set(), 142 data={constants.TI_REL_CONFIG: uc.CONFIG_FILE, 143 constants.TI_FILTER: frozenset([METHOD2_FILTER])}) 144 145INT_INFO = test_info.TestInfo( 146 uc.INT_NAME, 147 atf_tr.AtestTradefedTestRunner.NAME, 148 set(), 149 test_finder='INTEGRATION') 150 151MOD_INFO = test_info.TestInfo( 152 uc.MODULE_NAME, 153 atf_tr.AtestTradefedTestRunner.NAME, 154 set(), 155 test_finder='MODULE') 156 157MOD_INFO_NO_TEST_FINDER = test_info.TestInfo( 158 uc.MODULE_NAME, 159 atf_tr.AtestTradefedTestRunner.NAME, 160 set()) 161 162EVENTS_NORMAL = [ 163 ('TEST_MODULE_STARTED', { 164 'moduleContextFileName':'serial-util1146216{974}2772610436.ser', 165 'moduleName':'someTestModule'}), 166 ('TEST_RUN_STARTED', {'testCount': 2}), 167 ('TEST_STARTED', {'start_time':52, 'className':'someClassName', 168 'testName':'someTestName'}), 169 ('TEST_ENDED', {'end_time':1048, 'className':'someClassName', 170 'testName':'someTestName'}), 171 ('TEST_STARTED', {'start_time':48, 'className':'someClassName2', 172 'testName':'someTestName2'}), 173 ('TEST_FAILED', {'className':'someClassName2', 'testName':'someTestName2', 174 'trace': 'someTrace'}), 175 ('TEST_ENDED', {'end_time':9876450, 'className':'someClassName2', 176 'testName':'someTestName2'}), 177 ('TEST_RUN_ENDED', {}), 178 ('TEST_MODULE_ENDED', {'foo': 'bar'}), 179] 180 181class AtestTradefedTestRunnerUnittests(unittest.TestCase): 182 """Unit tests for atest_tf_test_runner.py""" 183 184 #pylint: disable=arguments-differ 185 @mock.patch.object(atf_tr.AtestTradefedTestRunner, '_get_ld_library_path') 186 @mock.patch.dict('os.environ', {constants.ANDROID_BUILD_TOP:'/'}) 187 def setUp(self, mock_get_ld_library_path): 188 mock_get_ld_library_path.return_value = RUN_ENV_STR 189 self.tr = atf_tr.AtestTradefedTestRunner(results_dir=uc.TEST_INFO_DIR) 190 191 def tearDown(self): 192 mock.patch.stopall() 193 194 @mock.patch.object(atf_tr.AtestTradefedTestRunner, 195 '_start_socket_server') 196 @mock.patch.object(atf_tr.AtestTradefedTestRunner, 197 'run') 198 @mock.patch.object(atf_tr.AtestTradefedTestRunner, 199 '_create_test_args', return_value=['some_args']) 200 @mock.patch.object(atf_tr.AtestTradefedTestRunner, 201 'generate_run_commands', return_value='some_cmd') 202 @mock.patch.object(atf_tr.AtestTradefedTestRunner, 203 '_process_connection', return_value=None) 204 @mock.patch('select.select') 205 @mock.patch('os.killpg', return_value=None) 206 @mock.patch('os.getpgid', return_value=None) 207 @mock.patch('signal.signal', return_value=None) 208 def test_run_tests_pretty(self, _signal, _pgid, _killpg, mock_select, 209 _process, _run_cmd, _test_args, 210 mock_run, mock_start_socket_server): 211 """Test _run_tests_pretty method.""" 212 mock_subproc = mock.Mock() 213 mock_run.return_value = mock_subproc 214 mock_subproc.returncode = 0 215 mock_subproc.poll.side_effect = [True, True, None] 216 mock_server = mock.Mock() 217 mock_server.getsockname.return_value = ('', '') 218 mock_start_socket_server.return_value = mock_server 219 mock_reporter = mock.Mock() 220 221 # Test no early TF exit 222 mock_conn = mock.Mock() 223 mock_server.accept.return_value = (mock_conn, 'some_addr') 224 mock_server.close.return_value = True 225 mock_select.side_effect = [([mock_server], None, None), 226 ([mock_conn], None, None)] 227 self.tr.run_tests_pretty([MODULE2_INFO], {}, mock_reporter) 228 229 # Test early TF exit 230 tmp_file = tempfile.NamedTemporaryFile() 231 with open(tmp_file.name, 'w') as f: 232 f.write("tf msg") 233 self.tr.test_log_file = tmp_file 234 mock_select.side_effect = [([], None, None)] 235 mock_subproc.poll.side_effect = None 236 capture_output = StringIO() 237 sys.stdout = capture_output 238 self.assertRaises(atf_tr.TradeFedExitError, self.tr.run_tests_pretty, 239 [MODULE2_INFO], {}, mock_reporter) 240 sys.stdout = sys.__stdout__ 241 self.assertTrue('tf msg' in capture_output.getvalue()) 242 243 @mock.patch.object(atf_tr.AtestTradefedTestRunner, '_process_connection') 244 @mock.patch('select.select') 245 def test_start_monitor_2_connection(self, mock_select, mock_process): 246 """Test _start_monitor method.""" 247 mock_server = mock.Mock() 248 mock_subproc = mock.Mock() 249 mock_reporter = mock.Mock() 250 mock_conn1 = mock.Mock() 251 mock_conn2 = mock.Mock() 252 mock_server.accept.side_effect = [(mock_conn1, 'addr 1'), 253 (mock_conn2, 'addr 2')] 254 mock_select.side_effect = [([mock_server], None, None), 255 ([mock_server], None, None), 256 ([mock_conn1], None, None), 257 ([mock_conn2], None, None), 258 ([mock_conn1], None, None), 259 ([mock_conn2], None, None)] 260 mock_process.side_effect = ['abc', 'def', False, False] 261 mock_subproc.poll.side_effect = [None, None, None, None, 262 None, True] 263 self.tr._start_monitor(mock_server, mock_subproc, mock_reporter, {}) 264 self.assertEqual(mock_process.call_count, 4) 265 calls = [mock.call.accept(), mock.call.close()] 266 mock_server.assert_has_calls(calls) 267 mock_conn1.assert_has_calls([mock.call.close()]) 268 mock_conn2.assert_has_calls([mock.call.close()]) 269 270 @mock.patch.object(atf_tr.AtestTradefedTestRunner, '_process_connection') 271 @mock.patch('select.select') 272 def test_start_monitor_tf_exit_before_2nd_connection(self, 273 mock_select, 274 mock_process): 275 """Test _start_monitor method.""" 276 mock_server = mock.Mock() 277 mock_subproc = mock.Mock() 278 mock_reporter = mock.Mock() 279 mock_conn1 = mock.Mock() 280 mock_conn2 = mock.Mock() 281 mock_server.accept.side_effect = [(mock_conn1, 'addr 1'), 282 (mock_conn2, 'addr 2')] 283 mock_select.side_effect = [([mock_server], None, None), 284 ([mock_server], None, None), 285 ([mock_conn1], None, None), 286 ([mock_conn2], None, None), 287 ([mock_conn1], None, None), 288 ([mock_conn2], None, None)] 289 mock_process.side_effect = ['abc', 'def', False, False] 290 # TF exit early but have not processed data in socket buffer. 291 mock_subproc.poll.side_effect = [None, None, True, True, 292 True, True] 293 self.tr._start_monitor(mock_server, mock_subproc, mock_reporter, {}) 294 self.assertEqual(mock_process.call_count, 4) 295 calls = [mock.call.accept(), mock.call.close()] 296 mock_server.assert_has_calls(calls) 297 mock_conn1.assert_has_calls([mock.call.close()]) 298 mock_conn2.assert_has_calls([mock.call.close()]) 299 300 301 def test_start_socket_server(self): 302 """Test start_socket_server method.""" 303 server = self.tr._start_socket_server() 304 host, port = server.getsockname() 305 self.assertEqual(host, atf_tr.SOCKET_HOST) 306 self.assertLessEqual(port, 65535) 307 self.assertGreaterEqual(port, 1024) 308 server.close() 309 310 @mock.patch('os.path.exists') 311 @mock.patch.dict('os.environ', {'APE_API_KEY':'/tmp/123.json'}) 312 def test_try_set_gts_authentication_key_is_set_by_user(self, mock_exist): 313 """Test try_set_authentication_key_is_set_by_user method.""" 314 # Test key is set by user. 315 self.tr._try_set_gts_authentication_key() 316 mock_exist.assert_not_called() 317 318 @mock.patch('os.path.join', return_value='/tmp/file_not_exist.json') 319 def test_try_set_gts_authentication_key_not_set(self, _): 320 """Test try_set_authentication_key_not_set method.""" 321 # Delete the environment variable if it's set. This is fine for this 322 # method because it's for validating the APE_API_KEY isn't set. 323 if os.environ.get('APE_API_KEY'): 324 del os.environ['APE_API_KEY'] 325 self.tr._try_set_gts_authentication_key() 326 self.assertEqual(os.environ.get('APE_API_KEY'), None) 327 328 @mock.patch.object(event_handler.EventHandler, 'process_event') 329 def test_process_connection(self, mock_pe): 330 """Test _process_connection method.""" 331 mock_socket = mock.Mock() 332 for name, data in EVENTS_NORMAL: 333 data_map = {mock_socket: ''} 334 socket_data = '%s %s' % (name, json.dumps(data)) 335 mock_socket.recv.return_value = socket_data 336 self.tr._process_connection(data_map, mock_socket, mock_pe) 337 338 calls = [mock.call.process_event(name, data) for name, data in EVENTS_NORMAL] 339 mock_pe.assert_has_calls(calls) 340 mock_socket.recv.return_value = '' 341 self.assertFalse(self.tr._process_connection(data_map, mock_socket, mock_pe)) 342 343 @mock.patch.object(event_handler.EventHandler, 'process_event') 344 def test_process_connection_multiple_lines_in_single_recv(self, mock_pe): 345 """Test _process_connection when recv reads multiple lines in one go.""" 346 mock_socket = mock.Mock() 347 squashed_events = '\n'.join(['%s %s' % (name, json.dumps(data)) 348 for name, data in EVENTS_NORMAL]) 349 socket_data = [squashed_events, ''] 350 mock_socket.recv.side_effect = socket_data 351 data_map = {mock_socket: ''} 352 self.tr._process_connection(data_map, mock_socket, mock_pe) 353 calls = [mock.call.process_event(name, data) for name, data in EVENTS_NORMAL] 354 mock_pe.assert_has_calls(calls) 355 356 @mock.patch.object(event_handler.EventHandler, 'process_event') 357 def test_process_connection_with_buffering(self, mock_pe): 358 """Test _process_connection when events overflow socket buffer size.""" 359 mock_socket = mock.Mock() 360 module_events = [EVENTS_NORMAL[0], EVENTS_NORMAL[-1]] 361 socket_events = ['%s %s' % (name, json.dumps(data)) 362 for name, data in module_events] 363 # test try-block code by breaking apart first event after first } 364 index = socket_events[0].index('}') + 1 365 socket_data = [socket_events[0][:index], socket_events[0][index:]] 366 # test non-try block buffering with second event 367 socket_data.extend([socket_events[1][:-4], socket_events[1][-4:], '']) 368 mock_socket.recv.side_effect = socket_data 369 data_map = {mock_socket: ''} 370 self.tr._process_connection(data_map, mock_socket, mock_pe) 371 self.tr._process_connection(data_map, mock_socket, mock_pe) 372 self.tr._process_connection(data_map, mock_socket, mock_pe) 373 self.tr._process_connection(data_map, mock_socket, mock_pe) 374 calls = [mock.call.process_event(name, data) for name, data in module_events] 375 mock_pe.assert_has_calls(calls) 376 377 @mock.patch.object(event_handler.EventHandler, 'process_event') 378 def test_process_connection_with_not_completed_event_data(self, mock_pe): 379 """Test _process_connection when event have \n prefix.""" 380 mock_socket = mock.Mock() 381 mock_socket.recv.return_value = ('\n%s %s' 382 %(EVENTS_NORMAL[0][0], 383 json.dumps(EVENTS_NORMAL[0][1]))) 384 data_map = {mock_socket: ''} 385 self.tr._process_connection(data_map, mock_socket, mock_pe) 386 calls = [mock.call.process_event(EVENTS_NORMAL[0][0], 387 EVENTS_NORMAL[0][1])] 388 mock_pe.assert_has_calls(calls) 389 390 @mock.patch('os.environ.get', return_value=None) 391 @mock.patch.object(atf_tr.AtestTradefedTestRunner, '_generate_metrics_folder') 392 @mock.patch('atest_utils.get_result_server_args') 393 def test_generate_run_commands_without_serial_env(self, mock_resultargs, mock_mertrics, _): 394 """Test generate_run_command method.""" 395 # Basic Run Cmd 396 mock_resultargs.return_value = [] 397 mock_mertrics.return_value = '' 398 unittest_utils.assert_strict_equal( 399 self, 400 self.tr.generate_run_commands([], {}), 401 [RUN_CMD.format(env=RUN_ENV_STR, 402 metrics='', 403 serial='', 404 tf_customize_template='', 405 device_early_release=' --no-early-device-release')]) 406 mock_mertrics.return_value = METRICS_DIR 407 unittest_utils.assert_strict_equal( 408 self, 409 self.tr.generate_run_commands([], {}), 410 [RUN_CMD.format(metrics=METRICS_DIR_ARG, 411 serial='', 412 tf_customize_template='', 413 device_early_release=' --no-early-device-release')]) 414 # Run cmd with result server args. 415 result_arg = '--result_arg' 416 mock_resultargs.return_value = [result_arg] 417 mock_mertrics.return_value = '' 418 unittest_utils.assert_strict_equal( 419 self, 420 self.tr.generate_run_commands( 421 [], {}), 422 [RUN_CMD.format(metrics='', 423 serial='', 424 tf_customize_template='', 425 device_early_release=' --no-early-device-release') + ' ' + result_arg]) 426 427 @mock.patch('os.environ.get') 428 @mock.patch.object(atf_tr.AtestTradefedTestRunner, '_generate_metrics_folder') 429 @mock.patch('atest_utils.get_result_server_args') 430 def test_generate_run_commands_with_serial_env(self, mock_resultargs, mock_mertrics, mock_env): 431 """Test generate_run_command method.""" 432 # Basic Run Cmd 433 env_device_serial = 'env-device-0' 434 mock_resultargs.return_value = [] 435 mock_mertrics.return_value = '' 436 mock_env.return_value = env_device_serial 437 env_serial_arg = ' --serial %s' % env_device_serial 438 # Serial env be set and without --serial arg. 439 unittest_utils.assert_strict_equal( 440 self, 441 self.tr.generate_run_commands([], {}), 442 [RUN_CMD.format(metrics='', 443 serial=env_serial_arg, 444 tf_customize_template='', 445 device_early_release=' --no-early-device-release')]) 446 # Serial env be set but with --serial arg. 447 arg_device_serial = 'arg-device-0' 448 arg_serial_arg = ' --serial %s' % arg_device_serial 449 unittest_utils.assert_strict_equal( 450 self, 451 self.tr.generate_run_commands([], {constants.SERIAL:arg_device_serial}), 452 [RUN_CMD.format(metrics='', 453 serial=arg_serial_arg, 454 tf_customize_template='', 455 device_early_release=' --no-early-device-release')]) 456 # Serial env be set but with -n arg 457 unittest_utils.assert_strict_equal( 458 self, 459 self.tr.generate_run_commands([], {constants.HOST: True}), 460 [RUN_CMD.format(metrics='', 461 serial='', 462 tf_customize_template='', 463 device_early_release=' --no-early-device-release') + 464 ' -n --prioritize-host-config --skip-host-arch-check']) 465 466 467 def test_flatten_test_filters(self): 468 """Test _flatten_test_filters method.""" 469 # No Flattening 470 filters = self.tr._flatten_test_filters({uc.CLASS_FILTER}) 471 unittest_utils.assert_strict_equal(self, frozenset([uc.CLASS_FILTER]), 472 filters) 473 filters = self.tr._flatten_test_filters({CLASS2_FILTER}) 474 unittest_utils.assert_strict_equal( 475 self, frozenset([CLASS2_FILTER]), filters) 476 filters = self.tr._flatten_test_filters({uc.METHOD_FILTER}) 477 unittest_utils.assert_strict_equal( 478 self, frozenset([uc.METHOD_FILTER]), filters) 479 filters = self.tr._flatten_test_filters({uc.METHOD_FILTER, 480 CLASS2_METHOD_FILTER}) 481 unittest_utils.assert_strict_equal( 482 self, frozenset([uc.METHOD_FILTER, CLASS2_METHOD_FILTER]), filters) 483 # Flattening 484 filters = self.tr._flatten_test_filters({uc.METHOD_FILTER, 485 METHOD2_FILTER}) 486 unittest_utils.assert_strict_equal( 487 self, filters, frozenset([uc.FLAT_METHOD_FILTER])) 488 filters = self.tr._flatten_test_filters({uc.METHOD_FILTER, 489 METHOD2_FILTER, 490 CLASS2_METHOD_FILTER,}) 491 unittest_utils.assert_strict_equal( 492 self, filters, frozenset([uc.FLAT_METHOD_FILTER, 493 CLASS2_METHOD_FILTER])) 494 495 def test_flatten_test_infos(self): 496 """Test _flatten_test_infos method.""" 497 # No Flattening 498 test_infos = self.tr._flatten_test_infos({uc.MODULE_INFO}) 499 unittest_utils.assert_equal_testinfo_sets(self, test_infos, 500 {uc.MODULE_INFO}) 501 502 test_infos = self.tr._flatten_test_infos([uc.MODULE_INFO, MODULE2_INFO]) 503 unittest_utils.assert_equal_testinfo_sets( 504 self, test_infos, {uc.MODULE_INFO, MODULE2_INFO}) 505 506 test_infos = self.tr._flatten_test_infos({CLASS1_INFO}) 507 unittest_utils.assert_equal_testinfo_sets(self, test_infos, 508 {CLASS1_INFO}) 509 510 test_infos = self.tr._flatten_test_infos({uc.INT_INFO}) 511 unittest_utils.assert_equal_testinfo_sets(self, test_infos, 512 {uc.INT_INFO}) 513 514 test_infos = self.tr._flatten_test_infos({uc.METHOD_INFO}) 515 unittest_utils.assert_equal_testinfo_sets(self, test_infos, 516 {uc.METHOD_INFO}) 517 518 # Flattening 519 test_infos = self.tr._flatten_test_infos({CLASS1_INFO, CLASS2_INFO}) 520 unittest_utils.assert_equal_testinfo_sets(self, test_infos, 521 {FLAT_CLASS_INFO}) 522 523 test_infos = self.tr._flatten_test_infos({CLASS1_INFO, uc.INT_INFO, 524 CLASS2_INFO}) 525 unittest_utils.assert_equal_testinfo_sets(self, test_infos, 526 {uc.INT_INFO, 527 FLAT_CLASS_INFO}) 528 529 test_infos = self.tr._flatten_test_infos({CLASS1_INFO, uc.MODULE_INFO, 530 CLASS2_INFO}) 531 unittest_utils.assert_equal_testinfo_sets(self, test_infos, 532 {CLASS1_CLASS2_MODULE_INFO}) 533 534 test_infos = self.tr._flatten_test_infos({MODULE2_INFO, uc.INT_INFO, 535 CLASS1_INFO, CLASS2_INFO, 536 uc.GTF_INT_INFO}) 537 unittest_utils.assert_equal_testinfo_sets(self, test_infos, 538 {uc.INT_INFO, uc.GTF_INT_INFO, 539 FLAT_CLASS_INFO, 540 MODULE2_INFO}) 541 542 test_infos = self.tr._flatten_test_infos({uc.METHOD_INFO, 543 CLASS2_METHOD_INFO}) 544 unittest_utils.assert_equal_testinfo_sets(self, test_infos, 545 {METHOD_AND_CLASS2_METHOD}) 546 547 test_infos = self.tr._flatten_test_infos({uc.METHOD_INFO, METHOD2_INFO, 548 CLASS2_METHOD_INFO}) 549 unittest_utils.assert_equal_testinfo_sets( 550 self, test_infos, {METHOD_METHOD2_AND_CLASS2_METHOD}) 551 test_infos = self.tr._flatten_test_infos({uc.METHOD_INFO, METHOD2_INFO, 552 CLASS2_METHOD_INFO, 553 MODULE2_INFO, 554 uc.INT_INFO}) 555 unittest_utils.assert_equal_testinfo_sets( 556 self, test_infos, {uc.INT_INFO, MODULE2_INFO, 557 METHOD_METHOD2_AND_CLASS2_METHOD}) 558 559 test_infos = self.tr._flatten_test_infos({CLASS3_INFO, CLASS4_INFO}) 560 unittest_utils.assert_equal_testinfo_sets(self, test_infos, 561 {FLAT2_CLASS_INFO}) 562 563 @mock.patch.object(test_finder_utils, 'get_test_config_and_srcs') 564 def test_create_test_args(self, mock_config): 565 """Test _create_test_args method.""" 566 # Only compile '--skip-loading-config-jar' in TF if it's not 567 # INTEGRATION finder or the finder property isn't set. 568 mock_config.return_value = '', '' 569 args = self.tr._create_test_args([MOD_INFO]) 570 self.assertTrue(constants.TF_SKIP_LOADING_CONFIG_JAR in args) 571 572 args = self.tr._create_test_args([INT_INFO]) 573 self.assertFalse(constants.TF_SKIP_LOADING_CONFIG_JAR in args) 574 575 args = self.tr._create_test_args([MOD_INFO_NO_TEST_FINDER]) 576 self.assertFalse(constants.TF_SKIP_LOADING_CONFIG_JAR in args) 577 578 args = self.tr._create_test_args([MOD_INFO_NO_TEST_FINDER, INT_INFO]) 579 self.assertFalse(constants.TF_SKIP_LOADING_CONFIG_JAR in args) 580 581 args = self.tr._create_test_args([MOD_INFO_NO_TEST_FINDER]) 582 self.assertFalse(constants.TF_SKIP_LOADING_CONFIG_JAR in args) 583 584 args = self.tr._create_test_args([MOD_INFO_NO_TEST_FINDER, INT_INFO, MOD_INFO]) 585 self.assertFalse(constants.TF_SKIP_LOADING_CONFIG_JAR in args) 586 587 @mock.patch('os.environ.get', return_value=None) 588 @mock.patch.object(atf_tr.AtestTradefedTestRunner, '_generate_metrics_folder') 589 @mock.patch('atest_utils.get_result_server_args') 590 def test_generate_run_commands_collect_tests_only(self, 591 mock_resultargs, 592 mock_mertrics, _): 593 """Test generate_run_command method.""" 594 # Testing without collect-tests-only 595 mock_resultargs.return_value = [] 596 mock_mertrics.return_value = '' 597 extra_args = {} 598 unittest_utils.assert_strict_equal( 599 self, 600 self.tr.generate_run_commands([], extra_args), 601 [RUN_CMD.format( 602 metrics='', 603 serial='', 604 tf_customize_template='', 605 device_early_release=' --no-early-device-release')]) 606 # Testing with collect-tests-only 607 mock_resultargs.return_value = [] 608 mock_mertrics.return_value = '' 609 extra_args = {constants.COLLECT_TESTS_ONLY: True} 610 unittest_utils.assert_strict_equal( 611 self, 612 self.tr.generate_run_commands([], extra_args), 613 [RUN_CMD.format( 614 metrics='', 615 serial=' --collect-tests-only', 616 tf_customize_template='', 617 device_early_release=' --no-early-device-release')]) 618 619 620 @mock.patch('os.environ.get', return_value=None) 621 @mock.patch.object(atf_tr.AtestTradefedTestRunner, '_generate_metrics_folder') 622 @mock.patch('atest_utils.get_result_server_args') 623 def test_generate_run_commands_with_tf_template(self, mock_resultargs, mock_mertrics, _): 624 """Test generate_run_command method.""" 625 tf_tmplate_key1 = 'tf_tmplate_key1' 626 tf_tmplate_val1 = 'tf_tmplate_val1' 627 tf_tmplate_key2 = 'tf_tmplate_key2' 628 tf_tmplate_val2 = 'tf_tmplate_val2' 629 # Testing with only one tradefed template command 630 mock_resultargs.return_value = [] 631 mock_mertrics.return_value = '' 632 extra_args = {constants.TF_TEMPLATE: 633 ['{}={}'.format(tf_tmplate_key1, 634 tf_tmplate_val1)]} 635 unittest_utils.assert_strict_equal( 636 self, 637 self.tr.generate_run_commands([], extra_args), 638 [RUN_CMD.format( 639 metrics='', 640 serial='', 641 device_early_release=' --no-early-device-release', 642 tf_customize_template= 643 '--template:map {}={}').format(tf_tmplate_key1, 644 tf_tmplate_val1)]) 645 # Testing with two tradefed template commands 646 extra_args = {constants.TF_TEMPLATE: 647 ['{}={}'.format(tf_tmplate_key1, 648 tf_tmplate_val1), 649 '{}={}'.format(tf_tmplate_key2, 650 tf_tmplate_val2)]} 651 unittest_utils.assert_strict_equal( 652 self, 653 self.tr.generate_run_commands([], extra_args), 654 [RUN_CMD.format( 655 metrics='', 656 serial='', 657 device_early_release=' --no-early-device-release', 658 tf_customize_template= 659 '--template:map {}={} --template:map {}={}').format( 660 tf_tmplate_key1, 661 tf_tmplate_val1, 662 tf_tmplate_key2, 663 tf_tmplate_val2)]) 664 665 @mock.patch.object(atest_gcp_utils.GCPHelper, 'get_credential_with_auth_flow') 666 @mock.patch('builtins.input') 667 def test_request_consent_of_upload_test_result_yes(self, 668 mock_input, 669 mock_get_credential_with_auth_flow): 670 """test request_consent_of_upload_test_result method.""" 671 constants.CREDENTIAL_FILE_NAME = 'cred_file' 672 constants.GCP_ACCESS_TOKEN = 'access_token' 673 tmp_folder = tempfile.mkdtemp() 674 mock_input.return_value = 'Y' 675 not_upload_file = os.path.join(tmp_folder, 676 constants.DO_NOT_UPLOAD) 677 678 self.tr._request_consent_of_upload_test_result(tmp_folder, True) 679 self.assertEqual(1, mock_get_credential_with_auth_flow.call_count) 680 self.assertFalse(os.path.exists(not_upload_file)) 681 682 self.tr._request_consent_of_upload_test_result(tmp_folder, True) 683 self.assertEqual(2, mock_get_credential_with_auth_flow.call_count) 684 self.assertFalse(os.path.exists(not_upload_file)) 685 686 @mock.patch.object(atest_gcp_utils.GCPHelper, 'get_credential_with_auth_flow') 687 @mock.patch('builtins.input') 688 def test_request_consent_of_upload_test_result_no(self, 689 mock_input, 690 mock_get_credential_with_auth_flow): 691 """test request_consent_of_upload_test_result method.""" 692 mock_input.return_value = 'N' 693 constants.CREDENTIAL_FILE_NAME = 'cred_file' 694 constants.GCP_ACCESS_TOKEN = 'access_token' 695 tmp_folder = tempfile.mkdtemp() 696 not_upload_file = os.path.join(tmp_folder, 697 constants.DO_NOT_UPLOAD) 698 699 self.tr._request_consent_of_upload_test_result(tmp_folder, True) 700 self.assertTrue(os.path.exists(not_upload_file)) 701 self.assertEqual(0, mock_get_credential_with_auth_flow.call_count) 702 self.tr._request_consent_of_upload_test_result(tmp_folder, True) 703 self.assertEqual(0, mock_get_credential_with_auth_flow.call_count) 704 705 @mock.patch('os.environ.get', return_value=None) 706 @mock.patch.object(atf_tr.AtestTradefedTestRunner, '_generate_metrics_folder') 707 @mock.patch('atest_utils.get_result_server_args') 708 def test_generate_run_commands_with_tf_early_device_release( 709 self, mock_resultargs, mock_mertrics, _): 710 """Test generate_run_command method.""" 711 # Testing without collect-tests-only 712 mock_resultargs.return_value = [] 713 mock_mertrics.return_value = '' 714 extra_args = {constants.TF_EARLY_DEVICE_RELEASE: True} 715 unittest_utils.assert_strict_equal( 716 self, 717 self.tr.generate_run_commands([], extra_args), 718 [RUN_CMD.format( 719 metrics='', 720 serial='', 721 tf_customize_template='', 722 device_early_release='')]) 723 724 @mock.patch.object(atf_tr.AtestTradefedTestRunner, '_prepare_data') 725 @mock.patch.object(atf_tr.AtestTradefedTestRunner, '_request_consent_of_upload_test_result') 726 def test_do_upload_flow(self, mock_request, mock_prepare): 727 """test _do_upload_flow method.""" 728 fake_extra_args = {} 729 fake_creds = mock.Mock() 730 fake_creds.token_response = {'access_token': 'fake_token'} 731 mock_request.return_value = fake_creds 732 fake_inv = {'invocationId': 'inv_id'} 733 fake_workunit = {'id': 'workunit_id'} 734 mock_prepare.return_value = fake_inv, fake_workunit 735 constants.TOKEN_FILE_PATH = tempfile.NamedTemporaryFile().name 736 creds, inv = self.tr._do_upload_flow(fake_extra_args) 737 self.assertEqual(fake_creds, creds) 738 self.assertEqual(fake_inv, inv) 739 self.assertEqual(fake_extra_args[constants.INVOCATION_ID], 740 fake_inv['invocationId']) 741 self.assertEqual(fake_extra_args[constants.WORKUNIT_ID], 742 fake_workunit['id']) 743 744 mock_request.return_value = None 745 creds, inv = self.tr._do_upload_flow(fake_extra_args) 746 self.assertEqual(None, creds) 747 self.assertEqual(None, inv) 748 749 @mock.patch.object(test_finder_utils, 'get_test_config_and_srcs') 750 def test_has_instant_app_config(self, mock_config): 751 """test _has_instant_app_config method.""" 752 no_instant_config = os.path.join( 753 uc.TEST_DATA_DIR, "parameter_config", "parameter.cfg") 754 instant_config = os.path.join( 755 uc.TEST_DATA_DIR, "parameter_config", "instant_app_parameter.cfg") 756 # Test find instant app config 757 mock_config.return_value = instant_config, '' 758 self.assertTrue( 759 atf_tr.AtestTradefedTestRunner._has_instant_app_config( 760 ['test_info'], 'module_info_obj')) 761 # Test not find instant app config 762 mock_config.return_value = no_instant_config, '' 763 self.assertFalse( 764 atf_tr.AtestTradefedTestRunner._has_instant_app_config( 765 ['test_info'], 'module_info_obj')) 766 767 @mock.patch.object(atf_tr.AtestTradefedTestRunner, 768 '_has_instant_app_config', return_value=True) 769 @mock.patch('os.environ.get', return_value=None) 770 @mock.patch.object(atf_tr.AtestTradefedTestRunner, 771 '_generate_metrics_folder') 772 @mock.patch('atest_utils.get_result_server_args') 773 def test_generate_run_commands_has_instant_app_config( 774 self, mock_resultargs, mock_mertrics, _, _mock_has_config): 775 """Test generate_run_command method which has instant app config.""" 776 # Basic Run Cmd 777 mock_resultargs.return_value = [] 778 mock_mertrics.return_value = '' 779 extra_tf_arg = ( 780 '{tf_test_arg} {tf_class}:{option_name}:{option_value}'.format( 781 tf_test_arg = constants.TF_TEST_ARG, 782 tf_class=constants.TF_AND_JUNIT_CLASS, 783 option_name=constants.TF_EXCLUDE_ANNOTATE, 784 option_value=constants.INSTANT_MODE_ANNOTATE)) 785 unittest_utils.assert_strict_equal( 786 self, 787 self.tr.generate_run_commands([], {}), 788 [RUN_CMD.format(env=RUN_ENV_STR, 789 metrics='', 790 serial='', 791 tf_customize_template='', 792 device_early_release=' --no-early-device-release ' 793 + extra_tf_arg)]) 794 795 @mock.patch.object(atest_utils, 'get_config_parameter') 796 @mock.patch.object(test_finder_utils, 'get_test_config_and_srcs') 797 def test_is_parameter_auto_enabled_cfg(self, mock_config, mock_cfg_para): 798 """test _is_parameter_auto_enabled_cfg method.""" 799 # Test if TF_PARA_INSTANT_APP is match 800 mock_config.return_value = 'test_config', '' 801 mock_cfg_para.return_value = {list(constants.DEFAULT_EXCLUDE_PARAS)[1], 802 list(constants.DEFAULT_EXCLUDE_PARAS)[0]} 803 self.assertFalse( 804 atf_tr.AtestTradefedTestRunner._is_parameter_auto_enabled_cfg( 805 ['test_info'], 'module_info_obj')) 806 # Test if DEFAULT_EXCLUDE_NOT_PARAS is match 807 mock_cfg_para.return_value = { 808 list(constants.DEFAULT_EXCLUDE_NOT_PARAS)[2], 809 list(constants.DEFAULT_EXCLUDE_NOT_PARAS)[0]} 810 self.assertFalse( 811 atf_tr.AtestTradefedTestRunner._is_parameter_auto_enabled_cfg( 812 ['test_info'], 'module_info_obj')) 813 # Test if have parameter not in default exclude paras 814 mock_cfg_para.return_value = { 815 'not match parameter', 816 list(constants.DEFAULT_EXCLUDE_PARAS)[1], 817 list(constants.DEFAULT_EXCLUDE_NOT_PARAS)[2]} 818 self.assertTrue( 819 atf_tr.AtestTradefedTestRunner._is_parameter_auto_enabled_cfg( 820 ['test_info'], 'module_info_obj')) 821 822 @mock.patch.object(atf_tr.AtestTradefedTestRunner, 823 '_is_parameter_auto_enabled_cfg', 824 return_value=True) 825 @mock.patch.object(test_finder_utils, 'get_test_config_and_srcs') 826 def test_create_test_args_with_auto_enable_parameter( 827 self, mock_config, _mock_is_enable): 828 """Test _create_test_args method with auto enabled parameter config.""" 829 # Should have --m on args and should not have --include-filter. 830 mock_config.return_value = '', '' 831 args = self.tr._create_test_args([MOD_INFO]) 832 self.assertTrue(constants.TF_MODULE_FILTER in args) 833 self.assertFalse(constants.TF_INCLUDE_FILTER in args) 834 835 @mock.patch.object(atf_tr.AtestTradefedTestRunner, 836 '_is_parameter_auto_enabled_cfg') 837 @mock.patch.object(test_finder_utils, 'get_test_config_and_srcs') 838 def test_parse_extra_args(self, mock_config, _mock_is_enable): 839 """Test _parse_extra_args .""" 840 # If extra_arg enable instant_app or secondary users, should not have 841 # --exclude-module-rameters even though test config parameter is auto 842 # enabled. 843 mock_config.return_value = '', '' 844 _mock_is_enable.return_value = True 845 args, _ = self.tr._parse_extra_args([MOD_INFO], [constants.INSTANT]) 846 self.assertFalse('--exclude-module-parameters' in args) 847 848 # If extra_arg not enable instant_app or secondary users, should have 849 # --exclude-module-rameters if config parameter is auto enabled. 850 _mock_is_enable.return_value = True 851 args, _ = self.tr._parse_extra_args([MOD_INFO], [constants.ALL_ABI]) 852 self.assertTrue('--exclude-module-parameters' in args) 853 854 # If extra_arg not enable instant_app or secondary users, should not 855 # have --exclude-module-rameters if config parameter is not auto enabled 856 _mock_is_enable.return_value = False 857 args, _ = self.tr._parse_extra_args([MOD_INFO], [constants.ALL_ABI]) 858 self.assertFalse('--exclude-module-parameters' in args) 859 860if __name__ == '__main__': 861 unittest.main() 862