1#!/usr/bin/python 2# Copyright 2015 The Chromium OS Authors. All rights reserved. 3# Use of this source code is governed by a BSD-style license that can be 4# found in the LICENSE file. 5# pylint: disable-msg=C0111 6 7import os, unittest 8import mox 9import common 10import subprocess 11import shutil 12import tempfile 13import types 14from autotest_lib.client.common_lib import control_data 15from autotest_lib.server import utils 16from autotest_lib.server.cros.dynamic_suite import constants 17from autotest_lib.server.cros.dynamic_suite import control_file_getter 18from autotest_lib.server.cros.dynamic_suite import suite as suite_module 19from autotest_lib.site_utils import test_runner_utils 20 21 22class StartsWithList(mox.Comparator): 23 def __init__(self, start_of_list): 24 """Mox comparator which returns True if the argument 25 to the mocked function is a list that begins with the elements 26 in start_of_list. 27 """ 28 self._lhs = start_of_list 29 30 def equals(self, rhs): 31 if len(rhs)<len(self._lhs): 32 return False 33 for (x, y) in zip(self._lhs, rhs): 34 if x != y: 35 return False 36 return True 37 38 39class ContainsSublist(mox.Comparator): 40 def __init__(self, sublist): 41 """Mox comparator which returns True if the argument 42 to the mocked function is a list that contains sublist 43 as a sub-list. 44 """ 45 self._sublist = sublist 46 47 def equals(self, rhs): 48 n = len(self._sublist) 49 if len(rhs)<n: 50 return False 51 return any((self._sublist == rhs[i:i+n]) 52 for i in xrange(len(rhs) - n + 1)) 53 54class DummyJob(object): 55 def __init__(self, id=1): 56 self.id = id 57 58class TestRunnerUnittests(mox.MoxTestBase): 59 60 def setUp(self): 61 mox.MoxTestBase.setUp(self) 62 63 64 def test_fetch_local_suite(self): 65 # Deferred until fetch_local_suite knows about non-local builds. 66 pass 67 68 69 def _results_directory_from_results_list(self, results_list): 70 """Generate a temp directory filled with provided test results. 71 72 @param results_list: List of results, each result is a tuple of strings 73 (test_name, test_status_message). 74 @returns: Absolute path to the results directory. 75 """ 76 global_dir = tempfile.mkdtemp() 77 for index, (test_name, test_status_message) in enumerate(results_list): 78 dir_name = '-'.join(['results', 79 "%02.f" % (index + 1), 80 test_name]) 81 local_dir = os.path.join(global_dir, dir_name) 82 os.mkdir(local_dir) 83 os.mkdir('%s/debug' % local_dir) 84 with open("%s/status.log" % local_dir, mode='w+') as status: 85 status.write(test_status_message) 86 status.flush() 87 return global_dir 88 89 90 def test_handle_local_result_for_good_test(self): 91 getter = self.mox.CreateMock(control_file_getter.DevServerGetter) 92 getter.get_control_file_list(suite_name=mox.IgnoreArg()).AndReturn([]) 93 job = DummyJob() 94 test = self.mox.CreateMock(control_data.ControlData) 95 test.job_retries = 5 96 self.mox.StubOutWithMock(test_runner_utils.LocalSuite, 97 '_retry_local_result') 98 self.mox.ReplayAll() 99 suite = test_runner_utils.LocalSuite([], "tag", [], None, getter, 100 job_retry=True) 101 suite._retry_handler = suite_module.RetryHandler({job.id: test}) 102 103 #No calls, should not be retried 104 directory = self._results_directory_from_results_list([ 105 ("dummy_Good", "GOOD: nonexistent test completed successfully")]) 106 new_id = suite.handle_local_result( 107 job.id, directory, 108 lambda log_entry, log_in_subdir=False: None) 109 self.assertIsNone(new_id) 110 shutil.rmtree(directory) 111 112 113 def test_handle_local_result_for_bad_test(self): 114 getter = self.mox.CreateMock(control_file_getter.DevServerGetter) 115 getter.get_control_file_list(suite_name=mox.IgnoreArg()).AndReturn([]) 116 job = DummyJob() 117 test = self.mox.CreateMock(control_data.ControlData) 118 test.job_retries = 5 119 self.mox.StubOutWithMock(test_runner_utils.LocalSuite, 120 '_retry_local_result') 121 test_runner_utils.LocalSuite._retry_local_result( 122 job.id, mox.IgnoreArg()).AndReturn(42) 123 self.mox.ReplayAll() 124 suite = test_runner_utils.LocalSuite([], "tag", [], None, getter, 125 job_retry=True) 126 suite._retry_handler = suite_module.RetryHandler({job.id: test}) 127 128 directory = self._results_directory_from_results_list([ 129 ("dummy_Bad", "FAIL")]) 130 new_id = suite.handle_local_result( 131 job.id, directory, 132 lambda log_entry, log_in_subdir=False: None) 133 self.assertIsNotNone(new_id) 134 shutil.rmtree(directory) 135 136 137 def test_generate_report_status_code_success_with_retries(self): 138 global_dir = self._results_directory_from_results_list([ 139 ("dummy_Flaky", "FAIL"), 140 ("dummy_Flaky", "GOOD: nonexistent test completed successfully")]) 141 status_code = test_runner_utils.generate_report( 142 global_dir, just_status_code=True) 143 self.assertEquals(status_code, 0) 144 shutil.rmtree(global_dir) 145 146 147 def test_generate_report_status_code_failure_with_retries(self): 148 global_dir = self._results_directory_from_results_list([ 149 ("dummy_Good", "GOOD: nonexistent test completed successfully"), 150 ("dummy_Bad", "FAIL"), 151 ("dummy_Bad", "FAIL")]) 152 status_code = test_runner_utils.generate_report( 153 global_dir, just_status_code=True) 154 self.assertNotEquals(status_code, 0) 155 shutil.rmtree(global_dir) 156 157 158 def test_get_predicate_for_test_arg(self): 159 # Assert the type signature of get_predicate_for_test(...) 160 # Because control.test_utils_wrapper calls this function, 161 # it is imperative for backwards compatilbility that 162 # the return type of the tested function does not change. 163 tests = ['dummy_test', 'e:name_expression', 'f:expression', 164 'suite:suitename'] 165 for test in tests: 166 pred, desc = test_runner_utils.get_predicate_for_test_arg(test) 167 self.assertTrue(isinstance(pred, types.FunctionType)) 168 self.assertTrue(isinstance(desc, str)) 169 170 def test_run_job(self): 171 class Object(): 172 pass 173 174 autotest_path = 'htap_tsetotua' 175 autoserv_command = os.path.join(autotest_path, 'server', 'autoserv') 176 remote = 'etomer' 177 results_dir = '/tmp/fakeresults' 178 fast_mode = False 179 job1_results_dir = '/tmp/fakeresults/results-1-gilbert' 180 job2_results_dir = '/tmp/fakeresults/results-2-sullivan' 181 args = 'matey' 182 expected_args_sublist = ['--args', args] 183 experimental_keyval = {constants.JOB_EXPERIMENTAL_KEY: False} 184 185 # Create some dummy job objects. 186 job1 = Object() 187 job2 = Object() 188 setattr(job1, 'control_type', 'cLiEnT') 189 setattr(job1, 'control_file', 'c1') 190 setattr(job1, 'id', 1) 191 setattr(job1, 'name', 'gilbert') 192 setattr(job1, 'keyvals', experimental_keyval) 193 194 setattr(job2, 'control_type', 'Server') 195 setattr(job2, 'control_file', 'c2') 196 setattr(job2, 'id', 2) 197 setattr(job2, 'name', 'sullivan') 198 setattr(job2, 'keyvals', experimental_keyval) 199 200 id_digits = 1 201 202 # Stub out subprocess.Popen and wait calls. 203 # Make them expect correct arguments. 204 def fake_readline(): 205 return b'' 206 mock_process_1 = self.mox.CreateMock(subprocess.Popen) 207 mock_process_2 = self.mox.CreateMock(subprocess.Popen) 208 fake_stdout = self.mox.CreateMock(file) 209 fake_returncode = 0 210 mock_process_1.stdout = fake_stdout 211 mock_process_1.returncode = fake_returncode 212 mock_process_2.stdout = fake_stdout 213 mock_process_2.returncode = fake_returncode 214 215 self.mox.StubOutWithMock(os, 'makedirs') 216 self.mox.StubOutWithMock(utils, 'write_keyval') 217 self.mox.StubOutWithMock(subprocess, 'Popen') 218 219 os.makedirs(job1_results_dir) 220 utils.write_keyval(job1_results_dir, experimental_keyval) 221 arglist_1 = [autoserv_command, '-p', '-r', job1_results_dir, 222 '-m', remote, '--no_console_prefix', '-l', 'gilbert', 223 '-c'] 224 subprocess.Popen(mox.And(StartsWithList(arglist_1), 225 ContainsSublist(expected_args_sublist)), 226 stdout=subprocess.PIPE, 227 stderr=subprocess.STDOUT 228 ).AndReturn(mock_process_1) 229 mock_process_1.stdout.readline().AndReturn(b'') 230 mock_process_1.wait().AndReturn(0) 231 232 os.makedirs(job2_results_dir) 233 utils.write_keyval(job2_results_dir, experimental_keyval) 234 arglist_2 = [autoserv_command, '-p', '-r', job2_results_dir, 235 '-m', remote, '--no_console_prefix', '-l', 'sullivan', 236 '-s'] 237 subprocess.Popen(mox.And(StartsWithList(arglist_2), 238 ContainsSublist(expected_args_sublist)), 239 stdout=subprocess.PIPE, 240 stderr=subprocess.STDOUT 241 ).AndReturn(mock_process_2) 242 mock_process_2.stdout.readline().AndReturn(b'') 243 mock_process_2.wait().AndReturn(0) 244 245 # Test run_job. 246 self.mox.ReplayAll() 247 code, job_res = test_runner_utils.run_job( 248 job1, remote, autotest_path,results_dir, fast_mode, id_digits, 249 0, None, args) 250 self.assertEqual(job_res, job1_results_dir) 251 self.assertEqual(code, 0) 252 code, job_res = test_runner_utils.run_job( 253 job2, remote, autotest_path, results_dir, fast_mode, id_digits, 254 0, None, args) 255 256 self.assertEqual(job_res, job2_results_dir) 257 self.assertEqual(code, 0) 258 self.mox.ResetAll() 259 260 def test_perform_local_run(self): 261 afe = test_runner_utils.setup_local_afe() 262 autotest_path = 'ottotest_path' 263 suite_name = 'sweet_name' 264 test_arg = 'suite:' + suite_name 265 remote = 'remoat' 266 build = 'bild' 267 board = 'bored' 268 fast_mode = False 269 suite_control_files = ['c1', 'c2', 'c3', 'c4'] 270 results_dir = '/tmp/test_that_results_fake' 271 id_digits = 1 272 ssh_verbosity = 2 273 ssh_options = '-F /dev/null -i /dev/null' 274 args = 'matey' 275 ignore_deps = False 276 277 # Fake suite objects that will be returned by fetch_local_suite 278 class fake_suite(object): 279 def __init__(self, suite_control_files, hosts): 280 self._suite_control_files = suite_control_files 281 self._hosts = hosts 282 self._jobs = [] 283 self._jobs_to_tests = {} 284 self.retry_hack = True 285 286 def schedule(self, *args, **kwargs): 287 for control_file in self._suite_control_files: 288 job_id = afe.create_job(control_file, hosts=self._hosts) 289 self._jobs.append(job_id) 290 self._jobs_to_tests[job_id] = control_file 291 292 def handle_local_result(self, job_id, results_dir, logger, 293 **kwargs): 294 if results_dir == "success_directory": 295 return None 296 retries = True 297 if 'retries' in kwargs: 298 retries = kwargs['retries'] 299 if retries and self.retry_hack: 300 self.retry_hack = False 301 else: 302 return None 303 control_file = self._jobs_to_tests.get(job_id) 304 job_id = afe.create_job(control_file, hosts=self._hosts) 305 self._jobs.append(job_id) 306 self._jobs_to_tests[job_id] = control_file 307 return job_id 308 309 @property 310 def jobs(self): 311 return self._jobs 312 313 def test_name_from_job(self, id): 314 return "" 315 316 # Mock out scheduling of suite and running of jobs. 317 self.mox.StubOutWithMock(test_runner_utils, 'fetch_local_suite') 318 test_runner_utils.fetch_local_suite(autotest_path, mox.IgnoreArg(), 319 afe, test_arg=test_arg, remote=remote, build=build, 320 board=board, results_directory=results_dir, 321 no_experimental=False, 322 ignore_deps=ignore_deps 323 ).AndReturn(fake_suite(suite_control_files, [remote])) 324 self.mox.StubOutWithMock(test_runner_utils, 'run_job') 325 self.mox.StubOutWithMock(test_runner_utils, 'run_provisioning_job') 326 self.mox.StubOutWithMock(test_runner_utils, '_auto_detect_labels') 327 328 test_runner_utils._auto_detect_labels(afe, remote) 329 # Test perform_local_run. Enforce that run_provisioning_job, 330 # run_job and _auto_detect_labels are called correctly. 331 test_runner_utils.run_provisioning_job( 332 'cros-version:' + build, remote, autotest_path, 333 results_dir, fast_mode, 334 ssh_verbosity, ssh_options, 335 False, False) 336 337 for control_file in suite_control_files: 338 test_runner_utils.run_job( 339 mox.ContainsAttributeValue('control_file', control_file), 340 remote, 341 autotest_path, 342 results_dir, 343 fast_mode, 344 id_digits, 345 ssh_verbosity, 346 ssh_options, 347 mox.StrContains(args), 348 False, 349 False, 350 {}, 351 ).AndReturn((0, '/fake/dir')) 352 self.mox.ReplayAll() 353 test_runner_utils.perform_local_run( 354 afe, autotest_path, ['suite:'+suite_name], remote, fast_mode, 355 build=build, board=board, ignore_deps=False, 356 ssh_verbosity=ssh_verbosity, ssh_options=ssh_options, 357 args=args, results_directory=results_dir) 358 359 360if __name__ == '__main__': 361 unittest.main() 362