1#!/usr/bin/python 2# 3# Copyright (c) 2012 The Chromium OS Authors. All rights reserved. 4# Use of this source code is governed by a BSD-style license that can be 5# found in the LICENSE file. 6 7 8"""Unit tests for server/cros/dynamic_suite/dynamic_suite.py.""" 9 10import collections 11from collections import OrderedDict 12import os 13import shutil 14import tempfile 15import unittest 16 17import mock 18import mox 19 20import common 21 22from autotest_lib.client.common_lib import base_job 23from autotest_lib.client.common_lib import control_data 24from autotest_lib.client.common_lib import error 25from autotest_lib.client.common_lib import priorities 26from autotest_lib.client.common_lib import utils 27from autotest_lib.client.common_lib.cros import dev_server 28from autotest_lib.server import frontend 29from autotest_lib.server.cros import provision 30from autotest_lib.server.cros.dynamic_suite import control_file_getter 31from autotest_lib.server.cros.dynamic_suite import constants 32from autotest_lib.server.cros.dynamic_suite import job_status 33from autotest_lib.server.cros.dynamic_suite import suite as SuiteBase 34from autotest_lib.server.cros.dynamic_suite import suite_common 35from autotest_lib.server.cros.dynamic_suite.comparators import StatusContains 36from autotest_lib.server.cros.dynamic_suite.fakes import FakeControlData 37from autotest_lib.server.cros.dynamic_suite.fakes import FakeJob 38from autotest_lib.server.cros.dynamic_suite.fakes import FakeMultiprocessingPool 39from autotest_lib.server.cros.dynamic_suite.suite import RetryHandler 40from autotest_lib.server.cros.dynamic_suite.suite import Suite 41 42class SuiteTest(mox.MoxTestBase): 43 """Unit tests for dynamic_suite Suite class. 44 45 @var _BUILDS: fake build 46 @var _TAG: fake suite tag 47 """ 48 49 _BOARD = 'board:board' 50 _BUILDS = {provision.CROS_VERSION_PREFIX:'build_1', 51 provision.FW_RW_VERSION_PREFIX:'fwrw_build_1'} 52 _TAG = 'au' 53 _ATTR = {'attr:attr'} 54 _DEVSERVER_HOST = 'http://dontcare:8080' 55 _FAKE_JOB_ID = 10 56 57 58 def setUp(self): 59 """Setup.""" 60 super(SuiteTest, self).setUp() 61 self.maxDiff = None 62 self.use_batch = suite_common.ENABLE_CONTROLS_IN_BATCH 63 suite_common.ENABLE_CONTROLS_IN_BATCH = False 64 self.afe = self.mox.CreateMock(frontend.AFE) 65 self.tko = self.mox.CreateMock(frontend.TKO) 66 67 self.tmpdir = tempfile.mkdtemp(suffix=type(self).__name__) 68 69 self.getter = self.mox.CreateMock(control_file_getter.ControlFileGetter) 70 self.devserver = dev_server.ImageServer(self._DEVSERVER_HOST) 71 72 self.files = OrderedDict( 73 [('one', FakeControlData(self._TAG, self._ATTR, 'data_one', 74 'FAST', job_retries=None)), 75 ('two', FakeControlData(self._TAG, self._ATTR, 'data_two', 76 'SHORT', dependencies=['feta'])), 77 ('three', FakeControlData(self._TAG, self._ATTR, 'data_three', 78 'MEDIUM')), 79 ('four', FakeControlData('other', self._ATTR, 'data_four', 80 'LONG', dependencies=['arugula'])), 81 ('five', FakeControlData(self._TAG, {'other'}, 'data_five', 82 'LONG', dependencies=['arugula', 83 'caligula'])), 84 ('six', FakeControlData(self._TAG, self._ATTR, 'data_six', 85 'LENGTHY')), 86 ('seven', FakeControlData(self._TAG, self._ATTR, 'data_seven', 87 'FAST', job_retries=1))]) 88 89 self.files_to_filter = { 90 'with/deps/...': FakeControlData(self._TAG, self._ATTR, 91 'gets filtered'), 92 'with/profilers/...': FakeControlData(self._TAG, self._ATTR, 93 'gets filtered')} 94 95 96 def tearDown(self): 97 """Teardown.""" 98 suite_common.ENABLE_CONTROLS_IN_BATCH = self.use_batch 99 super(SuiteTest, self).tearDown() 100 shutil.rmtree(self.tmpdir, ignore_errors=True) 101 102 103 def expect_control_file_parsing(self, suite_name=_TAG): 104 """Expect an attempt to parse the 'control files' in |self.files|. 105 106 @param suite_name: The suite name to parse control files for. 107 """ 108 all_files = self.files.keys() + self.files_to_filter.keys() 109 self._set_control_file_parsing_expectations(False, all_files, 110 self.files, suite_name) 111 112 113 def _set_control_file_parsing_expectations(self, already_stubbed, 114 file_list, files_to_parse, 115 suite_name): 116 """Expect an attempt to parse the 'control files' in |files|. 117 118 @param already_stubbed: parse_control_string already stubbed out. 119 @param file_list: the files the dev server returns 120 @param files_to_parse: the {'name': FakeControlData} dict of files we 121 expect to get parsed. 122 """ 123 if not already_stubbed: 124 self.mox.StubOutWithMock(control_data, 'parse_control_string') 125 126 self.mox.StubOutWithMock(suite_common.multiprocessing, 'Pool') 127 suite_common.multiprocessing.Pool( 128 processes=suite_common.get_process_limit()).AndReturn( 129 FakeMultiprocessingPool()) 130 131 self.getter.get_control_file_list( 132 suite_name=suite_name).AndReturn(file_list) 133 for file, data in files_to_parse.iteritems(): 134 self.getter.get_control_file_contents( 135 file).InAnyOrder().AndReturn(data.string) 136 control_data.parse_control_string( 137 data.string, 138 raise_warnings=True, 139 path=file).InAnyOrder().AndReturn(data) 140 141 142 def expect_control_file_parsing_in_batch(self, suite_name=_TAG): 143 """Expect an attempt to parse the contents of all control files in 144 |self.files| and |self.files_to_filter|, form them to a dict. 145 146 @param suite_name: The suite name to parse control files for. 147 """ 148 self.getter = self.mox.CreateMock(control_file_getter.DevServerGetter) 149 self.mox.StubOutWithMock(control_data, 'parse_control_string') 150 151 self.mox.StubOutWithMock(suite_common.multiprocessing, 'Pool') 152 suite_common.multiprocessing.Pool( 153 processes=suite_common.get_process_limit()).AndReturn( 154 FakeMultiprocessingPool()) 155 156 suite_info = {} 157 for k, v in self.files.iteritems(): 158 suite_info[k] = v.string 159 control_data.parse_control_string( 160 v.string, 161 raise_warnings=True, 162 path=k).InAnyOrder().AndReturn(v) 163 for k, v in self.files_to_filter.iteritems(): 164 suite_info[k] = v.string 165 self.getter._dev_server = self._DEVSERVER_HOST 166 self.getter.get_suite_info( 167 suite_name=suite_name).AndReturn(suite_info) 168 169 170 def testFindAllTestInBatch(self): 171 """Test switch on enable_getting_controls_in_batch for function 172 find_all_test.""" 173 self.use_batch = suite_common.ENABLE_CONTROLS_IN_BATCH 174 self.expect_control_file_parsing_in_batch() 175 suite_common.ENABLE_CONTROLS_IN_BATCH = True 176 177 self.mox.ReplayAll() 178 179 predicate = lambda d: d.suite == self._TAG 180 tests = SuiteBase.find_and_parse_tests(self.getter, 181 predicate, 182 self._TAG) 183 self.assertEquals(len(tests), 6) 184 self.assertTrue(self.files['one'] in tests) 185 self.assertTrue(self.files['two'] in tests) 186 self.assertTrue(self.files['three'] in tests) 187 self.assertTrue(self.files['five'] in tests) 188 self.assertTrue(self.files['six'] in tests) 189 self.assertTrue(self.files['seven'] in tests) 190 suite_common.ENABLE_CONTROLS_IN_BATCH = self.use_batch 191 192 193 def testFindAndParseStableTests(self): 194 """Should find only tests that match a predicate.""" 195 self.expect_control_file_parsing() 196 self.mox.ReplayAll() 197 198 predicate = lambda d: d.text == self.files['two'].string 199 tests = SuiteBase.find_and_parse_tests(self.getter, 200 predicate, 201 self._TAG) 202 self.assertEquals(len(tests), 1) 203 self.assertEquals(tests[0], self.files['two']) 204 205 206 def testFindSuiteSyntaxErrors(self): 207 """Check all control files for syntax errors. 208 209 This test actually parses all control files in the autotest directory 210 for syntax errors, by using the un-forgiving parser and pretending to 211 look for all control files with the suite attribute. 212 """ 213 autodir = os.path.abspath( 214 os.path.join(os.path.dirname(__file__), '..', '..', '..')) 215 fs_getter = SuiteBase.create_fs_getter(autodir) 216 predicate = lambda t: hasattr(t, 'suite') 217 SuiteBase.find_and_parse_tests(fs_getter, predicate, 218 forgiving_parser=False) 219 220 221 def testFindAndParseTestsSuite(self): 222 """Should find all tests that match a predicate.""" 223 self.expect_control_file_parsing() 224 self.mox.ReplayAll() 225 226 predicate = lambda d: d.suite == self._TAG 227 tests = SuiteBase.find_and_parse_tests(self.getter, 228 predicate, 229 self._TAG) 230 self.assertEquals(len(tests), 6) 231 self.assertTrue(self.files['one'] in tests) 232 self.assertTrue(self.files['two'] in tests) 233 self.assertTrue(self.files['three'] in tests) 234 self.assertTrue(self.files['five'] in tests) 235 self.assertTrue(self.files['six'] in tests) 236 self.assertTrue(self.files['seven'] in tests) 237 238 239 def testFindAndParseTestsAttr(self): 240 """Should find all tests that match a predicate.""" 241 self.expect_control_file_parsing() 242 self.mox.ReplayAll() 243 244 predicate = SuiteBase.matches_attribute_expression_predicate('attr:attr') 245 tests = SuiteBase.find_and_parse_tests(self.getter, 246 predicate, 247 self._TAG) 248 self.assertEquals(len(tests), 6) 249 self.assertTrue(self.files['one'] in tests) 250 self.assertTrue(self.files['two'] in tests) 251 self.assertTrue(self.files['three'] in tests) 252 self.assertTrue(self.files['four'] in tests) 253 self.assertTrue(self.files['six'] in tests) 254 self.assertTrue(self.files['seven'] in tests) 255 256 257 def testAdHocSuiteCreation(self): 258 """Should be able to schedule an ad-hoc suite by specifying 259 a single test name.""" 260 self.expect_control_file_parsing(suite_name='ad_hoc_suite') 261 self.mox.ReplayAll() 262 predicate = SuiteBase.test_name_equals_predicate('name-data_five') 263 suite = Suite.create_from_predicates([predicate], self._BUILDS, 264 self._BOARD, devserver=None, 265 cf_getter=self.getter, 266 afe=self.afe, tko=self.tko) 267 268 self.assertFalse(self.files['one'] in suite.tests) 269 self.assertFalse(self.files['two'] in suite.tests) 270 self.assertFalse(self.files['four'] in suite.tests) 271 self.assertTrue(self.files['five'] in suite.tests) 272 273 274 def mock_control_file_parsing(self): 275 """Fake out find_and_parse_tests(), returning content from |self.files|. 276 """ 277 for test in self.files.values(): 278 test.text = test.string # mimic parsing. 279 self.mox.StubOutWithMock(SuiteBase, 'find_and_parse_tests') 280 SuiteBase.find_and_parse_tests( 281 mox.IgnoreArg(), 282 mox.IgnoreArg(), 283 mox.IgnoreArg(), 284 forgiving_parser=True, 285 run_prod_code=False, 286 test_args=None).AndReturn(self.files.values()) 287 288 289 def expect_job_scheduling(self, recorder, 290 tests_to_skip=[], ignore_deps=False, 291 raises=False, suite_deps=[], suite=None, 292 extra_keyvals={}): 293 """Expect jobs to be scheduled for 'tests' in |self.files|. 294 295 @param recorder: object with a record_entry to be used to record test 296 results. 297 @param tests_to_skip: [list, of, test, names] that we expect to skip. 298 @param ignore_deps: If true, ignore tests' dependencies. 299 @param raises: If True, expect exceptions. 300 @param suite_deps: If True, add suite level dependencies. 301 @param extra_keyvals: Extra keyvals set to tests. 302 """ 303 record_job_id = suite and suite._results_dir 304 if record_job_id: 305 self.mox.StubOutWithMock(suite, '_remember_job_keyval') 306 recorder.record_entry( 307 StatusContains.CreateFromStrings('INFO', 'Start %s' % self._TAG), 308 log_in_subdir=False) 309 tests = self.files.values() 310 n = 1 311 for test in tests: 312 if test.name in tests_to_skip: 313 continue 314 dependencies = [] 315 if not ignore_deps: 316 dependencies.extend(test.dependencies) 317 if suite_deps: 318 dependencies.extend(suite_deps) 319 dependencies.append(self._BOARD) 320 build = self._BUILDS[provision.CROS_VERSION_PREFIX] 321 keyvals = { 322 'build': build, 323 'suite': self._TAG, 324 'builds': SuiteTest._BUILDS, 325 'experimental':test.experimental, 326 } 327 keyvals.update(extra_keyvals) 328 job_mock = self.afe.create_job( 329 control_file=test.text, 330 name=mox.And(mox.StrContains(build), 331 mox.StrContains(test.name)), 332 control_type=mox.IgnoreArg(), 333 meta_hosts=[self._BOARD], 334 dependencies=dependencies, 335 keyvals=keyvals, 336 max_runtime_mins=24*60, 337 timeout_mins=1440, 338 parent_job_id=None, 339 reboot_before=mox.IgnoreArg(), 340 run_reset=mox.IgnoreArg(), 341 priority=priorities.Priority.DEFAULT, 342 synch_count=test.sync_count, 343 require_ssp=test.require_ssp 344 ) 345 if raises: 346 job_mock.AndRaise(error.NoEligibleHostException()) 347 recorder.record_entry( 348 StatusContains.CreateFromStrings('START', test.name), 349 log_in_subdir=False) 350 recorder.record_entry( 351 StatusContains.CreateFromStrings('TEST_NA', test.name), 352 log_in_subdir=False) 353 recorder.record_entry( 354 StatusContains.CreateFromStrings('END', test.name), 355 log_in_subdir=False) 356 else: 357 fake_job = FakeJob(id=n) 358 job_mock.AndReturn(fake_job) 359 if record_job_id: 360 suite._remember_job_keyval(fake_job) 361 n += 1 362 363 364 def testScheduleTestsAndRecord(self): 365 """Should schedule stable and experimental tests with the AFE.""" 366 name_list = ['name-data_one', 'name-data_two', 'name-data_three', 367 'name-data_four', 'name-data_five', 'name-data_six', 368 'name-data_seven'] 369 keyval_dict = {constants.SCHEDULED_TEST_COUNT_KEY: 7, 370 constants.SCHEDULED_TEST_NAMES_KEY: repr(name_list)} 371 372 self.mock_control_file_parsing() 373 self.mox.ReplayAll() 374 suite = Suite.create_from_name(self._TAG, self._BUILDS, self._BOARD, 375 self.devserver, 376 afe=self.afe, tko=self.tko, 377 results_dir=self.tmpdir) 378 self.mox.ResetAll() 379 recorder = self.mox.CreateMock(base_job.base_job) 380 self.expect_job_scheduling(recorder, suite=suite) 381 382 self.mox.StubOutWithMock(utils, 'write_keyval') 383 utils.write_keyval(self.tmpdir, keyval_dict) 384 self.mox.ReplayAll() 385 suite.schedule(recorder.record_entry) 386 for job in suite._jobs: 387 self.assertTrue(hasattr(job, 'test_name')) 388 389 390 def testScheduleTests(self): 391 """Should schedule tests with the AFE.""" 392 name_list = ['name-data_one', 'name-data_two', 'name-data_three', 393 'name-data_four', 'name-data_five', 'name-data_six', 394 'name-data_seven'] 395 keyval_dict = {constants.SCHEDULED_TEST_COUNT_KEY: len(name_list), 396 constants.SCHEDULED_TEST_NAMES_KEY: repr(name_list)} 397 398 self.mock_control_file_parsing() 399 recorder = self.mox.CreateMock(base_job.base_job) 400 self.expect_job_scheduling(recorder) 401 self.mox.StubOutWithMock(utils, 'write_keyval') 402 utils.write_keyval(None, keyval_dict) 403 404 self.mox.ReplayAll() 405 suite = Suite.create_from_name(self._TAG, self._BUILDS, self._BOARD, 406 self.devserver, 407 afe=self.afe, tko=self.tko) 408 suite.schedule(recorder.record_entry) 409 410 411 def testScheduleTestsIgnoreDeps(self): 412 """Test scheduling tests ignoring deps.""" 413 name_list = ['name-data_one', 'name-data_two', 'name-data_three', 414 'name-data_four', 'name-data_five', 'name-data_six', 415 'name-data_seven'] 416 keyval_dict = {constants.SCHEDULED_TEST_COUNT_KEY: len(name_list), 417 constants.SCHEDULED_TEST_NAMES_KEY: repr(name_list)} 418 419 self.mock_control_file_parsing() 420 recorder = self.mox.CreateMock(base_job.base_job) 421 self.expect_job_scheduling(recorder, ignore_deps=True) 422 self.mox.StubOutWithMock(utils, 'write_keyval') 423 utils.write_keyval(None, keyval_dict) 424 425 self.mox.ReplayAll() 426 suite = Suite.create_from_name(self._TAG, self._BUILDS, self._BOARD, 427 self.devserver, 428 afe=self.afe, tko=self.tko, 429 ignore_deps=True) 430 suite.schedule(recorder.record_entry) 431 432 433 def testScheduleUnrunnableTestsTESTNA(self): 434 """Tests which fail to schedule should be TEST_NA.""" 435 # Since all tests will be fail to schedule, the num of scheduled tests 436 # will be zero. 437 name_list = [] 438 keyval_dict = {constants.SCHEDULED_TEST_COUNT_KEY: 0, 439 constants.SCHEDULED_TEST_NAMES_KEY: repr(name_list)} 440 441 self.mock_control_file_parsing() 442 recorder = self.mox.CreateMock(base_job.base_job) 443 self.expect_job_scheduling(recorder, raises=True) 444 self.mox.StubOutWithMock(utils, 'write_keyval') 445 utils.write_keyval(None, keyval_dict) 446 self.mox.ReplayAll() 447 suite = Suite.create_from_name(self._TAG, self._BUILDS, self._BOARD, 448 self.devserver, 449 afe=self.afe, tko=self.tko) 450 suite.schedule(recorder.record_entry) 451 452 453 def testRetryMapAfterScheduling(self): 454 """Test job-test and test-job mapping are correctly updated.""" 455 name_list = ['name-data_one', 'name-data_two', 'name-data_three', 456 'name-data_four', 'name-data_five', 'name-data_six', 457 'name-data_seven'] 458 keyval_dict = {constants.SCHEDULED_TEST_COUNT_KEY: 7, 459 constants.SCHEDULED_TEST_NAMES_KEY: repr(name_list)} 460 461 self.mock_control_file_parsing() 462 recorder = self.mox.CreateMock(base_job.base_job) 463 self.expect_job_scheduling(recorder) 464 self.mox.StubOutWithMock(utils, 'write_keyval') 465 utils.write_keyval(None, keyval_dict) 466 467 all_files = self.files.items() 468 # Sort tests in self.files so that they are in the same 469 # order as they are scheduled. 470 expected_retry_map = {} 471 for n in range(len(all_files)): 472 test = all_files[n][1] 473 job_id = n + 1 474 job_retries = 1 if test.job_retries is None else test.job_retries 475 if job_retries > 0: 476 expected_retry_map[job_id] = { 477 'state': RetryHandler.States.NOT_ATTEMPTED, 478 'retry_max': job_retries} 479 480 self.mox.ReplayAll() 481 suite = Suite.create_from_name(self._TAG, self._BUILDS, self._BOARD, 482 self.devserver, 483 afe=self.afe, tko=self.tko, 484 job_retry=True) 485 suite.schedule(recorder.record_entry) 486 487 self.assertEqual(expected_retry_map, suite._retry_handler._retry_map) 488 489 490 def testSuiteMaxRetries(self): 491 """Test suite max retries.""" 492 name_list = ['name-data_one', 'name-data_two', 'name-data_three', 493 'name-data_four', 'name-data_five', 494 'name-data_six', 'name-data_seven'] 495 keyval_dict = {constants.SCHEDULED_TEST_COUNT_KEY: 7, 496 constants.SCHEDULED_TEST_NAMES_KEY: repr(name_list)} 497 498 self.mock_control_file_parsing() 499 recorder = self.mox.CreateMock(base_job.base_job) 500 self.expect_job_scheduling(recorder) 501 self.mox.StubOutWithMock(utils, 'write_keyval') 502 utils.write_keyval(None, keyval_dict) 503 self.mox.ReplayAll() 504 suite = Suite.create_from_name(self._TAG, self._BUILDS, self._BOARD, 505 self.devserver, 506 afe=self.afe, tko=self.tko, 507 job_retry=True, max_retries=1) 508 suite.schedule(recorder.record_entry) 509 self.assertEqual(suite._retry_handler._max_retries, 1) 510 # Find the job_id of the test that allows retry 511 job_id = suite._retry_handler._retry_map.iterkeys().next() 512 suite._retry_handler.add_retry(old_job_id=job_id, new_job_id=10) 513 self.assertEqual(suite._retry_handler._max_retries, 0) 514 515 516 def testSuiteDependencies(self): 517 """Should add suite dependencies to tests scheduled.""" 518 name_list = ['name-data_one', 'name-data_two', 'name-data_three', 519 'name-data_four', 'name-data_five', 'name-data_six', 520 'name-data_seven'] 521 keyval_dict = {constants.SCHEDULED_TEST_COUNT_KEY: len(name_list), 522 constants.SCHEDULED_TEST_NAMES_KEY: repr(name_list)} 523 524 self.mock_control_file_parsing() 525 recorder = self.mox.CreateMock(base_job.base_job) 526 self.expect_job_scheduling(recorder, suite_deps=['extra']) 527 self.mox.StubOutWithMock(utils, 'write_keyval') 528 utils.write_keyval(None, keyval_dict) 529 530 self.mox.ReplayAll() 531 suite = Suite.create_from_name(self._TAG, self._BUILDS, self._BOARD, 532 self.devserver, extra_deps=['extra'], 533 afe=self.afe, tko=self.tko) 534 suite.schedule(recorder.record_entry) 535 536 537 def testInheritedKeyvals(self): 538 """Tests should inherit some whitelisted job keyvals.""" 539 # Only keyvals in constants.INHERITED_KEYVALS are inherited to tests. 540 job_keyvals = { 541 constants.KEYVAL_CIDB_BUILD_ID: '111', 542 constants.KEYVAL_CIDB_BUILD_STAGE_ID: '222', 543 'your': 'name', 544 } 545 test_keyvals = { 546 constants.KEYVAL_CIDB_BUILD_ID: '111', 547 constants.KEYVAL_CIDB_BUILD_STAGE_ID: '222', 548 } 549 550 self.mock_control_file_parsing() 551 recorder = self.mox.CreateMock(base_job.base_job) 552 self.expect_job_scheduling( 553 recorder, 554 extra_keyvals=test_keyvals) 555 self.mox.StubOutWithMock(utils, 'write_keyval') 556 utils.write_keyval(None, job_keyvals) 557 utils.write_keyval(None, mox.IgnoreArg()) 558 559 self.mox.ReplayAll() 560 suite = Suite.create_from_name(self._TAG, self._BUILDS, self._BOARD, 561 self.devserver, 562 afe=self.afe, tko=self.tko, 563 job_keyvals=job_keyvals) 564 suite.schedule(recorder.record_entry) 565 566 567 def _createSuiteWithMockedTestsAndControlFiles(self, file_bugs=False): 568 """Create a Suite, using mocked tests and control file contents. 569 570 @return Suite object, after mocking out behavior needed to create it. 571 """ 572 self.result_reporter = _MemoryResultReporter() 573 self.expect_control_file_parsing() 574 self.mox.ReplayAll() 575 suite = Suite.create_from_name( 576 self._TAG, 577 self._BUILDS, 578 self._BOARD, 579 self.devserver, 580 self.getter, 581 afe=self.afe, 582 tko=self.tko, 583 file_bugs=file_bugs, 584 job_retry=True, 585 result_reporter=self.result_reporter, 586 ) 587 self.mox.ResetAll() 588 return suite 589 590 591 def _createSuiteMockResults(self, results_dir=None, result_status='FAIL'): 592 """Create a suite, returned a set of mocked results to expect. 593 594 @param results_dir: A mock results directory. 595 @param result_status: A desired result status, e.g. 'FAIL', 'WARN'. 596 597 @return List of mocked results to wait on. 598 """ 599 self.suite = self._createSuiteWithMockedTestsAndControlFiles( 600 file_bugs=True) 601 self.suite._results_dir = results_dir 602 test_report = self._get_bad_test_report(result_status) 603 test_predicates = test_report.predicates 604 test_fallout = test_report.fallout 605 606 self.recorder = self.mox.CreateMock(base_job.base_job) 607 self.recorder.record_entry = self.mox.CreateMock( 608 base_job.base_job.record_entry) 609 self._mock_recorder_with_results([test_predicates], self.recorder) 610 return [test_predicates, test_fallout] 611 612 613 def _mock_recorder_with_results(self, results, recorder): 614 """ 615 Checks that results are recoded in order, eg: 616 START, (status, name, reason) END 617 618 @param results: list of results 619 @param recorder: status recorder 620 """ 621 for result in results: 622 status = result[0] 623 test_name = result[1] 624 recorder.record_entry( 625 StatusContains.CreateFromStrings('START', test_name), 626 log_in_subdir=False) 627 recorder.record_entry( 628 StatusContains.CreateFromStrings(*result), 629 log_in_subdir=False).InAnyOrder('results') 630 recorder.record_entry( 631 StatusContains.CreateFromStrings('END %s' % status, test_name), 632 log_in_subdir=False) 633 634 635 def schedule_and_expect_these_results(self, suite, results, recorder): 636 """Create mox stubs for call to suite.schedule and 637 job_status.wait_for_results 638 639 @param suite: suite object for which to stub out schedule(...) 640 @param results: results object to be returned from 641 job_stats_wait_for_results(...) 642 @param recorder: mocked recorder object to replay status messages 643 """ 644 def result_generator(results): 645 """A simple generator which generates results as Status objects. 646 647 This generator handles 'send' by simply ignoring it. 648 649 @param results: results object to be returned from 650 job_stats_wait_for_results(...) 651 @yield: job_status.Status objects. 652 """ 653 results = map(lambda r: job_status.Status(*r), results) 654 for r in results: 655 new_input = (yield r) 656 if new_input: 657 yield None 658 659 self.mox.StubOutWithMock(suite, 'schedule') 660 suite.schedule(recorder.record_entry) 661 suite._retry_handler = RetryHandler({}) 662 663 waiter_patch = mock.patch.object( 664 job_status.JobResultWaiter, 'wait_for_results', autospec=True) 665 waiter_mock = waiter_patch.start() 666 waiter_mock.return_value = result_generator(results) 667 self.addCleanup(waiter_patch.stop) 668 669 670 def testRunAndWaitSuccess(self): 671 """Should record successful results.""" 672 suite = self._createSuiteWithMockedTestsAndControlFiles() 673 674 recorder = self.mox.CreateMock(base_job.base_job) 675 676 results = [('GOOD', 'good'), ('FAIL', 'bad', 'reason')] 677 self._mock_recorder_with_results(results, recorder) 678 self.schedule_and_expect_these_results(suite, results, recorder) 679 self.mox.ReplayAll() 680 681 suite.schedule(recorder.record_entry) 682 suite.wait(recorder.record_entry) 683 684 685 def testRunAndWaitFailure(self): 686 """Should record failure to gather results.""" 687 suite = self._createSuiteWithMockedTestsAndControlFiles() 688 689 recorder = self.mox.CreateMock(base_job.base_job) 690 recorder.record_entry( 691 StatusContains.CreateFromStrings('FAIL', self._TAG, 'waiting'), 692 log_in_subdir=False) 693 694 self.mox.StubOutWithMock(suite, 'schedule') 695 suite.schedule(recorder.record_entry) 696 self.mox.ReplayAll() 697 698 with mock.patch.object( 699 job_status.JobResultWaiter, 'wait_for_results', 700 autospec=True) as wait_mock: 701 wait_mock.side_effect = Exception 702 suite.schedule(recorder.record_entry) 703 suite.wait(recorder.record_entry) 704 705 706 def testRunAndWaitScheduleFailure(self): 707 """Should record failure to schedule jobs.""" 708 suite = self._createSuiteWithMockedTestsAndControlFiles() 709 710 recorder = self.mox.CreateMock(base_job.base_job) 711 recorder.record_entry( 712 StatusContains.CreateFromStrings('INFO', 'Start %s' % self._TAG), 713 log_in_subdir=False) 714 715 recorder.record_entry( 716 StatusContains.CreateFromStrings('FAIL', self._TAG, 'scheduling'), 717 log_in_subdir=False) 718 719 self.mox.StubOutWithMock(suite._job_creator, 'create_job') 720 suite._job_creator.create_job( 721 mox.IgnoreArg(), retry_for=mox.IgnoreArg()).AndRaise( 722 Exception('Expected during test.')) 723 self.mox.ReplayAll() 724 725 suite.schedule(recorder.record_entry) 726 suite.wait(recorder.record_entry) 727 728 729 def testGetTestsSortedByTime(self): 730 """Should find all tests and sorted by TIME setting.""" 731 self.expect_control_file_parsing() 732 self.mox.ReplayAll() 733 # Get all tests. 734 tests = SuiteBase.find_and_parse_tests(self.getter, 735 lambda d: True, 736 self._TAG) 737 self.assertEquals(len(tests), 7) 738 times = [control_data.ControlData.get_test_time_index(test.time) 739 for test in tests] 740 self.assertTrue(all(x>=y for x, y in zip(times, times[1:])), 741 'Tests are not ordered correctly.') 742 743 744 def _get_bad_test_report(self, result_status='FAIL'): 745 """ 746 Fetch the predicates of a failing test, and the parameters 747 that are a fallout of this test failing. 748 """ 749 predicates = collections.namedtuple('predicates', 750 'status, testname, reason') 751 fallout = collections.namedtuple('fallout', 752 ('time_start, time_end, job_id,' 753 'username, hostname')) 754 test_report = collections.namedtuple('test_report', 755 'predicates, fallout') 756 return test_report(predicates(result_status, 'bad_test', 757 'dreadful_reason'), 758 fallout('2014-01-01 01:01:01', 'None', 759 self._FAKE_JOB_ID, 'user', 'myhost')) 760 761 762 def testJobRetryTestFail(self): 763 """Test retry works.""" 764 test_to_retry = self.files['seven'] 765 fake_new_job_id = self._FAKE_JOB_ID + 1 766 fake_job = FakeJob(id=self._FAKE_JOB_ID) 767 fake_new_job = FakeJob(id=fake_new_job_id) 768 769 test_results = self._createSuiteMockResults() 770 self.schedule_and_expect_these_results( 771 self.suite, 772 [test_results[0] + test_results[1]], 773 self.recorder) 774 self.mox.StubOutWithMock(self.suite._job_creator, 'create_job') 775 self.suite._job_creator.create_job( 776 test_to_retry, 777 retry_for=self._FAKE_JOB_ID).AndReturn(fake_new_job) 778 self.mox.ReplayAll() 779 self.suite.schedule(self.recorder.record_entry) 780 self.suite._retry_handler._retry_map = { 781 self._FAKE_JOB_ID: {'state': RetryHandler.States.NOT_ATTEMPTED, 782 'retry_max': 1} 783 } 784 self.suite._jobs_to_tests[self._FAKE_JOB_ID] = test_to_retry 785 self.suite.wait(self.recorder.record_entry) 786 expected_retry_map = { 787 self._FAKE_JOB_ID: {'state': RetryHandler.States.RETRIED, 788 'retry_max': 1}, 789 fake_new_job_id: {'state': RetryHandler.States.NOT_ATTEMPTED, 790 'retry_max': 0} 791 } 792 # Check retry map is correctly updated 793 self.assertEquals(self.suite._retry_handler._retry_map, 794 expected_retry_map) 795 # Check _jobs_to_tests is correctly updated 796 self.assertEquals(self.suite._jobs_to_tests[fake_new_job_id], 797 test_to_retry) 798 799 800 def testJobRetryTestWarn(self): 801 """Test that no retry is scheduled if test warns.""" 802 test_to_retry = self.files['seven'] 803 fake_job = FakeJob(id=self._FAKE_JOB_ID) 804 test_results = self._createSuiteMockResults(result_status='WARN') 805 self.schedule_and_expect_these_results( 806 self.suite, 807 [test_results[0] + test_results[1]], 808 self.recorder) 809 self.mox.ReplayAll() 810 self.suite.schedule(self.recorder.record_entry) 811 self.suite._retry_handler._retry_map = { 812 self._FAKE_JOB_ID: {'state': RetryHandler.States.NOT_ATTEMPTED, 813 'retry_max': 1} 814 } 815 self.suite._jobs_to_tests[self._FAKE_JOB_ID] = test_to_retry 816 expected_jobs_to_tests = self.suite._jobs_to_tests.copy() 817 expected_retry_map = self.suite._retry_handler._retry_map.copy() 818 self.suite.wait(self.recorder.record_entry) 819 self.assertTrue(self.result_reporter.results) 820 # Check retry map and _jobs_to_tests, ensure no retry was scheduled. 821 self.assertEquals(self.suite._retry_handler._retry_map, 822 expected_retry_map) 823 self.assertEquals(self.suite._jobs_to_tests, expected_jobs_to_tests) 824 825 826 def testFailedJobRetry(self): 827 """Make sure the suite survives even if the retry failed.""" 828 test_to_retry = self.files['seven'] 829 fake_job = FakeJob(id=self._FAKE_JOB_ID) 830 831 test_results = self._createSuiteMockResults() 832 self.schedule_and_expect_these_results( 833 self.suite, 834 [test_results[0] + test_results[1]], 835 self.recorder) 836 self.mox.StubOutWithMock(self.suite._job_creator, 'create_job') 837 self.suite._job_creator.create_job( 838 test_to_retry, retry_for=self._FAKE_JOB_ID).AndRaise( 839 error.RPCException('Expected during test')) 840 # Do not file a bug. 841 self.mox.StubOutWithMock(self.suite, '_should_report') 842 self.suite._should_report(mox.IgnoreArg()).AndReturn(False) 843 844 self.mox.ReplayAll() 845 846 self.suite.schedule(self.recorder.record_entry) 847 self.suite._retry_handler._retry_map = { 848 self._FAKE_JOB_ID: { 849 'state': RetryHandler.States.NOT_ATTEMPTED, 850 'retry_max': 1}} 851 self.suite._jobs_to_tests[self._FAKE_JOB_ID] = test_to_retry 852 self.suite.wait(self.recorder.record_entry) 853 expected_retry_map = { 854 self._FAKE_JOB_ID: { 855 'state': RetryHandler.States.ATTEMPTED, 856 'retry_max': 1}} 857 expected_jobs_to_tests = self.suite._jobs_to_tests.copy() 858 self.assertEquals(self.suite._retry_handler._retry_map, 859 expected_retry_map) 860 self.assertEquals(self.suite._jobs_to_tests, expected_jobs_to_tests) 861 862 863class _MemoryResultReporter(SuiteBase._ResultReporter): 864 """Reporter that stores results internally for testing.""" 865 def __init__(self): 866 self.results = [] 867 868 def report(self, result): 869 """Reports the result by storing it internally.""" 870 self.results.append(result) 871 872 873if __name__ == '__main__': 874 unittest.main() 875