1#!/usr/bin/python 2# 3# Copyright (c) 2012 The Chromium OS Authors. All rights reserved. 4# Use of this source code is governed by a BSD-style license that can be 5# found in the LICENSE file. 6 7 8"""Unit tests for server/cros/dynamic_suite/dynamic_suite.py.""" 9 10import collections 11from collections import OrderedDict 12import os 13import shutil 14import tempfile 15import unittest 16 17import mock 18import mox 19 20import common 21 22from autotest_lib.client.common_lib import base_job 23from autotest_lib.client.common_lib import control_data 24from autotest_lib.client.common_lib import error 25from autotest_lib.client.common_lib import priorities 26from autotest_lib.client.common_lib import utils 27from autotest_lib.client.common_lib.cros import dev_server 28from autotest_lib.server import frontend 29from autotest_lib.server.cros import provision 30from autotest_lib.server.cros.dynamic_suite import control_file_getter 31from autotest_lib.server.cros.dynamic_suite import constants 32from autotest_lib.server.cros.dynamic_suite import job_status 33from autotest_lib.server.cros.dynamic_suite import suite as SuiteBase 34from autotest_lib.server.cros.dynamic_suite.comparators import StatusContains 35from autotest_lib.server.cros.dynamic_suite.fakes import FakeControlData 36from autotest_lib.server.cros.dynamic_suite.fakes import FakeJob 37from autotest_lib.server.cros.dynamic_suite.suite import RetryHandler 38from autotest_lib.server.cros.dynamic_suite.suite import Suite 39 40 41class SuiteTest(mox.MoxTestBase): 42 """Unit tests for dynamic_suite Suite class. 43 44 @var _BUILDS: fake build 45 @var _TAG: fake suite tag 46 """ 47 48 _BOARD = 'board:board' 49 _BUILDS = {provision.CROS_VERSION_PREFIX:'build_1', 50 provision.FW_RW_VERSION_PREFIX:'fwrw_build_1'} 51 _TAG = 'au' 52 _ATTR = {'attr:attr'} 53 _DEVSERVER_HOST = 'http://dontcare:8080' 54 _FAKE_JOB_ID = 10 55 56 57 def setUp(self): 58 """Setup.""" 59 super(SuiteTest, self).setUp() 60 self.maxDiff = None 61 self.use_batch = SuiteBase.ENABLE_CONTROLS_IN_BATCH 62 SuiteBase.ENABLE_CONTROLS_IN_BATCH = False 63 self.afe = self.mox.CreateMock(frontend.AFE) 64 self.tko = self.mox.CreateMock(frontend.TKO) 65 66 self.tmpdir = tempfile.mkdtemp(suffix=type(self).__name__) 67 68 self.getter = self.mox.CreateMock(control_file_getter.ControlFileGetter) 69 self.devserver = dev_server.ImageServer(self._DEVSERVER_HOST) 70 71 self.files = OrderedDict( 72 [('one', FakeControlData(self._TAG, self._ATTR, 'data_one', 73 'FAST', job_retries=None)), 74 ('two', FakeControlData(self._TAG, self._ATTR, 'data_two', 75 'SHORT', dependencies=['feta'])), 76 ('three', FakeControlData(self._TAG, self._ATTR, 'data_three', 77 'MEDIUM')), 78 ('four', FakeControlData('other', self._ATTR, 'data_four', 79 'LONG', dependencies=['arugula'])), 80 ('five', FakeControlData(self._TAG, {'other'}, 'data_five', 81 'LONG', dependencies=['arugula', 82 'caligula'])), 83 ('six', FakeControlData(self._TAG, self._ATTR, 'data_six', 84 'LENGTHY')), 85 ('seven', FakeControlData(self._TAG, self._ATTR, 'data_seven', 86 'FAST', job_retries=1))]) 87 88 self.files_to_filter = { 89 'with/deps/...': FakeControlData(self._TAG, self._ATTR, 90 'gets filtered'), 91 'with/profilers/...': FakeControlData(self._TAG, self._ATTR, 92 'gets filtered')} 93 94 95 def tearDown(self): 96 """Teardown.""" 97 SuiteBase.ENABLE_CONTROLS_IN_BATCH = self.use_batch 98 super(SuiteTest, self).tearDown() 99 shutil.rmtree(self.tmpdir, ignore_errors=True) 100 101 102 def expect_control_file_parsing(self, suite_name=_TAG): 103 """Expect an attempt to parse the 'control files' in |self.files|. 104 105 @param suite_name: The suite name to parse control files for. 106 """ 107 all_files = self.files.keys() + self.files_to_filter.keys() 108 self._set_control_file_parsing_expectations(False, all_files, 109 self.files, suite_name) 110 111 112 def _set_control_file_parsing_expectations(self, already_stubbed, 113 file_list, files_to_parse, 114 suite_name): 115 """Expect an attempt to parse the 'control files' in |files|. 116 117 @param already_stubbed: parse_control_string already stubbed out. 118 @param file_list: the files the dev server returns 119 @param files_to_parse: the {'name': FakeControlData} dict of files we 120 expect to get parsed. 121 """ 122 if not already_stubbed: 123 self.mox.StubOutWithMock(control_data, 'parse_control_string') 124 125 self.getter.get_control_file_list( 126 suite_name=suite_name).AndReturn(file_list) 127 for file, data in files_to_parse.iteritems(): 128 self.getter.get_control_file_contents( 129 file).InAnyOrder().AndReturn(data.string) 130 control_data.parse_control_string( 131 data.string, 132 raise_warnings=True, 133 path=file).InAnyOrder().AndReturn(data) 134 135 136 def expect_control_file_parsing_in_batch(self, suite_name=_TAG): 137 """Expect an attempt to parse the contents of all control files in 138 |self.files| and |self.files_to_filter|, form them to a dict. 139 140 @param suite_name: The suite name to parse control files for. 141 """ 142 self.getter = self.mox.CreateMock(control_file_getter.DevServerGetter) 143 self.mox.StubOutWithMock(control_data, 'parse_control_string') 144 suite_info = {} 145 for k, v in self.files.iteritems(): 146 suite_info[k] = v.string 147 control_data.parse_control_string( 148 v.string, 149 raise_warnings=True, 150 path=k).InAnyOrder().AndReturn(v) 151 for k, v in self.files_to_filter.iteritems(): 152 suite_info[k] = v.string 153 self.getter._dev_server = self._DEVSERVER_HOST 154 self.getter.get_suite_info( 155 suite_name=suite_name).AndReturn(suite_info) 156 157 158 def testFindAllTestInBatch(self): 159 """Test switch on enable_getting_controls_in_batch for function 160 find_all_test.""" 161 self.use_batch = SuiteBase.ENABLE_CONTROLS_IN_BATCH 162 self.expect_control_file_parsing_in_batch() 163 SuiteBase.ENABLE_CONTROLS_IN_BATCH = True 164 165 self.mox.ReplayAll() 166 167 predicate = lambda d: d.suite == self._TAG 168 tests = SuiteBase.find_and_parse_tests(self.getter, 169 predicate, 170 self._TAG) 171 self.assertEquals(len(tests), 6) 172 self.assertTrue(self.files['one'] in tests) 173 self.assertTrue(self.files['two'] in tests) 174 self.assertTrue(self.files['three'] in tests) 175 self.assertTrue(self.files['five'] in tests) 176 self.assertTrue(self.files['six'] in tests) 177 self.assertTrue(self.files['seven'] in tests) 178 SuiteBase.ENABLE_CONTROLS_IN_BATCH = self.use_batch 179 180 181 def testFindAndParseStableTests(self): 182 """Should find only tests that match a predicate.""" 183 self.expect_control_file_parsing() 184 self.mox.ReplayAll() 185 186 predicate = lambda d: d.text == self.files['two'].string 187 tests = SuiteBase.find_and_parse_tests(self.getter, 188 predicate, 189 self._TAG) 190 self.assertEquals(len(tests), 1) 191 self.assertEquals(tests[0], self.files['two']) 192 193 194 def testFindSuiteSyntaxErrors(self): 195 """Check all control files for syntax errors. 196 197 This test actually parses all control files in the autotest directory 198 for syntax errors, by using the un-forgiving parser and pretending to 199 look for all control files with the suite attribute. 200 """ 201 autodir = os.path.abspath( 202 os.path.join(os.path.dirname(__file__), '..', '..', '..')) 203 fs_getter = SuiteBase.create_fs_getter(autodir) 204 predicate = lambda t: hasattr(t, 'suite') 205 SuiteBase.find_and_parse_tests(fs_getter, predicate, 206 forgiving_parser=False) 207 208 209 def testFindAndParseTestsSuite(self): 210 """Should find all tests that match a predicate.""" 211 self.expect_control_file_parsing() 212 self.mox.ReplayAll() 213 214 predicate = lambda d: d.suite == self._TAG 215 tests = SuiteBase.find_and_parse_tests(self.getter, 216 predicate, 217 self._TAG) 218 self.assertEquals(len(tests), 6) 219 self.assertTrue(self.files['one'] in tests) 220 self.assertTrue(self.files['two'] in tests) 221 self.assertTrue(self.files['three'] in tests) 222 self.assertTrue(self.files['five'] in tests) 223 self.assertTrue(self.files['six'] in tests) 224 self.assertTrue(self.files['seven'] in tests) 225 226 227 def testFindAndParseTestsAttr(self): 228 """Should find all tests that match a predicate.""" 229 self.expect_control_file_parsing() 230 self.mox.ReplayAll() 231 232 predicate = SuiteBase.matches_attribute_expression_predicate('attr:attr') 233 tests = SuiteBase.find_and_parse_tests(self.getter, 234 predicate, 235 self._TAG) 236 self.assertEquals(len(tests), 6) 237 self.assertTrue(self.files['one'] in tests) 238 self.assertTrue(self.files['two'] in tests) 239 self.assertTrue(self.files['three'] in tests) 240 self.assertTrue(self.files['four'] in tests) 241 self.assertTrue(self.files['six'] in tests) 242 self.assertTrue(self.files['seven'] in tests) 243 244 245 def testAdHocSuiteCreation(self): 246 """Should be able to schedule an ad-hoc suite by specifying 247 a single test name.""" 248 self.expect_control_file_parsing(suite_name='ad_hoc_suite') 249 self.mox.ReplayAll() 250 predicate = SuiteBase.test_name_equals_predicate('name-data_five') 251 suite = Suite.create_from_predicates([predicate], self._BUILDS, 252 self._BOARD, devserver=None, 253 cf_getter=self.getter, 254 afe=self.afe, tko=self.tko) 255 256 self.assertFalse(self.files['one'] in suite.tests) 257 self.assertFalse(self.files['two'] in suite.tests) 258 self.assertFalse(self.files['four'] in suite.tests) 259 self.assertTrue(self.files['five'] in suite.tests) 260 261 262 def mock_control_file_parsing(self): 263 """Fake out find_and_parse_tests(), returning content from |self.files|. 264 """ 265 for test in self.files.values(): 266 test.text = test.string # mimic parsing. 267 self.mox.StubOutWithMock(SuiteBase, 'find_and_parse_tests') 268 SuiteBase.find_and_parse_tests( 269 mox.IgnoreArg(), 270 mox.IgnoreArg(), 271 mox.IgnoreArg(), 272 forgiving_parser=True, 273 run_prod_code=False, 274 test_args=None).AndReturn(self.files.values()) 275 276 277 def expect_job_scheduling(self, recorder, 278 tests_to_skip=[], ignore_deps=False, 279 raises=False, suite_deps=[], suite=None, 280 extra_keyvals={}): 281 """Expect jobs to be scheduled for 'tests' in |self.files|. 282 283 @param recorder: object with a record_entry to be used to record test 284 results. 285 @param tests_to_skip: [list, of, test, names] that we expect to skip. 286 @param ignore_deps: If true, ignore tests' dependencies. 287 @param raises: If True, expect exceptions. 288 @param suite_deps: If True, add suite level dependencies. 289 @param extra_keyvals: Extra keyvals set to tests. 290 """ 291 record_job_id = suite and suite._results_dir 292 if record_job_id: 293 self.mox.StubOutWithMock(suite, '_remember_job_keyval') 294 recorder.record_entry( 295 StatusContains.CreateFromStrings('INFO', 'Start %s' % self._TAG), 296 log_in_subdir=False) 297 tests = self.files.values() 298 n = 1 299 for test in tests: 300 if test.name in tests_to_skip: 301 continue 302 dependencies = [] 303 if not ignore_deps: 304 dependencies.extend(test.dependencies) 305 if suite_deps: 306 dependencies.extend(suite_deps) 307 dependencies.append(self._BOARD) 308 build = self._BUILDS[provision.CROS_VERSION_PREFIX] 309 keyvals = { 310 'build': build, 311 'suite': self._TAG, 312 'builds': SuiteTest._BUILDS, 313 'experimental':test.experimental, 314 } 315 keyvals.update(extra_keyvals) 316 job_mock = self.afe.create_job( 317 control_file=test.text, 318 name=mox.And(mox.StrContains(build), 319 mox.StrContains(test.name)), 320 control_type=mox.IgnoreArg(), 321 meta_hosts=[self._BOARD], 322 dependencies=dependencies, 323 keyvals=keyvals, 324 max_runtime_mins=24*60, 325 timeout_mins=1440, 326 parent_job_id=None, 327 test_retry=0, 328 reboot_before=mox.IgnoreArg(), 329 run_reset=mox.IgnoreArg(), 330 priority=priorities.Priority.DEFAULT, 331 synch_count=test.sync_count, 332 require_ssp=test.require_ssp 333 ) 334 if raises: 335 job_mock.AndRaise(error.NoEligibleHostException()) 336 recorder.record_entry( 337 StatusContains.CreateFromStrings('START', test.name), 338 log_in_subdir=False) 339 recorder.record_entry( 340 StatusContains.CreateFromStrings('TEST_NA', test.name), 341 log_in_subdir=False) 342 recorder.record_entry( 343 StatusContains.CreateFromStrings('END', test.name), 344 log_in_subdir=False) 345 else: 346 fake_job = FakeJob(id=n) 347 job_mock.AndReturn(fake_job) 348 if record_job_id: 349 suite._remember_job_keyval(fake_job) 350 n += 1 351 352 353 def testScheduleTestsAndRecord(self): 354 """Should schedule stable and experimental tests with the AFE.""" 355 name_list = ['name-data_one', 'name-data_two', 'name-data_three', 356 'name-data_four', 'name-data_five', 'name-data_six', 357 'name-data_seven'] 358 keyval_dict = {constants.SCHEDULED_TEST_COUNT_KEY: 7, 359 constants.SCHEDULED_TEST_NAMES_KEY: repr(name_list)} 360 361 self.mock_control_file_parsing() 362 self.mox.ReplayAll() 363 suite = Suite.create_from_name(self._TAG, self._BUILDS, self._BOARD, 364 self.devserver, 365 afe=self.afe, tko=self.tko, 366 results_dir=self.tmpdir) 367 self.mox.ResetAll() 368 recorder = self.mox.CreateMock(base_job.base_job) 369 self.expect_job_scheduling(recorder, suite=suite) 370 371 self.mox.StubOutWithMock(utils, 'write_keyval') 372 utils.write_keyval(self.tmpdir, keyval_dict) 373 self.mox.ReplayAll() 374 suite.schedule(recorder.record_entry) 375 for job in suite._jobs: 376 self.assertTrue(hasattr(job, 'test_name')) 377 378 379 def testScheduleTests(self): 380 """Should schedule tests with the AFE.""" 381 name_list = ['name-data_one', 'name-data_two', 'name-data_three', 382 'name-data_four', 'name-data_five', 'name-data_six', 383 'name-data_seven'] 384 keyval_dict = {constants.SCHEDULED_TEST_COUNT_KEY: len(name_list), 385 constants.SCHEDULED_TEST_NAMES_KEY: repr(name_list)} 386 387 self.mock_control_file_parsing() 388 recorder = self.mox.CreateMock(base_job.base_job) 389 self.expect_job_scheduling(recorder) 390 self.mox.StubOutWithMock(utils, 'write_keyval') 391 utils.write_keyval(None, keyval_dict) 392 393 self.mox.ReplayAll() 394 suite = Suite.create_from_name(self._TAG, self._BUILDS, self._BOARD, 395 self.devserver, 396 afe=self.afe, tko=self.tko) 397 suite.schedule(recorder.record_entry) 398 399 400 def testScheduleTestsIgnoreDeps(self): 401 """Test scheduling tests ignoring deps.""" 402 name_list = ['name-data_one', 'name-data_two', 'name-data_three', 403 'name-data_four', 'name-data_five', 'name-data_six', 404 'name-data_seven'] 405 keyval_dict = {constants.SCHEDULED_TEST_COUNT_KEY: len(name_list), 406 constants.SCHEDULED_TEST_NAMES_KEY: repr(name_list)} 407 408 self.mock_control_file_parsing() 409 recorder = self.mox.CreateMock(base_job.base_job) 410 self.expect_job_scheduling(recorder, ignore_deps=True) 411 self.mox.StubOutWithMock(utils, 'write_keyval') 412 utils.write_keyval(None, keyval_dict) 413 414 self.mox.ReplayAll() 415 suite = Suite.create_from_name(self._TAG, self._BUILDS, self._BOARD, 416 self.devserver, 417 afe=self.afe, tko=self.tko, 418 ignore_deps=True) 419 suite.schedule(recorder.record_entry) 420 421 422 def testScheduleUnrunnableTestsTESTNA(self): 423 """Tests which fail to schedule should be TEST_NA.""" 424 # Since all tests will be fail to schedule, the num of scheduled tests 425 # will be zero. 426 name_list = [] 427 keyval_dict = {constants.SCHEDULED_TEST_COUNT_KEY: 0, 428 constants.SCHEDULED_TEST_NAMES_KEY: repr(name_list)} 429 430 self.mock_control_file_parsing() 431 recorder = self.mox.CreateMock(base_job.base_job) 432 self.expect_job_scheduling(recorder, raises=True) 433 self.mox.StubOutWithMock(utils, 'write_keyval') 434 utils.write_keyval(None, keyval_dict) 435 self.mox.ReplayAll() 436 suite = Suite.create_from_name(self._TAG, self._BUILDS, self._BOARD, 437 self.devserver, 438 afe=self.afe, tko=self.tko) 439 suite.schedule(recorder.record_entry) 440 441 442 def testRetryMapAfterScheduling(self): 443 """Test job-test and test-job mapping are correctly updated.""" 444 name_list = ['name-data_one', 'name-data_two', 'name-data_three', 445 'name-data_four', 'name-data_five', 'name-data_six', 446 'name-data_seven'] 447 keyval_dict = {constants.SCHEDULED_TEST_COUNT_KEY: 7, 448 constants.SCHEDULED_TEST_NAMES_KEY: repr(name_list)} 449 450 self.mock_control_file_parsing() 451 recorder = self.mox.CreateMock(base_job.base_job) 452 self.expect_job_scheduling(recorder) 453 self.mox.StubOutWithMock(utils, 'write_keyval') 454 utils.write_keyval(None, keyval_dict) 455 456 all_files = self.files.items() 457 # Sort tests in self.files so that they are in the same 458 # order as they are scheduled. 459 expected_retry_map = {} 460 for n in range(len(all_files)): 461 test = all_files[n][1] 462 job_id = n + 1 463 job_retries = 1 if test.job_retries is None else test.job_retries 464 if job_retries > 0: 465 expected_retry_map[job_id] = { 466 'state': RetryHandler.States.NOT_ATTEMPTED, 467 'retry_max': job_retries} 468 469 self.mox.ReplayAll() 470 suite = Suite.create_from_name(self._TAG, self._BUILDS, self._BOARD, 471 self.devserver, 472 afe=self.afe, tko=self.tko, 473 job_retry=True) 474 suite.schedule(recorder.record_entry) 475 476 self.assertEqual(expected_retry_map, suite._retry_handler._retry_map) 477 478 479 def testSuiteMaxRetries(self): 480 """Test suite max retries.""" 481 name_list = ['name-data_one', 'name-data_two', 'name-data_three', 482 'name-data_four', 'name-data_five', 483 'name-data_six', 'name-data_seven'] 484 keyval_dict = {constants.SCHEDULED_TEST_COUNT_KEY: 7, 485 constants.SCHEDULED_TEST_NAMES_KEY: repr(name_list)} 486 487 self.mock_control_file_parsing() 488 recorder = self.mox.CreateMock(base_job.base_job) 489 self.expect_job_scheduling(recorder) 490 self.mox.StubOutWithMock(utils, 'write_keyval') 491 utils.write_keyval(None, keyval_dict) 492 self.mox.ReplayAll() 493 suite = Suite.create_from_name(self._TAG, self._BUILDS, self._BOARD, 494 self.devserver, 495 afe=self.afe, tko=self.tko, 496 job_retry=True, max_retries=1) 497 suite.schedule(recorder.record_entry) 498 self.assertEqual(suite._retry_handler._max_retries, 1) 499 # Find the job_id of the test that allows retry 500 job_id = suite._retry_handler._retry_map.iterkeys().next() 501 suite._retry_handler.add_retry(old_job_id=job_id, new_job_id=10) 502 self.assertEqual(suite._retry_handler._max_retries, 0) 503 504 505 def testSuiteDependencies(self): 506 """Should add suite dependencies to tests scheduled.""" 507 name_list = ['name-data_one', 'name-data_two', 'name-data_three', 508 'name-data_four', 'name-data_five', 'name-data_six', 509 'name-data_seven'] 510 keyval_dict = {constants.SCHEDULED_TEST_COUNT_KEY: len(name_list), 511 constants.SCHEDULED_TEST_NAMES_KEY: repr(name_list)} 512 513 self.mock_control_file_parsing() 514 recorder = self.mox.CreateMock(base_job.base_job) 515 self.expect_job_scheduling(recorder, suite_deps=['extra']) 516 self.mox.StubOutWithMock(utils, 'write_keyval') 517 utils.write_keyval(None, keyval_dict) 518 519 self.mox.ReplayAll() 520 suite = Suite.create_from_name(self._TAG, self._BUILDS, self._BOARD, 521 self.devserver, extra_deps=['extra'], 522 afe=self.afe, tko=self.tko) 523 suite.schedule(recorder.record_entry) 524 525 526 def testInheritedKeyvals(self): 527 """Tests should inherit some whitelisted job keyvals.""" 528 # Only keyvals in constants.INHERITED_KEYVALS are inherited to tests. 529 job_keyvals = { 530 constants.KEYVAL_CIDB_BUILD_ID: '111', 531 constants.KEYVAL_CIDB_BUILD_STAGE_ID: '222', 532 'your': 'name', 533 } 534 test_keyvals = { 535 constants.KEYVAL_CIDB_BUILD_ID: '111', 536 constants.KEYVAL_CIDB_BUILD_STAGE_ID: '222', 537 } 538 539 self.mock_control_file_parsing() 540 recorder = self.mox.CreateMock(base_job.base_job) 541 self.expect_job_scheduling( 542 recorder, 543 extra_keyvals=test_keyvals) 544 self.mox.StubOutWithMock(utils, 'write_keyval') 545 utils.write_keyval(None, job_keyvals) 546 utils.write_keyval(None, mox.IgnoreArg()) 547 548 self.mox.ReplayAll() 549 suite = Suite.create_from_name(self._TAG, self._BUILDS, self._BOARD, 550 self.devserver, 551 afe=self.afe, tko=self.tko, 552 job_keyvals=job_keyvals) 553 suite.schedule(recorder.record_entry) 554 555 556 def _createSuiteWithMockedTestsAndControlFiles(self, file_bugs=False): 557 """Create a Suite, using mocked tests and control file contents. 558 559 @return Suite object, after mocking out behavior needed to create it. 560 """ 561 self.result_reporter = _MemoryResultReporter() 562 self.expect_control_file_parsing() 563 self.mox.ReplayAll() 564 suite = Suite.create_from_name( 565 self._TAG, 566 self._BUILDS, 567 self._BOARD, 568 self.devserver, 569 self.getter, 570 afe=self.afe, 571 tko=self.tko, 572 file_bugs=file_bugs, 573 job_retry=True, 574 result_reporter=self.result_reporter, 575 ) 576 self.mox.ResetAll() 577 return suite 578 579 580 def _createSuiteMockResults(self, results_dir=None, result_status='FAIL'): 581 """Create a suite, returned a set of mocked results to expect. 582 583 @param results_dir: A mock results directory. 584 @param result_status: A desired result status, e.g. 'FAIL', 'WARN'. 585 586 @return List of mocked results to wait on. 587 """ 588 self.suite = self._createSuiteWithMockedTestsAndControlFiles( 589 file_bugs=True) 590 self.suite._results_dir = results_dir 591 test_report = self._get_bad_test_report(result_status) 592 test_predicates = test_report.predicates 593 test_fallout = test_report.fallout 594 595 self.recorder = self.mox.CreateMock(base_job.base_job) 596 self.recorder.record_entry = self.mox.CreateMock( 597 base_job.base_job.record_entry) 598 self._mock_recorder_with_results([test_predicates], self.recorder) 599 return [test_predicates, test_fallout] 600 601 602 def _mock_recorder_with_results(self, results, recorder): 603 """ 604 Checks that results are recoded in order, eg: 605 START, (status, name, reason) END 606 607 @param results: list of results 608 @param recorder: status recorder 609 """ 610 for result in results: 611 status = result[0] 612 test_name = result[1] 613 recorder.record_entry( 614 StatusContains.CreateFromStrings('START', test_name), 615 log_in_subdir=False) 616 recorder.record_entry( 617 StatusContains.CreateFromStrings(*result), 618 log_in_subdir=False).InAnyOrder('results') 619 recorder.record_entry( 620 StatusContains.CreateFromStrings('END %s' % status, test_name), 621 log_in_subdir=False) 622 623 624 def schedule_and_expect_these_results(self, suite, results, recorder): 625 """Create mox stubs for call to suite.schedule and 626 job_status.wait_for_results 627 628 @param suite: suite object for which to stub out schedule(...) 629 @param results: results object to be returned from 630 job_stats_wait_for_results(...) 631 @param recorder: mocked recorder object to replay status messages 632 """ 633 def result_generator(results): 634 """A simple generator which generates results as Status objects. 635 636 This generator handles 'send' by simply ignoring it. 637 638 @param results: results object to be returned from 639 job_stats_wait_for_results(...) 640 @yield: job_status.Status objects. 641 """ 642 results = map(lambda r: job_status.Status(*r), results) 643 for r in results: 644 new_input = (yield r) 645 if new_input: 646 yield None 647 648 self.mox.StubOutWithMock(suite, 'schedule') 649 suite.schedule(recorder.record_entry) 650 suite._retry_handler = RetryHandler({}) 651 652 waiter_patch = mock.patch.object( 653 job_status.JobResultWaiter, 'wait_for_results', autospec=True) 654 waiter_mock = waiter_patch.start() 655 waiter_mock.return_value = result_generator(results) 656 self.addCleanup(waiter_patch.stop) 657 658 659 def testRunAndWaitSuccess(self): 660 """Should record successful results.""" 661 suite = self._createSuiteWithMockedTestsAndControlFiles() 662 663 recorder = self.mox.CreateMock(base_job.base_job) 664 665 results = [('GOOD', 'good'), ('FAIL', 'bad', 'reason')] 666 self._mock_recorder_with_results(results, recorder) 667 self.schedule_and_expect_these_results(suite, results, recorder) 668 self.mox.ReplayAll() 669 670 suite.schedule(recorder.record_entry) 671 suite.wait(recorder.record_entry) 672 673 674 def testRunAndWaitFailure(self): 675 """Should record failure to gather results.""" 676 suite = self._createSuiteWithMockedTestsAndControlFiles() 677 678 recorder = self.mox.CreateMock(base_job.base_job) 679 recorder.record_entry( 680 StatusContains.CreateFromStrings('FAIL', self._TAG, 'waiting'), 681 log_in_subdir=False) 682 683 self.mox.StubOutWithMock(suite, 'schedule') 684 suite.schedule(recorder.record_entry) 685 self.mox.ReplayAll() 686 687 with mock.patch.object( 688 job_status.JobResultWaiter, 'wait_for_results', 689 autospec=True) as wait_mock: 690 wait_mock.side_effect = Exception 691 suite.schedule(recorder.record_entry) 692 suite.wait(recorder.record_entry) 693 694 695 def testRunAndWaitScheduleFailure(self): 696 """Should record failure to schedule jobs.""" 697 suite = self._createSuiteWithMockedTestsAndControlFiles() 698 699 recorder = self.mox.CreateMock(base_job.base_job) 700 recorder.record_entry( 701 StatusContains.CreateFromStrings('INFO', 'Start %s' % self._TAG), 702 log_in_subdir=False) 703 704 recorder.record_entry( 705 StatusContains.CreateFromStrings('FAIL', self._TAG, 'scheduling'), 706 log_in_subdir=False) 707 708 self.mox.StubOutWithMock(suite._job_creator, 'create_job') 709 suite._job_creator.create_job( 710 mox.IgnoreArg(), retry_for=mox.IgnoreArg()).AndRaise( 711 Exception('Expected during test.')) 712 self.mox.ReplayAll() 713 714 suite.schedule(recorder.record_entry) 715 suite.wait(recorder.record_entry) 716 717 718 def testGetTestsSortedByTime(self): 719 """Should find all tests and sorted by TIME setting.""" 720 self.expect_control_file_parsing() 721 self.mox.ReplayAll() 722 # Get all tests. 723 tests = SuiteBase.find_and_parse_tests(self.getter, 724 lambda d: True, 725 self._TAG) 726 self.assertEquals(len(tests), 7) 727 times = [control_data.ControlData.get_test_time_index(test.time) 728 for test in tests] 729 self.assertTrue(all(x>=y for x, y in zip(times, times[1:])), 730 'Tests are not ordered correctly.') 731 732 733 def _get_bad_test_report(self, result_status='FAIL'): 734 """ 735 Fetch the predicates of a failing test, and the parameters 736 that are a fallout of this test failing. 737 """ 738 predicates = collections.namedtuple('predicates', 739 'status, testname, reason') 740 fallout = collections.namedtuple('fallout', 741 ('time_start, time_end, job_id,' 742 'username, hostname')) 743 test_report = collections.namedtuple('test_report', 744 'predicates, fallout') 745 return test_report(predicates(result_status, 'bad_test', 746 'dreadful_reason'), 747 fallout('2014-01-01 01:01:01', 'None', 748 self._FAKE_JOB_ID, 'user', 'myhost')) 749 750 751 def testJobRetryTestFail(self): 752 """Test retry works.""" 753 test_to_retry = self.files['seven'] 754 fake_new_job_id = self._FAKE_JOB_ID + 1 755 fake_job = FakeJob(id=self._FAKE_JOB_ID) 756 fake_new_job = FakeJob(id=fake_new_job_id) 757 758 test_results = self._createSuiteMockResults() 759 self.schedule_and_expect_these_results( 760 self.suite, 761 [test_results[0] + test_results[1]], 762 self.recorder) 763 self.mox.StubOutWithMock(self.suite._job_creator, 'create_job') 764 self.suite._job_creator.create_job( 765 test_to_retry, 766 retry_for=self._FAKE_JOB_ID).AndReturn(fake_new_job) 767 self.mox.ReplayAll() 768 self.suite.schedule(self.recorder.record_entry) 769 self.suite._retry_handler._retry_map = { 770 self._FAKE_JOB_ID: {'state': RetryHandler.States.NOT_ATTEMPTED, 771 'retry_max': 1} 772 } 773 self.suite._jobs_to_tests[self._FAKE_JOB_ID] = test_to_retry 774 self.suite.wait(self.recorder.record_entry) 775 expected_retry_map = { 776 self._FAKE_JOB_ID: {'state': RetryHandler.States.RETRIED, 777 'retry_max': 1}, 778 fake_new_job_id: {'state': RetryHandler.States.NOT_ATTEMPTED, 779 'retry_max': 0} 780 } 781 # Check retry map is correctly updated 782 self.assertEquals(self.suite._retry_handler._retry_map, 783 expected_retry_map) 784 # Check _jobs_to_tests is correctly updated 785 self.assertEquals(self.suite._jobs_to_tests[fake_new_job_id], 786 test_to_retry) 787 788 789 def testJobRetryTestWarn(self): 790 """Test that no retry is scheduled if test warns.""" 791 test_to_retry = self.files['seven'] 792 fake_job = FakeJob(id=self._FAKE_JOB_ID) 793 test_results = self._createSuiteMockResults(result_status='WARN') 794 self.schedule_and_expect_these_results( 795 self.suite, 796 [test_results[0] + test_results[1]], 797 self.recorder) 798 self.mox.ReplayAll() 799 self.suite.schedule(self.recorder.record_entry) 800 self.suite._retry_handler._retry_map = { 801 self._FAKE_JOB_ID: {'state': RetryHandler.States.NOT_ATTEMPTED, 802 'retry_max': 1} 803 } 804 self.suite._jobs_to_tests[self._FAKE_JOB_ID] = test_to_retry 805 expected_jobs_to_tests = self.suite._jobs_to_tests.copy() 806 expected_retry_map = self.suite._retry_handler._retry_map.copy() 807 self.suite.wait(self.recorder.record_entry) 808 self.assertTrue(self.result_reporter.results) 809 # Check retry map and _jobs_to_tests, ensure no retry was scheduled. 810 self.assertEquals(self.suite._retry_handler._retry_map, 811 expected_retry_map) 812 self.assertEquals(self.suite._jobs_to_tests, expected_jobs_to_tests) 813 814 815 def testFailedJobRetry(self): 816 """Make sure the suite survives even if the retry failed.""" 817 test_to_retry = self.files['seven'] 818 fake_job = FakeJob(id=self._FAKE_JOB_ID) 819 820 test_results = self._createSuiteMockResults() 821 self.schedule_and_expect_these_results( 822 self.suite, 823 [test_results[0] + test_results[1]], 824 self.recorder) 825 self.mox.StubOutWithMock(self.suite._job_creator, 'create_job') 826 self.suite._job_creator.create_job( 827 test_to_retry, retry_for=self._FAKE_JOB_ID).AndRaise( 828 error.RPCException('Expected during test')) 829 # Do not file a bug. 830 self.mox.StubOutWithMock(self.suite, '_should_report') 831 self.suite._should_report(mox.IgnoreArg()).AndReturn(False) 832 833 self.mox.ReplayAll() 834 835 self.suite.schedule(self.recorder.record_entry) 836 self.suite._retry_handler._retry_map = { 837 self._FAKE_JOB_ID: { 838 'state': RetryHandler.States.NOT_ATTEMPTED, 839 'retry_max': 1}} 840 self.suite._jobs_to_tests[self._FAKE_JOB_ID] = test_to_retry 841 self.suite.wait(self.recorder.record_entry) 842 expected_retry_map = { 843 self._FAKE_JOB_ID: { 844 'state': RetryHandler.States.ATTEMPTED, 845 'retry_max': 1}} 846 expected_jobs_to_tests = self.suite._jobs_to_tests.copy() 847 self.assertEquals(self.suite._retry_handler._retry_map, 848 expected_retry_map) 849 self.assertEquals(self.suite._jobs_to_tests, expected_jobs_to_tests) 850 851 852class _MemoryResultReporter(SuiteBase._ResultReporter): 853 """Reporter that stores results internally for testing.""" 854 def __init__(self): 855 self.results = [] 856 857 def report(self, result): 858 """Reports the result by storing it internally.""" 859 self.results.append(result) 860 861 862if __name__ == '__main__': 863 unittest.main() 864