1#!/usr/bin/python2 2# 3# Copyright (c) 2012 The Chromium OS Authors. All rights reserved. 4# Use of this source code is governed by a BSD-style license that can be 5# found in the LICENSE file. 6 7 8"""Unit tests for server/cros/dynamic_suite/dynamic_suite.py.""" 9 10import collections 11from collections import OrderedDict 12import os 13import shutil 14import tempfile 15import unittest 16 17import mock 18import mox 19 20import common 21 22from autotest_lib.client.common_lib import base_job 23from autotest_lib.client.common_lib import control_data 24from autotest_lib.client.common_lib import error 25from autotest_lib.client.common_lib import priorities 26from autotest_lib.client.common_lib import utils 27from autotest_lib.client.common_lib.cros import dev_server 28from autotest_lib.server import frontend 29from autotest_lib.server.cros import provision 30from autotest_lib.server.cros.dynamic_suite import control_file_getter 31from autotest_lib.server.cros.dynamic_suite import constants 32from autotest_lib.server.cros.dynamic_suite import job_status 33from autotest_lib.server.cros.dynamic_suite import suite as SuiteBase 34from autotest_lib.server.cros.dynamic_suite import suite_common 35from autotest_lib.server.cros.dynamic_suite.comparators import StatusContains 36from autotest_lib.server.cros.dynamic_suite.fakes import FakeControlData 37from autotest_lib.server.cros.dynamic_suite.fakes import FakeJob 38from autotest_lib.server.cros.dynamic_suite.fakes import FakeMultiprocessingPool 39from autotest_lib.server.cros.dynamic_suite.suite import RetryHandler 40from autotest_lib.server.cros.dynamic_suite.suite import Suite 41 42class SuiteTest(mox.MoxTestBase): 43 """Unit tests for dynamic_suite Suite class. 44 45 @var _BUILDS: fake build 46 @var _TAG: fake suite tag 47 """ 48 49 _BOARD = 'board:board' 50 _BUILDS = {provision.CROS_VERSION_PREFIX:'build_1', 51 provision.FW_RW_VERSION_PREFIX:'fwrw_build_1'} 52 _TAG = 'au' 53 _ATTR = {'attr:attr'} 54 _DEVSERVER_HOST = 'http://dontcare:8080' 55 _FAKE_JOB_ID = 10 56 57 58 def setUp(self): 59 """Setup.""" 60 super(SuiteTest, self).setUp() 61 self.maxDiff = None 62 self.use_batch = suite_common.ENABLE_CONTROLS_IN_BATCH 63 suite_common.ENABLE_CONTROLS_IN_BATCH = False 64 self.afe = self.mox.CreateMock(frontend.AFE) 65 self.tko = self.mox.CreateMock(frontend.TKO) 66 67 self.tmpdir = tempfile.mkdtemp(suffix=type(self).__name__) 68 69 self.getter = self.mox.CreateMock(control_file_getter.ControlFileGetter) 70 self.devserver = dev_server.ImageServer(self._DEVSERVER_HOST) 71 72 self.files = OrderedDict( 73 [('one', FakeControlData(self._TAG, self._ATTR, 'data_one', 74 'FAST', job_retries=None)), 75 ('two', FakeControlData(self._TAG, self._ATTR, 'data_two', 76 'SHORT', dependencies=['feta'])), 77 ('three', FakeControlData(self._TAG, self._ATTR, 'data_three', 78 'MEDIUM')), 79 ('four', FakeControlData('other', self._ATTR, 'data_four', 80 'LONG', dependencies=['arugula'])), 81 ('five', FakeControlData(self._TAG, {'other'}, 'data_five', 82 'LONG', dependencies=['arugula', 83 'caligula'])), 84 ('six', FakeControlData(self._TAG, self._ATTR, 'data_six', 85 'LENGTHY')), 86 ('seven', FakeControlData(self._TAG, self._ATTR, 'data_seven', 87 'FAST', job_retries=1))]) 88 89 self.files_to_filter = { 90 'with/deps/...': FakeControlData(self._TAG, self._ATTR, 91 'gets filtered'), 92 'with/profilers/...': FakeControlData(self._TAG, self._ATTR, 93 'gets filtered')} 94 95 96 def tearDown(self): 97 """Teardown.""" 98 suite_common.ENABLE_CONTROLS_IN_BATCH = self.use_batch 99 super(SuiteTest, self).tearDown() 100 shutil.rmtree(self.tmpdir, ignore_errors=True) 101 102 103 def expect_control_file_parsing(self, suite_name=_TAG): 104 """Expect an attempt to parse the 'control files' in |self.files|. 105 106 @param suite_name: The suite name to parse control files for. 107 """ 108 all_files = self.files.keys() + self.files_to_filter.keys() 109 self._set_control_file_parsing_expectations(False, all_files, 110 self.files, suite_name) 111 112 113 def _set_control_file_parsing_expectations(self, already_stubbed, 114 file_list, files_to_parse, 115 suite_name): 116 """Expect an attempt to parse the 'control files' in |files|. 117 118 @param already_stubbed: parse_control_string already stubbed out. 119 @param file_list: the files the dev server returns 120 @param files_to_parse: the {'name': FakeControlData} dict of files we 121 expect to get parsed. 122 """ 123 if not already_stubbed: 124 self.mox.StubOutWithMock(control_data, 'parse_control_string') 125 126 self.mox.StubOutWithMock(suite_common.multiprocessing, 'Pool') 127 suite_common.multiprocessing.Pool( 128 processes=suite_common.get_process_limit()).AndReturn( 129 FakeMultiprocessingPool()) 130 131 self.getter.get_control_file_list( 132 suite_name=suite_name).AndReturn(file_list) 133 for file, data in files_to_parse.iteritems(): 134 self.getter.get_control_file_contents( 135 file).InAnyOrder().AndReturn(data.string) 136 control_data.parse_control_string( 137 data.string, 138 raise_warnings=True, 139 path=file).InAnyOrder().AndReturn(data) 140 141 142 def expect_control_file_parsing_in_batch(self, suite_name=_TAG): 143 """Expect an attempt to parse the contents of all control files in 144 |self.files| and |self.files_to_filter|, form them to a dict. 145 146 @param suite_name: The suite name to parse control files for. 147 """ 148 self.getter = self.mox.CreateMock(control_file_getter.DevServerGetter) 149 self.mox.StubOutWithMock(control_data, 'parse_control_string') 150 151 self.mox.StubOutWithMock(suite_common.multiprocessing, 'Pool') 152 suite_common.multiprocessing.Pool( 153 processes=suite_common.get_process_limit()).AndReturn( 154 FakeMultiprocessingPool()) 155 156 suite_info = {} 157 for k, v in self.files.iteritems(): 158 suite_info[k] = v.string 159 control_data.parse_control_string( 160 v.string, 161 raise_warnings=True, 162 path=k).InAnyOrder().AndReturn(v) 163 for k, v in self.files_to_filter.iteritems(): 164 suite_info[k] = v.string 165 self.getter._dev_server = self._DEVSERVER_HOST 166 self.getter.get_suite_info( 167 suite_name=suite_name).AndReturn(suite_info) 168 169 170 def testFindAllTestInBatch(self): 171 """Test switch on enable_getting_controls_in_batch for function 172 find_all_test.""" 173 self.use_batch = suite_common.ENABLE_CONTROLS_IN_BATCH 174 self.expect_control_file_parsing_in_batch() 175 suite_common.ENABLE_CONTROLS_IN_BATCH = True 176 177 self.mox.ReplayAll() 178 179 predicate = lambda d: d.suite == self._TAG 180 tests = SuiteBase.find_and_parse_tests(self.getter, 181 predicate, 182 self._TAG) 183 self.assertEquals(len(tests), 6) 184 self.assertTrue(self.files['one'] in tests) 185 self.assertTrue(self.files['two'] in tests) 186 self.assertTrue(self.files['three'] in tests) 187 self.assertTrue(self.files['five'] in tests) 188 self.assertTrue(self.files['six'] in tests) 189 self.assertTrue(self.files['seven'] in tests) 190 suite_common.ENABLE_CONTROLS_IN_BATCH = self.use_batch 191 192 193 def testFindAndParseStableTests(self): 194 """Should find only tests that match a predicate.""" 195 self.expect_control_file_parsing() 196 self.mox.ReplayAll() 197 198 predicate = lambda d: d.text == self.files['two'].string 199 tests = SuiteBase.find_and_parse_tests(self.getter, 200 predicate, 201 self._TAG) 202 self.assertEquals(len(tests), 1) 203 self.assertEquals(tests[0], self.files['two']) 204 205 206 def testFindSuiteSyntaxErrors(self): 207 """Check all control files for syntax errors. 208 209 This test actually parses all control files in the autotest directory 210 for syntax errors, by using the un-forgiving parser and pretending to 211 look for all control files with the suite attribute. 212 """ 213 autodir = os.path.abspath( 214 os.path.join(os.path.dirname(__file__), '..', '..', '..')) 215 fs_getter = SuiteBase.create_fs_getter(autodir) 216 predicate = lambda t: hasattr(t, 'suite') 217 SuiteBase.find_and_parse_tests(fs_getter, predicate, 218 forgiving_parser=False) 219 220 221 def testFindAndParseTestsSuite(self): 222 """Should find all tests that match a predicate.""" 223 self.expect_control_file_parsing() 224 self.mox.ReplayAll() 225 226 predicate = lambda d: d.suite == self._TAG 227 tests = SuiteBase.find_and_parse_tests(self.getter, 228 predicate, 229 self._TAG) 230 self.assertEquals(len(tests), 6) 231 self.assertTrue(self.files['one'] in tests) 232 self.assertTrue(self.files['two'] in tests) 233 self.assertTrue(self.files['three'] in tests) 234 self.assertTrue(self.files['five'] in tests) 235 self.assertTrue(self.files['six'] in tests) 236 self.assertTrue(self.files['seven'] in tests) 237 238 239 def testFindAndParseTestsAttr(self): 240 """Should find all tests that match a predicate.""" 241 self.expect_control_file_parsing() 242 self.mox.ReplayAll() 243 244 predicate = SuiteBase.matches_attribute_expression_predicate('attr:attr') 245 tests = SuiteBase.find_and_parse_tests(self.getter, 246 predicate, 247 self._TAG) 248 self.assertEquals(len(tests), 6) 249 self.assertTrue(self.files['one'] in tests) 250 self.assertTrue(self.files['two'] in tests) 251 self.assertTrue(self.files['three'] in tests) 252 self.assertTrue(self.files['four'] in tests) 253 self.assertTrue(self.files['six'] in tests) 254 self.assertTrue(self.files['seven'] in tests) 255 256 257 def testAdHocSuiteCreation(self): 258 """Should be able to schedule an ad-hoc suite by specifying 259 a single test name.""" 260 self.expect_control_file_parsing(suite_name='ad_hoc_suite') 261 self.mox.ReplayAll() 262 predicate = SuiteBase.test_name_equals_predicate('name-data_five') 263 suite = Suite.create_from_predicates([predicate], self._BUILDS, 264 self._BOARD, devserver=None, 265 cf_getter=self.getter, 266 afe=self.afe, tko=self.tko) 267 268 self.assertFalse(self.files['one'] in suite.tests) 269 self.assertFalse(self.files['two'] in suite.tests) 270 self.assertFalse(self.files['four'] in suite.tests) 271 self.assertTrue(self.files['five'] in suite.tests) 272 273 274 def mock_control_file_parsing(self): 275 """Fake out find_and_parse_tests(), returning content from |self.files|. 276 """ 277 for test in self.files.values(): 278 test.text = test.string # mimic parsing. 279 self.mox.StubOutWithMock(SuiteBase, 'find_and_parse_tests') 280 SuiteBase.find_and_parse_tests( 281 mox.IgnoreArg(), 282 mox.IgnoreArg(), 283 mox.IgnoreArg(), 284 forgiving_parser=True, 285 run_prod_code=False, 286 test_args=None).AndReturn(self.files.values()) 287 288 289 def expect_job_scheduling(self, recorder, 290 tests_to_skip=[], ignore_deps=False, 291 raises=False, suite_deps=[], suite=None, 292 extra_keyvals={}): 293 """Expect jobs to be scheduled for 'tests' in |self.files|. 294 295 @param recorder: object with a record_entry to be used to record test 296 results. 297 @param tests_to_skip: [list, of, test, names] that we expect to skip. 298 @param ignore_deps: If true, ignore tests' dependencies. 299 @param raises: If True, expect exceptions. 300 @param suite_deps: If True, add suite level dependencies. 301 @param extra_keyvals: Extra keyvals set to tests. 302 """ 303 record_job_id = suite and suite._results_dir 304 if record_job_id: 305 self.mox.StubOutWithMock(suite, '_remember_job_keyval') 306 recorder.record_entry( 307 StatusContains.CreateFromStrings('INFO', 'Start %s' % self._TAG), 308 log_in_subdir=False) 309 tests = self.files.values() 310 n = 1 311 for test in tests: 312 if test.name in tests_to_skip: 313 continue 314 dependencies = [] 315 if not ignore_deps: 316 dependencies.extend(test.dependencies) 317 if suite_deps: 318 dependencies.extend(suite_deps) 319 dependencies.append(self._BOARD) 320 build = self._BUILDS[provision.CROS_VERSION_PREFIX] 321 keyvals = { 322 'build': build, 323 'suite': self._TAG, 324 'builds': SuiteTest._BUILDS, 325 'experimental':test.experimental, 326 } 327 keyvals.update(extra_keyvals) 328 job_mock = self.afe.create_job( 329 control_file=test.text, 330 name=mox.And(mox.StrContains(build), 331 mox.StrContains(test.name)), 332 control_type=mox.IgnoreArg(), 333 meta_hosts=[self._BOARD], 334 dependencies=dependencies, 335 keyvals=keyvals, 336 max_runtime_mins=24*60, 337 timeout_mins=1440, 338 parent_job_id=None, 339 reboot_before=mox.IgnoreArg(), 340 run_reset=mox.IgnoreArg(), 341 priority=priorities.Priority.DEFAULT, 342 synch_count=test.sync_count, 343 require_ssp=test.require_ssp 344 ) 345 if raises: 346 job_mock.AndRaise(error.NoEligibleHostException()) 347 recorder.record_entry( 348 StatusContains.CreateFromStrings('START', test.name), 349 log_in_subdir=False) 350 recorder.record_entry( 351 StatusContains.CreateFromStrings('TEST_NA', test.name), 352 log_in_subdir=False) 353 recorder.record_entry( 354 StatusContains.CreateFromStrings('END', test.name), 355 log_in_subdir=False) 356 else: 357 fake_job = FakeJob(id=n) 358 job_mock.AndReturn(fake_job) 359 if record_job_id: 360 suite._remember_job_keyval(fake_job) 361 n += 1 362 363 364 def testScheduleTestsAndRecord(self): 365 """Should schedule stable and experimental tests with the AFE.""" 366 name_list = ['name-data_one', 'name-data_two', 'name-data_three', 367 'name-data_four', 'name-data_five', 'name-data_six', 368 'name-data_seven'] 369 keyval_dict = {constants.SCHEDULED_TEST_COUNT_KEY: 7, 370 constants.SCHEDULED_TEST_NAMES_KEY: repr(name_list)} 371 372 self.mock_control_file_parsing() 373 self.mox.ReplayAll() 374 suite = Suite.create_from_name(self._TAG, self._BUILDS, self._BOARD, 375 self.devserver, 376 afe=self.afe, tko=self.tko, 377 results_dir=self.tmpdir) 378 self.mox.ResetAll() 379 recorder = self.mox.CreateMock(base_job.base_job) 380 self.expect_job_scheduling(recorder, suite=suite) 381 382 self.mox.StubOutWithMock(utils, 'write_keyval') 383 utils.write_keyval(self.tmpdir, keyval_dict) 384 self.mox.ReplayAll() 385 suite.schedule(recorder.record_entry) 386 for job in suite._jobs: 387 self.assertTrue(hasattr(job, 'test_name')) 388 389 390 def testScheduleTests(self): 391 """Should schedule tests with the AFE.""" 392 name_list = ['name-data_one', 'name-data_two', 'name-data_three', 393 'name-data_four', 'name-data_five', 'name-data_six', 394 'name-data_seven'] 395 keyval_dict = {constants.SCHEDULED_TEST_COUNT_KEY: len(name_list), 396 constants.SCHEDULED_TEST_NAMES_KEY: repr(name_list)} 397 398 self.mock_control_file_parsing() 399 recorder = self.mox.CreateMock(base_job.base_job) 400 self.expect_job_scheduling(recorder) 401 self.mox.StubOutWithMock(utils, 'write_keyval') 402 utils.write_keyval(None, keyval_dict) 403 404 self.mox.ReplayAll() 405 suite = Suite.create_from_name(self._TAG, self._BUILDS, self._BOARD, 406 self.devserver, 407 afe=self.afe, tko=self.tko) 408 suite.schedule(recorder.record_entry) 409 410 411 def testScheduleTestsIgnoreDeps(self): 412 """Test scheduling tests ignoring deps.""" 413 name_list = ['name-data_one', 'name-data_two', 'name-data_three', 414 'name-data_four', 'name-data_five', 'name-data_six', 415 'name-data_seven'] 416 keyval_dict = {constants.SCHEDULED_TEST_COUNT_KEY: len(name_list), 417 constants.SCHEDULED_TEST_NAMES_KEY: repr(name_list)} 418 419 self.mock_control_file_parsing() 420 recorder = self.mox.CreateMock(base_job.base_job) 421 self.expect_job_scheduling(recorder, ignore_deps=True) 422 self.mox.StubOutWithMock(utils, 'write_keyval') 423 utils.write_keyval(None, keyval_dict) 424 425 self.mox.ReplayAll() 426 suite = Suite.create_from_name(self._TAG, self._BUILDS, self._BOARD, 427 self.devserver, 428 afe=self.afe, tko=self.tko, 429 ignore_deps=True) 430 suite.schedule(recorder.record_entry) 431 432 433 def testScheduleUnrunnableTestsTESTNA(self): 434 """Tests which fail to schedule should be TEST_NA.""" 435 # Since all tests will be fail to schedule, the num of scheduled tests 436 # will be zero. 437 name_list = [] 438 keyval_dict = {constants.SCHEDULED_TEST_COUNT_KEY: 0, 439 constants.SCHEDULED_TEST_NAMES_KEY: repr(name_list)} 440 441 self.mock_control_file_parsing() 442 recorder = self.mox.CreateMock(base_job.base_job) 443 self.expect_job_scheduling(recorder, raises=True) 444 self.mox.StubOutWithMock(utils, 'write_keyval') 445 utils.write_keyval(None, keyval_dict) 446 self.mox.ReplayAll() 447 suite = Suite.create_from_name(self._TAG, self._BUILDS, self._BOARD, 448 self.devserver, 449 afe=self.afe, tko=self.tko) 450 suite.schedule(recorder.record_entry) 451 452 453 def testRetryMapAfterScheduling(self): 454 """Test job-test and test-job mapping are correctly updated.""" 455 name_list = ['name-data_one', 'name-data_two', 'name-data_three', 456 'name-data_four', 'name-data_five', 'name-data_six', 457 'name-data_seven'] 458 keyval_dict = {constants.SCHEDULED_TEST_COUNT_KEY: 7, 459 constants.SCHEDULED_TEST_NAMES_KEY: repr(name_list)} 460 461 self.mock_control_file_parsing() 462 recorder = self.mox.CreateMock(base_job.base_job) 463 self.expect_job_scheduling(recorder) 464 self.mox.StubOutWithMock(utils, 'write_keyval') 465 utils.write_keyval(None, keyval_dict) 466 467 all_files = self.files.items() 468 # Sort tests in self.files so that they are in the same 469 # order as they are scheduled. 470 expected_retry_map = {} 471 for n in range(len(all_files)): 472 test = all_files[n][1] 473 job_id = n + 1 474 job_retries = 1 if test.job_retries is None else test.job_retries 475 if job_retries > 0: 476 expected_retry_map[job_id] = { 477 'state': RetryHandler.States.NOT_ATTEMPTED, 478 'retry_max': job_retries} 479 480 self.mox.ReplayAll() 481 suite = Suite.create_from_name(self._TAG, self._BUILDS, self._BOARD, 482 self.devserver, 483 afe=self.afe, tko=self.tko, 484 job_retry=True) 485 suite.schedule(recorder.record_entry) 486 487 self.assertEqual(expected_retry_map, suite._retry_handler._retry_map) 488 489 490 def testSuiteMaxRetries(self): 491 """Test suite max retries.""" 492 name_list = ['name-data_one', 'name-data_two', 'name-data_three', 493 'name-data_four', 'name-data_five', 494 'name-data_six', 'name-data_seven'] 495 keyval_dict = {constants.SCHEDULED_TEST_COUNT_KEY: 7, 496 constants.SCHEDULED_TEST_NAMES_KEY: repr(name_list)} 497 498 self.mock_control_file_parsing() 499 recorder = self.mox.CreateMock(base_job.base_job) 500 self.expect_job_scheduling(recorder) 501 self.mox.StubOutWithMock(utils, 'write_keyval') 502 utils.write_keyval(None, keyval_dict) 503 self.mox.ReplayAll() 504 suite = Suite.create_from_name(self._TAG, self._BUILDS, self._BOARD, 505 self.devserver, 506 afe=self.afe, tko=self.tko, 507 job_retry=True, max_retries=1) 508 suite.schedule(recorder.record_entry) 509 self.assertEqual(suite._retry_handler._max_retries, 1) 510 # Find the job_id of the test that allows retry 511 job_id = suite._retry_handler._retry_map.iterkeys().next() 512 suite._retry_handler.add_retry(old_job_id=job_id, new_job_id=10) 513 self.assertEqual(suite._retry_handler._max_retries, 0) 514 515 516 def testSuiteDependencies(self): 517 """Should add suite dependencies to tests scheduled.""" 518 name_list = ['name-data_one', 'name-data_two', 'name-data_three', 519 'name-data_four', 'name-data_five', 'name-data_six', 520 'name-data_seven'] 521 keyval_dict = {constants.SCHEDULED_TEST_COUNT_KEY: len(name_list), 522 constants.SCHEDULED_TEST_NAMES_KEY: repr(name_list)} 523 524 self.mock_control_file_parsing() 525 recorder = self.mox.CreateMock(base_job.base_job) 526 self.expect_job_scheduling(recorder, suite_deps=['extra']) 527 self.mox.StubOutWithMock(utils, 'write_keyval') 528 utils.write_keyval(None, keyval_dict) 529 530 self.mox.ReplayAll() 531 suite = Suite.create_from_name(self._TAG, self._BUILDS, self._BOARD, 532 self.devserver, extra_deps=['extra'], 533 afe=self.afe, tko=self.tko) 534 suite.schedule(recorder.record_entry) 535 536 537 def testInheritedKeyvals(self): 538 """Tests should inherit some whitelisted job keyvals.""" 539 # Only keyvals in constants.INHERITED_KEYVALS are inherited to tests. 540 job_keyvals = { 541 constants.KEYVAL_CIDB_BUILD_ID: '111', 542 constants.KEYVAL_CIDB_BUILD_STAGE_ID: '222', 543 constants.KEYVAL_BRANCH: 'dummy_branch', 544 constants.KEYVAL_BUILDER_NAME: 'model-dummy', 545 constants.KEYVAL_MASTER_BUILDER_NAME: 'master-dummy', 546 'your': 'name', 547 } 548 test_keyvals = { 549 constants.KEYVAL_CIDB_BUILD_ID: '111', 550 constants.KEYVAL_CIDB_BUILD_STAGE_ID: '222', 551 constants.KEYVAL_BRANCH: 'dummy_branch', 552 constants.KEYVAL_BUILDER_NAME: 'model-dummy', 553 constants.KEYVAL_MASTER_BUILDER_NAME: 'master-dummy', 554 } 555 556 self.mock_control_file_parsing() 557 recorder = self.mox.CreateMock(base_job.base_job) 558 self.expect_job_scheduling( 559 recorder, 560 extra_keyvals=test_keyvals) 561 self.mox.StubOutWithMock(utils, 'write_keyval') 562 utils.write_keyval(None, job_keyvals) 563 utils.write_keyval(None, mox.IgnoreArg()) 564 565 self.mox.ReplayAll() 566 suite = Suite.create_from_name(self._TAG, self._BUILDS, self._BOARD, 567 self.devserver, 568 afe=self.afe, tko=self.tko, 569 job_keyvals=job_keyvals) 570 suite.schedule(recorder.record_entry) 571 572 573 def _createSuiteWithMockedTestsAndControlFiles(self, file_bugs=False): 574 """Create a Suite, using mocked tests and control file contents. 575 576 @return Suite object, after mocking out behavior needed to create it. 577 """ 578 self.result_reporter = _MemoryResultReporter() 579 self.expect_control_file_parsing() 580 self.mox.ReplayAll() 581 suite = Suite.create_from_name( 582 self._TAG, 583 self._BUILDS, 584 self._BOARD, 585 self.devserver, 586 self.getter, 587 afe=self.afe, 588 tko=self.tko, 589 file_bugs=file_bugs, 590 job_retry=True, 591 result_reporter=self.result_reporter, 592 ) 593 self.mox.ResetAll() 594 return suite 595 596 597 def _createSuiteMockResults(self, results_dir=None, result_status='FAIL'): 598 """Create a suite, returned a set of mocked results to expect. 599 600 @param results_dir: A mock results directory. 601 @param result_status: A desired result status, e.g. 'FAIL', 'WARN'. 602 603 @return List of mocked results to wait on. 604 """ 605 self.suite = self._createSuiteWithMockedTestsAndControlFiles( 606 file_bugs=True) 607 self.suite._results_dir = results_dir 608 test_report = self._get_bad_test_report(result_status) 609 test_predicates = test_report.predicates 610 test_fallout = test_report.fallout 611 612 self.recorder = self.mox.CreateMock(base_job.base_job) 613 self.recorder.record_entry = self.mox.CreateMock( 614 base_job.base_job.record_entry) 615 self._mock_recorder_with_results([test_predicates], self.recorder) 616 return [test_predicates, test_fallout] 617 618 619 def _mock_recorder_with_results(self, results, recorder): 620 """ 621 Checks that results are recoded in order, eg: 622 START, (status, name, reason) END 623 624 @param results: list of results 625 @param recorder: status recorder 626 """ 627 for result in results: 628 status = result[0] 629 test_name = result[1] 630 recorder.record_entry( 631 StatusContains.CreateFromStrings('START', test_name), 632 log_in_subdir=False) 633 recorder.record_entry( 634 StatusContains.CreateFromStrings(*result), 635 log_in_subdir=False).InAnyOrder('results') 636 recorder.record_entry( 637 StatusContains.CreateFromStrings('END %s' % status, test_name), 638 log_in_subdir=False) 639 640 641 def schedule_and_expect_these_results(self, suite, results, recorder): 642 """Create mox stubs for call to suite.schedule and 643 job_status.wait_for_results 644 645 @param suite: suite object for which to stub out schedule(...) 646 @param results: results object to be returned from 647 job_stats_wait_for_results(...) 648 @param recorder: mocked recorder object to replay status messages 649 """ 650 def result_generator(results): 651 """A simple generator which generates results as Status objects. 652 653 This generator handles 'send' by simply ignoring it. 654 655 @param results: results object to be returned from 656 job_stats_wait_for_results(...) 657 @yield: job_status.Status objects. 658 """ 659 results = map(lambda r: job_status.Status(*r), results) 660 for r in results: 661 new_input = (yield r) 662 if new_input: 663 yield None 664 665 self.mox.StubOutWithMock(suite, 'schedule') 666 suite.schedule(recorder.record_entry) 667 suite._retry_handler = RetryHandler({}) 668 669 waiter_patch = mock.patch.object( 670 job_status.JobResultWaiter, 'wait_for_results', autospec=True) 671 waiter_mock = waiter_patch.start() 672 waiter_mock.return_value = result_generator(results) 673 self.addCleanup(waiter_patch.stop) 674 675 676 def testRunAndWaitSuccess(self): 677 """Should record successful results.""" 678 suite = self._createSuiteWithMockedTestsAndControlFiles() 679 680 recorder = self.mox.CreateMock(base_job.base_job) 681 682 results = [('GOOD', 'good'), ('FAIL', 'bad', 'reason')] 683 self._mock_recorder_with_results(results, recorder) 684 self.schedule_and_expect_these_results(suite, results, recorder) 685 self.mox.ReplayAll() 686 687 suite.schedule(recorder.record_entry) 688 suite.wait(recorder.record_entry) 689 690 691 def testRunAndWaitFailure(self): 692 """Should record failure to gather results.""" 693 suite = self._createSuiteWithMockedTestsAndControlFiles() 694 695 recorder = self.mox.CreateMock(base_job.base_job) 696 recorder.record_entry( 697 StatusContains.CreateFromStrings('FAIL', self._TAG, 'waiting'), 698 log_in_subdir=False) 699 700 self.mox.StubOutWithMock(suite, 'schedule') 701 suite.schedule(recorder.record_entry) 702 self.mox.ReplayAll() 703 704 with mock.patch.object( 705 job_status.JobResultWaiter, 'wait_for_results', 706 autospec=True) as wait_mock: 707 wait_mock.side_effect = Exception 708 suite.schedule(recorder.record_entry) 709 suite.wait(recorder.record_entry) 710 711 712 def testRunAndWaitScheduleFailure(self): 713 """Should record failure to schedule jobs.""" 714 suite = self._createSuiteWithMockedTestsAndControlFiles() 715 716 recorder = self.mox.CreateMock(base_job.base_job) 717 recorder.record_entry( 718 StatusContains.CreateFromStrings('INFO', 'Start %s' % self._TAG), 719 log_in_subdir=False) 720 721 recorder.record_entry( 722 StatusContains.CreateFromStrings('FAIL', self._TAG, 'scheduling'), 723 log_in_subdir=False) 724 725 self.mox.StubOutWithMock(suite._job_creator, 'create_job') 726 suite._job_creator.create_job( 727 mox.IgnoreArg(), retry_for=mox.IgnoreArg()).AndRaise( 728 Exception('Expected during test.')) 729 self.mox.ReplayAll() 730 731 suite.schedule(recorder.record_entry) 732 suite.wait(recorder.record_entry) 733 734 735 def testGetTestsSortedByTime(self): 736 """Should find all tests and sorted by TIME setting.""" 737 self.expect_control_file_parsing() 738 self.mox.ReplayAll() 739 # Get all tests. 740 tests = SuiteBase.find_and_parse_tests(self.getter, 741 lambda d: True, 742 self._TAG) 743 self.assertEquals(len(tests), 7) 744 times = [control_data.ControlData.get_test_time_index(test.time) 745 for test in tests] 746 self.assertTrue(all(x>=y for x, y in zip(times, times[1:])), 747 'Tests are not ordered correctly.') 748 749 750 def _get_bad_test_report(self, result_status='FAIL'): 751 """ 752 Fetch the predicates of a failing test, and the parameters 753 that are a fallout of this test failing. 754 """ 755 predicates = collections.namedtuple('predicates', 756 'status, testname, reason') 757 fallout = collections.namedtuple('fallout', 758 ('time_start, time_end, job_id,' 759 'username, hostname')) 760 test_report = collections.namedtuple('test_report', 761 'predicates, fallout') 762 return test_report(predicates(result_status, 'bad_test', 763 'dreadful_reason'), 764 fallout('2014-01-01 01:01:01', 'None', 765 self._FAKE_JOB_ID, 'user', 'myhost')) 766 767 768 def testJobRetryTestFail(self): 769 """Test retry works.""" 770 test_to_retry = self.files['seven'] 771 fake_new_job_id = self._FAKE_JOB_ID + 1 772 fake_job = FakeJob(id=self._FAKE_JOB_ID) 773 fake_new_job = FakeJob(id=fake_new_job_id) 774 775 test_results = self._createSuiteMockResults() 776 self.schedule_and_expect_these_results( 777 self.suite, 778 [test_results[0] + test_results[1]], 779 self.recorder) 780 self.mox.StubOutWithMock(self.suite._job_creator, 'create_job') 781 self.suite._job_creator.create_job( 782 test_to_retry, 783 retry_for=self._FAKE_JOB_ID).AndReturn(fake_new_job) 784 self.mox.ReplayAll() 785 self.suite.schedule(self.recorder.record_entry) 786 self.suite._retry_handler._retry_map = { 787 self._FAKE_JOB_ID: {'state': RetryHandler.States.NOT_ATTEMPTED, 788 'retry_max': 1} 789 } 790 self.suite._jobs_to_tests[self._FAKE_JOB_ID] = test_to_retry 791 self.suite.wait(self.recorder.record_entry) 792 expected_retry_map = { 793 self._FAKE_JOB_ID: {'state': RetryHandler.States.RETRIED, 794 'retry_max': 1}, 795 fake_new_job_id: {'state': RetryHandler.States.NOT_ATTEMPTED, 796 'retry_max': 0} 797 } 798 # Check retry map is correctly updated 799 self.assertEquals(self.suite._retry_handler._retry_map, 800 expected_retry_map) 801 # Check _jobs_to_tests is correctly updated 802 self.assertEquals(self.suite._jobs_to_tests[fake_new_job_id], 803 test_to_retry) 804 805 806 def testJobRetryTestWarn(self): 807 """Test that no retry is scheduled if test warns.""" 808 test_to_retry = self.files['seven'] 809 fake_job = FakeJob(id=self._FAKE_JOB_ID) 810 test_results = self._createSuiteMockResults(result_status='WARN') 811 self.schedule_and_expect_these_results( 812 self.suite, 813 [test_results[0] + test_results[1]], 814 self.recorder) 815 self.mox.ReplayAll() 816 self.suite.schedule(self.recorder.record_entry) 817 self.suite._retry_handler._retry_map = { 818 self._FAKE_JOB_ID: {'state': RetryHandler.States.NOT_ATTEMPTED, 819 'retry_max': 1} 820 } 821 self.suite._jobs_to_tests[self._FAKE_JOB_ID] = test_to_retry 822 expected_jobs_to_tests = self.suite._jobs_to_tests.copy() 823 expected_retry_map = self.suite._retry_handler._retry_map.copy() 824 self.suite.wait(self.recorder.record_entry) 825 self.assertTrue(self.result_reporter.results) 826 # Check retry map and _jobs_to_tests, ensure no retry was scheduled. 827 self.assertEquals(self.suite._retry_handler._retry_map, 828 expected_retry_map) 829 self.assertEquals(self.suite._jobs_to_tests, expected_jobs_to_tests) 830 831 832 def testFailedJobRetry(self): 833 """Make sure the suite survives even if the retry failed.""" 834 test_to_retry = self.files['seven'] 835 fake_job = FakeJob(id=self._FAKE_JOB_ID) 836 837 test_results = self._createSuiteMockResults() 838 self.schedule_and_expect_these_results( 839 self.suite, 840 [test_results[0] + test_results[1]], 841 self.recorder) 842 self.mox.StubOutWithMock(self.suite._job_creator, 'create_job') 843 self.suite._job_creator.create_job( 844 test_to_retry, retry_for=self._FAKE_JOB_ID).AndRaise( 845 error.RPCException('Expected during test')) 846 # Do not file a bug. 847 self.mox.StubOutWithMock(self.suite, '_should_report') 848 self.suite._should_report(mox.IgnoreArg()).AndReturn(False) 849 850 self.mox.ReplayAll() 851 852 self.suite.schedule(self.recorder.record_entry) 853 self.suite._retry_handler._retry_map = { 854 self._FAKE_JOB_ID: { 855 'state': RetryHandler.States.NOT_ATTEMPTED, 856 'retry_max': 1}} 857 self.suite._jobs_to_tests[self._FAKE_JOB_ID] = test_to_retry 858 self.suite.wait(self.recorder.record_entry) 859 expected_retry_map = { 860 self._FAKE_JOB_ID: { 861 'state': RetryHandler.States.ATTEMPTED, 862 'retry_max': 1}} 863 expected_jobs_to_tests = self.suite._jobs_to_tests.copy() 864 self.assertEquals(self.suite._retry_handler._retry_map, 865 expected_retry_map) 866 self.assertEquals(self.suite._jobs_to_tests, expected_jobs_to_tests) 867 868 869class _MemoryResultReporter(SuiteBase._ResultReporter): 870 """Reporter that stores results internally for testing.""" 871 def __init__(self): 872 self.results = [] 873 874 def report(self, result): 875 """Reports the result by storing it internally.""" 876 self.results.append(result) 877 878 879if __name__ == '__main__': 880 unittest.main() 881