1#!/usr/bin/python2 2# 3# Copyright (c) 2012 The Chromium OS Authors. All rights reserved. 4# Use of this source code is governed by a BSD-style license that can be 5# found in the LICENSE file. 6 7 8"""Unit tests for server/cros/dynamic_suite/dynamic_suite.py.""" 9 10from __future__ import absolute_import 11from __future__ import division 12from __future__ import print_function 13 14import collections 15from collections import OrderedDict 16import mock 17import mox 18import os 19import six 20from six.moves import range 21from six.moves import zip 22import shutil 23import tempfile 24import unittest 25 26import common 27 28from autotest_lib.client.common_lib import base_job 29from autotest_lib.client.common_lib import control_data 30from autotest_lib.client.common_lib import error 31from autotest_lib.client.common_lib import priorities 32from autotest_lib.client.common_lib import utils 33from autotest_lib.client.common_lib.cros import dev_server 34from autotest_lib.server import frontend 35from autotest_lib.server.cros import provision 36from autotest_lib.server.cros.dynamic_suite import control_file_getter 37from autotest_lib.server.cros.dynamic_suite import constants 38from autotest_lib.server.cros.dynamic_suite import job_status 39from autotest_lib.server.cros.dynamic_suite import suite as SuiteBase 40from autotest_lib.server.cros.dynamic_suite import suite_common 41from autotest_lib.server.cros.dynamic_suite.comparators import StatusContains 42from autotest_lib.server.cros.dynamic_suite.fakes import FakeControlData 43from autotest_lib.server.cros.dynamic_suite.fakes import FakeJob 44from autotest_lib.server.cros.dynamic_suite.fakes import FakeMultiprocessingPool 45from autotest_lib.server.cros.dynamic_suite.suite import RetryHandler 46from autotest_lib.server.cros.dynamic_suite.suite import Suite 47 48 49class SuiteTest(mox.MoxTestBase): 50 """Unit tests for dynamic_suite Suite class. 51 52 @var _BUILDS: fake build 53 @var _TAG: fake suite tag 54 """ 55 56 _BOARD = 'board:board' 57 _BUILDS = {provision.CROS_VERSION_PREFIX:'build_1', 58 provision.FW_RW_VERSION_PREFIX:'fwrw_build_1'} 59 _TAG = 'au' 60 _ATTR = {'attr:attr'} 61 _DEVSERVER_HOST = 'http://dontcare:8080' 62 _FAKE_JOB_ID = 10 63 64 65 def setUp(self): 66 """Setup.""" 67 super(SuiteTest, self).setUp() 68 self.maxDiff = None 69 self.use_batch = suite_common.ENABLE_CONTROLS_IN_BATCH 70 suite_common.ENABLE_CONTROLS_IN_BATCH = False 71 self.afe = self.mox.CreateMock(frontend.AFE) 72 self.tko = self.mox.CreateMock(frontend.TKO) 73 74 self.tmpdir = tempfile.mkdtemp(suffix=type(self).__name__) 75 76 self.getter = self.mox.CreateMock(control_file_getter.ControlFileGetter) 77 self.devserver = dev_server.ImageServer(self._DEVSERVER_HOST) 78 79 self.files = OrderedDict( 80 [('one', FakeControlData(self._TAG, self._ATTR, 'data_one', 81 'FAST', job_retries=None)), 82 ('two', FakeControlData(self._TAG, self._ATTR, 'data_two', 83 'SHORT', dependencies=['feta'])), 84 ('three', FakeControlData(self._TAG, self._ATTR, 'data_three', 85 'MEDIUM')), 86 ('four', FakeControlData('other', self._ATTR, 'data_four', 87 'LONG', dependencies=['arugula'])), 88 ('five', FakeControlData(self._TAG, {'other'}, 'data_five', 89 'LONG', dependencies=['arugula', 90 'caligula'])), 91 ('six', FakeControlData(self._TAG, self._ATTR, 'data_six', 92 'LENGTHY')), 93 ('seven', FakeControlData(self._TAG, self._ATTR, 'data_seven', 94 'FAST', job_retries=1))]) 95 96 self.files_to_filter = { 97 'with/deps/...': FakeControlData(self._TAG, self._ATTR, 98 'gets filtered'), 99 'with/profilers/...': FakeControlData(self._TAG, self._ATTR, 100 'gets filtered')} 101 102 103 def tearDown(self): 104 """Teardown.""" 105 suite_common.ENABLE_CONTROLS_IN_BATCH = self.use_batch 106 super(SuiteTest, self).tearDown() 107 shutil.rmtree(self.tmpdir, ignore_errors=True) 108 109 110 def expect_control_file_parsing(self, suite_name=_TAG): 111 """Expect an attempt to parse the 'control files' in |self.files|. 112 113 @param suite_name: The suite name to parse control files for. 114 """ 115 all_files = list(self.files.keys()) + list(self.files_to_filter.keys()) 116 self._set_control_file_parsing_expectations(False, all_files, 117 self.files, suite_name) 118 119 120 def _set_control_file_parsing_expectations(self, already_stubbed, 121 file_list, files_to_parse, 122 suite_name): 123 """Expect an attempt to parse the 'control files' in |files|. 124 125 @param already_stubbed: parse_control_string already stubbed out. 126 @param file_list: the files the dev server returns 127 @param files_to_parse: the {'name': FakeControlData} dict of files we 128 expect to get parsed. 129 """ 130 if not already_stubbed: 131 self.mox.StubOutWithMock(control_data, 'parse_control_string') 132 133 self.mox.StubOutWithMock(suite_common.multiprocessing, 'Pool') 134 suite_common.multiprocessing.Pool( 135 processes=suite_common.get_process_limit()).AndReturn( 136 FakeMultiprocessingPool()) 137 138 self.getter.get_control_file_list( 139 suite_name=suite_name).AndReturn(file_list) 140 for file, data in six.iteritems(files_to_parse): 141 self.getter.get_control_file_contents( 142 file).InAnyOrder().AndReturn(data.string) 143 control_data.parse_control_string( 144 data.string, 145 raise_warnings=True, 146 path=file).InAnyOrder().AndReturn(data) 147 148 149 def expect_control_file_parsing_in_batch(self, suite_name=_TAG): 150 """Expect an attempt to parse the contents of all control files in 151 |self.files| and |self.files_to_filter|, form them to a dict. 152 153 @param suite_name: The suite name to parse control files for. 154 """ 155 self.getter = self.mox.CreateMock(control_file_getter.DevServerGetter) 156 self.mox.StubOutWithMock(control_data, 'parse_control_string') 157 158 self.mox.StubOutWithMock(suite_common.multiprocessing, 'Pool') 159 suite_common.multiprocessing.Pool( 160 processes=suite_common.get_process_limit()).AndReturn( 161 FakeMultiprocessingPool()) 162 163 suite_info = {} 164 for k, v in six.iteritems(self.files): 165 suite_info[k] = v.string 166 control_data.parse_control_string( 167 v.string, 168 raise_warnings=True, 169 path=k).InAnyOrder().AndReturn(v) 170 for k, v in six.iteritems(self.files_to_filter): 171 suite_info[k] = v.string 172 self.getter._dev_server = self._DEVSERVER_HOST 173 self.getter.get_suite_info( 174 suite_name=suite_name).AndReturn(suite_info) 175 176 177 def testFindAllTestInBatch(self): 178 """Test switch on enable_getting_controls_in_batch for function 179 find_all_test.""" 180 self.use_batch = suite_common.ENABLE_CONTROLS_IN_BATCH 181 self.expect_control_file_parsing_in_batch() 182 suite_common.ENABLE_CONTROLS_IN_BATCH = True 183 184 self.mox.ReplayAll() 185 186 predicate = lambda d: d.suite == self._TAG 187 tests = SuiteBase.find_and_parse_tests(self.getter, 188 predicate, 189 self._TAG) 190 self.assertEquals(len(tests), 6) 191 self.assertTrue(self.files['one'] in tests) 192 self.assertTrue(self.files['two'] in tests) 193 self.assertTrue(self.files['three'] in tests) 194 self.assertTrue(self.files['five'] in tests) 195 self.assertTrue(self.files['six'] in tests) 196 self.assertTrue(self.files['seven'] in tests) 197 suite_common.ENABLE_CONTROLS_IN_BATCH = self.use_batch 198 199 200 def testFindAndParseStableTests(self): 201 """Should find only tests that match a predicate.""" 202 self.expect_control_file_parsing() 203 self.mox.ReplayAll() 204 205 predicate = lambda d: d.text == self.files['two'].string 206 tests = SuiteBase.find_and_parse_tests(self.getter, 207 predicate, 208 self._TAG) 209 self.assertEquals(len(tests), 1) 210 self.assertEquals(tests[0], self.files['two']) 211 212 213 def testFindSuiteSyntaxErrors(self): 214 """Check all control files for syntax errors. 215 216 This test actually parses all control files in the autotest directory 217 for syntax errors, by using the un-forgiving parser and pretending to 218 look for all control files with the suite attribute. 219 """ 220 autodir = os.path.abspath( 221 os.path.join(os.path.dirname(__file__), '..', '..', '..')) 222 fs_getter = SuiteBase.create_fs_getter(autodir) 223 predicate = lambda t: hasattr(t, 'suite') 224 SuiteBase.find_and_parse_tests(fs_getter, predicate, 225 forgiving_parser=False) 226 227 228 def testFindAndParseTestsSuite(self): 229 """Should find all tests that match a predicate.""" 230 self.expect_control_file_parsing() 231 self.mox.ReplayAll() 232 233 predicate = lambda d: d.suite == self._TAG 234 tests = SuiteBase.find_and_parse_tests(self.getter, 235 predicate, 236 self._TAG) 237 self.assertEquals(len(tests), 6) 238 self.assertTrue(self.files['one'] in tests) 239 self.assertTrue(self.files['two'] in tests) 240 self.assertTrue(self.files['three'] in tests) 241 self.assertTrue(self.files['five'] in tests) 242 self.assertTrue(self.files['six'] in tests) 243 self.assertTrue(self.files['seven'] in tests) 244 245 246 def testFindAndParseTestsAttr(self): 247 """Should find all tests that match a predicate.""" 248 self.expect_control_file_parsing() 249 self.mox.ReplayAll() 250 251 predicate = SuiteBase.matches_attribute_expression_predicate('attr:attr') 252 tests = SuiteBase.find_and_parse_tests(self.getter, 253 predicate, 254 self._TAG) 255 self.assertEquals(len(tests), 6) 256 self.assertTrue(self.files['one'] in tests) 257 self.assertTrue(self.files['two'] in tests) 258 self.assertTrue(self.files['three'] in tests) 259 self.assertTrue(self.files['four'] in tests) 260 self.assertTrue(self.files['six'] in tests) 261 self.assertTrue(self.files['seven'] in tests) 262 263 264 def testAdHocSuiteCreation(self): 265 """Should be able to schedule an ad-hoc suite by specifying 266 a single test name.""" 267 self.expect_control_file_parsing(suite_name='ad_hoc_suite') 268 self.mox.ReplayAll() 269 predicate = SuiteBase.test_name_equals_predicate('name-data_five') 270 suite = Suite.create_from_predicates([predicate], self._BUILDS, 271 self._BOARD, devserver=None, 272 cf_getter=self.getter, 273 afe=self.afe, tko=self.tko) 274 275 self.assertFalse(self.files['one'] in suite.tests) 276 self.assertFalse(self.files['two'] in suite.tests) 277 self.assertFalse(self.files['four'] in suite.tests) 278 self.assertTrue(self.files['five'] in suite.tests) 279 280 281 def mock_control_file_parsing(self): 282 """Fake out find_and_parse_tests(), returning content from |self.files|. 283 """ 284 for test in self.files.values(): 285 test.text = test.string # mimic parsing. 286 self.mox.StubOutWithMock(SuiteBase, 'find_and_parse_tests') 287 SuiteBase.find_and_parse_tests( 288 mox.IgnoreArg(), 289 mox.IgnoreArg(), 290 mox.IgnoreArg(), 291 forgiving_parser=True, 292 run_prod_code=False, 293 test_args=None).AndReturn(list(self.files.values())) 294 295 296 def expect_job_scheduling(self, recorder, 297 tests_to_skip=[], ignore_deps=False, 298 raises=False, suite_deps=[], suite=None, 299 extra_keyvals={}): 300 """Expect jobs to be scheduled for 'tests' in |self.files|. 301 302 @param recorder: object with a record_entry to be used to record test 303 results. 304 @param tests_to_skip: [list, of, test, names] that we expect to skip. 305 @param ignore_deps: If true, ignore tests' dependencies. 306 @param raises: If True, expect exceptions. 307 @param suite_deps: If True, add suite level dependencies. 308 @param extra_keyvals: Extra keyvals set to tests. 309 """ 310 record_job_id = suite and suite._results_dir 311 if record_job_id: 312 self.mox.StubOutWithMock(suite, '_remember_job_keyval') 313 recorder.record_entry( 314 StatusContains.CreateFromStrings('INFO', 'Start %s' % self._TAG), 315 log_in_subdir=False) 316 tests = list(self.files.values()) 317 n = 1 318 for test in tests: 319 if test.name in tests_to_skip: 320 continue 321 dependencies = [] 322 if not ignore_deps: 323 dependencies.extend(test.dependencies) 324 if suite_deps: 325 dependencies.extend(suite_deps) 326 dependencies.append(self._BOARD) 327 build = self._BUILDS[provision.CROS_VERSION_PREFIX] 328 keyvals = { 329 'build': build, 330 'suite': self._TAG, 331 'builds': SuiteTest._BUILDS, 332 'experimental':test.experimental, 333 } 334 keyvals.update(extra_keyvals) 335 job_mock = self.afe.create_job( 336 control_file=test.text, 337 name=mox.And(mox.StrContains(build), 338 mox.StrContains(test.name)), 339 control_type=mox.IgnoreArg(), 340 meta_hosts=[self._BOARD], 341 dependencies=dependencies, 342 keyvals=keyvals, 343 max_runtime_mins=24*60, 344 timeout_mins=1440, 345 parent_job_id=None, 346 reboot_before=mox.IgnoreArg(), 347 run_reset=mox.IgnoreArg(), 348 priority=priorities.Priority.DEFAULT, 349 synch_count=test.sync_count, 350 require_ssp=test.require_ssp 351 ) 352 if raises: 353 job_mock.AndRaise(error.NoEligibleHostException()) 354 recorder.record_entry( 355 StatusContains.CreateFromStrings('START', test.name), 356 log_in_subdir=False) 357 recorder.record_entry( 358 StatusContains.CreateFromStrings('TEST_NA', test.name), 359 log_in_subdir=False) 360 recorder.record_entry( 361 StatusContains.CreateFromStrings('END', test.name), 362 log_in_subdir=False) 363 else: 364 fake_job = FakeJob(id=n) 365 job_mock.AndReturn(fake_job) 366 if record_job_id: 367 suite._remember_job_keyval(fake_job) 368 n += 1 369 370 371 def testScheduleTestsAndRecord(self): 372 """Should schedule stable and experimental tests with the AFE.""" 373 name_list = ['name-data_one', 'name-data_two', 'name-data_three', 374 'name-data_four', 'name-data_five', 'name-data_six', 375 'name-data_seven'] 376 keyval_dict = {constants.SCHEDULED_TEST_COUNT_KEY: 7, 377 constants.SCHEDULED_TEST_NAMES_KEY: repr(name_list)} 378 379 self.mock_control_file_parsing() 380 self.mox.ReplayAll() 381 suite = Suite.create_from_name(self._TAG, self._BUILDS, self._BOARD, 382 self.devserver, 383 afe=self.afe, tko=self.tko, 384 results_dir=self.tmpdir) 385 self.mox.ResetAll() 386 recorder = self.mox.CreateMock(base_job.base_job) 387 self.expect_job_scheduling(recorder, suite=suite) 388 389 self.mox.StubOutWithMock(utils, 'write_keyval') 390 utils.write_keyval(self.tmpdir, keyval_dict) 391 self.mox.ReplayAll() 392 suite.schedule(recorder.record_entry) 393 for job in suite._jobs: 394 self.assertTrue(hasattr(job, 'test_name')) 395 396 397 def testScheduleTests(self): 398 """Should schedule tests with the AFE.""" 399 name_list = ['name-data_one', 'name-data_two', 'name-data_three', 400 'name-data_four', 'name-data_five', 'name-data_six', 401 'name-data_seven'] 402 keyval_dict = {constants.SCHEDULED_TEST_COUNT_KEY: len(name_list), 403 constants.SCHEDULED_TEST_NAMES_KEY: repr(name_list)} 404 405 self.mock_control_file_parsing() 406 recorder = self.mox.CreateMock(base_job.base_job) 407 self.expect_job_scheduling(recorder) 408 self.mox.StubOutWithMock(utils, 'write_keyval') 409 utils.write_keyval(None, keyval_dict) 410 411 self.mox.ReplayAll() 412 suite = Suite.create_from_name(self._TAG, self._BUILDS, self._BOARD, 413 self.devserver, 414 afe=self.afe, tko=self.tko) 415 suite.schedule(recorder.record_entry) 416 417 418 def testScheduleTestsIgnoreDeps(self): 419 """Test scheduling tests ignoring deps.""" 420 name_list = ['name-data_one', 'name-data_two', 'name-data_three', 421 'name-data_four', 'name-data_five', 'name-data_six', 422 'name-data_seven'] 423 keyval_dict = {constants.SCHEDULED_TEST_COUNT_KEY: len(name_list), 424 constants.SCHEDULED_TEST_NAMES_KEY: repr(name_list)} 425 426 self.mock_control_file_parsing() 427 recorder = self.mox.CreateMock(base_job.base_job) 428 self.expect_job_scheduling(recorder, ignore_deps=True) 429 self.mox.StubOutWithMock(utils, 'write_keyval') 430 utils.write_keyval(None, keyval_dict) 431 432 self.mox.ReplayAll() 433 suite = Suite.create_from_name(self._TAG, self._BUILDS, self._BOARD, 434 self.devserver, 435 afe=self.afe, tko=self.tko, 436 ignore_deps=True) 437 suite.schedule(recorder.record_entry) 438 439 440 def testScheduleUnrunnableTestsTESTNA(self): 441 """Tests which fail to schedule should be TEST_NA.""" 442 # Since all tests will be fail to schedule, the num of scheduled tests 443 # will be zero. 444 name_list = [] 445 keyval_dict = {constants.SCHEDULED_TEST_COUNT_KEY: 0, 446 constants.SCHEDULED_TEST_NAMES_KEY: repr(name_list)} 447 448 self.mock_control_file_parsing() 449 recorder = self.mox.CreateMock(base_job.base_job) 450 self.expect_job_scheduling(recorder, raises=True) 451 self.mox.StubOutWithMock(utils, 'write_keyval') 452 utils.write_keyval(None, keyval_dict) 453 self.mox.ReplayAll() 454 suite = Suite.create_from_name(self._TAG, self._BUILDS, self._BOARD, 455 self.devserver, 456 afe=self.afe, tko=self.tko) 457 suite.schedule(recorder.record_entry) 458 459 460 def testRetryMapAfterScheduling(self): 461 """Test job-test and test-job mapping are correctly updated.""" 462 name_list = ['name-data_one', 'name-data_two', 'name-data_three', 463 'name-data_four', 'name-data_five', 'name-data_six', 464 'name-data_seven'] 465 keyval_dict = {constants.SCHEDULED_TEST_COUNT_KEY: 7, 466 constants.SCHEDULED_TEST_NAMES_KEY: repr(name_list)} 467 468 self.mock_control_file_parsing() 469 recorder = self.mox.CreateMock(base_job.base_job) 470 self.expect_job_scheduling(recorder) 471 self.mox.StubOutWithMock(utils, 'write_keyval') 472 utils.write_keyval(None, keyval_dict) 473 474 all_files = list(self.files.items()) 475 # Sort tests in self.files so that they are in the same 476 # order as they are scheduled. 477 expected_retry_map = {} 478 for n in range(len(all_files)): 479 test = all_files[n][1] 480 job_id = n + 1 481 job_retries = 1 if test.job_retries is None else test.job_retries 482 if job_retries > 0: 483 expected_retry_map[job_id] = { 484 'state': RetryHandler.States.NOT_ATTEMPTED, 485 'retry_max': job_retries} 486 487 self.mox.ReplayAll() 488 suite = Suite.create_from_name(self._TAG, self._BUILDS, self._BOARD, 489 self.devserver, 490 afe=self.afe, tko=self.tko, 491 job_retry=True) 492 suite.schedule(recorder.record_entry) 493 494 self.assertEqual(expected_retry_map, suite._retry_handler._retry_map) 495 496 497 def testSuiteMaxRetries(self): 498 """Test suite max retries.""" 499 name_list = ['name-data_one', 'name-data_two', 'name-data_three', 500 'name-data_four', 'name-data_five', 501 'name-data_six', 'name-data_seven'] 502 keyval_dict = {constants.SCHEDULED_TEST_COUNT_KEY: 7, 503 constants.SCHEDULED_TEST_NAMES_KEY: repr(name_list)} 504 505 self.mock_control_file_parsing() 506 recorder = self.mox.CreateMock(base_job.base_job) 507 self.expect_job_scheduling(recorder) 508 self.mox.StubOutWithMock(utils, 'write_keyval') 509 utils.write_keyval(None, keyval_dict) 510 self.mox.ReplayAll() 511 suite = Suite.create_from_name(self._TAG, self._BUILDS, self._BOARD, 512 self.devserver, 513 afe=self.afe, tko=self.tko, 514 job_retry=True, max_retries=1) 515 suite.schedule(recorder.record_entry) 516 self.assertEqual(suite._retry_handler._max_retries, 1) 517 # Find the job_id of the test that allows retry 518 job_id = next(six.iterkeys(suite._retry_handler._retry_map)) 519 suite._retry_handler.add_retry(old_job_id=job_id, new_job_id=10) 520 self.assertEqual(suite._retry_handler._max_retries, 0) 521 522 523 def testSuiteDependencies(self): 524 """Should add suite dependencies to tests scheduled.""" 525 name_list = ['name-data_one', 'name-data_two', 'name-data_three', 526 'name-data_four', 'name-data_five', 'name-data_six', 527 'name-data_seven'] 528 keyval_dict = {constants.SCHEDULED_TEST_COUNT_KEY: len(name_list), 529 constants.SCHEDULED_TEST_NAMES_KEY: repr(name_list)} 530 531 self.mock_control_file_parsing() 532 recorder = self.mox.CreateMock(base_job.base_job) 533 self.expect_job_scheduling(recorder, suite_deps=['extra']) 534 self.mox.StubOutWithMock(utils, 'write_keyval') 535 utils.write_keyval(None, keyval_dict) 536 537 self.mox.ReplayAll() 538 suite = Suite.create_from_name(self._TAG, self._BUILDS, self._BOARD, 539 self.devserver, extra_deps=['extra'], 540 afe=self.afe, tko=self.tko) 541 suite.schedule(recorder.record_entry) 542 543 544 def testInheritedKeyvals(self): 545 """Tests should inherit some allowlisted job keyvals.""" 546 # Only keyvals in constants.INHERITED_KEYVALS are inherited to tests. 547 job_keyvals = { 548 constants.KEYVAL_CIDB_BUILD_ID: '111', 549 constants.KEYVAL_CIDB_BUILD_STAGE_ID: '222', 550 constants.KEYVAL_BRANCH: 'dummy_branch', 551 constants.KEYVAL_BUILDER_NAME: 'model-dummy', 552 constants.KEYVAL_MASTER_BUILDER_NAME: 'master-dummy', 553 'your': 'name', 554 } 555 test_keyvals = { 556 constants.KEYVAL_CIDB_BUILD_ID: '111', 557 constants.KEYVAL_CIDB_BUILD_STAGE_ID: '222', 558 constants.KEYVAL_BRANCH: 'dummy_branch', 559 constants.KEYVAL_BUILDER_NAME: 'model-dummy', 560 constants.KEYVAL_MASTER_BUILDER_NAME: 'master-dummy', 561 } 562 563 self.mock_control_file_parsing() 564 recorder = self.mox.CreateMock(base_job.base_job) 565 self.expect_job_scheduling( 566 recorder, 567 extra_keyvals=test_keyvals) 568 self.mox.StubOutWithMock(utils, 'write_keyval') 569 utils.write_keyval(None, job_keyvals) 570 utils.write_keyval(None, mox.IgnoreArg()) 571 572 self.mox.ReplayAll() 573 suite = Suite.create_from_name(self._TAG, self._BUILDS, self._BOARD, 574 self.devserver, 575 afe=self.afe, tko=self.tko, 576 job_keyvals=job_keyvals) 577 suite.schedule(recorder.record_entry) 578 579 580 def _createSuiteWithMockedTestsAndControlFiles(self, file_bugs=False): 581 """Create a Suite, using mocked tests and control file contents. 582 583 @return Suite object, after mocking out behavior needed to create it. 584 """ 585 self.result_reporter = _MemoryResultReporter() 586 self.expect_control_file_parsing() 587 self.mox.ReplayAll() 588 suite = Suite.create_from_name( 589 self._TAG, 590 self._BUILDS, 591 self._BOARD, 592 self.devserver, 593 self.getter, 594 afe=self.afe, 595 tko=self.tko, 596 file_bugs=file_bugs, 597 job_retry=True, 598 result_reporter=self.result_reporter, 599 ) 600 self.mox.ResetAll() 601 return suite 602 603 604 def _createSuiteMockResults(self, results_dir=None, result_status='FAIL'): 605 """Create a suite, returned a set of mocked results to expect. 606 607 @param results_dir: A mock results directory. 608 @param result_status: A desired result status, e.g. 'FAIL', 'WARN'. 609 610 @return List of mocked results to wait on. 611 """ 612 self.suite = self._createSuiteWithMockedTestsAndControlFiles( 613 file_bugs=True) 614 self.suite._results_dir = results_dir 615 test_report = self._get_bad_test_report(result_status) 616 test_predicates = test_report.predicates 617 test_fallout = test_report.fallout 618 619 self.recorder = self.mox.CreateMock(base_job.base_job) 620 self.recorder.record_entry = self.mox.CreateMock( 621 base_job.base_job.record_entry) 622 self._mock_recorder_with_results([test_predicates], self.recorder) 623 return [test_predicates, test_fallout] 624 625 626 def _mock_recorder_with_results(self, results, recorder): 627 """ 628 Checks that results are recoded in order, eg: 629 START, (status, name, reason) END 630 631 @param results: list of results 632 @param recorder: status recorder 633 """ 634 for result in results: 635 status = result[0] 636 test_name = result[1] 637 recorder.record_entry( 638 StatusContains.CreateFromStrings('START', test_name), 639 log_in_subdir=False) 640 recorder.record_entry( 641 StatusContains.CreateFromStrings(*result), 642 log_in_subdir=False).InAnyOrder('results') 643 recorder.record_entry( 644 StatusContains.CreateFromStrings('END %s' % status, test_name), 645 log_in_subdir=False) 646 647 648 def schedule_and_expect_these_results(self, suite, results, recorder): 649 """Create mox stubs for call to suite.schedule and 650 job_status.wait_for_results 651 652 @param suite: suite object for which to stub out schedule(...) 653 @param results: results object to be returned from 654 job_stats_wait_for_results(...) 655 @param recorder: mocked recorder object to replay status messages 656 """ 657 def result_generator(results): 658 """A simple generator which generates results as Status objects. 659 660 This generator handles 'send' by simply ignoring it. 661 662 @param results: results object to be returned from 663 job_stats_wait_for_results(...) 664 @yield: job_status.Status objects. 665 """ 666 results = [job_status.Status(*r) for r in results] 667 for r in results: 668 new_input = (yield r) 669 if new_input: 670 yield None 671 672 self.mox.StubOutWithMock(suite, 'schedule') 673 suite.schedule(recorder.record_entry) 674 suite._retry_handler = RetryHandler({}) 675 676 waiter_patch = mock.patch.object( 677 job_status.JobResultWaiter, 'wait_for_results', autospec=True) 678 waiter_mock = waiter_patch.start() 679 waiter_mock.return_value = result_generator(results) 680 self.addCleanup(waiter_patch.stop) 681 682 683 def testRunAndWaitSuccess(self): 684 """Should record successful results.""" 685 suite = self._createSuiteWithMockedTestsAndControlFiles() 686 687 recorder = self.mox.CreateMock(base_job.base_job) 688 689 results = [('GOOD', 'good'), ('FAIL', 'bad', 'reason')] 690 self._mock_recorder_with_results(results, recorder) 691 self.schedule_and_expect_these_results(suite, results, recorder) 692 self.mox.ReplayAll() 693 694 suite.schedule(recorder.record_entry) 695 suite.wait(recorder.record_entry) 696 697 698 def testRunAndWaitFailure(self): 699 """Should record failure to gather results.""" 700 suite = self._createSuiteWithMockedTestsAndControlFiles() 701 702 recorder = self.mox.CreateMock(base_job.base_job) 703 recorder.record_entry( 704 StatusContains.CreateFromStrings('FAIL', self._TAG, 'waiting'), 705 log_in_subdir=False) 706 707 self.mox.StubOutWithMock(suite, 'schedule') 708 suite.schedule(recorder.record_entry) 709 self.mox.ReplayAll() 710 711 with mock.patch.object( 712 job_status.JobResultWaiter, 'wait_for_results', 713 autospec=True) as wait_mock: 714 wait_mock.side_effect = Exception 715 suite.schedule(recorder.record_entry) 716 suite.wait(recorder.record_entry) 717 718 719 def testRunAndWaitScheduleFailure(self): 720 """Should record failure to schedule jobs.""" 721 suite = self._createSuiteWithMockedTestsAndControlFiles() 722 723 recorder = self.mox.CreateMock(base_job.base_job) 724 recorder.record_entry( 725 StatusContains.CreateFromStrings('INFO', 'Start %s' % self._TAG), 726 log_in_subdir=False) 727 728 recorder.record_entry( 729 StatusContains.CreateFromStrings('FAIL', self._TAG, 'scheduling'), 730 log_in_subdir=False) 731 732 self.mox.StubOutWithMock(suite._job_creator, 'create_job') 733 suite._job_creator.create_job( 734 mox.IgnoreArg(), retry_for=mox.IgnoreArg()).AndRaise( 735 Exception('Expected during test.')) 736 self.mox.ReplayAll() 737 738 suite.schedule(recorder.record_entry) 739 suite.wait(recorder.record_entry) 740 741 742 def testGetTestsSortedByTime(self): 743 """Should find all tests and sorted by TIME setting.""" 744 self.expect_control_file_parsing() 745 self.mox.ReplayAll() 746 # Get all tests. 747 tests = SuiteBase.find_and_parse_tests(self.getter, 748 lambda d: True, 749 self._TAG) 750 self.assertEquals(len(tests), 7) 751 times = [control_data.ControlData.get_test_time_index(test.time) 752 for test in tests] 753 self.assertTrue(all(x>=y for x, y in zip(times, times[1:])), 754 'Tests are not ordered correctly.') 755 756 757 def _get_bad_test_report(self, result_status='FAIL'): 758 """ 759 Fetch the predicates of a failing test, and the parameters 760 that are a fallout of this test failing. 761 """ 762 predicates = collections.namedtuple('predicates', 763 'status, testname, reason') 764 fallout = collections.namedtuple('fallout', 765 ('time_start, time_end, job_id,' 766 'username, hostname')) 767 test_report = collections.namedtuple('test_report', 768 'predicates, fallout') 769 return test_report(predicates(result_status, 'bad_test', 770 'dreadful_reason'), 771 fallout('2014-01-01 01:01:01', 'None', 772 self._FAKE_JOB_ID, 'user', 'myhost')) 773 774 775 def testJobRetryTestFail(self): 776 """Test retry works.""" 777 test_to_retry = self.files['seven'] 778 fake_new_job_id = self._FAKE_JOB_ID + 1 779 fake_job = FakeJob(id=self._FAKE_JOB_ID) 780 fake_new_job = FakeJob(id=fake_new_job_id) 781 782 test_results = self._createSuiteMockResults() 783 self.schedule_and_expect_these_results( 784 self.suite, 785 [test_results[0] + test_results[1]], 786 self.recorder) 787 self.mox.StubOutWithMock(self.suite._job_creator, 'create_job') 788 self.suite._job_creator.create_job( 789 test_to_retry, 790 retry_for=self._FAKE_JOB_ID).AndReturn(fake_new_job) 791 self.mox.ReplayAll() 792 self.suite.schedule(self.recorder.record_entry) 793 self.suite._retry_handler._retry_map = { 794 self._FAKE_JOB_ID: {'state': RetryHandler.States.NOT_ATTEMPTED, 795 'retry_max': 1} 796 } 797 self.suite._jobs_to_tests[self._FAKE_JOB_ID] = test_to_retry 798 self.suite.wait(self.recorder.record_entry) 799 expected_retry_map = { 800 self._FAKE_JOB_ID: {'state': RetryHandler.States.RETRIED, 801 'retry_max': 1}, 802 fake_new_job_id: {'state': RetryHandler.States.NOT_ATTEMPTED, 803 'retry_max': 0} 804 } 805 # Check retry map is correctly updated 806 self.assertEquals(self.suite._retry_handler._retry_map, 807 expected_retry_map) 808 # Check _jobs_to_tests is correctly updated 809 self.assertEquals(self.suite._jobs_to_tests[fake_new_job_id], 810 test_to_retry) 811 812 813 def testJobRetryTestWarn(self): 814 """Test that no retry is scheduled if test warns.""" 815 test_to_retry = self.files['seven'] 816 fake_job = FakeJob(id=self._FAKE_JOB_ID) 817 test_results = self._createSuiteMockResults(result_status='WARN') 818 self.schedule_and_expect_these_results( 819 self.suite, 820 [test_results[0] + test_results[1]], 821 self.recorder) 822 self.mox.ReplayAll() 823 self.suite.schedule(self.recorder.record_entry) 824 self.suite._retry_handler._retry_map = { 825 self._FAKE_JOB_ID: {'state': RetryHandler.States.NOT_ATTEMPTED, 826 'retry_max': 1} 827 } 828 self.suite._jobs_to_tests[self._FAKE_JOB_ID] = test_to_retry 829 expected_jobs_to_tests = self.suite._jobs_to_tests.copy() 830 expected_retry_map = self.suite._retry_handler._retry_map.copy() 831 self.suite.wait(self.recorder.record_entry) 832 self.assertTrue(self.result_reporter.results) 833 # Check retry map and _jobs_to_tests, ensure no retry was scheduled. 834 self.assertEquals(self.suite._retry_handler._retry_map, 835 expected_retry_map) 836 self.assertEquals(self.suite._jobs_to_tests, expected_jobs_to_tests) 837 838 839 def testFailedJobRetry(self): 840 """Make sure the suite survives even if the retry failed.""" 841 test_to_retry = self.files['seven'] 842 fake_job = FakeJob(id=self._FAKE_JOB_ID) 843 844 test_results = self._createSuiteMockResults() 845 self.schedule_and_expect_these_results( 846 self.suite, 847 [test_results[0] + test_results[1]], 848 self.recorder) 849 self.mox.StubOutWithMock(self.suite._job_creator, 'create_job') 850 self.suite._job_creator.create_job( 851 test_to_retry, retry_for=self._FAKE_JOB_ID).AndRaise( 852 error.RPCException('Expected during test')) 853 # Do not file a bug. 854 self.mox.StubOutWithMock(self.suite, '_should_report') 855 self.suite._should_report(mox.IgnoreArg()).AndReturn(False) 856 857 self.mox.ReplayAll() 858 859 self.suite.schedule(self.recorder.record_entry) 860 self.suite._retry_handler._retry_map = { 861 self._FAKE_JOB_ID: { 862 'state': RetryHandler.States.NOT_ATTEMPTED, 863 'retry_max': 1}} 864 self.suite._jobs_to_tests[self._FAKE_JOB_ID] = test_to_retry 865 self.suite.wait(self.recorder.record_entry) 866 expected_retry_map = { 867 self._FAKE_JOB_ID: { 868 'state': RetryHandler.States.ATTEMPTED, 869 'retry_max': 1}} 870 expected_jobs_to_tests = self.suite._jobs_to_tests.copy() 871 self.assertEquals(self.suite._retry_handler._retry_map, 872 expected_retry_map) 873 self.assertEquals(self.suite._jobs_to_tests, expected_jobs_to_tests) 874 875 876class _MemoryResultReporter(SuiteBase._ResultReporter): 877 """Reporter that stores results internally for testing.""" 878 def __init__(self): 879 self.results = [] 880 881 def report(self, result): 882 """Reports the result by storing it internally.""" 883 self.results.append(result) 884 885 886if __name__ == '__main__': 887 unittest.main() 888