1#!/usr/bin/python 2# 3# Copyright (c) 2012 The Chromium OS Authors. All rights reserved. 4# Use of this source code is governed by a BSD-style license that can be 5# found in the LICENSE file. 6 7 8"""Unit tests for server/cros/dynamic_suite/dynamic_suite.py.""" 9 10import collections 11from collections import OrderedDict 12import os 13import shutil 14import tempfile 15import unittest 16 17import mox 18 19import common 20 21from autotest_lib.client.common_lib import base_job 22from autotest_lib.client.common_lib import control_data 23from autotest_lib.client.common_lib import error 24from autotest_lib.client.common_lib import priorities 25from autotest_lib.client.common_lib import utils 26from autotest_lib.client.common_lib.cros import dev_server 27from autotest_lib.server import frontend 28from autotest_lib.server.cros import provision 29from autotest_lib.server.cros.dynamic_suite import control_file_getter 30from autotest_lib.server.cros.dynamic_suite import constants 31from autotest_lib.server.cros.dynamic_suite import job_status 32from autotest_lib.server.cros.dynamic_suite import reporting 33from autotest_lib.server.cros.dynamic_suite import suite as SuiteBase 34from autotest_lib.server.cros.dynamic_suite.comparators import StatusContains 35from autotest_lib.server.cros.dynamic_suite.fakes import FakeControlData 36from autotest_lib.server.cros.dynamic_suite.fakes import FakeJob 37from autotest_lib.server.cros.dynamic_suite.suite import RetryHandler 38from autotest_lib.server.cros.dynamic_suite.suite import Suite 39from autotest_lib.site_utils import phapi_lib 40 41 42class SuiteTest(mox.MoxTestBase): 43 """Unit tests for dynamic_suite Suite class. 44 45 @var _BUILDS: fake build 46 @var _TAG: fake suite tag 47 """ 48 49 _BOARD = 'board:board' 50 _BUILDS = {provision.CROS_VERSION_PREFIX:'build_1', 51 provision.FW_RW_VERSION_PREFIX:'fwrw_build_1'} 52 _TAG = 'au' 53 _ATTR = {'attr:attr'} 54 _DEVSERVER_HOST = 'http://dontcare:8080' 55 _FAKE_JOB_ID = 10 56 57 58 def setUp(self): 59 super(SuiteTest, self).setUp() 60 self.maxDiff = None 61 self.use_batch = SuiteBase.ENABLE_CONTROLS_IN_BATCH 62 SuiteBase.ENABLE_CONTROLS_IN_BATCH = False 63 self.afe = self.mox.CreateMock(frontend.AFE) 64 self.tko = self.mox.CreateMock(frontend.TKO) 65 66 self.tmpdir = tempfile.mkdtemp(suffix=type(self).__name__) 67 68 self.getter = self.mox.CreateMock(control_file_getter.ControlFileGetter) 69 self.devserver = dev_server.ImageServer(self._DEVSERVER_HOST) 70 71 self.files = OrderedDict( 72 [('one', FakeControlData(self._TAG, self._ATTR, 'data_one', 73 'FAST', expr=True)), 74 ('two', FakeControlData(self._TAG, self._ATTR, 'data_two', 75 'SHORT', dependencies=['feta'])), 76 ('three', FakeControlData(self._TAG, self._ATTR, 'data_three', 77 'MEDIUM')), 78 ('four', FakeControlData('other', self._ATTR, 'data_four', 79 'LONG', dependencies=['arugula'])), 80 ('five', FakeControlData(self._TAG, {'other'}, 'data_five', 81 'LONG', dependencies=['arugula', 82 'caligula'])), 83 ('six', FakeControlData(self._TAG, self._ATTR, 'data_six', 84 'LENGTHY')), 85 ('seven', FakeControlData(self._TAG, self._ATTR, 'data_seven', 86 'FAST', job_retries=1))]) 87 88 self.files_to_filter = { 89 'with/deps/...': FakeControlData(self._TAG, self._ATTR, 90 'gets filtered'), 91 'with/profilers/...': FakeControlData(self._TAG, self._ATTR, 92 'gets filtered')} 93 94 95 def tearDown(self): 96 SuiteBase.ENABLE_CONTROLS_IN_BATCH = self.use_batch 97 super(SuiteTest, self).tearDown() 98 shutil.rmtree(self.tmpdir, ignore_errors=True) 99 100 101 def expect_control_file_parsing(self, suite_name=_TAG): 102 """Expect an attempt to parse the 'control files' in |self.files|. 103 104 @param suite_name: The suite name to parse control files for. 105 """ 106 all_files = self.files.keys() + self.files_to_filter.keys() 107 self._set_control_file_parsing_expectations(False, all_files, 108 self.files, suite_name) 109 110 111 def _set_control_file_parsing_expectations(self, already_stubbed, 112 file_list, files_to_parse, 113 suite_name): 114 """Expect an attempt to parse the 'control files' in |files|. 115 116 @param already_stubbed: parse_control_string already stubbed out. 117 @param file_list: the files the dev server returns 118 @param files_to_parse: the {'name': FakeControlData} dict of files we 119 expect to get parsed. 120 """ 121 if not already_stubbed: 122 self.mox.StubOutWithMock(control_data, 'parse_control_string') 123 124 self.getter.get_control_file_list( 125 suite_name=suite_name).AndReturn(file_list) 126 for file, data in files_to_parse.iteritems(): 127 self.getter.get_control_file_contents( 128 file).InAnyOrder().AndReturn(data.string) 129 control_data.parse_control_string( 130 data.string, 131 raise_warnings=True, 132 path=file).InAnyOrder().AndReturn(data) 133 134 135 def expect_control_file_parsing_in_batch(self, suite_name=_TAG): 136 """Expect an attempt to parse the contents of all control files in 137 |self.files| and |self.files_to_filter|, form them to a dict. 138 139 @param suite_name: The suite name to parse control files for. 140 """ 141 self.mox.StubOutWithMock(control_data, 'parse_control_string') 142 suite_info = {} 143 for k, v in self.files.iteritems(): 144 suite_info[k] = v.string 145 control_data.parse_control_string( 146 v.string, 147 raise_warnings=True, 148 path=k).InAnyOrder().AndReturn(v) 149 for k, v in self.files_to_filter.iteritems(): 150 suite_info[k] = v.string 151 self.getter._dev_server = self._DEVSERVER_HOST 152 self.getter.get_suite_info( 153 suite_name=suite_name).AndReturn(suite_info) 154 155 156 def testFindAllTestInBatch(self): 157 """Test switch on enable_getting_controls_in_batch for function 158 find_all_test.""" 159 self.use_batch = SuiteBase.ENABLE_CONTROLS_IN_BATCH 160 self.expect_control_file_parsing_in_batch() 161 SuiteBase.ENABLE_CONTROLS_IN_BATCH = True 162 163 self.mox.ReplayAll() 164 165 predicate = lambda d: d.suite == self._TAG 166 tests = Suite.find_and_parse_tests(self.getter, 167 predicate, 168 self._TAG, 169 add_experimental=True) 170 self.assertEquals(len(tests), 6) 171 self.assertTrue(self.files['one'] in tests) 172 self.assertTrue(self.files['two'] in tests) 173 self.assertTrue(self.files['three'] in tests) 174 self.assertTrue(self.files['five'] in tests) 175 self.assertTrue(self.files['six'] in tests) 176 self.assertTrue(self.files['seven'] in tests) 177 SuiteBase.ENABLE_CONTROLS_IN_BATCH = self.use_batch 178 179 180 def testFindAndParseStableTests(self): 181 """Should find only non-experimental tests that match a predicate.""" 182 self.expect_control_file_parsing() 183 self.mox.ReplayAll() 184 185 predicate = lambda d: d.text == self.files['two'].string 186 tests = Suite.find_and_parse_tests(self.getter, predicate, self._TAG) 187 self.assertEquals(len(tests), 1) 188 self.assertEquals(tests[0], self.files['two']) 189 190 191 def testFindSuiteSyntaxErrors(self): 192 """Check all control files for syntax errors. 193 194 This test actually parses all control files in the autotest directory 195 for syntax errors, by using the un-forgiving parser and pretending to 196 look for all control files with the suite attribute. 197 """ 198 autodir = os.path.abspath( 199 os.path.join(os.path.dirname(__file__), '..', '..', '..')) 200 fs_getter = Suite.create_fs_getter(autodir) 201 predicate = lambda t: hasattr(t, 'suite') 202 Suite.find_and_parse_tests(fs_getter, predicate, add_experimental=True, 203 forgiving_parser=False) 204 205 206 def testFindAndParseTestsSuite(self): 207 """Should find all tests that match a predicate.""" 208 self.expect_control_file_parsing() 209 self.mox.ReplayAll() 210 211 predicate = lambda d: d.suite == self._TAG 212 tests = Suite.find_and_parse_tests(self.getter, 213 predicate, 214 self._TAG, 215 add_experimental=True) 216 self.assertEquals(len(tests), 6) 217 self.assertTrue(self.files['one'] in tests) 218 self.assertTrue(self.files['two'] in tests) 219 self.assertTrue(self.files['three'] in tests) 220 self.assertTrue(self.files['five'] in tests) 221 self.assertTrue(self.files['six'] in tests) 222 self.assertTrue(self.files['seven'] in tests) 223 224 225 def testFindAndParseTestsAttr(self): 226 """Should find all tests that match a predicate.""" 227 self.expect_control_file_parsing() 228 self.mox.ReplayAll() 229 230 predicate = Suite.matches_attribute_expression_predicate('attr:attr') 231 tests = Suite.find_and_parse_tests(self.getter, 232 predicate, 233 self._TAG, 234 add_experimental=True) 235 self.assertEquals(len(tests), 6) 236 self.assertTrue(self.files['one'] in tests) 237 self.assertTrue(self.files['two'] in tests) 238 self.assertTrue(self.files['three'] in tests) 239 self.assertTrue(self.files['four'] in tests) 240 self.assertTrue(self.files['six'] in tests) 241 self.assertTrue(self.files['seven'] in tests) 242 243 244 def testAdHocSuiteCreation(self): 245 """Should be able to schedule an ad-hoc suite by specifying 246 a single test name.""" 247 self.expect_control_file_parsing(suite_name='ad_hoc_suite') 248 self.mox.ReplayAll() 249 predicate = Suite.test_name_equals_predicate('name-data_five') 250 suite = Suite.create_from_predicates([predicate], self._BUILDS, 251 self._BOARD, devserver=None, 252 cf_getter=self.getter, 253 afe=self.afe, tko=self.tko) 254 255 self.assertFalse(self.files['one'] in suite.tests) 256 self.assertFalse(self.files['two'] in suite.tests) 257 self.assertFalse(self.files['four'] in suite.tests) 258 self.assertTrue(self.files['five'] in suite.tests) 259 260 discoverer = SuiteBase._DynamicSuiteDiscoverer(suite.tests) 261 self.assertFalse(self.files['one'] in discoverer.unstable_tests) 262 self.assertFalse(self.files['two'] in discoverer.stable_tests) 263 self.assertFalse(self.files['one'] in discoverer.stable_tests) 264 self.assertFalse(self.files['two'] in discoverer.unstable_tests) 265 266 267 def testStableUnstableFilter(self): 268 """Should distinguish between experimental and stable tests.""" 269 self.expect_control_file_parsing() 270 self.mox.ReplayAll() 271 suite = Suite.create_from_name(self._TAG, self._BUILDS, self._BOARD, 272 devserver=None, 273 cf_getter=self.getter, 274 afe=self.afe, tko=self.tko) 275 276 self.assertTrue(self.files['one'] in suite.tests) 277 self.assertTrue(self.files['two'] in suite.tests) 278 # Sanity check. 279 self.assertFalse(self.files['four'] in suite.tests) 280 281 discoverer = SuiteBase._DynamicSuiteDiscoverer(suite.tests) 282 self.assertTrue(self.files['one'] in discoverer.unstable_tests) 283 self.assertTrue(self.files['two'] in discoverer.stable_tests) 284 self.assertFalse(self.files['one'] in discoverer.stable_tests) 285 self.assertFalse(self.files['two'] in discoverer.unstable_tests) 286 287 288 def mock_control_file_parsing(self): 289 """Fake out find_and_parse_tests(), returning content from |self.files|. 290 """ 291 for test in self.files.values(): 292 test.text = test.string # mimic parsing. 293 self.mox.StubOutWithMock(Suite, 'find_and_parse_tests') 294 Suite.find_and_parse_tests( 295 mox.IgnoreArg(), 296 mox.IgnoreArg(), 297 mox.IgnoreArg(), 298 add_experimental=True, 299 forgiving_parser=True, 300 run_prod_code=False, 301 test_args=None).AndReturn(self.files.values()) 302 303 304 def expect_job_scheduling(self, recorder, add_experimental, 305 tests_to_skip=[], ignore_deps=False, 306 raises=False, suite_deps=[], suite=None): 307 """Expect jobs to be scheduled for 'tests' in |self.files|. 308 309 @param add_experimental: expect jobs for experimental tests as well. 310 @param recorder: object with a record_entry to be used to record test 311 results. 312 @param tests_to_skip: [list, of, test, names] that we expect to skip. 313 @param ignore_deps: If true, ignore tests' dependencies. 314 @param raises: If True, expect exceptions. 315 @param suite_deps: If True, add suite level dependencies. 316 """ 317 record_job_id = suite and suite._results_dir 318 if record_job_id: 319 self.mox.StubOutWithMock(suite, '_remember_job_keyval') 320 recorder.record_entry( 321 StatusContains.CreateFromStrings('INFO', 'Start %s' % self._TAG), 322 log_in_subdir=False) 323 tests = self.files.values() 324 tests.sort(key=lambda test: test.experimental) 325 n = 1 326 for test in tests: 327 if not add_experimental and test.experimental: 328 continue 329 if test.name in tests_to_skip: 330 continue 331 dependencies = [] 332 if not ignore_deps: 333 dependencies.extend(test.dependencies) 334 if suite_deps: 335 dependencies.extend(suite_deps) 336 dependencies.append(self._BOARD) 337 build = self._BUILDS[provision.CROS_VERSION_PREFIX] 338 job_mock = self.afe.create_job( 339 control_file=test.text, 340 name=mox.And(mox.StrContains(build), 341 mox.StrContains(test.name)), 342 control_type=mox.IgnoreArg(), 343 meta_hosts=[self._BOARD], 344 dependencies=dependencies, 345 keyvals={'build': build, 'suite': self._TAG, 346 'builds': SuiteTest._BUILDS, 347 'experimental':test.experimental}, 348 max_runtime_mins=24*60, 349 timeout_mins=1440, 350 parent_job_id=None, 351 test_retry=0, 352 priority=priorities.Priority.DEFAULT, 353 synch_count=test.sync_count, 354 require_ssp=test.require_ssp 355 ) 356 if raises: 357 job_mock.AndRaise(error.NoEligibleHostException()) 358 recorder.record_entry( 359 StatusContains.CreateFromStrings('START', test.name), 360 log_in_subdir=False) 361 recorder.record_entry( 362 StatusContains.CreateFromStrings('TEST_NA', test.name), 363 log_in_subdir=False) 364 recorder.record_entry( 365 StatusContains.CreateFromStrings('END', test.name), 366 log_in_subdir=False) 367 else: 368 fake_job = FakeJob(id=n) 369 job_mock.AndReturn(fake_job) 370 if record_job_id: 371 suite._remember_job_keyval(fake_job) 372 n += 1 373 374 375 def testScheduleTestsAndRecord(self): 376 """Should schedule stable and experimental tests with the AFE.""" 377 name_list = ['name-data_two', 'name-data_three', 378 'name-data_four', 'name-data_five', 'name-data_six', 379 'name-data_seven', 'experimental_name-data_one'] 380 keyval_dict = {constants.SCHEDULED_TEST_COUNT_KEY: 7, 381 constants.SCHEDULED_TEST_NAMES_KEY: repr(name_list)} 382 383 self.mock_control_file_parsing() 384 self.mox.ReplayAll() 385 suite = Suite.create_from_name(self._TAG, self._BUILDS, self._BOARD, 386 self.devserver, 387 afe=self.afe, tko=self.tko, 388 results_dir=self.tmpdir) 389 self.mox.ResetAll() 390 recorder = self.mox.CreateMock(base_job.base_job) 391 self.expect_job_scheduling(recorder, add_experimental=True, suite=suite) 392 393 self.mox.StubOutWithMock(utils, 'write_keyval') 394 utils.write_keyval(self.tmpdir, keyval_dict) 395 self.mox.ReplayAll() 396 suite.schedule(recorder.record_entry, True) 397 for job in suite._jobs: 398 self.assertTrue(hasattr(job, 'test_name')) 399 400 401 def testScheduleStableTests(self): 402 """Should schedule only stable tests with the AFE.""" 403 # Since test data_one is experimental, it will be skip. 404 name_list = ['name-data_two', 'name-data_three', 405 'name-data_four', 'name-data_five', 'name-data_six', 406 'name-data_seven'] 407 keyval_dict = {constants.SCHEDULED_TEST_COUNT_KEY: 6, 408 constants.SCHEDULED_TEST_NAMES_KEY: repr(name_list)} 409 410 self.mock_control_file_parsing() 411 recorder = self.mox.CreateMock(base_job.base_job) 412 self.expect_job_scheduling(recorder, add_experimental=False) 413 self.mox.StubOutWithMock(utils, 'write_keyval') 414 utils.write_keyval(None, keyval_dict) 415 416 self.mox.ReplayAll() 417 suite = Suite.create_from_name(self._TAG, self._BUILDS, self._BOARD, 418 self.devserver, 419 afe=self.afe, tko=self.tko) 420 suite.schedule(recorder.record_entry, add_experimental=False) 421 422 423 def testScheduleStableTestsIgnoreDeps(self): 424 """Should schedule only stable tests with the AFE.""" 425 # Since test data_one is experimental, it will be skip. 426 name_list = ['name-data_two', 'name-data_three', 427 'name-data_four', 'name-data_five', 'name-data_six', 428 'name-data_seven'] 429 keyval_dict = {constants.SCHEDULED_TEST_COUNT_KEY: 6, 430 constants.SCHEDULED_TEST_NAMES_KEY: repr(name_list)} 431 432 self.mock_control_file_parsing() 433 recorder = self.mox.CreateMock(base_job.base_job) 434 self.expect_job_scheduling(recorder, add_experimental=False, 435 ignore_deps=True) 436 self.mox.StubOutWithMock(utils, 'write_keyval') 437 utils.write_keyval(None, keyval_dict) 438 439 self.mox.ReplayAll() 440 suite = Suite.create_from_name(self._TAG, self._BUILDS, self._BOARD, 441 self.devserver, 442 afe=self.afe, tko=self.tko, 443 ignore_deps=True) 444 suite.schedule(recorder.record_entry, add_experimental=False) 445 446 447 def testScheduleUnrunnableTestsTESTNA(self): 448 """Tests which fail to schedule should be TEST_NA.""" 449 # Since all tests will be fail to schedule, the num of scheduled tests 450 # will be zero. 451 name_list = [] 452 keyval_dict = {constants.SCHEDULED_TEST_COUNT_KEY: 0, 453 constants.SCHEDULED_TEST_NAMES_KEY: repr(name_list)} 454 455 self.mock_control_file_parsing() 456 recorder = self.mox.CreateMock(base_job.base_job) 457 self.expect_job_scheduling(recorder, add_experimental=True, raises=True) 458 self.mox.StubOutWithMock(utils, 'write_keyval') 459 utils.write_keyval(None, keyval_dict) 460 self.mox.ReplayAll() 461 suite = Suite.create_from_name(self._TAG, self._BUILDS, self._BOARD, 462 self.devserver, 463 afe=self.afe, tko=self.tko) 464 suite.schedule(recorder.record_entry, add_experimental=True) 465 466 467 def testRetryMapAfterScheduling(self): 468 """Test job-test and test-job mapping are correctly updated.""" 469 name_list = ['name-data_two', 'name-data_three', 470 'name-data_four', 'name-data_five', 'name-data_six', 471 'name-data_seven', 'experimental_name-data_one'] 472 keyval_dict = {constants.SCHEDULED_TEST_COUNT_KEY: 7, 473 constants.SCHEDULED_TEST_NAMES_KEY: repr(name_list)} 474 475 self.mock_control_file_parsing() 476 recorder = self.mox.CreateMock(base_job.base_job) 477 self.expect_job_scheduling(recorder, add_experimental=True) 478 self.mox.StubOutWithMock(utils, 'write_keyval') 479 utils.write_keyval(None, keyval_dict) 480 self.mox.ReplayAll() 481 suite = Suite.create_from_name(self._TAG, self._BUILDS, self._BOARD, 482 self.devserver, 483 afe=self.afe, tko=self.tko, 484 job_retry=True) 485 suite.schedule(recorder.record_entry, add_experimental=True) 486 all_files = self.files.items() 487 # Sort tests in self.files so that they are in the same 488 # order as they are scheduled. 489 all_files.sort(key=lambda record: record[1].experimental) 490 expected_retry_map = {} 491 for n in range(len(all_files)): 492 test = all_files[n][1] 493 job_id = n + 1 494 if test.job_retries > 0: 495 expected_retry_map[job_id] = { 496 'state': RetryHandler.States.NOT_ATTEMPTED, 497 'retry_max': test.job_retries} 498 self.assertEqual(expected_retry_map, suite._retry_handler._retry_map) 499 500 501 def testSuiteMaxRetries(self): 502 """Test suite max retries.""" 503 name_list = ['name-data_two', 'name-data_three', 504 'name-data_four', 'name-data_five', 'name-data_six', 505 'name-data_seven', 'experimental_name-data_one'] 506 keyval_dict = {constants.SCHEDULED_TEST_COUNT_KEY: 7, 507 constants.SCHEDULED_TEST_NAMES_KEY: repr(name_list)} 508 509 self.mock_control_file_parsing() 510 recorder = self.mox.CreateMock(base_job.base_job) 511 self.expect_job_scheduling(recorder, add_experimental=True) 512 self.mox.StubOutWithMock(utils, 'write_keyval') 513 utils.write_keyval(None, keyval_dict) 514 self.mox.ReplayAll() 515 suite = Suite.create_from_name(self._TAG, self._BUILDS, self._BOARD, 516 self.devserver, 517 afe=self.afe, tko=self.tko, 518 job_retry=True, max_retries=1) 519 suite.schedule(recorder.record_entry, add_experimental=True) 520 self.assertEqual(suite._retry_handler._max_retries, 1) 521 # Find the job_id of the test that allows retry 522 job_id = suite._retry_handler._retry_map.iterkeys().next() 523 suite._retry_handler.add_retry(old_job_id=job_id, new_job_id=10) 524 self.assertEqual(suite._retry_handler._max_retries, 0) 525 526 527 def testSuiteDependencies(self): 528 """Should add suite dependencies to tests scheduled.""" 529 # Since add_experimental set to False, will skip experimental data_one. 530 name_list = ['name-data_two', 'name-data_three', 531 'name-data_four', 'name-data_five', 'name-data_six', 532 'name-data_seven'] 533 keyval_dict = {constants.SCHEDULED_TEST_COUNT_KEY: 6, 534 constants.SCHEDULED_TEST_NAMES_KEY: repr(name_list)} 535 536 self.mock_control_file_parsing() 537 recorder = self.mox.CreateMock(base_job.base_job) 538 self.expect_job_scheduling(recorder, add_experimental=False, 539 suite_deps=['extra']) 540 self.mox.StubOutWithMock(utils, 'write_keyval') 541 utils.write_keyval(None, keyval_dict) 542 543 self.mox.ReplayAll() 544 suite = Suite.create_from_name(self._TAG, self._BUILDS, self._BOARD, 545 self.devserver, extra_deps=['extra'], 546 afe=self.afe, tko=self.tko) 547 suite.schedule(recorder.record_entry, add_experimental=False) 548 549 550 def _createSuiteWithMockedTestsAndControlFiles(self, file_bugs=False): 551 """Create a Suite, using mocked tests and control file contents. 552 553 @return Suite object, after mocking out behavior needed to create it. 554 """ 555 self.expect_control_file_parsing() 556 self.mox.ReplayAll() 557 suite = Suite.create_from_name(self._TAG, self._BUILDS, self._BOARD, 558 self.devserver, 559 self.getter, 560 afe=self.afe, tko=self.tko, 561 file_bugs=file_bugs, job_retry=True) 562 self.mox.ResetAll() 563 return suite 564 565 566 def _createSuiteMockResults(self, results_dir=None, will_file_bug=True, 567 result_status='FAIL'): 568 """Create a suite, returned a set of mocked results to expect. 569 570 @param results_dir: A mock results directory. 571 @param will_file_bug: Whether later a bug will be filed. 572 If true, will mock out tko method. 573 @param result_status: A desired result status, e.g. 'FAIL', 'WARN'. 574 575 @return List of mocked results to wait on. 576 """ 577 self.suite = self._createSuiteWithMockedTestsAndControlFiles( 578 file_bugs=True) 579 self.suite._results_dir = results_dir 580 test_report = self._get_bad_test_report(result_status) 581 test_predicates = test_report.predicates 582 test_fallout = test_report.fallout 583 584 self.recorder = self.mox.CreateMock(base_job.base_job) 585 self.recorder.record_entry = self.mox.CreateMock( 586 base_job.base_job.record_entry) 587 self._mock_recorder_with_results([test_predicates], self.recorder) 588 if will_file_bug: 589 self.suite._tko.run = self.mox.CreateMock(frontend.RpcClient.run) 590 self.suite._tko.run('get_detailed_test_views', 591 afe_job_id=self._FAKE_JOB_ID) 592 return [test_predicates, test_fallout] 593 594 595 def _mock_recorder_with_results(self, results, recorder): 596 """ 597 Checks that results are recoded in order, eg: 598 START, (status, name, reason) END 599 600 @param results: list of results 601 @param recorder: status recorder 602 """ 603 for result in results: 604 status = result[0] 605 test_name = result[1] 606 recorder.record_entry( 607 StatusContains.CreateFromStrings('START', test_name), 608 log_in_subdir=False) 609 recorder.record_entry( 610 StatusContains.CreateFromStrings(*result), 611 log_in_subdir=False).InAnyOrder('results') 612 recorder.record_entry( 613 StatusContains.CreateFromStrings('END %s' % status, test_name), 614 log_in_subdir=False) 615 616 617 def schedule_and_expect_these_results(self, suite, results, recorder): 618 """Create mox stubs for call to suite.schedule and 619 job_status.wait_for_results 620 621 @param suite: suite object for which to stub out schedule(...) 622 @param results: results object to be returned from 623 job_stats_wait_for_results(...) 624 @param recorder: mocked recorder object to replay status messages 625 """ 626 def result_generator(results): 627 """A simple generator which generates results as Status objects. 628 629 This generator handles 'send' by simply ignoring it. 630 631 @param results: results object to be returned from 632 job_stats_wait_for_results(...) 633 @yield: job_status.Status objects. 634 """ 635 results = map(lambda r: job_status.Status(*r), results) 636 for r in results: 637 new_input = (yield r) 638 if new_input: 639 yield None 640 641 self.mox.StubOutWithMock(suite, 'schedule') 642 suite.schedule(recorder.record_entry, True) 643 suite._retry_handler = RetryHandler({}) 644 645 self.mox.StubOutWithMock(job_status, 'wait_for_results') 646 job_status.wait_for_results( 647 self.afe, self.tko, suite._jobs).AndReturn( 648 result_generator(results)) 649 650 651 def testRunAndWaitSuccess(self): 652 """Should record successful results.""" 653 suite = self._createSuiteWithMockedTestsAndControlFiles() 654 655 recorder = self.mox.CreateMock(base_job.base_job) 656 657 results = [('GOOD', 'good'), ('FAIL', 'bad', 'reason')] 658 self._mock_recorder_with_results(results, recorder) 659 self.schedule_and_expect_these_results(suite, results, recorder) 660 self.mox.ReplayAll() 661 662 suite.schedule(recorder.record_entry, True) 663 suite.wait(recorder.record_entry) 664 665 666 def testRunAndWaitFailure(self): 667 """Should record failure to gather results.""" 668 suite = self._createSuiteWithMockedTestsAndControlFiles() 669 670 recorder = self.mox.CreateMock(base_job.base_job) 671 recorder.record_entry( 672 StatusContains.CreateFromStrings('FAIL', self._TAG, 'waiting'), 673 log_in_subdir=False) 674 675 self.mox.StubOutWithMock(suite, 'schedule') 676 suite.schedule(recorder.record_entry, True) 677 self.mox.StubOutWithMock(job_status, 'wait_for_results') 678 job_status.wait_for_results(mox.IgnoreArg(), 679 mox.IgnoreArg(), 680 mox.IgnoreArg()).AndRaise( 681 Exception('Expected during test.')) 682 self.mox.ReplayAll() 683 684 suite.schedule(recorder.record_entry, True) 685 suite.wait(recorder.record_entry) 686 687 688 def testRunAndWaitScheduleFailure(self): 689 """Should record failure to schedule jobs.""" 690 suite = self._createSuiteWithMockedTestsAndControlFiles() 691 692 recorder = self.mox.CreateMock(base_job.base_job) 693 recorder.record_entry( 694 StatusContains.CreateFromStrings('INFO', 'Start %s' % self._TAG), 695 log_in_subdir=False) 696 697 recorder.record_entry( 698 StatusContains.CreateFromStrings('FAIL', self._TAG, 'scheduling'), 699 log_in_subdir=False) 700 701 self.mox.StubOutWithMock(suite, '_create_job') 702 suite._create_job(mox.IgnoreArg(), retry_for=mox.IgnoreArg()).AndRaise( 703 Exception('Expected during test.')) 704 self.mox.ReplayAll() 705 706 suite.schedule(recorder.record_entry, True) 707 suite.wait(recorder.record_entry) 708 709 710 def testGetTestsSortedByTime(self): 711 """Should find all tests and sorted by TIME setting.""" 712 self.expect_control_file_parsing() 713 self.mox.ReplayAll() 714 # Get all tests. 715 tests = Suite.find_and_parse_tests(self.getter, 716 lambda d: True, 717 self._TAG, 718 add_experimental=True) 719 self.assertEquals(len(tests), 7) 720 times = [control_data.ControlData.get_test_time_index(test.time) 721 for test in tests] 722 self.assertTrue(all(x>=y for x, y in zip(times, times[1:])), 723 'Tests are not ordered correctly.') 724 725 726 def _get_bad_test_report(self, result_status='FAIL'): 727 """ 728 Fetch the predicates of a failing test, and the parameters 729 that are a fallout of this test failing. 730 """ 731 predicates = collections.namedtuple('predicates', 732 'status, testname, reason') 733 fallout = collections.namedtuple('fallout', 734 ('time_start, time_end, job_id,' 735 'username, hostname')) 736 test_report = collections.namedtuple('test_report', 737 'predicates, fallout') 738 return test_report(predicates(result_status, 'bad_test', 739 'dreadful_reason'), 740 fallout('2014-01-01 01:01:01', 'None', 741 self._FAKE_JOB_ID, 'user', 'myhost')) 742 743 744 def mock_bug_filing(self, test_results): 745 """A helper function that mocks bug filing. 746 747 @param test_results: A named tuple (predicates, fallout) representing 748 a bad test report. 749 """ 750 def check_result(result): 751 """ 752 Checks to see if the status passed to the bug reporter contains all 753 the arguments required to file bugs. 754 755 @param result: The result we get when a test fails. 756 """ 757 test_predicates = test_results[0] 758 test_fallout = test_results[1] 759 expected_result = job_status.Status( 760 test_predicates.status, test_predicates.testname, 761 reason=test_predicates.reason, 762 job_id=test_fallout.job_id, owner=test_fallout.username, 763 hostname=test_fallout.hostname, 764 begin_time_str=test_fallout.time_start) 765 766 return all(getattr(result, k, None) == v for k, v in 767 expected_result.__dict__.iteritems() 768 if 'timestamp' not in str(k)) 769 770 self.mox.StubOutWithMock(reporting, 'TestBug') 771 reporting.TestBug(self._BUILDS[provision.CROS_VERSION_PREFIX], 772 mox.IgnoreArg(), mox.IgnoreArg(), 773 mox.Func(check_result)) 774 775 self.mox.StubOutClassWithMocks(phapi_lib, 'ProjectHostingApiClient') 776 mock_host = phapi_lib.ProjectHostingApiClient(mox.IgnoreArg(), 777 mox.IgnoreArg(), 778 mox.IgnoreArg()) 779 self.mox.StubOutWithMock(reporting.Reporter, 'report') 780 reporting.Reporter.report(mox.IgnoreArg(), 781 mox.IgnoreArg()).AndReturn((0, 0)) 782 783 self.mox.StubOutWithMock(utils, 'write_keyval') 784 utils.write_keyval(mox.IgnoreArg(), mox.IgnoreArg()) 785 786 787 def testBugFiling(self): 788 """ 789 Confirm that all the necessary predicates are passed on to the 790 bug reporter when a test fails. 791 """ 792 test_results = self._createSuiteMockResults() 793 self.schedule_and_expect_these_results( 794 self.suite, 795 [test_results[0] + test_results[1]], 796 self.recorder) 797 798 self.mock_bug_filing(test_results) 799 self.mox.ReplayAll() 800 801 self.suite.schedule(self.recorder.record_entry, True) 802 self.suite._jobs_to_tests[self._FAKE_JOB_ID] = self.files['seven'] 803 self.suite.wait(self.recorder.record_entry) 804 805 806 def testFailedBugFiling(self): 807 """ 808 Make sure the suite survives even if we cannot file bugs. 809 """ 810 test_results = self._createSuiteMockResults(self.tmpdir) 811 self.schedule_and_expect_these_results( 812 self.suite, 813 [test_results[0] + test_results[1]], 814 self.recorder) 815 self.mox.StubOutWithMock(reporting.Reporter, '_check_tracker') 816 self.mox.StubOutClassWithMocks(phapi_lib, 'ProjectHostingApiClient') 817 mock_host = phapi_lib.ProjectHostingApiClient(mox.IgnoreArg(), 818 mox.IgnoreArg(), 819 mox.IgnoreArg()) 820 reporting.Reporter._check_tracker().AndReturn(False) 821 822 self.mox.ReplayAll() 823 824 self.suite.schedule(self.recorder.record_entry, True) 825 self.suite._jobs_to_tests[self._FAKE_JOB_ID] = self.files['seven'] 826 self.suite.wait(self.recorder.record_entry) 827 828 829 def testJobRetryTestFail(self): 830 """Test retry works.""" 831 test_to_retry = self.files['seven'] 832 fake_new_job_id = self._FAKE_JOB_ID + 1 833 fake_job = FakeJob(id=self._FAKE_JOB_ID) 834 fake_new_job = FakeJob(id=fake_new_job_id) 835 836 test_results = self._createSuiteMockResults(will_file_bug=False) 837 self.schedule_and_expect_these_results( 838 self.suite, 839 [test_results[0] + test_results[1]], 840 self.recorder) 841 self.mox.StubOutWithMock(self.suite, '_create_job') 842 self.suite._create_job( 843 test_to_retry, 844 retry_for=self._FAKE_JOB_ID).AndReturn(fake_new_job) 845 self.mox.ReplayAll() 846 self.suite.schedule(self.recorder.record_entry, True) 847 self.suite._retry_handler._retry_map = { 848 self._FAKE_JOB_ID: {'state': RetryHandler.States.NOT_ATTEMPTED, 849 'retry_max': 1} 850 } 851 self.suite._jobs_to_tests[self._FAKE_JOB_ID] = test_to_retry 852 self.suite.wait(self.recorder.record_entry) 853 expected_retry_map = { 854 self._FAKE_JOB_ID: {'state': RetryHandler.States.RETRIED, 855 'retry_max': 1}, 856 fake_new_job_id: {'state': RetryHandler.States.NOT_ATTEMPTED, 857 'retry_max': 0} 858 } 859 # Check retry map is correctly updated 860 self.assertEquals(self.suite._retry_handler._retry_map, 861 expected_retry_map) 862 # Check _jobs_to_tests is correctly updated 863 self.assertEquals(self.suite._jobs_to_tests[fake_new_job_id], 864 test_to_retry) 865 866 867 def testJobRetryTestWarn(self): 868 """Test that no retry is scheduled if test warns.""" 869 test_to_retry = self.files['seven'] 870 fake_job = FakeJob(id=self._FAKE_JOB_ID) 871 test_results = self._createSuiteMockResults( 872 will_file_bug=True, result_status='WARN') 873 self.schedule_and_expect_these_results( 874 self.suite, 875 [test_results[0] + test_results[1]], 876 self.recorder) 877 # A bug should be filed if test warns. 878 self.mock_bug_filing(test_results) 879 self.mox.ReplayAll() 880 self.suite.schedule(self.recorder.record_entry, True) 881 self.suite._retry_handler._retry_map = { 882 self._FAKE_JOB_ID: {'state': RetryHandler.States.NOT_ATTEMPTED, 883 'retry_max': 1} 884 } 885 self.suite._jobs_to_tests[self._FAKE_JOB_ID] = test_to_retry 886 expected_jobs_to_tests = self.suite._jobs_to_tests.copy() 887 expected_retry_map = self.suite._retry_handler._retry_map.copy() 888 self.suite.wait(self.recorder.record_entry) 889 # Check retry map and _jobs_to_tests, ensure no retry was scheduled. 890 self.assertEquals(self.suite._retry_handler._retry_map, 891 expected_retry_map) 892 self.assertEquals(self.suite._jobs_to_tests, expected_jobs_to_tests) 893 894 895 def testFailedJobRetry(self): 896 """Make sure the suite survives even if the retry failed.""" 897 test_to_retry = self.files['seven'] 898 fake_job = FakeJob(id=self._FAKE_JOB_ID) 899 900 test_results = self._createSuiteMockResults(will_file_bug=False) 901 self.schedule_and_expect_these_results( 902 self.suite, 903 [test_results[0] + test_results[1]], 904 self.recorder) 905 self.mox.StubOutWithMock(self.suite, '_create_job') 906 self.suite._create_job( 907 test_to_retry, retry_for=self._FAKE_JOB_ID).AndRaise( 908 error.RPCException('Expected during test')) 909 # Do not file a bug. 910 self.mox.StubOutWithMock(self.suite, '_should_report') 911 self.suite._should_report(mox.IgnoreArg()).AndReturn(False) 912 913 self.mox.ReplayAll() 914 915 self.suite.schedule(self.recorder.record_entry, True) 916 self.suite._retry_handler._retry_map = { 917 self._FAKE_JOB_ID: { 918 'state': RetryHandler.States.NOT_ATTEMPTED, 919 'retry_max': 1}} 920 self.suite._jobs_to_tests[self._FAKE_JOB_ID] = test_to_retry 921 self.suite.wait(self.recorder.record_entry) 922 expected_retry_map = { 923 self._FAKE_JOB_ID: { 924 'state': RetryHandler.States.ATTEMPTED, 925 'retry_max': 1}} 926 expected_jobs_to_tests = self.suite._jobs_to_tests.copy() 927 self.assertEquals(self.suite._retry_handler._retry_map, 928 expected_retry_map) 929 self.assertEquals(self.suite._jobs_to_tests, expected_jobs_to_tests) 930 931 932if __name__ == '__main__': 933 unittest.main() 934