1# Copyright 2016 The Chromium OS Authors. All rights reserved. 2# Use of this source code is governed by a BSD-style license that can be 3# found in the LICENSE file. 4 5"""Unit tests for the `repair` module.""" 6 7import functools 8import logging 9import unittest 10 11import common 12from autotest_lib.client.common_lib import hosts 13from autotest_lib.client.common_lib.hosts import repair 14 15 16class _StubHost(object): 17 """ 18 Stub class to fill in the relevant methods of `Host`. 19 20 This class provides mocking and stub behaviors for `Host` for use by 21 tests within this module. The class implements only those methods 22 that `Verifier` and `RepairAction` actually use. 23 """ 24 25 def __init__(self): 26 self._record_sequence = [] 27 28 29 def record(self, status_code, subdir, operation, status=''): 30 """ 31 Mock method to capture records written to `status.log`. 32 33 Each record is remembered in order to be checked for correctness 34 by individual tests later. 35 36 @param status_code As for `Host.record()`. 37 @param subdir As for `Host.record()`. 38 @param operation As for `Host.record()`. 39 @param status As for `Host.record()`. 40 """ 41 full_record = (status_code, subdir, operation, status) 42 self._record_sequence.append(full_record) 43 44 45 def get_log_records(self): 46 """ 47 Return the records logged for this fake host. 48 49 The returned list of records excludes records where the 50 `operation` parameter is not in `tagset`. 51 52 @param tagset Only include log records with these tags. 53 """ 54 return self._record_sequence 55 56 57 def reset_log_records(self): 58 """Clear our history of log records to allow re-testing.""" 59 self._record_sequence = [] 60 61 62class _StubVerifier(hosts.Verifier): 63 """ 64 Stub implementation of `Verifier` for testing purposes. 65 66 This is a full implementation of a concrete `Verifier` subclass 67 designed to allow calling unit tests control over whether verify 68 passes or fails. 69 70 A `_StubVerifier()` will pass whenever the value of `_fail_count` 71 is non-zero. Calls to `try_repair()` (typically made by a 72 `_StubRepairAction()`) will reduce this count, eventually 73 "repairing" the verifier. 74 75 @property verify_count The number of calls made to the instance's 76 `verify()` method. 77 @property message If verification fails, the exception raised, 78 when converted to a string, will have this 79 value. 80 @property _fail_count The number of repair attempts required 81 before this verifier will succeed. A 82 non-zero value means verification will fail. 83 @property _description The value of the `description` property. 84 """ 85 86 def __init__(self, tag, deps, fail_count): 87 super(_StubVerifier, self).__init__(tag, deps) 88 self.verify_count = 0 89 self.message = 'Failing "%s" by request' % tag 90 self._fail_count = fail_count 91 self._description = 'Testing verify() for "%s"' % tag 92 self._log_record_map = { 93 r[0]: r for r in [ 94 ('GOOD', None, self._verify_tag, ''), 95 ('FAIL', None, self._verify_tag, self.message), 96 ] 97 } 98 99 100 def __repr__(self): 101 return '_StubVerifier(%r, %r, %r)' % ( 102 self.tag, self._dependency_list, self._fail_count) 103 104 105 def verify(self, host): 106 self.verify_count += 1 107 if self._fail_count: 108 raise hosts.AutoservVerifyError(self.message) 109 110 111 def try_repair(self): 112 """Bring ourselves one step closer to working.""" 113 if self._fail_count: 114 self._fail_count -= 1 115 116 117 def unrepair(self): 118 """Make ourselves more broken.""" 119 self._fail_count += 1 120 121 122 def get_log_record(self, status): 123 """ 124 Return a host log record for this verifier. 125 126 Calculates the arguments expected to be passed to 127 `Host.record()` by `Verifier._verify_host()` when this verifier 128 runs. The passed in `status` corresponds to the argument of the 129 same name to be passed to `Host.record()`. 130 131 @param status Status value of the log record. 132 """ 133 return self._log_record_map[status] 134 135 136 @property 137 def description(self): 138 return self._description 139 140 141class _StubRepairFailure(Exception): 142 """Exception to be raised by `_StubRepairAction.repair()`.""" 143 pass 144 145 146class _StubRepairAction(hosts.RepairAction): 147 """Stub implementation of `RepairAction` for testing purposes. 148 149 This is a full implementation of a concrete `RepairAction` subclass 150 designed to allow calling unit tests control over whether repair 151 passes or fails. 152 153 The behavior of `repair()` depends on the `_success` property of a 154 `_StubRepairAction`. When the property is true, repair will call 155 `try_repair()` for all triggers, and then report success. When the 156 property is false, repair reports failure. 157 158 @property repair_count The number of calls made to the instance's 159 `repair()` method. 160 @property message If repair fails, the exception raised, when 161 converted to a string, will have this value. 162 @property _success Whether repair will follow its "success" or 163 "failure" paths. 164 @property _description The value of the `description` property. 165 """ 166 167 def __init__(self, tag, deps, triggers, success): 168 super(_StubRepairAction, self).__init__(tag, deps, triggers) 169 self.repair_count = 0 170 self.message = 'Failed repair for "%s"' % tag 171 self._success = success 172 self._description = 'Testing repair for "%s"' % tag 173 self._log_record_map = { 174 r[0]: r for r in [ 175 ('START', None, self._repair_tag, ''), 176 ('FAIL', None, self._repair_tag, self.message), 177 ('END FAIL', None, self._repair_tag, ''), 178 ('END GOOD', None, self._repair_tag, ''), 179 ] 180 } 181 182 183 def __repr__(self): 184 return '_StubRepairAction(%r, %r, %r, %r)' % ( 185 self.tag, self._dependency_list, 186 self._trigger_list, self._success) 187 188 189 def repair(self, host): 190 self.repair_count += 1 191 if not self._success: 192 raise _StubRepairFailure(self.message) 193 for v in self._trigger_list: 194 v.try_repair() 195 196 197 def get_log_record(self, status): 198 """ 199 Return a host log record for this repair action. 200 201 Calculates the arguments expected to be passed to 202 `Host.record()` by `RepairAction._repair_host()` when repair 203 runs. The passed in `status` corresponds to the argument of the 204 same name to be passed to `Host.record()`. 205 206 @param status Status value of the log record. 207 """ 208 return self._log_record_map[status] 209 210 211 @property 212 def description(self): 213 return self._description 214 215 216class _DependencyNodeTestCase(unittest.TestCase): 217 """ 218 Abstract base class for `RepairAction` and `Verifier` test cases. 219 220 This class provides `_make_verifier()` and `_make_repair_action()` 221 methods to create `_StubVerifier` and `_StubRepairAction` instances, 222 respectively, for testing. Constructed verifiers and repair actions 223 are remembered in `self.nodes`, a dictionary indexed by the tag 224 used to construct the object. 225 """ 226 227 def setUp(self): 228 logging.disable(logging.CRITICAL) 229 self._fake_host = _StubHost() 230 self.nodes = {} 231 232 233 def tearDown(self): 234 logging.disable(logging.NOTSET) 235 236 237 def _make_verifier(self, count, tag, deps): 238 """ 239 Make a `_StubVerifier` and remember it in `self.nodes`. 240 241 @param count As for the `_StubVerifer` constructor. 242 @param tag As for the `_StubVerifer` constructor. 243 @param deps As for the `_StubVerifer` constructor. 244 """ 245 verifier = _StubVerifier(tag, deps, count) 246 self.nodes[tag] = verifier 247 return verifier 248 249 250 def _make_repair_action(self, success, tag, deps, triggers): 251 """ 252 Make a `_StubRepairAction` and remember it in `self.nodes`. 253 254 @param success As for the `_StubRepairAction` constructor. 255 @param tag As for the `_StubRepairAction` constructor. 256 @param deps As for the `_StubRepairAction` constructor. 257 @param triggers As for the `_StubRepairAction` constructor. 258 """ 259 repair_action = _StubRepairAction(tag, deps, triggers, success) 260 self.nodes[tag] = repair_action 261 return repair_action 262 263 264 def _make_expected_failures(self, *verifiers): 265 """ 266 Make a set of `_DependencyFailure` objects from `verifiers`. 267 268 Return the set of `_DependencyFailure` objects that we would 269 expect to see in the `failures` attribute of an 270 `AutoservVerifyDependencyError` if all of the given verifiers 271 report failure. 272 273 @param verifiers A list of `_StubVerifier` objects that are 274 expected to fail. 275 276 @return A set of `_DependencyFailure` objects. 277 """ 278 failures = [repair._DependencyFailure(v.description, v.message) 279 for v in verifiers] 280 return set(failures) 281 282 283 def _generate_silent(self): 284 """ 285 Iterator to test different settings of the `silent` parameter. 286 287 This iterator exists to standardize testing assertions that 288 This iterator exists to standardize testing common 289 assertions about the `silent` parameter: 290 * When the parameter is true, no calls are made to the 291 `record()` method on the target host. 292 * When the parameter is false, certain expected calls are made 293 to the `record()` method on the target host. 294 295 The iterator is meant to be used like this: 296 297 for silent in self._generate_silent(): 298 # run test case that uses the silent parameter 299 self._check_log_records(silent, ... expected records ... ) 300 301 The code above will run its test case twice, once with 302 `silent=True` and once with `silent=False`. In between the 303 calls, log records are cleared. 304 305 @yields A boolean setting for `silent`. 306 """ 307 for silent in [False, True]: 308 yield silent 309 self._fake_host.reset_log_records() 310 311 312 def _check_log_records(self, silent, *record_data): 313 """ 314 Assert that log records occurred as expected. 315 316 Elements of `record_data` should be tuples of the form 317 `(tag, status)`, describing one expected log record. 318 The verifier or repair action for `tag` provides the expected 319 log record based on the status value. 320 321 The `silent` parameter is the value that was passed to the 322 verifier or repair action that did the logging. When true, 323 it indicates that no records should have been logged. 324 325 @param record_data List describing the expected record events. 326 @param silent When true, ignore `record_data` and assert 327 that nothing was logged. 328 """ 329 expected_records = [] 330 if not silent: 331 for tag, status in record_data: 332 expected_records.append( 333 self.nodes[tag].get_log_record(status)) 334 actual_records = self._fake_host.get_log_records() 335 self.assertEqual(expected_records, actual_records) 336 337 338class VerifyTests(_DependencyNodeTestCase): 339 """ 340 Unit tests for `Verifier`. 341 342 The tests in this class test the fundamental behaviors of the 343 `Verifier` class: 344 * Results from the `verify()` method are cached; the method is 345 only called the first time that `_verify_host()` is called. 346 * The `_verify_host()` method makes the expected calls to 347 `Host.record()` for every call to the `verify()` method. 348 * When a dependency fails, the dependent verifier isn't called. 349 * Verifier calls are made in the order required by the DAG. 350 351 The test cases don't use `RepairStrategy` to build DAG structures, 352 but instead rely on custom-built DAGs. 353 """ 354 355 def _generate_verify_count(self, verifier): 356 """ 357 Iterator to force a standard sequence with calls to `_reverify()`. 358 359 This iterator exists to standardize testing two common 360 assertions: 361 * The side effects from calling `_verify_host()` only 362 happen on the first call to the method, except... 363 * Calling `_reverify()` resets a verifier so that the 364 next call to `_verify_host()` will repeat the side 365 effects. 366 367 The iterator is meant to be used like this: 368 369 for count in self._generate_verify_cases(verifier): 370 # run a verifier._verify_host() test case 371 self.assertEqual(verifier.verify_count, count) 372 self._check_log_records(silent, ... expected records ... ) 373 374 The code above will run the `_verify_host()` test case twice, 375 then call `_reverify()` to clear cached results, then re-run 376 the test case two more times. 377 378 @param verifier The verifier to be tested and reverified. 379 @yields Each iteration yields the number of times `_reverify()` 380 has been called. 381 """ 382 for i in range(1, 3): 383 for _ in range(0, 2): 384 yield i 385 verifier._reverify() 386 self._fake_host.reset_log_records() 387 388 389 def test_success(self): 390 """ 391 Test proper handling of a successful verification. 392 393 Construct and call a simple, single-node verification that will 394 pass. Assert the following: 395 * The `verify()` method is called once. 396 * The expected 'GOOD' record is logged via `Host.record()`. 397 * If `_verify_host()` is called more than once, there are no 398 visible side-effects after the first call. 399 * Calling `_reverify()` clears all cached results. 400 """ 401 for silent in self._generate_silent(): 402 verifier = self._make_verifier(0, 'pass', []) 403 for count in self._generate_verify_count(verifier): 404 verifier._verify_host(self._fake_host, silent) 405 self.assertEqual(verifier.verify_count, count) 406 self._check_log_records(silent, ('pass', 'GOOD')) 407 408 409 def test_fail(self): 410 """ 411 Test proper handling of verification failure. 412 413 Construct and call a simple, single-node verification that will 414 fail. Assert the following: 415 * The failure is reported with the actual exception raised 416 by the verifier. 417 * The `verify()` method is called once. 418 * The expected 'FAIL' record is logged via `Host.record()`. 419 * If `_verify_host()` is called more than once, there are no 420 visible side-effects after the first call. 421 * Calling `_reverify()` clears all cached results. 422 """ 423 for silent in self._generate_silent(): 424 verifier = self._make_verifier(1, 'fail', []) 425 for count in self._generate_verify_count(verifier): 426 with self.assertRaises(hosts.AutoservVerifyError) as e: 427 verifier._verify_host(self._fake_host, silent) 428 self.assertEqual(verifier.verify_count, count) 429 self.assertEqual(verifier.message, str(e.exception)) 430 self._check_log_records(silent, ('fail', 'FAIL')) 431 432 433 def test_dependency_success(self): 434 """ 435 Test proper handling of dependencies that succeed. 436 437 Construct and call a two-node verification with one node 438 dependent on the other, where both nodes will pass. Assert the 439 following: 440 * The `verify()` method for both nodes is called once. 441 * The expected 'GOOD' record is logged via `Host.record()` 442 for both nodes. 443 * If `_verify_host()` is called more than once, there are no 444 visible side-effects after the first call. 445 * Calling `_reverify()` clears all cached results. 446 """ 447 for silent in self._generate_silent(): 448 child = self._make_verifier(0, 'pass', []) 449 parent = self._make_verifier(0, 'parent', [child]) 450 for count in self._generate_verify_count(parent): 451 parent._verify_host(self._fake_host, silent) 452 self.assertEqual(parent.verify_count, count) 453 self.assertEqual(child.verify_count, count) 454 self._check_log_records(silent, 455 ('pass', 'GOOD'), 456 ('parent', 'GOOD')) 457 458 459 def test_dependency_fail(self): 460 """ 461 Test proper handling of dependencies that fail. 462 463 Construct and call a two-node verification with one node 464 dependent on the other, where the dependency will fail. Assert 465 the following: 466 * The verification exception is `AutoservVerifyDependencyError`, 467 and the exception argument is the description of the failed 468 node. 469 * The `verify()` method for the failing node is called once, 470 and for the other node, not at all. 471 * The expected 'FAIL' record is logged via `Host.record()` 472 for the single failed node. 473 * If `_verify_host()` is called more than once, there are no 474 visible side-effects after the first call. 475 * Calling `_reverify()` clears all cached results. 476 """ 477 for silent in self._generate_silent(): 478 child = self._make_verifier(1, 'fail', []) 479 parent = self._make_verifier(0, 'parent', [child]) 480 failures = self._make_expected_failures(child) 481 for count in self._generate_verify_count(parent): 482 expected_exception = hosts.AutoservVerifyDependencyError 483 with self.assertRaises(expected_exception) as e: 484 parent._verify_host(self._fake_host, silent) 485 self.assertEqual(e.exception.failures, failures) 486 self.assertEqual(child.verify_count, count) 487 self.assertEqual(parent.verify_count, 0) 488 self._check_log_records(silent, ('fail', 'FAIL')) 489 490 491 def test_two_dependencies_pass(self): 492 """ 493 Test proper handling with two passing dependencies. 494 495 Construct and call a three-node verification with one node 496 dependent on the other two, where all nodes will pass. Assert 497 the following: 498 * The `verify()` method for all nodes is called once. 499 * The expected 'GOOD' records are logged via `Host.record()` 500 for all three nodes. 501 * If `_verify_host()` is called more than once, there are no 502 visible side-effects after the first call. 503 * Calling `_reverify()` clears all cached results. 504 """ 505 for silent in self._generate_silent(): 506 left = self._make_verifier(0, 'left', []) 507 right = self._make_verifier(0, 'right', []) 508 top = self._make_verifier(0, 'top', [left, right]) 509 for count in self._generate_verify_count(top): 510 top._verify_host(self._fake_host, silent) 511 self.assertEqual(top.verify_count, count) 512 self.assertEqual(left.verify_count, count) 513 self.assertEqual(right.verify_count, count) 514 self._check_log_records(silent, 515 ('left', 'GOOD'), 516 ('right', 'GOOD'), 517 ('top', 'GOOD')) 518 519 520 def test_two_dependencies_fail(self): 521 """ 522 Test proper handling with two failing dependencies. 523 524 Construct and call a three-node verification with one node 525 dependent on the other two, where both dependencies will fail. 526 Assert the following: 527 * The verification exception is `AutoservVerifyDependencyError`, 528 and the exception argument has the descriptions of both the 529 failed nodes. 530 * The `verify()` method for each failing node is called once, 531 and for the parent node not at all. 532 * The expected 'FAIL' records are logged via `Host.record()` 533 for the failing nodes. 534 * If `_verify_host()` is called more than once, there are no 535 visible side-effects after the first call. 536 * Calling `_reverify()` clears all cached results. 537 """ 538 for silent in self._generate_silent(): 539 left = self._make_verifier(1, 'left', []) 540 right = self._make_verifier(1, 'right', []) 541 top = self._make_verifier(0, 'top', [left, right]) 542 failures = self._make_expected_failures(left, right) 543 for count in self._generate_verify_count(top): 544 expected_exception = hosts.AutoservVerifyDependencyError 545 with self.assertRaises(expected_exception) as e: 546 top._verify_host(self._fake_host, silent) 547 self.assertEqual(e.exception.failures, failures) 548 self.assertEqual(top.verify_count, 0) 549 self.assertEqual(left.verify_count, count) 550 self.assertEqual(right.verify_count, count) 551 self._check_log_records(silent, 552 ('left', 'FAIL'), 553 ('right', 'FAIL')) 554 555 556 def test_two_dependencies_mixed(self): 557 """ 558 Test proper handling with mixed dependencies. 559 560 Construct and call a three-node verification with one node 561 dependent on the other two, where one dependency will pass, 562 and one will fail. Assert the following: 563 * The verification exception is `AutoservVerifyDependencyError`, 564 and the exception argument has the descriptions of the 565 single failed node. 566 * The `verify()` method for each dependency is called once, 567 and for the parent node not at all. 568 * The expected 'GOOD' and 'FAIL' records are logged via 569 `Host.record()` for the dependencies. 570 * If `_verify_host()` is called more than once, there are no 571 visible side-effects after the first call. 572 * Calling `_reverify()` clears all cached results. 573 """ 574 for silent in self._generate_silent(): 575 left = self._make_verifier(1, 'left', []) 576 right = self._make_verifier(0, 'right', []) 577 top = self._make_verifier(0, 'top', [left, right]) 578 failures = self._make_expected_failures(left) 579 for count in self._generate_verify_count(top): 580 expected_exception = hosts.AutoservVerifyDependencyError 581 with self.assertRaises(expected_exception) as e: 582 top._verify_host(self._fake_host, silent) 583 self.assertEqual(e.exception.failures, failures) 584 self.assertEqual(top.verify_count, 0) 585 self.assertEqual(left.verify_count, count) 586 self.assertEqual(right.verify_count, count) 587 self._check_log_records(silent, 588 ('left', 'FAIL'), 589 ('right', 'GOOD')) 590 591 592 def test_diamond_pass(self): 593 """ 594 Test a "diamond" structure DAG with all nodes passing. 595 596 Construct and call a "diamond" structure DAG where all nodes 597 will pass: 598 599 TOP 600 / \ 601 LEFT RIGHT 602 \ / 603 BOTTOM 604 605 Assert the following: 606 * The `verify()` method for all nodes is called once. 607 * The expected 'GOOD' records are logged via `Host.record()` 608 for all nodes. 609 * If `_verify_host()` is called more than once, there are no 610 visible side-effects after the first call. 611 * Calling `_reverify()` clears all cached results. 612 """ 613 for silent in self._generate_silent(): 614 bottom = self._make_verifier(0, 'bottom', []) 615 left = self._make_verifier(0, 'left', [bottom]) 616 right = self._make_verifier(0, 'right', [bottom]) 617 top = self._make_verifier(0, 'top', [left, right]) 618 for count in self._generate_verify_count(top): 619 top._verify_host(self._fake_host, silent) 620 self.assertEqual(top.verify_count, count) 621 self.assertEqual(left.verify_count, count) 622 self.assertEqual(right.verify_count, count) 623 self.assertEqual(bottom.verify_count, count) 624 self._check_log_records(silent, 625 ('bottom', 'GOOD'), 626 ('left', 'GOOD'), 627 ('right', 'GOOD'), 628 ('top', 'GOOD')) 629 630 631 def test_diamond_fail(self): 632 """ 633 Test a "diamond" structure DAG with the bottom node failing. 634 635 Construct and call a "diamond" structure DAG where the bottom 636 node will fail: 637 638 TOP 639 / \ 640 LEFT RIGHT 641 \ / 642 BOTTOM 643 644 Assert the following: 645 * The verification exception is `AutoservVerifyDependencyError`, 646 and the exception argument has the description of the 647 "bottom" node. 648 * The `verify()` method for the "bottom" node is called once, 649 and for the other nodes not at all. 650 * The expected 'FAIL' record is logged via `Host.record()` 651 for the "bottom" node. 652 * If `_verify_host()` is called more than once, there are no 653 visible side-effects after the first call. 654 * Calling `_reverify()` clears all cached results. 655 """ 656 for silent in self._generate_silent(): 657 bottom = self._make_verifier(1, 'bottom', []) 658 left = self._make_verifier(0, 'left', [bottom]) 659 right = self._make_verifier(0, 'right', [bottom]) 660 top = self._make_verifier(0, 'top', [left, right]) 661 failures = self._make_expected_failures(bottom) 662 for count in self._generate_verify_count(top): 663 expected_exception = hosts.AutoservVerifyDependencyError 664 with self.assertRaises(expected_exception) as e: 665 top._verify_host(self._fake_host, silent) 666 self.assertEqual(e.exception.failures, failures) 667 self.assertEqual(top.verify_count, 0) 668 self.assertEqual(left.verify_count, 0) 669 self.assertEqual(right.verify_count, 0) 670 self.assertEqual(bottom.verify_count, count) 671 self._check_log_records(silent, ('bottom', 'FAIL')) 672 673 674class RepairActionTests(_DependencyNodeTestCase): 675 """ 676 Unit tests for `RepairAction`. 677 678 The tests in this class test the fundamental behaviors of the 679 `RepairAction` class: 680 * Repair doesn't run unless all dependencies pass. 681 * Repair doesn't run unless at least one trigger fails. 682 * The `_repair_host()` method makes the expected calls to 683 `Host.record()` for every call to the `repair()` method. 684 685 The test cases don't use `RepairStrategy` to build repair 686 graphs, but instead rely on custom-built structures. 687 """ 688 689 def test_repair_not_triggered(self): 690 """ 691 Test a repair that doesn't trigger. 692 693 Construct and call a repair action with a verification trigger 694 that passes. Assert the following: 695 * The `verify()` method for the trigger is called. 696 * The `repair()` method is not called. 697 * The verifier logs the expected 'GOOD' message with 698 `Host.record()`. 699 * The repair action logs no messages with `Host.record()`. 700 """ 701 for silent in self._generate_silent(): 702 verifier = self._make_verifier(0, 'check', []) 703 repair_action = self._make_repair_action(True, 'unneeded', 704 [], [verifier]) 705 repair_action._repair_host(self._fake_host, silent) 706 self.assertEqual(verifier.verify_count, 1) 707 self.assertEqual(repair_action.repair_count, 0) 708 self._check_log_records(silent, ('check', 'GOOD')) 709 710 711 def test_repair_fails(self): 712 """ 713 Test a repair that triggers and fails. 714 715 Construct and call a repair action with a verification trigger 716 that fails. The repair fails by raising `_StubRepairFailure`. 717 Assert the following: 718 * The repair action fails with the `_StubRepairFailure` raised 719 by `repair()`. 720 * The `verify()` method for the trigger is called once. 721 * The `repair()` method is called once. 722 * The expected 'START', 'FAIL', and 'END FAIL' messages are 723 logged with `Host.record()` for the failed verifier and the 724 failed repair. 725 """ 726 for silent in self._generate_silent(): 727 verifier = self._make_verifier(1, 'fail', []) 728 repair_action = self._make_repair_action(False, 'nofix', 729 [], [verifier]) 730 with self.assertRaises(_StubRepairFailure) as e: 731 repair_action._repair_host(self._fake_host, silent) 732 self.assertEqual(repair_action.message, str(e.exception)) 733 self.assertEqual(verifier.verify_count, 1) 734 self.assertEqual(repair_action.repair_count, 1) 735 self._check_log_records(silent, 736 ('fail', 'FAIL'), 737 ('nofix', 'START'), 738 ('nofix', 'FAIL'), 739 ('nofix', 'END FAIL')) 740 741 742 def test_repair_success(self): 743 """ 744 Test a repair that fixes its trigger. 745 746 Construct and call a repair action that raises no exceptions, 747 using a repair trigger that fails first, then passes after 748 repair. Assert the following: 749 * The `repair()` method is called once. 750 * The trigger's `verify()` method is called twice. 751 * The expected 'START', 'FAIL', 'GOOD', and 'END GOOD' 752 messages are logged with `Host.record()` for the verifier 753 and the repair. 754 """ 755 for silent in self._generate_silent(): 756 verifier = self._make_verifier(1, 'fail', []) 757 repair_action = self._make_repair_action(True, 'fix', 758 [], [verifier]) 759 repair_action._repair_host(self._fake_host, silent) 760 self.assertEqual(repair_action.repair_count, 1) 761 self.assertEqual(verifier.verify_count, 2) 762 self._check_log_records(silent, 763 ('fail', 'FAIL'), 764 ('fix', 'START'), 765 ('fail', 'GOOD'), 766 ('fix', 'END GOOD')) 767 768 769 def test_repair_noop(self): 770 """ 771 Test a repair that doesn't fix a failing trigger. 772 773 Construct and call a repair action with a trigger that fails. 774 The repair action raises no exceptions, and after repair, the 775 trigger still fails. Assert the following: 776 * The `_repair_host()` call fails with `AutoservRepairError`. 777 * The `repair()` method is called once. 778 * The trigger's `verify()` method is called twice. 779 * The expected 'START', 'FAIL', and 'END FAIL' messages are 780 logged with `Host.record()` for the verifier and the repair. 781 """ 782 for silent in self._generate_silent(): 783 verifier = self._make_verifier(2, 'fail', []) 784 repair_action = self._make_repair_action(True, 'nofix', 785 [], [verifier]) 786 with self.assertRaises(hosts.AutoservRepairError) as e: 787 repair_action._repair_host(self._fake_host, silent) 788 self.assertEqual(repair_action.repair_count, 1) 789 self.assertEqual(verifier.verify_count, 2) 790 self._check_log_records(silent, 791 ('fail', 'FAIL'), 792 ('nofix', 'START'), 793 ('fail', 'FAIL'), 794 ('nofix', 'END FAIL')) 795 796 797 def test_dependency_pass(self): 798 """ 799 Test proper handling of repair dependencies that pass. 800 801 Construct and call a repair action with a dependency and a 802 trigger. The dependency will pass and the trigger will fail and 803 be repaired. Assert the following: 804 * Repair passes. 805 * The `verify()` method for the dependency is called once. 806 * The `verify()` method for the trigger is called twice. 807 * The `repair()` method is called once. 808 * The expected records are logged via `Host.record()` 809 for the successful dependency, the failed trigger, and 810 the successful repair. 811 """ 812 for silent in self._generate_silent(): 813 dep = self._make_verifier(0, 'dep', []) 814 trigger = self._make_verifier(1, 'trig', []) 815 repair = self._make_repair_action(True, 'fixit', 816 [dep], [trigger]) 817 repair._repair_host(self._fake_host, silent) 818 self.assertEqual(dep.verify_count, 1) 819 self.assertEqual(trigger.verify_count, 2) 820 self.assertEqual(repair.repair_count, 1) 821 self._check_log_records(silent, 822 ('dep', 'GOOD'), 823 ('trig', 'FAIL'), 824 ('fixit', 'START'), 825 ('trig', 'GOOD'), 826 ('fixit', 'END GOOD')) 827 828 829 def test_dependency_fail(self): 830 """ 831 Test proper handling of repair dependencies that fail. 832 833 Construct and call a repair action with a dependency and a 834 trigger, both of which fail. Assert the following: 835 * Repair fails with `AutoservVerifyDependencyError`, 836 and the exception argument is the description of the failed 837 dependency. 838 * The `verify()` method for the failing dependency is called 839 once. 840 * The trigger and the repair action aren't invoked at all. 841 * The expected 'FAIL' record is logged via `Host.record()` 842 for the single failed dependency. 843 """ 844 for silent in self._generate_silent(): 845 dep = self._make_verifier(1, 'dep', []) 846 trigger = self._make_verifier(1, 'trig', []) 847 repair = self._make_repair_action(True, 'fixit', 848 [dep], [trigger]) 849 expected_exception = hosts.AutoservVerifyDependencyError 850 with self.assertRaises(expected_exception) as e: 851 repair._repair_host(self._fake_host, silent) 852 self.assertEqual(e.exception.failures, 853 self._make_expected_failures(dep)) 854 self.assertEqual(dep.verify_count, 1) 855 self.assertEqual(trigger.verify_count, 0) 856 self.assertEqual(repair.repair_count, 0) 857 self._check_log_records(silent, ('dep', 'FAIL')) 858 859 860class _RepairStrategyTestCase(_DependencyNodeTestCase): 861 """Shared base class for testing `RepairStrategy` methods.""" 862 863 def _make_verify_data(self, *input_data): 864 """ 865 Create `verify_data` for the `RepairStrategy` constructor. 866 867 `RepairStrategy` expects `verify_data` as a list of tuples 868 of the form `(constructor, tag, deps)`. Each item in 869 `input_data` is a tuple of the form `(tag, count, deps)` that 870 creates one entry in the returned list of `verify_data` tuples 871 as follows: 872 * `count` is used to create a constructor function that calls 873 `self._make_verifier()` with that value plus plus the 874 arguments provided by the `RepairStrategy` constructor. 875 * `tag` and `deps` will be passed as-is to the `RepairStrategy` 876 constructor. 877 878 @param input_data A list of tuples, each representing one 879 tuple in the `verify_data` list. 880 @return A list suitable to be the `verify_data` parameter for 881 the `RepairStrategy` constructor. 882 """ 883 strategy_data = [] 884 for tag, count, deps in input_data: 885 construct = functools.partial(self._make_verifier, count) 886 strategy_data.append((construct, tag, deps)) 887 return strategy_data 888 889 890 def _make_repair_data(self, *input_data): 891 """ 892 Create `repair_data` for the `RepairStrategy` constructor. 893 894 `RepairStrategy` expects `repair_data` as a list of tuples 895 of the form `(constructor, tag, deps, triggers)`. Each item in 896 `input_data` is a tuple of the form `(tag, success, deps, triggers)` 897 that creates one entry in the returned list of `repair_data` 898 tuples as follows: 899 * `success` is used to create a constructor function that calls 900 `self._make_verifier()` with that value plus plus the 901 arguments provided by the `RepairStrategy` constructor. 902 * `tag`, `deps`, and `triggers` will be passed as-is to the 903 `RepairStrategy` constructor. 904 905 @param input_data A list of tuples, each representing one 906 tuple in the `repair_data` list. 907 @return A list suitable to be the `repair_data` parameter for 908 the `RepairStrategy` constructor. 909 """ 910 strategy_data = [] 911 for tag, success, deps, triggers in input_data: 912 construct = functools.partial(self._make_repair_action, success) 913 strategy_data.append((construct, tag, deps, triggers)) 914 return strategy_data 915 916 917 def _make_strategy(self, verify_input, repair_input): 918 """ 919 Create a `RepairStrategy` from the given arguments. 920 921 @param verify_input As for `input_data` in 922 `_make_verify_data()`. 923 @param repair_input As for `input_data` in 924 `_make_repair_data()`. 925 """ 926 verify_data = self._make_verify_data(*verify_input) 927 repair_data = self._make_repair_data(*repair_input) 928 return hosts.RepairStrategy(verify_data, repair_data) 929 930 def _check_silent_records(self, silent): 931 """ 932 Check that logging honored the `silent` parameter. 933 934 Asserts that logging with `Host.record()` occurred (or did not 935 occur) in accordance with the value of `silent`. 936 937 This method only asserts the presence or absence of log records. 938 Coverage for the contents of the log records is handled in other 939 test cases. 940 941 @param silent When true, there should be no log records; 942 otherwise there should be records present. 943 """ 944 log_records = self._fake_host.get_log_records() 945 if silent: 946 self.assertEqual(log_records, []) 947 else: 948 self.assertNotEqual(log_records, []) 949 950 951class RepairStrategyVerifyTests(_RepairStrategyTestCase): 952 """ 953 Unit tests for `RepairStrategy.verify()`. 954 955 These unit tests focus on verifying that the `RepairStrategy` 956 constructor creates the expected DAG structure from given 957 `verify_data`. Functional testing here is mainly confined to 958 asserting that `RepairStrategy.verify()` properly distinguishes 959 success from failure. Testing the behavior of specific DAG 960 structures is left to tests in `VerifyTests`. 961 """ 962 963 def test_single_node(self): 964 """ 965 Test construction of a single-node verification DAG. 966 967 Assert that the structure looks like this: 968 969 Root Node -> Main Node 970 """ 971 verify_data = self._make_verify_data(('main', 0, ())) 972 strategy = hosts.RepairStrategy(verify_data, []) 973 verifier = self.nodes['main'] 974 self.assertEqual( 975 strategy._verify_root._dependency_list, 976 [verifier]) 977 self.assertEqual(verifier._dependency_list, []) 978 979 980 def test_single_dependency(self): 981 """ 982 Test construction of a two-node dependency chain. 983 984 Assert that the structure looks like this: 985 986 Root Node -> Parent Node -> Child Node 987 """ 988 verify_data = self._make_verify_data( 989 ('child', 0, ()), 990 ('parent', 0, ('child',))) 991 strategy = hosts.RepairStrategy(verify_data, []) 992 parent = self.nodes['parent'] 993 child = self.nodes['child'] 994 self.assertEqual( 995 strategy._verify_root._dependency_list, [parent]) 996 self.assertEqual( 997 parent._dependency_list, [child]) 998 self.assertEqual( 999 child._dependency_list, []) 1000 1001 1002 def test_two_nodes_and_dependency(self): 1003 """ 1004 Test construction of two nodes with a shared dependency. 1005 1006 Assert that the structure looks like this: 1007 1008 Root Node -> Left Node ---\ 1009 \ -> Bottom Node 1010 -> Right Node / 1011 """ 1012 verify_data = self._make_verify_data( 1013 ('bottom', 0, ()), 1014 ('left', 0, ('bottom',)), 1015 ('right', 0, ('bottom',))) 1016 strategy = hosts.RepairStrategy(verify_data, []) 1017 bottom = self.nodes['bottom'] 1018 left = self.nodes['left'] 1019 right = self.nodes['right'] 1020 self.assertEqual( 1021 strategy._verify_root._dependency_list, 1022 [left, right]) 1023 self.assertEqual(left._dependency_list, [bottom]) 1024 self.assertEqual(right._dependency_list, [bottom]) 1025 self.assertEqual(bottom._dependency_list, []) 1026 1027 1028 def test_three_nodes(self): 1029 """ 1030 Test construction of three nodes with no dependencies. 1031 1032 Assert that the structure looks like this: 1033 1034 -> Node One 1035 / 1036 Root Node -> Node Two 1037 \ 1038 -> Node Three 1039 1040 N.B. This test exists to enforce ordering expectations of 1041 root-level DAG nodes. Three nodes are used to make it unlikely 1042 that randomly ordered roots will match expectations. 1043 """ 1044 verify_data = self._make_verify_data( 1045 ('one', 0, ()), 1046 ('two', 0, ()), 1047 ('three', 0, ())) 1048 strategy = hosts.RepairStrategy(verify_data, []) 1049 one = self.nodes['one'] 1050 two = self.nodes['two'] 1051 three = self.nodes['three'] 1052 self.assertEqual( 1053 strategy._verify_root._dependency_list, 1054 [one, two, three]) 1055 self.assertEqual(one._dependency_list, []) 1056 self.assertEqual(two._dependency_list, []) 1057 self.assertEqual(three._dependency_list, []) 1058 1059 1060 def test_verify(self): 1061 """ 1062 Test behavior of the `verify()` method. 1063 1064 Build a `RepairStrategy` with a single verifier. Assert the 1065 following: 1066 * If the verifier passes, `verify()` passes. 1067 * If the verifier fails, `verify()` fails. 1068 * The verifier is reinvoked with every call to `verify()`; 1069 cached results are not re-used. 1070 """ 1071 verify_data = self._make_verify_data(('tester', 0, ())) 1072 strategy = hosts.RepairStrategy(verify_data, []) 1073 verifier = self.nodes['tester'] 1074 count = 0 1075 for silent in self._generate_silent(): 1076 for i in range(0, 2): 1077 for j in range(0, 2): 1078 strategy.verify(self._fake_host, silent) 1079 self._check_silent_records(silent) 1080 count += 1 1081 self.assertEqual(verifier.verify_count, count) 1082 verifier.unrepair() 1083 for j in range(0, 2): 1084 with self.assertRaises(Exception) as e: 1085 strategy.verify(self._fake_host, silent) 1086 self._check_silent_records(silent) 1087 count += 1 1088 self.assertEqual(verifier.verify_count, count) 1089 verifier.try_repair() 1090 1091 1092class RepairStrategyRepairTests(_RepairStrategyTestCase): 1093 """ 1094 Unit tests for `RepairStrategy.repair()`. 1095 1096 These unit tests focus on verifying that the `RepairStrategy` 1097 constructor creates the expected repair list from given 1098 `repair_data`. Functional testing here is confined to asserting 1099 that `RepairStrategy.repair()` properly distinguishes success from 1100 failure. Testing the behavior of specific repair structures is left 1101 to tests in `RepairActionTests`. 1102 """ 1103 1104 def _check_common_trigger(self, strategy, repair_tags, triggers): 1105 self.assertEqual(strategy._repair_actions, 1106 [self.nodes[tag] for tag in repair_tags]) 1107 for tag in repair_tags: 1108 self.assertEqual(self.nodes[tag]._trigger_list, 1109 triggers) 1110 self.assertEqual(self.nodes[tag]._dependency_list, []) 1111 1112 1113 def test_single_repair_with_trigger(self): 1114 """ 1115 Test constructing a strategy with a single repair trigger. 1116 1117 Build a `RepairStrategy` with a single repair action and a 1118 single trigger. Assert that the trigger graph looks like this: 1119 1120 Repair -> Trigger 1121 1122 Assert that there are no repair dependencies. 1123 """ 1124 verify_input = (('base', 0, ()),) 1125 repair_input = (('fixit', True, (), ('base',)),) 1126 strategy = self._make_strategy(verify_input, repair_input) 1127 self._check_common_trigger(strategy, 1128 ['fixit'], 1129 [self.nodes['base']]) 1130 1131 1132 def test_repair_with_root_trigger(self): 1133 """ 1134 Test construction of a repair triggering on the root verifier. 1135 1136 Build a `RepairStrategy` with a single repair action that 1137 triggers on the root verifier. Assert that the trigger graph 1138 looks like this: 1139 1140 Repair -> Root Verifier 1141 1142 Assert that there are no repair dependencies. 1143 """ 1144 root_tag = hosts.RepairStrategy.ROOT_TAG 1145 repair_input = (('fixit', True, (), (root_tag,)),) 1146 strategy = self._make_strategy([], repair_input) 1147 self._check_common_trigger(strategy, 1148 ['fixit'], 1149 [strategy._verify_root]) 1150 1151 1152 def test_three_repairs(self): 1153 """ 1154 Test constructing a strategy with three repair actions. 1155 1156 Build a `RepairStrategy` with a three repair actions sharing a 1157 single trigger. Assert that the trigger graph looks like this: 1158 1159 Repair A -> Trigger 1160 Repair B -> Trigger 1161 Repair C -> Trigger 1162 1163 Assert that there are no repair dependencies. 1164 1165 N.B. This test exists to enforce ordering expectations of 1166 repair nodes. Three nodes are used to make it unlikely that 1167 randomly ordered actions will match expectations. 1168 """ 1169 verify_input = (('base', 0, ()),) 1170 repair_tags = ['a', 'b', 'c'] 1171 repair_input = ( 1172 (tag, True, (), ('base',)) for tag in repair_tags) 1173 strategy = self._make_strategy(verify_input, repair_input) 1174 self._check_common_trigger(strategy, 1175 repair_tags, 1176 [self.nodes['base']]) 1177 1178 1179 def test_repair_dependency(self): 1180 """ 1181 Test construction of a repair with a dependency. 1182 1183 Build a `RepairStrategy` with a single repair action that 1184 depends on a single verifier. Assert that the dependency graph 1185 looks like this: 1186 1187 Repair -> Verifier 1188 1189 Assert that there are no repair triggers. 1190 """ 1191 verify_input = (('base', 0, ()),) 1192 repair_input = (('fixit', True, ('base',), ()),) 1193 strategy = self._make_strategy(verify_input, repair_input) 1194 self.assertEqual(strategy._repair_actions, 1195 [self.nodes['fixit']]) 1196 self.assertEqual(self.nodes['fixit']._trigger_list, []) 1197 self.assertEqual(self.nodes['fixit']._dependency_list, 1198 [self.nodes['base']]) 1199 1200 1201 def _check_repair_failure(self, strategy, silent): 1202 """ 1203 Check the effects of a call to `repair()` that fails. 1204 1205 For the given strategy object, call the `repair()` method; the 1206 call is expected to fail and all repair actions are expected to 1207 trigger. 1208 1209 Assert the following: 1210 * The call raises an exception. 1211 * For each repair action in the strategy, its `repair()` 1212 method is called exactly once. 1213 1214 @param strategy The strategy to be tested. 1215 """ 1216 action_counts = [(a, a.repair_count) 1217 for a in strategy._repair_actions] 1218 with self.assertRaises(Exception) as e: 1219 strategy.repair(self._fake_host, silent) 1220 self._check_silent_records(silent) 1221 for action, count in action_counts: 1222 self.assertEqual(action.repair_count, count + 1) 1223 1224 1225 def _check_repair_success(self, strategy, silent): 1226 """ 1227 Check the effects of a call to `repair()` that succeeds. 1228 1229 For the given strategy object, call the `repair()` method; the 1230 call is expected to succeed without raising an exception and all 1231 repair actions are expected to trigger. 1232 1233 Assert that for each repair action in the strategy, its 1234 `repair()` method is called exactly once. 1235 1236 @param strategy The strategy to be tested. 1237 """ 1238 action_counts = [(a, a.repair_count) 1239 for a in strategy._repair_actions] 1240 strategy.repair(self._fake_host, silent) 1241 self._check_silent_records(silent) 1242 for action, count in action_counts: 1243 self.assertEqual(action.repair_count, count + 1) 1244 1245 1246 def test_repair(self): 1247 """ 1248 Test behavior of the `repair()` method. 1249 1250 Build a `RepairStrategy` with two repair actions each depending 1251 on its own verifier. Set up calls to `repair()` for each of 1252 the following conditions: 1253 * Both repair actions trigger and fail. 1254 * Both repair actions trigger and succeed. 1255 * Both repair actions trigger; the first one fails, but the 1256 second one succeeds. 1257 * Both repair actions trigger; the first one succeeds, but the 1258 second one fails. 1259 1260 Assert the following: 1261 * When both repair actions succeed, `repair()` succeeds. 1262 * When either repair action fails, `repair()` fails. 1263 * After each call to the strategy's `repair()` method, each 1264 repair action triggered exactly once. 1265 """ 1266 verify_input = (('a', 2, ()), ('b', 2, ())) 1267 repair_input = (('afix', True, (), ('a',)), 1268 ('bfix', True, (), ('b',))) 1269 strategy = self._make_strategy(verify_input, repair_input) 1270 1271 for silent in self._generate_silent(): 1272 # call where both 'afix' and 'bfix' fail 1273 self._check_repair_failure(strategy, silent) 1274 # repair counts are now 1 for both verifiers 1275 1276 # call where both 'afix' and 'bfix' succeed 1277 self._check_repair_success(strategy, silent) 1278 # repair counts are now 0 for both verifiers 1279 1280 # call where 'afix' fails and 'bfix' succeeds 1281 for tag in ['a', 'a', 'b']: 1282 self.nodes[tag].unrepair() 1283 self._check_repair_failure(strategy, silent) 1284 # 'a' repair count is 1; 'b' count is 0 1285 1286 # call where 'afix' succeeds and 'bfix' fails 1287 for tag in ['b', 'b']: 1288 self.nodes[tag].unrepair() 1289 self._check_repair_failure(strategy, silent) 1290 # 'a' repair count is 0; 'b' count is 1 1291 1292 for tag in ['a', 'a', 'b']: 1293 self.nodes[tag].unrepair() 1294 # repair counts are now 2 for both verifiers 1295 1296 1297if __name__ == '__main__': 1298 unittest.main() 1299