• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1# Copyright 2016 The Chromium OS Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5"""Unit tests for the `repair` module."""
6
7import functools
8import logging
9import unittest
10
11import common
12from autotest_lib.client.common_lib import hosts
13from autotest_lib.client.common_lib.hosts import repair
14
15
16class _StubHost(object):
17    """
18    Stub class to fill in the relevant methods of `Host`.
19
20    This class provides mocking and stub behaviors for `Host` for use by
21    tests within this module.  The class implements only those methods
22    that `Verifier` and `RepairAction` actually use.
23    """
24
25    def __init__(self):
26        self._record_sequence = []
27
28
29    def record(self, status_code, subdir, operation, status=''):
30        """
31        Mock method to capture records written to `status.log`.
32
33        Each record is remembered in order to be checked for correctness
34        by individual tests later.
35
36        @param status_code  As for `Host.record()`.
37        @param subdir       As for `Host.record()`.
38        @param operation    As for `Host.record()`.
39        @param status       As for `Host.record()`.
40        """
41        full_record = (status_code, subdir, operation, status)
42        self._record_sequence.append(full_record)
43
44
45    def get_log_records(self):
46        """
47        Return the records logged for this fake host.
48
49        The returned list of records excludes records where the
50        `operation` parameter is not in `tagset`.
51
52        @param tagset   Only include log records with these tags.
53        """
54        return self._record_sequence
55
56
57    def reset_log_records(self):
58        """Clear our history of log records to allow re-testing."""
59        self._record_sequence = []
60
61
62class _StubVerifier(hosts.Verifier):
63    """
64    Stub implementation of `Verifier` for testing purposes.
65
66    This is a full implementation of a concrete `Verifier` subclass
67    designed to allow calling unit tests control over whether verify
68    passes or fails.
69
70    A `_StubVerifier()` will pass whenever the value of `_fail_count`
71    is non-zero.  Calls to `try_repair()` (typically made by a
72    `_StubRepairAction()`) will reduce this count, eventually
73    "repairing" the verifier.
74
75    @property verify_count  The number of calls made to the instance's
76                            `verify()` method.
77    @property message       If verification fails, the exception raised,
78                            when converted to a string, will have this
79                            value.
80    @property _fail_count   The number of repair attempts required
81                            before this verifier will succeed.  A
82                            non-zero value means verification will fail.
83    @property _description  The value of the `description` property.
84    """
85
86    def __init__(self, tag, deps, fail_count):
87        super(_StubVerifier, self).__init__(tag, deps)
88        self.verify_count = 0
89        self.message = 'Failing "%s" by request' % tag
90        self._fail_count = fail_count
91        self._description = 'Testing verify() for "%s"' % tag
92        self._log_record_map = {
93            r[0]: r for r in [
94                ('GOOD', None, self._verify_tag, ''),
95                ('FAIL', None, self._verify_tag, self.message),
96            ]
97        }
98
99
100    def __repr__(self):
101        return '_StubVerifier(%r, %r, %r)' % (
102                self.tag, self._dependency_list, self._fail_count)
103
104
105    def verify(self, host):
106        self.verify_count += 1
107        if self._fail_count:
108            raise hosts.AutoservVerifyError(self.message)
109
110
111    def try_repair(self):
112        """Bring ourselves one step closer to working."""
113        if self._fail_count:
114            self._fail_count -= 1
115
116
117    def unrepair(self):
118        """Make ourselves more broken."""
119        self._fail_count += 1
120
121
122    def get_log_record(self, status):
123        """
124        Return a host log record for this verifier.
125
126        Calculates the arguments expected to be passed to
127        `Host.record()` by `Verifier._verify_host()` when this verifier
128        runs.  The passed in `status` corresponds to the argument of the
129        same name to be passed to `Host.record()`.
130
131        @param status   Status value of the log record.
132        """
133        return self._log_record_map[status]
134
135
136    @property
137    def description(self):
138        return self._description
139
140
141class _StubRepairFailure(Exception):
142    """Exception to be raised by `_StubRepairAction.repair()`."""
143    pass
144
145
146class _StubRepairAction(hosts.RepairAction):
147    """Stub implementation of `RepairAction` for testing purposes.
148
149    This is a full implementation of a concrete `RepairAction` subclass
150    designed to allow calling unit tests control over whether repair
151    passes or fails.
152
153    The behavior of `repair()` depends on the `_success` property of a
154    `_StubRepairAction`.  When the property is true, repair will call
155    `try_repair()` for all triggers, and then report success.  When the
156    property is false, repair reports failure.
157
158    @property repair_count  The number of calls made to the instance's
159                            `repair()` method.
160    @property message       If repair fails, the exception raised, when
161                            converted to a string, will have this value.
162    @property _success      Whether repair will follow its "success" or
163                            "failure" paths.
164    @property _description  The value of the `description` property.
165    """
166
167    def __init__(self, tag, deps, triggers, success):
168        super(_StubRepairAction, self).__init__(tag, deps, triggers)
169        self.repair_count = 0
170        self.message = 'Failed repair for "%s"' % tag
171        self._success = success
172        self._description = 'Testing repair for "%s"' % tag
173        self._log_record_map = {
174            r[0]: r for r in [
175                ('START', None, self._repair_tag, ''),
176                ('FAIL', None, self._repair_tag, self.message),
177                ('END FAIL', None, self._repair_tag, ''),
178                ('END GOOD', None, self._repair_tag, ''),
179            ]
180        }
181
182
183    def __repr__(self):
184        return '_StubRepairAction(%r, %r, %r, %r)' % (
185                self.tag, self._dependency_list,
186                self._trigger_list, self._success)
187
188
189    def repair(self, host):
190        self.repair_count += 1
191        if not self._success:
192            raise _StubRepairFailure(self.message)
193        for v in self._trigger_list:
194            v.try_repair()
195
196
197    def get_log_record(self, status):
198        """
199        Return a host log record for this repair action.
200
201        Calculates the arguments expected to be passed to
202        `Host.record()` by `RepairAction._repair_host()` when repair
203        runs.  The passed in `status` corresponds to the argument of the
204        same name to be passed to `Host.record()`.
205
206        @param status   Status value of the log record.
207        """
208        return self._log_record_map[status]
209
210
211    @property
212    def description(self):
213        return self._description
214
215
216class _DependencyNodeTestCase(unittest.TestCase):
217    """
218    Abstract base class for `RepairAction` and `Verifier` test cases.
219
220    This class provides `_make_verifier()` and `_make_repair_action()`
221    methods to create `_StubVerifier` and `_StubRepairAction` instances,
222    respectively, for testing.  Constructed verifiers and repair actions
223    are remembered in `self.nodes`, a dictionary indexed by the tag
224    used to construct the object.
225    """
226
227    def setUp(self):
228        logging.disable(logging.CRITICAL)
229        self._fake_host = _StubHost()
230        self.nodes = {}
231
232
233    def tearDown(self):
234        logging.disable(logging.NOTSET)
235
236
237    def _make_verifier(self, count, tag, deps):
238        """
239        Make a `_StubVerifier` and remember it in `self.nodes`.
240
241        @param count  As for the `_StubVerifer` constructor.
242        @param tag    As for the `_StubVerifer` constructor.
243        @param deps   As for the `_StubVerifer` constructor.
244        """
245        verifier = _StubVerifier(tag, deps, count)
246        self.nodes[tag] = verifier
247        return verifier
248
249
250    def _make_repair_action(self, success, tag, deps, triggers):
251        """
252        Make a `_StubRepairAction` and remember it in `self.nodes`.
253
254        @param success    As for the `_StubRepairAction` constructor.
255        @param tag        As for the `_StubRepairAction` constructor.
256        @param deps       As for the `_StubRepairAction` constructor.
257        @param triggers   As for the `_StubRepairAction` constructor.
258        """
259        repair_action = _StubRepairAction(tag, deps, triggers, success)
260        self.nodes[tag] = repair_action
261        return repair_action
262
263
264    def _make_expected_failures(self, *verifiers):
265        """
266        Make a set of `_DependencyFailure` objects from `verifiers`.
267
268        Return the set of `_DependencyFailure` objects that we would
269        expect to see in the `failures` attribute of an
270        `AutoservVerifyDependencyError` if all of the given verifiers
271        report failure.
272
273        @param verifiers  A list of `_StubVerifier` objects that are
274                          expected to fail.
275
276        @return A set of `_DependencyFailure` objects.
277        """
278        failures = [repair._DependencyFailure(v.description, v.message)
279                    for v in verifiers]
280        return set(failures)
281
282
283    def _generate_silent(self):
284        """
285        Iterator to test different settings of the `silent` parameter.
286
287        This iterator exists to standardize testing assertions that
288        This iterator exists to standardize testing common
289        assertions about the `silent` parameter:
290          * When the parameter is true, no calls are made to the
291            `record()` method on the target host.
292          * When the parameter is false, certain expected calls are made
293            to the `record()` method on the target host.
294
295        The iterator is meant to be used like this:
296
297            for silent in self._generate_silent():
298                # run test case that uses the silent parameter
299                self._check_log_records(silent, ... expected records ... )
300
301        The code above will run its test case twice, once with
302        `silent=True` and once with `silent=False`.  In between the
303        calls, log records are cleared.
304
305        @yields A boolean setting for `silent`.
306        """
307        for silent in [False, True]:
308            yield silent
309            self._fake_host.reset_log_records()
310
311
312    def _check_log_records(self, silent, *record_data):
313        """
314        Assert that log records occurred as expected.
315
316        Elements of `record_data` should be tuples of the form
317        `(tag, status)`, describing one expected log record.
318        The verifier or repair action for `tag` provides the expected
319        log record based on the status value.
320
321        The `silent` parameter is the value that was passed to the
322        verifier or repair action that did the logging.  When true,
323        it indicates that no records should have been logged.
324
325        @param record_data  List describing the expected record events.
326        @param silent       When true, ignore `record_data` and assert
327                            that nothing was logged.
328        """
329        expected_records = []
330        if not silent:
331            for tag, status in record_data:
332                expected_records.append(
333                        self.nodes[tag].get_log_record(status))
334        actual_records = self._fake_host.get_log_records()
335        self.assertEqual(expected_records, actual_records)
336
337
338class VerifyTests(_DependencyNodeTestCase):
339    """
340    Unit tests for `Verifier`.
341
342    The tests in this class test the fundamental behaviors of the
343    `Verifier` class:
344      * Results from the `verify()` method are cached; the method is
345        only called the first time that `_verify_host()` is called.
346      * The `_verify_host()` method makes the expected calls to
347        `Host.record()` for every call to the `verify()` method.
348      * When a dependency fails, the dependent verifier isn't called.
349      * Verifier calls are made in the order required by the DAG.
350
351    The test cases don't use `RepairStrategy` to build DAG structures,
352    but instead rely on custom-built DAGs.
353    """
354
355    def _generate_verify_count(self, verifier):
356        """
357        Iterator to force a standard sequence with calls to `_reverify()`.
358
359        This iterator exists to standardize testing two common
360        assertions:
361          * The side effects from calling `_verify_host()` only
362            happen on the first call to the method, except...
363          * Calling `_reverify()` resets a verifier so that the
364            next call to `_verify_host()` will repeat the side
365            effects.
366
367        The iterator is meant to be used like this:
368
369            for count in self._generate_verify_cases(verifier):
370                # run a verifier._verify_host() test case
371                self.assertEqual(verifier.verify_count, count)
372                self._check_log_records(silent, ... expected records ... )
373
374        The code above will run the `_verify_host()` test case twice,
375        then call `_reverify()` to clear cached results, then re-run
376        the test case two more times.
377
378        @param verifier   The verifier to be tested and reverified.
379        @yields Each iteration yields the number of times `_reverify()`
380                has been called.
381        """
382        for i in range(1, 3):
383            for _ in range(0, 2):
384                yield i
385            verifier._reverify()
386            self._fake_host.reset_log_records()
387
388
389    def test_success(self):
390        """
391        Test proper handling of a successful verification.
392
393        Construct and call a simple, single-node verification that will
394        pass.  Assert the following:
395          * The `verify()` method is called once.
396          * The expected 'GOOD' record is logged via `Host.record()`.
397          * If `_verify_host()` is called more than once, there are no
398            visible side-effects after the first call.
399          * Calling `_reverify()` clears all cached results.
400        """
401        for silent in self._generate_silent():
402            verifier = self._make_verifier(0, 'pass', [])
403            for count in self._generate_verify_count(verifier):
404                verifier._verify_host(self._fake_host, silent)
405                self.assertEqual(verifier.verify_count, count)
406                self._check_log_records(silent, ('pass', 'GOOD'))
407
408
409    def test_fail(self):
410        """
411        Test proper handling of verification failure.
412
413        Construct and call a simple, single-node verification that will
414        fail.  Assert the following:
415          * The failure is reported with the actual exception raised
416            by the verifier.
417          * The `verify()` method is called once.
418          * The expected 'FAIL' record is logged via `Host.record()`.
419          * If `_verify_host()` is called more than once, there are no
420            visible side-effects after the first call.
421          * Calling `_reverify()` clears all cached results.
422        """
423        for silent in self._generate_silent():
424            verifier = self._make_verifier(1, 'fail', [])
425            for count in self._generate_verify_count(verifier):
426                with self.assertRaises(hosts.AutoservVerifyError) as e:
427                    verifier._verify_host(self._fake_host, silent)
428                self.assertEqual(verifier.verify_count, count)
429                self.assertEqual(verifier.message, str(e.exception))
430                self._check_log_records(silent, ('fail', 'FAIL'))
431
432
433    def test_dependency_success(self):
434        """
435        Test proper handling of dependencies that succeed.
436
437        Construct and call a two-node verification with one node
438        dependent on the other, where both nodes will pass.  Assert the
439        following:
440          * The `verify()` method for both nodes is called once.
441          * The expected 'GOOD' record is logged via `Host.record()`
442            for both nodes.
443          * If `_verify_host()` is called more than once, there are no
444            visible side-effects after the first call.
445          * Calling `_reverify()` clears all cached results.
446        """
447        for silent in self._generate_silent():
448            child = self._make_verifier(0, 'pass', [])
449            parent = self._make_verifier(0, 'parent', [child])
450            for count in self._generate_verify_count(parent):
451                parent._verify_host(self._fake_host, silent)
452                self.assertEqual(parent.verify_count, count)
453                self.assertEqual(child.verify_count, count)
454                self._check_log_records(silent,
455                                        ('pass', 'GOOD'),
456                                        ('parent', 'GOOD'))
457
458
459    def test_dependency_fail(self):
460        """
461        Test proper handling of dependencies that fail.
462
463        Construct and call a two-node verification with one node
464        dependent on the other, where the dependency will fail.  Assert
465        the following:
466          * The verification exception is `AutoservVerifyDependencyError`,
467            and the exception argument is the description of the failed
468            node.
469          * The `verify()` method for the failing node is called once,
470            and for the other node, not at all.
471          * The expected 'FAIL' record is logged via `Host.record()`
472            for the single failed node.
473          * If `_verify_host()` is called more than once, there are no
474            visible side-effects after the first call.
475          * Calling `_reverify()` clears all cached results.
476        """
477        for silent in self._generate_silent():
478            child = self._make_verifier(1, 'fail', [])
479            parent = self._make_verifier(0, 'parent', [child])
480            failures = self._make_expected_failures(child)
481            for count in self._generate_verify_count(parent):
482                expected_exception = hosts.AutoservVerifyDependencyError
483                with self.assertRaises(expected_exception) as e:
484                    parent._verify_host(self._fake_host, silent)
485                self.assertEqual(e.exception.failures, failures)
486                self.assertEqual(child.verify_count, count)
487                self.assertEqual(parent.verify_count, 0)
488                self._check_log_records(silent, ('fail', 'FAIL'))
489
490
491    def test_two_dependencies_pass(self):
492        """
493        Test proper handling with two passing dependencies.
494
495        Construct and call a three-node verification with one node
496        dependent on the other two, where all nodes will pass.  Assert
497        the following:
498          * The `verify()` method for all nodes is called once.
499          * The expected 'GOOD' records are logged via `Host.record()`
500            for all three nodes.
501          * If `_verify_host()` is called more than once, there are no
502            visible side-effects after the first call.
503          * Calling `_reverify()` clears all cached results.
504        """
505        for silent in self._generate_silent():
506            left = self._make_verifier(0, 'left', [])
507            right = self._make_verifier(0, 'right', [])
508            top = self._make_verifier(0, 'top', [left, right])
509            for count in self._generate_verify_count(top):
510                top._verify_host(self._fake_host, silent)
511                self.assertEqual(top.verify_count, count)
512                self.assertEqual(left.verify_count, count)
513                self.assertEqual(right.verify_count, count)
514                self._check_log_records(silent,
515                                        ('left', 'GOOD'),
516                                        ('right', 'GOOD'),
517                                        ('top', 'GOOD'))
518
519
520    def test_two_dependencies_fail(self):
521        """
522        Test proper handling with two failing dependencies.
523
524        Construct and call a three-node verification with one node
525        dependent on the other two, where both dependencies will fail.
526        Assert the following:
527          * The verification exception is `AutoservVerifyDependencyError`,
528            and the exception argument has the descriptions of both the
529            failed nodes.
530          * The `verify()` method for each failing node is called once,
531            and for the parent node not at all.
532          * The expected 'FAIL' records are logged via `Host.record()`
533            for the failing nodes.
534          * If `_verify_host()` is called more than once, there are no
535            visible side-effects after the first call.
536          * Calling `_reverify()` clears all cached results.
537        """
538        for silent in self._generate_silent():
539            left = self._make_verifier(1, 'left', [])
540            right = self._make_verifier(1, 'right', [])
541            top = self._make_verifier(0, 'top', [left, right])
542            failures = self._make_expected_failures(left, right)
543            for count in self._generate_verify_count(top):
544                expected_exception = hosts.AutoservVerifyDependencyError
545                with self.assertRaises(expected_exception) as e:
546                    top._verify_host(self._fake_host, silent)
547                self.assertEqual(e.exception.failures, failures)
548                self.assertEqual(top.verify_count, 0)
549                self.assertEqual(left.verify_count, count)
550                self.assertEqual(right.verify_count, count)
551                self._check_log_records(silent,
552                                        ('left', 'FAIL'),
553                                        ('right', 'FAIL'))
554
555
556    def test_two_dependencies_mixed(self):
557        """
558        Test proper handling with mixed dependencies.
559
560        Construct and call a three-node verification with one node
561        dependent on the other two, where one dependency will pass,
562        and one will fail.  Assert the following:
563          * The verification exception is `AutoservVerifyDependencyError`,
564            and the exception argument has the descriptions of the
565            single failed node.
566          * The `verify()` method for each dependency is called once,
567            and for the parent node not at all.
568          * The expected 'GOOD' and 'FAIL' records are logged via
569            `Host.record()` for the dependencies.
570          * If `_verify_host()` is called more than once, there are no
571            visible side-effects after the first call.
572          * Calling `_reverify()` clears all cached results.
573        """
574        for silent in self._generate_silent():
575            left = self._make_verifier(1, 'left', [])
576            right = self._make_verifier(0, 'right', [])
577            top = self._make_verifier(0, 'top', [left, right])
578            failures = self._make_expected_failures(left)
579            for count in self._generate_verify_count(top):
580                expected_exception = hosts.AutoservVerifyDependencyError
581                with self.assertRaises(expected_exception) as e:
582                    top._verify_host(self._fake_host, silent)
583                self.assertEqual(e.exception.failures, failures)
584                self.assertEqual(top.verify_count, 0)
585                self.assertEqual(left.verify_count, count)
586                self.assertEqual(right.verify_count, count)
587                self._check_log_records(silent,
588                                        ('left', 'FAIL'),
589                                        ('right', 'GOOD'))
590
591
592    def test_diamond_pass(self):
593        """
594        Test a "diamond" structure DAG with all nodes passing.
595
596        Construct and call a "diamond" structure DAG where all nodes
597        will pass:
598
599                TOP
600               /   \
601            LEFT   RIGHT
602               \   /
603               BOTTOM
604
605       Assert the following:
606          * The `verify()` method for all nodes is called once.
607          * The expected 'GOOD' records are logged via `Host.record()`
608            for all nodes.
609          * If `_verify_host()` is called more than once, there are no
610            visible side-effects after the first call.
611          * Calling `_reverify()` clears all cached results.
612        """
613        for silent in self._generate_silent():
614            bottom = self._make_verifier(0, 'bottom', [])
615            left = self._make_verifier(0, 'left', [bottom])
616            right = self._make_verifier(0, 'right', [bottom])
617            top = self._make_verifier(0, 'top', [left, right])
618            for count in self._generate_verify_count(top):
619                top._verify_host(self._fake_host, silent)
620                self.assertEqual(top.verify_count, count)
621                self.assertEqual(left.verify_count, count)
622                self.assertEqual(right.verify_count, count)
623                self.assertEqual(bottom.verify_count, count)
624                self._check_log_records(silent,
625                                        ('bottom', 'GOOD'),
626                                        ('left', 'GOOD'),
627                                        ('right', 'GOOD'),
628                                        ('top', 'GOOD'))
629
630
631    def test_diamond_fail(self):
632        """
633        Test a "diamond" structure DAG with the bottom node failing.
634
635        Construct and call a "diamond" structure DAG where the bottom
636        node will fail:
637
638                TOP
639               /   \
640            LEFT   RIGHT
641               \   /
642               BOTTOM
643
644        Assert the following:
645          * The verification exception is `AutoservVerifyDependencyError`,
646            and the exception argument has the description of the
647            "bottom" node.
648          * The `verify()` method for the "bottom" node is called once,
649            and for the other nodes not at all.
650          * The expected 'FAIL' record is logged via `Host.record()`
651            for the "bottom" node.
652          * If `_verify_host()` is called more than once, there are no
653            visible side-effects after the first call.
654          * Calling `_reverify()` clears all cached results.
655        """
656        for silent in self._generate_silent():
657            bottom = self._make_verifier(1, 'bottom', [])
658            left = self._make_verifier(0, 'left', [bottom])
659            right = self._make_verifier(0, 'right', [bottom])
660            top = self._make_verifier(0, 'top', [left, right])
661            failures = self._make_expected_failures(bottom)
662            for count in self._generate_verify_count(top):
663                expected_exception = hosts.AutoservVerifyDependencyError
664                with self.assertRaises(expected_exception) as e:
665                    top._verify_host(self._fake_host, silent)
666                self.assertEqual(e.exception.failures, failures)
667                self.assertEqual(top.verify_count, 0)
668                self.assertEqual(left.verify_count, 0)
669                self.assertEqual(right.verify_count, 0)
670                self.assertEqual(bottom.verify_count, count)
671                self._check_log_records(silent, ('bottom', 'FAIL'))
672
673
674class RepairActionTests(_DependencyNodeTestCase):
675    """
676    Unit tests for `RepairAction`.
677
678    The tests in this class test the fundamental behaviors of the
679    `RepairAction` class:
680      * Repair doesn't run unless all dependencies pass.
681      * Repair doesn't run unless at least one trigger fails.
682      * The `_repair_host()` method makes the expected calls to
683        `Host.record()` for every call to the `repair()` method.
684
685    The test cases don't use `RepairStrategy` to build repair
686    graphs, but instead rely on custom-built structures.
687    """
688
689    def test_repair_not_triggered(self):
690        """
691        Test a repair that doesn't trigger.
692
693        Construct and call a repair action with a verification trigger
694        that passes.  Assert the following:
695          * The `verify()` method for the trigger is called.
696          * The `repair()` method is not called.
697          * The verifier logs the expected 'GOOD' message with
698            `Host.record()`.
699          * The repair action logs no messages with `Host.record()`.
700        """
701        for silent in self._generate_silent():
702            verifier = self._make_verifier(0, 'check', [])
703            repair_action = self._make_repair_action(True, 'unneeded',
704                                                     [], [verifier])
705            repair_action._repair_host(self._fake_host, silent)
706            self.assertEqual(verifier.verify_count, 1)
707            self.assertEqual(repair_action.repair_count, 0)
708            self._check_log_records(silent, ('check', 'GOOD'))
709
710
711    def test_repair_fails(self):
712        """
713        Test a repair that triggers and fails.
714
715        Construct and call a repair action with a verification trigger
716        that fails.  The repair fails by raising `_StubRepairFailure`.
717        Assert the following:
718          * The repair action fails with the `_StubRepairFailure` raised
719            by `repair()`.
720          * The `verify()` method for the trigger is called once.
721          * The `repair()` method is called once.
722          * The expected 'START', 'FAIL', and 'END FAIL' messages are
723            logged with `Host.record()` for the failed verifier and the
724            failed repair.
725        """
726        for silent in self._generate_silent():
727            verifier = self._make_verifier(1, 'fail', [])
728            repair_action = self._make_repair_action(False, 'nofix',
729                                                     [], [verifier])
730            with self.assertRaises(_StubRepairFailure) as e:
731                repair_action._repair_host(self._fake_host, silent)
732            self.assertEqual(repair_action.message, str(e.exception))
733            self.assertEqual(verifier.verify_count, 1)
734            self.assertEqual(repair_action.repair_count, 1)
735            self._check_log_records(silent,
736                                    ('fail', 'FAIL'),
737                                    ('nofix', 'START'),
738                                    ('nofix', 'FAIL'),
739                                    ('nofix', 'END FAIL'))
740
741
742    def test_repair_success(self):
743        """
744        Test a repair that fixes its trigger.
745
746        Construct and call a repair action that raises no exceptions,
747        using a repair trigger that fails first, then passes after
748        repair.  Assert the following:
749          * The `repair()` method is called once.
750          * The trigger's `verify()` method is called twice.
751          * The expected 'START', 'FAIL', 'GOOD', and 'END GOOD'
752            messages are logged with `Host.record()` for the verifier
753            and the repair.
754        """
755        for silent in self._generate_silent():
756            verifier = self._make_verifier(1, 'fail', [])
757            repair_action = self._make_repair_action(True, 'fix',
758                                                     [], [verifier])
759            repair_action._repair_host(self._fake_host, silent)
760            self.assertEqual(repair_action.repair_count, 1)
761            self.assertEqual(verifier.verify_count, 2)
762            self._check_log_records(silent,
763                                    ('fail', 'FAIL'),
764                                    ('fix', 'START'),
765                                    ('fail', 'GOOD'),
766                                    ('fix', 'END GOOD'))
767
768
769    def test_repair_noop(self):
770        """
771        Test a repair that doesn't fix a failing trigger.
772
773        Construct and call a repair action with a trigger that fails.
774        The repair action raises no exceptions, and after repair, the
775        trigger still fails.  Assert the following:
776          * The `_repair_host()` call fails with `AutoservRepairError`.
777          * The `repair()` method is called once.
778          * The trigger's `verify()` method is called twice.
779          * The expected 'START', 'FAIL', and 'END FAIL' messages are
780            logged with `Host.record()` for the verifier and the repair.
781        """
782        for silent in self._generate_silent():
783            verifier = self._make_verifier(2, 'fail', [])
784            repair_action = self._make_repair_action(True, 'nofix',
785                                                     [], [verifier])
786            with self.assertRaises(hosts.AutoservRepairError) as e:
787                repair_action._repair_host(self._fake_host, silent)
788            self.assertEqual(repair_action.repair_count, 1)
789            self.assertEqual(verifier.verify_count, 2)
790            self._check_log_records(silent,
791                                    ('fail', 'FAIL'),
792                                    ('nofix', 'START'),
793                                    ('fail', 'FAIL'),
794                                    ('nofix', 'END FAIL'))
795
796
797    def test_dependency_pass(self):
798        """
799        Test proper handling of repair dependencies that pass.
800
801        Construct and call a repair action with a dependency and a
802        trigger.  The dependency will pass and the trigger will fail and
803        be repaired.  Assert the following:
804          * Repair passes.
805          * The `verify()` method for the dependency is called once.
806          * The `verify()` method for the trigger is called twice.
807          * The `repair()` method is called once.
808          * The expected records are logged via `Host.record()`
809            for the successful dependency, the failed trigger, and
810            the successful repair.
811        """
812        for silent in self._generate_silent():
813            dep = self._make_verifier(0, 'dep', [])
814            trigger = self._make_verifier(1, 'trig', [])
815            repair = self._make_repair_action(True, 'fixit',
816                                              [dep], [trigger])
817            repair._repair_host(self._fake_host, silent)
818            self.assertEqual(dep.verify_count, 1)
819            self.assertEqual(trigger.verify_count, 2)
820            self.assertEqual(repair.repair_count, 1)
821            self._check_log_records(silent,
822                                    ('dep', 'GOOD'),
823                                    ('trig', 'FAIL'),
824                                    ('fixit', 'START'),
825                                    ('trig', 'GOOD'),
826                                    ('fixit', 'END GOOD'))
827
828
829    def test_dependency_fail(self):
830        """
831        Test proper handling of repair dependencies that fail.
832
833        Construct and call a repair action with a dependency and a
834        trigger, both of which fail.  Assert the following:
835          * Repair fails with `AutoservVerifyDependencyError`,
836            and the exception argument is the description of the failed
837            dependency.
838          * The `verify()` method for the failing dependency is called
839            once.
840          * The trigger and the repair action aren't invoked at all.
841          * The expected 'FAIL' record is logged via `Host.record()`
842            for the single failed dependency.
843        """
844        for silent in self._generate_silent():
845            dep = self._make_verifier(1, 'dep', [])
846            trigger = self._make_verifier(1, 'trig', [])
847            repair = self._make_repair_action(True, 'fixit',
848                                              [dep], [trigger])
849            expected_exception = hosts.AutoservVerifyDependencyError
850            with self.assertRaises(expected_exception) as e:
851                repair._repair_host(self._fake_host, silent)
852            self.assertEqual(e.exception.failures,
853                             self._make_expected_failures(dep))
854            self.assertEqual(dep.verify_count, 1)
855            self.assertEqual(trigger.verify_count, 0)
856            self.assertEqual(repair.repair_count, 0)
857            self._check_log_records(silent, ('dep', 'FAIL'))
858
859
860class _RepairStrategyTestCase(_DependencyNodeTestCase):
861    """Shared base class for testing `RepairStrategy` methods."""
862
863    def _make_verify_data(self, *input_data):
864        """
865        Create `verify_data` for the `RepairStrategy` constructor.
866
867        `RepairStrategy` expects `verify_data` as a list of tuples
868        of the form `(constructor, tag, deps)`.  Each item in
869        `input_data` is a tuple of the form `(tag, count, deps)` that
870        creates one entry in the returned list of `verify_data` tuples
871        as follows:
872          * `count` is used to create a constructor function that calls
873            `self._make_verifier()` with that value plus plus the
874            arguments provided by the `RepairStrategy` constructor.
875          * `tag` and `deps` will be passed as-is to the `RepairStrategy`
876            constructor.
877
878        @param input_data   A list of tuples, each representing one
879                            tuple in the `verify_data` list.
880        @return   A list suitable to be the `verify_data` parameter for
881                  the `RepairStrategy` constructor.
882        """
883        strategy_data = []
884        for tag, count, deps in input_data:
885            construct = functools.partial(self._make_verifier, count)
886            strategy_data.append((construct, tag, deps))
887        return strategy_data
888
889
890    def _make_repair_data(self, *input_data):
891        """
892        Create `repair_data` for the `RepairStrategy` constructor.
893
894        `RepairStrategy` expects `repair_data` as a list of tuples
895        of the form `(constructor, tag, deps, triggers)`.  Each item in
896        `input_data` is a tuple of the form `(tag, success, deps, triggers)`
897        that creates one entry in the returned list of `repair_data`
898        tuples as follows:
899          * `success` is used to create a constructor function that calls
900            `self._make_verifier()` with that value plus plus the
901            arguments provided by the `RepairStrategy` constructor.
902          * `tag`, `deps`, and `triggers` will be passed as-is to the
903            `RepairStrategy` constructor.
904
905        @param input_data   A list of tuples, each representing one
906                            tuple in the `repair_data` list.
907        @return   A list suitable to be the `repair_data` parameter for
908                  the `RepairStrategy` constructor.
909        """
910        strategy_data = []
911        for tag, success, deps, triggers in input_data:
912            construct = functools.partial(self._make_repair_action, success)
913            strategy_data.append((construct, tag, deps, triggers))
914        return strategy_data
915
916
917    def _make_strategy(self, verify_input, repair_input):
918        """
919        Create a `RepairStrategy` from the given arguments.
920
921        @param verify_input   As for `input_data` in
922                              `_make_verify_data()`.
923        @param repair_input   As for `input_data` in
924                              `_make_repair_data()`.
925        """
926        verify_data = self._make_verify_data(*verify_input)
927        repair_data = self._make_repair_data(*repair_input)
928        return hosts.RepairStrategy(verify_data, repair_data)
929
930    def _check_silent_records(self, silent):
931        """
932        Check that logging honored the `silent` parameter.
933
934        Asserts that logging with `Host.record()` occurred (or did not
935        occur) in accordance with the value of `silent`.
936
937        This method only asserts the presence or absence of log records.
938        Coverage for the contents of the log records is handled in other
939        test cases.
940
941        @param silent   When true, there should be no log records;
942                        otherwise there should be records present.
943        """
944        log_records = self._fake_host.get_log_records()
945        if silent:
946            self.assertEqual(log_records, [])
947        else:
948            self.assertNotEqual(log_records, [])
949
950
951class RepairStrategyVerifyTests(_RepairStrategyTestCase):
952    """
953    Unit tests for `RepairStrategy.verify()`.
954
955    These unit tests focus on verifying that the `RepairStrategy`
956    constructor creates the expected DAG structure from given
957    `verify_data`.  Functional testing here is mainly confined to
958    asserting that `RepairStrategy.verify()` properly distinguishes
959    success from failure.  Testing the behavior of specific DAG
960    structures is left to tests in `VerifyTests`.
961    """
962
963    def test_single_node(self):
964        """
965        Test construction of a single-node verification DAG.
966
967        Assert that the structure looks like this:
968
969            Root Node -> Main Node
970        """
971        verify_data = self._make_verify_data(('main', 0, ()))
972        strategy = hosts.RepairStrategy(verify_data, [])
973        verifier = self.nodes['main']
974        self.assertEqual(
975                strategy._verify_root._dependency_list,
976                [verifier])
977        self.assertEqual(verifier._dependency_list, [])
978
979
980    def test_single_dependency(self):
981        """
982        Test construction of a two-node dependency chain.
983
984        Assert that the structure looks like this:
985
986            Root Node -> Parent Node -> Child Node
987        """
988        verify_data = self._make_verify_data(
989                ('child', 0, ()),
990                ('parent', 0, ('child',)))
991        strategy = hosts.RepairStrategy(verify_data, [])
992        parent = self.nodes['parent']
993        child = self.nodes['child']
994        self.assertEqual(
995                strategy._verify_root._dependency_list, [parent])
996        self.assertEqual(
997                parent._dependency_list, [child])
998        self.assertEqual(
999                child._dependency_list, [])
1000
1001
1002    def test_two_nodes_and_dependency(self):
1003        """
1004        Test construction of two nodes with a shared dependency.
1005
1006        Assert that the structure looks like this:
1007
1008            Root Node -> Left Node ---\
1009                      \                -> Bottom Node
1010                        -> Right Node /
1011        """
1012        verify_data = self._make_verify_data(
1013                ('bottom', 0, ()),
1014                ('left', 0, ('bottom',)),
1015                ('right', 0, ('bottom',)))
1016        strategy = hosts.RepairStrategy(verify_data, [])
1017        bottom = self.nodes['bottom']
1018        left = self.nodes['left']
1019        right = self.nodes['right']
1020        self.assertEqual(
1021                strategy._verify_root._dependency_list,
1022                [left, right])
1023        self.assertEqual(left._dependency_list, [bottom])
1024        self.assertEqual(right._dependency_list, [bottom])
1025        self.assertEqual(bottom._dependency_list, [])
1026
1027
1028    def test_three_nodes(self):
1029        """
1030        Test construction of three nodes with no dependencies.
1031
1032        Assert that the structure looks like this:
1033
1034                       -> Node One
1035                      /
1036            Root Node -> Node Two
1037                      \
1038                       -> Node Three
1039
1040        N.B.  This test exists to enforce ordering expectations of
1041        root-level DAG nodes.  Three nodes are used to make it unlikely
1042        that randomly ordered roots will match expectations.
1043        """
1044        verify_data = self._make_verify_data(
1045                ('one', 0, ()),
1046                ('two', 0, ()),
1047                ('three', 0, ()))
1048        strategy = hosts.RepairStrategy(verify_data, [])
1049        one = self.nodes['one']
1050        two = self.nodes['two']
1051        three = self.nodes['three']
1052        self.assertEqual(
1053                strategy._verify_root._dependency_list,
1054                [one, two, three])
1055        self.assertEqual(one._dependency_list, [])
1056        self.assertEqual(two._dependency_list, [])
1057        self.assertEqual(three._dependency_list, [])
1058
1059
1060    def test_verify(self):
1061        """
1062        Test behavior of the `verify()` method.
1063
1064        Build a `RepairStrategy` with a single verifier.  Assert the
1065        following:
1066          * If the verifier passes, `verify()` passes.
1067          * If the verifier fails, `verify()` fails.
1068          * The verifier is reinvoked with every call to `verify()`;
1069            cached results are not re-used.
1070        """
1071        verify_data = self._make_verify_data(('tester', 0, ()))
1072        strategy = hosts.RepairStrategy(verify_data, [])
1073        verifier = self.nodes['tester']
1074        count = 0
1075        for silent in self._generate_silent():
1076            for i in range(0, 2):
1077                for j in range(0, 2):
1078                    strategy.verify(self._fake_host, silent)
1079                    self._check_silent_records(silent)
1080                    count += 1
1081                    self.assertEqual(verifier.verify_count, count)
1082                verifier.unrepair()
1083                for j in range(0, 2):
1084                    with self.assertRaises(Exception) as e:
1085                        strategy.verify(self._fake_host, silent)
1086                    self._check_silent_records(silent)
1087                    count += 1
1088                    self.assertEqual(verifier.verify_count, count)
1089                verifier.try_repair()
1090
1091
1092class RepairStrategyRepairTests(_RepairStrategyTestCase):
1093    """
1094    Unit tests for `RepairStrategy.repair()`.
1095
1096    These unit tests focus on verifying that the `RepairStrategy`
1097    constructor creates the expected repair list from given
1098    `repair_data`.  Functional testing here is confined to asserting
1099    that `RepairStrategy.repair()` properly distinguishes success from
1100    failure.  Testing the behavior of specific repair structures is left
1101    to tests in `RepairActionTests`.
1102    """
1103
1104    def _check_common_trigger(self, strategy, repair_tags, triggers):
1105        self.assertEqual(strategy._repair_actions,
1106                         [self.nodes[tag] for tag in repair_tags])
1107        for tag in repair_tags:
1108            self.assertEqual(self.nodes[tag]._trigger_list,
1109                             triggers)
1110            self.assertEqual(self.nodes[tag]._dependency_list, [])
1111
1112
1113    def test_single_repair_with_trigger(self):
1114        """
1115        Test constructing a strategy with a single repair trigger.
1116
1117        Build a `RepairStrategy` with a single repair action and a
1118        single trigger.  Assert that the trigger graph looks like this:
1119
1120            Repair -> Trigger
1121
1122        Assert that there are no repair dependencies.
1123        """
1124        verify_input = (('base', 0, ()),)
1125        repair_input = (('fixit', True, (), ('base',)),)
1126        strategy = self._make_strategy(verify_input, repair_input)
1127        self._check_common_trigger(strategy,
1128                                   ['fixit'],
1129                                   [self.nodes['base']])
1130
1131
1132    def test_repair_with_root_trigger(self):
1133        """
1134        Test construction of a repair triggering on the root verifier.
1135
1136        Build a `RepairStrategy` with a single repair action that
1137        triggers on the root verifier.  Assert that the trigger graph
1138        looks like this:
1139
1140            Repair -> Root Verifier
1141
1142        Assert that there are no repair dependencies.
1143        """
1144        root_tag = hosts.RepairStrategy.ROOT_TAG
1145        repair_input = (('fixit', True, (), (root_tag,)),)
1146        strategy = self._make_strategy([], repair_input)
1147        self._check_common_trigger(strategy,
1148                                   ['fixit'],
1149                                   [strategy._verify_root])
1150
1151
1152    def test_three_repairs(self):
1153        """
1154        Test constructing a strategy with three repair actions.
1155
1156        Build a `RepairStrategy` with a three repair actions sharing a
1157        single trigger.  Assert that the trigger graph looks like this:
1158
1159            Repair A -> Trigger
1160            Repair B -> Trigger
1161            Repair C -> Trigger
1162
1163        Assert that there are no repair dependencies.
1164
1165        N.B.  This test exists to enforce ordering expectations of
1166        repair nodes.  Three nodes are used to make it unlikely that
1167        randomly ordered actions will match expectations.
1168        """
1169        verify_input = (('base', 0, ()),)
1170        repair_tags = ['a', 'b', 'c']
1171        repair_input = (
1172            (tag, True, (), ('base',)) for tag in repair_tags)
1173        strategy = self._make_strategy(verify_input, repair_input)
1174        self._check_common_trigger(strategy,
1175                                   repair_tags,
1176                                   [self.nodes['base']])
1177
1178
1179    def test_repair_dependency(self):
1180        """
1181        Test construction of a repair with a dependency.
1182
1183        Build a `RepairStrategy` with a single repair action that
1184        depends on a single verifier.  Assert that the dependency graph
1185        looks like this:
1186
1187            Repair -> Verifier
1188
1189        Assert that there are no repair triggers.
1190        """
1191        verify_input = (('base', 0, ()),)
1192        repair_input = (('fixit', True, ('base',), ()),)
1193        strategy = self._make_strategy(verify_input, repair_input)
1194        self.assertEqual(strategy._repair_actions,
1195                         [self.nodes['fixit']])
1196        self.assertEqual(self.nodes['fixit']._trigger_list, [])
1197        self.assertEqual(self.nodes['fixit']._dependency_list,
1198                         [self.nodes['base']])
1199
1200
1201    def _check_repair_failure(self, strategy, silent):
1202        """
1203        Check the effects of a call to `repair()` that fails.
1204
1205        For the given strategy object, call the `repair()` method; the
1206        call is expected to fail and all repair actions are expected to
1207        trigger.
1208
1209        Assert the following:
1210          * The call raises an exception.
1211          * For each repair action in the strategy, its `repair()`
1212            method is called exactly once.
1213
1214        @param strategy   The strategy to be tested.
1215        """
1216        action_counts = [(a, a.repair_count)
1217                                 for a in strategy._repair_actions]
1218        with self.assertRaises(Exception) as e:
1219            strategy.repair(self._fake_host, silent)
1220        self._check_silent_records(silent)
1221        for action, count in action_counts:
1222              self.assertEqual(action.repair_count, count + 1)
1223
1224
1225    def _check_repair_success(self, strategy, silent):
1226        """
1227        Check the effects of a call to `repair()` that succeeds.
1228
1229        For the given strategy object, call the `repair()` method; the
1230        call is expected to succeed without raising an exception and all
1231        repair actions are expected to trigger.
1232
1233        Assert that for each repair action in the strategy, its
1234        `repair()` method is called exactly once.
1235
1236        @param strategy   The strategy to be tested.
1237        """
1238        action_counts = [(a, a.repair_count)
1239                                 for a in strategy._repair_actions]
1240        strategy.repair(self._fake_host, silent)
1241        self._check_silent_records(silent)
1242        for action, count in action_counts:
1243              self.assertEqual(action.repair_count, count + 1)
1244
1245
1246    def test_repair(self):
1247        """
1248        Test behavior of the `repair()` method.
1249
1250        Build a `RepairStrategy` with two repair actions each depending
1251        on its own verifier.  Set up calls to `repair()` for each of
1252        the following conditions:
1253          * Both repair actions trigger and fail.
1254          * Both repair actions trigger and succeed.
1255          * Both repair actions trigger; the first one fails, but the
1256            second one succeeds.
1257          * Both repair actions trigger; the first one succeeds, but the
1258            second one fails.
1259
1260        Assert the following:
1261          * When both repair actions succeed, `repair()` succeeds.
1262          * When either repair action fails, `repair()` fails.
1263          * After each call to the strategy's `repair()` method, each
1264            repair action triggered exactly once.
1265        """
1266        verify_input = (('a', 2, ()), ('b', 2, ()))
1267        repair_input = (('afix', True, (), ('a',)),
1268                        ('bfix', True, (), ('b',)))
1269        strategy = self._make_strategy(verify_input, repair_input)
1270
1271        for silent in self._generate_silent():
1272            # call where both 'afix' and 'bfix' fail
1273            self._check_repair_failure(strategy, silent)
1274            # repair counts are now 1 for both verifiers
1275
1276            # call where both 'afix' and 'bfix' succeed
1277            self._check_repair_success(strategy, silent)
1278            # repair counts are now 0 for both verifiers
1279
1280            # call where 'afix' fails and 'bfix' succeeds
1281            for tag in ['a', 'a', 'b']:
1282                self.nodes[tag].unrepair()
1283            self._check_repair_failure(strategy, silent)
1284            # 'a' repair count is 1; 'b' count is 0
1285
1286            # call where 'afix' succeeds and 'bfix' fails
1287            for tag in ['b', 'b']:
1288                self.nodes[tag].unrepair()
1289            self._check_repair_failure(strategy, silent)
1290            # 'a' repair count is 0; 'b' count is 1
1291
1292            for tag in ['a', 'a', 'b']:
1293                self.nodes[tag].unrepair()
1294            # repair counts are now 2 for both verifiers
1295
1296
1297if __name__ == '__main__':
1298    unittest.main()
1299