• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1# Copyright 2017 The Chromium OS Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5from __future__ import absolute_import
6from __future__ import print_function
7
8import logging
9
10import common
11from autotest_lib.server.hosts import host_info
12from chromite.lib import metrics
13
14
15_METRICS_PREFIX = 'chromeos/autotest/autoserv/host_info/shadowing_store/'
16_REFRESH_METRIC_NAME = _METRICS_PREFIX + 'refresh_count'
17_COMMIT_METRIC_NAME = _METRICS_PREFIX + 'commit_count'
18
19
20logger = logging.getLogger(__file__)
21
22class ShadowingStore(host_info.CachingHostInfoStore):
23    """A composite CachingHostInfoStore that maintains a main and shadow store.
24
25    ShadowingStore accepts two CachingHostInfoStore objects - primary_store and
26    shadow_store. All refresh/commit operations are serviced through
27    primary_store.  In addition, shadow_store is updated and compared with this
28    information, leaving breadcrumbs when the two differ. Any errors in
29    shadow_store operations are logged and ignored so as to not affect the user.
30
31    This is a transitional CachingHostInfoStore that allows us to continue to
32    use an AfeStore in practice, but also create a backing FileStore so that we
33    can validate the use of FileStore in prod.
34    """
35
36    def __init__(self, primary_store, shadow_store,
37                 mismatch_callback=None):
38        """
39        @param primary_store: A CachingHostInfoStore to be used as the primary
40                store.
41        @param shadow_store: A CachingHostInfoStore to be used to shadow the
42                primary store.
43        @param mismatch_callback: A callback used to notify whenever we notice a
44                mismatch between primary_store and shadow_store. The signature
45                of the callback must match:
46                    callback(primary_info, shadow_info)
47                where primary_info and shadow_info are HostInfo objects obtained
48                from the two stores respectively.
49                Mostly used by unittests. Actual users don't know / nor care
50                that they're using a ShadowingStore.
51        """
52        super(ShadowingStore, self).__init__()
53        self._primary_store = primary_store
54        self._shadow_store = shadow_store
55        self._mismatch_callback = (
56                mismatch_callback if mismatch_callback is not None
57                else _log_info_mismatch)
58        try:
59            self._shadow_store.commit(self._primary_store.get())
60        except host_info.StoreError as e:
61            metrics.Counter(
62                    _METRICS_PREFIX + 'initialization_fail_count').increment()
63            logger.exception(
64                    'Failed to initialize shadow store. '
65                    'Expect primary / shadow desync in the future.')
66
67    def commit_with_substitute(self, info, primary_store=None,
68                               shadow_store=None):
69        """Commit host information using alternative stores.
70
71        This is used to commit using an alternative store implementation
72        to work around some issues (crbug.com/903589).
73
74        Don't set cached_info in this function.
75
76        @param info: A HostInfo object to set.
77        @param primary_store: A CachingHostInfoStore object to commit instead of
78            the original primary_store.
79        @param shadow_store: A CachingHostInfoStore object to commit instead of
80            the original shadow store.
81        """
82        if primary_store is not None:
83            primary_store.commit(info)
84        else:
85            self._commit_to_primary_store(info)
86
87        if shadow_store is not None:
88            shadow_store.commit(info)
89        else:
90            self._commit_to_shadow_store(info)
91
92    def __str__(self):
93        return '%s[%s, %s]' % (type(self).__name__, self._primary_store,
94                               self._shadow_store)
95
96    def _refresh_impl(self):
97        """Obtains HostInfo from the primary and compares against shadow"""
98        primary_info = self._refresh_from_primary_store()
99        try:
100            shadow_info = self._refresh_from_shadow_store()
101        except host_info.StoreError:
102            logger.exception('Shadow refresh failed. '
103                             'Skipping comparison with primary.')
104            return primary_info
105        self._verify_store_infos(primary_info, shadow_info)
106        return primary_info
107
108    def _commit_impl(self, info):
109        """Commits HostInfo to both the primary and shadow store"""
110        self._commit_to_primary_store(info)
111        self._commit_to_shadow_store(info)
112
113    def _commit_to_primary_store(self, info):
114        try:
115            self._primary_store.commit(info)
116        except host_info.StoreError:
117            metrics.Counter(_COMMIT_METRIC_NAME).increment(
118                    fields={'file_commit_result': 'skipped'})
119            raise
120
121    def _commit_to_shadow_store(self, info):
122        try:
123            self._shadow_store.commit(info)
124        except host_info.StoreError:
125            logger.exception(
126                    'shadow commit failed. '
127                    'Expect primary / shadow desync in the future.')
128            metrics.Counter(_COMMIT_METRIC_NAME).increment(
129                    fields={'file_commit_result': 'fail'})
130        else:
131            metrics.Counter(_COMMIT_METRIC_NAME).increment(
132                    fields={'file_commit_result': 'success'})
133
134    def _refresh_from_primary_store(self):
135        try:
136            return self._primary_store.get(force_refresh=True)
137        except host_info.StoreError:
138            metrics.Counter(_REFRESH_METRIC_NAME).increment(
139                    fields={'validation_result': 'skipped'})
140            raise
141
142    def _refresh_from_shadow_store(self):
143        try:
144            return self._shadow_store.get(force_refresh=True)
145        except host_info.StoreError:
146            metrics.Counter(_REFRESH_METRIC_NAME).increment(fields={
147                    'validation_result': 'fail_shadow_store_refresh'})
148            raise
149
150    def _verify_store_infos(self, primary_info, shadow_info):
151        if primary_info == shadow_info:
152            metrics.Counter(_REFRESH_METRIC_NAME).increment(
153                    fields={'validation_result': 'success'})
154        else:
155            self._mismatch_callback(primary_info, shadow_info)
156            metrics.Counter(_REFRESH_METRIC_NAME).increment(
157                    fields={'validation_result': 'fail_mismatch'})
158            self._shadow_store.commit(primary_info)
159
160
161def _log_info_mismatch(primary_info, shadow_info):
162    """Log the two HostInfo instances.
163
164    Used as the default mismatch_callback.
165    """
166    logger.warning('primary / shadow disagree on refresh.')
167    logger.warning('primary: %s', primary_info)
168    logger.warning('shadow: %s', shadow_info)
169