1# Copyright 2017 The Chromium OS Authors. All rights reserved. 2# Use of this source code is governed by a BSD-style license that can be 3# found in the LICENSE file. 4 5from __future__ import absolute_import 6from __future__ import print_function 7 8import logging 9 10import common 11from autotest_lib.server.hosts import host_info 12from chromite.lib import metrics 13 14 15_METRICS_PREFIX = 'chromeos/autotest/autoserv/host_info/shadowing_store/' 16_REFRESH_METRIC_NAME = _METRICS_PREFIX + 'refresh_count' 17_COMMIT_METRIC_NAME = _METRICS_PREFIX + 'commit_count' 18 19 20logger = logging.getLogger(__file__) 21 22class ShadowingStore(host_info.CachingHostInfoStore): 23 """A composite CachingHostInfoStore that maintains a main and shadow store. 24 25 ShadowingStore accepts two CachingHostInfoStore objects - primary_store and 26 shadow_store. All refresh/commit operations are serviced through 27 primary_store. In addition, shadow_store is updated and compared with this 28 information, leaving breadcrumbs when the two differ. Any errors in 29 shadow_store operations are logged and ignored so as to not affect the user. 30 31 This is a transitional CachingHostInfoStore that allows us to continue to 32 use an AfeStore in practice, but also create a backing FileStore so that we 33 can validate the use of FileStore in prod. 34 """ 35 36 def __init__(self, primary_store, shadow_store, 37 mismatch_callback=None): 38 """ 39 @param primary_store: A CachingHostInfoStore to be used as the primary 40 store. 41 @param shadow_store: A CachingHostInfoStore to be used to shadow the 42 primary store. 43 @param mismatch_callback: A callback used to notify whenever we notice a 44 mismatch between primary_store and shadow_store. The signature 45 of the callback must match: 46 callback(primary_info, shadow_info) 47 where primary_info and shadow_info are HostInfo objects obtained 48 from the two stores respectively. 49 Mostly used by unittests. Actual users don't know / nor care 50 that they're using a ShadowingStore. 51 """ 52 super(ShadowingStore, self).__init__() 53 self._primary_store = primary_store 54 self._shadow_store = shadow_store 55 self._mismatch_callback = ( 56 mismatch_callback if mismatch_callback is not None 57 else _log_info_mismatch) 58 try: 59 self._shadow_store.commit(self._primary_store.get()) 60 except host_info.StoreError as e: 61 metrics.Counter( 62 _METRICS_PREFIX + 'initialization_fail_count').increment() 63 logger.exception( 64 'Failed to initialize shadow store. ' 65 'Expect primary / shadow desync in the future.') 66 67 def commit_with_substitute(self, info, primary_store=None, 68 shadow_store=None): 69 """Commit host information using alternative stores. 70 71 This is used to commit using an alternative store implementation 72 to work around some issues (crbug.com/903589). 73 74 Don't set cached_info in this function. 75 76 @param info: A HostInfo object to set. 77 @param primary_store: A CachingHostInfoStore object to commit instead of 78 the original primary_store. 79 @param shadow_store: A CachingHostInfoStore object to commit instead of 80 the original shadow store. 81 """ 82 if primary_store is not None: 83 primary_store.commit(info) 84 else: 85 self._commit_to_primary_store(info) 86 87 if shadow_store is not None: 88 shadow_store.commit(info) 89 else: 90 self._commit_to_shadow_store(info) 91 92 def __str__(self): 93 return '%s[%s, %s]' % (type(self).__name__, self._primary_store, 94 self._shadow_store) 95 96 def _refresh_impl(self): 97 """Obtains HostInfo from the primary and compares against shadow""" 98 primary_info = self._refresh_from_primary_store() 99 try: 100 shadow_info = self._refresh_from_shadow_store() 101 except host_info.StoreError: 102 logger.exception('Shadow refresh failed. ' 103 'Skipping comparison with primary.') 104 return primary_info 105 self._verify_store_infos(primary_info, shadow_info) 106 return primary_info 107 108 def _commit_impl(self, info): 109 """Commits HostInfo to both the primary and shadow store""" 110 self._commit_to_primary_store(info) 111 self._commit_to_shadow_store(info) 112 113 def _commit_to_primary_store(self, info): 114 try: 115 self._primary_store.commit(info) 116 except host_info.StoreError: 117 metrics.Counter(_COMMIT_METRIC_NAME).increment( 118 fields={'file_commit_result': 'skipped'}) 119 raise 120 121 def _commit_to_shadow_store(self, info): 122 try: 123 self._shadow_store.commit(info) 124 except host_info.StoreError: 125 logger.exception( 126 'shadow commit failed. ' 127 'Expect primary / shadow desync in the future.') 128 metrics.Counter(_COMMIT_METRIC_NAME).increment( 129 fields={'file_commit_result': 'fail'}) 130 else: 131 metrics.Counter(_COMMIT_METRIC_NAME).increment( 132 fields={'file_commit_result': 'success'}) 133 134 def _refresh_from_primary_store(self): 135 try: 136 return self._primary_store.get(force_refresh=True) 137 except host_info.StoreError: 138 metrics.Counter(_REFRESH_METRIC_NAME).increment( 139 fields={'validation_result': 'skipped'}) 140 raise 141 142 def _refresh_from_shadow_store(self): 143 try: 144 return self._shadow_store.get(force_refresh=True) 145 except host_info.StoreError: 146 metrics.Counter(_REFRESH_METRIC_NAME).increment(fields={ 147 'validation_result': 'fail_shadow_store_refresh'}) 148 raise 149 150 def _verify_store_infos(self, primary_info, shadow_info): 151 if primary_info == shadow_info: 152 metrics.Counter(_REFRESH_METRIC_NAME).increment( 153 fields={'validation_result': 'success'}) 154 else: 155 self._mismatch_callback(primary_info, shadow_info) 156 metrics.Counter(_REFRESH_METRIC_NAME).increment( 157 fields={'validation_result': 'fail_mismatch'}) 158 self._shadow_store.commit(primary_info) 159 160 161def _log_info_mismatch(primary_info, shadow_info): 162 """Log the two HostInfo instances. 163 164 Used as the default mismatch_callback. 165 """ 166 logger.warning('primary / shadow disagree on refresh.') 167 logger.warning('primary: %s', primary_info) 168 logger.warning('shadow: %s', shadow_info) 169