1# Copyright 2016 The Chromium OS Authors. All rights reserved. 2# Use of this source code is governed by a BSD-style license that can be 3# found in the LICENSE file. 4 5""" 6Repair actions and verifiers relating to CrOS firmware. 7 8This contains the repair actions and verifiers need to find problems 9with the firmware installed on Chrome OS DUTs, and when necessary, to 10fix problems by updating or re-installing the firmware. 11 12The operations in the module support two distinct use cases: 13 * DUTs used for FAFT tests can in some cases have problems with 14 corrupted firmware. The module supplies `FirmwareStatusVerifier` 15 to check for corruption, and supplies `FirmwareRepair` to re-install 16 firmware via servo when needed. 17 * DUTs used for general testing normally should be running a 18 designated "stable" firmware version. This module supplies 19 `FirmwareVersionVerifier` to detect and automatically update 20 firmware that is out-of-date from the designated version. 21 22For purposes of the operations in the module, we distinguish three kinds 23of DUT, based on pool assignments: 24 * DUTs used for general testing. These DUTs automatically check for 25 and install the stable firmware using `FirmwareVersionVerifier`. 26 * DUTs in pools used for FAFT testing. These check for bad firmware 27 builds with `FirmwareStatusVerifier`, and will fix problems using 28 `FirmwareRepair`. These DUTs don't check for or install the 29 stable firmware. 30 * DUTs not in general pools, and not used for FAFT. These DUTs 31 are expected to be managed by separate processes and are excluded 32 from all of the verification and repair code in this module. 33""" 34 35import logging 36import re 37 38import common 39from autotest_lib.client.common_lib import global_config 40from autotest_lib.client.common_lib import hosts 41from autotest_lib.server import afe_utils 42from autotest_lib.site_utils.suite_scheduler import constants 43 44 45# _FIRMWARE_REPAIR_POOLS - The set of pools that should be 46# managed by `FirmwareStatusVerifier` and `FirmwareRepair`. 47# 48_FIRMWARE_REPAIR_POOLS = set( 49 global_config.global_config.get_config_value( 50 'CROS', 51 'pools_support_firmware_repair', 52 type=str).split(',')) 53 54 55# _FIRMWARE_UPDATE_POOLS - The set of pools that should be 56# managed by `FirmwareVersionVerifier`. 57# 58_FIRMWARE_UPDATE_POOLS = set(constants.Pools.MANAGED_POOLS) 59 60 61def _is_firmware_repair_supported(host): 62 """ 63 Check if a host supports firmware repair. 64 65 When this function returns true, the DUT should be managed by 66 `FirmwareStatusVerifier` and `FirmwareRepair`, but not 67 `FirmwareVersionVerifier`. In general, this applies to DUTs 68 used for firmware testing. 69 70 @return A true value if the host should use `FirmwareStatusVerifier` 71 and `FirmwareRepair`; a false value otherwise. 72 """ 73 info = host.host_info_store.get() 74 return bool(info.pools & _FIRMWARE_REPAIR_POOLS) 75 76 77def _is_firmware_update_supported(host): 78 """ 79 Return whether a DUT should be running the standard firmware. 80 81 In the test lab, DUTs used for general testing, (e.g. the `bvt` 82 pool) need their firmware kept up-to-date with 83 `FirmwareVersionVerifier`. However, some pools have alternative 84 policies for firmware management. This returns whether a given DUT 85 should be updated via the standard stable version update, or 86 managed by some other procedure. 87 88 @param host The host to be checked for update policy. 89 @return A true value if the host should use 90 `FirmwareVersionVerifier`; a false value otherwise. 91 """ 92 info = host.host_info_store.get() 93 return bool(info.pools & _FIRMWARE_UPDATE_POOLS) 94 95 96class FirmwareStatusVerifier(hosts.Verifier): 97 """ 98 Verify that a host's firmware is in a good state. 99 100 For DUTs that run firmware tests, it's possible that the firmware 101 on the DUT can get corrupted. This verifier checks whether it 102 appears that firmware should be re-flashed using servo. 103 """ 104 105 def verify(self, host): 106 if not _is_firmware_repair_supported(host): 107 return 108 try: 109 # Read the AP firmware and dump the sections that we're 110 # interested in. 111 cmd = ('mkdir /tmp/verify_firmware; ' 112 'cd /tmp/verify_firmware; ' 113 'for section in VBLOCK_A VBLOCK_B FW_MAIN_A FW_MAIN_B; ' 114 'do flashrom -r image.bin -i $section:$section; ' 115 'done') 116 host.run(cmd) 117 118 # Verify the firmware blocks A and B. 119 cmd = ('vbutil_firmware --verify /tmp/verify_firmware/VBLOCK_%c' 120 ' --signpubkey /usr/share/vboot/devkeys/root_key.vbpubk' 121 ' --fv /tmp/verify_firmware/FW_MAIN_%c') 122 for c in ('A', 'B'): 123 rv = host.run(cmd % (c, c), ignore_status=True) 124 if rv.exit_status: 125 raise hosts.AutoservVerifyError( 126 'Firmware %c is in a bad state.' % c) 127 finally: 128 # Remove the temporary files. 129 host.run('rm -rf /tmp/verify_firmware') 130 131 @property 132 def description(self): 133 return 'Firmware on this DUT is clean' 134 135 136class FirmwareRepair(hosts.RepairAction): 137 """ 138 Reinstall the firmware image using servo. 139 140 This repair function attempts to use servo to install the DUT's 141 designated "stable firmware version". 142 143 This repair method only applies to DUTs used for FAFT. 144 """ 145 146 def repair(self, host): 147 if not _is_firmware_repair_supported(host): 148 raise hosts.AutoservRepairError( 149 'Firmware repair is not applicable to host %s.' % 150 host.hostname) 151 if not host.servo: 152 raise hosts.AutoservRepairError( 153 '%s has no servo support.' % host.hostname) 154 host.firmware_install() 155 156 @property 157 def description(self): 158 return 'Re-install the stable firmware via servo' 159 160 161class FirmwareVersionVerifier(hosts.Verifier): 162 """ 163 Check for a firmware update, and apply it if appropriate. 164 165 This verifier checks to ensure that either the firmware on the DUT 166 is up-to-date, or that the target firmware can be installed from the 167 currently running build. 168 169 Failure occurs when all of the following apply: 170 1. The DUT is not excluded from updates. For example, DUTs used 171 for FAFT testing use `FirmwareRepair` instead. 172 2. The DUT's board has an assigned stable firmware version. 173 3. The DUT is not running the assigned stable firmware. 174 4. The firmware supplied in the running OS build is not the 175 assigned stable firmware. 176 177 If the DUT needs an upgrade and the currently running OS build 178 supplies the necessary firmware, the verifier installs the new 179 firmware using `chromeos-firmwareupdate`. Failure to install will 180 cause the verifier to fail. 181 182 This verifier nominally breaks the rule that "verifiers must succeed 183 quickly", since it can invoke `reboot()` during the success code 184 path. We're doing it anyway for two reasons: 185 * The time between updates will typically be measured in months, 186 so the amortized cost is low. 187 * The reason we distinguish repair from verify is to allow 188 rescheduling work immediately while the expensive repair happens 189 out-of-band. But a firmware update will likely hit all DUTs at 190 once, so it's pointless to pass the buck to repair. 191 192 N.B. This verifier is a trigger for all repair actions that install 193 the stable repair image. If the firmware is out-of-date, but the 194 stable repair image does *not* contain the proper firmware version, 195 _the target DUT will fail repair, and will be unable to fix itself_. 196 """ 197 198 @staticmethod 199 def _get_rw_firmware(host): 200 result = host.run('crossystem fwid', ignore_status=True) 201 if result.exit_status == 0: 202 return result.stdout 203 else: 204 return None 205 206 @staticmethod 207 def _get_available_firmware(host): 208 result = host.run('chromeos-firmwareupdate -V', 209 ignore_status=True) 210 if result.exit_status == 0: 211 # At one point, the chromeos-firmwareupdate script was updated to 212 # add "RW" version fields. The old string, "BIOS version:" still 213 # appears in the new output, however it now refers to the RO 214 # firmware version. Therefore, we try searching for the new string 215 # first, "BIOS (RW) version". If that string isn't found, we then 216 # fallback to searching for old string. 217 version = re.search(r'BIOS \(RW\) version:\s*(?P<version>.*)', 218 result.stdout) 219 if not version: 220 version = re.search(r'BIOS version:\s*(?P<version>.*)', 221 result.stdout) 222 if version is not None: 223 return version.group('version') 224 return None 225 226 @staticmethod 227 def _check_hardware_match(version_a, version_b): 228 """ 229 Check that two firmware versions identify the same hardware. 230 231 Firmware version strings look like this: 232 Google_Gnawty.5216.239.34 233 The part before the numbers identifies the hardware for which 234 the firmware was built. This function checks that the hardware 235 identified by `version_a` and `version_b` is the same. 236 237 This is a sanity check to protect us from installing the wrong 238 firmware on a DUT when a board label has somehow gone astray. 239 240 @param version_a First firmware version for the comparison. 241 @param version_b Second firmware version for the comparison. 242 """ 243 hardware_a = version_a.split('.')[0] 244 hardware_b = version_b.split('.')[0] 245 if hardware_a != hardware_b: 246 message = 'Hardware/Firmware mismatch updating %s to %s' 247 raise hosts.AutoservVerifyError( 248 message % (version_a, version_b)) 249 250 def verify(self, host): 251 # Test 1 - The DUT is not excluded from updates. 252 if not _is_firmware_update_supported(host): 253 return 254 # Test 2 - The DUT has an assigned stable firmware version. 255 info = host.host_info_store.get() 256 if info.board is None: 257 raise hosts.AutoservVerifyError( 258 'Can not verify firmware version. ' 259 'No board label value found') 260 261 stable_firmware = afe_utils.get_stable_firmware_version(info.board) 262 if stable_firmware is None: 263 # This DUT doesn't have a firmware update target 264 return 265 266 # For tests 3 and 4: If the output from `crossystem` or 267 # `chromeos-firmwareupdate` isn't what we expect, we log an 268 # error, but don't fail: We don't want DUTs unable to test a 269 # build merely because of a bug or change in either of those 270 # commands. 271 272 # Test 3 - The DUT is not running the target stable firmware. 273 current_firmware = self._get_rw_firmware(host) 274 if current_firmware is None: 275 logging.error('DUT firmware version can\'t be determined.') 276 return 277 if current_firmware == stable_firmware: 278 return 279 # Test 4 - The firmware supplied in the running OS build is not 280 # the assigned stable firmware. 281 available_firmware = self._get_available_firmware(host) 282 if available_firmware is None: 283 logging.error('Supplied firmware version in OS can\'t be ' 284 'determined.') 285 return 286 if available_firmware != stable_firmware: 287 raise hosts.AutoservVerifyError( 288 'DUT firmware requires update from %s to %s' % 289 (current_firmware, stable_firmware)) 290 # Time to update the firmware. 291 logging.info('Updating firmware from %s to %s', 292 current_firmware, stable_firmware) 293 self._check_hardware_match(current_firmware, stable_firmware) 294 try: 295 host.run('chromeos-firmwareupdate --mode=autoupdate') 296 host.reboot() 297 except Exception as e: 298 message = ('chromeos-firmwareupdate failed: from ' 299 '%s to %s') 300 logging.exception(message, current_firmware, stable_firmware) 301 raise hosts.AutoservVerifyError( 302 message % (current_firmware, stable_firmware)) 303 final_firmware = self._get_rw_firmware(host) 304 if final_firmware != stable_firmware: 305 message = ('chromeos-firmwareupdate failed: tried upgrade ' 306 'to %s, now running %s instead') 307 raise hosts.AutoservVerifyError( 308 message % (stable_firmware, final_firmware)) 309 310 @property 311 def description(self): 312 return 'The firmware on this DUT is up-to-date' 313