1#!/usr/bin/env python2 2# Copyright 2020 The Chromium OS Authors. All rights reserved. 3# Use of this source code is governed by a BSD-style license that can be 4# found in the LICENSE file. 5 6import logging 7 8import common 9import base 10import constants 11import servo_updater 12import time 13import os 14import re 15 16from autotest_lib.client.common_lib import error 17from autotest_lib.client.common_lib import utils as client_utils 18from autotest_lib.server.cros.storage import storage_validate as storage 19from autotest_lib.server.cros import servo_keyboard_utils 20from autotest_lib.site_utils.admin_audit import rpm_validator 21 22try: 23 from chromite.lib import metrics 24except ImportError: 25 metrics = client_utils.metrics_mock 26 27# Common status used for statistics. 28STATUS_FAIL = 'fail' 29STATUS_SUCCESS = 'success' 30STATUS_SKIPPED = 'skipped' 31 32 33class VerifyDutStorage(base._BaseDUTVerifier): 34 """Verify the state of the storage on the DUT 35 36 The process to determine the type of storage and read metrics 37 of usage and EOL(end-of-life) information to determine the 38 state. 39 Supported storage types: MMS, NVME, SSD. 40 Possible states are: 41 UNKNOWN - not access to the DUT, not determine type of storage, 42 not information to determine metrics 43 NORMAL - the storage is in good shape and will work stable 44 device will work stable. (supported for all types) 45 ACCEPTABLE - the storage almost used all resources, device will 46 work stable but it is better be ready for replacement 47 device will work stable. (supported by MMS, NVME) 48 NEED_REPLACEMENT - the storage broken or worn off the life limit 49 device can work by not stable and can cause the 50 flakiness on the tests. (supported by all types) 51 """ 52 def __init__(self, dut_host): 53 super(VerifyDutStorage, self).__init__(dut_host) 54 self._state = None 55 56 def _verify(self, set_label=True, run_badblocks=None): 57 if not self.host_is_up(): 58 logging.info('Host is down; Skipping the verification') 59 return 60 try: 61 validator = storage.StorageStateValidator(self.get_host()) 62 storage_type = validator.get_type() 63 logging.debug('Detected storage type: %s', storage_type) 64 storage_state = validator.get_state(run_badblocks=run_badblocks) 65 logging.debug('Detected storage state: %s', storage_state) 66 state = self.convert_state(storage_state) 67 if state and set_label: 68 self._set_host_info_state(constants.DUT_STORAGE_STATE_PREFIX, 69 state) 70 if state == constants.HW_STATE_NEED_REPLACEMENT: 71 self.get_host().set_device_needs_replacement( 72 resultdir=self.get_result_dir()) 73 self._state = state 74 except Exception as e: 75 raise base.AuditError('Exception during getting state of' 76 ' storage %s' % str(e)) 77 78 def convert_state(self, state): 79 """Mapping state from validator to verifier""" 80 if state == storage.STORAGE_STATE_NORMAL: 81 return constants.HW_STATE_NORMAL 82 if state == storage.STORAGE_STATE_WARNING: 83 return constants.HW_STATE_ACCEPTABLE 84 if state == storage.STORAGE_STATE_CRITICAL: 85 return constants.HW_STATE_NEED_REPLACEMENT 86 return None 87 88 def get_state(self): 89 return self._state 90 91 92class VerifyServoUsb(base._BaseServoVerifier): 93 """Verify the state of the USB-drive on the Servo 94 95 The process to determine by checking the USB-drive on having any 96 bad sectors on it. 97 Possible states are: 98 UNKNOWN - not access to the device or servo, not available 99 software on the servo. 100 NORMAL - the device available for testing and not bad sectors. 101 was found on it, device will work stable 102 NEED_REPLACEMENT - the device available for testing and 103 some bad sectors were found on it. The device can 104 work but cause flakiness in the tests or repair process. 105 106 badblocks errors: 107 No such device or address while trying to determine device size 108 """ 109 def _verify(self): 110 if not self.servo_is_up(): 111 logging.info('Servo not initialized; Skipping the verification') 112 return 113 try: 114 usb = self.get_host()._probe_and_validate_usb_dev() 115 logging.debug('USB path: %s', usb) 116 except Exception as e: 117 usb = '' 118 logging.debug('(Not critical) %s', e) 119 if not usb: 120 self._set_state(constants.HW_STATE_NOT_DETECTED) 121 return 122 # basic readonly check 123 124 # path to USB if DUT is sshable 125 logging.info('Starting verification of USB drive...') 126 dut_usb = None 127 if self.host_is_up(): 128 dut_usb = self._usb_path_on_dut() 129 state = None 130 try: 131 if dut_usb: 132 logging.info('Try run check on DUT side.') 133 state = self._run_check_on_host(self._dut_host, dut_usb) 134 else: 135 logging.info('Try run check on ServoHost side.') 136 servo = self.get_host().get_servo() 137 servo_usb = servo.probe_host_usb_dev() 138 state = self._run_check_on_host(self.get_host(), servo_usb) 139 except Exception as e: 140 if 'Timeout encountered:' in str(e): 141 logging.info('Timeout during running action') 142 metrics.Counter( 143 'chromeos/autotest/audit/servo/usb/timeout' 144 ).increment(fields={'host': self._dut_host.hostname}) 145 else: 146 # badblocks generate errors when device not reachable or 147 # cannot read system information to execute process 148 state = constants.HW_STATE_NEED_REPLACEMENT 149 logging.debug(str(e)) 150 151 self._set_state(state) 152 logging.info('Finished verification of USB drive.') 153 154 self._install_stable_image() 155 156 def _usb_path_on_dut(self): 157 """Return path to the USB detected on DUT side.""" 158 servo = self.get_host().get_servo() 159 servo.switch_usbkey('dut') 160 result = self._dut_host.run('ls /dev/sd[a-z]') 161 for path in result.stdout.splitlines(): 162 cmd = ('. /usr/share/misc/chromeos-common.sh; get_device_type %s' % 163 path) 164 check_run = self._dut_host.run(cmd, timeout=30, ignore_status=True) 165 if check_run.stdout.strip() != 'USB': 166 continue 167 if self._quick_check_if_device_responsive(self._dut_host, path): 168 logging.info('USB drive detected on DUT side as %s', path) 169 return path 170 return None 171 172 def _quick_check_if_device_responsive(self, host, usb_path): 173 """Verify that device """ 174 validate_cmd = 'fdisk -l %s' % usb_path 175 try: 176 resp = host.run(validate_cmd, ignore_status=True, timeout=30) 177 if resp.exit_status == 0: 178 return True 179 logging.error('USB %s is not detected by fdisk!', usb_path) 180 except error.AutoservRunError as e: 181 if 'Timeout encountered' in str(e): 182 logging.warning('Timeout encountered during fdisk run.') 183 else: 184 logging.error('(Not critical) fdisk check fail for %s; %s', 185 usb_path, str(e)) 186 return False 187 188 def _run_check_on_host(self, host, usb): 189 """Run badblocks on the provided host. 190 191 @params host: Host where USB drive mounted 192 @params usb: Path to USB drive. (e.g. /dev/sda) 193 """ 194 command = 'badblocks -w -e 5 -b 4096 -t random %s' % usb 195 logging.info('Running command: %s', command) 196 # The response is the list of bad block on USB. 197 # Extended time for 2 hour to run USB verification. 198 # TODO (otabek@) (b:153661014#comment2) bring F3 to run 199 # check faster if badblocks cannot finish in 2 hours. 200 result = host.run(command, timeout=7200).stdout.strip() 201 logging.info("Check result: '%s'", result) 202 if result: 203 # So has result is Bad and empty is Good. 204 return constants.HW_STATE_NEED_REPLACEMENT 205 return constants.HW_STATE_NORMAL 206 207 def _install_stable_image(self): 208 """Install stable image to the USB drive.""" 209 # install fresh image to the USB because badblocks formats it 210 # https://crbug.com/1091406 211 try: 212 logging.debug('Started to install test image to USB-drive') 213 _, image_path = self._dut_host.stage_image_for_servo() 214 self.get_host().get_servo().image_to_servo_usb(image_path, 215 power_off_dut=False) 216 logging.debug('Finished installing test image to USB-drive') 217 except: 218 # ignore any error which happined during install image 219 # it not relative to the main goal 220 logging.info('Fail to install test image to USB-drive') 221 222 def _set_state(self, state): 223 if state: 224 self._set_host_info_state(constants.SERVO_USB_STATE_PREFIX, state) 225 226 227class VerifyServoFw(base._BaseServoVerifier): 228 """Force update Servo firmware if it not up-to-date. 229 230 This is rarely case when servo firmware was not updated by labstation 231 when servod started. This should ensure that the servo_v4 and 232 servo_micro is up-to-date. 233 """ 234 def _verify(self): 235 if not self.servo_host_is_up(): 236 logging.info('Servo host is down; Skipping the verification') 237 return 238 servo_updater.update_servo_firmware( 239 self.get_host(), 240 force_update=True) 241 242 243class VerifyRPMConfig(base._BaseDUTVerifier): 244 """Check RPM config of the setup. 245 246 This check run against RPM configs settings. 247 """ 248 249 def _verify(self): 250 if not self.host_is_up(): 251 logging.info('Host is down; Skipping the verification') 252 return 253 rpm_validator.verify_unsafe(self.get_host()) 254 255 256class FlashServoKeyboardMapVerifier(base._BaseDUTVerifier): 257 """Flash the keyboard map on servo.""" 258 259 _ATMEGA_RESET_DELAY = 0.2 260 _USB_PRESENT_DELAY = 1 261 262 # Command to detect LUFA Keyboard Demo by VID. 263 LSUSB_CMD = 'lsusb -d %s:' % servo_keyboard_utils.ATMEL_USB_VENDOR_ID 264 265 def _verify(self): 266 if not self.host_is_up(): 267 logging.info('Host is down; Skipping the action') 268 return 269 if not self.servo_is_up(): 270 logging.info('Servo not initialized; Skipping the action') 271 return 272 273 host = self.get_host() 274 servo = host.servo 275 try: 276 logging.info('Starting flashing the keyboard map.') 277 status = self._flash_keyboard_map(host, servo) 278 logging.info('Set status: %s', status) 279 if status == STATUS_FAIL: 280 self._send_metrics() 281 except Exception as e: 282 # The possible errors is timeout of commands. 283 logging.debug('Failed to flash servo keyboard map; %s', e) 284 self._send_metrics() 285 finally: 286 # Restore the default settings. 287 # Select the chip on the USB mux unless using Servo V4 288 if 'servo_v4' not in servo.get_servo_version(): 289 servo.set('usb_mux_sel4', 'on') 290 291 def _flash_keyboard_map(self, host, servo): 292 if host.run('hash dfu-programmer', ignore_status=True).exit_status: 293 logging.info( 294 'The image is too old that does not have dfu-programmer.') 295 return STATUS_SKIPPED 296 297 servo.set_nocheck('init_usb_keyboard', 'on') 298 299 if self._is_keyboard_present(host): 300 logging.info('Already using the new keyboard map.') 301 return STATUS_SUCCESS 302 303 # Boot AVR into DFU mode by enabling the HardWareBoot mode 304 # strapping and reset. 305 servo.set_get_all(['at_hwb:on', 306 'atmega_rst:on', 307 'sleep:%f' % self._ATMEGA_RESET_DELAY, 308 'atmega_rst:off', 309 'sleep:%f' % self._ATMEGA_RESET_DELAY, 310 'at_hwb:off']) 311 312 result = host.run(self.LSUSB_CMD, timeout=30).stdout.strip() 313 if not 'Atmel Corp. atmega32u4 DFU bootloader' in result: 314 logging.info('Not an expected chip: %s', result) 315 return STATUS_FAIL 316 317 # Update the keyboard map. 318 bindir = os.path.dirname(os.path.realpath(__file__)) 319 local_path = os.path.join(bindir, 'data', 'keyboard.hex') 320 host.send_file(local_path, '/tmp') 321 logging.info('Updating the keyboard map...') 322 host.run('dfu-programmer atmega32u4 erase --force', timeout=120) 323 host.run('dfu-programmer atmega32u4 flash /tmp/keyboard.hex', 324 timeout=120) 325 326 # Reset the chip. 327 servo.set_get_all(['atmega_rst:on', 328 'sleep:%f' % self._ATMEGA_RESET_DELAY, 329 'atmega_rst:off']) 330 if self._is_keyboard_present(host): 331 logging.info('Update successfully!') 332 return STATUS_SUCCESS 333 334 logging.info('Update failed!') 335 return STATUS_FAIL 336 337 def _is_keyboard_present(self, host): 338 # Check the result of lsusb. 339 time.sleep(self._USB_PRESENT_DELAY) 340 result = host.run(self.LSUSB_CMD, timeout=30).stdout.strip() 341 logging.info('got the result: %s', result) 342 if ('LUFA Keyboard Demo' in result and 343 servo_keyboard_utils.is_servo_usb_wake_capable(host)): 344 return True 345 return False 346 347 def _send_metrics(self): 348 host = self.get_host() 349 data = {'host': host.hostname, 'status': STATUS_FAIL} 350 metrics.Counter( 351 'chromeos/autotest/audit/servo_keyboard').increment(fields=data) 352 353 354class VerifyDUTMacAddress(base._BaseDUTVerifier): 355 """Verify and update cached NIC mac address on servo. 356 357 Servo_v4 plugged to the DUT and providing NIC for that. We caching mac 358 address on servod side to better debugging. 359 """ 360 361 # HUB and NIC VID/PID. 362 # Values presented as the string of the hex without 0x to match 363 # representation in sysfs (idVendor/idProduct). 364 HUB_VID = '04b4' 365 HUB_PID = '6502' 366 NIC_VID = '0bda' 367 NIC_PID = '8153' 368 369 # Regex to check mac address format. 370 # eg: f4:f5:e8:50:e9:45 371 RE_MACADDR = re.compile('^([0-9A-Fa-f]{2}[:-]){5}([0-9A-Fa-f]{2})$') 372 373 def _verify(self): 374 if not self.host_is_up(): 375 logging.info('Host is down; Skipping the action') 376 return 377 if not self.servo_is_up(): 378 logging.info('Servo host is down; Skipping the action') 379 return 380 host = self.get_host() 381 servo = host.servo 382 if not host._servo_host.is_labstation(): 383 logging.info('Only servo_v4 has NIC; ' 384 'Skipping the action') 385 return 386 if not servo.has_control('macaddr'): 387 logging.info('"macaddr" control not supported;' 388 'Skipping the action') 389 return 390 391 # Path to the NIC has to be located in the HUB. 392 # eg. 393 # HUB: /sys/bus/usb/devices/1-1 394 # NIC: /sys/bus/usb/devices/1-1.1 395 hub_path = self._get_device_path(None, self.HUB_VID, self.HUB_PID) 396 if not hub_path or hub_path == '.': 397 logging.info('The servo_v4 HUB not detected from DUT') 398 self._send_metrics() 399 return 400 logging.info('Path to the servo_v4 HUB device: %s', hub_path) 401 nic_path = self._get_device_path(hub_path, self.NIC_VID, self.NIC_PID) 402 if not nic_path or nic_path == '.': 403 logging.info('The servo_v4 NIC not detected in HUB folder') 404 self._send_metrics() 405 return 406 logging.info('Path to the servo_v4 NIC device: %s', nic_path) 407 if hub_path == nic_path or not nic_path.startswith(hub_path): 408 logging.info('The servo_v4 NIC was detect out of servo_v4 HUB;' 409 ' Skipping the action.') 410 self._send_metrics() 411 return 412 413 macaddr = self._get_mac_address(host, nic_path) 414 if not macaddr: 415 self._send_metrics() 416 return 417 418 cached_mac = self._get_cached_mac_address() 419 if not cached_mac or macaddr != cached_mac: 420 try: 421 servo.set('macaddr', macaddr) 422 logging.info('Successfully updated the servo "macaddr"!') 423 except error.TestFail as e: 424 logging.debug('Fail to update macaddr value; %s', e) 425 logging.info('Fail to update the "macaddr" value!') 426 self._send_metrics() 427 else: 428 logging.info('The servo "macaddr" doe not need update.') 429 430 def _get_cached_mac_address(self): 431 try: 432 return self.get_host().servo.get('macaddr') 433 except error.TestFail as e: 434 logging.error('(Non-critical) Fail to get macaddr: %s', e) 435 return None 436 437 def _get_mac_address(self, host, nic_path): 438 cmd = r'find %s/ | grep /net/ | grep /address' % nic_path 439 res = host.run(cmd, 440 timeout=30, 441 ignore_status=True, 442 ignore_timeout=True) 443 if not res: 444 logging.info('Timeout during retriving NIC address files.') 445 return None 446 addrs = res.stdout.splitlines() 447 if not addrs or len(addrs) == 0: 448 logging.info('No NIC address file found.') 449 return None 450 if len(addrs) > 1: 451 logging.info('More than one NIC address file found.') 452 return None 453 logging.info('Found NIC address file: %s', addrs[0]) 454 cmd = r'cat %s' % addrs[0] 455 res = host.run(cmd, 456 timeout=30, 457 ignore_status=True, 458 ignore_timeout=True) 459 if not res: 460 logging.info('Timeout during attemp read NIC address file: %s', 461 addrs[0]) 462 return None 463 mac_addr = res.stdout.strip() 464 if not self.RE_MACADDR.match(mac_addr): 465 logging.info('incorrect format of the mac address: %s', mac_addr) 466 return None 467 logging.info('Servo_v4 NIC mac address from DUT side: %s', mac_addr) 468 return mac_addr 469 470 def _get_device_path(self, base_path, vid, pid): 471 """Find a device by VID/PID under particular path. 472 473 1) Get path to the unique idVendor file with VID 474 2) Get path to the unique idProduct file with PID 475 3) Get directions of both file and compare them 476 477 @param base_path: Path to the directory where to look for the device. 478 @param vid: Vendor ID of the looking device. 479 @param pid: Product ID of the looking device. 480 481 @returns: path to the folder of the device 482 """ 483 host = self.get_host() 484 def _run(cmd): 485 res = host.run(cmd, timeout=30, 486 ignore_status=True, 487 ignore_timeout=True) 488 l = res.stdout.splitlines() 489 if not l or len(l) != 1: 490 return None 491 return l[0] 492 493 if not base_path: 494 base_path = '/sys/bus/usb/devices/*/' 495 else: 496 base_path += '*/' 497 cmd_template = 'grep -l %s $(find %s -maxdepth 1 -name %s)' 498 vid_path = _run(cmd_template % (vid, base_path, 'idVendor')) 499 if not vid_path: 500 return None 501 502 pid_path = _run(cmd_template % (pid, base_path, 'idProduct')) 503 if not pid_path: 504 return None 505 506 # check if both files locates in the same folder 507 return _run('LC_ALL=C comm -12 <(dirname %s) <(dirname %s)' % 508 (vid_path, pid_path)) 509 510 def _send_metrics(self): 511 host = self.get_host() 512 data = {'host': host.hostname, 'status': STATUS_FAIL} 513 metrics.Counter( 514 'chromeos/autotest/audit/servo_macaddr').increment(fields=data) 515