1# Lint as: python2, python3 2# Copyright 2016 The Chromium OS Authors. All rights reserved. 3# Use of this source code is governed by a BSD-style license that can be 4# found in the LICENSE file. 5 6from __future__ import absolute_import 7from __future__ import division 8from __future__ import print_function 9 10import functools 11import logging 12import math 13import sys 14import time 15 16import common 17from autotest_lib.client.common_lib import error 18from autotest_lib.client.common_lib import hosts 19from autotest_lib.client.common_lib import utils 20from autotest_lib.server.cros.servo import servo 21from autotest_lib.server.hosts import cros_constants 22from autotest_lib.server.hosts import repair_utils 23from autotest_lib.server.hosts import servo_constants 24from autotest_lib.server.cros.servo.topology import servo_topology 25from autotest_lib.site_utils.admin_audit import servo_updater 26import six 27 28try: 29 from autotest_lib.utils.frozen_chromite.lib import metrics 30except ImportError: 31 metrics = utils.metrics_mock 32 33from autotest_lib.utils.frozen_chromite.lib import timeout_util 34 35def ignore_exception_for_non_cros_host(func): 36 """ 37 Decorator to ignore ControlUnavailableError if servo host is not cros host. 38 When using test_that command on a workstation, this enables usage of 39 additional servo devices such as servo micro and Sweetberry. This shall not 40 change any lab behavior. 41 """ 42 @functools.wraps(func) 43 def wrapper(self, host): 44 """ 45 Wrapper around func. 46 """ 47 try: 48 func(self, host) 49 except servo.ControlUnavailableError as e: 50 if host.is_cros_host(): 51 raise 52 logging.warning("Servo host is not cros host, ignore %s: %s", 53 type(e).__name__, e) 54 return wrapper 55 56 57class _UpdateVerifier(hosts.Verifier): 58 """ 59 Verifier to trigger a servo host update, if necessary. 60 61 The verifier works only for servo_v3. 62 The operation doesn't wait for the update to complete and is 63 considered a success whether or not the servo is currently 64 up-to-date. 65 """ 66 67 @timeout_util.TimeoutDecorator(cros_constants.LONG_VERIFY_TIMEOUT_SEC) 68 def verify(self, host): 69 try: 70 if ( 71 not host.get_dut_host_info() 72 or not host.get_dut_host_info().servo_cros_stable_version): 73 logging.info('Servo stable version missed.' 74 ' Skip update check action.') 75 return 76 # We have seen cases that invalid GPT headers/entries block 77 # v3s from been update, so always try to repair here. 78 # See crbug.com/994396, crbug.com/1057302. 79 host.run('cgpt repair /dev/mmcblk0', ignore_status=True) 80 host.update_image() 81 # We don't want failure from update block DUT repair action. 82 # See crbug.com/1029950. 83 except Exception as e: 84 six.reraise(hosts.AutoservNonCriticalVerifyError, 85 hosts.AutoservNonCriticalVerifyError(e), 86 sys.exc_info()[2]) 87 88 def _is_applicable(self, host): 89 # Run only for servo_v3 host. 90 if host.is_labstation() or host.is_containerized_servod(): 91 return False 92 # Only run if the host is in the physical lab. 93 return host.is_in_lab() 94 95 @property 96 def description(self): 97 return 'Servo_v3 host software is up-to-date' 98 99 100class _StartServodVerifier(hosts.Verifier): 101 """First start of servod on the host. 102 103 Single running action to start servod in the first time. 104 This verifier created to fit current flow and will be revisited later. 105 Action never fails! 106 """ 107 108 @timeout_util.TimeoutDecorator(cros_constants.VERIFY_TIMEOUT_SEC) 109 def verify(self, host): 110 if not hasattr(self, 'started'): 111 logging.info('Starting servod!') 112 host.restart_servod(quick_startup=True) 113 # caching the value to prevent restart service when trigger verifier. 114 self.started = True 115 116 @property 117 def description(self): 118 return 'Initial servod start' 119 120 121class _RootServoPresentVerifier(hosts.Verifier): 122 """Verifier that first servo is present.""" 123 124 @timeout_util.TimeoutDecorator(cros_constants.VERIFY_TIMEOUT_SEC) 125 def verify(self, host): 126 device = None 127 topology = host.get_topology() 128 topology.read(host.get_dut_host_info()) 129 try: 130 device = topology.get_root_servo() 131 except Exception as e: 132 if host.is_containerized_servod(): 133 host.restart_servod() 134 logging.debug('Restarting servod container (Not critical) %s', 135 e) 136 else: 137 host.request_reboot() 138 logging.info( 139 'Reboot labstation requested, it will be handled' 140 ' by labstation AdminRepair task.' 141 ' Unable to detect root servo info from topology.') 142 logging.debug('(Not critical) %s', e) 143 if device: 144 logging.info('Root servo is present') 145 return 146 device = topology.get_root_servo_from_cache() 147 if device: 148 logging.debug('Found device: %s', device) 149 if device.get_serial_number() != host.servo_serial: 150 self.serial_mismatch = True 151 raise hosts.AutoservVerifyError('Serial mismatch detected') 152 logging.info('Root servo is present') 153 return 154 # Leaving error in case we got empty device. 155 raise hosts.AutoservVerifyError('Root servo not found!') 156 157 def _is_applicable(self, host): 158 if host.is_containerized_servod(): 159 logging.info('Servod is running within a container.') 160 return True 161 if not host.is_labstation(): 162 logging.info('Not supported for servo_v3.') 163 return False 164 # Only run if the host is in the physical lab. 165 return host.is_in_lab() 166 167 @property 168 def description(self): 169 return 'Root servo is present' 170 171 172class _RootServoV3PresentVerifier(hosts.Verifier): 173 """Verifier that first servo is present.""" 174 175 RETRY_COUNT = 3 176 177 @timeout_util.TimeoutDecorator(cros_constants.VERIFY_TIMEOUT_SEC) 178 def verify(self, host): 179 for a in range(self.RETRY_COUNT): 180 logging.debug('Attempt: %s find servo board on servo_v3.', a + 1) 181 present = host.is_servo_board_present_on_servo_v3() 182 if present == False: 183 raise hosts.AutoservVerifyError('Servo board not found!') 184 elif present == True: 185 logging.debug('Servo board is present') 186 return 187 raise hosts.AutoservVerifyError('Fail to find servo board!') 188 189 def _is_applicable(self, host): 190 if host.is_containerized_servod(): 191 logging.info('Servod is running within a container.') 192 return False 193 # Do not run for servos under labstations. 194 if host.is_labstation(): 195 logging.info('Servod is running on labstation.') 196 return False 197 # Only run if the host is in the physical lab. 198 return host.is_in_lab() 199 200 @property 201 def description(self): 202 return 'Servo board on servo_v3 is present' 203 204 205class _ServoFwVerifier(hosts.Verifier): 206 """Verifier to check is a servo fw is up-to-date.""" 207 208 @timeout_util.TimeoutDecorator(cros_constants.VERIFY_TIMEOUT_SEC) 209 def verify(self, host): 210 try: 211 if servo_updater.any_servo_needs_firmware_update(host): 212 raise hosts.AutoservVerifyError( 213 'Some servo requires firmware update') 214 except servo_updater.ServoFwVersionMissedError as e: 215 # Do not fail as it will trigger re-flash fw on the servo 216 logging.info( 217 'Issue with detect new version of firmware for servo.' 218 ' Please file a bug agains Fleet Automation team (go/fleet-bug)' 219 ) 220 221 def _is_applicable(self, host): 222 if host.is_containerized_servod(): 223 logging.info('Servod is running within a container.') 224 return True 225 # Run only for servos under labstations. 226 if not host.is_labstation(): 227 logging.info('Not supported for servo_v3.') 228 return False 229 # Only run if the host is in the physical lab. 230 return host.is_in_lab() 231 232 @property 233 def description(self): 234 return 'Servo fw is up-to-date' 235 236 237class _ConfigVerifier(hosts.Verifier): 238 """ 239 Base verifier for the servo config file verifiers. 240 """ 241 242 CONFIG_FILE = '/var/lib/servod/config' 243 ATTR = '' 244 245 @staticmethod 246 def _get_config_val(host, config_file, attr): 247 """ 248 Get the `attr` for `host` from `config_file`. 249 250 @param host Host to be checked for `config_file`. 251 @param config_file Path to the config file to be tested. 252 @param attr Attribute to get from config file. 253 254 @return The attr val as set in the config file, or `None` if 255 the file was absent. 256 """ 257 getboard = ('CONFIG=%s ; [ -f $CONFIG ] && ' 258 '. $CONFIG && echo $%s' % (config_file, attr)) 259 attr_val = host.run(getboard, ignore_status=True).stdout 260 return attr_val.strip('\n') if attr_val else None 261 262 @staticmethod 263 def _validate_attr(host, val, expected_val, attr, config_file): 264 """ 265 Check that the attr setting is valid for the host. 266 267 This presupposes that a valid config file was found. Raise an 268 execption if: 269 * There was no attr setting from the file (i.e. the setting 270 is an empty string), or 271 * The attr setting is valid, the attr is known, 272 and the setting doesn't match the DUT. 273 274 @param host Host to be checked for `config_file`. 275 @param val Value to be tested. 276 @param expected_val Expected value. 277 @param attr Attribute we're validating. 278 @param config_file Path to the config file to be tested. 279 """ 280 if not val: 281 raise hosts.AutoservVerifyError( 282 'config file %s exists, but %s ' 283 'is not set' % (attr, config_file)) 284 if expected_val is not None and val != expected_val: 285 raise hosts.AutoservVerifyError( 286 '%s is %s; it should be %s' % (attr, val, expected_val)) 287 288 289 def _get_config(self, host): 290 """ 291 Return the config file to check. 292 293 @param host Host object. 294 295 @return The config file to check. 296 """ 297 return '%s_%d' % (self.CONFIG_FILE, host.servo_port) 298 299 @property 300 def description(self): 301 return 'servo %s setting is correct' % self.ATTR 302 303 304class _SerialConfigVerifier(_ConfigVerifier): 305 """ 306 Verifier for the servo SERIAL configuration. 307 """ 308 309 ATTR = 'SERIAL' 310 311 @timeout_util.TimeoutDecorator(cros_constants.SHORT_VERIFY_TIMEOUT_SEC) 312 def verify(self, host): 313 """ 314 Test whether the `host` has a `SERIAL` setting configured. 315 316 This tests the config file names used by the `servod` upstart 317 job for a valid setting of the `SERIAL` variable. The following 318 conditions raise errors: 319 * The SERIAL setting doesn't match the DUT's entry in the AFE 320 database. 321 * There is no config file. 322 """ 323 if not host.is_cros_host(): 324 return 325 # Not all servo hosts will have a servo serial so don't verify if it's 326 # not set. 327 if host.servo_serial is None: 328 return 329 config = self._get_config(host) 330 serialval = self._get_config_val(host, config, self.ATTR) 331 if serialval is None: 332 raise hosts.AutoservVerifyError( 333 'Servo serial is unconfigured; should be %s' 334 % host.servo_serial 335 ) 336 337 self._validate_attr(host, serialval, host.servo_serial, self.ATTR, 338 config) 339 340 341 342class _BoardConfigVerifier(_ConfigVerifier): 343 """ 344 Verifier for the servo BOARD configuration. 345 """ 346 347 ATTR = 'BOARD' 348 349 @timeout_util.TimeoutDecorator(cros_constants.SHORT_VERIFY_TIMEOUT_SEC) 350 def verify(self, host): 351 """ 352 Test whether the `host` has a `BOARD` setting configured. 353 354 This tests the config file names used by the `servod` upstart 355 job for a valid setting of the `BOARD` variable. The following 356 conditions raise errors: 357 * A config file exists, but the content contains no setting 358 for BOARD. 359 * The BOARD setting doesn't match the DUT's entry in the AFE 360 database. 361 * There is no config file. 362 """ 363 if not host.is_cros_host(): 364 return 365 config = self._get_config(host) 366 boardval = self._get_config_val(host, config, self.ATTR) 367 if boardval is None: 368 msg = 'Servo board is unconfigured' 369 if host.servo_board is not None: 370 msg += '; should be %s' % host.servo_board 371 raise hosts.AutoservVerifyError(msg) 372 373 self._validate_attr(host, boardval, host.servo_board, self.ATTR, 374 config) 375 376 377class _ServodJobVerifier(hosts.Verifier): 378 """ 379 Verifier to check that the `servod` upstart job is running. 380 """ 381 382 @timeout_util.TimeoutDecorator(cros_constants.SHORT_VERIFY_TIMEOUT_SEC) 383 def verify(self, host): 384 if not host.is_cros_host(): 385 return 386 status_cmd = 'status servod PORT=%d' % host.servo_port 387 job_status = host.run(status_cmd, ignore_status=True).stdout 388 if 'start/running' not in job_status: 389 raise hosts.AutoservVerifyError( 390 'servod not running on %s port %d' % 391 (host.hostname, host.servo_port)) 392 393 @property 394 def description(self): 395 return 'servod upstart job is running' 396 397 398class _ServodEchoVerifier(hosts.Verifier): 399 """ 400 Verifier to check that the `servod` upstart job is responsible. 401 """ 402 403 SERVOD_INITIALIZED = 'servodtool instance wait-for-active -p %d --timeout 60' 404 SERVOD_RESPONSIVE = 'dut-control -p %d serialname' 405 406 @timeout_util.TimeoutDecorator(cros_constants.VERIFY_TIMEOUT_SEC) 407 def verify(self, host): 408 self._verify_servod_initialized(host) 409 self._verify_servod_responsive(host) 410 411 def _verify_servod_initialized(self, host): 412 # Verify that servod initialized. 413 cmd = self.SERVOD_INITIALIZED % host.servo_port 414 res = host.run(cmd, ignore_status=True, timeout=120) 415 if res.exit_status != 0: 416 raise hosts.AutoservVerifyError( 417 'Servod instance is not initialized') 418 logging.debug("Presented instance: %s", res.stdout.strip()) 419 420 def _verify_servod_responsive(self, host): 421 # Verify if servod started and process is responsible. 422 cmd = self.SERVOD_RESPONSIVE % host.servo_port 423 res = host.run(cmd, ignore_status=True, timeout=120) 424 if res.exit_status != 0: 425 raise hosts.AutoservVerifyError( 426 'Servod is not responsive for dut-control commands') 427 logging.info('Servod responsive: %s', res.stdout) 428 429 @property 430 def description(self): 431 return 'Servod is running and responsive to dut-control run.' 432 433 434class _DiskSpaceVerifier(hosts.Verifier): 435 """ 436 Verifier to make sure there is enough disk space left on servohost. 437 """ 438 439 @timeout_util.TimeoutDecorator(cros_constants.SHORT_VERIFY_TIMEOUT_SEC) 440 def verify(self, host): 441 # Check available space of stateful is greater than threshold, in Gib. 442 host.check_diskspace('/mnt/stateful_partition', 0.5) 443 444 @property 445 def description(self): 446 return 'servohost has enough disk space.' 447 448 def _is_applicable(self, host): 449 if host.is_containerized_servod(): 450 logging.info('Servod is running within a container.') 451 return False 452 return True 453 454 455class _ServodConnectionVerifier(hosts.Verifier): 456 """ 457 Verifier to check that we can connect to servod server. 458 459 If this verifier failed, it most likely servod was crashed or in a 460 crashing loop. For servo_v4 it's usually caused by not able to detect 461 CCD or servo_micro. 462 """ 463 464 @timeout_util.TimeoutDecorator(cros_constants.VERIFY_TIMEOUT_SEC) 465 def verify(self, host): 466 host.initialize_servo() 467 468 @property 469 def description(self): 470 return 'servod service is taking calls' 471 472 473class _ServodControlVerifier(hosts.Verifier): 474 """ 475 Verifier to check basic servo control functionality. 476 477 This tests the connection to the target servod service with a simple 478 method call. As a side-effect, all servo signals are initialized to 479 default values. 480 481 N.B. Initializing servo signals is necessary because the power 482 button and lid switch verifiers both test against expected initial 483 values. 484 """ 485 486 @timeout_util.TimeoutDecorator(cros_constants.VERIFY_TIMEOUT_SEC) 487 def verify(self, host): 488 try: 489 host.initialize_dut_for_servo() 490 except Exception as e: 491 six.reraise(hosts.AutoservNonCriticalVerifyError, 492 hosts.AutoservNonCriticalVerifyError(e), 493 sys.exc_info()[2]) 494 495 @property 496 def description(self): 497 return 'Basic servod control is working' 498 499 500class _Cr50ConsoleVerifier(hosts.Verifier): 501 """Verifier to check if cr50 console is present and working. 502 503 Validating based by running commands and expect they will not fail. 504 If any command fail then console is not working as expected. 505 """ 506 507 COMMAND_TO_CHECK_CONSOLE = ( 508 'gsc_ccd_level', 509 'cr50_testlab', 510 'cr50_ccd_state_flags', 511 ) 512 513 @ignore_exception_for_non_cros_host 514 @timeout_util.TimeoutDecorator(cros_constants.VERIFY_TIMEOUT_SEC) 515 def verify(self, host): 516 try: 517 for command in self.COMMAND_TO_CHECK_CONSOLE: 518 if host.get_servo().has_control(command): 519 # Response of command is not important. 520 host.get_servo().get(command) 521 except Exception as e: 522 six.reraise(hosts.AutoservNonCriticalVerifyError, 523 hosts.AutoservNonCriticalVerifyError(e), 524 sys.exc_info()[2]) 525 526 def _is_applicable(self, host): 527 # Only when DUT is running through ccd or c2d2. 528 # TODO(coconutruben): replace with ccd API when available in servo.py 529 return host.get_servo() and host.get_servo().main_device_uses_gsc_drv() 530 531 @property 532 def description(self): 533 return 'CR50 console is working' 534 535 536class _CCDTestlabVerifier(hosts.Verifier): 537 """ 538 Verifier to check that ccd testlab is enabled. 539 540 All DUT connected by ccd has to supported cr50 with enabled testlab 541 to allow manipulation by servo. The flag testlab is sticky and will 542 stay enabled if was set up. The testlab can be enabled when ccd is 543 open. (go/ccd-setup) 544 """ 545 @ignore_exception_for_non_cros_host 546 @timeout_util.TimeoutDecorator(cros_constants.VERIFY_TIMEOUT_SEC) 547 def verify(self, host): 548 if not host.get_servo().has_control('cr50_testlab'): 549 raise hosts.AutoservVerifyError( 550 'gsc has to be supported when use servo with ' 551 'ccd_*/type-c connection') 552 553 status = host.get_servo().get('cr50_testlab') 554 # check by 'on' to fail when get unexpected value 555 if status == 'on': 556 # If servo uses cr50 to control the dut, open ccd so repair actions 557 # that reset the dut will work (cr50_reboot, cold_reset, warm_reset) 558 if host.get_servo().main_device_uses_gsc_drv(): 559 host.get_servo().set_nocheck('cr50_testlab', 'open') 560 # ccd testlab enabled 561 return 562 raise hosts.AutoservNonCriticalVerifyError( 563 'The ccd testlab is disabled; DUT requires manual work ' 564 'to enable it (go/ccd-setup).') 565 566 def _is_applicable(self, host): 567 # Only when DUT is running through ccd. 568 # TODO(coconutruben): replace with ccd API when available in servo.py 569 return host.get_servo() and host.get_servo().main_device_is_ccd() 570 571 @property 572 def description(self): 573 return 'ccd testlab enabled' 574 575class _CCDPowerDeliveryVerifier(hosts.Verifier): 576 """Verifier to check and reset servo_v4_role for servos that support 577 power delivery feature(a.k.a power pass through). 578 579 There are currently two position of servo_v4_role, src and snk: 580 src -- servo in power delivery mode and passes power to the DUT. 581 snk -- servo in normal mode and not passes power to DUT. 582 We want to ensure that servo_v4_role is set to src. 583 """ 584 @timeout_util.TimeoutDecorator(cros_constants.VERIFY_TIMEOUT_SEC) 585 def verify(self, host): 586 if host.get_servo(): 587 self._printControl(host.get_servo(), 'ppdut5_mv') 588 self._printControl(host.get_servo(), 'ppchg5_mv') 589 if host.get_servo().get('servo_pd_role') == 'snk': 590 raise hosts.AutoservNonCriticalVerifyError( 591 'Power delivery not in src role.') 592 593 def _printControl(self, servo, control): 594 if servo.has_control(control): 595 logging.info("%s: %s", control, servo.get(control)) 596 597 def _is_applicable(self, host): 598 return (host.is_in_lab() and 599 host.get_servo().supports_built_in_pd_control()) 600 601 @property 602 def description(self): 603 return 'ensure applicable servo is in "src" mode for power delivery' 604 605 606class _BaseDUTConnectionVerifier(hosts.Verifier): 607 """Verifier to check connection between DUT and servo.""" 608 609 # Bus voltage on ppdut5. Value can be: 610 # - less than 500 - DUT is likely not connected 611 # - between 500 and 4000 - unexpected value 612 # - more than 4000 - DUT is likely connected 613 MAX_PPDUT5_MV_WHEN_NOT_CONNECTED = 500 614 MIN_PPDUT5_MV_WHEN_CONNECTED = 4000 615 616 def _is_usb_hub_connected(self, host): 617 """Checking bus voltage on ppdut5. 618 619 Supported only on servo_v4 boards. 620 If voltage value is lower than 500 then device is not connected. 621 When value higher 4000 means the device is connected. If value 622 between 500 and 4000 is not expected and will be marked as connected 623 and collected information which DUT has this exception. 624 625 @returns: bool 626 """ 627 logging.debug('Started check by ppdut5_mv:on') 628 try: 629 val = host.get_servo().get('ppdut5_mv') 630 logging.info('ppdut5_mv=%s', val) 631 if val < self.MAX_PPDUT5_MV_WHEN_NOT_CONNECTED: 632 # servo is not connected to the DUT 633 return False 634 if val < self.MIN_PPDUT5_MV_WHEN_CONNECTED: 635 # is unexpected value. 636 # collecting metrics to look case by case 637 # TODO(otabek) for analysis b:163845694 638 data = host._get_host_metrics_data() 639 metrics.Counter('chromeos/autotest/repair/ppdut5_mv_case' 640 ).increment(fields=data) 641 # else: 642 # servo is physical connected to the DUT 643 except Exception as e: 644 logging.debug('(Not critical) %s', e) 645 return True 646 647 def _is_ribbon_cable_connected(self, host): 648 """Check if ribbon cable is connected to the DUT. 649 650 The servo_micro/flex - can be checked by `cold_reset` signal. 651 When `cold_reset` is `on` it commonly indicates that the DUT 652 is disconnected. To avoid mistake of real signal we try 653 switch it off and if is cannot then servo is not connected. 654 655 @returns: bool 656 """ 657 logging.debug('Started check by cold_reset:on') 658 try: 659 val = host.get_servo().get('cold_reset') 660 logging.info('cold_reset=%s', val) 661 if val == 'on': 662 # If cold_reset has is on can be right signal 663 # or caused by missing connection between servo_micro and DUT. 664 # if we can switch it to the off then it was signal. 665 host.get_servo().set('cold_reset', 'off') 666 except error.TestFail: 667 logging.debug('Ribbon cable is not connected to the DUT.') 668 return False 669 except Exception as e: 670 logging.debug('(Not critical) %s', e) 671 return True 672 673 def _is_dut_power_on(self, host): 674 # DUT is running in normal state. 675 # if EC not supported by board then we expect error 676 try: 677 return host.get_servo().get('ec_system_powerstate') == 'S0' 678 except Exception as e: 679 logging.debug('(Not critical) %s', e) 680 return False 681 682 def _is_servo_v4_type_a(self, host): 683 return host.is_labstation() and host.get_servo().is_servo_v4_type_a() 684 685 def _is_servo_v4_type_c(self, host): 686 return host.is_labstation() and host.get_servo().is_servo_v4_type_c() 687 688 def _is_servo_v3(self, host): 689 return not host.is_labstation() 690 691 692class _DUTConnectionVerifier(_BaseDUTConnectionVerifier): 693 """Verifier to check connection Servo to the DUT. 694 695 Servo_v4 type-a connected to the DUT by: 696 1) servo_micro - checked by `cold_reset`. 697 Servo_v4 type-c connected to the DUT by: 698 1) ccd - checked by ppdut5_mv. 699 Servo_v3 connected to the DUT by: 700 1) legacy servo header - can be checked by `cold_reset`. 701 """ 702 703 @ignore_exception_for_non_cros_host 704 @timeout_util.TimeoutDecorator(cros_constants.VERIFY_TIMEOUT_SEC) 705 def verify(self, host): 706 if self._is_servo_v4_type_a(host): 707 if not self._is_ribbon_cable_connected(host): 708 raise hosts.AutoservVerifyError( 709 'Servo_micro is likely not connected to the DUT.') 710 elif self._is_servo_v4_type_c(host): 711 if (host.get_servo().supports_built_in_pd_control() 712 and not self._is_usb_hub_connected(host)): 713 raise hosts.AutoservVerifyError( 714 'Servo_v4 is likely not connected to the DUT.') 715 elif self._is_servo_v3(host): 716 if not self._is_ribbon_cable_connected(host): 717 raise hosts.AutoservVerifyError( 718 'Servo_v3 is likely not connected to the DUT.') 719 720 @property 721 def description(self): 722 return 'Ensure the Servo connected to the DUT.' 723 724 725class _ServoHubConnectionVerifier(_BaseDUTConnectionVerifier): 726 """Verifier to check connection ServoHub to DUT. 727 728 Servo_v4 type-a connected to the DUT by: 729 1) USB hub - checked by ppdut5_mv. 730 """ 731 732 @ignore_exception_for_non_cros_host 733 @timeout_util.TimeoutDecorator(cros_constants.VERIFY_TIMEOUT_SEC) 734 def verify(self, host): 735 if self._is_servo_v4_type_a(host): 736 if (self._is_dut_power_on(host) 737 and not self._is_usb_hub_connected(host)): 738 raise hosts.AutoservVerifyError( 739 'Servo USB hub is likely not connected to the DUT.') 740 741 def _is_applicable(self, host): 742 if host.is_ec_supported(): 743 return True 744 logging.info('Host does not support EC.') 745 return False 746 747 @property 748 def description(self): 749 return 'Ensure the Servo HUB connected to the DUT.' 750 751 752class _BaseCr50SBUVerifier(_BaseDUTConnectionVerifier): 753 """Check servod issue related to SBU voltage.""" 754 755 # Min SBU voltage to detect usb-device 756 SBU_THRESHOLD = 2500.0 757 # How many times collect SBU voltage to calc AVG value. 758 _TOTAL_CHECK_SBU_VOLTAGE = 10 759 760 def _is_applicable(self, host): 761 if host.is_localhost(): 762 logging.info('Target servo is not in a lab,' 763 ' action is not applicable.') 764 return False 765 if not self._is_servo_v4_type_c(host): 766 logging.info('Check support only servo-v4 (type-c),' 767 ' action is not applicable.') 768 return False 769 return True 770 771 def _is_sbu_voltage_issue(self, host): 772 """Check if servo does not detected by SBU voltage issue.""" 773 command = 'dut_sbu_voltage_float_fault' 774 if host.get_servo().has_control(command): 775 if host.get_servo().get(command) == 'on': 776 return True 777 return False 778 779 def _get_max_sbu_value(self, host): 780 """Get average voltage on SBU lines.""" 781 servo = host.get_servo() 782 if not servo.has_control('servo_dut_sbu1_mv'): 783 return -1 784 s1 = 0 785 s2 = 0 786 for i in range(self._TOTAL_CHECK_SBU_VOLTAGE): 787 try: 788 sbu1 = int(servo.get('servo_dut_sbu1_mv')) 789 sbu2 = int(servo.get('servo_dut_sbu2_mv')) 790 logging.debug('Attempt:%2d, sbu1 %4d sbu2 %4d', i, sbu1, sbu2) 791 s1 += sbu1 792 s2 += sbu2 793 except error.TestFail as e: 794 # This is a nice to have but if reading this fails, it 795 # shouldn't interfere with the test. 796 logging.exception(e) 797 logging.debug('Total: sbu1 %4d sbu2 %4d', s1, s2) 798 # Use float to get values with changes 799 s1 = s1 / float(self._TOTAL_CHECK_SBU_VOLTAGE) 800 s2 = s2 / float(self._TOTAL_CHECK_SBU_VOLTAGE) 801 logging.debug('Avg: sbu1 %7.2f sbu2 %7.2f', s1, s2) 802 max_sbu = max(s1, s2) 803 logging.info('Max sbu: %7.2f', max_sbu) 804 return max_sbu 805 806 807class _Cr50OffVerifier(_BaseCr50SBUVerifier): 808 """Check if CR50 is in deep sleep and fail to detected. 809 810 If SBU voltage is higher threshold but still cannot be detected 811 as usb device then probably CR50 is in deep sleep. 812 Threshold is 2500 mV on any SBU lines. 813 """ 814 815 @ignore_exception_for_non_cros_host 816 @timeout_util.TimeoutDecorator(cros_constants.VERIFY_TIMEOUT_SEC) 817 def verify(self, host): 818 if self._is_sbu_voltage_issue(host): 819 if self._get_max_sbu_value(host) > self.SBU_THRESHOLD: 820 raise hosts.AutoservVerifyError( 821 'CR50 voltage detected but usb device not enumerated') 822 823 @property 824 def description(self): 825 return 'CR50 voltage detected but not enumerated.' 826 827 828class _Cr50LowSBUVerifier(_BaseCr50SBUVerifier): 829 """Check if servod fail to detect CR50 due low voltage. 830 831 CR50 cannot be enumerated as SBU voltage line lower then 832 threshold. 833 Threshold is 2500 mV on any SBU lines. 834 """ 835 836 @ignore_exception_for_non_cros_host 837 @timeout_util.TimeoutDecorator(cros_constants.VERIFY_TIMEOUT_SEC) 838 def verify(self, host): 839 if self._is_sbu_voltage_issue(host): 840 v = self._get_max_sbu_value(host) 841 if v > 1 and v <= self.SBU_THRESHOLD: 842 raise hosts.AutoservVerifyError( 843 'Cr50 is not detected due to SBU voltages' 844 ' being below %dmV' % self.SBU_THRESHOLD) 845 846 @property 847 def description(self): 848 return 'Cr50 not detected as both SBU voltages are below threshold.' 849 850 851class _TopologyVerifier(hosts.Verifier): 852 """Verifier that all servo component is presented.""" 853 854 @ignore_exception_for_non_cros_host 855 @timeout_util.TimeoutDecorator(cros_constants.VERIFY_TIMEOUT_SEC) 856 def verify(self, host): 857 topology = host.get_topology() 858 topology.read(host.get_dut_host_info()) 859 try: 860 # Linux takes 1 second to detect and enumerate USB device since 861 # 2010 year. We take 10 seconds to be sure as old standard was 862 # 5 seconds. 863 time.sleep(10) 864 topology.validate(raise_error=True, 865 dual_set=host.is_dual_setup(), 866 compare=True) 867 except servo_topology.ServoTopologyError as e: 868 six.reraise(hosts.AutoservVerifyError, 869 hosts.AutoservVerifyError(e), 870 sys.exc_info()[2]) 871 872 def _is_applicable(self, host): 873 if host.is_localhost(): 874 logging.info('Target servo is not in a lab,' 875 ' action is not applicable.') 876 return False 877 if not host.is_servo_topology_supported(): 878 logging.info('Target servo-topology is not supported,' 879 ' action is not applicable.') 880 return False 881 return True 882 883 @property 884 def description(self): 885 return 'Ensure all Servo component present.' 886 887 888class _PowerButtonVerifier(hosts.Verifier): 889 """ 890 Verifier to check the `pwr_button` signal. 891 892 Tests that the `pwr_button` signal shows the power button has been 893 released. When `pwr_button` is stuck at `press`, it commonly 894 indicates that the ribbon cable is disconnected. 895 """ 896 # TODO (crbug.com/646593) - Remove list below once servo has been updated 897 # with a fake pwr_button signal. 898 _BOARDS_WO_PWR_BUTTON = ['arkham', 'gale', 'mistral', 'storm', 'whirlwind'] 899 900 @ignore_exception_for_non_cros_host 901 @timeout_util.TimeoutDecorator(cros_constants.SHORT_VERIFY_TIMEOUT_SEC) 902 def verify(self, host): 903 if host.servo_board in self._BOARDS_WO_PWR_BUTTON: 904 return 905 try: 906 button = host.get_servo().get('pwr_button') 907 except Exception as e: 908 six.reraise(hosts.AutoservNonCriticalVerifyError, 909 hosts.AutoservNonCriticalVerifyError(e), 910 sys.exc_info()[2]) 911 912 if button != 'release': 913 raise hosts.AutoservNonCriticalVerifyError( 914 'Check ribbon cable: \'pwr_button\' is stuck') 915 916 def _is_applicable(self, host): 917 return (host.get_servo() and host.get_servo().main_device_is_flex()) 918 919 @property 920 def description(self): 921 return 'pwr_button control is normal' 922 923 924class _BatteryVerifier(hosts.Verifier): 925 """Collect battery info for analysis.""" 926 927 @ignore_exception_for_non_cros_host 928 @timeout_util.TimeoutDecorator(cros_constants.VERIFY_TIMEOUT_SEC) 929 def verify(self, host): 930 try: 931 servo = host.get_servo() 932 charging = False 933 if servo.has_control('battery_is_charging'): 934 charging = servo.get('battery_is_charging') 935 level = -1 936 if servo.has_control('battery_charge_percent'): 937 level = servo.get('battery_charge_percent') 938 design_mah = servo.get('battery_full_design_mah') 939 charge_mah = servo.get('battery_full_charge_mah') 940 logging.info('Charging: %s', charging) 941 logging.info('Percentage: %s', level) 942 logging.info('Full charge max: %s', charge_mah) 943 logging.info('Full design max: %s', design_mah) 944 # based on analysis of ratio we can find out what is 945 # the level when we can say that battery is dead 946 ratio = int(math.floor(charge_mah / design_mah * 100.0)) 947 logging.info('Ratio: %s', ratio) 948 data = { 949 'board': host.servo_board or 'unknown', 950 'model': host.servo_model or 'unknown', 951 'ratio': ratio 952 } 953 metrics.Counter('chromeos/autotest/battery/ratio').increment( 954 fields=data) 955 except Exception as e: 956 # Keeping it with info level because we do not expect it. 957 logging.info('(Not critical) %s', e) 958 959 def _is_applicable(self, host): 960 if not host.is_ec_supported(): 961 logging.info('The board not support EC') 962 return False 963 dut_info = host.get_dut_host_info() 964 if dut_info: 965 host_info = host.get_dut_host_info() 966 if host_info.get_label_value('power') != 'battery': 967 logging.info('The board does not have battery') 968 return False 969 servo = host.get_servo() 970 if (not servo.has_control('battery_full_design_mah') 971 or not servo.has_control('battery_full_charge_mah')): 972 logging.info('The board is not supported battery controls...') 973 return False 974 return True 975 976 @property 977 def description(self): 978 return 'Logs battery levels' 979 980 981class _LidVerifier(hosts.Verifier): 982 """ 983 Verifier to check the `lid_open` signal. 984 """ 985 986 @ignore_exception_for_non_cros_host 987 @timeout_util.TimeoutDecorator(cros_constants.SHORT_VERIFY_TIMEOUT_SEC) 988 def verify(self, host): 989 try: 990 lid_open = host.get_servo().get('lid_open') 991 except Exception as e: 992 six.reraise(hosts.AutoservNonCriticalVerifyError, 993 hosts.AutoservNonCriticalVerifyError(e), 994 sys.exc_info()[2]) 995 996 if lid_open != 'yes' and lid_open != 'not_applicable': 997 raise hosts.AutoservNonCriticalVerifyError( 998 'Check lid switch: lid_open is %s' % lid_open) 999 1000 @property 1001 def description(self): 1002 return 'lid_open control is normal' 1003 1004 1005class ECConsoleVerifier(hosts.Verifier): 1006 """ 1007 Verifier response from the EC console. 1008 """ 1009 1010 COMMAND_TO_CHECK_CONSOLE = ( 1011 'ec_system_powerstate', 1012 'ec_board', 1013 ) 1014 1015 @ignore_exception_for_non_cros_host 1016 @timeout_util.TimeoutDecorator(cros_constants.VERIFY_TIMEOUT_SEC) 1017 def verify(self, host): 1018 if not host.is_ec_supported(): 1019 logging.info('The board does not support EC') 1020 return 1021 1022 for command in self.COMMAND_TO_CHECK_CONSOLE: 1023 if host.get_servo().has_control(command): 1024 try: 1025 # Response of command is not important. 1026 r = host.get_servo().get(command) 1027 logging.debug('Result %s:%s', command, r) 1028 # Exiting as we confirmed that console is working. 1029 return 1030 except Exception as e: 1031 logging.error('Fail to read %s control. Error: %s', 1032 command, e) 1033 # If we reached this point then no command succeeded. 1034 raise hosts.AutoservNonCriticalVerifyError( 1035 'EC console is not responding; ' 1036 'may be caused of broken EC firmware') 1037 1038 @property 1039 def description(self): 1040 return 'Check EC console' 1041 1042 1043class ServodDutControllerMissingVerifier(hosts.Verifier): 1044 """Verifier to check whether the servod dut controller is missing or not. 1045 1046 When servod is initializing, it checks if DUT controller is 1047 missing. If yes,then it sets 'dut_controller_missing_fault' to 1048 'on', otherwise, to 'off'. Missing controller means servo 1049 component connected to the DUT is missing, or is not responsive. 1050 """ 1051 1052 @timeout_util.TimeoutDecorator(cros_constants.VERIFY_TIMEOUT_SEC) 1053 def verify(self, host): 1054 logging.debug('ServodDutControllerMissingVerifier: Starting verifier.') 1055 if host.get_servo().get('dut_controller_missing_fault') == 'on': 1056 logging.debug('ServodDutControllerMissingVerifier: DUT Controller missing fault is on.') 1057 raise hosts.AutoservVerifyError('Servod is missing dut controller') 1058 else: 1059 logging.debug('ServodDutControllerMissingVerifier: DUT Controller missing fault is not on.') 1060 1061 def _is_applicable(self, host): 1062 if host.is_containerized_servod(): 1063 logging.debug('ServodDutControllerMissingVerifier: Detected containerized servod.') 1064 logging.info('Servod is running within a container') 1065 return True 1066 if not host.is_labstation(): 1067 logging.debug('ServodDutControllerMissingVerifier: Detected non-labstation.') 1068 logging.info('Not supported for servo_v3.') 1069 return False 1070 return host.is_in_lab() 1071 1072 @property 1073 def description(self): 1074 return 'ensure servod does not have missing dut controller' 1075 1076 1077class _ConnectionVerifier(repair_utils.SshVerifier): 1078 """ 1079 Ensure the servo host container is up. 1080 """ 1081 1082 def verify(self, host): 1083 if host.is_containerized_servod(): 1084 # We need start servod container first before check it-is present 1085 host.start_containerized_servod() 1086 return super(_ConnectionVerifier, self).verify(host) 1087 1088 @property 1089 def description(self): 1090 return 'Check the connection to the machine or container running servod.' 1091 1092 1093class _RestartServod(hosts.RepairAction): 1094 """Restart `servod` with the proper BOARD setting.""" 1095 1096 @timeout_util.TimeoutDecorator(cros_constants.REPAIR_TIMEOUT_SEC) 1097 def repair(self, host): 1098 if host.is_containerized_servod(): 1099 logging.debug('Restarting servod container') 1100 elif not host.is_cros_host(): 1101 raise hosts.AutoservRepairError( 1102 'Can\'t restart servod: not running ' 1103 'embedded ChromeOS.', 1104 'servo_not_applicable_to_non_cros_host') 1105 host.restart_servod() 1106 1107 @property 1108 def description(self): 1109 return 'Start servod with the proper config settings.' 1110 1111 1112class _ServoRebootRepair(repair_utils.RebootRepair): 1113 """Try repair servo by reboot servohost. 1114 1115 This is the same as the standard `RebootRepair`, for servo_v3 it will 1116 reboot the beaglebone board immediately while for labstation it will 1117 request a reboot by touch a flag file on its labstation, then 1118 labstation reboot will be handled by labstation AdminRepair task as 1119 labstation host multiple servos and need do an synchronized reboot. 1120 """ 1121 1122 @timeout_util.TimeoutDecorator(cros_constants.REPAIR_TIMEOUT_SEC) 1123 def repair(self, host): 1124 super(_ServoRebootRepair, self).repair(host) 1125 # restart servod for v3 after reboot. 1126 host.restart_servod() 1127 1128 def _is_applicable(self, host): 1129 if host.is_localhost() or not host.is_cros_host(): 1130 logging.info('Target servo is not in a lab, the reboot repair' 1131 ' action is not applicable.') 1132 return False 1133 1134 if host.is_labstation(): 1135 host.request_reboot() 1136 logging.info('Reboot labstation requested, it will be handled' 1137 ' by labstation AdminRepair task.') 1138 return False 1139 return True 1140 1141 @property 1142 def description(self): 1143 return 'Reboot the servo host.' 1144 1145 1146class _ToggleCCLineRepair(hosts.RepairAction): 1147 """Try repair servod by toggle cc. 1148 1149 When cr50 is not enumerated we can try to recover it by toggle cc line. 1150 """ 1151 # Timeout for shut down configuration channel. 1152 CC_OFF_TIMEOUT = 10 1153 # Timeout for initialize configuration channel. 1154 CC_ON_TIMEOUT = 30 1155 1156 @timeout_util.TimeoutDecorator(cros_constants.REPAIR_TIMEOUT_SEC) 1157 def repair(self, host): 1158 logging.info('Turn off configuration channel and wait 10 seconds.') 1159 servo_uart_cmd = 'servo_v4_uart_cmd' 1160 if not host.get_servo().has_control(servo_uart_cmd): 1161 servo_uart_cmd = 'servo_v4p1_uart_cmd' 1162 host.get_servo().set_nocheck(servo_uart_cmd, 'cc off') 1163 # wait till command will be effected 1164 time.sleep(self.CC_OFF_TIMEOUT) 1165 1166 logging.info('Turn on configuration channel and wait 30 seconds.') 1167 # alternative option to turn line on is by `cc srcdts` 1168 host.get_servo().set_nocheck('servo_pd_role', 'src') 1169 host.get_servo().set_nocheck('servo_dts_mode', 'on') 1170 # wait till command will be effected 1171 time.sleep(self.CC_ON_TIMEOUT) 1172 host.restart_servod() 1173 1174 def _is_applicable(self, host): 1175 if host.is_localhost(): 1176 logging.debug('Not supported for localhost.') 1177 return False 1178 if not host.servo_serial: 1179 logging.debug('Servod does not have serial.') 1180 return False 1181 if not host.servo_recovery: 1182 logging.debug('Servod is not running in recovery mode.') 1183 return False 1184 if not (host.is_labstation() or host.is_containerized_servod()): 1185 logging.debug('Not supported for servo_v3.') 1186 return False 1187 if not host.get_servo(): 1188 logging.debug('Servo is not initialized.') 1189 return False 1190 return self._is_type_c(host) 1191 1192 def _is_type_c(self, host): 1193 if host.get_dut_host_info(): 1194 servo_type = host.get_dut_host_info().get_label_value( 1195 servo_constants.SERVO_TYPE_LABEL_PREFIX) 1196 return 'ccd' in servo_type 1197 return False 1198 1199 @property 1200 def description(self): 1201 return 'Toggle cc lines' 1202 1203 1204class _FakedisconnectRepair(hosts.RepairAction): 1205 """Try repair servod by mimic reconnection of servo. 1206 1207 When cr50 is not enumerated as we can try to recover it by reconnect to DUT. 1208 """ 1209 # Delay to disconnect. 1210 DISC_DELAY_MS = 100 1211 # Timeout to wait to restore the connection. 1212 DISC_TIMEOUT_MS = 2000 1213 # Timeout to wait to execute the command and apply effect. 1214 EXEC_TIMEOUT = (DISC_DELAY_MS + DISC_TIMEOUT_MS) / 1000 + 2 1215 1216 @timeout_util.TimeoutDecorator(cros_constants.REPAIR_TIMEOUT_SEC) 1217 def repair(self, host): 1218 disc_cmd = ('fakedisconnect %d %d' % 1219 (self.DISC_DELAY_MS, self.DISC_TIMEOUT_MS)) 1220 # cannot use 'set' as control is not returned executed commands 1221 servo_uart_cmd = 'servo_v4_uart_cmd' 1222 if not host.get_servo().has_control(servo_uart_cmd): 1223 servo_uart_cmd = 'servo_v4p1_uart_cmd' 1224 host.get_servo().set_nocheck(servo_uart_cmd, disc_cmd) 1225 logging.debug('Waiting %ss for affect of action', self.EXEC_TIMEOUT) 1226 time.sleep(self.EXEC_TIMEOUT) 1227 host.restart_servod() 1228 1229 def _is_applicable(self, host): 1230 if host.is_localhost(): 1231 logging.debug('Not supported for localhost.') 1232 return False 1233 if not host.servo_serial: 1234 logging.debug('Servod does not have serial.') 1235 return False 1236 if not host.servo_recovery: 1237 logging.debug('Servod is not running in recovery mode.') 1238 return False 1239 if not (host.is_labstation() or host.is_containerized_servod()): 1240 logging.debug('Not supported for servo_v3.') 1241 return False 1242 if not host.get_servo(): 1243 logging.debug('Servo is not initialized.') 1244 return False 1245 return self._is_type_c(host) 1246 1247 def _is_type_c(self, host): 1248 if host.get_dut_host_info(): 1249 servo_type = host.get_dut_host_info().get_label_value( 1250 servo_constants.SERVO_TYPE_LABEL_PREFIX) 1251 return 'ccd' in servo_type 1252 return False 1253 1254 @property 1255 def description(self): 1256 return 'Fake reconnect to DUT' 1257 1258 1259class _PowerDeliveryRepair(hosts.RepairAction): 1260 """Repair to check servo_v4_role for servos that support 1261 power delivery feature(a.k.a power pass through). 1262 1263 There are currently two position of servo_v4_role, src and snk: 1264 src -- servo in power delivery mode and passes power to the DUT. 1265 snk -- servo in normal mode and not passes power to DUT. 1266 """ 1267 # How many time retry to set PD in correct mode and verify that is stay. 1268 # Set 5 as each attempt has 10 attempts inside 'set' method. 1269 _SET_ATTEMPT_COUNT = 5 1270 1271 @timeout_util.TimeoutDecorator(cros_constants.REPAIR_TIMEOUT_SEC) 1272 def repair(self, host): 1273 host.get_servo().set_nocheck('servo_pd_role', 'snk') 1274 time.sleep(1) 1275 for x in range(self._SET_ATTEMPT_COUNT): 1276 logging.debug('Try set servo_v4_role to src.' 1277 ' Attempt: %s', x + 1) 1278 try: 1279 host.get_servo().set('servo_pd_role', 'src') 1280 # Waiting a few seconds as it can be change to snk if PD 1281 # on servo has issue. 1282 time.sleep(5) 1283 except BaseException as e: 1284 logging.debug('Setting PD with retries failed %s', e) 1285 if host.get_servo().get('servo_pd_role') == 'src': 1286 break 1287 if host.get_servo().get('servo_pd_role') == 'snk': 1288 raise hosts.AutoservNonCriticalVerifyError( 1289 'Cannot switch power delivery to the src role') 1290 # Restart servod to re-initialize servos. 1291 # In some cases if device did not receive power can block detection 1292 # of servo components. 1293 host.restart_servod() 1294 1295 def _is_type_c(self, host): 1296 return (host.is_in_lab() and host.get_servo() 1297 and host.get_servo().supports_built_in_pd_control()) 1298 1299 @property 1300 def description(self): 1301 return 'Recover power delivery on servo' 1302 1303 1304class _ECRebootRepair(hosts.RepairAction): 1305 """ 1306 Reboot EC on DUT from servo. 1307 """ 1308 1309 def _is_applicable(self, host): 1310 return (not host.is_localhost()) and host.is_ec_supported() 1311 1312 @timeout_util.TimeoutDecorator(cros_constants.REPAIR_TIMEOUT_SEC) 1313 def repair(self, host): 1314 host.get_servo().ec_reboot() 1315 1316 @property 1317 def description(self): 1318 return 'Reboot EC' 1319 1320 1321class _DutRebootRepair(hosts.RepairAction): 1322 """ 1323 Reboot DUT to recover some servo controls depending on EC console. 1324 1325 Some servo controls, like lid_open, requires communicating with DUT through 1326 EC UART console. Failure of this kinds of controls can be recovered by 1327 rebooting the DUT. 1328 """ 1329 1330 @timeout_util.TimeoutDecorator(cros_constants.REPAIR_TIMEOUT_SEC) 1331 def repair(self, host): 1332 host.get_servo().get_power_state_controller().reset() 1333 # Get the lid_open value which requires EC console. 1334 lid_open = host.get_servo().get('lid_open') 1335 if lid_open != 'yes' and lid_open != 'not_applicable': 1336 raise hosts.AutoservVerifyError( 1337 'Still fail to contact EC console after rebooting DUT') 1338 1339 @property 1340 def description(self): 1341 return 'Reset the DUT via servo' 1342 1343 1344class _DiskCleanupRepair(hosts.RepairAction): 1345 """ 1346 Remove old logs/metrics/crash_dumps on servohost to free up disk space. 1347 """ 1348 KEEP_LOGS_MAX_DAYS = 5 1349 1350 FILE_TO_REMOVE = [ 1351 '/var/lib/metrics/uma-events', '/var/spool/crash/*', 1352 '/var/log/chrome/*', '/var/log/ui/*', 1353 '/home/chronos/BrowserMetrics/*' 1354 ] 1355 1356 @timeout_util.TimeoutDecorator(cros_constants.SHORT_REPAIR_TIMEOUT_SEC) 1357 def repair(self, host): 1358 if host.is_localhost(): 1359 # we don't want to remove anything from local testing. 1360 return 1361 1362 # Remove old servod logs. 1363 host.run('/usr/bin/find /var/log/servod_* -mtime +%d -print -delete' 1364 % self.KEEP_LOGS_MAX_DAYS, ignore_status=True) 1365 1366 # Remove pre-defined metrics and crash dumps. 1367 for path in self.FILE_TO_REMOVE: 1368 host.run('rm %s' % path, ignore_status=True) 1369 1370 @property 1371 def description(self): 1372 return 'Clean up old logs/metrics on servohost to free up disk space.' 1373 1374 1375class _ServoFwUpdateRepair(hosts.RepairAction): 1376 """Update firmware for servos. 1377 1378 We try to update servo 3 times and then try to force update it. 1379 """ 1380 1381 @timeout_util.TimeoutDecorator(cros_constants.REPAIR_TIMEOUT_SEC) 1382 def repair(self, host): 1383 try: 1384 servo_updater.update_servo_firmware(host, 1385 try_attempt_count=3, 1386 force_update=False, 1387 try_force_update=True) 1388 except servo_updater.ServoUpdaterError as er: 1389 # Catch servo_updater issue to cache it. 1390 self.servo_updater_issue_detected = True 1391 raise hosts.AutoservVerifyError('ServoUpdater issue detected') 1392 1393 def _is_applicable(self, host): 1394 # Run only for servo_v4 and servo_v4p1. 1395 return host.is_labstation() or host.is_containerized_servod() 1396 1397 @property 1398 def description(self): 1399 return 'Update servo-fw if required.' 1400 1401 1402class _ServoMicroFlashRepair(hosts.RepairAction): 1403 """ 1404 Remove old logs/metrics/crash_dumps on servohost to free up disk space. 1405 """ 1406 _TARGET_SERVO = 'servo_micro' 1407 1408 @timeout_util.TimeoutDecorator(cros_constants.REPAIR_TIMEOUT_SEC) 1409 def repair(self, host): 1410 if not host.is_cros_host(): 1411 raise hosts.AutoservRepairError( 1412 'Can\'t restart servod: not running ' 1413 'embedded ChromeOS.', 1414 'servo_not_applicable_to_non_cros_host') 1415 servo = host.get_servo() 1416 if not servo or self._TARGET_SERVO not in servo.get_servo_type(): 1417 logging.info("Servo-micro is not present on set-up") 1418 return 1419 1420 try: 1421 servo_updater.update_servo_firmware(host, 1422 boards=(self._TARGET_SERVO, ), 1423 force_update=True, 1424 ignore_version=True) 1425 except Exception as e: 1426 logging.debug("(Not critical) Servo device update error: %s", e) 1427 raise hosts.AutoservVerifyError( 1428 'Still fail to contact EC console after rebooting DUT') 1429 # Update time when we reflashed the fw on the device 1430 dhp = host.get_dut_health_profile() 1431 dhp.refresh_servo_miro_fw_update_run_time() 1432 host.restart_servod() 1433 1434 def is_time_to_try(self, dhp): 1435 """Verify that it is time when we can try to re-flash fw on servo_micro. 1436 1437 Re-flashing limited to once per 2 weeks to avoid over-flashing 1438 the servo device. 1439 """ 1440 today_time = int(time.time()) 1441 last_check = dhp.get_servo_micro_fw_update_time_epoch() 1442 can_run = today_time > (last_check + (14 * 24 * 60 * 60)) 1443 if not can_run: 1444 logging.info("The servo_micro fw updated in las 2 weeks ago.") 1445 return can_run 1446 1447 def _is_applicable(self, host): 1448 return (not host.is_localhost() and host.get_dut_health_profile() 1449 and self.is_time_to_try(host.get_dut_health_profile())) 1450 1451 @property 1452 def description(self): 1453 return 'Re-flash servo_micro firmware.' 1454 1455 1456def _servo_verifier_actions(): 1457 """ 1458 Return a verifiers for a `ServoHost`. 1459 """ 1460 return ( 1461 (_ConnectionVerifier, 'connection', []), 1462 (_RootServoPresentVerifier, 'servo_root_present', ['connection']), 1463 (_RootServoV3PresentVerifier, 'servo_v3_root_present', 1464 ['connection']), 1465 (_ServoFwVerifier, 'servo_fw', ['servo_root_present']), 1466 (_StartServodVerifier, 'start_servod', 1467 ['servo_fw', 'servo_v3_root_present']), 1468 (_DiskSpaceVerifier, 'servo_disk_space', ['connection']), 1469 (_UpdateVerifier, 'servo_update', ['servo_v3_root_present']), 1470 (_BoardConfigVerifier, 'servo_config_board', ['connection']), 1471 (_SerialConfigVerifier, 'servo_config_serial', ['connection']), 1472 (_ServodJobVerifier, 'servod_started', [ 1473 'start_servod', 'servo_config_board', 1474 'servo_config_serial', 'servo_disk_space' 1475 ]), 1476 (_ServodEchoVerifier, 'servod_echo', ['servod_started']), 1477 (_TopologyVerifier, 'servo_topology', ['servod_echo']), 1478 (_ServodConnectionVerifier, 'servod_connection', ['servod_echo']), 1479 (_Cr50LowSBUVerifier, 'servo_cr50_low_sbu', ['servod_connection']), 1480 (ServodDutControllerMissingVerifier, 1481 'servod_dut_controller_missing', ['servod_connection']), 1482 (_Cr50OffVerifier, 'servo_cr50_off', ['servod_connection']), 1483 (_ServodControlVerifier, 'servod_control', ['servod_connection']), 1484 (_DUTConnectionVerifier, 'servo_dut_connected', 1485 ['servod_connection']), 1486 (_ServoHubConnectionVerifier, 'servo_hub_connected', 1487 ['servo_dut_connected']), 1488 (_PowerButtonVerifier, 'servo_pwr_button', ['servo_hub_connected' 1489 ]), 1490 (_BatteryVerifier, 'servo_battery', ['servo_hub_connected']), 1491 (_LidVerifier, 'servo_lid_open', ['servo_hub_connected']), 1492 (ECConsoleVerifier, 'servo_ec_console', ['servo_dut_connected']), 1493 (_Cr50ConsoleVerifier, 'servo_cr50_console', 1494 ['servo_dut_connected']), 1495 (_CCDTestlabVerifier, 'servo_ccd_testlab', ['servo_cr50_console']), 1496 (_CCDPowerDeliveryVerifier, 'servo_power_delivery', 1497 ['servod_connection']), 1498 ) 1499 1500 1501def _servo_repair_actions(): 1502 """ 1503 Return a `RepairStrategy` for a `ServoHost`. 1504 """ 1505 config = ['servo_config_board', 'servo_config_serial', 'start_servod'] 1506 base_triggers = [ 1507 'servod_started', 'servo_topology', 'servod_connection', 1508 'servod_echo', 'servod_control', 'servo_dut_connected', 1509 'servo_hub_connected', 'servo_pwr_button', 'servo_cr50_console', 1510 'servo_cr50_low_sbu', 'servo_cr50_off', 'servo_power_delivery', 1511 'servod_dut_controller_missing' 1512 ] 1513 dut_triggers = [ 1514 'servod_control', 'servo_lid_open', 'servo_ec_console', 1515 'servo_topology', 'servo_dut_connected', 'servo_hub_connected', 1516 'servo_cr50_low_sbu', 'servo_cr50_off', 'servo_cr50_console', 1517 'servo_power_delivery', 'servod_dut_controller_missing' 1518 ] 1519 reboot_triggers = [ 1520 'servo_topology', 'servo_root_present', 'servo_disk_space', 1521 'servo_power_delivery' 1522 ] 1523 return ( 1524 (_ServoFwUpdateRepair, 'servo_fw_update', ['connection'], 1525 ['servo_fw']), 1526 (_DiskCleanupRepair, 'servo_disk_cleanup', ['connection'], 1527 ['servo_disk_space']), 1528 (_ServoMicroFlashRepair, 'servo_micro_flash', 1529 ['connection', 'servo_topology'], ['servo_dut_connected']), 1530 (_RestartServod, 'servod_restart', ['connection', 'servo_fw'], 1531 config + base_triggers), 1532 (_ServoRebootRepair, 'servo_reboot', ['connection'], 1533 reboot_triggers), 1534 (_PowerDeliveryRepair, 'servo_pd_recover', ['servod_connection'], 1535 base_triggers), 1536 (_FakedisconnectRepair, 'servo_fakedisconnect', 1537 ['servod_connection'], base_triggers), 1538 (_ToggleCCLineRepair, 'servo_cc', ['servod_connection'], 1539 base_triggers), 1540 (_DutRebootRepair, 'servo_dut_reboot', ['servod_connection'], 1541 dut_triggers), 1542 (_ECRebootRepair, 'servo_ec_reboot', ['servod_connection'], 1543 dut_triggers), 1544 ) 1545 1546 1547def create_servo_repair_strategy(): 1548 """ 1549 Return a `RepairStrategy` for a `ServoHost`. 1550 """ 1551 return hosts.RepairStrategy(_servo_verifier_actions(), 1552 _servo_repair_actions(), 'servo') 1553