1# Lint as: python2, python3 2# Copyright (c) 2019 The Chromium OS Authors. All rights reserved. 3# Use of this source code is governed by a BSD-style license that can be 4# found in the LICENSE file. 5# 6# Expects to be run in an environment with sudo and no interactive password 7# prompt, such as within the Chromium OS development chroot. 8 9 10"""This is a base host class for servohost and labstation.""" 11 12 13import six.moves.http_client 14import logging 15import socket 16import six.moves.xmlrpc_client 17import time 18import os 19 20try: 21 import docker 22 from autotest_lib.site_utils.docker import utils as docker_utils 23except ImportError: 24 logging.info("Docker API is not installed in this environment") 25 26from autotest_lib.client.bin import utils 27from autotest_lib.client.common_lib import autotest_enum 28from autotest_lib.client.common_lib import error 29from autotest_lib.client.common_lib import hosts 30from autotest_lib.client.common_lib import lsbrelease_utils 31from autotest_lib.client.common_lib.cros import dev_server 32from autotest_lib.client.common_lib.cros import kernel_utils 33from autotest_lib.client.cros import constants as client_constants 34from autotest_lib.server import autotest 35from autotest_lib.server import site_utils as server_utils 36from autotest_lib.server.cros import provisioner 37from autotest_lib.server.hosts import ssh_host 38from autotest_lib.site_utils.rpm_control_system import rpm_client 39 40 41class BaseServoHost(ssh_host.SSHHost): 42 """Base host class for a host that manage servo(s). 43 E.g. beaglebone, labstation. 44 """ 45 REBOOT_CMD = 'sleep 5; reboot & sleep 10; reboot -f' 46 47 TEMP_FILE_DIR = '/var/lib/servod/' 48 49 LOCK_FILE_POSTFIX = '_in_use' 50 REBOOT_FILE_POSTFIX = '_reboot' 51 52 # Time to wait a rebooting servohost, in seconds. 53 REBOOT_TIMEOUT = 240 54 55 # Timeout value to power cycle a servohost, in seconds. 56 BOOT_TIMEOUT = 240 57 58 # Constants that reflect current host update state. 59 UPDATE_STATE = autotest_enum.AutotestEnum('IDLE', 'RUNNING', 60 'PENDING_REBOOT') 61 62 def _initialize(self, 63 hostname, 64 is_in_lab=None, 65 servo_host_ssh_port=None, 66 servod_docker=None, 67 *args, 68 **dargs): 69 """Construct a BaseServoHost object. 70 71 @param is_in_lab: True if the servo host is in Cros Lab. Default is set 72 to None, for which utils.host_is_in_lab_zone will be 73 called to check if the servo host is in Cros lab. 74 75 """ 76 if servo_host_ssh_port is not None: 77 dargs['port'] = int(servo_host_ssh_port) 78 79 super(BaseServoHost, self)._initialize(hostname=hostname, 80 *args, **dargs) 81 82 self.servod_container_name = None 83 self._is_containerized_servod = False 84 if bool(servod_docker): 85 self._is_containerized_servod = True 86 self.servod_container_name = servod_docker 87 elif self.hostname.endswith('docker_servod'): 88 # For backward compatibility 89 self.servod_container_name = self.hostname 90 self._is_containerized_servod = True 91 92 self._is_localhost = (self.hostname == 'localhost' 93 and servo_host_ssh_port is None) 94 if self._is_localhost or self._is_containerized_servod: 95 self._is_in_lab = False 96 elif is_in_lab is None: 97 self._is_in_lab = (utils.host_is_in_lab_zone(self.hostname) 98 or self.is_satlab()) 99 else: 100 self._is_in_lab = is_in_lab 101 102 # Commands on the servo host must be run by the superuser. 103 # Our account on a remote host is root, but if our target is 104 # localhost then we might be running unprivileged. If so, 105 # `sudo` will have to be added to the commands. 106 if self._is_localhost: 107 self._sudo_required = utils.system_output('id -u') != '0' 108 else: 109 self._sudo_required = False 110 111 self._is_labstation = None 112 self._dut_host_info = None 113 self._dut_hostname = None 114 115 116 def get_board(self): 117 """Determine the board for this servo host. E.g. fizz-labstation 118 119 @returns a string representing this labstation's board or None if 120 target host is not using a ChromeOS image(e.g. test in chroot). 121 """ 122 output = self.run('cat /etc/lsb-release', ignore_status=True).stdout 123 return lsbrelease_utils.get_current_board(lsb_release_content=output) 124 125 126 def set_dut_host_info(self, dut_host_info): 127 """ 128 @param dut_host_info: A HostInfo object. 129 """ 130 logging.info('setting dut_host_info field to (%s)', dut_host_info) 131 self._dut_host_info = dut_host_info 132 133 134 def get_dut_host_info(self): 135 """ 136 @return A HostInfo object. 137 """ 138 return self._dut_host_info 139 140 141 def set_dut_hostname(self, dut_hostname): 142 """ 143 @param dut_hostname: hostname of the DUT that connected to this servo. 144 """ 145 logging.info('setting dut_hostname as (%s)', dut_hostname) 146 self._dut_hostname = dut_hostname 147 148 149 def get_dut_hostname(self): 150 """ 151 @returns hostname of the DUT that connected to this servo. 152 """ 153 return self._dut_hostname 154 155 156 def is_labstation(self): 157 """Determine if the host is a labstation 158 159 @returns True if ths host is a labstation otherwise False. 160 """ 161 if self.is_containerized_servod(): 162 return False 163 164 if self._is_labstation is None: 165 if 'labstation' in self.hostname: 166 logging.info('Based on hostname, the servohost is' 167 ' a labstation.') 168 self._is_labstation = True 169 else: 170 logging.info( 171 'Cannot determine if %s is a labstation from' 172 ' hostname, getting board info from the' 173 ' servohost.', self.hostname) 174 board = self.get_board() 175 self._is_labstation = bool(board) and 'labstation' in board 176 177 return self._is_labstation 178 179 180 def _get_lsb_release_content(self): 181 """Return the content of lsb-release file of host.""" 182 return self.run( 183 'cat "%s"' % client_constants.LSB_RELEASE).stdout.strip() 184 185 186 def get_release_version(self): 187 """Get the value of attribute CHROMEOS_RELEASE_VERSION from lsb-release. 188 189 @returns The version string in lsb-release, under attribute 190 CHROMEOS_RELEASE_VERSION(e.g. 12900.0.0). None on fail. 191 """ 192 return lsbrelease_utils.get_chromeos_release_version( 193 lsb_release_content=self._get_lsb_release_content() 194 ) 195 196 197 def get_full_release_path(self): 198 """Get full release path from servohost as string. 199 200 @returns full release path as a string 201 (e.g. fizz-labstation-release/R82.12900.0.0). None on fail. 202 """ 203 return lsbrelease_utils.get_chromeos_release_builder_path( 204 lsb_release_content=self._get_lsb_release_content() 205 ) 206 207 208 def _check_update_status(self): 209 """ Check servohost's current update state. 210 211 @returns: one of below state of from self.UPDATE_STATE 212 IDLE -- if the target host is not currently updating and not 213 pending on a reboot. 214 RUNNING -- if there is another updating process that running on 215 target host(note: we don't expect to hit this scenario). 216 PENDING_REBOOT -- if the target host had an update and pending 217 on reboot. 218 """ 219 result = self.run('pgrep -f quick-provision | grep -v $$', 220 ignore_status=True) 221 # We don't expect any output unless there are another quick 222 # provision process is running. 223 if result.exit_status == 0: 224 return self.UPDATE_STATE.RUNNING 225 226 # Determine if we have an update that pending on reboot by check if 227 # the current inactive kernel has priority for the next boot. 228 try: 229 inactive_kernel = kernel_utils.get_kernel_state(self)[1] 230 next_kernel = kernel_utils.get_next_kernel(self) 231 if inactive_kernel == next_kernel: 232 return self.UPDATE_STATE.PENDING_REBOOT 233 except Exception as e: 234 logging.error('Unexpected error while checking kernel info; %s', e) 235 return self.UPDATE_STATE.IDLE 236 237 238 def is_in_lab(self): 239 """Check whether the servo host is a lab device. 240 241 @returns: True if the servo host is in Cros Lab, otherwise False. 242 243 """ 244 return self._is_in_lab 245 246 247 def is_localhost(self): 248 """Checks whether the servo host points to localhost. 249 250 @returns: True if it points to localhost, otherwise False. 251 252 """ 253 return self._is_localhost 254 255 256 def is_containerized_servod(self): 257 """Checks whether the servo host is a containerized servod. 258 259 @returns: True if using containerized servod, otherwise False. 260 261 """ 262 return self._is_containerized_servod 263 264 def is_cros_host(self): 265 """Check if a servo host is running chromeos. 266 267 @return: True if the servo host is running chromeos. 268 False if it isn't, or we don't have enough information. 269 """ 270 if self.is_containerized_servod(): 271 return False 272 try: 273 result = self.run('grep -q CHROMEOS /etc/lsb-release', 274 ignore_status=True, timeout=10) 275 except (error.AutoservRunError, error.AutoservSSHTimeout): 276 return False 277 return result.exit_status == 0 278 279 280 def prepare_for_update(self): 281 """Prepares the DUT for an update. 282 Subclasses may override this to perform any special actions 283 required before updating. 284 """ 285 pass 286 287 288 def reboot(self, *args, **dargs): 289 """Reboot using special servo host reboot command.""" 290 super(BaseServoHost, self).reboot(reboot_cmd=self.REBOOT_CMD, 291 *args, **dargs) 292 293 294 def update_image(self, stable_version=None): 295 """Update the image on the servo host, if needed. 296 297 This method recognizes the following cases: 298 * If the Host is not running ChromeOS, do nothing. 299 * If a previously triggered update is now complete, reboot 300 to the new version. 301 * If the host is processing an update do nothing. 302 * If the host has an update that pending on reboot, do nothing. 303 * If the host is running a version of ChromeOS different 304 from the default for servo Hosts, start an update. 305 306 @stable_version the target build number.(e.g. R82-12900.0.0) 307 308 @raises dev_server.DevServerException: If all the devservers are down. 309 @raises site_utils.ParseBuildNameException: If the devserver returns 310 an invalid build name. 311 """ 312 # servod could be running in a Ubuntu workstation. 313 if not self.is_cros_host(): 314 logging.info('Not attempting an update, either %s is not running ' 315 'chromeos or we cannot find enough information about ' 316 'the host.', self.hostname) 317 return 318 319 if lsbrelease_utils.is_moblab(): 320 logging.info('Not attempting an update, %s is running moblab.', 321 self.hostname) 322 return 323 324 if not stable_version: 325 logging.debug("BaseServoHost::update_image attempting to get" 326 " servo cros stable version") 327 try: 328 stable_version = (self.get_dut_host_info(). 329 servo_cros_stable_version) 330 except AttributeError: 331 logging.error("BaseServoHost::update_image failed to get" 332 " servo cros stable version.") 333 334 target_build = "%s-release/%s" % (self.get_board(), stable_version) 335 target_build_number = server_utils.ParseBuildName( 336 target_build)[3] 337 current_build_number = self.get_release_version() 338 339 if current_build_number == target_build_number: 340 logging.info('servo host %s does not require an update.', 341 self.hostname) 342 return 343 344 status = self._check_update_status() 345 if status == self.UPDATE_STATE.RUNNING: 346 logging.info('servo host %s already processing an update', 347 self.hostname) 348 return 349 if status == self.UPDATE_STATE.PENDING_REBOOT: 350 # Labstation reboot is handled separately here as it require 351 # synchronized reboot among all managed DUTs. For servo_v3, we'll 352 # reboot when initialize Servohost, if there is a update pending. 353 logging.info('An update has been completed and pending reboot.') 354 return 355 356 ds = dev_server.ImageServer.resolve(self.hostname, 357 hostname=self.hostname) 358 url = ds.get_update_url(target_build) 359 cros_provisioner = provisioner.ChromiumOSProvisioner(update_url=url, 360 host=self, 361 is_servohost=True) 362 logging.info('Using devserver url: %s to trigger update on ' 363 'servo host %s, from %s to %s', url, self.hostname, 364 current_build_number, target_build_number) 365 cros_provisioner.run_provision() 366 367 368 def has_power(self): 369 """Return whether or not the servo host is powered by PoE or RPM.""" 370 # TODO(fdeng): See crbug.com/302791 371 # For now, assume all servo hosts in the lab have power. 372 return self.is_in_lab() 373 374 375 def _post_update_reboot(self): 376 """ Reboot servohost after an quick provision. 377 378 We need to do some specifal cleanup before and after reboot 379 when there is an update pending. 380 """ 381 # Regarding the 'crossystem' command below: In some cases, 382 # the update flow puts the TPM into a state such that it 383 # fails verification. We don't know why. However, this 384 # call papers over the problem by clearing the TPM during 385 # the reboot. 386 # 387 # We ignore failures from 'crossystem'. Although failure 388 # here is unexpected, and could signal a bug, the point of 389 # the exercise is to paper over problems; allowing this to 390 # fail would defeat the purpose. 391 392 # Preserve critical files before reboot since post-provision 393 # clobbering will wipe the stateful partition. 394 # TODO(xianuowang@) Remove this logic once we have updated to 395 # a image with https://crrev.com/c/2485908. 396 path_to_preserve = [ 397 '/var/lib/servod', 398 '/var/lib/device_health_profile', 399 ] 400 safe_location = '/mnt/stateful_partition/unencrypted/preserve/' 401 for item in path_to_preserve: 402 dest = os.path.join(safe_location, item.split('/')[-1]) 403 self.run('rm -rf %s' % dest, ignore_status=True) 404 self.run('mv %s %s' % (item, safe_location), ignore_status=True) 405 406 self.run('crossystem clear_tpm_owner_request=1', ignore_status=True) 407 self._servo_host_reboot() 408 logging.debug('Cleaning up autotest directories if exist.') 409 try: 410 installed_autodir = autotest.Autotest.get_installed_autodir(self) 411 self.run('rm -rf ' + installed_autodir) 412 except autotest.AutodirNotFoundError: 413 logging.debug('No autotest installed directory found.') 414 415 # Recover preserved files to original location. 416 # TODO(xianuowang@) Remove this logic once we have updated to 417 # a image with https://crrev.com/c/2485908. 418 for item in path_to_preserve: 419 src = os.path.join(safe_location, item.split('/')[-1]) 420 dest = '/'.join(item.split('/')[:-1]) 421 self.run('mv %s %s' % (src, dest), ignore_status=True) 422 423 def power_cycle(self): 424 """Cycle power to this host via PoE(servo v3) or RPM(labstation) 425 if it is a lab device. 426 427 @raises AutoservRepairError if it fails to power cycle the 428 servo host. 429 430 """ 431 if self.has_power(): 432 try: 433 rpm_client.set_power(self, 'CYCLE') 434 except (socket.error, six.moves.xmlrpc_client.Error, 435 six.moves.http_client.BadStatusLine, 436 rpm_client.RemotePowerException) as e: 437 raise hosts.AutoservRepairError( 438 'Power cycling %s failed: %s' % (self.hostname, e), 439 'power_cycle_via_rpm_failed' 440 ) 441 else: 442 logging.info('Skipping power cycling, not a lab device.') 443 444 445 def _servo_host_reboot(self): 446 """Reboot this servo host because a reboot is requested.""" 447 try: 448 # TODO(otabek) remove if found the fix for b/174514811 449 # The default factory firmware remember the latest chromeboxes 450 # status after power off. If box was in sleep mode before the 451 # break, the box will stay at sleep mode after power on. 452 # Disable power manager has make chromebox to boot always when 453 # we deliver the power to the device. 454 logging.info('Stoping powerd service on device') 455 self.run('stop powerd', ignore_status=True, timeout=30) 456 except Exception as e: 457 logging.debug('(Not critical) Fail to stop powerd; %s', e) 458 459 logging.info('Rebooting servo host %s from build %s', self.hostname, 460 self.get_release_version()) 461 # Tell the reboot() call not to wait for completion. 462 # Otherwise, the call will log reboot failure if servo does 463 # not come back. The logged reboot failure will lead to 464 # test job failure. If the test does not require servo, we 465 # don't want servo failure to fail the test with error: 466 # `Host did not return from reboot` in status.log. 467 self.reboot(fastsync=True, wait=False) 468 469 # We told the reboot() call not to wait, but we need to wait 470 # for the reboot before we continue. Alas. The code from 471 # here below is basically a copy of Host.wait_for_restart(), 472 # with the logging bits ripped out, so that they can't cause 473 # the failure logging problem described above. 474 # 475 # The stain that this has left on my soul can never be 476 # erased. 477 old_boot_id = self.get_boot_id() 478 if not self.wait_down(timeout=self.WAIT_DOWN_REBOOT_TIMEOUT, 479 warning_timer=self.WAIT_DOWN_REBOOT_WARNING, 480 old_boot_id=old_boot_id): 481 raise error.AutoservHostError( 482 'servo host %s failed to shut down.' % 483 self.hostname) 484 if self.wait_up(timeout=self.REBOOT_TIMEOUT): 485 logging.info('servo host %s back from reboot, with build %s', 486 self.hostname, self.get_release_version()) 487 else: 488 raise error.AutoservHostError( 489 'servo host %s failed to come back from reboot.' % 490 self.hostname) 491 492 493 def make_ssh_command(self, user='root', port=22, opts='', hosts_file=None, 494 connect_timeout=None, alive_interval=None, alive_count_max=None, 495 connection_attempts=None): 496 """Override default make_ssh_command to use tuned options. 497 498 Tuning changes: 499 - ConnectTimeout=30; maximum of 30 seconds allowed for an SSH 500 connection failure. Consistency with remote_access.py. 501 502 - ServerAliveInterval=180; which causes SSH to ping connection every 503 180 seconds. In conjunction with ServerAliveCountMax ensures 504 that if the connection dies, Autotest will bail out quickly. 505 506 - ServerAliveCountMax=3; consistency with remote_access.py. 507 508 - ConnectAttempts=4; reduce flakiness in connection errors; 509 consistency with remote_access.py. 510 511 - UserKnownHostsFile=/dev/null; we don't care about the keys. 512 513 - SSH protocol forced to 2; needed for ServerAliveInterval. 514 515 @param user User name to use for the ssh connection. 516 @param port Port on the target host to use for ssh connection. 517 @param opts Additional options to the ssh command. 518 @param hosts_file Ignored. 519 @param connect_timeout Ignored. 520 @param alive_interval Ignored. 521 @param alive_count_max Ignored. 522 @param connection_attempts Ignored. 523 524 @returns: An ssh command with the requested settings. 525 526 """ 527 options = ' '.join([opts, '-o Protocol=2']) 528 return super(BaseServoHost, self).make_ssh_command( 529 user=user, port=port, opts=options, hosts_file='/dev/null', 530 connect_timeout=30, alive_interval=180, alive_count_max=3, 531 connection_attempts=4) 532 533 534 def _make_scp_cmd(self, sources, dest): 535 """Format scp command. 536 537 Given a list of source paths and a destination path, produces the 538 appropriate scp command for encoding it. Remote paths must be 539 pre-encoded. Overrides _make_scp_cmd in AbstractSSHHost 540 to allow additional ssh options. 541 542 @param sources: A list of source paths to copy from. 543 @param dest: Destination path to copy to. 544 545 @returns: An scp command that copies |sources| on local machine to 546 |dest| on the remote servo host. 547 548 """ 549 command = ('scp -rq %s -o BatchMode=yes -o StrictHostKeyChecking=no ' 550 '-o UserKnownHostsFile=/dev/null %s %s "%s"') 551 port = self.port 552 if port is None: 553 logging.info('BaseServoHost: defaulting to port 22. See b/204502754.') 554 port = 22 555 args = ( 556 self._main_ssh.ssh_option, 557 ("-P %s" % port), 558 sources, 559 dest, 560 ) 561 return command % args 562 563 564 def run(self, command, timeout=3600, ignore_status=False, 565 stdout_tee=utils.TEE_TO_LOGS, stderr_tee=utils.TEE_TO_LOGS, 566 connect_timeout=30, ssh_failure_retry_ok=False, 567 options='', stdin=None, verbose=True, args=()): 568 """Run a command on the servo host. 569 570 Extends method `run` in SSHHost. If the servo host is a remote device, 571 it will call `run` in SSHost without changing anything. 572 If the servo host is 'localhost', it will call utils.system_output. 573 574 @param command: The command line string. 575 @param timeout: Time limit in seconds before attempting to 576 kill the running process. The run() function 577 will take a few seconds longer than 'timeout' 578 to complete if it has to kill the process. 579 @param ignore_status: Do not raise an exception, no matter 580 what the exit code of the command is. 581 @param stdout_tee/stderr_tee: Where to tee the stdout/stderr. 582 @param connect_timeout: SSH connection timeout (in seconds) 583 Ignored if host is 'localhost'. 584 @param options: String with additional ssh command options 585 Ignored if host is 'localhost'. 586 @param ssh_failure_retry_ok: when True and ssh connection failure is 587 suspected, OK to retry command (but not 588 compulsory, and likely not needed here) 589 @param stdin: Stdin to pass (a string) to the executed command. 590 @param verbose: Log the commands. 591 @param args: Sequence of strings to pass as arguments to command by 592 quoting them in " and escaping their contents if necessary. 593 594 @returns: A utils.CmdResult object. 595 596 @raises AutoservRunError if the command failed. 597 @raises AutoservSSHTimeout SSH connection has timed out. Only applies 598 when servo host is not 'localhost'. 599 600 """ 601 run_args = { 602 'command' : command, 603 'timeout' : timeout, 604 'ignore_status' : ignore_status, 605 'stdout_tee' : stdout_tee, 606 'stderr_tee' : stderr_tee, 607 # connect_timeout n/a for localhost 608 # options n/a for localhost 609 # ssh_failure_retry_ok n/a for localhost 610 'stdin' : stdin, 611 'verbose' : verbose, 612 'args' : args, 613 } 614 if self.is_containerized_servod(): 615 logging.debug("Trying to run the command %s", command) 616 client = docker_utils.get_docker_client(timeout=timeout) 617 container = client.containers.get(self.servod_container_name) 618 try: 619 (exit_code, 620 output) = container.exec_run("bash -c '%s'" % command) 621 # b/217780680, Make this compatible with python3, 622 if isinstance(output, bytes): 623 output = output.decode(errors='replace') 624 except docker.errors.APIError: 625 logging.exception("Failed to run command %s", command) 626 for line in container.logs().split(b'\n'): 627 logging.error(line) 628 return utils.CmdResult(command=command, 629 stdout="", 630 exit_status=-1) 631 return utils.CmdResult(command=command, 632 stdout=output, 633 exit_status=exit_code) 634 elif self.is_localhost(): 635 if self._sudo_required: 636 run_args['command'] = 'sudo -n sh -c "%s"' % utils.sh_escape( 637 command) 638 try: 639 return utils.run(**run_args) 640 except error.CmdError as e: 641 logging.error(e) 642 raise error.AutoservRunError('command execution error', 643 e.result_obj) 644 else: 645 run_args['connect_timeout'] = connect_timeout 646 run_args['options'] = options 647 run_args['ssh_failure_retry_ok'] = ssh_failure_retry_ok 648 return super(BaseServoHost, self).run(**run_args) 649 650 def _mount_drive(self, src_path, dst_path): 651 """Mount an external drive on servohost. 652 653 @param: src_path the drive path to mount(e.g. /dev/sda3). 654 @param: dst_path the destination directory on servohost to mount 655 the drive. 656 657 @returns: True if mount success otherwise False. 658 """ 659 # Make sure the dst dir exists. 660 self.run('mkdir -p %s' % dst_path) 661 662 result = self.run('mount -o ro %s %s' % (src_path, dst_path), 663 ignore_status=True) 664 return result.exit_status == 0 665 666 def _unmount_drive(self, mount_path): 667 """Unmount a drive from servohost. 668 669 @param: mount_path the path on servohost to unmount. 670 671 @returns: True if unmount success otherwise False. 672 """ 673 result = self.run('umount %s' % mount_path, ignore_status=True) 674 return result.exit_status == 0 675 676 def wait_ready(self, required_uptime=300): 677 """Wait ready for a servohost if it has been rebooted recently. 678 679 It may take a few minutes until all servos and their componments 680 re-enumerated and become ready after a servohost(especially labstation 681 as it supports multiple servos) reboot, so we need to make sure the 682 servohost has been up for a given a mount of time before trying to 683 start any actions. 684 685 @param required_uptime: Minimum uptime in seconds that we can 686 consdier a servohost be ready. 687 """ 688 uptime = float(self.check_uptime()) 689 # To prevent unexpected output from check_uptime() that causes long 690 # sleep, make sure the maximum wait time <= required_uptime. 691 diff = min(required_uptime - uptime, required_uptime) 692 if diff > 0: 693 logging.info( 694 'The servohost was just rebooted, wait %s' 695 ' seconds for it to become ready', diff) 696 time.sleep(diff) 697 698 def is_up(self, 699 timeout=60, 700 connect_timeout=None, 701 base_cmd="true", 702 with_servod=True): 703 """ 704 Check if the remote host is up by ssh-ing and running a base command. 705 706 @param timeout: command execution timeout in seconds. 707 @param connect_timeout: ssh connection timeout in seconds. 708 @param base_cmd: a base command to run with ssh. The default is 'true'. 709 @returns True if the remote host is up before the timeout expires, 710 False otherwise. 711 """ 712 if self.is_containerized_servod(): 713 client = docker_utils.get_docker_client(timeout=timeout) 714 # Look up the container list with hostname and with/without servod process by label. 715 containers = client.containers.list( 716 filters={ 717 'name': self.hostname, 718 'label': ["WITH_SERVOD=%s" % str(with_servod)] 719 }) 720 if not containers: 721 return False 722 elif with_servod: 723 # For container with servod process, check if servod process started. 724 (exit_code, output) = containers[0].exec_run("ps") 725 logging.info("Is Up output %s", output) 726 if b"servod" not in output: 727 return False 728 return True 729 else: 730 return super(BaseServoHost, self).is_up(timeout, connect_timeout, 731 base_cmd) 732