1# Lint as: python2, python3 2# Copyright (c) 2008 The Chromium OS Authors. All rights reserved. 3# Use of this source code is governed by a BSD-style license that can be 4# found in the LICENSE file. 5 6from __future__ import absolute_import 7from __future__ import division 8from __future__ import print_function 9 10import os, time, socket, shutil, glob, logging, tempfile, re 11import shlex 12import subprocess 13 14from autotest_lib.client.bin.result_tools import runner as result_tools_runner 15from autotest_lib.client.common_lib import error 16from autotest_lib.client.common_lib import utils 17from autotest_lib.client.common_lib.cros.network import ping_runner 18from autotest_lib.client.common_lib.global_config import global_config 19from autotest_lib.server import autoserv_parser 20from autotest_lib.server import utils, autotest 21from autotest_lib.server.hosts import host_info 22from autotest_lib.server.hosts import remote 23from autotest_lib.server.hosts import rpc_server_tracker 24from autotest_lib.server.hosts import ssh_multiplex 25from autotest_lib.server.hosts.tls_client import exec_dut_command 26 27import six 28from six.moves import filter 29 30try: 31 from autotest_lib.utils.frozen_chromite.lib import metrics 32except ImportError: 33 metrics = utils.metrics_mock 34 35# pylint: disable=C0111 36 37get_value = global_config.get_config_value 38enable_main_ssh = get_value('AUTOSERV', 39 'enable_main_ssh', 40 type=bool, 41 default=False) 42 43ENABLE_EXEC_DUT_COMMAND = get_value('AUTOSERV', 44 'enable_tls', 45 type=bool, 46 default=False) 47 48# Number of seconds to use the cached up status. 49_DEFAULT_UP_STATUS_EXPIRATION_SECONDS = 300 50_DEFAULT_SSH_PORT = None 51 52# Number of seconds to wait for the host to shut down in wait_down(). 53_DEFAULT_WAIT_DOWN_TIME_SECONDS = 120 54 55# Number of seconds to wait for the host to boot up in wait_up(). 56_DEFAULT_WAIT_UP_TIME_SECONDS = 120 57 58# Timeout in seconds for a single call of get_boot_id() in wait_down() 59# and a single ssh ping in wait_up(). 60_DEFAULT_MAX_PING_TIMEOUT = 10 61 62# The client symlink directory. 63AUTOTEST_CLIENT_SYMLINK_END = 'client/autotest_lib' 64 65 66class AbstractSSHHost(remote.RemoteHost): 67 """ 68 This class represents a generic implementation of most of the 69 framework necessary for controlling a host via ssh. It implements 70 almost all of the abstract Host methods, except for the core 71 Host.run method. 72 """ 73 VERSION_PREFIX = '' 74 # Timeout for main ssh connection setup, in seconds. 75 DEFAULT_START_MAIN_SSH_TIMEOUT_S = 5 76 77 def _initialize(self, 78 hostname, 79 user="root", 80 port=_DEFAULT_SSH_PORT, 81 password="", 82 is_client_install_supported=True, 83 afe_host=None, 84 host_info_store=None, 85 connection_pool=None, 86 *args, 87 **dargs): 88 super(AbstractSSHHost, self)._initialize(hostname=hostname, 89 *args, **dargs) 90 """ 91 @param hostname: The hostname of the host. 92 @param user: The username to use when ssh'ing into the host. 93 @param password: The password to use when ssh'ing into the host. 94 @param port: The port to use for ssh. 95 @param is_client_install_supported: Boolean to indicate if we can 96 install autotest on the host. 97 @param afe_host: The host object attained from the AFE (get_hosts). 98 @param host_info_store: Optional host_info.CachingHostInfoStore object 99 to obtain / update host information. 100 @param connection_pool: ssh_multiplex.ConnectionPool instance to share 101 the main ssh connection across control scripts. 102 """ 103 self._track_class_usage() 104 # IP address is retrieved only on demand. Otherwise the host 105 # initialization will fail for host is not online. 106 self._ip = None 107 self.user = user 108 self.port = port 109 self.password = password 110 self._is_client_install_supported = is_client_install_supported 111 self._use_rsync = None 112 self.known_hosts_file = tempfile.mkstemp()[1] 113 self._rpc_server_tracker = rpc_server_tracker.RpcServerTracker(self); 114 self._tls_exec_dut_command_client = None 115 self._tls_unstable = False 116 117 # Read the value of the use_icmp flag, setting to true if missing. 118 args_string = autoserv_parser.autoserv_parser.options.args 119 args_dict = utils.args_to_dict( 120 args_string.split() if args_string is not None else '') 121 value = args_dict.get('use_icmp', 'true').lower() 122 if value == 'true': 123 self._use_icmp = True 124 elif value == 'false': 125 self._use_icmp = False 126 else: 127 raise ValueError( 128 'use_icmp must be true or false: {}'.format(value)) 129 """ 130 Main SSH connection background job, socket temp directory and socket 131 control path option. If main-SSH is enabled, these fields will be 132 initialized by start_main_ssh when a new SSH connection is initiated. 133 """ 134 self._connection_pool = connection_pool 135 if connection_pool: 136 self._main_ssh = connection_pool.get(hostname, user, port) 137 else: 138 self._main_ssh = ssh_multiplex.MainSsh(hostname, user, port) 139 140 self._afe_host = afe_host or utils.EmptyAFEHost() 141 self.host_info_store = (host_info_store or 142 host_info.InMemoryHostInfoStore()) 143 144 # The cached status of whether the DUT responded to ping. 145 self._cached_up_status = None 146 # The timestamp when the value of _cached_up_status is set. 147 self._cached_up_status_updated = None 148 149 150 @property 151 def ip(self): 152 """@return IP address of the host. 153 """ 154 if not self._ip: 155 self._ip = socket.getaddrinfo(self.hostname, None)[0][4][0] 156 return self._ip 157 158 159 @property 160 def is_client_install_supported(self): 161 """" 162 Returns True if the host supports autotest client installs, False 163 otherwise. 164 """ 165 return self._is_client_install_supported 166 167 def is_satlab(self): 168 """Determine if the host is part of satlab 169 170 TODO(otabek@): Remove or update to better logic to determime Satlab. 171 172 @returns True if ths host is running under satlab otherwise False. 173 """ 174 if not hasattr(self, '_is_satlab'): 175 self._is_satlab = self.hostname.startswith('satlab-') 176 return self._is_satlab 177 178 @property 179 def rpc_server_tracker(self): 180 """" 181 @return The RPC server tracker associated with this host. 182 """ 183 return self._rpc_server_tracker 184 185 186 @property 187 def is_default_port(self): 188 """Returns True if its port is default SSH port.""" 189 return self.port == _DEFAULT_SSH_PORT or self.port is None 190 191 @property 192 def host_port(self): 193 """Returns hostname if port is default. Otherwise, hostname:port. 194 """ 195 if self.is_default_port: 196 return self.hostname 197 else: 198 return '%s:%d' % (self.hostname, self.port) 199 200 @property 201 def use_icmp(self): 202 """Returns True if icmp pings are allowed.""" 203 return self._use_icmp 204 205 206 # Though it doesn't use self here, it is not declared as staticmethod 207 # because its subclass may use self to access member variables. 208 def make_ssh_command(self, user="root", port=_DEFAULT_SSH_PORT, opts='', 209 hosts_file='/dev/null', connect_timeout=30, 210 alive_interval=300, alive_count_max=3, 211 connection_attempts=1): 212 ssh_options = " ".join([ 213 opts, 214 self.make_ssh_options( 215 hosts_file=hosts_file, connect_timeout=connect_timeout, 216 alive_interval=alive_interval, alive_count_max=alive_count_max, 217 connection_attempts=connection_attempts)]) 218 return ("/usr/bin/ssh -a -x %s -l %s %s" % 219 (ssh_options, user, "-p %d " % port if port else "")) 220 221 222 @staticmethod 223 def make_ssh_options(hosts_file='/dev/null', connect_timeout=30, 224 alive_interval=300, alive_count_max=3, 225 connection_attempts=1): 226 """Composes SSH -o options.""" 227 assert isinstance(connect_timeout, six.integer_types) 228 assert connect_timeout > 0 # can't disable the timeout 229 230 options = [("StrictHostKeyChecking", "no"), 231 ("UserKnownHostsFile", hosts_file), 232 ("BatchMode", "yes"), 233 ("ConnectTimeout", str(connect_timeout)), 234 ("ServerAliveInterval", str(alive_interval)), 235 ("ServerAliveCountMax", str(alive_count_max)), 236 ("ConnectionAttempts", str(connection_attempts))] 237 return " ".join("-o %s=%s" % kv for kv in options) 238 239 240 def use_rsync(self): 241 if self._use_rsync is not None: 242 return self._use_rsync 243 244 # Check if rsync is available on the remote host. If it's not, 245 # don't try to use it for any future file transfers. 246 self._use_rsync = self.check_rsync() 247 if not self._use_rsync: 248 logging.warning("rsync not available on remote host %s -- disabled", 249 self.host_port) 250 return self._use_rsync 251 252 253 def check_rsync(self): 254 """ 255 Check if rsync is available on the remote host. 256 """ 257 try: 258 self.run("rsync --version", stdout_tee=None, stderr_tee=None) 259 except error.AutoservRunError: 260 return False 261 return True 262 263 264 def _encode_remote_paths(self, paths, escape=True, use_scp=False): 265 """ 266 Given a list of file paths, encodes it as a single remote path, in 267 the style used by rsync and scp. 268 escape: add \\ to protect special characters. 269 use_scp: encode for scp if true, rsync if false. 270 """ 271 if escape: 272 paths = [utils.scp_remote_escape(path) for path in paths] 273 274 remote = self.hostname 275 276 # rsync and scp require IPv6 brackets, even when there isn't any 277 # trailing port number (ssh doesn't support IPv6 brackets). 278 # In the Python >= 3.3 future, 'import ipaddress' will parse addresses. 279 if re.search(r':.*:', remote): 280 remote = '[%s]' % remote 281 282 if use_scp: 283 return '%s@%s:"%s"' % (self.user, remote, " ".join(paths)) 284 else: 285 return '%s@%s:%s' % ( 286 self.user, remote, 287 " :".join('"%s"' % p for p in paths)) 288 289 def _encode_local_paths(self, paths, escape=True): 290 """ 291 Given a list of file paths, encodes it as a single local path. 292 escape: add \\ to protect special characters. 293 """ 294 if escape: 295 paths = [utils.sh_escape(path) for path in paths] 296 297 return " ".join('"%s"' % p for p in paths) 298 299 300 def rsync_options(self, delete_dest=False, preserve_symlinks=False, 301 safe_symlinks=False, excludes=None): 302 """Obtains rsync options for the remote.""" 303 ssh_cmd = self.make_ssh_command(user=self.user, port=self.port, 304 opts=self._main_ssh.ssh_option, 305 hosts_file=self.known_hosts_file) 306 if delete_dest: 307 delete_flag = "--delete" 308 else: 309 delete_flag = "" 310 if safe_symlinks: 311 symlink_flag = "-l --safe-links" 312 elif preserve_symlinks: 313 symlink_flag = "-l" 314 else: 315 symlink_flag = "-L" 316 exclude_args = '' 317 if excludes: 318 exclude_args = ' '.join( 319 ["--exclude '%s'" % exclude for exclude in excludes]) 320 return "%s %s --timeout=1800 --rsh='%s' -az --no-o --no-g %s" % ( 321 symlink_flag, delete_flag, ssh_cmd, exclude_args) 322 323 324 def _make_rsync_cmd(self, sources, dest, delete_dest, 325 preserve_symlinks, safe_symlinks, excludes=None): 326 """ 327 Given a string of source paths and a destination path, produces the 328 appropriate rsync command for copying them. Remote paths must be 329 pre-encoded. 330 """ 331 rsync_options = self.rsync_options( 332 delete_dest=delete_dest, preserve_symlinks=preserve_symlinks, 333 safe_symlinks=safe_symlinks, excludes=excludes) 334 return 'rsync %s %s "%s"' % (rsync_options, sources, dest) 335 336 337 def _make_ssh_cmd(self, cmd): 338 """ 339 Create a base ssh command string for the host which can be used 340 to run commands directly on the machine 341 """ 342 base_cmd = self.make_ssh_command(user=self.user, port=self.port, 343 opts=self._main_ssh.ssh_option, 344 hosts_file=self.known_hosts_file) 345 346 return '%s %s "%s"' % (base_cmd, self.hostname, utils.sh_escape(cmd)) 347 348 def _make_scp_cmd(self, sources, dest): 349 """ 350 Given a string of source paths and a destination path, produces the 351 appropriate scp command for encoding it. Remote paths must be 352 pre-encoded. 353 """ 354 command = ("scp -rq %s -o StrictHostKeyChecking=no " 355 "-o UserKnownHostsFile=%s %s%s '%s'") 356 return command % (self._main_ssh.ssh_option, self.known_hosts_file, 357 "-P %d " % self.port if self.port else '', sources, 358 dest) 359 360 361 def _make_rsync_compatible_globs(self, path, is_local): 362 """ 363 Given an rsync-style path, returns a list of globbed paths 364 that will hopefully provide equivalent behaviour for scp. Does not 365 support the full range of rsync pattern matching behaviour, only that 366 exposed in the get/send_file interface (trailing slashes). 367 368 The is_local param is flag indicating if the paths should be 369 interpreted as local or remote paths. 370 """ 371 372 # non-trailing slash paths should just work 373 if len(path) == 0 or path[-1] != "/": 374 return [path] 375 376 # make a function to test if a pattern matches any files 377 if is_local: 378 def glob_matches_files(path, pattern): 379 return len(glob.glob(path + pattern)) > 0 380 else: 381 def glob_matches_files(path, pattern): 382 result = self.run("ls \"%s\"%s" % (utils.sh_escape(path), 383 pattern), 384 stdout_tee=None, ignore_status=True) 385 return result.exit_status == 0 386 387 # take a set of globs that cover all files, and see which are needed 388 patterns = ["*", ".[!.]*"] 389 patterns = [p for p in patterns if glob_matches_files(path, p)] 390 391 # convert them into a set of paths suitable for the commandline 392 if is_local: 393 return ["\"%s\"%s" % (utils.sh_escape(path), pattern) 394 for pattern in patterns] 395 else: 396 return [utils.scp_remote_escape(path) + pattern 397 for pattern in patterns] 398 399 400 def _make_rsync_compatible_source(self, source, is_local): 401 """ 402 Applies the same logic as _make_rsync_compatible_globs, but 403 applies it to an entire list of sources, producing a new list of 404 sources, properly quoted. 405 """ 406 return sum((self._make_rsync_compatible_globs(path, is_local) 407 for path in source), []) 408 409 410 def _set_umask_perms(self, dest): 411 """ 412 Given a destination file/dir (recursively) set the permissions on 413 all the files and directories to the max allowed by running umask. 414 """ 415 416 # now this looks strange but I haven't found a way in Python to _just_ 417 # get the umask, apparently the only option is to try to set it 418 umask = os.umask(0) 419 os.umask(umask) 420 421 max_privs = 0o777 & ~umask 422 423 def set_file_privs(filename): 424 """Sets mode of |filename|. Assumes |filename| exists.""" 425 file_stat = os.stat(filename) 426 427 file_privs = max_privs 428 # if the original file permissions do not have at least one 429 # executable bit then do not set it anywhere 430 if not file_stat.st_mode & 0o111: 431 file_privs &= ~0o111 432 433 os.chmod(filename, file_privs) 434 435 # try a bottom-up walk so changes on directory permissions won't cut 436 # our access to the files/directories inside it 437 for root, dirs, files in os.walk(dest, topdown=False): 438 # when setting the privileges we emulate the chmod "X" behaviour 439 # that sets to execute only if it is a directory or any of the 440 # owner/group/other already has execute right 441 for dirname in dirs: 442 os.chmod(os.path.join(root, dirname), max_privs) 443 444 # Filter out broken symlinks as we go. 445 for filename in filter(os.path.exists, files): 446 set_file_privs(os.path.join(root, filename)) 447 448 449 # now set privs for the dest itself 450 if os.path.isdir(dest): 451 os.chmod(dest, max_privs) 452 else: 453 set_file_privs(dest) 454 455 456 def get_file(self, source, dest, delete_dest=False, preserve_perm=True, 457 preserve_symlinks=False, retry=True, safe_symlinks=False, 458 try_rsync=True): 459 """ 460 Copy files from the remote host to a local path. 461 462 Directories will be copied recursively. 463 If a source component is a directory with a trailing slash, 464 the content of the directory will be copied, otherwise, the 465 directory itself and its content will be copied. This 466 behavior is similar to that of the program 'rsync'. 467 468 Args: 469 source: either 470 1) a single file or directory, as a string 471 2) a list of one or more (possibly mixed) 472 files or directories 473 dest: a file or a directory (if source contains a 474 directory or more than one element, you must 475 supply a directory dest) 476 delete_dest: if this is true, the command will also clear 477 out any old files at dest that are not in the 478 source 479 preserve_perm: tells get_file() to try to preserve the sources 480 permissions on files and dirs 481 preserve_symlinks: try to preserve symlinks instead of 482 transforming them into files/dirs on copy 483 safe_symlinks: same as preserve_symlinks, but discard links 484 that may point outside the copied tree 485 try_rsync: set to False to skip directly to using scp 486 Raises: 487 AutoservRunError: the scp command failed 488 """ 489 logging.debug('get_file. source: %s, dest: %s, delete_dest: %s,' 490 'preserve_perm: %s, preserve_symlinks:%s', source, dest, 491 delete_dest, preserve_perm, preserve_symlinks) 492 493 # Start a main SSH connection if necessary. 494 self.start_main_ssh() 495 496 if isinstance(source, six.string_types): 497 source = [source] 498 dest = os.path.abspath(dest) 499 500 # If rsync is disabled or fails, try scp. 501 try_scp = True 502 if try_rsync and self.use_rsync(): 503 logging.debug('Using Rsync.') 504 try: 505 remote_source = self._encode_remote_paths(source) 506 local_dest = utils.sh_escape(dest) 507 rsync = self._make_rsync_cmd(remote_source, local_dest, 508 delete_dest, preserve_symlinks, 509 safe_symlinks) 510 utils.run(rsync) 511 try_scp = False 512 except error.CmdError as e: 513 # retry on rsync exit values which may be caused by transient 514 # network problems: 515 # 516 # rc 10: Error in socket I/O 517 # rc 12: Error in rsync protocol data stream 518 # rc 23: Partial transfer due to error 519 # rc 255: Ssh error 520 # 521 # Note that rc 23 includes dangling symlinks. In this case 522 # retrying is useless, but not very damaging since rsync checks 523 # for those before starting the transfer (scp does not). 524 status = e.result_obj.exit_status 525 if status in [10, 12, 23, 255] and retry: 526 logging.warning('rsync status %d, retrying', status) 527 self.get_file(source, dest, delete_dest, preserve_perm, 528 preserve_symlinks, retry=False) 529 # The nested get_file() does all that's needed. 530 return 531 else: 532 logging.warning("trying scp, rsync failed: %s (%d)", 533 e, status) 534 535 if try_scp: 536 logging.debug('Trying scp.') 537 # scp has no equivalent to --delete, just drop the entire dest dir 538 if delete_dest and os.path.isdir(dest): 539 shutil.rmtree(dest) 540 os.mkdir(dest) 541 542 remote_source = self._make_rsync_compatible_source(source, False) 543 if remote_source: 544 # _make_rsync_compatible_source() already did the escaping 545 remote_source = self._encode_remote_paths( 546 remote_source, escape=False, use_scp=True) 547 local_dest = utils.sh_escape(dest) 548 scp = self._make_scp_cmd(remote_source, local_dest) 549 try: 550 utils.run(scp) 551 except error.CmdError as e: 552 logging.debug('scp failed: %s', e) 553 raise error.AutoservRunError(e.args[0], e.args[1]) 554 555 if not preserve_perm: 556 # we have no way to tell scp to not try to preserve the 557 # permissions so set them after copy instead. 558 # for rsync we could use "--no-p --chmod=ugo=rwX" but those 559 # options are only in very recent rsync versions 560 self._set_umask_perms(dest) 561 562 563 def send_file(self, source, dest, delete_dest=False, 564 preserve_symlinks=False, excludes=None): 565 """ 566 Copy files from a local path to the remote host. 567 568 Directories will be copied recursively. 569 If a source component is a directory with a trailing slash, 570 the content of the directory will be copied, otherwise, the 571 directory itself and its content will be copied. This 572 behavior is similar to that of the program 'rsync'. 573 574 Args: 575 source: either 576 1) a single file or directory, as a string 577 2) a list of one or more (possibly mixed) 578 files or directories 579 dest: a file or a directory (if source contains a 580 directory or more than one element, you must 581 supply a directory dest) 582 delete_dest: if this is true, the command will also clear 583 out any old files at dest that are not in the 584 source 585 preserve_symlinks: controls if symlinks on the source will be 586 copied as such on the destination or transformed into the 587 referenced file/directory 588 excludes: A list of file pattern that matches files not to be 589 sent. `send_file` will fail if exclude is set, since 590 local copy does not support --exclude, e.g., when 591 using scp to copy file. 592 593 Raises: 594 AutoservRunError: the scp command failed 595 """ 596 logging.debug('send_file. source: %s, dest: %s, delete_dest: %s,' 597 'preserve_symlinks:%s', source, dest, 598 delete_dest, preserve_symlinks) 599 # Start a main SSH connection if necessary. 600 self.start_main_ssh() 601 602 if isinstance(source, six.string_types): 603 source = [source] 604 605 client_symlink = _client_symlink(source) 606 # The client symlink *must* be preserved, and should not be sent with 607 # the main send_file in case scp is used, which does not support symlink 608 if client_symlink: 609 source.remove(client_symlink) 610 611 local_sources = self._encode_local_paths(source) 612 if not local_sources: 613 raise error.TestError('source |%s| yielded an empty string' % ( 614 source)) 615 if local_sources.find('\x00') != -1: 616 raise error.TestError('one or more sources include NUL char') 617 618 self._send_file( 619 dest=dest, 620 source=source, 621 local_sources=local_sources, 622 delete_dest=delete_dest, 623 excludes=excludes, 624 preserve_symlinks=preserve_symlinks) 625 626 # Send the client symlink after the rest of the autotest repo has been 627 # sent. 628 if client_symlink: 629 self._send_client_symlink(dest=dest, 630 source=[client_symlink], 631 local_sources=client_symlink, 632 delete_dest=delete_dest, 633 excludes=excludes, 634 preserve_symlinks=True) 635 636 def _send_client_symlink(self, dest, source, local_sources, delete_dest, 637 excludes, preserve_symlinks): 638 if self.use_rsync(): 639 if self._send_using_rsync(dest=dest, 640 local_sources=local_sources, 641 delete_dest=delete_dest, 642 preserve_symlinks=preserve_symlinks, 643 excludes=excludes): 644 return 645 # Manually create the symlink if rsync is not available, or fails. 646 try: 647 self.run('mkdir {f} && touch {f}/__init__.py && cd {f} && ' 648 'ln -s ../ client'.format( 649 f=os.path.join(dest, 'autotest_lib'))) 650 except Exception as e: 651 raise error.AutotestHostRunError( 652 "Could not create client symlink on host: %s" % e) 653 654 def _send_file(self, dest, source, local_sources, delete_dest, excludes, 655 preserve_symlinks): 656 """Send file(s), trying rsync first, then scp.""" 657 if self.use_rsync(): 658 rsync_success = self._send_using_rsync( 659 dest=dest, 660 local_sources=local_sources, 661 delete_dest=delete_dest, 662 preserve_symlinks=preserve_symlinks, 663 excludes=excludes) 664 if rsync_success: 665 return 666 667 # Send using scp if you cannot via rsync, or rsync fails. 668 self._send_using_scp(dest=dest, 669 source=source, 670 delete_dest=delete_dest, 671 excludes=excludes) 672 673 def _send_using_rsync(self, dest, local_sources, delete_dest, 674 preserve_symlinks, excludes): 675 """Send using rsync. 676 677 Args: 678 dest: a file or a directory (if source contains a 679 directory or more than one element, you must 680 supply a directory dest) 681 local_sources: a string of files/dirs to send separated with spaces 682 delete_dest: if this is true, the command will also clear 683 out any old files at dest that are not in the 684 source 685 preserve_symlinks: controls if symlinks on the source will be 686 copied as such on the destination or transformed into the 687 referenced file/directory 688 excludes: A list of file pattern that matches files not to be 689 sent. `send_file` will fail if exclude is set, since 690 local copy does not support --exclude, e.g., when 691 using scp to copy file. 692 Returns: 693 bool: True if the cmd succeeded, else False 694 695 """ 696 logging.debug('Using Rsync.') 697 remote_dest = self._encode_remote_paths([dest]) 698 try: 699 rsync = self._make_rsync_cmd(local_sources, 700 remote_dest, 701 delete_dest, 702 preserve_symlinks, 703 False, 704 excludes=excludes) 705 utils.run(rsync) 706 return True 707 except error.CmdError as e: 708 logging.warning("trying scp, rsync failed: %s", e) 709 return False 710 711 def _send_using_scp(self, dest, source, delete_dest, excludes): 712 """Send using scp. 713 714 Args: 715 source: either 716 1) a single file or directory, as a string 717 2) a list of one or more (possibly mixed) 718 files or directories 719 dest: a file or a directory (if source contains a 720 directory or more than one element, you must 721 supply a directory dest) 722 delete_dest: if this is true, the command will also clear 723 out any old files at dest that are not in the 724 source 725 excludes: A list of file pattern that matches files not to be 726 sent. `send_file` will fail if exclude is set, since 727 local copy does not support --exclude, e.g., when 728 using scp to copy file. 729 730 Raises: 731 AutoservRunError: the scp command failed 732 """ 733 logging.debug('Trying scp.') 734 if excludes: 735 raise error.AutotestHostRunError( 736 '--exclude is not supported in scp, try to use rsync. ' 737 'excludes: %s' % ','.join(excludes), None) 738 739 # scp has no equivalent to --delete, just drop the entire dest dir 740 if delete_dest: 741 is_dir = self.run("ls -d %s/" % dest, 742 ignore_status=True).exit_status == 0 743 if is_dir: 744 cmd = "rm -rf %s && mkdir %s" 745 cmd %= (dest, dest) 746 self.run(cmd) 747 748 remote_dest = self._encode_remote_paths([dest], use_scp=True) 749 local_sources = self._make_rsync_compatible_source(source, True) 750 if local_sources: 751 sources = self._encode_local_paths(local_sources, escape=False) 752 scp = self._make_scp_cmd(sources, remote_dest) 753 try: 754 utils.run(scp) 755 except error.CmdError as e: 756 logging.debug('scp failed: %s', e) 757 raise error.AutoservRunError(e.args[0], e.args[1]) 758 else: 759 logging.debug('skipping scp for empty source list') 760 761 def verify_ssh_user_access(self): 762 """Verify ssh access to this host. 763 764 @returns False if ssh_ping fails due to Permissions error, True 765 otherwise. 766 """ 767 try: 768 self.ssh_ping() 769 except (error.AutoservSshPermissionDeniedError, 770 error.AutoservSshPingHostError): 771 return False 772 return True 773 774 775 def ssh_ping(self, timeout=60, connect_timeout=None, base_cmd='true'): 776 """ 777 Pings remote host via ssh. 778 779 @param timeout: Command execution timeout in seconds. 780 Defaults to 60 seconds. 781 @param connect_timeout: ssh connection timeout in seconds. 782 @param base_cmd: The base command to run with the ssh ping. 783 Defaults to true. 784 @raise AutoservSSHTimeout: If the ssh ping times out. 785 @raise AutoservSshPermissionDeniedError: If ssh ping fails due to 786 permissions. 787 @raise AutoservSshPingHostError: For other AutoservRunErrors. 788 """ 789 ctimeout = min(timeout, connect_timeout or timeout) 790 try: 791 self.run(base_cmd, timeout=timeout, connect_timeout=ctimeout, 792 ssh_failure_retry_ok=True) 793 except error.AutoservSSHTimeout: 794 msg = "Host (ssh) verify timed out (timeout = %d)" % timeout 795 raise error.AutoservSSHTimeout(msg) 796 except error.AutoservSshPermissionDeniedError: 797 #let AutoservSshPermissionDeniedError be visible to the callers 798 raise 799 except error.AutoservRunError as e: 800 # convert the generic AutoservRunError into something more 801 # specific for this context 802 raise error.AutoservSshPingHostError(e.description + '\n' + 803 repr(e.result_obj)) 804 805 806 def is_up(self, timeout=60, connect_timeout=None, base_cmd='true'): 807 """ 808 Check if the remote host is up by ssh-ing and running a base command. 809 810 @param timeout: command execution timeout in seconds. 811 @param connect_timeout: ssh connection timeout in seconds. 812 @param base_cmd: a base command to run with ssh. The default is 'true'. 813 @returns True if the remote host is up before the timeout expires, 814 False otherwise. 815 """ 816 try: 817 self.ssh_ping(timeout=timeout, 818 connect_timeout=connect_timeout, 819 base_cmd=base_cmd) 820 except error.AutoservError: 821 return False 822 else: 823 return True 824 825 826 def is_up_fast(self, count=1): 827 """Return True if the host can be pinged. 828 829 @param count How many time try to ping before decide that host is not 830 reachable by ping. 831 """ 832 if not self._use_icmp: 833 stack = self._get_server_stack_state(lowest_frames=1, 834 highest_frames=7) 835 logging.warning("is_up_fast called with icmp disabled from %s!", 836 stack) 837 return True 838 ping_config = ping_runner.PingConfig(self.hostname, 839 count=1, 840 ignore_result=True, 841 ignore_status=True) 842 843 # Run up to the amount specified, but also exit as soon as the first 844 # reply is found. 845 loops_remaining = count 846 while loops_remaining > 0: 847 loops_remaining -= 1 848 if ping_runner.PingRunner().ping(ping_config).received > 0: 849 return True 850 return False 851 852 853 def wait_up(self, 854 timeout=_DEFAULT_WAIT_UP_TIME_SECONDS, 855 host_is_down=False): 856 """ 857 Wait until the remote host is up or the timeout expires. 858 859 In fact, it will wait until an ssh connection to the remote 860 host can be established, and getty is running. 861 862 @param timeout time limit in seconds before returning even 863 if the host is not up. 864 @param host_is_down set to True if the host is known to be down before 865 wait_up. 866 867 @returns True if the host was found to be up before the timeout expires, 868 False otherwise 869 """ 870 if host_is_down: 871 # Since we expect the host to be down when this is called, if there is 872 # an existing ssh main connection close it. 873 self.close_main_ssh() 874 current_time = int(time.time()) 875 end_time = current_time + timeout 876 877 ssh_success_logged = False 878 autoserv_error_logged = False 879 while current_time < end_time: 880 ping_timeout = min(_DEFAULT_MAX_PING_TIMEOUT, 881 end_time - current_time) 882 if self.is_up(timeout=ping_timeout, connect_timeout=ping_timeout): 883 if not ssh_success_logged: 884 logging.debug('Successfully pinged host %s', 885 self.host_port) 886 wait_procs = self.get_wait_up_processes() 887 if wait_procs: 888 logging.debug('Waiting for processes: %s', wait_procs) 889 else: 890 logging.debug('No wait_up processes to wait for') 891 ssh_success_logged = True 892 try: 893 if self.are_wait_up_processes_up(): 894 logging.debug('Host %s is now up', self.host_port) 895 return True 896 except error.AutoservError as e: 897 if not autoserv_error_logged: 898 logging.debug('Ignoring failure to reach %s: %s %s', 899 self.host_port, e, 900 '(and further similar failures)') 901 autoserv_error_logged = True 902 time.sleep(1) 903 current_time = int(time.time()) 904 905 logging.debug('Host %s is still down after waiting %d seconds', 906 self.host_port, int(timeout + time.time() - end_time)) 907 return False 908 909 910 def wait_down(self, timeout=_DEFAULT_WAIT_DOWN_TIME_SECONDS, 911 warning_timer=None, old_boot_id=None, 912 max_ping_timeout=_DEFAULT_MAX_PING_TIMEOUT): 913 """ 914 Wait until the remote host is down or the timeout expires. 915 916 If old_boot_id is provided, waits until either the machine is 917 unpingable or self.get_boot_id() returns a value different from 918 old_boot_id. If the boot_id value has changed then the function 919 returns True under the assumption that the machine has shut down 920 and has now already come back up. 921 922 If old_boot_id is None then until the machine becomes unreachable the 923 method assumes the machine has not yet shut down. 924 925 @param timeout Time limit in seconds before returning even if the host 926 is still up. 927 @param warning_timer Time limit in seconds that will generate a warning 928 if the host is not down yet. Can be None for no warning. 929 @param old_boot_id A string containing the result of self.get_boot_id() 930 prior to the host being told to shut down. Can be None if this is 931 not available. 932 @param max_ping_timeout Maximum timeout in seconds for each 933 self.get_boot_id() call. If this timeout is hit, it is assumed that 934 the host went down and became unreachable. 935 936 @returns True if the host was found to be down (max_ping_timeout timeout 937 expired or boot_id changed if provided) and False if timeout 938 expired. 939 """ 940 #TODO: there is currently no way to distinguish between knowing 941 #TODO: boot_id was unsupported and not knowing the boot_id. 942 current_time = int(time.time()) 943 end_time = current_time + timeout 944 945 if warning_timer: 946 warn_time = current_time + warning_timer 947 948 if old_boot_id is not None: 949 logging.debug('Host %s pre-shutdown boot_id is %s', 950 self.host_port, old_boot_id) 951 952 # Impose semi real-time deadline constraints, since some clients 953 # (eg: watchdog timer tests) expect strict checking of time elapsed. 954 # Each iteration of this loop is treated as though it atomically 955 # completes within current_time, this is needed because if we used 956 # inline time.time() calls instead then the following could happen: 957 # 958 # while time.time() < end_time: [23 < 30] 959 # some code. [takes 10 secs] 960 # try: 961 # new_boot_id = self.get_boot_id(timeout=end_time - time.time()) 962 # [30 - 33] 963 # The last step will lead to a return True, when in fact the machine 964 # went down at 32 seconds (>30). Hence we need to pass get_boot_id 965 # the same time that allowed us into that iteration of the loop. 966 while current_time < end_time: 967 ping_timeout = min(end_time - current_time, max_ping_timeout) 968 try: 969 new_boot_id = self.get_boot_id(timeout=ping_timeout) 970 except error.AutoservError: 971 logging.debug('Host %s is now unreachable over ssh, is down', 972 self.host_port) 973 return True 974 else: 975 # if the machine is up but the boot_id value has changed from 976 # old boot id, then we can assume the machine has gone down 977 # and then already come back up 978 if old_boot_id is not None and old_boot_id != new_boot_id: 979 logging.debug('Host %s now has boot_id %s and so must ' 980 'have rebooted', self.host_port, new_boot_id) 981 return True 982 983 if warning_timer and current_time > warn_time: 984 self.record("INFO", None, "shutdown", 985 "Shutdown took longer than %ds" % warning_timer) 986 # Print the warning only once. 987 warning_timer = None 988 # If a machine is stuck switching runlevels 989 # This may cause the machine to reboot. 990 self.run('kill -HUP 1', ignore_status=True) 991 992 time.sleep(1) 993 current_time = int(time.time()) 994 995 return False 996 997 998 # tunable constants for the verify & repair code 999 AUTOTEST_GB_DISKSPACE_REQUIRED = get_value("SERVER", 1000 "gb_diskspace_required", 1001 type=float, 1002 default=20.0) 1003 1004 1005 def verify_connectivity(self): 1006 super(AbstractSSHHost, self).verify_connectivity() 1007 1008 logging.info('Pinging host %s', self.host_port) 1009 self.ssh_ping() 1010 logging.info("Host (ssh) %s is alive", self.host_port) 1011 1012 if self.is_shutting_down(): 1013 raise error.AutoservHostIsShuttingDownError("Host is shutting down") 1014 1015 1016 def verify_software(self): 1017 super(AbstractSSHHost, self).verify_software() 1018 try: 1019 self.check_diskspace(autotest.Autotest.get_install_dir(self), 1020 self.AUTOTEST_GB_DISKSPACE_REQUIRED) 1021 except error.AutoservDiskFullHostError: 1022 # only want to raise if it's a space issue 1023 raise 1024 except (error.AutoservHostError, autotest.AutodirNotFoundError): 1025 logging.exception('autodir space check exception, this is probably ' 1026 'safe to ignore\n') 1027 1028 def close(self): 1029 super(AbstractSSHHost, self).close() 1030 self.rpc_server_tracker.disconnect_all() 1031 if not self._connection_pool: 1032 self._main_ssh.close() 1033 if os.path.exists(self.known_hosts_file): 1034 os.remove(self.known_hosts_file) 1035 self.tls_exec_dut_command = None 1036 1037 def close_main_ssh(self): 1038 """Stop the ssh main connection. 1039 1040 Intended for situations when the host is known to be down and we don't 1041 need a ssh timeout to tell us it is down. For example, if you just 1042 instructed the host to shutdown or hibernate. 1043 """ 1044 logging.debug("Stopping main ssh connection") 1045 self._main_ssh.close() 1046 1047 def restart_main_ssh(self): 1048 """ 1049 Stop and restart the ssh main connection. This is meant as a last 1050 resort when ssh commands fail and we don't understand why. 1051 """ 1052 logging.debug("Restarting main ssh connection") 1053 self._main_ssh.close() 1054 self._main_ssh.maybe_start(timeout=30) 1055 1056 def start_main_ssh(self, timeout=DEFAULT_START_MAIN_SSH_TIMEOUT_S): 1057 """ 1058 Called whenever a non-main SSH connection needs to be initiated (e.g., 1059 by run, rsync, scp). If main SSH support is enabled and a main SSH 1060 connection is not active already, start a new one in the background. 1061 Also, cleanup any zombie main SSH connections (e.g., dead due to 1062 reboot). 1063 1064 timeout: timeout in seconds (default 5) to wait for main ssh 1065 connection to be established. If timeout is reached, a 1066 warning message is logged, but no other action is taken. 1067 """ 1068 if not enable_main_ssh: 1069 return 1070 self._main_ssh.maybe_start(timeout=timeout) 1071 1072 @property 1073 def tls_unstable(self): 1074 # A single test will rebuild remote many times. Its safe to assume if 1075 # TLS unstable for one try, it will be for others. If we check each, 1076 # it adds ~60 seconds per test (if its dead). 1077 if os.getenv('TLS_UNSTABLE'): 1078 return bool(os.getenv('TLS_UNSTABLE')) 1079 if self._tls_unstable is not None: 1080 return self._tls_unstable 1081 1082 @tls_unstable.setter 1083 def tls_unstable(self, v): 1084 if not isinstance(v, bool): 1085 raise error.AutoservError('tls_stable setting must be bool, got %s' 1086 % (type(v))) 1087 os.environ['TLS_UNSTABLE'] = str(v) 1088 self._tls_unstable = v 1089 1090 @property 1091 def tls_exec_dut_command_client(self): 1092 # If client is already initialized, return that. 1093 if not ENABLE_EXEC_DUT_COMMAND: 1094 return None 1095 if self.tls_unstable: 1096 return None 1097 if self._tls_exec_dut_command_client is not None: 1098 return self._tls_exec_dut_command_client 1099 # If the TLS connection is alive, create a new client. 1100 if self.tls_connection is None: 1101 return None 1102 return exec_dut_command.TLSExecDutCommandClient( 1103 tlsconnection=self.tls_connection, 1104 hostname=self.hostname) 1105 1106 def clear_known_hosts(self): 1107 """Clears out the temporary ssh known_hosts file. 1108 1109 This is useful if the test SSHes to the machine, then reinstalls it, 1110 then SSHes to it again. It can be called after the reinstall to 1111 reduce the spam in the logs. 1112 """ 1113 logging.info("Clearing known hosts for host '%s', file '%s'.", 1114 self.host_port, self.known_hosts_file) 1115 # Clear out the file by opening it for writing and then closing. 1116 fh = open(self.known_hosts_file, "w") 1117 fh.close() 1118 1119 1120 def collect_logs(self, remote_src_dir, local_dest_dir, ignore_errors=True): 1121 """Copy log directories from a host to a local directory. 1122 1123 @param remote_src_dir: A destination directory on the host. 1124 @param local_dest_dir: A path to a local destination directory. 1125 If it doesn't exist it will be created. 1126 @param ignore_errors: If True, ignore exceptions. 1127 1128 @raises OSError: If there were problems creating the local_dest_dir and 1129 ignore_errors is False. 1130 @raises AutoservRunError, AutotestRunError: If something goes wrong 1131 while copying the directories and ignore_errors is False. 1132 """ 1133 if not self.check_cached_up_status(): 1134 logging.warning('Host %s did not answer to ping, skip collecting ' 1135 'logs.', self.host_port) 1136 return 1137 1138 locally_created_dest = False 1139 if (not os.path.exists(local_dest_dir) 1140 or not os.path.isdir(local_dest_dir)): 1141 try: 1142 os.makedirs(local_dest_dir) 1143 locally_created_dest = True 1144 except OSError as e: 1145 logging.warning('Unable to collect logs from host ' 1146 '%s: %s', self.host_port, e) 1147 if not ignore_errors: 1148 raise 1149 return 1150 1151 # Build test result directory summary 1152 try: 1153 result_tools_runner.run_on_client(self, remote_src_dir) 1154 except (error.AutotestRunError, error.AutoservRunError, 1155 error.AutoservSSHTimeout) as e: 1156 logging.exception( 1157 'Non-critical failure: Failed to collect and throttle ' 1158 'results at %s from host %s', remote_src_dir, 1159 self.host_port) 1160 1161 try: 1162 self.get_file(remote_src_dir, local_dest_dir, safe_symlinks=True) 1163 except (error.AutotestRunError, error.AutoservRunError, 1164 error.AutoservSSHTimeout) as e: 1165 logging.warning('Collection of %s to local dir %s from host %s ' 1166 'failed: %s', remote_src_dir, local_dest_dir, 1167 self.host_port, e) 1168 if locally_created_dest: 1169 shutil.rmtree(local_dest_dir, ignore_errors=ignore_errors) 1170 if not ignore_errors: 1171 raise 1172 1173 # Clean up directory summary file on the client side. 1174 try: 1175 result_tools_runner.run_on_client(self, remote_src_dir, 1176 cleanup_only=True) 1177 except (error.AutotestRunError, error.AutoservRunError, 1178 error.AutoservSSHTimeout) as e: 1179 logging.exception( 1180 'Non-critical failure: Failed to cleanup result summary ' 1181 'files at %s in host %s', remote_src_dir, self.hostname) 1182 1183 1184 def create_ssh_tunnel(self, port, local_port): 1185 """Create an ssh tunnel from local_port to port. 1186 1187 This is used to forward a port securely through a tunnel process from 1188 the server to the DUT for RPC server connection. 1189 1190 @param port: remote port on the host. 1191 @param local_port: local forwarding port. 1192 1193 @return: the tunnel process. 1194 """ 1195 tunnel_options = '-n -N -q -L %d:localhost:%d' % (local_port, port) 1196 ssh_cmd = self.make_ssh_command(opts=tunnel_options, port=self.port) 1197 tunnel_cmd = '%s %s' % (ssh_cmd, self.hostname) 1198 logging.debug('Full tunnel command: %s', tunnel_cmd) 1199 # Exec the ssh process directly here rather than using a shell. 1200 # Using a shell leaves a dangling ssh process, because we deliver 1201 # signals to the shell wrapping ssh, not the ssh process itself. 1202 args = shlex.split(tunnel_cmd) 1203 with open('/dev/null', 'w') as devnull: 1204 tunnel_proc = subprocess.Popen(args, stdout=devnull, stderr=devnull, 1205 close_fds=True) 1206 logging.debug('Started ssh tunnel, local = %d' 1207 ' remote = %d, pid = %d', 1208 local_port, port, tunnel_proc.pid) 1209 return tunnel_proc 1210 1211 1212 def disconnect_ssh_tunnel(self, tunnel_proc): 1213 """ 1214 Disconnects a previously forwarded port from the server to the DUT for 1215 RPC server connection. 1216 1217 @param tunnel_proc: a tunnel process returned from |create_ssh_tunnel|. 1218 """ 1219 if tunnel_proc.poll() is None: 1220 tunnel_proc.terminate() 1221 logging.debug('Terminated tunnel, pid %d', tunnel_proc.pid) 1222 else: 1223 logging.debug('Tunnel pid %d terminated early, status %d', 1224 tunnel_proc.pid, tunnel_proc.returncode) 1225 1226 1227 def get_os_type(self): 1228 """Returns the host OS descriptor (to be implemented in subclasses). 1229 1230 @return A string describing the OS type. 1231 """ 1232 raise NotImplementedError 1233 1234 1235 def check_cached_up_status( 1236 self, expiration_seconds=_DEFAULT_UP_STATUS_EXPIRATION_SECONDS): 1237 """Check if the DUT responded to ping in the past `expiration_seconds`. 1238 1239 @param expiration_seconds: The number of seconds to keep the cached 1240 status of whether the DUT responded to ping. 1241 @return: True if the DUT has responded to ping during the past 1242 `expiration_seconds`. 1243 """ 1244 # Refresh the up status if any of following conditions is true: 1245 # * cached status is never set 1246 # * cached status is False, so the method can check if the host is up 1247 # again. 1248 # * If the cached status is older than `expiration_seconds` 1249 # If we have icmp disabled, treat that as a cached ping. 1250 if not self._use_icmp: 1251 return True 1252 expire_time = time.time() - expiration_seconds 1253 if (self._cached_up_status_updated is None or 1254 not self._cached_up_status or 1255 self._cached_up_status_updated < expire_time): 1256 self._cached_up_status = self.is_up_fast() 1257 self._cached_up_status_updated = time.time() 1258 return self._cached_up_status 1259 1260 1261 def _track_class_usage(self): 1262 """Tracking which class was used. 1263 1264 The idea to identify unused classes to be able clean them up. 1265 We skip names with dynamic created classes where the name is 1266 hostname of the device. 1267 """ 1268 class_name = None 1269 if 'chrome' not in self.__class__.__name__: 1270 class_name = self.__class__.__name__ 1271 else: 1272 for base in self.__class__.__bases__: 1273 if 'chrome' not in base.__name__: 1274 class_name = base.__name__ 1275 break 1276 if class_name: 1277 data = {'host_class': class_name} 1278 metrics.Counter( 1279 'chromeos/autotest/used_hosts').increment(fields=data) 1280 1281 def is_file_exists(self, file_path): 1282 """Check whether a given file is exist on the host. 1283 """ 1284 result = self.run('test -f ' + file_path, 1285 timeout=30, 1286 ignore_status=True) 1287 return result.exit_status == 0 1288 1289 1290def _client_symlink(sources): 1291 """Return the client symlink if in sources.""" 1292 for source in sources: 1293 if source.endswith(AUTOTEST_CLIENT_SYMLINK_END): 1294 return source 1295 return None 1296