1# Copyright 2019 - The Android Open Source Project 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); 4# you may not use this file except in compliance with the License. 5# You may obtain a copy of the License at 6# 7# http://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12# See the License for the specific language governing permissions and 13# limitations under the License. 14"""Ssh Utilities.""" 15from __future__ import print_function 16import logging 17 18import re 19import subprocess 20import sys 21import threading 22 23from acloud import errors 24from acloud.internal import constants 25from acloud.internal.lib import utils 26 27logger = logging.getLogger(__name__) 28 29_SSH_CMD = ("-i %(rsa_key_file)s -o LogLevel=ERROR -o ControlPath=none " 30 "-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no") 31_SSH_IDENTITY = "-l %(login_user)s %(ip_addr)s" 32_SSH_CMD_MAX_RETRY = 5 33_SSH_CMD_RETRY_SLEEP = 3 34_CONNECTION_TIMEOUT = 10 35_MAX_REPORTED_ERROR_LINES = 10 36_ERROR_MSG_RE = re.compile(r".*]\s*\"(?:message|response)\"\s:\s\"(?P<content>.*)\"") 37_ERROR_MSG_TO_QUOTE_RE = r"(\\u2019)|(\\u2018)" 38_ERROR_MSG_DEL_STYLE_RE = r"(<style.+\/style>)" 39_ERROR_MSG_DEL_TAGS_RE = (r"(<[\/]*(a|b|p|span|ins|code|title)>)|" 40 r"(<(a|span|meta|html|!)[^>]*>)") 41 42 43def _SshCallWait(cmd, timeout=None): 44 """Runs a single SSH command. 45 46 - SSH returns code 0 for "Successful execution". 47 - Use wait() until the process is complete without receiving any output. 48 49 Args: 50 cmd: String of the full SSH command to run, including the SSH binary 51 and its arguments. 52 timeout: Optional integer, number of seconds to give 53 54 Returns: 55 An exit status of 0 indicates that it ran successfully. 56 """ 57 logger.info("Running command \"%s\"", cmd) 58 process = subprocess.Popen(cmd, shell=True, stdin=None, 59 stdout=subprocess.PIPE, stderr=subprocess.STDOUT) 60 if timeout: 61 # TODO: if process is killed, out error message to log. 62 timer = threading.Timer(timeout, process.kill) 63 timer.start() 64 process.wait() 65 if timeout: 66 timer.cancel() 67 return process.returncode 68 69 70def _SshCall(cmd, timeout=None): 71 """Runs a single SSH command. 72 73 - SSH returns code 0 for "Successful execution". 74 - Use communicate() until the process and the child thread are complete. 75 76 Args: 77 cmd: String of the full SSH command to run, including the SSH binary 78 and its arguments. 79 timeout: Optional integer, number of seconds to give 80 81 Returns: 82 An exit status of 0 indicates that it ran successfully. 83 """ 84 logger.info("Running command \"%s\"", cmd) 85 process = subprocess.Popen(cmd, shell=True, stdin=None, 86 stdout=subprocess.PIPE, stderr=subprocess.STDOUT) 87 if timeout: 88 # TODO: if process is killed, out error message to log. 89 timer = threading.Timer(timeout, process.kill) 90 timer.start() 91 process.communicate() 92 if timeout: 93 timer.cancel() 94 return process.returncode 95 96 97def _SshLogOutput(cmd, timeout=None, show_output=False, hide_error_msg=False): 98 """Runs a single SSH command while logging its output and processes its return code. 99 100 Output is streamed to the log at the debug level for more interactive debugging. 101 SSH returns error code 255 for "failed to connect", so this is interpreted as a failure in 102 SSH rather than a failure on the target device and this is converted to a different exception 103 type. 104 105 Args: 106 cmd: String of the full SSH command to run, including the SSH binary and its arguments. 107 timeout: Optional integer, number of seconds to give. 108 show_output: Boolean, True to show command output in screen. 109 hide_error_msg: Boolean, True to hide error message. 110 111 Raises: 112 errors.DeviceConnectionError: Failed to connect to the GCE instance. 113 subprocess.CalledProcessError: The process exited with an error on the instance. 114 errors.LaunchCVDFail: Happened on launch_cvd with specific pattern of error message. 115 """ 116 # Use "exec" to let cmd to inherit the shell process, instead of having the 117 # shell launch a child process which does not get killed. 118 cmd = "exec " + cmd 119 logger.info("Running command \"%s\"", cmd) 120 process = subprocess.Popen(cmd, shell=True, stdin=None, 121 stdout=subprocess.PIPE, stderr=subprocess.STDOUT, 122 universal_newlines=True) 123 if timeout: 124 # TODO: if process is killed, out error message to log. 125 timer = threading.Timer(timeout, process.kill) 126 timer.start() 127 stdout, _ = process.communicate() 128 if stdout: 129 if (show_output or process.returncode != 0) and not hide_error_msg: 130 print(stdout.strip(), file=sys.stderr) 131 else: 132 # fetch_cvd and launch_cvd can be noisy, so left at debug 133 logger.debug(stdout.strip()) 134 if timeout: 135 timer.cancel() 136 if process.returncode == 255: 137 error_msg = (f"Failed to send command to instance {cmd}\n" 138 f"Error message: {_GetErrorMessage(stdout)}") 139 if constants.ERROR_MSG_SSO_INVALID in stdout: 140 raise errors.SshConnectFail(error_msg) 141 raise errors.DeviceConnectionError(error_msg) 142 if process.returncode != 0: 143 if constants.ERROR_MSG_VNC_NOT_SUPPORT in stdout: 144 raise errors.LaunchCVDFail(constants.ERROR_MSG_VNC_NOT_SUPPORT) 145 if constants.ERROR_MSG_WEBRTC_NOT_SUPPORT in stdout: 146 raise errors.LaunchCVDFail(constants.ERROR_MSG_WEBRTC_NOT_SUPPORT) 147 raise subprocess.CalledProcessError(process.returncode, cmd) 148 149 150def _GetErrorMessage(stdout): 151 """Get error message. 152 153 Fetch the content of "message" or "response" from the ssh output and filter 154 unused content then log into report. Once the two fields didn't match, to 155 log last _MAX_REPORTED_ERROR_LINES lines into report. 156 157 Args: 158 stdout: String of the ssh output. 159 160 Returns: 161 String of the formatted ssh output. 162 """ 163 matches = _ERROR_MSG_RE.finditer(stdout) 164 for match in matches: 165 return _FilterUnusedContent(match.group("content")) 166 split_stdout = stdout.splitlines()[-_MAX_REPORTED_ERROR_LINES::] 167 return "\n".join(split_stdout) 168 169def _FilterUnusedContent(content): 170 """Filter unused content from html. 171 172 Remove the html tags and style from content. 173 174 Args: 175 content: String, html content. 176 177 Returns: 178 String without html style or tags. 179 """ 180 content = re.sub(_ERROR_MSG_TO_QUOTE_RE, "'", content) 181 content = re.sub(_ERROR_MSG_DEL_STYLE_RE, "", content, flags=re.DOTALL) 182 content = re.sub(_ERROR_MSG_DEL_TAGS_RE, "", content) 183 content = re.sub(r"\\n", " ", content) 184 return content 185 186 187def ShellCmdWithRetry(cmd, timeout=None, show_output=False, 188 retry=_SSH_CMD_MAX_RETRY): 189 """Runs a shell command on remote device. 190 191 If the network is unstable and causes SSH connect fail, it will retry. When 192 it retry in a short time, you may encounter unstable network. We will use 193 the mechanism of RETRY_BACKOFF_FACTOR. The retry time for each failure is 194 times * retries. 195 196 Args: 197 cmd: String of the full SSH command to run, including the SSH binary and its arguments. 198 timeout: Optional integer, number of seconds to give. 199 show_output: Boolean, True to show command output in screen. 200 retry: Integer, the retry times. 201 202 Raises: 203 errors.DeviceConnectionError: For any non-zero return code of remote_cmd. 204 errors.LaunchCVDFail: Happened on launch_cvd with specific pattern of error message. 205 subprocess.CalledProcessError: The process exited with an error on the instance. 206 """ 207 utils.RetryExceptionType( 208 exception_types=(errors.DeviceConnectionError, 209 errors.LaunchCVDFail, 210 subprocess.CalledProcessError), 211 max_retries=retry, 212 functor=_SshLogOutput, 213 sleep_multiplier=_SSH_CMD_RETRY_SLEEP, 214 retry_backoff_factor=utils.DEFAULT_RETRY_BACKOFF_FACTOR, 215 cmd=cmd, 216 timeout=timeout, 217 show_output=show_output) 218 219 220class IP(): 221 """ A class that control the IP address.""" 222 def __init__(self, external=None, internal=None, ip=None): 223 """Init for IP. 224 Args: 225 external: String, external ip. 226 internal: String, internal ip. 227 ip: String, default ip to set for either external and internal 228 if neither is set. 229 """ 230 self.external = external or ip 231 self.internal = internal or ip 232 233 234class Ssh(): 235 """A class that control the remote instance via the IP address. 236 237 Attributes: 238 _ip: an IP object. 239 _user: String of user login into the instance. 240 _ssh_private_key_path: Path to the private key file. 241 _extra_args_ssh_tunnel: String, extra args for ssh or scp. 242 _report_internal_ip: Boolean, True to use internal ip. 243 _gce_hostname: String, the hostname for ssh connect. 244 """ 245 def __init__(self, ip, user, ssh_private_key_path, 246 extra_args_ssh_tunnel=None, report_internal_ip=False, 247 gce_hostname=None): 248 self._ip = ip.internal if report_internal_ip else ip.external 249 self._user = user 250 self._ssh_private_key_path = ssh_private_key_path 251 self._extra_args_ssh_tunnel = extra_args_ssh_tunnel 252 if gce_hostname: 253 self._ip = gce_hostname 254 self._extra_args_ssh_tunnel = None 255 logger.debug( 256 "To connect with hostname, erase the extra_args_ssh_tunnel: %s", 257 extra_args_ssh_tunnel) 258 259 def Run(self, target_command, timeout=None, show_output=False, 260 retry=_SSH_CMD_MAX_RETRY): 261 """Run a shell command over SSH on a remote instance. 262 263 Example: 264 ssh: 265 base_cmd_list is ["ssh", "-i", "~/private_key_path" ,"-l" , "user", "1.1.1.1"] 266 target_command is "remote command" 267 scp: 268 base_cmd_list is ["scp", "-i", "~/private_key_path"] 269 target_command is "{src_file} {dst_file}" 270 271 Args: 272 target_command: String, text of command to run on the remote instance. 273 timeout: Integer, the maximum time to wait for the command to respond. 274 show_output: Boolean, True to show command output in screen. 275 retry: Integer, the retry times. 276 """ 277 ShellCmdWithRetry(self.GetBaseCmd(constants.SSH_BIN) + " " + target_command, 278 timeout, 279 show_output, 280 retry) 281 282 def GetBaseCmd(self, execute_bin): 283 """Get a base command over SSH on a remote instance. 284 285 Example: 286 execute bin is ssh: 287 ssh -i ~/private_key_path $extra_args -l user 1.1.1.1 288 execute bin is scp: 289 scp -i ~/private_key_path $extra_args 290 291 Args: 292 execute_bin: String, execute type, e.g. ssh or scp. 293 294 Returns: 295 Strings of base connection command. 296 297 Raises: 298 errors.UnknownType: Don't support the execute bin. 299 """ 300 base_cmd = [utils.FindExecutable(execute_bin)] 301 base_cmd.append(_SSH_CMD % {"rsa_key_file": self._ssh_private_key_path}) 302 if self._extra_args_ssh_tunnel: 303 base_cmd.append(self._extra_args_ssh_tunnel) 304 305 if execute_bin == constants.SSH_BIN: 306 base_cmd.append(_SSH_IDENTITY % 307 {"login_user":self._user, "ip_addr":self._ip}) 308 return " ".join(base_cmd) 309 if execute_bin == constants.SCP_BIN: 310 return " ".join(base_cmd) 311 312 raise errors.UnknownType("Don't support the execute bin %s." % execute_bin) 313 314 def GetCmdOutput(self, cmd): 315 """Runs a single SSH command and get its output. 316 317 Args: 318 cmd: String, text of command to run on the remote instance. 319 320 Returns: 321 String of the command output. 322 """ 323 ssh_cmd = "exec " + self.GetBaseCmd(constants.SSH_BIN) + " " + cmd 324 logger.info("Running command \"%s\"", ssh_cmd) 325 process = subprocess.Popen(ssh_cmd, shell=True, stdin=None, 326 stdout=subprocess.PIPE, stderr=subprocess.STDOUT, 327 universal_newlines=True) 328 stdout, _ = process.communicate() 329 return stdout 330 331 def CheckSshConnection(self, timeout): 332 """Run remote 'uptime' ssh command to check ssh connection. 333 334 Args: 335 timeout: Integer, the maximum time to wait for the command to respond. 336 337 Raises: 338 errors.DeviceConnectionError: Ssh isn't ready in the remote instance. 339 """ 340 remote_cmd = [self.GetBaseCmd(constants.SSH_BIN)] 341 remote_cmd.append("uptime") 342 try: 343 _SshLogOutput(" ".join(remote_cmd), timeout, hide_error_msg=True) 344 except subprocess.CalledProcessError as e: 345 raise errors.DeviceConnectionError( 346 "Ssh isn't ready in the remote instance.") from e 347 348 @utils.TimeExecute(function_description="Waiting for SSH server") 349 def WaitForSsh(self, timeout=None, max_retry=_SSH_CMD_MAX_RETRY): 350 """Wait until the remote instance is ready to accept commands over SSH. 351 352 Args: 353 timeout: Integer, the maximum time in seconds to wait for the 354 command to respond. 355 max_retry: Integer, the maximum number of retry. 356 357 Raises: 358 errors.DeviceConnectionError: Ssh isn't ready in the remote instance. 359 """ 360 ssh_timeout = timeout or constants.DEFAULT_SSH_TIMEOUT 361 sleep_multiplier = ssh_timeout / sum(range(max_retry + 1)) 362 logger.debug("Retry with interval time: %s secs", str(sleep_multiplier)) 363 try: 364 utils.RetryExceptionType( 365 exception_types=errors.DeviceConnectionError, 366 max_retries=max_retry, 367 functor=self.CheckSshConnection, 368 sleep_multiplier=sleep_multiplier, 369 retry_backoff_factor=utils.DEFAULT_RETRY_BACKOFF_FACTOR, 370 timeout=_CONNECTION_TIMEOUT) 371 except errors.DeviceConnectionError as ssh_timeout: 372 ssh_cmd = "%s uptime" % self.GetBaseCmd(constants.SSH_BIN) 373 _SshLogOutput(ssh_cmd, timeout=_CONNECTION_TIMEOUT) 374 raise errors.DeviceConnectionError( 375 "Ssh connect timeout.\nYou can try the ssh connect command to " 376 "get detail information: '%s'" % ssh_cmd) from ssh_timeout 377 378 def ScpPushFile(self, src_file, dst_file): 379 """Scp push file to remote. 380 381 Args: 382 src_file: The source file path to be pulled. 383 dst_file: The destination file path the file is pulled to. 384 """ 385 scp_command = [self.GetBaseCmd(constants.SCP_BIN)] 386 scp_command.append(src_file) 387 scp_command.append("%s@%s:%s" %(self._user, self._ip, dst_file)) 388 ShellCmdWithRetry(" ".join(scp_command)) 389 390 def ScpPushFiles(self, src_files, dst_dir): 391 """Push files to one specific folder of remote instance via scp command. 392 393 Args: 394 src_files: The source file path list to be pushed. 395 dst_dir: The destination directory the files to be pushed to. 396 """ 397 scp_command = [self.GetBaseCmd(constants.SCP_BIN)] 398 scp_command.extend(src_files) 399 scp_command.append("%s@%s:%s" % (self._user, self._ip, dst_dir)) 400 ShellCmdWithRetry(" ".join(scp_command)) 401 402 def ScpPullFile(self, src_file, dst_file): 403 """Scp pull file from remote. 404 405 Args: 406 src_file: The source file path to be pulled. 407 dst_file: The destination file path the file is pulled to. 408 """ 409 scp_command = [self.GetBaseCmd(constants.SCP_BIN)] 410 scp_command.append("%s@%s:%s" %(self._user, self._ip, src_file)) 411 scp_command.append(dst_file) 412 ShellCmdWithRetry(" ".join(scp_command)) 413