1# Copyright 2019 - The Android Open Source Project 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); 4# you may not use this file except in compliance with the License. 5# You may obtain a copy of the License at 6# 7# http://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12# See the License for the specific language governing permissions and 13# limitations under the License. 14"""A client that manages Cuttlefish Virtual Device on compute engine. 15 16** CvdComputeClient ** 17 18CvdComputeClient derives from AndroidComputeClient. It manges a google 19compute engine project that is setup for running Cuttlefish Virtual Devices. 20It knows how to create a host instance from Cuttlefish Stable Host Image, fetch 21Android build, and start Android within the host instance. 22 23** Class hierarchy ** 24 25 base_cloud_client.BaseCloudApiClient 26 ^ 27 | 28 gcompute_client.ComputeClient 29 ^ 30 | 31 android_compute_client.AndroidComputeClient 32 ^ 33 | 34 cvd_compute_client_multi_stage.CvdComputeClient 35 36""" 37 38import logging 39import os 40import re 41import subprocess 42import tempfile 43import time 44 45from acloud import errors 46from acloud.internal import constants 47from acloud.internal.lib import android_build_client 48from acloud.internal.lib import android_compute_client 49from acloud.internal.lib import cvd_utils 50from acloud.internal.lib import gcompute_client 51from acloud.internal.lib import utils 52from acloud.internal.lib.ssh import Ssh 53from acloud.setup import mkcert 54 55 56logger = logging.getLogger(__name__) 57 58_DEFAULT_WEBRTC_DEVICE_ID = "cvd-1" 59_FETCHER_NAME = "fetch_cvd" 60_NO_RETRY = 0 61# Launch cvd command for acloud report 62_LAUNCH_CVD_COMMAND = "launch_cvd_command" 63_CONFIG_RE = re.compile(r"^config=(?P<config>.+)") 64_TRUST_REMOTE_INSTANCE_COMMAND = ( 65 f"\"sudo cp -p ~/{constants.WEBRTC_CERTS_PATH}/{constants.SSL_CA_NAME}.pem " 66 f"{constants.SSL_TRUST_CA_DIR}/{constants.SSL_CA_NAME}.crt;" 67 "sudo update-ca-certificates;\"") 68 69 70class CvdComputeClient(android_compute_client.AndroidComputeClient): 71 """Client that manages Android Virtual Device.""" 72 73 DATA_POLICY_CREATE_IF_MISSING = "create_if_missing" 74 # Data policy to customize disk size. 75 DATA_POLICY_ALWAYS_CREATE = "always_create" 76 77 def __init__(self, 78 acloud_config, 79 oauth2_credentials, 80 ins_timeout_secs=None, 81 report_internal_ip=None, 82 gpu=None): 83 """Initialize. 84 85 Args: 86 acloud_config: An AcloudConfig object. 87 oauth2_credentials: An oauth2client.OAuth2Credentials instance. 88 ins_timeout_secs: Integer, the maximum time to wait for the 89 instance ready. 90 report_internal_ip: Boolean to report the internal ip instead of 91 external ip. 92 gpu: String, GPU to attach to the device. 93 """ 94 super().__init__(acloud_config, oauth2_credentials) 95 96 self._build_api = ( 97 android_build_client.AndroidBuildClient(oauth2_credentials)) 98 self._ssh_private_key_path = acloud_config.ssh_private_key_path 99 self._ins_timeout_secs = ins_timeout_secs 100 self._report_internal_ip = report_internal_ip 101 self._gpu = gpu 102 # Store all failures result when creating one or multiple instances. 103 # This attribute is only used by the deprecated create_cf command. 104 self._all_failures = {} 105 self._extra_args_ssh_tunnel = acloud_config.extra_args_ssh_tunnel 106 self._ssh = None 107 self._ip = None 108 self._user = constants.GCE_USER 109 self._openwrt = None 110 self._stage = constants.STAGE_INIT 111 self._execution_time = {constants.TIME_ARTIFACT: 0, 112 constants.TIME_GCE: 0, 113 constants.TIME_LAUNCH: 0} 114 115 def InitRemoteHost(self, ssh, ip, user, base_dir): 116 """Init remote host. 117 118 Check if we can ssh to the remote host, stop any cf instances running 119 on it, and remove existing files. 120 121 Args: 122 ssh: Ssh object. 123 ip: namedtuple (internal, external) that holds IP address of the 124 remote host, e.g. "external:140.110.20.1, internal:10.0.0.1" 125 user: String of user log in to the instance. 126 base_dir: The remote directory containing the images and tools. 127 """ 128 self.SetStage(constants.STAGE_SSH_CONNECT) 129 self._ssh = ssh 130 self._ip = ip 131 self._user = user 132 self._ssh.WaitForSsh(timeout=self._ins_timeout_secs) 133 cvd_utils.CleanUpRemoteCvd(self._ssh, base_dir, raise_error=False) 134 135 # pylint: disable=arguments-differ,broad-except 136 def CreateInstance(self, instance, image_name, image_project, 137 avd_spec, extra_scopes=None): 138 """Create/Reuse a GCE instance. 139 140 Args: 141 instance: instance name. 142 image_name: A string, the name of the GCE image. 143 image_project: A string, name of the project where the image lives. 144 Assume the default project if None. 145 avd_spec: An AVDSpec instance. 146 extra_scopes: A list of extra scopes to be passed to the instance. 147 148 Returns: 149 A string, representing instance name. 150 """ 151 # A blank data disk would be created on the host. Make sure the size of 152 # the boot disk is large enough to hold it. 153 boot_disk_size_gb = ( 154 int(self.GetImage(image_name, image_project)["diskSizeGb"]) + 155 avd_spec.cfg.extra_data_disk_size_gb) 156 157 if avd_spec.instance_name_to_reuse: 158 self._ip = self._ReusingGceInstance(avd_spec) 159 else: 160 self._VerifyZoneByQuota() 161 self._ip = self._CreateGceInstance(instance, image_name, image_project, 162 extra_scopes, boot_disk_size_gb, 163 avd_spec) 164 if avd_spec.connect_hostname: 165 self._gce_hostname = gcompute_client.GetGCEHostName( 166 self._project, instance, self._zone) 167 self._ssh = Ssh(ip=self._ip, 168 user=constants.GCE_USER, 169 ssh_private_key_path=self._ssh_private_key_path, 170 extra_args_ssh_tunnel=self._extra_args_ssh_tunnel, 171 report_internal_ip=self._report_internal_ip, 172 gce_hostname=self._gce_hostname) 173 try: 174 self.SetStage(constants.STAGE_SSH_CONNECT) 175 self._ssh.WaitForSsh(timeout=self._ins_timeout_secs) 176 if avd_spec.instance_name_to_reuse: 177 cvd_utils.CleanUpRemoteCvd(self._ssh, cvd_utils.GCE_BASE_DIR, 178 raise_error=False) 179 except Exception as e: 180 self._all_failures[instance] = e 181 return instance 182 183 def _GetGCEHostName(self, instance): 184 """Get the GCE host name with specific rule. 185 186 Args: 187 instance: Sting, instance name. 188 189 Returns: 190 One host name coverted by instance name, project name, and zone. 191 """ 192 if ":" in self._project: 193 domain = self._project.split(":")[0] 194 project_no_domain = self._project.split(":")[1] 195 project = f"{project_no_domain}.{domain}" 196 return f"nic0.{instance}.{self._zone}.c.{project}.internal.gcpnode.com" 197 return f"nic0.{instance}.{self._zone}.c.{self._project}.internal.gcpnode.com" 198 199 def _GetConfigFromAndroidInfo(self, base_dir): 200 """Get config value from android-info.txt. 201 202 The config in android-info.txt would like "config=phone". 203 204 Args: 205 base_dir: The remote directory containing the images. 206 207 Returns: 208 Strings of config value. 209 """ 210 android_info = self._ssh.GetCmdOutput( 211 f"cat {base_dir}/{constants.ANDROID_INFO_FILE}") 212 logger.debug("Android info: %s", android_info) 213 config_match = _CONFIG_RE.match(android_info) 214 if config_match: 215 return config_match.group("config") 216 return None 217 218 @utils.TimeExecute(function_description="Launching AVD(s) and waiting for boot up", 219 result_evaluator=utils.BootEvaluator) 220 def LaunchCvd(self, instance, avd_spec, base_dir, extra_args): 221 """Launch CVD. 222 223 Launch AVD with launch_cvd. If the process is failed, acloud would show 224 error messages and auto download log files from remote instance. 225 226 Args: 227 instance: String, instance name. 228 avd_spec: An AVDSpec instance. 229 base_dir: The remote directory containing the images and tools. 230 extra_args: Collection of strings, the extra arguments generated by 231 acloud. e.g., remote image paths. 232 233 Returns: 234 dict of faliures, return this dict for BootEvaluator to handle 235 LaunchCvd success or fail messages. 236 """ 237 self.SetStage(constants.STAGE_BOOT_UP) 238 timestart = time.time() 239 error_msg = "" 240 launch_cvd_args = list(extra_args) 241 config = self._GetConfigFromAndroidInfo(base_dir) 242 launch_cvd_args.extend(cvd_utils.GetLaunchCvdArgs(avd_spec, config)) 243 244 boot_timeout_secs = self._GetBootTimeout( 245 avd_spec.boot_timeout_secs or constants.DEFAULT_CF_BOOT_TIMEOUT) 246 ssh_command = (f"'HOME=$HOME/{base_dir} " 247 f"{base_dir}/bin/launch_cvd -daemon " 248 f"{' '.join(launch_cvd_args)}'") 249 try: 250 self.ExtendReportData(_LAUNCH_CVD_COMMAND, ssh_command) 251 self._ssh.Run(ssh_command, boot_timeout_secs, retry=_NO_RETRY) 252 self._UpdateOpenWrtStatus(avd_spec) 253 except (subprocess.CalledProcessError, errors.DeviceConnectionError, 254 errors.LaunchCVDFail) as e: 255 error_msg = (f"Device {instance} did not finish on boot within " 256 f"timeout ({boot_timeout_secs} secs)") 257 if constants.ERROR_MSG_VNC_NOT_SUPPORT in str(e): 258 error_msg = ( 259 "VNC is not supported in the current build. Please try WebRTC such " 260 "as '$acloud create' or '$acloud create --autoconnect webrtc'") 261 if constants.ERROR_MSG_WEBRTC_NOT_SUPPORT in str(e): 262 error_msg = ( 263 "WEBRTC is not supported in the current build. Please try VNC such " 264 "as '$acloud create --autoconnect vnc'") 265 utils.PrintColorString(str(e), utils.TextColors.FAIL) 266 267 self._execution_time[constants.TIME_LAUNCH] = time.time() - timestart 268 return {instance: error_msg} if error_msg else {} 269 270 def _GetBootTimeout(self, timeout_secs): 271 """Get boot timeout. 272 273 Timeout settings includes download artifacts and boot up. 274 275 Args: 276 timeout_secs: integer of timeout value. 277 278 Returns: 279 The timeout values for device boots up. 280 """ 281 boot_timeout_secs = timeout_secs - self._execution_time[constants.TIME_ARTIFACT] 282 logger.debug("Timeout for boot: %s secs", boot_timeout_secs) 283 return boot_timeout_secs 284 285 @utils.TimeExecute(function_description="Reusing GCE instance") 286 def _ReusingGceInstance(self, avd_spec): 287 """Reusing a cuttlefish existing instance. 288 289 Args: 290 avd_spec: An AVDSpec instance. 291 292 Returns: 293 ssh.IP object, that stores internal and external ip of the instance. 294 """ 295 gcompute_client.ComputeClient.AddSshRsaInstanceMetadata( 296 self, constants.GCE_USER, avd_spec.cfg.ssh_public_key_path, 297 avd_spec.instance_name_to_reuse) 298 ip = gcompute_client.ComputeClient.GetInstanceIP( 299 self, instance=avd_spec.instance_name_to_reuse, zone=self._zone) 300 301 return ip 302 303 @utils.TimeExecute(function_description="Creating GCE instance") 304 def _CreateGceInstance(self, instance, image_name, image_project, 305 extra_scopes, boot_disk_size_gb, avd_spec): 306 """Create a single configured cuttlefish device. 307 308 Override method from parent class. 309 Args: 310 instance: String, instance name. 311 image_name: String, the name of the GCE image. 312 image_project: String, the name of the project where the image. 313 extra_scopes: A list of extra scopes to be passed to the instance. 314 boot_disk_size_gb: Integer, size of the boot disk in GB. 315 avd_spec: An AVDSpec instance. 316 317 Returns: 318 ssh.IP object, that stores internal and external ip of the instance. 319 """ 320 self.SetStage(constants.STAGE_GCE) 321 timestart = time.time() 322 metadata = self._metadata.copy() 323 324 if avd_spec: 325 metadata[constants.INS_KEY_AVD_TYPE] = avd_spec.avd_type 326 metadata[constants.INS_KEY_AVD_FLAVOR] = avd_spec.flavor 327 metadata[constants.INS_KEY_WEBRTC_DEVICE_ID] = ( 328 avd_spec.webrtc_device_id or _DEFAULT_WEBRTC_DEVICE_ID) 329 metadata[constants.INS_KEY_DISPLAY] = ("%sx%s (%s)" % ( 330 avd_spec.hw_property[constants.HW_X_RES], 331 avd_spec.hw_property[constants.HW_Y_RES], 332 avd_spec.hw_property[constants.HW_ALIAS_DPI])) 333 if avd_spec.gce_metadata: 334 for key, value in avd_spec.gce_metadata.items(): 335 metadata[key] = value 336 # Record webrtc port, it will be removed if cvd support to show it. 337 if avd_spec.connect_webrtc: 338 metadata[constants.INS_KEY_WEBRTC_PORT] = constants.WEBRTC_LOCAL_PORT 339 340 disk_args = self._GetDiskArgs( 341 instance, image_name, image_project, boot_disk_size_gb) 342 disable_external_ip = avd_spec.disable_external_ip if avd_spec else False 343 gcompute_client.ComputeClient.CreateInstance( 344 self, 345 instance=instance, 346 image_name=image_name, 347 image_project=image_project, 348 disk_args=disk_args, 349 metadata=metadata, 350 machine_type=self._machine_type, 351 network=self._network, 352 zone=self._zone, 353 gpu=self._gpu, 354 disk_type=avd_spec.disk_type if avd_spec else None, 355 extra_scopes=extra_scopes, 356 disable_external_ip=disable_external_ip) 357 ip = gcompute_client.ComputeClient.GetInstanceIP( 358 self, instance=instance, zone=self._zone) 359 logger.debug("'instance_ip': %s", ip.internal 360 if self._report_internal_ip else ip.external) 361 362 self._execution_time[constants.TIME_GCE] = time.time() - timestart 363 return ip 364 365 @utils.TimeExecute(function_description="Uploading build fetcher to instance") 366 def UpdateFetchCvd(self, fetch_cvd_version): 367 """Download fetch_cvd from the Build API, and upload it to a remote instance. 368 369 The version of fetch_cvd to use is retrieved from the configuration file. Once fetch_cvd 370 is on the instance, future commands can use it to download relevant Cuttlefish files from 371 the Build API on the instance itself. 372 373 Args: 374 fetch_cvd_version: String. The build id of fetch_cvd. 375 """ 376 self.SetStage(constants.STAGE_ARTIFACT) 377 download_dir = tempfile.mkdtemp() 378 download_target = os.path.join(download_dir, _FETCHER_NAME) 379 self._build_api.DownloadFetchcvd(download_target, fetch_cvd_version) 380 self._ssh.ScpPushFile(src_file=download_target, dst_file=_FETCHER_NAME) 381 os.remove(download_target) 382 os.rmdir(download_dir) 383 384 @utils.TimeExecute(function_description="Downloading build on instance") 385 def FetchBuild(self, default_build_info, system_build_info, 386 kernel_build_info, boot_build_info, bootloader_build_info, 387 ota_build_info): 388 """Execute fetch_cvd on the remote instance to get Cuttlefish runtime files. 389 390 Args: 391 default_build_info: The build that provides full cuttlefish images. 392 system_build_info: The build that provides the system image. 393 kernel_build_info: The build that provides the kernel. 394 boot_build_info: The build that provides the boot image. 395 bootloader_build_info: The build that provides the bootloader. 396 ota_build_info: The build that provides the OTA tools. 397 398 Returns: 399 List of string args for fetch_cvd. 400 """ 401 timestart = time.time() 402 fetch_cvd_args = ["-credential_source=gce"] 403 fetch_cvd_build_args = self._build_api.GetFetchBuildArgs( 404 default_build_info, system_build_info, kernel_build_info, 405 boot_build_info, bootloader_build_info, ota_build_info) 406 fetch_cvd_args.extend(fetch_cvd_build_args) 407 408 self._ssh.Run("./fetch_cvd " + " ".join(fetch_cvd_args), 409 timeout=constants.DEFAULT_SSH_TIMEOUT) 410 self._execution_time[constants.TIME_ARTIFACT] = time.time() - timestart 411 412 @utils.TimeExecute(function_description="Update instance's certificates") 413 def UpdateCertificate(self): 414 """Update webrtc default certificates of the remote instance. 415 416 For trusting both the localhost and remote instance, the process will 417 upload certificates(rootCA.pem, server.crt, server.key) and the mkcert 418 tool from the client workstation to remote instance where running the 419 mkcert with the uploaded rootCA file and replace the webrtc frontend 420 default certificates for connecting to a remote webrtc AVD without the 421 insecure warning. 422 """ 423 local_cert_dir = os.path.join(os.path.expanduser("~"), 424 constants.SSL_DIR) 425 if mkcert.AllocateLocalHostCert(): 426 upload_files = [] 427 for cert_file in (constants.WEBRTC_CERTS_FILES + 428 [f"{constants.SSL_CA_NAME}.pem"]): 429 upload_files.append(os.path.join(local_cert_dir, 430 cert_file)) 431 try: 432 self._ssh.ScpPushFiles(upload_files, constants.WEBRTC_CERTS_PATH) 433 self._ssh.Run(_TRUST_REMOTE_INSTANCE_COMMAND) 434 except subprocess.CalledProcessError: 435 logger.debug("Update WebRTC frontend certificate failed.") 436 437 @utils.TimeExecute(function_description="Upload extra files to instance") 438 def UploadExtraFiles(self, extra_files): 439 """Upload extra files into GCE instance. 440 441 Args: 442 extra_files: List of namedtuple ExtraFile. 443 444 Raises: 445 errors.CheckPathError: The provided path doesn't exist. 446 """ 447 for extra_file in extra_files: 448 if not os.path.exists(extra_file.source): 449 raise errors.CheckPathError( 450 f"The path doesn't exist: {extra_file.source}") 451 self._ssh.ScpPushFile(extra_file.source, extra_file.target) 452 453 def GetSshConnectCmd(self): 454 """Get ssh connect command. 455 456 Returns: 457 String of ssh connect command. 458 """ 459 return self._ssh.GetBaseCmd(constants.SSH_BIN) 460 461 def GetInstanceIP(self, instance=None): 462 """Override method from parent class. 463 464 It need to get the IP address in the common_operation. If the class 465 already defind the ip address, return the ip address. 466 467 Args: 468 instance: String, representing instance name. 469 470 Returns: 471 ssh.IP object, that stores internal and external ip of the instance. 472 """ 473 if self._ip: 474 return self._ip 475 return gcompute_client.ComputeClient.GetInstanceIP( 476 self, instance=instance, zone=self._zone) 477 478 def GetHostImageName(self, stable_image_name, image_family, image_project): 479 """Get host image name. 480 481 Args: 482 stable_image_name: String of stable host image name. 483 image_family: String of image family. 484 image_project: String of image project. 485 486 Returns: 487 String of stable host image name. 488 489 Raises: 490 errors.ConfigError: There is no host image name in config file. 491 """ 492 if stable_image_name: 493 return stable_image_name 494 495 if image_family: 496 image_name = gcompute_client.ComputeClient.GetImageFromFamily( 497 self, image_family, image_project)["name"] 498 logger.debug("Get the host image name from image family: %s", image_name) 499 return image_name 500 501 raise errors.ConfigError( 502 "Please specify 'stable_host_image_name' or 'stable_host_image_family'" 503 " in config.") 504 505 def SetStage(self, stage): 506 """Set stage to know the create progress. 507 508 Args: 509 stage: Integer, the stage would like STAGE_INIT, STAGE_GCE. 510 """ 511 self._stage = stage 512 513 def _UpdateOpenWrtStatus(self, avd_spec): 514 """Update the OpenWrt device status. 515 516 Args: 517 avd_spec: An AVDSpec instance. 518 """ 519 self._openwrt = avd_spec.openwrt if avd_spec else False 520 521 @property 522 def all_failures(self): 523 """Return all_failures""" 524 return self._all_failures 525 526 @property 527 def execution_time(self): 528 """Return execution_time""" 529 return self._execution_time 530 531 @property 532 def stage(self): 533 """Return stage""" 534 return self._stage 535 536 @property 537 def openwrt(self): 538 """Return openwrt""" 539 return self._openwrt 540 541 @property 542 def build_api(self): 543 """Return build_api""" 544 return self._build_api 545