1# Copyright 2022 - The Android Open Source Project 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); 4# you may not use this file except in compliance with the License. 5# You may obtain a copy of the License at 6# 7# http://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12# See the License for the specific language governing permissions and 13# limitations under the License. 14 15"""RemoteHostDeviceFactory implements the device factory interface and creates 16cuttlefish instances on a remote host.""" 17 18import glob 19import json 20import logging 21import os 22import posixpath as remote_path 23import shutil 24import subprocess 25import tempfile 26import time 27 28from acloud import errors 29from acloud.internal import constants 30from acloud.internal.lib import auth 31from acloud.internal.lib import cvd_compute_client_multi_stage 32from acloud.internal.lib import cvd_utils 33from acloud.internal.lib import utils 34from acloud.internal.lib import ssh 35from acloud.public.actions import base_device_factory 36from acloud.pull import pull 37 38 39logger = logging.getLogger(__name__) 40_ALL_FILES = "*" 41_HOME_FOLDER = os.path.expanduser("~") 42_SCREEN_CONSOLE_COMMAND = "screen ~/cuttlefish_runtime/console" 43 44 45class RemoteHostDeviceFactory(base_device_factory.BaseDeviceFactory): 46 """A class that can produce a cuttlefish device. 47 48 Attributes: 49 avd_spec: AVDSpec object that tells us what we're going to create. 50 local_image_artifact: A string, path to local image. 51 cvd_host_package_artifact: A string, path to cvd host package. 52 all_failures: A dictionary mapping instance names to errors. 53 all_logs: A dictionary mapping instance names to lists of 54 report.LogFile. 55 compute_client: An object of cvd_compute_client.CvdComputeClient. 56 ssh: An Ssh object. 57 """ 58 59 _USER_BUILD = "userbuild" 60 61 def __init__(self, avd_spec, local_image_artifact=None, 62 cvd_host_package_artifact=None): 63 """Initialize attributes.""" 64 self._avd_spec = avd_spec 65 self._local_image_artifact = local_image_artifact 66 self._cvd_host_package_artifact = cvd_host_package_artifact 67 self._all_failures = {} 68 self._all_logs = {} 69 credentials = auth.CreateCredentials(avd_spec.cfg) 70 compute_client = cvd_compute_client_multi_stage.CvdComputeClient( 71 acloud_config=avd_spec.cfg, 72 oauth2_credentials=credentials, 73 ins_timeout_secs=avd_spec.ins_timeout_secs, 74 report_internal_ip=avd_spec.report_internal_ip, 75 gpu=avd_spec.gpu) 76 super().__init__(compute_client) 77 self._ssh = None 78 79 def CreateInstance(self): 80 """Create a single configured cuttlefish device. 81 82 Returns: 83 A string, representing instance name. 84 """ 85 init_remote_host_timestart = time.time() 86 instance = self._InitRemotehost() 87 self._compute_client.execution_time[constants.TIME_GCE] = ( 88 time.time() - init_remote_host_timestart) 89 90 process_artifacts_timestart = time.time() 91 image_args = self._ProcessRemoteHostArtifacts() 92 self._compute_client.execution_time[constants.TIME_ARTIFACT] = ( 93 time.time() - process_artifacts_timestart) 94 95 launch_cvd_timestart = time.time() 96 failures = self._compute_client.LaunchCvd( 97 instance, self._avd_spec, self._GetInstancePath(), image_args) 98 self._compute_client.execution_time[constants.TIME_LAUNCH] = ( 99 time.time() - launch_cvd_timestart) 100 101 self._all_failures.update(failures) 102 self._FindLogFiles( 103 instance, instance in failures and not self._avd_spec.no_pull_log) 104 return instance 105 106 def _GetInstancePath(self, relative_path=""): 107 """Append a relative path to the remote base directory. 108 109 Args: 110 relative_path: The remote relative path. 111 112 Returns: 113 The remote base directory if relative_path is empty. 114 The remote path under the base directory otherwise. 115 """ 116 base_dir = cvd_utils.GetRemoteHostBaseDir( 117 self._avd_spec.base_instance_num) 118 return (remote_path.join(base_dir, relative_path) if relative_path else 119 base_dir) 120 121 def _InitRemotehost(self): 122 """Initialize remote host. 123 124 Determine the remote host instance name, and activate ssh. It need to 125 get the IP address in the common_operation. So need to pass the IP and 126 ssh to compute_client. 127 128 build_target: The format is like "aosp_cf_x86_phone". We only get info 129 from the user build image file name. If the file name is 130 not custom format (no "-"), we will use $TARGET_PRODUCT 131 from environment variable as build_target. 132 133 Returns: 134 A string, representing instance name. 135 """ 136 image_name = os.path.basename( 137 self._local_image_artifact) if self._local_image_artifact else "" 138 build_target = (os.environ.get(constants.ENV_BUILD_TARGET) 139 if "-" not in image_name else 140 image_name.split("-", maxsplit=1)[0]) 141 build_id = self._USER_BUILD 142 if self._avd_spec.image_source == constants.IMAGE_SRC_REMOTE: 143 build_id = self._avd_spec.remote_image[constants.BUILD_ID] 144 145 instance = cvd_utils.FormatRemoteHostInstanceName( 146 self._avd_spec.remote_host, self._avd_spec.base_instance_num, 147 build_id, build_target) 148 ip = ssh.IP(ip=self._avd_spec.remote_host) 149 self._ssh = ssh.Ssh( 150 ip=ip, 151 user=self._avd_spec.host_user, 152 ssh_private_key_path=(self._avd_spec.host_ssh_private_key_path or 153 self._avd_spec.cfg.ssh_private_key_path), 154 extra_args_ssh_tunnel=self._avd_spec.cfg.extra_args_ssh_tunnel, 155 report_internal_ip=self._avd_spec.report_internal_ip) 156 self._compute_client.InitRemoteHost( 157 self._ssh, ip, self._avd_spec.host_user, self._GetInstancePath()) 158 return instance 159 160 def _ProcessRemoteHostArtifacts(self): 161 """Process remote host artifacts. 162 163 - If images source is local, tool will upload images from local site to 164 remote host. 165 - If images source is remote, tool will download images from android 166 build to local and unzip it then upload to remote host, because there 167 is no permission to fetch build rom on the remote host. 168 169 Returns: 170 A list of strings, the launch_cvd arguments. 171 """ 172 self._compute_client.SetStage(constants.STAGE_ARTIFACT) 173 self._ssh.Run(f"mkdir -p {self._GetInstancePath()}") 174 if self._avd_spec.image_source == constants.IMAGE_SRC_LOCAL: 175 cvd_utils.UploadArtifacts( 176 self._ssh, self._GetInstancePath(), 177 self._local_image_artifact or self._avd_spec.local_image_dir, 178 self._cvd_host_package_artifact) 179 else: 180 try: 181 artifacts_path = tempfile.mkdtemp() 182 logger.debug("Extracted path of artifacts: %s", artifacts_path) 183 if self._avd_spec.remote_fetch: 184 # TODO: Check fetch cvd wrapper file is valid. 185 if self._avd_spec.fetch_cvd_wrapper: 186 self._UploadFetchCvd(artifacts_path) 187 self._DownloadArtifactsByFetchWrapper() 188 else: 189 self._UploadFetchCvd(artifacts_path) 190 self._DownloadArtifactsRemotehost() 191 else: 192 self._DownloadArtifacts(artifacts_path) 193 self._UploadRemoteImageArtifacts(artifacts_path) 194 finally: 195 shutil.rmtree(artifacts_path) 196 197 return cvd_utils.UploadExtraImages(self._ssh, self._GetInstancePath(), 198 self._avd_spec) 199 200 def _GetRemoteFetchCredentialArg(self): 201 """Get the credential source argument for remote fetch_cvd. 202 203 Remote fetch_cvd uses the service account key uploaded by 204 _UploadFetchCvd if it is available. Otherwise, fetch_cvd uses the 205 token extracted from the local credential file. 206 207 Returns: 208 A string, the credential source argument. 209 """ 210 cfg = self._avd_spec.cfg 211 if cfg.service_account_json_private_key_path: 212 return "-credential_source=" + self._GetInstancePath( 213 constants.FETCH_CVD_CREDENTIAL_SOURCE) 214 215 return self._compute_client.build_api.GetFetchCertArg( 216 os.path.join(_HOME_FOLDER, cfg.creds_cache_file)) 217 218 @utils.TimeExecute( 219 function_description="Downloading artifacts on remote host by fetch cvd wrapper.") 220 def _DownloadArtifactsByFetchWrapper(self): 221 """Generate fetch_cvd args and run fetch cvd wrapper on remote host to download artifacts. 222 223 Fetch cvd wrapper will fetch from cluster cached artifacts, and fallback to fetch_cvd if 224 the artifacts not exist. 225 """ 226 fetch_cvd_build_args = self._compute_client.build_api.GetFetchBuildArgs( 227 self._avd_spec.remote_image, 228 self._avd_spec.system_build_info, 229 self._avd_spec.kernel_build_info, 230 self._avd_spec.boot_build_info, 231 self._avd_spec.bootloader_build_info, 232 self._avd_spec.ota_build_info) 233 234 fetch_cvd_args = self._avd_spec.fetch_cvd_wrapper.split(',') + [ 235 f"-directory={self._GetInstancePath()}", 236 f"-fetch_cvd_path={self._GetInstancePath(constants.FETCH_CVD)}", 237 self._GetRemoteFetchCredentialArg()] 238 fetch_cvd_args.extend(fetch_cvd_build_args) 239 240 ssh_cmd = self._ssh.GetBaseCmd(constants.SSH_BIN) 241 cmd = (f"{ssh_cmd} -- " + " ".join(fetch_cvd_args)) 242 logger.debug("cmd:\n %s", cmd) 243 ssh.ShellCmdWithRetry(cmd) 244 245 @utils.TimeExecute(function_description="Downloading artifacts on remote host") 246 def _DownloadArtifactsRemotehost(self): 247 """Generate fetch_cvd args and run fetch_cvd on remote host to download artifacts.""" 248 fetch_cvd_build_args = self._compute_client.build_api.GetFetchBuildArgs( 249 self._avd_spec.remote_image, 250 self._avd_spec.system_build_info, 251 self._avd_spec.kernel_build_info, 252 self._avd_spec.boot_build_info, 253 self._avd_spec.bootloader_build_info, 254 self._avd_spec.ota_build_info) 255 256 fetch_cvd_args = [self._GetInstancePath(constants.FETCH_CVD), 257 f"-directory={self._GetInstancePath()}", 258 self._GetRemoteFetchCredentialArg()] 259 fetch_cvd_args.extend(fetch_cvd_build_args) 260 261 ssh_cmd = self._ssh.GetBaseCmd(constants.SSH_BIN) 262 cmd = (f"{ssh_cmd} -- " + " ".join(fetch_cvd_args)) 263 logger.debug("cmd:\n %s", cmd) 264 ssh.ShellCmdWithRetry(cmd) 265 266 @utils.TimeExecute(function_description="Download and upload fetch_cvd") 267 def _UploadFetchCvd(self, extract_path): 268 """Download fetch_cvd, duplicate service account json private key when available and upload 269 to remote host. 270 271 Args: 272 extract_path: String, a path include extracted files. 273 """ 274 cfg = self._avd_spec.cfg 275 is_arm_img = (cvd_utils.IsArmImage(self._avd_spec.remote_image) 276 and self._avd_spec.remote_fetch) 277 fetch_cvd = os.path.join(extract_path, constants.FETCH_CVD) 278 self._compute_client.build_api.DownloadFetchcvd( 279 fetch_cvd, self._avd_spec.fetch_cvd_version, is_arm_img) 280 # Duplicate fetch_cvd API key when available 281 if cfg.service_account_json_private_key_path: 282 shutil.copyfile( 283 cfg.service_account_json_private_key_path, 284 os.path.join(extract_path, constants.FETCH_CVD_CREDENTIAL_SOURCE)) 285 286 self._UploadRemoteImageArtifacts(extract_path) 287 288 @utils.TimeExecute(function_description="Downloading Android Build artifact") 289 def _DownloadArtifacts(self, extract_path): 290 """Download the CF image artifacts and process them. 291 292 - Download images from the Android Build system. 293 - Download cvd host package from the Android Build system. 294 295 Args: 296 extract_path: String, a path include extracted files. 297 298 Raises: 299 errors.GetRemoteImageError: Fails to download rom images. 300 """ 301 cfg = self._avd_spec.cfg 302 303 # Download images with fetch_cvd 304 fetch_cvd = os.path.join(extract_path, constants.FETCH_CVD) 305 self._compute_client.build_api.DownloadFetchcvd( 306 fetch_cvd, self._avd_spec.fetch_cvd_version) 307 fetch_cvd_build_args = self._compute_client.build_api.GetFetchBuildArgs( 308 self._avd_spec.remote_image, 309 self._avd_spec.system_build_info, 310 self._avd_spec.kernel_build_info, 311 self._avd_spec.boot_build_info, 312 self._avd_spec.bootloader_build_info, 313 self._avd_spec.ota_build_info) 314 creds_cache_file = os.path.join(_HOME_FOLDER, cfg.creds_cache_file) 315 fetch_cvd_cert_arg = self._compute_client.build_api.GetFetchCertArg( 316 creds_cache_file) 317 fetch_cvd_args = [fetch_cvd, f"-directory={extract_path}", 318 fetch_cvd_cert_arg] 319 fetch_cvd_args.extend(fetch_cvd_build_args) 320 logger.debug("Download images command: %s", fetch_cvd_args) 321 try: 322 subprocess.check_call(fetch_cvd_args) 323 except subprocess.CalledProcessError as e: 324 raise errors.GetRemoteImageError(f"Fails to download images: {e}") 325 326 @utils.TimeExecute(function_description="Uploading remote image artifacts") 327 def _UploadRemoteImageArtifacts(self, images_dir): 328 """Upload remote image artifacts to instance. 329 330 Args: 331 images_dir: String, directory of local artifacts downloaded by 332 fetch_cvd. 333 """ 334 artifact_files = [ 335 os.path.basename(image) 336 for image in glob.glob(os.path.join(images_dir, _ALL_FILES)) 337 ] 338 ssh_cmd = self._ssh.GetBaseCmd(constants.SSH_BIN) 339 # TODO(b/182259589): Refactor upload image command into a function. 340 cmd = (f"tar -cf - --lzop -S -C {images_dir} " 341 f"{' '.join(artifact_files)} | " 342 f"{ssh_cmd} -- " 343 f"tar -xf - --lzop -S -C {self._GetInstancePath()}") 344 logger.debug("cmd:\n %s", cmd) 345 ssh.ShellCmdWithRetry(cmd) 346 347 def _FindLogFiles(self, instance, download): 348 """Find and pull all log files from instance. 349 350 Args: 351 instance: String, instance name. 352 download: Whether to download the files to a temporary directory 353 and show messages to the user. 354 """ 355 logs = [] 356 if (self._avd_spec.image_source == constants.IMAGE_SRC_REMOTE and 357 self._avd_spec.remote_fetch): 358 logs.append( 359 cvd_utils.GetRemoteFetcherConfigJson(self._GetInstancePath())) 360 logs.extend(cvd_utils.FindRemoteLogs( 361 self._ssh, 362 self._GetInstancePath(), 363 self._avd_spec.base_instance_num, 364 self._avd_spec.num_avds_per_instance)) 365 self._all_logs[instance] = logs 366 367 if download: 368 # To avoid long download time, fetch from the first device only. 369 log_files = pull.GetAllLogFilePaths( 370 self._ssh, self._GetInstancePath(constants.REMOTE_LOG_FOLDER)) 371 error_log_folder = pull.PullLogs(self._ssh, log_files, instance) 372 self._compute_client.ExtendReportData(constants.ERROR_LOG_FOLDER, 373 error_log_folder) 374 375 def GetOpenWrtInfoDict(self): 376 """Get openwrt info dictionary. 377 378 Returns: 379 A openwrt info dictionary. None for the case is not openwrt device. 380 """ 381 if not self._avd_spec.openwrt: 382 return None 383 return {"ssh_command": self._compute_client.GetSshConnectCmd(), 384 "screen_command": _SCREEN_CONSOLE_COMMAND} 385 386 def GetBuildInfoDict(self): 387 """Get build info dictionary. 388 389 Returns: 390 A build info dictionary. None for local image case. 391 """ 392 if self._avd_spec.image_source == constants.IMAGE_SRC_LOCAL: 393 return None 394 return cvd_utils.GetRemoteBuildInfoDict(self._avd_spec) 395 396 def GetAdbPorts(self): 397 """Get ADB ports of the created devices. 398 399 Returns: 400 The port numbers as a list of integers. 401 """ 402 return cvd_utils.GetAdbPorts(self._avd_spec.base_instance_num, 403 self._avd_spec.num_avds_per_instance) 404 405 def GetFastbootPorts(self): 406 """Get Fastboot ports of the created devices. 407 408 Returns: 409 The port numbers as a list of integers. 410 """ 411 return cvd_utils.GetFastbootPorts(self._avd_spec.base_instance_num, 412 self._avd_spec.num_avds_per_instance) 413 414 def GetVncPorts(self): 415 """Get VNC ports of the created devices. 416 417 Returns: 418 The port numbers as a list of integers. 419 """ 420 return cvd_utils.GetVncPorts(self._avd_spec.base_instance_num, 421 self._avd_spec.num_avds_per_instance) 422 423 def GetFailures(self): 424 """Get failures from all devices. 425 426 Returns: 427 A dictionary that contains all the failures. 428 The key is the name of the instance that fails to boot, 429 and the value is a string or an errors.DeviceBootError object. 430 """ 431 return self._all_failures 432 433 def GetLogs(self): 434 """Get all device logs. 435 436 Returns: 437 A dictionary that maps instance names to lists of report.LogFile. 438 """ 439 return self._all_logs 440 441 def GetFetchCvdWrapperLogIfExist(self): 442 """Get FetchCvdWrapper log if exist. 443 444 Returns: 445 A dictionary that includes FetchCvdWrapper logs. 446 """ 447 if not self._avd_spec.fetch_cvd_wrapper: 448 return {} 449 path = os.path.join(self._GetInstancePath(), "fetch_cvd_wrapper_log.json") 450 ssh_cmd = self._ssh.GetBaseCmd(constants.SSH_BIN) + " cat " + path 451 proc = subprocess.run(ssh_cmd, shell=True, capture_output=True, 452 check=False) 453 if proc.stderr: 454 logger.debug("`%s` stderr: %s", ssh_cmd, proc.stderr.decode()) 455 if proc.stdout: 456 try: 457 return json.loads(proc.stdout) 458 except ValueError as e: 459 return {"status": "FETCH_WRAPPER_REPORT_PARSE_ERROR"} 460 return {} 461