1# Copyright 2022 - The Android Open Source Project 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); 4# you may not use this file except in compliance with the License. 5# You may obtain a copy of the License at 6# 7# http://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12# See the License for the specific language governing permissions and 13# limitations under the License. 14 15"""Utility functions that process cuttlefish images.""" 16 17import glob 18import logging 19import os 20import posixpath as remote_path 21import subprocess 22 23from acloud import errors 24from acloud.create import create_common 25from acloud.internal import constants 26from acloud.internal.lib import ssh 27from acloud.internal.lib import utils 28from acloud.public import report 29 30 31logger = logging.getLogger(__name__) 32 33# bootloader and kernel are files required to launch AVD. 34_ARTIFACT_FILES = ["*.img", "bootloader", "kernel"] 35_REMOTE_IMAGE_DIR = "acloud_cf" 36# The boot image name pattern corresponds to the use cases: 37# - In a cuttlefish build environment, ANDROID_PRODUCT_OUT conatins boot.img 38# and boot-debug.img. The former is the default boot image. The latter is not 39# useful for cuttlefish. 40# - In an officially released GKI (Generic Kernel Image) package, the image 41# name is boot-<kernel version>.img. 42_BOOT_IMAGE_NAME_PATTERN = r"boot(-[\d.]+)?\.img" 43_VENDOR_BOOT_IMAGE_NAME = "vendor_boot.img" 44_KERNEL_IMAGE_NAMES = ("kernel", "bzImage", "Image") 45_INITRAMFS_IMAGE_NAME = "initramfs.img" 46_REMOTE_BOOT_IMAGE_PATH = remote_path.join(_REMOTE_IMAGE_DIR, "boot.img") 47_REMOTE_VENDOR_BOOT_IMAGE_PATH = remote_path.join( 48 _REMOTE_IMAGE_DIR, _VENDOR_BOOT_IMAGE_NAME) 49_REMOTE_KERNEL_IMAGE_PATH = remote_path.join( 50 _REMOTE_IMAGE_DIR, _KERNEL_IMAGE_NAMES[0]) 51_REMOTE_INITRAMFS_IMAGE_PATH = remote_path.join( 52 _REMOTE_IMAGE_DIR, _INITRAMFS_IMAGE_NAME) 53 54_ANDROID_BOOT_IMAGE_MAGIC = b"ANDROID!" 55 56HOST_KERNEL_LOG = report.LogFile( 57 "/var/log/kern.log", constants.LOG_TYPE_KERNEL_LOG, "host_kernel.log") 58TOMBSTONES = report.LogFile( 59 constants.REMOTE_LOG_FOLDER + "/tombstones", constants.LOG_TYPE_DIR, 60 "tombstones-zip") 61FETCHER_CONFIG_JSON = report.LogFile( 62 "fetcher_config.json", constants.LOG_TYPE_TEXT) 63 64 65def _UploadImageZip(ssh_obj, image_zip): 66 """Upload an image zip to a remote host and a GCE instance. 67 68 Args: 69 ssh_obj: An Ssh object. 70 image_zip: The path to the image zip. 71 """ 72 remote_cmd = f"/usr/bin/install_zip.sh . < {image_zip}" 73 logger.debug("remote_cmd:\n %s", remote_cmd) 74 ssh_obj.Run(remote_cmd) 75 76 77def _UploadImageDir(ssh_obj, image_dir): 78 """Upload an image directory to a remote host or a GCE instance. 79 80 The images are compressed for faster upload. 81 82 Args: 83 ssh_obj: An Ssh object. 84 image_dir: The directory containing the files to be uploaded. 85 """ 86 try: 87 images_path = os.path.join(image_dir, "required_images") 88 with open(images_path, "r", encoding="utf-8") as images: 89 artifact_files = images.read().splitlines() 90 except IOError: 91 # Older builds may not have a required_images file. In this case 92 # we fall back to *.img. 93 artifact_files = [] 94 for file_name in _ARTIFACT_FILES: 95 artifact_files.extend( 96 os.path.basename(image) for image in glob.glob( 97 os.path.join(image_dir, file_name))) 98 # Upload android-info.txt to parse config value. 99 artifact_files.append(constants.ANDROID_INFO_FILE) 100 cmd = (f"tar -cf - --lzop -S -C {image_dir} {' '.join(artifact_files)} | " 101 f"{ssh_obj.GetBaseCmd(constants.SSH_BIN)} -- tar -xf - --lzop -S") 102 logger.debug("cmd:\n %s", cmd) 103 ssh.ShellCmdWithRetry(cmd) 104 105 106def _UploadCvdHostPackage(ssh_obj, cvd_host_package): 107 """Upload a CVD host package to a remote host or a GCE instance. 108 109 Args: 110 ssh_obj: An Ssh object. 111 cvd_host_package: The path to the CVD host package. 112 """ 113 remote_cmd = f"tar -x -z -f - < {cvd_host_package}" 114 logger.debug("remote_cmd:\n %s", remote_cmd) 115 ssh_obj.Run(remote_cmd) 116 117 118@utils.TimeExecute(function_description="Processing and uploading local images") 119def UploadArtifacts(ssh_obj, image_path, cvd_host_package): 120 """Upload images and a CVD host package to a remote host or a GCE instance. 121 122 Args: 123 ssh_obj: An Ssh object. 124 image_path: A string, the path to the image zip built by `m dist` or 125 the directory containing the images built by `m`. 126 cvd_host_package: A string, the path to the CVD host package in gzip. 127 """ 128 if os.path.isdir(image_path): 129 _UploadImageDir(ssh_obj, image_path) 130 else: 131 _UploadImageZip(ssh_obj, image_path) 132 _UploadCvdHostPackage(ssh_obj, cvd_host_package) 133 134 135def _IsBootImage(image_path): 136 """Check if a file is an Android boot image by reading the magic bytes. 137 138 Args: 139 image_path: The file path. 140 141 Returns: 142 A boolean, whether the file is a boot image. 143 """ 144 if not os.path.isfile(image_path): 145 return False 146 with open(image_path, "rb") as image_file: 147 return image_file.read(8) == _ANDROID_BOOT_IMAGE_MAGIC 148 149 150def FindBootImages(search_path): 151 """Find boot and vendor_boot images in a path. 152 153 Args: 154 search_path: A path to an image file or an image directory. 155 156 Returns: 157 The boot image path and the vendor_boot image path. Each value can be 158 None if the path doesn't exist. 159 160 Raises: 161 errors.GetLocalImageError if search_path contains more than one boot 162 image or the file format is not correct. 163 """ 164 boot_image_path = create_common.FindLocalImage( 165 search_path, _BOOT_IMAGE_NAME_PATTERN, raise_error=False) 166 if boot_image_path and not _IsBootImage(boot_image_path): 167 raise errors.GetLocalImageError( 168 f"{boot_image_path} is not a boot image.") 169 170 vendor_boot_image_path = os.path.join(search_path, _VENDOR_BOOT_IMAGE_NAME) 171 if not os.path.isfile(vendor_boot_image_path): 172 vendor_boot_image_path = None 173 174 return boot_image_path, vendor_boot_image_path 175 176 177def _FindKernelImages(search_path): 178 """Find kernel and initramfs images in a path. 179 180 Args: 181 search_path: A path to an image directory. 182 183 Returns: 184 The kernel image path and the initramfs image path. Each value can be 185 None if the path doesn't exist. 186 """ 187 paths = [os.path.join(search_path, name) for name in _KERNEL_IMAGE_NAMES] 188 kernel_image_path = next((path for path in paths if os.path.isfile(path)), 189 None) 190 191 initramfs_image_path = os.path.join(search_path, _INITRAMFS_IMAGE_NAME) 192 if not os.path.isfile(initramfs_image_path): 193 initramfs_image_path = None 194 195 return kernel_image_path, initramfs_image_path 196 197 198@utils.TimeExecute(function_description="Uploading local kernel images.") 199def _UploadKernelImages(ssh_obj, search_path): 200 """Find and upload kernel images to a remote host or a GCE instance. 201 202 Args: 203 ssh_obj: An Ssh object. 204 search_path: A path to an image file or an image directory. 205 206 Returns: 207 A list of strings, the launch_cvd arguments including the remote paths. 208 209 Raises: 210 errors.GetLocalImageError if search_path does not contain kernel 211 images. 212 """ 213 # Assume that the caller cleaned up the remote home directory. 214 ssh_obj.Run("mkdir -p " + _REMOTE_IMAGE_DIR) 215 216 boot_image_path, vendor_boot_image_path = FindBootImages(search_path) 217 if boot_image_path: 218 ssh_obj.ScpPushFile(boot_image_path, _REMOTE_BOOT_IMAGE_PATH) 219 launch_cvd_args = ["-boot_image", _REMOTE_BOOT_IMAGE_PATH] 220 if vendor_boot_image_path: 221 ssh_obj.ScpPushFile(vendor_boot_image_path, 222 _REMOTE_VENDOR_BOOT_IMAGE_PATH) 223 launch_cvd_args.extend(["-vendor_boot_image", 224 _REMOTE_VENDOR_BOOT_IMAGE_PATH]) 225 return launch_cvd_args 226 227 kernel_image_path, initramfs_image_path = _FindKernelImages(search_path) 228 if kernel_image_path and initramfs_image_path: 229 ssh_obj.ScpPushFile(kernel_image_path, _REMOTE_KERNEL_IMAGE_PATH) 230 ssh_obj.ScpPushFile(initramfs_image_path, _REMOTE_INITRAMFS_IMAGE_PATH) 231 return ["-kernel_path", _REMOTE_KERNEL_IMAGE_PATH, 232 "-initramfs_path", _REMOTE_INITRAMFS_IMAGE_PATH] 233 234 raise errors.GetLocalImageError( 235 f"{search_path} is not a boot image or a directory containing images.") 236 237 238def UploadExtraImages(ssh_obj, avd_spec): 239 """Find and upload the images specified in avd_spec. 240 241 Args: 242 ssh_obj: An Ssh object. 243 avd_spec: An AvdSpec object containing extra image paths. 244 245 Returns: 246 A list of strings, the launch_cvd arguments including the remote paths. 247 248 Raises: 249 errors.GetLocalImageError if any specified image path does not exist. 250 """ 251 if avd_spec.local_kernel_image: 252 return _UploadKernelImages(ssh_obj, avd_spec.local_kernel_image) 253 return [] 254 255 256def CleanUpRemoteCvd(ssh_obj, raise_error): 257 """Call stop_cvd and delete the files on a remote host or a GCE instance. 258 259 Args: 260 ssh_obj: An Ssh object. 261 raise_error: Whether to raise an error if the remote instance is not 262 running. 263 264 Raises: 265 subprocess.CalledProcessError if any command fails. 266 """ 267 stop_cvd_cmd = "./bin/stop_cvd" 268 if raise_error: 269 ssh_obj.Run(stop_cvd_cmd) 270 else: 271 try: 272 ssh_obj.Run(stop_cvd_cmd, retry=0) 273 except subprocess.CalledProcessError as e: 274 logger.debug( 275 "Failed to stop_cvd (possibly no running device): %s", e) 276 277 # This command deletes all files except hidden files under HOME. 278 # It does not raise an error if no files can be deleted. 279 ssh_obj.Run("'rm -rf ./*'") 280 281 282def ConvertRemoteLogs(log_paths): 283 """Convert paths on a remote host or a GCE instance to log objects. 284 285 Args: 286 log_paths: A collection of strings, the remote paths to the logs. 287 288 Returns: 289 A list of report.LogFile objects. 290 """ 291 logs = [] 292 for log_path in log_paths: 293 log = report.LogFile(log_path, constants.LOG_TYPE_TEXT) 294 if log_path.endswith("kernel.log"): 295 log = report.LogFile(log_path, constants.LOG_TYPE_KERNEL_LOG) 296 elif log_path.endswith("logcat"): 297 log = report.LogFile(log_path, constants.LOG_TYPE_LOGCAT, 298 "full_gce_logcat") 299 elif not (log_path.endswith(".log") or 300 log_path.endswith("cuttlefish_config.json")): 301 continue 302 logs.append(log) 303 return logs 304 305 306def GetRemoteBuildInfoDict(avd_spec): 307 """Convert remote build infos to a dictionary for reporting. 308 309 Args: 310 avd_spec: An AvdSpec object containing the build infos. 311 312 Returns: 313 A dict containing the build infos. 314 """ 315 build_info_dict = { 316 key: val for key, val in avd_spec.remote_image.items() if val} 317 318 # kernel_target has a default value. If the user provides kernel_build_id 319 # or kernel_branch, then convert kernel build info. 320 if (avd_spec.kernel_build_info.get(constants.BUILD_ID) or 321 avd_spec.kernel_build_info.get(constants.BUILD_BRANCH)): 322 build_info_dict.update( 323 {"kernel_" + key: val 324 for key, val in avd_spec.kernel_build_info.items() if val} 325 ) 326 build_info_dict.update( 327 {"system_" + key: val 328 for key, val in avd_spec.system_build_info.items() if val} 329 ) 330 build_info_dict.update( 331 {"bootloader_" + key: val 332 for key, val in avd_spec.bootloader_build_info.items() if val} 333 ) 334 return build_info_dict 335