1#!/usr/bin/env python 2# 3# Copyright 2016 - The Android Open Source Project 4# 5# Licensed under the Apache License, Version 2.0 (the "License"); 6# you may not use this file except in compliance with the License. 7# You may obtain a copy of the License at 8# 9# http://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an "AS IS" BASIS, 13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14# See the License for the specific language governing permissions and 15# limitations under the License. 16 17"""A client that talks to Android Build APIs.""" 18 19import collections 20import io 21import json 22import logging 23import os 24import ssl 25import stat 26 27import apiclient 28 29from acloud import errors 30from acloud.internal import constants 31from acloud.internal.lib import base_cloud_client 32from acloud.internal.lib import utils 33 34 35logger = logging.getLogger(__name__) 36 37# The BuildInfo namedtuple data structure. 38# It will be the data structure returned by GetBuildInfo method. 39BuildInfo = collections.namedtuple("BuildInfo", [ 40 "branch", # The branch name string 41 "build_id", # The build id string 42 "build_target", # The build target string 43 "release_build_id"]) # The release build id string 44_DEFAULT_BRANCH = "aosp-master" 45 46 47class AndroidBuildClient(base_cloud_client.BaseCloudApiClient): 48 """Client that manages Android Build.""" 49 50 # API settings, used by BaseCloudApiClient. 51 API_NAME = "androidbuildinternal" 52 API_VERSION = "v2beta1" 53 SCOPE = "https://www.googleapis.com/auth/androidbuild.internal" 54 55 # other variables. 56 DEFAULT_RESOURCE_ID = "0" 57 # TODO(b/27269552): We should use "latest". 58 DEFAULT_ATTEMPT_ID = "0" 59 DEFAULT_CHUNK_SIZE = 20 * 1024 * 1024 60 NO_ACCESS_ERROR_PATTERN = "does not have storage.objects.create access" 61 # LKGB variables. 62 BUILD_STATUS_COMPLETE = "complete" 63 BUILD_TYPE_SUBMITTED = "submitted" 64 ONE_RESULT = 1 65 BUILD_SUCCESSFUL = True 66 LATEST = "latest" 67 # FETCH_CVD variables. 68 FETCHER_NAME = "fetch_cvd" 69 FETCHER_BUILD_TARGET = "aosp_cf_x86_64_phone-trunk_staging-userdebug" 70 FETCHER_BUILD_TARGET_ARM = "aosp_cf_arm64_only_phone-trunk_staging-userdebug" 71 # TODO(b/297085994): cvd fetch is migrating from AOSP to github artifacts, so 72 # temporary returning hardcoded values instead of LKGB 73 FETCHER_BUILD_ID = 11559438 74 FETCHER_BUILD_ID_ARM = 11559085 75 MAX_RETRY = 3 76 RETRY_SLEEP_SECS = 3 77 78 # Message constant 79 COPY_TO_MSG = ("build artifact (target: %s, build_id: %s, " 80 "artifact: %s, attempt_id: %s) to " 81 "google storage (bucket: %s, path: %s)") 82 # pylint: disable=invalid-name 83 def DownloadArtifact(self, 84 build_target, 85 build_id, 86 resource_id, 87 local_dest, 88 attempt_id=None): 89 """Get Android build attempt information. 90 91 Args: 92 build_target: Target name, e.g. "aosp_cf_x86_64_phone-userdebug" 93 build_id: Build id, a string, e.g. "2263051", "P2804227" 94 resource_id: Id of the resource, e.g "avd-system.tar.gz". 95 local_dest: A local path where the artifact should be stored. 96 e.g. "/tmp/avd-system.tar.gz" 97 attempt_id: String, attempt id, will default to DEFAULT_ATTEMPT_ID. 98 """ 99 attempt_id = attempt_id or self.DEFAULT_ATTEMPT_ID 100 api = self.service.buildartifact().get_media( 101 buildId=build_id, 102 target=build_target, 103 attemptId=attempt_id, 104 resourceId=resource_id) 105 logger.info("Downloading artifact: target: %s, build_id: %s, " 106 "resource_id: %s, dest: %s", build_target, build_id, 107 resource_id, local_dest) 108 try: 109 with io.FileIO(local_dest, mode="wb") as fh: 110 downloader = apiclient.http.MediaIoBaseDownload( 111 fh, api, chunksize=self.DEFAULT_CHUNK_SIZE) 112 done = False 113 while not done: 114 _, done = downloader.next_chunk() 115 logger.info("Downloaded artifact: %s", local_dest) 116 except (OSError, apiclient.errors.HttpError) as e: 117 logger.error("Downloading artifact failed: %s", str(e)) 118 raise errors.DriverError(str(e)) 119 120 def DownloadFetchcvd( 121 self, 122 local_dest, 123 fetch_cvd_version, 124 is_arm_version=False): 125 """Get fetch_cvd from Android Build. 126 127 Args: 128 local_dest: A local path where the artifact should be stored. 129 e.g. "/tmp/fetch_cvd" 130 fetch_cvd_version: String of fetch_cvd version. 131 is_arm_version: is ARM version fetch_cvd. 132 """ 133 if fetch_cvd_version == constants.LKGB: 134 fetch_cvd_version = self.GetFetcherVersion(is_arm_version) 135 fetch_cvd_build_target = ( 136 self.FETCHER_BUILD_TARGET_ARM if is_arm_version 137 else self.FETCHER_BUILD_TARGET) 138 try: 139 utils.RetryExceptionType( 140 exception_types=(ssl.SSLError, errors.DriverError), 141 max_retries=self.MAX_RETRY, 142 functor=self.DownloadArtifact, 143 sleep_multiplier=self.RETRY_SLEEP_SECS, 144 retry_backoff_factor=utils.DEFAULT_RETRY_BACKOFF_FACTOR, 145 build_target=fetch_cvd_build_target, 146 build_id=fetch_cvd_version, 147 resource_id=self.FETCHER_NAME, 148 local_dest=local_dest, 149 attempt_id=self.LATEST) 150 except Exception: 151 logger.debug("Download fetch_cvd with build id: %s", 152 constants.FETCH_CVD_SECOND_VERSION) 153 utils.RetryExceptionType( 154 exception_types=(ssl.SSLError, errors.DriverError), 155 max_retries=self.MAX_RETRY, 156 functor=self.DownloadArtifact, 157 sleep_multiplier=self.RETRY_SLEEP_SECS, 158 retry_backoff_factor=utils.DEFAULT_RETRY_BACKOFF_FACTOR, 159 build_target=fetch_cvd_build_target, 160 build_id=constants.FETCH_CVD_SECOND_VERSION, 161 resource_id=self.FETCHER_NAME, 162 local_dest=local_dest, 163 attempt_id=self.LATEST) 164 fetch_cvd_stat = os.stat(local_dest) 165 os.chmod(local_dest, fetch_cvd_stat.st_mode | stat.S_IEXEC) 166 167 @staticmethod 168 def ProcessBuild(build_info, ignore_artifact=False): 169 """Create a Cuttlefish fetch_cvd build string. 170 171 Args: 172 build_info: The dictionary that contains build information. 173 ignore_artifact: Avoid adding artifact part to fetch_cvd build string 174 175 Returns: 176 A string, used in the fetch_cvd cmd or None if all args are None. 177 """ 178 build_id = build_info.get(constants.BUILD_ID) 179 build_target = build_info.get(constants.BUILD_TARGET) 180 branch = build_info.get(constants.BUILD_BRANCH) 181 artifact = build_info.get(constants.BUILD_ARTIFACT) 182 183 result = build_id or branch 184 if build_target is not None: 185 result = result or _DEFAULT_BRANCH 186 result += "/" + build_target 187 188 if not ignore_artifact and artifact: 189 result += "{" + artifact + "}" 190 191 return result 192 193 def GetFetchBuildArgs(self, default_build_info, system_build_info, 194 kernel_build_info, boot_build_info, 195 bootloader_build_info, android_efi_loader_build_info, 196 ota_build_info, host_package_build_info): 197 """Get args from build information for fetch_cvd. 198 199 Each build_info is a dictionary that contains 3 items, for example, 200 { 201 constants.BUILD_ID: "2263051", 202 constants.BUILD_TARGET: "aosp_cf_x86_64_phone-userdebug", 203 constants.BUILD_BRANCH: "aosp-master", 204 } 205 206 Args: 207 default_build_info: The build that provides full cuttlefish images. 208 system_build_info: The build that provides the system image. 209 kernel_build_info: The build that provides the kernel. 210 boot_build_info: The build that provides the boot image. This 211 dictionary may contain an additional key 212 constants.BUILD_ARTIFACT which is mapped to the 213 boot image name. 214 bootloader_build_info: The build that provides the bootloader. 215 android_efi_loader_build_info: The build that provides the Android EFI loader. 216 ota_build_info: The build that provides the OTA tools. 217 host_package_build_info: The build that provides the host package. 218 219 Returns: 220 List of string args for fetch_cvd. 221 """ 222 fetch_cvd_args = [] 223 224 default_build = self.ProcessBuild(default_build_info) 225 if default_build: 226 fetch_cvd_args.append(f"-default_build={default_build}") 227 system_build = self.ProcessBuild(system_build_info) 228 if system_build: 229 fetch_cvd_args.append(f"-system_build={system_build}") 230 bootloader_build = self.ProcessBuild(bootloader_build_info) 231 if bootloader_build: 232 fetch_cvd_args.append(f"-bootloader_build={bootloader_build}") 233 android_efi_loader_build = self.ProcessBuild(android_efi_loader_build_info) 234 if android_efi_loader_build: 235 fetch_cvd_args.append(f"-android_efi_loader_build {android_efi_loader_build}") 236 kernel_build = self.GetKernelBuild(kernel_build_info) 237 if kernel_build: 238 fetch_cvd_args.append(f"-kernel_build={kernel_build}") 239 boot_build = self.ProcessBuild(boot_build_info, ignore_artifact=True) 240 if boot_build: 241 fetch_cvd_args.append(f"-boot_build={boot_build}") 242 boot_artifact = boot_build_info.get(constants.BUILD_ARTIFACT) 243 if boot_artifact: 244 fetch_cvd_args.append(f"-boot_artifact={boot_artifact}") 245 ota_build = self.ProcessBuild(ota_build_info) 246 if ota_build: 247 fetch_cvd_args.append(f"-otatools_build={ota_build}") 248 host_package_build = self.ProcessBuild(host_package_build_info) 249 if host_package_build: 250 fetch_cvd_args.append(f"-host_package_build={host_package_build}") 251 252 return fetch_cvd_args 253 254 def GetFetcherVersion(self, is_arm_version=False): 255 """Get fetch_cvd build id from LKGB. 256 257 Returns: 258 The build id of fetch_cvd. 259 """ 260 # TODO(b/297085994): currently returning hardcoded values 261 # For more information, please check the BUILD_ID constant definition 262 # comment section 263 return self.FETCHER_BUILD_ID_ARM if is_arm_version else self.FETCHER_BUILD_ID 264 265 @staticmethod 266 # pylint: disable=broad-except 267 def GetFetchCertArg(certification_file): 268 """Get cert arg from certification file for fetch_cvd. 269 270 Parse the certification file to get access token of the latest 271 credential data and pass it to fetch_cvd command. 272 Example of certification file: 273 { 274 "data": [ 275 { 276 "credential": { 277 "_class": "OAuth2Credentials", 278 "_module": "oauth2client.client", 279 "access_token": "token_strings", 280 "client_id": "179485041932", 281 } 282 }] 283 } 284 285 286 Args: 287 certification_file: String of certification file path. 288 289 Returns: 290 String of certificate arg for fetch_cvd. If there is no 291 certification file, return empty string for aosp branch. 292 """ 293 cert_arg = "" 294 try: 295 with open(certification_file) as cert_file: 296 auth_token = json.load(cert_file).get("data")[-1].get( 297 "credential").get("access_token") 298 if auth_token: 299 cert_arg = f"-credential_source={auth_token}" 300 except Exception as e: 301 utils.PrintColorString( 302 f"Fail to open the certification file " 303 f"({certification_file}): {e}", 304 utils.TextColors.WARNING) 305 return cert_arg 306 307 def GetKernelBuild(self, kernel_build_info): 308 """Get kernel build args for fetch_cvd. 309 310 Args: 311 kernel_build_info: The dictionary that contains build information. 312 313 Returns: 314 String of kernel build args for fetch_cvd. 315 If no kernel build then return None. 316 """ 317 # kernel_target have default value "kernel". If user provide kernel_build_id 318 # or kernel_branch, then start to process kernel image. 319 if (kernel_build_info.get(constants.BUILD_ID) or 320 kernel_build_info.get(constants.BUILD_BRANCH)): 321 return self.ProcessBuild(kernel_build_info) 322 return None 323 324 def CopyTo(self, 325 build_target, 326 build_id, 327 artifact_name, 328 destination_bucket, 329 destination_path, 330 attempt_id=None): 331 """Copy an Android Build artifact to a storage bucket. 332 333 Args: 334 build_target: Target name, e.g. "aosp_cf_x86_64_phone-userdebug" 335 build_id: Build id, a string, e.g. "2263051", "P2804227" 336 artifact_name: Name of the artifact, e.g "avd-system.tar.gz". 337 destination_bucket: String, a google storage bucket name. 338 destination_path: String, "path/inside/bucket" 339 attempt_id: String, attempt id, will default to DEFAULT_ATTEMPT_ID. 340 """ 341 attempt_id = attempt_id or self.DEFAULT_ATTEMPT_ID 342 copy_msg = "Copying %s" % self.COPY_TO_MSG 343 logger.info(copy_msg, build_target, build_id, artifact_name, 344 attempt_id, destination_bucket, destination_path) 345 api = self.service.buildartifact().copyTo( 346 buildId=build_id, 347 target=build_target, 348 attemptId=attempt_id, 349 artifactName=artifact_name, 350 destinationBucket=destination_bucket, 351 destinationPath=destination_path) 352 try: 353 self.Execute(api) 354 finish_msg = "Finished copying %s" % self.COPY_TO_MSG 355 logger.info(finish_msg, build_target, build_id, artifact_name, 356 attempt_id, destination_bucket, destination_path) 357 except errors.HttpError as e: 358 if e.code == 503: 359 if self.NO_ACCESS_ERROR_PATTERN in str(e): 360 error_msg = "Please grant android build team's service account " 361 error_msg += "write access to bucket %s. Original error: %s" 362 error_msg %= (destination_bucket, str(e)) 363 raise errors.HttpError(e.code, message=error_msg) 364 raise 365 366 def GetBranch(self, build_target, build_id): 367 """Derives branch name. 368 369 Args: 370 build_target: Target name, e.g. "aosp_cf_x86_64_phone-userdebug" 371 build_id: Build ID, a string, e.g. "2263051", "P2804227" 372 373 Returns: 374 A string, the name of the branch 375 """ 376 api = self.service.build().get(buildId=build_id, target=build_target) 377 build = self.Execute(api) 378 return build.get("branch", "") 379 380 def GetLKGB(self, build_target, build_branch): 381 """Get latest successful build id. 382 383 From branch and target, we can use api to query latest successful build id. 384 e.g. {u'nextPageToken':..., u'builds': [{u'completionTimestamp':u'1534157869286', 385 ... u'buildId': u'4949805', u'machineName'...}]} 386 387 Args: 388 build_target: String, target name, e.g. "aosp_cf_x86_64_phone-userdebug" 389 build_branch: String, git branch name, e.g. "aosp-master" 390 391 Returns: 392 A string, string of build id number. 393 394 Raises: 395 errors.CreateError: Can't get build id. 396 """ 397 api = self.service.build().list( 398 branch=build_branch, 399 target=build_target, 400 buildAttemptStatus=self.BUILD_STATUS_COMPLETE, 401 buildType=self.BUILD_TYPE_SUBMITTED, 402 maxResults=self.ONE_RESULT, 403 successful=self.BUILD_SUCCESSFUL) 404 build = self.Execute(api) 405 logger.info("GetLKGB build API response: %s", build) 406 if build: 407 return str(build.get("builds")[0].get("buildId")) 408 raise errors.GetBuildIDError( 409 "No available good builds for branch: %s target: %s" 410 % (build_branch, build_target) 411 ) 412 413 def GetBuildInfo(self, build_target, build_id, branch): 414 """Get build info namedtuple. 415 416 Args: 417 build_target: Target name. 418 build_id: Build id, a string or None, e.g. "2263051", "P2804227" 419 If None or latest, the last green build id will be 420 returned. 421 branch: Branch name, a string or None, e.g. git_master. If None, the 422 returned branch will be searched by given build_id. 423 424 Returns: 425 A build info namedtuple with keys build_target, build_id, branch and 426 gcs_bucket_build_id 427 """ 428 if build_id and build_id != self.LATEST: 429 # Get build from build_id and build_target 430 api = self.service.build().get(buildId=build_id, 431 target=build_target) 432 build = self.Execute(api) or {} 433 elif branch: 434 # Get last green build in the branch 435 api = self.service.build().list( 436 branch=branch, 437 target=build_target, 438 successful=True, 439 maxResults=1, 440 buildType="submitted") 441 builds = self.Execute(api).get("builds", []) 442 build = builds[0] if builds else {} 443 else: 444 build = {} 445 446 build_id = build.get("buildId") 447 build_target = build_target if build_id else None 448 return BuildInfo(build.get("branch"), build_id, build_target, 449 build.get("releaseCandidateName")) 450