1#!/usr/bin/env python 2# 3# Copyright 2016 - The Android Open Source Project 4# 5# Licensed under the Apache License, Version 2.0 (the "License"); 6# you may not use this file except in compliance with the License. 7# You may obtain a copy of the License at 8# 9# http://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an "AS IS" BASIS, 13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14# See the License for the specific language governing permissions and 15# limitations under the License. 16 17"""A client that talks to Android Build APIs.""" 18 19import collections 20import io 21import json 22import logging 23import os 24import ssl 25import stat 26 27import apiclient 28 29from acloud import errors 30from acloud.internal import constants 31from acloud.internal.lib import base_cloud_client 32from acloud.internal.lib import utils 33 34 35logger = logging.getLogger(__name__) 36 37# The BuildInfo namedtuple data structure. 38# It will be the data structure returned by GetBuildInfo method. 39BuildInfo = collections.namedtuple("BuildInfo", [ 40 "branch", # The branch name string 41 "build_id", # The build id string 42 "build_target", # The build target string 43 "release_build_id"]) # The release build id string 44_DEFAULT_BRANCH = "aosp-main" 45 46 47class AndroidBuildClient(base_cloud_client.BaseCloudApiClient): 48 """Client that manages Android Build.""" 49 50 # API settings, used by BaseCloudApiClient. 51 API_NAME = "androidbuildinternal" 52 API_VERSION = "v2beta1" 53 SCOPE = "https://www.googleapis.com/auth/androidbuild.internal" 54 55 # other variables. 56 DEFAULT_RESOURCE_ID = "0" 57 # TODO(b/27269552): We should use "latest". 58 DEFAULT_ATTEMPT_ID = "0" 59 DEFAULT_CHUNK_SIZE = 20 * 1024 * 1024 60 NO_ACCESS_ERROR_PATTERN = "does not have storage.objects.create access" 61 # LKGB variables. 62 BUILD_STATUS_COMPLETE = "complete" 63 BUILD_TYPE_SUBMITTED = "submitted" 64 ONE_RESULT = 1 65 BUILD_SUCCESSFUL = True 66 LATEST = "latest" 67 MAX_RETRY = 3 68 RETRY_SLEEP_SECS = 3 69 70 # Message constant 71 COPY_TO_MSG = ("build artifact (target: %s, build_id: %s, " 72 "artifact: %s, attempt_id: %s) to " 73 "google storage (bucket: %s, path: %s)") 74 # pylint: disable=invalid-name 75 def DownloadArtifact(self, 76 build_target, 77 build_id, 78 resource_id, 79 local_dest, 80 attempt_id=None): 81 """Get Android build attempt information. 82 83 Args: 84 build_target: Target name, e.g. "aosp_cf_x86_64_phone-trunk_staging-userdebug" 85 build_id: Build id, a string, e.g. "2263051", "P2804227" 86 resource_id: Id of the resource, e.g "avd-system.tar.gz". 87 local_dest: A local path where the artifact should be stored. 88 e.g. "/tmp/avd-system.tar.gz" 89 attempt_id: String, attempt id, will default to DEFAULT_ATTEMPT_ID. 90 """ 91 attempt_id = attempt_id or self.DEFAULT_ATTEMPT_ID 92 api = self.service.buildartifact().get_media( 93 buildId=build_id, 94 target=build_target, 95 attemptId=attempt_id, 96 resourceId=resource_id) 97 logger.info("Downloading artifact: target: %s, build_id: %s, " 98 "resource_id: %s, dest: %s", build_target, build_id, 99 resource_id, local_dest) 100 try: 101 with io.FileIO(local_dest, mode="wb") as fh: 102 downloader = apiclient.http.MediaIoBaseDownload( 103 fh, api, chunksize=self.DEFAULT_CHUNK_SIZE) 104 done = False 105 while not done: 106 _, done = downloader.next_chunk() 107 logger.info("Downloaded artifact: %s", local_dest) 108 except (OSError, apiclient.errors.HttpError) as e: 109 logger.error("Downloading artifact failed: %s", str(e)) 110 raise errors.DriverError(str(e)) 111 112 @staticmethod 113 def ProcessBuild(build_info, ignore_artifact=False): 114 """Create a Cuttlefish fetch_cvd build string. 115 116 Args: 117 build_info: The dictionary that contains build information. 118 ignore_artifact: Avoid adding artifact part to fetch_cvd build string 119 120 Returns: 121 A string, used in the fetch_cvd cmd or None if all args are None. 122 """ 123 build_id = build_info.get(constants.BUILD_ID) 124 build_target = build_info.get(constants.BUILD_TARGET) 125 branch = build_info.get(constants.BUILD_BRANCH) 126 artifact = build_info.get(constants.BUILD_ARTIFACT) 127 128 result = build_id or branch 129 if build_target is not None: 130 result = result or _DEFAULT_BRANCH 131 result += "/" + build_target 132 133 if not ignore_artifact and artifact: 134 result += "{" + artifact + "}" 135 136 return result 137 138 def GetFetchBuildArgs(self, default_build_info, system_build_info, 139 kernel_build_info, boot_build_info, 140 bootloader_build_info, android_efi_loader_build_info, 141 ota_build_info, host_package_build_info): 142 """Get args from build information for fetch_cvd. 143 144 Each build_info is a dictionary that contains 3 items, for example, 145 { 146 constants.BUILD_ID: "2263051", 147 constants.BUILD_TARGET: "aosp_cf_x86_64_phone-trunk_staging-userdebug", 148 constants.BUILD_BRANCH: "aosp-main", 149 } 150 151 Args: 152 default_build_info: The build that provides full cuttlefish images. 153 system_build_info: The build that provides the system image. 154 kernel_build_info: The build that provides the kernel. 155 boot_build_info: The build that provides the boot image. This 156 dictionary may contain an additional key 157 constants.BUILD_ARTIFACT which is mapped to the 158 boot image name. 159 bootloader_build_info: The build that provides the bootloader. 160 android_efi_loader_build_info: The build that provides the Android EFI loader. 161 ota_build_info: The build that provides the OTA tools. 162 host_package_build_info: The build that provides the host package. 163 164 Returns: 165 List of string args for fetch_cvd. 166 """ 167 fetch_cvd_args = [] 168 169 default_build = self.ProcessBuild(default_build_info) 170 if default_build: 171 fetch_cvd_args.append(f"-default_build={default_build}") 172 system_build = self.ProcessBuild(system_build_info) 173 if system_build: 174 fetch_cvd_args.append(f"-system_build={system_build}") 175 bootloader_build = self.ProcessBuild(bootloader_build_info) 176 if bootloader_build: 177 fetch_cvd_args.append(f"-bootloader_build={bootloader_build}") 178 android_efi_loader_build = self.ProcessBuild(android_efi_loader_build_info) 179 if android_efi_loader_build: 180 fetch_cvd_args.append(f"-android_efi_loader_build {android_efi_loader_build}") 181 kernel_build = self.GetKernelBuild(kernel_build_info) 182 if kernel_build: 183 fetch_cvd_args.append(f"-kernel_build={kernel_build}") 184 boot_build = self.ProcessBuild(boot_build_info, ignore_artifact=True) 185 if boot_build: 186 fetch_cvd_args.append(f"-boot_build={boot_build}") 187 boot_artifact = boot_build_info.get(constants.BUILD_ARTIFACT) 188 if boot_artifact: 189 fetch_cvd_args.append(f"-boot_artifact={boot_artifact}") 190 ota_build = self.ProcessBuild(ota_build_info) 191 if ota_build: 192 fetch_cvd_args.append(f"-otatools_build={ota_build}") 193 host_package_build = self.ProcessBuild(host_package_build_info) 194 if host_package_build: 195 fetch_cvd_args.append(f"-host_package_build={host_package_build}") 196 197 return fetch_cvd_args 198 199 def GetFetcherVersion(self, is_arm_version=False): 200 """Get fetch_cvd build id from LKGB. 201 202 Returns: 203 The build id of fetch_cvd. 204 """ 205 # TODO(b/297085994): currently returning hardcoded values 206 # For more information, please check the BUILD_ID constant definition 207 # comment section 208 return self.FETCHER_BUILD_ID_ARM if is_arm_version else self.FETCHER_BUILD_ID 209 210 @staticmethod 211 # pylint: disable=broad-except 212 def GetFetchCertArg(certification_file): 213 """Get cert arg from certification file for fetch_cvd. 214 215 Parse the certification file to get access token of the latest 216 credential data and pass it to fetch_cvd command. 217 Example of certification file: 218 { 219 "data": [ 220 { 221 "credential": { 222 "_class": "OAuth2Credentials", 223 "_module": "oauth2client.client", 224 "access_token": "token_strings", 225 "client_id": "179485041932", 226 } 227 }] 228 } 229 230 231 Args: 232 certification_file: String of certification file path. 233 234 Returns: 235 String of certificate arg for fetch_cvd. If there is no 236 certification file, return empty string for aosp branch. 237 """ 238 cert_arg = "" 239 try: 240 with open(certification_file) as cert_file: 241 auth_token = json.load(cert_file).get("data")[-1].get( 242 "credential").get("access_token") 243 if auth_token: 244 cert_arg = f"-credential_source={auth_token}" 245 except Exception as e: 246 utils.PrintColorString( 247 f"Fail to open the certification file " 248 f"({certification_file}): {e}", 249 utils.TextColors.WARNING) 250 return cert_arg 251 252 def GetKernelBuild(self, kernel_build_info): 253 """Get kernel build args for fetch_cvd. 254 255 Args: 256 kernel_build_info: The dictionary that contains build information. 257 258 Returns: 259 String of kernel build args for fetch_cvd. 260 If no kernel build then return None. 261 """ 262 # kernel_target have default value "kernel". If user provide kernel_build_id 263 # or kernel_branch, then start to process kernel image. 264 if (kernel_build_info.get(constants.BUILD_ID) or 265 kernel_build_info.get(constants.BUILD_BRANCH)): 266 return self.ProcessBuild(kernel_build_info) 267 return None 268 269 def CopyTo(self, 270 build_target, 271 build_id, 272 artifact_name, 273 destination_bucket, 274 destination_path, 275 attempt_id=None): 276 """Copy an Android Build artifact to a storage bucket. 277 278 Args: 279 build_target: Target name, e.g. "aosp_cf_x86_64_phone-trunk_staging-userdebug" 280 build_id: Build id, a string, e.g. "2263051", "P2804227" 281 artifact_name: Name of the artifact, e.g "avd-system.tar.gz". 282 destination_bucket: String, a google storage bucket name. 283 destination_path: String, "path/inside/bucket" 284 attempt_id: String, attempt id, will default to DEFAULT_ATTEMPT_ID. 285 """ 286 attempt_id = attempt_id or self.DEFAULT_ATTEMPT_ID 287 copy_msg = "Copying %s" % self.COPY_TO_MSG 288 logger.info(copy_msg, build_target, build_id, artifact_name, 289 attempt_id, destination_bucket, destination_path) 290 api = self.service.buildartifact().copyTo( 291 buildId=build_id, 292 target=build_target, 293 attemptId=attempt_id, 294 artifactName=artifact_name, 295 destinationBucket=destination_bucket, 296 destinationPath=destination_path) 297 try: 298 self.Execute(api) 299 finish_msg = "Finished copying %s" % self.COPY_TO_MSG 300 logger.info(finish_msg, build_target, build_id, artifact_name, 301 attempt_id, destination_bucket, destination_path) 302 except errors.HttpError as e: 303 if e.code == 503: 304 if self.NO_ACCESS_ERROR_PATTERN in str(e): 305 error_msg = "Please grant android build team's service account " 306 error_msg += "write access to bucket %s. Original error: %s" 307 error_msg %= (destination_bucket, str(e)) 308 raise errors.HttpError(e.code, message=error_msg) 309 raise 310 311 def GetBranch(self, build_target, build_id): 312 """Derives branch name. 313 314 Args: 315 build_target: Target name, e.g. "aosp_cf_x86_64_phone-trunk_staging-userdebug" 316 build_id: Build ID, a string, e.g. "2263051", "P2804227" 317 318 Returns: 319 A string, the name of the branch 320 """ 321 api = self.service.build().get(buildId=build_id, target=build_target) 322 build = self.Execute(api) 323 return build.get("branch", "") 324 325 def GetLKGB(self, build_target, build_branch): 326 """Get latest successful build id. 327 328 From branch and target, we can use api to query latest successful build id. 329 e.g. {u'nextPageToken':..., u'builds': [{u'completionTimestamp':u'1534157869286', 330 ... u'buildId': u'4949805', u'machineName'...}]} 331 332 Args: 333 build_target: String, target name, e.g. "aosp_cf_x86_64_phone-trunk_staging-userdebug" 334 build_branch: String, git branch name, e.g. "aosp-main" 335 336 Returns: 337 A string, string of build id number. 338 339 Raises: 340 errors.CreateError: Can't get build id. 341 """ 342 api = self.service.build().list( 343 branch=build_branch, 344 target=build_target, 345 buildAttemptStatus=self.BUILD_STATUS_COMPLETE, 346 buildType=self.BUILD_TYPE_SUBMITTED, 347 maxResults=self.ONE_RESULT, 348 successful=self.BUILD_SUCCESSFUL) 349 build = self.Execute(api) 350 logger.info("GetLKGB build API response: %s", build) 351 if build: 352 return str(build.get("builds")[0].get("buildId")) 353 raise errors.GetBuildIDError( 354 "No available good builds for branch: %s target: %s" 355 % (build_branch, build_target) 356 ) 357 358 def GetBuildInfo(self, build_target, build_id, branch): 359 """Get build info namedtuple. 360 361 Args: 362 build_target: Target name. 363 build_id: Build id, a string or None, e.g. "2263051", "P2804227" 364 If None or latest, the last green build id will be 365 returned. 366 branch: Branch name, a string or None, e.g. git_master. If None, the 367 returned branch will be searched by given build_id. 368 369 Returns: 370 A build info namedtuple with keys build_target, build_id, branch and 371 gcs_bucket_build_id 372 """ 373 if build_id and build_id != self.LATEST: 374 # Get build from build_id and build_target 375 api = self.service.build().get(buildId=build_id, 376 target=build_target) 377 build = self.Execute(api) or {} 378 elif branch: 379 # Get last green build in the branch 380 api = self.service.build().list( 381 branch=branch, 382 target=build_target, 383 successful=True, 384 maxResults=1, 385 buildType="submitted") 386 builds = self.Execute(api).get("builds", []) 387 build = builds[0] if builds else {} 388 else: 389 build = {} 390 391 build_id = build.get("buildId") 392 build_target = build_target if build_id else None 393 return BuildInfo(build.get("branch"), build_id, build_target, 394 build.get("releaseCandidateName")) 395