1#!/usr/bin/env python 2# 3# Copyright 2016 - The Android Open Source Project 4# 5# Licensed under the Apache License, Version 2.0 (the "License"); 6# you may not use this file except in compliance with the License. 7# You may obtain a copy of the License at 8# 9# http://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an "AS IS" BASIS, 13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14# See the License for the specific language governing permissions and 15# limitations under the License. 16 17"""A client that talks to Android Build APIs.""" 18 19import collections 20import io 21import json 22import logging 23import os 24import ssl 25import stat 26 27import apiclient 28 29from acloud import errors 30from acloud.internal import constants 31from acloud.internal.lib import base_cloud_client 32from acloud.internal.lib import utils 33 34 35logger = logging.getLogger(__name__) 36 37# The BuildInfo namedtuple data structure. 38# It will be the data structure returned by GetBuildInfo method. 39BuildInfo = collections.namedtuple("BuildInfo", [ 40 "branch", # The branch name string 41 "build_id", # The build id string 42 "build_target", # The build target string 43 "release_build_id"]) # The release build id string 44_DEFAULT_BRANCH = "aosp-master" 45 46 47class AndroidBuildClient(base_cloud_client.BaseCloudApiClient): 48 """Client that manages Android Build.""" 49 50 # API settings, used by BaseCloudApiClient. 51 API_NAME = "androidbuildinternal" 52 API_VERSION = "v2beta1" 53 SCOPE = "https://www.googleapis.com/auth/androidbuild.internal" 54 55 # other variables. 56 DEFAULT_RESOURCE_ID = "0" 57 # TODO(b/27269552): We should use "latest". 58 DEFAULT_ATTEMPT_ID = "0" 59 DEFAULT_CHUNK_SIZE = 20 * 1024 * 1024 60 NO_ACCESS_ERROR_PATTERN = "does not have storage.objects.create access" 61 # LKGB variables. 62 BUILD_STATUS_COMPLETE = "complete" 63 BUILD_TYPE_SUBMITTED = "submitted" 64 ONE_RESULT = 1 65 BUILD_SUCCESSFUL = True 66 LATEST = "latest" 67 # FETCH_CVD variables. 68 FETCHER_NAME = "fetch_cvd" 69 FETCHER_BRANCH = "aosp-master" 70 FETCHER_BUILD_TARGET = "aosp_cf_x86_64_phone-userdebug" 71 FETCHER_ARM_VERSION_BUILD_TARGET = "aosp_cf_arm64_phone-userdebug" 72 MAX_RETRY = 3 73 RETRY_SLEEP_SECS = 3 74 75 # Message constant 76 COPY_TO_MSG = ("build artifact (target: %s, build_id: %s, " 77 "artifact: %s, attempt_id: %s) to " 78 "google storage (bucket: %s, path: %s)") 79 # pylint: disable=invalid-name 80 def DownloadArtifact(self, 81 build_target, 82 build_id, 83 resource_id, 84 local_dest, 85 attempt_id=None): 86 """Get Android build attempt information. 87 88 Args: 89 build_target: Target name, e.g. "aosp_cf_x86_64_phone-userdebug" 90 build_id: Build id, a string, e.g. "2263051", "P2804227" 91 resource_id: Id of the resource, e.g "avd-system.tar.gz". 92 local_dest: A local path where the artifact should be stored. 93 e.g. "/tmp/avd-system.tar.gz" 94 attempt_id: String, attempt id, will default to DEFAULT_ATTEMPT_ID. 95 """ 96 attempt_id = attempt_id or self.DEFAULT_ATTEMPT_ID 97 api = self.service.buildartifact().get_media( 98 buildId=build_id, 99 target=build_target, 100 attemptId=attempt_id, 101 resourceId=resource_id) 102 logger.info("Downloading artifact: target: %s, build_id: %s, " 103 "resource_id: %s, dest: %s", build_target, build_id, 104 resource_id, local_dest) 105 try: 106 with io.FileIO(local_dest, mode="wb") as fh: 107 downloader = apiclient.http.MediaIoBaseDownload( 108 fh, api, chunksize=self.DEFAULT_CHUNK_SIZE) 109 done = False 110 while not done: 111 _, done = downloader.next_chunk() 112 logger.info("Downloaded artifact: %s", local_dest) 113 except (OSError, apiclient.errors.HttpError) as e: 114 logger.error("Downloading artifact failed: %s", str(e)) 115 raise errors.DriverError(str(e)) 116 117 def DownloadFetchcvd( 118 self, 119 local_dest, 120 fetch_cvd_version, 121 is_arm_version=False): 122 """Get fetch_cvd from Android Build. 123 124 Args: 125 local_dest: A local path where the artifact should be stored. 126 e.g. "/tmp/fetch_cvd" 127 fetch_cvd_version: String of fetch_cvd version. 128 is_arm_version: is ARM version fetch_cvd. 129 """ 130 if fetch_cvd_version == constants.LKGB: 131 fetch_cvd_version = self.GetFetcherVersion() 132 utils.RetryExceptionType( 133 exception_types=(ssl.SSLError, errors.DriverError), 134 max_retries=self.MAX_RETRY, 135 functor=self.DownloadArtifact, 136 sleep_multiplier=self.RETRY_SLEEP_SECS, 137 retry_backoff_factor=utils.DEFAULT_RETRY_BACKOFF_FACTOR, 138 build_target=(self.FETCHER_ARM_VERSION_BUILD_TARGET 139 if is_arm_version else self.FETCHER_BUILD_TARGET), 140 build_id=fetch_cvd_version, 141 resource_id=self.FETCHER_NAME, 142 local_dest=local_dest, 143 attempt_id=self.LATEST) 144 fetch_cvd_stat = os.stat(local_dest) 145 os.chmod(local_dest, fetch_cvd_stat.st_mode | stat.S_IEXEC) 146 147 @staticmethod 148 def ProcessBuild(build_info): 149 """Create a Cuttlefish fetch_cvd build string. 150 151 Args: 152 build_info: The dictionary that contains build information. 153 154 Returns: 155 A string, used in the fetch_cvd cmd or None if all args are None. 156 """ 157 build_id = build_info.get(constants.BUILD_ID) 158 build_target = build_info.get(constants.BUILD_TARGET) 159 branch = build_info.get(constants.BUILD_BRANCH) 160 if not build_target: 161 return build_id or branch 162 163 if build_target and not branch: 164 branch = _DEFAULT_BRANCH 165 return (build_id or branch) + "/" + build_target 166 167 def GetFetchBuildArgs(self, default_build_info, system_build_info, 168 kernel_build_info, boot_build_info, 169 bootloader_build_info, ota_build_info): 170 """Get args from build information for fetch_cvd. 171 172 Each build_info is a dictionary that contains 3 items, for example, 173 { 174 constants.BUILD_ID: "2263051", 175 constants.BUILD_TARGET: "aosp_cf_x86_64_phone-userdebug", 176 constants.BUILD_BRANCH: "aosp-master", 177 } 178 179 Args: 180 default_build_info: The build that provides full cuttlefish images. 181 system_build_info: The build that provides the system image. 182 kernel_build_info: The build that provides the kernel. 183 boot_build_info: The build that provides the boot image. This 184 dictionary may contain an additional key 185 constants.BUILD_ARTIFACT which is mapped to the 186 boot image name. 187 bootloader_build_info: The build that provides the bootloader. 188 ota_build_info: The build that provides the OTA tools. 189 190 Returns: 191 List of string args for fetch_cvd. 192 """ 193 fetch_cvd_args = [] 194 195 default_build = self.ProcessBuild(default_build_info) 196 if default_build: 197 fetch_cvd_args.append(f"-default_build={default_build}") 198 system_build = self.ProcessBuild(system_build_info) 199 if system_build: 200 fetch_cvd_args.append(f"-system_build={system_build}") 201 bootloader_build = self.ProcessBuild(bootloader_build_info) 202 if bootloader_build: 203 fetch_cvd_args.append(f"-bootloader_build={bootloader_build}") 204 kernel_build = self.GetKernelBuild(kernel_build_info) 205 if kernel_build: 206 fetch_cvd_args.append(f"-kernel_build={kernel_build}") 207 boot_build = self.ProcessBuild(boot_build_info) 208 if boot_build: 209 fetch_cvd_args.append(f"-boot_build={boot_build}") 210 boot_artifact = boot_build_info.get(constants.BUILD_ARTIFACT) 211 if boot_artifact: 212 fetch_cvd_args.append(f"-boot_artifact={boot_artifact}") 213 ota_build = self.ProcessBuild(ota_build_info) 214 if ota_build: 215 fetch_cvd_args.append(f"-otatools_build={ota_build}") 216 217 return fetch_cvd_args 218 219 def GetFetcherVersion(self): 220 """Get fetch_cvd build id from LKGB. 221 222 Returns: 223 The build id of fetch_cvd. 224 """ 225 return self.GetLKGB(self.FETCHER_BUILD_TARGET, self.FETCHER_BRANCH) 226 227 @staticmethod 228 # pylint: disable=broad-except 229 def GetFetchCertArg(certification_file): 230 """Get cert arg from certification file for fetch_cvd. 231 232 Parse the certification file to get access token of the latest 233 credential data and pass it to fetch_cvd command. 234 Example of certification file: 235 { 236 "data": [ 237 { 238 "credential": { 239 "_class": "OAuth2Credentials", 240 "_module": "oauth2client.client", 241 "access_token": "token_strings", 242 "client_id": "179485041932", 243 } 244 }] 245 } 246 247 248 Args: 249 certification_file: String of certification file path. 250 251 Returns: 252 String of certificate arg for fetch_cvd. If there is no 253 certification file, return empty string for aosp branch. 254 """ 255 cert_arg = "" 256 try: 257 with open(certification_file) as cert_file: 258 auth_token = json.load(cert_file).get("data")[-1].get( 259 "credential").get("access_token") 260 if auth_token: 261 cert_arg = f"-credential_source={auth_token}" 262 except Exception as e: 263 utils.PrintColorString( 264 f"Fail to open the certification file " 265 f"({certification_file}): {e}", 266 utils.TextColors.WARNING) 267 return cert_arg 268 269 def GetKernelBuild(self, kernel_build_info): 270 """Get kernel build args for fetch_cvd. 271 272 Args: 273 kernel_build_info: The dictionary that contains build information. 274 275 Returns: 276 String of kernel build args for fetch_cvd. 277 If no kernel build then return None. 278 """ 279 # kernel_target have default value "kernel". If user provide kernel_build_id 280 # or kernel_branch, then start to process kernel image. 281 if (kernel_build_info.get(constants.BUILD_ID) or 282 kernel_build_info.get(constants.BUILD_BRANCH)): 283 return self.ProcessBuild(kernel_build_info) 284 return None 285 286 def CopyTo(self, 287 build_target, 288 build_id, 289 artifact_name, 290 destination_bucket, 291 destination_path, 292 attempt_id=None): 293 """Copy an Android Build artifact to a storage bucket. 294 295 Args: 296 build_target: Target name, e.g. "aosp_cf_x86_64_phone-userdebug" 297 build_id: Build id, a string, e.g. "2263051", "P2804227" 298 artifact_name: Name of the artifact, e.g "avd-system.tar.gz". 299 destination_bucket: String, a google storage bucket name. 300 destination_path: String, "path/inside/bucket" 301 attempt_id: String, attempt id, will default to DEFAULT_ATTEMPT_ID. 302 """ 303 attempt_id = attempt_id or self.DEFAULT_ATTEMPT_ID 304 copy_msg = "Copying %s" % self.COPY_TO_MSG 305 logger.info(copy_msg, build_target, build_id, artifact_name, 306 attempt_id, destination_bucket, destination_path) 307 api = self.service.buildartifact().copyTo( 308 buildId=build_id, 309 target=build_target, 310 attemptId=attempt_id, 311 artifactName=artifact_name, 312 destinationBucket=destination_bucket, 313 destinationPath=destination_path) 314 try: 315 self.Execute(api) 316 finish_msg = "Finished copying %s" % self.COPY_TO_MSG 317 logger.info(finish_msg, build_target, build_id, artifact_name, 318 attempt_id, destination_bucket, destination_path) 319 except errors.HttpError as e: 320 if e.code == 503: 321 if self.NO_ACCESS_ERROR_PATTERN in str(e): 322 error_msg = "Please grant android build team's service account " 323 error_msg += "write access to bucket %s. Original error: %s" 324 error_msg %= (destination_bucket, str(e)) 325 raise errors.HttpError(e.code, message=error_msg) 326 raise 327 328 def GetBranch(self, build_target, build_id): 329 """Derives branch name. 330 331 Args: 332 build_target: Target name, e.g. "aosp_cf_x86_64_phone-userdebug" 333 build_id: Build ID, a string, e.g. "2263051", "P2804227" 334 335 Returns: 336 A string, the name of the branch 337 """ 338 api = self.service.build().get(buildId=build_id, target=build_target) 339 build = self.Execute(api) 340 return build.get("branch", "") 341 342 def GetLKGB(self, build_target, build_branch): 343 """Get latest successful build id. 344 345 From branch and target, we can use api to query latest successful build id. 346 e.g. {u'nextPageToken':..., u'builds': [{u'completionTimestamp':u'1534157869286', 347 ... u'buildId': u'4949805', u'machineName'...}]} 348 349 Args: 350 build_target: String, target name, e.g. "aosp_cf_x86_64_phone-userdebug" 351 build_branch: String, git branch name, e.g. "aosp-master" 352 353 Returns: 354 A string, string of build id number. 355 356 Raises: 357 errors.CreateError: Can't get build id. 358 """ 359 api = self.service.build().list( 360 branch=build_branch, 361 target=build_target, 362 buildAttemptStatus=self.BUILD_STATUS_COMPLETE, 363 buildType=self.BUILD_TYPE_SUBMITTED, 364 maxResults=self.ONE_RESULT, 365 successful=self.BUILD_SUCCESSFUL) 366 build = self.Execute(api) 367 logger.info("GetLKGB build API response: %s", build) 368 if build: 369 return str(build.get("builds")[0].get("buildId")) 370 raise errors.GetBuildIDError( 371 "No available good builds for branch: %s target: %s" 372 % (build_branch, build_target) 373 ) 374 375 def GetBuildInfo(self, build_target, build_id, branch): 376 """Get build info namedtuple. 377 378 Args: 379 build_target: Target name. 380 build_id: Build id, a string or None, e.g. "2263051", "P2804227" 381 If None or latest, the last green build id will be 382 returned. 383 branch: Branch name, a string or None, e.g. git_master. If None, the 384 returned branch will be searched by given build_id. 385 386 Returns: 387 A build info namedtuple with keys build_target, build_id, branch and 388 gcs_bucket_build_id 389 """ 390 if build_id and build_id != self.LATEST: 391 # Get build from build_id and build_target 392 api = self.service.build().get(buildId=build_id, 393 target=build_target) 394 build = self.Execute(api) or {} 395 elif branch: 396 # Get last green build in the branch 397 api = self.service.build().list( 398 branch=branch, 399 target=build_target, 400 successful=True, 401 maxResults=1, 402 buildType="submitted") 403 builds = self.Execute(api).get("builds", []) 404 build = builds[0] if builds else {} 405 else: 406 build = {} 407 408 build_id = build.get("buildId") 409 build_target = build_target if build_id else None 410 return BuildInfo(build.get("branch"), build_id, build_target, 411 build.get("releaseCandidateName")) 412