1#!/usr/bin/env python 2# 3# Copyright 2016 - The Android Open Source Project 4# 5# Licensed under the Apache License, Version 2.0 (the "License"); 6# you may not use this file except in compliance with the License. 7# You may obtain a copy of the License at 8# 9# http://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an "AS IS" BASIS, 13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14# See the License for the specific language governing permissions and 15# limitations under the License. 16 17"""A client that talks to Android Build APIs.""" 18 19import collections 20import io 21import json 22import logging 23import os 24import ssl 25import stat 26 27import apiclient 28 29from acloud import errors 30from acloud.internal.lib import base_cloud_client 31from acloud.internal.lib import utils 32 33 34logger = logging.getLogger(__name__) 35 36# The BuildInfo namedtuple data structure. 37# It will be the data structure returned by GetBuildInfo method. 38BuildInfo = collections.namedtuple("BuildInfo", [ 39 "branch", # The branch name string 40 "build_id", # The build id string 41 "build_target", # The build target string 42 "release_build_id"]) # The release build id string 43_DEFAULT_BRANCH = "aosp-master" 44 45 46class AndroidBuildClient(base_cloud_client.BaseCloudApiClient): 47 """Client that manages Android Build.""" 48 49 # API settings, used by BaseCloudApiClient. 50 API_NAME = "androidbuildinternal" 51 API_VERSION = "v2beta1" 52 SCOPE = "https://www.googleapis.com/auth/androidbuild.internal" 53 54 # other variables. 55 DEFAULT_RESOURCE_ID = "0" 56 # TODO(b/27269552): We should use "latest". 57 DEFAULT_ATTEMPT_ID = "0" 58 DEFAULT_CHUNK_SIZE = 20 * 1024 * 1024 59 NO_ACCESS_ERROR_PATTERN = "does not have storage.objects.create access" 60 # LKGB variables. 61 BUILD_STATUS_COMPLETE = "complete" 62 BUILD_TYPE_SUBMITTED = "submitted" 63 ONE_RESULT = 1 64 BUILD_SUCCESSFUL = True 65 LATEST = "latest" 66 # FETCH_CVD variables. 67 FETCHER_NAME = "fetch_cvd" 68 FETCHER_BUILD_TARGET = "aosp_cf_x86_64_phone-userdebug" 69 MAX_RETRY = 3 70 RETRY_SLEEP_SECS = 3 71 72 # Message constant 73 COPY_TO_MSG = ("build artifact (target: %s, build_id: %s, " 74 "artifact: %s, attempt_id: %s) to " 75 "google storage (bucket: %s, path: %s)") 76 # pylint: disable=invalid-name 77 def DownloadArtifact(self, 78 build_target, 79 build_id, 80 resource_id, 81 local_dest, 82 attempt_id=None): 83 """Get Android build attempt information. 84 85 Args: 86 build_target: Target name, e.g. "aosp_cf_x86_64_phone-userdebug" 87 build_id: Build id, a string, e.g. "2263051", "P2804227" 88 resource_id: Id of the resource, e.g "avd-system.tar.gz". 89 local_dest: A local path where the artifact should be stored. 90 e.g. "/tmp/avd-system.tar.gz" 91 attempt_id: String, attempt id, will default to DEFAULT_ATTEMPT_ID. 92 """ 93 attempt_id = attempt_id or self.DEFAULT_ATTEMPT_ID 94 api = self.service.buildartifact().get_media( 95 buildId=build_id, 96 target=build_target, 97 attemptId=attempt_id, 98 resourceId=resource_id) 99 logger.info("Downloading artifact: target: %s, build_id: %s, " 100 "resource_id: %s, dest: %s", build_target, build_id, 101 resource_id, local_dest) 102 try: 103 with io.FileIO(local_dest, mode="wb") as fh: 104 downloader = apiclient.http.MediaIoBaseDownload( 105 fh, api, chunksize=self.DEFAULT_CHUNK_SIZE) 106 done = False 107 while not done: 108 _, done = downloader.next_chunk() 109 logger.info("Downloaded artifact: %s", local_dest) 110 except (OSError, apiclient.errors.HttpError) as e: 111 logger.error("Downloading artifact failed: %s", str(e)) 112 raise errors.DriverError(str(e)) 113 114 def DownloadFetchcvd(self, local_dest, fetch_cvd_version): 115 """Get fetch_cvd from Android Build. 116 117 Args: 118 local_dest: A local path where the artifact should be stored. 119 e.g. "/tmp/fetch_cvd" 120 fetch_cvd_version: String of fetch_cvd version. 121 """ 122 utils.RetryExceptionType( 123 exception_types=ssl.SSLError, 124 max_retries=self.MAX_RETRY, 125 functor=self.DownloadArtifact, 126 sleep_multiplier=self.RETRY_SLEEP_SECS, 127 retry_backoff_factor=utils.DEFAULT_RETRY_BACKOFF_FACTOR, 128 build_target=self.FETCHER_BUILD_TARGET, 129 build_id=fetch_cvd_version, 130 resource_id=self.FETCHER_NAME, 131 local_dest=local_dest, 132 attempt_id=self.LATEST) 133 fetch_cvd_stat = os.stat(local_dest) 134 os.chmod(local_dest, fetch_cvd_stat.st_mode | stat.S_IEXEC) 135 136 @staticmethod 137 def ProcessBuild(build_id=None, branch=None, build_target=None): 138 """Create a Cuttlefish fetch_cvd build string. 139 140 Args: 141 build_id: A specific build number to load from. Takes precedence over `branch`. 142 branch: A manifest-branch at which to get the latest build. 143 build_target: A particular device to load at the desired build. 144 145 Returns: 146 A string, used in the fetch_cvd cmd or None if all args are None. 147 """ 148 if not build_target: 149 return build_id or branch 150 151 if build_target and not branch: 152 branch = _DEFAULT_BRANCH 153 return (build_id or branch) + "/" + build_target 154 155 # pylint: disable=too-many-locals 156 def GetFetchBuildArgs(self, build_id, branch, build_target, system_build_id, 157 system_branch, system_build_target, kernel_build_id, 158 kernel_branch, kernel_build_target, bootloader_build_id, 159 bootloader_branch, bootloader_build_target, 160 ota_build_id, ota_branch, ota_build_target): 161 """Get args from build information for fetch_cvd. 162 163 Args: 164 build_id: String of build id, e.g. "2263051", "P2804227" 165 branch: String of branch name, e.g. "aosp-master" 166 build_target: String of target name. 167 e.g. "aosp_cf_x86_64_phone-userdebug" 168 system_build_id: String of the system image build id. 169 system_branch: String of the system image branch name. 170 system_build_target: String of the system image target name, 171 e.g. "cf_x86_phone-userdebug" 172 kernel_build_id: String of the kernel image build id. 173 kernel_branch: String of the kernel image branch name. 174 kernel_build_target: String of the kernel image target name, 175 bootloader_build_id: String of the bootloader build id. 176 bootloader_branch: String of the bootloader branch name. 177 bootloader_build_target: String of the bootloader target name. 178 ota_build_id: String of the bootloader build id. 179 ota_branch: String of the bootloader branch name. 180 ota_build_target: String of the bootloader target name. 181 182 Returns: 183 List of string args for fetch_cvd. 184 """ 185 fetch_cvd_args = [] 186 187 default_build = self.ProcessBuild(build_id, branch, build_target) 188 if default_build: 189 fetch_cvd_args.append("-default_build=%s" % default_build) 190 system_build = self.ProcessBuild( 191 system_build_id, system_branch, system_build_target) 192 if system_build: 193 fetch_cvd_args.append("-system_build=%s" % system_build) 194 bootloader_build = self.ProcessBuild(bootloader_build_id, 195 bootloader_branch, 196 bootloader_build_target) 197 if bootloader_build: 198 fetch_cvd_args.append("-bootloader_build=%s" % bootloader_build) 199 kernel_build = self.GetKernelBuild(kernel_build_id, 200 kernel_branch, 201 kernel_build_target) 202 if kernel_build: 203 fetch_cvd_args.append("-kernel_build=%s" % kernel_build) 204 ota_build = self.ProcessBuild( 205 ota_build_id, ota_branch, ota_build_target) 206 if ota_build: 207 fetch_cvd_args.append("-otatools_build=%s" % ota_build) 208 209 return fetch_cvd_args 210 211 @staticmethod 212 # pylint: disable=broad-except 213 def GetFetchCertArg(certification_file): 214 """Get cert arg from certification file for fetch_cvd. 215 216 Parse the certification file to get access token of the latest 217 credential data and pass it to fetch_cvd command. 218 Example of certification file: 219 { 220 "data": [ 221 { 222 "credential": { 223 "_class": "OAuth2Credentials", 224 "_module": "oauth2client.client", 225 "access_token": "token_strings", 226 "client_id": "179485041932", 227 } 228 }] 229 } 230 231 232 Args: 233 certification_file: String of certification file path. 234 235 Returns: 236 String of certificate arg for fetch_cvd. If there is no 237 certification file, return empty string for aosp branch. 238 """ 239 cert_arg = "" 240 241 try: 242 with open(certification_file) as cert_file: 243 auth_token = json.load(cert_file).get("data")[-1].get( 244 "credential").get("access_token") 245 if auth_token: 246 cert_arg = "-credential_source=%s" % auth_token 247 except Exception as e: 248 utils.PrintColorString( 249 "Fail to open the certification file(%s): %s" % 250 (certification_file, e), utils.TextColors.WARNING) 251 return cert_arg 252 253 def GetKernelBuild(self, kernel_build_id, kernel_branch, kernel_build_target): 254 """Get kernel build args for fetch_cvd. 255 256 Args: 257 kernel_branch: Kernel branch name, e.g. "kernel-common-android-4.14" 258 kernel_build_id: Kernel build id, a string, e.g. "223051", "P280427" 259 kernel_build_target: String, Kernel build target name. 260 261 Returns: 262 String of kernel build args for fetch_cvd. 263 If no kernel build then return None. 264 """ 265 # kernel_target have default value "kernel". If user provide kernel_build_id 266 # or kernel_branch, then start to process kernel image. 267 if kernel_build_id or kernel_branch: 268 return self.ProcessBuild(kernel_build_id, kernel_branch, kernel_build_target) 269 return None 270 271 def CopyTo(self, 272 build_target, 273 build_id, 274 artifact_name, 275 destination_bucket, 276 destination_path, 277 attempt_id=None): 278 """Copy an Android Build artifact to a storage bucket. 279 280 Args: 281 build_target: Target name, e.g. "aosp_cf_x86_64_phone-userdebug" 282 build_id: Build id, a string, e.g. "2263051", "P2804227" 283 artifact_name: Name of the artifact, e.g "avd-system.tar.gz". 284 destination_bucket: String, a google storage bucket name. 285 destination_path: String, "path/inside/bucket" 286 attempt_id: String, attempt id, will default to DEFAULT_ATTEMPT_ID. 287 """ 288 attempt_id = attempt_id or self.DEFAULT_ATTEMPT_ID 289 copy_msg = "Copying %s" % self.COPY_TO_MSG 290 logger.info(copy_msg, build_target, build_id, artifact_name, 291 attempt_id, destination_bucket, destination_path) 292 api = self.service.buildartifact().copyTo( 293 buildId=build_id, 294 target=build_target, 295 attemptId=attempt_id, 296 artifactName=artifact_name, 297 destinationBucket=destination_bucket, 298 destinationPath=destination_path) 299 try: 300 self.Execute(api) 301 finish_msg = "Finished copying %s" % self.COPY_TO_MSG 302 logger.info(finish_msg, build_target, build_id, artifact_name, 303 attempt_id, destination_bucket, destination_path) 304 except errors.HttpError as e: 305 if e.code == 503: 306 if self.NO_ACCESS_ERROR_PATTERN in str(e): 307 error_msg = "Please grant android build team's service account " 308 error_msg += "write access to bucket %s. Original error: %s" 309 error_msg %= (destination_bucket, str(e)) 310 raise errors.HttpError(e.code, message=error_msg) 311 raise 312 313 def GetBranch(self, build_target, build_id): 314 """Derives branch name. 315 316 Args: 317 build_target: Target name, e.g. "aosp_cf_x86_64_phone-userdebug" 318 build_id: Build ID, a string, e.g. "2263051", "P2804227" 319 320 Returns: 321 A string, the name of the branch 322 """ 323 api = self.service.build().get(buildId=build_id, target=build_target) 324 build = self.Execute(api) 325 return build.get("branch", "") 326 327 def GetLKGB(self, build_target, build_branch): 328 """Get latest successful build id. 329 330 From branch and target, we can use api to query latest successful build id. 331 e.g. {u'nextPageToken':..., u'builds': [{u'completionTimestamp':u'1534157869286', 332 ... u'buildId': u'4949805', u'machineName'...}]} 333 334 Args: 335 build_target: String, target name, e.g. "aosp_cf_x86_64_phone-userdebug" 336 build_branch: String, git branch name, e.g. "aosp-master" 337 338 Returns: 339 A string, string of build id number. 340 341 Raises: 342 errors.CreateError: Can't get build id. 343 """ 344 api = self.service.build().list( 345 branch=build_branch, 346 target=build_target, 347 buildAttemptStatus=self.BUILD_STATUS_COMPLETE, 348 buildType=self.BUILD_TYPE_SUBMITTED, 349 maxResults=self.ONE_RESULT, 350 successful=self.BUILD_SUCCESSFUL) 351 build = self.Execute(api) 352 logger.info("GetLKGB build API response: %s", build) 353 if build: 354 return str(build.get("builds")[0].get("buildId")) 355 raise errors.GetBuildIDError( 356 "No available good builds for branch: %s target: %s" 357 % (build_branch, build_target) 358 ) 359 360 def GetBuildInfo(self, build_target, build_id, branch): 361 """Get build info namedtuple. 362 363 Args: 364 build_target: Target name. 365 build_id: Build id, a string or None, e.g. "2263051", "P2804227" 366 If None or latest, the last green build id will be 367 returned. 368 branch: Branch name, a string or None, e.g. git_master. If None, the 369 returned branch will be searched by given build_id. 370 371 Returns: 372 A build info namedtuple with keys build_target, build_id, branch and 373 gcs_bucket_build_id 374 """ 375 if build_id and build_id != self.LATEST: 376 # Get build from build_id and build_target 377 api = self.service.build().get(buildId=build_id, 378 target=build_target) 379 build = self.Execute(api) or {} 380 elif branch: 381 # Get last green build in the branch 382 api = self.service.build().list( 383 branch=branch, 384 target=build_target, 385 successful=True, 386 maxResults=1, 387 buildType="submitted") 388 builds = self.Execute(api).get("builds", []) 389 build = builds[0] if builds else {} 390 else: 391 build = {} 392 393 build_id = build.get("buildId") 394 build_target = build_target if build_id else None 395 return BuildInfo(build.get("branch"), build_id, build_target, 396 build.get("releaseCandidateName")) 397