1#!/usr/bin/env python 2# 3# Copyright 2016 - The Android Open Source Project 4# 5# Licensed under the Apache License, Version 2.0 (the "License"); 6# you may not use this file except in compliance with the License. 7# You may obtain a copy of the License at 8# 9# http://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an "AS IS" BASIS, 13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14# See the License for the specific language governing permissions and 15# limitations under the License. 16 17"""A client that talks to Android Build APIs.""" 18 19import collections 20import io 21import json 22import logging 23import os 24import ssl 25import stat 26 27import apiclient 28 29from acloud import errors 30from acloud.internal.lib import base_cloud_client 31from acloud.internal.lib import utils 32 33 34logger = logging.getLogger(__name__) 35 36# The BuildInfo namedtuple data structure. 37# It will be the data structure returned by GetBuildInfo method. 38BuildInfo = collections.namedtuple("BuildInfo", [ 39 "branch", # The branch name string 40 "build_id", # The build id string 41 "build_target", # The build target string 42 "release_build_id"]) # The release build id string 43_DEFAULT_BRANCH = "aosp-master" 44 45 46class AndroidBuildClient(base_cloud_client.BaseCloudApiClient): 47 """Client that manages Android Build.""" 48 49 # API settings, used by BaseCloudApiClient. 50 API_NAME = "androidbuildinternal" 51 API_VERSION = "v2beta1" 52 SCOPE = "https://www.googleapis.com/auth/androidbuild.internal" 53 54 # other variables. 55 DEFAULT_RESOURCE_ID = "0" 56 # TODO(b/27269552): We should use "latest". 57 DEFAULT_ATTEMPT_ID = "0" 58 DEFAULT_CHUNK_SIZE = 20 * 1024 * 1024 59 NO_ACCESS_ERROR_PATTERN = "does not have storage.objects.create access" 60 # LKGB variables. 61 BUILD_STATUS_COMPLETE = "complete" 62 BUILD_TYPE_SUBMITTED = "submitted" 63 ONE_RESULT = 1 64 BUILD_SUCCESSFUL = True 65 LATEST = "latest" 66 # FETCH_CVD variables. 67 FETCHER_NAME = "fetch_cvd" 68 FETCHER_BUILD_TARGET = "aosp_cf_x86_phone-userdebug" 69 MAX_RETRY = 3 70 RETRY_SLEEP_SECS = 3 71 72 # Message constant 73 COPY_TO_MSG = ("build artifact (target: %s, build_id: %s, " 74 "artifact: %s, attempt_id: %s) to " 75 "google storage (bucket: %s, path: %s)") 76 # pylint: disable=invalid-name 77 def DownloadArtifact(self, 78 build_target, 79 build_id, 80 resource_id, 81 local_dest, 82 attempt_id=None): 83 """Get Android build attempt information. 84 85 Args: 86 build_target: Target name, e.g. "aosp_cf_x86_phone-userdebug" 87 build_id: Build id, a string, e.g. "2263051", "P2804227" 88 resource_id: Id of the resource, e.g "avd-system.tar.gz". 89 local_dest: A local path where the artifact should be stored. 90 e.g. "/tmp/avd-system.tar.gz" 91 attempt_id: String, attempt id, will default to DEFAULT_ATTEMPT_ID. 92 """ 93 attempt_id = attempt_id or self.DEFAULT_ATTEMPT_ID 94 api = self.service.buildartifact().get_media( 95 buildId=build_id, 96 target=build_target, 97 attemptId=attempt_id, 98 resourceId=resource_id) 99 logger.info("Downloading artifact: target: %s, build_id: %s, " 100 "resource_id: %s, dest: %s", build_target, build_id, 101 resource_id, local_dest) 102 try: 103 with io.FileIO(local_dest, mode="wb") as fh: 104 downloader = apiclient.http.MediaIoBaseDownload( 105 fh, api, chunksize=self.DEFAULT_CHUNK_SIZE) 106 done = False 107 while not done: 108 _, done = downloader.next_chunk() 109 logger.info("Downloaded artifact: %s", local_dest) 110 except (OSError, apiclient.errors.HttpError) as e: 111 logger.error("Downloading artifact failed: %s", str(e)) 112 raise errors.DriverError(str(e)) 113 114 def DownloadFetchcvd(self, local_dest, fetch_cvd_version): 115 """Get fetch_cvd from Android Build. 116 117 Args: 118 local_dest: A local path where the artifact should be stored. 119 e.g. "/tmp/fetch_cvd" 120 fetch_cvd_version: String of fetch_cvd version. 121 """ 122 utils.RetryExceptionType( 123 exception_types=ssl.SSLError, 124 max_retries=self.MAX_RETRY, 125 functor=self.DownloadArtifact, 126 sleep_multiplier=self.RETRY_SLEEP_SECS, 127 retry_backoff_factor=utils.DEFAULT_RETRY_BACKOFF_FACTOR, 128 build_target=self.FETCHER_BUILD_TARGET, 129 build_id=fetch_cvd_version, 130 resource_id=self.FETCHER_NAME, 131 local_dest=local_dest, 132 attempt_id=self.LATEST) 133 fetch_cvd_stat = os.stat(local_dest) 134 os.chmod(local_dest, fetch_cvd_stat.st_mode | stat.S_IEXEC) 135 136 @staticmethod 137 def ProcessBuild(build_id=None, branch=None, build_target=None): 138 """Create a Cuttlefish fetch_cvd build string. 139 140 Args: 141 build_id: A specific build number to load from. Takes precedence over `branch`. 142 branch: A manifest-branch at which to get the latest build. 143 build_target: A particular device to load at the desired build. 144 145 Returns: 146 A string, used in the fetch_cvd cmd or None if all args are None. 147 """ 148 if not build_target: 149 return build_id or branch 150 151 if build_target and not branch: 152 branch = _DEFAULT_BRANCH 153 return (build_id or branch) + "/" + build_target 154 155 # pylint: disable=too-many-locals 156 def GetFetchBuildArgs(self, build_id, branch, build_target, system_build_id, 157 system_branch, system_build_target, kernel_build_id, 158 kernel_branch, kernel_build_target, bootloader_build_id, 159 bootloader_branch, bootloader_build_target): 160 """Get args from build information for fetch_cvd. 161 162 Args: 163 build_id: String of build id, e.g. "2263051", "P2804227" 164 branch: String of branch name, e.g. "aosp-master" 165 build_target: String of target name. 166 e.g. "aosp_cf_x86_phone-userdebug" 167 system_build_id: String of the system image build id. 168 system_branch: String of the system image branch name. 169 system_build_target: String of the system image target name, 170 e.g. "cf_x86_phone-userdebug" 171 kernel_build_id: String of the kernel image build id. 172 kernel_branch: String of the kernel image branch name. 173 kernel_build_target: String of the kernel image target name, 174 bootloader_build_id: String of the bootloader build id. 175 bootloader_branch: String of the bootloader branch name. 176 bootloader_build_target: String of the bootloader target name. 177 178 Returns: 179 List of string args for fetch_cvd. 180 """ 181 fetch_cvd_args = [] 182 183 default_build = self.ProcessBuild(build_id, branch, build_target) 184 if default_build: 185 fetch_cvd_args.append("-default_build=" + default_build) 186 system_build = self.ProcessBuild( 187 system_build_id, system_branch, system_build_target) 188 if system_build: 189 fetch_cvd_args.append("-system_build=" + system_build) 190 bootloader_build = self.ProcessBuild(bootloader_build_id, 191 bootloader_branch, 192 bootloader_build_target) 193 if bootloader_build: 194 fetch_cvd_args.append("-bootloader_build=%s" % bootloader_build) 195 kernel_build = self.GetKernelBuild(kernel_build_id, 196 kernel_branch, 197 kernel_build_target) 198 if kernel_build: 199 fetch_cvd_args.append("-kernel_build=" + kernel_build) 200 201 return fetch_cvd_args 202 203 @staticmethod 204 # pylint: disable=broad-except 205 def GetFetchCertArg(certification_file): 206 """Get cert arg from certification file for fetch_cvd. 207 208 Parse the certification file to get access token of the latest 209 credential data and pass it to fetch_cvd command. 210 Example of certification file: 211 { 212 "data": [ 213 { 214 "credential": { 215 "_class": "OAuth2Credentials", 216 "_module": "oauth2client.client", 217 "access_token": "token_strings", 218 "client_id": "179485041932", 219 } 220 }] 221 } 222 223 224 Args: 225 certification_file: String of certification file path. 226 227 Returns: 228 String of certificate arg for fetch_cvd. If there is no 229 certification file, return empty string for aosp branch. 230 """ 231 cert_arg = "" 232 233 try: 234 with open(certification_file) as cert_file: 235 auth_token = json.load(cert_file).get("data")[-1].get( 236 "credential").get("access_token") 237 if auth_token: 238 cert_arg = "-credential_source=%s" % auth_token 239 except Exception as e: 240 utils.PrintColorString( 241 "Fail to open the certification file(%s): %s" % 242 (certification_file, e), utils.TextColors.WARNING) 243 return cert_arg 244 245 def GetKernelBuild(self, kernel_build_id, kernel_branch, kernel_build_target): 246 """Get kernel build args for fetch_cvd. 247 248 Args: 249 kernel_branch: Kernel branch name, e.g. "kernel-common-android-4.14" 250 kernel_build_id: Kernel build id, a string, e.g. "223051", "P280427" 251 kernel_build_target: String, Kernel build target name. 252 253 Returns: 254 String of kernel build args for fetch_cvd. 255 If no kernel build then return None. 256 """ 257 # kernel_target have default value "kernel". If user provide kernel_build_id 258 # or kernel_branch, then start to process kernel image. 259 if kernel_build_id or kernel_branch: 260 return self.ProcessBuild(kernel_build_id, kernel_branch, kernel_build_target) 261 return None 262 263 def CopyTo(self, 264 build_target, 265 build_id, 266 artifact_name, 267 destination_bucket, 268 destination_path, 269 attempt_id=None): 270 """Copy an Android Build artifact to a storage bucket. 271 272 Args: 273 build_target: Target name, e.g. "aosp_cf_x86_phone-userdebug" 274 build_id: Build id, a string, e.g. "2263051", "P2804227" 275 artifact_name: Name of the artifact, e.g "avd-system.tar.gz". 276 destination_bucket: String, a google storage bucket name. 277 destination_path: String, "path/inside/bucket" 278 attempt_id: String, attempt id, will default to DEFAULT_ATTEMPT_ID. 279 """ 280 attempt_id = attempt_id or self.DEFAULT_ATTEMPT_ID 281 copy_msg = "Copying %s" % self.COPY_TO_MSG 282 logger.info(copy_msg, build_target, build_id, artifact_name, 283 attempt_id, destination_bucket, destination_path) 284 api = self.service.buildartifact().copyTo( 285 buildId=build_id, 286 target=build_target, 287 attemptId=attempt_id, 288 artifactName=artifact_name, 289 destinationBucket=destination_bucket, 290 destinationPath=destination_path) 291 try: 292 self.Execute(api) 293 finish_msg = "Finished copying %s" % self.COPY_TO_MSG 294 logger.info(finish_msg, build_target, build_id, artifact_name, 295 attempt_id, destination_bucket, destination_path) 296 except errors.HttpError as e: 297 if e.code == 503: 298 if self.NO_ACCESS_ERROR_PATTERN in str(e): 299 error_msg = "Please grant android build team's service account " 300 error_msg += "write access to bucket %s. Original error: %s" 301 error_msg %= (destination_bucket, str(e)) 302 raise errors.HttpError(e.code, message=error_msg) 303 raise 304 305 def GetBranch(self, build_target, build_id): 306 """Derives branch name. 307 308 Args: 309 build_target: Target name, e.g. "aosp_cf_x86_phone-userdebug" 310 build_id: Build ID, a string, e.g. "2263051", "P2804227" 311 312 Returns: 313 A string, the name of the branch 314 """ 315 api = self.service.build().get(buildId=build_id, target=build_target) 316 build = self.Execute(api) 317 return build.get("branch", "") 318 319 def GetLKGB(self, build_target, build_branch): 320 """Get latest successful build id. 321 322 From branch and target, we can use api to query latest successful build id. 323 e.g. {u'nextPageToken':..., u'builds': [{u'completionTimestamp':u'1534157869286', 324 ... u'buildId': u'4949805', u'machineName'...}]} 325 326 Args: 327 build_target: String, target name, e.g. "aosp_cf_x86_phone-userdebug" 328 build_branch: String, git branch name, e.g. "aosp-master" 329 330 Returns: 331 A string, string of build id number. 332 333 Raises: 334 errors.CreateError: Can't get build id. 335 """ 336 api = self.service.build().list( 337 branch=build_branch, 338 target=build_target, 339 buildAttemptStatus=self.BUILD_STATUS_COMPLETE, 340 buildType=self.BUILD_TYPE_SUBMITTED, 341 maxResults=self.ONE_RESULT, 342 successful=self.BUILD_SUCCESSFUL) 343 build = self.Execute(api) 344 logger.info("GetLKGB build API response: %s", build) 345 if build: 346 return str(build.get("builds")[0].get("buildId")) 347 raise errors.GetBuildIDError( 348 "No available good builds for branch: %s target: %s" 349 % (build_branch, build_target) 350 ) 351 352 def GetBuildInfo(self, build_target, build_id, branch): 353 """Get build info namedtuple. 354 355 Args: 356 build_target: Target name. 357 build_id: Build id, a string or None, e.g. "2263051", "P2804227" 358 If None or latest, the last green build id will be 359 returned. 360 branch: Branch name, a string or None, e.g. git_master. If None, the 361 returned branch will be searched by given build_id. 362 363 Returns: 364 A build info namedtuple with keys build_target, build_id, branch and 365 gcs_bucket_build_id 366 """ 367 if build_id and build_id != self.LATEST: 368 # Get build from build_id and build_target 369 api = self.service.build().get(buildId=build_id, 370 target=build_target) 371 build = self.Execute(api) or {} 372 elif branch: 373 # Get last green build in the branch 374 api = self.service.build().list( 375 branch=branch, 376 target=build_target, 377 successful=True, 378 maxResults=1, 379 buildType="submitted") 380 builds = self.Execute(api).get("builds", []) 381 build = builds[0] if builds else {} 382 else: 383 build = {} 384 385 build_id = build.get("buildId") 386 build_target = build_target if build_id else None 387 return BuildInfo(build.get("branch"), build_id, build_target, 388 build.get("releaseCandidateName")) 389