• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#
2# Copyright (C) 2016 The Android Open Source Project
3#
4# Licensed under the Apache License, Version 2.0 (the "License");
5# you may not use this file except in compliance with the License.
6# You may obtain a copy of the License at
7#
8#      http://www.apache.org/licenses/LICENSE-2.0
9#
10# Unless required by applicable law or agreed to in writing, software
11# distributed under the License is distributed on an "AS IS" BASIS,
12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13# See the License for the specific language governing permissions and
14# limitations under the License.
15#
16"""Class to fetch artifacts from internal build server.
17"""
18
19import apiclient
20import httplib2
21import io
22import json
23import logging
24import re
25import time
26from apiclient.discovery import build
27from oauth2client import client as oauth2_client
28from oauth2client.service_account import ServiceAccountCredentials
29from vts.utils.python.retry import retry
30
31logger = logging.getLogger('artifact_fetcher')
32
33class DriverError(Exception):
34    """Base Android GCE driver exception."""
35
36class AndroidBuildClient(object):
37    """Client that manages Android Build.
38
39    Attributes:
40        service: object, initialized and authorized service object for the
41                 androidbuildinternal API.
42        API_NAME: string, name of internal API accessed by the client.
43        API_VERSION: string, version of the internal API accessed by the client.
44        SCOPE: string, URL for which to request access via oauth2.
45        DEFAULT_RESOURCE_ID: string, default artifact name to request.
46        DEFAULT_ATTEMPT_ID: string, default attempt to request for the artifact.
47        DEFAULT_CHUNK_SIZE: int, number of bytes to download at a time.
48        RETRY_COUNT: int, max number of retries.
49        RETRY_BACKOFF_FACTOR: float, base of exponential determining sleep time.
50                              total_time = (backoff_factor^(attempt - 1))*sleep
51        RETRY_SLEEP_MULTIPLIER: float, multiplier for how long to sleep between
52                                attempts.
53        RETRY_HTTP_CODES: int array, HTTP codes for which a retry will be
54                          attempted.
55        RETRIABLE_AUTH_ERRORS: class tuple, list of error classes for which a
56                               retry will be attempted.
57
58    """
59
60    API_NAME = "androidbuildinternal"
61    API_VERSION = "v2beta1"
62    SCOPE = "https://www.googleapis.com/auth/androidbuild.internal"
63
64    # other variables.
65    DEFAULT_RESOURCE_ID = "0"
66    DEFAULT_ATTEMPT_ID = "latest"
67    DEFAULT_CHUNK_SIZE = 20 * 1024 * 1024
68
69    # Defaults for retry.
70    RETRY_COUNT = 5
71    RETRY_BACKOFF_FACTOR = 1.5
72    RETRY_SLEEP_MULTIPLIER = 1
73    RETRY_HTTP_CODES = [
74        500,  # Internal Server Error
75        502,  # Bad Gateway
76        503,  # Service Unavailable
77    ]
78
79    RETRIABLE_AUTH_ERRORS = (oauth2_client.AccessTokenRefreshError,)
80
81    def __init__(self, oauth2_service_json):
82        """Initialize.
83
84        Args:
85          oauth2_service_json: Path to service account json file.
86        """
87        authToken = ServiceAccountCredentials.from_json_keyfile_name(
88            oauth2_service_json, [self.SCOPE])
89        http_auth = authToken.authorize(httplib2.Http())
90        self.service = retry.RetryException(
91            exc_retry=self.RETRIABLE_AUTH_ERRORS,
92            max_retry=self.RETRY_COUNT,
93            functor=build,
94            sleep=self.RETRY_SLEEP_MULTIPLIER,
95            backoff_factor=self.RETRY_BACKOFF_FACTOR,
96            serviceName=self.API_NAME,
97            version=self.API_VERSION,
98            http=http_auth)
99
100    def GetArtifact(self, branch, build_target, build_id, resource_id,
101                         attempt_id=None):
102        """Get artifact from android build server.
103
104        Args:
105            branch: Branch from which the code was built, e.g. "master"
106            build_target: Target name, e.g. "gce_x86-userdebug"
107            build_id: Build id, a string, e.g. "2263051", "P2804227"
108            resource_id: Name of resource to be downloaded, a string.
109            attempt_id: String, attempt id, will default to DEFAULT_ATTEMPT_ID.
110
111
112        Returns:
113            Contents of the requested resource as a string.
114        """
115        attempt_id = attempt_id or self.DEFAULT_ATTEMPT_ID
116        api = self.service.buildartifact().get_media(
117            buildId=build_id,
118            target=build_target, attemptId=attempt_id,
119            resourceId=resource_id)
120        logger.info("Downloading artifact: target: %s, build_id: %s, "
121                    "resource_id: %s", build_target, build_id, resource_id)
122        try:
123            with io.BytesIO() as mem_buffer:
124                downloader = apiclient.http.MediaIoBaseDownload(
125                    mem_buffer, api, chunksize=self.DEFAULT_CHUNK_SIZE)
126                done = False
127                while not done:
128                    _, done = downloader.next_chunk()
129                logger.info("Downloaded artifact %s" % resource_id)
130                return mem_buffer.getvalue()
131        except OSError as e:
132            logger.error("Downloading artifact failed: %s", str(e))
133            raise DriverError(str(e))
134
135    def GetManifest(self, branch, build_target, build_id, attempt_id=None):
136        """Get Android build manifest XML file.
137
138        Args:
139            branch: Branch from which the code was built, e.g. "master"
140            build_target: Target name, e.g. "gce_x86-userdebug"
141            build_id: Build id, a string, e.g. "2263051", "P2804227"
142            attempt_id: String, attempt id, will default to DEFAULT_ATTEMPT_ID.
143
144
145        Returns:
146            Contents of the requested XML file as a string.
147        """
148        resource_id = "manifest_%s.xml" % build_id
149        return self.GetArtifact(branch, build_target, build_id, resource_id,
150                                attempt_id)
151
152    def GetRepoDictionary(self, branch, build_target, build_id, attempt_id=None):
153        """Get dictionary of repositories and git revision IDs
154
155        Args:
156            branch: Branch from which the code was built, e.g. "master"
157            build_target: Target name, e.g. "gce_x86-userdebug"
158            build_id: Build id, a string, e.g. "2263051", "P2804227"
159            attempt_id: String, attempt id, will default to DEFAULT_ATTEMPT_ID.
160
161
162        Returns:
163            Dictionary of project names (string) to commit ID (string)
164        """
165        resource_id = "BUILD_INFO"
166        build_info = self.GetArtifact(branch, build_target, build_id,
167                                      resource_id, attempt_id)
168        try:
169            return json.loads(build_info)["repo-dict"]
170        except (ValueError, KeyError):
171            logger.warn("Could not find repo dictionary.")
172            return {}
173
174    def GetCoverage(self, branch, build_target, build_id, product,
175                    attempt_id=None):
176        """Get Android build coverage zip file.
177
178        Args:
179            branch: Branch from which the code was built, e.g. "master"
180            build_target: Target name, e.g. "gce_x86-userdebug"
181            build_id: Build id, a string, e.g. "2263051", "P2804227"
182            product: Name of product for build target, e.g. "bullhead", "angler"
183            attempt_id: String, attempt id, will default to DEFAULT_ATTEMPT_ID.
184
185
186        Returns:
187            Contents of the requested zip file as a string.
188        """
189        resource_id = ("%s-coverage-%s.zip" % (product, build_id))
190        return self.GetArtifact(branch, build_target, build_id, resource_id,
191                                attempt_id)
192