• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1"""Provides helper functions for fetching artifacts."""
2
3import io
4import os
5import re
6import sys
7import sysconfig
8import time
9
10# This is a workaround to put '/usr/lib/python3.X' ahead of googleapiclient
11# Using embedded_launcher won't work since py3-cmd doesn't contain _ssl module.
12if sys.version_info.major == 3:
13  sys.path.insert(0, os.path.dirname(sysconfig.get_paths()['purelib']))
14
15# pylint: disable=import-error,g-bad-import-order,g-import-not-at-top
16import apiclient
17from googleapiclient.discovery import build
18from six.moves import http_client
19
20import httplib2
21from oauth2client.service_account import ServiceAccountCredentials
22
23_SCOPE_URL = 'https://www.googleapis.com/auth/androidbuild.internal'
24_DEF_JSON_KEYFILE = '.config/gcloud/application_default_credentials.json'
25
26
27# 20 MB default chunk size -- used in Buildbot
28_DEFAULT_CHUNK_SIZE = 20 * 1024 * 1024
29
30# HTTP errors -- used in Builbot
31_DEFAULT_MASKED_ERRORS = [404]
32_DEFAULT_RETRIED_ERRORS = [503]
33_DEFAULT_RETRIES = 10
34
35
36def _create_http_from_p12(robot_credentials_file, robot_username):
37  """Creates a credentialed HTTP object for requests.
38
39  Args:
40    robot_credentials_file: The path to the robot credentials file.
41    robot_username: A string containing the username of the robot account.
42
43  Returns:
44    An authorized httplib2.Http object.
45  """
46  try:
47    credentials = ServiceAccountCredentials.from_p12_keyfile(
48        service_account_email=robot_username,
49        filename=robot_credentials_file,
50        scopes=_SCOPE_URL)
51  except AttributeError:
52    raise ValueError('Machine lacks openssl or pycrypto support')
53  http = httplib2.Http()
54  return credentials.authorize(http)
55
56
57def _simple_execute(http_request,
58                    masked_errors=None,
59                    retried_errors=None,
60                    retry_delay_seconds=5,
61                    max_tries=_DEFAULT_RETRIES):
62  """Execute http request and return None on specified errors.
63
64  Args:
65    http_request: the apiclient provided http request
66    masked_errors: list of errors to return None on
67    retried_errors: list of erros to retry the request on
68    retry_delay_seconds: how many seconds to sleep before retrying
69    max_tries: maximum number of attmpts to make request
70
71  Returns:
72    The result on success or None on masked errors.
73  """
74  if not masked_errors:
75    masked_errors = _DEFAULT_MASKED_ERRORS
76  if not retried_errors:
77    retried_errors = _DEFAULT_RETRIED_ERRORS
78
79  last_error = None
80  for _ in range(max_tries):
81    try:
82      return http_request.execute()
83    except http_client.errors.HttpError as e:
84      last_error = e
85      if e.resp.status in masked_errors:
86        return None
87      elif e.resp.status in retried_errors:
88        time.sleep(retry_delay_seconds)
89      else:
90        # Server Error is server error
91        raise e
92
93  # We've gone through the max_retries, raise the last error
94  raise last_error  # pylint: disable=raising-bad-type
95
96
97def create_client(http):
98  """Creates an Android build api client from an authorized http object.
99
100  Args:
101     http: An authorized httplib2.Http object.
102
103  Returns:
104    An authorized android build api client.
105  """
106  return build(serviceName='androidbuildinternal', version='v2beta1', http=http)
107
108
109def create_client_from_json_keyfile(json_keyfile_name=None):
110  """Creates an Android build api client from a json keyfile.
111
112  Args:
113    json_keyfile_name: The location of the keyfile, if None is provided use
114                       default location.
115
116  Returns:
117    An authorized android build api client.
118  """
119  if not json_keyfile_name:
120    json_keyfile_name = os.path.join(os.getenv('HOME'), _DEF_JSON_KEYFILE)
121
122  credentials = ServiceAccountCredentials.from_json_keyfile_name(
123      filename=json_keyfile_name, scopes=_SCOPE_URL)
124  http = httplib2.Http()
125  credentials.authorize(http)
126  return create_client(http)
127
128
129def create_client_from_p12(robot_credentials_file, robot_username):
130  """Creates an Android build api client from a config file.
131
132  Args:
133    robot_credentials_file: The path to the robot credentials file.
134    robot_username: A string containing the username of the robot account.
135
136  Returns:
137    An authorized android build api client.
138  """
139  http = _create_http_from_p12(robot_credentials_file, robot_username)
140  return create_client(http)
141
142
143def fetch_artifact(client, build_id, target, resource_id, dest):
144  """Fetches an artifact.
145
146  Args:
147    client: An authorized android build api client.
148    build_id: AB build id
149    target: the target name to download from
150    resource_id: the resource id of the artifact
151    dest: path to store the artifact
152  """
153  out_dir = os.path.dirname(dest)
154  if not os.path.exists(out_dir):
155    os.makedirs(out_dir)
156
157  dl_req = client.buildartifact().get_media(
158      buildId=build_id,
159      target=target,
160      attemptId='latest',
161      resourceId=resource_id)
162
163  print('Fetching %s to %s...' % (resource_id, dest))
164  with io.FileIO(dest, mode='wb') as fh:
165    downloader = apiclient.http.MediaIoBaseDownload(
166        fh, dl_req, chunksize=_DEFAULT_CHUNK_SIZE)
167    done = False
168    while not done:
169      status, done = downloader.next_chunk(num_retries=_DEFAULT_RETRIES)
170      print('Fetching...' + str(status.progress() * 100))
171
172  print('Done Fetching %s to %s' % (resource_id, dest))
173
174
175def get_build_list(client, **kwargs):
176  """Get a list of builds from the android build api that matches parameters.
177
178  Args:
179    client: An authorized android build api client.
180    **kwargs: keyworded arguments to pass to build api.
181
182  Returns:
183    Response from build api.
184  """
185  build_request = client.build().list(**kwargs)
186
187  return _simple_execute(build_request)
188
189
190def list_artifacts(client, regex, **kwargs):
191  """List artifacts from the android build api that matches parameters.
192
193  Args:
194    client: An authorized android build api client.
195    regex: Regular expression pattern to match artifact name.
196    **kwargs: keyworded arguments to pass to buildartifact.list api.
197
198  Returns:
199    List of matching artifact names.
200  """
201  matching_artifacts = []
202  kwargs.setdefault('attemptId', 'latest')
203  regex = re.compile(regex)
204  req = client.buildartifact().list(**kwargs)
205  while req:
206    result = _simple_execute(req)
207    if result and 'artifacts' in result:
208      for a in result['artifacts']:
209        if regex.match(a['name']):
210          matching_artifacts.append(a['name'])
211    req = client.buildartifact().list_next(req, result)
212  return matching_artifacts
213
214
215def fetch_artifacts(client, out_dir, target, pattern, build_id):
216  """Fetches target files artifacts matching patterns.
217
218  Args:
219    client: An authorized instance of an android build api client for making
220      requests.
221    out_dir: The directory to store the fetched artifacts to.
222    target: The target name to download from.
223    pattern: A regex pattern to match to artifacts filename.
224    build_id: The Android Build id.
225  """
226  if not os.path.exists(out_dir):
227    os.makedirs(out_dir)
228
229  # Build a list of needed artifacts
230  artifacts = list_artifacts(
231      client=client,
232      regex=pattern,
233      buildId=build_id,
234      target=target)
235
236  for artifact in artifacts:
237    fetch_artifact(
238        client=client,
239        build_id=build_id,
240        target=target,
241        resource_id=artifact,
242        dest=os.path.join(out_dir, artifact))
243
244
245def get_latest_build_id(client, branch, target):
246  """Get the latest build id.
247
248  Args:
249    client: An authorized instance of an android build api client for making
250      requests.
251    branch: The branch to download from
252    target: The target name to download from.
253  Returns:
254    The build id.
255  """
256  build_response = get_build_list(
257      client=client,
258      branch=branch,
259      target=target,
260      maxResults=1,
261      successful=True,
262      buildType='submitted')
263
264  if not build_response:
265    raise ValueError('Unable to determine latest build ID!')
266
267  return build_response['builds'][0]['buildId']
268
269
270def fetch_latest_artifacts(client, out_dir, target, pattern, branch):
271  """Fetches target files artifacts matching patterns from the latest build.
272
273  Args:
274    client: An authorized instance of an android build api client for making
275      requests.
276    out_dir: The directory to store the fetched artifacts to.
277    target: The target name to download from.
278    pattern: A regex pattern to match to artifacts filename
279    branch: The branch to download from
280  """
281  build_id = get_latest_build_id(
282      client=client, branch=branch, target=target)
283
284  fetch_artifacts(client, out_dir, target, pattern, build_id)
285