• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1"""Provides helper functions for fetching artifacts."""
2
3import io
4import os
5import re
6import sys
7import sysconfig
8import time
9
10# This is a workaround to put '/usr/lib/python3.X' ahead of googleapiclient
11# Using embedded_launcher won't work since py3-cmd doesn't contain _ssl module.
12if sys.version_info.major == 3:
13  sys.path.insert(0, os.path.dirname(sysconfig.get_paths()['purelib']))
14
15# pylint: disable=import-error,g-bad-import-order,g-import-not-at-top
16import apiclient
17from googleapiclient.discovery import build
18from six.moves import http_client
19
20import httplib2
21from oauth2client.service_account import ServiceAccountCredentials
22
23_SCOPE_URL = 'https://www.googleapis.com/auth/androidbuild.internal'
24_DEF_JSON_KEYFILE = '.config/gcloud/application_default_credentials.json'
25
26
27# 20 MB default chunk size -- used in Buildbot
28_DEFAULT_CHUNK_SIZE = 20 * 1024 * 1024
29
30# HTTP errors -- used in Builbot
31_DEFAULT_MASKED_ERRORS = [404]
32_DEFAULT_RETRIED_ERRORS = [503]
33_DEFAULT_RETRIES = 10
34
35
36def _create_http_from_p12(robot_credentials_file, robot_username):
37  """Creates a credentialed HTTP object for requests.
38
39  Args:
40    robot_credentials_file: The path to the robot credentials file.
41    robot_username: A string containing the username of the robot account.
42
43  Returns:
44    An authorized httplib2.Http object.
45  """
46  try:
47    credentials = ServiceAccountCredentials.from_p12_keyfile(
48        service_account_email=robot_username,
49        filename=robot_credentials_file,
50        scopes=_SCOPE_URL)
51  except AttributeError:
52    raise ValueError('Machine lacks openssl or pycrypto support')
53  http = httplib2.Http()
54  return credentials.authorize(http)
55
56
57def _simple_execute(http_request,
58                    masked_errors=None,
59                    retried_errors=None,
60                    retry_delay_seconds=5,
61                    max_tries=_DEFAULT_RETRIES):
62  """Execute http request and return None on specified errors.
63
64  Args:
65    http_request: the apiclient provided http request
66    masked_errors: list of errors to return None on
67    retried_errors: list of erros to retry the request on
68    retry_delay_seconds: how many seconds to sleep before retrying
69    max_tries: maximum number of attmpts to make request
70
71  Returns:
72    The result on success or None on masked errors.
73  """
74  if not masked_errors:
75    masked_errors = _DEFAULT_MASKED_ERRORS
76  if not retried_errors:
77    retried_errors = _DEFAULT_RETRIED_ERRORS
78
79  last_error = None
80  for _ in range(max_tries):
81    try:
82      return http_request.execute()
83    except http_client.errors.HttpError as e:
84      last_error = e
85      if e.resp.status in masked_errors:
86        return None
87      elif e.resp.status in retried_errors:
88        time.sleep(retry_delay_seconds)
89      else:
90        # Server Error is server error
91        raise e
92
93  # We've gone through the max_retries, raise the last error
94  raise last_error  # pylint: disable=raising-bad-type
95
96
97def create_client(http):
98  """Creates an Android build api client from an authorized http object.
99
100  Args:
101     http: An authorized httplib2.Http object.
102
103  Returns:
104    An authorized android build api client.
105  """
106  return build(serviceName='androidbuildinternal', version='v2beta1', http=http,
107               static_discovery=False)
108
109
110def create_client_from_json_keyfile(json_keyfile_name=None):
111  """Creates an Android build api client from a json keyfile.
112
113  Args:
114    json_keyfile_name: The location of the keyfile, if None is provided use
115                       default location.
116
117  Returns:
118    An authorized android build api client.
119  """
120  if not json_keyfile_name:
121    json_keyfile_name = os.path.join(os.getenv('HOME'), _DEF_JSON_KEYFILE)
122
123  credentials = ServiceAccountCredentials.from_json_keyfile_name(
124      filename=json_keyfile_name, scopes=_SCOPE_URL)
125  http = httplib2.Http()
126  credentials.authorize(http)
127  return create_client(http)
128
129
130def create_client_from_p12(robot_credentials_file, robot_username):
131  """Creates an Android build api client from a config file.
132
133  Args:
134    robot_credentials_file: The path to the robot credentials file.
135    robot_username: A string containing the username of the robot account.
136
137  Returns:
138    An authorized android build api client.
139  """
140  http = _create_http_from_p12(robot_credentials_file, robot_username)
141  return create_client(http)
142
143
144def fetch_artifact(client, build_id, target, resource_id, dest):
145  """Fetches an artifact.
146
147  Args:
148    client: An authorized android build api client.
149    build_id: AB build id
150    target: the target name to download from
151    resource_id: the resource id of the artifact
152    dest: path to store the artifact
153  """
154  out_dir = os.path.dirname(dest)
155  if not os.path.exists(out_dir):
156    os.makedirs(out_dir)
157
158  dl_req = client.buildartifact().get_media(
159      buildId=build_id,
160      target=target,
161      attemptId='latest',
162      resourceId=resource_id)
163
164  print('Fetching %s to %s...' % (resource_id, dest))
165  with io.FileIO(dest, mode='wb') as fh:
166    downloader = apiclient.http.MediaIoBaseDownload(
167        fh, dl_req, chunksize=_DEFAULT_CHUNK_SIZE)
168    done = False
169    while not done:
170      status, done = downloader.next_chunk(num_retries=_DEFAULT_RETRIES)
171      print('Fetching...' + str(status.progress() * 100))
172
173  print('Done Fetching %s to %s' % (resource_id, dest))
174
175
176def get_build_list(client, **kwargs):
177  """Get a list of builds from the android build api that matches parameters.
178
179  Args:
180    client: An authorized android build api client.
181    **kwargs: keyworded arguments to pass to build api.
182
183  Returns:
184    Response from build api.
185  """
186  build_request = client.build().list(**kwargs)
187
188  return _simple_execute(build_request)
189
190
191def list_artifacts(client, regex, **kwargs):
192  """List artifacts from the android build api that matches parameters.
193
194  Args:
195    client: An authorized android build api client.
196    regex: Regular expression pattern to match artifact name.
197    **kwargs: keyworded arguments to pass to buildartifact.list api.
198
199  Returns:
200    List of matching artifact names.
201  """
202  matching_artifacts = []
203  kwargs.setdefault('attemptId', 'latest')
204  regex = re.compile(regex)
205  req = client.buildartifact().list(**kwargs)
206  while req:
207    result = _simple_execute(req)
208    if result and 'artifacts' in result:
209      for a in result['artifacts']:
210        if regex.match(a['name']):
211          matching_artifacts.append(a['name'])
212    req = client.buildartifact().list_next(req, result)
213  return matching_artifacts
214
215
216def fetch_artifacts(client, out_dir, target, pattern, build_id):
217  """Fetches target files artifacts matching patterns.
218
219  Args:
220    client: An authorized instance of an android build api client for making
221      requests.
222    out_dir: The directory to store the fetched artifacts to.
223    target: The target name to download from.
224    pattern: A regex pattern to match to artifacts filename.
225    build_id: The Android Build id.
226  """
227  if not os.path.exists(out_dir):
228    os.makedirs(out_dir)
229
230  # Build a list of needed artifacts
231  artifacts = list_artifacts(
232      client=client,
233      regex=pattern,
234      buildId=build_id,
235      target=target)
236
237  for artifact in artifacts:
238    fetch_artifact(
239        client=client,
240        build_id=build_id,
241        target=target,
242        resource_id=artifact,
243        dest=os.path.join(out_dir, artifact))
244
245
246def get_latest_build_id(client, branch, target):
247  """Get the latest build id.
248
249  Args:
250    client: An authorized instance of an android build api client for making
251      requests.
252    branch: The branch to download from
253    target: The target name to download from.
254  Returns:
255    The build id.
256  """
257  build_response = get_build_list(
258      client=client,
259      branch=branch,
260      target=target,
261      maxResults=1,
262      successful=True,
263      buildType='submitted')
264
265  if not build_response:
266    raise ValueError('Unable to determine latest build ID!')
267
268  return build_response['builds'][0]['buildId']
269
270
271def fetch_latest_artifacts(client, out_dir, target, pattern, branch):
272  """Fetches target files artifacts matching patterns from the latest build.
273
274  Args:
275    client: An authorized instance of an android build api client for making
276      requests.
277    out_dir: The directory to store the fetched artifacts to.
278    target: The target name to download from.
279    pattern: A regex pattern to match to artifacts filename
280    branch: The branch to download from
281  """
282  build_id = get_latest_build_id(
283      client=client, branch=branch, target=target)
284
285  fetch_artifacts(client, out_dir, target, pattern, build_id)
286