• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1"""Provides helper functions for fetching artifacts."""
2
3import io
4import os
5import re
6import sys
7import sysconfig
8import time
9
10# This is a workaround to put '/usr/lib/python3.X' ahead of googleapiclient
11# Using embedded_launcher won't work since py3-cmd doesn't contain _ssl module.
12if sys.version_info.major == 3:
13  sys.path.insert(0, os.path.dirname(sysconfig.get_paths()['purelib']))
14
15# pylint: disable=import-error,g-bad-import-order,g-import-not-at-top
16import apiclient
17from googleapiclient.discovery import build
18from googleapiclient.errors import HttpError
19
20import httplib2
21from oauth2client.service_account import ServiceAccountCredentials
22
23_SCOPE_URL = 'https://www.googleapis.com/auth/androidbuild.internal'
24_DEF_JSON_KEYFILE = '.config/gcloud/application_default_credentials.json'
25
26
27# 20 MB default chunk size -- used in Buildbot
28_DEFAULT_CHUNK_SIZE = 20 * 1024 * 1024
29
30# HTTP errors -- used in Builbot
31_DEFAULT_MASKED_ERRORS = [404]
32_DEFAULT_RETRIED_ERRORS = [503]
33_DEFAULT_RETRIES = 10
34
35
36def _create_http_from_p12(robot_credentials_file, robot_username):
37  """Creates a credentialed HTTP object for requests.
38
39  Args:
40    robot_credentials_file: The path to the robot credentials file.
41    robot_username: A string containing the username of the robot account.
42
43  Returns:
44    An authorized httplib2.Http object.
45  """
46  try:
47    credentials = ServiceAccountCredentials.from_p12_keyfile(
48        service_account_email=robot_username,
49        filename=robot_credentials_file,
50        scopes=_SCOPE_URL)
51  except AttributeError:
52    raise ValueError('Machine lacks openssl or pycrypto support')
53  http = httplib2.Http()
54  return credentials.authorize(http)
55
56
57def _simple_execute(http_request,
58                    masked_errors=None,
59                    retried_errors=None,
60                    retry_delay_seconds=5,
61                    max_tries=_DEFAULT_RETRIES):
62  """Execute http request and return None on specified errors.
63
64  Args:
65    http_request: the apiclient provided http request
66    masked_errors: list of errors to return None on
67    retried_errors: list of erros to retry the request on
68    retry_delay_seconds: how many seconds to sleep before retrying
69    max_tries: maximum number of attmpts to make request
70
71  Returns:
72    The result on success or None on masked errors.
73  """
74  if not masked_errors:
75    masked_errors = _DEFAULT_MASKED_ERRORS
76  if not retried_errors:
77    retried_errors = _DEFAULT_RETRIED_ERRORS
78
79  last_error = None
80  for _ in range(max_tries):
81    try:
82      return http_request.execute()
83    except HttpError as e:
84      last_error = e
85      if e.resp.status in masked_errors:
86        return None
87      elif e.resp.status in retried_errors:
88        time.sleep(retry_delay_seconds)
89      else:
90        # Server Error is server error
91        raise e
92
93  # We've gone through the max_retries, raise the last error
94  raise last_error  # pylint: disable=raising-bad-type
95
96
97def create_client(http):
98  """Creates an Android build api client from an authorized http object.
99
100  Args:
101     http: An authorized httplib2.Http object.
102
103  Returns:
104    An authorized android build api client.
105  """
106  return build(serviceName='androidbuildinternal', version='v3', http=http,
107               static_discovery=False)
108
109
110def create_client_from_json_keyfile(json_keyfile_name=None):
111  """Creates an Android build api client from a json keyfile.
112
113  Args:
114    json_keyfile_name: The location of the keyfile, if None is provided use
115                       default location.
116
117  Returns:
118    An authorized android build api client.
119  """
120  if not json_keyfile_name:
121    json_keyfile_name = os.path.join(os.getenv('HOME'), _DEF_JSON_KEYFILE)
122
123  credentials = ServiceAccountCredentials.from_json_keyfile_name(
124      filename=json_keyfile_name, scopes=_SCOPE_URL)
125  http = httplib2.Http()
126  credentials.authorize(http)
127  return create_client(http)
128
129
130def create_client_from_p12(robot_credentials_file, robot_username):
131  """Creates an Android build api client from a config file.
132
133  Args:
134    robot_credentials_file: The path to the robot credentials file.
135    robot_username: A string containing the username of the robot account.
136
137  Returns:
138    An authorized android build api client.
139  """
140  http = _create_http_from_p12(robot_credentials_file, robot_username)
141  return create_client(http)
142
143
144def fetch_artifact(client, build_id, target, resource_id, dest):
145  """Fetches an artifact.
146
147  Args:
148    client: An authorized android build api client.
149    build_id: AB build id
150    target: the target name to download from
151    resource_id: the resource id of the artifact
152    dest: path to store the artifact
153  """
154  out_dir = os.path.dirname(dest)
155  if not os.path.exists(out_dir):
156    os.makedirs(out_dir)
157
158  dl_req = client.buildartifact().get_media(
159      buildId=build_id,
160      target=target,
161      attemptId='latest',
162      resourceId=resource_id)
163
164  print('Fetching %s to %s...' % (resource_id, dest))
165  with io.FileIO(dest, mode='wb') as fh:
166    downloader = apiclient.http.MediaIoBaseDownload(
167        fh, dl_req, chunksize=_DEFAULT_CHUNK_SIZE)
168    done = False
169    while not done:
170      status, done = downloader.next_chunk(num_retries=_DEFAULT_RETRIES)
171      print('Fetching...' + str(status.progress() * 100))
172
173  print('Done Fetching %s to %s' % (resource_id, dest))
174
175
176def get_build_list(client, **kwargs):
177  """Get a list of builds from the android build api that matches parameters.
178
179  Args:
180    client: An authorized android build api client.
181    **kwargs: keyworded arguments to pass to build api.
182
183  Returns:
184    Response from build api.
185  """
186  build_request = client.build().list(**kwargs)
187
188  return _simple_execute(build_request)
189
190
191def list_artifacts(client, regex, **kwargs):
192  """List artifacts from the android build api that matches parameters.
193
194  Args:
195    client: An authorized android build api client.
196    regex: Regular expression pattern to match artifact name.
197    **kwargs: keyworded arguments to pass to buildartifact.list api.
198
199  Returns:
200    List of matching artifact names.
201  """
202  matching_artifacts = []
203  kwargs.setdefault('attemptId', 'latest')
204  req = client.buildartifact().list(nameRegexp=regex, **kwargs)
205  while req:
206    result = _simple_execute(req)
207    if result and 'artifacts' in result:
208      for a in result['artifacts']:
209        matching_artifacts.append(a['name'])
210    req = client.buildartifact().list_next(req, result)
211  return matching_artifacts
212
213
214def fetch_artifacts(client, out_dir, target, pattern, build_id):
215  """Fetches target files artifacts matching patterns.
216
217  Args:
218    client: An authorized instance of an android build api client for making
219      requests.
220    out_dir: The directory to store the fetched artifacts to.
221    target: The target name to download from.
222    pattern: A regex pattern to match to artifacts filename.
223    build_id: The Android Build id.
224  """
225  if not os.path.exists(out_dir):
226    os.makedirs(out_dir)
227
228  # Build a list of needed artifacts
229  artifacts = list_artifacts(
230      client=client,
231      regex=pattern,
232      buildId=build_id,
233      target=target)
234
235  for artifact in artifacts:
236    fetch_artifact(
237        client=client,
238        build_id=build_id,
239        target=target,
240        resource_id=artifact,
241        dest=os.path.join(out_dir, artifact))
242
243
244def get_latest_build_id(client, branch, target):
245  """Get the latest build id.
246
247  Args:
248    client: An authorized instance of an android build api client for making
249      requests.
250    branch: The branch to download from
251    target: The target name to download from.
252  Returns:
253    The build id.
254  """
255  build_response = get_build_list(
256      client=client,
257      branch=branch,
258      target=target,
259      maxResults=1,
260      successful=True,
261      buildType='submitted')
262
263  if not build_response:
264    raise ValueError('Unable to determine latest build ID!')
265
266  return build_response['builds'][0]['buildId']
267
268
269def fetch_latest_artifacts(client, out_dir, target, pattern, branch):
270  """Fetches target files artifacts matching patterns from the latest build.
271
272  Args:
273    client: An authorized instance of an android build api client for making
274      requests.
275    out_dir: The directory to store the fetched artifacts to.
276    target: The target name to download from.
277    pattern: A regex pattern to match to artifacts filename
278    branch: The branch to download from
279  """
280  build_id = get_latest_build_id(
281      client=client, branch=branch, target=target)
282
283  fetch_artifacts(client, out_dir, target, pattern, build_id)
284