• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1# Copyright 2015 The Chromium Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5"""Defines common functionality used for interacting with Rietveld."""
6
7import json
8import logging
9import mimetypes
10import urllib
11
12from google.appengine.ext import ndb
13
14from dashboard import utils
15
16_DESCRIPTION = """This patch was automatically uploaded by the Chrome Perf
17Dashboard (https://chromeperf.appspot.com). It is being used to run a perf
18bisect try job. It should not be submitted."""
19
20
21class ResponseObject(object):
22  """Class for Response Object.
23
24  This class holds attributes similar to response object returned by
25  google.appengine.api.urlfetch. This is used to convert response object
26  returned by httplib2.Http.request.
27  """
28
29  def __init__(self, status_code, content):
30    self.status_code = int(status_code)
31    self.content = content
32
33
34class RietveldConfig(ndb.Model):
35  """Configuration info for a Rietveld service account.
36
37  The data is stored only in the App Engine datastore (and the cloud console)
38  and not the code because it contains sensitive information like private keys.
39  """
40  # TODO(qyearsley): Remove RietveldConfig and store the server URL in
41  # datastore.
42  client_email = ndb.TextProperty()
43  service_account_key = ndb.TextProperty()
44
45  # The protocol and domain of the Rietveld host. Should not contain path.
46  server_url = ndb.TextProperty()
47
48  # The protocol and domain of the Internal Rietveld host which is used
49  # to create issues for internal only tests.
50  internal_server_url = ndb.TextProperty()
51
52
53def GetDefaultRietveldConfig():
54  """Returns the default rietveld config entity from the datastore."""
55  return ndb.Key(RietveldConfig, 'default_rietveld_config').get()
56
57
58class RietveldService(object):
59  """Implements a Python API to Rietveld via HTTP.
60
61  Authentication is handled via an OAuth2 access token minted from an RSA key
62  associated with a service account (which can be created via the Google API
63  console). For this to work, the Rietveld instance to talk to must be
64  configured to allow the service account client ID as OAuth2 audience (see
65  Rietveld source). Both the RSA key and the server URL are provided via static
66  application configuration.
67  """
68
69  def __init__(self, internal_only=False):
70    self.internal_only = internal_only
71    self._config = None
72    self._http = None
73
74  def Config(self):
75    if not self._config:
76      self._config = GetDefaultRietveldConfig()
77    return self._config
78
79  def MakeRequest(self, path, *args, **kwargs):
80    """Makes a request to the Rietveld server."""
81    if self.internal_only:
82      server_url = self.Config().internal_server_url
83    else:
84      server_url = self.Config().server_url
85    url = '%s/%s' % (server_url, path)
86    response, content = utils.ServiceAccountHttp().request(url, *args, **kwargs)
87    return ResponseObject(response.get('status'), content)
88
89  def _XsrfToken(self):
90    """Requests a XSRF token from Rietveld."""
91    return self.MakeRequest(
92        'xsrf_token', headers={'X-Requesting-XSRF-Token': 1}).content
93
94  def _EncodeMultipartFormData(self, fields, files):
95    """Encode form fields for multipart/form-data.
96
97    Args:
98      fields: A sequence of (name, value) elements for regular form fields.
99      files: A sequence of (name, filename, value) elements for data to be
100             uploaded as files.
101    Returns:
102      (content_type, body) ready for httplib.HTTP instance.
103
104    Source:
105      http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/146306
106    """
107    boundary = '-M-A-G-I-C---B-O-U-N-D-A-R-Y-'
108    crlf = '\r\n'
109    lines = []
110    for (key, value) in fields:
111      lines.append('--' + boundary)
112      lines.append('Content-Disposition: form-data; name="%s"' % key)
113      lines.append('')
114      if isinstance(value, unicode):
115        value = value.encode('utf-8')
116      lines.append(value)
117    for (key, filename, value) in files:
118      lines.append('--' + boundary)
119      lines.append('Content-Disposition: form-data; name="%s"; filename="%s"' %
120                   (key, filename))
121      content_type = (mimetypes.guess_type(filename)[0] or
122                      'application/octet-stream')
123      lines.append('Content-Type: %s' % content_type)
124      lines.append('')
125      if isinstance(value, unicode):
126        value = value.encode('utf-8')
127      lines.append(value)
128    lines.append('--' + boundary + '--')
129    lines.append('')
130    body = crlf.join(lines)
131    content_type = 'multipart/form-data; boundary=%s' % boundary
132    return content_type, body
133
134  def UploadPatch(self, subject, patch, base_checksum, base_hashes,
135                  base_content, config_path):
136    """Uploads the given patch file contents to Rietveld.
137
138    The process of creating an issue and uploading the patch requires several
139    HTTP requests to Rietveld.
140
141    Rietveld API documentation: https://code.google.com/p/rietveld/wiki/APIs
142    For specific implementation in Rietveld codebase, see http://goo.gl/BW205J.
143
144    Args:
145      subject: Title of the job, as it will appear in rietveld.
146      patch: The patch, which is a specially-formatted string.
147      base_checksum: Base md5 checksum to send.
148      base_hashes: "Base hashes" string to send.
149      base_content: Base config file contents.
150      config_path: Path to the config file.
151
152    Returns:
153      A (issue ID, patchset ID) pair. These are strings that contain numerical
154      IDs. If the patch upload was unsuccessful, then (None, None) is returned.
155    """
156    base = 'https://chromium.googlesource.com/chromium/src.git@master'
157    repo_guid = 'c14d891d44f0afff64e56ed7c9702df1d807b1ee'
158    form_fields = [
159        ('subject', subject),
160        ('description', _DESCRIPTION),
161        ('base', base),
162        ('xsrf_token', self._XsrfToken()),
163        ('repo_guid', repo_guid),
164        ('content_upload', '1'),
165        ('base_hashes', base_hashes),
166    ]
167    uploaded_diff_file = [('data', 'data.diff', patch)]
168    ctype, body = self._EncodeMultipartFormData(
169        form_fields, uploaded_diff_file)
170    response = self.MakeRequest(
171        'upload', method='POST', body=body, headers={'content-type': ctype})
172    if response.status_code != 200:
173      logging.error('Error %s uploading to /upload', response.status_code)
174      logging.error(response.content)
175      return (None, None)
176
177    # There should always be 3 lines in the request, but sometimes Rietveld
178    # returns 2 lines. Log the content so we can debug further.
179    logging.info('Response from Rietveld /upload:\n%s', response.content)
180    if not response.content.startswith('Issue created.'):
181      logging.error('Unexpected response: %s', response.content)
182      return (None, None)
183    lines = response.content.splitlines()
184    if len(lines) < 2:
185      logging.error('Unexpected response %s', response.content)
186      return (None, None)
187
188    msg = lines[0]
189    issue_id = msg[msg.rfind('/') + 1:]
190    patchset_id = lines[1].strip()
191    patches = [x.split(' ', 1) for x in lines[2:]]
192    request_path = '%d/upload_content/%d/%d' % (
193        int(issue_id), int(patchset_id), int(patches[0][0]))
194    form_fields = [
195        ('filename', config_path),
196        ('status', 'M'),
197        ('checksum', base_checksum),
198        ('is_binary', str(False)),
199        ('is_current', str(False)),
200    ]
201    uploaded_diff_file = [('data', config_path, base_content)]
202    ctype, body = self._EncodeMultipartFormData(form_fields, uploaded_diff_file)
203    response = self.MakeRequest(
204        request_path, method='POST', body=body, headers={'content-type': ctype})
205    if response.status_code != 200:
206      logging.error(
207          'Error %s uploading to %s', response.status_code, request_path)
208      logging.error(response.content)
209      return (None, None)
210
211    request_path = '%s/upload_complete/%s' % (issue_id, patchset_id)
212    response = self.MakeRequest(request_path, method='POST')
213    if response.status_code != 200:
214      logging.error(
215          'Error %s uploading to %s', response.status_code, request_path)
216      logging.error(response.content)
217      return (None, None)
218    return issue_id, patchset_id
219
220  def TryPatch(self, tryserver_master, issue_id, patchset_id, bot):
221    """Sends a request to try the given patchset on the given trybot.
222
223    To see exactly how this request is handled, you can see the try_patchset
224    handler in the Chromium branch of Rietveld: http://goo.gl/U6tJQZ
225
226    Args:
227      tryserver_master: Master name, e.g. "tryserver.chromium.perf".
228      issue_id: Rietveld issue ID.
229      patchset_id: Patchset ID (returned when a patch is uploaded).
230      bot: Bisect bot name.
231
232    Returns:
233      True if successful, False otherwise.
234    """
235    args = {
236        'xsrf_token': self._XsrfToken(),
237        'builders': json.dumps({bot: ['defaulttests']}),
238        'master': tryserver_master,
239        'reason': 'Perf bisect',
240        'clobber': 'False',
241    }
242    request_path = '%s/try/%s' % (issue_id, patchset_id)
243    response = self.MakeRequest(
244        request_path, method='POST', body=urllib.urlencode(args))
245    if response.status_code != 200:
246      logging.error(
247          'Error %s POSTing to /%s/try/%s', response.status_code, issue_id,
248          patchset_id)
249      logging.error(response.content)
250      return False
251    return True
252