1# Copyright 2015 The Chromium Authors. All rights reserved. 2# Use of this source code is governed by a BSD-style license that can be 3# found in the LICENSE file. 4 5"""Defines common functionality used for interacting with Rietveld.""" 6 7import json 8import logging 9import mimetypes 10import urllib 11 12from google.appengine.ext import ndb 13 14from dashboard import utils 15 16_DESCRIPTION = """This patch was automatically uploaded by the Chrome Perf 17Dashboard (https://chromeperf.appspot.com). It is being used to run a perf 18bisect try job. It should not be submitted.""" 19 20 21class ResponseObject(object): 22 """Class for Response Object. 23 24 This class holds attributes similar to response object returned by 25 google.appengine.api.urlfetch. This is used to convert response object 26 returned by httplib2.Http.request. 27 """ 28 29 def __init__(self, status_code, content): 30 self.status_code = int(status_code) 31 self.content = content 32 33 34class RietveldConfig(ndb.Model): 35 """Configuration info for a Rietveld service account. 36 37 The data is stored only in the App Engine datastore (and the cloud console) 38 and not the code because it contains sensitive information like private keys. 39 """ 40 # TODO(qyearsley): Remove RietveldConfig and store the server URL in 41 # datastore. 42 client_email = ndb.TextProperty() 43 service_account_key = ndb.TextProperty() 44 45 # The protocol and domain of the Rietveld host. Should not contain path. 46 server_url = ndb.TextProperty() 47 48 # The protocol and domain of the Internal Rietveld host which is used 49 # to create issues for internal only tests. 50 internal_server_url = ndb.TextProperty() 51 52 53def GetDefaultRietveldConfig(): 54 """Returns the default rietveld config entity from the datastore.""" 55 return ndb.Key(RietveldConfig, 'default_rietveld_config').get() 56 57 58class RietveldService(object): 59 """Implements a Python API to Rietveld via HTTP. 60 61 Authentication is handled via an OAuth2 access token minted from an RSA key 62 associated with a service account (which can be created via the Google API 63 console). For this to work, the Rietveld instance to talk to must be 64 configured to allow the service account client ID as OAuth2 audience (see 65 Rietveld source). Both the RSA key and the server URL are provided via static 66 application configuration. 67 """ 68 69 def __init__(self, internal_only=False): 70 self.internal_only = internal_only 71 self._config = None 72 self._http = None 73 74 def Config(self): 75 if not self._config: 76 self._config = GetDefaultRietveldConfig() 77 return self._config 78 79 def MakeRequest(self, path, *args, **kwargs): 80 """Makes a request to the Rietveld server.""" 81 if self.internal_only: 82 server_url = self.Config().internal_server_url 83 else: 84 server_url = self.Config().server_url 85 url = '%s/%s' % (server_url, path) 86 response, content = utils.ServiceAccountHttp().request(url, *args, **kwargs) 87 return ResponseObject(response.get('status'), content) 88 89 def _XsrfToken(self): 90 """Requests a XSRF token from Rietveld.""" 91 return self.MakeRequest( 92 'xsrf_token', headers={'X-Requesting-XSRF-Token': 1}).content 93 94 def _EncodeMultipartFormData(self, fields, files): 95 """Encode form fields for multipart/form-data. 96 97 Args: 98 fields: A sequence of (name, value) elements for regular form fields. 99 files: A sequence of (name, filename, value) elements for data to be 100 uploaded as files. 101 Returns: 102 (content_type, body) ready for httplib.HTTP instance. 103 104 Source: 105 http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/146306 106 """ 107 boundary = '-M-A-G-I-C---B-O-U-N-D-A-R-Y-' 108 crlf = '\r\n' 109 lines = [] 110 for (key, value) in fields: 111 lines.append('--' + boundary) 112 lines.append('Content-Disposition: form-data; name="%s"' % key) 113 lines.append('') 114 if isinstance(value, unicode): 115 value = value.encode('utf-8') 116 lines.append(value) 117 for (key, filename, value) in files: 118 lines.append('--' + boundary) 119 lines.append('Content-Disposition: form-data; name="%s"; filename="%s"' % 120 (key, filename)) 121 content_type = (mimetypes.guess_type(filename)[0] or 122 'application/octet-stream') 123 lines.append('Content-Type: %s' % content_type) 124 lines.append('') 125 if isinstance(value, unicode): 126 value = value.encode('utf-8') 127 lines.append(value) 128 lines.append('--' + boundary + '--') 129 lines.append('') 130 body = crlf.join(lines) 131 content_type = 'multipart/form-data; boundary=%s' % boundary 132 return content_type, body 133 134 def UploadPatch(self, subject, patch, base_checksum, base_hashes, 135 base_content, config_path): 136 """Uploads the given patch file contents to Rietveld. 137 138 The process of creating an issue and uploading the patch requires several 139 HTTP requests to Rietveld. 140 141 Rietveld API documentation: https://code.google.com/p/rietveld/wiki/APIs 142 For specific implementation in Rietveld codebase, see http://goo.gl/BW205J. 143 144 Args: 145 subject: Title of the job, as it will appear in rietveld. 146 patch: The patch, which is a specially-formatted string. 147 base_checksum: Base md5 checksum to send. 148 base_hashes: "Base hashes" string to send. 149 base_content: Base config file contents. 150 config_path: Path to the config file. 151 152 Returns: 153 A (issue ID, patchset ID) pair. These are strings that contain numerical 154 IDs. If the patch upload was unsuccessful, then (None, None) is returned. 155 """ 156 base = 'https://chromium.googlesource.com/chromium/src.git@master' 157 repo_guid = 'c14d891d44f0afff64e56ed7c9702df1d807b1ee' 158 form_fields = [ 159 ('subject', subject), 160 ('description', _DESCRIPTION), 161 ('base', base), 162 ('xsrf_token', self._XsrfToken()), 163 ('repo_guid', repo_guid), 164 ('content_upload', '1'), 165 ('base_hashes', base_hashes), 166 ] 167 uploaded_diff_file = [('data', 'data.diff', patch)] 168 ctype, body = self._EncodeMultipartFormData( 169 form_fields, uploaded_diff_file) 170 response = self.MakeRequest( 171 'upload', method='POST', body=body, headers={'content-type': ctype}) 172 if response.status_code != 200: 173 logging.error('Error %s uploading to /upload', response.status_code) 174 logging.error(response.content) 175 return (None, None) 176 177 # There should always be 3 lines in the request, but sometimes Rietveld 178 # returns 2 lines. Log the content so we can debug further. 179 logging.info('Response from Rietveld /upload:\n%s', response.content) 180 if not response.content.startswith('Issue created.'): 181 logging.error('Unexpected response: %s', response.content) 182 return (None, None) 183 lines = response.content.splitlines() 184 if len(lines) < 2: 185 logging.error('Unexpected response %s', response.content) 186 return (None, None) 187 188 msg = lines[0] 189 issue_id = msg[msg.rfind('/') + 1:] 190 patchset_id = lines[1].strip() 191 patches = [x.split(' ', 1) for x in lines[2:]] 192 request_path = '%d/upload_content/%d/%d' % ( 193 int(issue_id), int(patchset_id), int(patches[0][0])) 194 form_fields = [ 195 ('filename', config_path), 196 ('status', 'M'), 197 ('checksum', base_checksum), 198 ('is_binary', str(False)), 199 ('is_current', str(False)), 200 ] 201 uploaded_diff_file = [('data', config_path, base_content)] 202 ctype, body = self._EncodeMultipartFormData(form_fields, uploaded_diff_file) 203 response = self.MakeRequest( 204 request_path, method='POST', body=body, headers={'content-type': ctype}) 205 if response.status_code != 200: 206 logging.error( 207 'Error %s uploading to %s', response.status_code, request_path) 208 logging.error(response.content) 209 return (None, None) 210 211 request_path = '%s/upload_complete/%s' % (issue_id, patchset_id) 212 response = self.MakeRequest(request_path, method='POST') 213 if response.status_code != 200: 214 logging.error( 215 'Error %s uploading to %s', response.status_code, request_path) 216 logging.error(response.content) 217 return (None, None) 218 return issue_id, patchset_id 219 220 def TryPatch(self, tryserver_master, issue_id, patchset_id, bot): 221 """Sends a request to try the given patchset on the given trybot. 222 223 To see exactly how this request is handled, you can see the try_patchset 224 handler in the Chromium branch of Rietveld: http://goo.gl/U6tJQZ 225 226 Args: 227 tryserver_master: Master name, e.g. "tryserver.chromium.perf". 228 issue_id: Rietveld issue ID. 229 patchset_id: Patchset ID (returned when a patch is uploaded). 230 bot: Bisect bot name. 231 232 Returns: 233 True if successful, False otherwise. 234 """ 235 args = { 236 'xsrf_token': self._XsrfToken(), 237 'builders': json.dumps({bot: ['defaulttests']}), 238 'master': tryserver_master, 239 'reason': 'Perf bisect', 240 'clobber': 'False', 241 } 242 request_path = '%s/try/%s' % (issue_id, patchset_id) 243 response = self.MakeRequest( 244 request_path, method='POST', body=urllib.urlencode(args)) 245 if response.status_code != 200: 246 logging.error( 247 'Error %s POSTing to /%s/try/%s', response.status_code, issue_id, 248 patchset_id) 249 logging.error(response.content) 250 return False 251 return True 252