1#!/usr/bin/python 2 3# Copyright (c) 2013 The Chromium OS Authors. All rights reserved. 4# Use of this source code is governed by a BSD-style license that can be 5# found in the LICENSE file. 6import logging 7 8import common 9 10import httplib 11import httplib2 12from autotest_lib.server.cros.dynamic_suite import constants 13from chromite.lib import gdata_lib 14 15try: 16 from apiclient.discovery import build as apiclient_build 17 from apiclient import errors as apiclient_errors 18 from oauth2client import file as oauth_client_fileio 19except ImportError as e: 20 apiclient_build = None 21 logging.debug("API client for bug filing disabled. %s", e) 22 23 24class ProjectHostingApiException(Exception): 25 """ 26 Raised when an api call fails, since the actual 27 HTTP error can be cryptic. 28 """ 29 30 31class BaseIssue(gdata_lib.Issue): 32 """Base issue class with the minimum data to describe a tracker bug. 33 """ 34 def __init__(self, t_issue): 35 kwargs = {} 36 kwargs.update((keys, t_issue.get(keys)) 37 for keys in gdata_lib.Issue.SlotDefaults.keys()) 38 super(BaseIssue, self).__init__(**kwargs) 39 40 41class Issue(BaseIssue): 42 """ 43 Class representing an Issue and it's related metadata. 44 """ 45 def __init__(self, t_issue): 46 """ 47 Initialize |self| from tracker issue |t_issue| 48 49 @param t_issue: The base issue we want to use to populate 50 the member variables of this object. 51 52 @raises ProjectHostingApiException: If the tracker issue doesn't 53 contain all expected fields needed to create a complete issue. 54 """ 55 super(Issue, self).__init__(t_issue) 56 57 try: 58 # The value keyed under 'summary' in the tracker issue 59 # is, unfortunately, not the summary but the title. The 60 # actual summary is the update at index 0. 61 self.summary = t_issue.get('updates')[0] 62 self.comments = t_issue.get('updates')[1:] 63 64 # open or closed statuses are classified according to labels like 65 # unconfirmed, verified, fixed etc just like through the front end. 66 self.state = t_issue.get(constants.ISSUE_STATE) 67 self.merged_into = None 68 if (t_issue.get(constants.ISSUE_STATUS) 69 == constants.ISSUE_DUPLICATE and 70 constants.ISSUE_MERGEDINTO in t_issue): 71 parent_issue_dict = t_issue.get(constants.ISSUE_MERGEDINTO) 72 self.merged_into = parent_issue_dict.get('issueId') 73 except KeyError as e: 74 raise ProjectHostingApiException('Cannot create a ' 75 'complete issue %s, tracker issue: %s' % (e, t_issue)) 76 77 78class ProjectHostingApiClient(): 79 """ 80 Client class for interaction with the project hosting api. 81 """ 82 83 # Maximum number of results we would like when querying the tracker. 84 _max_results_for_issue = 50 85 _start_index = 1 86 87 88 def __init__(self, oauth_credentials, project_name, 89 monorail_server='staging'): 90 if apiclient_build is None: 91 raise ProjectHostingApiException('Cannot get apiclient library.') 92 93 if not oauth_credentials: 94 raise ProjectHostingApiException('No oauth_credentials is provided.') 95 96 # TODO(akeshet): This try-except is due to incompatibility of phapi_lib 97 # with oauth2client > 2. Until this is fixed, this is expected to fail 98 # and bug filing will be effectively disabled. crbug.com/648489 99 try: 100 storage = oauth_client_fileio.Storage(oauth_credentials) 101 credentials = storage.get() 102 except Exception as e: 103 raise ProjectHostingApiException('Incompaible credentials format, ' 104 'or other exception. Will not file ' 105 'bugs.') 106 if credentials is None or credentials.invalid: 107 raise ProjectHostingApiException('Invalid credentials for Project ' 108 'Hosting api. Cannot file bugs.') 109 110 http = credentials.authorize(httplib2.Http()) 111 try: 112 url = ('https://monorail-%s.appspot.com/_ah/api/discovery/v1/' 113 'apis/{api}/{apiVersion}/rest' % monorail_server) 114 self._codesite_service = apiclient_build( 115 "monorail", "v1", http=http, 116 discoveryServiceUrl=url) 117 except (apiclient_errors.Error, httplib2.HttpLib2Error, 118 httplib.BadStatusLine) as e: 119 raise ProjectHostingApiException(str(e)) 120 self._project_name = project_name 121 122 123 def _execute_request(self, request): 124 """ 125 Executes an api request. 126 127 @param request: An apiclient.http.HttpRequest object representing the 128 request to be executed. 129 @raises: ProjectHostingApiException if we fail to execute the request. 130 This could happen if we receive an http response that is not a 131 2xx, or if the http object itself encounters an error. 132 133 @return: A deserialized object model of the response body returned for 134 the request. 135 """ 136 try: 137 return request.execute() 138 except (apiclient_errors.Error, httplib2.HttpLib2Error, 139 httplib.BadStatusLine) as e: 140 msg = 'Unable to execute your request: %s' 141 raise ProjectHostingApiException(msg % e) 142 143 144 def _get_field(self, field): 145 """ 146 Gets a field from the project. 147 148 This method directly queries the project hosting API using bugdroids1's, 149 api key. 150 151 @param field: A selector, which corresponds loosely to a field in the 152 new bug description of the crosbug frontend. 153 @raises: ProjectHostingApiException, if the request execution fails. 154 155 @return: A json formatted python dict of the specified field's options, 156 or None if we can't find the api library. This dictionary 157 represents the javascript literal used by the front end tracker 158 and can hold multiple filds. 159 160 The returned dictionary follows a template, but it's structure 161 is only loosely defined as it needs to match whatever the front 162 end describes via javascript. 163 For a new issue interface which looks like: 164 165 field 1: text box 166 drop down: predefined value 1 = description 167 predefined value 2 = description 168 field 2: text box 169 similar structure as field 1 170 171 you will get a dictionary like: 172 { 173 'field name 1': { 174 'project realted config': 'config value' 175 'property': [ 176 {predefined value for property 1, description}, 177 {predefined value for property 2, description} 178 ] 179 }, 180 181 'field name 2': { 182 similar structure 183 } 184 ... 185 } 186 """ 187 project = self._codesite_service.projects() 188 request = project.get(projectId=self._project_name, 189 fields=field) 190 return self._execute_request(request) 191 192 193 def _list_updates(self, issue_id): 194 """ 195 Retrieve all updates for a given issue including comments, changes to 196 it's labels, status etc. The first element in the dictionary returned 197 by this method, is by default, the 0th update on the bug; which is the 198 entry that created it. All the text in a given update is keyed as 199 'content', and updates that contain no text, eg: a change to the status 200 of a bug, will contain the emtpy string instead. 201 202 @param issue_id: The id of the issue we want detailed information on. 203 @raises: ProjectHostingApiException, if the request execution fails. 204 205 @return: A json formatted python dict that has an entry for each update 206 performed on this issue. 207 """ 208 issue_comments = self._codesite_service.issues().comments() 209 request = issue_comments.list(projectId=self._project_name, 210 issueId=issue_id, 211 maxResults=self._max_results_for_issue) 212 return self._execute_request(request) 213 214 215 def _get_issue(self, issue_id): 216 """ 217 Gets an issue given it's id. 218 219 @param issue_id: A string representing the issue id. 220 @raises: ProjectHostingApiException, if failed to get the issue. 221 222 @return: A json formatted python dict that has the issue content. 223 """ 224 issues = self._codesite_service.issues() 225 try: 226 request = issues.get(projectId=self._project_name, 227 issueId=issue_id) 228 except TypeError as e: 229 raise ProjectHostingApiException( 230 'Unable to get issue %s from project %s: %s' % 231 (issue_id, self._project_name, str(e))) 232 return self._execute_request(request) 233 234 235 def set_max_results(self, max_results): 236 """Set the max results to return. 237 238 @param max_results: An integer representing the maximum number of 239 matching results to return per query. 240 """ 241 self._max_results_for_issue = max_results 242 243 244 def set_start_index(self, start_index): 245 """Set the start index, for paging. 246 247 @param start_index: The new start index to use. 248 """ 249 self._start_index = start_index 250 251 252 def list_issues(self, **kwargs): 253 """ 254 List issues containing the search marker. This method will only list 255 the summary, title and id of an issue, though it searches through the 256 comments. Eg: if we're searching for the marker '123', issues that 257 contain a comment of '123' will appear in the output, but the string 258 '123' itself may not, because the output only contains issue summaries. 259 260 @param kwargs: 261 q: The anchor string used in the search. 262 can: a string representing the search space that is passed to the 263 google api, can be 'all', 'new', 'open', 'owned', 'reported', 264 'starred', or 'to-verify', defaults to 'all'. 265 label: A string representing a single label to match. 266 267 @return: A json formatted python dict of all matching issues. 268 269 @raises: ProjectHostingApiException, if the request execution fails. 270 """ 271 issues = self._codesite_service.issues() 272 273 # Asking for issues with None or '' labels will restrict the query 274 # to those issues without labels. 275 if not kwargs['label']: 276 del kwargs['label'] 277 278 request = issues.list(projectId=self._project_name, 279 startIndex=self._start_index, 280 maxResults=self._max_results_for_issue, 281 **kwargs) 282 return self._execute_request(request) 283 284 285 def _get_property_values(self, prop_dict): 286 """ 287 Searches a dictionary as returned by _get_field for property lists, 288 then returns each value in the list. Effectively this gives us 289 all the accepted values for a property. For example, in crosbug, 290 'properties' map to things like Status, Labels, Owner etc, each of these 291 will have a list within the issuesConfig dict. 292 293 @param prop_dict: dictionary which contains a list of properties. 294 @yield: each value in a property list. This can be a dict or any other 295 type of datastructure, the caller is responsible for handling 296 it correctly. 297 """ 298 for name, property in prop_dict.iteritems(): 299 if isinstance(property, list): 300 for values in property: 301 yield values 302 303 304 def _get_cros_labels(self, prop_dict): 305 """ 306 Helper function to isolate labels from the labels dictionary. This 307 dictionary is of the form: 308 { 309 "label": "Cr-OS-foo", 310 "description": "description" 311 }, 312 And maps to the frontend like so: 313 Labels: Cr-??? 314 Cr-OS-foo = description 315 where Cr-OS-foo is a conveniently predefined value for Label Cr-OS-???. 316 317 @param prop_dict: a dictionary we expect the Cros label to be in. 318 @return: A lower case product area, eg: video, factory, ui. 319 """ 320 label = prop_dict.get('label') 321 if label and 'Cr-OS-' in label: 322 return label.split('Cr-OS-')[1] 323 324 325 def get_areas(self): 326 """ 327 Parse issue options and return a list of 'Cr-OS' labels. 328 329 @return: a list of Cr-OS labels from crosbug, eg: ['kernel', 'systems'] 330 """ 331 if apiclient_build is None: 332 logging.error('Missing Api-client import. Cannot get area-labels.') 333 return [] 334 335 try: 336 issue_options_dict = self._get_field('issuesConfig') 337 except ProjectHostingApiException as e: 338 logging.error('Unable to determine area labels: %s', str(e)) 339 return [] 340 341 # Since we can request multiple fields at once we need to 342 # retrieve each one from the field options dictionary, even if we're 343 # really only asking for one field. 344 issue_options = issue_options_dict.get('issuesConfig') 345 if issue_options is None: 346 logging.error('The IssueConfig field does not contain issue ' 347 'configuration as a member anymore; The project ' 348 'hosting api might have changed.') 349 return [] 350 351 return filter(None, [self._get_cros_labels(each) 352 for each in self._get_property_values(issue_options) 353 if isinstance(each, dict)]) 354 355 356 def create_issue(self, request_body): 357 """ 358 Convert the request body into an issue on the frontend tracker. 359 360 @param request_body: A python dictionary with key-value pairs 361 that represent the fields of the issue. 362 eg: { 363 'title': 'bug title', 364 'description': 'bug description', 365 'labels': ['Type-Bug'], 366 'owner': {'name': 'owner@'}, 367 'cc': [{'name': 'cc1'}, {'name': 'cc2'}], 368 'components': ["Internals->Components"] 369 } 370 Note the title and descriptions fields of a 371 new bug are not optional, all other fields are. 372 @raises: ProjectHostingApiException, if request execution fails. 373 374 @return: The response body, which will contain the metadata of the 375 issue created, or an error response code and information 376 about a failure. 377 """ 378 issues = self._codesite_service.issues() 379 request = issues.insert(projectId=self._project_name, sendEmail=True, 380 body=request_body) 381 return self._execute_request(request) 382 383 384 def update_issue(self, issue_id, request_body): 385 """ 386 Convert the request body into an update on an issue. 387 388 @param request_body: A python dictionary with key-value pairs 389 that represent the fields of the update. 390 eg: 391 { 392 'content': 'comment to add', 393 'updates': 394 { 395 'labels': ['Type-Bug', 'another label'], 396 'owner': 'owner@', 397 'cc': ['cc1@', cc2@'], 398 } 399 } 400 Note the owner and cc fields need to be email 401 addresses the tracker recognizes. 402 @param issue_id: The id of the issue to update. 403 @raises: ProjectHostingApiException, if request execution fails. 404 405 @return: The response body, which will contain information about the 406 update of said issue, or an error response code and information 407 about a failure. 408 """ 409 issues = self._codesite_service.issues() 410 request = issues.comments().insert(projectId=self._project_name, 411 issueId=issue_id, sendEmail=False, 412 body=request_body) 413 return self._execute_request(request) 414 415 416 def _populate_issue_updates(self, t_issue): 417 """ 418 Populates a tracker issue with updates. 419 420 Any issue is useless without it's updates, since the updates will 421 contain both the summary and the comments. We need at least one of 422 those to successfully dedupe. The Api doesn't allow us to grab all this 423 information in one shot because viewing the comments on an issue 424 requires more authority than just viewing it's title. 425 426 @param t_issue: The basic tracker issue, to populate with updates. 427 @raises: ProjectHostingApiException, if request execution fails. 428 429 @returns: A tracker issue, with it's updates. 430 """ 431 updates = self._list_updates(t_issue['id']) 432 t_issue['updates'] = [update['content'] for update in 433 self._get_property_values(updates) 434 if update.get('content')] 435 return t_issue 436 437 438 def get_tracker_issues_by_text(self, search_text, full_text=True, 439 include_dupes=False, label=None): 440 """ 441 Find all Tracker issues that contain the specified search text. 442 443 @param search_text: Anchor text to use in the search. 444 @param full_text: True if we would like an extensive search through 445 issue comments. If False the search will be restricted 446 to just summaries and titles. 447 @param include_dupes: If True, search over both open issues as well as 448 closed issues whose status is 'Duplicate'. If False, 449 only search over open issues. 450 @param label: A string representing a single label to match. 451 452 @return: A list of issues that contain the search text, or an empty list 453 when we're either unable to list issues or none match the text. 454 """ 455 issue_list = [] 456 try: 457 search_space = 'all' if include_dupes else 'open' 458 feed = self.list_issues(q=search_text, can=search_space, 459 label=label) 460 except ProjectHostingApiException as e: 461 logging.error('Unable to search for issues with marker %s: %s', 462 search_text, e) 463 return issue_list 464 465 for t_issue in self._get_property_values(feed): 466 state = t_issue.get(constants.ISSUE_STATE) 467 status = t_issue.get(constants.ISSUE_STATUS) 468 is_open_or_dup = (state == constants.ISSUE_OPEN or 469 (state == constants.ISSUE_CLOSED 470 and status == constants.ISSUE_DUPLICATE)) 471 # All valid issues will have an issue id we can use to retrieve 472 # more information about it. If we encounter a failure mode that 473 # returns a bad Http response code but doesn't throw an exception 474 # we won't find an issue id in the returned json. 475 if t_issue.get('id') and is_open_or_dup: 476 # TODO(beeps): If this method turns into a performance 477 # bottle neck yield each issue and refactor the reporter. 478 # For now passing all issues allows us to detect when 479 # deduping fails, because multiple issues will match a 480 # given query exactly. 481 try: 482 if full_text: 483 issue = Issue(self._populate_issue_updates(t_issue)) 484 else: 485 issue = BaseIssue(t_issue) 486 except ProjectHostingApiException as e: 487 logging.error('Unable to list the updates of issue %s: %s', 488 t_issue.get('id'), str(e)) 489 else: 490 issue_list.append(issue) 491 return issue_list 492 493 494 def get_tracker_issue_by_id(self, issue_id): 495 """ 496 Returns an issue object given the id. 497 498 @param issue_id: A string representing the issue id. 499 500 @return: An Issue object on success or None on failure. 501 """ 502 try: 503 t_issue = self._get_issue(issue_id) 504 return Issue(self._populate_issue_updates(t_issue)) 505 except ProjectHostingApiException as e: 506 logging.error('Creation of an Issue object for %s fails: %s', 507 issue_id, str(e)) 508 return None 509