1#pylint: disable-msg=W0611 2# Copyright (c) 2012 The Chromium OS Authors. All rights reserved. 3# Use of this source code is governed by a BSD-style license that can be 4# found in the LICENSE file. 5 6import cgi 7import collections 8import HTMLParser 9import logging 10import re 11import textwrap 12 13from xml.parsers import expat 14 15import common 16 17from autotest_lib.client.common_lib import error 18from autotest_lib.client.common_lib import global_config 19from autotest_lib.server import afe_urls 20from autotest_lib.server import site_utils 21from autotest_lib.server.cros.dynamic_suite import constants 22from autotest_lib.server.cros.dynamic_suite import job_status 23from autotest_lib.server.cros.dynamic_suite import reporting_utils 24from autotest_lib.server.cros.dynamic_suite import tools 25from autotest_lib.site_utils import gmail_lib 26 27try: 28 from chromite.lib import metrics 29except ImportError: 30 metrics = site_utils.metrics_mock 31 32 33# Try importing the essential bug reporting libraries. 34try: 35 from autotest_lib.site_utils import phapi_lib 36except ImportError, e: 37 fundamental_libs = False 38 logging.debug('Bug filing disabled. %s', e) 39else: 40 fundamental_libs = True 41 42BUG_CONFIG_SECTION = 'BUG_REPORTING' 43 44CHROMIUM_EMAIL_ADDRESS = global_config.global_config.get_config_value( 45 BUG_CONFIG_SECTION, 'chromium_email_address', default='') 46EMAIL_CREDS_FILE = global_config.global_config.get_config_value( 47 'NOTIFICATIONS', 'gmail_api_credentials_test_failure', default=None) 48 49 50class Bug(object): 51 """Holds the minimum information needed to make a dedupable bug report.""" 52 53 def __init__(self, title, summary, search_marker=None, labels=None, 54 owner='', cc=None, components=None): 55 """ 56 Initializes Bug object. 57 58 @param title: The title of the bug. 59 @param summary: The summary of the bug. 60 @param search_marker: The string used to determine if a bug is a 61 duplicate report or not. All Bugs with the same 62 search_marker are considered to be for the same 63 bug. Make this None if you do not want to dedupe. 64 @param labels: The labels that the filed bug will have. 65 @param owner: The owner/asignee of this bug. Typically left blank. 66 @param cc: Who to cc'd for this bug. 67 @param components: The components that the filed bug will have. 68 """ 69 self._title = title 70 self._summary = summary 71 self._search_marker = search_marker 72 self.owner = owner 73 74 self.labels = labels if labels is not None else [] 75 self.components = components if components is not None else [] 76 self.cc = cc if cc is not None else [] 77 78 79 def title(self): 80 """Combines information about this bug into a title string.""" 81 return self._title 82 83 84 def summary(self): 85 """Combines information about this bug into a summary string.""" 86 return self._summary 87 88 89 def search_marker(self): 90 """Return an Anchor that we can use to dedupe this exact bug.""" 91 return self._search_marker 92 93 94class TestBug(Bug): 95 """ 96 Wrap up all information needed to make an intelligent report about an 97 issue. Each TestBug has a search marker associated with it that can be 98 used to find similar reports. 99 """ 100 101 def __init__(self, build, chrome_version, suite, result): 102 """ 103 @param build: The build type, of the form <board>/<milestone>-<release>. 104 eg: x86-mario-release/R25-4321.0.0 105 @param chrome_version: The chrome version associated with the build. 106 eg: 28.0.1498.1 107 @param suite: The name of the suite that this test run is a part of. 108 @param result: The status of the job associated with this issue. 109 This contains the status, job id, test name, hostname 110 and reason for issue. 111 """ 112 self.build = build 113 self.chrome_version = chrome_version 114 self.suite = suite 115 self.name = tools.get_test_name(build, suite, result.test_name) 116 self.reason = result.reason 117 # The result_owner is used to find results and logs. 118 self.result_owner = result.owner 119 self.hostname = result.hostname 120 self.job_id = result.id 121 122 # Aborts, server/client job failures or a test failure without a 123 # reason field need lab attention. Lab bugs for the aborted case 124 # are disabled till crbug.com/188217 is resolved. 125 self.lab_error = job_status.is_for_infrastructure_fail(result) 126 127 # The owner is who the bug is assigned to. 128 self.owner = '' 129 self.cc = [] 130 self.components = [] 131 132 if result.is_warn(): 133 self.labels = ['Test-Warning'] 134 self.status = 'Warning' 135 else: 136 self.labels = [] 137 self.status = 'Failure' 138 139 140 def title(self): 141 """Combines information about this bug into a title string.""" 142 return '[%s] %s %s on %s' % (self.suite, self.name, 143 self.status, self.build) 144 145 146 def summary(self): 147 """Combines information about this bug into a summary string.""" 148 149 links = self._get_links_for_failure() 150 template = ('This report is automatically generated to track the ' 151 'following %(status)s:\n' 152 'Test: %(test)s.\n' 153 'Suite: %(suite)s.\n' 154 'Chrome Version: %(chrome_version)s.\n' 155 'Build: %(build)s.\n\nReason:\n%(reason)s.\n' 156 'build artifacts: %(build_artifacts)s.\n' 157 'results log: %(results_log)s.\n' 158 'status log: %(status_log)s.\n' 159 'buildbot stages: %(buildbot_stages)s.\n' 160 'job link: %(job)s.\n\n' 161 'You may want to check the test history on wmatrix: ' 162 '%(test_history_url)s\n' 163 'You may also want to check the test retry dashboard in ' 164 'case this is a flakey test: %(retry_url)s\n') 165 166 specifics = { 167 'status': self.status, 168 'test': self.name, 169 'suite': self.suite, 170 'build': self.build, 171 'chrome_version': self.chrome_version, 172 'reason': self.reason, 173 'build_artifacts': links.artifacts, 174 'results_log': links.results, 175 'status_log': links.status_log, 176 'buildbot_stages': links.buildbot, 177 'job': links.job, 178 'test_history_url': links.test_history_url, 179 'retry_url': links.retry_url, 180 } 181 182 return template % specifics 183 184 185 # TO-DO(shuqianz) Fix the dedupe failing issue because reason contains 186 # special characters after 187 # https://bugs.chromium.org/p/monorail/issues/detail?id=806 being fixed. 188 def search_marker(self): 189 """Return an Anchor that we can use to dedupe this exact bug.""" 190 board = '' 191 try: 192 board = site_utils.ParseBuildName(self.build)[0] 193 except site_utils.ParseBuildNameException as e: 194 logging.error(str(e)) 195 196 # Substitute the board name for a placeholder. We try both build and 197 # release board name variants. 198 reason = self.reason 199 if board: 200 for b in (board, board.replace('_', '-')): 201 reason = reason.replace(b, 'BOARD_PLACEHOLDER') 202 203 return "%s{%s,%s,%s}" % ('Test%s' % self.status, self.suite, 204 self.name, reason) 205 206 207 def _get_links_for_failure(self): 208 """Returns a named tuple of links related to this failure.""" 209 links = collections.namedtuple('links', ('results,' 210 'status_log,' 211 'artifacts,' 212 'buildbot,' 213 'job,' 214 'test_history_url,' 215 'retry_url')) 216 return links(reporting_utils.link_result_logs( 217 self.job_id, self.result_owner, self.hostname), 218 reporting_utils.link_status_log( 219 self.job_id, self.result_owner, self.hostname), 220 reporting_utils.link_build_artifacts(self.build), 221 reporting_utils.link_buildbot_stages(self.build), 222 reporting_utils.link_job(self.job_id), 223 reporting_utils.link_test_history(self.name), 224 reporting_utils.link_retry_url(self.name)) 225 226 227class MachineKillerBug(Bug): 228 """Wrap up information needed to report a test killing a machine.""" 229 230 # Label used by the bug-filer to categorize machine killers 231 _MACHINE_KILLER_LABEL = 'machine-killer' 232 # Address to which this bug will be cc'd 233 _CC_ADDRESS = global_config.global_config.get_config_value( 234 'SCHEDULER', 'notify_email_errors', default='') 235 236 237 def __init__(self, job_id, job_name, machine): 238 """Initialize MachineKillerBug. 239 240 @param job_id: The id of the job, this should be an afe job id. 241 @param job_name: the name of the job 242 @param machine: The hostname of a machine that has been put 243 in Repair Failed by the job. 244 245 """ 246 # Name of test job may contain information like build and suite. 247 # e.g. lumpy-release/R31-1234.0.0/bvt/dummy_Pass_SERVER_JOB 248 # Try to split job_name with '/' and use the last part 249 # as test name. Note this assumes test name must not contains '/'. 250 self._test_name = job_name.rsplit('/', 1)[-1] 251 self._job_id = job_id 252 self._machine = machine 253 self.owner='' 254 self.cc=[self._CC_ADDRESS] 255 self.labels=[self._MACHINE_KILLER_LABEL] 256 self.components = [] 257 258 259 def title(self): 260 return ('%s suspected of putting machines in Repair Failed state.' 261 % self._test_name) 262 263 def summary(self): 264 """Combines information about this bug into a summary string.""" 265 266 template = ('This bug has been automatically filed to track the ' 267 'following issue:\n\n' 268 'Test: %(test)s.\n' 269 'Machine: %(machine)s.\n' 270 'Issue: It is suspected that the test has put the ' 271 'machine in the Repair Failed State.\n' 272 'Suggested Actions: Investigate to determine if this ' 273 'test is at fault and then either fix or disable the ' 274 'test if appropriate.\n' 275 'Job link: %(job)s.\n') 276 disclaimer = ('\n\nNote that the autofiled count on this bug indicates ' 277 'the number of times we have attempted to repair the ' 278 'machine, not the number of times it has gone into ' 279 'the repair failed state.\n') 280 specifics = { 281 'test': self._test_name, 282 'machine': self._machine, 283 'job': reporting_utils.link_job(self._job_id), 284 } 285 return template % specifics + disclaimer 286 287 288 def search_marker(self): 289 """Returns an Anchor that we can use to dedupe this bug.""" 290 return 'MachineKiller{%s}' % self._test_name 291 292 293class PoolHealthBug(Bug): 294 """Report information about a critical pool of DUTs in the lab.""" 295 296 _POOL_HEALTH_LABELS = global_config.global_config.get_config_value( 297 'BUG_REPORTING', 'pool_health_labels', type=list, default=[]) 298 _POOL_HEALTH_COMPONENTS = global_config.global_config.get_config_value( 299 'BUG_REPORTING', 'pool_health_components', type=list, default=[]) 300 _CC_ADDRESS = global_config.global_config.get_config_value( 301 'BUG_REPORTING', 'pool_health_cc', type=list, default=[]) 302 _SUMMARY_TEMPLATE = textwrap.dedent("""\ 303 This bug has been automatically filed to track the following issue: 304 305 Not enough DUTs available. 306 Pool: {this._pool} 307 Board: {this._board} 308 DUTs needed: {this._num_required} 309 DUTs available: {this._num_available} 310 Suite: {this._suite_name} 311 Build: {this._build} 312 313 Hosts: 314 315 {host_summaries} 316 """) 317 _HOST_TEMPLATE = '{host.hostname} {locked_status} {host.status} {afe_link}' 318 319 def __init__(self, exception): 320 """Initialize a PoolHealthBug. 321 322 @param exception: NotEnoughDutsError with context information. 323 @param hosts: An Iterable of all Hosts with the 324 board, in the given pool. 325 """ 326 self._exception = exception 327 self._board = exception.board 328 self._pool = exception.pool 329 self._num_available = exception.num_available 330 self._num_required = exception.num_required 331 self._bug_id = exception.bug_id 332 self._hosts = exception.hosts 333 self._suite_name = exception.suite_name 334 self._build = exception.build 335 336 self.owner = '' 337 self.cc = self._CC_ADDRESS 338 self.labels = self._POOL_HEALTH_LABELS 339 self.components = self._POOL_HEALTH_COMPONENTS 340 341 342 def title(self): 343 return ('pool: %s, board: %s in critical state' % 344 (self._pool, self._board)) 345 346 347 def summary(self): 348 """Combines information about this bug into a summary string.""" 349 return self._SUMMARY_TEMPLATE.format( 350 this=self, 351 host_summaries='\n'.join(self._make_host_summaries())) 352 353 354 def _make_host_summaries(self): 355 """Yield hosts summary strings.""" 356 host_template = self._HOST_TEMPLATE 357 for host in self._hosts: 358 yield host_template.format( 359 host=host, 360 locked_status='Locked' if host.locked else 'Unlocked', 361 afe_link=afe_urls.get_host_url(host.id)) 362 363 364 def search_marker(self): 365 """Returns an Anchor that we can use to dedupe this bug.""" 366 return 'PoolHealthBug{%s, %s}' % (self._pool, self._board) 367 368 369class SuiteSchedulerBug(Bug): 370 """Bug filed for suite scheduler.""" 371 372 _SUITE_SCHEDULER_LABELS = ['Build-HardwareLab', 'Pri-1', 'suite_scheduler'] 373 374 def __init__(self, suite, build, board, control_file_exception): 375 self._suite = suite 376 self._build = build 377 self._board = board 378 self._exception = control_file_exception 379 # TODO(fdeng): fix get_sheriffs crbug.com/483254 380 lab_deputies = site_utils.get_sheriffs(lab_only=True) 381 self.owner = lab_deputies[0] if lab_deputies else '' 382 self.labels = self._SUITE_SCHEDULER_LABELS 383 self.cc = lab_deputies[1:] if lab_deputies else [] 384 self.components = [] 385 386 387 def title(self): 388 """Return Title of the bug""" 389 if isinstance(self._exception, error.ControlFileNotFound): 390 t = 'Missing control file' 391 else: 392 t = 'Problem with getting control file' 393 return '[suite scheduler] %s for suite: "%s", build: %s' % ( 394 t, self._suite, self._build) 395 396 397 def summary(self): 398 """Combines information about this bug into a summary string.""" 399 template = ('Suite scheduler could not schedule suite due to ' 400 'a control file problem:\n\n' 401 'Suite:\t%(suite)s\n' 402 'Build:\t%(build)s\n' 403 'Board:\t%(board)s (The problem may happen for other ' 404 'boards as well, only the first board is reported.)\n' 405 'Diagnose:\n%(diagnose)s\n') 406 407 if isinstance(self._exception, error.ControlFileNotFound): 408 diagnose = ( 409 '\tThe suite\'s control file does not exist in the build.\n' 410 '\tDo you expect the suite to run for the said build?\n' 411 '\t- If yes, please add/backport the control file to ' 412 'the build,\n' 413 '\t- If not, please fix the entry for this suite in ' 414 'suite_scheduler.ini so that it specifies the ' 415 'right builds to run;\n' 416 '\t and request a push to prod.') 417 else: 418 diagnose = ('\tNo suggestion. Please ask infra deputy ' 419 'to triage.\n%s\n') % str(self._exception) 420 specifics = {'suite': self._suite, 421 'build': self._build, 422 'board': self._board, 423 'error': type(self._exception), 424 'diagnose': diagnose,} 425 return template % specifics 426 427 428 def search_marker(self): 429 """Returns an Anchor that we can use to dedupe this bug.""" 430 # TODO(fdeng): flaky deduping behavior, see crbug.com/486895 431 return 'SuiteSchedulerBug{%s, %s}' % ( 432 self._suite, type(self._exception).__name__) 433 434 435ReportResult = collections.namedtuple('ReportResult', ['bug_id', 'update_count']) 436 437 438class Reporter(object): 439 """ 440 Files external reports about bugs that happened inside autotest. 441 """ 442 # Credentials for access to the project hosting api 443 _project_name = global_config.global_config.get_config_value( 444 BUG_CONFIG_SECTION, 'project_name', default='') 445 _oauth_credentials = global_config.global_config.get_config_value( 446 BUG_CONFIG_SECTION, 'credentials', default='') 447 _monorail_server= global_config.global_config.get_config_value( 448 BUG_CONFIG_SECTION, 'monorail_server', default='staging') 449 450 # AUTOFILED_COUNT is a label prefix used to indicate how 451 # many times we think we've updated an issue automatically. 452 AUTOFILED_COUNT = 'autofiled-count-' 453 _PREDEFINED_LABELS = ['autofiled', '%s%d' % (AUTOFILED_COUNT, 1), 454 'OS-Chrome', 'Type-Bug', 455 'Restrict-View-Google'] 456 457 _SEARCH_MARKER = 'ANCHOR ' 458 459 460 @classmethod 461 def _get_creds_abspath(cls): 462 """Returns the abspath of the bug filer credentials file. 463 464 @return: A path to the oauth2 credentials file. 465 """ 466 return site_utils.get_creds_abspath(cls._oauth_credentials) 467 468 469 def __init__(self): 470 if not fundamental_libs: 471 logging.warning("Bug filing disabled due to missing imports.") 472 return 473 try: 474 self._phapi_client = phapi_lib.ProjectHostingApiClient( 475 self._get_creds_abspath(), self._project_name, 476 self._monorail_server) 477 except phapi_lib.ProjectHostingApiException as e: 478 logging.error('Unable to create project hosting api client: %s', e) 479 self._phapi_client = None 480 481 482 def _check_tracker(self): 483 """Returns True if we have a tracker object to use for filing bugs.""" 484 return fundamental_libs and self._phapi_client 485 486 487 def get_bug_tracker_client(self): 488 """Returns the client used to communicate with the project hosting api. 489 490 @return: The instance of the ProjectHostingApiClient associated with 491 this reporter. 492 """ 493 if self._check_tracker(): 494 return self._phapi_client 495 raise phapi_lib.ProjectHostingApiException('Project hosting client not ' 496 'initialized for project:%s, using auth file: %s' % 497 (self._project_name, self._get_creds_abspath())) 498 499 500 def _get_lab_error_template(self): 501 """Return the lab error template. 502 503 @return: A dictionary representing the bug options for an issue that 504 requires investigation from the lab team. 505 """ 506 lab_sheriff = site_utils.get_sheriffs(lab_only=True) 507 return {'labels': ['Build-HardwareLab'], 508 'owner': lab_sheriff[0] if lab_sheriff else '',} 509 510 511 def _format_issue_options(self, override, **kwargs): 512 """ 513 Override the default issue configuration with a suite specific 514 configuration when one is specified in the suite's bug_template. 515 The bug_template is specified in the suite control file. After 516 overriding the correct options, format them in a way that's understood 517 by the project hosting api. 518 519 @param override: Suite specific dictionary with issue config operations. 520 @param kwargs: Keyword args containing the default issue config options. 521 @return: A dictionary which contains the suite specific options, and the 522 default option when a suite specific option isn't specified. 523 """ 524 if override: 525 kwargs.update((k,v) for k,v in override.iteritems() if v) 526 527 kwargs['summary'] = kwargs['title'] 528 kwargs['labels'] = list(set(kwargs['labels'] + self._PREDEFINED_LABELS)) 529 kwargs['cc'] = list(map(lambda cc: {'name': cc}, 530 set(kwargs['cc'] + kwargs['sheriffs']))) 531 532 # The existence of an owner key will cause the api to try and match 533 # the value under the key to a member of the project, resulting in a 534 # 404 or 500 Http response when the owner is invalid. 535 if (CHROMIUM_EMAIL_ADDRESS not in kwargs['owner']): 536 del(kwargs['owner']) 537 else: 538 kwargs['owner'] = {'name': kwargs['owner']} 539 return kwargs 540 541 542 def _anchor_summary(self, bug): 543 """ 544 Creates the summary that can be used for bug deduplication. 545 546 Only attaches the anchor if the search_marker on the bug is not None. 547 548 @param: The bug to create the anchored summary for. 549 550 @return the summary with the anchor appened if the search marker is not 551 None, otherwise return the summary. 552 """ 553 if bug.search_marker() is None: 554 return bug.summary() 555 else: 556 return '%s\n\n%s%s\n' % (bug.summary(), self._SEARCH_MARKER, 557 bug.search_marker()) 558 559 560 def _create_bug_report(self, bug, bug_template={}, sheriffs=[]): 561 """ 562 Creates a new bug report. 563 564 @param bug: The Bug instance to create the report for. 565 @param bug_template: A template of options to use for filing bugs. 566 @param sheriffs: A list of chromium email addresses (of sheriffs) 567 to cc on this bug. Since the list of sheriffs is 568 dynamic it needs to be determined at runtime, as 569 opposed to the normal cc list which is available 570 through the bug template. 571 @return: id of the created issue as a string, or None if an issue 572 wasn't created. Note that if either the description or title 573 fields are missing we won't be able to create a bug. 574 """ 575 anchored_summary = self._anchor_summary(bug) 576 577 issue = self._format_issue_options(bug_template, title=bug.title(), 578 description=anchored_summary, labels=bug.labels, 579 status='Untriaged', owner=bug.owner, cc=bug.cc, 580 sheriffs=sheriffs, components=bug.components) 581 582 try: 583 filed_bug = self._phapi_client.create_issue(issue) 584 except phapi_lib.ProjectHostingApiException as e: 585 logging.error('Unable to create a bug for issue with title: %s and ' 586 'description %s and owner: %s. To file a new bug you ' 587 'need both a description and a title, and to assign ' 588 'it to an owner, that person must be known to the ' 589 'bug tracker', bug.title(), anchored_summary, 590 issue.get('owner')) 591 else: 592 logging.info('Filing new bug %s, with description %s', 593 filed_bug.get('id'), anchored_summary) 594 return filed_bug.get('id') 595 596 597 def modify_bug_report(self, issue_id, comment, label_update, status=''): 598 """Modifies an existing bug report with a new comment. 599 600 Adds the given comment and applies the given list of label 601 updates. 602 603 @param issue_id Id of the issue to update with. 604 @param comment Comment to update the issue with. 605 @param label_update List with label updates. 606 @param status New status of the issue. 607 """ 608 updates = { 609 'content': comment, 610 'updates': { 'labels': label_update, 'status': status } 611 } 612 try: 613 self._phapi_client.update_issue(issue_id, updates) 614 except phapi_lib.ProjectHostingApiException as e: 615 logging.warning('Unable to update issue %s, comment %s, ' 616 'labels %r, status %s: %s', issue_id, comment, 617 label_update, status, e) 618 else: 619 logging.info('Updated issue %s, comment %s, labels %r, status %s.', 620 issue_id, comment, label_update, status) 621 622 623 def _find_issue_by_marker(self, marker): 624 """ 625 Queries the tracker to find if there is a bug filed for this issue. 626 627 1. 'Escape' the string: cgi.escape is the easiest way to achieve this, 628 though it doesn't handle all html escape characters. 629 eg: replace '"<' with '"<' 630 2. Perform an exact search for the escaped string, if this returns an 631 empty issue list perform a more relaxed query and finally fall back 632 to a query devoid of the reason field. Between these 3 queries we 633 should retrieve the super set of all issues that this marker can be 634 in. In most cases the first search should return a result, examples 635 where this might not be the case are when the reason field contains 636 information that varies between test runs. Since the second search 637 has raw escape characters it will match comments too, and the last 638 should match all similar issues regardless. 639 3. Look through the issues for an exact match between clean versions 640 of the marker and summary; for now 'clean' means bereft of numbers. 641 4. If no match is found look through a list of comments for each issue. 642 643 @param marker The marker string to search for to find a duplicate of 644 this issue. 645 @return A phapi_lib.Issue instance of the issue that was found, or 646 None if no issue was found. Also returns None if the marker 647 is None. 648 """ 649 650 if marker is None: 651 logging.info('No search marker specified, will create new issue.') 652 return None 653 654 # Note that this method cannot handle markers which have already been 655 # html escaped, as it will try and unescape them by converting the & 656 # to & again, thereby failing deduplication. 657 marker = HTMLParser.HTMLParser().unescape(marker) 658 html_escaped_marker = cgi.escape(marker, quote=True) 659 660 # The tracker frontend stores summaries and comments as html elements, 661 # specifically, a summary turns into a span and a comment into 662 # preformatted text. Eg: 663 # 1. A summary of >& would become <span>>&</span> 664 # 2. A comment of >& would become <pre>>&</pre> 665 # When searching for exact matches in text, the gdata api gets this 666 # feed and parses all <pre> tags unescaping html, then matching your 667 # exact string to that. However it does not unescape all <span> tags, 668 # presumably for reasons of performance. Therefore a search for the 669 # exact string ">&" would match issue 2, but not issue 1, and a search 670 # for ">&" would match issue 1 but not issue 2. This problem is 671 # further exacerbated when we have quotes within our search string, 672 # which is common when the reason field contains a python dictionary. 673 # 674 # Our searching strategy prioritizes exact matches in the summary, since 675 # the first bug thats filed will have a summary with the anchor. If we 676 # do not find an exact match in any summary we search through all 677 # related issues of the same bug/suite in the hope of finding an exact 678 # match in the comments. Note that the comments are returned as 679 # unescaped text. 680 # 681 # TODO(beeps): when we start merging issues this could return bloated 682 # results, but for now we have to include duplicate issues so that 683 # we can find the original one with the hook. 684 markers = ['"' + self._SEARCH_MARKER + html_escaped_marker + '"', 685 self._SEARCH_MARKER + marker, 686 self._SEARCH_MARKER + ','.join(marker.split(',')[:2])] 687 for decorated_marker in markers: 688 issues = self._phapi_client.get_tracker_issues_by_text( 689 decorated_marker, include_dupes=True) 690 if issues: 691 break 692 693 if not issues: 694 return 695 696 # Breadth first, since open issues/bugs probably < comments/issue. 697 # If we find more than one issue matching a particular anchor assign 698 # a mystery bug with all relevent information on the owner and return 699 # the first matching issue. 700 clean_marker = re.sub('[0-9]+', '', html_escaped_marker) 701 all_issues = [issue for issue in issues 702 if clean_marker in re.sub('[0-9]+', '', issue.summary)] 703 704 if len(all_issues) > 1: 705 issue_ids = [issue.id for issue in all_issues] 706 logging.warning('Multiple results for a specific query. Query: %s, ' 707 'results: %s', marker, issue_ids) 708 709 if all_issues: 710 return all_issues[0] 711 712 unescaped_clean_marker = re.sub('[0-9]+', '', marker) 713 for issue in issues: 714 if any(unescaped_clean_marker in re.sub('[0-9]+', '', comment) 715 for comment in issue.comments): 716 return issue 717 718 719 def _dedupe_issue(self, marker): 720 """Finds an issue, then checks if it has a parent that's still open. 721 722 @param marker: The marker string to search for to find a duplicate of 723 a issue. 724 @return An Issue instance, representing an open issue that is a 725 duplicate of the one being searched for. 726 """ 727 issue = self._find_issue_by_marker(marker) 728 if not issue or issue.state == constants.ISSUE_OPEN: 729 return issue 730 731 # Iterativly look through the chain of parents, until we find one whose 732 # state is 'open' or reach the end of the chain. 733 # It is possible that the chain forms a circle. Record the visited 734 # issues to prevent loop on a circle. 735 visited_issues = set([issue.id]) 736 while issue.merged_into is not None: 737 issue = self._phapi_client.get_tracker_issue_by_id( 738 issue.merged_into) 739 if not issue or issue.id in visited_issues: 740 break 741 elif issue.state == constants.ISSUE_OPEN: 742 logging.debug('Return the active issue %d that duplicated ' 743 'issue(s) have been merged into.', issue.id) 744 return issue 745 else: 746 visited_issues.add(issue.id) 747 logging.debug('All merged issues %s have been closed, marked ' 748 'invalid etc, will create a new issue instead.', 749 list(visited_issues)) 750 return None 751 752 753 def _get_count_labels_and_max(self, issue): 754 """Read the current autofiled count labels and count. 755 756 Automatically filed issues have a label of the form 757 `autofiled-count-<number>` that indicates about how many 758 times the autofiling code has updated the issue. This 759 routine goes through the labels for the given issue to find 760 the existing count label(s). 761 762 Old bugs may not have a count; this routine implicitly 763 assigns those bugs an initial count of one. 764 765 Usually, only one count label should exist. But 766 this method is written to take care of the case 767 where multiple count labels exist. In such case, 768 All the labels and the max count is returned. 769 770 @param issue: Issue whose 'autofiled-count' is to be read. 771 772 @returns: 2-tuple with a list of labels and 773 the max count. 774 """ 775 count_labels = [] 776 count_max = 1 777 is_count_label = lambda l: l.startswith(self.AUTOFILED_COUNT) 778 for label in filter(is_count_label, issue.labels): 779 try: 780 count = int(label[len(self.AUTOFILED_COUNT):]) 781 except ValueError: 782 continue 783 count_max = max(count, count_max) 784 count_labels.append(label) 785 return count_labels, count_max 786 787 788 def _create_autofiled_count_update(self, issue): 789 """Calculate an 'autofiled-count' label update. 790 791 Remove all the existing autofiled count labels 792 and calculate a new count label. 793 794 Updates to issues aren't guaranteed to be atomic, so in 795 some cases count labels may (in theory at least) be dropped 796 or duplicated. 797 798 The return values are a list of label updates and the 799 count value of the new count label. For the label updates, 800 all existing count labels will be prefixed with '-' to 801 remove them, and a new label with a new count will be added 802 to the set. Labels not related to the count aren't updated. 803 804 @param issue Issue whose 'autofiled-count' is to be updated. 805 @return 2-tuple with a list of label updates and the 806 new count value. 807 """ 808 count_labels, count_max = self._get_count_labels_and_max(issue) 809 label_updates = [] 810 for label in count_labels: 811 label_updates.append('-%s' % label) 812 new_count = count_max + 1 813 label_updates.append('%s%d' % (self.AUTOFILED_COUNT, new_count)) 814 return label_updates, new_count 815 816 817 @classmethod 818 def _get_project_label_from_title(cls, title): 819 """Extract a project label for the device being tested from 820 provided bug title. If no project is found, return empty string. 821 822 E.g. For the following bug title: 823 824 [stress] platform_BootDevice Failure on rikku-release/R44-7075.0.0 825 826 we extract 'rikku' and return a string 'Proj-rikku'. 827 828 Note1: For certain boards, they contain the reference name as well: 829 830 veyron_minnie-release/R44-7075.0.0 831 832 in these cases, we only extract and use the subboard (minnie) and not 833 the whole string (veyron_minnie). 834 835 Note2: some builds have different names like tot-release, 836 freon-build, etc. This function needs to handle these cases as well. 837 838 @param title: A string of the bug title, from which to extract 839 the project label for the device being tested. 840 @return '' if no valid label is found, or a label of the 841 form 'proj-samus' if found. 842 """ 843 m = re.search('.* on (?:.*_)?(?P<proj>[^-]*)-[\S]+/.*', title) 844 if m and m.group('proj'): 845 return 'Proj-%s' % m.group('proj') 846 else: 847 return '' 848 849 850 def report(self, bug, bug_template=None, ignore_duplicate=False): 851 """Report an issue to the bug tracker. 852 853 If this issue has happened before, post a comment on the 854 existing bug about it occurring again, and update the 855 'autofiled-count' label. If this is a new issue, create a 856 new bug for it. 857 858 @param bug A Bug instance about the issue. 859 @param bug_template A template dictionary specifying the 860 default bug filing options for an issue 861 with this suite. 862 @param ignore_duplicate If True, when a duplicate is found, 863 simply ignore the new one rather than 864 posting an update. 865 @return A ReportResult namedtuple containing: 866 867 - the issue id as a string or None 868 - the number of times the bug has been updated. For a new 869 bug, the count is 1. If we could not file a bug for some 870 reason, the count is 0. 871 """ 872 if bug_template is None: 873 bug_template = {} 874 875 if not self._check_tracker(): 876 logging.error("Can't file %s", bug.title()) 877 return ReportResult(None, 0) 878 879 project_label = self._get_project_label_from_title(bug.title()) 880 881 issue = None 882 try: 883 issue = self._dedupe_issue(bug.search_marker()) 884 except expat.ExpatError as e: 885 # If our search string sends python's xml module into a 886 # state which it believes will lead to an xml syntax 887 # error, it will give up and throw an exception. This 888 # might happen with aborted jobs that contain weird 889 # escape characters in their reason fields. We'd rather 890 # create a new issue than fail in deduplicating such cases. 891 logging.warning('Unable to deduplicate, creating new issue: %s', 892 str(e)) 893 894 if issue and ignore_duplicate: 895 logging.debug('Duplicate found for %s, not filing as requested.', 896 bug.search_marker()) 897 _, bug_count = self._get_count_labels_and_max(issue) 898 return ReportResult(issue.id, bug_count) 899 900 if issue: 901 comment = '%s\n\n%s' % (bug.title(), self._anchor_summary(bug)) 902 label_update, bug_count = ( 903 self._create_autofiled_count_update(issue)) 904 if project_label: 905 label_update.append(project_label) 906 self.modify_bug_report(issue.id, comment, label_update) 907 return ReportResult(issue.id, bug_count) 908 909 sheriffs = [] 910 911 # TODO(beeps): crbug.com/254256 912 try: 913 if bug.lab_error and bug.suite == 'bvt': 914 lab_error_template = self._get_lab_error_template() 915 if bug_template.get('labels'): 916 lab_error_template['labels'] += bug_template.get('labels') 917 bug_template = lab_error_template 918 elif bug.suite == 'bvt': 919 sheriffs = site_utils.get_sheriffs() 920 except AttributeError: 921 pass 922 923 if project_label: 924 bug_template.get('labels', []).append(project_label) 925 bug_id = self._create_bug_report(bug, bug_template, sheriffs) 926 bug_count = 1 if bug_id else 0 927 return ReportResult(bug_id, bug_count) 928 929 930class NullReporter(object): 931 """Null object for bug reporter.""" 932 933 def report(self, bug, bug_template=None, ignore_duplicate=False): 934 """Report an issue to the bug tracker. 935 936 If this issue has happened before, post a comment on the 937 existing bug about it occurring again, and update the 938 'autofiled-count' label. If this is a new issue, create a 939 new bug for it. 940 941 @param bug A Bug instance about the issue. 942 @param bug_template A template dictionary specifying the 943 default bug filing options for an issue 944 with this suite. 945 @param ignore_duplicate If True, when a duplicate is found, 946 simply ignore the new one rather than 947 posting an update. 948 @return A ReportResult namedtuple containing: 949 950 - the issue id as a string or None 951 - the number of times the bug has been updated. For a new 952 bug, the count is 1. If we could not file a bug for some 953 reason, the count is 0. 954 """ 955 return ReportResult(None, 0) 956 957 958# TODO(beeps): Move this to server/site_utils after crbug.com/281906 is fixed. 959def submit_generic_bug_report(*args, **kwargs): 960 """ 961 Submit a generic bug report. 962 963 See server.cros.dynamic_suite.reporting.Bug for valid arguments. 964 965 @params args: List of arguments to pass to the Bug creation. 966 @params kwargs: Keyword arguments to pass to Bug creation. 967 968 @returns the filed bug's id. 969 """ 970 bug = Bug(*args, **kwargs) 971 reporter = Reporter() 972 return reporter.report(bug)[0] 973 974 975def send_email(bug, bug_template): 976 """Send email to the owner and cc's to notify the TestBug. 977 978 @param bug: TestBug instance. 979 @param bug_template: A template dictionary specifying the default bug 980 filing options for failures in this suite. 981 """ 982 to_set = set(bug.cc) if bug.cc else set() 983 if bug.owner: 984 to_set.add(bug.owner) 985 if bug_template.get('cc'): 986 to_set = to_set.union(bug_template.get('cc')) 987 if bug_template.get('owner'): 988 to_set.add(bug_template.get('owner')) 989 recipients = ', '.join(to_set) 990 if not recipients: 991 logging.warning('No owner/cc found. Will skip sending a mail.') 992 return 993 success = False 994 try: 995 gmail_lib.send_email( 996 recipients, bug.title(), bug.summary(), retry=False, 997 creds_path=site_utils.get_creds_abspath(EMAIL_CREDS_FILE)) 998 success = True 999 finally: 1000 (metrics.Counter('chromeos/autotest/errors/send_bug_email') 1001 .increment(fields={'success': success})) 1002