1#!/usr/bin/python 2# 3# Copyright (c) 2013 The Chromium OS Authors. All rights reserved. 4# Use of this source code is governed by a BSD-style license that can be 5# found in the LICENSE file. 6 7import unittest 8 9import mox 10 11import common 12from autotest_lib.server.cros.dynamic_suite import constants 13from autotest_lib.server.cros.dynamic_suite import job_status 14from autotest_lib.server.cros.dynamic_suite import reporting 15from autotest_lib.server.cros.dynamic_suite import reporting_utils 16from autotest_lib.server.cros.dynamic_suite import tools 17from autotest_lib.site_utils import phapi_lib 18from chromite.lib import gdata_lib 19 20 21class ReportingTest(mox.MoxTestBase): 22 """Unittests to verify basic control flow for automatic bug filing.""" 23 24 # fake issue id to use in testing duplicate issues 25 _FAKE_ISSUE_ID = 123 26 27 # test report used to generate failure 28 test_report = { 29 'build':'build-build/R1-1', 30 'chrome_version':'28.0', 31 'suite':'suite', 32 'test':'bad_test', 33 'reason':'dreadful_reason', 34 'owner':'user', 35 'hostname':'myhost', 36 'job_id':'myjob', 37 'status': 'FAIL', 38 } 39 40 bug_template = { 41 'labels': ['Cr-Internals-WebRTC'], 42 'owner': 'myself', 43 'status': 'Fixed', 44 'summary': 'This is a short summary', 45 'title': None, 46 } 47 48 def _get_failure(self, is_server_job=False): 49 """Get a TestBug so we can report it. 50 51 @param is_server_job: Set to True of failed job is a server job. Server 52 job's test name is formated as build/suite/test_name. 53 @return: a failure object initialized with values from test_report. 54 """ 55 if is_server_job: 56 test_name = tools.create_job_name( 57 self.test_report.get('build'), 58 self.test_report.get('suite'), 59 self.test_report.get('test')) 60 else: 61 test_name = self.test_report.get('test') 62 expected_result = job_status.Status(self.test_report.get('status'), 63 test_name, 64 reason=self.test_report.get('reason'), 65 job_id=self.test_report.get('job_id'), 66 owner=self.test_report.get('owner'), 67 hostname=self.test_report.get('hostname')) 68 69 return reporting.TestBug(self.test_report.get('build'), 70 self.test_report.get('chrome_version'), 71 self.test_report.get('suite'), expected_result) 72 73 74 def setUp(self): 75 super(ReportingTest, self).setUp() 76 self.mox.StubOutClassWithMocks(phapi_lib, 'ProjectHostingApiClient') 77 self._orig_project_name = reporting.Reporter._project_name 78 self._orig_monorail_server = reporting.Reporter._monorail_server 79 80 # We want to have some data so that the Reporter doesn't fail at 81 # initialization. 82 reporting.Reporter._project_name = 'project' 83 reporting.Reporter._monorail_server = 'staging' 84 85 86 def tearDown(self): 87 reporting.Reporter._project_name = self._orig_project_name 88 reporting.Reporter._monorail_server = self._orig_monorail_server 89 super(ReportingTest, self).tearDown() 90 91 92 def testNewIssue(self): 93 """Add a new issue to the tracker when a matching issue isn't found. 94 95 Confirms that we call CreateTrackerIssue when an Issue search 96 returns None. 97 """ 98 self.mox.StubOutWithMock(reporting.Reporter, '_find_issue_by_marker') 99 self.mox.StubOutWithMock(reporting.TestBug, 'summary') 100 101 client = phapi_lib.ProjectHostingApiClient(mox.IgnoreArg(), 102 mox.IgnoreArg(), 103 mox.IgnoreArg()) 104 client.create_issue(mox.IgnoreArg()).AndReturn( 105 {'id': self._FAKE_ISSUE_ID}) 106 reporting.Reporter._find_issue_by_marker(mox.IgnoreArg()).AndReturn( 107 None) 108 reporting.TestBug.summary().AndReturn('') 109 110 self.mox.ReplayAll() 111 bug_id, bug_count = reporting.Reporter().report(self._get_failure()) 112 113 self.assertEqual(bug_id, self._FAKE_ISSUE_ID) 114 self.assertEqual(bug_count, 1) 115 116 117 def testDuplicateIssue(self): 118 """Dedupe to an existing issue when one is found. 119 120 Confirms that we call AppendTrackerIssueById with the same issue 121 returned by the issue search. 122 """ 123 self.mox.StubOutWithMock(reporting.Reporter, '_find_issue_by_marker') 124 self.mox.StubOutWithMock(reporting.TestBug, 'summary') 125 126 issue = self.mox.CreateMock(phapi_lib.Issue) 127 issue.id = self._FAKE_ISSUE_ID 128 issue.labels = [] 129 issue.state = constants.ISSUE_OPEN 130 131 client = phapi_lib.ProjectHostingApiClient(mox.IgnoreArg(), 132 mox.IgnoreArg(), 133 mox.IgnoreArg()) 134 client.update_issue(self._FAKE_ISSUE_ID, mox.IgnoreArg()) 135 reporting.Reporter._find_issue_by_marker(mox.IgnoreArg()).AndReturn( 136 issue) 137 138 reporting.TestBug.summary().AndReturn('') 139 140 self.mox.ReplayAll() 141 bug_id, bug_count = reporting.Reporter().report(self._get_failure()) 142 143 self.assertEqual(bug_id, self._FAKE_ISSUE_ID) 144 self.assertEqual(bug_count, 2) 145 146 147 def testSuiteIssueConfig(self): 148 """Test that the suite bug template values are not overridden.""" 149 150 def check_suite_options(issue): 151 """ 152 Checks to see if the options specified in bug_template reflect in 153 the issue we're about to file, and that the autofiled label was not 154 lost in the process. 155 156 @param issue: issue to check labels on. 157 """ 158 assert('autofiled' in issue.labels) 159 for k, v in self.bug_template.iteritems(): 160 if (isinstance(v, list) 161 and all(item in getattr(issue, k) for item in v)): 162 continue 163 if v and getattr(issue, k) is not v: 164 return False 165 return True 166 167 self.mox.StubOutWithMock(reporting.Reporter, '_find_issue_by_marker') 168 self.mox.StubOutWithMock(reporting.TestBug, 'summary') 169 170 reporting.Reporter._find_issue_by_marker(mox.IgnoreArg()).AndReturn( 171 None) 172 reporting.TestBug.summary().AndReturn('Summary') 173 174 mock_host = phapi_lib.ProjectHostingApiClient(mox.IgnoreArg(), 175 mox.IgnoreArg(), 176 mox.IgnoreArg()) 177 mock_host.create_issue(mox.IgnoreArg()).AndReturn( 178 {'id': self._FAKE_ISSUE_ID}) 179 180 self.mox.ReplayAll() 181 bug_id, bug_count = reporting.Reporter().report(self._get_failure(), 182 self.bug_template) 183 184 self.assertEqual(bug_id, self._FAKE_ISSUE_ID) 185 self.assertEqual(bug_count, 1) 186 187 188 def testGenericBugCanBeFiled(self): 189 """Test that we can use a Bug object to file a bug report.""" 190 self.mox.StubOutWithMock(reporting.Reporter, '_find_issue_by_marker') 191 192 bug = reporting.Bug('title', 'summary', 'marker') 193 194 reporting.Reporter._find_issue_by_marker(mox.IgnoreArg()).AndReturn( 195 None) 196 197 mock_host = phapi_lib.ProjectHostingApiClient(mox.IgnoreArg(), 198 mox.IgnoreArg(), 199 mox.IgnoreArg()) 200 mock_host.create_issue(mox.IgnoreArg()).AndReturn( 201 {'id': self._FAKE_ISSUE_ID}) 202 203 self.mox.ReplayAll() 204 bug_id, bug_count = reporting.Reporter().report(bug) 205 206 self.assertEqual(bug_id, self._FAKE_ISSUE_ID) 207 self.assertEqual(bug_count, 1) 208 209 210 def testWithSearchMarkerSetToNoneIsNotDeduped(self): 211 """Test that we do not dedupe bugs that have no search marker.""" 212 213 bug = reporting.Bug('title', 'summary', search_marker=None) 214 215 mock_host = phapi_lib.ProjectHostingApiClient(mox.IgnoreArg(), 216 mox.IgnoreArg(), 217 mox.IgnoreArg()) 218 mock_host.create_issue(mox.IgnoreArg()).AndReturn( 219 {'id': self._FAKE_ISSUE_ID}) 220 221 self.mox.ReplayAll() 222 bug_id, bug_count = reporting.Reporter().report(bug) 223 224 self.assertEqual(bug_id, self._FAKE_ISSUE_ID) 225 self.assertEqual(bug_count, 1) 226 227 228 def testSearchMarkerNoBuildSuiteInfo(self): 229 """Test that the search marker does not include build and suite info.""" 230 test_failure = self._get_failure(is_server_job=True) 231 search_marker = test_failure.search_marker() 232 self.assertFalse(test_failure.build in search_marker, 233 ('Build information should not be presented in search ' 234 'marker.')) 235 236 237class FindIssueByMarkerTests(mox.MoxTestBase): 238 """Tests the _find_issue_by_marker function.""" 239 240 def setUp(self): 241 super(FindIssueByMarkerTests, self).setUp() 242 self.mox.StubOutClassWithMocks(phapi_lib, 'ProjectHostingApiClient') 243 self._orig_project_name = reporting.Reporter._project_name 244 self._orig_monorail_server = reporting.Reporter._monorail_server 245 246 # We want to have some data so that the Reporter doesn't fail at 247 # initialization. 248 reporting.Reporter._project_name = 'project' 249 reporting.Reporter._monorail_server = 'staging' 250 251 def tearDown(self): 252 reporting.Reporter._project_name = self._orig_project_name 253 reporting.Reporter._monorail_server = self._orig_monorail_server 254 super(FindIssueByMarkerTests, self).tearDown() 255 256 257 def testReturnNoneIfMarkerIsNone(self): 258 """Test that we do not look up an issue if the search marker is None.""" 259 mock_host = phapi_lib.ProjectHostingApiClient(mox.IgnoreArg(), 260 mox.IgnoreArg(), 261 mox.IgnoreArg()) 262 263 self.mox.ReplayAll() 264 result = reporting.Reporter()._find_issue_by_marker(None) 265 self.assertTrue(result is None) 266 267 268class AnchorSummaryTests(mox.MoxTestBase): 269 """Tests the _anchor_summary function.""" 270 271 def setUp(self): 272 super(AnchorSummaryTests, self).setUp() 273 self.mox.StubOutClassWithMocks(phapi_lib, 'ProjectHostingApiClient') 274 self._orig_project_name = reporting.Reporter._project_name 275 self._orig_monorail_server = reporting.Reporter._monorail_server 276 277 # We want to have some data so that the Reporter doesn't fail at 278 # initialization. 279 reporting.Reporter._project_name = 'project' 280 reporting.Reporter._monorail_server = 'staging' 281 282 283 def tearDown(self): 284 reporting.Reporter._project_name = self._orig_project_name 285 reporting.Reporter._monorail_server = self._orig_monorail_server 286 super(AnchorSummaryTests, self).tearDown() 287 288 289 def test_summary_returned_untouched_if_no_search_maker(self): 290 """Test that we just return the summary if we have no search marker.""" 291 mock_host = phapi_lib.ProjectHostingApiClient(mox.IgnoreArg(), 292 mox.IgnoreArg(), 293 mox.IgnoreArg()) 294 295 bug = reporting.Bug('title', 'summary', None) 296 297 self.mox.ReplayAll() 298 result = reporting.Reporter()._anchor_summary(bug) 299 300 self.assertEqual(result, 'summary') 301 302 303 def test_append_anchor_to_summary_if_search_marker(self): 304 """Test that we add an anchor to the search marker.""" 305 mock_host = phapi_lib.ProjectHostingApiClient(mox.IgnoreArg(), 306 mox.IgnoreArg(), 307 mox.IgnoreArg()) 308 309 bug = reporting.Bug('title', 'summary', 'marker') 310 311 self.mox.ReplayAll() 312 result = reporting.Reporter()._anchor_summary(bug) 313 314 self.assertEqual(result, 'summary\n\n%smarker\n' % 315 reporting.Reporter._SEARCH_MARKER) 316 317 318class LabelUpdateTests(mox.MoxTestBase): 319 """Test the _create_autofiled_count_update() function.""" 320 321 def setUp(self): 322 super(LabelUpdateTests, self).setUp() 323 self.mox.StubOutClassWithMocks(phapi_lib, 'ProjectHostingApiClient') 324 self._orig_project_name = reporting.Reporter._project_name 325 self._orig_monorail_server = reporting.Reporter._monorail_server 326 327 # We want to have some data so that the Reporter doesn't fail at 328 # initialization. 329 reporting.Reporter._project_name = 'project' 330 reporting.Reporter._monorail_server = 'staging' 331 332 333 def tearDown(self): 334 reporting.Reporter._project_name = self._orig_project_name 335 reporting.Reporter._monorail_server = self._orig_monorail_server 336 super(LabelUpdateTests, self).tearDown() 337 338 339 def _create_count_label(self, n): 340 return '%s%d' % (reporting.Reporter.AUTOFILED_COUNT, n) 341 342 343 def _test_count_label_update(self, labels, remove, expected_count): 344 """Utility to test _create_autofiled_count_update(). 345 346 @param labels Input list of labels. 347 @param remove List of labels expected to be removed 348 in the result. 349 @param expected_count Count value expected to be returned 350 from the call. 351 """ 352 client = phapi_lib.ProjectHostingApiClient(mox.IgnoreArg(), 353 mox.IgnoreArg(), 354 mox.IgnoreArg()) 355 self.mox.ReplayAll() 356 issue = self.mox.CreateMock(gdata_lib.Issue) 357 issue.labels = labels 358 359 reporter = reporting.Reporter() 360 new_labels, count = reporter._create_autofiled_count_update(issue) 361 expected = map(lambda l: '-' + l, remove) 362 expected.append(self._create_count_label(expected_count)) 363 self.assertEqual(new_labels, expected) 364 self.assertEqual(count, expected_count) 365 366 367 def testProjectLabelExtraction(self): 368 """Test that the project label is correctly extracted from the title.""" 369 TITLE_EMPTY = '' 370 TITLE_NO_PROJ = '[stress] platformDevice Failure on release/47-75.0.0' 371 TITLE_PROJ = '[stress] p_Device Failure on rikku-release/R44-7075.0.0' 372 TITLE_PROJ2 = '[stress] p_Device Failure on ' \ 373 'rikku-freon-release/R44-7075.0.0' 374 TITLE_PROJ_SUBBOARD = '[stress] p_Device Failure on ' \ 375 'veyron_rikku-release/R44-7075.0.0' 376 377 client = phapi_lib.ProjectHostingApiClient(mox.IgnoreArg(), 378 mox.IgnoreArg(), 379 mox.IgnoreArg()) 380 self.mox.ReplayAll() 381 382 reporter = reporting.Reporter() 383 self.assertEqual(reporter._get_project_label_from_title(TITLE_EMPTY), 384 '') 385 self.assertEqual(reporter._get_project_label_from_title( 386 TITLE_NO_PROJ), '') 387 self.assertEqual(reporter._get_project_label_from_title(TITLE_PROJ), 388 'Proj-rikku') 389 self.assertEqual(reporter._get_project_label_from_title(TITLE_PROJ2), 390 'Proj-rikku') 391 self.assertEqual(reporter._get_project_label_from_title( 392 TITLE_PROJ_SUBBOARD), 'Proj-rikku') 393 394 395 def testCountLabelIncrement(self): 396 """Test that incrementing an autofiled-count label should work.""" 397 n = 3 398 old_label = self._create_count_label(n) 399 self._test_count_label_update([old_label], [old_label], n + 1) 400 401 402 def testCountLabelIncrementPredefined(self): 403 """Test that Reporter._PREDEFINED_LABELS has a sane autofiled-count.""" 404 self._test_count_label_update( 405 reporting.Reporter._PREDEFINED_LABELS, 406 [self._create_count_label(1)], 2) 407 408 409 def testCountLabelCreate(self): 410 """Test that old bugs should get a correct autofiled-count.""" 411 self._test_count_label_update([], [], 2) 412 413 414 def testCountLabelIncrementMultiple(self): 415 """Test that duplicate autofiled-count labels are handled.""" 416 old_count1 = self._create_count_label(2) 417 old_count2 = self._create_count_label(3) 418 self._test_count_label_update([old_count1, old_count2], 419 [old_count1, old_count2], 4) 420 421 422 def testCountLabelSkipUnknown(self): 423 """Test that autofiled-count increment ignores unknown labels.""" 424 old_count = self._create_count_label(3) 425 self._test_count_label_update(['unknown-label', old_count], 426 [old_count], 4) 427 428 429 def testCountLabelSkipMalformed(self): 430 """Test that autofiled-count increment ignores unusual labels.""" 431 old_count = self._create_count_label(3) 432 self._test_count_label_update( 433 [reporting.Reporter.AUTOFILED_COUNT + 'bogus', 434 self._create_count_label(8) + '-bogus', 435 old_count], 436 [old_count], 4) 437 438 439class TestSubmitGenericBugReport(mox.MoxTestBase, unittest.TestCase): 440 """Test the submit_generic_bug_report function.""" 441 442 def setUp(self): 443 super(TestSubmitGenericBugReport, self).setUp() 444 self.mox.StubOutClassWithMocks(reporting, 'Reporter') 445 446 447 def test_accepts_required_arguments(self): 448 """ 449 Test that the function accepts the required arguments. 450 451 This basically tests that no exceptions are thrown. 452 453 """ 454 reporter = reporting.Reporter() 455 reporter.report(mox.IgnoreArg()).AndReturn((11,1)) 456 457 self.mox.ReplayAll() 458 reporting.submit_generic_bug_report('title', 'summary') 459 460 461 def test_rejects_too_few_required_arguments(self): 462 """Test that the function rejects too few required arguments.""" 463 self.mox.ReplayAll() 464 self.assertRaises(TypeError, 465 reporting.submit_generic_bug_report, 'too_few') 466 467 468 def test_accepts_key_word_arguments(self): 469 """ 470 Test that the functions accepts the key_word arguments. 471 472 This basically tests that no exceptions are thrown. 473 474 """ 475 reporter = reporting.Reporter() 476 reporter.report(mox.IgnoreArg()).AndReturn((11,1)) 477 478 self.mox.ReplayAll() 479 reporting.submit_generic_bug_report('test', 'summary', labels=[]) 480 481 482 def test_rejects_invalid_keyword_arguments(self): 483 """Test that the function rejects invalid keyword arguments.""" 484 self.mox.ReplayAll() 485 self.assertRaises(TypeError, reporting.submit_generic_bug_report, 486 'title', 'summary', wrong='wrong') 487 488 489class TestMergeBugTemplate(mox.MoxTestBase): 490 """Test bug can be properly merged and validated.""" 491 def test_validate_success(self): 492 """Test a valid bug can be verified successfully.""" 493 bug_template= {} 494 bug_template['owner'] = 'someone@company.com' 495 reporting_utils.BugTemplate.validate_bug_template(bug_template) 496 497 498 def test_validate_success(self): 499 """Test a valid bug can be verified successfully.""" 500 # Bug template must be a dictionary. 501 bug_template = ['test'] 502 self.assertRaises(reporting_utils.InvalidBugTemplateException, 503 reporting_utils.BugTemplate.validate_bug_template, 504 bug_template) 505 506 # Bug template must contain value for essential attribute, e.g., owner. 507 bug_template= {'no-owner': 'user1'} 508 self.assertRaises(reporting_utils.InvalidBugTemplateException, 509 reporting_utils.BugTemplate.validate_bug_template, 510 bug_template) 511 512 # Bug template must contain value for essential attribute, e.g., owner. 513 bug_template= {'owner': 'invalid_email_address'} 514 self.assertRaises(reporting_utils.InvalidBugTemplateException, 515 reporting_utils.BugTemplate.validate_bug_template, 516 bug_template) 517 518 # Check unexpected attributes. 519 bug_template= {} 520 bug_template['random tag'] = 'test' 521 self.assertRaises(reporting_utils.InvalidBugTemplateException, 522 reporting_utils.BugTemplate.validate_bug_template, 523 bug_template) 524 525 # Value for cc must be a list 526 bug_template= {} 527 bug_template['cc'] = 'test' 528 self.assertRaises(reporting_utils.InvalidBugTemplateException, 529 reporting_utils.BugTemplate.validate_bug_template, 530 bug_template) 531 532 # Value for labels must be a list 533 bug_template= {} 534 bug_template['labels'] = 'test' 535 self.assertRaises(reporting_utils.InvalidBugTemplateException, 536 reporting_utils.BugTemplate.validate_bug_template, 537 bug_template) 538 539 540 def test_merge_success(self): 541 """Test test and suite bug templates can be merged successfully.""" 542 test_bug_template = { 543 'labels': ['l1'], 544 'owner': 'user1@chromium.org', 545 'status': 'Assigned', 546 'title': None, 547 'cc': ['cc1@chromium.org', 'cc2@chromium.org'] 548 } 549 suite_bug_template = { 550 'labels': ['l2'], 551 'owner': 'user2@chromium.org', 552 'status': 'Fixed', 553 'summary': 'This is a short summary for suite bug', 554 'title': 'Title for suite bug', 555 'cc': ['cc2@chromium.org', 'cc3@chromium.org'] 556 } 557 bug_template = reporting_utils.BugTemplate(suite_bug_template) 558 merged_bug_template = bug_template.finalize_bug_template( 559 test_bug_template) 560 self.assertEqual(merged_bug_template['owner'], 561 test_bug_template['owner'], 562 'Value in test bug template should prevail.') 563 564 self.assertEqual(merged_bug_template['title'], 565 suite_bug_template['title'], 566 'If an attribute has value None in test bug template, ' 567 'use the value given in suite bug template.') 568 569 self.assertEqual(merged_bug_template['summary'], 570 suite_bug_template['summary'], 571 'If an attribute does not exist in test bug template, ' 572 'but exists in suite bug template, it should be ' 573 'included in the merged template.') 574 575 self.assertEqual(merged_bug_template['cc'], 576 test_bug_template['cc'] + suite_bug_template['cc'], 577 'List values for an attribute should be merged.') 578 579 self.assertEqual(merged_bug_template['labels'], 580 test_bug_template['labels'] + 581 suite_bug_template['labels'], 582 'List values for an attribute should be merged.') 583 584 test_bug_template['owner'] = '' 585 test_bug_template['cc'] = [''] 586 suite_bug_template['owner'] = '' 587 suite_bug_template['cc'] = [''] 588 bug_template = reporting_utils.BugTemplate(suite_bug_template) 589 merged_bug_template = bug_template.finalize_bug_template( 590 test_bug_template) 591 self.assertFalse('owner' in merged_bug_template, 592 'owner should be removed from the merged template.') 593 self.assertFalse('cc' in merged_bug_template, 594 'cc should be removed from the merged template.') 595 596 597if __name__ == '__main__': 598 unittest.main() 599