1# Copyright (C) 2009 Google Inc. All rights reserved. 2# 3# Redistribution and use in source and binary forms, with or without 4# modification, are permitted provided that the following conditions are 5# met: 6# 7# * Redistributions of source code must retain the above copyright 8# notice, this list of conditions and the following disclaimer. 9# * Redistributions in binary form must reproduce the above 10# copyright notice, this list of conditions and the following disclaimer 11# in the documentation and/or other materials provided with the 12# distribution. 13# * Neither the name of Google Inc. nor the names of its 14# contributors may be used to endorse or promote products derived from 15# this software without specific prior written permission. 16# 17# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 18# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 19# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 20# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 21# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 22# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 23# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 24# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 25# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 29import os 30import StringIO 31 32from webkitpy.common.checkout.scm import CheckoutNeedsUpdate 33from webkitpy.common.net.bugzilla import Attachment 34from webkitpy.common.system.filesystem_mock import MockFileSystem 35from webkitpy.common.system.outputcapture import OutputCapture 36from webkitpy.layout_tests.layout_package import test_results 37from webkitpy.layout_tests.layout_package import test_failures 38from webkitpy.thirdparty.mock import Mock 39from webkitpy.tool.commands.commandtest import CommandsTest 40from webkitpy.tool.commands.queues import * 41from webkitpy.tool.commands.queuestest import QueuesTest 42from webkitpy.tool.commands.stepsequence import StepSequence 43from webkitpy.tool.mocktool import MockTool, MockSCM, MockStatusServer 44 45 46class TestQueue(AbstractPatchQueue): 47 name = "test-queue" 48 49 50class TestReviewQueue(AbstractReviewQueue): 51 name = "test-review-queue" 52 53 54class TestFeederQueue(FeederQueue): 55 _sleep_duration = 0 56 57 58class AbstractQueueTest(CommandsTest): 59 def test_log_directory(self): 60 self.assertEquals(TestQueue()._log_directory(), os.path.join("..", "test-queue-logs")) 61 62 def _assert_run_webkit_patch(self, run_args, port=None): 63 queue = TestQueue() 64 tool = MockTool() 65 tool.status_server.bot_id = "gort" 66 tool.executive = Mock() 67 queue.bind_to_tool(tool) 68 queue._options = Mock() 69 queue._options.port = port 70 71 queue.run_webkit_patch(run_args) 72 expected_run_args = ["echo", "--status-host=example.com", "--bot-id=gort"] 73 if port: 74 expected_run_args.append("--port=%s" % port) 75 expected_run_args.extend(run_args) 76 tool.executive.run_and_throw_if_fail.assert_called_with(expected_run_args) 77 78 def test_run_webkit_patch(self): 79 self._assert_run_webkit_patch([1]) 80 self._assert_run_webkit_patch(["one", 2]) 81 self._assert_run_webkit_patch([1], port="mockport") 82 83 def test_iteration_count(self): 84 queue = TestQueue() 85 queue._options = Mock() 86 queue._options.iterations = 3 87 self.assertTrue(queue.should_continue_work_queue()) 88 self.assertTrue(queue.should_continue_work_queue()) 89 self.assertTrue(queue.should_continue_work_queue()) 90 self.assertFalse(queue.should_continue_work_queue()) 91 92 def test_no_iteration_count(self): 93 queue = TestQueue() 94 queue._options = Mock() 95 self.assertTrue(queue.should_continue_work_queue()) 96 self.assertTrue(queue.should_continue_work_queue()) 97 self.assertTrue(queue.should_continue_work_queue()) 98 self.assertTrue(queue.should_continue_work_queue()) 99 100 def _assert_log_message(self, script_error, log_message): 101 failure_log = AbstractQueue._log_from_script_error_for_upload(script_error, output_limit=10) 102 self.assertTrue(failure_log.read(), log_message) 103 104 def test_log_from_script_error_for_upload(self): 105 self._assert_log_message(ScriptError("test"), "test") 106 # In python 2.5 unicode(Exception) is busted. See: 107 # http://bugs.python.org/issue2517 108 # With no good workaround, we just ignore these tests. 109 if not hasattr(Exception, "__unicode__"): 110 return 111 112 unicode_tor = u"WebKit \u2661 Tor Arne Vestb\u00F8!" 113 utf8_tor = unicode_tor.encode("utf-8") 114 self._assert_log_message(ScriptError(unicode_tor), utf8_tor) 115 script_error = ScriptError(unicode_tor, output=unicode_tor) 116 expected_output = "%s\nLast %s characters of output:\n%s" % (utf8_tor, 10, utf8_tor[-10:]) 117 self._assert_log_message(script_error, expected_output) 118 119 120class FeederQueueTest(QueuesTest): 121 def test_feeder_queue(self): 122 queue = TestFeederQueue() 123 tool = MockTool(log_executive=True) 124 expected_stderr = { 125 "begin_work_queue": self._default_begin_work_queue_stderr("feeder-queue", MockSCM.fake_checkout_root), 126 "should_proceed_with_work_item": "", 127 "next_work_item": "", 128 "process_work_item": """Warning, attachment 128 on bug 42 has invalid committer (non-committer@example.com) 129Warning, attachment 128 on bug 42 has invalid committer (non-committer@example.com) 130MOCK setting flag 'commit-queue' to '-' on attachment '128' with comment 'Rejecting attachment 128 from commit-queue.' and additional comment 'non-committer@example.com does not have committer permissions according to http://trac.webkit.org/browser/trunk/Tools/Scripts/webkitpy/common/config/committers.py. 131 132- If you do not have committer rights please read http://webkit.org/coding/contributing.html for instructions on how to use bugzilla flags. 133 134- If you have committer rights please correct the error in Tools/Scripts/webkitpy/common/config/committers.py by adding yourself to the file (no review needed). The commit-queue restarts itself every 2 hours. After restart the commit-queue will correctly respect your committer rights.' 135MOCK: update_work_items: commit-queue [106, 197] 136Feeding commit-queue items [106, 197] 137Feeding EWS (1 r? patch, 1 new) 138MOCK: submit_to_ews: 103 139""", 140 "handle_unexpected_error": "Mock error message\n", 141 } 142 self.assert_queue_outputs(queue, tool=tool, expected_stderr=expected_stderr) 143 144 145class AbstractPatchQueueTest(CommandsTest): 146 def test_next_patch(self): 147 queue = AbstractPatchQueue() 148 tool = MockTool() 149 queue.bind_to_tool(tool) 150 queue._options = Mock() 151 queue._options.port = None 152 self.assertEquals(queue._next_patch(), None) 153 tool.status_server = MockStatusServer(work_items=[2, 197]) 154 expected_stdout = "MOCK: fetch_attachment: 2 is not a known attachment id\n" # A mock-only message to prevent us from making mistakes. 155 expected_stderr = "MOCK: release_work_item: None 2\n" 156 patch_id = OutputCapture().assert_outputs(self, queue._next_patch, [], expected_stdout=expected_stdout, expected_stderr=expected_stderr) 157 self.assertEquals(patch_id, None) # 2 is an invalid patch id 158 self.assertEquals(queue._next_patch().id(), 197) 159 160 161class NeedsUpdateSequence(StepSequence): 162 def _run(self, tool, options, state): 163 raise CheckoutNeedsUpdate([], 1, "", None) 164 165 166class AlwaysCommitQueueTool(object): 167 def __init__(self): 168 self.status_server = MockStatusServer() 169 170 def command_by_name(self, name): 171 return CommitQueue 172 173 174class SecondThoughtsCommitQueue(CommitQueue): 175 def __init__(self): 176 self._reject_patch = False 177 CommitQueue.__init__(self) 178 179 def run_command(self, command): 180 # We want to reject the patch after the first validation, 181 # so wait to reject it until after some other command has run. 182 self._reject_patch = True 183 return CommitQueue.run_command(self, command) 184 185 def refetch_patch(self, patch): 186 if not self._reject_patch: 187 return self._tool.bugs.fetch_attachment(patch.id()) 188 189 attachment_dictionary = { 190 "id": patch.id(), 191 "bug_id": patch.bug_id(), 192 "name": "Rejected", 193 "is_obsolete": True, 194 "is_patch": False, 195 "review": "-", 196 "reviewer_email": "foo@bar.com", 197 "commit-queue": "-", 198 "committer_email": "foo@bar.com", 199 "attacher_email": "Contributer1", 200 } 201 return Attachment(attachment_dictionary, None) 202 203 204class CommitQueueTest(QueuesTest): 205 def _mock_test_result(self, testname): 206 return test_results.TestResult(testname, [test_failures.FailureTextMismatch()]) 207 208 def test_commit_queue(self): 209 expected_stderr = { 210 "begin_work_queue": self._default_begin_work_queue_stderr("commit-queue", MockSCM.fake_checkout_root), 211 "should_proceed_with_work_item": "MOCK: update_status: commit-queue Processing patch\n", 212 "next_work_item": "", 213 "process_work_item": """MOCK: update_status: commit-queue Cleaned working directory 214MOCK: update_status: commit-queue Updated working directory 215MOCK: update_status: commit-queue Applied patch 216MOCK: update_status: commit-queue Built patch 217MOCK: update_status: commit-queue Passed tests 218MOCK: update_status: commit-queue Landed patch 219MOCK: update_status: commit-queue Pass 220MOCK: release_work_item: commit-queue 197 221""", 222 "handle_unexpected_error": "MOCK setting flag 'commit-queue' to '-' on attachment '197' with comment 'Rejecting attachment 197 from commit-queue.' and additional comment 'Mock error message'\n", 223 "handle_script_error": "ScriptError error message\n", 224 } 225 self.assert_queue_outputs(CommitQueue(), expected_stderr=expected_stderr) 226 227 def test_commit_queue_failure(self): 228 expected_stderr = { 229 "begin_work_queue": self._default_begin_work_queue_stderr("commit-queue", MockSCM.fake_checkout_root), 230 "should_proceed_with_work_item": "MOCK: update_status: commit-queue Processing patch\n", 231 "next_work_item": "", 232 "process_work_item": """MOCK: update_status: commit-queue Cleaned working directory 233MOCK: update_status: commit-queue Updated working directory 234MOCK: update_status: commit-queue Patch does not apply 235MOCK setting flag 'commit-queue' to '-' on attachment '197' with comment 'Rejecting attachment 197 from commit-queue.' and additional comment 'MOCK script error' 236MOCK: update_status: commit-queue Fail 237MOCK: release_work_item: commit-queue 197 238""", 239 "handle_unexpected_error": "MOCK setting flag 'commit-queue' to '-' on attachment '197' with comment 'Rejecting attachment 197 from commit-queue.' and additional comment 'Mock error message'\n", 240 "handle_script_error": "ScriptError error message\n", 241 } 242 queue = CommitQueue() 243 244 def mock_run_webkit_patch(command): 245 if command == ['clean'] or command == ['update']: 246 # We want cleaning to succeed so we can error out on a step 247 # that causes the commit-queue to reject the patch. 248 return 249 raise ScriptError('MOCK script error') 250 251 queue.run_webkit_patch = mock_run_webkit_patch 252 self.assert_queue_outputs(queue, expected_stderr=expected_stderr) 253 254 def test_rollout(self): 255 tool = MockTool(log_executive=True) 256 tool.filesystem.write_text_file('/mock/results.html', '') # Otherwise the commit-queue will hit a KeyError trying to read the results from the MockFileSystem. 257 tool.buildbot.light_tree_on_fire() 258 expected_stderr = { 259 "begin_work_queue": self._default_begin_work_queue_stderr("commit-queue", MockSCM.fake_checkout_root), 260 "should_proceed_with_work_item": "MOCK: update_status: commit-queue Processing patch\n", 261 "next_work_item": "", 262 "process_work_item": """MOCK run_and_throw_if_fail: ['echo', '--status-host=example.com', 'clean'] 263MOCK: update_status: commit-queue Cleaned working directory 264MOCK run_and_throw_if_fail: ['echo', '--status-host=example.com', 'update'] 265MOCK: update_status: commit-queue Updated working directory 266MOCK run_and_throw_if_fail: ['echo', '--status-host=example.com', 'apply-attachment', '--no-update', '--non-interactive', 197] 267MOCK: update_status: commit-queue Applied patch 268MOCK run_and_throw_if_fail: ['echo', '--status-host=example.com', 'build', '--no-clean', '--no-update', '--build-style=both'] 269MOCK: update_status: commit-queue Built patch 270MOCK run_and_throw_if_fail: ['echo', '--status-host=example.com', 'build-and-test', '--no-clean', '--no-update', '--test', '--non-interactive'] 271MOCK: update_status: commit-queue Passed tests 272MOCK run_and_throw_if_fail: ['echo', '--status-host=example.com', 'land-attachment', '--force-clean', '--ignore-builders', '--non-interactive', '--parent-command=commit-queue', 197] 273MOCK: update_status: commit-queue Landed patch 274MOCK: update_status: commit-queue Pass 275MOCK: release_work_item: commit-queue 197 276""", 277 "handle_unexpected_error": "MOCK setting flag 'commit-queue' to '-' on attachment '197' with comment 'Rejecting attachment 197 from commit-queue.' and additional comment 'Mock error message'\n", 278 "handle_script_error": "ScriptError error message\n", 279 } 280 self.assert_queue_outputs(CommitQueue(), tool=tool, expected_stderr=expected_stderr) 281 282 def test_rollout_lands(self): 283 tool = MockTool(log_executive=True) 284 tool.buildbot.light_tree_on_fire() 285 rollout_patch = tool.bugs.fetch_attachment(106) # _patch6, a rollout patch. 286 assert(rollout_patch.is_rollout()) 287 expected_stderr = { 288 "begin_work_queue": self._default_begin_work_queue_stderr("commit-queue", MockSCM.fake_checkout_root), 289 "should_proceed_with_work_item": "MOCK: update_status: commit-queue Processing rollout patch\n", 290 "next_work_item": "", 291 "process_work_item": """MOCK run_and_throw_if_fail: ['echo', '--status-host=example.com', 'clean'] 292MOCK: update_status: commit-queue Cleaned working directory 293MOCK run_and_throw_if_fail: ['echo', '--status-host=example.com', 'update'] 294MOCK: update_status: commit-queue Updated working directory 295MOCK run_and_throw_if_fail: ['echo', '--status-host=example.com', 'apply-attachment', '--no-update', '--non-interactive', 106] 296MOCK: update_status: commit-queue Applied patch 297MOCK run_and_throw_if_fail: ['echo', '--status-host=example.com', 'land-attachment', '--force-clean', '--ignore-builders', '--non-interactive', '--parent-command=commit-queue', 106] 298MOCK: update_status: commit-queue Landed patch 299MOCK: update_status: commit-queue Pass 300MOCK: release_work_item: commit-queue 106 301""", 302 "handle_unexpected_error": "MOCK setting flag 'commit-queue' to '-' on attachment '106' with comment 'Rejecting attachment 106 from commit-queue.' and additional comment 'Mock error message'\n", 303 "handle_script_error": "ScriptError error message\n", 304 } 305 self.assert_queue_outputs(CommitQueue(), tool=tool, work_item=rollout_patch, expected_stderr=expected_stderr) 306 307 def test_auto_retry(self): 308 queue = CommitQueue() 309 options = Mock() 310 options.parent_command = "commit-queue" 311 tool = AlwaysCommitQueueTool() 312 sequence = NeedsUpdateSequence(None) 313 314 expected_stderr = "Commit failed because the checkout is out of date. Please update and try again.\nMOCK: update_status: commit-queue Tests passed, but commit failed (checkout out of date). Updating, then landing without building or re-running tests.\n" 315 state = {'patch': None} 316 OutputCapture().assert_outputs(self, sequence.run_and_handle_errors, [tool, options, state], expected_exception=TryAgain, expected_stderr=expected_stderr) 317 318 self.assertEquals(options.update, True) 319 self.assertEquals(options.build, False) 320 self.assertEquals(options.test, False) 321 322 def test_manual_reject_during_processing(self): 323 queue = SecondThoughtsCommitQueue() 324 queue.bind_to_tool(MockTool()) 325 queue._tool.filesystem.write_text_file('/mock/results.html', '') # Otherwise the commit-queue will hit a KeyError trying to read the results from the MockFileSystem. 326 queue._options = Mock() 327 queue._options.port = None 328 expected_stderr = """MOCK: update_status: commit-queue Cleaned working directory 329MOCK: update_status: commit-queue Updated working directory 330MOCK: update_status: commit-queue Applied patch 331MOCK: update_status: commit-queue Built patch 332MOCK: update_status: commit-queue Passed tests 333MOCK: update_status: commit-queue Retry 334MOCK: release_work_item: commit-queue 197 335""" 336 OutputCapture().assert_outputs(self, queue.process_work_item, [QueuesTest.mock_work_item], expected_stderr=expected_stderr) 337 338 def test_report_flaky_tests(self): 339 queue = CommitQueue() 340 queue.bind_to_tool(MockTool()) 341 expected_stderr = """MOCK bug comment: bug_id=76, cc=None 342--- Begin comment --- 343The commit-queue just saw foo/bar.html flake (Text diff mismatch) while processing attachment 197 on bug 42. 344Port: MockPort Platform: MockPlatform 1.0 345--- End comment --- 346 347MOCK add_attachment_to_bug: bug_id=76, description=Failure diff from bot filename=failure.diff 348MOCK bug comment: bug_id=76, cc=None 349--- Begin comment --- 350The commit-queue just saw bar/baz.html flake (Text diff mismatch) while processing attachment 197 on bug 42. 351Port: MockPort Platform: MockPlatform 1.0 352--- End comment --- 353 354MOCK add_attachment_to_bug: bug_id=76, description=Archive of layout-test-results from bot filename=layout-test-results.zip 355MOCK bug comment: bug_id=42, cc=None 356--- Begin comment --- 357The commit-queue encountered the following flaky tests while processing attachment 197: 358 359foo/bar.html bug 76 (author: abarth@webkit.org) 360bar/baz.html bug 76 (author: abarth@webkit.org) 361The commit-queue is continuing to process your patch. 362--- End comment --- 363 364""" 365 test_names = ["foo/bar.html", "bar/baz.html"] 366 test_results = [self._mock_test_result(name) for name in test_names] 367 368 class MockZipFile(object): 369 def __init__(self): 370 self.fp = StringIO() 371 372 def read(self, path): 373 return "" 374 375 def namelist(self): 376 # This is intentionally missing one diffs.txt to exercise the "upload the whole zip" codepath. 377 return ['foo/bar-diffs.txt'] 378 379 OutputCapture().assert_outputs(self, queue.report_flaky_tests, [QueuesTest.mock_work_item, test_results, MockZipFile()], expected_stderr=expected_stderr) 380 381 def test_missing_layout_test_results(self): 382 queue = CommitQueue() 383 tool = MockTool() 384 results_path = '/mock/results.html' 385 tool.filesystem = MockFileSystem({results_path: None}) 386 queue.bind_to_tool(tool) 387 # Make sure that our filesystem mock functions as we expect. 388 self.assertRaises(IOError, tool.filesystem.read_text_file, results_path) 389 # layout_test_results shouldn't raise even if the results.html file is missing. 390 self.assertEquals(queue.layout_test_results(), None) 391 392 def test_layout_test_results(self): 393 queue = CommitQueue() 394 queue.bind_to_tool(MockTool()) 395 queue._read_file_contents = lambda path: None 396 self.assertEquals(queue.layout_test_results(), None) 397 queue._read_file_contents = lambda path: "" 398 self.assertEquals(queue.layout_test_results(), None) 399 queue._create_layout_test_results = lambda: LayoutTestResults([]) 400 results = queue.layout_test_results() 401 self.assertNotEquals(results, None) 402 self.assertEquals(results.failure_limit_count(), 10) # This value matches RunTests.NON_INTERACTIVE_FAILURE_LIMIT_COUNT 403 404 def test_archive_last_layout_test_results(self): 405 queue = CommitQueue() 406 queue.bind_to_tool(MockTool()) 407 patch = queue._tool.bugs.fetch_attachment(128) 408 # This is just to test that the method doesn't raise. 409 queue.archive_last_layout_test_results(patch) 410 411 def test_upload_results_archive_for_patch(self): 412 queue = CommitQueue() 413 queue.bind_to_tool(MockTool()) 414 patch = queue._tool.bugs.fetch_attachment(128) 415 expected_stderr = """MOCK add_attachment_to_bug: bug_id=42, description=Archive of layout-test-results from bot filename=layout-test-results.zip 416-- Begin comment -- 417The attached test failures were seen while running run-webkit-tests on the commit-queue. 418Port: MockPort Platform: MockPlatform 1.0 419-- End comment -- 420""" 421 OutputCapture().assert_outputs(self, queue._upload_results_archive_for_patch, [patch, Mock()], expected_stderr=expected_stderr) 422 423 424class StyleQueueTest(QueuesTest): 425 def test_style_queue(self): 426 expected_stderr = { 427 "begin_work_queue": self._default_begin_work_queue_stderr("style-queue", MockSCM.fake_checkout_root), 428 "next_work_item": "", 429 "should_proceed_with_work_item": "MOCK: update_status: style-queue Checking style\n", 430 "process_work_item": "MOCK: update_status: style-queue Pass\nMOCK: release_work_item: style-queue 197\n", 431 "handle_unexpected_error": "Mock error message\n", 432 "handle_script_error": "MOCK: update_status: style-queue ScriptError error message\nMOCK bug comment: bug_id=42, cc=[]\n--- Begin comment ---\nAttachment 197 did not pass style-queue:\n\nScriptError error message\n\nIf any of these errors are false positives, please file a bug against check-webkit-style.\n--- End comment ---\n\n", 433 } 434 expected_exceptions = { 435 "handle_script_error": SystemExit, 436 } 437 self.assert_queue_outputs(StyleQueue(), expected_stderr=expected_stderr, expected_exceptions=expected_exceptions) 438