1# Copyright (c) 2010 Google Inc. All rights reserved. 2# 3# Redistribution and use in source and binary forms, with or without 4# modification, are permitted provided that the following conditions are 5# met: 6# 7# * Redistributions of source code must retain the above copyright 8# notice, this list of conditions and the following disclaimer. 9# * Redistributions in binary form must reproduce the above 10# copyright notice, this list of conditions and the following disclaimer 11# in the documentation and/or other materials provided with the 12# distribution. 13# * Neither the name of Google Inc. nor the names of its 14# contributors may be used to endorse or promote products derived from 15# this software without specific prior written permission. 16# 17# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 18# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 19# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 20# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 21# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 22# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 23# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 24# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 25# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 29from datetime import datetime 30import unittest 31 32from webkitpy.common.net import bugzilla 33from webkitpy.common.net.layouttestresults import LayoutTestResults 34from webkitpy.common.system.deprecated_logging import error, log 35from webkitpy.common.system.outputcapture import OutputCapture 36from webkitpy.layout_tests.layout_package import test_results 37from webkitpy.layout_tests.layout_package import test_failures 38from webkitpy.thirdparty.mock import Mock 39from webkitpy.tool.bot.commitqueuetask import * 40from webkitpy.tool.mocktool import MockTool 41 42 43class MockCommitQueue(CommitQueueTaskDelegate): 44 def __init__(self, error_plan): 45 self._error_plan = error_plan 46 47 def run_command(self, command): 48 log("run_webkit_patch: %s" % command) 49 if self._error_plan: 50 error = self._error_plan.pop(0) 51 if error: 52 raise error 53 54 def command_passed(self, success_message, patch): 55 log("command_passed: success_message='%s' patch='%s'" % ( 56 success_message, patch.id())) 57 58 def command_failed(self, failure_message, script_error, patch): 59 log("command_failed: failure_message='%s' script_error='%s' patch='%s'" % ( 60 failure_message, script_error, patch.id())) 61 return 3947 62 63 def refetch_patch(self, patch): 64 return patch 65 66 def layout_test_results(self): 67 return None 68 69 def report_flaky_tests(self, patch, flaky_results, results_archive): 70 flaky_tests = [result.filename for result in flaky_results] 71 log("report_flaky_tests: patch='%s' flaky_tests='%s' archive='%s'" % (patch.id(), flaky_tests, results_archive.filename)) 72 73 def archive_last_layout_test_results(self, patch): 74 log("archive_last_layout_test_results: patch='%s'" % patch.id()) 75 archive = Mock() 76 archive.filename = "mock-archive-%s.zip" % patch.id() 77 return archive 78 79 80class CommitQueueTaskTest(unittest.TestCase): 81 def _run_through_task(self, commit_queue, expected_stderr, expected_exception=None, expect_retry=False): 82 tool = MockTool(log_executive=True) 83 patch = tool.bugs.fetch_attachment(197) 84 task = CommitQueueTask(commit_queue, patch) 85 success = OutputCapture().assert_outputs(self, task.run, expected_stderr=expected_stderr, expected_exception=expected_exception) 86 if not expected_exception: 87 self.assertEqual(success, not expect_retry) 88 89 def test_success_case(self): 90 commit_queue = MockCommitQueue([]) 91 expected_stderr = """run_webkit_patch: ['clean'] 92command_passed: success_message='Cleaned working directory' patch='197' 93run_webkit_patch: ['update'] 94command_passed: success_message='Updated working directory' patch='197' 95run_webkit_patch: ['apply-attachment', '--no-update', '--non-interactive', 197] 96command_passed: success_message='Applied patch' patch='197' 97run_webkit_patch: ['build', '--no-clean', '--no-update', '--build-style=both'] 98command_passed: success_message='Built patch' patch='197' 99run_webkit_patch: ['build-and-test', '--no-clean', '--no-update', '--test', '--non-interactive'] 100command_passed: success_message='Passed tests' patch='197' 101run_webkit_patch: ['land-attachment', '--force-clean', '--ignore-builders', '--non-interactive', '--parent-command=commit-queue', 197] 102command_passed: success_message='Landed patch' patch='197' 103""" 104 self._run_through_task(commit_queue, expected_stderr) 105 106 def test_clean_failure(self): 107 commit_queue = MockCommitQueue([ 108 ScriptError("MOCK clean failure"), 109 ]) 110 expected_stderr = """run_webkit_patch: ['clean'] 111command_failed: failure_message='Unable to clean working directory' script_error='MOCK clean failure' patch='197' 112""" 113 self._run_through_task(commit_queue, expected_stderr, expect_retry=True) 114 115 def test_update_failure(self): 116 commit_queue = MockCommitQueue([ 117 None, 118 ScriptError("MOCK update failure"), 119 ]) 120 expected_stderr = """run_webkit_patch: ['clean'] 121command_passed: success_message='Cleaned working directory' patch='197' 122run_webkit_patch: ['update'] 123command_failed: failure_message='Unable to update working directory' script_error='MOCK update failure' patch='197' 124""" 125 self._run_through_task(commit_queue, expected_stderr, expect_retry=True) 126 127 def test_apply_failure(self): 128 commit_queue = MockCommitQueue([ 129 None, 130 None, 131 ScriptError("MOCK apply failure"), 132 ]) 133 expected_stderr = """run_webkit_patch: ['clean'] 134command_passed: success_message='Cleaned working directory' patch='197' 135run_webkit_patch: ['update'] 136command_passed: success_message='Updated working directory' patch='197' 137run_webkit_patch: ['apply-attachment', '--no-update', '--non-interactive', 197] 138command_failed: failure_message='Patch does not apply' script_error='MOCK apply failure' patch='197' 139""" 140 self._run_through_task(commit_queue, expected_stderr, ScriptError) 141 142 def test_build_failure(self): 143 commit_queue = MockCommitQueue([ 144 None, 145 None, 146 None, 147 ScriptError("MOCK build failure"), 148 ]) 149 expected_stderr = """run_webkit_patch: ['clean'] 150command_passed: success_message='Cleaned working directory' patch='197' 151run_webkit_patch: ['update'] 152command_passed: success_message='Updated working directory' patch='197' 153run_webkit_patch: ['apply-attachment', '--no-update', '--non-interactive', 197] 154command_passed: success_message='Applied patch' patch='197' 155run_webkit_patch: ['build', '--no-clean', '--no-update', '--build-style=both'] 156command_failed: failure_message='Patch does not build' script_error='MOCK build failure' patch='197' 157run_webkit_patch: ['build', '--force-clean', '--no-update', '--build-style=both'] 158command_passed: success_message='Able to build without patch' patch='197' 159""" 160 self._run_through_task(commit_queue, expected_stderr, ScriptError) 161 162 def test_red_build_failure(self): 163 commit_queue = MockCommitQueue([ 164 None, 165 None, 166 None, 167 ScriptError("MOCK build failure"), 168 ScriptError("MOCK clean build failure"), 169 ]) 170 expected_stderr = """run_webkit_patch: ['clean'] 171command_passed: success_message='Cleaned working directory' patch='197' 172run_webkit_patch: ['update'] 173command_passed: success_message='Updated working directory' patch='197' 174run_webkit_patch: ['apply-attachment', '--no-update', '--non-interactive', 197] 175command_passed: success_message='Applied patch' patch='197' 176run_webkit_patch: ['build', '--no-clean', '--no-update', '--build-style=both'] 177command_failed: failure_message='Patch does not build' script_error='MOCK build failure' patch='197' 178run_webkit_patch: ['build', '--force-clean', '--no-update', '--build-style=both'] 179command_failed: failure_message='Unable to build without patch' script_error='MOCK clean build failure' patch='197' 180""" 181 self._run_through_task(commit_queue, expected_stderr, expect_retry=True) 182 183 def test_flaky_test_failure(self): 184 commit_queue = MockCommitQueue([ 185 None, 186 None, 187 None, 188 None, 189 ScriptError("MOCK tests failure"), 190 ]) 191 # CommitQueueTask will only report flaky tests if we successfully parsed 192 # results.html and returned a LayoutTestResults object, so we fake one. 193 commit_queue.layout_test_results = lambda: LayoutTestResults([]) 194 expected_stderr = """run_webkit_patch: ['clean'] 195command_passed: success_message='Cleaned working directory' patch='197' 196run_webkit_patch: ['update'] 197command_passed: success_message='Updated working directory' patch='197' 198run_webkit_patch: ['apply-attachment', '--no-update', '--non-interactive', 197] 199command_passed: success_message='Applied patch' patch='197' 200run_webkit_patch: ['build', '--no-clean', '--no-update', '--build-style=both'] 201command_passed: success_message='Built patch' patch='197' 202run_webkit_patch: ['build-and-test', '--no-clean', '--no-update', '--test', '--non-interactive'] 203command_failed: failure_message='Patch does not pass tests' script_error='MOCK tests failure' patch='197' 204archive_last_layout_test_results: patch='197' 205run_webkit_patch: ['build-and-test', '--no-clean', '--no-update', '--test', '--non-interactive'] 206command_passed: success_message='Passed tests' patch='197' 207report_flaky_tests: patch='197' flaky_tests='[]' archive='mock-archive-197.zip' 208run_webkit_patch: ['land-attachment', '--force-clean', '--ignore-builders', '--non-interactive', '--parent-command=commit-queue', 197] 209command_passed: success_message='Landed patch' patch='197' 210""" 211 self._run_through_task(commit_queue, expected_stderr) 212 213 def test_failed_archive(self): 214 commit_queue = MockCommitQueue([ 215 None, 216 None, 217 None, 218 None, 219 ScriptError("MOCK tests failure"), 220 ]) 221 commit_queue.layout_test_results = lambda: LayoutTestResults([]) 222 # It's possible delegate to fail to archive layout tests, don't try to report 223 # flaky tests when that happens. 224 commit_queue.archive_last_layout_test_results = lambda patch: None 225 expected_stderr = """run_webkit_patch: ['clean'] 226command_passed: success_message='Cleaned working directory' patch='197' 227run_webkit_patch: ['update'] 228command_passed: success_message='Updated working directory' patch='197' 229run_webkit_patch: ['apply-attachment', '--no-update', '--non-interactive', 197] 230command_passed: success_message='Applied patch' patch='197' 231run_webkit_patch: ['build', '--no-clean', '--no-update', '--build-style=both'] 232command_passed: success_message='Built patch' patch='197' 233run_webkit_patch: ['build-and-test', '--no-clean', '--no-update', '--test', '--non-interactive'] 234command_failed: failure_message='Patch does not pass tests' script_error='MOCK tests failure' patch='197' 235run_webkit_patch: ['build-and-test', '--no-clean', '--no-update', '--test', '--non-interactive'] 236command_passed: success_message='Passed tests' patch='197' 237run_webkit_patch: ['land-attachment', '--force-clean', '--ignore-builders', '--non-interactive', '--parent-command=commit-queue', 197] 238command_passed: success_message='Landed patch' patch='197' 239""" 240 self._run_through_task(commit_queue, expected_stderr) 241 242 def test_double_flaky_test_failure(self): 243 class DoubleFlakyCommitQueue(MockCommitQueue): 244 def __init__(self, error_plan): 245 MockCommitQueue.__init__(self, error_plan) 246 self._double_flaky_test_counter = 0 247 248 def run_command(self, command): 249 self._double_flaky_test_counter += 1 250 MockCommitQueue.run_command(self, command) 251 252 def _mock_test_result(self, testname): 253 return test_results.TestResult(testname, [test_failures.FailureTextMismatch()]) 254 255 def layout_test_results(self): 256 if self._double_flaky_test_counter % 2: 257 return LayoutTestResults([self._mock_test_result('foo.html')]) 258 return LayoutTestResults([self._mock_test_result('bar.html')]) 259 260 commit_queue = DoubleFlakyCommitQueue([ 261 None, 262 None, 263 None, 264 None, 265 ScriptError("MOCK test failure"), 266 ScriptError("MOCK test failure again"), 267 ]) 268 # The (subtle) point of this test is that report_flaky_tests does not appear 269 # in the expected_stderr for this run. 270 # Note also that there is no attempt to run the tests w/o the patch. 271 expected_stderr = """run_webkit_patch: ['clean'] 272command_passed: success_message='Cleaned working directory' patch='197' 273run_webkit_patch: ['update'] 274command_passed: success_message='Updated working directory' patch='197' 275run_webkit_patch: ['apply-attachment', '--no-update', '--non-interactive', 197] 276command_passed: success_message='Applied patch' patch='197' 277run_webkit_patch: ['build', '--no-clean', '--no-update', '--build-style=both'] 278command_passed: success_message='Built patch' patch='197' 279run_webkit_patch: ['build-and-test', '--no-clean', '--no-update', '--test', '--non-interactive'] 280command_failed: failure_message='Patch does not pass tests' script_error='MOCK test failure' patch='197' 281archive_last_layout_test_results: patch='197' 282run_webkit_patch: ['build-and-test', '--no-clean', '--no-update', '--test', '--non-interactive'] 283command_failed: failure_message='Patch does not pass tests' script_error='MOCK test failure again' patch='197' 284""" 285 tool = MockTool(log_executive=True) 286 patch = tool.bugs.fetch_attachment(197) 287 task = CommitQueueTask(commit_queue, patch) 288 success = OutputCapture().assert_outputs(self, task.run, expected_stderr=expected_stderr) 289 self.assertEqual(success, False) 290 291 def test_test_failure(self): 292 commit_queue = MockCommitQueue([ 293 None, 294 None, 295 None, 296 None, 297 ScriptError("MOCK test failure"), 298 ScriptError("MOCK test failure again"), 299 ]) 300 expected_stderr = """run_webkit_patch: ['clean'] 301command_passed: success_message='Cleaned working directory' patch='197' 302run_webkit_patch: ['update'] 303command_passed: success_message='Updated working directory' patch='197' 304run_webkit_patch: ['apply-attachment', '--no-update', '--non-interactive', 197] 305command_passed: success_message='Applied patch' patch='197' 306run_webkit_patch: ['build', '--no-clean', '--no-update', '--build-style=both'] 307command_passed: success_message='Built patch' patch='197' 308run_webkit_patch: ['build-and-test', '--no-clean', '--no-update', '--test', '--non-interactive'] 309command_failed: failure_message='Patch does not pass tests' script_error='MOCK test failure' patch='197' 310archive_last_layout_test_results: patch='197' 311run_webkit_patch: ['build-and-test', '--no-clean', '--no-update', '--test', '--non-interactive'] 312command_failed: failure_message='Patch does not pass tests' script_error='MOCK test failure again' patch='197' 313archive_last_layout_test_results: patch='197' 314run_webkit_patch: ['build-and-test', '--force-clean', '--no-update', '--build', '--test', '--non-interactive'] 315command_passed: success_message='Able to pass tests without patch' patch='197' 316""" 317 self._run_through_task(commit_queue, expected_stderr, ScriptError) 318 319 def test_red_test_failure(self): 320 commit_queue = MockCommitQueue([ 321 None, 322 None, 323 None, 324 None, 325 ScriptError("MOCK test failure"), 326 ScriptError("MOCK test failure again"), 327 ScriptError("MOCK clean test failure"), 328 ]) 329 expected_stderr = """run_webkit_patch: ['clean'] 330command_passed: success_message='Cleaned working directory' patch='197' 331run_webkit_patch: ['update'] 332command_passed: success_message='Updated working directory' patch='197' 333run_webkit_patch: ['apply-attachment', '--no-update', '--non-interactive', 197] 334command_passed: success_message='Applied patch' patch='197' 335run_webkit_patch: ['build', '--no-clean', '--no-update', '--build-style=both'] 336command_passed: success_message='Built patch' patch='197' 337run_webkit_patch: ['build-and-test', '--no-clean', '--no-update', '--test', '--non-interactive'] 338command_failed: failure_message='Patch does not pass tests' script_error='MOCK test failure' patch='197' 339archive_last_layout_test_results: patch='197' 340run_webkit_patch: ['build-and-test', '--no-clean', '--no-update', '--test', '--non-interactive'] 341command_failed: failure_message='Patch does not pass tests' script_error='MOCK test failure again' patch='197' 342archive_last_layout_test_results: patch='197' 343run_webkit_patch: ['build-and-test', '--force-clean', '--no-update', '--build', '--test', '--non-interactive'] 344command_failed: failure_message='Unable to pass tests without patch (tree is red?)' script_error='MOCK clean test failure' patch='197' 345""" 346 self._run_through_task(commit_queue, expected_stderr, expect_retry=True) 347 348 def test_land_failure(self): 349 commit_queue = MockCommitQueue([ 350 None, 351 None, 352 None, 353 None, 354 None, 355 ScriptError("MOCK land failure"), 356 ]) 357 expected_stderr = """run_webkit_patch: ['clean'] 358command_passed: success_message='Cleaned working directory' patch='197' 359run_webkit_patch: ['update'] 360command_passed: success_message='Updated working directory' patch='197' 361run_webkit_patch: ['apply-attachment', '--no-update', '--non-interactive', 197] 362command_passed: success_message='Applied patch' patch='197' 363run_webkit_patch: ['build', '--no-clean', '--no-update', '--build-style=both'] 364command_passed: success_message='Built patch' patch='197' 365run_webkit_patch: ['build-and-test', '--no-clean', '--no-update', '--test', '--non-interactive'] 366command_passed: success_message='Passed tests' patch='197' 367run_webkit_patch: ['land-attachment', '--force-clean', '--ignore-builders', '--non-interactive', '--parent-command=commit-queue', 197] 368command_failed: failure_message='Unable to land patch' script_error='MOCK land failure' patch='197' 369""" 370 # FIXME: This should really be expect_retry=True for a better user experiance. 371 self._run_through_task(commit_queue, expected_stderr, ScriptError) 372 373 def _expect_validate(self, patch, is_valid): 374 class MockDelegate(object): 375 def refetch_patch(self, patch): 376 return patch 377 378 task = CommitQueueTask(MockDelegate(), patch) 379 self.assertEquals(task._validate(), is_valid) 380 381 def _mock_patch(self, attachment_dict={}, bug_dict={'bug_status': 'NEW'}, committer="fake"): 382 bug = bugzilla.Bug(bug_dict, None) 383 patch = bugzilla.Attachment(attachment_dict, bug) 384 patch._committer = committer 385 return patch 386 387 def test_validate(self): 388 self._expect_validate(self._mock_patch(), True) 389 self._expect_validate(self._mock_patch({'is_obsolete': True}), False) 390 self._expect_validate(self._mock_patch(bug_dict={'bug_status': 'CLOSED'}), False) 391 self._expect_validate(self._mock_patch(committer=None), False) 392 self._expect_validate(self._mock_patch({'review': '-'}), False) 393