1# Copyright (c) 2010 Google Inc. All rights reserved. 2# 3# Redistribution and use in source and binary forms, with or without 4# modification, are permitted provided that the following conditions are 5# met: 6# 7# * Redistributions of source code must retain the above copyright 8# notice, this list of conditions and the following disclaimer. 9# * Redistributions in binary form must reproduce the above 10# copyright notice, this list of conditions and the following disclaimer 11# in the documentation and/or other materials provided with the 12# distribution. 13# * Neither the name of Google Inc. nor the names of its 14# contributors may be used to endorse or promote products derived from 15# this software without specific prior written permission. 16# 17# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 18# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 19# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 20# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 21# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 22# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 23# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 24# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 25# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 29from webkitpy.common.system.executive import ScriptError 30from webkitpy.common.net.layouttestresults import LayoutTestResults 31from webkitpy.tool.bot.expectedfailures import ExpectedFailures 32 33 34class CommitQueueTaskDelegate(object): 35 def run_command(self, command): 36 raise NotImplementedError("subclasses must implement") 37 38 def command_passed(self, message, patch): 39 raise NotImplementedError("subclasses must implement") 40 41 def command_failed(self, message, script_error, patch): 42 raise NotImplementedError("subclasses must implement") 43 44 def refetch_patch(self, patch): 45 raise NotImplementedError("subclasses must implement") 46 47 def layout_test_results(self): 48 raise NotImplementedError("subclasses must implement") 49 50 def archive_last_layout_test_results(self, patch): 51 raise NotImplementedError("subclasses must implement") 52 53 # We could make results_archive optional, but for now it's required. 54 def report_flaky_tests(self, patch, flaky_tests, results_archive): 55 raise NotImplementedError("subclasses must implement") 56 57 58class CommitQueueTask(object): 59 def __init__(self, delegate, patch): 60 self._delegate = delegate 61 self._patch = patch 62 self._script_error = None 63 self._results_archive_from_patch_test_run = None 64 self._expected_failures = ExpectedFailures() 65 66 def _validate(self): 67 # Bugs might get closed, or patches might be obsoleted or r-'d while the 68 # commit-queue is processing. 69 self._patch = self._delegate.refetch_patch(self._patch) 70 if self._patch.is_obsolete(): 71 return False 72 if self._patch.bug().is_closed(): 73 return False 74 if not self._patch.committer(): 75 return False 76 if not self._patch.review() != "-": 77 return False 78 # Reviewer is not required. Missing reviewers will be caught during 79 # the ChangeLog check during landing. 80 return True 81 82 def _run_command(self, command, success_message, failure_message): 83 try: 84 self._delegate.run_command(command) 85 self._delegate.command_passed(success_message, patch=self._patch) 86 return True 87 except ScriptError, e: 88 self._script_error = e 89 self.failure_status_id = self._delegate.command_failed(failure_message, script_error=self._script_error, patch=self._patch) 90 return False 91 92 def _clean(self): 93 return self._run_command([ 94 "clean", 95 ], 96 "Cleaned working directory", 97 "Unable to clean working directory") 98 99 def _update(self): 100 # FIXME: Ideally the status server log message should include which revision we updated to. 101 return self._run_command([ 102 "update", 103 ], 104 "Updated working directory", 105 "Unable to update working directory") 106 107 def _apply(self): 108 return self._run_command([ 109 "apply-attachment", 110 "--no-update", 111 "--non-interactive", 112 self._patch.id(), 113 ], 114 "Applied patch", 115 "Patch does not apply") 116 117 def _build(self): 118 return self._run_command([ 119 "build", 120 "--no-clean", 121 "--no-update", 122 "--build-style=both", 123 ], 124 "Built patch", 125 "Patch does not build") 126 127 def _build_without_patch(self): 128 return self._run_command([ 129 "build", 130 "--force-clean", 131 "--no-update", 132 "--build-style=both", 133 ], 134 "Able to build without patch", 135 "Unable to build without patch") 136 137 def _test(self): 138 success = self._run_command([ 139 "build-and-test", 140 "--no-clean", 141 "--no-update", 142 # Notice that we don't pass --build, which means we won't build! 143 "--test", 144 "--non-interactive", 145 ], 146 "Passed tests", 147 "Patch does not pass tests") 148 149 self._expected_failures.shrink_expected_failures(self._delegate.layout_test_results(), success) 150 return success 151 152 def _build_and_test_without_patch(self): 153 success = self._run_command([ 154 "build-and-test", 155 "--force-clean", 156 "--no-update", 157 "--build", 158 "--test", 159 "--non-interactive", 160 ], 161 "Able to pass tests without patch", 162 "Unable to pass tests without patch (tree is red?)") 163 164 self._expected_failures.shrink_expected_failures(self._delegate.layout_test_results(), success) 165 return success 166 167 def _land(self): 168 # Unclear if this should pass --quiet or not. If --parent-command always does the reporting, then it should. 169 return self._run_command([ 170 "land-attachment", 171 "--force-clean", 172 "--ignore-builders", 173 "--non-interactive", 174 "--parent-command=commit-queue", 175 self._patch.id(), 176 ], 177 "Landed patch", 178 "Unable to land patch") 179 180 def _report_flaky_tests(self, flaky_test_results, results_archive): 181 self._delegate.report_flaky_tests(self._patch, flaky_test_results, results_archive) 182 183 def _results_failed_different_tests(self, first, second): 184 first_failing_tests = [] if not first else first.failing_tests() 185 second_failing_tests = [] if not second else second.failing_tests() 186 return first_failing_tests != second_failing_tests 187 188 def _test_patch(self): 189 if self._test(): 190 return True 191 192 # Note: archive_last_layout_test_results deletes the results directory, making these calls order-sensitve. 193 # We could remove this dependency by building the layout_test_results from the archive. 194 first_results = self._delegate.layout_test_results() 195 first_results_archive = self._delegate.archive_last_layout_test_results(self._patch) 196 197 if self._expected_failures.failures_were_expected(first_results): 198 return True 199 200 if self._test(): 201 # Only report flaky tests if we were successful at parsing results.html and archiving results. 202 if first_results and first_results_archive: 203 self._report_flaky_tests(first_results.failing_test_results(), first_results_archive) 204 return True 205 206 second_results = self._delegate.layout_test_results() 207 if self._results_failed_different_tests(first_results, second_results): 208 # We could report flaky tests here, but we would need to be careful 209 # to use similar checks to ExpectedFailures._can_trust_results 210 # to make sure we don't report constant failures as flakes when 211 # we happen to hit the --exit-after-N-failures limit. 212 # See https://bugs.webkit.org/show_bug.cgi?id=51272 213 return False 214 215 # Archive (and remove) second results so layout_test_results() after 216 # build_and_test_without_patch won't use second results instead of the clean-tree results. 217 second_results_archive = self._delegate.archive_last_layout_test_results(self._patch) 218 219 if self._build_and_test_without_patch(): 220 # The error from the previous ._test() run is real, report it. 221 return self.report_failure(first_results_archive) 222 223 clean_tree_results = self._delegate.layout_test_results() 224 self._expected_failures.grow_expected_failures(clean_tree_results) 225 226 return False # Tree must be redder than we expected, just retry later. 227 228 def results_archive_from_patch_test_run(self, patch): 229 assert(self._patch.id() == patch.id()) # CommitQueueTask is not currently re-useable. 230 return self._results_archive_from_patch_test_run 231 232 def report_failure(self, results_archive=None): 233 if not self._validate(): 234 return False 235 self._results_archive_from_patch_test_run = results_archive 236 raise self._script_error 237 238 def run(self): 239 if not self._validate(): 240 return False 241 if not self._clean(): 242 return False 243 if not self._update(): 244 return False 245 if not self._apply(): 246 return self.report_failure() 247 if not self._patch.is_rollout(): 248 if not self._build(): 249 if not self._build_without_patch(): 250 return False 251 return self.report_failure() 252 if not self._test_patch(): 253 return False 254 # Make sure the patch is still valid before landing (e.g., make sure 255 # no one has set commit-queue- since we started working on the patch.) 256 if not self._validate(): 257 return False 258 # FIXME: We should understand why the land failure occured and retry if possible. 259 if not self._land(): 260 return self.report_failure() 261 return True 262