1# Copyright (C) 2010 Google Inc. All rights reserved. 2# 3# Redistribution and use in source and binary forms, with or without 4# modification, are permitted provided that the following conditions are 5# met: 6# 7# * Redistributions of source code must retain the above copyright 8# notice, this list of conditions and the following disclaimer. 9# * Redistributions in binary form must reproduce the above 10# copyright notice, this list of conditions and the following disclaimer 11# in the documentation and/or other materials provided with the 12# distribution. 13# * Neither the name of Google Inc. nor the names of its 14# contributors may be used to endorse or promote products derived from 15# this software without specific prior written permission. 16# 17# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 18# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 19# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 20# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 21# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 22# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 23# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 24# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 25# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 29import webkitpy.thirdparty.unittest2 as unittest 30 31from webkitpy.common.checkout.baselineoptimizer import BaselineOptimizer 32from webkitpy.common.checkout.scm.scm_mock import MockSCM 33from webkitpy.common.host_mock import MockHost 34from webkitpy.common.net.buildbot.buildbot_mock import MockBuilder 35from webkitpy.common.net.layouttestresults import LayoutTestResults 36from webkitpy.common.system.executive_mock import MockExecutive 37from webkitpy.common.system.executive_mock import MockExecutive2 38from webkitpy.common.system.outputcapture import OutputCapture 39from webkitpy.thirdparty.mock import Mock 40from webkitpy.tool.commands.rebaseline import * 41from webkitpy.tool.mocktool import MockTool, MockOptions 42 43 44class _BaseTestCase(unittest.TestCase): 45 MOCK_WEB_RESULT = 'MOCK Web result, convert 404 to None=True' 46 WEB_PREFIX = 'http://example.com/f/builders/WebKit Mac10.7/results/layout-test-results' 47 48 command_constructor = None 49 50 def setUp(self): 51 self.tool = MockTool() 52 self.command = self.command_constructor() # lint warns that command_constructor might not be set, but this is intentional; pylint: disable=E1102 53 self.command.bind_to_tool(self.tool) 54 self.lion_port = self.tool.port_factory.get_from_builder_name("WebKit Mac10.7") 55 self.lion_expectations_path = self.lion_port.path_to_generic_test_expectations_file() 56 57 # FIXME: crbug.com/279494. We should override builders._exact_matches 58 # here to point to a set of test ports and restore the value in 59 # tearDown(), and that way the individual tests wouldn't have to worry 60 # about it. 61 62 def _expand(self, path): 63 if self.tool.filesystem.isabs(path): 64 return path 65 return self.tool.filesystem.join(self.lion_port.layout_tests_dir(), path) 66 67 def _read(self, path): 68 return self.tool.filesystem.read_text_file(self._expand(path)) 69 70 def _write(self, path, contents): 71 self.tool.filesystem.write_text_file(self._expand(path), contents) 72 73 def _zero_out_test_expectations(self): 74 for port_name in self.tool.port_factory.all_port_names(): 75 port = self.tool.port_factory.get(port_name) 76 for path in port.expectations_files(): 77 self._write(path, '') 78 self.tool.filesystem.written_files = {} 79 80 def _setup_mock_builder_data(self): 81 data = LayoutTestResults.results_from_string("""ADD_RESULTS({ 82 "tests": { 83 "userscripts": { 84 "first-test.html": { 85 "expected": "PASS", 86 "actual": "IMAGE+TEXT" 87 }, 88 "second-test.html": { 89 "expected": "FAIL", 90 "actual": "IMAGE+TEXT" 91 } 92 } 93 } 94});""") 95 # FIXME: crbug.com/279494 - we shouldn't be mixing mock and real builder names. 96 for builder in ['MOCK builder', 'MOCK builder (Debug)', 'WebKit Mac10.7']: 97 self.command._builder_data[builder] = data 98 99 100class TestCopyExistingBaselinesInternal(_BaseTestCase): 101 command_constructor = CopyExistingBaselinesInternal 102 103 def setUp(self): 104 super(TestCopyExistingBaselinesInternal, self).setUp() 105 106 def test_copying_overwritten_baseline(self): 107 self.tool.executive = MockExecutive2() 108 109 # FIXME: crbug.com/279494. it's confusing that this is the test- port, and not the regular lion port. Really all of the tests should be using the test ports. 110 port = self.tool.port_factory.get('test-mac-snowleopard') 111 self._write(port._filesystem.join(port.layout_tests_dir(), 'platform/test-mac-snowleopard/failures/expected/image-expected.txt'), 'original snowleopard result') 112 113 old_exact_matches = builders._exact_matches 114 oc = OutputCapture() 115 try: 116 builders._exact_matches = { 117 "MOCK Leopard": {"port_name": "test-mac-leopard", "specifiers": set(["mock-specifier"])}, 118 "MOCK SnowLeopard": {"port_name": "test-mac-snowleopard", "specifiers": set(["mock-specifier"])}, 119 } 120 121 options = MockOptions(builder="MOCK SnowLeopard", suffixes="txt", verbose=True, test="failures/expected/image.html", results_directory=None) 122 123 oc.capture_output() 124 self.command.execute(options, [], self.tool) 125 finally: 126 out, _, _ = oc.restore_output() 127 builders._exact_matches = old_exact_matches 128 129 self.assertMultiLineEqual(self._read(self.tool.filesystem.join(port.layout_tests_dir(), 'platform/test-mac-leopard/failures/expected/image-expected.txt')), 'original snowleopard result') 130 self.assertMultiLineEqual(out, '{"add": [], "remove-lines": [], "delete": []}\n') 131 132 def test_copying_overwritten_baseline_to_multiple_locations(self): 133 self.tool.executive = MockExecutive2() 134 135 # FIXME: crbug.com/279494. it's confusing that this is the test- port, and not the regular win port. Really all of the tests should be using the test ports. 136 port = self.tool.port_factory.get('test-win-win7') 137 self._write(port._filesystem.join(port.layout_tests_dir(), 'platform/test-win-win7/failures/expected/image-expected.txt'), 'original win7 result') 138 139 old_exact_matches = builders._exact_matches 140 oc = OutputCapture() 141 try: 142 builders._exact_matches = { 143 "MOCK Leopard": {"port_name": "test-mac-leopard", "specifiers": set(["mock-specifier"])}, 144 "MOCK Linux": {"port_name": "test-linux-x86_64", "specifiers": set(["mock-specifier"])}, 145 "MOCK Win7": {"port_name": "test-win-win7", "specifiers": set(["mock-specifier"])}, 146 } 147 148 options = MockOptions(builder="MOCK Win7", suffixes="txt", verbose=True, test="failures/expected/image.html", results_directory=None) 149 150 oc.capture_output() 151 self.command.execute(options, [], self.tool) 152 finally: 153 out, _, _ = oc.restore_output() 154 builders._exact_matches = old_exact_matches 155 156 self.assertMultiLineEqual(self._read(self.tool.filesystem.join(port.layout_tests_dir(), 'platform/test-linux-x86_64/failures/expected/image-expected.txt')), 'original win7 result') 157 self.assertFalse(self.tool.filesystem.exists(self.tool.filesystem.join(port.layout_tests_dir(), 'platform/mac-leopard/userscripts/another-test-expected.txt'))) 158 self.assertMultiLineEqual(out, '{"add": [], "remove-lines": [], "delete": []}\n') 159 160 def test_no_copy_existing_baseline(self): 161 self.tool.executive = MockExecutive2() 162 163 # FIXME: it's confusing that this is the test- port, and not the regular win port. Really all of the tests should be using the test ports. 164 port = self.tool.port_factory.get('test-win-win7') 165 self._write(port._filesystem.join(port.layout_tests_dir(), 'platform/test-win-win7/failures/expected/image-expected.txt'), 'original win7 result') 166 167 old_exact_matches = builders._exact_matches 168 oc = OutputCapture() 169 try: 170 builders._exact_matches = { 171 "MOCK Leopard": {"port_name": "test-mac-leopard", "specifiers": set(["mock-specifier"])}, 172 "MOCK Linux": {"port_name": "test-linux-x86_64", "specifiers": set(["mock-specifier"])}, 173 "MOCK Win7": {"port_name": "test-win-win7", "specifiers": set(["mock-specifier"])}, 174 } 175 176 options = MockOptions(builder="MOCK Win7", suffixes="txt", verbose=True, test="failures/expected/image.html", results_directory=None) 177 178 oc.capture_output() 179 self.command.execute(options, [], self.tool) 180 finally: 181 out, _, _ = oc.restore_output() 182 builders._exact_matches = old_exact_matches 183 184 self.assertMultiLineEqual(self._read(self.tool.filesystem.join(port.layout_tests_dir(), 'platform/test-linux-x86_64/failures/expected/image-expected.txt')), 'original win7 result') 185 self.assertMultiLineEqual(self._read(self.tool.filesystem.join(port.layout_tests_dir(), 'platform/test-win-win7/failures/expected/image-expected.txt')), 'original win7 result') 186 self.assertFalse(self.tool.filesystem.exists(self.tool.filesystem.join(port.layout_tests_dir(), 'platform/mac-leopard/userscripts/another-test-expected.txt'))) 187 self.assertMultiLineEqual(out, '{"add": [], "remove-lines": [], "delete": []}\n') 188 189 def test_no_copy_skipped_test(self): 190 self.tool.executive = MockExecutive2() 191 192 port = self.tool.port_factory.get('test-win-win7') 193 fs = self.tool.filesystem 194 self._write(fs.join(port.layout_tests_dir(), 'platform/test-win-win7/failures/expected/image-expected.txt'), 'original win7 result') 195 expectations_path = fs.join(port.path_to_generic_test_expectations_file()) 196 self._write(expectations_path, ( 197 "[ Win ] failures/expected/image.html [ Failure ]\n" 198 "[ Linux ] failures/expected/image.html [ Skip ]\n")) 199 old_exact_matches = builders._exact_matches 200 oc = OutputCapture() 201 try: 202 builders._exact_matches = { 203 "MOCK Linux": {"port_name": "test-linux-x86_64", "specifiers": set(["mock-specifier"])}, 204 "MOCK Leopard": {"port_name": "test-mac-leopard", "specifiers": set(["mock-specifier"])}, 205 "MOCK Win7": {"port_name": "test-win-win7", "specifiers": set(["mock-specifier"])}, 206 } 207 208 options = MockOptions(builder="MOCK Win7", suffixes="txt", verbose=True, test="failures/expected/image.html", results_directory=None) 209 210 oc.capture_output() 211 self.command.execute(options, [], self.tool) 212 finally: 213 out, _, _ = oc.restore_output() 214 builders._exact_matches = old_exact_matches 215 216 self.assertFalse(fs.exists(fs.join(port.layout_tests_dir(), 'platform/test-linux-x86_64/failures/expected/image-expected.txt'))) 217 self.assertEqual(self._read(fs.join(port.layout_tests_dir(), 'platform/test-win-win7/failures/expected/image-expected.txt')), 218 'original win7 result') 219 220 221class TestRebaselineTest(_BaseTestCase): 222 command_constructor = RebaselineTest # AKA webkit-patch rebaseline-test-internal 223 224 def setUp(self): 225 super(TestRebaselineTest, self).setUp() 226 self.options = MockOptions(builder="WebKit Mac10.7", test="userscripts/another-test.html", suffixes="txt", results_directory=None) 227 228 def test_baseline_directory(self): 229 command = self.command 230 self.assertMultiLineEqual(command._baseline_directory("WebKit Mac10.7"), "/mock-checkout/third_party/WebKit/LayoutTests/platform/mac-lion") 231 self.assertMultiLineEqual(command._baseline_directory("WebKit Mac10.6"), "/mock-checkout/third_party/WebKit/LayoutTests/platform/mac-snowleopard") 232 233 def test_rebaseline_updates_expectations_file_noop(self): 234 self._zero_out_test_expectations() 235 self._write(self.lion_expectations_path, """Bug(B) [ Mac Linux XP Debug ] fast/dom/Window/window-postmessage-clone-really-deep-array.html [ Pass ] 236Bug(A) [ Debug ] : fast/css/large-list-of-rules-crash.html [ Failure ] 237""") 238 self._write("fast/dom/Window/window-postmessage-clone-really-deep-array.html", "Dummy test contents") 239 self._write("fast/css/large-list-of-rules-crash.html", "Dummy test contents") 240 self._write("userscripts/another-test.html", "Dummy test contents") 241 242 self.options.suffixes = "png,wav,txt" 243 self.command._rebaseline_test_and_update_expectations(self.options) 244 245 self.assertItemsEqual(self.tool.web.urls_fetched, 246 [self.WEB_PREFIX + '/userscripts/another-test-actual.png', 247 self.WEB_PREFIX + '/userscripts/another-test-actual.wav', 248 self.WEB_PREFIX + '/userscripts/another-test-actual.txt']) 249 new_expectations = self._read(self.lion_expectations_path) 250 self.assertMultiLineEqual(new_expectations, """Bug(B) [ Mac Linux XP Debug ] fast/dom/Window/window-postmessage-clone-really-deep-array.html [ Pass ] 251Bug(A) [ Debug ] : fast/css/large-list-of-rules-crash.html [ Failure ] 252""") 253 254 def test_rebaseline_test(self): 255 self.command._rebaseline_test("WebKit Linux", "userscripts/another-test.html", "txt", self.WEB_PREFIX) 256 self.assertItemsEqual(self.tool.web.urls_fetched, [self.WEB_PREFIX + '/userscripts/another-test-actual.txt']) 257 258 def test_rebaseline_test_with_results_directory(self): 259 self._write("userscripts/another-test.html", "test data") 260 self._write(self.lion_expectations_path, "Bug(x) [ Mac ] userscripts/another-test.html [ ImageOnlyFailure ]\nbug(z) [ Linux ] userscripts/another-test.html [ ImageOnlyFailure ]\n") 261 self.options.results_directory = '/tmp' 262 self.command._rebaseline_test_and_update_expectations(self.options) 263 self.assertItemsEqual(self.tool.web.urls_fetched, ['file:///tmp/userscripts/another-test-actual.txt']) 264 265 def test_rebaseline_reftest(self): 266 self._write("userscripts/another-test.html", "test data") 267 self._write("userscripts/another-test-expected.html", "generic result") 268 OutputCapture().assert_outputs(self, self.command._rebaseline_test_and_update_expectations, args=[self.options], 269 expected_logs="Cannot rebaseline reftest: userscripts/another-test.html\n") 270 self.assertDictEqual(self.command._scm_changes, {'add': [], 'remove-lines': [], "delete": []}) 271 272 def test_rebaseline_test_and_print_scm_changes(self): 273 self.command._print_scm_changes = True 274 self.command._scm_changes = {'add': [], 'delete': []} 275 self.tool._scm.exists = lambda x: False 276 277 self.command._rebaseline_test("WebKit Linux", "userscripts/another-test.html", "txt", None) 278 279 self.assertDictEqual(self.command._scm_changes, {'add': ['/mock-checkout/third_party/WebKit/LayoutTests/platform/linux/userscripts/another-test-expected.txt'], 'delete': []}) 280 281 def test_rebaseline_test_internal_with_port_that_lacks_buildbot(self): 282 self.tool.executive = MockExecutive2() 283 284 # FIXME: it's confusing that this is the test- port, and not the regular win port. Really all of the tests should be using the test ports. 285 port = self.tool.port_factory.get('test-win-win7') 286 self._write(port._filesystem.join(port.layout_tests_dir(), 'platform/test-win-win7/failures/expected/image-expected.txt'), 'original win7 result') 287 288 old_exact_matches = builders._exact_matches 289 oc = OutputCapture() 290 try: 291 builders._exact_matches = { 292 "MOCK XP": {"port_name": "test-win-xp"}, 293 "MOCK Win7": {"port_name": "test-win-win7"}, 294 } 295 296 options = MockOptions(optimize=True, builder="MOCK Win7", suffixes="txt", 297 verbose=True, test="failures/expected/image.html", results_directory=None) 298 299 oc.capture_output() 300 self.command.execute(options, [], self.tool) 301 finally: 302 out, _, _ = oc.restore_output() 303 builders._exact_matches = old_exact_matches 304 305 self.assertMultiLineEqual(self._read(self.tool.filesystem.join(port.layout_tests_dir(), 'platform/test-win-win7/failures/expected/image-expected.txt')), 'MOCK Web result, convert 404 to None=True') 306 self.assertFalse(self.tool.filesystem.exists(self.tool.filesystem.join(port.layout_tests_dir(), 'platform/test-win-xp/failures/expected/image-expected.txt'))) 307 self.assertMultiLineEqual(out, '{"add": [], "remove-lines": [{"test": "failures/expected/image.html", "builder": "MOCK Win7"}], "delete": []}\n') 308 309 310class TestAbstractParallelRebaselineCommand(_BaseTestCase): 311 command_constructor = AbstractParallelRebaselineCommand 312 313 def test_builders_to_fetch_from(self): 314 old_exact_matches = builders._exact_matches 315 try: 316 builders._exact_matches = { 317 "MOCK XP": {"port_name": "test-win-xp"}, 318 "MOCK Win7": {"port_name": "test-win-win7"}, 319 "MOCK Win7 (dbg)(1)": {"port_name": "test-win-win7"}, 320 "MOCK Win7 (dbg)(2)": {"port_name": "test-win-win7"}, 321 } 322 323 builders_to_fetch = self.command._builders_to_fetch_from(["MOCK XP", "MOCK Win7 (dbg)(1)", "MOCK Win7 (dbg)(2)", "MOCK Win7"]) 324 self.assertEqual(builders_to_fetch, ["MOCK XP", "MOCK Win7"]) 325 finally: 326 builders._exact_matches = old_exact_matches 327 328 329class TestRebaselineJson(_BaseTestCase): 330 command_constructor = RebaselineJson 331 332 def setUp(self): 333 super(TestRebaselineJson, self).setUp() 334 self.tool.executive = MockExecutive2() 335 self.old_exact_matches = builders._exact_matches 336 builders._exact_matches = { 337 "MOCK builder": {"port_name": "test-mac-snowleopard"}, 338 "MOCK builder (Debug)": {"port_name": "test-mac-snowleopard"}, 339 } 340 341 def tearDown(self): 342 builders._exact_matches = self.old_exact_matches 343 super(TestRebaselineJson, self).tearDown() 344 345 def test_rebaseline_test_passes_on_all_builders(self): 346 self._setup_mock_builder_data() 347 348 def builder_data(): 349 self.command._builder_data['MOCK builder'] = LayoutTestResults.results_from_string("""ADD_RESULTS({ 350 "tests": { 351 "userscripts": { 352 "first-test.html": { 353 "expected": "NEEDSREBASELINE", 354 "actual": "PASS" 355 } 356 } 357 } 358});""") 359 return self.command._builder_data 360 361 self.command.builder_data = builder_data 362 363 options = MockOptions(optimize=True, verbose=True, results_directory=None) 364 365 self._write(self.lion_expectations_path, "Bug(x) userscripts/first-test.html [ ImageOnlyFailure ]\n") 366 self._write("userscripts/first-test.html", "Dummy test contents") 367 368 self.command._rebaseline(options, {"userscripts/first-test.html": {"MOCK builder": ["txt", "png"]}}) 369 370 # Note that we have one run_in_parallel() call followed by a run_command() 371 self.assertEqual(self.tool.executive.calls, 372 [[['echo', 'optimize-baselines', '--no-modify-scm', '--suffixes', '', 'userscripts/first-test.html', '--verbose']]]) 373 374 def test_rebaseline_all(self): 375 self._setup_mock_builder_data() 376 377 options = MockOptions(optimize=True, verbose=True, results_directory=None) 378 self._write("userscripts/first-test.html", "Dummy test contents") 379 self.command._rebaseline(options, {"userscripts/first-test.html": {"MOCK builder": ["txt", "png"]}}) 380 381 # Note that we have one run_in_parallel() call followed by a run_command() 382 self.assertEqual(self.tool.executive.calls, 383 [[['echo', 'copy-existing-baselines-internal', '--suffixes', 'txt,png', '--builder', 'MOCK builder', '--test', 'userscripts/first-test.html', '--verbose']], 384 [['echo', 'rebaseline-test-internal', '--suffixes', 'txt,png', '--builder', 'MOCK builder', '--test', 'userscripts/first-test.html', '--verbose']], 385 [['echo', 'optimize-baselines', '--no-modify-scm', '--suffixes', 'txt,png', 'userscripts/first-test.html', '--verbose']]]) 386 387 def test_rebaseline_debug(self): 388 self._setup_mock_builder_data() 389 390 options = MockOptions(optimize=True, verbose=True, results_directory=None) 391 self._write("userscripts/first-test.html", "Dummy test contents") 392 self.command._rebaseline(options, {"userscripts/first-test.html": {"MOCK builder (Debug)": ["txt", "png"]}}) 393 394 # Note that we have one run_in_parallel() call followed by a run_command() 395 self.assertEqual(self.tool.executive.calls, 396 [[['echo', 'copy-existing-baselines-internal', '--suffixes', 'txt,png', '--builder', 'MOCK builder (Debug)', '--test', 'userscripts/first-test.html', '--verbose']], 397 [['echo', 'rebaseline-test-internal', '--suffixes', 'txt,png', '--builder', 'MOCK builder (Debug)', '--test', 'userscripts/first-test.html', '--verbose']], 398 [['echo', 'optimize-baselines', '--no-modify-scm', '--suffixes', 'txt,png', 'userscripts/first-test.html', '--verbose']]]) 399 400 def test_no_optimize(self): 401 self._setup_mock_builder_data() 402 403 options = MockOptions(optimize=False, verbose=True, results_directory=None) 404 self._write("userscripts/first-test.html", "Dummy test contents") 405 self.command._rebaseline(options, {"userscripts/first-test.html": {"MOCK builder (Debug)": ["txt", "png"]}}) 406 407 # Note that we have only one run_in_parallel() call 408 self.assertEqual(self.tool.executive.calls, 409 [[['echo', 'copy-existing-baselines-internal', '--suffixes', 'txt,png', '--builder', 'MOCK builder (Debug)', '--test', 'userscripts/first-test.html', '--verbose']], 410 [['echo', 'rebaseline-test-internal', '--suffixes', 'txt,png', '--builder', 'MOCK builder (Debug)', '--test', 'userscripts/first-test.html', '--verbose']]]) 411 412 def test_results_directory(self): 413 self._setup_mock_builder_data() 414 415 options = MockOptions(optimize=False, verbose=True, results_directory='/tmp') 416 self._write("userscripts/first-test.html", "Dummy test contents") 417 self.command._rebaseline(options, {"userscripts/first-test.html": {"MOCK builder": ["txt", "png"]}}) 418 419 # Note that we have only one run_in_parallel() call 420 self.assertEqual(self.tool.executive.calls, 421 [[['echo', 'copy-existing-baselines-internal', '--suffixes', 'txt,png', '--builder', 'MOCK builder', '--test', 'userscripts/first-test.html', '--results-directory', '/tmp', '--verbose']], 422 [['echo', 'rebaseline-test-internal', '--suffixes', 'txt,png', '--builder', 'MOCK builder', '--test', 'userscripts/first-test.html', '--results-directory', '/tmp', '--verbose']]]) 423 424class TestRebaselineJsonUpdatesExpectationsFiles(_BaseTestCase): 425 command_constructor = RebaselineJson 426 427 def setUp(self): 428 super(TestRebaselineJsonUpdatesExpectationsFiles, self).setUp() 429 self.tool.executive = MockExecutive2() 430 431 def mock_run_command(args, 432 cwd=None, 433 input=None, 434 error_handler=None, 435 return_exit_code=False, 436 return_stderr=True, 437 decode_output=False, 438 env=None): 439 return '{"add": [], "remove-lines": [{"test": "userscripts/first-test.html", "builder": "WebKit Mac10.7"}]}\n' 440 self.tool.executive.run_command = mock_run_command 441 442 def test_rebaseline_updates_expectations_file(self): 443 options = MockOptions(optimize=False, verbose=True, results_directory=None) 444 445 self._write(self.lion_expectations_path, "Bug(x) [ Mac ] userscripts/first-test.html [ ImageOnlyFailure ]\nbug(z) [ Linux ] userscripts/first-test.html [ ImageOnlyFailure ]\n") 446 self._write("userscripts/first-test.html", "Dummy test contents") 447 self._setup_mock_builder_data() 448 449 self.command._rebaseline(options, {"userscripts/first-test.html": {"WebKit Mac10.7": ["txt", "png"]}}) 450 451 new_expectations = self._read(self.lion_expectations_path) 452 self.assertMultiLineEqual(new_expectations, "Bug(x) [ Mavericks MountainLion Retina SnowLeopard ] userscripts/first-test.html [ ImageOnlyFailure ]\nbug(z) [ Linux ] userscripts/first-test.html [ ImageOnlyFailure ]\n") 453 454 def test_rebaseline_updates_expectations_file_all_platforms(self): 455 options = MockOptions(optimize=False, verbose=True, results_directory=None) 456 457 self._write(self.lion_expectations_path, "Bug(x) userscripts/first-test.html [ ImageOnlyFailure ]\n") 458 self._write("userscripts/first-test.html", "Dummy test contents") 459 self._setup_mock_builder_data() 460 461 self.command._rebaseline(options, {"userscripts/first-test.html": {"WebKit Mac10.7": ["txt", "png"]}}) 462 463 new_expectations = self._read(self.lion_expectations_path) 464 self.assertMultiLineEqual(new_expectations, "Bug(x) [ Android Linux Mavericks MountainLion Retina SnowLeopard Win ] userscripts/first-test.html [ ImageOnlyFailure ]\n") 465 466 def test_rebaseline_handles_platform_skips(self): 467 # This test is just like test_rebaseline_updates_expectations_file_all_platforms(), 468 # except that if a particular port happens to SKIP a test in an overrides file, 469 # we count that as passing, and do not think that we still need to rebaseline it. 470 options = MockOptions(optimize=False, verbose=True, results_directory=None) 471 472 self._write(self.lion_expectations_path, "Bug(x) userscripts/first-test.html [ ImageOnlyFailure ]\n") 473 self._write("NeverFixTests", "Bug(y) [ Android ] userscripts [ Skip ]\n") 474 self._write("userscripts/first-test.html", "Dummy test contents") 475 self._setup_mock_builder_data() 476 477 self.command._rebaseline(options, {"userscripts/first-test.html": {"WebKit Mac10.7": ["txt", "png"]}}) 478 479 new_expectations = self._read(self.lion_expectations_path) 480 self.assertMultiLineEqual(new_expectations, "Bug(x) [ Linux Mavericks MountainLion Retina SnowLeopard Win ] userscripts/first-test.html [ ImageOnlyFailure ]\n") 481 482 def test_rebaseline_handles_skips_in_file(self): 483 # This test is like test_Rebaseline_handles_platform_skips, except that the 484 # Skip is in the same (generic) file rather than a platform file. In this case, 485 # the Skip line should be left unmodified. Note that the first line is now 486 # qualified as "[Linux Mac Win]"; if it was unqualified, it would conflict with 487 # the second line. 488 options = MockOptions(optimize=False, verbose=True, results_directory=None) 489 490 self._write(self.lion_expectations_path, 491 ("Bug(x) [ Linux Mac Win ] userscripts/first-test.html [ ImageOnlyFailure ]\n" 492 "Bug(y) [ Android ] userscripts/first-test.html [ Skip ]\n")) 493 self._write("userscripts/first-test.html", "Dummy test contents") 494 self._setup_mock_builder_data() 495 496 self.command._rebaseline(options, {"userscripts/first-test.html": {"WebKit Mac10.7": ["txt", "png"]}}) 497 498 new_expectations = self._read(self.lion_expectations_path) 499 self.assertMultiLineEqual(new_expectations, 500 ("Bug(x) [ Linux Mavericks MountainLion Retina SnowLeopard Win ] userscripts/first-test.html [ ImageOnlyFailure ]\n" 501 "Bug(y) [ Android ] userscripts/first-test.html [ Skip ]\n")) 502 503 def test_rebaseline_handles_smoke_tests(self): 504 # This test is just like test_rebaseline_handles_platform_skips, except that we check for 505 # a test not being in the SmokeTests file, instead of using overrides files. 506 # If a test is not part of the smoke tests, we count that as passing on ports that only 507 # run smoke tests, and do not think that we still need to rebaseline it. 508 options = MockOptions(optimize=False, verbose=True, results_directory=None) 509 510 self._write(self.lion_expectations_path, "Bug(x) userscripts/first-test.html [ ImageOnlyFailure ]\n") 511 self._write("SmokeTests", "fast/html/article-element.html") 512 self._write("userscripts/first-test.html", "Dummy test contents") 513 self._setup_mock_builder_data() 514 515 self.command._rebaseline(options, {"userscripts/first-test.html": {"WebKit Mac10.7": ["txt", "png"]}}) 516 517 new_expectations = self._read(self.lion_expectations_path) 518 self.assertMultiLineEqual(new_expectations, "Bug(x) [ Linux Mavericks MountainLion Retina SnowLeopard Win ] userscripts/first-test.html [ ImageOnlyFailure ]\n") 519 520 521class TestRebaseline(_BaseTestCase): 522 # This command shares most of its logic with RebaselineJson, so these tests just test what is different. 523 524 command_constructor = Rebaseline # AKA webkit-patch rebaseline 525 526 def test_rebaseline(self): 527 self.command._builders_to_pull_from = lambda: [MockBuilder('MOCK builder')] 528 529 self._write("userscripts/first-test.html", "test data") 530 531 self._zero_out_test_expectations() 532 self._setup_mock_builder_data() 533 534 old_exact_matches = builders._exact_matches 535 try: 536 builders._exact_matches = { 537 "MOCK builder": {"port_name": "test-mac-leopard", "specifiers": set(["mock-specifier"])}, 538 } 539 self.command.execute(MockOptions(results_directory=False, optimize=False, builders=None, suffixes="txt,png", verbose=True), ['userscripts/first-test.html'], self.tool) 540 finally: 541 builders._exact_matches = old_exact_matches 542 543 calls = filter(lambda x: x != ['qmake', '-v'] and x[0] != 'perl', self.tool.executive.calls) 544 self.assertEqual(calls, 545 [[['echo', 'copy-existing-baselines-internal', '--suffixes', 'txt,png', '--builder', 'MOCK builder', '--test', 'userscripts/first-test.html', '--verbose']], 546 [['echo', 'rebaseline-test-internal', '--suffixes', 'txt,png', '--builder', 'MOCK builder', '--test', 'userscripts/first-test.html', '--verbose']]]) 547 548 def test_rebaseline_directory(self): 549 self.command._builders_to_pull_from = lambda: [MockBuilder('MOCK builder')] 550 551 self._write("userscripts/first-test.html", "test data") 552 self._write("userscripts/second-test.html", "test data") 553 554 self._setup_mock_builder_data() 555 556 old_exact_matches = builders._exact_matches 557 try: 558 builders._exact_matches = { 559 "MOCK builder": {"port_name": "test-mac-leopard", "specifiers": set(["mock-specifier"])}, 560 } 561 self.command.execute(MockOptions(results_directory=False, optimize=False, builders=None, suffixes="txt,png", verbose=True), ['userscripts'], self.tool) 562 finally: 563 builders._exact_matches = old_exact_matches 564 565 calls = filter(lambda x: x != ['qmake', '-v'] and x[0] != 'perl', self.tool.executive.calls) 566 self.assertEqual(calls, 567 [[['echo', 'copy-existing-baselines-internal', '--suffixes', 'txt,png', '--builder', 'MOCK builder', '--test', 'userscripts/first-test.html', '--verbose'], 568 ['echo', 'copy-existing-baselines-internal', '--suffixes', 'txt,png', '--builder', 'MOCK builder', '--test', 'userscripts/second-test.html', '--verbose']], 569 [['echo', 'rebaseline-test-internal', '--suffixes', 'txt,png', '--builder', 'MOCK builder', '--test', 'userscripts/first-test.html', '--verbose'], 570 ['echo', 'rebaseline-test-internal', '--suffixes', 'txt,png', '--builder', 'MOCK builder', '--test', 'userscripts/second-test.html', '--verbose']]]) 571 572 573class MockLineRemovingExecutive(MockExecutive): 574 def run_in_parallel(self, commands): 575 assert len(commands) 576 577 num_previous_calls = len(self.calls) 578 command_outputs = [] 579 for cmd_line, cwd in commands: 580 out = self.run_command(cmd_line, cwd=cwd) 581 if 'rebaseline-test-internal' in cmd_line: 582 out = '{"add": [], "remove-lines": [{"test": "%s", "builder": "%s"}], "delete": []}\n' % (cmd_line[7], cmd_line[5]) 583 command_outputs.append([0, out, '']) 584 585 new_calls = self.calls[num_previous_calls:] 586 self.calls = self.calls[:num_previous_calls] 587 self.calls.append(new_calls) 588 return command_outputs 589 590 591class TestRebaselineExpectations(_BaseTestCase): 592 command_constructor = RebaselineExpectations 593 594 def setUp(self): 595 super(TestRebaselineExpectations, self).setUp() 596 self.options = MockOptions(optimize=False, builders=None, suffixes=['txt'], verbose=False, platform=None, results_directory=None) 597 598 def _write_test_file(self, port, path, contents): 599 abs_path = self.tool.filesystem.join(port.layout_tests_dir(), path) 600 self.tool.filesystem.write_text_file(abs_path, contents) 601 602 def _setup_test_port(self): 603 test_port = self.tool.port_factory.get('test') 604 original_get = self.tool.port_factory.get 605 606 def get_test_port(port_name=None, options=None, **kwargs): 607 if not port_name: 608 return test_port 609 return original_get(port_name, options, **kwargs) 610 # Need to make sure all the ports grabbed use the test checkout path instead of the mock checkout path. 611 # FIXME: crbug.com/279494 - we shouldn't be doing this. 612 self.tool.port_factory.get = get_test_port 613 614 return test_port 615 616 def test_rebaseline_expectations(self): 617 self._zero_out_test_expectations() 618 619 self.tool.executive = MockExecutive2() 620 621 def builder_data(): 622 self.command._builder_data['MOCK SnowLeopard'] = self.command._builder_data['MOCK Leopard'] = LayoutTestResults.results_from_string("""ADD_RESULTS({ 623 "tests": { 624 "userscripts": { 625 "another-test.html": { 626 "expected": "PASS", 627 "actual": "PASS TEXT" 628 }, 629 "images.svg": { 630 "expected": "FAIL", 631 "actual": "IMAGE+TEXT" 632 } 633 } 634 } 635});""") 636 return self.command._builder_data 637 638 self.command.builder_data = builder_data 639 640 self._write("userscripts/another-test.html", "Dummy test contents") 641 self._write("userscripts/images.svg", "Dummy test contents") 642 self.command._tests_to_rebaseline = lambda port: { 643 'userscripts/another-test.html': set(['txt']), 644 'userscripts/images.svg': set(['png']), 645 'userscripts/not-actually-failing.html': set(['txt', 'png', 'wav']), 646 } 647 648 old_exact_matches = builders._exact_matches 649 try: 650 builders._exact_matches = { 651 "MOCK Leopard": {"port_name": "test-mac-leopard", "specifiers": set(["mock-specifier"])}, 652 "MOCK SnowLeopard": {"port_name": "test-mac-snowleopard", "specifiers": set(["mock-specifier"])}, 653 } 654 self.command.execute(self.options, [], self.tool) 655 finally: 656 builders._exact_matches = old_exact_matches 657 658 # FIXME: change this to use the test- ports. 659 calls = filter(lambda x: x != ['qmake', '-v'], self.tool.executive.calls) 660 self.assertEqual(self.tool.executive.calls, [ 661 [ 662 ['echo', 'copy-existing-baselines-internal', '--suffixes', 'txt', '--builder', 'MOCK Leopard', '--test', 'userscripts/another-test.html'], 663 ['echo', 'copy-existing-baselines-internal', '--suffixes', 'txt', '--builder', 'MOCK SnowLeopard', '--test', 'userscripts/another-test.html'], 664 ['echo', 'copy-existing-baselines-internal', '--suffixes', 'png', '--builder', 'MOCK Leopard', '--test', 'userscripts/images.svg'], 665 ['echo', 'copy-existing-baselines-internal', '--suffixes', 'png', '--builder', 'MOCK SnowLeopard', '--test', 'userscripts/images.svg'] 666 ], 667 [ 668 ['echo', 'rebaseline-test-internal', '--suffixes', 'txt', '--builder', 'MOCK Leopard', '--test', 'userscripts/another-test.html'], 669 ['echo', 'rebaseline-test-internal', '--suffixes', 'txt', '--builder', 'MOCK SnowLeopard', '--test', 'userscripts/another-test.html'], 670 ['echo', 'rebaseline-test-internal', '--suffixes', 'png', '--builder', 'MOCK Leopard', '--test', 'userscripts/images.svg'], 671 ['echo', 'rebaseline-test-internal', '--suffixes', 'png', '--builder', 'MOCK SnowLeopard', '--test', 'userscripts/images.svg'] 672 ] 673 ]) 674 675 def test_rebaseline_expectations_noop(self): 676 self._zero_out_test_expectations() 677 678 oc = OutputCapture() 679 try: 680 oc.capture_output() 681 self.command.execute(self.options, [], self.tool) 682 finally: 683 _, _, logs = oc.restore_output() 684 self.assertEqual(self.tool.filesystem.written_files, {}) 685 self.assertEqual(logs, 'Did not find any tests marked Rebaseline.\n') 686 687 def disabled_test_overrides_are_included_correctly(self): 688 # This tests that the any tests marked as REBASELINE in the overrides are found, but 689 # that the overrides do not get written into the main file. 690 self._zero_out_test_expectations() 691 692 self._write(self.lion_expectations_path, '') 693 self.lion_port.expectations_dict = lambda: { 694 self.lion_expectations_path: '', 695 'overrides': ('Bug(x) userscripts/another-test.html [ Failure Rebaseline ]\n' 696 'Bug(y) userscripts/test.html [ Crash ]\n')} 697 self._write('/userscripts/another-test.html', '') 698 699 self.assertDictEqual(self.command._tests_to_rebaseline(self.lion_port), {'userscripts/another-test.html': set(['png', 'txt', 'wav'])}) 700 self.assertEqual(self._read(self.lion_expectations_path), '') 701 702 def test_rebaseline_without_other_expectations(self): 703 self._write("userscripts/another-test.html", "Dummy test contents") 704 self._write(self.lion_expectations_path, "Bug(x) userscripts/another-test.html [ Rebaseline ]\n") 705 self.assertDictEqual(self.command._tests_to_rebaseline(self.lion_port), {'userscripts/another-test.html': ('png', 'wav', 'txt')}) 706 707 def test_rebaseline_test_passes_everywhere(self): 708 test_port = self._setup_test_port() 709 710 old_builder_data = self.command.builder_data 711 712 def builder_data(): 713 self.command._builder_data['MOCK Leopard'] = self.command._builder_data['MOCK SnowLeopard'] = LayoutTestResults.results_from_string("""ADD_RESULTS({ 714 "tests": { 715 "fast": { 716 "dom": { 717 "prototype-taco.html": { 718 "expected": "FAIL", 719 "actual": "PASS", 720 "is_unexpected": true 721 } 722 } 723 } 724 } 725});""") 726 return self.command._builder_data 727 728 self.command.builder_data = builder_data 729 730 self.tool.filesystem.write_text_file(test_port.path_to_generic_test_expectations_file(), """ 731Bug(foo) fast/dom/prototype-taco.html [ Rebaseline ] 732""") 733 734 self._write_test_file(test_port, 'fast/dom/prototype-taco.html', "Dummy test contents") 735 736 self.tool.executive = MockLineRemovingExecutive() 737 738 old_exact_matches = builders._exact_matches 739 try: 740 builders._exact_matches = { 741 "MOCK Leopard": {"port_name": "test-mac-leopard", "specifiers": set(["mock-specifier"])}, 742 "MOCK SnowLeopard": {"port_name": "test-mac-snowleopard", "specifiers": set(["mock-specifier"])}, 743 } 744 745 self.command.execute(self.options, [], self.tool) 746 self.assertEqual(self.tool.executive.calls, []) 747 748 # The mac ports should both be removed since they're the only ones in builders._exact_matches. 749 self.assertEqual(self.tool.filesystem.read_text_file(test_port.path_to_generic_test_expectations_file()), """ 750Bug(foo) [ Linux Win ] fast/dom/prototype-taco.html [ Rebaseline ] 751""") 752 finally: 753 builders._exact_matches = old_exact_matches 754 755 756class _FakeOptimizer(BaselineOptimizer): 757 def read_results_by_directory(self, baseline_name): 758 if baseline_name.endswith('txt'): 759 return {'LayoutTests/passes/text.html': '123456'} 760 return {} 761 762 763class TestOptimizeBaselines(_BaseTestCase): 764 command_constructor = OptimizeBaselines 765 766 def _write_test_file(self, port, path, contents): 767 abs_path = self.tool.filesystem.join(port.layout_tests_dir(), path) 768 self.tool.filesystem.write_text_file(abs_path, contents) 769 770 def setUp(self): 771 super(TestOptimizeBaselines, self).setUp() 772 773 def test_modify_scm(self): 774 # FIXME: This is a hack to get the unittest and the BaselineOptimize to both use /mock-checkout 775 # instead of one using /mock-checkout and one using /test-checkout. 776 default_port = self.tool.port_factory.get() 777 self.tool.port_factory.get = lambda port_name=None: default_port 778 779 test_port = self.tool.port_factory.get('test') 780 self._write_test_file(test_port, 'another/test.html', "Dummy test contents") 781 self._write_test_file(test_port, 'platform/mac-snowleopard/another/test-expected.txt', "result A") 782 self._write_test_file(test_port, 'another/test-expected.txt', "result A") 783 784 old_exact_matches = builders._exact_matches 785 try: 786 builders._exact_matches = { 787 "MOCK Leopard Debug": {"port_name": "test-mac-snowleopard", "specifiers": set(["mock-specifier"])}, 788 } 789 OutputCapture().assert_outputs(self, self.command.execute, args=[ 790 MockOptions(suffixes='txt', no_modify_scm=False, platform='test-mac-snowleopard'), 791 ['another/test.html'], 792 self.tool, 793 ], expected_stdout='{"add": [], "remove-lines": [], "delete": []}\n') 794 finally: 795 builders._exact_matches = old_exact_matches 796 797 self.assertFalse(self.tool.filesystem.exists(self.tool.filesystem.join(test_port.layout_tests_dir(), 'platform/mac/another/test-expected.txt'))) 798 self.assertTrue(self.tool.filesystem.exists(self.tool.filesystem.join(test_port.layout_tests_dir(), 'another/test-expected.txt'))) 799 800 def test_no_modify_scm(self): 801 # FIXME: This is a hack to get the unittest and the BaselineOptimize to both use /mock-checkout 802 # instead of one using /mock-checkout and one using /test-checkout. 803 default_port = self.tool.port_factory.get() 804 self.tool.port_factory.get = lambda port_name=None: default_port 805 806 test_port = self.tool.port_factory.get('test') 807 self._write_test_file(test_port, 'another/test.html', "Dummy test contents") 808 self._write_test_file(test_port, 'platform/mac-snowleopard/another/test-expected.txt', "result A") 809 self._write_test_file(test_port, 'another/test-expected.txt', "result A") 810 811 old_exact_matches = builders._exact_matches 812 try: 813 builders._exact_matches = { 814 "MOCK Leopard Debug": {"port_name": "test-mac-snowleopard", "specifiers": set(["mock-specifier"])}, 815 } 816 OutputCapture().assert_outputs(self, self.command.execute, args=[ 817 MockOptions(suffixes='txt', no_modify_scm=True, platform='test-mac-snowleopard'), 818 ['another/test.html'], 819 self.tool, 820 ], expected_stdout='{"add": [], "remove-lines": [], "delete": ["/mock-checkout/third_party/WebKit/LayoutTests/platform/mac-snowleopard/another/test-expected.txt"]}\n') 821 finally: 822 builders._exact_matches = old_exact_matches 823 824 self.assertFalse(self.tool.filesystem.exists(self.tool.filesystem.join(test_port.layout_tests_dir(), 'platform/mac/another/test-expected.txt'))) 825 self.assertTrue(self.tool.filesystem.exists(self.tool.filesystem.join(test_port.layout_tests_dir(), 'another/test-expected.txt'))) 826 827 828class TestAnalyzeBaselines(_BaseTestCase): 829 command_constructor = AnalyzeBaselines 830 831 def setUp(self): 832 super(TestAnalyzeBaselines, self).setUp() 833 self.port = self.tool.port_factory.get('test') 834 self.tool.port_factory.get = (lambda port_name=None, options=None: self.port) 835 self.lines = [] 836 self.command._optimizer_class = _FakeOptimizer 837 self.command._write = (lambda msg: self.lines.append(msg)) # pylint bug warning about unnecessary lambda? pylint: disable=W0108 838 839 def test_default(self): 840 self.command.execute(MockOptions(suffixes='txt', missing=False, platform=None), ['passes/text.html'], self.tool) 841 self.assertEqual(self.lines, 842 ['passes/text-expected.txt:', 843 ' (generic): 123456']) 844 845 def test_missing_baselines(self): 846 self.command.execute(MockOptions(suffixes='png,txt', missing=True, platform=None), ['passes/text.html'], self.tool) 847 self.assertEqual(self.lines, 848 ['passes/text-expected.png: (no baselines found)', 849 'passes/text-expected.txt:', 850 ' (generic): 123456']) 851 852 853class TestAutoRebaseline(_BaseTestCase): 854 command_constructor = AutoRebaseline 855 856 def _write_test_file(self, port, path, contents): 857 abs_path = self.tool.filesystem.join(port.layout_tests_dir(), path) 858 self.tool.filesystem.write_text_file(abs_path, contents) 859 860 def _setup_test_port(self): 861 test_port = self.tool.port_factory.get('test') 862 original_get = self.tool.port_factory.get 863 864 def get_test_port(port_name=None, options=None, **kwargs): 865 if not port_name: 866 return test_port 867 return original_get(port_name, options, **kwargs) 868 # Need to make sure all the ports grabbed use the test checkout path instead of the mock checkout path. 869 # FIXME: crbug.com/279494 - we shouldn't be doing this. 870 self.tool.port_factory.get = get_test_port 871 872 return test_port 873 874 def setUp(self): 875 super(TestAutoRebaseline, self).setUp() 876 self.command.latest_revision_processed_on_all_bots = lambda: 9000 877 self.command.bot_revision_data = lambda: [{"builder": "Mock builder", "revision": "9000"}] 878 879 def test_release_builders(self): 880 old_exact_matches = builders._exact_matches 881 try: 882 builders._exact_matches = { 883 "MOCK Leopard": {"port_name": "test-mac-leopard", "specifiers": set(["mock-specifier"])}, 884 "MOCK Leopard Debug": {"port_name": "test-mac-snowleopard", "specifiers": set(["mock-specifier"])}, 885 "MOCK Leopard ASAN": {"port_name": "test-mac-snowleopard", "specifiers": set(["mock-specifier"])}, 886 } 887 self.assertEqual(self.command._release_builders(), ['MOCK Leopard']) 888 finally: 889 builders._exact_matches = old_exact_matches 890 891 def test_tests_to_rebaseline(self): 892 def blame(path): 893 return """ 894624c3081c0 path/to/TestExpectations (foobarbaz1@chromium.org 2013-06-14 20:18:46 +0000 11) crbug.com/24182 [ Debug ] path/to/norebaseline.html [ ImageOnlyFailure ] 895624c3081c0 path/to/TestExpectations (foobarbaz1@chromium.org 2013-04-28 04:52:41 +0000 13) Bug(foo) path/to/rebaseline-without-bug-number.html [ NeedsRebaseline ] 896624c3081c0 path/to/TestExpectations (foobarbaz1@chromium.org 2013-06-14 20:18:46 +0000 11) crbug.com/24182 [ Debug ] path/to/rebaseline-with-modifiers.html [ NeedsRebaseline ] 897624c3081c0 path/to/TestExpectations (foobarbaz1@chromium.org 2013-04-28 04:52:41 +0000 12) crbug.com/24182 crbug.com/234 path/to/rebaseline-without-modifiers.html [ NeedsRebaseline ] 8986469e754a1 path/to/TestExpectations (foobarbaz1@chromium.org 2013-04-28 04:52:41 +0000 12) crbug.com/24182 path/to/rebaseline-new-revision.html [ NeedsRebaseline ] 899624caaaaaa path/to/TestExpectations (foo@chromium.org 2013-04-28 04:52:41 +0000 12) crbug.com/24182 path/to/not-cycled-through-bots.html [ NeedsRebaseline ] 9000000000000 path/to/TestExpectations (foo@chromium.org 2013-04-28 04:52:41 +0000 12) crbug.com/24182 path/to/locally-changed-lined.html [ NeedsRebaseline ] 901""" 902 self.tool.scm().blame = blame 903 904 min_revision = 9000 905 self.assertEqual(self.command.tests_to_rebaseline(self.tool, min_revision, print_revisions=False), ( 906 set(['path/to/rebaseline-without-bug-number.html', 'path/to/rebaseline-with-modifiers.html', 'path/to/rebaseline-without-modifiers.html']), 907 5678, 908 'foobarbaz1@chromium.org', 909 set(['24182', '234']), 910 True)) 911 912 def test_tests_to_rebaseline_over_limit(self): 913 def blame(path): 914 result = "" 915 for i in range(0, self.command.MAX_LINES_TO_REBASELINE + 1): 916 result += "624c3081c0 path/to/TestExpectations (foobarbaz1@chromium.org 2013-04-28 04:52:41 +0000 13) crbug.com/24182 path/to/rebaseline-%s.html [ NeedsRebaseline ]\n" % i 917 return result 918 self.tool.scm().blame = blame 919 920 expected_list_of_tests = [] 921 for i in range(0, self.command.MAX_LINES_TO_REBASELINE): 922 expected_list_of_tests.append("path/to/rebaseline-%s.html" % i) 923 924 min_revision = 9000 925 self.assertEqual(self.command.tests_to_rebaseline(self.tool, min_revision, print_revisions=False), ( 926 set(expected_list_of_tests), 927 5678, 928 'foobarbaz1@chromium.org', 929 set(['24182']), 930 True)) 931 932 def test_commit_message(self): 933 author = "foo@chromium.org" 934 revision = 1234 935 bugs = set() 936 self.assertEqual(self.command.commit_message(author, revision, bugs), 937 """Auto-rebaseline for r1234 938 939http://src.chromium.org/viewvc/blink?view=revision&revision=1234 940 941TBR=foo@chromium.org 942""") 943 944 bugs = set(["234", "345"]) 945 self.assertEqual(self.command.commit_message(author, revision, bugs), 946 """Auto-rebaseline for r1234 947 948http://src.chromium.org/viewvc/blink?view=revision&revision=1234 949 950BUG=234,345 951TBR=foo@chromium.org 952""") 953 954 def test_no_needs_rebaseline_lines(self): 955 def blame(path): 956 return """ 9576469e754a1 path/to/TestExpectations (foobarbaz1@chromium.org 2013-06-14 20:18:46 +0000 11) crbug.com/24182 [ Debug ] path/to/norebaseline.html [ ImageOnlyFailure ] 958""" 959 self.tool.scm().blame = blame 960 961 self.command.execute(MockOptions(optimize=True, verbose=False, move_overwritten_baselines=False, results_directory=False), [], self.tool) 962 self.assertEqual(self.tool.executive.calls, []) 963 964 def test_execute(self): 965 def blame(path): 966 return """ 9676469e754a1 path/to/TestExpectations (foobarbaz1@chromium.org 2013-06-14 20:18:46 +0000 11) # Test NeedsRebaseline being in a comment doesn't bork parsing. 9686469e754a1 path/to/TestExpectations (foobarbaz1@chromium.org 2013-06-14 20:18:46 +0000 11) crbug.com/24182 [ Debug ] path/to/norebaseline.html [ ImageOnlyFailure ] 9696469e754a1 path/to/TestExpectations (foobarbaz1@chromium.org 2013-04-28 04:52:41 +0000 13) Bug(foo) fast/dom/prototype-taco.html [ NeedsRebaseline ] 9706469e754a1 path/to/TestExpectations (foobarbaz1@chromium.org 2013-06-14 20:18:46 +0000 11) crbug.com/24182 [ SnowLeopard ] fast/dom/prototype-strawberry.html [ NeedsRebaseline ] 9716469e754a1 path/to/TestExpectations (foobarbaz1@chromium.org 2013-04-28 04:52:41 +0000 12) crbug.com/24182 fast/dom/prototype-chocolate.html [ NeedsRebaseline ] 972624caaaaaa path/to/TestExpectations (foo@chromium.org 2013-04-28 04:52:41 +0000 12) crbug.com/24182 path/to/not-cycled-through-bots.html [ NeedsRebaseline ] 9730000000000 path/to/TestExpectations (foo@chromium.org 2013-04-28 04:52:41 +0000 12) crbug.com/24182 path/to/locally-changed-lined.html [ NeedsRebaseline ] 974""" 975 self.tool.scm().blame = blame 976 977 test_port = self._setup_test_port() 978 979 old_builder_data = self.command.builder_data 980 981 def builder_data(): 982 old_builder_data() 983 # have prototype-chocolate only fail on "MOCK Leopard". 984 self.command._builder_data['MOCK SnowLeopard'] = LayoutTestResults.results_from_string("""ADD_RESULTS({ 985 "tests": { 986 "fast": { 987 "dom": { 988 "prototype-taco.html": { 989 "expected": "PASS", 990 "actual": "PASS TEXT", 991 "is_unexpected": true 992 }, 993 "prototype-chocolate.html": { 994 "expected": "FAIL", 995 "actual": "PASS" 996 }, 997 "prototype-strawberry.html": { 998 "expected": "PASS", 999 "actual": "IMAGE PASS", 1000 "is_unexpected": true 1001 } 1002 } 1003 } 1004 } 1005});""") 1006 return self.command._builder_data 1007 1008 self.command.builder_data = builder_data 1009 1010 self.tool.filesystem.write_text_file(test_port.path_to_generic_test_expectations_file(), """ 1011crbug.com/24182 [ Debug ] path/to/norebaseline.html [ Rebaseline ] 1012Bug(foo) fast/dom/prototype-taco.html [ NeedsRebaseline ] 1013crbug.com/24182 [ SnowLeopard ] fast/dom/prototype-strawberry.html [ NeedsRebaseline ] 1014crbug.com/24182 fast/dom/prototype-chocolate.html [ NeedsRebaseline ] 1015crbug.com/24182 path/to/not-cycled-through-bots.html [ NeedsRebaseline ] 1016crbug.com/24182 path/to/locally-changed-lined.html [ NeedsRebaseline ] 1017""") 1018 1019 self._write_test_file(test_port, 'fast/dom/prototype-taco.html', "Dummy test contents") 1020 self._write_test_file(test_port, 'fast/dom/prototype-strawberry.html', "Dummy test contents") 1021 self._write_test_file(test_port, 'fast/dom/prototype-chocolate.html', "Dummy test contents") 1022 1023 self.tool.executive = MockLineRemovingExecutive() 1024 1025 old_exact_matches = builders._exact_matches 1026 try: 1027 builders._exact_matches = { 1028 "MOCK Leopard": {"port_name": "test-mac-leopard", "specifiers": set(["mock-specifier"])}, 1029 "MOCK SnowLeopard": {"port_name": "test-mac-snowleopard", "specifiers": set(["mock-specifier"])}, 1030 } 1031 1032 self.command.tree_status = lambda: 'closed' 1033 self.command.execute(MockOptions(optimize=True, verbose=False, move_overwritten_baselines=False, results_directory=False), [], self.tool) 1034 self.assertEqual(self.tool.executive.calls, []) 1035 1036 self.command.tree_status = lambda: 'open' 1037 self.tool.executive.calls = [] 1038 self.command.execute(MockOptions(optimize=True, verbose=False, move_overwritten_baselines=False, results_directory=False), [], self.tool) 1039 1040 self.assertEqual(self.tool.executive.calls, [ 1041 [ 1042 ['echo', 'copy-existing-baselines-internal', '--suffixes', 'txt,png', '--builder', 'MOCK Leopard', '--test', 'fast/dom/prototype-chocolate.html'], 1043 ['echo', 'copy-existing-baselines-internal', '--suffixes', 'png', '--builder', 'MOCK SnowLeopard', '--test', 'fast/dom/prototype-strawberry.html'], 1044 ['echo', 'copy-existing-baselines-internal', '--suffixes', 'txt', '--builder', 'MOCK Leopard', '--test', 'fast/dom/prototype-taco.html'], 1045 ['echo', 'copy-existing-baselines-internal', '--suffixes', 'txt', '--builder', 'MOCK SnowLeopard', '--test', 'fast/dom/prototype-taco.html'], 1046 ], 1047 [ 1048 ['echo', 'rebaseline-test-internal', '--suffixes', 'txt,png', '--builder', 'MOCK Leopard', '--test', 'fast/dom/prototype-chocolate.html'], 1049 ['echo', 'rebaseline-test-internal', '--suffixes', 'png', '--builder', 'MOCK SnowLeopard', '--test', 'fast/dom/prototype-strawberry.html'], 1050 ['echo', 'rebaseline-test-internal', '--suffixes', 'txt', '--builder', 'MOCK Leopard', '--test', 'fast/dom/prototype-taco.html'], 1051 ['echo', 'rebaseline-test-internal', '--suffixes', 'txt', '--builder', 'MOCK SnowLeopard', '--test', 'fast/dom/prototype-taco.html'], 1052 ], 1053 [ 1054 ['echo', 'optimize-baselines', '--no-modify-scm', '--suffixes', 'txt,png', 'fast/dom/prototype-chocolate.html'], 1055 ['echo', 'optimize-baselines', '--no-modify-scm', '--suffixes', 'png', 'fast/dom/prototype-strawberry.html'], 1056 ['echo', 'optimize-baselines', '--no-modify-scm', '--suffixes', 'txt', 'fast/dom/prototype-taco.html'], 1057 ], 1058 ['git', 'cl', 'upload', '-f'], 1059 ['git', 'pull'], 1060 ['git', 'cl', 'dcommit', '-f'], 1061 ['git', 'cl', 'set_close'], 1062 ]) 1063 1064 # The mac ports should both be removed since they're the only ones in builders._exact_matches. 1065 self.assertEqual(self.tool.filesystem.read_text_file(test_port.path_to_generic_test_expectations_file()), """ 1066crbug.com/24182 [ Debug ] path/to/norebaseline.html [ Rebaseline ] 1067Bug(foo) [ Linux Win ] fast/dom/prototype-taco.html [ NeedsRebaseline ] 1068crbug.com/24182 [ Linux Win ] fast/dom/prototype-chocolate.html [ NeedsRebaseline ] 1069crbug.com/24182 path/to/not-cycled-through-bots.html [ NeedsRebaseline ] 1070crbug.com/24182 path/to/locally-changed-lined.html [ NeedsRebaseline ] 1071""") 1072 finally: 1073 builders._exact_matches = old_exact_matches 1074 1075 def test_execute_git_cl_hangs(self): 1076 def blame(path): 1077 return """ 10786469e754a1 path/to/TestExpectations (foobarbaz1@chromium.org 2013-04-28 04:52:41 +0000 13) Bug(foo) fast/dom/prototype-taco.html [ NeedsRebaseline ] 1079""" 1080 self.tool.scm().blame = blame 1081 1082 test_port = self._setup_test_port() 1083 1084 old_builder_data = self.command.builder_data 1085 1086 def builder_data(): 1087 old_builder_data() 1088 # have prototype-chocolate only fail on "MOCK Leopard". 1089 self.command._builder_data['MOCK SnowLeopard'] = LayoutTestResults.results_from_string("""ADD_RESULTS({ 1090 "tests": { 1091 "fast": { 1092 "dom": { 1093 "prototype-taco.html": { 1094 "expected": "PASS", 1095 "actual": "PASS TEXT", 1096 "is_unexpected": true 1097 } 1098 } 1099 } 1100 } 1101});""") 1102 return self.command._builder_data 1103 1104 self.command.builder_data = builder_data 1105 1106 self.tool.filesystem.write_text_file(test_port.path_to_generic_test_expectations_file(), """ 1107Bug(foo) fast/dom/prototype-taco.html [ NeedsRebaseline ] 1108""") 1109 1110 self._write_test_file(test_port, 'fast/dom/prototype-taco.html', "Dummy test contents") 1111 1112 old_exact_matches = builders._exact_matches 1113 try: 1114 builders._exact_matches = { 1115 "MOCK SnowLeopard": {"port_name": "test-mac-snowleopard", "specifiers": set(["mock-specifier"])}, 1116 } 1117 1118 self.command.SECONDS_BEFORE_GIVING_UP = 0 1119 self.command.tree_status = lambda: 'open' 1120 self.tool.executive.calls = [] 1121 self.command.execute(MockOptions(optimize=True, verbose=False, move_overwritten_baselines=False, results_directory=False), [], self.tool) 1122 1123 self.assertEqual(self.tool.executive.calls, [ 1124 [ 1125 ['echo', 'copy-existing-baselines-internal', '--suffixes', 'txt', '--builder', 'MOCK SnowLeopard', '--test', 'fast/dom/prototype-taco.html'], 1126 ], 1127 [ 1128 ['echo', 'rebaseline-test-internal', '--suffixes', 'txt', '--builder', 'MOCK SnowLeopard', '--test', 'fast/dom/prototype-taco.html'], 1129 ], 1130 [['echo', 'optimize-baselines', '--no-modify-scm', '--suffixes', 'txt', 'fast/dom/prototype-taco.html']], 1131 ['git', 'cl', 'upload', '-f'], 1132 ['git', 'cl', 'set_close'], 1133 ]) 1134 finally: 1135 builders._exact_matches = old_exact_matches 1136 1137 def test_execute_test_passes_everywhere(self): 1138 def blame(path): 1139 return """ 11406469e754a1 path/to/TestExpectations (foobarbaz1@chromium.org 2013-04-28 04:52:41 +0000 13) Bug(foo) fast/dom/prototype-taco.html [ NeedsRebaseline ] 1141""" 1142 self.tool.scm().blame = blame 1143 1144 test_port = self._setup_test_port() 1145 1146 old_builder_data = self.command.builder_data 1147 1148 def builder_data(): 1149 self.command._builder_data['MOCK Leopard'] = self.command._builder_data['MOCK SnowLeopard'] = LayoutTestResults.results_from_string("""ADD_RESULTS({ 1150 "tests": { 1151 "fast": { 1152 "dom": { 1153 "prototype-taco.html": { 1154 "expected": "FAIL", 1155 "actual": "PASS", 1156 "is_unexpected": true 1157 } 1158 } 1159 } 1160 } 1161});""") 1162 return self.command._builder_data 1163 1164 self.command.builder_data = builder_data 1165 1166 self.tool.filesystem.write_text_file(test_port.path_to_generic_test_expectations_file(), """ 1167Bug(foo) fast/dom/prototype-taco.html [ NeedsRebaseline ] 1168""") 1169 1170 self._write_test_file(test_port, 'fast/dom/prototype-taco.html', "Dummy test contents") 1171 1172 self.tool.executive = MockLineRemovingExecutive() 1173 1174 old_exact_matches = builders._exact_matches 1175 try: 1176 builders._exact_matches = { 1177 "MOCK Leopard": {"port_name": "test-mac-leopard", "specifiers": set(["mock-specifier"])}, 1178 "MOCK SnowLeopard": {"port_name": "test-mac-snowleopard", "specifiers": set(["mock-specifier"])}, 1179 } 1180 1181 self.command.tree_status = lambda: 'open' 1182 self.command.execute(MockOptions(optimize=True, verbose=False, move_overwritten_baselines=False, results_directory=False), [], self.tool) 1183 self.assertEqual(self.tool.executive.calls, [ 1184 [['echo', 'optimize-baselines', '--no-modify-scm', '--suffixes', '', 'fast/dom/prototype-taco.html']], 1185 ['git', 'cl', 'upload', '-f'], 1186 ['git', 'pull'], 1187 ['git', 'cl', 'dcommit', '-f'], 1188 ['git', 'cl', 'set_close'], 1189 ]) 1190 1191 # The mac ports should both be removed since they're the only ones in builders._exact_matches. 1192 self.assertEqual(self.tool.filesystem.read_text_file(test_port.path_to_generic_test_expectations_file()), """ 1193Bug(foo) [ Linux Win ] fast/dom/prototype-taco.html [ NeedsRebaseline ] 1194""") 1195 finally: 1196 builders._exact_matches = old_exact_matches 1197 1198 1199class TestRebaselineOMatic(_BaseTestCase): 1200 command_constructor = RebaselineOMatic 1201 1202 def setUp(self): 1203 super(TestRebaselineOMatic, self).setUp() 1204 self._logs = [] 1205 1206 def _mock_log_to_server(self, log='', is_new_entry=False): 1207 self._logs.append({'log': log, 'newentry': is_new_entry}) 1208 1209 def test_run_logged_command(self): 1210 self.command._verbose = False 1211 self.command._log_to_server = self._mock_log_to_server 1212 self.command._run_logged_command(['echo', 'foo']) 1213 self.assertEqual(self.tool.executive.calls, [['echo', 'foo']]) 1214 self.assertEqual(self._logs, [{'log': 'MOCK STDOUT', 'newentry': False}]) 1215 1216 def test_do_one_rebaseline(self): 1217 self.command._verbose = False 1218 self.command._log_to_server = self._mock_log_to_server 1219 1220 oc = OutputCapture() 1221 oc.capture_output() 1222 self.command._do_one_rebaseline() 1223 out, _, _ = oc.restore_output() 1224 1225 self.assertEqual(out, '') 1226 self.assertEqual(self.tool.executive.calls, [ 1227 ['git', 'pull'], 1228 ['/mock-checkout/third_party/WebKit/Tools/Scripts/webkit-patch', 'auto-rebaseline'], 1229 ]) 1230 self.assertEqual(self._logs, [ 1231 {'log': '', 'newentry': True}, 1232 {'log': 'MOCK STDOUT', 'newentry': False}, 1233 ]) 1234 1235 def test_do_one_rebaseline_verbose(self): 1236 self.command._verbose = True 1237 self.command._log_to_server = self._mock_log_to_server 1238 1239 oc = OutputCapture() 1240 oc.capture_output() 1241 self.command._do_one_rebaseline() 1242 out, _, _ = oc.restore_output() 1243 1244 self.assertEqual(out, 'MOCK STDOUT\n') 1245 self.assertEqual(self.tool.executive.calls, [ 1246 ['git', 'pull'], 1247 ['/mock-checkout/third_party/WebKit/Tools/Scripts/webkit-patch', 'auto-rebaseline', '--verbose'], 1248 ]) 1249 self.assertEqual(self._logs, [ 1250 {'log': '', 'newentry': True}, 1251 {'log': 'MOCK STDOUT', 'newentry': False}, 1252 ]) 1253
