1# Copyright (C) 2010 Google Inc. All rights reserved. 2# 3# Redistribution and use in source and binary forms, with or without 4# modification, are permitted provided that the following conditions are 5# met: 6# 7# * Redistributions of source code must retain the above copyright 8# notice, this list of conditions and the following disclaimer. 9# * Redistributions in binary form must reproduce the above 10# copyright notice, this list of conditions and the following disclaimer 11# in the documentation and/or other materials provided with the 12# distribution. 13# * Neither the name of Google Inc. nor the names of its 14# contributors may be used to endorse or promote products derived from 15# this software without specific prior written permission. 16# 17# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 18# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 19# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 20# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 21# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 22# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 23# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 24# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 25# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 29"""Unit testing base class for Port implementations.""" 30 31import errno 32import logging 33import os 34import socket 35import sys 36import time 37import webkitpy.thirdparty.unittest2 as unittest 38 39from webkitpy.common.system.executive_mock import MockExecutive, MockExecutive2 40from webkitpy.common.system.filesystem_mock import MockFileSystem 41from webkitpy.common.system.outputcapture import OutputCapture 42from webkitpy.common.system.systemhost_mock import MockSystemHost 43from webkitpy.layout_tests.models import test_run_results 44from webkitpy.layout_tests.port.base import Port, TestConfiguration 45from webkitpy.layout_tests.port.server_process_mock import MockServerProcess 46from webkitpy.tool.mocktool import MockOptions 47 48 49# FIXME: get rid of this fixture 50class TestWebKitPort(Port): 51 port_name = "testwebkitport" 52 53 def __init__(self, port_name=None, symbols_string=None, 54 expectations_file=None, skips_file=None, host=None, config=None, 55 **kwargs): 56 port_name = port_name or TestWebKitPort.port_name 57 self.symbols_string = symbols_string # Passing "" disables all staticly-detectable features. 58 host = host or MockSystemHost() 59 super(TestWebKitPort, self).__init__(host, port_name=port_name, **kwargs) 60 61 def all_test_configurations(self): 62 return [self.test_configuration()] 63 64 def _symbols_string(self): 65 return self.symbols_string 66 67 def _tests_for_disabled_features(self): 68 return ["accessibility", ] 69 70 71class FakePrinter(object): 72 def write_update(self, msg): 73 pass 74 75 def write_throttled_update(self, msg): 76 pass 77 78 79 80class PortTestCase(unittest.TestCase): 81 """Tests that all Port implementations must pass.""" 82 HTTP_PORTS = (8000, 8080, 8443) 83 WEBSOCKET_PORTS = (8880,) 84 85 # Subclasses override this to point to their Port subclass. 86 os_name = None 87 os_version = None 88 port_maker = TestWebKitPort 89 port_name = None 90 91 def make_port(self, host=None, port_name=None, options=None, os_name=None, os_version=None, **kwargs): 92 host = host or MockSystemHost(os_name=(os_name or self.os_name), os_version=(os_version or self.os_version)) 93 options = options or MockOptions(configuration='Release') 94 port_name = port_name or self.port_name 95 port_name = self.port_maker.determine_full_port_name(host, options, port_name) 96 port = self.port_maker(host, port_name, options=options, **kwargs) 97 port._config.build_directory = lambda configuration: '/mock-build' 98 return port 99 100 def make_wdiff_available(self, port): 101 port._wdiff_available = True 102 103 def test_check_build(self): 104 port = self.make_port() 105 port._check_file_exists = lambda path, desc: True 106 if port._dump_reader: 107 port._dump_reader.check_is_functional = lambda: True 108 port._options.build = True 109 port._check_driver_build_up_to_date = lambda config: True 110 port.check_httpd = lambda: True 111 oc = OutputCapture() 112 try: 113 oc.capture_output() 114 self.assertEqual(port.check_build(needs_http=True, printer=FakePrinter()), 115 test_run_results.OK_EXIT_STATUS) 116 finally: 117 out, err, logs = oc.restore_output() 118 self.assertIn('pretty patches', logs) # We should get a warning about PrettyPatch being missing, 119 self.assertNotIn('build requirements', logs) # but not the driver itself. 120 121 port._check_file_exists = lambda path, desc: False 122 port._check_driver_build_up_to_date = lambda config: False 123 try: 124 oc.capture_output() 125 self.assertEqual(port.check_build(needs_http=True, printer=FakePrinter()), 126 test_run_results.UNEXPECTED_ERROR_EXIT_STATUS) 127 finally: 128 out, err, logs = oc.restore_output() 129 self.assertIn('pretty patches', logs) # And, hereere we should get warnings about both. 130 self.assertIn('build requirements', logs) 131 132 def test_default_max_locked_shards(self): 133 port = self.make_port() 134 port.default_child_processes = lambda: 16 135 self.assertEqual(port.default_max_locked_shards(), 4) 136 port.default_child_processes = lambda: 2 137 self.assertEqual(port.default_max_locked_shards(), 1) 138 139 def test_default_timeout_ms(self): 140 self.assertEqual(self.make_port(options=MockOptions(configuration='Release')).default_timeout_ms(), 6000) 141 self.assertEqual(self.make_port(options=MockOptions(configuration='Debug')).default_timeout_ms(), 18000) 142 143 def test_default_pixel_tests(self): 144 self.assertEqual(self.make_port().default_pixel_tests(), True) 145 146 def test_driver_cmd_line(self): 147 port = self.make_port() 148 self.assertTrue(len(port.driver_cmd_line())) 149 150 options = MockOptions(additional_drt_flag=['--foo=bar', '--foo=baz']) 151 port = self.make_port(options=options) 152 cmd_line = port.driver_cmd_line() 153 self.assertTrue('--foo=bar' in cmd_line) 154 self.assertTrue('--foo=baz' in cmd_line) 155 156 def test_uses_apache(self): 157 self.assertTrue(self.make_port().uses_apache()) 158 159 def assert_servers_are_down(self, host, ports): 160 for port in ports: 161 try: 162 test_socket = socket.socket() 163 test_socket.connect((host, port)) 164 self.fail() 165 except IOError, e: 166 self.assertTrue(e.errno in (errno.ECONNREFUSED, errno.ECONNRESET)) 167 finally: 168 test_socket.close() 169 170 def assert_servers_are_up(self, host, ports): 171 for port in ports: 172 try: 173 test_socket = socket.socket() 174 test_socket.connect((host, port)) 175 except IOError, e: 176 self.fail('failed to connect to %s:%d' % (host, port)) 177 finally: 178 test_socket.close() 179 180 def test_diff_image__missing_both(self): 181 port = self.make_port() 182 self.assertEqual(port.diff_image(None, None), (None, None)) 183 self.assertEqual(port.diff_image(None, ''), (None, None)) 184 self.assertEqual(port.diff_image('', None), (None, None)) 185 186 self.assertEqual(port.diff_image('', ''), (None, None)) 187 188 def test_diff_image__missing_actual(self): 189 port = self.make_port() 190 self.assertEqual(port.diff_image(None, 'foo'), ('foo', None)) 191 self.assertEqual(port.diff_image('', 'foo'), ('foo', None)) 192 193 def test_diff_image__missing_expected(self): 194 port = self.make_port() 195 self.assertEqual(port.diff_image('foo', None), ('foo', None)) 196 self.assertEqual(port.diff_image('foo', ''), ('foo', None)) 197 198 def test_diff_image(self): 199 def _path_to_image_diff(): 200 return "/path/to/image_diff" 201 202 port = self.make_port() 203 port._path_to_image_diff = _path_to_image_diff 204 205 mock_image_diff = "MOCK Image Diff" 206 207 def mock_run_command(args): 208 port._filesystem.write_binary_file(args[4], mock_image_diff) 209 return 1 210 211 # Images are different. 212 port._executive = MockExecutive2(run_command_fn=mock_run_command) 213 self.assertEqual(mock_image_diff, port.diff_image("EXPECTED", "ACTUAL")[0]) 214 215 # Images are the same. 216 port._executive = MockExecutive2(exit_code=0) 217 self.assertEqual(None, port.diff_image("EXPECTED", "ACTUAL")[0]) 218 219 # There was some error running image_diff. 220 port._executive = MockExecutive2(exit_code=2) 221 exception_raised = False 222 try: 223 port.diff_image("EXPECTED", "ACTUAL") 224 except ValueError, e: 225 exception_raised = True 226 self.assertFalse(exception_raised) 227 228 def test_diff_image_crashed(self): 229 port = self.make_port() 230 port._executive = MockExecutive2(exit_code=2) 231 self.assertEqual(port.diff_image("EXPECTED", "ACTUAL"), (None, 'Image diff returned an exit code of 2. See http://crbug.com/278596')) 232 233 def test_check_wdiff(self): 234 port = self.make_port() 235 port.check_wdiff() 236 237 def test_wdiff_text_fails(self): 238 host = MockSystemHost(os_name=self.os_name, os_version=self.os_version) 239 host.executive = MockExecutive(should_throw=True) 240 port = self.make_port(host=host) 241 port._executive = host.executive # AndroidPortTest.make_port sets its own executive, so reset that as well. 242 243 # This should raise a ScriptError that gets caught and turned into the 244 # error text, and also mark wdiff as not available. 245 self.make_wdiff_available(port) 246 self.assertTrue(port.wdiff_available()) 247 diff_txt = port.wdiff_text("/tmp/foo.html", "/tmp/bar.html") 248 self.assertEqual(diff_txt, port._wdiff_error_html) 249 self.assertFalse(port.wdiff_available()) 250 251 def test_missing_symbol_to_skipped_tests(self): 252 # Test that we get the chromium skips and not the webkit default skips 253 port = self.make_port() 254 skip_dict = port._missing_symbol_to_skipped_tests() 255 if port.PORT_HAS_AUDIO_CODECS_BUILT_IN: 256 self.assertEqual(skip_dict, {}) 257 else: 258 self.assertTrue('ff_mp3_decoder' in skip_dict) 259 self.assertFalse('WebGLShader' in skip_dict) 260 261 def test_test_configuration(self): 262 port = self.make_port() 263 self.assertTrue(port.test_configuration()) 264 265 def test_all_test_configurations(self): 266 """Validate the complete set of configurations this port knows about.""" 267 port = self.make_port() 268 self.assertEqual(set(port.all_test_configurations()), set([ 269 TestConfiguration('snowleopard', 'x86', 'debug'), 270 TestConfiguration('snowleopard', 'x86', 'release'), 271 TestConfiguration('lion', 'x86', 'debug'), 272 TestConfiguration('lion', 'x86', 'release'), 273 TestConfiguration('retina', 'x86', 'debug'), 274 TestConfiguration('retina', 'x86', 'release'), 275 TestConfiguration('mountainlion', 'x86', 'debug'), 276 TestConfiguration('mountainlion', 'x86', 'release'), 277 TestConfiguration('mavericks', 'x86', 'debug'), 278 TestConfiguration('mavericks', 'x86', 'release'), 279 TestConfiguration('xp', 'x86', 'debug'), 280 TestConfiguration('xp', 'x86', 'release'), 281 TestConfiguration('win7', 'x86', 'debug'), 282 TestConfiguration('win7', 'x86', 'release'), 283 TestConfiguration('lucid', 'x86', 'debug'), 284 TestConfiguration('lucid', 'x86', 'release'), 285 TestConfiguration('lucid', 'x86_64', 'debug'), 286 TestConfiguration('lucid', 'x86_64', 'release'), 287 TestConfiguration('icecreamsandwich', 'x86', 'debug'), 288 TestConfiguration('icecreamsandwich', 'x86', 'release'), 289 ])) 290 def test_get_crash_log(self): 291 port = self.make_port() 292 self.assertEqual(port._get_crash_log(None, None, None, None, newer_than=None), 293 (None, 294 'crash log for <unknown process name> (pid <unknown>):\n' 295 'STDOUT: <empty>\n' 296 'STDERR: <empty>\n')) 297 298 self.assertEqual(port._get_crash_log('foo', 1234, 'out bar\nout baz', 'err bar\nerr baz\n', newer_than=None), 299 ('err bar\nerr baz\n', 300 'crash log for foo (pid 1234):\n' 301 'STDOUT: out bar\n' 302 'STDOUT: out baz\n' 303 'STDERR: err bar\n' 304 'STDERR: err baz\n')) 305 306 self.assertEqual(port._get_crash_log('foo', 1234, 'foo\xa6bar', 'foo\xa6bar', newer_than=None), 307 ('foo\xa6bar', 308 u'crash log for foo (pid 1234):\n' 309 u'STDOUT: foo\ufffdbar\n' 310 u'STDERR: foo\ufffdbar\n')) 311 312 self.assertEqual(port._get_crash_log('foo', 1234, 'foo\xa6bar', 'foo\xa6bar', newer_than=1.0), 313 ('foo\xa6bar', 314 u'crash log for foo (pid 1234):\n' 315 u'STDOUT: foo\ufffdbar\n' 316 u'STDERR: foo\ufffdbar\n')) 317 318 def assert_build_path(self, options, dirs, expected_path): 319 port = self.make_port(options=options) 320 for directory in dirs: 321 port.host.filesystem.maybe_make_directory(directory) 322 self.assertEqual(port._build_path(), expected_path) 323 324 def test_expectations_files(self): 325 port = self.make_port() 326 327 generic_path = port.path_to_generic_test_expectations_file() 328 chromium_overrides_path = port.path_from_chromium_base( 329 'webkit', 'tools', 'layout_tests', 'test_expectations.txt') 330 never_fix_tests_path = port._filesystem.join(port.layout_tests_dir(), 'NeverFixTests') 331 stale_tests_path = port._filesystem.join(port.layout_tests_dir(), 'StaleTestExpectations') 332 slow_tests_path = port._filesystem.join(port.layout_tests_dir(), 'SlowTests') 333 flaky_tests_path = port._filesystem.join(port.layout_tests_dir(), 'FlakyTests') 334 skia_overrides_path = port.path_from_chromium_base( 335 'skia', 'skia_test_expectations.txt') 336 337 port._filesystem.write_text_file(skia_overrides_path, 'dummy text') 338 339 w3c_overrides_path = port.path_from_chromium_base( 340 'webkit', 'tools', 'layout_tests', 'test_expectations_w3c.txt') 341 port._filesystem.write_text_file(w3c_overrides_path, 'dummy text') 342 343 port._options.builder_name = 'DUMMY_BUILDER_NAME' 344 self.assertEqual(port.expectations_files(), 345 [generic_path, skia_overrides_path, w3c_overrides_path, 346 never_fix_tests_path, stale_tests_path, slow_tests_path, 347 flaky_tests_path, chromium_overrides_path]) 348 349 port._options.builder_name = 'builder (deps)' 350 self.assertEqual(port.expectations_files(), 351 [generic_path, skia_overrides_path, w3c_overrides_path, 352 never_fix_tests_path, stale_tests_path, slow_tests_path, 353 flaky_tests_path, chromium_overrides_path]) 354 355 # A builder which does NOT observe the Chromium test_expectations, 356 # but still observes the Skia test_expectations... 357 port._options.builder_name = 'builder' 358 self.assertEqual(port.expectations_files(), 359 [generic_path, skia_overrides_path, w3c_overrides_path, 360 never_fix_tests_path, stale_tests_path, slow_tests_path, 361 flaky_tests_path]) 362 363 def test_check_sys_deps(self): 364 port = self.make_port() 365 port._executive = MockExecutive2(exit_code=0) 366 self.assertEqual(port.check_sys_deps(needs_http=False), test_run_results.OK_EXIT_STATUS) 367 port._executive = MockExecutive2(exit_code=1, output='testing output failure') 368 self.assertEqual(port.check_sys_deps(needs_http=False), test_run_results.SYS_DEPS_EXIT_STATUS) 369 370 def test_expectations_ordering(self): 371 port = self.make_port() 372 for path in port.expectations_files(): 373 port._filesystem.write_text_file(path, '') 374 ordered_dict = port.expectations_dict() 375 self.assertEqual(port.path_to_generic_test_expectations_file(), ordered_dict.keys()[0]) 376 377 options = MockOptions(additional_expectations=['/tmp/foo', '/tmp/bar']) 378 port = self.make_port(options=options) 379 for path in port.expectations_files(): 380 port._filesystem.write_text_file(path, '') 381 port._filesystem.write_text_file('/tmp/foo', 'foo') 382 port._filesystem.write_text_file('/tmp/bar', 'bar') 383 ordered_dict = port.expectations_dict() 384 self.assertEqual(ordered_dict.keys()[-2:], options.additional_expectations) # pylint: disable=E1101 385 self.assertEqual(ordered_dict.values()[-2:], ['foo', 'bar']) 386 387 def test_skipped_directories_for_symbols(self): 388 # This first test confirms that the commonly found symbols result in the expected skipped directories. 389 symbols_string = " ".join(["fooSymbol"]) 390 expected_directories = set([ 391 "webaudio/codec-tests/mp3", 392 "webaudio/codec-tests/aac", 393 ]) 394 395 result_directories = set(TestWebKitPort(symbols_string=symbols_string)._skipped_tests_for_unsupported_features(test_list=['webaudio/codec-tests/mp3/foo.html'])) 396 self.assertEqual(result_directories, expected_directories) 397 398 # Test that the nm string parsing actually works: 399 symbols_string = """ 400000000000124f498 s __ZZN7WebCore13ff_mp3_decoder12replaceChildEPS0_S1_E19__PRETTY_FUNCTION__ 401000000000124f500 s __ZZN7WebCore13ff_mp3_decoder13addChildAboveEPS0_S1_E19__PRETTY_FUNCTION__ 402000000000124f670 s __ZZN7WebCore13ff_mp3_decoder13addChildBelowEPS0_S1_E19__PRETTY_FUNCTION__ 403""" 404 # Note 'compositing' is not in the list of skipped directories (hence the parsing of GraphicsLayer worked): 405 expected_directories = set([ 406 "webaudio/codec-tests/aac", 407 ]) 408 result_directories = set(TestWebKitPort(symbols_string=symbols_string)._skipped_tests_for_unsupported_features(test_list=['webaudio/codec-tests/mp3/foo.html'])) 409 self.assertEqual(result_directories, expected_directories) 410 411 def _assert_config_file_for_platform(self, port, platform, config_file): 412 self.assertEqual(port._apache_config_file_name_for_platform(platform), config_file) 413 414 def test_linux_distro_detection(self): 415 port = TestWebKitPort() 416 self.assertFalse(port._is_redhat_based()) 417 self.assertFalse(port._is_debian_based()) 418 419 port._filesystem = MockFileSystem({'/etc/redhat-release': ''}) 420 self.assertTrue(port._is_redhat_based()) 421 self.assertFalse(port._is_debian_based()) 422 423 port._filesystem = MockFileSystem({'/etc/debian_version': ''}) 424 self.assertFalse(port._is_redhat_based()) 425 self.assertTrue(port._is_debian_based()) 426 427 def test_apache_config_file_name_for_platform(self): 428 port = TestWebKitPort() 429 self._assert_config_file_for_platform(port, 'cygwin', 'cygwin-httpd.conf') 430 431 self._assert_config_file_for_platform(port, 'linux2', 'apache2-httpd.conf') 432 self._assert_config_file_for_platform(port, 'linux3', 'apache2-httpd.conf') 433 434 port._is_redhat_based = lambda: True 435 port._apache_version = lambda: '2.2' 436 self._assert_config_file_for_platform(port, 'linux2', 'fedora-httpd-2.2.conf') 437 438 port = TestWebKitPort() 439 port._is_debian_based = lambda: True 440 port._apache_version = lambda: '2.2' 441 self._assert_config_file_for_platform(port, 'linux2', 'debian-httpd-2.2.conf') 442 443 self._assert_config_file_for_platform(port, 'mac', 'apache2-httpd.conf') 444 self._assert_config_file_for_platform(port, 'win32', 'apache2-httpd.conf') # win32 isn't a supported sys.platform. AppleWin/WinCairo/WinCE ports all use cygwin. 445 self._assert_config_file_for_platform(port, 'barf', 'apache2-httpd.conf') 446 447 def test_path_to_apache_config_file(self): 448 port = TestWebKitPort() 449 450 saved_environ = os.environ.copy() 451 try: 452 os.environ['WEBKIT_HTTP_SERVER_CONF_PATH'] = '/path/to/httpd.conf' 453 self.assertRaises(IOError, port.path_to_apache_config_file) 454 port._filesystem.write_text_file('/existing/httpd.conf', 'Hello, world!') 455 os.environ['WEBKIT_HTTP_SERVER_CONF_PATH'] = '/existing/httpd.conf' 456 self.assertEqual(port.path_to_apache_config_file(), '/existing/httpd.conf') 457 finally: 458 os.environ = saved_environ.copy() 459 460 # Mock out _apache_config_file_name_for_platform to ignore the passed sys.platform value. 461 port._apache_config_file_name_for_platform = lambda platform: 'httpd.conf' 462 self.assertEqual(port.path_to_apache_config_file(), '/mock-checkout/third_party/WebKit/LayoutTests/http/conf/httpd.conf') 463 464 # Check that even if we mock out _apache_config_file_name, the environment variable takes precedence. 465 saved_environ = os.environ.copy() 466 try: 467 os.environ['WEBKIT_HTTP_SERVER_CONF_PATH'] = '/existing/httpd.conf' 468 self.assertEqual(port.path_to_apache_config_file(), '/existing/httpd.conf') 469 finally: 470 os.environ = saved_environ.copy() 471 472 def test_additional_platform_directory(self): 473 port = self.make_port(options=MockOptions(additional_platform_directory=['/tmp/foo'])) 474 self.assertEqual(port.baseline_search_path()[0], '/tmp/foo') 475