• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1# Copyright (C) 2010, 2012 Google Inc. All rights reserved.
2#
3# Redistribution and use in source and binary forms, with or without
4# modification, are permitted provided that the following conditions are
5# met:
6#
7#     * Redistributions of source code must retain the above copyright
8# notice, this list of conditions and the following disclaimer.
9#     * Redistributions in binary form must reproduce the above
10# copyright notice, this list of conditions and the following disclaimer
11# in the documentation and/or other materials provided with the
12# distribution.
13#     * Neither the name of Google Inc. nor the names of its
14# contributors may be used to endorse or promote products derived from
15# this software without specific prior written permission.
16#
17# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
18# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
19# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
20# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
21# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
22# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
23# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28
29"""Unit tests for printing.py."""
30
31import optparse
32import StringIO
33import sys
34import time
35import webkitpy.thirdparty.unittest2 as unittest
36
37from webkitpy.common.host_mock import MockHost
38
39from webkitpy.common.system import logtesting
40from webkitpy.layout_tests import port
41from webkitpy.layout_tests.controllers import manager
42from webkitpy.layout_tests.models import test_expectations
43from webkitpy.layout_tests.models import test_failures
44from webkitpy.layout_tests.models import test_results
45from webkitpy.layout_tests.views import printing
46
47
48def get_options(args):
49    print_options = printing.print_options()
50    option_parser = optparse.OptionParser(option_list=print_options)
51    return option_parser.parse_args(args)
52
53
54class TestUtilityFunctions(unittest.TestCase):
55    def test_print_options(self):
56        options, args = get_options([])
57        self.assertIsNotNone(options)
58
59
60class FakeRunResults(object):
61    def __init__(self, total=1, expected=1, unexpected=0, fake_results=None):
62        fake_results = fake_results or []
63        self.total = total
64        self.expected = expected
65        self.expected_failures = 0
66        self.unexpected = unexpected
67        self.expected_skips = 0
68        self.results_by_name = {}
69        total_run_time = 0
70        for result in fake_results:
71            self.results_by_name[result.shard_name] = result
72            total_run_time += result.total_run_time
73        self.run_time = total_run_time + 1
74
75
76class FakeShard(object):
77    def __init__(self, shard_name, total_run_time):
78        self.shard_name = shard_name
79        self.total_run_time = total_run_time
80
81
82class  Testprinter(unittest.TestCase):
83    def assertEmpty(self, stream):
84        self.assertFalse(stream.getvalue())
85
86    def assertNotEmpty(self, stream):
87        self.assertTrue(stream.getvalue())
88
89    def assertWritten(self, stream, contents):
90        self.assertEqual(stream.buflist, contents)
91
92    def reset(self, stream):
93        stream.buflist = []
94        stream.buf = ''
95
96    def get_printer(self, args=None):
97        args = args or []
98        printing_options = printing.print_options()
99        option_parser = optparse.OptionParser(option_list=printing_options)
100        options, args = option_parser.parse_args(args)
101        host = MockHost()
102        self._port = host.port_factory.get('test', options)
103        nproc = 2
104
105        regular_output = StringIO.StringIO()
106        printer = printing.Printer(self._port, options, regular_output)
107        return printer, regular_output
108
109    def get_result(self, test_name, result_type=test_expectations.PASS, run_time=0):
110        failures = []
111        if result_type == test_expectations.TIMEOUT:
112            failures = [test_failures.FailureTimeout()]
113        elif result_type == test_expectations.CRASH:
114            failures = [test_failures.FailureCrash()]
115        return test_results.TestResult(test_name, failures=failures, test_run_time=run_time)
116
117    def test_configure_and_cleanup(self):
118        # This test verifies that calling cleanup repeatedly and deleting
119        # the object is safe.
120        printer, err = self.get_printer()
121        printer.cleanup()
122        printer.cleanup()
123        printer = None
124
125    def test_print_config(self):
126        printer, err = self.get_printer()
127        # FIXME: it's lame that i have to set these options directly.
128        printer._options.pixel_tests = True
129        printer._options.new_baseline = True
130        printer._options.time_out_ms = 6000
131        printer._options.slow_time_out_ms = 12000
132        printer.print_config('/tmp')
133        self.assertIn("Using port 'test-mac-leopard'", err.getvalue())
134        self.assertIn('Test configuration: <leopard, x86, release>', err.getvalue())
135        self.assertIn('View the test results at file:///tmp', err.getvalue())
136        self.assertIn('Baseline search path: test-mac-leopard -> test-mac-snowleopard -> generic', err.getvalue())
137        self.assertIn('Using Release build', err.getvalue())
138        self.assertIn('Pixel tests enabled', err.getvalue())
139        self.assertIn('Command line:', err.getvalue())
140        self.assertIn('Regular timeout: ', err.getvalue())
141
142        self.reset(err)
143        printer._options.quiet = True
144        printer.print_config('/tmp')
145        self.assertNotIn('Baseline search path: test-mac-leopard -> test-mac-snowleopard -> generic', err.getvalue())
146
147    def test_print_directory_timings(self):
148        printer, err = self.get_printer()
149        printer._options.debug_rwt_logging = True
150
151        run_results = FakeRunResults()
152        run_results.results_by_name = {
153            "slowShard": FakeShard("slowShard", 16),
154            "borderlineShard": FakeShard("borderlineShard", 15),
155            "fastShard": FakeShard("fastShard", 1),
156        }
157
158        printer._print_directory_timings(run_results)
159        self.assertWritten(err, ['Time to process slowest subdirectories:\n', '  slowShard took 16.0 seconds to run 1 tests.\n', '\n'])
160
161        printer, err = self.get_printer()
162        printer._options.debug_rwt_logging = True
163
164        run_results.results_by_name = {
165            "borderlineShard": FakeShard("borderlineShard", 15),
166            "fastShard": FakeShard("fastShard", 1),
167        }
168
169        printer._print_directory_timings(run_results)
170        self.assertWritten(err, [])
171
172    def test_print_one_line_summary(self):
173        def run_test(total, exp, unexp, shards, result):
174            printer, err = self.get_printer(['--timing'] if shards else None)
175            fake_results = FakeRunResults(total, exp, unexp, shards)
176            total_time = fake_results.run_time + 1
177            printer._print_one_line_summary(total_time, fake_results)
178            self.assertWritten(err, result)
179
180        # Without times:
181        run_test(1, 1, 0, [], ["The test ran as expected.\n", "\n"])
182        run_test(2, 1, 1, [], ["\n", "1 test ran as expected, 1 didn't:\n", "\n"])
183        run_test(3, 2, 1, [], ["\n", "2 tests ran as expected, 1 didn't:\n", "\n"])
184        run_test(3, 2, 0, [], ["\n", "2 tests ran as expected (1 didn't run).\n", "\n"])
185
186        # With times:
187        fake_shards = [FakeShard("foo", 1), FakeShard("bar", 2)]
188        run_test(1, 1, 0, fake_shards, ["The test ran as expected in 5.00s (2.00s in rwt, 1x).\n", "\n"])
189        run_test(2, 1, 1, fake_shards, ["\n", "1 test ran as expected, 1 didn't in 5.00s (2.00s in rwt, 1x):\n", "\n"])
190        run_test(3, 2, 1, fake_shards, ["\n", "2 tests ran as expected, 1 didn't in 5.00s (2.00s in rwt, 1x):\n", "\n"])
191        run_test(3, 2, 0, fake_shards, ["\n", "2 tests ran as expected (1 didn't run) in 5.00s (2.00s in rwt, 1x).\n", "\n"])
192
193    def test_test_status_line(self):
194        printer, _ = self.get_printer()
195        printer._meter.number_of_columns = lambda: 80
196        actual = printer._test_status_line('fast/dom/HTMLFormElement/associated-elements-after-index-assertion-fail1.html', ' passed')
197        self.assertEqual(80, len(actual))
198        self.assertEqual(actual, '[0/0] fast/dom/HTMLFormElement/associa...after-index-assertion-fail1.html passed')
199
200        printer._meter.number_of_columns = lambda: 89
201        actual = printer._test_status_line('fast/dom/HTMLFormElement/associated-elements-after-index-assertion-fail1.html', ' passed')
202        self.assertEqual(89, len(actual))
203        self.assertEqual(actual, '[0/0] fast/dom/HTMLFormElement/associated-...ents-after-index-assertion-fail1.html passed')
204
205        printer._meter.number_of_columns = lambda: sys.maxint
206        actual = printer._test_status_line('fast/dom/HTMLFormElement/associated-elements-after-index-assertion-fail1.html', ' passed')
207        self.assertEqual(90, len(actual))
208        self.assertEqual(actual, '[0/0] fast/dom/HTMLFormElement/associated-elements-after-index-assertion-fail1.html passed')
209
210        printer._meter.number_of_columns = lambda: 18
211        actual = printer._test_status_line('fast/dom/HTMLFormElement/associated-elements-after-index-assertion-fail1.html', ' passed')
212        self.assertEqual(18, len(actual))
213        self.assertEqual(actual, '[0/0] f...l passed')
214
215        printer._meter.number_of_columns = lambda: 10
216        actual = printer._test_status_line('fast/dom/HTMLFormElement/associated-elements-after-index-assertion-fail1.html', ' passed')
217        self.assertEqual(actual, '[0/0] associated-elements-after-index-assertion-fail1.html passed')
218
219    def test_details(self):
220        printer, err = self.get_printer(['--details'])
221        result = self.get_result('passes/image.html')
222        printer.print_started_test('passes/image.html')
223        printer.print_finished_test(result, expected=False, exp_str='', got_str='')
224        self.assertNotEmpty(err)
225
226    def test_print_found(self):
227        printer, err = self.get_printer()
228
229        printer.print_found(100, 10, 1, 1)
230        self.assertWritten(err, ["Found 100 tests; running 10, skipping 90.\n"])
231
232        self.reset(err)
233        printer.print_found(100, 10, 2, 3)
234        self.assertWritten(err, ["Found 100 tests; running 10 (6 times each: --repeat-each=2 --iterations=3), skipping 90.\n"])
235
236    def test_debug_rwt_logging_is_throttled(self):
237        printer, err = self.get_printer(['--debug-rwt-logging'])
238
239        result = self.get_result('passes/image.html')
240        printer.print_started_test('passes/image.html')
241        printer.print_finished_test(result, expected=True, exp_str='', got_str='')
242
243        printer.print_started_test('passes/text.html')
244        result = self.get_result('passes/text.html')
245        printer.print_finished_test(result, expected=True, exp_str='', got_str='')
246
247        # Only the first test's start should be printed.
248        lines = err.buflist
249        self.assertEqual(len(lines), 1)
250        self.assertTrue(lines[0].endswith('passes/image.html\n'))
251