• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1# Copyright (C) 2011 Google Inc. All rights reserved.
2#
3# Redistribution and use in source and binary forms, with or without
4# modification, are permitted provided that the following conditions are
5# met:
6#
7#     * Redistributions of source code must retain the above copyright
8# notice, this list of conditions and the following disclaimer.
9#     * Redistributions in binary form must reproduce the above
10# copyright notice, this list of conditions and the following disclaimer
11# in the documentation and/or other materials provided with the
12# distribution.
13#     * Neither the name of Google Inc. nor the names of its
14# contributors may be used to endorse or promote products derived from
15# this software without specific prior written permission.
16#
17# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
18# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
19# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
20# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
21# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
22# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
23# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28
29
30import logging
31
32from webkitpy.layout_tests.controllers import repaint_overlay
33from webkitpy.layout_tests.models import test_failures
34
35
36_log = logging.getLogger(__name__)
37
38
39def write_test_result(filesystem, port, results_directory, test_name, driver_output,
40                      expected_driver_output, failures):
41    """Write the test result to the result output directory."""
42    root_output_dir = results_directory
43    writer = TestResultWriter(filesystem, port, root_output_dir, test_name)
44
45    if driver_output.error:
46        writer.write_stderr(driver_output.error)
47
48    for failure in failures:
49        # FIXME: Instead of this long 'if' block, each failure class might
50        # have a responsibility for writing a test result.
51        if isinstance(failure, (test_failures.FailureMissingResult,
52                                test_failures.FailureTextMismatch,
53                                test_failures.FailureTestHarnessAssertion)):
54            writer.write_text_files(driver_output.text, expected_driver_output.text)
55            writer.create_text_diff_and_write_result(driver_output.text, expected_driver_output.text)
56            writer.create_repaint_overlay_result(driver_output.text, expected_driver_output.text)
57        elif isinstance(failure, test_failures.FailureMissingImage):
58            writer.write_image_files(driver_output.image, expected_image=None)
59        elif isinstance(failure, test_failures.FailureMissingImageHash):
60            writer.write_image_files(driver_output.image, expected_driver_output.image)
61        elif isinstance(failure, test_failures.FailureImageHashMismatch):
62            writer.write_image_files(driver_output.image, expected_driver_output.image)
63            writer.write_image_diff_files(driver_output.image_diff)
64        elif isinstance(failure, (test_failures.FailureAudioMismatch,
65                                  test_failures.FailureMissingAudio)):
66            writer.write_audio_files(driver_output.audio, expected_driver_output.audio)
67        elif isinstance(failure, test_failures.FailureCrash):
68            crashed_driver_output = expected_driver_output if failure.is_reftest else driver_output
69            writer.write_crash_log(crashed_driver_output.crash_log)
70        elif isinstance(failure, test_failures.FailureLeak):
71            writer.write_leak_log(driver_output.leak_log)
72        elif isinstance(failure, test_failures.FailureReftestMismatch):
73            writer.write_image_files(driver_output.image, expected_driver_output.image)
74            # FIXME: This work should be done earlier in the pipeline (e.g., when we compare images for non-ref tests).
75            # FIXME: We should always have 2 images here.
76            if driver_output.image and expected_driver_output.image:
77                diff_image, err_str = port.diff_image(expected_driver_output.image, driver_output.image)
78                if diff_image:
79                    writer.write_image_diff_files(diff_image)
80                else:
81                    _log.warn('ref test mismatch did not produce an image diff.')
82            writer.write_image_files(driver_output.image, expected_image=None)
83            if filesystem.exists(failure.reference_filename):
84                writer.write_reftest(failure.reference_filename)
85            else:
86                _log.warn("reference %s was not found" % failure.reference_filename)
87        elif isinstance(failure, test_failures.FailureReftestMismatchDidNotOccur):
88            writer.write_image_files(driver_output.image, expected_image=None)
89            if filesystem.exists(failure.reference_filename):
90                writer.write_reftest(failure.reference_filename)
91            else:
92                _log.warn("reference %s was not found" % failure.reference_filename)
93        else:
94            assert isinstance(failure, (test_failures.FailureTimeout, test_failures.FailureReftestNoImagesGenerated))
95
96
97class TestResultWriter(object):
98    """A class which handles all writing operations to the result directory."""
99
100    # Filename pieces when writing failures to the test results directory.
101    FILENAME_SUFFIX_ACTUAL = "-actual"
102    FILENAME_SUFFIX_EXPECTED = "-expected"
103    FILENAME_SUFFIX_DIFF = "-diff"
104    FILENAME_SUFFIX_STDERR = "-stderr"
105    FILENAME_SUFFIX_CRASH_LOG = "-crash-log"
106    FILENAME_SUFFIX_SAMPLE = "-sample"
107    FILENAME_SUFFIX_LEAK_LOG = "-leak-log"
108    FILENAME_SUFFIX_WDIFF = "-wdiff.html"
109    FILENAME_SUFFIX_PRETTY_PATCH = "-pretty-diff.html"
110    FILENAME_SUFFIX_IMAGE_DIFF = "-diff.png"
111    FILENAME_SUFFIX_IMAGE_DIFFS_HTML = "-diffs.html"
112    FILENAME_SUFFIX_OVERLAY = "-overlay.html"
113
114    def __init__(self, filesystem, port, root_output_dir, test_name):
115        self._filesystem = filesystem
116        self._port = port
117        self._root_output_dir = root_output_dir
118        self._test_name = test_name
119
120    def _make_output_directory(self):
121        """Creates the output directory (if needed) for a given test filename."""
122        fs = self._filesystem
123        output_filename = fs.join(self._root_output_dir, self._test_name)
124        fs.maybe_make_directory(fs.dirname(output_filename))
125
126    def output_filename(self, modifier):
127        """Returns a filename inside the output dir that contains modifier.
128
129        For example, if test name is "fast/dom/foo.html" and modifier is "-expected.txt",
130        the return value is "/<path-to-root-output-dir>/fast/dom/foo-expected.txt".
131
132        Args:
133          modifier: a string to replace the extension of filename with
134
135        Return:
136          The absolute path to the output filename
137        """
138        fs = self._filesystem
139        output_filename = fs.join(self._root_output_dir, self._test_name)
140        return fs.splitext(output_filename)[0] + modifier
141
142    def _write_file(self, path, contents):
143        if contents is not None:
144            self._make_output_directory()
145            self._filesystem.write_binary_file(path, contents)
146
147    def _output_testname(self, modifier):
148        fs = self._filesystem
149        return fs.splitext(fs.basename(self._test_name))[0] + modifier
150
151    def write_output_files(self, file_type, output, expected):
152        """Writes the test output, the expected output in the results directory.
153
154        The full output filename of the actual, for example, will be
155          <filename>-actual<file_type>
156        For instance,
157          my_test-actual.txt
158
159        Args:
160          file_type: A string describing the test output file type, e.g. ".txt"
161          output: A string containing the test output
162          expected: A string containing the expected test output
163        """
164        actual_filename = self.output_filename(self.FILENAME_SUFFIX_ACTUAL + file_type)
165        expected_filename = self.output_filename(self.FILENAME_SUFFIX_EXPECTED + file_type)
166
167        self._write_file(actual_filename, output)
168        self._write_file(expected_filename, expected)
169
170    def write_stderr(self, error):
171        filename = self.output_filename(self.FILENAME_SUFFIX_STDERR + ".txt")
172        self._write_file(filename, error)
173
174    def write_crash_log(self, crash_log):
175        filename = self.output_filename(self.FILENAME_SUFFIX_CRASH_LOG + ".txt")
176        self._write_file(filename, crash_log.encode('utf8', 'replace'))
177
178    def write_leak_log(self, leak_log):
179        filename = self.output_filename(self.FILENAME_SUFFIX_LEAK_LOG + ".txt")
180        self._write_file(filename, leak_log)
181
182    def copy_sample_file(self, sample_file):
183        filename = self.output_filename(self.FILENAME_SUFFIX_SAMPLE + ".txt")
184        self._filesystem.copyfile(sample_file, filename)
185
186    def write_text_files(self, actual_text, expected_text):
187        self.write_output_files(".txt", actual_text, expected_text)
188
189    def create_text_diff_and_write_result(self, actual_text, expected_text):
190        # FIXME: This function is actually doing the diffs as well as writing results.
191        # It might be better to extract code which does 'diff' and make it a separate function.
192        if not actual_text or not expected_text:
193            return
194
195        file_type = '.txt'
196        actual_filename = self.output_filename(self.FILENAME_SUFFIX_ACTUAL + file_type)
197        expected_filename = self.output_filename(self.FILENAME_SUFFIX_EXPECTED + file_type)
198        # We treat diff output as binary. Diff output may contain multiple files
199        # in conflicting encodings.
200        diff = self._port.diff_text(expected_text, actual_text, expected_filename, actual_filename)
201        diff_filename = self.output_filename(self.FILENAME_SUFFIX_DIFF + file_type)
202        self._write_file(diff_filename, diff)
203
204        # Shell out to wdiff to get colored inline diffs.
205        if self._port.wdiff_available():
206            wdiff = self._port.wdiff_text(expected_filename, actual_filename)
207            wdiff_filename = self.output_filename(self.FILENAME_SUFFIX_WDIFF)
208            self._write_file(wdiff_filename, wdiff)
209
210        # Use WebKit's PrettyPatch.rb to get an HTML diff.
211        if self._port.pretty_patch_available():
212            pretty_patch = self._port.pretty_patch_text(diff_filename)
213            pretty_patch_filename = self.output_filename(self.FILENAME_SUFFIX_PRETTY_PATCH)
214            self._write_file(pretty_patch_filename, pretty_patch)
215
216    def create_repaint_overlay_result(self, actual_text, expected_text):
217        html = repaint_overlay.generate_repaint_overlay_html(self._test_name, actual_text, expected_text)
218        if html:
219            overlay_filename = self.output_filename(self.FILENAME_SUFFIX_OVERLAY)
220            self._write_file(overlay_filename, html)
221
222    def write_audio_files(self, actual_audio, expected_audio):
223        self.write_output_files('.wav', actual_audio, expected_audio)
224
225    def write_image_files(self, actual_image, expected_image):
226        self.write_output_files('.png', actual_image, expected_image)
227
228    def write_image_diff_files(self, image_diff):
229        diff_filename = self.output_filename(self.FILENAME_SUFFIX_IMAGE_DIFF)
230        self._write_file(diff_filename, image_diff)
231
232        diffs_html_filename = self.output_filename(self.FILENAME_SUFFIX_IMAGE_DIFFS_HTML)
233        # FIXME: old-run-webkit-tests shows the diff percentage as the text contents of the "diff" link.
234        # FIXME: old-run-webkit-tests include a link to the test file.
235        html = """<!DOCTYPE HTML>
236<html>
237<head>
238<title>%(title)s</title>
239<style>.label{font-weight:bold}</style>
240</head>
241<body>
242Difference between images: <a href="%(diff_filename)s">diff</a><br>
243<div class=imageText></div>
244<div class=imageContainer data-prefix="%(prefix)s">Loading...</div>
245<script>
246(function() {
247    var preloadedImageCount = 0;
248    function preloadComplete() {
249        ++preloadedImageCount;
250        if (preloadedImageCount < 2)
251            return;
252        toggleImages();
253        setInterval(toggleImages, 2000)
254    }
255
256    function preloadImage(url) {
257        image = new Image();
258        image.addEventListener('load', preloadComplete);
259        image.src = url;
260        return image;
261    }
262
263    function toggleImages() {
264        if (text.textContent == 'Expected Image') {
265            text.textContent = 'Actual Image';
266            container.replaceChild(actualImage, container.firstChild);
267        } else {
268            text.textContent = 'Expected Image';
269            container.replaceChild(expectedImage, container.firstChild);
270        }
271    }
272
273    var text = document.querySelector('.imageText');
274    var container = document.querySelector('.imageContainer');
275    var actualImage = preloadImage(container.getAttribute('data-prefix') + '-actual.png');
276    var expectedImage = preloadImage(container.getAttribute('data-prefix') + '-expected.png');
277})();
278</script>
279</body>
280</html>
281""" % {
282            'title': self._test_name,
283            'diff_filename': self._output_testname(self.FILENAME_SUFFIX_IMAGE_DIFF),
284            'prefix': self._output_testname(''),
285        }
286        self._write_file(diffs_html_filename, html)
287
288    def write_reftest(self, src_filepath):
289        fs = self._filesystem
290        dst_dir = fs.dirname(fs.join(self._root_output_dir, self._test_name))
291        dst_filepath = fs.join(dst_dir, fs.basename(src_filepath))
292        self._write_file(dst_filepath, fs.read_binary_file(src_filepath))
293