1#!/usr/bin/env python 2# Copyright (C) 2010 Google Inc. All rights reserved. 3# 4# Redistribution and use in source and binary forms, with or without 5# modification, are permitted provided that the following conditions are 6# met: 7# 8# * Redistributions of source code must retain the above copyright 9# notice, this list of conditions and the following disclaimer. 10# * Redistributions in binary form must reproduce the above 11# copyright notice, this list of conditions and the following disclaimer 12# in the documentation and/or other materials provided with the 13# distribution. 14# * Neither the name of Google Inc. nor the names of its 15# contributors may be used to endorse or promote products derived from 16# this software without specific prior written permission. 17# 18# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 19# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 20# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 21# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 22# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 23# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 24# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 25# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 26# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 27# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 28# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 30"""Compares the image output of a test to the expected image output. 31 32Compares hashes for the generated and expected images. If the output doesn't 33match, returns FailureImageHashMismatch and outputs both hashes into the layout 34test results directory. 35""" 36 37import errno 38import logging 39import os 40import shutil 41 42from layout_package import test_failures 43from test_types import test_type_base 44 45# Cache whether we have the image_diff executable available. 46_compare_available = True 47_compare_msg_printed = False 48 49 50class ImageDiff(test_type_base.TestTypeBase): 51 52 def _copy_output_png(self, test_filename, source_image, extension): 53 """Copies result files into the output directory with appropriate 54 names. 55 56 Args: 57 test_filename: the test filename 58 source_file: path to the image file (either actual or expected) 59 extension: extension to indicate -actual.png or -expected.png 60 """ 61 self._make_output_directory(test_filename) 62 dest_image = self.output_filename(test_filename, extension) 63 64 try: 65 shutil.copyfile(source_image, dest_image) 66 except IOError, e: 67 # A missing expected PNG has already been recorded as an error. 68 if errno.ENOENT != e.errno: 69 raise 70 71 def _save_baseline_files(self, filename, png_path, checksum): 72 """Saves new baselines for the PNG and checksum. 73 74 Args: 75 filename: test filename 76 png_path: path to the actual PNG result file 77 checksum: value of the actual checksum result 78 """ 79 png_file = open(png_path, "rb") 80 png_data = png_file.read() 81 png_file.close() 82 self._save_baseline_data(filename, png_data, ".png") 83 self._save_baseline_data(filename, checksum, ".checksum") 84 85 def _create_image_diff(self, port, filename, target): 86 """Creates the visual diff of the expected/actual PNGs. 87 88 Args: 89 filename: the name of the test 90 target: Debug or Release 91 """ 92 diff_filename = self.output_filename(filename, 93 self.FILENAME_SUFFIX_COMPARE) 94 actual_filename = self.output_filename(filename, 95 self.FILENAME_SUFFIX_ACTUAL + '.png') 96 expected_filename = self.output_filename(filename, 97 self.FILENAME_SUFFIX_EXPECTED + '.png') 98 99 try: 100 _compare_available = True 101 result = port.diff_image(actual_filename, expected_filename, 102 diff_filename) 103 except ValueError: 104 _compare_available = False 105 106 global _compare_msg_printed 107 if not _compare_available and not _compare_msg_printed: 108 _compare_msg_printed = True 109 print('image_diff not found. Make sure you have a ' + target + 110 ' build of the image_diff executable.') 111 112 return result 113 114 def compare_output(self, port, filename, output, test_args, target): 115 """Implementation of CompareOutput that checks the output image and 116 checksum against the expected files from the LayoutTest directory. 117 """ 118 failures = [] 119 120 # If we didn't produce a hash file, this test must be text-only. 121 if test_args.hash is None: 122 return failures 123 124 # If we're generating a new baseline, we pass. 125 if test_args.new_baseline: 126 self._save_baseline_files(filename, test_args.png_path, 127 test_args.hash) 128 return failures 129 130 # Compare hashes. 131 expected_hash_file = self._port.expected_filename(filename, 132 '.checksum') 133 expected_png_file = self._port.expected_filename(filename, '.png') 134 135 if test_args.show_sources: 136 logging.debug('Using %s' % expected_hash_file) 137 logging.debug('Using %s' % expected_png_file) 138 139 try: 140 expected_hash = open(expected_hash_file, "r").read() 141 except IOError, e: 142 if errno.ENOENT != e.errno: 143 raise 144 expected_hash = '' 145 146 147 if not os.path.isfile(expected_png_file): 148 # Report a missing expected PNG file. 149 self.write_output_files(port, filename, '', '.checksum', 150 test_args.hash, expected_hash, 151 diff=False, wdiff=False) 152 self._copy_output_png(filename, test_args.png_path, '-actual.png') 153 failures.append(test_failures.FailureMissingImage(self)) 154 return failures 155 elif test_args.hash == expected_hash: 156 # Hash matched (no diff needed, okay to return). 157 return failures 158 159 160 self.write_output_files(port, filename, '', '.checksum', 161 test_args.hash, expected_hash, 162 diff=False, wdiff=False) 163 self._copy_output_png(filename, test_args.png_path, '-actual.png') 164 self._copy_output_png(filename, expected_png_file, '-expected.png') 165 166 # Even though we only use result in one codepath below but we 167 # still need to call CreateImageDiff for other codepaths. 168 result = self._create_image_diff(port, filename, target) 169 if expected_hash == '': 170 failures.append(test_failures.FailureMissingImageHash(self)) 171 elif test_args.hash != expected_hash: 172 # Hashes don't match, so see if the images match. If they do, then 173 # the hash is wrong. 174 if result == 0: 175 failures.append(test_failures.FailureImageHashIncorrect(self)) 176 else: 177 failures.append(test_failures.FailureImageHashMismatch(self)) 178 179 return failures 180 181 def diff_files(self, port, file1, file2): 182 """Diff two image files. 183 184 Args: 185 file1, file2: full paths of the files to compare. 186 187 Returns: 188 True if two files are different. 189 False otherwise. 190 """ 191 192 try: 193 result = port.diff_image(file1, file2) 194 except ValueError, e: 195 return True 196 197 return result == 1 198