1# Copyright 2013 the V8 project authors. All rights reserved. 2# Redistribution and use in source and binary forms, with or without 3# modification, are permitted provided that the following conditions are 4# met: 5# 6# * Redistributions of source code must retain the above copyright 7# notice, this list of conditions and the following disclaimer. 8# * Redistributions in binary form must reproduce the above 9# copyright notice, this list of conditions and the following 10# disclaimer in the documentation and/or other materials provided 11# with the distribution. 12# * Neither the name of Google Inc. nor the names of its 13# contributors may be used to endorse or promote products derived 14# from this software without specific prior written permission. 15# 16# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 17# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 18# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 19# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 20# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 21# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 22# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 23# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 24# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 25# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 26# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27 28import itertools 29import os 30import re 31 32from testrunner.local import testsuite 33from testrunner.objects import testcase 34 35FLAGS_PATTERN = re.compile(r"//\s+Flags:(.*)") 36FILES_PATTERN = re.compile(r"//\s+Files:(.*)") 37SELF_SCRIPT_PATTERN = re.compile(r"//\s+Env: TEST_FILE_NAME") 38 39 40# TODO (machenbach): Share commonalities with mjstest. 41class WebkitTestSuite(testsuite.TestSuite): 42 43 def __init__(self, name, root): 44 super(WebkitTestSuite, self).__init__(name, root) 45 46 def ListTests(self, context): 47 tests = [] 48 for dirname, dirs, files in os.walk(self.root): 49 for dotted in [x for x in dirs if x.startswith('.')]: 50 dirs.remove(dotted) 51 if 'resources' in dirs: 52 dirs.remove('resources') 53 54 dirs.sort() 55 files.sort() 56 for filename in files: 57 if filename.endswith(".js"): 58 testname = os.path.join(dirname[len(self.root) + 1:], filename[:-3]) 59 test = testcase.TestCase(self, testname) 60 tests.append(test) 61 return tests 62 63 def GetFlagsForTestCase(self, testcase, context): 64 source = self.GetSourceForTest(testcase) 65 flags = [] + context.mode_flags 66 flags_match = re.findall(FLAGS_PATTERN, source) 67 for match in flags_match: 68 flags += match.strip().split() 69 70 files_list = [] # List of file names to append to command arguments. 71 files_match = FILES_PATTERN.search(source); 72 # Accept several lines of 'Files:'. 73 while True: 74 if files_match: 75 files_list += files_match.group(1).strip().split() 76 files_match = FILES_PATTERN.search(source, files_match.end()) 77 else: 78 break 79 files = [ os.path.normpath(os.path.join(self.root, '..', '..', f)) 80 for f in files_list ] 81 testfilename = os.path.join(self.root, testcase.path + self.suffix()) 82 if SELF_SCRIPT_PATTERN.search(source): 83 env = ["-e", "TEST_FILE_NAME=\"%s\"" % testfilename.replace("\\", "\\\\")] 84 files = env + files 85 files.append(os.path.join(self.root, "resources/standalone-pre.js")) 86 files.append(testfilename) 87 files.append(os.path.join(self.root, "resources/standalone-post.js")) 88 89 flags += files 90 if context.isolates: 91 flags.append("--isolate") 92 flags += files 93 94 return testcase.flags + flags 95 96 def GetSourceForTest(self, testcase): 97 filename = os.path.join(self.root, testcase.path + self.suffix()) 98 with open(filename) as f: 99 return f.read() 100 101 # TODO(machenbach): Share with test/message/testcfg.py 102 def _IgnoreLine(self, string): 103 """Ignore empty lines, valgrind output and Android output.""" 104 if not string: return True 105 return (string.startswith("==") or string.startswith("**") or 106 string.startswith("ANDROID") or 107 # These five patterns appear in normal Native Client output. 108 string.startswith("DEBUG MODE ENABLED") or 109 string.startswith("tools/nacl-run.py") or 110 string.find("BYPASSING ALL ACL CHECKS") > 0 or 111 string.find("Native Client module will be loaded") > 0 or 112 string.find("NaClHostDescOpen:") > 0) 113 114 def IsFailureOutput(self, output, testpath): 115 if super(WebkitTestSuite, self).IsFailureOutput(output, testpath): 116 return True 117 file_name = os.path.join(self.root, testpath) + "-expected.txt" 118 with file(file_name, "r") as expected: 119 expected_lines = expected.readlines() 120 121 def ExpIterator(): 122 for line in expected_lines: 123 if line.startswith("#") or not line.strip(): continue 124 yield line.strip() 125 126 def ActIterator(lines): 127 for line in lines: 128 if self._IgnoreLine(line.strip()): continue 129 yield line.strip() 130 131 def ActBlockIterator(): 132 """Iterates over blocks of actual output lines.""" 133 lines = output.stdout.splitlines() 134 start_index = 0 135 found_eqeq = False 136 for index, line in enumerate(lines): 137 # If a stress test separator is found: 138 if line.startswith("=="): 139 # Iterate over all lines before a separator except the first. 140 if not found_eqeq: 141 found_eqeq = True 142 else: 143 yield ActIterator(lines[start_index:index]) 144 # The next block of ouput lines starts after the separator. 145 start_index = index + 1 146 # Iterate over complete output if no separator was found. 147 if not found_eqeq: 148 yield ActIterator(lines) 149 150 for act_iterator in ActBlockIterator(): 151 for (expected, actual) in itertools.izip_longest( 152 ExpIterator(), act_iterator, fillvalue=''): 153 if expected != actual: 154 return True 155 return False 156 157 158def GetSuite(name, root): 159 return WebkitTestSuite(name, root) 160