1#!/usr/bin/env python 2# Copyright 2020 The Tint Authors. 3# 4# Licensed under the Apache License, Version 2.0 (the "License"); 5# you may not use this file except in compliance with the License. 6# You may obtain a copy of the License at 7# 8# http://www.apache.org/licenses/LICENSE-2.0 9# 10# Unless required by applicable law or agreed to in writing, software 11# distributed under the License is distributed on an "AS IS" BASIS, 12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13# See the License for the specific language governing permissions and 14# limitations under the License. 15 16# Test runner for executing a test of tests with Tint. The runner will 17# find all .wgsl files in the given folder and attempt to convert them 18# to each of the backend formats. If the file contains a '.fail.' in the 19# name then the runner will expect the file to fail conversion. 20 21import base64 22import copy 23import difflib 24import optparse 25import os 26import platform 27import re 28import subprocess 29import sys 30import tempfile 31 32 33""" 34A single test case to be executed. Stores the path to the test file 35and the result of executing the test. 36""" 37 38 39class TestCase: 40 def __init__(self, input_path, parse_only): 41 self.input_path = input_path 42 self.parse_only = parse_only 43 self.results = {} 44 45 def IsExpectedFail(self): 46 fail_re = re.compile('^.+[\.]fail[\.]wgsl') 47 return fail_re.match(self.GetInputPath()) 48 49 def IsParseOnly(self): 50 return self.parse_only 51 52 def GetInputPath(self): 53 return self.input_path 54 55 def GetResult(self, fmt): 56 return self.results[fmt] 57 58 def GetReason(self): 59 with open(self.GetInputPath()) as test: 60 first_line = test.readline() 61 if (first_line.startswith("# v-")): 62 reason = first_line[2:8] 63 else: 64 reason = '' 65 return reason 66 67 68""" 69The test runner, will execute a series of test cases and record the 70results. 71""" 72 73 74class TestRunner: 75 def RunTest(self, tc): 76 """Runs a single test.""" 77 print("Testing {}".format(tc.GetInputPath())) 78 79 cmd = [self.options.test_prog_path] 80 if tc.IsParseOnly(): 81 cmd += ['--parse-only'] 82 83 languages = ["wgsl", "spvasm", "msl", "hlsl"] 84 try: 85 for lang in languages: 86 lang_cmd = copy.copy(cmd) 87 lang_cmd += ['--format', lang] 88 lang_cmd += [tc.GetInputPath()] 89 err = subprocess.check_output(lang_cmd, 90 stderr=subprocess.STDOUT) 91 92 except Exception as e: 93 failure_reason = "{}".format("".join(map(chr, 94 bytearray(e.output)))) 95 if tc.IsExpectedFail(): 96 right_reason = tc.GetReason() 97 if (right_reason in failure_reason): 98 return False, "" 99 else: 100 return False, right_reason 101 102 if not tc.IsExpectedFail(): 103 print(failure_reason) 104 print(e) 105 return False, "" 106 107 return True, "" 108 109 def RunTests(self): 110 """Runs a set of test cases""" 111 for tc in self.test_cases: 112 result, reason = self.RunTest(tc) 113 """evaluate final result based on result, tc.IsExpectedFail() and reason""" 114 if not result: 115 # result == false, expected true, reason:don't care 116 if not tc.IsExpectedFail(): 117 print("Expected: " + tc.GetInputPath() + 118 " to pass but failed.") 119 self.failures.append(tc.GetInputPath()) 120 # result == false, expected false, reason: wrong 121 else: 122 if reason.startswith("v-"): 123 print("Failed for a wrong reason: " + 124 tc.GetInputPath() + 125 " expected with error code: " + reason) 126 self.failures_wrong_reason.append(tc.GetInputPath()) 127 # result == true, expected false, reason:don't care 128 elif tc.IsExpectedFail(): 129 print("Expected: " + tc.GetInputPath() + 130 " to fail but passed.") 131 self.failures.append(tc.GetInputPath()) 132 133 def GetUnexpectedFailures(self): 134 for failure in self.failures + self.failures_wrong_reason: 135 if failure not in self.known_failures: 136 self.unexpected_failures.append(failure) 137 return 138 139 def SummarizeResults(self): 140 """Prints a summarization of the test results to STDOUT""" 141 if len(self.unexpected_failures): 142 self.unexpected_failures.sort() 143 print('\nSummary of unexpected failures:') 144 for unexpected_fail in self.unexpected_failures: 145 print(unexpected_fail) 146 147 for f in self.known_failures: 148 if f not in self.failures_wrong_reason + self.failures: 149 self.unexpected_successes.append(f) 150 151 if len(self.unexpected_successes): 152 print('\nSummary of unexpected successes:') 153 for s in self.unexpected_successes: 154 print(s) 155 156 print('') 157 print('Test cases executed: {}'.format(len(self.test_cases))) 158 print(' Successes: {}'.format( 159 (len(self.test_cases) - len(self.failures) - 160 len(self.failures_wrong_reason)))) 161 print(' Failures: {}'.format( 162 len(self.failures) + len(self.failures_wrong_reason))) 163 print(' Unexpected Failures: {}'.format(len( 164 self.unexpected_failures))) 165 print(' Unexpected Successes: {}'.format( 166 len(self.unexpected_successes))) 167 print('') 168 169 def Run(self): 170 """Executes the test runner.""" 171 base_path = os.path.abspath( 172 os.path.join(os.path.dirname(__file__), '..')) 173 174 usage = 'usage: %prog [options] (file)' 175 parser = optparse.OptionParser(usage=usage) 176 parser.add_option('--build-dir', 177 default=os.path.join(base_path, 'out', 'Debug'), 178 help='path to build directory') 179 parser.add_option('--test-dir', 180 default=os.path.join(os.path.dirname(__file__), '..', 181 'third_party', 'gpuweb-cts', 182 'src', 'webgpu', 'shader', 183 'validation', 'wgsl'), 184 help='path to directory containing test files') 185 parser.add_option( 186 '--known-failures-file', 187 default=os.path.join(base_path, 'tools', 'known_tint_failures'), 188 help='path to directory containing the known failures file') 189 parser.add_option( 190 '--test-prog-path', 191 default=None, 192 help='path to program to test (default build-dir/tint)') 193 parser.add_option('--parse-only', 194 action="store_true", 195 default=False, 196 help='only parse test cases; do not compile') 197 198 self.options, self.args = parser.parse_args() 199 200 if self.options.test_prog_path == None: 201 test_prog = os.path.abspath( 202 os.path.join(self.options.build_dir, 'tint')) 203 if not os.path.isfile(test_prog): 204 print("Cannot find test program {}".format(test_prog)) 205 return 1 206 207 self.options.test_prog_path = test_prog 208 209 if not os.path.isfile(self.options.test_prog_path): 210 print("Cannot find test program '{}'".format( 211 self.options.test_prog_path)) 212 return 1 213 214 input_file_re = re.compile('^.+[\.]wgsl') 215 self.test_cases = [] 216 217 if self.args: 218 for filename in self.args: 219 input_path = os.path.join(self.options.test_dir, filename) 220 if not os.path.isfile(input_path): 221 print("Cannot find test file '{}'".format(filename)) 222 return 1 223 224 self.test_cases.append( 225 TestCase(input_path, self.options.parse_only)) 226 227 else: 228 for file_dir, _, filename_list in os.walk(self.options.test_dir): 229 for input_filename in filename_list: 230 if input_file_re.match(input_filename): 231 input_path = os.path.join(file_dir, input_filename) 232 if os.path.isfile(input_path): 233 self.test_cases.append( 234 TestCase(input_path, self.options.parse_only)) 235 known_failure_file = self.options.known_failures_file 236 self.known_failures = [] 237 with open(known_failure_file, 'r') as f: 238 for failure_filename in f.read().splitlines(): 239 self.known_failures.append( 240 os.path.join(self.options.test_dir, failure_filename)) 241 242 self.failures = [] 243 self.failures_wrong_reason = [] 244 self.unexpected_failures = [] 245 self.unexpected_successes = [] 246 247 self.RunTests() 248 self.GetUnexpectedFailures() 249 self.SummarizeResults() 250 251 return not len(self.unexpected_failures + self.unexpected_successes) 252 253 254def main(): 255 runner = TestRunner() 256 return runner.Run() 257 258 259if __name__ == '__main__': 260 sys.exit(main()) 261