1# Copyright (c) 2018 Google LLC 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); 4# you may not use this file except in compliance with the License. 5# You may obtain a copy of the License at 6# 7# http://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12# See the License for the specific language governing permissions and 13# limitations under the License. 14"""Manages and runs tests from the current working directory. 15 16This will traverse the current working directory and look for python files that 17contain subclasses of SpirvTest. 18 19If a class has an @inside_spirv_testsuite decorator, an instance of that 20class will be created and serve as a test case in that testsuite. The test 21case is then run by the following steps: 22 23 1. A temporary directory will be created. 24 2. The spirv_args member variable will be inspected and all placeholders in it 25 will be expanded by calling instantiate_for_spirv_args() on placeholders. 26 The transformed list elements are then supplied as arguments to the spirv-* 27 tool under test. 28 3. If the environment member variable exists, its write() method will be 29 invoked. 30 4. All expected_* member variables will be inspected and all placeholders in 31 them will be expanded by calling instantiate_for_expectation() on those 32 placeholders. After placeholder expansion, if the expected_* variable is 33 a list, its element will be joined together with '' to form a single 34 string. These expected_* variables are to be used by the check_*() methods. 35 5. The spirv-* tool will be run with the arguments supplied in spirv_args. 36 6. All check_*() member methods will be called by supplying a TestStatus as 37 argument. Each check_*() method is expected to return a (Success, Message) 38 pair where Success is a boolean indicating success and Message is an error 39 message. 40 7. If any check_*() method fails, the error message is output and the 41 current test case fails. 42 43If --leave-output was not specified, all temporary files and directories will 44be deleted. 45""" 46 47from __future__ import print_function 48 49import argparse 50import fnmatch 51import inspect 52import os 53import shutil 54import subprocess 55import sys 56import tempfile 57from collections import defaultdict 58from placeholder import PlaceHolder 59 60EXPECTED_BEHAVIOR_PREFIX = 'expected_' 61VALIDATE_METHOD_PREFIX = 'check_' 62 63 64def get_all_variables(instance): 65 """Returns the names of all the variables in instance.""" 66 return [v for v in dir(instance) if not callable(getattr(instance, v))] 67 68 69def get_all_methods(instance): 70 """Returns the names of all methods in instance.""" 71 return [m for m in dir(instance) if callable(getattr(instance, m))] 72 73 74def get_all_superclasses(cls): 75 """Returns all superclasses of a given class. 76 77 Returns: 78 A list of superclasses of the given class. The order guarantees that 79 * A Base class precedes its derived classes, e.g., for "class B(A)", it 80 will be [..., A, B, ...]. 81 * When there are multiple base classes, base classes declared first 82 precede those declared later, e.g., for "class C(A, B), it will be 83 [..., A, B, C, ...] 84 """ 85 classes = [] 86 for superclass in cls.__bases__: 87 for c in get_all_superclasses(superclass): 88 if c not in classes: 89 classes.append(c) 90 for superclass in cls.__bases__: 91 if superclass not in classes: 92 classes.append(superclass) 93 return classes 94 95 96def get_all_test_methods(test_class): 97 """Gets all validation methods. 98 99 Returns: 100 A list of validation methods. The order guarantees that 101 * A method defined in superclass precedes one defined in subclass, 102 e.g., for "class A(B)", methods defined in B precedes those defined 103 in A. 104 * If a subclass has more than one superclass, e.g., "class C(A, B)", 105 then methods defined in A precedes those defined in B. 106 """ 107 classes = get_all_superclasses(test_class) 108 classes.append(test_class) 109 all_tests = [ 110 m for c in classes for m in get_all_methods(c) 111 if m.startswith(VALIDATE_METHOD_PREFIX) 112 ] 113 unique_tests = [] 114 for t in all_tests: 115 if t not in unique_tests: 116 unique_tests.append(t) 117 return unique_tests 118 119 120class SpirvTest: 121 """Base class for spirv test cases. 122 123 Subclasses define test cases' facts (shader source code, spirv command, 124 result validation), which will be used by the TestCase class for running 125 tests. Subclasses should define spirv_args (specifying spirv_tool command 126 arguments), and at least one check_*() method (for result validation) for 127 a full-fledged test case. All check_*() methods should take a TestStatus 128 parameter and return a (Success, Message) pair, in which Success is a 129 boolean indicating success and Message is an error message. The test passes 130 iff all check_*() methods returns true. 131 132 Often, a test case class will delegate the check_* behaviors by inheriting 133 from other classes. 134 """ 135 136 def name(self): 137 return self.__class__.__name__ 138 139 140class TestStatus: 141 """A struct for holding run status of a test case.""" 142 143 def __init__(self, test_manager, returncode, stdout, stderr, directory, 144 inputs, input_filenames): 145 self.test_manager = test_manager 146 self.returncode = returncode 147 self.stdout = stdout 148 self.stderr = stderr 149 # temporary directory where the test runs 150 self.directory = directory 151 # List of inputs, as PlaceHolder objects. 152 self.inputs = inputs 153 # the names of input shader files (potentially including paths) 154 self.input_filenames = input_filenames 155 156 157class SpirvTestException(Exception): 158 """SpirvTest exception class.""" 159 pass 160 161 162def inside_spirv_testsuite(testsuite_name): 163 """Decorator for subclasses of SpirvTest. 164 165 This decorator checks that a class meets the requirements (see below) 166 for a test case class, and then puts the class in a certain testsuite. 167 * The class needs to be a subclass of SpirvTest. 168 * The class needs to have spirv_args defined as a list. 169 * The class needs to define at least one check_*() methods. 170 * All expected_* variables required by check_*() methods can only be 171 of bool, str, or list type. 172 * Python runtime will throw an exception if the expected_* member 173 attributes required by check_*() methods are missing. 174 """ 175 176 def actual_decorator(cls): 177 if not inspect.isclass(cls): 178 raise SpirvTestException('Test case should be a class') 179 if not issubclass(cls, SpirvTest): 180 raise SpirvTestException( 181 'All test cases should be subclasses of SpirvTest') 182 if 'spirv_args' not in get_all_variables(cls): 183 raise SpirvTestException('No spirv_args found in the test case') 184 if not isinstance(cls.spirv_args, list): 185 raise SpirvTestException('spirv_args needs to be a list') 186 if not any( 187 [m.startswith(VALIDATE_METHOD_PREFIX) for m in get_all_methods(cls)]): 188 raise SpirvTestException('No check_*() methods found in the test case') 189 if not all( 190 [isinstance(v, (bool, str, list)) for v in get_all_variables(cls)]): 191 raise SpirvTestException( 192 'expected_* variables are only allowed to be bool, str, or ' 193 'list type.') 194 cls.parent_testsuite = testsuite_name 195 return cls 196 197 return actual_decorator 198 199 200class TestManager: 201 """Manages and runs a set of tests.""" 202 203 def __init__(self, executable_path, assembler_path, disassembler_path): 204 self.executable_path = executable_path 205 self.assembler_path = assembler_path 206 self.disassembler_path = disassembler_path 207 self.num_successes = 0 208 self.num_failures = 0 209 self.num_tests = 0 210 self.leave_output = False 211 self.tests = defaultdict(list) 212 213 def notify_result(self, test_case, success, message): 214 """Call this to notify the manager of the results of a test run.""" 215 self.num_successes += 1 if success else 0 216 self.num_failures += 0 if success else 1 217 counter_string = str(self.num_successes + self.num_failures) + '/' + str( 218 self.num_tests) 219 print('%-10s %-40s ' % (counter_string, test_case.test.name()) + 220 ('Passed' if success else '-Failed-')) 221 if not success: 222 print(' '.join(test_case.command)) 223 print(message) 224 225 def add_test(self, testsuite, test): 226 """Add this to the current list of test cases.""" 227 self.tests[testsuite].append(TestCase(test, self)) 228 self.num_tests += 1 229 230 def run_tests(self): 231 for suite in self.tests: 232 print('SPIRV tool test suite: "{suite}"'.format(suite=suite)) 233 for x in self.tests[suite]: 234 x.runTest() 235 236 237class TestCase: 238 """A single test case that runs in its own directory.""" 239 240 def __init__(self, test, test_manager): 241 self.test = test 242 self.test_manager = test_manager 243 self.inputs = [] # inputs, as PlaceHolder objects. 244 self.file_shaders = [] # filenames of shader files. 245 self.stdin_shader = None # text to be passed to spirv_tool as stdin 246 247 def setUp(self): 248 """Creates environment and instantiates placeholders for the test case.""" 249 250 self.directory = tempfile.mkdtemp(dir=os.getcwd()) 251 spirv_args = self.test.spirv_args 252 # Instantiate placeholders in spirv_args 253 self.test.spirv_args = [ 254 arg.instantiate_for_spirv_args(self) 255 if isinstance(arg, PlaceHolder) else arg for arg in self.test.spirv_args 256 ] 257 # Get all shader files' names 258 self.inputs = [arg for arg in spirv_args if isinstance(arg, PlaceHolder)] 259 self.file_shaders = [arg.filename for arg in self.inputs] 260 261 if 'environment' in get_all_variables(self.test): 262 self.test.environment.write(self.directory) 263 264 expectations = [ 265 v for v in get_all_variables(self.test) 266 if v.startswith(EXPECTED_BEHAVIOR_PREFIX) 267 ] 268 # Instantiate placeholders in expectations 269 for expectation_name in expectations: 270 expectation = getattr(self.test, expectation_name) 271 if isinstance(expectation, list): 272 expanded_expections = [ 273 element.instantiate_for_expectation(self) 274 if isinstance(element, PlaceHolder) else element 275 for element in expectation 276 ] 277 setattr(self.test, expectation_name, expanded_expections) 278 elif isinstance(expectation, PlaceHolder): 279 setattr(self.test, expectation_name, 280 expectation.instantiate_for_expectation(self)) 281 282 def tearDown(self): 283 """Removes the directory if we were not instructed to do otherwise.""" 284 if not self.test_manager.leave_output: 285 shutil.rmtree(self.directory) 286 287 def runTest(self): 288 """Sets up and runs a test, reports any failures and then cleans up.""" 289 self.setUp() 290 success = False 291 message = '' 292 try: 293 self.command = [self.test_manager.executable_path] 294 self.command.extend(self.test.spirv_args) 295 296 process = subprocess.Popen( 297 args=self.command, 298 stdin=subprocess.PIPE, 299 stdout=subprocess.PIPE, 300 stderr=subprocess.PIPE, 301 cwd=self.directory) 302 output = process.communicate(self.stdin_shader) 303 test_status = TestStatus(self.test_manager, process.returncode, output[0], 304 output[1], self.directory, self.inputs, 305 self.file_shaders) 306 run_results = [ 307 getattr(self.test, test_method)(test_status) 308 for test_method in get_all_test_methods(self.test.__class__) 309 ] 310 success, message = zip(*run_results) 311 success = all(success) 312 message = '\n'.join(message) 313 except Exception as e: 314 success = False 315 message = str(e) 316 self.test_manager.notify_result( 317 self, success, 318 message + '\nSTDOUT:\n%s\nSTDERR:\n%s' % (output[0], output[1])) 319 self.tearDown() 320 321 322def main(): 323 parser = argparse.ArgumentParser() 324 parser.add_argument( 325 'spirv_tool', 326 metavar='path/to/spirv_tool', 327 type=str, 328 nargs=1, 329 help='Path to the spirv-* tool under test') 330 parser.add_argument( 331 'spirv_as', 332 metavar='path/to/spirv-as', 333 type=str, 334 nargs=1, 335 help='Path to spirv-as') 336 parser.add_argument( 337 'spirv_dis', 338 metavar='path/to/spirv-dis', 339 type=str, 340 nargs=1, 341 help='Path to spirv-dis') 342 parser.add_argument( 343 '--leave-output', 344 action='store_const', 345 const=1, 346 help='Do not clean up temporary directories') 347 parser.add_argument( 348 '--test-dir', nargs=1, help='Directory to gather the tests from') 349 args = parser.parse_args() 350 default_path = sys.path 351 root_dir = os.getcwd() 352 if args.test_dir: 353 root_dir = args.test_dir[0] 354 manager = TestManager(args.spirv_tool[0], args.spirv_as[0], args.spirv_dis[0]) 355 if args.leave_output: 356 manager.leave_output = True 357 for root, _, filenames in os.walk(root_dir): 358 for filename in fnmatch.filter(filenames, '*.py'): 359 if filename.endswith('nosetest.py'): 360 # Skip nose tests, which are for testing functions of 361 # the test framework. 362 continue 363 sys.path = default_path 364 sys.path.append(root) 365 mod = __import__(os.path.splitext(filename)[0]) 366 for _, obj, in inspect.getmembers(mod): 367 if inspect.isclass(obj) and hasattr(obj, 'parent_testsuite'): 368 manager.add_test(obj.parent_testsuite, obj()) 369 manager.run_tests() 370 if manager.num_failures > 0: 371 sys.exit(-1) 372 373 374if __name__ == '__main__': 375 main() 376