1#!/usr/bin/env python 2# Copyright 2018 The Amber Authors. 3# 4# Licensed under the Apache License, Version 2.0 (the "License"); 5# you may not use this file except in compliance with the License. 6# You may obtain a copy of the License at 7# 8# http://www.apache.org/licenses/LICENSE-2.0 9# 10# Unless required by applicable law or agreed to in writing, software 11# distributed under the License is distributed on an "AS IS" BASIS, 12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13# See the License for the specific language governing permissions and 14# limitations under the License. 15 16import base64 17import difflib 18import optparse 19import os 20import platform 21import re 22import subprocess 23import sys 24import tempfile 25 26SUPPRESSIONS = { 27 "Darwin": [ 28 # No geometry shader on MoltenVK 29 "draw_triangle_list_using_geom_shader.vkscript", 30 # No tessellation shader on MoltenVK 31 "draw_triangle_list_using_tessellation.vkscript", 32 # No std140 support for matrices in SPIRV-Cross 33 "compute_mat2x2.vkscript", 34 "compute_mat2x2float.vkscript", 35 "compute_mat2x2.amber", 36 "compute_mat3x2.vkscript", 37 "compute_mat3x2float.vkscript", 38 "compute_mat3x2.amber", 39 # Metal vertex shaders cannot simultaneously write to a buffer and return 40 # a value to the rasterizer rdar://48348476 41 # https://github.com/KhronosGroup/MoltenVK/issues/527 42 "multiple_ssbo_update_with_graphics_pipeline.vkscript", 43 "multiple_ubo_update_with_graphics_pipeline.vkscript", 44 ], 45 "Linux": [ 46 ], 47 "Win": [ 48 ] 49} 50 51SUPPRESSIONS_SWIFTSHADER = [ 52 # Incorrect rendering: github.com/google/amber/issues/727 53 "draw_array_instanced.vkscript", 54 # Exceeds device limit maxComputeWorkGroupInvocations 55 "draw_sampled_image.amber", 56 # No geometry shader support 57 "draw_triangle_list_using_geom_shader.vkscript", 58 # No tessellation shader support 59 "draw_triangle_list_using_tessellation.vkscript", 60 # Vertex buffer format not supported 61 "draw_triangle_list_in_r8g8b8a8_srgb_color_frame.vkscript", 62 "draw_triangle_list_in_r32g32b32a32_sfloat_color_frame.vkscript", 63 "draw_triangle_list_in_r16g16b16a16_uint_color_frame.vkscript", 64 # Color attachment format is not supported 65 "draw_triangle_list_in_r16g16b16a16_snorm_color_frame.vkscript", 66 "draw_triangle_list_in_r8g8b8a8_snorm_color_frame.vkscript", 67 # No supporting device for Float16Int8Features 68 "float16.amber", 69 "int8.amber", 70 # No supporting device for the required 16-bit storage features 71 "storage16.amber", 72 # Exceeded maxBoundDescriptorSets limit of physical device 73 "multiple_ssbo_with_sparse_descriptor_set_in_compute_pipeline.vkscript", 74 # shaderStorageImageWriteWithoutFormat but is not enabled on the device 75 "opencl_read_and_write_image3d_rgba32i.amber", 76 "opencl_write_image.amber", 77 "glsl_read_and_write_image3d_rgba32i.amber", 78 # shaderStorageImageMultisample feature not supported 79 "draw_storageimage_multisample.amber", 80 # Unsupported depth/stencil formats 81 "draw_rectangles_depth_test_d24s8.amber", 82 "draw_rectangles_depth_test_x8d24.amber", 83 # Tessellation not supported 84 "tessellation_isolines.amber", 85 # 8 bit indices not supported 86 "draw_indexed_uint8.amber", 87 # Intermittent failures (https://github.com/google/amber/issues/1019). 88 "draw_polygon_mode.amber", 89 # Missing opcapability abort 90 "draw_triangle_list_hlsl.amber", 91] 92 93OPENCL_CASES = [ 94 "opencl_bind_buffer.amber", 95 "opencl_c_copy.amber", 96 "opencl_generated_push_constants.amber", 97 "opencl_read_and_write_image3d_rgba32i.amber", 98 "opencl_read_image.amber", 99 "opencl_read_image_literal_sampler.amber", 100 "opencl_set_arg.amber", 101 "opencl_write_image.amber", 102 ] 103 104DXC_CASES = [ 105 "draw_triangle_list_hlsl.amber", 106 "relative_includes_hlsl.amber", 107] 108 109SUPPRESSIONS_DAWN = [ 110 # Dawn does not support push constants 111 "graphics_push_constants.amber", 112 "graphics_push_constants.vkscript", 113 "compute_push_const_mat2x2.vkscript", 114 "compute_push_const_mat2x2float.vkscript", 115 "compute_push_const_mat2x3.vkscript", 116 "compute_push_const_mat2x3float.vkscript", 117 "compute_push_const_mat3x2.vkscript", 118 "compute_push_const_mat3x2float.vkscript", 119 "compute_push_const_mat3x3.vkscript", 120 "compute_push_const_mat3x3float.vkscript", 121 "compute_push_const_mat3x4.vkscript", 122 "compute_push_const_mat3x4float.vkscript", 123 "compute_push_const_mat4x3.vkscript", 124 "compute_push_const_mat4x3float.vkscript", 125 "compute_push_constant_and_ssbo.amber", 126 "compute_push_constant_and_ssbo.vkscript", 127 # Dawn does not support tessellation or geometry shader 128 "draw_triangle_list_using_geom_shader.vkscript", 129 "draw_triangle_list_using_tessellation.vkscript", 130 # Dawn requires a fragmentStage now and in the medium term. 131 # issue #556 (temp dawn limitation) 132 "position_to_ssbo.amber", 133 # DrawRect command is not supported in a pipeline with more than one vertex 134 # buffer attached 135 "draw_array_after_draw_rect.vkscript", 136 "draw_rect_after_draw_array.vkscript", 137 "draw_rect_and_draw_array_mixed.vkscript", 138 # Dawn DoCommands require a pipeline 139 "probe_no_compute_with_multiple_ssbo_commands.vkscript", 140 "probe_no_compute_with_ssbo.vkscript", 141 # Max number of descriptor sets is 4 in Dawn 142 "multiple_ssbo_with_sparse_descriptor_set_in_compute_pipeline.vkscript", 143 # Dawn entry point has to be "main" as a result it does not support 144 # doEntryPoint or opencl (in opencl using "main" as entry point is invalid). 145 # issue #607 (temp dawn limitation) 146 "compute_ssbo_with_entrypoint_command.vkscript", 147 "entry_point.amber", 148 "non_default_entry_point.amber", 149 "opencl_bind_buffer.amber", 150 "opencl_c_copy.amber", 151 "opencl_set_arg.amber", 152 "shader_specialization.amber", 153 # framebuffer format is not supported according to table "Mandatory format 154 # support" in Vulkan spec: VK_FORMAT_FEATURE_COLOR_ATTACHMENT_BIT = 0 155 "draw_triangle_list_in_r16g16b16a16_snorm_color_frame.vkscript", 156 "draw_triangle_list_in_r16g16b16a16_uint_color_frame.vkscript", 157 "draw_triangle_list_in_r32g32b32a32_sfloat_color_frame.vkscript", 158 "draw_triangle_list_in_r8g8b8a8_snorm_color_frame.vkscript", 159 "draw_triangle_list_in_r8g8b8a8_srgb_color_frame.vkscript", 160 # Dawn does not support vertexPipelineStoresAndAtomics 161 "multiple_ubo_update_with_graphics_pipeline.vkscript", 162 "multiple_ssbo_update_with_graphics_pipeline.vkscript", 163 # Currently not working, under investigation 164 "draw_triangle_list_with_depth.vkscript", 165 # draw_grid not implemented for dawn yet 166 "draw_grid.amber", 167 "draw_grid_multiple_color_attachment.amber", 168 "draw_grid_multiple_pipeline.amber", 169 "draw_grids.amber", 170 "draw_grid.vkscript", 171 "draw_grid_with_buffer.amber", 172 "draw_grid_with_two_vertex_data_attached.expect_fail.amber", 173] 174 175class TestCase: 176 def __init__(self, input_path, parse_only, use_dawn, use_opencl, use_dxc, 177 use_swiftshader): 178 self.input_path = input_path 179 self.parse_only = parse_only 180 self.use_dawn = use_dawn 181 self.use_opencl = use_opencl 182 self.use_dxc = use_dxc 183 self.use_swiftshader = use_swiftshader 184 185 self.results = {} 186 187 def IsExpectedFail(self): 188 fail_re = re.compile('^.+[.]expect_fail[.][amber|vkscript]') 189 return fail_re.match(self.GetInputPath()) 190 191 def IsSuppressed(self): 192 system = platform.system() 193 194 base = os.path.basename(self.input_path) 195 is_dawn_suppressed = base in SUPPRESSIONS_DAWN 196 if self.use_dawn and is_dawn_suppressed: 197 return True 198 199 is_swiftshader_suppressed = base in SUPPRESSIONS_SWIFTSHADER 200 if self.use_swiftshader and is_swiftshader_suppressed: 201 return True 202 203 is_opencl_test = base in OPENCL_CASES 204 if not self.use_opencl and is_opencl_test: 205 return True 206 207 is_dxc_test = base in DXC_CASES 208 if not self.use_dxc and is_dxc_test: 209 return True 210 211 if system in SUPPRESSIONS.keys(): 212 is_system_suppressed = base in SUPPRESSIONS[system] 213 return is_system_suppressed 214 215 return False 216 217 def IsParseOnly(self): 218 return self.parse_only 219 220 def IsUseDawn(self): 221 return self.use_dawn 222 223 def GetInputPath(self): 224 return self.input_path 225 226 def GetResult(self, fmt): 227 return self.results[fmt] 228 229 230class TestRunner: 231 def RunTest(self, tc): 232 print("Testing {}".format(tc.GetInputPath())) 233 234 cmd = [self.options.test_prog_path, '-q'] 235 if tc.IsParseOnly(): 236 cmd += ['-p'] 237 if tc.IsUseDawn(): 238 cmd += ['-e', 'dawn'] 239 cmd += [tc.GetInputPath()] 240 241 try: 242 err = subprocess.check_output(cmd, stderr=subprocess.STDOUT) 243 if len(err) != 0 and not tc.IsExpectedFail() and not tc.IsSuppressed(): 244 sys.stdout.write(err.decode('utf-8')) 245 return False 246 247 except Exception as e: 248 if not tc.IsExpectedFail() and not tc.IsSuppressed(): 249 print("{}".format("".join(map(chr, bytearray(e.output))))) 250 print(e) 251 return False 252 253 return True 254 255 256 def RunTests(self): 257 for tc in self.test_cases: 258 result = self.RunTest(tc) 259 260 if tc.IsSuppressed(): 261 self.suppressed.append(tc.GetInputPath()) 262 else: 263 if not tc.IsExpectedFail() and not result: 264 self.failures.append(tc.GetInputPath()) 265 elif tc.IsExpectedFail() and result: 266 print("Expected: " + tc.GetInputPath() + " to fail but passed.") 267 self.failures.append(tc.GetInputPath()) 268 269 def SummarizeResults(self): 270 if len(self.failures) > 0: 271 self.failures.sort() 272 273 print('\nSummary of Failures:') 274 for failure in self.failures: 275 print(failure) 276 277 if len(self.suppressed) > 0: 278 self.suppressed.sort() 279 280 print('\nSummary of Suppressions:') 281 for suppression in self.suppressed: 282 print(suppression) 283 284 print('') 285 print('Test cases executed: {}'.format(len(self.test_cases))) 286 print(' Successes: {}'.format((len(self.test_cases) - len(self.suppressed) - len(self.failures)))) 287 print(' Failures: {}'.format(len(self.failures))) 288 print(' Suppressed: {}'.format(len(self.suppressed))) 289 print('') 290 291 292 def Run(self): 293 base_path = os.path.abspath(os.path.join(os.path.dirname(__file__), '..')) 294 295 usage = 'usage: %prog [options] (file)' 296 parser = optparse.OptionParser(usage=usage) 297 parser.add_option('--build-dir', 298 default=os.path.join(base_path, 'out', 'Debug'), 299 help='path to build directory') 300 parser.add_option('--test-dir', 301 default=os.path.join(os.path.dirname(__file__), 'cases'), 302 help='path to directory containing test files') 303 parser.add_option('--test-prog-path', default=None, 304 help='path to program to test') 305 parser.add_option('--parse-only', 306 action="store_true", default=False, 307 help='only parse test cases; do not execute') 308 parser.add_option('--use-dawn', 309 action="store_true", default=False, 310 help='Use dawn as the backend; Default is Vulkan') 311 parser.add_option('--use-opencl', 312 action="store_true", default=False, 313 help='Enable OpenCL tests') 314 parser.add_option('--use-dxc', 315 action="store_true", default=False, 316 help='Enable DXC tests') 317 parser.add_option('--use-swiftshader', 318 action="store_true", default=False, 319 help='Tells test runner swiftshader is the device') 320 321 self.options, self.args = parser.parse_args() 322 323 if self.options.test_prog_path == None: 324 test_prog = os.path.abspath(os.path.join(self.options.build_dir, 'amber')) 325 if not os.path.isfile(test_prog): 326 print("Cannot find test program {}".format(test_prog)) 327 return 1 328 329 self.options.test_prog_path = test_prog 330 331 if not os.path.isfile(self.options.test_prog_path): 332 print("--test-prog-path must point to an executable") 333 return 1 334 335 input_file_re = re.compile('^.+[\.](amber|vkscript)') 336 self.test_cases = [] 337 338 if self.args: 339 for filename in self.args: 340 input_path = os.path.join(self.options.test_dir, filename) 341 if not os.path.isfile(input_path): 342 print("Cannot find test file '{}'".format(filename)) 343 return 1 344 345 self.test_cases.append(TestCase(input_path, self.options.parse_only, 346 self.options.use_dawn, self.options.use_opencl, 347 self.options.use_dxc, self.options.use_swiftshader)) 348 349 else: 350 for file_dir, _, filename_list in os.walk(self.options.test_dir): 351 for input_filename in filename_list: 352 if input_file_re.match(input_filename): 353 input_path = os.path.join(file_dir, input_filename) 354 if os.path.isfile(input_path): 355 self.test_cases.append( 356 TestCase(input_path, self.options.parse_only, 357 self.options.use_dawn, self.options.use_opencl, 358 self.options.use_dxc, self.options.use_swiftshader)) 359 360 self.failures = [] 361 self.suppressed = [] 362 363 self.RunTests() 364 self.SummarizeResults() 365 366 return len(self.failures) != 0 367 368def main(): 369 runner = TestRunner() 370 return runner.Run() 371 372 373if __name__ == '__main__': 374 sys.exit(main()) 375