• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/env python
2# Copyright 2018 The Amber Authors.
3#
4# Licensed under the Apache License, Version 2.0 (the "License");
5# you may not use this file except in compliance with the License.
6# You may obtain a copy of the License at
7#
8#     http://www.apache.org/licenses/LICENSE-2.0
9#
10# Unless required by applicable law or agreed to in writing, software
11# distributed under the License is distributed on an "AS IS" BASIS,
12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13# See the License for the specific language governing permissions and
14# limitations under the License.
15
16import base64
17import difflib
18import optparse
19import os
20import platform
21import re
22import subprocess
23import sys
24import tempfile
25
26SUPPRESSIONS = {
27  "Darwin": [
28    # No geometry shader on MoltenVK
29    "draw_triangle_list_using_geom_shader.vkscript",
30    # No tessellation shader on MoltenVK
31    "draw_triangle_list_using_tessellation.vkscript",
32    # No std140 support for matrices in SPIRV-Cross
33    "compute_mat2x2.vkscript",
34    "compute_mat2x2float.vkscript",
35    "compute_mat2x2.amber",
36    "compute_mat3x2.vkscript",
37    "compute_mat3x2float.vkscript",
38    "compute_mat3x2.amber",
39    # Metal vertex shaders cannot simultaneously write to a buffer and return
40    # a value to the rasterizer rdar://48348476
41    # https://github.com/KhronosGroup/MoltenVK/issues/527
42    "multiple_ssbo_update_with_graphics_pipeline.vkscript",
43    "multiple_ubo_update_with_graphics_pipeline.vkscript",
44    # DXC not currently building on bot
45    "draw_triangle_list_hlsl.amber",
46    # CLSPV not built by default
47    "opencl_bind_buffer.amber",
48    "opencl_c_copy.amber",
49    "opencl_set_arg.amber"
50  ],
51  "Linux": [
52    # DXC not currently building on bot
53    "draw_triangle_list_hlsl.amber",
54    # CLSPV not built by default
55    "opencl_bind_buffer.amber",
56    "opencl_c_copy.amber",
57    "opencl_set_arg.amber"
58  ],
59  "Win": [
60    # DXC not currently building on bot
61    "draw_triangle_list_hlsl.amber",
62    # CLSPV not built by default
63    "opencl_bind_buffer.amber",
64    "opencl_c_copy.amber",
65    "opencl_set_arg.amber"
66   ]
67 }
68
69SUPPRESSIONS_DAWN = [
70  # Dawn does not support push constants
71  "graphics_push_constants.amber",
72  "graphics_push_constants.vkscript",
73  "compute_push_const_mat2x2.vkscript",
74  "compute_push_const_mat2x2float.vkscript",
75  "compute_push_const_mat2x3.vkscript",
76  "compute_push_const_mat2x3float.vkscript",
77  "compute_push_const_mat3x2.vkscript",
78  "compute_push_const_mat3x2float.vkscript",
79  "compute_push_const_mat3x3.vkscript",
80  "compute_push_const_mat3x3float.vkscript",
81  "compute_push_const_mat3x4.vkscript",
82  "compute_push_const_mat3x4float.vkscript",
83  "compute_push_const_mat4x3.vkscript",
84  "compute_push_const_mat4x3float.vkscript",
85  "compute_push_constant_and_ssbo.amber",
86  "compute_push_constant_and_ssbo.vkscript",
87  # Dawn does not support tessellation or geometry shader
88  "draw_triangle_list_using_geom_shader.vkscript",
89  "draw_triangle_list_using_tessellation.vkscript",
90  # Dawn requires a fragmentStage now and in the medium term.
91  # issue #556 (temp dawn limitation)
92  "position_to_ssbo.amber",
93  # DrawRect command is not supported in a pipeline with more than one vertex
94  # buffer attached
95  "draw_array_after_draw_rect.vkscript",
96  "draw_rect_after_draw_array.vkscript",
97  "draw_rect_and_draw_array_mixed.vkscript",
98  # Dawn DoCommands require a pipeline
99  "probe_no_compute_with_multiple_ssbo_commands.vkscript",
100  "probe_no_compute_with_ssbo.vkscript",
101  # Max number of descriptor sets is 4 in Dawn
102  "multiple_ssbo_with_sparse_descriptor_set_in_compute_pipeline.vkscript",
103  # Dawn entry point has to be "main" as a result it does not support
104  # doEntryPoint or opencl (in opencl using "main" as entry point is invalid).
105  # issue #607 (temp dawn limitation)
106  "compute_ssbo_with_entrypoint_command.vkscript",
107  "entry_point.amber",
108  "non_default_entry_point.amber",
109  "opencl_bind_buffer.amber",
110  "opencl_c_copy.amber",
111  "opencl_set_arg.amber",
112  "shader_specialization.amber",
113  # framebuffer format is not supported according to table "Mandatory format
114  # support" in Vulkan spec: VK_FORMAT_FEATURE_COLOR_ATTACHMENT_BIT = 0
115  "draw_triangle_list_in_r16g16b16a16_snorm_color_frame.vkscript",
116  "draw_triangle_list_in_r16g16b16a16_uint_color_frame.vkscript",
117  "draw_triangle_list_in_r32g32b32a32_sfloat_color_frame.vkscript",
118  "draw_triangle_list_in_r8g8b8a8_snorm_color_frame.vkscript",
119  "draw_triangle_list_in_r8g8b8a8_srgb_color_frame.vkscript",
120  # Dawn does not support vertexPipelineStoresAndAtomics
121  "multiple_ubo_update_with_graphics_pipeline.vkscript",
122  "multiple_ssbo_update_with_graphics_pipeline.vkscript",
123  # Currently not working, under investigation
124  "draw_triangle_list_with_depth.vkscript",
125]
126
127class TestCase:
128  def __init__(self, input_path, parse_only, use_dawn):
129    self.input_path = input_path
130    self.parse_only = parse_only
131    self.use_dawn = use_dawn
132
133    self.results = {}
134
135  def IsExpectedFail(self):
136    fail_re = re.compile('^.+[.]expect_fail[.][amber|vkscript]')
137    return fail_re.match(self.GetInputPath())
138
139  def IsSuppressed(self):
140    system = platform.system()
141    if system in SUPPRESSIONS.keys():
142      is_system_suppressed = os.path.basename(self.input_path) in SUPPRESSIONS[system]
143      is_dawn_suppressed = os.path.basename(self.input_path) in SUPPRESSIONS_DAWN
144      return is_system_suppressed | (self.use_dawn & is_dawn_suppressed)
145
146    return False
147
148  def IsParseOnly(self):
149    return self.parse_only
150
151  def IsUseDawn(self):
152    return self.use_dawn
153
154  def GetInputPath(self):
155    return self.input_path
156
157  def GetResult(self, fmt):
158    return self.results[fmt]
159
160
161class TestRunner:
162  def RunTest(self, tc):
163    print "Testing %s" % tc.GetInputPath()
164
165    cmd = [self.options.test_prog_path, '-q']
166    if tc.IsParseOnly():
167      cmd += ['-p']
168    if tc.IsUseDawn():
169      cmd += ['-e', 'dawn']
170    cmd += [tc.GetInputPath()]
171
172    try:
173      err = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
174      if err != "" and not tc.IsExpectedFail() and not tc.IsSuppressed():
175        sys.stdout.write(err)
176        return False
177
178    except Exception as e:
179      print e.output
180      if not tc.IsExpectedFail() and not tc.IsSuppressed():
181        print e
182      return False
183
184    return True
185
186
187  def RunTests(self):
188    for tc in self.test_cases:
189      result = self.RunTest(tc)
190
191      if tc.IsSuppressed():
192        self.suppressed.append(tc.GetInputPath())
193      else:
194        if not tc.IsExpectedFail() and not result:
195          self.failures.append(tc.GetInputPath())
196        elif tc.IsExpectedFail() and result:
197          print("Expected: " + tc.GetInputPath() + " to fail but passed.")
198          self.failures.append(tc.GetInputPath())
199
200  def SummarizeResults(self):
201    if len(self.failures) > 0:
202      self.failures.sort()
203
204      print '\nSummary of Failures:'
205      for failure in self.failures:
206        print failure
207
208    if len(self.suppressed) > 0:
209      self.suppressed.sort()
210
211      print '\nSummary of Suppressions:'
212      for suppression in self.suppressed:
213        print suppression
214
215    print
216    print 'Test cases executed: %d' % len(self.test_cases)
217    print '  Successes:  %d' % (len(self.test_cases) - len(self.suppressed) - len(self.failures))
218    print '  Failures:   %d' % len(self.failures)
219    print '  Suppressed: %d' % len(self.suppressed)
220    print
221
222
223  def Run(self):
224    base_path = os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))
225
226    usage = 'usage: %prog [options] (file)'
227    parser = optparse.OptionParser(usage=usage)
228    parser.add_option('--build-dir',
229                      default=os.path.join(base_path, 'out', 'Debug'),
230                      help='path to build directory')
231    parser.add_option('--test-dir',
232                      default=os.path.join(os.path.dirname(__file__), 'cases'),
233                      help='path to directory containing test files')
234    parser.add_option('--test-prog-path', default=None,
235                      help='path to program to test')
236    parser.add_option('--parse-only',
237                      action="store_true", default=False,
238                      help='only parse test cases; do not execute')
239    parser.add_option('--use-dawn',
240                      action="store_true", default=False,
241                      help='Use dawn as the backend; Default is Vulkan')
242
243    self.options, self.args = parser.parse_args()
244
245    if self.options.test_prog_path == None:
246      test_prog = os.path.abspath(os.path.join(self.options.build_dir, 'amber'))
247      if not os.path.isfile(test_prog):
248        print "Cannot find test program %s" % test_prog
249        return 1
250
251      self.options.test_prog_path = test_prog
252
253    if not os.path.isfile(self.options.test_prog_path):
254      print "--test-prog-path must point to an executable"
255      return 1
256
257    input_file_re = re.compile('^.+[.][amber|vkscript]')
258    self.test_cases = []
259
260    if self.args:
261      for filename in self.args:
262        input_path = os.path.join(self.options.test_dir, filename)
263        if not os.path.isfile(input_path):
264          print "Cannot find test file '%s'" % filename
265          return 1
266
267        self.test_cases.append(TestCase(input_path, self.options.parse_only, self.options.use_dawn))
268
269    else:
270      for file_dir, _, filename_list in os.walk(self.options.test_dir):
271        for input_filename in filename_list:
272          if input_file_re.match(input_filename):
273            input_path = os.path.join(file_dir, input_filename)
274            if os.path.isfile(input_path):
275              self.test_cases.append(
276                  TestCase(input_path, self.options.parse_only, self.options.use_dawn))
277
278    self.failures = []
279    self.suppressed = []
280
281    self.RunTests()
282    self.SummarizeResults()
283
284    return len(self.failures) != 0
285
286def main():
287  runner = TestRunner()
288  return runner.Run()
289
290
291if __name__ == '__main__':
292  sys.exit(main())
293