1#! /usr/bin/env python3 2# Taken from Crucible and modified to parse declarations 3 4import argparse 5import io 6import os 7import re 8import shutil 9import struct 10import subprocess 11import sys 12import tempfile 13from textwrap import dedent 14 15class ShaderCompileError(RuntimeError): 16 def __init__(self, *args): 17 super(ShaderCompileError, self).__init__(*args) 18 19target_env_re = re.compile(r'QO_TARGET_ENV\s+(\S+)') 20 21stage_to_glslang_stage = { 22 'VERTEX': 'vert', 23 'TESS_CONTROL': 'tesc', 24 'TESS_EVALUATION': 'tese', 25 'GEOMETRY': 'geom', 26 'FRAGMENT': 'frag', 27 'COMPUTE': 'comp', 28} 29 30base_layout_qualifier_id_re = r'({0}\s*=\s*(?P<{0}>\d+))' 31id_re = '(?P<name_%d>[^(gl_)]\S+)' 32type_re = '(?P<dtype_%d>\S+)' 33location_re = base_layout_qualifier_id_re.format('location') 34component_re = base_layout_qualifier_id_re.format('component') 35binding_re = base_layout_qualifier_id_re.format('binding') 36set_re = base_layout_qualifier_id_re.format('set') 37unk_re = r'\S+(=\d+)?' 38layout_qualifier_re = r'layout\W*\((%s)+\)' % '|'.join([location_re, binding_re, set_re, unk_re, '[, ]+']) 39ubo_decl_re = 'uniform\W+%s(\W*{)?(?P<type_ubo>)' % (id_re%0) 40ssbo_decl_re = 'buffer\W+%s(\W*{)?(?P<type_ssbo>)' % (id_re%1) 41image_buffer_decl_re = r'uniform\W+imageBuffer\w+%s;(?P<type_img_buf>)' % (id_re%2) 42image_decl_re = r'uniform\W+image\S+\W+%s;(?P<type_img>)' % (id_re%3) 43texture_buffer_decl_re = r'uniform\W+textureBuffer\w+%s;(?P<type_tex_buf>)' % (id_re%4) 44combined_texture_sampler_decl_re = r'uniform\W+sampler\S+\W+%s;(?P<type_combined>)' % (id_re%5) 45texture_decl_re = r'uniform\W+texture\S+\W+%s;(?P<type_tex>)' % (id_re%6) 46sampler_decl_re = r'uniform\W+sampler\w+%s;(?P<type_samp>)' % (id_re%7) 47input_re = r'in\W+%s\W+%s;(?P<type_in>)' % (type_re%0, id_re%8) 48output_re = r'out\W+%s\W+%s;(?P<type_out>)' % (type_re%1, id_re%9) 49match_decl_re = re.compile(layout_qualifier_re + r'\W*((' + r')|('.join([ubo_decl_re, ssbo_decl_re, image_buffer_decl_re, image_decl_re, texture_buffer_decl_re, combined_texture_sampler_decl_re, texture_decl_re, sampler_decl_re, input_re, output_re]) + r'))$') 50 51class Shader: 52 def __init__(self, stage): 53 self.glsl = None 54 self.stream = io.StringIO() 55 self.stage = stage 56 self.dwords = None 57 self.target_env = "" 58 self.declarations = [] 59 60 def add_text(self, s): 61 self.stream.write(s) 62 63 def finish_text(self, start_line, end_line): 64 self.glsl = self.stream.getvalue() 65 self.stream = None 66 67 # Handle the QO_EXTENSION macro 68 self.glsl = self.glsl.replace('QO_EXTENSION', '#extension') 69 70 # Handle the QO_DEFINE macro 71 self.glsl = self.glsl.replace('QO_DEFINE', '#define') 72 73 m = target_env_re.search(self.glsl) 74 if m: 75 self.target_env = m.group(1) 76 self.glsl = self.glsl.replace('QO_TARGET_ENV', '// --target-env') 77 78 self.start_line = start_line 79 self.end_line = end_line 80 81 def __run_glslang(self, extra_args=[]): 82 stage = stage_to_glslang_stage[self.stage] 83 stage_flags = ['-S', stage] 84 85 in_file = tempfile.NamedTemporaryFile(suffix='.'+stage) 86 src = ('#version 450\n' + self.glsl).encode('utf-8') 87 in_file.write(src) 88 in_file.flush() 89 out_file = tempfile.NamedTemporaryFile(suffix='.spirv') 90 args = [glslang, '-H'] + extra_args + stage_flags 91 if self.target_env: 92 args += ['--target-env', self.target_env] 93 args += ['-o', out_file.name, in_file.name] 94 with subprocess.Popen(args, 95 stdout = subprocess.PIPE, 96 stderr = subprocess.PIPE, 97 stdin = subprocess.PIPE) as proc: 98 99 out, err = proc.communicate(timeout=30) 100 in_file.close() 101 102 if proc.returncode != 0: 103 # Unfortunately, glslang dumps errors to standard out. 104 # However, since we don't really want to count on that, 105 # we'll grab the output of both 106 message = out.decode('utf-8') + '\n' + err.decode('utf-8') 107 raise ShaderCompileError(message.strip()) 108 109 out_file.seek(0) 110 spirv = out_file.read() 111 out_file.close() 112 return (spirv, out) 113 114 def _parse_declarations(self): 115 for line in self.glsl.splitlines(): 116 res = re.match(match_decl_re, line.lstrip().rstrip()) 117 if res == None: 118 continue 119 res = {k:v for k, v in res.groupdict().items() if v != None} 120 name = [v for k, v in res.items() if k.startswith('name_')][0] 121 data_type = ([v for k, v in res.items() if k.startswith('dtype_')] + [''])[0] 122 decl_type = [k for k, v in res.items() if k.startswith('type_')][0][5:] 123 location = int(res.get('location', 0)) 124 component = int(res.get('component', 0)) 125 binding = int(res.get('binding', 0)) 126 desc_set = int(res.get('set', 0)) 127 self.declarations.append('{"%s", "%s", QoShaderDeclType_%s, %d, %d, %d, %d}' % 128 (name, data_type, decl_type, location, component, binding, desc_set)) 129 130 def compile(self): 131 def dwords(f): 132 while True: 133 dword_str = f.read(4) 134 if not dword_str: 135 return 136 assert len(dword_str) == 4 137 yield struct.unpack('I', dword_str)[0] 138 139 (spirv, assembly) = self.__run_glslang() 140 self.dwords = list(dwords(io.BytesIO(spirv))) 141 self.assembly = str(assembly, 'utf-8') 142 143 self._parse_declarations() 144 145 def _dump_glsl_code(self, f): 146 # Dump GLSL code for reference. Use // instead of /* */ 147 # comments so we don't need to escape the GLSL code. 148 f.write('// GLSL code:\n') 149 f.write('//') 150 for line in self.glsl.splitlines(): 151 f.write('\n// {0}'.format(line)) 152 f.write('\n\n') 153 154 def _dump_spirv_code(self, f, var_name): 155 f.write('/* SPIR-V Assembly:\n') 156 f.write(' *\n') 157 for line in self.assembly.splitlines(): 158 f.write(' * ' + line + '\n') 159 f.write(' */\n') 160 161 f.write('static const uint32_t {0}[] = {{'.format(var_name)) 162 line_start = 0 163 while line_start < len(self.dwords): 164 f.write('\n ') 165 for i in range(line_start, min(line_start + 6, len(self.dwords))): 166 f.write(' 0x{:08x},'.format(self.dwords[i])) 167 line_start += 6 168 f.write('\n};\n') 169 170 def dump_c_code(self, f): 171 f.write('\n\n') 172 var_prefix = '__qonos_shader{0}'.format(self.end_line) 173 174 self._dump_glsl_code(f) 175 self._dump_spirv_code(f, var_prefix + '_spir_v_src') 176 f.write('static const QoShaderDecl {0}_decls[] = {{{1}}};\n'.format(var_prefix, ', '.join(self.declarations))) 177 178 f.write(dedent("""\ 179 static const QoShaderModuleCreateInfo {0}_info = {{ 180 .spirvSize = sizeof({0}_spir_v_src), 181 .pSpirv = {0}_spir_v_src, 182 .declarationCount = sizeof({0}_decls) / sizeof({0}_decls[0]), 183 .pDeclarations = {0}_decls, 184 """.format(var_prefix))) 185 186 f.write(" .stage = VK_SHADER_STAGE_{0}_BIT,\n".format(self.stage)) 187 188 f.write('};\n') 189 190 f.write('#define __qonos_shader{0}_info __qonos_shader{1}_info\n'\ 191 .format(self.start_line, self.end_line)) 192 193token_exp = re.compile(r'(qoShaderModuleCreateInfoGLSL|qoCreateShaderModuleGLSL|\(|\)|,)') 194 195class Parser: 196 def __init__(self, f): 197 self.infile = f 198 self.paren_depth = 0 199 self.shader = None 200 self.line_number = 1 201 self.shaders = [] 202 203 def tokenize(f): 204 leftover = '' 205 for line in f: 206 pos = 0 207 while True: 208 m = token_exp.search(line, pos) 209 if m: 210 if m.start() > pos: 211 leftover += line[pos:m.start()] 212 pos = m.end() 213 214 if leftover: 215 yield leftover 216 leftover = '' 217 218 yield m.group(0) 219 220 else: 221 leftover += line[pos:] 222 break 223 224 self.line_number += 1 225 226 if leftover: 227 yield leftover 228 229 self.token_iter = tokenize(self.infile) 230 231 def handle_shader_src(self): 232 paren_depth = 1 233 for t in self.token_iter: 234 if t == '(': 235 paren_depth += 1 236 elif t == ')': 237 paren_depth -= 1 238 if paren_depth == 0: 239 return 240 241 self.current_shader.add_text(t) 242 243 def handle_macro(self, macro): 244 t = next(self.token_iter) 245 assert t == '(' 246 247 start_line = self.line_number 248 249 if macro == 'qoCreateShaderModuleGLSL': 250 # Throw away the device parameter 251 t = next(self.token_iter) 252 t = next(self.token_iter) 253 assert t == ',' 254 255 stage = next(self.token_iter).strip() 256 257 t = next(self.token_iter) 258 assert t == ',' 259 260 self.current_shader = Shader(stage) 261 self.handle_shader_src() 262 self.current_shader.finish_text(start_line, self.line_number) 263 264 self.shaders.append(self.current_shader) 265 self.current_shader = None 266 267 def run(self): 268 for t in self.token_iter: 269 if t in ('qoShaderModuleCreateInfoGLSL', 'qoCreateShaderModuleGLSL'): 270 self.handle_macro(t) 271 272def open_file(name, mode): 273 if name == '-': 274 if mode == 'w': 275 return sys.stdout 276 elif mode == 'r': 277 return sys.stdin 278 else: 279 assert False 280 else: 281 return open(name, mode) 282 283def parse_args(): 284 description = dedent("""\ 285 This program scrapes a C file for any instance of the 286 qoShaderModuleCreateInfoGLSL and qoCreateShaderModuleGLSL macaros, 287 grabs the GLSL source code, compiles it to SPIR-V. The resulting 288 SPIR-V code is written to another C file as an array of 32-bit 289 words. 290 291 If '-' is passed as the input file or output file, stdin or stdout 292 will be used instead of a file on disc.""") 293 294 p = argparse.ArgumentParser( 295 description=description, 296 formatter_class=argparse.RawDescriptionHelpFormatter) 297 p.add_argument('-o', '--outfile', default='-', 298 help='Output to the given file (default: stdout).') 299 p.add_argument('--with-glslang', metavar='PATH', 300 default='glslangValidator', 301 dest='glslang', 302 help='Full path to the glslangValidator shader compiler.') 303 p.add_argument('infile', metavar='INFILE') 304 305 return p.parse_args() 306 307 308args = parse_args() 309infname = args.infile 310outfname = args.outfile 311glslang = args.glslang 312 313with open_file(infname, 'r') as infile: 314 parser = Parser(infile) 315 parser.run() 316 317for shader in parser.shaders: 318 shader.compile() 319 320with open_file(outfname, 'w') as outfile: 321 outfile.write(dedent("""\ 322 /* ========================== DO NOT EDIT! ========================== 323 * This file is autogenerated by glsl_scraper.py. 324 */ 325 326 #include <stdint.h> 327 328 #define __QO_SHADER_INFO_VAR2(_line) __qonos_shader ## _line ## _info 329 #define __QO_SHADER_INFO_VAR(_line) __QO_SHADER_INFO_VAR2(_line) 330 331 #define qoShaderModuleCreateInfoGLSL(stage, ...) \\ 332 __QO_SHADER_INFO_VAR(__LINE__) 333 334 #define qoCreateShaderModuleGLSL(dev, stage, ...) \\ 335 __qoCreateShaderModule((dev), &__QO_SHADER_INFO_VAR(__LINE__)) 336 """)) 337 338 for shader in parser.shaders: 339 shader.dump_c_code(outfile) 340