#!/usr/bin/python3 # Copyright 2018 The ANGLE Project Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. # # gen_vk_internal_shaders.py: # Code generation for internal Vulkan shaders. Should be run when an internal # shader program is changed, added or removed. # Because this script can be slow direct invocation is supported. But before # code upload please run scripts/run_code_generation.py. import io import json import multiprocessing import os import platform import re import subprocess import sys import gzip out_file_cpp = 'vk_internal_shaders_autogen.cpp' out_file_h = 'vk_internal_shaders_autogen.h' out_file_gni = 'vk_internal_shaders_autogen.gni' is_windows = platform.system() == 'Windows' is_linux = platform.system() == 'Linux' # Templates for the generated files: template_shader_library_cpp = u"""// GENERATED FILE - DO NOT EDIT. // Generated by {script_name} using data from {input_file_name} // // Copyright 2018 The ANGLE Project Authors. All rights reserved. // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. // // {out_file_name}: // Pre-generated shader library for the ANGLE Vulkan back-end. #include "libANGLE/renderer/vulkan/vk_internal_shaders_autogen.h" #define USE_SYSTEM_ZLIB #include "compression_utils_portable.h" namespace rx {{ namespace vk {{ namespace {{ {internal_shader_includes} // This is compressed SPIR-V binary blob and size struct CompressedShaderBlob {{ const uint8_t *code; uint32_t size; }}; {shader_tables_cpp} angle::Result GetShader(ErrorContext *context, ShaderModulePtr shaders[], const CompressedShaderBlob *compressedShaderBlobs, size_t shadersCount, uint32_t shaderFlags, ShaderModulePtr *shaderOut) {{ ASSERT(shaderFlags < shadersCount); ShaderModulePtr &shader = shaders[shaderFlags]; if (shader) {{ ASSERT(shader->valid()); *shaderOut = shader; return angle::Result::Continue; }} // Create shader lazily. Access will need to be locked for multi-threading. const CompressedShaderBlob &compressedShaderCode = compressedShaderBlobs[shaderFlags]; ASSERT(compressedShaderCode.code != nullptr); uLong uncompressedSize = zlib_internal::GetGzipUncompressedSize(compressedShaderCode.code, compressedShaderCode.size); std::vector shaderCode((uncompressedSize + 3) / 4, 0); // Note: we assume a little-endian environment throughout ANGLE. int zResult = zlib_internal::GzipUncompressHelper(reinterpret_cast(shaderCode.data()), &uncompressedSize, compressedShaderCode.code, compressedShaderCode.size); if (zResult != Z_OK) {{ ERR() << "Failure to decompressed internal shader: " << zResult << "\\n"; return angle::Result::Stop; }} ANGLE_TRY(InitShaderModule(context, &shader, shaderCode.data(), shaderCode.size() * 4)); ASSERT(shader); ASSERT(shader->valid()); *shaderOut = shader; return angle::Result::Continue; }} }} // anonymous namespace ShaderLibrary::ShaderLibrary() {{ }} ShaderLibrary::~ShaderLibrary() {{ }} void ShaderLibrary::destroy(VkDevice device) {{ }} {shader_get_functions_cpp} }} // namespace vk }} // namespace rx """ template_shader_library_h = u"""// GENERATED FILE - DO NOT EDIT. // Generated by {script_name} using data from {input_file_name} // // Copyright 2018 The ANGLE Project Authors. All rights reserved. // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. // // {out_file_name}: // Pre-generated shader library for the ANGLE Vulkan back-end. #ifndef LIBANGLE_RENDERER_VULKAN_VK_INTERNAL_SHADERS_AUTOGEN_H_ #define LIBANGLE_RENDERER_VULKAN_VK_INTERNAL_SHADERS_AUTOGEN_H_ #include "libANGLE/renderer/vulkan/vk_utils.h" namespace rx {{ namespace vk {{ namespace InternalShader {{ {shader_variation_definitions} }} // namespace InternalShader class ShaderLibrary final : angle::NonCopyable {{ public: ShaderLibrary(); ~ShaderLibrary(); void destroy(VkDevice device); {shader_get_functions_h} private: {shader_tables_h} }}; }} // namespace vk }} // namespace rx #endif // LIBANGLE_RENDERER_VULKAN_VK_INTERNAL_SHADERS_AUTOGEN_H_ """ template_shader_includes_gni = u"""# GENERATED FILE - DO NOT EDIT. # Generated by {script_name} using data from {input_file_name} # # Copyright 2018 The ANGLE Project Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. # # {out_file_name}: # List of generated shaders for inclusion in ANGLE's build process. angle_vulkan_internal_shaders = [ {shaders_list} ] """ template_spirv_blob_inc = u"""// GENERATED FILE - DO NOT EDIT. // Generated by {script_name}. // // Copyright 2018 The ANGLE Project Authors. All rights reserved. // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. // // {out_file_name}: // Pre-generated shader for the ANGLE Vulkan back-end. #pragma once constexpr uint8_t {variable_name}[] = {{ {blob} }}; // Generated from: // {preprocessed_source} """ # Gets the constant variable name for a generated shader. def get_var_name(output, prefix='k'): return prefix + output.replace(".", "_") # Gets the namespace name given to constants generated from shader_file def get_namespace_name(shader_file): return get_var_name(os.path.basename(shader_file), '') # Gets the namespace name given to constants generated from shader_file def get_variation_table_name(shader_file, prefix='k'): return get_var_name(os.path.basename(shader_file), prefix) + '_shaders' # Gets the internal ID string for a particular shader. def get_shader_id(shader): file = os.path.splitext(os.path.basename(shader))[0] return file.replace(".", "_") # Returns the name of the generated SPIR-V file for a shader. def get_output_path(name): return os.path.join('shaders', 'gen', name + ".inc") # Finds a path to GN's out directory def get_linux_glslang_exe_path(): return '../../../../tools/glslang/glslang_validator' def get_win_glslang_exe_path(): return get_linux_glslang_exe_path() + '.exe' def get_glslang_exe_path(): glslang_exe = get_win_glslang_exe_path() if is_windows else get_linux_glslang_exe_path() if not os.path.isfile(glslang_exe): raise Exception('Could not find %s' % glslang_exe) return glslang_exe # Generates the code for a shader blob array entry. def gen_shader_blob_entry(shader): var_name = get_var_name(os.path.basename(shader))[0:-4] return "{%s, %s}" % (var_name, "sizeof(%s)" % var_name) def slash(s): return s.replace('\\', '/') def gen_shader_include(shader): return '#include "libANGLE/renderer/vulkan/%s"' % slash(shader) def get_variations_path(shader): variation_file = shader + '.json' return variation_file if os.path.exists(variation_file) else None def get_shader_variations(shader): variation_file = get_variations_path(shader) if variation_file is None: # If there is no variation file, assume none. return ({}, []) with open(variation_file) as fin: variations = json.loads(fin.read()) flags = {} enums = [] for key, value in variations.items(): if key == "Description": continue elif key == "Flags": flags = value elif len(value) > 0: enums.append((key, value)) def bits(enum): return (1 << (len(enum) - 1).bit_length()) / float(len(enum)) # sort enums so the ones with the most waste ends up last, reducing the table size enums.sort(key=lambda enum: (bits(enum[1]), enum[0])) return (flags, enums) def get_variation_bits(flags, enums): flags_bits = len(flags) enum_bits = [(len(enum[1]) - 1).bit_length() for enum in enums] return (flags_bits, enum_bits) def next_enum_variation(enums, enum_indices): """Loop through indices from [0, 0, ...] to [L0-1, L1-1, ...] where Li is len(enums[i]). The list can be thought of as a number with many digits, where each digit is in [0, Li), and this function effectively implements the increment operation, with the least-significant digit being the first item.""" for i in range(len(enums)): current = enum_indices[i] # if current digit has room, increment it. if current + 1 < len(enums[i][1]): enum_indices[i] = current + 1 return True # otherwise reset it to 0 and carry to the next digit. enum_indices[i] = 0 # if this is reached, the number has overflowed and the loop is finished. return False compact_newlines_regex = re.compile(r"\n\s*\n", re.MULTILINE) def cleanup_preprocessed_shader(shader_text): return compact_newlines_regex.sub('\n\n', shader_text.strip()) def read_and_compress_spirv_blob(blob_path): with open(blob_path, 'rb') as blob_file: blob = blob_file.read() buf = io.BytesIO() with gzip.GzipFile(fileobj=buf, mode='wb', compresslevel=9, mtime=0) as f: f.write(blob) return buf.getvalue() def write_compressed_spirv_blob_as_c_array(output_path, variable_name, compressed_blob, preprocessed_source): hex_array = ['0x{:02x}'.format(byte) for byte in compressed_blob] blob = ',\n '.join(','.join(hex_array[i:i + 16]) for i in range(0, len(hex_array), 16)) text = template_spirv_blob_inc.format( script_name=os.path.basename(__file__), out_file_name=output_path.replace('\\', '/'), variable_name=variable_name, blob=blob, preprocessed_source=preprocessed_source) with open(output_path, 'wb') as incfile: incfile.write(str.encode(text)) class CompileQueue: class CompressAndAppendPreprocessorOutput: def __init__(self, shader_file, preprocessor_args, output_path, variable_name): # Asynchronously launch the preprocessor job. self.process = subprocess.Popen( preprocessor_args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) # Store the file name for output to be appended to. self.output_path = output_path self.variable_name = variable_name # Store info for error description. self.shader_file = shader_file def wait(self, queue): (out, err) = self.process.communicate() if self.process.returncode == 0: # Use unix line endings. out = out.replace('\r\n', '\n') # Use Linux-style slashes in #line directives. out = out.replace('shaders\\src\\', 'shaders/src/') # Clean up excessive empty lines. out = cleanup_preprocessed_shader(out) # Comment it out! out = '\n'.join([('// ' + line).strip() for line in out.splitlines()]) # Read the SPIR-V blob and compress it. compressed_blob = read_and_compress_spirv_blob(self.output_path) # Write the compressed blob as a C array in the output file, followed by the # preprocessor output. write_compressed_spirv_blob_as_c_array(self.output_path, self.variable_name, compressed_blob, out) out = None return (out, err, self.process.returncode, None, "Error running preprocessor on " + self.shader_file) class CompileToSPIRV: def __init__(self, shader_file, shader_basename, variation_string, output_path, compile_args, preprocessor_args, variable_name): # Asynchronously launch the compile job. self.process = subprocess.Popen( compile_args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) # Store info for launching the preprocessor. self.preprocessor_args = preprocessor_args self.output_path = output_path # Store info for job and error description. self.shader_file = shader_file self.shader_basename = shader_basename self.variation_string = variation_string self.variable_name = variable_name def wait(self, queue): (out, err) = self.process.communicate() if self.process.returncode == 0: # Insert the preprocessor job in the queue. queue.append( CompileQueue.CompressAndAppendPreprocessorOutput(self.shader_file, self.preprocessor_args, self.output_path, self.variable_name)) # If all the output says is the source file name, don't bother printing it. if out.strip() == self.shader_file: out = None description = self.output_path + ': ' + self.shader_basename + self.variation_string return (out, err, self.process.returncode, description, "Error compiling " + self.shader_file) def __init__(self): # Compile with as many CPU threads are detected. Once a shader is compiled, another job is # automatically added to the queue to append the preprocessor output to the generated file. self.queue = [] self.thread_count = multiprocessing.cpu_count() def _wait_first(self, ignore_output=False): (out, err, returncode, description, exception_description) = self.queue[0].wait(self.queue) self.queue.pop(0) if not ignore_output: if description: print(description) if out and out.strip(): print(out.strip()) if err and err.strip(): print(err) if returncode != 0: return exception_description return None # Wait for all pending tasks. If called after error is detected, ignore_output can be used to # make sure errors in later jobs are suppressed to avoid cluttering the output. This is # because the same compile error is likely present in other variations of the same shader and # outputting the same error multiple times is not useful. def _wait_all(self, ignore_output=False): exception_description = None while len(self.queue) > 0: this_job_exception = self._wait_first(ignore_output) # If encountered an error, keep it to be raised, ignoring errors from following jobs. if this_job_exception and not ignore_output: exception_description = this_job_exception ignore_output = True return exception_description def add_job(self, shader_file, shader_basename, variation_string, output_path, compile_args, preprocessor_args, variable_name): # If the queue is full, wait until there is at least one slot available. while len(self.queue) >= self.thread_count: exception = self._wait_first(False) # If encountered an exception, cleanup following jobs and raise it. if exception: self._wait_all(True) raise Exception(exception) # Add a compile job self.queue.append( CompileQueue.CompileToSPIRV(shader_file, shader_basename, variation_string, output_path, compile_args, preprocessor_args, variable_name)) def finish(self): exception = self._wait_all(False) # If encountered an exception, cleanup following jobs and raise it. if exception is not None: raise Exception(exception) # If the option is just a string, that's the name. Otherwise, it could be # [ name, arg1, ..., argN ]. In that case, name is option[0] and option[1:] are extra arguments # that need to be passed to glslang_validator for this variation. def get_variation_name(option): return option if isinstance(option, str) else option[0] def get_variation_args(option): return [] if isinstance(option, str) else option[1:] def compile_variation(glslang_path, compile_queue, shader_file, shader_basename, flags, enums, flags_active, enum_indices, flags_bits, enum_bits, output_shaders): glslang_args = [glslang_path] # generate -D defines and the output file name # # The variations are given a bit pattern to be able to OR different flags into a variation. The # least significant bits are the flags, where there is one bit per flag. After that, each enum # takes up as few bits as needed to count that many enum values. variation_bits = 0 variation_string = '' variation_extra_args = [] for f in range(len(flags)): if flags_active & (1 << f): flag = flags[f] flag_name = get_variation_name(flag) variation_extra_args += get_variation_args(flag) glslang_args.append('-D' + flag_name + '=1') variation_bits |= 1 << f variation_string += '|' + flag_name current_bit_start = flags_bits for e in range(len(enums)): enum = enums[e][1][enum_indices[e]] enum_name = get_variation_name(enum) variation_extra_args += get_variation_args(enum) glslang_args.append('-D' + enum_name + '=1') variation_bits |= enum_indices[e] << current_bit_start current_bit_start += enum_bits[e] variation_string += '|' + enum_name output_name = '%s.%08X' % (shader_basename, variation_bits) output_path = get_output_path(output_name) output_shaders.append(output_path) if glslang_path is not None: glslang_preprocessor_output_args = glslang_args + ['-E'] glslang_preprocessor_output_args.append(shader_file) # Input GLSL shader glslang_args += ['-V'] # Output mode is Vulkan glslang_args += ['-Os'] # Optimize by default. glslang_args += ['-g0'] # Strip debug info to save on binary size. glslang_args += variation_extra_args # Add other flags, or override -Os or -g0 glslang_args += ['-o', output_path] # Output file glslang_args.append(shader_file) # Input GLSL shader compile_queue.add_job(shader_file, shader_basename, variation_string, output_path, glslang_args, glslang_preprocessor_output_args, get_var_name(output_name)) class ShaderAndVariations: def __init__(self, shader_file): self.shader_file = shader_file (self.flags, self.enums) = get_shader_variations(shader_file) get_variation_bits(self.flags, self.enums) (self.flags_bits, self.enum_bits) = get_variation_bits(self.flags, self.enums) # Maximum index value has all flags set and all enums at max value. max_index = (1 << self.flags_bits) - 1 current_bit_start = self.flags_bits for (name, values), bits in zip(self.enums, self.enum_bits): max_index |= (len(values) - 1) << current_bit_start current_bit_start += bits # Minimum array size is one more than the maximum value. self.array_len = max_index + 1 def get_variation_definition(shader_and_variation): shader_file = shader_and_variation.shader_file flags = shader_and_variation.flags enums = shader_and_variation.enums flags_bits = shader_and_variation.flags_bits enum_bits = shader_and_variation.enum_bits array_len = shader_and_variation.array_len namespace_name = get_namespace_name(shader_file) definition = 'namespace %s\n{\n' % namespace_name if len(flags) > 0: definition += 'enum flags\n{\n' definition += ''.join([ 'k%s = 0x%08X,\n' % (get_variation_name(flags[f]), 1 << f) for f in range(len(flags)) ]) definition += '};\n' current_bit_start = flags_bits for e in range(len(enums)): enum = enums[e] enum_name = enum[0] definition += 'enum %s\n{\n' % enum_name definition += ''.join([ 'k%s = 0x%08X,\n' % (get_variation_name(enum[1][v]), v << current_bit_start) for v in range(len(enum[1])) ]) definition += '};\n' current_bit_start += enum_bits[e] definition += 'constexpr size_t kArrayLen = 0x%08X;\n' % array_len definition += '} // namespace %s\n' % namespace_name return definition def get_shader_table_h(shader_and_variation): shader_file = shader_and_variation.shader_file flags = shader_and_variation.flags enums = shader_and_variation.enums table_name = get_variation_table_name(shader_file, 'm') table = 'ShaderModulePtr %s[' % table_name namespace_name = "InternalShader::" + get_namespace_name(shader_file) table += '%s::kArrayLen' % namespace_name table += '];' return table def get_shader_table_cpp(shader_and_variation): shader_file = shader_and_variation.shader_file enums = shader_and_variation.enums flags_bits = shader_and_variation.flags_bits enum_bits = shader_and_variation.enum_bits array_len = shader_and_variation.array_len # Cache max and mask value of each enum to quickly know when a possible variation is invalid enum_maxes = [] enum_masks = [] current_bit_start = flags_bits for e in range(len(enums)): enum_values = enums[e][1] enum_maxes.append((len(enum_values) - 1) << current_bit_start) enum_masks.append(((1 << enum_bits[e]) - 1) << current_bit_start) current_bit_start += enum_bits[e] table_name = get_variation_table_name(shader_file) var_name = get_var_name(os.path.basename(shader_file)) table = 'constexpr CompressedShaderBlob %s[] = {\n' % table_name for variation in range(array_len): # if any variation is invalid, output an empty entry if any([(variation & enum_masks[e]) > enum_maxes[e] for e in range(len(enums))]): table += '{nullptr, 0}, // 0x%08X\n' % variation else: entry = '%s_%08X' % (var_name, variation) table += '{%s, sizeof(%s)},\n' % (entry, entry) table += '};' return table def get_get_function_h(shader_and_variation): shader_file = shader_and_variation.shader_file function_name = get_var_name(os.path.basename(shader_file), 'get') definition = 'angle::Result %s' % function_name definition += '(ErrorContext *context, uint32_t shaderFlags, ShaderModulePtr *shaderOut);' return definition def get_get_function_cpp(shader_and_variation): shader_file = shader_and_variation.shader_file enums = shader_and_variation.enums function_name = get_var_name(os.path.basename(shader_file), 'get') namespace_name = "InternalShader::" + get_namespace_name(shader_file) member_table_name = get_variation_table_name(shader_file, 'm') constant_table_name = get_variation_table_name(shader_file) definition = 'angle::Result ShaderLibrary::%s' % function_name definition += '(ErrorContext *context, uint32_t shaderFlags, ShaderModulePtr *shaderOut)\n{\n' definition += 'return GetShader(context, %s, %s, ArraySize(%s), shaderFlags, shaderOut);\n}\n' % ( member_table_name, constant_table_name, constant_table_name) return definition def shader_path(shader): return '"%s"' % slash(shader) def main(): # STEP 0: Handle inputs/outputs for run_code_generation.py's auto_script shaders_dir = os.path.join('shaders', 'src') if not os.path.isdir(shaders_dir): raise Exception("Could not find shaders directory") print_inputs = len(sys.argv) == 2 and sys.argv[1] == 'inputs' print_outputs = len(sys.argv) == 2 and sys.argv[1] == 'outputs' # If an argument X is given that's not inputs or outputs, compile shaders that match *X*. # This is useful in development to build only the shader of interest. shader_files_to_compile = os.listdir(shaders_dir) if not (print_inputs or print_outputs or len(sys.argv) < 2): shader_files_to_compile = [f for f in shader_files_to_compile if f.find(sys.argv[1]) != -1] valid_extensions = ['.vert', '.frag', '.comp'] input_shaders = sorted([ os.path.join(shaders_dir, shader) for shader in os.listdir(shaders_dir) if any([os.path.splitext(shader)[1] == ext for ext in valid_extensions]) ]) shader_headers = sorted([ os.path.join(shaders_dir, shader) for shader in os.listdir(shaders_dir) if os.path.splitext(shader)[1] == '.inc' ]) if print_inputs: glslang_binaries = [get_linux_glslang_exe_path(), get_win_glslang_exe_path()] glslang_binary_hashes = [path + '.sha1' for path in glslang_binaries] input_shaders_variations = [get_variations_path(shader) for shader in input_shaders] input_shaders_variations = [ variations for variations in input_shaders_variations if variations is not None ] print(",".join(input_shaders + shader_headers + input_shaders_variations + glslang_binary_hashes)) return 0 # STEP 1: Call glslang to generate the internal shaders into small .inc files. # Iterates over the shaders and call glslang with the right arguments. glslang_path = None if not print_outputs: glslang_path = get_glslang_exe_path() output_shaders = [] input_shaders_and_variations = [ ShaderAndVariations(shader_file) for shader_file in input_shaders ] compile_queue = CompileQueue() for shader_and_variation in input_shaders_and_variations: shader_file = shader_and_variation.shader_file flags = shader_and_variation.flags enums = shader_and_variation.enums flags_bits = shader_and_variation.flags_bits enum_bits = shader_and_variation.enum_bits # an array where each element i is in [0, len(enums[i])), # telling which enum is currently selected enum_indices = [0] * len(enums) output_name = os.path.basename(shader_file) while True: do_compile = not print_outputs and output_name in shader_files_to_compile # a number where each bit says whether a flag is active or not, # with values in [0, 2^len(flags)) for flags_active in range(1 << len(flags)): compile_variation(glslang_path if do_compile else None, compile_queue, shader_file, output_name, flags, enums, flags_active, enum_indices, flags_bits, enum_bits, output_shaders) if not next_enum_variation(enums, enum_indices): break output_shaders = sorted(output_shaders) outputs = output_shaders + [out_file_cpp, out_file_h] if print_outputs: print(','.join(outputs)) return 0 compile_queue.finish() # STEP 2: Consolidate the .inc files into an auto-generated cpp/h library. with open(out_file_cpp, 'w') as outfile: includes = "\n".join([gen_shader_include(shader) for shader in output_shaders]) shader_tables_cpp = '\n'.join( [get_shader_table_cpp(s) for s in input_shaders_and_variations]) shader_get_functions_cpp = '\n'.join( [get_get_function_cpp(s) for s in input_shaders_and_variations]) outcode = template_shader_library_cpp.format( script_name=os.path.basename(__file__), out_file_name=out_file_cpp.replace('\\', '/'), input_file_name='shaders/src/*', internal_shader_includes=includes, shader_tables_cpp=shader_tables_cpp, shader_get_functions_cpp=shader_get_functions_cpp) outfile.write(outcode) outfile.close() with open(out_file_h, 'w') as outfile: shader_variation_definitions = '\n'.join( [get_variation_definition(s) for s in input_shaders_and_variations]) shader_get_functions_h = '\n'.join( [get_get_function_h(s) for s in input_shaders_and_variations]) shader_tables_h = '\n'.join([get_shader_table_h(s) for s in input_shaders_and_variations]) outcode = template_shader_library_h.format( script_name=os.path.basename(__file__), out_file_name=out_file_h.replace('\\', '/'), input_file_name='shaders/src/*', shader_variation_definitions=shader_variation_definitions, shader_get_functions_h=shader_get_functions_h, shader_tables_h=shader_tables_h) outfile.write(outcode) outfile.close() # STEP 3: Create a gni file with the generated files. with io.open(out_file_gni, 'w', newline='\n') as outfile: outcode = template_shader_includes_gni.format( script_name=os.path.basename(__file__), out_file_name=out_file_gni.replace('\\', '/'), input_file_name='shaders/src/*', shaders_list=',\n'.join([shader_path(shader) for shader in output_shaders])) outfile.write(outcode) outfile.close() return 0 if __name__ == '__main__': sys.exit(main())