1#!/usr/bin/python3 2# 3# Copyright 2017 The ANGLE Project Authors. All rights reserved. 4# Use of this source code is governed by a BSD-style license that can be 5# found in the LICENSE file. 6# 7# run_code_generation.py: 8# Runs ANGLE format table and other script code generation scripts. 9 10import argparse 11import hashlib 12import json 13import os 14import subprocess 15import sys 16import platform 17 18script_dir = sys.path[0] 19root_dir = os.path.abspath(os.path.join(script_dir, '..')) 20 21hash_dir = 'code_generation_hashes' 22 23# auto_script is a standard way for scripts to return their inputs and outputs. 24 25 26def get_child_script_dirname(script): 27 # All script names are relative to ANGLE's root 28 return os.path.dirname(os.path.abspath(os.path.join(root_dir, script))) 29 30 31# Replace all backslashes with forward slashes to be platform independent 32def clean_path_slashes(path): 33 return path.replace("\\", "/") 34 35 36# Takes a script file name which is relative to the code generation script's directory and 37# changes it to be relative to the angle root directory 38def rebase_script_path(script_path, relative_path): 39 return os.path.relpath(os.path.join(os.path.dirname(script_path), relative_path), root_dir) 40 41 42# Check if we need a module from vpython 43def get_executable_name(first_line): 44 binary = os.path.basename(first_line.strip().replace(' ', '/')) 45 if platform.system() == 'Windows': 46 if binary == 'python2': 47 return 'python.bat' 48 else: 49 return binary + '.bat' 50 else: 51 return binary 52 53 54def grab_from_script(script, param): 55 res = '' 56 f = open(os.path.basename(script), 'r') 57 exe = get_executable_name(f.readline()) 58 try: 59 res = subprocess.check_output([exe, script, param]).decode().strip() 60 except Exception: 61 print('Error grabbing script output: %s, executable %s' % (script, exe)) 62 raise 63 f.close() 64 if res == '': 65 return [] 66 return [clean_path_slashes(rebase_script_path(script, name)) for name in res.split(',')] 67 68 69def auto_script(script): 70 # Set the CWD to the script directory. 71 os.chdir(get_child_script_dirname(script)) 72 base_script = os.path.basename(script) 73 info = { 74 'inputs': grab_from_script(base_script, 'inputs'), 75 'outputs': grab_from_script(base_script, 'outputs') 76 } 77 # Reset the CWD to the root ANGLE directory. 78 os.chdir(root_dir) 79 return info 80 81 82generators = { 83 'ANGLE features': 84 'include/platform/gen_features.py', 85 'ANGLE format': 86 'src/libANGLE/renderer/gen_angle_format_table.py', 87 'ANGLE load functions table': 88 'src/libANGLE/renderer/gen_load_functions_table.py', 89 'ANGLE shader preprocessor': 90 'src/compiler/preprocessor/generate_parser.py', 91 'ANGLE shader translator': 92 'src/compiler/translator/generate_parser.py', 93 'D3D11 blit shader selection': 94 'src/libANGLE/renderer/d3d/d3d11/gen_blit11helper.py', 95 'D3D11 format': 96 'src/libANGLE/renderer/d3d/d3d11/gen_texture_format_table.py', 97 'DXGI format': 98 'src/libANGLE/renderer/gen_dxgi_format_table.py', 99 'DXGI format support': 100 'src/libANGLE/renderer/gen_dxgi_support_tables.py', 101 'Emulated HLSL functions': 102 'src/compiler/translator/gen_emulated_builtin_function_tables.py', 103 'Extension files': 104 'src/libANGLE/gen_extensions.py', 105 'GL copy conversion table': 106 'src/libANGLE/gen_copy_conversion_table.py', 107 'GL CTS (dEQP) build files': 108 'scripts/gen_vk_gl_cts_build.py', 109 'GL/EGL/WGL loader': 110 'scripts/generate_loader.py', 111 'GL/EGL entry points': 112 'scripts/generate_entry_points.py', 113 'GLenum value to string map': 114 'scripts/gen_gl_enum_utils.py', 115 'GL format map': 116 'src/libANGLE/gen_format_map.py', 117 'Metal format table': 118 'src/libANGLE/renderer/metal/gen_mtl_format_table.py', 119 'Metal default shaders': 120 'src/libANGLE/renderer/metal/shaders/gen_mtl_internal_shaders.py', 121 'OpenGL dispatch table': 122 'src/libANGLE/renderer/gl/generate_gl_dispatch_table.py', 123 'overlay fonts': 124 'src/libANGLE/gen_overlay_fonts.py', 125 'overlay widgets': 126 'src/libANGLE/gen_overlay_widgets.py', 127 'packed enum': 128 'src/common/gen_packed_gl_enums.py', 129 'proc table': 130 'scripts/gen_proc_table.py', 131 'restricted traces': 132 'src/tests/restricted_traces/gen_restricted_traces.py', 133 'SPIR-V helpers': 134 'src/common/spirv/gen_spirv_builder_and_parser.py', 135 'Static builtins': 136 'src/compiler/translator/gen_builtin_symbols.py', 137 'Test spec JSON': 138 'infra/specs/generate_test_spec_json.py', 139 'uniform type': 140 'src/common/gen_uniform_type_table.py', 141 'Vulkan format': 142 'src/libANGLE/renderer/vulkan/gen_vk_format_table.py', 143 'Vulkan internal shader programs': 144 'src/libANGLE/renderer/vulkan/gen_vk_internal_shaders.py', 145 'Vulkan mandatory format support table': 146 'src/libANGLE/renderer/vulkan/gen_vk_mandatory_format_support_table.py', 147} 148 149 150def md5(fname): 151 hash_md5 = hashlib.md5() 152 with open(fname, "r") as f: 153 for chunk in iter(lambda: f.read(4096), ""): 154 hash_md5.update(chunk.encode()) 155 return hash_md5.hexdigest() 156 157 158def get_hash_file_name(name): 159 return name.replace(' ', '_').replace('/', '_') + '.json' 160 161 162def any_hash_dirty(name, filenames, new_hashes, old_hashes): 163 found_dirty_hash = False 164 165 for fname in filenames: 166 if not os.path.isfile(fname): 167 print('File not found: "%s". Code gen dirty for %s' % (fname, name)) 168 found_dirty_hash = True 169 else: 170 new_hashes[fname] = md5(fname) 171 if (not fname in old_hashes) or (old_hashes[fname] != new_hashes[fname]): 172 print('Hash for "%s" dirty for %s generator.' % (fname, name)) 173 found_dirty_hash = True 174 return found_dirty_hash 175 176 177def any_old_hash_missing(all_new_hashes, all_old_hashes): 178 result = False 179 for file, old_hashes in all_old_hashes.items(): 180 if file not in all_new_hashes: 181 print('"%s" does not exist. Code gen dirty.' % file) 182 result = True 183 else: 184 for name, _ in old_hashes.items(): 185 if name not in all_new_hashes[file]: 186 print('Hash for %s is missing from "%s". Code gen is dirty.' % (name, file)) 187 result = True 188 return result 189 190 191def update_output_hashes(script, outputs, new_hashes): 192 for output in outputs: 193 if not os.path.isfile(output): 194 print('Output is missing from %s: %s' % (script, output)) 195 sys.exit(1) 196 new_hashes[output] = md5(output) 197 198 199def load_hashes(): 200 hashes = {} 201 for file in os.listdir(hash_dir): 202 hash_fname = os.path.join(hash_dir, file) 203 with open(hash_fname) as hash_file: 204 try: 205 hashes[file] = json.load(hash_file) 206 except ValueError: 207 raise Exception("Could not decode JSON from %s" % file) 208 return hashes 209 210 211def main(): 212 os.chdir(script_dir) 213 214 all_old_hashes = load_hashes() 215 all_new_hashes = {} 216 any_dirty = False 217 218 parser = argparse.ArgumentParser(description='Generate ANGLE internal code.') 219 parser.add_argument( 220 '-v', 221 '--verify-no-dirty', 222 dest='verify_only', 223 action='store_true', 224 help='verify hashes are not dirty') 225 parser.add_argument( 226 '-g', '--generator', action='append', nargs='*', type=str, dest='specified_generators'), 227 228 args = parser.parse_args() 229 230 ranGenerators = generators 231 runningSingleGenerator = False 232 if (args.specified_generators): 233 ranGenerators = {k: v for k, v in generators.items() if k in args.specified_generators[0]} 234 runningSingleGenerator = True 235 236 if len(ranGenerators) == 0: 237 print("No valid generators specified.") 238 return 1 239 240 for name, script in sorted(ranGenerators.items()): 241 info = auto_script(script) 242 fname = get_hash_file_name(name) 243 filenames = info['inputs'] + info['outputs'] + [script] 244 new_hashes = {} 245 if fname not in all_old_hashes: 246 all_old_hashes[fname] = {} 247 if any_hash_dirty(name, filenames, new_hashes, all_old_hashes[fname]): 248 any_dirty = True 249 250 if not args.verify_only: 251 print('Running ' + name + ' code generator') 252 253 # Set the CWD to the script directory. 254 os.chdir(get_child_script_dirname(script)) 255 256 f = open(os.path.basename(script), "r") 257 if subprocess.call([get_executable_name(f.readline()), 258 os.path.basename(script)]) != 0: 259 sys.exit(1) 260 f.close() 261 262 # Update the hash dictionary. 263 all_new_hashes[fname] = new_hashes 264 265 if not runningSingleGenerator and any_old_hash_missing(all_new_hashes, all_old_hashes): 266 any_dirty = True 267 268 if args.verify_only: 269 sys.exit(any_dirty) 270 271 if any_dirty: 272 args = ['git.bat'] if os.name == 'nt' else ['git'] 273 args += ['cl', 'format'] 274 print('Calling git cl format') 275 if subprocess.call(args) != 0: 276 sys.exit(1) 277 278 # Update the output hashes again since they can be formatted. 279 for name, script in sorted(ranGenerators.items()): 280 info = auto_script(script) 281 fname = get_hash_file_name(name) 282 update_output_hashes(name, info['outputs'], all_new_hashes[fname]) 283 284 os.chdir(script_dir) 285 286 for fname, new_hashes in all_new_hashes.items(): 287 hash_fname = os.path.join(hash_dir, fname) 288 json.dump( 289 new_hashes, 290 open(hash_fname, "w"), 291 indent=2, 292 sort_keys=True, 293 separators=(',', ':\n ')) 294 295 296if __name__ == '__main__': 297 sys.exit(main()) 298