• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/python2
2#
3# Copyright 2017 The ANGLE Project Authors. All rights reserved.
4# Use of this source code is governed by a BSD-style license that can be
5# found in the LICENSE file.
6#
7# run_code_generation.py:
8#   Runs ANGLE format table and other script code generation scripts.
9
10import hashlib
11import json
12import os
13import subprocess
14import sys
15import platform
16
17script_dir = sys.path[0]
18root_dir = os.path.abspath(os.path.join(script_dir, '..'))
19
20hash_dir = 'code_generation_hashes'
21
22# auto_script is a standard way for scripts to return their inputs and outputs.
23
24
25def get_child_script_dirname(script):
26    # All script names are relative to ANGLE's root
27    return os.path.dirname(os.path.abspath(os.path.join(root_dir, script)))
28
29
30# Replace all backslashes with forward slashes to be platform independent
31def clean_path_slashes(path):
32    return path.replace("\\", "/")
33
34
35# Takes a script file name which is relative to the code generation script's directory and
36# changes it to be relative to the angle root directory
37def rebase_script_path(script_path, relative_path):
38    return os.path.relpath(os.path.join(os.path.dirname(script_path), relative_path), root_dir)
39
40
41# Check if we need a module from vpython
42def get_executable_name(first_line):
43    if 'vpython' in first_line:
44        return 'vpython.bat' if platform.system() == 'Windows' else 'vpython'
45    return 'python'
46
47
48def grab_from_script(script, param):
49    res = ''
50    f = open(os.path.basename(script), "r")
51    res = subprocess.check_output([get_executable_name(f.readline()), script, param]).strip()
52    f.close()
53    if res == '':
54        return []
55    return [clean_path_slashes(rebase_script_path(script, name)) for name in res.split(',')]
56
57
58def auto_script(script):
59    # Set the CWD to the script directory.
60    os.chdir(get_child_script_dirname(script))
61    base_script = os.path.basename(script)
62    info = {
63        'inputs': grab_from_script(base_script, 'inputs'),
64        'outputs': grab_from_script(base_script, 'outputs')
65    }
66    # Reset the CWD to the root ANGLE directory.
67    os.chdir(root_dir)
68    return info
69
70
71generators = {
72    'ANGLE format':
73        'src/libANGLE/renderer/gen_angle_format_table.py',
74    'ANGLE load functions table':
75        'src/libANGLE/renderer/gen_load_functions_table.py',
76    'ANGLE shader preprocessor':
77        'src/compiler/preprocessor/generate_parser.py',
78    'ANGLE shader translator':
79        'src/compiler/translator/generate_parser.py',
80    'D3D11 blit shader selection':
81        'src/libANGLE/renderer/d3d/d3d11/gen_blit11helper.py',
82    'D3D11 format':
83        'src/libANGLE/renderer/d3d/d3d11/gen_texture_format_table.py',
84    'DXGI format':
85        'src/libANGLE/renderer/d3d/d3d11/gen_dxgi_format_table.py',
86    'DXGI format support':
87        'src/libANGLE/renderer/d3d/d3d11/gen_dxgi_support_tables.py',
88    'GL copy conversion table':
89        'src/libANGLE/gen_copy_conversion_table.py',
90    'GL/EGL/WGL loader':
91        'scripts/generate_loader.py',
92    'GL/EGL entry points':
93        'scripts/generate_entry_points.py',
94    'GLenum value to string map':
95        'scripts/gen_gl_enum_utils.py',
96    'GL format map':
97        'src/libANGLE/gen_format_map.py',
98    'uniform type':
99        'src/common/gen_uniform_type_table.py',
100    'OpenGL dispatch table':
101        'src/libANGLE/renderer/gl/generate_gl_dispatch_table.py',
102    'packed enum':
103        'src/common/gen_packed_gl_enums.py',
104    'proc table':
105        'scripts/gen_proc_table.py',
106    'Vulkan format':
107        'src/libANGLE/renderer/vulkan/gen_vk_format_table.py',
108    'Vulkan mandatory format support table':
109        'src/libANGLE/renderer/vulkan/gen_vk_mandatory_format_support_table.py',
110    'Vulkan internal shader programs':
111        'src/libANGLE/renderer/vulkan/gen_vk_internal_shaders.py',
112    'overlay fonts':
113        'src/libANGLE/gen_overlay_fonts.py',
114    'overlay widgets':
115        'src/libANGLE/gen_overlay_widgets.py',
116    'Emulated HLSL functions':
117        'src/compiler/translator/gen_emulated_builtin_function_tables.py',
118    'Static builtins':
119        'src/compiler/translator/gen_builtin_symbols.py',
120    'Metal format table':
121        'src/libANGLE/renderer/metal/gen_mtl_format_table.py',
122    'Metal default shaders':
123        'src/libANGLE/renderer/metal/shaders/gen_mtl_internal_shaders.py',
124    'GL CTS (dEQP) build files':
125        'scripts/gen_vk_gl_cts_build.py',
126}
127
128
129def md5(fname):
130    hash_md5 = hashlib.md5()
131    with open(fname, "r") as f:
132        for chunk in iter(lambda: f.read(4096), b""):
133            hash_md5.update(chunk)
134    return hash_md5.hexdigest()
135
136
137def get_hash_file_name(name):
138    return name.replace(' ', '_').replace('/', '_') + '.json'
139
140
141def any_hash_dirty(name, filenames, new_hashes, old_hashes):
142    found_dirty_hash = False
143
144    for fname in filenames:
145        if not os.path.isfile(fname):
146            print('File not found: "%s". Code gen dirty for %s' % (fname, name))
147            found_dirty_hash = True
148        else:
149            new_hashes[fname] = md5(fname)
150            if (not fname in old_hashes) or (old_hashes[fname] != new_hashes[fname]):
151                print('Hash for "%s" dirty for %s generator.' % (fname, name))
152                found_dirty_hash = True
153    return found_dirty_hash
154
155
156def any_old_hash_missing(all_new_hashes, all_old_hashes):
157    result = False
158    for file, old_hashes in all_old_hashes.iteritems():
159        if file not in all_new_hashes:
160            print('"%s" does not exist. Code gen dirty.' % file)
161            result = True
162        else:
163            for name, _ in old_hashes.iteritems():
164                if name not in all_new_hashes[file]:
165                    print('Hash for %s is missing from "%s". Code gen is dirty.' % (name, file))
166                    result = True
167    return result
168
169
170def update_output_hashes(script, outputs, new_hashes):
171    for output in outputs:
172        if not os.path.isfile(output):
173            print('Output is missing from %s: %s' % (script, output))
174            sys.exit(1)
175        new_hashes[output] = md5(output)
176
177
178def load_hashes():
179    hashes = {}
180    for file in os.listdir(hash_dir):
181        hash_fname = os.path.join(hash_dir, file)
182        with open(hash_fname) as hash_file:
183            hashes[file] = json.load(open(hash_fname))
184    return hashes
185
186
187def main():
188    os.chdir(script_dir)
189
190    all_old_hashes = load_hashes()
191    all_new_hashes = {}
192    any_dirty = False
193
194    verify_only = False
195    if len(sys.argv) > 1 and sys.argv[1] == '--verify-no-dirty':
196        verify_only = True
197
198    for name, script in sorted(generators.iteritems()):
199        info = auto_script(script)
200        fname = get_hash_file_name(name)
201        filenames = info['inputs'] + info['outputs'] + [script]
202        new_hashes = {}
203        if fname not in all_old_hashes:
204            all_old_hashes[fname] = {}
205        if any_hash_dirty(name, filenames, new_hashes, all_old_hashes[fname]):
206            any_dirty = True
207
208            if not verify_only:
209                # Set the CWD to the script directory.
210                os.chdir(get_child_script_dirname(script))
211
212                print('Running ' + name + ' code generator')
213
214                f = open(os.path.basename(script), "r")
215                if subprocess.call([get_executable_name(f.readline()),
216                                    os.path.basename(script)]) != 0:
217                    sys.exit(1)
218                f.close()
219
220        # Update the hash dictionary.
221        all_new_hashes[fname] = new_hashes
222
223    if any_old_hash_missing(all_new_hashes, all_old_hashes):
224        any_dirty = True
225
226    if verify_only:
227        sys.exit(any_dirty)
228
229    if any_dirty:
230        args = ['git.bat'] if os.name == 'nt' else ['git']
231        # The diff can be so large the arguments to clang-format can break the Windows command
232        # line length limits. Work around this by calling git cl format with --full.
233        args += ['cl', 'format', '--full']
234        print('Calling git cl format')
235        if subprocess.call(args) != 0:
236            sys.exit(1)
237
238        # Update the output hashes again since they can be formatted.
239        for name, script in sorted(generators.iteritems()):
240            info = auto_script(script)
241            fname = get_hash_file_name(name)
242            update_output_hashes(name, info['outputs'], all_new_hashes[fname])
243
244        os.chdir(script_dir)
245
246        for fname, new_hashes in all_new_hashes.iteritems():
247            hash_fname = os.path.join(hash_dir, fname)
248            json.dump(
249                new_hashes,
250                open(hash_fname, "w"),
251                indent=2,
252                sort_keys=True,
253                separators=(',', ':\n    '))
254
255
256if __name__ == '__main__':
257    sys.exit(main())
258