• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#! /usr/bin/env python3
2assert __name__ == '__main__'
3
4'''
5To update ANGLE in Gecko, use Windows with git-bash, and setup depot_tools, python2, and
6python3. Because depot_tools expects `python` to be `python2` (shame!), python2 must come
7before python3 in your path.
8
9Upstream: https://chromium.googlesource.com/angle/angle
10
11Our repo: https://github.com/mozilla/angle
12It has branches like 'firefox-60' which is the branch we use for pulling into
13Gecko with this script.
14
15This script leaves a record of the merge-base and cherry-picks that we pull into
16Gecko. (gfx/angle/cherries.log)
17
18ANGLE<->Chrome version mappings are here: https://omahaproxy.appspot.com/
19An easy choice is to grab Chrome's Beta's ANGLE branch.
20
21## Usage
22
23Prepare your env:
24
25~~~
26export PATH="$PATH:/path/to/depot_tools"
27~~~
28
29If this is a new repo, don't forget:
30
31~~~
32# In the angle repo:
33./scripts/bootstrap.py
34gclient sync
35~~~
36
37Update: (in the angle repo)
38
39~~~
40# In the angle repo:
41/path/to/gecko/gfx/angle/update-angle.py origin/chromium/XXXX
42git push moz # Push the firefox-XX branch to github.com/mozilla/angle
43~~~~
44
45'''
46
47import json
48import os
49import pathlib
50import re
51import shutil
52import subprocess
53import sys
54from typing import * # mypy annotations
55
56SCRIPT_DIR = os.path.dirname(__file__)
57
58GN_ENV = dict(os.environ)
59# We need to set DEPOT_TOOLS_WIN_TOOLCHAIN to 0 for non-Googlers, but otherwise
60# leave it unset since vs_toolchain.py assumes that the user is a Googler with
61# the Visual Studio files in depot_tools if DEPOT_TOOLS_WIN_TOOLCHAIN is not
62# explicitly set to 0.
63vs_found = False
64vs_dir = os.path.join(SCRIPT_DIR, '..', 'third_party', 'depot_tools', 'win_toolchain', 'vs_files')
65if not os.path.isdir(vs_dir):
66    GN_ENV['DEPOT_TOOLS_WIN_TOOLCHAIN'] = '0'
67
68if len(sys.argv) < 3:
69    sys.exit('Usage: export_targets.py OUT_DIR ROOTS...')
70
71(OUT_DIR, *ROOTS) = sys.argv[1:]
72for x in ROOTS:
73    assert x.startswith('//:')
74
75# ------------------------------------------------------------------------------
76
77def run_checked(*args, **kwargs):
78    print(' ', args, file=sys.stderr)
79    sys.stderr.flush()
80    return subprocess.run(args, check=True, **kwargs)
81
82
83def sortedi(x):
84    return sorted(x, key=str.lower)
85
86
87def dag_traverse(root_keys: Sequence[str], pre_recurse_func: Callable[[str], list]):
88    visited_keys: Set[str] = set()
89
90    def recurse(key):
91        if key in visited_keys:
92            return
93        visited_keys.add(key)
94
95        t = pre_recurse_func(key)
96        try:
97            (next_keys, post_recurse_func) = t
98        except ValueError:
99            (next_keys,) = t
100            post_recurse_func = None
101
102        for x in next_keys:
103            recurse(x)
104
105        if post_recurse_func:
106            post_recurse_func(key)
107        return
108
109    for x in root_keys:
110        recurse(x)
111    return
112
113# ------------------------------------------------------------------------------
114
115print('Importing graph', file=sys.stderr)
116
117try:
118    p = run_checked('gn', 'desc', '--format=json', str(OUT_DIR), '*', stdout=subprocess.PIPE,
119                env=GN_ENV, shell=(True if sys.platform == 'win32' else False))
120except subprocess.CalledProcessError:
121    sys.stderr.buffer.write(b'"gn desc" failed. Is depot_tools in your PATH?\n')
122    exit(1)
123
124# -
125
126print('\nProcessing graph', file=sys.stderr)
127descs = json.loads(p.stdout.decode())
128
129# Ready to traverse
130# ------------------------------------------------------------------------------
131
132LIBRARY_TYPES = ('shared_library', 'static_library')
133
134def flattened_target(target_name: str, descs: dict, stop_at_lib: bool =True) -> dict:
135    flattened = dict(descs[target_name])
136
137    EXPECTED_TYPES = LIBRARY_TYPES + ('source_set', 'group', 'action')
138
139    def pre(k):
140        dep = descs[k]
141
142        dep_type = dep['type']
143        deps = dep['deps']
144        if stop_at_lib and dep_type in LIBRARY_TYPES:
145            return ((),)
146
147        if dep_type == 'copy':
148            assert not deps, (target_name, dep['deps'])
149        else:
150            assert dep_type in EXPECTED_TYPES, (k, dep_type)
151            for (k,v) in dep.items():
152                if type(v) in (list, tuple, set):
153                    # This is a workaround for
154                    # https://bugs.chromium.org/p/gn/issues/detail?id=196, where
155                    # the value of "public" can be a string instead of a list.
156                    existing = flattened.get(k, [])
157                    if isinstance(existing, str):
158                      existing = [existing]
159                    flattened[k] = sortedi(set(existing + v))
160                else:
161                    #flattened.setdefault(k, v)
162                    pass
163        return (deps,)
164
165    dag_traverse(descs[target_name]['deps'], pre)
166    return flattened
167
168# ------------------------------------------------------------------------------
169# Check that includes are valid. (gn's version of this check doesn't seem to work!)
170
171INCLUDE_REGEX = re.compile(b'(?:^|\\n) *# *include +([<"])([^>"]+)[>"]')
172assert INCLUDE_REGEX.match(b'#include "foo"')
173assert INCLUDE_REGEX.match(b'\n#include "foo"')
174
175# Most of these are ignored because this script does not currently handle
176# #includes in #ifdefs properly, so they will erroneously be marked as being
177# included, but not part of the source list.
178IGNORED_INCLUDES = {
179    b'absl/container/flat_hash_map.h',
180    b'absl/container/flat_hash_set.h',
181    b'compiler/translator/TranslatorESSL.h',
182    b'compiler/translator/TranslatorGLSL.h',
183    b'compiler/translator/TranslatorHLSL.h',
184    b'compiler/translator/TranslatorMetal.h',
185    b'compiler/translator/TranslatorMetalDirect.h',
186    b'compiler/translator/TranslatorVulkan.h',
187    b'contrib/optimizations/slide_hash_neon.h',
188    b'dirent_on_windows.h',
189    b'dlopen_fuchsia.h',
190    b'kernel/image.h',
191    b'libANGLE/renderer/d3d/d3d11/winrt/NativeWindow11WinRT.h',
192    b'libANGLE/renderer/d3d/DeviceD3D.h',
193    b'libANGLE/renderer/d3d/DisplayD3D.h',
194    b'libANGLE/renderer/d3d/RenderTargetD3D.h',
195    b'libANGLE/renderer/gl/apple/DisplayApple_api.h',
196    b'libANGLE/renderer/gl/cgl/DisplayCGL.h',
197    b'libANGLE/renderer/gl/eagl/DisplayEAGL.h',
198    b'libANGLE/renderer/gl/egl/android/DisplayAndroid.h',
199    b'libANGLE/renderer/gl/egl/DisplayEGL.h',
200    b'libANGLE/renderer/gl/egl/gbm/DisplayGbm.h',
201    b'libANGLE/renderer/gl/glx/DisplayGLX.h',
202    b'libANGLE/renderer/gl/wgl/DisplayWGL.h',
203    b'libANGLE/renderer/metal/DisplayMtl_api.h',
204    b'libANGLE/renderer/null/DisplayNULL.h',
205    b'libANGLE/renderer/vulkan/android/AHBFunctions.h',
206    b'libANGLE/renderer/vulkan/android/DisplayVkAndroid.h',
207    b'libANGLE/renderer/vulkan/DisplayVk_api.h',
208    b'libANGLE/renderer/vulkan/fuchsia/DisplayVkFuchsia.h',
209    b'libANGLE/renderer/vulkan/ggp/DisplayVkGGP.h',
210    b'libANGLE/renderer/vulkan/mac/DisplayVkMac.h',
211    b'libANGLE/renderer/vulkan/win32/DisplayVkWin32.h',
212    b'libANGLE/renderer/vulkan/xcb/DisplayVkXcb.h',
213    b'libANGLE/renderer/vulkan/wayland/DisplayVkWayland.h',
214    b'loader_cmake_config.h',
215    b'loader_linux.h',
216    b'loader_windows.h',
217    b'optick.h',
218    b'spirv-tools/libspirv.h',
219    b'third_party/volk/volk.h',
220    b'vk_loader_extensions.c',
221    b'vk_snippets.h',
222    b'vulkan_android.h',
223    b'vulkan_beta.h',
224    b'vulkan_directfb.h',
225    b'vulkan_fuchsia.h',
226    b'vulkan_ggp.h',
227    b'vulkan_ios.h',
228    b'vulkan_macos.h',
229    b'vulkan_metal.h',
230    b'vulkan_vi.h',
231    b'vulkan_wayland.h',
232    b'vulkan_win32.h',
233    b'vulkan_xcb.h',
234    b'vulkan_xlib.h',
235    b'vulkan_xlib_xrandr.h',
236# rapidjson adds these include stubs into their documentation
237# comments. Since the script doesn't skip comments they are
238# erroneously marked as valid includes
239    b'rapidjson/...',
240    # Validation layers support building with robin hood hashing, but we are not enabling that
241    # See http://anglebug.com/5791
242    b'robin_hood.h',
243    # From the Vulkan-Loader
244    b'winres.h',
245}
246
247IGNORED_INCLUDE_PREFIXES = {
248    b'android',
249    b'Carbon',
250    b'CoreFoundation',
251    b'CoreServices',
252    b'IOSurface',
253    b'mach',
254    b'mach-o',
255    b'OpenGL',
256    b'pci',
257    b'sys',
258    b'wrl',
259    b'X11',
260}
261
262IGNORED_DIRECTORIES = {
263    '//buildtools/third_party/libc++',
264    '//third_party/abseil-cpp',
265    '//third_party/SwiftShader',
266}
267
268def has_all_includes(target_name: str, descs: dict) -> bool:
269    for ignored_directory in IGNORED_DIRECTORIES:
270        if target_name.startswith(ignored_directory):
271            return True
272
273    flat = flattened_target(target_name, descs, stop_at_lib=False)
274    acceptable_sources = flat.get('sources', []) + flat.get('outputs', [])
275    acceptable_sources = {x.rsplit('/', 1)[-1].encode() for x in acceptable_sources}
276
277    ret = True
278    desc = descs[target_name]
279    for cur_file in desc.get('sources', []):
280        assert cur_file.startswith('/'), cur_file
281        if not cur_file.startswith('//'):
282            continue
283        cur_file = pathlib.Path(cur_file[2:])
284        text = cur_file.read_bytes()
285        for m in INCLUDE_REGEX.finditer(text):
286            if m.group(1) == b'<':
287                continue
288            include = m.group(2)
289            if include in IGNORED_INCLUDES:
290                continue
291            try:
292                (prefix, _) = include.split(b'/', 1)
293                if prefix in IGNORED_INCLUDE_PREFIXES:
294                    continue
295            except ValueError:
296                pass
297
298            include_file = include.rsplit(b'/', 1)[-1]
299            if include_file not in acceptable_sources:
300                #print('  acceptable_sources:')
301                #for x in sorted(acceptable_sources):
302                #    print('   ', x)
303                print('Warning in {}: {}: Included file must be listed in the GN target or its public dependency: {}'.format(target_name, cur_file, include), file=sys.stderr)
304                ret = False
305            #print('Looks valid:', m.group())
306            continue
307
308    return ret
309
310# -
311# Gather real targets:
312
313def gather_libraries(roots: Sequence[str], descs: dict) -> Set[str]:
314    libraries = set()
315    def fn(target_name):
316        cur = descs[target_name]
317        print('  ' + cur['type'], target_name, file=sys.stderr)
318        assert has_all_includes(target_name, descs), target_name
319
320        if cur['type'] in ('shared_library', 'static_library'):
321            libraries.add(target_name)
322        return (cur['deps'], )
323
324    dag_traverse(roots, fn)
325    return libraries
326
327# -
328
329libraries = gather_libraries(ROOTS, descs)
330print(f'\n{len(libraries)} libraries:', file=sys.stderr)
331for k in libraries:
332    print(f'  {k}', file=sys.stderr)
333print('\nstdout begins:', file=sys.stderr)
334sys.stderr.flush()
335
336# ------------------------------------------------------------------------------
337# Output
338
339out = {k: flattened_target(k, descs) for k in libraries}
340
341for (k,desc) in out.items():
342    dep_libs: Set[str] = set()
343    for dep_name in set(desc['deps']):
344        dep = descs[dep_name]
345        if dep['type'] in LIBRARY_TYPES:
346            dep_libs.add(dep_name)
347    desc['dep_libs'] = sortedi(dep_libs)
348
349json.dump(out, sys.stdout, indent='  ')
350exit(0)
351