1#! /usr/bin/env python3 2assert __name__ == '__main__' 3 4''' 5To update ANGLE in Gecko, use Windows with git-bash, and setup depot_tools, python2, and 6python3. Because depot_tools expects `python` to be `python2` (shame!), python2 must come 7before python3 in your path. 8 9Upstream: https://chromium.googlesource.com/angle/angle 10 11Our repo: https://github.com/mozilla/angle 12It has branches like 'firefox-60' which is the branch we use for pulling into 13Gecko with this script. 14 15This script leaves a record of the merge-base and cherry-picks that we pull into 16Gecko. (gfx/angle/cherries.log) 17 18ANGLE<->Chrome version mappings are here: https://omahaproxy.appspot.com/ 19An easy choice is to grab Chrome's Beta's ANGLE branch. 20 21## Usage 22 23Prepare your env: 24 25~~~ 26export PATH="$PATH:/path/to/depot_tools" 27~~~ 28 29If this is a new repo, don't forget: 30 31~~~ 32# In the angle repo: 33./scripts/bootstrap.py 34gclient sync 35~~~ 36 37Update: (in the angle repo) 38 39~~~ 40# In the angle repo: 41/path/to/gecko/gfx/angle/update-angle.py origin/chromium/XXXX 42git push moz # Push the firefox-XX branch to github.com/mozilla/angle 43~~~~ 44 45''' 46 47import json 48import os 49import pathlib 50import re 51import shutil 52import subprocess 53import sys 54from typing import * # mypy annotations 55 56REPO_DIR = pathlib.Path.cwd() 57 58GN_ENV = dict(os.environ) 59# We need to set DEPOT_TOOLS_WIN_TOOLCHAIN to 0 for non-Googlers, but otherwise 60# leave it unset since vs_toolchain.py assumes that the user is a Googler with 61# the Visual Studio files in depot_tools if DEPOT_TOOLS_WIN_TOOLCHAIN is not 62# explicitly set to 0. 63vs_found = False 64for directory in os.environ['PATH'].split(os.pathsep): 65 vs_dir = os.path.join(directory, 'win_toolchain', 'vs_files') 66 if os.path.exists(vs_dir): 67 vs_found = True 68 break 69if not vs_found: 70 GN_ENV['DEPOT_TOOLS_WIN_TOOLCHAIN'] = '0' 71 72if len(sys.argv) < 3: 73 sys.exit('Usage: export_targets.py OUT_DIR ROOTS...') 74 75(OUT_DIR, *ROOTS) = sys.argv[1:] 76for x in ROOTS: 77 assert x.startswith('//:') 78 79# ------------------------------------------------------------------------------ 80 81def run_checked(*args, **kwargs): 82 print(' ', args, file=sys.stderr) 83 sys.stderr.flush() 84 return subprocess.run(args, check=True, **kwargs) 85 86 87def sortedi(x): 88 return sorted(x, key=str.lower) 89 90 91def dag_traverse(root_keys: Sequence[str], pre_recurse_func: Callable[[str], list]): 92 visited_keys: Set[str] = set() 93 94 def recurse(key): 95 if key in visited_keys: 96 return 97 visited_keys.add(key) 98 99 t = pre_recurse_func(key) 100 try: 101 (next_keys, post_recurse_func) = t 102 except ValueError: 103 (next_keys,) = t 104 post_recurse_func = None 105 106 for x in next_keys: 107 recurse(x) 108 109 if post_recurse_func: 110 post_recurse_func(key) 111 return 112 113 for x in root_keys: 114 recurse(x) 115 return 116 117# ------------------------------------------------------------------------------ 118 119print('Importing graph', file=sys.stderr) 120 121try: 122 p = run_checked('gn', 'desc', '--format=json', str(OUT_DIR), '*', stdout=subprocess.PIPE, 123 env=GN_ENV, shell=(True if sys.platform == 'win32' else False)) 124except subprocess.CalledProcessError: 125 sys.stderr.buffer.write(b'"gn desc" failed. Is depot_tools in your PATH?\n') 126 exit(1) 127 128# - 129 130print('\nProcessing graph', file=sys.stderr) 131descs = json.loads(p.stdout.decode()) 132 133# Ready to traverse 134# ------------------------------------------------------------------------------ 135 136LIBRARY_TYPES = ('shared_library', 'static_library') 137 138def flattened_target(target_name: str, descs: dict, stop_at_lib: bool =True) -> dict: 139 flattened = dict(descs[target_name]) 140 141 EXPECTED_TYPES = LIBRARY_TYPES + ('source_set', 'group', 'action') 142 143 def pre(k): 144 dep = descs[k] 145 146 dep_type = dep['type'] 147 deps = dep['deps'] 148 if stop_at_lib and dep_type in LIBRARY_TYPES: 149 return ((),) 150 151 if dep_type == 'copy': 152 assert not deps, (target_name, dep['deps']) 153 else: 154 assert dep_type in EXPECTED_TYPES, (k, dep_type) 155 for (k,v) in dep.items(): 156 if type(v) in (list, tuple, set): 157 flattened[k] = sortedi(set(flattened.get(k, []) + v)) 158 else: 159 #flattened.setdefault(k, v) 160 pass 161 return (deps,) 162 163 dag_traverse(descs[target_name]['deps'], pre) 164 return flattened 165 166# ------------------------------------------------------------------------------ 167# Check that includes are valid. (gn's version of this check doesn't seem to work!) 168 169INCLUDE_REGEX = re.compile(b'(?:^|\\n) *# *include +([<"])([^>"]+)[>"]') 170assert INCLUDE_REGEX.match(b'#include "foo"') 171assert INCLUDE_REGEX.match(b'\n#include "foo"') 172 173# Most of these are ignored because this script does not currently handle 174# #includes in #ifdefs properly, so they will erroneously be marked as being 175# included, but not part of the source list. 176IGNORED_INCLUDES = { 177 b'compiler/translator/TranslatorESSL.h', 178 b'compiler/translator/TranslatorGLSL.h', 179 b'compiler/translator/TranslatorHLSL.h', 180 b'compiler/translator/TranslatorMetal.h', 181 b'compiler/translator/TranslatorVulkan.h', 182 b'libANGLE/renderer/d3d/DeviceD3D.h', 183 b'libANGLE/renderer/d3d/DisplayD3D.h', 184 b'libANGLE/renderer/d3d/RenderTargetD3D.h', 185 b'libANGLE/renderer/d3d/d3d11/winrt/NativeWindow11WinRT.h', 186 b'libANGLE/renderer/gl/cgl/DisplayCGL.h', 187 b'libANGLE/renderer/gl/eagl/DisplayEAGL.h', 188 b'libANGLE/renderer/gl/egl/android/DisplayAndroid.h', 189 b'libANGLE/renderer/gl/egl/DisplayEGL.h', 190 b'libANGLE/renderer/gl/egl/ozone/DisplayOzone.h', 191 b'libANGLE/renderer/gl/glx/DisplayGLX.h', 192 b'libANGLE/renderer/gl/wgl/DisplayWGL.h', 193 b'libANGLE/renderer/metal/DisplayMtl_api.h', 194 b'libANGLE/renderer/null/DisplayNULL.h', 195 b'libANGLE/renderer/vulkan/android/DisplayVkAndroid.h', 196 b'libANGLE/renderer/vulkan/fuchsia/DisplayVkFuchsia.h', 197 b'libANGLE/renderer/vulkan/ggp/DisplayVkGGP.h', 198 b'libANGLE/renderer/vulkan/mac/DisplayVkMac.h', 199 b'libANGLE/renderer/vulkan/win32/DisplayVkWin32.h', 200 b'libANGLE/renderer/vulkan/xcb/DisplayVkXcb.h', 201 b'kernel/image.h', 202} 203 204IGNORED_INCLUDE_PREFIXES = { 205 b'android', 206 b'Carbon', 207 b'CoreFoundation', 208 b'CoreServices', 209 b'IOSurface', 210 b'mach', 211 b'mach-o', 212 b'OpenGL', 213 b'pci', 214 b'sys', 215 b'wrl', 216 b'X11', 217} 218 219IGNORED_DIRECTORIES = { 220 '//third_party/SwiftShader', 221 '//third_party/vulkan-headers', 222 '//third_party/vulkan-loader', 223 '//third_party/vulkan-tools', 224 '//third_party/vulkan-validation-layers', 225} 226 227def has_all_includes(target_name: str, descs: dict) -> bool: 228 for ignored_directory in IGNORED_DIRECTORIES: 229 if target_name.startswith(ignored_directory): 230 return True 231 232 flat = flattened_target(target_name, descs, stop_at_lib=False) 233 acceptable_sources = flat.get('sources', []) + flat.get('outputs', []) 234 acceptable_sources = {x.rsplit('/', 1)[-1].encode() for x in acceptable_sources} 235 236 ret = True 237 desc = descs[target_name] 238 for cur_file in desc.get('sources', []): 239 assert cur_file.startswith('/'), cur_file 240 if not cur_file.startswith('//'): 241 continue 242 cur_file = pathlib.Path(cur_file[2:]) 243 text = cur_file.read_bytes() 244 for m in INCLUDE_REGEX.finditer(text): 245 if m.group(1) == b'<': 246 continue 247 include = m.group(2) 248 if include in IGNORED_INCLUDES: 249 continue 250 try: 251 (prefix, _) = include.split(b'/', 1) 252 if prefix in IGNORED_INCLUDE_PREFIXES: 253 continue 254 except ValueError: 255 pass 256 257 include_file = include.rsplit(b'/', 1)[-1] 258 if include_file not in acceptable_sources: 259 #print(' acceptable_sources:') 260 #for x in sorted(acceptable_sources): 261 # print(' ', x) 262 print('Warning in {}: {}: Invalid include: {}'.format(target_name, cur_file, include), file=sys.stderr) 263 ret = False 264 #print('Looks valid:', m.group()) 265 continue 266 267 return ret 268 269# - 270# Gather real targets: 271 272def gather_libraries(roots: Sequence[str], descs: dict) -> Set[str]: 273 libraries = set() 274 def fn(target_name): 275 cur = descs[target_name] 276 print(' ' + cur['type'], target_name, file=sys.stderr) 277 assert has_all_includes(target_name, descs), target_name 278 279 if cur['type'] in ('shared_library', 'static_library'): 280 libraries.add(target_name) 281 return (cur['deps'], ) 282 283 dag_traverse(roots, fn) 284 return libraries 285 286# - 287 288libraries = gather_libraries(ROOTS, descs) 289print(f'\n{len(libraries)} libraries:', file=sys.stderr) 290for k in libraries: 291 print(f' {k}', file=sys.stderr) 292print('\nstdout begins:', file=sys.stderr) 293sys.stderr.flush() 294 295# ------------------------------------------------------------------------------ 296# Output 297 298out = {k: flattened_target(k, descs) for k in libraries} 299 300for (k,desc) in out.items(): 301 dep_libs: Set[str] = set() 302 for dep_name in set(desc['deps']): 303 dep = descs[dep_name] 304 if dep['type'] in LIBRARY_TYPES: 305 dep_libs.add(dep_name[3:]) 306 desc['deps'] = sortedi(dep_libs) 307 308json.dump(out, sys.stdout, indent=' ') 309exit(0) 310