1#! /usr/bin/env python3 2assert __name__ == '__main__' 3 4''' 5To update ANGLE in Gecko, use Windows with git-bash, and setup depot_tools, python2, and 6python3. Because depot_tools expects `python` to be `python2` (shame!), python2 must come 7before python3 in your path. 8 9Upstream: https://chromium.googlesource.com/angle/angle 10 11Our repo: https://github.com/mozilla/angle 12It has branches like 'firefox-60' which is the branch we use for pulling into 13Gecko with this script. 14 15This script leaves a record of the merge-base and cherry-picks that we pull into 16Gecko. (gfx/angle/cherries.log) 17 18ANGLE<->Chrome version mappings are here: https://omahaproxy.appspot.com/ 19An easy choice is to grab Chrome's Beta's ANGLE branch. 20 21## Usage 22 23Prepare your env: 24 25~~~ 26export PATH="$PATH:/path/to/depot_tools" 27~~~ 28 29If this is a new repo, don't forget: 30 31~~~ 32# In the angle repo: 33./scripts/bootstrap.py 34gclient sync 35~~~ 36 37Update: (in the angle repo) 38 39~~~ 40# In the angle repo: 41/path/to/gecko/gfx/angle/update-angle.py origin/chromium/XXXX 42git push moz # Push the firefox-XX branch to github.com/mozilla/angle 43~~~~ 44 45''' 46 47import json 48import os 49import pathlib 50import re 51import shutil 52import subprocess 53import sys 54from typing import * # mypy annotations 55 56SCRIPT_DIR = os.path.dirname(__file__) 57 58GN_ENV = dict(os.environ) 59# We need to set DEPOT_TOOLS_WIN_TOOLCHAIN to 0 for non-Googlers, but otherwise 60# leave it unset since vs_toolchain.py assumes that the user is a Googler with 61# the Visual Studio files in depot_tools if DEPOT_TOOLS_WIN_TOOLCHAIN is not 62# explicitly set to 0. 63vs_found = False 64vs_dir = os.path.join(SCRIPT_DIR, '..', 'third_party', 'depot_tools', 'win_toolchain', 'vs_files') 65if not os.path.isdir(vs_dir): 66 GN_ENV['DEPOT_TOOLS_WIN_TOOLCHAIN'] = '0' 67 68if len(sys.argv) < 3: 69 sys.exit('Usage: export_targets.py OUT_DIR ROOTS...') 70 71(OUT_DIR, *ROOTS) = sys.argv[1:] 72for x in ROOTS: 73 assert x.startswith('//:') 74 75# ------------------------------------------------------------------------------ 76 77def run_checked(*args, **kwargs): 78 print(' ', args, file=sys.stderr) 79 sys.stderr.flush() 80 return subprocess.run(args, check=True, **kwargs) 81 82 83def sortedi(x): 84 return sorted(x, key=str.lower) 85 86 87def dag_traverse(root_keys: Sequence[str], pre_recurse_func: Callable[[str], list]): 88 visited_keys: Set[str] = set() 89 90 def recurse(key): 91 if key in visited_keys: 92 return 93 visited_keys.add(key) 94 95 t = pre_recurse_func(key) 96 try: 97 (next_keys, post_recurse_func) = t 98 except ValueError: 99 (next_keys,) = t 100 post_recurse_func = None 101 102 for x in next_keys: 103 recurse(x) 104 105 if post_recurse_func: 106 post_recurse_func(key) 107 return 108 109 for x in root_keys: 110 recurse(x) 111 return 112 113# ------------------------------------------------------------------------------ 114 115print('Importing graph', file=sys.stderr) 116 117try: 118 p = run_checked('gn', 'desc', '--format=json', str(OUT_DIR), '*', stdout=subprocess.PIPE, 119 env=GN_ENV, shell=(True if sys.platform == 'win32' else False)) 120except subprocess.CalledProcessError: 121 sys.stderr.buffer.write(b'"gn desc" failed. Is depot_tools in your PATH?\n') 122 exit(1) 123 124# - 125 126print('\nProcessing graph', file=sys.stderr) 127descs = json.loads(p.stdout.decode()) 128 129# Ready to traverse 130# ------------------------------------------------------------------------------ 131 132LIBRARY_TYPES = ('shared_library', 'static_library') 133 134def flattened_target(target_name: str, descs: dict, stop_at_lib: bool =True) -> dict: 135 flattened = dict(descs[target_name]) 136 137 EXPECTED_TYPES = LIBRARY_TYPES + ('source_set', 'group', 'action') 138 139 def pre(k): 140 dep = descs[k] 141 142 dep_type = dep['type'] 143 deps = dep['deps'] 144 if stop_at_lib and dep_type in LIBRARY_TYPES: 145 return ((),) 146 147 if dep_type == 'copy': 148 assert not deps, (target_name, dep['deps']) 149 else: 150 assert dep_type in EXPECTED_TYPES, (k, dep_type) 151 for (k,v) in dep.items(): 152 if type(v) in (list, tuple, set): 153 # This is a workaround for 154 # https://bugs.chromium.org/p/gn/issues/detail?id=196, where 155 # the value of "public" can be a string instead of a list. 156 existing = flattened.get(k, []) 157 if isinstance(existing, str): 158 existing = [existing] 159 # Use temporary sets then sort them to avoid a bottleneck here 160 if not isinstance(existing, set): 161 flattened[k] = set(existing) 162 flattened[k].update(v) 163 else: 164 #flattened.setdefault(k, v) 165 pass 166 return (deps,) 167 168 dag_traverse(descs[target_name]['deps'], pre) 169 170 for k, v in flattened.items(): 171 if isinstance(v, set): 172 flattened[k] = sortedi(v) 173 return flattened 174 175# ------------------------------------------------------------------------------ 176# Check that includes are valid. (gn's version of this check doesn't seem to work!) 177 178INCLUDE_REGEX = re.compile(b'^ *# *include +([<"])([^>"]+)[>"].*$', re.MULTILINE) 179assert INCLUDE_REGEX.findall(b' # include <foo> //comment\n#include "bar"') == [(b'<', b'foo'), (b'"', b'bar')] 180 181# Most of these are ignored because this script does not currently handle 182# #includes in #ifdefs properly, so they will erroneously be marked as being 183# included, but not part of the source list. 184IGNORED_INCLUDES = { 185 b'absl/container/flat_hash_map.h', 186 b'absl/container/flat_hash_set.h', 187 b'compiler/translator/glsl/TranslatorESSL.h', 188 b'compiler/translator/glsl/TranslatorGLSL.h', 189 b'compiler/translator/hlsl/TranslatorHLSL.h', 190 b'compiler/translator/msl/TranslatorMSL.h', 191 b'compiler/translator/null/TranslatorNULL.h', 192 b'compiler/translator/spirv/TranslatorSPIRV.h', 193 b'compiler/translator/wgsl/TranslatorWGSL.h', 194 b'contrib/optimizations/slide_hash_neon.h', 195 b'dirent_on_windows.h', 196 b'dlopen_fuchsia.h', 197 b'kernel/image.h', 198 b'libANGLE/renderer/d3d/d3d11/Device11.h', 199 b'libANGLE/renderer/d3d/d3d11/winrt/NativeWindow11WinRT.h', 200 b'libANGLE/renderer/d3d/DisplayD3D.h', 201 b'libANGLE/renderer/d3d/RenderTargetD3D.h', 202 b'libANGLE/renderer/gl/cgl/DisplayCGL.h', 203 b'libANGLE/renderer/gl/egl/android/DisplayAndroid.h', 204 b'libANGLE/renderer/gl/egl/DisplayEGL.h', 205 b'libANGLE/renderer/gl/egl/gbm/DisplayGbm.h', 206 b'libANGLE/renderer/gl/glx/DisplayGLX.h', 207 b'libANGLE/renderer/gl/glx/DisplayGLX_api.h', 208 b'libANGLE/renderer/gl/wgl/DisplayWGL.h', 209 b'libANGLE/renderer/metal/DisplayMtl_api.h', 210 b'libANGLE/renderer/null/DisplayNULL.h', 211 b'libANGLE/renderer/vulkan/android/AHBFunctions.h', 212 b'libANGLE/renderer/vulkan/android/DisplayVkAndroid.h', 213 b'libANGLE/renderer/vulkan/DisplayVk_api.h', 214 b'libANGLE/renderer/vulkan/fuchsia/DisplayVkFuchsia.h', 215 b'libANGLE/renderer/vulkan/mac/DisplayVkMac.h', 216 b'libANGLE/renderer/vulkan/win32/DisplayVkWin32.h', 217 b'libANGLE/renderer/vulkan/xcb/DisplayVkXcb.h', 218 b'libANGLE/renderer/vulkan/wayland/DisplayVkWayland.h', 219 b'loader_cmake_config.h', 220 b'loader_linux.h', 221 b'loader_windows.h', 222 b'optick.h', 223 b'spirv-tools/libspirv.h', 224 b'third_party/volk/volk.h', 225 b'vk_loader_extensions.c', 226 b'vk_snippets.h', 227 b'vulkan_android.h', 228 b'vulkan_beta.h', 229 b'vulkan_directfb.h', 230 b'vulkan_fuchsia.h', 231 b'vulkan_ggp.h', 232 b'vulkan_ios.h', 233 b'vulkan_macos.h', 234 b'vulkan_metal.h', 235 b'vulkan_sci.h', 236 b'vulkan_vi.h', 237 b'vulkan_wayland.h', 238 b'vulkan_win32.h', 239 b'vulkan_xcb.h', 240 b'vulkan_xlib.h', 241 b'vulkan_xlib_xrandr.h', 242 # rapidjson adds these include stubs into their documentation 243 # comments. Since the script doesn't skip comments they are 244 # erroneously marked as valid includes 245 b'rapidjson/...', 246 # Validation layers support building with robin hood hashing, but we are not enabling that 247 # See http://anglebug.com/42264327 248 b'robin_hood.h', 249 # Validation layers optionally use mimalloc 250 b'mimalloc-new-delete.h', 251 # From the Vulkan-Loader 252 b'winres.h', 253 # From a comment in vulkan-validation-layers/src/layers/vk_mem_alloc.h 254 b'my_custom_assert.h', 255 b'my_custom_min.h', 256 # https://bugs.chromium.org/p/gn/issues/detail?id=311 257 b'spirv/unified1/spirv.hpp11', 258 # Behind #if defined(QAT_COMPRESSION_ENABLED) in third_party/zlib/deflate.c 259 b'contrib/qat/deflate_qat.h', 260 # Behind #if defined(TRACY_ENABLE) in third_party/vulkan-validation-layers/src/layers/vulkan/generated/chassis.cpp 261 b'profiling/profiling.h', 262} 263 264IGNORED_INCLUDE_PREFIXES = { 265 b'android', 266 b'Carbon', 267 b'CoreFoundation', 268 b'CoreServices', 269 b'IOSurface', 270 b'mach', 271 b'mach-o', 272 b'OpenGL', 273 b'pci', 274 b'sys', 275 b'wrl', 276 b'X11', 277} 278 279IGNORED_DIRECTORIES = { 280 '//buildtools/third_party/libc++', 281 '//third_party/libc++/src', 282 '//third_party/abseil-cpp', 283 '//third_party/SwiftShader', 284 '//third_party/dawn', 285} 286 287def has_all_includes(target_name: str, descs: dict) -> bool: 288 for ignored_directory in IGNORED_DIRECTORIES: 289 if target_name.startswith(ignored_directory): 290 return True 291 292 flat = flattened_target(target_name, descs, stop_at_lib=False) 293 acceptable_sources = flat.get('sources', []) + flat.get('outputs', []) 294 acceptable_sources = {x.rsplit('/', 1)[-1].encode() for x in acceptable_sources} 295 296 ret = True 297 desc = descs[target_name] 298 for cur_file in desc.get('sources', []): 299 assert cur_file.startswith('/'), cur_file 300 if not cur_file.startswith('//'): 301 continue 302 cur_file = pathlib.Path(cur_file[2:]) 303 text = cur_file.read_bytes() 304 for m in INCLUDE_REGEX.finditer(text): 305 if m.group(1) == b'<': 306 continue 307 include = m.group(2) 308 if include in IGNORED_INCLUDES: 309 continue 310 try: 311 (prefix, _) = include.split(b'/', 1) 312 if prefix in IGNORED_INCLUDE_PREFIXES: 313 continue 314 except ValueError: 315 pass 316 317 include_file = include.rsplit(b'/', 1)[-1] 318 if include_file not in acceptable_sources: 319 #print(' acceptable_sources:') 320 #for x in sorted(acceptable_sources): 321 # print(' ', x) 322 print('Warning in {}: {}: Included file must be listed in the GN target or its public dependency: {}'.format(target_name, cur_file, include), file=sys.stderr) 323 ret = False 324 #print('Looks valid:', m.group()) 325 continue 326 327 return ret 328 329# - 330# Gather real targets: 331 332def gather_libraries(roots: Sequence[str], descs: dict) -> Set[str]: 333 libraries = set() 334 def fn(target_name): 335 cur = descs[target_name] 336 print(' ' + cur['type'], target_name, file=sys.stderr) 337 assert has_all_includes(target_name, descs), target_name 338 339 if cur['type'] in ('shared_library', 'static_library'): 340 libraries.add(target_name) 341 return (cur['deps'], ) 342 343 dag_traverse(roots, fn) 344 return libraries 345 346# - 347 348libraries = gather_libraries(ROOTS, descs) 349print(f'\n{len(libraries)} libraries:', file=sys.stderr) 350for k in libraries: 351 print(f' {k}', file=sys.stderr) 352print('\nstdout begins:', file=sys.stderr) 353sys.stderr.flush() 354 355# ------------------------------------------------------------------------------ 356# Output 357 358out = {k: flattened_target(k, descs) for k in libraries} 359 360for (k,desc) in out.items(): 361 dep_libs: Set[str] = set() 362 for dep_name in set(desc['deps']): 363 dep = descs[dep_name] 364 if dep['type'] in LIBRARY_TYPES: 365 dep_libs.add(dep_name) 366 desc['dep_libs'] = sortedi(dep_libs) 367 368json.dump(out, sys.stdout, indent=' ') 369exit(0) 370