1#!/usr/bin/env python3 2# Copyright 2019 Google LLC 3# 4# Licensed under the Apache License, Version 2.0 (the "License"); 5# you may not use this file except in compliance with the License. 6# You may obtain a copy of the License at 7# 8# http://www.apache.org/licenses/LICENSE-2.0 9# 10# Unless required by applicable law or agreed to in writing, software 11# distributed under the License is distributed on an "AS IS" BASIS, 12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13# See the License for the specific language governing permissions and 14# limitations under the License. 15 16"""Parse a DEPS file and git checkout all of the dependencies. 17 18Args: 19 An optional list of deps_os values. 20 21 --with-swiftshader Checkout Swiftshader dependencies 22 --with-clspv Checkout clspv dependencies 23 --with-dxc Checkout dxc dependencies 24 25Environment Variables: 26 GIT_EXECUTABLE: path to "git" binary; if unset, will look for one of 27 ['git', 'git.exe', 'git.bat'] in your default path. 28 29 GIT_SYNC_DEPS_PATH: file to get the dependency list from; if unset, 30 will use the file ../DEPS relative to this script's directory. 31 32 GIT_SYNC_DEPS_QUIET: if set to non-empty string, suppress messages. 33 34Git Config: 35 To disable syncing of a single repository: 36 cd path/to/repository 37 git config sync-deps.disable true 38 39 To re-enable sync: 40 cd path/to/repository 41 git config --unset sync-deps.disable 42""" 43 44import os 45import re 46import subprocess 47import sys 48import threading 49from builtins import bytes 50 51with_clspv = False 52with_dxc = False 53with_swiftshader = False 54 55def git_executable(): 56 """Find the git executable. 57 58 Returns: 59 A string suitable for passing to subprocess functions, or None. 60 """ 61 envgit = os.environ.get('GIT_EXECUTABLE') 62 searchlist = ['git', 'git.exe', 'git.bat'] 63 if envgit: 64 searchlist.insert(0, envgit) 65 with open(os.devnull, 'w') as devnull: 66 for git in searchlist: 67 try: 68 subprocess.call([git, '--version'], stdout=devnull) 69 except (OSError,): 70 continue 71 return git 72 return None 73 74 75DEFAULT_DEPS_PATH = os.path.normpath( 76 os.path.join(os.path.dirname(__file__), os.pardir, 'DEPS')) 77 78 79def usage(deps_file_path = None): 80 sys.stderr.write( 81 'Usage: run to grab dependencies, with optional platform support:\n') 82 sys.stderr.write(' %s %s' % (sys.executable, __file__)) 83 if deps_file_path: 84 parsed_deps = parse_file_to_dict(deps_file_path) 85 if 'deps_os' in parsed_deps: 86 for deps_os in parsed_deps['deps_os']: 87 sys.stderr.write(' [%s]' % deps_os) 88 sys.stderr.write('\n\n') 89 sys.stderr.write(__doc__) 90 91 92def git_repository_sync_is_disabled(git, directory): 93 try: 94 disable = subprocess.check_output( 95 [git, 'config', 'sync-deps.disable'], cwd=directory) 96 return disable.lower().strip() in ['true', '1', 'yes', 'on'] 97 except subprocess.CalledProcessError: 98 return False 99 100 101def is_git_toplevel(git, directory): 102 """Return true iff the directory is the top level of a Git repository. 103 104 Args: 105 git (string) the git executable 106 107 directory (string) the path into which the repository 108 is expected to be checked out. 109 """ 110 try: 111 toplevel = subprocess.check_output( 112 [git, 'rev-parse', '--show-toplevel'], cwd=directory).strip() 113 return os.path.realpath(bytes(directory, 'utf8')) == os.path.realpath(toplevel) 114 except subprocess.CalledProcessError: 115 return False 116 117 118def status(directory, checkoutable): 119 def tail_truncate(s, length): 120 return s if len(s) <= length else '...' + s[-(length - 3):] 121 dlen = 36 122 directory = tail_truncate(directory, dlen) 123 checkoutable = tail_truncate(checkoutable, 40) 124 sys.stdout.write('%-*s @ %s\n' % (dlen, directory, checkoutable)) 125 126 127def git_checkout_to_directory(git, repo, checkoutable, directory, verbose): 128 """Checkout (and clone if needed) a Git repository. 129 130 Args: 131 git (string) the git executable 132 133 repo (string) the location of the repository, suitable 134 for passing to `git clone`. 135 136 checkoutable (string) a tag, branch, or commit, suitable for 137 passing to `git checkout` 138 139 directory (string) the path into which the repository 140 should be checked out. 141 142 verbose (boolean) 143 144 Raises an exception if any calls to git fail. 145 """ 146 if verbose: 147 status(directory, checkoutable) 148 149 if not os.path.isdir(directory): 150 subprocess.check_call( 151 [git, 'clone', '--quiet', repo, directory]) 152 153 if not is_git_toplevel(git, directory): 154 # if the directory exists, but isn't a git repo, you will modify 155 # the parent repostory, which isn't what you want. 156 sys.stdout.write('%s\n IS NOT TOP-LEVEL GIT DIRECTORY.\n' % directory) 157 return 158 159 # Check to see if this repo is disabled. Quick return. 160 if git_repository_sync_is_disabled(git, directory): 161 sys.stdout.write('%s\n SYNC IS DISABLED.\n' % directory) 162 return 163 164 with open(os.devnull, 'w') as devnull: 165 # If this fails, we will fetch before trying again. Don't spam user 166 # with error infomation. 167 if 0 == subprocess.call([git, 'checkout', '--quiet', checkoutable], 168 cwd=directory, stderr=devnull): 169 # if this succeeds, skip slow `git fetch`. 170 return 171 172 # If the repo has changed, always force use of the correct repo. 173 # If origin already points to repo, this is a quick no-op. 174 subprocess.check_call( 175 [git, 'remote', 'set-url', 'origin', repo], cwd=directory) 176 177 subprocess.check_call([git, 'fetch', '--quiet'], cwd=directory) 178 179 subprocess.check_call([git, 'checkout', '--quiet', checkoutable], cwd=directory) 180 181 182def parse_file_to_dict(path): 183 dictionary = {} 184 contents = open(path).read() 185 # Need to convert Var() to vars[], so that the DEPS is actually Python. Var() 186 # comes from Autoroller using gclient which has a slightly different DEPS 187 # format. 188 contents = re.sub(r"Var\((.*?)\)", r"vars[\1]", contents) 189 exec(contents, dictionary) 190 return dictionary 191 192 193def git_sync_deps(deps_file_path, command_line_os_requests, verbose): 194 """Grab dependencies, with optional platform support. 195 196 Args: 197 deps_file_path (string) Path to the DEPS file. 198 199 command_line_os_requests (list of strings) Can be empty list. 200 List of strings that should each be a key in the deps_os 201 dictionary in the DEPS file. 202 203 Raises git Exceptions. 204 """ 205 git = git_executable() 206 assert git 207 208 deps_file_directory = os.path.dirname(deps_file_path) 209 deps_file = parse_file_to_dict(deps_file_path) 210 dependencies = deps_file['deps'].copy() 211 os_specific_dependencies = deps_file.get('deps_os', dict()) 212 if 'all' in command_line_os_requests: 213 for value in os_specific_dependencies.values(): 214 dependencies.update(value) 215 else: 216 for os_name in command_line_os_requests: 217 # Add OS-specific dependencies 218 if os_name in os_specific_dependencies: 219 dependencies.update(os_specific_dependencies[os_name]) 220 for directory in dependencies: 221 for other_dir in dependencies: 222 if directory.startswith(other_dir + '/'): 223 raise Exception('%r is parent of %r' % (other_dir, directory)) 224 list_of_arg_lists = [] 225 for directory in sorted(dependencies): 226 if '@' in dependencies[directory]: 227 repo, checkoutable = dependencies[directory].rsplit('@', 1) 228 else: 229 raise Exception("please specify commit or tag") 230 231 if not with_clspv and directory == 'third_party/clspv': 232 continue 233 234 if not with_clspv and directory == 'third_party/clspv-llvm': 235 continue 236 237 if not with_dxc and directory == 'third_party/dxc': 238 continue 239 240 if not with_dxc and directory == 'third_party/DirectX-Headers': 241 continue 242 243 if not with_swiftshader and directory == 'third_party/swiftshader': 244 continue 245 246 relative_directory = os.path.join(deps_file_directory, directory) 247 248 list_of_arg_lists.append( 249 (git, repo, checkoutable, relative_directory, verbose)) 250 251 multithread(git_checkout_to_directory, list_of_arg_lists) 252 253 for directory in deps_file.get('recursedeps', []): 254 recursive_path = os.path.join(deps_file_directory, directory, 'DEPS') 255 git_sync_deps(recursive_path, command_line_os_requests, verbose) 256 257 258def multithread(function, list_of_arg_lists): 259 # for args in list_of_arg_lists: 260 # function(*args) 261 # return 262 threads = [] 263 for args in list_of_arg_lists: 264 thread = threading.Thread(None, function, None, args) 265 thread.start() 266 threads.append(thread) 267 for thread in threads: 268 thread.join() 269 270 271def main(argv): 272 global with_clspv 273 global with_dxc 274 global with_swiftshader 275 deps_file_path = os.environ.get('GIT_SYNC_DEPS_PATH', DEFAULT_DEPS_PATH) 276 verbose = not bool(os.environ.get('GIT_SYNC_DEPS_QUIET', False)) 277 278 if '--help' in argv or '-h' in argv: 279 usage(deps_file_path) 280 return 1 281 282 if '--with-clspv' in argv: 283 with_clspv = True 284 285 if '--with-dxc' in argv: 286 with_dxc = True 287 288 if '--with-swiftshader' in argv: 289 with_swiftshader = True 290 291 git_sync_deps(deps_file_path, argv, verbose) 292 # subprocess.check_call( 293 # [sys.executable, 294 # os.path.join(os.path.dirname(deps_file_path), 'bin', 'fetch-gn')]) 295 return 0 296 297 298if __name__ == '__main__': 299 exit(main(sys.argv[1:])) 300