1#!/usr/bin/env python3 2# Copyright 2019 Google LLC 3# 4# Licensed under the Apache License, Version 2.0 (the "License"); 5# you may not use this file except in compliance with the License. 6# You may obtain a copy of the License at 7# 8# http://www.apache.org/licenses/LICENSE-2.0 9# 10# Unless required by applicable law or agreed to in writing, software 11# distributed under the License is distributed on an "AS IS" BASIS, 12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13# See the License for the specific language governing permissions and 14# limitations under the License. 15 16"""Parse a DEPS file and git checkout all of the dependencies. 17 18Args: 19 An optional list of deps_os values. 20 21 --with-swiftshader Checkout Swiftshader dependencies 22 --with-clspv Checkout clspv dependencies 23 --with-dxc Checkout dxc dependencies 24 25Environment Variables: 26 GIT_EXECUTABLE: path to "git" binary; if unset, will look for one of 27 ['git', 'git.exe', 'git.bat'] in your default path. 28 29 GIT_SYNC_DEPS_PATH: file to get the dependency list from; if unset, 30 will use the file ../DEPS relative to this script's directory. 31 32 GIT_SYNC_DEPS_QUIET: if set to non-empty string, suppress messages. 33 34Git Config: 35 To disable syncing of a single repository: 36 cd path/to/repository 37 git config sync-deps.disable true 38 39 To re-enable sync: 40 cd path/to/repository 41 git config --unset sync-deps.disable 42""" 43 44import os 45import re 46import subprocess 47import sys 48import threading 49from builtins import bytes 50 51with_clspv = False 52with_dxc = False 53with_swiftshader = False 54 55def git_executable(): 56 """Find the git executable. 57 58 Returns: 59 A string suitable for passing to subprocess functions, or None. 60 """ 61 envgit = os.environ.get('GIT_EXECUTABLE') 62 searchlist = ['git', 'git.exe', 'git.bat'] 63 if envgit: 64 searchlist.insert(0, envgit) 65 with open(os.devnull, 'w') as devnull: 66 for git in searchlist: 67 try: 68 subprocess.call([git, '--version'], stdout=devnull) 69 except (OSError,): 70 continue 71 return git 72 return None 73 74 75DEFAULT_DEPS_PATH = os.path.normpath( 76 os.path.join(os.path.dirname(__file__), os.pardir, 'DEPS')) 77 78 79def usage(deps_file_path = None): 80 sys.stderr.write( 81 'Usage: run to grab dependencies, with optional platform support:\n') 82 sys.stderr.write(' %s %s' % (sys.executable, __file__)) 83 if deps_file_path: 84 parsed_deps = parse_file_to_dict(deps_file_path) 85 if 'deps_os' in parsed_deps: 86 for deps_os in parsed_deps['deps_os']: 87 sys.stderr.write(' [%s]' % deps_os) 88 sys.stderr.write('\n\n') 89 sys.stderr.write(__doc__) 90 91 92def git_repository_sync_is_disabled(git, directory): 93 try: 94 disable = subprocess.check_output( 95 [git, 'config', 'sync-deps.disable'], cwd=directory) 96 return disable.lower().strip() in ['true', '1', 'yes', 'on'] 97 except subprocess.CalledProcessError: 98 return False 99 100 101def is_git_toplevel(git, directory): 102 """Return true iff the directory is the top level of a Git repository. 103 104 Args: 105 git (string) the git executable 106 107 directory (string) the path into which the repository 108 is expected to be checked out. 109 """ 110 try: 111 toplevel = subprocess.check_output( 112 [git, 'rev-parse', '--show-toplevel'], cwd=directory).strip() 113 return os.path.realpath(bytes(directory, 'utf8')) == os.path.realpath(toplevel) 114 except subprocess.CalledProcessError: 115 return False 116 117 118def status(directory, checkoutable): 119 def truncate(s, length): 120 return s if len(s) <= length else s[:(length - 3)] + '...' 121 dlen = 36 122 directory = truncate(directory, dlen) 123 checkoutable = truncate(checkoutable, 40) 124 sys.stdout.write('%-*s @ %s\n' % (dlen, directory, checkoutable)) 125 126 127def git_checkout_to_directory(git, repo, checkoutable, directory, verbose): 128 """Checkout (and clone if needed) a Git repository. 129 130 Args: 131 git (string) the git executable 132 133 repo (string) the location of the repository, suitable 134 for passing to `git clone`. 135 136 checkoutable (string) a tag, branch, or commit, suitable for 137 passing to `git checkout` 138 139 directory (string) the path into which the repository 140 should be checked out. 141 142 verbose (boolean) 143 144 Raises an exception if any calls to git fail. 145 """ 146 if verbose: 147 status(directory, checkoutable) 148 149 if not os.path.isdir(directory): 150 subprocess.check_call( 151 [git, 'clone', '--quiet', repo, directory]) 152 153 if not is_git_toplevel(git, directory): 154 # if the directory exists, but isn't a git repo, you will modify 155 # the parent repostory, which isn't what you want. 156 sys.stdout.write('%s\n IS NOT TOP-LEVEL GIT DIRECTORY.\n' % directory) 157 return 158 159 # Check to see if this repo is disabled. Quick return. 160 if git_repository_sync_is_disabled(git, directory): 161 sys.stdout.write('%s\n SYNC IS DISABLED.\n' % directory) 162 return 163 164 with open(os.devnull, 'w') as devnull: 165 # If this fails, we will fetch before trying again. Don't spam user 166 # with error infomation. 167 if 0 == subprocess.call([git, 'checkout', '--quiet', checkoutable], 168 cwd=directory, stderr=devnull): 169 # if this succeeds, skip slow `git fetch`. 170 return 171 172 # If the repo has changed, always force use of the correct repo. 173 # If origin already points to repo, this is a quick no-op. 174 subprocess.check_call( 175 [git, 'remote', 'set-url', 'origin', repo], cwd=directory) 176 177 subprocess.check_call([git, 'fetch', '--quiet'], cwd=directory) 178 179 subprocess.check_call([git, 'checkout', '--quiet', checkoutable], cwd=directory) 180 181 182def parse_file_to_dict(path): 183 dictionary = {} 184 contents = open(path).read() 185 # Need to convert Var() to vars[], so that the DEPS is actually Python. Var() 186 # comes from Autoroller using gclient which has a slightly different DEPS 187 # format. 188 contents = re.sub(r"Var\((.*?)\)", r"vars[\1]", contents) 189 exec(contents, dictionary) 190 return dictionary 191 192 193def git_sync_deps(deps_file_path, command_line_os_requests, verbose): 194 """Grab dependencies, with optional platform support. 195 196 Args: 197 deps_file_path (string) Path to the DEPS file. 198 199 command_line_os_requests (list of strings) Can be empty list. 200 List of strings that should each be a key in the deps_os 201 dictionary in the DEPS file. 202 203 Raises git Exceptions. 204 """ 205 git = git_executable() 206 assert git 207 208 deps_file_directory = os.path.dirname(deps_file_path) 209 deps_file = parse_file_to_dict(deps_file_path) 210 dependencies = deps_file['deps'].copy() 211 os_specific_dependencies = deps_file.get('deps_os', dict()) 212 if 'all' in command_line_os_requests: 213 for value in os_specific_dependencies.values(): 214 dependencies.update(value) 215 else: 216 for os_name in command_line_os_requests: 217 # Add OS-specific dependencies 218 if os_name in os_specific_dependencies: 219 dependencies.update(os_specific_dependencies[os_name]) 220 for directory in dependencies: 221 for other_dir in dependencies: 222 if directory.startswith(other_dir + '/'): 223 raise Exception('%r is parent of %r' % (other_dir, directory)) 224 list_of_arg_lists = [] 225 for directory in sorted(dependencies): 226 if '@' in dependencies[directory]: 227 repo, checkoutable = dependencies[directory].rsplit('@', 1) 228 else: 229 raise Exception("please specify commit or tag") 230 231 if not with_clspv and directory == 'third_party/clspv': 232 continue 233 234 if not with_clspv and directory == 'third_party/clspv-llvm': 235 continue 236 237 if not with_dxc and directory == 'third_party/dxc': 238 continue 239 240 if not with_swiftshader and directory == 'third_party/swiftshader': 241 continue 242 243 relative_directory = os.path.join(deps_file_directory, directory) 244 245 list_of_arg_lists.append( 246 (git, repo, checkoutable, relative_directory, verbose)) 247 248 multithread(git_checkout_to_directory, list_of_arg_lists) 249 250 for directory in deps_file.get('recursedeps', []): 251 recursive_path = os.path.join(deps_file_directory, directory, 'DEPS') 252 git_sync_deps(recursive_path, command_line_os_requests, verbose) 253 254 255def multithread(function, list_of_arg_lists): 256 # for args in list_of_arg_lists: 257 # function(*args) 258 # return 259 threads = [] 260 for args in list_of_arg_lists: 261 thread = threading.Thread(None, function, None, args) 262 thread.start() 263 threads.append(thread) 264 for thread in threads: 265 thread.join() 266 267 268def main(argv): 269 global with_clspv 270 global with_dxc 271 global with_swiftshader 272 deps_file_path = os.environ.get('GIT_SYNC_DEPS_PATH', DEFAULT_DEPS_PATH) 273 verbose = not bool(os.environ.get('GIT_SYNC_DEPS_QUIET', False)) 274 275 if '--help' in argv or '-h' in argv: 276 usage(deps_file_path) 277 return 1 278 279 if '--with-clspv' in argv: 280 with_clspv = True 281 282 if '--with-dxc' in argv: 283 with_dxc = True 284 285 if '--with-swiftshader' in argv: 286 with_swiftshader = True 287 288 git_sync_deps(deps_file_path, argv, verbose) 289 # subprocess.check_call( 290 # [sys.executable, 291 # os.path.join(os.path.dirname(deps_file_path), 'bin', 'fetch-gn')]) 292 return 0 293 294 295if __name__ == '__main__': 296 exit(main(sys.argv[1:])) 297