1#!/usr/bin/python3 2# Copyright 2014 Google Inc. 3# 4# Use of this source code is governed by a BSD-style license that can be 5# found in the LICENSE file. 6 7 8"""Parse a DEPS file and git checkout all of the dependencies. 9 10Args: 11 An optional list of deps_os values. 12 13Environment Variables: 14 GIT_EXECUTABLE: path to "git" binary; if unset, will look for git in 15 your default path. 16 17 GIT_SYNC_DEPS_PATH: file to get the dependency list from; if unset, 18 will use the file ../DEPS relative to this script's directory. 19 20 GIT_SYNC_DEPS_QUIET: if set to non-empty string, suppress messages. 21 22Git Config: 23 To disable syncing of a single repository: 24 cd path/to/repository 25 git config sync-deps.disable true 26 27 To re-enable sync: 28 cd path/to/repository 29 git config --unset sync-deps.disable 30""" 31 32 33import os 34import subprocess 35import sys 36import threading 37 38 39def git_executable(): 40 """Find the git executable. 41 42 Returns: 43 A string suitable for passing to subprocess functions, or None. 44 """ 45 envgit = os.environ.get('GIT_EXECUTABLE') 46 searchlist = ['git', 'git.bat'] 47 if envgit: 48 searchlist.insert(0, envgit) 49 with open(os.devnull, 'w') as devnull: 50 for git in searchlist: 51 try: 52 subprocess.call([git, '--version'], stdout=devnull) 53 except (OSError,): 54 continue 55 return git 56 return None 57 58 59DEFAULT_DEPS_PATH = os.path.normpath( 60 os.path.join(os.path.dirname(__file__), os.pardir, 'DEPS')) 61 62 63def usage(deps_file_path = None): 64 sys.stderr.write( 65 'Usage: run to grab dependencies, with optional platform support:\n') 66 sys.stderr.write(' %s %s' % (sys.executable, __file__)) 67 if deps_file_path: 68 parsed_deps = parse_file_to_dict(deps_file_path) 69 if 'deps_os' in parsed_deps: 70 for deps_os in parsed_deps['deps_os']: 71 sys.stderr.write(' [%s]' % deps_os) 72 sys.stderr.write('\n\n') 73 sys.stderr.write(__doc__) 74 75 76def git_repository_sync_is_disabled(git, directory): 77 try: 78 disable = subprocess.check_output( 79 [git, 'config', 'sync-deps.disable'], cwd=directory) 80 return disable.lower().strip() in ['true', '1', 'yes', 'on'] 81 except subprocess.CalledProcessError: 82 return False 83 84 85def is_git_toplevel(git, directory): 86 """Return true iff the directory is the top level of a Git repository. 87 88 Args: 89 git (string) the git executable 90 91 directory (string) the path into which the repository 92 is expected to be checked out. 93 """ 94 try: 95 toplevel = subprocess.check_output( 96 [git, 'rev-parse', '--show-toplevel'], cwd=directory).strip() 97 return (os.path.normcase(os.path.realpath(directory)) == 98 os.path.normcase(os.path.realpath(toplevel.decode()))) 99 except subprocess.CalledProcessError: 100 return False 101 102 103def status(directory, commithash, change): 104 def truncate_beginning(s, length): 105 return s if len(s) <= length else '...' + s[-(length-3):] 106 def truncate_end(s, length): 107 return s if len(s) <= length else s[:(length - 3)] + '...' 108 109 dlen = 36 110 directory = truncate_beginning(directory, dlen) 111 commithash = truncate_end(commithash, 40) 112 symbol = '>' if change else '@' 113 sys.stdout.write('%-*s %s %s\n' % (dlen, directory, symbol, commithash)) 114 115 116def git_checkout_to_directory(git, repo, commithash, directory, verbose): 117 """Checkout (and clone if needed) a Git repository. 118 119 Args: 120 git (string) the git executable 121 122 repo (string) the location of the repository, suitable 123 for passing to `git clone`. 124 125 commithash (string) a commit, suitable for passing to `git checkout` 126 127 directory (string) the path into which the repository 128 should be checked out. 129 130 verbose (boolean) 131 132 Raises an exception if any calls to git fail. 133 """ 134 if not os.path.isdir(directory): 135 subprocess.check_call( 136 [git, 'clone', '--quiet', '--no-checkout', repo, directory]) 137 subprocess.check_call([git, 'checkout', '--quiet', commithash], 138 cwd=directory) 139 if verbose: 140 status(directory, commithash, True) 141 return 142 143 if not is_git_toplevel(git, directory): 144 # if the directory exists, but isn't a git repo, you will modify 145 # the parent repository, which isn't what you want. 146 sys.stdout.write('%s\n IS NOT TOP-LEVEL GIT DIRECTORY.\n' % directory) 147 return 148 149 # Check to see if this repo is disabled. Quick return. 150 if git_repository_sync_is_disabled(git, directory): 151 sys.stdout.write('%s\n SYNC IS DISABLED.\n' % directory) 152 return 153 154 with open(os.devnull, 'w') as devnull: 155 # If this fails, we will fetch before trying again. Don't spam user 156 # with error infomation. 157 if 0 == subprocess.call([git, 'checkout', '--quiet', commithash], 158 cwd=directory, stderr=devnull): 159 # if this succeeds, skip slow `git fetch`. 160 if verbose: 161 status(directory, commithash, False) # Success. 162 return 163 164 # If the repo has changed, always force use of the correct repo. 165 # If origin already points to repo, this is a quick no-op. 166 subprocess.check_call( 167 [git, 'remote', 'set-url', 'origin', repo], cwd=directory) 168 169 subprocess.check_call([git, 'fetch', '--quiet'], cwd=directory) 170 171 subprocess.check_call([git, 'checkout', '--quiet', commithash], cwd=directory) 172 173 if verbose: 174 status(directory, commithash, True) # Success. 175 176 177def parse_file_to_dict(path): 178 dictionary = {} 179 with open(path) as f: 180 exec('def Var(x): return vars[x]\n' + f.read(), dictionary) 181 return dictionary 182 183 184def is_sha1_sum(s): 185 """SHA1 sums are 160 bits, encoded as lowercase hexadecimal.""" 186 return len(s) == 40 and all(c in '0123456789abcdef' for c in s) 187 188 189def git_sync_deps(deps_file_path, command_line_os_requests, verbose): 190 """Grab dependencies, with optional platform support. 191 192 Args: 193 deps_file_path (string) Path to the DEPS file. 194 195 command_line_os_requests (list of strings) Can be empty list. 196 List of strings that should each be a key in the deps_os 197 dictionary in the DEPS file. 198 199 Raises git Exceptions. 200 """ 201 git = git_executable() 202 assert git 203 204 deps_file_directory = os.path.dirname(deps_file_path) 205 deps_file = parse_file_to_dict(deps_file_path) 206 dependencies = deps_file['deps'].copy() 207 os_specific_dependencies = deps_file.get('deps_os', dict()) 208 if 'all' in command_line_os_requests: 209 for value in os_specific_dependencies.itervalues(): 210 dependencies.update(value) 211 else: 212 for os_name in command_line_os_requests: 213 # Add OS-specific dependencies 214 if os_name in os_specific_dependencies: 215 dependencies.update(os_specific_dependencies[os_name]) 216 for directory in dependencies: 217 for other_dir in dependencies: 218 if directory.startswith(other_dir + '/'): 219 raise Exception('%r is parent of %r' % (other_dir, directory)) 220 list_of_arg_lists = [] 221 for directory in sorted(dependencies): 222 if not isinstance(dependencies[directory], str): 223 if verbose: 224 sys.stdout.write( 'Skipping "%s".\n' % directory) 225 continue 226 if '@' in dependencies[directory]: 227 repo, commithash = dependencies[directory].split('@', 1) 228 else: 229 raise Exception("please specify commit") 230 if not is_sha1_sum(commithash): 231 raise Exception("poorly formed commit hash: %r" % commithash) 232 233 relative_directory = os.path.join(deps_file_directory, directory) 234 235 list_of_arg_lists.append( 236 (git, repo, commithash, relative_directory, verbose)) 237 238 multithread(git_checkout_to_directory, list_of_arg_lists) 239 240 241def multithread(function, list_of_arg_lists): 242 anything_failed = False 243 threads = [] 244 def hook(args): 245 nonlocal anything_failed 246 anything_failed = True 247 threading.excepthook = hook 248 for args in list_of_arg_lists: 249 thread = threading.Thread(None, function, None, args) 250 thread.start() 251 threads.append(thread) 252 for thread in threads: 253 thread.join() 254 if anything_failed: 255 raise Exception("Thread failure detected") 256 257 258def main(argv): 259 deps_file_path = os.environ.get('GIT_SYNC_DEPS_PATH', DEFAULT_DEPS_PATH) 260 verbose = not bool(os.environ.get('GIT_SYNC_DEPS_QUIET', False)) 261 262 if '--help' in argv or '-h' in argv: 263 usage(deps_file_path) 264 return 1 265 266 git_sync_deps(deps_file_path, argv, verbose) 267 subprocess.check_call( 268 [sys.executable, 269 os.path.join(os.path.dirname(deps_file_path), 'bin', 'fetch-gn')]) 270 subprocess.check_call( 271 [sys.executable, 272 os.path.join(os.path.dirname(deps_file_path), 'bin', 'activate-emsdk')]) 273 return 0 274 275 276if __name__ == '__main__': 277 exit(main(sys.argv[1:])) 278