1#!/usr/bin/env python 2# Copyright 2014 Google Inc. 3# 4# Use of this source code is governed by a BSD-style license that can be 5# found in the LICENSE file. 6 7 8"""Parse a DEPS file and git checkout all of the dependencies. 9 10Args: 11 An optional list of deps_os values. 12 13Environment Variables: 14 GIT_EXECUTABLE: path to "git" binary; if unset, will look for git in 15 your default path. 16 17 GIT_SYNC_DEPS_PATH: file to get the dependency list from; if unset, 18 will use the file ../DEPS relative to this script's directory. 19 20 GIT_SYNC_DEPS_QUIET: if set to non-empty string, suppress messages. 21 22Git Config: 23 To disable syncing of a single repository: 24 cd path/to/repository 25 git config sync-deps.disable true 26 27 To re-enable sync: 28 cd path/to/repository 29 git config --unset sync-deps.disable 30""" 31 32 33import os 34import subprocess 35import sys 36import threading 37 38 39def git_executable(): 40 """Find the git executable. 41 42 Returns: 43 A string suitable for passing to subprocess functions, or None. 44 """ 45 envgit = os.environ.get('GIT_EXECUTABLE') 46 searchlist = ['git'] 47 if envgit: 48 searchlist.insert(0, envgit) 49 with open(os.devnull, 'w') as devnull: 50 for git in searchlist: 51 try: 52 subprocess.call([git, '--version'], stdout=devnull) 53 except (OSError,): 54 continue 55 return git 56 return None 57 58 59DEFAULT_DEPS_PATH = os.path.normpath( 60 os.path.join(os.path.dirname(__file__), os.pardir, 'DEPS')) 61 62 63def usage(deps_file_path = None): 64 sys.stderr.write( 65 'Usage: run to grab dependencies, with optional platform support:\n') 66 sys.stderr.write(' %s %s' % (sys.executable, __file__)) 67 if deps_file_path: 68 parsed_deps = parse_file_to_dict(deps_file_path) 69 if 'deps_os' in parsed_deps: 70 for deps_os in parsed_deps['deps_os']: 71 sys.stderr.write(' [%s]' % deps_os) 72 sys.stderr.write('\n\n') 73 sys.stderr.write(__doc__) 74 75 76def git_repository_sync_is_disabled(git, directory): 77 try: 78 disable = subprocess.check_output( 79 [git, 'config', 'sync-deps.disable'], cwd=directory) 80 return disable.lower().strip() in ['true', '1', 'yes', 'on'] 81 except subprocess.CalledProcessError: 82 return False 83 84 85def is_git_toplevel(git, directory): 86 """Return true iff the directory is the top level of a Git repository. 87 88 Args: 89 git (string) the git executable 90 91 directory (string) the path into which the repository 92 is expected to be checked out. 93 """ 94 try: 95 toplevel = subprocess.check_output( 96 [git, 'rev-parse', '--show-toplevel'], cwd=directory).strip() 97 return os.path.realpath(directory) == os.path.realpath(toplevel.decode()) 98 except subprocess.CalledProcessError: 99 return False 100 101 102def status(directory, commithash, change): 103 def truncate_beginning(s, length): 104 return s if len(s) <= length else '...' + s[-(length-3):] 105 def truncate_end(s, length): 106 return s if len(s) <= length else s[:(length - 3)] + '...' 107 108 dlen = 36 109 directory = truncate_beginning(directory, dlen) 110 commithash = truncate_end(commithash, 40) 111 symbol = '>' if change else '@' 112 sys.stdout.write('%-*s %s %s\n' % (dlen, directory, symbol, commithash)) 113 114 115def git_checkout_to_directory(git, repo, commithash, directory, verbose): 116 """Checkout (and clone if needed) a Git repository. 117 118 Args: 119 git (string) the git executable 120 121 repo (string) the location of the repository, suitable 122 for passing to `git clone`. 123 124 commithash (string) a commit, suitable for passing to `git checkout` 125 126 directory (string) the path into which the repository 127 should be checked out. 128 129 verbose (boolean) 130 131 Raises an exception if any calls to git fail. 132 """ 133 if not os.path.isdir(directory): 134 subprocess.check_call( 135 [git, 'clone', '--quiet', '--no-checkout', repo, directory]) 136 subprocess.check_call([git, 'checkout', '--quiet', commithash], 137 cwd=directory) 138 if verbose: 139 status(directory, commithash, True) 140 return 141 142 if not is_git_toplevel(git, directory): 143 # if the directory exists, but isn't a git repo, you will modify 144 # the parent repostory, which isn't what you want. 145 sys.stdout.write('%s\n IS NOT TOP-LEVEL GIT DIRECTORY.\n' % directory) 146 return 147 148 # Check to see if this repo is disabled. Quick return. 149 if git_repository_sync_is_disabled(git, directory): 150 sys.stdout.write('%s\n SYNC IS DISABLED.\n' % directory) 151 return 152 153 with open(os.devnull, 'w') as devnull: 154 # If this fails, we will fetch before trying again. Don't spam user 155 # with error infomation. 156 if 0 == subprocess.call([git, 'checkout', '--quiet', commithash], 157 cwd=directory, stderr=devnull): 158 # if this succeeds, skip slow `git fetch`. 159 if verbose: 160 status(directory, commithash, False) # Success. 161 return 162 163 # If the repo has changed, always force use of the correct repo. 164 # If origin already points to repo, this is a quick no-op. 165 subprocess.check_call( 166 [git, 'remote', 'set-url', 'origin', repo], cwd=directory) 167 168 subprocess.check_call([git, 'fetch', '--quiet'], cwd=directory) 169 170 subprocess.check_call([git, 'checkout', '--quiet', commithash], cwd=directory) 171 172 if verbose: 173 status(directory, commithash, True) # Success. 174 175 176def parse_file_to_dict(path): 177 dictionary = {} 178 with open(path) as f: 179 exec('def Var(x): return vars[x]\n' + f.read(), dictionary) 180 return dictionary 181 182 183def is_sha1_sum(s): 184 """SHA1 sums are 160 bits, encoded as lowercase hexadecimal.""" 185 return len(s) == 40 and all(c in '0123456789abcdef' for c in s) 186 187 188def git_sync_deps(deps_file_path, command_line_os_requests, verbose): 189 """Grab dependencies, with optional platform support. 190 191 Args: 192 deps_file_path (string) Path to the DEPS file. 193 194 command_line_os_requests (list of strings) Can be empty list. 195 List of strings that should each be a key in the deps_os 196 dictionary in the DEPS file. 197 198 Raises git Exceptions. 199 """ 200 git = git_executable() 201 assert git 202 203 deps_file_directory = os.path.dirname(deps_file_path) 204 deps_file = parse_file_to_dict(deps_file_path) 205 dependencies = deps_file['deps'].copy() 206 os_specific_dependencies = deps_file.get('deps_os', dict()) 207 if 'all' in command_line_os_requests: 208 for value in os_specific_dependencies.itervalues(): 209 dependencies.update(value) 210 else: 211 for os_name in command_line_os_requests: 212 # Add OS-specific dependencies 213 if os_name in os_specific_dependencies: 214 dependencies.update(os_specific_dependencies[os_name]) 215 for directory in dependencies: 216 for other_dir in dependencies: 217 if directory.startswith(other_dir + '/'): 218 raise Exception('%r is parent of %r' % (other_dir, directory)) 219 list_of_arg_lists = [] 220 for directory in sorted(dependencies): 221 if not isinstance(dependencies[directory], str): 222 if verbose: 223 sys.stdout.write( 'Skipping "%s".\n' % directory) 224 continue 225 if '@' in dependencies[directory]: 226 repo, commithash = dependencies[directory].split('@', 1) 227 else: 228 raise Exception("please specify commit") 229 if not is_sha1_sum(commithash): 230 raise Exception("poorly formed commit hash: %r" % commithash) 231 232 relative_directory = os.path.join(deps_file_directory, directory) 233 234 list_of_arg_lists.append( 235 (git, repo, commithash, relative_directory, verbose)) 236 237 multithread(git_checkout_to_directory, list_of_arg_lists) 238 239 240def multithread(function, list_of_arg_lists): 241 # for args in list_of_arg_lists: 242 # function(*args) 243 # return 244 threads = [] 245 for args in list_of_arg_lists: 246 thread = threading.Thread(None, function, None, args) 247 thread.start() 248 threads.append(thread) 249 for thread in threads: 250 thread.join() 251 252 253def main(argv): 254 deps_file_path = os.environ.get('GIT_SYNC_DEPS_PATH', DEFAULT_DEPS_PATH) 255 verbose = not bool(os.environ.get('GIT_SYNC_DEPS_QUIET', False)) 256 257 if '--help' in argv or '-h' in argv: 258 usage(deps_file_path) 259 return 1 260 261 git_sync_deps(deps_file_path, argv, verbose) 262 subprocess.check_call( 263 [sys.executable, 264 os.path.join(os.path.dirname(deps_file_path), 'bin', 'fetch-gn')]) 265 return 0 266 267 268if __name__ == '__main__': 269 exit(main(sys.argv[1:])) 270