1 #!/usr/bin/env python 2 # Copyright 2014 Google Inc. 3 # 4 # Use of this source code is governed by a BSD-style license that can be 5 # found in the LICENSE file. 6 7 8 """Parse a DEPS file and git checkout all of the dependencies. 9 10 Args: 11 An optional list of deps_os values. 12 13 Environment Variables: 14 GIT_EXECUTABLE: path to "git" binary; if unset, will look for git in 15 your default path. 16 17 GIT_SYNC_DEPS_PATH: file to get the dependency list from; if unset, 18 will use the file ../DEPS relative to this script's directory. 19 20 GIT_SYNC_DEPS_QUIET: if set to non-empty string, suppress messages. 21 22 Git Config: 23 To disable syncing of a single repository: 24 cd path/to/repository 25 git config sync-deps.disable true 26 27 To re-enable sync: 28 cd path/to/repository 29 git config --unset sync-deps.disable 30 """ 31 32 33 import os 34 import subprocess 35 import sys 36 import threading 37 38 39 def git_executable(): 40 """Find the git executable. 41 42 Returns: 43 A string suitable for passing to subprocess functions, or None. 44 """ 45 envgit = os.environ.get('GIT_EXECUTABLE') 46 searchlist = ['git'] 47 if envgit: 48 searchlist.insert(0, envgit) 49 with open(os.devnull, 'w') as devnull: 50 for git in searchlist: 51 try: 52 subprocess.call([git, '--version'], stdout=devnull) 53 except (OSError,): 54 continue 55 return git 56 return None 57 58 59 DEFAULT_DEPS_PATH = os.path.normpath( 60 os.path.join(os.path.dirname(__file__), os.pardir, 'DEPS')) 61 62 63 def usage(deps_file_path = None): 64 sys.stderr.write( 65 'Usage: run to grab dependencies, with optional platform support:\n') 66 sys.stderr.write(' %s %s' % (sys.executable, __file__)) 67 if deps_file_path: 68 parsed_deps = parse_file_to_dict(deps_file_path) 69 if 'deps_os' in parsed_deps: 70 for deps_os in parsed_deps['deps_os']: 71 sys.stderr.write(' [%s]' % deps_os) 72 sys.stderr.write('\n\n') 73 sys.stderr.write(__doc__) 74 75 76 def git_repository_sync_is_disabled(git, directory): 77 try: 78 disable = subprocess.check_output( 79 [git, 'config', 'sync-deps.disable'], cwd=directory) 80 return disable.lower().strip() in ['true', '1', 'yes', 'on'] 81 except subprocess.CalledProcessError: 82 return False 83 84 85 def is_git_toplevel(git, directory): 86 """Return true iff the directory is the top level of a Git repository. 87 88 Args: 89 git (string) the git executable 90 91 directory (string) the path into which the repository 92 is expected to be checked out. 93 """ 94 try: 95 toplevel = subprocess.check_output( 96 [git, 'rev-parse', '--show-toplevel'], cwd=directory).strip() 97 return os.path.realpath(directory) == os.path.realpath(toplevel) 98 except subprocess.CalledProcessError: 99 return False 100 101 102 def status(directory, checkoutable): 103 def truncate(s, length): 104 return s if len(s) <= length else s[:(length - 3)] + '...' 105 dlen = 36 106 directory = truncate(directory, dlen) 107 checkoutable = truncate(checkoutable, 40) 108 sys.stdout.write('%-*s @ %s\n' % (dlen, directory, checkoutable)) 109 110 111 def git_checkout_to_directory(git, repo, checkoutable, directory, verbose): 112 """Checkout (and clone if needed) a Git repository. 113 114 Args: 115 git (string) the git executable 116 117 repo (string) the location of the repository, suitable 118 for passing to `git clone`. 119 120 checkoutable (string) a tag, branch, or commit, suitable for 121 passing to `git checkout` 122 123 directory (string) the path into which the repository 124 should be checked out. 125 126 verbose (boolean) 127 128 Raises an exception if any calls to git fail. 129 """ 130 if not os.path.isdir(directory): 131 subprocess.check_call( 132 [git, 'clone', '--quiet', repo, directory]) 133 134 if not is_git_toplevel(git, directory): 135 # if the directory exists, but isn't a git repo, you will modify 136 # the parent repostory, which isn't what you want. 137 sys.stdout.write('%s\n IS NOT TOP-LEVEL GIT DIRECTORY.\n' % directory) 138 return 139 140 # Check to see if this repo is disabled. Quick return. 141 if git_repository_sync_is_disabled(git, directory): 142 sys.stdout.write('%s\n SYNC IS DISABLED.\n' % directory) 143 return 144 145 with open(os.devnull, 'w') as devnull: 146 # If this fails, we will fetch before trying again. Don't spam user 147 # with error infomation. 148 if 0 == subprocess.call([git, 'checkout', '--quiet', checkoutable], 149 cwd=directory, stderr=devnull): 150 # if this succeeds, skip slow `git fetch`. 151 if verbose: 152 status(directory, checkoutable) # Success. 153 return 154 155 # If the repo has changed, always force use of the correct repo. 156 # If origin already points to repo, this is a quick no-op. 157 subprocess.check_call( 158 [git, 'remote', 'set-url', 'origin', repo], cwd=directory) 159 160 subprocess.check_call([git, 'fetch', '--quiet'], cwd=directory) 161 162 subprocess.check_call([git, 'checkout', '--quiet', checkoutable], cwd=directory) 163 164 if verbose: 165 status(directory, checkoutable) # Success. 166 167 168 def parse_file_to_dict(path): 169 dictionary = {} 170 execfile(path, dictionary) 171 return dictionary 172 173 174 def git_sync_deps(deps_file_path, command_line_os_requests, verbose): 175 """Grab dependencies, with optional platform support. 176 177 Args: 178 deps_file_path (string) Path to the DEPS file. 179 180 command_line_os_requests (list of strings) Can be empty list. 181 List of strings that should each be a key in the deps_os 182 dictionary in the DEPS file. 183 184 Raises git Exceptions. 185 """ 186 git = git_executable() 187 assert git 188 189 deps_file_directory = os.path.dirname(deps_file_path) 190 deps_file = parse_file_to_dict(deps_file_path) 191 dependencies = deps_file['deps'].copy() 192 os_specific_dependencies = deps_file.get('deps_os', dict()) 193 if 'all' in command_line_os_requests: 194 for value in os_specific_dependencies.itervalues(): 195 dependencies.update(value) 196 else: 197 for os_name in command_line_os_requests: 198 # Add OS-specific dependencies 199 if os_name in os_specific_dependencies: 200 dependencies.update(os_specific_dependencies[os_name]) 201 for directory in dependencies: 202 for other_dir in dependencies: 203 if directory.startswith(other_dir + '/'): 204 raise Exception('%r is parent of %r' % (other_dir, directory)) 205 list_of_arg_lists = [] 206 for directory in sorted(dependencies): 207 if not isinstance(dependencies[directory], basestring): 208 if verbose: 209 print 'Skipping "%s".' % directory 210 continue 211 if '@' in dependencies[directory]: 212 repo, checkoutable = dependencies[directory].split('@', 1) 213 else: 214 raise Exception("please specify commit or tag") 215 216 relative_directory = os.path.join(deps_file_directory, directory) 217 218 list_of_arg_lists.append( 219 (git, repo, checkoutable, relative_directory, verbose)) 220 221 multithread(git_checkout_to_directory, list_of_arg_lists) 222 223 224 def multithread(function, list_of_arg_lists): 225 # for args in list_of_arg_lists: 226 # function(*args) 227 # return 228 threads = [] 229 for args in list_of_arg_lists: 230 thread = threading.Thread(None, function, None, args) 231 thread.start() 232 threads.append(thread) 233 for thread in threads: 234 thread.join() 235 236 237 def main(argv): 238 deps_file_path = os.environ.get('GIT_SYNC_DEPS_PATH', DEFAULT_DEPS_PATH) 239 verbose = not bool(os.environ.get('GIT_SYNC_DEPS_QUIET', False)) 240 241 if '--help' in argv or '-h' in argv: 242 usage(deps_file_path) 243 return 1 244 245 git_sync_deps(deps_file_path, argv, verbose) 246 subprocess.check_call( 247 [sys.executable, 248 os.path.join(os.path.dirname(deps_file_path), 'bin', 'fetch-gn')]) 249 return 0 250 251 252 if __name__ == '__main__': 253 exit(main(sys.argv[1:])) 254