• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/env python3
2# Copyright 2014 Google Inc.
3#
4# Redistribution and use in source and binary forms, with or without
5# modification, are permitted provided that the following conditions are
6# met:
7#
8#    * Redistributions of source code must retain the above copyright
9# notice, this list of conditions and the following disclaimer.
10#    * Redistributions in binary form must reproduce the above
11# copyright notice, this list of conditions and the following disclaimer
12# in the documentation and/or other materials provided with the
13# distribution.
14#    * Neither the name of Google Inc. nor the names of its
15# contributors may be used to endorse or promote products derived from
16# this software without specific prior written permission.
17#
18# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29
30"""Parse a DEPS file and git checkout all of the dependencies.
31
32Args:
33  An optional list of deps_os values.
34
35Environment Variables:
36  GIT_EXECUTABLE: path to "git" binary; if unset, will look for one of
37  ['git', 'git.exe', 'git.bat'] in your default path.
38
39  GIT_SYNC_DEPS_PATH: file to get the dependency list from; if unset,
40  will use the file ../DEPS relative to this script's directory.
41
42  GIT_SYNC_DEPS_QUIET: if set to non-empty string, suppress messages.
43
44Git Config:
45  To disable syncing of a single repository:
46      cd path/to/repository
47      git config sync-deps.disable true
48
49  To re-enable sync:
50      cd path/to/repository
51      git config --unset sync-deps.disable
52"""
53
54
55import os
56import re
57import subprocess
58import sys
59import threading
60from builtins import bytes
61
62
63def git_executable():
64  """Find the git executable.
65
66  Returns:
67      A string suitable for passing to subprocess functions, or None.
68  """
69  envgit = os.environ.get('GIT_EXECUTABLE')
70  searchlist = ['git', 'git.exe', 'git.bat']
71  if envgit:
72    searchlist.insert(0, envgit)
73  with open(os.devnull, 'w') as devnull:
74    for git in searchlist:
75      try:
76        subprocess.call([git, '--version'], stdout=devnull)
77      except (OSError,):
78        continue
79      return git
80  return None
81
82
83DEFAULT_DEPS_PATH = os.path.normpath(
84  os.path.join(os.path.dirname(__file__), os.pardir, 'DEPS'))
85
86
87def usage(deps_file_path = None):
88  sys.stderr.write(
89    'Usage: run to grab dependencies, with optional platform support:\n')
90  sys.stderr.write('  %s %s' % (sys.executable, __file__))
91  if deps_file_path:
92    parsed_deps = parse_file_to_dict(deps_file_path)
93    if 'deps_os' in parsed_deps:
94      for deps_os in parsed_deps['deps_os']:
95        sys.stderr.write(' [%s]' % deps_os)
96  sys.stderr.write('\n\n')
97  sys.stderr.write(__doc__)
98
99
100def git_repository_sync_is_disabled(git, directory):
101  try:
102    disable = subprocess.check_output(
103      [git, 'config', 'sync-deps.disable'], cwd=directory)
104    return disable.lower().strip() in ['true', '1', 'yes', 'on']
105  except subprocess.CalledProcessError:
106    return False
107
108
109def is_git_toplevel(git, directory):
110  """Return true iff the directory is the top level of a Git repository.
111
112  Args:
113    git (string) the git executable
114
115    directory (string) the path into which the repository
116              is expected to be checked out.
117  """
118  try:
119    toplevel = subprocess.check_output(
120      [git, 'rev-parse', '--show-toplevel'], cwd=directory).strip()
121    return os.path.realpath(bytes(directory, 'utf8')) == os.path.realpath(toplevel)
122  except subprocess.CalledProcessError:
123    return False
124
125
126def status(directory, checkoutable):
127  def truncate(s, length):
128    return s if len(s) <= length else s[:(length - 3)] + '...'
129  dlen = 36
130  directory = truncate(directory, dlen)
131  checkoutable = truncate(checkoutable, 40)
132  sys.stdout.write('%-*s @ %s\n' % (dlen, directory, checkoutable))
133
134
135def git_checkout_to_directory(git, repo, checkoutable, directory, verbose):
136  """Checkout (and clone if needed) a Git repository.
137
138  Args:
139    git (string) the git executable
140
141    repo (string) the location of the repository, suitable
142         for passing to `git clone`.
143
144    checkoutable (string) a tag, branch, or commit, suitable for
145                 passing to `git checkout`
146
147    directory (string) the path into which the repository
148              should be checked out.
149
150    verbose (boolean)
151
152  Raises an exception if any calls to git fail.
153  """
154  if not os.path.isdir(directory):
155    subprocess.check_call(
156      [git, 'clone', '--quiet', repo, directory])
157
158  if not is_git_toplevel(git, directory):
159    # if the directory exists, but isn't a git repo, you will modify
160    # the parent repostory, which isn't what you want.
161    sys.stdout.write('%s\n  IS NOT TOP-LEVEL GIT DIRECTORY.\n' % directory)
162    return
163
164  # Check to see if this repo is disabled.  Quick return.
165  if git_repository_sync_is_disabled(git, directory):
166    sys.stdout.write('%s\n  SYNC IS DISABLED.\n' % directory)
167    return
168
169  with open(os.devnull, 'w') as devnull:
170    # If this fails, we will fetch before trying again.  Don't spam user
171    # with error infomation.
172    if 0 == subprocess.call([git, 'checkout', '--quiet', checkoutable],
173                            cwd=directory, stderr=devnull):
174      # if this succeeds, skip slow `git fetch`.
175      if verbose:
176        status(directory, checkoutable)  # Success.
177      return
178
179  # If the repo has changed, always force use of the correct repo.
180  # If origin already points to repo, this is a quick no-op.
181  subprocess.check_call(
182      [git, 'remote', 'set-url', 'origin', repo], cwd=directory)
183
184  subprocess.check_call([git, 'fetch', '--quiet'], cwd=directory)
185
186  subprocess.check_call([git, 'checkout', '--quiet', checkoutable], cwd=directory)
187
188  if verbose:
189    status(directory, checkoutable)  # Success.
190
191
192def parse_file_to_dict(path):
193  dictionary = {}
194  contents = open(path).read()
195  # Need to convert Var() to vars[], so that the DEPS is actually Python. Var()
196  # comes from Autoroller using gclient which has a slightly different DEPS
197  # format.
198  contents = re.sub(r"Var\((.*?)\)", r"vars[\1]", contents)
199  exec(contents, dictionary)
200  return dictionary
201
202
203def git_sync_deps(deps_file_path, command_line_os_requests, verbose):
204  """Grab dependencies, with optional platform support.
205
206  Args:
207    deps_file_path (string) Path to the DEPS file.
208
209    command_line_os_requests (list of strings) Can be empty list.
210        List of strings that should each be a key in the deps_os
211        dictionary in the DEPS file.
212
213  Raises git Exceptions.
214  """
215  git = git_executable()
216  assert git
217
218  deps_file_directory = os.path.dirname(deps_file_path)
219  deps_file = parse_file_to_dict(deps_file_path)
220  dependencies = deps_file['deps'].copy()
221  os_specific_dependencies = deps_file.get('deps_os', dict())
222  if 'all' in command_line_os_requests:
223    for value in list(os_specific_dependencies.values()):
224      dependencies.update(value)
225  else:
226    for os_name in command_line_os_requests:
227      # Add OS-specific dependencies
228      if os_name in os_specific_dependencies:
229        dependencies.update(os_specific_dependencies[os_name])
230  for directory in dependencies:
231    for other_dir in dependencies:
232      if directory.startswith(other_dir + '/'):
233        raise Exception('%r is parent of %r' % (other_dir, directory))
234  list_of_arg_lists = []
235  for directory in sorted(dependencies):
236    if '@' in dependencies[directory]:
237      repo, checkoutable = dependencies[directory].split('@', 1)
238    else:
239      raise Exception("please specify commit or tag")
240
241    relative_directory = os.path.join(deps_file_directory, directory)
242
243    list_of_arg_lists.append(
244      (git, repo, checkoutable, relative_directory, verbose))
245
246  multithread(git_checkout_to_directory, list_of_arg_lists)
247
248  for directory in deps_file.get('recursedeps', []):
249    recursive_path = os.path.join(deps_file_directory, directory, 'DEPS')
250    git_sync_deps(recursive_path, command_line_os_requests, verbose)
251
252
253def multithread(function, list_of_arg_lists):
254  # for args in list_of_arg_lists:
255  #   function(*args)
256  # return
257  threads = []
258  for args in list_of_arg_lists:
259    thread = threading.Thread(None, function, None, args)
260    thread.start()
261    threads.append(thread)
262  for thread in threads:
263    thread.join()
264
265
266def main(argv):
267  deps_file_path = os.environ.get('GIT_SYNC_DEPS_PATH', DEFAULT_DEPS_PATH)
268  verbose = not bool(os.environ.get('GIT_SYNC_DEPS_QUIET', False))
269
270  if '--help' in argv or '-h' in argv:
271    usage(deps_file_path)
272    return 1
273
274  git_sync_deps(deps_file_path, argv, verbose)
275  # subprocess.check_call(
276  #     [sys.executable,
277  #      os.path.join(os.path.dirname(deps_file_path), 'bin', 'fetch-gn')])
278  return 0
279
280
281if __name__ == '__main__':
282  exit(main(sys.argv[1:]))
283