• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/env python2
2#
3# Copyright 2010 Google Inc. All Rights Reserved.
4"""Module for transferring files between various types of repositories."""
5
6from __future__ import print_function
7
8__author__ = 'asharif@google.com (Ahmad Sharif)'
9
10import argparse
11import datetime
12import json
13import os
14import re
15import socket
16import sys
17import tempfile
18
19from automation.clients.helper import perforce
20from cros_utils import command_executer
21from cros_utils import logger
22from cros_utils import misc
23
24# pylint: disable=anomalous-backslash-in-string
25
26
27def GetCanonicalMappings(mappings):
28  canonical_mappings = []
29  for mapping in mappings:
30    remote_path, local_path = mapping.split()
31    if local_path.endswith('/') and not remote_path.endswith('/'):
32      local_path = os.path.join(local_path, os.path.basename(remote_path))
33    remote_path = remote_path.lstrip('/').split('/', 1)[1]
34    canonical_mappings.append(perforce.PathMapping(remote_path, local_path))
35  return canonical_mappings
36
37
38def SplitMapping(mapping):
39  parts = mapping.split()
40  assert len(parts) <= 2, 'Mapping %s invalid' % mapping
41  remote_path = parts[0]
42  if len(parts) == 2:
43    local_path = parts[1]
44  else:
45    local_path = '.'
46  return remote_path, local_path
47
48
49class Repo(object):
50  """Basic repository base class."""
51
52  def __init__(self, no_create_tmp_dir=False):
53    self.repo_type = None
54    self.address = None
55    self.mappings = None
56    self.revision = None
57    self.ignores = ['.gitignore', '.p4config', 'README.google']
58    if no_create_tmp_dir:
59      self._root_dir = None
60    else:
61      self._root_dir = tempfile.mkdtemp()
62    self._ce = command_executer.GetCommandExecuter()
63    self._logger = logger.GetLogger()
64
65  def PullSources(self):
66    """Pull all sources into an internal dir."""
67    pass
68
69  def SetupForPush(self):
70    """Setup a repository for pushing later."""
71    pass
72
73  def PushSources(self, commit_message=None, dry_run=False, message_file=None):
74    """Push to the external repo with the commit message."""
75    pass
76
77  def _RsyncExcludingRepoDirs(self, source_dir, dest_dir):
78    for f in os.listdir(source_dir):
79      if f in ['.git', '.svn', '.p4config']:
80        continue
81      dest_file = os.path.join(dest_dir, f)
82      source_file = os.path.join(source_dir, f)
83      if os.path.exists(dest_file):
84        command = 'rm -rf %s' % dest_file
85        self._ce.RunCommand(command)
86      command = 'rsync -a %s %s' % (source_file, dest_dir)
87      self._ce.RunCommand(command)
88    return 0
89
90  def MapSources(self, dest_dir):
91    """Copy sources from the internal dir to root_dir."""
92    return self._RsyncExcludingRepoDirs(self._root_dir, dest_dir)
93
94  def GetRoot(self):
95    return self._root_dir
96
97  def SetRoot(self, directory):
98    self._root_dir = directory
99
100  def CleanupRoot(self):
101    command = 'rm -rf %s' % self._root_dir
102    return self._ce.RunCommand(command)
103
104  def __str__(self):
105    return '\n'.join(
106        str(s) for s in [self.repo_type, self.address, self.mappings])
107
108
109# Note - this type of repo is used only for "readonly", in other words, this
110# only serves as a incoming repo.
111class FileRepo(Repo):
112  """Class for file repositories."""
113
114  def __init__(self, address, ignores=None):
115    Repo.__init__(self, no_create_tmp_dir=True)
116    self.repo_type = 'file'
117    self.address = address
118    self.mappings = None
119    self.branch = None
120    self.revision = '{0} (as of "{1}")'.format(address, datetime.datetime.now())
121    self.gerrit = None
122    self._root_dir = self.address
123    if ignores:
124      self.ignores += ignores
125
126  def CleanupRoot(self):
127    """Override to prevent deletion."""
128    pass
129
130
131class P4Repo(Repo):
132  """Class for P4 repositories."""
133
134  def __init__(self, address, mappings, revision=None):
135    Repo.__init__(self)
136    self.repo_type = 'p4'
137    self.address = address
138    self.mappings = mappings
139    self.revision = revision
140
141  def PullSources(self):
142    client_name = socket.gethostname()
143    client_name += tempfile.mkstemp()[1].replace('/', '-')
144    mappings = self.mappings
145    p4view = perforce.View('depot2', GetCanonicalMappings(mappings))
146    p4client = perforce.CommandsFactory(
147        self._root_dir, p4view, name=client_name)
148    command = p4client.SetupAndDo(p4client.Sync(self.revision))
149    ret = self._ce.RunCommand(command)
150    assert ret == 0, 'Could not setup client.'
151    command = p4client.InCheckoutDir(p4client.SaveCurrentCLNumber())
152    ret, o, _ = self._ce.RunCommandWOutput(command)
153    assert ret == 0, 'Could not get version from client.'
154    self.revision = re.search('^\d+$', o.strip(), re.MULTILINE).group(0)
155    command = p4client.InCheckoutDir(p4client.Remove())
156    ret = self._ce.RunCommand(command)
157    assert ret == 0, 'Could not delete client.'
158    return 0
159
160
161class SvnRepo(Repo):
162  """Class for svn repositories."""
163
164  def __init__(self, address, mappings):
165    Repo.__init__(self)
166    self.repo_type = 'svn'
167    self.address = address
168    self.mappings = mappings
169
170  def PullSources(self):
171    with misc.WorkingDirectory(self._root_dir):
172      for mapping in self.mappings:
173        remote_path, local_path = SplitMapping(mapping)
174        command = 'svn co %s/%s %s' % (self.address, remote_path, local_path)
175      ret = self._ce.RunCommand(command)
176      if ret:
177        return ret
178
179      self.revision = ''
180      for mapping in self.mappings:
181        remote_path, local_path = SplitMapping(mapping)
182        command = 'cd %s && svnversion -c .' % (local_path)
183        ret, o, _ = self._ce.RunCommandWOutput(command)
184        self.revision += o.strip().split(':')[-1]
185        if ret:
186          return ret
187    return 0
188
189
190class GitRepo(Repo):
191  """Class for git repositories."""
192
193  def __init__(self, address, branch, mappings=None, ignores=None, gerrit=None):
194    Repo.__init__(self)
195    self.repo_type = 'git'
196    self.address = address
197    self.branch = branch or 'master'
198    if ignores:
199      self.ignores += ignores
200    self.mappings = mappings
201    self.gerrit = gerrit
202
203  def _CloneSources(self):
204    with misc.WorkingDirectory(self._root_dir):
205      command = 'git clone %s .' % (self.address)
206      return self._ce.RunCommand(command)
207
208  def PullSources(self):
209    with misc.WorkingDirectory(self._root_dir):
210      ret = self._CloneSources()
211      if ret:
212        return ret
213
214      command = 'git checkout %s' % self.branch
215      ret = self._ce.RunCommand(command)
216      if ret:
217        return ret
218
219      command = 'git describe --always'
220      ret, o, _ = self._ce.RunCommandWOutput(command)
221      self.revision = o.strip()
222      return ret
223
224  def SetupForPush(self):
225    with misc.WorkingDirectory(self._root_dir):
226      ret = self._CloneSources()
227      logger.GetLogger().LogFatalIf(
228          ret, 'Could not clone git repo %s.' % self.address)
229
230      command = 'git branch -a | grep -wq %s' % self.branch
231      ret = self._ce.RunCommand(command)
232
233      if ret == 0:
234        if self.branch != 'master':
235          command = ('git branch --track %s remotes/origin/%s' % (self.branch,
236                                                                  self.branch))
237        else:
238          command = 'pwd'
239        command += '&& git checkout %s' % self.branch
240      else:
241        command = 'git symbolic-ref HEAD refs/heads/%s' % self.branch
242      command += '&& rm -rf *'
243      ret = self._ce.RunCommand(command)
244      return ret
245
246  def CommitLocally(self, commit_message=None, message_file=None):
247    with misc.WorkingDirectory(self._root_dir):
248      command = 'pwd'
249      for ignore in self.ignores:
250        command += '&& echo \'%s\' >> .git/info/exclude' % ignore
251      command += '&& git add -Av .'
252      if message_file:
253        message_arg = '-F %s' % message_file
254      elif commit_message:
255        message_arg = '-m \'%s\'' % commit_message
256      else:
257        raise RuntimeError('No commit message given!')
258      command += '&& git commit -v %s' % message_arg
259      return self._ce.RunCommand(command)
260
261  def PushSources(self, commit_message=None, dry_run=False, message_file=None):
262    ret = self.CommitLocally(commit_message, message_file)
263    if ret:
264      return ret
265    push_args = ''
266    if dry_run:
267      push_args += ' -n '
268    with misc.WorkingDirectory(self._root_dir):
269      if self.gerrit:
270        label = 'somelabel'
271        command = 'git remote add %s %s' % (label, self.address)
272        command += ('&& git push %s %s HEAD:refs/for/master' % (push_args,
273                                                                label))
274      else:
275        command = 'git push -v %s origin %s:%s' % (push_args, self.branch,
276                                                   self.branch)
277      ret = self._ce.RunCommand(command)
278    return ret
279
280  def MapSources(self, root_dir):
281    if not self.mappings:
282      self._RsyncExcludingRepoDirs(self._root_dir, root_dir)
283      return
284    with misc.WorkingDirectory(self._root_dir):
285      for mapping in self.mappings:
286        remote_path, local_path = SplitMapping(mapping)
287        remote_path.rstrip('...')
288        local_path.rstrip('...')
289        full_local_path = os.path.join(root_dir, local_path)
290        ret = self._RsyncExcludingRepoDirs(remote_path, full_local_path)
291        if ret:
292          return ret
293    return 0
294
295
296class RepoReader(object):
297  """Class for reading repositories."""
298
299  def __init__(self, filename):
300    self.filename = filename
301    self.main_dict = {}
302    self.input_repos = []
303    self.output_repos = []
304
305  def ParseFile(self):
306    with open(self.filename) as f:
307      self.main_dict = json.load(f)
308      self.CreateReposFromDict(self.main_dict)
309    return [self.input_repos, self.output_repos]
310
311  def CreateReposFromDict(self, main_dict):
312    for key, repo_list in main_dict.items():
313      for repo_dict in repo_list:
314        repo = self.CreateRepoFromDict(repo_dict)
315        if key == 'input':
316          self.input_repos.append(repo)
317        elif key == 'output':
318          self.output_repos.append(repo)
319        else:
320          logger.GetLogger().LogFatal('Unknown key: %s found' % key)
321
322  def CreateRepoFromDict(self, repo_dict):
323    repo_type = repo_dict.get('type', None)
324    repo_address = repo_dict.get('address', None)
325    repo_mappings = repo_dict.get('mappings', None)
326    repo_ignores = repo_dict.get('ignores', None)
327    repo_branch = repo_dict.get('branch', None)
328    gerrit = repo_dict.get('gerrit', None)
329    revision = repo_dict.get('revision', None)
330
331    if repo_type == 'p4':
332      repo = P4Repo(repo_address, repo_mappings, revision=revision)
333    elif repo_type == 'svn':
334      repo = SvnRepo(repo_address, repo_mappings)
335    elif repo_type == 'git':
336      repo = GitRepo(
337          repo_address,
338          repo_branch,
339          mappings=repo_mappings,
340          ignores=repo_ignores,
341          gerrit=gerrit)
342    elif repo_type == 'file':
343      repo = FileRepo(repo_address)
344    else:
345      logger.GetLogger().LogFatal('Unknown repo type: %s' % repo_type)
346    return repo
347
348
349@logger.HandleUncaughtExceptions
350def Main(argv):
351  parser = argparse.ArgumentParser()
352  parser.add_argument(
353      '-i',
354      '--input_file',
355      dest='input_file',
356      help='The input file that contains repo descriptions.')
357
358  parser.add_argument(
359      '-n',
360      '--dry_run',
361      dest='dry_run',
362      action='store_true',
363      default=False,
364      help='Do a dry run of the push.')
365
366  parser.add_argument(
367      '-F',
368      '--message_file',
369      dest='message_file',
370      default=None,
371      help=('Use contents of the log file as the commit '
372            'message.'))
373
374  options = parser.parse_args(argv)
375  if not options.input_file:
376    parser.print_help()
377    return 1
378  rr = RepoReader(options.input_file)
379  [input_repos, output_repos] = rr.ParseFile()
380
381  # Make sure FileRepo is not used as output destination.
382  for output_repo in output_repos:
383    if output_repo.repo_type == 'file':
384      logger.GetLogger().LogFatal(
385          'FileRepo is only supported as an input repo.')
386
387  for output_repo in output_repos:
388    ret = output_repo.SetupForPush()
389    if ret:
390      return ret
391
392  input_revisions = []
393  for input_repo in input_repos:
394    ret = input_repo.PullSources()
395    if ret:
396      return ret
397    input_revisions.append(input_repo.revision)
398
399  for input_repo in input_repos:
400    for output_repo in output_repos:
401      ret = input_repo.MapSources(output_repo.GetRoot())
402      if ret:
403        return ret
404
405  commit_message = 'Synced repos to: %s' % ','.join(input_revisions)
406  for output_repo in output_repos:
407    ret = output_repo.PushSources(
408        commit_message=commit_message,
409        dry_run=options.dry_run,
410        message_file=options.message_file)
411    if ret:
412      return ret
413
414  if not options.dry_run:
415    for output_repo in output_repos:
416      output_repo.CleanupRoot()
417    for input_repo in input_repos:
418      input_repo.CleanupRoot()
419
420  return ret
421
422
423if __name__ == '__main__':
424  retval = Main(sys.argv[1:])
425  sys.exit(retval)
426