• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/env python
2# Copyright (C) 2017 The Android Open Source Project
3#
4# Licensed under the Apache License, Version 2.0 (the "License");
5# you may not use this file except in compliance with the License.
6# You may obtain a copy of the License at
7#
8#      http://www.apache.org/licenses/LICENSE-2.0
9#
10# Unless required by applicable law or agreed to in writing, software
11# distributed under the License is distributed on an "AS IS" BASIS,
12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13# See the License for the specific language governing permissions and
14# limitations under the License.
15""" Mirrors a Gerrit repo into GitHub.
16
17Mirrors all the branches (refs/heads/foo) from Gerrit to Github as-is, taking
18care of propagating also deletions.
19
20This script used to be more complex, turning all the Gerrit CLs
21(refs/changes/NN/cl_number/patchset_number) into Github branches
22(refs/heads/cl_number). This use case was dropped as we moved away from Travis.
23See the git history of this file for more.
24"""
25
26import argparse
27import logging
28import os
29import re
30import shutil
31import subprocess
32import sys
33import time
34
35CUR_DIR = os.path.dirname(os.path.abspath(__file__))
36GIT_UPSTREAM = 'https://android.googlesource.com/platform/external/perfetto/'
37GIT_MIRROR = 'git@github.com:google/perfetto.git'
38WORKDIR = os.path.join(CUR_DIR, 'repo')
39
40# Min delay (in seconds) between two consecutive git poll cycles. This is to
41# avoid hitting gerrit API quota limits.
42POLL_PERIOD_SEC = 60
43
44# The actual key is stored into the Google Cloud project metadata.
45ENV = {
46    'GIT_SSH_COMMAND': 'ssh -i ' + os.path.join(CUR_DIR, 'deploy_key'),
47    'GIT_DIR': WORKDIR,
48}
49
50
51def GitCmd(*args, **kwargs):
52  cmd = ['git'] + list(args)
53  p = subprocess.Popen(
54      cmd,
55      stdin=subprocess.PIPE,
56      stdout=subprocess.PIPE,
57      stderr=sys.stderr,
58      cwd=WORKDIR,
59      env=ENV)
60  out = p.communicate(kwargs.get('stdin'))[0]
61  assert p.returncode == 0, 'FAIL: ' + ' '.join(cmd)
62  return out
63
64
65# Create a git repo that mirrors both the upstream and the mirror repos.
66def Setup(args):
67  if os.path.exists(WORKDIR):
68    if args.no_clean:
69      return
70    shutil.rmtree(WORKDIR)
71  os.makedirs(WORKDIR)
72  GitCmd('init', '--bare', '--quiet')
73  GitCmd('remote', 'add', 'upstream', GIT_UPSTREAM)
74  GitCmd('config', 'remote.upstream.tagOpt', '--no-tags')
75  GitCmd('config', '--add', 'remote.upstream.fetch',
76         '+refs/heads/*:refs/remotes/upstream/heads/*')
77  GitCmd('config', '--add', 'remote.upstream.fetch',
78         '+refs/tags/*:refs/remotes/upstream/tags/*')
79  GitCmd('remote', 'add', 'mirror', GIT_MIRROR, '--mirror=fetch')
80
81
82def Sync(args):
83  logging.info('Fetching git remotes')
84  GitCmd('fetch', '--all', '--quiet')
85  all_refs = GitCmd('show-ref')
86  future_heads = {}
87  current_heads = {}
88
89  # List all refs from both repos and:
90  # 1. Keep track of all branch heads refnames and sha1s from the (github)
91  #    mirror into |current_heads|.
92  # 2. Keep track of all upstream (AOSP) branch heads into |future_heads|. Note:
93  #    this includes only pure branches and NOT CLs. CLs and their patchsets are
94  #    stored in a hidden ref (refs/changes) which is NOT under refs/heads.
95  # 3. Keep track of all upstream (AOSP) CLs from the refs/changes namespace
96  #    into changes[cl_number][patchset_number].
97  for line in all_refs.splitlines():
98    ref_sha1, ref = line.split()
99
100    FILT_REGEX = r'(heads/main|heads/master|heads/releases/.*|tags/v\d+\.\d+)$'
101    m = re.match('refs/' + FILT_REGEX, ref)
102    if m is not None:
103      branch = m.group(1)
104      current_heads['refs/' + branch] = ref_sha1
105      continue
106
107    m = re.match('refs/remotes/upstream/' + FILT_REGEX, ref)
108    if m is not None:
109      branch = m.group(1)
110      future_heads['refs/' + branch] = ref_sha1
111      continue
112
113  deleted_heads = set(current_heads) - set(future_heads)
114  logging.info('current_heads: %d, future_heads: %d, deleted_heads: %d',
115               len(current_heads), len(future_heads), len(deleted_heads))
116
117  # Now compute:
118  # 1. The set of branches in the mirror (github) that have been deleted on the
119  #    upstream (AOSP) repo. These will be deleted also from the mirror.
120  # 2. The set of rewritten branches to be updated.
121  update_ref_cmd = ''
122  for ref_to_delete in deleted_heads:
123    update_ref_cmd += 'delete %s\n' % ref_to_delete
124  for ref_to_update, ref_sha1 in future_heads.iteritems():
125    if current_heads.get(ref_to_update) != ref_sha1:
126      update_ref_cmd += 'update %s %s\n' % (ref_to_update, ref_sha1)
127
128  GitCmd('update-ref', '--stdin', stdin=update_ref_cmd)
129
130  if args.push:
131    logging.info('Pushing updates')
132    GitCmd('push', 'mirror', '--all', '--prune', '--force', '--follow-tags')
133    GitCmd('gc', '--prune=all', '--aggressive', '--quiet')
134  else:
135    logging.info('Dry-run mode, skipping git push. Pass --push for prod mode.')
136
137
138def Main():
139  parser = argparse.ArgumentParser()
140  parser.add_argument('--push', default=False, action='store_true')
141  parser.add_argument('--no-clean', default=False, action='store_true')
142  parser.add_argument('-v', dest='verbose', default=False, action='store_true')
143  args = parser.parse_args()
144
145  logging.basicConfig(
146      format='%(asctime)s %(levelname)-8s %(message)s',
147      level=logging.DEBUG if args.verbose else logging.INFO,
148      datefmt='%Y-%m-%d %H:%M:%S')
149
150  logging.info('Setting up git repo one-off')
151  Setup(args)
152  while True:
153    logging.info('------- BEGINNING OF SYNC CYCLE -------')
154    Sync(args)
155    logging.info('------- END OF SYNC CYCLE -------')
156    time.sleep(POLL_PERIOD_SEC)
157
158
159if __name__ == '__main__':
160  sys.exit(Main())
161