• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/python
2"""Diff a repo (downstream) and its upstream.
3
4This script:
5  1. Downloads a repo source tree with specified manifest URL, branch
6     and release tag.
7  2. Retrieves the BUILD_ID from $downstream/build/core/build_id.mk.
8  3. Downloads the upstream using the BUILD_ID.
9  4. Diffs each project in these two repos.
10"""
11
12import argparse
13import datetime
14import os
15import subprocess
16import repo_diff_trees
17
18HELP_MSG = "Diff a repo (downstream) and its upstream"
19
20DOWNSTREAM_WORKSPACE = "downstream"
21UPSTREAM_WORKSPACE = "upstream"
22
23DEFAULT_MANIFEST_URL = "https://android.googlesource.com/platform/manifest"
24DEFAULT_MANIFEST_BRANCH = "android-8.0.0_r10"
25DEFAULT_UPSTREAM_MANIFEST_URL = "https://android.googlesource.com/platform/manifest"
26DEFAULT_UPSTREAM_MANIFEST_BRANCH = "android-8.0.0_r1"
27SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
28DEFAULT_EXCLUSIONS_FILE = os.path.join(SCRIPT_DIR, "android_exclusions.txt")
29
30
31def parse_args():
32  """Parse args."""
33
34  parser = argparse.ArgumentParser(description=HELP_MSG)
35
36  parser.add_argument("-u", "--manifest-url",
37                      help="manifest url",
38                      default=DEFAULT_MANIFEST_URL)
39  parser.add_argument("-b", "--manifest-branch",
40                      help="manifest branch",
41                      default=DEFAULT_MANIFEST_BRANCH)
42  parser.add_argument("-r", "--upstream-manifest-url",
43                      help="upstream manifest url",
44                      default=DEFAULT_UPSTREAM_MANIFEST_URL)
45  parser.add_argument("-a", "--upstream-manifest-branch",
46                      help="upstream manifest branch",
47                      default=DEFAULT_UPSTREAM_MANIFEST_BRANCH)
48  parser.add_argument("-e", "--exclusions-file",
49                      help="exclusions file",
50                      default=DEFAULT_EXCLUSIONS_FILE)
51  parser.add_argument("-t", "--tag",
52                      help="release tag (optional). If not set then will "
53                      "sync the latest in the branch.")
54  parser.add_argument("-i", "--ignore_error_during_sync",
55                      action="store_true",
56                      help="repo sync might fail due to varios reasons. "
57                      "Ignore these errors and move on. Use with caution.")
58
59  return parser.parse_args()
60
61
62def repo_init(url, rev, workspace):
63  """Repo init with specific url and rev.
64
65  Args:
66    url: manifest url
67    rev: manifest branch, or rev
68    workspace: the folder to init and sync code
69  """
70
71  print("repo init:\n  url: %s\n  rev: %s\n  workspace: %s" %
72        (url, rev, workspace))
73
74  subprocess.check_output("repo init --manifest-url=%s --manifest-branch=%s" %
75                          (url, rev), cwd=workspace, shell=True)
76
77
78def repo_sync(workspace, ignore_error, retry=5):
79  """Repo sync."""
80
81  count = 0
82  while count < retry:
83    count += 1
84    print("repo sync (retry=%d/%d):\n  workspace: %s" %
85          (count, retry, workspace))
86
87    try:
88      command = "repo sync --jobs=24 --current-branch --quiet"
89      command += " --no-tags --no-clone-bundle"
90      if ignore_error:
91        command += " --force-broken"
92      subprocess.check_output(command, cwd=workspace, shell=True)
93    except subprocess.CalledProcessError as e:
94      print "Error: %s" % e.output
95      if count == retry and not ignore_error:
96        raise e
97    # Stop retrying if the repo sync was successful
98    else:
99      break
100
101
102def get_commit_with_keyword(project_path, keyword):
103  """Get the latest commit in $project_path with the specific keyword."""
104
105  return subprocess.check_output(("git -C %s "
106                                  "rev-list --max-count=1 --grep=\"%s\" "
107                                  "HEAD") %
108                                 (project_path, keyword), shell=True).rstrip()
109
110
111def get_build_id(workspace):
112  """Get BUILD_ID defined in $workspace/build/core/build_id.mk."""
113
114  path = os.path.join(workspace, "build", "core", "build_id.mk")
115  return subprocess.check_output("source %s && echo $BUILD_ID" % path,
116                                 shell=True).rstrip()
117
118
119def repo_sync_specific_release(url, branch, tag, workspace, ignore_error):
120  """Repo sync source with the specific release tag."""
121
122  if not os.path.exists(workspace):
123    os.makedirs(workspace)
124
125  manifest_path = os.path.join(workspace, ".repo", "manifests")
126
127  repo_init(url, branch, workspace)
128
129  if tag:
130    rev = get_commit_with_keyword(manifest_path, tag)
131    if not rev:
132      raise(ValueError("could not find a manifest revision for tag " + tag))
133    repo_init(url, rev, workspace)
134
135  repo_sync(workspace, ignore_error)
136
137
138def diff(manifest_url, manifest_branch, tag,
139         upstream_manifest_url, upstream_manifest_branch,
140         exclusions_file, ignore_error_during_sync):
141  """Syncs and diffs an Android workspace against an upstream workspace."""
142
143  workspace = os.path.abspath(DOWNSTREAM_WORKSPACE)
144  upstream_workspace = os.path.abspath(UPSTREAM_WORKSPACE)
145  # repo sync downstream source tree
146  repo_sync_specific_release(
147      manifest_url,
148      manifest_branch,
149      tag,
150      workspace,
151      ignore_error_during_sync)
152
153  build_id = None
154
155  if tag:
156    # get the build_id so that we know which rev of upstream we need
157    build_id = get_build_id(workspace)
158    if not build_id:
159      raise(ValueError("Error: could not find the Build ID of " + workspace))
160
161  # repo sync upstream source tree
162  repo_sync_specific_release(
163      upstream_manifest_url,
164      upstream_manifest_branch,
165      build_id,
166      upstream_workspace,
167      ignore_error_during_sync)
168
169
170  # make output folder
171  if tag:
172    output_folder = os.path.abspath(tag.replace(" ", "_"))
173  else:
174    current_time = datetime.datetime.today().strftime('%Y%m%d_%H%M%S')
175    output_folder = os.path.abspath(current_time)
176
177  if not os.path.exists(output_folder):
178      os.makedirs(output_folder)
179
180  # do the comparison
181  repo_diff_trees.diff(
182      upstream_workspace,
183      workspace,
184      os.path.join(output_folder, "project.csv"),
185      os.path.join(output_folder, "commit.csv"),
186      os.path.abspath(exclusions_file),
187  )
188
189
190def main():
191  args = parse_args()
192
193  diff(args.manifest_url,
194       args.manifest_branch,
195       args.tag,
196       args.upstream_manifest_url,
197       args.upstream_manifest_branch,
198       args.exclusions_file,
199       args.ignore_error_during_sync)
200
201if __name__ == "__main__":
202  main()
203