• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/env -S python3 -B
2#
3# Copyright (C) 2021 The Android Open Source Project
4#
5# Licensed under the Apache License, Version 2.0 (the "License");
6# you may not use this file except in compliance with the License.
7# You may obtain a copy of the License at
8#
9#      http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16
17"""Downloads ART Module prebuilts and creates CLs to update them in git."""
18
19import argparse
20import collections
21import os
22import re
23import subprocess
24import sys
25import tempfile
26
27
28# Prebuilt description used in commit message
29PREBUILT_DESCR = "ART Module"
30
31# fetch_artifact branch and targets
32BRANCH = "aosp-master-art"
33MODULE_TARGET = "DOES_NOT_EXIST" # There is currently no CI build in AOSP.
34SDK_TARGET = "mainline_modules_sdks"
35
36# Where to install the APEX modules
37MODULE_PATH = "packages/modules/ArtPrebuilt"
38
39# Where to install the SDKs and module exports
40SDK_PATH = "prebuilts/module_sdk/art"
41
42SDK_VERSION = "current"
43
44# Paths to git projects to prepare CLs in
45GIT_PROJECT_ROOTS = [MODULE_PATH, SDK_PATH]
46
47SCRIPT_PATH = MODULE_PATH + "/update-art-module-prebuilts.py"
48
49
50InstallEntry = collections.namedtuple("InstallEntry", [
51    # Artifact path in the build, passed to fetch_target
52    "source_path",
53    # Local install path
54    "install_path",
55    # True if this is a module SDK, to be skipped by --skip-module-sdk.
56    "module_sdk",
57    # True if the entry is a zip file that should be unzipped to install_path
58    "install_unzipped",
59])
60
61
62def install_apks_entry(apex_name):
63  return [InstallEntry(
64      os.path.join(apex_name + ".apks"),
65      os.path.join(MODULE_PATH, apex_name + ".apks"),
66      module_sdk=False,
67      install_unzipped=False)]
68
69
70def install_sdk_entries(apex_name, mainline_sdk_name, sdk_dir):
71  return [InstallEntry(
72      os.path.join("mainline-sdks",
73                   SDK_VERSION,
74                   apex_name,
75                   sdk_dir,
76                   mainline_sdk_name + "-" + SDK_VERSION + ".zip"),
77      os.path.join(SDK_PATH, SDK_VERSION, sdk_dir),
78      module_sdk=True,
79      install_unzipped=True)]
80
81
82install_entries = (
83    install_apks_entry("com.android.art") +
84    install_sdk_entries("com.android.art",
85                        "art-module-sdk", "sdk") +
86    install_sdk_entries("com.android.art",
87                        "art-module-host-exports", "host-exports") +
88    install_sdk_entries("com.android.art",
89                        "art-module-test-exports", "test-exports")
90)
91
92
93def check_call(cmd, **kwargs):
94  """Proxy for subprocess.check_call with logging."""
95  msg = " ".join(cmd) if isinstance(cmd, list) else cmd
96  if "cwd" in kwargs:
97    msg = "In " + kwargs["cwd"] + ": " + msg
98  print(msg)
99  subprocess.check_call(cmd, **kwargs)
100
101
102def fetch_artifact(branch, target, build, fetch_pattern, local_dir):
103  """Fetches artifact from the build server."""
104  fetch_artifact_path = "/google/data/ro/projects/android/fetch_artifact"
105  cmd = [fetch_artifact_path, "--branch", branch, "--target", target,
106         "--bid", build, fetch_pattern]
107  check_call(cmd, cwd=local_dir)
108
109
110def start_branch(git_branch_name, git_dirs):
111  """Creates a new repo branch in the given projects."""
112  check_call(["repo", "start", git_branch_name] + git_dirs)
113  # In case the branch already exists we reset it to upstream, to get a clean
114  # update CL.
115  for git_dir in git_dirs:
116    check_call(["git", "reset", "--hard", "@{upstream}"], cwd=git_dir)
117
118
119def upload_branch(git_root, git_branch_name):
120  """Uploads the CLs in the given branch in the given project."""
121  # Set the branch as topic to bundle with the CLs in other git projects (if
122  # any).
123  check_call(["repo", "upload", "-t", "--br=" + git_branch_name, git_root])
124
125
126def remove_files(git_root, subpaths, stage_removals):
127  """Removes files in the work tree, optionally staging them in git."""
128  if stage_removals:
129    check_call(["git", "rm", "-qrf", "--ignore-unmatch"] + subpaths, cwd=git_root)
130  # Need a plain rm afterwards even if git rm was executed, because git won't
131  # remove directories if they have non-git files in them.
132  check_call(["rm", "-rf"] + subpaths, cwd=git_root)
133
134
135def commit(git_root, prebuilt_descr, branch, target, build, add_paths, bug_number):
136  """Commits the new prebuilts."""
137  check_call(["git", "add"] + add_paths, cwd=git_root)
138
139  if build:
140    message = (
141        "Update {prebuilt_descr} prebuilts to build {build}.\n\n"
142        "Taken from branch {branch}, target {target}."
143        .format(prebuilt_descr=prebuilt_descr, branch=branch, target=target,
144                build=build))
145  else:
146    message = (
147        "DO NOT SUBMIT: Update {prebuilt_descr} prebuilts from local build."
148        .format(prebuilt_descr=prebuilt_descr))
149  message += ("\n\nCL prepared by {}."
150              "\n\nTest: Presubmits".format(SCRIPT_PATH))
151  if bug_number:
152    message += ("\nBug: {}".format(bug_number))
153  msg_fd, msg_path = tempfile.mkstemp()
154  with os.fdopen(msg_fd, "w") as f:
155    f.write(message)
156
157  # Do a diff first to skip the commit without error if there are no changes to
158  # commit.
159  check_call("git diff-index --quiet --cached HEAD -- || "
160             "git commit -F " + msg_path, shell=True, cwd=git_root)
161  os.unlink(msg_path)
162
163
164def install_entry(branch, target, build, local_dist, entry):
165  """Installs one file specified by entry."""
166
167  install_dir, install_file = os.path.split(entry.install_path)
168  if install_dir and not os.path.exists(install_dir):
169    os.makedirs(install_dir)
170
171  if build:
172    fetch_artifact(branch, target, build, entry.source_path, install_dir)
173  else:
174    check_call(["cp", os.path.join(local_dist, entry.source_path), install_dir])
175  source_file = os.path.basename(entry.source_path)
176
177  if entry.install_unzipped:
178    check_call(["mkdir", install_file], cwd=install_dir)
179    # Add -DD to not extract timestamps that may confuse the build system.
180    check_call(["unzip", "-DD", source_file, "-d", install_file],
181               cwd=install_dir)
182    check_call(["rm", source_file], cwd=install_dir)
183
184  elif source_file != install_file:
185    check_call(["mv", source_file, install_file], cwd=install_dir)
186
187
188def install_paths_per_git_root(roots, paths):
189  """Partitions the given paths into subpaths within the given roots.
190
191  Args:
192    roots: List of root paths.
193    paths: List of paths relative to the same directory as the root paths.
194
195  Returns:
196    A dict mapping each root to the subpaths under it. It's an error if some
197    path doesn't go into any root.
198  """
199  res = collections.defaultdict(list)
200  for path in paths:
201    found = False
202    for root in roots:
203      if path.startswith(root + "/"):
204        res[root].append(path[len(root) + 1:])
205        found = True
206        break
207    if not found:
208      sys.exit("Install path {} is not in any of the git roots: {}"
209               .format(path, " ".join(roots)))
210  return res
211
212
213def get_args():
214  """Parses and returns command line arguments."""
215  parser = argparse.ArgumentParser(
216      epilog="Either --build or --local-dist is required.")
217
218  parser.add_argument("--branch", default=BRANCH,
219                      help="Branch to fetch, defaults to " + BRANCH)
220  parser.add_argument("--module-target", default=MODULE_TARGET,
221                      help="Target to fetch modules from, defaults to " +
222                      MODULE_TARGET)
223  parser.add_argument("--sdk-target", default=SDK_TARGET,
224                      help="Target to fetch SDKs from, defaults to " +
225                      SDK_TARGET)
226  parser.add_argument("--build", metavar="NUMBER",
227                      help="Build number to fetch")
228  parser.add_argument("--local-dist", metavar="PATH",
229                      help="Take prebuilts from this local dist dir instead of "
230                      "using fetch_artifact")
231  parser.add_argument("--skip-apex", default=True, action="store_true",
232                      help="Do not fetch .apex files. Defaults to true.")
233  parser.add_argument("--skip-module-sdk", action="store_true",
234                      help="Do not fetch and unpack sdk and module_export zips.")
235  parser.add_argument("--skip-cls", action="store_true",
236                      help="Do not create branches or git commits")
237  parser.add_argument("--bug", metavar="NUMBER",
238                      help="Add a 'Bug' line with this number to commit "
239                      "messages.")
240  parser.add_argument("--upload", action="store_true",
241                      help="Upload the CLs to Gerrit")
242
243  args = parser.parse_args()
244  if ((not args.build and not args.local_dist) or
245      (args.build and args.local_dist)):
246    sys.exit(parser.format_help())
247  return args
248
249
250def main():
251  """Program entry point."""
252  args = get_args()
253
254  if any(path for path in GIT_PROJECT_ROOTS if not os.path.exists(path)):
255    sys.exit("This script must be run in the root of the Android build tree.")
256
257  entries = install_entries
258  if args.skip_apex:
259    entries = [entry for entry in entries if entry.module_sdk]
260  if args.skip_module_sdk:
261    entries = [entry for entry in entries if not entry.module_sdk]
262  if not entries:
263    sys.exit("Both APEXes and SDKs skipped - nothing to do.")
264
265  install_paths = [entry.install_path for entry in entries]
266  install_paths_per_root = install_paths_per_git_root(
267      GIT_PROJECT_ROOTS, install_paths)
268
269  git_branch_name = PREBUILT_DESCR.lower().replace(" ", "-") + "-update"
270  if args.build:
271    git_branch_name += "-" + args.build
272
273  if not args.skip_cls:
274    git_paths = list(install_paths_per_root.keys())
275    start_branch(git_branch_name, git_paths)
276
277  for git_root, subpaths in install_paths_per_root.items():
278    remove_files(git_root, subpaths, not args.skip_cls)
279  for entry in entries:
280    target = args.sdk_target if entry.module_sdk else args.module_target
281    install_entry(args.branch, target, args.build, args.local_dist, entry)
282
283  if not args.skip_cls:
284    for git_root, subpaths in install_paths_per_root.items():
285      target = args.sdk_target if git_root == SDK_PATH else args.module_target
286      commit(git_root, PREBUILT_DESCR, args.branch, target, args.build, subpaths,
287             args.bug)
288
289    if args.upload:
290      # Don't upload all projects in a single repo upload call, because that
291      # makes it pop up an interactive editor.
292      for git_root in install_paths_per_root:
293        upload_branch(git_root, git_branch_name)
294
295
296if __name__ == "__main__":
297  main()
298