• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/env -S python3 -B
2#
3# Copyright (C) 2021 The Android Open Source Project
4#
5# Licensed under the Apache License, Version 2.0 (the "License");
6# you may not use this file except in compliance with the License.
7# You may obtain a copy of the License at
8#
9#      http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16
17"""Downloads ART Module prebuilts and creates CLs to update them in git."""
18
19import argparse
20import collections
21import os
22import re
23import subprocess
24import sys
25import tempfile
26
27
28# Prebuilt description used in commit message
29PREBUILT_DESCR = "ART Module"
30
31# fetch_artifact branch and target
32BRANCH = "aosp-master-art"
33TARGET = "aosp_art_module"
34
35ARCHES = ["arm", "arm64", "x86", "x86_64"]
36
37# Where to install the APEX packages
38PACKAGE_PATH = "packages/modules/ArtPrebuilt"
39
40# Where to install the SDKs and module exports
41SDK_PATH = "prebuilts/module_sdk/art"
42
43SDK_VERSION = "current"
44
45# Paths to git projects to prepare CLs in
46GIT_PROJECT_ROOTS = [PACKAGE_PATH, SDK_PATH]
47
48SCRIPT_PATH = PACKAGE_PATH + "/update-art-module-prebuilts.py"
49
50
51InstallEntry = collections.namedtuple("InstallEntry", [
52    # Artifact path in the build, passed to fetch_target
53    "source_path",
54    # Local install path
55    "install_path",
56    # True if this is a module SDK, to be skipped by --skip-module-sdk.
57    "module_sdk",
58    # True if the entry is a zip file that should be unzipped to install_path
59    "install_unzipped",
60])
61
62
63def install_apex_entries(apex_name):
64  res = []
65  for arch in ARCHES:
66    res.append(InstallEntry(
67        os.path.join(arch, apex_name + ".apex"),
68        os.path.join(PACKAGE_PATH, apex_name + "-" + arch + ".apex"),
69        module_sdk=False,
70        install_unzipped=False))
71  return res
72
73
74def install_sdk_entries(mainline_sdk_name, sdk_dir):
75  return [InstallEntry(
76      os.path.join("mainline-sdks",
77                   mainline_sdk_name + "-" + SDK_VERSION + ".zip"),
78      os.path.join(SDK_PATH, SDK_VERSION, sdk_dir),
79      module_sdk=True,
80      install_unzipped=True)]
81
82
83install_entries = (
84    install_apex_entries("com.android.art") +
85    install_apex_entries("com.android.art.debug") +
86    install_sdk_entries("art-module-sdk", "sdk") +
87    install_sdk_entries("art-module-host-exports", "host-exports") +
88    install_sdk_entries("art-module-test-exports", "test-exports")
89)
90
91
92def rewrite_bp_for_art_module_source_build(bp_path):
93  """Rewrites an Android.bp file to conditionally prefer prebuilts."""
94  print("Rewriting {} for SOONG_CONFIG_art_module_source_build use."
95        .format(bp_path))
96  bp_file = open(bp_path, "r+")
97
98  # TODO(b/174997203): Remove this when we have a proper way to control prefer
99  # flags in Mainline modules.
100
101  header_lines = []
102  for line in bp_file:
103    line = line.rstrip("\n")
104    if not line.startswith("//"):
105      break
106    header_lines.append(line)
107
108  art_module_types = set()
109
110  content_lines = []
111  for line in bp_file:
112    line = line.rstrip("\n")
113    module_header = re.match("([a-z0-9_]+) +{$", line)
114    if not module_header:
115      content_lines.append(line)
116    else:
117      # Iterate over one Soong module.
118      module_start = line
119      soong_config_clause = False
120      module_content = []
121
122      for module_line in bp_file:
123        module_line = module_line.rstrip("\n")
124        if module_line == "}":
125          break
126        if module_line == "    prefer: false,":
127          module_content.extend([
128              ("    // Do not prefer prebuilt if "
129               "SOONG_CONFIG_art_module_source_build is true."),
130              "    prefer: true,",
131              "    soong_config_variables: {",
132              "        source_build: {",
133              "            prefer: false,",
134              "        },",
135              "    },"])
136          soong_config_clause = True
137        else:
138          module_content.append(module_line)
139
140      if soong_config_clause:
141        module_type = "art_prebuilt_" + module_header.group(1)
142        module_start = module_type + " {"
143        art_module_types.add(module_type)
144
145      content_lines.append(module_start)
146      content_lines.extend(module_content)
147      content_lines.append("}")
148
149  header_lines.extend(
150      ["",
151       "// Soong config variable stanza added by {}.".format(SCRIPT_PATH),
152       "soong_config_module_type_import {",
153       "    from: \"prebuilts/module_sdk/art/SoongConfig.bp\",",
154       "    module_types: ["] +
155      ["        \"{}\",".format(art_module)
156       for art_module in sorted(art_module_types)] +
157      ["    ],",
158       "}",
159       ""])
160
161  bp_file.seek(0)
162  bp_file.truncate()
163  bp_file.write("\n".join(header_lines + content_lines))
164  bp_file.close()
165
166
167def check_call(cmd, **kwargs):
168  """Proxy for subprocess.check_call with logging."""
169  msg = " ".join(cmd) if isinstance(cmd, list) else cmd
170  if "cwd" in kwargs:
171    msg = "In " + kwargs["cwd"] + ": " + msg
172  print(msg)
173  subprocess.check_call(cmd, **kwargs)
174
175
176def fetch_artifact(branch, target, build, fetch_pattern, local_dir):
177  """Fetches artifact from the build server."""
178  fetch_artifact_path = "/google/data/ro/projects/android/fetch_artifact"
179  cmd = [fetch_artifact_path, "--branch", branch, "--target", target,
180         "--bid", build, fetch_pattern]
181  check_call(cmd, cwd=local_dir)
182
183
184def start_branch(branch_name, git_dirs):
185  """Creates a new repo branch in the given projects."""
186  check_call(["repo", "start", branch_name] + git_dirs)
187  # In case the branch already exists we reset it to upstream, to get a clean
188  # update CL.
189  for git_dir in git_dirs:
190    check_call(["git", "reset", "--hard", "@{upstream}"], cwd=git_dir)
191
192
193def upload_branch(git_root, branch_name):
194  """Uploads the CLs in the given branch in the given project."""
195  # Set the branch as topic to bundle with the CLs in other git projects (if
196  # any).
197  check_call(["repo", "upload", "-t", "--br=" + branch_name, git_root])
198
199
200def remove_files(git_root, subpaths, stage_removals):
201  """Removes files in the work tree, optionally staging them in git."""
202  if stage_removals:
203    check_call(["git", "rm", "-qrf", "--ignore-unmatch"] + subpaths, cwd=git_root)
204  # Need a plain rm afterwards even if git rm was executed, because git won't
205  # remove directories if they have non-git files in them.
206  check_call(["rm", "-rf"] + subpaths, cwd=git_root)
207
208
209def commit(git_root, prebuilt_descr, branch, target, build, add_paths, bug_number):
210  """Commits the new prebuilts."""
211  check_call(["git", "add"] + add_paths, cwd=git_root)
212
213  if build:
214    message = (
215        "Update {prebuilt_descr} prebuilts to build {build}.\n\n"
216        "Taken from branch {branch}, target {target}."
217        .format(prebuilt_descr=prebuilt_descr, branch=branch, target=target,
218                build=build))
219  else:
220    message = (
221        "DO NOT SUBMIT: Update {prebuilt_descr} prebuilts from local build."
222        .format(prebuilt_descr=prebuilt_descr))
223  message += ("\n\nCL prepared by {}."
224              "\n\nTest: Presubmits".format(SCRIPT_PATH))
225  if bug_number:
226    message += ("\nBug: {}".format(bug_number))
227  msg_fd, msg_path = tempfile.mkstemp()
228  with os.fdopen(msg_fd, "w") as f:
229    f.write(message)
230
231  # Do a diff first to skip the commit without error if there are no changes to
232  # commit.
233  check_call("git diff-index --quiet --cached HEAD -- || "
234             "git commit -F " + msg_path, shell=True, cwd=git_root)
235  os.unlink(msg_path)
236
237
238def install_entry(build, local_dist, entry):
239  """Installs one file specified by entry."""
240
241  install_dir, install_file = os.path.split(entry.install_path)
242  if install_dir and not os.path.exists(install_dir):
243    os.makedirs(install_dir)
244
245  if build:
246    fetch_artifact(BRANCH, TARGET, build, entry.source_path, install_dir)
247  else:
248    check_call(["cp", os.path.join(local_dist, entry.source_path), install_dir])
249  source_file = os.path.basename(entry.source_path)
250
251  if entry.install_unzipped:
252    check_call(["mkdir", install_file], cwd=install_dir)
253    # Add -DD to not extract timestamps that may confuse the build system.
254    check_call(["unzip", "-DD", source_file, "-d", install_file],
255               cwd=install_dir)
256    check_call(["rm", source_file], cwd=install_dir)
257
258  elif source_file != install_file:
259    check_call(["mv", source_file, install_file], cwd=install_dir)
260
261
262def install_paths_per_git_root(roots, paths):
263  """Partitions the given paths into subpaths within the given roots.
264
265  Args:
266    roots: List of root paths.
267    paths: List of paths relative to the same directory as the root paths.
268
269  Returns:
270    A dict mapping each root to the subpaths under it. It's an error if some
271    path doesn't go into any root.
272  """
273  res = collections.defaultdict(list)
274  for path in paths:
275    found = False
276    for root in roots:
277      if path.startswith(root + "/"):
278        res[root].append(path[len(root) + 1:])
279        found = True
280        break
281    if not found:
282      sys.exit("Install path {} is not in any of the git roots: {}"
283               .format(path, " ".join(roots)))
284  return res
285
286
287def get_args():
288  """Parses and returns command line arguments."""
289  parser = argparse.ArgumentParser(
290      epilog="Either --build or --local-dist is required.")
291
292  parser.add_argument("--build", metavar="NUMBER",
293                      help="Build number to fetch from branch {}, target {}"
294                      .format(BRANCH, TARGET))
295  parser.add_argument("--local-dist", metavar="PATH",
296                      help="Take prebuilts from this local dist dir instead of "
297                      "using fetch_artifact")
298  parser.add_argument("--skip-apex", action="store_true",
299                      help="Do not fetch .apex files.")
300  parser.add_argument("--skip-module-sdk", action="store_true",
301                      help="Do not fetch and unpack sdk and module_export zips.")
302  parser.add_argument("--skip-cls", action="store_true",
303                      help="Do not create branches or git commits")
304  parser.add_argument("--bug", metavar="NUMBER",
305                      help="Add a 'Bug' line with this number to commit "
306                      "messages.")
307  parser.add_argument("--upload", action="store_true",
308                      help="Upload the CLs to Gerrit")
309
310  args = parser.parse_args()
311  if ((not args.build and not args.local_dist) or
312      (args.build and args.local_dist)):
313    sys.exit(parser.format_help())
314  return args
315
316
317def main():
318  """Program entry point."""
319  args = get_args()
320
321  if any(path for path in GIT_PROJECT_ROOTS if not os.path.exists(path)):
322    sys.exit("This script must be run in the root of the Android build tree.")
323
324  entries = install_entries
325  if args.skip_apex:
326    entries = [entry for entry in entries if entry.module_sdk]
327  if args.skip_module_sdk:
328    entries = [entry for entry in entries if not entry.module_sdk]
329  if not entries:
330    sys.exit("Both APEXes and SDKs skipped - nothing to do.")
331
332  install_paths = [entry.install_path for entry in entries]
333  install_paths_per_root = install_paths_per_git_root(
334      GIT_PROJECT_ROOTS, install_paths)
335
336  branch_name = PREBUILT_DESCR.lower().replace(" ", "-") + "-update"
337  if args.build:
338    branch_name += "-" + args.build
339
340  if not args.skip_cls:
341    git_paths = list(install_paths_per_root.keys())
342    start_branch(branch_name, git_paths)
343
344  for git_root, subpaths in install_paths_per_root.items():
345    remove_files(git_root, subpaths, not args.skip_cls)
346  for entry in entries:
347    install_entry(args.build, args.local_dist, entry)
348
349  # Postprocess the Android.bp files in the SDK snapshot to control prefer flags
350  # on the prebuilts through SOONG_CONFIG_art_module_source_build.
351  # TODO(b/174997203): Replace this with a better way to control prefer flags on
352  # Mainline module prebuilts.
353  for entry in entries:
354    if entry.install_unzipped:
355      bp_path = os.path.join(entry.install_path, "Android.bp")
356      if os.path.exists(bp_path):
357        rewrite_bp_for_art_module_source_build(bp_path)
358
359  if not args.skip_cls:
360    for git_root, subpaths in install_paths_per_root.items():
361      commit(git_root, PREBUILT_DESCR, BRANCH, TARGET, args.build, subpaths,
362             args.bug)
363
364    if args.upload:
365      # Don't upload all projects in a single repo upload call, because that
366      # makes it pop up an interactive editor.
367      for git_root in install_paths_per_root:
368        upload_branch(git_root, branch_name)
369
370
371if __name__ == "__main__":
372  main()
373