• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/python3
2
3import argparse
4import glob
5import json
6import os
7import re
8import shlex
9import shutil
10import subprocess
11import sys
12import tempfile
13import zipfile
14
15from collections import defaultdict
16from pathlib import Path
17
18# See go/fetch_artifact for details on this script.
19FETCH_ARTIFACT = '/google/data/ro/projects/android/fetch_artifact'
20COMPAT_REPO = Path('prebuilts/sdk')
21COMPAT_README = Path('extensions/README.md')
22# This build target is used when fetching from a train build (TXXXXXXXX)
23BUILD_TARGET_TRAIN = 'train_build'
24# This build target is used when fetching from a non-train build (XXXXXXXX)
25BUILD_TARGET_CONTINUOUS = 'mainline_modules_sdks-userdebug'
26BUILD_TARGET_CONTINUOUS_MAIN = 'mainline_modules_sdks-{release_config}-userdebug'
27# The glob of sdk artifacts to fetch from remote build
28ARTIFACT_PATTERN = 'mainline-sdks/for-next-build/current/{module_name}/sdk/*.zip'
29# The glob of sdk artifacts to fetch from local build
30ARTIFACT_LOCAL_PATTERN = 'out/dist/mainline-sdks/for-next-build/current/{module_name}/sdk/*.zip'
31ARTIFACT_MODULES_INFO = 'mainline-modules-info.json'
32ARTIFACT_LOCAL_MODULES_INFO = 'out/dist/mainline-modules-info.json'
33COMMIT_TEMPLATE = """Finalize artifacts for extension SDK %d
34
35Import from build id %s.
36
37Generated with:
38$ %s
39
40Bug: %d
41Test: presubmit"""
42
43def fail(*args, **kwargs):
44    print(*args, file=sys.stderr, **kwargs)
45    sys.exit(1)
46
47def fetch_artifacts(build_id, target, artifact_path, dest):
48    print('Fetching %s from %s ...' % (artifact_path, target))
49    fetch_cmd = [FETCH_ARTIFACT]
50    fetch_cmd.extend(['--bid', str(build_id)])
51    fetch_cmd.extend(['--target', target])
52    fetch_cmd.append(artifact_path)
53    fetch_cmd.append(str(dest))
54    print("Running: " + ' '.join(fetch_cmd))
55    try:
56        subprocess.check_output(fetch_cmd, stderr=subprocess.STDOUT)
57    except subprocess.CalledProcessError as e:
58        fail(
59            'FAIL: Unable to retrieve %s artifact for build ID %s for %s target\n Error: %s'
60            % (artifact_path, build_id, target, e.output.decode())
61        )
62
63def fetch_mainline_modules_info_artifact(target, build_id):
64    tmpdir = Path(tempfile.TemporaryDirectory().name)
65    tmpdir.mkdir()
66    if args.local_mode:
67        artifact_path = ARTIFACT_LOCAL_MODULES_INFO
68        print('Copying %s to %s ...' % (artifact_path, tmpdir))
69        shutil.copy(artifact_path, tmpdir)
70    else:
71        artifact_path = ARTIFACT_MODULES_INFO
72        fetch_artifacts(build_id, target, artifact_path, tmpdir)
73    return tmpdir / ARTIFACT_MODULES_INFO
74
75def fetch_module_sdk_artifacts(target, build_id, module_name):
76    tmpdir = Path(tempfile.TemporaryDirectory().name)
77    tmpdir.mkdir()
78    if args.local_mode:
79        artifact_path = ARTIFACT_LOCAL_PATTERN.format(module_name='*')
80        print('Copying %s to %s ...' % (artifact_path, tmpdir))
81        for file in glob.glob(artifact_path):
82            shutil.copy(file, tmpdir)
83    else:
84        artifact_path = ARTIFACT_PATTERN.format(module_name=module_name)
85        fetch_artifacts(build_id, target, artifact_path, tmpdir)
86    return tmpdir
87
88def repo_for_sdk(sdk_filename, mainline_modules_info):
89    for module in mainline_modules_info:
90        if mainline_modules_info[module]["sdk_name"] in sdk_filename:
91            project_path = Path(mainline_modules_info[module]["module_sdk_project"])
92            if args.gantry_download_dir:
93                project_path = args.gantry_download_dir / project_path
94                os.makedirs(project_path , exist_ok = True, mode = 0o777)
95            print(f"module_sdk_path for {module}: {project_path}")
96            return project_path
97
98    fail('"%s" has no valid mapping to any mainline module.' % sdk_filename)
99
100def dir_for_sdk(filename, version):
101    base = str(version)
102    if 'test-exports' in filename:
103        return os.path.join(base, 'test-exports')
104    if 'host-exports' in filename:
105        return os.path.join(base, 'host-exports')
106    return base
107
108def is_ignored(file):
109    # Conscrypt has some legacy API tracking files that we don't consider for extensions.
110    bad_stem_prefixes = ['conscrypt.module.intra.core.api', 'conscrypt.module.platform.api']
111    return any([file.stem.startswith(p) for p in bad_stem_prefixes])
112
113
114def maybe_tweak_compat_stem(file):
115    # For legacy reasons, art and conscrypt txt file names in the SDKs (*.module.public.api)
116    # do not match their expected filename in prebuilts/sdk (art, conscrypt). So rename them
117    # to match.
118    new_stem = file.stem
119    new_stem = new_stem.replace('art.module.public.api', 'art')
120    new_stem = new_stem.replace('conscrypt.module.public.api', 'conscrypt')
121
122    # The stub jar artifacts from official builds are named '*-stubs.jar', but
123    # the convention for the copies in prebuilts/sdk is just '*.jar'. Fix that.
124    new_stem = new_stem.replace('-stubs', '')
125
126    return file.with_stem(new_stem)
127
128parser = argparse.ArgumentParser(description=('Finalize an extension SDK with prebuilts'))
129parser.add_argument('-a', '--amend_last_commit', action="store_true", help='Amend current HEAD commits instead of making new commits.')
130parser.add_argument('-b', '--bug', type=int, required=True, help='The bug number to add to the commit message.')
131parser.add_argument('-c', '--release_config', type=str, help='The release config to use to finalize.')
132parser.add_argument('-d', '--dry_run', action='store_true', help='Leaves git and repo out it')
133parser.add_argument('-f', '--finalize_sdk', type=int, required=True, help='The numbered SDK to finalize.')
134# This flag is only required when executed via Gantry. It points to the downloaded directory to be used.
135parser.add_argument('-g', '--gantry_download_dir', type=str, help=argparse.SUPPRESS)
136parser.add_argument('-l', '--local_mode', action="store_true", help='Local mode: use locally built artifacts and don\'t upload the result to Gerrit.')
137parser.add_argument('-m', '--modules', action='append', help='Modules to include. Can be provided multiple times, or not at all for all modules.')
138parser.add_argument('-r', '--readme', required=True, help='Version history entry to add to %s' % (COMPAT_REPO / COMPAT_README))
139parser.add_argument('bid', help='Build server build ID')
140args = parser.parse_args()
141
142if not os.path.isdir('build/soong') and not args.gantry_download_dir:
143    fail("This script must be run from the top of an Android source tree.")
144
145if args.release_config:
146    BUILD_TARGET_CONTINUOUS = BUILD_TARGET_CONTINUOUS_MAIN.format(release_config=args.release_config)
147build_target = BUILD_TARGET_TRAIN if args.bid[0] == 'T' else BUILD_TARGET_CONTINUOUS
148branch_name = 'finalize-%d' % args.finalize_sdk
149cmdline = shlex.join([x for x in sys.argv if x not in ['-a', '--amend_last_commit', '-l', '--local_mode']])
150commit_message = COMMIT_TEMPLATE % (args.finalize_sdk, args.bid, cmdline, args.bug)
151module_names = args.modules or ['*']
152
153if args.gantry_download_dir:
154    args.gantry_download_dir = Path(args.gantry_download_dir)
155    COMPAT_REPO = args.gantry_download_dir / COMPAT_REPO
156    mainline_modules_info_file = args.gantry_download_dir / ARTIFACT_MODULES_INFO
157else:
158    mainline_modules_info_file = fetch_mainline_modules_info_artifact(build_target, args.bid)
159
160compat_dir = COMPAT_REPO.joinpath('extensions/%d' % args.finalize_sdk)
161if compat_dir.is_dir():
162    print('Removing existing dir %s' % compat_dir)
163    shutil.rmtree(compat_dir)
164
165created_dirs = defaultdict(set)
166with open(mainline_modules_info_file, "r", encoding="utf8",) as file:
167    mainline_modules_info = json.load(file)
168
169for m in module_names:
170    if args.gantry_download_dir:
171        tmpdir = args.gantry_download_dir / "sdk_artifacts"
172    else:
173        tmpdir = fetch_module_sdk_artifacts(build_target, args.bid, m)
174    for f in tmpdir.iterdir():
175        repo = repo_for_sdk(f.name, mainline_modules_info)
176        dir = dir_for_sdk(f.name, args.finalize_sdk)
177        target_dir = repo.joinpath(dir)
178        if target_dir.is_dir():
179            print('Removing existing dir %s' % target_dir)
180            shutil.rmtree(target_dir)
181        with zipfile.ZipFile(tmpdir.joinpath(f)) as zipFile:
182            zipFile.extractall(target_dir)
183
184        # Disable the Android.bp, but keep it for reference / potential future use.
185        shutil.move(target_dir.joinpath('Android.bp'), target_dir.joinpath('Android.bp.auto'))
186
187        print('Created %s' % target_dir)
188        created_dirs[repo].add(dir)
189
190        # Copy api txt files to compat tracking dir
191        src_files = [Path(p) for p in glob.glob(os.path.join(target_dir, 'sdk_library/*/*.txt')) + glob.glob(os.path.join(target_dir, 'sdk_library/*/*.jar'))]
192        for src_file in src_files:
193            if is_ignored(src_file):
194                continue
195            api_type = src_file.parts[-2]
196            dest_dir = compat_dir.joinpath(api_type, 'api') if src_file.suffix == '.txt' else compat_dir.joinpath(api_type)
197            dest_file = maybe_tweak_compat_stem(dest_dir.joinpath(src_file.name))
198            os.makedirs(dest_dir, exist_ok = True)
199            shutil.copy(src_file, dest_file)
200            created_dirs[COMPAT_REPO].add(dest_dir.relative_to(COMPAT_REPO))
201
202if args.local_mode:
203    print('Updated prebuilts using locally built artifacts. Don\'t submit or use for anything besides local testing.')
204    sys.exit(0)
205
206# Do not commit any changes when the script is executed via Gantry.
207if args.gantry_download_dir:
208    sys.exit(0)
209
210if args.dry_run:
211    sys.exit(0)
212
213subprocess.check_output(['repo', 'start', branch_name] + list(created_dirs.keys()))
214print('Running git commit')
215for repo in created_dirs:
216    git = ['git', '-C', str(repo)]
217    subprocess.check_output(git + ['add'] + list(created_dirs[repo]))
218
219    if repo == COMPAT_REPO:
220        with open(COMPAT_REPO / COMPAT_README, "a") as readme:
221            readme.write(f"- {args.finalize_sdk}: {args.readme}\n")
222        subprocess.check_output(git + ['add', COMPAT_README])
223
224    if args.amend_last_commit:
225        change_id_match = re.search(r'Change-Id: [^\\n]+', str(subprocess.check_output(git + ['log', '-1'])))
226        if change_id_match:
227            change_id = '\n' + change_id_match.group(0)
228        else:
229            fail('FAIL: Unable to find change_id of the last commit.')
230        subprocess.check_output(git + ['commit', '--amend', '-m', commit_message + change_id])
231    else:
232        subprocess.check_output(git + ['commit', '-m', commit_message])
233