1#!/usr/bin/python3 2 3import argparse 4import glob 5import os 6import re 7import shlex 8import shutil 9import subprocess 10import sys 11import tempfile 12import zipfile 13 14from collections import defaultdict 15from pathlib import Path 16 17# See go/fetch_artifact for details on this script. 18FETCH_ARTIFACT = '/google/data/ro/projects/android/fetch_artifact' 19COMPAT_REPO = Path('prebuilts/sdk') 20COMPAT_README = Path('extensions/README.md') 21# This build target is used when fetching from a train build (TXXXXXXXX) 22BUILD_TARGET_TRAIN = 'train_build' 23# This build target is used when fetching from a non-train build (XXXXXXXX) 24BUILD_TARGET_CONTINUOUS = 'mainline_modules_sdks-userdebug' 25# The glob of sdk artifacts to fetch from remote build 26ARTIFACT_PATTERN = 'mainline-sdks/for-latest-build/current/{module_name}/sdk/*.zip' 27# The glob of sdk artifacts to fetch from local build 28ARTIFACT_LOCAL_PATTERN = 'out/dist/mainline-sdks/for-latest-build/current/{module_name}/sdk/*.zip' 29COMMIT_TEMPLATE = """Finalize artifacts for extension SDK %d 30 31Import from build id %s. 32 33Generated with: 34$ %s 35 36Bug: %d 37Test: presubmit""" 38 39def fail(*args, **kwargs): 40 print(*args, file=sys.stderr, **kwargs) 41 sys.exit(1) 42 43def fetch_artifacts(target, build_id, module_name): 44 tmpdir = Path(tempfile.TemporaryDirectory().name) 45 tmpdir.mkdir() 46 if args.local_mode: 47 artifact_path = ARTIFACT_LOCAL_PATTERN.format(module_name='*') 48 print('Copying %s to %s ...' % (artifact_path, tmpdir)) 49 for file in glob.glob(artifact_path): 50 shutil.copy(file, tmpdir) 51 else: 52 artifact_path = ARTIFACT_PATTERN.format(module_name=module_name) 53 print('Fetching %s from %s ...' % (artifact_path, target)) 54 fetch_cmd = [FETCH_ARTIFACT] 55 fetch_cmd.extend(['--bid', str(build_id)]) 56 fetch_cmd.extend(['--target', target]) 57 fetch_cmd.append(artifact_path) 58 fetch_cmd.append(str(tmpdir)) 59 print("Running: " + ' '.join(fetch_cmd)) 60 try: 61 subprocess.check_output(fetch_cmd, stderr=subprocess.STDOUT) 62 except subprocess.CalledProcessError: 63 fail('FAIL: Unable to retrieve %s artifact for build ID %s' % (artifact_path, build_id)) 64 return tmpdir 65 66def repo_for_sdk(filename): 67 module = filename.split('-')[0] 68 target_dir = '' 69 if module == 'btservices': return Path('prebuilts/module_sdk/Bluetooth') 70 if module == 'media': return Path('prebuilts/module_sdk/Media') 71 if module == 'rkpd': return Path('prebuilts/module_sdk/RemoteKeyProvisioning') 72 if module == 'tethering': return Path('prebuilts/module_sdk/Connectivity') 73 for dir in os.listdir('prebuilts/module_sdk/'): 74 if module.lower() in dir.lower(): 75 if target_dir: 76 fail('Multiple target dirs matched "%s": %s' % (module, (target_dir, dir))) 77 target_dir = dir 78 if not target_dir: 79 fail('Could not find a target dir for %s' % filename) 80 81 return Path('prebuilts/module_sdk/%s' % target_dir) 82 83def dir_for_sdk(filename, version): 84 base = str(version) 85 if 'test-exports' in filename: 86 return os.path.join(base, 'test-exports') 87 if 'host-exports' in filename: 88 return os.path.join(base, 'host-exports') 89 return base 90 91def is_ignored(file): 92 # Conscrypt has some legacy API tracking files that we don't consider for extensions. 93 bad_stem_prefixes = ['conscrypt.module.intra.core.api', 'conscrypt.module.platform.api'] 94 return any([file.stem.startswith(p) for p in bad_stem_prefixes]) 95 96 97def maybe_tweak_compat_stem(file): 98 # For legacy reasons, art and conscrypt txt file names in the SDKs (*.module.public.api) 99 # do not match their expected filename in prebuilts/sdk (art, conscrypt). So rename them 100 # to match. 101 new_stem = file.stem 102 new_stem = new_stem.replace('art.module.public.api', 'art') 103 new_stem = new_stem.replace('conscrypt.module.public.api', 'conscrypt') 104 105 # The stub jar artifacts from official builds are named '*-stubs.jar', but 106 # the convention for the copies in prebuilts/sdk is just '*.jar'. Fix that. 107 new_stem = new_stem.replace('-stubs', '') 108 109 return file.with_stem(new_stem) 110 111if not os.path.isdir('build/soong'): 112 fail("This script must be run from the top of an Android source tree.") 113 114parser = argparse.ArgumentParser(description=('Finalize an extension SDK with prebuilts')) 115parser.add_argument('-f', '--finalize_sdk', type=int, required=True, help='The numbered SDK to finalize.') 116parser.add_argument('-b', '--bug', type=int, required=True, help='The bug number to add to the commit message.') 117parser.add_argument('-r', '--readme', required=True, help='Version history entry to add to %s' % (COMPAT_REPO / COMPAT_README)) 118parser.add_argument('-a', '--amend_last_commit', action="store_true", help='Amend current HEAD commits instead of making new commits.') 119parser.add_argument('-m', '--modules', action='append', help='Modules to include. Can be provided multiple times, or not at all for all modules.') 120parser.add_argument('-l', '--local_mode', action="store_true", help='Local mode: use locally built artifacts and don\'t upload the result to Gerrit.') 121parser.add_argument('bid', help='Build server build ID') 122args = parser.parse_args() 123 124build_target = BUILD_TARGET_TRAIN if args.bid[0] == 'T' else BUILD_TARGET_CONTINUOUS 125branch_name = 'finalize-%d' % args.finalize_sdk 126cmdline = shlex.join([x for x in sys.argv if x not in ['-a', '--amend_last_commit', '-l', '--local_mode']]) 127commit_message = COMMIT_TEMPLATE % (args.finalize_sdk, args.bid, cmdline, args.bug) 128module_names = args.modules or ['*'] 129 130compat_dir = COMPAT_REPO.joinpath('extensions/%d' % args.finalize_sdk) 131if compat_dir.is_dir(): 132 print('Removing existing dir %s' % compat_dir) 133 shutil.rmtree(compat_dir) 134 135created_dirs = defaultdict(set) 136for m in module_names: 137 tmpdir = fetch_artifacts(build_target, args.bid, m) 138 for f in tmpdir.iterdir(): 139 repo = repo_for_sdk(f.name) 140 dir = dir_for_sdk(f.name, args.finalize_sdk) 141 target_dir = repo.joinpath(dir) 142 if target_dir.is_dir(): 143 print('Removing existing dir %s' % target_dir) 144 shutil.rmtree(target_dir) 145 with zipfile.ZipFile(tmpdir.joinpath(f)) as zipFile: 146 zipFile.extractall(target_dir) 147 148 # Disable the Android.bp, but keep it for reference / potential future use. 149 shutil.move(target_dir.joinpath('Android.bp'), target_dir.joinpath('Android.bp.auto')) 150 151 print('Created %s' % target_dir) 152 created_dirs[repo].add(dir) 153 154 # Copy api txt files to compat tracking dir 155 src_files = [Path(p) for p in glob.glob(os.path.join(target_dir, 'sdk_library/*/*.txt')) + glob.glob(os.path.join(target_dir, 'sdk_library/*/*.jar'))] 156 for src_file in src_files: 157 if is_ignored(src_file): 158 continue 159 api_type = src_file.parts[-2] 160 dest_dir = compat_dir.joinpath(api_type, 'api') if src_file.suffix == '.txt' else compat_dir.joinpath(api_type) 161 dest_file = maybe_tweak_compat_stem(dest_dir.joinpath(src_file.name)) 162 os.makedirs(dest_dir, exist_ok = True) 163 shutil.copy(src_file, dest_file) 164 created_dirs[COMPAT_REPO].add(dest_dir.relative_to(COMPAT_REPO)) 165 166if args.local_mode: 167 print('Updated prebuilts using locally built artifacts. Don\'t submit or use for anything besides local testing.') 168 sys.exit(0) 169 170subprocess.check_output(['repo', 'start', branch_name] + list(created_dirs.keys())) 171print('Running git commit') 172for repo in created_dirs: 173 git = ['git', '-C', str(repo)] 174 subprocess.check_output(git + ['add'] + list(created_dirs[repo])) 175 176 if repo == COMPAT_REPO: 177 with open(COMPAT_REPO / COMPAT_README, "a") as readme: 178 readme.write(f"- {args.finalize_sdk}: {args.readme}\n") 179 subprocess.check_output(git + ['add', COMPAT_README]) 180 181 if args.amend_last_commit: 182 change_id = '\n' + re.search(r'Change-Id: [^\\n]+', str(subprocess.check_output(git + ['log', '-1']))).group(0) 183 subprocess.check_output(git + ['commit', '--amend', '-m', commit_message + change_id]) 184 else: 185 subprocess.check_output(git + ['commit', '-m', commit_message]) 186