1# 2# Copyright (C) 2018 The Android Open Source Project 3# 4# Licensed under the Apache License, Version 2.0 (the "License"); 5# you may not use this file except in compliance with the License. 6# You may obtain a copy of the License at 7# 8# http://www.apache.org/licenses/LICENSE-2.0 9# 10# Unless required by applicable law or agreed to in writing, software 11# distributed under the License is distributed on an "AS IS" BASIS, 12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13# See the License for the specific language governing permissions and 14# limitations under the License. 15"""A commandline tool to check and update packages in external/ 16 17Example usage: 18updater.sh checkall 19updater.sh update kotlinc 20updater.sh update --refresh --keep_date rust/crates/libc 21""" 22 23import argparse 24from collections.abc import Iterable 25import json 26import logging 27import os 28import shutil 29import subprocess 30import textwrap 31import time 32from typing import Dict, Iterator, List, Union, Tuple, Type 33from pathlib import Path 34 35from base_updater import Updater 36from color import Color, color_string 37from crates_updater import CratesUpdater 38from git_updater import GitUpdater 39from github_archive_updater import GithubArchiveUpdater 40import fileutils 41import git_utils 42# pylint: disable=import-error 43import metadata_pb2 # type: ignore 44import updater_utils 45 46UPDATERS: List[Type[Updater]] = [ 47 CratesUpdater, 48 GithubArchiveUpdater, 49 GitUpdater, 50] 51 52TMP_BRANCH_NAME = 'tmp_auto_upgrade' 53 54 55def build_updater(proj_path: Path) -> Tuple[Updater, metadata_pb2.MetaData]: 56 """Build updater for a project specified by proj_path. 57 58 Reads and parses METADATA file. And builds updater based on the information. 59 60 Args: 61 proj_path: Absolute or relative path to the project. 62 63 Returns: 64 The updater object built. None if there's any error. 65 """ 66 git_utils.repo_sync(proj_path) 67 proj_path = fileutils.get_absolute_project_path(proj_path) 68 metadata = fileutils.read_metadata(proj_path) 69 metadata = fileutils.convert_url_to_identifier(metadata) 70 updater = updater_utils.create_updater(metadata, proj_path, UPDATERS) 71 return updater, metadata 72 73 74def commit_message_generator(project_name: str, version: str, path: str, bug: int | None = None) -> str: 75 header = f"Upgrade {project_name} to {version}\n" 76 body = textwrap.dedent(f""" 77 This project was upgraded with external_updater. 78 Usage: tools/external_updater/updater.sh update external/{path} 79 For more info, check https://cs.android.com/android/platform/superproject/main/+/main:tools/external_updater/README.md\n\n""") 80 if bug is None: 81 footer = "Test: TreeHugger" 82 else: 83 footer = f"Bug: {bug}\nTest: TreeHugger" 84 return header + body + footer 85 86 87def _do_update(args: argparse.Namespace, updater: Updater, 88 metadata: metadata_pb2.MetaData) -> None: 89 full_path = updater.project_path 90 91 if not args.keep_local_changes: 92 git_utils.detach_to_android_head(full_path) 93 if TMP_BRANCH_NAME in git_utils.list_local_branches(full_path): 94 git_utils.delete_branch(full_path, TMP_BRANCH_NAME) 95 git_utils.reset_hard(full_path) 96 git_utils.clean(full_path) 97 git_utils.start_branch(full_path, TMP_BRANCH_NAME) 98 try: 99 tmp_dir_of_old_version = updater.update() 100 bp_files = fileutils.find_local_bp_files(full_path, updater.latest_version) 101 fileutils.bpfmt(full_path, bp_files) 102 updated_metadata = updater.update_metadata(metadata) 103 fileutils.write_metadata(full_path, updated_metadata, args.keep_date) 104 105 try: 106 rel_proj_path = str(fileutils.get_relative_project_path(full_path)) 107 except ValueError: 108 # Absolute paths to other trees will not be relative to our tree. 109 # There are no portable instructions for upgrading that project, 110 # since the path will differ between machines (or checkouts). 111 rel_proj_path = "<absolute path to project>" 112 commit_message = commit_message_generator(metadata.name, updater.latest_version, rel_proj_path, args.bug) 113 git_utils.remove_gitmodules(full_path) 114 git_utils.add_file(full_path, '*') 115 git_utils.commit(full_path, commit_message, args.no_verify) 116 117 if not args.skip_post_update: 118 if tmp_dir_of_old_version: 119 updater_utils.run_post_update(full_path, tmp_dir_of_old_version) 120 else: 121 updater_utils.run_post_update(full_path) 122 git_utils.add_file(full_path, '*') 123 git_utils.commit_amend(full_path) 124 125 if args.build: 126 try: 127 updater_utils.build(full_path) 128 except subprocess.CalledProcessError: 129 logging.exception("Build failed, aborting upload") 130 return 131 except Exception as err: 132 if updater.rollback(): 133 print('Rolled back.') 134 raise err 135 136 if not args.no_upload: 137 git_utils.push(full_path, args.remote_name, updater.has_errors) 138 139 140def has_new_version(updater: Updater) -> bool: 141 """Checks if a newer version of the project is available.""" 142 if updater.latest_version is not None and updater.current_version != updater.latest_version: 143 return True 144 return False 145 146 147def print_project_status(updater: Updater) -> None: 148 """Prints the current status of the project on console.""" 149 150 current_version = updater.current_version 151 latest_version = updater.latest_version 152 alternative_latest_version = updater.alternative_latest_version 153 154 print(f'Current version: {current_version}') 155 print('Latest version: ', end='') 156 if not latest_version: 157 print(color_string('Not available', Color.STALE)) 158 else: 159 print(latest_version) 160 if alternative_latest_version is not None: 161 print(f'Alternative latest version: {alternative_latest_version}') 162 if has_new_version(updater): 163 print(color_string('Out of date!', Color.STALE)) 164 else: 165 print(color_string('Up to date.', Color.FRESH)) 166 167 168def find_ver_types(current_version: str) -> Tuple[str, str]: 169 if git_utils.is_commit(current_version): 170 alternative_ver_type = 'tag' 171 latest_ver_type = 'sha' 172 else: 173 alternative_ver_type = 'sha' 174 latest_ver_type = 'tag' 175 return latest_ver_type, alternative_ver_type 176 177 178def use_alternative_version(updater: Updater) -> bool: 179 """This function only runs when there is an alternative version available.""" 180 181 latest_ver_type, alternative_ver_type = find_ver_types(updater.current_version) 182 latest_version = updater.latest_version 183 alternative_version = updater.alternative_latest_version 184 new_version_available = has_new_version(updater) 185 186 out_of_date_question = f'Would you like to upgrade to {alternative_ver_type} {alternative_version} instead of {latest_ver_type} {latest_version}? (yes/no)\n' 187 up_to_date_question = f'Would you like to upgrade to {alternative_ver_type} {alternative_version}? (yes/no)\n' 188 recom_message = color_string(f'We recommend upgrading to {alternative_ver_type} {alternative_version} instead. ', Color.FRESH) 189 not_recom_message = color_string(f'We DO NOT recommend upgrading to {alternative_ver_type} {alternative_version}. ', Color.STALE) 190 191 # If alternative_version is not None, there are four possible 192 # scenarios: 193 # Scenario 1, out of date, we recommend switching to tag: 194 # Current version: sha1 195 # Latest version: sha2 196 # Alternative latest version: tag 197 198 # Scenario 2, up to date, we DO NOT recommend switching to sha. 199 # Current version: tag1 200 # Latest version: tag1 201 # Alternative latest version: sha 202 203 # Scenario 3, out of date, we DO NOT recommend switching to sha. 204 # Current version: tag1 205 # Latest version: tag2 206 # Alternative latest version: sha 207 208 # Scenario 4, out of date, no recommendations at all 209 # Current version: sha1 210 # Latest version: No tag found or a tag that doesn't belong to any branch 211 # Alternative latest version: sha 212 213 if alternative_ver_type == 'tag': 214 warning = out_of_date_question + recom_message 215 else: 216 if not new_version_available: 217 warning = up_to_date_question + not_recom_message 218 else: 219 if not latest_version: 220 warning = up_to_date_question 221 else: 222 warning = out_of_date_question + not_recom_message 223 224 answer = input(warning) 225 if "yes".startswith(answer.lower()): 226 return True 227 elif answer.lower().startswith("no"): 228 return False 229 # If user types something that is not "yes" or "no" or something similar, abort. 230 else: 231 raise ValueError(f"Invalid input: {answer}") 232 233 234def check_and_update(args: argparse.Namespace, 235 proj_path: Path, 236 update_lib=False) -> Union[Updater, str]: 237 """Checks updates for a project. 238 239 Args: 240 args: commandline arguments 241 proj_path: Absolute or relative path to the project. 242 update_lib: If false, will only check for new version, but not update. 243 """ 244 245 try: 246 canonical_path = fileutils.canonicalize_project_path(proj_path) 247 print(f'Checking {canonical_path}...') 248 updater, metadata = build_updater(proj_path) 249 updater.check() 250 251 new_version_available = has_new_version(updater) 252 print_project_status(updater) 253 254 if update_lib: 255 if args.custom_version is not None: 256 updater.set_custom_version(args.custom_version) 257 print(f"Upgrading to custom version {args.custom_version}") 258 elif args.refresh: 259 updater.refresh_without_upgrading() 260 elif new_version_available: 261 if updater.alternative_latest_version is not None: 262 if use_alternative_version(updater): 263 updater.set_new_version(updater.alternative_latest_version) 264 else: 265 return updater 266 _do_update(args, updater, metadata) 267 return updater 268 269 # pylint: disable=broad-except 270 except Exception as err: 271 logging.exception("Failed to check or update %s", proj_path) 272 return str(err) 273 274 275def check_and_update_path(args: argparse.Namespace, paths: Iterable[Path], 276 update_lib: bool, 277 delay: int) -> Dict[str, Dict[str, str]]: 278 results = {} 279 for path in paths: 280 res = {} 281 updater = check_and_update(args, path, update_lib) 282 if isinstance(updater, str): 283 res['error'] = updater 284 else: 285 res['current'] = updater.current_version 286 res['latest'] = updater.latest_version 287 results[str(fileutils.canonicalize_project_path(path))] = res 288 time.sleep(delay) 289 return results 290 291 292def _list_all_metadata() -> Iterator[str]: 293 for path, dirs, files in os.walk(fileutils.external_path()): 294 if fileutils.METADATA_FILENAME in files: 295 # Skip sub directories. 296 dirs[:] = [] 297 yield path 298 dirs.sort(key=lambda d: d.lower()) 299 300 301def write_json(json_file: str, results: Dict[str, Dict[str, str]]) -> None: 302 """Output a JSON report.""" 303 with Path(json_file).open('w', encoding='utf-8') as res_file: 304 json.dump(results, res_file, sort_keys=True, indent=4) 305 306 307def validate(args: argparse.Namespace) -> None: 308 """Handler for validate command.""" 309 paths = fileutils.resolve_command_line_paths(args.paths) 310 try: 311 canonical_path = fileutils.canonicalize_project_path(paths[0]) 312 print(f'Validating {canonical_path}') 313 updater, _ = build_updater(paths[0]) 314 print(updater.validate()) 315 except Exception: # pylint: disable=broad-exception-caught 316 logging.exception("Failed to check or update %s", paths) 317 318 319def check(args: argparse.Namespace) -> None: 320 """Handler for check command.""" 321 if args.all: 322 paths = [Path(p) for p in _list_all_metadata()] 323 else: 324 paths = fileutils.resolve_command_line_paths(args.paths) 325 results = check_and_update_path(args, paths, False, args.delay) 326 327 if args.json_output is not None: 328 write_json(args.json_output, results) 329 330 331def update(args: argparse.Namespace) -> None: 332 """Handler for update command.""" 333 all_paths = fileutils.resolve_command_line_paths(args.paths) 334 # Remove excluded paths. 335 excludes = set() if args.exclude is None else set(args.exclude) 336 filtered_paths = [path for path in all_paths 337 if not path.name in excludes] 338 # Now we can update each path. 339 results = check_and_update_path(args, filtered_paths, True, 0) 340 341 if args.json_output is not None: 342 write_json(args.json_output, results) 343 344 345def parse_args() -> argparse.Namespace: 346 """Parses commandline arguments.""" 347 348 parser = argparse.ArgumentParser( 349 prog='tools/external_updater/updater.sh', 350 description='Check updates for third party projects in external/.') 351 subparsers = parser.add_subparsers(dest='cmd') 352 subparsers.required = True 353 354 # Creates parser for check command. 355 check_parser = subparsers.add_parser( 356 'check', 357 help='Check update for one project.') 358 check_parser.add_argument( 359 'paths', 360 nargs='*', 361 help='Paths of the project. ' 362 'Relative paths will be resolved from external/.') 363 check_parser.add_argument( 364 '--json-output', 365 help='Path of a json file to write result to.') 366 check_parser.add_argument( 367 '--all', 368 action='store_true', 369 help='If set, check updates for all supported projects.') 370 check_parser.add_argument( 371 '--delay', 372 default=0, 373 type=int, 374 help='Time in seconds to wait between checking two projects.') 375 check_parser.set_defaults(func=check) 376 377 # Creates parser for update command. 378 update_parser = subparsers.add_parser( 379 'update', 380 help='Update one project.') 381 update_parser.add_argument( 382 'paths', 383 nargs='*', 384 help='Paths of the project as globs.') 385 update_parser.add_argument( 386 '--no-build', 387 action='store_false', 388 dest='build', 389 help='Skip building') 390 update_parser.add_argument( 391 '--no-upload', 392 action='store_true', 393 help='Does not upload to Gerrit after upgrade') 394 update_parser.add_argument( 395 '--bug', 396 type=int, 397 help='Bug number for this update') 398 update_parser.add_argument( 399 '--custom-version', 400 type=str, 401 help='Custom version we want to upgrade to.') 402 update_parser.add_argument( 403 '--skip-post-update', 404 action='store_true', 405 help='Skip post_update script if post_update script exists') 406 update_parser.add_argument( 407 '--keep-local-changes', 408 action='store_true', 409 help='Updates the current branch instead of creating a new branch') 410 update_parser.add_argument( 411 '--no-verify', 412 action='store_true', 413 help='Pass --no-verify to git commit') 414 update_parser.add_argument( 415 '--remote-name', 416 default='aosp', 417 required=False, 418 help='Remote repository name, the default is set to aosp') 419 update_parser.add_argument( 420 '--exclude', 421 action='append', 422 help='Names of projects to exclude. ' 423 'These are just the final part of the path ' 424 'with no directories.') 425 update_parser.add_argument( 426 '--refresh', 427 help='Run update and refresh to the current version.', 428 action='store_true') 429 update_parser.add_argument( 430 '--keep-date', 431 help='Run update and do not change date in METADATA.', 432 action='store_true') 433 update_parser.add_argument( 434 '--json-output', 435 help='Path of a json file to write result to.') 436 update_parser.set_defaults(func=update) 437 438 diff_parser = subparsers.add_parser( 439 'validate', 440 help='Check if aosp version is what it claims to be.') 441 diff_parser.add_argument( 442 'paths', 443 nargs='*', 444 help='Paths of the project.' 445 'Relative paths will be resolved from external/.') 446 diff_parser.set_defaults(func=validate) 447 448 return parser.parse_args() 449 450 451def main() -> None: 452 """The main entry.""" 453 454 args = parse_args() 455 args.func(args) 456 457 458if __name__ == '__main__': 459 main() 460