1#!/usr/bin/env python3 2# -*- coding: utf-8 -*- 3# Copyright 2019 The Chromium OS Authors. All rights reserved. 4# Use of this source code is governed by a BSD-style license that can be 5# found in the LICENSE file. 6 7"""Updates the LLVM hash and uprevs the build of the specified packages. 8 9For each package, a temporary repo is created and the changes are uploaded 10for review. 11""" 12 13from __future__ import print_function 14 15import argparse 16import datetime 17import enum 18import os 19import re 20import subprocess 21 22import chroot 23import failure_modes 24import get_llvm_hash 25import git 26import llvm_patch_management 27 28DEFAULT_PACKAGES = [ 29 'dev-util/lldb-server', 30 'sys-devel/llvm', 31 'sys-libs/compiler-rt', 32 'sys-libs/libcxx', 33 'sys-libs/libcxxabi', 34 'sys-libs/llvm-libunwind', 35] 36 37 38# Specify which LLVM hash to update 39class LLVMVariant(enum.Enum): 40 """Represent the LLVM hash in an ebuild file to update.""" 41 42 current = 'LLVM_HASH' 43 next = 'LLVM_NEXT_HASH' 44 45 46# If set to `True`, then the contents of `stdout` after executing a command will 47# be displayed to the terminal. 48verbose = False 49 50 51def defaultCrosRoot(): 52 """Get default location of chroot_path. 53 54 The logic assumes that the cros_root is ~/chromiumos, unless llvm_tools is 55 inside of a CrOS checkout, in which case that checkout should be used. 56 57 Returns: 58 The best guess location for the cros checkout. 59 """ 60 llvm_tools_path = os.path.realpath(os.path.dirname(__file__)) 61 if llvm_tools_path.endswith('src/third_party/toolchain-utils/llvm_tools'): 62 return os.path.join(llvm_tools_path, '../../../../') 63 return '~/chromiumos' 64 65 66def GetCommandLineArgs(): 67 """Parses the command line for the optional command line arguments. 68 69 Returns: 70 The log level to use when retrieving the LLVM hash or google3 LLVM version, 71 the chroot path to use for executing chroot commands, 72 a list of a package or packages to update their LLVM next hash, 73 and the LLVM version to use when retrieving the LLVM hash. 74 """ 75 76 # Create parser and add optional command-line arguments. 77 parser = argparse.ArgumentParser( 78 description="Updates the build's hash for llvm-next.") 79 80 # Add argument for a specific chroot path. 81 parser.add_argument('--chroot_path', 82 default=defaultCrosRoot(), 83 help='the path to the chroot (default: %(default)s)') 84 85 # Add argument for specific builds to uprev and update their llvm-next hash. 86 parser.add_argument('--update_packages', 87 default=DEFAULT_PACKAGES, 88 required=False, 89 nargs='+', 90 help='the ebuilds to update their hash for llvm-next ' 91 '(default: %(default)s)') 92 93 # Add argument for whether to display command contents to `stdout`. 94 parser.add_argument('--verbose', 95 action='store_true', 96 help='display contents of a command to the terminal ' 97 '(default: %(default)s)') 98 99 # Add argument for the LLVM hash to update 100 parser.add_argument( 101 '--is_llvm_next', 102 action='store_true', 103 help='which llvm hash to update. If specified, update LLVM_NEXT_HASH. ' 104 'Otherwise, update LLVM_HASH') 105 106 # Add argument for the LLVM version to use. 107 parser.add_argument( 108 '--llvm_version', 109 type=get_llvm_hash.IsSvnOption, 110 required=True, 111 help='which git hash to use. Either a svn revision, or one ' 112 'of %s' % sorted(get_llvm_hash.KNOWN_HASH_SOURCES)) 113 114 # Add argument for the mode of the patch management when handling patches. 115 parser.add_argument( 116 '--failure_mode', 117 default=failure_modes.FailureModes.FAIL.value, 118 choices=[ 119 failure_modes.FailureModes.FAIL.value, 120 failure_modes.FailureModes.CONTINUE.value, 121 failure_modes.FailureModes.DISABLE_PATCHES.value, 122 failure_modes.FailureModes.REMOVE_PATCHES.value 123 ], 124 help='the mode of the patch manager when handling failed patches ' 125 '(default: %(default)s)') 126 127 # Add argument for the patch metadata file. 128 parser.add_argument( 129 '--patch_metadata_file', 130 default='PATCHES.json', 131 help='the .json file that has all the patches and their ' 132 'metadata if applicable (default: PATCHES.json inside $FILESDIR)') 133 134 # Parse the command line. 135 args_output = parser.parse_args() 136 137 # FIXME: We shouldn't be using globals here, but until we fix it, make pylint 138 # stop complaining about it. 139 # pylint: disable=global-statement 140 global verbose 141 142 verbose = args_output.verbose 143 144 return args_output 145 146 147def GetEbuildPathsFromSymLinkPaths(symlinks): 148 """Reads the symlink(s) to get the ebuild path(s) to the package(s). 149 150 Args: 151 symlinks: A list of absolute path symlink/symlinks that point 152 to the package's ebuild. 153 154 Returns: 155 A dictionary where the key is the absolute path of the symlink and the value 156 is the absolute path to the ebuild that was read from the symlink. 157 158 Raises: 159 ValueError: Invalid symlink(s) were provided. 160 """ 161 162 # A dictionary that holds: 163 # key: absolute symlink path 164 # value: absolute ebuild path 165 resolved_paths = {} 166 167 # Iterate through each symlink. 168 # 169 # For each symlink, check that it is a valid symlink, 170 # and then construct the ebuild path, and 171 # then add the ebuild path to the dict. 172 for cur_symlink in symlinks: 173 if not os.path.islink(cur_symlink): 174 raise ValueError('Invalid symlink provided: %s' % cur_symlink) 175 176 # Construct the absolute path to the ebuild. 177 ebuild_path = os.path.realpath(cur_symlink) 178 179 if cur_symlink not in resolved_paths: 180 resolved_paths[cur_symlink] = ebuild_path 181 182 return resolved_paths 183 184 185def UpdateEbuildLLVMHash(ebuild_path, llvm_variant, git_hash, svn_version): 186 """Updates the LLVM hash in the ebuild. 187 188 The build changes are staged for commit in the temporary repo. 189 190 Args: 191 ebuild_path: The absolute path to the ebuild. 192 llvm_variant: Which LLVM hash to update. 193 git_hash: The new git hash. 194 svn_version: The SVN-style revision number of git_hash. 195 196 Raises: 197 ValueError: Invalid ebuild path provided or failed to stage the commit 198 of the changes or failed to update the LLVM hash. 199 """ 200 201 # Iterate through each ebuild. 202 # 203 # For each ebuild, read the file in 204 # advance and then create a temporary file 205 # that gets updated with the new LLVM hash 206 # and revision number and then the ebuild file 207 # gets updated to the temporary file. 208 209 if not os.path.isfile(ebuild_path): 210 raise ValueError('Invalid ebuild path provided: %s' % ebuild_path) 211 212 temp_ebuild_file = '%s.temp' % ebuild_path 213 214 with open(ebuild_path) as ebuild_file: 215 # write updates to a temporary file in case of interrupts 216 with open(temp_ebuild_file, 'w') as temp_file: 217 for cur_line in ReplaceLLVMHash(ebuild_file, llvm_variant, git_hash, 218 svn_version): 219 temp_file.write(cur_line) 220 os.rename(temp_ebuild_file, ebuild_path) 221 222 # Get the path to the parent directory. 223 parent_dir = os.path.dirname(ebuild_path) 224 225 # Stage the changes. 226 subprocess.check_output(['git', '-C', parent_dir, 'add', ebuild_path]) 227 228 229def ReplaceLLVMHash(ebuild_lines, llvm_variant, git_hash, svn_version): 230 """Updates the LLVM git hash. 231 232 Args: 233 ebuild_lines: The contents of the ebuild file. 234 llvm_variant: The LLVM hash to update. 235 git_hash: The new git hash. 236 svn_version: The SVN-style revision number of git_hash. 237 238 Yields: 239 lines of the modified ebuild file 240 """ 241 is_updated = False 242 llvm_regex = re.compile('^' + re.escape(llvm_variant.value) + 243 '=\"[a-z0-9]+\"') 244 for cur_line in ebuild_lines: 245 if not is_updated and llvm_regex.search(cur_line): 246 # Update the git hash and revision number. 247 cur_line = '%s=\"%s\" # r%d\n' % (llvm_variant.value, git_hash, 248 svn_version) 249 250 is_updated = True 251 252 yield cur_line 253 254 if not is_updated: 255 raise ValueError('Failed to update %s' % llvm_variant.value) 256 257 258def UprevEbuildSymlink(symlink): 259 """Uprevs the symlink's revision number. 260 261 Increases the revision number by 1 and stages the change in 262 the temporary repo. 263 264 Args: 265 symlink: The absolute path of an ebuild symlink. 266 267 Raises: 268 ValueError: Failed to uprev the symlink or failed to stage the changes. 269 """ 270 271 if not os.path.islink(symlink): 272 raise ValueError('Invalid symlink provided: %s' % symlink) 273 274 new_symlink, is_changed = re.subn( 275 r'r([0-9]+).ebuild', 276 lambda match: 'r%s.ebuild' % str(int(match.group(1)) + 1), 277 symlink, 278 count=1) 279 280 if not is_changed: 281 raise ValueError('Failed to uprev the symlink.') 282 283 # rename the symlink 284 subprocess.check_output( 285 ['git', '-C', 286 os.path.dirname(symlink), 'mv', symlink, new_symlink]) 287 288 289def UprevEbuildToVersion(symlink, svn_version, git_hash): 290 """Uprevs the ebuild's revision number. 291 292 Increases the revision number by 1 and stages the change in 293 the temporary repo. 294 295 Args: 296 symlink: The absolute path of an ebuild symlink. 297 svn_version: The SVN-style revision number of git_hash. 298 git_hash: The new git hash. 299 300 Raises: 301 ValueError: Failed to uprev the ebuild or failed to stage the changes. 302 AssertionError: No llvm version provided for an LLVM uprev 303 """ 304 305 if not os.path.islink(symlink): 306 raise ValueError('Invalid symlink provided: %s' % symlink) 307 308 ebuild = os.path.realpath(symlink) 309 llvm_major_version = get_llvm_hash.GetLLVMMajorVersion(git_hash) 310 # llvm 311 package = os.path.basename(os.path.dirname(symlink)) 312 if not package: 313 raise ValueError('Tried to uprev an unknown package') 314 if package == 'llvm': 315 new_ebuild, is_changed = re.subn( 316 r'(\d+)\.(\d+)_pre([0-9]+)_p([0-9]+)', 317 '%s.\\2_pre%s_p%s' % (llvm_major_version, svn_version, 318 datetime.datetime.today().strftime('%Y%m%d')), 319 ebuild, 320 count=1) 321 # any other package 322 else: 323 new_ebuild, is_changed = re.subn(r'(\d+)\.(\d+)_pre([0-9]+)', 324 '%s.\\2_pre%s' % 325 (llvm_major_version, svn_version), 326 ebuild, 327 count=1) 328 329 if not is_changed: # failed to increment the revision number 330 raise ValueError('Failed to uprev the ebuild.') 331 332 symlink_dir = os.path.dirname(symlink) 333 334 # Rename the ebuild 335 subprocess.check_output(['git', '-C', symlink_dir, 'mv', ebuild, new_ebuild]) 336 337 # Create a symlink of the renamed ebuild 338 new_symlink = new_ebuild[:-len('.ebuild')] + '-r1.ebuild' 339 subprocess.check_output(['ln', '-s', '-r', new_ebuild, new_symlink]) 340 341 if not os.path.islink(new_symlink): 342 raise ValueError('Invalid symlink name: %s' % new_ebuild[:-len('.ebuild')]) 343 344 subprocess.check_output(['git', '-C', symlink_dir, 'add', new_symlink]) 345 346 # Remove the old symlink 347 subprocess.check_output(['git', '-C', symlink_dir, 'rm', symlink]) 348 349 350def CreatePathDictionaryFromPackages(chroot_path, update_packages): 351 """Creates a symlink and ebuild path pair dictionary from the packages. 352 353 Args: 354 chroot_path: The absolute path to the chroot. 355 update_packages: The filtered packages to be updated. 356 357 Returns: 358 A dictionary where the key is the absolute path to the symlink 359 of the package and the value is the absolute path to the ebuild of 360 the package. 361 """ 362 363 # Construct a list containing the chroot file paths of the package(s). 364 chroot_file_paths = chroot.GetChrootEbuildPaths(chroot_path, update_packages) 365 366 # Construct a list containing the symlink(s) of the package(s). 367 symlink_file_paths = chroot.ConvertChrootPathsToAbsolutePaths( 368 chroot_path, chroot_file_paths) 369 370 # Create a dictionary where the key is the absolute path of the symlink to 371 # the package and the value is the absolute path to the ebuild of the package. 372 return GetEbuildPathsFromSymLinkPaths(symlink_file_paths) 373 374 375def RemovePatchesFromFilesDir(patches): 376 """Removes the patches from $FILESDIR of a package. 377 378 Args: 379 patches: A list of absolute pathes of patches to remove 380 381 Raises: 382 ValueError: Failed to remove a patch in $FILESDIR. 383 """ 384 385 for patch in patches: 386 subprocess.check_output( 387 ['git', '-C', os.path.dirname(patch), 'rm', '-f', patch]) 388 389 390def StagePatchMetadataFileForCommit(patch_metadata_file_path): 391 """Stages the updated patch metadata file for commit. 392 393 Args: 394 patch_metadata_file_path: The absolute path to the patch metadata file. 395 396 Raises: 397 ValueError: Failed to stage the patch metadata file for commit or invalid 398 patch metadata file. 399 """ 400 401 if not os.path.isfile(patch_metadata_file_path): 402 raise ValueError('Invalid patch metadata file provided: %s' % 403 patch_metadata_file_path) 404 405 # Cmd to stage the patch metadata file for commit. 406 subprocess.check_output([ 407 'git', '-C', 408 os.path.dirname(patch_metadata_file_path), 'add', 409 patch_metadata_file_path 410 ]) 411 412 413def StagePackagesPatchResultsForCommit(package_info_dict, commit_messages): 414 """Stages the patch results of the packages to the commit message. 415 416 Args: 417 package_info_dict: A dictionary where the key is the package name and the 418 value is a dictionary that contains information about the patches of the 419 package (key). 420 commit_messages: The commit message that has the updated ebuilds and 421 upreving information. 422 423 Returns: 424 commit_messages with new additions 425 """ 426 427 # For each package, check if any patches for that package have 428 # changed, if so, add which patches have changed to the commit 429 # message. 430 for package_name, patch_info_dict in package_info_dict.items(): 431 if (patch_info_dict['disabled_patches'] 432 or patch_info_dict['removed_patches'] 433 or patch_info_dict['modified_metadata']): 434 cur_package_header = '\nFor the package %s:' % package_name 435 commit_messages.append(cur_package_header) 436 437 # Add to the commit message that the patch metadata file was modified. 438 if patch_info_dict['modified_metadata']: 439 patch_metadata_path = patch_info_dict['modified_metadata'] 440 commit_messages.append('The patch metadata file %s was modified' % 441 os.path.basename(patch_metadata_path)) 442 443 StagePatchMetadataFileForCommit(patch_metadata_path) 444 445 # Add each disabled patch to the commit message. 446 if patch_info_dict['disabled_patches']: 447 commit_messages.append('The following patches were disabled:') 448 449 for patch_path in patch_info_dict['disabled_patches']: 450 commit_messages.append(os.path.basename(patch_path)) 451 452 # Add each removed patch to the commit message. 453 if patch_info_dict['removed_patches']: 454 commit_messages.append('The following patches were removed:') 455 456 for patch_path in patch_info_dict['removed_patches']: 457 commit_messages.append(os.path.basename(patch_path)) 458 459 RemovePatchesFromFilesDir(patch_info_dict['removed_patches']) 460 461 return commit_messages 462 463 464def UpdatePackages(packages, llvm_variant, git_hash, svn_version, chroot_path, 465 patch_metadata_file, mode, git_hash_source, 466 extra_commit_msg): 467 """Updates an LLVM hash and uprevs the ebuild of the packages. 468 469 A temporary repo is created for the changes. The changes are 470 then uploaded for review. 471 472 Args: 473 packages: A list of all the packages that are going to be updated. 474 llvm_variant: The LLVM hash to update. 475 git_hash: The new git hash. 476 svn_version: The SVN-style revision number of git_hash. 477 chroot_path: The absolute path to the chroot. 478 patch_metadata_file: The name of the .json file in '$FILESDIR/' that has 479 the patches and its metadata. 480 mode: The mode of the patch manager when handling an applicable patch 481 that failed to apply. 482 Ex. 'FailureModes.FAIL' 483 git_hash_source: The source of which git hash to use based off of. 484 Ex. 'google3', 'tot', or <version> such as 365123 485 extra_commit_msg: extra test to append to the commit message. 486 487 Returns: 488 A nametuple that has two (key, value) pairs, where the first pair is the 489 Gerrit commit URL and the second pair is the change list number. 490 """ 491 492 # Determines whether to print the result of each executed command. 493 llvm_patch_management.verbose = verbose 494 495 # Construct a dictionary where the key is the absolute path of the symlink to 496 # the package and the value is the absolute path to the ebuild of the package. 497 paths_dict = CreatePathDictionaryFromPackages(chroot_path, packages) 498 499 repo_path = os.path.dirname(next(iter(paths_dict.values()))) 500 501 branch = 'update-' + llvm_variant.value + '-' + git_hash 502 503 git.CreateBranch(repo_path, branch) 504 505 try: 506 commit_message_header = 'llvm' 507 if llvm_variant == LLVMVariant.next: 508 commit_message_header = 'llvm-next' 509 if git_hash_source in get_llvm_hash.KNOWN_HASH_SOURCES: 510 commit_message_header += ('/%s: upgrade to %s (r%d)' % 511 (git_hash_source, git_hash, svn_version)) 512 else: 513 commit_message_header += (': upgrade to %s (r%d)' % 514 (git_hash, svn_version)) 515 516 commit_messages = [ 517 commit_message_header + '\n', 518 'The following packages have been updated:', 519 ] 520 521 # Holds the list of packages that are updating. 522 packages = [] 523 524 # Iterate through the dictionary. 525 # 526 # For each iteration: 527 # 1) Update the ebuild's LLVM hash. 528 # 2) Uprev the ebuild (symlink). 529 # 3) Add the modified package to the commit message. 530 for symlink_path, ebuild_path in paths_dict.items(): 531 path_to_ebuild_dir = os.path.dirname(ebuild_path) 532 533 UpdateEbuildLLVMHash(ebuild_path, llvm_variant, git_hash, svn_version) 534 535 if llvm_variant == LLVMVariant.current: 536 UprevEbuildToVersion(symlink_path, svn_version, git_hash) 537 else: 538 UprevEbuildSymlink(symlink_path) 539 540 cur_dir_name = os.path.basename(path_to_ebuild_dir) 541 parent_dir_name = os.path.basename(os.path.dirname(path_to_ebuild_dir)) 542 543 packages.append('%s/%s' % (parent_dir_name, cur_dir_name)) 544 commit_messages.append('%s/%s' % (parent_dir_name, cur_dir_name)) 545 546 EnsurePackageMaskContains(chroot_path, git_hash) 547 548 # Handle the patches for each package. 549 package_info_dict = llvm_patch_management.UpdatePackagesPatchMetadataFile( 550 chroot_path, svn_version, patch_metadata_file, packages, mode) 551 552 # Update the commit message if changes were made to a package's patches. 553 commit_messages = StagePackagesPatchResultsForCommit( 554 package_info_dict, commit_messages) 555 556 if extra_commit_msg: 557 commit_messages.append(extra_commit_msg) 558 559 change_list = git.UploadChanges(repo_path, branch, commit_messages) 560 561 finally: 562 git.DeleteBranch(repo_path, branch) 563 564 return change_list 565 566 567def EnsurePackageMaskContains(chroot_path, git_hash): 568 """Adds the major version of llvm to package.mask if it's not already present. 569 570 Args: 571 chroot_path: The absolute path to the chroot. 572 git_hash: The new git hash. 573 574 Raises: 575 FileExistsError: package.mask not found in ../../chromiumos-overlay 576 """ 577 578 llvm_major_version = get_llvm_hash.GetLLVMMajorVersion(git_hash) 579 580 overlay_dir = os.path.join(chroot_path, 'src/third_party/chromiumos-overlay') 581 mask_path = os.path.join(overlay_dir, 582 'profiles/targets/chromeos/package.mask') 583 with open(mask_path, 'r+') as mask_file: 584 mask_contents = mask_file.read() 585 expected_line = '=sys-devel/llvm-%s.0_pre*\n' % llvm_major_version 586 if expected_line not in mask_contents: 587 mask_file.write(expected_line) 588 589 subprocess.check_output(['git', '-C', overlay_dir, 'add', mask_path]) 590 591 592def main(): 593 """Updates the LLVM next hash for each package. 594 595 Raises: 596 AssertionError: The script was run inside the chroot. 597 """ 598 599 chroot.VerifyOutsideChroot() 600 601 args_output = GetCommandLineArgs() 602 603 llvm_variant = LLVMVariant.current 604 if args_output.is_llvm_next: 605 llvm_variant = LLVMVariant.next 606 607 git_hash_source = args_output.llvm_version 608 609 git_hash, svn_version = get_llvm_hash.GetLLVMHashAndVersionFromSVNOption( 610 git_hash_source) 611 612 change_list = UpdatePackages(args_output.update_packages, 613 llvm_variant, 614 git_hash, 615 svn_version, 616 args_output.chroot_path, 617 args_output.patch_metadata_file, 618 failure_modes.FailureModes( 619 args_output.failure_mode), 620 git_hash_source, 621 extra_commit_msg=None) 622 623 print('Successfully updated packages to %s (%d)' % (git_hash, svn_version)) 624 print('Gerrit URL: %s' % change_list.url) 625 print('Change list number: %d' % change_list.cl_number) 626 627 628if __name__ == '__main__': 629 main() 630