1#!/usr/bin/env python3 2# Copyright 2019 The ANGLE project authors. All Rights Reserved. 3# 4# Use of this source code is governed by a BSD-style license 5# that can be found in the LICENSE file in the root of the source 6# tree. An additional intellectual property rights grant can be found 7# in the file PATENTS. All contributing project authors may 8# be found in the AUTHORS file in the root of the source tree. 9 10# This is a modified copy of the script in 11# https://webrtc.googlesource.com/src/+/main/tools_webrtc/autoroller/roll_deps.py 12# customized for ANGLE. 13"""Script to automatically roll Chromium dependencies in the ANGLE DEPS file.""" 14 15import argparse 16import base64 17import collections 18import logging 19import os 20import platform 21import re 22import subprocess 23import sys 24import urllib.request 25 26 27def FindSrcDirPath(): 28 """Returns the abs path to the root dir of the project.""" 29 # Special cased for ANGLE. 30 return os.path.dirname(os.path.abspath(os.path.join(__file__, '..'))) 31 32ANGLE_CHROMIUM_DEPS = [ 33 'build', 34 'buildtools', 35 'buildtools/linux64', 36 'buildtools/mac', 37 'buildtools/reclient', 38 'buildtools/win', 39 'testing', 40 'third_party/abseil-cpp', 41 'third_party/android_build_tools', 42 'third_party/android_build_tools/aapt2/cipd', 43 'third_party/android_build_tools/art', 44 'third_party/android_build_tools/bundletool', 45 'third_party/android_build_tools/error_prone/cipd', 46 'third_party/android_build_tools/error_prone_javac/cipd', 47 'third_party/android_build_tools/lint/cipd', 48 'third_party/android_build_tools/manifest_merger/cipd', 49 'third_party/android_build_tools/nullaway/cipd', 50 'third_party/android_deps', 51 'third_party/android_platform', 52 'third_party/android_sdk', 53 'third_party/android_sdk/public', 54 'third_party/android_system_sdk/cipd', 55 'third_party/android_toolchain/ndk', 56 'third_party/bazel', 57 'third_party/catapult', 58 'third_party/clang-format/script', 59 'third_party/colorama/src', 60 'third_party/cpu_features/src', 61 'third_party/depot_tools', 62 'third_party/flatbuffers/src', 63 'third_party/fuchsia-sdk/sdk', 64 'third_party/ijar', 65 'third_party/jdk', 66 'third_party/jdk/extras', 67 'third_party/jinja2', 68 'third_party/kotlin_stdlib', 69 'third_party/libc++/src', 70 'third_party/libc++abi/src', 71 'third_party/libdrm/src', 72 'third_party/libjpeg_turbo', 73 'third_party/libunwind/src', 74 'third_party/llvm-libc/src', 75 'third_party/markupsafe', 76 'third_party/nasm', 77 'third_party/ninja', 78 'third_party/proguard', 79 'third_party/protobuf', 80 'third_party/Python-Markdown', 81 'third_party/qemu-linux-x64', 82 'third_party/qemu-mac-x64', 83 'third_party/r8/cipd', 84 'third_party/r8/d8/cipd', 85 'third_party/requests/src', 86 'third_party/rust', 87 'third_party/siso/cipd', 88 'third_party/six', 89 'third_party/turbine/cipd', 90 'third_party/zlib', 91 'tools/android', 92 'tools/clang', 93 'tools/clang/dsymutil', 94 'tools/luci-go', 95 'tools/mb', 96 'tools/md_browser', 97 'tools/memory', 98 'tools/perf', 99 'tools/protoc_wrapper', 100 'tools/python', 101 'tools/rust', 102 'tools/skia_goldctl/linux', 103 'tools/skia_goldctl/mac_amd64', 104 'tools/skia_goldctl/mac_arm64', 105 'tools/skia_goldctl/win', 106 'tools/valgrind', 107] 108 109ANGLE_URL = 'https://chromium.googlesource.com/angle/angle' 110CHROMIUM_SRC_URL = 'https://chromium.googlesource.com/chromium/src' 111CHROMIUM_COMMIT_TEMPLATE = CHROMIUM_SRC_URL + '/+/%s' 112CHROMIUM_LOG_TEMPLATE = CHROMIUM_SRC_URL + '/+log/%s' 113CHROMIUM_FILE_TEMPLATE = CHROMIUM_SRC_URL + '/+/%s/%s' 114 115COMMIT_POSITION_RE = re.compile('^Cr-Commit-Position: .*#([0-9]+).*$') 116CLANG_REVISION_RE = re.compile(r'^CLANG_REVISION = \'([-0-9a-z]+)\'') 117ROLL_BRANCH_NAME = 'roll_chromium_revision' 118 119SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) 120CHECKOUT_SRC_DIR = FindSrcDirPath() 121CHECKOUT_ROOT_DIR = CHECKOUT_SRC_DIR 122 123# Copied from tools/android/roll/android_deps/.../BuildConfigGenerator.groovy. 124ANDROID_DEPS_START = r'=== ANDROID_DEPS Generated Code Start ===' 125ANDROID_DEPS_END = r'=== ANDROID_DEPS Generated Code End ===' 126# Location of automically gathered android deps. 127ANDROID_DEPS_PATH = 'src/third_party/android_deps/' 128 129NOTIFY_EMAIL = 'angle-wrangler@grotations.appspotmail.com' 130 131CLANG_TOOLS_URL = 'https://chromium.googlesource.com/chromium/src/tools/clang' 132CLANG_FILE_TEMPLATE = CLANG_TOOLS_URL + '/+/%s/%s' 133 134CLANG_TOOLS_PATH = 'tools/clang' 135CLANG_UPDATE_SCRIPT_URL_PATH = 'scripts/update.py' 136CLANG_UPDATE_SCRIPT_LOCAL_PATH = os.path.join(CHECKOUT_SRC_DIR, 'tools', 'clang', 'scripts', 137 'update.py') 138 139DepsEntry = collections.namedtuple('DepsEntry', 'path url revision') 140ChangedDep = collections.namedtuple('ChangedDep', 'path url current_rev new_rev') 141ClangChange = collections.namedtuple('ClangChange', 'mirror_change clang_change') 142CipdDepsEntry = collections.namedtuple('CipdDepsEntry', 'path packages') 143ChangedCipdPackage = collections.namedtuple('ChangedCipdPackage', 144 'path package current_version new_version') 145 146ChromiumRevisionUpdate = collections.namedtuple('ChromiumRevisionUpdate', ('current_chromium_rev ' 147 'new_chromium_rev ')) 148 149 150def AddDepotToolsToPath(): 151 sys.path.append(os.path.join(CHECKOUT_SRC_DIR, 'build')) 152 import find_depot_tools 153 find_depot_tools.add_depot_tools_to_path() 154 155 156class RollError(Exception): 157 pass 158 159 160def StrExpansion(): 161 return lambda str_value: str_value 162 163 164def VarLookup(local_scope): 165 return lambda var_name: local_scope['vars'][var_name] 166 167 168def ParseDepsDict(deps_content): 169 local_scope = {} 170 global_scope = { 171 'Str': StrExpansion(), 172 'Var': VarLookup(local_scope), 173 'deps_os': {}, 174 } 175 exec (deps_content, global_scope, local_scope) 176 return local_scope 177 178 179def ParseLocalDepsFile(filename): 180 with open(filename, 'rb') as f: 181 deps_content = f.read() 182 return ParseDepsDict(deps_content) 183 184 185def ParseCommitPosition(commit_message): 186 for line in reversed(commit_message.splitlines()): 187 m = COMMIT_POSITION_RE.match(line.strip()) 188 if m: 189 return int(m.group(1)) 190 logging.error('Failed to parse commit position id from:\n%s\n', commit_message) 191 sys.exit(-1) 192 193 194def _RunCommand(command, working_dir=None, ignore_exit_code=False, extra_env=None, 195 input_data=None): 196 """Runs a command and returns the output from that command. 197 198 If the command fails (exit code != 0), the function will exit the process. 199 200 Returns: 201 A tuple containing the stdout and stderr outputs as strings. 202 """ 203 working_dir = working_dir or CHECKOUT_SRC_DIR 204 logging.debug('CMD: %s CWD: %s', ' '.join(command), working_dir) 205 env = os.environ.copy() 206 if extra_env: 207 assert all(isinstance(value, str) for value in extra_env.values()) 208 logging.debug('extra env: %s', extra_env) 209 env.update(extra_env) 210 p = subprocess.Popen( 211 command, 212 stdin=subprocess.PIPE, 213 stdout=subprocess.PIPE, 214 stderr=subprocess.PIPE, 215 env=env, 216 cwd=working_dir, 217 universal_newlines=True) 218 std_output, err_output = p.communicate(input_data) 219 p.stdout.close() 220 p.stderr.close() 221 if not ignore_exit_code and p.returncode != 0: 222 logging.error('Command failed: %s\n' 223 'stdout:\n%s\n' 224 'stderr:\n%s\n', ' '.join(command), std_output, err_output) 225 sys.exit(p.returncode) 226 return std_output, err_output 227 228 229def _GetBranches(): 230 """Returns a tuple of active,branches. 231 232 The 'active' is the name of the currently active branch and 'branches' is a 233 list of all branches. 234 """ 235 lines = _RunCommand(['git', 'branch'])[0].split('\n') 236 branches = [] 237 active = '' 238 for line in lines: 239 if '*' in line: 240 # The assumption is that the first char will always be the '*'. 241 active = line[1:].strip() 242 branches.append(active) 243 else: 244 branch = line.strip() 245 if branch: 246 branches.append(branch) 247 return active, branches 248 249 250def _ReadGitilesContent(url): 251 # Download and decode BASE64 content until 252 # https://code.google.com/p/gitiles/issues/detail?id=7 is fixed. 253 logging.debug('Reading gitiles URL %s' % url) 254 base64_content = ReadUrlContent(url + '?format=TEXT') 255 return base64.b64decode(base64_content[0]).decode('utf-8') 256 257 258def ReadRemoteCrFile(path_below_src, revision): 259 """Reads a remote Chromium file of a specific revision. Returns a string.""" 260 return _ReadGitilesContent(CHROMIUM_FILE_TEMPLATE % (revision, path_below_src)) 261 262 263def ReadRemoteCrCommit(revision): 264 """Reads a remote Chromium commit message. Returns a string.""" 265 return _ReadGitilesContent(CHROMIUM_COMMIT_TEMPLATE % revision) 266 267 268def ReadRemoteClangFile(path_below_src, revision): 269 """Reads a remote Clang file of a specific revision. Returns a string.""" 270 return _ReadGitilesContent(CLANG_FILE_TEMPLATE % (revision, path_below_src)) 271 272 273def ReadUrlContent(url): 274 """Connect to a remote host and read the contents. Returns a list of lines.""" 275 conn = urllib.request.urlopen(url) 276 try: 277 return conn.readlines() 278 except IOError as e: 279 logging.exception('Error connecting to %s. Error: %s', url, e) 280 raise 281 finally: 282 conn.close() 283 284 285def GetMatchingDepsEntries(depsentry_dict, dir_path): 286 """Gets all deps entries matching the provided path. 287 288 This list may contain more than one DepsEntry object. 289 Example: dir_path='src/testing' would give results containing both 290 'src/testing/gtest' and 'src/testing/gmock' deps entries for Chromium's DEPS. 291 Example 2: dir_path='src/build' should return 'src/build' but not 292 'src/buildtools'. 293 294 Returns: 295 A list of DepsEntry objects. 296 """ 297 result = [] 298 for path, depsentry in depsentry_dict.items(): 299 if path == dir_path: 300 result.append(depsentry) 301 else: 302 parts = path.split('/') 303 if all(part == parts[i] for i, part in enumerate(dir_path.split('/'))): 304 result.append(depsentry) 305 return result 306 307 308def BuildDepsentryDict(deps_dict): 309 """Builds a dict of paths to DepsEntry objects from a raw parsed deps dict.""" 310 result = {} 311 312 def AddDepsEntries(deps_subdict): 313 for path, dep in deps_subdict.items(): 314 if path in result: 315 continue 316 if not isinstance(dep, dict): 317 dep = {'url': dep} 318 if dep.get('dep_type') == 'cipd': 319 result[path] = CipdDepsEntry(path, dep['packages']) 320 elif dep.get('dep_type') == 'gcs': 321 # Ignore GCS deps - there aren't any that we want to sync yet 322 continue 323 else: 324 if '@' not in dep['url']: 325 continue 326 url, revision = dep['url'].split('@') 327 result[path] = DepsEntry(path, url, revision) 328 329 AddDepsEntries(deps_dict['deps']) 330 for deps_os in ['win', 'mac', 'unix', 'android', 'ios', 'unix']: 331 AddDepsEntries(deps_dict.get('deps_os', {}).get(deps_os, {})) 332 return result 333 334 335def _FindChangedCipdPackages(path, old_pkgs, new_pkgs): 336 pkgs_equal = ({p['package'] for p in old_pkgs} == {p['package'] for p in new_pkgs}) 337 assert pkgs_equal, ('Old: %s\n New: %s.\nYou need to do a manual roll ' 338 'and remove/add entries in DEPS so the old and new ' 339 'list match.' % (old_pkgs, new_pkgs)) 340 for old_pkg in old_pkgs: 341 for new_pkg in new_pkgs: 342 old_version = old_pkg['version'] 343 new_version = new_pkg['version'] 344 if (old_pkg['package'] == new_pkg['package'] and old_version != new_version): 345 logging.debug('Roll dependency %s to %s', path, new_version) 346 yield ChangedCipdPackage(path, old_pkg['package'], old_version, new_version) 347 348 349def _FindNewDeps(old, new): 350 """ Gather dependencies only in |new| and return corresponding paths. """ 351 old_entries = set(BuildDepsentryDict(old)) 352 new_entries = set(BuildDepsentryDict(new)) 353 return [path for path in new_entries - old_entries if path in ANGLE_CHROMIUM_DEPS] 354 355 356def CalculateChangedDeps(angle_deps, new_cr_deps): 357 """ 358 Calculate changed deps entries based on entries defined in the ANGLE DEPS 359 file: 360 - If a shared dependency with the Chromium DEPS file: roll it to the same 361 revision as Chromium (i.e. entry in the new_cr_deps dict) 362 - If it's a Chromium sub-directory, roll it to the HEAD revision (notice 363 this means it may be ahead of the chromium_revision, but generally these 364 should be close). 365 - If it's another DEPS entry (not shared with Chromium), roll it to HEAD 366 unless it's configured to be skipped. 367 368 Returns: 369 A list of ChangedDep objects representing the changed deps. 370 """ 371 372 def ChromeURL(angle_deps_entry): 373 # Perform variable substitutions. 374 # This is a hack to get around the unsupported way this script parses DEPS. 375 # A better fix would be to use the gclient APIs to query and update DEPS. 376 # However this is complicated by how this script downloads DEPS remotely. 377 return angle_deps_entry.url.replace('{chromium_git}', 'https://chromium.googlesource.com') 378 379 result = [] 380 angle_entries = BuildDepsentryDict(angle_deps) 381 new_cr_entries = BuildDepsentryDict(new_cr_deps) 382 for path, angle_deps_entry in angle_entries.items(): 383 if path not in ANGLE_CHROMIUM_DEPS: 384 continue 385 386 # All ANGLE Chromium dependencies are located in src/. 387 chrome_path = 'src/%s' % path 388 cr_deps_entry = new_cr_entries.get(chrome_path) 389 390 if cr_deps_entry: 391 assert type(cr_deps_entry) is type(angle_deps_entry) 392 393 if isinstance(cr_deps_entry, CipdDepsEntry): 394 result.extend( 395 _FindChangedCipdPackages(path, angle_deps_entry.packages, 396 cr_deps_entry.packages)) 397 continue 398 399 # Use the revision from Chromium's DEPS file. 400 new_rev = cr_deps_entry.revision 401 assert ChromeURL(angle_deps_entry) == cr_deps_entry.url, ( 402 'ANGLE DEPS entry %s has a different URL (%s) than Chromium (%s).' % 403 (path, ChromeURL(angle_deps_entry), cr_deps_entry.url)) 404 else: 405 if isinstance(angle_deps_entry, DepsEntry): 406 # Use the HEAD of the deps repo. 407 stdout, _ = _RunCommand(['git', 'ls-remote', ChromeURL(angle_deps_entry), 'HEAD']) 408 new_rev = stdout.strip().split('\t')[0] 409 else: 410 # The dependency has been removed from chromium. 411 # This is handled by FindRemovedDeps. 412 continue 413 414 # Check if an update is necessary. 415 if angle_deps_entry.revision != new_rev: 416 logging.debug('Roll dependency %s to %s', path, new_rev) 417 result.append( 418 ChangedDep(path, ChromeURL(angle_deps_entry), angle_deps_entry.revision, new_rev)) 419 return sorted(result) 420 421 422def CalculateChangedClang(changed_deps, autoroll): 423 mirror_change = [change for change in changed_deps if change.path == CLANG_TOOLS_PATH] 424 if not mirror_change: 425 return None 426 427 mirror_change = mirror_change[0] 428 429 def GetClangRev(lines): 430 for line in lines: 431 match = CLANG_REVISION_RE.match(line) 432 if match: 433 return match.group(1) 434 raise RollError('Could not parse Clang revision!') 435 436 old_clang_update_py = ReadRemoteClangFile(CLANG_UPDATE_SCRIPT_URL_PATH, 437 mirror_change.current_rev).splitlines() 438 old_clang_rev = GetClangRev(old_clang_update_py) 439 logging.debug('Found old clang rev: %s' % old_clang_rev) 440 441 new_clang_update_py = ReadRemoteClangFile(CLANG_UPDATE_SCRIPT_URL_PATH, 442 mirror_change.new_rev).splitlines() 443 new_clang_rev = GetClangRev(new_clang_update_py) 444 logging.debug('Found new clang rev: %s' % new_clang_rev) 445 clang_change = ChangedDep(CLANG_UPDATE_SCRIPT_LOCAL_PATH, None, old_clang_rev, new_clang_rev) 446 return ClangChange(mirror_change, clang_change) 447 448 449def GenerateCommitMessage( 450 rev_update, 451 current_commit_pos, 452 new_commit_pos, 453 changed_deps_list, 454 autoroll, 455 clang_change, 456): 457 current_cr_rev = rev_update.current_chromium_rev[0:10] 458 new_cr_rev = rev_update.new_chromium_rev[0:10] 459 rev_interval = '%s..%s' % (current_cr_rev, new_cr_rev) 460 git_number_interval = '%s:%s' % (current_commit_pos, new_commit_pos) 461 462 commit_msg = [] 463 # Autoroll already adds chromium_revision changes to commit message 464 if not autoroll: 465 commit_msg.extend([ 466 'Roll chromium_revision %s (%s)\n' % (rev_interval, git_number_interval), 467 'Change log: %s' % (CHROMIUM_LOG_TEMPLATE % rev_interval), 468 'Full diff: %s\n' % (CHROMIUM_COMMIT_TEMPLATE % rev_interval) 469 ]) 470 471 def Section(adjective, deps): 472 noun = 'dependency' if len(deps) == 1 else 'dependencies' 473 commit_msg.append('%s %s' % (adjective, noun)) 474 475 tbr_authors = '' 476 if changed_deps_list: 477 Section('Changed', changed_deps_list) 478 479 for c in changed_deps_list: 480 if isinstance(c, ChangedCipdPackage): 481 commit_msg.append('* %s: %s..%s' % (c.path, c.current_version, c.new_version)) 482 else: 483 commit_msg.append('* %s: %s/+log/%s..%s' % 484 (c.path, c.url, c.current_rev[0:10], c.new_rev[0:10])) 485 486 if changed_deps_list: 487 # rev_interval is empty for autoroll, since we are starting from a state 488 # in which chromium_revision is already modified in DEPS 489 if not autoroll: 490 change_url = CHROMIUM_FILE_TEMPLATE % (rev_interval, 'DEPS') 491 commit_msg.append('DEPS diff: %s\n' % change_url) 492 else: 493 commit_msg.append('No dependencies changed.') 494 495 c = clang_change 496 if (c and (c.clang_change.current_rev != c.clang_change.new_rev)): 497 commit_msg.append('Clang version changed %s:%s' % 498 (c.clang_change.current_rev, c.clang_change.new_rev)) 499 500 rev_clang = rev_interval = '%s..%s' % (c.mirror_change.current_rev, 501 c.mirror_change.new_rev) 502 change_url = CLANG_FILE_TEMPLATE % (rev_clang, CLANG_UPDATE_SCRIPT_URL_PATH) 503 commit_msg.append('Details: %s\n' % change_url) 504 else: 505 commit_msg.append('No update to Clang.\n') 506 507 # Autoroll takes care of BUG and TBR in commit message 508 if not autoroll: 509 # TBR needs to be non-empty for Gerrit to process it. 510 git_author = _RunCommand(['git', 'config', 'user.email'], 511 working_dir=CHECKOUT_SRC_DIR)[0].splitlines()[0] 512 tbr_authors = git_author + ',' + tbr_authors 513 514 commit_msg.append('TBR=%s' % tbr_authors) 515 commit_msg.append('BUG=None') 516 517 return '\n'.join(commit_msg) 518 519 520def UpdateDepsFile(deps_filename, rev_update, changed_deps, new_cr_content, autoroll): 521 """Update the DEPS file with the new revision.""" 522 523 with open(deps_filename, 'rb') as deps_file: 524 deps_content = deps_file.read().decode('utf-8') 525 # Autoroll takes care of updating 'chromium_revision', thus we don't need to. 526 if not autoroll: 527 # Update the chromium_revision variable. 528 deps_content = deps_content.replace(rev_update.current_chromium_rev, 529 rev_update.new_chromium_rev) 530 531 # Add and remove dependencies. For now: only generated android deps. 532 # Since gclient cannot add or remove deps, we rely on the fact that 533 # these android deps are located in one place to copy/paste. 534 deps_re = re.compile(ANDROID_DEPS_START + '.*' + ANDROID_DEPS_END, re.DOTALL) 535 new_deps = deps_re.search(new_cr_content) 536 old_deps = deps_re.search(deps_content) 537 if not new_deps or not old_deps: 538 faulty = 'Chromium' if not new_deps else 'ANGLE' 539 raise RollError('Was expecting to find "%s" and "%s"\n' 540 'in %s DEPS' % (ANDROID_DEPS_START, ANDROID_DEPS_END, faulty)) 541 542 replacement = new_deps.group(0).replace('src/third_party/android_deps', 543 'third_party/android_deps') 544 replacement = replacement.replace('checkout_android', 545 'checkout_android and not build_with_chromium') 546 547 deps_content = deps_re.sub(replacement, deps_content) 548 549 with open(deps_filename, 'wb') as deps_file: 550 deps_file.write(deps_content.encode('utf-8')) 551 552 # Update each individual DEPS entry. 553 for dep in changed_deps: 554 # We don't sync deps on autoroller, so ignore missing local deps 555 if not autoroll: 556 local_dep_dir = os.path.join(CHECKOUT_ROOT_DIR, dep.path) 557 if not os.path.isdir(local_dep_dir): 558 raise RollError('Cannot find local directory %s. Either run\n' 559 'gclient sync --deps=all\n' 560 'or make sure the .gclient file for your solution contains all ' 561 'platforms in the target_os list, i.e.\n' 562 'target_os = ["android", "unix", "mac", "ios", "win"];\n' 563 'Then run "gclient sync" again.' % local_dep_dir) 564 if isinstance(dep, ChangedCipdPackage): 565 package = dep.package.format() # Eliminate double curly brackets 566 update = '%s:%s@%s' % (dep.path, package, dep.new_version) 567 else: 568 update = '%s@%s' % (dep.path, dep.new_rev) 569 gclient_cmd = 'gclient' 570 if platform.system() == 'Windows': 571 gclient_cmd += '.bat' 572 _RunCommand([gclient_cmd, 'setdep', '--revision', update], working_dir=CHECKOUT_SRC_DIR) 573 574 575def _IsTreeClean(): 576 stdout, _ = _RunCommand(['git', 'status', '--porcelain']) 577 if len(stdout) == 0: 578 return True 579 580 logging.error('Dirty/unversioned files:\n%s', stdout) 581 return False 582 583 584def _EnsureUpdatedMainBranch(dry_run): 585 current_branch = _RunCommand(['git', 'rev-parse', '--abbrev-ref', 'HEAD'])[0].splitlines()[0] 586 if current_branch != 'main': 587 logging.error('Please checkout the main branch and re-run this script.') 588 if not dry_run: 589 sys.exit(-1) 590 591 logging.info('Updating main branch...') 592 _RunCommand(['git', 'pull']) 593 594 595def _CreateRollBranch(dry_run): 596 logging.info('Creating roll branch: %s', ROLL_BRANCH_NAME) 597 if not dry_run: 598 _RunCommand(['git', 'checkout', '-b', ROLL_BRANCH_NAME]) 599 600 601def _RemovePreviousRollBranch(dry_run): 602 active_branch, branches = _GetBranches() 603 if active_branch == ROLL_BRANCH_NAME: 604 active_branch = 'main' 605 if ROLL_BRANCH_NAME in branches: 606 logging.info('Removing previous roll branch (%s)', ROLL_BRANCH_NAME) 607 if not dry_run: 608 _RunCommand(['git', 'checkout', active_branch]) 609 _RunCommand(['git', 'branch', '-D', ROLL_BRANCH_NAME]) 610 611 612def _LocalCommit(commit_msg, dry_run): 613 logging.info('Committing changes locally.') 614 if not dry_run: 615 _RunCommand(['git', 'add', '--update', '.']) 616 _RunCommand(['git', 'commit', '-m', commit_msg]) 617 618 619def _LocalCommitAmend(commit_msg, dry_run): 620 logging.info('Amending changes to local commit.') 621 if not dry_run: 622 old_commit_msg = _RunCommand(['git', 'log', '-1', '--pretty=%B'])[0].strip() 623 logging.debug('Existing commit message:\n%s\n', old_commit_msg) 624 625 bug_index = old_commit_msg.rfind('Bug:') 626 if bug_index == -1: 627 logging.error('"Bug:" not found in commit message.') 628 if not dry_run: 629 sys.exit(-1) 630 new_commit_msg = old_commit_msg[:bug_index] + commit_msg + '\n' + old_commit_msg[bug_index:] 631 632 _RunCommand(['git', 'commit', '-a', '--amend', '-m', new_commit_msg]) 633 634 635def ChooseCQMode(skip_cq, cq_over, current_commit_pos, new_commit_pos): 636 if skip_cq: 637 return 0 638 if (new_commit_pos - current_commit_pos) < cq_over: 639 return 1 640 return 2 641 642 643def _UploadCL(commit_queue_mode): 644 """Upload the committed changes as a changelist to Gerrit. 645 646 commit_queue_mode: 647 - 2: Submit to commit queue. 648 - 1: Run trybots but do not submit to CQ. 649 - 0: Skip CQ, upload only. 650 """ 651 cmd = ['git', 'cl', 'upload', '--force', '--bypass-hooks', '--send-mail'] 652 cmd.extend(['--cc', NOTIFY_EMAIL]) 653 if commit_queue_mode >= 2: 654 logging.info('Sending the CL to the CQ...') 655 cmd.extend(['--use-commit-queue']) 656 elif commit_queue_mode >= 1: 657 logging.info('Starting CQ dry run...') 658 cmd.extend(['--cq-dry-run']) 659 extra_env = { 660 'EDITOR': 'true', 661 'SKIP_GCE_AUTH_FOR_GIT': '1', 662 } 663 stdout, stderr = _RunCommand(cmd, extra_env=extra_env) 664 logging.debug('Output from "git cl upload":\nstdout:\n%s\n\nstderr:\n%s', stdout, stderr) 665 666 667def GetRollRevisionRanges(opts, angle_deps): 668 current_cr_rev = angle_deps['vars']['chromium_revision'] 669 new_cr_rev = opts.revision 670 if not new_cr_rev: 671 stdout, _ = _RunCommand(['git', 'ls-remote', CHROMIUM_SRC_URL, 'HEAD']) 672 head_rev = stdout.strip().split('\t')[0] 673 logging.info('No revision specified. Using HEAD: %s', head_rev) 674 new_cr_rev = head_rev 675 676 return ChromiumRevisionUpdate(current_cr_rev, new_cr_rev) 677 678 679def main(): 680 p = argparse.ArgumentParser() 681 p.add_argument( 682 '--clean', 683 action='store_true', 684 default=False, 685 help='Removes any previous local roll branch.') 686 p.add_argument( 687 '-r', 688 '--revision', 689 help=('Chromium Git revision to roll to. Defaults to the ' 690 'Chromium HEAD revision if omitted.')) 691 p.add_argument( 692 '--dry-run', 693 action='store_true', 694 default=False, 695 help=('Calculate changes and modify DEPS, but don\'t create ' 696 'any local branch, commit, upload CL or send any ' 697 'tryjobs.')) 698 p.add_argument( 699 '-i', 700 '--ignore-unclean-workdir', 701 action='store_true', 702 default=False, 703 help=('Ignore if the current branch is not main or if there ' 704 'are uncommitted changes (default: %(default)s).')) 705 grp = p.add_mutually_exclusive_group() 706 grp.add_argument( 707 '--skip-cq', 708 action='store_true', 709 default=False, 710 help='Skip sending the CL to the CQ (default: %(default)s)') 711 grp.add_argument( 712 '--cq-over', 713 type=int, 714 default=1, 715 help=('Commit queue dry run if the revision difference ' 716 'is below this number (default: %(default)s)')) 717 grp.add_argument( 718 '--autoroll', 719 action='store_true', 720 default=False, 721 help='Autoroller mode - amend existing commit, ' 722 'do not create nor upload a CL (default: %(default)s)') 723 p.add_argument( 724 '-v', 725 '--verbose', 726 action='store_true', 727 default=False, 728 help='Be extra verbose in printing of log messages.') 729 opts = p.parse_args() 730 731 if opts.verbose: 732 logging.basicConfig(level=logging.DEBUG) 733 else: 734 logging.basicConfig(level=logging.INFO) 735 736 # We don't have locally sync'ed deps on autoroller, 737 # so trust it to have depot_tools in path 738 if not opts.autoroll: 739 AddDepotToolsToPath() 740 741 if not opts.ignore_unclean_workdir and not _IsTreeClean(): 742 logging.error('Please clean your local checkout first.') 743 return 1 744 745 if opts.clean: 746 _RemovePreviousRollBranch(opts.dry_run) 747 748 if not opts.ignore_unclean_workdir: 749 _EnsureUpdatedMainBranch(opts.dry_run) 750 751 deps_filename = os.path.join(CHECKOUT_SRC_DIR, 'DEPS') 752 angle_deps = ParseLocalDepsFile(deps_filename) 753 754 rev_update = GetRollRevisionRanges(opts, angle_deps) 755 756 current_commit_pos = ParseCommitPosition(ReadRemoteCrCommit(rev_update.current_chromium_rev)) 757 new_commit_pos = ParseCommitPosition(ReadRemoteCrCommit(rev_update.new_chromium_rev)) 758 759 new_cr_content = ReadRemoteCrFile('DEPS', rev_update.new_chromium_rev) 760 new_cr_deps = ParseDepsDict(new_cr_content) 761 changed_deps = CalculateChangedDeps(angle_deps, new_cr_deps) 762 clang_change = CalculateChangedClang(changed_deps, opts.autoroll) 763 commit_msg = GenerateCommitMessage(rev_update, current_commit_pos, new_commit_pos, 764 changed_deps, opts.autoroll, clang_change) 765 logging.debug('Commit message:\n%s', commit_msg) 766 767 # We are updating a commit that autoroll has created, using existing branch 768 if not opts.autoroll: 769 _CreateRollBranch(opts.dry_run) 770 771 if not opts.dry_run: 772 UpdateDepsFile(deps_filename, rev_update, changed_deps, new_cr_content, opts.autoroll) 773 774 if opts.autoroll: 775 _LocalCommitAmend(commit_msg, opts.dry_run) 776 else: 777 if _IsTreeClean(): 778 logging.info("No DEPS changes detected, skipping CL creation.") 779 else: 780 _LocalCommit(commit_msg, opts.dry_run) 781 commit_queue_mode = ChooseCQMode(opts.skip_cq, opts.cq_over, current_commit_pos, 782 new_commit_pos) 783 logging.info('Uploading CL...') 784 if not opts.dry_run: 785 _UploadCL(commit_queue_mode) 786 return 0 787 788 789if __name__ == '__main__': 790 sys.exit(main()) 791