1#!/usr/bin/env python3 2# Copyright 2019 The ANGLE project authors. All Rights Reserved. 3# 4# Use of this source code is governed by a BSD-style license 5# that can be found in the LICENSE file in the root of the source 6# tree. An additional intellectual property rights grant can be found 7# in the file PATENTS. All contributing project authors may 8# be found in the AUTHORS file in the root of the source tree. 9 10# This is a modified copy of the script in 11# https://webrtc.googlesource.com/src/+/main/tools_webrtc/autoroller/roll_deps.py 12# customized for ANGLE. 13"""Script to automatically roll Chromium dependencies in the ANGLE DEPS file.""" 14 15import argparse 16import base64 17import collections 18import logging 19import os 20import platform 21import re 22import subprocess 23import sys 24import urllib.request 25 26 27def FindSrcDirPath(): 28 """Returns the abs path to the root dir of the project.""" 29 # Special cased for ANGLE. 30 return os.path.dirname(os.path.abspath(os.path.join(__file__, '..'))) 31 32 33ANGLE_CHROMIUM_DEPS = [ 34 'build', 35 'buildtools', 36 'buildtools/clang_format/script', 37 'buildtools/linux64', 38 'buildtools/mac', 39 'buildtools/third_party/libc++/trunk', 40 'buildtools/third_party/libc++abi/trunk', 41 'buildtools/third_party/libunwind/trunk', 42 'buildtools/win', 43 'testing', 44 'third_party/abseil-cpp', 45 'third_party/android_build_tools', 46 'third_party/android_build_tools/aapt2', 47 'third_party/android_build_tools/art', 48 'third_party/android_build_tools/bundletool', 49 'third_party/android_deps', 50 'third_party/android_ndk', 51 'third_party/android_platform', 52 'third_party/android_sdk', 53 'third_party/android_sdk/androidx_browser/src', 54 'third_party/android_sdk/public', 55 'third_party/android_system_sdk', 56 'third_party/bazel', 57 'third_party/catapult', 58 'third_party/colorama/src', 59 'third_party/depot_tools', 60 'third_party/ijar', 61 'third_party/jdk', 62 'third_party/jdk/extras', 63 'third_party/jinja2', 64 'third_party/libjpeg_turbo', 65 'third_party/markupsafe', 66 'third_party/nasm', 67 'third_party/proguard', 68 'third_party/protobuf', 69 'third_party/Python-Markdown', 70 'third_party/qemu-linux-x64', 71 'third_party/qemu-mac-x64', 72 'third_party/r8', 73 'third_party/requests/src', 74 'third_party/six', 75 'third_party/turbine', 76 'third_party/zlib', 77 'tools/android/errorprone_plugin', 78 'tools/clang', 79 'tools/clang/dsymutil', 80 'tools/luci-go', 81 'tools/mb', 82 'tools/md_browser', 83 'tools/memory', 84 'tools/perf', 85 'tools/protoc_wrapper', 86 'tools/python', 87 'tools/skia_goldctl/linux', 88 'tools/skia_goldctl/mac_amd64', 89 'tools/skia_goldctl/mac_arm64', 90 'tools/skia_goldctl/win', 91 'tools/valgrind', 92] 93 94ANGLE_URL = 'https://chromium.googlesource.com/angle/angle' 95CHROMIUM_SRC_URL = 'https://chromium.googlesource.com/chromium/src' 96CHROMIUM_COMMIT_TEMPLATE = CHROMIUM_SRC_URL + '/+/%s' 97CHROMIUM_LOG_TEMPLATE = CHROMIUM_SRC_URL + '/+log/%s' 98CHROMIUM_FILE_TEMPLATE = CHROMIUM_SRC_URL + '/+/%s/%s' 99 100COMMIT_POSITION_RE = re.compile('^Cr-Commit-Position: .*#([0-9]+).*$') 101CLANG_REVISION_RE = re.compile(r'^CLANG_REVISION = \'([-0-9a-z]+)\'') 102ROLL_BRANCH_NAME = 'roll_chromium_revision' 103 104SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) 105CHECKOUT_SRC_DIR = FindSrcDirPath() 106CHECKOUT_ROOT_DIR = CHECKOUT_SRC_DIR 107 108# Copied from tools/android/roll/android_deps/.../BuildConfigGenerator.groovy. 109ANDROID_DEPS_START = r'=== ANDROID_DEPS Generated Code Start ===' 110ANDROID_DEPS_END = r'=== ANDROID_DEPS Generated Code End ===' 111# Location of automically gathered android deps. 112ANDROID_DEPS_PATH = 'src/third_party/android_deps/' 113 114NOTIFY_EMAIL = 'angle-wrangler@grotations.appspotmail.com' 115 116CLANG_TOOLS_URL = 'https://chromium.googlesource.com/chromium/src/tools/clang' 117CLANG_FILE_TEMPLATE = CLANG_TOOLS_URL + '/+/%s/%s' 118 119CLANG_TOOLS_PATH = 'tools/clang' 120CLANG_UPDATE_SCRIPT_URL_PATH = 'scripts/update.py' 121CLANG_UPDATE_SCRIPT_LOCAL_PATH = os.path.join(CHECKOUT_SRC_DIR, 'tools', 'clang', 'scripts', 122 'update.py') 123 124DepsEntry = collections.namedtuple('DepsEntry', 'path url revision') 125ChangedDep = collections.namedtuple('ChangedDep', 'path url current_rev new_rev') 126ClangChange = collections.namedtuple('ClangChange', 'mirror_change clang_change') 127CipdDepsEntry = collections.namedtuple('CipdDepsEntry', 'path packages') 128ChangedCipdPackage = collections.namedtuple('ChangedCipdPackage', 129 'path package current_version new_version') 130 131ChromiumRevisionUpdate = collections.namedtuple('ChromiumRevisionUpdate', ('current_chromium_rev ' 132 'new_chromium_rev ')) 133 134 135def AddDepotToolsToPath(): 136 sys.path.append(os.path.join(CHECKOUT_SRC_DIR, 'build')) 137 import find_depot_tools 138 find_depot_tools.add_depot_tools_to_path() 139 140 141class RollError(Exception): 142 pass 143 144 145def StrExpansion(): 146 return lambda str_value: str_value 147 148 149def VarLookup(local_scope): 150 return lambda var_name: local_scope['vars'][var_name] 151 152 153def ParseDepsDict(deps_content): 154 local_scope = {} 155 global_scope = { 156 'Str': StrExpansion(), 157 'Var': VarLookup(local_scope), 158 'deps_os': {}, 159 } 160 exec (deps_content, global_scope, local_scope) 161 return local_scope 162 163 164def ParseLocalDepsFile(filename): 165 with open(filename, 'rb') as f: 166 deps_content = f.read() 167 return ParseDepsDict(deps_content) 168 169 170def ParseCommitPosition(commit_message): 171 for line in reversed(commit_message.splitlines()): 172 m = COMMIT_POSITION_RE.match(line.strip()) 173 if m: 174 return int(m.group(1)) 175 logging.error('Failed to parse commit position id from:\n%s\n', commit_message) 176 sys.exit(-1) 177 178 179def _RunCommand(command, working_dir=None, ignore_exit_code=False, extra_env=None, 180 input_data=None): 181 """Runs a command and returns the output from that command. 182 183 If the command fails (exit code != 0), the function will exit the process. 184 185 Returns: 186 A tuple containing the stdout and stderr outputs as strings. 187 """ 188 working_dir = working_dir or CHECKOUT_SRC_DIR 189 logging.debug('CMD: %s CWD: %s', ' '.join(command), working_dir) 190 env = os.environ.copy() 191 if extra_env: 192 assert all(isinstance(value, str) for value in extra_env.values()) 193 logging.debug('extra env: %s', extra_env) 194 env.update(extra_env) 195 p = subprocess.Popen( 196 command, 197 stdin=subprocess.PIPE, 198 stdout=subprocess.PIPE, 199 stderr=subprocess.PIPE, 200 env=env, 201 cwd=working_dir, 202 universal_newlines=True) 203 std_output, err_output = p.communicate(input_data) 204 p.stdout.close() 205 p.stderr.close() 206 if not ignore_exit_code and p.returncode != 0: 207 logging.error('Command failed: %s\n' 208 'stdout:\n%s\n' 209 'stderr:\n%s\n', ' '.join(command), std_output, err_output) 210 sys.exit(p.returncode) 211 return std_output, err_output 212 213 214def _GetBranches(): 215 """Returns a tuple of active,branches. 216 217 The 'active' is the name of the currently active branch and 'branches' is a 218 list of all branches. 219 """ 220 lines = _RunCommand(['git', 'branch'])[0].split('\n') 221 branches = [] 222 active = '' 223 for line in lines: 224 if '*' in line: 225 # The assumption is that the first char will always be the '*'. 226 active = line[1:].strip() 227 branches.append(active) 228 else: 229 branch = line.strip() 230 if branch: 231 branches.append(branch) 232 return active, branches 233 234 235def _ReadGitilesContent(url): 236 # Download and decode BASE64 content until 237 # https://code.google.com/p/gitiles/issues/detail?id=7 is fixed. 238 logging.debug('Reading gitiles URL %s' % url) 239 base64_content = ReadUrlContent(url + '?format=TEXT') 240 return base64.b64decode(base64_content[0]).decode('utf-8') 241 242 243def ReadRemoteCrFile(path_below_src, revision): 244 """Reads a remote Chromium file of a specific revision. Returns a string.""" 245 return _ReadGitilesContent(CHROMIUM_FILE_TEMPLATE % (revision, path_below_src)) 246 247 248def ReadRemoteCrCommit(revision): 249 """Reads a remote Chromium commit message. Returns a string.""" 250 return _ReadGitilesContent(CHROMIUM_COMMIT_TEMPLATE % revision) 251 252 253def ReadRemoteClangFile(path_below_src, revision): 254 """Reads a remote Clang file of a specific revision. Returns a string.""" 255 return _ReadGitilesContent(CLANG_FILE_TEMPLATE % (revision, path_below_src)) 256 257 258def ReadUrlContent(url): 259 """Connect to a remote host and read the contents. Returns a list of lines.""" 260 conn = urllib.request.urlopen(url) 261 try: 262 return conn.readlines() 263 except IOError as e: 264 logging.exception('Error connecting to %s. Error: %s', url, e) 265 raise 266 finally: 267 conn.close() 268 269 270def GetMatchingDepsEntries(depsentry_dict, dir_path): 271 """Gets all deps entries matching the provided path. 272 273 This list may contain more than one DepsEntry object. 274 Example: dir_path='src/testing' would give results containing both 275 'src/testing/gtest' and 'src/testing/gmock' deps entries for Chromium's DEPS. 276 Example 2: dir_path='src/build' should return 'src/build' but not 277 'src/buildtools'. 278 279 Returns: 280 A list of DepsEntry objects. 281 """ 282 result = [] 283 for path, depsentry in depsentry_dict.items(): 284 if path == dir_path: 285 result.append(depsentry) 286 else: 287 parts = path.split('/') 288 if all(part == parts[i] for i, part in enumerate(dir_path.split('/'))): 289 result.append(depsentry) 290 return result 291 292 293def BuildDepsentryDict(deps_dict): 294 """Builds a dict of paths to DepsEntry objects from a raw parsed deps dict.""" 295 result = {} 296 297 def AddDepsEntries(deps_subdict): 298 for path, dep in deps_subdict.items(): 299 if path in result: 300 continue 301 if not isinstance(dep, dict): 302 dep = {'url': dep} 303 if dep.get('dep_type') == 'cipd': 304 result[path] = CipdDepsEntry(path, dep['packages']) 305 else: 306 if '@' not in dep['url']: 307 continue 308 url, revision = dep['url'].split('@') 309 result[path] = DepsEntry(path, url, revision) 310 311 AddDepsEntries(deps_dict['deps']) 312 for deps_os in ['win', 'mac', 'unix', 'android', 'ios', 'unix']: 313 AddDepsEntries(deps_dict.get('deps_os', {}).get(deps_os, {})) 314 return result 315 316 317def _FindChangedCipdPackages(path, old_pkgs, new_pkgs): 318 pkgs_equal = ({p['package'] for p in old_pkgs} == {p['package'] for p in new_pkgs}) 319 assert pkgs_equal, ('Old: %s\n New: %s.\nYou need to do a manual roll ' 320 'and remove/add entries in DEPS so the old and new ' 321 'list match.' % (old_pkgs, new_pkgs)) 322 for old_pkg in old_pkgs: 323 for new_pkg in new_pkgs: 324 old_version = old_pkg['version'] 325 new_version = new_pkg['version'] 326 if (old_pkg['package'] == new_pkg['package'] and old_version != new_version): 327 logging.debug('Roll dependency %s to %s', path, new_version) 328 yield ChangedCipdPackage(path, old_pkg['package'], old_version, new_version) 329 330 331def _FindNewDeps(old, new): 332 """ Gather dependencies only in |new| and return corresponding paths. """ 333 old_entries = set(BuildDepsentryDict(old)) 334 new_entries = set(BuildDepsentryDict(new)) 335 return [path for path in new_entries - old_entries if path in ANGLE_CHROMIUM_DEPS] 336 337 338def CalculateChangedDeps(angle_deps, new_cr_deps): 339 """ 340 Calculate changed deps entries based on entries defined in the ANGLE DEPS 341 file: 342 - If a shared dependency with the Chromium DEPS file: roll it to the same 343 revision as Chromium (i.e. entry in the new_cr_deps dict) 344 - If it's a Chromium sub-directory, roll it to the HEAD revision (notice 345 this means it may be ahead of the chromium_revision, but generally these 346 should be close). 347 - If it's another DEPS entry (not shared with Chromium), roll it to HEAD 348 unless it's configured to be skipped. 349 350 Returns: 351 A list of ChangedDep objects representing the changed deps. 352 """ 353 354 def ChromeURL(angle_deps_entry): 355 # Perform variable substitutions. 356 # This is a hack to get around the unsupported way this script parses DEPS. 357 # A better fix would be to use the gclient APIs to query and update DEPS. 358 # However this is complicated by how this script downloads DEPS remotely. 359 return angle_deps_entry.url.replace('{chromium_git}', 'https://chromium.googlesource.com') 360 361 result = [] 362 angle_entries = BuildDepsentryDict(angle_deps) 363 new_cr_entries = BuildDepsentryDict(new_cr_deps) 364 for path, angle_deps_entry in angle_entries.items(): 365 if path not in ANGLE_CHROMIUM_DEPS: 366 continue 367 368 # All ANGLE Chromium dependencies are located in src/. 369 chrome_path = 'src/%s' % path 370 cr_deps_entry = new_cr_entries.get(chrome_path) 371 372 if cr_deps_entry: 373 assert type(cr_deps_entry) is type(angle_deps_entry) 374 375 if isinstance(cr_deps_entry, CipdDepsEntry): 376 result.extend( 377 _FindChangedCipdPackages(path, angle_deps_entry.packages, 378 cr_deps_entry.packages)) 379 continue 380 381 # Use the revision from Chromium's DEPS file. 382 new_rev = cr_deps_entry.revision 383 assert ChromeURL(angle_deps_entry) == cr_deps_entry.url, ( 384 'ANGLE DEPS entry %s has a different URL (%s) than Chromium (%s).' % 385 (path, ChromeURL(angle_deps_entry), cr_deps_entry.url)) 386 else: 387 if isinstance(angle_deps_entry, DepsEntry): 388 # Use the HEAD of the deps repo. 389 stdout, _ = _RunCommand(['git', 'ls-remote', ChromeURL(angle_deps_entry), 'HEAD']) 390 new_rev = stdout.strip().split('\t')[0] 391 else: 392 # The dependency has been removed from chromium. 393 # This is handled by FindRemovedDeps. 394 continue 395 396 # Check if an update is necessary. 397 if angle_deps_entry.revision != new_rev: 398 logging.debug('Roll dependency %s to %s', path, new_rev) 399 result.append( 400 ChangedDep(path, ChromeURL(angle_deps_entry), angle_deps_entry.revision, new_rev)) 401 return sorted(result) 402 403 404def CalculateChangedClang(changed_deps, autoroll): 405 mirror_change = [change for change in changed_deps if change.path == CLANG_TOOLS_PATH] 406 if not mirror_change: 407 return None 408 409 mirror_change = mirror_change[0] 410 411 def GetClangRev(lines): 412 for line in lines: 413 match = CLANG_REVISION_RE.match(line) 414 if match: 415 return match.group(1) 416 raise RollError('Could not parse Clang revision!') 417 418 old_clang_update_py = ReadRemoteClangFile(CLANG_UPDATE_SCRIPT_URL_PATH, 419 mirror_change.current_rev).splitlines() 420 old_clang_rev = GetClangRev(old_clang_update_py) 421 logging.debug('Found old clang rev: %s' % old_clang_rev) 422 423 new_clang_update_py = ReadRemoteClangFile(CLANG_UPDATE_SCRIPT_URL_PATH, 424 mirror_change.new_rev).splitlines() 425 new_clang_rev = GetClangRev(new_clang_update_py) 426 logging.debug('Found new clang rev: %s' % new_clang_rev) 427 clang_change = ChangedDep(CLANG_UPDATE_SCRIPT_LOCAL_PATH, None, old_clang_rev, new_clang_rev) 428 return ClangChange(mirror_change, clang_change) 429 430 431def GenerateCommitMessage( 432 rev_update, 433 current_commit_pos, 434 new_commit_pos, 435 changed_deps_list, 436 autoroll, 437 clang_change, 438): 439 current_cr_rev = rev_update.current_chromium_rev[0:10] 440 new_cr_rev = rev_update.new_chromium_rev[0:10] 441 rev_interval = '%s..%s' % (current_cr_rev, new_cr_rev) 442 git_number_interval = '%s:%s' % (current_commit_pos, new_commit_pos) 443 444 commit_msg = [] 445 # Autoroll already adds chromium_revision changes to commit message 446 if not autoroll: 447 commit_msg.extend([ 448 'Roll chromium_revision %s (%s)\n' % (rev_interval, git_number_interval), 449 'Change log: %s' % (CHROMIUM_LOG_TEMPLATE % rev_interval), 450 'Full diff: %s\n' % (CHROMIUM_COMMIT_TEMPLATE % rev_interval) 451 ]) 452 453 def Section(adjective, deps): 454 noun = 'dependency' if len(deps) == 1 else 'dependencies' 455 commit_msg.append('%s %s' % (adjective, noun)) 456 457 tbr_authors = '' 458 if changed_deps_list: 459 Section('Changed', changed_deps_list) 460 461 for c in changed_deps_list: 462 if isinstance(c, ChangedCipdPackage): 463 commit_msg.append('* %s: %s..%s' % (c.path, c.current_version, c.new_version)) 464 else: 465 commit_msg.append('* %s: %s/+log/%s..%s' % 466 (c.path, c.url, c.current_rev[0:10], c.new_rev[0:10])) 467 468 if changed_deps_list: 469 # rev_interval is empty for autoroll, since we are starting from a state 470 # in which chromium_revision is already modified in DEPS 471 if not autoroll: 472 change_url = CHROMIUM_FILE_TEMPLATE % (rev_interval, 'DEPS') 473 commit_msg.append('DEPS diff: %s\n' % change_url) 474 else: 475 commit_msg.append('No dependencies changed.') 476 477 c = clang_change 478 if (c and (c.clang_change.current_rev != c.clang_change.new_rev)): 479 commit_msg.append('Clang version changed %s:%s' % 480 (c.clang_change.current_rev, c.clang_change.new_rev)) 481 482 rev_clang = rev_interval = '%s..%s' % (c.mirror_change.current_rev, 483 c.mirror_change.new_rev) 484 change_url = CLANG_FILE_TEMPLATE % (rev_clang, CLANG_UPDATE_SCRIPT_URL_PATH) 485 commit_msg.append('Details: %s\n' % change_url) 486 else: 487 commit_msg.append('No update to Clang.\n') 488 489 # Autoroll takes care of BUG and TBR in commit message 490 if not autoroll: 491 # TBR needs to be non-empty for Gerrit to process it. 492 git_author = _RunCommand(['git', 'config', 'user.email'], 493 working_dir=CHECKOUT_SRC_DIR)[0].splitlines()[0] 494 tbr_authors = git_author + ',' + tbr_authors 495 496 commit_msg.append('TBR=%s' % tbr_authors) 497 commit_msg.append('BUG=None') 498 499 return '\n'.join(commit_msg) 500 501 502def UpdateDepsFile(deps_filename, rev_update, changed_deps, new_cr_content, autoroll): 503 """Update the DEPS file with the new revision.""" 504 505 with open(deps_filename, 'rb') as deps_file: 506 deps_content = deps_file.read().decode('utf-8') 507 # Autoroll takes care of updating 'chromium_revision', thus we don't need to. 508 if not autoroll: 509 # Update the chromium_revision variable. 510 deps_content = deps_content.replace(rev_update.current_chromium_rev, 511 rev_update.new_chromium_rev) 512 513 # Add and remove dependencies. For now: only generated android deps. 514 # Since gclient cannot add or remove deps, we rely on the fact that 515 # these android deps are located in one place to copy/paste. 516 deps_re = re.compile(ANDROID_DEPS_START + '.*' + ANDROID_DEPS_END, re.DOTALL) 517 new_deps = deps_re.search(new_cr_content) 518 old_deps = deps_re.search(deps_content) 519 if not new_deps or not old_deps: 520 faulty = 'Chromium' if not new_deps else 'ANGLE' 521 raise RollError('Was expecting to find "%s" and "%s"\n' 522 'in %s DEPS' % (ANDROID_DEPS_START, ANDROID_DEPS_END, faulty)) 523 524 replacement = new_deps.group(0).replace('src/third_party/android_deps', 525 'third_party/android_deps') 526 replacement = replacement.replace('checkout_android', 527 'checkout_android and not build_with_chromium') 528 529 deps_content = deps_re.sub(replacement, deps_content) 530 531 with open(deps_filename, 'wb') as deps_file: 532 deps_file.write(deps_content.encode('utf-8')) 533 534 # Update each individual DEPS entry. 535 for dep in changed_deps: 536 # We don't sync deps on autoroller, so ignore missing local deps 537 if not autoroll: 538 local_dep_dir = os.path.join(CHECKOUT_ROOT_DIR, dep.path) 539 if not os.path.isdir(local_dep_dir): 540 raise RollError('Cannot find local directory %s. Either run\n' 541 'gclient sync --deps=all\n' 542 'or make sure the .gclient file for your solution contains all ' 543 'platforms in the target_os list, i.e.\n' 544 'target_os = ["android", "unix", "mac", "ios", "win"];\n' 545 'Then run "gclient sync" again.' % local_dep_dir) 546 if isinstance(dep, ChangedCipdPackage): 547 package = dep.package.format() # Eliminate double curly brackets 548 update = '%s:%s@%s' % (dep.path, package, dep.new_version) 549 else: 550 update = '%s@%s' % (dep.path, dep.new_rev) 551 gclient_cmd = 'gclient' 552 if platform.system() == 'Windows': 553 gclient_cmd += '.bat' 554 _RunCommand([gclient_cmd, 'setdep', '--revision', update], working_dir=CHECKOUT_SRC_DIR) 555 556 557def _IsTreeClean(): 558 stdout, _ = _RunCommand(['git', 'status', '--porcelain']) 559 if len(stdout) == 0: 560 return True 561 562 logging.error('Dirty/unversioned files:\n%s', stdout) 563 return False 564 565 566def _EnsureUpdatedMainBranch(dry_run): 567 current_branch = _RunCommand(['git', 'rev-parse', '--abbrev-ref', 'HEAD'])[0].splitlines()[0] 568 if current_branch != 'main': 569 logging.error('Please checkout the main branch and re-run this script.') 570 if not dry_run: 571 sys.exit(-1) 572 573 logging.info('Updating main branch...') 574 _RunCommand(['git', 'pull']) 575 576 577def _CreateRollBranch(dry_run): 578 logging.info('Creating roll branch: %s', ROLL_BRANCH_NAME) 579 if not dry_run: 580 _RunCommand(['git', 'checkout', '-b', ROLL_BRANCH_NAME]) 581 582 583def _RemovePreviousRollBranch(dry_run): 584 active_branch, branches = _GetBranches() 585 if active_branch == ROLL_BRANCH_NAME: 586 active_branch = 'main' 587 if ROLL_BRANCH_NAME in branches: 588 logging.info('Removing previous roll branch (%s)', ROLL_BRANCH_NAME) 589 if not dry_run: 590 _RunCommand(['git', 'checkout', active_branch]) 591 _RunCommand(['git', 'branch', '-D', ROLL_BRANCH_NAME]) 592 593 594def _LocalCommit(commit_msg, dry_run): 595 logging.info('Committing changes locally.') 596 if not dry_run: 597 _RunCommand(['git', 'add', '--update', '.']) 598 _RunCommand(['git', 'commit', '-m', commit_msg]) 599 600 601def _LocalCommitAmend(commit_msg, dry_run): 602 logging.info('Amending changes to local commit.') 603 if not dry_run: 604 old_commit_msg = _RunCommand(['git', 'log', '-1', '--pretty=%B'])[0].strip() 605 logging.debug('Existing commit message:\n%s\n', old_commit_msg) 606 607 bug_index = old_commit_msg.rfind('Bug:') 608 if bug_index == -1: 609 logging.error('"Bug:" not found in commit message.') 610 if not dry_run: 611 sys.exit(-1) 612 new_commit_msg = old_commit_msg[:bug_index] + commit_msg + '\n' + old_commit_msg[bug_index:] 613 614 _RunCommand(['git', 'commit', '-a', '--amend', '-m', new_commit_msg]) 615 616 617def ChooseCQMode(skip_cq, cq_over, current_commit_pos, new_commit_pos): 618 if skip_cq: 619 return 0 620 if (new_commit_pos - current_commit_pos) < cq_over: 621 return 1 622 return 2 623 624 625def _UploadCL(commit_queue_mode): 626 """Upload the committed changes as a changelist to Gerrit. 627 628 commit_queue_mode: 629 - 2: Submit to commit queue. 630 - 1: Run trybots but do not submit to CQ. 631 - 0: Skip CQ, upload only. 632 """ 633 cmd = ['git', 'cl', 'upload', '--force', '--bypass-hooks', '--send-mail'] 634 cmd.extend(['--cc', NOTIFY_EMAIL]) 635 if commit_queue_mode >= 2: 636 logging.info('Sending the CL to the CQ...') 637 cmd.extend(['--use-commit-queue']) 638 elif commit_queue_mode >= 1: 639 logging.info('Starting CQ dry run...') 640 cmd.extend(['--cq-dry-run']) 641 extra_env = { 642 'EDITOR': 'true', 643 'SKIP_GCE_AUTH_FOR_GIT': '1', 644 } 645 stdout, stderr = _RunCommand(cmd, extra_env=extra_env) 646 logging.debug('Output from "git cl upload":\nstdout:\n%s\n\nstderr:\n%s', stdout, stderr) 647 648 649def GetRollRevisionRanges(opts, angle_deps): 650 current_cr_rev = angle_deps['vars']['chromium_revision'] 651 new_cr_rev = opts.revision 652 if not new_cr_rev: 653 stdout, _ = _RunCommand(['git', 'ls-remote', CHROMIUM_SRC_URL, 'HEAD']) 654 head_rev = stdout.strip().split('\t')[0] 655 logging.info('No revision specified. Using HEAD: %s', head_rev) 656 new_cr_rev = head_rev 657 658 return ChromiumRevisionUpdate(current_cr_rev, new_cr_rev) 659 660 661def main(): 662 p = argparse.ArgumentParser() 663 p.add_argument( 664 '--clean', 665 action='store_true', 666 default=False, 667 help='Removes any previous local roll branch.') 668 p.add_argument( 669 '-r', 670 '--revision', 671 help=('Chromium Git revision to roll to. Defaults to the ' 672 'Chromium HEAD revision if omitted.')) 673 p.add_argument( 674 '--dry-run', 675 action='store_true', 676 default=False, 677 help=('Calculate changes and modify DEPS, but don\'t create ' 678 'any local branch, commit, upload CL or send any ' 679 'tryjobs.')) 680 p.add_argument( 681 '-i', 682 '--ignore-unclean-workdir', 683 action='store_true', 684 default=False, 685 help=('Ignore if the current branch is not main or if there ' 686 'are uncommitted changes (default: %(default)s).')) 687 grp = p.add_mutually_exclusive_group() 688 grp.add_argument( 689 '--skip-cq', 690 action='store_true', 691 default=False, 692 help='Skip sending the CL to the CQ (default: %(default)s)') 693 grp.add_argument( 694 '--cq-over', 695 type=int, 696 default=1, 697 help=('Commit queue dry run if the revision difference ' 698 'is below this number (default: %(default)s)')) 699 grp.add_argument( 700 '--autoroll', 701 action='store_true', 702 default=False, 703 help='Autoroller mode - amend existing commit, ' 704 'do not create nor upload a CL (default: %(default)s)') 705 p.add_argument( 706 '-v', 707 '--verbose', 708 action='store_true', 709 default=False, 710 help='Be extra verbose in printing of log messages.') 711 opts = p.parse_args() 712 713 if opts.verbose: 714 logging.basicConfig(level=logging.DEBUG) 715 else: 716 logging.basicConfig(level=logging.INFO) 717 718 # We don't have locally sync'ed deps on autoroller, 719 # so trust it to have depot_tools in path 720 if not opts.autoroll: 721 AddDepotToolsToPath() 722 723 if not opts.ignore_unclean_workdir and not _IsTreeClean(): 724 logging.error('Please clean your local checkout first.') 725 return 1 726 727 if opts.clean: 728 _RemovePreviousRollBranch(opts.dry_run) 729 730 if not opts.ignore_unclean_workdir: 731 _EnsureUpdatedMainBranch(opts.dry_run) 732 733 deps_filename = os.path.join(CHECKOUT_SRC_DIR, 'DEPS') 734 angle_deps = ParseLocalDepsFile(deps_filename) 735 736 rev_update = GetRollRevisionRanges(opts, angle_deps) 737 738 current_commit_pos = ParseCommitPosition(ReadRemoteCrCommit(rev_update.current_chromium_rev)) 739 new_commit_pos = ParseCommitPosition(ReadRemoteCrCommit(rev_update.new_chromium_rev)) 740 741 new_cr_content = ReadRemoteCrFile('DEPS', rev_update.new_chromium_rev) 742 new_cr_deps = ParseDepsDict(new_cr_content) 743 changed_deps = CalculateChangedDeps(angle_deps, new_cr_deps) 744 clang_change = CalculateChangedClang(changed_deps, opts.autoroll) 745 commit_msg = GenerateCommitMessage(rev_update, current_commit_pos, new_commit_pos, 746 changed_deps, opts.autoroll, clang_change) 747 logging.debug('Commit message:\n%s', commit_msg) 748 749 # We are updating a commit that autoroll has created, using existing branch 750 if not opts.autoroll: 751 _CreateRollBranch(opts.dry_run) 752 753 if not opts.dry_run: 754 UpdateDepsFile(deps_filename, rev_update, changed_deps, new_cr_content, opts.autoroll) 755 756 if opts.autoroll: 757 _LocalCommitAmend(commit_msg, opts.dry_run) 758 else: 759 if _IsTreeClean(): 760 logging.info("No DEPS changes detected, skipping CL creation.") 761 else: 762 _LocalCommit(commit_msg, opts.dry_run) 763 commit_queue_mode = ChooseCQMode(opts.skip_cq, opts.cq_over, current_commit_pos, 764 new_commit_pos) 765 logging.info('Uploading CL...') 766 if not opts.dry_run: 767 _UploadCL(commit_queue_mode) 768 return 0 769 770 771if __name__ == '__main__': 772 sys.exit(main()) 773