1#!/usr/bin/env vpython3 2 3# Copyright 2017 The LibYuv Project Authors. All rights reserved. 4# 5# Use of this source code is governed by a BSD-style license 6# that can be found in the LICENSE file in the root of the source 7# tree. An additional intellectual property rights grant can be found 8# in the file PATENTS. All contributing project authors may 9# be found in the AUTHORS file in the root of the source tree. 10 11# This is a modified copy of the script in 12# https://webrtc.googlesource.com/src/+/master/tools_webrtc/autoroller/roll_deps.py 13# customized for libyuv. 14 15"""Script to automatically roll dependencies in the libyuv DEPS file.""" 16 17import argparse 18import base64 19import collections 20import logging 21import os 22import re 23import subprocess 24import sys 25import urllib.request 26 27 28# Skip these dependencies (list without solution name prefix). 29DONT_AUTOROLL_THESE = [ 30 'src/third_party/gflags/src', 31] 32 33LIBYUV_URL = 'https://chromium.googlesource.com/libyuv/libyuv' 34CHROMIUM_SRC_URL = 'https://chromium.googlesource.com/chromium/src' 35CHROMIUM_COMMIT_TEMPLATE = CHROMIUM_SRC_URL + '/+/%s' 36CHROMIUM_LOG_TEMPLATE = CHROMIUM_SRC_URL + '/+log/%s' 37CHROMIUM_FILE_TEMPLATE = CHROMIUM_SRC_URL + '/+/%s/%s' 38 39COMMIT_POSITION_RE = re.compile('^Cr-Commit-Position: .*#([0-9]+).*$') 40CLANG_REVISION_RE = re.compile(r'^CLANG_REVISION = \'([0-9a-z-]+)\'$') 41ROLL_BRANCH_NAME = 'roll_chromium_revision' 42 43SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) 44CHECKOUT_SRC_DIR = os.path.realpath(os.path.join(SCRIPT_DIR, os.pardir, 45 os.pardir)) 46CHECKOUT_ROOT_DIR = os.path.realpath(os.path.join(CHECKOUT_SRC_DIR, os.pardir)) 47 48sys.path.append(os.path.join(CHECKOUT_SRC_DIR, 'build')) 49import find_depot_tools # pylint: disable=wrong-import-position 50find_depot_tools.add_depot_tools_to_path() 51 52CLANG_UPDATE_SCRIPT_URL_PATH = 'tools/clang/scripts/update.py' 53CLANG_UPDATE_SCRIPT_LOCAL_PATH = os.path.join(CHECKOUT_SRC_DIR, 'tools', 54 'clang', 'scripts', 'update.py') 55 56DepsEntry = collections.namedtuple('DepsEntry', 'path url revision') 57ChangedDep = collections.namedtuple('ChangedDep', 58 'path url current_rev new_rev') 59 60class RollError(Exception): 61 pass 62 63 64def VarLookup(local_scope): 65 return lambda var_name: local_scope['vars'][var_name] 66 67 68def ParseDepsDict(deps_content): 69 local_scope = {} 70 global_scope = { 71 'Var': VarLookup(local_scope), 72 'Str': lambda s: s, 73 'deps_os': {}, 74 } 75 exec(deps_content, global_scope, local_scope) 76 return local_scope 77 78 79def ParseLocalDepsFile(filename): 80 with open(filename, 'rb') as f: 81 deps_content = f.read().decode('utf-8') 82 return ParseDepsDict(deps_content) 83 84 85def ParseRemoteCrDepsFile(revision): 86 deps_content = ReadRemoteCrFile('DEPS', revision) 87 return ParseDepsDict(deps_content) 88 89 90def ParseCommitPosition(commit_message): 91 for line in reversed(commit_message.splitlines()): 92 m = COMMIT_POSITION_RE.match(line.strip()) 93 if m: 94 return int(m.group(1)) 95 logging.error('Failed to parse commit position id from:\n%s\n', 96 commit_message) 97 sys.exit(-1) 98 99 100def _RunCommand(command, working_dir=None, ignore_exit_code=False, 101 extra_env=None, input_data=None): 102 """Runs a command and returns the output from that command. 103 104 If the command fails (exit code != 0), the function will exit the process. 105 106 Returns: 107 A tuple containing the stdout and stderr outputs as strings. 108 """ 109 working_dir = working_dir or CHECKOUT_SRC_DIR 110 logging.debug('CMD: %s CWD: %s', ' '.join(command), working_dir) 111 env = os.environ.copy() 112 if extra_env: 113 assert all(isinstance(value, str) for value in extra_env.values()) 114 logging.debug('extra env: %s', extra_env) 115 env.update(extra_env) 116 p = subprocess.Popen(command, 117 stdin=subprocess.PIPE, 118 stdout=subprocess.PIPE, 119 stderr=subprocess.PIPE, 120 env=env, 121 cwd=working_dir, 122 universal_newlines=True) 123 std_output, err_output = p.communicate(input_data) 124 p.stdout.close() 125 p.stderr.close() 126 if not ignore_exit_code and p.returncode != 0: 127 logging.error('Command failed: %s\n' 128 'stdout:\n%s\n' 129 'stderr:\n%s\n', ' '.join(command), std_output, err_output) 130 sys.exit(p.returncode) 131 return std_output, err_output 132 133 134def _GetBranches(): 135 """Returns a tuple of active,branches. 136 137 The 'active' is the name of the currently active branch and 'branches' is a 138 list of all branches. 139 """ 140 lines = _RunCommand(['git', 'branch'])[0].split('\n') 141 branches = [] 142 active = '' 143 for line in lines: 144 if '*' in line: 145 # The assumption is that the first char will always be the '*'. 146 active = line[1:].strip() 147 branches.append(active) 148 else: 149 branch = line.strip() 150 if branch: 151 branches.append(branch) 152 return active, branches 153 154 155def _ReadGitilesContent(url): 156 # Download and decode BASE64 content until 157 # https://code.google.com/p/gitiles/issues/detail?id=7 is fixed. 158 base64_content = ReadUrlContent(url + '?format=TEXT') 159 return base64.b64decode(base64_content[0]).decode('utf-8') 160 161 162def ReadRemoteCrFile(path_below_src, revision): 163 """Reads a remote Chromium file of a specific revision. Returns a string.""" 164 return _ReadGitilesContent(CHROMIUM_FILE_TEMPLATE % (revision, 165 path_below_src)) 166 167 168def ReadRemoteCrCommit(revision): 169 """Reads a remote Chromium commit message. Returns a string.""" 170 return _ReadGitilesContent(CHROMIUM_COMMIT_TEMPLATE % revision) 171 172 173def ReadUrlContent(url): 174 """Connect to a remote host and read the contents. Returns a list of lines.""" 175 conn = urllib.request.urlopen(url) 176 try: 177 return conn.readlines() 178 except IOError as e: 179 logging.exception('Error connecting to %s. Error: %s', url, e) 180 raise 181 finally: 182 conn.close() 183 184 185def GetMatchingDepsEntries(depsentry_dict, dir_path): 186 """Gets all deps entries matching the provided path. 187 188 This list may contain more than one DepsEntry object. 189 Example: dir_path='src/testing' would give results containing both 190 'src/testing/gtest' and 'src/testing/gmock' deps entries for Chromium's DEPS. 191 Example 2: dir_path='src/build' should return 'src/build' but not 192 'src/buildtools'. 193 194 Returns: 195 A list of DepsEntry objects. 196 """ 197 result = [] 198 for path, depsentry in depsentry_dict.items(): 199 if path == dir_path: 200 result.append(depsentry) 201 else: 202 parts = path.split('/') 203 if all(part == parts[i] 204 for i, part in enumerate(dir_path.split('/'))): 205 result.append(depsentry) 206 return result 207 208def BuildDepsentryDict(deps_dict): 209 """Builds a dict of paths to DepsEntry objects from a raw deps dict.""" 210 result = {} 211 212 def AddDepsEntries(deps_subdict): 213 for path, deps_url_spec in deps_subdict.items(): 214 if isinstance(deps_url_spec, dict): 215 if deps_url_spec.get('dep_type') == 'cipd': 216 continue 217 deps_url = deps_url_spec['url'] 218 else: 219 deps_url = deps_url_spec 220 if not path in result: 221 url, revision = deps_url.split('@') if deps_url else (None, None) 222 result[path] = DepsEntry(path, url, revision) 223 224 AddDepsEntries(deps_dict['deps']) 225 for deps_os in ['win', 'mac', 'linux', 'android', 'ios', 'unix']: 226 AddDepsEntries(deps_dict.get('deps_os', {}).get(deps_os, {})) 227 return result 228 229 230def CalculateChangedDeps(libyuv_deps, new_cr_deps): 231 """ 232 Calculate changed deps entries based on entries defined in the libyuv DEPS 233 file: 234 - If a shared dependency with the Chromium DEPS file: roll it to the same 235 revision as Chromium (i.e. entry in the new_cr_deps dict) 236 - If it's a Chromium sub-directory, roll it to the HEAD revision (notice 237 this means it may be ahead of the chromium_revision, but generally these 238 should be close). 239 - If it's another DEPS entry (not shared with Chromium), roll it to HEAD 240 unless it's configured to be skipped. 241 242 Returns: 243 A list of ChangedDep objects representing the changed deps. 244 """ 245 result = [] 246 libyuv_entries = BuildDepsentryDict(libyuv_deps) 247 new_cr_entries = BuildDepsentryDict(new_cr_deps) 248 for path, libyuv_deps_entry in libyuv_entries.items(): 249 if path in DONT_AUTOROLL_THESE: 250 continue 251 cr_deps_entry = new_cr_entries.get(path) 252 if cr_deps_entry: 253 # Use the revision from Chromium's DEPS file. 254 new_rev = cr_deps_entry.revision 255 assert libyuv_deps_entry.url == cr_deps_entry.url, ( 256 'Libyuv DEPS entry %s has a different URL (%s) than Chromium (%s).' % 257 (path, libyuv_deps_entry.url, cr_deps_entry.url)) 258 else: 259 # Use the HEAD of the deps repo. 260 stdout, _ = _RunCommand(['git', 'ls-remote', libyuv_deps_entry.url, 261 'HEAD']) 262 new_rev = stdout.strip().split('\t')[0] 263 264 # Check if an update is necessary. 265 if libyuv_deps_entry.revision != new_rev: 266 logging.debug('Roll dependency %s to %s', path, new_rev) 267 result.append(ChangedDep(path, libyuv_deps_entry.url, 268 libyuv_deps_entry.revision, new_rev)) 269 return sorted(result) 270 271 272def CalculateChangedClang(new_cr_rev): 273 def GetClangRev(lines): 274 for line in lines: 275 match = CLANG_REVISION_RE.match(line) 276 if match: 277 return match.group(1) 278 raise RollError('Could not parse Clang revision from:\n' + '\n'.join(' ' + l for l in lines)) 279 280 with open(CLANG_UPDATE_SCRIPT_LOCAL_PATH, 'r') as f: 281 current_lines = f.readlines() 282 current_rev = GetClangRev(current_lines) 283 284 new_clang_update_py = ReadRemoteCrFile(CLANG_UPDATE_SCRIPT_URL_PATH, 285 new_cr_rev).splitlines() 286 new_rev = GetClangRev(new_clang_update_py) 287 return ChangedDep(CLANG_UPDATE_SCRIPT_LOCAL_PATH, None, current_rev, new_rev) 288 289 290def GenerateCommitMessage(current_cr_rev, new_cr_rev, current_commit_pos, 291 new_commit_pos, changed_deps_list, clang_change): 292 current_cr_rev = current_cr_rev[0:10] 293 new_cr_rev = new_cr_rev[0:10] 294 rev_interval = '%s..%s' % (current_cr_rev, new_cr_rev) 295 git_number_interval = '%s:%s' % (current_commit_pos, new_commit_pos) 296 297 commit_msg = ['Roll chromium_revision %s (%s)\n' % (rev_interval, 298 git_number_interval)] 299 commit_msg.append('Change log: %s' % (CHROMIUM_LOG_TEMPLATE % rev_interval)) 300 commit_msg.append('Full diff: %s\n' % (CHROMIUM_COMMIT_TEMPLATE % 301 rev_interval)) 302 if changed_deps_list: 303 commit_msg.append('Changed dependencies:') 304 305 for c in changed_deps_list: 306 commit_msg.append('* %s: %s/+log/%s..%s' % (c.path, c.url, 307 c.current_rev[0:10], 308 c.new_rev[0:10])) 309 change_url = CHROMIUM_FILE_TEMPLATE % (rev_interval, 'DEPS') 310 commit_msg.append('DEPS diff: %s\n' % change_url) 311 else: 312 commit_msg.append('No dependencies changed.') 313 314 if clang_change.current_rev != clang_change.new_rev: 315 commit_msg.append('Clang version changed %s:%s' % 316 (clang_change.current_rev, clang_change.new_rev)) 317 change_url = CHROMIUM_FILE_TEMPLATE % (rev_interval, 318 CLANG_UPDATE_SCRIPT_URL_PATH) 319 commit_msg.append('Details: %s\n' % change_url) 320 else: 321 commit_msg.append('No update to Clang.\n') 322 323 # TBR needs to be non-empty for Gerrit to process it. 324 git_author = _RunCommand(['git', 'config', 'user.email'], 325 working_dir=CHECKOUT_SRC_DIR)[0].strip() 326 commit_msg.append('TBR=%s' % git_author) 327 328 commit_msg.append('BUG=None') 329 return '\n'.join(commit_msg) 330 331 332def UpdateDepsFile(deps_filename, old_cr_revision, new_cr_revision, 333 changed_deps): 334 """Update the DEPS file with the new revision.""" 335 336 # Update the chromium_revision variable. 337 with open(deps_filename, 'rb') as deps_file: 338 deps_content = deps_file.read().decode('utf-8') 339 deps_content = deps_content.replace(old_cr_revision, new_cr_revision) 340 with open(deps_filename, 'wb') as deps_file: 341 deps_file.write(deps_content.encode('utf-8')) 342 343 # Update each individual DEPS entry. 344 for dep in changed_deps: 345 local_dep_dir = os.path.join(CHECKOUT_ROOT_DIR, dep.path) 346 if not os.path.isdir(local_dep_dir): 347 raise RollError( 348 'Cannot find local directory %s. Make sure the .gclient file\n' 349 'contains all platforms in the target_os list, i.e.\n' 350 'target_os = ["android", "unix", "mac", "ios", "win"];\n' 351 'Then run "gclient sync" again.' % local_dep_dir) 352 _RunCommand( 353 ['gclient', 'setdep', '--revision', '%s@%s' % (dep.path, dep.new_rev)], 354 working_dir=CHECKOUT_SRC_DIR) 355 356 357def _IsTreeClean(): 358 stdout, _ = _RunCommand(['git', 'status', '--porcelain']) 359 if len(stdout) == 0: 360 return True 361 362 logging.error('Dirty/unversioned files:\n%s', stdout) 363 return False 364 365 366def _EnsureUpdatedMasterBranch(dry_run): 367 current_branch = _RunCommand( 368 ['git', 'rev-parse', '--abbrev-ref', 'HEAD'])[0].splitlines()[0] 369 if current_branch != 'main': 370 logging.error('Please checkout the main branch and re-run this script.') 371 if not dry_run: 372 sys.exit(-1) 373 374 logging.info('Updating main branch...') 375 _RunCommand(['git', 'pull']) 376 377 378def _CreateRollBranch(dry_run): 379 logging.info('Creating roll branch: %s', ROLL_BRANCH_NAME) 380 if not dry_run: 381 _RunCommand(['git', 'checkout', '-b', ROLL_BRANCH_NAME]) 382 383 384def _RemovePreviousRollBranch(dry_run): 385 active_branch, branches = _GetBranches() 386 if active_branch == ROLL_BRANCH_NAME: 387 active_branch = 'main' 388 if ROLL_BRANCH_NAME in branches: 389 logging.info('Removing previous roll branch (%s)', ROLL_BRANCH_NAME) 390 if not dry_run: 391 _RunCommand(['git', 'checkout', active_branch]) 392 _RunCommand(['git', 'branch', '-D', ROLL_BRANCH_NAME]) 393 394 395def _LocalCommit(commit_msg, dry_run): 396 logging.info('Committing changes locally.') 397 if not dry_run: 398 _RunCommand(['git', 'add', '--update', '.']) 399 _RunCommand(['git', 'commit', '-m', commit_msg]) 400 401 402def ChooseCQMode(skip_cq, cq_over, current_commit_pos, new_commit_pos): 403 if skip_cq: 404 return 0 405 if (new_commit_pos - current_commit_pos) < cq_over: 406 return 1 407 return 2 408 409 410def _UploadCL(commit_queue_mode): 411 """Upload the committed changes as a changelist to Gerrit. 412 413 commit_queue_mode: 414 - 2: Submit to commit queue. 415 - 1: Run trybots but do not submit to CQ. 416 - 0: Skip CQ, upload only. 417 """ 418 cmd = ['git', 'cl', 'upload', '--force', '--bypass-hooks', '--send-mail'] 419 if commit_queue_mode >= 2: 420 logging.info('Sending the CL to the CQ...') 421 cmd.extend(['-o', 'label=Bot-Commit+1']) 422 cmd.extend(['-o', 'label=Commit-Queue+2']) 423 elif commit_queue_mode >= 1: 424 logging.info('Starting CQ dry run...') 425 cmd.extend(['-o', 'label=Commit-Queue+1']) 426 extra_env = { 427 'EDITOR': 'true', 428 'SKIP_GCE_AUTH_FOR_GIT': '1', 429 } 430 stdout, stderr = _RunCommand(cmd, extra_env=extra_env) 431 logging.debug('Output from "git cl upload":\nstdout:\n%s\n\nstderr:\n%s', 432 stdout, stderr) 433 434 435def main(): 436 p = argparse.ArgumentParser() 437 p.add_argument('--clean', action='store_true', default=False, 438 help='Removes any previous local roll branch.') 439 p.add_argument('-r', '--revision', 440 help=('Chromium Git revision to roll to. Defaults to the ' 441 'Chromium HEAD revision if omitted.')) 442 p.add_argument('--dry-run', action='store_true', default=False, 443 help=('Calculate changes and modify DEPS, but don\'t create ' 444 'any local branch, commit, upload CL or send any ' 445 'tryjobs.')) 446 p.add_argument('-i', '--ignore-unclean-workdir', action='store_true', 447 default=False, 448 help=('Ignore if the current branch is not main or if there ' 449 'are uncommitted changes (default: %(default)s).')) 450 grp = p.add_mutually_exclusive_group() 451 grp.add_argument('--skip-cq', action='store_true', default=False, 452 help='Skip sending the CL to the CQ (default: %(default)s)') 453 grp.add_argument('--cq-over', type=int, default=1, 454 help=('Commit queue dry run if the revision difference ' 455 'is below this number (default: %(default)s)')) 456 p.add_argument('-v', '--verbose', action='store_true', default=False, 457 help='Be extra verbose in printing of log messages.') 458 opts = p.parse_args() 459 460 if opts.verbose: 461 logging.basicConfig(level=logging.DEBUG) 462 else: 463 logging.basicConfig(level=logging.INFO) 464 465 if not opts.ignore_unclean_workdir and not _IsTreeClean(): 466 logging.error('Please clean your local checkout first.') 467 return 1 468 469 if opts.clean: 470 _RemovePreviousRollBranch(opts.dry_run) 471 472 if not opts.ignore_unclean_workdir: 473 _EnsureUpdatedMasterBranch(opts.dry_run) 474 475 new_cr_rev = opts.revision 476 if not new_cr_rev: 477 stdout, _ = _RunCommand(['git', 'ls-remote', CHROMIUM_SRC_URL, 'HEAD']) 478 head_rev = stdout.strip().split('\t')[0] 479 logging.info('No revision specified. Using HEAD: %s', head_rev) 480 new_cr_rev = head_rev 481 482 deps_filename = os.path.join(CHECKOUT_SRC_DIR, 'DEPS') 483 libyuv_deps = ParseLocalDepsFile(deps_filename) 484 current_cr_rev = libyuv_deps['vars']['chromium_revision'] 485 486 current_commit_pos = ParseCommitPosition(ReadRemoteCrCommit(current_cr_rev)) 487 new_commit_pos = ParseCommitPosition(ReadRemoteCrCommit(new_cr_rev)) 488 489 new_cr_deps = ParseRemoteCrDepsFile(new_cr_rev) 490 changed_deps = CalculateChangedDeps(libyuv_deps, new_cr_deps) 491 clang_change = CalculateChangedClang(new_cr_rev) 492 commit_msg = GenerateCommitMessage(current_cr_rev, new_cr_rev, 493 current_commit_pos, new_commit_pos, 494 changed_deps, clang_change) 495 logging.debug('Commit message:\n%s', commit_msg) 496 497 _CreateRollBranch(opts.dry_run) 498 UpdateDepsFile(deps_filename, current_cr_rev, new_cr_rev, changed_deps) 499 _LocalCommit(commit_msg, opts.dry_run) 500 commit_queue_mode = ChooseCQMode(opts.skip_cq, opts.cq_over, 501 current_commit_pos, new_commit_pos) 502 logging.info('Uploading CL...') 503 if not opts.dry_run: 504 _UploadCL(commit_queue_mode) 505 return 0 506 507 508if __name__ == '__main__': 509 sys.exit(main()) 510