1#!/usr/bin/python 2# 3# Copyright (C) 2012 The Android Open Source Project 4# 5# Licensed under the Apache License, Version 2.0 (the "License"); 6# you may not use this file except in compliance with the License. 7# You may obtain a copy of the License at 8# 9# http://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an "AS IS" BASIS, 13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14# See the License for the specific language governing permissions and 15# limitations under the License. 16 17"""Merge Chromium into the Android tree.""" 18 19import contextlib 20import logging 21import optparse 22import os 23import re 24import sys 25import urllib2 26 27import merge_common 28 29 30# We need to import this *after* merging from upstream to get the latest 31# version. Set it to none here to catch uses before it's imported. 32webview_licenses = None 33 34 35AUTOGEN_MESSAGE = 'This commit was generated by merge_from_chromium.py.' 36SRC_GIT_BRANCH = 'refs/remotes/history/upstream-master' 37 38 39def _ReadGitFile(sha1, path, git_url=None, git_branch=None): 40 """Reads a file from a (possibly remote) git project at a specific revision. 41 42 Args: 43 sha1: The SHA1 at which to read. 44 path: The relative path of the file to read. 45 git_url: The URL of the git server, if reading a remote project. 46 git_branch: The branch to fetch, if reading a remote project. 47 Returns: 48 The contents of the specified file. 49 """ 50 if git_url: 51 merge_common.GetCommandStdout(['git', 'fetch', '-f', git_url, git_branch]) 52 return merge_common.GetCommandStdout(['git', 'show', '%s:%s' % (sha1, path)]) 53 54 55def _ParseDEPS(deps_content): 56 """Parses the .DEPS.git file from Chromium and returns its contents. 57 58 Args: 59 deps_content: The contents of the .DEPS.git file as text. 60 Returns: 61 A dictionary of the contents of .DEPS.git at the specified revision 62 """ 63 64 class FromImpl(object): 65 """Used to implement the From syntax.""" 66 67 def __init__(self, module_name): 68 self.module_name = module_name 69 70 def __str__(self): 71 return 'From("%s")' % self.module_name 72 73 class _VarImpl(object): 74 def __init__(self, custom_vars, local_scope): 75 self._custom_vars = custom_vars 76 self._local_scope = local_scope 77 78 def Lookup(self, var_name): 79 """Implements the Var syntax.""" 80 if var_name in self._custom_vars: 81 return self._custom_vars[var_name] 82 elif var_name in self._local_scope.get('vars', {}): 83 return self._local_scope['vars'][var_name] 84 raise Exception('Var is not defined: %s' % var_name) 85 86 tmp_locals = {} 87 var = _VarImpl({}, tmp_locals) 88 tmp_globals = {'From': FromImpl, 'Var': var.Lookup, 'deps_os': {}} 89 exec(deps_content) in tmp_globals, tmp_locals 90 return tmp_locals 91 92 93def _GetProjectMergeInfo(projects, deps_vars): 94 """Gets the git URL and SHA1 for each project based on .DEPS.git. 95 96 Args: 97 projects: The list of projects to consider. 98 deps_vars: The dictionary of dependencies from .DEPS.git. 99 Returns: 100 A dictionary from project to git URL and SHA1 - 'path: (url, sha1)' 101 Raises: 102 TemporaryMergeError: if a project to be merged is not found in .DEPS.git. 103 """ 104 deps_fallback_order = [ 105 deps_vars['deps'], 106 deps_vars['deps_os']['unix'], 107 deps_vars['deps_os']['android'], 108 ] 109 result = {} 110 for path in projects: 111 for deps in deps_fallback_order: 112 if len(path) > 0: 113 upstream_path = os.path.join('src', path) 114 else: 115 upstream_path = 'src' 116 url_plus_sha1 = deps.get(upstream_path) 117 if url_plus_sha1: 118 break 119 else: 120 raise merge_common.TemporaryMergeError( 121 'Could not find .DEPS.git entry for project %s. This probably ' 122 'means that the project list in merge_from_chromium.py needs to be ' 123 'updated.' % path) 124 match = re.match('(.*?)@(.*)', url_plus_sha1) 125 url = match.group(1) 126 sha1 = match.group(2) 127 logging.debug(' Got URL %s and SHA1 %s for project %s', url, sha1, path) 128 result[path] = {'url': url, 'sha1': sha1} 129 return result 130 131 132def _MergeProjects(version, root_sha1, target, unattended, buildspec_url): 133 """Merges each required Chromium project into the Android repository. 134 135 .DEPS.git is consulted to determine which revision each project must be merged 136 at. Only a whitelist of required projects are merged. 137 138 Args: 139 version: The version to mention in generated commit messages. 140 root_sha1: The git hash to merge in the root repository. 141 target: The target branch to merge to. 142 unattended: Run in unattended mode. 143 buildspec_url: URL for buildspec repository, when merging a branch. 144 Raises: 145 TemporaryMergeError: If incompatibly licensed code is left after pruning. 146 """ 147 # The logic for this step lives here, in the Android tree, as it makes no 148 # sense for a Chromium tree to know about this merge. 149 150 if unattended: 151 branch_create_flag = '-B' 152 else: 153 branch_create_flag = '-b' 154 branch_name = 'merge-from-chromium-%s' % version 155 156 logging.debug('Parsing DEPS ...') 157 if root_sha1: 158 deps_content = _ReadGitFile(root_sha1, '.DEPS.git') 159 else: 160 deps_content = _ReadGitFile('FETCH_HEAD', 161 'releases/' + version + '/.DEPS.git', 162 buildspec_url, 163 'master') 164 165 deps_vars = _ParseDEPS(deps_content) 166 167 merge_info = _GetProjectMergeInfo(merge_common.THIRD_PARTY_PROJECTS, 168 deps_vars) 169 170 for path in merge_info: 171 # webkit needs special handling as we have a local mirror 172 local_mirrored = path == 'third_party/WebKit' 173 url = merge_info[path]['url'] 174 sha1 = merge_info[path]['sha1'] 175 dest_dir = os.path.join(merge_common.REPOSITORY_ROOT, path) 176 if local_mirrored: 177 remote = 'history' 178 else: 179 remote = 'goog' 180 merge_common.GetCommandStdout(['git', 'checkout', 181 branch_create_flag, branch_name, 182 '-t', remote + '/' + target], 183 cwd=dest_dir) 184 if not local_mirrored or not root_sha1: 185 logging.debug('Fetching project %s at %s ...', path, sha1) 186 fetch_args = ['git', 'fetch', url, sha1] 187 merge_common.GetCommandStdout(fetch_args, cwd=dest_dir) 188 if merge_common.GetCommandStdout(['git', 'rev-list', '-1', 'HEAD..' + sha1], 189 cwd=dest_dir): 190 logging.debug('Merging project %s at %s ...', path, sha1) 191 # Merge conflicts make git merge return 1, so ignore errors 192 merge_common.GetCommandStdout(['git', 'merge', '--no-commit', sha1], 193 cwd=dest_dir, ignore_errors=True) 194 merge_common.CheckNoConflictsAndCommitMerge( 195 'Merge %s from %s at %s\n\n%s' % (path, url, sha1, AUTOGEN_MESSAGE), 196 cwd=dest_dir, unattended=unattended) 197 else: 198 logging.debug('No new commits to merge in project %s', path) 199 200 # Handle root repository separately. 201 merge_common.GetCommandStdout(['git', 'checkout', 202 branch_create_flag, branch_name, 203 '-t', 'history/' + target]) 204 if not root_sha1: 205 merge_info = _GetProjectMergeInfo([''], deps_vars) 206 url = merge_info['']['url'] 207 root_sha1 = merge_info['']['sha1'] 208 merge_common.GetCommandStdout(['git', 'fetch', url, root_sha1]) 209 logging.debug('Merging Chromium at %s ...', root_sha1) 210 # Merge conflicts make git merge return 1, so ignore errors 211 merge_common.GetCommandStdout(['git', 'merge', '--no-commit', root_sha1], 212 ignore_errors=True) 213 merge_common.CheckNoConflictsAndCommitMerge( 214 'Merge Chromium at %s (%s)\n\n%s' 215 % (version, root_sha1, AUTOGEN_MESSAGE), unattended=unattended) 216 217 logging.debug('Getting directories to exclude ...') 218 219 # We import this now that we have merged the latest version. 220 # It imports to a global in order that it can be used to generate NOTICE 221 # later. We also disable writing bytecode to keep the source tree clean. 222 sys.path.append(os.path.join(merge_common.REPOSITORY_ROOT, 'android_webview', 223 'tools')) 224 sys.dont_write_bytecode = True 225 global webview_licenses 226 import webview_licenses 227 import known_issues 228 229 for path, exclude_list in known_issues.KNOWN_INCOMPATIBLE.iteritems(): 230 logging.debug(' %s', '\n '.join(os.path.join(path, x) for x in 231 exclude_list)) 232 dest_dir = os.path.join(merge_common.REPOSITORY_ROOT, path) 233 merge_common.GetCommandStdout(['git', 'rm', '-rf', '--ignore-unmatch'] + 234 exclude_list, cwd=dest_dir) 235 if _ModifiedFilesInIndex(dest_dir): 236 merge_common.GetCommandStdout(['git', 'commit', '-m', 237 'Exclude unwanted directories'], 238 cwd=dest_dir) 239 240 241def _CheckLicenses(): 242 """Check that no incompatibly licensed directories exist.""" 243 directories_left_over = webview_licenses.GetIncompatibleDirectories() 244 if directories_left_over: 245 raise merge_common.TemporaryMergeError( 246 'Incompatibly licensed directories remain: ' + 247 '\n'.join(directories_left_over)) 248 249 250def _GenerateMakefiles(version, unattended): 251 """Run gyp to generate the Android build system makefiles. 252 253 Args: 254 version: The version to mention in generated commit messages. 255 unattended: Run in unattended mode. 256 """ 257 logging.debug('Generating makefiles ...') 258 259 # TODO(torne): come up with a way to deal with hooks from DEPS properly 260 261 # TODO(torne): The .tmp files are generated by 262 # third_party/WebKit/Source/WebCore/WebCore.gyp/WebCore.gyp into the source 263 # tree. We should avoid this, or at least use a more specific name to avoid 264 # accidentally removing or adding other files. 265 for path in merge_common.ALL_PROJECTS: 266 dest_dir = os.path.join(merge_common.REPOSITORY_ROOT, path) 267 merge_common.GetCommandStdout(['git', 'rm', '--ignore-unmatch', 268 'GypAndroid.*.mk', '*.target.*.mk', 269 '*.host.*.mk', '*.tmp'], cwd=dest_dir) 270 271 try: 272 merge_common.GetCommandStdout(['android_webview/tools/gyp_webview', 'all']) 273 except merge_common.MergeError as e: 274 if not unattended: 275 raise 276 else: 277 for path in merge_common.ALL_PROJECTS: 278 merge_common.GetCommandStdout( 279 ['git', 'reset', '--hard'], 280 cwd=os.path.join(merge_common.REPOSITORY_ROOT, path)) 281 raise merge_common.TemporaryMergeError('Makefile generation failed: ' + 282 str(e)) 283 284 for path in merge_common.ALL_PROJECTS: 285 dest_dir = os.path.join(merge_common.REPOSITORY_ROOT, path) 286 # git add doesn't have an --ignore-unmatch so we have to do this instead: 287 merge_common.GetCommandStdout(['git', 'add', '-f', 'GypAndroid.*.mk'], 288 ignore_errors=True, cwd=dest_dir) 289 merge_common.GetCommandStdout(['git', 'add', '-f', '*.target.*.mk'], 290 ignore_errors=True, cwd=dest_dir) 291 merge_common.GetCommandStdout(['git', 'add', '-f', '*.host.*.mk'], 292 ignore_errors=True, cwd=dest_dir) 293 merge_common.GetCommandStdout(['git', 'add', '-f', '*.tmp'], 294 ignore_errors=True, cwd=dest_dir) 295 # Only try to commit the makefiles if something has actually changed. 296 if _ModifiedFilesInIndex(dest_dir): 297 merge_common.GetCommandStdout( 298 ['git', 'commit', '-m', 299 'Update makefiles after merge of Chromium at %s\n\n%s' % 300 (version, AUTOGEN_MESSAGE)], cwd=dest_dir) 301 302 303def _ModifiedFilesInIndex(cwd=merge_common.REPOSITORY_ROOT): 304 """Returns true if git's index contains any changes.""" 305 status = merge_common.GetCommandStdout(['git', 'status', '--porcelain'], 306 cwd=cwd) 307 return re.search(r'^[MADRC]', status, flags=re.MULTILINE) is not None 308 309 310def _GenerateNoticeFile(version): 311 """Generates and commits a NOTICE file containing code licenses. 312 313 This covers all third-party code (from Android's perspective) that lives in 314 the Chromium tree. 315 316 Args: 317 version: The version to mention in generated commit messages. 318 """ 319 logging.debug('Regenerating NOTICE file ...') 320 321 contents = webview_licenses.GenerateNoticeFile() 322 323 with open(os.path.join(merge_common.REPOSITORY_ROOT, 'NOTICE'), 'w') as f: 324 f.write(contents) 325 merge_common.GetCommandStdout(['git', 'add', 'NOTICE']) 326 # Only try to commit the NOTICE update if the file has actually changed. 327 if _ModifiedFilesInIndex(): 328 merge_common.GetCommandStdout([ 329 'git', 'commit', '-m', 330 'Update NOTICE file after merge of Chromium at %s\n\n%s' 331 % (version, AUTOGEN_MESSAGE)]) 332 333 334def _GenerateLastChange(version): 335 """Write a build/util/LASTCHANGE file containing the current revision. 336 337 The revision number is compiled into the binary at build time from this file. 338 339 Args: 340 version: The version to mention in generated commit messages. 341 """ 342 logging.debug('Updating LASTCHANGE ...') 343 svn_revision, sha1 = _GetSVNRevisionAndSHA1('HEAD', 'HEAD') 344 with open(os.path.join(merge_common.REPOSITORY_ROOT, 'build/util/LASTCHANGE'), 345 'w') as f: 346 f.write('LASTCHANGE=%s\n' % svn_revision) 347 merge_common.GetCommandStdout(['git', 'add', '-f', 'build/util/LASTCHANGE']) 348 logging.debug('Updating LASTCHANGE.blink ...') 349 with open(os.path.join(merge_common.REPOSITORY_ROOT, 350 'build/util/LASTCHANGE.blink'), 'w') as f: 351 f.write('LASTCHANGE=%s\n' % _GetBlinkRevision()) 352 merge_common.GetCommandStdout(['git', 'add', '-f', 353 'build/util/LASTCHANGE.blink']) 354 if _ModifiedFilesInIndex(): 355 merge_common.GetCommandStdout([ 356 'git', 'commit', '-m', 357 'Update LASTCHANGE file after merge of Chromium at %s\n\n%s' 358 % (version, AUTOGEN_MESSAGE)]) 359 360 361def GetLKGR(): 362 """Fetch the last known good release from Chromium's dashboard. 363 364 Returns: 365 The last known good SVN revision. 366 """ 367 with contextlib.closing( 368 urllib2.urlopen('https://chromium-status.appspot.com/lkgr')) as lkgr: 369 return int(lkgr.read()) 370 371 372def GetHEAD(): 373 """Fetch the latest HEAD revision from the git mirror of the Chromium svn 374 repo. 375 376 Returns: 377 The latest HEAD SVN revision. 378 """ 379 (svn_revision, root_sha1) = _GetSVNRevisionAndSHA1(SRC_GIT_BRANCH, 380 'HEAD') 381 return int(svn_revision) 382 383 384def _ParseSvnRevisionFromGitCommitMessage(commit_message): 385 return re.search(r'^git-svn-id: .*@([0-9]+)', commit_message, 386 flags=re.MULTILINE).group(1) 387 388 389def _GetSVNRevisionFromSha(sha1): 390 commit = merge_common.GetCommandStdout([ 391 'git', 'show', '--format=%H%n%b', sha1]) 392 return _ParseSvnRevisionFromGitCommitMessage(commit) 393 394 395def _GetSVNRevisionAndSHA1(git_branch, svn_revision): 396 logging.debug('Getting SVN revision and SHA1 ...') 397 398 if svn_revision == 'HEAD': 399 # Just use the latest commit. 400 commit = merge_common.GetCommandStdout([ 401 'git', 'log', '-n1', '--grep=git-svn-id:', '--format=%H%n%b', 402 git_branch]) 403 sha1 = commit.split()[0] 404 svn_revision = _ParseSvnRevisionFromGitCommitMessage(commit) 405 return (svn_revision, sha1) 406 407 if svn_revision is None: 408 # Fetch LKGR from upstream. 409 svn_revision = GetLKGR() 410 output = merge_common.GetCommandStdout([ 411 'git', 'log', '--grep=git-svn-id: .*@%s' % svn_revision, 412 '--format=%H', git_branch]) 413 if not output: 414 raise merge_common.TemporaryMergeError('Revision %s not found in git repo.' 415 % svn_revision) 416 # The log grep will sometimes match reverts/reapplies of commits. We take the 417 # oldest (last) match because the first time it appears in history is 418 # overwhelmingly likely to be the correct commit. 419 sha1 = output.split()[-1] 420 return (svn_revision, sha1) 421 422 423def _GetBlinkRevision(): 424 commit = merge_common.GetCommandStdout([ 425 'git', 'log', '-n1', '--grep=git-svn-id:', '--format=%H%n%b'], 426 cwd=os.path.join(merge_common.REPOSITORY_ROOT, 'third_party', 'WebKit')) 427 return _ParseSvnRevisionFromGitCommitMessage(commit) 428 429 430def Snapshot(svn_revision, root_sha1, release, target, unattended, 431 buildspec_url): 432 """Takes a snapshot of the Chromium tree and merges it into Android. 433 434 Android makefiles and a top-level NOTICE file are generated and committed 435 after the merge. 436 437 Args: 438 svn_revision: The SVN revision in the Chromium repository to merge from. 439 root_sha1: The sha1 in the Chromium git mirror to merge from. 440 release: The Chromium release version to merge from (e.g. "30.0.1599.20"). 441 Only one of svn_revision, root_sha1 and release should be 442 specified. 443 target: The target branch to merge to. 444 unattended: Run in unattended mode. 445 buildspec_url: URL for buildspec repository, used when merging a release. 446 447 Returns: 448 True if new commits were merged; False if no new commits were present. 449 """ 450 if svn_revision: 451 svn_revision, root_sha1 = _GetSVNRevisionAndSHA1(SRC_GIT_BRANCH, 452 svn_revision) 453 elif root_sha1: 454 svn_revision = _GetSVNRevisionFromSha(root_sha1) 455 456 if svn_revision and root_sha1: 457 version = svn_revision 458 if not merge_common.GetCommandStdout(['git', 'rev-list', '-1', 459 'HEAD..' + root_sha1]): 460 logging.info('No new commits to merge at %s (%s)', 461 svn_revision, root_sha1) 462 return False 463 elif release: 464 version = release 465 root_sha1 = None 466 else: 467 raise merge_common.MergeError('No merge source specified') 468 469 logging.info('Snapshotting Chromium at %s (%s)', version, root_sha1) 470 471 # 1. Merge, accounting for excluded directories 472 _MergeProjects(version, root_sha1, target, unattended, buildspec_url) 473 474 # 2. Generate Android makefiles 475 _GenerateMakefiles(version, unattended) 476 477 # 3. Check for incompatible licenses 478 _CheckLicenses() 479 480 # 4. Generate Android NOTICE file 481 _GenerateNoticeFile(version) 482 483 # 5. Generate LASTCHANGE file 484 _GenerateLastChange(version) 485 486 return True 487 488 489def Push(version, target): 490 """Push the finished snapshot to the Android repository.""" 491 src = 'merge-from-chromium-%s' % version 492 # Use forced pushes ('+' prefix) for the temporary and archive branches in 493 # case they already got updated by a previous (possibly failed?) merge, but 494 # do not force push to the real master-chromium branch as this could erase 495 # downstream changes. 496 refspecs = ['%s:%s' % (src, target), 497 '+%s:refs/archive/chromium-%s' % (src, version)] 498 if target == 'master-chromium': 499 refspecs.insert(0, '+%s:master-chromium-merge' % src) 500 for refspec in refspecs: 501 logging.debug('Pushing to server (%s) ...' % refspec) 502 for path in merge_common.ALL_PROJECTS: 503 if path in merge_common.PROJECTS_WITH_FLAT_HISTORY: 504 remote = 'history' 505 else: 506 remote = 'goog' 507 logging.debug('Pushing %s', path) 508 dest_dir = os.path.join(merge_common.REPOSITORY_ROOT, path) 509 merge_common.GetCommandStdout(['git', 'push', remote, refspec], 510 cwd=dest_dir) 511 512 513def main(): 514 parser = optparse.OptionParser(usage='%prog [options]') 515 parser.epilog = ('Takes a snapshot of the Chromium tree at the specified ' 516 'Chromium SVN revision and merges it into this repository. ' 517 'Paths marked as excluded for license reasons are removed ' 518 'as part of the merge. Also generates Android makefiles and ' 519 'generates a top-level NOTICE file suitable for use in the ' 520 'Android build.') 521 parser.add_option( 522 '', '--svn_revision', 523 default=None, 524 help=('Merge to the specified chromium SVN revision, rather than using ' 525 'the current LKGR. Can also pass HEAD to merge from tip of tree. ' 526 'Only one of svn_revision, sha1 and release should be specified')) 527 parser.add_option( 528 '', '--sha1', 529 default=None, 530 help=('Merge to the specified chromium sha1 revision from ' + SRC_GIT_BRANCH 531 + ' branch, rather than using the current LKGR. Only one of' 532 'svn_revision, sha1 and release should be specified.')) 533 parser.add_option( 534 '', '--release', 535 default=None, 536 help=('Merge to the specified chromium release buildspec (e.g. ' 537 '"30.0.1599.20"). Only one of svn_revision, sha1 and release ' 538 'should be specified.')) 539 parser.add_option( 540 '', '--buildspec_url', 541 default=None, 542 help=('Git URL for buildspec repository.')) 543 parser.add_option( 544 '', '--target', 545 default='master-chromium', metavar='BRANCH', 546 help=('Target branch to push to. Defaults to master-chromium.')) 547 parser.add_option( 548 '', '--push', 549 default=False, action='store_true', 550 help=('Push the result of a previous merge to the server. Note ' 551 'svn_revision must be given.')) 552 parser.add_option( 553 '', '--get_lkgr', 554 default=False, action='store_true', 555 help=('Just print the current LKGR on stdout and exit.')) 556 parser.add_option( 557 '', '--get_head', 558 default=False, action='store_true', 559 help=('Just print the current HEAD revision on stdout and exit.')) 560 parser.add_option( 561 '', '--unattended', 562 default=False, action='store_true', 563 help=('Run in unattended mode.')) 564 parser.add_option( 565 '', '--no_changes_exit', 566 default=0, type='int', 567 help=('Exit code to use if there are no changes to merge, for scripts.')) 568 (options, args) = parser.parse_args() 569 if args: 570 parser.print_help() 571 return 1 572 573 if 'ANDROID_BUILD_TOP' not in os.environ: 574 print >>sys.stderr, 'You need to run the Android envsetup.sh and lunch.' 575 return 1 576 577 logging.basicConfig(format='%(message)s', level=logging.DEBUG, 578 stream=sys.stdout) 579 580 if options.get_lkgr: 581 print GetLKGR() 582 elif options.get_head: 583 logging.disable(logging.CRITICAL) # Prevent log messages 584 print GetHEAD() 585 elif options.push: 586 if options.release: 587 Push(options.release, options.target) 588 elif options.svn_revision: 589 Push(options.svn_revision, options.target) 590 else: 591 print >>sys.stderr, 'You need to pass the version to push.' 592 return 1 593 else: 594 if not Snapshot(options.svn_revision, options.sha1, options.release, 595 options.target, options.unattended, options.buildspec_url): 596 return options.no_changes_exit 597 598 return 0 599 600if __name__ == '__main__': 601 sys.exit(main()) 602