1 #!/usr/bin/env python 2 # Copyright 2014 the V8 project authors. All rights reserved. 3 # Use of this source code is governed by a BSD-style license that can be 4 # found in the LICENSE file. 5 6 # This script retrieves the history of all V8 branches and 7 # their corresponding Chromium revisions. 8 9 # Requires a chromium checkout with branch heads: 10 # gclient sync --with_branch_heads 11 # gclient fetch 12 13 import argparse 14 import csv 15 import itertools 16 import json 17 import os 18 import re 19 import sys 20 21 from common_includes import * 22 23 CONFIG = { 24 "BRANCHNAME": "retrieve-v8-releases", 25 "PERSISTFILE_BASENAME": "/tmp/v8-releases-tempfile", 26 } 27 28 # Expression for retrieving the bleeding edge revision from a commit message. 29 PUSH_MSG_SVN_RE = re.compile(r".* \(based on bleeding_edge revision r(\d+)\)$") 30 PUSH_MSG_GIT_RE = re.compile(r".* \(based on ([a-fA-F0-9]+)\)$") 31 32 # Expression for retrieving the merged patches from a merge commit message 33 # (old and new format). 34 MERGE_MESSAGE_RE = re.compile(r"^.*[M|m]erged (.+)(\)| into).*$", re.M) 35 36 CHERRY_PICK_TITLE_GIT_RE = re.compile(r"^.* \(cherry\-pick\)\.?$") 37 38 # New git message for cherry-picked CLs. One message per line. 39 MERGE_MESSAGE_GIT_RE = re.compile(r"^Merged ([a-fA-F0-9]+)\.?$") 40 41 # Expression for retrieving reverted patches from a commit message (old and 42 # new format). 43 ROLLBACK_MESSAGE_RE = re.compile(r"^.*[R|r]ollback of (.+)(\)| in).*$", re.M) 44 45 # New git message for reverted CLs. One message per line. 46 ROLLBACK_MESSAGE_GIT_RE = re.compile(r"^Rollback of ([a-fA-F0-9]+)\.?$") 47 48 # Expression for retrieving the code review link. 49 REVIEW_LINK_RE = re.compile(r"^Review URL: (.+)$", re.M) 50 51 # Expression with three versions (historical) for extracting the v8 revision 52 # from the chromium DEPS file. 53 DEPS_RE = re.compile(r"""^\s*(?:["']v8_revision["']: ["']""" 54 """|\(Var\("googlecode_url"\) % "v8"\) \+ "\/trunk@""" 55 """|"http\:\/\/v8\.googlecode\.com\/svn\/trunk@)""" 56 """([^"']+)["'].*$""", re.M) 57 58 # Expression to pick tag and revision for bleeding edge tags. To be used with 59 # output of 'svn log'. 60 BLEEDING_EDGE_TAGS_RE = re.compile( 61 r"A \/tags\/([^\s]+) \(from \/branches\/bleeding_edge\:(\d+)\)") 62 63 OMAHA_PROXY_URL = "http://omahaproxy.appspot.com/" 64 65 def SortBranches(branches): 66 """Sort branches with version number names.""" 67 return sorted(branches, key=SortingKey, reverse=True) 68 69 70 def FilterDuplicatesAndReverse(cr_releases): 71 """Returns the chromium releases in reverse order filtered by v8 revision 72 duplicates. 73 74 cr_releases is a list of [cr_rev, v8_hsh] reverse-sorted by cr_rev. 75 """ 76 last = "" 77 result = [] 78 for release in reversed(cr_releases): 79 if last == release[1]: 80 continue 81 last = release[1] 82 result.append(release) 83 return result 84 85 86 def BuildRevisionRanges(cr_releases): 87 """Returns a mapping of v8 revision -> chromium ranges. 88 The ranges are comma-separated, each range has the form R1:R2. The newest 89 entry is the only one of the form R1, as there is no end range. 90 91 cr_releases is a list of [cr_rev, v8_hsh] reverse-sorted by cr_rev. 92 cr_rev either refers to a chromium commit position or a chromium branch 93 number. 94 """ 95 range_lists = {} 96 cr_releases = FilterDuplicatesAndReverse(cr_releases) 97 98 # Visit pairs of cr releases from oldest to newest. 99 for cr_from, cr_to in itertools.izip( 100 cr_releases, itertools.islice(cr_releases, 1, None)): 101 102 # Assume the chromium revisions are all different. 103 assert cr_from[0] != cr_to[0] 104 105 ran = "%s:%d" % (cr_from[0], int(cr_to[0]) - 1) 106 107 # Collect the ranges in lists per revision. 108 range_lists.setdefault(cr_from[1], []).append(ran) 109 110 # Add the newest revision. 111 if cr_releases: 112 range_lists.setdefault(cr_releases[-1][1], []).append(cr_releases[-1][0]) 113 114 # Stringify and comma-separate the range lists. 115 return dict((hsh, ", ".join(ran)) for hsh, ran in range_lists.iteritems()) 116 117 118 def MatchSafe(match): 119 if match: 120 return match.group(1) 121 else: 122 return "" 123 124 125 class Preparation(Step): 126 MESSAGE = "Preparation." 127 128 def RunStep(self): 129 self.CommonPrepare() 130 self.PrepareBranch() 131 132 133 class RetrieveV8Releases(Step): 134 MESSAGE = "Retrieve all V8 releases." 135 136 def ExceedsMax(self, releases): 137 return (self._options.max_releases > 0 138 and len(releases) > self._options.max_releases) 139 140 def GetMasterHashFromPush(self, title): 141 return MatchSafe(PUSH_MSG_GIT_RE.match(title)) 142 143 def GetMergedPatches(self, body): 144 patches = MatchSafe(MERGE_MESSAGE_RE.search(body)) 145 if not patches: 146 patches = MatchSafe(ROLLBACK_MESSAGE_RE.search(body)) 147 if patches: 148 # Indicate reverted patches with a "-". 149 patches = "-%s" % patches 150 return patches 151 152 def GetMergedPatchesGit(self, body): 153 patches = [] 154 for line in body.splitlines(): 155 patch = MatchSafe(MERGE_MESSAGE_GIT_RE.match(line)) 156 if patch: 157 patches.append(patch) 158 patch = MatchSafe(ROLLBACK_MESSAGE_GIT_RE.match(line)) 159 if patch: 160 patches.append("-%s" % patch) 161 return ", ".join(patches) 162 163 164 def GetReleaseDict( 165 self, git_hash, master_position, master_hash, branch, version, 166 patches, cl_body): 167 revision = self.GetCommitPositionNumber(git_hash) 168 return { 169 # The cr commit position number on the branch. 170 "revision": revision, 171 # The git revision on the branch. 172 "revision_git": git_hash, 173 # The cr commit position number on master. 174 "master_position": master_position, 175 # The same for git. 176 "master_hash": master_hash, 177 # The branch name. 178 "branch": branch, 179 # The version for displaying in the form 3.26.3 or 3.26.3.12. 180 "version": version, 181 # The date of the commit. 182 "date": self.GitLog(n=1, format="%ci", git_hash=git_hash), 183 # Merged patches if available in the form 'r1234, r2345'. 184 "patches_merged": patches, 185 # Default for easier output formatting. 186 "chromium_revision": "", 187 # Default for easier output formatting. 188 "chromium_branch": "", 189 # Link to the CL on code review. Candiates pushes are not uploaded, 190 # so this field will be populated below with the recent roll CL link. 191 "review_link": MatchSafe(REVIEW_LINK_RE.search(cl_body)), 192 # Link to the commit message on google code. 193 "revision_link": ("https://code.google.com/p/v8/source/detail?r=%s" 194 % revision), 195 } 196 197 def GetRelease(self, git_hash, branch): 198 self.ReadAndPersistVersion() 199 base_version = [self["major"], self["minor"], self["build"]] 200 version = ".".join(base_version) 201 body = self.GitLog(n=1, format="%B", git_hash=git_hash) 202 203 patches = "" 204 if self["patch"] != "0": 205 version += ".%s" % self["patch"] 206 if CHERRY_PICK_TITLE_GIT_RE.match(body.splitlines()[0]): 207 patches = self.GetMergedPatchesGit(body) 208 else: 209 patches = self.GetMergedPatches(body) 210 211 if SortingKey("4.2.69") <= SortingKey(version): 212 master_hash = self.GetLatestReleaseBase(version=version) 213 else: 214 # Legacy: Before version 4.2.69, the master revision was determined 215 # by commit message. 216 title = self.GitLog(n=1, format="%s", git_hash=git_hash) 217 master_hash = self.GetMasterHashFromPush(title) 218 master_position = "" 219 if master_hash: 220 master_position = self.GetCommitPositionNumber(master_hash) 221 return self.GetReleaseDict( 222 git_hash, master_position, master_hash, branch, version, 223 patches, body), self["patch"] 224 225 def GetReleasesFromBranch(self, branch): 226 self.GitReset(self.vc.RemoteBranch(branch)) 227 if branch == self.vc.MasterBranch(): 228 return self.GetReleasesFromMaster() 229 230 releases = [] 231 try: 232 for git_hash in self.GitLog(format="%H").splitlines(): 233 if VERSION_FILE not in self.GitChangedFiles(git_hash): 234 continue 235 if self.ExceedsMax(releases): 236 break # pragma: no cover 237 if not self.GitCheckoutFileSafe(VERSION_FILE, git_hash): 238 break # pragma: no cover 239 240 release, patch_level = self.GetRelease(git_hash, branch) 241 releases.append(release) 242 243 # Follow branches only until their creation point. 244 # TODO(machenbach): This omits patches if the version file wasn't 245 # manipulated correctly. Find a better way to detect the point where 246 # the parent of the branch head leads to the trunk branch. 247 if branch != self.vc.CandidateBranch() and patch_level == "0": 248 break 249 250 # Allow Ctrl-C interrupt. 251 except (KeyboardInterrupt, SystemExit): # pragma: no cover 252 pass 253 254 # Clean up checked-out version file. 255 self.GitCheckoutFileSafe(VERSION_FILE, "HEAD") 256 return releases 257 258 def GetReleaseFromRevision(self, revision): 259 releases = [] 260 try: 261 if (VERSION_FILE not in self.GitChangedFiles(revision) or 262 not self.GitCheckoutFileSafe(VERSION_FILE, revision)): 263 print "Skipping revision %s" % revision 264 return [] # pragma: no cover 265 266 branches = map( 267 str.strip, 268 self.Git("branch -r --contains %s" % revision).strip().splitlines(), 269 ) 270 branch = "" 271 for b in branches: 272 if b.startswith("origin/"): 273 branch = b.split("origin/")[1] 274 break 275 if b.startswith("branch-heads/"): 276 branch = b.split("branch-heads/")[1] 277 break 278 else: 279 print "Could not determine branch for %s" % revision 280 281 release, _ = self.GetRelease(revision, branch) 282 releases.append(release) 283 284 # Allow Ctrl-C interrupt. 285 except (KeyboardInterrupt, SystemExit): # pragma: no cover 286 pass 287 288 # Clean up checked-out version file. 289 self.GitCheckoutFileSafe(VERSION_FILE, "HEAD") 290 return releases 291 292 293 def RunStep(self): 294 self.GitCreateBranch(self._config["BRANCHNAME"]) 295 releases = [] 296 if self._options.branch == 'recent': 297 # List every release from the last 7 days. 298 revisions = self.GetRecentReleases(max_age=7 * DAY_IN_SECONDS) 299 for revision in revisions: 300 releases += self.GetReleaseFromRevision(revision) 301 elif self._options.branch == 'all': # pragma: no cover 302 # Retrieve the full release history. 303 for branch in self.vc.GetBranches(): 304 releases += self.GetReleasesFromBranch(branch) 305 releases += self.GetReleasesFromBranch(self.vc.CandidateBranch()) 306 releases += self.GetReleasesFromBranch(self.vc.MasterBranch()) 307 else: # pragma: no cover 308 # Retrieve history for a specified branch. 309 assert self._options.branch in (self.vc.GetBranches() + 310 [self.vc.CandidateBranch(), self.vc.MasterBranch()]) 311 releases += self.GetReleasesFromBranch(self._options.branch) 312 313 self["releases"] = sorted(releases, 314 key=lambda r: SortingKey(r["version"]), 315 reverse=True) 316 317 318 class UpdateChromiumCheckout(Step): 319 MESSAGE = "Update the chromium checkout." 320 321 def RunStep(self): 322 cwd = self._options.chromium 323 self.GitFetchOrigin("+refs/heads/*:refs/remotes/origin/*", 324 "+refs/branch-heads/*:refs/remotes/branch-heads/*", 325 cwd=cwd) 326 # Update v8 checkout in chromium. 327 self.GitFetchOrigin(cwd=os.path.join(cwd, "v8")) 328 329 330 def ConvertToCommitNumber(step, revision): 331 # Simple check for git hashes. 332 if revision.isdigit() and len(revision) < 8: 333 return revision 334 return step.GetCommitPositionNumber( 335 revision, cwd=os.path.join(step._options.chromium, "v8")) 336 337 338 class RetrieveChromiumV8Releases(Step): 339 MESSAGE = "Retrieve V8 releases from Chromium DEPS." 340 341 def RunStep(self): 342 cwd = self._options.chromium 343 344 # All v8 revisions we are interested in. 345 releases_dict = dict((r["revision_git"], r) for r in self["releases"]) 346 347 cr_releases = [] 348 count_past_last_v8 = 0 349 try: 350 for git_hash in self.GitLog( 351 format="%H", grep="V8", branch="origin/master", 352 path="DEPS", cwd=cwd).splitlines(): 353 deps = self.GitShowFile(git_hash, "DEPS", cwd=cwd) 354 match = DEPS_RE.search(deps) 355 if match: 356 cr_rev = self.GetCommitPositionNumber(git_hash, cwd=cwd) 357 if cr_rev: 358 v8_hsh = match.group(1) 359 cr_releases.append([cr_rev, v8_hsh]) 360 361 if count_past_last_v8: 362 count_past_last_v8 += 1 # pragma: no cover 363 364 if count_past_last_v8 > 20: 365 break # pragma: no cover 366 367 # Stop as soon as we find a v8 revision that we didn't fetch in the 368 # v8-revision-retrieval part above (i.e. a revision that's too old). 369 # Just iterate a few more times in case there were reverts. 370 if v8_hsh not in releases_dict: 371 count_past_last_v8 += 1 # pragma: no cover 372 373 # Allow Ctrl-C interrupt. 374 except (KeyboardInterrupt, SystemExit): # pragma: no cover 375 pass 376 377 # Add the chromium ranges to the v8 candidates and master releases. 378 all_ranges = BuildRevisionRanges(cr_releases) 379 380 for hsh, ranges in all_ranges.iteritems(): 381 releases_dict.get(hsh, {})["chromium_revision"] = ranges 382 383 384 # TODO(machenbach): Unify common code with method above. 385 class RetrieveChromiumBranches(Step): 386 MESSAGE = "Retrieve Chromium branch information." 387 388 def RunStep(self): 389 cwd = self._options.chromium 390 391 # All v8 revisions we are interested in. 392 releases_dict = dict((r["revision_git"], r) for r in self["releases"]) 393 394 # Filter out irrelevant branches. 395 branches = filter(lambda r: re.match(r"branch-heads/\d+", r), 396 self.GitRemotes(cwd=cwd)) 397 398 # Transform into pure branch numbers. 399 branches = map(lambda r: int(re.match(r"branch-heads/(\d+)", r).group(1)), 400 branches) 401 402 branches = sorted(branches, reverse=True) 403 404 cr_branches = [] 405 count_past_last_v8 = 0 406 try: 407 for branch in branches: 408 deps = self.GitShowFile( 409 "refs/branch-heads/%d" % branch, "DEPS", cwd=cwd) 410 match = DEPS_RE.search(deps) 411 if match: 412 v8_hsh = match.group(1) 413 cr_branches.append([str(branch), v8_hsh]) 414 415 if count_past_last_v8: 416 count_past_last_v8 += 1 # pragma: no cover 417 418 if count_past_last_v8 > 20: 419 break # pragma: no cover 420 421 # Stop as soon as we find a v8 revision that we didn't fetch in the 422 # v8-revision-retrieval part above (i.e. a revision that's too old). 423 # Just iterate a few more times in case there were reverts. 424 if v8_hsh not in releases_dict: 425 count_past_last_v8 += 1 # pragma: no cover 426 427 # Allow Ctrl-C interrupt. 428 except (KeyboardInterrupt, SystemExit): # pragma: no cover 429 pass 430 431 # Add the chromium branches to the v8 candidate releases. 432 all_ranges = BuildRevisionRanges(cr_branches) 433 for revision, ranges in all_ranges.iteritems(): 434 releases_dict.get(revision, {})["chromium_branch"] = ranges 435 436 437 class RetrieveInformationOnChromeReleases(Step): 438 MESSAGE = 'Retrieves relevant information on the latest Chrome releases' 439 440 def Run(self): 441 442 params = None 443 result_raw = self.ReadURL( 444 OMAHA_PROXY_URL + "all.json", 445 params, 446 wait_plan=[5, 20] 447 ) 448 recent_releases = json.loads(result_raw) 449 450 canaries = [] 451 452 for current_os in recent_releases: 453 for current_version in current_os["versions"]: 454 if current_version["channel"] != "canary": 455 continue 456 457 current_candidate = self._CreateCandidate(current_version) 458 canaries.append(current_candidate) 459 460 chrome_releases = {"canaries": canaries} 461 self["chrome_releases"] = chrome_releases 462 463 def _GetGitHashForV8Version(self, v8_version): 464 if v8_version == "N/A": 465 return "" 466 467 real_v8_version = v8_version 468 if v8_version.split(".")[3]== "0": 469 real_v8_version = v8_version[:-2] 470 471 try: 472 return self.GitGetHashOfTag(real_v8_version) 473 except GitFailedException: 474 return "" 475 476 def _CreateCandidate(self, current_version): 477 params = None 478 url_to_call = (OMAHA_PROXY_URL + "v8.json?version=" 479 + current_version["previous_version"]) 480 result_raw = self.ReadURL( 481 url_to_call, 482 params, 483 wait_plan=[5, 20] 484 ) 485 previous_v8_version = json.loads(result_raw)["v8_version"] 486 v8_previous_version_hash = self._GetGitHashForV8Version(previous_v8_version) 487 488 current_v8_version = current_version["v8_version"] 489 v8_version_hash = self._GetGitHashForV8Version(current_v8_version) 490 491 current_candidate = { 492 "chrome_version": current_version["version"], 493 "os": current_version["os"], 494 "release_date": current_version["current_reldate"], 495 "v8_version": current_v8_version, 496 "v8_version_hash": v8_version_hash, 497 "v8_previous_version": previous_v8_version, 498 "v8_previous_version_hash": v8_previous_version_hash, 499 } 500 return current_candidate 501 502 503 class CleanUp(Step): 504 MESSAGE = "Clean up." 505 506 def RunStep(self): 507 self.CommonCleanup() 508 509 510 class WriteOutput(Step): 511 MESSAGE = "Print output." 512 513 def Run(self): 514 515 output = { 516 "releases": self["releases"], 517 "chrome_releases": self["chrome_releases"], 518 } 519 520 if self._options.csv: 521 with open(self._options.csv, "w") as f: 522 writer = csv.DictWriter(f, 523 ["version", "branch", "revision", 524 "chromium_revision", "patches_merged"], 525 restval="", 526 extrasaction="ignore") 527 for release in self["releases"]: 528 writer.writerow(release) 529 if self._options.json: 530 with open(self._options.json, "w") as f: 531 f.write(json.dumps(output)) 532 if not self._options.csv and not self._options.json: 533 print output # pragma: no cover 534 535 536 class Releases(ScriptsBase): 537 def _PrepareOptions(self, parser): 538 parser.add_argument("-b", "--branch", default="recent", 539 help=("The branch to analyze. If 'all' is specified, " 540 "analyze all branches. If 'recent' (default) " 541 "is specified, track beta, stable and " 542 "candidates.")) 543 parser.add_argument("-c", "--chromium", 544 help=("The path to your Chromium src/ " 545 "directory to automate the V8 roll.")) 546 parser.add_argument("--csv", help="Path to a CSV file for export.") 547 parser.add_argument("-m", "--max-releases", type=int, default=0, 548 help="The maximum number of releases to track.") 549 parser.add_argument("--json", help="Path to a JSON file for export.") 550 551 def _ProcessOptions(self, options): # pragma: no cover 552 options.force_readline_defaults = True 553 return True 554 555 def _Config(self): 556 return { 557 "BRANCHNAME": "retrieve-v8-releases", 558 "PERSISTFILE_BASENAME": "/tmp/v8-releases-tempfile", 559 } 560 561 def _Steps(self): 562 563 return [ 564 Preparation, 565 RetrieveV8Releases, 566 UpdateChromiumCheckout, 567 RetrieveChromiumV8Releases, 568 RetrieveChromiumBranches, 569 RetrieveInformationOnChromeReleases, 570 CleanUp, 571 WriteOutput, 572 ] 573 574 575 if __name__ == "__main__": # pragma: no cover 576 sys.exit(Releases().Run()) 577