tools/release/releases.py

   1 #!/usr/bin/env python
   2 # Copyright 2014 the V8 project authors. All rights reserved.
   3 # Use of this source code is governed by a BSD-style license that can be
   4 # found in the LICENSE file.
   5
   6 # This script retrieves the history of all V8 branches and
   7 # their corresponding Chromium revisions.
   8
   9 # Requires a chromium checkout with branch heads:
  10 # gclient sync --with_branch_heads
  11 # gclient fetch
  12
  13 import argparse
  14 import csv
  15 import itertools
  16 import json
  17 import os
  18 import re
  19 import sys
  20
  21 from common_includes import *
  22
  23 CONFIG = {
  24   "BRANCHNAME": "retrieve-v8-releases",
  25   "PERSISTFILE_BASENAME": "/tmp/v8-releases-tempfile",
  26 }
  27
  28 # Expression for retrieving the bleeding edge revision from a commit message.
  29 PUSH_MSG_SVN_RE = re.compile(r".* \(based on bleeding_edge revision r(\d+)\)$")
  30 PUSH_MSG_GIT_RE = re.compile(r".* \(based on ([a-fA-F0-9]+)\)$")
  31
  32 # Expression for retrieving the merged patches from a merge commit message
  33 # (old and new format).
  34 MERGE_MESSAGE_RE = re.compile(r"^.*[M|m]erged (.+)(\)| into).*$", re.M)
  35
  36 CHERRY_PICK_TITLE_GIT_RE = re.compile(r"^.* \(cherry\-pick\)\.?$")
  37
  38 # New git message for cherry-picked CLs. One message per line.
  39 MERGE_MESSAGE_GIT_RE = re.compile(r"^Merged ([a-fA-F0-9]+)\.?$")
  40
  41 # Expression for retrieving reverted patches from a commit message (old and
  42 # new format).
  43 ROLLBACK_MESSAGE_RE = re.compile(r"^.*[R|r]ollback of (.+)(\)| in).*$", re.M)
  44
  45 # New git message for reverted CLs. One message per line.
  46 ROLLBACK_MESSAGE_GIT_RE = re.compile(r"^Rollback of ([a-fA-F0-9]+)\.?$")
  47
  48 # Expression for retrieving the code review link.
  49 REVIEW_LINK_RE = re.compile(r"^Review URL: (.+)$", re.M)
  50
  51 # Expression with three versions (historical) for extracting the v8 revision
  52 # from the chromium DEPS file.
  53 DEPS_RE = re.compile(r"""^\s*(?:["']v8_revision["']: ["']"""
  54                      """|\(Var\("googlecode_url"\) % "v8"\) \+ "\/trunk@"""
  55                      """|"http\:\/\/v8\.googlecode\.com\/svn\/trunk@)"""
  56                      """([^"']+)["'].*$""", re.M)
  57
  58 # Expression to pick tag and revision for bleeding edge tags. To be used with
  59 # output of 'svn log'.
  60 BLEEDING_EDGE_TAGS_RE = re.compile(
  61     r"A \/tags\/([^\s]+) \(from \/branches\/bleeding_edge\:(\d+)\)")
  62
  63 OMAHA_PROXY_URL = "http://omahaproxy.appspot.com/"
  64
  65 def SortBranches(branches):
  66   """Sort branches with version number names."""
  67   return sorted(branches, key=SortingKey, reverse=True)
  68
  69
  70 def FilterDuplicatesAndReverse(cr_releases):
  71   """Returns the chromium releases in reverse order filtered by v8 revision
  72   duplicates.
  73
  74   cr_releases is a list of [cr_rev, v8_hsh] reverse-sorted by cr_rev.
  75   """
  76   last = ""
  77   result = []
  78   for release in reversed(cr_releases):
  79     if last == release[1]:
  80       continue
  81     last = release[1]
  82     result.append(release)
  83   return result
  84
  85
  86 def BuildRevisionRanges(cr_releases):
  87   """Returns a mapping of v8 revision -> chromium ranges.
  88   The ranges are comma-separated, each range has the form R1:R2. The newest
  89   entry is the only one of the form R1, as there is no end range.
  90
  91   cr_releases is a list of [cr_rev, v8_hsh] reverse-sorted by cr_rev.
  92   cr_rev either refers to a chromium commit position or a chromium branch
  93   number.
  94   """
  95   range_lists = {}
  96   cr_releases = FilterDuplicatesAndReverse(cr_releases)
  97
  98   # Visit pairs of cr releases from oldest to newest.
  99   for cr_from, cr_to in itertools.izip(
 100       cr_releases, itertools.islice(cr_releases, 1, None)):
 101
 102     # Assume the chromium revisions are all different.
 103     assert cr_from[0] != cr_to[0]
 104
 105     ran = "%s:%d" % (cr_from[0], int(cr_to[0]) - 1)
 106
 107     # Collect the ranges in lists per revision.
 108     range_lists.setdefault(cr_from[1], []).append(ran)
 109
 110   # Add the newest revision.
 111   if cr_releases:
 112     range_lists.setdefault(cr_releases[-1][1], []).append(cr_releases[-1][0])
 113
 114   # Stringify and comma-separate the range lists.
 115   return dict((hsh, ", ".join(ran)) for hsh, ran in range_lists.iteritems())
 116
 117
 118 def MatchSafe(match):
 119   if match:
 120     return match.group(1)
 121   else:
 122     return ""
 123
 124
 125 class Preparation(Step):
 126   MESSAGE = "Preparation."
 127
 128   def RunStep(self):
 129     self.CommonPrepare()
 130     self.PrepareBranch()
 131
 132
 133 class RetrieveV8Releases(Step):
 134   MESSAGE = "Retrieve all V8 releases."
 135
 136   def ExceedsMax(self, releases):
 137     return (self._options.max_releases > 0
 138             and len(releases) > self._options.max_releases)
 139
 140   def GetMasterHashFromPush(self, title):
 141     return MatchSafe(PUSH_MSG_GIT_RE.match(title))
 142
 143   def GetMergedPatches(self, body):
 144     patches = MatchSafe(MERGE_MESSAGE_RE.search(body))
 145     if not patches:
 146       patches = MatchSafe(ROLLBACK_MESSAGE_RE.search(body))
 147       if patches:
 148         # Indicate reverted patches with a "-".
 149         patches = "-%s" % patches
 150     return patches
 151
 152   def GetMergedPatchesGit(self, body):
 153     patches = []
 154     for line in body.splitlines():
 155       patch = MatchSafe(MERGE_MESSAGE_GIT_RE.match(line))
 156       if patch:
 157         patches.append(patch)
 158       patch = MatchSafe(ROLLBACK_MESSAGE_GIT_RE.match(line))
 159       if patch:
 160         patches.append("-%s" % patch)
 161     return ", ".join(patches)
 162
 163
 164   def GetReleaseDict(
 165       self, git_hash, master_position, master_hash, branch, version,
 166       patches, cl_body):
 167     revision = self.GetCommitPositionNumber(git_hash)
 168     return {
 169       # The cr commit position number on the branch.
 170       "revision": revision,
 171       # The git revision on the branch.
 172       "revision_git": git_hash,
 173       # The cr commit position number on master.
 174       "master_position": master_position,
 175       # The same for git.
 176       "master_hash": master_hash,
 177       # The branch name.
 178       "branch": branch,
 179       # The version for displaying in the form 3.26.3 or 3.26.3.12.
 180       "version": version,
 181       # The date of the commit.
 182       "date": self.GitLog(n=1, format="%ci", git_hash=git_hash),
 183       # Merged patches if available in the form 'r1234, r2345'.
 184       "patches_merged": patches,
 185       # Default for easier output formatting.
 186       "chromium_revision": "",
 187       # Default for easier output formatting.
 188       "chromium_branch": "",
 189       # Link to the CL on code review. Candiates pushes are not uploaded,
 190       # so this field will be populated below with the recent roll CL link.
 191       "review_link": MatchSafe(REVIEW_LINK_RE.search(cl_body)),
 192       # Link to the commit message on google code.
 193       "revision_link": ("https://code.google.com/p/v8/source/detail?r=%s"
 194                         % revision),
 195     }
 196
 197   def GetRelease(self, git_hash, branch):
 198     self.ReadAndPersistVersion()
 199     base_version = [self["major"], self["minor"], self["build"]]
 200     version = ".".join(base_version)
 201     body = self.GitLog(n=1, format="%B", git_hash=git_hash)
 202
 203     patches = ""
 204     if self["patch"] != "0":
 205       version += ".%s" % self["patch"]
 206       if CHERRY_PICK_TITLE_GIT_RE.match(body.splitlines()[0]):
 207         patches = self.GetMergedPatchesGit(body)
 208       else:
 209         patches = self.GetMergedPatches(body)
 210
 211     if SortingKey("4.2.69") <= SortingKey(version):
 212       master_hash = self.GetLatestReleaseBase(version=version)
 213     else:
 214       # Legacy: Before version 4.2.69, the master revision was determined
 215       # by commit message.
 216       title = self.GitLog(n=1, format="%s", git_hash=git_hash)
 217       master_hash = self.GetMasterHashFromPush(title)
 218     master_position = ""
 219     if master_hash:
 220       master_position = self.GetCommitPositionNumber(master_hash)
 221     return self.GetReleaseDict(
 222         git_hash, master_position, master_hash, branch, version,
 223         patches, body), self["patch"]
 224
 225   def GetReleasesFromBranch(self, branch):
 226     self.GitReset(self.vc.RemoteBranch(branch))
 227     if branch == self.vc.MasterBranch():
 228       return self.GetReleasesFromMaster()
 229
 230     releases = []
 231     try:
 232       for git_hash in self.GitLog(format="%H").splitlines():
 233         if VERSION_FILE not in self.GitChangedFiles(git_hash):
 234           continue
 235         if self.ExceedsMax(releases):
 236           break  # pragma: no cover
 237         if not self.GitCheckoutFileSafe(VERSION_FILE, git_hash):
 238           break  # pragma: no cover
 239
 240         release, patch_level = self.GetRelease(git_hash, branch)
 241         releases.append(release)
 242
 243         # Follow branches only until their creation point.
 244         # TODO(machenbach): This omits patches if the version file wasn't
 245         # manipulated correctly. Find a better way to detect the point where
 246         # the parent of the branch head leads to the trunk branch.
 247         if branch != self.vc.CandidateBranch() and patch_level == "0":
 248           break
 249
 250     # Allow Ctrl-C interrupt.
 251     except (KeyboardInterrupt, SystemExit):  # pragma: no cover
 252       pass
 253
 254     # Clean up checked-out version file.
 255     self.GitCheckoutFileSafe(VERSION_FILE, "HEAD")
 256     return releases
 257
 258   def GetReleaseFromRevision(self, revision):
 259     releases = []
 260     try:
 261       if (VERSION_FILE not in self.GitChangedFiles(revision) or
 262           not self.GitCheckoutFileSafe(VERSION_FILE, revision)):
 263         print "Skipping revision %s" % revision
 264         return []  # pragma: no cover
 265
 266       branches = map(
 267           str.strip,
 268           self.Git("branch -r --contains %s" % revision).strip().splitlines(),
 269       )
 270       branch = ""
 271       for b in branches:
 272         if b.startswith("origin/"):
 273           branch = b.split("origin/")[1]
 274           break
 275         if b.startswith("branch-heads/"):
 276           branch = b.split("branch-heads/")[1]
 277           break
 278       else:
 279         print "Could not determine branch for %s" % revision
 280
 281       release, _ = self.GetRelease(revision, branch)
 282       releases.append(release)
 283
 284     # Allow Ctrl-C interrupt.
 285     except (KeyboardInterrupt, SystemExit):  # pragma: no cover
 286       pass
 287
 288     # Clean up checked-out version file.
 289     self.GitCheckoutFileSafe(VERSION_FILE, "HEAD")
 290     return releases
 291
 292
 293   def RunStep(self):
 294     self.GitCreateBranch(self._config["BRANCHNAME"])
 295     releases = []
 296     if self._options.branch == 'recent':
 297       # List every release from the last 7 days.
 298       revisions = self.GetRecentReleases(max_age=7 * DAY_IN_SECONDS)
 299       for revision in revisions:
 300         releases += self.GetReleaseFromRevision(revision)
 301     elif self._options.branch == 'all':  # pragma: no cover
 302       # Retrieve the full release history.
 303       for branch in self.vc.GetBranches():
 304         releases += self.GetReleasesFromBranch(branch)
 305       releases += self.GetReleasesFromBranch(self.vc.CandidateBranch())
 306       releases += self.GetReleasesFromBranch(self.vc.MasterBranch())
 307     else:  # pragma: no cover
 308       # Retrieve history for a specified branch.
 309       assert self._options.branch in (self.vc.GetBranches() +
 310           [self.vc.CandidateBranch(), self.vc.MasterBranch()])
 311       releases += self.GetReleasesFromBranch(self._options.branch)
 312
 313     self["releases"] = sorted(releases,
 314                               key=lambda r: SortingKey(r["version"]),
 315                               reverse=True)
 316
 317
 318 class UpdateChromiumCheckout(Step):
 319   MESSAGE = "Update the chromium checkout."
 320
 321   def RunStep(self):
 322     cwd = self._options.chromium
 323     self.GitFetchOrigin("+refs/heads/*:refs/remotes/origin/*",
 324                         "+refs/branch-heads/*:refs/remotes/branch-heads/*",
 325                         cwd=cwd)
 326     # Update v8 checkout in chromium.
 327     self.GitFetchOrigin(cwd=os.path.join(cwd, "v8"))
 328
 329
 330 def ConvertToCommitNumber(step, revision):
 331   # Simple check for git hashes.
 332   if revision.isdigit() and len(revision) < 8:
 333     return revision
 334   return step.GetCommitPositionNumber(
 335       revision, cwd=os.path.join(step._options.chromium, "v8"))
 336
 337
 338 class RetrieveChromiumV8Releases(Step):
 339   MESSAGE = "Retrieve V8 releases from Chromium DEPS."
 340
 341   def RunStep(self):
 342     cwd = self._options.chromium
 343
 344     # All v8 revisions we are interested in.
 345     releases_dict = dict((r["revision_git"], r) for r in self["releases"])
 346
 347     cr_releases = []
 348     count_past_last_v8 = 0
 349     try:
 350       for git_hash in self.GitLog(
 351           format="%H", grep="V8", branch="origin/master",
 352           path="DEPS", cwd=cwd).splitlines():
 353         deps = self.GitShowFile(git_hash, "DEPS", cwd=cwd)
 354         match = DEPS_RE.search(deps)
 355         if match:
 356           cr_rev = self.GetCommitPositionNumber(git_hash, cwd=cwd)
 357           if cr_rev:
 358             v8_hsh = match.group(1)
 359             cr_releases.append([cr_rev, v8_hsh])
 360
 361           if count_past_last_v8:
 362             count_past_last_v8 += 1  # pragma: no cover
 363
 364           if count_past_last_v8 > 20:
 365             break  # pragma: no cover
 366
 367           # Stop as soon as we find a v8 revision that we didn't fetch in the
 368           # v8-revision-retrieval part above (i.e. a revision that's too old).
 369           # Just iterate a few more times in case there were reverts.
 370           if v8_hsh not in releases_dict:
 371             count_past_last_v8 += 1  # pragma: no cover
 372
 373     # Allow Ctrl-C interrupt.
 374     except (KeyboardInterrupt, SystemExit):  # pragma: no cover
 375       pass
 376
 377     # Add the chromium ranges to the v8 candidates and master releases.
 378     all_ranges = BuildRevisionRanges(cr_releases)
 379
 380     for hsh, ranges in all_ranges.iteritems():
 381       releases_dict.get(hsh, {})["chromium_revision"] = ranges
 382
 383
 384 # TODO(machenbach): Unify common code with method above.
 385 class RetrieveChromiumBranches(Step):
 386   MESSAGE = "Retrieve Chromium branch information."
 387
 388   def RunStep(self):
 389     cwd = self._options.chromium
 390
 391     # All v8 revisions we are interested in.
 392     releases_dict = dict((r["revision_git"], r) for r in self["releases"])
 393
 394     # Filter out irrelevant branches.
 395     branches = filter(lambda r: re.match(r"branch-heads/\d+", r),
 396                       self.GitRemotes(cwd=cwd))
 397
 398     # Transform into pure branch numbers.
 399     branches = map(lambda r: int(re.match(r"branch-heads/(\d+)", r).group(1)),
 400                    branches)
 401
 402     branches = sorted(branches, reverse=True)
 403
 404     cr_branches = []
 405     count_past_last_v8 = 0
 406     try:
 407       for branch in branches:
 408         deps = self.GitShowFile(
 409             "refs/branch-heads/%d" % branch, "DEPS", cwd=cwd)
 410         match = DEPS_RE.search(deps)
 411         if match:
 412           v8_hsh = match.group(1)
 413           cr_branches.append([str(branch), v8_hsh])
 414
 415           if count_past_last_v8:
 416             count_past_last_v8 += 1  # pragma: no cover
 417
 418           if count_past_last_v8 > 20:
 419             break  # pragma: no cover
 420
 421           # Stop as soon as we find a v8 revision that we didn't fetch in the
 422           # v8-revision-retrieval part above (i.e. a revision that's too old).
 423           # Just iterate a few more times in case there were reverts.
 424           if v8_hsh not in releases_dict:
 425             count_past_last_v8 += 1  # pragma: no cover
 426
 427     # Allow Ctrl-C interrupt.
 428     except (KeyboardInterrupt, SystemExit):  # pragma: no cover
 429       pass
 430
 431     # Add the chromium branches to the v8 candidate releases.
 432     all_ranges = BuildRevisionRanges(cr_branches)
 433     for revision, ranges in all_ranges.iteritems():
 434       releases_dict.get(revision, {})["chromium_branch"] = ranges
 435
 436
 437 class RetrieveInformationOnChromeReleases(Step):
 438   MESSAGE = 'Retrieves relevant information on the latest Chrome releases'
 439
 440   def Run(self):
 441
 442     params = None
 443     result_raw = self.ReadURL(
 444                              OMAHA_PROXY_URL + "all.json",
 445                              params,
 446                              wait_plan=[5, 20]
 447                              )
 448     recent_releases = json.loads(result_raw)
 449
 450     canaries = []
 451
 452     for current_os in recent_releases:
 453       for current_version in current_os["versions"]:
 454         if current_version["channel"] != "canary":
 455           continue
 456
 457         current_candidate = self._CreateCandidate(current_version)
 458         canaries.append(current_candidate)
 459
 460     chrome_releases = {"canaries": canaries}
 461     self["chrome_releases"] = chrome_releases
 462
 463   def _GetGitHashForV8Version(self, v8_version):
 464     if v8_version.split(".")[3]== "0":
 465       return self.GitGetHashOfTag(v8_version[:-2])
 466
 467     return self.GitGetHashOfTag(v8_version)
 468
 469   def _CreateCandidate(self, current_version):
 470     params = None
 471     url_to_call = (OMAHA_PROXY_URL + "v8.json?version="
 472                    + current_version["previous_version"])
 473     result_raw = self.ReadURL(
 474                          url_to_call,
 475                          params,
 476                          wait_plan=[5, 20]
 477                          )
 478     previous_v8_version = json.loads(result_raw)["v8_version"]
 479     v8_previous_version_hash = self._GetGitHashForV8Version(previous_v8_version)
 480
 481     current_v8_version = current_version["v8_version"]
 482     v8_version_hash = self._GetGitHashForV8Version(current_v8_version)
 483
 484     current_candidate = {
 485                         "chrome_version": current_version["version"],
 486                         "os": current_version["os"],
 487                         "release_date": current_version["current_reldate"],
 488                         "v8_version": current_v8_version,
 489                         "v8_version_hash": v8_version_hash,
 490                         "v8_previous_version": previous_v8_version,
 491                         "v8_previous_version_hash": v8_previous_version_hash,
 492                        }
 493     return current_candidate
 494
 495
 496 class CleanUp(Step):
 497   MESSAGE = "Clean up."
 498
 499   def RunStep(self):
 500     self.CommonCleanup()
 501
 502
 503 class WriteOutput(Step):
 504   MESSAGE = "Print output."
 505
 506   def Run(self):
 507
 508     output = {
 509               "releases": self["releases"],
 510               "chrome_releases": self["chrome_releases"],
 511               }
 512
 513     if self._options.csv:
 514       with open(self._options.csv, "w") as f:
 515         writer = csv.DictWriter(f,
 516                                 ["version", "branch", "revision",
 517                                  "chromium_revision", "patches_merged"],
 518                                 restval="",
 519                                 extrasaction="ignore")
 520         for release in self["releases"]:
 521           writer.writerow(release)
 522     if self._options.json:
 523       with open(self._options.json, "w") as f:
 524         f.write(json.dumps(output))
 525     if not self._options.csv and not self._options.json:
 526       print output  # pragma: no cover
 527
 528
 529 class Releases(ScriptsBase):
 530   def _PrepareOptions(self, parser):
 531     parser.add_argument("-b", "--branch", default="recent",
 532                         help=("The branch to analyze. If 'all' is specified, "
 533                               "analyze all branches. If 'recent' (default) "
 534                               "is specified, track beta, stable and "
 535                               "candidates."))
 536     parser.add_argument("-c", "--chromium",
 537                         help=("The path to your Chromium src/ "
 538                               "directory to automate the V8 roll."))
 539     parser.add_argument("--csv", help="Path to a CSV file for export.")
 540     parser.add_argument("-m", "--max-releases", type=int, default=0,
 541                         help="The maximum number of releases to track.")
 542     parser.add_argument("--json", help="Path to a JSON file for export.")
 543
 544   def _ProcessOptions(self, options):  # pragma: no cover
 545     options.force_readline_defaults = True
 546     return True
 547
 548   def _Config(self):
 549     return {
 550       "BRANCHNAME": "retrieve-v8-releases",
 551       "PERSISTFILE_BASENAME": "/tmp/v8-releases-tempfile",
 552     }
 553
 554   def _Steps(self):
 555
 556     return [
 557       Preparation,
 558       RetrieveV8Releases,
 559       UpdateChromiumCheckout,
 560       RetrieveChromiumV8Releases,
 561       RetrieveChromiumBranches,
 562       RetrieveInformationOnChromeReleases,
 563       CleanUp,
 564       WriteOutput,
 565     ]
 566
 567
 568 if __name__ == "__main__":  # pragma: no cover
 569   sys.exit(Releases().Run())