From a6a0bd884655f26b4b56b9ac2992bd91cb7b4d6d Mon Sep 17 00:00:00 2001 From: machenbach Date: Thu, 5 Mar 2015 02:50:00 -0800 Subject: [PATCH] Make auto-roller roll recent release based on timestamp. Before this change, out of two versions 4.2.13.1 and 4.2.14, the latter would have been considered the newest. Now, the timestamp of the commit determines the age, which allows to roll a patched version first. The auto-roller will only roll forward (i.e. there needs to be a commit range between the last roll and the new candidate). Additionally, this CL does some cleanups: - Use the sheriff detection mechanism in the auto-roller. - Require the roll revision parameter in the chromium_roll script to avoid redundancy. The auto_roll script determines that revision automatically. - Simplify the revision summary in the commit message. The summary will now show last_roll..new_roll, which e.g. includes the version change CL. It'll now show useful information for cherry-picks, which it didn't before. - Remove unused clusterfuzz check. That check is part of the release process script. TBR=tandrii@chromium.org NOTRY=true TEST=./script_test.py TEST=./tools/release/chromium_roll.py --dry-run --sheriff -c ~/tmp/chromium/src --last-roll 55b9049ea3978a589d8db2aa191d21222eef737f fc263505535a175c8efa18f4c787bd92a1d1ab3d --use-commit-queue -r me -a you Review URL: https://codereview.chromium.org/977903002 Cr-Commit-Position: refs/heads/master@{#27009} --- tools/release/auto_roll.py | 51 ++++++++++++++++------------------------ tools/release/chromium_roll.py | 49 ++++++++++++++++---------------------- tools/release/common_includes.py | 1 + tools/release/releases.py | 2 +- tools/release/test_scripts.py | 46 ++++++++++++++++-------------------- 5 files changed, 62 insertions(+), 87 deletions(-) diff --git a/tools/release/auto_roll.py b/tools/release/auto_roll.py index ba8d298..8d21280 100755 --- a/tools/release/auto_roll.py +++ b/tools/release/auto_roll.py @@ -42,8 +42,11 @@ class DetectLastRoll(Step): MESSAGE = "Detect commit ID of the last Chromium roll." def RunStep(self): - # The revision that should be rolled. - latest_release = self.GetLatestRelease() + # The revision that should be rolled. Check for the latest of the most + # recent releases based on commit timestamp. + revisions = self.GetRecentReleases( + max_age=self._options.max_age * DAY_IN_SECONDS) + assert revisions, "Didn't find any recent release." # Interpret the DEPS file to retrieve the v8 revision. # TODO(machenbach): This should be part or the roll-deps api of @@ -54,38 +57,23 @@ class DetectLastRoll(Step): # The revision rolled last. self["last_roll"] = vars['v8_revision'] - # TODO(machenbach): It is possible that the auto-push script made a new - # fast-forward release (e.g. 4.2.3) while somebody patches the last - # candidate (e.g. 4.2.2.1). In this case, the auto-roller would pick - # the fast-forward release. Should there be a way to prioritize the - # patched version? - - if latest_release == self["last_roll"]: - # We always try to roll if the latest revision is not the revision in - # chromium. + # There must be some progress between the last roll and the new candidate + # revision (i.e. we don't go backwards). The revisions are ordered newest + # to oldest. It is possible that the newest timestamp has no progress + # compared to the last roll, i.e. if the newest release is a cherry-pick + # on a release branch. Then we look further. + for revision in revisions: + commits = self.GitLog( + format="%H", git_hash="%s..%s" % (self["last_roll"], revision)) + if commits: + self["roll"] = revision + break + else: print("There is no newer v8 revision than the one in Chromium (%s)." % self["last_roll"]) return True -class CheckClusterFuzz(Step): - MESSAGE = "Check ClusterFuzz api for new problems." - - def RunStep(self): - if not os.path.exists(self.Config("CLUSTERFUZZ_API_KEY_FILE")): - print "Skipping ClusterFuzz check. No api key file found." - return False - api_key = FileToText(self.Config("CLUSTERFUZZ_API_KEY_FILE")) - # Check for open, reproducible issues that have no associated bug. - result = self._side_effect_handler.ReadClusterFuzzAPI( - api_key, job_type="linux_asan_d8_dbg", reproducible="True", - open="True", bug_information="", - revision_greater_or_equal=str(self["last_push"])) - if result: - print "Stop due to pending ClusterFuzz issues." - return True - - class RollChromium(Step): MESSAGE = "Roll V8 into Chromium." @@ -97,6 +85,7 @@ class RollChromium(Step): "--chromium", self._options.chromium, "--last-roll", self["last_roll"], "--use-commit-queue", + self["roll"], ] if self._options.sheriff: args.append("--sheriff") @@ -112,6 +101,8 @@ class AutoRoll(ScriptsBase): parser.add_argument("-c", "--chromium", required=True, help=("The path to your Chromium src/ " "directory to automate the V8 roll.")) + parser.add_argument("--max-age", default=3, type=int, + help="Maximum age in days of the latest release.") parser.add_argument("--roll", help="Call Chromium roll script.", default=False, action="store_true") @@ -127,14 +118,12 @@ class AutoRoll(ScriptsBase): def _Config(self): return { "PERSISTFILE_BASENAME": "/tmp/v8-auto-roll-tempfile", - "CLUSTERFUZZ_API_KEY_FILE": ".cf_api_key", } def _Steps(self): return [ CheckActiveRoll, DetectLastRoll, - CheckClusterFuzz, RollChromium, ] diff --git a/tools/release/chromium_roll.py b/tools/release/chromium_roll.py index eb7c3b5..de0a569 100755 --- a/tools/release/chromium_roll.py +++ b/tools/release/chromium_roll.py @@ -24,30 +24,21 @@ class Preparation(Step): def RunStep(self): # Update v8 remote tracking branches. self.GitFetchOrigin() + self.Git("fetch origin +refs/tags/*:refs/tags/*") -class DetectLastPush(Step): - MESSAGE = "Detect commit ID of last release." +class PrepareRollCandidate(Step): + MESSAGE = "Robustness checks of the roll candidate." def RunStep(self): - # The revision that should be rolled. - self["last_push"] = self._options.last_push or self.GetLatestRelease() - self["push_title"] = self.GitLog(n=1, format="%s", - git_hash=self["last_push"]) + self["roll_title"] = self.GitLog(n=1, format="%s", + git_hash=self._options.roll) - # The master revision this release is based on. - self["push_base"] = self.GetLatestReleaseBase() - - # FIXME(machenbach): Manually specifying a revision doesn't work at the - # moment. Needs more complicated logic to find the correct push_base above. - # Maybe delete that parameter entirely? - assert not self._options.last_push - - # Determine the master revision of the last roll. + # Make sure the last roll and the roll candidate are releases. + version = self.GetVersionTag(self._options.roll) + assert version, "The revision to roll is not tagged." version = self.GetVersionTag(self._options.last_roll) - assert version - self["last_rolled_base"] = self.GetLatestReleaseBase(version=version) - assert self["last_rolled_base"] + assert version, "The revision used as last roll is not tagged." class SwitchChromium(Step): @@ -77,7 +68,7 @@ class UpdateChromiumCheckout(Step): # Update v8 remotes. self.GitFetchOrigin() - self.GitCreateBranch("v8-roll-%s" % self["last_push"], + self.GitCreateBranch("v8-roll-%s" % self._options.roll, cwd=self._options.chromium) @@ -87,15 +78,15 @@ class UploadCL(Step): def RunStep(self): # Patch DEPS file. if self.Command( - "roll-dep", "v8 %s" % self["last_push"], + "roll-dep", "v8 %s" % self._options.roll, cwd=self._options.chromium) is None: - self.Die("Failed to create deps for %s" % self["last_push"]) + self.Die("Failed to create deps for %s" % self._options.roll) message = [] - message.append("Update V8 to %s." % self["push_title"].lower()) + message.append("Update V8 to %s." % self["roll_title"].lower()) message.append( - ROLL_SUMMARY % (self["last_rolled_base"][:8], self["push_base"][:8])) + ROLL_SUMMARY % (self._options.last_roll[:8], self._options.roll[:8])) message.append(ISSUE_MSG) @@ -111,7 +102,7 @@ class UploadCL(Step): print "CL uploaded." else: self.GitCheckout("master", cwd=self._options.chromium) - self.GitDeleteBranch("v8-roll-%s" % self["last_push"], + self.GitDeleteBranch("v8-roll-%s" % self._options.roll, cwd=self._options.chromium) print "Dry run - don't upload." @@ -130,8 +121,8 @@ class CleanUp(Step): def RunStep(self): print("Congratulations, you have successfully rolled %s into " - "Chromium. Please don't forget to update the v8rel spreadsheet." - % self["last_push"]) + "Chromium." + % self._options.roll) # Clean up all temporary files. Command("rm", "-f %s*" % self._config["PERSISTFILE_BASENAME"]) @@ -142,10 +133,9 @@ class ChromiumRoll(ScriptsBase): parser.add_argument("-c", "--chromium", required=True, help=("The path to your Chromium src/ " "directory to automate the V8 roll.")) - parser.add_argument("-l", "--last-push", - help="The git commit ID of the last candidates push.") parser.add_argument("--last-roll", required=True, help="The git commit ID of the last rolled version.") + parser.add_argument("roll", nargs=1, help="Revision to roll."), parser.add_argument("--use-commit-queue", help="Check the CQ bit on upload.", default=False, action="store_true") @@ -158,6 +148,7 @@ class ChromiumRoll(ScriptsBase): options.requires_editor = False options.force = True options.manual = False + options.roll = options.roll[0] return True def _Config(self): @@ -168,7 +159,7 @@ class ChromiumRoll(ScriptsBase): def _Steps(self): return [ Preparation, - DetectLastPush, + PrepareRollCandidate, DetermineV8Sheriff, SwitchChromium, UpdateChromiumCheckout, diff --git a/tools/release/common_includes.py b/tools/release/common_includes.py index 895a23c..c967384 100644 --- a/tools/release/common_includes.py +++ b/tools/release/common_includes.py @@ -46,6 +46,7 @@ from git_recipes import GitRecipesMixin from git_recipes import GitFailedException CHANGELOG_FILE = "ChangeLog" +DAY_IN_SECONDS = 24 * 60 * 60 PUSH_MSG_GIT_RE = re.compile(r".* \(based on (?P[a-fA-F0-9]+)\)$") PUSH_MSG_NEW_RE = re.compile(r"^Version \d+\.\d+\.\d+$") VERSION_FILE = os.path.join("include", "v8-version.h") diff --git a/tools/release/releases.py b/tools/release/releases.py index 0f35e7c..10bf4d0 100755 --- a/tools/release/releases.py +++ b/tools/release/releases.py @@ -294,7 +294,7 @@ class RetrieveV8Releases(Step): releases = [] if self._options.branch == 'recent': # List every release from the last 7 days. - revisions = self.GetRecentReleases(max_age=7 * 24 * 60 * 60) + revisions = self.GetRecentReleases(max_age=7 * DAY_IN_SECONDS) for revision in revisions: releases += self.GetReleaseFromRevision(revision) elif self._options.branch == 'all': # pragma: no cover diff --git a/tools/release/test_scripts.py b/tools/release/test_scripts.py index fb495d1..68d30cd 100644 --- a/tools/release/test_scripts.py +++ b/tools/release/test_scripts.py @@ -64,7 +64,6 @@ TEST_CONFIG = { "ALREADY_MERGING_SENTINEL_FILE": "/tmp/test-merge-to-branch-tempfile-already-merging", "TEMPORARY_PATCH_FILE": "/tmp/test-merge-to-branch-tempfile-temporary-patch", - "CLUSTERFUZZ_API_KEY_FILE": "/tmp/test-fake-cf-api-key", } @@ -397,11 +396,6 @@ class ScriptTest(unittest.TestCase): else: return self._mock.Call("readurl", url) - def ReadClusterFuzzAPI(self, api_key, **params): - # TODO(machenbach): Use a mock for this and add a test that stops rolling - # due to clustefuzz results. - return [] - def Sleep(self, seconds): pass @@ -994,7 +988,7 @@ git-svn-id: https://v8.googlecode.com/svn/branches/bleeding_edge@123456 123 ROLL_COMMIT_MSG = """Update V8 to version 3.22.4 (based on abc). Summary of changes available at: -https://chromium.googlesource.com/v8/v8/+log/last_rol..abc +https://chromium.googlesource.com/v8/v8/+log/last_rol..roll_hsh Please follow these instructions for assigning/CC'ing issues: https://code.google.com/p/v8-wiki/wiki/TriagingIssues @@ -1018,18 +1012,10 @@ TBR=g_name@chromium.org,reviewer@chromium.org""" expectations = [ Cmd("git fetch origin", ""), Cmd("git fetch origin +refs/tags/*:refs/tags/*", ""), - Cmd("git tag", self.TAGS), - Cmd("git log -1 --format=%H 3.22.4", "push_hash\n"), - Cmd("git log -1 --format=%s push_hash", + Cmd("git log -1 --format=%s roll_hsh", "Version 3.22.4 (based on abc)\n"), - Cmd("git log -1 --format=%H 3.22.4", "push_hash\n"), - Cmd("git log -1 --format=%s push_hash", - "Version 3.22.4 (based on abc)"), + Cmd("git describe --tags roll_hsh", "3.22.4"), Cmd("git describe --tags last_roll_hsh", "3.22.2.1"), - Cmd("git log -1 --format=%H 3.22.2", "last_roll_base_hash"), - Cmd("git log -1 --format=%s last_roll_base_hash", "Version 3.22.2"), - Cmd("git log -1 --format=%H last_roll_base_hash^", - "last_roll_master_hash"), URL("https://chromium-build.appspot.com/p/chromium/sheriff_v8.js", "document.write('g_name')"), Cmd("git status -s -uno", "", cwd=chrome_dir), @@ -1037,8 +1023,8 @@ TBR=g_name@chromium.org,reviewer@chromium.org""" Cmd("gclient sync --nohooks", "syncing...", cwd=chrome_dir), Cmd("git pull", "", cwd=chrome_dir), Cmd("git fetch origin", ""), - Cmd("git new-branch v8-roll-push_hash", "", cwd=chrome_dir), - Cmd("roll-dep v8 push_hash", "rolled", cb=WriteDeps, cwd=chrome_dir), + Cmd("git new-branch v8-roll-roll_hsh", "", cwd=chrome_dir), + Cmd("roll-dep v8 roll_hsh", "rolled", cb=WriteDeps, cwd=chrome_dir), Cmd(("git commit -am \"%s\" " "--author \"author@chromium.org \"" % self.ROLL_COMMIT_MSG), @@ -1051,7 +1037,8 @@ TBR=g_name@chromium.org,reviewer@chromium.org""" args = ["-a", "author@chromium.org", "-c", chrome_dir, "--sheriff", "-r", "reviewer@chromium.org", - "--last-roll", "last_roll_hsh"] + "--last-roll", "last_roll_hsh", + "roll_hsh"] ChromiumRoll(TEST_CONFIG, self).Run(args) deps = FileToText(os.path.join(chrome_dir, "DEPS")) @@ -1123,8 +1110,13 @@ deps = { "owner=author%40chromium.org&limit=30&closed=3&format=json", ("{\"results\": [{\"subject\": \"different\"}]}")), Cmd("git fetch origin +refs/tags/*:refs/tags/*", ""), - Cmd("git tag", self.TAGS), - Cmd("git log -1 --format=%H 3.22.4", "push_hash\n"), + Cmd("git rev-list --max-age=740800 --tags", + "bad_tag\nhash_234\nhash_123"), + Cmd("git describe --tags bad_tag", ""), + Cmd("git describe --tags hash_234", "3.22.4"), + Cmd("git describe --tags hash_123", "3.22.3"), + Cmd("git log --format=%H abcd123455..hash_234", ""), + Cmd("git log --format=%H abcd123455..hash_123", ""), ]) result = auto_roll.AutoRoll(TEST_CONFIG, self).Run( @@ -1134,16 +1126,18 @@ deps = { def testAutoRoll(self): TEST_CONFIG["CHROMIUM"] = self.MakeEmptyTempDirectory() TextToFile(self.FAKE_DEPS, os.path.join(TEST_CONFIG["CHROMIUM"], "DEPS")) - TEST_CONFIG["CLUSTERFUZZ_API_KEY_FILE"] = self.MakeEmptyTempFile() - TextToFile("fake key", TEST_CONFIG["CLUSTERFUZZ_API_KEY_FILE"]) self.Expect([ URL("https://codereview.chromium.org/search", "owner=author%40chromium.org&limit=30&closed=3&format=json", ("{\"results\": [{\"subject\": \"different\"}]}")), Cmd("git fetch origin +refs/tags/*:refs/tags/*", ""), - Cmd("git tag", self.TAGS), - Cmd("git log -1 --format=%H 3.22.4", "push_hash\n"), + Cmd("git rev-list --max-age=740800 --tags", + "bad_tag\nhash_234\nhash_123"), + Cmd("git describe --tags bad_tag", ""), + Cmd("git describe --tags hash_234", "3.22.4"), + Cmd("git describe --tags hash_123", "3.22.3"), + Cmd("git log --format=%H abcd123455..hash_234", "hash1\nhash2\n"), ]) result = auto_roll.AutoRoll(TEST_CONFIG, self).Run( -- 2.7.4