1 # Copyright (c) 2011-2012 The Chromium OS Authors. All rights reserved.
2 # Use of this source code is governed by a BSD-style license that can be
3 # found in the LICENSE file.
5 """Module that handles interactions with a Validation Pool.
7 The validation pool is the set of commits that are ready to be validated i.e.
8 ready for the commit queue to try.
21 from xml.dom import minidom
23 from chromite.buildbot import cbuildbot_results as results_lib
24 from chromite.buildbot import constants
25 from chromite.buildbot import lkgm_manager
26 from chromite.buildbot import manifest_version
27 from chromite.buildbot import portage_utilities
28 from chromite.lib import cros_build_lib
29 from chromite.lib import gerrit
30 from chromite.lib import git
31 from chromite.lib import gob_util
32 from chromite.lib import gs
33 from chromite.lib import parallel
34 from chromite.lib import patch as cros_patch
35 from chromite.lib import timeout_util
37 # Third-party libraries bundled with chromite need to be listed after the
38 # first chromite import.
41 # We import mox so that w/in ApplyPoolIntoRepo, if a mox exception is
42 # thrown, we don't cover it up.
49 PRE_CQ = constants.PRE_CQ
52 # The gerrit-on-borg team tells us that delays up to 2 minutes can be
53 # normal. Setting timeout to 3 minutes to be safe-ish.
54 SUBMITTED_WAIT_TIMEOUT = 3 * 60 # Time in seconds.
56 class TreeIsClosedException(Exception):
57 """Raised when the tree is closed and we wanted to submit changes."""
59 def __init__(self, closed_or_throttled=False):
63 closed_or_throttled: True if the exception is being thrown on a
64 possibly 'throttled' tree. False if only
65 thrown on a 'closed' tree. Default: False
67 if closed_or_throttled:
68 status_text = 'closed or throttled'
69 opposite_status_text = 'open'
71 status_text = 'closed'
72 opposite_status_text = 'throttled or open'
74 super(TreeIsClosedException, self).__init__(
75 'Tree is %s. Please set tree status to %s to '
76 'proceed.' % (status_text, opposite_status_text))
79 class FailedToSubmitAllChangesException(results_lib.StepFailure):
80 """Raised if we fail to submit any change."""
82 def __init__(self, changes):
83 super(FailedToSubmitAllChangesException, self).__init__(
84 'FAILED TO SUBMIT ALL CHANGES: Could not verify that changes %s were '
85 'submitted' % ' '.join(str(c) for c in changes))
88 class FailedToSubmitAllChangesNonFatalException(
89 FailedToSubmitAllChangesException):
90 """Raised if we fail to submit any change due to non-fatal errors."""
93 class InternalCQError(cros_patch.PatchException):
94 """Exception thrown when CQ has an unexpected/unhandled error."""
96 def __init__(self, patch, message):
97 cros_patch.PatchException.__init__(self, patch, message=message)
99 def ShortExplanation(self):
100 return 'failed to apply due to a CQ issue: %s' % (self.message,)
103 class NoMatchingChangeFoundException(Exception):
104 """Raised if we try to apply a non-existent change."""
107 class ChangeNotInManifestException(Exception):
108 """Raised if we try to apply a not-in-manifest change."""
111 class PatchNotCommitReady(cros_patch.PatchException):
112 """Raised if a patch is not marked as commit ready."""
114 def ShortExplanation(self):
115 return 'isn\'t marked as Commit-Ready anymore.'
118 class PatchNotPublished(cros_patch.PatchException):
119 """Raised if a patch is not published."""
121 def ShortExplanation(self):
122 return 'has not been published.'
125 class PatchRejected(cros_patch.PatchException):
126 """Raised if a patch was rejected by the CQ because the CQ failed."""
128 def ShortExplanation(self):
129 return 'was rejected by the CQ.'
132 class PatchFailedToSubmit(cros_patch.PatchException):
133 """Raised if we fail to submit a change."""
135 def ShortExplanation(self):
136 error = 'could not be submitted by the CQ.'
138 error += ' The error message from Gerrit was: %s' % (self.message,)
140 error += ' The Gerrit server might be having trouble.'
144 class PatchConflict(cros_patch.PatchException):
145 """Raised if a patch needs to be rebased."""
147 def ShortExplanation(self):
148 return ('no longer applies cleanly to tip of tree. Please rebase '
149 'and re-upload your patch.')
152 class PatchSubmittedWithoutDeps(cros_patch.DependencyError):
153 """Exception thrown when a patch was submitted incorrectly."""
155 def ShortExplanation(self):
156 dep_error = cros_patch.DependencyError.ShortExplanation(self)
157 return ('was submitted, even though it %s\n'
159 'You may want to revert your patch, and investigate why its'
160 'dependencies failed to submit.\n'
162 'This error only occurs when we have a dependency cycle, and we '
163 'submit one change before realizing that a later change cannot '
164 'be submitted.' % (dep_error,))
167 class PatchSeriesTooLong(cros_patch.PatchException):
168 """Exception thrown when a required dep isn't satisfied."""
170 def __init__(self, patch, max_length):
171 cros_patch.PatchException.__init__(self, patch)
172 self.max_length = max_length
174 def ShortExplanation(self):
175 return ("The Pre-CQ cannot handle a patch series longer than %s patches. "
176 "Please wait for some patches to be submitted before marking more "
177 "patches as ready. " % (self.max_length,))
180 return self.ShortExplanation()
183 def _RunCommand(cmd, dryrun):
184 """Runs the specified shell cmd if dryrun=False.
186 Errors are ignored, but logged.
189 logging.info('Would have run: %s', ' '.join(cmd))
193 cros_build_lib.RunCommand(cmd)
194 except cros_build_lib.RunCommandError:
195 cros_build_lib.Error('Command failed', exc_info=True)
198 def GetStagesToIgnoreFromConfigFile(config_path):
199 """Get a list of stage name prefixes to ignore from |config_path|.
201 This function reads the specified config file and returns the list
202 of stage name prefixes to ignore in the CQ. See GetStagesToIgnoreForChange
206 config_path: The path to the config file to read.
209 parser = ConfigParser.SafeConfigParser()
211 parser.read(config_path)
212 if parser.has_option('GENERAL', 'ignored-stages'):
213 ignored_stages = parser.get('GENERAL', 'ignored-stages').split()
214 except ConfigParser.Error:
215 cros_build_lib.Error('Error parsing %r', config_path, exc_info=True)
217 return ignored_stages
220 def GetStagesToIgnoreForChange(build_root, change):
221 """Get a list of stages that the CQ should ignore for a given |change|.
223 The list of stage name prefixes to ignore for each project is specified in a
224 config file inside the project, named COMMIT-QUEUE.ini. The file would look
228 ignored-stages: HWTest VMTest
230 The CQ will submit changes to the given project even if the listed stages
231 failed. These strings are stage name prefixes, meaning that "HWTest" would
232 match any HWTest stage (e.g. "HWTest [bvt]" or "HWTest [foo]")
235 build_root: The root of the checkout.
236 change: Change to examine.
239 A list of stages to ignore for the given |change|.
241 manifest = git.ManifestCheckout.Cached(build_root)
242 checkout = change.GetCheckout(manifest)
244 dirname = checkout.GetPath(absolute=True)
245 path = os.path.join(dirname, 'COMMIT-QUEUE.ini')
246 return GetStagesToIgnoreFromConfigFile(path)
250 class GerritHelperNotAvailable(gerrit.GerritException):
251 """Exception thrown when a specific helper is requested but unavailable."""
253 def __init__(self, remote=constants.EXTERNAL_REMOTE):
254 gerrit.GerritException.__init__(self)
255 # Stringify the pool so that serialization doesn't try serializing
256 # the actual HelperPool.
258 self.args = (remote,)
262 "Needed a remote=%s gerrit_helper, but one isn't allowed by this "
263 "HelperPool instance.") % (self.remote,)
266 class HelperPool(object):
267 """Pool of allowed GerritHelpers to be used by CQ/PatchSeries."""
269 def __init__(self, cros_internal=None, cros=None):
270 """Initialize this instance with the given handlers.
272 Most likely you want the classmethod SimpleCreate which takes boolean
275 If a given handler is None, then it's disabled; else the passed in
279 constants.EXTERNAL_REMOTE : cros,
280 constants.INTERNAL_REMOTE : cros_internal
284 def SimpleCreate(cls, cros_internal=True, cros=True):
285 """Classmethod helper for creating a HelperPool from boolean options.
288 cros_internal: If True, allow access to a GerritHelper for internal.
289 cros: If True, allow access to a GerritHelper for external.
292 An appropriately configured HelperPool instance.
295 cros = gerrit.GetGerritHelper(constants.EXTERNAL_REMOTE)
300 cros_internal = gerrit.GetGerritHelper(constants.INTERNAL_REMOTE)
304 return cls(cros_internal=cros_internal, cros=cros)
306 def ForChange(self, change):
307 """Return the helper to use for a particular change.
309 If no helper is configured, an Exception is raised.
311 return self.GetHelper(change.remote)
313 def GetHelper(self, remote):
314 """Return the helper to use for a given remote.
316 If no helper is configured, an Exception is raised.
318 helper = self.pool.get(remote)
320 raise GerritHelperNotAvailable(remote)
325 for helper in self.pool.itervalues():
330 def _PatchWrapException(functor):
331 """Decorator to intercept patch exceptions and wrap them.
333 Specifically, for known/handled Exceptions, it intercepts and
334 converts it into a DependencyError- via that, preserving the
335 cause, while casting it into an easier to use form (one that can
336 be chained in addition).
338 def f(self, parent, *args, **kwargs):
340 return functor(self, parent, *args, **kwargs)
341 except gerrit.GerritException as e:
342 if isinstance(e, gerrit.QueryNotSpecific):
343 e = ("%s\nSuggest you use gerrit numbers instead (prefixed with a * "
344 "if it's an internal change)." % e)
345 new_exc = cros_patch.PatchException(parent, e)
346 raise new_exc.__class__, new_exc, sys.exc_info()[2]
347 except cros_patch.PatchException as e:
348 if e.patch.id == parent.id:
350 new_exc = cros_patch.DependencyError(parent, e)
351 raise new_exc.__class__, new_exc, sys.exc_info()[2]
353 f.__name__ = functor.__name__
357 class PatchSeries(object):
358 """Class representing a set of patches applied to a single git repository."""
360 def __init__(self, path, helper_pool=None, force_content_merging=False,
361 forced_manifest=None, deps_filter_fn=None, is_submitting=False):
365 path: Path to the buildroot.
366 helper_pool: Pool of allowed GerritHelpers to be used for fetching
367 patches. Defaults to allowing both internal and external fetches.
368 force_content_merging: Allow merging of trivial conflicts, even if they
369 are disabled by Gerrit.
370 forced_manifest: A manifest object to use for mapping projects to
371 repositories. Defaults to the buildroot.
372 deps_filter_fn: A function which specifies what patches you would
373 like to accept. It is passed a patch and is expected to return
375 is_submitting: Whether we are currently submitting patchsets. This is
376 used to print better error messages.
378 self.manifest = forced_manifest
379 self._content_merging_projects = {}
380 self.force_content_merging = force_content_merging
382 if helper_pool is None:
383 helper_pool = HelperPool.SimpleCreate(cros_internal=True, cros=True)
384 self._helper_pool = helper_pool
386 if deps_filter_fn is None:
387 deps_filter_fn = lambda x:x
388 self.deps_filter_fn = deps_filter_fn
389 self._is_submitting = is_submitting
395 # A mapping of ChangeId to exceptions if the patch failed against
396 # ToT. Primarily used to keep the resolution/applying from going
397 # down known bad paths.
398 self._committed_cache = cros_patch.PatchCache()
399 self._lookup_cache = cros_patch.PatchCache()
400 self._change_deps_cache = {}
402 def _ManifestDecorator(functor):
403 """Method decorator that sets self.manifest automatically.
405 This function automatically initializes the manifest, and allows callers to
406 override the manifest if needed.
408 # pylint: disable=E0213,W0212,E1101,E1102
409 def f(self, *args, **kwargs):
410 manifest = kwargs.pop('manifest', None)
411 # Wipe is used to track if we need to reset manifest to None, and
412 # to identify if we already had a forced_manifest via __init__.
413 wipe = self.manifest is None
416 raise ValueError("manifest can't be specified when one is forced "
419 manifest = git.ManifestCheckout.Cached(self._path)
421 manifest = self.manifest
424 self.manifest = manifest
425 return functor(self, *args, **kwargs)
430 f.__name__ = functor.__name__
431 f.__doc__ = functor.__doc__
435 def GetGitRepoForChange(self, change, strict=False):
436 """Get the project path associated with the specified change.
439 change: The change to operate on.
440 strict: If True, throw ChangeNotInManifest rather than returning
441 None. Default: False.
444 The project path if found in the manifest. Otherwise returns
445 None (if strict=False).
449 checkout = change.GetCheckout(self.manifest, strict=strict)
450 if checkout is not None:
451 project_dir = checkout.GetPath(absolute=True)
456 def _IsContentMerging(self, change):
457 """Discern if the given change has Content Merging enabled in gerrit.
459 Note if the instance was created w/ force_content_merging=True,
460 then this function will lie and always return True to avoid the
461 admin-level access required of <=gerrit-2.1.
464 True if the change's project has content merging enabled, False if not.
467 AssertionError: If the gerrit helper requested is disallowed.
468 GerritException: If there is a failure in querying gerrit.
470 if self.force_content_merging:
472 return self.manifest.ProjectIsContentMerging(change.project)
475 def ApplyChange(self, change, dryrun=False):
476 # If we're in dryrun mode, then 3way is always allowed.
477 # Otherwise, allow 3way only if the gerrit project allows it.
478 trivial = False if dryrun else not self._IsContentMerging(change)
479 return change.ApplyAgainstManifest(self.manifest, trivial=trivial)
481 def _LookupHelper(self, patch):
482 """Returns the helper for the given cros_patch.PatchQuery object."""
483 return self._helper_pool.GetHelper(patch.remote)
485 def _GetGerritPatch(self, query):
486 """Query the configured helpers looking for a given change.
489 project: The gerrit project to query.
490 query: A cros_patch.PatchQuery object.
493 A GerritPatch object.
495 helper = self._LookupHelper(query)
496 query_text = query.ToGerritQueryText()
497 change = helper.QuerySingleRecord(
498 query_text, must_match=not git.IsSHA1(query_text))
503 # If the query was a gerrit number based query, check the projects/change-id
504 # to see if we already have it locally, but couldn't map it since we didn't
505 # know the gerrit number at the time of the initial injection.
506 existing = self._lookup_cache[change]
507 if cros_patch.ParseGerritNumber(query_text) and existing is not None:
508 keys = change.LookupAliases()
509 self._lookup_cache.InjectCustomKeys(keys, existing)
512 self.InjectLookupCache([change])
513 if change.IsAlreadyMerged():
514 self.InjectCommittedPatches([change])
517 def _LookupUncommittedChanges(self, deps, limit_to=None):
518 """Given a set of deps (changes), return unsatisfied dependencies.
521 deps: A list of cros_patch.PatchQuery objects representing
522 sequence of dependencies for the leaf that we need to identify
523 as either merged, or needing resolving.
524 limit_to: If non-None, then this must be a mapping (preferably a
525 cros_patch.PatchCache for translation reasons) of which non-committed
526 changes are allowed to be used for a transaction.
529 A sequence of cros_patch.GitRepoPatch instances (or derivatives) that
530 need to be resolved for this change to be mergable.
534 if dep in self._committed_cache:
538 self._LookupHelper(dep)
539 except GerritHelperNotAvailable:
540 # Internal dependencies are irrelevant to external builders.
541 logging.info("Skipping internal dependency: %s", dep)
544 dep_change = self._lookup_cache[dep]
546 if dep_change is None:
547 dep_change = self._GetGerritPatch(dep)
548 if dep_change is None:
550 if getattr(dep_change, 'IsAlreadyMerged', lambda: False)():
552 elif limit_to is not None and dep_change not in limit_to:
553 if self._is_submitting:
554 raise PatchRejected(dep_change)
556 raise PatchNotCommitReady(dep_change)
558 unsatisfied.append(dep_change)
560 # Perform last minute custom filtering.
561 return [x for x in unsatisfied if self.deps_filter_fn(x)]
563 def CreateTransaction(self, change, limit_to=None):
564 """Given a change, resolve it into a transaction.
566 In this case, a transaction is defined as a group of commits that
567 must land for the given change to be merged- specifically its
568 parent deps, and its CQ-DEPEND.
571 change: A cros_patch.GitRepoPatch instance to generate a transaction
573 limit_to: If non-None, limit the allowed uncommitted patches to
574 what's in that container/mapping.
577 A sequence of the necessary cros_patch.GitRepoPatch objects for
580 plan, seen = [], cros_patch.PatchCache()
581 self._AddChangeToPlanWithDeps(change, plan, seen, limit_to=limit_to)
584 def CreateTransactions(self, changes, limit_to=None):
585 """Create a list of transactions from a list of changes.
588 changes: A list of cros_patch.GitRepoPatch instances to generate
590 limit_to: See CreateTransaction docs.
593 A list of (change, plan, e) tuples for the given list of changes. The
594 plan represents the necessary GitRepoPatch objects for a given change. If
595 an exception occurs while creating the transaction, e will contain the
596 exception. (Otherwise, e will be None.)
598 for change in changes:
600 plan = self.CreateTransaction(change, limit_to=limit_to)
601 except cros_patch.PatchException as e:
602 yield (change, (), e)
604 yield (change, plan, None)
606 def CreateDisjointTransactions(self, changes, max_txn_length=None):
607 """Create a list of disjoint transactions from a list of changes.
610 changes: A list of cros_patch.GitRepoPatch instances to generate
612 max_txn_length: The maximum length of any given transaction. Optional.
613 By default, do not limit the length of transactions.
616 A list of disjoint transactions and a list of exceptions. Each transaction
617 can be tried independently, without involving patches from other
618 transactions. Each change in the pool will included in exactly one of the
619 transactions, unless the patch does not apply for some reason.
621 # Gather the dependency graph for the specified changes.
622 deps, edges, failed = {}, {}, []
623 for change, plan, ex in self.CreateTransactions(changes, limit_to=changes):
625 logging.info('Failed creating transaction for %s: %s', change, ex)
628 # Save off the ordered dependencies of this change.
631 # Mark every change in the transaction as bidirectionally connected.
632 for change_dep in plan:
633 edges.setdefault(change_dep, set()).update(plan)
635 # Calculate an unordered group of strongly connected components.
636 unordered_plans = digraph.StronglyConnectedComponents(list(edges), edges)
638 # Sort the groups according to our ordered dependency graph.
640 for unordered_plan in unordered_plans:
641 ordered_plan, seen = [], set()
642 for change in unordered_plan:
643 # Iterate over the required CLs, adding them to our plan in order.
644 new_changes = list(dep_change for dep_change in deps[change]
645 if dep_change not in seen)
646 new_plan_size = len(ordered_plan) + len(new_changes)
647 if not max_txn_length or new_plan_size <= max_txn_length:
648 seen.update(new_changes)
649 ordered_plan.extend(new_changes)
652 # We found a transaction that is <= max_txn_length. Process the
653 # transaction. Ignore the remaining patches for now; they will be
654 # processed later (once the current transaction has been pushed).
655 ordered_plans.append(ordered_plan)
657 # We couldn't find any transactions that were <= max_txn_length.
658 # This should only happen if circular dependencies prevent us from
659 # truncating a long list of patches. Reject the whole set of patches
661 for change in unordered_plan:
662 failed.append(PatchSeriesTooLong(change, max_txn_length))
664 return ordered_plans, failed
667 def _AddChangeToPlanWithDeps(self, change, plan, seen, limit_to=None):
668 """Add a change and its dependencies into a |plan|.
671 change: The change to add to the plan.
672 plan: The list of changes to apply, in order. This function will append
673 |change| and any necessary dependencies to |plan|.
674 seen: The changes whose Gerrit dependencies have already been processed.
675 limit_to: If non-None, limit the allowed uncommitted patches to
676 what's in that container/mapping.
679 DependencyError: If we could not resolve a dependency.
680 GerritException or GOBError: If there is a failure in querying gerrit.
682 if change in self._committed_cache or change in plan:
683 # If the requested change is already in the plan, then we have already
684 # processed its dependencies.
687 # Get a list of the changes that haven't been committed.
688 # These are returned as cros_patch.PatchQuery objects.
689 gerrit_deps, paladin_deps = self.GetDepsForChange(change)
691 # Only process the dependencies for each change once. We prioritize Gerrit
692 # dependencies over CQ dependencies, since Gerrit dependencies might be
693 # required in order for the change to apply.
694 if change not in seen:
695 gerrit_deps = self._LookupUncommittedChanges(
696 gerrit_deps, limit_to=limit_to)
698 for dep in gerrit_deps:
699 self._AddChangeToPlanWithDeps(dep, plan, seen, limit_to=limit_to)
701 # If there are cyclic dependencies, we might have already applied this
702 # patch as part of dependency resolution. If not, apply this patch.
703 if change not in plan:
706 # Process paladin deps last, so as to avoid circular dependencies between
707 # gerrit dependencies and paladin dependencies.
708 paladin_deps = self._LookupUncommittedChanges(
709 paladin_deps, limit_to=limit_to)
710 for dep in paladin_deps:
711 # Add the requested change (plus deps) to our plan, if it we aren't
712 # already in the process of doing that.
714 self._AddChangeToPlanWithDeps(dep, plan, seen, limit_to=limit_to)
717 def GetDepChangesForChange(self, change):
718 """Look up the gerrit/paladin dependency changes for |change|.
721 A tuple of GerritPatch objects which are change's Gerrit
722 dependencies, and Paladin dependencies.
725 DependencyError: If we could not resolve a dependency.
726 GerritException or GOBError: If there is a failure in querying gerrit.
728 gerrit_deps, paladin_deps = self.GetDepsForChange(change)
730 def _DepsToChanges(deps):
732 unprocessed_deps = []
734 dep_change = self._committed_cache[dep]
736 dep_changes.append(dep_change)
738 unprocessed_deps.append(dep)
740 for dep in unprocessed_deps:
741 dep_changes.extend(self._LookupUncommittedChanges(deps))
745 return _DepsToChanges(gerrit_deps), _DepsToChanges(paladin_deps)
748 def GetDepsForChange(self, change):
749 """Look up the gerrit/paladin deps for |change|.
752 A tuple of PatchQuery objects representing change's Gerrit
753 dependencies, and Paladin dependencies.
756 DependencyError: If we could not resolve a dependency.
757 GerritException or GOBError: If there is a failure in querying gerrit.
759 val = self._change_deps_cache.get(change)
761 git_repo = self.GetGitRepoForChange(change)
762 val = self._change_deps_cache[change] = (
763 change.GerritDependencies(),
764 change.PaladinDependencies(git_repo))
768 def InjectCommittedPatches(self, changes):
769 """Record that the given patches are already committed.
771 This is primarily useful for external code to notify this object
772 that changes were applied to the tree outside its purview- specifically
773 useful for dependency resolution.
775 self._committed_cache.Inject(*changes)
777 def InjectLookupCache(self, changes):
778 """Inject into the internal lookup cache the given changes, using them
779 (rather than asking gerrit for them) as needed for dependencies.
781 self._lookup_cache.Inject(*changes)
783 def FetchChanges(self, changes):
784 """Fetch the specified changes, if needed.
786 If we're an external builder, internal changes are filtered out.
789 An iterator over a list of the filtered changes.
791 for change in changes:
793 self._helper_pool.ForChange(change)
794 except GerritHelperNotAvailable:
795 # Internal patches are irrelevant to external builders.
796 logging.info("Skipping internal patch: %s", change)
798 change.Fetch(self.GetGitRepoForChange(change, strict=True))
802 def Apply(self, changes, dryrun=False, frozen=True,
803 honor_ordering=False, changes_filter=None):
804 """Applies changes from pool into the build root specified by the manifest.
806 This method resolves each given change down into a set of transactions-
807 the change and its dependencies- that must go in, then tries to apply
808 the largest transaction first, working its way down.
810 If a transaction cannot be applied, then it is rolled back
811 in full- note that if a change is involved in multiple transactions,
812 if an earlier attempt fails, that change can be retried in a new
813 transaction if the failure wasn't caused by the patch being incompatible
817 changes: A sequence of cros_patch.GitRepoPatch instances to resolve
819 dryrun: If True, then content-merging is explicitly forced,
820 and no modifications to gerrit will occur.
821 frozen: If True, then resolving of the given changes is explicitly
822 limited to just the passed in changes, or known committed changes.
823 This is basically CQ/Paladin mode, used to limit the changes being
824 pulled in/committed to just what we allow.
825 honor_ordering: Apply normally will reorder the transactions it
826 computes, trying the largest first, then degrading through smaller
827 transactions if the larger of the two fails. If honor_ordering
828 is False, then the ordering given via changes is preserved-
829 this is mainly of use for cbuildbot induced patching, and shouldn't
830 be used for CQ patching.
831 changes_filter: If not None, must be a functor taking two arguments:
832 series, changes; it must return the changes to work on.
833 This is invoked after the initial changes have been fetched,
834 thus this is a way for consumers to do last minute checking of the
835 changes being inspected, and expand the changes if necessary.
836 Primarily this is of use for cbuildbot patching when dealing w/
837 uploaded/remote patches.
840 A tuple of changes-applied, Exceptions for the changes that failed
841 against ToT, and Exceptions that failed inflight; These exceptions
842 are cros_patch.PatchException instances.
844 # Prefetch the changes; we need accurate change_id/id's, which is
845 # guaranteed via Fetch.
846 changes = list(self.FetchChanges(changes))
848 changes = changes_filter(self, changes)
850 self.InjectLookupCache(changes)
851 limit_to = cros_patch.PatchCache(changes) if frozen else None
852 resolved, applied, failed = [], [], []
853 for change, plan, ex in self.CreateTransactions(changes, limit_to=limit_to):
855 logging.info("Failed creating transaction for %s: %s", change, ex)
858 resolved.append((change, plan))
859 logging.info("Transaction for %s is %s.",
860 change, ', '.join(map(str, resolved[-1][-1])))
863 # No work to do; either no changes were given to us, or all failed
865 return [], failed, []
867 if not honor_ordering:
868 # Sort by length, falling back to the order the changes were given to us.
869 # This is done to prefer longer transactions (more painful to rebase)
870 # over shorter transactions.
871 position = dict((change, idx) for idx, change in enumerate(changes))
873 ids = [x.id for x in data[1]]
874 return -len(ids), position[data[0]]
875 resolved.sort(key=mk_key)
877 for inducing_change, transaction_changes in resolved:
879 with self._Transaction(transaction_changes):
880 logging.debug("Attempting transaction for %s: changes: %s",
882 ', '.join(map(str, transaction_changes)))
883 self._ApplyChanges(inducing_change, transaction_changes,
885 except cros_patch.PatchException as e:
886 logging.info("Failed applying transaction for %s: %s",
890 applied.extend(transaction_changes)
891 self.InjectCommittedPatches(transaction_changes)
893 # Uniquify while maintaining order.
901 applied = list(_uniq(applied))
902 self._is_submitting = True
904 failed = [x for x in failed if x.patch not in applied]
905 failed_tot = [x for x in failed if not x.inflight]
906 failed_inflight = [x for x in failed if x.inflight]
907 return applied, failed_tot, failed_inflight
909 @contextlib.contextmanager
910 def _Transaction(self, commits):
911 """ContextManager used to rollback changes to a build root if necessary.
913 Specifically, if an unhandled non system exception occurs, this context
914 manager will roll back all relevant modifications to the git repos
918 commits: A sequence of cros_patch.GitRepoPatch instances that compromise
919 this transaction- this is used to identify exactly what may be changed,
920 thus what needs to be tracked and rolled back if the transaction fails.
922 # First, the book keeping code; gather required data so we know what
923 # to rollback to should this transaction fail. Specifically, we track
924 # what was checked out for each involved repo, and if it was a branch,
925 # the sha1 of the branch; that information is enough to rewind us back
926 # to the original repo state.
928 map(functools.partial(self.GetGitRepoForChange, strict=True), commits))
930 for project_dir in project_state:
931 current_sha1 = git.RunGit(
932 project_dir, ['rev-list', '-n1', 'HEAD']).output.strip()
933 resets.append((project_dir, current_sha1))
936 committed_cache = self._committed_cache.copy()
940 # Reaching here means it was applied cleanly, thus return.
943 logging.info("Rewinding transaction: failed changes: %s .",
944 ', '.join(map(str, commits)), exc_info=True)
946 for project_dir, sha1 in resets:
947 git.RunGit(project_dir, ['reset', '--hard', sha1])
949 self._committed_cache = committed_cache
953 def _ApplyChanges(self, _inducing_change, changes, dryrun=False):
954 """Apply a given ordered sequence of changes.
957 _inducing_change: The core GitRepoPatch instance that lead to this
958 sequence of changes; basically what this transaction was computed from.
959 Needs to be passed in so that the exception wrapping machinery can
960 convert any failures, assigning blame appropriately.
961 manifest: A ManifestCheckout instance representing what we're working on.
962 changes: A ordered sequence of GitRepoPatch instances to apply.
963 dryrun: Whether or not this is considered a production run.
965 # Bail immediately if we know one of the requisite patches won't apply.
966 for change in changes:
967 failure = self.failed_tot.get(change.id)
968 if failure is not None:
972 for change in changes:
973 if change in self._committed_cache:
977 self.ApplyChange(change, dryrun=dryrun)
978 except cros_patch.PatchException as e:
980 self.failed_tot[change.id] = e
982 applied.append(change)
984 logging.debug('Done investigating changes. Applied %s',
985 ' '.join([c.id for c in applied]))
988 def WorkOnSingleRepo(cls, git_repo, tracking_branch, **kwargs):
989 """Classmethod to generate a PatchSeries that targets a single git repo.
991 It does this via forcing a fake manifest, which in turn points
992 tracking branch/paths/content-merging at what is passed through here.
995 git_repo: Absolute path to the git repository to operate upon.
996 tracking_branch: Which tracking branch patches should apply against.
997 kwargs: See PatchSeries.__init__ for the various optional args;
998 not forced_manifest cannot be used here, and force_content_merging
999 defaults to True in this usage.
1002 A PatchSeries instance w/ a forced manifest.
1005 if 'forced_manifest' in kwargs:
1006 raise ValueError("RawPatchSeries doesn't allow a forced_manifest "
1008 merging = kwargs.setdefault('force_content_merging', True)
1009 kwargs['forced_manifest'] = _ManifestShim(
1010 git_repo, tracking_branch, content_merging=merging)
1012 return cls(git_repo, **kwargs)
1015 class _ManifestShim(object):
1016 """A fake manifest that only contains a single repository.
1018 This fake manifest is used to allow us to filter out patches for
1019 the PatchSeries class. It isn't a complete implementation -- we just
1020 implement the functions that PatchSeries uses. It works via duck typing.
1022 All of the below methods accept the same arguments as the corresponding
1023 methods in git.ManifestCheckout.*, but they do not make any use of the
1024 arguments -- they just always return information about this project.
1027 def __init__(self, path, tracking_branch, remote='origin',
1028 content_merging=True):
1030 tracking_branch = 'refs/remotes/%s/%s' % (
1031 remote, git.StripRefs(tracking_branch),
1033 attrs = dict(local_path=path, path=path, tracking_branch=tracking_branch)
1034 self.checkout = git.ProjectCheckout(attrs)
1035 self.content_merging = content_merging
1037 def FindCheckouts(self, *_args, **_kwargs):
1038 """Returns the list of checkouts.
1040 In this case, we only have one repository so we just return that repository.
1041 We accept the same arguments as git.ManifestCheckout.FindCheckouts, but we
1042 do not make any use of them.
1045 A list of ProjectCheckout objects.
1047 return [self.checkout]
1049 def ProjectIsContentMerging(self, *_args, **_kwargs):
1050 """Check whether this project has content-merging enabled."""
1051 return self.content_merging
1054 class ValidationFailedMessage(object):
1055 """Message indicating that changes failed to be validated."""
1057 def __init__(self, message, tracebacks, internal):
1058 """Create a ValidationFailedMessage object.
1061 message: The message to print.
1062 tracebacks: Exceptions received by individual builders, if any.
1063 internal: Whether this failure occurred on an internal builder.
1065 # Convert each of the input arguments into simple Python datastructures
1066 # (i.e. not generators) that can be easily pickled.
1067 self.message = str(message)
1068 self.tracebacks = tuple(tracebacks)
1069 self.internal = bool(internal)
1074 def MightBeFlakyFailure(self):
1075 """Check if there is a good chance this is a flaky failure."""
1076 # We only consider a failed build to be flaky if there is only one failure,
1077 # and that failure is a flaky failure.
1079 if len(self.tracebacks) == 1:
1080 # TimeoutErrors are often flaky.
1081 exc = self.tracebacks[0].exception
1082 if (isinstance(exc, results_lib.StepFailure) and exc.possibly_flaky or
1083 isinstance(exc, timeout_util.TimeoutError)):
1087 def _MatchesFailureType(self, cls):
1088 """Check if all of the tracebacks match the specified failure type."""
1089 for traceback in self.tracebacks:
1090 if not isinstance(traceback.exception, cls):
1094 def IsPackageBuildFailure(self):
1095 """Check if all of the failures are package build failures."""
1096 return self._MatchesFailureType(results_lib.PackageBuildFailure)
1098 def FindPackageBuildFailureSuspects(self, changes):
1099 """Figure out what changes probably caused our failures.
1101 We use a fairly simplistic algorithm to calculate breakage: If you changed
1102 a package, and that package broke, you probably broke the build. If there
1103 were multiple changes to a broken package, we fail them all.
1105 Some safeguards are implemented to ensure that bad changes are kicked out:
1106 1) Changes to overlays (e.g. ebuilds, eclasses, etc.) are always kicked
1107 out if the build fails.
1108 2) If a package fails that nobody changed, we kick out all of the
1110 3) If any failures occur that we can't explain, we kick out all of the
1113 It is certainly possible to trick this algorithm: If one developer submits
1114 a change to libchromeos that breaks the power_manager, and another developer
1115 submits a change to the power_manager at the same time, only the
1116 power_manager change will be kicked out. That said, in that situation, the
1117 libchromeos change will likely be kicked out on the next run, thanks to
1121 changes: List of changes to examine.
1124 Set of changes that likely caused the failure.
1126 blame_everything = False
1128 for traceback in self.tracebacks:
1129 for package in traceback.exception.failed_packages:
1130 failed_projects = portage_utilities.FindWorkonProjects([package])
1131 blame_assigned = False
1132 for change in changes:
1133 if change.project in failed_projects:
1134 blame_assigned = True
1135 suspects.add(change)
1136 if not blame_assigned:
1137 blame_everything = True
1139 if blame_everything or not suspects:
1140 suspects = changes[:]
1142 # Never treat changes to overlays as innocent.
1143 suspects.update(change for change in changes
1144 if '/overlays/' in change.project)
1148 class CalculateSuspects(object):
1149 """Diagnose the cause for a given set of failures."""
1152 def _FindPackageBuildFailureSuspects(cls, changes, messages):
1153 """Figure out what CLs are at fault for a set of build failures."""
1155 for message in messages:
1156 suspects.update(message.FindPackageBuildFailureSuspects(changes))
1160 def _MightBeFlakyFailure(cls, messages):
1161 """Check if there is a good chance this is a flaky failure."""
1162 # We consider a failed commit queue run to be flaky if only one builder
1163 # failed, and that failure is flaky.
1164 return len(messages) == 1 and messages[0].MightBeFlakyFailure()
1167 def _FindPreviouslyFailedChanges(cls, candidates):
1168 """Find what changes that have previously failed the CQ.
1170 The first time a change is included in a build that fails due to a
1171 flaky (or apparently unrelated) failure, we assume that it is innocent. If
1172 this happens more than once, we kick out the CL.
1175 for change in candidates:
1176 if ValidationPool.GetCLStatusCount(
1177 CQ, change, ValidationPool.STATUS_FAILED):
1178 suspects.add(change)
1182 def FindSuspects(cls, changes, messages):
1183 """Find out what changes probably caused our failure.
1185 In cases where there were no internal failures, we can assume that the
1186 external failures are at fault. Otherwise, this function just defers to
1187 _FindPackagedBuildFailureSuspects and FindPreviouslyFailedChanges as needed.
1188 If the failures don't match either case, just fail everything.
1193 # If there were no internal failures, only kick out external changes.
1194 if any(message.internal for message in messages):
1195 candidates = changes
1197 candidates = [change for change in changes if not change.internal]
1199 if all(message.IsPackageBuildFailure() for message in messages):
1200 suspects = cls._FindPackageBuildFailureSuspects(candidates, messages)
1201 elif cls._MightBeFlakyFailure(messages):
1202 suspects = cls._FindPreviouslyFailedChanges(changes)
1204 suspects.update(candidates)
1209 class ValidationPool(object):
1210 """Class that handles interactions with a validation pool.
1212 This class can be used to acquire a set of commits that form a pool of
1213 commits ready to be validated and committed.
1215 Usage: Use ValidationPool.AcquirePool -- a static
1216 method that grabs the commits that are ready for validation.
1219 GLOBAL_DRYRUN = False
1220 MAX_TIMEOUT = 60 * 60 * 4
1222 STATUS_URL = 'https://chromiumos-status.appspot.com/current?format=json'
1223 STATUS_FAILED = manifest_version.BuilderStatus.STATUS_FAILED
1224 STATUS_INFLIGHT = manifest_version.BuilderStatus.STATUS_INFLIGHT
1225 STATUS_PASSED = manifest_version.BuilderStatus.STATUS_PASSED
1226 STATUS_LAUNCHING = 'launching'
1227 STATUS_WAITING = 'waiting'
1228 INCONSISTENT_SUBMIT_MSG = ('Gerrit thinks that the change was not submitted, '
1229 'even though we hit the submit button.')
1231 # The grace period (in seconds) before we reject a patch due to dependency
1233 REJECTION_GRACE_PERIOD = 30 * 60
1235 # Cache for the status of CLs.
1236 _CL_STATUS_CACHE = {}
1238 def __init__(self, overlays, build_root, build_number, builder_name,
1239 is_master, dryrun, changes=None, non_os_changes=None,
1240 conflicting_changes=None, pre_cq=False, metadata=None):
1241 """Initializes an instance by setting default variables to instance vars.
1243 Generally use AcquirePool as an entry pool to a pool rather than this
1247 overlays: One of constants.VALID_OVERLAYS.
1248 build_root: Build root directory.
1249 build_number: Build number for this validation attempt.
1250 builder_name: Builder name on buildbot dashboard.
1251 is_master: True if this is the master builder for the Commit Queue.
1252 dryrun: If set to True, do not submit anything to Gerrit.
1254 changes: List of changes for this validation pool.
1255 non_os_changes: List of changes that are part of this validation
1256 pool but aren't part of the cros checkout.
1257 conflicting_changes: Changes that failed to apply but we're keeping around
1258 because they conflict with other changes in flight.
1259 pre_cq: If set to True, this builder is verifying CLs before they go to
1261 metadata: Optional CBuildbotMetadata instance where CL actions will
1265 self.build_root = build_root
1267 # These instances can be instantiated via both older, or newer pickle
1268 # dumps. Thus we need to assert the given args since we may be getting
1269 # a value we no longer like (nor work with).
1270 if overlays not in constants.VALID_OVERLAYS:
1271 raise ValueError("Unknown/unsupported overlay: %r" % (overlays,))
1273 self._helper_pool = self.GetGerritHelpersForOverlays(overlays)
1275 if not isinstance(build_number, int):
1276 raise ValueError("Invalid build_number: %r" % (build_number,))
1278 if not isinstance(builder_name, basestring):
1279 raise ValueError("Invalid builder_name: %r" % (builder_name,))
1281 for changes_name, changes_value in (
1282 ('changes', changes), ('non_os_changes', non_os_changes)):
1283 if not changes_value:
1285 if not all(isinstance(x, cros_patch.GitRepoPatch) for x in changes_value):
1287 'Invalid %s: all elements must be a GitRepoPatch derivative, got %r'
1288 % (changes_name, changes_value))
1290 if conflicting_changes and not all(
1291 isinstance(x, cros_patch.PatchException)
1292 for x in conflicting_changes):
1294 'Invalid conflicting_changes: all elements must be a '
1295 'cros_patch.PatchException derivative, got %r'
1296 % (conflicting_changes,))
1298 self.build_log = self.ConstructDashboardURL(overlays, pre_cq, builder_name,
1301 self.is_master = bool(is_master)
1302 self.pre_cq = pre_cq
1303 self._metadata = metadata
1304 self.dryrun = bool(dryrun) or self.GLOBAL_DRYRUN
1305 self.queue = 'A trybot' if pre_cq else 'The Commit Queue'
1306 self.bot = PRE_CQ if pre_cq else CQ
1308 # See optional args for types of changes.
1309 self.changes = changes or []
1310 self.non_manifest_changes = non_os_changes or []
1311 # Note, we hold onto these CLs since they conflict against our current CLs
1312 # being tested; if our current ones succeed, we notify the user to deal
1313 # w/ the conflict. If the CLs we're testing fail, then there is no
1314 # reason we can't try these again in the next run.
1315 self.changes_that_failed_to_apply_earlier = conflicting_changes or []
1317 # Private vars only used for pickling.
1318 self._overlays = overlays
1319 self._build_number = build_number
1320 self._builder_name = builder_name
1323 def GetBuildDashboardForOverlays(overlays, trybot):
1324 """Discern the dashboard to use based on the given overlay."""
1326 return constants.TRYBOT_DASHBOARD
1327 if overlays in [constants.PRIVATE_OVERLAYS, constants.BOTH_OVERLAYS]:
1328 return constants.BUILD_INT_DASHBOARD
1329 return constants.BUILD_DASHBOARD
1332 def ConstructDashboardURL(cls, overlays, trybot, builder_name, build_number,
1334 """Return the dashboard (buildbot) URL for this run
1337 overlays: One of constants.VALID_OVERLAYS.
1338 trybot: Boolean: is this a remote trybot?
1339 builder_name: Builder name on buildbot dashboard.
1340 build_number: Build number for this validation attempt.
1341 stage: Link directly to a stage log, else use the general landing page.
1344 The fully formed URL
1346 build_dashboard = cls.GetBuildDashboardForOverlays(overlays, trybot)
1347 url_suffix = 'builders/%s/builds/%s' % (builder_name, str(build_number))
1349 url_suffix += '/steps/%s/logs/stdio' % (stage,)
1350 url_suffix = urllib.quote(url_suffix)
1351 return os.path.join(build_dashboard, url_suffix)
1354 def GetGerritHelpersForOverlays(overlays):
1355 """Discern the allowed GerritHelpers to use based on the given overlay."""
1356 cros_internal = cros = False
1357 if overlays in [constants.PUBLIC_OVERLAYS, constants.BOTH_OVERLAYS, False]:
1360 if overlays in [constants.PRIVATE_OVERLAYS, constants.BOTH_OVERLAYS]:
1361 cros_internal = True
1363 return HelperPool.SimpleCreate(cros_internal=cros_internal, cros=cros)
1365 def __reduce__(self):
1366 """Used for pickling to re-create validation pool."""
1367 # NOTE: self._metadata is specifically excluded from the validation pool
1368 # pickle. We do not want the un-pickled validation pool to have a reference
1369 # to its own un-pickled metadata instance. Instead, we want to to refer
1370 # to the builder run's metadata instance. This is accomplished by setting
1371 # metadata at un-pickle time, in ValidationPool.Load(...).
1376 self.build_root, self._build_number, self._builder_name,
1377 self.is_master, self.dryrun, self.changes,
1378 self.non_manifest_changes,
1379 self.changes_that_failed_to_apply_earlier,
1383 def FilterDraftChanges(cls, changes):
1384 """Filter out draft changes based on the status of the latest patch set.
1386 Our Gerrit query cannot exclude changes whose latest patch set has
1387 not yet been published as long as there is one published patchset
1388 in the change. Such changes will fail when we try to merge them,
1389 which may lead to undesirable consequence (e.g. dependencies not
1393 changes: List of changes to filter.
1396 List of published changes.
1398 return [x for x in changes if not x.patch_dict['currentPatchSet']['draft']]
1401 def FilterNonMatchingChanges(cls, changes):
1402 """Filter out changes that don't actually match our query.
1404 Generally, Gerrit should only return patches that match our
1405 query. However, Gerrit keeps a query cache and the cached data may
1408 There are also race conditions (bugs in Gerrit) where the final
1409 patch won't match our query. Here's an example problem that this
1410 code fixes: If the Pre-CQ launcher picks up a CL while the CQ is
1411 committing the CL, it may catch a race condition where a new
1412 patchset has been created and committed by the CQ, but the CL is
1413 still treated as if it matches the query (which it doesn't,
1417 changes: List of changes to filter.
1420 List of changes that match our query.
1422 filtered_changes = []
1423 for change in changes:
1424 # Because the gerrit cache sometimes gets stale, double-check that the
1425 # change hasn't already been merged.
1426 if change.status != 'NEW':
1428 # Check whether the change should be rejected (e.g. verified:
1429 # -1, code-review: -2).
1430 should_reject = False
1431 for field, value in constants.DEFAULT_CQ_SHOULD_REJECT_FIELDS.iteritems():
1432 if change.HasApproval(field, value):
1433 should_reject = True
1438 # Check that the user (or chrome-bot) uploaded a new change under our
1439 # feet while Gerrit was in the middle of answering our query.
1440 for field, value in constants.DEFAULT_CQ_READY_FIELDS.iteritems():
1441 if not change.HasApproval(field, value):
1444 filtered_changes.append(change)
1446 return filtered_changes
1449 def AcquirePreCQPool(cls, *args, **kwargs):
1450 """See ValidationPool.__init__ for arguments."""
1451 kwargs.setdefault('pre_cq', True)
1452 kwargs.setdefault('is_master', True)
1453 pool = cls(*args, **kwargs)
1454 pool.RecordPatchesInMetadata()
1458 def AcquirePool(cls, overlays, repo, build_number, builder_name,
1459 dryrun=False, changes_query=None, check_tree_open=True,
1460 change_filter=None, throttled_ok=False, metadata=None):
1461 """Acquires the current pool from Gerrit.
1463 Polls Gerrit and checks for which changes are ready to be committed.
1464 Should only be called from master builders.
1467 overlays: One of constants.VALID_OVERLAYS.
1468 repo: The repo used to sync, to filter projects, and to apply patches
1470 build_number: Corresponding build number for the build.
1471 builder_name: Builder name on buildbot dashboard.
1472 dryrun: Don't submit anything to gerrit.
1473 changes_query: The gerrit query to use to identify changes; if None,
1474 uses the internal defaults.
1475 check_tree_open: If True, only return when the tree is open.
1476 change_filter: If set, use change_filter(pool, changes,
1477 non_manifest_changes) to filter out unwanted patches.
1478 throttled_ok: if |check_tree_open|, treat a throttled tree as open.
1480 metadata: Optional CBuildbotMetadata instance where CL actions will
1484 ValidationPool object.
1487 TreeIsClosedException: if the tree is closed (or throttled, if not
1490 if change_filter is None:
1491 change_filter = lambda _, x, y: (x, y)
1493 # We choose a longer wait here as we haven't committed to anything yet. By
1494 # doing this here we can reduce the number of builder cycles.
1495 end_time = time.time() + cls.MAX_TIMEOUT
1497 time_left = end_time - time.time()
1499 # Wait until the tree becomes open (or throttled, if |throttled_ok|,
1500 # and record the tree status in tree_status).
1503 tree_status = timeout_util.WaitForTreeStatus(
1504 cls.STATUS_URL, cls.SLEEP_TIMEOUT, timeout=time_left,
1505 throttled_ok=throttled_ok)
1506 except timeout_util.TimeoutError:
1507 raise TreeIsClosedException(closed_or_throttled=not throttled_ok)
1509 tree_status = constants.TREE_OPEN
1511 waiting_for = 'new CLs'
1513 # Select the right default gerrit query based on the the tree
1514 # status, or use custom |changes_query| if it was provided.
1515 using_default_query = (changes_query is None)
1516 if not using_default_query:
1517 query = changes_query
1518 elif tree_status == constants.TREE_THROTTLED:
1519 query = constants.THROTTLED_CQ_READY_QUERY
1520 waiting_for = 'new CQ+2 CLs or the tree to open'
1522 query = constants.DEFAULT_CQ_READY_QUERY
1524 # Sync so that we are up-to-date on what is committed.
1527 # Only master configurations should call this method.
1528 pool = ValidationPool(overlays, repo.directory, build_number,
1529 builder_name, True, dryrun, metadata=metadata)
1532 # Iterate through changes from all gerrit instances we care about.
1533 for helper in cls.GetGerritHelpersForOverlays(overlays):
1534 raw_changes = helper.Query(query, sort='lastUpdated')
1535 raw_changes.reverse()
1537 # Reload the changes because the data in the Gerrit cache may be stale.
1538 raw_changes = list(cls.ReloadChanges(raw_changes))
1540 # If we used a default query, verify the results match the query, to
1541 # prevent race conditions. Note, this filters using the conditions
1542 # of DEFAULT_CQ_READY_QUERY even if the tree is throttled. Since that
1543 # query is strictly more permissive than the throttled query, we are
1544 # not at risk of incorrectly losing any patches here. We only expose
1545 # ourselves to the minor race condititon that a CQ+2 patch could have
1546 # been marked as CQ+1 out from under us, but still end up being picked
1547 # up in a throttled CQ run.
1548 if using_default_query:
1549 published_changes = cls.FilterDraftChanges(raw_changes)
1550 draft_changes.extend(set(raw_changes) - set(published_changes))
1551 raw_changes = cls.FilterNonMatchingChanges(published_changes)
1553 changes, non_manifest_changes = ValidationPool._FilterNonCrosProjects(
1554 raw_changes, git.ManifestCheckout.Cached(repo.directory))
1555 pool.changes.extend(changes)
1556 pool.non_manifest_changes.extend(non_manifest_changes)
1558 for change in draft_changes:
1559 pool.HandleDraftChange(change)
1561 # Filter out unwanted changes.
1562 pool.changes, pool.non_manifest_changes = change_filter(
1563 pool, pool.changes, pool.non_manifest_changes)
1565 if (pool.changes or pool.non_manifest_changes or dryrun or time_left < 0
1566 or cls.ShouldExitEarly()):
1569 logging.info('Waiting for %s (%d minutes left)...', waiting_for,
1571 time.sleep(cls.SLEEP_TIMEOUT)
1573 pool.RecordPatchesInMetadata()
1577 def AcquirePoolFromManifest(cls, manifest, overlays, repo, build_number,
1578 builder_name, is_master, dryrun, metadata=None):
1579 """Acquires the current pool from a given manifest.
1581 This function assumes that you have already synced to the given manifest.
1584 manifest: path to the manifest where the pool resides.
1585 overlays: One of constants.VALID_OVERLAYS.
1586 repo: The repo used to filter projects and to apply patches against.
1587 build_number: Corresponding build number for the build.
1588 builder_name: Builder name on buildbot dashboard.
1589 is_master: Boolean that indicates whether this is a pool for a master.
1591 dryrun: Don't submit anything to gerrit.
1592 metadata: Optional CBuildbotMetadata instance where CL actions will
1596 ValidationPool object.
1598 pool = ValidationPool(overlays, repo.directory, build_number, builder_name,
1599 is_master, dryrun, metadata=metadata)
1600 manifest_dom = minidom.parse(manifest)
1601 pending_commits = manifest_dom.getElementsByTagName(
1602 lkgm_manager.PALADIN_COMMIT_ELEMENT)
1603 for pc in pending_commits:
1604 patch = cros_patch.GerritFetchOnlyPatch(
1605 pc.getAttribute(lkgm_manager.PALADIN_PROJECT_URL_ATTR),
1606 pc.getAttribute(lkgm_manager.PALADIN_PROJECT_ATTR),
1607 pc.getAttribute(lkgm_manager.PALADIN_REF_ATTR),
1608 pc.getAttribute(lkgm_manager.PALADIN_BRANCH_ATTR),
1609 pc.getAttribute(lkgm_manager.PALADIN_REMOTE_ATTR),
1610 pc.getAttribute(lkgm_manager.PALADIN_COMMIT_ATTR),
1611 pc.getAttribute(lkgm_manager.PALADIN_CHANGE_ID_ATTR),
1612 pc.getAttribute(lkgm_manager.PALADIN_GERRIT_NUMBER_ATTR),
1613 pc.getAttribute(lkgm_manager.PALADIN_PATCH_NUMBER_ATTR),
1614 owner_email=pc.getAttribute(lkgm_manager.PALADIN_OWNER_EMAIL_ATTR),)
1615 pool.changes.append(patch)
1617 pool.RecordPatchesInMetadata()
1621 def ShouldExitEarly(cls):
1622 """Return whether we should exit early.
1624 This function is intended to be overridden by tests or by subclasses.
1629 def _FilterNonCrosProjects(changes, manifest):
1630 """Filters changes to a tuple of relevant changes.
1632 There are many code reviews that are not part of Chromium OS and/or
1633 only relevant on a different branch. This method returns a tuple of (
1634 relevant reviews in a manifest, relevant reviews not in the manifest). Note
1635 that this function must be run while chromite is checked out in a
1636 repo-managed checkout.
1639 changes: List of GerritPatch objects.
1640 manifest: The manifest to check projects/branches against.
1643 Tuple of (relevant reviews in a manifest,
1644 relevant reviews not in the manifest).
1647 def IsCrosReview(change):
1648 return (change.project.startswith('chromiumos') or
1649 change.project.startswith('chromeos'))
1651 # First we filter to only Chromium OS repositories.
1652 changes = [c for c in changes if IsCrosReview(c)]
1654 changes_in_manifest = []
1655 changes_not_in_manifest = []
1656 for change in changes:
1657 if change.GetCheckout(manifest, strict=False):
1658 changes_in_manifest.append(change)
1660 changes_not_in_manifest.append(change)
1661 logging.info('Filtered change %s', change)
1663 return changes_in_manifest, changes_not_in_manifest
1666 def _FilterDependencyErrors(cls, errors):
1667 """Filter out ignorable DependencyError exceptions.
1669 If a dependency isn't marked as ready, or a dependency fails to apply,
1670 we only complain after REJECTION_GRACE_PERIOD has passed since the patch
1673 This helps in two situations:
1674 1) If the developer is in the middle of marking a stack of changes as
1675 ready, we won't reject their work until the grace period has passed.
1676 2) If the developer marks a big circular stack of changes as ready, and
1677 some change in the middle of the stack doesn't apply, the user will
1678 get a chance to rebase their change before we mark all the changes as
1681 This function filters out dependency errors that can be ignored due to
1685 errors: List of exceptions to filter.
1688 List of unfiltered exceptions.
1690 reject_timestamp = time.time() - cls.REJECTION_GRACE_PERIOD
1692 for error in errors:
1693 results.append(error)
1694 if reject_timestamp < error.patch.approval_timestamp:
1695 while error is not None:
1696 if isinstance(error, cros_patch.DependencyError):
1697 logging.info('Ignoring dependency errors for %s due to grace '
1698 'period', error.patch)
1701 error = getattr(error, 'error', None)
1705 def PrintLinksToChanges(cls, changes):
1706 """Print links to the specified |changes|.
1709 changes: A list of cros_patch.GerritPatch instances to generate
1712 # Completely fill the status cache in parallel.
1713 cls.FillCLStatusCache(CQ, changes)
1715 def SortKeyForChanges(change):
1716 all_failures = cls.GetCLStatusCount(CQ, change, cls.STATUS_FAILED,
1717 latest_patchset_only=False)
1718 failures = cls.GetCLStatusCount(CQ, change, cls.STATUS_FAILED)
1719 return (-all_failures, -failures,
1720 os.path.basename(change.project), change.gerrit_number)
1722 # Now, sort and print the changes.
1723 for change in sorted(changes, key=SortKeyForChanges):
1724 project = os.path.basename(change.project)
1725 gerrit_number = cros_patch.AddPrefix(change, change.gerrit_number)
1726 # We cannot print '@' in the link because it is used to separate
1727 # the display text and the URL.
1728 author = change.owner_email.replace('@', '-AT-')
1729 if (change.owner_email.endswith(constants.GOOGLE_EMAIL) or
1730 change.owner_email.endswith(constants.CHROMIUM_EMAIL)):
1731 author = change.owner
1733 s = '%s | %s | %s' % (project, author, gerrit_number)
1735 # Print a count of how many times a given CL has failed the CQ.
1736 all_failures = cls.GetCLStatusCount(CQ, change, cls.STATUS_FAILED,
1737 latest_patchset_only=False)
1738 failures = cls.GetCLStatusCount(CQ, change, cls.STATUS_FAILED)
1740 s += ' | fails:%d' % (failures,)
1741 if all_failures > failures:
1742 s += '(%d)' % (all_failures,)
1744 # Add a note if the latest patchset has already passed the CQ.
1745 passed = cls.GetCLStatusCount(CQ, change, cls.STATUS_PASSED)
1747 s += ' | passed:%d' % passed
1749 cros_build_lib.PrintBuildbotLink(s, change.url)
1751 def ApplyPoolIntoRepo(self, manifest=None):
1752 """Applies changes from pool into the directory specified by the buildroot.
1754 This method applies changes in the order specified. If the build
1755 is running as the master, it also respects the dependency
1756 order. Otherwise, the changes should already be listed in an order
1757 that will not break the dependency order.
1760 True if we managed to apply any changes.
1763 failed_tot = failed_inflight = {}
1764 patch_series = PatchSeries(self.build_root, helper_pool=self._helper_pool)
1767 # pylint: disable=E1123
1768 applied, failed_tot, failed_inflight = patch_series.Apply(
1769 self.changes, dryrun=self.dryrun, manifest=manifest)
1770 except (KeyboardInterrupt, RuntimeError, SystemExit):
1772 except Exception as e:
1773 if mox is not None and isinstance(e, mox.Error):
1777 'Unhandled exception occurred while applying changes: %s\n\n'
1778 'To be safe, we have kicked out all of the CLs, so that the '
1779 'commit queue does not go into an infinite loop retrying '
1782 links = ', '.join('CL:%s' % x.gerrit_number_str for x in self.changes)
1783 cros_build_lib.Error('%s\nAffected Patches are: %s', msg, links)
1784 errors = [InternalCQError(patch, msg) for patch in self.changes]
1785 self._HandleApplyFailure(errors)
1788 # Slaves do not need to create transactions and should simply
1789 # apply the changes serially, based on the order that the
1790 # changes were listed on the manifest.
1791 for change in self.changes:
1793 # pylint: disable=E1123
1794 patch_series.ApplyChange(change, dryrun=self.dryrun,
1796 except cros_patch.PatchException as e:
1797 # Fail if any patch cannot be applied.
1798 self._HandleApplyFailure([InternalCQError(change, e)])
1801 applied.append(change)
1803 self.PrintLinksToChanges(applied)
1806 inputs = [[change] for change in applied]
1807 parallel.RunTasksInProcessPool(self._HandleApplySuccess, inputs)
1809 failed_tot = self._FilterDependencyErrors(failed_tot)
1812 'The following changes could not cleanly be applied to ToT: %s',
1813 ' '.join([c.patch.id for c in failed_tot]))
1814 self._HandleApplyFailure(failed_tot)
1816 failed_inflight = self._FilterDependencyErrors(failed_inflight)
1819 'The following changes could not cleanly be applied against the '
1820 'current stack of patches; if this stack fails, they will be tried '
1821 'in the next run. Inflight failed changes: %s',
1822 ' '.join([c.patch.id for c in failed_inflight]))
1824 self.changes_that_failed_to_apply_earlier.extend(failed_inflight)
1825 self.changes = applied
1827 return bool(self.changes)
1830 def Load(filename, metadata=None):
1831 """Loads the validation pool from the file.
1834 filename: path of file to load from.
1835 metadata: Optional CBuildbotInstance to use as metadata object
1836 for loaded pool (as metadata instances do not survive
1839 with open(filename, 'rb') as p_file:
1840 pool = cPickle.load(p_file)
1841 pool._metadata = metadata
1842 # Because metadata is currently not surviving cbuildbot re-execution,
1843 # re-record that patches were picked up in the non-skipped run of
1845 # TODO(akeshet): Remove this code once metadata is being pickled and
1846 # passed across re-executions. See crbug.com/356930
1847 pool.RecordPatchesInMetadata()
1850 def Save(self, filename):
1851 """Serializes the validation pool."""
1852 with open(filename, 'wb') as p_file:
1853 cPickle.dump(self, p_file, protocol=cPickle.HIGHEST_PROTOCOL)
1855 # Note: All submit code, all gerrit code, and basically everything other
1856 # than patch resolution/applying needs to use .change_id from patch objects.
1857 # Basically all code from this point forward.
1858 def _SubmitChangeWithDeps(self, patch_series, change, errors, limit_to):
1859 """Submit |change| and its dependencies.
1861 If you call this function multiple times with the same PatchSeries, each
1862 CL will only be submitted once.
1865 patch_series: A PatchSeries() object.
1866 change: The change (a GerritPatch object) to submit.
1867 errors: A dictionary. This dictionary should contain all patches that have
1868 encountered errors, and map them to the associated exception object.
1869 limit_to: The list of patches that were approved by this CQ run. We will
1870 only consider submitting patches that are in this list.
1873 A copy of the errors object. If new errors have occurred while submitting
1874 this change, and those errors have prevented a change from being
1875 submitted, they will be added to the errors object.
1877 # Find out what patches we need to submit.
1878 errors = errors.copy()
1880 plan = patch_series.CreateTransaction(change, limit_to=limit_to)
1881 except cros_patch.PatchException as e:
1885 error_stack, submitted = [], []
1886 for dep_change in plan:
1887 # Has this change failed to submit before?
1888 dep_error = errors.get(dep_change)
1889 if dep_error is None and error_stack:
1890 # One of the dependencies failed to submit. Report an error.
1891 dep_error = cros_patch.DependencyError(dep_change, error_stack[-1])
1893 # If there were no errors, submit the patch.
1894 if dep_error is None:
1896 if self._SubmitChange(dep_change) or self.dryrun:
1897 submitted.append(dep_change)
1899 msg = self.INCONSISTENT_SUBMIT_MSG
1900 dep_error = PatchFailedToSubmit(dep_change, msg)
1901 except (gob_util.GOBError, gerrit.GerritException) as e:
1902 if getattr(e, 'http_status', None) == httplib.CONFLICT:
1903 dep_error = PatchConflict(dep_change)
1905 dep_error = PatchFailedToSubmit(dep_change, str(e))
1906 logging.error('%s', dep_error)
1908 # Add any error we saw to the stack.
1909 if dep_error is not None:
1910 logging.info('%s', dep_error)
1911 errors[dep_change] = dep_error
1912 error_stack.append(dep_error)
1914 # Track submitted patches so that we don't submit them again.
1915 patch_series.InjectCommittedPatches(submitted)
1917 # Look for incorrectly submitted patches. We only have this problem
1918 # when we have a dependency cycle, and we submit one change before
1919 # realizing that a later change cannot be submitted. For now, just
1920 # print an error message and notify the developers.
1922 # If you see this error a lot, consider implementing a best-effort
1923 # attempt at reverting changes.
1924 for submitted_change in submitted:
1925 gdeps, pdeps = patch_series.GetDepChangesForChange(submitted_change)
1926 for dep in gdeps + pdeps:
1927 dep_error = errors.get(dep)
1928 if dep_error is not None:
1929 error = PatchSubmittedWithoutDeps(submitted_change, dep_error)
1930 self._HandleIncorrectSubmission(error)
1931 logging.error('%s was erroneously submitted.', submitted_change)
1935 def SubmitChanges(self, changes, check_tree_open=True, throttled_ok=True):
1936 """Submits the given changes to Gerrit.
1939 changes: GerritPatch's to submit.
1940 check_tree_open: Whether to check that the tree is open before submitting
1941 changes. If this is False, TreeIsClosedException will never be raised.
1942 throttled_ok: if |check_tree_open|, treat a throttled tree as open
1945 A list of the changes that failed to submit.
1948 TreeIsClosedException: if the tree is closed.
1950 assert self.is_master, 'Non-master builder calling SubmitPool'
1951 assert not self.pre_cq, 'Trybot calling SubmitPool'
1953 # Mark all changes as successful.
1954 inputs = [[self.bot, change, self.STATUS_PASSED, self.dryrun]
1955 for change in changes]
1956 parallel.RunTasksInProcessPool(self.UpdateCLStatus, inputs)
1958 if (check_tree_open and not self.dryrun and not
1959 timeout_util.IsTreeOpen(self.STATUS_URL, self.SLEEP_TIMEOUT,
1960 timeout=self.MAX_TIMEOUT,
1961 throttled_ok=throttled_ok)):
1962 raise TreeIsClosedException(close_or_throttled=not throttled_ok)
1964 # First, reload all of the changes from the Gerrit server so that we have a
1965 # fresh view of their approval status. This is needed so that our filtering
1966 # that occurs below will be mostly up-to-date.
1968 changes = list(self.ReloadChanges(changes))
1970 # Filter out changes that are already merged (e.g. dev chumped the
1971 # CL during the CQ run). We do not consider these as errors, and
1972 # print out warnings instead.
1973 uncommitted_changes = [x for x in changes if not x.IsAlreadyMerged()]
1974 for change in set(changes) - set(uncommitted_changes):
1975 logging.warning('%s is already merged. It was most likely chumped during '
1976 'the current CQ run.', change)
1978 # Filter out the draft changes here to prevent the race condition
1979 # where user uploads a new draft patch set during the CQ run.
1980 published_changes = self.FilterDraftChanges(uncommitted_changes)
1981 for change in set(uncommitted_changes) - set(published_changes):
1982 errors[change] = PatchNotPublished(change)
1984 # Filter out changes that aren't marked as CR=+2, CQ=+1, V=+1 anymore, in
1985 # case the patch status changed during the CQ run.
1986 filtered_changes = self.FilterNonMatchingChanges(published_changes)
1987 for change in set(published_changes) - set(filtered_changes):
1988 errors[change] = PatchNotCommitReady(change)
1990 patch_series = PatchSeries(self.build_root, helper_pool=self._helper_pool)
1991 patch_series.InjectLookupCache(filtered_changes)
1992 for change in filtered_changes:
1993 errors = self._SubmitChangeWithDeps(patch_series, change, errors,
1996 for patch, error in errors.iteritems():
1997 logging.error('Could not submit %s', patch)
1998 self._HandleCouldNotSubmit(patch, error)
2002 def RecordPatchesInMetadata(self):
2003 """Mark all patches as having been picked up in metadata.json."""
2005 timestamp = int(time.time())
2006 for change in self.changes:
2007 self._metadata.RecordCLAction(change, constants.CL_ACTION_PICKED_UP,
2010 def ReloadChanges(cls, changes):
2011 """Reload the specified |changes| from the server.
2014 changes: A list of PatchQuery objects.
2017 A list of GerritPatch objects.
2019 return gerrit.GetGerritPatchInfoWithPatchQueries(changes)
2021 def _SubmitChange(self, change):
2022 """Submits patch using Gerrit Review."""
2023 logging.info('Change %s will be submitted', change)
2024 was_change_submitted = False
2025 helper = self._helper_pool.ForChange(change)
2026 helper.SubmitChange(change, dryrun=self.dryrun)
2027 updated_change = helper.QuerySingleRecord(change.gerrit_number)
2029 # If change is 'SUBMITTED' give gerrit some time to resolve that
2030 # to 'MERGED' or fail outright.
2031 if updated_change.status == 'SUBMITTED':
2033 return helper.QuerySingleRecord(change.gerrit_number)
2035 return value and value.status == 'SUBMITTED'
2038 updated_change = timeout_util.WaitForSuccess(
2039 _Retry, _Query, timeout=SUBMITTED_WAIT_TIMEOUT, period=1)
2040 except timeout_util.TimeoutError:
2041 # The change really is stuck on submitted, not merged, then.
2042 logging.warning('Timed out waiting for gerrit to finish submitting'
2043 ' change %s, but status is still "%s".',
2044 change.gerrit_number_str, updated_change.status)
2046 was_change_submitted = updated_change.status == 'MERGED'
2047 if not was_change_submitted:
2049 'Change %s was submitted to gerrit without errors, but gerrit is'
2050 ' reporting it with status "%s" (expected "MERGED").',
2051 change.gerrit_number_str, updated_change.status)
2052 if updated_change.status == 'SUBMITTED':
2053 # So far we have never seen a SUBMITTED CL that did not eventually
2054 # transition to MERGED. If it is stuck on SUBMITTED treat as MERGED.
2055 was_change_submitted = True
2056 logging.info('Proceeding now with the assumption that change %s'
2057 ' will eventually transition to "MERGED".',
2058 change.gerrit_number_str)
2060 logging.error('Most likely gerrit was unable to merge change %s.',
2061 change.gerrit_number_str)
2064 if was_change_submitted:
2065 action = constants.CL_ACTION_SUBMITTED
2067 action = constants.CL_ACTION_SUBMIT_FAILED
2068 self._metadata.RecordCLAction(change, action)
2070 return was_change_submitted
2072 def RemoveCommitReady(self, change):
2073 """Remove the commit ready bit for the specified |change|."""
2074 self._helper_pool.ForChange(change).RemoveCommitReady(change,
2077 self._metadata.RecordCLAction(change, constants.CL_ACTION_KICKED_OUT)
2079 def SubmitNonManifestChanges(self, check_tree_open=True):
2080 """Commits changes to Gerrit from Pool that aren't part of the checkout.
2083 check_tree_open: Whether to check that the tree is open before submitting
2084 changes. If this is False, TreeIsClosedException will never be raised.
2087 TreeIsClosedException: if the tree is closed.
2089 self.SubmitChanges(self.non_manifest_changes,
2090 check_tree_open=check_tree_open)
2092 def SubmitPool(self, check_tree_open=True, throttled_ok=True):
2093 """Commits changes to Gerrit from Pool. This is only called by a master.
2096 check_tree_open: Whether to check that the tree is open before submitting
2097 changes. If this is False, TreeIsClosedException will never be raised.
2098 throttled_ok: if |check_tree_open|, treat a throttled tree as open
2101 TreeIsClosedException: if the tree is closed.
2102 FailedToSubmitAllChangesException: if we can't submit a change.
2103 FailedToSubmitAllChangesNonFatalException: if we can't submit a change
2104 due to non-fatal errors.
2106 # Note that SubmitChanges can throw an exception if it can't
2107 # submit all changes; in that particular case, don't mark the inflight
2108 # failures patches as failed in gerrit- some may apply next time we do
2109 # a CQ run (since the submit state has changed, we have no way of
2110 # knowing). They *likely* will still fail, but this approach tries
2111 # to minimize wasting the developers time.
2112 errors = self.SubmitChanges(self.changes, check_tree_open=check_tree_open,
2113 throttled_ok=throttled_ok)
2115 # We don't throw a fatal error for the whitelisted
2116 # exceptions. These exceptions are mostly caused by human
2117 # intervention during the current run and have limited impact on
2119 whitelisted_exceptions = (PatchNotCommitReady,
2122 cros_patch.DependencyError,)
2124 if all(isinstance(e, whitelisted_exceptions) for e in errors.values()):
2125 raise FailedToSubmitAllChangesNonFatalException(errors)
2127 raise FailedToSubmitAllChangesException(errors)
2129 if self.changes_that_failed_to_apply_earlier:
2130 self._HandleApplyFailure(self.changes_that_failed_to_apply_earlier)
2132 def SubmitPartialPool(self, tracebacks):
2133 """If the build failed, push any CLs that don't care about the failure.
2135 Each project can specify a list of stages it does not care about in its
2136 COMMIT-QUEUE.ini file. Changes to that project will be submitted even if
2140 tracebacks: A list of RecordedTraceback objects. These objects represent
2141 the exceptions that failed the build.
2144 A list of the rejected changes.
2146 # Create a list of the failing stage prefixes.
2147 failing_stages = set(traceback.failed_prefix for traceback in tracebacks)
2149 # For each CL, look at whether it cares about the failures. Based on this,
2150 # categorize the CL as accepted or rejected.
2151 accepted, rejected = [], []
2152 for change in self.changes:
2153 ignored_stages = GetStagesToIgnoreForChange(self.build_root, change)
2154 if failing_stages.issubset(ignored_stages):
2155 accepted.append(change)
2157 rejected.append(change)
2159 # Actually submit the accepted changes.
2160 self.SubmitChanges(accepted)
2162 # Return the list of rejected changes.
2165 def _HandleApplyFailure(self, failures):
2166 """Handles changes that were not able to be applied cleanly.
2169 failures: GerritPatch changes to handle.
2171 for failure in failures:
2172 logging.info('Change %s did not apply cleanly.', failure.patch)
2174 self._HandleCouldNotApply(failure)
2176 def _HandleCouldNotApply(self, failure):
2177 """Handler for when Paladin fails to apply a change.
2179 This handler notifies set CodeReview-2 to the review forcing the developer
2180 to re-upload a rebased change.
2183 failure: GerritPatch instance to operate upon.
2185 msg = ('%(queue)s failed to apply your change in %(build_log)s .'
2187 self.SendNotification(failure.patch, msg, failure=failure)
2188 self.RemoveCommitReady(failure.patch)
2190 def _HandleIncorrectSubmission(self, failure):
2191 """Handler for when Paladin incorrectly submits a change."""
2192 msg = ('%(queue)s incorrectly submitted your change in %(build_log)s .'
2194 self.SendNotification(failure.patch, msg, failure=failure)
2195 self.RemoveCommitReady(failure.patch)
2197 def HandleDraftChange(self, change):
2198 """Handler for when the latest patch set of |change| is not published.
2200 This handler removes the commit ready bit from the specified changes and
2201 sends the developer a message explaining why.
2204 change: GerritPatch instance to operate upon.
2206 msg = ('%(queue)s could not apply your change because the latest patch '
2207 'set is not published. Please publish your draft patch set before '
2208 'marking your commit as ready.')
2209 self.SendNotification(change, msg)
2210 self.RemoveCommitReady(change)
2212 def HandleValidationTimeout(self, changes=None, sanity=True):
2213 """Handles changes that timed out.
2215 This handler removes the commit ready bit from the specified changes and
2216 sends the developer a message explaining why.
2219 changes: A list of cros_patch.GerritPatch instances to mark as failed.
2220 By default, mark all of the changes as failed.
2221 sanity: A boolean indicating whether the build was considered sane by
2222 any relevant sanity check builders (True = sane). If not sane,
2223 none of the changes will have their CommitReady bit modified.
2227 changes = self.changes
2229 logging.info('Validation timed out for all changes.')
2230 msg = ('%(queue)s timed out while verifying your change in '
2231 '%(build_log)s . This means that a supporting builder did not '
2232 'finish building your change within the specified timeout.')
2234 msg += ('If you believe this happened in error, just re-mark your '
2235 'commit as ready. Your change will then get automatically '
2238 msg += ('The sanity check builder in this run failed, so no changes '
2239 'will be blamed for the failure.')
2241 for change in changes:
2242 logging.info('Validation timed out for change %s.', change)
2243 self.SendNotification(change, msg)
2245 self.RemoveCommitReady(change)
2247 def SendNotification(self, change, msg, **kwargs):
2248 d = dict(build_log=self.build_log, queue=self.queue, **kwargs)
2251 except (TypeError, ValueError) as e:
2253 "Generation of message %s for change %s failed: dict was %r, "
2254 "exception %s", msg, change, d, e)
2256 "Generation of message %s for change %s failed: dict was %r, "
2257 "exception %s" % (msg, change, d, e))
2258 PaladinMessage(msg, change, self._helper_pool.ForChange(change)).Send(
2261 def HandlePreCQSuccess(self):
2262 """Handler that is called when the Pre-CQ successfully verifies a change."""
2263 msg = '%(queue)s successfully verified your change in %(build_log)s .'
2264 for change in self.changes:
2265 if self.GetCLStatus(self.bot, change) != self.STATUS_PASSED:
2266 self.SendNotification(change, msg)
2267 self.UpdateCLStatus(self.bot, change, self.STATUS_PASSED,
2268 dry_run=self.dryrun)
2270 def _HandleCouldNotSubmit(self, change, error=''):
2271 """Handler that is called when Paladin can't submit a change.
2273 This should be rare, but if an admin overrides the commit queue and commits
2274 a change that conflicts with this change, it'll apply, build/validate but
2275 receive an error when submitting.
2278 change: GerritPatch instance to operate upon.
2279 error: The reason why the change could not be submitted.
2281 self.SendNotification(change,
2282 '%(queue)s failed to submit your change in %(build_log)s . '
2283 '%(error)s', error=error)
2284 self.RemoveCommitReady(change)
2287 def _CreateValidationFailureMessage(pre_cq, change, suspects, messages,
2289 """Create a message explaining why a validation failure occurred.
2292 pre_cq: Whether this builder is a Pre-CQ builder.
2293 change: The change we want to create a message for.
2294 suspects: The set of suspect changes that we think broke the build.
2295 messages: A list of build failure messages from supporting builders.
2296 sanity: A boolean indicating whether the build was considered sane by
2297 any relevant sanity check builders (True = sane). If not sane,
2298 none of the changes will have their CommitReady bit modified.
2300 # Build a list of error messages. We don't want to build a ridiculously
2301 # long comment, as Gerrit will reject it. See http://crbug.com/236831
2302 max_error_len = 20000 / max(1, len(messages))
2303 msg = ['The following build(s) failed:']
2304 for message in map(str, messages):
2305 if len(message) > max_error_len:
2306 message = message[:max_error_len] + '... (truncated)'
2309 # Create a list of changes other than this one that might be guilty.
2310 # Limit the number of suspects to 20 so that the list of suspects isn't
2311 # ridiculously long.
2313 other_suspects = suspects - set([change])
2314 if len(other_suspects) < max_suspects:
2315 other_suspects_str = ', '.join(sorted(
2316 'CL:%s' % x.gerrit_number_str for x in other_suspects))
2318 other_suspects_str = ('%d other changes. See the blamelist for more '
2319 'details.' % (len(other_suspects),))
2321 will_retry_automatically = False
2323 msg.append('The sanity check builder in this run failed, implying that '
2324 'either ToT or the build infrastructure itself was broken '
2325 'even without the tested patches. Thus, no changes will be '
2326 'blamed for the failure.')
2327 will_retry_automatically = True
2328 elif change in suspects:
2329 if other_suspects_str:
2330 msg.append('Your change may have caused this failure. There are '
2331 'also other changes that may be at fault: %s'
2332 % other_suspects_str)
2334 msg.append('This failure was probably caused by your change.')
2336 msg.append('Please check whether the failure is your fault. If your '
2337 'change is not at fault, you may mark it as ready again.')
2339 if len(suspects) == 1:
2340 msg.append('This failure was probably caused by %s'
2341 % other_suspects_str)
2343 msg.append('One of the following changes is probably at fault: %s'
2344 % other_suspects_str)
2346 will_retry_automatically = not pre_cq
2348 if will_retry_automatically:
2350 0, 'NOTE: The Commit Queue will retry your change automatically.')
2352 return '\n\n'.join(msg)
2354 def _ChangeFailedValidation(self, change, messages, suspects, sanity):
2355 """Handles a validation failure for an individual change.
2358 change: The change to mark as failed.
2359 messages: A list of build failure messages from supporting builders.
2360 These must be ValidationFailedMessage objects.
2361 suspects: The list of changes that are suspected of breaking the build.
2362 sanity: A boolean indicating whether the build was considered sane by
2363 any relevant sanity check builders (True = sane). If not sane,
2364 none of the changes will have their CommitReady bit modified.
2366 msg = self._CreateValidationFailureMessage(self.pre_cq, change, suspects,
2368 self.SendNotification(change, '%(details)s', details=msg)
2370 if change in suspects:
2371 self.RemoveCommitReady(change)
2373 # Mark the change as failed. If the Ready bit is still set, the change
2374 # will be retried automatically.
2375 self.UpdateCLStatus(self.bot, change, self.STATUS_FAILED,
2376 dry_run=self.dryrun)
2378 def HandleValidationFailure(self, messages, changes=None, sanity=True):
2379 """Handles a list of validation failure messages from slave builders.
2381 This handler parses a list of failure messages from our list of builders
2382 and calculates which changes were likely responsible for the failure. The
2383 changes that were responsible for the failure have their Commit Ready bit
2384 stripped and the other changes are left marked as Commit Ready.
2387 messages: A list of build failure messages from supporting builders.
2388 These must be ValidationFailedMessage objects.
2389 changes: A list of cros_patch.GerritPatch instances to mark as failed.
2390 By default, mark all of the changes as failed.
2391 sanity: A boolean indicating whether the build was considered sane by
2392 any relevant sanity check builders (True = sane). If not sane,
2393 none of the changes will have their CommitReady bit modified.
2396 changes = self.changes
2399 for change in changes:
2400 # Ignore changes that were already verified.
2401 if self.pre_cq and self.GetCLStatus(PRE_CQ, change) == self.STATUS_PASSED:
2403 candidates.append(change)
2405 # First, calculate which changes are likely at fault for the failure.
2406 suspects = CalculateSuspects.FindSuspects(candidates, messages)
2408 # Send out failure notifications for each change.
2409 inputs = [[change, messages, suspects, sanity] for change in candidates]
2410 parallel.RunTasksInProcessPool(self._ChangeFailedValidation, inputs)
2412 def GetValidationFailedMessage(self):
2413 """Returns message indicating these changes failed to be validated."""
2414 logging.info('Validation failed for all changes.')
2415 internal = self._overlays in [constants.PRIVATE_OVERLAYS,
2416 constants.BOTH_OVERLAYS]
2418 tracebacks = tuple(results_lib.Results.GetTracebacks())
2419 for x in tracebacks:
2420 details.append('The %s stage failed: %s' % (x.failed_stage, x.exception))
2422 details = ['cbuildbot failed']
2423 details.append('in %s' % (self.build_log,))
2424 msg = '%s: %s' % (self._builder_name, ' '.join(details))
2425 return ValidationFailedMessage(msg, tracebacks, internal)
2427 def HandleCouldNotApply(self, change):
2428 """Handler for when Paladin fails to apply a change.
2430 This handler strips the Commit Ready bit forcing the developer
2431 to re-upload a rebased change as this theirs failed to apply cleanly.
2434 change: GerritPatch instance to operate upon.
2436 msg = '%(queue)s failed to apply your change in %(build_log)s . '
2437 # This is written this way to protect against bugs in CQ itself. We log
2438 # it both to the build output, and mark the change w/ it.
2439 extra_msg = getattr(change, 'apply_error_message', None)
2440 if extra_msg is None:
2442 'Change %s was passed to HandleCouldNotApply without an appropriate '
2443 'apply_error_message set. Internal bug.', change)
2445 'Internal CQ issue: extra error info was not given, Please contact '
2446 'the build team and ensure they are aware of this specific change '
2450 self.SendNotification(change, msg)
2451 self.RemoveCommitReady(change)
2453 def _HandleApplySuccess(self, change):
2454 """Handler for when Paladin successfully applies a change.
2456 This handler notifies a developer that their change is being tried as
2457 part of a Paladin run defined by a build_log.
2460 change: GerritPatch instance to operate upon.
2463 status = self.GetCLStatus(self.bot, change)
2464 if status == self.STATUS_PASSED:
2466 msg = ('%(queue)s has picked up your change. '
2467 'You can follow along at %(build_log)s .')
2468 self.SendNotification(change, msg)
2469 if not self.pre_cq or status == self.STATUS_LAUNCHING:
2470 self.UpdateCLStatus(self.bot, change, self.STATUS_INFLIGHT,
2471 dry_run=self.dryrun)
2474 def GetCLStatusURL(cls, bot, change, latest_patchset_only=True):
2475 """Get the status URL for |change| on |bot|.
2478 bot: Which bot to look at. Can be CQ or PRE_CQ.
2479 change: GerritPatch instance to operate upon.
2480 latest_patchset_only: If True, return the URL for tracking the latest
2481 patchset. If False, return the URL for tracking all patchsets. Defaults
2485 The status URL, as a string.
2487 internal = 'int' if change.internal else 'ext'
2488 components = [constants.MANIFEST_VERSIONS_GS_URL, bot,
2489 internal, str(change.gerrit_number)]
2490 if latest_patchset_only:
2491 components.append(str(change.patch_number))
2492 return '/'.join(components)
2495 def GetCLStatus(cls, bot, change):
2496 """Get the status for |change| on |bot|.
2499 change: GerritPatch instance to operate upon.
2500 bot: Which bot to look at. Can be CQ or PRE_CQ.
2503 The status, as a string.
2505 url = cls.GetCLStatusURL(bot, change)
2506 ctx = gs.GSContext()
2508 return ctx.Cat(url).output
2509 except gs.GSNoSuchKey:
2510 logging.debug('No status yet for %r', url)
2514 def UpdateCLStatus(cls, bot, change, status, dry_run):
2515 """Update the |status| of |change| on |bot|."""
2516 for latest_patchset_only in (False, True):
2517 url = cls.GetCLStatusURL(bot, change, latest_patchset_only)
2518 ctx = gs.GSContext(dry_run=dry_run)
2519 ctx.Copy('-', url, input=status)
2520 ctx.Counter('%s/%s' % (url, status)).Increment()
2523 def GetCLStatusCount(cls, bot, change, status, latest_patchset_only=True):
2524 """Return how many times |change| has been set to |status| on |bot|.
2527 bot: Which bot to look at. Can be CQ or PRE_CQ.
2528 change: GerritPatch instance to operate upon.
2529 status: The status string to look for.
2530 latest_patchset_only: If True, only how many times the latest patchset has
2531 been set to |status|. If False, count how many times any patchset has
2532 been set to |status|. Defaults to False.
2535 The number of times |change| has been set to |status| on |bot|, as an
2538 cache_key = (bot, change, status, latest_patchset_only)
2539 if cache_key not in cls._CL_STATUS_CACHE:
2540 base_url = cls.GetCLStatusURL(bot, change, latest_patchset_only)
2541 url = '%s/%s' % (base_url, status)
2542 cls._CL_STATUS_CACHE[cache_key] = gs.GSContext().Counter(url).Get()
2543 return cls._CL_STATUS_CACHE[cache_key]
2546 def FillCLStatusCache(cls, bot, changes, statuses=None):
2547 """Cache all of the stats about the given |changes| in parallel.
2550 bot: Bot to pull down stats for.
2551 changes: Changes to cache.
2552 statuses: Statuses to cache. By default, cache the PASSED and FAILED
2555 if statuses is None:
2556 statuses = (cls.STATUS_PASSED, cls.STATUS_FAILED)
2558 for change in changes:
2559 for status in statuses:
2560 for latest_patchset_only in (False, True):
2561 cache_key = (bot, change, status, latest_patchset_only)
2562 if cache_key not in cls._CL_STATUS_CACHE:
2563 inputs.append(cache_key)
2565 with parallel.Manager() as manager:
2566 # Grab the CL status of all of the CLs in the background, into a proxied
2568 cls._CL_STATUS_CACHE = manager.dict(cls._CL_STATUS_CACHE)
2569 parallel.RunTasksInProcessPool(cls.GetCLStatusCount, inputs)
2571 # Convert the cache back into a regular dictionary before we shut down
2573 cls._CL_STATUS_CACHE = dict(cls._CL_STATUS_CACHE)
2575 def CreateDisjointTransactions(self, manifest, max_txn_length=None):
2576 """Create a list of disjoint transactions from the changes in the pool.
2579 manifest: Manifest to use.
2580 max_txn_length: The maximum length of any given transaction. Optional.
2581 By default, do not limit the length of transactions.
2584 A list of disjoint transactions. Each transaction can be tried
2585 independently, without involving patches from other transactions.
2586 Each change in the pool will included in exactly one of transactions,
2587 unless the patch does not apply for some reason.
2589 patches = PatchSeries(self.build_root, forced_manifest=manifest)
2590 plans, failed = patches.CreateDisjointTransactions(
2591 self.changes, max_txn_length=max_txn_length)
2592 failed = self._FilterDependencyErrors(failed)
2594 self._HandleApplyFailure(failed)
2598 class PaladinMessage():
2599 """An object that is used to send messages to developers about their changes.
2601 # URL where Paladin documentation is stored.
2602 _PALADIN_DOCUMENTATION_URL = ('http://www.chromium.org/developers/'
2603 'tree-sheriffs/sheriff-details-chromium-os/'
2604 'commit-queue-overview')
2606 # Gerrit can't handle commands over 32768 bytes. See http://crbug.com/236831
2607 MAX_MESSAGE_LEN = 32000
2609 def __init__(self, message, patch, helper):
2610 if len(message) > self.MAX_MESSAGE_LEN:
2611 message = message[:self.MAX_MESSAGE_LEN] + '... (truncated)'
2612 self.message = message
2614 self.helper = helper
2616 def _ConstructPaladinMessage(self):
2617 """Adds any standard Paladin messaging to an existing message."""
2618 return self.message + ('\n\nCommit queue documentation: %s' %
2619 self._PALADIN_DOCUMENTATION_URL)
2621 def Send(self, dryrun):
2622 """Posts a comment to a gerrit review."""
2624 'message': self._ConstructPaladinMessage(),
2627 path = 'changes/%s/revisions/%s/review' % (
2628 self.patch.gerrit_number, self.patch.revision)
2630 logging.info('Would have sent %r to %s', body, path)
2632 gob_util.FetchUrl(self.helper.host, path, reqtype='POST', body=body)