1 # Copyright (c) 2011-2012 The Chromium OS Authors. All rights reserved.
2 # Use of this source code is governed by a BSD-style license that can be
3 # found in the LICENSE file.
5 """Module that handles interactions with a Validation Pool.
7 The validation pool is the set of commits that are ready to be validated i.e.
8 ready for the commit queue to try.
21 from xml.dom import minidom
23 from chromite.cbuildbot import cbuildbot_config
24 from chromite.cbuildbot import failures_lib
25 from chromite.cbuildbot import constants
26 from chromite.cbuildbot import portage_utilities
27 from chromite.cbuildbot import lkgm_manager
28 from chromite.cbuildbot import manifest_version
29 from chromite.cbuildbot import tree_status
30 from chromite.lib import cros_build_lib
31 from chromite.lib import gerrit
32 from chromite.lib import git
33 from chromite.lib import gob_util
34 from chromite.lib import gs
35 from chromite.lib import parallel
36 from chromite.lib import patch as cros_patch
37 from chromite.lib import timeout_util
39 # Third-party libraries bundled with chromite need to be listed after the
40 # first chromite import.
43 # We import mox so that w/in ApplyPoolIntoRepo, if a mox exception is
44 # thrown, we don't cover it up.
51 PRE_CQ = constants.PRE_CQ
54 # The gerrit-on-borg team tells us that delays up to 2 minutes can be
55 # normal. Setting timeout to 3 minutes to be safe-ish.
56 SUBMITTED_WAIT_TIMEOUT = 3 * 60 # Time in seconds.
59 class TreeIsClosedException(Exception):
60 """Raised when the tree is closed and we wanted to submit changes."""
62 def __init__(self, closed_or_throttled=False):
66 closed_or_throttled: True if the exception is being thrown on a
67 possibly 'throttled' tree. False if only
68 thrown on a 'closed' tree. Default: False
70 if closed_or_throttled:
71 status_text = 'closed or throttled'
72 opposite_status_text = 'open'
74 status_text = 'closed'
75 opposite_status_text = 'throttled or open'
77 super(TreeIsClosedException, self).__init__(
78 'Tree is %s. Please set tree status to %s to '
79 'proceed.' % (status_text, opposite_status_text))
82 class FailedToSubmitAllChangesException(failures_lib.StepFailure):
83 """Raised if we fail to submit any change."""
85 def __init__(self, changes):
86 super(FailedToSubmitAllChangesException, self).__init__(
87 'FAILED TO SUBMIT ALL CHANGES: Could not verify that changes %s were '
88 'submitted' % ' '.join(str(c) for c in changes))
91 class FailedToSubmitAllChangesNonFatalException(
92 FailedToSubmitAllChangesException):
93 """Raised if we fail to submit any change due to non-fatal errors."""
96 class InternalCQError(cros_patch.PatchException):
97 """Exception thrown when CQ has an unexpected/unhandled error."""
99 def __init__(self, patch, message):
100 cros_patch.PatchException.__init__(self, patch, message=message)
102 def ShortExplanation(self):
103 return 'failed to apply due to a CQ issue: %s' % (self.message,)
106 class NoMatchingChangeFoundException(Exception):
107 """Raised if we try to apply a non-existent change."""
110 class ChangeNotInManifestException(Exception):
111 """Raised if we try to apply a not-in-manifest change."""
114 class PatchNotCommitReady(cros_patch.PatchException):
115 """Raised if a patch is not marked as commit ready."""
117 def ShortExplanation(self):
118 return 'isn\'t marked as Commit-Ready anymore.'
121 class PatchModified(cros_patch.PatchException):
122 """Raised if a patch is modified while the CQ is running."""
124 def ShortExplanation(self):
125 return 'was modified while the CQ was in the middle of testing it.'
128 class PatchRejected(cros_patch.PatchException):
129 """Raised if a patch was rejected by the CQ because the CQ failed."""
131 def ShortExplanation(self):
132 return 'was rejected by the CQ.'
135 class PatchFailedToSubmit(cros_patch.PatchException):
136 """Raised if we fail to submit a change."""
138 def ShortExplanation(self):
139 error = 'could not be submitted by the CQ.'
141 error += ' The error message from Gerrit was: %s' % (self.message,)
143 error += ' The Gerrit server might be having trouble.'
147 class PatchConflict(cros_patch.PatchException):
148 """Raised if a patch needs to be rebased."""
150 def ShortExplanation(self):
151 return ('could not be submitted because Gerrit reported a conflict. Did '
152 'you modify your patch during the CQ run? Or do you just need to '
156 class PatchSubmittedWithoutDeps(cros_patch.DependencyError):
157 """Exception thrown when a patch was submitted incorrectly."""
159 def ShortExplanation(self):
160 dep_error = cros_patch.DependencyError.ShortExplanation(self)
161 return ('was submitted, even though it %s\n'
163 'You may want to revert your patch, and investigate why its'
164 'dependencies failed to submit.\n'
166 'This error only occurs when we have a dependency cycle, and we '
167 'submit one change before realizing that a later change cannot '
168 'be submitted.' % (dep_error,))
171 class PatchSeriesTooLong(cros_patch.PatchException):
172 """Exception thrown when a required dep isn't satisfied."""
174 def __init__(self, patch, max_length):
175 cros_patch.PatchException.__init__(self, patch)
176 self.max_length = max_length
178 def ShortExplanation(self):
179 return ("The Pre-CQ cannot handle a patch series longer than %s patches. "
180 "Please wait for some patches to be submitted before marking more "
181 "patches as ready. " % (self.max_length,))
184 return self.ShortExplanation()
187 def _RunCommand(cmd, dryrun):
188 """Runs the specified shell cmd if dryrun=False.
190 Errors are ignored, but logged.
193 logging.info('Would have run: %s', ' '.join(cmd))
197 cros_build_lib.RunCommand(cmd)
198 except cros_build_lib.RunCommandError:
199 cros_build_lib.Error('Command failed', exc_info=True)
202 def GetStagesToIgnoreFromConfigFile(config_path):
203 """Get a list of stage name prefixes to ignore from |config_path|.
205 This function reads the specified config file and returns the list
206 of stage name prefixes to ignore in the CQ. See GetStagesToIgnoreForChange
210 config_path: The path to the config file to read.
213 parser = ConfigParser.SafeConfigParser()
215 parser.read(config_path)
216 if parser.has_option('GENERAL', 'ignored-stages'):
217 ignored_stages = parser.get('GENERAL', 'ignored-stages').split()
218 except ConfigParser.Error:
219 cros_build_lib.Error('Error parsing %r', config_path, exc_info=True)
221 return ignored_stages
224 def GetStagesToIgnoreForChange(build_root, change):
225 """Get a list of stages that the CQ should ignore for a given |change|.
227 The list of stage name prefixes to ignore for each project is specified in a
228 config file inside the project, named COMMIT-QUEUE.ini. The file would look
232 ignored-stages: HWTest VMTest
234 The CQ will submit changes to the given project even if the listed stages
235 failed. These strings are stage name prefixes, meaning that "HWTest" would
236 match any HWTest stage (e.g. "HWTest [bvt]" or "HWTest [foo]")
239 build_root: The root of the checkout.
240 change: Change to examine.
243 A list of stages to ignore for the given |change|.
245 manifest = git.ManifestCheckout.Cached(build_root)
246 checkout = change.GetCheckout(manifest)
248 dirname = checkout.GetPath(absolute=True)
249 path = os.path.join(dirname, 'COMMIT-QUEUE.ini')
250 return GetStagesToIgnoreFromConfigFile(path)
254 class GerritHelperNotAvailable(gerrit.GerritException):
255 """Exception thrown when a specific helper is requested but unavailable."""
257 def __init__(self, remote=constants.EXTERNAL_REMOTE):
258 gerrit.GerritException.__init__(self)
259 # Stringify the pool so that serialization doesn't try serializing
260 # the actual HelperPool.
262 self.args = (remote,)
266 "Needed a remote=%s gerrit_helper, but one isn't allowed by this "
267 "HelperPool instance.") % (self.remote,)
270 class HelperPool(object):
271 """Pool of allowed GerritHelpers to be used by CQ/PatchSeries."""
273 def __init__(self, cros_internal=None, cros=None):
274 """Initialize this instance with the given handlers.
276 Most likely you want the classmethod SimpleCreate which takes boolean
279 If a given handler is None, then it's disabled; else the passed in
283 constants.EXTERNAL_REMOTE : cros,
284 constants.INTERNAL_REMOTE : cros_internal
288 def SimpleCreate(cls, cros_internal=True, cros=True):
289 """Classmethod helper for creating a HelperPool from boolean options.
292 cros_internal: If True, allow access to a GerritHelper for internal.
293 cros: If True, allow access to a GerritHelper for external.
296 An appropriately configured HelperPool instance.
299 cros = gerrit.GetGerritHelper(constants.EXTERNAL_REMOTE)
304 cros_internal = gerrit.GetGerritHelper(constants.INTERNAL_REMOTE)
308 return cls(cros_internal=cros_internal, cros=cros)
310 def ForChange(self, change):
311 """Return the helper to use for a particular change.
313 If no helper is configured, an Exception is raised.
315 return self.GetHelper(change.remote)
317 def GetHelper(self, remote):
318 """Return the helper to use for a given remote.
320 If no helper is configured, an Exception is raised.
322 helper = self.pool.get(remote)
324 raise GerritHelperNotAvailable(remote)
329 for helper in self.pool.itervalues():
334 def _PatchWrapException(functor):
335 """Decorator to intercept patch exceptions and wrap them.
337 Specifically, for known/handled Exceptions, it intercepts and
338 converts it into a DependencyError- via that, preserving the
339 cause, while casting it into an easier to use form (one that can
340 be chained in addition).
342 def f(self, parent, *args, **kwargs):
344 return functor(self, parent, *args, **kwargs)
345 except gerrit.GerritException as e:
346 if isinstance(e, gerrit.QueryNotSpecific):
347 e = ("%s\nSuggest you use gerrit numbers instead (prefixed with a * "
348 "if it's an internal change)." % e)
349 new_exc = cros_patch.PatchException(parent, e)
350 raise new_exc.__class__, new_exc, sys.exc_info()[2]
351 except cros_patch.PatchException as e:
352 if e.patch.id == parent.id:
354 new_exc = cros_patch.DependencyError(parent, e)
355 raise new_exc.__class__, new_exc, sys.exc_info()[2]
357 f.__name__ = functor.__name__
361 class PatchSeries(object):
362 """Class representing a set of patches applied to a single git repository."""
364 def __init__(self, path, helper_pool=None, forced_manifest=None,
365 deps_filter_fn=None, is_submitting=False):
369 path: Path to the buildroot.
370 helper_pool: Pool of allowed GerritHelpers to be used for fetching
371 patches. Defaults to allowing both internal and external fetches.
372 forced_manifest: A manifest object to use for mapping projects to
373 repositories. Defaults to the buildroot.
374 deps_filter_fn: A function which specifies what patches you would
375 like to accept. It is passed a patch and is expected to return
377 is_submitting: Whether we are currently submitting patchsets. This is
378 used to print better error messages.
380 self.manifest = forced_manifest
382 if helper_pool is None:
383 helper_pool = HelperPool.SimpleCreate(cros_internal=True, cros=True)
384 self._helper_pool = helper_pool
386 if deps_filter_fn is None:
387 deps_filter_fn = lambda x: True
388 self.deps_filter_fn = deps_filter_fn
389 self._is_submitting = is_submitting
395 # A mapping of ChangeId to exceptions if the patch failed against
396 # ToT. Primarily used to keep the resolution/applying from going
397 # down known bad paths.
398 self._committed_cache = cros_patch.PatchCache()
399 self._lookup_cache = cros_patch.PatchCache()
400 self._change_deps_cache = {}
402 def _ManifestDecorator(functor):
403 """Method decorator that sets self.manifest automatically.
405 This function automatically initializes the manifest, and allows callers to
406 override the manifest if needed.
408 # pylint: disable=E0213,W0212,E1101,E1102
409 def f(self, *args, **kwargs):
410 manifest = kwargs.pop('manifest', None)
411 # Wipe is used to track if we need to reset manifest to None, and
412 # to identify if we already had a forced_manifest via __init__.
413 wipe = self.manifest is None
416 raise ValueError("manifest can't be specified when one is forced "
419 manifest = git.ManifestCheckout.Cached(self._path)
421 manifest = self.manifest
424 self.manifest = manifest
425 return functor(self, *args, **kwargs)
430 f.__name__ = functor.__name__
431 f.__doc__ = functor.__doc__
435 def GetGitRepoForChange(self, change, strict=False):
436 """Get the project path associated with the specified change.
439 change: The change to operate on.
440 strict: If True, throw ChangeNotInManifest rather than returning
441 None. Default: False.
444 The project path if found in the manifest. Otherwise returns
445 None (if strict=False).
449 checkout = change.GetCheckout(self.manifest, strict=strict)
450 if checkout is not None:
451 project_dir = checkout.GetPath(absolute=True)
456 def ApplyChange(self, change):
457 # Always enable content merging.
458 return change.ApplyAgainstManifest(self.manifest, trivial=False)
460 def _LookupHelper(self, patch):
461 """Returns the helper for the given cros_patch.PatchQuery object."""
462 return self._helper_pool.GetHelper(patch.remote)
464 def _GetGerritPatch(self, query):
465 """Query the configured helpers looking for a given change.
468 project: The gerrit project to query.
469 query: A cros_patch.PatchQuery object.
472 A GerritPatch object.
474 helper = self._LookupHelper(query)
475 query_text = query.ToGerritQueryText()
476 change = helper.QuerySingleRecord(
477 query_text, must_match=not git.IsSHA1(query_text))
482 # If the query was a gerrit number based query, check the projects/change-id
483 # to see if we already have it locally, but couldn't map it since we didn't
484 # know the gerrit number at the time of the initial injection.
485 existing = self._lookup_cache[change]
486 if cros_patch.ParseGerritNumber(query_text) and existing is not None:
487 keys = change.LookupAliases()
488 self._lookup_cache.InjectCustomKeys(keys, existing)
491 self.InjectLookupCache([change])
492 if change.IsAlreadyMerged():
493 self.InjectCommittedPatches([change])
496 def _LookupUncommittedChanges(self, deps, limit_to=None):
497 """Given a set of deps (changes), return unsatisfied dependencies.
500 deps: A list of cros_patch.PatchQuery objects representing
501 sequence of dependencies for the leaf that we need to identify
502 as either merged, or needing resolving.
503 limit_to: If non-None, then this must be a mapping (preferably a
504 cros_patch.PatchCache for translation reasons) of which non-committed
505 changes are allowed to be used for a transaction.
508 A sequence of cros_patch.GitRepoPatch instances (or derivatives) that
509 need to be resolved for this change to be mergable.
513 if dep in self._committed_cache:
517 self._LookupHelper(dep)
518 except GerritHelperNotAvailable:
519 # Internal dependencies are irrelevant to external builders.
520 logging.info("Skipping internal dependency: %s", dep)
523 dep_change = self._lookup_cache[dep]
525 if dep_change is None:
526 dep_change = self._GetGerritPatch(dep)
527 if dep_change is None:
529 if getattr(dep_change, 'IsAlreadyMerged', lambda: False)():
531 elif limit_to is not None and dep_change not in limit_to:
532 if self._is_submitting:
533 raise PatchRejected(dep_change)
535 raise PatchNotCommitReady(dep_change)
537 unsatisfied.append(dep_change)
539 # Perform last minute custom filtering.
540 return [x for x in unsatisfied if self.deps_filter_fn(x)]
542 def CreateTransaction(self, change, limit_to=None):
543 """Given a change, resolve it into a transaction.
545 In this case, a transaction is defined as a group of commits that
546 must land for the given change to be merged- specifically its
547 parent deps, and its CQ-DEPEND.
550 change: A cros_patch.GitRepoPatch instance to generate a transaction
552 limit_to: If non-None, limit the allowed uncommitted patches to
553 what's in that container/mapping.
556 A sequence of the necessary cros_patch.GitRepoPatch objects for
560 gerrit_deps_seen = cros_patch.PatchCache()
561 cq_deps_seen = cros_patch.PatchCache()
562 self._AddChangeToPlanWithDeps(change, plan, gerrit_deps_seen,
563 cq_deps_seen, limit_to=limit_to)
566 def CreateTransactions(self, changes, limit_to=None):
567 """Create a list of transactions from a list of changes.
570 changes: A list of cros_patch.GitRepoPatch instances to generate
572 limit_to: See CreateTransaction docs.
575 A list of (change, plan, e) tuples for the given list of changes. The
576 plan represents the necessary GitRepoPatch objects for a given change. If
577 an exception occurs while creating the transaction, e will contain the
578 exception. (Otherwise, e will be None.)
580 for change in changes:
582 plan = self.CreateTransaction(change, limit_to=limit_to)
583 except cros_patch.PatchException as e:
584 yield (change, (), e)
586 yield (change, plan, None)
588 def CreateDisjointTransactions(self, changes, max_txn_length=None):
589 """Create a list of disjoint transactions from a list of changes.
592 changes: A list of cros_patch.GitRepoPatch instances to generate
594 max_txn_length: The maximum length of any given transaction. Optional.
595 By default, do not limit the length of transactions.
598 A list of disjoint transactions and a list of exceptions. Each transaction
599 can be tried independently, without involving patches from other
600 transactions. Each change in the pool will included in exactly one of the
601 transactions, unless the patch does not apply for some reason.
603 # Gather the dependency graph for the specified changes.
604 deps, edges, failed = {}, {}, []
605 for change, plan, ex in self.CreateTransactions(changes, limit_to=changes):
607 logging.info('Failed creating transaction for %s: %s', change, ex)
610 # Save off the ordered dependencies of this change.
613 # Mark every change in the transaction as bidirectionally connected.
614 for change_dep in plan:
615 edges.setdefault(change_dep, set()).update(plan)
617 # Calculate an unordered group of strongly connected components.
618 unordered_plans = digraph.StronglyConnectedComponents(list(edges), edges)
620 # Sort the groups according to our ordered dependency graph.
622 for unordered_plan in unordered_plans:
623 ordered_plan, seen = [], set()
624 for change in unordered_plan:
625 # Iterate over the required CLs, adding them to our plan in order.
626 new_changes = list(dep_change for dep_change in deps[change]
627 if dep_change not in seen)
628 new_plan_size = len(ordered_plan) + len(new_changes)
629 if not max_txn_length or new_plan_size <= max_txn_length:
630 seen.update(new_changes)
631 ordered_plan.extend(new_changes)
634 # We found a transaction that is <= max_txn_length. Process the
635 # transaction. Ignore the remaining patches for now; they will be
636 # processed later (once the current transaction has been pushed).
637 ordered_plans.append(ordered_plan)
639 # We couldn't find any transactions that were <= max_txn_length.
640 # This should only happen if circular dependencies prevent us from
641 # truncating a long list of patches. Reject the whole set of patches
643 for change in unordered_plan:
644 failed.append(PatchSeriesTooLong(change, max_txn_length))
646 return ordered_plans, failed
649 def _AddChangeToPlanWithDeps(self, change, plan, gerrit_deps_seen,
650 cq_deps_seen, limit_to=None,
651 include_cq_deps=True):
652 """Add a change and its dependencies into a |plan|.
655 change: The change to add to the plan.
656 plan: The list of changes to apply, in order. This function will append
657 |change| and any necessary dependencies to |plan|.
658 gerrit_deps_seen: The changes whose Gerrit dependencies have already been
660 cq_deps_seen: The changes whose CQ-DEPEND and Gerrit dependencies have
661 already been processed.
662 limit_to: If non-None, limit the allowed uncommitted patches to
663 what's in that container/mapping.
664 include_cq_deps: If True, include CQ dependencies in the list
665 of dependencies. Defaults to True.
668 DependencyError: If we could not resolve a dependency.
669 GerritException or GOBError: If there is a failure in querying gerrit.
671 if change in self._committed_cache:
674 # Get a list of the changes that haven't been committed.
675 # These are returned as cros_patch.PatchQuery objects.
676 gerrit_deps, cq_deps = self.GetDepsForChange(change)
678 # Only process the Gerrit dependencies for each change once. We prioritize
679 # Gerrit dependencies over CQ dependencies, since Gerrit dependencies might
680 # be required in order for the change to apply.
681 old_plan_len = len(plan)
682 if change not in gerrit_deps_seen:
683 gerrit_deps = self._LookupUncommittedChanges(
684 gerrit_deps, limit_to=limit_to)
685 gerrit_deps_seen.Inject(change)
686 for dep in gerrit_deps:
687 self._AddChangeToPlanWithDeps(dep, plan, gerrit_deps_seen, cq_deps_seen,
688 limit_to=limit_to, include_cq_deps=False)
690 # If there are cyclic dependencies, we might have already applied this
691 # patch as part of dependency resolution. If not, apply this patch.
692 if change not in plan:
695 # Process CQ deps last, so as to avoid circular dependencies between
696 # Gerrit dependencies and CQ dependencies.
697 if include_cq_deps and change not in cq_deps_seen:
698 cq_deps = self._LookupUncommittedChanges(
699 cq_deps, limit_to=limit_to)
700 cq_deps_seen.Inject(change)
701 for dep in plan[old_plan_len:] + cq_deps:
702 # Add the requested change (plus deps) to our plan, if it we aren't
703 # already in the process of doing that.
704 if dep not in cq_deps_seen:
705 self._AddChangeToPlanWithDeps(dep, plan, gerrit_deps_seen,
706 cq_deps_seen, limit_to=limit_to)
709 def GetDepChangesForChange(self, change):
710 """Look up the Gerrit/CQ dependency changes for |change|.
713 (gerrit_deps, cq_deps): The change's Gerrit dependencies and CQ
714 dependencies, as lists of GerritPatch objects.
717 DependencyError: If we could not resolve a dependency.
718 GerritException or GOBError: If there is a failure in querying gerrit.
720 gerrit_deps, cq_deps = self.GetDepsForChange(change)
722 def _DepsToChanges(deps):
724 unprocessed_deps = []
726 dep_change = self._committed_cache[dep]
728 dep_changes.append(dep_change)
730 unprocessed_deps.append(dep)
732 for dep in unprocessed_deps:
733 dep_changes.extend(self._LookupUncommittedChanges(deps))
737 return _DepsToChanges(gerrit_deps), _DepsToChanges(cq_deps)
740 def GetDepsForChange(self, change):
741 """Look up the Gerrit/CQ deps for |change|.
744 A tuple of PatchQuery objects representing change's Gerrit
745 dependencies, and CQ dependencies.
748 DependencyError: If we could not resolve a dependency.
749 GerritException or GOBError: If there is a failure in querying gerrit.
751 val = self._change_deps_cache.get(change)
753 git_repo = self.GetGitRepoForChange(change)
754 val = self._change_deps_cache[change] = (
755 change.GerritDependencies(),
756 change.PaladinDependencies(git_repo))
760 def InjectCommittedPatches(self, changes):
761 """Record that the given patches are already committed.
763 This is primarily useful for external code to notify this object
764 that changes were applied to the tree outside its purview- specifically
765 useful for dependency resolution.
767 self._committed_cache.Inject(*changes)
769 def InjectLookupCache(self, changes):
770 """Inject into the internal lookup cache the given changes, using them
771 (rather than asking gerrit for them) as needed for dependencies.
773 self._lookup_cache.Inject(*changes)
775 def FetchChanges(self, changes):
776 """Fetch the specified changes, if needed.
778 If we're an external builder, internal changes are filtered out.
781 An iterator over a list of the filtered changes.
783 for change in changes:
785 self._helper_pool.ForChange(change)
786 except GerritHelperNotAvailable:
787 # Internal patches are irrelevant to external builders.
788 logging.info("Skipping internal patch: %s", change)
790 change.Fetch(self.GetGitRepoForChange(change, strict=True))
794 def Apply(self, changes, frozen=True, honor_ordering=False,
795 changes_filter=None):
796 """Applies changes from pool into the build root specified by the manifest.
798 This method resolves each given change down into a set of transactions-
799 the change and its dependencies- that must go in, then tries to apply
800 the largest transaction first, working its way down.
802 If a transaction cannot be applied, then it is rolled back
803 in full- note that if a change is involved in multiple transactions,
804 if an earlier attempt fails, that change can be retried in a new
805 transaction if the failure wasn't caused by the patch being incompatible
809 changes: A sequence of cros_patch.GitRepoPatch instances to resolve
811 frozen: If True, then resolving of the given changes is explicitly
812 limited to just the passed in changes, or known committed changes.
813 This is basically CQ/Paladin mode, used to limit the changes being
814 pulled in/committed to just what we allow.
815 honor_ordering: Apply normally will reorder the transactions it
816 computes, trying the largest first, then degrading through smaller
817 transactions if the larger of the two fails. If honor_ordering
818 is False, then the ordering given via changes is preserved-
819 this is mainly of use for cbuildbot induced patching, and shouldn't
820 be used for CQ patching.
821 changes_filter: If not None, must be a functor taking two arguments:
822 series, changes; it must return the changes to work on.
823 This is invoked after the initial changes have been fetched,
824 thus this is a way for consumers to do last minute checking of the
825 changes being inspected, and expand the changes if necessary.
826 Primarily this is of use for cbuildbot patching when dealing w/
827 uploaded/remote patches.
830 A tuple of changes-applied, Exceptions for the changes that failed
831 against ToT, and Exceptions that failed inflight; These exceptions
832 are cros_patch.PatchException instances.
834 # Prefetch the changes; we need accurate change_id/id's, which is
835 # guaranteed via Fetch.
836 changes = list(self.FetchChanges(changes))
838 changes = changes_filter(self, changes)
840 self.InjectLookupCache(changes)
841 limit_to = cros_patch.PatchCache(changes) if frozen else None
842 resolved, applied, failed = [], [], []
843 for change, plan, ex in self.CreateTransactions(changes, limit_to=limit_to):
845 logging.info("Failed creating transaction for %s: %s", change, ex)
848 resolved.append((change, plan))
849 logging.info("Transaction for %s is %s.",
850 change, ', '.join(map(str, resolved[-1][-1])))
853 # No work to do; either no changes were given to us, or all failed
855 return [], failed, []
857 if not honor_ordering:
858 # Sort by length, falling back to the order the changes were given to us.
859 # This is done to prefer longer transactions (more painful to rebase)
860 # over shorter transactions.
861 position = dict((change, idx) for idx, change in enumerate(changes))
863 ids = [x.id for x in data[1]]
864 return -len(ids), position[data[0]]
865 resolved.sort(key=mk_key)
867 for inducing_change, transaction_changes in resolved:
869 with self._Transaction(transaction_changes):
870 logging.debug("Attempting transaction for %s: changes: %s",
872 ', '.join(map(str, transaction_changes)))
873 self._ApplyChanges(inducing_change, transaction_changes)
874 except cros_patch.PatchException as e:
875 logging.info("Failed applying transaction for %s: %s",
879 applied.extend(transaction_changes)
880 self.InjectCommittedPatches(transaction_changes)
882 # Uniquify while maintaining order.
890 applied = list(_uniq(applied))
891 self._is_submitting = True
893 failed = [x for x in failed if x.patch not in applied]
894 failed_tot = [x for x in failed if not x.inflight]
895 failed_inflight = [x for x in failed if x.inflight]
896 return applied, failed_tot, failed_inflight
898 @contextlib.contextmanager
899 def _Transaction(self, commits):
900 """ContextManager used to rollback changes to a build root if necessary.
902 Specifically, if an unhandled non system exception occurs, this context
903 manager will roll back all relevant modifications to the git repos
907 commits: A sequence of cros_patch.GitRepoPatch instances that compromise
908 this transaction- this is used to identify exactly what may be changed,
909 thus what needs to be tracked and rolled back if the transaction fails.
911 # First, the book keeping code; gather required data so we know what
912 # to rollback to should this transaction fail. Specifically, we track
913 # what was checked out for each involved repo, and if it was a branch,
914 # the sha1 of the branch; that information is enough to rewind us back
915 # to the original repo state.
917 map(functools.partial(self.GetGitRepoForChange, strict=True), commits))
919 for project_dir in project_state:
920 current_sha1 = git.RunGit(
921 project_dir, ['rev-list', '-n1', 'HEAD']).output.strip()
922 resets.append((project_dir, current_sha1))
925 committed_cache = self._committed_cache.copy()
929 # Reaching here means it was applied cleanly, thus return.
932 logging.info("Rewinding transaction: failed changes: %s .",
933 ', '.join(map(str, commits)), exc_info=True)
935 for project_dir, sha1 in resets:
936 git.RunGit(project_dir, ['reset', '--hard', sha1])
938 self._committed_cache = committed_cache
942 def _ApplyChanges(self, _inducing_change, changes):
943 """Apply a given ordered sequence of changes.
946 _inducing_change: The core GitRepoPatch instance that lead to this
947 sequence of changes; basically what this transaction was computed from.
948 Needs to be passed in so that the exception wrapping machinery can
949 convert any failures, assigning blame appropriately.
950 manifest: A ManifestCheckout instance representing what we're working on.
951 changes: A ordered sequence of GitRepoPatch instances to apply.
953 # Bail immediately if we know one of the requisite patches won't apply.
954 for change in changes:
955 failure = self.failed_tot.get(change.id)
956 if failure is not None:
960 for change in changes:
961 if change in self._committed_cache:
965 self.ApplyChange(change)
966 except cros_patch.PatchException as e:
968 self.failed_tot[change.id] = e
970 applied.append(change)
972 logging.debug('Done investigating changes. Applied %s',
973 ' '.join([c.id for c in applied]))
976 def WorkOnSingleRepo(cls, git_repo, tracking_branch, **kwargs):
977 """Classmethod to generate a PatchSeries that targets a single git repo.
979 It does this via forcing a fake manifest, which in turn points
980 tracking branch/paths/content-merging at what is passed through here.
983 git_repo: Absolute path to the git repository to operate upon.
984 tracking_branch: Which tracking branch patches should apply against.
985 kwargs: See PatchSeries.__init__ for the various optional args;
986 note forced_manifest cannot be used here.
989 A PatchSeries instance w/ a forced manifest.
992 if 'forced_manifest' in kwargs:
993 raise ValueError("RawPatchSeries doesn't allow a forced_manifest "
995 kwargs['forced_manifest'] = _ManifestShim(git_repo, tracking_branch)
997 return cls(git_repo, **kwargs)
1000 class _ManifestShim(object):
1001 """A fake manifest that only contains a single repository.
1003 This fake manifest is used to allow us to filter out patches for
1004 the PatchSeries class. It isn't a complete implementation -- we just
1005 implement the functions that PatchSeries uses. It works via duck typing.
1007 All of the below methods accept the same arguments as the corresponding
1008 methods in git.ManifestCheckout.*, but they do not make any use of the
1009 arguments -- they just always return information about this project.
1012 def __init__(self, path, tracking_branch, remote='origin'):
1014 tracking_branch = 'refs/remotes/%s/%s' % (
1015 remote, git.StripRefs(tracking_branch),
1017 attrs = dict(local_path=path, path=path, tracking_branch=tracking_branch)
1018 self.checkout = git.ProjectCheckout(attrs)
1020 def FindCheckouts(self, *_args, **_kwargs):
1021 """Returns the list of checkouts.
1023 In this case, we only have one repository so we just return that repository.
1024 We accept the same arguments as git.ManifestCheckout.FindCheckouts, but we
1025 do not make any use of them.
1028 A list of ProjectCheckout objects.
1030 return [self.checkout]
1033 class CalculateSuspects(object):
1034 """Diagnose the cause for a given set of failures."""
1037 def _FindPackageBuildFailureSuspects(cls, changes, messages):
1038 """Figure out what CLs are at fault for a set of build failures.
1041 changes: A list of cros_patch.GerritPatch instances to consider.
1042 messages: A list of build failure messages, of type
1043 BuildFailureMessage.
1046 for message in messages:
1047 suspects.update(message.FindPackageBuildFailureSuspects(changes))
1051 def _FindPreviouslyFailedChanges(cls, candidates):
1052 """Find what changes that have previously failed the CQ.
1054 The first time a change is included in a build that fails due to a
1055 flaky (or apparently unrelated) failure, we assume that it is innocent. If
1056 this happens more than once, we kick out the CL.
1059 for change in candidates:
1060 if ValidationPool.GetCLStatusCount(
1061 CQ, change, ValidationPool.STATUS_FAILED):
1062 suspects.add(change)
1066 def FilterChromiteChanges(cls, changes):
1067 """Returns a list of chromite changes in |changes|."""
1068 return [x for x in changes if x.project == constants.CHROMITE_PROJECT]
1071 def _MatchesFailureType(cls, messages, fail_type, strict=True):
1072 """Returns True if all failures are instances of |fail_type|.
1075 messages: A list of BuildFailureMessage or NoneType objects
1076 from the failed slaves.
1077 fail_type: The exception class to look for.
1078 strict: If False, treat NoneType message as a match.
1081 True if all objects in |messages| are non-None and all failures are
1082 instances of |fail_type|.
1084 return ((not strict or all(messages)) and
1085 all(x.MatchesFailureType(fail_type) for x in messages if x))
1088 def OnlyLabFailures(cls, messages, no_stat):
1089 """Determine if the cause of build failure was lab failure.
1092 messages: A list of BuildFailureMessage or NoneType objects
1093 from the failed slaves.
1094 no_stat: A list of builders which failed prematurely without reporting
1098 True if the build failed purely due to lab failures.
1100 # If any builder failed prematuely, lab failure was not the only cause.
1101 return (not no_stat and
1102 cls._MatchesFailureType(messages, failures_lib.TestLabFailure))
1105 def OnlyInfraFailures(cls, messages, no_stat):
1106 """Determine if the cause of build failure was infrastructure failure.
1109 messages: A list of BuildFailureMessage or NoneType objects
1110 from the failed slaves.
1111 no_stat: A list of builders which failed prematurely without reporting
1115 True if the build failed purely due to infrastructure failures.
1117 # "Failed to report status" and "NoneType" messages are considered
1119 return ((not messages and no_stat) or
1120 cls._MatchesFailureType(
1121 messages, failures_lib.InfrastructureFailure, strict=False))
1124 def FindSuspects(cls, build_root, changes, messages, infra_fail=False,
1126 """Find out what changes probably caused our failure.
1128 In cases where there were no internal failures, we can assume that the
1129 external failures are at fault. Otherwise, this function just defers to
1130 _FindPackageBuildFailureSuspects and FindPreviouslyFailedChanges as needed.
1131 If the failures don't match either case, just fail everything.
1134 build_root: Build root directory.
1135 changes: A list of cros_patch.GerritPatch instances to consider.
1136 messages: A list of build failure messages, of type
1137 BuildFailureMessage or of type NoneType.
1138 infra_fail: The build failed purely due to infrastructure failures.
1139 lab_fail: The build failed purely due to test lab infrastructure
1143 A set of changes as suspects.
1146 logging.warning('Detected that the build failed purely due to HW '
1147 'Test Lab failure(s). Will not reject any changes')
1150 if not lab_fail and infra_fail:
1151 # The non-lab infrastructure errors might have been caused
1152 # by chromite changes.
1154 'Detected that the build failed due to non-lab infrastructure '
1155 'issue(s). Will only reject chromite changes')
1156 return set(cls.FilterChromiteChanges(changes))
1159 # If there were no internal failures, only kick out external changes.
1160 # Treat None messages as external for this purpose.
1161 if any(message and message.internal for message in messages):
1162 candidates = changes
1164 candidates = [change for change in changes if not change.internal]
1166 # Filter out innocent internal overlay changes from our list of candidates.
1167 candidates = cls.FilterInnocentOverlayChanges(
1168 build_root, candidates, messages)
1170 bad_changes = ValidationPool.GetShouldRejectChanges(candidates)
1172 # If there are changes that have been set verified=-1 or
1173 # code-review=-2, these changes are suspects of the failed build.
1174 suspects.update(bad_changes)
1175 elif all(message and message.IsPackageBuildFailure()
1176 for message in messages):
1177 # If we are here, there are no None messages.
1178 suspects = cls._FindPackageBuildFailureSuspects(candidates, messages)
1180 suspects.update(candidates)
1185 def GetResponsibleOverlays(cls, build_root, messages):
1186 """Get the set of overlays that could have caused failures.
1188 This loops through the set of builders that failed in a given run and
1189 finds what overlays could have been responsible for the failure.
1192 build_root: Build root directory.
1193 messages: A list of build failure messages from supporting builders.
1194 These must be BuildFailureMessage objects or NoneType objects.
1197 The set of overlays that could have caused the failures. If we can't
1198 determine what overlays are responsible, returns None.
1200 responsible_overlays = set()
1201 for message in messages:
1204 bot_id = message.builder
1205 config = cbuildbot_config.config.get(bot_id)
1208 for board in config.boards:
1209 overlays = portage_utilities.FindOverlays(
1210 constants.BOTH_OVERLAYS, board, build_root)
1211 responsible_overlays.update(overlays)
1212 return responsible_overlays
1215 def GetAffectedOverlays(cls, change, manifest, all_overlays):
1216 """Get the set of overlays affected by a given change.
1219 change: The change to look at.
1220 manifest: A ManifestCheckout instance representing our build directory.
1221 all_overlays: The set of all valid overlays.
1224 The set of overlays affected by the specified |change|. If the change
1225 affected something other than an overlay, return None.
1227 checkout = change.GetCheckout(manifest, strict=False)
1229 git_repo = checkout.GetPath(absolute=True)
1231 # The whole git repo is an overlay. Return it.
1232 # Example: src/private-overlays/overlay-x86-zgb-private
1233 if git_repo in all_overlays:
1234 return set([git_repo])
1236 # Get the set of immediate subdirs affected by the change.
1237 # Example: src/overlays/overlay-x86-zgb
1238 subdirs = set([os.path.join(git_repo, path.split(os.path.sep)[0])
1239 for path in change.GetDiffStatus(git_repo)])
1241 # If all of the subdirs are overlays, return them.
1242 if subdirs.issubset(all_overlays):
1246 def FilterInnocentOverlayChanges(cls, build_root, changes, messages):
1247 """Filter out clearly innocent overlay changes based on failure messages.
1249 It is not possible to break a x86-generic builder via a change to an
1250 unrelated overlay (e.g. amd64-generic). Filter out changes that are
1251 known to be innocent.
1254 build_root: Build root directory.
1255 changes: Changes to filter.
1256 messages: A list of build failure messages from supporting builders.
1257 These must be BuildFailureMessage objects or NoneType objects.
1260 The list of changes that are potentially guilty.
1262 responsible_overlays = cls.GetResponsibleOverlays(build_root, messages)
1263 if responsible_overlays is None:
1265 all_overlays = set(portage_utilities.FindOverlays(
1266 constants.BOTH_OVERLAYS, None, build_root))
1267 manifest = git.ManifestCheckout.Cached(build_root)
1269 for change in changes:
1270 overlays = cls.GetAffectedOverlays(change, manifest, all_overlays)
1271 if overlays is None or overlays.issubset(responsible_overlays):
1272 candidates.append(change)
1276 class ValidationPool(object):
1277 """Class that handles interactions with a validation pool.
1279 This class can be used to acquire a set of commits that form a pool of
1280 commits ready to be validated and committed.
1282 Usage: Use ValidationPool.AcquirePool -- a static
1283 method that grabs the commits that are ready for validation.
1286 GLOBAL_DRYRUN = False
1287 MAX_TIMEOUT = 60 * 60 * 4
1289 STATUS_FAILED = manifest_version.BuilderStatus.STATUS_FAILED
1290 STATUS_INFLIGHT = manifest_version.BuilderStatus.STATUS_INFLIGHT
1291 STATUS_PASSED = manifest_version.BuilderStatus.STATUS_PASSED
1292 STATUS_LAUNCHING = 'launching'
1293 STATUS_WAITING = 'waiting'
1294 INCONSISTENT_SUBMIT_MSG = ('Gerrit thinks that the change was not submitted, '
1295 'even though we hit the submit button.')
1297 # The grace period (in seconds) before we reject a patch due to dependency
1299 REJECTION_GRACE_PERIOD = 30 * 60
1301 # Cache for the status of CLs.
1302 _CL_STATUS_CACHE = {}
1304 def __init__(self, overlays, build_root, build_number, builder_name,
1305 is_master, dryrun, changes=None, non_os_changes=None,
1306 conflicting_changes=None, pre_cq=False, metadata=None):
1307 """Initializes an instance by setting default variables to instance vars.
1309 Generally use AcquirePool as an entry pool to a pool rather than this
1313 overlays: One of constants.VALID_OVERLAYS.
1314 build_root: Build root directory.
1315 build_number: Build number for this validation attempt.
1316 builder_name: Builder name on buildbot dashboard.
1317 is_master: True if this is the master builder for the Commit Queue.
1318 dryrun: If set to True, do not submit anything to Gerrit.
1320 changes: List of changes for this validation pool.
1321 non_os_changes: List of changes that are part of this validation
1322 pool but aren't part of the cros checkout.
1323 conflicting_changes: Changes that failed to apply but we're keeping around
1324 because they conflict with other changes in flight.
1325 pre_cq: If set to True, this builder is verifying CLs before they go to
1327 metadata: Optional CBuildbotMetadata instance where CL actions will
1331 self.build_root = build_root
1333 # These instances can be instantiated via both older, or newer pickle
1334 # dumps. Thus we need to assert the given args since we may be getting
1335 # a value we no longer like (nor work with).
1336 if overlays not in constants.VALID_OVERLAYS:
1337 raise ValueError("Unknown/unsupported overlay: %r" % (overlays,))
1339 self._helper_pool = self.GetGerritHelpersForOverlays(overlays)
1341 if not isinstance(build_number, int):
1342 raise ValueError("Invalid build_number: %r" % (build_number,))
1344 if not isinstance(builder_name, basestring):
1345 raise ValueError("Invalid builder_name: %r" % (builder_name,))
1347 for changes_name, changes_value in (
1348 ('changes', changes), ('non_os_changes', non_os_changes)):
1349 if not changes_value:
1351 if not all(isinstance(x, cros_patch.GitRepoPatch) for x in changes_value):
1353 'Invalid %s: all elements must be a GitRepoPatch derivative, got %r'
1354 % (changes_name, changes_value))
1356 if conflicting_changes and not all(
1357 isinstance(x, cros_patch.PatchException)
1358 for x in conflicting_changes):
1360 'Invalid conflicting_changes: all elements must be a '
1361 'cros_patch.PatchException derivative, got %r'
1362 % (conflicting_changes,))
1364 self.build_log = self.ConstructDashboardURL(overlays, pre_cq, builder_name,
1367 self.is_master = bool(is_master)
1368 self.pre_cq = pre_cq
1369 self._metadata = metadata
1370 self.dryrun = bool(dryrun) or self.GLOBAL_DRYRUN
1371 self.queue = 'A trybot' if pre_cq else 'The Commit Queue'
1372 self.bot = PRE_CQ if pre_cq else CQ
1374 # See optional args for types of changes.
1375 self.changes = changes or []
1376 self.non_manifest_changes = non_os_changes or []
1377 # Note, we hold onto these CLs since they conflict against our current CLs
1378 # being tested; if our current ones succeed, we notify the user to deal
1379 # w/ the conflict. If the CLs we're testing fail, then there is no
1380 # reason we can't try these again in the next run.
1381 self.changes_that_failed_to_apply_earlier = conflicting_changes or []
1383 # Private vars only used for pickling.
1384 self._overlays = overlays
1385 self._build_number = build_number
1386 self._builder_name = builder_name
1389 def GetBuildDashboardForOverlays(overlays, trybot):
1390 """Discern the dashboard to use based on the given overlay."""
1392 return constants.TRYBOT_DASHBOARD
1393 if overlays in [constants.PRIVATE_OVERLAYS, constants.BOTH_OVERLAYS]:
1394 return constants.BUILD_INT_DASHBOARD
1395 return constants.BUILD_DASHBOARD
1398 def ConstructDashboardURL(cls, overlays, trybot, builder_name, build_number,
1400 """Return the dashboard (buildbot) URL for this run
1403 overlays: One of constants.VALID_OVERLAYS.
1404 trybot: Boolean: is this a remote trybot?
1405 builder_name: Builder name on buildbot dashboard.
1406 build_number: Build number for this validation attempt.
1407 stage: Link directly to a stage log, else use the general landing page.
1410 The fully formed URL
1412 build_dashboard = cls.GetBuildDashboardForOverlays(overlays, trybot)
1413 url_suffix = 'builders/%s/builds/%s' % (builder_name, str(build_number))
1415 url_suffix += '/steps/%s/logs/stdio' % (stage,)
1416 url_suffix = urllib.quote(url_suffix)
1417 return os.path.join(build_dashboard, url_suffix)
1420 def GetGerritHelpersForOverlays(overlays):
1421 """Discern the allowed GerritHelpers to use based on the given overlay."""
1422 cros_internal = cros = False
1423 if overlays in [constants.PUBLIC_OVERLAYS, constants.BOTH_OVERLAYS, False]:
1426 if overlays in [constants.PRIVATE_OVERLAYS, constants.BOTH_OVERLAYS]:
1427 cros_internal = True
1429 return HelperPool.SimpleCreate(cros_internal=cros_internal, cros=cros)
1431 def __reduce__(self):
1432 """Used for pickling to re-create validation pool."""
1433 # NOTE: self._metadata is specifically excluded from the validation pool
1434 # pickle. We do not want the un-pickled validation pool to have a reference
1435 # to its own un-pickled metadata instance. Instead, we want to to refer
1436 # to the builder run's metadata instance. This is accomplished by setting
1437 # metadata at un-pickle time, in ValidationPool.Load(...).
1442 self.build_root, self._build_number, self._builder_name,
1443 self.is_master, self.dryrun, self.changes,
1444 self.non_manifest_changes,
1445 self.changes_that_failed_to_apply_earlier,
1449 def FilterDraftChanges(cls, changes):
1450 """Filter out draft changes based on the status of the latest patch set.
1452 Our Gerrit query cannot exclude changes whose latest patch set has
1453 not yet been published as long as there is one published patchset
1454 in the change. Such changes will fail when we try to merge them,
1455 which may lead to undesirable consequence (e.g. dependencies not
1459 changes: List of changes to filter.
1462 List of published changes.
1464 return [x for x in changes if not x.patch_dict['currentPatchSet']['draft']]
1467 def GetShouldRejectChanges(cls, changes):
1468 """Returns the changes that should be rejected.
1470 Check whether the change should be rejected (e.g. verified: -1,
1474 changes: List of changes.
1477 A list of changes that should be rejected.
1479 return [x for x in changes if
1480 any(x.HasApproval(f, v) for f, v in
1481 constants.DEFAULT_CQ_SHOULD_REJECT_FIELDS.iteritems())]
1484 def FilterNonMatchingChanges(cls, changes):
1485 """Filter out changes that don't actually match our query.
1487 Generally, Gerrit should only return patches that match our
1488 query. However, Gerrit keeps a query cache and the cached data may
1491 There are also race conditions (bugs in Gerrit) where the final
1492 patch won't match our query. Here's an example problem that this
1493 code fixes: If the Pre-CQ launcher picks up a CL while the CQ is
1494 committing the CL, it may catch a race condition where a new
1495 patchset has been created and committed by the CQ, but the CL is
1496 still treated as if it matches the query (which it doesn't,
1500 changes: List of changes to filter.
1503 List of changes that match our query.
1505 filtered_changes = []
1506 should_reject_changes = cls.GetShouldRejectChanges(changes)
1507 for change in changes:
1508 if change in should_reject_changes:
1510 # Because the gerrit cache sometimes gets stale, double-check that the
1511 # change hasn't already been merged.
1512 if change.status != 'NEW':
1514 # Check that the user (or chrome-bot) uploaded a new change under our
1515 # feet while Gerrit was in the middle of answering our query.
1516 for field, value in constants.DEFAULT_CQ_READY_FIELDS.iteritems():
1517 if not change.HasApproval(field, value):
1520 filtered_changes.append(change)
1522 return filtered_changes
1525 @failures_lib.SetFailureType(failures_lib.BuilderFailure)
1526 def AcquirePreCQPool(cls, *args, **kwargs):
1527 """See ValidationPool.__init__ for arguments."""
1528 kwargs.setdefault('pre_cq', True)
1529 kwargs.setdefault('is_master', True)
1530 pool = cls(*args, **kwargs)
1531 pool.RecordPatchesInMetadata()
1535 def AcquirePool(cls, overlays, repo, build_number, builder_name,
1536 dryrun=False, changes_query=None, check_tree_open=True,
1537 change_filter=None, throttled_ok=False, metadata=None):
1538 """Acquires the current pool from Gerrit.
1540 Polls Gerrit and checks for which changes are ready to be committed.
1541 Should only be called from master builders.
1544 overlays: One of constants.VALID_OVERLAYS.
1545 repo: The repo used to sync, to filter projects, and to apply patches
1547 build_number: Corresponding build number for the build.
1548 builder_name: Builder name on buildbot dashboard.
1549 dryrun: Don't submit anything to gerrit.
1550 changes_query: The gerrit query to use to identify changes; if None,
1551 uses the internal defaults.
1552 check_tree_open: If True, only return when the tree is open.
1553 change_filter: If set, use change_filter(pool, changes,
1554 non_manifest_changes) to filter out unwanted patches.
1555 throttled_ok: if |check_tree_open|, treat a throttled tree as open.
1557 metadata: Optional CBuildbotMetadata instance where CL actions will
1561 ValidationPool object.
1564 TreeIsClosedException: if the tree is closed (or throttled, if not
1567 if change_filter is None:
1568 change_filter = lambda _, x, y: (x, y)
1570 # We choose a longer wait here as we haven't committed to anything yet. By
1571 # doing this here we can reduce the number of builder cycles.
1572 end_time = time.time() + cls.MAX_TIMEOUT
1574 time_left = end_time - time.time()
1576 # Wait until the tree becomes open (or throttled, if |throttled_ok|,
1577 # and record the tree status).
1580 status = tree_status.WaitForTreeStatus(
1581 period=cls.SLEEP_TIMEOUT, timeout=time_left,
1582 throttled_ok=throttled_ok)
1583 except timeout_util.TimeoutError:
1584 raise TreeIsClosedException(closed_or_throttled=not throttled_ok)
1586 status = constants.TREE_OPEN
1588 waiting_for = 'new CLs'
1590 # Select the right default gerrit query based on the the tree
1591 # status, or use custom |changes_query| if it was provided.
1592 using_default_query = (changes_query is None)
1593 if not using_default_query:
1594 query = changes_query
1595 elif status == constants.TREE_THROTTLED:
1596 query = constants.THROTTLED_CQ_READY_QUERY
1597 waiting_for = 'new CQ+2 CLs or the tree to open'
1599 query = constants.DEFAULT_CQ_READY_QUERY
1601 # Sync so that we are up-to-date on what is committed.
1604 # Only master configurations should call this method.
1605 pool = ValidationPool(overlays, repo.directory, build_number,
1606 builder_name, True, dryrun, metadata=metadata)
1609 # Iterate through changes from all gerrit instances we care about.
1610 for helper in cls.GetGerritHelpersForOverlays(overlays):
1611 raw_changes = helper.Query(query, sort='lastUpdated')
1612 raw_changes.reverse()
1614 # Reload the changes because the data in the Gerrit cache may be stale.
1615 raw_changes = list(cls.ReloadChanges(raw_changes))
1617 # If we used a default query, verify the results match the query, to
1618 # prevent race conditions. Note, this filters using the conditions
1619 # of DEFAULT_CQ_READY_QUERY even if the tree is throttled. Since that
1620 # query is strictly more permissive than the throttled query, we are
1621 # not at risk of incorrectly losing any patches here. We only expose
1622 # ourselves to the minor race condititon that a CQ+2 patch could have
1623 # been marked as CQ+1 out from under us, but still end up being picked
1624 # up in a throttled CQ run.
1625 if using_default_query:
1626 published_changes = cls.FilterDraftChanges(raw_changes)
1627 draft_changes.extend(set(raw_changes) - set(published_changes))
1628 raw_changes = cls.FilterNonMatchingChanges(published_changes)
1630 changes, non_manifest_changes = ValidationPool._FilterNonCrosProjects(
1631 raw_changes, git.ManifestCheckout.Cached(repo.directory))
1632 pool.changes.extend(changes)
1633 pool.non_manifest_changes.extend(non_manifest_changes)
1635 for change in draft_changes:
1636 pool.HandleDraftChange(change)
1638 # Filter out unwanted changes.
1639 pool.changes, pool.non_manifest_changes = change_filter(
1640 pool, pool.changes, pool.non_manifest_changes)
1642 if (pool.changes or pool.non_manifest_changes or dryrun or time_left < 0
1643 or cls.ShouldExitEarly()):
1646 logging.info('Waiting for %s (%d minutes left)...', waiting_for,
1648 time.sleep(cls.SLEEP_TIMEOUT)
1650 pool.RecordPatchesInMetadata()
1653 def AddPendingCommitsIntoPool(self, manifest):
1654 """Add the pending commits from |manifest| into pool.
1657 manifest: path to the manifest.
1659 manifest_dom = minidom.parse(manifest)
1660 pending_commits = manifest_dom.getElementsByTagName(
1661 lkgm_manager.PALADIN_COMMIT_ELEMENT)
1662 for pc in pending_commits:
1663 patch = cros_patch.GerritFetchOnlyPatch(
1664 pc.getAttribute(lkgm_manager.PALADIN_PROJECT_URL_ATTR),
1665 pc.getAttribute(lkgm_manager.PALADIN_PROJECT_ATTR),
1666 pc.getAttribute(lkgm_manager.PALADIN_REF_ATTR),
1667 pc.getAttribute(lkgm_manager.PALADIN_BRANCH_ATTR),
1668 pc.getAttribute(lkgm_manager.PALADIN_REMOTE_ATTR),
1669 pc.getAttribute(lkgm_manager.PALADIN_COMMIT_ATTR),
1670 pc.getAttribute(lkgm_manager.PALADIN_CHANGE_ID_ATTR),
1671 pc.getAttribute(lkgm_manager.PALADIN_GERRIT_NUMBER_ATTR),
1672 pc.getAttribute(lkgm_manager.PALADIN_PATCH_NUMBER_ATTR),
1673 owner_email=pc.getAttribute(lkgm_manager.PALADIN_OWNER_EMAIL_ATTR),
1674 fail_count=int(pc.getAttribute(lkgm_manager.PALADIN_FAIL_COUNT_ATTR)),
1675 pass_count=int(pc.getAttribute(lkgm_manager.PALADIN_PASS_COUNT_ATTR)),
1676 total_fail_count=int(pc.getAttribute(
1677 lkgm_manager.PALADIN_TOTAL_FAIL_COUNT_ATTR)),)
1679 self.changes.append(patch)
1682 def AcquirePoolFromManifest(cls, manifest, overlays, repo, build_number,
1683 builder_name, is_master, dryrun, metadata=None):
1684 """Acquires the current pool from a given manifest.
1686 This function assumes that you have already synced to the given manifest.
1689 manifest: path to the manifest where the pool resides.
1690 overlays: One of constants.VALID_OVERLAYS.
1691 repo: The repo used to filter projects and to apply patches against.
1692 build_number: Corresponding build number for the build.
1693 builder_name: Builder name on buildbot dashboard.
1694 is_master: Boolean that indicates whether this is a pool for a master.
1696 dryrun: Don't submit anything to gerrit.
1697 metadata: Optional CBuildbotMetadata instance where CL actions will
1701 ValidationPool object.
1703 pool = ValidationPool(overlays, repo.directory, build_number, builder_name,
1704 is_master, dryrun, metadata=metadata)
1705 pool.AddPendingCommitsIntoPool(manifest)
1706 pool.RecordPatchesInMetadata()
1710 def ShouldExitEarly(cls):
1711 """Return whether we should exit early.
1713 This function is intended to be overridden by tests or by subclasses.
1718 def _FilterNonCrosProjects(changes, manifest):
1719 """Filters changes to a tuple of relevant changes.
1721 There are many code reviews that are not part of Chromium OS and/or
1722 only relevant on a different branch. This method returns a tuple of (
1723 relevant reviews in a manifest, relevant reviews not in the manifest). Note
1724 that this function must be run while chromite is checked out in a
1725 repo-managed checkout.
1728 changes: List of GerritPatch objects.
1729 manifest: The manifest to check projects/branches against.
1732 Tuple of (relevant reviews in a manifest,
1733 relevant reviews not in the manifest).
1736 def IsCrosReview(change):
1737 return (change.project.startswith('chromiumos') or
1738 change.project.startswith('chromeos'))
1740 # First we filter to only Chromium OS repositories.
1741 changes = [c for c in changes if IsCrosReview(c)]
1743 changes_in_manifest = []
1744 changes_not_in_manifest = []
1745 for change in changes:
1746 if change.GetCheckout(manifest, strict=False):
1747 changes_in_manifest.append(change)
1749 changes_not_in_manifest.append(change)
1750 logging.info('Filtered change %s', change)
1752 return changes_in_manifest, changes_not_in_manifest
1755 def _FilterDependencyErrors(cls, errors):
1756 """Filter out ignorable DependencyError exceptions.
1758 If a dependency isn't marked as ready, or a dependency fails to apply,
1759 we only complain after REJECTION_GRACE_PERIOD has passed since the patch
1762 This helps in two situations:
1763 1) If the developer is in the middle of marking a stack of changes as
1764 ready, we won't reject their work until the grace period has passed.
1765 2) If the developer marks a big circular stack of changes as ready, and
1766 some change in the middle of the stack doesn't apply, the user will
1767 get a chance to rebase their change before we mark all the changes as
1770 This function filters out dependency errors that can be ignored due to
1774 errors: List of exceptions to filter.
1777 List of unfiltered exceptions.
1779 reject_timestamp = time.time() - cls.REJECTION_GRACE_PERIOD
1781 for error in errors:
1782 results.append(error)
1783 if reject_timestamp < error.patch.approval_timestamp:
1784 while error is not None:
1785 if isinstance(error, cros_patch.DependencyError):
1786 logging.info('Ignoring dependency errors for %s due to grace '
1787 'period', error.patch)
1790 error = getattr(error, 'error', None)
1794 def PrintLinksToChanges(cls, changes):
1795 """Print links to the specified |changes|.
1797 This method prints a link to list of |changes| by using the
1798 information stored in |changes|. It should not attempt to query
1799 Google Storage or Gerrit.
1802 changes: A list of cros_patch.GerritPatch instances to generate
1805 def SortKeyForChanges(change):
1806 return (-change.total_fail_count, -change.fail_count,
1807 os.path.basename(change.project), change.gerrit_number)
1809 # Now, sort and print the changes.
1810 for change in sorted(changes, key=SortKeyForChanges):
1811 project = os.path.basename(change.project)
1812 gerrit_number = cros_patch.AddPrefix(change, change.gerrit_number)
1813 # We cannot print '@' in the link because it is used to separate
1814 # the display text and the URL by the buildbot annotator.
1815 author = change.owner_email.replace('@', '-AT-')
1816 if (change.owner_email.endswith(constants.GOOGLE_EMAIL) or
1817 change.owner_email.endswith(constants.CHROMIUM_EMAIL)):
1818 author = change.owner
1820 s = '%s | %s | %s' % (project, author, gerrit_number)
1822 # Print a count of how many times a given CL has failed the CQ.
1823 if change.total_fail_count:
1824 s += ' | fails:%d' % (change.fail_count,)
1825 if change.total_fail_count > change.fail_count:
1826 s += '(%d)' % (change.total_fail_count,)
1828 # Add a note if the latest patchset has already passed the CQ.
1829 if change.pass_count > 0:
1830 s += ' | passed:%d' % change.pass_count
1832 cros_build_lib.PrintBuildbotLink(s, change.url)
1834 def ApplyPoolIntoRepo(self, manifest=None):
1835 """Applies changes from pool into the directory specified by the buildroot.
1837 This method applies changes in the order specified. If the build
1838 is running as the master, it also respects the dependency
1839 order. Otherwise, the changes should already be listed in an order
1840 that will not break the dependency order.
1843 True if we managed to apply any changes.
1846 failed_tot = failed_inflight = {}
1847 patch_series = PatchSeries(self.build_root, helper_pool=self._helper_pool)
1850 # pylint: disable=E1123
1851 applied, failed_tot, failed_inflight = patch_series.Apply(
1852 self.changes, manifest=manifest)
1853 except (KeyboardInterrupt, RuntimeError, SystemExit):
1855 except Exception as e:
1856 if mox is not None and isinstance(e, mox.Error):
1860 'Unhandled exception occurred while applying changes: %s\n\n'
1861 'To be safe, we have kicked out all of the CLs, so that the '
1862 'commit queue does not go into an infinite loop retrying '
1865 links = ', '.join('CL:%s' % x.gerrit_number_str for x in self.changes)
1866 cros_build_lib.Error('%s\nAffected Patches are: %s', msg, links)
1867 errors = [InternalCQError(patch, msg) for patch in self.changes]
1868 self._HandleApplyFailure(errors)
1871 # Completely fill the status cache in parallel.
1872 self.FillCLStatusCache(CQ, applied)
1873 for change in applied:
1874 change.total_fail_count = self.GetCLStatusCount(
1875 CQ, change, self.STATUS_FAILED, latest_patchset_only=False)
1876 change.fail_count = self.GetCLStatusCount(
1877 CQ, change, self.STATUS_FAILED)
1878 change.pass_count = self.GetCLStatusCount(
1879 CQ, change, self.STATUS_PASSED)
1882 # Slaves do not need to create transactions and should simply
1883 # apply the changes serially, based on the order that the
1884 # changes were listed on the manifest.
1885 for change in self.changes:
1887 # pylint: disable=E1123
1888 patch_series.ApplyChange(change, manifest=manifest)
1889 except cros_patch.PatchException as e:
1890 # Fail if any patch cannot be applied.
1891 self._HandleApplyFailure([InternalCQError(change, e)])
1894 applied.append(change)
1896 self.PrintLinksToChanges(applied)
1899 inputs = [[change] for change in applied]
1900 parallel.RunTasksInProcessPool(self._HandleApplySuccess, inputs)
1902 failed_tot = self._FilterDependencyErrors(failed_tot)
1905 'The following changes could not cleanly be applied to ToT: %s',
1906 ' '.join([c.patch.id for c in failed_tot]))
1907 self._HandleApplyFailure(failed_tot)
1909 failed_inflight = self._FilterDependencyErrors(failed_inflight)
1912 'The following changes could not cleanly be applied against the '
1913 'current stack of patches; if this stack fails, they will be tried '
1914 'in the next run. Inflight failed changes: %s',
1915 ' '.join([c.patch.id for c in failed_inflight]))
1917 self.changes_that_failed_to_apply_earlier.extend(failed_inflight)
1918 self.changes = applied
1920 return bool(self.changes)
1923 def Load(filename, metadata=None, record_patches=True):
1924 """Loads the validation pool from the file.
1927 filename: path of file to load from.
1928 metadata: Optional CBuildbotInstance to use as metadata object
1929 for loaded pool (as metadata instances do not survive
1931 record_patches: Optional, defaults to True. If True, patches
1932 picked up in this pool will be recorded in
1935 with open(filename, 'rb') as p_file:
1936 pool = cPickle.load(p_file)
1937 pool._metadata = metadata
1938 # Because metadata is currently not surviving cbuildbot re-execution,
1939 # re-record that patches were picked up in the non-skipped run of
1941 # TODO(akeshet): Remove this code once metadata is being pickled and
1942 # passed across re-executions. See crbug.com/356930
1944 pool.RecordPatchesInMetadata()
1947 def Save(self, filename):
1948 """Serializes the validation pool."""
1949 with open(filename, 'wb') as p_file:
1950 cPickle.dump(self, p_file, protocol=cPickle.HIGHEST_PROTOCOL)
1952 # Note: All submit code, all gerrit code, and basically everything other
1953 # than patch resolution/applying needs to use .change_id from patch objects.
1954 # Basically all code from this point forward.
1955 def _SubmitChangeWithDeps(self, patch_series, change, errors, limit_to):
1956 """Submit |change| and its dependencies.
1958 If you call this function multiple times with the same PatchSeries, each
1959 CL will only be submitted once.
1962 patch_series: A PatchSeries() object.
1963 change: The change (a GerritPatch object) to submit.
1964 errors: A dictionary. This dictionary should contain all patches that have
1965 encountered errors, and map them to the associated exception object.
1966 limit_to: The list of patches that were approved by this CQ run. We will
1967 only consider submitting patches that are in this list.
1970 A copy of the errors object. If new errors have occurred while submitting
1971 this change, and those errors have prevented a change from being
1972 submitted, they will be added to the errors object.
1974 # Find out what patches we need to submit.
1975 errors = errors.copy()
1977 plan = patch_series.CreateTransaction(change, limit_to=limit_to)
1978 except cros_patch.PatchException as e:
1982 error_stack, submitted = [], []
1983 for dep_change in plan:
1984 # Has this change failed to submit before?
1985 dep_error = errors.get(dep_change)
1986 if dep_error is None and error_stack:
1987 # One of the dependencies failed to submit. Report an error.
1988 dep_error = cros_patch.DependencyError(dep_change, error_stack[-1])
1990 # If there were no errors, submit the patch.
1991 if dep_error is None:
1993 if self._SubmitChange(dep_change) or self.dryrun:
1994 submitted.append(dep_change)
1996 msg = self.INCONSISTENT_SUBMIT_MSG
1997 dep_error = PatchFailedToSubmit(dep_change, msg)
1998 except (gob_util.GOBError, gerrit.GerritException) as e:
1999 if getattr(e, 'http_status', None) == httplib.CONFLICT:
2000 dep_error = PatchConflict(dep_change)
2002 dep_error = PatchFailedToSubmit(dep_change, str(e))
2003 logging.error('%s', dep_error)
2005 # Add any error we saw to the stack.
2006 if dep_error is not None:
2007 logging.info('%s', dep_error)
2008 errors[dep_change] = dep_error
2009 error_stack.append(dep_error)
2011 # Track submitted patches so that we don't submit them again.
2012 patch_series.InjectCommittedPatches(submitted)
2014 # Look for incorrectly submitted patches. We only have this problem
2015 # when we have a dependency cycle, and we submit one change before
2016 # realizing that a later change cannot be submitted. For now, just
2017 # print an error message and notify the developers.
2019 # If you see this error a lot, consider implementing a best-effort
2020 # attempt at reverting changes.
2021 for submitted_change in submitted:
2022 gdeps, pdeps = patch_series.GetDepChangesForChange(submitted_change)
2023 for dep in gdeps + pdeps:
2024 dep_error = errors.get(dep)
2025 if dep_error is not None:
2026 error = PatchSubmittedWithoutDeps(submitted_change, dep_error)
2027 self._HandleIncorrectSubmission(error)
2028 logging.error('%s was erroneously submitted.', submitted_change)
2032 def SubmitChanges(self, changes, check_tree_open=True, throttled_ok=True):
2033 """Submits the given changes to Gerrit.
2036 changes: GerritPatch's to submit.
2037 check_tree_open: Whether to check that the tree is open before submitting
2038 changes. If this is False, TreeIsClosedException will never be raised.
2039 throttled_ok: if |check_tree_open|, treat a throttled tree as open
2042 A list of the changes that failed to submit.
2045 TreeIsClosedException: if the tree is closed.
2047 assert self.is_master, 'Non-master builder calling SubmitPool'
2048 assert not self.pre_cq, 'Trybot calling SubmitPool'
2050 # Mark all changes as successful.
2051 inputs = [[self.bot, change, self.STATUS_PASSED, self.dryrun]
2052 for change in changes]
2053 parallel.RunTasksInProcessPool(self.UpdateCLStatus, inputs)
2055 if (check_tree_open and not self.dryrun and not
2056 tree_status.IsTreeOpen(period=self.SLEEP_TIMEOUT,
2057 timeout=self.MAX_TIMEOUT,
2058 throttled_ok=throttled_ok)):
2059 raise TreeIsClosedException(close_or_throttled=not throttled_ok)
2061 # Filter out changes that were modified during the CQ run.
2062 unmodified_changes, errors = self.FilterModifiedChanges(changes)
2064 # Filter out changes that aren't marked as CR=+2, CQ=+1, V=+1 anymore, in
2065 # case the patch status changed during the CQ run.
2066 filtered_changes = self.FilterNonMatchingChanges(unmodified_changes)
2067 for change in set(unmodified_changes) - set(filtered_changes):
2068 errors[change] = PatchNotCommitReady(change)
2070 patch_series = PatchSeries(self.build_root, helper_pool=self._helper_pool)
2071 patch_series.InjectLookupCache(filtered_changes)
2072 for change in filtered_changes:
2073 errors = self._SubmitChangeWithDeps(patch_series, change, errors,
2076 for patch, error in errors.iteritems():
2077 logging.error('Could not submit %s', patch)
2078 self._HandleCouldNotSubmit(patch, error)
2082 def RecordPatchesInMetadata(self):
2083 """Mark all patches as having been picked up in metadata.json.
2085 If self._metadata is None, then this function does nothing.
2088 timestamp = int(time.time())
2089 for change in self.changes:
2090 self._metadata.RecordCLAction(change, constants.CL_ACTION_PICKED_UP,
2094 def FilterModifiedChanges(cls, changes):
2095 """Filter out changes that were modified while the CQ was in-flight.
2098 changes: A list of changes (as PatchQuery objects).
2101 This returns a tuple (unmodified_changes, errors).
2103 unmodified_changes: A reloaded list of changes, only including unmodified
2104 and unsubmitted changes.
2105 errors: A dictionary. This dictionary will contain all patches that have
2106 encountered errors, and map them to the associated exception object.
2108 # Reload all of the changes from the Gerrit server so that we have a
2109 # fresh view of their approval status. This is needed so that our filtering
2110 # that occurs below will be mostly up-to-date.
2111 unmodified_changes, errors = [], {}
2112 reloaded_changes = list(cls.ReloadChanges(changes))
2113 old_changes = cros_patch.PatchCache(changes)
2114 for change in reloaded_changes:
2115 if change.IsAlreadyMerged():
2116 logging.warning('%s is already merged. It was most likely chumped '
2117 'during the current CQ run.', change)
2118 elif change.patch_number != old_changes[change].patch_number:
2119 # If users upload new versions of a CL while the CQ is in-flight, then
2120 # their CLs are no longer tested. These CLs should be rejected.
2121 errors[change] = PatchModified(change)
2123 unmodified_changes.append(change)
2125 return unmodified_changes, errors
2128 def ReloadChanges(cls, changes):
2129 """Reload the specified |changes| from the server.
2132 changes: A list of PatchQuery objects.
2135 A list of GerritPatch objects.
2137 return gerrit.GetGerritPatchInfoWithPatchQueries(changes)
2139 def _SubmitChange(self, change):
2140 """Submits patch using Gerrit Review."""
2141 logging.info('Change %s will be submitted', change)
2142 was_change_submitted = False
2143 helper = self._helper_pool.ForChange(change)
2144 helper.SubmitChange(change, dryrun=self.dryrun)
2145 updated_change = helper.QuerySingleRecord(change.gerrit_number)
2147 # If change is 'SUBMITTED' give gerrit some time to resolve that
2148 # to 'MERGED' or fail outright.
2149 if updated_change.status == 'SUBMITTED':
2151 return helper.QuerySingleRecord(change.gerrit_number)
2153 return value and value.status == 'SUBMITTED'
2156 updated_change = timeout_util.WaitForSuccess(
2157 _Retry, _Query, timeout=SUBMITTED_WAIT_TIMEOUT, period=1)
2158 except timeout_util.TimeoutError:
2159 # The change really is stuck on submitted, not merged, then.
2160 logging.warning('Timed out waiting for gerrit to finish submitting'
2161 ' change %s, but status is still "%s".',
2162 change.gerrit_number_str, updated_change.status)
2164 was_change_submitted = updated_change.status == 'MERGED'
2165 if not was_change_submitted:
2167 'Change %s was submitted to gerrit without errors, but gerrit is'
2168 ' reporting it with status "%s" (expected "MERGED").',
2169 change.gerrit_number_str, updated_change.status)
2170 if updated_change.status == 'SUBMITTED':
2171 # So far we have never seen a SUBMITTED CL that did not eventually
2172 # transition to MERGED. If it is stuck on SUBMITTED treat as MERGED.
2173 was_change_submitted = True
2174 logging.info('Proceeding now with the assumption that change %s'
2175 ' will eventually transition to "MERGED".',
2176 change.gerrit_number_str)
2178 logging.error('Most likely gerrit was unable to merge change %s.',
2179 change.gerrit_number_str)
2182 if was_change_submitted:
2183 action = constants.CL_ACTION_SUBMITTED
2185 action = constants.CL_ACTION_SUBMIT_FAILED
2186 self._metadata.RecordCLAction(change, action)
2188 return was_change_submitted
2190 def RemoveCommitReady(self, change):
2191 """Remove the commit ready bit for the specified |change|."""
2192 self._helper_pool.ForChange(change).RemoveCommitReady(change,
2195 self._metadata.RecordCLAction(change, constants.CL_ACTION_KICKED_OUT)
2197 def SubmitNonManifestChanges(self, check_tree_open=True):
2198 """Commits changes to Gerrit from Pool that aren't part of the checkout.
2201 check_tree_open: Whether to check that the tree is open before submitting
2202 changes. If this is False, TreeIsClosedException will never be raised.
2205 TreeIsClosedException: if the tree is closed.
2207 self.SubmitChanges(self.non_manifest_changes,
2208 check_tree_open=check_tree_open)
2210 def SubmitPool(self, check_tree_open=True, throttled_ok=True):
2211 """Commits changes to Gerrit from Pool. This is only called by a master.
2214 check_tree_open: Whether to check that the tree is open before submitting
2215 changes. If this is False, TreeIsClosedException will never be raised.
2216 throttled_ok: if |check_tree_open|, treat a throttled tree as open
2219 TreeIsClosedException: if the tree is closed.
2220 FailedToSubmitAllChangesException: if we can't submit a change.
2221 FailedToSubmitAllChangesNonFatalException: if we can't submit a change
2222 due to non-fatal errors.
2224 # Note that SubmitChanges can throw an exception if it can't
2225 # submit all changes; in that particular case, don't mark the inflight
2226 # failures patches as failed in gerrit- some may apply next time we do
2227 # a CQ run (since the submit state has changed, we have no way of
2228 # knowing). They *likely* will still fail, but this approach tries
2229 # to minimize wasting the developers time.
2230 errors = self.SubmitChanges(self.changes, check_tree_open=check_tree_open,
2231 throttled_ok=throttled_ok)
2233 # We don't throw a fatal error for the whitelisted
2234 # exceptions. These exceptions are mostly caused by human
2235 # intervention during the current run and have limited impact on
2237 whitelisted_exceptions = (PatchConflict,
2239 PatchNotCommitReady,
2240 cros_patch.DependencyError,)
2242 if all(isinstance(e, whitelisted_exceptions) for e in errors.values()):
2243 raise FailedToSubmitAllChangesNonFatalException(errors)
2245 raise FailedToSubmitAllChangesException(errors)
2247 if self.changes_that_failed_to_apply_earlier:
2248 self._HandleApplyFailure(self.changes_that_failed_to_apply_earlier)
2250 def SubmitPartialPool(self, tracebacks):
2251 """If the build failed, push any CLs that don't care about the failure.
2253 Each project can specify a list of stages it does not care about in its
2254 COMMIT-QUEUE.ini file. Changes to that project will be submitted even if
2258 tracebacks: A list of RecordedTraceback objects. These objects represent
2259 the exceptions that failed the build.
2262 A list of the rejected changes.
2264 # Create a list of the failing stage prefixes.
2265 failing_stages = set(traceback.failed_prefix for traceback in tracebacks)
2267 # For each CL, look at whether it cares about the failures. Based on this,
2268 # categorize the CL as accepted or rejected.
2269 accepted, rejected = [], []
2270 for change in self.changes:
2271 ignored_stages = GetStagesToIgnoreForChange(self.build_root, change)
2272 if failing_stages.issubset(ignored_stages):
2273 accepted.append(change)
2275 rejected.append(change)
2277 # Actually submit the accepted changes.
2278 self.SubmitChanges(accepted)
2280 # Return the list of rejected changes.
2283 def _HandleApplyFailure(self, failures):
2284 """Handles changes that were not able to be applied cleanly.
2287 failures: GerritPatch changes to handle.
2289 for failure in failures:
2290 logging.info('Change %s did not apply cleanly.', failure.patch)
2292 self._HandleCouldNotApply(failure)
2294 def _HandleCouldNotApply(self, failure):
2295 """Handler for when Paladin fails to apply a change.
2297 This handler notifies set CodeReview-2 to the review forcing the developer
2298 to re-upload a rebased change.
2301 failure: GerritPatch instance to operate upon.
2303 msg = ('%(queue)s failed to apply your change in %(build_log)s .'
2305 self.SendNotification(failure.patch, msg, failure=failure)
2306 self.RemoveCommitReady(failure.patch)
2308 def _HandleIncorrectSubmission(self, failure):
2309 """Handler for when Paladin incorrectly submits a change."""
2310 msg = ('%(queue)s incorrectly submitted your change in %(build_log)s .'
2312 self.SendNotification(failure.patch, msg, failure=failure)
2313 self.RemoveCommitReady(failure.patch)
2315 def HandleDraftChange(self, change):
2316 """Handler for when the latest patch set of |change| is not published.
2318 This handler removes the commit ready bit from the specified changes and
2319 sends the developer a message explaining why.
2322 change: GerritPatch instance to operate upon.
2324 msg = ('%(queue)s could not apply your change because the latest patch '
2325 'set is not published. Please publish your draft patch set before '
2326 'marking your commit as ready.')
2327 self.SendNotification(change, msg)
2328 self.RemoveCommitReady(change)
2330 def HandleValidationTimeout(self, changes=None, sanity=True):
2331 """Handles changes that timed out.
2333 This handler removes the commit ready bit from the specified changes and
2334 sends the developer a message explaining why.
2337 changes: A list of cros_patch.GerritPatch instances to mark as failed.
2338 By default, mark all of the changes as failed.
2339 sanity: A boolean indicating whether the build was considered sane. If
2340 not sane, none of the changes will have their CommitReady bit modified.
2343 changes = self.changes
2345 logging.info('Validation timed out for all changes.')
2346 msg = ('%(queue)s timed out while verifying your change in '
2347 '%(build_log)s . This means that a supporting builder did not '
2348 'finish building your change within the specified timeout.')
2350 msg += ('If you believe this happened in error, just re-mark your '
2351 'commit as ready. Your change will then get automatically '
2354 msg += ('The build failure may have been caused by infrastructure '
2355 'issues, so no changes will be blamed for the failure.')
2357 for change in changes:
2358 logging.info('Validation timed out for change %s.', change)
2359 self.SendNotification(change, msg)
2361 self.RemoveCommitReady(change)
2363 def SendNotification(self, change, msg, **kwargs):
2364 d = dict(build_log=self.build_log, queue=self.queue, **kwargs)
2367 except (TypeError, ValueError) as e:
2369 "Generation of message %s for change %s failed: dict was %r, "
2370 "exception %s", msg, change, d, e)
2372 "Generation of message %s for change %s failed: dict was %r, "
2373 "exception %s" % (msg, change, d, e))
2374 PaladinMessage(msg, change, self._helper_pool.ForChange(change)).Send(
2377 def HandlePreCQSuccess(self):
2378 """Handler that is called when the Pre-CQ successfully verifies a change."""
2379 msg = '%(queue)s successfully verified your change in %(build_log)s .'
2380 for change in self.changes:
2381 if self.GetCLStatus(self.bot, change) != self.STATUS_PASSED:
2382 self.SendNotification(change, msg)
2383 self.UpdateCLStatus(self.bot, change, self.STATUS_PASSED,
2384 dry_run=self.dryrun)
2386 def _HandleCouldNotSubmit(self, change, error=''):
2387 """Handler that is called when Paladin can't submit a change.
2389 This should be rare, but if an admin overrides the commit queue and commits
2390 a change that conflicts with this change, it'll apply, build/validate but
2391 receive an error when submitting.
2394 change: GerritPatch instance to operate upon.
2395 error: The reason why the change could not be submitted.
2397 self.SendNotification(change,
2398 '%(queue)s failed to submit your change in %(build_log)s . '
2399 '%(error)s', error=error)
2400 self.RemoveCommitReady(change)
2403 def _CreateValidationFailureMessage(pre_cq, change, suspects, messages,
2404 sanity=True, infra_fail=False,
2405 lab_fail=False, no_stat=None):
2406 """Create a message explaining why a validation failure occurred.
2409 pre_cq: Whether this builder is a Pre-CQ builder.
2410 change: The change we want to create a message for.
2411 suspects: The set of suspect changes that we think broke the build.
2412 messages: A list of build failure messages from supporting builders.
2413 These must be BuildFailureMessage objects or NoneType objects.
2414 sanity: A boolean indicating whether the build was considered sane. If
2415 not sane, none of the changes will have their CommitReady bit modified.
2416 infra_fail: The build failed purely due to infrastructure failures.
2417 lab_fail: The build failed purely due to test lab infrastructure failures.
2418 no_stat: A list of builders which failed prematurely without reporting
2423 msg.append('The following build(s) did not start or failed prematurely:')
2424 msg.append(', '.join(no_stat))
2427 # Build a list of error messages. We don't want to build a ridiculously
2428 # long comment, as Gerrit will reject it. See http://crbug.com/236831
2429 max_error_len = 20000 / max(1, len(messages))
2430 msg.append('The following build(s) failed:')
2431 for message in map(str, messages):
2432 if len(message) > max_error_len:
2433 message = message[:max_error_len] + '... (truncated)'
2436 # Create a list of changes other than this one that might be guilty.
2437 # Limit the number of suspects to 20 so that the list of suspects isn't
2438 # ridiculously long.
2440 other_suspects = suspects - set([change])
2441 if len(other_suspects) < max_suspects:
2442 other_suspects_str = ', '.join(sorted(
2443 'CL:%s' % x.gerrit_number_str for x in other_suspects))
2445 other_suspects_str = ('%d other changes. See the blamelist for more '
2446 'details.' % (len(other_suspects),))
2448 will_retry_automatically = False
2450 msg.append('The build was consider not sane because the sanity check '
2451 'builder(s) failed. Your change will not be blamed for the '
2453 will_retry_automatically = True
2455 msg.append('The build encountered Chrome OS Lab infrastructure issues. '
2456 ' Your change will not be blamed for the failure.')
2457 will_retry_automatically = True
2460 msg.append('The build failure may have been caused by infrastructure '
2461 'issues and/or bad chromite changes.')
2463 if change in suspects:
2464 if other_suspects_str:
2465 msg.append('Your change may have caused this failure. There are '
2466 'also other changes that may be at fault: %s'
2467 % other_suspects_str)
2469 msg.append('This failure was probably caused by your change.')
2471 msg.append('Please check whether the failure is your fault. If your '
2472 'change is not at fault, you may mark it as ready again.')
2474 if len(suspects) == 1:
2475 msg.append('This failure was probably caused by %s'
2476 % other_suspects_str)
2477 elif len(suspects) > 0:
2478 msg.append('One of the following changes is probably at fault: %s'
2479 % other_suspects_str)
2481 will_retry_automatically = not pre_cq
2483 if will_retry_automatically:
2485 0, 'NOTE: The Commit Queue will retry your change automatically.')
2487 return '\n\n'.join(msg)
2489 def _ChangeFailedValidation(self, change, messages, suspects, sanity,
2490 infra_fail, lab_fail, no_stat):
2491 """Handles a validation failure for an individual change.
2494 change: The change to mark as failed.
2495 messages: A list of build failure messages from supporting builders.
2496 These must be BuildFailureMessage objects.
2497 suspects: The list of changes that are suspected of breaking the build.
2498 sanity: A boolean indicating whether the build was considered sane. If
2499 not sane, none of the changes will have their CommitReady bit modified.
2500 infra_fail: The build failed purely due to infrastructure failures.
2501 lab_fail: The build failed purely due to test lab infrastructure failures.
2502 no_stat: A list of builders which failed prematurely without reporting
2505 msg = self._CreateValidationFailureMessage(
2506 self.pre_cq, change, suspects, messages,
2507 sanity, infra_fail, lab_fail, no_stat)
2508 self.SendNotification(change, '%(details)s', details=msg)
2510 if change in suspects:
2511 self.RemoveCommitReady(change)
2513 # Mark the change as failed. If the Ready bit is still set, the change
2514 # will be retried automatically.
2515 self.UpdateCLStatus(self.bot, change, self.STATUS_FAILED,
2516 dry_run=self.dryrun)
2518 def HandleValidationFailure(self, messages, changes=None, sanity=True,
2520 """Handles a list of validation failure messages from slave builders.
2522 This handler parses a list of failure messages from our list of builders
2523 and calculates which changes were likely responsible for the failure. The
2524 changes that were responsible for the failure have their Commit Ready bit
2525 stripped and the other changes are left marked as Commit Ready.
2528 messages: A list of build failure messages from supporting builders.
2529 These must be BuildFailureMessage objects or NoneType objects.
2530 changes: A list of cros_patch.GerritPatch instances to mark as failed.
2531 By default, mark all of the changes as failed.
2532 sanity: A boolean indicating whether the build was considered sane. If
2533 not sane, none of the changes will have their CommitReady bit modified.
2534 no_stat: A list of builders which failed prematurely without reporting
2535 status. If not None, this implies there were infrastructure issues.
2538 changes = self.changes
2540 # Reload the changes to get latest statuses of the changes.
2541 changes = self.ReloadChanges(changes)
2544 for change in changes:
2545 # Pre-CQ ignores changes that were already verified.
2546 if self.pre_cq and self.GetCLStatus(PRE_CQ, change) == self.STATUS_PASSED:
2548 candidates.append(change)
2551 infra_fail = lab_fail = False
2553 # If the build was sane, determine the cause of the failures and
2554 # the changes that are likely at fault for the failure.
2555 lab_fail = CalculateSuspects.OnlyLabFailures(messages, no_stat)
2556 infra_fail = CalculateSuspects.OnlyInfraFailures(messages, no_stat)
2557 suspects = CalculateSuspects.FindSuspects(
2558 self.build_root, candidates, messages, infra_fail=infra_fail,
2560 # Send out failure notifications for each change.
2561 inputs = [[change, messages, suspects, sanity, infra_fail,
2562 lab_fail, no_stat] for change in candidates]
2563 parallel.RunTasksInProcessPool(self._ChangeFailedValidation, inputs)
2565 def HandleCouldNotApply(self, change):
2566 """Handler for when Paladin fails to apply a change.
2568 This handler strips the Commit Ready bit forcing the developer
2569 to re-upload a rebased change as this theirs failed to apply cleanly.
2572 change: GerritPatch instance to operate upon.
2574 msg = '%(queue)s failed to apply your change in %(build_log)s . '
2575 # This is written this way to protect against bugs in CQ itself. We log
2576 # it both to the build output, and mark the change w/ it.
2577 extra_msg = getattr(change, 'apply_error_message', None)
2578 if extra_msg is None:
2580 'Change %s was passed to HandleCouldNotApply without an appropriate '
2581 'apply_error_message set. Internal bug.', change)
2583 'Internal CQ issue: extra error info was not given, Please contact '
2584 'the build team and ensure they are aware of this specific change '
2588 self.SendNotification(change, msg)
2589 self.RemoveCommitReady(change)
2591 def _HandleApplySuccess(self, change):
2592 """Handler for when Paladin successfully applies a change.
2594 This handler notifies a developer that their change is being tried as
2595 part of a Paladin run defined by a build_log.
2598 change: GerritPatch instance to operate upon.
2601 status = self.GetCLStatus(self.bot, change)
2602 if status == self.STATUS_PASSED:
2604 msg = ('%(queue)s has picked up your change. '
2605 'You can follow along at %(build_log)s .')
2606 self.SendNotification(change, msg)
2607 if not self.pre_cq or status == self.STATUS_LAUNCHING:
2608 self.UpdateCLStatus(self.bot, change, self.STATUS_INFLIGHT,
2609 dry_run=self.dryrun)
2612 def GetCLStatusURL(cls, bot, change, latest_patchset_only=True):
2613 """Get the status URL for |change| on |bot|.
2616 bot: Which bot to look at. Can be CQ or PRE_CQ.
2617 change: GerritPatch instance to operate upon.
2618 latest_patchset_only: If True, return the URL for tracking the latest
2619 patchset. If False, return the URL for tracking all patchsets. Defaults
2623 The status URL, as a string.
2625 internal = 'int' if change.internal else 'ext'
2626 components = [constants.MANIFEST_VERSIONS_GS_URL, bot,
2627 internal, str(change.gerrit_number)]
2628 if latest_patchset_only:
2629 components.append(str(change.patch_number))
2630 return '/'.join(components)
2633 def GetCLStatus(cls, bot, change):
2634 """Get the status for |change| on |bot|.
2637 change: GerritPatch instance to operate upon.
2638 bot: Which bot to look at. Can be CQ or PRE_CQ.
2641 The status, as a string.
2643 url = cls.GetCLStatusURL(bot, change)
2644 ctx = gs.GSContext()
2646 return ctx.Cat(url).output
2647 except gs.GSNoSuchKey:
2648 logging.debug('No status yet for %r', url)
2652 def UpdateCLStatus(cls, bot, change, status, dry_run):
2653 """Update the |status| of |change| on |bot|."""
2654 for latest_patchset_only in (False, True):
2655 url = cls.GetCLStatusURL(bot, change, latest_patchset_only)
2656 ctx = gs.GSContext(dry_run=dry_run)
2657 ctx.Copy('-', url, input=status)
2658 ctx.Counter('%s/%s' % (url, status)).Increment()
2661 def GetCLStatusCount(cls, bot, change, status, latest_patchset_only=True):
2662 """Return how many times |change| has been set to |status| on |bot|.
2665 bot: Which bot to look at. Can be CQ or PRE_CQ.
2666 change: GerritPatch instance to operate upon.
2667 status: The status string to look for.
2668 latest_patchset_only: If True, only how many times the latest patchset has
2669 been set to |status|. If False, count how many times any patchset has
2670 been set to |status|. Defaults to False.
2673 The number of times |change| has been set to |status| on |bot|, as an
2676 cache_key = (bot, change, status, latest_patchset_only)
2677 if cache_key not in cls._CL_STATUS_CACHE:
2678 base_url = cls.GetCLStatusURL(bot, change, latest_patchset_only)
2679 url = '%s/%s' % (base_url, status)
2680 cls._CL_STATUS_CACHE[cache_key] = gs.GSContext().Counter(url).Get()
2681 return cls._CL_STATUS_CACHE[cache_key]
2684 def FillCLStatusCache(cls, bot, changes, statuses=None):
2685 """Cache all of the stats about the given |changes| in parallel.
2688 bot: Bot to pull down stats for.
2689 changes: Changes to cache.
2690 statuses: Statuses to cache. By default, cache the PASSED and FAILED
2693 if statuses is None:
2694 statuses = (cls.STATUS_PASSED, cls.STATUS_FAILED)
2696 for change in changes:
2697 for status in statuses:
2698 for latest_patchset_only in (False, True):
2699 cache_key = (bot, change, status, latest_patchset_only)
2700 if cache_key not in cls._CL_STATUS_CACHE:
2701 inputs.append(cache_key)
2703 with parallel.Manager() as manager:
2704 # Grab the CL status of all of the CLs in the background, into a proxied
2706 cls._CL_STATUS_CACHE = manager.dict(cls._CL_STATUS_CACHE)
2707 parallel.RunTasksInProcessPool(cls.GetCLStatusCount, inputs)
2709 # Convert the cache back into a regular dictionary before we shut down
2711 cls._CL_STATUS_CACHE = dict(cls._CL_STATUS_CACHE)
2713 def CreateDisjointTransactions(self, manifest, max_txn_length=None):
2714 """Create a list of disjoint transactions from the changes in the pool.
2717 manifest: Manifest to use.
2718 max_txn_length: The maximum length of any given transaction. Optional.
2719 By default, do not limit the length of transactions.
2722 A list of disjoint transactions. Each transaction can be tried
2723 independently, without involving patches from other transactions.
2724 Each change in the pool will included in exactly one of transactions,
2725 unless the patch does not apply for some reason.
2727 patches = PatchSeries(self.build_root, forced_manifest=manifest)
2728 plans, failed = patches.CreateDisjointTransactions(
2729 self.changes, max_txn_length=max_txn_length)
2730 failed = self._FilterDependencyErrors(failed)
2732 self._HandleApplyFailure(failed)
2736 class PaladinMessage():
2737 """An object that is used to send messages to developers about their changes.
2739 # URL where Paladin documentation is stored.
2740 _PALADIN_DOCUMENTATION_URL = ('http://www.chromium.org/developers/'
2741 'tree-sheriffs/sheriff-details-chromium-os/'
2742 'commit-queue-overview')
2744 # Gerrit can't handle commands over 32768 bytes. See http://crbug.com/236831
2745 MAX_MESSAGE_LEN = 32000
2747 def __init__(self, message, patch, helper):
2748 if len(message) > self.MAX_MESSAGE_LEN:
2749 message = message[:self.MAX_MESSAGE_LEN] + '... (truncated)'
2750 self.message = message
2752 self.helper = helper
2754 def _ConstructPaladinMessage(self):
2755 """Adds any standard Paladin messaging to an existing message."""
2756 return self.message + ('\n\nCommit queue documentation: %s' %
2757 self._PALADIN_DOCUMENTATION_URL)
2759 def Send(self, dryrun):
2760 """Posts a comment to a gerrit review."""
2762 'message': self._ConstructPaladinMessage(),
2765 path = 'changes/%s/revisions/%s/review' % (
2766 self.patch.gerrit_number, self.patch.revision)
2768 logging.info('Would have sent %r to %s', body, path)
2770 gob_util.FetchUrl(self.helper.host, path, reqtype='POST', body=body)