src/third_party/chromite/cbuildbot/validation_pool_unittest.py

   1 #!/usr/bin/python
   2
   3 # Copyright (c) 2011-2012 The Chromium OS Authors. All rights reserved.
   4 # Use of this source code is governed by a BSD-style license that can be
   5 # found in the LICENSE file.
   6
   7 """Module that contains unittests for validation_pool module."""
   8
   9 import contextlib
  10 import copy
  11 import functools
  12 import httplib
  13 import itertools
  14 import mox
  15 import os
  16 import pickle
  17 import sys
  18 import tempfile
  19 import time
  20 import unittest
  21
  22 import constants
  23 sys.path.insert(0, constants.SOURCE_ROOT)
  24
  25 from chromite.cbuildbot import failures_lib
  26 from chromite.cbuildbot import results_lib
  27 from chromite.cbuildbot import metadata_lib
  28 from chromite.cbuildbot import repository
  29 from chromite.cbuildbot import tree_status
  30 from chromite.cbuildbot import validation_pool
  31 from chromite.lib import cros_build_lib
  32 from chromite.lib import cros_build_lib_unittest
  33 from chromite.lib import cros_test_lib
  34 from chromite.lib import gerrit
  35 from chromite.lib import gob_util
  36 from chromite.lib import gs
  37 from chromite.lib import osutils
  38 from chromite.lib import parallel_unittest
  39 from chromite.lib import partial_mock
  40 from chromite.lib import patch as cros_patch
  41 from chromite.lib import patch_unittest
  42
  43
  44 import mock
  45
  46
  47 _GetNumber = iter(itertools.count()).next
  48
  49 # Some tests require the kernel, and fail with buildtools only repo.
  50 KERNEL_AVAILABLE = os.path.exists(os.path.join(
  51     constants.SOURCE_ROOT, 'src', 'third_party', 'kernel'))
  52
  53 def GetTestJson(change_id=None):
  54   """Get usable fake Gerrit patch json data
  55
  56   Args:
  57     change_id: If given, force this ChangeId
  58   """
  59   data = copy.deepcopy(patch_unittest.FAKE_PATCH_JSON)
  60   if change_id is not None:
  61     data['id'] = str(change_id)
  62   return data
  63
  64
  65 class MockManifest(object):
  66   """Helper class for Mocking Manifest objects."""
  67
  68   def __init__(self, path, **kwargs):
  69     self.root = path
  70     for key, attr in kwargs.iteritems():
  71       setattr(self, key, attr)
  72
  73
  74 # pylint: disable=W0212,R0904
  75 class Base(cros_test_lib.MockTestCase):
  76   """Test case base class with helpers for other test suites."""
  77
  78   def setUp(self):
  79     self.patch_mock = None
  80     self._patch_counter = (itertools.count(1)).next
  81     self.build_root = 'fakebuildroot'
  82     self.PatchObject(gob_util, 'CreateHttpConn',
  83                      side_effect=AssertionError('Test should not contact GoB'))
  84     self.PatchObject(tree_status, 'IsTreeOpen', return_value=True)
  85     self.PatchObject(tree_status, 'WaitForTreeStatus',
  86                      return_value=constants.TREE_OPEN)
  87
  88   def MockPatch(self, change_id=None, patch_number=None, is_merged=False,
  89                 project='chromiumos/chromite', remote=constants.EXTERNAL_REMOTE,
  90                 tracking_branch='refs/heads/master', is_draft=False,
  91                 approvals=()):
  92     """Helper function to create mock GerritPatch objects."""
  93     if change_id is None:
  94       change_id = self._patch_counter()
  95     gerrit_number = str(change_id)
  96     change_id = hex(change_id)[2:].rstrip('L').lower()
  97     change_id = 'I%s' % change_id.rjust(40, '0')
  98     sha1 = hex(_GetNumber())[2:].rstrip('L').lower().rjust(40, '0')
  99     patch_number = (patch_number if patch_number is not None else _GetNumber())
 100     fake_url = 'http://foo/bar'
 101     if not approvals:
 102       approvals = [{'type': 'VRIF', 'value': '1', 'grantedOn': 1391733002},
 103                    {'type': 'CRVW', 'value': '2', 'grantedOn': 1391733002},
 104                    {'type': 'COMR', 'value': '1', 'grantedOn': 1391733002},]
 105
 106     current_patch_set = {
 107       'number': patch_number,
 108       'revision': sha1,
 109       'draft': is_draft,
 110       'approvals': approvals,
 111     }
 112     patch_dict = {
 113       'currentPatchSet': current_patch_set,
 114       'id': change_id,
 115       'number': gerrit_number,
 116       'project': project,
 117       'branch': tracking_branch,
 118       'owner': {'email': 'elmer.fudd@chromium.org'},
 119       'remote': remote,
 120       'status': 'MERGED' if is_merged else 'NEW',
 121       'url': '%s/%s' % (fake_url, change_id),
 122     }
 123
 124     patch = cros_patch.GerritPatch(patch_dict, remote, fake_url)
 125     patch.pass_count = 0
 126     patch.fail_count = 1
 127     patch.total_fail_count = 3
 128     return patch
 129
 130   def GetPatches(self, how_many=1, always_use_list=False, **kwargs):
 131     """Get a sequential list of patches.
 132
 133     Args:
 134       how_many: How many patches to return.
 135       always_use_list: Whether to use a list for a single item list.
 136       **kwargs: Keyword arguments for self.MockPatch.
 137     """
 138     patches = [self.MockPatch(**kwargs) for _ in xrange(how_many)]
 139     if self.patch_mock:
 140       for i, patch in enumerate(patches):
 141         self.patch_mock.SetGerritDependencies(patch, patches[:i + 1])
 142     if how_many == 1 and not always_use_list:
 143       return patches[0]
 144     return patches
 145
 146
 147 class MoxBase(Base, cros_test_lib.MoxTestCase):
 148   """Base class for other test suites with numbers mocks patched in."""
 149
 150   def setUp(self):
 151     self.mox.StubOutWithMock(validation_pool, '_RunCommand')
 152     # Suppress all gerrit access; having this occur is generally a sign
 153     # the code is either misbehaving, or that the tests are bad.
 154     self.mox.StubOutWithMock(gerrit.GerritHelper, 'Query')
 155     self.PatchObject(gs.GSContext, 'Cat', side_effect=gs.GSNoSuchKey())
 156     self.PatchObject(gs.GSContext, 'Copy')
 157     self.PatchObject(gs.GSContext, 'Exists', return_value=False)
 158     self.PatchObject(gs.GSCounter, 'Increment')
 159
 160   def MakeHelper(self, cros_internal=None, cros=None):
 161     # pylint: disable=W0201
 162     if cros_internal:
 163       cros_internal = self.mox.CreateMock(gerrit.GerritHelper)
 164       cros_internal.version = '2.2'
 165       cros_internal.remote = constants.INTERNAL_REMOTE
 166     if cros:
 167       cros = self.mox.CreateMock(gerrit.GerritHelper)
 168       cros.remote = constants.EXTERNAL_REMOTE
 169       cros.version = '2.2'
 170     return validation_pool.HelperPool(cros_internal=cros_internal,
 171                                       cros=cros)
 172
 173
 174 class IgnoredStagesTest(Base):
 175   """Tests for functions that calculate what stages to ignore."""
 176
 177   def testBadConfigFile(self):
 178     """Test if we can handle an incorrectly formatted config file."""
 179     with osutils.TempDir(set_global=True) as tempdir:
 180       path = os.path.join(tempdir, 'foo.ini')
 181       osutils.WriteFile(path, 'foobar')
 182       ignored = validation_pool.GetStagesToIgnoreFromConfigFile(path)
 183       self.assertEqual([], ignored)
 184
 185   def testMissingConfigFile(self):
 186     """Test if we can handle a missing config file."""
 187     with osutils.TempDir(set_global=True) as tempdir:
 188       path = os.path.join(tempdir, 'foo.ini')
 189       ignored = validation_pool.GetStagesToIgnoreFromConfigFile(path)
 190       self.assertEqual([], ignored)
 191
 192   def testGoodConfigFile(self):
 193     """Test if we can handle a good config file."""
 194     with osutils.TempDir(set_global=True) as tempdir:
 195       path = os.path.join(tempdir, 'foo.ini')
 196       osutils.WriteFile(path, '[GENERAL]\nignored-stages: bar baz\n')
 197       ignored = validation_pool.GetStagesToIgnoreFromConfigFile(path)
 198       self.assertEqual(['bar', 'baz'], ignored)
 199
 200
 201 class TestPatchSeries(MoxBase):
 202   """Tests resolution and applying logic of validation_pool.ValidationPool."""
 203
 204   @contextlib.contextmanager
 205   def _ValidateTransactionCall(self, _changes):
 206     yield
 207
 208   def GetPatchSeries(self, helper_pool=None):
 209     if helper_pool is None:
 210       helper_pool = self.MakeHelper(cros_internal=True, cros=True)
 211     series = validation_pool.PatchSeries(self.build_root, helper_pool)
 212
 213     # Suppress transactions.
 214     series._Transaction = self._ValidateTransactionCall
 215     series.GetGitRepoForChange = \
 216         lambda change, **kwargs: os.path.join(self.build_root, change.project)
 217
 218     return series
 219
 220   def assertPath(self, _patch, return_value, path):
 221     self.assertEqual(path, os.path.join(self.build_root, _patch.project))
 222     if isinstance(return_value, Exception):
 223       raise return_value
 224     return return_value
 225
 226   def SetPatchDeps(self, patch, parents=(), cq=()):
 227     """Set the dependencies of |patch|.
 228
 229     Args:
 230       patch: The patch to process.
 231       parents: A set of strings to set as parents of |patch|.
 232       cq: A set of strings to set as paladin dependencies of |patch|.
 233     """
 234     patch.GerritDependencies = (
 235         lambda: [cros_patch.ParsePatchDep(x) for x in parents])
 236     patch.PaladinDependencies = functools.partial(
 237         self.assertPath, patch, [cros_patch.ParsePatchDep(x) for x in cq])
 238     patch.Fetch = functools.partial(
 239         self.assertPath, patch, patch.sha1)
 240
 241   def _ValidatePatchApplyManifest(self, value):
 242     self.assertTrue(isinstance(value, MockManifest))
 243     self.assertEqual(value.root, self.build_root)
 244     return True
 245
 246   def SetPatchApply(self, patch, trivial=False):
 247     self.mox.StubOutWithMock(patch, 'ApplyAgainstManifest')
 248     return patch.ApplyAgainstManifest(
 249         mox.Func(self._ValidatePatchApplyManifest),
 250         trivial=trivial)
 251
 252   def assertResults(self, series, changes, applied=(), failed_tot=(),
 253                     failed_inflight=(), frozen=True):
 254     manifest = MockManifest(self.build_root)
 255     result = series.Apply(changes, frozen=frozen, manifest=manifest)
 256
 257     _GetIds = lambda seq:[x.id for x in seq]
 258     _GetFailedIds = lambda seq: _GetIds(x.patch for x in seq)
 259
 260     applied_result = _GetIds(result[0])
 261     failed_tot_result, failed_inflight_result = map(_GetFailedIds, result[1:])
 262
 263     applied = _GetIds(applied)
 264     failed_tot = _GetIds(failed_tot)
 265     failed_inflight = _GetIds(failed_inflight)
 266
 267     self.maxDiff = None
 268     self.assertEqual(applied, applied_result)
 269     self.assertItemsEqual(failed_inflight, failed_inflight_result)
 270     self.assertItemsEqual(failed_tot, failed_tot_result)
 271     return result
 272
 273   def testApplyWithDeps(self):
 274     """Test that we can apply changes correctly and respect deps.
 275
 276     This tests a simple out-of-order change where change1 depends on change2
 277     but tries to get applied before change2.  What should happen is that
 278     we should notice change2 is a dep of change1 and apply it first.
 279     """
 280     series = self.GetPatchSeries()
 281
 282     patch1, patch2 = patches = self.GetPatches(2)
 283
 284     self.SetPatchDeps(patch2)
 285     self.SetPatchDeps(patch1, [patch2.id])
 286
 287     self.SetPatchApply(patch2)
 288     self.SetPatchApply(patch1)
 289
 290     self.mox.ReplayAll()
 291     self.assertResults(series, patches, [patch2, patch1])
 292     self.mox.VerifyAll()
 293
 294   def testSha1Deps(self):
 295     """Test that we can apply changes correctly and respect sha1 deps.
 296
 297     This tests a simple out-of-order change where change1 depends on change2
 298     but tries to get applied before change2.  What should happen is that
 299     we should notice change2 is a dep of change1 and apply it first.
 300     """
 301     series = self.GetPatchSeries()
 302
 303     patch1, patch2, patch3 = patches = self.GetPatches(3)
 304     patch2.change_id = patch2.id = patch2.sha1
 305     patch3.change_id = patch3.id = '*' + patch3.sha1
 306     patch3.remote = constants.INTERNAL_REMOTE
 307
 308     self.SetPatchDeps(patch1, [patch2.sha1])
 309     self.SetPatchDeps(patch2, ['*%s' % patch3.sha1])
 310     self.SetPatchDeps(patch3)
 311
 312     self.SetPatchApply(patch2)
 313     self.SetPatchApply(patch3)
 314     self.SetPatchApply(patch1)
 315
 316     self.mox.ReplayAll()
 317     self.assertResults(series, patches, [patch3, patch2, patch1])
 318     self.mox.VerifyAll()
 319
 320   def testGerritNumberDeps(self):
 321     """Test that we can apply changes correctly and respect gerrit number deps.
 322
 323     This tests a simple out-of-order change where change1 depends on change2
 324     but tries to get applied before change2.  What should happen is that
 325     we should notice change2 is a dep of change1 and apply it first.
 326     """
 327     series = self.GetPatchSeries()
 328
 329     patch1, patch2, patch3 = patches = self.GetPatches(3)
 330
 331     self.SetPatchDeps(patch3, cq=[patch1.gerrit_number])
 332     self.SetPatchDeps(patch2, cq=[patch3.gerrit_number])
 333     self.SetPatchDeps(patch1, cq=[patch2.id])
 334
 335     self.SetPatchApply(patch3)
 336     self.SetPatchApply(patch2)
 337     self.SetPatchApply(patch1)
 338
 339     self.mox.ReplayAll()
 340     self.assertResults(series, patches, patches)
 341     self.mox.VerifyAll()
 342
 343   def testGerritLazyMapping(self):
 344     """Given a patch lacking a gerrit number, via gerrit, map it to that change.
 345
 346     Literally, this ensures that local patches pushed up- lacking a gerrit
 347     number- are mapped back to a changeid via asking gerrit for that number,
 348     then the local matching patch is used if available.
 349     """
 350     series = self.GetPatchSeries()
 351
 352     patch1 = self.MockPatch()
 353     self.PatchObject(patch1, 'LookupAliases', return_value=[patch1.id])
 354     patch2 = self.MockPatch(change_id=int(patch1.change_id[1:]))
 355     patch3 = self.MockPatch()
 356
 357     self.SetPatchDeps(patch3, cq=[patch2.gerrit_number])
 358     self.SetPatchDeps(patch2)
 359     self.SetPatchDeps(patch1)
 360
 361     self.SetPatchApply(patch1)
 362     self.SetPatchApply(patch3)
 363
 364     self._SetQuery(series, patch2, query=patch2.gerrit_number).AndReturn(patch2)
 365
 366     self.mox.ReplayAll()
 367     applied = self.assertResults(series, [patch1, patch3], [patch3, patch1])[0]
 368     self.assertTrue(applied[0] is patch3)
 369     self.assertTrue(applied[1] is patch1)
 370     self.mox.VerifyAll()
 371
 372   def testCrosGerritDeps(self, cros_internal=True):
 373     """Test that we can apply changes correctly and respect deps.
 374
 375     This tests a simple out-of-order change where change1 depends on change3
 376     but tries to get applied before change2.  What should happen is that
 377     we should notice change2 is a dep of change1 and apply it first.
 378     """
 379     helper_pool = self.MakeHelper(cros_internal=cros_internal, cros=True)
 380     series = self.GetPatchSeries(helper_pool=helper_pool)
 381
 382     patch1 = self.MockPatch(remote=constants.EXTERNAL_REMOTE)
 383     patch2 = self.MockPatch(remote=constants.INTERNAL_REMOTE)
 384     patch3 = self.MockPatch(remote=constants.EXTERNAL_REMOTE)
 385     patches = [patch1, patch2, patch3]
 386     if cros_internal:
 387       applied_patches = [patch3, patch1, patch2]
 388     else:
 389       applied_patches = [patch3, patch1]
 390
 391     self.SetPatchDeps(patch1, [patch3.id])
 392     self.SetPatchDeps(patch2)
 393     self.SetPatchDeps(patch3, cq=[patch2.id])
 394
 395     if cros_internal:
 396       self.SetPatchApply(patch2)
 397     self.SetPatchApply(patch1)
 398     self.SetPatchApply(patch3)
 399
 400     self.mox.ReplayAll()
 401     self.assertResults(series, patches, applied_patches)
 402     self.mox.VerifyAll()
 403
 404   def testExternalCrosGerritDeps(self):
 405     """Test that we exclude internal deps on external trybot."""
 406     self.testCrosGerritDeps(cros_internal=False)
 407
 408   @staticmethod
 409   def _SetQuery(series, change, query=None):
 410     helper = series._helper_pool.GetHelper(change.remote)
 411     query = change.id if query is None else query
 412     return helper.QuerySingleRecord(query, must_match=True)
 413
 414   def testApplyMissingDep(self):
 415     """Test that we don't try to apply a change without met dependencies.
 416
 417     Patch2 is in the validation pool that depends on Patch1 (which is not)
 418     Nothing should get applied.
 419     """
 420     series = self.GetPatchSeries()
 421
 422     patch1, patch2 = self.GetPatches(2)
 423
 424     self.SetPatchDeps(patch2, [patch1.id])
 425     self._SetQuery(series, patch1).AndReturn(patch1)
 426
 427     self.mox.ReplayAll()
 428     self.assertResults(series, [patch2],
 429                        [], [patch2])
 430     self.mox.VerifyAll()
 431
 432   def testApplyWithCommittedDeps(self):
 433     """Test that we apply a change with dependency already committed."""
 434     series = self.GetPatchSeries()
 435
 436     # Use for basic commit check.
 437     patch1 = self.GetPatches(1, is_merged=True)
 438     patch2 = self.GetPatches(1)
 439
 440     self.SetPatchDeps(patch2, [patch1.id])
 441     self._SetQuery(series, patch1).AndReturn(patch1)
 442     self.SetPatchApply(patch2)
 443
 444     # Used to ensure that an uncommitted change put in the lookup cache
 445     # isn't invalidly pulled into the graph...
 446     patch3, patch4, patch5 = self.GetPatches(3)
 447
 448     self._SetQuery(series, patch3).AndReturn(patch3)
 449     self.SetPatchDeps(patch4, [patch3.id])
 450     self.SetPatchDeps(patch5, [patch3.id])
 451
 452     self.mox.ReplayAll()
 453     self.assertResults(series, [patch2, patch4, patch5], [patch2],
 454                        [patch4, patch5])
 455     self.mox.VerifyAll()
 456
 457   def testCyclicalDeps(self):
 458     """Verify that the machinery handles cycles correctly."""
 459     series = self.GetPatchSeries()
 460
 461     patch1, patch2, patch3 = patches = self.GetPatches(3)
 462
 463     self.SetPatchDeps(patch1, [patch2.id])
 464     self.SetPatchDeps(patch2, cq=[patch3.id])
 465     self.SetPatchDeps(patch3, [patch1.id])
 466
 467     self.SetPatchApply(patch1)
 468     self.SetPatchApply(patch2)
 469     self.SetPatchApply(patch3)
 470
 471     self.mox.ReplayAll()
 472     self.assertResults(series, patches, [patch2, patch1, patch3])
 473     self.mox.VerifyAll()
 474
 475   def testComplexCyclicalDeps(self, fail=False):
 476     """Verify handling of two interdependent cycles."""
 477     series = self.GetPatchSeries()
 478
 479     # Create two cyclically interdependent patch chains.
 480     # Example: Two patch series A1<-A2<-A3<-A4 and B1<-B2<-B3<-B4. A1 has a
 481     # CQ-DEPEND on B4 and B1 has a CQ-DEPEND on A4, so all of the patches must
 482     # be committed together.
 483     chain1, chain2 = chains = self.GetPatches(4), self.GetPatches(4)
 484     for chain in chains:
 485       (other_chain,) = [x for x in chains if x != chain]
 486       self.SetPatchDeps(chain[0], [], cq=[other_chain[-1].id])
 487       for i in range(1, len(chain)):
 488         self.SetPatchDeps(chain[i], [chain[i-1].id])
 489
 490     # Apply the second-last patch first, so that the last patch in the series
 491     # will be pulled in via the CQ-DEPEND on the other patch chain.
 492     to_apply = [chain1[-2]] + [x for x in (chain1 + chain2) if x != chain1[-2]]
 493
 494     # All of the patches but chain[-1] were applied successfully.
 495     for patch in chain1[:-1] + chain2:
 496       self.SetPatchApply(patch)
 497
 498     if fail:
 499       # Pretend that chain[-1] failed to apply.
 500       res = self.SetPatchApply(chain1[-1])
 501       res.AndRaise(cros_patch.ApplyPatchException(chain1[-1]))
 502       applied = []
 503       failed_tot = to_apply
 504     else:
 505       # We apply the patches in this order since the last patch in chain1
 506       # is pulled in via CQ-DEPEND.
 507       self.SetPatchApply(chain1[-1])
 508       applied = chain1[:-1] + chain2 + [chain1[-1]]
 509       failed_tot = []
 510
 511     self.mox.ReplayAll()
 512     self.assertResults(series, to_apply, applied=applied, failed_tot=failed_tot)
 513     self.mox.VerifyAll()
 514
 515   def testFailingComplexCyclicalDeps(self):
 516     """Verify handling of failing interlocked cycles."""
 517     self.testComplexCyclicalDeps(fail=True)
 518
 519   def testApplyPartialFailures(self):
 520     """Test that can apply changes correctly when one change fails to apply.
 521
 522     This tests a simple change order where 1 depends on 2 and 1 fails to apply.
 523     Only 1 should get tried as 2 will abort once it sees that 1 can't be
 524     applied.  3 with no dependencies should go through fine.
 525
 526     Since patch1 fails to apply, we should also get a call to handle the
 527     failure.
 528     """
 529     series = self.GetPatchSeries()
 530
 531     patch1, patch2, patch3, patch4 = patches = self.GetPatches(4)
 532
 533     self.SetPatchDeps(patch1)
 534     self.SetPatchDeps(patch2, [patch1.id])
 535     self.SetPatchDeps(patch3)
 536     self.SetPatchDeps(patch4)
 537
 538     self.SetPatchApply(patch1).AndRaise(
 539         cros_patch.ApplyPatchException(patch1))
 540
 541     self.SetPatchApply(patch3)
 542     self.SetPatchApply(patch4).AndRaise(
 543         cros_patch.ApplyPatchException(patch1, inflight=True))
 544
 545     self.mox.ReplayAll()
 546     self.assertResults(series, patches,
 547                        [patch3], [patch2, patch1], [patch4])
 548     self.mox.VerifyAll()
 549
 550   def testComplexApply(self):
 551     """More complex deps test.
 552
 553     This tests a total of 2 change chains where the first change we see
 554     only has a partial chain with the 3rd change having the whole chain i.e.
 555     1->2, 3->1->2.  Since we get these in the order 1,2,3,4,5 the order we
 556     should apply is 2,1,3,4,5.
 557
 558     This test also checks the patch order to verify that Apply re-orders
 559     correctly based on the chain.
 560     """
 561     series = self.GetPatchSeries()
 562
 563     patch1, patch2, patch3, patch4, patch5 = patches = self.GetPatches(5)
 564
 565     self.SetPatchDeps(patch1, [patch2.id])
 566     self.SetPatchDeps(patch2)
 567     self.SetPatchDeps(patch3, [patch1.id, patch2.id])
 568     self.SetPatchDeps(patch4, cq=[patch5.id])
 569     self.SetPatchDeps(patch5)
 570
 571     for patch in (patch2, patch1, patch3, patch4, patch5):
 572       self.SetPatchApply(patch)
 573
 574     self.mox.ReplayAll()
 575     self.assertResults(
 576         series, patches, [patch2, patch1, patch3, patch4, patch5])
 577     self.mox.VerifyAll()
 578
 579   def testApplyStandalonePatches(self):
 580     """Simple apply of two changes with no dependent CL's."""
 581     series = self.GetPatchSeries()
 582
 583     patches = self.GetPatches(3)
 584
 585     for patch in patches:
 586       self.SetPatchDeps(patch)
 587
 588     for patch in patches:
 589       self.SetPatchApply(patch)
 590
 591     self.mox.ReplayAll()
 592     self.assertResults(series, patches, patches)
 593     self.mox.VerifyAll()
 594
 595
 596 def MakePool(overlays=constants.PUBLIC_OVERLAYS, build_number=1,
 597              builder_name='foon', is_master=True, dryrun=True, **kwargs):
 598   """Helper for creating ValidationPool objects for tests."""
 599   kwargs.setdefault('changes', [])
 600   build_root = kwargs.pop('build_root', '/fake_root')
 601
 602   pool = validation_pool.ValidationPool(
 603       overlays, build_root, build_number, builder_name, is_master,
 604       dryrun, **kwargs)
 605   return pool
 606
 607
 608 class MockPatchSeries(partial_mock.PartialMock):
 609   """Mock the PatchSeries functions."""
 610   TARGET = 'chromite.cbuildbot.validation_pool.PatchSeries'
 611   ATTRS = ('GetDepsForChange', '_GetGerritPatch', '_LookupHelper')
 612
 613   def __init__(self):
 614     partial_mock.PartialMock.__init__(self)
 615     self.deps = {}
 616     self.cq_deps = {}
 617
 618   def SetGerritDependencies(self, patch, deps):
 619     """Add |deps| to the Gerrit dependencies of |patch|."""
 620     self.deps[patch] = deps
 621
 622   def SetCQDependencies(self, patch, deps):
 623     """Add |deps| to the CQ dependencies of |patch|."""
 624     self.cq_deps[patch] = deps
 625
 626   def GetDepsForChange(self, _inst, patch):
 627     return self.deps.get(patch, []), self.cq_deps.get(patch, [])
 628
 629   def _GetGerritPatch(self, _inst, dep, **_kwargs):
 630     return dep
 631
 632   _LookupHelper = mock.MagicMock()
 633
 634
 635 class TestSubmitChange(MoxBase):
 636   """Test suite related to submitting changes."""
 637
 638   def setUp(self):
 639     self.orig_timeout = validation_pool.SUBMITTED_WAIT_TIMEOUT
 640     validation_pool.SUBMITTED_WAIT_TIMEOUT = 4
 641
 642   def tearDown(self):
 643     validation_pool.SUBMITTED_WAIT_TIMEOUT = self.orig_timeout
 644
 645   def _TestSubmitChange(self, results):
 646     """Test submitting a change with the given results."""
 647     results = [cros_test_lib.EasyAttr(status=r) for r in results]
 648     change = self.MockPatch(change_id=12345, patch_number=1)
 649     pool = self.mox.CreateMock(validation_pool.ValidationPool)
 650     pool.dryrun = False
 651     pool._metadata = metadata_lib.CBuildbotMetadata()
 652     pool._helper_pool = self.mox.CreateMock(validation_pool.HelperPool)
 653     helper = self.mox.CreateMock(validation_pool.gerrit.GerritHelper)
 654
 655     # Prepare replay script.
 656     pool._helper_pool.ForChange(change).AndReturn(helper)
 657     helper.SubmitChange(change, dryrun=False)
 658     for result in results:
 659       helper.QuerySingleRecord(change.gerrit_number).AndReturn(result)
 660     self.mox.ReplayAll()
 661
 662     # Verify results.
 663     retval = validation_pool.ValidationPool._SubmitChange(pool, change)
 664     self.mox.VerifyAll()
 665     return retval
 666
 667   def testSubmitChangeMerged(self):
 668     """Submit one change to gerrit, status MERGED."""
 669     self.assertTrue(self._TestSubmitChange(['MERGED']))
 670
 671   def testSubmitChangeSubmitted(self):
 672     """Submit one change to gerrit, stuck on SUBMITTED."""
 673     # The query will be retried 1 more time than query timeout.
 674     results = ['SUBMITTED' for _i in
 675                xrange(validation_pool.SUBMITTED_WAIT_TIMEOUT + 1)]
 676     self.assertTrue(self._TestSubmitChange(results))
 677
 678   def testSubmitChangeSubmittedToMerged(self):
 679     """Submit one change to gerrit, status SUBMITTED then MERGED."""
 680     results = ['SUBMITTED', 'SUBMITTED', 'MERGED']
 681     self.assertTrue(self._TestSubmitChange(results))
 682
 683   def testSubmitChangeFailed(self):
 684     """Submit one change to gerrit, reported back as NEW."""
 685     self.assertFalse(self._TestSubmitChange(['NEW']))
 686
 687
 688 class ValidationFailureOrTimeout(MoxBase):
 689   """Tests that HandleValidationFailure and HandleValidationTimeout functions.
 690
 691   These tests check that HandleValidationTimeout and HandleValidationFailure
 692   reject (i.e. zero out the CQ field) of the correct number of patches, under
 693   various circumstances.
 694   """
 695
 696   _PATCH_MESSAGE = 'Your patch failed.'
 697   _BUILD_MESSAGE = 'Your build failed.'
 698
 699   def setUp(self):
 700     self._patches = self.GetPatches(3)
 701     self._pool = MakePool(changes=self._patches)
 702
 703     self.PatchObject(
 704         validation_pool.ValidationPool, 'GetCLStatus',
 705         return_value=validation_pool.ValidationPool.STATUS_PASSED)
 706     self.PatchObject(
 707         validation_pool.CalculateSuspects, 'FindSuspects',
 708         return_value=self._patches)
 709     self.PatchObject(
 710         validation_pool.ValidationPool, '_CreateValidationFailureMessage',
 711         return_value=self._PATCH_MESSAGE)
 712     self.PatchObject(validation_pool.ValidationPool, 'SendNotification')
 713     self.PatchObject(validation_pool.ValidationPool, 'RemoveCommitReady')
 714     self.PatchObject(validation_pool.ValidationPool, 'UpdateCLStatus')
 715     self.PatchObject(validation_pool.ValidationPool, 'ReloadChanges',
 716                      return_value=self._patches)
 717     self.PatchObject(validation_pool.CalculateSuspects, 'OnlyLabFailures',
 718                      return_value=False)
 719     self.PatchObject(validation_pool.CalculateSuspects, 'OnlyInfraFailures',
 720                      return_value=False)
 721     self.StartPatcher(parallel_unittest.ParallelMock())
 722
 723   def testPatchesWereRejectedByFailure(self):
 724     """Tests that all patches are rejected by failure."""
 725     self._pool.HandleValidationFailure([self._BUILD_MESSAGE])
 726     self.assertEqual(
 727         len(self._patches), self._pool.RemoveCommitReady.call_count)
 728
 729   def testPatchesWereRejectedByTimeout(self):
 730     self._pool.HandleValidationTimeout()
 731     self.assertEqual(
 732         len(self._patches), self._pool.RemoveCommitReady.call_count)
 733
 734   def testNoSuspectsWithFailure(self):
 735     """Tests no change is blamed when there is no suspect."""
 736     self.PatchObject(validation_pool.CalculateSuspects, 'FindSuspects',
 737                      return_value=[])
 738     self._pool.HandleValidationFailure([self._BUILD_MESSAGE])
 739     self.assertEqual(0, self._pool.RemoveCommitReady.call_count)
 740
 741   def testPreCQ(self):
 742     self._pool.pre_cq = True
 743     self._pool.HandleValidationFailure([self._BUILD_MESSAGE])
 744     self.assertEqual(0, self._pool.RemoveCommitReady.call_count)
 745
 746   def testPatchesWereNotRejectedByInsaneFailure(self):
 747     self._pool.HandleValidationFailure([self._BUILD_MESSAGE], sanity=False)
 748     self.assertEqual(0, self._pool.RemoveCommitReady.call_count)
 749
 750
 751 class TestCoreLogic(MoxBase):
 752   """Tests resolution and applying logic of validation_pool.ValidationPool."""
 753
 754   def setUp(self):
 755     self.mox.StubOutWithMock(validation_pool.PatchSeries, 'Apply')
 756     self.mox.StubOutWithMock(validation_pool.PatchSeries, 'ApplyChange')
 757     self.patch_mock = self.StartPatcher(MockPatchSeries())
 758     funcs = ['SendNotification', '_SubmitChange']
 759     for func in funcs:
 760       self.mox.StubOutWithMock(validation_pool.ValidationPool, func)
 761     self.PatchObject(validation_pool.ValidationPool, 'ReloadChanges',
 762                      side_effect=lambda x: x)
 763     self.StartPatcher(parallel_unittest.ParallelMock())
 764
 765   def MakePool(self, *args, **kwargs):
 766     """Helper for creating ValidationPool objects for Mox tests."""
 767     handlers = kwargs.pop('handlers', False)
 768     kwargs['build_root'] = self.build_root
 769     pool = MakePool(*args, **kwargs)
 770     funcs = ['_HandleApplySuccess', '_HandleApplyFailure',
 771              '_HandleCouldNotApply', '_HandleCouldNotSubmit']
 772     if handlers:
 773       for func in funcs:
 774         self.mox.StubOutWithMock(pool, func)
 775     return pool
 776
 777   def MakeFailure(self, patch, inflight=True):
 778     return cros_patch.ApplyPatchException(patch, inflight=inflight)
 779
 780   def GetPool(self, changes, applied=(), tot=(), inflight=(), **kwargs):
 781     pool = self.MakePool(changes=changes, **kwargs)
 782     applied = list(applied)
 783     tot = [self.MakeFailure(x, inflight=False) for x in tot]
 784     inflight = [self.MakeFailure(x, inflight=True) for x in inflight]
 785     # pylint: disable=E1120,E1123
 786     validation_pool.PatchSeries.Apply(
 787         changes, manifest=mox.IgnoreArg()
 788         ).AndReturn((applied, tot, inflight))
 789
 790     for patch in applied:
 791       pool._HandleApplySuccess(patch).AndReturn(None)
 792
 793     if tot:
 794       pool._HandleApplyFailure(tot).AndReturn(None)
 795
 796     # We stash this on the pool object so we can reuse it during validation.
 797     # We could stash this in the test instances, but that would break
 798     # for any tests that do multiple pool instances.
 799
 800     pool._test_data = (changes, applied, tot, inflight)
 801
 802     return pool
 803
 804   def testApplySlavePool(self):
 805     """Verifies that slave calls ApplyChange() directly for each patch."""
 806     slave_pool = self.MakePool(is_master=False)
 807     patches = self.GetPatches(3)
 808     slave_pool.changes = patches
 809     for patch in patches:
 810       # pylint: disable=E1120, E1123
 811       validation_pool.PatchSeries.ApplyChange(patch, manifest=mox.IgnoreArg())
 812
 813     self.mox.ReplayAll()
 814     self.assertEqual(True, slave_pool.ApplyPoolIntoRepo())
 815     self.mox.VerifyAll()
 816
 817   def runApply(self, pool, result):
 818     self.assertEqual(result, pool.ApplyPoolIntoRepo())
 819     self.assertEqual(pool.changes, pool._test_data[1])
 820     failed_inflight = pool.changes_that_failed_to_apply_earlier
 821     expected_inflight = set(pool._test_data[3])
 822     # Intersect the results, since it's possible there were results failed
 823     # results that weren't related to the ApplyPoolIntoRepo call.
 824     self.assertEqual(set(failed_inflight).intersection(expected_inflight),
 825                      expected_inflight)
 826
 827     self.assertEqual(pool.changes, pool._test_data[1])
 828
 829   def testPatchSeriesInteraction(self):
 830     """Verify the interaction between PatchSeries and ValidationPool.
 831
 832     Effectively, this validates data going into PatchSeries, and coming back
 833     out; verifies the hand off to _Handle* functions, but no deeper.
 834     """
 835     patches = self.GetPatches(3)
 836
 837     apply_pool = self.GetPool(patches, applied=patches, handlers=True)
 838     all_inflight = self.GetPool(patches, inflight=patches, handlers=True)
 839     all_tot = self.GetPool(patches, tot=patches, handlers=True)
 840     mixed = self.GetPool(patches, tot=patches[0:1], inflight=patches[1:2],
 841                          applied=patches[2:3], handlers=True)
 842
 843     self.mox.ReplayAll()
 844     self.runApply(apply_pool, True)
 845     self.runApply(all_inflight, False)
 846     self.runApply(all_tot, False)
 847     self.runApply(mixed, True)
 848     self.mox.VerifyAll()
 849
 850   def testHandleApplySuccess(self):
 851     """Validate steps taken for successfull application."""
 852     patch = self.GetPatches(1)
 853     pool = self.MakePool()
 854     pool.SendNotification(patch, mox.StrContains('has picked up your change'))
 855     self.mox.ReplayAll()
 856     pool._HandleApplySuccess(patch)
 857     self.mox.VerifyAll()
 858
 859   def testHandleApplyFailure(self):
 860     failures = [cros_patch.ApplyPatchException(x) for x in self.GetPatches(4)]
 861
 862     notified_patches = failures[:2]
 863     unnotified_patches = failures[2:]
 864     master_pool = self.MakePool(dryrun=False)
 865     slave_pool = self.MakePool(is_master=False)
 866
 867     self.mox.StubOutWithMock(gerrit.GerritHelper, 'RemoveCommitReady')
 868
 869     for failure in notified_patches:
 870       master_pool.SendNotification(
 871           failure.patch,
 872           mox.StrContains('failed to apply your change'),
 873           failure=mox.IgnoreArg())
 874       # This pylint suppressin shouldn't be necessary, but pylint is invalidly
 875       # thinking that the first arg isn't passed in; we suppress it to suppress
 876       # the pylnt bug.
 877       # pylint: disable=E1120
 878       gerrit.GerritHelper.RemoveCommitReady(failure.patch, dryrun=False)
 879
 880     self.mox.ReplayAll()
 881     master_pool._HandleApplyFailure(notified_patches)
 882     slave_pool._HandleApplyFailure(unnotified_patches)
 883     self.mox.VerifyAll()
 884
 885   def _setUpSubmit(self):
 886     pool = self.MakePool(dryrun=False, handlers=True)
 887     patches = self.GetPatches(3)
 888     failed = self.GetPatches(3)
 889     pool.changes = patches[:]
 890     # While we don't do anything w/ these patches, that's
 891     # intentional; we're verifying that it isn't submitted
 892     # if there is a failure.
 893     pool.changes_that_failed_to_apply_earlier = failed[:]
 894
 895     return (pool, patches, failed)
 896
 897   def testSubmitPoolFailures(self):
 898     """Tests that a fatal exception is raised."""
 899     pool, patches, _failed = self._setUpSubmit()
 900     patch1, patch2, patch3 = patches
 901
 902     pool._SubmitChange(patch1).AndReturn(True)
 903     pool._SubmitChange(patch2).AndReturn(False)
 904
 905     pool._HandleCouldNotSubmit(patch2, mox.IgnoreArg()).InAnyOrder()
 906     pool._HandleCouldNotSubmit(patch3, mox.IgnoreArg()).InAnyOrder()
 907
 908     self.mox.ReplayAll()
 909     self.assertRaises(validation_pool.FailedToSubmitAllChangesException,
 910                       pool.SubmitPool)
 911     self.mox.VerifyAll()
 912
 913   def testSubmitPartialPass(self):
 914     """Tests that a non-fatal exception is raised."""
 915     pool, patches, _failed = self._setUpSubmit()
 916     patch1, patch2, patch3 = patches
 917     # Make patch2 not commit-ready.
 918     patch2._approvals = []
 919
 920     pool._SubmitChange(patch1).AndReturn(True)
 921
 922     pool._HandleCouldNotSubmit(patch2, mox.IgnoreArg()).InAnyOrder()
 923     pool._HandleCouldNotSubmit(patch3, mox.IgnoreArg()).InAnyOrder()
 924
 925     self.mox.ReplayAll()
 926     self.assertRaises(validation_pool.FailedToSubmitAllChangesNonFatalException,
 927                       pool.SubmitPool)
 928     self.mox.VerifyAll()
 929
 930   def testSubmitPool(self):
 931     """Tests that we can submit a pool of patches."""
 932     pool, patches, failed = self._setUpSubmit()
 933
 934     for patch in patches:
 935       pool._SubmitChange(patch).AndReturn(True)
 936
 937     pool._HandleApplyFailure(failed)
 938
 939     self.mox.ReplayAll()
 940     pool.SubmitPool()
 941     self.mox.VerifyAll()
 942
 943   def testSubmitNonManifestChanges(self):
 944     """Simple test to make sure we can submit non-manifest changes."""
 945     pool, patches, _failed = self._setUpSubmit()
 946     pool.non_manifest_changes = patches[:]
 947
 948     for patch in patches:
 949       pool._SubmitChange(patch).AndReturn(True)
 950
 951     self.mox.ReplayAll()
 952     pool.SubmitNonManifestChanges()
 953     self.mox.VerifyAll()
 954
 955   def testUnhandledExceptions(self):
 956     """Test that CQ doesn't loop due to unhandled Exceptions."""
 957     pool, patches, _failed = self._setUpSubmit()
 958
 959     class MyException(Exception):
 960       """"Unique Exception used for testing."""
 961
 962     def VerifyCQError(patch, error):
 963       cq_error = validation_pool.InternalCQError(patch, error.message)
 964       return str(error) == str(cq_error)
 965
 966     # pylint: disable=E1120,E1123
 967     validation_pool.PatchSeries.Apply(
 968         patches, manifest=mox.IgnoreArg()).AndRaise(MyException)
 969     errors = [mox.Func(functools.partial(VerifyCQError, x)) for x in patches]
 970     pool._HandleApplyFailure(errors).AndReturn(None)
 971
 972     self.mox.ReplayAll()
 973     self.assertRaises(MyException, pool.ApplyPoolIntoRepo)
 974     self.mox.VerifyAll()
 975
 976   def testFilterDependencyErrors(self):
 977     """Verify that dependency errors are correctly filtered out."""
 978     failures = [cros_patch.ApplyPatchException(x) for x in self.GetPatches(2)]
 979     failures += [cros_patch.DependencyError(x, y) for x, y in
 980                  zip(self.GetPatches(2), failures)]
 981     failures[0].patch.approval_timestamp = time.time()
 982     failures[-1].patch.approval_timestamp = time.time()
 983     self.mox.ReplayAll()
 984     result = validation_pool.ValidationPool._FilterDependencyErrors(failures)
 985     self.assertEquals(set(failures[:-1]), set(result))
 986     self.mox.VerifyAll()
 987
 988   def testFilterNonCrosProjects(self):
 989     """Runs through a filter of own manifest and fake changes.
 990
 991     This test should filter out the tacos/chromite project as its not real.
 992     """
 993     base_func = itertools.cycle(['chromiumos', 'chromeos']).next
 994     patches = self.GetPatches(8)
 995     for patch in patches:
 996       patch.project = '%s/%i' % (base_func(), _GetNumber())
 997       patch.tracking_branch = str(_GetNumber())
 998
 999     non_cros_patches = self.GetPatches(2)
1000     for patch in non_cros_patches:
1001       patch.project = str(_GetNumber())
1002
1003     filtered_patches = patches[:4]
1004     allowed_patches = []
1005     projects = {}
1006     for idx, patch in enumerate(patches[4:]):
1007       fails = bool(idx % 2)
1008       # Vary the revision so we can validate that it checks the branch.
1009       revision = ('monkeys' if fails
1010                   else 'refs/heads/%s' % patch.tracking_branch)
1011       if fails:
1012         filtered_patches.append(patch)
1013       else:
1014         allowed_patches.append(patch)
1015       projects.setdefault(patch.project, {})['revision'] = revision
1016
1017     manifest = MockManifest(self.build_root, projects=projects)
1018     for patch in allowed_patches:
1019       patch.GetCheckout = lambda *_args, **_kwargs: True
1020     for patch in filtered_patches:
1021       patch.GetCheckout = lambda *_args, **_kwargs: False
1022
1023     self.mox.ReplayAll()
1024     results = validation_pool.ValidationPool._FilterNonCrosProjects(
1025         patches + non_cros_patches, manifest)
1026
1027     def compare(list1, list2):
1028       mangle = lambda c:(c.id, c.project, c.tracking_branch)
1029       self.assertEqual(list1, list2,
1030         msg="Comparison failed:\n list1: %r\n list2: %r"
1031             % (map(mangle, list1), map(mangle, list2)))
1032
1033     compare(results[0], allowed_patches)
1034     compare(results[1], filtered_patches)
1035
1036
1037 class TestPickling(cros_test_lib.TempDirTestCase):
1038   """Tests to validate pickling of ValidationPool, covering CQ's needs"""
1039
1040   def testSelfCompatibility(self):
1041     """Verify compatibility of current git HEAD against itself."""
1042     self._CheckTestData(self._GetTestData())
1043
1044   def testToTCompatibility(self):
1045     """Validate that ToT can use our pickles, and that we can use ToT's data."""
1046     repo = os.path.join(self.tempdir, 'chromite')
1047     reference = os.path.abspath(__file__)
1048     reference = os.path.normpath(os.path.join(reference, '../../'))
1049
1050     repository.CloneGitRepo(
1051         repo,
1052         '%s/chromiumos/chromite' % constants.EXTERNAL_GOB_URL,
1053         reference=reference)
1054
1055     code = """
1056 import sys
1057 from chromite.cbuildbot import validation_pool_unittest
1058 if not hasattr(validation_pool_unittest, 'TestPickling'):
1059   sys.exit(0)
1060 sys.stdout.write(validation_pool_unittest.TestPickling.%s)
1061 """
1062
1063     # Verify ToT can take our pickle.
1064     cros_build_lib.RunCommand(
1065         ['python', '-c', code % '_CheckTestData(sys.stdin.read())'],
1066         cwd=self.tempdir, print_cmd=False, capture_output=True,
1067         input=self._GetTestData())
1068
1069     # Verify we can handle ToT's pickle.
1070     ret = cros_build_lib.RunCommand(
1071         ['python', '-c', code % '_GetTestData()'],
1072         cwd=self.tempdir, print_cmd=False, capture_output=True)
1073
1074     self._CheckTestData(ret.output)
1075
1076   @staticmethod
1077   def _GetCrosInternalPatch(patch_info):
1078     return cros_patch.GerritPatch(
1079         patch_info,
1080         constants.INTERNAL_REMOTE,
1081         constants.INTERNAL_GERRIT_URL)
1082
1083   @staticmethod
1084   def _GetCrosPatch(patch_info):
1085     return cros_patch.GerritPatch(
1086         patch_info,
1087         constants.EXTERNAL_REMOTE,
1088         constants.EXTERNAL_GERRIT_URL)
1089
1090   @classmethod
1091   def _GetTestData(cls):
1092     ids = [cros_patch.MakeChangeId() for _ in xrange(3)]
1093     changes = [cls._GetCrosInternalPatch(GetTestJson(ids[0]))]
1094     non_os = [cls._GetCrosPatch(GetTestJson(ids[1]))]
1095     conflicting = [cls._GetCrosInternalPatch(GetTestJson(ids[2]))]
1096     conflicting = [cros_patch.PatchException(x)for x in conflicting]
1097     pool = validation_pool.ValidationPool(
1098         constants.PUBLIC_OVERLAYS,
1099         '/fake/pathway', 1,
1100         'testing', True, True,
1101         changes=changes, non_os_changes=non_os,
1102         conflicting_changes=conflicting)
1103     return pickle.dumps([pool, changes, non_os, conflicting])
1104
1105   @staticmethod
1106   def _CheckTestData(data):
1107     results = pickle.loads(data)
1108     pool, changes, non_os, conflicting = results
1109     def _f(source, value, getter=None):
1110       if getter is None:
1111         getter = lambda x: x
1112       assert len(source) == len(value)
1113       for s_item, v_item in zip(source, value):
1114         assert getter(s_item).id == getter(v_item).id
1115         assert getter(s_item).remote == getter(v_item).remote
1116     _f(pool.changes, changes)
1117     _f(pool.non_manifest_changes, non_os)
1118     _f(pool.changes_that_failed_to_apply_earlier, conflicting,
1119        getter=lambda s:getattr(s, 'patch', s))
1120     return ''
1121
1122
1123 class TestFindSuspects(MoxBase):
1124   """Tests validation_pool.ValidationPool._FindSuspects"""
1125
1126   def setUp(self):
1127     overlay = 'chromiumos/overlays/chromiumos-overlay'
1128     self.overlay_patch = self.GetPatches(project=overlay)
1129     chromite = 'chromiumos/chromite'
1130     self.chromite_patch = self.GetPatches(project=chromite)
1131     self.power_manager = 'chromiumos/platform2/power_manager'
1132     self.power_manager_pkg = 'chromeos-base/power_manager'
1133     self.power_manager_patch = self.GetPatches(project=self.power_manager)
1134     self.kernel = 'chromiumos/third_party/kernel'
1135     self.kernel_pkg = 'sys-kernel/chromeos-kernel'
1136     self.kernel_patch = self.GetPatches(project=self.kernel)
1137     self.secret = 'chromeos/secret'
1138     self.secret_patch = self.GetPatches(project=self.secret,
1139                                         remote=constants.INTERNAL_REMOTE)
1140     self.PatchObject(cros_patch.GitRepoPatch, 'GetCheckout')
1141     self.PatchObject(cros_patch.GitRepoPatch, 'GetDiffStatus')
1142
1143   @staticmethod
1144   def _GetBuildFailure(pkg):
1145     """Create a PackageBuildFailure for the specified |pkg|.
1146
1147     Args:
1148       pkg: Package that failed to build.
1149     """
1150     ex = cros_build_lib.RunCommandError('foo', cros_build_lib.CommandResult())
1151     return failures_lib.PackageBuildFailure(ex, 'bar', [pkg])
1152
1153   def _GetFailedMessage(self, exceptions, stage='Build', internal=False,
1154                         bot='daisy_spring-paladin'):
1155     """Returns a BuildFailureMessage object."""
1156     tracebacks = []
1157     for ex in exceptions:
1158       tracebacks.append(results_lib.RecordedTraceback('Build', 'Build', ex,
1159                                                       str(ex)))
1160     reason = 'failure reason string'
1161     return failures_lib.BuildFailureMessage(
1162         'Stage %s failed' % stage, tracebacks, internal, reason, bot)
1163
1164   def _AssertSuspects(self, patches, suspects, pkgs=(), exceptions=(),
1165                       internal=False, infra_fail=False, lab_fail=False):
1166     """Run _FindSuspects and verify its output.
1167
1168     Args:
1169       patches: List of patches to look at.
1170       suspects: Expected list of suspects returned by _FindSuspects.
1171       pkgs: List of packages that failed with exceptions in the build.
1172       exceptions: List of other exceptions that occurred during the build.
1173       internal: Whether the failures occurred on an internal bot.
1174       infra_fail: Whether the build failed due to infrastructure issues.
1175       lab_fail: Whether the build failed due to lab infrastructure issues.
1176     """
1177     all_exceptions = list(exceptions) + [self._GetBuildFailure(x) for x in pkgs]
1178     message = self._GetFailedMessage(all_exceptions, internal=internal)
1179     results = validation_pool.CalculateSuspects.FindSuspects(
1180         constants.SOURCE_ROOT, patches, [message], lab_fail=lab_fail,
1181         infra_fail=infra_fail)
1182     self.assertEquals(set(suspects), results)
1183
1184   @unittest.skipIf(not KERNEL_AVAILABLE, 'Full checkout is required.')
1185   def testFailSameProject(self):
1186     """Patches to the package that failed should be marked as failing."""
1187     suspects = [self.kernel_patch]
1188     patches = suspects + [self.power_manager_patch, self.secret_patch]
1189     self._AssertSuspects(patches, suspects, [self.kernel_pkg])
1190
1191   @unittest.skipIf(not KERNEL_AVAILABLE, 'Full checkout is required.')
1192   def testFailSameProjectPlusOverlay(self):
1193     """Patches to the overlay should be marked as failing."""
1194     suspects = [self.overlay_patch, self.kernel_patch]
1195     patches = suspects + [self.power_manager_patch, self.secret_patch]
1196     self._AssertSuspects(patches, suspects, [self.kernel_pkg])
1197
1198   def testFailUnknownPackage(self):
1199     """If no patches changed the package, all patches should fail."""
1200     suspects = [self.overlay_patch, self.power_manager_patch]
1201     changes = suspects + [self.secret_patch]
1202     self._AssertSuspects(changes, suspects, [self.kernel_pkg])
1203
1204   def testFailUnknownException(self):
1205     """An unknown exception should cause all [public] patches to fail."""
1206     suspects = [self.kernel_patch, self.power_manager_patch]
1207     changes = suspects + [self.secret_patch]
1208     self._AssertSuspects(changes, suspects, exceptions=[Exception('foo bar')])
1209
1210   def testFailUnknownInternalException(self):
1211     """An unknown exception should cause all [internal] patches to fail."""
1212     suspects = [self.kernel_patch, self.power_manager_patch, self.secret_patch]
1213     self._AssertSuspects(suspects, suspects, exceptions=[Exception('foo bar')],
1214                          internal=True)
1215
1216   def testFailUnknownCombo(self):
1217     """Unknown exceptions should cause all patches to fail.
1218
1219     Even if there are also build failures that we can explain.
1220     """
1221     suspects = [self.kernel_patch, self.power_manager_patch]
1222     changes = suspects + [self.secret_patch]
1223     self._AssertSuspects(changes, suspects, [self.kernel_pkg],
1224                          [Exception('foo bar')])
1225
1226   def testFailNoExceptions(self):
1227     """If there are no exceptions, all patches should be failed."""
1228     suspects = [self.kernel_patch, self.power_manager_patch]
1229     changes = suspects + [self.secret_patch]
1230     self._AssertSuspects(changes, suspects)
1231
1232   def testLabFail(self):
1233     """If there are only lab failures, no suspect is chosen."""
1234     suspects = []
1235     changes = [self.kernel_patch, self.power_manager_patch]
1236     self._AssertSuspects(changes, suspects, lab_fail=True, infra_fail=True)
1237
1238   def testInfraFail(self):
1239     """If there are only non-lab infra faliures, pick chromite changes."""
1240     suspects = [self.chromite_patch]
1241     changes = [self.kernel_patch, self.power_manager_patch] + suspects
1242     self._AssertSuspects(changes, suspects, lab_fail=False, infra_fail=True)
1243
1244   def _GetMessages(self, lab_fail=0, infra_fail=0, other_fail=0):
1245     """Returns a list of BuildFailureMessage objects."""
1246     messages = []
1247     messages.extend(
1248         [self._GetFailedMessage([failures_lib.TestLabFailure()])
1249          for _ in range(lab_fail)])
1250     messages.extend(
1251         [self._GetFailedMessage([failures_lib.InfrastructureFailure()])
1252          for _ in range(infra_fail)])
1253     messages.extend(
1254         [self._GetFailedMessage(Exception())
1255          for _ in range(other_fail)])
1256     return messages
1257
1258   def testOnlyLabFailures(self):
1259     """Tests the OnlyLabFailures function."""
1260     messages = self._GetMessages(lab_fail=2)
1261     no_stat = []
1262     self.assertTrue(
1263         validation_pool.CalculateSuspects.OnlyLabFailures(messages, no_stat))
1264
1265     no_stat = ['foo', 'bar']
1266     # Some builders did not start. This is not a lab failure.
1267     self.assertFalse(
1268         validation_pool.CalculateSuspects.OnlyLabFailures(messages, no_stat))
1269
1270     messages = self._GetMessages(lab_fail=1, infra_fail=1)
1271     no_stat = []
1272     # Non-lab infrastructure failures are present.
1273     self.assertFalse(
1274         validation_pool.CalculateSuspects.OnlyLabFailures(messages, no_stat))
1275
1276   def testOnlyInfraFailures(self):
1277     """Tests the OnlyInfraFailures function."""
1278     messages = self._GetMessages(infra_fail=2)
1279     no_stat = []
1280     self.assertTrue(
1281         validation_pool.CalculateSuspects.OnlyInfraFailures(messages, no_stat))
1282
1283     messages = self._GetMessages(lab_fail=2)
1284     no_stat = []
1285     # Lab failures are infrastructure failures.
1286     self.assertTrue(
1287         validation_pool.CalculateSuspects.OnlyInfraFailures(messages, no_stat))
1288
1289     no_stat = ['orange']
1290     messages = []
1291     # 'Builders failed to report statuses' belong to infrastructure failures.
1292     self.assertTrue(
1293         validation_pool.CalculateSuspects.OnlyInfraFailures(messages, no_stat))
1294
1295   def testSkipInnocentOverlayPatches(self):
1296     """Test that we don't blame innocent overlay patches."""
1297     changes = self.GetPatches(4)
1298     overlay_dir = os.path.join(constants.SOURCE_ROOT, 'src/overlays')
1299     m = mock.MagicMock()
1300     self.PatchObject(cros_patch.GitRepoPatch, 'GetCheckout', return_value=m)
1301     self.PatchObject(m, 'GetPath', return_value=overlay_dir)
1302     self.PatchObject(changes[0], 'GetDiffStatus',
1303         return_value={'overlay-x86-generic/make.conf': 'M'})
1304     self.PatchObject(changes[1], 'GetDiffStatus',
1305         return_value={'make.conf': 'M'})
1306     self.PatchObject(changes[2], 'GetDiffStatus',
1307         return_value={'overlay-daisy/make.conf': 'M'})
1308     self.PatchObject(changes[3], 'GetDiffStatus',
1309         return_value={'overlay-daisy_spring/make.conf': 'M'})
1310
1311     self._AssertSuspects(changes, changes[1:], [self.kernel_pkg])
1312
1313
1314 class TestCLStatus(MoxBase):
1315   """Tests methods that get the CL status."""
1316
1317   def testPrintLinks(self):
1318     changes = self.GetPatches(3)
1319     with parallel_unittest.ParallelMock():
1320       validation_pool.ValidationPool.PrintLinksToChanges(changes)
1321
1322   def testStatusCache(self):
1323     validation_pool.ValidationPool._CL_STATUS_CACHE = {}
1324     changes = self.GetPatches(3)
1325     with parallel_unittest.ParallelMock():
1326       validation_pool.ValidationPool.FillCLStatusCache(validation_pool.CQ,
1327                                                        changes)
1328       self.assertEqual(len(validation_pool.ValidationPool._CL_STATUS_CACHE), 12)
1329       validation_pool.ValidationPool.PrintLinksToChanges(changes)
1330       self.assertEqual(len(validation_pool.ValidationPool._CL_STATUS_CACHE), 12)
1331
1332
1333 class TestCreateValidationFailureMessage(Base):
1334   """Tests validation_pool.ValidationPool._CreateValidationFailureMessage"""
1335
1336   def _AssertMessage(self, change, suspects, messages, sanity=True,
1337                      infra_fail=False, lab_fail=False, no_stat=None):
1338     """Call the _CreateValidationFailureMessage method.
1339
1340     Args:
1341       change: The change we are commenting on.
1342       suspects: List of suspected changes.
1343       messages: List of messages should appear in the failure message.
1344       sanity: Bool indicating sanity of build, default: True.
1345       infra_fail: True if build failed due to infrastructure issues.
1346       lab_fail: True if build failed due to lab infrastructure issues.
1347       no_stat: List of builders that did not start.
1348     """
1349     msg = validation_pool.ValidationPool._CreateValidationFailureMessage(
1350       False, change, set(suspects), [], sanity=sanity,
1351       infra_fail=infra_fail, lab_fail=lab_fail, no_stat=no_stat)
1352     for x in messages:
1353       self.assertTrue(x in msg)
1354     return msg
1355
1356   def testSuspectChange(self):
1357     """Test case where 1 is the only change and is suspect."""
1358     patch = self.GetPatches(1)
1359     self._AssertMessage(patch, [patch], ['probably caused by your change'])
1360
1361   def testInnocentChange(self):
1362     """Test case where 1 is innocent."""
1363     patch1, patch2 = self.GetPatches(2)
1364     self._AssertMessage(patch1, [patch2],
1365                         ['This failure was probably caused by',
1366                          'retry your change automatically'])
1367
1368   def testSuspectChanges(self):
1369     """Test case where 1 is suspected, but so is 2."""
1370     patches = self.GetPatches(2)
1371     self._AssertMessage(patches[0], patches,
1372                         ['may have caused this failure'])
1373
1374   def testInnocentChangeWithMultipleSuspects(self):
1375     """Test case where 2 and 3 are suspected."""
1376     patches = self.GetPatches(3)
1377     self._AssertMessage(patches[0], patches[1:],
1378                         ['One of the following changes is probably'])
1379
1380   def testNoMessages(self):
1381     """Test case where there are no messages."""
1382     patch1 = self.GetPatches(1)
1383     self._AssertMessage(patch1, [patch1], [])
1384
1385   def testInsaneBuild(self):
1386     """Test case where the build was not sane."""
1387     patches = self.GetPatches(3)
1388     self._AssertMessage(
1389         patches[0], patches, ['The build was consider not sane',
1390                               'retry your change automatically'],
1391         sanity=False)
1392
1393   def testLabFailMessage(self):
1394     """Test case where the build failed due to lab failures."""
1395     patches = self.GetPatches(3)
1396     self._AssertMessage(
1397         patches[0], patches, ['Lab infrastructure',
1398                               'retry your change automatically'],
1399         lab_fail=True)
1400
1401   def testInfraFailMessage(self):
1402     """Test case where the build failed due to infrastructure failures."""
1403     patches = self.GetPatches(2)
1404     self._AssertMessage(
1405         patches[0], [patches[0]],
1406         ['may have been caused by infrastructure',
1407          'This failure was probably caused by your change'],
1408         infra_fail=True)
1409     self._AssertMessage(
1410         patches[1], [patches[0]], ['may have been caused by infrastructure',
1411                                    'retry your change automatically'],
1412         infra_fail=True)
1413
1414
1415 class TestCreateDisjointTransactions(Base):
1416   """Test the CreateDisjointTransactions function."""
1417
1418   def setUp(self):
1419     self.patch_mock = self.StartPatcher(MockPatchSeries())
1420
1421   def GetPatches(self, how_many, **kwargs):
1422     return Base.GetPatches(self, how_many, always_use_list=True, **kwargs)
1423
1424   def verifyTransactions(self, txns, max_txn_length=None, circular=False):
1425     """Verify the specified list of transactions are processed correctly.
1426
1427     Args:
1428       txns: List of transactions to process.
1429       max_txn_length: Maximum length of any given transaction. This is passed
1430         to the CreateDisjointTransactions function.
1431       circular: Whether the transactions contain circular dependencies.
1432     """
1433     remove = self.PatchObject(gerrit.GerritHelper, 'RemoveCommitReady')
1434     patches = list(itertools.chain.from_iterable(txns))
1435     expected_plans = txns
1436     if max_txn_length is not None:
1437       # When max_txn_length is specified, transactions should be truncated to
1438       # the specified length, ignoring any remaining patches.
1439       expected_plans = [txn[:max_txn_length] for txn in txns]
1440
1441     pool = MakePool(changes=patches)
1442     plans = pool.CreateDisjointTransactions(None, max_txn_length=max_txn_length)
1443
1444     # If the dependencies are circular, the order of the patches is not
1445     # guaranteed, so compare them in sorted order.
1446     if circular:
1447       plans = [sorted(plan) for plan in plans]
1448       expected_plans = [sorted(plan) for plan in expected_plans]
1449
1450     # Verify the plans match, and that no changes were rejected.
1451     self.assertEqual(set(map(str, plans)), set(map(str, expected_plans)))
1452     self.assertEqual(0, remove.call_count)
1453
1454   def testPlans(self, max_txn_length=None):
1455     """Verify that independent sets are distinguished."""
1456     for num in range(0, 5):
1457       txns = [self.GetPatches(num) for _ in range(0, num)]
1458       self.verifyTransactions(txns, max_txn_length=max_txn_length)
1459
1460   def runUnresolvedPlan(self, changes, max_txn_length=None):
1461     """Helper for testing unresolved plans."""
1462     notify = self.PatchObject(validation_pool.ValidationPool,
1463                               'SendNotification')
1464     remove = self.PatchObject(gerrit.GerritHelper, 'RemoveCommitReady')
1465     pool = MakePool(changes=changes)
1466     plans = pool.CreateDisjointTransactions(None, max_txn_length=max_txn_length)
1467     self.assertEqual(plans, [])
1468     self.assertEqual(remove.call_count, notify.call_count)
1469     return remove.call_count
1470
1471   def testUnresolvedPlan(self):
1472     """Test plan with old approval_timestamp."""
1473     changes = self.GetPatches(5)[1:]
1474     with cros_test_lib.LoggingCapturer():
1475       call_count = self.runUnresolvedPlan(changes)
1476     self.assertEqual(4, call_count)
1477
1478   def testRecentUnresolvedPlan(self):
1479     """Test plan with recent approval_timestamp."""
1480     changes = self.GetPatches(5)[1:]
1481     for change in changes:
1482       change.approval_timestamp = time.time()
1483     with cros_test_lib.LoggingCapturer():
1484       call_count = self.runUnresolvedPlan(changes)
1485     self.assertEqual(0, call_count)
1486
1487   def testTruncatedPlan(self):
1488     """Test that plans can be truncated correctly."""
1489     # Long lists of patches should be truncated, and we should not see any
1490     # errors when this happens.
1491     self.testPlans(max_txn_length=3)
1492
1493   def testCircularPlans(self):
1494     """Verify that circular plans are handled correctly."""
1495     patches = self.GetPatches(5)
1496     self.patch_mock.SetGerritDependencies(patches[0], [patches[-1]])
1497
1498     # Verify that all patches can be submitted normally.
1499     self.verifyTransactions([patches], circular=True)
1500
1501     # It is not possible to truncate a circular plan. Verify that an error
1502     # is reported in this case.
1503     with cros_test_lib.LoggingCapturer():
1504       call_count = self.runUnresolvedPlan(patches, max_txn_length=3)
1505     self.assertEqual(5, call_count)
1506
1507
1508 class MockValidationPool(partial_mock.PartialMock):
1509   """Mock out a ValidationPool instance."""
1510
1511   TARGET = 'chromite.cbuildbot.validation_pool.ValidationPool'
1512   ATTRS = ('ReloadChanges', 'RemoveCommitReady', '_SubmitChange',
1513            'SendNotification')
1514
1515   def __init__(self):
1516     partial_mock.PartialMock.__init__(self)
1517     self.submit_results = {}
1518     self.max_submits = None
1519
1520   def GetSubmittedChanges(self):
1521     calls = []
1522     for call in self.patched['_SubmitChange'].call_args_list:
1523       call_args, _ = call
1524       calls.append(call_args[1])
1525     return calls
1526
1527   def _SubmitChange(self, _inst, change):
1528     result = self.submit_results.get(change, True)
1529     if isinstance(result, Exception):
1530       raise result
1531     if result and self.max_submits is not None:
1532       if self.max_submits <= 0:
1533         return False
1534       self.max_submits -= 1
1535     return result
1536
1537   @classmethod
1538   def ReloadChanges(cls, changes):
1539     return changes
1540
1541   RemoveCommitReady = None
1542   SendNotification = None
1543
1544
1545 class BaseSubmitPoolTestCase(Base, cros_build_lib_unittest.RunCommandTestCase):
1546   """Test full ability to submit and reject CL pools."""
1547
1548   def setUp(self):
1549     self.pool_mock = self.StartPatcher(MockValidationPool())
1550     self.patch_mock = self.StartPatcher(MockPatchSeries())
1551     self.PatchObject(gerrit.GerritHelper, 'QuerySingleRecord')
1552     self.patches = self.GetPatches(2)
1553
1554     # By default, don't ignore any errors.
1555     self.ignores = dict((patch, []) for patch in self.patches)
1556
1557   def SetUpPatchPool(self, failed_to_apply=False):
1558     pool = MakePool(changes=self.patches, dryrun=False)
1559     if failed_to_apply:
1560       # Create some phony errors and add them to the pool.
1561       errors = []
1562       for patch in self.GetPatches(2):
1563         errors.append(validation_pool.InternalCQError(patch, str('foo')))
1564       pool.changes_that_failed_to_apply_earlier = errors[:]
1565     return pool
1566
1567   def GetTracebacks(self):
1568     return []
1569
1570   def SubmitPool(self, submitted=(), rejected=(), **kwargs):
1571     """Helper function for testing that we can submit a pool successfully.
1572
1573     Args:
1574       submitted: List of changes that we expect to be submitted.
1575       rejected: List of changes that we expect to be rejected.
1576       **kwargs: Keyword arguments for SetUpPatchPool.
1577     """
1578     # self.ignores maps changes to a list of stages to ignore. Use it.
1579     self.PatchObject(
1580         validation_pool, 'GetStagesToIgnoreForChange',
1581         side_effect=lambda _, change: self.ignores[change])
1582
1583     # Set up our pool and submit the patches.
1584     pool = self.SetUpPatchPool(**kwargs)
1585     tracebacks = self.GetTracebacks()
1586     if tracebacks:
1587       actually_rejected = sorted(pool.SubmitPartialPool(self.GetTracebacks()))
1588     else:
1589       actually_rejected = pool.SubmitChanges(self.patches)
1590
1591     # Check that the right patches were submitted and rejected.
1592     self.assertItemsEqual(list(rejected), list(actually_rejected))
1593     actually_submitted = self.pool_mock.GetSubmittedChanges()
1594     self.assertEqual(list(submitted), actually_submitted)
1595
1596
1597 class SubmitPoolTest(BaseSubmitPoolTestCase):
1598   """Test suite related to the Submit Pool."""
1599
1600   def GetNotifyArg(self, change, key):
1601     """Look up a call to notify about |change| and grab |key| from it.
1602
1603     Args:
1604       change: The change to look up.
1605       key: The key to look up. If this is an integer, look up a positional
1606         argument by index. Otherwise, look up a keyword argument.
1607     """
1608     names = []
1609     for call in self.pool_mock.patched['SendNotification'].call_args_list:
1610       call_args, call_kwargs = call
1611       if change == call_args[1]:
1612         if isinstance(key, int):
1613           return call_args[key]
1614         return call_kwargs[key]
1615       names.append(call_args[1])
1616
1617     # Verify that |change| is present at all. This should always fail.
1618     self.assertIn(change, names)
1619
1620   def assertEqualNotifyArg(self, value, change, idx):
1621     """Verify that |value| equals self.GetNotifyArg(|change|, |idx|)."""
1622     self.assertEqual(str(value), str(self.GetNotifyArg(change, idx)))
1623
1624   def testSubmitPool(self):
1625     """Test that we can submit a pool successfully."""
1626     self.SubmitPool(submitted=self.patches)
1627
1628   def testRejectCLs(self):
1629     """Test that we can reject a CL successfully."""
1630     self.SubmitPool(submitted=self.patches, failed_to_apply=True)
1631
1632   def testSubmitCycle(self):
1633     """Submit a cyclic set of dependencies"""
1634     self.patch_mock.SetCQDependencies(self.patches[0], [self.patches[1]])
1635     self.SubmitPool(submitted=self.patches)
1636
1637   def testSubmitReverseCycle(self):
1638     """Submit a cyclic set of dependencies, specified in reverse order."""
1639     self.patch_mock.SetCQDependencies(self.patches[1], [self.patches[0]])
1640     self.patch_mock.SetGerritDependencies(self.patches[1], [])
1641     self.patch_mock.SetGerritDependencies(self.patches[0], [self.patches[1]])
1642     self.SubmitPool(submitted=self.patches[::-1])
1643
1644   def testRedundantCQDepend(self):
1645     """Submit a cycle with redundant CQ-DEPEND specifications."""
1646     self.patches = self.GetPatches(4)
1647     self.patch_mock.SetCQDependencies(self.patches[0], [self.patches[-1]])
1648     self.patch_mock.SetCQDependencies(self.patches[1], [self.patches[-1]])
1649     self.SubmitPool(submitted=self.patches)
1650
1651   def testSubmitPartialCycle(self):
1652     """Submit a failed cyclic set of dependencies"""
1653     self.pool_mock.max_submits = 1
1654     self.patch_mock.SetCQDependencies(self.patches[0], [self.patches[1]])
1655     self.SubmitPool(submitted=self.patches, rejected=[self.patches[1]])
1656     (submitted, rejected) = self.pool_mock.GetSubmittedChanges()
1657     failed_submit = validation_pool.PatchFailedToSubmit(
1658         rejected, validation_pool.ValidationPool.INCONSISTENT_SUBMIT_MSG)
1659     bad_submit = validation_pool.PatchSubmittedWithoutDeps(
1660         submitted, failed_submit)
1661     self.assertEqualNotifyArg(failed_submit, rejected, 'error')
1662     self.assertEqualNotifyArg(bad_submit, submitted, 'failure')
1663
1664   def testSubmitFailedCycle(self):
1665     """Submit a failed cyclic set of dependencies"""
1666     self.pool_mock.max_submits = 0
1667     self.patch_mock.SetCQDependencies(self.patches[0], [self.patches[1]])
1668     self.SubmitPool(submitted=[self.patches[0]], rejected=self.patches)
1669     (attempted,) = self.pool_mock.GetSubmittedChanges()
1670     (rejected,) = [x for x in self.patches if x != attempted]
1671     failed_submit = validation_pool.PatchFailedToSubmit(
1672         attempted, validation_pool.ValidationPool.INCONSISTENT_SUBMIT_MSG)
1673     dep_failed = cros_patch.DependencyError(rejected, failed_submit)
1674     self.assertEqualNotifyArg(failed_submit, attempted, 'error')
1675     self.assertEqualNotifyArg(dep_failed, rejected, 'error')
1676
1677   def testConflict(self):
1678     """Submit a change that conflicts with TOT."""
1679     error = gob_util.GOBError(httplib.CONFLICT, 'Conflict')
1680     self.pool_mock.submit_results[self.patches[0]] = error
1681     self.SubmitPool(submitted=[self.patches[0]], rejected=self.patches[::-1])
1682     notify_error = validation_pool.PatchConflict(self.patches[0])
1683     self.assertEqualNotifyArg(notify_error, self.patches[0], 'error')
1684
1685   def testServerError(self):
1686     """Test case where GOB returns a server error."""
1687     error = gerrit.GerritException('Internal server error')
1688     self.pool_mock.submit_results[self.patches[0]] = error
1689     self.SubmitPool(submitted=[self.patches[0]], rejected=self.patches[::-1])
1690     notify_error = validation_pool.PatchFailedToSubmit(self.patches[0], error)
1691     self.assertEqualNotifyArg(notify_error, self.patches[0], 'error')
1692
1693   def testNotCommitReady(self):
1694     """Test that a CL is rejected if its approvals were pulled."""
1695     def _ReloadPatches(patches):
1696       reloaded = copy.deepcopy(patches)
1697       self.PatchObject(reloaded[1], 'HasApproval', return_value=False)
1698       return reloaded
1699     self.PatchObject(validation_pool.ValidationPool, 'ReloadChanges',
1700                      side_effect=_ReloadPatches)
1701     self.SubmitPool(submitted=self.patches[:1], rejected=self.patches[1:])
1702
1703   def testAlreadyMerged(self):
1704     """Test that a CL that was chumped during the run was not rejected."""
1705     self.PatchObject(self.patches[0], 'IsAlreadyMerged', return_value=True)
1706     self.SubmitPool(submitted=self.patches[1:], rejected=[])
1707
1708   def testModified(self):
1709     """Test that a CL that was modified during the run is rejected."""
1710     def _ReloadPatches(patches):
1711       reloaded = copy.deepcopy(patches)
1712       reloaded[1].patch_number += 1
1713       return reloaded
1714     self.PatchObject(validation_pool.ValidationPool, 'ReloadChanges',
1715                      side_effect=_ReloadPatches)
1716     self.SubmitPool(submitted=self.patches[:1], rejected=self.patches[1:])
1717
1718
1719 class SubmitPartialPoolTest(BaseSubmitPoolTestCase):
1720   """Test the SubmitPartialPool function."""
1721
1722   def setUp(self):
1723     # Set up each patch to be in its own project, so that we can easily
1724     # request to ignore failures for the specified patch.
1725     for patch in self.patches:
1726       patch.project = str(patch)
1727
1728     self.stage_name = 'MyHWTest'
1729
1730   def GetTracebacks(self):
1731     """Return a list containing a single traceback."""
1732     traceback = results_lib.RecordedTraceback(
1733         self.stage_name, self.stage_name, Exception(), '')
1734     return [traceback]
1735
1736   def IgnoreFailures(self, patch):
1737     """Set us up to ignore failures for the specified |patch|."""
1738     self.ignores[patch] = [self.stage_name]
1739
1740   def testSubmitNone(self):
1741     """Submit no changes."""
1742     self.SubmitPool(submitted=(), rejected=self.patches)
1743
1744   def testSubmitAll(self):
1745     """Submit all changes."""
1746     self.IgnoreFailures(self.patches[0])
1747     self.IgnoreFailures(self.patches[1])
1748     self.SubmitPool(submitted=self.patches, rejected=[])
1749
1750   def testSubmitFirst(self):
1751     """Submit the first change in a series."""
1752     self.IgnoreFailures(self.patches[0])
1753     self.SubmitPool(submitted=[self.patches[0]], rejected=[self.patches[1]])
1754
1755   def testSubmitSecond(self):
1756     """Attempt to submit the second change in a series."""
1757     self.IgnoreFailures(self.patches[1])
1758     self.SubmitPool(submitted=[], rejected=[self.patches[0]])
1759
1760
1761 class LoadManifestTest(cros_test_lib.TempDirTestCase):
1762   """Tests loading the manifest."""
1763
1764   manifest_content = (
1765       '<?xml version="1.0" ?><manifest>'
1766       '<pending_commit branch="master" '
1767       'change_id="Ieeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee1" '
1768       'commit="1ddddddddddddddddddddddddddddddddddddddd" '
1769       'fail_count="2" gerrit_number="17000" owner_email="foo@chromium.org" '
1770       'pass_count="0" patch_number="2" project="chromiumos/taco/bar" '
1771       'project_url="https://base_url/chromiumos/taco/bar" '
1772       'ref="refs/changes/51/17000/2" remote="cros" total_fail_count="3"/>'
1773       '</manifest>')
1774
1775   def setUp(self):
1776     """Sets up a pool."""
1777     self.pool = MakePool()
1778
1779   def testAddPendingCommitsIntoPool(self):
1780     """Test reading the pending commits and add them into the pool."""
1781     with tempfile.NamedTemporaryFile() as f:
1782       f.write(self.manifest_content)
1783       f.flush()
1784       self.pool.AddPendingCommitsIntoPool(f.name)
1785
1786     self.assertEqual(self.pool.changes[0].owner_email, 'foo@chromium.org')
1787     self.assertEqual(self.pool.changes[0].tracking_branch, 'master')
1788     self.assertEqual(self.pool.changes[0].remote, 'cros')
1789     self.assertEqual(self.pool.changes[0].gerrit_number, '17000')
1790     self.assertEqual(self.pool.changes[0].project, 'chromiumos/taco/bar')
1791     self.assertEqual(self.pool.changes[0].project_url,
1792                      'https://base_url/chromiumos/taco/bar')
1793     self.assertEqual(self.pool.changes[0].change_id,
1794                      'Ieeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee1')
1795     self.assertEqual(self.pool.changes[0].commit,
1796                      '1ddddddddddddddddddddddddddddddddddddddd')
1797     self.assertEqual(self.pool.changes[0].fail_count, 2)
1798     self.assertEqual(self.pool.changes[0].pass_count, 0)
1799     self.assertEqual(self.pool.changes[0].total_fail_count, 3)
1800
1801
1802 if __name__ == '__main__':
1803   cros_test_lib.main()