src/third_party/chromite/cbuildbot/validation_pool_unittest.py

   1 #!/usr/bin/python
   2 # Copyright (c) 2011-2012 The Chromium OS Authors. All rights reserved.
   3 # Use of this source code is governed by a BSD-style license that can be
   4 # found in the LICENSE file.
   5
   6 """Module that contains unittests for validation_pool module."""
   7
   8 from __future__ import print_function
   9
  10 import ConfigParser
  11 import contextlib
  12 import copy
  13 import functools
  14 import httplib
  15 import itertools
  16 import mox
  17 import os
  18 import pickle
  19 import sys
  20 import tempfile
  21 import time
  22 import unittest
  23
  24 import constants
  25 sys.path.insert(0, constants.SOURCE_ROOT)
  26
  27 from chromite.cbuildbot import failures_lib
  28 from chromite.cbuildbot import results_lib
  29 from chromite.cbuildbot import metadata_lib
  30 from chromite.cbuildbot import repository
  31 from chromite.cbuildbot import tree_status
  32 from chromite.cbuildbot import validation_pool
  33 from chromite.lib import cros_build_lib
  34 from chromite.lib import cros_build_lib_unittest
  35 from chromite.lib import cros_test_lib
  36 from chromite.lib import gerrit
  37 from chromite.lib import gob_util
  38 from chromite.lib import gs
  39 from chromite.lib import osutils
  40 from chromite.lib import parallel
  41 from chromite.lib import parallel_unittest
  42 from chromite.lib import partial_mock
  43 from chromite.lib import patch as cros_patch
  44 from chromite.lib import patch_unittest
  45
  46
  47 import mock
  48
  49
  50 _GetNumber = iter(itertools.count()).next
  51
  52 # Some tests require the kernel, and fail with buildtools only repo.
  53 KERNEL_AVAILABLE = os.path.exists(os.path.join(
  54     constants.SOURCE_ROOT, 'src', 'third_party', 'kernel'))
  55
  56 def GetTestJson(change_id=None):
  57   """Get usable fake Gerrit patch json data
  58
  59   Args:
  60     change_id: If given, force this ChangeId
  61   """
  62   data = copy.deepcopy(patch_unittest.FAKE_PATCH_JSON)
  63   if change_id is not None:
  64     data['id'] = str(change_id)
  65   return data
  66
  67
  68 class MockManifest(object):
  69   """Helper class for Mocking Manifest objects."""
  70
  71   def __init__(self, path, **kwargs):
  72     self.root = path
  73     for key, attr in kwargs.iteritems():
  74       setattr(self, key, attr)
  75
  76
  77 # pylint: disable=W0212,R0904
  78 class Base(cros_test_lib.MockTestCase):
  79   """Test case base class with helpers for other test suites."""
  80
  81   def setUp(self):
  82     self.manager = parallel.Manager()
  83     self.patch_mock = None
  84     self._patch_counter = (itertools.count(1)).next
  85     self.build_root = 'fakebuildroot'
  86     self.PatchObject(gob_util, 'CreateHttpConn',
  87                      side_effect=AssertionError('Test should not contact GoB'))
  88     self.PatchObject(tree_status, 'IsTreeOpen', return_value=True)
  89     self.PatchObject(tree_status, 'WaitForTreeStatus',
  90                      return_value=constants.TREE_OPEN)
  91
  92   def MockPatch(self, change_id=None, patch_number=None, is_merged=False,
  93                 project='chromiumos/chromite', remote=constants.EXTERNAL_REMOTE,
  94                 tracking_branch='refs/heads/master', is_draft=False,
  95                 approvals=()):
  96     """Helper function to create mock GerritPatch objects."""
  97     if change_id is None:
  98       change_id = self._patch_counter()
  99     gerrit_number = str(change_id)
 100     change_id = hex(change_id)[2:].rstrip('L').lower()
 101     change_id = 'I%s' % change_id.rjust(40, '0')
 102     sha1 = hex(_GetNumber())[2:].rstrip('L').lower().rjust(40, '0')
 103     patch_number = (patch_number if patch_number is not None else _GetNumber())
 104     fake_url = 'http://foo/bar'
 105     if not approvals:
 106       approvals = [{'type': 'VRIF', 'value': '1', 'grantedOn': 1391733002},
 107                    {'type': 'CRVW', 'value': '2', 'grantedOn': 1391733002},
 108                    {'type': 'COMR', 'value': '1', 'grantedOn': 1391733002},]
 109
 110     current_patch_set = {
 111       'number': patch_number,
 112       'revision': sha1,
 113       'draft': is_draft,
 114       'approvals': approvals,
 115     }
 116     patch_dict = {
 117       'currentPatchSet': current_patch_set,
 118       'id': change_id,
 119       'number': gerrit_number,
 120       'project': project,
 121       'branch': tracking_branch,
 122       'owner': {'email': 'elmer.fudd@chromium.org'},
 123       'remote': remote,
 124       'status': 'MERGED' if is_merged else 'NEW',
 125       'url': '%s/%s' % (fake_url, change_id),
 126     }
 127
 128     patch = cros_patch.GerritPatch(patch_dict, remote, fake_url)
 129     patch.pass_count = 0
 130     patch.fail_count = 1
 131     patch.total_fail_count = 3
 132     return patch
 133
 134   def GetPatches(self, how_many=1, always_use_list=False, **kwargs):
 135     """Get a sequential list of patches.
 136
 137     Args:
 138       how_many: How many patches to return.
 139       always_use_list: Whether to use a list for a single item list.
 140       **kwargs: Keyword arguments for self.MockPatch.
 141     """
 142     patches = [self.MockPatch(**kwargs) for _ in xrange(how_many)]
 143     if self.patch_mock:
 144       for i, patch in enumerate(patches):
 145         self.patch_mock.SetGerritDependencies(patch, patches[:i + 1])
 146     if how_many == 1 and not always_use_list:
 147       return patches[0]
 148     return patches
 149
 150
 151 class MoxBase(Base, cros_test_lib.MoxTestCase):
 152   """Base class for other test suites with numbers mocks patched in."""
 153
 154   def setUp(self):
 155     self.mox.StubOutWithMock(validation_pool, '_RunCommand')
 156     # Suppress all gerrit access; having this occur is generally a sign
 157     # the code is either misbehaving, or that the tests are bad.
 158     self.mox.StubOutWithMock(gerrit.GerritHelper, 'Query')
 159     self.PatchObject(gs.GSContext, 'Cat', side_effect=gs.GSNoSuchKey())
 160     self.PatchObject(gs.GSContext, 'Copy')
 161     self.PatchObject(gs.GSContext, 'Exists', return_value=False)
 162     self.PatchObject(gs.GSCounter, 'Increment')
 163
 164   def MakeHelper(self, cros_internal=None, cros=None):
 165     # pylint: disable=W0201
 166     if cros_internal:
 167       cros_internal = self.mox.CreateMock(gerrit.GerritHelper)
 168       cros_internal.version = '2.2'
 169       cros_internal.remote = constants.INTERNAL_REMOTE
 170     if cros:
 171       cros = self.mox.CreateMock(gerrit.GerritHelper)
 172       cros.remote = constants.EXTERNAL_REMOTE
 173       cros.version = '2.2'
 174     return validation_pool.HelperPool(cros_internal=cros_internal,
 175                                       cros=cros)
 176
 177
 178 class IgnoredStagesTest(Base):
 179   """Tests for functions that calculate what stages to ignore."""
 180
 181   def GetOption(self, path, section='GENERAL', option='ignored-stages'):
 182     return validation_pool._GetOptionFromConfigFile(path, section, option)
 183
 184   def testBadConfigFile(self):
 185     """Test if we can handle an incorrectly formatted config file."""
 186     with osutils.TempDir(set_global=True) as tempdir:
 187       path = os.path.join(tempdir, 'foo.ini')
 188       osutils.WriteFile(path, 'foobar')
 189       self.assertRaises(ConfigParser.Error, self.GetOption, path)
 190
 191   def testMissingConfigFile(self):
 192     """Test if we can handle a missing config file."""
 193     with osutils.TempDir(set_global=True) as tempdir:
 194       path = os.path.join(tempdir, 'foo.ini')
 195       self.assertEqual(None, self.GetOption(path))
 196
 197   def testGoodConfigFile(self):
 198     """Test if we can handle a good config file."""
 199     with osutils.TempDir(set_global=True) as tempdir:
 200       path = os.path.join(tempdir, 'foo.ini')
 201       osutils.WriteFile(path, '[GENERAL]\nignored-stages: bar baz\n')
 202       ignored = self.GetOption(path)
 203       self.assertEqual('bar baz', ignored)
 204
 205
 206 class TestPatchSeries(MoxBase):
 207   """Tests resolution and applying logic of validation_pool.ValidationPool."""
 208
 209   @contextlib.contextmanager
 210   def _ValidateTransactionCall(self, _changes):
 211     yield
 212
 213   def GetPatchSeries(self, helper_pool=None):
 214     if helper_pool is None:
 215       helper_pool = self.MakeHelper(cros_internal=True, cros=True)
 216     series = validation_pool.PatchSeries(self.build_root, helper_pool)
 217
 218     # Suppress transactions.
 219     series._Transaction = self._ValidateTransactionCall
 220     series.GetGitRepoForChange = \
 221         lambda change, **kwargs: os.path.join(self.build_root, change.project)
 222
 223     return series
 224
 225   def assertPath(self, _patch, return_value, path):
 226     self.assertEqual(path, os.path.join(self.build_root, _patch.project))
 227     if isinstance(return_value, Exception):
 228       raise return_value
 229     return return_value
 230
 231   def SetPatchDeps(self, patch, parents=(), cq=()):
 232     """Set the dependencies of |patch|.
 233
 234     Args:
 235       patch: The patch to process.
 236       parents: A set of strings to set as parents of |patch|.
 237       cq: A set of strings to set as paladin dependencies of |patch|.
 238     """
 239     patch.GerritDependencies = (
 240         lambda: [cros_patch.ParsePatchDep(x) for x in parents])
 241     patch.PaladinDependencies = functools.partial(
 242         self.assertPath, patch, [cros_patch.ParsePatchDep(x) for x in cq])
 243     patch.Fetch = functools.partial(
 244         self.assertPath, patch, patch.sha1)
 245
 246   def _ValidatePatchApplyManifest(self, value):
 247     self.assertTrue(isinstance(value, MockManifest))
 248     self.assertEqual(value.root, self.build_root)
 249     return True
 250
 251   def SetPatchApply(self, patch, trivial=False):
 252     self.mox.StubOutWithMock(patch, 'ApplyAgainstManifest')
 253     return patch.ApplyAgainstManifest(
 254         mox.Func(self._ValidatePatchApplyManifest),
 255         trivial=trivial)
 256
 257   def assertResults(self, series, changes, applied=(), failed_tot=(),
 258                     failed_inflight=(), frozen=True):
 259     manifest = MockManifest(self.build_root)
 260     result = series.Apply(changes, frozen=frozen, manifest=manifest)
 261
 262     _GetIds = lambda seq:[x.id for x in seq]
 263     _GetFailedIds = lambda seq: _GetIds(x.patch for x in seq)
 264
 265     applied_result = _GetIds(result[0])
 266     failed_tot_result, failed_inflight_result = map(_GetFailedIds, result[1:])
 267
 268     applied = _GetIds(applied)
 269     failed_tot = _GetIds(failed_tot)
 270     failed_inflight = _GetIds(failed_inflight)
 271
 272     self.maxDiff = None
 273     self.assertEqual(applied, applied_result)
 274     self.assertItemsEqual(failed_inflight, failed_inflight_result)
 275     self.assertItemsEqual(failed_tot, failed_tot_result)
 276     return result
 277
 278   def testApplyWithDeps(self):
 279     """Test that we can apply changes correctly and respect deps.
 280
 281     This tests a simple out-of-order change where change1 depends on change2
 282     but tries to get applied before change2.  What should happen is that
 283     we should notice change2 is a dep of change1 and apply it first.
 284     """
 285     series = self.GetPatchSeries()
 286
 287     patch1, patch2 = patches = self.GetPatches(2)
 288
 289     self.SetPatchDeps(patch2)
 290     self.SetPatchDeps(patch1, [patch2.id])
 291
 292     self.SetPatchApply(patch2)
 293     self.SetPatchApply(patch1)
 294
 295     self.mox.ReplayAll()
 296     self.assertResults(series, patches, [patch2, patch1])
 297     self.mox.VerifyAll()
 298
 299   def testSha1Deps(self):
 300     """Test that we can apply changes correctly and respect sha1 deps.
 301
 302     This tests a simple out-of-order change where change1 depends on change2
 303     but tries to get applied before change2.  What should happen is that
 304     we should notice change2 is a dep of change1 and apply it first.
 305     """
 306     series = self.GetPatchSeries()
 307
 308     patch1, patch2, patch3 = patches = self.GetPatches(3)
 309     patch2.change_id = patch2.id = patch2.sha1
 310     patch3.change_id = patch3.id = '*' + patch3.sha1
 311     patch3.remote = constants.INTERNAL_REMOTE
 312
 313     self.SetPatchDeps(patch1, [patch2.sha1])
 314     self.SetPatchDeps(patch2, ['*%s' % patch3.sha1])
 315     self.SetPatchDeps(patch3)
 316
 317     self.SetPatchApply(patch2)
 318     self.SetPatchApply(patch3)
 319     self.SetPatchApply(patch1)
 320
 321     self.mox.ReplayAll()
 322     self.assertResults(series, patches, [patch3, patch2, patch1])
 323     self.mox.VerifyAll()
 324
 325   def testGerritNumberDeps(self):
 326     """Test that we can apply changes correctly and respect gerrit number deps.
 327
 328     This tests a simple out-of-order change where change1 depends on change2
 329     but tries to get applied before change2.  What should happen is that
 330     we should notice change2 is a dep of change1 and apply it first.
 331     """
 332     series = self.GetPatchSeries()
 333
 334     patch1, patch2, patch3 = patches = self.GetPatches(3)
 335
 336     self.SetPatchDeps(patch3, cq=[patch1.gerrit_number])
 337     self.SetPatchDeps(patch2, cq=[patch3.gerrit_number])
 338     self.SetPatchDeps(patch1, cq=[patch2.id])
 339
 340     self.SetPatchApply(patch3)
 341     self.SetPatchApply(patch2)
 342     self.SetPatchApply(patch1)
 343
 344     self.mox.ReplayAll()
 345     self.assertResults(series, patches, patches)
 346     self.mox.VerifyAll()
 347
 348   def testGerritLazyMapping(self):
 349     """Given a patch lacking a gerrit number, via gerrit, map it to that change.
 350
 351     Literally, this ensures that local patches pushed up- lacking a gerrit
 352     number- are mapped back to a changeid via asking gerrit for that number,
 353     then the local matching patch is used if available.
 354     """
 355     series = self.GetPatchSeries()
 356
 357     patch1 = self.MockPatch()
 358     self.PatchObject(patch1, 'LookupAliases', return_value=[patch1.id])
 359     patch2 = self.MockPatch(change_id=int(patch1.change_id[1:]))
 360     patch3 = self.MockPatch()
 361
 362     self.SetPatchDeps(patch3, cq=[patch2.gerrit_number])
 363     self.SetPatchDeps(patch2)
 364     self.SetPatchDeps(patch1)
 365
 366     self.SetPatchApply(patch1)
 367     self.SetPatchApply(patch3)
 368
 369     self._SetQuery(series, patch2, query=patch2.gerrit_number).AndReturn(patch2)
 370
 371     self.mox.ReplayAll()
 372     applied = self.assertResults(series, [patch1, patch3], [patch3, patch1])[0]
 373     self.assertTrue(applied[0] is patch3)
 374     self.assertTrue(applied[1] is patch1)
 375     self.mox.VerifyAll()
 376
 377   def testCrosGerritDeps(self, cros_internal=True):
 378     """Test that we can apply changes correctly and respect deps.
 379
 380     This tests a simple out-of-order change where change1 depends on change3
 381     but tries to get applied before change2.  What should happen is that
 382     we should notice change2 is a dep of change1 and apply it first.
 383     """
 384     helper_pool = self.MakeHelper(cros_internal=cros_internal, cros=True)
 385     series = self.GetPatchSeries(helper_pool=helper_pool)
 386
 387     patch1 = self.MockPatch(remote=constants.EXTERNAL_REMOTE)
 388     patch2 = self.MockPatch(remote=constants.INTERNAL_REMOTE)
 389     patch3 = self.MockPatch(remote=constants.EXTERNAL_REMOTE)
 390     patches = [patch1, patch2, patch3]
 391     if cros_internal:
 392       applied_patches = [patch3, patch1, patch2]
 393     else:
 394       applied_patches = [patch3, patch1]
 395
 396     self.SetPatchDeps(patch1, [patch3.id])
 397     self.SetPatchDeps(patch2)
 398     self.SetPatchDeps(patch3, cq=[patch2.id])
 399
 400     if cros_internal:
 401       self.SetPatchApply(patch2)
 402     self.SetPatchApply(patch1)
 403     self.SetPatchApply(patch3)
 404
 405     self.mox.ReplayAll()
 406     self.assertResults(series, patches, applied_patches)
 407     self.mox.VerifyAll()
 408
 409   def testExternalCrosGerritDeps(self):
 410     """Test that we exclude internal deps on external trybot."""
 411     self.testCrosGerritDeps(cros_internal=False)
 412
 413   @staticmethod
 414   def _SetQuery(series, change, query=None):
 415     helper = series._helper_pool.GetHelper(change.remote)
 416     query = change.id if query is None else query
 417     return helper.QuerySingleRecord(query, must_match=True)
 418
 419   def testApplyMissingDep(self):
 420     """Test that we don't try to apply a change without met dependencies.
 421
 422     Patch2 is in the validation pool that depends on Patch1 (which is not)
 423     Nothing should get applied.
 424     """
 425     series = self.GetPatchSeries()
 426
 427     patch1, patch2 = self.GetPatches(2)
 428
 429     self.SetPatchDeps(patch2, [patch1.id])
 430     self._SetQuery(series, patch1).AndReturn(patch1)
 431
 432     self.mox.ReplayAll()
 433     self.assertResults(series, [patch2],
 434                        [], [patch2])
 435     self.mox.VerifyAll()
 436
 437   def testApplyWithCommittedDeps(self):
 438     """Test that we apply a change with dependency already committed."""
 439     series = self.GetPatchSeries()
 440
 441     # Use for basic commit check.
 442     patch1 = self.GetPatches(1, is_merged=True)
 443     patch2 = self.GetPatches(1)
 444
 445     self.SetPatchDeps(patch2, [patch1.id])
 446     self._SetQuery(series, patch1).AndReturn(patch1)
 447     self.SetPatchApply(patch2)
 448
 449     # Used to ensure that an uncommitted change put in the lookup cache
 450     # isn't invalidly pulled into the graph...
 451     patch3, patch4, patch5 = self.GetPatches(3)
 452
 453     self._SetQuery(series, patch3).AndReturn(patch3)
 454     self.SetPatchDeps(patch4, [patch3.id])
 455     self.SetPatchDeps(patch5, [patch3.id])
 456
 457     self.mox.ReplayAll()
 458     self.assertResults(series, [patch2, patch4, patch5], [patch2],
 459                        [patch4, patch5])
 460     self.mox.VerifyAll()
 461
 462   def testCyclicalDeps(self):
 463     """Verify that the machinery handles cycles correctly."""
 464     series = self.GetPatchSeries()
 465
 466     patch1, patch2, patch3 = patches = self.GetPatches(3)
 467
 468     self.SetPatchDeps(patch1, [patch2.id])
 469     self.SetPatchDeps(patch2, cq=[patch3.id])
 470     self.SetPatchDeps(patch3, [patch1.id])
 471
 472     self.SetPatchApply(patch1)
 473     self.SetPatchApply(patch2)
 474     self.SetPatchApply(patch3)
 475
 476     self.mox.ReplayAll()
 477     self.assertResults(series, patches, [patch2, patch1, patch3])
 478     self.mox.VerifyAll()
 479
 480   def testComplexCyclicalDeps(self, fail=False):
 481     """Verify handling of two interdependent cycles."""
 482     series = self.GetPatchSeries()
 483
 484     # Create two cyclically interdependent patch chains.
 485     # Example: Two patch series A1<-A2<-A3<-A4 and B1<-B2<-B3<-B4. A1 has a
 486     # CQ-DEPEND on B4 and B1 has a CQ-DEPEND on A4, so all of the patches must
 487     # be committed together.
 488     chain1, chain2 = chains = self.GetPatches(4), self.GetPatches(4)
 489     for chain in chains:
 490       (other_chain,) = [x for x in chains if x != chain]
 491       self.SetPatchDeps(chain[0], [], cq=[other_chain[-1].id])
 492       for i in range(1, len(chain)):
 493         self.SetPatchDeps(chain[i], [chain[i-1].id])
 494
 495     # Apply the second-last patch first, so that the last patch in the series
 496     # will be pulled in via the CQ-DEPEND on the other patch chain.
 497     to_apply = [chain1[-2]] + [x for x in (chain1 + chain2) if x != chain1[-2]]
 498
 499     # All of the patches but chain[-1] were applied successfully.
 500     for patch in chain1[:-1] + chain2:
 501       self.SetPatchApply(patch)
 502
 503     if fail:
 504       # Pretend that chain[-1] failed to apply.
 505       res = self.SetPatchApply(chain1[-1])
 506       res.AndRaise(cros_patch.ApplyPatchException(chain1[-1]))
 507       applied = []
 508       failed_tot = to_apply
 509     else:
 510       # We apply the patches in this order since the last patch in chain1
 511       # is pulled in via CQ-DEPEND.
 512       self.SetPatchApply(chain1[-1])
 513       applied = chain1[:-1] + chain2 + [chain1[-1]]
 514       failed_tot = []
 515
 516     self.mox.ReplayAll()
 517     self.assertResults(series, to_apply, applied=applied, failed_tot=failed_tot)
 518     self.mox.VerifyAll()
 519
 520   def testFailingComplexCyclicalDeps(self):
 521     """Verify handling of failing interlocked cycles."""
 522     self.testComplexCyclicalDeps(fail=True)
 523
 524   def testApplyPartialFailures(self):
 525     """Test that can apply changes correctly when one change fails to apply.
 526
 527     This tests a simple change order where 1 depends on 2 and 1 fails to apply.
 528     Only 1 should get tried as 2 will abort once it sees that 1 can't be
 529     applied.  3 with no dependencies should go through fine.
 530
 531     Since patch1 fails to apply, we should also get a call to handle the
 532     failure.
 533     """
 534     series = self.GetPatchSeries()
 535
 536     patch1, patch2, patch3, patch4 = patches = self.GetPatches(4)
 537
 538     self.SetPatchDeps(patch1)
 539     self.SetPatchDeps(patch2, [patch1.id])
 540     self.SetPatchDeps(patch3)
 541     self.SetPatchDeps(patch4)
 542
 543     self.SetPatchApply(patch1).AndRaise(
 544         cros_patch.ApplyPatchException(patch1))
 545
 546     self.SetPatchApply(patch3)
 547     self.SetPatchApply(patch4).AndRaise(
 548         cros_patch.ApplyPatchException(patch1, inflight=True))
 549
 550     self.mox.ReplayAll()
 551     self.assertResults(series, patches,
 552                        [patch3], [patch2, patch1], [patch4])
 553     self.mox.VerifyAll()
 554
 555   def testComplexApply(self):
 556     """More complex deps test.
 557
 558     This tests a total of 2 change chains where the first change we see
 559     only has a partial chain with the 3rd change having the whole chain i.e.
 560     1->2, 3->1->2.  Since we get these in the order 1,2,3,4,5 the order we
 561     should apply is 2,1,3,4,5.
 562
 563     This test also checks the patch order to verify that Apply re-orders
 564     correctly based on the chain.
 565     """
 566     series = self.GetPatchSeries()
 567
 568     patch1, patch2, patch3, patch4, patch5 = patches = self.GetPatches(5)
 569
 570     self.SetPatchDeps(patch1, [patch2.id])
 571     self.SetPatchDeps(patch2)
 572     self.SetPatchDeps(patch3, [patch1.id, patch2.id])
 573     self.SetPatchDeps(patch4, cq=[patch5.id])
 574     self.SetPatchDeps(patch5)
 575
 576     for patch in (patch2, patch1, patch3, patch4, patch5):
 577       self.SetPatchApply(patch)
 578
 579     self.mox.ReplayAll()
 580     self.assertResults(
 581         series, patches, [patch2, patch1, patch3, patch4, patch5])
 582     self.mox.VerifyAll()
 583
 584   def testApplyStandalonePatches(self):
 585     """Simple apply of two changes with no dependent CL's."""
 586     series = self.GetPatchSeries()
 587
 588     patches = self.GetPatches(3)
 589
 590     for patch in patches:
 591       self.SetPatchDeps(patch)
 592
 593     for patch in patches:
 594       self.SetPatchApply(patch)
 595
 596     self.mox.ReplayAll()
 597     self.assertResults(series, patches, patches)
 598     self.mox.VerifyAll()
 599
 600
 601 def MakePool(overlays=constants.PUBLIC_OVERLAYS, build_number=1,
 602              builder_name='foon', is_master=True, dryrun=True, **kwargs):
 603   """Helper for creating ValidationPool objects for tests."""
 604   kwargs.setdefault('changes', [])
 605   build_root = kwargs.pop('build_root', '/fake_root')
 606
 607   pool = validation_pool.ValidationPool(
 608       overlays, build_root, build_number, builder_name, is_master,
 609       dryrun, **kwargs)
 610   return pool
 611
 612
 613 class MockPatchSeries(partial_mock.PartialMock):
 614   """Mock the PatchSeries functions."""
 615   TARGET = 'chromite.cbuildbot.validation_pool.PatchSeries'
 616   ATTRS = ('GetDepsForChange', '_GetGerritPatch', '_LookupHelper')
 617
 618   def __init__(self):
 619     partial_mock.PartialMock.__init__(self)
 620     self.deps = {}
 621     self.cq_deps = {}
 622
 623   def SetGerritDependencies(self, patch, deps):
 624     """Add |deps| to the Gerrit dependencies of |patch|."""
 625     self.deps[patch] = deps
 626
 627   def SetCQDependencies(self, patch, deps):
 628     """Add |deps| to the CQ dependencies of |patch|."""
 629     self.cq_deps[patch] = deps
 630
 631   def GetDepsForChange(self, _inst, patch):
 632     return self.deps.get(patch, []), self.cq_deps.get(patch, [])
 633
 634   def _GetGerritPatch(self, _inst, dep, **_kwargs):
 635     return dep
 636
 637   _LookupHelper = mock.MagicMock()
 638
 639
 640 class TestSubmitChange(MoxBase):
 641   """Test suite related to submitting changes."""
 642
 643   def setUp(self):
 644     self.orig_timeout = validation_pool.SUBMITTED_WAIT_TIMEOUT
 645     validation_pool.SUBMITTED_WAIT_TIMEOUT = 4
 646
 647   def tearDown(self):
 648     validation_pool.SUBMITTED_WAIT_TIMEOUT = self.orig_timeout
 649
 650   def _TestSubmitChange(self, results, build_id=31337):
 651     """Test submitting a change with the given results."""
 652     results = [cros_test_lib.EasyAttr(status=r) for r in results]
 653     change = self.MockPatch(change_id=12345, patch_number=1)
 654     pool = self.mox.CreateMock(validation_pool.ValidationPool)
 655     pool.dryrun = False
 656     pool._metadata = metadata_lib.CBuildbotMetadata()
 657     pool._metadata.UpdateWithDict({'build_id': build_id})
 658     pool._helper_pool = self.mox.CreateMock(validation_pool.HelperPool)
 659     helper = self.mox.CreateMock(validation_pool.gerrit.GerritHelper)
 660
 661     self.mox.StubOutWithMock(validation_pool.ValidationPool,
 662                              '_InsertCLActionToDatabase')
 663
 664     # Prepare replay script.
 665     pool._helper_pool.ForChange(change).AndReturn(helper)
 666     helper.SubmitChange(change, dryrun=False)
 667     validation_pool.ValidationPool._InsertCLActionToDatabase(build_id, change,
 668                                                              mox.IgnoreArg())
 669     for result in results:
 670       helper.QuerySingleRecord(change.gerrit_number).AndReturn(result)
 671     self.mox.ReplayAll()
 672
 673     # Verify results.
 674     retval = validation_pool.ValidationPool._SubmitChange(pool, change)
 675     self.mox.VerifyAll()
 676     return retval
 677
 678   def testSubmitChangeMerged(self):
 679     """Submit one change to gerrit, status MERGED."""
 680     self.assertTrue(self._TestSubmitChange(['MERGED']))
 681
 682   def testSubmitChangeSubmitted(self):
 683     """Submit one change to gerrit, stuck on SUBMITTED."""
 684     # The query will be retried 1 more time than query timeout.
 685     results = ['SUBMITTED' for _i in
 686                xrange(validation_pool.SUBMITTED_WAIT_TIMEOUT + 1)]
 687     self.assertTrue(self._TestSubmitChange(results))
 688
 689   def testSubmitChangeSubmittedToMerged(self):
 690     """Submit one change to gerrit, status SUBMITTED then MERGED."""
 691     results = ['SUBMITTED', 'SUBMITTED', 'MERGED']
 692     self.assertTrue(self._TestSubmitChange(results))
 693
 694   def testSubmitChangeFailed(self):
 695     """Submit one change to gerrit, reported back as NEW."""
 696     self.assertFalse(self._TestSubmitChange(['NEW']))
 697
 698
 699 class ValidationFailureOrTimeout(MoxBase):
 700   """Tests that HandleValidationFailure and HandleValidationTimeout functions.
 701
 702   These tests check that HandleValidationTimeout and HandleValidationFailure
 703   reject (i.e. zero out the CQ field) of the correct number of patches, under
 704   various circumstances.
 705   """
 706
 707   _PATCH_MESSAGE = 'Your patch failed.'
 708   _BUILD_MESSAGE = 'Your build failed.'
 709
 710   def setUp(self):
 711     self._patches = self.GetPatches(3)
 712     self._pool = MakePool(changes=self._patches)
 713
 714     self.PatchObject(
 715         validation_pool.ValidationPool, 'GetCLStatus',
 716         return_value=validation_pool.ValidationPool.STATUS_PASSED)
 717     self.PatchObject(
 718         validation_pool.CalculateSuspects, 'FindSuspects',
 719         return_value=self._patches)
 720     self.PatchObject(
 721         validation_pool.ValidationPool, '_CreateValidationFailureMessage',
 722         return_value=self._PATCH_MESSAGE)
 723     self.PatchObject(validation_pool.ValidationPool, 'SendNotification')
 724     self.PatchObject(validation_pool.ValidationPool, 'RemoveCommitReady')
 725     self.PatchObject(validation_pool.ValidationPool, 'UpdateCLStatus')
 726     self.PatchObject(validation_pool.ValidationPool, 'ReloadChanges',
 727                      return_value=self._patches)
 728     self.PatchObject(validation_pool.CalculateSuspects, 'OnlyLabFailures',
 729                      return_value=False)
 730     self.PatchObject(validation_pool.CalculateSuspects, 'OnlyInfraFailures',
 731                      return_value=False)
 732     self.StartPatcher(parallel_unittest.ParallelMock())
 733
 734   def testPatchesWereRejectedByFailure(self):
 735     """Tests that all patches are rejected by failure."""
 736     self._pool.HandleValidationFailure([self._BUILD_MESSAGE])
 737     self.assertEqual(
 738         len(self._patches), self._pool.RemoveCommitReady.call_count)
 739
 740   def testPatchesWereRejectedByTimeout(self):
 741     self._pool.HandleValidationTimeout()
 742     self.assertEqual(
 743         len(self._patches), self._pool.RemoveCommitReady.call_count)
 744
 745   def testNoSuspectsWithFailure(self):
 746     """Tests no change is blamed when there is no suspect."""
 747     self.PatchObject(validation_pool.CalculateSuspects, 'FindSuspects',
 748                      return_value=[])
 749     self._pool.HandleValidationFailure([self._BUILD_MESSAGE])
 750     self.assertEqual(0, self._pool.RemoveCommitReady.call_count)
 751
 752   def testPreCQ(self):
 753     self._pool.pre_cq = True
 754     self._pool.HandleValidationFailure([self._BUILD_MESSAGE])
 755     self.assertEqual(0, self._pool.RemoveCommitReady.call_count)
 756
 757   def testPatchesWereNotRejectedByInsaneFailure(self):
 758     self._pool.HandleValidationFailure([self._BUILD_MESSAGE], sanity=False)
 759     self.assertEqual(0, self._pool.RemoveCommitReady.call_count)
 760
 761
 762 class TestCoreLogic(MoxBase):
 763   """Tests resolution and applying logic of validation_pool.ValidationPool."""
 764
 765   def setUp(self):
 766     self.mox.StubOutWithMock(validation_pool.PatchSeries, 'Apply')
 767     self.mox.StubOutWithMock(validation_pool.PatchSeries, 'ApplyChange')
 768     self.patch_mock = self.StartPatcher(MockPatchSeries())
 769     funcs = ['SendNotification', '_SubmitChange']
 770     for func in funcs:
 771       self.mox.StubOutWithMock(validation_pool.ValidationPool, func)
 772     self.PatchObject(validation_pool.ValidationPool, 'ReloadChanges',
 773                      side_effect=lambda x: x)
 774     self.StartPatcher(parallel_unittest.ParallelMock())
 775
 776   def MakePool(self, *args, **kwargs):
 777     """Helper for creating ValidationPool objects for Mox tests."""
 778     handlers = kwargs.pop('handlers', False)
 779     kwargs['build_root'] = self.build_root
 780     pool = MakePool(*args, **kwargs)
 781     funcs = ['_HandleApplySuccess', '_HandleApplyFailure',
 782              '_HandleCouldNotApply', '_HandleCouldNotSubmit']
 783     if handlers:
 784       for func in funcs:
 785         self.mox.StubOutWithMock(pool, func)
 786     return pool
 787
 788   def MakeFailure(self, patch, inflight=True):
 789     return cros_patch.ApplyPatchException(patch, inflight=inflight)
 790
 791   def GetPool(self, changes, applied=(), tot=(), inflight=(), **kwargs):
 792     pool = self.MakePool(changes=changes, **kwargs)
 793     applied = list(applied)
 794     tot = [self.MakeFailure(x, inflight=False) for x in tot]
 795     inflight = [self.MakeFailure(x, inflight=True) for x in inflight]
 796     # pylint: disable=E1120,E1123
 797     validation_pool.PatchSeries.Apply(
 798         changes, manifest=mox.IgnoreArg()
 799         ).AndReturn((applied, tot, inflight))
 800
 801     for patch in applied:
 802       pool._HandleApplySuccess(patch).AndReturn(None)
 803
 804     if tot:
 805       pool._HandleApplyFailure(tot).AndReturn(None)
 806
 807     # We stash this on the pool object so we can reuse it during validation.
 808     # We could stash this in the test instances, but that would break
 809     # for any tests that do multiple pool instances.
 810
 811     pool._test_data = (changes, applied, tot, inflight)
 812
 813     return pool
 814
 815   def testApplySlavePool(self):
 816     """Verifies that slave calls ApplyChange() directly for each patch."""
 817     slave_pool = self.MakePool(is_master=False)
 818     patches = self.GetPatches(3)
 819     slave_pool.changes = patches
 820     for patch in patches:
 821       # pylint: disable=E1120, E1123
 822       validation_pool.PatchSeries.ApplyChange(patch, manifest=mox.IgnoreArg())
 823
 824     self.mox.ReplayAll()
 825     self.assertEqual(True, slave_pool.ApplyPoolIntoRepo())
 826     self.mox.VerifyAll()
 827
 828   def runApply(self, pool, result):
 829     self.assertEqual(result, pool.ApplyPoolIntoRepo())
 830     self.assertEqual(pool.changes, pool._test_data[1])
 831     failed_inflight = pool.changes_that_failed_to_apply_earlier
 832     expected_inflight = set(pool._test_data[3])
 833     # Intersect the results, since it's possible there were results failed
 834     # results that weren't related to the ApplyPoolIntoRepo call.
 835     self.assertEqual(set(failed_inflight).intersection(expected_inflight),
 836                      expected_inflight)
 837
 838     self.assertEqual(pool.changes, pool._test_data[1])
 839
 840   def testPatchSeriesInteraction(self):
 841     """Verify the interaction between PatchSeries and ValidationPool.
 842
 843     Effectively, this validates data going into PatchSeries, and coming back
 844     out; verifies the hand off to _Handle* functions, but no deeper.
 845     """
 846     patches = self.GetPatches(3)
 847
 848     apply_pool = self.GetPool(patches, applied=patches, handlers=True)
 849     all_inflight = self.GetPool(patches, inflight=patches, handlers=True)
 850     all_tot = self.GetPool(patches, tot=patches, handlers=True)
 851     mixed = self.GetPool(patches, tot=patches[0:1], inflight=patches[1:2],
 852                          applied=patches[2:3], handlers=True)
 853
 854     self.mox.ReplayAll()
 855     self.runApply(apply_pool, True)
 856     self.runApply(all_inflight, False)
 857     self.runApply(all_tot, False)
 858     self.runApply(mixed, True)
 859     self.mox.VerifyAll()
 860
 861   def testHandleApplySuccess(self):
 862     """Validate steps taken for successfull application."""
 863     patch = self.GetPatches(1)
 864     pool = self.MakePool()
 865     pool.SendNotification(patch, mox.StrContains('has picked up your change'))
 866     self.mox.ReplayAll()
 867     pool._HandleApplySuccess(patch)
 868     self.mox.VerifyAll()
 869
 870   def testHandleApplyFailure(self):
 871     failures = [cros_patch.ApplyPatchException(x) for x in self.GetPatches(4)]
 872
 873     notified_patches = failures[:2]
 874     unnotified_patches = failures[2:]
 875     master_pool = self.MakePool(dryrun=False)
 876     slave_pool = self.MakePool(is_master=False)
 877
 878     self.mox.StubOutWithMock(gerrit.GerritHelper, 'RemoveCommitReady')
 879
 880     for failure in notified_patches:
 881       master_pool.SendNotification(
 882           failure.patch,
 883           mox.StrContains('failed to apply your change'),
 884           failure=mox.IgnoreArg())
 885       # This pylint suppressin shouldn't be necessary, but pylint is invalidly
 886       # thinking that the first arg isn't passed in; we suppress it to suppress
 887       # the pylnt bug.
 888       # pylint: disable=E1120
 889       gerrit.GerritHelper.RemoveCommitReady(failure.patch, dryrun=False)
 890
 891     self.mox.ReplayAll()
 892     master_pool._HandleApplyFailure(notified_patches)
 893     slave_pool._HandleApplyFailure(unnotified_patches)
 894     self.mox.VerifyAll()
 895
 896   def _setUpSubmit(self):
 897     pool = self.MakePool(dryrun=False, handlers=True)
 898     patches = self.GetPatches(3)
 899     failed = self.GetPatches(3)
 900     pool.changes = patches[:]
 901     # While we don't do anything w/ these patches, that's
 902     # intentional; we're verifying that it isn't submitted
 903     # if there is a failure.
 904     pool.changes_that_failed_to_apply_earlier = failed[:]
 905
 906     return (pool, patches, failed)
 907
 908   def testSubmitPoolFailures(self):
 909     """Tests that a fatal exception is raised."""
 910     pool, patches, _failed = self._setUpSubmit()
 911     patch1, patch2, patch3 = patches
 912
 913     pool._SubmitChange(patch1).AndReturn(True)
 914     pool._SubmitChange(patch2).AndReturn(False)
 915
 916     pool._HandleCouldNotSubmit(patch2, mox.IgnoreArg()).InAnyOrder()
 917     pool._HandleCouldNotSubmit(patch3, mox.IgnoreArg()).InAnyOrder()
 918
 919     self.mox.ReplayAll()
 920     self.assertRaises(validation_pool.FailedToSubmitAllChangesException,
 921                       pool.SubmitPool)
 922     self.mox.VerifyAll()
 923
 924   def testSubmitPartialPass(self):
 925     """Tests that a non-fatal exception is raised."""
 926     pool, patches, _failed = self._setUpSubmit()
 927     patch1, patch2, patch3 = patches
 928     # Make patch2 not commit-ready.
 929     patch2._approvals = []
 930
 931     pool._SubmitChange(patch1).AndReturn(True)
 932
 933     pool._HandleCouldNotSubmit(patch2, mox.IgnoreArg()).InAnyOrder()
 934     pool._HandleCouldNotSubmit(patch3, mox.IgnoreArg()).InAnyOrder()
 935
 936     self.mox.ReplayAll()
 937     self.assertRaises(validation_pool.FailedToSubmitAllChangesNonFatalException,
 938                       pool.SubmitPool)
 939     self.mox.VerifyAll()
 940
 941   def testSubmitPool(self):
 942     """Tests that we can submit a pool of patches."""
 943     pool, patches, failed = self._setUpSubmit()
 944
 945     for patch in patches:
 946       pool._SubmitChange(patch).AndReturn(True)
 947
 948     pool._HandleApplyFailure(failed)
 949
 950     self.mox.ReplayAll()
 951     pool.SubmitPool()
 952     self.mox.VerifyAll()
 953
 954   def testSubmitNonManifestChanges(self):
 955     """Simple test to make sure we can submit non-manifest changes."""
 956     pool, patches, _failed = self._setUpSubmit()
 957     pool.non_manifest_changes = patches[:]
 958
 959     for patch in patches:
 960       pool._SubmitChange(patch).AndReturn(True)
 961
 962     self.mox.ReplayAll()
 963     pool.SubmitNonManifestChanges()
 964     self.mox.VerifyAll()
 965
 966   def testUnhandledExceptions(self):
 967     """Test that CQ doesn't loop due to unhandled Exceptions."""
 968     pool, patches, _failed = self._setUpSubmit()
 969
 970     class MyException(Exception):
 971       """"Unique Exception used for testing."""
 972
 973     def VerifyCQError(patch, error):
 974       cq_error = validation_pool.InternalCQError(patch, error.message)
 975       return str(error) == str(cq_error)
 976
 977     # pylint: disable=E1120,E1123
 978     validation_pool.PatchSeries.Apply(
 979         patches, manifest=mox.IgnoreArg()).AndRaise(MyException)
 980     errors = [mox.Func(functools.partial(VerifyCQError, x)) for x in patches]
 981     pool._HandleApplyFailure(errors).AndReturn(None)
 982
 983     self.mox.ReplayAll()
 984     self.assertRaises(MyException, pool.ApplyPoolIntoRepo)
 985     self.mox.VerifyAll()
 986
 987   def testFilterDependencyErrors(self):
 988     """Verify that dependency errors are correctly filtered out."""
 989     failures = [cros_patch.ApplyPatchException(x) for x in self.GetPatches(2)]
 990     failures += [cros_patch.DependencyError(x, y) for x, y in
 991                  zip(self.GetPatches(2), failures)]
 992     failures[0].patch.approval_timestamp = time.time()
 993     failures[-1].patch.approval_timestamp = time.time()
 994     self.mox.ReplayAll()
 995     result = validation_pool.ValidationPool._FilterDependencyErrors(failures)
 996     self.assertEquals(set(failures[:-1]), set(result))
 997     self.mox.VerifyAll()
 998
 999   def testFilterNonCrosProjects(self):
1000     """Runs through a filter of own manifest and fake changes.
1001
1002     This test should filter out the tacos/chromite project as its not real.
1003     """
1004     base_func = itertools.cycle(['chromiumos', 'chromeos']).next
1005     patches = self.GetPatches(8)
1006     for patch in patches:
1007       patch.project = '%s/%i' % (base_func(), _GetNumber())
1008       patch.tracking_branch = str(_GetNumber())
1009
1010     non_cros_patches = self.GetPatches(2)
1011     for patch in non_cros_patches:
1012       patch.project = str(_GetNumber())
1013
1014     filtered_patches = patches[:4]
1015     allowed_patches = []
1016     projects = {}
1017     for idx, patch in enumerate(patches[4:]):
1018       fails = bool(idx % 2)
1019       # Vary the revision so we can validate that it checks the branch.
1020       revision = ('monkeys' if fails
1021                   else 'refs/heads/%s' % patch.tracking_branch)
1022       if fails:
1023         filtered_patches.append(patch)
1024       else:
1025         allowed_patches.append(patch)
1026       projects.setdefault(patch.project, {})['revision'] = revision
1027
1028     manifest = MockManifest(self.build_root, projects=projects)
1029     for patch in allowed_patches:
1030       patch.GetCheckout = lambda *_args, **_kwargs: True
1031     for patch in filtered_patches:
1032       patch.GetCheckout = lambda *_args, **_kwargs: False
1033
1034     self.mox.ReplayAll()
1035     results = validation_pool.ValidationPool._FilterNonCrosProjects(
1036         patches + non_cros_patches, manifest)
1037
1038     def compare(list1, list2):
1039       mangle = lambda c:(c.id, c.project, c.tracking_branch)
1040       self.assertEqual(list1, list2,
1041         msg="Comparison failed:\n list1: %r\n list2: %r"
1042             % (map(mangle, list1), map(mangle, list2)))
1043
1044     compare(results[0], allowed_patches)
1045     compare(results[1], filtered_patches)
1046
1047
1048 class TestPickling(cros_test_lib.TempDirTestCase):
1049   """Tests to validate pickling of ValidationPool, covering CQ's needs"""
1050
1051   def testSelfCompatibility(self):
1052     """Verify compatibility of current git HEAD against itself."""
1053     self._CheckTestData(self._GetTestData())
1054
1055   def testToTCompatibility(self):
1056     """Validate that ToT can use our pickles, and that we can use ToT's data."""
1057     repo = os.path.join(self.tempdir, 'chromite')
1058     reference = os.path.abspath(__file__)
1059     reference = os.path.normpath(os.path.join(reference, '../../'))
1060
1061     repository.CloneGitRepo(
1062         repo,
1063         '%s/chromiumos/chromite' % constants.EXTERNAL_GOB_URL,
1064         reference=reference)
1065
1066     code = """
1067 import sys
1068 from chromite.cbuildbot import validation_pool_unittest
1069 if not hasattr(validation_pool_unittest, 'TestPickling'):
1070   sys.exit(0)
1071 sys.stdout.write(validation_pool_unittest.TestPickling.%s)
1072 """
1073
1074     # Verify ToT can take our pickle.
1075     cros_build_lib.RunCommand(
1076         ['python', '-c', code % '_CheckTestData(sys.stdin.read())'],
1077         cwd=self.tempdir, print_cmd=False, capture_output=True,
1078         input=self._GetTestData())
1079
1080     # Verify we can handle ToT's pickle.
1081     ret = cros_build_lib.RunCommand(
1082         ['python', '-c', code % '_GetTestData()'],
1083         cwd=self.tempdir, print_cmd=False, capture_output=True)
1084
1085     self._CheckTestData(ret.output)
1086
1087   @staticmethod
1088   def _GetCrosInternalPatch(patch_info):
1089     return cros_patch.GerritPatch(
1090         patch_info,
1091         constants.INTERNAL_REMOTE,
1092         constants.INTERNAL_GERRIT_URL)
1093
1094   @staticmethod
1095   def _GetCrosPatch(patch_info):
1096     return cros_patch.GerritPatch(
1097         patch_info,
1098         constants.EXTERNAL_REMOTE,
1099         constants.EXTERNAL_GERRIT_URL)
1100
1101   @classmethod
1102   def _GetTestData(cls):
1103     ids = [cros_patch.MakeChangeId() for _ in xrange(3)]
1104     changes = [cls._GetCrosInternalPatch(GetTestJson(ids[0]))]
1105     non_os = [cls._GetCrosPatch(GetTestJson(ids[1]))]
1106     conflicting = [cls._GetCrosInternalPatch(GetTestJson(ids[2]))]
1107     conflicting = [cros_patch.PatchException(x)for x in conflicting]
1108     pool = validation_pool.ValidationPool(
1109         constants.PUBLIC_OVERLAYS,
1110         '/fake/pathway', 1,
1111         'testing', True, True,
1112         changes=changes, non_os_changes=non_os,
1113         conflicting_changes=conflicting)
1114     return pickle.dumps([pool, changes, non_os, conflicting])
1115
1116   @staticmethod
1117   def _CheckTestData(data):
1118     results = pickle.loads(data)
1119     pool, changes, non_os, conflicting = results
1120     def _f(source, value, getter=None):
1121       if getter is None:
1122         getter = lambda x: x
1123       assert len(source) == len(value)
1124       for s_item, v_item in zip(source, value):
1125         assert getter(s_item).id == getter(v_item).id
1126         assert getter(s_item).remote == getter(v_item).remote
1127     _f(pool.changes, changes)
1128     _f(pool.non_manifest_changes, non_os)
1129     _f(pool.changes_that_failed_to_apply_earlier, conflicting,
1130        getter=lambda s:getattr(s, 'patch', s))
1131     return ''
1132
1133
1134 class TestFindSuspects(MoxBase):
1135   """Tests validation_pool.ValidationPool._FindSuspects"""
1136
1137   def setUp(self):
1138     overlay = 'chromiumos/overlays/chromiumos-overlay'
1139     self.overlay_patch = self.GetPatches(project=overlay)
1140     chromite = 'chromiumos/chromite'
1141     self.chromite_patch = self.GetPatches(project=chromite)
1142     self.power_manager = 'chromiumos/platform2/power_manager'
1143     self.power_manager_pkg = 'chromeos-base/power_manager'
1144     self.power_manager_patch = self.GetPatches(project=self.power_manager)
1145     self.kernel = 'chromiumos/third_party/kernel'
1146     self.kernel_pkg = 'sys-kernel/chromeos-kernel'
1147     self.kernel_patch = self.GetPatches(project=self.kernel)
1148     self.secret = 'chromeos/secret'
1149     self.secret_patch = self.GetPatches(project=self.secret,
1150                                         remote=constants.INTERNAL_REMOTE)
1151     self.PatchObject(cros_patch.GitRepoPatch, 'GetCheckout')
1152     self.PatchObject(cros_patch.GitRepoPatch, 'GetDiffStatus')
1153
1154   @staticmethod
1155   def _GetBuildFailure(pkg):
1156     """Create a PackageBuildFailure for the specified |pkg|.
1157
1158     Args:
1159       pkg: Package that failed to build.
1160     """
1161     ex = cros_build_lib.RunCommandError('foo', cros_build_lib.CommandResult())
1162     return failures_lib.PackageBuildFailure(ex, 'bar', [pkg])
1163
1164   def _GetFailedMessage(self, exceptions, stage='Build', internal=False,
1165                         bot='daisy_spring-paladin'):
1166     """Returns a BuildFailureMessage object."""
1167     tracebacks = []
1168     for ex in exceptions:
1169       tracebacks.append(results_lib.RecordedTraceback('Build', 'Build', ex,
1170                                                       str(ex)))
1171     reason = 'failure reason string'
1172     return failures_lib.BuildFailureMessage(
1173         'Stage %s failed' % stage, tracebacks, internal, reason, bot)
1174
1175   def _AssertSuspects(self, patches, suspects, pkgs=(), exceptions=(),
1176                       internal=False, infra_fail=False, lab_fail=False):
1177     """Run _FindSuspects and verify its output.
1178
1179     Args:
1180       patches: List of patches to look at.
1181       suspects: Expected list of suspects returned by _FindSuspects.
1182       pkgs: List of packages that failed with exceptions in the build.
1183       exceptions: List of other exceptions that occurred during the build.
1184       internal: Whether the failures occurred on an internal bot.
1185       infra_fail: Whether the build failed due to infrastructure issues.
1186       lab_fail: Whether the build failed due to lab infrastructure issues.
1187     """
1188     all_exceptions = list(exceptions) + [self._GetBuildFailure(x) for x in pkgs]
1189     message = self._GetFailedMessage(all_exceptions, internal=internal)
1190     results = validation_pool.CalculateSuspects.FindSuspects(
1191         constants.SOURCE_ROOT, patches, [message], lab_fail=lab_fail,
1192         infra_fail=infra_fail)
1193     self.assertEquals(set(suspects), results)
1194
1195   @unittest.skipIf(not KERNEL_AVAILABLE, 'Full checkout is required.')
1196   def testFailSameProject(self):
1197     """Patches to the package that failed should be marked as failing."""
1198     suspects = [self.kernel_patch]
1199     patches = suspects + [self.power_manager_patch, self.secret_patch]
1200     self._AssertSuspects(patches, suspects, [self.kernel_pkg])
1201
1202   @unittest.skipIf(not KERNEL_AVAILABLE, 'Full checkout is required.')
1203   def testFailSameProjectPlusOverlay(self):
1204     """Patches to the overlay should be marked as failing."""
1205     suspects = [self.overlay_patch, self.kernel_patch]
1206     patches = suspects + [self.power_manager_patch, self.secret_patch]
1207     self._AssertSuspects(patches, suspects, [self.kernel_pkg])
1208
1209   def testFailUnknownPackage(self):
1210     """If no patches changed the package, all patches should fail."""
1211     suspects = [self.overlay_patch, self.power_manager_patch]
1212     changes = suspects + [self.secret_patch]
1213     self._AssertSuspects(changes, suspects, [self.kernel_pkg])
1214
1215   def testFailUnknownException(self):
1216     """An unknown exception should cause all [public] patches to fail."""
1217     suspects = [self.kernel_patch, self.power_manager_patch]
1218     changes = suspects + [self.secret_patch]
1219     self._AssertSuspects(changes, suspects, exceptions=[Exception('foo bar')])
1220
1221   def testFailUnknownInternalException(self):
1222     """An unknown exception should cause all [internal] patches to fail."""
1223     suspects = [self.kernel_patch, self.power_manager_patch, self.secret_patch]
1224     self._AssertSuspects(suspects, suspects, exceptions=[Exception('foo bar')],
1225                          internal=True)
1226
1227   def testFailUnknownCombo(self):
1228     """Unknown exceptions should cause all patches to fail.
1229
1230     Even if there are also build failures that we can explain.
1231     """
1232     suspects = [self.kernel_patch, self.power_manager_patch]
1233     changes = suspects + [self.secret_patch]
1234     self._AssertSuspects(changes, suspects, [self.kernel_pkg],
1235                          [Exception('foo bar')])
1236
1237   def testFailNoExceptions(self):
1238     """If there are no exceptions, all patches should be failed."""
1239     suspects = [self.kernel_patch, self.power_manager_patch]
1240     changes = suspects + [self.secret_patch]
1241     self._AssertSuspects(changes, suspects)
1242
1243   def testLabFail(self):
1244     """If there are only lab failures, no suspect is chosen."""
1245     suspects = []
1246     changes = [self.kernel_patch, self.power_manager_patch]
1247     self._AssertSuspects(changes, suspects, lab_fail=True, infra_fail=True)
1248
1249   def testInfraFail(self):
1250     """If there are only non-lab infra faliures, pick chromite changes."""
1251     suspects = [self.chromite_patch]
1252     changes = [self.kernel_patch, self.power_manager_patch] + suspects
1253     self._AssertSuspects(changes, suspects, lab_fail=False, infra_fail=True)
1254
1255   def testManualBlame(self):
1256     """If there are changes that were manually blamed, pick those changes."""
1257     approvals1 = [{'type': 'VRIF', 'value': '-1', 'grantedOn': 1391733002},
1258                   {'type': 'CRVW', 'value': '2', 'grantedOn': 1391733002},
1259                   {'type': 'COMR', 'value': '1', 'grantedOn': 1391733002},]
1260     approvals2 = [{'type': 'VRIF', 'value': '1', 'grantedOn': 1391733002},
1261                   {'type': 'CRVW', 'value': '-2', 'grantedOn': 1391733002},
1262                   {'type': 'COMR', 'value': '1', 'grantedOn': 1391733002},]
1263     suspects = [self.MockPatch(approvals=approvals1),
1264                 self.MockPatch(approvals=approvals2)]
1265     changes = [self.kernel_patch, self.chromite_patch] + suspects
1266     self._AssertSuspects(changes, suspects, lab_fail=False, infra_fail=False)
1267     self._AssertSuspects(changes, suspects, lab_fail=True, infra_fail=False)
1268     self._AssertSuspects(changes, suspects, lab_fail=True, infra_fail=True)
1269     self._AssertSuspects(changes, suspects, lab_fail=False, infra_fail=True)
1270
1271   def _GetMessages(self, lab_fail=0, infra_fail=0, other_fail=0):
1272     """Returns a list of BuildFailureMessage objects."""
1273     messages = []
1274     messages.extend(
1275         [self._GetFailedMessage([failures_lib.TestLabFailure()])
1276          for _ in range(lab_fail)])
1277     messages.extend(
1278         [self._GetFailedMessage([failures_lib.InfrastructureFailure()])
1279          for _ in range(infra_fail)])
1280     messages.extend(
1281         [self._GetFailedMessage(Exception())
1282          for _ in range(other_fail)])
1283     return messages
1284
1285   def testOnlyLabFailures(self):
1286     """Tests the OnlyLabFailures function."""
1287     messages = self._GetMessages(lab_fail=2)
1288     no_stat = []
1289     self.assertTrue(
1290         validation_pool.CalculateSuspects.OnlyLabFailures(messages, no_stat))
1291
1292     no_stat = ['foo', 'bar']
1293     # Some builders did not start. This is not a lab failure.
1294     self.assertFalse(
1295         validation_pool.CalculateSuspects.OnlyLabFailures(messages, no_stat))
1296
1297     messages = self._GetMessages(lab_fail=1, infra_fail=1)
1298     no_stat = []
1299     # Non-lab infrastructure failures are present.
1300     self.assertFalse(
1301         validation_pool.CalculateSuspects.OnlyLabFailures(messages, no_stat))
1302
1303   def testOnlyInfraFailures(self):
1304     """Tests the OnlyInfraFailures function."""
1305     messages = self._GetMessages(infra_fail=2)
1306     no_stat = []
1307     self.assertTrue(
1308         validation_pool.CalculateSuspects.OnlyInfraFailures(messages, no_stat))
1309
1310     messages = self._GetMessages(lab_fail=2)
1311     no_stat = []
1312     # Lab failures are infrastructure failures.
1313     self.assertTrue(
1314         validation_pool.CalculateSuspects.OnlyInfraFailures(messages, no_stat))
1315
1316     no_stat = ['orange']
1317     messages = []
1318     # 'Builders failed to report statuses' belong to infrastructure failures.
1319     self.assertTrue(
1320         validation_pool.CalculateSuspects.OnlyInfraFailures(messages, no_stat))
1321
1322   def testSkipInnocentOverlayPatches(self):
1323     """Test that we don't blame innocent overlay patches."""
1324     changes = self.GetPatches(4)
1325     overlay_dir = os.path.join(constants.SOURCE_ROOT, 'src/overlays')
1326     m = mock.MagicMock()
1327     self.PatchObject(cros_patch.GitRepoPatch, 'GetCheckout', return_value=m)
1328     self.PatchObject(m, 'GetPath', return_value=overlay_dir)
1329     self.PatchObject(changes[0], 'GetDiffStatus',
1330         return_value={'overlay-x86-generic/make.conf': 'M'})
1331     self.PatchObject(changes[1], 'GetDiffStatus',
1332         return_value={'make.conf': 'M'})
1333     self.PatchObject(changes[2], 'GetDiffStatus',
1334         return_value={'overlay-daisy/make.conf': 'M'})
1335     self.PatchObject(changes[3], 'GetDiffStatus',
1336         return_value={'overlay-daisy_spring/make.conf': 'M'})
1337
1338     self._AssertSuspects(changes, changes[1:], [self.kernel_pkg])
1339
1340
1341 class TestCLStatus(MoxBase):
1342   """Tests methods that get the CL status."""
1343
1344   def testPrintLinks(self):
1345     changes = self.GetPatches(3)
1346     with parallel_unittest.ParallelMock():
1347       validation_pool.ValidationPool.PrintLinksToChanges(changes)
1348
1349   def testStatusCache(self):
1350     validation_pool.ValidationPool._CL_STATUS_CACHE = {}
1351     changes = self.GetPatches(3)
1352     with parallel_unittest.ParallelMock():
1353       validation_pool.ValidationPool.FillCLStatusCache(validation_pool.CQ,
1354                                                        changes)
1355       self.assertEqual(len(validation_pool.ValidationPool._CL_STATUS_CACHE), 12)
1356       validation_pool.ValidationPool.PrintLinksToChanges(changes)
1357       self.assertEqual(len(validation_pool.ValidationPool._CL_STATUS_CACHE), 12)
1358
1359
1360 class TestCreateValidationFailureMessage(Base):
1361   """Tests validation_pool.ValidationPool._CreateValidationFailureMessage"""
1362
1363   def _AssertMessage(self, change, suspects, messages, sanity=True,
1364                      infra_fail=False, lab_fail=False, no_stat=None):
1365     """Call the _CreateValidationFailureMessage method.
1366
1367     Args:
1368       change: The change we are commenting on.
1369       suspects: List of suspected changes.
1370       messages: List of messages should appear in the failure message.
1371       sanity: Bool indicating sanity of build, default: True.
1372       infra_fail: True if build failed due to infrastructure issues.
1373       lab_fail: True if build failed due to lab infrastructure issues.
1374       no_stat: List of builders that did not start.
1375     """
1376     msg = validation_pool.ValidationPool._CreateValidationFailureMessage(
1377       False, change, set(suspects), [], sanity=sanity,
1378       infra_fail=infra_fail, lab_fail=lab_fail, no_stat=no_stat)
1379     for x in messages:
1380       self.assertTrue(x in msg)
1381     return msg
1382
1383   def testSuspectChange(self):
1384     """Test case where 1 is the only change and is suspect."""
1385     patch = self.GetPatches(1)
1386     self._AssertMessage(patch, [patch], ['probably caused by your change'])
1387
1388   def testInnocentChange(self):
1389     """Test case where 1 is innocent."""
1390     patch1, patch2 = self.GetPatches(2)
1391     self._AssertMessage(patch1, [patch2],
1392                         ['This failure was probably caused by',
1393                          'retry your change automatically'])
1394
1395   def testSuspectChanges(self):
1396     """Test case where 1 is suspected, but so is 2."""
1397     patches = self.GetPatches(2)
1398     self._AssertMessage(patches[0], patches,
1399                         ['may have caused this failure'])
1400
1401   def testInnocentChangeWithMultipleSuspects(self):
1402     """Test case where 2 and 3 are suspected."""
1403     patches = self.GetPatches(3)
1404     self._AssertMessage(patches[0], patches[1:],
1405                         ['One of the following changes is probably'])
1406
1407   def testNoMessages(self):
1408     """Test case where there are no messages."""
1409     patch1 = self.GetPatches(1)
1410     self._AssertMessage(patch1, [patch1], [])
1411
1412   def testInsaneBuild(self):
1413     """Test case where the build was not sane."""
1414     patches = self.GetPatches(3)
1415     self._AssertMessage(
1416         patches[0], patches, ['The build was consider not sane',
1417                               'retry your change automatically'],
1418         sanity=False)
1419
1420   def testLabFailMessage(self):
1421     """Test case where the build failed due to lab failures."""
1422     patches = self.GetPatches(3)
1423     self._AssertMessage(
1424         patches[0], patches, ['Lab infrastructure',
1425                               'retry your change automatically'],
1426         lab_fail=True)
1427
1428   def testInfraFailMessage(self):
1429     """Test case where the build failed due to infrastructure failures."""
1430     patches = self.GetPatches(2)
1431     self._AssertMessage(
1432         patches[0], [patches[0]],
1433         ['may have been caused by infrastructure',
1434          'This failure was probably caused by your change'],
1435         infra_fail=True)
1436     self._AssertMessage(
1437         patches[1], [patches[0]], ['may have been caused by infrastructure',
1438                                    'retry your change automatically'],
1439         infra_fail=True)
1440
1441
1442 class TestCreateDisjointTransactions(Base):
1443   """Test the CreateDisjointTransactions function."""
1444
1445   def setUp(self):
1446     self.patch_mock = self.StartPatcher(MockPatchSeries())
1447
1448   def GetPatches(self, how_many, **kwargs):
1449     return Base.GetPatches(self, how_many, always_use_list=True, **kwargs)
1450
1451   def verifyTransactions(self, txns, max_txn_length=None, circular=False):
1452     """Verify the specified list of transactions are processed correctly.
1453
1454     Args:
1455       txns: List of transactions to process.
1456       max_txn_length: Maximum length of any given transaction. This is passed
1457         to the CreateDisjointTransactions function.
1458       circular: Whether the transactions contain circular dependencies.
1459     """
1460     remove = self.PatchObject(gerrit.GerritHelper, 'RemoveCommitReady')
1461     patches = list(itertools.chain.from_iterable(txns))
1462     expected_plans = txns
1463     if max_txn_length is not None:
1464       # When max_txn_length is specified, transactions should be truncated to
1465       # the specified length, ignoring any remaining patches.
1466       expected_plans = [txn[:max_txn_length] for txn in txns]
1467
1468     pool = MakePool(changes=patches)
1469     plans = pool.CreateDisjointTransactions(None, max_txn_length=max_txn_length)
1470
1471     # If the dependencies are circular, the order of the patches is not
1472     # guaranteed, so compare them in sorted order.
1473     if circular:
1474       plans = [sorted(plan) for plan in plans]
1475       expected_plans = [sorted(plan) for plan in expected_plans]
1476
1477     # Verify the plans match, and that no changes were rejected.
1478     self.assertEqual(set(map(str, plans)), set(map(str, expected_plans)))
1479     self.assertEqual(0, remove.call_count)
1480
1481   def testPlans(self, max_txn_length=None):
1482     """Verify that independent sets are distinguished."""
1483     for num in range(0, 5):
1484       txns = [self.GetPatches(num) for _ in range(0, num)]
1485       self.verifyTransactions(txns, max_txn_length=max_txn_length)
1486
1487   def runUnresolvedPlan(self, changes, max_txn_length=None):
1488     """Helper for testing unresolved plans."""
1489     notify = self.PatchObject(validation_pool.ValidationPool,
1490                               'SendNotification')
1491     remove = self.PatchObject(gerrit.GerritHelper, 'RemoveCommitReady')
1492     pool = MakePool(changes=changes)
1493     plans = pool.CreateDisjointTransactions(None, max_txn_length=max_txn_length)
1494     self.assertEqual(plans, [])
1495     self.assertEqual(remove.call_count, notify.call_count)
1496     return remove.call_count
1497
1498   def testUnresolvedPlan(self):
1499     """Test plan with old approval_timestamp."""
1500     changes = self.GetPatches(5)[1:]
1501     with cros_test_lib.LoggingCapturer():
1502       call_count = self.runUnresolvedPlan(changes)
1503     self.assertEqual(4, call_count)
1504
1505   def testRecentUnresolvedPlan(self):
1506     """Test plan with recent approval_timestamp."""
1507     changes = self.GetPatches(5)[1:]
1508     for change in changes:
1509       change.approval_timestamp = time.time()
1510     with cros_test_lib.LoggingCapturer():
1511       call_count = self.runUnresolvedPlan(changes)
1512     self.assertEqual(0, call_count)
1513
1514   def testTruncatedPlan(self):
1515     """Test that plans can be truncated correctly."""
1516     # Long lists of patches should be truncated, and we should not see any
1517     # errors when this happens.
1518     self.testPlans(max_txn_length=3)
1519
1520   def testCircularPlans(self):
1521     """Verify that circular plans are handled correctly."""
1522     patches = self.GetPatches(5)
1523     self.patch_mock.SetGerritDependencies(patches[0], [patches[-1]])
1524
1525     # Verify that all patches can be submitted normally.
1526     self.verifyTransactions([patches], circular=True)
1527
1528     # It is not possible to truncate a circular plan. Verify that an error
1529     # is reported in this case.
1530     with cros_test_lib.LoggingCapturer():
1531       call_count = self.runUnresolvedPlan(patches, max_txn_length=3)
1532     self.assertEqual(5, call_count)
1533
1534
1535 class MockValidationPool(partial_mock.PartialMock):
1536   """Mock out a ValidationPool instance."""
1537
1538   TARGET = 'chromite.cbuildbot.validation_pool.ValidationPool'
1539   ATTRS = ('ReloadChanges', 'RemoveCommitReady', '_SubmitChange',
1540            'SendNotification')
1541
1542   def __init__(self, manager):
1543     partial_mock.PartialMock.__init__(self)
1544     self.submit_results = {}
1545     self.max_submits = manager.Value('i', -1)
1546     self.submitted = manager.list()
1547     self.notification_calls = manager.list()
1548
1549   def GetSubmittedChanges(self):
1550     return list(self.submitted)
1551
1552   def _SubmitChange(self, _inst, change):
1553     result = self.submit_results.get(change, True)
1554     self.submitted.append(change)
1555     if isinstance(result, Exception):
1556       raise result
1557     if result and self.max_submits.value != -1:
1558       if self.max_submits.value <= 0:
1559         return False
1560       self.max_submits.value -= 1
1561     return result
1562
1563   def SendNotification(self, *args, **kwargs):
1564     self.notification_calls.append((args, kwargs))
1565
1566   @classmethod
1567   def ReloadChanges(cls, changes):
1568     return changes
1569
1570   RemoveCommitReady = None
1571
1572
1573 class BaseSubmitPoolTestCase(Base, cros_build_lib_unittest.RunCommandTestCase):
1574   """Test full ability to submit and reject CL pools."""
1575
1576   def setUp(self):
1577     self.pool_mock = self.StartPatcher(MockValidationPool(self.manager))
1578     self.patch_mock = self.StartPatcher(MockPatchSeries())
1579     self.PatchObject(gerrit.GerritHelper, 'QuerySingleRecord')
1580     self.patches = self.GetPatches(2)
1581
1582     # By default, don't ignore any errors.
1583     self.ignores = dict((patch, []) for patch in self.patches)
1584
1585   def SetUpPatchPool(self, failed_to_apply=False):
1586     pool = MakePool(changes=self.patches, dryrun=False)
1587     if failed_to_apply:
1588       # Create some phony errors and add them to the pool.
1589       errors = []
1590       for patch in self.GetPatches(2):
1591         errors.append(validation_pool.InternalCQError(patch, str('foo')))
1592       pool.changes_that_failed_to_apply_earlier = errors[:]
1593     return pool
1594
1595   def GetTracebacks(self):
1596     return []
1597
1598   def SubmitPool(self, submitted=(), rejected=(), **kwargs):
1599     """Helper function for testing that we can submit a pool successfully.
1600
1601     Args:
1602       submitted: List of changes that we expect to be submitted.
1603       rejected: List of changes that we expect to be rejected.
1604       **kwargs: Keyword arguments for SetUpPatchPool.
1605     """
1606     # self.ignores maps changes to a list of stages to ignore. Use it.
1607     self.PatchObject(
1608         validation_pool, 'GetStagesToIgnoreForChange',
1609         side_effect=lambda _, change: self.ignores[change])
1610
1611     # Set up our pool and submit the patches.
1612     pool = self.SetUpPatchPool(**kwargs)
1613     tracebacks = self.GetTracebacks()
1614     if tracebacks:
1615       actually_rejected = sorted(pool.SubmitPartialPool(self.GetTracebacks()))
1616     else:
1617       actually_rejected = pool.SubmitChanges(self.patches)
1618
1619     # Check that the right patches were submitted and rejected.
1620     self.assertItemsEqual(list(rejected), list(actually_rejected))
1621     actually_submitted = self.pool_mock.GetSubmittedChanges()
1622     self.assertEqual(list(submitted), actually_submitted)
1623
1624
1625 class SubmitPoolTest(BaseSubmitPoolTestCase):
1626   """Test suite related to the Submit Pool."""
1627
1628   def GetNotifyArg(self, change, key):
1629     """Look up a call to notify about |change| and grab |key| from it.
1630
1631     Args:
1632       change: The change to look up.
1633       key: The key to look up. If this is an integer, look up a positional
1634         argument by index. Otherwise, look up a keyword argument.
1635     """
1636     names = []
1637     for call in self.pool_mock.notification_calls:
1638       call_args, call_kwargs = call
1639       if change == call_args[1]:
1640         if isinstance(key, int):
1641           return call_args[key]
1642         return call_kwargs[key]
1643       names.append(call_args[1])
1644
1645     # Verify that |change| is present at all. This should always fail.
1646     self.assertIn(change, names)
1647
1648   def assertEqualNotifyArg(self, value, change, idx):
1649     """Verify that |value| equals self.GetNotifyArg(|change|, |idx|)."""
1650     self.assertEqual(str(value), str(self.GetNotifyArg(change, idx)))
1651
1652   def testSubmitPool(self):
1653     """Test that we can submit a pool successfully."""
1654     self.SubmitPool(submitted=self.patches)
1655
1656   def testRejectCLs(self):
1657     """Test that we can reject a CL successfully."""
1658     self.SubmitPool(submitted=self.patches, failed_to_apply=True)
1659
1660   def testSubmitCycle(self):
1661     """Submit a cyclic set of dependencies"""
1662     self.patch_mock.SetCQDependencies(self.patches[0], [self.patches[1]])
1663     self.SubmitPool(submitted=self.patches)
1664
1665   def testSubmitReverseCycle(self):
1666     """Submit a cyclic set of dependencies, specified in reverse order."""
1667     self.patch_mock.SetCQDependencies(self.patches[1], [self.patches[0]])
1668     self.patch_mock.SetGerritDependencies(self.patches[1], [])
1669     self.patch_mock.SetGerritDependencies(self.patches[0], [self.patches[1]])
1670     self.SubmitPool(submitted=self.patches[::-1])
1671
1672   def testRedundantCQDepend(self):
1673     """Submit a cycle with redundant CQ-DEPEND specifications."""
1674     self.patches = self.GetPatches(4)
1675     self.patch_mock.SetCQDependencies(self.patches[0], [self.patches[-1]])
1676     self.patch_mock.SetCQDependencies(self.patches[1], [self.patches[-1]])
1677     self.SubmitPool(submitted=self.patches)
1678
1679   def testSubmitPartialCycle(self):
1680     """Submit a failed cyclic set of dependencies"""
1681     self.pool_mock.max_submits.value = 1
1682     self.patch_mock.SetCQDependencies(self.patches[0], [self.patches[1]])
1683     self.SubmitPool(submitted=self.patches, rejected=[self.patches[1]])
1684     (submitted, rejected) = self.pool_mock.GetSubmittedChanges()
1685     failed_submit = validation_pool.PatchFailedToSubmit(
1686         rejected, validation_pool.ValidationPool.INCONSISTENT_SUBMIT_MSG)
1687     bad_submit = validation_pool.PatchSubmittedWithoutDeps(
1688         submitted, failed_submit)
1689     self.assertEqualNotifyArg(failed_submit, rejected, 'error')
1690     self.assertEqualNotifyArg(bad_submit, submitted, 'failure')
1691
1692   def testSubmitFailedCycle(self):
1693     """Submit a failed cyclic set of dependencies"""
1694     self.pool_mock.max_submits.value = 0
1695     self.patch_mock.SetCQDependencies(self.patches[0], [self.patches[1]])
1696     self.SubmitPool(submitted=[self.patches[0]], rejected=self.patches)
1697     (attempted,) = self.pool_mock.GetSubmittedChanges()
1698     (rejected,) = [x for x in self.patches if x.id != attempted.id]
1699     failed_submit = validation_pool.PatchFailedToSubmit(
1700         attempted, validation_pool.ValidationPool.INCONSISTENT_SUBMIT_MSG)
1701     dep_failed = cros_patch.DependencyError(rejected, failed_submit)
1702     self.assertEqualNotifyArg(failed_submit, attempted, 'error')
1703     self.assertEqualNotifyArg(dep_failed, rejected, 'error')
1704
1705   def testConflict(self):
1706     """Submit a change that conflicts with TOT."""
1707     error = gob_util.GOBError(httplib.CONFLICT, 'Conflict')
1708     self.pool_mock.submit_results[self.patches[0]] = error
1709     self.SubmitPool(submitted=[self.patches[0]], rejected=self.patches[::-1])
1710     notify_error = validation_pool.PatchConflict(self.patches[0])
1711     self.assertEqualNotifyArg(notify_error, self.patches[0], 'error')
1712
1713   def testServerError(self):
1714     """Test case where GOB returns a server error."""
1715     error = gerrit.GerritException('Internal server error')
1716     self.pool_mock.submit_results[self.patches[0]] = error
1717     self.SubmitPool(submitted=[self.patches[0]], rejected=self.patches[::-1])
1718     notify_error = validation_pool.PatchFailedToSubmit(self.patches[0], error)
1719     self.assertEqualNotifyArg(notify_error, self.patches[0], 'error')
1720
1721   def testNotCommitReady(self):
1722     """Test that a CL is rejected if its approvals were pulled."""
1723     def _ReloadPatches(patches):
1724       reloaded = copy.deepcopy(patches)
1725       self.PatchObject(reloaded[1], 'HasApproval', return_value=False)
1726       return reloaded
1727     self.PatchObject(validation_pool.ValidationPool, 'ReloadChanges',
1728                      side_effect=_ReloadPatches)
1729     self.SubmitPool(submitted=self.patches[:1], rejected=self.patches[1:])
1730
1731   def testAlreadyMerged(self):
1732     """Test that a CL that was chumped during the run was not rejected."""
1733     self.PatchObject(self.patches[0], 'IsAlreadyMerged', return_value=True)
1734     self.SubmitPool(submitted=self.patches[1:], rejected=[])
1735
1736   def testModified(self):
1737     """Test that a CL that was modified during the run is rejected."""
1738     def _ReloadPatches(patches):
1739       reloaded = copy.deepcopy(patches)
1740       reloaded[1].patch_number += 1
1741       return reloaded
1742     self.PatchObject(validation_pool.ValidationPool, 'ReloadChanges',
1743                      side_effect=_ReloadPatches)
1744     self.SubmitPool(submitted=self.patches[:1], rejected=self.patches[1:])
1745
1746
1747 class SubmitPartialPoolTest(BaseSubmitPoolTestCase):
1748   """Test the SubmitPartialPool function."""
1749
1750   def setUp(self):
1751     # Set up each patch to be in its own project, so that we can easily
1752     # request to ignore failures for the specified patch.
1753     for patch in self.patches:
1754       patch.project = str(patch)
1755
1756     self.stage_name = 'MyHWTest'
1757
1758   def GetTracebacks(self):
1759     """Return a list containing a single traceback."""
1760     traceback = results_lib.RecordedTraceback(
1761         self.stage_name, self.stage_name, Exception(), '')
1762     return [traceback]
1763
1764   def IgnoreFailures(self, patch):
1765     """Set us up to ignore failures for the specified |patch|."""
1766     self.ignores[patch] = [self.stage_name]
1767
1768   def testSubmitNone(self):
1769     """Submit no changes."""
1770     self.SubmitPool(submitted=(), rejected=self.patches)
1771
1772   def testSubmitAll(self):
1773     """Submit all changes."""
1774     self.IgnoreFailures(self.patches[0])
1775     self.IgnoreFailures(self.patches[1])
1776     self.SubmitPool(submitted=self.patches, rejected=[])
1777
1778   def testSubmitFirst(self):
1779     """Submit the first change in a series."""
1780     self.IgnoreFailures(self.patches[0])
1781     self.SubmitPool(submitted=[self.patches[0]], rejected=[self.patches[1]])
1782
1783   def testSubmitSecond(self):
1784     """Attempt to submit the second change in a series."""
1785     self.IgnoreFailures(self.patches[1])
1786     self.SubmitPool(submitted=[], rejected=[self.patches[0]])
1787
1788
1789 class LoadManifestTest(cros_test_lib.TempDirTestCase):
1790   """Tests loading the manifest."""
1791
1792   manifest_content = (
1793       '<?xml version="1.0" ?><manifest>'
1794       '<pending_commit branch="master" '
1795       'change_id="Ieeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee1" '
1796       'commit="1ddddddddddddddddddddddddddddddddddddddd" '
1797       'fail_count="2" gerrit_number="17000" owner_email="foo@chromium.org" '
1798       'pass_count="0" patch_number="2" project="chromiumos/taco/bar" '
1799       'project_url="https://base_url/chromiumos/taco/bar" '
1800       'ref="refs/changes/51/17000/2" remote="cros" total_fail_count="3"/>'
1801       '</manifest>')
1802
1803   def setUp(self):
1804     """Sets up a pool."""
1805     self.pool = MakePool()
1806
1807   def testAddPendingCommitsIntoPool(self):
1808     """Test reading the pending commits and add them into the pool."""
1809     with tempfile.NamedTemporaryFile() as f:
1810       f.write(self.manifest_content)
1811       f.flush()
1812       self.pool.AddPendingCommitsIntoPool(f.name)
1813
1814     self.assertEqual(self.pool.changes[0].owner_email, 'foo@chromium.org')
1815     self.assertEqual(self.pool.changes[0].tracking_branch, 'master')
1816     self.assertEqual(self.pool.changes[0].remote, 'cros')
1817     self.assertEqual(self.pool.changes[0].gerrit_number, '17000')
1818     self.assertEqual(self.pool.changes[0].project, 'chromiumos/taco/bar')
1819     self.assertEqual(self.pool.changes[0].project_url,
1820                      'https://base_url/chromiumos/taco/bar')
1821     self.assertEqual(self.pool.changes[0].change_id,
1822                      'Ieeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee1')
1823     self.assertEqual(self.pool.changes[0].commit,
1824                      '1ddddddddddddddddddddddddddddddddddddddd')
1825     self.assertEqual(self.pool.changes[0].fail_count, 2)
1826     self.assertEqual(self.pool.changes[0].pass_count, 0)
1827     self.assertEqual(self.pool.changes[0].total_fail_count, 3)
1828
1829
1830 if __name__ == '__main__':
1831   cros_test_lib.main()