src/third_party/chromite/buildbot/validation_pool_unittest.py

   1 #!/usr/bin/python
   2
   3 # Copyright (c) 2011-2012 The Chromium OS Authors. All rights reserved.
   4 # Use of this source code is governed by a BSD-style license that can be
   5 # found in the LICENSE file.
   6
   7 """Module that contains unittests for validation_pool module."""
   8
   9 import contextlib
  10 import copy
  11 import functools
  12 import httplib
  13 import itertools
  14 import mox
  15 import os
  16 import pickle
  17 import sys
  18 import time
  19
  20 import constants
  21 sys.path.insert(0, constants.SOURCE_ROOT)
  22
  23 from chromite.buildbot import cbuildbot_results as results_lib
  24 from chromite.buildbot import cbuildbot_metadata
  25 from chromite.buildbot import repository
  26 from chromite.buildbot import validation_pool
  27 from chromite.lib import cros_build_lib
  28 from chromite.lib import cros_build_lib_unittest
  29 from chromite.lib import cros_test_lib
  30 from chromite.lib import gerrit
  31 from chromite.lib import gob_util
  32 from chromite.lib import gs
  33 from chromite.lib import osutils
  34 from chromite.lib import parallel_unittest
  35 from chromite.lib import partial_mock
  36 from chromite.lib import patch as cros_patch
  37 from chromite.lib import patch_unittest
  38 from chromite.lib import timeout_util
  39
  40
  41 import mock
  42
  43
  44 _GetNumber = iter(itertools.count()).next
  45
  46
  47 def GetTestJson(change_id=None):
  48   """Get usable fake Gerrit patch json data
  49
  50   Args:
  51     change_id: If given, force this ChangeId
  52   """
  53   data = copy.deepcopy(patch_unittest.FAKE_PATCH_JSON)
  54   if change_id is not None:
  55     data['id'] = str(change_id)
  56   return data
  57
  58
  59 class MockManifest(object):
  60   """Helper class for Mocking Manifest objects."""
  61
  62   def __init__(self, path, **kwargs):
  63     self.root = path
  64     for key, attr in kwargs.iteritems():
  65       setattr(self, key, attr)
  66
  67   def ProjectIsContentMerging(self, _project):
  68     return False
  69
  70
  71 # pylint: disable=W0212,R0904
  72 class Base(cros_test_lib.MockTestCase):
  73   """Test case base class with helpers for other test suites."""
  74
  75   def setUp(self):
  76     self.patch_mock = None
  77     self._patch_counter = (itertools.count(1)).next
  78     self.build_root = 'fakebuildroot'
  79     self.PatchObject(gob_util, 'CreateHttpConn',
  80                      side_effect=AssertionError('Test should not contact GoB'))
  81     self.PatchObject(timeout_util, 'IsTreeOpen', return_value=True)
  82     self.PatchObject(timeout_util, 'WaitForTreeStatus',
  83                      return_value=constants.TREE_OPEN)
  84
  85   def MockPatch(self, change_id=None, patch_number=None, is_merged=False,
  86                 project='chromiumos/chromite', remote=constants.EXTERNAL_REMOTE,
  87                 tracking_branch='refs/heads/master', is_draft=False,
  88                 approvals=()):
  89     """Helper function to create mock GerritPatch objects."""
  90     if change_id is None:
  91       change_id = self._patch_counter()
  92     gerrit_number = str(change_id)
  93     change_id = hex(change_id)[2:].rstrip('L').lower()
  94     change_id = 'I%s' % change_id.rjust(40, '0')
  95     sha1 = hex(_GetNumber())[2:].rstrip('L').lower().rjust(40, '0')
  96     patch_number = (patch_number if patch_number is not None else _GetNumber())
  97     fake_url = 'http://foo/bar'
  98     if not approvals:
  99       approvals = [{'type': 'VRIF', 'value': '1', 'grantedOn': 1391733002},
 100                    {'type': 'CRVW', 'value': '2', 'grantedOn': 1391733002},
 101                    {'type': 'COMR', 'value': '1', 'grantedOn': 1391733002},]
 102
 103     current_patch_set = {
 104       'number': patch_number,
 105       'revision': sha1,
 106       'draft': is_draft,
 107       'approvals': approvals,
 108     }
 109     patch_dict = {
 110       'currentPatchSet': current_patch_set,
 111       'id': change_id,
 112       'number': gerrit_number,
 113       'project': project,
 114       'branch': tracking_branch,
 115       'owner': {'email': 'elmer.fudd@chromium.org'},
 116       'remote': remote,
 117       'status': 'MERGED' if is_merged else 'NEW',
 118       'url': '%s/%s' % (fake_url, change_id),
 119     }
 120
 121     return cros_patch.GerritPatch(patch_dict, remote, fake_url)
 122
 123   def GetPatches(self, how_many=1, always_use_list=False, **kwargs):
 124     """Get a sequential list of patches.
 125
 126     Args:
 127       how_many: How many patches to return.
 128       always_use_list: Whether to use a list for a single item list.
 129       **kwargs: Keyword arguments for self.MockPatch.
 130     """
 131     patches = [self.MockPatch(**kwargs) for _ in xrange(how_many)]
 132     if self.patch_mock:
 133       for i, patch in enumerate(patches):
 134         self.patch_mock.SetGerritDependencies(patch, patches[:i + 1])
 135     if how_many == 1 and not always_use_list:
 136       return patches[0]
 137     return patches
 138
 139
 140 class MoxBase(Base, cros_test_lib.MoxTestCase):
 141   """Base class for other test suites with numbers mocks patched in."""
 142
 143   def setUp(self):
 144     self.mox.StubOutWithMock(validation_pool, '_RunCommand')
 145     # Suppress all gerrit access; having this occur is generally a sign
 146     # the code is either misbehaving, or that the tests are bad.
 147     self.mox.StubOutWithMock(gerrit.GerritHelper, 'Query')
 148     self.PatchObject(gs.GSContext, 'Cat', side_effect=gs.GSNoSuchKey())
 149     self.PatchObject(gs.GSContext, 'Copy')
 150     self.PatchObject(gs.GSContext, 'Exists', return_value=False)
 151     self.PatchObject(gs.GSCounter, 'Increment')
 152
 153   def MakeHelper(self, cros_internal=None, cros=None):
 154     # pylint: disable=W0201
 155     if cros_internal:
 156       cros_internal = self.mox.CreateMock(gerrit.GerritHelper)
 157       cros_internal.version = '2.2'
 158       cros_internal.remote = constants.INTERNAL_REMOTE
 159     if cros:
 160       cros = self.mox.CreateMock(gerrit.GerritHelper)
 161       cros.remote = constants.EXTERNAL_REMOTE
 162       cros.version = '2.2'
 163     return validation_pool.HelperPool(cros_internal=cros_internal,
 164                                       cros=cros)
 165
 166
 167 class IgnoredStagesTest(Base):
 168   """Tests for functions that calculate what stages to ignore."""
 169
 170   def testBadConfigFile(self):
 171     """Test if we can handle an incorrectly formatted config file."""
 172     with osutils.TempDir(set_global=True) as tempdir:
 173       path = os.path.join(tempdir, 'foo.ini')
 174       osutils.WriteFile(path, 'foobar')
 175       ignored = validation_pool.GetStagesToIgnoreFromConfigFile(path)
 176       self.assertEqual([], ignored)
 177
 178   def testMissingConfigFile(self):
 179     """Test if we can handle a missing config file."""
 180     with osutils.TempDir(set_global=True) as tempdir:
 181       path = os.path.join(tempdir, 'foo.ini')
 182       ignored = validation_pool.GetStagesToIgnoreFromConfigFile(path)
 183       self.assertEqual([], ignored)
 184
 185   def testGoodConfigFile(self):
 186     """Test if we can handle a good config file."""
 187     with osutils.TempDir(set_global=True) as tempdir:
 188       path = os.path.join(tempdir, 'foo.ini')
 189       osutils.WriteFile(path, '[GENERAL]\nignored-stages: bar baz\n')
 190       ignored = validation_pool.GetStagesToIgnoreFromConfigFile(path)
 191       self.assertEqual(['bar', 'baz'], ignored)
 192
 193
 194 class TestPatchSeries(MoxBase):
 195   """Tests resolution and applying logic of validation_pool.ValidationPool."""
 196
 197   @contextlib.contextmanager
 198   def _ValidateTransactionCall(self, _changes):
 199     yield
 200
 201   def GetPatchSeries(self, helper_pool=None, force_content_merging=False):
 202     if helper_pool is None:
 203       helper_pool = self.MakeHelper(cros_internal=True, cros=True)
 204     series = validation_pool.PatchSeries(self.build_root, helper_pool,
 205                                          force_content_merging)
 206
 207     # Suppress transactions.
 208     series._Transaction = self._ValidateTransactionCall
 209     series.GetGitRepoForChange = \
 210         lambda change, **kwargs: os.path.join(self.build_root, change.project)
 211     series._IsContentMerging = lambda change: False
 212
 213     return series
 214
 215   def assertPath(self, _patch, return_value, path):
 216     self.assertEqual(path, os.path.join(self.build_root, _patch.project))
 217     if isinstance(return_value, Exception):
 218       raise return_value
 219     return return_value
 220
 221   def SetPatchDeps(self, patch, parents=(), cq=()):
 222     """Set the dependencies of |patch|.
 223
 224     Args:
 225       patch: The patch to process.
 226       parents: A set of strings to set as parents of |patch|.
 227       cq: A set of strings to set as paladin dependencies of |patch|.
 228     """
 229     patch.GerritDependencies = (
 230         lambda: [cros_patch.ParsePatchDep(x) for x in parents])
 231     patch.PaladinDependencies = functools.partial(
 232         self.assertPath, patch, [cros_patch.ParsePatchDep(x) for x in cq])
 233     patch.Fetch = functools.partial(
 234         self.assertPath, patch, patch.sha1)
 235
 236   def _ValidatePatchApplyManifest(self, value):
 237     self.assertTrue(isinstance(value, MockManifest))
 238     self.assertEqual(value.root, self.build_root)
 239     return True
 240
 241   def SetPatchApply(self, patch, trivial=True):
 242     self.mox.StubOutWithMock(patch, 'ApplyAgainstManifest')
 243     return patch.ApplyAgainstManifest(
 244         mox.Func(self._ValidatePatchApplyManifest),
 245         trivial=trivial)
 246
 247   def assertResults(self, series, changes, applied=(), failed_tot=(),
 248                     failed_inflight=(), frozen=True, dryrun=False):
 249     # Convenience; set the content pool as necessary.
 250     for remote in set(x.remote for x in changes):
 251       try:
 252         helper = series._helper_pool.GetHelper(remote)
 253         series._content_merging_projects.setdefault(helper, frozenset())
 254       except validation_pool.GerritHelperNotAvailable:
 255         continue
 256
 257     manifest = MockManifest(self.build_root)
 258     result = series.Apply(changes, dryrun=dryrun,
 259                           frozen=frozen, manifest=manifest)
 260
 261     _GetIds = lambda seq:[x.id for x in seq]
 262     _GetFailedIds = lambda seq: _GetIds(x.patch for x in seq)
 263
 264     applied_result = _GetIds(result[0])
 265     failed_tot_result, failed_inflight_result = map(_GetFailedIds, result[1:])
 266
 267     applied = _GetIds(applied)
 268     failed_tot = _GetIds(failed_tot)
 269     failed_inflight = _GetIds(failed_inflight)
 270
 271     self.assertEqual(
 272         [applied, failed_tot, failed_inflight],
 273         [applied_result, failed_tot_result, failed_inflight_result])
 274     return result
 275
 276   def testApplyWithDeps(self):
 277     """Test that we can apply changes correctly and respect deps.
 278
 279     This tests a simple out-of-order change where change1 depends on change2
 280     but tries to get applied before change2.  What should happen is that
 281     we should notice change2 is a dep of change1 and apply it first.
 282     """
 283     series = self.GetPatchSeries()
 284
 285     patch1, patch2 = patches = self.GetPatches(2)
 286
 287     self.SetPatchDeps(patch2)
 288     self.SetPatchDeps(patch1, [patch2.id])
 289
 290     self.SetPatchApply(patch2)
 291     self.SetPatchApply(patch1)
 292
 293     self.mox.ReplayAll()
 294     self.assertResults(series, patches, [patch2, patch1])
 295     self.mox.VerifyAll()
 296
 297   def testSha1Deps(self):
 298     """Test that we can apply changes correctly and respect sha1 deps.
 299
 300     This tests a simple out-of-order change where change1 depends on change2
 301     but tries to get applied before change2.  What should happen is that
 302     we should notice change2 is a dep of change1 and apply it first.
 303     """
 304     series = self.GetPatchSeries()
 305
 306     patch1, patch2, patch3 = patches = self.GetPatches(3)
 307     patch2.change_id = patch2.id = patch2.sha1
 308     patch3.change_id = patch3.id = '*' + patch3.sha1
 309     patch3.remote = constants.INTERNAL_REMOTE
 310
 311     self.SetPatchDeps(patch1, [patch2.sha1])
 312     self.SetPatchDeps(patch2, ['*%s' % patch3.sha1])
 313     self.SetPatchDeps(patch3)
 314
 315     self.SetPatchApply(patch2)
 316     self.SetPatchApply(patch3)
 317     self.SetPatchApply(patch1)
 318
 319     self.mox.ReplayAll()
 320     self.assertResults(series, patches, [patch3, patch2, patch1])
 321     self.mox.VerifyAll()
 322
 323   def testGerritNumberDeps(self):
 324     """Test that we can apply changes correctly and respect gerrit number deps.
 325
 326     This tests a simple out-of-order change where change1 depends on change2
 327     but tries to get applied before change2.  What should happen is that
 328     we should notice change2 is a dep of change1 and apply it first.
 329     """
 330     series = self.GetPatchSeries()
 331
 332     patch1, patch2, patch3 = patches = self.GetPatches(3)
 333
 334     self.SetPatchDeps(patch3, cq=[patch1.gerrit_number])
 335     self.SetPatchDeps(patch2, cq=[patch3.gerrit_number])
 336     self.SetPatchDeps(patch1, cq=[patch2.id])
 337
 338     self.SetPatchApply(patch3)
 339     self.SetPatchApply(patch2)
 340     self.SetPatchApply(patch1)
 341
 342     self.mox.ReplayAll()
 343     self.assertResults(series, patches, patches)
 344     self.mox.VerifyAll()
 345
 346   def testGerritLazyMapping(self):
 347     """Given a patch lacking a gerrit number, via gerrit, map it to that change.
 348
 349     Literally, this ensures that local patches pushed up- lacking a gerrit
 350     number- are mapped back to a changeid via asking gerrit for that number,
 351     then the local matching patch is used if available.
 352     """
 353     series = self.GetPatchSeries()
 354
 355     patch1 = self.MockPatch()
 356     self.PatchObject(patch1, 'LookupAliases', return_value=[patch1.id])
 357     patch2 = self.MockPatch(change_id=int(patch1.change_id[1:]))
 358     patch3 = self.MockPatch()
 359
 360     self.SetPatchDeps(patch3, cq=[patch2.gerrit_number])
 361     self.SetPatchDeps(patch2)
 362     self.SetPatchDeps(patch1)
 363
 364     self.SetPatchApply(patch1)
 365     self.SetPatchApply(patch3)
 366
 367     self._SetQuery(series, patch2, query=patch2.gerrit_number).AndReturn(patch2)
 368
 369     self.mox.ReplayAll()
 370     applied = self.assertResults(series, [patch1, patch3], [patch3, patch1])[0]
 371     self.assertTrue(applied[0] is patch3)
 372     self.assertTrue(applied[1] is patch1)
 373     self.mox.VerifyAll()
 374
 375   def testCrosGerritDeps(self, cros_internal=True):
 376     """Test that we can apply changes correctly and respect deps.
 377
 378     This tests a simple out-of-order change where change1 depends on change3
 379     but tries to get applied before change2.  What should happen is that
 380     we should notice change2 is a dep of change1 and apply it first.
 381     """
 382     helper_pool = self.MakeHelper(cros_internal=cros_internal, cros=True)
 383     series = self.GetPatchSeries(helper_pool=helper_pool)
 384
 385     patch1 = self.MockPatch(remote=constants.EXTERNAL_REMOTE)
 386     patch2 = self.MockPatch(remote=constants.INTERNAL_REMOTE)
 387     patch3 = self.MockPatch(remote=constants.EXTERNAL_REMOTE)
 388     patches = [patch1, patch2, patch3]
 389     applied_patches = patches[::-1] if cros_internal else [patch3, patch1]
 390
 391     self.SetPatchDeps(patch1, [patch3.id])
 392     self.SetPatchDeps(patch2)
 393     self.SetPatchDeps(patch3, cq=[patch2.id])
 394
 395     if cros_internal:
 396       self.SetPatchApply(patch2)
 397     self.SetPatchApply(patch1)
 398     self.SetPatchApply(patch3)
 399
 400     self.mox.ReplayAll()
 401     self.assertResults(series, patches, applied_patches)
 402     self.mox.VerifyAll()
 403
 404   def testExternalCrosGerritDeps(self):
 405     """Test that we exclude internal deps on external trybot."""
 406     self.testCrosGerritDeps(cros_internal=False)
 407
 408   @staticmethod
 409   def _SetQuery(series, change, query=None):
 410     helper = series._helper_pool.GetHelper(change.remote)
 411     query = change.id if query is None else query
 412     return helper.QuerySingleRecord(query, must_match=True)
 413
 414   def testApplyMissingDep(self):
 415     """Test that we don't try to apply a change without met dependencies.
 416
 417     Patch2 is in the validation pool that depends on Patch1 (which is not)
 418     Nothing should get applied.
 419     """
 420     series = self.GetPatchSeries()
 421
 422     patch1, patch2 = self.GetPatches(2)
 423
 424     self.SetPatchDeps(patch2, [patch1.id])
 425     self._SetQuery(series, patch1).AndReturn(patch1)
 426
 427     self.mox.ReplayAll()
 428     self.assertResults(series, [patch2],
 429                        [], [patch2])
 430     self.mox.VerifyAll()
 431
 432   def testApplyWithCommittedDeps(self):
 433     """Test that we apply a change with dependency already committed."""
 434     series = self.GetPatchSeries()
 435
 436     # Use for basic commit check.
 437     patch1 = self.GetPatches(1, is_merged=True)
 438     patch2 = self.GetPatches(1)
 439
 440     self.SetPatchDeps(patch2, [patch1.id])
 441     self._SetQuery(series, patch1).AndReturn(patch1)
 442     self.SetPatchApply(patch2)
 443
 444     # Used to ensure that an uncommitted change put in the lookup cache
 445     # isn't invalidly pulled into the graph...
 446     patch3, patch4, patch5 = self.GetPatches(3)
 447
 448     self._SetQuery(series, patch3).AndReturn(patch3)
 449     self.SetPatchDeps(patch4, [patch3.id])
 450     self.SetPatchDeps(patch5, [patch3.id])
 451
 452     self.mox.ReplayAll()
 453     self.assertResults(series, [patch2, patch4, patch5], [patch2],
 454                        [patch4, patch5])
 455     self.mox.VerifyAll()
 456
 457   def testCyclicalDeps(self):
 458     """Verify that the machinery handles cycles correctly."""
 459     series = self.GetPatchSeries()
 460
 461     patch1, patch2 = patches = self.GetPatches(2)
 462
 463     self.SetPatchDeps(patch1, [patch1.id])
 464     self.SetPatchDeps(patch2, cq=[patch1.id])
 465
 466     self.SetPatchApply(patch2)
 467     self.SetPatchApply(patch1)
 468
 469     self.mox.ReplayAll()
 470     self.assertResults(series, patches, patches[::-1])
 471
 472   def testApplyPartialFailures(self):
 473     """Test that can apply changes correctly when one change fails to apply.
 474
 475     This tests a simple change order where 1 depends on 2 and 1 fails to apply.
 476     Only 1 should get tried as 2 will abort once it sees that 1 can't be
 477     applied.  3 with no dependencies should go through fine.
 478
 479     Since patch1 fails to apply, we should also get a call to handle the
 480     failure.
 481     """
 482     series = self.GetPatchSeries()
 483
 484     patch1, patch2, patch3, patch4 = patches = self.GetPatches(4)
 485
 486     self.SetPatchDeps(patch1)
 487     self.SetPatchDeps(patch2, [patch1.id])
 488     self.SetPatchDeps(patch3)
 489     self.SetPatchDeps(patch4)
 490
 491     self.SetPatchApply(patch1).AndRaise(
 492         cros_patch.ApplyPatchException(patch1))
 493
 494     self.SetPatchApply(patch3)
 495     self.SetPatchApply(patch4).AndRaise(
 496         cros_patch.ApplyPatchException(patch1, inflight=True))
 497
 498     self.mox.ReplayAll()
 499     self.assertResults(series, patches,
 500                        [patch3], [patch2, patch1], [patch4])
 501     self.mox.VerifyAll()
 502
 503   def testComplexApply(self):
 504     """More complex deps test.
 505
 506     This tests a total of 2 change chains where the first change we see
 507     only has a partial chain with the 3rd change having the whole chain i.e.
 508     1->2, 3->1->2.  Since we get these in the order 1,2,3,4,5 the order we
 509     should apply is 2,1,3,4,5.
 510
 511     This test also checks the patch order to verify that Apply re-orders
 512     correctly based on the chain.
 513     """
 514     series = self.GetPatchSeries()
 515
 516     patch1, patch2, patch3, patch4, patch5 = patches = self.GetPatches(5)
 517
 518     self.SetPatchDeps(patch1, [patch2.id])
 519     self.SetPatchDeps(patch2)
 520     self.SetPatchDeps(patch3, [patch1.id, patch2.id])
 521     self.SetPatchDeps(patch4, cq=[patch5.id])
 522     self.SetPatchDeps(patch5)
 523
 524     for patch in (patch2, patch1, patch3, patch4, patch5):
 525       self.SetPatchApply(patch)
 526
 527     self.mox.ReplayAll()
 528     self.assertResults(
 529         series, patches, [patch2, patch1, patch3, patch4, patch5])
 530     self.mox.VerifyAll()
 531
 532   def testApplyStandalonePatches(self):
 533     """Simple apply of two changes with no dependent CL's."""
 534     series = self.GetPatchSeries()
 535
 536     patches = self.GetPatches(3)
 537
 538     for patch in patches:
 539       self.SetPatchDeps(patch)
 540
 541     for patch in patches:
 542       self.SetPatchApply(patch)
 543
 544     self.mox.ReplayAll()
 545     self.assertResults(series, patches, patches)
 546     self.mox.VerifyAll()
 547
 548
 549 def MakePool(overlays=constants.PUBLIC_OVERLAYS, build_number=1,
 550              builder_name='foon', is_master=True, dryrun=True, **kwargs):
 551   """Helper for creating ValidationPool objects for tests."""
 552   kwargs.setdefault('changes', [])
 553   build_root = kwargs.pop('build_root', '/fake_root')
 554
 555   pool = validation_pool.ValidationPool(
 556       overlays, build_root, build_number, builder_name, is_master,
 557       dryrun, **kwargs)
 558   return pool
 559
 560
 561 class MockPatchSeries(partial_mock.PartialMock):
 562   """Mock the PatchSeries functions."""
 563   TARGET = 'chromite.buildbot.validation_pool.PatchSeries'
 564   ATTRS = ('GetDepsForChange', '_GetGerritPatch', '_LookupHelper')
 565
 566   def __init__(self):
 567     partial_mock.PartialMock.__init__(self)
 568     self.deps = {}
 569     self.cq_deps = {}
 570
 571   def SetGerritDependencies(self, patch, deps):
 572     """Add |deps| to the Gerrit dependencies of |patch|."""
 573     self.deps[patch] = deps
 574
 575   def SetCQDependencies(self, patch, deps):
 576     """Add |deps| to the CQ dependencies of |patch|."""
 577     self.cq_deps[patch] = deps
 578
 579   def GetDepsForChange(self, _inst, patch):
 580     return self.deps.get(patch, []), self.cq_deps.get(patch, [])
 581
 582   def _GetGerritPatch(self, _inst, dep, **_kwargs):
 583     return dep
 584
 585   _LookupHelper = mock.MagicMock()
 586
 587
 588 class TestSubmitChange(MoxBase):
 589   """Test suite related to submitting changes."""
 590
 591   def setUp(self):
 592     self.orig_timeout = validation_pool.SUBMITTED_WAIT_TIMEOUT
 593     validation_pool.SUBMITTED_WAIT_TIMEOUT = 4
 594
 595   def tearDown(self):
 596     validation_pool.SUBMITTED_WAIT_TIMEOUT = self.orig_timeout
 597
 598   def _TestSubmitChange(self, results):
 599     """Test submitting a change with the given results."""
 600     results = [cros_test_lib.EasyAttr(status=r) for r in results]
 601     change = self.MockPatch(change_id=12345, patch_number=1)
 602     pool = self.mox.CreateMock(validation_pool.ValidationPool)
 603     pool.dryrun = False
 604     pool._metadata = cbuildbot_metadata.CBuildbotMetadata()
 605     pool._helper_pool = self.mox.CreateMock(validation_pool.HelperPool)
 606     helper = self.mox.CreateMock(validation_pool.gerrit.GerritHelper)
 607
 608     # Prepare replay script.
 609     pool._helper_pool.ForChange(change).AndReturn(helper)
 610     helper.SubmitChange(change, dryrun=False)
 611     for result in results:
 612       helper.QuerySingleRecord(change.gerrit_number).AndReturn(result)
 613     self.mox.ReplayAll()
 614
 615     # Verify results.
 616     retval = validation_pool.ValidationPool._SubmitChange(pool, change)
 617     self.mox.VerifyAll()
 618     return retval
 619
 620   def testSubmitChangeMerged(self):
 621     """Submit one change to gerrit, status MERGED."""
 622     self.assertTrue(self._TestSubmitChange(['MERGED']))
 623
 624   def testSubmitChangeSubmitted(self):
 625     """Submit one change to gerrit, stuck on SUBMITTED."""
 626     # The query will be retried 1 more time than query timeout.
 627     results = ['SUBMITTED' for _i in
 628                xrange(validation_pool.SUBMITTED_WAIT_TIMEOUT + 1)]
 629     self.assertTrue(self._TestSubmitChange(results))
 630
 631   def testSubmitChangeSubmittedToMerged(self):
 632     """Submit one change to gerrit, status SUBMITTED then MERGED."""
 633     results = ['SUBMITTED', 'SUBMITTED', 'MERGED']
 634     self.assertTrue(self._TestSubmitChange(results))
 635
 636   def testSubmitChangeFailed(self):
 637     """Submit one change to gerrit, reported back as NEW."""
 638     self.assertFalse(self._TestSubmitChange(['NEW']))
 639
 640
 641 class ValidationFailureOrTimeout(MoxBase):
 642   """Tests that HandleValidationFailure and HandleValidationTimeout functions.
 643
 644   These tests check that HandleValidationTimeout and HandleValidationFailure
 645   reject (i.e. zero out the CQ field) of the correct number of patches, under
 646   various circumstances.
 647   """
 648
 649   _PATCH_MESSAGE = 'Your patch failed.'
 650   _BUILD_MESSAGE = 'Your build failed.'
 651
 652   def setUp(self):
 653     self._patches = self.GetPatches(3)
 654     self._pool = MakePool(changes=self._patches)
 655
 656     self.PatchObject(
 657         validation_pool.ValidationPool, 'GetCLStatus',
 658         return_value=validation_pool.ValidationPool.STATUS_PASSED)
 659     self.PatchObject(
 660         validation_pool.CalculateSuspects, 'FindSuspects',
 661         return_value=self._patches)
 662     self.PatchObject(
 663         validation_pool.ValidationPool, '_CreateValidationFailureMessage',
 664         return_value=self._PATCH_MESSAGE)
 665     self.PatchObject(validation_pool.ValidationPool, 'SendNotification')
 666     self.PatchObject(validation_pool.ValidationPool, 'RemoveCommitReady')
 667     self.PatchObject(validation_pool.ValidationPool, 'UpdateCLStatus')
 668     self.StartPatcher(parallel_unittest.ParallelMock())
 669
 670
 671   def testPatchesWereRejectedByFailure(self):
 672     self._pool.HandleValidationFailure([self._BUILD_MESSAGE])
 673     self.assertEqual(
 674         len(self._patches), self._pool.RemoveCommitReady.call_count)
 675
 676   def testPatchesWereRejectedByTimeout(self):
 677     self._pool.HandleValidationTimeout()
 678     self.assertEqual(
 679         len(self._patches), self._pool.RemoveCommitReady.call_count)
 680
 681   def testNoSuspectsWithFailure(self):
 682     self.PatchObject(
 683         validation_pool.CalculateSuspects, 'FindSuspects',
 684         return_value=[])
 685     self._pool.HandleValidationFailure([self._BUILD_MESSAGE])
 686     self.assertEqual(0, self._pool.RemoveCommitReady.call_count)
 687
 688   def testPreCQ(self):
 689     self._pool.pre_cq = True
 690     self._pool.HandleValidationFailure([self._BUILD_MESSAGE])
 691     self.assertEqual(0, self._pool.RemoveCommitReady.call_count)
 692
 693   def testPatchesWereNotRejectedByInsaneFailure(self):
 694     self._pool.HandleValidationFailure([self._BUILD_MESSAGE], sanity=False)
 695     self.assertEqual(0, self._pool.RemoveCommitReady.call_count)
 696
 697
 698 class TestCoreLogic(MoxBase):
 699   """Tests resolution and applying logic of validation_pool.ValidationPool."""
 700
 701   def setUp(self):
 702     self.mox.StubOutWithMock(validation_pool.PatchSeries, 'Apply')
 703     self.mox.StubOutWithMock(validation_pool.PatchSeries, 'ApplyChange')
 704     self.patch_mock = self.StartPatcher(MockPatchSeries())
 705     funcs = ['SendNotification', '_SubmitChange']
 706     for func in funcs:
 707       self.mox.StubOutWithMock(validation_pool.ValidationPool, func)
 708     self.PatchObject(validation_pool.ValidationPool, 'ReloadChanges',
 709                      side_effect=lambda x: x)
 710     self.StartPatcher(parallel_unittest.ParallelMock())
 711
 712   def MakePool(self, *args, **kwargs):
 713     """Helper for creating ValidationPool objects for Mox tests."""
 714     handlers = kwargs.pop('handlers', False)
 715     kwargs['build_root'] = self.build_root
 716     pool = MakePool(*args, **kwargs)
 717     funcs = ['_HandleApplySuccess', '_HandleApplyFailure',
 718              '_HandleCouldNotApply', '_HandleCouldNotSubmit']
 719     if handlers:
 720       for func in funcs:
 721         self.mox.StubOutWithMock(pool, func)
 722     return pool
 723
 724   def MakeFailure(self, patch, inflight=True):
 725     return cros_patch.ApplyPatchException(patch, inflight=inflight)
 726
 727   def GetPool(self, changes, applied=(), tot=(),
 728               inflight=(), dryrun=True, **kwargs):
 729     pool = self.MakePool(changes=changes, **kwargs)
 730     applied = list(applied)
 731     tot = [self.MakeFailure(x, inflight=False) for x in tot]
 732     inflight = [self.MakeFailure(x, inflight=True) for x in inflight]
 733     # pylint: disable=E1120,E1123
 734     validation_pool.PatchSeries.Apply(
 735         changes, dryrun=dryrun, manifest=mox.IgnoreArg()
 736         ).AndReturn((applied, tot, inflight))
 737
 738     for patch in applied:
 739       pool._HandleApplySuccess(patch).AndReturn(None)
 740
 741     if tot:
 742       pool._HandleApplyFailure(tot).AndReturn(None)
 743
 744     # We stash this on the pool object so we can reuse it during validation.
 745     # We could stash this in the test instances, but that would break
 746     # for any tests that do multiple pool instances.
 747
 748     pool._test_data = (changes, applied, tot, inflight)
 749
 750     return pool
 751
 752   def testApplySlavePool(self):
 753     """Verifies that slave calls ApplyChange() directly for each patch."""
 754     slave_pool = self.MakePool(is_master=False)
 755     patches = self.GetPatches(3)
 756     slave_pool.changes = patches
 757     for patch in patches:
 758       # pylint: disable=E1120, E1123
 759       validation_pool.PatchSeries.ApplyChange(
 760           patch, dryrun=mox.IgnoreArg(), manifest=mox.IgnoreArg())
 761
 762     self.mox.ReplayAll()
 763     self.assertEqual(True, slave_pool.ApplyPoolIntoRepo())
 764     self.mox.VerifyAll()
 765
 766   def runApply(self, pool, result):
 767     self.assertEqual(result, pool.ApplyPoolIntoRepo())
 768     self.assertEqual(pool.changes, pool._test_data[1])
 769     failed_inflight = pool.changes_that_failed_to_apply_earlier
 770     expected_inflight = set(pool._test_data[3])
 771     # Intersect the results, since it's possible there were results failed
 772     # results that weren't related to the ApplyPoolIntoRepo call.
 773     self.assertEqual(set(failed_inflight).intersection(expected_inflight),
 774                      expected_inflight)
 775
 776     self.assertEqual(pool.changes, pool._test_data[1])
 777
 778   def testPatchSeriesInteraction(self):
 779     """Verify the interaction between PatchSeries and ValidationPool.
 780
 781     Effectively, this validates data going into PatchSeries, and coming back
 782     out; verifies the hand off to _Handle* functions, but no deeper.
 783     """
 784     patches = self.GetPatches(3)
 785
 786     apply_pool = self.GetPool(patches, applied=patches, handlers=True)
 787     all_inflight = self.GetPool(patches, inflight=patches, handlers=True)
 788     all_tot = self.GetPool(patches, tot=patches, handlers=True)
 789     mixed = self.GetPool(patches, tot=patches[0:1], inflight=patches[1:2],
 790                          applied=patches[2:3], handlers=True)
 791
 792     self.mox.ReplayAll()
 793     self.runApply(apply_pool, True)
 794     self.runApply(all_inflight, False)
 795     self.runApply(all_tot, False)
 796     self.runApply(mixed, True)
 797     self.mox.VerifyAll()
 798
 799   def testHandleApplySuccess(self):
 800     """Validate steps taken for successfull application."""
 801     patch = self.GetPatches(1)
 802     pool = self.MakePool()
 803     pool.SendNotification(patch, mox.StrContains('has picked up your change'))
 804     self.mox.ReplayAll()
 805     pool._HandleApplySuccess(patch)
 806     self.mox.VerifyAll()
 807
 808   def testHandleApplyFailure(self):
 809     failures = [cros_patch.ApplyPatchException(x) for x in self.GetPatches(4)]
 810
 811     notified_patches = failures[:2]
 812     unnotified_patches = failures[2:]
 813     master_pool = self.MakePool(dryrun=False)
 814     slave_pool = self.MakePool(is_master=False)
 815
 816     self.mox.StubOutWithMock(gerrit.GerritHelper, 'RemoveCommitReady')
 817
 818     for failure in notified_patches:
 819       master_pool.SendNotification(
 820           failure.patch,
 821           mox.StrContains('failed to apply your change'),
 822           failure=mox.IgnoreArg())
 823       # This pylint suppressin shouldn't be necessary, but pylint is invalidly
 824       # thinking that the first arg isn't passed in; we suppress it to suppress
 825       # the pylnt bug.
 826       # pylint: disable=E1120
 827       gerrit.GerritHelper.RemoveCommitReady(failure.patch, dryrun=False)
 828
 829     self.mox.ReplayAll()
 830     master_pool._HandleApplyFailure(notified_patches)
 831     slave_pool._HandleApplyFailure(unnotified_patches)
 832     self.mox.VerifyAll()
 833
 834   def _setUpSubmit(self):
 835     pool = self.MakePool(dryrun=False, handlers=True)
 836     patches = self.GetPatches(3)
 837     failed = self.GetPatches(3)
 838     pool.changes = patches[:]
 839     # While we don't do anything w/ these patches, that's
 840     # intentional; we're verifying that it isn't submitted
 841     # if there is a failure.
 842     pool.changes_that_failed_to_apply_earlier = failed[:]
 843
 844     return (pool, patches, failed)
 845
 846   def testSubmitPoolFailures(self):
 847     """Tests that a fatal exception is raised."""
 848     pool, patches, _failed = self._setUpSubmit()
 849     patch1, patch2, patch3 = patches
 850
 851     pool._SubmitChange(patch1).AndReturn(True)
 852     pool._SubmitChange(patch2).AndReturn(False)
 853
 854     pool._HandleCouldNotSubmit(patch2, mox.IgnoreArg()).InAnyOrder()
 855     pool._HandleCouldNotSubmit(patch3, mox.IgnoreArg()).InAnyOrder()
 856
 857     self.mox.ReplayAll()
 858     self.assertRaises(validation_pool.FailedToSubmitAllChangesException,
 859                       pool.SubmitPool)
 860     self.mox.VerifyAll()
 861
 862   def testSubmitPartialPass(self):
 863     """Tests that a non-fatal exception is raised."""
 864     pool, patches, _failed = self._setUpSubmit()
 865     patch1, patch2, patch3 = patches
 866     # Make patch2 not commit-ready.
 867     patch2._approvals = []
 868
 869     pool._SubmitChange(patch1).AndReturn(True)
 870
 871     pool._HandleCouldNotSubmit(patch2, mox.IgnoreArg()).InAnyOrder()
 872     pool._HandleCouldNotSubmit(patch3, mox.IgnoreArg()).InAnyOrder()
 873
 874     self.mox.ReplayAll()
 875     self.assertRaises(validation_pool.FailedToSubmitAllChangesNonFatalException,
 876                       pool.SubmitPool)
 877     self.mox.VerifyAll()
 878
 879   def testSubmitPool(self):
 880     """Tests that we can submit a pool of patches."""
 881     pool, patches, failed = self._setUpSubmit()
 882
 883     for patch in patches:
 884       pool._SubmitChange(patch).AndReturn(True)
 885
 886     pool._HandleApplyFailure(failed)
 887
 888     self.mox.ReplayAll()
 889     pool.SubmitPool()
 890     self.mox.VerifyAll()
 891
 892   def testSubmitNonManifestChanges(self):
 893     """Simple test to make sure we can submit non-manifest changes."""
 894     pool, patches, _failed = self._setUpSubmit()
 895     pool.non_manifest_changes = patches[:]
 896
 897     for patch in patches:
 898       pool._SubmitChange(patch).AndReturn(True)
 899
 900     self.mox.ReplayAll()
 901     pool.SubmitNonManifestChanges()
 902     self.mox.VerifyAll()
 903
 904   def testUnhandledExceptions(self):
 905     """Test that CQ doesn't loop due to unhandled Exceptions."""
 906     pool, patches, _failed = self._setUpSubmit()
 907
 908     class MyException(Exception):
 909       """"Unique Exception used for testing."""
 910
 911     def VerifyCQError(patch, error):
 912       cq_error = validation_pool.InternalCQError(patch, error.message)
 913       return str(error) == str(cq_error)
 914
 915     # pylint: disable=E1120,E1123
 916     validation_pool.PatchSeries.Apply(
 917         patches, dryrun=False, manifest=mox.IgnoreArg()).AndRaise(
 918         MyException)
 919     errors = [mox.Func(functools.partial(VerifyCQError, x)) for x in patches]
 920     pool._HandleApplyFailure(errors).AndReturn(None)
 921
 922     self.mox.ReplayAll()
 923     self.assertRaises(MyException, pool.ApplyPoolIntoRepo)
 924     self.mox.VerifyAll()
 925
 926   def testFilterDependencyErrors(self):
 927     """Verify that dependency errors are correctly filtered out."""
 928     failures = [cros_patch.ApplyPatchException(x) for x in self.GetPatches(2)]
 929     failures += [cros_patch.DependencyError(x, y) for x, y in
 930                  zip(self.GetPatches(2), failures)]
 931     failures[0].patch.approval_timestamp = time.time()
 932     failures[-1].patch.approval_timestamp = time.time()
 933     self.mox.ReplayAll()
 934     result = validation_pool.ValidationPool._FilterDependencyErrors(failures)
 935     self.assertEquals(set(failures[:-1]), set(result))
 936     self.mox.VerifyAll()
 937
 938   def testFilterNonCrosProjects(self):
 939     """Runs through a filter of own manifest and fake changes.
 940
 941     This test should filter out the tacos/chromite project as its not real.
 942     """
 943     base_func = itertools.cycle(['chromiumos', 'chromeos']).next
 944     patches = self.GetPatches(8)
 945     for patch in patches:
 946       patch.project = '%s/%i' % (base_func(), _GetNumber())
 947       patch.tracking_branch = str(_GetNumber())
 948
 949     non_cros_patches = self.GetPatches(2)
 950     for patch in non_cros_patches:
 951       patch.project = str(_GetNumber())
 952
 953     filtered_patches = patches[:4]
 954     allowed_patches = []
 955     projects = {}
 956     for idx, patch in enumerate(patches[4:]):
 957       fails = bool(idx % 2)
 958       # Vary the revision so we can validate that it checks the branch.
 959       revision = ('monkeys' if fails
 960                   else 'refs/heads/%s' % patch.tracking_branch)
 961       if fails:
 962         filtered_patches.append(patch)
 963       else:
 964         allowed_patches.append(patch)
 965       projects.setdefault(patch.project, {})['revision'] = revision
 966
 967     manifest = MockManifest(self.build_root, projects=projects)
 968     for patch in allowed_patches:
 969       patch.GetCheckout = lambda *_args, **_kwargs: True
 970     for patch in filtered_patches:
 971       patch.GetCheckout = lambda *_args, **_kwargs: False
 972
 973     self.mox.ReplayAll()
 974     results = validation_pool.ValidationPool._FilterNonCrosProjects(
 975         patches + non_cros_patches, manifest)
 976
 977     def compare(list1, list2):
 978       mangle = lambda c:(c.id, c.project, c.tracking_branch)
 979       self.assertEqual(list1, list2,
 980         msg="Comparison failed:\n list1: %r\n list2: %r"
 981             % (map(mangle, list1), map(mangle, list2)))
 982
 983     compare(results[0], allowed_patches)
 984     compare(results[1], filtered_patches)
 985
 986
 987 class TestPickling(cros_test_lib.TempDirTestCase):
 988   """Tests to validate pickling of ValidationPool, covering CQ's needs"""
 989
 990   def testSelfCompatibility(self):
 991     """Verify compatibility of current git HEAD against itself."""
 992     self._CheckTestData(self._GetTestData())
 993
 994   def testToTCompatibility(self):
 995     """Validate that ToT can use our pickles, and that we can use ToT's data."""
 996     repo = os.path.join(self.tempdir, 'chromite')
 997     reference = os.path.abspath(__file__)
 998     reference = os.path.normpath(os.path.join(reference, '../../'))
 999
1000     repository.CloneGitRepo(
1001         repo,
1002         '%s/chromiumos/chromite' % constants.EXTERNAL_GOB_URL,
1003         reference=reference)
1004
1005     code = """
1006 import sys
1007 from chromite.buildbot import validation_pool_unittest
1008 if not hasattr(validation_pool_unittest, 'TestPickling'):
1009   sys.exit(0)
1010 sys.stdout.write(validation_pool_unittest.TestPickling.%s)
1011 """
1012
1013     # Verify ToT can take our pickle.
1014     cros_build_lib.RunCommand(
1015         ['python', '-c', code % '_CheckTestData(sys.stdin.read())'],
1016         cwd=self.tempdir, print_cmd=False, capture_output=True,
1017         input=self._GetTestData())
1018
1019     # Verify we can handle ToT's pickle.
1020     ret = cros_build_lib.RunCommand(
1021         ['python', '-c', code % '_GetTestData()'],
1022         cwd=self.tempdir, print_cmd=False, capture_output=True)
1023
1024     self._CheckTestData(ret.output)
1025
1026   @staticmethod
1027   def _GetCrosInternalPatch(patch_info):
1028     return cros_patch.GerritPatch(
1029         patch_info,
1030         constants.INTERNAL_REMOTE,
1031         constants.INTERNAL_GERRIT_URL)
1032
1033   @staticmethod
1034   def _GetCrosPatch(patch_info):
1035     return cros_patch.GerritPatch(
1036         patch_info,
1037         constants.EXTERNAL_REMOTE,
1038         constants.EXTERNAL_GERRIT_URL)
1039
1040   @classmethod
1041   def _GetTestData(cls):
1042     ids = [cros_patch.MakeChangeId() for _ in xrange(3)]
1043     changes = [cls._GetCrosInternalPatch(GetTestJson(ids[0]))]
1044     non_os = [cls._GetCrosPatch(GetTestJson(ids[1]))]
1045     conflicting = [cls._GetCrosInternalPatch(GetTestJson(ids[2]))]
1046     conflicting = [cros_patch.PatchException(x)for x in conflicting]
1047     pool = validation_pool.ValidationPool(
1048         constants.PUBLIC_OVERLAYS,
1049         '/fake/pathway', 1,
1050         'testing', True, True,
1051         changes=changes, non_os_changes=non_os,
1052         conflicting_changes=conflicting)
1053     return pickle.dumps([pool, changes, non_os, conflicting])
1054
1055   @staticmethod
1056   def _CheckTestData(data):
1057     results = pickle.loads(data)
1058     pool, changes, non_os, conflicting = results
1059     def _f(source, value, getter=None):
1060       if getter is None:
1061         getter = lambda x: x
1062       assert len(source) == len(value)
1063       for s_item, v_item in zip(source, value):
1064         assert getter(s_item).id == getter(v_item).id
1065         assert getter(s_item).remote == getter(v_item).remote
1066     _f(pool.changes, changes)
1067     _f(pool.non_manifest_changes, non_os)
1068     _f(pool.changes_that_failed_to_apply_earlier, conflicting,
1069        getter=lambda s:getattr(s, 'patch', s))
1070     return ''
1071
1072
1073 class TestFindSuspects(MoxBase):
1074   """Tests validation_pool.ValidationPool._FindSuspects"""
1075
1076   def setUp(self):
1077     overlay = 'chromiumos/overlays/chromiumos-overlay'
1078     self.overlay_patch = self.GetPatches(project=overlay)
1079     self.power_manager = 'chromiumos/platform/power_manager'
1080     self.power_manager_pkg = 'chromeos-base/power_manager'
1081     self.power_manager_patch = self.GetPatches(project=self.power_manager)
1082     self.kernel = 'chromiumos/third_party/kernel'
1083     self.kernel_pkg = 'sys-kernel/chromeos-kernel'
1084     self.kernel_patch = self.GetPatches(project=self.kernel)
1085     self.secret = 'chromeos/secret'
1086     self.secret_patch = self.GetPatches(project=self.secret,
1087                                         remote=constants.INTERNAL_REMOTE)
1088
1089   @staticmethod
1090   def _GetBuildFailure(pkg):
1091     """Create a PackageBuildFailure for the specified |pkg|.
1092
1093     Args:
1094       pkg: Package that failed to build.
1095     """
1096     ex = cros_build_lib.RunCommandError('foo', cros_build_lib.CommandResult())
1097     return results_lib.PackageBuildFailure(ex, 'bar', [pkg])
1098
1099   def _AssertSuspects(self, patches, suspects, pkgs=(), exceptions=(),
1100                       internal=False):
1101     """Run _FindSuspects and verify its output.
1102
1103     Args:
1104       patches: List of patches to look at.
1105       suspects: Expected list of suspects returned by _FindSuspects.
1106       pkgs: List of packages that failed with exceptions in the build.
1107       exceptions: List of other exceptions that occurred during the build.
1108       internal: Whether the failures occurred on an internal bot.
1109     """
1110     all_exceptions = list(exceptions) + [self._GetBuildFailure(x) for x in pkgs]
1111     tracebacks = []
1112     for ex in all_exceptions:
1113       tracebacks.append(results_lib.RecordedTraceback('Build', 'Build', ex,
1114                                                       str(ex)))
1115     message = validation_pool.ValidationFailedMessage(
1116         'foo bar %r' % tracebacks, tracebacks, internal)
1117     results = validation_pool.CalculateSuspects.FindSuspects(patches, [message])
1118     self.assertEquals(set(suspects), results)
1119
1120   def testFailSameProject(self):
1121     """Patches to the package that failed should be marked as failing."""
1122     suspects = [self.kernel_patch]
1123     patches = suspects + [self.power_manager_patch, self.secret_patch]
1124     self._AssertSuspects(patches, suspects, [self.kernel_pkg])
1125
1126   def testFailSameProjectPlusOverlay(self):
1127     """Patches to the overlay should be marked as failing."""
1128     suspects = [self.overlay_patch, self.kernel_patch]
1129     patches = suspects + [self.power_manager_patch, self.secret_patch]
1130     self._AssertSuspects(patches, suspects, [self.kernel_pkg])
1131
1132   def testFailUnknownPackage(self):
1133     """If no patches changed the package, all patches should fail."""
1134     suspects = [self.overlay_patch, self.power_manager_patch]
1135     changes = suspects + [self.secret_patch]
1136     self._AssertSuspects(changes, suspects, [self.kernel_pkg])
1137
1138   def testFailUnknownException(self):
1139     """An unknown exception should cause all [public] patches to fail."""
1140     suspects = [self.kernel_patch, self.power_manager_patch]
1141     changes = suspects + [self.secret_patch]
1142     self._AssertSuspects(changes, suspects, exceptions=[Exception('foo bar')])
1143
1144   def testFailUnknownInternalException(self):
1145     """An unknown exception should cause all [internal] patches to fail."""
1146     suspects = [self.kernel_patch, self.power_manager_patch, self.secret_patch]
1147     self._AssertSuspects(suspects, suspects, exceptions=[Exception('foo bar')],
1148                          internal=True)
1149
1150   def testFailUnknownCombo(self):
1151     """Unknown exceptions should cause all patches to fail.
1152
1153     Even if there are also build failures that we can explain.
1154     """
1155     suspects = [self.kernel_patch, self.power_manager_patch]
1156     changes = suspects + [self.secret_patch]
1157     self._AssertSuspects(changes, suspects, [self.kernel_pkg],
1158                          [Exception('foo bar')])
1159
1160   def testFailNoExceptions(self):
1161     """If there are no exceptions, all patches should be failed."""
1162     suspects = [self.kernel_patch, self.power_manager_patch]
1163     changes = suspects + [self.secret_patch]
1164     self._AssertSuspects(changes, suspects)
1165
1166
1167 class TestCLStatus(MoxBase):
1168   """Tests methods that get the CL status."""
1169
1170   def testPrintLinks(self):
1171     changes = self.GetPatches(3)
1172     with parallel_unittest.ParallelMock():
1173       validation_pool.ValidationPool.PrintLinksToChanges(changes)
1174
1175   def testStatusCache(self):
1176     validation_pool.ValidationPool._CL_STATUS_CACHE = {}
1177     changes = self.GetPatches(3)
1178     with parallel_unittest.ParallelMock():
1179       validation_pool.ValidationPool.FillCLStatusCache(validation_pool.CQ,
1180                                                        changes)
1181       self.assertEqual(len(validation_pool.ValidationPool._CL_STATUS_CACHE), 12)
1182       validation_pool.ValidationPool.PrintLinksToChanges(changes)
1183       self.assertEqual(len(validation_pool.ValidationPool._CL_STATUS_CACHE), 12)
1184
1185
1186 class TestCreateValidationFailureMessage(Base):
1187   """Tests validation_pool.ValidationPool._CreateValidationFailureMessage"""
1188
1189   def _AssertMessage(self, change, suspects, messages, sanity=True):
1190     """Call the _CreateValidationFailureMessage method.
1191
1192     Args:
1193       change: The change we are commenting on.
1194       suspects: List of suspected changes.
1195       messages: List of messages to include in comment.
1196       sanity: Bool indicating sanity of build, default: True.
1197     """
1198     msg = validation_pool.ValidationPool._CreateValidationFailureMessage(
1199       False, change, set(suspects), messages, sanity=sanity)
1200     for x in messages:
1201       self.assertTrue(x in msg)
1202     return msg
1203
1204   def testSuspectChange(self):
1205     """Test case where 1 is the only change and is suspect."""
1206     patch = self.GetPatches(1)
1207     self._AssertMessage(patch, [patch], ['%s failed' % patch])
1208
1209   def testInnocentChange(self):
1210     """Test case where 1 is innocent."""
1211     patch1, patch2 = self.GetPatches(2)
1212     self._AssertMessage(patch1, [patch2], ['%s failed' % patch2])
1213
1214   def testSuspectChanges(self):
1215     """Test case where 1 is suspected, but so is 2."""
1216     patches = self.GetPatches(2)
1217     self._AssertMessage(patches[0], patches,
1218                         ['%s and %s failed' % tuple(patches)])
1219
1220   def testInnocentChangeWithMultipleSuspects(self):
1221     """Test case where 2 and 3 are suspected."""
1222     patches = self.GetPatches(3)
1223     self._AssertMessage(patches[0], patches[1:],
1224                         ['%s and %s failed' % tuple(patches[1:])])
1225
1226   def testNoSuspects(self):
1227     """Test case where there are no suspects."""
1228     self._AssertMessage(self.GetPatches(1), [], ['Internal error'])
1229
1230   def testNoMessages(self):
1231     """Test case where there are no messages."""
1232     patch1 = self.GetPatches(1)
1233     self._AssertMessage(patch1, [patch1], [])
1234
1235   def testInsaneBuild(self):
1236     patches = self.GetPatches(3)
1237     self._AssertMessage(
1238         patches[0], patches, ['sanity check builder',
1239                               'retry your change automatically'],
1240         sanity=False)
1241
1242 class TestCreateDisjointTransactions(Base):
1243   """Test the CreateDisjointTransactions function."""
1244
1245   def setUp(self):
1246     self.patch_mock = self.StartPatcher(MockPatchSeries())
1247
1248   def GetPatches(self, how_many, **kwargs):
1249     return Base.GetPatches(self, how_many, always_use_list=True, **kwargs)
1250
1251   def verifyTransactions(self, txns, max_txn_length=None, circular=False):
1252     """Verify the specified list of transactions are processed correctly.
1253
1254     Args:
1255       txns: List of transactions to process.
1256       max_txn_length: Maximum length of any given transaction. This is passed
1257         to the CreateDisjointTransactions function.
1258       circular: Whether the transactions contain circular dependencies.
1259     """
1260     remove = self.PatchObject(gerrit.GerritHelper, 'RemoveCommitReady')
1261     patches = list(itertools.chain.from_iterable(txns))
1262     expected_plans = txns
1263     if max_txn_length is not None:
1264       # When max_txn_length is specified, transactions should be truncated to
1265       # the specified length, ignoring any remaining patches.
1266       expected_plans = [txn[:max_txn_length] for txn in txns]
1267
1268     pool = MakePool(changes=patches)
1269     plans = pool.CreateDisjointTransactions(None, max_txn_length=max_txn_length)
1270
1271     # If the dependencies are circular, the order of the patches is not
1272     # guaranteed, so compare them in sorted order.
1273     if circular:
1274       plans = [sorted(plan) for plan in plans]
1275       expected_plans = [sorted(plan) for plan in expected_plans]
1276
1277     # Verify the plans match, and that no changes were rejected.
1278     self.assertEqual(set(map(str, plans)), set(map(str, expected_plans)))
1279     self.assertEqual(0, remove.call_count)
1280
1281   def testPlans(self, max_txn_length=None):
1282     """Verify that independent sets are distinguished."""
1283     for num in range(0, 5):
1284       txns = [self.GetPatches(num) for _ in range(0, num)]
1285       self.verifyTransactions(txns, max_txn_length=max_txn_length)
1286
1287   def runUnresolvedPlan(self, changes, max_txn_length=None):
1288     """Helper for testing unresolved plans."""
1289     notify = self.PatchObject(validation_pool.ValidationPool,
1290                               'SendNotification')
1291     remove = self.PatchObject(gerrit.GerritHelper, 'RemoveCommitReady')
1292     pool = MakePool(changes=changes)
1293     plans = pool.CreateDisjointTransactions(None, max_txn_length=max_txn_length)
1294     self.assertEqual(plans, [])
1295     self.assertEqual(remove.call_count, notify.call_count)
1296     return remove.call_count
1297
1298   def testUnresolvedPlan(self):
1299     """Test plan with old approval_timestamp."""
1300     changes = self.GetPatches(5)[1:]
1301     with cros_test_lib.LoggingCapturer():
1302       call_count = self.runUnresolvedPlan(changes)
1303     self.assertEqual(4, call_count)
1304
1305   def testRecentUnresolvedPlan(self):
1306     """Test plan with recent approval_timestamp."""
1307     changes = self.GetPatches(5)[1:]
1308     for change in changes:
1309       change.approval_timestamp = time.time()
1310     with cros_test_lib.LoggingCapturer():
1311       call_count = self.runUnresolvedPlan(changes)
1312     self.assertEqual(0, call_count)
1313
1314   def testTruncatedPlan(self):
1315     """Test that plans can be truncated correctly."""
1316     # Long lists of patches should be truncated, and we should not see any
1317     # errors when this happens.
1318     self.testPlans(max_txn_length=3)
1319
1320   def testCircularPlans(self):
1321     """Verify that circular plans are handled correctly."""
1322     patches = self.GetPatches(5)
1323     self.patch_mock.SetGerritDependencies(patches[0], [patches[-1]])
1324
1325     # Verify that all patches can be submitted normally.
1326     self.verifyTransactions([patches], circular=True)
1327
1328     # It is not possible to truncate a circular plan. Verify that an error
1329     # is reported in this case.
1330     with cros_test_lib.LoggingCapturer():
1331       call_count = self.runUnresolvedPlan(patches, max_txn_length=3)
1332     self.assertEqual(5, call_count)
1333
1334
1335 class MockValidationPool(partial_mock.PartialMock):
1336   """Mock out a ValidationPool instance."""
1337
1338   TARGET = 'chromite.buildbot.validation_pool.ValidationPool'
1339   ATTRS = ('ReloadChanges', 'RemoveCommitReady', '_SubmitChange',
1340            'SendNotification')
1341
1342   def __init__(self):
1343     partial_mock.PartialMock.__init__(self)
1344     self.submit_results = {}
1345     self.max_submits = None
1346
1347   def GetSubmittedChanges(self):
1348     calls = []
1349     for call in self.patched['_SubmitChange'].call_args_list:
1350       call_args, _ = call
1351       calls.append(call_args[1])
1352     return calls
1353
1354   def _SubmitChange(self, _inst, change):
1355     result = self.submit_results.get(change, True)
1356     if isinstance(result, Exception):
1357       raise result
1358     if result and self.max_submits is not None:
1359       if self.max_submits <= 0:
1360         return False
1361       self.max_submits -= 1
1362     return result
1363
1364   @classmethod
1365   def ReloadChanges(cls, changes):
1366     return changes
1367
1368   RemoveCommitReady = None
1369   SendNotification = None
1370
1371
1372 class BaseSubmitPoolTestCase(Base, cros_build_lib_unittest.RunCommandTestCase):
1373   """Test full ability to submit and reject CL pools."""
1374
1375   def setUp(self):
1376     self.pool_mock = self.StartPatcher(MockValidationPool())
1377     self.patch_mock = self.StartPatcher(MockPatchSeries())
1378     self.PatchObject(gerrit.GerritHelper, 'QuerySingleRecord')
1379     self.patches = self.GetPatches(2)
1380
1381     # By default, don't ignore any errors.
1382     self.ignores = dict((patch, []) for patch in self.patches)
1383
1384   def SetUpPatchPool(self, failed_to_apply=False):
1385     pool = MakePool(changes=self.patches, dryrun=False)
1386     if failed_to_apply:
1387       # Create some phony errors and add them to the pool.
1388       errors = []
1389       for patch in self.GetPatches(2):
1390         errors.append(validation_pool.InternalCQError(patch, str('foo')))
1391       pool.changes_that_failed_to_apply_earlier = errors[:]
1392     return pool
1393
1394   def GetTracebacks(self):
1395     return []
1396
1397   def SubmitPool(self, submitted=(), rejected=(), **kwargs):
1398     """Helper function for testing that we can submit a pool successfully.
1399
1400     Args:
1401       submitted: List of changes that we expect to be submitted.
1402       rejected: List of changes that we expect to be rejected.
1403       **kwargs: Keyword arguments for SetUpPatchPool.
1404     """
1405     # self.ignores maps changes to a list of stages to ignore. Use it.
1406     self.PatchObject(
1407         validation_pool, 'GetStagesToIgnoreForChange',
1408         side_effect=lambda _, change: self.ignores[change])
1409
1410     # Set up our pool and submit the patches.
1411     pool = self.SetUpPatchPool(**kwargs)
1412     tracebacks = self.GetTracebacks()
1413     if tracebacks:
1414       actually_rejected = sorted(pool.SubmitPartialPool(self.GetTracebacks()))
1415     else:
1416       actually_rejected = pool.SubmitChanges(self.patches)
1417
1418     # Check that the right patches were submitted and rejected.
1419     self.assertItemsEqual(list(rejected), list(actually_rejected))
1420     actually_submitted = self.pool_mock.GetSubmittedChanges()
1421     self.assertEqual(list(submitted), actually_submitted)
1422
1423
1424 class SubmitPoolTest(BaseSubmitPoolTestCase):
1425   """Test suite related to the Submit Pool."""
1426
1427   def GetNotifyArg(self, change, key):
1428     """Look up a call to notify about |change| and grab |key| from it.
1429
1430     Args:
1431       change: The change to look up.
1432       key: The key to look up. If this is an integer, look up a positional
1433         argument by index. Otherwise, look up a keyword argument.
1434     """
1435     names = []
1436     for call in self.pool_mock.patched['SendNotification'].call_args_list:
1437       call_args, call_kwargs = call
1438       if change == call_args[1]:
1439         if isinstance(key, int):
1440           return call_args[key]
1441         return call_kwargs[key]
1442       names.append(call_args[1])
1443
1444     # Verify that |change| is present at all. This should always fail.
1445     self.assertIn(change, names)
1446
1447   def assertEqualNotifyArg(self, value, change, idx):
1448     """Verify that |value| equals self.GetNotifyArg(|change|, |idx|)."""
1449     self.assertEqual(str(value), str(self.GetNotifyArg(change, idx)))
1450
1451   def testSubmitPool(self):
1452     """Test that we can submit a pool successfully."""
1453     self.SubmitPool(submitted=self.patches)
1454
1455   def testRejectCLs(self):
1456     """Test that we can reject a CL successfully."""
1457     self.SubmitPool(submitted=self.patches, failed_to_apply=True)
1458
1459   def testSubmitCycle(self):
1460     """Submit a cyclic set of dependencies"""
1461     self.patch_mock.SetCQDependencies(self.patches[0], [self.patches[1]])
1462     self.SubmitPool(submitted=self.patches)
1463
1464   def testSubmitReverseCycle(self):
1465     """Submit a cyclic set of dependencies, specified in reverse order."""
1466     self.patch_mock.SetCQDependencies(self.patches[1], [self.patches[0]])
1467     self.patch_mock.SetGerritDependencies(self.patches[1], [])
1468     self.patch_mock.SetGerritDependencies(self.patches[0], [self.patches[1]])
1469     self.SubmitPool(submitted=self.patches[::-1])
1470
1471   def testRedundantCQDepend(self):
1472     """Submit a cycle with redundant CQ-DEPEND specifications."""
1473     self.patches = self.GetPatches(4)
1474     self.patch_mock.SetCQDependencies(self.patches[0], [self.patches[-1]])
1475     self.patch_mock.SetCQDependencies(self.patches[1], [self.patches[-1]])
1476     self.SubmitPool(submitted=self.patches)
1477
1478   def testSubmitPartialCycle(self):
1479     """Submit a failed cyclic set of dependencies"""
1480     self.pool_mock.max_submits = 1
1481     self.patch_mock.SetCQDependencies(self.patches[0], [self.patches[1]])
1482     self.SubmitPool(submitted=self.patches, rejected=[self.patches[1]])
1483     (submitted, rejected) = self.pool_mock.GetSubmittedChanges()
1484     failed_submit = validation_pool.PatchFailedToSubmit(
1485         rejected, validation_pool.ValidationPool.INCONSISTENT_SUBMIT_MSG)
1486     bad_submit = validation_pool.PatchSubmittedWithoutDeps(
1487         submitted, failed_submit)
1488     self.assertEqualNotifyArg(failed_submit, rejected, 'error')
1489     self.assertEqualNotifyArg(bad_submit, submitted, 'failure')
1490
1491   def testSubmitFailedCycle(self):
1492     """Submit a failed cyclic set of dependencies"""
1493     self.pool_mock.max_submits = 0
1494     self.patch_mock.SetCQDependencies(self.patches[0], [self.patches[1]])
1495     self.SubmitPool(submitted=[self.patches[0]], rejected=self.patches)
1496     (attempted,) = self.pool_mock.GetSubmittedChanges()
1497     (rejected,) = [x for x in self.patches if x != attempted]
1498     failed_submit = validation_pool.PatchFailedToSubmit(
1499         attempted, validation_pool.ValidationPool.INCONSISTENT_SUBMIT_MSG)
1500     dep_failed = cros_patch.DependencyError(rejected, failed_submit)
1501     self.assertEqualNotifyArg(failed_submit, attempted, 'error')
1502     self.assertEqualNotifyArg(dep_failed, rejected, 'error')
1503
1504   def testConflict(self):
1505     """Submit a change that conflicts with TOT."""
1506     error = gob_util.GOBError(httplib.CONFLICT, 'Conflict')
1507     self.pool_mock.submit_results[self.patches[0]] = error
1508     self.SubmitPool(submitted=[self.patches[0]], rejected=self.patches[::-1])
1509     notify_error = validation_pool.PatchConflict(self.patches[0])
1510     self.assertEqualNotifyArg(notify_error, self.patches[0], 'error')
1511
1512   def testServerError(self):
1513     """Test case where GOB returns a server error."""
1514     error = gerrit.GerritException('Internal server error')
1515     self.pool_mock.submit_results[self.patches[0]] = error
1516     self.SubmitPool(submitted=[self.patches[0]], rejected=self.patches[::-1])
1517     notify_error = validation_pool.PatchFailedToSubmit(self.patches[0], error)
1518     self.assertEqualNotifyArg(notify_error, self.patches[0], 'error')
1519
1520   def testDraftCL(self):
1521     """Test that a draft CL is rejected."""
1522     self.patches[1].patch_dict['currentPatchSet']['draft'] = True
1523     self.SubmitPool(submitted=self.patches[:1], rejected=self.patches[1:])
1524
1525   def testNotCommitReady(self):
1526     """Test that a CL without the commit ready bit is rejected."""
1527     self.PatchObject(self.patches[1], 'HasApproval', return_value=False)
1528     self.SubmitPool(submitted=self.patches[:1], rejected=self.patches[1:])
1529
1530   def testAlreadyMerged(self):
1531     """Test that a CL that was chumped during the run was not rejected."""
1532     self.PatchObject(self.patches[0], 'IsAlreadyMerged', return_value=True)
1533     self.SubmitPool(submitted=self.patches[1:], rejected=[])
1534
1535
1536 class SubmitPartialPoolTest(BaseSubmitPoolTestCase):
1537   """Test the SubmitPartialPool function."""
1538
1539   def setUp(self):
1540     # Set up each patch to be in its own project, so that we can easily
1541     # request to ignore failures for the specified patch.
1542     for patch in self.patches:
1543       patch.project = str(patch)
1544
1545     self.stage_name = 'MyHWTest'
1546
1547   def GetTracebacks(self):
1548     """Return a list containing a single traceback."""
1549     traceback = results_lib.RecordedTraceback(
1550         self.stage_name, self.stage_name, Exception(), '')
1551     return [traceback]
1552
1553   def IgnoreFailures(self, patch):
1554     """Set us up to ignore failures for the specified |patch|."""
1555     self.ignores[patch] = [self.stage_name]
1556
1557   def testSubmitNone(self):
1558     """Submit no changes."""
1559     self.SubmitPool(submitted=(), rejected=self.patches)
1560
1561   def testSubmitAll(self):
1562     """Submit all changes."""
1563     self.IgnoreFailures(self.patches[0])
1564     self.IgnoreFailures(self.patches[1])
1565     self.SubmitPool(submitted=self.patches, rejected=[])
1566
1567   def testSubmitFirst(self):
1568     """Submit the first change in a series."""
1569     self.IgnoreFailures(self.patches[0])
1570     self.SubmitPool(submitted=[self.patches[0]], rejected=[self.patches[1]])
1571
1572   def testSubmitSecond(self):
1573     """Attempt to submit the second change in a series."""
1574     self.IgnoreFailures(self.patches[1])
1575     self.SubmitPool(submitted=[], rejected=[self.patches[0]])
1576
1577
1578 if __name__ == '__main__':
1579   cros_test_lib.main()