src/third_party/chromite/buildbot/cbuildbot_stages_unittest.py

   1 #!/usr/bin/python
   2 # Copyright (c) 2012 The Chromium OS Authors. All rights reserved.
   3 # Use of this source code is governed by a BSD-style license that can be
   4 # found in the LICENSE file.
   5
   6 """Unittests for build stages."""
   7
   8 import contextlib
   9 import copy
  10 import cPickle
  11 import itertools
  12 import json
  13 import mox
  14 import os
  15 import signal
  16 import StringIO
  17 import sys
  18 import tempfile
  19 import time
  20 import unittest
  21
  22 import constants
  23 sys.path.insert(0, constants.SOURCE_ROOT)
  24 from chromite.buildbot import builderstage as bs
  25 from chromite.buildbot import cbuildbot_config as config
  26 from chromite.buildbot import cbuildbot_commands as commands
  27 from chromite.buildbot import cbuildbot_results as results_lib
  28 from chromite.buildbot import cbuildbot_run
  29 from chromite.buildbot import cbuildbot_stages as stages
  30 from chromite.buildbot import lab_status
  31 from chromite.buildbot import lkgm_manager
  32 from chromite.buildbot import manifest_version
  33 from chromite.buildbot import manifest_version_unittest
  34 from chromite.buildbot import portage_utilities
  35 from chromite.buildbot import repository
  36 from chromite.buildbot import validation_pool
  37 from chromite.lib import alerts
  38 from chromite.lib import cros_build_lib
  39 from chromite.lib import cros_build_lib_unittest
  40 from chromite.lib import cros_test_lib
  41 from chromite.lib import gerrit
  42 from chromite.lib import git
  43 from chromite.lib import git_unittest
  44 from chromite.lib import gob_util
  45 from chromite.lib import gs_unittest
  46 from chromite.lib import osutils
  47 from chromite.lib import parallel
  48 from chromite.lib import parallel_unittest
  49 from chromite.lib import partial_mock
  50 from chromite.lib import timeout_util
  51 from chromite.scripts import cbuildbot
  52
  53 try:
  54   # pylint: disable=F0401
  55   from crostools.lib import gspaths
  56   from crostools.lib import paygen_build_lib
  57   CROSTOOLS_AVAILABLE = True
  58 except ImportError:
  59   CROSTOOLS_AVAILABLE = False
  60
  61
  62 # TODO(build): Finish test wrapper (http://crosbug.com/37517).
  63 # Until then, this has to be after the chromite imports.
  64 import mock
  65
  66
  67 MANIFEST_CONTENTS = """\
  68 <?xml version="1.0" encoding="UTF-8"?>
  69 <manifest>
  70   <remote fetch="https://chromium.googlesource.com"
  71           name="cros"
  72           review="chromium-review.googlesource.com"/>
  73
  74   <default remote="cros" revision="refs/heads/master" sync-j="8"/>
  75
  76   <project groups="minilayout,buildtools"
  77            name="chromiumos/chromite"
  78            path="chromite"
  79            revision="refs/heads/special-branch"/>
  80
  81   <project name="chromiumos/special"
  82            path="src/special-new"
  83            revision="new-special-branch"/>
  84
  85   <project name="chromiumos/special"
  86            path="src/special-old"
  87            revision="old-special-branch"/>
  88
  89   <project name="faraway/external"
  90            path="external"
  91            revision="refs/heads/master"/>
  92
  93   <project name="faraway/unpinned"
  94            path="unpinned"
  95            revision="refs/heads/master"
  96            pin="False" />
  97
  98 </manifest>"""
  99
 100 CHROMITE_REVISION = "fb46d34d7cd4b9c167b74f494f2a99b68df50b18"
 101 SPECIAL_REVISION1 = "7bc42f093d644eeaf1c77fab60883881843c3c65"
 102 SPECIAL_REVISION2 = "6270eb3b4f78d9bffec77df50f374f5aae72b370"
 103
 104 VERSIONED_MANIFEST_CONTENTS = """\
 105 <?xml version="1.0" encoding="UTF-8"?>
 106 <manifest revision="fe72f0912776fa4596505e236e39286fb217961b">
 107   <remote fetch="https://chrome-internal.googlesource.com" name="chrome"/>
 108   <remote fetch="https://chromium.googlesource.com/" name="chromium"/>
 109   <remote fetch="https://chromium.googlesource.com" name="cros" \
 110 review="chromium-review.googlesource.com"/>
 111   <remote fetch="https://chrome-internal.googlesource.com" name="cros-internal" \
 112 review="https://chrome-internal-review.googlesource.com"/>
 113   <remote fetch="https://special.googlesource.com/" name="special" \
 114 review="https://special.googlesource.com/"/>
 115
 116   <default remote="cros" revision="refs/heads/master" sync-j="8"/>
 117
 118   <project name="chromeos/manifest-internal" path="manifest-internal" \
 119 remote="cros-internal" revision="fe72f0912776fa4596505e236e39286fb217961b" \
 120 upstream="refs/heads/master"/>
 121   <project groups="minilayout,buildtools" name="chromiumos/chromite" \
 122 path="chromite" revision="%(chromite_revision)s" \
 123 upstream="refs/heads/master"/>
 124   <project name="chromiumos/manifest" path="manifest" \
 125 revision="f24b69176b16bf9153f53883c0cc752df8e07d8b" \
 126 upstream="refs/heads/master"/>
 127   <project groups="minilayout" name="chromiumos/overlays/chromiumos-overlay" \
 128 path="src/third_party/chromiumos-overlay" \
 129 revision="3ac713c65b5d18585e606a0ee740385c8ec83e44" \
 130 upstream="refs/heads/master"/>
 131   <project name="chromiumos/special" path="src/special-new" \
 132 revision="%(special_revision1)s" \
 133 upstream="new-special-branch"/>
 134   <project name="chromiumos/special" path="src/special-old" \
 135 revision="%(special_revision2)s" \
 136 upstream="old-special-branch"/>
 137 </manifest>""" % dict(chromite_revision=CHROMITE_REVISION,
 138                       special_revision1=SPECIAL_REVISION1,
 139                       special_revision2=SPECIAL_REVISION2)
 140
 141
 142 DEFAULT_CHROME_BRANCH = '27'
 143
 144
 145 class BuilderRunMock(partial_mock.PartialMock):
 146   """Partial mock for BuilderRun class."""
 147
 148   TARGET = 'chromite.buildbot.cbuildbot_run._BuilderRunBase'
 149   ATTRS = ('GetVersionInfo', )
 150   VERSION = '3333.1.0'
 151
 152   def GetVersionInfo(self, _build_root):
 153     return manifest_version.VersionInfo(
 154         version_string=self.VERSION, chrome_branch=DEFAULT_CHROME_BRANCH)
 155
 156
 157 # pylint: disable=E1111,E1120,W0212,R0901,R0904
 158 class StageTest(cros_test_lib.MoxTempDirTestCase,
 159                 cros_test_lib.MockOutputTestCase):
 160   """Test running a single stage in isolation."""
 161
 162   TARGET_MANIFEST_BRANCH = 'ooga_booga'
 163   BUILDROOT = 'buildroot'
 164
 165   # Subclass should override this to default to a different build config
 166   # for its tests.
 167   BOT_ID = 'x86-generic-paladin'
 168
 169   # Subclasses can override this.  If non-None, value is inserted into
 170   # self.run.attrs.release_tag.
 171   RELEASE_TAG = None
 172
 173   def setUp(self):
 174     # Prepare a fake build root in self.tempdir, save at self.build_root.
 175     self.build_root = os.path.join(self.tempdir, self.BUILDROOT)
 176     osutils.SafeMakedirs(os.path.join(self.build_root, '.repo'))
 177
 178     self._manager = parallel.Manager()
 179     self._manager.__enter__()
 180
 181     # These are here to make pylint happy.  Values filled in by _Prepare.
 182     self.bot_id = None
 183     self._current_board = None
 184     self._boards = None
 185
 186   def _Prepare(self, bot_id=None, extra_config=None, cmd_args=None,
 187                extra_cmd_args=None):
 188     """Prepare a BuilderRun at self.run for this test.
 189
 190     This method must allow being called more than once.  Subclasses can
 191     override this method, but those subclass methods should also call this one.
 192
 193     The idea is that all test preparation that falls out from the choice of
 194     build config and cbuildbot options should go in _Prepare.
 195
 196     This will populate the following attributes on self:
 197       run: A BuilderRun object.
 198       bot_id: The bot id (name) that was used from config.config.
 199       self._boards: Same as self.run.config.boards.  TODO(mtennant): remove.
 200       self._current_board: First board in list, if there is one.
 201
 202     Args:
 203       bot_id: Name of build config to use, defaults to self.BOT_ID.
 204       extra_config: Dict used to add to the build config for the given
 205         bot_id.  Example: {'push_image': True}.
 206       cmd_args: List to override the default cbuildbot command args.
 207       extra_cmd_args: List to add to default cbuildbot command args.  This
 208         is a good way to adjust an options value for your test.
 209         Example: ['branch-name', 'some-branch-name'] will effectively cause
 210         self.run.options.branch_name to be set to 'some-branch-name'.
 211     """
 212     # Use cbuildbot parser to create options object and populate default values.
 213     parser = cbuildbot._CreateParser()
 214     if not cmd_args:
 215       # Fill in default command args.
 216       cmd_args = [
 217           '-r', self.build_root, '--buildbot', '--noprebuilts',
 218           '--buildnumber', '1234',
 219           '--branch', self.TARGET_MANIFEST_BRANCH,
 220       ]
 221     if extra_cmd_args:
 222       cmd_args += extra_cmd_args
 223     (options, args) = parser.parse_args(cmd_args)
 224
 225     # The bot_id can either be specified as arg to _Prepare method or in the
 226     # cmd_args (as cbuildbot normally accepts it from command line).
 227     if args:
 228       self.bot_id = args[0]
 229       if bot_id:
 230         # This means bot_id was specified as _Prepare arg and in cmd_args.
 231         # Make sure they are the same.
 232         self.assertEquals(self.bot_id, bot_id)
 233     else:
 234       self.bot_id = bot_id or self.BOT_ID
 235       args = [self.bot_id]
 236     cbuildbot._FinishParsing(options, args)
 237
 238     # Populate build_config corresponding to self.bot_id.
 239     build_config = copy.deepcopy(config.config[self.bot_id])
 240     build_config['manifest_repo_url'] = 'fake_url'
 241     if extra_config:
 242       build_config.update(extra_config)
 243     if options.remote_trybot:
 244       build_config = config.OverrideConfigForTrybot(build_config, options)
 245
 246     self._boards = build_config['boards']
 247     self._current_board = self._boards[0] if self._boards else None
 248
 249     # Some preliminary sanity checks.
 250     self.assertEquals(options.buildroot, self.build_root)
 251
 252     # Construct a real BuilderRun using options and build_config.
 253     self.run = cbuildbot_run.BuilderRun(options, build_config, self._manager)
 254
 255     if self.RELEASE_TAG is not None:
 256       self.run.attrs.release_tag = self.RELEASE_TAG
 257
 258     portage_utilities._OVERLAY_LIST_CMD = '/bin/true'
 259
 260   def tearDown(self):
 261     # Mimic exiting with statement for self._manager.
 262     self._manager.__exit__(None, None, None)
 263
 264   def AutoPatch(self, to_patch):
 265     """Patch a list of objects with autospec=True.
 266
 267     Args:
 268       to_patch: A list of tuples in the form (target, attr) to patch.  Will be
 269       directly passed to mock.patch.object.
 270     """
 271     for item in to_patch:
 272       self.PatchObject(*item, autospec=True)
 273
 274   def GetHWTestSuite(self):
 275     """Get the HW test suite for the current bot."""
 276     hw_tests = self.run.config['hw_tests']
 277     if not hw_tests:
 278       # TODO(milleral): Add HWTests back to lumpy-chrome-perf.
 279       raise unittest.SkipTest('Missing HWTest for %s' % (self.bot_id,))
 280
 281     return hw_tests[0]
 282
 283
 284 class AbstractStageTest(StageTest):
 285   """Base class for tests that test a particular build stage.
 286
 287   Abstract base class that sets up the build config and options with some
 288   default values for testing BuilderStage and its derivatives.
 289   """
 290
 291   def ConstructStage(self):
 292     """Returns an instance of the stage to be tested.
 293     Implement in subclasses.
 294     """
 295     raise NotImplementedError(self, "ConstructStage: Implement in your test")
 296
 297   def RunStage(self):
 298     """Creates and runs an instance of the stage to be tested.
 299     Requires ConstructStage() to be implemented.
 300
 301     Raises:
 302       NotImplementedError: ConstructStage() was not implemented.
 303     """
 304
 305     # Stage construction is usually done as late as possible because the tests
 306     # set up the build configuration and options used in constructing the stage.
 307     results_lib.Results.Clear()
 308     stage = self.ConstructStage()
 309     stage.Run()
 310     self.assertTrue(results_lib.Results.BuildSucceededSoFar())
 311
 312
 313 def patch(*args, **kwargs):
 314   """Convenience wrapper for mock.patch.object.
 315
 316   Sets autospec=True by default.
 317   """
 318   kwargs.setdefault('autospec', True)
 319   return mock.patch.object(*args, **kwargs)
 320
 321
 322 @contextlib.contextmanager
 323 def patches(*args):
 324   """Context manager for a list of patch objects."""
 325   with cros_build_lib.ContextManagerStack() as stack:
 326     for arg in args:
 327       stack.Add(lambda: arg)
 328     yield
 329
 330
 331 class BuilderStageTest(AbstractStageTest):
 332   """Tests for BuilderStage class."""
 333
 334   def setUp(self):
 335     self._Prepare()
 336
 337   def ConstructStage(self):
 338     return bs.BuilderStage(self.run)
 339
 340   def testGetPortageEnvVar(self):
 341     """Basic test case for _GetPortageEnvVar function."""
 342     self.mox.StubOutWithMock(cros_build_lib, 'RunCommand')
 343     envvar = 'EXAMPLE'
 344     obj = cros_test_lib.EasyAttr(output='RESULT\n')
 345     cros_build_lib.RunCommand(mox.And(mox.IsA(list), mox.In(envvar)),
 346                               cwd='%s/src/scripts' % self.build_root,
 347                               redirect_stdout=True, enter_chroot=True,
 348                               error_code_ok=True).AndReturn(obj)
 349     self.mox.ReplayAll()
 350
 351     stage = self.ConstructStage()
 352     board = self._current_board
 353     result = stage._GetPortageEnvVar(envvar, board)
 354     self.mox.VerifyAll()
 355
 356     self.assertEqual(result, 'RESULT')
 357
 358   def testStageNamePrefixSmoke(self):
 359     """Basic test for the StageNamePrefix() function."""
 360     stage = self.ConstructStage()
 361     self.assertEqual(stage.StageNamePrefix(), 'Builder')
 362
 363   def testGetStageNamesSmoke(self):
 364     """Basic test for the GetStageNames() function."""
 365     stage = self.ConstructStage()
 366     self.assertEqual(stage.GetStageNames(), ['Builder'])
 367
 368   def testConstructDashboardURLSmoke(self):
 369     """Basic test for the ConstructDashboardURL() function."""
 370     stage = self.ConstructStage()
 371
 372     exp_url = ('http://build.chromium.org/p/chromiumos/builders/'
 373                'x86-generic-paladin/builds/1234')
 374     self.assertEqual(stage.ConstructDashboardURL(), exp_url)
 375
 376     stage_name = 'Archive'
 377     exp_url = '%s/steps/%s/logs/stdio' % (exp_url, stage_name)
 378     self.assertEqual(stage.ConstructDashboardURL(stage=stage_name), exp_url)
 379
 380   def test_ExtractOverlaysSmoke(self):
 381     """Basic test for the _ExtractOverlays() function."""
 382     stage = self.ConstructStage()
 383     self.assertEqual(stage._ExtractOverlays(), ([], []))
 384
 385   def test_PrintSmoke(self):
 386     """Basic test for the _Print() function."""
 387     stage = self.ConstructStage()
 388     with self.OutputCapturer():
 389       stage._Print('hi there')
 390     self.AssertOutputContainsLine('hi there', check_stderr=True)
 391
 392   def test_PrintLoudlySmoke(self):
 393     """Basic test for the _PrintLoudly() function."""
 394     stage = self.ConstructStage()
 395     with self.OutputCapturer():
 396       stage._PrintLoudly('hi there')
 397     self.AssertOutputContainsLine(r'\*{10}', check_stderr=True)
 398     self.AssertOutputContainsLine('hi there', check_stderr=True)
 399
 400   def testRunSmoke(self):
 401     """Basic passing test for the Run() function."""
 402     stage = self.ConstructStage()
 403     with self.OutputCapturer():
 404       stage.Run()
 405
 406   def _RunCapture(self, stage):
 407     """Helper method to run Run() with captured output."""
 408     output = self.OutputCapturer()
 409     output.StartCapturing()
 410     try:
 411       stage.Run()
 412     finally:
 413       output.StopCapturing()
 414
 415   def testRunException(self):
 416     """Verify stage exceptions are handled."""
 417     class TestError(Exception):
 418       """Unique test exception"""
 419
 420     perform_mock = self.PatchObject(bs.BuilderStage, 'PerformStage')
 421     perform_mock.side_effect = TestError('fail!')
 422
 423     stage = self.ConstructStage()
 424     results_lib.Results.Clear()
 425     self.assertRaises(results_lib.StepFailure, self._RunCapture, stage)
 426
 427     results = results_lib.Results.Get()[0]
 428     self.assertTrue(isinstance(results.result, TestError))
 429     self.assertEqual(str(results.result), 'fail!')
 430
 431   def testHandleExceptionException(self):
 432     """Verify exceptions in HandleException handlers are themselves handled."""
 433     class TestError(Exception):
 434       """Unique test exception"""
 435
 436     class BadStage(bs.BuilderStage):
 437       """Stage that throws an exception when PerformStage is called."""
 438
 439       handled_exceptions = []
 440
 441       def PerformStage(self):
 442         raise TestError('first fail')
 443
 444       def _HandleStageException(self, exc_info):
 445         self.handled_exceptions.append(str(exc_info[1]))
 446         raise TestError('nested')
 447
 448     stage = BadStage(self.run)
 449     results_lib.Results.Clear()
 450     self.assertRaises(results_lib.StepFailure, self._RunCapture, stage)
 451
 452     # Verify the results tracked the original exception.
 453     results = results_lib.Results.Get()[0]
 454     self.assertTrue(isinstance(results.result, TestError))
 455     self.assertEqual(str(results.result), 'first fail')
 456
 457     self.assertEqual(stage.handled_exceptions, ['first fail'])
 458
 459
 460 class ManifestVersionedSyncStageTest(AbstractStageTest):
 461   """Tests the two (heavily related) stages ManifestVersionedSync, and
 462      ManifestVersionedSyncCompleted.
 463   """
 464   # pylint: disable=W0223
 465
 466   def setUp(self):
 467     self.source_repo = 'ssh://source/repo'
 468     self.manifest_version_url = 'fake manifest url'
 469     self.branch = 'master'
 470     self.build_name = 'x86-generic'
 471     self.incr_type = 'branch'
 472     self.next_version = 'next_version'
 473     self.sync_stage = None
 474
 475     repo = repository.RepoRepository(
 476       self.source_repo, self.tempdir, self.branch)
 477     self.manager = manifest_version.BuildSpecsManager(
 478       repo, self.manifest_version_url, self.build_name, self.incr_type,
 479       force=False, branch=self.branch, dry_run=True)
 480
 481     self._Prepare()
 482
 483   def _Prepare(self, bot_id=None, **kwargs):
 484     super(ManifestVersionedSyncStageTest, self)._Prepare(bot_id, **kwargs)
 485
 486     self.run.config['manifest_version'] = self.manifest_version_url
 487     self.sync_stage = stages.ManifestVersionedSyncStage(self.run)
 488     self.sync_stage.manifest_manager = self.manager
 489     self.run.attrs.manifest_manager = self.manager
 490
 491   def testManifestVersionedSyncOnePartBranch(self):
 492     """Tests basic ManifestVersionedSyncStage with branch ooga_booga"""
 493     self.mox.StubOutWithMock(stages.ManifestVersionedSyncStage,
 494                              'Initialize')
 495     self.mox.StubOutWithMock(manifest_version.BuildSpecsManager,
 496                              'GetNextBuildSpec')
 497     self.mox.StubOutWithMock(manifest_version.BuildSpecsManager,
 498                              'GetLatestPassingSpec')
 499     self.mox.StubOutWithMock(stages.SyncStage, 'ManifestCheckout')
 500
 501     stages.ManifestVersionedSyncStage.Initialize()
 502     self.manager.GetNextBuildSpec(
 503         dashboard_url=self.sync_stage.ConstructDashboardURL()
 504         ).AndReturn(self.next_version)
 505     self.manager.GetLatestPassingSpec().AndReturn(None)
 506
 507     stages.SyncStage.ManifestCheckout(self.next_version)
 508
 509     self.mox.ReplayAll()
 510     self.sync_stage.Run()
 511     self.mox.VerifyAll()
 512
 513   def testManifestVersionedSyncCompletedSuccess(self):
 514     """Tests basic ManifestVersionedSyncStageCompleted on success"""
 515     self.mox.StubOutWithMock(manifest_version.BuildSpecsManager, 'UpdateStatus')
 516
 517     self.manager.UpdateStatus(message=None, success=True,
 518                               dashboard_url=mox.IgnoreArg())
 519
 520     self.mox.ReplayAll()
 521     stage = stages.ManifestVersionedSyncCompletionStage(self.run,
 522                                                         self.sync_stage,
 523                                                         success=True)
 524     stage.Run()
 525     self.mox.VerifyAll()
 526
 527   def testManifestVersionedSyncCompletedFailure(self):
 528     """Tests basic ManifestVersionedSyncStageCompleted on failure"""
 529     self.mox.StubOutWithMock(manifest_version.BuildSpecsManager, 'UpdateStatus')
 530
 531     self.manager.UpdateStatus(message=None, success=False,
 532                               dashboard_url=mox.IgnoreArg())
 533
 534
 535     self.mox.ReplayAll()
 536     stage = stages.ManifestVersionedSyncCompletionStage(self.run,
 537                                                         self.sync_stage,
 538                                                         success=False)
 539     stage.Run()
 540     self.mox.VerifyAll()
 541
 542   def testManifestVersionedSyncCompletedIncomplete(self):
 543     """Tests basic ManifestVersionedSyncStageCompleted on incomplete build."""
 544     self.mox.ReplayAll()
 545     stage = stages.ManifestVersionedSyncCompletionStage(self.run,
 546                                                         self.sync_stage,
 547                                                         success=False)
 548     stage.Run()
 549     self.mox.VerifyAll()
 550
 551
 552 class CommitQueueCompletionStageTest(cros_test_lib.TestCase):
 553   """Test partial functionality of CommitQueueCompletionStage."""
 554
 555   def testSanityDetection(self):
 556     """Test the _WasBuildSane function."""
 557     sanity_slaves = ['sanity_1', 'sanity_2', 'sanity_3']
 558
 559     passed = manifest_version.BuilderStatus(
 560         manifest_version.BuilderStatus.STATUS_PASSED, '')
 561     failed = manifest_version.BuilderStatus(
 562         manifest_version.BuilderStatus.STATUS_FAILED, '')
 563     missing = manifest_version.BuilderStatus(
 564         manifest_version.BuilderStatus.STATUS_MISSING, '')
 565
 566     # If any sanity builder failed, build was not sane.
 567     slave_statuses = {'builder_a': passed,
 568                       'sanity_1' : missing,
 569                       'sanity_2' : passed,
 570                       'sanity_3' : failed}
 571     self.assertFalse(
 572         stages.CommitQueueCompletionStage._WasBuildSane(sanity_slaves,
 573                                                         slave_statuses))
 574
 575     # If some sanity builders did not report a status but the others passed,
 576     # then build was sane.
 577     slave_statuses = {'builder_a': passed,
 578                       'sanity_1' : missing,
 579                       'sanity_2' : passed}
 580
 581     self.assertTrue(
 582         stages.CommitQueueCompletionStage._WasBuildSane(sanity_slaves,
 583                                                         slave_statuses))
 584
 585     # If all sanity builders passed, build was sane.
 586     slave_statuses = {'builder_a': failed,
 587                       'sanity_1' : passed,
 588                       'sanity_2' : passed,
 589                       'sanity_3' : passed}
 590     self.assertTrue(
 591         stages.CommitQueueCompletionStage._WasBuildSane(sanity_slaves,
 592                                                         slave_statuses))
 593
 594
 595 class MasterSlaveSyncCompletionStage(AbstractStageTest):
 596   """Tests the two (heavily related) stages ManifestVersionedSync, and
 597      ManifestVersionedSyncCompleted.
 598   """
 599   BOT_ID = 'x86-generic-paladin'
 600
 601   def setUp(self):
 602     self.source_repo = 'ssh://source/repo'
 603     self.manifest_version_url = 'fake manifest url'
 604     self.branch = 'master'
 605     self.build_type = constants.PFQ_TYPE
 606
 607     self._Prepare()
 608
 609   def _Prepare(self, bot_id=None, **kwargs):
 610     super(MasterSlaveSyncCompletionStage, self)._Prepare(bot_id, **kwargs)
 611
 612     self.run.config['manifest_version'] = True
 613     self.run.config['build_type'] = self.build_type
 614     self.run.config['master'] = True
 615
 616   def ConstructStage(self):
 617     sync_stage = stages.MasterSlaveSyncStage(self.run)
 618     return stages.MasterSlaveSyncCompletionStage(self.run, sync_stage,
 619                                                  success=True)
 620
 621   def _GetTestConfig(self):
 622     test_config = {}
 623     test_config['test1'] = {
 624         'manifest_version': True,
 625         'build_type': constants.PFQ_TYPE,
 626         'overlays': 'public',
 627         'important': False,
 628         'chrome_rev': None,
 629         'branch': False,
 630         'internal': False,
 631         'master': False,
 632     }
 633     test_config['test2'] = {
 634         'manifest_version': False,
 635         'build_type': constants.PFQ_TYPE,
 636         'overlays': 'public',
 637         'important': True,
 638         'chrome_rev': None,
 639         'branch': False,
 640         'internal': False,
 641         'master': False,
 642     }
 643     test_config['test3'] = {
 644         'manifest_version': True,
 645         'build_type': constants.PFQ_TYPE,
 646         'overlays': 'both',
 647         'important': True,
 648         'chrome_rev': None,
 649         'branch': False,
 650         'internal': True,
 651         'master': False,
 652     }
 653     test_config['test4'] = {
 654         'manifest_version': True,
 655         'build_type': constants.PFQ_TYPE,
 656         'overlays': 'both',
 657         'important': True,
 658         'chrome_rev': None,
 659         'branch': True,
 660         'internal': True,
 661         'master': False,
 662     }
 663     test_config['test5'] = {
 664         'manifest_version': True,
 665         'build_type': constants.PFQ_TYPE,
 666         'overlays': 'public',
 667         'important': True,
 668         'chrome_rev': None,
 669         'branch': False,
 670         'internal': False,
 671         'master': False,
 672     }
 673     return test_config
 674
 675   def testGetSlavesForMaster(self):
 676     """Tests that we get the slaves for a fake unified master configuration."""
 677     orig_config = stages.cbuildbot_config.config
 678     try:
 679       stages.cbuildbot_config.config = test_config = self._GetTestConfig()
 680
 681       self.mox.ReplayAll()
 682
 683       stage = self.ConstructStage()
 684       p = stage._GetSlaveConfigs()
 685       self.mox.VerifyAll()
 686
 687       self.assertTrue(test_config['test3'] in p)
 688       self.assertTrue(test_config['test5'] in p)
 689       self.assertFalse(test_config['test1'] in p)
 690       self.assertFalse(test_config['test2'] in p)
 691       self.assertFalse(test_config['test4'] in p)
 692
 693     finally:
 694       stages.cbuildbot_config.config = orig_config
 695
 696   def testIsFailureFatal(self):
 697     """Tests the correctness of the _IsFailureFatal method"""
 698     stage = self.ConstructStage()
 699
 700     # Test behavior when there are no sanity check builders
 701     self.assertFalse(stage._IsFailureFatal(set(), set(), set()))
 702     self.assertTrue(stage._IsFailureFatal(set(['test3']), set(), set()))
 703     self.assertTrue(stage._IsFailureFatal(set(), set(['test5']), set()))
 704     self.assertTrue(stage._IsFailureFatal(set(), set(), set(['test1'])))
 705
 706     # Test behavior where there is a sanity check builder
 707     stage._run.config.sanity_check_slaves = ['sanity']
 708     self.assertTrue(stage._IsFailureFatal(set(['test5']), set(['sanity']),
 709                                           set()))
 710     self.assertFalse(stage._IsFailureFatal(set(), set(['sanity']), set()))
 711     self.assertTrue(stage._IsFailureFatal(set(), set(['sanity']),
 712                                           set(['test1'])))
 713     self.assertFalse(stage._IsFailureFatal(set(), set(),
 714                                            set(['sanity'])))
 715
 716   def testAnnotateFailingBuilders(self):
 717     """Tests that _AnnotateFailingBuilders is free of syntax errors."""
 718     stage = self.ConstructStage()
 719
 720     failing = {'a'}
 721     inflight = {}
 722     status = manifest_version.BuilderStatus('failed', 'message', 'url')
 723     statuses = {'a' : status}
 724     no_stat = set()
 725     stage._AnnotateFailingBuilders(failing, inflight, no_stat, statuses)
 726
 727   def testExceptionHandler(self):
 728     """Verify _HandleStageException is sane."""
 729     stage = self.ConstructStage()
 730     e = ValueError('foo')
 731     try:
 732       raise e
 733     except ValueError:
 734       ret = stage._HandleStageException(sys.exc_info())
 735       self.assertTrue(isinstance(ret, tuple))
 736       self.assertEqual(len(ret), 3)
 737       self.assertEqual(ret[0], e)
 738
 739
 740 # pylint: disable=W0223
 741 class RunCommandAbstractStageTest(AbstractStageTest,
 742                                   cros_build_lib_unittest.RunCommandTestCase):
 743   """Base test class for testing a stage and mocking RunCommand."""
 744
 745   FULL_BOT_ID = 'x86-generic-full'
 746   BIN_BOT_ID = 'x86-generic-paladin'
 747
 748   def _Prepare(self, bot_id, **kwargs):
 749     super(RunCommandAbstractStageTest, self)._Prepare(bot_id, **kwargs)
 750
 751   def _PrepareFull(self, **kwargs):
 752     self._Prepare(self.FULL_BOT_ID, **kwargs)
 753
 754   def _PrepareBin(self, **kwargs):
 755     self._Prepare(self.BIN_BOT_ID, **kwargs)
 756
 757   def _Run(self, dir_exists):
 758     """Helper for running the build."""
 759     with patch(os.path, 'isdir', return_value=dir_exists):
 760       self.RunStage()
 761
 762
 763 class InitSDKTest(RunCommandAbstractStageTest):
 764   """Test building the SDK"""
 765
 766   def setUp(self):
 767     self.PatchObject(cros_build_lib, 'GetChrootVersion', return_value='12')
 768
 769   def ConstructStage(self):
 770     return stages.InitSDKStage(self.run)
 771
 772   def testFullBuildWithExistingChroot(self):
 773     """Tests whether we create chroots for full builds."""
 774     self._PrepareFull()
 775     self._Run(dir_exists=True)
 776     self.assertCommandContains(['cros_sdk'])
 777
 778   def testBinBuildWithMissingChroot(self):
 779     """Tests whether we create chroots when needed."""
 780     self._PrepareBin()
 781     # Do not force chroot replacement in build config.
 782     self.run._config.chroot_replace = False
 783     self._Run(dir_exists=False)
 784     self.assertCommandContains(['cros_sdk'])
 785
 786   def testFullBuildWithMissingChroot(self):
 787     """Tests whether we create chroots when needed."""
 788     self._PrepareFull()
 789     self._Run(dir_exists=True)
 790     self.assertCommandContains(['cros_sdk'])
 791
 792   def testFullBuildWithNoSDK(self):
 793     """Tests whether the --nosdk option works."""
 794     self._PrepareFull(extra_cmd_args=['--nosdk'])
 795     self._Run(dir_exists=False)
 796     self.assertCommandContains(['cros_sdk', '--bootstrap'])
 797
 798   def testBinBuildWithExistingChroot(self):
 799     """Tests whether the --nosdk option works."""
 800     self._PrepareFull(extra_cmd_args=['--nosdk'])
 801     # Do not force chroot replacement in build config.
 802     self.run._config.chroot_replace = False
 803     self._Run(dir_exists=True)
 804     self.assertCommandContains(['cros_sdk'], expected=False)
 805
 806
 807 class SetupBoardTest(RunCommandAbstractStageTest):
 808   """Test building the board"""
 809
 810   def ConstructStage(self):
 811     return stages.SetupBoardStage(self.run, self._current_board)
 812
 813   def _RunFull(self, dir_exists=False):
 814     """Helper for testing a full builder."""
 815     self._Run(dir_exists)
 816     cmd = ['./setup_board', '--board=%s' % self._current_board, '--nousepkg']
 817     self.assertCommandContains(cmd, expected=not dir_exists)
 818     cmd = ['./setup_board', '--skip_chroot_upgrade']
 819     self.assertCommandContains(cmd, expected=False)
 820
 821   def testFullBuildWithProfile(self):
 822     """Tests whether full builds add profile flag when requested."""
 823     self._PrepareFull(extra_config={'profile': 'foo'})
 824     self._RunFull(dir_exists=False)
 825     self.assertCommandContains(['./setup_board', '--profile=foo'])
 826
 827   def testFullBuildWithOverriddenProfile(self):
 828     """Tests whether full builds add overridden profile flag when requested."""
 829     self._PrepareFull(extra_cmd_args=['--profile', 'smock'])
 830     self._RunFull(dir_exists=False)
 831     self.assertCommandContains(['./setup_board', '--profile=smock'])
 832
 833   def testFullBuildWithLatestToolchain(self):
 834     """Tests whether we use --nousepkg for creating the board"""
 835     self._PrepareFull()
 836     self._RunFull(dir_exists=False)
 837
 838   def _RunBin(self, dir_exists):
 839     """Helper for testing a binary builder."""
 840     self._Run(dir_exists)
 841     self.assertCommandContains(['./setup_board'])
 842     cmd = ['./setup_board', '--nousepkg']
 843     self.assertCommandContains(cmd, expected=self.run.options.latest_toolchain)
 844     cmd = ['./setup_board', '--skip_chroot_upgrade']
 845     self.assertCommandContains(cmd, expected=False)
 846
 847   def testBinBuildWithBoard(self):
 848     """Tests whether we don't create the board when it's there."""
 849     self._PrepareBin()
 850     self._RunBin(dir_exists=True)
 851
 852   def testBinBuildWithMissingBoard(self):
 853     """Tests whether we create the board when it's missing."""
 854     self._PrepareBin()
 855     self._RunBin(dir_exists=False)
 856
 857   def testBinBuildWithLatestToolchain(self):
 858     """Tests whether we use --nousepkg for creating the board."""
 859     self._PrepareBin()
 860     self._RunBin(dir_exists=False)
 861
 862   def testSDKBuild(self):
 863     """Tests whether we use --skip_chroot_upgrade for SDK builds."""
 864     extra_config = {'build_type': constants.CHROOT_BUILDER_TYPE}
 865     self._PrepareFull(extra_config=extra_config)
 866     self._Run(dir_exists=False)
 867     self.assertCommandContains(['./setup_board', '--skip_chroot_upgrade'])
 868
 869
 870 class SDKStageTest(AbstractStageTest):
 871   """Tests SDK package and Manifest creation."""
 872   fake_packages = [('cat1/package', '1'), ('cat1/package', '2'),
 873                    ('cat2/package', '3'), ('cat2/package', '4')]
 874   fake_json_data = {}
 875   fake_chroot = None
 876
 877   def setUp(self):
 878     # Replace SudoRunCommand, since we don't care about sudo.
 879     self._OriginalSudoRunCommand = cros_build_lib.SudoRunCommand
 880     cros_build_lib.SudoRunCommand = cros_build_lib.RunCommand
 881
 882     # Prepare a fake chroot.
 883     self.fake_chroot = os.path.join(self.build_root, 'chroot/build/amd64-host')
 884     osutils.SafeMakedirs(self.fake_chroot)
 885     osutils.Touch(os.path.join(self.fake_chroot, 'file'))
 886     for package, v in self.fake_packages:
 887       cpv = portage_utilities.SplitCPV('%s-%s' % (package, v))
 888       key = '%s/%s' % (cpv.category, cpv.package)
 889       self.fake_json_data.setdefault(key, []).append([v, {}])
 890
 891   def tearDown(self):
 892     cros_build_lib.SudoRunCommand = self._OriginalSudoRunCommand
 893
 894   def ConstructStage(self):
 895     return stages.SDKPackageStage(self.run)
 896
 897   def testTarballCreation(self):
 898     """Tests whether we package the tarball and correctly create a Manifest."""
 899     self._Prepare('chromiumos-sdk')
 900     fake_tarball = os.path.join(self.build_root, 'built-sdk.tar.xz')
 901     fake_manifest = os.path.join(self.build_root,
 902                                  'built-sdk.tar.xz.Manifest')
 903     self.mox.StubOutWithMock(portage_utilities, 'ListInstalledPackages')
 904     self.mox.StubOutWithMock(stages.SDKPackageStage,
 905                              'CreateRedistributableToolchains')
 906
 907     portage_utilities.ListInstalledPackages(self.fake_chroot).AndReturn(
 908         self.fake_packages)
 909     # This code has its own unit tests, so no need to go testing it here.
 910     stages.SDKPackageStage.CreateRedistributableToolchains(mox.IgnoreArg())
 911
 912     self.mox.ReplayAll()
 913     self.RunStage()
 914     self.mox.VerifyAll()
 915
 916     # Check tarball for the correct contents.
 917     output = cros_build_lib.RunCommand(
 918         ['tar', '-I', 'xz', '-tvf', fake_tarball],
 919         capture_output=True).output.splitlines()
 920     # First line is './', use it as an anchor, count the chars, and strip as
 921     # much from all other lines.
 922     stripchars = len(output[0]) - 1
 923     tar_lines = [x[stripchars:] for x in output]
 924     # TODO(ferringb): replace with assertIn.
 925     self.assertFalse('/build/amd64-host/' in tar_lines)
 926     self.assertTrue('/file' in tar_lines)
 927     # Verify manifest contents.
 928     real_json_data = json.loads(osutils.ReadFile(fake_manifest))
 929     self.assertEqual(real_json_data['packages'],
 930                      self.fake_json_data)
 931
 932
 933 class VMTestStageTest(AbstractStageTest):
 934   """Tests for the VMTest stage."""
 935
 936   BOT_ID = 'x86-generic-full'
 937   RELEASE_TAG = ''
 938
 939   def setUp(self):
 940     for cmd in ('RunTestSuite', 'CreateTestRoot', 'GenerateStackTraces',
 941                 'ArchiveFile', 'ArchiveTestResults', 'UploadArchivedFile',
 942                 'RunDevModeTest', 'RunCrosVMTest', 'ListFailedTests',
 943                 'GetTestResultsDir', 'BuildAndArchiveTestResultsTarball'):
 944       self.PatchObject(commands, cmd, autospec=True)
 945
 946     self.PatchObject(osutils, 'RmDir', autospec=True)
 947     self.PatchObject(os.path, 'isdir', autospec=True)
 948     self.PatchObject(os, 'listdir', autospec=True)
 949     self.StartPatcher(BuilderRunMock())
 950
 951     self._Prepare()
 952
 953     # Simulate breakpad symbols being ready.
 954     board_runattrs = self.run.GetBoardRunAttrs(self._current_board)
 955     board_runattrs.SetParallel('breakpad_symbols_generated', True)
 956
 957   def ConstructStage(self):
 958     self.run.GetArchive().SetupArchivePath()
 959     return stages.VMTestStage(self.run, self._current_board)
 960
 961   def testFullTests(self):
 962     """Tests if full unit and cros_au_test_harness tests are run correctly."""
 963     self.run.config['vm_tests'] = constants.FULL_AU_TEST_TYPE
 964     self.RunStage()
 965
 966   def testQuickTests(self):
 967     """Tests if quick unit and cros_au_test_harness tests are run correctly."""
 968     self.run.config['vm_tests'] = constants.SIMPLE_AU_TEST_TYPE
 969     self.RunStage()
 970
 971
 972 class UnitTestStageTest(AbstractStageTest):
 973   """Tests for the UnitTest stage."""
 974
 975   BOT_ID = 'x86-generic-full'
 976
 977   def setUp(self):
 978     self.mox.StubOutWithMock(commands, 'RunUnitTests')
 979     self.mox.StubOutWithMock(commands, 'TestAuZip')
 980
 981     self._Prepare()
 982
 983   def ConstructStage(self):
 984     return stages.UnitTestStage(self.run, self._current_board)
 985
 986   def testQuickTests(self):
 987     self.mox.StubOutWithMock(os.path, 'exists')
 988     self.run.config['quick_unit'] = True
 989     commands.RunUnitTests(self.build_root, self._current_board, full=False,
 990                           blacklist=[], extra_env=mox.IgnoreArg())
 991     image_dir = os.path.join(self.build_root,
 992                              'src/build/images/x86-generic/latest-cbuildbot')
 993     os.path.exists(os.path.join(image_dir,
 994                                 'au-generator.zip')).AndReturn(True)
 995     commands.TestAuZip(self.build_root, image_dir)
 996     self.mox.ReplayAll()
 997
 998     self.RunStage()
 999     self.mox.VerifyAll()
1000
1001   def testQuickTestsAuGeneratorZipMissing(self):
1002     self.mox.StubOutWithMock(os.path, 'exists')
1003     self.run.config['quick_unit'] = True
1004     commands.RunUnitTests(self.build_root, self._current_board, full=False,
1005                           blacklist=[], extra_env=mox.IgnoreArg())
1006     image_dir = os.path.join(self.build_root,
1007                              'src/build/images/x86-generic/latest-cbuildbot')
1008     os.path.exists(os.path.join(image_dir,
1009                                 'au-generator.zip')).AndReturn(False)
1010     self.mox.ReplayAll()
1011
1012     self.RunStage()
1013     self.mox.VerifyAll()
1014
1015   def testFullTests(self):
1016     """Tests if full unit and cros_au_test_harness tests are run correctly."""
1017     self.mox.StubOutWithMock(os.path, 'exists')
1018     self.run.config['quick_unit'] = False
1019     commands.RunUnitTests(self.build_root, self._current_board, full=True,
1020                           blacklist=[], extra_env=mox.IgnoreArg())
1021     image_dir = os.path.join(self.build_root,
1022                              'src/build/images/x86-generic/latest-cbuildbot')
1023     os.path.exists(os.path.join(image_dir,
1024                                 'au-generator.zip')).AndReturn(True)
1025     commands.TestAuZip(self.build_root, image_dir)
1026     self.mox.ReplayAll()
1027
1028     self.RunStage()
1029     self.mox.VerifyAll()
1030
1031
1032 class HWTestStageTest(AbstractStageTest):
1033   """Tests for the HWTest stage."""
1034
1035   BOT_ID = 'x86-mario-release'
1036   RELEASE_TAG = ''
1037
1038   def setUp(self):
1039     self.StartPatcher(BuilderRunMock())
1040
1041     self.mox.StubOutWithMock(lab_status, 'CheckLabStatus')
1042     self.mox.StubOutWithMock(commands, 'HaveCQHWTestsBeenAborted')
1043     self.mox.StubOutWithMock(commands, 'RunHWTestSuite')
1044     self.mox.StubOutWithMock(cros_build_lib, 'PrintBuildbotStepWarnings')
1045     self.mox.StubOutWithMock(cros_build_lib, 'PrintBuildbotStepFailure')
1046     self.mox.StubOutWithMock(cros_build_lib, 'Warning')
1047     self.mox.StubOutWithMock(cros_build_lib, 'Error')
1048
1049     self.suite_config = None
1050     self.suite = None
1051
1052     self._Prepare()
1053
1054   def _Prepare(self, bot_id=None, **kwargs):
1055     super(HWTestStageTest, self)._Prepare(bot_id, **kwargs)
1056
1057     self.run.options.log_dir = '/b/cbuild/mylogdir'
1058
1059     self.suite_config = self.GetHWTestSuite()
1060     self.suite = self.suite_config.suite
1061
1062   def ConstructStage(self):
1063     self.run.GetArchive().SetupArchivePath()
1064     return stages.HWTestStage(self.run, self._current_board, self.suite_config)
1065
1066   def _RunHWTestSuite(self, debug=False, returncode=0, fails=False,
1067                       timeout=False):
1068     """Pretend to run the HWTest suite to assist with tests.
1069
1070     Args:
1071       debug: Whether the HWTest suite should be run in debug mode.
1072       returncode: The return value of the HWTest command.
1073       fails: Whether the command as a whole should fail.
1074       timeout: Whether the the hw tests should time out.
1075     """
1076     if config.IsCQType(self.run.config.build_type):
1077       version = self.run.GetVersion()
1078       for _ in range(1 + int(fails)):
1079         commands.HaveCQHWTestsBeenAborted(version).AndReturn(False)
1080
1081     lab_status.CheckLabStatus(mox.IgnoreArg())
1082     m = commands.RunHWTestSuite(mox.IgnoreArg(),
1083                                 self.suite,
1084                                 self._current_board, mox.IgnoreArg(),
1085                                 mox.IgnoreArg(), mox.IgnoreArg(), True,
1086                                 mox.IgnoreArg(), mox.IgnoreArg(), debug)
1087
1088     # Raise an exception if the user wanted the command to fail.
1089     if timeout:
1090       m.AndRaise(timeout_util.TimeoutError('Timed out'))
1091       cros_build_lib.PrintBuildbotStepFailure()
1092       cros_build_lib.Error(mox.IgnoreArg())
1093     elif returncode != 0:
1094       result = cros_build_lib.CommandResult(cmd='run_hw_tests',
1095                                             returncode=returncode)
1096       m.AndRaise(cros_build_lib.RunCommandError('HWTests failed', result))
1097
1098       # Make sure failures are logged correctly.
1099       if fails:
1100         cros_build_lib.PrintBuildbotStepFailure()
1101         cros_build_lib.Error(mox.IgnoreArg())
1102       else:
1103         cros_build_lib.Warning(mox.IgnoreArg())
1104         cros_build_lib.PrintBuildbotStepWarnings()
1105         cros_build_lib.Warning(mox.IgnoreArg())
1106
1107     self.mox.ReplayAll()
1108     if fails or timeout:
1109       self.assertRaises(results_lib.StepFailure, self.RunStage)
1110     else:
1111       self.RunStage()
1112     self.mox.VerifyAll()
1113
1114   def testRemoteTrybotWithHWTest(self):
1115     """Test remote trybot with hw test enabled"""
1116     cmd_args = ['--remote-trybot', '-r', self.build_root, '--hwtest']
1117     self._Prepare(cmd_args=cmd_args)
1118     self._RunHWTestSuite()
1119
1120   def testRemoteTrybotNoHWTest(self):
1121     """Test remote trybot with no hw test"""
1122     cmd_args = ['--remote-trybot', '-r', self.build_root]
1123     self._Prepare(cmd_args=cmd_args)
1124     self._RunHWTestSuite(debug=True)
1125
1126   def testWithSuite(self):
1127     """Test if run correctly with a test suite."""
1128     self._RunHWTestSuite()
1129
1130   def testWithTimeout(self):
1131     """Test if run correctly with a critical timeout."""
1132     self._Prepare('x86-alex-paladin')
1133     self._RunHWTestSuite(timeout=True)
1134
1135   def testWithSuiteWithInfrastructureFailure(self):
1136     """Tests that we warn correctly if we get a returncode of 2."""
1137     self._RunHWTestSuite(returncode=2)
1138
1139   def testWithSuiteWithFatalFailure(self):
1140     """Tests that we fail if we get a returncode of 1."""
1141     self._RunHWTestSuite(returncode=1, fails=True)
1142
1143   def testSendPerfResults(self):
1144     """Tests that we can send perf results back correctly."""
1145     self._Prepare('lumpy-chrome-perf')
1146     self.suite = 'perf_v2'
1147
1148     self.mox.StubOutWithMock(stages.HWTestStage, '_PrintFile')
1149
1150     with gs_unittest.GSContextMock() as gs_mock:
1151       gs_mock.SetDefaultCmdResult()
1152       self._RunHWTestSuite()
1153
1154   def testHandleLabDownAsWarning(self):
1155     """Test that buildbot warn when lab is down."""
1156     cros_build_lib.Warning(mox.IgnoreArg())
1157     check_lab = lab_status.CheckLabStatus(mox.IgnoreArg())
1158     check_lab.AndRaise(lab_status.LabIsDownException('Lab is not up.'))
1159     cros_build_lib.PrintBuildbotStepWarnings()
1160     cros_build_lib.Warning(mox.IgnoreArg())
1161     self.mox.ReplayAll()
1162     self.RunStage()
1163     self.mox.VerifyAll()
1164
1165
1166 class SignerResultsStageTest(AbstractStageTest):
1167   """Test the SignerResultsStage."""
1168
1169   BOT_ID = 'x86-mario-release'
1170   RELEASE_TAG = '0.0.1'
1171
1172   def setUp(self):
1173     self.StartPatcher(BuilderRunMock())
1174     self._Prepare()
1175
1176     self.signer_result = """
1177       { "status": { "status": "passed" }, "board": "link",
1178       "keyset": "link-mp-v4", "type": "recovery", "channel": "stable" }
1179       """
1180
1181     self.insns_urls_per_channel = {
1182         'chan1': ['chan1_uri1', 'chan1_uri2'],
1183         'chan2': ['chan2_uri1'],
1184     }
1185
1186
1187   def ConstructStage(self):
1188     archive_stage = stages.ArchiveStage(self.run, self._current_board)
1189     stage = stages.SignerResultsStage(
1190         self.run, self._current_board, archive_stage)
1191
1192     return stage
1193
1194   def testPerformStageSuccess(self):
1195     """Test that SignerResultsStage works when signing works."""
1196     results = ['chan1_uri1.json', 'chan1_uri2.json', 'chan2_uri1.json']
1197
1198     with patch(stages.gs, 'GSContext') as mock_gs_ctx_init:
1199       mock_gs_ctx = mock_gs_ctx_init.return_value
1200       mock_gs_ctx.Cat.return_value.output = self.signer_result
1201
1202       stage = self.ConstructStage()
1203       stage.archive_stage._push_image_status_queue.put(
1204           self.insns_urls_per_channel)
1205
1206       stage.PerformStage()
1207       for result in results:
1208         mock_gs_ctx.Cat.assert_any_call(result)
1209
1210       self.assertEqual(stage.archive_stage.WaitForChannelSigning(), 'chan1')
1211       self.assertEqual(stage.archive_stage.WaitForChannelSigning(), 'chan2')
1212       self.assertEqual(stage.archive_stage.WaitForChannelSigning(),
1213                        stages.SignerResultsStage.FINISHED)
1214
1215   def testPerformStageSuccessNothingSigned(self):
1216     """Test that SignerResultsStage passes when there are no signed images."""
1217     with patch(stages.gs, 'GSContext') as mock_gs_ctx_init:
1218       mock_gs_ctx = mock_gs_ctx_init.return_value
1219       mock_gs_ctx.Cat.return_value.output = self.signer_result
1220
1221       stage = self.ConstructStage()
1222       stage.archive_stage._push_image_status_queue.put({})
1223
1224       stage.PerformStage()
1225       self.assertFalse(mock_gs_ctx.Cat.called)
1226       self.assertEqual(stage.archive_stage.WaitForChannelSigning(),
1227                        stages.SignerResultsStage.FINISHED)
1228
1229   def testPerformStageFailure(self):
1230     """Test that SignerResultsStage errors when the signers report an error."""
1231     with patch(stages.gs, 'GSContext') as mock_gs_ctx_init:
1232       mock_gs_ctx = mock_gs_ctx_init.return_value
1233       mock_gs_ctx.Cat.return_value.output = """
1234           { "status": { "status": "failed" }, "board": "link",
1235             "keyset": "link-mp-v4", "type": "recovery", "channel": "stable" }
1236           """
1237       stage = self.ConstructStage()
1238       stage.archive_stage._push_image_status_queue.put({
1239           'chan1': ['chan1_uri1'],
1240       })
1241       self.assertRaises(stages.SignerFailure, stage.PerformStage)
1242
1243   def testPerformStageMalformedJson(self):
1244     """Test that SignerResultsStage errors when invalid Json is received.."""
1245     with patch(stages.gs, 'GSContext') as mock_gs_ctx_init:
1246       mock_gs_ctx = mock_gs_ctx_init.return_value
1247       mock_gs_ctx.Cat.return_value.output = "{"
1248
1249       stage = self.ConstructStage()
1250       stage.archive_stage._push_image_status_queue.put({
1251           'chan1': ['chan1_uri1'],
1252       })
1253       self.assertRaises(stages.MalformedResultsException, stage.PerformStage)
1254
1255   def testPerformStageTimeout(self):
1256     """Test that SignerResultsStage reports timeouts correctly."""
1257     with patch(stages.timeout_util, 'WaitForSuccess') as mock_wait:
1258       mock_wait.side_effect = timeout_util.TimeoutError
1259
1260       stage = self.ConstructStage()
1261       stage.archive_stage._push_image_status_queue.put({
1262           'chan1': ['chan1_uri1'],
1263       })
1264       self.assertRaises(stages.SignerResultsTimeout, stage.PerformStage)
1265
1266   def testCheckForResultsSuccess(self):
1267     """Test that SignerResultsStage works when signing works."""
1268     with patch(stages.gs, 'GSContext') as mock_gs_ctx_init:
1269       mock_gs_ctx = mock_gs_ctx_init.return_value
1270       mock_gs_ctx.Cat.return_value.output = self.signer_result
1271
1272       stage = self.ConstructStage()
1273       self.assertTrue(
1274           stage._CheckForResults(mock_gs_ctx,
1275           self.insns_urls_per_channel))
1276
1277   def testCheckForResultsSuccessNoChannels(self):
1278     """Test that SignerResultsStage works when there is nothing to check for."""
1279     with patch(stages.gs, 'GSContext') as mock_gs_ctx_init:
1280       mock_gs_ctx = mock_gs_ctx_init.return_value
1281
1282       stage = self.ConstructStage()
1283
1284       # Ensure we find that we are ready if there are no channels to look for.
1285       self.assertTrue(stage._CheckForResults(mock_gs_ctx, {}))
1286
1287       # Ensure we didn't contact GS while checking for no channels.
1288       self.assertFalse(mock_gs_ctx.Cat.called)
1289
1290   def testCheckForResultsPartialComplete(self):
1291     """Verify _CheckForResults handles partial signing results."""
1292     def catChan2Success(url):
1293       if url.startswith('chan2'):
1294         result = mock.Mock()
1295         result.output = self.signer_result
1296         return result
1297       else:
1298         raise stages.gs.GSNoSuchKey()
1299
1300     with patch(stages.gs, 'GSContext') as mock_gs_ctx_init:
1301       mock_gs_ctx = mock_gs_ctx_init.return_value
1302       mock_gs_ctx.Cat.side_effect = catChan2Success
1303
1304       stage = self.ConstructStage()
1305       self.assertFalse(
1306           stage._CheckForResults(mock_gs_ctx,
1307           self.insns_urls_per_channel))
1308       self.assertEqual(stage.signing_results, {
1309           'chan1': {},
1310           'chan2': {
1311               'chan2_uri1.json': {
1312                   'board': 'link',
1313                   'channel': 'stable',
1314                   'keyset': 'link-mp-v4',
1315                   'status': {'status': 'passed'},
1316                   'type': 'recovery'
1317               }
1318           }
1319       })
1320
1321   def testCheckForResultsUnexpectedJson(self):
1322     """Verify _CheckForResults handles unexpected Json values."""
1323     with patch(stages.gs, 'GSContext') as mock_gs_ctx_init:
1324       mock_gs_ctx = mock_gs_ctx_init.return_value
1325       mock_gs_ctx.Cat.return_value.output = "{}"
1326
1327       stage = self.ConstructStage()
1328       self.assertFalse(
1329           stage._CheckForResults(mock_gs_ctx,
1330           self.insns_urls_per_channel))
1331       self.assertEqual(stage.signing_results, {
1332           'chan1': {}, 'chan2': {}
1333       })
1334
1335   def testCheckForResultsNoResult(self):
1336     """Verify _CheckForResults handles missing signer results."""
1337     with patch(stages.gs, 'GSContext') as mock_gs_ctx_init:
1338       mock_gs_ctx = mock_gs_ctx_init.return_value
1339       mock_gs_ctx.Cat.side_effect = stages.gs.GSNoSuchKey
1340
1341       stage = self.ConstructStage()
1342       self.assertFalse(
1343           stage._CheckForResults(mock_gs_ctx,
1344           self.insns_urls_per_channel))
1345       self.assertEqual(stage.signing_results, {
1346           'chan1': {}, 'chan2': {}
1347       })
1348
1349
1350 class PaygenStageTest(StageTest):
1351   """Test the PaygenStageStage."""
1352
1353   BOT_ID = 'x86-mario-release'
1354   RELEASE_TAG = '0.0.1'
1355
1356   def setUp(self):
1357     self.StartPatcher(BuilderRunMock())
1358     self._Prepare()
1359
1360   def ConstructStage(self):
1361     archive_stage = stages.ArchiveStage(self.run, self._current_board)
1362     return stages.PaygenStage(self.run, self._current_board, archive_stage)
1363
1364   def testPerformStageSuccess(self):
1365     """Test that SignerResultsStage works when signing works."""
1366
1367     with patch(stages.parallel, 'BackgroundTaskRunner') as background:
1368       queue = background().__enter__()
1369
1370       stage = self.ConstructStage()
1371       stage.archive_stage.AnnounceChannelSigned('stable')
1372       stage.archive_stage.AnnounceChannelSigned('beta')
1373       stage.archive_stage.AnnounceChannelSigned(
1374           stages.SignerResultsStage.FINISHED)
1375       stage.PerformStage()
1376
1377       # Verify that we queue up work
1378       queue.put.assert_any_call(('stable', 'x86-mario', '0.0.1', False, True))
1379       queue.put.assert_any_call(('beta', 'x86-mario', '0.0.1', False, True))
1380
1381   def testPerformStageNoChannels(self):
1382     """Test that SignerResultsStage works when signing works."""
1383     with patch(stages.parallel, 'BackgroundTaskRunner') as background:
1384       queue = background().__enter__()
1385
1386       stage = self.ConstructStage()
1387       stage.archive_stage.AnnounceChannelSigned(
1388           stages.SignerResultsStage.FINISHED)
1389       stage.PerformStage()
1390
1391       # Ensure no work was queued up.
1392       self.assertFalse(queue.put.called)
1393
1394   def testPerformSigningFailed(self):
1395     """Test that SignerResultsStage works when signing works."""
1396     with patch(stages.parallel, 'BackgroundTaskRunner') as background:
1397       queue = background().__enter__()
1398
1399       stage = self.ConstructStage()
1400       stage.archive_stage.AnnounceChannelSigned(None)
1401
1402       self.assertRaises(stages.PaygenSigningRequirementsError,
1403                         stage.PerformStage)
1404
1405       # Ensure no work was queued up.
1406       self.assertFalse(queue.put.called)
1407
1408   def testPerformTrybot(self):
1409     """Test the PerformStage alternate behavior for trybot runs."""
1410     with patch(stages.parallel, 'BackgroundTaskRunner') as background:
1411       queue = background().__enter__()
1412
1413       # The stage is constructed differently for trybots, so don't use
1414       # ConstructStage.
1415       stage = stages.PaygenStage(self.run, self._current_board,
1416                                  archive_stage=None, channels=['foo', 'bar'])
1417       stage.PerformStage()
1418
1419       # Notice that we didn't put anything in _wait_for_channel_signing, but
1420       # still got results right away.
1421       queue.put.assert_any_call(('foo', 'x86-mario', '0.0.1', False, True))
1422       queue.put.assert_any_call(('bar', 'x86-mario', '0.0.1', False, True))
1423
1424   @unittest.skipIf(not CROSTOOLS_AVAILABLE,
1425                    'Internal crostools repository needed.')
1426   def testRunPaygenInProcess(self):
1427     """Test that SignerResultsStage works when signing works."""
1428     with patch(paygen_build_lib, 'CreatePayloads') as create_payloads:
1429       # Call the method under test.
1430       stage = self.ConstructStage()
1431       stage._RunPaygenInProcess('foo', 'foo-board', 'foo-version', False, True)
1432
1433       # Ensure arguments are properly converted and passed along.
1434       create_payloads.assert_called_with(gspaths.Build(version='foo-version',
1435                                                        board='foo-board',
1436                                                        channel='foo-channel'),
1437                                          dry_run=False,
1438                                          work_dir=mock.ANY,
1439                                          run_parallel=True,
1440                                          run_on_builder=True,
1441                                          skip_test_payloads=False,
1442                                          skip_autotest=False)
1443
1444   @unittest.skipIf(not CROSTOOLS_AVAILABLE,
1445                    'Internal crostools repository needed.')
1446   def testRunPaygenInProcessComplex(self):
1447     """Test that SignerResultsStage with arguments that are more unusual."""
1448     with patch(paygen_build_lib, 'CreatePayloads') as create_payloads:
1449       # Call the method under test.
1450       # Use release tools channel naming, and a board name including a variant.
1451       stage = self.ConstructStage()
1452       stage._RunPaygenInProcess('foo-channel', 'foo-board_variant',
1453                                 'foo-version', True, False)
1454
1455       # Ensure arguments are properly converted and passed along.
1456       create_payloads.assert_called_with(
1457           gspaths.Build(version='foo-version',
1458                         board='foo-board-variant',
1459                         channel='foo-channel'),
1460           dry_run=True,
1461           work_dir=mock.ANY,
1462           run_parallel=True,
1463           run_on_builder=True,
1464           skip_test_payloads=True,
1465           skip_autotest=True)
1466
1467
1468 class AUTestStageTest(AbstractStageTest,
1469                       cros_build_lib_unittest.RunCommandTestCase):
1470   """Test only custom methods in AUTestStageTest."""
1471   BOT_ID = 'x86-mario-release'
1472   RELEASE_TAG = '0.0.1'
1473
1474   def setUp(self):
1475     self.StartPatcher(BuilderRunMock())
1476     self.PatchObject(commands, 'ArchiveFile', autospec=True,
1477                      return_value='foo.txt')
1478     self.PatchObject(commands, 'HaveCQHWTestsBeenAborted', autospec=True,
1479                      return_value=False)
1480     self.PatchObject(lab_status, 'CheckLabStatus', autospec=True)
1481
1482     self.archive_stage = None
1483     self.suite_config = None
1484     self.suite = None
1485
1486     self._Prepare()
1487
1488   def _Prepare(self, bot_id=None, **kwargs):
1489     super(AUTestStageTest, self)._Prepare(bot_id, **kwargs)
1490
1491     self.run.GetArchive().SetupArchivePath()
1492     self.archive_stage = stages.ArchiveStage(self.run, self._current_board)
1493     self.suite_config = self.GetHWTestSuite()
1494     self.suite = self.suite_config.suite
1495
1496   def ConstructStage(self):
1497     return stages.AUTestStage(self.run, self._current_board, self.suite_config)
1498
1499   def testPerformStage(self):
1500     """Tests that we correctly generate a tarball and archive it."""
1501     stage = self.ConstructStage()
1502     stage.PerformStage()
1503     cmd = ['site_utils/autoupdate/full_release_test.py', '--npo', '--dump',
1504            '--archive_url', self.archive_stage.upload_url,
1505            self.archive_stage.release_tag, self._current_board]
1506     self.assertCommandContains(cmd)
1507     self.assertCommandContains([commands._AUTOTEST_RPC_CLIENT, self.suite])
1508
1509
1510 class UprevStageTest(AbstractStageTest):
1511   """Tests for the UprevStage class."""
1512
1513   def setUp(self):
1514     self.mox.StubOutWithMock(commands, 'UprevPackages')
1515
1516     self._Prepare()
1517
1518   def ConstructStage(self):
1519     return stages.UprevStage(self.run)
1520
1521   def testBuildRev(self):
1522     """Uprevving the build without uprevving chrome."""
1523     self.run.config['uprev'] = True
1524     commands.UprevPackages(self.build_root, self._boards, [], enter_chroot=True)
1525     self.mox.ReplayAll()
1526     self.RunStage()
1527     self.mox.VerifyAll()
1528
1529   def testNoRev(self):
1530     """No paths are enabled."""
1531     self.run.config['uprev'] = False
1532     self.mox.ReplayAll()
1533     self.RunStage()
1534     self.mox.VerifyAll()
1535
1536
1537 class ArchivingMock(partial_mock.PartialMock):
1538   """Partial mock for ArchivingStage."""
1539
1540   TARGET = 'chromite.buildbot.cbuildbot_stages.ArchivingStage'
1541   ATTRS = ('UploadArtifact',)
1542
1543   def UploadArtifact(self, *args, **kwargs):
1544     with patch(commands, 'ArchiveFile', return_value='foo.txt'):
1545       with patch(commands, 'UploadArchivedFile'):
1546         self.backup['UploadArtifact'](*args, **kwargs)
1547
1548
1549 class BuildPackagesStageTest(AbstractStageTest):
1550   """Tests BuildPackagesStage."""
1551
1552   def setUp(self):
1553     self._release_tag = None
1554
1555     self.StartPatcher(BuilderRunMock())
1556
1557   def ConstructStage(self):
1558     self.run.attrs.release_tag = self._release_tag
1559     return stages.BuildPackagesStage(self.run, self._current_board)
1560
1561   @contextlib.contextmanager
1562   def RunStageWithConfig(self):
1563     """Run the given config"""
1564     try:
1565       with cros_build_lib_unittest.RunCommandMock() as rc:
1566         rc.SetDefaultCmdResult()
1567         with cros_test_lib.OutputCapturer():
1568           with cros_test_lib.LoggingCapturer():
1569             self.RunStage()
1570
1571         yield rc
1572
1573     except AssertionError as ex:
1574       msg = '%s failed the following test:\n%s' % (self.bot_id, ex)
1575       raise AssertionError(msg)
1576
1577   def RunTestsWithBotId(self, bot_id, options_tests=True):
1578     """Test with the config for the specified bot_id."""
1579     self._Prepare(bot_id)
1580     self.run.options.tests = options_tests
1581
1582     with self.RunStageWithConfig() as rc:
1583       cfg = self.run.config
1584       rc.assertCommandContains(['./build_packages'])
1585       rc.assertCommandContains(['./build_packages', '--skip_chroot_upgrade'])
1586       rc.assertCommandContains(['./build_packages', '--nousepkg'],
1587                                expected=not cfg['usepkg_build_packages'])
1588       build_tests = cfg['build_tests'] and self.run.options.tests
1589       rc.assertCommandContains(['./build_packages', '--nowithautotest'],
1590                                expected=not build_tests)
1591
1592   def testAllConfigs(self):
1593     """Test all major configurations"""
1594     task = self.RunTestsWithBotId
1595     with parallel.BackgroundTaskRunner(task) as queue:
1596       # Loop through all major configuration types and pick one from each.
1597       for bot_type in config.CONFIG_TYPE_DUMP_ORDER:
1598         for bot_id in config.config:
1599           if bot_id.endswith(bot_type):
1600             # Skip any config without a board, since those configs do not
1601             # build packages.
1602             cfg = config.config[bot_id]
1603             if cfg.boards:
1604               queue.put([bot_id])
1605               break
1606
1607   def testNoTests(self):
1608     """Test that self.options.tests = False works."""
1609     self.RunTestsWithBotId('x86-generic-paladin', options_tests=False)
1610
1611
1612 class BuildImageStageMock(ArchivingMock):
1613   """Partial mock for BuildImageStage."""
1614
1615   TARGET = 'chromite.buildbot.cbuildbot_stages.BuildImageStage'
1616   ATTRS = ArchivingMock.ATTRS + ('_BuildImages', '_GenerateAuZip')
1617
1618   def _BuildImages(self, *args, **kwargs):
1619     with patches(
1620         patch(os, 'symlink'),
1621         patch(os, 'readlink', return_value='foo.txt')):
1622       self.backup['_BuildImages'](*args, **kwargs)
1623
1624   def _GenerateAuZip(self, *args, **kwargs):
1625     with patch(git, 'ReinterpretPathForChroot', return_value='/chroot/path'):
1626       self.backup['_GenerateAuZip'](*args, **kwargs)
1627
1628
1629 class BuildImageStageTest(BuildPackagesStageTest):
1630   """Tests BuildImageStage."""
1631
1632   def setUp(self):
1633     self.StartPatcher(BuildImageStageMock())
1634
1635   def ConstructStage(self):
1636     return stages.BuildImageStage(self.run, self._current_board)
1637
1638   def RunTestsWithReleaseConfig(self, release_tag):
1639     self._release_tag = release_tag
1640
1641     with parallel_unittest.ParallelMock():
1642       with self.RunStageWithConfig() as rc:
1643         cfg = self.run.config
1644         cmd = ['./build_image', '--version=%s' % (self._release_tag or '')]
1645         rc.assertCommandContains(cmd, expected=cfg['images'])
1646         rc.assertCommandContains(['./image_to_vm.sh'],
1647                                  expected=cfg['vm_tests'])
1648         cmd = ['./build_library/generate_au_zip.py', '-o', '/chroot/path']
1649         rc.assertCommandContains(cmd, expected=cfg['images'])
1650
1651   def RunTestsWithBotId(self, bot_id, options_tests=True):
1652     """Test with the config for the specified bot_id."""
1653     release_tag = '0.0.1'
1654     self._Prepare(bot_id)
1655     self.run.options.tests = options_tests
1656     self.run.attrs.release_tag = release_tag
1657
1658     task = self.RunTestsWithReleaseConfig
1659     steps = [lambda: task(tag) for tag in (None, release_tag)]
1660     parallel.RunParallelSteps(steps)
1661
1662
1663 class UploadTestArtifactsStageMock(ArchivingMock):
1664   """Partial mock for BuildImageStage."""
1665
1666   TARGET = 'chromite.buildbot.cbuildbot_stages.UploadTestArtifactsStage'
1667   ATTRS = ArchivingMock.ATTRS + ('BuildAutotestTarballs',)
1668
1669   def BuildAutotestTarballs(self, *args, **kwargs):
1670     with patches(
1671         patch(commands, 'BuildTarball'),
1672         patch(commands, 'FindFilesWithPattern', return_value=['foo.txt'])):
1673       self.backup['BuildAutotestTarballs'](*args, **kwargs)
1674
1675
1676 class UploadTestArtifactsStageTest(BuildPackagesStageTest):
1677   """Tests UploadTestArtifactsStage."""
1678
1679   def setUp(self):
1680     self.StartPatcher(UploadTestArtifactsStageMock())
1681
1682   def ConstructStage(self):
1683     return stages.UploadTestArtifactsStage(self.run, self._current_board)
1684
1685   def RunTestsWithBotId(self, bot_id, options_tests=True):
1686     """Test with the config for the specified bot_id."""
1687     self._Prepare(bot_id)
1688     self.run.options.tests = options_tests
1689     self.run.attrs.release_tag = '0.0.1'
1690
1691     # Simulate images being ready.
1692     board_runattrs = self.run.GetBoardRunAttrs(self._current_board)
1693     board_runattrs.SetParallel('images_generated', True)
1694
1695     with parallel_unittest.ParallelMock():
1696       with self.RunStageWithConfig() as rc:
1697         cfg = self.run.config
1698         hw = cfg['upload_hw_test_artifacts']
1699         canary = (cfg['build_type'] == constants.CANARY_TYPE)
1700         rc.assertCommandContains(['--full_payload'], expected=hw and not canary)
1701         rc.assertCommandContains(['--nplus1'], expected=hw and canary)
1702
1703
1704 class ArchivingStageTest(AbstractStageTest):
1705   """Excerise ArchivingStage functionality."""
1706   RELEASE_TAG = ''
1707
1708   def setUp(self):
1709     self.StartPatcher(BuilderRunMock())
1710     self.StartPatcher(ArchivingMock())
1711
1712     self._Prepare()
1713
1714   def ConstructStage(self):
1715     self.run.GetArchive().SetupArchivePath()
1716     archive_stage = stages.ArchiveStage(self.run, self._current_board)
1717     return stages.ArchivingStage(self.run, self._current_board, archive_stage)
1718
1719   def testMetadataJson(self):
1720     """Test that the json metadata is built correctly"""
1721     # First add some results to make sure we can handle various types.
1722     results_lib.Results.Clear()
1723     results_lib.Results.Record('Sync', results_lib.Results.SUCCESS, time=1)
1724     results_lib.Results.Record('Build', results_lib.Results.SUCCESS, time=2)
1725     results_lib.Results.Record('Test', FailStage.FAIL_EXCEPTION, time=3)
1726     results_lib.Results.Record('SignerTests', results_lib.Results.SKIPPED)
1727
1728     # Now run the code.
1729     stage = self.ConstructStage()
1730     stage.UploadMetadata(stage='tests')
1731
1732     # Now check the results.
1733     json_file = os.path.join(
1734         stage.archive_path,
1735         constants.METADATA_STAGE_JSON % { 'stage': 'tests' } )
1736     json_data = json.loads(osutils.ReadFile(json_file))
1737
1738     important_keys = (
1739         'boards',
1740         'bot-config',
1741         'metadata-version',
1742         'results',
1743         'sdk-version',
1744         'toolchain-tuple',
1745         'toolchain-url',
1746         'version',
1747     )
1748     for key in important_keys:
1749       self.assertTrue(key in json_data)
1750
1751     self.assertEquals(json_data['boards'], ['x86-generic'])
1752     self.assertEquals(json_data['bot-config'], 'x86-generic-paladin')
1753     self.assertEquals(json_data['version']['full'], stage.version)
1754     self.assertEquals(json_data['metadata-version'], '2')
1755
1756     results_passed = ('Sync', 'Build',)
1757     results_failed = ('Test',)
1758     results_skipped = ('SignerTests',)
1759     for result in json_data['results']:
1760       if result['name'] in results_passed:
1761         self.assertEquals(result['status'], constants.FINAL_STATUS_PASSED)
1762       elif result['name'] in results_failed:
1763         self.assertEquals(result['status'], constants.FINAL_STATUS_FAILED)
1764       elif result['name'] in results_skipped:
1765         self.assertEquals(result['status'], constants.FINAL_STATUS_PASSED)
1766         self.assertTrue('skipped' in result['summary'].lower())
1767
1768     # The buildtools manifest doesn't have any overlays. In this case, we can't
1769     # find any toolchains.
1770     overlays = portage_utilities.FindOverlays(
1771         constants.BOTH_OVERLAYS, board=None, buildroot=self.build_root)
1772     overlay_tuples = ['i686-pc-linux-gnu', 'arm-none-eabi']
1773     self.assertEquals(json_data['toolchain-tuple'],
1774                       overlay_tuples if overlays else [])
1775
1776
1777 class ArchiveStageTest(AbstractStageTest):
1778   """Exercise ArchiveStage functionality."""
1779   RELEASE_TAG = ''
1780   VERSION = '3333.1.0'
1781
1782   def _PatchDependencies(self):
1783     """Patch dependencies of ArchiveStage.PerformStage()."""
1784     to_patch = [
1785         (parallel, 'RunParallelSteps'), (commands, 'PushImages'),
1786         (commands, 'UploadArchivedFile')]
1787     self.AutoPatch(to_patch)
1788
1789   def setUp(self):
1790     self.StartPatcher(BuilderRunMock())
1791     self._PatchDependencies()
1792
1793     self._Prepare()
1794
1795   def _Prepare(self, bot_id=None, **kwargs):
1796     extra_config = {'upload_symbols': True, 'push_image': True}
1797     super(ArchiveStageTest, self)._Prepare(bot_id, extra_config=extra_config,
1798                                            **kwargs)
1799
1800   def ConstructStage(self):
1801     self.run.GetArchive().SetupArchivePath()
1802     return stages.ArchiveStage(self.run, self._current_board)
1803
1804   def testArchive(self):
1805     """Simple did-it-run test."""
1806     # TODO(davidjames): Test the individual archive steps as well.
1807     self.RunStage()
1808
1809   # TODO(build): This test is not actually testing anything real.  It confirms
1810   # that PushImages is not called, but the mock for RunParallelSteps already
1811   # prevents PushImages from being called, regardless of whether this is a
1812   # trybot flow.
1813   def testNoPushImagesForRemoteTrybot(self):
1814     """Test that remote trybot overrides work to disable push images."""
1815     self._Prepare('x86-mario-release',
1816                   cmd_args=['--remote-trybot', '-r', self.build_root,
1817                             '--buildnumber=1234'])
1818     self.RunStage()
1819     # pylint: disable=E1101
1820     self.assertEquals(commands.PushImages.call_count, 0)
1821
1822   def ConstructStageForArchiveStep(self):
1823     """Stage construction for archive steps."""
1824     stage = self.ConstructStage()
1825     self.PatchObject(stage._upload_queue, 'put', autospec=True)
1826     self.PatchObject(git, 'ReinterpretPathForChroot', return_value='',
1827                      autospec=True)
1828     return stage
1829
1830   def testBuildAndArchiveDeltaSysroot(self):
1831     """Test tarball is added to upload queue."""
1832     stage = self.ConstructStageForArchiveStep()
1833     with cros_build_lib_unittest.RunCommandMock() as rc:
1834       rc.SetDefaultCmdResult()
1835       stage.BuildAndArchiveDeltaSysroot()
1836     stage._upload_queue.put.assert_called_with([constants.DELTA_SYSROOT_TAR])
1837
1838   def testBuildAndArchiveDeltaSysrootFailure(self):
1839     """Test tarball not added to upload queue on command exception."""
1840     stage = self.ConstructStageForArchiveStep()
1841     with cros_build_lib_unittest.RunCommandMock() as rc:
1842       rc.AddCmdResult(partial_mock.In('generate_delta_sysroot'), returncode=1,
1843                       error='generate_delta_sysroot: error')
1844       self.assertRaises2(cros_build_lib.RunCommandError,
1845                         stage.BuildAndArchiveDeltaSysroot)
1846     self.assertFalse(stage._upload_queue.put.called)
1847
1848
1849 class UploadPrebuiltsStageTest(RunCommandAbstractStageTest):
1850   """Tests for the UploadPrebuilts stage."""
1851
1852   CMD = './upload_prebuilts'
1853   RELEASE_TAG = ''
1854
1855   def setUp(self):
1856     self.StartPatcher(BuilderRunMock())
1857
1858   def _Prepare(self, bot_id=None, **kwargs):
1859     super(UploadPrebuiltsStageTest, self)._Prepare(bot_id, **kwargs)
1860
1861     self.run.options.prebuilts = True
1862
1863   def ConstructStage(self):
1864     return stages.UploadPrebuiltsStage(self.run,
1865                                        self.run.config.boards[-1])
1866
1867   def _VerifyBoardMap(self, bot_id, count, board_map, public_args=None,
1868                       private_args=None):
1869     """Verify that the prebuilts are uploaded for the specified bot.
1870
1871     Args:
1872       bot_id: Bot to upload prebuilts for.
1873       count: Number of assert checks that should be performed.
1874       board_map: Map from slave boards to whether the bot is public.
1875       public_args: List of extra arguments for public boards.
1876       private_args: List of extra arguments for private boards.
1877     """
1878     self._Prepare(bot_id)
1879     self.RunStage()
1880     public_prefix = [self.CMD] + (public_args or [])
1881     private_prefix = [self.CMD] + (private_args or [])
1882     for board, public in board_map.iteritems():
1883       if public or public_args:
1884         public_cmd = public_prefix + ['--slave-board', board]
1885         self.assertCommandContains(public_cmd, expected=public)
1886         count -= 1
1887       private_cmd = private_prefix + ['--slave-board', board, '--private']
1888       self.assertCommandContains(private_cmd, expected=not public)
1889       count -= 1
1890     if board_map:
1891       self.assertCommandContains([self.CMD, '--set-version',
1892                                   self.run.GetVersion()])
1893       count -= 1
1894     self.assertEqual(count, 0,
1895         'Number of asserts performed does not match (%d remaining)' % count)
1896
1897   def testFullPrebuiltsUpload(self):
1898     """Test uploading of full builder prebuilts."""
1899     self._VerifyBoardMap('x86-generic-full', 0, {})
1900     self.assertCommandContains([self.CMD, '--git-sync'])
1901
1902   def testIncorrectCount(self):
1903     """Test that _VerifyBoardMap asserts when the count is wrong."""
1904     self.assertRaises(AssertionError, self._VerifyBoardMap, 'x86-generic-full',
1905                       1, {})
1906
1907   def testChromeUpload(self):
1908     """Test uploading of prebuilts for chrome build."""
1909     board_map = {'amd64-generic': True, 'daisy': True,
1910                  'x86-alex': False, 'lumpy': False}
1911     self._VerifyBoardMap('x86-generic-chromium-pfq', 9, board_map,
1912                          public_args=['--board', 'x86-generic'])
1913
1914
1915 class MasterUploadPrebuiltsStageTest(RunCommandAbstractStageTest):
1916   """Tests for the MasterUploadPrebuilts stage."""
1917
1918   CMD = './upload_prebuilts'
1919   RELEASE_TAG = '1234.5.6'
1920   VERSION = 'R%s-%s' % (DEFAULT_CHROME_BRANCH, RELEASE_TAG)
1921
1922   def setUp(self):
1923     self.StartPatcher(BuilderRunMock())
1924
1925   def _Prepare(self, bot_id=None, **kwargs):
1926     super(MasterUploadPrebuiltsStageTest, self)._Prepare(bot_id, **kwargs)
1927
1928     self.run.options.prebuilts = True
1929
1930   def ConstructStage(self):
1931     return stages.MasterUploadPrebuiltsStage(self.run)
1932
1933   def _RunStage(self, bot_id):
1934     """Run the stage under test with the given |bot_id| config.
1935
1936     Args:
1937       bot_id: Builder config target name.
1938     """
1939     self._Prepare(bot_id)
1940     self.RunStage()
1941
1942   def _VerifyResults(self, public_slave_boards=(), private_slave_boards=()):
1943     """Verify that the expected prebuilt commands were run.
1944
1945     Do various assertions on the two RunCommands that were run by stage.
1946     There should be one private (--private) and one public (default) run.
1947
1948     Args:
1949       public_slave_boards: List of public slave boards.
1950       private_slave_boards: List of private slave boards.
1951     """
1952     # TODO(mtennant): Add functionality in partial_mock to support more flexible
1953     # asserting.  For example here, asserting that '--sync-host' appears in
1954     # the command that did not include '--public'.
1955
1956     # Some args are expected for any public run.
1957     if public_slave_boards:
1958       # It would be nice to confirm that --private is not in command, but note
1959       # that --sync-host should not appear in the --private command.
1960       cmd = [self.CMD, '--sync-binhost-conf', '--sync-host']
1961       self.assertCommandContains(cmd, expected=True)
1962
1963     # Some args are expected for any private run.
1964     if private_slave_boards:
1965       cmd = [self.CMD, '--sync-binhost-conf', '--private']
1966       self.assertCommandContains(cmd, expected=True)
1967
1968     # Assert public slave boards are mentioned in public run.
1969     for board in public_slave_boards:
1970       # This check does not actually confirm that this board was in the public
1971       # run rather than the private run, unfortunately.
1972       cmd = [self.CMD, '--slave-board', board]
1973       self.assertCommandContains(cmd, expected=True)
1974
1975     # Assert private slave boards are mentioned in private run.
1976     for board in private_slave_boards:
1977       cmd = [self.CMD, '--slave-board', board, '--private']
1978       self.assertCommandContains(cmd, expected=True)
1979
1980     # We expect --set-version so long as build config has manifest_version=True.
1981     self.assertCommandContains([self.CMD, '--set-version', self.VERSION],
1982                                expected=self.run.config.manifest_version)
1983
1984   def testMasterPaladinUpload(self):
1985     self._RunStage('master-paladin')
1986
1987     # Provide a sample of private/public slave boards that are expected.
1988     public_slave_boards = ('amd64-generic', 'x86-generic')
1989     private_slave_boards = ('x86-mario', 'x86-alex', 'lumpy', 'daisy_spring')
1990
1991     self._VerifyResults(public_slave_boards=public_slave_boards,
1992                         private_slave_boards=private_slave_boards)
1993
1994
1995 class UploadDevInstallerPrebuiltsStageTest(AbstractStageTest):
1996   """Tests for the UploadDevInstallerPrebuilts stage."""
1997
1998   RELEASE_TAG = 'RT'
1999
2000   def setUp(self):
2001     self.mox.StubOutWithMock(commands, 'UploadDevInstallerPrebuilts')
2002
2003     self.StartPatcher(BuilderRunMock())
2004
2005     self._Prepare()
2006
2007   def _Prepare(self, bot_id=None, **kwargs):
2008     super(UploadDevInstallerPrebuiltsStageTest, self)._Prepare(bot_id, **kwargs)
2009
2010     self.run.options.chrome_rev = None
2011     self.run.options.prebuilts = True
2012     self.run.config['dev_installer_prebuilts'] = True
2013     self.run.config['binhost_bucket'] = 'gs://testbucket'
2014     self.run.config['binhost_key'] = 'dontcare'
2015     self.run.config['binhost_base_url'] = 'https://dontcare/here'
2016
2017   def ConstructStage(self):
2018     return stages.DevInstallerPrebuiltsStage(self.run,
2019                                              self._current_board)
2020
2021   def testDevInstallerUpload(self):
2022     """Basic sanity test testing uploads of dev installer prebuilts."""
2023     version = 'R%s-%s' % (DEFAULT_CHROME_BRANCH, self.RELEASE_TAG)
2024
2025     commands.UploadDevInstallerPrebuilts(
2026         binhost_bucket=self.run.config.binhost_bucket,
2027         binhost_key=self.run.config.binhost_key,
2028         binhost_base_url=self.run.config.binhost_base_url,
2029         buildroot=self.build_root,
2030         board=self._current_board,
2031         extra_args=mox.And(mox.IsA(list),
2032                            mox.In(version)))
2033
2034     self.mox.ReplayAll()
2035     self.RunStage()
2036     self.mox.VerifyAll()
2037
2038
2039 class PublishUprevChangesStageTest(AbstractStageTest):
2040   """Tests for the PublishUprevChanges stage."""
2041
2042   def setUp(self):
2043     self.mox.StubOutWithMock(stages.PublishUprevChangesStage,
2044                              '_GetPortageEnvVar')
2045     self.mox.StubOutWithMock(commands, 'UploadPrebuilts')
2046     self.mox.StubOutWithMock(commands, 'UprevPush')
2047     self.mox.StubOutWithMock(stages.PublishUprevChangesStage,
2048                              '_ExtractOverlays')
2049     stages.PublishUprevChangesStage._ExtractOverlays().AndReturn(
2050         [['foo'], ['bar']])
2051
2052   def ConstructStage(self):
2053     return stages.PublishUprevChangesStage(self.run, success=True)
2054
2055   def testPush(self):
2056     """Test values for PublishUprevChanges."""
2057     self._Prepare(extra_config={'build_type': constants.BUILD_FROM_SOURCE_TYPE,
2058                                 'push_overlays': constants.PUBLIC_OVERLAYS,
2059                                 'master': True},
2060                   extra_cmd_args=['--chrome_rev', constants.CHROME_REV_TOT])
2061     self.run.options.prebuilts = True
2062     stages.commands.UprevPush(self.build_root, ['bar'], False)
2063
2064     self.mox.ReplayAll()
2065     self.RunStage()
2066     self.mox.VerifyAll()
2067
2068
2069 class CPEExportStageTest(AbstractStageTest):
2070   """Test CPEExportStage"""
2071
2072   def setUp(self):
2073     self.StartPatcher(BuilderRunMock())
2074     self.StartPatcher(ArchivingMock())
2075     self.StartPatcher(parallel_unittest.ParallelMock())
2076
2077     self.rc_mock = self.StartPatcher(cros_build_lib_unittest.RunCommandMock())
2078     self.rc_mock.SetDefaultCmdResult(output='')
2079
2080     self.stage = None
2081
2082   def ConstructStage(self):
2083     """Create a CPEExportStage instance for testing"""
2084     self.run.GetArchive().SetupArchivePath()
2085     return stages.CPEExportStage(self.run, self._current_board)
2086
2087   def assertBoardAttrEqual(self, attr, expected_value):
2088     """Assert the value of a board run |attr| against |expected_value|."""
2089     value = self.stage.board_runattrs.GetParallel(attr)
2090     self.assertEqual(expected_value, value)
2091
2092   def _TestPerformStage(self):
2093     """Run PerformStage for the stage."""
2094     self._Prepare()
2095     self.run.attrs.release_tag = BuilderRunMock.VERSION
2096
2097     self.stage = self.ConstructStage()
2098     self.stage.PerformStage()
2099
2100   def testCPEExport(self):
2101     """Test that CPEExport stage runs without syntax errors."""
2102     self._TestPerformStage()
2103
2104
2105 class DebugSymbolsStageTest(AbstractStageTest):
2106   """Test DebugSymbolsStage"""
2107
2108   def setUp(self):
2109     self.StartPatcher(BuilderRunMock())
2110     self.StartPatcher(ArchivingMock())
2111     self.StartPatcher(parallel_unittest.ParallelMock())
2112
2113     self.gen_mock = self.PatchObject(commands, 'GenerateBreakpadSymbols')
2114     self.upload_mock = self.PatchObject(commands, 'UploadSymbols')
2115     self.tar_mock = self.PatchObject(commands, 'GenerateDebugTarball')
2116
2117     self.rc_mock = self.StartPatcher(cros_build_lib_unittest.RunCommandMock())
2118     self.rc_mock.SetDefaultCmdResult(output='')
2119
2120     self.stage = None
2121
2122   def ConstructStage(self):
2123     """Create a DebugSymbolsStage instance for testing"""
2124     self.run.GetArchive().SetupArchivePath()
2125     return stages.DebugSymbolsStage(self.run, self._current_board)
2126
2127   def assertBoardAttrEqual(self, attr, expected_value):
2128     """Assert the value of a board run |attr| against |expected_value|."""
2129     value = self.stage.board_runattrs.GetParallel(attr)
2130     self.assertEqual(expected_value, value)
2131
2132   def _TestPerformStage(self, extra_config=None):
2133     """Run PerformStage for the stage with the given extra config."""
2134     if not extra_config:
2135       extra_config = {
2136           'archive_build_debug': True,
2137           'vm_tests': True,
2138           'upload_symbols': True,
2139       }
2140
2141     self._Prepare(extra_config=extra_config)
2142     self.run.attrs.release_tag = BuilderRunMock.VERSION
2143
2144     self.tar_mock.side_effect = '/my/tar/ball'
2145     self.stage = self.ConstructStage()
2146     try:
2147       self.stage.PerformStage()
2148     except Exception:
2149       self.stage._HandleStageException(sys.exc_info())
2150       raise
2151
2152   def testPerformStageWithSymbols(self):
2153     """Smoke test for an PerformStage when debugging is enabled"""
2154     self._TestPerformStage()
2155
2156     self.assertEqual(self.gen_mock.call_count, 1)
2157     self.assertEqual(self.upload_mock.call_count, 1)
2158     self.assertEqual(self.tar_mock.call_count, 1)
2159
2160     self.assertBoardAttrEqual('breakpad_symbols_generated', True)
2161     self.assertBoardAttrEqual('debug_tarball_generated', True)
2162
2163   def testPerformStageNoSymbols(self):
2164     """Smoke test for an PerformStage when debugging is disabled"""
2165     extra_config = {
2166         'archive_build_debug': False,
2167         'vm_tests': False,
2168         'upload_symbols': False,
2169     }
2170     self._TestPerformStage(extra_config)
2171
2172     self.assertEqual(self.gen_mock.call_count, 1)
2173     self.assertEqual(self.upload_mock.call_count, 0)
2174     self.assertEqual(self.tar_mock.call_count, 1)
2175
2176     self.assertBoardAttrEqual('breakpad_symbols_generated', True)
2177     self.assertBoardAttrEqual('debug_tarball_generated', True)
2178
2179   def testGenerateCrashStillNotifies(self):
2180     """Crashes in symbol generation should still notify external events."""
2181     class TestError(Exception):
2182       """Unique test exception"""
2183
2184     self.gen_mock.side_effect = TestError('mew')
2185     self.assertRaises(TestError, self._TestPerformStage)
2186
2187     self.assertEqual(self.gen_mock.call_count, 1)
2188     self.assertEqual(self.upload_mock.call_count, 0)
2189     self.assertEqual(self.tar_mock.call_count, 0)
2190
2191     self.assertBoardAttrEqual('breakpad_symbols_generated', False)
2192     self.assertBoardAttrEqual('debug_tarball_generated', False)
2193
2194   def testUploadCrashStillNotifies(self):
2195     """Crashes in symbol upload should still notify external events."""
2196     class TestError(Exception):
2197       """Unique test exception"""
2198
2199     self.upload_mock.side_effect = TestError('mew')
2200     self.assertRaises(TestError, self._TestPerformStage)
2201
2202     self.assertEqual(self.gen_mock.call_count, 1)
2203     self.assertEqual(self.upload_mock.call_count, 1)
2204     self.assertEqual(self.tar_mock.call_count, 1)
2205
2206     self.assertBoardAttrEqual('breakpad_symbols_generated', True)
2207     self.assertBoardAttrEqual('debug_tarball_generated', True)
2208
2209
2210 class PassStage(bs.BuilderStage):
2211   """PassStage always works"""
2212
2213
2214 class Pass2Stage(bs.BuilderStage):
2215   """Pass2Stage always works"""
2216
2217
2218 class FailStage(bs.BuilderStage):
2219   """FailStage always throws an exception"""
2220
2221   FAIL_EXCEPTION = results_lib.StepFailure("Fail stage needs to fail.")
2222
2223   def PerformStage(self):
2224     """Throw the exception to make us fail."""
2225     raise self.FAIL_EXCEPTION
2226
2227
2228 class SkipStage(bs.BuilderStage):
2229   """SkipStage is skipped."""
2230   config_name = 'signer_tests'
2231
2232
2233 class SneakyFailStage(bs.BuilderStage):
2234   """SneakyFailStage exits with an error."""
2235
2236   def PerformStage(self):
2237     """Exit without reporting back."""
2238     os._exit(1)
2239
2240
2241 class SuicideStage(bs.BuilderStage):
2242   """SuicideStage kills itself with kill -9."""
2243
2244   def PerformStage(self):
2245     """Exit without reporting back."""
2246     os.kill(os.getpid(), signal.SIGKILL)
2247
2248
2249 class SetAttrStage(bs.BuilderStage):
2250   """Stage that sets requested run attribute to a value."""
2251
2252   DEFAULT_ATTR = 'unittest_value'
2253   VALUE = 'HereTakeThis'
2254
2255   def __init__(self, builder_run, delay=2, attr=DEFAULT_ATTR, *args, **kwargs):
2256     super(SetAttrStage, self).__init__(builder_run, *args, **kwargs)
2257     self.delay = delay
2258     self.attr = attr
2259
2260   def PerformStage(self):
2261     """Wait self.delay seconds then set requested run attribute."""
2262     time.sleep(self.delay)
2263     self._run.attrs.SetParallel(self.attr, self.VALUE)
2264
2265   def QueueableException(self):
2266     return cbuildbot_run.ParallelAttributeError(self.attr)
2267
2268
2269 class GetAttrStage(bs.BuilderStage):
2270   """Stage that accesses requested run attribute and confirms value."""
2271
2272   DEFAULT_ATTR = 'unittest_value'
2273
2274   def __init__(self, builder_run, tester=None, timeout=5, attr=DEFAULT_ATTR,
2275                *args, **kwargs):
2276     super(GetAttrStage, self).__init__(builder_run, *args, **kwargs)
2277     self.tester = tester
2278     self.timeout = timeout
2279     self.attr = attr
2280
2281   def PerformStage(self):
2282     """Wait for attrs.test value to show up."""
2283     assert not self._run.attrs.HasParallel(self.attr)
2284     value = self._run.attrs.GetParallel(self.attr, self.timeout)
2285     if self.tester:
2286       self.tester(value)
2287
2288   def QueueableException(self):
2289     return cbuildbot_run.ParallelAttributeError(self.attr)
2290
2291   def TimeoutException(self):
2292     return cbuildbot_run.AttrTimeoutError(self.attr)
2293
2294
2295 class BuildStagesResultsTest(cros_test_lib.TestCase):
2296   """Tests for stage results and reporting."""
2297
2298   def setUp(self):
2299     # Always stub RunCommmand out as we use it in every method.
2300     self.bot_id = 'x86-generic-paladin'
2301     build_config = config.config[self.bot_id]
2302     self.build_root = '/fake_root'
2303
2304     # Create a class to hold
2305     class Options(object):
2306       """Dummy class to hold option values."""
2307
2308     options = Options()
2309     options.archive_base = 'gs://dontcare'
2310     options.buildroot = self.build_root
2311     options.debug = False
2312     options.prebuilts = False
2313     options.clobber = False
2314     options.nosdk = False
2315     options.remote_trybot = False
2316     options.latest_toolchain = False
2317     options.buildnumber = 1234
2318     options.chrome_rev = None
2319     options.branch = 'dontcare'
2320
2321     self._manager = parallel.Manager()
2322     self._manager.__enter__()
2323
2324     self.run = cbuildbot_run.BuilderRun(options, build_config, self._manager)
2325
2326     results_lib.Results.Clear()
2327
2328   def tearDown(self):
2329     # Mimic exiting with statement for self._manager.
2330     self._manager.__exit__(None, None, None)
2331
2332   def _runStages(self):
2333     """Run a couple of stages so we can capture the results"""
2334     # Run two pass stages, and one fail stage.
2335     PassStage(self.run).Run()
2336     Pass2Stage(self.run).Run()
2337     self.assertRaises(
2338       results_lib.StepFailure,
2339       FailStage(self.run).Run)
2340
2341   def _verifyRunResults(self, expectedResults, max_time=2.0):
2342     actualResults = results_lib.Results.Get()
2343
2344     # Break out the asserts to be per item to make debugging easier
2345     self.assertEqual(len(expectedResults), len(actualResults))
2346     for i in xrange(len(expectedResults)):
2347       entry = actualResults[i]
2348       xname, xresult = expectedResults[i]
2349
2350       if entry.result not in results_lib.Results.NON_FAILURE_TYPES:
2351         self.assertTrue(isinstance(entry.result, BaseException))
2352         if isinstance(entry.result, results_lib.StepFailure):
2353           self.assertEqual(str(entry.result), entry.description)
2354
2355       self.assertTrue(entry.time >= 0 and entry.time < max_time)
2356       self.assertEqual(xname, entry.name)
2357       self.assertEqual(type(xresult), type(entry.result))
2358       self.assertEqual(repr(xresult), repr(entry.result))
2359
2360   def _PassString(self):
2361     record = results_lib.Result('Pass', results_lib.Results.SUCCESS, 'None',
2362                                 'Pass', '', '0')
2363     return results_lib.Results.SPLIT_TOKEN.join(record) + '\n'
2364
2365   def testRunStages(self):
2366     """Run some stages and verify the captured results"""
2367
2368     self.assertEqual(results_lib.Results.Get(), [])
2369
2370     self._runStages()
2371
2372     # Verify that the results are what we expect.
2373     expectedResults = [
2374         ('Pass', results_lib.Results.SUCCESS),
2375         ('Pass2', results_lib.Results.SUCCESS),
2376         ('Fail', FailStage.FAIL_EXCEPTION),
2377     ]
2378     self._verifyRunResults(expectedResults)
2379
2380   def testSuccessTest(self):
2381     """Run some stages and verify the captured results"""
2382
2383     results_lib.Results.Record('Pass', results_lib.Results.SUCCESS)
2384
2385     self.assertTrue(results_lib.Results.BuildSucceededSoFar())
2386
2387     results_lib.Results.Record('Fail', FailStage.FAIL_EXCEPTION, time=1)
2388
2389     self.assertFalse(results_lib.Results.BuildSucceededSoFar())
2390
2391     results_lib.Results.Record('Pass2', results_lib.Results.SUCCESS)
2392
2393     self.assertFalse(results_lib.Results.BuildSucceededSoFar())
2394
2395   def _TestParallelStages(self, stage_objs):
2396     builder = cbuildbot.SimpleBuilder(self.run)
2397     error = None
2398     with mock.patch.multiple(parallel._BackgroundTask, PRINT_INTERVAL=0.01):
2399       try:
2400         builder._RunParallelStages(stage_objs)
2401       except parallel.BackgroundFailure as ex:
2402         error = ex
2403
2404     return error
2405
2406   def testParallelStages(self):
2407     stage_objs = [stage(self.run) for stage in
2408                   (PassStage, SneakyFailStage, FailStage, SuicideStage,
2409                    Pass2Stage)]
2410     error = self._TestParallelStages(stage_objs)
2411     self.assertTrue(error)
2412     expectedResults = [
2413         ('Pass', results_lib.Results.SUCCESS),
2414         ('Fail', FailStage.FAIL_EXCEPTION),
2415         ('Pass2', results_lib.Results.SUCCESS),
2416         ('SneakyFail', error),
2417         ('Suicide', error),
2418     ]
2419     self._verifyRunResults(expectedResults)
2420
2421   def testParallelStageCommunicationOK(self):
2422     """Test run attr communication betweeen parallel stages."""
2423     def assert_test(value):
2424       self.assertEqual(value, SetAttrStage.VALUE,
2425                        'Expected value %r to be passed between stages, but'
2426                        ' got %r.' % (SetAttrStage.VALUE, value))
2427     stage_objs = [
2428         SetAttrStage(self.run),
2429         GetAttrStage(self.run, assert_test, timeout=30),
2430         GetAttrStage(self.run, assert_test, timeout=30),
2431     ]
2432     error = self._TestParallelStages(stage_objs)
2433     self.assertFalse(error)
2434     expectedResults = [
2435         ('SetAttr', results_lib.Results.SUCCESS),
2436         ('GetAttr', results_lib.Results.SUCCESS),
2437         ('GetAttr', results_lib.Results.SUCCESS),
2438     ]
2439     self._verifyRunResults(expectedResults, max_time=30.0)
2440
2441     # Make sure run attribute propagated up to the top, too.
2442     value = self.run.attrs.GetParallel('unittest_value')
2443     self.assertEqual(SetAttrStage.VALUE, value)
2444
2445   def testParallelStageCommunicationTimeout(self):
2446     """Test run attr communication between parallel stages that times out."""
2447     def assert_test(value):
2448       self.assertEqual(value, SetAttrStage.VALUE,
2449                        'Expected value %r to be passed between stages, but'
2450                        ' got %r.' % (SetAttrStage.VALUE, value))
2451     stage_objs = [SetAttrStage(self.run, delay=11),
2452                   GetAttrStage(self.run, assert_test, timeout=1),
2453                  ]
2454     error = self._TestParallelStages(stage_objs)
2455     self.assertTrue(error)
2456     expectedResults = [
2457         ('SetAttr', results_lib.Results.SUCCESS),
2458         ('GetAttr', stage_objs[1].TimeoutException()),
2459     ]
2460     self._verifyRunResults(expectedResults, max_time=12.0)
2461
2462   def testParallelStageCommunicationNotQueueable(self):
2463     """Test setting non-queueable run attr in parallel stage."""
2464     stage_objs = [SetAttrStage(self.run, attr='release_tag'),
2465                   GetAttrStage(self.run, timeout=2),
2466                  ]
2467     error = self._TestParallelStages(stage_objs)
2468     self.assertTrue(error)
2469     expectedResults = [
2470         ('SetAttr', stage_objs[0].QueueableException()),
2471         ('GetAttr', stage_objs[1].TimeoutException()),
2472     ]
2473     self._verifyRunResults(expectedResults, max_time=12.0)
2474
2475   def testStagesReportSuccess(self):
2476     """Tests Stage reporting."""
2477
2478     stages.ManifestVersionedSyncStage.manifest_manager = None
2479
2480     # Store off a known set of results and generate a report
2481     results_lib.Results.Record('Sync', results_lib.Results.SUCCESS, time=1)
2482     results_lib.Results.Record('Build', results_lib.Results.SUCCESS, time=2)
2483     results_lib.Results.Record('Test', FailStage.FAIL_EXCEPTION, time=3)
2484     results_lib.Results.Record('SignerTests', results_lib.Results.SKIPPED)
2485     result = cros_build_lib.CommandResult(cmd=['/bin/false', '/nosuchdir'],
2486                                           returncode=2)
2487     results_lib.Results.Record(
2488         'Archive',
2489         cros_build_lib.RunCommandError(
2490             'Command "/bin/false /nosuchdir" failed.\n',
2491             result), time=4)
2492
2493     results = StringIO.StringIO()
2494
2495     results_lib.Results.Report(results)
2496
2497     expectedResults = (
2498         "************************************************************\n"
2499         "** Stage Results\n"
2500         "************************************************************\n"
2501         "** PASS Sync (0:00:01)\n"
2502         "************************************************************\n"
2503         "** PASS Build (0:00:02)\n"
2504         "************************************************************\n"
2505         "** FAIL Test (0:00:03) with StepFailure\n"
2506         "************************************************************\n"
2507         "** FAIL Archive (0:00:04) in /bin/false\n"
2508         "************************************************************\n"
2509     )
2510
2511     expectedLines = expectedResults.split('\n')
2512     actualLines = results.getvalue().split('\n')
2513
2514     # Break out the asserts to be per item to make debugging easier
2515     for i in xrange(min(len(actualLines), len(expectedLines))):
2516       self.assertEqual(expectedLines[i], actualLines[i])
2517     self.assertEqual(len(expectedLines), len(actualLines))
2518
2519   def testStagesReportError(self):
2520     """Tests Stage reporting with exceptions."""
2521
2522     stages.ManifestVersionedSyncStage.manifest_manager = None
2523
2524     # Store off a known set of results and generate a report
2525     results_lib.Results.Record('Sync', results_lib.Results.SUCCESS, time=1)
2526     results_lib.Results.Record('Build', results_lib.Results.SUCCESS, time=2)
2527     results_lib.Results.Record('Test', FailStage.FAIL_EXCEPTION,
2528                                'failException Msg\nLine 2', time=3)
2529     result = cros_build_lib.CommandResult(cmd=['/bin/false', '/nosuchdir'],
2530                                           returncode=2)
2531     results_lib.Results.Record(
2532         'Archive',
2533         cros_build_lib.RunCommandError(
2534             'Command "/bin/false /nosuchdir" failed.\n',
2535             result),
2536         'FailRunCommand msg', time=4)
2537
2538     results = StringIO.StringIO()
2539
2540     results_lib.Results.Report(results)
2541
2542     expectedResults = (
2543         "************************************************************\n"
2544         "** Stage Results\n"
2545         "************************************************************\n"
2546         "** PASS Sync (0:00:01)\n"
2547         "************************************************************\n"
2548         "** PASS Build (0:00:02)\n"
2549         "************************************************************\n"
2550         "** FAIL Test (0:00:03) with StepFailure\n"
2551         "************************************************************\n"
2552         "** FAIL Archive (0:00:04) in /bin/false\n"
2553         "************************************************************\n"
2554         "\n"
2555         "Failed in stage Test:\n"
2556         "\n"
2557         "failException Msg\n"
2558         "Line 2\n"
2559         "\n"
2560         "Failed in stage Archive:\n"
2561         "\n"
2562         "FailRunCommand msg\n"
2563    )
2564
2565     expectedLines = expectedResults.split('\n')
2566     actualLines = results.getvalue().split('\n')
2567
2568     # Break out the asserts to be per item to make debugging easier
2569     for i in xrange(min(len(actualLines), len(expectedLines))):
2570       self.assertEqual(expectedLines[i], actualLines[i])
2571     self.assertEqual(len(expectedLines), len(actualLines))
2572
2573   def testStagesReportReleaseTag(self):
2574     """Tests Release Tag entry in stages report."""
2575
2576     current_version = "release_tag_string"
2577     archive_urls = {
2578         'board1': 'http://foo.com/bucket/bot-id1/version/index.html',
2579         'board2': 'http://foo.com/bucket/bot-id2/version/index.html',}
2580     # Store off a known set of results and generate a report
2581     results_lib.Results.Record('Pass', results_lib.Results.SUCCESS, time=1)
2582
2583     results = StringIO.StringIO()
2584
2585     results_lib.Results.Report(results, archive_urls, current_version)
2586
2587     expectedResults = (
2588         "************************************************************\n"
2589         "** RELEASE VERSION: release_tag_string\n"
2590         "************************************************************\n"
2591         "** Stage Results\n"
2592         "************************************************************\n"
2593         "** PASS Pass (0:00:01)\n"
2594         "************************************************************\n"
2595         "** BUILD ARTIFACTS FOR THIS BUILD CAN BE FOUND AT:\n"
2596         "**  board1: %s\n"
2597         "@@@STEP_LINK@Artifacts[board1]: bot-id1/version@%s@@@\n"
2598         "**  board2: %s\n"
2599         "@@@STEP_LINK@Artifacts[board2]: bot-id2/version@%s@@@\n"
2600         "************************************************************\n"
2601         % (archive_urls['board1'], archive_urls['board1'],
2602            archive_urls['board2'], archive_urls['board2']))
2603
2604     expectedLines = expectedResults.split('\n')
2605     actualLines = results.getvalue().split('\n')
2606
2607     # Break out the asserts to be per item to make debugging easier
2608     for i in xrange(len(expectedLines)):
2609       self.assertEqual(expectedLines[i], actualLines[i])
2610     self.assertEqual(len(expectedLines), len(actualLines))
2611
2612   def testSaveCompletedStages(self):
2613     """Tests that we can save out completed stages."""
2614
2615     # Run this again to make sure we have the expected results stored
2616     results_lib.Results.Record('Pass', results_lib.Results.SUCCESS)
2617     results_lib.Results.Record('Fail', FailStage.FAIL_EXCEPTION)
2618     results_lib.Results.Record('Pass2', results_lib.Results.SUCCESS)
2619
2620     saveFile = StringIO.StringIO()
2621     results_lib.Results.SaveCompletedStages(saveFile)
2622     self.assertEqual(saveFile.getvalue(), self._PassString())
2623
2624   def testRestoreCompletedStages(self):
2625     """Tests that we can read in completed stages."""
2626
2627     results_lib.Results.RestoreCompletedStages(
2628         StringIO.StringIO(self._PassString()))
2629
2630     previous = results_lib.Results.GetPrevious()
2631     self.assertEqual(previous.keys(), ['Pass'])
2632
2633   def testRunAfterRestore(self):
2634     """Tests that we skip previously completed stages."""
2635
2636     # Fake results_lib.Results.RestoreCompletedStages
2637     results_lib.Results.RestoreCompletedStages(
2638         StringIO.StringIO(self._PassString()))
2639
2640     self._runStages()
2641
2642     # Verify that the results are what we expect.
2643     expectedResults = [
2644         ('Pass', results_lib.Results.SUCCESS),
2645         ('Pass2', results_lib.Results.SUCCESS),
2646         ('Fail', FailStage.FAIL_EXCEPTION),
2647     ]
2648     self._verifyRunResults(expectedResults)
2649
2650   def testFailedButForgiven(self):
2651     """Tests that warnings are flagged as such."""
2652     results_lib.Results.Record('Warn', results_lib.Results.FORGIVEN, time=1)
2653     results = StringIO.StringIO()
2654     results_lib.Results.Report(results)
2655     self.assertTrue('@@@STEP_WARNINGS@@@' in results.getvalue())
2656
2657
2658 class ReportStageTest(AbstractStageTest):
2659   """Test the Report stage."""
2660
2661   RELEASE_TAG = ''
2662
2663   def setUp(self):
2664     for cmd in ((osutils, 'ReadFile'), (osutils, 'WriteFile'),
2665                 (commands, 'UploadArchivedFile'),
2666                 (alerts, 'SendEmail')):
2667       self.StartPatcher(mock.patch.object(*cmd, autospec=True))
2668
2669     self.StartPatcher(BuilderRunMock())
2670     self.cq = CLStatusMock()
2671     self.StartPatcher(self.cq)
2672     self.sync_stage = None
2673
2674     self._Prepare()
2675
2676   def _SetupUpdateStreakCounter(self, counter_value=-1):
2677     self.PatchObject(stages.ReportStage, '_UpdateStreakCounter',
2678                      autospec=True, return_value=counter_value)
2679
2680   def _SetupCommitQueueSyncPool(self):
2681     self.sync_stage = stages.CommitQueueSyncStage(self.run)
2682     pool = validation_pool.ValidationPool(constants.BOTH_OVERLAYS,
2683         self.build_root, build_number=3, builder_name=self.bot_id,
2684         is_master=True, dryrun=True)
2685     pool.changes = [MockPatch()]
2686     self.sync_stage.pool = pool
2687
2688   def ConstructStage(self):
2689     return stages.ReportStage(self.run, self.sync_stage, None)
2690
2691   def testCheckResults(self):
2692     """Basic sanity check for results stage functionality"""
2693     self._SetupUpdateStreakCounter()
2694     self.PatchObject(stages.ReportStage, '_UploadArchiveIndex',
2695                      return_value={'any': 'dict'})
2696     self.RunStage()
2697     filenames = (
2698         'LATEST-%s' % self.TARGET_MANIFEST_BRANCH,
2699         'LATEST-%s' % BuilderRunMock.VERSION,
2700     )
2701     calls = [mock.call(mock.ANY, mock.ANY, 'metadata.json', False,
2702                        update_list=True, acl=mock.ANY)]
2703     calls += [mock.call(mock.ANY, mock.ANY, filename, False,
2704                         acl=mock.ANY) for filename in filenames]
2705     # pylint: disable=E1101
2706     self.assertEquals(calls, commands.UploadArchivedFile.call_args_list)
2707
2708   def testCommitQueueResults(self):
2709     """Check that commit queue patches get serialized"""
2710     self._SetupUpdateStreakCounter()
2711     self._SetupCommitQueueSyncPool()
2712     self.RunStage()
2713
2714   def testAlertEmail(self):
2715     """Send out alerts when streak counter reaches the threshold."""
2716     self._Prepare(extra_config={'health_threshold': 3,
2717                                 'health_alert_recipients': ['fake_recipient']})
2718     self._SetupUpdateStreakCounter(counter_value=-3)
2719     self._SetupCommitQueueSyncPool()
2720     self.RunStage()
2721     # pylint: disable=E1101
2722     self.assertGreater(alerts.SendEmail.call_count, 0,
2723                        'CQ health alerts emails were not sent.')
2724
2725   def testAlertEmailOnFailingStreak(self):
2726     """Continue sending out alerts when streak counter exceeds the threshold."""
2727     self._Prepare(extra_config={'health_threshold': 3,
2728                                 'health_alert_recipients': ['fake_recipient']})
2729     self._SetupUpdateStreakCounter(counter_value=-5)
2730     self._SetupCommitQueueSyncPool()
2731     self.RunStage()
2732     # pylint: disable=E1101
2733     self.assertGreater(alerts.SendEmail.call_count, 0,
2734                        'CQ health alerts emails were not sent.')
2735
2736
2737 class BoardSpecificBuilderStageTest(cros_test_lib.TestCase):
2738   """Tests option/config settings on board-specific stages."""
2739
2740   def testCheckOptions(self):
2741     """Makes sure options/config settings are setup correctly."""
2742
2743     parser = cbuildbot._CreateParser()
2744     (options, _) = parser.parse_args([])
2745
2746     for attr in dir(stages):
2747       obj = eval('stages.' + attr)
2748       if not hasattr(obj, '__base__'):
2749         continue
2750       if not obj.__base__ is stages.BoardSpecificBuilderStage:
2751         continue
2752       if obj.option_name:
2753         self.assertTrue(getattr(options, obj.option_name))
2754       if obj.config_name:
2755         if not obj.config_name in config._settings:
2756           self.fail(('cbuildbot_stages.%s.config_name "%s" is missing from '
2757                      'cbuildbot_config._settings') % (attr, obj.config_name))
2758
2759
2760 class MockPatch(mock.MagicMock):
2761   """MagicMock for a GerritPatch-like object."""
2762
2763   gerrit_number = '1234'
2764   patch_number = '1'
2765   project = 'chromiumos/chromite'
2766   status = 'NEW'
2767   internal = False
2768   current_patch_set = {
2769     'number': patch_number,
2770     'draft': False,
2771   }
2772   patch_dict = {
2773     'currentPatchSet': current_patch_set,
2774   }
2775
2776   def HasApproval(self, field, value):
2777     """Pretends the patch is good.
2778
2779     Pretend the patch has all of the values listed in
2780     constants.DEFAULT_CQ_READY_FIELDS, but not any other fields.
2781     """
2782     return constants.DEFAULT_CQ_READY_FIELDS.get(field, 0) == value
2783
2784
2785 class BaseCQTest(StageTest):
2786   """Helper class for testing the CommitQueueSync stage"""
2787   PALADIN_BOT_ID = None
2788   MANIFEST_CONTENTS = '<manifest/>'
2789
2790   def setUp(self):
2791     """Setup patchers for specified bot id."""
2792     # Mock out methods as needed.
2793     self.PatchObject(lkgm_manager, 'GenerateBlameList')
2794     self.PatchObject(repository.RepoRepository, 'ExportManifest',
2795                      return_value=MANIFEST_CONTENTS, autospec=True)
2796     self.StartPatcher(git_unittest.ManifestMock())
2797     self.StartPatcher(git_unittest.ManifestCheckoutMock())
2798     version_file = os.path.join(self.build_root, constants.VERSION_FILE)
2799     manifest_version_unittest.VersionInfoTest.WriteFakeVersionFile(version_file)
2800     rc_mock = self.StartPatcher(cros_build_lib_unittest.RunCommandMock())
2801     rc_mock.SetDefaultCmdResult()
2802
2803     # Block the CQ from contacting GoB.
2804     self.PatchObject(gerrit.GerritHelper, 'RemoveCommitReady')
2805     self.PatchObject(gerrit.GerritHelper, 'SubmitChange')
2806     self.PatchObject(validation_pool.PaladinMessage, 'Send')
2807
2808     # If a test is still contacting GoB, something is busted.
2809     self.PatchObject(gob_util, 'CreateHttpConn',
2810                      side_effect=AssertionError('Test should not contact GoB'))
2811
2812     # Create a fake repo / manifest on disk that is used by subclasses.
2813     for subdir in ('repo', 'manifests'):
2814       osutils.SafeMakedirs(os.path.join(self.build_root, '.repo', subdir))
2815     self.manifest_path = os.path.join(self.build_root, '.repo', 'manifest.xml')
2816     osutils.WriteFile(self.manifest_path, MANIFEST_CONTENTS)
2817     self.PatchObject(validation_pool.ValidationPool, 'ReloadChanges',
2818                      side_effect=lambda x: x)
2819
2820     self.sync_stage = None
2821     self._Prepare()
2822
2823   def _Prepare(self, bot_id=None, **kwargs):
2824     super(BaseCQTest, self)._Prepare(bot_id, **kwargs)
2825
2826     self.sync_stage = stages.CommitQueueSyncStage(self.run)
2827
2828   def PerformSync(self, remote='cros', committed=False, tree_open=True,
2829                   tree_throttled=False, tracking_branch='master',
2830                   num_patches=1, runs=0):
2831     """Helper to perform a basic sync for master commit queue.
2832
2833     Args:
2834       remote: Remote name to use for mock patches. Default: 'cros'.
2835       committed: Value to be returned by mock patches' IsChangeCommitted.
2836                  Default: False.
2837       tree_open: If True, behave as if tree is open. Default: True.
2838       tree_throttled: If True, behave as if tree is throttled
2839                       (overriding the tree_open arg). Default: False.
2840       tracking_branch: Tracking branch name for mock patches.
2841       num_patches: The number of mock patches to create. Default: 1.
2842       runs: The maximum number of times to allow validation_pool.AcquirePool
2843             to wait for additional changes. runs=0 means never wait for
2844             additional changes. Default: 0.
2845     """
2846     p = MockPatch(remote=remote, tracking_branch=tracking_branch)
2847     my_patches = [p] * num_patches
2848     self.PatchObject(gerrit.GerritHelper, 'IsChangeCommitted',
2849                      return_value=committed, autospec=True)
2850     self.PatchObject(gerrit.GerritHelper, 'Query',
2851                      return_value=my_patches, autospec=True)
2852     if tree_throttled:
2853       self.PatchObject(timeout_util, 'WaitForTreeStatus',
2854                        return_value=constants.TREE_THROTTLED, autospec=True)
2855     elif tree_open:
2856       self.PatchObject(timeout_util, 'WaitForTreeStatus',
2857                        return_value=constants.TREE_OPEN, autospec=True)
2858     else:
2859       self.PatchObject(timeout_util, 'WaitForTreeStatus',
2860                        side_effect=timeout_util.TimeoutError())
2861
2862     exit_it = itertools.chain([False] * runs, itertools.repeat(True))
2863     self.PatchObject(validation_pool.ValidationPool, 'ShouldExitEarly',
2864                      side_effect=exit_it)
2865     self.sync_stage.PerformStage()
2866
2867   def ReloadPool(self):
2868     """Save the pool to disk and reload it."""
2869     with tempfile.NamedTemporaryFile() as f:
2870       cPickle.dump(self.sync_stage.pool, f)
2871       f.flush()
2872       self.run.options.validation_pool = f.name
2873       self.sync_stage = stages.CommitQueueSyncStage(self.run)
2874       self.sync_stage.HandleSkip()
2875
2876
2877 class SlaveCQSyncTest(BaseCQTest):
2878   """Tests the CommitQueueSync stage for the paladin slaves."""
2879   BOT_ID = 'x86-alex-paladin'
2880
2881   def testReload(self):
2882     """Test basic ability to sync and reload the patches from disk."""
2883     self.PatchObject(lkgm_manager.LKGMManager, 'GetLatestCandidate',
2884                      return_value=self.manifest_path, autospec=True)
2885     self.sync_stage.PerformStage()
2886     self.ReloadPool()
2887
2888
2889 class MasterCQSyncTest(BaseCQTest):
2890   """Tests the CommitQueueSync stage for the paladin masters.
2891
2892   Tests in this class should apply both to the paladin masters and to the
2893   Pre-CQ Launcher.
2894   """
2895   BOT_ID = 'master-paladin'
2896
2897   def setUp(self):
2898     """Setup patchers for specified bot id."""
2899     self.AutoPatch([[validation_pool.ValidationPool, 'ApplyPoolIntoRepo']])
2900     self.PatchObject(lkgm_manager.LKGMManager, 'CreateNewCandidate',
2901                      return_value=self.manifest_path, autospec=True)
2902     self.PatchObject(lkgm_manager.LKGMManager, 'CreateFromManifest',
2903                      return_value=self.manifest_path, autospec=True)
2904
2905   def testCommitNonManifestChange(self, **kwargs):
2906     """Test the commit of a non-manifest change."""
2907     # Setting tracking_branch=foo makes this a non-manifest change.
2908     kwargs.setdefault('committed', True)
2909     self.PerformSync(tracking_branch='foo', **kwargs)
2910
2911   def testFailedCommitOfNonManifestChange(self):
2912     """Test that the commit of a non-manifest change fails."""
2913     self.testCommitNonManifestChange(committed=False)
2914
2915   def testCommitManifestChange(self, **kwargs):
2916     """Test committing a change to a project that's part of the manifest."""
2917     self.PatchObject(validation_pool.ValidationPool, '_FilterNonCrosProjects',
2918                      side_effect=lambda x, _: (x, []))
2919     self.PerformSync(**kwargs)
2920
2921   def testDefaultSync(self):
2922     """Test basic ability to sync with standard options."""
2923     self.PerformSync()
2924
2925
2926 class ExtendedMasterCQSyncTest(MasterCQSyncTest):
2927   """Additional tests for the CommitQueueSync stage.
2928
2929   These only apply to the paladin master and not to any other stages.
2930   """
2931
2932   def testReload(self):
2933     """Test basic ability to sync and reload the patches from disk."""
2934     # Use zero patches because MockPatches can't be pickled.
2935     self.PerformSync(num_patches=0, runs=0)
2936     self.ReloadPool()
2937
2938   def testTreeClosureBlocksCommit(self):
2939     """Test that tree closures block commits."""
2940     self.assertRaises(SystemExit, self.testCommitNonManifestChange,
2941                       tree_open=False)
2942
2943   def testTreeThrottleUsesAlternateGerritQuery(self):
2944     """Test that if the tree is throttled, we use an alternate gerrit query."""
2945     self.PerformSync(tree_throttled=True)
2946     gerrit.GerritHelper.Query.assert_called_with(
2947         mock.ANY, constants.THROTTLED_CQ_READY_QUERY,
2948         sort='lastUpdated')
2949
2950
2951 class CLStatusMock(partial_mock.PartialMock):
2952   """Partial mock for CLStatus methods in ValidationPool."""
2953
2954   TARGET = 'chromite.buildbot.validation_pool.ValidationPool'
2955   ATTRS = ('GetCLStatus', 'GetCLStatusCount', 'UpdateCLStatus',)
2956
2957   def __init__(self, treat_launching_as_inflight=False):
2958     """CLStatusMock constructor.
2959
2960     Args:
2961       treat_launching_as_inflight: When getting a CL's status via
2962         GetCLStatus, treat any change with status LAUNCHING as if
2963         it has status INFLIGHT. This simulates pre-cq tryjobs getting
2964         immediately launched. Default: False.
2965     """
2966     partial_mock.PartialMock.__init__(self)
2967     self.calls = {}
2968     self.status = {}
2969     self.status_count = {}
2970     self._treat_launching_as_inflight = treat_launching_as_inflight
2971
2972   def GetCLStatus(self, _bot, change):
2973     status = self.status.get(change)
2974     if (self._treat_launching_as_inflight and
2975         status == validation_pool.ValidationPool.STATUS_LAUNCHING):
2976       return validation_pool.ValidationPool.STATUS_INFLIGHT
2977     return status
2978
2979   def GetCLStatusCount(self, _bot, change, count, latest_patchset_only=True):
2980     # pylint: disable=W0613
2981     return self.status_count.get(change, 0)
2982
2983   def UpdateCLStatus(self, _bot, change, status, dry_run):
2984     # pylint: disable=W0613
2985     self.calls[status] = self.calls.get(status, 0) + 1
2986     self.status[change] = status
2987     self.status_count[change] = self.status_count.get(change, 0) + 1
2988
2989
2990 class PreCQLauncherStageTest(MasterCQSyncTest):
2991   """Tests for the PreCQLauncherStage."""
2992   BOT_ID = 'pre-cq-launcher'
2993   STATUS_LAUNCHING = validation_pool.ValidationPool.STATUS_LAUNCHING
2994   STATUS_WAITING = validation_pool.ValidationPool.STATUS_WAITING
2995   STATUS_FAILED = validation_pool.ValidationPool.STATUS_FAILED
2996
2997   def setUp(self):
2998     self.PatchObject(time, 'sleep', autospec=True)
2999
3000   def _PrepareValidationPoolMock(self, auto_launch=False):
3001     # pylint: disable-msg=W0201
3002     self.pre_cq = CLStatusMock(treat_launching_as_inflight=auto_launch)
3003     self.StartPatcher(self.pre_cq)
3004
3005   def _Prepare(self, bot_id=None, **kwargs):
3006     super(PreCQLauncherStageTest, self)._Prepare(bot_id, **kwargs)
3007
3008     self.sync_stage = stages.PreCQLauncherStage(self.run)
3009
3010   def testTreeClosureIsOK(self):
3011     """Test that tree closures block commits."""
3012     self._PrepareValidationPoolMock()
3013     self.testCommitNonManifestChange(tree_open=False)
3014
3015   def testLaunchTrybot(self):
3016     """Test launching a trybot."""
3017     self._PrepareValidationPoolMock()
3018     self.testCommitManifestChange()
3019     self.assertEqual(self.pre_cq.status.values(), [self.STATUS_LAUNCHING])
3020     self.assertEqual(self.pre_cq.calls.keys(), [self.STATUS_LAUNCHING])
3021
3022   def runTrybotTest(self, launching, waiting, failed, runs):
3023     """Helper function for testing PreCQLauncher.
3024
3025     Create a mock patch to be picked up by the PreCQ. Allow up to
3026     |runs|+1 calls to ProcessChanges. Assert that the patch received status
3027     LAUNCHING, WAITING, and FAILED |launching|, |waiting|, and |failed| times
3028     respectively.
3029     """
3030     self.testCommitManifestChange(runs=runs)
3031     self.assertEqual(self.pre_cq.calls.get(self.STATUS_LAUNCHING, 0), launching)
3032     self.assertEqual(self.pre_cq.calls.get(self.STATUS_WAITING, 0), waiting)
3033     self.assertEqual(self.pre_cq.calls.get(self.STATUS_FAILED, 0), failed)
3034
3035   def testLaunchTrybotTimesOutOnce(self):
3036     """Test what happens when a trybot launch times out."""
3037     self._PrepareValidationPoolMock()
3038     it = itertools.chain([True], itertools.repeat(False))
3039     self.PatchObject(stages.PreCQLauncherStage, '_HasLaunchTimedOut',
3040                      side_effect=it)
3041     self.runTrybotTest(launching=2, waiting=1, failed=0, runs=3)
3042
3043   def testLaunchTrybotTimesOutTwice(self):
3044     """Test what happens when a trybot launch times out."""
3045     self._PrepareValidationPoolMock()
3046     self.PatchObject(stages.PreCQLauncherStage, '_HasLaunchTimedOut',
3047                      return_value=True)
3048     self.runTrybotTest(launching=2, waiting=1, failed=1, runs=3)
3049
3050   def testInflightTrybotTimesOutOnce(self):
3051     """Test what happens when an inflight trybot times out."""
3052     self._PrepareValidationPoolMock(auto_launch=True)
3053     it = itertools.chain([True], itertools.repeat(False))
3054     self.PatchObject(stages.PreCQLauncherStage, '_HasInflightTimedOut',
3055                      side_effect=it)
3056     self.runTrybotTest(launching=1, waiting=0, failed=1, runs=1)
3057
3058
3059 class ChromeSDKStageTest(AbstractStageTest, cros_test_lib.LoggingTestCase):
3060   """Verify stage that creates the chrome-sdk and builds chrome with it."""
3061   BOT_ID = 'link-paladin'
3062   RELEASE_TAG = ''
3063
3064   def setUp(self):
3065     self.StartPatcher(BuilderRunMock())
3066     self.StartPatcher(parallel_unittest.ParallelMock())
3067
3068     self._Prepare()
3069
3070   def _Prepare(self, bot_id=None, **kwargs):
3071     super(ChromeSDKStageTest, self)._Prepare(bot_id, **kwargs)
3072
3073     self.run.options.chrome_root = '/tmp/non-existent'
3074
3075   def ConstructStage(self):
3076     self.run.GetArchive().SetupArchivePath()
3077     return stages.ChromeSDKStage(self.run, self._current_board)
3078
3079   def testIt(self):
3080     """A simple run-through test."""
3081     rc_mock = self.StartPatcher(cros_build_lib_unittest.RunCommandMock())
3082     rc_mock.SetDefaultCmdResult()
3083     self.PatchObject(stages.ChromeSDKStage, '_ArchiveChromeEbuildEnv',
3084                      autospec=True)
3085     self.PatchObject(stages.ChromeSDKStage, '_VerifyChromeDeployed',
3086                      autospec=True)
3087     self.PatchObject(stages.ChromeSDKStage, '_VerifySDKEnvironment',
3088                      autospec=True)
3089     self.RunStage()
3090
3091   def testChromeEnvironment(self):
3092     """Test that the Chrome environment is built."""
3093     # Create the chrome environment compressed file.
3094     stage = self.ConstructStage()
3095     chrome_env_dir = os.path.join(
3096         stage._pkg_dir, constants.CHROME_CP + '-25.3643.0_rc1')
3097     env_file = os.path.join(chrome_env_dir, 'environment')
3098     osutils.Touch(env_file, makedirs=True)
3099
3100     cros_build_lib.RunCommand(['bzip2', env_file])
3101
3102     # Run the code.
3103     stage._ArchiveChromeEbuildEnv()
3104
3105     env_tar_base = stage._upload_queue.get()[0]
3106     env_tar = os.path.join(stage.archive_path, env_tar_base)
3107     self.assertTrue(os.path.exists(env_tar))
3108     cros_test_lib.VerifyTarball(env_tar, ['./', 'environment'])
3109
3110
3111 class BranchUtilStageTest(AbstractStageTest, cros_test_lib.LoggingTestCase):
3112   """Tests for branch creation/deletion."""
3113
3114   BOT_ID = constants.BRANCH_UTIL_CONFIG
3115   DEFAULT_VERSION = '111.0.0'
3116   RELEASE_BRANCH_NAME = 'release-test-branch'
3117
3118   def _CreateVersionFile(self, version=None):
3119     if version is None:
3120       version = self.DEFAULT_VERSION
3121     version_file = os.path.join(self.build_root, constants.VERSION_FILE)
3122     manifest_version_unittest.VersionInfoTest.WriteFakeVersionFile(
3123         version_file, version=version)
3124
3125   def setUp(self):
3126     """Setup patchers for specified bot id."""
3127     # Mock out methods as needed.
3128     self.StartPatcher(parallel_unittest.ParallelMock())
3129     self.StartPatcher(git_unittest.ManifestCheckoutMock())
3130     self._CreateVersionFile()
3131     self.rc_mock = self.StartPatcher(cros_build_lib_unittest.RunCommandMock())
3132     self.rc_mock.SetDefaultCmdResult()
3133
3134     # We have a versioned manifest (generated by ManifestVersionSyncStage) and
3135     # the regular, user-maintained manifests.
3136     manifests = {
3137         '.repo/manifest.xml': VERSIONED_MANIFEST_CONTENTS,
3138         'manifest/default.xml': MANIFEST_CONTENTS,
3139         'manifest-internal/official.xml': MANIFEST_CONTENTS,
3140     }
3141     for m_path, m_content in manifests.iteritems():
3142       full_path = os.path.join(self.build_root, m_path)
3143       osutils.SafeMakedirs(os.path.dirname(full_path))
3144       osutils.WriteFile(full_path, m_content)
3145
3146     self.norm_name = git.NormalizeRef(self.RELEASE_BRANCH_NAME)
3147
3148   def _Prepare(self, bot_id=None, **kwargs):
3149     if 'cmd_args' not in kwargs:
3150       # Fill in cmd_args so we do not use the default, which specifies
3151       # --branch.  That is incompatible with some branch-util flows.
3152       kwargs['cmd_args'] = ['-r', self.build_root]
3153     super(BranchUtilStageTest, self)._Prepare(bot_id, **kwargs)
3154
3155   def ConstructStage(self):
3156     return stages.BranchUtilStage(self.run)
3157
3158   def _VerifyPush(self, new_branch, rename_from=None, delete=False):
3159     """Verify that |new_branch| has been created.
3160
3161     Args:
3162       new_branch: The new remote branch to create (or delete).
3163       rename_from: If set, |rename_from| is being renamed to |new_branch|.
3164       delete: If set, |new_branch| is being deleted.
3165     """
3166     # Pushes all operate on remote branch refs.
3167     new_branch = git.NormalizeRef(new_branch)
3168
3169     # Calculate source and destination revisions.
3170     suffixes = ['', '-new-special-branch', '-old-special-branch']
3171     if delete:
3172       src_revs = [''] * len(suffixes)
3173     elif rename_from is not None:
3174       rename_from = git.NormalizeRef(rename_from)
3175       rename_from_tracking = git.NormalizeRemoteRef('cros', rename_from)
3176       src_revs = [
3177           '%s%s' % (rename_from_tracking, suffix) for suffix in suffixes
3178       ]
3179     else:
3180       src_revs = [CHROMITE_REVISION, SPECIAL_REVISION1, SPECIAL_REVISION2]
3181     dest_revs = ['%s%s' % (new_branch, suffix) for suffix in suffixes]
3182
3183     # Verify pushes happened correctly.
3184     for src_rev, dest_rev in zip(src_revs, dest_revs):
3185       cmd = ['push', '%s:%s' % (src_rev, dest_rev)]
3186       self.rc_mock.assertCommandContains(cmd)
3187       if rename_from is not None:
3188         cmd = ['push', ':%s' % (rename_from,)]
3189         self.rc_mock.assertCommandContains(cmd)
3190
3191   def testRelease(self):
3192     """Run-through of branch creation."""
3193     self._Prepare(extra_cmd_args=['--branch-name', self.RELEASE_BRANCH_NAME,
3194                                   '--version', self.DEFAULT_VERSION])
3195     # Simulate branch not existing.
3196     self.rc_mock.AddCmdResult(
3197         partial_mock.ListRegex('git show-ref .*%s' % self.RELEASE_BRANCH_NAME),
3198         returncode=1)
3199     # SHA1 of HEAD for pinned branches.
3200     self.rc_mock.AddCmdResult(
3201         partial_mock.ListRegex('git rev-parse HEAD'),
3202         output='12345')
3203
3204     before = manifest_version.VersionInfo.from_repo(self.build_root)
3205     self.RunStage()
3206     after = manifest_version.VersionInfo.from_repo(self.build_root)
3207     # Verify Chrome version was bumped.
3208     self.assertEquals(int(after.chrome_branch) - int(before.chrome_branch), 1)
3209     self.assertEquals(int(after.build_number) - int(before.build_number), 1)
3210
3211     # Verify that manifests were branched properly. Notice that external
3212     # is pinned to a SHA1, not an actual branch.
3213     branch_names = {
3214       'chromite': self.norm_name,
3215       'external': '12345',
3216       'src/special-new': self.norm_name + '-new-special-branch',
3217       'src/special-old': self.norm_name + '-old-special-branch',
3218       'unpinned': 'refs/heads/master',
3219     }
3220     for m in ['manifest/default.xml', 'manifest-internal/official.xml']:
3221       manifest = git.Manifest(os.path.join(self.build_root, m))
3222       for project_data in manifest.checkouts_by_path.itervalues():
3223         branch_name = branch_names[project_data['path']]
3224         msg = (
3225             'Branch name for %s should be %r, but got %r' %
3226                 (project_data['path'], branch_name, project_data['revision'])
3227         )
3228         self.assertEquals(project_data['revision'], branch_name, msg)
3229
3230     self._VerifyPush(self.norm_name)
3231
3232   def testNonRelease(self):
3233     """Non-release branch creation."""
3234     self._Prepare(extra_cmd_args=['--branch-name', 'refs/heads/test-branch',
3235                                   '--version', self.DEFAULT_VERSION])
3236     # Simulate branch not existing.
3237     self.rc_mock.AddCmdResult(
3238         partial_mock.ListRegex('git show-ref .*test-branch'),
3239         returncode=1)
3240
3241     before = manifest_version.VersionInfo.from_repo(self.build_root)
3242     # Disable the new branch increment so that
3243     # IncrementVersionOnDiskForSourceBranch detects we need to bump the version.
3244     self.PatchObject(stages.BranchUtilStage,
3245                      '_IncrementVersionOnDiskForNewBranch', autospec=True)
3246     self.RunStage()
3247     after = manifest_version.VersionInfo.from_repo(self.build_root)
3248     # Verify only branch number is bumped.
3249     self.assertEquals(after.chrome_branch, before.chrome_branch)
3250     self.assertEquals(int(after.build_number) - int(before.build_number), 1)
3251     self._VerifyPush(self.run.options.branch_name)
3252
3253   def testDeletion(self):
3254     """Branch deletion."""
3255     self._Prepare(extra_cmd_args=['--branch-name', self.RELEASE_BRANCH_NAME,
3256                                   '--delete-branch'])
3257     self.rc_mock.AddCmdResult(
3258         partial_mock.ListRegex('git show-ref .*release-test-branch.*'),
3259         output='SomeSHA1Value'
3260     )
3261     self.RunStage()
3262     self._VerifyPush(self.norm_name, delete=True)
3263
3264   def testRename(self):
3265     """Branch rename."""
3266     self._Prepare(extra_cmd_args=['--branch-name', self.RELEASE_BRANCH_NAME,
3267                                   '--rename-to', 'refs/heads/release-rename'])
3268     # Simulate source branch existing and destination branch not existing.
3269     self.rc_mock.AddCmdResult(
3270         partial_mock.ListRegex('git show-ref .*%s' % self.RELEASE_BRANCH_NAME),
3271         output='SomeSHA1Value')
3272     self.rc_mock.AddCmdResult(
3273         partial_mock.ListRegex('git show-ref .*release-rename'),
3274         returncode=1)
3275     self.RunStage()
3276     self._VerifyPush(self.run.options.rename_to, rename_from=self.norm_name)
3277
3278   def testDryRun(self):
3279     """Verify all pushes are done with --dryrun when --debug is set."""
3280     def VerifyDryRun(cmd, *_args, **_kwargs):
3281       self.assertTrue('--dry-run' in cmd)
3282
3283     # Simulate branch not existing.
3284     self.rc_mock.AddCmdResult(
3285         partial_mock.ListRegex('git show-ref .*%s' % self.RELEASE_BRANCH_NAME),
3286         returncode=1)
3287
3288     self._Prepare(extra_cmd_args=['--branch-name', self.RELEASE_BRANCH_NAME,
3289                                   '--debug',
3290                                   '--version', self.DEFAULT_VERSION])
3291     self.rc_mock.AddCmdResult(partial_mock.In('push'),
3292                               side_effect=VerifyDryRun)
3293     self.RunStage()
3294     self.rc_mock.assertCommandContains(['push', '--dry-run'])
3295
3296   def _DetermineIncrForVersion(self, version):
3297     version_info = manifest_version.VersionInfo(version)
3298     stage_cls = stages.BranchUtilStage
3299     return stage_cls.DetermineBranchIncrParams(version_info)
3300
3301   def testDetermineIncrBranch(self):
3302     """Verify branch increment detection."""
3303     incr_type, _ = self._DetermineIncrForVersion(self.DEFAULT_VERSION)
3304     self.assertEquals(incr_type, 'branch')
3305
3306   def testDetermineIncrPatch(self):
3307     """Verify patch increment detection."""
3308     incr_type, _ = self._DetermineIncrForVersion('111.1.0')
3309     self.assertEquals(incr_type, 'patch')
3310
3311   def testDetermineBranchIncrError(self):
3312     """Detect unbranchable version."""
3313     self.assertRaises(stages.BranchError, self._DetermineIncrForVersion,
3314                       '111.1.1')
3315
3316   def _SimulateIncrementFailure(self):
3317     """Simulates a git push failure during source branch increment."""
3318     self._Prepare(extra_cmd_args=['--branch-name', self.RELEASE_BRANCH_NAME,
3319                                   '--version', self.DEFAULT_VERSION])
3320     overlay_dir = os.path.join(
3321         self.build_root, constants.CHROMIUMOS_OVERLAY_DIR)
3322     self.rc_mock.AddCmdResult(partial_mock.In('push'), returncode=128)
3323     stage = self.ConstructStage()
3324     args = (overlay_dir, 'gerrit', 'refs/heads/master')
3325     stage._IncrementVersionOnDiskForSourceBranch(*args)
3326
3327   def testSourceIncrementWarning(self):
3328     """Test the warning case for incrementing failure."""
3329     # Since all git commands are mocked out, the _FetchAndCheckoutTo function
3330     # does nothing, and leaves the chromeos_version.sh file in the bumped state,
3331     # so it looks like TOT version was indeed bumped by another bot.
3332     with cros_test_lib.LoggingCapturer() as logger:
3333       self._SimulateIncrementFailure()
3334       self.AssertLogsContain(logger, 'bumped by another')
3335
3336   def testSourceIncrementFailure(self):
3337     """Test the failure case for incrementing failure."""
3338     def FetchAndCheckoutTo(*_args, **_kwargs):
3339       self._CreateVersionFile()
3340
3341     # Simulate a git checkout of TOT.
3342     self.PatchObject(stages.BranchUtilStage, '_FetchAndCheckoutTo',
3343                      side_effect=FetchAndCheckoutTo, autospec=True)
3344     self.assertRaises(cros_build_lib.RunCommandError,
3345                       self._SimulateIncrementFailure)
3346
3347
3348 class PatchChromeStageTest(AbstractStageTest):
3349   """Tests for PatchChromeStage."""
3350
3351   def setUp(self):
3352     self._Prepare(cmd_args=[
3353         '-r', self.build_root,
3354         '--rietveld-patches=1234',
3355         '--rietveld-patches=555:adir',
3356     ])
3357     self.PatchObject(commands, 'PatchChrome')
3358
3359   def ConstructStage(self):
3360     return stages.PatchChromeStage(self.run)
3361
3362   def testBasic(self):
3363     """Verify requested patches are applied."""
3364     stage = self.ConstructStage()
3365     stage.PerformStage()
3366
3367
3368 if __name__ == '__main__':
3369   cros_test_lib.main()