twisted/trial/test/test_runner.py

   1 # Copyright (c) Twisted Matrix Laboratories.
   2 # See LICENSE for details.
   3 #
   4 # Maintainer: Jonathan Lange
   5 # Author: Robert Collins
   6
   7
   8 import StringIO, os, sys
   9 from zope.interface import implements
  10 from zope.interface.verify import verifyObject
  11
  12 from twisted.trial.itrial import IReporter, ITestCase
  13 from twisted.trial import unittest, runner, reporter, util
  14 from twisted.python import failure, log, reflect, filepath
  15 from twisted.python.filepath import FilePath
  16 from twisted.scripts import trial
  17 from twisted.plugins import twisted_trial
  18 from twisted import plugin
  19 from twisted.internet import defer
  20
  21
  22 pyunit = __import__('unittest')
  23
  24
  25 class CapturingDebugger(object):
  26
  27     def __init__(self):
  28         self._calls = []
  29
  30     def runcall(self, *args, **kwargs):
  31         self._calls.append('runcall')
  32         args[0](*args[1:], **kwargs)
  33
  34
  35
  36 class CapturingReporter(object):
  37     """
  38     Reporter that keeps a log of all actions performed on it.
  39     """
  40
  41     implements(IReporter)
  42
  43     stream = None
  44     tbformat = None
  45     args = None
  46     separator = None
  47     testsRun = None
  48
  49     def __init__(self, stream=None, tbformat=None, rterrors=None,
  50                  publisher=None):
  51         """
  52         Create a capturing reporter.
  53         """
  54         self._calls = []
  55         self.shouldStop = False
  56         self._stream = stream
  57         self._tbformat = tbformat
  58         self._rterrors = rterrors
  59         self._publisher = publisher
  60
  61
  62     def startTest(self, method):
  63         """
  64         Report the beginning of a run of a single test method
  65         @param method: an object that is adaptable to ITestMethod
  66         """
  67         self._calls.append('startTest')
  68
  69
  70     def stopTest(self, method):
  71         """
  72         Report the status of a single test method
  73         @param method: an object that is adaptable to ITestMethod
  74         """
  75         self._calls.append('stopTest')
  76
  77
  78     def cleanupErrors(self, errs):
  79         """called when the reactor has been left in a 'dirty' state
  80         @param errs: a list of L{twisted.python.failure.Failure}s
  81         """
  82         self._calls.append('cleanupError')
  83
  84
  85     def addSuccess(self, test):
  86         self._calls.append('addSuccess')
  87
  88
  89     def done(self):
  90         """
  91         Do nothing. These tests don't care about done.
  92         """
  93
  94
  95
  96 class TrialRunnerTestsMixin:
  97     """
  98     Mixin defining tests for L{runner.TrialRunner}.
  99     """
 100     def tearDown(self):
 101         self.runner._tearDownLogFile()
 102
 103
 104     def test_empty(self):
 105         """
 106         Empty test method, used by the other tests.
 107         """
 108
 109
 110     def _getObservers(self):
 111         return log.theLogPublisher.observers
 112
 113
 114     def test_addObservers(self):
 115         """
 116         Any log system observers L{TrialRunner.run} adds are removed by the
 117         time it returns.
 118         """
 119         originalCount = len(self._getObservers())
 120         self.runner.run(self.test)
 121         newCount = len(self._getObservers())
 122         self.assertEqual(newCount, originalCount)
 123
 124
 125     def test_logFileAlwaysActive(self):
 126         """
 127         Test that a new file is opened on each run.
 128         """
 129         oldSetUpLogFile = self.runner._setUpLogFile
 130         l = []
 131         def setUpLogFile():
 132             oldSetUpLogFile()
 133             l.append(self.runner._logFileObserver)
 134         self.runner._setUpLogFile = setUpLogFile
 135         self.runner.run(self.test)
 136         self.runner.run(self.test)
 137         self.assertEqual(len(l), 2)
 138         self.failIf(l[0] is l[1], "Should have created a new file observer")
 139
 140
 141     def test_logFileGetsClosed(self):
 142         """
 143         Test that file created is closed during the run.
 144         """
 145         oldSetUpLogFile = self.runner._setUpLogFile
 146         l = []
 147         def setUpLogFile():
 148             oldSetUpLogFile()
 149             l.append(self.runner._logFileObject)
 150         self.runner._setUpLogFile = setUpLogFile
 151         self.runner.run(self.test)
 152         self.assertEqual(len(l), 1)
 153         self.failUnless(l[0].closed)
 154
 155
 156
 157 class TestTrialRunner(TrialRunnerTestsMixin, unittest.TestCase):
 158     """
 159     Tests for L{runner.TrialRunner} with the feature to turn unclean errors
 160     into warnings disabled.
 161     """
 162     def setUp(self):
 163         self.stream = StringIO.StringIO()
 164         self.runner = runner.TrialRunner(CapturingReporter, stream=self.stream)
 165         self.test = TestTrialRunner('test_empty')
 166
 167
 168     def test_publisher(self):
 169         """
 170         The reporter constructed by L{runner.TrialRunner} is passed
 171         L{twisted.python.log} as the value for the C{publisher} parameter.
 172         """
 173         result = self.runner._makeResult()
 174         self.assertIdentical(result._publisher, log)
 175
 176
 177
 178 class TrialRunnerWithUncleanWarningsReporter(TrialRunnerTestsMixin,
 179                                              unittest.TestCase):
 180     """
 181     Tests for the TrialRunner's interaction with an unclean-error suppressing
 182     reporter.
 183     """
 184
 185     def setUp(self):
 186         self.stream = StringIO.StringIO()
 187         self.runner = runner.TrialRunner(CapturingReporter, stream=self.stream,
 188                                          uncleanWarnings=True)
 189         self.test = TestTrialRunner('test_empty')
 190
 191
 192
 193 class DryRunMixin(object):
 194
 195     suppress = [util.suppress(
 196         category=DeprecationWarning,
 197         message="Test visitors deprecated in Twisted 8.0")]
 198
 199
 200     def setUp(self):
 201         self.log = []
 202         self.stream = StringIO.StringIO()
 203         self.runner = runner.TrialRunner(CapturingReporter,
 204                                          runner.TrialRunner.DRY_RUN,
 205                                          stream=self.stream)
 206         self.makeTestFixtures()
 207
 208
 209     def makeTestFixtures(self):
 210         """
 211         Set C{self.test} and C{self.suite}, where C{self.suite} is an empty
 212         TestSuite.
 213         """
 214
 215
 216     def test_empty(self):
 217         """
 218         If there are no tests, the reporter should not receive any events to
 219         report.
 220         """
 221         result = self.runner.run(runner.TestSuite())
 222         self.assertEqual(result._calls, [])
 223
 224
 225     def test_singleCaseReporting(self):
 226         """
 227         If we are running a single test, check the reporter starts, passes and
 228         then stops the test during a dry run.
 229         """
 230         result = self.runner.run(self.test)
 231         self.assertEqual(result._calls, ['startTest', 'addSuccess', 'stopTest'])
 232
 233
 234     def test_testsNotRun(self):
 235         """
 236         When we are doing a dry run, the tests should not actually be run.
 237         """
 238         self.runner.run(self.test)
 239         self.assertEqual(self.log, [])
 240
 241
 242
 243 class DryRunTest(DryRunMixin, unittest.TestCase):
 244     """
 245     Check that 'dry run' mode works well with Trial tests.
 246     """
 247     def makeTestFixtures(self):
 248         class MockTest(unittest.TestCase):
 249             def test_foo(test):
 250                 self.log.append('test_foo')
 251         self.test = MockTest('test_foo')
 252         self.suite = runner.TestSuite()
 253
 254
 255
 256 class PyUnitDryRunTest(DryRunMixin, unittest.TestCase):
 257     """
 258     Check that 'dry run' mode works well with stdlib unittest tests.
 259     """
 260     def makeTestFixtures(self):
 261         class PyunitCase(pyunit.TestCase):
 262             def test_foo(self):
 263                 pass
 264         self.test = PyunitCase('test_foo')
 265         self.suite = pyunit.TestSuite()
 266
 267
 268
 269 class TestRunner(unittest.TestCase):
 270     def setUp(self):
 271         self.config = trial.Options()
 272         # whitebox hack a reporter in, because plugins are CACHED and will
 273         # only reload if the FILE gets changed.
 274
 275         parts = reflect.qual(CapturingReporter).split('.')
 276         package = '.'.join(parts[:-1])
 277         klass = parts[-1]
 278         plugins = [twisted_trial._Reporter(
 279             "Test Helper Reporter",
 280             package,
 281             description="Utility for unit testing.",
 282             longOpt="capturing",
 283             shortOpt=None,
 284             klass=klass)]
 285
 286
 287         # XXX There should really be a general way to hook the plugin system
 288         # for tests.
 289         def getPlugins(iface, *a, **kw):
 290             self.assertEqual(iface, IReporter)
 291             return plugins + list(self.original(iface, *a, **kw))
 292
 293         self.original = plugin.getPlugins
 294         plugin.getPlugins = getPlugins
 295
 296         self.standardReport = ['startTest', 'addSuccess', 'stopTest',
 297                                'startTest', 'addSuccess', 'stopTest',
 298                                'startTest', 'addSuccess', 'stopTest',
 299                                'startTest', 'addSuccess', 'stopTest',
 300                                'startTest', 'addSuccess', 'stopTest',
 301                                'startTest', 'addSuccess', 'stopTest',
 302                                'startTest', 'addSuccess', 'stopTest',
 303                                'startTest', 'addSuccess', 'stopTest',
 304                                'startTest', 'addSuccess', 'stopTest',
 305                                'startTest', 'addSuccess', 'stopTest']
 306
 307
 308     def tearDown(self):
 309         plugin.getPlugins = self.original
 310
 311
 312     def parseOptions(self, args):
 313         self.config.parseOptions(args)
 314
 315
 316     def getRunner(self):
 317         r = trial._makeRunner(self.config)
 318         r.stream = StringIO.StringIO()
 319         # XXX The runner should always take care of cleaning this up itself.
 320         # It's not clear why this is necessary.  The runner always tears down
 321         # its log file.
 322         self.addCleanup(r._tearDownLogFile)
 323         # XXX The runner should always take care of cleaning this up itself as
 324         # well.  It's necessary because TrialRunner._setUpTestdir might raise
 325         # an exception preventing Reporter.done from being run, leaving the
 326         # observer added by Reporter.__init__ still present in the system.
 327         # Something better needs to happen inside
 328         # TrialRunner._runWithoutDecoration to remove the need for this cludge.
 329         r._log = log.LogPublisher()
 330         return r
 331
 332
 333     def test_runner_can_get_reporter(self):
 334         self.parseOptions([])
 335         result = self.config['reporter']
 336         runner = self.getRunner()
 337         self.assertEqual(result, runner._makeResult().__class__)
 338
 339
 340     def test_runner_get_result(self):
 341         self.parseOptions([])
 342         runner = self.getRunner()
 343         result = runner._makeResult()
 344         self.assertEqual(result.__class__, self.config['reporter'])
 345
 346
 347     def test_uncleanWarningsOffByDefault(self):
 348         """
 349         By default Trial sets the 'uncleanWarnings' option on the runner to
 350         False. This means that dirty reactor errors will be reported as
 351         errors. See L{test_reporter.TestDirtyReactor}.
 352         """
 353         self.parseOptions([])
 354         runner = self.getRunner()
 355         self.assertNotIsInstance(runner._makeResult(),
 356                                  reporter.UncleanWarningsReporterWrapper)
 357
 358
 359     def test_getsUncleanWarnings(self):
 360         """
 361         Specifying '--unclean-warnings' on the trial command line will cause
 362         reporters to be wrapped in a device which converts unclean errors to
 363         warnings.  See L{test_reporter.TestDirtyReactor} for implications.
 364         """
 365         self.parseOptions(['--unclean-warnings'])
 366         runner = self.getRunner()
 367         self.assertIsInstance(runner._makeResult(),
 368                               reporter.UncleanWarningsReporterWrapper)
 369
 370
 371     def test_runner_working_directory(self):
 372         self.parseOptions(['--temp-directory', 'some_path'])
 373         runner = self.getRunner()
 374         self.assertEqual(runner.workingDirectory, 'some_path')
 375
 376
 377     def test_concurrentImplicitWorkingDirectory(self):
 378         """
 379         If no working directory is explicitly specified and the default
 380         working directory is in use by another runner, L{TrialRunner.run}
 381         selects a different default working directory to use.
 382         """
 383         self.parseOptions([])
 384
 385         # Make sure we end up with the same working directory after this test
 386         # as we had before it.
 387         self.addCleanup(os.chdir, os.getcwd())
 388
 389         # Make a new directory and change into it.  This isolates us from state
 390         # that other tests might have dumped into this process's temp
 391         # directory.
 392         runDirectory = FilePath(self.mktemp())
 393         runDirectory.makedirs()
 394         os.chdir(runDirectory.path)
 395
 396         firstRunner = self.getRunner()
 397         secondRunner = self.getRunner()
 398
 399         where = {}
 400
 401         class ConcurrentCase(unittest.TestCase):
 402             def test_first(self):
 403                 """
 404                 Start a second test run which will have a default working
 405                 directory which is the same as the working directory of the
 406                 test run already in progress.
 407                 """
 408                 # Change the working directory to the value it had before this
 409                 # test suite was started.
 410                 where['concurrent'] = subsequentDirectory = os.getcwd()
 411                 os.chdir(runDirectory.path)
 412                 self.addCleanup(os.chdir, subsequentDirectory)
 413
 414                 secondRunner.run(ConcurrentCase('test_second'))
 415
 416             def test_second(self):
 417                 """
 418                 Record the working directory for later analysis.
 419                 """
 420                 where['record'] = os.getcwd()
 421
 422         result = firstRunner.run(ConcurrentCase('test_first'))
 423         bad = result.errors + result.failures
 424         if bad:
 425             self.fail(bad[0][1])
 426         self.assertEqual(
 427             where, {
 428                 'concurrent': runDirectory.child('_trial_temp').path,
 429                 'record': runDirectory.child('_trial_temp-1').path})
 430
 431
 432     def test_concurrentExplicitWorkingDirectory(self):
 433         """
 434         If a working directory which is already in use is explicitly specified,
 435         L{TrialRunner.run} raises L{_WorkingDirectoryBusy}.
 436         """
 437         self.parseOptions(['--temp-directory', os.path.abspath(self.mktemp())])
 438
 439         initialDirectory = os.getcwd()
 440         self.addCleanup(os.chdir, initialDirectory)
 441
 442         firstRunner = self.getRunner()
 443         secondRunner = self.getRunner()
 444
 445         class ConcurrentCase(unittest.TestCase):
 446             def test_concurrent(self):
 447                 """
 448                 Try to start another runner in the same working directory and
 449                 assert that it raises L{_WorkingDirectoryBusy}.
 450                 """
 451                 self.assertRaises(
 452                     util._WorkingDirectoryBusy,
 453                     secondRunner.run, ConcurrentCase('test_failure'))
 454
 455             def test_failure(self):
 456                 """
 457                 Should not be called, always fails.
 458                 """
 459                 self.fail("test_failure should never be called.")
 460
 461         result = firstRunner.run(ConcurrentCase('test_concurrent'))
 462         bad = result.errors + result.failures
 463         if bad:
 464             self.fail(bad[0][1])
 465
 466
 467     def test_runner_normal(self):
 468         self.parseOptions(['--temp-directory', self.mktemp(),
 469                            '--reporter', 'capturing',
 470                            'twisted.trial.test.sample'])
 471         my_runner = self.getRunner()
 472         loader = runner.TestLoader()
 473         suite = loader.loadByName('twisted.trial.test.sample', True)
 474         result = my_runner.run(suite)
 475         self.assertEqual(self.standardReport, result._calls)
 476
 477
 478     def test_runner_debug(self):
 479         self.parseOptions(['--reporter', 'capturing',
 480                            '--debug', 'twisted.trial.test.sample'])
 481         my_runner = self.getRunner()
 482         debugger = CapturingDebugger()
 483         def get_debugger():
 484             return debugger
 485         my_runner._getDebugger = get_debugger
 486         loader = runner.TestLoader()
 487         suite = loader.loadByName('twisted.trial.test.sample', True)
 488         result = my_runner.run(suite)
 489         self.assertEqual(self.standardReport, result._calls)
 490         self.assertEqual(['runcall'], debugger._calls)
 491
 492
 493
 494 class RemoveSafelyTests(unittest.TestCase):
 495     """
 496     Tests for L{_removeSafely}.
 497     """
 498     def test_removeSafelyNoTrialMarker(self):
 499         """
 500         If a path doesn't contain a node named C{"_trial_marker"}, that path is
 501         not removed by L{runner._removeSafely} and a L{runner._NoTrialMarker}
 502         exception is raised instead.
 503         """
 504         directory = self.mktemp()
 505         os.mkdir(directory)
 506         dirPath = filepath.FilePath(directory)
 507         self.assertRaises(util._NoTrialMarker, util._removeSafely, dirPath)
 508
 509
 510     def test_removeSafelyRemoveFailsMoveSucceeds(self):
 511         """
 512         If an L{OSError} is raised while removing a path in
 513         L{runner._removeSafely}, an attempt is made to move the path to a new
 514         name.
 515         """
 516         def dummyRemove():
 517             """
 518             Raise an C{OSError} to emulate the branch of L{runner._removeSafely}
 519             in which path removal fails.
 520             """
 521             raise OSError()
 522
 523         # Patch stdout so we can check the print statements in _removeSafely
 524         out = StringIO.StringIO()
 525         self.patch(sys, 'stdout', out)
 526
 527         # Set up a trial directory with a _trial_marker
 528         directory = self.mktemp()
 529         os.mkdir(directory)
 530         dirPath = filepath.FilePath(directory)
 531         dirPath.child('_trial_marker').touch()
 532         # Ensure that path.remove() raises an OSError
 533         dirPath.remove = dummyRemove
 534
 535         util._removeSafely(dirPath)
 536         self.assertIn("could not remove FilePath", out.getvalue())
 537
 538
 539     def test_removeSafelyRemoveFailsMoveFails(self):
 540         """
 541         If an L{OSError} is raised while removing a path in
 542         L{runner._removeSafely}, an attempt is made to move the path to a new
 543         name. If that attempt fails, the L{OSError} is re-raised.
 544         """
 545         def dummyRemove():
 546             """
 547             Raise an C{OSError} to emulate the branch of L{runner._removeSafely}
 548             in which path removal fails.
 549             """
 550             raise OSError("path removal failed")
 551
 552         def dummyMoveTo(path):
 553             """
 554             Raise an C{OSError} to emulate the branch of L{runner._removeSafely}
 555             in which path movement fails.
 556             """
 557             raise OSError("path movement failed")
 558
 559         # Patch stdout so we can check the print statements in _removeSafely
 560         out = StringIO.StringIO()
 561         self.patch(sys, 'stdout', out)
 562
 563         # Set up a trial directory with a _trial_marker
 564         directory = self.mktemp()
 565         os.mkdir(directory)
 566         dirPath = filepath.FilePath(directory)
 567         dirPath.child('_trial_marker').touch()
 568
 569         # Ensure that path.remove() and path.moveTo() both raise OSErrors
 570         dirPath.remove = dummyRemove
 571         dirPath.moveTo = dummyMoveTo
 572
 573         error = self.assertRaises(OSError, util._removeSafely, dirPath)
 574         self.assertEqual(str(error), "path movement failed")
 575         self.assertIn("could not remove FilePath", out.getvalue())
 576
 577
 578
 579 class TestTrialSuite(unittest.TestCase):
 580
 581     def test_imports(self):
 582         # FIXME, HTF do you test the reactor can be cleaned up ?!!!
 583         from twisted.trial.runner import TrialSuite
 584
 585
 586
 587
 588 class TestUntilFailure(unittest.TestCase):
 589     class FailAfter(unittest.TestCase):
 590         """
 591         A test  case that fails when run 3 times in a row.
 592         """
 593         count = []
 594         def test_foo(self):
 595             self.count.append(None)
 596             if len(self.count) == 3:
 597                 self.fail('Count reached 3')
 598
 599
 600     def setUp(self):
 601         TestUntilFailure.FailAfter.count = []
 602         self.test = TestUntilFailure.FailAfter('test_foo')
 603         self.stream = StringIO.StringIO()
 604         self.runner = runner.TrialRunner(reporter.Reporter, stream=self.stream)
 605
 606
 607     def test_runUntilFailure(self):
 608         """
 609         Test that the runUntilFailure method of the runner actually fail after
 610         a few runs.
 611         """
 612         result = self.runner.runUntilFailure(self.test)
 613         self.assertEqual(result.testsRun, 1)
 614         self.failIf(result.wasSuccessful())
 615         self.assertEqual(self._getFailures(result), 1)
 616
 617
 618     def _getFailures(self, result):
 619         """
 620         Get the number of failures that were reported to a result.
 621         """
 622         return len(result.failures)
 623
 624
 625     def test_runUntilFailureDecorate(self):
 626         """
 627         C{runUntilFailure} doesn't decorate the tests uselessly: it does it one
 628         time when run starts, but not at each turn.
 629         """
 630         decorated = []
 631         def decorate(test, interface):
 632             decorated.append((test, interface))
 633             return test
 634         self.patch(unittest, "decorate", decorate)
 635         result = self.runner.runUntilFailure(self.test)
 636         self.assertEqual(result.testsRun, 1)
 637
 638         self.assertEqual(len(decorated), 1)
 639         self.assertEqual(decorated, [(self.test, ITestCase)])
 640
 641
 642     def test_runUntilFailureForceGCDecorate(self):
 643         """
 644         C{runUntilFailure} applies the force-gc decoration after the standard
 645         L{ITestCase} decoration, but only one time.
 646         """
 647         decorated = []
 648         def decorate(test, interface):
 649             decorated.append((test, interface))
 650             return test
 651         self.patch(unittest, "decorate", decorate)
 652         self.runner._forceGarbageCollection = True
 653         result = self.runner.runUntilFailure(self.test)
 654         self.assertEqual(result.testsRun, 1)
 655
 656         self.assertEqual(len(decorated), 2)
 657         self.assertEqual(decorated,
 658             [(self.test, ITestCase),
 659              (self.test, unittest._ForceGarbageCollectionDecorator)])
 660
 661
 662
 663 class UncleanUntilFailureTests(TestUntilFailure):
 664     """
 665     Test that the run-until-failure feature works correctly with the unclean
 666     error suppressor.
 667     """
 668
 669     def setUp(self):
 670         TestUntilFailure.setUp(self)
 671         self.runner = runner.TrialRunner(reporter.Reporter, stream=self.stream,
 672                                          uncleanWarnings=True)
 673
 674     def _getFailures(self, result):
 675         """
 676         Get the number of failures that were reported to a result that
 677         is wrapped in an UncleanFailureWrapper.
 678         """
 679         return len(result._originalReporter.failures)
 680
 681
 682
 683 class BreakingSuite(runner.TestSuite):
 684     """
 685     A L{TestSuite} that logs an error when it is run.
 686     """
 687
 688     def run(self, result):
 689         try:
 690             raise RuntimeError("error that occurs outside of a test")
 691         except RuntimeError:
 692             log.err(failure.Failure())
 693
 694
 695
 696 class TestLoggedErrors(unittest.TestCase):
 697     """
 698     It is possible for an error generated by a test to be logged I{outside} of
 699     any test. The log observers constructed by L{TestCase} won't catch these
 700     errors. Here we try to generate such errors and ensure they are reported to
 701     a L{TestResult} object.
 702     """
 703
 704     def tearDown(self):
 705         self.flushLoggedErrors(RuntimeError)
 706
 707
 708     def test_construct(self):
 709         """
 710         Check that we can construct a L{runner.LoggedSuite} and that it
 711         starts empty.
 712         """
 713         suite = runner.LoggedSuite()
 714         self.assertEqual(suite.countTestCases(), 0)
 715
 716
 717     def test_capturesError(self):
 718         """
 719         Chek that a L{LoggedSuite} reports any logged errors to its result.
 720         """
 721         result = reporter.TestResult()
 722         suite = runner.LoggedSuite([BreakingSuite()])
 723         suite.run(result)
 724         self.assertEqual(len(result.errors), 1)
 725         self.assertEqual(result.errors[0][0].id(), runner.NOT_IN_TEST)
 726         self.failUnless(result.errors[0][1].check(RuntimeError))
 727
 728
 729
 730 class TestTestHolder(unittest.TestCase):
 731
 732     def setUp(self):
 733         self.description = "description"
 734         self.holder = runner.TestHolder(self.description)
 735
 736
 737     def test_holder(self):
 738         """
 739         Check that L{runner.TestHolder} takes a description as a parameter
 740         and that this description is returned by the C{id} and
 741         C{shortDescription} methods.
 742         """
 743         self.assertEqual(self.holder.id(), self.description)
 744         self.assertEqual(self.holder.shortDescription(), self.description)
 745
 746
 747     def test_holderImplementsITestCase(self):
 748         """
 749         L{runner.TestHolder} implements L{ITestCase}.
 750         """
 751         self.assertIdentical(self.holder, ITestCase(self.holder))
 752         self.assertTrue(
 753             verifyObject(ITestCase, self.holder),
 754             "%r claims to provide %r but does not do so correctly."
 755             % (self.holder, ITestCase))
 756
 757
 758     def test_runsWithStandardResult(self):
 759         """
 760         A L{runner.TestHolder} can run against the standard Python
 761         C{TestResult}.
 762         """
 763         result = pyunit.TestResult()
 764         self.holder.run(result)
 765         self.assertTrue(result.wasSuccessful())
 766         self.assertEqual(1, result.testsRun)
 767
 768
 769
 770 class ErrorHolderTestsMixin(object):
 771     """
 772     This mixin defines test methods which can be applied to a
 773     L{runner.ErrorHolder} constructed with either a L{Failure} or a
 774     C{exc_info}-style tuple.
 775
 776     Subclass this and implement C{setUp} to create C{self.holder} referring to a
 777     L{runner.ErrorHolder} instance and C{self.error} referring to a L{Failure}
 778     which the holder holds.
 779     """
 780     exceptionForTests = ZeroDivisionError('integer division or modulo by zero')
 781
 782     class TestResultStub(object):
 783         """
 784         Stub for L{TestResult}.
 785         """
 786         def __init__(self):
 787             self.errors = []
 788
 789         def startTest(self, test):
 790             pass
 791
 792         def stopTest(self, test):
 793             pass
 794
 795         def addError(self, test, error):
 796             self.errors.append((test, error))
 797
 798
 799     def test_runsWithStandardResult(self):
 800         """
 801         A L{runner.ErrorHolder} can run against the standard Python
 802         C{TestResult}.
 803         """
 804         result = pyunit.TestResult()
 805         self.holder.run(result)
 806         self.assertFalse(result.wasSuccessful())
 807         self.assertEqual(1, result.testsRun)
 808
 809
 810     def test_run(self):
 811         """
 812         L{runner.ErrorHolder} adds an error to the result when run.
 813         """
 814         self.holder.run(self.result)
 815         self.assertEqual(
 816             self.result.errors,
 817             [(self.holder, (self.error.type, self.error.value, self.error.tb))])
 818
 819
 820     def test_call(self):
 821         """
 822         L{runner.ErrorHolder} adds an error to the result when called.
 823         """
 824         self.holder(self.result)
 825         self.assertEqual(
 826             self.result.errors,
 827             [(self.holder, (self.error.type, self.error.value, self.error.tb))])
 828
 829
 830     def test_countTestCases(self):
 831         """
 832         L{runner.ErrorHolder.countTestCases} always returns 0.
 833         """
 834         self.assertEqual(self.holder.countTestCases(), 0)
 835
 836
 837     def test_repr(self):
 838         """
 839         L{runner.ErrorHolder.__repr__} returns a string describing the error it
 840         holds.
 841         """
 842         self.assertEqual(repr(self.holder),
 843             "<ErrorHolder description='description' "
 844             "error=ZeroDivisionError('integer division or modulo by zero',)>")
 845
 846
 847
 848 class FailureHoldingErrorHolderTests(ErrorHolderTestsMixin, TestTestHolder):
 849     """
 850     Tests for L{runner.ErrorHolder} behaving similarly to L{runner.TestHolder}
 851     when constructed with a L{Failure} representing its error.
 852     """
 853     def setUp(self):
 854         self.description = "description"
 855         # make a real Failure so we can construct ErrorHolder()
 856         try:
 857             raise self.exceptionForTests
 858         except ZeroDivisionError:
 859             self.error = failure.Failure()
 860         self.holder = runner.ErrorHolder(self.description, self.error)
 861         self.result = self.TestResultStub()
 862
 863
 864
 865 class ExcInfoHoldingErrorHolderTests(ErrorHolderTestsMixin, TestTestHolder):
 866     """
 867     Tests for L{runner.ErrorHolder} behaving similarly to L{runner.TestHolder}
 868     when constructed with a C{exc_info}-style tuple representing its error.
 869     """
 870     def setUp(self):
 871         self.description = "description"
 872         # make a real Failure so we can construct ErrorHolder()
 873         try:
 874             raise self.exceptionForTests
 875         except ZeroDivisionError:
 876             exceptionInfo = sys.exc_info()
 877             self.error = failure.Failure()
 878         self.holder = runner.ErrorHolder(self.description, exceptionInfo)
 879         self.result = self.TestResultStub()
 880
 881
 882
 883 class TestMalformedMethod(unittest.TestCase):
 884     """
 885     Test that trial manages when test methods don't have correct signatures.
 886     """
 887     class ContainMalformed(unittest.TestCase):
 888         """
 889         This TestCase holds malformed test methods that trial should handle.
 890         """
 891         def test_foo(self, blah):
 892             pass
 893         def test_bar():
 894             pass
 895         test_spam = defer.deferredGenerator(test_bar)
 896
 897     def _test(self, method):
 898         """
 899         Wrapper for one of the test method of L{ContainMalformed}.
 900         """
 901         stream = StringIO.StringIO()
 902         trialRunner = runner.TrialRunner(reporter.Reporter, stream=stream)
 903         test = TestMalformedMethod.ContainMalformed(method)
 904         result = trialRunner.run(test)
 905         self.assertEqual(result.testsRun, 1)
 906         self.failIf(result.wasSuccessful())
 907         self.assertEqual(len(result.errors), 1)
 908
 909     def test_extraArg(self):
 910         """
 911         Test when the method has extra (useless) arguments.
 912         """
 913         self._test('test_foo')
 914
 915     def test_noArg(self):
 916         """
 917         Test when the method doesn't have even self as argument.
 918         """
 919         self._test('test_bar')
 920
 921     def test_decorated(self):
 922         """
 923         Test a decorated method also fails.
 924         """
 925         self._test('test_spam')
 926
 927
 928
 929 class DestructiveTestSuiteTestCase(unittest.TestCase):
 930     """
 931     Test for L{runner.DestructiveTestSuite}.
 932     """
 933
 934     def test_basic(self):
 935         """
 936         Thes destructive test suite should run the tests normally.
 937         """
 938         called = []
 939         class MockTest(unittest.TestCase):
 940             def test_foo(test):
 941                 called.append(True)
 942         test = MockTest('test_foo')
 943         result = reporter.TestResult()
 944         suite = runner.DestructiveTestSuite([test])
 945         self.assertEqual(called, [])
 946         suite.run(result)
 947         self.assertEqual(called, [True])
 948         self.assertEqual(suite.countTestCases(), 0)
 949
 950
 951     def test_shouldStop(self):
 952         """
 953         Test the C{shouldStop} management: raising a C{KeyboardInterrupt} must
 954         interrupt the suite.
 955         """
 956         called = []
 957         class MockTest(unittest.TestCase):
 958             def test_foo1(test):
 959                 called.append(1)
 960             def test_foo2(test):
 961                 raise KeyboardInterrupt()
 962             def test_foo3(test):
 963                 called.append(2)
 964         result = reporter.TestResult()
 965         loader = runner.TestLoader()
 966         loader.suiteFactory = runner.DestructiveTestSuite
 967         suite = loader.loadClass(MockTest)
 968         self.assertEqual(called, [])
 969         suite.run(result)
 970         self.assertEqual(called, [1])
 971         # The last test shouldn't have been run
 972         self.assertEqual(suite.countTestCases(), 1)
 973
 974
 975     def test_cleanup(self):
 976         """
 977         Checks that the test suite cleanups its tests during the run, so that
 978         it ends empty.
 979         """
 980         class MockTest(unittest.TestCase):
 981             def test_foo(test):
 982                 pass
 983         test = MockTest('test_foo')
 984         result = reporter.TestResult()
 985         suite = runner.DestructiveTestSuite([test])
 986         self.assertEqual(suite.countTestCases(), 1)
 987         suite.run(result)
 988         self.assertEqual(suite.countTestCases(), 0)
 989
 990
 991
 992 class TestRunnerDeprecation(unittest.TestCase):
 993
 994     class FakeReporter(reporter.Reporter):
 995         """
 996         Fake reporter that does *not* implement done() but *does* implement
 997         printErrors, separator, printSummary, stream, write and writeln
 998         without deprecations.
 999         """
1000
1001         done = None
1002         separator = None
1003         stream = None
1004
1005         def printErrors(self, *args):
1006             pass
1007
1008         def printSummary(self, *args):
1009             pass
1010
1011         def write(self, *args):
1012             pass
1013
1014         def writeln(self, *args):
1015             pass
1016
1017
1018     def test_reporterDeprecations(self):
1019         """
1020         The runner emits a warning if it is using a result that doesn't
1021         implement 'done'.
1022         """
1023         trialRunner = runner.TrialRunner(None)
1024         result = self.FakeReporter()
1025         trialRunner._makeResult = lambda: result
1026         def f():
1027             # We have to use a pyunit test, otherwise we'll get deprecation
1028             # warnings about using iterate() in a test.
1029             trialRunner.run(pyunit.TestCase('id'))
1030         self.assertWarns(
1031             DeprecationWarning,
1032             "%s should implement done() but doesn't. Falling back to "
1033             "printErrors() and friends." % reflect.qual(result.__class__),
1034             __file__, f)