twisted/trial/test/test_loader.py

   1 # Copyright (c) Twisted Matrix Laboratories.
   2 # See LICENSE for details.
   3
   4 """
   5 Tests for loading tests by name.
   6 """
   7
   8 import os
   9 import shutil
  10 import sys
  11
  12 from twisted.python import util
  13 from twisted.python.hashlib import md5
  14 from twisted.trial.test import packages
  15 from twisted.trial import runner, reporter, unittest
  16 from twisted.trial.itrial import ITestCase
  17
  18 from twisted.python.modules import getModule
  19
  20
  21
  22 def testNames(tests):
  23     """
  24     Return the id of each test within the given test suite or case.
  25     """
  26     names = []
  27     for test in unittest._iterateTests(tests):
  28         names.append(test.id())
  29     return names
  30
  31
  32
  33 class FinderTest(packages.PackageTest):
  34     def setUp(self):
  35         packages.PackageTest.setUp(self)
  36         self.loader = runner.TestLoader()
  37
  38     def tearDown(self):
  39         packages.PackageTest.tearDown(self)
  40
  41     def test_findPackage(self):
  42         sample1 = self.loader.findByName('twisted')
  43         import twisted as sample2
  44         self.assertEqual(sample1, sample2)
  45
  46     def test_findModule(self):
  47         sample1 = self.loader.findByName('twisted.trial.test.sample')
  48         import sample as sample2
  49         self.assertEqual(sample1, sample2)
  50
  51     def test_findFile(self):
  52         path = util.sibpath(__file__, 'sample.py')
  53         sample1 = self.loader.findByName(path)
  54         import sample as sample2
  55         self.assertEqual(sample1, sample2)
  56
  57     def test_findObject(self):
  58         sample1 = self.loader.findByName('twisted.trial.test.sample.FooTest')
  59         import sample
  60         self.assertEqual(sample.FooTest, sample1)
  61
  62     def test_findNonModule(self):
  63         self.failUnlessRaises(AttributeError,
  64                               self.loader.findByName,
  65                               'twisted.trial.test.nonexistent')
  66
  67     def test_findNonPackage(self):
  68         self.failUnlessRaises(ValueError,
  69                               self.loader.findByName,
  70                               'nonextant')
  71
  72     def test_findNonFile(self):
  73         path = util.sibpath(__file__, 'nonexistent.py')
  74         self.failUnlessRaises(ValueError, self.loader.findByName, path)
  75
  76
  77
  78 class FileTest(packages.SysPathManglingTest):
  79     """
  80     Tests for L{runner.filenameToModule}.
  81     """
  82     def test_notFile(self):
  83         self.failUnlessRaises(ValueError,
  84                               runner.filenameToModule, 'doesntexist')
  85
  86     def test_moduleInPath(self):
  87         sample1 = runner.filenameToModule(util.sibpath(__file__, 'sample.py'))
  88         import sample as sample2
  89         self.assertEqual(sample2, sample1)
  90
  91
  92     def test_moduleNotInPath(self):
  93         """
  94         If passed the path to a file containing the implementation of a
  95         module within a package which is not on the import path,
  96         L{runner.filenameToModule} returns a module object loosely
  97         resembling the module defined by that file anyway.
  98         """
  99         # "test_sample" isn't actually the name of this module.  However,
 100         # filenameToModule can't seem to figure that out.  So clean up this
 101         # mis-named module.  It would be better if this weren't necessary
 102         # and filenameToModule either didn't exist or added a correctly
 103         # named module to sys.modules.
 104         self.addCleanup(sys.modules.pop, 'test_sample', None)
 105
 106         self.mangleSysPath(self.oldPath)
 107         sample1 = runner.filenameToModule(
 108             os.path.join(self.parent, 'goodpackage', 'test_sample.py'))
 109         self.mangleSysPath(self.newPath)
 110         from goodpackage import test_sample as sample2
 111         self.assertEqual(os.path.splitext(sample2.__file__)[0],
 112                              os.path.splitext(sample1.__file__)[0])
 113
 114
 115     def test_packageInPath(self):
 116         package1 = runner.filenameToModule(os.path.join(self.parent,
 117                                                         'goodpackage'))
 118         import goodpackage
 119         self.assertEqual(goodpackage, package1)
 120
 121
 122     def test_packageNotInPath(self):
 123         """
 124         If passed the path to a directory which represents a package which
 125         is not on the import path, L{runner.filenameToModule} returns a
 126         module object loosely resembling the package defined by that
 127         directory anyway.
 128         """
 129         # "__init__" isn't actually the name of the package!  However,
 130         # filenameToModule is pretty stupid and decides that is its name
 131         # after all.  Make sure it gets cleaned up.  See the comment in
 132         # test_moduleNotInPath for possible courses of action related to
 133         # this.
 134         self.addCleanup(sys.modules.pop, "__init__")
 135
 136         self.mangleSysPath(self.oldPath)
 137         package1 = runner.filenameToModule(
 138             os.path.join(self.parent, 'goodpackage'))
 139         self.mangleSysPath(self.newPath)
 140         import goodpackage
 141         self.assertEqual(os.path.splitext(goodpackage.__file__)[0],
 142                              os.path.splitext(package1.__file__)[0])
 143
 144
 145     def test_directoryNotPackage(self):
 146         self.failUnlessRaises(ValueError, runner.filenameToModule,
 147                               util.sibpath(__file__, 'directory'))
 148
 149     def test_filenameNotPython(self):
 150         self.failUnlessRaises(ValueError, runner.filenameToModule,
 151                               util.sibpath(__file__, 'notpython.py'))
 152
 153     def test_filenameMatchesPackage(self):
 154         filename = os.path.join(self.parent, 'goodpackage.py')
 155         fd = open(filename, 'w')
 156         fd.write(packages.testModule)
 157         fd.close()
 158         try:
 159             module = runner.filenameToModule(filename)
 160             self.assertEqual(filename, module.__file__)
 161         finally:
 162             os.remove(filename)
 163
 164     def test_directory(self):
 165         """
 166         Test loader against a filesystem directory. It should handle
 167         'path' and 'path/' the same way.
 168         """
 169         path  = util.sibpath(__file__, 'goodDirectory')
 170         os.mkdir(path)
 171         f = file(os.path.join(path, '__init__.py'), "w")
 172         f.close()
 173         try:
 174             module = runner.filenameToModule(path)
 175             self.assert_(module.__name__.endswith('goodDirectory'))
 176             module = runner.filenameToModule(path + os.path.sep)
 177             self.assert_(module.__name__.endswith('goodDirectory'))
 178         finally:
 179             shutil.rmtree(path)
 180
 181
 182
 183 class LoaderTest(packages.SysPathManglingTest):
 184     """
 185     Tests for L{trial.TestLoader}.
 186     """
 187
 188     def setUp(self):
 189         self.loader = runner.TestLoader()
 190         packages.SysPathManglingTest.setUp(self)
 191
 192
 193     def test_sortCases(self):
 194         import sample
 195         suite = self.loader.loadClass(sample.AlphabetTest)
 196         self.assertEqual(['test_a', 'test_b', 'test_c'],
 197                              [test._testMethodName for test in suite._tests])
 198         newOrder = ['test_b', 'test_c', 'test_a']
 199         sortDict = dict(zip(newOrder, range(3)))
 200         self.loader.sorter = lambda x : sortDict.get(x.shortDescription(), -1)
 201         suite = self.loader.loadClass(sample.AlphabetTest)
 202         self.assertEqual(newOrder,
 203                              [test._testMethodName for test in suite._tests])
 204
 205
 206     def test_loadMethod(self):
 207         import sample
 208         suite = self.loader.loadMethod(sample.FooTest.test_foo)
 209         self.assertEqual(1, suite.countTestCases())
 210         self.assertEqual('test_foo', suite._testMethodName)
 211
 212
 213     def test_loadFailingMethod(self):
 214         # test added for issue1353
 215         import erroneous
 216         suite = self.loader.loadMethod(erroneous.TestRegularFail.test_fail)
 217         result = reporter.TestResult()
 218         suite.run(result)
 219         self.assertEqual(result.testsRun, 1)
 220         self.assertEqual(len(result.failures), 1)
 221
 222
 223     def test_loadNonMethod(self):
 224         import sample
 225         self.failUnlessRaises(TypeError, self.loader.loadMethod, sample)
 226         self.failUnlessRaises(TypeError,
 227                               self.loader.loadMethod, sample.FooTest)
 228         self.failUnlessRaises(TypeError, self.loader.loadMethod, "string")
 229         self.failUnlessRaises(TypeError,
 230                               self.loader.loadMethod, ('foo', 'bar'))
 231
 232
 233     def test_loadBadDecorator(self):
 234         """
 235         A decorated test method for which the decorator has failed to set the
 236         method's __name__ correctly is loaded and its name in the class scope
 237         discovered.
 238         """
 239         import sample
 240         suite = self.loader.loadMethod(sample.DecorationTest.test_badDecorator)
 241         self.assertEqual(1, suite.countTestCases())
 242         self.assertEqual('test_badDecorator', suite._testMethodName)
 243
 244
 245     def test_loadGoodDecorator(self):
 246         """
 247         A decorated test method for which the decorator has set the method's
 248         __name__ correctly is loaded and the only name by which it goes is used.
 249         """
 250         import sample
 251         suite = self.loader.loadMethod(
 252             sample.DecorationTest.test_goodDecorator)
 253         self.assertEqual(1, suite.countTestCases())
 254         self.assertEqual('test_goodDecorator', suite._testMethodName)
 255
 256
 257     def test_loadRenamedDecorator(self):
 258         """
 259         Load a decorated method which has been copied to a new name inside the
 260         class.  Thus its __name__ and its key in the class's __dict__ no
 261         longer match.
 262         """
 263         import sample
 264         suite = self.loader.loadMethod(
 265             sample.DecorationTest.test_renamedDecorator)
 266         self.assertEqual(1, suite.countTestCases())
 267         self.assertEqual('test_renamedDecorator', suite._testMethodName)
 268
 269
 270     def test_loadClass(self):
 271         import sample
 272         suite = self.loader.loadClass(sample.FooTest)
 273         self.assertEqual(2, suite.countTestCases())
 274         self.assertEqual(['test_bar', 'test_foo'],
 275                              [test._testMethodName for test in suite._tests])
 276
 277
 278     def test_loadNonClass(self):
 279         import sample
 280         self.failUnlessRaises(TypeError, self.loader.loadClass, sample)
 281         self.failUnlessRaises(TypeError,
 282                               self.loader.loadClass, sample.FooTest.test_foo)
 283         self.failUnlessRaises(TypeError, self.loader.loadClass, "string")
 284         self.failUnlessRaises(TypeError,
 285                               self.loader.loadClass, ('foo', 'bar'))
 286
 287
 288     def test_loadNonTestCase(self):
 289         import sample
 290         self.failUnlessRaises(ValueError, self.loader.loadClass,
 291                               sample.NotATest)
 292
 293
 294     def test_loadModule(self):
 295         import sample
 296         suite = self.loader.loadModule(sample)
 297         self.assertEqual(10, suite.countTestCases())
 298
 299
 300     def test_loadNonModule(self):
 301         import sample
 302         self.failUnlessRaises(TypeError,
 303                               self.loader.loadModule, sample.FooTest)
 304         self.failUnlessRaises(TypeError,
 305                               self.loader.loadModule, sample.FooTest.test_foo)
 306         self.failUnlessRaises(TypeError, self.loader.loadModule, "string")
 307         self.failUnlessRaises(TypeError,
 308                               self.loader.loadModule, ('foo', 'bar'))
 309
 310
 311     def test_loadPackage(self):
 312         import goodpackage
 313         suite = self.loader.loadPackage(goodpackage)
 314         self.assertEqual(7, suite.countTestCases())
 315
 316
 317     def test_loadNonPackage(self):
 318         import sample
 319         self.failUnlessRaises(TypeError,
 320                               self.loader.loadPackage, sample.FooTest)
 321         self.failUnlessRaises(TypeError,
 322                               self.loader.loadPackage, sample.FooTest.test_foo)
 323         self.failUnlessRaises(TypeError, self.loader.loadPackage, "string")
 324         self.failUnlessRaises(TypeError,
 325                               self.loader.loadPackage, ('foo', 'bar'))
 326
 327
 328     def test_loadModuleAsPackage(self):
 329         import sample
 330         ## XXX -- should this instead raise a ValueError? -- jml
 331         self.failUnlessRaises(TypeError, self.loader.loadPackage, sample)
 332
 333
 334     def test_loadPackageRecursive(self):
 335         import goodpackage
 336         suite = self.loader.loadPackage(goodpackage, recurse=True)
 337         self.assertEqual(14, suite.countTestCases())
 338
 339
 340     def test_loadAnythingOnModule(self):
 341         import sample
 342         suite = self.loader.loadAnything(sample)
 343         self.assertEqual(sample.__name__,
 344                              suite._tests[0]._tests[0].__class__.__module__)
 345
 346
 347     def test_loadAnythingOnClass(self):
 348         import sample
 349         suite = self.loader.loadAnything(sample.FooTest)
 350         self.assertEqual(2, suite.countTestCases())
 351
 352
 353     def test_loadAnythingOnMethod(self):
 354         import sample
 355         suite = self.loader.loadAnything(sample.FooTest.test_foo)
 356         self.assertEqual(1, suite.countTestCases())
 357
 358
 359     def test_loadAnythingOnPackage(self):
 360         import goodpackage
 361         suite = self.loader.loadAnything(goodpackage)
 362         self.failUnless(isinstance(suite, self.loader.suiteFactory))
 363         self.assertEqual(7, suite.countTestCases())
 364
 365
 366     def test_loadAnythingOnPackageRecursive(self):
 367         import goodpackage
 368         suite = self.loader.loadAnything(goodpackage, recurse=True)
 369         self.failUnless(isinstance(suite, self.loader.suiteFactory))
 370         self.assertEqual(14, suite.countTestCases())
 371
 372
 373     def test_loadAnythingOnString(self):
 374         # the important thing about this test is not the string-iness
 375         # but the non-handledness.
 376         self.failUnlessRaises(TypeError,
 377                               self.loader.loadAnything, "goodpackage")
 378
 379
 380     def test_importErrors(self):
 381         import package
 382         suite = self.loader.loadPackage(package, recurse=True)
 383         result = reporter.Reporter()
 384         suite.run(result)
 385         self.assertEqual(False, result.wasSuccessful())
 386         self.assertEqual(2, len(result.errors))
 387         errors = [test.id() for test, error in result.errors]
 388         errors.sort()
 389         self.assertEqual(errors, ['package.test_bad_module',
 390                                       'package.test_import_module'])
 391
 392
 393     def test_differentInstances(self):
 394         """
 395         L{TestLoader.loadClass} returns a suite with each test method
 396         represented by a different instances of the L{TestCase} they are
 397         defined on.
 398         """
 399         class DistinctInstances(unittest.TestCase):
 400             def test_1(self):
 401                 self.first = 'test1Run'
 402
 403             def test_2(self):
 404                 self.assertFalse(hasattr(self, 'first'))
 405
 406         suite = self.loader.loadClass(DistinctInstances)
 407         result = reporter.Reporter()
 408         suite.run(result)
 409         self.assertTrue(result.wasSuccessful())
 410
 411
 412     def test_loadModuleWith_test_suite(self):
 413         """
 414         Check that C{test_suite} is used when present and other L{TestCase}s are
 415         not included.
 416         """
 417         from twisted.trial.test import mockcustomsuite
 418         suite = self.loader.loadModule(mockcustomsuite)
 419         self.assertEqual(0, suite.countTestCases())
 420         self.assertEqual("MyCustomSuite", getattr(suite, 'name', None))
 421
 422
 423     def test_loadModuleWith_testSuite(self):
 424         """
 425         Check that C{testSuite} is used when present and other L{TestCase}s are
 426         not included.
 427         """
 428         from twisted.trial.test import mockcustomsuite2
 429         suite = self.loader.loadModule(mockcustomsuite2)
 430         self.assertEqual(0, suite.countTestCases())
 431         self.assertEqual("MyCustomSuite", getattr(suite, 'name', None))
 432
 433
 434     def test_loadModuleWithBothCustom(self):
 435         """
 436         Check that if C{testSuite} and C{test_suite} are both present in a
 437         module then C{testSuite} gets priority.
 438         """
 439         from twisted.trial.test import mockcustomsuite3
 440         suite = self.loader.loadModule(mockcustomsuite3)
 441         self.assertEqual('testSuite', getattr(suite, 'name', None))
 442
 443
 444     def test_customLoadRaisesAttributeError(self):
 445         """
 446         Make sure that any C{AttributeError}s raised by C{testSuite} are not
 447         swallowed by L{TestLoader}.
 448         """
 449         def testSuite():
 450             raise AttributeError('should be reraised')
 451         from twisted.trial.test import mockcustomsuite2
 452         mockcustomsuite2.testSuite, original = (testSuite,
 453                                                 mockcustomsuite2.testSuite)
 454         try:
 455             self.assertRaises(AttributeError, self.loader.loadModule,
 456                               mockcustomsuite2)
 457         finally:
 458             mockcustomsuite2.testSuite = original
 459
 460
 461     # XXX - duplicated and modified from test_script
 462     def assertSuitesEqual(self, test1, test2):
 463         names1 = testNames(test1)
 464         names2 = testNames(test2)
 465         names1.sort()
 466         names2.sort()
 467         self.assertEqual(names1, names2)
 468
 469
 470     def test_loadByNamesDuplicate(self):
 471         """
 472         Check that loadByNames ignores duplicate names
 473         """
 474         module = 'twisted.trial.test.test_test_visitor'
 475         suite1 = self.loader.loadByNames([module, module], True)
 476         suite2 = self.loader.loadByName(module, True)
 477         self.assertSuitesEqual(suite1, suite2)
 478
 479
 480     def test_loadDifferentNames(self):
 481         """
 482         Check that loadByNames loads all the names that it is given
 483         """
 484         modules = ['goodpackage', 'package.test_module']
 485         suite1 = self.loader.loadByNames(modules)
 486         suite2 = runner.TestSuite(map(self.loader.loadByName, modules))
 487         self.assertSuitesEqual(suite1, suite2)
 488
 489     def test_loadInheritedMethods(self):
 490         """
 491         Check that test methods names which are inherited from are all
 492         loaded rather than just one.
 493         """
 494         methods = ['inheritancepackage.test_x.A.test_foo',
 495                    'inheritancepackage.test_x.B.test_foo']
 496         suite1 = self.loader.loadByNames(methods)
 497         suite2 = runner.TestSuite(map(self.loader.loadByName, methods))
 498         self.assertSuitesEqual(suite1, suite2)
 499
 500
 501
 502 class ZipLoadingTest(LoaderTest):
 503     def setUp(self):
 504         from twisted.test.test_paths import zipit
 505         LoaderTest.setUp(self)
 506         zipit(self.parent, self.parent+'.zip')
 507         self.parent += '.zip'
 508         self.mangleSysPath(self.oldPath+[self.parent])
 509
 510
 511
 512 class PackageOrderingTest(packages.SysPathManglingTest):
 513     if sys.version_info < (2, 4):
 514         skip = (
 515             "Python 2.3 import semantics make this behavior incorrect on that "
 516             "version of Python as well as difficult to test.  The second "
 517             "import of a package which raised an exception the first time it "
 518             "was imported will succeed on Python 2.3, whereas it will fail on "
 519             "later versions of Python.  Trial does not account for this, so "
 520             "this test fails with inconsistencies between the expected and "
 521             "the received loader errors.")
 522
 523     def setUp(self):
 524         self.loader = runner.TestLoader()
 525         self.topDir = self.mktemp()
 526         parent = os.path.join(self.topDir, "uberpackage")
 527         os.makedirs(parent)
 528         file(os.path.join(parent, "__init__.py"), "wb").close()
 529         packages.SysPathManglingTest.setUp(self, parent)
 530         self.mangleSysPath(self.oldPath + [self.topDir])
 531
 532     def _trialSortAlgorithm(self, sorter):
 533         """
 534         Right now, halfway by accident, trial sorts like this:
 535
 536             1. all modules are grouped together in one list and sorted.
 537
 538             2. within each module, the classes are grouped together in one list
 539                and sorted.
 540
 541             3. finally within each class, each test method is grouped together
 542                in a list and sorted.
 543
 544         This attempts to return a sorted list of testable thingies following
 545         those rules, so that we can compare the behavior of loadPackage.
 546
 547         The things that show as 'cases' are errors from modules which failed to
 548         import, and test methods.  Let's gather all those together.
 549         """
 550         pkg = getModule('uberpackage')
 551         testModules = []
 552         for testModule in pkg.walkModules():
 553             if testModule.name.split(".")[-1].startswith("test_"):
 554                 testModules.append(testModule)
 555         sortedModules = sorted(testModules, key=sorter) # ONE
 556         for modinfo in sortedModules:
 557             # Now let's find all the classes.
 558             module = modinfo.load(None)
 559             if module is None:
 560                 yield modinfo
 561             else:
 562                 testClasses = []
 563                 for attrib in modinfo.iterAttributes():
 564                     if runner.isTestCase(attrib.load()):
 565                         testClasses.append(attrib)
 566                 sortedClasses = sorted(testClasses, key=sorter) # TWO
 567                 for clsinfo in sortedClasses:
 568                     testMethods = []
 569                     for attr in clsinfo.iterAttributes():
 570                         if attr.name.split(".")[-1].startswith('test'):
 571                             testMethods.append(attr)
 572                     sortedMethods = sorted(testMethods, key=sorter) # THREE
 573                     for methinfo in sortedMethods:
 574                         yield methinfo
 575
 576
 577     def loadSortedPackages(self, sorter=runner.name):
 578         """
 579         Verify that packages are loaded in the correct order.
 580         """
 581         import uberpackage
 582         self.loader.sorter = sorter
 583         suite = self.loader.loadPackage(uberpackage, recurse=True)
 584         # XXX: Work around strange, unexplained Zope crap.
 585         # jml, 2007-11-15.
 586         suite = unittest.decorate(suite, ITestCase)
 587         resultingTests = list(unittest._iterateTests(suite))
 588         manifest = list(self._trialSortAlgorithm(sorter))
 589         for number, (manifestTest, actualTest) in enumerate(
 590             zip(manifest, resultingTests)):
 591             self.assertEqual(
 592                  manifestTest.name, actualTest.id(),
 593                  "#%d: %s != %s" %
 594                  (number, manifestTest.name, actualTest.id()))
 595         self.assertEqual(len(manifest), len(resultingTests))
 596
 597
 598     def test_sortPackagesDefaultOrder(self):
 599         self.loadSortedPackages()
 600
 601
 602     def test_sortPackagesSillyOrder(self):
 603         def sillySorter(s):
 604             # This has to work on fully-qualified class names and class
 605             # objects, which is silly, but it's the "spec", such as it is.
 606 #             if isinstance(s, type) or isinstance(s, types.ClassType):
 607 #                 return s.__module__+'.'+s.__name__
 608             n = runner.name(s)
 609             d = md5(n).hexdigest()
 610             return d
 611         self.loadSortedPackages(sillySorter)