xstc/xstc.py

   1 #!/usr/bin/env python
   2
   3 #
   4 # This is the MS subset of the W3C test suite for XML Schemas.
   5 # This file is generated from the MS W3c test suite description file.
   6 #
   7
   8 import sys, os
   9 import exceptions, optparse
  10 import libxml2
  11
  12 opa = optparse.OptionParser()
  13
  14 opa.add_option("-b", "--base", action="store", type="string", dest="baseDir",
  15                            default="",
  16                            help="""The base directory; i.e. the parent folder of the
  17                            "nisttest", "suntest" and "msxsdtest" directories.""")
  18
  19 opa.add_option("-o", "--out", action="store", type="string", dest="logFile",
  20                            default="test.log",
  21                            help="The filepath of the log file to be created")
  22
  23 opa.add_option("--log", action="store_true", dest="enableLog",
  24                            default=False,
  25                            help="Create the log file")
  26
  27 opa.add_option("--no-test-out", action="store_true", dest="disableTestStdOut",
  28                            default=False,
  29                            help="Don't output test results")
  30
  31 opa.add_option("-s", "--silent", action="store_true", dest="silent", default=False,
  32                            help="Disables display of all tests")
  33
  34 opa.add_option("-v", "--verbose", action="store_true", dest="verbose",
  35                            default=False,
  36                            help="Displays all tests (only if --silent is not set)")
  37
  38 opa.add_option("-x", "--max", type="int", dest="maxTestCount",
  39                            default="-1",
  40                            help="The maximum number of tests to be run")
  41
  42 opa.add_option("-t", "--test", type="string", dest="singleTest",
  43                            default=None,
  44                            help="Runs the specified test only")
  45
  46 opa.add_option("--tsw", "--test-starts-with", type="string", dest="testStartsWith",
  47                            default=None,
  48                            help="Runs the specified test(s), starting with the given string")
  49
  50 opa.add_option("--rieo", "--report-internal-errors-only", action="store_true",
  51                            dest="reportInternalErrOnly", default=False,
  52                            help="Display erroneous tests of type 'internal' only")
  53
  54 opa.add_option("--rueo", "--report-unimplemented-errors-only", action="store_true",
  55                            dest="reportUnimplErrOnly", default=False,
  56                            help="Display erroneous tests of type 'unimplemented' only")
  57
  58 opa.add_option("--rmleo", "--report-mem-leak-errors-only", action="store_true",
  59                            dest="reportMemLeakErrOnly", default=False,
  60                            help="Display erroneous tests of type 'memory leak' only")
  61
  62 opa.add_option("-c", "--combines", type="string", dest="combines",
  63                            default=None,
  64                            help="Combines to be run (all if omitted)")
  65
  66 opa.add_option("--csw", "--csw", type="string", dest="combineStartsWith",
  67                            default=None,
  68                            help="Combines to be run (all if omitted)")
  69
  70 opa.add_option("--rc", "--report-combines", action="store_true",
  71                            dest="reportCombines", default=False,
  72                            help="Display combine reports")
  73
  74 opa.add_option("--rec", "--report-err-combines", action="store_true",
  75                            dest="reportErrCombines", default=False,
  76                            help="Display erroneous combine reports only")
  77
  78 opa.add_option("--debug", action="store_true",
  79                            dest="debugEnabled", default=False,
  80                            help="Displays debug messages")
  81
  82 opa.add_option("--info", action="store_true",
  83                            dest="info", default=False,
  84                            help="Displays info on the suite only. Does not run any test.")
  85 opa.add_option("--sax", action="store_true",
  86                            dest="validationSAX", default=False,
  87                            help="Use SAX2-driven validation.")
  88 opa.add_option("--tn", action="store_true",
  89                            dest="displayTestName", default=False,
  90                            help="Display the test name in every case.")
  91
  92 (options, args) = opa.parse_args()
  93
  94 if options.combines is not None:
  95         options.combines = options.combines.split()
  96
  97 ################################################
  98 # The vars below are not intended to be changed.
  99 #
 100
 101 msgSchemaNotValidButShould =  "The schema should be valid."
 102 msgSchemaValidButShouldNot = "The schema should be invalid."
 103 msgInstanceNotValidButShould = "The instance should be valid."
 104 msgInstanceValidButShouldNot = "The instance should be invalid."
 105 vendorNIST = "NIST"
 106 vendorNIST_2 = "NIST-2"
 107 vendorSUN  = "SUN"
 108 vendorMS   = "MS"
 109
 110 ###################
 111 # Helper functions.
 112 #
 113 vendor = None
 114
 115 def handleError(test, msg):
 116         global options
 117         if not options.silent:
 118                 test.addLibLog("'%s'   LIB: %s" % (test.name, msg))
 119         if msg.find("Unimplemented") > -1:
 120                 test.failUnimplemented()
 121         elif msg.find("Internal") > -1:
 122                 test.failInternal()
 123
 124
 125 def fixFileNames(fileName):
 126         if (fileName is None) or (fileName == ""):
 127                 return ""
 128         dirs = fileName.split("/")
 129         if dirs[1] != "Tests":
 130                 fileName = os.path.join(".", "Tests")
 131                 for dir in dirs[1:]:
 132                         fileName = os.path.join(fileName, dir)
 133         return fileName
 134
 135 class XSTCTestGroup:
 136         def __init__(self, name, schemaFileName, descr):
 137                 global vendor, vendorNIST_2
 138                 self.name = name
 139                 self.descr = descr
 140                 self.mainSchema = True
 141                 self.schemaFileName = fixFileNames(schemaFileName)
 142                 self.schemaParsed = False
 143                 self.schemaTried = False
 144
 145         def setSchema(self, schemaFileName, parsed):
 146                 if not self.mainSchema:
 147                         return
 148                 self.mainSchema = False
 149                 self.schemaParsed = parsed
 150                 self.schemaTried = True
 151
 152 class XSTCTestCase:
 153
 154                    # <!-- groupName, Name, Accepted, File, Val, Descr
 155         def __init__(self, isSchema, groupName, name, accepted, file, val, descr):
 156                 global options
 157                 #
 158                 # Constructor.
 159                 #
 160                 self.testRunner = None
 161                 self.isSchema = isSchema
 162                 self.groupName = groupName
 163                 self.name = name
 164                 self.accepted = accepted
 165                 self.fileName = fixFileNames(file)
 166                 self.val = val
 167                 self.descr = descr
 168                 self.failed = False
 169                 self.combineName = None
 170
 171                 self.log = []
 172                 self.libLog = []
 173                 self.initialMemUsed = 0
 174                 self.memLeak = 0
 175                 self.excepted = False
 176                 self.bad = False
 177                 self.unimplemented = False
 178                 self.internalErr = False
 179                 self.noSchemaErr = False
 180                 self.failed = False
 181                 #
 182                 # Init the log.
 183                 #
 184                 if not options.silent:
 185                         if self.descr is not None:
 186                                 self.log.append("'%s'   descr: %s\n" % (self.name, self.descr))
 187                         self.log.append("'%s'   exp validity: %d\n" % (self.name, self.val))
 188
 189         def initTest(self, runner):
 190                 global vendorNIST, vendorSUN, vendorMS, vendorNIST_2, options, vendor
 191                 #
 192                 # Get the test-group.
 193                 #
 194                 self.runner = runner
 195                 self.group = runner.getGroup(self.groupName)
 196                 if vendor == vendorMS or vendor == vendorSUN:
 197                         #
 198                         # Use the last given directory for the combine name.
 199                         #
 200                         dirs = self.fileName.split("/")
 201                         self.combineName = dirs[len(dirs) -2]
 202                 elif vendor == vendorNIST:
 203                         #
 204                         # NIST files are named in the following form:
 205                         # "NISTSchema-short-pattern-1.xsd"
 206                         #
 207                         tokens = self.name.split("-")
 208                         self.combineName = tokens[1]
 209                 elif vendor == vendorNIST_2:
 210                         #
 211                         # Group-names have the form: "atomic-normalizedString-length-1"
 212                         #
 213                         tokens = self.groupName.split("-")
 214                         self.combineName = "%s-%s" % (tokens[0], tokens[1])
 215                 else:
 216                         self.combineName = "unkown"
 217                         raise Exception("Could not compute the combine name of a test.")
 218                 if (not options.silent) and (self.group.descr is not None):
 219                         self.log.append("'%s'   group-descr: %s\n" % (self.name, self.group.descr))
 220
 221
 222         def addLibLog(self, msg):
 223                 """This one is intended to be used by the error handler
 224                 function"""
 225                 global options
 226                 if not options.silent:
 227                         self.libLog.append(msg)
 228
 229         def fail(self, msg):
 230                 global options
 231                 self.failed = True
 232                 if not options.silent:
 233                         self.log.append("'%s' ( FAILED: %s\n" % (self.name, msg))
 234
 235         def failNoSchema(self):
 236                 global options
 237                 self.failed = True
 238                 self.noSchemaErr = True
 239                 if not options.silent:
 240                         self.log.append("'%s' X NO-SCHEMA\n" % (self.name))
 241
 242         def failInternal(self):
 243                 global options
 244                 self.failed = True
 245                 self.internalErr = True
 246                 if not options.silent:
 247                         self.log.append("'%s' * INTERNAL\n" % self.name)
 248
 249         def failUnimplemented(self):
 250                 global options
 251                 self.failed = True
 252                 self.unimplemented = True
 253                 if not options.silent:
 254                         self.log.append("'%s' ? UNIMPLEMENTED\n" % self.name)
 255
 256         def failCritical(self, msg):
 257                 global options
 258                 self.failed = True
 259                 self.bad = True
 260                 if not options.silent:
 261                         self.log.append("'%s' ! BAD: %s\n" % (self.name, msg))
 262
 263         def failExcept(self, e):
 264                 global options
 265                 self.failed = True
 266                 self.excepted = True
 267                 if not options.silent:
 268                         self.log.append("'%s' # EXCEPTION: %s\n" % (self.name, e.__str__()))
 269
 270         def setUp(self):
 271                 #
 272                 # Set up Libxml2.
 273                 #
 274                 self.initialMemUsed = libxml2.debugMemory(1)
 275                 libxml2.initParser()
 276                 libxml2.lineNumbersDefault(1)
 277                 libxml2.registerErrorHandler(handleError, self)
 278
 279         def tearDown(self):
 280                 libxml2.schemaCleanupTypes()
 281                 libxml2.cleanupParser()
 282                 self.memLeak = libxml2.debugMemory(1) - self.initialMemUsed
 283
 284         def isIOError(self, file, docType):
 285                 err = None
 286                 try:
 287                         err = libxml2.lastError()
 288                 except:
 289                         # Suppress exceptions.
 290                         pass
 291                 if (err is None):
 292                         return False
 293                 if err.domain() == libxml2.XML_FROM_IO:
 294                         self.failCritical("failed to access the %s resource '%s'\n" % (docType, file))
 295
 296         def debugMsg(self, msg):
 297                 global options
 298                 if options.debugEnabled:
 299                         sys.stdout.write("'%s'   DEBUG: %s\n" % (self.name, msg))
 300
 301         def finalize(self):
 302                 global options
 303                 """Adds additional info to the log."""
 304                 #
 305                 # Add libxml2 messages.
 306                 #
 307                 if not options.silent:
 308                         self.log.extend(self.libLog)
 309                         #
 310                         # Add memory leaks.
 311                         #
 312                         if self.memLeak != 0:
 313                                 self.log.append("%s + memory leak: %d bytes\n" % (self.name, self.memLeak))
 314
 315         def run(self):
 316                 """Runs a test."""
 317                 global options
 318
 319                 ##filePath = os.path.join(options.baseDir, self.fileName)
 320                 # filePath = "%s/%s/%s/%s" % (options.baseDir, self.test_Folder, self.schema_Folder, self.schema_File)
 321                 if options.displayTestName:
 322                         sys.stdout.write("'%s'\n" % self.name)
 323                 try:
 324                         self.validate()
 325                 except (Exception, libxml2.parserError, libxml2.treeError), e:
 326                         self.failExcept(e)
 327
 328 def parseSchema(fileName):
 329         schema = None
 330         ctxt = libxml2.schemaNewParserCtxt(fileName)
 331         try:
 332                 try:
 333                         schema = ctxt.schemaParse()
 334                 except:
 335                         pass
 336         finally:
 337                 del ctxt
 338                 return schema
 339
 340
 341 class XSTCSchemaTest(XSTCTestCase):
 342
 343         def __init__(self, groupName, name, accepted, file, val, descr):
 344                 XSTCTestCase.__init__(self, 1, groupName, name, accepted, file, val, descr)
 345
 346         def validate(self):
 347                 global msgSchemaNotValidButShould, msgSchemaValidButShouldNot
 348                 schema = None
 349                 filePath = self.fileName
 350                 # os.path.join(options.baseDir, self.fileName)
 351                 valid = 0
 352                 try:
 353                         #
 354                         # Parse the schema.
 355                         #
 356                         self.debugMsg("loading schema: %s" % filePath)
 357                         schema = parseSchema(filePath)
 358                         self.debugMsg("after loading schema")
 359                         if schema is None:
 360                                 self.debugMsg("schema is None")
 361                                 self.debugMsg("checking for IO errors...")
 362                                 if self.isIOError(file, "schema"):
 363                                         return
 364                         self.debugMsg("checking schema result")
 365                         if (schema is None and self.val) or (schema is not None and self.val == 0):
 366                                 self.debugMsg("schema result is BAD")
 367                                 if (schema == None):
 368                                         self.fail(msgSchemaNotValidButShould)
 369                                 else:
 370                                         self.fail(msgSchemaValidButShouldNot)
 371                         else:
 372                                 self.debugMsg("schema result is OK")
 373                 finally:
 374                         self.group.setSchema(self.fileName, schema is not None)
 375                         del schema
 376
 377 class XSTCInstanceTest(XSTCTestCase):
 378
 379         def __init__(self, groupName, name, accepted, file, val, descr):
 380                 XSTCTestCase.__init__(self, 0, groupName, name, accepted, file, val, descr)
 381
 382         def validate(self):
 383                 instance = None
 384                 schema = None
 385                 filePath = self.fileName
 386                 # os.path.join(options.baseDir, self.fileName)
 387
 388                 if not self.group.schemaParsed and self.group.schemaTried:
 389                         self.failNoSchema()
 390                         return
 391
 392                 self.debugMsg("loading instance: %s" % filePath)
 393                 parserCtxt = libxml2.newParserCtxt()
 394                 if (parserCtxt is None):
 395                         # TODO: Is this one necessary, or will an exception
 396                         # be already raised?
 397                         raise Exception("Could not create the instance parser context.")
 398                 if not options.validationSAX:
 399                         try:
 400                                 try:
 401                                         instance = parserCtxt.ctxtReadFile(filePath, None, libxml2.XML_PARSE_NOWARNING)
 402                                 except:
 403                                         # Suppress exceptions.
 404                                         pass
 405                         finally:
 406                                 del parserCtxt
 407                         self.debugMsg("after loading instance")
 408                         if instance is None:
 409                                 self.debugMsg("instance is None")
 410                                 self.failCritical("Failed to parse the instance for unknown reasons.")
 411                                 return
 412                 try:
 413                         #
 414                         # Validate the instance.
 415                         #
 416                         self.debugMsg("loading schema: %s" % self.group.schemaFileName)
 417                         schema = parseSchema(self.group.schemaFileName)
 418                         try:
 419                                 validationCtxt = schema.schemaNewValidCtxt()
 420                                 #validationCtxt = libxml2.schemaNewValidCtxt(None)
 421                                 if (validationCtxt is None):
 422                                         self.failCritical("Could not create the validation context.")
 423                                         return
 424                                 try:
 425                                         self.debugMsg("validating instance")
 426                                         if options.validationSAX:
 427                                                 instance_Err = validationCtxt.schemaValidateFile(filePath, 0)
 428                                         else:
 429                                                 instance_Err = validationCtxt.schemaValidateDoc(instance)
 430                                         self.debugMsg("after instance validation")
 431                                         self.debugMsg("instance-err: %d" % instance_Err)
 432                                         if (instance_Err != 0 and self.val == 1) or (instance_Err == 0 and self.val == 0):
 433                                                 self.debugMsg("instance result is BAD")
 434                                                 if (instance_Err != 0):
 435                                                         self.fail(msgInstanceNotValidButShould)
 436                                                 else:
 437                                                         self.fail(msgInstanceValidButShouldNot)
 438
 439                                         else:
 440                                                                 self.debugMsg("instance result is OK")
 441                                 finally:
 442                                         del validationCtxt
 443                         finally:
 444                                 del schema
 445                 finally:
 446                         if instance is not None:
 447                                 instance.freeDoc()
 448
 449
 450 ####################
 451 # Test runner class.
 452 #
 453
 454 class XSTCTestRunner:
 455
 456         CNT_TOTAL = 0
 457         CNT_RAN = 1
 458         CNT_SUCCEEDED = 2
 459         CNT_FAILED = 3
 460         CNT_UNIMPLEMENTED = 4
 461         CNT_INTERNAL = 5
 462         CNT_BAD = 6
 463         CNT_EXCEPTED = 7
 464         CNT_MEMLEAK = 8
 465         CNT_NOSCHEMA = 9
 466         CNT_NOTACCEPTED = 10
 467         CNT_SCHEMA_TEST = 11
 468
 469         def __init__(self):
 470                 self.logFile = None
 471                 self.counters = self.createCounters()
 472                 self.testList = []
 473                 self.combinesRan = {}
 474                 self.groups = {}
 475                 self.curGroup = None
 476
 477         def createCounters(self):
 478                 counters = {self.CNT_TOTAL:0, self.CNT_RAN:0, self.CNT_SUCCEEDED:0,
 479                 self.CNT_FAILED:0, self.CNT_UNIMPLEMENTED:0, self.CNT_INTERNAL:0, self.CNT_BAD:0,
 480                 self.CNT_EXCEPTED:0, self.CNT_MEMLEAK:0, self.CNT_NOSCHEMA:0, self.CNT_NOTACCEPTED:0,
 481                 self.CNT_SCHEMA_TEST:0}
 482
 483                 return counters
 484
 485         def addTest(self, test):
 486                 self.testList.append(test)
 487                 test.initTest(self)
 488
 489         def getGroup(self, groupName):
 490                 return self.groups[groupName]
 491
 492         def addGroup(self, group):
 493                 self.groups[group.name] = group
 494
 495         def updateCounters(self, test, counters):
 496                 if test.memLeak != 0:
 497                         counters[self.CNT_MEMLEAK] += 1
 498                 if not test.failed:
 499                         counters[self.CNT_SUCCEEDED] +=1
 500                 if test.failed:
 501                         counters[self.CNT_FAILED] += 1
 502                 if test.bad:
 503                         counters[self.CNT_BAD] += 1
 504                 if test.unimplemented:
 505                         counters[self.CNT_UNIMPLEMENTED] += 1
 506                 if test.internalErr:
 507                         counters[self.CNT_INTERNAL] += 1
 508                 if test.noSchemaErr:
 509                         counters[self.CNT_NOSCHEMA] += 1
 510                 if test.excepted:
 511                         counters[self.CNT_EXCEPTED] += 1
 512                 if not test.accepted:
 513                         counters[self.CNT_NOTACCEPTED] += 1
 514                 if test.isSchema:
 515                         counters[self.CNT_SCHEMA_TEST] += 1
 516                 return counters
 517
 518         def displayResults(self, out, all, combName, counters):
 519                 out.write("\n")
 520                 if all:
 521                         if options.combines is not None:
 522                                 out.write("combine(s): %s\n" % str(options.combines))
 523                 elif combName is not None:
 524                         out.write("combine : %s\n" % combName)
 525                 out.write("  total           : %d\n" % counters[self.CNT_TOTAL])
 526                 if all or options.combines is not None:
 527                         out.write("  ran             : %d\n" % counters[self.CNT_RAN])
 528                         out.write("    (schemata)    : %d\n" % counters[self.CNT_SCHEMA_TEST])
 529                 # out.write("    succeeded       : %d\n" % counters[self.CNT_SUCCEEDED])
 530                 out.write("  not accepted    : %d\n" % counters[self.CNT_NOTACCEPTED])
 531                 if counters[self.CNT_FAILED] > 0:
 532                         out.write("    failed                  : %d\n" % counters[self.CNT_FAILED])
 533                         out.write("     -> internal            : %d\n" % counters[self.CNT_INTERNAL])
 534                         out.write("     -> unimpl.             : %d\n" % counters[self.CNT_UNIMPLEMENTED])
 535                         out.write("     -> skip-invalid-schema : %d\n" % counters[self.CNT_NOSCHEMA])
 536                         out.write("     -> bad                 : %d\n" % counters[self.CNT_BAD])
 537                         out.write("     -> exceptions          : %d\n" % counters[self.CNT_EXCEPTED])
 538                         out.write("    memory leaks            : %d\n" % counters[self.CNT_MEMLEAK])
 539
 540         def displayShortResults(self, out, all, combName, counters):
 541                 out.write("Ran %d of %d tests (%d schemata):" % (counters[self.CNT_RAN],
 542                                   counters[self.CNT_TOTAL], counters[self.CNT_SCHEMA_TEST]))
 543                 # out.write("    succeeded       : %d\n" % counters[self.CNT_SUCCEEDED])
 544                 if counters[self.CNT_NOTACCEPTED] > 0:
 545                         out.write(" %d not accepted" % (counters[self.CNT_NOTACCEPTED]))
 546                 if counters[self.CNT_FAILED] > 0 or counters[self.CNT_MEMLEAK] > 0:
 547                         if counters[self.CNT_FAILED] > 0:
 548                                 out.write(" %d failed" % (counters[self.CNT_FAILED]))
 549                                 out.write(" (")
 550                                 if counters[self.CNT_INTERNAL] > 0:
 551                                         out.write(" %d internal" % (counters[self.CNT_INTERNAL]))
 552                                 if counters[self.CNT_UNIMPLEMENTED] > 0:
 553                                         out.write(" %d unimplemented" % (counters[self.CNT_UNIMPLEMENTED]))
 554                                 if counters[self.CNT_NOSCHEMA] > 0:
 555                                         out.write(" %d skip-invalid-schema" % (counters[self.CNT_NOSCHEMA]))
 556                                 if counters[self.CNT_BAD] > 0:
 557                                         out.write(" %d bad" % (counters[self.CNT_BAD]))
 558                                 if counters[self.CNT_EXCEPTED] > 0:
 559                                         out.write(" %d exception" % (counters[self.CNT_EXCEPTED]))
 560                                 out.write(" )")
 561                         if counters[self.CNT_MEMLEAK] > 0:
 562                                 out.write(" %d leaks" % (counters[self.CNT_MEMLEAK]))
 563                         out.write("\n")
 564                 else:
 565                         out.write(" all passed\n")
 566
 567         def reportCombine(self, combName):
 568                 global options
 569
 570                 counters = self.createCounters()
 571                 #
 572                 # Compute evaluation counters.
 573                 #
 574                 for test in self.combinesRan[combName]:
 575                         counters[self.CNT_TOTAL] += 1
 576                         counters[self.CNT_RAN] += 1
 577                         counters = self.updateCounters(test, counters)
 578                 if options.reportErrCombines and (counters[self.CNT_FAILED] == 0) and (counters[self.CNT_MEMLEAK] == 0):
 579                         pass
 580                 else:
 581                         if options.enableLog:
 582                                 self.displayResults(self.logFile, False, combName, counters)
 583                         self.displayResults(sys.stdout, False, combName, counters)
 584
 585         def displayTestLog(self, test):
 586                 sys.stdout.writelines(test.log)
 587                 sys.stdout.write("~~~~~~~~~~\n")
 588
 589         def reportTest(self, test):
 590                 global options
 591
 592                 error = test.failed or test.memLeak != 0
 593                 #
 594                 # Only erroneous tests will be written to the log,
 595                 # except @verbose is switched on.
 596                 #
 597                 if options.enableLog and (options.verbose or error):
 598                         self.logFile.writelines(test.log)
 599                         self.logFile.write("~~~~~~~~~~\n")
 600                 #
 601                 # if not @silent, only erroneous tests will be
 602                 # written to stdout, except @verbose is switched on.
 603                 #
 604                 if not options.silent:
 605                         if options.reportInternalErrOnly and test.internalErr:
 606                                 self.displayTestLog(test)
 607                         if options.reportMemLeakErrOnly and test.memLeak != 0:
 608                                 self.displayTestLog(test)
 609                         if options.reportUnimplErrOnly and test.unimplemented:
 610                                 self.displayTestLog(test)
 611                         if (options.verbose or error) and (not options.reportInternalErrOnly) and (not options.reportMemLeakErrOnly) and (not options.reportUnimplErrOnly):
 612                                 self.displayTestLog(test)
 613
 614
 615         def addToCombines(self, test):
 616                 found = False
 617                 if self.combinesRan.has_key(test.combineName):
 618                         self.combinesRan[test.combineName].append(test)
 619                 else:
 620                         self.combinesRan[test.combineName] = [test]
 621
 622         def run(self):
 623
 624                 global options
 625
 626                 if options.info:
 627                         for test in self.testList:
 628                                 self.addToCombines(test)
 629                         sys.stdout.write("Combines: %d\n" % len(self.combinesRan))
 630                         sys.stdout.write("%s\n" % self.combinesRan.keys())
 631                         return
 632
 633                 if options.enableLog:
 634                         self.logFile = open(options.logFile, "w")
 635                 try:
 636                         for test in self.testList:
 637                                 self.counters[self.CNT_TOTAL] += 1
 638                                 #
 639                                 # Filter tests.
 640                                 #
 641                                 if options.singleTest is not None and options.singleTest != "":
 642                                         if (test.name != options.singleTest):
 643                                                 continue
 644                                 elif options.combines is not None:
 645                                         if not options.combines.__contains__(test.combineName):
 646                                                 continue
 647                                 elif options.testStartsWith is not None:
 648                                         if not test.name.startswith(options.testStartsWith):
 649                                                 continue
 650                                 elif options.combineStartsWith is not None:
 651                                         if not test.combineName.startswith(options.combineStartsWith):
 652                                                 continue
 653
 654                                 if options.maxTestCount != -1 and self.counters[self.CNT_RAN] >= options.maxTestCount:
 655                                         break
 656                                 self.counters[self.CNT_RAN] += 1
 657                                 #
 658                                 # Run the thing, dammit.
 659                                 #
 660                                 try:
 661                                         test.setUp()
 662                                         try:
 663                                                 test.run()
 664                                         finally:
 665                                                 test.tearDown()
 666                                 finally:
 667                                         #
 668                                         # Evaluate.
 669                                         #
 670                                         test.finalize()
 671                                         self.reportTest(test)
 672                                         if options.reportCombines or options.reportErrCombines:
 673                                                 self.addToCombines(test)
 674                                         self.counters = self.updateCounters(test, self.counters)
 675                 finally:
 676                         if options.reportCombines or options.reportErrCombines:
 677                                 #
 678                                 # Build a report for every single combine.
 679                                 #
 680                                 # TODO: How to sort a dict?
 681                                 #
 682                                 self.combinesRan.keys().sort(None)
 683                                 for key in self.combinesRan.keys():
 684                                         self.reportCombine(key)
 685
 686                         #
 687                         # Display the final report.
 688                         #
 689                         if options.silent:
 690                                 self.displayShortResults(sys.stdout, True, None, self.counters)
 691                         else:
 692                                 sys.stdout.write("===========================\n")
 693                                 self.displayResults(sys.stdout, True, None, self.counters)