scripts/testset.py

   1 # -*- coding: utf-8 -*-
   2
   3 import sys
   4 import random
   5 import string
   6 import subprocess
   7 from optparse import OptionParser
   8
   9 def all (results, predicate):
  10         for result in results:
  11                 if not predicate(result):
  12                         return False
  13         return True
  14
  15 def any (results, predicate):
  16         for result in results:
  17                 if predicate(result):
  18                         return True
  19         return False
  20
  21 class FilterRule:
  22         def __init__ (self, name, description, filters):
  23                 self.name                       = name
  24                 self.description        = description
  25                 self.filters            = filters
  26
  27 class TestCaseResult:
  28         def __init__ (self, name, results):
  29                 self.name               = name
  30                 self.results    = results
  31
  32 class Group:
  33         def __init__ (self, name):
  34                 self.name               = name
  35                 self.cases              = []
  36
  37 def readCaseList (filename):
  38         f = open(filename, 'rb')
  39         cases = []
  40         for line in f:
  41                 if line[:6] == "TEST: ":
  42                         case = line[6:].strip()
  43                         if len(case) > 0:
  44                                 cases.append(case)
  45         return cases
  46
  47 def toResultList (caselist):
  48         results = []
  49         for case in caselist:
  50                 results.append(TestCaseResult(case, []))
  51         return results
  52
  53 def addResultsToCaseList (caselist, results):
  54         resultMap       = {}
  55         caseListRes     = toResultList(caselist)
  56
  57         for res in caseListRes:
  58                 resultMap[res.name] = res
  59
  60         for result in results:
  61                 if result.name in resultMap:
  62                         resultMap[result.name].results += result.results
  63
  64         return caseListRes
  65
  66 def readTestResults (filename):
  67         f                       = open(filename, 'rb')
  68         csvData         = f.read()
  69         csvLines        = csvData.splitlines()
  70         results         = []
  71
  72         f.close()
  73
  74         for line in csvLines[1:]:
  75                 args = line.split(',')
  76                 if len(args) == 1:
  77                         continue # Ignore
  78
  79                 results.append(TestCaseResult(args[0], args[1:]))
  80
  81         if len(results) == 0:
  82                 raise Exception("Empty result list")
  83
  84         # Sanity check for results
  85         numResultItems  = len(results[0].results)
  86         seenResults             = set()
  87         for result in results:
  88                 if result.name in seenResults:
  89                         raise Exception("Duplicate result row for test case '%s'" % result.name)
  90                 if len(result.results) != numResultItems:
  91                         raise Exception("Found %d results for test case '%s', expected %d" % (len(result.results), result.name, numResultItems))
  92                 seenResults.add(result.name)
  93
  94         return results
  95
  96 def readGroupList (filename):
  97         f = open(filename, 'rb')
  98         groups = []
  99         for line in f:
 100                 group = line.strip()
 101                 if group != "":
 102                         groups.append(group)
 103         return groups
 104
 105 def createGroups (results, groupNames):
 106         groups  = []
 107         matched = set()
 108
 109         for groupName in groupNames:
 110                 group = Group(groupName)
 111                 groups.append(group)
 112
 113                 prefix          = groupName + "."
 114                 prefixLen       = len(prefix)
 115                 for case in results:
 116                         if case.name[:prefixLen] == prefix:
 117                                 if case in matched:
 118                                         die("Case '%s' matched by multiple groups (when processing '%s')" % (case.name, group.name))
 119                                 group.cases.append(case)
 120                                 matched.add(case)
 121
 122         return groups
 123
 124 def createLeafGroups (results):
 125         groups = []
 126         groupMap = {}
 127
 128         for case in results:
 129                 parts           = case.name.split('.')
 130                 groupName       = string.join(parts[:-1], ".")
 131
 132                 if not groupName in groupMap:
 133                         group = Group(groupName)
 134                         groups.append(group)
 135                         groupMap[groupName] = group
 136                 else:
 137                         group = groupMap[groupName]
 138
 139                 group.cases.append(case)
 140
 141         return groups
 142
 143 def filterList (results, condition):
 144         filtered = []
 145         for case in results:
 146                 if condition(case.results):
 147                         filtered.append(case)
 148         return filtered
 149
 150 def getFilter (list, name):
 151         for filter in list:
 152                 if filter.name == name:
 153                         return filter
 154         return None
 155
 156 def getNumCasesInGroups (groups):
 157         numCases = 0
 158         for group in groups:
 159                 numCases += len(group.cases)
 160         return numCases
 161
 162 def getCasesInSet (results, caseSet):
 163         filtered = []
 164         for case in results:
 165                 if case in caseSet:
 166                         filtered.append(case)
 167         return filtered
 168
 169 def selectCasesInGroups (results, groups):
 170         casesInGroups = set()
 171         for group in groups:
 172                 for case in group.cases:
 173                         casesInGroups.add(case)
 174         return getCasesInSet(results, casesInGroups)
 175
 176 def selectRandomSubset (results, groups, limit, seed):
 177         selectedCases   = set()
 178         numSelect               = min(limit, getNumCasesInGroups(groups))
 179
 180         random.seed(seed)
 181         random.shuffle(groups)
 182
 183         groupNdx = 0
 184         while len(selectedCases) < numSelect:
 185                 group = groups[groupNdx]
 186                 if len(group.cases) == 0:
 187                         del groups[groupNdx]
 188                         if groupNdx == len(groups):
 189                                 groupNdx -= 1
 190                         continue # Try next
 191
 192                 selected = random.choice(group.cases)
 193                 selectedCases.add(selected)
 194                 group.cases.remove(selected)
 195
 196                 groupNdx = (groupNdx + 1) % len(groups)
 197
 198         return getCasesInSet(results, selectedCases)
 199
 200 def die (msg):
 201         print msg
 202         sys.exit(-1)
 203
 204 # Named filter lists
 205 FILTER_RULES = [
 206         FilterRule("all",                       "No filtering",                                                                                 []),
 207         FilterRule("all-pass",          "All results must be 'Pass'",                                                   [lambda l: all(l, lambda r: r == 'Pass')]),
 208         FilterRule("any-pass",          "Any of results is 'Pass'",                                                             [lambda l: any(l, lambda r: r == 'Pass')]),
 209         FilterRule("any-fail",          "Any of results is not 'Pass' or 'NotSupported'",               [lambda l: not all(l, lambda r: r == 'Pass' or r == 'NotSupported')]),
 210         FilterRule("prev-failing",      "Any except last result is failure",                                    [lambda l: l[-1] == 'Pass' and not all(l[:-1], lambda r: r == 'Pass')]),
 211         FilterRule("prev-passing",      "Any except last result is 'Pass'",                                             [lambda l: l[-1] != 'Pass' and any(l[:-1], lambda r: r == 'Pass')])
 212 ]
 213
 214 if __name__ == "__main__":
 215         parser = OptionParser(usage = "usage: %prog [options] [caselist] [result csv file]")
 216         parser.add_option("-f", "--filter", dest="filter", default="all", help="filter rule name")
 217         parser.add_option("-l", "--list", action="store_true", dest="list", default=False, help="list available rules")
 218         parser.add_option("-n", "--num", dest="limit", default=0, help="limit number of cases")
 219         parser.add_option("-s", "--seed", dest="seed", default=0, help="use selected seed for random selection")
 220         parser.add_option("-g", "--groups", dest="groups_file", default=None, help="select cases based on group list file")
 221
 222         (options, args) = parser.parse_args()
 223
 224         if options.list:
 225                 print "Available filter rules:"
 226                 for filter in FILTER_RULES:
 227                         print "  %s: %s" % (filter.name, filter.description)
 228                 sys.exit(0)
 229
 230         if len(args) == 0:
 231                 die("No input files specified")
 232         elif len(args) > 2:
 233                 die("Too many arguments")
 234
 235         # Fetch filter
 236         filter = getFilter(FILTER_RULES, options.filter)
 237         if filter == None:
 238                 die("Unknown filter '%s'" % options.filter)
 239
 240         # Read case list
 241         caselist = readCaseList(args[0])
 242         if len(args) > 1:
 243                 results = readTestResults(args[1])
 244                 results = addResultsToCaseList(caselist, results)
 245         else:
 246                 results = toResultList(caselist)
 247
 248         # Execute filters for results
 249         for rule in filter.filters:
 250                 results = filterList(results, rule)
 251
 252         if options.limit != 0:
 253                 if options.groups_file != None:
 254                         groups = createGroups(results, readGroupList(options.groups_file))
 255                 else:
 256                         groups = createLeafGroups(results)
 257                 results = selectRandomSubset(results, groups, int(options.limit), int(options.seed))
 258         elif options.groups_file != None:
 259                 groups = createGroups(results, readGroupList(options.groups_file))
 260                 results = selectCasesInGroups(results, groups)
 261
 262         # Print test set
 263         for result in results:
 264                 print result.name