external/vulkancts/verify_submission.py

   1 # -*- coding: utf-8 -*-
   2
   3 #-------------------------------------------------------------------------
   4 # Vulkan CTS
   5 # ----------
   6 #
   7 # Copyright (c) 2016 Google Inc.
   8 #
   9 # Licensed under the Apache License, Version 2.0 (the "License");
  10 # you may not use this file except in compliance with the License.
  11 # You may obtain a copy of the License at
  12 #
  13 #      http://www.apache.org/licenses/LICENSE-2.0
  14 #
  15 # Unless required by applicable law or agreed to in writing, software
  16 # distributed under the License is distributed on an "AS IS" BASIS,
  17 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  18 # See the License for the specific language governing permissions and
  19 # limitations under the License.
  20 #
  21 #-------------------------------------------------------------------------
  22
  23 import os
  24 import re
  25 import sys
  26
  27 from fnmatch import fnmatch
  28
  29 sys.path.append(os.path.join(os.path.dirname(__file__), "..", "..", "scripts"))
  30 sys.path.append(os.path.join(os.path.dirname(__file__), "..", "..", "scripts", "log"))
  31
  32 from build.common import readFile
  33 from log_parser import StatusCode, BatchResultParser
  34
  35 ALLOWED_STATUS_CODES = set([
  36                 StatusCode.PASS,
  37                 StatusCode.NOT_SUPPORTED,
  38                 StatusCode.QUALITY_WARNING,
  39                 StatusCode.COMPATIBILITY_WARNING
  40         ])
  41
  42 STATEMENT_PATTERN       = "STATEMENT-*"
  43 TEST_LOG_PATTERN        = "*.qpa"
  44 GIT_STATUS_PATTERN      = "git-status.txt"
  45 GIT_LOG_PATTERN         = "git-log.txt"
  46 PATCH_PATTERN           = "*.patch"
  47
  48 class PackageDescription:
  49         def __init__ (self, basePath, statement, testLogs, gitStatus, gitLog, patches, otherItems):
  50                 self.basePath           = basePath
  51                 self.statement          = statement
  52                 self.testLogs           = testLogs
  53                 self.gitStatus          = gitStatus
  54                 self.gitLog                     = gitLog
  55                 self.patches            = patches
  56                 self.otherItems         = otherItems
  57
  58 class ValidationMessage:
  59         TYPE_ERROR              = 0
  60         TYPE_WARNING    = 1
  61
  62         def __init__ (self, type, filename, message):
  63                 self.type               = type
  64                 self.filename   = filename
  65                 self.message    = message
  66
  67         def __str__ (self):
  68                 prefix = {self.TYPE_ERROR: "ERROR: ", self.TYPE_WARNING: "WARNING: "}
  69                 return prefix[self.type] + os.path.basename(self.filename) + ": " + self.message
  70
  71 def error (filename, message):
  72         return ValidationMessage(ValidationMessage.TYPE_ERROR, filename, message)
  73
  74 def warning (filename, message):
  75         return ValidationMessage(ValidationMessage.TYPE_WARNING, filename, message)
  76
  77 def getPackageDescription (packagePath):
  78         allItems        = os.listdir(packagePath)
  79         statement       = None
  80         testLogs        = []
  81         gitStatus       = None
  82         gitLog          = None
  83         patches         = []
  84         otherItems      = []
  85
  86         for item in allItems:
  87                 if fnmatch(item, STATEMENT_PATTERN):
  88                         assert statement == None
  89                         statement = item
  90                 elif fnmatch(item, TEST_LOG_PATTERN):
  91                         testLogs.append(item)
  92                 elif fnmatch(item, GIT_STATUS_PATTERN):
  93                         assert gitStatus == None
  94                         gitStatus = item
  95                 elif fnmatch(item, GIT_LOG_PATTERN):
  96                         assert gitLog == None
  97                         gitLog = item
  98                 elif fnmatch(item, PATCH_PATTERN):
  99                         patches.append(item)
 100                 else:
 101                         otherItems.append(item)
 102
 103         return PackageDescription(packagePath, statement, testLogs, gitStatus, gitLog, patches, otherItems)
 104
 105 def readMustpass (filename):
 106         f = open(filename, 'rb')
 107         cases = []
 108         for line in f:
 109                 s = line.strip()
 110                 if len(s) > 0:
 111                         cases.append(s)
 112         return cases
 113
 114 def readTestLog (filename):
 115         parser = BatchResultParser()
 116         return parser.parseFile(filename)
 117
 118 def verifyTestLog (filename, mustpass):
 119         results                 = readTestLog(filename)
 120         messages                        = []
 121         resultOrderOk   = True
 122
 123         # Mustpass case names must be unique
 124         assert len(mustpass) == len(set(mustpass))
 125
 126         # Verify number of results
 127         if len(results) != len(mustpass):
 128                 messages.append(error(filename, "Wrong number of test results, expected %d, found %d" % (len(mustpass), len(results))))
 129
 130         caseNameToResultNdx = {}
 131         for ndx in xrange(len(results)):
 132                 result = results[ndx]
 133                 if not result in caseNameToResultNdx:
 134                         caseNameToResultNdx[result.name] = ndx
 135                 else:
 136                         messages.append(error(filename, "Multiple results for " + result.name))
 137
 138         # Verify that all results are present and valid
 139         for ndx in xrange(len(mustpass)):
 140                 caseName = mustpass[ndx]
 141
 142                 if caseName in caseNameToResultNdx:
 143                         resultNdx       = caseNameToResultNdx[caseName]
 144                         result          = results[resultNdx]
 145
 146                         if resultNdx != ndx:
 147                                 resultOrderOk = False
 148
 149                         if not result.statusCode in ALLOWED_STATUS_CODES:
 150                                 messages.append(error(filename, result.name + ": " + result.statusCode))
 151                 else:
 152                         messages.append(error(filename, "Missing result for " + caseName))
 153
 154         if len(results) == len(mustpass) and not resultOrderOk:
 155                 messages.append(error(filename, "Results are not in the expected order"))
 156
 157         return messages
 158
 159 def beginsWith (str, prefix):
 160         return str[:len(prefix)] == prefix
 161
 162 def verifyStatement (package):
 163         messages        = []
 164
 165         if package.statement != None:
 166                 statementPath   = os.path.join(package.basePath, package.statement)
 167                 statement               = readFile(statementPath)
 168                 hasVersion              = False
 169                 hasProduct              = False
 170                 hasCpu                  = False
 171                 hasOs                   = False
 172
 173                 for line in statement.splitlines():
 174                         if beginsWith(line, "CONFORM_VERSION:"):
 175                                 if hasVersion:
 176                                         messages.append(error(statementPath, "Multiple CONFORM_VERSIONs"))
 177                                 else:
 178                                         hasVersion = True
 179                         elif beginsWith(line, "PRODUCT:"):
 180                                 hasProduct = True # Multiple products allowed
 181                         elif beginsWith(line, "CPU:"):
 182                                 if hasCpu:
 183                                         messages.append(error(statementPath, "Multiple PRODUCTs"))
 184                                 else:
 185                                         hasCpu = True
 186                         elif beginsWith(line, "OS:"):
 187                                 if hasOs:
 188                                         messages.append(error(statementPath, "Multiple OSes"))
 189                                 else:
 190                                         hasOs = True
 191
 192                 if not hasVersion:
 193                         messages.append(error(statementPath, "No CONFORM_VERSION"))
 194                 if not hasProduct:
 195                         messages.append(error(statementPath, "No PRODUCT"))
 196                 if not hasCpu:
 197                         messages.append(error(statementPath, "No CPU"))
 198                 if not hasOs:
 199                         messages.append(error(statementPath, "No OS"))
 200         else:
 201                 messages.append(error(package.basePath, "Missing conformance statement file"))
 202
 203         return messages
 204
 205 def verifyGitStatus (package):
 206         messages = []
 207
 208         if package.gitStatus != None:
 209                 statusPath      = os.path.join(package.basePath, package.gitStatus)
 210                 status          = readFile(statusPath)
 211
 212                 if status.find("nothing to commit, working directory clean") < 0:
 213                         messages.append(error(package.basePath, "Working directory is not clean"))
 214         else:
 215                 messages.append(error(package.basePath, "Missing git-status.txt"))
 216
 217         return messages
 218
 219 def isGitLogEmpty (package):
 220         assert package.gitLog != None
 221
 222         logPath = os.path.join(package.basePath, package.gitLog)
 223         log             = readFile(logPath)
 224
 225         return len(log.strip()) == 0
 226
 227 def verifyGitLog (package):
 228         messages = []
 229
 230         if package.gitLog != None:
 231                 if not isGitLogEmpty(package):
 232                         messages.append(warning(os.path.join(package.basePath, package.gitLog), "Log is not empty"))
 233         else:
 234                 messages.append(error(package.basePath, "Missing git-log.txt"))
 235
 236         return messages
 237
 238 def verifyPatches (package):
 239         messages        = []
 240         hasPatches      = len(package.patches)
 241         logEmpty        = package.gitLog and isGitLogEmpty(package)
 242
 243         if hasPatches and logEmpty:
 244                 messages.append(error(package.basePath, "Package includes patches but log is empty"))
 245         elif not hasPatches and not logEmpty:
 246                 messages.append(error(package.basePath, "Test log is not empty but package doesn't contain patches"))
 247
 248         return messages
 249
 250 def verifyTestLogs (package, mustpass):
 251         messages        = []
 252
 253         for testLogFile in package.testLogs:
 254                 messages += verifyTestLog(os.path.join(package.basePath, testLogFile), mustpass)
 255
 256         if len(package.testLogs) == 0:
 257                 messages.append(error(package.basePath, "No test log files found"))
 258
 259         return messages
 260
 261 def verifyPackage (package, mustpass):
 262         messages = []
 263
 264         messages += verifyStatement(package)
 265         messages += verifyGitStatus(package)
 266         messages += verifyGitLog(package)
 267         messages += verifyPatches(package)
 268         messages += verifyTestLogs(package, mustpass)
 269
 270         for item in package.otherItems:
 271                 messages.append(warning(os.path.join(package.basePath, item), "Unknown file"))
 272
 273         return messages
 274
 275 if __name__ == "__main__":
 276         if len(sys.argv) != 3:
 277                 print "%s: [extracted submission package] [mustpass]" % sys.argv[0]
 278                 sys.exit(-1)
 279
 280         packagePath             = os.path.normpath(sys.argv[1])
 281         mustpassPath    = sys.argv[2]
 282         package                 = getPackageDescription(packagePath)
 283         mustpass                = readMustpass(mustpassPath)
 284         messages                = verifyPackage(package, mustpass)
 285
 286         errors                  = [m for m in messages if m.type == ValidationMessage.TYPE_ERROR]
 287         warnings                = [m for m in messages if m.type == ValidationMessage.TYPE_WARNING]
 288
 289         for message in messages:
 290                 print str(message)
 291
 292         print ""
 293
 294         if len(errors) > 0:
 295                 print "Found %d validation errors and %d warnings!" % (len(errors), len(warnings))
 296                 sys.exit(-2)
 297         elif len(warnings) > 0:
 298                 print "Found %d warnings, manual review required" % len(warnings)
 299                 sys.exit(-1)
 300         else:
 301                 print "All validation checks passed"