scripts/verify/verify.py

   1 # -*- coding: utf-8 -*-
   2
   3 #-------------------------------------------------------------------------
   4 # Vulkan CTS
   5 # ----------
   6 #
   7 # Copyright (c) 2016 Google Inc.
   8 #
   9 # Licensed under the Apache License, Version 2.0 (the "License");
  10 # you may not use this file except in compliance with the License.
  11 # You may obtain a copy of the License at
  12 #
  13 #      http://www.apache.org/licenses/LICENSE-2.0
  14 #
  15 # Unless required by applicable law or agreed to in writing, software
  16 # distributed under the License is distributed on an "AS IS" BASIS,
  17 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  18 # See the License for the specific language governing permissions and
  19 # limitations under the License.
  20 #
  21 #-------------------------------------------------------------------------
  22
  23 import os
  24 import sys
  25
  26 sys.path.append(os.path.join(os.path.dirname(__file__), "..", "log"))
  27 sys.path.append(os.path.join(os.path.dirname(__file__), "..", "build"))
  28
  29 from common import readFile
  30 from message import *
  31 from log_parser import StatusCode, BatchResultParser
  32
  33 ALLOWED_STATUS_CODES = set([
  34                 StatusCode.PASS,
  35                 StatusCode.NOT_SUPPORTED,
  36                 StatusCode.QUALITY_WARNING,
  37                 StatusCode.COMPATIBILITY_WARNING
  38         ])
  39
  40 def readMustpass (filename):
  41         f = open(filename, 'rb')
  42         cases = []
  43         for line in f:
  44                 s = line.strip()
  45                 if len(s) > 0:
  46                         cases.append(s)
  47         return cases
  48
  49 def readTestLog (filename):
  50         parser = BatchResultParser()
  51         return parser.parseFile(filename)
  52
  53 def verifyTestLog (filename, mustpass):
  54         results                 = readTestLog(filename)
  55         messages                        = []
  56         resultOrderOk   = True
  57
  58         # Mustpass case names must be unique
  59         assert len(mustpass) == len(set(mustpass))
  60
  61         # Verify number of results
  62         if len(results) != len(mustpass):
  63                 messages.append(error(filename, "Wrong number of test results, expected %d, found %d" % (len(mustpass), len(results))))
  64
  65         caseNameToResultNdx = {}
  66         for ndx in xrange(len(results)):
  67                 result = results[ndx]
  68                 if not result in caseNameToResultNdx:
  69                         caseNameToResultNdx[result.name] = ndx
  70                 else:
  71                         messages.append(error(filename, "Multiple results for " + result.name))
  72
  73         # Verify that all results are present and valid
  74         for ndx in xrange(len(mustpass)):
  75                 caseName = mustpass[ndx]
  76
  77                 if caseName in caseNameToResultNdx:
  78                         resultNdx       = caseNameToResultNdx[caseName]
  79                         result          = results[resultNdx]
  80
  81                         if resultNdx != ndx:
  82                                 resultOrderOk = False
  83
  84                         if not result.statusCode in ALLOWED_STATUS_CODES:
  85                                 messages.append(error(filename, result.name + ": " + result.statusCode))
  86                 else:
  87                         messages.append(error(filename, "Missing result for " + caseName))
  88
  89         if len(results) == len(mustpass) and not resultOrderOk:
  90                 messages.append(error(filename, "Results are not in the expected order"))
  91
  92         return messages
  93
  94 def beginsWith (str, prefix):
  95         return str[:len(prefix)] == prefix
  96
  97 def verifyStatement (package):
  98         messages        = []
  99
 100         if package.statement != None:
 101                 statementPath   = os.path.join(package.basePath, package.statement)
 102                 statement               = readFile(statementPath)
 103                 hasVersion              = False
 104                 hasProduct              = False
 105                 hasCpu                  = False
 106                 hasOs                   = False
 107
 108                 for line in statement.splitlines():
 109                         if beginsWith(line, "CONFORM_VERSION:"):
 110                                 if hasVersion:
 111                                         messages.append(error(statementPath, "Multiple CONFORM_VERSIONs"))
 112                                 else:
 113                                         assert len(line.split()) >= 2
 114                                         package.conformVersion = line.split()[1]
 115                                         hasVersion = True
 116                         elif beginsWith(line, "PRODUCT:"):
 117                                 hasProduct = True # Multiple products allowed
 118                         elif beginsWith(line, "CPU:"):
 119                                 if hasCpu:
 120                                         messages.append(error(statementPath, "Multiple PRODUCTs"))
 121                                 else:
 122                                         hasCpu = True
 123                         elif beginsWith(line, "OS:"):
 124                                 if hasOs:
 125                                         messages.append(error(statementPath, "Multiple OSes"))
 126                                 else:
 127                                         assert len(line.split()) >= 2
 128                                         package.conformOs = line.split()[1]
 129                                         hasOs = True
 130
 131                 if not hasVersion:
 132                         messages.append(error(statementPath, "No CONFORM_VERSION"))
 133                 if not hasProduct:
 134                         messages.append(error(statementPath, "No PRODUCT"))
 135                 if not hasCpu:
 136                         messages.append(error(statementPath, "No CPU"))
 137                 if not hasOs:
 138                         messages.append(error(statementPath, "No OS"))
 139         else:
 140                 messages.append(error(package.basePath, "Missing conformance statement file"))
 141
 142         return messages
 143
 144 def verifyGitStatus (package):
 145         messages = []
 146
 147         if len(package.gitStatus) > 0:
 148                 for s in package.gitStatus:
 149                         statusPath      = os.path.join(package.basePath, s)
 150                         status          = readFile(statusPath)
 151
 152                         if status.find("nothing to commit, working directory clean") < 0 and status.find("nothing to commit, working tree clean") < 0:
 153                                 messages.append(error(package.basePath, "Working directory is not clean"))
 154         else:
 155                 messages.append(error(package.basePath, "Missing git status files"))
 156
 157         return messages
 158
 159 def isGitLogEmpty (package, gitLog):
 160         logPath = os.path.join(package.basePath, gitLog)
 161         log             = readFile(logPath)
 162
 163         return len(log.strip()) == 0
 164
 165 def verifyGitLog (package):
 166         messages = []
 167
 168         if len(package.gitLog) > 0:
 169                 for log, path in package.gitLog:
 170                         if not isGitLogEmpty(package, log):
 171                                 messages.append(warning(os.path.join(package.basePath, log), "Log is not empty"))
 172         else:
 173                 messages.append(error(package.basePath, "Missing git log files"))
 174
 175         return messages
 176
 177 def verifyPatches (package):
 178         messages        = []
 179         hasPatches      = len(package.patches)
 180         logEmpty        = True
 181         for log, path in package.gitLog:
 182                 logEmpty &= isGitLogEmpty(package, log)
 183
 184         if hasPatches and logEmpty:
 185                 messages.append(error(package.basePath, "Package includes patches but log is empty"))
 186         elif not hasPatches and not logEmpty:
 187                 messages.append(error(package.basePath, "Test log is not empty but package doesn't contain patches"))
 188
 189         return messages