1# -*- coding: utf-8 -*- 2 3#------------------------------------------------------------------------- 4# Vulkan CTS 5# ---------- 6# 7# Copyright (c) 2016 Google Inc. 8# 9# Licensed under the Apache License, Version 2.0 (the "License"); 10# you may not use this file except in compliance with the License. 11# You may obtain a copy of the License at 12# 13# http://www.apache.org/licenses/LICENSE-2.0 14# 15# Unless required by applicable law or agreed to in writing, software 16# distributed under the License is distributed on an "AS IS" BASIS, 17# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 18# See the License for the specific language governing permissions and 19# limitations under the License. 20# 21#------------------------------------------------------------------------- 22 23import os 24import sys 25 26sys.path.append(os.path.join(os.path.dirname(__file__), "..", "log")) 27sys.path.append(os.path.join(os.path.dirname(__file__), "..", "build")) 28 29from common import readFile 30from message import * 31from log_parser import StatusCode, BatchResultParser 32 33ALLOWED_STATUS_CODES = set([ 34 StatusCode.PASS, 35 StatusCode.NOT_SUPPORTED, 36 StatusCode.QUALITY_WARNING, 37 StatusCode.COMPATIBILITY_WARNING 38 ]) 39 40def readMustpass (filename): 41 f = open(filename, 'rb') 42 cases = [] 43 for line in f: 44 s = line.strip() 45 if len(s) > 0: 46 cases.append(s) 47 return cases 48 49def readTestLog (filename): 50 parser = BatchResultParser() 51 return parser.parseFile(filename) 52 53def verifyTestLog (filename, mustpass): 54 results = readTestLog(filename) 55 messages = [] 56 resultOrderOk = True 57 58 # Mustpass case names must be unique 59 assert len(mustpass) == len(set(mustpass)) 60 61 # Verify number of results 62 if len(results) != len(mustpass): 63 messages.append(error(filename, "Wrong number of test results, expected %d, found %d" % (len(mustpass), len(results)))) 64 65 caseNameToResultNdx = {} 66 for ndx in xrange(len(results)): 67 result = results[ndx] 68 if not result in caseNameToResultNdx: 69 caseNameToResultNdx[result.name] = ndx 70 else: 71 messages.append(error(filename, "Multiple results for " + result.name)) 72 73 # Verify that all results are present and valid 74 for ndx in xrange(len(mustpass)): 75 caseName = mustpass[ndx] 76 77 if caseName in caseNameToResultNdx: 78 resultNdx = caseNameToResultNdx[caseName] 79 result = results[resultNdx] 80 81 if resultNdx != ndx: 82 resultOrderOk = False 83 84 if not result.statusCode in ALLOWED_STATUS_CODES: 85 messages.append(error(filename, result.name + ": " + result.statusCode)) 86 else: 87 messages.append(error(filename, "Missing result for " + caseName)) 88 89 if len(results) == len(mustpass) and not resultOrderOk: 90 messages.append(error(filename, "Results are not in the expected order")) 91 92 return messages 93 94def beginsWith (str, prefix): 95 return str[:len(prefix)] == prefix 96 97def verifyStatement (package): 98 messages = [] 99 100 if package.statement != None: 101 statementPath = os.path.join(package.basePath, package.statement) 102 statement = readFile(statementPath) 103 hasVersion = False 104 hasProduct = False 105 hasCpu = False 106 hasOs = False 107 108 for line in statement.splitlines(): 109 if beginsWith(line, "CONFORM_VERSION:"): 110 if hasVersion: 111 messages.append(error(statementPath, "Multiple CONFORM_VERSIONs")) 112 else: 113 assert len(line.split()) >= 2 114 package.conformVersion = line.split()[1] 115 hasVersion = True 116 elif beginsWith(line, "PRODUCT:"): 117 hasProduct = True # Multiple products allowed 118 elif beginsWith(line, "CPU:"): 119 if hasCpu: 120 messages.append(error(statementPath, "Multiple PRODUCTs")) 121 else: 122 hasCpu = True 123 elif beginsWith(line, "OS:"): 124 if hasOs: 125 messages.append(error(statementPath, "Multiple OSes")) 126 else: 127 assert len(line.split()) >= 2 128 package.conformOs = line.split()[1] 129 hasOs = True 130 131 if not hasVersion: 132 messages.append(error(statementPath, "No CONFORM_VERSION")) 133 if not hasProduct: 134 messages.append(error(statementPath, "No PRODUCT")) 135 if not hasCpu: 136 messages.append(error(statementPath, "No CPU")) 137 if not hasOs: 138 messages.append(error(statementPath, "No OS")) 139 else: 140 messages.append(error(package.basePath, "Missing conformance statement file")) 141 142 return messages 143 144def verifyGitStatus (package): 145 messages = [] 146 147 if len(package.gitStatus) > 0: 148 for s in package.gitStatus: 149 statusPath = os.path.join(package.basePath, s) 150 status = readFile(statusPath) 151 152 if status.find("nothing to commit, working directory clean") < 0 and status.find("nothing to commit, working tree clean") < 0: 153 messages.append(error(package.basePath, "Working directory is not clean")) 154 else: 155 messages.append(error(package.basePath, "Missing git status files")) 156 157 return messages 158 159def isGitLogEmpty (package, gitLog): 160 logPath = os.path.join(package.basePath, gitLog) 161 log = readFile(logPath) 162 163 return len(log.strip()) == 0 164 165def verifyGitLog (package): 166 messages = [] 167 168 if len(package.gitLog) > 0: 169 for log, path in package.gitLog: 170 if not isGitLogEmpty(package, log): 171 messages.append(warning(os.path.join(package.basePath, log), "Log is not empty")) 172 else: 173 messages.append(error(package.basePath, "Missing git log files")) 174 175 return messages 176 177def verifyPatches (package): 178 messages = [] 179 hasPatches = len(package.patches) 180 logEmpty = True 181 for log, path in package.gitLog: 182 logEmpty &= isGitLogEmpty(package, log) 183 184 if hasPatches and logEmpty: 185 messages.append(error(package.basePath, "Package includes patches but log is empty")) 186 elif not hasPatches and not logEmpty: 187 messages.append(error(package.basePath, "Test log is not empty but package doesn't contain patches")) 188 189 return messages 190