1# -*- coding: utf-8 -*- 2 3#------------------------------------------------------------------------- 4# drawElements Quality Program utilities 5# -------------------------------------- 6# 7# Copyright 2016 The Android Open Source Project 8# 9# Licensed under the Apache License, Version 2.0 (the "License"); 10# you may not use this file except in compliance with the License. 11# You may obtain a copy of the License at 12# 13# http://www.apache.org/licenses/LICENSE-2.0 14# 15# Unless required by applicable law or agreed to in writing, software 16# distributed under the License is distributed on an "AS IS" BASIS, 17# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 18# See the License for the specific language governing permissions and 19# limitations under the License. 20# 21#------------------------------------------------------------------------- 22 23from build.common import * 24from build.config import ANY_GENERATOR 25from build.build import build 26from build_caselists import Module, getModuleByName, getBuildConfig, genCaseList, getCaseListPath, DEFAULT_BUILD_DIR, DEFAULT_TARGET 27from fnmatch import fnmatch 28from copy import copy 29 30import argparse 31import xml.etree.cElementTree as ElementTree 32import xml.dom.minidom as minidom 33 34APK_NAME = "com.drawelements.deqp.apk" 35 36GENERATED_FILE_WARNING = """ 37 This file has been automatically generated. Edit with caution. 38 """ 39 40class Project: 41 def __init__ (self, path, copyright = None): 42 self.path = path 43 self.copyright = copyright 44 45class Configuration: 46 def __init__ (self, name, filters, glconfig = None, rotation = None, surfacetype = None, required = False, runtime = None): 47 self.name = name 48 self.glconfig = glconfig 49 self.rotation = rotation 50 self.surfacetype = surfacetype 51 self.required = required 52 self.filters = filters 53 self.expectedRuntime = runtime 54 55class Package: 56 def __init__ (self, module, configurations): 57 self.module = module 58 self.configurations = configurations 59 60class Mustpass: 61 def __init__ (self, project, version, packages): 62 self.project = project 63 self.version = version 64 self.packages = packages 65 66class Filter: 67 TYPE_INCLUDE = 0 68 TYPE_EXCLUDE = 1 69 70 def __init__ (self, type, filename): 71 self.type = type 72 self.filename = filename 73 74class TestRoot: 75 def __init__ (self): 76 self.children = [] 77 78class TestGroup: 79 def __init__ (self, name): 80 self.name = name 81 self.children = [] 82 83class TestCase: 84 def __init__ (self, name): 85 self.name = name 86 self.configurations = [] 87 88class GLESVersion: 89 def __init__(self, major, minor): 90 self.major = major 91 self.minor = minor 92 93 def encode (self): 94 return (self.major << 16) | (self.minor) 95 96def getModuleGLESVersion (module): 97 versions = { 98 'dEQP-EGL': GLESVersion(2,0), 99 'dEQP-GLES2': GLESVersion(2,0), 100 'dEQP-GLES3': GLESVersion(3,0), 101 'dEQP-GLES31': GLESVersion(3,1) 102 } 103 return versions[module.name] if module.name in versions else None 104 105def getSrcDir (mustpass): 106 return os.path.join(mustpass.project.path, mustpass.version, "src") 107 108def getTmpDir (mustpass): 109 return os.path.join(mustpass.project.path, mustpass.version, "tmp") 110 111def getModuleShorthand (module): 112 assert module.name[:5] == "dEQP-" 113 return module.name[5:].lower() 114 115def getCaseListFileName (package, configuration): 116 return "%s-%s.txt" % (getModuleShorthand(package.module), configuration.name) 117 118def getDstCaseListPath (mustpass, package, configuration): 119 return os.path.join(mustpass.project.path, mustpass.version, getCaseListFileName(package, configuration)) 120 121def getCTSPackageName (package): 122 return "com.drawelements.deqp." + getModuleShorthand(package.module) 123 124def getCommandLine (config): 125 cmdLine = "" 126 127 if config.glconfig != None: 128 cmdLine += "--deqp-gl-config-name=%s " % config.glconfig 129 130 if config.rotation != None: 131 cmdLine += "--deqp-screen-rotation=%s " % config.rotation 132 133 if config.surfacetype != None: 134 cmdLine += "--deqp-surface-type=%s " % config.surfacetype 135 136 cmdLine += "--deqp-watchdog=enable" 137 138 return cmdLine 139 140def readCaseList (filename): 141 cases = [] 142 with open(filename, 'rb') as f: 143 for line in f: 144 if line[:6] == "TEST: ": 145 cases.append(line[6:].strip()) 146 return cases 147 148def getCaseList (buildCfg, generator, module): 149 build(buildCfg, generator, [module.binName]) 150 genCaseList(buildCfg, generator, module, "txt") 151 return readCaseList(getCaseListPath(buildCfg, module, "txt")) 152 153def readPatternList (filename): 154 ptrns = [] 155 with open(filename, 'rb') as f: 156 for line in f: 157 line = line.strip() 158 if len(line) > 0 and line[0] != '#': 159 ptrns.append(line) 160 return ptrns 161 162def applyPatterns (caseList, patterns, filename, op): 163 matched = set() 164 errors = [] 165 curList = copy(caseList) 166 trivialPtrns = [p for p in patterns if p.find('*') < 0] 167 regularPtrns = [p for p in patterns if p.find('*') >= 0] 168 169 # Apply trivial (just case paths) 170 allCasesSet = set(caseList) 171 for path in trivialPtrns: 172 if path in allCasesSet: 173 if path in matched: 174 errors.append((path, "Same case specified more than once")) 175 matched.add(path) 176 else: 177 errors.append((path, "Test case not found")) 178 179 curList = [c for c in curList if c not in matched] 180 181 for pattern in regularPtrns: 182 matchedThisPtrn = set() 183 184 for case in curList: 185 if fnmatch(case, pattern): 186 matchedThisPtrn.add(case) 187 188 if len(matchedThisPtrn) == 0: 189 errors.append((pattern, "Pattern didn't match any cases")) 190 191 matched = matched | matchedThisPtrn 192 curList = [c for c in curList if c not in matched] 193 194 for pattern, reason in errors: 195 print "ERROR: %s: %s" % (reason, pattern) 196 197 if len(errors) > 0: 198 die("Found %s invalid patterns while processing file %s" % (len(errors), filename)) 199 200 return [c for c in caseList if op(c in matched)] 201 202def applyInclude (caseList, patterns, filename): 203 return applyPatterns(caseList, patterns, filename, lambda b: b) 204 205def applyExclude (caseList, patterns, filename): 206 return applyPatterns(caseList, patterns, filename, lambda b: not b) 207 208def readPatternLists (mustpass): 209 lists = {} 210 for package in mustpass.packages: 211 for cfg in package.configurations: 212 for filter in cfg.filters: 213 if not filter.filename in lists: 214 lists[filter.filename] = readPatternList(os.path.join(getSrcDir(mustpass), filter.filename)) 215 return lists 216 217def applyFilters (caseList, patternLists, filters): 218 res = copy(caseList) 219 for filter in filters: 220 ptrnList = patternLists[filter.filename] 221 if filter.type == Filter.TYPE_INCLUDE: 222 res = applyInclude(res, ptrnList, filter.filename) 223 else: 224 assert filter.type == Filter.TYPE_EXCLUDE 225 res = applyExclude(res, ptrnList, filter.filename) 226 return res 227 228def appendToHierarchy (root, casePath): 229 def findChild (node, name): 230 for child in node.children: 231 if child.name == name: 232 return child 233 return None 234 235 curNode = root 236 components = casePath.split('.') 237 238 for component in components[:-1]: 239 nextNode = findChild(curNode, component) 240 if not nextNode: 241 nextNode = TestGroup(component) 242 curNode.children.append(nextNode) 243 curNode = nextNode 244 245 if not findChild(curNode, components[-1]): 246 curNode.children.append(TestCase(components[-1])) 247 248def buildTestHierachy (caseList): 249 root = TestRoot() 250 for case in caseList: 251 appendToHierarchy(root, case) 252 return root 253 254def buildTestCaseMap (root): 255 caseMap = {} 256 257 def recursiveBuild (curNode, prefix): 258 curPath = prefix + curNode.name 259 if isinstance(curNode, TestCase): 260 caseMap[curPath] = curNode 261 else: 262 for child in curNode.children: 263 recursiveBuild(child, curPath + '.') 264 265 for child in root.children: 266 recursiveBuild(child, '') 267 268 return caseMap 269 270def include (filename): 271 return Filter(Filter.TYPE_INCLUDE, filename) 272 273def exclude (filename): 274 return Filter(Filter.TYPE_EXCLUDE, filename) 275 276def insertXMLHeaders (mustpass, doc): 277 if mustpass.project.copyright != None: 278 doc.insert(0, ElementTree.Comment(mustpass.project.copyright)) 279 doc.insert(1, ElementTree.Comment(GENERATED_FILE_WARNING)) 280 281def prettifyXML (doc): 282 uglyString = ElementTree.tostring(doc, 'utf-8') 283 reparsed = minidom.parseString(uglyString) 284 return reparsed.toprettyxml(indent='\t', encoding='utf-8') 285 286def genSpecXML (mustpass): 287 mustpassElem = ElementTree.Element("Mustpass", version = mustpass.version) 288 insertXMLHeaders(mustpass, mustpassElem) 289 290 for package in mustpass.packages: 291 packageElem = ElementTree.SubElement(mustpassElem, "TestPackage", name = package.module.name) 292 293 for config in package.configurations: 294 configElem = ElementTree.SubElement(packageElem, "Configuration", 295 name = config.name, 296 caseListFile = getCaseListFileName(package, config), 297 commandLine = getCommandLine(config)) 298 299 return mustpassElem 300 301def addOptionElement (parent, optionName, optionValue): 302 ElementTree.SubElement(parent, "option", name=optionName, value=optionValue) 303 304def genAndroidTestXml (mustpass): 305 RUNNER_CLASS = "com.drawelements.deqp.runner.DeqpTestRunner" 306 configElement = ElementTree.Element("configuration") 307 308 # have the deqp package installed on the device for us 309 preparerElement = ElementTree.SubElement(configElement, "target_preparer") 310 preparerElement.set("class", "com.android.tradefed.targetprep.suite.SuiteApkInstaller") 311 addOptionElement(preparerElement, "cleanup-apks", "true") 312 addOptionElement(preparerElement, "test-file-name", "com.drawelements.deqp.apk") 313 314 # add in metadata option for component name 315 ElementTree.SubElement(configElement, "option", name="test-suite-tag", value="cts") 316 ElementTree.SubElement(configElement, "option", name="config-descriptor:metadata", key="component", value="deqp") 317 ElementTree.SubElement(configElement, "option", name="config-descriptor:metadata", key="parameter", value="not_instant_app") 318 ElementTree.SubElement(configElement, "option", name="config-descriptor:metadata", key="parameter", value="multi_abi") 319 controllerElement = ElementTree.SubElement(configElement, "object") 320 controllerElement.set("type", "module_controller") 321 controllerElement.set("class", "com.android.tradefed.testtype.suite.module.TestFailureModuleController") 322 addOptionElement(controllerElement, "screenshot-on-failure", "false") 323 324 for package in mustpass.packages: 325 for config in package.configurations: 326 testElement = ElementTree.SubElement(configElement, "test") 327 testElement.set("class", RUNNER_CLASS) 328 addOptionElement(testElement, "deqp-package", package.module.name) 329 addOptionElement(testElement, "deqp-caselist-file", getCaseListFileName(package,config)) 330 # \todo [2015-10-16 kalle]: Replace with just command line? - requires simplifications in the runner/tests as well. 331 if config.glconfig != None: 332 addOptionElement(testElement, "deqp-gl-config-name", config.glconfig) 333 334 if config.surfacetype != None: 335 addOptionElement(testElement, "deqp-surface-type", config.surfacetype) 336 337 if config.rotation != None: 338 addOptionElement(testElement, "deqp-screen-rotation", config.rotation) 339 340 if config.expectedRuntime != None: 341 addOptionElement(testElement, "runtime-hint", config.expectedRuntime) 342 343 if config.required: 344 addOptionElement(testElement, "deqp-config-required", "true") 345 346 insertXMLHeaders(mustpass, configElement) 347 348 return configElement 349 350def genMustpass (mustpass, moduleCaseLists): 351 print "Generating mustpass '%s'" % mustpass.version 352 353 patternLists = readPatternLists(mustpass) 354 355 for package in mustpass.packages: 356 allCasesInPkg = moduleCaseLists[package.module] 357 358 for config in package.configurations: 359 filtered = applyFilters(allCasesInPkg, patternLists, config.filters) 360 dstFile = getDstCaseListPath(mustpass, package, config) 361 362 print " Writing deqp caselist: " + dstFile 363 writeFile(dstFile, "\n".join(filtered) + "\n") 364 365 specXML = genSpecXML(mustpass) 366 specFilename = os.path.join(mustpass.project.path, mustpass.version, "mustpass.xml") 367 368 print " Writing spec: " + specFilename 369 writeFile(specFilename, prettifyXML(specXML)) 370 371 # TODO: Which is the best selector mechanism? 372 if (mustpass.version == "master"): 373 androidTestXML = genAndroidTestXml(mustpass) 374 androidTestFilename = os.path.join(mustpass.project.path, "AndroidTest.xml") 375 376 print " Writing AndroidTest.xml: " + androidTestFilename 377 writeFile(androidTestFilename, prettifyXML(androidTestXML)) 378 379 print "Done!" 380 381def genMustpassLists (mustpassLists, generator, buildCfg): 382 moduleCaseLists = {} 383 384 # Getting case lists involves invoking build, so we want to cache the results 385 for mustpass in mustpassLists: 386 for package in mustpass.packages: 387 if not package.module in moduleCaseLists: 388 moduleCaseLists[package.module] = getCaseList(buildCfg, generator, package.module) 389 390 for mustpass in mustpassLists: 391 genMustpass(mustpass, moduleCaseLists) 392 393def parseCmdLineArgs (): 394 parser = argparse.ArgumentParser(description = "Build Android CTS mustpass", 395 formatter_class=argparse.ArgumentDefaultsHelpFormatter) 396 parser.add_argument("-b", 397 "--build-dir", 398 dest="buildDir", 399 default=DEFAULT_BUILD_DIR, 400 help="Temporary build directory") 401 parser.add_argument("-t", 402 "--build-type", 403 dest="buildType", 404 default="Debug", 405 help="Build type") 406 parser.add_argument("-c", 407 "--deqp-target", 408 dest="targetName", 409 default=DEFAULT_TARGET, 410 help="dEQP build target") 411 return parser.parse_args() 412 413def parseBuildConfigFromCmdLineArgs (): 414 args = parseCmdLineArgs() 415 return getBuildConfig(args.buildDir, args.targetName, args.buildType) 416