1# -*- coding: utf-8 -*- 2 3#------------------------------------------------------------------------- 4# drawElements Quality Program utilities 5# -------------------------------------- 6# 7# Copyright 2016 The Android Open Source Project 8# 9# Licensed under the Apache License, Version 2.0 (the "License"); 10# you may not use this file except in compliance with the License. 11# You may obtain a copy of the License at 12# 13# http://www.apache.org/licenses/LICENSE-2.0 14# 15# Unless required by applicable law or agreed to in writing, software 16# distributed under the License is distributed on an "AS IS" BASIS, 17# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 18# See the License for the specific language governing permissions and 19# limitations under the License. 20# 21#------------------------------------------------------------------------- 22 23from build.common import * 24from build.config import ANY_GENERATOR 25from build.build import build 26from build_caselists import Module, getModuleByName, getBuildConfig, genCaseList, getCaseListPath, DEFAULT_BUILD_DIR, DEFAULT_TARGET 27from fnmatch import fnmatch 28from copy import copy 29 30import argparse 31import xml.etree.cElementTree as ElementTree 32import xml.dom.minidom as minidom 33 34APK_NAME = "com.drawelements.deqp.apk" 35 36GENERATED_FILE_WARNING = """ 37 This file has been automatically generated. Edit with caution. 38 """ 39 40class Project: 41 def __init__ (self, path, copyright = None): 42 self.path = path 43 self.copyright = copyright 44 45class Configuration: 46 def __init__ (self, name, filters, glconfig = None, rotation = None, surfacetype = None, required = False, runtime = None, runByDefault = True): 47 self.name = name 48 self.glconfig = glconfig 49 self.rotation = rotation 50 self.surfacetype = surfacetype 51 self.required = required 52 self.filters = filters 53 self.expectedRuntime = runtime 54 self.runByDefault = runByDefault 55 56class Package: 57 def __init__ (self, module, configurations): 58 self.module = module 59 self.configurations = configurations 60 61class Mustpass: 62 def __init__ (self, project, version, packages): 63 self.project = project 64 self.version = version 65 self.packages = packages 66 67class Filter: 68 TYPE_INCLUDE = 0 69 TYPE_EXCLUDE = 1 70 71 def __init__ (self, type, filename): 72 self.type = type 73 self.filename = filename 74 75class TestRoot: 76 def __init__ (self): 77 self.children = [] 78 79class TestGroup: 80 def __init__ (self, name): 81 self.name = name 82 self.children = [] 83 84class TestCase: 85 def __init__ (self, name): 86 self.name = name 87 self.configurations = [] 88 89class GLESVersion: 90 def __init__(self, major, minor): 91 self.major = major 92 self.minor = minor 93 94 def encode (self): 95 return (self.major << 16) | (self.minor) 96 97def getModuleGLESVersion (module): 98 versions = { 99 'dEQP-EGL': GLESVersion(2,0), 100 'dEQP-GLES2': GLESVersion(2,0), 101 'dEQP-GLES3': GLESVersion(3,0), 102 'dEQP-GLES31': GLESVersion(3,1) 103 } 104 return versions[module.name] if module.name in versions else None 105 106def getSrcDir (mustpass): 107 return os.path.join(mustpass.project.path, mustpass.version, "src") 108 109def getTmpDir (mustpass): 110 return os.path.join(mustpass.project.path, mustpass.version, "tmp") 111 112def getModuleShorthand (module): 113 assert module.name[:5] == "dEQP-" 114 return module.name[5:].lower() 115 116def getCaseListFileName (package, configuration): 117 return "%s-%s.txt" % (getModuleShorthand(package.module), configuration.name) 118 119def getDstCaseListPath (mustpass, package, configuration): 120 return os.path.join(mustpass.project.path, mustpass.version, getCaseListFileName(package, configuration)) 121 122def getCTSPackageName (package): 123 return "com.drawelements.deqp." + getModuleShorthand(package.module) 124 125def getCommandLine (config): 126 cmdLine = "" 127 128 if config.glconfig != None: 129 cmdLine += "--deqp-gl-config-name=%s " % config.glconfig 130 131 if config.rotation != None: 132 cmdLine += "--deqp-screen-rotation=%s " % config.rotation 133 134 if config.surfacetype != None: 135 cmdLine += "--deqp-surface-type=%s " % config.surfacetype 136 137 cmdLine += "--deqp-watchdog=enable" 138 139 return cmdLine 140 141def readCaseList (filename): 142 cases = [] 143 with open(filename, 'rt') as f: 144 for line in f: 145 if line[:6] == "TEST: ": 146 cases.append(line[6:].strip()) 147 return cases 148 149def getCaseList (buildCfg, generator, module): 150 build(buildCfg, generator, [module.binName]) 151 genCaseList(buildCfg, generator, module, "txt") 152 return readCaseList(getCaseListPath(buildCfg, module, "txt")) 153 154def readPatternList (filename): 155 ptrns = [] 156 with open(filename, 'rt') as f: 157 for line in f: 158 line = line.strip() 159 if len(line) > 0 and line[0] != '#': 160 ptrns.append(line) 161 return ptrns 162 163def applyPatterns (caseList, patterns, filename, op): 164 matched = set() 165 errors = [] 166 curList = copy(caseList) 167 trivialPtrns = [p for p in patterns if p.find('*') < 0] 168 regularPtrns = [p for p in patterns if p.find('*') >= 0] 169 170 # Apply trivial (just case paths) 171 allCasesSet = set(caseList) 172 for path in trivialPtrns: 173 if path in allCasesSet: 174 if path in matched: 175 errors.append((path, "Same case specified more than once")) 176 matched.add(path) 177 else: 178 errors.append((path, "Test case not found")) 179 180 curList = [c for c in curList if c not in matched] 181 182 for pattern in regularPtrns: 183 matchedThisPtrn = set() 184 185 for case in curList: 186 if fnmatch(case, pattern): 187 matchedThisPtrn.add(case) 188 189 if len(matchedThisPtrn) == 0: 190 errors.append((pattern, "Pattern didn't match any cases")) 191 192 matched = matched | matchedThisPtrn 193 curList = [c for c in curList if c not in matched] 194 195 for pattern, reason in errors: 196 print("ERROR: %s: %s" % (reason, pattern)) 197 198 if len(errors) > 0: 199 die("Found %s invalid patterns while processing file %s" % (len(errors), filename)) 200 201 return [c for c in caseList if op(c in matched)] 202 203def applyInclude (caseList, patterns, filename): 204 return applyPatterns(caseList, patterns, filename, lambda b: b) 205 206def applyExclude (caseList, patterns, filename): 207 return applyPatterns(caseList, patterns, filename, lambda b: not b) 208 209def readPatternLists (mustpass): 210 lists = {} 211 for package in mustpass.packages: 212 for cfg in package.configurations: 213 for filter in cfg.filters: 214 if not filter.filename in lists: 215 lists[filter.filename] = readPatternList(os.path.join(getSrcDir(mustpass), filter.filename)) 216 return lists 217 218def applyFilters (caseList, patternLists, filters): 219 res = copy(caseList) 220 for filter in filters: 221 ptrnList = patternLists[filter.filename] 222 if filter.type == Filter.TYPE_INCLUDE: 223 res = applyInclude(res, ptrnList, filter.filename) 224 else: 225 assert filter.type == Filter.TYPE_EXCLUDE 226 res = applyExclude(res, ptrnList, filter.filename) 227 return res 228 229def appendToHierarchy (root, casePath): 230 def findChild (node, name): 231 for child in node.children: 232 if child.name == name: 233 return child 234 return None 235 236 curNode = root 237 components = casePath.split('.') 238 239 for component in components[:-1]: 240 nextNode = findChild(curNode, component) 241 if not nextNode: 242 nextNode = TestGroup(component) 243 curNode.children.append(nextNode) 244 curNode = nextNode 245 246 if not findChild(curNode, components[-1]): 247 curNode.children.append(TestCase(components[-1])) 248 249def buildTestHierachy (caseList): 250 root = TestRoot() 251 for case in caseList: 252 appendToHierarchy(root, case) 253 return root 254 255def buildTestCaseMap (root): 256 caseMap = {} 257 258 def recursiveBuild (curNode, prefix): 259 curPath = prefix + curNode.name 260 if isinstance(curNode, TestCase): 261 caseMap[curPath] = curNode 262 else: 263 for child in curNode.children: 264 recursiveBuild(child, curPath + '.') 265 266 for child in root.children: 267 recursiveBuild(child, '') 268 269 return caseMap 270 271def include (filename): 272 return Filter(Filter.TYPE_INCLUDE, filename) 273 274def exclude (filename): 275 return Filter(Filter.TYPE_EXCLUDE, filename) 276 277def insertXMLHeaders (mustpass, doc): 278 if mustpass.project.copyright != None: 279 doc.insert(0, ElementTree.Comment(mustpass.project.copyright)) 280 doc.insert(1, ElementTree.Comment(GENERATED_FILE_WARNING)) 281 282def prettifyXML (doc): 283 uglyString = ElementTree.tostring(doc, 'utf-8') 284 reparsed = minidom.parseString(uglyString) 285 return reparsed.toprettyxml(indent='\t', encoding='utf-8') 286 287def genSpecXML (mustpass): 288 mustpassElem = ElementTree.Element("Mustpass", version = mustpass.version) 289 insertXMLHeaders(mustpass, mustpassElem) 290 291 for package in mustpass.packages: 292 packageElem = ElementTree.SubElement(mustpassElem, "TestPackage", name = package.module.name) 293 294 for config in package.configurations: 295 configElem = ElementTree.SubElement(packageElem, "Configuration", 296 caseListFile = getCaseListFileName(package, config), 297 commandLine = getCommandLine(config), 298 name = config.name) 299 300 return mustpassElem 301 302def addOptionElement (parent, optionName, optionValue): 303 ElementTree.SubElement(parent, "option", name=optionName, value=optionValue) 304 305def genAndroidTestXml (mustpass): 306 RUNNER_CLASS = "com.drawelements.deqp.runner.DeqpTestRunner" 307 configElement = ElementTree.Element("configuration") 308 309 # have the deqp package installed on the device for us 310 preparerElement = ElementTree.SubElement(configElement, "target_preparer") 311 preparerElement.set("class", "com.android.tradefed.targetprep.suite.SuiteApkInstaller") 312 addOptionElement(preparerElement, "cleanup-apks", "true") 313 addOptionElement(preparerElement, "test-file-name", "com.drawelements.deqp.apk") 314 315 # add in metadata option for component name 316 ElementTree.SubElement(configElement, "option", name="test-suite-tag", value="cts") 317 ElementTree.SubElement(configElement, "option", key="component", name="config-descriptor:metadata", value="deqp") 318 ElementTree.SubElement(configElement, "option", key="parameter", name="config-descriptor:metadata", value="not_instant_app") 319 ElementTree.SubElement(configElement, "option", key="parameter", name="config-descriptor:metadata", value="multi_abi") 320 ElementTree.SubElement(configElement, "option", key="parameter", name="config-descriptor:metadata", value="secondary_user") 321 controllerElement = ElementTree.SubElement(configElement, "object") 322 controllerElement.set("class", "com.android.tradefed.testtype.suite.module.TestFailureModuleController") 323 controllerElement.set("type", "module_controller") 324 addOptionElement(controllerElement, "screenshot-on-failure", "false") 325 326 for package in mustpass.packages: 327 for config in package.configurations: 328 if not config.runByDefault: 329 continue 330 331 testElement = ElementTree.SubElement(configElement, "test") 332 testElement.set("class", RUNNER_CLASS) 333 addOptionElement(testElement, "deqp-package", package.module.name) 334 addOptionElement(testElement, "deqp-caselist-file", getCaseListFileName(package,config)) 335 # \todo [2015-10-16 kalle]: Replace with just command line? - requires simplifications in the runner/tests as well. 336 if config.glconfig != None: 337 addOptionElement(testElement, "deqp-gl-config-name", config.glconfig) 338 339 if config.surfacetype != None: 340 addOptionElement(testElement, "deqp-surface-type", config.surfacetype) 341 342 if config.rotation != None: 343 addOptionElement(testElement, "deqp-screen-rotation", config.rotation) 344 345 if config.expectedRuntime != None: 346 addOptionElement(testElement, "runtime-hint", config.expectedRuntime) 347 348 if config.required: 349 addOptionElement(testElement, "deqp-config-required", "true") 350 351 insertXMLHeaders(mustpass, configElement) 352 353 return configElement 354 355def genMustpass (mustpass, moduleCaseLists): 356 print("Generating mustpass '%s'" % mustpass.version) 357 358 patternLists = readPatternLists(mustpass) 359 360 for package in mustpass.packages: 361 allCasesInPkg = moduleCaseLists[package.module] 362 363 for config in package.configurations: 364 filtered = applyFilters(allCasesInPkg, patternLists, config.filters) 365 dstFile = getDstCaseListPath(mustpass, package, config) 366 367 print(" Writing deqp caselist: " + dstFile) 368 writeFile(dstFile, "\n".join(filtered) + "\n") 369 370 specXML = genSpecXML(mustpass) 371 specFilename = os.path.join(mustpass.project.path, mustpass.version, "mustpass.xml") 372 373 print(" Writing spec: " + specFilename) 374 writeFile(specFilename, prettifyXML(specXML).decode()) 375 376 # TODO: Which is the best selector mechanism? 377 if (mustpass.version == "master"): 378 androidTestXML = genAndroidTestXml(mustpass) 379 androidTestFilename = os.path.join(mustpass.project.path, "AndroidTest.xml") 380 381 print(" Writing AndroidTest.xml: " + androidTestFilename) 382 writeFile(androidTestFilename, prettifyXML(androidTestXML).decode()) 383 384 print("Done!") 385 386def genMustpassLists (mustpassLists, generator, buildCfg): 387 moduleCaseLists = {} 388 389 # Getting case lists involves invoking build, so we want to cache the results 390 for mustpass in mustpassLists: 391 for package in mustpass.packages: 392 if not package.module in moduleCaseLists: 393 moduleCaseLists[package.module] = getCaseList(buildCfg, generator, package.module) 394 395 for mustpass in mustpassLists: 396 genMustpass(mustpass, moduleCaseLists) 397 398def parseCmdLineArgs (): 399 parser = argparse.ArgumentParser(description = "Build Android CTS mustpass", 400 formatter_class=argparse.ArgumentDefaultsHelpFormatter) 401 parser.add_argument("-b", 402 "--build-dir", 403 dest="buildDir", 404 default=DEFAULT_BUILD_DIR, 405 help="Temporary build directory") 406 parser.add_argument("-t", 407 "--build-type", 408 dest="buildType", 409 default="Debug", 410 help="Build type") 411 parser.add_argument("-c", 412 "--deqp-target", 413 dest="targetName", 414 default=DEFAULT_TARGET, 415 help="dEQP build target") 416 return parser.parse_args() 417 418def parseBuildConfigFromCmdLineArgs (): 419 args = parseCmdLineArgs() 420 return getBuildConfig(args.buildDir, args.targetName, args.buildType) 421