1# -*- coding: utf-8 -*- 2#------------------------------------------------------------------------- 3# drawElements Quality Program utilities 4# -------------------------------------- 5# 6# Copyright 2016 The Android Open Source Project 7# 8# Licensed under the Apache License, Version 2.0 (the "License"); 9# you may not use this file except in compliance with the License. 10# You may obtain a copy of the License at 11# 12# http://www.apache.org/licenses/LICENSE-2.0 13# 14# Unless required by applicable law or agreed to in writing, software 15# distributed under the License is distributed on an "AS IS" BASIS, 16# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 17# See the License for the specific language governing permissions and 18# limitations under the License. 19# 20#------------------------------------------------------------------------- 21 22from ctsbuild.common import * 23from ctsbuild.build import build 24from build_caselists import Module, getModuleByName, getBuildConfig, genCaseList, getCaseListPath, DEFAULT_BUILD_DIR, DEFAULT_TARGET 25from fnmatch import fnmatch 26from copy import copy 27from collections import defaultdict 28import logging 29import argparse 30import re 31import xml.etree.cElementTree as ElementTree 32import xml.dom.minidom as minidom 33 34GENERATED_FILE_WARNING = """ 35 This file has been automatically generated. Edit with caution. 36 """ 37 38class Project: 39 def __init__ (self, path, copyright = None): 40 self.path = path 41 self.copyright = copyright 42 43class Configuration: 44 def __init__ (self, name, filters, glconfig = None, rotation = None, surfacetype = None, required = False, runtime = None, runByDefault = True, listOfGroupsToSplit = []): 45 self.name = name 46 self.glconfig = glconfig 47 self.rotation = rotation 48 self.surfacetype = surfacetype 49 self.required = required 50 self.filters = filters 51 self.expectedRuntime = runtime 52 self.runByDefault = runByDefault 53 self.listOfGroupsToSplit = listOfGroupsToSplit 54 55class Package: 56 def __init__ (self, module, configurations): 57 self.module = module 58 self.configurations = configurations 59 60class Mustpass: 61 def __init__ (self, project, version, packages): 62 self.project = project 63 self.version = version 64 self.packages = packages 65 66class Filter: 67 TYPE_INCLUDE = 0 68 TYPE_EXCLUDE = 1 69 70 def __init__ (self, type, filenames): 71 self.type = type 72 self.filenames = filenames 73 self.key = ",".join(filenames) 74 75class TestRoot: 76 def __init__ (self): 77 self.children = [] 78 79class TestGroup: 80 def __init__ (self, name): 81 self.name = name 82 self.children = [] 83 84class TestCase: 85 def __init__ (self, name): 86 self.name = name 87 self.configurations = [] 88 89def getSrcDir (mustpass): 90 return os.path.join(mustpass.project.path, mustpass.version, "src") 91 92def getModuleShorthand (module): 93 assert module.name[:5] == "dEQP-" 94 return module.name[5:].lower() 95 96def getCaseListFileName (package, configuration): 97 return "%s-%s.txt" % (getModuleShorthand(package.module), configuration.name) 98 99def getDstCaseListPath (mustpass): 100 return os.path.join(mustpass.project.path, mustpass.version) 101 102def getCommandLine (config): 103 cmdLine = "" 104 105 if config.glconfig != None: 106 cmdLine += "--deqp-gl-config-name=%s " % config.glconfig 107 108 if config.rotation != None: 109 cmdLine += "--deqp-screen-rotation=%s " % config.rotation 110 111 if config.surfacetype != None: 112 cmdLine += "--deqp-surface-type=%s " % config.surfacetype 113 114 cmdLine += "--deqp-watchdog=enable" 115 116 return cmdLine 117 118class CaseList: 119 def __init__(self, filePath, sortedLines): 120 self.filePath = filePath 121 self.sortedLines = sortedLines 122 123def readAndSortCaseList (buildCfg, generator, module): 124 build(buildCfg, generator, [module.binName]) 125 genCaseList(buildCfg, generator, module, "txt") 126 filePath = getCaseListPath(buildCfg, module, "txt") 127 with open(filePath, 'r') as first_file: 128 lines = first_file.readlines() 129 lines.sort() 130 caseList = CaseList(filePath, lines) 131 return caseList 132 133def readPatternList (filename, patternList): 134 with open(filename, 'rt') as f: 135 for line in f: 136 line = line.strip() 137 if len(line) > 0 and line[0] != '#': 138 patternList.append(line) 139 140def include (*filenames): 141 return Filter(Filter.TYPE_INCLUDE, filenames) 142 143def exclude (*filenames): 144 return Filter(Filter.TYPE_EXCLUDE, filenames) 145 146def insertXMLHeaders (mustpass, doc): 147 if mustpass.project.copyright != None: 148 doc.insert(0, ElementTree.Comment(mustpass.project.copyright)) 149 doc.insert(1, ElementTree.Comment(GENERATED_FILE_WARNING)) 150 151def prettifyXML (doc): 152 uglyString = ElementTree.tostring(doc, 'utf-8') 153 reparsed = minidom.parseString(uglyString) 154 return reparsed.toprettyxml(indent='\t', encoding='utf-8') 155 156def genSpecXML (mustpass): 157 mustpassElem = ElementTree.Element("Mustpass", version = mustpass.version) 158 insertXMLHeaders(mustpass, mustpassElem) 159 160 for package in mustpass.packages: 161 packageElem = ElementTree.SubElement(mustpassElem, "TestPackage", name = package.module.name) 162 163 for config in package.configurations: 164 configElem = ElementTree.SubElement(packageElem, "Configuration", 165 caseListFile = getCaseListFileName(package, config), 166 commandLine = getCommandLine(config), 167 name = config.name) 168 169 return mustpassElem 170 171def addOptionElement (parent, optionName, optionValue): 172 ElementTree.SubElement(parent, "option", name=optionName, value=optionValue) 173 174def genAndroidTestXml (mustpass): 175 RUNNER_CLASS = "com.drawelements.deqp.runner.DeqpTestRunner" 176 configElement = ElementTree.Element("configuration") 177 178 # have the deqp package installed on the device for us 179 preparerElement = ElementTree.SubElement(configElement, "target_preparer") 180 preparerElement.set("class", "com.android.tradefed.targetprep.suite.SuiteApkInstaller") 181 addOptionElement(preparerElement, "cleanup-apks", "true") 182 addOptionElement(preparerElement, "test-file-name", "com.drawelements.deqp.apk") 183 184 # Target preparer for incremental dEQP 185 preparerElement = ElementTree.SubElement(configElement, "target_preparer") 186 preparerElement.set("class", "com.android.compatibility.common.tradefed.targetprep.FilePusher") 187 addOptionElement(preparerElement, "cleanup", "true") 188 addOptionElement(preparerElement, "disable", "true") 189 addOptionElement(preparerElement, "push", "deqp-binary32->/data/local/tmp/deqp-binary32") 190 addOptionElement(preparerElement, "push", "deqp-binary64->/data/local/tmp/deqp-binary64") 191 addOptionElement(preparerElement, "push", "gles2->/data/local/tmp/gles2") 192 addOptionElement(preparerElement, "push", "gles3->/data/local/tmp/gles3") 193 addOptionElement(preparerElement, "push", "gles3-incremental-deqp.txt->/data/local/tmp/gles3-incremental-deqp.txt") 194 addOptionElement(preparerElement, "push", "gles31->/data/local/tmp/gles31") 195 addOptionElement(preparerElement, "push", "internal->/data/local/tmp/internal") 196 addOptionElement(preparerElement, "push", "vk-incremental-deqp.txt->/data/local/tmp/vk-incremental-deqp.txt") 197 addOptionElement(preparerElement, "push", "vulkan->/data/local/tmp/vulkan") 198 preparerElement = ElementTree.SubElement(configElement, "target_preparer") 199 preparerElement.set("class", "com.android.compatibility.common.tradefed.targetprep.IncrementalDeqpPreparer") 200 addOptionElement(preparerElement, "disable", "true") 201 202 # add in metadata option for component name 203 ElementTree.SubElement(configElement, "option", name="test-suite-tag", value="cts") 204 ElementTree.SubElement(configElement, "option", key="component", name="config-descriptor:metadata", value="deqp") 205 ElementTree.SubElement(configElement, "option", key="parameter", name="config-descriptor:metadata", value="not_instant_app") 206 ElementTree.SubElement(configElement, "option", key="parameter", name="config-descriptor:metadata", value="multi_abi") 207 ElementTree.SubElement(configElement, "option", key="parameter", name="config-descriptor:metadata", value="secondary_user") 208 ElementTree.SubElement(configElement, "option", key="parameter", name="config-descriptor:metadata", value="no_foldable_states") 209 controllerElement = ElementTree.SubElement(configElement, "object") 210 controllerElement.set("class", "com.android.tradefed.testtype.suite.module.TestFailureModuleController") 211 controllerElement.set("type", "module_controller") 212 addOptionElement(controllerElement, "screenshot-on-failure", "false") 213 214 for package in mustpass.packages: 215 for config in package.configurations: 216 if not config.runByDefault: 217 continue 218 219 testElement = ElementTree.SubElement(configElement, "test") 220 testElement.set("class", RUNNER_CLASS) 221 addOptionElement(testElement, "deqp-package", package.module.name) 222 caseListFile = getCaseListFileName(package,config) 223 addOptionElement(testElement, "deqp-caselist-file", caseListFile) 224 if caseListFile.startswith("gles3"): 225 addOptionElement(testElement, "incremental-deqp-include-file", "gles3-incremental-deqp.txt") 226 elif caseListFile.startswith("vk"): 227 addOptionElement(testElement, "incremental-deqp-include-file", "vk-incremental-deqp.txt") 228 # \todo [2015-10-16 kalle]: Replace with just command line? - requires simplifications in the runner/tests as well. 229 if config.glconfig != None: 230 addOptionElement(testElement, "deqp-gl-config-name", config.glconfig) 231 232 if config.surfacetype != None: 233 addOptionElement(testElement, "deqp-surface-type", config.surfacetype) 234 235 if config.rotation != None: 236 addOptionElement(testElement, "deqp-screen-rotation", config.rotation) 237 238 if config.expectedRuntime != None: 239 addOptionElement(testElement, "runtime-hint", config.expectedRuntime) 240 241 if config.required: 242 addOptionElement(testElement, "deqp-config-required", "true") 243 244 insertXMLHeaders(mustpass, configElement) 245 246 return configElement 247 248class PatternSet: 249 def __init__(self): 250 self.namedPatternsTree = {} 251 self.namedPatternsDict = {} 252 self.wildcardPatternsDict = {} 253 254def readPatternSets (mustpass): 255 patternSets = {} 256 for package in mustpass.packages: 257 for cfg in package.configurations: 258 for filter in cfg.filters: 259 if not filter.key in patternSets: 260 patternList = [] 261 for filename in filter.filenames: 262 readPatternList(os.path.join(getSrcDir(mustpass), filename), patternList) 263 patternSet = PatternSet() 264 for pattern in patternList: 265 if pattern.find('*') == -1: 266 patternSet.namedPatternsDict[pattern] = 0 267 t = patternSet.namedPatternsTree 268 parts = pattern.split('.') 269 for part in parts: 270 t = t.setdefault(part, {}) 271 else: 272 # We use regex instead of fnmatch because it's faster 273 patternSet.wildcardPatternsDict[re.compile("^" + pattern.replace(".", r"\.").replace("*", ".*?") + "$")] = 0 274 patternSets[filter.key] = patternSet 275 return patternSets 276 277def genMustpassFromLists (mustpass, moduleCaseLists): 278 print("Generating mustpass '%s'" % mustpass.version) 279 patternSets = readPatternSets(mustpass) 280 281 for package in mustpass.packages: 282 currentCaseList = moduleCaseLists[package.module] 283 logging.debug("Reading " + currentCaseList.filePath) 284 285 for config in package.configurations: 286 # construct components of path to main destination file 287 mainDstFileDir = getDstCaseListPath(mustpass) 288 mainDstFileName = getCaseListFileName(package, config) 289 mainDstFilePath = os.path.join(mainDstFileDir, mainDstFileName) 290 mainGroupSubDir = mainDstFileName[:-4] 291 292 if not os.path.exists(mainDstFileDir): 293 os.makedirs(mainDstFileDir) 294 mainDstFile = open(mainDstFilePath, 'w') 295 print(mainDstFilePath) 296 output_files = {} 297 def openAndStoreFile(filePath, testFilePath, parentFile): 298 if filePath not in output_files: 299 try: 300 print(" " + filePath) 301 parentFile.write(mainGroupSubDir + "/" + testFilePath + "\n") 302 currentDir = os.path.dirname(filePath) 303 if not os.path.exists(currentDir): 304 os.makedirs(currentDir) 305 output_files[filePath] = open(filePath, 'w') 306 307 except FileNotFoundError: 308 print(f"File not found: {filePath}") 309 return output_files[filePath] 310 311 lastOutputFile = "" 312 currentOutputFile = None 313 for line in currentCaseList.sortedLines: 314 if not line.startswith("TEST: "): 315 continue 316 caseName = line.replace("TEST: ", "").strip("\n") 317 caseParts = caseName.split(".") 318 keep = True 319 # Do the includes with the complex patterns first 320 for filter in config.filters: 321 if filter.type == Filter.TYPE_INCLUDE: 322 keep = False 323 patterns = patternSets[filter.key].wildcardPatternsDict 324 for pattern in patterns.keys(): 325 keep = pattern.match(caseName) 326 if keep: 327 patterns[pattern] += 1 328 break 329 330 if not keep: 331 t = patternSets[filter.key].namedPatternsTree 332 if len(t.keys()) == 0: 333 continue 334 for part in caseParts: 335 if part in t: 336 t = t[part] 337 else: 338 t = None # Not found 339 break 340 keep = t == {} 341 if keep: 342 patternSets[filter.key].namedPatternsDict[caseName] += 1 343 344 # Do the excludes 345 if filter.type == Filter.TYPE_EXCLUDE: 346 patterns = patternSets[filter.key].wildcardPatternsDict 347 for pattern in patterns.keys(): 348 discard = pattern.match(caseName) 349 if discard: 350 patterns[pattern] += 1 351 keep = False 352 break 353 if keep: 354 t = patternSets[filter.key].namedPatternsTree 355 if len(t.keys()) == 0: 356 continue 357 for part in caseParts: 358 if part in t: 359 t = t[part] 360 else: 361 t = None # Not found 362 break 363 if t == {}: 364 patternSets[filter.key].namedPatternsDict[caseName] += 1 365 keep = False 366 if not keep: 367 break 368 if not keep: 369 continue 370 371 parts = caseName.split('.') 372 if len(config.listOfGroupsToSplit) > 0: 373 if len(parts) > 2: 374 groupName = parts[1].replace("_", "-") 375 for splitPattern in config.listOfGroupsToSplit: 376 splitParts = splitPattern.split(".") 377 if len(splitParts) > 1 and caseName.startswith(splitPattern + "."): 378 groupName = groupName + "/" + parts[2].replace("_", "-") 379 filePath = os.path.join(mainDstFileDir, mainGroupSubDir, groupName + ".txt") 380 if lastOutputFile != filePath: 381 currentOutputFile = openAndStoreFile(filePath, groupName + ".txt", mainDstFile) 382 lastOutputFile = filePath 383 currentOutputFile.write(caseName + "\n") 384 else: 385 mainDstFile.write(caseName + "\n") 386 387 # Check that all patterns have been used in the filters 388 # This check will help identifying typos and patterns becoming stale 389 for filter in config.filters: 390 if filter.type == Filter.TYPE_INCLUDE: 391 patternSet = patternSets[filter.key] 392 for pattern, usage in patternSet.namedPatternsDict.items(): 393 if usage == 0: 394 logging.warning("Case %s in file %s for module %s was never used!" % (pattern, filter.key, config.name)) 395 for pattern, usage in patternSet.wildcardPatternsDict.items(): 396 if usage == 0: 397 logging.warning("Pattern %s in file %s for module %s was never used!" % (pattern, filter.key, config.name)) 398 399 # Generate XML 400 specXML = genSpecXML(mustpass) 401 specFilename = os.path.join(mustpass.project.path, mustpass.version, "mustpass.xml") 402 403 print(" Writing spec: " + specFilename) 404 writeFile(specFilename, prettifyXML(specXML).decode()) 405 406 # TODO: Which is the best selector mechanism? 407 if (mustpass.version == "main"): 408 androidTestXML = genAndroidTestXml(mustpass) 409 androidTestFilename = os.path.join(mustpass.project.path, "AndroidTest.xml") 410 411 print(" Writing AndroidTest.xml: " + androidTestFilename) 412 writeFile(androidTestFilename, prettifyXML(androidTestXML).decode()) 413 414 print("Done!") 415 416 417def genMustpassLists (mustpassLists, generator, buildCfg): 418 moduleCaseLists = {} 419 420 # Getting case lists involves invoking build, so we want to cache the results 421 for mustpass in mustpassLists: 422 for package in mustpass.packages: 423 if not package.module in moduleCaseLists: 424 moduleCaseLists[package.module] = readAndSortCaseList(buildCfg, generator, package.module) 425 426 for mustpass in mustpassLists: 427 genMustpassFromLists(mustpass, moduleCaseLists) 428 429def parseCmdLineArgs (): 430 parser = argparse.ArgumentParser(description = "Build Android CTS mustpass", 431 formatter_class=argparse.ArgumentDefaultsHelpFormatter) 432 parser.add_argument("-b", 433 "--build-dir", 434 dest="buildDir", 435 default=DEFAULT_BUILD_DIR, 436 help="Temporary build directory") 437 parser.add_argument("-t", 438 "--build-type", 439 dest="buildType", 440 default="Debug", 441 help="Build type") 442 parser.add_argument("-c", 443 "--deqp-target", 444 dest="targetName", 445 default=DEFAULT_TARGET, 446 help="dEQP build target") 447 parser.add_argument("-v", "--verbose", 448 dest="verbose", 449 action="store_true", 450 help="Enable verbose logging") 451 return parser.parse_args() 452 453def parseBuildConfigFromCmdLineArgs (): 454 args = parseCmdLineArgs() 455 initializeLogger(args.verbose) 456 return getBuildConfig(args.buildDir, args.targetName, args.buildType) 457