xref: /third_party/vk-gl-cts/scripts/mustpass.py

# -*- coding: utf-8 -*-

#-------------------------------------------------------------------------
# drawElements Quality Program utilities
# --------------------------------------
#
# Copyright 2016 The Android Open Source Project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
#-------------------------------------------------------------------------

from ctsbuild.common import *
from ctsbuild.config import ANY_GENERATOR
from ctsbuild.build import build
from build_caselists import Module, getModuleByName, getBuildConfig, genCaseList, getCaseListPath, DEFAULT_BUILD_DIR, DEFAULT_TARGET
from fnmatch import fnmatch
from copy import copy
from collections import defaultdict

import argparse
import xml.etree.cElementTree as ElementTree
import xml.dom.minidom as minidom

APK_NAME		= "com.drawelements.deqp.apk"

GENERATED_FILE_WARNING = """
     This file has been automatically generated. Edit with caution.
     """

class Project:
	def __init__ (self, path, copyright = None):
		self.path		= path
		self.copyright	= copyright

class Configuration:
	def __init__ (self, name, filters, glconfig = None, rotation = None, surfacetype = None, required = False, runtime = None, runByDefault = True, listOfGroupsToSplit = []):
		self.name					= name
		self.glconfig				= glconfig
		self.rotation				= rotation
		self.surfacetype			= surfacetype
		self.required				= required
		self.filters				= filters
		self.expectedRuntime		= runtime
		self.runByDefault			= runByDefault
		self.listOfGroupsToSplit	= listOfGroupsToSplit

class Package:
	def __init__ (self, module, configurations):
		self.module			= module
		self.configurations	= configurations

class Mustpass:
	def __init__ (self, project, version, packages):
		self.project	= project
		self.version	= version
		self.packages	= packages

class Filter:
	TYPE_INCLUDE = 0
	TYPE_EXCLUDE = 1

	def __init__ (self, type, filename):
		self.type		= type
		self.filename	= filename

class TestRoot:
	def __init__ (self):
		self.children	= []

class TestGroup:
	def __init__ (self, name):
		self.name		= name
		self.children	= []

class TestCase:
	def __init__ (self, name):
		self.name			= name
		self.configurations	= []

class GLESVersion:
	def __init__(self, major, minor):
		self.major = major
		self.minor = minor

	def encode (self):
		return (self.major << 16) | (self.minor)

def getModuleGLESVersion (module):
	versions = {
		'dEQP-EGL':		GLESVersion(2,0),
		'dEQP-GLES2':	GLESVersion(2,0),
		'dEQP-GLES3':	GLESVersion(3,0),
		'dEQP-GLES31':	GLESVersion(3,1)
	}
	return versions[module.name] if module.name in versions else None

def getSrcDir (mustpass):
	return os.path.join(mustpass.project.path, mustpass.version, "src")

def getTmpDir (mustpass):
	return os.path.join(mustpass.project.path, mustpass.version, "tmp")

def getModuleShorthand (module):
	assert module.name[:5] == "dEQP-"
	return module.name[5:].lower()

def getCaseListFileName (package, configuration):
	return "%s-%s.txt" % (getModuleShorthand(package.module), configuration.name)

def getDstCaseListPath (mustpass):
	return os.path.join(mustpass.project.path, mustpass.version)

def getCTSPackageName (package):
	return "com.drawelements.deqp." + getModuleShorthand(package.module)

def getCommandLine (config):
	cmdLine = ""

	if config.glconfig != None:
		cmdLine += "--deqp-gl-config-name=%s " % config.glconfig

	if config.rotation != None:
		cmdLine += "--deqp-screen-rotation=%s " % config.rotation

	if config.surfacetype != None:
		cmdLine += "--deqp-surface-type=%s " % config.surfacetype

	cmdLine += "--deqp-watchdog=enable"

	return cmdLine

def readCaseDict (filename):
	# read all cases and organize them in a tree; this is needed for chunked mustpass
	# groups are stored as dictionaries and cases as list of strings with full case paths
	groupTree = {}
	# limit how deep constructed tree should be - this later simplifies applying filters;
	# if in future we will need to split to separate .txt files deeper groups thet this value should be increased
	limitGroupTreeDepth = 3
	# create helper stack that will contain references to currently filled groups, from top to bottom
	groupStack = None
	# cretae variable that will hold currentlt processed line from the file
	processedLine = None
	with open(filename, 'rt') as f:
		for nextLine in f:
			# to be able to build tree structure we need to know what is the next line in the file this is
			# why the first line read from the file will be actually processed during the second iteration
			if processedLine is None:
				processedLine		= nextLine
				# to simplify code use this section to also extract root node name
				rootName			= processedLine[7:processedLine.rfind('.')]
				groupTree[rootName]	= {}
				groupStack			= [groupTree[rootName]]
				continue
			# check if currently processed line is a test case or a group
			processedEntryType = processedLine[:6]
			if processedEntryType == "TEST: ":
				# append this test case to the last group on the stack
				groupStack[-1].append(processedLine[6:].strip())
			elif processedEntryType == "GROUP:":
				# count number of dots in path to determine what is the depth of current group in the tree
				processedGroupDepth = processedLine.count('.')
				# limit tree construction just to specified level
				availableLimit = limitGroupTreeDepth - processedGroupDepth
				if availableLimit > 0:
					# check how deep is stack currently
					groupStackDepth = len(groupStack)
					# if stack is deeper then depth of current group then we need to pop number of items
					if processedGroupDepth < groupStackDepth:
						groupStack = groupStack[:groupStackDepth-(groupStackDepth-processedGroupDepth)]
					# get group that will have new child - this is the last group on the stack
					parentGroup = groupStack[-1]
					# add new dict that will contain other groups or list of cases depending on the next line
					# and available depth limit (if are about to reach limit we won't add group dictionaries
					# but just add all cases from deeper groups to the group at this depth)
					processedGroupName = processedLine[7:-1]
					parentGroup[processedGroupName] = {} if (nextLine[:6] == "GROUP:") and (availableLimit > 1) else []
					# add new group to the stack (items in groupStack can be either list or dict)
					groupStack.append(parentGroup[processedGroupName])
			# before going to the next line set procesedLine for the next iteration
			processedLine = nextLine
	# handle last test case - we need to do it after the loop as in the loop we needed to know what is the next line
	assert(processedLine[:6] == "TEST: ")
	groupStack[-1].append(processedLine[6:].strip())
	return groupTree

def getCaseDict (buildCfg, generator, module):
	build(buildCfg, generator, [module.binName])
	genCaseList(buildCfg, generator, module, "txt")
	return readCaseDict(getCaseListPath(buildCfg, module, "txt"))

def readPatternList (filename):
	ptrns = []
	with open(filename, 'rt') as f:
		for line in f:
			line = line.strip()
			if len(line) > 0 and line[0] != '#':
				ptrns.append(line)
	return ptrns


def constructNewDict(oldDict, listOfCases, op = lambda a: not a):
	# Helper function used to construct case dictionary without specific cases
	rootName		= list(oldDict.keys())[0]
	newDict			= {rootName : {}}
	newDictStack	= [newDict]
	oldDictStack	= [oldDict]
	while True:
		# mak sure that both stacks have same number of items
		assert(len(oldDictStack) == len(newDictStack))
		# when all items from stack were processed then we can exit the loop
		if len(oldDictStack) == 0:
			break
		# grab last item from both stacks
		itemOnOldStack = oldDictStack.pop()
		itemOnNewStack = newDictStack.pop()
		# if item on stack is dictionary then it represents groups and
		# we need to reconstruct them in new dictionary
		if type(itemOnOldStack) is dict:
			assert(type(itemOnNewStack) is dict)
			listOfGroups = list(itemOnOldStack.keys())
			for groupName in listOfGroups:
				# create list or dictionary depending on contnent of child group
				doesGroupsContainCases = type(itemOnOldStack[groupName]) is list
				itemOnNewStack[groupName] = [] if doesGroupsContainCases else {}
				# append groups on stacks
				assert(type(itemOnNewStack[groupName]) == type(itemOnOldStack[groupName]))
				newDictStack.append(itemOnNewStack[groupName])
				oldDictStack.append(itemOnOldStack[groupName])
		else:
			# if item on stack is list then it represents group that contain cases we need
			# to apply filter on each of them to make sure only proper cases are appended
			assert(type(itemOnOldStack) is list)
			assert(type(itemOnNewStack) is list)
			for caseName in itemOnOldStack:
				if op(caseName in listOfCases):
					itemOnNewStack.append(caseName)
	return newDict

def constructSet(caseDict, perGroupOperation):
	casesSet		= set()
	dictStack		= [caseDict]
	while True:
		# when all items from stack were processed then we can exit the loop
		if len(dictStack) == 0:
			break
		# grab last item from stack
		itemOnStack = dictStack.pop()
		# if item on stack is dictionary then it represents groups and we need to add them to stack
		if type(itemOnStack) is dict:
			for groupName in itemOnStack.keys():
				dictStack.append(itemOnStack[groupName])
		else:
			# if item on stack is a list of cases we can add them to set containing all cases
			assert(type(itemOnStack) is list)
			casesSet = perGroupOperation(casesSet, itemOnStack)
	return casesSet

def applyPatterns (caseDict, patterns, filename, op):
	matched			= set()
	errors			= []
	trivialPtrns	= [p for p in patterns if p.find('*') < 0]
	regularPtrns	= [p for p in patterns if p.find('*') >= 0]

	# Construct helper set that contains cases from all groups
	unionOperation	= lambda resultCasesSet, groupCaseList: resultCasesSet.union(set(groupCaseList))
	allCasesSet		= constructSet(caseDict, unionOperation)

	# Apply trivial patterns - plain case paths without wildcard
	for path in trivialPtrns:
		if path in allCasesSet:
			if path in matched:
				errors.append((path, "Same case specified more than once"))
			matched.add(path)
		else:
			errors.append((path, "Test case not found"))

	# Construct new dictionary but without already matched paths
	curDict = constructNewDict(caseDict, matched)

	# Apply regular patterns - paths with wildcard
	for pattern in regularPtrns:

		# Helper function that checks if cases from case group match pattern
		def matchOperation(resultCasesSet, groupCaseList):
			for caseName in groupCaseList:
				if fnmatch(caseName, pattern):
					resultCasesSet.add(caseName)
			return resultCasesSet

		matchedThisPtrn = constructSet(curDict, matchOperation)

		if len(matchedThisPtrn) == 0:
			errors.append((pattern, "Pattern didn't match any cases"))

		matched = matched | matchedThisPtrn

		# To speed up search construct smaller case dictionary without already matched paths
		curDict = constructNewDict(curDict, matched)

	for pattern, reason in errors:
		print("ERROR: %s: %s" % (reason, pattern))

	if len(errors) > 0:
		die("Found %s invalid patterns while processing file %s" % (len(errors), filename))

	# Construct final dictionary using aproperiate operation
	return constructNewDict(caseDict, matched, op)

def applyInclude (caseDict, patterns, filename):
	return applyPatterns(caseDict, patterns, filename, lambda b: b)

def applyExclude (caseDict, patterns, filename):
	return applyPatterns(caseDict, patterns, filename, lambda b: not b)

def readPatternLists (mustpass):
	lists = {}
	for package in mustpass.packages:
		for cfg in package.configurations:
			for filter in cfg.filters:
				if not filter.filename in lists:
					lists[filter.filename] = readPatternList(os.path.join(getSrcDir(mustpass), filter.filename))
	return lists

def applyFilters (caseDict, patternLists, filters):
	res = copy(caseDict)
	for filter in filters:
		ptrnList = patternLists[filter.filename]
		if filter.type == Filter.TYPE_INCLUDE:
			res = applyInclude(res, ptrnList, filter.filename)
		else:
			assert filter.type == Filter.TYPE_EXCLUDE
			res = applyExclude(res, ptrnList, filter.filename)
	return res

def appendToHierarchy (root, casePath):
	def findChild (node, name):
		for child in node.children:
			if child.name == name:
				return child
		return None

	curNode		= root
	components	= casePath.split('.')

	for component in components[:-1]:
		nextNode = findChild(curNode, component)
		if not nextNode:
			nextNode = TestGroup(component)
			curNode.children.append(nextNode)
		curNode = nextNode

	if not findChild(curNode, components[-1]):
		curNode.children.append(TestCase(components[-1]))

def buildTestHierachy (caseList):
	root = TestRoot()
	for case in caseList:
		appendToHierarchy(root, case)
	return root

def buildTestCaseMap (root):
	caseMap = {}

	def recursiveBuild (curNode, prefix):
		curPath = prefix + curNode.name
		if isinstance(curNode, TestCase):
			caseMap[curPath] = curNode
		else:
			for child in curNode.children:
				recursiveBuild(child, curPath + '.')

	for child in root.children:
		recursiveBuild(child, '')

	return caseMap

def include (filename):
	return Filter(Filter.TYPE_INCLUDE, filename)

def exclude (filename):
	return Filter(Filter.TYPE_EXCLUDE, filename)

def insertXMLHeaders (mustpass, doc):
	if mustpass.project.copyright != None:
		doc.insert(0, ElementTree.Comment(mustpass.project.copyright))
	doc.insert(1, ElementTree.Comment(GENERATED_FILE_WARNING))

def prettifyXML (doc):
	uglyString	= ElementTree.tostring(doc, 'utf-8')
	reparsed	= minidom.parseString(uglyString)
	return reparsed.toprettyxml(indent='\t', encoding='utf-8')

def genSpecXML (mustpass):
	mustpassElem = ElementTree.Element("Mustpass", version = mustpass.version)
	insertXMLHeaders(mustpass, mustpassElem)

	for package in mustpass.packages:
		packageElem = ElementTree.SubElement(mustpassElem, "TestPackage", name = package.module.name)

		for config in package.configurations:
			configElem = ElementTree.SubElement(packageElem, "Configuration",
												caseListFile	= getCaseListFileName(package, config),
												commandLine		= getCommandLine(config),
												name			= config.name)

	return mustpassElem

def addOptionElement (parent, optionName, optionValue):
	ElementTree.SubElement(parent, "option", name=optionName, value=optionValue)

def genAndroidTestXml (mustpass):
	RUNNER_CLASS = "com.drawelements.deqp.runner.DeqpTestRunner"
	configElement = ElementTree.Element("configuration")

	# have the deqp package installed on the device for us
	preparerElement = ElementTree.SubElement(configElement, "target_preparer")
	preparerElement.set("class", "com.android.tradefed.targetprep.suite.SuiteApkInstaller")
	addOptionElement(preparerElement, "cleanup-apks", "true")
	addOptionElement(preparerElement, "test-file-name", "com.drawelements.deqp.apk")

	# Target preparer for incremental dEQP
	preparerElement = ElementTree.SubElement(configElement, "target_preparer")
	preparerElement.set("class", "com.android.compatibility.common.tradefed.targetprep.IncrementalDeqpPreparer")
	addOptionElement(preparerElement, "disable", "true")

	# add in metadata option for component name
	ElementTree.SubElement(configElement, "option", name="test-suite-tag", value="cts")
	ElementTree.SubElement(configElement, "option", key="component", name="config-descriptor:metadata", value="deqp")
	ElementTree.SubElement(configElement, "option", key="parameter", name="config-descriptor:metadata", value="not_instant_app")
	ElementTree.SubElement(configElement, "option", key="parameter", name="config-descriptor:metadata", value="multi_abi")
	ElementTree.SubElement(configElement, "option", key="parameter", name="config-descriptor:metadata", value="secondary_user")
	ElementTree.SubElement(configElement, "option", key="parameter", name="config-descriptor:metadata", value="no_foldable_states")
	controllerElement = ElementTree.SubElement(configElement, "object")
	controllerElement.set("class", "com.android.tradefed.testtype.suite.module.TestFailureModuleController")
	controllerElement.set("type", "module_controller")
	addOptionElement(controllerElement, "screenshot-on-failure", "false")

	for package in mustpass.packages:
		for config in package.configurations:
			if not config.runByDefault:
				continue

			testElement = ElementTree.SubElement(configElement, "test")
			testElement.set("class", RUNNER_CLASS)
			addOptionElement(testElement, "deqp-package", package.module.name)
			caseListFile = getCaseListFileName(package,config)
			addOptionElement(testElement, "deqp-caselist-file", caseListFile)
			if caseListFile.startswith("gles3"):
				addOptionElement(testElement, "incremental-deqp-include-file", "gles3-incremental-deqp.txt")
			elif caseListFile.startswith("vk"):
				addOptionElement(testElement, "incremental-deqp-include-file", "vk-incremental-deqp.txt")
			# \todo [2015-10-16 kalle]: Replace with just command line? - requires simplifications in the runner/tests as well.
			if config.glconfig != None:
				addOptionElement(testElement, "deqp-gl-config-name", config.glconfig)

			if config.surfacetype != None:
				addOptionElement(testElement, "deqp-surface-type", config.surfacetype)

			if config.rotation != None:
				addOptionElement(testElement, "deqp-screen-rotation", config.rotation)

			if config.expectedRuntime != None:
				addOptionElement(testElement, "runtime-hint", config.expectedRuntime)

			if config.required:
				addOptionElement(testElement, "deqp-config-required", "true")

	insertXMLHeaders(mustpass, configElement)

	return configElement

def genMustpass (mustpass, moduleCaseDicts):
	print("Generating mustpass '%s'" % mustpass.version)

	patternLists = readPatternLists(mustpass)

	for package in mustpass.packages:
		allCasesInPkgDict	= moduleCaseDicts[package.module]

		for config in package.configurations:

			# construct dictionary with all filters applyed
			filteredCaseDict	= applyFilters(allCasesInPkgDict, patternLists, config.filters)

			# construct components of path to main destination file
			mainDstFilePath		= getDstCaseListPath(mustpass)
			mainDstFileName		= getCaseListFileName(package, config)
			mainDstFile			= os.path.join(mainDstFilePath, mainDstFileName)
			mainGruopSubDir		= mainDstFileName[:-4]

			# if case paths should be split to multiple files then main
			# destination file will contain paths to individual files containing cases
			if len(config.listOfGroupsToSplit) > 0:
				# make sure directory for group files exists
				rootGroupPath = os.path.join(mainDstFilePath, mainGruopSubDir)
				if not os.path.exists(rootGroupPath):
					os.makedirs(rootGroupPath)

				# iterate over case dictionary and split it to .txt files acording to
				# groups that were specified in config.listOfGroupsToSplit
				splitedGroupsDict	= {}
				dictStack			= [filteredCaseDict]
				helperListStack		= [ [] ]
				while True:
					# when all items from stack were processed then we can exit the loop
					if len(dictStack) == 0:
						break
					assert(len(dictStack) == len(helperListStack))
					# grab last item from stack
					itemOnStack = dictStack.pop()
					caseListFromHelperStack = helperListStack.pop()
					# if item on stack is dictionary then it represents groups and we need to add them to stack
					if type(itemOnStack) is dict:
						for groupName in sorted(itemOnStack):

							# check if this group should be split to multiple .txt files
							if groupName in config.listOfGroupsToSplit:
								# we can split only groups that contain other groups,
								# listOfGroupsToSplit should not contain groups that contain test cases
								assert(type(itemOnStack[groupName]) is dict)
								# add child groups of this group to splitedGroupsDict
								for childGroupName in itemOnStack[groupName]:
									# make sure that child group should not be splited
									# (if it should then this will be handle in one of the next iterations)
									if childGroupName not in config.listOfGroupsToSplit:
										splitedGroupsDict[childGroupName] = []

							# add this group to stack used for iteration over casses tree
							dictStack.append(itemOnStack[groupName])

							# decide what list we should append to helperListStack;
							# if this group represents one of individual .txt files then grab
							# propper array of cases from splitedGroupsDict and add it to helper stack;
							# if groupName is not in splitedGroupsDict then use the same list as was used
							# by parent group (we are merging casses from those groups to single .txt file)
							helperListStack.append(splitedGroupsDict.get(groupName, caseListFromHelperStack))
					else:
						# if item on stack is a list of cases we can add them to proper list
						assert(type(itemOnStack) is list)
						caseListFromHelperStack.extend(itemOnStack)

				print("  Writing separated caselists:")
				groupPathsList = []
				for groupPath in splitedGroupsDict:
					# skip groups that after filtering have no casses left
					if len(splitedGroupsDict[groupPath]) == 0:
						continue
					# remove root node name from the beginning of group and replace all '_' with '-'
					processedGroupPath = groupPath[groupPath.find('.')+1:].replace('_', '-')
					# split group paths
					groupList = processedGroupPath.split('.')
					groupSubDir = '/'
					# create subdirectories if there is more then one group name in groupList
					path = rootGroupPath
					if len(groupList) > 1:
						for groupName in groupList[:-1]:
							# make sure directory for group files exists
							groupSubDir	= groupSubDir + groupName + '/'
							path		= os.path.join(path, groupName)
							if not os.path.exists(path):
								os.makedirs(path)
					# construct path to .txt file and save all cases
					groupDstFileName	= groupList[-1] + ".txt"
					groupDstFileFullDir	= os.path.join(path, groupDstFileName)
					groupPathsList.append(mainGruopSubDir + groupSubDir + groupDstFileName)
					print("    " + groupDstFileFullDir)
					writeFile(groupDstFileFullDir, "\n".join(splitedGroupsDict[groupPath]) + "\n")

				# write file containing names of all group files
				print("  Writing file containing list of separated case files: " + mainDstFile)
				groupPathsList.sort()
				writeFile(mainDstFile, "\n".join(groupPathsList) + "\n")
			else:
				# merge all cases to single case list
				filteredCaseList	= []
				dictStack			= [filteredCaseDict]
				while True:
					# when all items from stack were processed then we can exit the loop
					if len(dictStack) == 0:
						break
					# grab last item from stack
					itemOnStack = dictStack.pop()
					# if item on stack is dictionary then it represents groups and we need to add them to stack
					if type(itemOnStack) is dict:
						for groupName in itemOnStack.keys():
							dictStack.append(itemOnStack[groupName])
					else:
						# if item on stack is a list of cases we can add them to filteredCaseList
						assert(type(itemOnStack) is list)
						filteredCaseList.extend(itemOnStack)
				# write file containing all cases
				if len(filteredCaseList) > 0:
					print("  Writing deqp caselist: " + mainDstFile)
					writeFile(mainDstFile, "\n".join(filteredCaseList) + "\n")

	specXML = genSpecXML(mustpass)
	specFilename = os.path.join(mustpass.project.path, mustpass.version, "mustpass.xml")

	print("  Writing spec: " + specFilename)
	writeFile(specFilename, prettifyXML(specXML).decode())

	# TODO: Which is the best selector mechanism?
	if (mustpass.version == "master"):
		androidTestXML		= genAndroidTestXml(mustpass)
		androidTestFilename	= os.path.join(mustpass.project.path, "AndroidTest.xml")

		print("  Writing AndroidTest.xml: " + androidTestFilename)
		writeFile(androidTestFilename, prettifyXML(androidTestXML).decode())

	print("Done!")

def genMustpassLists (mustpassLists, generator, buildCfg):
	moduleCaseDicts = {}

	# Getting case lists involves invoking build, so we want to cache the results
	for mustpass in mustpassLists:
		for package in mustpass.packages:
			if not package.module in moduleCaseDicts:
				moduleCaseDicts[package.module] = getCaseDict(buildCfg, generator, package.module)

	for mustpass in mustpassLists:
		genMustpass(mustpass, moduleCaseDicts)

def parseCmdLineArgs ():
	parser = argparse.ArgumentParser(description = "Build Android CTS mustpass",
									 formatter_class=argparse.ArgumentDefaultsHelpFormatter)
	parser.add_argument("-b",
						"--build-dir",
						dest="buildDir",
						default=DEFAULT_BUILD_DIR,
						help="Temporary build directory")
	parser.add_argument("-t",
						"--build-type",
						dest="buildType",
						default="Debug",
						help="Build type")
	parser.add_argument("-c",
						"--deqp-target",
						dest="targetName",
						default=DEFAULT_TARGET,
						help="dEQP build target")
	return parser.parse_args()

def parseBuildConfigFromCmdLineArgs ():
	args = parseCmdLineArgs()
	return getBuildConfig(args.buildDir, args.targetName, args.buildType)