• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1# -*- coding: utf-8 -*-
2
3#-------------------------------------------------------------------------
4# drawElements Quality Program utilities
5# --------------------------------------
6#
7# Copyright 2015 The Android Open Source Project
8#
9# Licensed under the Apache License, Version 2.0 (the "License");
10# you may not use this file except in compliance with the License.
11# You may obtain a copy of the License at
12#
13#      http://www.apache.org/licenses/LICENSE-2.0
14#
15# Unless required by applicable law or agreed to in writing, software
16# distributed under the License is distributed on an "AS IS" BASIS,
17# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
18# See the License for the specific language governing permissions and
19# limitations under the License.
20#
21#-------------------------------------------------------------------------
22
23import sys
24import random
25import string
26import subprocess
27from optparse import OptionParser
28
29def all (results, predicate):
30	for result in results:
31		if not predicate(result):
32			return False
33	return True
34
35def any (results, predicate):
36	for result in results:
37		if predicate(result):
38			return True
39	return False
40
41class FilterRule:
42	def __init__ (self, name, description, filters):
43		self.name			= name
44		self.description	= description
45		self.filters		= filters
46
47class TestCaseResult:
48	def __init__ (self, name, results):
49		self.name		= name
50		self.results	= results
51
52class Group:
53	def __init__ (self, name):
54		self.name		= name
55		self.cases		= []
56
57def readCaseList (filename):
58	f = open(filename, 'rb')
59	cases = []
60	for line in f:
61		if line[:6] == "TEST: ":
62			case = line[6:].strip()
63			if len(case) > 0:
64				cases.append(case)
65	return cases
66
67def toResultList (caselist):
68	results = []
69	for case in caselist:
70		results.append(TestCaseResult(case, []))
71	return results
72
73def addResultsToCaseList (caselist, results):
74	resultMap	= {}
75	caseListRes	= toResultList(caselist)
76
77	for res in caseListRes:
78		resultMap[res.name] = res
79
80	for result in results:
81		if result.name in resultMap:
82			resultMap[result.name].results += result.results
83
84	return caseListRes
85
86def readTestResults (filename):
87	f			= open(filename, 'rb')
88	csvData		= f.read()
89	csvLines	= csvData.splitlines()
90	results		= []
91
92	f.close()
93
94	for line in csvLines[1:]:
95		args = line.split(',')
96		if len(args) == 1:
97			continue # Ignore
98
99		results.append(TestCaseResult(args[0], args[1:]))
100
101	if len(results) == 0:
102		raise Exception("Empty result list")
103
104	# Sanity check for results
105	numResultItems	= len(results[0].results)
106	seenResults		= set()
107	for result in results:
108		if result.name in seenResults:
109			raise Exception("Duplicate result row for test case '%s'" % result.name)
110		if len(result.results) != numResultItems:
111			raise Exception("Found %d results for test case '%s', expected %d" % (len(result.results), result.name, numResultItems))
112		seenResults.add(result.name)
113
114	return results
115
116def readGroupList (filename):
117	f = open(filename, 'rb')
118	groups = []
119	for line in f:
120		group = line.strip()
121		if group != "":
122			groups.append(group)
123	return groups
124
125def createGroups (results, groupNames):
126	groups	= []
127	matched	= set()
128
129	for groupName in groupNames:
130		group = Group(groupName)
131		groups.append(group)
132
133		prefix		= groupName + "."
134		prefixLen	= len(prefix)
135		for case in results:
136			if case.name[:prefixLen] == prefix:
137				if case in matched:
138					die("Case '%s' matched by multiple groups (when processing '%s')" % (case.name, group.name))
139				group.cases.append(case)
140				matched.add(case)
141
142	return groups
143
144def createLeafGroups (results):
145	groups = []
146	groupMap = {}
147
148	for case in results:
149		parts		= case.name.split('.')
150		groupName	= string.join(parts[:-1], ".")
151
152		if not groupName in groupMap:
153			group = Group(groupName)
154			groups.append(group)
155			groupMap[groupName] = group
156		else:
157			group = groupMap[groupName]
158
159		group.cases.append(case)
160
161	return groups
162
163def filterList (results, condition):
164	filtered = []
165	for case in results:
166		if condition(case.results):
167			filtered.append(case)
168	return filtered
169
170def getFilter (list, name):
171	for filter in list:
172		if filter.name == name:
173			return filter
174	return None
175
176def getNumCasesInGroups (groups):
177	numCases = 0
178	for group in groups:
179		numCases += len(group.cases)
180	return numCases
181
182def getCasesInSet (results, caseSet):
183	filtered = []
184	for case in results:
185		if case in caseSet:
186			filtered.append(case)
187	return filtered
188
189def selectCasesInGroups (results, groups):
190	casesInGroups = set()
191	for group in groups:
192		for case in group.cases:
193			casesInGroups.add(case)
194	return getCasesInSet(results, casesInGroups)
195
196def selectRandomSubset (results, groups, limit, seed):
197	selectedCases	= set()
198	numSelect		= min(limit, getNumCasesInGroups(groups))
199
200	random.seed(seed)
201	random.shuffle(groups)
202
203	groupNdx = 0
204	while len(selectedCases) < numSelect:
205		group = groups[groupNdx]
206		if len(group.cases) == 0:
207			del groups[groupNdx]
208			if groupNdx == len(groups):
209				groupNdx -= 1
210			continue # Try next
211
212		selected = random.choice(group.cases)
213		selectedCases.add(selected)
214		group.cases.remove(selected)
215
216		groupNdx = (groupNdx + 1) % len(groups)
217
218	return getCasesInSet(results, selectedCases)
219
220def die (msg):
221	print msg
222	sys.exit(-1)
223
224# Named filter lists
225FILTER_RULES = [
226	FilterRule("all",			"No filtering",											[]),
227	FilterRule("all-pass",		"All results must be 'Pass'",							[lambda l: all(l, lambda r: r == 'Pass')]),
228	FilterRule("any-pass",		"Any of results is 'Pass'",								[lambda l: any(l, lambda r: r == 'Pass')]),
229	FilterRule("any-fail",		"Any of results is not 'Pass' or 'NotSupported'",		[lambda l: not all(l, lambda r: r == 'Pass' or r == 'NotSupported')]),
230	FilterRule("prev-failing",	"Any except last result is failure",					[lambda l: l[-1] == 'Pass' and not all(l[:-1], lambda r: r == 'Pass')]),
231	FilterRule("prev-passing",	"Any except last result is 'Pass'",						[lambda l: l[-1] != 'Pass' and any(l[:-1], lambda r: r == 'Pass')])
232]
233
234if __name__ == "__main__":
235	parser = OptionParser(usage = "usage: %prog [options] [caselist] [result csv file]")
236	parser.add_option("-f", "--filter", dest="filter", default="all", help="filter rule name")
237	parser.add_option("-l", "--list", action="store_true", dest="list", default=False, help="list available rules")
238	parser.add_option("-n", "--num", dest="limit", default=0, help="limit number of cases")
239	parser.add_option("-s", "--seed", dest="seed", default=0, help="use selected seed for random selection")
240	parser.add_option("-g", "--groups", dest="groups_file", default=None, help="select cases based on group list file")
241
242	(options, args)	= parser.parse_args()
243
244	if options.list:
245		print "Available filter rules:"
246		for filter in FILTER_RULES:
247			print "  %s: %s" % (filter.name, filter.description)
248		sys.exit(0)
249
250	if len(args) == 0:
251		die("No input files specified")
252	elif len(args) > 2:
253		die("Too many arguments")
254
255	# Fetch filter
256	filter = getFilter(FILTER_RULES, options.filter)
257	if filter == None:
258		die("Unknown filter '%s'" % options.filter)
259
260	# Read case list
261	caselist = readCaseList(args[0])
262	if len(args) > 1:
263		results = readTestResults(args[1])
264		results = addResultsToCaseList(caselist, results)
265	else:
266		results = toResultList(caselist)
267
268	# Execute filters for results
269	for rule in filter.filters:
270		results = filterList(results, rule)
271
272	if options.limit != 0:
273		if options.groups_file != None:
274			groups = createGroups(results, readGroupList(options.groups_file))
275		else:
276			groups = createLeafGroups(results)
277		results = selectRandomSubset(results, groups, int(options.limit), int(options.seed))
278	elif options.groups_file != None:
279		groups = createGroups(results, readGroupList(options.groups_file))
280		results = selectCasesInGroups(results, groups)
281
282	# Print test set
283	for result in results:
284		print result.name
285