• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1// Copyright 2019 The SwiftShader Authors. All Rights Reserved.
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//    http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15// Package testlist provides utilities for handling test lists.
16package testlist
17
18import (
19	"bytes"
20	"crypto/sha1"
21	"encoding/gob"
22	"encoding/hex"
23	"encoding/json"
24	"io/ioutil"
25	"path/filepath"
26	"sort"
27	"strings"
28
29	"../cause"
30)
31
32// API is an enumerator of graphics APIs.
33type API string
34
35// Graphics APIs.
36const (
37	EGL    = API("egl")
38	GLES2  = API("gles2")
39	GLES3  = API("gles3")
40	Vulkan = API("vulkan")
41)
42
43// Group is a list of tests to be run for a single API.
44type Group struct {
45	Name  string
46	File  string
47	API   API
48	Tests []string
49}
50
51// Load loads the test list file and appends all tests to the Group.
52func (g *Group) Load() error {
53	return g.LoadFile(g.File)
54}
55
56func (g *Group) LoadFile(file string) error {
57	dir, _ := filepath.Split(file)
58	tests, err := ioutil.ReadFile(file)
59	if err != nil {
60		return cause.Wrap(err, "Couldn't read '%s'", file)
61	}
62	for _, line := range strings.Split(string(tests), "\n") {
63		line = strings.TrimSpace(line)
64		// The test list file can contain references to other .txt files
65		// containing the individual tests.
66		if strings.HasSuffix(line, ".txt") {
67			g.LoadFile(filepath.Join(dir, line))
68		} else if line != "" && !strings.HasPrefix(line, "#") {
69			g.Tests = append(g.Tests, line)
70		}
71	}
72	sort.Strings(g.Tests)
73	return nil
74}
75
76// Filter returns a new Group that contains only tests that match the predicate.
77func (g Group) Filter(pred func(string) bool) Group {
78	out := Group{
79		Name: g.Name,
80		File: g.File,
81		API:  g.API,
82	}
83	for _, test := range g.Tests {
84		if pred(test) {
85			out.Tests = append(out.Tests, test)
86		}
87	}
88	return out
89}
90
91// Limit returns a new Group that contains a maximum of limit tests.
92func (g Group) Limit(limit int) Group {
93	out := Group{
94		Name:  g.Name,
95		File:  g.File,
96		API:   g.API,
97		Tests: g.Tests,
98	}
99	if len(g.Tests) > limit {
100		out.Tests = g.Tests[:limit]
101	}
102	return out
103}
104
105// Lists is the full list of tests to be run.
106type Lists []Group
107
108// Filter returns a new Lists that contains only tests that match the predicate.
109func (l Lists) Filter(pred func(string) bool) Lists {
110	out := Lists{}
111	for _, group := range l {
112		filtered := group.Filter(pred)
113		if len(filtered.Tests) > 0 {
114			out = append(out, filtered)
115		}
116	}
117	return out
118}
119
120// Hash returns a SHA1 hash of the set of tests.
121func (l Lists) Hash() string {
122	h := sha1.New()
123	if err := gob.NewEncoder(h).Encode(l); err != nil {
124		panic(cause.Wrap(err, "Could not encode testlist to produce hash"))
125	}
126	return hex.EncodeToString(h.Sum(nil))
127}
128
129// Load loads the test list json file and returns the full set of tests.
130func Load(root, jsonPath string) (Lists, error) {
131	root, err := filepath.Abs(root)
132	if err != nil {
133		return nil, cause.Wrap(err, "Couldn't get absolute path of '%s'", root)
134	}
135
136	jsonPath, err = filepath.Abs(jsonPath)
137	if err != nil {
138		return nil, cause.Wrap(err, "Couldn't get absolute path of '%s'", jsonPath)
139	}
140
141	i, err := ioutil.ReadFile(jsonPath)
142	if err != nil {
143		return nil, cause.Wrap(err, "Couldn't read test list from '%s'", jsonPath)
144	}
145
146	var jsonGroups []struct {
147		Name     string
148		API      string
149		TestFile string `json:"tests"`
150	}
151	if err := json.NewDecoder(bytes.NewReader(i)).Decode(&jsonGroups); err != nil {
152		return nil, cause.Wrap(err, "Couldn't parse '%s'", jsonPath)
153	}
154
155	dir := filepath.Dir(jsonPath)
156
157	out := make(Lists, len(jsonGroups))
158	for i, jsonGroup := range jsonGroups {
159		group := Group{
160			Name: jsonGroup.Name,
161			File: filepath.Join(dir, jsonGroup.TestFile),
162			API:  API(jsonGroup.API),
163		}
164		if err := group.Load(); err != nil {
165			return nil, err
166		}
167
168		// Make the path relative before displaying it to the world.
169		relPath, err := filepath.Rel(root, group.File)
170		if err != nil {
171			return nil, cause.Wrap(err, "Couldn't get relative path for '%s'", group.File)
172		}
173		group.File = relPath
174
175		out[i] = group
176	}
177
178	return out, nil
179}
180
181// Status is an enumerator of test results.
182type Status string
183
184const (
185	// Pass is the status of a successful test.
186	Pass = Status("PASS")
187	// Fail is the status of a failed test.
188	Fail = Status("FAIL")
189	// Timeout is the status of a test that failed to complete in the alloted
190	// time.
191	Timeout = Status("TIMEOUT")
192	// Crash is the status of a test that crashed.
193	Crash = Status("CRASH")
194	// Unimplemented is the status of a test that failed with UNIMPLEMENTED().
195	Unimplemented = Status("UNIMPLEMENTED")
196	// Unsupported is the status of a test that failed with UNSUPPORTED().
197	Unsupported = Status("UNSUPPORTED")
198	// Unreachable is the status of a test that failed with UNREACHABLE().
199	Unreachable = Status("UNREACHABLE")
200	// Assert is the status of a test that failed with ASSERT() or ASSERT_MSG().
201	Assert = Status("ASSERT")
202	// Abort is the status of a test that failed with ABORT().
203	Abort = Status("ABORT")
204	// NotSupported is the status of a test feature not supported by the driver.
205	NotSupported = Status("NOT_SUPPORTED")
206	// CompatibilityWarning is the status passing test with a warning.
207	CompatibilityWarning = Status("COMPATIBILITY_WARNING")
208	// QualityWarning is the status passing test with a warning.
209	QualityWarning = Status("QUALITY_WARNING")
210	// InternalError is the status of a test that failed on an API usage error.
211	InternalError = Status("INTERNAL_ERROR")
212)
213
214// Statuses is the full list of status types
215var Statuses = []Status{
216	Pass,
217	Fail,
218	Timeout,
219	Crash,
220	Unimplemented,
221	Unsupported,
222	Unreachable,
223	Assert,
224	Abort,
225	NotSupported,
226	CompatibilityWarning,
227	QualityWarning,
228	InternalError,
229}
230
231// Failing returns true if the task status requires fixing.
232func (s Status) Failing() bool {
233	switch s {
234	case Fail, Timeout, Crash, Unimplemented, Unreachable, Assert, Abort, InternalError:
235		return true
236	case Unsupported:
237		// This may seem surprising that this should be a failure, however these
238		// should not be reached, as dEQP should not be using features that are
239		// not advertised.
240		return true
241	default:
242		return false
243	}
244}
245
246// Passing returns true if the task status is considered a pass.
247func (s Status) Passing() bool {
248	switch s {
249	case Pass, CompatibilityWarning, QualityWarning:
250		return true
251	default:
252		return false
253	}
254}
255
256// FilePathWithStatus returns the path to the test list file with the status
257// appended before the file extension.
258func FilePathWithStatus(listPath string, status Status) string {
259	ext := filepath.Ext(listPath)
260	name := listPath[:len(listPath)-len(ext)]
261	return name + "-" + string(status) + ext
262}
263