1// Copyright 2019 The SwiftShader Authors. All Rights Reserved. 2// 3// Licensed under the Apache License, Version 2.0 (the "License"); 4// you may not use this file except in compliance with the License. 5// You may obtain a copy of the License at 6// 7// http://www.apache.org/licenses/LICENSE-2.0 8// 9// Unless required by applicable law or agreed to in writing, software 10// distributed under the License is distributed on an "AS IS" BASIS, 11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12// See the License for the specific language governing permissions and 13// limitations under the License. 14 15// Package testlist provides utilities for handling test lists. 16package testlist 17 18import ( 19 "bytes" 20 "crypto/sha1" 21 "encoding/gob" 22 "encoding/hex" 23 "encoding/json" 24 "io/ioutil" 25 "path/filepath" 26 "sort" 27 "strings" 28 29 "../cause" 30) 31 32// API is an enumerator of graphics APIs. 33type API string 34 35// Graphics APIs. 36const ( 37 EGL = API("egl") 38 GLES2 = API("gles2") 39 GLES3 = API("gles3") 40 Vulkan = API("vulkan") 41) 42 43// Group is a list of tests to be run for a single API. 44type Group struct { 45 Name string 46 File string 47 API API 48 Tests []string 49} 50 51// Load loads the test list file and appends all tests to the Group. 52func (g *Group) Load() error { 53 return g.LoadFile(g.File) 54} 55 56func (g *Group) LoadFile(file string) error { 57 dir, _ := filepath.Split(file) 58 tests, err := ioutil.ReadFile(file) 59 if err != nil { 60 return cause.Wrap(err, "Couldn't read '%s'", file) 61 } 62 for _, line := range strings.Split(string(tests), "\n") { 63 line = strings.TrimSpace(line) 64 // The test list file can contain references to other .txt files 65 // containing the individual tests. 66 if strings.HasSuffix(line, ".txt") { 67 g.LoadFile(filepath.Join(dir, line)) 68 } else if line != "" && !strings.HasPrefix(line, "#") { 69 g.Tests = append(g.Tests, line) 70 } 71 } 72 sort.Strings(g.Tests) 73 return nil 74} 75 76// Filter returns a new Group that contains only tests that match the predicate. 77func (g Group) Filter(pred func(string) bool) Group { 78 out := Group{ 79 Name: g.Name, 80 File: g.File, 81 API: g.API, 82 } 83 for _, test := range g.Tests { 84 if pred(test) { 85 out.Tests = append(out.Tests, test) 86 } 87 } 88 return out 89} 90 91// Limit returns a new Group that contains a maximum of limit tests. 92func (g Group) Limit(limit int) Group { 93 out := Group{ 94 Name: g.Name, 95 File: g.File, 96 API: g.API, 97 Tests: g.Tests, 98 } 99 if len(g.Tests) > limit { 100 out.Tests = g.Tests[:limit] 101 } 102 return out 103} 104 105// Lists is the full list of tests to be run. 106type Lists []Group 107 108// Filter returns a new Lists that contains only tests that match the predicate. 109func (l Lists) Filter(pred func(string) bool) Lists { 110 out := Lists{} 111 for _, group := range l { 112 filtered := group.Filter(pred) 113 if len(filtered.Tests) > 0 { 114 out = append(out, filtered) 115 } 116 } 117 return out 118} 119 120// Hash returns a SHA1 hash of the set of tests. 121func (l Lists) Hash() string { 122 h := sha1.New() 123 if err := gob.NewEncoder(h).Encode(l); err != nil { 124 panic(cause.Wrap(err, "Could not encode testlist to produce hash")) 125 } 126 return hex.EncodeToString(h.Sum(nil)) 127} 128 129// Load loads the test list json file and returns the full set of tests. 130func Load(root, jsonPath string) (Lists, error) { 131 root, err := filepath.Abs(root) 132 if err != nil { 133 return nil, cause.Wrap(err, "Couldn't get absolute path of '%s'", root) 134 } 135 136 jsonPath, err = filepath.Abs(jsonPath) 137 if err != nil { 138 return nil, cause.Wrap(err, "Couldn't get absolute path of '%s'", jsonPath) 139 } 140 141 i, err := ioutil.ReadFile(jsonPath) 142 if err != nil { 143 return nil, cause.Wrap(err, "Couldn't read test list from '%s'", jsonPath) 144 } 145 146 var jsonGroups []struct { 147 Name string 148 API string 149 TestFile string `json:"tests"` 150 } 151 if err := json.NewDecoder(bytes.NewReader(i)).Decode(&jsonGroups); err != nil { 152 return nil, cause.Wrap(err, "Couldn't parse '%s'", jsonPath) 153 } 154 155 dir := filepath.Dir(jsonPath) 156 157 out := make(Lists, len(jsonGroups)) 158 for i, jsonGroup := range jsonGroups { 159 group := Group{ 160 Name: jsonGroup.Name, 161 File: filepath.Join(dir, jsonGroup.TestFile), 162 API: API(jsonGroup.API), 163 } 164 if err := group.Load(); err != nil { 165 return nil, err 166 } 167 168 // Make the path relative before displaying it to the world. 169 relPath, err := filepath.Rel(root, group.File) 170 if err != nil { 171 return nil, cause.Wrap(err, "Couldn't get relative path for '%s'", group.File) 172 } 173 group.File = relPath 174 175 out[i] = group 176 } 177 178 return out, nil 179} 180 181// Status is an enumerator of test results. 182type Status string 183 184const ( 185 // Pass is the status of a successful test. 186 Pass = Status("PASS") 187 // Fail is the status of a failed test. 188 Fail = Status("FAIL") 189 // Timeout is the status of a test that failed to complete in the alloted 190 // time. 191 Timeout = Status("TIMEOUT") 192 // Crash is the status of a test that crashed. 193 Crash = Status("CRASH") 194 // Unimplemented is the status of a test that failed with UNIMPLEMENTED(). 195 Unimplemented = Status("UNIMPLEMENTED") 196 // Unsupported is the status of a test that failed with UNSUPPORTED(). 197 Unsupported = Status("UNSUPPORTED") 198 // Unreachable is the status of a test that failed with UNREACHABLE(). 199 Unreachable = Status("UNREACHABLE") 200 // Assert is the status of a test that failed with ASSERT() or ASSERT_MSG(). 201 Assert = Status("ASSERT") 202 // Abort is the status of a test that failed with ABORT(). 203 Abort = Status("ABORT") 204 // NotSupported is the status of a test feature not supported by the driver. 205 NotSupported = Status("NOT_SUPPORTED") 206 // CompatibilityWarning is the status passing test with a warning. 207 CompatibilityWarning = Status("COMPATIBILITY_WARNING") 208 // QualityWarning is the status passing test with a warning. 209 QualityWarning = Status("QUALITY_WARNING") 210 // InternalError is the status of a test that failed on an API usage error. 211 InternalError = Status("INTERNAL_ERROR") 212) 213 214// Statuses is the full list of status types 215var Statuses = []Status{ 216 Pass, 217 Fail, 218 Timeout, 219 Crash, 220 Unimplemented, 221 Unsupported, 222 Unreachable, 223 Assert, 224 Abort, 225 NotSupported, 226 CompatibilityWarning, 227 QualityWarning, 228 InternalError, 229} 230 231// Failing returns true if the task status requires fixing. 232func (s Status) Failing() bool { 233 switch s { 234 case Fail, Timeout, Crash, Unimplemented, Unreachable, Assert, Abort, InternalError: 235 return true 236 case Unsupported: 237 // This may seem surprising that this should be a failure, however these 238 // should not be reached, as dEQP should not be using features that are 239 // not advertised. 240 return true 241 default: 242 return false 243 } 244} 245 246// Passing returns true if the task status is considered a pass. 247func (s Status) Passing() bool { 248 switch s { 249 case Pass, CompatibilityWarning, QualityWarning: 250 return true 251 default: 252 return false 253 } 254} 255 256// FilePathWithStatus returns the path to the test list file with the status 257// appended before the file extension. 258func FilePathWithStatus(listPath string, status Status) string { 259 ext := filepath.Ext(listPath) 260 name := listPath[:len(listPath)-len(ext)] 261 return name + "-" + string(status) + ext 262} 263