1// Copyright 2019 The SwiftShader Authors. All Rights Reserved. 2// 3// Licensed under the Apache License, Version 2.0 (the "License"); 4// you may not use this file except in compliance with the License. 5// You may obtain a copy of the License at 6// 7// http://www.apache.org/licenses/LICENSE-2.0 8// 9// Unless required by applicable law or agreed to in writing, software 10// distributed under the License is distributed on an "AS IS" BASIS, 11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12// See the License for the specific language governing permissions and 13// limitations under the License. 14 15// run_testlist is a tool runs a dEQP test list, using multiple sand-boxed 16// processes. 17// 18// Unlike simply running deqp with its --deqp-caselist-file flag, run_testlist 19// uses multiple sand-boxed processes, which greatly reduces testing time, and 20// gracefully handles crashing processes. 21package main 22 23import ( 24 "bytes" 25 "encoding/json" 26 "errors" 27 "flag" 28 "fmt" 29 "io/ioutil" 30 "log" 31 "math/rand" 32 "os" 33 "path/filepath" 34 "regexp" 35 "runtime" 36 "strings" 37 "time" 38 39 "swiftshader.googlesource.com/SwiftShader/tests/regres/cov" 40 "swiftshader.googlesource.com/SwiftShader/tests/regres/deqp" 41 "swiftshader.googlesource.com/SwiftShader/tests/regres/llvm" 42 "swiftshader.googlesource.com/SwiftShader/tests/regres/shell" 43 "swiftshader.googlesource.com/SwiftShader/tests/regres/testlist" 44 "swiftshader.googlesource.com/SwiftShader/tests/regres/util" 45) 46 47func min(a, b int) int { 48 if a < b { 49 return a 50 } 51 return b 52} 53 54var ( 55 deqpVkBinary = flag.String("deqp-vk", "deqp-vk", "path to the deqp-vk binary") 56 testList = flag.String("test-list", "vk-master-PASS.txt", "path to a test list file") 57 numThreads = flag.Int("num-threads", min(runtime.NumCPU(), 100), "number of parallel test runner processes") 58 maxTestsPerProc = flag.Int("max-tests-per-proc", 1, "maximum number of tests running in a single process") 59 maxProcMemory = flag.Uint64("max-proc-mem", shell.MaxProcMemory, "maximum virtual memory per child process") 60 output = flag.String("output", "results.json", "path to an output JSON results file") 61 filter = flag.String("filter", "", "filter for test names. Start with a '/' to indicate regex") 62 limit = flag.Int("limit", 0, "only run a maximum of this number of tests") 63 shuffle = flag.Bool("shuffle", false, "shuffle tests") 64 noResults = flag.Bool("no-results", false, "disable generation of results.json file") 65 genCoverage = flag.Bool("coverage", false, "generate test coverage") 66 enableValidation = flag.Bool("validation", false, "run deqp-vk with Vulkan validation layers") 67) 68 69const testTimeout = time.Minute * 2 70 71func run() error { 72 group := testlist.Group{ 73 Name: "", 74 File: *testList, 75 API: testlist.Vulkan, 76 } 77 if err := group.Load(); err != nil { 78 return err 79 } 80 81 if *filter != "" { 82 if strings.HasPrefix(*filter, "/") { 83 re := regexp.MustCompile((*filter)[1:]) 84 group = group.Filter(re.MatchString) 85 } else { 86 group = group.Filter(func(name string) bool { 87 ok, _ := filepath.Match(*filter, name) 88 return ok 89 }) 90 } 91 } 92 93 shell.MaxProcMemory = *maxProcMemory 94 95 if *shuffle { 96 rnd := rand.New(rand.NewSource(time.Now().UnixNano())) 97 rnd.Shuffle(len(group.Tests), func(i, j int) { group.Tests[i], group.Tests[j] = group.Tests[j], group.Tests[i] }) 98 } 99 100 if *limit != 0 && len(group.Tests) > *limit { 101 group.Tests = group.Tests[:*limit] 102 } 103 104 log.Printf("Running %d tests...\n", len(group.Tests)) 105 106 config := deqp.Config{ 107 ExeEgl: "", 108 ExeGles2: "", 109 ExeGles3: "", 110 ExeVulkan: *deqpVkBinary, 111 Env: os.Environ(), 112 NumParallelTests: *numThreads, 113 MaxTestsPerProc: *maxTestsPerProc, 114 TestLists: testlist.Lists{group}, 115 TestTimeout: testTimeout, 116 ValidationLayer: *enableValidation, 117 } 118 119 if *genCoverage { 120 icdPath := findSwiftshaderICD() 121 t := findToolchain(icdPath) 122 config.CoverageEnv = &cov.Env{ 123 LLVM: t.llvm, 124 TurboCov: t.turbocov, 125 RootDir: projectRootDir(), 126 ExePath: findSwiftshaderSO(icdPath), 127 } 128 } 129 130 res, err := config.Run() 131 if err != nil { 132 return err 133 } 134 135 counts := map[testlist.Status]int{} 136 for _, r := range res.Tests { 137 counts[r.Status] = counts[r.Status] + 1 138 } 139 for _, s := range testlist.Statuses { 140 if count := counts[s]; count > 0 { 141 log.Printf("%s: %d\n", string(s), count) 142 } 143 } 144 145 if *genCoverage { 146 f, err := os.Create("coverage.dat") 147 if err != nil { 148 return fmt.Errorf("failed to open coverage.dat file: %w", err) 149 } 150 if err := res.Coverage.Encode("master", f); err != nil { 151 return fmt.Errorf("failed to encode coverage data: %w", err) 152 } 153 } 154 155 if !*noResults { 156 err = res.Save(*output) 157 if err != nil { 158 return err 159 } 160 } 161 162 return nil 163} 164 165func findSwiftshaderICD() string { 166 icdPaths := strings.Split(os.Getenv("VK_ICD_FILENAMES"), ";") 167 for _, icdPath := range icdPaths { 168 _, file := filepath.Split(icdPath) 169 if file == "vk_swiftshader_icd.json" { 170 return icdPath 171 } 172 } 173 panic("Cannot find vk_swiftshader_icd.json in VK_ICD_FILENAMES") 174} 175 176func findSwiftshaderSO(vkSwiftshaderICD string) string { 177 root := struct { 178 ICD struct { 179 Path string `json:"library_path"` 180 } 181 }{} 182 183 icd, err := ioutil.ReadFile(vkSwiftshaderICD) 184 if err != nil { 185 panic(fmt.Errorf("Could not read '%v'. %v", vkSwiftshaderICD, err)) 186 } 187 188 if err := json.NewDecoder(bytes.NewReader(icd)).Decode(&root); err != nil { 189 panic(fmt.Errorf("Could not parse '%v'. %v", vkSwiftshaderICD, err)) 190 } 191 192 if util.IsFile(root.ICD.Path) { 193 return root.ICD.Path 194 } 195 dir := filepath.Dir(vkSwiftshaderICD) 196 path, err := filepath.Abs(filepath.Join(dir, root.ICD.Path)) 197 if err != nil { 198 panic(fmt.Errorf("Could not locate ICD so at '%v'. %v", root.ICD.Path, err)) 199 } 200 201 return path 202} 203 204type toolchain struct { 205 llvm llvm.Toolchain 206 turbocov string 207} 208 209func findToolchain(vkSwiftshaderICD string) toolchain { 210 minVersion := llvm.Version{Major: 7} 211 212 // Try finding the llvm toolchain via the CMake generated 213 // coverage-toolchain.txt file that sits next to vk_swiftshader_icd.json. 214 dir := filepath.Dir(vkSwiftshaderICD) 215 toolchainInfoPath := filepath.Join(dir, "coverage-toolchain.txt") 216 if util.IsFile(toolchainInfoPath) { 217 if file, err := os.Open(toolchainInfoPath); err == nil { 218 defer file.Close() 219 content := struct { 220 LLVM string `json:"llvm"` 221 TurboCov string `json:"turbo-cov"` 222 }{} 223 err := json.NewDecoder(file).Decode(&content) 224 if err != nil { 225 log.Fatalf("Couldn't read 'toolchainInfoPath': %v", err) 226 } 227 if t := llvm.Search(content.LLVM).FindAtLeast(minVersion); t != nil { 228 return toolchain{*t, content.TurboCov} 229 } 230 } 231 } 232 233 // Fallback, try searching PATH. 234 if t := llvm.Search().FindAtLeast(minVersion); t != nil { 235 return toolchain{*t, ""} 236 } 237 238 log.Fatal("Could not find LLVM toolchain") 239 return toolchain{} 240} 241 242func projectRootDir() string { 243 _, thisFile, _, _ := runtime.Caller(1) 244 thisDir := filepath.Dir(thisFile) 245 root, err := filepath.Abs(filepath.Join(thisDir, "../../../..")) 246 if err != nil { 247 panic(err) 248 } 249 return root 250} 251 252func main() { 253 flag.ErrHelp = errors.New("regres is a tool to detect regressions between versions of SwiftShader") 254 flag.Parse() 255 if err := run(); err != nil { 256 _, _ = fmt.Fprintln(os.Stderr, err) 257 os.Exit(-1) 258 } 259} 260