xref: /external/swiftshader/tests/regres/deqp/deqp.go

// Copyright 2019 The SwiftShader Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package deqp provides functions for running dEQP, as well as loading and storing the results.
package deqp

import (
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"log"
	"math/rand"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"strconv"
	"strings"
	"sync"
	"time"

	"swiftshader.googlesource.com/SwiftShader/tests/regres/cov"
	"swiftshader.googlesource.com/SwiftShader/tests/regres/shell"
	"swiftshader.googlesource.com/SwiftShader/tests/regres/testlist"
	"swiftshader.googlesource.com/SwiftShader/tests/regres/util"
)

const dataVersion = 1

var (
	// Regular expression to parse the output of a dEQP test.
	deqpRE = regexp.MustCompile(`(Fail|Pass|NotSupported|CompatibilityWarning|QualityWarning|InternalError) \(([^\)]*)\)`)
	// Regular expression to parse a test that failed due to UNIMPLEMENTED()
	unimplementedRE = regexp.MustCompile(`[^\n]*UNIMPLEMENTED:[^\n]*`)
	// Regular expression to parse a test that failed due to UNSUPPORTED()
	unsupportedRE = regexp.MustCompile(`[^\n]*UNSUPPORTED:[^\n]*`)
	// Regular expression to parse a test that failed due to UNREACHABLE()
	unreachableRE = regexp.MustCompile(`[^\n]*UNREACHABLE:[^\n]*`)
	// Regular expression to parse a test that failed due to ASSERT()
	assertRE = regexp.MustCompile(`[^\n]*ASSERT\([^\)]*\)[^\n]*`)
	// Regular expression to parse a test that failed due to ABORT()
	abortRE = regexp.MustCompile(`[^\n]*ABORT:[^\n]*`)
	// Regular expression to parse individual test names and output
	caseOutputRE = regexp.MustCompile("Test case '([^']*)'..")
)

// Config contains the inputs required for running dEQP on a group of test lists.
type Config struct {
	ExeEgl           string
	ExeGles2         string
	ExeGles3         string
	ExeVulkan        string
	TempDir          string // Directory for temporary log files, coverage output.
	TestLists        testlist.Lists
	Env              []string
	LogReplacements  map[string]string
	NumParallelTests int
	MaxTestsPerProc  int
	CoverageEnv      *cov.Env
	TestTimeout      time.Duration
	ValidationLayer  bool
}

// Results holds the results of tests across all APIs.
// The Results structure may be serialized to cache results.
type Results struct {
	Version  int
	Error    string
	Tests    map[string]TestResult
	Coverage *cov.Tree
	Duration time.Duration
}

// TestResult holds the results of a single dEQP test.
type TestResult struct {
	Test      string
	Status    testlist.Status
	TimeTaken time.Duration
	Err       string `json:",omitempty"`
	Coverage  *cov.Coverage
}

func (r TestResult) String() string {
	if r.Err != "" {
		return fmt.Sprintf("%s: %s (%s)", r.Test, r.Status, r.Err)
	}
	return fmt.Sprintf("%s: %s", r.Test, r.Status)
}

// LoadResults loads cached test results from disk.
func LoadResults(path string) (*Results, error) {
	f, err := os.Open(path)
	if err != nil {
		return nil, fmt.Errorf("failed to open '%s' for loading test results: %w", path, err)
	}
	defer f.Close()

	var out Results
	if err := json.NewDecoder(f).Decode(&out); err != nil {
		return nil, err
	}
	if out.Version != dataVersion {
		return nil, errors.New("Data is from an old version")
	}
	return &out, nil
}

// Save saves (caches) test results to disk.
func (r *Results) Save(path string) error {
	if err := os.MkdirAll(filepath.Dir(path), 0777); err != nil {
		return fmt.Errorf("failed to make '%s' for saving test results: %w", filepath.Dir(path), err)
	}

	f, err := os.Create(path)
	if err != nil {
		return fmt.Errorf("failed to open '%s' for saving test results: %w", path, err)
	}
	defer f.Close()

	enc := json.NewEncoder(f)
	enc.SetIndent("", "  ")
	if err := enc.Encode(r); err != nil {
		return fmt.Errorf("failed to encode test results: %w", err)
	}

	return nil
}

// Run runs all the tests.
func (c *Config) Run() (*Results, error) {
	start := time.Now()

	if c.TempDir == "" {
		dir, err := ioutil.TempDir("", "deqp")
		if err != nil {
			return nil, fmt.Errorf("failed to generate temporary directory: %w", err)
		}
		c.TempDir = dir
	}

	// Wait group that completes once all the tests have finished.
	wg := sync.WaitGroup{}
	results := make(chan TestResult, 256)

	numTests := 0

	goroutineIndex := 0

	// For each API that we are testing
	for _, list := range c.TestLists {
		// Resolve the test runner
		exe, supportsCoverage := "", false

		switch list.API {
		case testlist.EGL:
			exe = c.ExeEgl
		case testlist.GLES2:
			exe = c.ExeGles2
		case testlist.GLES3:
			exe = c.ExeGles3
		case testlist.Vulkan:
			exe, supportsCoverage = c.ExeVulkan, true
		default:
			return nil, fmt.Errorf("Unknown API '%v'", list.API)
		}
		if !util.IsFile(exe) {
			return nil, fmt.Errorf("failed to find dEQP executable at '%s'", exe)
		}

		// Build a chan for the test names to be run.
		tests := make(chan string, len(list.Tests))

		numParallelTests := c.NumParallelTests
		if list.API != testlist.Vulkan {
			// OpenGL tests attempt to open lots of X11 display connections,
			// which may cause us to run out of handles. This maximum was
			// determined experimentally on a 72-core system.
			maxParallelGLTests := 16

			if numParallelTests > maxParallelGLTests {
				numParallelTests = maxParallelGLTests
			}
		}

		// Start a number of go routines to run the tests.
		wg.Add(numParallelTests)
		for i := 0; i < numParallelTests; i++ {
			go func(index int) {
				c.TestRoutine(exe, tests, results, index, supportsCoverage)
				wg.Done()
			}(goroutineIndex)
			goroutineIndex++
		}

		// Shuffle the test list.
		// This attempts to mix heavy-load tests with lighter ones.
		shuffled := make([]string, len(list.Tests))
		for i, j := range rand.New(rand.NewSource(42)).Perm(len(list.Tests)) {
			shuffled[i] = list.Tests[j]
		}

		// Hand the tests to the TestRoutines.
		for _, t := range shuffled {
			tests <- t
		}

		// Close the tests chan to indicate that there are no more tests to run.
		// The TestRoutine functions will return once all tests have been
		// run.
		close(tests)

		numTests += len(list.Tests)
	}

	out := Results{
		Version: dataVersion,
		Tests:   map[string]TestResult{},
	}

	if c.CoverageEnv != nil {
		out.Coverage = &cov.Tree{}
		out.Coverage.Add(cov.Path{}, c.CoverageEnv.AllSourceFiles())
	}

	// Collect the results.
	finished := make(chan struct{})
	lastUpdate := time.Now()
	go func() {
		start, i := time.Now(), 0
		for r := range results {
			i++
			if time.Since(lastUpdate) > time.Minute {
				lastUpdate = time.Now()
				remaining := numTests - i
				log.Printf("Ran %d/%d tests (%v%%). Estimated completion in %v.\n",
					i, numTests, util.Percent(i, numTests),
					(time.Since(start)/time.Duration(i))*time.Duration(remaining))
			}
			out.Tests[r.Test] = r
			if r.Coverage != nil {
				path := strings.Split(r.Test, ".")
				out.Coverage.Add(cov.Path(path), r.Coverage)
				r.Coverage = nil // Free memory
			}
		}
		close(finished)
	}()

	wg.Wait()      // Block until all the deqpTestRoutines have finished.
	close(results) // Signal no more results.
	<-finished     // And wait for the result collecting go-routine to finish.

	out.Duration = time.Since(start)

	return &out, nil
}

// TestRoutine repeatedly runs the dEQP test executable exe with the tests
// taken from tests. The output of the dEQP test is parsed, and the test result
// is written to results.
// TestRoutine only returns once the tests chan has been closed.
// TestRoutine does not close the results chan.
func (c *Config) TestRoutine(exe string, tests <-chan string, results chan<- TestResult, goroutineIndex int, supportsCoverage bool) {
	// Context for the GCOV_PREFIX environment variable:
	// If you compile SwiftShader with gcc and the --coverage flag, the build will contain coverage instrumentation.
	// We can use this to get the code coverage of SwiftShader from running dEQP.
	// The coverage instrumentation reads the existing coverage files on start-up (at a hardcoded path alongside the
	// SwiftShader build), updates coverage info as the programs runs, then (over)writes the coverage files on exit.
	// Thus, multiple parallel processes will race when updating coverage information. The GCOV_PREFIX environment
	// variable adds a prefix to the hardcoded paths.
	// E.g. Given GCOV_PREFIX=/tmp/coverage, the hardcoded path /ss/build/a.gcno becomes /tmp/coverage/ss/build/a.gcno.
	// This is mainly intended for running the target program on a different machine where the hardcoded paths don't
	// make sense. It can also be used to avoid races. It would be trivial to avoid races if the GCOV_PREFIX variable
	// supported macro variables like the Clang code coverage "%p" variable that expands to the process ID; in this
	// case, we could use GCOV_PREFIX=/tmp/coverage/%p to avoid races. Unfortunately, gcc does not support this.
	// Furthermore, processing coverage information from many directories can be slow; we start a lot of dEQP child
	// processes, each of which will likely get a unique process ID. In practice, we only need one directory per go
	// routine.

	// If GCOV_PREFIX is in Env, replace occurrences of "PROC_ID" in GCOV_PREFIX with goroutineIndex.
	// This avoids races between parallel child processes reading and writing coverage output files.
	// For example, GCOV_PREFIX="/tmp/gcov_output/PROC_ID" becomes GCOV_PREFIX="/tmp/gcov_output/1" in the first go routine.
	// You might expect PROC_ID to be the process ID of some process, but the only real requirement is that
	// it is a unique ID between the *parallel* child processes.
	env := make([]string, 0, len(c.Env))
	for _, v := range c.Env {
		if strings.HasPrefix(v, "GCOV_PREFIX=") {
			v = strings.ReplaceAll(v, "PROC_ID", strconv.Itoa(goroutineIndex))
		}
		env = append(env, v)
	}

	coverageFile := filepath.Join(c.TempDir, fmt.Sprintf("%v.profraw", goroutineIndex))
	if supportsCoverage {
		if c.CoverageEnv != nil {
			env = cov.AppendRuntimeEnv(env, coverageFile)
		}
	}

	logPath := "/dev/null" // TODO(bclayton): Try "nul" on windows.
	if !util.IsFile(logPath) {
		logPath = filepath.Join(c.TempDir, fmt.Sprintf("%v.log", goroutineIndex))
	}

	testNames := []string{}
	for name := range tests {
		testNames = append(testNames, name)
		if len(testNames) >= c.MaxTestsPerProc {
			c.PerformTests(exe, env, coverageFile, logPath, testNames, supportsCoverage, results)
			// Clear list of test names
			testNames = testNames[:0]
		}
	}
	if len(testNames) > 0 {
		c.PerformTests(exe, env, coverageFile, logPath, testNames, supportsCoverage, results)
	}
}

func (c *Config) PerformTests(exe string, env []string, coverageFile string, logPath string, testNames []string, supportsCoverage bool, results chan<- TestResult) {
	// log.Printf("Running test(s) '%s'\n", testNames)

	start := time.Now()
	// Set validation layer according to flag.
	validation := "disable"
	if c.ValidationLayer {
		validation = "enable"
	}

	// The list of test names will be passed to stdin, since the deqp-stdin-caselist option is used
	stdin := strings.Join(testNames, "\n") + "\n"

	numTests := len(testNames)
	timeout := c.TestTimeout * time.Duration(numTests)
	outRaw, deqpErr := shell.Exec(timeout, exe, filepath.Dir(exe), env, stdin,
		"--deqp-validation="+validation,
		"--deqp-surface-type=pbuffer",
		"--deqp-shadercache=disable",
		"--deqp-log-images=disable",
		"--deqp-log-shader-sources=disable",
		"--deqp-log-decompiled-spirv=disable",
		"--deqp-log-empty-loginfo=disable",
		"--deqp-log-flush=disable",
		"--deqp-log-filename="+logPath,
		"--deqp-stdin-caselist")
	duration := time.Since(start)
	out := string(outRaw)
	out = strings.ReplaceAll(out, exe, "<dEQP>")
	for k, v := range c.LogReplacements {
		out = strings.ReplaceAll(out, k, v)
	}

	var coverage *cov.Coverage
	if c.CoverageEnv != nil && supportsCoverage {
		var covErr error
		coverage, covErr = c.CoverageEnv.Import(coverageFile)
		if covErr != nil {
			log.Printf("Warning: Failed to process test coverage for test '%v'. %v", testNames, covErr)
		}
		os.Remove(coverageFile)
	}

	if numTests > 1 {
		// Separate output per test case
		caseOutputs := caseOutputRE.Split(out, -1)

		// If the output isn't as expected, a crash may have happened
		isCrash := (len(caseOutputs) != (numTests + 1))

		// Verify the exit code to see if a crash has happened
		var exitErr *exec.ExitError
		if errors.As(deqpErr, &exitErr) {
			if exitErr.ExitCode() == 255 {
				isCrash = true
			}
		}

		// If a crash has happened, re-run tests separately
		if isCrash {
			// Re-run tests one by one
			for _, testName := range testNames {
				singleTest := []string{testName}
				c.PerformTests(exe, env, coverageFile, logPath, singleTest, supportsCoverage, results)
			}
		} else {
			caseOutputs = caseOutputs[1:] // Ignore text up to first "Test case '...'"
			caseNameMatches := caseOutputRE.FindAllStringSubmatch(out, -1)
			caseNames := make([]string, len(caseNameMatches))
			for i, m := range caseNameMatches {
				caseNames[i] = m[1]
			}

			averageDuration := duration / time.Duration(numTests)
			for i, caseOutput := range caseOutputs {
				results <- c.AnalyzeOutput(caseNames[i], caseOutput, averageDuration, coverage, deqpErr)
			}
		}
	} else {
		results <- c.AnalyzeOutput(testNames[0], out, duration, coverage, deqpErr)
	}
}

func (c *Config) AnalyzeOutput(name string, out string, duration time.Duration, coverage *cov.Coverage, err error) TestResult {
	for _, test := range []struct {
		re *regexp.Regexp
		s  testlist.Status
	}{
		{unimplementedRE, testlist.Unimplemented},
		{unsupportedRE, testlist.Unsupported},
		{unreachableRE, testlist.Unreachable},
		{assertRE, testlist.Assert},
		{abortRE, testlist.Abort},
	} {
		if s := test.re.FindString(out); s != "" {
			return TestResult{
				Test:      name,
				Status:    test.s,
				TimeTaken: duration,
				Err:       s,
				Coverage:  coverage,
			}
		}
	}

	// Don't treat non-zero error codes as crashes.
	var exitErr *exec.ExitError
	if errors.As(err, &exitErr) {
		if exitErr.ExitCode() != 255 {
			out += fmt.Sprintf("\nProcess terminated with code %d", exitErr.ExitCode())
			err = nil
		}
	}

	switch err.(type) {
	default:
		return TestResult{
			Test:      name,
			Status:    testlist.Crash,
			TimeTaken: duration,
			Err:       out,
			Coverage:  coverage,
		}
	case shell.ErrTimeout:
		log.Printf("Timeout for test '%v'\n", name)
		return TestResult{
			Test:      name,
			Status:    testlist.Timeout,
			TimeTaken: duration,
			Coverage:  coverage,
		}
	case nil:
		toks := deqpRE.FindStringSubmatch(out)
		if len(toks) < 3 {
			err := fmt.Sprintf("Couldn't parse test '%v' output:\n%s", name, out)
			log.Println("Warning: ", err)
			return TestResult{Test: name, Status: testlist.Fail, Err: err, Coverage: coverage}
		}
		switch toks[1] {
		case "Pass":
			return TestResult{Test: name, Status: testlist.Pass, TimeTaken: duration, Coverage: coverage}
		case "NotSupported":
			return TestResult{Test: name, Status: testlist.NotSupported, TimeTaken: duration, Coverage: coverage}
		case "CompatibilityWarning":
			return TestResult{Test: name, Status: testlist.CompatibilityWarning, TimeTaken: duration, Coverage: coverage}
		case "QualityWarning":
			return TestResult{Test: name, Status: testlist.QualityWarning, TimeTaken: duration, Coverage: coverage}
		case "Fail":
			var err string
			if toks[2] != "Fail" {
				err = toks[2]
			}
			return TestResult{Test: name, Status: testlist.Fail, Err: err, TimeTaken: duration, Coverage: coverage}
		case "InternalError":
			var err string
			if toks[2] != "InternalError" {
				err = toks[2]
			}
			return TestResult{Test: name, Status: testlist.InternalError, Err: err, TimeTaken: duration, Coverage: coverage}
		default:
			err := fmt.Sprintf("Couldn't parse test output:\n%s", out)
			log.Println("Warning: ", err)
			return TestResult{Test: name, Status: testlist.Fail, Err: err, TimeTaken: duration, Coverage: coverage}
		}
	}
}