• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1/* Copyright (c) 2015, Google Inc.
2 *
3 * Permission to use, copy, modify, and/or distribute this software for any
4 * purpose with or without fee is hereby granted, provided that the above
5 * copyright notice and this permission notice appear in all copies.
6 *
7 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
8 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
9 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
10 * SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
11 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION
12 * OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
13 * CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */
14
15package main
16
17import (
18	"bytes"
19	"encoding/json"
20	"flag"
21	"fmt"
22	"os"
23	"os/exec"
24	"path"
25	"strconv"
26	"strings"
27	"syscall"
28	"time"
29)
30
31// TODO(davidben): Link tests with the malloc shim and port -malloc-test to this runner.
32
33var (
34	useValgrind     = flag.Bool("valgrind", false, "If true, run code under valgrind")
35	useGDB          = flag.Bool("gdb", false, "If true, run BoringSSL code under gdb")
36	buildDir        = flag.String("build-dir", "build", "The build directory to run the tests from.")
37	jsonOutput      = flag.String("json-output", "", "The file to output JSON results to.")
38	mallocTest      = flag.Int64("malloc-test", -1, "If non-negative, run each test with each malloc in turn failing from the given number onwards.")
39	mallocTestDebug = flag.Bool("malloc-test-debug", false, "If true, ask each test to abort rather than fail a malloc. This can be used with a specific value for --malloc-test to identity the malloc failing that is causing problems.")
40)
41
42type test []string
43
44// testOutput is a representation of Chromium's JSON test result format. See
45// https://www.chromium.org/developers/the-json-test-results-format
46type testOutput struct {
47	Version           int                   `json:"version"`
48	Interrupted       bool                  `json:"interrupted"`
49	PathDelimiter     string                `json:"path_delimiter"`
50	SecondsSinceEpoch float64               `json:"seconds_since_epoch"`
51	NumFailuresByType map[string]int        `json:"num_failures_by_type"`
52	Tests             map[string]testResult `json:"tests"`
53}
54
55type testResult struct {
56	Actual       string `json:"actual"`
57	Expected     string `json:"expected"`
58	IsUnexpected bool   `json:"is_unexpected"`
59}
60
61func newTestOutput() *testOutput {
62	return &testOutput{
63		Version:           3,
64		PathDelimiter:     ".",
65		SecondsSinceEpoch: float64(time.Now().UnixNano()) / float64(time.Second/time.Nanosecond),
66		NumFailuresByType: make(map[string]int),
67		Tests:             make(map[string]testResult),
68	}
69}
70
71func (t *testOutput) addResult(name, result string) {
72	if _, found := t.Tests[name]; found {
73		panic(name)
74	}
75	t.Tests[name] = testResult{
76		Actual:       result,
77		Expected:     "PASS",
78		IsUnexpected: result != "PASS",
79	}
80	t.NumFailuresByType[result]++
81}
82
83func (t *testOutput) writeTo(name string) error {
84	file, err := os.Create(name)
85	if err != nil {
86		return err
87	}
88	defer file.Close()
89	out, err := json.MarshalIndent(t, "", "  ")
90	if err != nil {
91		return err
92	}
93	_, err = file.Write(out)
94	return err
95}
96
97func valgrindOf(dbAttach bool, path string, args ...string) *exec.Cmd {
98	valgrindArgs := []string{"--error-exitcode=99", "--track-origins=yes", "--leak-check=full"}
99	if dbAttach {
100		valgrindArgs = append(valgrindArgs, "--db-attach=yes", "--db-command=xterm -e gdb -nw %f %p")
101	}
102	valgrindArgs = append(valgrindArgs, path)
103	valgrindArgs = append(valgrindArgs, args...)
104
105	return exec.Command("valgrind", valgrindArgs...)
106}
107
108func gdbOf(path string, args ...string) *exec.Cmd {
109	xtermArgs := []string{"-e", "gdb", "--args"}
110	xtermArgs = append(xtermArgs, path)
111	xtermArgs = append(xtermArgs, args...)
112
113	return exec.Command("xterm", xtermArgs...)
114}
115
116type moreMallocsError struct{}
117
118func (moreMallocsError) Error() string {
119	return "child process did not exhaust all allocation calls"
120}
121
122var errMoreMallocs = moreMallocsError{}
123
124func runTestOnce(test test, mallocNumToFail int64) (passed bool, err error) {
125	prog := path.Join(*buildDir, test[0])
126	args := test[1:]
127	var cmd *exec.Cmd
128	if *useValgrind {
129		cmd = valgrindOf(false, prog, args...)
130	} else if *useGDB {
131		cmd = gdbOf(prog, args...)
132	} else {
133		cmd = exec.Command(prog, args...)
134	}
135	var stdoutBuf bytes.Buffer
136	var stderrBuf bytes.Buffer
137	cmd.Stdout = &stdoutBuf
138	cmd.Stderr = &stderrBuf
139	if mallocNumToFail >= 0 {
140		cmd.Env = os.Environ()
141		cmd.Env = append(cmd.Env, "MALLOC_NUMBER_TO_FAIL="+strconv.FormatInt(mallocNumToFail, 10))
142		if *mallocTestDebug {
143			cmd.Env = append(cmd.Env, "MALLOC_ABORT_ON_FAIL=1")
144		}
145		cmd.Env = append(cmd.Env, "_MALLOC_CHECK=1")
146	}
147
148	if err := cmd.Start(); err != nil {
149		return false, err
150	}
151	if err := cmd.Wait(); err != nil {
152		if exitError, ok := err.(*exec.ExitError); ok {
153			if exitError.Sys().(syscall.WaitStatus).ExitStatus() == 88 {
154				return false, errMoreMallocs
155			}
156		}
157		fmt.Print(string(stderrBuf.Bytes()))
158		return false, err
159	}
160	fmt.Print(string(stderrBuf.Bytes()))
161
162	// Account for Windows line-endings.
163	stdout := bytes.Replace(stdoutBuf.Bytes(), []byte("\r\n"), []byte("\n"), -1)
164
165	if bytes.HasSuffix(stdout, []byte("PASS\n")) &&
166		(len(stdout) == 5 || stdout[len(stdout)-6] == '\n') {
167		return true, nil
168	}
169	return false, nil
170}
171
172func runTest(test test) (bool, error) {
173	if *mallocTest < 0 {
174		return runTestOnce(test, -1)
175	}
176
177	for mallocNumToFail := int64(*mallocTest); ; mallocNumToFail++ {
178		if passed, err := runTestOnce(test, mallocNumToFail); err != errMoreMallocs {
179			if err != nil {
180				err = fmt.Errorf("at malloc %d: %s", mallocNumToFail, err)
181			}
182			return passed, err
183		}
184	}
185}
186
187// shortTestName returns the short name of a test. Except for evp_test, it
188// assumes that any argument which ends in .txt is a path to a data file and not
189// relevant to the test's uniqueness.
190func shortTestName(test test) string {
191	var args []string
192	for _, arg := range test {
193		if test[0] == "crypto/evp/evp_test" || !strings.HasSuffix(arg, ".txt") {
194			args = append(args, arg)
195		}
196	}
197	return strings.Join(args, " ")
198}
199
200// setWorkingDirectory walks up directories as needed until the current working
201// directory is the top of a BoringSSL checkout.
202func setWorkingDirectory() {
203	for i := 0; i < 64; i++ {
204		if _, err := os.Stat("BUILDING.md"); err == nil {
205			return
206		}
207		os.Chdir("..")
208	}
209
210	panic("Couldn't find BUILDING.md in a parent directory!")
211}
212
213func parseTestConfig(filename string) ([]test, error) {
214	in, err := os.Open(filename)
215	if err != nil {
216		return nil, err
217	}
218	defer in.Close()
219
220	decoder := json.NewDecoder(in)
221	var result []test
222	if err := decoder.Decode(&result); err != nil {
223		return nil, err
224	}
225	return result, nil
226}
227
228func main() {
229	flag.Parse()
230	setWorkingDirectory()
231
232	tests, err := parseTestConfig("util/all_tests.json")
233	if err != nil {
234		fmt.Printf("Failed to parse input: %s\n", err)
235		os.Exit(1)
236	}
237
238	testOutput := newTestOutput()
239	var failed []test
240	for _, test := range tests {
241		fmt.Printf("%s\n", strings.Join([]string(test), " "))
242
243		name := shortTestName(test)
244		passed, err := runTest(test)
245		if err != nil {
246			fmt.Printf("%s failed to complete: %s\n", test[0], err)
247			failed = append(failed, test)
248			testOutput.addResult(name, "CRASHED")
249		} else if !passed {
250			fmt.Printf("%s failed to print PASS on the last line.\n", test[0])
251			failed = append(failed, test)
252			testOutput.addResult(name, "FAIL")
253		} else {
254			testOutput.addResult(name, "PASS")
255		}
256	}
257
258	if *jsonOutput != "" {
259		if err := testOutput.writeTo(*jsonOutput); err != nil {
260			fmt.Fprintf(os.Stderr, "Error: %s\n", err)
261		}
262	}
263
264	if len(failed) > 0 {
265		fmt.Printf("\n%d of %d tests failed:\n", len(failed), len(tests))
266		for _, test := range failed {
267			fmt.Printf("\t%s\n", strings.Join([]string(test), " "))
268		}
269		os.Exit(1)
270	}
271
272	fmt.Printf("\nAll tests passed!\n")
273}
274