// Copyright 2017 syzkaller project authors. All rights reserved.
// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.
// syz-benchcmp visualizes syz-manager benchmarking results.
// First, run syz-manager with -bench=old flag.
// Then, do experimental modifications and run syz-manager again with -bench=new flag.
// Then, run syz-benchcmp old new.
package main
import (
"bufio"
"encoding/json"
"flag"
"fmt"
"html/template"
"io/ioutil"
"os"
"os/exec"
"path/filepath"
"sort"
)
var (
flagSkip = flag.Int("skip", -30, "skip that many seconds after start (skip first 20% by default)")
)
type Graph struct {
Name string
Headers []string
Points []Point
}
type Point struct {
Time uint64
Vals []uint64
}
func main() {
flag.Parse()
if len(flag.Args()) == 0 {
fmt.Fprintf(os.Stderr, "usage: syz-benchcmp [flags] bench_file0 [bench_file1 [bench_file2]]...\n")
flag.PrintDefaults()
os.Exit(1)
}
graphs := []*Graph{
{Name: "coverage"},
{Name: "corpus"},
{Name: "exec total"},
{Name: "crash types"},
}
for i, fname := range flag.Args() {
data := readFile(fname)
addExecSpeed(data)
for _, g := range graphs {
g.Headers = append(g.Headers, filepath.Base(fname))
for _, v := range data {
pt := Point{
Time: v["fuzzing"],
Vals: make([]uint64, len(flag.Args())),
}
pt.Vals[i] = v[g.Name]
g.Points = append(g.Points, pt)
}
}
}
for _, g := range graphs {
if len(g.Points) == 0 {
failf("no data points")
}
sort.Sort(pointSlice(g.Points))
skipStart(g)
restoreMissingPoints(g)
}
printFinalStats(graphs)
display(graphs)
}
func readFile(fname string) (data []map[string]uint64) {
f, err := os.Open(fname)
if err != nil {
failf("failed to open input file: %v", err)
}
defer f.Close()
dec := json.NewDecoder(bufio.NewReader(f))
for dec.More() {
v := make(map[string]uint64)
if err := dec.Decode(&v); err != nil {
failf("failed to decode input file %v: %v", fname, err)
}
data = append(data, v)
}
return
}
func addExecSpeed(data []map[string]uint64) {
// Speed between consecutive samples is very unstable.
const (
window = 100
step = 10
)
for i := window; i < len(data); i += step {
cur := data[i]
prev := data[i-window]
dx := cur["exec total"] - prev["exec total"]
dt := cur["fuzzing"] - prev["fuzzing"]
cur["exec speed"] = dx * 1000 / dt
}
}
func skipStart(g *Graph) {
skipTime := uint64(*flagSkip)
if *flagSkip < 0 {
// Negative skip means percents.
max := g.Points[len(g.Points)-1].Time
skipTime = max * -skipTime / 100
}
if skipTime > 0 {
skip := sort.Search(len(g.Points), func(i int) bool {
return g.Points[i].Time > skipTime
})
g.Points = g.Points[skip:]
}
}
func restoreMissingPoints(g *Graph) {
for i := range g.Headers {
// Find previous and next non-zero point for each zero point,
// and restore its value with linear inerpolation.
type Pt struct {
Time uint64
Val uint64
}
var prev Pt
prevs := make(map[uint64]Pt)
for _, pt := range g.Points {
if pt.Vals[i] != 0 {
prev = Pt{pt.Time, pt.Vals[i]}
continue
}
prevs[pt.Time] = prev
}
var next Pt
for pti := len(g.Points) - 1; pti >= 0; pti-- {
pt := g.Points[pti]
if pt.Vals[i] != 0 {
next = Pt{pt.Time, pt.Vals[i]}
continue
}
prev := prevs[pt.Time]
if prev.Val == 0 || next.Val == 0 {
continue
}
pt.Vals[i] = prev.Val
if next.Time != prev.Time {
// Use signed calculations as corpus can go backwards.
pt.Vals[i] += uint64(int64(next.Val-prev.Val) * int64(pt.Time-prev.Time) / int64(next.Time-prev.Time))
}
}
}
}
func printFinalStats(graphs []*Graph) {
for i := 1; i < len(graphs[0].Headers); i++ {
fmt.Printf("%-12v%16v%16v%16v\n", "", graphs[0].Headers[0], graphs[0].Headers[i], "diff")
for _, g := range graphs {
lastNonZero := func(x int) uint64 {
for j := len(g.Points) - 1; j >= 0; j-- {
if v := g.Points[j].Vals[x]; v != 0 {
return v
}
}
return 0
}
old := lastNonZero(0)
new := lastNonZero(i)
fmt.Printf("%-12v%16v%16v%+16d\n", g.Name, old, new, int64(new-old))
}
fmt.Printf("\n")
}
}
func display(graphs []*Graph) {
outf, err := ioutil.TempFile("", "")
if err != nil {
failf("failed to create temp file: %v", err)
}
if err := htmlTemplate.Execute(outf, graphs); err != nil {
failf("failed to execute template: %v", err)
}
outf.Close()
name := outf.Name() + ".html"
if err := os.Rename(outf.Name(), name); err != nil {
failf("failed to rename file: %v", err)
}
if err := exec.Command("xdg-open", name).Start(); err != nil {
failf("failed to start browser: %v", err)
}
}
type pointSlice []Point
func (a pointSlice) Len() int { return len(a) }
func (a pointSlice) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
func (a pointSlice) Less(i, j int) bool { return a[i].Time < a[j].Time }
func failf(msg string, args ...interface{}) {
fmt.Fprintf(os.Stderr, msg+"\n", args...)
os.Exit(1)
}
var htmlTemplate = template.Must(
template.New("").Parse(`
Syzkaller Bench
`))