// Copyright 2018 syzkaller project authors. All rights reserved. // Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file. package bisect import ( "fmt" "io" "path/filepath" "time" "github.com/google/syzkaller/pkg/build" "github.com/google/syzkaller/pkg/instance" "github.com/google/syzkaller/pkg/mgrconfig" "github.com/google/syzkaller/pkg/osutil" "github.com/google/syzkaller/pkg/vcs" ) type Config struct { Trace io.Writer Fix bool BinDir string DebugDir string Kernel KernelConfig Syzkaller SyzkallerConfig Repro ReproConfig Manager mgrconfig.Config } type KernelConfig struct { Repo string Branch string Commit string Cmdline string Sysctl string Config []byte Userspace string } type SyzkallerConfig struct { Repo string Commit string Descriptions string } type ReproConfig struct { Opts []byte Syz []byte C []byte } type env struct { cfg *Config repo vcs.Repo head *vcs.Commit inst *instance.Env numTests int buildTime time.Duration testTime time.Duration } type buildEnv struct { compiler string } func Run(cfg *Config) (*vcs.Commit, error) { repo, err := vcs.NewRepo(cfg.Manager.TargetOS, cfg.Manager.Type, cfg.Manager.KernelSrc) if err != nil { return nil, err } env := &env{ cfg: cfg, repo: repo, } if cfg.Fix { env.log("searching for fixing commit since %v", cfg.Kernel.Commit) } else { env.log("searching for guilty commit starting from %v", cfg.Kernel.Commit) } start := time.Now() res, err := env.bisect() env.log("revisions tested: %v, total time: %v (build: %v, test: %v)", env.numTests, time.Since(start), env.buildTime, env.testTime) if err != nil { env.log("error: %v", err) return nil, err } if res == nil { env.log("the crash is still unfixed") return nil, nil } what := "bad" if cfg.Fix { what = "good" } env.log("first %v commit: %v %v", what, res.Hash, res.Title) env.log("cc: %q", res.CC) return res, nil } func (env *env) bisect() (*vcs.Commit, error) { cfg := env.cfg var err error if env.inst, err = instance.NewEnv(&cfg.Manager); err != nil { return nil, err } if env.head, err = env.repo.Poll(cfg.Kernel.Repo, cfg.Kernel.Branch); err != nil { return nil, err } if err := build.Clean(cfg.Manager.TargetOS, cfg.Manager.TargetVMArch, cfg.Manager.Type, cfg.Manager.KernelSrc); err != nil { return nil, fmt.Errorf("kernel clean failed: %v", err) } env.log("building syzkaller on %v", cfg.Syzkaller.Commit) if err := env.inst.BuildSyzkaller(cfg.Syzkaller.Repo, cfg.Syzkaller.Commit); err != nil { return nil, err } if _, err := env.repo.SwitchCommit(cfg.Kernel.Commit); err != nil { return nil, err } if res, err := env.test(); err != nil { return nil, err } else if res != vcs.BisectBad { return nil, fmt.Errorf("the crash wasn't reproduced on the original commit") } res, bad, good, err := env.commitRange() if err != nil { return nil, err } if res != nil { return res, nil // happens on the oldest release } if good == "" { return nil, nil // still not fixed } return env.repo.Bisect(bad, good, cfg.Trace, func() (vcs.BisectResult, error) { res, err := env.test() if cfg.Fix { if res == vcs.BisectBad { res = vcs.BisectGood } else if res == vcs.BisectGood { res = vcs.BisectBad } } return res, err }) } func (env *env) commitRange() (*vcs.Commit, string, string, error) { if env.cfg.Fix { return env.commitRangeForFix() } return env.commitRangeForBug() } func (env *env) commitRangeForFix() (*vcs.Commit, string, string, error) { env.log("testing current HEAD %v", env.head.Hash) if _, err := env.repo.SwitchCommit(env.head.Hash); err != nil { return nil, "", "", err } res, err := env.test() if err != nil { return nil, "", "", err } if res != vcs.BisectGood { return nil, "", "", nil } return nil, env.head.Hash, env.cfg.Kernel.Commit, nil } func (env *env) commitRangeForBug() (*vcs.Commit, string, string, error) { cfg := env.cfg tags, err := env.repo.PreviousReleaseTags(cfg.Kernel.Commit) if err != nil { return nil, "", "", err } for i, tag := range tags { if tag == "v3.8" { // v3.8 does not work with modern perl, and as we go further in history // make stops to work, then binutils, glibc, etc. So we stop at v3.8. // Up to that point we only need an ancient gcc. tags = tags[:i] break } } if len(tags) == 0 { return nil, "", "", fmt.Errorf("no release tags before this commit") } lastBad := cfg.Kernel.Commit for i, tag := range tags { env.log("testing release %v", tag) commit, err := env.repo.SwitchCommit(tag) if err != nil { return nil, "", "", err } res, err := env.test() if err != nil { return nil, "", "", err } if res == vcs.BisectGood { return nil, lastBad, tag, nil } if res == vcs.BisectBad { lastBad = tag } if i == len(tags)-1 { return commit, "", "", nil } } panic("unreachable") } func (env *env) test() (vcs.BisectResult, error) { cfg := env.cfg env.numTests++ current, err := env.repo.HeadCommit() if err != nil { return 0, err } be, err := env.buildEnvForCommit(current.Hash) if err != nil { return 0, err } compilerID, err := build.CompilerIdentity(be.compiler) if err != nil { return 0, err } env.log("testing commit %v with %v", current.Hash, compilerID) buildStart := time.Now() if err := build.Clean(cfg.Manager.TargetOS, cfg.Manager.TargetVMArch, cfg.Manager.Type, cfg.Manager.KernelSrc); err != nil { return 0, fmt.Errorf("kernel clean failed: %v", err) } err = env.inst.BuildKernel(be.compiler, cfg.Kernel.Userspace, cfg.Kernel.Cmdline, cfg.Kernel.Sysctl, cfg.Kernel.Config) env.buildTime += time.Since(buildStart) if err != nil { if verr, ok := err.(*osutil.VerboseError); ok { env.log("%v", verr.Title) env.saveDebugFile(current.Hash, 0, verr.Output) } else { env.log("%v", err) } return vcs.BisectSkip, nil } testStart := time.Now() results, err := env.inst.Test(8, cfg.Repro.Syz, cfg.Repro.Opts, cfg.Repro.C) env.testTime += time.Since(testStart) if err != nil { env.log("failed: %v", err) return vcs.BisectSkip, nil } bad, good := env.processResults(current, results) res := vcs.BisectSkip if bad != 0 { res = vcs.BisectBad } else if good != 0 { res = vcs.BisectGood } return res, nil } func (env *env) processResults(current *vcs.Commit, results []error) (bad, good int) { var verdicts []string for i, res := range results { if res == nil { good++ verdicts = append(verdicts, "OK") continue } switch err := res.(type) { case *instance.TestError: if err.Boot { verdicts = append(verdicts, fmt.Sprintf("boot failed: %v", err)) } else { verdicts = append(verdicts, fmt.Sprintf("basic kernel testing failed: %v", err)) } output := err.Output if err.Report != nil { output = err.Report.Output } env.saveDebugFile(current.Hash, i, output) case *instance.CrashError: bad++ verdicts = append(verdicts, fmt.Sprintf("crashed: %v", err)) output := err.Report.Report if len(output) == 0 { output = err.Report.Output } env.saveDebugFile(current.Hash, i, output) default: verdicts = append(verdicts, fmt.Sprintf("failed: %v", err)) } } unique := make(map[string]bool) for _, verdict := range verdicts { unique[verdict] = true } if len(unique) == 1 { env.log("all runs: %v", verdicts[0]) } else { for i, verdict := range verdicts { env.log("run #%v: %v", i, verdict) } } return } // Note: linux-specific. func (env *env) buildEnvForCommit(commit string) (*buildEnv, error) { cfg := env.cfg tags, err := env.repo.PreviousReleaseTags(commit) if err != nil { return nil, err } be := &buildEnv{ compiler: filepath.Join(cfg.BinDir, "gcc-"+linuxCompilerVersion(tags), "bin", "gcc"), } return be, nil } func linuxCompilerVersion(tags []string) string { for _, tag := range tags { switch tag { case "v4.12": return "8.1.0" case "v4.11": return "7.3.0" case "v3.19": return "5.5.0" } } return "4.9.4" } func (env *env) saveDebugFile(hash string, idx int, data []byte) { if env.cfg.DebugDir == "" || len(data) == 0 { return } osutil.WriteFile(filepath.Join(env.cfg.DebugDir, fmt.Sprintf("%v.%v", hash, idx)), data) } func (env *env) log(msg string, args ...interface{}) { fmt.Fprintf(env.cfg.Trace, msg+"\n", args...) }