// Copyright 2015 syzkaller project authors. All rights reserved. // Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file. // Package vm provides an abstract test machine (VM, physical machine, etc) // interface for the rest of the system. // For convenience test machines are subsequently collectively called VMs. // Package wraps vmimpl package interface with some common functionality // and higher-level interface. package vm import ( "bytes" "fmt" "os" "time" "github.com/google/syzkaller/pkg/mgrconfig" "github.com/google/syzkaller/pkg/osutil" "github.com/google/syzkaller/pkg/report" "github.com/google/syzkaller/vm/vmimpl" // Import all VM implementations, so that users only need to import vm. _ "github.com/google/syzkaller/vm/adb" _ "github.com/google/syzkaller/vm/gce" _ "github.com/google/syzkaller/vm/gvisor" _ "github.com/google/syzkaller/vm/isolated" _ "github.com/google/syzkaller/vm/kvm" _ "github.com/google/syzkaller/vm/odroid" _ "github.com/google/syzkaller/vm/qemu" ) type Pool struct { impl vmimpl.Pool workdir string } type Instance struct { impl vmimpl.Instance workdir string index int } var ( Shutdown = vmimpl.Shutdown ErrTimeout = vmimpl.ErrTimeout ) type BootErrorer interface { BootError() (string, []byte) } func Create(cfg *mgrconfig.Config, debug bool) (*Pool, error) { env := &vmimpl.Env{ Name: cfg.Name, OS: cfg.TargetOS, Arch: cfg.TargetVMArch, Workdir: cfg.Workdir, Image: cfg.Image, SSHKey: cfg.SSHKey, SSHUser: cfg.SSHUser, Debug: debug, Config: cfg.VM, } impl, err := vmimpl.Create(cfg.Type, env) if err != nil { return nil, err } return &Pool{ impl: impl, workdir: env.Workdir, }, nil } func (pool *Pool) Count() int { return pool.impl.Count() } func (pool *Pool) Create(index int) (*Instance, error) { if index < 0 || index >= pool.Count() { return nil, fmt.Errorf("invalid VM index %v (count %v)", index, pool.Count()) } workdir, err := osutil.ProcessTempDir(pool.workdir) if err != nil { return nil, fmt.Errorf("failed to create instance temp dir: %v", err) } impl, err := pool.impl.Create(workdir, index) if err != nil { os.RemoveAll(workdir) return nil, err } return &Instance{ impl: impl, workdir: workdir, index: index, }, nil } func (inst *Instance) Copy(hostSrc string) (string, error) { return inst.impl.Copy(hostSrc) } func (inst *Instance) Forward(port int) (string, error) { return inst.impl.Forward(port) } func (inst *Instance) Run(timeout time.Duration, stop <-chan bool, command string) ( outc <-chan []byte, errc <-chan error, err error) { return inst.impl.Run(timeout, stop, command) } func (inst *Instance) Diagnose() bool { return inst.impl.Diagnose() } func (inst *Instance) Close() { inst.impl.Close() os.RemoveAll(inst.workdir) } // MonitorExecution monitors execution of a program running inside of a VM. // It detects kernel oopses in output, lost connections, hangs, etc. // outc/errc is what vm.Instance.Run returns, reporter parses kernel output for oopses. // If canExit is false and the program exits, it is treated as an error. // Returns a non-symbolized crash report, or nil if no error happens. func (inst *Instance) MonitorExecution(outc <-chan []byte, errc <-chan error, reporter report.Reporter, canExit bool) (rep *report.Report) { mon := &monitor{ inst: inst, outc: outc, errc: errc, reporter: reporter, canExit: canExit, } lastExecuteTime := time.Now() ticker := time.NewTicker(10 * time.Second) defer ticker.Stop() for { select { case err := <-errc: switch err { case nil: // The program has exited without errors, // but wait for kernel output in case there is some delayed oops. return mon.extractError("") case ErrTimeout: return nil default: // Note: connection lost can race with a kernel oops message. // In such case we want to return the kernel oops. return mon.extractError("lost connection to test machine") } case out := <-outc: lastPos := len(mon.output) mon.output = append(mon.output, out...) if bytes.Contains(mon.output[lastPos:], executingProgram1) || bytes.Contains(mon.output[lastPos:], executingProgram2) { lastExecuteTime = time.Now() } if reporter.ContainsCrash(mon.output[mon.matchPos:]) { return mon.extractError("unknown error") } if len(mon.output) > 2*beforeContext { copy(mon.output, mon.output[len(mon.output)-beforeContext:]) mon.output = mon.output[:beforeContext] } mon.matchPos = len(mon.output) - maxErrorLength if mon.matchPos < 0 { mon.matchPos = 0 } case <-ticker.C: // Detect both "not output whatsoever" and "kernel episodically prints // something to console, but fuzzer is not actually executing programs". // The timeout used to be 3 mins for a long time. // But (1) we were seeing flakes on linux where net namespace // destruction can be really slow, and (2) gVisor watchdog timeout // is 3 mins + 1/4 of that for checking period = 3m45s. // Current linux max timeout is CONFIG_DEFAULT_HUNG_TASK_TIMEOUT=140 // and workqueue.watchdog_thresh=140 which both actually result // in 140-280s detection delay. // So the current timeout is 5 mins (300s). // We don't want it to be too long too because it will waste time on real hangs. if time.Since(lastExecuteTime) < 5*time.Minute { break } if inst.Diagnose() { mon.waitForOutput() } rep := &report.Report{ Title: "no output from test machine", Output: mon.output, Suppressed: report.IsSuppressed(mon.reporter, mon.output), } return rep case <-Shutdown: return nil } } } type monitor struct { inst *Instance outc <-chan []byte errc <-chan error reporter report.Reporter canExit bool output []byte matchPos int } func (mon *monitor) extractError(defaultError string) *report.Report { // Give it some time to finish writing the error message. mon.inst.Diagnose() mon.waitForOutput() if bytes.Contains(mon.output, []byte("SYZ-FUZZER: PREEMPTED")) { return nil } if !mon.reporter.ContainsCrash(mon.output[mon.matchPos:]) { if defaultError == "" { if mon.canExit { return nil } defaultError = "lost connection to test machine" } rep := &report.Report{ Title: defaultError, Output: mon.output, Suppressed: report.IsSuppressed(mon.reporter, mon.output), } return rep } rep := mon.reporter.Parse(mon.output[mon.matchPos:]) if rep == nil { panic(fmt.Sprintf("reporter.ContainsCrash/Parse disagree:\n%s", mon.output[mon.matchPos:])) } start := mon.matchPos + rep.StartPos - beforeContext if start < 0 { start = 0 } end := mon.matchPos + rep.EndPos + afterContext if end > len(mon.output) { end = len(mon.output) } rep.Output = mon.output[start:end] rep.StartPos += mon.matchPos - start rep.EndPos += mon.matchPos - start return rep } func (mon *monitor) waitForOutput() { timer := time.NewTimer(10 * time.Second) for { select { case out, ok := <-mon.outc: if !ok { timer.Stop() return } mon.output = append(mon.output, out...) case <-timer.C: return } } } const ( beforeContext = 1024 << 10 afterContext = 128 << 10 maxErrorLength = 512 ) var ( executingProgram1 = []byte("executing program") // syz-fuzzer output executingProgram2 = []byte("executed programs:") // syz-execprog output )