1// Copyright 2018 syzkaller project authors. All rights reserved. 2// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file. 3 4// Package gvisor provides support for gVisor, user-space kernel, testing. 5// See https://github.com/google/gvisor 6package gvisor 7 8import ( 9 "bytes" 10 "fmt" 11 "io" 12 "net" 13 "os" 14 "os/exec" 15 "path/filepath" 16 "strings" 17 "syscall" 18 "time" 19 20 "github.com/google/syzkaller/pkg/config" 21 "github.com/google/syzkaller/pkg/osutil" 22 "github.com/google/syzkaller/vm/vmimpl" 23) 24 25func init() { 26 vmimpl.Register("gvisor", ctor) 27} 28 29type Config struct { 30 Count int `json:"count"` // number of VMs to use 31 RunscArgs string `json:"runsc_args"` 32} 33 34type Pool struct { 35 env *vmimpl.Env 36 cfg *Config 37} 38 39type instance struct { 40 cfg *Config 41 image string 42 debug bool 43 rootDir string 44 imageDir string 45 name string 46 port int 47 cmd *exec.Cmd 48 merger *vmimpl.OutputMerger 49} 50 51func ctor(env *vmimpl.Env) (vmimpl.Pool, error) { 52 cfg := &Config{ 53 Count: 1, 54 } 55 if err := config.LoadData(env.Config, cfg); err != nil { 56 return nil, fmt.Errorf("failed to parse vm config: %v", err) 57 } 58 if cfg.Count < 1 || cfg.Count > 1000 { 59 return nil, fmt.Errorf("invalid config param count: %v, want [1, 1000]", cfg.Count) 60 } 61 if env.Debug { 62 cfg.Count = 1 63 } 64 if !osutil.IsExist(env.Image) { 65 return nil, fmt.Errorf("image file %q does not exist", env.Image) 66 } 67 pool := &Pool{ 68 cfg: cfg, 69 env: env, 70 } 71 return pool, nil 72} 73 74func (pool *Pool) Count() int { 75 return pool.cfg.Count 76} 77 78func (pool *Pool) Create(workdir string, index int) (vmimpl.Instance, error) { 79 rootDir := filepath.Clean(filepath.Join(workdir, "..", "gvisor_root")) 80 imageDir := filepath.Join(workdir, "image") 81 bundleDir := filepath.Join(workdir, "bundle") 82 osutil.MkdirAll(rootDir) 83 osutil.MkdirAll(bundleDir) 84 osutil.MkdirAll(imageDir) 85 86 caps := "" 87 for _, c := range sandboxCaps { 88 if caps != "" { 89 caps += ", " 90 } 91 caps += "\"" + c + "\"" 92 } 93 vmConfig := fmt.Sprintf(configTempl, imageDir, caps) 94 if err := osutil.WriteFile(filepath.Join(bundleDir, "config.json"), []byte(vmConfig)); err != nil { 95 return nil, err 96 } 97 bin, err := exec.LookPath(os.Args[0]) 98 if err != nil { 99 return nil, fmt.Errorf("failed to lookup %v: %v", os.Args[0], err) 100 } 101 if err := osutil.CopyFile(bin, filepath.Join(imageDir, "init")); err != nil { 102 return nil, err 103 } 104 105 rpipe, wpipe, err := osutil.LongPipe() 106 if err != nil { 107 return nil, err 108 } 109 var tee io.Writer 110 if pool.env.Debug { 111 tee = os.Stdout 112 } 113 merger := vmimpl.NewOutputMerger(tee) 114 merger.Add("gvisor", rpipe) 115 116 inst := &instance{ 117 cfg: pool.cfg, 118 image: pool.env.Image, 119 debug: pool.env.Debug, 120 rootDir: rootDir, 121 imageDir: imageDir, 122 name: fmt.Sprintf("%v-%v", pool.env.Name, index), 123 merger: merger, 124 } 125 126 // Kill the previous instance in case it's still running. 127 osutil.Run(time.Minute, inst.runscCmd("delete", "-force", inst.name)) 128 time.Sleep(3 * time.Second) 129 130 cmd := inst.runscCmd("run", "-bundle", bundleDir, inst.name) 131 cmd.Stdout = wpipe 132 cmd.Stderr = wpipe 133 if err := cmd.Start(); err != nil { 134 wpipe.Close() 135 merger.Wait() 136 return nil, err 137 } 138 inst.cmd = cmd 139 wpipe.Close() 140 141 if err := inst.waitBoot(); err != nil { 142 inst.Close() 143 return nil, err 144 } 145 return inst, nil 146} 147 148func (inst *instance) waitBoot() error { 149 errorMsg := []byte("FATAL ERROR:") 150 bootedMsg := []byte(initStartMsg) 151 timeout := time.NewTimer(time.Minute) 152 defer timeout.Stop() 153 var output []byte 154 for { 155 select { 156 case out := <-inst.merger.Output: 157 output = append(output, out...) 158 if pos := bytes.Index(output, errorMsg); pos != -1 { 159 end := bytes.IndexByte(output[pos:], '\n') 160 if end == -1 { 161 end = len(output) 162 } else { 163 end += pos 164 } 165 return vmimpl.BootError{ 166 Title: string(output[pos:end]), 167 Output: output, 168 } 169 } 170 if bytes.Contains(output, bootedMsg) { 171 return nil 172 } 173 case err := <-inst.merger.Err: 174 return vmimpl.BootError{ 175 Title: fmt.Sprintf("runsc failed: %v", err), 176 Output: output, 177 } 178 case <-timeout.C: 179 return vmimpl.BootError{ 180 Title: "init process did not start", 181 Output: output, 182 } 183 } 184 } 185} 186 187func (inst *instance) runscCmd(add ...string) *exec.Cmd { 188 args := []string{ 189 "-root", inst.rootDir, 190 "-watchdog-action=panic", 191 "-network=none", 192 } 193 if inst.cfg.RunscArgs != "" { 194 args = append(args, strings.Split(inst.cfg.RunscArgs, " ")...) 195 } 196 args = append(args, add...) 197 cmd := osutil.Command(inst.image, args...) 198 cmd.Env = []string{ 199 "GOTRACEBACK=all", 200 "GORACE=halt_on_error=1", 201 } 202 return cmd 203} 204 205func (inst *instance) Close() { 206 time.Sleep(3 * time.Second) 207 osutil.Run(time.Minute, inst.runscCmd("delete", "-force", inst.name)) 208 inst.cmd.Process.Kill() 209 inst.merger.Wait() 210 inst.cmd.Wait() 211 osutil.Run(time.Minute, inst.runscCmd("delete", "-force", inst.name)) 212 time.Sleep(3 * time.Second) 213} 214 215func (inst *instance) Forward(port int) (string, error) { 216 if inst.port != 0 { 217 return "", fmt.Errorf("forward port is already setup") 218 } 219 inst.port = port 220 return "stdin", nil 221} 222 223func (inst *instance) Copy(hostSrc string) (string, error) { 224 fname := filepath.Base(hostSrc) 225 if err := osutil.CopyFile(hostSrc, filepath.Join(inst.imageDir, fname)); err != nil { 226 return "", err 227 } 228 if err := os.Chmod(inst.imageDir, 0777); err != nil { 229 return "", err 230 } 231 return filepath.Join("/", fname), nil 232} 233 234func (inst *instance) Run(timeout time.Duration, stop <-chan bool, command string) ( 235 <-chan []byte, <-chan error, error) { 236 args := []string{"exec", "-user=0:0"} 237 for _, c := range sandboxCaps { 238 args = append(args, "-cap", c) 239 } 240 args = append(args, inst.name) 241 args = append(args, strings.Split(command, " ")...) 242 cmd := inst.runscCmd(args...) 243 244 rpipe, wpipe, err := osutil.LongPipe() 245 if err != nil { 246 return nil, nil, err 247 } 248 defer wpipe.Close() 249 inst.merger.Add("cmd", rpipe) 250 cmd.Stdout = wpipe 251 cmd.Stderr = wpipe 252 253 guestSock, err := inst.guestProxy() 254 if err != nil { 255 return nil, nil, err 256 } 257 if guestSock != nil { 258 defer guestSock.Close() 259 cmd.Stdin = guestSock 260 } 261 262 if err := cmd.Start(); err != nil { 263 return nil, nil, err 264 } 265 errc := make(chan error, 1) 266 signal := func(err error) { 267 select { 268 case errc <- err: 269 default: 270 } 271 } 272 273 go func() { 274 select { 275 case <-time.After(timeout): 276 signal(vmimpl.ErrTimeout) 277 case <-stop: 278 signal(vmimpl.ErrTimeout) 279 case err := <-inst.merger.Err: 280 cmd.Process.Kill() 281 if cmdErr := cmd.Wait(); cmdErr == nil { 282 // If the command exited successfully, we got EOF error from merger. 283 // But in this case no error has happened and the EOF is expected. 284 err = nil 285 } 286 signal(err) 287 return 288 } 289 cmd.Process.Kill() 290 cmd.Wait() 291 }() 292 return inst.merger.Output, errc, nil 293} 294 295func (inst *instance) guestProxy() (*os.File, error) { 296 if inst.port == 0 { 297 return nil, nil 298 } 299 // One does not simply let gvisor guest connect to host tcp port. 300 // We create a unix socket, pass it to guest in stdin. 301 // Guest will use it instead of dialing manager directly. 302 // On host we connect to manager tcp port and proxy between the tcp and unix connections. 303 socks, err := syscall.Socketpair(syscall.AF_UNIX, syscall.SOCK_STREAM, 0) 304 if err != nil { 305 return nil, err 306 } 307 hostSock := os.NewFile(uintptr(socks[0]), "host unix proxy") 308 guestSock := os.NewFile(uintptr(socks[1]), "guest unix proxy") 309 conn, err := net.Dial("tcp", fmt.Sprintf("127.0.0.1:%v", inst.port)) 310 if err != nil { 311 conn.Close() 312 hostSock.Close() 313 guestSock.Close() 314 return nil, err 315 } 316 go func() { 317 io.Copy(hostSock, conn) 318 hostSock.Close() 319 }() 320 go func() { 321 io.Copy(conn, hostSock) 322 conn.Close() 323 }() 324 return guestSock, nil 325} 326 327func (inst *instance) Diagnose() bool { 328 osutil.Run(time.Minute, inst.runscCmd("debug", "-stacks", inst.name)) 329 return true 330} 331 332func init() { 333 if os.Getenv("SYZ_GVISOR_PROXY") != "" { 334 fmt.Fprintf(os.Stderr, initStartMsg) 335 select {} 336 } 337} 338 339const initStartMsg = "SYZKALLER INIT STARTED\n" 340 341const configTempl = ` 342{ 343 "root": { 344 "path": "%[1]v", 345 "readonly": true 346 }, 347 "process":{ 348 "args": ["/init"], 349 "cwd": "/tmp", 350 "env": ["SYZ_GVISOR_PROXY=1"], 351 "capabilities": { 352 "bounding": [%[2]v], 353 "effective": [%[2]v], 354 "inheritable": [%[2]v], 355 "permitted": [%[2]v], 356 "ambient": [%[2]v] 357 } 358 } 359} 360` 361 362var sandboxCaps = []string{ 363 "CAP_CHOWN", "CAP_DAC_OVERRIDE", "CAP_DAC_READ_SEARCH", "CAP_FOWNER", "CAP_FSETID", 364 "CAP_KILL", "CAP_SETGID", "CAP_SETUID", "CAP_SETPCAP", "CAP_LINUX_IMMUTABLE", 365 "CAP_NET_BIND_SERVICE", "CAP_NET_BROADCAST", "CAP_NET_ADMIN", "CAP_NET_RAW", 366 "CAP_IPC_LOCK", "CAP_IPC_OWNER", "CAP_SYS_MODULE", "CAP_SYS_RAWIO", "CAP_SYS_CHROOT", 367 "CAP_SYS_PTRACE", "CAP_SYS_PACCT", "CAP_SYS_ADMIN", "CAP_SYS_BOOT", "CAP_SYS_NICE", 368 "CAP_SYS_RESOURCE", "CAP_SYS_TIME", "CAP_SYS_TTY_CONFIG", "CAP_MKNOD", "CAP_LEASE", 369 "CAP_AUDIT_WRITE", "CAP_AUDIT_CONTROL", "CAP_SETFCAP", "CAP_MAC_OVERRIDE", "CAP_MAC_ADMIN", 370 "CAP_SYSLOG", "CAP_WAKE_ALARM", "CAP_BLOCK_SUSPEND", "CAP_AUDIT_READ", 371} 372