1// Copyright 2017 syzkaller project authors. All rights reserved. 2// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file. 3 4// syz-ci is a continuous fuzzing system for syzkaller. 5// It runs several syz-manager's, polls and rebuilds images for managers 6// and polls and rebuilds syzkaller binaries. 7// For usage instructions see: docs/ci.md 8package main 9 10// Implementation details: 11// 12// 2 main components: 13// - SyzUpdater: handles syzkaller updates 14// - Manager: handles kernel build and syz-manager process (one per manager) 15// Both operate in a similar way and keep 2 builds: 16// - latest: latest known good build (i.e. we tested it) 17// preserved across restarts/reboots, i.e. we can start fuzzing even when 18// current syzkaller/kernel git head is broken, or git is down, or anything else 19// - current: currently used build (a copy of one of the latest builds) 20// Other important points: 21// - syz-ci is always built on the same revision as the rest of syzkaller binaries, 22// this allows us to handle e.g. changes in manager config format. 23// - consequently, syzkaller binaries are never updated on-the-fly, 24// instead we re-exec and then update 25// - we understand when the latest build is fresh even after reboot, 26// i.e. we store enough information to identify it (git hash, compiler identity, etc), 27// so we don't rebuild unnecessary (kernel builds take time) 28// - we generally avoid crashing the process and handle all errors gracefully 29// (this is a continuous system), except for some severe/user errors during start 30// (e.g. bad config file, or can't create necessary dirs) 31// 32// Directory/file structure: 33// syz-ci : current executable 34// syz-ci.tag : tag of the current executable (syzkaller git hash) 35// syzkaller/ 36// latest/ : latest good syzkaller build 37// current/ : syzkaller build currently in use 38// managers/ 39// manager1/ : one dir per manager 40// kernel/ : kernel checkout 41// workdir/ : manager workdir (never deleted) 42// latest/ : latest good kernel image build 43// current/ : kernel image currently in use 44// jobs/ 45// linux/ : one dir per target OS 46// kernel/ : kernel checkout 47// image/ : currently used image 48// workdir/ : some temp files 49// 50// Current executable, syzkaller and kernel builds are marked with tag files. 51// Tag files uniquely identify the build (git hash, compiler identity, kernel config, etc). 52// For tag files both contents and modification time are important, 53// modification time allows us to understand if we need to rebuild after a restart. 54 55import ( 56 "encoding/json" 57 "flag" 58 "fmt" 59 "os" 60 "sync" 61 62 "github.com/google/syzkaller/pkg/config" 63 "github.com/google/syzkaller/pkg/log" 64 "github.com/google/syzkaller/pkg/mgrconfig" 65 "github.com/google/syzkaller/pkg/osutil" 66) 67 68var flagConfig = flag.String("config", "", "config file") 69 70type Config struct { 71 Name string `json:"name"` 72 HTTP string `json:"http"` 73 DashboardAddr string `json:"dashboard_addr"` // Optional. 74 DashboardClient string `json:"dashboard_client"` // Optional. 75 DashboardKey string `json:"dashboard_key"` // Optional. 76 HubAddr string `json:"hub_addr"` // Optional. 77 HubKey string `json:"hub_key"` // Optional. 78 Goroot string `json:"goroot"` // Go 1.8+ toolchain dir. 79 SyzkallerRepo string `json:"syzkaller_repo"` 80 SyzkallerBranch string `json:"syzkaller_branch"` 81 // Dir with additional syscall descriptions (.txt and .const files). 82 SyzkallerDescriptions string `json:"syzkaller_descriptions"` 83 // Enable patch testing jobs. 84 EnableJobs bool `json:"enable_jobs"` 85 Managers []*ManagerConfig `json:"managers"` 86} 87 88type ManagerConfig struct { 89 Name string `json:"name"` 90 DashboardClient string `json:"dashboard_client"` 91 DashboardKey string `json:"dashboard_key"` 92 Repo string `json:"repo"` 93 // Short name of the repo (e.g. "linux-next"), used only for reporting. 94 RepoAlias string `json:"repo_alias"` 95 Branch string `json:"branch"` 96 Compiler string `json:"compiler"` 97 Userspace string `json:"userspace"` 98 KernelConfig string `json:"kernel_config"` 99 // File with kernel cmdline values (optional). 100 KernelCmdline string `json:"kernel_cmdline"` 101 // File with sysctl values (e.g. output of sysctl -a, optional). 102 KernelSysctl string `json:"kernel_sysctl"` 103 ManagerConfig json.RawMessage `json:"manager_config"` 104} 105 106func main() { 107 flag.Parse() 108 log.EnableLogCaching(1000, 1<<20) 109 cfg, err := loadConfig(*flagConfig) 110 if err != nil { 111 log.Fatalf("failed to load config: %v", err) 112 } 113 114 shutdownPending := make(chan struct{}) 115 osutil.HandleInterrupts(shutdownPending) 116 117 updater := NewSyzUpdater(cfg) 118 updater.UpdateOnStart(shutdownPending) 119 updatePending := make(chan struct{}) 120 go func() { 121 updater.WaitForUpdate() 122 close(updatePending) 123 }() 124 125 var wg sync.WaitGroup 126 wg.Add(1) 127 stop := make(chan struct{}) 128 go func() { 129 select { 130 case <-shutdownPending: 131 case <-updatePending: 132 } 133 close(stop) 134 wg.Done() 135 }() 136 137 managers := make([]*Manager, len(cfg.Managers)) 138 for i, mgrcfg := range cfg.Managers { 139 managers[i] = createManager(cfg, mgrcfg, stop) 140 } 141 for _, mgr := range managers { 142 mgr := mgr 143 wg.Add(1) 144 go func() { 145 defer wg.Done() 146 mgr.loop() 147 }() 148 } 149 if cfg.EnableJobs { 150 jp := newJobProcessor(cfg, managers) 151 wg.Add(1) 152 go func() { 153 defer wg.Done() 154 jp.loop(stop) 155 }() 156 } 157 158 wg.Wait() 159 160 select { 161 case <-shutdownPending: 162 case <-updatePending: 163 updater.UpdateAndRestart() 164 } 165} 166 167func loadConfig(filename string) (*Config, error) { 168 cfg := &Config{ 169 SyzkallerRepo: "https://github.com/google/syzkaller.git", 170 SyzkallerBranch: "master", 171 Goroot: os.Getenv("GOROOT"), 172 } 173 if err := config.LoadFile(filename, cfg); err != nil { 174 return nil, err 175 } 176 if cfg.Name == "" { 177 return nil, fmt.Errorf("param 'name' is empty") 178 } 179 if cfg.HTTP == "" { 180 return nil, fmt.Errorf("param 'http' is empty") 181 } 182 if len(cfg.Managers) == 0 { 183 return nil, fmt.Errorf("no managers specified") 184 } 185 for i, mgr := range cfg.Managers { 186 if mgr.Name == "" { 187 return nil, fmt.Errorf("param 'managers[%v].name' is empty", i) 188 } 189 mgrcfg := new(mgrconfig.Config) 190 if err := config.LoadData(mgr.ManagerConfig, mgrcfg); err != nil { 191 return nil, fmt.Errorf("manager %v: %v", mgr.Name, err) 192 } 193 } 194 return cfg, nil 195} 196