1// Copyright 2017 syzkaller project authors. All rights reserved. 2// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file. 3 4package dash 5 6import ( 7 "fmt" 8 "regexp" 9 "strconv" 10 "time" 11 12 "github.com/google/syzkaller/dashboard/dashapi" 13 "github.com/google/syzkaller/pkg/hash" 14 "golang.org/x/net/context" 15 "google.golang.org/appengine/datastore" 16) 17 18// This file contains definitions of entities stored in datastore. 19 20const ( 21 maxTextLen = 200 22 MaxStringLen = 1024 23 24 maxCrashes = 40 25) 26 27type Manager struct { 28 Namespace string 29 Name string 30 Link string 31 CurrentBuild string 32 FailedBuildBug string 33 LastAlive time.Time 34 CurrentUpTime time.Duration 35} 36 37// ManagerStats holds per-day manager runtime stats. 38// Has Manager as parent entity. Keyed by Date. 39type ManagerStats struct { 40 Date int // YYYYMMDD 41 MaxCorpus int64 42 MaxCover int64 43 TotalFuzzingTime time.Duration 44 TotalCrashes int64 45 TotalExecs int64 46} 47 48type Build struct { 49 Namespace string 50 Manager string 51 ID string // unique ID generated by syz-ci 52 Type BuildType 53 Time time.Time 54 OS string 55 Arch string 56 VMArch string 57 SyzkallerCommit string 58 CompilerID string 59 KernelRepo string 60 KernelBranch string 61 KernelCommit string 62 KernelCommitTitle string `datastore:",noindex"` 63 KernelCommitDate time.Time `datastore:",noindex"` 64 KernelConfig int64 // reference to KernelConfig text entity 65} 66 67type Bug struct { 68 Namespace string 69 Seq int64 // sequences of the bug with the same title 70 Title string 71 Status int 72 DupOf string 73 NumCrashes int64 74 NumRepro int64 75 ReproLevel dashapi.ReproLevel 76 HasReport bool 77 FirstTime time.Time 78 LastTime time.Time 79 LastSavedCrash time.Time 80 LastReproTime time.Time 81 Closed time.Time 82 Reporting []BugReporting 83 Commits []string 84 HappenedOn []string `datastore:",noindex"` // list of managers 85 PatchedOn []string `datastore:",noindex"` // list of managers 86} 87 88type BugReporting struct { 89 Name string // refers to Reporting.Name 90 ID string // unique ID per BUG/BugReporting used in commucation with external systems 91 ExtID string // arbitrary reporting ID that is passed back in dashapi.BugReport 92 Link string 93 CC string // additional emails added to CC list (|-delimited list) 94 CrashID int64 // crash that we've last reported in this reporting 95 ReproLevel dashapi.ReproLevel 96 Reported time.Time 97 Closed time.Time 98} 99 100type Crash struct { 101 Manager string 102 BuildID string 103 Time time.Time 104 Reported time.Time // set if this crash was ever reported 105 Maintainers []string `datastore:",noindex"` 106 Log int64 // reference to CrashLog text entity 107 Report int64 // reference to CrashReport text entity 108 ReproOpts []byte `datastore:",noindex"` 109 ReproSyz int64 // reference to ReproSyz text entity 110 ReproC int64 // reference to ReproC text entity 111 // Custom crash priority for reporting (greater values are higher priority). 112 // For example, a crash in mainline kernel has higher priority than a crash in a side branch. 113 // For historical reasons this is called ReportLen. 114 ReportLen int64 115} 116 117// ReportingState holds dynamic info associated with reporting. 118type ReportingState struct { 119 Entries []ReportingStateEntry 120} 121 122type ReportingStateEntry struct { 123 Namespace string 124 Name string 125 // Current reporting quota consumption. 126 Sent int 127 Date int // YYYYMMDD 128} 129 130// Job represent a single patch testing job for syz-ci. 131// Later we may want to extend this to other types of jobs (hense the generic name): 132// - test of a committed fix 133// - reproduce crash 134// - test that crash still happens on HEAD 135// - crash bisect 136// Job has Bug as parent entity. 137type Job struct { 138 Created time.Time 139 User string 140 CC []string 141 Reporting string 142 ExtID string // email Message-ID 143 Link string // web link for the job (e.g. email in the group) 144 Namespace string 145 Manager string 146 BugTitle string 147 CrashID int64 148 149 // Provided by user: 150 KernelRepo string 151 KernelBranch string 152 Patch int64 // reference to Patch text entity 153 154 Attempts int // number of times we tried to execute this job 155 Started time.Time 156 Finished time.Time // if set, job is finished 157 158 // Result of execution: 159 CrashTitle string // if empty, we did not hit crash during testing 160 CrashLog int64 // reference to CrashLog text entity 161 CrashReport int64 // reference to CrashReport text entity 162 BuildID string 163 Error int64 // reference to Error text entity, if set job failed 164 165 Reported bool // have we reported result back to user? 166} 167 168// Text holds text blobs (crash logs, reports, reproducers, etc). 169type Text struct { 170 Namespace string 171 Text []byte `datastore:",noindex"` // gzip-compressed text 172} 173 174const ( 175 textCrashLog = "CrashLog" 176 textCrashReport = "CrashReport" 177 textReproSyz = "ReproSyz" 178 textReproC = "ReproC" 179 textKernelConfig = "KernelConfig" 180 textPatch = "Patch" 181 textError = "Error" 182) 183 184const ( 185 BugStatusOpen = iota 186) 187 188const ( 189 BugStatusFixed = 1000 + iota 190 BugStatusInvalid 191 BugStatusDup 192) 193 194const ( 195 ReproLevelNone = dashapi.ReproLevelNone 196 ReproLevelSyz = dashapi.ReproLevelSyz 197 ReproLevelC = dashapi.ReproLevelC 198) 199 200type BuildType int 201 202const ( 203 BuildNormal BuildType = iota 204 BuildFailed 205 BuildJob 206) 207 208// updateManager does transactional compare-and-swap on the manager and its current stats. 209func updateManager(c context.Context, ns, name string, fn func(mgr *Manager, stats *ManagerStats)) error { 210 date := timeDate(timeNow(c)) 211 tx := func(c context.Context) error { 212 mgr := new(Manager) 213 mgrKey := datastore.NewKey(c, "Manager", fmt.Sprintf("%v-%v", ns, name), 0, nil) 214 if err := datastore.Get(c, mgrKey, mgr); err != nil { 215 if err != datastore.ErrNoSuchEntity { 216 return fmt.Errorf("failed to get manager %v/%v: %v", ns, name, err) 217 } 218 mgr = &Manager{ 219 Namespace: ns, 220 Name: name, 221 } 222 } 223 stats := new(ManagerStats) 224 statsKey := datastore.NewKey(c, "ManagerStats", "", int64(date), mgrKey) 225 if err := datastore.Get(c, statsKey, stats); err != nil { 226 if err != datastore.ErrNoSuchEntity { 227 return fmt.Errorf("failed to get stats %v/%v/%v: %v", ns, name, date, err) 228 } 229 stats = &ManagerStats{ 230 Date: date, 231 } 232 } 233 234 fn(mgr, stats) 235 236 if _, err := datastore.Put(c, mgrKey, mgr); err != nil { 237 return fmt.Errorf("failed to put manager: %v", err) 238 } 239 if _, err := datastore.Put(c, statsKey, stats); err != nil { 240 return fmt.Errorf("failed to put manager stats: %v", err) 241 } 242 return nil 243 } 244 return datastore.RunInTransaction(c, tx, &datastore.TransactionOptions{Attempts: 10}) 245} 246 247func loadAllManagers(c context.Context) ([]*Manager, []*datastore.Key, error) { 248 var managers []*Manager 249 keys, err := datastore.NewQuery("Manager"). 250 GetAll(c, &managers) 251 if err != nil { 252 return nil, nil, fmt.Errorf("failed to query managers: %v", err) 253 } 254 var result []*Manager 255 var resultKeys []*datastore.Key 256 257 for i, mgr := range managers { 258 if config.Namespaces[mgr.Namespace].Managers[mgr.Name].Decommissioned { 259 continue 260 } 261 result = append(result, mgr) 262 resultKeys = append(resultKeys, keys[i]) 263 } 264 return result, resultKeys, nil 265} 266 267func buildKey(c context.Context, ns, id string) *datastore.Key { 268 if ns == "" { 269 panic("requesting build key outside of namespace") 270 } 271 h := hash.String([]byte(fmt.Sprintf("%v-%v", ns, id))) 272 return datastore.NewKey(c, "Build", h, 0, nil) 273} 274 275func loadBuild(c context.Context, ns, id string) (*Build, error) { 276 build := new(Build) 277 if err := datastore.Get(c, buildKey(c, ns, id), build); err != nil { 278 if err == datastore.ErrNoSuchEntity { 279 return nil, fmt.Errorf("unknown build %v/%v", ns, id) 280 } 281 return nil, fmt.Errorf("failed to get build %v/%v: %v", ns, id, err) 282 } 283 return build, nil 284} 285 286func lastManagerBuild(c context.Context, ns, manager string) (*Build, error) { 287 var builds []*Build 288 _, err := datastore.NewQuery("Build"). 289 Filter("Namespace=", ns). 290 Filter("Manager=", manager). 291 Filter("Type=", BuildNormal). 292 Order("-Time"). 293 Limit(1). 294 GetAll(c, &builds) 295 if err != nil { 296 return nil, fmt.Errorf("failed to fetch manager build: %v", err) 297 } 298 if len(builds) == 0 { 299 return nil, fmt.Errorf("failed to fetch manager build: no builds") 300 } 301 return builds[0], nil 302} 303 304func (bug *Bug) displayTitle() string { 305 if bug.Seq == 0 { 306 return bug.Title 307 } 308 return fmt.Sprintf("%v (%v)", bug.Title, bug.Seq+1) 309} 310 311var displayTitleRe = regexp.MustCompile(`^(.*) \(([0-9]+)\)$`) 312 313func splitDisplayTitle(display string) (string, int64, error) { 314 match := displayTitleRe.FindStringSubmatchIndex(display) 315 if match == nil { 316 return display, 0, nil 317 } 318 title := display[match[2]:match[3]] 319 seqStr := display[match[4]:match[5]] 320 seq, err := strconv.ParseInt(seqStr, 10, 64) 321 if err != nil { 322 return "", 0, fmt.Errorf("failed to parse bug title: %v", err) 323 } 324 if seq <= 0 || seq > 1e6 { 325 return "", 0, fmt.Errorf("failed to parse bug title: seq=%v", seq) 326 } 327 return title, seq - 1, nil 328} 329 330func canonicalBug(c context.Context, bug *Bug) (*Bug, error) { 331 for { 332 if bug.Status != BugStatusDup { 333 return bug, nil 334 } 335 canon := new(Bug) 336 bugKey := datastore.NewKey(c, "Bug", bug.DupOf, 0, nil) 337 if err := datastore.Get(c, bugKey, canon); err != nil { 338 return nil, fmt.Errorf("failed to get dup bug %q for %q: %v", 339 bug.DupOf, bugKeyHash(bug.Namespace, bug.Title, bug.Seq), err) 340 } 341 bug = canon 342 } 343} 344 345func bugKeyHash(ns, title string, seq int64) string { 346 return hash.String([]byte(fmt.Sprintf("%v-%v-%v-%v", config.Namespaces[ns].Key, ns, title, seq))) 347} 348 349func bugReportingHash(bugHash, reporting string) string { 350 // Since these IDs appear in Reported-by tags in commit, we slightly limit their size. 351 const hashLen = 20 352 return hash.String([]byte(fmt.Sprintf("%v-%v", bugHash, reporting)))[:hashLen] 353} 354 355func kernelRepoInfo(build *Build) KernelRepo { 356 return kernelRepoInfoRaw(build.KernelRepo, build.KernelBranch) 357} 358 359func kernelRepoInfoRaw(repo, branch string) KernelRepo { 360 repoID := repo 361 if branch != "" { 362 repoID += "/" + branch 363 } 364 info := config.KernelRepos[repoID] 365 if info.Alias == "" { 366 info.Alias = repoID 367 } 368 return info 369} 370 371func textLink(tag string, id int64) string { 372 if id == 0 { 373 return "" 374 } 375 return fmt.Sprintf("/text?tag=%v&x=%v", tag, strconv.FormatUint(uint64(id), 16)) 376} 377 378// timeDate returns t's date as a single int YYYYMMDD. 379func timeDate(t time.Time) int { 380 year, month, day := t.Date() 381 return year*10000 + int(month)*100 + day 382} 383