1// Copyright 2017 syzkaller project authors. All rights reserved. 2// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file. 3 4package dash 5 6import ( 7 "encoding/json" 8 "fmt" 9 "strconv" 10 "strings" 11 "time" 12 13 "github.com/google/syzkaller/dashboard/dashapi" 14 "github.com/google/syzkaller/pkg/email" 15 "github.com/google/syzkaller/pkg/vcs" 16 "golang.org/x/net/context" 17 "google.golang.org/appengine/datastore" 18 "google.golang.org/appengine/log" 19) 20 21// handleTestRequest added new job to datastore. 22// Returns empty string if job added successfully, or reason why it wasn't added. 23func handleTestRequest(c context.Context, bugID, user, extID, link, patch, repo, branch string, 24 jobCC []string) string { 25 log.Infof(c, "test request: bug=%q user=%q extID=%q patch=%v, repo=%q branch=%q", 26 bugID, user, extID, len(patch), repo, branch) 27 for _, blacklisted := range config.EmailBlacklist { 28 if user == blacklisted { 29 log.Errorf(c, "test request from blacklisted user: %v", user) 30 return "" 31 } 32 } 33 bug, bugKey, err := findBugByReportingID(c, bugID) 34 if err != nil { 35 log.Errorf(c, "can't find bug: %v", err) 36 if link != "" { 37 return "" // don't send duplicate error reply 38 } 39 myEmail, _ := email.AddAddrContext(ownEmail(c), "hash") 40 return fmt.Sprintf("can't find the associated bug (do you have %v in To/CC?)", myEmail) 41 } 42 bugReporting, _ := bugReportingByID(bug, bugID) 43 reply, err := addTestJob(c, bug, bugKey, bugReporting, user, extID, link, patch, repo, branch, jobCC) 44 if err != nil { 45 log.Errorf(c, "test request failed: %v", err) 46 if reply == "" { 47 reply = internalError 48 } 49 } 50 // Update bug CC list in any case. 51 if !stringsInList(strings.Split(bugReporting.CC, "|"), jobCC) { 52 tx := func(c context.Context) error { 53 bug := new(Bug) 54 if err := datastore.Get(c, bugKey, bug); err != nil { 55 return err 56 } 57 bugReporting = bugReportingByName(bug, bugReporting.Name) 58 bugCC := strings.Split(bugReporting.CC, "|") 59 merged := email.MergeEmailLists(bugCC, jobCC) 60 bugReporting.CC = strings.Join(merged, "|") 61 if _, err := datastore.Put(c, bugKey, bug); err != nil { 62 return fmt.Errorf("failed to put bug: %v", err) 63 } 64 return nil 65 } 66 if err := datastore.RunInTransaction(c, tx, nil); err != nil { 67 // We've already stored the job, so just log the error. 68 log.Errorf(c, "failed to update bug: %v", err) 69 } 70 } 71 if link != "" { 72 reply = "" // don't send duplicate error reply 73 } 74 return reply 75} 76 77func addTestJob(c context.Context, bug *Bug, bugKey *datastore.Key, bugReporting *BugReporting, 78 user, extID, link, patch, repo, branch string, jobCC []string) (string, error) { 79 crash, crashKey, err := findCrashForBug(c, bug) 80 if err != nil { 81 return "", err 82 } 83 if reason := checkTestJob(c, bug, bugReporting, crash, repo, branch); reason != "" { 84 return reason, nil 85 } 86 87 manager := crash.Manager 88 for _, ns := range config.Namespaces { 89 if mgr, ok := ns.Managers[manager]; ok { 90 if mgr.RestrictedTestingRepo != "" && repo != mgr.RestrictedTestingRepo { 91 return mgr.RestrictedTestingReason, nil 92 } 93 if mgr.Decommissioned { 94 manager = mgr.DelegatedTo 95 } 96 break 97 } 98 } 99 100 patchID, err := putText(c, bug.Namespace, textPatch, []byte(patch), false) 101 if err != nil { 102 return "", err 103 } 104 105 job := &Job{ 106 Created: timeNow(c), 107 User: user, 108 CC: jobCC, 109 Reporting: bugReporting.Name, 110 ExtID: extID, 111 Link: link, 112 Namespace: bug.Namespace, 113 Manager: manager, 114 BugTitle: bug.displayTitle(), 115 CrashID: crashKey.IntID(), 116 KernelRepo: repo, 117 KernelBranch: branch, 118 Patch: patchID, 119 } 120 121 deletePatch := false 122 tx := func(c context.Context) error { 123 deletePatch = false 124 // We can get 2 emails for the same request: one direct and one from a mailing list. 125 // Filter out such duplicates (for dup we only need link update). 126 var jobs []*Job 127 keys, err := datastore.NewQuery("Job"). 128 Ancestor(bugKey). 129 Filter("ExtID=", extID). 130 GetAll(c, &jobs) 131 if len(jobs) > 1 || err != nil { 132 return fmt.Errorf("failed to query jobs: jobs=%v err=%v", len(jobs), err) 133 } 134 if len(jobs) != 0 { 135 // The job is already present, update link. 136 deletePatch = true 137 existingJob, jobKey := jobs[0], keys[0] 138 if existingJob.Link != "" || link == "" { 139 return nil 140 } 141 existingJob.Link = link 142 if _, err := datastore.Put(c, jobKey, existingJob); err != nil { 143 return fmt.Errorf("failed to put job: %v", err) 144 } 145 return nil 146 } 147 // Create a new job. 148 jobKey := datastore.NewIncompleteKey(c, "Job", bugKey) 149 if _, err := datastore.Put(c, jobKey, job); err != nil { 150 return fmt.Errorf("failed to put job: %v", err) 151 } 152 return nil 153 } 154 err = datastore.RunInTransaction(c, tx, &datastore.TransactionOptions{XG: true, Attempts: 30}) 155 if patchID != 0 && deletePatch || err != nil { 156 if err := datastore.Delete(c, datastore.NewKey(c, textPatch, "", patchID, nil)); err != nil { 157 log.Errorf(c, "failed to delete patch for dup job: %v", err) 158 } 159 } 160 if err != nil { 161 return "", fmt.Errorf("job tx failed: %v", err) 162 } 163 return "", nil 164} 165 166func checkTestJob(c context.Context, bug *Bug, bugReporting *BugReporting, crash *Crash, 167 repo, branch string) string { 168 switch { 169 case crash.ReproC == 0 && crash.ReproSyz == 0: 170 return "This crash does not have a reproducer. I cannot test it." 171 case !vcs.CheckRepoAddress(repo): 172 return fmt.Sprintf("%q does not look like a valid git repo address.", repo) 173 case !vcs.CheckBranch(branch) && !vcs.CheckCommitHash(branch): 174 return fmt.Sprintf("%q does not look like a valid git branch or commit.", branch) 175 case crash.ReproC == 0 && crash.ReproSyz == 0: 176 return "This crash does not have a reproducer. I cannot test it." 177 case bug.Status == BugStatusFixed: 178 return "This bug is already marked as fixed. No point in testing." 179 case bug.Status == BugStatusInvalid: 180 return "This bug is already marked as invalid. No point in testing." 181 // TODO(dvyukov): for BugStatusDup check status of the canonical bug. 182 case !bugReporting.Closed.IsZero(): 183 return "This bug is already upstreamed. Please test upstream." 184 } 185 return "" 186} 187 188// pollPendingJobs returns the next job to execute for the provided list of managers. 189func pollPendingJobs(c context.Context, managers []string) (interface{}, error) { 190retry: 191 job, jobKey, err := loadPendingJob(c, managers) 192 if job == nil || err != nil { 193 return job, err 194 } 195 jobID := extJobID(jobKey) 196 patch, _, err := getText(c, textPatch, job.Patch) 197 if err != nil { 198 return nil, err 199 } 200 bugKey := jobKey.Parent() 201 crashKey := datastore.NewKey(c, "Crash", "", job.CrashID, bugKey) 202 crash := new(Crash) 203 if err := datastore.Get(c, crashKey, crash); err != nil { 204 return nil, fmt.Errorf("job %v: failed to get crash: %v", jobID, err) 205 } 206 207 build, err := loadBuild(c, job.Namespace, crash.BuildID) 208 if err != nil { 209 return nil, err 210 } 211 kernelConfig, _, err := getText(c, textKernelConfig, build.KernelConfig) 212 if err != nil { 213 return nil, err 214 } 215 216 reproC, _, err := getText(c, textReproC, crash.ReproC) 217 if err != nil { 218 return nil, err 219 } 220 reproSyz, _, err := getText(c, textReproSyz, crash.ReproSyz) 221 if err != nil { 222 return nil, err 223 } 224 225 now := timeNow(c) 226 stale := false 227 tx := func(c context.Context) error { 228 stale = false 229 job = new(Job) 230 if err := datastore.Get(c, jobKey, job); err != nil { 231 return fmt.Errorf("job %v: failed to get in tx: %v", jobID, err) 232 } 233 if !job.Finished.IsZero() { 234 // This happens sometimes due to inconsistent datastore. 235 stale = true 236 return nil 237 } 238 job.Attempts++ 239 job.Started = now 240 if _, err := datastore.Put(c, jobKey, job); err != nil { 241 return fmt.Errorf("job %v: failed to put: %v", jobID, err) 242 } 243 return nil 244 } 245 if err := datastore.RunInTransaction(c, tx, nil); err != nil { 246 return nil, err 247 } 248 if stale { 249 goto retry 250 } 251 resp := &dashapi.JobPollResp{ 252 ID: jobID, 253 Manager: job.Manager, 254 KernelRepo: job.KernelRepo, 255 KernelBranch: job.KernelBranch, 256 KernelConfig: kernelConfig, 257 SyzkallerCommit: build.SyzkallerCommit, 258 Patch: patch, 259 ReproOpts: crash.ReproOpts, 260 ReproSyz: reproSyz, 261 ReproC: reproC, 262 } 263 return resp, nil 264} 265 266// doneJob is called by syz-ci to mark completion of a job. 267func doneJob(c context.Context, req *dashapi.JobDoneReq) error { 268 jobID := req.ID 269 jobKey, err := jobID2Key(c, req.ID) 270 if err != nil { 271 return err 272 } 273 now := timeNow(c) 274 tx := func(c context.Context) error { 275 job := new(Job) 276 if err := datastore.Get(c, jobKey, job); err != nil { 277 return fmt.Errorf("job %v: failed to get job: %v", jobID, err) 278 } 279 if !job.Finished.IsZero() { 280 return fmt.Errorf("job %v: already finished", jobID) 281 } 282 ns := job.Namespace 283 if isNewBuild, err := uploadBuild(c, now, ns, &req.Build, BuildJob); err != nil { 284 return err 285 } else if !isNewBuild { 286 log.Errorf(c, "job %v: duplicate build %v", jobID, req.Build.ID) 287 } 288 if job.Error, err = putText(c, ns, textError, req.Error, false); err != nil { 289 return err 290 } 291 if job.CrashLog, err = putText(c, ns, textCrashLog, req.CrashLog, false); err != nil { 292 return err 293 } 294 if job.CrashReport, err = putText(c, ns, textCrashReport, req.CrashReport, false); err != nil { 295 return err 296 } 297 job.BuildID = req.Build.ID 298 job.CrashTitle = req.CrashTitle 299 job.Finished = now 300 if _, err := datastore.Put(c, jobKey, job); err != nil { 301 return fmt.Errorf("failed to put job: %v", err) 302 } 303 return nil 304 } 305 return datastore.RunInTransaction(c, tx, &datastore.TransactionOptions{XG: true, Attempts: 30}) 306} 307 308func pollCompletedJobs(c context.Context, typ string) ([]*dashapi.BugReport, error) { 309 var jobs []*Job 310 keys, err := datastore.NewQuery("Job"). 311 Filter("Finished>", time.Time{}). 312 Filter("Reported=", false). 313 GetAll(c, &jobs) 314 if err != nil { 315 return nil, fmt.Errorf("failed to query jobs: %v", err) 316 } 317 var reports []*dashapi.BugReport 318 for i, job := range jobs { 319 reporting := config.Namespaces[job.Namespace].ReportingByName(job.Reporting) 320 if reporting.Config.Type() != typ { 321 continue 322 } 323 rep, err := createBugReportForJob(c, job, keys[i], reporting.Config) 324 if err != nil { 325 log.Errorf(c, "failed to create report for job: %v", err) 326 continue 327 } 328 reports = append(reports, rep) 329 } 330 return reports, nil 331} 332 333func createBugReportForJob(c context.Context, job *Job, jobKey *datastore.Key, config interface{}) ( 334 *dashapi.BugReport, error) { 335 reportingConfig, err := json.Marshal(config) 336 if err != nil { 337 return nil, err 338 } 339 crashLog, _, err := getText(c, textCrashLog, job.CrashLog) 340 if err != nil { 341 return nil, err 342 } 343 if len(crashLog) > maxMailLogLen { 344 crashLog = crashLog[len(crashLog)-maxMailLogLen:] 345 } 346 report, _, err := getText(c, textCrashReport, job.CrashReport) 347 if err != nil { 348 return nil, err 349 } 350 if len(report) > maxMailReportLen { 351 report = report[:maxMailReportLen] 352 } 353 jobError, _, err := getText(c, textError, job.Error) 354 if err != nil { 355 return nil, err 356 } 357 patch, _, err := getText(c, textPatch, job.Patch) 358 if err != nil { 359 return nil, err 360 } 361 build, err := loadBuild(c, job.Namespace, job.BuildID) 362 if err != nil { 363 return nil, err 364 } 365 kernelConfig, _, err := getText(c, textKernelConfig, build.KernelConfig) 366 if err != nil { 367 return nil, err 368 } 369 bug := new(Bug) 370 if err := datastore.Get(c, jobKey.Parent(), bug); err != nil { 371 return nil, fmt.Errorf("failed to load job parent bug: %v", err) 372 } 373 bugReporting := bugReportingByName(bug, job.Reporting) 374 if bugReporting == nil { 375 return nil, fmt.Errorf("job bug has no reporting %q", job.Reporting) 376 } 377 rep := &dashapi.BugReport{ 378 Namespace: job.Namespace, 379 Config: reportingConfig, 380 ID: bugReporting.ID, 381 JobID: extJobID(jobKey), 382 ExtID: job.ExtID, 383 Title: bug.displayTitle(), 384 CC: job.CC, 385 Log: crashLog, 386 LogLink: externalLink(c, textCrashLog, job.CrashLog), 387 Report: report, 388 ReportLink: externalLink(c, textCrashReport, job.CrashReport), 389 OS: build.OS, 390 Arch: build.Arch, 391 VMArch: build.VMArch, 392 CompilerID: build.CompilerID, 393 KernelRepo: build.KernelRepo, 394 KernelRepoAlias: kernelRepoInfo(build).Alias, 395 KernelBranch: build.KernelBranch, 396 KernelCommit: build.KernelCommit, 397 KernelCommitTitle: build.KernelCommitTitle, 398 KernelCommitDate: build.KernelCommitDate, 399 KernelConfig: kernelConfig, 400 KernelConfigLink: externalLink(c, textKernelConfig, build.KernelConfig), 401 CrashTitle: job.CrashTitle, 402 Error: jobError, 403 ErrorLink: externalLink(c, textError, job.Error), 404 Patch: patch, 405 PatchLink: externalLink(c, textPatch, job.Patch), 406 } 407 return rep, nil 408} 409 410func jobReported(c context.Context, jobID string) error { 411 jobKey, err := jobID2Key(c, jobID) 412 if err != nil { 413 return err 414 } 415 tx := func(c context.Context) error { 416 job := new(Job) 417 if err := datastore.Get(c, jobKey, job); err != nil { 418 return fmt.Errorf("job %v: failed to get job: %v", jobID, err) 419 } 420 job.Reported = true 421 if _, err := datastore.Put(c, jobKey, job); err != nil { 422 return fmt.Errorf("failed to put job: %v", err) 423 } 424 return nil 425 } 426 return datastore.RunInTransaction(c, tx, nil) 427} 428 429func loadPendingJob(c context.Context, managers []string) (*Job, *datastore.Key, error) { 430 var jobs []*Job 431 keys, err := datastore.NewQuery("Job"). 432 Filter("Finished=", time.Time{}). 433 Order("Attempts"). 434 Order("Created"). 435 GetAll(c, &jobs) 436 if err != nil { 437 return nil, nil, fmt.Errorf("failed to query jobs: %v", err) 438 } 439 mgrs := make(map[string]bool) 440 for _, mgr := range managers { 441 mgrs[mgr] = true 442 } 443 for i, job := range jobs { 444 if !mgrs[job.Manager] { 445 continue 446 } 447 return job, keys[i], nil 448 } 449 return nil, nil, nil 450} 451 452func extJobID(jobKey *datastore.Key) string { 453 return fmt.Sprintf("%v|%v", jobKey.Parent().StringID(), jobKey.IntID()) 454} 455 456func jobID2Key(c context.Context, id string) (*datastore.Key, error) { 457 keyStr := strings.Split(id, "|") 458 if len(keyStr) != 2 { 459 return nil, fmt.Errorf("bad job id %q", id) 460 } 461 jobKeyID, err := strconv.ParseInt(keyStr[1], 10, 64) 462 if err != nil { 463 return nil, fmt.Errorf("bad job id %q", id) 464 } 465 bugKey := datastore.NewKey(c, "Bug", keyStr[0], 0, nil) 466 jobKey := datastore.NewKey(c, "Job", "", jobKeyID, bugKey) 467 return jobKey, nil 468} 469