1// Copyright 2017 syzkaller project authors. All rights reserved. 2// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file. 3 4package dash 5 6import ( 7 "bytes" 8 "encoding/json" 9 "fmt" 10 "io/ioutil" 11 "net/http" 12 "net/mail" 13 "regexp" 14 "strconv" 15 "strings" 16 "text/template" 17 "time" 18 19 "github.com/google/syzkaller/dashboard/dashapi" 20 "github.com/google/syzkaller/pkg/email" 21 "golang.org/x/net/context" 22 "google.golang.org/appengine" 23 "google.golang.org/appengine/log" 24 aemail "google.golang.org/appengine/mail" 25) 26 27// Email reporting interface. 28 29func initEmailReporting() { 30 http.HandleFunc("/email_poll", handleEmailPoll) 31 http.HandleFunc("/_ah/mail/", handleIncomingMail) 32 http.HandleFunc("/_ah/bounce", handleEmailBounce) 33 34 mailingLists = make(map[string]bool) 35 for _, cfg := range config.Namespaces { 36 for _, reporting := range cfg.Reporting { 37 if cfg, ok := reporting.Config.(*EmailConfig); ok { 38 mailingLists[email.CanonicalEmail(cfg.Email)] = true 39 } 40 } 41 } 42} 43 44const ( 45 emailType = "email" 46 // This plays an important role at least for job replies. 47 // If we CC a kernel mailing list and it uses Patchwork, 48 // then any emails with a patch attached create a new patch 49 // entry pending for review. The prefix makes Patchwork 50 // treat it as a comment for a previous patch. 51 replySubjectPrefix = "Re: " 52 commitHashLen = 12 53 commitTitleLen = 47 // so that whole line fits into 78 chars 54) 55 56var mailingLists map[string]bool 57 58type EmailConfig struct { 59 Email string 60 Moderation bool 61 MailMaintainers bool 62 DefaultMaintainers []string 63} 64 65func (cfg *EmailConfig) Type() string { 66 return emailType 67} 68 69func (cfg *EmailConfig) NeedMaintainers() bool { 70 return cfg.MailMaintainers && len(cfg.DefaultMaintainers) == 0 71} 72 73func (cfg *EmailConfig) Validate() error { 74 if _, err := mail.ParseAddress(cfg.Email); err != nil { 75 return fmt.Errorf("bad email address %q: %v", cfg.Email, err) 76 } 77 for _, email := range cfg.DefaultMaintainers { 78 if _, err := mail.ParseAddress(email); err != nil { 79 return fmt.Errorf("bad email address %q: %v", email, err) 80 } 81 } 82 if cfg.Moderation && cfg.MailMaintainers { 83 return fmt.Errorf("both Moderation and MailMaintainers set") 84 } 85 return nil 86} 87 88// handleEmailPoll is called by cron and sends emails for new bugs, if any. 89func handleEmailPoll(w http.ResponseWriter, r *http.Request) { 90 c := appengine.NewContext(r) 91 if err := emailPollBugs(c); err != nil { 92 log.Errorf(c, "bug poll failed: %v", err) 93 http.Error(w, err.Error(), http.StatusInternalServerError) 94 return 95 } 96 if err := emailPollJobs(c); err != nil { 97 log.Errorf(c, "job poll failed: %v", err) 98 http.Error(w, err.Error(), http.StatusInternalServerError) 99 return 100 } 101 w.Write([]byte("OK")) 102} 103 104func emailPollBugs(c context.Context) error { 105 reports := reportingPollBugs(c, emailType) 106 for _, rep := range reports { 107 cfg := new(EmailConfig) 108 if err := json.Unmarshal(rep.Config, cfg); err != nil { 109 log.Errorf(c, "failed to unmarshal email config: %v", err) 110 continue 111 } 112 if cfg.MailMaintainers { 113 rep.CC = email.MergeEmailLists(rep.CC, rep.Maintainers, cfg.DefaultMaintainers) 114 } 115 if err := emailReport(c, rep, "mail_bug.txt"); err != nil { 116 log.Errorf(c, "failed to report bug: %v", err) 117 continue 118 } 119 cmd := &dashapi.BugUpdate{ 120 ID: rep.ID, 121 Status: dashapi.BugStatusOpen, 122 ReproLevel: dashapi.ReproLevelNone, 123 CrashID: rep.CrashID, 124 } 125 if len(rep.ReproC) != 0 { 126 cmd.ReproLevel = dashapi.ReproLevelC 127 } else if len(rep.ReproSyz) != 0 { 128 cmd.ReproLevel = dashapi.ReproLevelSyz 129 } 130 ok, reason, err := incomingCommand(c, cmd) 131 if !ok || err != nil { 132 log.Errorf(c, "failed to update reported bug: ok=%v reason=%v err=%v", ok, reason, err) 133 } 134 } 135 return nil 136} 137 138func emailPollJobs(c context.Context) error { 139 jobs, err := pollCompletedJobs(c, emailType) 140 if err != nil { 141 return err 142 } 143 for _, job := range jobs { 144 if err := emailReport(c, job, "mail_test_result.txt"); err != nil { 145 log.Errorf(c, "failed to report job: %v", err) 146 continue 147 } 148 if err := jobReported(c, job.JobID); err != nil { 149 log.Errorf(c, "failed to mark job reported: %v", err) 150 continue 151 } 152 } 153 return nil 154} 155 156func emailReport(c context.Context, rep *dashapi.BugReport, templ string) error { 157 cfg := new(EmailConfig) 158 if err := json.Unmarshal(rep.Config, cfg); err != nil { 159 return fmt.Errorf("failed to unmarshal email config: %v", err) 160 } 161 to := email.MergeEmailLists([]string{cfg.Email}, rep.CC) 162 // Build error output and failing VM boot log can be way too long to inline. 163 if len(rep.Error) > maxInlineError { 164 rep.Error = rep.Error[len(rep.Error)-maxInlineError:] 165 } else { 166 rep.ErrorLink = "" 167 } 168 from, err := email.AddAddrContext(fromAddr(c), rep.ID) 169 if err != nil { 170 return err 171 } 172 creditEmail, err := email.AddAddrContext(ownEmail(c), rep.ID) 173 if err != nil { 174 return err 175 } 176 userspaceArch := "" 177 if rep.Arch == "386" { 178 userspaceArch = "i386" 179 } 180 link := fmt.Sprintf("%v/bug?extid=%v", appURL(c), rep.ID) 181 // Data passed to the template. 182 type BugReportData struct { 183 First bool 184 Link string 185 CreditEmail string 186 Moderation bool 187 Maintainers []string 188 CompilerID string 189 KernelRepo string 190 KernelCommit string 191 KernelCommitTitle string 192 KernelCommitDate string 193 UserSpaceArch string 194 CrashTitle string 195 Report []byte 196 Error []byte 197 ErrorLink string 198 LogLink string 199 KernelConfigLink string 200 ReproSyzLink string 201 ReproCLink string 202 NumCrashes int64 203 HappenedOn []string 204 PatchLink string 205 } 206 data := &BugReportData{ 207 First: rep.First, 208 Link: link, 209 CreditEmail: creditEmail, 210 Moderation: cfg.Moderation, 211 Maintainers: rep.Maintainers, 212 CompilerID: rep.CompilerID, 213 KernelRepo: rep.KernelRepoAlias, 214 KernelCommit: rep.KernelCommit, 215 KernelCommitTitle: rep.KernelCommitTitle, 216 KernelCommitDate: formatKernelTime(rep.KernelCommitDate), 217 UserSpaceArch: userspaceArch, 218 CrashTitle: rep.CrashTitle, 219 Report: rep.Report, 220 Error: rep.Error, 221 ErrorLink: rep.ErrorLink, 222 LogLink: rep.LogLink, 223 KernelConfigLink: rep.KernelConfigLink, 224 ReproSyzLink: rep.ReproSyzLink, 225 ReproCLink: rep.ReproCLink, 226 NumCrashes: rep.NumCrashes, 227 HappenedOn: rep.HappenedOn, 228 PatchLink: rep.PatchLink, 229 } 230 if len(data.KernelCommit) > commitHashLen { 231 data.KernelCommit = data.KernelCommit[:commitHashLen] 232 } 233 if len(data.KernelCommitTitle) > commitTitleLen { 234 data.KernelCommitTitle = data.KernelCommitTitle[:commitTitleLen-2] + ".." 235 } 236 log.Infof(c, "sending email %q to %q", rep.Title, to) 237 return sendMailTemplate(c, rep.Title, from, to, rep.ExtID, nil, templ, data) 238} 239 240// handleIncomingMail is the entry point for incoming emails. 241func handleIncomingMail(w http.ResponseWriter, r *http.Request) { 242 c := appengine.NewContext(r) 243 if err := incomingMail(c, r); err != nil { 244 log.Errorf(c, "%v", err) 245 } 246} 247 248func incomingMail(c context.Context, r *http.Request) error { 249 msg, err := email.Parse(r.Body, ownEmails(c)) 250 if err != nil { 251 return err 252 } 253 log.Infof(c, "received email: subject %q, from %q, cc %q, msg %q, bug %q, cmd %q, link %q", 254 msg.Subject, msg.From, msg.Cc, msg.MessageID, msg.BugID, msg.Command, msg.Link) 255 if msg.Command == "fix:" && msg.CommandArgs == "exact-commit-title" { 256 // Sometimes it happens that somebody sends us our own text back, ignore it. 257 msg.Command, msg.CommandArgs = "", "" 258 } 259 bug, _, reporting := loadBugInfo(c, msg) 260 if bug == nil { 261 return nil // error was already logged 262 } 263 emailConfig := reporting.Config.(*EmailConfig) 264 // A mailing list can send us a duplicate email, to not process/reply 265 // to such duplicate emails, we ignore emails coming from our mailing lists. 266 mailingList := email.CanonicalEmail(emailConfig.Email) 267 fromMailingList := email.CanonicalEmail(msg.From) == mailingList 268 mailingListInCC := checkMailingListInCC(c, msg, mailingList) 269 log.Infof(c, "from/cc mailing list: %v/%v", fromMailingList, mailingListInCC) 270 if msg.Command == "test:" { 271 args := strings.Split(msg.CommandArgs, " ") 272 if len(args) != 2 { 273 return replyTo(c, msg, fmt.Sprintf("want 2 args (repo, branch), got %v", 274 len(args)), nil) 275 } 276 reply := handleTestRequest(c, msg.BugID, email.CanonicalEmail(msg.From), 277 msg.MessageID, msg.Link, msg.Patch, args[0], args[1], msg.Cc) 278 if reply != "" { 279 return replyTo(c, msg, reply, nil) 280 } 281 return nil 282 } 283 if fromMailingList && msg.Command != "" { 284 log.Infof(c, "duplicate email from mailing list, ignoring") 285 return nil 286 } 287 cmd := &dashapi.BugUpdate{ 288 ID: msg.BugID, 289 ExtID: msg.MessageID, 290 Link: msg.Link, 291 CC: msg.Cc, 292 } 293 switch msg.Command { 294 case "": 295 cmd.Status = dashapi.BugStatusUpdate 296 case "upstream": 297 cmd.Status = dashapi.BugStatusUpstream 298 case "invalid": 299 cmd.Status = dashapi.BugStatusInvalid 300 case "undup": 301 cmd.Status = dashapi.BugStatusOpen 302 case "fix:": 303 if msg.CommandArgs == "" { 304 return replyTo(c, msg, fmt.Sprintf("no commit title"), nil) 305 } 306 cmd.Status = dashapi.BugStatusOpen 307 cmd.FixCommits = []string{msg.CommandArgs} 308 case "dup:": 309 if msg.CommandArgs == "" { 310 return replyTo(c, msg, fmt.Sprintf("no dup title"), nil) 311 } 312 cmd.Status = dashapi.BugStatusDup 313 cmd.DupOf = msg.CommandArgs 314 default: 315 return replyTo(c, msg, fmt.Sprintf("unknown command %q", msg.Command), nil) 316 } 317 ok, reply, err := incomingCommand(c, cmd) 318 if err != nil { 319 return nil // the error was already logged 320 } 321 if !ok && reply != "" { 322 return replyTo(c, msg, reply, nil) 323 } 324 if !mailingListInCC && msg.Command != "" { 325 warnMailingListInCC(c, msg, mailingList) 326 } 327 return nil 328} 329 330func handleEmailBounce(w http.ResponseWriter, r *http.Request) { 331 c := appengine.NewContext(r) 332 body, err := ioutil.ReadAll(r.Body) 333 if err != nil { 334 log.Errorf(c, "email bounced: failed to read body: %v", err) 335 return 336 } 337 if nonCriticalBounceRe.Match(body) { 338 log.Infof(c, "email bounced: address not found") 339 } else { 340 log.Errorf(c, "email bounced") 341 } 342 log.Infof(c, "%s", body) 343} 344 345// These are just stale emails in MAINTAINERS. 346var nonCriticalBounceRe = regexp.MustCompile(`\*\* Address not found \*\*|550 #5\.1\.0 Address rejected`) 347 348func loadBugInfo(c context.Context, msg *email.Email) (bug *Bug, bugReporting *BugReporting, reporting *Reporting) { 349 if msg.BugID == "" { 350 if msg.Command == "" { 351 // This happens when people CC syzbot on unrelated emails. 352 log.Infof(c, "no bug ID (%q)", msg.Subject) 353 } else { 354 log.Errorf(c, "no bug ID (%q)", msg.Subject) 355 if err := replyTo(c, msg, "Can't find the corresponding bug.", nil); err != nil { 356 log.Errorf(c, "failed to send reply: %v", err) 357 } 358 } 359 return nil, nil, nil 360 } 361 bug, _, err := findBugByReportingID(c, msg.BugID) 362 if err != nil { 363 log.Errorf(c, "can't find bug: %v", err) 364 if err := replyTo(c, msg, "Can't find the corresponding bug.", nil); err != nil { 365 log.Errorf(c, "failed to send reply: %v", err) 366 } 367 return nil, nil, nil 368 } 369 bugReporting, _ = bugReportingByID(bug, msg.BugID) 370 if bugReporting == nil { 371 log.Errorf(c, "can't find bug reporting: %v", err) 372 if err := replyTo(c, msg, "Can't find the corresponding bug.", nil); err != nil { 373 log.Errorf(c, "failed to send reply: %v", err) 374 } 375 return nil, nil, nil 376 } 377 reporting = config.Namespaces[bug.Namespace].ReportingByName(bugReporting.Name) 378 if reporting == nil { 379 log.Errorf(c, "can't find reporting for this bug: namespace=%q reporting=%q", 380 bug.Namespace, bugReporting.Name) 381 return nil, nil, nil 382 } 383 if reporting.Config.Type() != emailType { 384 log.Errorf(c, "reporting is not email: namespace=%q reporting=%q config=%q", 385 bug.Namespace, bugReporting.Name, reporting.Config.Type()) 386 return nil, nil, nil 387 } 388 return bug, bugReporting, reporting 389} 390 391func checkMailingListInCC(c context.Context, msg *email.Email, mailingList string) bool { 392 if email.CanonicalEmail(msg.From) == mailingList { 393 return true 394 } 395 for _, cc := range msg.Cc { 396 if email.CanonicalEmail(cc) == mailingList { 397 return true 398 } 399 } 400 msg.Cc = append(msg.Cc, mailingList) 401 return false 402} 403 404func warnMailingListInCC(c context.Context, msg *email.Email, mailingList string) { 405 reply := fmt.Sprintf("Your '%v' command is accepted, but please keep %v mailing list"+ 406 " in CC next time. It serves as a history of what happened with each bug report."+ 407 " Thank you.", 408 msg.Command, mailingList) 409 if err := replyTo(c, msg, reply, nil); err != nil { 410 log.Errorf(c, "failed to send email reply: %v", err) 411 } 412} 413 414func sendMailTemplate(c context.Context, subject, from string, to []string, replyTo string, 415 attachments []aemail.Attachment, template string, data interface{}) error { 416 body := new(bytes.Buffer) 417 if err := mailTemplates.ExecuteTemplate(body, template, data); err != nil { 418 return fmt.Errorf("failed to execute %v template: %v", template, err) 419 } 420 msg := &aemail.Message{ 421 Sender: from, 422 To: to, 423 Subject: subject, 424 Body: body.String(), 425 Attachments: attachments, 426 } 427 if replyTo != "" { 428 msg.Headers = mail.Header{"In-Reply-To": []string{replyTo}} 429 msg.Subject = replySubjectPrefix + msg.Subject 430 } 431 return sendEmail(c, msg) 432} 433 434func replyTo(c context.Context, msg *email.Email, reply string, attachment *aemail.Attachment) error { 435 var attachments []aemail.Attachment 436 if attachment != nil { 437 attachments = append(attachments, *attachment) 438 } 439 from, err := email.AddAddrContext(fromAddr(c), msg.BugID) 440 if err != nil { 441 return err 442 } 443 log.Infof(c, "sending reply: to=%q cc=%q subject=%q reply=%q", 444 msg.From, msg.Cc, msg.Subject, reply) 445 replyMsg := &aemail.Message{ 446 Sender: from, 447 To: []string{msg.From}, 448 Cc: msg.Cc, 449 Subject: replySubjectPrefix + msg.Subject, 450 Body: email.FormReply(msg.Body, reply), 451 Attachments: attachments, 452 Headers: mail.Header{"In-Reply-To": []string{msg.MessageID}}, 453 } 454 return sendEmail(c, replyMsg) 455} 456 457// Sends email, can be stubbed for testing. 458var sendEmail = func(c context.Context, msg *aemail.Message) error { 459 if err := aemail.Send(c, msg); err != nil { 460 return fmt.Errorf("failed to send email: %v", err) 461 } 462 return nil 463} 464 465func ownEmail(c context.Context) string { 466 return fmt.Sprintf("syzbot@%v.appspotmail.com", appengine.AppID(c)) 467} 468 469func fromAddr(c context.Context) string { 470 return fmt.Sprintf("\"syzbot\" <%v>", ownEmail(c)) 471} 472 473func ownEmails(c context.Context) []string { 474 // Now we use syzbot@ but we used to use bot@, so we add them both. 475 return []string{ 476 ownEmail(c), 477 fmt.Sprintf("bot@%v.appspotmail.com", appengine.AppID(c)), 478 } 479} 480 481func externalLink(c context.Context, tag string, id int64) string { 482 if id == 0 { 483 return "" 484 } 485 return fmt.Sprintf("%v/x/%v?x=%v", appURL(c), textFilename(tag), strconv.FormatUint(uint64(id), 16)) 486} 487 488func appURL(c context.Context) string { 489 return fmt.Sprintf("https://%v.appspot.com", appengine.AppID(c)) 490} 491 492func formatKernelTime(t time.Time) string { 493 if t.IsZero() { 494 return "" 495 } 496 // This is how dates appear in git log. 497 return t.Format("Mon Jan 2 15:04:05 2006 -0700") 498} 499 500func formatStringList(list []string) string { 501 return strings.Join(list, ", ") 502} 503 504var ( 505 mailTemplates = template.Must(template.New("").Funcs(mailFuncs).ParseGlob("mail_*.txt")) 506 507 mailFuncs = template.FuncMap{ 508 "formatTime": formatKernelTime, 509 "formatList": formatStringList, 510 } 511) 512