• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1// Copyright 2017 syzkaller project authors. All rights reserved.
2// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.
3
4package dash
5
6import (
7	"fmt"
8	"regexp"
9	"strconv"
10	"time"
11
12	"github.com/google/syzkaller/dashboard/dashapi"
13	"github.com/google/syzkaller/pkg/hash"
14	"golang.org/x/net/context"
15	"google.golang.org/appengine/datastore"
16)
17
18// This file contains definitions of entities stored in datastore.
19
20const (
21	maxTextLen   = 200
22	MaxStringLen = 1024
23
24	maxCrashes = 40
25)
26
27type Manager struct {
28	Namespace      string
29	Name           string
30	Link           string
31	CurrentBuild   string
32	FailedBuildBug string
33	LastAlive      time.Time
34	CurrentUpTime  time.Duration
35}
36
37// ManagerStats holds per-day manager runtime stats.
38// Has Manager as parent entity. Keyed by Date.
39type ManagerStats struct {
40	Date             int // YYYYMMDD
41	MaxCorpus        int64
42	MaxCover         int64
43	TotalFuzzingTime time.Duration
44	TotalCrashes     int64
45	TotalExecs       int64
46}
47
48type Build struct {
49	Namespace         string
50	Manager           string
51	ID                string // unique ID generated by syz-ci
52	Type              BuildType
53	Time              time.Time
54	OS                string
55	Arch              string
56	VMArch            string
57	SyzkallerCommit   string
58	CompilerID        string
59	KernelRepo        string
60	KernelBranch      string
61	KernelCommit      string
62	KernelCommitTitle string    `datastore:",noindex"`
63	KernelCommitDate  time.Time `datastore:",noindex"`
64	KernelConfig      int64     // reference to KernelConfig text entity
65}
66
67type Bug struct {
68	Namespace      string
69	Seq            int64 // sequences of the bug with the same title
70	Title          string
71	Status         int
72	DupOf          string
73	NumCrashes     int64
74	NumRepro       int64
75	ReproLevel     dashapi.ReproLevel
76	HasReport      bool
77	FirstTime      time.Time
78	LastTime       time.Time
79	LastSavedCrash time.Time
80	LastReproTime  time.Time
81	Closed         time.Time
82	Reporting      []BugReporting
83	Commits        []string
84	HappenedOn     []string `datastore:",noindex"` // list of managers
85	PatchedOn      []string `datastore:",noindex"` // list of managers
86}
87
88type BugReporting struct {
89	Name       string // refers to Reporting.Name
90	ID         string // unique ID per BUG/BugReporting used in commucation with external systems
91	ExtID      string // arbitrary reporting ID that is passed back in dashapi.BugReport
92	Link       string
93	CC         string // additional emails added to CC list (|-delimited list)
94	CrashID    int64  // crash that we've last reported in this reporting
95	ReproLevel dashapi.ReproLevel
96	Reported   time.Time
97	Closed     time.Time
98}
99
100type Crash struct {
101	Manager     string
102	BuildID     string
103	Time        time.Time
104	Reported    time.Time // set if this crash was ever reported
105	Maintainers []string  `datastore:",noindex"`
106	Log         int64     // reference to CrashLog text entity
107	Report      int64     // reference to CrashReport text entity
108	ReproOpts   []byte    `datastore:",noindex"`
109	ReproSyz    int64     // reference to ReproSyz text entity
110	ReproC      int64     // reference to ReproC text entity
111	// Custom crash priority for reporting (greater values are higher priority).
112	// For example, a crash in mainline kernel has higher priority than a crash in a side branch.
113	// For historical reasons this is called ReportLen.
114	ReportLen int64
115}
116
117// ReportingState holds dynamic info associated with reporting.
118type ReportingState struct {
119	Entries []ReportingStateEntry
120}
121
122type ReportingStateEntry struct {
123	Namespace string
124	Name      string
125	// Current reporting quota consumption.
126	Sent int
127	Date int // YYYYMMDD
128}
129
130// Job represent a single patch testing job for syz-ci.
131// Later we may want to extend this to other types of jobs (hense the generic name):
132//   - test of a committed fix
133//   - reproduce crash
134//   - test that crash still happens on HEAD
135//   - crash bisect
136// Job has Bug as parent entity.
137type Job struct {
138	Created   time.Time
139	User      string
140	CC        []string
141	Reporting string
142	ExtID     string // email Message-ID
143	Link      string // web link for the job (e.g. email in the group)
144	Namespace string
145	Manager   string
146	BugTitle  string
147	CrashID   int64
148
149	// Provided by user:
150	KernelRepo   string
151	KernelBranch string
152	Patch        int64 // reference to Patch text entity
153
154	Attempts int // number of times we tried to execute this job
155	Started  time.Time
156	Finished time.Time // if set, job is finished
157
158	// Result of execution:
159	CrashTitle  string // if empty, we did not hit crash during testing
160	CrashLog    int64  // reference to CrashLog text entity
161	CrashReport int64  // reference to CrashReport text entity
162	BuildID     string
163	Error       int64 // reference to Error text entity, if set job failed
164
165	Reported bool // have we reported result back to user?
166}
167
168// Text holds text blobs (crash logs, reports, reproducers, etc).
169type Text struct {
170	Namespace string
171	Text      []byte `datastore:",noindex"` // gzip-compressed text
172}
173
174const (
175	textCrashLog     = "CrashLog"
176	textCrashReport  = "CrashReport"
177	textReproSyz     = "ReproSyz"
178	textReproC       = "ReproC"
179	textKernelConfig = "KernelConfig"
180	textPatch        = "Patch"
181	textError        = "Error"
182)
183
184const (
185	BugStatusOpen = iota
186)
187
188const (
189	BugStatusFixed = 1000 + iota
190	BugStatusInvalid
191	BugStatusDup
192)
193
194const (
195	ReproLevelNone = dashapi.ReproLevelNone
196	ReproLevelSyz  = dashapi.ReproLevelSyz
197	ReproLevelC    = dashapi.ReproLevelC
198)
199
200type BuildType int
201
202const (
203	BuildNormal BuildType = iota
204	BuildFailed
205	BuildJob
206)
207
208// updateManager does transactional compare-and-swap on the manager and its current stats.
209func updateManager(c context.Context, ns, name string, fn func(mgr *Manager, stats *ManagerStats)) error {
210	date := timeDate(timeNow(c))
211	tx := func(c context.Context) error {
212		mgr := new(Manager)
213		mgrKey := datastore.NewKey(c, "Manager", fmt.Sprintf("%v-%v", ns, name), 0, nil)
214		if err := datastore.Get(c, mgrKey, mgr); err != nil {
215			if err != datastore.ErrNoSuchEntity {
216				return fmt.Errorf("failed to get manager %v/%v: %v", ns, name, err)
217			}
218			mgr = &Manager{
219				Namespace: ns,
220				Name:      name,
221			}
222		}
223		stats := new(ManagerStats)
224		statsKey := datastore.NewKey(c, "ManagerStats", "", int64(date), mgrKey)
225		if err := datastore.Get(c, statsKey, stats); err != nil {
226			if err != datastore.ErrNoSuchEntity {
227				return fmt.Errorf("failed to get stats %v/%v/%v: %v", ns, name, date, err)
228			}
229			stats = &ManagerStats{
230				Date: date,
231			}
232		}
233
234		fn(mgr, stats)
235
236		if _, err := datastore.Put(c, mgrKey, mgr); err != nil {
237			return fmt.Errorf("failed to put manager: %v", err)
238		}
239		if _, err := datastore.Put(c, statsKey, stats); err != nil {
240			return fmt.Errorf("failed to put manager stats: %v", err)
241		}
242		return nil
243	}
244	return datastore.RunInTransaction(c, tx, &datastore.TransactionOptions{Attempts: 10})
245}
246
247func loadAllManagers(c context.Context) ([]*Manager, []*datastore.Key, error) {
248	var managers []*Manager
249	keys, err := datastore.NewQuery("Manager").
250		GetAll(c, &managers)
251	if err != nil {
252		return nil, nil, fmt.Errorf("failed to query managers: %v", err)
253	}
254	var result []*Manager
255	var resultKeys []*datastore.Key
256
257	for i, mgr := range managers {
258		if config.Namespaces[mgr.Namespace].Managers[mgr.Name].Decommissioned {
259			continue
260		}
261		result = append(result, mgr)
262		resultKeys = append(resultKeys, keys[i])
263	}
264	return result, resultKeys, nil
265}
266
267func buildKey(c context.Context, ns, id string) *datastore.Key {
268	if ns == "" {
269		panic("requesting build key outside of namespace")
270	}
271	h := hash.String([]byte(fmt.Sprintf("%v-%v", ns, id)))
272	return datastore.NewKey(c, "Build", h, 0, nil)
273}
274
275func loadBuild(c context.Context, ns, id string) (*Build, error) {
276	build := new(Build)
277	if err := datastore.Get(c, buildKey(c, ns, id), build); err != nil {
278		if err == datastore.ErrNoSuchEntity {
279			return nil, fmt.Errorf("unknown build %v/%v", ns, id)
280		}
281		return nil, fmt.Errorf("failed to get build %v/%v: %v", ns, id, err)
282	}
283	return build, nil
284}
285
286func lastManagerBuild(c context.Context, ns, manager string) (*Build, error) {
287	var builds []*Build
288	_, err := datastore.NewQuery("Build").
289		Filter("Namespace=", ns).
290		Filter("Manager=", manager).
291		Filter("Type=", BuildNormal).
292		Order("-Time").
293		Limit(1).
294		GetAll(c, &builds)
295	if err != nil {
296		return nil, fmt.Errorf("failed to fetch manager build: %v", err)
297	}
298	if len(builds) == 0 {
299		return nil, fmt.Errorf("failed to fetch manager build: no builds")
300	}
301	return builds[0], nil
302}
303
304func (bug *Bug) displayTitle() string {
305	if bug.Seq == 0 {
306		return bug.Title
307	}
308	return fmt.Sprintf("%v (%v)", bug.Title, bug.Seq+1)
309}
310
311var displayTitleRe = regexp.MustCompile(`^(.*) \(([0-9]+)\)$`)
312
313func splitDisplayTitle(display string) (string, int64, error) {
314	match := displayTitleRe.FindStringSubmatchIndex(display)
315	if match == nil {
316		return display, 0, nil
317	}
318	title := display[match[2]:match[3]]
319	seqStr := display[match[4]:match[5]]
320	seq, err := strconv.ParseInt(seqStr, 10, 64)
321	if err != nil {
322		return "", 0, fmt.Errorf("failed to parse bug title: %v", err)
323	}
324	if seq <= 0 || seq > 1e6 {
325		return "", 0, fmt.Errorf("failed to parse bug title: seq=%v", seq)
326	}
327	return title, seq - 1, nil
328}
329
330func canonicalBug(c context.Context, bug *Bug) (*Bug, error) {
331	for {
332		if bug.Status != BugStatusDup {
333			return bug, nil
334		}
335		canon := new(Bug)
336		bugKey := datastore.NewKey(c, "Bug", bug.DupOf, 0, nil)
337		if err := datastore.Get(c, bugKey, canon); err != nil {
338			return nil, fmt.Errorf("failed to get dup bug %q for %q: %v",
339				bug.DupOf, bugKeyHash(bug.Namespace, bug.Title, bug.Seq), err)
340		}
341		bug = canon
342	}
343}
344
345func bugKeyHash(ns, title string, seq int64) string {
346	return hash.String([]byte(fmt.Sprintf("%v-%v-%v-%v", config.Namespaces[ns].Key, ns, title, seq)))
347}
348
349func bugReportingHash(bugHash, reporting string) string {
350	// Since these IDs appear in Reported-by tags in commit, we slightly limit their size.
351	const hashLen = 20
352	return hash.String([]byte(fmt.Sprintf("%v-%v", bugHash, reporting)))[:hashLen]
353}
354
355func kernelRepoInfo(build *Build) KernelRepo {
356	return kernelRepoInfoRaw(build.KernelRepo, build.KernelBranch)
357}
358
359func kernelRepoInfoRaw(repo, branch string) KernelRepo {
360	repoID := repo
361	if branch != "" {
362		repoID += "/" + branch
363	}
364	info := config.KernelRepos[repoID]
365	if info.Alias == "" {
366		info.Alias = repoID
367	}
368	return info
369}
370
371func textLink(tag string, id int64) string {
372	if id == 0 {
373		return ""
374	}
375	return fmt.Sprintf("/text?tag=%v&x=%v", tag, strconv.FormatUint(uint64(id), 16))
376}
377
378// timeDate returns t's date as a single int YYYYMMDD.
379func timeDate(t time.Time) int {
380	year, month, day := t.Date()
381	return year*10000 + int(month)*100 + day
382}
383