dashboard/app: integrate tail reports

dashboard/app/api.go

@@ -916,6 +916,9 @@ func reportCrash(c context.Context, build *Build, req *dashapi.Crash) (*Bug, err
 			bug.SetAutoSubsystems(c, newSubsystems, now, subsystemService.Revision)
 		}
 		bug.increaseCrashStats(now)
+		if err = bug.addTitleStat(req.TailTitles); err != nil {
+			return fmt.Errorf("failed to add title stat: %w", err)
+		}
 		bug.HappenedOn = mergeString(bug.HappenedOn, build.Manager)
 		// Migration of older entities (for new bugs Title is always in MergedTitles).
 		bug.MergedTitles = mergeString(bug.MergedTitles, bug.Title)
@@ -975,10 +978,11 @@ func (crash *Crash) UpdateReportingPriority(c context.Context, build *Build, bug
 func saveCrash(c context.Context, ns string, req *dashapi.Crash, bug *Bug, bugKey *db.Key,
 	build *Build, assets []Asset) error {
 	crash := &Crash{
-		Title:   req.Title,
-		Manager: build.Manager,
-		BuildID: req.BuildID,
-		Time:    timeNow(c),
+		Title:      req.Title,
+		TailTitles: req.TailTitles,
+		Manager:    build.Manager,
+		BuildID:    req.BuildID,
+		Time:       timeNow(c),
 		Maintainers: email.MergeEmailLists(req.Maintainers,
 			GetEmails(req.Recipients, dashapi.To),
 			GetEmails(req.Recipients, dashapi.Cc)),
@@ -996,6 +1000,13 @@ func saveCrash(c context.Context, ns string, req *dashapi.Crash, bug *Bug, bugKe
 	if crash.Report, err = putText(c, ns, textCrashReport, req.Report); err != nil {
 		return err
 	}
+	for _, tailReport := range req.TailReports {
+		tailReportID, err := putText(c, ns, textCrashReport, tailReport)
+		if err != nil {
+			return err
+		}
+		crash.TailReports = append(crash.TailReports, tailReportID)
+	}
 	if crash.ReproSyz, err = putText(c, ns, textReproSyz, req.ReproSyz); err != nil {
 		return err
 	}
@@ -1073,6 +1084,11 @@ func purgeOldCrashes(c context.Context, bug *Bug, bugKey *db.Key) {
 		if crash.Report != 0 {
 			toDelete = append(toDelete, db.NewKey(c, textCrashReport, "", crash.Report, nil))
 		}
+		if len(crash.TailReports) != 0 {
+			for _, tailReport := range crash.TailReports {
+				toDelete = append(toDelete, db.NewKey(c, textCrashReport, "", tailReport, nil))
+			}
+		}
 		if crash.ReproSyz != 0 {
 			toDelete = append(toDelete, db.NewKey(c, textReproSyz, "", crash.ReproSyz, nil))
 		}

dashboard/app/app_test.go

@@ -16,9 +16,11 @@ import (
 	"github.com/google/go-cmp/cmp"
 	"github.com/google/syzkaller/dashboard/dashapi"
 	"github.com/google/syzkaller/pkg/auth"
+	"github.com/google/syzkaller/pkg/report"
 	"github.com/google/syzkaller/pkg/subsystem"
 	_ "github.com/google/syzkaller/pkg/subsystem/lists"
 	"github.com/google/syzkaller/sys/targets"
+	"github.com/stretchr/testify/assert"
 	"google.golang.org/appengine/v2/user"
 )
 
@@ -1139,3 +1141,43 @@ kernel BUG at <a href='https://github.com/google/syzkaller/blob/111222/fs/ext4/i
 		t.Fatal(diff)
 	}
 }
+
+func TestTailReports(t *testing.T) {
+	c := NewCtx(t)
+	defer c.Close()
+
+	build := testBuild(1)
+	err := c.client.UploadBuild(build)
+	assert.NoError(t, err)
+
+	crash := testCrash(build, 1)
+	_, err = c.client.ReportCrash(crash)
+	assert.NoError(t, err)
+	c.client.pollBug()
+
+	const someKASANTitle = "KASAN: slab-out-of-bounds Write in foo"
+	crash.TailTitles = []string{someKASANTitle}
+	crash.TailReports = [][]byte{[]byte("tail report")}
+	_, err = c.client.ReportCrash(crash)
+	assert.NoError(t, err)
+
+	res, _ := c.GET("/test1")
+	wantRank := fmt.Sprintf("[rank %d, freq  50.0%%] %s", report.TitlesToImpact(someKASANTitle), someKASANTitle)
+	if !strings.Contains(string(res), wantRank) {
+		t.Logf("%s", res)
+		t.Errorf("can't find rank string %q", wantRank)
+	}
+
+	res, _ = c.GET("/bug?extid=decf42d66dced481afc1")
+
+	// Bug page crash table has 2 lines. Checking the most complex only.
+	wantTitles := fmt.Sprintf(">%s<br>%s<", crash.Title, crash.TailTitles[0])
+	if !strings.Contains(string(res), wantTitles) {
+		t.Logf("%s", res)
+		t.Errorf("can't find titles string %q", wantTitles)
+	}
+	if !strings.Contains(string(res), ">tail report 1<") {
+		t.Logf("%s", res)
+		t.Error("can't find tail report string")
+	}
+}

dashboard/app/entities_datastore.go

@@ -13,6 +13,7 @@ import (
 
 	"github.com/google/syzkaller/dashboard/dashapi"
 	"github.com/google/syzkaller/pkg/hash"
+	"github.com/google/syzkaller/pkg/report"
 	"github.com/google/syzkaller/pkg/subsystem"
 	db "google.golang.org/appengine/v2/datastore"
 )
@@ -91,6 +92,7 @@ type Bug struct {
 	Title        string
 	MergedTitles []string // crash titles that we already merged into this bug
 	AltTitles    []string // alternative crash titles that we may merge into this bug
+	TitleStat    string   `datastore:",noindex"` // serialized report.TitleStat
 	Status       int
 	StatusReason dashapi.BugStatusReason // e.g. if the bug status is "invalid", here's the reason why
 	DupOf        string
@@ -348,6 +350,7 @@ type Crash struct {
 	// May be different from bug.Title due to AltTitles.
 	// May be empty for old bugs, in such case bug.Title is the right title.
 	Title           string
+	TailTitles      []string
 	Manager         string
 	BuildID         string
 	Time            time.Time
@@ -357,6 +360,7 @@ type Crash struct {
 	Log             int64               // reference to CrashLog text entity
 	Flags           int64               // properties of the Crash
 	Report          int64               // reference to CrashReport text entity
+	TailReports     []int64             // references to CrashReport text entity
 	ReportElements  CrashReportElements // parsed parts of the crash report
 	ReproOpts       []byte              `datastore:",noindex"`
 	ReproSyz        int64               // reference to ReproSyz text entity
@@ -985,6 +989,20 @@ func (bug *Bug) increaseCrashStats(now time.Time) {
 	}
 }
 
+func (bug *Bug) addTitleStat(titles []string) error {
+	ts, err := report.TitleStatFromBytes([]byte(bug.TitleStat))
+	if err != nil {
+		return err
+	}
+	ts.Add(append([]string{bug.Title}, titles...))
+	bytes, err := ts.ToBytes()
+	if err != nil {
+		return err
+	}
+	bug.TitleStat = string(bytes)
+	return nil
+}
+
 func (bug *Bug) dailyStatsTail(from time.Time) []BugDailyStats {
 	startDate := timeDate(from)
 	startPos := len(bug.DailyStats)

dashboard/app/main.go

@@ -368,6 +368,7 @@ type uiBug struct {
 	Namespace      string
 	Title          string
 	ImpactScore    int
+	RankTooltip    string
 	NumCrashes     int64
 	NumCrashesBad  bool
 	BisectCause    BisectStatus
@@ -398,19 +399,21 @@ type uiBugLabel struct {
 }
 
 type uiCrash struct {
-	Title           string
-	Manager         string
-	Time            time.Time
-	Maintainers     string
-	LogLink         string
-	LogHasStrace    bool
-	ReportLink      string
-	ReproSyzLink    string
-	ReproCLink      string
-	ReproIsRevoked  bool
-	ReproLogLink    string
-	MachineInfoLink string
-	Assets          []*uiAsset
+	Title            string
+	TailTitles       []string
+	Manager          string
+	Time             time.Time
+	Maintainers      string
+	LogLink          string
+	LogHasStrace     bool
+	ReportLink       string
+	TailReportsLinks []string
+	ReproSyzLink     string
+	ReproCLink       string
+	ReproIsRevoked   bool
+	ReproLogLink     string
+	MachineInfoLink  string
+	Assets           []*uiAsset
 	*uiBuild
 }
 
@@ -1938,10 +1941,16 @@ func createUIBug(c context.Context, bug *Bug, state *ReportingState, managers []
 			log.Errorf(c, "failed to generate credit email: %v", err)
 		}
 	}
+
+	titleStat, err := report.TitleStatFromBytes([]byte(bug.TitleStat))
+	if err != nil {
+		log.Errorf(c, "report.TitleStatFromBytes: %v", err)
+	}
 	uiBug := &uiBug{
 		Namespace:      bug.Namespace,
 		Title:          bug.displayTitle(),
 		ImpactScore:    report.TitlesToImpact(bug.Title, bug.AltTitles...),
+		RankTooltip:    report.HigherRankTooltip(bug.Title, titleStat.Explain()),
 		BisectCause:    bug.BisectCause,
 		BisectFix:      bug.BisectFix,
 		NumCrashes:     bug.NumCrashes,
@@ -2076,20 +2085,26 @@ func makeUIAssets(c context.Context, build *Build, crash *Crash, forReport bool)
 }
 
 func makeUICrash(c context.Context, crash *Crash, build *Build) *uiCrash {
+	var tailReportsLinks []string
+	for _, tailReportID := range crash.TailReports {
+		tailReportsLinks = append(tailReportsLinks, textLink(textCrashReport, tailReportID))
+	}
 	ui := &uiCrash{
-		Title:           crash.Title,
-		Manager:         crash.Manager,
-		Time:            crash.Time,
-		Maintainers:     strings.Join(crash.Maintainers, ", "),
-		LogLink:         textLink(textCrashLog, crash.Log),
-		LogHasStrace:    dashapi.CrashFlags(crash.Flags)&dashapi.CrashUnderStrace > 0,
-		ReportLink:      textLink(textCrashReport, crash.Report),
-		ReproSyzLink:    textLink(textReproSyz, crash.ReproSyz),
-		ReproCLink:      textLink(textReproC, crash.ReproC),
-		ReproLogLink:    textLink(textReproLog, crash.ReproLog),
-		ReproIsRevoked:  crash.ReproIsRevoked,
-		MachineInfoLink: textLink(textMachineInfo, crash.MachineInfo),
-		Assets:          makeUIAssets(c, build, crash, true),
+		Title:            crash.Title,
+		TailTitles:       crash.TailTitles,
+		Manager:          crash.Manager,
+		Time:             crash.Time,
+		Maintainers:      strings.Join(crash.Maintainers, ", "),
+		LogLink:          textLink(textCrashLog, crash.Log),
+		LogHasStrace:     dashapi.CrashFlags(crash.Flags)&dashapi.CrashUnderStrace > 0,
+		ReportLink:       textLink(textCrashReport, crash.Report),
+		TailReportsLinks: tailReportsLinks,
+		ReproSyzLink:     textLink(textReproSyz, crash.ReproSyz),
+		ReproCLink:       textLink(textReproC, crash.ReproC),
+		ReproLogLink:     textLink(textReproLog, crash.ReproLog),
+		ReproIsRevoked:   crash.ReproIsRevoked,
+		MachineInfoLink:  textLink(textMachineInfo, crash.MachineInfo),
+		Assets:           makeUIAssets(c, build, crash, true),
 	}
 	if build != nil {
 		ui.uiBuild = makeUIBuild(c, build, true)

dashboard/app/templates/templates.html

@@ -206,7 +206,14 @@ <h1><a href="/{{$.Namespace}}">syzbot</a></h1>
 					<span class="bug-label">{{link .Link .Name}}</span>
 				{{- end}}
 			</td>
-			<td class="stat">{{$b.ImpactScore}}</td>
+			<td class="rank">
+				{{if $b.RankTooltip}}
+					<b>{{$b.ImpactScore}}</b>
+					<pre class="tooltiptext">{{$b.RankTooltip}}</pre>
+				{{else}}
+					{{$b.ImpactScore}}
+				{{end}}
+			</td>
 			<td class="stat">{{formatReproLevel $b.ReproLevel}}</td>
 			<td class="bisect_status">{{print $b.BisectCause}}</td>
 			<td class="bisect_status">{{print $b.BisectFix}}</td>
@@ -456,15 +463,27 @@ <h1><a href="/{{$.Namespace}}">syzbot</a></h1>
 			<td class="tag">{{link $b.SyzkallerCommitLink (formatShortHash $b.SyzkallerCommit)}}</td>
 			<td class="config">{{if $b.KernelConfigLink}}<a href="{{$b.KernelConfigLink}}">.config</a>{{end}}</td>
 			<td class="repro">{{if $b.LogLink}}<a href="{{$b.LogLink}}">{{if $b.LogHasStrace}}strace{{else}}console{{end}} log</a>{{end}}</td>
-			<td class="repro">{{if $b.ReportLink}}<a href="{{$b.ReportLink}}">report</a>{{end}}</td>
+			<td class="repro">
+				{{- if $b.ReportLink -}}
+					<a href="{{$b.ReportLink}}">report</a>
+				{{- end -}}
+				{{- range $i, $trl := $b.TailReportsLinks -}}
+					<br><a href="{{$trl}}">tail report {{add $i 1}}</a>
+				{{- end -}}
+			</td>
 			<td class="repro{{if $b.ReproIsRevoked}} stale_repro{{end}}">{{if $b.ReproSyzLink}}<a href="{{$b.ReproSyzLink}}">syz</a>{{end}}{{if $b.ReproLogLink}} / <a href="{{$b.ReproLogLink}}">log</a>{{end}}</td>
 			<td class="repro{{if $b.ReproIsRevoked}} stale_repro{{end}}">{{if $b.ReproCLink}}<a href="{{$b.ReproCLink}}">C</a>{{end}}</td>
 			<td class="repro">{{if $b.MachineInfoLink}}<a href="{{$b.MachineInfoLink}}">info</a>{{end}}</td>
 			<td class="assets">{{range $i, $asset := .Assets}}
 				<span class="no-break">[<a href="{{$asset.DownloadURL}}">{{$asset.Title}}</a>{{if $asset.FsckLogURL}} (<a href="{{$asset.FsckLogURL}}">{{if $asset.FsIsClean}}clean{{else}}corrupt{{end}} fs</a>){{end}}]</span>
 			{{end}}</td>
 			<td class="manager">{{$b.Manager}}</td>
-			<td class="manager">{{$b.Title}}</td>
+			<td class="manager">
+				{{- $b.Title}}
+				{{- range $tt := $b.TailTitles -}}
+				<br>{{$tt}}
+				{{- end -}}
+			</td>
 		</tr>
 		{{end}}
 		</tbody>

dashboard/dashapi/dashapi.go

@@ -322,13 +322,17 @@ type Crash struct {
 	BuildID     string // refers to Build.ID
 	Title       string
 	AltTitles   []string // alternative titles, used for better deduplication
+	TailTitles  []string // titles of the tail reports, see TailReports field
 	Corrupted   bool     // report is corrupted (corrupted title, no stacks, etc)
 	Suppressed  bool
 	Maintainers []string // deprecated in favor of Recipients
 	Recipients  Recipients
 	Log         []byte
 	Flags       CrashFlags
 	Report      []byte
+	// Crashing machine may generate report chain like WARNING -> WARNING -> KASAN -> GPF.
+	// These additional reports are used to better understand the bug nature and impact.
+	TailReports [][]byte
 	MachineInfo []byte
 	Assets      []NewAsset
 	GuiltyFiles []string

pkg/html/html.go

@@ -52,24 +52,27 @@ func CreateTextGlob(glob string) *texttemplate.Template {
 }
 
 var Funcs = template.FuncMap{
-	"link":                   link,
-	"optlink":                optlink,
-	"formatTime":             FormatTime,
-	"formatDate":             FormatDate,
-	"formatKernelTime":       formatKernelTime,
-	"formatJSTime":           formatJSTime,
+	// keep-sorted start
+	"add":                    func(a, b int) int { return a + b },
+	"commitLink":             commitLink,
+	"dereference":            dereferencePointer,
 	"formatClock":            formatClock,
+	"formatCommitTableTitle": formatCommitTableTitle,
+	"formatDate":             FormatDate,
 	"formatDuration":         formatDuration,
+	"formatJSTime":           formatJSTime,
+	"formatKernelTime":       formatKernelTime,
 	"formatLateness":         formatLateness,
+	"formatList":             formatStringList,
 	"formatReproLevel":       formatReproLevel,
-	"formatStat":             formatStat,
 	"formatShortHash":        formatShortHash,
+	"formatStat":             formatStat,
 	"formatTagHash":          formatTagHash,
-	"formatCommitTableTitle": formatCommitTableTitle,
-	"formatList":             formatStringList,
+	"formatTime":             FormatTime,
+	"link":                   link,
+	"optlink":                optlink,
 	"selectBisect":           selectBisect,
-	"dereference":            dereferencePointer,
-	"commitLink":             commitLink,
+	// keep-sorted end
 }
 
 func selectBisect(rep *dashapi.BugReport) *dashapi.BisectResult {

pkg/manager/crash.go

@@ -91,8 +91,10 @@ func (cs *CrashStore) SaveCrash(crash *Crash) (bool, error) {
 	writeOrRemove("tag", []byte(cs.Tag))
 	writeOrRemove("report", report.MergeReportBytes(reps))
 	writeOrRemove("machineInfo", crash.MachineInfo)
-	if err := report.AddTitleStat(filepath.Join(dir, "title-stat"), reps); err != nil {
-		return false, fmt.Errorf("report.AddTitleStat: %w", err)
+	titleStatPath := filepath.Join(dir, "title-stat")
+	titles := append([]string{crash.Title}, crash.TailTitles()...)
+	if err := report.AddTitlesToStatFile(titleStatPath, titles); err != nil {
+		return false, fmt.Errorf("report.AddTitlesToStatFile: %w", err)
 	}
 
 	return first, nil
@@ -243,11 +245,13 @@ func (cs *CrashStore) BugInfo(id string, full bool) (*BugInfo, error) {
 
 	// Bug rank may go up over time if we observe higher ranked bugs as a consequence of the first failure.
 	ret.Rank = report.TitlesToImpact(ret.Title)
-	if titleStat, err := report.ReadStatFile(filepath.Join(dir, "title-stat")); err == nil {
-		ret.TailTitles = report.ExplainTitleStat(titleStat)
-		for _, ti := range ret.TailTitles {
-			ret.Rank = max(ret.Rank, ti.Rank)
-		}
+	titleStat, err := report.ReadStatFile(filepath.Join(dir, "title-stat"))
+	if err != nil {
+		return nil, err
+	}
+	ret.TailTitles = titleStat.Explain()
+	for _, ti := range ret.TailTitles {
+		ret.Rank = max(ret.Rank, ti.Rank)
 	}
 
 	ret.FirstTime = osutil.CreationTime(stat)