Dracon producer to parse yarn audit --json jsonlines (#126)

* Yarn audit producer

* Tidy up

* Correct docker image

* Fix typo

* Rename structs

* Fix Linting

* Make most types private
* Add public comments

* Fix tests on type rename

* Correct build comment

* Update main error handling

Co-authored-by: bintal <[email protected]>

producers/producer.go

@@ -3,6 +3,7 @@
 package producers
 
 import (
+	"bufio"
 	"bytes"
 	"encoding/json"
 	"flag"
@@ -53,12 +54,32 @@ func ParseFlags() error {
 	return nil
 }
 
+// ReadLines returns the lines of the contents of the file given by InResults
+func ReadLines() ([][]byte, error) {
+	file, err := os.Open(InResults)
+	if err != nil {
+		return nil, err
+	}
+	defer file.Close()
+
+	scanner := bufio.NewScanner(file)
+
+	var result [][]byte
+
+	for scanner.Scan() {
+		result = append(result, scanner.Bytes())
+	}
+
+	return result, nil
+}
+
 // ReadInFile returns the contents of the file given by InResults.
 func ReadInFile() ([]byte, error) {
 	file, err := os.Open(InResults)
 	if err != nil {
 		return nil, err
 	}
+	defer file.Close()
 
 	buffer := new(bytes.Buffer)
 	buffer.ReadFrom(file)

producers/yarn_audit/BUILD

@@ -0,0 +1,24 @@
+subinclude("//third_party/defs:docker")
+
+# this producer covers yarn audit https://classic.yarnpkg.com/lang/en/docs/cli/audit/
+
+go_binary(
+    name = "yarn_audit",
+    srcs = [
+        "main.go",
+    ],
+    deps = [
+        "//api/proto:v1",
+        "//producers",
+        "//producers/yarn_audit/types",
+    ],
+)
+
+docker_image(
+    name = "dracon-producer-yarn-audit",
+    srcs = [
+        ":yarn_audit",
+    ],
+    base_image = "//build/docker:dracon-base-go",
+    image = "dracon-producer-yarn-audit",
+)

producers/yarn_audit/Dockerfile

@@ -0,0 +1,5 @@
+FROM //build/docker:dracon-base-go
+
+COPY yarn_audit /parse
+
+ENTRYPOINT ["/parse"]

producers/yarn_audit/README.md

@@ -0,0 +1 @@
+For use with output of `yarn audit --json`

producers/yarn_audit/main.go

@@ -0,0 +1,40 @@
+package main
+
+import (
+	"github.com/thought-machine/dracon/producers"
+	"github.com/thought-machine/dracon/producers/yarn_audit/types"
+
+	"log"
+)
+
+func main() {
+	if err := producers.ParseFlags(); err != nil {
+		log.Fatal(err)
+	}
+
+	inLines, err := producers.ReadLines()
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	report, errors := types.NewReport(inLines)
+
+	// Individual errors should already be printed to logs
+	if len(errors) > 0 {
+		errorMessage := "Errors creating Yarn Audit report: %d"
+		if report != nil{
+			log.Printf(errorMessage, len(errors))
+		} else {
+			log.Fatalf(errorMessage, len(errors))
+		}
+	}
+
+	if report != nil {
+		if err := producers.WriteDraconOut(
+			"yarn-audit",
+			report.AsIssues(),
+		); err != nil {
+			log.Fatal(err)
+		}
+	}
+}

producers/yarn_audit/types/BUILD

@@ -0,0 +1,24 @@
+go_library(
+    name = "types",
+    srcs = [
+        "yarn-issue.go",
+    ],
+    visibility = ["//producers/yarn_audit/..."],
+    deps = [
+        "//api/proto:v1",
+        "//producers",
+    ],
+)
+
+go_test(
+    name = "types_test",
+    srcs = [
+        "yarn-issue_test.go",
+    ],
+    deps = [
+        ":types",
+        "//api/proto:v1",
+        "//producers",
+        "//third_party/go:stretchr_testify",
+    ],
+)

producers/yarn_audit/types/yarn-issue.go

@@ -0,0 +1,225 @@
+package types
+
+import (
+	"encoding/json"
+	"fmt"
+	"strings"
+
+	v1 "github.com/thought-machine/dracon/api/proto/v1"
+	"github.com/thought-machine/dracon/producers"
+
+	"log"
+)
+
+func yarnToIssueSeverity(severity string) v1.Severity {
+
+	switch severity {
+	case "low":
+		return v1.Severity_SEVERITY_LOW
+	case "moderate":
+		return v1.Severity_SEVERITY_MEDIUM
+	case "high":
+		return v1.Severity_SEVERITY_HIGH
+	case "critical":
+		return v1.Severity_SEVERITY_CRITICAL
+	default:
+		return v1.Severity_SEVERITY_INFO
+
+	}
+}
+
+type yarnAuditLine struct {
+	Type string      `json:"type"`
+	Data interface{} `json:"data"`
+}
+
+func (yl *yarnAuditLine) UnmarshalJSON(data []byte) error {
+	var typ struct {
+		Type string `json:"type"`
+	}
+
+	if err := json.Unmarshal(data, &typ); err != nil {
+		return err
+	}
+
+	switch typ.Type {
+	case "auditSummary":
+		yl.Data = new(auditSummaryData)
+	case "auditAdvisory":
+		yl.Data = new(auditAdvisoryData)
+	case "auditAction":
+		yl.Data = new(auditActionData)
+	default:
+		log.Printf("Parsed unsupported type: %s", typ.Type)
+	}
+
+	type tmp yarnAuditLine // avoids infinite recursion
+	return json.Unmarshal(data, (*tmp)(yl))
+
+}
+
+type auditActionData struct {
+	Cmd        string      `json:"cmd"`
+	IsBreaking bool        `json:"isBreaking"`
+	Action     auditAction `json:"action"`
+}
+
+type auditAdvisoryData struct {
+	Resolution auditResolution `json:"resolution"`
+	Advisory   yarnAdvisory        `json:"advisory"`
+}
+
+// AsIssue returns data as a Dracon v1.Issue
+func (audit *auditAdvisoryData) AsIssue() *v1.Issue {
+	var targetName string
+	if audit.Resolution.Path != "" {
+		targetName = audit.Resolution.Path + ": "
+	}
+	targetName += audit.Advisory.ModuleName
+
+	return &v1.Issue{
+		Target:      targetName,
+		Type:        audit.Advisory.Cwe,
+		Title:       audit.Advisory.Title,
+		Severity:    yarnToIssueSeverity(audit.Advisory.Severity),
+		Confidence:  v1.Confidence_CONFIDENCE_HIGH,
+		Description: fmt.Sprintf("%s", audit.Advisory.GetDescription()),
+		Cve:         strings.Join(audit.Advisory.Cves, ", "),
+	}
+}
+
+type auditSummaryData struct {
+	Vulnerabilities      vulnerabilities `json:"vulnerabilities"`
+	Dependencies         int             `json:"dependencies"`
+	DevDependencies      int             `json:"devDependencies"`
+	OptionalDependencies int             `json:"optionalDependencies"`
+	TotalDependencies    int             `json:"totalDependencies"`
+}
+
+type auditAction struct {
+	Action   string            `json:"action"`
+	Module   string            `json:"module"`
+	Target   string            `json:"target"`
+	IsMajor  bool              `json:"isMajor"`
+	Resolves []auditResolution `json:"resolves"`
+}
+
+type vulnerabilities struct {
+	Info     int `json:"info"`
+	Low      int `json:"low"`
+	Moderate int `json:"moderate"`
+	High     int `json:"high"`
+	Critical int `json:"critical"`
+}
+
+type yarnAdvisory struct {
+	Findings           []finding         `json:"findings"`
+	Metadata           *advisoryMetaData `json:"metadata"`
+	VulnerableVersions string            `json:"vulnerable_versions"`
+	ModuleName         string            `json:"module_name"`
+	Severity           string            `json:"severity"`
+	GithubAdvisoryID   string            `json:"github_advisory_id"`
+	Cves               []string          `json:"cves"`
+	Access             string            `json:"access"`
+	PatchedVersions    string            `json:"patched_versions"`
+	Updated            string            `json:"updated"`
+	Recommendation     string            `json:"recommendation"`
+	Cwe                string            `json:"cwe"`
+	FoundBy            *contact          `json:"found_by"`
+	Deleted            bool              `json:"deleted"`
+	ID                 int               `json:"id"`
+	References         string            `json:"references"`
+	Created            string            `json:"created"`
+	ReportedBy         *contact          `json:"reported_by"`
+	Title              string            `json:"title"`
+	NpmAdvisoryID      interface{}       `json:"npm_advisory_id"`
+	Overview           string            `json:"overview"`
+	URL                string            `json:"url"`
+}
+
+func (advisory *yarnAdvisory) GetDescription() string {
+	return fmt.Sprintf(
+		"Vulnerable Versions: %s\nRecommendation: %s\nOverview: %s\nReferences:\n%s\nAdvisory URL: %s\n",
+		advisory.VulnerableVersions,
+		advisory.Recommendation,
+		advisory.Overview,
+		advisory.References,
+		advisory.URL,
+	)
+}
+
+type finding struct {
+	Version  string   `json:"version"`
+	Paths    []string `json:"paths"`
+	Dev      bool     `json:"dev"`
+	Optional bool     `json:"optional"`
+	Bundled  bool     `json:"bundled"`
+}
+
+type auditResolution struct {
+	ID       int    `json:"id"`
+	Path     string `json:"path"`
+	Dev      bool   `json:"dev"`
+	Optional bool   `json:"optional"`
+	Bundled  bool   `json:"bundled"`
+}
+
+type advisoryMetaData struct {
+	ModuleType         string `json:"module_type"`
+	Exploitability      int    `json:"exploitability"`
+	AffectedComponents string `json:"affected_components"`
+}
+
+type contact struct {
+	Name string `json: name`
+}
+
+// YarnAuditReport includes yarn audit data grouped by advisories, actions and summary
+type YarnAuditReport struct {
+	AuditAdvisories []*auditAdvisoryData
+	AuditActions  []*auditActionData
+	AuditSummary       *auditSummaryData
+}
+
+// NewReport returns a YarnAuditReport, assuming each line is jsonline and returns any errors
+func NewReport(reportLines [][]byte) (*YarnAuditReport, []error) {
+
+	var report YarnAuditReport
+
+	var errors []error
+
+	for _, line := range reportLines {
+		var auditLine yarnAuditLine
+		if err := producers.ParseJSON(line, &auditLine); err != nil {
+			log.Printf("Error parsing JSON line '%s': %s\n", line, err)
+			errors = append(errors, err)
+		} else {
+
+			switch auditLine.Data.(type) {
+			case *auditSummaryData:
+				report.AuditSummary = auditLine.Data.(*auditSummaryData)
+			case *auditAdvisoryData:
+				report.AuditAdvisories = append(report.AuditAdvisories, auditLine.Data.(*auditAdvisoryData))
+			case *auditActionData:
+				report.AuditActions = append(report.AuditActions, auditLine.Data.(*auditActionData))
+			}
+		}
+	}
+
+	if report.AuditAdvisories != nil && len(report.AuditAdvisories) > 0 {
+		return &report, errors
+	}
+
+	return nil, errors
+}
+
+// AsIssues returns the YarnAuditReport as Dracon v1.Issue list. Currently only converts the YarnAuditReport.AuditAdvisories
+func (r *YarnAuditReport) AsIssues() []*v1.Issue {
+	issues := make([]*v1.Issue, 0)
+
+	for _, audit := range r.AuditAdvisories {
+		issues = append(issues, audit.AsIssue())
+	}
+
+	return issues
+}