chore: switch to yardstick validate from custom gate.py

Signed-off-by: Will Murphy <[email protected]>
anchore · Aug 29, 2024 · 832391c · 832391c
1 parent 7901a57
commit 832391c
Show file tree

Hide file tree

Showing 4 changed files with 9 additions and 347 deletions.
diff --git a/test/quality/.yardstick.yaml b/test/quality/.yardstick.yaml
@@ -91,6 +91,11 @@ default_max_year: 2021
 result-sets:
   pr_vs_latest_via_sbom:
     description: "latest released grype vs grype from the current build (via SBOM ingestion)"
+    validations:
+      - max-f1-regression: 0.0
+        max-new-false-negatives: 00
+        max-unlabeled-percent: 10
+        max_year: 2021
     matrix:
       images: *images
 
@@ -112,6 +117,7 @@ result-sets:
           # for local build of grype, use for example:
           version: path:../../+import-db=db.tar.gz
           takes: SBOM
+          label: candidate
 
         - name: grype
           # note: we import a static (pinned) DB as to prevent changes in the DB from affecting the results. The
@@ -121,3 +127,4 @@ result-sets:
           # are testing with is not too stale.
           version: latest+import-db=db.tar.gz
           takes: SBOM
+          label: reference
diff --git a/test/quality/Makefile b/test/quality/Makefile
@@ -27,7 +27,7 @@ all: capture validate ## Fetch or capture all data and run all quality checks
 
 .PHONY: validate
 validate: venv $(VULNERABILITY_LABELS)/Makefile ## Run all quality checks against already collected data
-	$(ACTIVATE_VENV) ./gate.py
+	$(ACTIVATE_VENV) yardstick validate -r $(RESULT_SET)
 
 .PHONY: capture
 capture: sboms vulns ## Collect and store all syft and grype results

diff --git a/test/quality/gate.py b/test/quality/gate.py
diff --git a/test/quality/requirements.txt b/test/quality/requirements.txt
@@ -1,3 +1,3 @@
-git+https://github.com/anchore/yardstick@v0.9.1
+git+https://github.com/anchore/yardstick@feat-validate-subcommand
 # ../../../yardstick
 tabulate==0.9.0