Merge pull request #3028 from cuipinghuo/vsa-acceptance-enhance

test: add VSA upload error handling acceptance tests

Author: cuipinghuo

acceptance/cli/cli.go

@@ -824,13 +824,29 @@ func createTrackBundleFile(ctx context.Context, name string, content *godog.DocS
 	return createGenericFile(ctx, name, content)
 }
 
+// theLogOutputShouldContain checks if the log output (stderr) contains the expected text
+func theLogOutputShouldContain(ctx context.Context, expected string) error {
+	status, err := ecStatusFrom(ctx)
+	if err != nil {
+		return err
+	}
+
+	stderr := status.stderr.String()
+	if !strings.Contains(stderr, expected) {
+		return fmt.Errorf("expected log output to contain:\n%s\nbut stderr was:\n%s", expected, stderr)
+	}
+
+	return nil
+}
+
 // AddStepsTo adds Gherkin steps to the godog ScenarioContext
 func AddStepsTo(sc *godog.ScenarioContext) {
 	sc.Step(`^ec command is run with "(.+)"$`, ecCommandIsRunWith)
 	sc.Step(`^the exit status should be (\d+)$`, theExitStatusIs)
 	sc.Step(`^the standard output should contain$`, theStandardOutputShouldContain)
 	sc.Step(`^the standard output should match baseline file "(.+)"$`, theStandardOutputShouldMatchBaseline)
 	sc.Step(`^the standard error should contain$`, theStandardErrorShouldContain)
+	sc.Step(`^the log output should contain "([^"]*)"$`, theLogOutputShouldContain)
 	sc.Step(`^the environment variable is set "([^"]*)"$`, theEnvironmentVarilableIsSet)
 	sc.Step(`^the output should match the snapshot$`, matchSnapshot)
 	sc.Step(`^the "([^"]*)" file should match the snapshot$`, matchFileSnapshot)

acceptance/rekor/rekor.go

@@ -357,6 +357,18 @@ func IsRunning(ctx context.Context) bool {
 	return testenv.HasState[rekorState](ctx)
 }
 
+// rekorUploadShouldFail creates WireMock stubs that simulate Rekor upload failures
+func rekorUploadShouldFail(ctx context.Context) error {
+	// Create a stub that returns a 500 Internal Server Error for VSA upload requests
+	return wiremock.StubFor(ctx, wiremock.Post(wiremock.URLPathEqualTo("/api/v1/log/entries")).
+		WillReturnResponse(wiremock.NewResponse().
+			WithStatus(500).
+			WithHeaders(map[string]string{
+				"Content-Type": "application/json",
+			}).
+			WithBody(`{"message":"Internal server error"}`)))
+}
+
 // AddStepsTo adds Gherkin steps to the godog ScenarioContext
 func AddStepsTo(sc *godog.ScenarioContext) {
 	sc.Step(`^stub rekord running$`, stubRekordRunning)
@@ -366,6 +378,7 @@ func AddStepsTo(sc *godog.ScenarioContext) {
 	sc.Step(`^VSA should be uploaded to Rekor successfully$`, vsaShouldBeUploadedToRekor)
 	sc.Step(`^VSA index search should return no results$`, stubVSAIndexSearch)
 	sc.Step(`^VSA index search should return valid VSA$`, stubVSAIndexSearchWithResult)
+	sc.Step(`^Rekor upload should fail$`, rekorUploadShouldFail)
 }
 
 // expectVSAUploadToRekor creates WireMock stubs to expect VSA upload requests to Rekor

features/__snapshots__/vsa.snap

@@ -307,3 +307,55 @@ time="${TIMESTAMP}" level=warning msg="invalid storage config 'invalid-backend@s
 [VSA generation with multiple storage backends:stderr - 1]
 
 ---
+
+[VSA generation without upload backends shows warning:stdout - 1]
+{
+  "success": true,
+  "components": [
+    {
+      "name": "Unnamed",
+      "containerImage": "${REGISTRY}/acceptance/vsa-no-upload-image@sha256:${REGISTRY_acceptance/vsa-no-upload-image:latest_DIGEST}",
+      "source": {},
+      "success": true,
+      "signatures": [
+        {
+          "keyid": "",
+          "sig": "${IMAGE_SIGNATURE_acceptance/vsa-no-upload-image}"
+        }
+      ],
+      "attestations": [
+        {
+          "type": "https://in-toto.io/Statement/v0.1",
+          "predicateType": "https://slsa.dev/provenance/v0.2",
+          "predicateBuildType": "https://tekton.dev/attestations/chains/pipelinerun@v2",
+          "signatures": [
+            {
+              "keyid": "",
+              "sig": "${ATTESTATION_SIGNATURE_acceptance/vsa-no-upload-image}"
+            }
+          ]
+        }
+      ]
+    }
+  ],
+  "key": "${vsa-no-upload_PUBLIC_KEY_JSON}",
+  "policy": {
+    "sources": [
+      {
+        "policy": [
+          "git::${GITHOST}/git/vsa-no-upload-policy.git?ref=${LATEST_COMMIT}"
+        ]
+      }
+    ],
+    "rekorUrl": "${REKOR}",
+    "publicKey": "${vsa-no-upload_PUBLIC_KEY}"
+  },
+  "ec-version": "${EC_VERSION}",
+  "effective-time": "${TIMESTAMP}"
+}
+---
+
+[VSA generation without upload backends shows warning:stderr - 1]
+time="${TIMESTAMP}" level=error msg="[VSA] VSA files generated but not uploaded (no --vsa-upload backends specified)"
+
+---

features/vsa.feature

@@ -171,3 +171,55 @@ Feature: VSA generation and storage
     When ec command is run with "validate image --image ${REGISTRY}/acceptance/vsa-existing-image@sha256:${REGISTRY_acceptance/vsa-existing-image:latest_DIGEST} --policy acceptance/vsa-existing-ec-policy --public-key ${vsa-existing_PUBLIC_KEY} --rekor-url ${REKOR} --vsa-expiration 24h --output json"
     Then the exit status should be 0
     Then the output should match the snapshot
+
+  Scenario: VSA generation without upload backends shows warning
+    Given a key pair named "vsa-no-upload"
+    Given an image named "acceptance/vsa-no-upload-image"
+    Given a valid image signature of "acceptance/vsa-no-upload-image" image signed by the "vsa-no-upload" key
+    Given a valid Rekor entry for image signature of "acceptance/vsa-no-upload-image"
+    Given a valid attestation of "acceptance/vsa-no-upload-image" signed by the "vsa-no-upload" key
+    Given a valid Rekor entry for attestation of "acceptance/vsa-no-upload-image"
+    Given a git repository named "vsa-no-upload-policy" with
+      | main.rego | examples/happy_day.rego |
+    Given policy configuration named "vsa-no-upload-ec-policy" with specification
+    """
+    {
+      "sources": [
+        {
+          "policy": [
+            "git::https://${GITHOST}/git/vsa-no-upload-policy.git"
+          ]
+        }
+      ]
+    }
+    """
+    When ec command is run with "validate image --image ${REGISTRY}/acceptance/vsa-no-upload-image --policy acceptance/vsa-no-upload-ec-policy --public-key ${vsa-no-upload_PUBLIC_KEY} --rekor-url ${REKOR} --vsa --vsa-signing-key ${vsa-no-upload_PRIVATE_KEY} --vsa-expiration 0 --output json"
+    Then the exit status should be 0
+    Then the output should match the snapshot
+    And the log output should contain "[VSA] VSA files generated but not uploaded (no --vsa-upload backends specified)"
+
+  Scenario: VSA upload failure handling
+    Given a key pair named "vsa-upload-fail"
+    Given an image named "acceptance/vsa-upload-fail-image"
+    Given a valid image signature of "acceptance/vsa-upload-fail-image" image signed by the "vsa-upload-fail" key
+    Given a valid Rekor entry for image signature of "acceptance/vsa-upload-fail-image"
+    Given a valid attestation of "acceptance/vsa-upload-fail-image" signed by the "vsa-upload-fail" key
+    Given a valid Rekor entry for attestation of "acceptance/vsa-upload-fail-image"
+    Given a git repository named "vsa-upload-fail-policy" with
+      | main.rego | examples/happy_day.rego |
+    Given policy configuration named "vsa-upload-fail-ec-policy" with specification
+    """
+    {
+      "sources": [
+        {
+          "policy": [
+            "git::https://${GITHOST}/git/vsa-upload-fail-policy.git"
+          ]
+        }
+      ]
+    }
+    """
+    Given Rekor upload should fail
+    When ec command is run with "validate image --image ${REGISTRY}/acceptance/vsa-upload-fail-image --policy acceptance/vsa-upload-fail-ec-policy --public-key ${vsa-upload-fail_PUBLIC_KEY} --rekor-url ${REKOR} --vsa --vsa-signing-key ${vsa-upload-fail_PRIVATE_KEY} --vsa-upload rekor@${REKOR} --vsa-expiration 0 --output json"
+    Then the exit status should be 0
+    And the log output should contain "[VSA] Failed to upload in-toto 0.0.2 entry"