Add NGF build information to test results (#2353)

Problem: We don't add the commit that was tested to the test results and
we don't have a way to confirm that the correct version was tested.

Solution: Add build info to the tests results

.github/workflows/functional.yml

@@ -118,19 +118,19 @@ jobs:
         run: |
           ngf_prefix=ghcr.io/nginxinc/nginx-gateway-fabric
           ngf_tag=${{ steps.ngf-meta.outputs.version }}
-          make test${{ inputs.image == 'plus' && '-with-plus' || ''}} PREFIX=${ngf_prefix} TAG=${ngf_tag} GINKGO_LABEL=telemetry GW_SERVICE_TYPE=LoadBalancer
+          make test${{ inputs.image == 'plus' && '-with-plus' || ''}} PREFIX=${ngf_prefix} TAG=${ngf_tag} GINKGO_LABEL=telemetry GW_SERVICE_TYPE=LoadBalancer CI=true
         working-directory: ./tests
 
       - name: Run functional graceful-recovery tests
         run: |
           ngf_prefix=ghcr.io/nginxinc/nginx-gateway-fabric
           ngf_tag=${{ steps.ngf-meta.outputs.version }}
-          make test${{ inputs.image == 'plus' && '-with-plus' || ''}} PREFIX=${ngf_prefix} TAG=${ngf_tag} GINKGO_LABEL=graceful-recovery GW_SERVICE_TYPE=LoadBalancer CLUSTER_NAME=${{ github.run_id }}
+          make test${{ inputs.image == 'plus' && '-with-plus' || ''}} PREFIX=${ngf_prefix} TAG=${ngf_tag} GINKGO_LABEL=graceful-recovery GW_SERVICE_TYPE=LoadBalancer CLUSTER_NAME=${{ github.run_id }} CI=true
         working-directory: ./tests
 
       - name: Run functional tests
         run: |
           ngf_prefix=ghcr.io/nginxinc/nginx-gateway-fabric
           ngf_tag=${{ steps.ngf-meta.outputs.version }}
-          make test${{ inputs.image == 'plus' && '-with-plus' || ''}} PREFIX=${ngf_prefix} TAG=${ngf_tag} GW_SERVICE_TYPE=LoadBalancer
+          make test${{ inputs.image == 'plus' && '-with-plus' || ''}} PREFIX=${ngf_prefix} TAG=${ngf_tag} GW_SERVICE_TYPE=LoadBalancer CI=true
         working-directory: ./tests

Makefile

@@ -45,8 +45,8 @@ endif
 
 .PHONY: help
 help: Makefile ## Display this help
-	@grep -hE '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "; printf "Usage:\n\n    make \033[36m<target>\033[0m [VARIABLE=value...]\n\nTargets:\n\n"}; {printf "    \033[36m%-30s\033[0m %s\n", $$1, $$2}'
-	@grep -hE '^(override )?[a-zA-Z_-]+ \??\+?= .*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = " \\??\\+?= .*?## "; printf "\nVariables:\n\n"}; {gsub(/override /, "", $$1); printf "    \033[36m%-30s\033[0m %s\n", $$1, $$2}'
+	@grep -hE '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "; printf "Usage:\n\n	make \033[36m<target>\033[0m [VARIABLE=value...]\n\nTargets:\n\n"}; {printf "	 \033[36m%-30s\033[0m %s\n", $$1, $$2}'
+	@grep -hE '^(override )?[a-zA-Z_-]+ \??\+?= .*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = " \\??\\+?= .*?## "; printf "\nVariables:\n\n"}; {gsub(/override /, "", $$1); printf "	 \033[36m%-30s\033[0m %s\n", $$1, $$2}'
 
 .PHONY: build-prod-images
 build-prod-images: build-prod-ngf-image build-prod-nginx-image ## Build the NGF and nginx docker images for production

tests/Makefile

@@ -1,10 +1,11 @@
-CI=false
+CI ?= false
 CLUSTER_NAME ?= kind
 CONFORMANCE_PREFIX = conformance-test-runner## Prefix for the conformance test runner image
 CONFORMANCE_TAG = latest## Tag for the conformance test runner image
 GATEWAY_CLASS = nginx## Gateway class to use
 GINKGO_FLAGS =
 GINKGO_LABEL =
+GITHUB_OUTPUT =
 GW_API_VERSION ?= $(shell sed -n 's/.*ref=v\(.*\)/\1/p' ../config/crd/gateway-api/standard/kustomization.yaml)## Supported Gateway API version from current NGF
 GW_API_PREV_VERSION ?= 1.1.0## Supported Gateway API version from previous NGF release
 GW_SERVICE_TYPE = NodePort## Service type to use for the gateway
@@ -15,7 +16,11 @@ PROVISIONER_MANIFEST = conformance/provisioner/provisioner.yaml
 SUPPORTED_FEATURES = HTTPRouteQueryParamMatching,HTTPRouteMethodMatching,HTTPRoutePortRedirect,HTTPRouteSchemeRedirect,HTTPRouteHostRewrite,HTTPRoutePathRewrite,GatewayPort8080,HTTPRouteResponseHeaderModification,GRPCExactMethodMatching,GRPCRouteListenerHostnameMatching,GRPCRouteHeaderMatching
 
 ifneq ($(GINKGO_LABEL),)
-    override GINKGO_FLAGS += --label-filter "$(GINKGO_LABEL)"
+	override GINKGO_FLAGS += --label-filter "$(GINKGO_LABEL)"
+endif
+
+ifeq ($(CI),true)
+	GITHUB_OUTPUT := --github-output
 endif
 
 .PHONY: update-go-modules
@@ -33,9 +38,9 @@ run-conformance-tests: ## Run conformance tests
 	kubectl apply -f conformance/conformance-rbac.yaml
 	kubectl run -i conformance \
 		--image=$(CONFORMANCE_PREFIX):$(CONFORMANCE_TAG) --image-pull-policy=Never \
-		--overrides='{ "spec": { "serviceAccountName": "conformance" }  }' \
+		--overrides='{ "spec": { "serviceAccountName": "conformance" }	}' \
 		--restart=Never -- sh -c "go test -v . -tags conformance,experimental -args --gateway-class=$(GATEWAY_CLASS) \
-						        --supported-features=$(SUPPORTED_FEATURES) --version=$(NGF_VERSION) \
+								--supported-features=$(SUPPORTED_FEATURES) --version=$(NGF_VERSION) \
 								--report-output=output.txt; cat output.txt" | tee output.txt
 	./scripts/check-pod-exit-code.sh
 	sed -e '1,/CONFORMANCE PROFILE/d' output.txt > conformance-profile.yaml
@@ -65,8 +70,8 @@ reset-go-modules: ## Reset the go modules changes
 
 # Check if PLUS_ENABLED is true
 ifeq ($(PLUS_ENABLED),true)
-    # If true, set NGINX_PREFIX to $NGINX_PLUS_PREFIX
-    NGINX_PREFIX := $(NGINX_PLUS_PREFIX)
+	# If true, set NGINX_PREFIX to $NGINX_PLUS_PREFIX
+	NGINX_PREFIX := $(NGINX_PLUS_PREFIX)
 endif
 
 .PHONY: setup-gcp-and-run-tests
@@ -107,18 +112,19 @@ start-longevity-test: nfr-test ## Start the longevity test to run for 4 days in
 stop-longevity-test: export STOP_LONGEVITY=true
 stop-longevity-test: nfr-test ## Stop the longevity test and collects results
 
+
 .PHONY: .vm-nfr-test
 .vm-nfr-test: ## Runs the NFR tests on the GCP VM (called by `nfr-test`)
-	go run github.com/onsi/ginkgo/v2/ginkgo --randomize-all --randomize-suites --keep-going --fail-on-pending --trace -r -v --force-newlines $(ifeq $(CI),true,--github-output) \
-	    --label-filter "nfr" $(GINKGO_FLAGS) --timeout 3h ./suite --  --gateway-api-version=$(GW_API_VERSION) \
+	go run github.com/onsi/ginkgo/v2/ginkgo --randomize-all --randomize-suites --keep-going --fail-on-pending --trace -r -v --buildvcs --force-newlines $(GITHUB_OUTPUT) \
+		--label-filter "nfr" $(GINKGO_FLAGS) --timeout 3h ./suite --  --gateway-api-version=$(GW_API_VERSION) \
 		--gateway-api-prev-version=$(GW_API_PREV_VERSION) --image-tag=$(TAG) --version-under-test=$(NGF_VERSION) \
 		--plus-enabled=$(PLUS_ENABLED) --ngf-image-repo=$(PREFIX) --nginx-image-repo=$(NGINX_PREFIX) --nginx-plus-image-repo=$(NGINX_PLUS_PREFIX) \
 		--pull-policy=$(PULL_POLICY) --service-type=$(GW_SERVICE_TYPE) \
 		--is-gke-internal-lb=$(GW_SVC_GKE_INTERNAL)
 
 .PHONY: test
 test: ## Runs the functional tests on your default k8s cluster
-	go run github.com/onsi/ginkgo/v2/ginkgo --randomize-all --randomize-suites --keep-going --fail-on-pending --trace -r -v --force-newlines \
+	go run github.com/onsi/ginkgo/v2/ginkgo --randomize-all --randomize-suites --keep-going --fail-on-pending --trace -r -v --buildvcs --force-newlines $(GITHUB_OUTPUT) \
 		--label-filter "functional" $(GINKGO_FLAGS) ./suite -- \
 		--gateway-api-version=$(GW_API_VERSION) --gateway-api-prev-version=$(GW_API_PREV_VERSION) \
 		--image-tag=$(TAG) --version-under-test=$(NGF_VERSION) \

tests/framework/info.go

@@ -2,11 +2,13 @@ package framework
 
 import (
 	"fmt"
+	"runtime/debug"
 
 	core "k8s.io/api/core/v1"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 )
 
+// GetLogs returns the logs for all containers in all pods for a release.
 func GetLogs(rm ResourceManager, namespace string, releaseName string) string {
 	var returnLogs string
 	pods, err := rm.GetPods(namespace, client.MatchingLabels{
@@ -32,6 +34,7 @@ func GetLogs(rm ResourceManager, namespace string, releaseName string) string {
 	return returnLogs
 }
 
+// GetEvents returns the events for a namespace.
 func GetEvents(rm ResourceManager, namespace string) string {
 	var returnEvents string
 	events, err := rm.GetEvents(namespace)
@@ -53,3 +56,28 @@ func GetEvents(rm ResourceManager, namespace string) string {
 	}
 	return returnEvents
 }
+
+// GetBuildInfo returns the build information.
+func GetBuildInfo() (commitHash string, commitTime string, dirtyBuild string) {
+	commitHash = "unknown"
+	commitTime = "unknown"
+	dirtyBuild = "unknown"
+
+	info, ok := debug.ReadBuildInfo()
+	if !ok {
+		return
+	}
+
+	for _, kv := range info.Settings {
+		switch kv.Key {
+		case "vcs.revision":
+			commitHash = kv.Value
+		case "vcs.time":
+			commitTime = kv.Value
+		case "vcs.modified":
+			dirtyBuild = kv.Value
+		}
+	}
+
+	return
+}

tests/framework/results.go

@@ -49,10 +49,13 @@ func WriteSystemInfoToFile(file *os.File, ci ClusterInfo, plus bool) error {
 	if ci.IsGKE {
 		clusterType = "GKE"
 	}
+
+	commit, date, dirty := GetBuildInfo()
+
+	//nolint:lll
 	text := fmt.Sprintf(
-		//nolint:lll
-		"# Results\n\n## Test environment\n\nNGINX Plus: %v\n\n%s Cluster:\n\n- Node count: %d\n- k8s version: %s\n- vCPUs per node: %d\n- RAM per node: %s\n- Max pods per node: %d\n",
-		plus, clusterType, ci.NodeCount, ci.K8sVersion, ci.CPUCountPerNode, ci.MemoryPerNode, ci.MaxPodsPerNode,
+		"# Results\n\n## Test environment\n\nNGINX Plus: %v\n\nNGINX Gateway Fabric:\n\n- Commit: %s\n- Date: %s\n- Dirty: %v\n\n%s Cluster:\n\n- Node count: %d\n- k8s version: %s\n- vCPUs per node: %d\n- RAM per node: %s\n- Max pods per node: %d\n",
+		plus, commit, date, dirty, clusterType, ci.NodeCount, ci.K8sVersion, ci.CPUCountPerNode, ci.MemoryPerNode, ci.MaxPodsPerNode,
 	)
 	if _, err := fmt.Fprint(file, text); err != nil {
 		return err

tests/scripts/remote-scripts/run-nfr-tests.sh

@@ -10,7 +10,7 @@ elif [ "${STOP_LONGEVITY}" == "true" ]; then
     GINKGO_LABEL="longevity-teardown"
 fi
 
-cd nginx-gateway-fabric/tests && make .vm-nfr-test TAG="${TAG}" PREFIX="${PREFIX}" NGINX_PREFIX="${NGINX_PREFIX}" NGINX_PLUS_PREFIX="${NGINX_PLUS_PREFIX}" PLUS_ENABLED="${PLUS_ENABLED}" GINKGO_LABEL=${GINKGO_LABEL} GINKGO_FLAGS="${GINKGO_FLAGS}" PULL_POLICY=Always GW_SERVICE_TYPE=LoadBalancer GW_SVC_GKE_INTERNAL=true NGF_VERSION="${NGF_VERSION}"
+cd nginx-gateway-fabric/tests && make .vm-nfr-test CI=${CI} TAG="${TAG}" PREFIX="${PREFIX}" NGINX_PREFIX="${NGINX_PREFIX}" NGINX_PLUS_PREFIX="${NGINX_PLUS_PREFIX}" PLUS_ENABLED="${PLUS_ENABLED}" GINKGO_LABEL=${GINKGO_LABEL} GINKGO_FLAGS="${GINKGO_FLAGS}" PULL_POLICY=Always GW_SERVICE_TYPE=LoadBalancer GW_SVC_GKE_INTERNAL=true NGF_VERSION="${NGF_VERSION}"
 
 if [ "${START_LONGEVITY}" == "true" ]; then
     suite/scripts/longevity-wrk.sh

tests/scripts/remote-scripts/run-tests.sh

@@ -4,4 +4,4 @@ set -e
 
 source "${HOME}"/vars.env
 
-cd nginx-gateway-fabric/tests && make test TAG="${TAG}" PREFIX="${PREFIX}" NGINX_PREFIX="${NGINX_PREFIX}" NGINX_PLUS_PREFIX="${NGINX_PLUS_PREFIX}" PLUS_ENABLED="${PLUS_ENABLED}" GINKGO_LABEL="${GINKGO_LABEL}" GINKGO_FLAGS="${GINKGO_FLAGS}" PULL_POLICY=Always GW_SERVICE_TYPE=LoadBalancer GW_SVC_GKE_INTERNAL=true NGF_VERSION="${NGF_VERSION}"
+cd nginx-gateway-fabric/tests && make test CI=${CI} TAG="${TAG}" PREFIX="${PREFIX}" NGINX_PREFIX="${NGINX_PREFIX}" NGINX_PLUS_PREFIX="${NGINX_PLUS_PREFIX}" PLUS_ENABLED="${PLUS_ENABLED}" GINKGO_LABEL="${GINKGO_LABEL}" GINKGO_FLAGS="${GINKGO_FLAGS}" PULL_POLICY=Always GW_SERVICE_TYPE=LoadBalancer GW_SVC_GKE_INTERNAL=true NGF_VERSION="${NGF_VERSION}"

tests/scripts/run-tests-gcp-vm.sh

@@ -16,6 +16,7 @@ gcloud compute scp --zone "${GKE_CLUSTER_ZONE}" --project="${GKE_PROJECT}" "${SC
 gcloud compute ssh --zone "${GKE_CLUSTER_ZONE}" --project="${GKE_PROJECT}" username@"${RESOURCE_NAME}" \
     --command="export START_LONGEVITY=${START_LONGEVITY} &&\
         export STOP_LONGEVITY=${STOP_LONGEVITY} &&\
+        export CI=${CI} &&\
         bash -s" <"${SCRIPT_DIR}"/remote-scripts/${SCRIPT}
 retcode=$?
 

tests/suite/client_settings_test.go

@@ -1,4 +1,4 @@
-package suite
+package main
 
 import (
 	"bytes"

tests/suite/dataplane_perf_test.go

@@ -1,4 +1,4 @@
-package suite
+package main
 
 import (
 	"fmt"

tests/suite/graceful_recovery_test.go

@@ -1,4 +1,4 @@
-package suite
+package main
 
 import (
 	"context"

tests/suite/longevity_test.go

@@ -1,4 +1,4 @@
-package suite
+package main
 
 import (
 	"fmt"

tests/suite/sample_test.go

@@ -1,4 +1,4 @@
-package suite
+package main
 
 import (
 	"fmt"

tests/suite/scale_test.go

@@ -1,4 +1,4 @@
-package suite
+package main
 
 import (
 	"bytes"
@@ -130,39 +130,39 @@ var _ = Describe("Scale test", Ordered, Label("nfr", "scale"), func() {
 	}
 
 	const scaleResultTemplate = `
-	## Test {{ .Name }}
+## Test {{ .Name }}
 
-	### Reloads
+### Reloads
 
-	- Total: {{ .ReloadCount }}
-	- Total Errors: {{ .ReloadErrsCount }}
-	- Average Time: {{ .ReloadAvgTime }}ms
-	- Reload distribution:
-	{{- range .ReloadBuckets }}
-		- {{ .Le }}ms: {{ .Val }}
-	{{- end }}
+- Total: {{ .ReloadCount }}
+- Total Errors: {{ .ReloadErrsCount }}
+- Average Time: {{ .ReloadAvgTime }}ms
+- Reload distribution:
+{{- range .ReloadBuckets }}
+	- {{ .Le }}ms: {{ .Val }}
+{{- end }}
 
-	### Event Batch Processing
+### Event Batch Processing
 
-	- Total: {{ .EventsCount }}
-	- Average Time: {{ .EventsAvgTime }}ms
-	- Event Batch Processing distribution:
-	{{- range .EventsBuckets }}
-		- {{ .Le }}ms: {{ .Val }}
-	{{- end }}
+- Total: {{ .EventsCount }}
+- Average Time: {{ .EventsAvgTime }}ms
+- Event Batch Processing distribution:
+{{- range .EventsBuckets }}
+	- {{ .Le }}ms: {{ .Val }}
+{{- end }}
 
-	### Errors
+### Errors
 
-	- NGF errors: {{ .NGFErrors }}
-	- NGF container restarts: {{ .NGFContainerRestarts }}
-	- NGINX errors: {{ .NginxErrors }}
-	- NGINX container restarts: {{ .NginxContainerRestarts }}
+- NGF errors: {{ .NGFErrors }}
+- NGF container restarts: {{ .NGFContainerRestarts }}
+- NGINX errors: {{ .NginxErrors }}
+- NGINX container restarts: {{ .NginxContainerRestarts }}
 
-	### Graphs and Logs
+### Graphs and Logs
 
-	See [output directory](./{{ .Name }}) for more details.
-	The logs are attached only if there are errors.
-	`
+See [output directory](./{{ .Name }}) for more details.
+The logs are attached only if there are errors.
+`
 
 	writeScaleResults := func(dest io.Writer, results scaleTestResults) error {
 		tmpl, err := template.New("results").Parse(scaleResultTemplate)
@@ -905,7 +905,7 @@ var _ = Describe("Zero downtime scale test", Ordered, Label("nfr", "zero-downtim
 
 		Expect(framework.WriteMetricsResults(outFile, res.metrics)).To(Succeed())
 
-		link := fmt.Sprintf("\n\n![%[1]v.png](%[1]v.png)\n", fmt.Sprintf("%s-%s", testFileNamePrefix, res.scheme))
+		link := fmt.Sprintf("\n\n![%[1]v-oss.png](%[1]v-oss.png)\n", fmt.Sprintf("%s-%s", testFileNamePrefix, res.scheme))
 		if *plusEnabled {
 			link = fmt.Sprintf("\n\n![%[1]v-plus.png](%[1]v-plus.png)\n", fmt.Sprintf("%s-%s", testFileNamePrefix, res.scheme))
 		}

tests/suite/system_suite_test.go

@@ -1,4 +1,6 @@
-package suite
+// This package needs to be named main to get build info
+// because of https://github.com/golang/go/issues/33976
+package main
 
 import (
 	"context"

tests/suite/telemetry_test.go

@@ -1,4 +1,4 @@
-package suite
+package main
 
 import (
 	"fmt"

tests/suite/tracing_test.go

@@ -1,4 +1,4 @@
-package suite
+package main
 
 import (
 	"context"

tests/suite/upgrade_test.go

@@ -1,4 +1,4 @@
-package suite
+package main
 
 import (
 	"bytes"