chore: benchmark job optimization with configurable toggles (#7045)

* Benchmark: pin Nighthawk image, skip PNGs by default, add CI-friendly toggles
Pin nighthawk client/server to non-latest (env/envoyproxy/nighthawk-dev with default v0.7.0)
Make PNG rendering optional; default disabled in CI
Add hook to optionally skip kind image load (USE_PUBLISHED_IMAGE)
Remove Graphviz from benchmark jobs; set BENCHMARK_RENDER_PNG=false
Wire client image override in benchmark suite
Keep behavior unchanged for local runs unless toggles are set

Signed-off-by: ADITYATIWARI342005 <[email protected]>

* Address review feedback: pin images in YAML, remove runtime overrides

- Update nighthawk-test-server.yaml and nighthawk-client.yaml to use stable SHA tag directly
- Remove runtime image override logic from suite.go (simplified approach)
- Remove unnecessary maybe-kube-install-image target and kubectl set image command
- Clean up unused NIGHTHAWK_IMAGE/NIGHTHAWK_TAG variables from Makefile
- Keep BENCHMARK_RENDER_PNG toggle for CI optimization

Signed-off-by: ADITYATIWARI342005 <[email protected]>

Author: ADITYATIWARI342005

.github/workflows/build_and_test.yaml

@@ -252,8 +252,6 @@ jobs:
     - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8  # v5.0.0
     - uses: ./tools/github-actions/setup-deps
 
-    - name: Setup Graphviz
-      uses: ts-graphviz/setup-graphviz@b1de5da23ed0a6d14e0aeee8ed52fdd87af2363c  # v2.0.2
 
     # Benchmark
     - name: Run Benchmark tests
@@ -266,6 +264,7 @@ jobs:
         BENCHMARK_CPU_LIMITS: 1000m
         BENCHMARK_MEMORY_LIMITS: 2000Mi
         BENCHMARK_REPORT_DIR: benchmark_report
+        BENCHMARK_RENDER_PNG: "false"
       run: make benchmark
 
     - name: Upload Benchmark report

.github/workflows/release.yaml

@@ -18,8 +18,6 @@ jobs:
       - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8  # v5.0.0
       - uses: ./tools/github-actions/setup-deps
 
-      - name: Setup Graphviz
-        uses: ts-graphviz/setup-graphviz@b1de5da23ed0a6d14e0aeee8ed52fdd87af2363c  # v2.0.2
 
       # Benchmark
       - name: Run Benchmark tests
@@ -33,6 +31,7 @@ jobs:
           BENCHMARK_CPU_LIMITS: 1000m
           BENCHMARK_MEMORY_LIMITS: 2000Mi
           BENCHMARK_REPORT_DIR: benchmark_report
+          BENCHMARK_RENDER_PNG: "false"
         run: make benchmark
 
       - name: Package benchmark report

test/benchmark/config/nighthawk-client.yaml

@@ -9,7 +9,7 @@ spec:
     spec:
       containers:
         - name: nighthawk-client
-          image: envoyproxy/nighthawk-dev:latest
+          image: envoyproxy/nighthawk-dev:6198df0e01dc85f8d5efa3ef188f3bcdcc6eecac
           imagePullPolicy: IfNotPresent
           command:
             - /usr/local/bin/nighthawk_client

test/benchmark/config/nighthawk-test-server.yaml

@@ -17,7 +17,7 @@ spec:
       serviceAccountName: default
       containers:
         - name: nighthawk-server
-          image: envoyproxy/nighthawk-dev:latest
+          image: envoyproxy/nighthawk-dev:6198df0e01dc85f8d5efa3ef188f3bcdcc6eecac
           imagePullPolicy: IfNotPresent
           args: ["-c", "/etc/test-server-config/nighthawk-test-server-config.yaml"]
           ports:

tools/make/kube.mk

@@ -25,6 +25,10 @@ BENCHMARK_CONNECTIONS ?= 100
 BENCHMARK_DURATION ?= 60
 BENCHMARK_REPORT_DIR ?= benchmark_report
 
+# Benchmark toggles
+# Disable PNG rendering by default to speed up CI
+BENCHMARK_RENDER_PNG ?= false
+
 CONFORMANCE_RUN_TEST ?=
 
 E2E_RUN_TEST ?=
@@ -253,8 +257,8 @@ run-benchmark: install-benchmark-server prepare-ip-family ## Run benchmark tests
 	kubectl apply -f test/benchmark/config/gatewayclass.yaml
 	go test -v -tags benchmark -timeout $(BENCHMARK_TIMEOUT) ./test/benchmark --rps=$(BENCHMARK_RPS) --connections=$(BENCHMARK_CONNECTIONS) --duration=$(BENCHMARK_DURATION) --report-save-dir=$(BENCHMARK_REPORT_DIR)
 	# render benchmark profiles into image
-	dot -V
-	find test/benchmark/$(BENCHMARK_REPORT_DIR)/profiles -name "*.pprof" -type f -exec sh -c 'go tool pprof -png "$$1" > "$${1%.pprof}.png"' _ {} \;
+	@if [ "$(BENCHMARK_RENDER_PNG)" != "false" ]; then dot -V; fi
+	@if [ "$(BENCHMARK_RENDER_PNG)" != "false" ]; then find test/benchmark/$(BENCHMARK_REPORT_DIR)/profiles -name "*.pprof" -type f -exec sh -c 'go tool pprof -png "$$1" > "$$${1%.pprof}.png"' _ {} \; ; fi
 
 .PHONY: install-benchmark-server
 install-benchmark-server: ## Install nighthawk server for benchmark test