diff --git a/tests/bdd/AGENTS.md b/tests/bdd/AGENTS.md index a4b3f022aa..30c44ef7b3 100644 --- a/tests/bdd/AGENTS.md +++ b/tests/bdd/AGENTS.md @@ -58,6 +58,10 @@ logic into `dsl/`. - Function lifecycle CLI option tables preserve row order, repeated options, empty values, and product-invalid values. Only the `option | value` table structure is validated before the command runs. +- The Helm function lifecycle uses `SAMPLE_HELM_FUNCTION_CHART`. The chart + must be publicly reachable by ReVal, expose an `entrypoint` Service on port + 8000, answer `/health`, and echo the request message from `/echo`. Keep the + chart reference outside the feature so release versions are not hard-coded. - Gateway API route readiness tables expose each route's kind, name, namespace, and intended Gateway parent plus the shared context and timeout. The step requires `Accepted=True` and `ResolvedRefs=True` for that parent but diff --git a/tests/bdd/README.md b/tests/bdd/README.md index 60b53509c7..c248c7f7f1 100644 --- a/tests/bdd/README.md +++ b/tests/bdd/README.md @@ -46,7 +46,10 @@ go test -short ./... Live runs build nvcf-cli, bring up a real k3d cluster, and exercise the feature end to end. They require an NGC API key and sample registry -coordinates. +coordinates. The single-cluster and multi-cluster Helmfile runs also require +`SAMPLE_HELM_FUNCTION_CHART`. It must identify a chart that ReVal can fetch +without credentials. The chart must expose an `entrypoint` Service on port +8000, answer `/health`, and echo the request message from `/echo`. Each `-run` argument is anchored with `^...$` so the live entry point runs without also matching its `...FeatureFileWiresToSteps` wiring @@ -67,9 +70,9 @@ go test -run '^TestSingleClusterUpOneClick$' -timeout 30m -v # Multi-cluster CLI feature go test -run '^TestMultiClusterUp$' -timeout 60m -v -# Single-cluster Helmfile feature (requires NGC_API_KEY, SAMPLE_NGC_ORG, -# SAMPLE_NGC_TEAM) +# Single-cluster Helmfile feature (also requires a public sample Helm chart) NGC_API_KEY= SAMPLE_NGC_ORG= SAMPLE_NGC_TEAM= \ + SAMPLE_HELM_FUNCTION_CHART= \ go test -run '^TestSingleClusterHelmfile$' -timeout 90m -v # Focused single-cluster Helmfile feature for the documented public Docker Hub @@ -86,6 +89,7 @@ BDD_CLEANUP_MODE=topology-multi \ # on k3d-ncp-local-compute-1. Same secrets as the single-cluster # Helmfile feature. NGC_API_KEY= SAMPLE_NGC_ORG= SAMPLE_NGC_TEAM= \ + SAMPLE_HELM_FUNCTION_CHART= \ go test -run '^TestMultiClusterHelmfile$' -timeout 90m -v ``` diff --git a/tests/bdd/features/multi-cluster-helmfile.feature b/tests/bdd/features/multi-cluster-helmfile.feature index ec0fa1cc8e..27e8a55b24 100644 --- a/tests/bdd/features/multi-cluster-helmfile.feature +++ b/tests/bdd/features/multi-cluster-helmfile.feature @@ -329,6 +329,44 @@ Feature: Install a local multi-cluster NVCF stack with Helmfile # Keep the simulated GPU capacity available for the next scenario. And I successfully undeploy the function selected by NVCF CLI + @function-lifecycle @helm-function + Scenario: Operator creates, deploys, and invokes a Helm chart function across clusters + Given environment variable "SAMPLE_HELM_FUNCTION_CHART" is set + And I use NVCF CLI config "${REPO_ROOT}/tests/bdd/fixtures/nvcf-cli-local.yaml" + + # ReVal renders the chart on the control plane before NVCA deploys the + # workload in the separate compute cluster. Successful invocation proves + # that the complete chart-backed path works across the cluster boundary. + # TODO(#1871): Replace this raw command with a table-backed Helm function + # create step so the options remain readable. + When I successfully run command: + """ + ${NVCF_CLI} --config ${REPO_ROOT}/tests/bdd/fixtures/nvcf-cli-local.yaml function create --name bdd-multi-helm-function --helm-chart ${SAMPLE_HELM_FUNCTION_CHART} --helm-chart-service entrypoint --inference-url /echo --inference-port 8000 --health-uri /health --health-port 8000 --health-timeout PT30S + """ + + And I successfully deploy the function selected by NVCF CLI with options: + | option | value | + | --gpu | H100 | + | --instance-type | NCP.GPU.H100_1x | + | --backend | ncp-local-compute-1 | + | --regions | us-west-1 | + | --min-instances | 1 | + | --max-instances | 1 | + | --timeout | 900 | + + And I successfully generate a function API key with CLI options: + | option | value | + | --description | bdd-multi-helm-function | + | --scopes | invoke_function,list_functions,queue_details,list_functions_details | + + When I successfully invoke the function selected by NVCF CLI over HTTP with timeout "120" seconds and poll duration "5" seconds: + """ + {"message":"bdd-multi-helm-echo","repeats":1} + """ + Then the command output should contain "bdd-multi-helm-echo" + + And I successfully undeploy the function selected by NVCF CLI + @function-lifecycle @grpc Scenario: Operator creates, deploys, and invokes the gRPC Load Tester Supreme sample function Given I use NVCF CLI config "${REPO_ROOT}/tests/bdd/fixtures/nvcf-cli-local.yaml" diff --git a/tests/bdd/features/single-cluster-helmfile.feature b/tests/bdd/features/single-cluster-helmfile.feature index 80e5bb4699..6e4b3318a0 100644 --- a/tests/bdd/features/single-cluster-helmfile.feature +++ b/tests/bdd/features/single-cluster-helmfile.feature @@ -222,6 +222,43 @@ Feature: Install a local single-cluster NVCF stack with Helmfile # scenario's deployment at once. And I successfully undeploy the function selected by NVCF CLI + @function-lifecycle @helm-function + Scenario: Operator creates, deploys, and invokes a Helm chart function + Given environment variable "SAMPLE_HELM_FUNCTION_CHART" is set + And I use NVCF CLI config "${REPO_ROOT}/tests/bdd/fixtures/nvcf-cli-local.yaml" + + # Function creation and deployment both cross the ReVal chart-rendering + # boundary. A deployed ReVal pod alone does not prove this path works. + # TODO(#1871): Replace this raw command with a table-backed Helm function + # create step so the options remain readable. + When I successfully run command: + """ + ${NVCF_CLI} --config ${REPO_ROOT}/tests/bdd/fixtures/nvcf-cli-local.yaml function create --name bdd-helm-function --helm-chart ${SAMPLE_HELM_FUNCTION_CHART} --helm-chart-service entrypoint --inference-url /echo --inference-port 8000 --health-uri /health --health-port 8000 --health-timeout PT30S + """ + + And I successfully deploy the function selected by NVCF CLI with options: + | option | value | + | --gpu | H100 | + | --instance-type | NCP.GPU.H100_1x | + | --backend | ncp-local | + | --regions | us-west-1 | + | --min-instances | 1 | + | --max-instances | 1 | + | --timeout | 900 | + + And I successfully generate a function API key with CLI options: + | option | value | + | --description | bdd-helm-function | + | --scopes | invoke_function,list_functions,queue_details,list_functions_details | + + When I successfully invoke the function selected by NVCF CLI over HTTP with timeout "120" seconds and poll duration "5" seconds: + """ + {"message":"bdd-helm-echo","repeats":1} + """ + Then the command output should contain "bdd-helm-echo" + + And I successfully undeploy the function selected by NVCF CLI + @function-lifecycle @grpc Scenario: Operator creates, deploys, and invokes the gRPC Load Tester Supreme sample function Given I use NVCF CLI config "${REPO_ROOT}/tests/bdd/fixtures/nvcf-cli-local.yaml" diff --git a/tests/bdd/godog_test.go b/tests/bdd/godog_test.go index 9e95d1d4bd..8973e5dbcf 100644 --- a/tests/bdd/godog_test.go +++ b/tests/bdd/godog_test.go @@ -438,11 +438,14 @@ func TestSingleClusterHelmfileFeatureFileWiresToSteps(t *testing.T) { const vanityInvokeCommand = "/usr/bin/nvcf-cli --config /repo-root-placeholder/tests/bdd/fixtures/nvcf-cli-local.yaml" + " function invoke --vanity-host vanity.localhost --path /bdd/echo --timeout 120" + " --request-body '{\"message\":\"bdd-vanity-echo\",\"repeats\":1}'" + const helmInvokeCommand = "/usr/bin/nvcf-cli --config /repo-root-placeholder/tests/bdd/fixtures/nvcf-cli-local.yaml" + + " function invoke --request-body '{\"message\":\"bdd-helm-echo\",\"repeats\":1}' --timeout 120 --poll-duration 5" home := t.TempDir() t.Setenv("HOME", home) t.Setenv("NGC_API_KEY", "test-key") t.Setenv("SAMPLE_NGC_ORG", "test-org") t.Setenv("SAMPLE_NGC_TEAM", "test-team") + t.Setenv("SAMPLE_HELM_FUNCTION_CHART", "https://charts.example.test/inference-test-0.1.0.tgz") t.Setenv("NVCF_CLI", "/usr/bin/nvcf-cli") t.Setenv("REPO_ROOT", "/repo-root-placeholder") runner := newFakeRunner(map[string]harness.Result{ @@ -463,6 +466,10 @@ func TestSingleClusterHelmfileFeatureFileWiresToSteps(t *testing.T) { ExitCode: 0, Stdout: "Function invocation completed!\n\nResponse:\n{\"rawResponse\":\"bdd-vanity-echo\"}\n", }, + helmInvokeCommand: { + ExitCode: 0, + Stdout: "Function invocation completed!\n\nResponse:\n{\"rawResponse\":\"bdd-helm-echo\"}\n", + }, "/usr/bin/nvcf-cli --config /repo-root-placeholder/tests/bdd/fixtures/nvcf-cli-local.yaml function invoke" + " --grpc --grpc-plaintext --grpc-service Echo --grpc-method EchoMessage" + " --request-body '{\"message\":\"bdd-grpc-echo\"}' --timeout 120 --poll-duration 5": { @@ -563,6 +570,16 @@ func TestSingleClusterHelmfileFeatureFileWiresToSteps(t *testing.T) { "--description bdd-grpc-load-tester-supreme") { t.Fatal("gRPC sample function API key was not generated for the function service") } + if !commandRanThatContainsAll(suite.Runner.(*fakeRunner).runs, + "function create --name bdd-helm-function", + "--helm-chart https://charts.example.test/inference-test-0.1.0.tgz", + "--helm-chart-service entrypoint", + "--inference-url /echo --inference-port 8000") { + t.Fatal("Helm sample function was not created through the chart-rendering path") + } + if !commandRanExactly(suite.Runner.(*fakeRunner).runs, helmInvokeCommand) { + t.Fatal("Helm sample function was not invoked") + } if !commandRanThatContainsAll(suite.Runner.(*fakeRunner).runs, "function create --name bdd-openai-compatible-sample", "nvcf-openai-compatible-sample:local", @@ -576,7 +593,7 @@ func TestSingleClusterHelmfileFeatureFileWiresToSteps(t *testing.T) { if !commandRanThatContains(suite.Runner.(*fakeRunner).runs, "function delete --deployment-only") { t.Fatal("function deployment cleanup was never invoked") } - assertFunctionDeploymentsUseInstanceType(t, suite.Runner.(*fakeRunner).runs, "NCP.GPU.H100_1x", 3) + assertFunctionDeploymentsUseInstanceType(t, suite.Runner.(*fakeRunner).runs, "NCP.GPU.H100_1x", 4) if !commandRanThatContains(suite.Runner.(*fakeRunner).runs, "http://llm.localhost:8080/v1/chat/completions") { t.Fatal("unauthenticated LLM gateway check was never invoked") } @@ -1118,9 +1135,12 @@ func observabilityAllHelmListJSON() string { // multi-cluster feature targets the cp and compute clusters // explicitly. func TestMultiClusterHelmfileFeatureFileWiresToSteps(t *testing.T) { + const multiHelmInvokeCommand = "/usr/bin/nvcf-cli --config /repo-root-placeholder/tests/bdd/fixtures/nvcf-cli-local.yaml" + + " function invoke --request-body '{\"message\":\"bdd-multi-helm-echo\",\"repeats\":1}' --timeout 120 --poll-duration 5" t.Setenv("NGC_API_KEY", "test-key") t.Setenv("SAMPLE_NGC_ORG", "test-org") t.Setenv("SAMPLE_NGC_TEAM", "test-team") + t.Setenv("SAMPLE_HELM_FUNCTION_CHART", "https://charts.example.test/inference-test-0.1.0.tgz") t.Setenv("NVCF_CLI", "/usr/bin/nvcf-cli") t.Setenv("REPO_ROOT", "/repo-root-placeholder") const taskSmokeCommand = "env NVCT_BDD_TASK_INSTANCE_TYPE=NCP.GPU.H100_1x tests/bdd/scripts/run-nvct-task-smoke.sh" @@ -1168,6 +1188,10 @@ func TestMultiClusterHelmfileFeatureFileWiresToSteps(t *testing.T) { ExitCode: 0, Stdout: "Function invocation completed!\n\nResponse:\n{\"rawResponse\":\"bdd-echo\"}\n", }, + multiHelmInvokeCommand: { + ExitCode: 0, + Stdout: "Function invocation completed!\n\nResponse:\n{\"rawResponse\":\"bdd-multi-helm-echo\"}\n", + }, "/usr/bin/nvcf-cli --config /repo-root-placeholder/tests/bdd/fixtures/nvcf-cli-local.yaml function invoke" + " --grpc --grpc-plaintext --grpc-service Echo --grpc-method EchoMessage" + " --request-body '{\"message\":\"bdd-grpc-echo\"}' --timeout 120 --poll-duration 5": { @@ -1290,6 +1314,16 @@ func TestMultiClusterHelmfileFeatureFileWiresToSteps(t *testing.T) { "--description bdd-grpc-load-tester-supreme") { t.Fatal("gRPC sample function API key was not generated for the function service") } + if !commandRanThatContainsAll(suite.Runner.(*fakeRunner).runs, + "function create --name bdd-multi-helm-function", + "--helm-chart https://charts.example.test/inference-test-0.1.0.tgz", + "--helm-chart-service entrypoint", + "--inference-url /echo --inference-port 8000") { + t.Fatal("multi-cluster Helm sample function was not created through the chart-rendering path") + } + if !commandRanExactly(suite.Runner.(*fakeRunner).runs, multiHelmInvokeCommand) { + t.Fatal("multi-cluster Helm sample function was not invoked") + } if !commandRanThatContainsAll(suite.Runner.(*fakeRunner).runs, "function create --name bdd-multi-openai-compatible-sample", "nvcf-openai-compatible-sample:local", @@ -1311,8 +1345,8 @@ func TestMultiClusterHelmfileFeatureFileWiresToSteps(t *testing.T) { cleanupCount++ } } - if cleanupCount != 3 { - t.Fatalf("function deployment cleanup commands = %d, want 3", cleanupCount) + if cleanupCount != 4 { + t.Fatalf("function deployment cleanup commands = %d, want 4", cleanupCount) } if commandRanThatContains(suite.Runner.(*fakeRunner).runs, "api-key generate --description bdd-nvct-task-smoke") { t.Fatal("NVCT task smoke should not use nvcf-cli api-key generate because it emits function resources") @@ -1320,7 +1354,7 @@ func TestMultiClusterHelmfileFeatureFileWiresToSteps(t *testing.T) { if !commandRanExactly(suite.Runner.(*fakeRunner).runs, taskSmokeCommand) { t.Fatal("NVCT task API smoke script was not invoked with the local instance type") } - assertFunctionDeploymentsUseInstanceType(t, suite.Runner.(*fakeRunner).runs, "NCP.GPU.H100_1x", 3) + assertFunctionDeploymentsUseInstanceType(t, suite.Runner.(*fakeRunner).runs, "NCP.GPU.H100_1x", 4) } // TestMultiClusterHelmfileLLMRegistrationMultiregionFeatureFileWiresToSteps