diff --git a/deploy/stacks/observability/charts/nvcf-default-monitors/values.yaml b/deploy/stacks/observability/charts/nvcf-default-monitors/values.yaml index 7a587a968..70a62cf1d 100644 --- a/deploy/stacks/observability/charts/nvcf-default-monitors/values.yaml +++ b/deploy/stacks/observability/charts/nvcf-default-monitors/values.yaml @@ -92,5 +92,5 @@ computePlane: operator: Exists interval: 30s path: /metrics - port: metrics + port: worker-metrics labels: {} diff --git a/deploy/stacks/observability/environments/base.yaml b/deploy/stacks/observability/environments/base.yaml index e896371e5..bfd3928b4 100644 --- a/deploy/stacks/observability/environments/base.yaml +++ b/deploy/stacks/observability/environments/base.yaml @@ -188,5 +188,5 @@ defaultMonitors: operator: Exists interval: 30s path: /metrics - port: metrics + port: worker-metrics labels: {} diff --git a/deploy/stacks/observability/tests/profile-defaults.sh b/deploy/stacks/observability/tests/profile-defaults.sh index 60cb80dc3..ebde2b28d 100755 --- a/deploy/stacks/observability/tests/profile-defaults.sh +++ b/deploy/stacks/observability/tests/profile-defaults.sh @@ -59,6 +59,7 @@ profile_releases_csv() { paste -sd, - } +# render_monitors renders default monitor manifests for an observability profile. render_monitors() { local profile="$1" local output_name="$2" @@ -124,6 +125,7 @@ release_needs_csv() { paste -sd, - } +# assert_equal fails the test when actual and expected values differ. assert_equal() { local actual="$1" local expected="$2" @@ -133,6 +135,7 @@ assert_equal() { fail "$description: expected '$expected', got '$actual'" } +# assert_yaml_value verifies that a yq expression resolves to the expected value. assert_yaml_value() { local file="$1" local expression="$2" @@ -353,6 +356,12 @@ assert_yaml_value "$worker_monitor_manifest" \ assert_yaml_value "$worker_monitor_manifest" \ '.spec.selector.matchExpressions[0].operator' Exists \ 'worker pod label expression operator' +assert_yaml_value "$worker_monitor_manifest" \ + '.spec.podMetricsEndpoints[0].port' worker-metrics \ + 'worker metrics port' +assert_yaml_value "$work_dir/compute-monitor-values.yaml" \ + '.computePlane.worker.port' worker-metrics \ + 'compute worker metrics Helmfile value' # The application chart owns its Service labels. Compare them with the shared # ServiceMonitor selector so an application label change cannot silently break diff --git a/src/compute-plane-services/nvca/pkg/operator/reconcile/manifests/netpol/allow-ingress-monitoring.yaml b/src/compute-plane-services/nvca/pkg/operator/reconcile/manifests/netpol/allow-ingress-monitoring.yaml index 470017777..d4abce35d 100644 --- a/src/compute-plane-services/nvca/pkg/operator/reconcile/manifests/netpol/allow-ingress-monitoring.yaml +++ b/src/compute-plane-services/nvca/pkg/operator/reconcile/manifests/netpol/allow-ingress-monitoring.yaml @@ -28,6 +28,8 @@ spec: protocol: TCP - port: 8888 protocol: TCP + - port: worker-metrics + protocol: TCP - port: 10103 protocol: TCP - port: 18888 diff --git a/src/compute-plane-services/nvca/pkg/operator/reconcile/nvcaagent_reconcile_test.go b/src/compute-plane-services/nvca/pkg/operator/reconcile/nvcaagent_reconcile_test.go index cc948a332..6b0e40de7 100644 --- a/src/compute-plane-services/nvca/pkg/operator/reconcile/nvcaagent_reconcile_test.go +++ b/src/compute-plane-services/nvca/pkg/operator/reconcile/nvcaagent_reconcile_test.go @@ -41,6 +41,7 @@ import ( "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/utils/ptr" "sigs.k8s.io/yaml" @@ -2479,6 +2480,7 @@ func TestGetInternalPersistentStorageConfig(t *testing.T) { } } +// TestGetNetworkPoliciesDataEmptyDDCSIPList verifies generated network policies without DDCS CIDRs. func TestGetNetworkPoliciesDataEmptyDDCSIPList(t *testing.T) { expNPNames := []string{ EgressNetworkPolicyNameKey, @@ -2501,7 +2503,15 @@ func TestGetNetworkPoliciesDataEmptyDDCSIPList(t *testing.T) { got, err := bc.getNetworkPoliciesData(newTestContext(), nb) require.NoError(t, err) assert.Len(t, got, len(expNPNames)) - assertNetworkPolicyAllowsTCPPort(t, got[IngressNetworkPolicyNameKey], IngressNetworkPolicyNameKey, 8888) + assertNetworkPolicyAllowsTCPPort( + t, got[IngressNetworkPolicyNameKey], IngressNetworkPolicyNameKey, intstr.FromInt32(8888), + ) + assertNetworkPolicyAllowsTCPPort( + t, got[IngressNetworkPolicyNameKey], IngressNetworkPolicyNameKey, intstr.FromString("worker-metrics"), + ) + assertNetworkPolicyOmitsTCPPort( + t, got[IngressNetworkPolicyNameKey], IngressNetworkPolicyNameKey, intstr.FromInt32(9089), + ) b := &bytes.Buffer{} require.NoError(t, err) for _, k := range expNPNames { @@ -2511,6 +2521,7 @@ func TestGetNetworkPoliciesDataEmptyDDCSIPList(t *testing.T) { assert.Equal(t, stripSPDXHeaders(readTestdataFile(t, filepath.Join("testdata", "netpols.yaml"))), stripSPDXHeaders(b.String())) } +// TestGetNetworkPoliciesDataWithDDCSIPList verifies generated network policies with DDCS CIDRs. func TestGetNetworkPoliciesDataWithDDCSIPList(t *testing.T) { expNPNames := []string{ EgressNetworkPolicyNameKey, @@ -2534,7 +2545,15 @@ func TestGetNetworkPoliciesDataWithDDCSIPList(t *testing.T) { got, err := bc.getNetworkPoliciesData(newTestContext(), nb) require.NoError(t, err) assert.Len(t, got, len(expNPNames)) - assertNetworkPolicyAllowsTCPPort(t, got[IngressNetworkPolicyNameKey], IngressNetworkPolicyNameKey, 8888) + assertNetworkPolicyAllowsTCPPort( + t, got[IngressNetworkPolicyNameKey], IngressNetworkPolicyNameKey, intstr.FromInt32(8888), + ) + assertNetworkPolicyAllowsTCPPort( + t, got[IngressNetworkPolicyNameKey], IngressNetworkPolicyNameKey, intstr.FromString("worker-metrics"), + ) + assertNetworkPolicyOmitsTCPPort( + t, got[IngressNetworkPolicyNameKey], IngressNetworkPolicyNameKey, intstr.FromInt32(9089), + ) b := &bytes.Buffer{} require.NoError(t, err) for _, k := range expNPNames { @@ -2544,7 +2563,8 @@ func TestGetNetworkPoliciesDataWithDDCSIPList(t *testing.T) { assert.Equal(t, stripSPDXHeaders(readTestdataFile(t, filepath.Join("testdata", "netpols_with_ddcs.yaml"))), stripSPDXHeaders(b.String())) } -func assertNetworkPolicyAllowsTCPPort(t *testing.T, policyYAML, policyName string, port int32) { +// assertNetworkPolicyAllowsTCPPort verifies a policy allows a numeric or named TCP destination port. +func assertNetworkPolicyAllowsTCPPort(t *testing.T, policyYAML, policyName string, port intstr.IntOrString) { t.Helper() var policy netv1.NetworkPolicy @@ -2556,15 +2576,41 @@ func assertNetworkPolicyAllowsTCPPort(t *testing.T, policyYAML, policyName strin if networkPolicyPort.Port == nil || networkPolicyPort.Protocol == nil { continue } - if networkPolicyPort.Port.IntVal == port && *networkPolicyPort.Protocol == corev1.ProtocolTCP { + if *networkPolicyPort.Port == port && *networkPolicyPort.Protocol == corev1.ProtocolTCP { return } } } - assert.Failf(t, "missing TCP port", "%s should allow TCP port %d", policyName, port) + assert.Failf(t, "missing TCP port", "%s should allow TCP port %q", policyName, port.String()) +} + +// assertNetworkPolicyOmitsTCPPort verifies a policy does not explicitly declare a numeric or named TCP port. +func assertNetworkPolicyOmitsTCPPort(t *testing.T, policyYAML, policyName string, port intstr.IntOrString) { + t.Helper() + + var policy netv1.NetworkPolicy + require.NoError(t, yaml.Unmarshal([]byte(policyYAML), &policy)) + require.Equal(t, policyName, policy.Name) + + for _, ingressRule := range policy.Spec.Ingress { + for _, networkPolicyPort := range ingressRule.Ports { + if networkPolicyPort.Port == nil { + continue + } + protocol := corev1.ProtocolTCP + if networkPolicyPort.Protocol != nil { + protocol = *networkPolicyPort.Protocol + } + assert.Falsef(t, + *networkPolicyPort.Port == port && protocol == corev1.ProtocolTCP, + "%s should not explicitly declare TCP port %q", policyName, port.String(), + ) + } + } } +// TestGetEffectiveK8sNetworkCIDRs verifies the network CIDRs used by generated policies. func TestGetEffectiveK8sNetworkCIDRs(t *testing.T) { tests := []struct { name string diff --git a/src/compute-plane-services/nvca/pkg/operator/reconcile/testdata/netpols.yaml b/src/compute-plane-services/nvca/pkg/operator/reconcile/testdata/netpols.yaml index b6378b786..5fde375cc 100644 --- a/src/compute-plane-services/nvca/pkg/operator/reconcile/testdata/netpols.yaml +++ b/src/compute-plane-services/nvca/pkg/operator/reconcile/testdata/netpols.yaml @@ -74,6 +74,8 @@ spec: protocol: TCP - port: 8888 protocol: TCP + - port: worker-metrics + protocol: TCP - port: 10103 protocol: TCP - port: 18888 diff --git a/src/compute-plane-services/nvca/pkg/operator/reconcile/testdata/netpols_with_ddcs.yaml b/src/compute-plane-services/nvca/pkg/operator/reconcile/testdata/netpols_with_ddcs.yaml index 3236c5560..c1a497f41 100644 --- a/src/compute-plane-services/nvca/pkg/operator/reconcile/testdata/netpols_with_ddcs.yaml +++ b/src/compute-plane-services/nvca/pkg/operator/reconcile/testdata/netpols_with_ddcs.yaml @@ -80,6 +80,8 @@ spec: protocol: TCP - port: 8888 protocol: TCP + - port: worker-metrics + protocol: TCP - port: 10103 protocol: TCP - port: 18888 diff --git a/src/compute-plane-services/nvca/vendor/github.com/NVIDIA/nvcf/src/libraries/go/lib/pkg/icms-translate/translate/common/translator.go b/src/compute-plane-services/nvca/vendor/github.com/NVIDIA/nvcf/src/libraries/go/lib/pkg/icms-translate/translate/common/translator.go index caf7921d2..081f13406 100644 --- a/src/compute-plane-services/nvca/vendor/github.com/NVIDIA/nvcf/src/libraries/go/lib/pkg/icms-translate/translate/common/translator.go +++ b/src/compute-plane-services/nvca/vendor/github.com/NVIDIA/nvcf/src/libraries/go/lib/pkg/icms-translate/translate/common/translator.go @@ -32,11 +32,15 @@ import ( const ( UtilsHealthPort int32 = 8080 + // UtilsMetricsPort is the TCP port exposed by worker-utils for Prometheus metrics. + UtilsMetricsPort int32 = 8010 UtilsContainerName = "utils" - UtilsPodName = "utils" - InitContainerName = "init" - ESSContainerName = "ess" + // WorkerMetricsPortName is the shared named port used by worker PodMonitor targets. + WorkerMetricsPortName = "worker-metrics" + UtilsPodName = "utils" + InitContainerName = "init" + ESSContainerName = "ess" TermLogPath = "/dev/termination-log" DevShmPath = "/dev/shm" @@ -330,6 +334,16 @@ func AddNVIDIAGPUNoScheduleToleration(podSpec *corev1.PodSpec) (added bool) { }) } +// MutateUtilsContainer declares the worker metrics port and configures the health probes. +func MutateUtilsContainer(containerSpec *corev1.Container) { + containerSpec.Ports = append(containerSpec.Ports, corev1.ContainerPort{ + Name: WorkerMetricsPortName, + ContainerPort: UtilsMetricsPort, + Protocol: corev1.ProtocolTCP, + }) + MutateUtilsProbes(containerSpec) +} + func MutateUtilsProbes(containerSpec *corev1.Container) { const ( liveEndpoint = "/v1/health/live" diff --git a/src/compute-plane-services/nvca/vendor/github.com/NVIDIA/nvcf/src/libraries/go/lib/pkg/icms-translate/translate/function/llm.go b/src/compute-plane-services/nvca/vendor/github.com/NVIDIA/nvcf/src/libraries/go/lib/pkg/icms-translate/translate/function/llm.go index f458147f9..337ef7096 100644 --- a/src/compute-plane-services/nvca/vendor/github.com/NVIDIA/nvcf/src/libraries/go/lib/pkg/icms-translate/translate/function/llm.go +++ b/src/compute-plane-services/nvca/vendor/github.com/NVIDIA/nvcf/src/libraries/go/lib/pkg/icms-translate/translate/function/llm.go @@ -30,6 +30,8 @@ import ( const ( LLMWorkerContainerName = "llm-worker" + // llmMetricsPort is the TCP port exposed by Pylon for Prometheus metrics. + llmMetricsPort int32 = 9089 //nolint:gosec llmCredentialManagerImageEnv = "LLM_CREDENTIAL_MANAGER_IMAGE" @@ -81,6 +83,7 @@ func upstreamHealthPath(allEnvSet map[string]string) string { return path } +// newLLMRouterClientContainer builds the Pylon sidecar for an LLM worker. func newLLMRouterClientContainer( ls *LaunchSpecification, allEnvSet map[string]string, @@ -169,8 +172,13 @@ func newLLMRouterClientContainer( Name: LLMWorkerContainerName, Image: llmRouterClientImage, ImagePullPolicy: corev1.PullIfNotPresent, - Args: args, - Env: common.SortEnvs(envs), + Ports: []corev1.ContainerPort{{ + Name: common.WorkerMetricsPortName, + ContainerPort: llmMetricsPort, + Protocol: corev1.ProtocolTCP, + }}, + Args: args, + Env: common.SortEnvs(envs), Resources: corev1.ResourceRequirements{ Requests: corev1.ResourceList{ corev1.ResourceCPU: *resource.NewMilliQuantity(500, resource.DecimalSI), diff --git a/src/compute-plane-services/nvca/vendor/github.com/NVIDIA/nvcf/src/libraries/go/lib/pkg/icms-translate/translate/function/translate_container.go b/src/compute-plane-services/nvca/vendor/github.com/NVIDIA/nvcf/src/libraries/go/lib/pkg/icms-translate/translate/function/translate_container.go index d00614633..931246439 100644 --- a/src/compute-plane-services/nvca/vendor/github.com/NVIDIA/nvcf/src/libraries/go/lib/pkg/icms-translate/translate/function/translate_container.go +++ b/src/compute-plane-services/nvca/vendor/github.com/NVIDIA/nvcf/src/libraries/go/lib/pkg/icms-translate/translate/function/translate_container.go @@ -33,6 +33,8 @@ import ( "github.com/NVIDIA/nvcf/src/libraries/go/lib/pkg/otelconfig/backendconfig" ) +// translateContainer converts a container function request into Kubernetes resources. +// //nolint:gocyclo // complex function with many conditional branches func translateContainer(t CreationQueueMessage, tcfg TranslateConfig) (objs []metav1.Object, err error) { if err := tcfg.ValidateContainer(); err != nil { @@ -431,7 +433,7 @@ func translateContainer(t CreationQueueMessage, tcfg TranslateConfig) (objs []me VolumeMounts: utilsContainerVolumeMounts, } // mutate startup / liveness / readiness probes - common.MutateUtilsProbes(&utilsContainer) + common.MutateUtilsContainer(&utilsContainer) pod.Spec.Containers = append(pod.Spec.Containers, utilsContainer) } else { // llm functions don't have a utils container. they have a router client container instead. diff --git a/src/compute-plane-services/nvca/vendor/github.com/NVIDIA/nvcf/src/libraries/go/lib/pkg/icms-translate/translate/function/translate_container_utdep.go b/src/compute-plane-services/nvca/vendor/github.com/NVIDIA/nvcf/src/libraries/go/lib/pkg/icms-translate/translate/function/translate_container_utdep.go index b5a6fbe86..a6fe3beed 100644 --- a/src/compute-plane-services/nvca/vendor/github.com/NVIDIA/nvcf/src/libraries/go/lib/pkg/icms-translate/translate/function/translate_container_utdep.go +++ b/src/compute-plane-services/nvca/vendor/github.com/NVIDIA/nvcf/src/libraries/go/lib/pkg/icms-translate/translate/function/translate_container_utdep.go @@ -32,12 +32,8 @@ import ( translateutil "github.com/NVIDIA/nvcf/src/libraries/go/lib/pkg/icms-translate/translate/util" ) -/* -Container Functions will be translated as the following set of artifacts - - an inference Pod - - a utils deployment - - a service endpoint for inference -*/ +// translateContainerUtilsDeploy converts a container function request into an +// inference pod, a worker-utils deployment, and an inference service. func translateContainerUtilsDeploy(t CreationQueueMessage, tcfg TranslateConfig) (objs []metav1.Object, err error) { if err := tcfg.ValidateContainer(); err != nil { return nil, fmt.Errorf("invalid container translate config: %v", err) @@ -285,6 +281,7 @@ func translateContainerUtilsDeploy(t CreationQueueMessage, tcfg TranslateConfig) return objs, nil } +// getUtilsDeploymentAndSecrets builds the worker-utils deployment and registry secrets. func getUtilsDeploymentAndSecrets(t CreationQueueMessage, tcfg TranslateConfig) (objs []metav1.Object, err error) { // Fail if LLS is enabled for utils pod as a separate pod from the inference pod if t.Details.FunctionType == FunctionTypeStreaming { @@ -553,7 +550,7 @@ func getUtilsDeploymentAndSecrets(t CreationQueueMessage, tcfg TranslateConfig) VolumeMounts: utilsContainerVolumeMounts, } // mutate startup / liveness / readiness probes - common.MutateUtilsProbes(&utilsContainer) + common.MutateUtilsContainer(&utilsContainer) utilsPod.Spec.Containers = append(utilsPod.Spec.Containers, utilsContainer) replicas := int32(1) diff --git a/src/compute-plane-services/nvca/vendor/github.com/NVIDIA/nvcf/src/libraries/go/lib/pkg/icms-translate/translate/function/translate_helm.go b/src/compute-plane-services/nvca/vendor/github.com/NVIDIA/nvcf/src/libraries/go/lib/pkg/icms-translate/translate/function/translate_helm.go index ef526fef5..54daa6539 100644 --- a/src/compute-plane-services/nvca/vendor/github.com/NVIDIA/nvcf/src/libraries/go/lib/pkg/icms-translate/translate/function/translate_helm.go +++ b/src/compute-plane-services/nvca/vendor/github.com/NVIDIA/nvcf/src/libraries/go/lib/pkg/icms-translate/translate/function/translate_helm.go @@ -31,6 +31,8 @@ import ( translateutil "github.com/NVIDIA/nvcf/src/libraries/go/lib/pkg/icms-translate/translate/util" ) +// translateHelmChart converts a Helm function request into Kubernetes resources. +// //nolint:gocyclo // complex function with many conditional branches func translateHelmChart(t CreationQueueMessage, tcfg TranslateConfig) (objs []metav1.Object, err error) { // Fail if LLS is enabled for utils pod as a separate pod from the inference pod @@ -351,7 +353,7 @@ func translateHelmChart(t CreationQueueMessage, tcfg TranslateConfig) (objs []me VolumeMounts: utilsContainerVolumeMounts, } // mutate startup / liveness / readiness probes - common.MutateUtilsProbes(&utilsContainer) + common.MutateUtilsContainer(&utilsContainer) utilsPod.Spec.Containers = append(utilsPod.Spec.Containers, utilsContainer) } else { // llm functions don't have a utils container. they have a router client container instead. diff --git a/src/compute-plane-services/nvca/vendor/github.com/NVIDIA/nvcf/src/libraries/go/lib/pkg/icms-translate/translate/function/translate_helm_utdep.go b/src/compute-plane-services/nvca/vendor/github.com/NVIDIA/nvcf/src/libraries/go/lib/pkg/icms-translate/translate/function/translate_helm_utdep.go index fc26782bd..c98e38d0a 100644 --- a/src/compute-plane-services/nvca/vendor/github.com/NVIDIA/nvcf/src/libraries/go/lib/pkg/icms-translate/translate/function/translate_helm_utdep.go +++ b/src/compute-plane-services/nvca/vendor/github.com/NVIDIA/nvcf/src/libraries/go/lib/pkg/icms-translate/translate/function/translate_helm_utdep.go @@ -30,6 +30,8 @@ import ( translateutil "github.com/NVIDIA/nvcf/src/libraries/go/lib/pkg/icms-translate/translate/util" ) +// translateHelmChartUtilsDeploy builds Helm workload resources with worker-utils +// in a separate deployment. func translateHelmChartUtilsDeploy(t CreationQueueMessage, tcfg TranslateConfig) (objs []metav1.Object, err error) { // Fail if LLS is enabled for utils pod as a separate pod from the inference pod if t.Details.FunctionType == FunctionTypeStreaming { @@ -319,7 +321,7 @@ func translateHelmChartUtilsDeploy(t CreationQueueMessage, tcfg TranslateConfig) VolumeMounts: utilsContainerVolumeMounts, } // mutate startup / liveness / readiness probes - common.MutateUtilsProbes(&utilsContainer) + common.MutateUtilsContainer(&utilsContainer) utilsPod.Spec.Containers = append(utilsPod.Spec.Containers, utilsContainer) replicas := int32(1) diff --git a/src/compute-plane-services/nvca/vendor/github.com/NVIDIA/nvcf/src/libraries/go/lib/pkg/icms-translate/translate/task/translate.go b/src/compute-plane-services/nvca/vendor/github.com/NVIDIA/nvcf/src/libraries/go/lib/pkg/icms-translate/translate/task/translate.go index d6cf3e419..3bfd20165 100644 --- a/src/compute-plane-services/nvca/vendor/github.com/NVIDIA/nvcf/src/libraries/go/lib/pkg/icms-translate/translate/task/translate.go +++ b/src/compute-plane-services/nvca/vendor/github.com/NVIDIA/nvcf/src/libraries/go/lib/pkg/icms-translate/translate/task/translate.go @@ -37,6 +37,8 @@ import ( // TODO: k8s version considerations. // For now, all task Pod features are supported by all recent k8s versions +// translateContainer converts a container task request into Kubernetes resources. +// //nolint:gocyclo // complex function with many conditional branches func translateContainer(t CreationQueueMessage, tcfg TranslateConfig) (objs []metav1.Object, err error) { tcfg.Default() @@ -435,7 +437,7 @@ func translateContainer(t CreationQueueMessage, tcfg TranslateConfig) (objs []me VolumeMounts: utilsContainerVolumeMounts, } // mutate startup / liveness / readiness probes - common.MutateUtilsProbes(&utilsContainer) + common.MutateUtilsContainer(&utilsContainer) pod.Spec.Containers = append(pod.Spec.Containers, utilsContainer) // Setup telemetry for the pod diff --git a/src/compute-plane-services/nvca/vendor/github.com/NVIDIA/nvcf/src/libraries/go/lib/pkg/icms-translate/translate/task/translate_helm.go b/src/compute-plane-services/nvca/vendor/github.com/NVIDIA/nvcf/src/libraries/go/lib/pkg/icms-translate/translate/task/translate_helm.go index 56332d37d..36a30ee4e 100644 --- a/src/compute-plane-services/nvca/vendor/github.com/NVIDIA/nvcf/src/libraries/go/lib/pkg/icms-translate/translate/task/translate_helm.go +++ b/src/compute-plane-services/nvca/vendor/github.com/NVIDIA/nvcf/src/libraries/go/lib/pkg/icms-translate/translate/task/translate_helm.go @@ -31,6 +31,8 @@ import ( translateutil "github.com/NVIDIA/nvcf/src/libraries/go/lib/pkg/icms-translate/translate/util" ) +// translateHelmChart converts a Helm task request into Kubernetes resources. +// //nolint:gocyclo // complex function with many conditional branches func translateHelmChart(t CreationQueueMessage, tcfg TranslateConfig) (objs []metav1.Object, err error) { if err := tcfg.ValidateHelmChart(); err != nil { @@ -356,7 +358,7 @@ func translateHelmChart(t CreationQueueMessage, tcfg TranslateConfig) (objs []me VolumeMounts: utilsContainerVolumeMounts, } // mutate startup / liveness / readiness probes - common.MutateUtilsProbes(&utilsContainer) + common.MutateUtilsContainer(&utilsContainer) utilsPod.Spec.Containers = append(utilsPod.Spec.Containers, utilsContainer) if hasTelemetries { diff --git a/src/libraries/go/lib/pkg/icms-translate/translate/common/translator.go b/src/libraries/go/lib/pkg/icms-translate/translate/common/translator.go index caf7921d2..081f13406 100644 --- a/src/libraries/go/lib/pkg/icms-translate/translate/common/translator.go +++ b/src/libraries/go/lib/pkg/icms-translate/translate/common/translator.go @@ -32,11 +32,15 @@ import ( const ( UtilsHealthPort int32 = 8080 + // UtilsMetricsPort is the TCP port exposed by worker-utils for Prometheus metrics. + UtilsMetricsPort int32 = 8010 UtilsContainerName = "utils" - UtilsPodName = "utils" - InitContainerName = "init" - ESSContainerName = "ess" + // WorkerMetricsPortName is the shared named port used by worker PodMonitor targets. + WorkerMetricsPortName = "worker-metrics" + UtilsPodName = "utils" + InitContainerName = "init" + ESSContainerName = "ess" TermLogPath = "/dev/termination-log" DevShmPath = "/dev/shm" @@ -330,6 +334,16 @@ func AddNVIDIAGPUNoScheduleToleration(podSpec *corev1.PodSpec) (added bool) { }) } +// MutateUtilsContainer declares the worker metrics port and configures the health probes. +func MutateUtilsContainer(containerSpec *corev1.Container) { + containerSpec.Ports = append(containerSpec.Ports, corev1.ContainerPort{ + Name: WorkerMetricsPortName, + ContainerPort: UtilsMetricsPort, + Protocol: corev1.ProtocolTCP, + }) + MutateUtilsProbes(containerSpec) +} + func MutateUtilsProbes(containerSpec *corev1.Container) { const ( liveEndpoint = "/v1/health/live" diff --git a/src/libraries/go/lib/pkg/icms-translate/translate/common/translator_test.go b/src/libraries/go/lib/pkg/icms-translate/translate/common/translator_test.go index e792f1f3b..38580f925 100644 --- a/src/libraries/go/lib/pkg/icms-translate/translate/common/translator_test.go +++ b/src/libraries/go/lib/pkg/icms-translate/translate/common/translator_test.go @@ -25,6 +25,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) +// TestAddNVIDIAGPUNoScheduleToleration verifies that the GPU toleration is added only when absent. func TestAddNVIDIAGPUNoScheduleToleration(t *testing.T) { tests := []struct { name string @@ -88,6 +89,22 @@ func TestAddNVIDIAGPUNoScheduleToleration(t *testing.T) { } } +// TestMutateUtilsContainer verifies the worker-utils metrics port and health probes. +func TestMutateUtilsContainer(t *testing.T) { + container := corev1.Container{} + + MutateUtilsContainer(&container) + + assert.Equal(t, []corev1.ContainerPort{{ + Name: WorkerMetricsPortName, + ContainerPort: UtilsMetricsPort, + Protocol: corev1.ProtocolTCP, + }}, container.Ports) + assert.NotNil(t, container.StartupProbe) + assert.NotNil(t, container.ReadinessProbe) + assert.NotNil(t, container.LivenessProbe) +} + func TestMergeTolerations(t *testing.T) { t.Run("adds_new_tolerations_and_skips_exact_duplicates", func(t *testing.T) { noExecuteSeconds := int64(30) diff --git a/src/libraries/go/lib/pkg/icms-translate/translate/function/llm.go b/src/libraries/go/lib/pkg/icms-translate/translate/function/llm.go index f458147f9..337ef7096 100644 --- a/src/libraries/go/lib/pkg/icms-translate/translate/function/llm.go +++ b/src/libraries/go/lib/pkg/icms-translate/translate/function/llm.go @@ -30,6 +30,8 @@ import ( const ( LLMWorkerContainerName = "llm-worker" + // llmMetricsPort is the TCP port exposed by Pylon for Prometheus metrics. + llmMetricsPort int32 = 9089 //nolint:gosec llmCredentialManagerImageEnv = "LLM_CREDENTIAL_MANAGER_IMAGE" @@ -81,6 +83,7 @@ func upstreamHealthPath(allEnvSet map[string]string) string { return path } +// newLLMRouterClientContainer builds the Pylon sidecar for an LLM worker. func newLLMRouterClientContainer( ls *LaunchSpecification, allEnvSet map[string]string, @@ -169,8 +172,13 @@ func newLLMRouterClientContainer( Name: LLMWorkerContainerName, Image: llmRouterClientImage, ImagePullPolicy: corev1.PullIfNotPresent, - Args: args, - Env: common.SortEnvs(envs), + Ports: []corev1.ContainerPort{{ + Name: common.WorkerMetricsPortName, + ContainerPort: llmMetricsPort, + Protocol: corev1.ProtocolTCP, + }}, + Args: args, + Env: common.SortEnvs(envs), Resources: corev1.ResourceRequirements{ Requests: corev1.ResourceList{ corev1.ResourceCPU: *resource.NewMilliQuantity(500, resource.DecimalSI), diff --git a/src/libraries/go/lib/pkg/icms-translate/translate/function/llm_test.go b/src/libraries/go/lib/pkg/icms-translate/translate/function/llm_test.go index 8157e6b0c..d7f91fa34 100644 --- a/src/libraries/go/lib/pkg/icms-translate/translate/function/llm_test.go +++ b/src/libraries/go/lib/pkg/icms-translate/translate/function/llm_test.go @@ -57,6 +57,7 @@ func healthPathArgs(args []string) []string { return healthPaths } +// TestNewLLMRouterClientContainer verifies Pylon configuration and validation. func TestNewLLMRouterClientContainer(t *testing.T) { type spec struct { name string @@ -381,6 +382,11 @@ func TestNewLLMRouterClientContainer(t *testing.T) { return } require.NoError(t, err) + assert.Equal(t, []corev1.ContainerPort{{ + Name: common.WorkerMetricsPortName, + ContainerPort: llmMetricsPort, + Protocol: corev1.ProtocolTCP, + }}, c.Ports) tt.validate(t, c) }) } diff --git a/src/libraries/go/lib/pkg/icms-translate/translate/function/translate_container.go b/src/libraries/go/lib/pkg/icms-translate/translate/function/translate_container.go index d00614633..931246439 100644 --- a/src/libraries/go/lib/pkg/icms-translate/translate/function/translate_container.go +++ b/src/libraries/go/lib/pkg/icms-translate/translate/function/translate_container.go @@ -33,6 +33,8 @@ import ( "github.com/NVIDIA/nvcf/src/libraries/go/lib/pkg/otelconfig/backendconfig" ) +// translateContainer converts a container function request into Kubernetes resources. +// //nolint:gocyclo // complex function with many conditional branches func translateContainer(t CreationQueueMessage, tcfg TranslateConfig) (objs []metav1.Object, err error) { if err := tcfg.ValidateContainer(); err != nil { @@ -431,7 +433,7 @@ func translateContainer(t CreationQueueMessage, tcfg TranslateConfig) (objs []me VolumeMounts: utilsContainerVolumeMounts, } // mutate startup / liveness / readiness probes - common.MutateUtilsProbes(&utilsContainer) + common.MutateUtilsContainer(&utilsContainer) pod.Spec.Containers = append(pod.Spec.Containers, utilsContainer) } else { // llm functions don't have a utils container. they have a router client container instead. diff --git a/src/libraries/go/lib/pkg/icms-translate/translate/function/translate_container_utdep.go b/src/libraries/go/lib/pkg/icms-translate/translate/function/translate_container_utdep.go index b5a6fbe86..a6fe3beed 100644 --- a/src/libraries/go/lib/pkg/icms-translate/translate/function/translate_container_utdep.go +++ b/src/libraries/go/lib/pkg/icms-translate/translate/function/translate_container_utdep.go @@ -32,12 +32,8 @@ import ( translateutil "github.com/NVIDIA/nvcf/src/libraries/go/lib/pkg/icms-translate/translate/util" ) -/* -Container Functions will be translated as the following set of artifacts - - an inference Pod - - a utils deployment - - a service endpoint for inference -*/ +// translateContainerUtilsDeploy converts a container function request into an +// inference pod, a worker-utils deployment, and an inference service. func translateContainerUtilsDeploy(t CreationQueueMessage, tcfg TranslateConfig) (objs []metav1.Object, err error) { if err := tcfg.ValidateContainer(); err != nil { return nil, fmt.Errorf("invalid container translate config: %v", err) @@ -285,6 +281,7 @@ func translateContainerUtilsDeploy(t CreationQueueMessage, tcfg TranslateConfig) return objs, nil } +// getUtilsDeploymentAndSecrets builds the worker-utils deployment and registry secrets. func getUtilsDeploymentAndSecrets(t CreationQueueMessage, tcfg TranslateConfig) (objs []metav1.Object, err error) { // Fail if LLS is enabled for utils pod as a separate pod from the inference pod if t.Details.FunctionType == FunctionTypeStreaming { @@ -553,7 +550,7 @@ func getUtilsDeploymentAndSecrets(t CreationQueueMessage, tcfg TranslateConfig) VolumeMounts: utilsContainerVolumeMounts, } // mutate startup / liveness / readiness probes - common.MutateUtilsProbes(&utilsContainer) + common.MutateUtilsContainer(&utilsContainer) utilsPod.Spec.Containers = append(utilsPod.Spec.Containers, utilsContainer) replicas := int32(1) diff --git a/src/libraries/go/lib/pkg/icms-translate/translate/function/translate_helm.go b/src/libraries/go/lib/pkg/icms-translate/translate/function/translate_helm.go index ef526fef5..54daa6539 100644 --- a/src/libraries/go/lib/pkg/icms-translate/translate/function/translate_helm.go +++ b/src/libraries/go/lib/pkg/icms-translate/translate/function/translate_helm.go @@ -31,6 +31,8 @@ import ( translateutil "github.com/NVIDIA/nvcf/src/libraries/go/lib/pkg/icms-translate/translate/util" ) +// translateHelmChart converts a Helm function request into Kubernetes resources. +// //nolint:gocyclo // complex function with many conditional branches func translateHelmChart(t CreationQueueMessage, tcfg TranslateConfig) (objs []metav1.Object, err error) { // Fail if LLS is enabled for utils pod as a separate pod from the inference pod @@ -351,7 +353,7 @@ func translateHelmChart(t CreationQueueMessage, tcfg TranslateConfig) (objs []me VolumeMounts: utilsContainerVolumeMounts, } // mutate startup / liveness / readiness probes - common.MutateUtilsProbes(&utilsContainer) + common.MutateUtilsContainer(&utilsContainer) utilsPod.Spec.Containers = append(utilsPod.Spec.Containers, utilsContainer) } else { // llm functions don't have a utils container. they have a router client container instead. diff --git a/src/libraries/go/lib/pkg/icms-translate/translate/function/translate_helm_utdep.go b/src/libraries/go/lib/pkg/icms-translate/translate/function/translate_helm_utdep.go index fc26782bd..c98e38d0a 100644 --- a/src/libraries/go/lib/pkg/icms-translate/translate/function/translate_helm_utdep.go +++ b/src/libraries/go/lib/pkg/icms-translate/translate/function/translate_helm_utdep.go @@ -30,6 +30,8 @@ import ( translateutil "github.com/NVIDIA/nvcf/src/libraries/go/lib/pkg/icms-translate/translate/util" ) +// translateHelmChartUtilsDeploy builds Helm workload resources with worker-utils +// in a separate deployment. func translateHelmChartUtilsDeploy(t CreationQueueMessage, tcfg TranslateConfig) (objs []metav1.Object, err error) { // Fail if LLS is enabled for utils pod as a separate pod from the inference pod if t.Details.FunctionType == FunctionTypeStreaming { @@ -319,7 +321,7 @@ func translateHelmChartUtilsDeploy(t CreationQueueMessage, tcfg TranslateConfig) VolumeMounts: utilsContainerVolumeMounts, } // mutate startup / liveness / readiness probes - common.MutateUtilsProbes(&utilsContainer) + common.MutateUtilsContainer(&utilsContainer) utilsPod.Spec.Containers = append(utilsPod.Spec.Containers, utilsContainer) replicas := int32(1) diff --git a/src/libraries/go/lib/pkg/icms-translate/translate/function/translate_tolerations_test.go b/src/libraries/go/lib/pkg/icms-translate/translate/function/translate_tolerations_test.go index ece4d14e8..e47b46610 100644 --- a/src/libraries/go/lib/pkg/icms-translate/translate/function/translate_tolerations_test.go +++ b/src/libraries/go/lib/pkg/icms-translate/translate/function/translate_tolerations_test.go @@ -34,6 +34,8 @@ import ( "github.com/NVIDIA/nvcf/src/libraries/go/lib/pkg/icms-translate/translate/common" ) +// TestTranslateContainer_AppliesConfiguredTolerations verifies configured +// tolerations on container function pods. func TestTranslateContainer_AppliesConfiguredTolerations(t *testing.T) { customToleration := corev1.Toleration{ Key: "dedicated", @@ -68,6 +70,8 @@ func TestTranslateContainer_AppliesConfiguredTolerations(t *testing.T) { ) } +// TestTranslateContainerUtilsDeploy_AppliesConfiguredTolerations verifies +// tolerations and metrics in split deployment mode. func TestTranslateContainerUtilsDeploy_AppliesConfiguredTolerations(t *testing.T) { customToleration := corev1.Toleration{ Key: "workload-type", @@ -104,8 +108,13 @@ func TestTranslateContainerUtilsDeploy_AppliesConfiguredTolerations(t *testing.T Effect: corev1.TaintEffectNoSchedule, }, ) + assertUtilsMetricsPort(t, findContainerByName( + t, utilsDeployment.Spec.Template.Spec.Containers, common.UtilsContainerName, + )) } +// TestTranslateHelmChartUtilsDeploy_AppliesConfiguredTolerations verifies Helm +// tolerations and worker metrics. func TestTranslateHelmChartUtilsDeploy_AppliesConfiguredTolerations(t *testing.T) { customToleration := corev1.Toleration{ Key: "workload-type", @@ -132,8 +141,13 @@ func TestTranslateHelmChartUtilsDeploy_AppliesConfiguredTolerations(t *testing.T Effect: corev1.TaintEffectNoSchedule, }, ) + assertUtilsMetricsPort(t, findContainerByName( + t, utilsDeployment.Spec.Template.Spec.Containers, common.UtilsContainerName, + )) } +// TestTranslateHelmChartLLM_AddsRouterAndCredentialContainers verifies the Helm +// LLM sidecars and metrics port. func TestTranslateHelmChartLLM_AddsRouterAndCredentialContainers(t *testing.T) { msg := newHelmFunctionMessage() msg.Details.FunctionType = FunctionTypeLLM @@ -169,6 +183,7 @@ func TestTranslateHelmChartLLM_AddsRouterAndCredentialContainers(t *testing.T) { require.Len(t, utilsPod.Spec.Containers, 2) router := findContainerByName(t, utilsPod.Spec.Containers, LLMWorkerContainerName) + assertLLMMetricsPort(t, router) assert.Equal(t, "nvcr.io/nvidia/router:latest", router.Image) assert.Contains(t, router.Args, "--upstream-http-base-url=http://inference-svc:8080") assert.Contains(t, router.Args, "--stargate-address=llm-router.example.com:443") @@ -194,6 +209,8 @@ func TestTranslateHelmChartLLM_AddsRouterAndCredentialContainers(t *testing.T) { require.NotNil(t, findVolumeByName(t, utilsPod.Spec.Volumes, "llm").EmptyDir) } +// TestTranslateContainerLLM_AddsRouterAndCredentialContainers verifies the +// container LLM sidecars and metrics port. func TestTranslateContainerLLM_AddsRouterAndCredentialContainers(t *testing.T) { msg := newContainerFunctionMessage() msg.Details.FunctionType = FunctionTypeLLM @@ -228,6 +245,7 @@ func TestTranslateContainerLLM_AddsRouterAndCredentialContainers(t *testing.T) { assert.Equal(t, "nvcr.io/nvidia/function:latest", findContainerByName(t, pod.Spec.Containers, inferenceContainerName).Image) router := findContainerByName(t, pod.Spec.Containers, LLMWorkerContainerName) + assertLLMMetricsPort(t, router) assert.Equal(t, "nvcr.io/nvidia/router:latest", router.Image) assert.Contains(t, router.Args, "--upstream-http-base-url=http://127.0.0.1:8080") assert.Contains(t, router.Args, "--stargate-address=llm-router.example.com:443") @@ -328,6 +346,7 @@ func TestTranslateContainerLLMWithSecretsSharesAssertionTokenWithCredentialManag require.NotNil(t, findVolumeByName(t, pod.Spec.Volumes, common.EssDataVolumeName).EmptyDir) } +// TestTranslateNonLLMFunctionsDoNotAddCredentialManager verifies non-LLM workloads omit LLM sidecars. func TestTranslateNonLLMFunctionsDoNotAddCredentialManager(t *testing.T) { for _, functionType := range []string{FunctionTypeDefault, FunctionTypeStreaming} { t.Run(functionType, func(t *testing.T) { @@ -350,6 +369,8 @@ func TestTranslateNonLLMFunctionsDoNotAddCredentialManager(t *testing.T) { } } +// TestTranslateContainerWithCacheAndSecrets verifies cache, secret, and +// worker-utils resources for container functions. func TestTranslateContainerWithCacheAndSecrets(t *testing.T) { msg := newContainerFunctionMessage() msg.LaunchSpecification.EnvironmentB64 = encodeTextEnv(map[string]string{ @@ -376,6 +397,7 @@ func TestTranslateContainerWithCacheAndSecrets(t *testing.T) { require.NoError(t, err) pod := findPodByName(t, objs, "0-request") + assertUtilsMetricsPort(t, findContainerByName(t, pod.Spec.Containers, common.UtilsContainerName)) assert.Equal(t, "value", envSliceToMap(findContainerByName(t, pod.Spec.Containers, inferenceContainerName).Env)["INFERENCE_ENV"]) assert.Equal(t, "ess", findContainerByName(t, pod.Spec.Containers, "ess").Name) require.Len(t, pod.Spec.InitContainers, 2) @@ -384,6 +406,8 @@ func TestTranslateContainerWithCacheAndSecrets(t *testing.T) { assert.NotNil(t, findObjectByName(t, objs, "writer-job-function-cache")) } +// TestTranslateHelmChartWithCacheAndSecrets verifies cache, secret, and +// worker-utils resources for Helm functions. func TestTranslateHelmChartWithCacheAndSecrets(t *testing.T) { msg := newHelmFunctionMessage() msg.LaunchSpecification.EnvironmentB64 = encodeTextEnv(map[string]string{ @@ -409,7 +433,7 @@ func TestTranslateHelmChartWithCacheAndSecrets(t *testing.T) { pod := findPodByName(t, objs, common.UtilsPodName) assert.Equal(t, "ess", findContainerByName(t, pod.Spec.Containers, "ess").Name) - assert.Equal(t, common.UtilsContainerName, findContainerByName(t, pod.Spec.Containers, common.UtilsContainerName).Name) + assertUtilsMetricsPort(t, findContainerByName(t, pod.Spec.Containers, common.UtilsContainerName)) require.Len(t, pod.Spec.InitContainers, 2) assert.Equal(t, "ess-init", pod.Spec.InitContainers[1].Name) assert.NotNil(t, findObjectByName(t, objs, "rw-pvc-helm-function-cache")) @@ -677,3 +701,23 @@ func assertTolerationsMatch(t *testing.T, got []corev1.Toleration, want ...corev t.Helper() assert.ElementsMatch(t, want, got) } + +// assertUtilsMetricsPort verifies the named worker-utils metrics port. +func assertUtilsMetricsPort(t *testing.T, container corev1.Container) { + t.Helper() + assert.Equal(t, []corev1.ContainerPort{{ + Name: common.WorkerMetricsPortName, + ContainerPort: common.UtilsMetricsPort, + Protocol: corev1.ProtocolTCP, + }}, container.Ports) +} + +// assertLLMMetricsPort verifies the named Pylon metrics port. +func assertLLMMetricsPort(t *testing.T, container corev1.Container) { + t.Helper() + assert.Equal(t, []corev1.ContainerPort{{ + Name: common.WorkerMetricsPortName, + ContainerPort: llmMetricsPort, + Protocol: corev1.ProtocolTCP, + }}, container.Ports) +} diff --git a/src/libraries/go/lib/pkg/icms-translate/translate/task/translate.go b/src/libraries/go/lib/pkg/icms-translate/translate/task/translate.go index d6cf3e419..3bfd20165 100644 --- a/src/libraries/go/lib/pkg/icms-translate/translate/task/translate.go +++ b/src/libraries/go/lib/pkg/icms-translate/translate/task/translate.go @@ -37,6 +37,8 @@ import ( // TODO: k8s version considerations. // For now, all task Pod features are supported by all recent k8s versions +// translateContainer converts a container task request into Kubernetes resources. +// //nolint:gocyclo // complex function with many conditional branches func translateContainer(t CreationQueueMessage, tcfg TranslateConfig) (objs []metav1.Object, err error) { tcfg.Default() @@ -435,7 +437,7 @@ func translateContainer(t CreationQueueMessage, tcfg TranslateConfig) (objs []me VolumeMounts: utilsContainerVolumeMounts, } // mutate startup / liveness / readiness probes - common.MutateUtilsProbes(&utilsContainer) + common.MutateUtilsContainer(&utilsContainer) pod.Spec.Containers = append(pod.Spec.Containers, utilsContainer) // Setup telemetry for the pod diff --git a/src/libraries/go/lib/pkg/icms-translate/translate/task/translate_all_test.go b/src/libraries/go/lib/pkg/icms-translate/translate/task/translate_all_test.go index 9837699ba..d9d3d9792 100644 --- a/src/libraries/go/lib/pkg/icms-translate/translate/task/translate_all_test.go +++ b/src/libraries/go/lib/pkg/icms-translate/translate/task/translate_all_test.go @@ -33,6 +33,7 @@ import ( "github.com/NVIDIA/nvcf/src/libraries/go/lib/pkg/icms-translate/translate/common" ) +// TestTranslateContainerCreatesTaskPod verifies resources generated for a container task. func TestTranslateContainerCreatesTaskPod(t *testing.T) { objs, err := Translate( newTaskMessage(false), @@ -51,10 +52,12 @@ func TestTranslateContainerCreatesTaskPod(t *testing.T) { assert.Equal(t, corev1.RestartPolicyNever, pod.Spec.RestartPolicy) utilsContainer := findTaskContainerByName(t, pod.Spec.Containers, common.UtilsContainerName) + assertTaskUtilsMetricsPort(t, utilsContainer) assert.Equal(t, "nvcr.io/nvidia/utils:latest", utilsContainer.Image) assert.Equal(t, string(common.UploadResult), envValue(utilsContainer.Env, resultHandlingStratEnvKey)) } +// TestTranslateContainerWithCacheAndSecrets verifies cache and secret resources for a container task. func TestTranslateContainerWithCacheAndSecrets(t *testing.T) { msg := newTaskMessage(false) msg.LaunchSpecification.EnvironmentB64 = encodeTaskTextEnv(map[string]string{ @@ -87,6 +90,7 @@ func TestTranslateContainerWithCacheAndSecrets(t *testing.T) { assert.NotNil(t, findTaskJobByName(t, objs, "writer-job-cache-handle")) } +// TestTranslateHelmChartCreatesUtilsPod verifies resources generated for a Helm task. func TestTranslateHelmChartCreatesUtilsPod(t *testing.T) { objs, err := Translate( newTaskMessage(true), @@ -107,6 +111,7 @@ func TestTranslateHelmChartCreatesUtilsPod(t *testing.T) { assert.Equal(t, "nvcr.io/nvidia/init:latest", pod.Spec.InitContainers[0].Image) utilsContainer := findTaskContainerByName(t, pod.Spec.Containers, common.UtilsContainerName) + assertTaskUtilsMetricsPort(t, utilsContainer) assert.Equal(t, "nvcr.io/nvidia/utils:latest", utilsContainer.Image) assert.Equal(t, "request-task", envValue(utilsContainer.Env, "INSTANCE_ID")) assert.Equal(t, string(common.UploadResult), envValue(utilsContainer.Env, resultHandlingStratEnvKey)) @@ -341,6 +346,17 @@ func findTaskContainerByName(t *testing.T, containers []corev1.Container, name s return corev1.Container{} } +// assertTaskUtilsMetricsPort verifies the named worker-utils metrics port for tasks. +func assertTaskUtilsMetricsPort(t *testing.T, container corev1.Container) { + t.Helper() + assert.Equal(t, []corev1.ContainerPort{{ + Name: common.WorkerMetricsPortName, + ContainerPort: common.UtilsMetricsPort, + Protocol: corev1.ProtocolTCP, + }}, container.Ports) +} + +// findTaskVolumeByName returns the named task volume source or fails the test. func findTaskVolumeByName(t *testing.T, volumes []corev1.Volume, name string) corev1.VolumeSource { t.Helper() diff --git a/src/libraries/go/lib/pkg/icms-translate/translate/task/translate_helm.go b/src/libraries/go/lib/pkg/icms-translate/translate/task/translate_helm.go index 56332d37d..36a30ee4e 100644 --- a/src/libraries/go/lib/pkg/icms-translate/translate/task/translate_helm.go +++ b/src/libraries/go/lib/pkg/icms-translate/translate/task/translate_helm.go @@ -31,6 +31,8 @@ import ( translateutil "github.com/NVIDIA/nvcf/src/libraries/go/lib/pkg/icms-translate/translate/util" ) +// translateHelmChart converts a Helm task request into Kubernetes resources. +// //nolint:gocyclo // complex function with many conditional branches func translateHelmChart(t CreationQueueMessage, tcfg TranslateConfig) (objs []metav1.Object, err error) { if err := tcfg.ValidateHelmChart(); err != nil { @@ -356,7 +358,7 @@ func translateHelmChart(t CreationQueueMessage, tcfg TranslateConfig) (objs []me VolumeMounts: utilsContainerVolumeMounts, } // mutate startup / liveness / readiness probes - common.MutateUtilsProbes(&utilsContainer) + common.MutateUtilsContainer(&utilsContainer) utilsPod.Spec.Containers = append(utilsPod.Spec.Containers, utilsContainer) if hasTelemetries { diff --git a/src/libraries/go/lib/testdata/icms-translate/function/container/basic/exp.yaml b/src/libraries/go/lib/testdata/icms-translate/function/container/basic/exp.yaml index 6b95cfb89..aba4431dd 100644 --- a/src/libraries/go/lib/testdata/icms-translate/function/container/basic/exp.yaml +++ b/src/libraries/go/lib/testdata/icms-translate/function/container/basic/exp.yaml @@ -280,6 +280,10 @@ spec: successThreshold: 1 timeoutSeconds: 1 name: utils + ports: + - containerPort: 8010 + name: worker-metrics + protocol: TCP readinessProbe: failureThreshold: 2 httpGet: diff --git a/src/libraries/go/lib/testdata/icms-translate/function/container/basic/exp_utdep.yaml b/src/libraries/go/lib/testdata/icms-translate/function/container/basic/exp_utdep.yaml index 8e7376946..086364e77 100644 --- a/src/libraries/go/lib/testdata/icms-translate/function/container/basic/exp_utdep.yaml +++ b/src/libraries/go/lib/testdata/icms-translate/function/container/basic/exp_utdep.yaml @@ -491,6 +491,10 @@ spec: successThreshold: 1 timeoutSeconds: 1 name: utils + ports: + - containerPort: 8010 + name: worker-metrics + protocol: TCP readinessProbe: failureThreshold: 2 httpGet: diff --git a/src/libraries/go/lib/testdata/icms-translate/function/container/cache/exp.yaml b/src/libraries/go/lib/testdata/icms-translate/function/container/cache/exp.yaml index 3931e30dd..23cec9438 100644 --- a/src/libraries/go/lib/testdata/icms-translate/function/container/cache/exp.yaml +++ b/src/libraries/go/lib/testdata/icms-translate/function/container/cache/exp.yaml @@ -453,6 +453,10 @@ spec: successThreshold: 1 timeoutSeconds: 1 name: utils + ports: + - containerPort: 8010 + name: worker-metrics + protocol: TCP readinessProbe: failureThreshold: 2 httpGet: diff --git a/src/libraries/go/lib/testdata/icms-translate/function/container/cache/exp_utdep.yaml b/src/libraries/go/lib/testdata/icms-translate/function/container/cache/exp_utdep.yaml index a6c9ce09a..9f8b9a178 100644 --- a/src/libraries/go/lib/testdata/icms-translate/function/container/cache/exp_utdep.yaml +++ b/src/libraries/go/lib/testdata/icms-translate/function/container/cache/exp_utdep.yaml @@ -479,6 +479,10 @@ spec: successThreshold: 1 timeoutSeconds: 1 name: utils + ports: + - containerPort: 8010 + name: worker-metrics + protocol: TCP readinessProbe: failureThreshold: 2 httpGet: diff --git a/src/libraries/go/lib/testdata/icms-translate/function/container/ess/exp.yaml b/src/libraries/go/lib/testdata/icms-translate/function/container/ess/exp.yaml index 2fa35392b..f094cf6e5 100644 --- a/src/libraries/go/lib/testdata/icms-translate/function/container/ess/exp.yaml +++ b/src/libraries/go/lib/testdata/icms-translate/function/container/ess/exp.yaml @@ -309,6 +309,10 @@ spec: successThreshold: 1 timeoutSeconds: 1 name: utils + ports: + - containerPort: 8010 + name: worker-metrics + protocol: TCP readinessProbe: failureThreshold: 2 httpGet: diff --git a/src/libraries/go/lib/testdata/icms-translate/function/container/ess/exp_utdep.yaml b/src/libraries/go/lib/testdata/icms-translate/function/container/ess/exp_utdep.yaml index 43135e9cc..cab2b1abc 100644 --- a/src/libraries/go/lib/testdata/icms-translate/function/container/ess/exp_utdep.yaml +++ b/src/libraries/go/lib/testdata/icms-translate/function/container/ess/exp_utdep.yaml @@ -487,6 +487,10 @@ spec: successThreshold: 1 timeoutSeconds: 1 name: utils + ports: + - containerPort: 8010 + name: worker-metrics + protocol: TCP readinessProbe: failureThreshold: 2 httpGet: diff --git a/src/libraries/go/lib/testdata/icms-translate/function/container/llm/exp.yaml b/src/libraries/go/lib/testdata/icms-translate/function/container/llm/exp.yaml index e421e46d7..f0e004f89 100644 --- a/src/libraries/go/lib/testdata/icms-translate/function/container/llm/exp.yaml +++ b/src/libraries/go/lib/testdata/icms-translate/function/container/llm/exp.yaml @@ -270,6 +270,10 @@ spec: image: nvcr.io/0651155215864979/ncp-dev/stargate-client:0.4.0 imagePullPolicy: IfNotPresent name: llm-worker + ports: + - containerPort: 9089 + name: worker-metrics + protocol: TCP resources: limits: cpu: "1" diff --git a/src/libraries/go/lib/testdata/icms-translate/function/container/lls/exp.yaml b/src/libraries/go/lib/testdata/icms-translate/function/container/lls/exp.yaml index 05c5d4ab7..ee7d90711 100644 --- a/src/libraries/go/lib/testdata/icms-translate/function/container/lls/exp.yaml +++ b/src/libraries/go/lib/testdata/icms-translate/function/container/lls/exp.yaml @@ -284,6 +284,10 @@ spec: successThreshold: 1 timeoutSeconds: 1 name: utils + ports: + - containerPort: 8010 + name: worker-metrics + protocol: TCP readinessProbe: failureThreshold: 2 httpGet: diff --git a/src/libraries/go/lib/testdata/icms-translate/function/container/no-cred/exp.yaml b/src/libraries/go/lib/testdata/icms-translate/function/container/no-cred/exp.yaml index c6708d12e..dafdd694b 100644 --- a/src/libraries/go/lib/testdata/icms-translate/function/container/no-cred/exp.yaml +++ b/src/libraries/go/lib/testdata/icms-translate/function/container/no-cred/exp.yaml @@ -242,6 +242,10 @@ spec: successThreshold: 1 timeoutSeconds: 1 name: utils + ports: + - containerPort: 8010 + name: worker-metrics + protocol: TCP readinessProbe: failureThreshold: 2 httpGet: diff --git a/src/libraries/go/lib/testdata/icms-translate/function/container/no-cred/exp_utdep.yaml b/src/libraries/go/lib/testdata/icms-translate/function/container/no-cred/exp_utdep.yaml index 43a076742..a7868793d 100644 --- a/src/libraries/go/lib/testdata/icms-translate/function/container/no-cred/exp_utdep.yaml +++ b/src/libraries/go/lib/testdata/icms-translate/function/container/no-cred/exp_utdep.yaml @@ -449,6 +449,10 @@ spec: successThreshold: 1 timeoutSeconds: 1 name: utils + ports: + - containerPort: 8010 + name: worker-metrics + protocol: TCP readinessProbe: failureThreshold: 2 httpGet: diff --git a/src/libraries/go/lib/testdata/icms-translate/function/container/otel-ess/exp.yaml b/src/libraries/go/lib/testdata/icms-translate/function/container/otel-ess/exp.yaml index 5b57cf0ae..487e608b3 100644 --- a/src/libraries/go/lib/testdata/icms-translate/function/container/otel-ess/exp.yaml +++ b/src/libraries/go/lib/testdata/icms-translate/function/container/otel-ess/exp.yaml @@ -319,6 +319,10 @@ spec: successThreshold: 1 timeoutSeconds: 1 name: utils + ports: + - containerPort: 8010 + name: worker-metrics + protocol: TCP readinessProbe: failureThreshold: 2 httpGet: diff --git a/src/libraries/go/lib/testdata/icms-translate/function/container/otel/exp.yaml b/src/libraries/go/lib/testdata/icms-translate/function/container/otel/exp.yaml index dcdd64896..f9746b620 100644 --- a/src/libraries/go/lib/testdata/icms-translate/function/container/otel/exp.yaml +++ b/src/libraries/go/lib/testdata/icms-translate/function/container/otel/exp.yaml @@ -313,6 +313,10 @@ spec: successThreshold: 1 timeoutSeconds: 1 name: utils + ports: + - containerPort: 8010 + name: worker-metrics + protocol: TCP readinessProbe: failureThreshold: 2 httpGet: diff --git a/src/libraries/go/lib/testdata/icms-translate/function/container/tolerations/exp.yaml b/src/libraries/go/lib/testdata/icms-translate/function/container/tolerations/exp.yaml index 41eabfe3d..c8b2293ce 100644 --- a/src/libraries/go/lib/testdata/icms-translate/function/container/tolerations/exp.yaml +++ b/src/libraries/go/lib/testdata/icms-translate/function/container/tolerations/exp.yaml @@ -280,6 +280,10 @@ spec: successThreshold: 1 timeoutSeconds: 1 name: utils + ports: + - containerPort: 8010 + name: worker-metrics + protocol: TCP readinessProbe: failureThreshold: 2 httpGet: diff --git a/src/libraries/go/lib/testdata/icms-translate/function/container/tolerations/exp_utdep.yaml b/src/libraries/go/lib/testdata/icms-translate/function/container/tolerations/exp_utdep.yaml index 9db4e69e6..71019fbee 100644 --- a/src/libraries/go/lib/testdata/icms-translate/function/container/tolerations/exp_utdep.yaml +++ b/src/libraries/go/lib/testdata/icms-translate/function/container/tolerations/exp_utdep.yaml @@ -495,6 +495,10 @@ spec: successThreshold: 1 timeoutSeconds: 1 name: utils + ports: + - containerPort: 8010 + name: worker-metrics + protocol: TCP readinessProbe: failureThreshold: 2 httpGet: diff --git a/src/libraries/go/lib/testdata/icms-translate/function/helmchart/basic/exp.yaml b/src/libraries/go/lib/testdata/icms-translate/function/helmchart/basic/exp.yaml index cef591472..dfd4fd941 100644 --- a/src/libraries/go/lib/testdata/icms-translate/function/helmchart/basic/exp.yaml +++ b/src/libraries/go/lib/testdata/icms-translate/function/helmchart/basic/exp.yaml @@ -242,6 +242,10 @@ spec: successThreshold: 1 timeoutSeconds: 1 name: utils + ports: + - containerPort: 8010 + name: worker-metrics + protocol: TCP readinessProbe: failureThreshold: 2 httpGet: diff --git a/src/libraries/go/lib/testdata/icms-translate/function/helmchart/basic/exp_utdep.yaml b/src/libraries/go/lib/testdata/icms-translate/function/helmchart/basic/exp_utdep.yaml index a50a744b9..f6b155cc4 100644 --- a/src/libraries/go/lib/testdata/icms-translate/function/helmchart/basic/exp_utdep.yaml +++ b/src/libraries/go/lib/testdata/icms-translate/function/helmchart/basic/exp_utdep.yaml @@ -282,6 +282,10 @@ spec: successThreshold: 1 timeoutSeconds: 1 name: utils + ports: + - containerPort: 8010 + name: worker-metrics + protocol: TCP readinessProbe: failureThreshold: 2 httpGet: diff --git a/src/libraries/go/lib/testdata/icms-translate/function/helmchart/cache/exp.yaml b/src/libraries/go/lib/testdata/icms-translate/function/helmchart/cache/exp.yaml index 2879277b8..759166beb 100644 --- a/src/libraries/go/lib/testdata/icms-translate/function/helmchart/cache/exp.yaml +++ b/src/libraries/go/lib/testdata/icms-translate/function/helmchart/cache/exp.yaml @@ -372,6 +372,10 @@ spec: successThreshold: 1 timeoutSeconds: 1 name: utils + ports: + - containerPort: 8010 + name: worker-metrics + protocol: TCP readinessProbe: failureThreshold: 2 httpGet: diff --git a/src/libraries/go/lib/testdata/icms-translate/function/helmchart/cache/exp_utdep.yaml b/src/libraries/go/lib/testdata/icms-translate/function/helmchart/cache/exp_utdep.yaml index 44d31ae6f..041dd3565 100644 --- a/src/libraries/go/lib/testdata/icms-translate/function/helmchart/cache/exp_utdep.yaml +++ b/src/libraries/go/lib/testdata/icms-translate/function/helmchart/cache/exp_utdep.yaml @@ -406,6 +406,10 @@ spec: successThreshold: 1 timeoutSeconds: 1 name: utils + ports: + - containerPort: 8010 + name: worker-metrics + protocol: TCP readinessProbe: failureThreshold: 2 httpGet: diff --git a/src/libraries/go/lib/testdata/icms-translate/function/helmchart/ess/exp.yaml b/src/libraries/go/lib/testdata/icms-translate/function/helmchart/ess/exp.yaml index 0dea252bd..1783fae50 100644 --- a/src/libraries/go/lib/testdata/icms-translate/function/helmchart/ess/exp.yaml +++ b/src/libraries/go/lib/testdata/icms-translate/function/helmchart/ess/exp.yaml @@ -231,6 +231,10 @@ spec: successThreshold: 1 timeoutSeconds: 1 name: utils + ports: + - containerPort: 8010 + name: worker-metrics + protocol: TCP readinessProbe: failureThreshold: 2 httpGet: diff --git a/src/libraries/go/lib/testdata/icms-translate/function/helmchart/ess/exp_utdep.yaml b/src/libraries/go/lib/testdata/icms-translate/function/helmchart/ess/exp_utdep.yaml index 437b5e780..97a87483a 100644 --- a/src/libraries/go/lib/testdata/icms-translate/function/helmchart/ess/exp_utdep.yaml +++ b/src/libraries/go/lib/testdata/icms-translate/function/helmchart/ess/exp_utdep.yaml @@ -239,6 +239,10 @@ spec: successThreshold: 1 timeoutSeconds: 1 name: utils + ports: + - containerPort: 8010 + name: worker-metrics + protocol: TCP readinessProbe: failureThreshold: 2 httpGet: diff --git a/src/libraries/go/lib/testdata/icms-translate/function/helmchart/llm/exp.yaml b/src/libraries/go/lib/testdata/icms-translate/function/helmchart/llm/exp.yaml index 8c6bc6e55..dfe1569af 100644 --- a/src/libraries/go/lib/testdata/icms-translate/function/helmchart/llm/exp.yaml +++ b/src/libraries/go/lib/testdata/icms-translate/function/helmchart/llm/exp.yaml @@ -238,6 +238,10 @@ spec: image: nvcr.io/nvcf-core/stargate-client:0.4.0 imagePullPolicy: IfNotPresent name: llm-worker + ports: + - containerPort: 9089 + name: worker-metrics + protocol: TCP resources: limits: cpu: "1" diff --git a/src/libraries/go/lib/testdata/icms-translate/function/helmchart/no-cred/exp.yaml b/src/libraries/go/lib/testdata/icms-translate/function/helmchart/no-cred/exp.yaml index bdc7498a2..59908363a 100644 --- a/src/libraries/go/lib/testdata/icms-translate/function/helmchart/no-cred/exp.yaml +++ b/src/libraries/go/lib/testdata/icms-translate/function/helmchart/no-cred/exp.yaml @@ -169,6 +169,10 @@ spec: successThreshold: 1 timeoutSeconds: 1 name: utils + ports: + - containerPort: 8010 + name: worker-metrics + protocol: TCP readinessProbe: failureThreshold: 2 httpGet: diff --git a/src/libraries/go/lib/testdata/icms-translate/function/helmchart/no-cred/exp_utdep.yaml b/src/libraries/go/lib/testdata/icms-translate/function/helmchart/no-cred/exp_utdep.yaml index 96371c836..e27450686 100644 --- a/src/libraries/go/lib/testdata/icms-translate/function/helmchart/no-cred/exp_utdep.yaml +++ b/src/libraries/go/lib/testdata/icms-translate/function/helmchart/no-cred/exp_utdep.yaml @@ -209,6 +209,10 @@ spec: successThreshold: 1 timeoutSeconds: 1 name: utils + ports: + - containerPort: 8010 + name: worker-metrics + protocol: TCP readinessProbe: failureThreshold: 2 httpGet: diff --git a/src/libraries/go/lib/testdata/icms-translate/function/helmchart/otel-ess/exp.yaml b/src/libraries/go/lib/testdata/icms-translate/function/helmchart/otel-ess/exp.yaml index 2e169668d..0a27dae24 100644 --- a/src/libraries/go/lib/testdata/icms-translate/function/helmchart/otel-ess/exp.yaml +++ b/src/libraries/go/lib/testdata/icms-translate/function/helmchart/otel-ess/exp.yaml @@ -296,6 +296,10 @@ spec: successThreshold: 1 timeoutSeconds: 1 name: utils + ports: + - containerPort: 8010 + name: worker-metrics + protocol: TCP readinessProbe: failureThreshold: 2 httpGet: diff --git a/src/libraries/go/lib/testdata/icms-translate/function/helmchart/otel/exp.yaml b/src/libraries/go/lib/testdata/icms-translate/function/helmchart/otel/exp.yaml index f98c94a8a..6cf96d024 100644 --- a/src/libraries/go/lib/testdata/icms-translate/function/helmchart/otel/exp.yaml +++ b/src/libraries/go/lib/testdata/icms-translate/function/helmchart/otel/exp.yaml @@ -290,6 +290,10 @@ spec: successThreshold: 1 timeoutSeconds: 1 name: utils + ports: + - containerPort: 8010 + name: worker-metrics + protocol: TCP readinessProbe: failureThreshold: 2 httpGet: diff --git a/src/libraries/go/lib/testdata/icms-translate/task/container/basic-icms/exp.yaml b/src/libraries/go/lib/testdata/icms-translate/task/container/basic-icms/exp.yaml index 8f8d7a018..35eccb122 100644 --- a/src/libraries/go/lib/testdata/icms-translate/task/container/basic-icms/exp.yaml +++ b/src/libraries/go/lib/testdata/icms-translate/task/container/basic-icms/exp.yaml @@ -277,6 +277,10 @@ spec: successThreshold: 1 timeoutSeconds: 1 name: utils + ports: + - containerPort: 8010 + name: worker-metrics + protocol: TCP readinessProbe: failureThreshold: 2 httpGet: diff --git a/src/libraries/go/lib/testdata/icms-translate/task/container/basic/exp.yaml b/src/libraries/go/lib/testdata/icms-translate/task/container/basic/exp.yaml index 8f8d7a018..35eccb122 100644 --- a/src/libraries/go/lib/testdata/icms-translate/task/container/basic/exp.yaml +++ b/src/libraries/go/lib/testdata/icms-translate/task/container/basic/exp.yaml @@ -277,6 +277,10 @@ spec: successThreshold: 1 timeoutSeconds: 1 name: utils + ports: + - containerPort: 8010 + name: worker-metrics + protocol: TCP readinessProbe: failureThreshold: 2 httpGet: diff --git a/src/libraries/go/lib/testdata/icms-translate/task/container/cache/exp.yaml b/src/libraries/go/lib/testdata/icms-translate/task/container/cache/exp.yaml index b34edc75f..9bf336693 100644 --- a/src/libraries/go/lib/testdata/icms-translate/task/container/cache/exp.yaml +++ b/src/libraries/go/lib/testdata/icms-translate/task/container/cache/exp.yaml @@ -452,6 +452,10 @@ spec: successThreshold: 1 timeoutSeconds: 1 name: utils + ports: + - containerPort: 8010 + name: worker-metrics + protocol: TCP readinessProbe: failureThreshold: 2 httpGet: diff --git a/src/libraries/go/lib/testdata/icms-translate/task/container/emptyargs/exp.yaml b/src/libraries/go/lib/testdata/icms-translate/task/container/emptyargs/exp.yaml index 8ff88af7b..53ea7450d 100644 --- a/src/libraries/go/lib/testdata/icms-translate/task/container/emptyargs/exp.yaml +++ b/src/libraries/go/lib/testdata/icms-translate/task/container/emptyargs/exp.yaml @@ -267,6 +267,10 @@ spec: successThreshold: 1 timeoutSeconds: 1 name: utils + ports: + - containerPort: 8010 + name: worker-metrics + protocol: TCP readinessProbe: failureThreshold: 2 httpGet: diff --git a/src/libraries/go/lib/testdata/icms-translate/task/container/ess/exp.yaml b/src/libraries/go/lib/testdata/icms-translate/task/container/ess/exp.yaml index 36f255035..b80dcab3e 100644 --- a/src/libraries/go/lib/testdata/icms-translate/task/container/ess/exp.yaml +++ b/src/libraries/go/lib/testdata/icms-translate/task/container/ess/exp.yaml @@ -312,6 +312,10 @@ spec: successThreshold: 1 timeoutSeconds: 1 name: utils + ports: + - containerPort: 8010 + name: worker-metrics + protocol: TCP readinessProbe: failureThreshold: 2 httpGet: diff --git a/src/libraries/go/lib/testdata/icms-translate/task/container/no-cred/exp.yaml b/src/libraries/go/lib/testdata/icms-translate/task/container/no-cred/exp.yaml index 43aa58e77..702f5a750 100644 --- a/src/libraries/go/lib/testdata/icms-translate/task/container/no-cred/exp.yaml +++ b/src/libraries/go/lib/testdata/icms-translate/task/container/no-cred/exp.yaml @@ -243,6 +243,10 @@ spec: successThreshold: 1 timeoutSeconds: 1 name: utils + ports: + - containerPort: 8010 + name: worker-metrics + protocol: TCP readinessProbe: failureThreshold: 2 httpGet: diff --git a/src/libraries/go/lib/testdata/icms-translate/task/container/otel-ess/exp.yaml b/src/libraries/go/lib/testdata/icms-translate/task/container/otel-ess/exp.yaml index 9c149e305..bed576f3b 100644 --- a/src/libraries/go/lib/testdata/icms-translate/task/container/otel-ess/exp.yaml +++ b/src/libraries/go/lib/testdata/icms-translate/task/container/otel-ess/exp.yaml @@ -316,6 +316,10 @@ spec: successThreshold: 1 timeoutSeconds: 1 name: utils + ports: + - containerPort: 8010 + name: worker-metrics + protocol: TCP readinessProbe: failureThreshold: 2 httpGet: diff --git a/src/libraries/go/lib/testdata/icms-translate/task/container/otel/exp.yaml b/src/libraries/go/lib/testdata/icms-translate/task/container/otel/exp.yaml index 217474438..c35e5c378 100644 --- a/src/libraries/go/lib/testdata/icms-translate/task/container/otel/exp.yaml +++ b/src/libraries/go/lib/testdata/icms-translate/task/container/otel/exp.yaml @@ -316,6 +316,10 @@ spec: successThreshold: 1 timeoutSeconds: 1 name: utils + ports: + - containerPort: 8010 + name: worker-metrics + protocol: TCP readinessProbe: failureThreshold: 2 httpGet: diff --git a/src/libraries/go/lib/testdata/icms-translate/task/helmchart/basic/exp.yaml b/src/libraries/go/lib/testdata/icms-translate/task/helmchart/basic/exp.yaml index b0aedb323..167face65 100644 --- a/src/libraries/go/lib/testdata/icms-translate/task/helmchart/basic/exp.yaml +++ b/src/libraries/go/lib/testdata/icms-translate/task/helmchart/basic/exp.yaml @@ -236,6 +236,10 @@ spec: successThreshold: 1 timeoutSeconds: 1 name: utils + ports: + - containerPort: 8010 + name: worker-metrics + protocol: TCP readinessProbe: failureThreshold: 2 httpGet: diff --git a/src/libraries/go/lib/testdata/icms-translate/task/helmchart/cache/exp.yaml b/src/libraries/go/lib/testdata/icms-translate/task/helmchart/cache/exp.yaml index 6050d01b2..2a8f122cd 100644 --- a/src/libraries/go/lib/testdata/icms-translate/task/helmchart/cache/exp.yaml +++ b/src/libraries/go/lib/testdata/icms-translate/task/helmchart/cache/exp.yaml @@ -374,6 +374,10 @@ spec: successThreshold: 1 timeoutSeconds: 1 name: utils + ports: + - containerPort: 8010 + name: worker-metrics + protocol: TCP readinessProbe: failureThreshold: 2 httpGet: diff --git a/src/libraries/go/lib/testdata/icms-translate/task/helmchart/ess/exp.yaml b/src/libraries/go/lib/testdata/icms-translate/task/helmchart/ess/exp.yaml index 41b3ec161..2d26d10c8 100644 --- a/src/libraries/go/lib/testdata/icms-translate/task/helmchart/ess/exp.yaml +++ b/src/libraries/go/lib/testdata/icms-translate/task/helmchart/ess/exp.yaml @@ -235,6 +235,10 @@ spec: successThreshold: 1 timeoutSeconds: 1 name: utils + ports: + - containerPort: 8010 + name: worker-metrics + protocol: TCP readinessProbe: failureThreshold: 2 httpGet: diff --git a/src/libraries/go/lib/testdata/icms-translate/task/helmchart/no-cred/exp.yaml b/src/libraries/go/lib/testdata/icms-translate/task/helmchart/no-cred/exp.yaml index c5f89d3eb..b5dbc2599 100644 --- a/src/libraries/go/lib/testdata/icms-translate/task/helmchart/no-cred/exp.yaml +++ b/src/libraries/go/lib/testdata/icms-translate/task/helmchart/no-cred/exp.yaml @@ -203,6 +203,10 @@ spec: successThreshold: 1 timeoutSeconds: 1 name: utils + ports: + - containerPort: 8010 + name: worker-metrics + protocol: TCP readinessProbe: failureThreshold: 2 httpGet: diff --git a/src/libraries/go/lib/testdata/icms-translate/task/helmchart/otel-ess/exp.yaml b/src/libraries/go/lib/testdata/icms-translate/task/helmchart/otel-ess/exp.yaml index 213c3ed11..dc808be9c 100644 --- a/src/libraries/go/lib/testdata/icms-translate/task/helmchart/otel-ess/exp.yaml +++ b/src/libraries/go/lib/testdata/icms-translate/task/helmchart/otel-ess/exp.yaml @@ -231,6 +231,10 @@ spec: successThreshold: 1 timeoutSeconds: 1 name: utils + ports: + - containerPort: 8010 + name: worker-metrics + protocol: TCP readinessProbe: failureThreshold: 2 httpGet: diff --git a/src/libraries/go/lib/testdata/icms-translate/task/helmchart/otel/exp.yaml b/src/libraries/go/lib/testdata/icms-translate/task/helmchart/otel/exp.yaml index fbcb13632..5abcc0b22 100644 --- a/src/libraries/go/lib/testdata/icms-translate/task/helmchart/otel/exp.yaml +++ b/src/libraries/go/lib/testdata/icms-translate/task/helmchart/otel/exp.yaml @@ -292,6 +292,10 @@ spec: successThreshold: 1 timeoutSeconds: 1 name: utils + ports: + - containerPort: 8010 + name: worker-metrics + protocol: TCP readinessProbe: failureThreshold: 2 httpGet: