diff --git a/k8s/helm/README.md b/k8s/helm/README.md index 1dd24fe97c..11aba5827f 100644 --- a/k8s/helm/README.md +++ b/k8s/helm/README.md @@ -161,10 +161,10 @@ For the complete default values, see [values.yaml](values.yaml). | Key | Type | Default | Description | |-----|------|---------|-------------| -| api | object | [See values.yaml](values.yaml#L768) | API configuration settings for the api deployment | +| api | object | [See values.yaml](values.yaml#L761) | API configuration settings for the api deployment | | api.affinity | object | `{}` | Affinity configuration for the API service. | | api.annotations | object | `{}` | Annotations to add to the API service deployment. | -| api.autoscaling | object | [See values.yaml](values.yaml#L897) | Specifies autoscaling configurations for the deployment. | +| api.autoscaling | object | [See values.yaml](values.yaml#L890) | Specifies autoscaling configurations for the deployment. | | api.autoscaling.annotations | object | `{}` | Annotations for the HorizontalPodAutoscaler. | | api.autoscaling.enabled | bool | `false` | Whether to enable horizontal pod autoscaler. | | api.autoscaling.maxReplicas | int | `10` | The maximum number of replicas for the deployment. | @@ -175,25 +175,25 @@ For the complete default values, see [values.yaml](values.yaml). | api.extraArgs | list | `[]` | Additional arguments to pass to the Platform API service | | api.extraVolumeMounts | list | `[]` | Additional volume mounts to add to the Platform API container. | | api.extraVolumes | list | `[]` | Additional volumes to add to the Platform API pod. | -| api.image | object | [See values.yaml](values.yaml#L774) | Container image configuration for the api deployment. | +| api.image | object | [See values.yaml](values.yaml#L767) | Container image configuration for the api deployment. | | api.image.pullPolicy | string | `"IfNotPresent"` | The image pull policy determining when to pull new images. | | api.image.repository | string | `"nvcr.io/nvidia/nemo-platform/nmp-api"` | The registry where the NeMo Platform image is located. | | api.image.tag | string | `""` | The image tag to use. | -| api.livenessProbe | object | [See values.yaml](values.yaml#L856) | Liveness probe configuration for the api service. | +| api.livenessProbe | object | [See values.yaml](values.yaml#L849) | Liveness probe configuration for the api service. | | api.livenessProbe.failureThreshold | int | `3` | The failure threshold for the liveness probe. | | api.livenessProbe.httpGet | object | `{"path":"/health/live","port":"http"}` | The HTTP GET request to use for the liveness probe. | | api.livenessProbe.periodSeconds | int | `10` | The frequency in seconds to perform the liveness probe. | | api.livenessProbe.timeoutSeconds | int | `5` | The timeout in seconds for the liveness probe. | | api.nodeSelector | object | `{}` | Node selector configuration for the API service. | | api.podAnnotations | object | `{}` | Annotations to add to the API service pod. | -| api.podDisruptionBudget | object | [See values.yaml](values.yaml#L884) | PodDisruptionBudget configuration for the API service. | +| api.podDisruptionBudget | object | [See values.yaml](values.yaml#L877) | PodDisruptionBudget configuration for the API service. | | api.podDisruptionBudget.annotations | object | `{}` | Annotations for the PodDisruptionBudget. | | api.podDisruptionBudget.enabled | bool | `false` | Whether to create a PodDisruptionBudget for the API pods. | | api.podDisruptionBudget.minAvailable | int | `1` | Minimum number of API pods that must remain available during voluntary disruptions. Only one of minAvailable or maxUnavailable may be set. | | api.podLabels | object | `{}` | Labels for the API service pod. | -| api.podSecurityContext | object | [See values.yaml](values.yaml#L821) | Pod-level security context settings for the API service. | +| api.podSecurityContext | object | [See values.yaml](values.yaml#L814) | Pod-level security context settings for the API service. | | api.podSecurityContext.fsGroup | int | `1000` | The file system group ID to use for all containers. | -| api.readinessProbe | object | [See values.yaml](values.yaml#L870) | Readiness probe configuration for the api service. | +| api.readinessProbe | object | [See values.yaml](values.yaml#L863) | Readiness probe configuration for the api service. | | api.readinessProbe.failureThreshold | int | `3` | The failure threshold for the readiness probe. | | api.readinessProbe.httpGet | object | `{"path":"/health/ready","port":"http"}` | The HTTP GET request to use for the readiness probe. | | api.readinessProbe.periodSeconds | int | `10` | The frequency in seconds to perform the readiness probe. | @@ -203,11 +203,11 @@ For the complete default values, see [values.yaml](values.yaml). | api.securityContext | object | `{}` | Container-level security context settings for the API service. | | api.server | object | `{"keepAliveTimeoutSeconds":5}` | Platform API server settings. | | api.server.keepAliveTimeoutSeconds | int | `5` | Seconds Uvicorn keeps idle HTTP connections open. Must be greater than envoyProxy.timeouts.upstreamIdle when Envoy is enabled. | -| api.service | object | [See values.yaml](values.yaml#L828) | Service configuration for the API service. | +| api.service | object | [See values.yaml](values.yaml#L821) | Service configuration for the API service. | | api.service.annotations | object | `{}` | Annotations for the API service. | | api.service.port | int | `8080` | The port number to expose for the service. | | api.service.type | string | `"ClusterIP"` | The Kubernetes service type to create. | -| api.serviceAccount | object | [See values.yaml](values.yaml#L804) | Service account configuration for the API service. | +| api.serviceAccount | object | [See values.yaml](values.yaml#L797) | Service account configuration for the API service. | | api.serviceAccount.annotations | object | `{}` | Annotations to add to the service account. | | api.serviceAccount.automount | bool | `true` | Automatically mount a ServiceAccount's API credentials. | | api.serviceAccount.create | bool | `true` | Specifies whether a service account should be created. | @@ -219,7 +219,7 @@ For the complete default values, see [values.yaml](values.yaml). | api.serviceMonitor.labels | object | `{}` | Additional labels to add to the ServiceMonitor | | api.serviceMonitor.scheme | string | `"http"` | Scheme to use for scraping metrics (http or https) | | api.services | list | `[]` | Explicit services passed to `nemo services run` with `--services`. When non-empty, overrides api.serviceGroup. Must be a list. | -| api.startupProbe | object | [See values.yaml](values.yaml#L840) | Startup probe configuration for the api service. | +| api.startupProbe | object | [See values.yaml](values.yaml#L833) | Startup probe configuration for the api service. | | api.startupProbe.failureThreshold | int | `24` | The failure threshold for the startup probe. | | api.startupProbe.httpGet | object | `{"path":"/health/ready","port":"http"}` | The HTTP GET request to use for the startup probe. | | api.startupProbe.initialDelaySeconds | int | `10` | Number of seconds to wait before the first startup probe. Allows time for DB connection retries (e.g. Postgres pod booting). | @@ -262,14 +262,14 @@ For the complete default values, see [values.yaml](values.yaml). | clickhouse.serviceAccount.name | string | `""` | The name of the service account to use. If not set and create is true, a name is generated from the release fullname. | | clickhouse.startupProbe | object | [See values.yaml](values.yaml#L408) | Startup probe configuration for the ClickHouse container. | | clickhouse.tolerations | list | `[]` | Tolerations for the ClickHouse pod. | -| core | object | [See values.yaml](values.yaml#L967) | Core deployment configuration settings | +| core | object | [See values.yaml](values.yaml#L960) | Core deployment configuration settings | | core.controller.affinity | object | `{}` | Affinity configuration for the controller service. | | core.controller.annotations | object | `{}` | Annotations to add to the controller service deployment. | | core.controller.controllerGroup | string | `"all"` | Predefined controller group passed to `nemo services run` with `--controller-group`. Ignored when core.controller.controllers is non-empty. | | core.controller.controllers | list | `[]` | Explicit controllers passed to `nemo services run` with `--controllers`. When non-empty, overrides core.controller.controllerGroup. Must be a list. | | core.controller.env | object | `{}` | Additional environment variables to pass to containers. This is an object formatted like NAME: value or NAME: valueFrom: {object}. | | core.controller.extraArgs | list | `[]` | Additional arguments to pass to the Core Controller service | -| core.controller.livenessProbe | object | [See values.yaml](values.yaml#L1076) | Liveness probe configuration for the controller service. | +| core.controller.livenessProbe | object | [See values.yaml](values.yaml#L1069) | Liveness probe configuration for the controller service. | | core.controller.livenessProbe.failureThreshold | int | `3` | The failure threshold for the liveness probe. | | core.controller.livenessProbe.httpGet | object | `{"path":"/health/live","port":"http"}` | The HTTP GET request to use for the liveness probe. | | core.controller.livenessProbe.periodSeconds | int | `10` | The frequency in seconds to perform the liveness probe. | @@ -277,24 +277,24 @@ For the complete default values, see [values.yaml](values.yaml). | core.controller.nodeSelector | object | `{}` | Node selector configuration for the controller service. | | core.controller.podAnnotations | object | `{}` | Annotations to add to the controller service pod. | | core.controller.podLabels | object | `{}` | Labels for the controller service pod. | -| core.controller.podSecurityContext | object | [See values.yaml](values.yaml#L1050) | Pod-level security context settings for the controller service. | +| core.controller.podSecurityContext | object | [See values.yaml](values.yaml#L1043) | Pod-level security context settings for the controller service. | | core.controller.podSecurityContext.fsGroup | int | `1000` | The file system group ID to use for all containers. | -| core.controller.readinessProbe | object | [See values.yaml](values.yaml#L1090) | Readiness probe configuration for the controller service. | +| core.controller.readinessProbe | object | [See values.yaml](values.yaml#L1083) | Readiness probe configuration for the controller service. | | core.controller.readinessProbe.failureThreshold | int | `3` | The failure threshold for the readiness probe. | | core.controller.readinessProbe.httpGet | object | `{"path":"/health/ready","port":"http"}` | The HTTP GET request to use for the readiness probe. | | core.controller.readinessProbe.periodSeconds | int | `10` | The frequency in seconds to perform the readiness probe. | | core.controller.readinessProbe.timeoutSeconds | int | `5` | The timeout in seconds for the readiness probe. | | core.controller.resources | object | `{}` | Kubernetes deployment resources configuration for the controller service. | | core.controller.securityContext | object | `{}` | Container-level security context settings for the controller service. | -| core.controller.service | object | [See values.yaml](values.yaml#L1037) | Service configuration for the controller service. This only configures a headless service for DNS resolution. | +| core.controller.service | object | [See values.yaml](values.yaml#L1030) | Service configuration for the controller service. This only configures a headless service for DNS resolution. | | core.controller.service.annotations | object | `{}` | Annotations for the headless controller service. | | core.controller.service.port | int | `8080` | The port for the service. | -| core.controller.serviceAccount | object | [See values.yaml](values.yaml#L1018) | Service account configuration for the controller service. | +| core.controller.serviceAccount | object | [See values.yaml](values.yaml#L1011) | Service account configuration for the controller service. | | core.controller.serviceAccount.annotations | object | `{}` | Annotations to add to the service account. | | core.controller.serviceAccount.automount | bool | `true` | Automatically mount a ServiceAccount's API credentials. | | core.controller.serviceAccount.create | bool | `true` | Specifies whether a service account should be created. | | core.controller.serviceAccount.name | string | `""` | The name of the service account to use. If not set and create is true, a name is generated using the fullname template. | -| core.controller.startupProbe | object | [See values.yaml](values.yaml#L1060) | Startup probe configuration for the core service. | +| core.controller.startupProbe | object | [See values.yaml](values.yaml#L1053) | Startup probe configuration for the core service. | | core.controller.startupProbe.failureThreshold | int | `24` | The failure threshold for the startup probe. | | core.controller.startupProbe.httpGet | object | `{"path":"/health/ready","port":"http"}` | The HTTP GET request to use for the startup probe. | | core.controller.startupProbe.initialDelaySeconds | int | `10` | Number of seconds to wait before the first startup probe. Allows time for DB connection retries (e.g. Postgres pod booting). | @@ -303,11 +303,11 @@ For the complete default values, see [values.yaml](values.yaml). | core.controller.tolerations | list | `[]` | Tolerations configuration for the controller service. | | core.controller.topologySpreadConstraints | list | `[]` | Topology spread constraints for the controller service pods. See https://kubernetes.io/docs/concepts/scheduling-eviction/topology-spread-constraints/ | | core.enabled | bool | `true` | Specifies whether to enable the core deployment. | -| core.image | object | [See values.yaml](values.yaml#L973) | Container image configuration for the core deployment. | +| core.image | object | [See values.yaml](values.yaml#L966) | Container image configuration for the core deployment. | | core.image.pullPolicy | string | `"IfNotPresent"` | The image pull policy determining when to pull new images. | | core.image.repository | string | `"nvcr.io/nvidia/nemo-platform/nmp-api"` | The registry where the NeMo Platform image is located. | | core.image.tag | string | `""` | The image tag to use. | -| core.jobs | object | [See values.yaml](values.yaml#L1003) | Service account configuration for pods created by the jobs controller (Kubernetes/Volcano job pods). | +| core.jobs | object | [See values.yaml](values.yaml#L996) | Service account configuration for pods created by the jobs controller (Kubernetes/Volcano job pods). | | core.jobs.serviceAccount.annotations | object | `{}` | Annotations to add to the service account. | | core.jobs.serviceAccount.automount | bool | `true` | Automatically mount a ServiceAccount's API credentials. | | core.jobs.serviceAccount.create | bool | `true` | Specifies whether a service account should be created for job pods. | @@ -326,11 +326,11 @@ For the complete default values, see [values.yaml](values.yaml). | core.telemetry | object | `{}` | OpenTelemetry configuration overrides for the platform deployment. | | env | object | `{}` | Environment variables that will be applied to every deployment pod. Uses a simple key value map structure like MY_ENV_VAR: the-key and works with valueFrom as well. | | envFromSecret | string | `""` | Optional. Name of an existing Kubernetes Secret to load as env vars (envFrom) for the API pod. When set, the chart does not create or generate the default api-env Secret; use your own Secret (for example, from Vault or sealed-secrets). | -| envoyProxy | object | [See values.yaml](values.yaml#L1128) | Envoy proxy configuration settings. Resources are created only when platform config has auth.enabled: true (see platformConfig.auth.enabled). | +| envoyProxy | object | [See values.yaml](values.yaml#L1121) | Envoy proxy configuration settings. Resources are created only when platform config has auth.enabled: true (see platformConfig.auth.enabled). | | envoyProxy.adminPort | int | `9901` | Envoy Admin port | | envoyProxy.affinity | object | `{}` | Affinity configuration for the Envoy pods. | | envoyProxy.annotations | object | `{}` | Annotations to add to the Envoy service deployment. | -| envoyProxy.autoscaling | object | [See values.yaml](values.yaml#L1253) | Specifies autoscaling configurations for the deployment. | +| envoyProxy.autoscaling | object | [See values.yaml](values.yaml#L1246) | Specifies autoscaling configurations for the deployment. | | envoyProxy.autoscaling.annotations | object | `{}` | Annotations for the HorizontalPodAutoscaler. | | envoyProxy.autoscaling.enabled | bool | `false` | Whether to enable horizontal pod autoscaler. | | envoyProxy.autoscaling.maxReplicas | int | `10` | The maximum number of replicas for the deployment. | @@ -343,25 +343,25 @@ For the complete default values, see [values.yaml](values.yaml). | envoyProxy.extraVolumeMounts | list | `[]` | Additional volume mounts to add to the Envoy container. | | envoyProxy.extraVolumes | list | `[]` | Additional volumes to add to the Envoy pod. | | envoyProxy.image.digest | string | `""` | Optional image digest. When set, the Envoy image renders as repository@digest. | -| envoyProxy.livenessProbe | object | [See values.yaml](values.yaml#L1214) | Liveness probe for the Envoy container (admin interface /ready). | +| envoyProxy.livenessProbe | object | [See values.yaml](values.yaml#L1207) | Liveness probe for the Envoy container (admin interface /ready). | | envoyProxy.nodeSelector | object | `{}` | Node selector configuration for the Envoy pods. | | envoyProxy.podAnnotations | object | `{}` | Annotations to add to the Envoy service pod. | -| envoyProxy.podDisruptionBudget | object | [See values.yaml](values.yaml#L1240) | PodDisruptionBudget configuration for the Envoy service. | +| envoyProxy.podDisruptionBudget | object | [See values.yaml](values.yaml#L1233) | PodDisruptionBudget configuration for the Envoy service. | | envoyProxy.podDisruptionBudget.annotations | object | `{}` | Annotations for the PodDisruptionBudget. | | envoyProxy.podDisruptionBudget.enabled | bool | `false` | Whether to create a PodDisruptionBudget for the Envoy pods. | | envoyProxy.podDisruptionBudget.minAvailable | int | `1` | Minimum number of Envoy pods that must remain available during voluntary disruptions. Only one of minAvailable or maxUnavailable may be set. | | envoyProxy.podLabels | object | `{}` | Labels for the Envoy service pod. | -| envoyProxy.podSecurityContext | object | [See values.yaml](values.yaml#L1174) | Pod-level security context settings for the Envoy service. | +| envoyProxy.podSecurityContext | object | [See values.yaml](values.yaml#L1167) | Pod-level security context settings for the Envoy service. | | envoyProxy.podSecurityContext.fsGroup | int | `1000` | The file system group ID to use for all containers. | -| envoyProxy.readinessProbe | object | [See values.yaml](values.yaml#L1222) | Readiness probe for the Envoy container (admin interface /ready). | +| envoyProxy.readinessProbe | object | [See values.yaml](values.yaml#L1215) | Readiness probe for the Envoy container (admin interface /ready). | | envoyProxy.resources | object | `{}` | Kubernetes deployment resources configuration for the Envoy service. Utilization-based autoscaling requires a matching resource request. | | envoyProxy.securityContext | object | `{}` | Container-level security context settings for the Envoy service. | -| envoyProxy.service | object | [See values.yaml](values.yaml#L1181) | Service configuration for the Envoy service. | +| envoyProxy.service | object | [See values.yaml](values.yaml#L1174) | Service configuration for the Envoy service. | | envoyProxy.service.annotations | object | `{}` | Annotations for the Envoy service. | | envoyProxy.service.exposeAdminPort | bool | `false` | Expose the Envoy admin port through the Kubernetes Service. Enable only for controlled in-cluster scraping or debugging. | | envoyProxy.service.port | int | `8080` | The port number to expose for the service. | | envoyProxy.service.type | string | `"ClusterIP"` | The Kubernetes service type to create. | -| envoyProxy.serviceAccount | object | [See values.yaml](values.yaml#L1157) | Service account configuration for the Envoy service. | +| envoyProxy.serviceAccount | object | [See values.yaml](values.yaml#L1150) | Service account configuration for the Envoy service. | | envoyProxy.serviceAccount.annotations | object | `{}` | Annotations to add to the service account. | | envoyProxy.serviceAccount.automount | bool | `true` | Automatically mount a ServiceAccount's API credentials. | | envoyProxy.serviceAccount.create | bool | `true` | Specifies whether a service account should be created. | @@ -371,8 +371,8 @@ For the complete default values, see [values.yaml](values.yaml). | envoyProxy.serviceMonitor.interval | string | `"30s"` | Scrape interval for the ServiceMonitor | | envoyProxy.serviceMonitor.labels | object | `{}` | Additional labels to add to the ServiceMonitor | | envoyProxy.serviceMonitor.scheme | string | `"http"` | Scheme to use for scraping metrics (http or https) | -| envoyProxy.startupProbe | object | [See values.yaml](values.yaml#L1230) | Startup probe for the Envoy container (admin interface /ready). | -| envoyProxy.timeouts | object | [See values.yaml](values.yaml#L1196) | Timeouts for proxying to long-lived streams (e.g. inference gateway). Use "0s" to disable a timeout. | +| envoyProxy.startupProbe | object | [See values.yaml](values.yaml#L1223) | Startup probe for the Envoy container (admin interface /ready). | +| envoyProxy.timeouts | object | [See values.yaml](values.yaml#L1189) | Timeouts for proxying to long-lived streams (e.g. inference gateway). Use "0s" to disable a timeout. | | envoyProxy.timeouts.connect | string | `"30s"` | Cluster connect timeout (time to establish connection to backend). | | envoyProxy.timeouts.request | string | `"0s"` | Total request timeout. 0 = disabled (required for streaming; not compatible with streaming if set). | | envoyProxy.timeouts.requestHeaders | string | `"60s"` | Time to receive full request headers. 0 = disabled. | @@ -407,13 +407,13 @@ For the complete default values, see [values.yaml](values.yaml). | httpRoute.hostnames | list | `[]` | If this has a specific hostname, add the name or names here in an array. | | httpRoute.labels | object | `{}` | Extra labels for the HTTP Route object. | | httpRoute.parentRefs | list | `[]` | A list of Gateways to enable this route on. This is required if httpRoute.enabled is true. | -| httpRoute.pathRules | list | [See values.yaml](values.yaml#L699) | Path matches to route queries. | +| httpRoute.pathRules | list | [See values.yaml](values.yaml#L692) | Path matches to route queries. | | imagePullSecrets | list | `[]` | Existing Kubernetes image pull secrets to use for pulling container images from private registries or mirrors. | | ingress.annotations | object | `{}` | Annotations for the ingress resource. | | ingress.className | string | `""` | The ingress class to use if your cluster has more than one class. | | ingress.defaultHost | string | `""` | Optional default hostname. When set, one rule is generated with this host and paths from the first entry in ingress.hosts. | | ingress.enabled | bool | `false` | Specifies whether to enable the ingress. | -| ingress.hosts[0] | object | [See values.yaml](values.yaml#L664) | Hostname used by ingress. If blank, use path-only routing. | +| ingress.hosts[0] | object | [See values.yaml](values.yaml#L657) | Hostname used by ingress. If blank, use path-only routing. | | ingress.tls | list | `[]` | TLS configurations. | | multinodeNetworking | object | [See values.yaml](values.yaml#L112) | Multi-node networking configuration for distributed GPU training. These settings control Kyverno policies that inject cloud-specific networking and NCCL configurations. Requirements: - Kyverno policy engine must be installed in your cluster (required for multi-node networking) - Kyverno is NOT included as a subchart dependency and must be installed separately To install Kyverno: helm install kyverno kyverno/kyverno --namespace kyverno --create-namespace --version 3.2.0 Documentation: https://kyverno.io/docs/installation/ Helm chart: https://kyverno.github.io/kyverno/ Note: Only enable ONE cloud provider per cluster deployment. | | multinodeNetworking.aws | object | `{"efaDevicesPerGPU":1,"enabled":false}` | AWS-specific configuration for EFA device injection | @@ -476,16 +476,16 @@ For the complete default values, see [values.yaml](values.yaml). | opensandbox.apiKeySecretKey | string | `"api-key"` | Key inside apiKeySecret. | | opensandbox.domain | string | [See values.yaml](values.yaml#L85) | In-cluster OpenSandbox Service DNS with no scheme. | | opensandbox.protocol | string | `"http"` | Scheme jobs use to reach the server. In-cluster Services speak http. | -| openshiftRoute | object | [See values.yaml](values.yaml#L718) | OpenShift Route (route.openshift.io/v1). Use on OpenShift to expose the API via a Route instead of Ingress. | +| openshiftRoute | object | [See values.yaml](values.yaml#L711) | OpenShift Route (route.openshift.io/v1). Use on OpenShift to expose the API via a Route instead of Ingress. | | openshiftRoute.annotations | object | `{}` | Annotations for the route resource. | | openshiftRoute.enabled | bool | `false` | Specifies whether to create an OpenShift Route for the API service. | | openshiftRoute.host | string | `""` | Hostname for the route. If empty, the OpenShift router may assign a default hostname. | | openshiftRoute.labels | object | `{}` | Labels for the route resource. | -| openshiftRoute.service | string | [See values.yaml](values.yaml#L724) | Service name to route to. Defaults to Envoy when auth+envoy enabled, otherwise API (tpl-evaluated). | -| openshiftRoute.targetPort | string | [See values.yaml](values.yaml#L726) | Target port on the service. Defaults to Envoy or API port depending on auth (tpl-evaluated). | +| openshiftRoute.service | string | [See values.yaml](values.yaml#L717) | Service name to route to. Defaults to Envoy when auth+envoy enabled, otherwise API (tpl-evaluated). | +| openshiftRoute.targetPort | string | [See values.yaml](values.yaml#L719) | Target port on the service. Defaults to Envoy or API port depending on auth (tpl-evaluated). | | openshiftRoute.tls | object | `{}` | Optional TLS configuration (termination, certificate, key, etc.). See OpenShift Route spec. | | platformConfig | object | [See values.yaml](values.yaml#L485) | Platform-wide configuration settings Set configuration here to apply custom, structured configuration across all services. Applied after the base platform config is evaluated for templates. Enables adding / overriding YAML-based elements in the evaluated platform config. It is usually recommended to use this config section instead of `basePlatformConfig` unless you need to use templating features. For example, you can set the NIM default StorageClass via models.controller.backends.deployments_plugin.default_storage_class. For full configuration reference, see https://docs.nvidia.com/nemo-platform | -| platformSeedJob | object | [See values.yaml](values.yaml#L939) | Platform seed Job (Helm hook: runs after install/upgrade) Runs the platform-seed task (guardrails configs, evaluator system entities, data designer filesets). Uses post-install,post-upgrade hooks so it runs on fresh installs and can be re-triggered on no-op upgrade. | +| platformSeedJob | object | [See values.yaml](values.yaml#L932) | Platform seed Job (Helm hook: runs after install/upgrade) Runs the platform-seed task (guardrails configs, evaluator system entities, data designer filesets). Uses post-install,post-upgrade hooks so it runs on fresh installs and can be re-triggered on no-op upgrade. | | platformSeedJob.activeDeadlineSeconds | int | `600` | Maximum time in seconds the Job can run. | | platformSeedJob.affinity | object | `{}` | Affinity for the platform seeding Job pod. | | platformSeedJob.backoffLimit | int | `6` | Number of retries before considering the Job failed. | @@ -498,7 +498,7 @@ For the complete default values, see [values.yaml](values.yaml). | platformSeedJob.securityContext | object | `{}` | Container-level security context for the platform-seed container. | | platformSeedJob.tolerations | list | `[]` | Tolerations for the platform seeding Job pod. | | platformSeedJob.ttlSecondsAfterFinished | int | `86400` | Seconds after the Job finishes (success or failure) before it is eligible for automatic deletion. | -| podSecurityContext | object | [See values.yaml](values.yaml#L759) | Pod security context settings applied to all services by default. These can be overridden in individual service configurations. | +| podSecurityContext | object | [See values.yaml](values.yaml#L752) | Pod security context settings applied to all services by default. These can be overridden in individual service configurations. | | postgresql | object | [See values.yaml](values.yaml#L286) | Local PostgreSQL configuration for the NeMo Platform. | | postgresql.affinity | object | `{}` | Affinity for the PostgreSQL pod. | | postgresql.auth | object | [See values.yaml](values.yaml#L296) | PostgreSQL authentication configuration. | @@ -541,7 +541,7 @@ For the complete default values, see [values.yaml](values.yaml). | secrets.defaultEncryptionKey.generated.tolerations | list | `[]` | Tolerations for the key generation hook. | | secrets.defaultEncryptionKey.generated.ttlSecondsAfterFinished | int | `300` | Seconds to keep the key generation hook Job after it finishes, if the hook is not deleted first. | | secrets.defaultEncryptionKey.value | string | `""` | Optional base64-encoded key for encrypting platform secrets. The decoded key must be at least 32 bytes. If empty and envFromSecret is not set, a pre-install hook generates a per-install key. | -| securityContext | object | [See values.yaml](values.yaml#L764) | Container security context settings applied to all services by default. These can be overridden in individual service configurations. | +| securityContext | object | [See values.yaml](values.yaml#L757) | Container security context settings applied to all services by default. These can be overridden in individual service configurations. | | telemetry.OTEL_EXPORTER_OTLP_ENDPOINT | string | `""` | The OpenTelemetry grpc collector endpoint to export traces and metrics to. | | telemetry.OTEL_EXPORTER_OTLP_INSECURE | bool | `true` | Whether to use an insecure connection (no TLS) to the OpenTelemetry collector endpoint. | | telemetry.OTEL_EXPORTER_OTLP_METRICS_ENDPOINT | string | `nil` | The OpenTelemetry metrics exporter endpoint to use. Defaults to `OTEL_EXPORTER_OTLP_ENDPOINT` if not set. | diff --git a/k8s/helm/values.yaml b/k8s/helm/values.yaml index c5a9f1d3c5..77a5208290 100644 --- a/k8s/helm/values.yaml +++ b/k8s/helm/values.yaml @@ -577,13 +577,6 @@ basePlatformConfig: | enabled: true k8s_executor: local-k8s default_executor: local-k8s - # Platform-default pod annotations applied to every k8s model - # deployment/job (all engines). The Istio native-sidecar annotation - # makes a mesh-injected istio-proxy terminate when a Job's main - # container exits, so the weight-puller Job completes while keeping - # mesh mTLS egress (inject: "false" is NOT viable in-mesh). - default_pod_annotations: - sidecar.istio.io/nativeSidecar: "true" # Bypass the image `nemo` ENTRYPOINT; invoke the adapters module directly. # Avoids PermissionError when the sidecar writes instance state under $HOME. lora_sidecar_command: diff --git a/plugins/nemo-deployments/src/nemo_deployments_plugin/backends/k8s/backend.py b/plugins/nemo-deployments/src/nemo_deployments_plugin/backends/k8s/backend.py index a0a2229e9f..c0e595eac1 100644 --- a/plugins/nemo-deployments/src/nemo_deployments_plugin/backends/k8s/backend.py +++ b/plugins/nemo-deployments/src/nemo_deployments_plugin/backends/k8s/backend.py @@ -139,6 +139,7 @@ async def create_deployment( backend_config=backend_config, config=config, executor_image_pull_secrets=self._executor_config.image_pull_secrets, + executor_defaults=self._executor_config.to_k8s_defaults(), secret_env=secret_env, auth_context=auth_context, workload_delegation_store=self._workload_delegations, @@ -154,6 +155,7 @@ async def create_deployment( backend_config=backend_config, config=config, executor_image_pull_secrets=self._executor_config.image_pull_secrets, + executor_defaults=self._executor_config.to_k8s_defaults(), secret_env=secret_env, auth_context=auth_context, workload_delegation_store=self._workload_delegations, diff --git a/plugins/nemo-deployments/src/nemo_deployments_plugin/backends/k8s/compiler.py b/plugins/nemo-deployments/src/nemo_deployments_plugin/backends/k8s/compiler.py index d079d70911..fcdb00d720 100644 --- a/plugins/nemo-deployments/src/nemo_deployments_plugin/backends/k8s/compiler.py +++ b/plugins/nemo-deployments/src/nemo_deployments_plugin/backends/k8s/compiler.py @@ -155,6 +155,24 @@ def build_container_spec( return k8s.client.V1Container(**kwargs) +@dataclass(frozen=True) +class ExecutorK8sDefaults: + """Executor-level k8s defaults applied to every workload the executor renders. + + These are the base layer shared by ALL deployments-plugin consumers (models, + agents, ...). A per-entity ``K8sDeploymentConfig`` value overrides the default: + annotations merge key-wise (per-entity key wins); node_selector / tolerations / + affinity / topology_spread_constraints are applied only when the entity leaves + them unset (per-entity wins wholesale). + """ + + pod_annotations: dict[str, str] = field(default_factory=dict) + node_selector: dict[str, str] = field(default_factory=dict) + tolerations: list[dict[str, Any]] = field(default_factory=list) + affinity: dict[str, Any] = field(default_factory=dict) + topology_spread_constraints: list[dict[str, Any]] = field(default_factory=list) + + @dataclass(frozen=True) class CompiledWorkload: """Kubernetes objects derived from a DeploymentConfig.""" @@ -531,6 +549,7 @@ def compile_workload( k8s_config: K8sDeploymentConfig | None, pod_restart_policy: RestartPolicy, executor_image_pull_secrets: list[ImagePullSecret] | None = None, + executor_defaults: ExecutorK8sDefaults | None = None, secret_env: dict[str, str] | None = None, ) -> CompiledWorkload: """Compile pod spec kwargs and optional ConfigMap/Secret for a Job or Deployment. @@ -618,7 +637,32 @@ def compile_workload( if effective_service_account_name: pod_spec_kwargs["service_account_name"] = effective_service_account_name - pod_annotations = dict(k8s_config.pod_annotations) if k8s_config is not None else {} + # Apply executor-level defaults as the BASE layer, shared by every consumer + # (models, agents, ...). A per-entity K8sDeploymentConfig value (set above) + # wins: node_selector / tolerations / affinity / topology_spread are applied + # only when the entity left them unset; annotations merge key-wise below. + entity_pod_annotations = dict(k8s_config.pod_annotations) if k8s_config is not None else {} + if executor_defaults is not None: + if executor_defaults.node_selector and "node_selector" not in pod_spec_kwargs: + pod_spec_kwargs["node_selector"] = dict(executor_defaults.node_selector) + if executor_defaults.tolerations and "tolerations" not in pod_spec_kwargs: + default_tolerations = build_tolerations( + [Toleration.model_validate(item) for item in executor_defaults.tolerations if item] + ) + if default_tolerations: + pod_spec_kwargs["tolerations"] = default_tolerations + if executor_defaults.affinity and "affinity" not in pod_spec_kwargs: + default_affinity = _deserialize_k8s(executor_defaults.affinity, "V1Affinity") + if default_affinity is not None: + pod_spec_kwargs["affinity"] = default_affinity + if executor_defaults.topology_spread_constraints and "topology_spread_constraints" not in pod_spec_kwargs: + default_tsc = build_topology_spread_constraints(executor_defaults.topology_spread_constraints) + if default_tsc: + pod_spec_kwargs["topology_spread_constraints"] = default_tsc + # Annotations merge key-wise: executor default first, per-entity wins. + pod_annotations = {**executor_defaults.pod_annotations, **entity_pod_annotations} + else: + pod_annotations = entity_pod_annotations return CompiledWorkload( pod_spec_kwargs=pod_spec_kwargs, diff --git a/plugins/nemo-deployments/src/nemo_deployments_plugin/backends/k8s/config.py b/plugins/nemo-deployments/src/nemo_deployments_plugin/backends/k8s/config.py index b51c1c46fc..871454cd28 100644 --- a/plugins/nemo-deployments/src/nemo_deployments_plugin/backends/k8s/config.py +++ b/plugins/nemo-deployments/src/nemo_deployments_plugin/backends/k8s/config.py @@ -7,7 +7,10 @@ import os import re +from copy import deepcopy +from typing import Any +from nemo_deployments_plugin.backends.k8s.compiler import ExecutorK8sDefaults from nemo_platform_plugin.config import ImagePullSecret from pydantic import BaseModel, Field, field_validator @@ -50,6 +53,47 @@ class K8sExecutorConfig(BaseModel): default_factory=list, description="Image pull secrets merged with platform image_pull_secrets on every pod.", ) + default_pod_annotations: dict[str, str] = Field( + default_factory=dict, + description=( + "Executor-level default pod annotations stamped onto every Job/Deployment pod this " + "executor renders (all consumers: models, agents, ...). Merged key-wise with per-entity " + "backend_config.k8s.podAnnotations, where the per-entity value wins for a shared key. " + "Ships the Istio native-sidecar annotation so a mesh-injected proxy terminates when a " + "Job's main container exits." + ), + ) + default_node_selector: dict[str, str] = Field( + default_factory=dict, + description=( + "Executor-level default nodeSelector applied to every Job/Deployment pod this executor " + "renders when the entity does not set backend_config.k8s.nodeSelector." + ), + ) + default_tolerations: list[dict[str, str | int]] = Field( + default_factory=list, + description=( + "Executor-level default pod tolerations applied to every Job/Deployment pod this executor " + "renders when the entity does not set backend_config.k8s.tolerations. Each entry is a raw " + "Kubernetes toleration object." + ), + ) + default_affinity: dict[str, Any] = Field( + default_factory=dict, + description=( + "Executor-level default pod affinity applied to every Job/Deployment pod this executor " + "renders when the entity does not set backend_config.k8s.affinity. Raw Kubernetes affinity " + "object (nodeAffinity / podAffinity / podAntiAffinity)." + ), + ) + default_topology_spread_constraints: list[dict[str, Any]] = Field( + default_factory=list, + description=( + "Executor-level default pod topology spread constraints applied to every Job/Deployment pod " + "this executor renders when the entity does not set backend_config.k8s.topologySpreadConstraints. " + "Each entry is a raw Kubernetes topologySpreadConstraint object." + ), + ) @field_validator("default_namespace") @classmethod @@ -73,3 +117,18 @@ def effective_namespace(self) -> str: if pod_namespace: return pod_namespace return _FALLBACK_NAMESPACE + + def to_k8s_defaults(self) -> ExecutorK8sDefaults: + """Bundle the executor-level pod defaults for the workload compiler. + + Shared by every deployments-plugin consumer (models, agents, ...): the + compiler applies these as the base layer, with per-entity + ``backend_config.k8s`` values overriding them. + """ + return ExecutorK8sDefaults( + pod_annotations=dict(self.default_pod_annotations), + node_selector=dict(self.default_node_selector), + tolerations=deepcopy(self.default_tolerations), + affinity=deepcopy(self.default_affinity), + topology_spread_constraints=deepcopy(self.default_topology_spread_constraints), + ) diff --git a/plugins/nemo-deployments/src/nemo_deployments_plugin/backends/k8s/deployments.py b/plugins/nemo-deployments/src/nemo_deployments_plugin/backends/k8s/deployments.py index 2bdd506851..f47331e1cf 100644 --- a/plugins/nemo-deployments/src/nemo_deployments_plugin/backends/k8s/deployments.py +++ b/plugins/nemo-deployments/src/nemo_deployments_plugin/backends/k8s/deployments.py @@ -21,6 +21,7 @@ from nemo_deployments_plugin.backends.k8s.compiler import ( CompiledWorkload, DeploymentConfigError, + ExecutorK8sDefaults, compile_workload, create_configmap, create_secret, @@ -101,6 +102,7 @@ def build_deployment_body( deployment_name: str, k8s_config: K8sDeploymentConfig | None, executor_image_pull_secrets: list | None = None, + executor_defaults: ExecutorK8sDefaults | None = None, secret_env: dict[str, str] | None = None, ) -> BuiltDeployment: """Build an ``apps/v1.Deployment`` for create and its compiled workload.""" @@ -115,6 +117,7 @@ def build_deployment_body( k8s_config=k8s_config, pod_restart_policy="Always", executor_image_pull_secrets=executor_image_pull_secrets, + executor_defaults=executor_defaults, secret_env=secret_env, ) deployment = k8s.client.V1Deployment( @@ -331,6 +334,7 @@ async def create_deployment( backend_config: dict[str, Any], config: DeploymentConfig, executor_image_pull_secrets: list | None = None, + executor_defaults: ExecutorK8sDefaults | None = None, secret_env: dict[str, str] | None = None, auth_context: AuthContext | None = None, workload_delegation_store: WorkloadDelegationStore | None = None, @@ -359,6 +363,7 @@ async def create_deployment( deployment_name=name, k8s_config=k8s_config, executor_image_pull_secrets=executor_image_pull_secrets, + executor_defaults=executor_defaults, secret_env=secret_env, ) deployment_body = built.deployment diff --git a/plugins/nemo-deployments/src/nemo_deployments_plugin/backends/k8s/jobs.py b/plugins/nemo-deployments/src/nemo_deployments_plugin/backends/k8s/jobs.py index 22d309847f..bb009cd0e1 100644 --- a/plugins/nemo-deployments/src/nemo_deployments_plugin/backends/k8s/jobs.py +++ b/plugins/nemo-deployments/src/nemo_deployments_plugin/backends/k8s/jobs.py @@ -18,6 +18,7 @@ from nemo_deployments_plugin.backends.k8s.compiler import ( CompiledWorkload, DeploymentConfigError, + ExecutorK8sDefaults, compile_workload, create_configmap, create_secret, @@ -147,6 +148,7 @@ def build_job_body( deployment_name: str, k8s_config: K8sDeploymentConfig | None, executor_image_pull_secrets: list | None = None, + executor_defaults: ExecutorK8sDefaults | None = None, secret_env: dict[str, str] | None = None, ) -> BuiltJob: """Build a ``batch/v1.Job`` for create.""" @@ -159,6 +161,7 @@ def build_job_body( k8s_config=k8s_config, pod_restart_policy=config.restart_policy, executor_image_pull_secrets=executor_image_pull_secrets, + executor_defaults=executor_defaults, secret_env=secret_env, ) job = k8s.client.V1Job( @@ -249,6 +252,7 @@ async def create_job( backend_config: dict[str, Any], config: DeploymentConfig, executor_image_pull_secrets: list | None = None, + executor_defaults: ExecutorK8sDefaults | None = None, secret_env: dict[str, str] | None = None, auth_context: AuthContext | None = None, workload_delegation_store: WorkloadDelegationStore | None = None, @@ -277,6 +281,7 @@ async def create_job( deployment_name=name, k8s_config=k8s_config, executor_image_pull_secrets=executor_image_pull_secrets, + executor_defaults=executor_defaults, secret_env=secret_env, ) body = built.job diff --git a/plugins/nemo-deployments/tests/unit/backends/k8s/test_backend.py b/plugins/nemo-deployments/tests/unit/backends/k8s/test_backend.py index 9b6f44167e..42a705b760 100644 --- a/plugins/nemo-deployments/tests/unit/backends/k8s/test_backend.py +++ b/plugins/nemo-deployments/tests/unit/backends/k8s/test_backend.py @@ -30,6 +30,45 @@ def test_default_namespace_rejects_invalid_dns_label() -> None: K8sExecutorConfig(default_namespace="X") +def test_to_k8s_defaults_bundles_executor_pod_defaults() -> None: + config = K8sExecutorConfig( + default_pod_annotations={"sidecar.istio.io/nativeSidecar": "true"}, + default_node_selector={"gpu": "a100"}, + default_tolerations=[{"key": "gpu", "operator": "Equal", "value": "true", "effect": "NoSchedule"}], + default_affinity={"nodeAffinity": {}}, + default_topology_spread_constraints=[{"maxSkew": 1, "topologyKey": "kubernetes.io/hostname"}], + ) + defaults = config.to_k8s_defaults() + assert defaults.pod_annotations == {"sidecar.istio.io/nativeSidecar": "true"} + assert defaults.node_selector == {"gpu": "a100"} + assert defaults.tolerations[0]["key"] == "gpu" + assert defaults.affinity == {"nodeAffinity": {}} + assert defaults.topology_spread_constraints[0]["topologyKey"] == "kubernetes.io/hostname" + + +def test_to_k8s_defaults_empty_by_default() -> None: + defaults = K8sExecutorConfig().to_k8s_defaults() + assert defaults.pod_annotations == {} + assert defaults.node_selector == {} + assert defaults.tolerations == [] + assert defaults.affinity == {} + assert defaults.topology_spread_constraints == [] + + +def test_to_k8s_defaults_deep_copies_nested_objects() -> None: + # Mutating the returned defaults must not corrupt the shared executor config + # for later workloads (nested nodeAffinity / labelSelector are deep-copied). + config = K8sExecutorConfig( + default_affinity={"nodeAffinity": {"key": "orig"}}, + default_topology_spread_constraints=[{"labelSelector": {"matchLabels": {"app": "orig"}}}], + ) + defaults = config.to_k8s_defaults() + defaults.affinity["nodeAffinity"]["key"] = "mutated" + defaults.topology_spread_constraints[0]["labelSelector"]["matchLabels"]["app"] = "mutated" + assert config.default_affinity["nodeAffinity"]["key"] == "orig" + assert config.default_topology_spread_constraints[0]["labelSelector"]["matchLabels"]["app"] == "orig" + + def test_effective_namespace_prefers_explicit_config(monkeypatch: pytest.MonkeyPatch) -> None: # An explicit config value wins even when POD_NAMESPACE is set. monkeypatch.setenv("POD_NAMESPACE", "pod-ns") diff --git a/plugins/nemo-deployments/tests/unit/backends/k8s/test_compiler.py b/plugins/nemo-deployments/tests/unit/backends/k8s/test_compiler.py index 8cb7503e68..26aedc80fb 100644 --- a/plugins/nemo-deployments/tests/unit/backends/k8s/test_compiler.py +++ b/plugins/nemo-deployments/tests/unit/backends/k8s/test_compiler.py @@ -11,6 +11,7 @@ from kubernetes.client import ApiClient from nemo_deployments_plugin.backends.k8s.compiler import ( DeploymentConfigError, + ExecutorK8sDefaults, _build_probe, build_configmap_body, build_env_vars, @@ -263,6 +264,99 @@ def test_compile_carries_pod_annotations() -> None: assert "annotations" not in compiled.pod_spec_kwargs +def test_compile_applies_executor_defaults() -> None: + # Executor-level defaults (shared by every consumer: models, agents, ...) land + # on a workload with no per-entity k8s config. + config = sample_always_config() + executor_defaults = ExecutorK8sDefaults( + pod_annotations={"sidecar.istio.io/nativeSidecar": "true"}, + node_selector={"gpu": "a100"}, + tolerations=[{"key": "gpu", "operator": "Equal", "value": "true", "effect": "NoSchedule"}], + affinity={ + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [{"matchExpressions": [{"key": "gpu", "operator": "In", "values": ["a100"]}]}] + } + } + }, + topology_spread_constraints=[ + {"maxSkew": 1, "topologyKey": "kubernetes.io/hostname", "whenUnsatisfiable": "DoNotSchedule"} + ], + ) + compiled = compile_workload( + config=config, + workspace="default", + deployment_name="task", + labels={"managed-by": "nemo-deployments"}, + k8s_config=None, + pod_restart_policy="Always", + executor_defaults=executor_defaults, + ) + pod_spec = _serialized(compiled.pod_spec_kwargs) + assert compiled.pod_annotations == {"sidecar.istio.io/nativeSidecar": "true"} + assert pod_spec["node_selector"] == {"gpu": "a100"} + assert pod_spec["tolerations"][0]["key"] == "gpu" + assert compiled.pod_spec_kwargs["affinity"].node_affinity is not None + assert pod_spec["topology_spread_constraints"][0]["topologyKey"] == "kubernetes.io/hostname" + + +def test_compile_per_entity_wins_over_executor_defaults() -> None: + # A per-entity K8sDeploymentConfig overrides the executor default: annotations + # merge key-wise (entity key wins), and node_selector / tolerations are applied + # from the entity wholesale (executor default not additionally applied). + config = sample_always_config() + k8s_config = K8sDeploymentConfig.model_validate( + { + "podAnnotations": {"sidecar.istio.io/nativeSidecar": "false", "team": "a"}, + "nodeSelector": {"zone": "us-west1-a"}, + "tolerations": [{"key": "entity", "operator": "Exists"}], + } + ) + executor_defaults = ExecutorK8sDefaults( + pod_annotations={"sidecar.istio.io/nativeSidecar": "true", "platform": "nmp"}, + node_selector={"gpu": "a100"}, + tolerations=[{"key": "platform", "operator": "Exists"}], + ) + compiled = compile_workload( + config=config, + workspace="default", + deployment_name="task", + labels={"managed-by": "nemo-deployments"}, + k8s_config=k8s_config, + pod_restart_policy="Always", + executor_defaults=executor_defaults, + ) + pod_spec = _serialized(compiled.pod_spec_kwargs) + # Annotations: entity value wins for the shared key; non-conflicting keys from + # both sides are retained. + assert compiled.pod_annotations == { + "sidecar.istio.io/nativeSidecar": "false", + "team": "a", + "platform": "nmp", + } + # node_selector / tolerations: entity wins wholesale. + assert pod_spec["node_selector"] == {"zone": "us-west1-a"} + assert [t["key"] for t in pod_spec["tolerations"]] == ["entity"] + + +def test_compile_empty_executor_defaults_are_noop() -> None: + config = sample_config(restart_policy="Never") + compiled = compile_workload( + config=config, + workspace="default", + deployment_name="task", + labels={"managed-by": "nemo-deployments"}, + k8s_config=None, + pod_restart_policy="Never", + executor_defaults=ExecutorK8sDefaults(), + ) + assert compiled.pod_annotations == {} + assert "node_selector" not in compiled.pod_spec_kwargs + assert "tolerations" not in compiled.pod_spec_kwargs + assert "affinity" not in compiled.pod_spec_kwargs + assert "topology_spread_constraints" not in compiled.pod_spec_kwargs + + def test_compile_pod_annotations_default_empty_without_k8s_config() -> None: config = sample_config(restart_policy="Never") compiled = compile_workload(