fix: set envoy upstream idle timeout

.github/workflows/ci.yaml

@@ -1060,6 +1060,7 @@ jobs:
       !cancelled() && (
         github.event_name == 'workflow_dispatch' ||
         needs.changes.outputs.cpu-smoke == 'true' ||
+        needs.changes.outputs.helm == 'true' ||
         needs.changes.outputs.auth-idp == 'true'
       )
     runs-on: ubuntu-latest
@@ -1078,6 +1079,9 @@ jobs:
           cache-dependency-glob: uv.lock
       - name: Run auth-idp static tests
         run: |
+          set -euo pipefail
+          docker pull docker.io/envoyproxy/envoy:v1.37.0
+          docker pull envoyproxy/envoy:v1.36.2
           helm dependency build k8s/helm
           helm dependency build contrib/auth/authentik/helm
           uv run --frozen pytest tests/auth_idp/static -v

contrib/auth/authentik/gateway/envoy.yaml

@@ -251,6 +251,16 @@ static_resources:
     - name: nemo
       connect_timeout: 5s
       type: LOGICAL_DNS
+      # Envoy's default upstream HTTP idle timeout is 1h. Keep this below the
+      # API keep-alive timeout so stale pooled API connections are retired
+      # before the backend closes them.
+      typed_extension_protocol_options:
+        envoy.extensions.upstreams.http.v3.HttpProtocolOptions:
+          "@type": type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions
+          common_http_protocol_options:
+            idle_timeout: "4s"
+          explicit_http_config:
+            http_protocol_options: {}
       load_assignment:
         cluster_name: nemo
         endpoints:

contrib/auth/authentik/helm/templates/_envoy-config.tpl

@@ -257,6 +257,16 @@ static_resources:
     - name: nemo
       connect_timeout: 5s
       type: LOGICAL_DNS
+      # Envoy's default upstream HTTP idle timeout is 1h. Keep this below the
+      # API keep-alive timeout so stale pooled API connections are retired
+      # before the backend closes them.
+      typed_extension_protocol_options:
+        envoy.extensions.upstreams.http.v3.HttpProtocolOptions:
+          "@type": type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions
+          common_http_protocol_options:
+            idle_timeout: {{ .Values.envoyProxy.timeouts.upstreamIdle | quote }}
+          explicit_http_config:
+            http_protocol_options: {}
       load_assignment:
         cluster_name: nemo
         endpoints:

contrib/auth/authentik/helm/values.yaml

@@ -188,6 +188,8 @@ nemo-platform:
       - name: workload-token-tls
         secret:
           secretName: *workloadTokenTlsSecretName
+    timeouts:
+      upstreamIdle: "4s"
     configOverride: '{{ include "nemo-platform-authentik.envoyConfig" . }}'
   platformConfig:
     auth:

docs/cli/reference.mdx

@@ -342,6 +342,7 @@ nemo services run [OPTIONS]
 * `--config`: Path to a platform configuration YAML file.
 * `--host`: Host to bind to. [default: 127.0.0.1]
 * `--port <INTEGER>`: Port to bind to. [default: 8080]
+* `--keep-alive-timeout-seconds <INTEGER RANGE>`: Seconds Uvicorn keeps idle HTTP connections open. [default: 5]
 * `--instance`: Instance name. Defaults to a name derived from the working directory and port.
 
 **Help:**
@@ -377,6 +378,7 @@ nemo services start [OPTIONS]
 * `--config`: Path to a platform configuration YAML file.
 * `--host`: Host to bind to. [default: 127.0.0.1]
 * `--port <INTEGER>`: Port to bind to. [default: 8080]
+* `--keep-alive-timeout-seconds <INTEGER RANGE>`: Seconds Uvicorn keeps idle HTTP connections open. [default: 5]
 * `--instance`: Instance name. Defaults to a name derived from the working directory and port.
 
 **Help:**
@@ -446,6 +448,7 @@ nemo services restart [OPTIONS]
 * `--config`: Path to a platform configuration YAML file.
 * `--host`: Host to bind to. Defaults to previous value or 127.0.0.1.
 * `--port <INTEGER>`: Port to bind to. Defaults to previous value or 8080.
+* `--keep-alive-timeout-seconds <INTEGER RANGE>`: Seconds Uvicorn keeps idle HTTP connections open. Defaults to the previous value or 5.
 * `--instance`: Instance name. Defaults to a name derived from the working directory and port.
 
 **Help:**

k8s/helm/README.md

@@ -160,6 +160,8 @@ and
 | api.replicaCount | int | `1` | Number of replicas for the API service. |
 | api.resources | object | `{}` | Kubernetes deployment resources configuration for the API service. Utilization-based autoscaling requires a matching resource request. |
 | api.securityContext | object | `{}` | Container-level security context settings for the API service. |
+| api.server | object | `{"keepAliveTimeoutSeconds":5}` | Platform API server settings. |
+| api.server.keepAliveTimeoutSeconds | int | `5` | Seconds Uvicorn keeps idle HTTP connections open. Must be greater than envoyProxy.timeouts.upstreamIdle when Envoy is enabled. |
 | api.service | object | This object has the following default values for the service configuration. | Service configuration for the API service. |
 | api.service.annotations | object | `{}` | Annotations for the API service. |
 | api.service.port | int | `8080` | The port number to expose for the service. |
@@ -331,6 +333,7 @@ and
 | envoyProxy.timeouts.requestHeaders | string | `"60s"` | Time to receive full request headers. 0 = disabled. |
 | envoyProxy.timeouts.route | string | `"0s"` | Per-route timeout for the passthrough to backend. 0 = disabled. |
 | envoyProxy.timeouts.streamIdle | string | `"0s"` | Stream idle timeout. Time with no activity before stream is closed. 0 = disabled (required for long-lived streams). |
+| envoyProxy.timeouts.upstreamIdle | string | `"4s"` | Positive whole-second upstream connection idle timeout. Must be less than api.server.keepAliveTimeoutSeconds when Envoy is enabled. |
 | envoyProxy.tolerations | list | `[]` | Tolerations configuration for the Envoy pods. |
 | envoyProxy.topologySpreadConstraints | list | `[]` | Topology spread constraints for the Envoy pods. See https://kubernetes.io/docs/concepts/scheduling-eviction/topology-spread-constraints/ |
 | existingSecret | string | `"ngc-api"` | You can use an existing Kubernetes secret for communicating with the NGC API for downloading models. The chart uses the `ngcAPIKey` value to generate the secret if you set this to an empty string. |

k8s/helm/templates/_helpers.tpl

@@ -141,6 +141,27 @@ local process instead of the cluster Service.
 {{- printf "http://localhost:%s" (toString .Values.api.service.port) -}}
 {{- end -}}
 
+{{/*
+Validate that Envoy retires idle upstream API connections before Uvicorn closes
+them. This avoids reusing a backend connection that the API already dropped.
+*/}}
+{{- define "nemo-platform.validateEnvoyKeepAliveTimeouts" -}}
+{{- if and .Values.api.enabled (include "nemo-platform.authEnabled" .) .Values.envoyProxy.enabled -}}
+{{- $apiKeepAliveSeconds := .Values.api.server.keepAliveTimeoutSeconds | int -}}
+{{- if lt $apiKeepAliveSeconds 1 -}}
+{{- fail "api.server.keepAliveTimeoutSeconds must be greater than 0" -}}
+{{- end -}}
+{{- $upstreamIdle := .Values.envoyProxy.timeouts.upstreamIdle | toString -}}
+{{- if not (regexMatch "^[1-9][0-9]*s$" $upstreamIdle) -}}
+{{- fail "envoyProxy.timeouts.upstreamIdle must be a positive whole-second duration like \"4s\"" -}}
+{{- end -}}
+{{- $upstreamIdleSeconds := trimSuffix "s" $upstreamIdle | int -}}
+{{- if ge $upstreamIdleSeconds $apiKeepAliveSeconds -}}
+{{- fail (printf "envoyProxy.timeouts.upstreamIdle (%s) must be less than api.server.keepAliveTimeoutSeconds (%ds)" $upstreamIdle $apiKeepAliveSeconds) -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+
 {{/*
 Pod annotations
 */}}

k8s/helm/templates/api/api-deployment.yaml

@@ -52,6 +52,7 @@ spec:
             - "--service-group=all"
             - "--host={{ include "nemo-platform.bindHost" . }}"
             - "--port={{ .Values.api.service.port }}"
+            - "--keep-alive-timeout-seconds={{ .Values.api.server.keepAliveTimeoutSeconds }}"
             {{- range .Values.api.extraArgs }}
             - {{ . | quote }}
             {{- end }}

k8s/helm/templates/proxy/envoy-configmap.yaml

@@ -1,3 +1,4 @@
+{{- include "nemo-platform.validateEnvoyKeepAliveTimeouts" . }}
 {{- if and (include "nemo-platform.authEnabled" .) .Values.envoyProxy.enabled }}
 apiVersion: v1
 kind: ConfigMap
@@ -59,6 +60,16 @@ data:
           type: STRICT_DNS
           lb_policy: ROUND_ROBIN
           connect_timeout: {{ .Values.envoyProxy.timeouts.connect | quote }}
+          # Envoy's default upstream HTTP idle timeout is 1h. The chart default
+          # stays below the API keep-alive timeout so stale pooled API
+          # connections are retired before the backend closes them.
+          typed_extension_protocol_options:
+            envoy.extensions.upstreams.http.v3.HttpProtocolOptions:
+              "@type": type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions
+              common_http_protocol_options:
+                idle_timeout: {{ .Values.envoyProxy.timeouts.upstreamIdle | quote }}
+              explicit_http_config:
+                http_protocol_options: {}
           load_assignment:
             cluster_name: backend_cluster
             endpoints:

k8s/helm/values.yaml

@@ -647,6 +647,10 @@ api:
 
   # -- Number of replicas for the API service.
   replicaCount: 1
+  # -- Platform API server settings.
+  server:
+    # -- Seconds Uvicorn keeps idle HTTP connections open. Must be greater than envoyProxy.timeouts.upstreamIdle when Envoy is enabled.
+    keepAliveTimeoutSeconds: 5
   # -- Additional arguments to pass to the Platform API service
   extraArgs: []
   # -- Additional volume mounts to add to the Platform API container.
@@ -1053,6 +1057,8 @@ envoyProxy:
     route: "0s"
     # -- Cluster connect timeout (time to establish connection to backend).
     connect: "30s"
+    # -- Positive whole-second upstream connection idle timeout. Must be less than api.server.keepAliveTimeoutSeconds when Envoy is enabled.
+    upstreamIdle: "4s"
 
   # -- Kubernetes deployment resources configuration for the Envoy service. Utilization-based autoscaling requires a matching resource request.
   resources: {}

packages/nemo_platform_ext/src/nemo_platform_ext/cli/commands/services/cli.py

@@ -37,7 +37,11 @@
     stop_instance,
     write_descriptor,
 )
-from nmp.platform_runner.config import DEFAULT_LOCAL_SERVICES_BIND_HOST, PlatformAppConfig
+from nmp.platform_runner.config import (
+    DEFAULT_LOCAL_SERVICES_BIND_HOST,
+    DEFAULT_UVICORN_KEEP_ALIVE_TIMEOUT_SECONDS,
+    PlatformAppConfig,
+)
 
 logger = logging.getLogger(__name__)
 
@@ -207,6 +211,14 @@ def run_services(
     ] = None,
     host: Annotated[str, typer.Option("--host", help="Host to bind to.")] = DEFAULT_LOCAL_SERVICES_BIND_HOST,
     port: Annotated[int, typer.Option("--port", help="Port to bind to.")] = _DEFAULT_PORT,
+    keep_alive_timeout_seconds: Annotated[
+        int,
+        typer.Option(
+            "--keep-alive-timeout-seconds",
+            min=1,
+            help="Seconds Uvicorn keeps idle HTTP connections open.",
+        ),
+    ] = DEFAULT_UVICORN_KEEP_ALIVE_TIMEOUT_SECONDS,
     instance: Annotated[
         str | None,
         typer.Option(
@@ -244,6 +256,7 @@ def run_services(
         scope=scope,
         host=host,
         port=port,
+        keep_alive_timeout_seconds=keep_alive_timeout_seconds,
         state_root=base_dir,
     )
 
@@ -317,6 +330,14 @@ def start_services(
     ] = None,
     host: Annotated[str, typer.Option("--host", help="Host to bind to.")] = DEFAULT_LOCAL_SERVICES_BIND_HOST,
     port: Annotated[int, typer.Option("--port", help="Port to bind to.")] = _DEFAULT_PORT,
+    keep_alive_timeout_seconds: Annotated[
+        int,
+        typer.Option(
+            "--keep-alive-timeout-seconds",
+            min=1,
+            help="Seconds Uvicorn keeps idle HTTP connections open.",
+        ),
+    ] = DEFAULT_UVICORN_KEEP_ALIVE_TIMEOUT_SECONDS,
     instance: Annotated[
         str | None,
         typer.Option(
@@ -358,6 +379,7 @@ def start_services(
         scope=scope,
         host=host,
         port=port,
+        keep_alive_timeout_seconds=keep_alive_timeout_seconds,
         state_root=base_dir,
     )
 
@@ -495,6 +517,14 @@ def restart_services(
         int | None,
         typer.Option("--port", help="Port to bind to. Defaults to previous value or 8080."),
     ] = None,
+    keep_alive_timeout_seconds: Annotated[
+        int | None,
+        typer.Option(
+            "--keep-alive-timeout-seconds",
+            min=1,
+            help="Seconds Uvicorn keeps idle HTTP connections open. Defaults to the previous value or 5.",
+        ),
+    ] = None,
     instance: Annotated[
         str | None,
         typer.Option(
@@ -561,6 +591,15 @@ def restart_services(
         host if host is not None else (previous_config.host if previous_config else DEFAULT_LOCAL_SERVICES_BIND_HOST)
     )
     effective_port = port if port is not None else (previous_config.port if previous_config else _DEFAULT_PORT)
+    effective_keep_alive_timeout_seconds = (
+        keep_alive_timeout_seconds
+        if keep_alive_timeout_seconds is not None
+        else (
+            previous_config.keep_alive_timeout_seconds
+            if previous_config
+            else DEFAULT_UVICORN_KEEP_ALIVE_TIMEOUT_SECONDS
+        )
+    )
 
     _warn_bind_all(effective_host)
 
@@ -575,6 +614,7 @@ def restart_services(
         scope=scope,
         host=effective_host,
         port=effective_port,
+        keep_alive_timeout_seconds=effective_keep_alive_timeout_seconds,
         state_root=base_dir,
     )
 

packages/nemo_platform_ext/src/nemo_platform_ext/local/process.py

@@ -827,6 +827,7 @@ def start_background(
     if config.config_path:
         args += ["--config", config.config_path]
     args += ["--host", config.host, "--port", str(config.port)]
+    args += ["--keep-alive-timeout-seconds", str(config.keep_alive_timeout_seconds)]
     args += ["--instance", config.scope]
 
     env = os.environ.copy()

packages/nemo_platform_ext/src/nemo_platform_ext/local/services.py

@@ -35,9 +35,11 @@
 )
 from nmp.platform_runner.config import (
     DEFAULT_SCOPE,
+    DEFAULT_UVICORN_KEEP_ALIVE_TIMEOUT_SECONDS,
     PlatformAppConfig,
     default_runtime_root,
     default_state_root,
+    validate_keep_alive_timeout_seconds,
     validate_scope,
 )
 
@@ -151,6 +153,7 @@ class ServiceRunConfig:
     data_dir: str | Path | None = None
     readiness_timeout: float = 60.0
     readiness_poll_interval: float = 0.5
+    keep_alive_timeout_seconds: int = DEFAULT_UVICORN_KEEP_ALIVE_TIMEOUT_SECONDS
     mode: ServiceMode | str = ServiceMode.DAEMON
 
     def __post_init__(self) -> None:
@@ -176,6 +179,7 @@ def __post_init__(self) -> None:
             raise ValueError("readiness_timeout must be greater than 0")
         if self.readiness_poll_interval <= 0:
             raise ValueError("readiness_poll_interval must be greater than 0")
+        self.keep_alive_timeout_seconds = validate_keep_alive_timeout_seconds(self.keep_alive_timeout_seconds)
         self.scope = validate_scope(self.scope)
 
     @property
@@ -215,6 +219,7 @@ def to_platform_app_config(self) -> PlatformAppConfig:
             socket_path=_optional_str(self.resolved_socket_path),
             state_root=_optional_str(self.state_root),
             runtime_root=_optional_str(self.runtime_dir),
+            keep_alive_timeout_seconds=self.keep_alive_timeout_seconds,
         )
 
     def to_child_payload(self) -> dict[str, object]:
@@ -239,6 +244,7 @@ def to_child_payload(self) -> dict[str, object]:
             "data_dir": _optional_str(self.data_dir),
             "readiness_timeout": self.readiness_timeout,
             "readiness_poll_interval": self.readiness_poll_interval,
+            "keep_alive_timeout_seconds": self.keep_alive_timeout_seconds,
         }
 
 
@@ -486,9 +492,21 @@ def serve_embedded_app(app: Any, cfg: ServiceRunConfig, socket_path: Path | None
     if socket_path is not None:
         from nmp.platform_runner.server import _run_server_on_bound_sockets
 
-        _run_server_on_bound_sockets(app, host=cfg.host, port=cfg.port, socket_path=str(socket_path))
+        _run_server_on_bound_sockets(
+            app,
+            host=cfg.host,
+            port=cfg.port,
+            socket_path=str(socket_path),
+            keep_alive_timeout_seconds=cfg.keep_alive_timeout_seconds,
+        )
     else:
-        uvicorn.run(app, host=cfg.host, port=cfg.port, log_config=None)
+        uvicorn.run(
+            app,
+            host=cfg.host,
+            port=cfg.port,
+            log_config=None,
+            timeout_keep_alive=cfg.keep_alive_timeout_seconds,
+        )
 
 
 def run_services(