fix: recycle inference pods without invalid deadlines

api/v1alpha1/inferenceservice_types.go

@@ -561,18 +561,32 @@ type InferenceServiceSpec struct {
 	// +optional
 	SLO *SLOSpec `json:"slo,omitempty"`
 
-	// MaxPodLifetimeSeconds sets the maximum lifetime (in seconds) for
-	// inference pods. When set, the operator copies this value to
-	// PodSpec.ActiveDeadlineSeconds on the generated Deployment's pod
-	// template, causing Kubernetes to terminate the pod after the
-	// specified duration even if it remains healthy. This is useful for
-	// workloads that need periodic process recycling to release driver
-	// memory (e.g. llama.cpp on AMD Vulkan with pinned GTT memory).
-	// When omitted, pods run indefinitely until manually restarted or
-	// the Deployment is updated.
+	// MaxPodLifetimeSeconds requests best-effort periodic recycling of
+	// deployment-backed inference pods. The controller evicts one expired pod
+	// at a time, using its status start time as the age reference; it is not a
+	// strict deadline and does not set PodSpec.ActiveDeadlineSeconds. This is
+	// useful for workloads that need periodic process recycling to release
+	// driver memory (e.g. llama.cpp on AMD Vulkan with pinned GTT memory).
+	// Eviction respects PodDisruptionBudgets, and when rolloutPolicy.waitForIdle
+	// is set it waits for the backend to go idle first. With a single replica
+	// recycling is a restart, not a rolling replacement: expect a downtime
+	// window while the model reloads. When omitted, pods run indefinitely until
+	// manually restarted or the Deployment is updated.
 	// +kubebuilder:validation:Minimum=1
 	// +optional
 	MaxPodLifetimeSeconds *int64 `json:"maxPodLifetimeSeconds,omitempty"`
+
+	// MaxPodLifetimeIdleTimeoutSeconds bounds how long recycling will wait for
+	// an idle backend before evicting anyway, measured from the moment the pod
+	// exceeded maxPodLifetimeSeconds. It only applies when
+	// rolloutPolicy.waitForIdle is set, which otherwise makes recycling wait
+	// indefinitely — safe for in-flight requests, but a saturated service then
+	// never recycles, which is exactly when leaked driver memory hurts most.
+	// Set 0 to recycle without waiting for idle at all. When omitted, recycling
+	// waits indefinitely.
+	// +kubebuilder:validation:Minimum=0
+	// +optional
+	MaxPodLifetimeIdleTimeoutSeconds *int64 `json:"maxPodLifetimeIdleTimeoutSeconds,omitempty"`
 }
 
 // RolloutPolicySpec defines how deployment updates should be gated on backend idleness.

charts/llmkube/templates/clusterrole.yaml

@@ -57,6 +57,12 @@ rules:
   - patch
   - update
   - watch
+- apiGroups:
+  - ""
+  resources:
+  - pods/eviction
+  verbs:
+  - create
 - apiGroups:
   - apps
   resources:

charts/llmkube/templates/crds/inferenceservices.yaml

@@ -3447,17 +3447,32 @@ spec:
                   Jinja enables Jinja2 chat template rendering for tool/function calling support.
                   Required when using the OpenAI-compatible API with tools. Maps to llama.cpp --jinja flag.
                 type: boolean
+              maxPodLifetimeIdleTimeoutSeconds:
+                description: |-
+                  MaxPodLifetimeIdleTimeoutSeconds bounds how long recycling will wait for
+                  an idle backend before evicting anyway, measured from the moment the pod
+                  exceeded maxPodLifetimeSeconds. It only applies when
+                  rolloutPolicy.waitForIdle is set, which otherwise makes recycling wait
+                  indefinitely — safe for in-flight requests, but a saturated service then
+                  never recycles, which is exactly when leaked driver memory hurts most.
+                  Set 0 to recycle without waiting for idle at all. When omitted, recycling
+                  waits indefinitely.
+                format: int64
+                minimum: 0
+                type: integer
               maxPodLifetimeSeconds:
                 description: |-
-                  MaxPodLifetimeSeconds sets the maximum lifetime (in seconds) for
-                  inference pods. When set, the operator copies this value to
-                  PodSpec.ActiveDeadlineSeconds on the generated Deployment's pod
-                  template, causing Kubernetes to terminate the pod after the
-                  specified duration even if it remains healthy. This is useful for
-                  workloads that need periodic process recycling to release driver
-                  memory (e.g. llama.cpp on AMD Vulkan with pinned GTT memory).
-                  When omitted, pods run indefinitely until manually restarted or
-                  the Deployment is updated.
+                  MaxPodLifetimeSeconds requests best-effort periodic recycling of
+                  deployment-backed inference pods. The controller evicts one expired pod
+                  at a time, using its status start time as the age reference; it is not a
+                  strict deadline and does not set PodSpec.ActiveDeadlineSeconds. This is
+                  useful for workloads that need periodic process recycling to release
+                  driver memory (e.g. llama.cpp on AMD Vulkan with pinned GTT memory).
+                  Eviction respects PodDisruptionBudgets, and when rolloutPolicy.waitForIdle
+                  is set it waits for the backend to go idle first. With a single replica
+                  recycling is a restart, not a rolling replacement: expect a downtime
+                  window while the model reloads. When omitted, pods run indefinitely until
+                  manually restarted or the Deployment is updated.
                 format: int64
                 minimum: 1
                 type: integer

config/crd/bases/inference.llmkube.dev_inferenceservices.yaml

@@ -3443,17 +3443,32 @@ spec:
                   Jinja enables Jinja2 chat template rendering for tool/function calling support.
                   Required when using the OpenAI-compatible API with tools. Maps to llama.cpp --jinja flag.
                 type: boolean
+              maxPodLifetimeIdleTimeoutSeconds:
+                description: |-
+                  MaxPodLifetimeIdleTimeoutSeconds bounds how long recycling will wait for
+                  an idle backend before evicting anyway, measured from the moment the pod
+                  exceeded maxPodLifetimeSeconds. It only applies when
+                  rolloutPolicy.waitForIdle is set, which otherwise makes recycling wait
+                  indefinitely — safe for in-flight requests, but a saturated service then
+                  never recycles, which is exactly when leaked driver memory hurts most.
+                  Set 0 to recycle without waiting for idle at all. When omitted, recycling
+                  waits indefinitely.
+                format: int64
+                minimum: 0
+                type: integer
               maxPodLifetimeSeconds:
                 description: |-
-                  MaxPodLifetimeSeconds sets the maximum lifetime (in seconds) for
-                  inference pods. When set, the operator copies this value to
-                  PodSpec.ActiveDeadlineSeconds on the generated Deployment's pod
-                  template, causing Kubernetes to terminate the pod after the
-                  specified duration even if it remains healthy. This is useful for
-                  workloads that need periodic process recycling to release driver
-                  memory (e.g. llama.cpp on AMD Vulkan with pinned GTT memory).
-                  When omitted, pods run indefinitely until manually restarted or
-                  the Deployment is updated.
+                  MaxPodLifetimeSeconds requests best-effort periodic recycling of
+                  deployment-backed inference pods. The controller evicts one expired pod
+                  at a time, using its status start time as the age reference; it is not a
+                  strict deadline and does not set PodSpec.ActiveDeadlineSeconds. This is
+                  useful for workloads that need periodic process recycling to release
+                  driver memory (e.g. llama.cpp on AMD Vulkan with pinned GTT memory).
+                  Eviction respects PodDisruptionBudgets, and when rolloutPolicy.waitForIdle
+                  is set it waits for the backend to go idle first. With a single replica
+                  recycling is a restart, not a rolling replacement: expect a downtime
+                  window while the model reloads. When omitted, pods run indefinitely until
+                  manually restarted or the Deployment is updated.
                 format: int64
                 minimum: 1
                 type: integer

config/rbac/role.yaml

@@ -39,6 +39,12 @@ rules:
   - patch
   - update
   - watch
+- apiGroups:
+  - ""
+  resources:
+  - pods/eviction
+  verbs:
+  - create
 - apiGroups:
   - ""
   - events.k8s.io

internal/controller/deployment_builder.go

@@ -432,16 +432,15 @@ func (r *InferenceServiceReconciler) constructDeployment(
 					Annotations: buildPodAnnotations(isvc),
 				},
 				Spec: corev1.PodSpec{
-					SecurityContext:       inferPodSecurityContext(isvc, r.DefaultFSGroup),
-					InitContainers:        storageConfig.initContainers,
-					Containers:            []corev1.Container{container},
-					Volumes:               storageConfig.volumes,
-					PriorityClassName:     r.resolvePriorityClassName(isvc),
-					RuntimeClassName:      isvc.Spec.RuntimeClassName,
-					ImagePullSecrets:      isvc.Spec.ImagePullSecrets,
-					EnableServiceLinks:    resolveEnableServiceLinks(backend),
-					ResourceClaims:        modelResourceClaims(model),
-					ActiveDeadlineSeconds: isvc.Spec.MaxPodLifetimeSeconds,
+					SecurityContext:    inferPodSecurityContext(isvc, r.DefaultFSGroup),
+					InitContainers:     storageConfig.initContainers,
+					Containers:         []corev1.Container{container},
+					Volumes:            storageConfig.volumes,
+					PriorityClassName:  r.resolvePriorityClassName(isvc),
+					RuntimeClassName:   isvc.Spec.RuntimeClassName,
+					ImagePullSecrets:   isvc.Spec.ImagePullSecrets,
+					EnableServiceLinks: resolveEnableServiceLinks(backend),
+					ResourceClaims:     modelResourceClaims(model),
 				},
 			},
 		},

internal/controller/inferenceservice_controller.go

@@ -147,6 +147,7 @@ func initContainerSecurityContext(isvc *inferencev1alpha1.InferenceService) *cor
 // +kubebuilder:rbac:groups=core,resources=services,verbs=get;list;watch;create;update;patch;delete
 // +kubebuilder:rbac:groups=discovery.k8s.io,resources=endpointslices,verbs=get;list;watch
 // +kubebuilder:rbac:groups=core,resources=pods,verbs=get;list;watch
+// +kubebuilder:rbac:groups=core,resources=pods/eviction,verbs=create
 // +kubebuilder:rbac:groups=core,resources=persistentvolumeclaims,verbs=get;list;watch;create;update;patch
 // +kubebuilder:rbac:groups=scheduling.k8s.io,resources=priorityclasses,verbs=get;list;watch
 // +kubebuilder:rbac:groups=autoscaling,resources=horizontalpodautoscalers,verbs=get;list;watch;create;update;patch;delete
@@ -248,26 +249,44 @@ func (r *InferenceServiceReconciler) Reconcile(ctx context.Context, req ctrl.Req
 		return finalResult, statusErr
 	}
 
+	lifetimeRequeue, err := r.reconcilePodLifetime(ctx, inferenceService, isMetal, time.Now())
+	if err != nil {
+		return finalResult, err
+	}
+	finalResult.RequeueAfter = earliestPositive(finalResult.RequeueAfter, lifetimeRequeue)
+
 	// When a rollout is deferred pending idle, reconcileRolloutPolicy set
 	// RolloutDeferred=True (persisted by the status update above). Drive a
 	// recheck so the controller notices when the backend goes idle or the
 	// idleTimeoutSeconds budget is spent. Metal-backed services never defer
 	// (no Deployment), so this does not conflict with the metal requeue below.
 	if cond := meta.FindStatusCondition(inferenceService.Status.Conditions, ConditionRolloutDeferred); cond != nil && cond.Status == metav1.ConditionTrue {
-		finalResult.RequeueAfter = inferencev1alpha1.DefaultIdleCheckInterval
+		finalResult.RequeueAfter = earliestPositive(finalResult.RequeueAfter, inferencev1alpha1.DefaultIdleCheckInterval)
 	}
 
 	// On the metal path a heartbeat going stale generates no watch event, so
 	// force a periodic requeue when the Endpoints carry the annotation.
 	if isMetal {
 		if requeue := metalHeartbeatRequeueDuration(metalSnap); requeue > 0 {
-			finalResult.RequeueAfter = requeue
+			finalResult.RequeueAfter = earliestPositive(finalResult.RequeueAfter, requeue)
 		}
 	}
 
 	return finalResult, nil
 }
 
+// earliestPositive merges the requeue timers Reconcile collects without
+// allowing a zero timer (which means "no requeue") to win a plain minimum.
+func earliestPositive(values ...time.Duration) time.Duration {
+	var earliest time.Duration
+	for _, value := range values {
+		if value > 0 && (earliest == 0 || value < earliest) {
+			earliest = value
+		}
+	}
+	return earliest
+}
+
 func (r *InferenceServiceReconciler) getModelForInferenceService(ctx context.Context, isvc *inferencev1alpha1.InferenceService) (*inferencev1alpha1.Model, bool, *ctrl.Result, error) {
 	log := logf.FromContext(ctx)
 

internal/controller/inferenceservice_deployment_test.go

@@ -594,22 +594,9 @@ var _ = Describe("Multi-GPU Deployment Construction", func() {
 			}
 		})
 
-		It("should leave ActiveDeadlineSeconds nil when maxPodLifetimeSeconds is unset", func() {
-			isvc := &inferencev1alpha1.InferenceService{
-				ObjectMeta: metav1.ObjectMeta{
-					Name:      "lifetime-service",
-					Namespace: "default",
-				},
-				Spec: inferencev1alpha1.InferenceServiceSpec{
-					ModelRef: "lifetime-model",
-					Image:    "ghcr.io/ggml-org/llama.cpp:server",
-				},
-			}
-			deployment := reconciler.constructDeployment(isvc, model, 1)
-			Expect(deployment.Spec.Template.Spec.ActiveDeadlineSeconds).To(BeNil())
-		})
-
-		It("should set ActiveDeadlineSeconds on the pod template when maxPodLifetimeSeconds is set", func() {
+		// Recycling is driven by the controller, not the pod template: the
+		// field must stay nil so the Deployment remains updatable.
+		It("should leave ActiveDeadlineSeconds nil when maxPodLifetimeSeconds is set", func() {
 			lifetime := int64(3600)
 			isvc := &inferencev1alpha1.InferenceService{
 				ObjectMeta: metav1.ObjectMeta{
@@ -623,8 +610,7 @@ var _ = Describe("Multi-GPU Deployment Construction", func() {
 				},
 			}
 			deployment := reconciler.constructDeployment(isvc, model, 1)
-			Expect(deployment.Spec.Template.Spec.ActiveDeadlineSeconds).NotTo(BeNil())
-			Expect(*deployment.Spec.Template.Spec.ActiveDeadlineSeconds).To(Equal(int64(3600)))
+			Expect(deployment.Spec.Template.Spec.ActiveDeadlineSeconds).To(BeNil())
 		})
 	})