From 0d3aef673dd9aefdc43e1d42034e9cb2b00ff79d Mon Sep 17 00:00:00 2001 From: thxCode Date: Tue, 6 Oct 2026 07:47:22 +0800 Subject: [PATCH] fix(worker): let the ConfigMap owner watch enqueue elastic state records The ModelDeployment controller watches ConfigMaps through Owns(), but the ownership predicate only accepted systemmeta resource notes. Elastic operation records carry gpustack.ai/elastic-operation=true and observation records carried no marker at all, so after an operator restart, persisted elastic state changes could not enqueue the owning ModelDeployment and a resize in progress froze (#721). Label observation records with gpustack.ai/elastic-observation=true and accept both elastic state labels in the predicate. Signed-off-by: thxCode --- pkg/worker/controllers/worker/model_deployment.go | 3 +++ .../worker/model_deployment_elastic_reconcile.go | 6 +++++- pkg/worker/controllers/worker/model_deployment_test.go | 4 ++++ 3 files changed, 12 insertions(+), 1 deletion(-) diff --git a/pkg/worker/controllers/worker/model_deployment.go b/pkg/worker/controllers/worker/model_deployment.go index 1d1153a9a..de0199034 100644 --- a/pkg/worker/controllers/worker/model_deployment.go +++ b/pkg/worker/controllers/worker/model_deployment.go @@ -2175,6 +2175,9 @@ func modelDeploymentOwns(obj ctrlcli.Object, md *workercore.ModelDeployment) boo } func modelDeploymentOwnedResource(obj ctrlcli.Object) bool { + if obj.GetLabels()[elasticOperationLabel] == "true" || obj.GetLabels()[elasticObservationLabel] == "true" { + return true + } if !systemmeta.MatchResource(obj, ModelDeploymentResourceType) { return false } diff --git a/pkg/worker/controllers/worker/model_deployment_elastic_reconcile.go b/pkg/worker/controllers/worker/model_deployment_elastic_reconcile.go index b3d9576a9..fc0efa5bf 100644 --- a/pkg/worker/controllers/worker/model_deployment_elastic_reconcile.go +++ b/pkg/worker/controllers/worker/model_deployment_elastic_reconcile.go @@ -25,7 +25,10 @@ import ( "gpustack.ai/gpustack/pkg/worker/elasticengine" ) -const modelDeploymentElasticWithdrawnAnnotation = "modeldeployment.gpustack.ai/elastic-withdrawn" +const ( + modelDeploymentElasticWithdrawnAnnotation = "modeldeployment.gpustack.ai/elastic-withdrawn" + elasticObservationLabel = "gpustack.ai/elastic-observation" +) // The internal observation document preserves independent values and a last proven width. // A previous value only selects the next observation. It never authorizes an engine action. @@ -98,6 +101,7 @@ func (r *ModelDeploymentReconciler) persistElasticObservation( cm = &core.ConfigMap{ ObjectMeta: meta.ObjectMeta{ Name: modelDeploymentElasticObservationName(md), Namespace: md.Namespace, + Labels: map[string]string{elasticObservationLabel: "true"}, OwnerReferences: []meta.OwnerReference{{ APIVersion: workercore.GroupVersion.String(), Kind: "ModelDeployment", Name: md.Name, UID: md.UID, Controller: elasticControllerRef(), diff --git a/pkg/worker/controllers/worker/model_deployment_test.go b/pkg/worker/controllers/worker/model_deployment_test.go index 3c4bccced..39488fbd5 100644 --- a/pkg/worker/controllers/worker/model_deployment_test.go +++ b/pkg/worker/controllers/worker/model_deployment_test.go @@ -277,9 +277,13 @@ func TestModelDeploymentOwnedResourceRequiresAResourceNote(t *testing.T) { systemmeta.NoteResource(withoutNote, ModelDeploymentResourceType, nil) wrongType := role.DeepCopy() systemmeta.NoteResource(wrongType, "instances", nil) + elasticOperation := &core.ConfigMap{ObjectMeta: meta.ObjectMeta{Labels: map[string]string{elasticOperationLabel: "true"}}} + elasticObservation := &core.ConfigMap{ObjectMeta: meta.ObjectMeta{Labels: map[string]string{elasticObservationLabel: "true"}}} assert.True(t, modelDeploymentOwnedResource(role)) assert.True(t, modelDeploymentOwnedResource(router)) + assert.True(t, modelDeploymentOwnedResource(elasticOperation)) + assert.True(t, modelDeploymentOwnedResource(elasticObservation)) assert.False(t, modelDeploymentOwnedResource(withoutNote)) assert.False(t, modelDeploymentOwnedResource(wrongType)) }