diff --git a/pkg/worker/controllers/worker/model_deployment.go b/pkg/worker/controllers/worker/model_deployment.go
index 1d1153a9a..de0199034 100644
--- a/pkg/worker/controllers/worker/model_deployment.go
+++ b/pkg/worker/controllers/worker/model_deployment.go
@@ -2175,6 +2175,9 @@ func modelDeploymentOwns(obj ctrlcli.Object, md *workercore.ModelDeployment) boo
 }
 
 func modelDeploymentOwnedResource(obj ctrlcli.Object) bool {
+	if obj.GetLabels()[elasticOperationLabel] == "true" || obj.GetLabels()[elasticObservationLabel] == "true" {
+		return true
+	}
 	if !systemmeta.MatchResource(obj, ModelDeploymentResourceType) {
 		return false
 	}
diff --git a/pkg/worker/controllers/worker/model_deployment_elastic_reconcile.go b/pkg/worker/controllers/worker/model_deployment_elastic_reconcile.go
index b3d9576a9..fc0efa5bf 100644
--- a/pkg/worker/controllers/worker/model_deployment_elastic_reconcile.go
+++ b/pkg/worker/controllers/worker/model_deployment_elastic_reconcile.go
@@ -25,7 +25,10 @@ import (
 	"gpustack.ai/gpustack/pkg/worker/elasticengine"
 )
 
-const modelDeploymentElasticWithdrawnAnnotation = "modeldeployment.gpustack.ai/elastic-withdrawn"
+const (
+	modelDeploymentElasticWithdrawnAnnotation = "modeldeployment.gpustack.ai/elastic-withdrawn"
+	elasticObservationLabel                   = "gpustack.ai/elastic-observation"
+)
 
 // The internal observation document preserves independent values and a last proven width.
 // A previous value only selects the next observation. It never authorizes an engine action.
@@ -98,6 +101,7 @@ func (r *ModelDeploymentReconciler) persistElasticObservation(
 		cm = &core.ConfigMap{
 			ObjectMeta: meta.ObjectMeta{
 				Name: modelDeploymentElasticObservationName(md), Namespace: md.Namespace,
+				Labels: map[string]string{elasticObservationLabel: "true"},
 				OwnerReferences: []meta.OwnerReference{{
 					APIVersion: workercore.GroupVersion.String(), Kind: "ModelDeployment",
 					Name: md.Name, UID: md.UID, Controller: elasticControllerRef(),
diff --git a/pkg/worker/controllers/worker/model_deployment_test.go b/pkg/worker/controllers/worker/model_deployment_test.go
index 3c4bccced..39488fbd5 100644
--- a/pkg/worker/controllers/worker/model_deployment_test.go
+++ b/pkg/worker/controllers/worker/model_deployment_test.go
@@ -277,9 +277,13 @@ func TestModelDeploymentOwnedResourceRequiresAResourceNote(t *testing.T) {
 	systemmeta.NoteResource(withoutNote, ModelDeploymentResourceType, nil)
 	wrongType := role.DeepCopy()
 	systemmeta.NoteResource(wrongType, "instances", nil)
+	elasticOperation := &core.ConfigMap{ObjectMeta: meta.ObjectMeta{Labels: map[string]string{elasticOperationLabel: "true"}}}
+	elasticObservation := &core.ConfigMap{ObjectMeta: meta.ObjectMeta{Labels: map[string]string{elasticObservationLabel: "true"}}}
 
 	assert.True(t, modelDeploymentOwnedResource(role))
 	assert.True(t, modelDeploymentOwnedResource(router))
+	assert.True(t, modelDeploymentOwnedResource(elasticOperation))
+	assert.True(t, modelDeploymentOwnedResource(elasticObservation))
 	assert.False(t, modelDeploymentOwnedResource(withoutNote))
 	assert.False(t, modelDeploymentOwnedResource(wrongType))
 }