telemetry: auto-detect push subscription support, opt-out exclusions

.gitignore

@@ -50,9 +50,11 @@ dashboards/redfish-telemetry/redfish-telemetry
 # percli intermediate output directories (dashboards/built/*.json are the canonical outputs)
 dashboards/built/*/
 
-# Local server lists for hack/verify-test-events contain real BMC credentials.
+# Local server lists and configs for hack/verify-test-events may contain real BMC credentials.
+hack/verify-test-events/servers.yaml
 hack/verify-test-events/servers.local.yaml
 hack/verify-test-events/*.local.yaml
+hack/verify-test-events/config.yaml
 plugins
 hack/verify-test-events/servers.yaml
 hack/verify-test-events/config.yaml

cmd/main.go

@@ -12,8 +12,6 @@ import (
 	"time"
 
 	"github.com/ironcore-dev/metal-maintenance-operator/internal/cli"
-	"github.com/ironcore-dev/metal-maintenance-operator/internal/configloader"
-	"github.com/ironcore-dev/metal-maintenance-operator/internal/controller/config"
 	"github.com/ironcore-dev/metal-maintenance-operator/internal/discovery"
 	"github.com/ironcore-dev/metal-maintenance-operator/internal/ignition"
 	"github.com/ironcore-dev/metal-maintenance-operator/internal/server"
@@ -59,6 +57,7 @@ import (
 	// +kubebuilder:scaffold:imports
 )
 
+const maxRepositoryPassesLimit = 5
 const maxFailedAutoRetryCount = 10
 
 var (
@@ -104,6 +103,8 @@ func main() {
 	var resyncInterval time.Duration
 	var biosSettingsTimeoutExpiry time.Duration
 	var rebootTimeoutExpiry time.Duration
+	var scheduledJobPowerOnTimeout time.Duration
+	var maxRepositoryPasses int
 	flag.StringVar(&metricsAddr, "metrics-bind-address", "0", "The address the metrics endpoint binds to. "+
 		"Use :8443 for HTTPS or :8080 for HTTP, or leave as 0 to disable the metrics service.")
 	flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.")
@@ -155,29 +156,15 @@ func main() {
 		"Timeout for BIOS settings application before the task is considered expired.")
 	flag.DurationVar(&rebootTimeoutExpiry, "reboot-timeout-expiry", 10*time.Minute,
 		"Timeout waiting for a server to complete a reboot/power-cycle before the task is considered expired.")
+	flag.IntVar(&maxRepositoryPasses, "max-repository-passes", maxRepositoryPassesLimit,
+		"Maximum number of check->apply->track->recheck passes a FirmwareUpdate may take before being marked Failed.")
 	var minBMCResetInterval time.Duration
 	flag.DurationVar(&minBMCResetInterval, "min-bmc-reset-interval", 24*time.Hour,
 		"Minimum time that must have passed since a BMC's last completed reset to reissue another reset.")
-
-	// FirmwareUpdate ConfigMap flags. When --firmwareupdate-config-name is
-	// set, a live-reloading configloader.Loader watches the named ConfigMap
-	// and the FirmwareUpdateReconciler prefers its values over its own
-	// built-in defaults (ResyncInterval, DefaultFailedAutoRetryCount,
-	// MaxRepositoryPasses, MinBMCResetInterval, ScheduledJobPowerOnTimeout,
-	// RepositoryCheckInterval) for any field the ConfigMap sets, falling back
-	// to those defaults otherwise. Empty (the default) disables the feature
-	// entirely, leaving the FirmwareUpdate controller on its built-in
-	// defaults.
-	var (
-		firmwareUpdateConfigName      string
-		firmwareUpdateConfigNamespace string
-	)
-	flag.StringVar(&firmwareUpdateConfigName, "firmwareupdate-config-name", "",
-		"Name of the ConfigMap holding FirmwareUpdate controller overrides. Leave empty (the default) to disable "+
-			"ConfigMap-based overrides.")
-	flag.StringVar(&firmwareUpdateConfigNamespace, "firmwareupdate-config-namespace", "",
-		"Namespace of the FirmwareUpdate ConfigMap. Defaults to POD_NAMESPACE. Ignored when "+
-			"--firmwareupdate-config-name is empty.")
+	flag.DurationVar(&scheduledJobPowerOnTimeout, "scheduled-job-power-on-timeout", 30*time.Minute,
+		"How long a firmware upgrade job may remain in the \"Scheduled\" state (queued, awaiting a reboot/power "+
+			"cycle to execute) with the server powered off and no observed progress, before the controller powers the "+
+			"server back on to let the job proceed. A value <= 0 disables this check.")
 
 	// Telemetry collector flags. The whole pipeline is gated by
 	// --enable-telemetry — when off (the default), zero telemetry
@@ -228,6 +215,13 @@ func main() {
 
 	ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts)))
 
+	if maxRepositoryPasses < 1 || maxRepositoryPasses > maxRepositoryPassesLimit {
+		setupLog.Error(
+			nil, "Failed to start FirmwareUpdate controller: invalid --max-repository-passes value",
+			"value", maxRepositoryPasses, "min", 1, "max", maxRepositoryPassesLimit)
+		os.Exit(1)
+	}
+
 	if defaultFailedAutoRetryCountInt < 0 || defaultFailedAutoRetryCountInt > maxFailedAutoRetryCount {
 		setupLog.Error(nil, "Failed to start FirmwareUpdate controller: invalid --default-failed-auto-retry-count value",
 			"value", defaultFailedAutoRetryCountInt, "min", 0, "max", maxFailedAutoRetryCount)
@@ -263,16 +257,6 @@ func main() {
 		}
 	}
 
-	if firmwareUpdateConfigName != "" && firmwareUpdateConfigNamespace == "" {
-		firmwareUpdateConfigNamespace = os.Getenv("POD_NAMESPACE")
-		if firmwareUpdateConfigNamespace == "" {
-			setupLog.Error(nil,
-				"FirmwareUpdate ConfigMap namespace resolution failed: neither "+
-					"--firmwareupdate-config-namespace nor POD_NAMESPACE is set")
-			os.Exit(1)
-		}
-	}
-
 	// if the enable-http2 flag is false (the default), http/2 should be disabled
 	// due to its vulnerabilities. More specifically, disabling http/2 will
 	// prevent from being vulnerable to the HTTP/2 Stream Cancellation and
@@ -351,7 +335,6 @@ func main() {
 				splitAndTrim(configMapCacheNamespaces),
 				namedConfigMapRefs(
 					enableTelemetry, telemetryConfigNamespace, telemetryConfigName,
-					firmwareUpdateConfigNamespace, firmwareUpdateConfigName,
 				)...,
 			),
 		},
@@ -543,34 +526,19 @@ func main() {
 		os.Exit(1)
 	}
 
-	var firmwareUpdateConfigLoader *configloader.Loader[config.FirmwareUpdateConfig]
-	if firmwareUpdateConfigName != "" && firmwareUpdateConfigNamespace != "" {
-		firmwareUpdateConfigLoader = &configloader.Loader[config.FirmwareUpdateConfig]{
-			Cache:         mgr.GetCache(),
-			Client:        mgr.GetClient(),
-			Namespace:     firmwareUpdateConfigNamespace,
-			Name:          firmwareUpdateConfigName,
-			Parse:         config.ParseFirmwareUpdateConfig,
-			Log:           ctrl.Log.WithName("firmwareupdate-config-loader"),
-			EventRecorder: mgr.GetEventRecorder("metal-maintenance-operator"),
-		}
-		if err := mgr.Add(firmwareUpdateConfigLoader); err != nil {
-			setupLog.Error(err, "Unable to register FirmwareUpdate ConfigMap loader")
-			os.Exit(1)
-		}
-		setupLog.Info("Enabled FirmwareUpdate ConfigMap overrides",
-			"configMap", firmwareUpdateConfigNamespace+"/"+firmwareUpdateConfigName)
-	}
-
 	if err = (&systemctrl.FirmwareUpdateReconciler{
-		Client:             mgr.GetClient(),
-		ManagerNamespace:   managerNamespace,
-		DefaultProtocol:    protocol,
-		SkipCertValidation: skipCertValidation,
-		Scheme:             mgr.GetScheme(),
-		Conditions:         accessor,
-		BMCOptions:         bmcOpts,
-		ConfigLoader:       firmwareUpdateConfigLoader,
+		Client:                      mgr.GetClient(),
+		ManagerNamespace:            managerNamespace,
+		DefaultProtocol:             protocol,
+		SkipCertValidation:          skipCertValidation,
+		Scheme:                      mgr.GetScheme(),
+		ResyncInterval:              resyncInterval,
+		Conditions:                  accessor,
+		BMCOptions:                  bmcOpts,
+		DefaultFailedAutoRetryCount: int32(defaultFailedAutoRetryCountInt),
+		MaxRepositoryPasses:         int32(maxRepositoryPasses),
+		MinBMCResetInterval:         minBMCResetInterval,
+		ScheduledJobPowerOnTimeout:  scheduledJobPowerOnTimeout,
 	}).SetupWithManager(mgr); err != nil {
 		setupLog.Error(err, "Unable to create FirmwareUpdate controller")
 		os.Exit(1)
@@ -667,31 +635,47 @@ func main() {
 	// +kubebuilder:scaffold:builder
 
 	if enableTelemetry {
-		if err := telemetryruntime.AddTo(mgr, telemetryruntime.Options{
+		telemetryOpts := telemetryruntime.Options{
 			ConfigName:                 telemetryConfigName,
 			ConfigNamespace:            telemetryConfigNamespace,
 			ReceiverURL:                telemetryReceiverURL,
 			EventsAddr:                 telemetryEventsAddr,
 			InsecureTLS:                telemetryInsecureTLS,
 			SubscriberID:               telemetrySubscriberID,
 			EnableCriticalEventHandler: telemetryEnableCriticalHandler,
-		}); err != nil {
+		}
+		if err := telemetryruntime.AddTo(mgr, &telemetryOpts); err != nil {
 			setupLog.Error(err, "Failed to add telemetry pipeline")
 			os.Exit(1)
 		}
 		setupLog.Info("Enabled telemetry pipeline",
 			"configMap", telemetryConfigNamespace+"/"+telemetryConfigName,
 			"receiverURL", telemetryReceiverURL)
-	}
 
-	// BMC service-discovery endpoint.
-	sdHandler := &discovery.Handler{
-		Client: mgr.GetClient(),
-		Log:    ctrl.Log.WithName("discovery"),
-	}
-	if err := mgr.AddMetricsServerExtraHandler(discovery.Path, sdHandler); err != nil {
-		setupLog.Error(err, "Failed to register BMC service discovery handler", "path", discovery.Path)
-		os.Exit(1)
+		// BMC service-discovery endpoint: omit BMCs with active push
+		// subscriptions so the exporter only scrapes what the push pipeline
+		// hasn't claimed.
+		sdHandler := &discovery.Handler{
+			Client:                 mgr.GetClient(),
+			Log:                    ctrl.Log.WithName("discovery"),
+			SubscribedBMCs:         telemetryOpts.ActiveSubscriptions,
+			UnsubscribedBMCReasons: telemetryOpts.UnsubscribedBMCReasons,
+		}
+		if err := mgr.AddMetricsServerExtraHandler(discovery.Path, sdHandler); err != nil {
+			setupLog.Error(err, "Failed to register BMC service discovery handler", "path", discovery.Path)
+			os.Exit(1)
+		}
+	} else {
+		// Telemetry disabled: register the SD endpoint without subscription
+		// filtering — the exporter scrapes all BMCs.
+		sdHandler := &discovery.Handler{
+			Client: mgr.GetClient(),
+			Log:    ctrl.Log.WithName("discovery"),
+		}
+		if err := mgr.AddMetricsServerExtraHandler(discovery.Path, sdHandler); err != nil {
+			setupLog.Error(err, "Failed to register BMC service discovery handler", "path", discovery.Path)
+			os.Exit(1)
+		}
 	}
 	if metricsAddr == "0" || metricsAddr == "" {
 		setupLog.Info("BMC service discovery registered but metrics server is disabled",
@@ -733,15 +717,11 @@ type namedConfigMapRef struct {
 // "no restriction needed" for that entry).
 func namedConfigMapRefs(
 	telemetryEnabled bool, telemetryNS, telemetryName string,
-	firmwareUpdateConfigNS, firmwareUpdateConfigName string,
 ) []namedConfigMapRef {
 	var refs []namedConfigMapRef
 	if telemetryEnabled && telemetryNS != "" && telemetryName != "" {
 		refs = append(refs, namedConfigMapRef{Namespace: telemetryNS, Name: telemetryName})
 	}
-	if firmwareUpdateConfigName != "" && firmwareUpdateConfigNS != "" {
-		refs = append(refs, namedConfigMapRef{Namespace: firmwareUpdateConfigNS, Name: firmwareUpdateConfigName})
-	}
 	return refs
 }
 

dist/chart/templates/extras/telemetry-configmap.yaml

@@ -17,8 +17,14 @@ data:
     {{- with .Values.telemetry.config.testEventTimeout }}
     testEventTimeout: {{ . }}
     {{- end }}
-    {{- if .Values.telemetry.config.eventBasedHardware }}
-    eventBasedHardware:
-{{ .Values.telemetry.config.eventBasedHardware | toYaml | indent 6 }}
+    {{- if .Values.telemetry.config.excludeHardware }}
+    excludeHardware:
+{{ .Values.telemetry.config.excludeHardware | toYaml | indent 6 }}
+    {{- end }}
+    {{- if .Values.telemetry.config.testEventOverrides }}
+    testEventOverrides:
+      {{- range $key := (.Values.telemetry.config.testEventOverrides | keys | sortAlpha) }}
+      - {{ get $.Values.telemetry.config.testEventOverrides $key | toYaml | trimSuffix "\n" | nindent 10 | trimPrefix "\n          " }}
+      {{- end }}
     {{- end }}
 {{- end }}

dist/chart/values.yaml

@@ -303,24 +303,41 @@ telemetry:
     ## Timeout for the health check round-trip. Bounds: 5s – 5m.
     ##
     testEventTimeout: ""
-    ## Vendor/model rows that opt into event-based subscription.
+    ## Vendor/model rows that opt OUT of event-based subscription (exclusions).
     ## See config/telemetry/configmap.yaml for full field documentation.
     ##
-    eventBasedHardware: []
-    ## Example:
-    # eventBasedHardware:
-    #   - vendor: "Dell Inc."
-    #     models: ["PowerEdge R760", "PowerEdge R660"]
-    #     testMessageId: "FAN0001"
-    #     testSeverity: "Informational"
-    #   - vendor: "HPE"
-    #     models: ["*"]
-    #     testMessageId: "iLOEvents.1.0.TestAlert"
-    #     testSeverity: "OK"
-    #     testOriginOfCondition: "/redfish/v1/Managers/1"
-    #   - vendor: "Lenovo"
-    #     models: ["*"]
-    #     testMessageId: "EventRegistry.1.0.0.FQXSPCA0012I"
+    excludeHardware: []
+    ## Named map of vendor/model rows that opt into event-based subscription.
+    ## Keys are arbitrary identifiers used only for Helm merge; they do not
+    ## appear in the rendered ConfigMap.  Because Helm merges maps (unlike
+    ## lists, which are replaced), end-users can add entries in their own
+    ## values.yaml without repeating the defaults:
+    ##
+    ##   testEventOverrides:
+    ##     dell-r840:
+    ##       vendor: "Dell Inc."
+    ##       models: ["PowerEdge R840"]
+    ##       testMessageId: "Storage.1.0.DriveFailed"
+    ##       testSeverity: "Informational"
+    ##
+    ## See config/telemetry/configmap.yaml for full field documentation.
+    ##
+    testEventOverrides:
+      dell:
+        vendor: "Dell Inc."
+        models: ["*"]
+        testMessageId: "FAN.1.0.FAN0001"
+        testSeverity: "Informational"
+      hpe:
+        vendor: "HPE"
+        models: ["*"]
+        testMessageId: "iLOEvents.1.0.TestAlert"
+        testSeverity: "OK"
+        testOriginOfCondition: "/redfish/v1/Managers/1"
+      lenovo:
+        vendor: "Lenovo"
+        models: ["*"]
+        testMessageId: "EventRegistry.1.0.0.FQXSPCA0012I"
 
 ## Additional Secret resources to render in the release namespace.
 ## Useful for operator-pattern credentials (e.g. Gardener token-requestor secrets).

docs/concepts/biossettings.md

@@ -18,21 +18,18 @@ checks, maintenance, and post-apply verification.
 - `status.state` reflects the overall lifecycle: `Pending`, `InProgress`, `Applied`, or `Failed`. Each entry in
   `status.flowState[]` mirrors this per settings-flow step, so a stuck rollout can be pinpointed to the exact step.
 - `spec.retryPolicy.maxAttempts` bounds automatic retries after a transient failure.
-- `spec.readinessGates`/`spec.completionConditionType` (optional) let this object participate in a manual
-  cross-resource sequencing chain — see [Readiness Gates](readiness-gates.md).
 
 ## Workflow
 
 1. The controller validates `spec.settingsFlow` for duplicate step names and duplicate setting keys across steps.
-2. It waits until the server's BIOS version matches `spec.version`, and until `spec.readinessGates` (if set) are
-   satisfied on the server (see [Readiness Gates](readiness-gates.md)), then requests (or reuses)
-   `ServerMaintenance` for the server per `spec.serverMaintenancePolicy`.
+2. It waits until the server's BIOS version matches `spec.version`, then requests (or reuses) `ServerMaintenance`
+   for the server per `spec.serverMaintenancePolicy`.
 3. Once the server is in maintenance, the controller applies each `settingsFlow` step in priority order, powering
    the server off/on when a reboot is required to make settings take effect.
 4. After each step is applied, the controller verifies it against the live BIOS settings before advancing to the
    next step, recording progress in `status.flowState[]`.
-5. Once all steps are verified, `status.state` becomes `Applied` (patching `spec.completionConditionType` `True`
-   on the server, if set); on unrecoverable failure, `Failed` (subject to `spec.retryPolicy`).
+5. Once all steps are verified, `status.state` becomes `Applied`; on unrecoverable failure, `Failed` (subject to
+   `spec.retryPolicy`).
 6. On deletion, the controller cleans up the `ServerMaintenance` it requested and removes its finalizer once no
    maintenance for this object is still in progress.
 

docs/concepts/biossettingsset.md

@@ -12,10 +12,6 @@ by creating and managing one child [`BIOSSettings`](biossettings.md) per selecte
   children whose server no longer matches the selector (once the child is no longer `InProgress`).
 - `status` aggregates rollout progress across all children: `fullyLabeledServers`, `availableBIOSSettings`,
   `pendingBIOSSettings`, `inProgressBIOSSettings`, `completedBIOSSettings`, and `failedBIOSSettings`.
-- `spec.biosSettingsTemplate.readinessGates`/`completionConditionType` (optional) are propagated as-is to every
-  child, so the whole fleet can participate in a manual [Readiness Gates](readiness-gates.md) chain; since each
-  child targets a distinct `Server`, using the same `completionConditionType` across all children is safe (no
-  collision, as each is patched onto that child's own server).
 
 ## Workflow
 

docs/concepts/biosversion.md

@@ -16,21 +16,17 @@ served from an external location.
   BMC-reported upgrade task (state, status, percent complete).
 - `status.state` reflects the overall lifecycle: `Pending`, `InProgress`, `Completed`, or `Failed`.
 - `spec.retryPolicy.maxAttempts` bounds automatic retries after a transient failure.
-- `spec.readinessGates`/`spec.completionConditionType` (optional) let this object participate in a manual
-  cross-resource sequencing chain — see [Readiness Gates](readiness-gates.md).
 
 ## Workflow
 
-1. If `spec.readinessGates` is set, the controller waits until those conditions are satisfied on the server before
-   proceeding (see [Readiness Gates](readiness-gates.md)).
-2. The controller requests (or reuses) `ServerMaintenance` for the server per `spec.serverMaintenancePolicy`.
-3. Once the server is in maintenance, it issues the firmware upgrade using `spec.image`, then powers the server
+1. The controller requests (or reuses) `ServerMaintenance` for the server per `spec.serverMaintenancePolicy`.
+2. Once the server is in maintenance, it issues the firmware upgrade using `spec.image`, then powers the server
    off/on as needed to complete the upgrade.
-4. It polls the BMC-reported task in `status.upgradeTask` until it completes, fails, or times out
+3. It polls the BMC-reported task in `status.upgradeTask` until it completes, fails, or times out
    (`spec.rebootTimeoutExpiry` bounds the reboot wait, configured at the operator level).
-5. On success, `status.state` becomes `Completed` (patching `spec.completionConditionType` `True` on the server, if
-   set); on unrecoverable failure, `Failed` (subject to `spec.retryPolicy`).
-6. On deletion, the controller cleans up the `ServerMaintenance` it requested and removes its finalizer once the
+4. On success, `status.state` becomes `Completed`; on unrecoverable failure, `Failed` (subject to
+   `spec.retryPolicy`).
+5. On deletion, the controller cleans up the `ServerMaintenance` it requested and removes its finalizer once the
    upgrade is no longer in progress.
 
 ## Example

docs/concepts/biosversionset.md

@@ -12,10 +12,6 @@ by creating and managing one child [`BIOSVersion`](biosversion.md) per selected
   children whose server no longer matches the selector (once the child is no longer `InProgress`).
 - `status` aggregates rollout progress across all children: `fullyLabeledServers`, `availableBIOSVersion`,
   `pendingBIOSVersion`, `inProgressBIOSVersion`, `completedBIOSVersion`, and `failedBIOSVersion`.
-- `spec.biosVersionTemplate.readinessGates`/`completionConditionType` (optional) are propagated as-is to every
-  child, so the whole fleet can participate in a manual [Readiness Gates](readiness-gates.md) chain; since each
-  child targets a distinct `Server`, using the same `completionConditionType` across all children is safe (no
-  collision, as each is patched onto that child's own server).
 
 ## Workflow