OCPBUGS-15430: remove Kubernetes API alerting rules

This commit removes the Kubernetes API alerting rules from the
`kubernetes-monitoring-rules` PrometheusRule resource. From now on,
these rules will be managed by the Kubernetes API server operator.

Signed-off-by: Simon Pasquier <[email protected]>

Author: simonpasquier

assets/control-plane/prometheus-rule.yaml

@@ -331,46 +331,6 @@ spec:
       for: 15m
       labels:
         severity: warning
-  - name: kubernetes-system-apiserver
-    rules:
-    - alert: KubeAggregatedAPIErrors
-      annotations:
-        description: Kubernetes aggregated API {{ $labels.instance }}/{{ $labels.name }} has reported {{ $labels.reason }} errors.
-        runbook_url: https://github.com/openshift/runbooks/blob/master/alerts/cluster-monitoring-operator/KubeAggregatedAPIErrors.md
-        summary: Kubernetes aggregated API has reported errors.
-      expr: |
-        sum by(cluster, instance, name, reason)(increase(aggregator_unavailable_apiservice_total{job="apiserver"}[1m])) > 0
-      for: 10m
-      labels:
-        severity: warning
-    - alert: KubeAggregatedAPIDown
-      annotations:
-        description: Kubernetes aggregated API {{ $labels.name }}/{{ $labels.namespace }} has been only {{ $value | humanize }}% available over the last 10m.
-        summary: Kubernetes aggregated API is down.
-      expr: |
-        (1 - max by(name, namespace, cluster)(avg_over_time(aggregator_unavailable_apiservice{job="apiserver"}[10m]))) * 100 < 85
-      for: 15m
-      labels:
-        severity: warning
-    - alert: KubeAPIDown
-      annotations:
-        description: KubeAPI has disappeared from Prometheus target discovery.
-        runbook_url: https://github.com/openshift/runbooks/blob/master/alerts/cluster-monitoring-operator/KubeAPIDown.md
-        summary: Target disappeared from Prometheus target discovery.
-      expr: |
-        absent(up{job="apiserver"} == 1)
-      for: 15m
-      labels:
-        severity: critical
-    - alert: KubeAPITerminatedRequests
-      annotations:
-        description: The kubernetes apiserver has terminated {{ $value | humanizePercentage }} of its incoming requests.
-        summary: The kubernetes apiserver has terminated {{ $value | humanizePercentage }} of its incoming requests.
-      expr: |
-        sum by(cluster) (rate(apiserver_request_terminations_total{job="apiserver"}[10m])) / ( sum by(cluster) (rate(apiserver_request_total{job="apiserver"}[10m])) + sum by(cluster) (rate(apiserver_request_terminations_total{job="apiserver"}[10m])) ) > 0.20
-      for: 5m
-      labels:
-        severity: warning
   - name: kubernetes-system-kubelet
     rules:
     - alert: KubeNodeNotReady

jsonnet/utils/sanitize-rules.libsonnet

@@ -2,7 +2,6 @@ local k8sMixinUtils = import 'github.com/kubernetes-monitoring/kubernetes-mixin/
 
 // List of rule groups which are dropped from the final manifests.
 local excludedRuleGroups = [
-  'kube-apiserver-availability.rules',
   // rules managed by openshift/cluster-kube-controller-manager-operator.
   'kubernetes-system-controller-manager',
   // rules managed by openshift/cluster-kube-scheduler-operator.
@@ -12,6 +11,8 @@ local excludedRuleGroups = [
   'kube-apiserver.rules',
   'kube-apiserver-burnrate.rules',
   'kube-apiserver-histogram.rules',
+  'kube-apiserver-availability.rules',
+  'kubernetes-system-apiserver',
   // Availability of kube-proxy depends on the selected CNO plugin hence the
   // rules should be managed by CNO directly.
   'kubernetes-system-kube-proxy',
@@ -64,15 +65,6 @@ local excludedRules = [
       { alert: 'KubeMemoryQuotaOvercommit' },
     ],
   },
-  {
-    name: 'kubernetes-system-apiserver',
-    rules: [
-      // KubeClientCertificateExpiration alert isn't
-      // actionable because the cluster admin has no way to
-      // prevent a client from using an expird certificate.
-      { alert: 'KubeClientCertificateExpiration' },
-    ],
-  },
   {
     name: 'kubernetes-system-kubelet',
     rules: [
@@ -383,15 +375,6 @@ local patchedRules = [
       },
     ],
   },
-  {
-    name: 'kubernetes-system-apiserver',
-    rules: [
-      {
-        alert: 'KubeAggregatedAPIDown',
-        'for': '15m',
-      },
-    ],
-  },
   {
     name: 'prometheus',
     rules: [
@@ -510,8 +493,6 @@ local includeRunbooks = {
   ClusterMonitoringOperatorDeprecatedConfig: openShiftRunbookCMO('ClusterMonitoringOperatorDeprecatedConfig.md'),
   ClusterOperatorDegraded: openShiftRunbookCMO('ClusterOperatorDegraded.md'),
   ClusterOperatorDown: openShiftRunbookCMO('ClusterOperatorDown.md'),
-  KubeAggregatedAPIErrors: openShiftRunbookCMO('KubeAggregatedAPIErrors.md'),
-  KubeAPIDown: openShiftRunbookCMO('KubeAPIDown.md'),
   KubeDeploymentReplicasMismatch: openShiftRunbookCMO('KubeDeploymentReplicasMismatch.md'),
   KubeJobFailed: openShiftRunbookCMO('KubeJobFailed.md'),
   KubeNodeNotReady: openShiftRunbookCMO('KubeNodeNotReady.md'),

pkg/manifests/manifests.go

@@ -2474,21 +2474,6 @@ func (f *Factory) ControlPlanePrometheusRule() (*monv1.PrometheusRule, error) {
 
 	r.Namespace = f.namespace
 
-	if f.infrastructure.HostedControlPlane() {
-		groups := []monv1.RuleGroup{}
-		for _, g := range r.Spec.Groups {
-			switch g.Name {
-			case "kubernetes-system-apiserver",
-				"kubernetes-system-controller-manager",
-				"kubernetes-system-scheduler":
-				// skip
-			default:
-				groups = append(groups, g)
-			}
-		}
-		r.Spec.Groups = groups
-	}
-
 	return r, nil
 }
 

pkg/manifests/manifests_test.go

@@ -3873,25 +3873,14 @@ func TestPrometheusK8sControlPlaneRulesFiltered(t *testing.T) {
 	tests := []struct {
 		name           string
 		infrastructure InfrastructureReader
-		verify         func(bool)
 	}{
 		{
 			name:           "default config",
 			infrastructure: defaultInfrastructureReader(),
-			verify: func(api bool) {
-				if !api {
-					t.Fatal("did not get all expected kubernetes control plane rules")
-				}
-			},
 		},
 		{
 			name:           "hosted control plane",
 			infrastructure: &fakeInfrastructureReader{highlyAvailableInfrastructure: true, hostedControlPlane: true},
-			verify: func(api bool) {
-				if api {
-					t.Fatalf("kubernetes control plane rules found, none expected")
-				}
-			},
 		},
 	}
 
@@ -3901,14 +3890,13 @@ func TestPrometheusK8sControlPlaneRulesFiltered(t *testing.T) {
 		if err != nil {
 			t.Fatal(err)
 		}
-		apiServerRulesFound := false
+
 		for _, g := range r.Spec.Groups {
 			switch g.Name {
-			case "kubernetes-system-apiserver":
-				apiServerRulesFound = true
+			case "kubernetes-system-apiserver", "kubernetes-system-controller-manager", "kubernetes-system-scheduler":
+				t.Fatalf("Kubernetes control plane rule group %s found, none expected", g.Name)
 			}
 		}
-		tc.verify(apiServerRulesFound)
 	}
 }