feat(operator): Added LokistackPendingComponents alert

Author: JoaoBraveCoding

operator/docs/lokistack/sop.md

@@ -365,3 +365,48 @@ The schema configuration does not contain the most recent schema version and nee
 ### Steps
 
 - Add a new object storage schema V13 with a future EffectiveDate
+
+## Lokistack Components Not Ready
+
+### Impact
+
+One or more LokiStack components are not ready, which can disrupt ingestion or querying and lead to degraded service.
+
+### Summary
+
+The LokiStack reports that some components have not reached the `Ready` state. This might be related to Kubernetes resources (Pods/Deployments), configuration, or external dependencies.
+
+### Severity
+
+`Critical`
+
+### Access Required
+
+- Console access to the cluster
+- Edit or view access in the namespace where the LokiStack is deployed:
+  - OpenShift
+    - `openshift-logging` (LokiStack)
+
+### Steps
+
+- Inspect the LokiStack conditions and events
+  - Describe the LokiStack resource and review status conditions:
+    - `kubectl -n <namespace> describe lokistack <name>`
+  - Check for conditions that would lead to some pods not being in the `Ready` state
+- Check operator and reconciliation status
+  - Ensure the Loki Operator is running and not reporting errors:
+    - `kubectl -n <operator-namespace> logs deploy/loki-operator`
+  - Look for reconcile errors related to missing permissions, invalid fields, or failed rollouts.
+- Verify component Pods and Deployments
+  - Ensure all core components are running and Ready in the LokiStack namespace:
+    - `distributor`, `ingester`, `querier`, `query-frontend`, `index-gateway`, `compactor`, `gateway`
+  - Check Pod readiness and recent restarts:
+    - `kubectl -n <namespace> get pods`
+    - `kubectl -n <namespace> describe pod <pod>`
+- Examine Kubernetes events for failures
+  - `kubectl -n <namespace> get events --sort-by=.lastTimestamp`
+  - Common causes: image pull backoffs, failed mounts, readiness probe failures, or insufficient resources
+- Validate configuration and referenced resources
+  - Confirm referenced `Secrets`, `ConfigMaps`, exist and have correct keys
+- Look into the Pod logs of the component that still not `Ready`:
+    - `kubectl -n <namespace> logs <pod>`

operator/internal/manifests/internal/alerts/prometheus-alerts.yaml

@@ -209,3 +209,14 @@ groups:
     for: 1m
     labels:
       severity: warning
+  - alert: LokistackComponentsNotReady
+    annotations:
+      description: |-
+        The LokiStack "{{ $labels.stack_name }}" in namespace "{{ $labels.stack_namespace }}" has components that are not ready.
+      summary: "One or more LokiStack components are not ready."
+      runbook_url: "[[ .RunbookURL ]]#Lokistack-Pending-Components"
+    expr: |
+      sum  (lokistack_status_condition{reason="ReadyComponents", status="false"}) by (stack_name, stack_namespace, reason, status) == 1
+    for: 10m
+    labels:
+      severity: critical

operator/internal/manifests/internal/alerts/testdata/test.yaml

@@ -66,6 +66,9 @@ tests:
       - series: 'loki_discarded_samples_total{namespace="my-ns", tenant="application", reason="line_too_long"}'
         values: '0x5 0+120x25 3000'
 
+      - series: 'lokistack_status_condition{stack_name="mystack", stack_namespace="my-ns", reason="ReadyComponents", status="false"}'
+        values: '0+0x15 1+0x10'
+
     alert_rule_test:
       - eval_time: 16m
         alertname: LokiRequestErrors
@@ -194,3 +197,16 @@ tests:
                 Samples are discarded because of "line_too_long" at a rate of 2 samples per second.
               summary: Loki is discarding samples during ingestion because they fail validation.
               runbook_url: "[[ .RunbookURL]]#Loki-Discarded-Samples-Warning"
+      - eval_time: 26m
+        alertname: LokistackComponentsNotReady
+        exp_alerts:
+          - exp_labels:
+              stack_namespace: my-ns
+              stack_name: mystack
+              reason: ReadyComponents
+              status: "false"
+              severity: critical
+            exp_annotations:
+              description: 'The LokiStack "mystack" in namespace "my-ns" has components that are not ready.'
+              summary: "One or more LokiStack components are not ready."
+              runbook_url: "[[ .RunbookURL ]]#Lokistack-Pending-Components"