Fix discrepancies concerning rule generations

Signed-off-by: Kemal Akkoyun <[email protected]>

Author: kakkoyun

README.md

@@ -1,15 +1,19 @@
 # observatorium/configuration
+
 This projects holds all the configuration files for our internal Observatorium deployments.
 
-## Installing jsonnet dependencies.
+## Installing jsonnet dependencies
+
 To install all dependencies:
-```
+
+```console
 jb install
 #installs pinned versions from `jsonnetfile.lock.json` file.
 ```
 
 To update a dependency:
-```
+
+```console
 jb update https://github.com/thanos-io/kube-thanos
 #updates `kube-thanos` to master and sets the new hash in `jsonnetfile.lock.json`.
 
@@ -18,35 +22,46 @@ jb update
 ```
 
 ## Grafana dashboards
+
 All dashboards are generated in `manifests/production/grafana` with:
-```
+
+```console
 make grafana
 ```
 
 **Staging**: deploys on every commit master.
 
 **Production**: update the commit hash ref in [`https://gitlab.cee.redhat.com/service/app-interface/blob/master/data/services/observability/cicd/saas/saas-grafana.yaml`](https://gitlab.cee.redhat.com/service/app-interface/blob/master/data/services/observability/cicd/saas/saas-grafana.yaml)
 
+## Prometheus Rules
+
+Use `synchronize.sh` to create a MR against `app-interface` to update dashboards.
+
 ## Components - Deployments, ServiceMonitors, ConfigMaps etc...
+
 All components manifests are generated in `manifests/production/` with:
-```
+
+```console
 make manifests
 ```
+
 **Staging**: deploys on every commit master.
 
 **Production**: update the commit hash ref in [`https://gitlab.cee.redhat.com/service/app-interface/blob/master/data/services/telemeter/cicd/saas.yaml`](https://gitlab.cee.redhat.com/service/app-interface/blob/master/data/services/telemeter/cicd/saas.yaml)
 
-
 ## CI Jobs
-Jobs runs are posted in:<br/>
-`#sd-app-sre-info` for grafana dashboards <br/>
-and <br/>
-`#team-monitoring-info` for everyrhing else.
+
+Jobs runs are posted in:
+
+`#sd-app-sre-info` for grafana dashboards
+
+and
+
+`#team-monitoring-info` for everything else.
 
 ## Troubleshooting
+
 1. Enable port forwarding for a user - [example](
 https://gitlab.cee.redhat.com/service/app-interface/-/blob/ee91aac666ee39a273332c59ad4bdf7e0f50eeba/data/teams/telemeter/users/fbranczy.yml#L14)
 2. Add a pod name to the allowed list for port forwarding - [example](
 https://gitlab.cee.redhat.com/service/app-interface/-/blob/ee91aac666ee39a273332c59ad4bdf7e0f50eeba/resources/app-sre/telemeter-production/observatorium-allow-port-forward.role.yaml#L10)
-
-

prometheusrules.jsonnet

@@ -29,7 +29,17 @@ local absent(name, job) = {
 
 // Add dashboards and runbook anntotations.
 // Overwrite severity to medium and high.
-local appSREOverwrites(prometheusAlerts, namespace) = {
+local appSREOverwrites(namespace) = {
+  local environment = std.split(namespace, '-')[1],
+  local dashboardDatasource = function(environment) {
+    datasource:
+      if
+        environment == 'stage' then 'app-sre-stage-01-prometheus'
+      else if
+        environment == 'production' then 'telemeter-prod-01-prometheus'
+      else error 'no datasource for environment %s' % environment,
+  },
+
   local dashboardID = function(name) {
     id:
       if
@@ -57,34 +67,54 @@ local appSREOverwrites(prometheusAlerts, namespace) = {
       else error 'no dashboard id for group %s' % name,
   },
 
-  local setSeverity = function(label, alertName) {
-    label: if std.startsWith(alertName, 'Loki') then 'info'
-    else if label == 'critical' then
-      // For thanos page only for `ThanosNoRuleEvaluations`.
-      if std.startsWith(alertName, 'Thanos') then
-        if alertName != 'ThanosNoRuleEvaluations' then 'high' else label
-      else label
-    else if label == 'warning' then 'medium'
-    else 'high',
+  local setSeverity = function(label, environment, alertName) {
+    label:
+      if
+        std.startsWith(alertName, 'Loki') then 'info'
+      else if
+        label == 'critical' then
+        if
+          environment == 'stage' then 'high'
+        else if
+          // For thanos, page only for `ThanosNoRuleEvaluations`.
+          std.startsWith(alertName, 'Thanos') && alertName != 'ThanosNoRuleEvaluations' then 'high'
+        else label
+      else if
+        label == 'warning' then 'medium'
+      else 'high',
   },
 
   groups: [
     g {
       rules: [
         if std.objectHas(r, 'alert') then
           r {
-            annotations+: {
-              runbook: 'https://gitlab.cee.redhat.com/observatorium/configuration/blob/master/docs/sop/observatorium.md#%s' % std.asciiLower(r.alert),
-              dashboard: if std.startsWith(g.name, 'telemeter') then 'https://grafana.app-sre.devshift.net/d/Tg-mH0rizaSJDKSADJ/telemeter?orgId=1&refresh=1m&var-datasource=telemeter-prod-01-prometheus'
-              else 'https://grafana.app-sre.devshift.net/d/%s/%s?orgId=1&refresh=10s&var-datasource=app-sre-prometheus&var-namespace=%s&var-job=All&var-pod=All&var-interval=5m' % [
-                dashboardID(g.name).id,
-                g.name,
-                namespace,
-              ],
-            },
+            annotations+:
+              {
+                // Message is a required field. Upstream thanos-mixin doesn't have it.
+                message: if std.objectHasAll(self, 'description') then self.description else r.annotations.message,
+              } +
+              if std.startsWith(g.name, 'telemeter') then
+                {
+                  runbook: 'https://gitlab.cee.redhat.com/observatorium/configuration/blob/master/docs/sop/telemeter.md#%s' % std.asciiLower(r.alert),
+                  dashboard: 'https://grafana.app-sre.devshift.net/d/%s/telemeter?orgId=1&refresh=1m&var-datasource=%s' % [
+                    dashboardID(g.name).id,
+                    dashboardDatasource(environment).datasource,
+                  ],
+                }
+              else
+                {
+                  runbook: 'https://gitlab.cee.redhat.com/observatorium/configuration/blob/master/docs/sop/observatorium.md#%s' % std.asciiLower(r.alert),
+                  dashboard: 'https://grafana.app-sre.devshift.net/d/%s/%s?orgId=1&refresh=10s&var-datasource=%s&var-namespace=%s&var-job=All&var-pod=All&var-interval=5m' % [
+                    dashboardID(g.name).id,
+                    g.name,
+                    dashboardDatasource(environment).datasource,
+                    namespace,
+                  ],
+                },
             labels+: {
               service: 'telemeter',
-              severity: setSeverity(r.labels.severity, r.alert).label,
+              severity: setSeverity(r.labels.severity, environment, r.alert).label,
             },
           } else r
         for r in super.rules
@@ -152,7 +182,7 @@ local renderAlerts(name, namespace, mixin) = {
   },
 
   spec: mixin {
-    prometheusAlerts+:: appSREOverwrites(super.prometheusAlerts, namespace),
+    prometheusAlerts+:: appSREOverwrites(namespace),
   }.prometheusAlerts,
 };
 
@@ -204,7 +234,7 @@ local renderAlerts(name, namespace, mixin) = {
   },
 
   'telemeter-slos-stage.prometheusrules': renderAlerts('telemeter-slos-stage', 'telemeter-stage', telemeter),
-  'telemeter-slos-production.prometheusrules': renderAlerts('telemeter-slos-production', 'telemeter-prodcution', telemeter),
+  'telemeter-slos-production.prometheusrules': renderAlerts('telemeter-slos-production', 'telemeter-production', telemeter),
 }
 
 {

resources/observability/prometheusrules/observatorium-api-production.prometheusrules.yaml

@@ -69,7 +69,7 @@ spec:
       record: http_requests_total:burnrate6h
     - alert: ObservatoriumAPIErrorsSLOBudgetBurn
       annotations:
-        dashboard: https://grafana.app-sre.devshift.net/d/Tg-mH0rizaSJDKSADX/observatorium-api-write-errors.slo.rules?orgId=1&refresh=10s&var-datasource=app-sre-prometheus&var-namespace=telemeter-production&var-job=All&var-pod=All&var-interval=5m
+        dashboard: https://grafana.app-sre.devshift.net/d/Tg-mH0rizaSJDKSADX/observatorium-api-write-errors.slo.rules?orgId=1&refresh=10s&var-datasource=telemeter-prod-01-prometheus&var-namespace=telemeter-production&var-job=All&var-pod=All&var-interval=5m
         message: 'High error budget burn for handler=receive,job=observatorium-observatorium-api (current value: {{ $value }})'
         runbook: https://gitlab.cee.redhat.com/observatorium/configuration/blob/master/docs/sop/observatorium.md#observatoriumapierrorsslobudgetburn
       expr: |
@@ -84,7 +84,7 @@ spec:
         severity: critical
     - alert: ObservatoriumAPIErrorsSLOBudgetBurn
       annotations:
-        dashboard: https://grafana.app-sre.devshift.net/d/Tg-mH0rizaSJDKSADX/observatorium-api-write-errors.slo.rules?orgId=1&refresh=10s&var-datasource=app-sre-prometheus&var-namespace=telemeter-production&var-job=All&var-pod=All&var-interval=5m
+        dashboard: https://grafana.app-sre.devshift.net/d/Tg-mH0rizaSJDKSADX/observatorium-api-write-errors.slo.rules?orgId=1&refresh=10s&var-datasource=telemeter-prod-01-prometheus&var-namespace=telemeter-production&var-job=All&var-pod=All&var-interval=5m
         message: 'High error budget burn for handler=receive,job=observatorium-observatorium-api (current value: {{ $value }})'
         runbook: https://gitlab.cee.redhat.com/observatorium/configuration/blob/master/docs/sop/observatorium.md#observatoriumapierrorsslobudgetburn
       expr: |
@@ -99,7 +99,7 @@ spec:
         severity: critical
     - alert: ObservatoriumAPIErrorsSLOBudgetBurn
       annotations:
-        dashboard: https://grafana.app-sre.devshift.net/d/Tg-mH0rizaSJDKSADX/observatorium-api-write-errors.slo.rules?orgId=1&refresh=10s&var-datasource=app-sre-prometheus&var-namespace=telemeter-production&var-job=All&var-pod=All&var-interval=5m
+        dashboard: https://grafana.app-sre.devshift.net/d/Tg-mH0rizaSJDKSADX/observatorium-api-write-errors.slo.rules?orgId=1&refresh=10s&var-datasource=telemeter-prod-01-prometheus&var-namespace=telemeter-production&var-job=All&var-pod=All&var-interval=5m
         message: 'High error budget burn for handler=receive,job=observatorium-observatorium-api (current value: {{ $value }})'
         runbook: https://gitlab.cee.redhat.com/observatorium/configuration/blob/master/docs/sop/observatorium.md#observatoriumapierrorsslobudgetburn
       expr: |
@@ -114,7 +114,7 @@ spec:
         severity: medium
     - alert: ObservatoriumAPIErrorsSLOBudgetBurn
       annotations:
-        dashboard: https://grafana.app-sre.devshift.net/d/Tg-mH0rizaSJDKSADX/observatorium-api-write-errors.slo.rules?orgId=1&refresh=10s&var-datasource=app-sre-prometheus&var-namespace=telemeter-production&var-job=All&var-pod=All&var-interval=5m
+        dashboard: https://grafana.app-sre.devshift.net/d/Tg-mH0rizaSJDKSADX/observatorium-api-write-errors.slo.rules?orgId=1&refresh=10s&var-datasource=telemeter-prod-01-prometheus&var-namespace=telemeter-production&var-job=All&var-pod=All&var-interval=5m
         message: 'High error budget burn for handler=receive,job=observatorium-observatorium-api (current value: {{ $value }})'
         runbook: https://gitlab.cee.redhat.com/observatorium/configuration/blob/master/docs/sop/observatorium.md#observatoriumapierrorsslobudgetburn
       expr: |
@@ -187,7 +187,7 @@ spec:
       record: http_requests_total:burnrate6h
     - alert: ObservatoriumAPIErrorsSLOBudgetBurn
       annotations:
-        dashboard: https://grafana.app-sre.devshift.net/d/Tg-mH0rizaSJDKSADX/observatorium-api-query-errors.slo.rules?orgId=1&refresh=10s&var-datasource=app-sre-prometheus&var-namespace=telemeter-production&var-job=All&var-pod=All&var-interval=5m
+        dashboard: https://grafana.app-sre.devshift.net/d/Tg-mH0rizaSJDKSADX/observatorium-api-query-errors.slo.rules?orgId=1&refresh=10s&var-datasource=telemeter-prod-01-prometheus&var-namespace=telemeter-production&var-job=All&var-pod=All&var-interval=5m
         message: 'High error budget burn for handler=~query|query_legacy,job=observatorium-observatorium-api (current value: {{ $value }})'
         runbook: https://gitlab.cee.redhat.com/observatorium/configuration/blob/master/docs/sop/observatorium.md#observatoriumapierrorsslobudgetburn
       expr: |
@@ -202,7 +202,7 @@ spec:
         severity: critical
     - alert: ObservatoriumAPIErrorsSLOBudgetBurn
       annotations:
-        dashboard: https://grafana.app-sre.devshift.net/d/Tg-mH0rizaSJDKSADX/observatorium-api-query-errors.slo.rules?orgId=1&refresh=10s&var-datasource=app-sre-prometheus&var-namespace=telemeter-production&var-job=All&var-pod=All&var-interval=5m
+        dashboard: https://grafana.app-sre.devshift.net/d/Tg-mH0rizaSJDKSADX/observatorium-api-query-errors.slo.rules?orgId=1&refresh=10s&var-datasource=telemeter-prod-01-prometheus&var-namespace=telemeter-production&var-job=All&var-pod=All&var-interval=5m
         message: 'High error budget burn for handler=~query|query_legacy,job=observatorium-observatorium-api (current value: {{ $value }})'
         runbook: https://gitlab.cee.redhat.com/observatorium/configuration/blob/master/docs/sop/observatorium.md#observatoriumapierrorsslobudgetburn
       expr: |
@@ -217,7 +217,7 @@ spec:
         severity: critical
     - alert: ObservatoriumAPIErrorsSLOBudgetBurn
       annotations:
-        dashboard: https://grafana.app-sre.devshift.net/d/Tg-mH0rizaSJDKSADX/observatorium-api-query-errors.slo.rules?orgId=1&refresh=10s&var-datasource=app-sre-prometheus&var-namespace=telemeter-production&var-job=All&var-pod=All&var-interval=5m
+        dashboard: https://grafana.app-sre.devshift.net/d/Tg-mH0rizaSJDKSADX/observatorium-api-query-errors.slo.rules?orgId=1&refresh=10s&var-datasource=telemeter-prod-01-prometheus&var-namespace=telemeter-production&var-job=All&var-pod=All&var-interval=5m
         message: 'High error budget burn for handler=~query|query_legacy,job=observatorium-observatorium-api (current value: {{ $value }})'
         runbook: https://gitlab.cee.redhat.com/observatorium/configuration/blob/master/docs/sop/observatorium.md#observatoriumapierrorsslobudgetburn
       expr: |
@@ -232,7 +232,7 @@ spec:
         severity: medium
     - alert: ObservatoriumAPIErrorsSLOBudgetBurn
       annotations:
-        dashboard: https://grafana.app-sre.devshift.net/d/Tg-mH0rizaSJDKSADX/observatorium-api-query-errors.slo.rules?orgId=1&refresh=10s&var-datasource=app-sre-prometheus&var-namespace=telemeter-production&var-job=All&var-pod=All&var-interval=5m
+        dashboard: https://grafana.app-sre.devshift.net/d/Tg-mH0rizaSJDKSADX/observatorium-api-query-errors.slo.rules?orgId=1&refresh=10s&var-datasource=telemeter-prod-01-prometheus&var-namespace=telemeter-production&var-job=All&var-pod=All&var-interval=5m
         message: 'High error budget burn for handler=~query|query_legacy,job=observatorium-observatorium-api (current value: {{ $value }})'
         runbook: https://gitlab.cee.redhat.com/observatorium/configuration/blob/master/docs/sop/observatorium.md#observatoriumapierrorsslobudgetburn
       expr: |
@@ -305,7 +305,7 @@ spec:
       record: http_requests_total:burnrate6h
     - alert: ObservatoriumAPIErrorsSLOBudgetBurn
       annotations:
-        dashboard: https://grafana.app-sre.devshift.net/d/Tg-mH0rizaSJDKSADX/observatorium-api-query-range-errors.slo.rules?orgId=1&refresh=10s&var-datasource=app-sre-prometheus&var-namespace=telemeter-production&var-job=All&var-pod=All&var-interval=5m
+        dashboard: https://grafana.app-sre.devshift.net/d/Tg-mH0rizaSJDKSADX/observatorium-api-query-range-errors.slo.rules?orgId=1&refresh=10s&var-datasource=telemeter-prod-01-prometheus&var-namespace=telemeter-production&var-job=All&var-pod=All&var-interval=5m
         message: 'High error budget burn for handler=query_range,job=observatorium-observatorium-api (current value: {{ $value }})'
         runbook: https://gitlab.cee.redhat.com/observatorium/configuration/blob/master/docs/sop/observatorium.md#observatoriumapierrorsslobudgetburn
       expr: |
@@ -320,7 +320,7 @@ spec:
         severity: critical
     - alert: ObservatoriumAPIErrorsSLOBudgetBurn
       annotations:
-        dashboard: https://grafana.app-sre.devshift.net/d/Tg-mH0rizaSJDKSADX/observatorium-api-query-range-errors.slo.rules?orgId=1&refresh=10s&var-datasource=app-sre-prometheus&var-namespace=telemeter-production&var-job=All&var-pod=All&var-interval=5m
+        dashboard: https://grafana.app-sre.devshift.net/d/Tg-mH0rizaSJDKSADX/observatorium-api-query-range-errors.slo.rules?orgId=1&refresh=10s&var-datasource=telemeter-prod-01-prometheus&var-namespace=telemeter-production&var-job=All&var-pod=All&var-interval=5m
         message: 'High error budget burn for handler=query_range,job=observatorium-observatorium-api (current value: {{ $value }})'
         runbook: https://gitlab.cee.redhat.com/observatorium/configuration/blob/master/docs/sop/observatorium.md#observatoriumapierrorsslobudgetburn
       expr: |
@@ -335,7 +335,7 @@ spec:
         severity: critical
     - alert: ObservatoriumAPIErrorsSLOBudgetBurn
       annotations:
-        dashboard: https://grafana.app-sre.devshift.net/d/Tg-mH0rizaSJDKSADX/observatorium-api-query-range-errors.slo.rules?orgId=1&refresh=10s&var-datasource=app-sre-prometheus&var-namespace=telemeter-production&var-job=All&var-pod=All&var-interval=5m
+        dashboard: https://grafana.app-sre.devshift.net/d/Tg-mH0rizaSJDKSADX/observatorium-api-query-range-errors.slo.rules?orgId=1&refresh=10s&var-datasource=telemeter-prod-01-prometheus&var-namespace=telemeter-production&var-job=All&var-pod=All&var-interval=5m
         message: 'High error budget burn for handler=query_range,job=observatorium-observatorium-api (current value: {{ $value }})'
         runbook: https://gitlab.cee.redhat.com/observatorium/configuration/blob/master/docs/sop/observatorium.md#observatoriumapierrorsslobudgetburn
       expr: |
@@ -350,7 +350,7 @@ spec:
         severity: medium
     - alert: ObservatoriumAPIErrorsSLOBudgetBurn
       annotations:
-        dashboard: https://grafana.app-sre.devshift.net/d/Tg-mH0rizaSJDKSADX/observatorium-api-query-range-errors.slo.rules?orgId=1&refresh=10s&var-datasource=app-sre-prometheus&var-namespace=telemeter-production&var-job=All&var-pod=All&var-interval=5m
+        dashboard: https://grafana.app-sre.devshift.net/d/Tg-mH0rizaSJDKSADX/observatorium-api-query-range-errors.slo.rules?orgId=1&refresh=10s&var-datasource=telemeter-prod-01-prometheus&var-namespace=telemeter-production&var-job=All&var-pod=All&var-interval=5m
         message: 'High error budget burn for handler=query_range,job=observatorium-observatorium-api (current value: {{ $value }})'
         runbook: https://gitlab.cee.redhat.com/observatorium/configuration/blob/master/docs/sop/observatorium.md#observatoriumapierrorsslobudgetburn
       expr: |