[ATMOSPHERE-48]fix: allow prometheus extend rule

.gitignore

@@ -11,6 +11,7 @@ molecule/*/host_vars/*
 ## Misc
 *.tar.gz
 __pycache__
+tests/.coverage
 tests/output
 *.orig
 *.rej

doc/source/admin/monitoring.rst

@@ -260,6 +260,40 @@ so by making the following changes to your inventory:
 In the example above, we are whitelisting the IP range ``10.0.0.0/24`` and the IP address
 ``172.10.0.1``.
 
+Extend Rules
+~~~~~~~~~~~~
+
+It's possible to extend existing prometheus rules
+by using ``kube_prometheus_stack_extend_rules`` variable to your inventory.
+Here is an example:
+
+.. code-block:: yaml
+
+   kube_prometheus_stack_extend_rules:
+     ipmi-exporter: {
+         groups: [
+             {
+                 name: "new_rules",
+                 rules: [
+                     {
+                         alert: "IpmiCollectorDown",
+                         expr: "ipmi_up == 0",
+                         for: "45m",
+                         labels: {
+                             severity: "P2"
+                         }
+                     }
+                 ]
+             }
+         ]
+     }
+
+This will add/replace all rules for group  ``new_rules`` for
+``ipmi-exporter``. The default behavior for
+``kube_prometheus_stack_extend_rules`` is  add new rule group and keep existing
+rule groups, and replace entire rule group if rule group have the same name
+(``new_rules`` in this case).
+
 AlertManager
 ============
 

plugins/filter/rules_merge.py

@@ -0,0 +1,47 @@
+# SPDX-License-Identifier: Apache-2.0
+# Copyright (c) VEXXHOST, Inc.
+
+from ansible.errors import AnsibleFilterError
+
+
+def rules_merge(rules, **kwargs):
+    list_merge = kwargs.pop("list_merge", "replace")
+    if list_merge not in ("replace", "append"):
+        raise AnsibleFilterError(
+            "'replace' and 'append' are the only valid value for 'list_merge'"
+        )
+    if kwargs:
+        raise AnsibleFilterError("'list_merge' is the only valid keyword argument")
+
+    for rule_name in rules:
+        merged_groups = dict()
+        rule = rules[rule_name]
+        if "groups" not in rule:
+            continue
+        for group in rule["groups"]:
+            if list_merge == "append" and group["name"] in merged_groups:
+                merged_groups[group["name"]]["rules"] += group["rules"]
+                # combine merged_groups[group["name"]]["rules"] and group["rules"]
+                counts = []
+                pops = []
+                for idx, ru in enumerate(merged_groups[group["name"]]["rules"]):
+                    ru_tup = tuple(sorted(ru.items()))
+                    if ru_tup in counts:
+                        pops.append(idx)
+                    else:
+                        counts.append(ru_tup)
+                for i in reversed(pops):
+                    del merged_groups[group["name"]]["rules"][i]
+            else:
+                merged_groups[group["name"]] = group
+
+        rule["groups"] = list(merged_groups.values())
+        rules[rule_name] = rule
+    return rules
+
+
+class FilterModule(object):
+    def filters(self):
+        return {
+            "rules_merge": rules_merge,
+        }

roles/kube_prometheus_stack/defaults/main.yml

@@ -30,6 +30,8 @@ kube_prometheus_stack_node_exporter_config:
     cert_file: /certs/tls.crt
     key_file: /certs/tls.key
 
+kube_prometheus_stack_extend_rules: {}
+
 kube_prometheus_stack_ingress_class_name: "{{ atmosphere_ingress_class_name }}"
 kube_prometheus_stack_ingress_cluster_issuer: "{{ atmosphere_ingress_cluster_issuer }}"
 
@@ -58,7 +60,6 @@ kube_prometheus_stack_keycloak_admin_user: admin
 kube_prometheus_stack_keycloak_admin_password: "{{ keycloak_admin_password }}"
 kube_prometheus_stack_keycloak_realm: atmosphere
 kube_prometheus_stack_keycloak_realm_name: Atmosphere
-
 kube_prometheus_stack_keycloak_clients:
   - id: alertmanager
     port: 9093

roles/kube_prometheus_stack/vars/main.yml

@@ -654,7 +654,7 @@ _kube_prometheus_stack_helm_values:
             {{ kube_prometheus_stack_node_exporter_config | to_nice_yaml | indent(4) }}
           certificate-template.yml: |
             {{ kube_prometheus_stack_node_exporter_tls_template | to_nice_yaml | indent(4) }}
-  additionalPrometheusRulesMap: "{{ lookup('vexxhost.atmosphere.jsonnet', 'jsonnet/rules.jsonnet') }}"
+  additionalPrometheusRulesMap: "{{ lookup('vexxhost.atmosphere.jsonnet', 'jsonnet/rules.jsonnet') | combine(kube_prometheus_stack_extend_rules, recursive=true, list_merge='append') | vexxhost.atmosphere.rules_merge(list_merge='replace') }}"
   extraManifests:
     - apiVersion: rbac.authorization.k8s.io/v1
       kind: Role

tests/unit/plugins/filter/test_rules_merge.py

@@ -0,0 +1,111 @@
+# SPDX-License-Identifier: Apache-2.0
+# Copyright (c) VEXXHOST, Inc.
+
+import copy
+
+import pytest
+from ansible.errors import AnsibleFilterError
+from ansible_collections.vexxhost.atmosphere.plugins.filter.rules_merge import (
+    rules_merge,
+)
+
+rules = {
+    "ipmi-exporter": {
+        "groups": [
+            {
+                "name": "rules",
+                "rules": [
+                    {
+                        "alert": "IpmiCollectorDown",
+                        "expr": "ipmi_up == 0",
+                        "for": "16m",
+                        "labels": {"severity": "P2"},
+                    }
+                ],
+            },
+            {
+                "name": "rules",
+                "rules": [
+                    {
+                        "alert": "IpmiCollectorDown",
+                        "expr": "ipmi_up == 0",
+                        "for": "15m",
+                        "labels": {"severity": "P3"},
+                    }
+                ],
+            },
+        ]
+    }
+}
+
+expectd_append_rules = {
+    "ipmi-exporter": {
+        "groups": [
+            {
+                "name": "rules",
+                "rules": [
+                    {
+                        "alert": "IpmiCollectorDown",
+                        "expr": "ipmi_up == 0",
+                        "for": "16m",
+                        "labels": {"severity": "P2"},
+                    },
+                    {
+                        "alert": "IpmiCollectorDown",
+                        "expr": "ipmi_up == 0",
+                        "for": "15m",
+                        "labels": {"severity": "P3"},
+                    },
+                ],
+            }
+        ]
+    }
+}
+
+expectd_replace_rules = {
+    "ipmi-exporter": {
+        "groups": [
+            {
+                "name": "rules",
+                "rules": [
+                    {
+                        "alert": "IpmiCollectorDown",
+                        "expr": "ipmi_up == 0",
+                        "for": "15m",
+                        "labels": {"severity": "P3"},
+                    }
+                ],
+            }
+        ]
+    }
+}
+
+
+@pytest.mark.parametrize(
+    "test_input,expected",
+    [
+        (
+            [copy.deepcopy(rules), {"list_merge": "replace"}],
+            expectd_replace_rules,
+        ),
+        (
+            [copy.deepcopy(rules), {"list_merge": "append"}],
+            expectd_append_rules,
+        ),
+        (
+            [dict(), {"list_merge": "foo"}],
+            "'replace' and 'append' are the only valid value for 'list_merge'",
+        ),
+        (
+            [dict(), {"list_merge": "append", "bar": "foo"}],
+            "'list_merge' is the only valid keyword argument",
+        ),
+    ],
+)
+def test_rules_merge(test_input, expected):
+    if isinstance(expected, dict):
+        with pytest.raises(AnsibleFilterError) as ex:
+            rules_merge(test_input[0], **test_input[1])
+        assert str(ex.value) == expected
+    else:
+        assert rules_merge(test_input[0], **test_input[1]) == expected