diff --git a/.gitignore b/.gitignore index d362686ff..43952d94e 100644 --- a/.gitignore +++ b/.gitignore @@ -11,6 +11,7 @@ molecule/*/host_vars/* ## Misc *.tar.gz __pycache__ +tests/.coverage tests/output *.orig *.rej diff --git a/doc/source/admin/monitoring.rst b/doc/source/admin/monitoring.rst index d196e15cf..886fb1a99 100644 --- a/doc/source/admin/monitoring.rst +++ b/doc/source/admin/monitoring.rst @@ -260,6 +260,40 @@ so by making the following changes to your inventory: In the example above, we are whitelisting the IP range ``10.0.0.0/24`` and the IP address ``172.10.0.1``. +Extend Rules +~~~~~~~~~~~~ + +It's possible to extend existing prometheus rules +by using ``kube_prometheus_stack_extend_rules`` variable to your inventory. +Here is an example: + +.. code-block:: yaml + + kube_prometheus_stack_extend_rules: + ipmi-exporter: { + groups: [ + { + name: "new_rules", + rules: [ + { + alert: "IpmiCollectorDown", + expr: "ipmi_up == 0", + for: "45m", + labels: { + severity: "P2" + } + } + ] + } + ] + } + +This will add/replace all rules for group ``new_rules`` for +``ipmi-exporter``. The default behavior for +``kube_prometheus_stack_extend_rules`` is add new rule group and keep existing +rule groups, and replace entire rule group if rule group have the same name +(``new_rules`` in this case). + AlertManager ============ diff --git a/plugins/filter/rules_merge.py b/plugins/filter/rules_merge.py new file mode 100644 index 000000000..2c04ee235 --- /dev/null +++ b/plugins/filter/rules_merge.py @@ -0,0 +1,47 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) VEXXHOST, Inc. + +from ansible.errors import AnsibleFilterError + + +def rules_merge(rules, **kwargs): + list_merge = kwargs.pop("list_merge", "replace") + if list_merge not in ("replace", "append"): + raise AnsibleFilterError( + "'replace' and 'append' are the only valid value for 'list_merge'" + ) + if kwargs: + raise AnsibleFilterError("'list_merge' is the only valid keyword argument") + + for rule_name in rules: + merged_groups = dict() + rule = rules[rule_name] + if "groups" not in rule: + continue + for group in rule["groups"]: + if list_merge == "append" and group["name"] in merged_groups: + merged_groups[group["name"]]["rules"] += group["rules"] + # combine merged_groups[group["name"]]["rules"] and group["rules"] + counts = [] + pops = [] + for idx, ru in enumerate(merged_groups[group["name"]]["rules"]): + ru_tup = tuple(sorted(ru.items())) + if ru_tup in counts: + pops.append(idx) + else: + counts.append(ru_tup) + for i in reversed(pops): + del merged_groups[group["name"]]["rules"][i] + else: + merged_groups[group["name"]] = group + + rule["groups"] = list(merged_groups.values()) + rules[rule_name] = rule + return rules + + +class FilterModule(object): + def filters(self): + return { + "rules_merge": rules_merge, + } diff --git a/roles/kube_prometheus_stack/defaults/main.yml b/roles/kube_prometheus_stack/defaults/main.yml index db6215c68..53b9e5efe 100644 --- a/roles/kube_prometheus_stack/defaults/main.yml +++ b/roles/kube_prometheus_stack/defaults/main.yml @@ -30,6 +30,8 @@ kube_prometheus_stack_node_exporter_config: cert_file: /certs/tls.crt key_file: /certs/tls.key +kube_prometheus_stack_extend_rules: {} + kube_prometheus_stack_ingress_class_name: "{{ atmosphere_ingress_class_name }}" kube_prometheus_stack_ingress_cluster_issuer: "{{ atmosphere_ingress_cluster_issuer }}" @@ -58,7 +60,6 @@ kube_prometheus_stack_keycloak_admin_user: admin kube_prometheus_stack_keycloak_admin_password: "{{ keycloak_admin_password }}" kube_prometheus_stack_keycloak_realm: atmosphere kube_prometheus_stack_keycloak_realm_name: Atmosphere - kube_prometheus_stack_keycloak_clients: - id: alertmanager port: 9093 diff --git a/roles/kube_prometheus_stack/vars/main.yml b/roles/kube_prometheus_stack/vars/main.yml index 80a74cb8c..5b1977ced 100644 --- a/roles/kube_prometheus_stack/vars/main.yml +++ b/roles/kube_prometheus_stack/vars/main.yml @@ -654,7 +654,7 @@ _kube_prometheus_stack_helm_values: {{ kube_prometheus_stack_node_exporter_config | to_nice_yaml | indent(4) }} certificate-template.yml: | {{ kube_prometheus_stack_node_exporter_tls_template | to_nice_yaml | indent(4) }} - additionalPrometheusRulesMap: "{{ lookup('vexxhost.atmosphere.jsonnet', 'jsonnet/rules.jsonnet') }}" + additionalPrometheusRulesMap: "{{ lookup('vexxhost.atmosphere.jsonnet', 'jsonnet/rules.jsonnet') | combine(kube_prometheus_stack_extend_rules, recursive=true, list_merge='append') | vexxhost.atmosphere.rules_merge(list_merge='replace') }}" extraManifests: - apiVersion: rbac.authorization.k8s.io/v1 kind: Role diff --git a/tests/unit/plugins/filter/test_rules_merge.py b/tests/unit/plugins/filter/test_rules_merge.py new file mode 100644 index 000000000..ada625c33 --- /dev/null +++ b/tests/unit/plugins/filter/test_rules_merge.py @@ -0,0 +1,111 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) VEXXHOST, Inc. + +import copy + +import pytest +from ansible.errors import AnsibleFilterError +from ansible_collections.vexxhost.atmosphere.plugins.filter.rules_merge import ( + rules_merge, +) + +rules = { + "ipmi-exporter": { + "groups": [ + { + "name": "rules", + "rules": [ + { + "alert": "IpmiCollectorDown", + "expr": "ipmi_up == 0", + "for": "16m", + "labels": {"severity": "P2"}, + } + ], + }, + { + "name": "rules", + "rules": [ + { + "alert": "IpmiCollectorDown", + "expr": "ipmi_up == 0", + "for": "15m", + "labels": {"severity": "P3"}, + } + ], + }, + ] + } +} + +expectd_append_rules = { + "ipmi-exporter": { + "groups": [ + { + "name": "rules", + "rules": [ + { + "alert": "IpmiCollectorDown", + "expr": "ipmi_up == 0", + "for": "16m", + "labels": {"severity": "P2"}, + }, + { + "alert": "IpmiCollectorDown", + "expr": "ipmi_up == 0", + "for": "15m", + "labels": {"severity": "P3"}, + }, + ], + } + ] + } +} + +expectd_replace_rules = { + "ipmi-exporter": { + "groups": [ + { + "name": "rules", + "rules": [ + { + "alert": "IpmiCollectorDown", + "expr": "ipmi_up == 0", + "for": "15m", + "labels": {"severity": "P3"}, + } + ], + } + ] + } +} + + +@pytest.mark.parametrize( + "test_input,expected", + [ + ( + [copy.deepcopy(rules), {"list_merge": "replace"}], + expectd_replace_rules, + ), + ( + [copy.deepcopy(rules), {"list_merge": "append"}], + expectd_append_rules, + ), + ( + [dict(), {"list_merge": "foo"}], + "'replace' and 'append' are the only valid value for 'list_merge'", + ), + ( + [dict(), {"list_merge": "append", "bar": "foo"}], + "'list_merge' is the only valid keyword argument", + ), + ], +) +def test_rules_merge(test_input, expected): + if isinstance(expected, dict): + with pytest.raises(AnsibleFilterError) as ex: + rules_merge(test_input[0], **test_input[1]) + assert str(ex.value) == expected + else: + assert rules_merge(test_input[0], **test_input[1]) == expected