Setup certificate monitoring

peterablehmann · May 25, 2024 · 79ceb35 · 79ceb35
1 parent 6463853
commit 79ceb35
Show file tree

Hide file tree

Showing 3 changed files with 96 additions and 0 deletions.
diff --git a/modules/monitoring/blackbox.nix b/modules/monitoring/blackbox.nix
@@ -0,0 +1,55 @@
+{ pkgs
+, config
+, ...
+}:
+let
+  domain = "blackbox.xnee.net";
+  tls-dir = config.security.acme.certs.${domain}.directory;
+  webConfig = pkgs.writeTextFile {
+    name = "web-config.yml";
+    text = ''
+      tls_server_config:
+        cert_file: ${tls-dir}/fullchain.pem
+        key_file: ${tls-dir}/key.pem
+      basic_auth_users:
+        prometheus: $2y$10$XnqpKDYhGVLgQaKzv8Lm9.0hZagMN7UB9Q/mIDU3t4tE4nBwYXnYC
+    '';
+  };
+in
+{
+  security.acme.certs."${domain}" = { };
+  networking.domains.subDomains."${domain}" = { };
+  services.nginx.virtualHosts."${domain}" = {
+    enableACME = true;
+    forceSSL = true;
+    locations."/" = {
+      proxyPass = "https://localhost:${builtins.toString config.services.prometheus.exporters.blackbox.port }";
+    };
+  };
+
+  services.prometheus.exporters.blackbox = {
+    enable = true;
+    port = 3044;
+    extraFlags = [
+      "--web.config.file=${webConfig}"
+    ];
+    configFile = (pkgs.formats.yaml { }).generate "blackbox.yml" {
+      modules = {
+        certs = {
+          prober = "http";
+          http = {
+            method = "GET";
+            fail_if_not_ssl = true;
+            preferred_ip_protocol = "ip6";
+            ip_protocol_fallback = true;
+          };
+        };
+      };
+    };
+  };
+
+  systemd.services.prometheus-blackbox-exporter.serviceConfig = {
+    SupplementaryGroups = [ config.security.acme.certs.${domain}.group ];
+    BindReadOnlyPaths = [ tls-dir ];
+  };
+}
diff --git a/modules/monitoring/default.nix b/modules/monitoring/default.nix
@@ -1,5 +1,6 @@
 {
   imports = [
+    ./blackbox.nix
     ./prometheus.nix
     ./grafana.nix
   ];

diff --git a/modules/monitoring/prometheus.nix b/modules/monitoring/prometheus.nix
@@ -29,6 +29,46 @@
             );
           }];
         }
+        {
+          job_name = "certs";
+          scrape_interval = "5m";
+          basic_auth = {
+            username = "prometheus";
+            password_file = config.sops.secrets."prometheus/basic_auth".path;
+          };
+          metrics_path = "/probe";
+          params = {
+            module = [ "certs" ];
+          };
+          static_configs = [{
+            targets = lib.flatten (lib.mapAttrsToList (n: v: builtins.attrNames v.config.security.acme.certs) inputs.self.nixosConfigurations);
+          }];
+          relabel_configs = [
+            {
+              source_labels = [ "__address__" ];
+              target_label = "__param_target";
+            }
+            {
+              source_labels = [ "__param_target" ];
+              target_label = "instance";
+            }
+            {
+              target_label = "__address__";
+              replacement = "blackbox.xnee.net";
+            }
+          ];
+        }
+        {
+          job_name = "blackbox_exporter";
+          scrape_interval = "1m";
+          basic_auth = {
+            username = "prometheus";
+            password_file = config.sops.secrets."prometheus/basic_auth".path;
+          };
+          static_configs = [{
+            targets = [ "blackbox.xnee.net" ];
+          }];
+        }
         {
           job_name = "prometheus";
           scrape_interval = "5s";