From 6474866fda12694917a3cafc9612283611ffeaab Mon Sep 17 00:00:00 2001
From: Chris Smith <1979423+chris13524@users.noreply.github.com>
Date: Tue, 23 Jan 2024 20:19:20 -0500
Subject: [PATCH] fix: prefer max/min for alarms (#330)

---
 terraform/monitoring/panels/rds/cpu.libsonnet    | 13 +++++++++++++
 .../panels/rds/database_connections.libsonnet    |  2 +-
 .../panels/rds/freeable_memory.libsonnet         | 16 ++++++++++++++--
 .../panels/rds/volume_bytes_used.libsonnet       | 11 +++++++++++
 4 files changed, 39 insertions(+), 3 deletions(-)

diff --git a/terraform/monitoring/panels/rds/cpu.libsonnet b/terraform/monitoring/panels/rds/cpu.libsonnet
index faf1df2d..513351e8 100644
--- a/terraform/monitoring/panels/rds/cpu.libsonnet
+++ b/terraform/monitoring/panels/rds/cpu.libsonnet
@@ -21,6 +21,7 @@ local targets   = grafana.targets;
         notifications = vars.notifications,
         refid         = 'CPU',
         limit         = 70,
+        reducer       = grafana.alertCondition.reducers.Max,
       )
     )
 
@@ -35,4 +36,16 @@ local targets   = grafana.targets;
       statistic     = 'Average',
       refId         = 'CPU_Avg'
     ))
+
+    .addTarget(targets.cloudwatch(
+      datasource    = ds.cloudwatch,
+      namespace     = 'AWS/RDS',
+      metricName    = 'CPUUtilization',
+      dimensions  = {
+        DBClusterIdentifier: vars.rds_cluster_id,
+      },
+      matchExact  = true,
+      statistic     = 'Maximum',
+      refId         = 'CPU_Max'
+    ))
 }
diff --git a/terraform/monitoring/panels/rds/database_connections.libsonnet b/terraform/monitoring/panels/rds/database_connections.libsonnet
index c5f40d79..7f2de4e0 100644
--- a/terraform/monitoring/panels/rds/database_connections.libsonnet
+++ b/terraform/monitoring/panels/rds/database_connections.libsonnet
@@ -20,6 +20,6 @@ local targets   = grafana.targets;
         DBClusterIdentifier: vars.rds_cluster_id,
       },
       matchExact  = true,
-      statistic     = 'Average',
+      statistic     = 'Maximum',
     ))
 }
diff --git a/terraform/monitoring/panels/rds/freeable_memory.libsonnet b/terraform/monitoring/panels/rds/freeable_memory.libsonnet
index 8d467af5..ad9a7a5a 100644
--- a/terraform/monitoring/panels/rds/freeable_memory.libsonnet
+++ b/terraform/monitoring/panels/rds/freeable_memory.libsonnet
@@ -25,10 +25,10 @@ local targets   = grafana.targets;
           evaluatorParams = [ 30 ],
           evaluatorType   = 'lt',
           operatorType    = 'or',
-          queryRefId      = 'Mem_Avg',
+          queryRefId      = 'Mem_Min',
           queryTimeStart  = '5m',
           queryTimeEnd    = 'now',
-          reducerType     = grafana.alert_reducers.Avg
+          reducerType     = grafana.alert_reducers.Min
         ),
       ],
     ))
@@ -44,4 +44,16 @@ local targets   = grafana.targets;
       statistic     = 'Average',
       refId         = 'Mem_Avg',
     ))
+
+    .addTarget(targets.cloudwatch(
+      datasource    = ds.cloudwatch,
+      namespace     = 'AWS/RDS',
+      metricName    = 'FreeableMemory',
+      dimensions  = {
+        DBClusterIdentifier: vars.rds_cluster_id,
+      },
+      matchExact  = true,
+      statistic     = 'Minimum',
+      refId         = 'Mem_Min',
+    ))
 }
diff --git a/terraform/monitoring/panels/rds/volume_bytes_used.libsonnet b/terraform/monitoring/panels/rds/volume_bytes_used.libsonnet
index a073a92c..e6887188 100644
--- a/terraform/monitoring/panels/rds/volume_bytes_used.libsonnet
+++ b/terraform/monitoring/panels/rds/volume_bytes_used.libsonnet
@@ -25,4 +25,15 @@ local targets   = grafana.targets;
       matchExact  = true,
       statistic     = 'Average',
     ))
+
+    .addTarget(targets.cloudwatch(
+      datasource    = ds.cloudwatch,
+      namespace     = 'AWS/RDS',
+      metricName    = 'VolumeBytesUsed',
+      dimensions  = {
+        DBClusterIdentifier: vars.rds_cluster_id,
+      },
+      matchExact  = true,
+      statistic     = 'Maximum',
+    ))
 }