From 8ded05a725668d3e1e00afd31b8f11f22a01a608 Mon Sep 17 00:00:00 2001 From: BenoitCote Date: Tue, 30 Jun 2026 16:03:18 -0700 Subject: [PATCH 1/3] adding alert for storage above a certain threshold --- cron_jobs/direct_health_monitor.py | 79 +++++++++++++++++++++++++++++- 1 file changed, 78 insertions(+), 1 deletion(-) diff --git a/cron_jobs/direct_health_monitor.py b/cron_jobs/direct_health_monitor.py index 092b7f14..1cc5fbcc 100644 --- a/cron_jobs/direct_health_monitor.py +++ b/cron_jobs/direct_health_monitor.py @@ -877,6 +877,76 @@ def query() -> Any: ) +DISK_USAGE_THRESHOLD_PCT = int(os.getenv("HEALTH_MONITOR_DISK_THRESHOLD", 80)) + + +async def check_disk_usage() -> list[HealthRecord]: + """Check VM disk usage via `df -h` and flag filesystems above the threshold.""" + + log.info("Checking disk usage...") + + try: + proc = await asyncio.create_subprocess_exec( + "df", "-h", + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + stdout, stderr = await asyncio.wait_for(proc.communicate(), timeout=10) + except asyncio.TimeoutError: + return [ + HealthRecord( + component="Disk Usage", + cluster="vm", + status=HealthStatus.FAILED, + detail="df -h timed out", + ) + ] + except Exception as e: + return [ + HealthRecord( + component="Disk Usage", + cluster="vm", + status=HealthStatus.FAILED, + detail=f"Failed to run df: {e}", + ) + ] + + records: list[HealthRecord] = [] + lines = stdout.decode().splitlines() + + for line in lines[1:]: + parts = line.split() + if len(parts) < 6: + continue + filesystem, size, used, avail, use_pct, mount = parts[0], parts[1], parts[2], parts[3], parts[4], parts[5] + try: + pct = int(use_pct.rstrip("%")) + except ValueError: + continue + + if pct > DISK_USAGE_THRESHOLD_PCT: + records.append( + HealthRecord( + component=f"Disk {mount} ({filesystem})", + cluster="vm", + status=HealthStatus.FAILED, + detail=f"{use_pct} used — {used} of {size} (avail: {avail})", + ) + ) + + if not records: + records.append( + HealthRecord( + component="Disk Usage", + cluster="vm", + status=HealthStatus.HEALTHY, + detail=f"All filesystems below {DISK_USAGE_THRESHOLD_PCT}%", + ) + ) + + return records + + async def check_globus_compute() -> HealthRecord: """Check Globus Compute connectivity""" @@ -984,6 +1054,7 @@ async def run_monitor() -> list[HealthRecord]: check_redis_health(), check_postgres_health(), check_globus_compute(), + check_disk_usage(), ), return_exceptions=True, ) @@ -1002,7 +1073,13 @@ async def run_monitor() -> list[HealthRecord]: ) ) elif isinstance(result, list): - records.extend(result) + for item in result: + if isinstance(item, list): + records.extend(item) + elif isinstance(item, HealthRecord): + records.append(item) + else: + log.error("Unexpected item in %s results: %r", cluster_name, item) else: log.error("Unexpected result for %s monitor: %r", cluster_name, result) records.append( From fb31b9d9805faa76236a1fa1ea021dda1e396151 Mon Sep 17 00:00:00 2001 From: BenoitCote Date: Tue, 30 Jun 2026 16:18:38 -0700 Subject: [PATCH 2/3] adding storage alerts for the VM --- cron_jobs/direct_health_monitor.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/cron_jobs/direct_health_monitor.py b/cron_jobs/direct_health_monitor.py index 1cc5fbcc..6c19e322 100644 --- a/cron_jobs/direct_health_monitor.py +++ b/cron_jobs/direct_health_monitor.py @@ -633,7 +633,6 @@ async def check_metis_models() -> list[HealthRecord]: return records - async def extract_minerva_models() -> list[str]: """Flatten Minerva status structure into a list of live model names.""" @@ -768,7 +767,6 @@ async def check_minerva_models() -> list[HealthRecord]: return records - async def check_gateway_health() -> HealthRecord: """Check resource_server /health""" From e0c2e2fa2119d476f797838264581d900e173af6 Mon Sep 17 00:00:00 2001 From: BenoitCote Date: Tue, 30 Jun 2026 16:19:19 -0700 Subject: [PATCH 3/3] formatting --- cron_jobs/direct_health_monitor.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/cron_jobs/direct_health_monitor.py b/cron_jobs/direct_health_monitor.py index 6c19e322..238e2ff4 100644 --- a/cron_jobs/direct_health_monitor.py +++ b/cron_jobs/direct_health_monitor.py @@ -633,6 +633,7 @@ async def check_metis_models() -> list[HealthRecord]: return records + async def extract_minerva_models() -> list[str]: """Flatten Minerva status structure into a list of live model names.""" @@ -767,6 +768,7 @@ async def check_minerva_models() -> list[HealthRecord]: return records + async def check_gateway_health() -> HealthRecord: """Check resource_server /health""" @@ -885,7 +887,8 @@ async def check_disk_usage() -> list[HealthRecord]: try: proc = await asyncio.create_subprocess_exec( - "df", "-h", + "df", + "-h", stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE, ) @@ -916,7 +919,14 @@ async def check_disk_usage() -> list[HealthRecord]: parts = line.split() if len(parts) < 6: continue - filesystem, size, used, avail, use_pct, mount = parts[0], parts[1], parts[2], parts[3], parts[4], parts[5] + filesystem, size, used, avail, use_pct, mount = ( + parts[0], + parts[1], + parts[2], + parts[3], + parts[4], + parts[5], + ) try: pct = int(use_pct.rstrip("%")) except ValueError: