fix: send notifications when UK database is down (#6498)

server/jobs.js

@@ -1,8 +1,22 @@
 const { UptimeKumaServer } = require("./uptime-kuma-server");
 const { clearOldData } = require("./jobs/clear-old-data");
 const { incrementalVacuum } = require("./jobs/incremental-vacuum");
+const { Notification } = require("./notification");
 const Cron = require("croner");
 
+/**
+ * Refresh notification cache periodically
+ * @returns {Promise<void>}
+ */
+async function refreshNotificationCache() {
+    try {
+        await Notification.refreshCache();
+        Notification.resetDatabaseDownFlag();
+    } catch (e) {
+        // Silently fail - cache refresh is not critical
+    }
+}
+
 const jobs = [
     {
         name: "clear-old-data",
@@ -15,6 +29,12 @@ const jobs = [
         interval: "*/5 * * * *",
         jobFunc: incrementalVacuum,
         croner: null,
+    },
+    {
+        name: "refresh-notification-cache",
+        interval: "*/30 * * * *", // Every 30 minutes
+        jobFunc: refreshNotificationCache,
+        croner: null,
     }
 ];
 

server/model/monitor.js

@@ -990,17 +990,49 @@ class Monitor extends BeanModel {
         };
 
         /**
-         * Get a heartbeat and handle errors7
+         * Get a heartbeat and handle errors
          * @returns {void}
          */
         const safeBeat = async () => {
             try {
                 await beat();
+                // If beat succeeds, database is likely back up, reset the flag
+                const { Notification } = require("../notification");
+                Notification.resetDatabaseDownFlag();
             } catch (e) {
                 console.trace(e);
                 UptimeKumaServer.errorLog(e, false);
                 log.error("monitor", "Please report to https://github.com/louislam/uptime-kuma/issues");
 
+                // Check if this is a database connection error
+                const isDatabaseError = e && (
+                    e.code === "EHOSTUNREACH" ||
+                    e.code === "ECONNREFUSED" ||
+                    e.code === "ETIMEDOUT" ||
+                    e.code === "ENOTFOUND" ||
+                    e.code === "ER_ACCESS_DENIED_ERROR" ||
+                    e.code === "ER_BAD_DB_ERROR" ||
+                    e.code === "ER_DBACCESS_DENIED_ERROR" ||
+                    e.message?.includes("database") ||
+                    e.message?.includes("Database") ||
+                    e.message?.includes("Connection lost") ||
+                    e.message?.includes("connect") ||
+                    (e.syscall === "connect" && (e.address || e.port))
+                );
+
+                if (isDatabaseError) {
+                    const errorMessage = e.message || `${e.code || "Unknown error"}`;
+                    log.error("monitor", `Database connection error detected in monitor: ${errorMessage}`);
+
+                    // Try to send notification using cached notifications
+                    try {
+                        const { Notification } = require("../notification");
+                        await Notification.sendDatabaseDownNotification(errorMessage);
+                    } catch (notifError) {
+                        log.error("monitor", `Failed to send database down notification: ${notifError.message}`);
+                    }
+                }
+
                 if (! this.isStop) {
                     log.info("monitor", "Try to restart the monitor");
                     this.heartbeatInterval = setTimeout(safeBeat, this.interval * 1000);

server/notification.js

@@ -87,6 +87,21 @@ const Webpush = require("./notification-providers/Webpush");
 
 class Notification {
     providerList = {};
+    /**
+     * Cache for all notifications to use when database is down
+     * @type {Array<object>}
+     */
+    static notificationCache = [];
+    /**
+     * Last time the cache was refreshed
+     * @type {number}
+     */
+    static cacheLastRefresh = 0;
+    /**
+     * Flag to track if we've already sent a database down notification
+     * @type {boolean}
+     */
+    static databaseDownNotificationSent = false;
 
     /**
      * Initialize the notification providers
@@ -256,6 +271,13 @@ class Notification {
             await applyNotificationEveryMonitor(bean.id, userID);
         }
 
+        // Refresh cache after saving
+        try {
+            await Notification.refreshCache();
+        } catch (e) {
+            // Silently fail - cache refresh is not critical
+        }
+
         return bean;
     }
 
@@ -276,6 +298,13 @@ class Notification {
         }
 
         await R.trash(bean);
+
+        // Refresh cache after deleting
+        try {
+            await Notification.refreshCache();
+        } catch (e) {
+            // Silently fail - cache refresh is not critical
+        }
     }
 
     /**
@@ -285,6 +314,100 @@ class Notification {
     static async checkApprise() {
         return await commandExists("apprise");
     }
+
+    /**
+     * Load all notifications into cache for use when database is down
+     * @returns {Promise<void>}
+     */
+    static async refreshCache() {
+        try {
+            // Get all notifications (including default ones)
+            const notifications = await R.getAll(`
+                SELECT notification.* 
+                FROM notification 
+                WHERE active = 1
+            `);
+
+            this.notificationCache = notifications.map(bean => {
+                try {
+                    const config = JSON.parse(bean.config || "{}");
+                    return {
+                        id: bean.id,
+                        name: bean.name,
+                        type: config.type,
+                        config: config,
+                        is_default: bean.is_default === 1,
+                        user_id: bean.user_id,
+                    };
+                } catch (e) {
+                    log.warn("notification", `Failed to parse notification config for ${bean.id}: ${e.message}`);
+                    return null;
+                }
+            }).filter(n => n !== null);
+
+            this.cacheLastRefresh = Date.now();
+            log.debug("notification", `Refreshed notification cache with ${this.notificationCache.length} notifications`);
+        } catch (e) {
+            log.error("notification", `Failed to refresh notification cache: ${e.message}`);
+            // Don't clear the cache if refresh fails, keep using old cache
+        }
+    }
+
+    /**
+     * Send notification about database being down using cached notifications
+     * @param {string} errorMessage Error message from database connection failure
+     * @returns {Promise<void>}
+     */
+    static async sendDatabaseDownNotification(errorMessage) {
+        // Only send once per database down event
+        if (this.databaseDownNotificationSent) {
+            return;
+        }
+
+        // Check if cache is empty or too old (older than 1 hour)
+        const cacheAge = Date.now() - this.cacheLastRefresh;
+        if (this.notificationCache.length === 0 || cacheAge > 60 * 60 * 1000) {
+            log.warn("notification", "Notification cache is empty or too old, cannot send database down notification");
+            return;
+        }
+
+        this.databaseDownNotificationSent = true;
+
+        const msg = `🔴 Uptime Kuma Database Connection Failed\n\nError: ${errorMessage}\n\nUptime Kuma is unable to connect to its database. Monitoring may be affected.`;
+
+        // Send to all cached notifications
+        for (const notification of this.notificationCache) {
+            try {
+                await this.send(
+                    notification.config,
+                    msg,
+                    {
+                        id: 0,
+                        name: "Uptime Kuma System",
+                        type: "system",
+                        active: true,
+                    },
+                    {
+                        status: "down",
+                        msg: errorMessage,
+                        time: new Date().toISOString(),
+                        ping: null,
+                    }
+                );
+                log.info("notification", `Sent database down notification via ${notification.name} (${notification.type})`);
+            } catch (e) {
+                log.error("notification", `Failed to send database down notification via ${notification.name}: ${e.message}`);
+            }
+        }
+    }
+
+    /**
+     * Reset the database down notification flag (call when database is back up)
+     * @returns {void}
+     */
+    static resetDatabaseDownFlag() {
+        this.databaseDownNotificationSent = false;
+    }
 }
 
 /**

server/server.js

@@ -197,6 +197,15 @@ let needSetup = false;
     server.entryPage = await Settings.get("entryPage");
     await StatusPage.loadDomainMappingList();
 
+    // Refresh notification cache after database is ready
+    try {
+        await Notification.refreshCache();
+        Notification.resetDatabaseDownFlag();
+        log.info("server", "Notification cache refreshed");
+    } catch (e) {
+        log.warn("server", `Failed to refresh notification cache on startup: ${e.message}`);
+    }
+
     log.debug("server", "Initializing Prometheus");
     await Prometheus.init();
 
@@ -1996,10 +2005,38 @@ gracefulShutdown(server.httpServer, {
 });
 
 // Catch unexpected errors here
-let unexpectedErrorHandler = (error, promise) => {
+let unexpectedErrorHandler = async (error, promise) => {
     console.trace(error);
     UptimeKumaServer.errorLog(error, false);
     console.error("If you keep encountering errors, please report to https://github.com/louislam/uptime-kuma/issues");
+
+    // Check if this is a database connection error
+    const isDatabaseError = error && (
+        error.code === "EHOSTUNREACH" ||
+        error.code === "ECONNREFUSED" ||
+        error.code === "ETIMEDOUT" ||
+        error.code === "ENOTFOUND" ||
+        error.code === "ER_ACCESS_DENIED_ERROR" ||
+        error.code === "ER_BAD_DB_ERROR" ||
+        error.code === "ER_DBACCESS_DENIED_ERROR" ||
+        error.message?.includes("database") ||
+        error.message?.includes("Database") ||
+        error.message?.includes("Connection lost") ||
+        error.message?.includes("connect") ||
+        (error.syscall === "connect" && (error.address || error.port))
+    );
+
+    if (isDatabaseError) {
+        const errorMessage = error.message || `${error.code || "Unknown error"}`;
+        log.error("server", `Database connection error detected: ${errorMessage}`);
+
+        // Try to send notification using cached notifications
+        try {
+            await Notification.sendDatabaseDownNotification(errorMessage);
+        } catch (e) {
+            log.error("server", `Failed to send database down notification: ${e.message}`);
+        }
+    }
 };
 process.addListener("unhandledRejection", unexpectedErrorHandler);
 process.addListener("uncaughtException", unexpectedErrorHandler);