fix: send notifications when UK database is down (#6498)

db/knex_migrations/2026-01-06-0331-add-send-database-down.js

@@ -0,0 +1,11 @@
+exports.up = async function (knex) {
+    await knex.schema.alterTable("notification", (table) => {
+        table.boolean("send_database_down").notNullable().defaultTo(false);
+    });
+};
+
+exports.down = async function (knex) {
+    await knex.schema.alterTable("notification", (table) => {
+        table.dropColumn("send_database_down");
+    });
+};

server/database-error-detector.js

@@ -0,0 +1,79 @@
+const Database = require("./database");
+
+/**
+ * Check if an error is specifically from a database operation
+ * This is more restrictive to avoid false positives (e.g., notification send failures)
+ * @param {Error} error The error to check
+ * @returns {boolean} True if this is a database error
+ */
+function isDatabaseError(error) {
+    if (!error) {
+        return false;
+    }
+
+    // Check for MySQL/MariaDB specific error codes (most reliable indicator)
+    if (
+        error.code &&
+        (error.code.startsWith("ER_") || // MySQL/MariaDB error codes
+            error.code === "PROTOCOL_CONNECTION_LOST" ||
+            error.code === "PROTOCOL_ENQUEUE_AFTER_QUIT" ||
+            error.code === "PROTOCOL_ENQUEUE_AFTER_FATAL_ERROR")
+    ) {
+        return true;
+    }
+
+    // Check if error originated from database operations by examining stack trace
+    const stack = error.stack || "";
+    const isFromDatabaseCode =
+        stack.includes("redbean-node") ||
+        stack.includes("/server/database.js") ||
+        (stack.includes("/server/model/") && (stack.includes("R.") || stack.includes("knex")));
+
+    // For connection errors, only consider them database errors if:
+    // 1. They're from database code (stack trace check above), AND
+    // 2. They match database connection patterns
+    if (isFromDatabaseCode) {
+        // Check for connection errors that could be database-related
+        if (
+            error.code &&
+            (error.code === "EHOSTUNREACH" ||
+                error.code === "ECONNREFUSED" ||
+                error.code === "ETIMEDOUT" ||
+                error.code === "ENOTFOUND")
+        ) {
+            // For MariaDB/MySQL, verify it's connecting to the actual database host/port
+            if (Database.dbConfig && Database.dbConfig.type && Database.dbConfig.type.endsWith("mariadb")) {
+                const dbPort = Database.dbConfig.port || 3306;
+                const dbHost = Database.dbConfig.hostname;
+                // Only match if port matches database port OR address matches database hostname
+                if ((error.port && error.port === dbPort) || (error.address && error.address === dbHost)) {
+                    return true;
+                }
+            }
+            // For SQLite, connection errors from database code are database errors
+            if (Database.dbConfig && Database.dbConfig.type === "sqlite" && error.syscall === "connect") {
+                return true;
+            }
+        }
+
+        // Check for database-specific error messages (only if from database code)
+        if (
+            error.message &&
+            (error.message.includes("SQLITE_") ||
+                error.message.includes("SQLITE_ERROR") ||
+                error.message.includes("SQLITE_BUSY") ||
+                error.message.includes("SQLITE_LOCKED") ||
+                error.message.includes("database is locked") ||
+                error.message.includes("no such table") ||
+                (error.message.includes("Connection lost") && stack.includes("mysql")))
+        ) {
+            return true;
+        }
+    }
+
+    return false;
+}
+
+module.exports = {
+    isDatabaseError,
+};

server/jobs.js

@@ -1,8 +1,22 @@
 const { UptimeKumaServer } = require("./uptime-kuma-server");
 const { clearOldData } = require("./jobs/clear-old-data");
 const { incrementalVacuum } = require("./jobs/incremental-vacuum");
+const { Notification } = require("./notification");
 const Cron = require("croner");
 
+/**
+ * Refresh notification cache periodically
+ * @returns {Promise<void>}
+ */
+async function refreshNotificationCache() {
+    try {
+        await Notification.refreshCache();
+        Notification.resetDatabaseDownFlag();
+    } catch (e) {
+        // Silently fail - cache refresh is not critical
+    }
+}
+
 const jobs = [
     {
         name: "clear-old-data",
@@ -16,6 +30,12 @@ const jobs = [
         jobFunc: incrementalVacuum,
         croner: null,
     },
+    {
+        name: "refresh-notification-cache",
+        interval: "*/30 * * * *", // Every 30 minutes
+        jobFunc: refreshNotificationCache,
+        croner: null,
+    },
 ];
 
 /**

server/model/monitor.js

@@ -1129,17 +1129,35 @@ class Monitor extends BeanModel {
         };
 
         /**
-         * Get a heartbeat and handle errors7
+         * Get a heartbeat and handle errors
          * @returns {void}
          */
         const safeBeat = async () => {
             try {
                 await beat();
+                // If beat succeeds, database is likely back up, reset the flag
+                const { Notification } = require("../notification");
+                Notification.resetDatabaseDownFlag();
             } catch (e) {
                 console.trace(e);
                 UptimeKumaServer.errorLog(e, false);
                 log.error("monitor", "Please report to https://github.com/louislam/uptime-kuma/issues");
 
+                // Check if this is a database connection error (only if error is from database operations)
+                const { isDatabaseError } = require("../database-error-detector");
+                if (isDatabaseError(e)) {
+                    const errorMessage = e.message || `${e.code || "Unknown error"}`;
+                    log.error("monitor", `Database connection error detected in monitor: ${errorMessage}`);
+
+                    // Try to send notification using cached notifications
+                    try {
+                        const { Notification } = require("../notification");
+                        await Notification.sendDatabaseDownNotification(errorMessage);
+                    } catch (notifError) {
+                        log.error("monitor", `Failed to send database down notification: ${notifError.message}`);
+                    }
+                }
+
                 if (!this.isStop) {
                     log.info("monitor", "Try to restart the monitor");
                     this.heartbeatInterval = setTimeout(safeBeat, this.interval * 1000);

server/notification.js

@@ -88,6 +88,26 @@ const HaloPSA = require("./notification-providers/HaloPSA");
 
 class Notification {
     providerList = {};
+    /**
+     * Cache for all notifications to use when database is down
+     * @type {Array<object>}
+     */
+    static notificationCache = [];
+    /**
+     * Last time the cache was refreshed
+     * @type {number}
+     */
+    static cacheLastRefresh = 0;
+    /**
+     * Flag to track if we've already sent a database down notification
+     * @type {boolean}
+     */
+    static databaseDownNotificationSent = false;
+    /**
+     * Timestamp of when we last sent a database down notification
+     * @type {number}
+     */
+    static lastDatabaseDownNotificationTime = 0;
 
     /**
      * Initialize the notification providers
@@ -239,12 +259,15 @@ class Notification {
         bean.user_id = userID;
         bean.config = JSON.stringify(notification);
         bean.is_default = notification.isDefault || false;
+        bean.send_database_down = notification.sendDatabaseDown || false;
         await R.store(bean);
 
         if (notification.applyExisting) {
             await applyNotificationEveryMonitor(bean.id, userID);
         }
 
+        await this.refreshCacheSafely();
+
         return bean;
     }
 
@@ -262,6 +285,8 @@ class Notification {
         }
 
         await R.trash(bean);
+
+        await this.refreshCacheSafely();
     }
 
     /**
@@ -271,6 +296,107 @@ class Notification {
     static async checkApprise() {
         return await commandExists("apprise");
     }
+
+    /**
+     * Refresh cache safely, silently handling any errors
+     * @returns {Promise<void>}
+     */
+    static async refreshCacheSafely() {
+        try {
+            await this.refreshCache();
+        } catch (e) {
+            // Silently fail - cache refresh is not critical
+        }
+    }
+
+    /**
+     * Load all notifications into cache for use when database is down
+     * @returns {Promise<void>}
+     */
+    static async refreshCache() {
+        try {
+            // Get only notifications that are opted-in for database down notifications
+            const notifications = await R.find("notification", " active = 1 AND send_database_down = 1 ");
+
+            this.notificationCache = notifications.map((bean) => {
+                return {
+                    id: bean.id,
+                    name: bean.name,
+                    config: bean.config, // Store raw config string, parse when needed
+                    is_default: bean.is_default === 1,
+                    user_id: bean.user_id,
+                };
+            });
+
+            this.cacheLastRefresh = Date.now();
+            log.debug(
+                "notification",
+                `Refreshed notification cache with ${this.notificationCache.length} notifications (database down opt-in)`
+            );
+        } catch (e) {
+            log.error("notification", `Failed to refresh notification cache: ${e.message}`);
+            // Don't clear the cache if refresh fails, keep using old cache
+        }
+    }
+
+    /**
+     * Send notification about database being down using cached notifications
+     * @param {string} errorMessage Error message from database connection failure
+     * @returns {Promise<void>}
+     */
+    static async sendDatabaseDownNotification(errorMessage) {
+        const now = Date.now();
+        const COOLDOWN_PERIOD = 24 * 60 * 60 * 1000; // 24 hours cooldown between notifications
+
+        // Check cooldown period - don't spam notifications (especially important for SMS)
+        if (this.lastDatabaseDownNotificationTime > 0) {
+            const timeSinceLastNotification = now - this.lastDatabaseDownNotificationTime;
+            if (timeSinceLastNotification < COOLDOWN_PERIOD) {
+                log.debug(
+                    "notification",
+                    `Skipping database down notification - cooldown period active (${Math.round(timeSinceLastNotification / 3600000)}h / 24h)`
+                );
+                return;
+            }
+        }
+
+        // Check if cache is empty or too old (older than 1 hour)
+        const cacheAge = now - this.cacheLastRefresh;
+        if (this.notificationCache.length === 0 || cacheAge > 60 * 60 * 1000) {
+            log.warn("notification", "Notification cache is empty or too old, cannot send database down notification");
+            return;
+        }
+
+        this.databaseDownNotificationSent = true;
+        this.lastDatabaseDownNotificationTime = now;
+
+        const msg = `🔴 Uptime Kuma Database Connection Failed\n\nError: ${errorMessage}\n\nUptime Kuma is unable to connect to its database. Monitoring may be affected.`;
+
+        // Send to all cached notifications
+        for (const notification of this.notificationCache) {
+            try {
+                const config = JSON.parse(notification.config || "{}");
+                await this.send(config, msg);
+                log.info("notification", `Sent database down notification via ${notification.name} (${config.type})`);
+            } catch (e) {
+                log.error(
+                    "notification",
+                    `Failed to send database down notification via ${notification.name}: ${e.message}`
+                );
+            }
+        }
+    }
+
+    /**
+     * Reset the database down notification flag (call when database is back up)
+     * Only resets the flag, not the timestamp, to maintain cooldown period
+     * @returns {void}
+     */
+    static resetDatabaseDownFlag() {
+        this.databaseDownNotificationSent = false;
+        // Note: We don't reset lastDatabaseDownNotificationTime here to maintain
+        // the cooldown period even if database recovers and fails again
+    }
 }
 
 /**

server/server.js

@@ -230,6 +230,15 @@ let needSetup = false;
     server.entryPage = await Settings.get("entryPage");
     await StatusPage.loadDomainMappingList();
 
+    // Refresh notification cache after database is ready
+    try {
+        await Notification.refreshCache();
+        Notification.resetDatabaseDownFlag();
+        log.info("server", "Notification cache refreshed");
+    } catch (e) {
+        log.warn("server", `Failed to refresh notification cache on startup: ${e.message}`);
+    }
+
     log.debug("server", "Initializing Prometheus");
     await Prometheus.init();
 
@@ -1975,10 +1984,24 @@ gracefulShutdown(server.httpServer, {
 });
 
 // Catch unexpected errors here
-let unexpectedErrorHandler = (error, promise) => {
+let unexpectedErrorHandler = async (error, promise) => {
     console.trace(error);
     UptimeKumaServer.errorLog(error, false);
     console.error("If you keep encountering errors, please report to https://github.com/louislam/uptime-kuma/issues");
+
+    // Check if this is a database connection error (only if error is from database operations)
+    const { isDatabaseError } = require("./database-error-detector");
+    if (isDatabaseError(error)) {
+        const errorMessage = error.message || `${error.code || "Unknown error"}`;
+        log.error("server", `Database connection error detected: ${errorMessage}`);
+
+        // Try to send notification using cached notifications
+        try {
+            await Notification.sendDatabaseDownNotification(errorMessage);
+        } catch (e) {
+            log.error("server", `Failed to send database down notification: ${e.message}`);
+        }
+    }
 };
 process.addListener("unhandledRejection", unexpectedErrorHandler);
 process.addListener("uncaughtException", unexpectedErrorHandler);