diff --git a/db/knex_migrations/2026-01-06-0331-add-send-database-down.js b/db/knex_migrations/2026-01-06-0331-add-send-database-down.js new file mode 100644 index 0000000000..e86f033c73 --- /dev/null +++ b/db/knex_migrations/2026-01-06-0331-add-send-database-down.js @@ -0,0 +1,11 @@ +exports.up = async function (knex) { + await knex.schema.alterTable("notification", (table) => { + table.boolean("send_database_down").notNullable().defaultTo(false); + }); +}; + +exports.down = async function (knex) { + await knex.schema.alterTable("notification", (table) => { + table.dropColumn("send_database_down"); + }); +}; diff --git a/server/database-error-detector.js b/server/database-error-detector.js new file mode 100644 index 0000000000..b88485895b --- /dev/null +++ b/server/database-error-detector.js @@ -0,0 +1,79 @@ +const Database = require("./database"); + +/** + * Check if an error is specifically from a database operation + * This is more restrictive to avoid false positives (e.g., notification send failures) + * @param {Error} error The error to check + * @returns {boolean} True if this is a database error + */ +function isDatabaseError(error) { + if (!error) { + return false; + } + + // Check for MySQL/MariaDB specific error codes (most reliable indicator) + if ( + error.code && + (error.code.startsWith("ER_") || // MySQL/MariaDB error codes + error.code === "PROTOCOL_CONNECTION_LOST" || + error.code === "PROTOCOL_ENQUEUE_AFTER_QUIT" || + error.code === "PROTOCOL_ENQUEUE_AFTER_FATAL_ERROR") + ) { + return true; + } + + // Check if error originated from database operations by examining stack trace + const stack = error.stack || ""; + const isFromDatabaseCode = + stack.includes("redbean-node") || + stack.includes("/server/database.js") || + (stack.includes("/server/model/") && (stack.includes("R.") || stack.includes("knex"))); + + // For connection errors, only consider them database errors if: + // 1. They're from database code (stack trace check above), AND + // 2. They match database connection patterns + if (isFromDatabaseCode) { + // Check for connection errors that could be database-related + if ( + error.code && + (error.code === "EHOSTUNREACH" || + error.code === "ECONNREFUSED" || + error.code === "ETIMEDOUT" || + error.code === "ENOTFOUND") + ) { + // For MariaDB/MySQL, verify it's connecting to the actual database host/port + if (Database.dbConfig && Database.dbConfig.type && Database.dbConfig.type.endsWith("mariadb")) { + const dbPort = Database.dbConfig.port || 3306; + const dbHost = Database.dbConfig.hostname; + // Only match if port matches database port OR address matches database hostname + if ((error.port && error.port === dbPort) || (error.address && error.address === dbHost)) { + return true; + } + } + // For SQLite, connection errors from database code are database errors + if (Database.dbConfig && Database.dbConfig.type === "sqlite" && error.syscall === "connect") { + return true; + } + } + + // Check for database-specific error messages (only if from database code) + if ( + error.message && + (error.message.includes("SQLITE_") || + error.message.includes("SQLITE_ERROR") || + error.message.includes("SQLITE_BUSY") || + error.message.includes("SQLITE_LOCKED") || + error.message.includes("database is locked") || + error.message.includes("no such table") || + (error.message.includes("Connection lost") && stack.includes("mysql"))) + ) { + return true; + } + } + + return false; +} + +module.exports = { + isDatabaseError, +}; diff --git a/server/jobs.js b/server/jobs.js index d9333fdb97..f01d209813 100644 --- a/server/jobs.js +++ b/server/jobs.js @@ -1,8 +1,22 @@ const { UptimeKumaServer } = require("./uptime-kuma-server"); const { clearOldData } = require("./jobs/clear-old-data"); const { incrementalVacuum } = require("./jobs/incremental-vacuum"); +const { Notification } = require("./notification"); const Cron = require("croner"); +/** + * Refresh notification cache periodically + * @returns {Promise} + */ +async function refreshNotificationCache() { + try { + await Notification.refreshCache(); + Notification.resetDatabaseDownFlag(); + } catch (e) { + // Silently fail - cache refresh is not critical + } +} + const jobs = [ { name: "clear-old-data", @@ -16,6 +30,12 @@ const jobs = [ jobFunc: incrementalVacuum, croner: null, }, + { + name: "refresh-notification-cache", + interval: "*/30 * * * *", // Every 30 minutes + jobFunc: refreshNotificationCache, + croner: null, + }, ]; /** diff --git a/server/model/monitor.js b/server/model/monitor.js index e01977133c..80af6258fa 100644 --- a/server/model/monitor.js +++ b/server/model/monitor.js @@ -1129,17 +1129,35 @@ class Monitor extends BeanModel { }; /** - * Get a heartbeat and handle errors7 + * Get a heartbeat and handle errors * @returns {void} */ const safeBeat = async () => { try { await beat(); + // If beat succeeds, database is likely back up, reset the flag + const { Notification } = require("../notification"); + Notification.resetDatabaseDownFlag(); } catch (e) { console.trace(e); UptimeKumaServer.errorLog(e, false); log.error("monitor", "Please report to https://github.com/louislam/uptime-kuma/issues"); + // Check if this is a database connection error (only if error is from database operations) + const { isDatabaseError } = require("../database-error-detector"); + if (isDatabaseError(e)) { + const errorMessage = e.message || `${e.code || "Unknown error"}`; + log.error("monitor", `Database connection error detected in monitor: ${errorMessage}`); + + // Try to send notification using cached notifications + try { + const { Notification } = require("../notification"); + await Notification.sendDatabaseDownNotification(errorMessage); + } catch (notifError) { + log.error("monitor", `Failed to send database down notification: ${notifError.message}`); + } + } + if (!this.isStop) { log.info("monitor", "Try to restart the monitor"); this.heartbeatInterval = setTimeout(safeBeat, this.interval * 1000); diff --git a/server/notification.js b/server/notification.js index de429388c0..465038f0da 100644 --- a/server/notification.js +++ b/server/notification.js @@ -88,6 +88,26 @@ const HaloPSA = require("./notification-providers/HaloPSA"); class Notification { providerList = {}; + /** + * Cache for all notifications to use when database is down + * @type {Array} + */ + static notificationCache = []; + /** + * Last time the cache was refreshed + * @type {number} + */ + static cacheLastRefresh = 0; + /** + * Flag to track if we've already sent a database down notification + * @type {boolean} + */ + static databaseDownNotificationSent = false; + /** + * Timestamp of when we last sent a database down notification + * @type {number} + */ + static lastDatabaseDownNotificationTime = 0; /** * Initialize the notification providers @@ -239,12 +259,15 @@ class Notification { bean.user_id = userID; bean.config = JSON.stringify(notification); bean.is_default = notification.isDefault || false; + bean.send_database_down = notification.sendDatabaseDown || false; await R.store(bean); if (notification.applyExisting) { await applyNotificationEveryMonitor(bean.id, userID); } + await this.refreshCacheSafely(); + return bean; } @@ -262,6 +285,8 @@ class Notification { } await R.trash(bean); + + await this.refreshCacheSafely(); } /** @@ -271,6 +296,107 @@ class Notification { static async checkApprise() { return await commandExists("apprise"); } + + /** + * Refresh cache safely, silently handling any errors + * @returns {Promise} + */ + static async refreshCacheSafely() { + try { + await this.refreshCache(); + } catch (e) { + // Silently fail - cache refresh is not critical + } + } + + /** + * Load all notifications into cache for use when database is down + * @returns {Promise} + */ + static async refreshCache() { + try { + // Get only notifications that are opted-in for database down notifications + const notifications = await R.find("notification", " active = 1 AND send_database_down = 1 "); + + this.notificationCache = notifications.map((bean) => { + return { + id: bean.id, + name: bean.name, + config: bean.config, // Store raw config string, parse when needed + is_default: bean.is_default === 1, + user_id: bean.user_id, + }; + }); + + this.cacheLastRefresh = Date.now(); + log.debug( + "notification", + `Refreshed notification cache with ${this.notificationCache.length} notifications (database down opt-in)` + ); + } catch (e) { + log.error("notification", `Failed to refresh notification cache: ${e.message}`); + // Don't clear the cache if refresh fails, keep using old cache + } + } + + /** + * Send notification about database being down using cached notifications + * @param {string} errorMessage Error message from database connection failure + * @returns {Promise} + */ + static async sendDatabaseDownNotification(errorMessage) { + const now = Date.now(); + const COOLDOWN_PERIOD = 24 * 60 * 60 * 1000; // 24 hours cooldown between notifications + + // Check cooldown period - don't spam notifications (especially important for SMS) + if (this.lastDatabaseDownNotificationTime > 0) { + const timeSinceLastNotification = now - this.lastDatabaseDownNotificationTime; + if (timeSinceLastNotification < COOLDOWN_PERIOD) { + log.debug( + "notification", + `Skipping database down notification - cooldown period active (${Math.round(timeSinceLastNotification / 3600000)}h / 24h)` + ); + return; + } + } + + // Check if cache is empty or too old (older than 1 hour) + const cacheAge = now - this.cacheLastRefresh; + if (this.notificationCache.length === 0 || cacheAge > 60 * 60 * 1000) { + log.warn("notification", "Notification cache is empty or too old, cannot send database down notification"); + return; + } + + this.databaseDownNotificationSent = true; + this.lastDatabaseDownNotificationTime = now; + + const msg = `🔴 Uptime Kuma Database Connection Failed\n\nError: ${errorMessage}\n\nUptime Kuma is unable to connect to its database. Monitoring may be affected.`; + + // Send to all cached notifications + for (const notification of this.notificationCache) { + try { + const config = JSON.parse(notification.config || "{}"); + await this.send(config, msg); + log.info("notification", `Sent database down notification via ${notification.name} (${config.type})`); + } catch (e) { + log.error( + "notification", + `Failed to send database down notification via ${notification.name}: ${e.message}` + ); + } + } + } + + /** + * Reset the database down notification flag (call when database is back up) + * Only resets the flag, not the timestamp, to maintain cooldown period + * @returns {void} + */ + static resetDatabaseDownFlag() { + this.databaseDownNotificationSent = false; + // Note: We don't reset lastDatabaseDownNotificationTime here to maintain + // the cooldown period even if database recovers and fails again + } } /** diff --git a/server/server.js b/server/server.js index 885e883403..7fe4e0c128 100644 --- a/server/server.js +++ b/server/server.js @@ -230,6 +230,15 @@ let needSetup = false; server.entryPage = await Settings.get("entryPage"); await StatusPage.loadDomainMappingList(); + // Refresh notification cache after database is ready + try { + await Notification.refreshCache(); + Notification.resetDatabaseDownFlag(); + log.info("server", "Notification cache refreshed"); + } catch (e) { + log.warn("server", `Failed to refresh notification cache on startup: ${e.message}`); + } + log.debug("server", "Initializing Prometheus"); await Prometheus.init(); @@ -1975,10 +1984,24 @@ gracefulShutdown(server.httpServer, { }); // Catch unexpected errors here -let unexpectedErrorHandler = (error, promise) => { +let unexpectedErrorHandler = async (error, promise) => { console.trace(error); UptimeKumaServer.errorLog(error, false); console.error("If you keep encountering errors, please report to https://github.com/louislam/uptime-kuma/issues"); + + // Check if this is a database connection error (only if error is from database operations) + const { isDatabaseError } = require("./database-error-detector"); + if (isDatabaseError(error)) { + const errorMessage = error.message || `${error.code || "Unknown error"}`; + log.error("server", `Database connection error detected: ${errorMessage}`); + + // Try to send notification using cached notifications + try { + await Notification.sendDatabaseDownNotification(errorMessage); + } catch (e) { + log.error("server", `Failed to send database down notification: ${e.message}`); + } + } }; process.addListener("unhandledRejection", unexpectedErrorHandler); process.addListener("uncaughtException", unexpectedErrorHandler); diff --git a/src/components/NotificationDialog.vue b/src/components/NotificationDialog.vue index 6d915255a3..4be3b400d1 100644 --- a/src/components/NotificationDialog.vue +++ b/src/components/NotificationDialog.vue @@ -128,6 +128,20 @@ + +
+ +
+ + +
+
+ {{ $t("sendDatabaseDownNotificationDescription") }} +
@@ -410,6 +424,13 @@ export default { for (let n of this.$root.notificationList) { if (n.id === notificationID) { this.notification = JSON.parse(n.config); + // Load send_database_down from database (not stored in config) + if (n.send_database_down !== undefined) { + this.notification.sendDatabaseDown = + n.send_database_down === 1 || n.send_database_down === true; + } else { + this.notification.sendDatabaseDown = false; + } break; } } @@ -419,6 +440,7 @@ export default { name: "", type: "telegram", isDefault: false, + sendDatabaseDown: false, }; } diff --git a/src/lang/en.json b/src/lang/en.json index ea1ea35a8f..9410cbb7a4 100644 --- a/src/lang/en.json +++ b/src/lang/en.json @@ -655,6 +655,8 @@ "rrtypeDescription": "Select the RR type you want to monitor", "pauseMonitorMsg": "Are you sure want to pause?", "enableDefaultNotificationDescription": "This notification will be enabled by default for new monitors. You can still disable the notification separately for each monitor.", + "Send database down notifications": "Send database down notifications", + "sendDatabaseDownNotificationDescription": "Enable this notification provider to receive alerts when Uptime Kuma's database connection fails. This helps you stay informed about critical system issues even when the database is unavailable. Notifications are sent with a 24-hour cooldown period to prevent spam.", "Clear All Events": "Clear All Events", "clearAllEventsMsg": "Are you sure want to delete all events?", "Events cleared successfully": "Events cleared successfully.", diff --git a/test/backend-test/test-database-down-notification.js b/test/backend-test/test-database-down-notification.js new file mode 100644 index 0000000000..a9ae2f1b93 --- /dev/null +++ b/test/backend-test/test-database-down-notification.js @@ -0,0 +1,201 @@ +const { describe, test, before, after } = require("node:test"); +const assert = require("node:assert"); +const { R } = require("redbean-node"); +const { Notification } = require("../../server/notification"); +const Database = require("../../server/database"); + +describe("Database Down Notification", () => { + let testNotification; + + before(async () => { + // Initialize data directory first (required before connecting) + Database.initDataDir({}); + + // Ensure database is connected (this copies template DB which has basic tables) + await Database.connect(true); // testMode = true + + // Ensure notification table exists with all required columns + // The template DB might be outdated, so we need to ensure schema is complete + const hasNotificationTable = await R.hasTable("notification"); + if (!hasNotificationTable) { + // If table doesn't exist, create all tables first + const { createTables } = require("../../db/knex_init_db.js"); + await createTables(); + } else { + // Table exists, but check if it has all required columns (template DB might be outdated) + // Add missing columns if they don't exist + const hasIsDefault = await R.knex.schema.hasColumn("notification", "is_default"); + if (!hasIsDefault) { + await R.knex.schema.alterTable("notification", (table) => { + table.boolean("is_default").notNullable().defaultTo(false); + }); + } + } + + // Run migrations to ensure schema is current (including send_database_down column) + // Database.patch() handles migrations properly with foreign key checks + try { + await Database.patch(undefined, undefined); + } catch (e) { + // Some migrations may fail if tables don't exist in template DB, that's okay + // But we still need to ensure our column exists, so add it manually if migration failed + if (!e.message.includes("the following files are missing:")) { + console.warn("Migration warning (may be expected):", e.message); + // Fallback: ensure the column exists if migration didn't complete + const hasColumn = await R.knex.schema.hasColumn("notification", "send_database_down"); + if (!hasColumn) { + await R.knex.schema.alterTable("notification", (table) => { + table.boolean("send_database_down").notNullable().defaultTo(false); + }); + } + } + } + + // Create a test notification with send_database_down enabled (opt-in) + const notificationBean = R.dispense("notification"); + notificationBean.name = "Test Notification"; + notificationBean.user_id = 1; + notificationBean.config = JSON.stringify({ + type: "webhook", + webhookURL: "https://example.com/webhook", + }); + notificationBean.active = 1; + notificationBean.is_default = 0; + notificationBean.send_database_down = 1; // Opt-in for database down notifications + await R.store(notificationBean); + testNotification = notificationBean; + }); + + after(async () => { + // Clean up test notification + if (testNotification) { + try { + await R.trash(testNotification); + } catch (e) { + // Ignore cleanup errors + } + } + await Database.close(); + }); + + test("refreshCache() loads only opt-in notifications into cache", async () => { + // Create a notification that is NOT opted-in + const nonOptInBean = R.dispense("notification"); + nonOptInBean.name = "Non-Opt-In Notification"; + nonOptInBean.user_id = 1; + nonOptInBean.config = JSON.stringify({ type: "webhook", webhookURL: "https://example.com/webhook2" }); + nonOptInBean.active = 1; + nonOptInBean.is_default = 0; + nonOptInBean.send_database_down = 0; // NOT opted-in + await R.store(nonOptInBean); + + try { + await Notification.refreshCache(); + + assert.ok(Notification.notificationCache.length > 0, "Cache should contain notifications"); + assert.ok(Notification.cacheLastRefresh > 0, "Cache refresh time should be set"); + + // Verify test notification (opt-in) is in cache + const cached = Notification.notificationCache.find((n) => n.id === testNotification.id); + assert.ok(cached, "Opt-in notification should be in cache"); + assert.strictEqual(cached.name, "Test Notification"); + // Config is stored as raw string, parse to verify + const config = JSON.parse(cached.config); + assert.strictEqual(config.type, "webhook"); + + // Verify non-opt-in notification is NOT in cache + const nonOptInCached = Notification.notificationCache.find((n) => n.id === nonOptInBean.id); + assert.strictEqual(nonOptInCached, undefined, "Non-opt-in notification should NOT be in cache"); + } finally { + // Clean up + await R.trash(nonOptInBean); + } + }); + + test("sendDatabaseDownNotification() uses cached notifications and prevents duplicates", async () => { + // Ensure cache is populated + await Notification.refreshCache(); + assert.ok(Notification.notificationCache.length > 0, "Cache should be populated"); + + // Reset the flag + Notification.resetDatabaseDownFlag(); + assert.strictEqual(Notification.databaseDownNotificationSent, false); + + // Mock the send method to track calls + let sendCallCount = 0; + const originalSend = Notification.send; + Notification.send = async (notification, msg) => { + sendCallCount++; + assert.ok(msg.includes("Database Connection Failed"), "Message should mention database failure"); + return "OK"; + }; + + try { + // First call should send + await Notification.sendDatabaseDownNotification("Test database error: ECONNREFUSED"); + assert.ok(sendCallCount > 0, "send() should have been called"); + assert.strictEqual(Notification.databaseDownNotificationSent, true, "Flag should be set"); + + const firstCallCount = sendCallCount; + + // Second call should not send again (duplicate prevention) + await Notification.sendDatabaseDownNotification("Test error 2"); + assert.strictEqual(sendCallCount, firstCallCount, "Should not send again on second call"); + } finally { + // Restore original send method + Notification.send = originalSend; + } + }); + + test("sendDatabaseDownNotification() handles empty cache gracefully", async () => { + // Clear cache + Notification.notificationCache = []; + Notification.cacheLastRefresh = 0; + Notification.resetDatabaseDownFlag(); + + // Should not throw + await Notification.sendDatabaseDownNotification("Test error"); + + // Flag should remain false since cache is empty + assert.strictEqual(Notification.databaseDownNotificationSent, false); + }); + + test("resetDatabaseDownFlag() resets the notification flag", () => { + Notification.databaseDownNotificationSent = true; + Notification.resetDatabaseDownFlag(); + assert.strictEqual(Notification.databaseDownNotificationSent, false); + }); + + test("refreshCache() handles database errors gracefully", async () => { + // Ensure cache is populated first + await Notification.refreshCache(); + const originalCacheLength = Notification.notificationCache.length; + const originalCacheItems = JSON.parse(JSON.stringify(Notification.notificationCache)); // Deep copy + + // Temporarily mock R.find to throw an error + const originalFind = R.find; + R.find = async () => { + throw new Error("Database connection lost"); + }; + + try { + // Should not throw + await Notification.refreshCache(); + + // Cache should remain unchanged (not cleared) + assert.ok(Array.isArray(Notification.notificationCache), "Cache should still be an array"); + assert.strictEqual( + Notification.notificationCache.length, + originalCacheLength, + "Cache should have same length" + ); + assert.deepStrictEqual( + Notification.notificationCache, + originalCacheItems, + "Cache should contain same items" + ); + } finally { + R.find = originalFind; + } + }); +});