diff --git a/client/src/Hooks/useMonitorForm.ts b/client/src/Hooks/useMonitorForm.ts index 963409fc8a..7f26d36a11 100644 --- a/client/src/Hooks/useMonitorForm.ts +++ b/client/src/Hooks/useMonitorForm.ts @@ -12,6 +12,8 @@ const getBaseDefaults = (data?: Monitor | null) => ({ description: data?.description || "", interval: data?.interval || 60000, notifications: data?.notifications || [], + escalationDelayMinutes: data?.escalationDelayMinutes ?? 0, + escalationChannels: data?.escalationChannels || [], statusWindowSize: data?.statusWindowSize || 5, statusWindowThreshold: data?.statusWindowThreshold || 60, geoCheckEnabled: data?.geoCheckEnabled ?? false, diff --git a/client/src/Pages/CreateMonitor/index.tsx b/client/src/Pages/CreateMonitor/index.tsx index 15b76eab36..36ee675935 100644 --- a/client/src/Pages/CreateMonitor/index.tsx +++ b/client/src/Pages/CreateMonitor/index.tsx @@ -765,6 +765,94 @@ const CreateMonitorPage = () => { } /> + + ( + field.onChange(Number(e.target.value))} + type="number" + fieldLabel={t( + "pages.createMonitor.form.escalation.option.delayMinutes.label" + )} + fullWidth + error={!!fieldState.error} + helperText={fieldState.error?.message ?? ""} + /> + )} + /> + { + const notificationOptions = (notifications ?? []).map((n) => ({ + ...n, + name: n.notificationName, + })); + const selectedEscalationChannels = notificationOptions.filter((n) => + (field.value ?? []).includes(n.id) + ); + return ( + + option.name} + onChange={(_: unknown, newValue: typeof notificationOptions) => { + field.onChange(newValue.map((n) => n.id)); + }} + isOptionEqualToValue={(option, value) => option.id === value.id} + fieldLabel={t( + "pages.createMonitor.form.escalation.option.channels.label" + )} + /> + {selectedEscalationChannels.length > 0 && ( + + {selectedEscalationChannels.map((notification, index) => ( + + + {notification.notificationName} + + { + field.onChange( + (field.value ?? []).filter( + (id: string) => id !== notification.id + ) + ); + }} + aria-label="Remove escalation channel" + > + + + {index < selectedEscalationChannels.length - 1 && ( + + )} + + ))} + + )} + + ); + }} + /> + + } + /> {(watchedType === "http" || watchedType === "grpc" || watchedType === "websocket") && ( diff --git a/client/src/Types/Monitor.ts b/client/src/Types/Monitor.ts index 053b517d1d..de4eb0343f 100644 --- a/client/src/Types/Monitor.ts +++ b/client/src/Types/Monitor.ts @@ -60,6 +60,8 @@ export interface Monitor { interval: number; uptimePercentage?: number; notifications: string[]; + escalationDelayMinutes?: number; + escalationChannels?: string[]; secret?: string; cpuAlertThreshold: number; cpuAlertCounter: number; diff --git a/client/src/Validation/monitor.ts b/client/src/Validation/monitor.ts index 9acffe6fed..7a96fa1f23 100644 --- a/client/src/Validation/monitor.ts +++ b/client/src/Validation/monitor.ts @@ -13,6 +13,8 @@ const baseSchema = z.object({ description: z.string().optional(), interval: z.number().min(15000, "Interval must be at least 15 seconds"), notifications: z.array(z.string()), + escalationDelayMinutes: z.number().min(0).optional(), + escalationChannels: z.array(z.string()).optional(), statusWindowSize: z .number({ message: "Status window size is required" }) .min(1, "Status window size must be at least 1") diff --git a/client/src/locales/en.json b/client/src/locales/en.json index 92a21939f3..aa55b4eee1 100644 --- a/client/src/locales/en.json +++ b/client/src/locales/en.json @@ -543,6 +543,18 @@ "description": "Select the notification channels you want to use", "title": "Notifications" }, + "escalation": { + "title": "Escalation Rules", + "description": "If the monitor stays down for the specified time, notify additional channels.", + "option": { + "delayMinutes": { + "label": "Escalate after (minutes)" + }, + "channels": { + "label": "Escalation notification channels" + } + } + }, "type": { "description": "Select the type of check to perform", "optionDockerDescription": "Use Docker to monitor if a container is running.", diff --git a/server/src/db/models/Monitor.ts b/server/src/db/models/Monitor.ts index 036aeadad6..3312fda427 100644 --- a/server/src/db/models/Monitor.ts +++ b/server/src/db/models/Monitor.ts @@ -18,11 +18,12 @@ type CheckSnapshotDocument = Omit & { createdAt: Dat type MonitorDocumentBase = Omit< Monitor, - "id" | "userId" | "teamId" | "notifications" | "selectedDisks" | "statusWindow" | "recentChecks" | "createdAt" | "updatedAt" + "id" | "userId" | "teamId" | "notifications" | "selectedDisks" | "statusWindow" | "recentChecks" | "createdAt" | "updatedAt" > & { statusWindow: boolean[]; recentChecks: CheckSnapshotDocument[]; notifications: Types.ObjectId[]; + escalationChannels: Types.ObjectId[]; selectedDisks: string[]; matchMethod?: MonitorMatchMethod; }; @@ -284,6 +285,16 @@ const MonitorSchema = new Schema( ref: "Notification", }, ], + escalationDelayMinutes: { + type: Number, + default: 0, + }, + escalationChannels: [ + { + type: Schema.Types.ObjectId, + ref: "Notification", + }, + ], secret: { type: String, }, diff --git a/server/src/repositories/monitors/MongoMonitorsRepository.ts b/server/src/repositories/monitors/MongoMonitorsRepository.ts index b2d7594483..32c7a3b334 100644 --- a/server/src/repositories/monitors/MongoMonitorsRepository.ts +++ b/server/src/repositories/monitors/MongoMonitorsRepository.ts @@ -351,6 +351,7 @@ class MongoMonitorsRepository implements IMonitorsRepository { }; const notificationIds = (doc.notifications ?? []).map((notification) => toStringId(notification)); + const escalationChannelIds = (doc.escalationChannels ?? []).map((id) => toStringId(id)); return { id: toStringId(doc._id), @@ -374,6 +375,8 @@ class MongoMonitorsRepository implements IMonitorsRepository { interval: doc.interval, uptimePercentage: doc.uptimePercentage ?? undefined, notifications: notificationIds, + escalationDelayMinutes: doc.escalationDelayMinutes ?? 0, + escalationChannels: escalationChannelIds, secret: doc.secret ?? undefined, cpuAlertThreshold: doc.cpuAlertThreshold, cpuAlertCounter: doc.cpuAlertCounter, @@ -410,6 +413,7 @@ class MongoMonitorsRepository implements IMonitorsRepository { }; const notificationIds = (doc.notifications ?? []).map((notification: unknown) => toStringId(notification)); + const escalationChannelIds = (doc.escalationChannels ?? []).map((id: unknown) => toStringId(id)); return { id: toStringId(doc._id), @@ -433,6 +437,8 @@ class MongoMonitorsRepository implements IMonitorsRepository { interval: doc.interval, uptimePercentage: doc.uptimePercentage ?? undefined, notifications: notificationIds, + escalationDelayMinutes: doc.escalationDelayMinutes ?? 0, + escalationChannels: escalationChannelIds, secret: doc.secret ?? undefined, cpuAlertThreshold: doc.cpuAlertThreshold, cpuAlertCounter: doc.cpuAlertCounter, diff --git a/server/src/service/infrastructure/SuperSimpleQueue/SuperSimpleQueueHelper.ts b/server/src/service/infrastructure/SuperSimpleQueue/SuperSimpleQueueHelper.ts index b6908127b2..bb527e7ab6 100644 --- a/server/src/service/infrastructure/SuperSimpleQueue/SuperSimpleQueueHelper.ts +++ b/server/src/service/infrastructure/SuperSimpleQueue/SuperSimpleQueueHelper.ts @@ -177,6 +177,43 @@ export class SuperSimpleQueueHelper implements ISuperSimpleQueueHelper { stack: error instanceof Error ? error.stack : undefined, }); }); + + //Step 8. Schedule escalation notification if monitor is down and escalation is configured + if ( + decision.shouldCreateIncident && + monitor.escalationChannels && + monitor.escalationChannels.length > 0 && + monitor.escalationDelayMinutes && + monitor.escalationDelayMinutes > 0 + ) { + const delayMs = monitor.escalationDelayMinutes * 60 * 1000; + setTimeout(async () => { + try { + const currentMonitor = await this.monitorsRepository.findById(monitor.id, monitor.teamId); + if (currentMonitor.status === "down" || currentMonitor.status === "breached") { + await this.notificationsService.handleEscalationNotifications(currentMonitor, status, decision); + this.logger.info({ + message: `Escalation notifications sent for monitor ${monitor.id} after ${monitor.escalationDelayMinutes} minutes`, + service: SERVICE_NAME, + method: "getMonitorJob", + }); + } else { + this.logger.debug({ + message: `Monitor ${monitor.id} recovered before escalation delay, skipping escalation`, + service: SERVICE_NAME, + method: "getMonitorJob", + }); + } + } catch (error: unknown) { + this.logger.error({ + message: `Error sending escalation notification for monitor ${monitor.id}: ${error instanceof Error ? error.message : "Unknown error"}`, + service: SERVICE_NAME, + method: "getMonitorJob", + stack: error instanceof Error ? error.stack : undefined, + }); + } + }, delayMs); + } } catch (error: unknown) { this.logger.warn({ message: error instanceof Error ? error.message : "Unknown error", diff --git a/server/src/service/infrastructure/notificationsService.ts b/server/src/service/infrastructure/notificationsService.ts index c75477c88c..2bc8f23c8a 100644 --- a/server/src/service/infrastructure/notificationsService.ts +++ b/server/src/service/infrastructure/notificationsService.ts @@ -14,7 +14,7 @@ export interface INotificationsService { updateById(id: string, teamId: string, updateData: Partial): Promise; deleteById: (id: string, teamId: string) => Promise; handleNotifications: (monitor: Monitor, monitorStatusResponse: MonitorStatusResponse, decision: MonitorActionDecision) => Promise; - + handleEscalationNotifications: (monitor: Monitor, monitorStatusResponse: MonitorStatusResponse, decision: MonitorActionDecision) => Promise; sendTestNotification: (notification: Partial) => Promise; testAllNotifications: (notificationIds: string[]) => Promise; } @@ -197,4 +197,38 @@ export class NotificationsService implements INotificationsService { await this.monitorsRepository.removeNotificationFromMonitors(id); return deleted; }; + + private sendEscalationNotifications = async (monitor: Monitor, monitorStatusResponse: MonitorStatusResponse, decision: MonitorActionDecision) => { + const escalationChannelIds = monitor.escalationChannels ?? []; + if (escalationChannelIds.length === 0) { + return false; + } + + const notifications = await this.notificationsRepository.findNotificationsByIds(escalationChannelIds); + + const settings = this.settingsService.getSettings(); + const clientHost = settings.clientHost || "Host not defined"; + const notificationMessage = this.notificationMessageBuilder.buildMessage(monitor, monitorStatusResponse, decision, clientHost); + + const tasks = notifications.map((notification) => this.send(notification, monitor, monitorStatusResponse, decision, notificationMessage)); + + const outcomes = await Promise.all(tasks); + const succeeded = outcomes.filter(Boolean).length; + const failed = outcomes.length - succeeded; + if (failed > 0) { + this.logger.warn({ + message: `Escalation notification send completed with ${succeeded} success, ${failed} failure(s)`, + service: SERVICE_NAME, + method: "sendEscalationNotifications", + }); + } + return succeeded === notifications.length; + }; + + handleEscalationNotifications = async (monitor: Monitor, monitorStatusResponse: MonitorStatusResponse, decision: MonitorActionDecision) => { + if (!decision.shouldSendNotification) { + return false; + } + return await this.sendEscalationNotifications(monitor, monitorStatusResponse, decision); + }; } diff --git a/server/src/types/monitor.ts b/server/src/types/monitor.ts index f29ce75d78..23c8653b1a 100644 --- a/server/src/types/monitor.ts +++ b/server/src/types/monitor.ts @@ -37,6 +37,8 @@ export interface Monitor { interval: number; uptimePercentage?: number; notifications: string[]; + escalationDelayMinutes?: number; + escalationChannels?: string[]; secret?: string; cpuAlertThreshold: number; cpuAlertCounter: number; diff --git a/server/src/validation/monitorValidation.ts b/server/src/validation/monitorValidation.ts index df000ecef2..24b2d21e1d 100644 --- a/server/src/validation/monitorValidation.ts +++ b/server/src/validation/monitorValidation.ts @@ -67,6 +67,8 @@ export const createMonitorBodyValidation = z.object({ diskAlertThreshold: z.number().optional(), tempAlertThreshold: z.number().optional(), notifications: z.array(z.string()).optional(), + escalationDelayMinutes: z.number().min(0).default(0), + escalationChannels: z.array(z.string()).optional(), secret: z.string().optional(), jsonPath: z.union([z.string(), z.literal("")]).optional(), expectedValue: z.union([z.string(), z.literal("")]).optional(), @@ -89,6 +91,8 @@ export const editMonitorBodyValidation = z.object({ description: z.union([z.string(), z.literal("")]).optional(), interval: z.number().optional(), notifications: z.array(z.string()).optional(), + escalationDelayMinutes: z.number().min(0).optional(), + escalationChannels: z.array(z.string()).optional(), secret: z.string().optional(), ignoreTlsErrors: z.boolean().optional(), useAdvancedMatching: z.boolean().optional(),