diff --git a/.changeset/silver-forks-rhyme.md b/.changeset/silver-forks-rhyme.md new file mode 100644 index 000000000..e7557ce27 --- /dev/null +++ b/.changeset/silver-forks-rhyme.md @@ -0,0 +1,5 @@ +--- +"@hyperdx/api": patch +--- + +fix: set a max size for alert timeranges diff --git a/packages/api/src/tasks/__tests__/util.test.ts b/packages/api/src/tasks/__tests__/util.test.ts index 8c9d261c0..fb44f0896 100644 --- a/packages/api/src/tasks/__tests__/util.test.ts +++ b/packages/api/src/tasks/__tests__/util.test.ts @@ -1,4 +1,5 @@ import { + calcAlertDateRange, escapeJsonString, roundDownTo, roundDownToXMinutes, @@ -335,4 +336,192 @@ describe('util', () => { expect(escapeJsonString('foo\u0000bar')).toBe('foo\\u0000bar'); }); }); + + describe('calcAlertDateRange', () => { + const now = Date.now(); + const oneMinuteMs = 60 * 1000; + const oneHourMs = 60 * oneMinuteMs; + + it('should return unchanged dates when range is within limits', () => { + const startTime = now - 10 * oneMinuteMs; // 10 minutes ago + const endTime = now; + const windowSizeInMins = 5; + + const [start, end] = calcAlertDateRange( + startTime, + endTime, + windowSizeInMins, + ); + + expect(start.getTime()).toBe(startTime); + expect(end.getTime()).toBe(endTime); + }); + + it('should truncate start time when too many windows (> 50)', () => { + const windowSizeInMins = 1; + const maxWindows = 50; + const tooManyWindowsMs = + (maxWindows + 10) * windowSizeInMins * oneMinuteMs; // 60 minutes + const startTime = now - tooManyWindowsMs; + const endTime = now; + + const [start, end] = calcAlertDateRange( + startTime, + endTime, + windowSizeInMins, + ); + + // Should truncate to exactly 50 windows + const expectedStartTime = + endTime - maxWindows * windowSizeInMins * oneMinuteMs; + expect(start.getTime()).toBe(expectedStartTime); + expect(end.getTime()).toBe(endTime); + }); + + it('should truncate start time when time range exceeds 6 hours for short windows (< 15 mins)', () => { + const windowSizeInMins = 10; + const maxLookbackTime = 6 * oneHourMs; // 6 hours for windows < 15 minutes + const tooLongRangeMs = maxLookbackTime + oneHourMs; // 7 hours + const startTime = now - tooLongRangeMs; + const endTime = now; + + const [start, end] = calcAlertDateRange( + startTime, + endTime, + windowSizeInMins, + ); + + const expectedStartTime = endTime - maxLookbackTime; + expect(start.getTime()).toBeGreaterThan(startTime); + expect(start.getTime()).toBe(expectedStartTime); + expect(end.getTime()).toBe(endTime); + }); + + it('should truncate start time when time range exceeds 24 hours for long windows (>= 15 mins)', () => { + const windowSizeInMins = 30; + const maxLookbackTime = 24 * oneHourMs; // 24 hours for windows >= 15 minutes + const tooLongRangeMs = maxLookbackTime + 2 * oneHourMs; // 26 hours + const startTime = now - tooLongRangeMs; + const endTime = now; + + const [start, end] = calcAlertDateRange( + startTime, + endTime, + windowSizeInMins, + ); + + const expectedStartTime = endTime - maxLookbackTime; + expect(start.getTime()).toBe(expectedStartTime); + expect(end.getTime()).toBe(endTime); + }); + + it('should apply the more restrictive truncation when both limits are exceeded', () => { + const windowSizeInMins = 1; + const maxWindows = 50; + const maxLookbackTime = 6 * oneHourMs; // 6 hours for 1-minute windows + + // Create a range that exceeds both limits + const excessiveRangeMs = Math.max( + (maxWindows + 100) * windowSizeInMins * oneMinuteMs, // 150 windows + maxLookbackTime + 2 * oneHourMs, // 8 hours + ); + const startTime = now - excessiveRangeMs; + const endTime = now; + + const [start, end] = calcAlertDateRange( + startTime, + endTime, + windowSizeInMins, + ); + + // Should use the more restrictive limit (maxWindows in this case) + const expectedStartTime = + endTime - maxWindows * windowSizeInMins * oneMinuteMs; + expect(start.getTime()).toBe(expectedStartTime); + expect(end.getTime()).toBe(endTime); + }); + + it('should handle very large window sizes correctly', () => { + const windowSizeInMins = 120; // 2 hours + const maxLookbackTime = 24 * oneHourMs; // 24 hours for large windows + const normalRange = 12 * oneHourMs; // 12 hours - within limit + const startTime = now - normalRange; + const endTime = now; + + const [start, end] = calcAlertDateRange( + startTime, + endTime, + windowSizeInMins, + ); + + // Should remain unchanged since within limits + expect(start.getTime()).toBe(startTime); + expect(end.getTime()).toBe(endTime); + }); + + it('should handle exactly 50 windows without truncation', () => { + const windowSizeInMins = 5; + const maxWindows = 50; + const exactlyMaxWindowsMs = maxWindows * windowSizeInMins * oneMinuteMs; // 250 minutes + const startTime = now - exactlyMaxWindowsMs; + const endTime = now; + + const [start, end] = calcAlertDateRange( + startTime, + endTime, + windowSizeInMins, + ); + + // Should remain unchanged since exactly at the limit + expect(start.getTime()).toBe(startTime); + expect(end.getTime()).toBe(endTime); + }); + + it('should handle fractional windows correctly', () => { + const windowSizeInMins = 7; + const partialWindowsMs = 7.5 * windowSizeInMins * oneMinuteMs; // 7.5 windows + const startTime = now - partialWindowsMs; + const endTime = now; + + const [start, end] = calcAlertDateRange( + startTime, + endTime, + windowSizeInMins, + ); + + // Should remain unchanged since well within limits + expect(start.getTime()).toBe(startTime); + expect(end.getTime()).toBe(endTime); + }); + + it('should handle zero time range', () => { + const startTime = now; + const endTime = now; + const windowSizeInMins = 5; + + const [start, end] = calcAlertDateRange( + startTime, + endTime, + windowSizeInMins, + ); + + expect(start.getTime()).toBe(startTime); + expect(end.getTime()).toBe(endTime); + }); + + it('should return Date objects', () => { + const startTime = now - 10 * oneMinuteMs; + const endTime = now; + const windowSizeInMins = 5; + + const [start, end] = calcAlertDateRange( + startTime, + endTime, + windowSizeInMins, + ); + + expect(start).toBeInstanceOf(Date); + expect(end).toBeInstanceOf(Date); + }); + }); }); diff --git a/packages/api/src/tasks/checkAlerts.ts b/packages/api/src/tasks/checkAlerts.ts index ccbe486aa..b93ce7b66 100644 --- a/packages/api/src/tasks/checkAlerts.ts +++ b/packages/api/src/tasks/checkAlerts.ts @@ -36,7 +36,11 @@ import { renderAlertTemplate, } from '@/tasks/template'; import { CheckAlertsTaskArgs, HdxTask } from '@/tasks/types'; -import { roundDownToXMinutes, unflattenObject } from '@/tasks/util'; +import { + calcAlertDateRange, + roundDownToXMinutes, + unflattenObject, +} from '@/tasks/util'; import logger from '@/utils/logger'; import { tasksTracer } from './tracer'; @@ -171,10 +175,14 @@ export const processAlert = async ( ); return; } - const checkStartTime = previous - ? previous.createdAt - : fns.subMinutes(nowInMinsRoundDown, windowSizeInMins); - const checkEndTime = nowInMinsRoundDown; + const dateRange = calcAlertDateRange( + (previous + ? previous.createdAt + : fns.subMinutes(nowInMinsRoundDown, windowSizeInMins) + ).getTime(), + nowInMinsRoundDown.getTime(), + windowSizeInMins, + ); let chartConfig: ChartConfigWithOptDateRange | undefined; if (details.taskType === AlertTaskType.SAVED_SEARCH) { @@ -182,7 +190,7 @@ export const processAlert = async ( chartConfig = { connection: connectionId, displayType: DisplayType.Line, - dateRange: [checkStartTime, checkEndTime], + dateRange, dateRangeStartInclusive: true, dateRangeEndInclusive: false, from: source.from, @@ -206,7 +214,7 @@ export const processAlert = async ( if (tile.config.displayType === DisplayType.Line) { chartConfig = { connection: connectionId, - dateRange: [checkStartTime, checkEndTime], + dateRange, dateRangeStartInclusive: true, dateRangeEndInclusive: false, displayType: tile.config.displayType, @@ -252,9 +260,10 @@ export const processAlert = async ( logger.info( { alertId: alert.id, + chartConfig, checksData, - checkStartTime, - checkEndTime, + checkStartTime: dateRange[0], + checkEndTime: dateRange[1], }, `Received alert metric [${alert.source} source]`, ); diff --git a/packages/api/src/tasks/util.ts b/packages/api/src/tasks/util.ts index 95d29ea14..ce6cbc5db 100644 --- a/packages/api/src/tasks/util.ts +++ b/packages/api/src/tasks/util.ts @@ -1,5 +1,7 @@ import { set } from 'lodash'; +import logger from '@/utils/logger'; + // transfer keys of attributes with dot into nested object // ex: { 'a.b': 'c', 'd.e.f': 'g' } -> { a: { b: 'c' }, d: { e: { f: 'g' } } } export const unflattenObject = ( @@ -38,3 +40,46 @@ export const roundDownToXMinutes = (x: number) => roundDownTo(1000 * 60 * x); export const escapeJsonString = (str: string) => { return JSON.stringify(str).slice(1, -1); }; + +const MAX_NUM_WINDOWS = 50; +const maxLookbackTime = (windowSizeInMins: number) => + 3600_000 * (windowSizeInMins < 15 ? 6 : 24); +export function calcAlertDateRange( + _startTime: number, + _endTime: number, + windowSizeInMins: number, +): [Date, Date] { + let startTime = _startTime; + const endTime = _endTime; + const numWindows = (endTime - startTime) / 60_000 / windowSizeInMins; + // Truncate if too many windows are present + if (numWindows > MAX_NUM_WINDOWS) { + startTime = endTime - MAX_NUM_WINDOWS * 1000 * 60 * windowSizeInMins; + logger.info( + { + requestedStartTime: _startTime, + startTime, + endTime, + windowSizeInMins, + numWindows, + }, + 'startTime truncated due to too many windows', + ); + } + // Truncate if time range is over threshold + const MAX_LOOKBACK_TIME = maxLookbackTime(windowSizeInMins); + if (endTime - startTime > MAX_LOOKBACK_TIME) { + startTime = endTime - MAX_LOOKBACK_TIME; + logger.info( + { + requestedStartTime: _startTime, + startTime, + endTime, + windowSizeInMins, + numWindows, + }, + 'startTime truncated due to long lookback time', + ); + } + return [new Date(startTime), new Date(endTime)]; +}