diff --git a/.github/workflows/slo-weekly-report.yml b/.github/workflows/slo-weekly-report.yml new file mode 100644 index 00000000..9389d13a --- /dev/null +++ b/.github/workflows/slo-weekly-report.yml @@ -0,0 +1,30 @@ +name: SLO Weekly Report + +on: + schedule: + # Every Monday at 09:00 UTC + - cron: "0 9 * * 1" + workflow_dispatch: + +jobs: + slo-report: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: "24" + cache: "npm" + cache-dependency-path: package-lock.json + + - name: Install dependencies + run: npm ci + + - name: Run SLO weekly report + env: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} + PROMETHEUS_URL: ${{ secrets.PROMETHEUS_URL }} + BURN_RATE_DASHBOARD_URL: ${{ secrets.BURN_RATE_DASHBOARD_URL }} + run: npx tsx scripts/slo-weekly-report.ts diff --git a/docs/observability.md b/docs/observability.md index f3b3d447..553048eb 100644 --- a/docs/observability.md +++ b/docs/observability.md @@ -20,6 +20,10 @@ ChronoPay uses `X-Request-Id` for end-to-end request correlation. ## Service Level Objectives (SLOs) +Route-level objectives and burn-rate metrics are defined in `src/metrics/sloMetrics.ts` (`booking_intent`, `slots_list`, `checkout`, `escrow_listener`). + +A **weekly error-budget report** is posted to Slack every Monday (see [slo-weekly-report.md](./slo-weekly-report.md) and `ops/dashboards/slo-burn-rate.json`). + ### Booking Intents - **Availability & Latency:** 99.9% of booking-intent creates (`POST /api/v1/booking-intents`) complete in under 500ms over a rolling 30-day window. diff --git a/docs/slo-weekly-report.md b/docs/slo-weekly-report.md new file mode 100644 index 00000000..23c6d3e9 --- /dev/null +++ b/docs/slo-weekly-report.md @@ -0,0 +1,73 @@ +# Weekly SLO Error-Budget Report + +Automated weekly Slack summary of SLO error-budget consumption per route. + +## Routes and objectives + +Aligned with `src/metrics/sloMetrics.ts`: + +| Route | Objective | +|-------|-----------| +| `booking_intent` | 99.9% | +| `slots_list` | 99.5% | +| `checkout` | 99.99% | +| `escrow_listener` | 99% | + +## Budget math + +For each route over the past 7 days: + +``` +errorBudget = 1 − sloObjective +observedErrorRate = badEvents / totalRequests (0 when no traffic) +consumedFraction = observedErrorRate / errorBudget +remainingBudget = clamp(1 − consumedFraction, 0, 1) +``` + +Weekly consumption shown in Slack is `consumedFraction × 100%` of the error budget. + +## Data sources + +The script (`scripts/slo-weekly-report.ts`) resolves data in order: + +1. **Prometheus counters** — `increase(slo_requests_total[7d])` and `increase(slo_bad_events_total[7d])` (or `slo_route_*` variants) when present. +2. **`slo_burn_rate` gauge** — max over the week per route when counters are absent. +3. **No-traffic defaults** — all four routes at 100% remaining with a _no traffic_ marker when Prometheus is empty or unreachable. + +## Slack message + +- Per-route leaderboard sorted worst-first (lowest remaining budget) +- Remaining budget %, weekly consumption %, status (Exhausted / Critical / At Risk / Healthy) +- Link to the [SLO burn-rate dashboard](../ops/dashboards/slo-burn-rate.json) (`uid: slo-burn-rate`) + +## Running locally + +```bash +SLACK_WEBHOOK_URL=https://hooks.slack.com/services/... \ +PROMETHEUS_URL=http://localhost:9090 \ +BURN_RATE_DASHBOARD_URL=https://grafana.example.com/d/slo-burn-rate \ +npx tsx scripts/slo-weekly-report.ts +``` + +Environment defaults (when unset): + +- `PROMETHEUS_URL` — empty (posts no-traffic defaults) +- `BURN_RATE_DASHBOARD_URL` — `https://grafana.example.com/d/slo-burn-rate` + +## CI schedule + +GitHub Actions workflow `.github/workflows/slo-weekly-report.yml` runs every **Monday at 09:00 UTC** and supports manual `workflow_dispatch`. + +Required secrets: + +- `SLACK_WEBHOOK_URL` (required) +- `PROMETHEUS_URL` (optional) +- `BURN_RATE_DASHBOARD_URL` (optional) + +Slack delivery failures throw so the cron job fails loudly. + +## Tests + +```bash +npm test -- --testPathPattern='slo-weekly-report' --coverage=false +``` diff --git a/ops/dashboards/slo-burn-rate.json b/ops/dashboards/slo-burn-rate.json new file mode 100644 index 00000000..3866caf5 --- /dev/null +++ b/ops/dashboards/slo-burn-rate.json @@ -0,0 +1,186 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "slo_burn_rate{window=\"5m\"}", + "legendFormat": "{{route}} (5m)", + "range": true, + "refId": "A" + } + ], + "title": "SLO Burn Rate — 5m window", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 8 + }, + "id": 2, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "slo_burn_rate{window=\"1h\"}", + "legendFormat": "{{route}} (1h)", + "range": true, + "refId": "A" + } + ], + "title": "SLO Burn Rate — 1h window", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 16 + }, + "id": 3, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "slo_burn_rate{window=\"6h\"}", + "legendFormat": "{{route}} (6h)", + "range": true, + "refId": "A" + } + ], + "title": "SLO Burn Rate — 6h window", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 24 + }, + "id": 4, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "max by (route) (slo_burn_rate)", + "legendFormat": "{{route}}", + "range": true, + "refId": "A" + } + ], + "title": "Max Burn Rate by Route", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "yellow", "value": 0.5 }, + { "color": "red", "value": 1 } + ] + } + } + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 24 + }, + "id": 5, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "clamp_max(1 - max by (route) (max_over_time(slo_burn_rate[7d])), 1)", + "legendFormat": "{{route}}", + "range": true, + "refId": "A" + } + ], + "title": "Estimated Weekly Budget Remaining", + "type": "stat" + } + ], + "title": "SLO Burn Rate", + "uid": "slo-burn-rate", + "version": 1 +} diff --git a/scripts/__tests__/slo-weekly-report.test.ts b/scripts/__tests__/slo-weekly-report.test.ts new file mode 100644 index 00000000..459a0ee7 --- /dev/null +++ b/scripts/__tests__/slo-weekly-report.test.ts @@ -0,0 +1,275 @@ +import { describe, it, expect, jest } from "@jest/globals"; +import { + budgetBar, + budgetStatus, + buildSlackReport, + computeRouteBudget, + computeWeeklyBudgets, + defaultWeeklyBudgets, + fetchWeeklySloData, + postToSlack, + runReport, + SloData, +} from "../slo-weekly-report.js"; + +describe("budgetBar", () => { + it("renders all filled for 1.0", () => { + expect(budgetBar(1.0)).toBe("██████████"); + }); + + it("renders all empty for 0.0", () => { + expect(budgetBar(0.0)).toBe("░░░░░░░░░░"); + }); + + it("renders half filled for 0.5", () => { + expect(budgetBar(0.5)).toBe("█████░░░░░"); + }); + + it("clamps to 10 segments", () => { + expect(budgetBar(1.5)).toBe("██████████"); + expect(budgetBar(-0.5)).toBe("░░░░░░░░░░"); + }); +}); + +describe("budgetStatus", () => { + it("returns Exhausted for 0", () => { + expect(budgetStatus(0)).toContain("Exhausted"); + }); + + it("returns Critical below 0.2", () => { + expect(budgetStatus(0.1)).toContain("Critical"); + expect(budgetStatus(0.19)).toContain("Critical"); + }); + + it("returns At Risk between 0.2 and 0.5", () => { + expect(budgetStatus(0.3)).toContain("At Risk"); + expect(budgetStatus(0.49)).toContain("At Risk"); + }); + + it("returns Healthy at or above 0.5", () => { + expect(budgetStatus(0.5)).toContain("Healthy"); + expect(budgetStatus(0.9)).toContain("Healthy"); + }); +}); + +describe("computeWeeklyBudgets", () => { + it("computes remaining budget from counts and objectives", () => { + // booking_intent SLO 0.999 → error budget 0.001 + // 1000 requests, 2 errors → error rate 0.002 → consumed 2x budget + const result = computeRouteBudget("booking_intent", 1000, 2); + expect(result.consumedFraction).toBeCloseTo(2, 5); + expect(result.remainingBudget).toBe(0); + expect(result.sloObjective).toBe(0.999); + }); + + it("returns 100% remaining for routes with no traffic", () => { + const budgets = computeWeeklyBudgets([ + { route: "checkout", totalRequests: 500, badEvents: 0 }, + ]); + const checkout = budgets.find((b) => b.route === "checkout"); + expect(checkout?.remainingBudget).toBe(1); + expect(checkout?.noTraffic).toBe(false); + expect(budgets.find((b) => b.route === "slots_list")?.noTraffic).toBe(true); + }); + + it("covers all four reported routes", () => { + const budgets = computeWeeklyBudgets([]); + expect(budgets).toHaveLength(4); + expect(budgets.every((b) => b.remainingBudget === 1 && b.noTraffic)).toBe(true); + }); +}); + +describe("buildSlackReport", () => { + it("returns a message with header and worst-first leaderboard", () => { + const slos: SloData[] = [ + { + route: "checkout", + totalRequests: 1000, + badEvents: 5, + sloObjective: 0.9999, + observedErrorRate: 0.005, + consumedFraction: 50, + remainingBudget: 0.5, + noTraffic: false, + }, + { + route: "slots_list", + totalRequests: 500, + badEvents: 50, + sloObjective: 0.995, + observedErrorRate: 0.1, + consumedFraction: 2, + remainingBudget: 0.1, + noTraffic: false, + }, + ]; + + const result = buildSlackReport(slos, "https://grafana.example.com/d/slo-burn-rate"); + + expect(result.text).toContain("critically low"); + const sectionBlocks = result.blocks.filter( + (b) => b.type === "section" && (b.text as { text?: string })?.text, + ); + const leaderboardText = (sectionBlocks[1].text as { text: string }).text; + + expect(leaderboardText.indexOf("slots_list")).toBeLessThan( + leaderboardText.indexOf("checkout"), + ); + expect(leaderboardText).toContain("5000.0%` consumed"); + expect(leaderboardText).toContain("`10.0%` remaining"); + + const contextBlock = result.blocks.find((b) => b.type === "context"); + expect(contextBlock).toBeDefined(); + expect( + (contextBlock!.elements as Array<{ text: string }>)[0].text, + ).toContain("slo-burn-rate"); + }); + + it("shows no-traffic markers for an empty week", () => { + const slos = defaultWeeklyBudgets(); + const result = buildSlackReport(slos, "https://grafana.example.com/d/slo-burn-rate"); + const sectionBlocks = result.blocks.filter( + (b) => b.type === "section" && (b.text as { text?: string })?.text, + ); + const leaderboardText = (sectionBlocks[1].text as { text: string }).text; + expect(leaderboardText).toContain("no traffic"); + expect(leaderboardText).toContain("`100.0%` remaining"); + expect(result.text).toContain("within error budget"); + }); + + it("flags exhausted budgets", () => { + const slos: SloData[] = [ + { + route: "escrow_listener", + totalRequests: 100, + badEvents: 10, + sloObjective: 0.99, + observedErrorRate: 0.1, + consumedFraction: 10, + remainingBudget: 0, + noTraffic: false, + }, + ]; + const result = buildSlackReport(slos, "https://grafana.example.com/d/slo"); + expect(result.text).toContain("exhausted"); + const sectionBlocks = result.blocks.filter( + (b) => b.type === "section" && (b.text as { text?: string })?.text, + ); + expect((sectionBlocks[1].text as { text: string }).text).toContain("Exhausted"); + }); + + it("returns no-data message when SLO list is empty", () => { + const result = buildSlackReport([], "https://grafana.example.com/d/slo"); + const sectionBlocks = result.blocks.filter( + (b) => b.type === "section" && (b.text as { text?: string })?.text, + ); + expect((sectionBlocks[1].text as { text: string }).text).toContain( + "No SLO data available", + ); + }); +}); + +describe("fetchWeeklySloData", () => { + it("returns default routes when prometheus URL is empty", async () => { + const slos = await fetchWeeklySloData(""); + expect(slos).toHaveLength(4); + expect(slos.every((s) => s.remainingBudget === 1 && s.noTraffic)).toBe(true); + }); + + it("falls back to defaults when prometheus returns empty results", async () => { + const fetchFn = jest.fn().mockResolvedValue({ + ok: true, + json: async () => ({ status: "success", data: { result: [] } }), + } as Response); + + const slos = await fetchWeeklySloData( + "http://prometheus:9090", + 7, + fetchFn as typeof fetch, + ); + expect(slos).toHaveLength(4); + expect(slos.every((s) => s.noTraffic)).toBe(true); + }); +}); + +describe("postToSlack", () => { + it("throws on non-2xx responses (Slack outage)", async () => { + const fetchFn = jest.fn().mockResolvedValue({ + ok: false, + status: 503, + text: async () => "Service Unavailable", + } as Response); + + await expect( + postToSlack( + "https://hooks.slack.com/test", + { text: "test", blocks: [] }, + fetchFn as typeof fetch, + ), + ).rejects.toThrow("Slack webhook returned 503"); + }); + + it("succeeds on 2xx", async () => { + const fetchFn = jest.fn().mockResolvedValue({ + ok: true, + status: 200, + text: async () => "ok", + } as Response); + + await expect( + postToSlack( + "https://hooks.slack.com/test", + { text: "test", blocks: [] }, + fetchFn as typeof fetch, + ), + ).resolves.toBeUndefined(); + }); +}); + +describe("runReport", () => { + it("posts a report using mocked fetch", async () => { + const fetchFn = jest.fn().mockImplementation(async (input, init) => { + const url = String(input); + if (url.includes("/api/v1/query")) { + return { + ok: true, + json: async () => ({ status: "success", data: { result: [] } }), + } as Response; + } + if (init?.method === "POST") { + return { ok: true, status: 200, text: async () => "ok" } as Response; + } + throw new Error(`Unexpected fetch: ${url}`); + }); + + const slos = await runReport({ + prometheusUrl: "", + slackWebhookUrl: "https://hooks.slack.com/test", + burnRateDashboardUrl: "https://grafana.example.com/d/slo-burn-rate", + fetchFn: fetchFn as typeof fetch, + }); + + expect(slos).toHaveLength(4); + expect(fetchFn).toHaveBeenCalledWith( + "https://hooks.slack.com/test", + expect.objectContaining({ method: "POST" }), + ); + }); + + it("propagates Slack failures", async () => { + const fetchFn = jest.fn().mockResolvedValue({ + ok: false, + status: 500, + text: async () => "error", + } as Response); + + await expect( + runReport({ + prometheusUrl: "", + slackWebhookUrl: "https://hooks.slack.com/test", + burnRateDashboardUrl: "https://grafana.example.com/d/slo-burn-rate", + fetchFn: fetchFn as typeof fetch, + }), + ).rejects.toThrow("Slack webhook returned 500"); + }); +}); diff --git a/scripts/slo-weekly-report.ts b/scripts/slo-weekly-report.ts new file mode 100644 index 00000000..34a80b9b --- /dev/null +++ b/scripts/slo-weekly-report.ts @@ -0,0 +1,462 @@ +/** + * slo-weekly-report.ts + * + * Computes weekly SLO error-budget consumption per route and posts a concise + * Slack report with a per-service leaderboard. + * + * Usage: + * SLACK_WEBHOOK_URL=https://hooks.slack.com/... \ + * PROMETHEUS_URL=http://localhost:9090 \ + * BURN_RATE_DASHBOARD_URL=https://grafana.example.com/d/slo-burn-rate \ + * npx tsx scripts/slo-weekly-report.ts + */ + +import { fileURLToPath } from "node:url"; +import { + RouteName, + SLO_OBJECTIVES, +} from "../src/metrics/sloMetrics.js"; +import { REPORTED_ROUTES } from "../src/simulator/sloHeadroomReporter.js"; + +// ─── Types ─────────────────────────────────────────────────────────────────── + +export interface RouteCountInput { + route: RouteName; + totalRequests: number; + badEvents: number; + sloObjective?: number; +} + +export interface SloData { + route: RouteName; + totalRequests: number; + badEvents: number; + sloObjective: number; + /** Observed error rate over the window (0 when no traffic). */ + observedErrorRate: number; + /** Fraction of the error budget consumed this week (0..∞). */ + consumedFraction: number; + /** Remaining error budget as a fraction of the total budget (0..1). */ + remainingBudget: number; + /** True when no requests were observed in the window. */ + noTraffic: boolean; +} + +export interface SlackMessage { + text: string; + blocks: Array>; +} + +export interface ReportOptions { + prometheusUrl?: string; + slackWebhookUrl: string; + burnRateDashboardUrl: string; + windowDays?: number; + /** Injectable fetch for tests and custom runtimes. */ + fetchFn?: typeof fetch; +} + +type FetchFn = typeof fetch; + +// ─── Budget math (aligned with sloHeadroomReporter) ───────────────────────── + +export function clamp(value: number, min: number, max: number): number { + return Math.max(min, Math.min(max, value)); +} + +/** + * Compute weekly error-budget consumption for a single route. + * + * errorBudget = 1 − sloObjective + * consumedFraction = observedErrorRate / errorBudget + * remainingBudget = clamp(1 − consumedFraction, 0, 1) + */ +export function computeRouteBudget( + route: RouteName, + totalRequests: number, + badEvents: number, + sloObjective: number = SLO_OBJECTIVES[route], +): SloData { + const observedErrorRate = + totalRequests > 0 ? badEvents / totalRequests : 0; + const errorBudget = 1 - sloObjective; + const noTraffic = totalRequests === 0; + + let consumedFraction = 0; + if (!noTraffic && errorBudget > 0) { + consumedFraction = observedErrorRate / errorBudget; + } else if (!noTraffic && errorBudget === 0 && observedErrorRate > 0) { + consumedFraction = Infinity; + } + + const remainingBudget = + noTraffic || !isFinite(consumedFraction) + ? noTraffic + ? 1 + : 0 + : clamp(1 - consumedFraction, 0, 1); + + return { + route, + totalRequests, + badEvents, + sloObjective, + observedErrorRate, + consumedFraction: noTraffic ? 0 : consumedFraction, + remainingBudget, + noTraffic, + }; +} + +/** + * Pure compute path used by tests and as a fallback when Prometheus is empty. + */ +export function computeWeeklyBudgets(routes: RouteCountInput[]): SloData[] { + const byRoute = new Map(); + for (const entry of routes) { + byRoute.set(entry.route, entry); + } + + return REPORTED_ROUTES.map((route) => { + const entry = byRoute.get(route); + return computeRouteBudget( + route, + entry?.totalRequests ?? 0, + entry?.badEvents ?? 0, + entry?.sloObjective ?? SLO_OBJECTIVES[route], + ); + }); +} + +export function defaultWeeklyBudgets(): SloData[] { + return computeWeeklyBudgets([]); +} + +// ─── Prometheus ───────────────────────────────────────────────────────────── + +export async function queryPrometheus( + prometheusUrl: string, + query: string, + fetchFn: FetchFn = fetch, +): Promise; value: [number, string] }>> { + const url = `${prometheusUrl.replace(/\/$/, "")}/api/v1/query?query=${encodeURIComponent(query)}`; + const response = await fetchFn(url); + + if (!response.ok) { + throw new Error( + `Prometheus query failed: ${response.status} ${response.statusText}`, + ); + } + + const data = (await response.json()) as { + status: string; + error?: string; + data?: { result: Array<{ metric: Record; value: [number, string] }> }; + }; + + if (data.status !== "success") { + throw new Error(`Prometheus returned error: ${data.error || "unknown"}`); + } + + return data.data?.result ?? []; +} + +function parseRouteLabel(metric: Record): RouteName | null { + const raw = metric.route || metric.service; + if (raw && raw in SLO_OBJECTIVES) { + return raw as RouteName; + } + return null; +} + +async function fetchCounterCounts( + prometheusUrl: string, + windowDays: number, + fetchFn: FetchFn, +): Promise> { + const range = `${windowDays}d`; + const map = new Map(); + + const requestQueries = [ + `increase(slo_requests_total[${range}])`, + `increase(slo_route_requests_total[${range}])`, + ]; + const errorQueries = [ + `increase(slo_bad_events_total[${range}])`, + `increase(slo_route_errors_total[${range}])`, + ]; + + for (const query of requestQueries) { + const results = await queryPrometheus(prometheusUrl, query, fetchFn); + for (const result of results) { + const route = parseRouteLabel(result.metric); + if (!route) continue; + const val = parseFloat(result.value[1]); + const existing = map.get(route) ?? { totalRequests: 0, badEvents: 0 }; + existing.totalRequests += Number.isNaN(val) ? 0 : Math.round(val); + map.set(route, existing); + } + if (map.size > 0) break; + } + + for (const query of errorQueries) { + const results = await queryPrometheus(prometheusUrl, query, fetchFn); + for (const result of results) { + const route = parseRouteLabel(result.metric); + if (!route) continue; + const val = parseFloat(result.value[1]); + const existing = map.get(route) ?? { totalRequests: 0, badEvents: 0 }; + existing.badEvents += Number.isNaN(val) ? 0 : Math.round(val); + map.set(route, existing); + } + if ([...map.values()].some((v) => v.badEvents > 0)) break; + } + + return map; +} + +async function fetchBurnRateBudgets( + prometheusUrl: string, + windowDays: number, + fetchFn: FetchFn, +): Promise> { + const range = `${windowDays}d`; + const query = `max by (route) (max_over_time(slo_burn_rate[${range}]))`; + const results = await queryPrometheus(prometheusUrl, query, fetchFn); + const map = new Map(); + + for (const result of results) { + const route = parseRouteLabel(result.metric); + if (!route) continue; + const burnRate = parseFloat(result.value[1]); + if (Number.isNaN(burnRate)) continue; + map.set(route, burnRate); + } + + return map; +} + +/** + * Fetch weekly SLO data from Prometheus when configured, otherwise return + * known routes at 100% remaining with a no-traffic marker. + */ +export async function fetchWeeklySloData( + prometheusUrl: string | undefined, + windowDays: number = 7, + fetchFn: FetchFn = fetch, +): Promise { + if (!prometheusUrl?.trim()) { + return defaultWeeklyBudgets(); + } + + try { + const counts = await fetchCounterCounts(prometheusUrl, windowDays, fetchFn); + if (counts.size > 0) { + return computeWeeklyBudgets( + REPORTED_ROUTES.map((route) => { + const count = counts.get(route); + return { + route, + totalRequests: count?.totalRequests ?? 0, + badEvents: count?.badEvents ?? 0, + }; + }), + ); + } + + const burnRates = await fetchBurnRateBudgets( + prometheusUrl, + windowDays, + fetchFn, + ); + if (burnRates.size > 0) { + return REPORTED_ROUTES.map((route) => { + const consumedFraction = burnRates.get(route) ?? 0; + const remainingBudget = clamp(1 - consumedFraction, 0, 1); + return { + route, + totalRequests: 0, + badEvents: 0, + sloObjective: SLO_OBJECTIVES[route], + observedErrorRate: 0, + consumedFraction, + remainingBudget, + noTraffic: false, + }; + }); + } + } catch (err) { + console.warn( + "Prometheus unavailable or query failed; posting no-traffic defaults:", + err instanceof Error ? err.message : err, + ); + } + + return defaultWeeklyBudgets(); +} + +// ─── Report construction ──────────────────────────────────────────────────── + +export function formatPercent(fraction: number, digits = 1): string { + if (!isFinite(fraction)) return "∞"; + return (fraction * 100).toFixed(digits); +} + +export function budgetBar(fraction: number): string { + const clamped = clamp(fraction, 0, 1); + const filled = Math.round(clamped * 10); + const empty = 10 - filled; + return "█".repeat(filled) + "░".repeat(empty); +} + +export function budgetStatus(remainingBudget: number): string { + if (remainingBudget <= 0) return "🔴 Exhausted"; + if (remainingBudget < 0.2) return "🔴 Critical"; + if (remainingBudget < 0.5) return "🟡 At Risk"; + return "🟢 Healthy"; +} + +function formatRouteLine(slo: SloData): string { + const remainingPct = formatPercent(slo.remainingBudget); + const consumedPct = formatPercent(slo.consumedFraction); + const bar = budgetBar(slo.remainingBudget); + const status = budgetStatus(slo.remainingBudget); + const trafficNote = slo.noTraffic ? " · _no traffic_" : ""; + return ( + `\`${slo.route}\` · ${bar} \`${remainingPct}%\` remaining · ` + + `\`${consumedPct}%\` consumed · ${status}${trafficNote}` + ); +} + +/** + * Builds a Slack message payload with a per-service leaderboard (worst first). + */ +export function buildSlackReport( + slos: SloData[], + burnRateDashboardUrl: string, + windowDays: number = 7, +): SlackMessage { + const sorted = [...slos].sort((a, b) => a.remainingBudget - b.remainingBudget); + const blocks: Array> = []; + + blocks.push({ + type: "section", + text: { + type: "mrkdwn", + text: `📊 *Weekly SLO Error-Budget Report*\nPeriod: past ${windowDays} days`, + }, + }); + blocks.push({ type: "divider" }); + + if (sorted.length === 0) { + blocks.push({ + type: "section", + text: { + type: "mrkdwn", + text: "No SLO data available for this period.", + }, + }); + } else { + const lines = [ + "`Service` · `Remaining` · `Weekly consumption` · `Status`", + ...sorted.map(formatRouteLine), + ]; + + blocks.push({ + type: "section", + text: { type: "mrkdwn", text: lines.join("\n") }, + }); + blocks.push({ type: "divider" }); + blocks.push({ + type: "context", + elements: [ + { + type: "mrkdwn", + text: `<${burnRateDashboardUrl}|View Burn-Rate Dashboard> · Generated ${new Date().toISOString()}`, + }, + ], + }); + } + + const criticalCount = sorted.filter((s) => s.remainingBudget < 0.2).length; + const exhaustedCount = sorted.filter((s) => s.remainingBudget <= 0).length; + + return { + text: + exhaustedCount > 0 + ? `Weekly SLO Report: ${exhaustedCount} route(s) exhausted error budget` + : criticalCount > 0 + ? `Weekly SLO Report: ${criticalCount} route(s) critically low on error budget` + : "Weekly SLO Report: all routes within error budget", + blocks, + }; +} + +// ─── Slack delivery ───────────────────────────────────────────────────────── + +export async function postToSlack( + webhookUrl: string, + message: SlackMessage, + fetchFn: FetchFn = fetch, +): Promise { + const response = await fetchFn(webhookUrl, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(message), + }); + + if (!response.ok) { + const body = await response.text(); + throw new Error(`Slack webhook returned ${response.status}: ${body}`); + } +} + +// ─── Main ─────────────────────────────────────────────────────────────────── + +export async function runReport(options: ReportOptions): Promise { + const windowDays = options.windowDays ?? 7; + const fetchFn = options.fetchFn ?? fetch; + + const slos = await fetchWeeklySloData( + options.prometheusUrl, + windowDays, + fetchFn, + ); + const message = buildSlackReport( + slos, + options.burnRateDashboardUrl, + windowDays, + ); + await postToSlack(options.slackWebhookUrl, message, fetchFn); + + console.log( + `SLO weekly report posted to Slack. ${slos.length} route(s) tracked.`, + ); + return slos; +} + +// ─── CLI entry ─────────────────────────────────────────────────────────────── + +const isMain = process.argv[1] === fileURLToPath(import.meta.url); + +if (isMain) { + const slackWebhookUrl = process.env.SLACK_WEBHOOK_URL; + const prometheusUrl = process.env.PROMETHEUS_URL ?? ""; + const burnRateDashboardUrl = + process.env.BURN_RATE_DASHBOARD_URL ?? + "https://grafana.example.com/d/slo-burn-rate"; + + if (!slackWebhookUrl) { + console.error("SLACK_WEBHOOK_URL environment variable is required."); + process.exit(1); + } + + runReport({ + prometheusUrl, + slackWebhookUrl, + burnRateDashboardUrl, + }).catch((err) => { + console.error("SLO weekly report failed:", err); + process.exit(1); + }); +}