diff --git a/src/index.ts b/src/index.ts index d95a5bfc..26025c38 100644 --- a/src/index.ts +++ b/src/index.ts @@ -87,6 +87,7 @@ import settingsRoutes from "./routes/settings"; import { statementsRoutes } from "./routes/statements"; import { paymentLinkRoutes } from "./routes/paymentLinkRoutes.js"; import providerStatusRouter from "./routes/providerStatus"; +import { costAllocationRoutes } from "./routes/costAllocation"; import { startHeartbeatService, stopHeartbeatService } from "./services/heartbeatService"; import { startStellarExporter } from "./services/stellarExporter"; @@ -392,6 +393,7 @@ app.use("/api/developer", developerDashboardRoutes); app.use("/api/admin", requireAuth, adminRoutes); app.use("/api/admin/providers/status", requireAuth, providerStatusRouter); app.use("/api/admin/kyc-upgrades", requireAuth, kycTierUpgradeRoutes); +app.use("/api/admin/cost-allocation", requireAuth, costAllocationRoutes); app.use("/api/admin/auth", createAdminSep10Router()); app.use("/sep10", createSep10Router()); app.use("/sep31", sep31Router); diff --git a/src/routes/costAllocation.ts b/src/routes/costAllocation.ts new file mode 100644 index 00000000..a69da962 --- /dev/null +++ b/src/routes/costAllocation.ts @@ -0,0 +1,81 @@ +/** + * Cost Allocation Metrics API — issue #261 + * + * Exposes unit economics data (cost per transaction by provider and feature) + * as a JSON endpoint consumed by analytics dashboards and data pipelines. + * + * Routes: + * GET /api/admin/cost-allocation — unit cost analytics (JSON) + * GET /api/admin/cost-allocation/export — CSV export for data warehouse + */ + +import { Router, Request, Response } from "express"; +import { requireAuth } from "../middleware/auth"; +import { costMetrics } from "../services/costAllocationMetrics"; + +const router = Router(); + +/** + * GET /api/admin/cost-allocation + * Returns cost-per-transaction analytics grouped by provider and feature. + * Identifies the most expensive features and enables capacity planning. + */ +router.get("/", requireAuth, (_req: Request, res: Response) => { + const records = costMetrics.getUnitCostAnalytics(); + + // Aggregate totals for the summary block + const summary = records.reduce( + (acc, r) => ({ + total_api_calls: acc.total_api_calls + r.api_calls, + total_db_queries: acc.total_db_queries + r.db_queries, + total_storage_bytes: acc.total_storage_bytes + r.storage_bytes, + total_transactions: acc.total_transactions + r.total_transactions, + }), + { + total_api_calls: 0, + total_db_queries: 0, + total_storage_bytes: 0, + total_transactions: 0, + }, + ); + + res.json({ + generated_at: new Date().toISOString(), + summary, + records, + note: "Costs are estimated from configurable per-unit rates. Override via COST_PER_API_CALL_USD_CENTS, COST_PER_DB_QUERY_USD_CENTS, COST_PER_STORAGE_KB_USD_CENTS env vars.", + }); +}); + +/** + * GET /api/admin/cost-allocation/export + * Returns cost analytics as a CSV for import into BigQuery / Redshift / Excel. + */ +router.get("/export", requireAuth, (_req: Request, res: Response) => { + const records = costMetrics.getUnitCostAnalytics(); + + const header = + "provider,feature,api_calls,db_queries,storage_bytes,total_transactions," + + "estimated_api_cost_usd_cents_per_tx,estimated_db_cost_usd_cents_per_tx," + + "estimated_total_cost_usd_cents_per_tx\n"; + + const rows = records + .map( + (r) => + `${r.provider},${r.feature},${r.api_calls},${r.db_queries},` + + `${r.storage_bytes},${r.total_transactions},` + + `${r.estimated_api_cost_usd_cents_per_tx},` + + `${r.estimated_db_cost_usd_cents_per_tx},` + + `${r.estimated_total_cost_usd_cents_per_tx}`, + ) + .join("\n"); + + res.setHeader("Content-Type", "text/csv"); + res.setHeader( + "Content-Disposition", + `attachment; filename="cost-allocation-${new Date().toISOString().slice(0, 10)}.csv"`, + ); + res.send(header + rows); +}); + +export { router as costAllocationRoutes }; diff --git a/src/services/costAllocationMetrics.ts b/src/services/costAllocationMetrics.ts new file mode 100644 index 00000000..8f9c3965 --- /dev/null +++ b/src/services/costAllocationMetrics.ts @@ -0,0 +1,326 @@ +/** + * Cost Allocation Metrics Service — issue #261 + * + * Tracks resource usage (API calls, DB queries, storage operations) broken + * down by provider (MTN, Airtel, Orange) and feature (deposit, withdraw, KYC, + * webhook, …) to calculate unit economics per transaction. + * + * Design: + * - Prometheus counters / histograms with `provider` + `feature` labels + * → zero runtime overhead (O(1) label lookup in prom-client) + * - Separate cost analytics helpers aggregate the raw Prometheus data into + * human-readable unit-economics objects that can be exported to dashboards + * or a data warehouse via the /metrics route. + * + * Usage: + * import { costMetrics } from "./costAllocationMetrics"; + * + * // Track a provider API call + * costMetrics.recordApiCall("mtn", "deposit", durationMs, "success"); + * + * // Track a DB query + * costMetrics.recordDbQuery("airtel", "withdraw", durationMs); + * + * // Track storage usage (S3, etc.) + * costMetrics.recordStorageOp("orange", "kyc", bytesWritten); + */ + +import { Counter, Histogram, Gauge, Registry } from "prom-client"; +import { register as globalRegistry } from "../utils/metrics"; + +// --------------------------------------------------------------------------- +// Types +// --------------------------------------------------------------------------- + +export type Provider = "mtn" | "airtel" | "orange" | "stellar" | "internal"; +export type Feature = + | "deposit" + | "withdraw" + | "kyc" + | "webhook" + | "aml" + | "reconciliation" + | "statement" + | "fee_calculation" + | "exchange_rate" + | "other"; + +export interface UnitCostRecord { + provider: Provider | string; + feature: Feature | string; + api_calls: number; + db_queries: number; + storage_bytes: number; + total_transactions: number; + /** + * Estimated API cost in USD-cents per transaction. + * Derived from api_calls * COST_PER_API_CALL / total_transactions. + */ + estimated_api_cost_usd_cents_per_tx: number; + /** + * Estimated DB cost in USD-cents per transaction. + * Derived from db_queries * COST_PER_DB_QUERY / total_transactions. + */ + estimated_db_cost_usd_cents_per_tx: number; + /** Sum of the two above for total cost. */ + estimated_total_cost_usd_cents_per_tx: number; +} + +// --------------------------------------------------------------------------- +// Cost constants (tuneable via env vars) +// These are conservative baselines — operators should override with their +// actual AWS / GCP / provider pricing. +// --------------------------------------------------------------------------- + +const COST_PER_API_CALL_USD_CENTS = parseFloat( + process.env.COST_PER_API_CALL_USD_CENTS ?? "0.001", +); +const COST_PER_DB_QUERY_USD_CENTS = parseFloat( + process.env.COST_PER_DB_QUERY_USD_CENTS ?? "0.0005", +); +const COST_PER_STORAGE_KB_USD_CENTS = parseFloat( + process.env.COST_PER_STORAGE_KB_USD_CENTS ?? "0.00002", +); + +// --------------------------------------------------------------------------- +// Prometheus metrics +// --------------------------------------------------------------------------- + +/** + * Total provider API calls broken down by provider, feature, and outcome. + * Enables: cost per provider, most expensive feature, error rates. + */ +export const costApiCallsTotal = new Counter({ + name: "cost_provider_api_calls_total", + help: "Total provider API calls tracked for cost allocation", + labelNames: ["provider", "feature", "status"] as const, + registers: [globalRegistry], +}); + +/** + * Duration histogram for provider API calls — p95/p99 latency per provider + * helps identify slow (and therefore expensive) integrations. + */ +export const costApiCallDurationMs = new Histogram({ + name: "cost_provider_api_call_duration_ms", + help: "Duration of provider API calls in milliseconds (cost allocation)", + labelNames: ["provider", "feature"] as const, + buckets: [10, 50, 100, 250, 500, 1000, 2500, 5000, 10000], + registers: [globalRegistry], +}); + +/** + * Database queries attributed to a provider + feature. + * Tracks read vs write separately for replica routing cost analysis. + */ +export const costDbQueriesTotal = new Counter({ + name: "cost_db_queries_total", + help: "Total database queries tracked for cost allocation", + labelNames: ["provider", "feature", "query_type"] as const, + registers: [globalRegistry], +}); + +/** + * Duration histogram for DB queries — identifies expensive DB operations per + * feature so the team can decide where to add caching or indexes. + */ +export const costDbQueryDurationMs = new Histogram({ + name: "cost_db_query_duration_ms", + help: "Duration of database queries in milliseconds (cost allocation)", + labelNames: ["provider", "feature", "query_type"] as const, + buckets: [1, 5, 10, 25, 50, 100, 250, 500, 1000], + registers: [globalRegistry], +}); + +/** + * Storage bytes written (S3 uploads, document storage, etc.) + * Labelled by provider + feature so KYC document costs are visible. + */ +export const costStorageBytesTotal = new Counter({ + name: "cost_storage_bytes_total", + help: "Total bytes written to storage for cost allocation", + labelNames: ["provider", "feature"] as const, + registers: [globalRegistry], +}); + +/** + * Completed transactions per provider + feature. + * Dividing cost counters by this gives unit cost per transaction. + */ +export const costTransactionsTotal = new Counter({ + name: "cost_transactions_total", + help: "Total completed transactions for unit cost calculation", + labelNames: ["provider", "feature", "status"] as const, + registers: [globalRegistry], +}); + +/** + * Estimated cost per transaction gauge (updated on each transaction). + * Exported directly so Grafana can display it without PromQL division. + */ +export const costEstimatedCentsPerTx = new Gauge({ + name: "cost_estimated_cents_per_transaction", + help: "Estimated cost in USD-cents per transaction by provider and feature", + labelNames: ["provider", "feature"] as const, + registers: [globalRegistry], +}); + +// --------------------------------------------------------------------------- +// In-memory accumulator for unit cost calculation +// (Prometheus counters are write-only so we track the values separately for +// the analytics helper. This is reset on restart — long-term data lives in +// the time-series DB that scrapes the /metrics endpoint.) +// --------------------------------------------------------------------------- + +interface AccumulatorEntry { + api_calls: number; + db_queries: number; + storage_bytes: number; + transactions: number; +} + +const accumulator = new Map(); + +function accKey(provider: string, feature: string): string { + return `${provider}::${feature}`; +} + +function getOrCreate(provider: string, feature: string): AccumulatorEntry { + const key = accKey(provider, feature); + if (!accumulator.has(key)) { + accumulator.set(key, { + api_calls: 0, + db_queries: 0, + storage_bytes: 0, + transactions: 0, + }); + } + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + return accumulator.get(key)!; +} + +// --------------------------------------------------------------------------- +// Public API +// --------------------------------------------------------------------------- + +export const costMetrics = { + /** + * Record a provider API call. + * Call this from provider service wrappers (MobileMoneyService, etc.). + */ + recordApiCall( + provider: Provider | string, + feature: Feature | string, + durationMs: number, + status: "success" | "error" | "timeout" = "success", + ): void { + costApiCallsTotal.inc({ provider, feature, status }); + costApiCallDurationMs.observe({ provider, feature }, durationMs); + getOrCreate(provider, feature).api_calls += 1; + }, + + /** + * Record a database query attributed to a provider + feature. + * Call this from query helpers when a provider can be inferred from context. + */ + recordDbQuery( + provider: Provider | string, + feature: Feature | string, + durationMs: number, + queryType: "read" | "write" = "read", + ): void { + costDbQueriesTotal.inc({ provider, feature, query_type: queryType }); + costDbQueryDurationMs.observe( + { provider, feature, query_type: queryType }, + durationMs, + ); + getOrCreate(provider, feature).db_queries += 1; + }, + + /** + * Record bytes written to storage (S3, local disk, etc.). + */ + recordStorageOp( + provider: Provider | string, + feature: Feature | string, + bytes: number, + ): void { + costStorageBytesTotal.inc({ provider, feature }, bytes); + getOrCreate(provider, feature).storage_bytes += bytes; + }, + + /** + * Record a completed transaction and update the cost-per-transaction gauge. + * Call this from the transaction completion path. + */ + recordTransaction( + provider: Provider | string, + feature: Feature | string, + status: "completed" | "failed" | "refunded" = "completed", + ): void { + costTransactionsTotal.inc({ provider, feature, status }); + + const entry = getOrCreate(provider, feature); + entry.transactions += 1; + + // Refresh the estimated cost gauge + const apiCost = entry.api_calls * COST_PER_API_CALL_USD_CENTS; + const dbCost = entry.db_queries * COST_PER_DB_QUERY_USD_CENTS; + const storageCost = + (entry.storage_bytes / 1024) * COST_PER_STORAGE_KB_USD_CENTS; + const totalCost = apiCost + dbCost + storageCost; + const perTx = entry.transactions > 0 ? totalCost / entry.transactions : 0; + + costEstimatedCentsPerTx.set({ provider, feature }, perTx); + }, + + /** + * Return unit cost analytics for all tracked provider + feature combinations. + * Suitable for export to a data warehouse or analytics dashboard. + */ + getUnitCostAnalytics(): UnitCostRecord[] { + const results: UnitCostRecord[] = []; + + for (const [key, entry] of accumulator.entries()) { + const [provider, feature] = key.split("::"); + + const apiCost = entry.api_calls * COST_PER_API_CALL_USD_CENTS; + const dbCost = entry.db_queries * COST_PER_DB_QUERY_USD_CENTS; + const storageCost = + (entry.storage_bytes / 1024) * COST_PER_STORAGE_KB_USD_CENTS; + const totalCost = apiCost + dbCost + storageCost; + const perTx = + entry.transactions > 0 ? totalCost / entry.transactions : 0; + const apiPerTx = + entry.transactions > 0 ? apiCost / entry.transactions : 0; + const dbPerTx = + entry.transactions > 0 ? dbCost / entry.transactions : 0; + + results.push({ + provider: provider ?? "unknown", + feature: feature ?? "unknown", + api_calls: entry.api_calls, + db_queries: entry.db_queries, + storage_bytes: entry.storage_bytes, + total_transactions: entry.transactions, + estimated_api_cost_usd_cents_per_tx: Math.round(apiPerTx * 10000) / 10000, + estimated_db_cost_usd_cents_per_tx: Math.round(dbPerTx * 10000) / 10000, + estimated_total_cost_usd_cents_per_tx: Math.round(perTx * 10000) / 10000, + }); + } + + // Sort by total cost descending so the most expensive shows first + results.sort( + (a, b) => + b.estimated_total_cost_usd_cents_per_tx - + a.estimated_total_cost_usd_cents_per_tx, + ); + + return results; + }, + + /** Reset in-memory accumulator (for testing). */ + _reset(): void { + accumulator.clear(); + }, +} as const; diff --git a/tests/services/costAllocationMetrics.test.ts b/tests/services/costAllocationMetrics.test.ts new file mode 100644 index 00000000..1b3c6644 --- /dev/null +++ b/tests/services/costAllocationMetrics.test.ts @@ -0,0 +1,138 @@ +/** + * Tests for cost allocation metrics service — issue #261 + */ + +// Mock prom-client so tests don't conflict with the global registry +jest.mock("prom-client", () => { + const makeCounter = () => ({ inc: jest.fn() }); + const makeHistogram = () => ({ observe: jest.fn() }); + const makeGauge = () => ({ set: jest.fn() }); + return { + Counter: jest.fn().mockImplementation(makeCounter), + Histogram: jest.fn().mockImplementation(makeHistogram), + Gauge: jest.fn().mockImplementation(makeGauge), + Registry: jest.fn().mockImplementation(() => ({})), + }; +}); + +// Mock the utils/metrics registry so the counters don't clash +jest.mock("../../../src/utils/metrics", () => ({ register: {} })); + +import { costMetrics } from "../../../src/services/costAllocationMetrics"; + +beforeEach(() => { + costMetrics._reset(); +}); + +describe("costMetrics.recordApiCall", () => { + it("accumulates api_calls in the analytics output", () => { + costMetrics.recordApiCall("mtn", "deposit", 120, "success"); + costMetrics.recordApiCall("mtn", "deposit", 95, "success"); + + const records = costMetrics.getUnitCostAnalytics(); + const row = records.find((r) => r.provider === "mtn" && r.feature === "deposit"); + expect(row?.api_calls).toBe(2); + }); + + it("tracks errors separately from successes via status label", () => { + costMetrics.recordApiCall("airtel", "withdraw", 300, "error"); + costMetrics.recordApiCall("airtel", "withdraw", 150, "success"); + + const records = costMetrics.getUnitCostAnalytics(); + const row = records.find((r) => r.provider === "airtel" && r.feature === "withdraw"); + // Both calls accumulate into api_calls (status is a Prometheus label, not split here) + expect(row?.api_calls).toBe(2); + }); +}); + +describe("costMetrics.recordDbQuery", () => { + it("accumulates db_queries", () => { + costMetrics.recordDbQuery("orange", "kyc", 10, "read"); + costMetrics.recordDbQuery("orange", "kyc", 8, "write"); + + const records = costMetrics.getUnitCostAnalytics(); + const row = records.find((r) => r.provider === "orange" && r.feature === "kyc"); + expect(row?.db_queries).toBe(2); + }); +}); + +describe("costMetrics.recordStorageOp", () => { + it("accumulates storage_bytes", () => { + costMetrics.recordStorageOp("mtn", "kyc", 1024 * 500); // 500 KB + costMetrics.recordStorageOp("mtn", "kyc", 1024 * 200); // 200 KB + + const records = costMetrics.getUnitCostAnalytics(); + const row = records.find((r) => r.provider === "mtn" && r.feature === "kyc"); + expect(row?.storage_bytes).toBe(1024 * 700); + }); +}); + +describe("costMetrics.recordTransaction + unit cost calculation", () => { + it("calculates non-zero cost per transaction after recording resources", () => { + // Simulate 10 API calls and 20 DB queries for 5 transactions + for (let i = 0; i < 10; i++) costMetrics.recordApiCall("mtn", "deposit", 100); + for (let i = 0; i < 20; i++) costMetrics.recordDbQuery("mtn", "deposit", 10); + for (let i = 0; i < 5; i++) costMetrics.recordTransaction("mtn", "deposit"); + + const records = costMetrics.getUnitCostAnalytics(); + const row = records.find((r) => r.provider === "mtn" && r.feature === "deposit"); + + expect(row?.total_transactions).toBe(5); + expect(row?.estimated_total_cost_usd_cents_per_tx).toBeGreaterThan(0); + expect(row?.estimated_api_cost_usd_cents_per_tx).toBeGreaterThan(0); + expect(row?.estimated_db_cost_usd_cents_per_tx).toBeGreaterThan(0); + }); + + it("returns zero cost per transaction when no transactions recorded", () => { + costMetrics.recordApiCall("airtel", "deposit", 100); + + const records = costMetrics.getUnitCostAnalytics(); + const row = records.find((r) => r.provider === "airtel" && r.feature === "deposit"); + expect(row?.estimated_total_cost_usd_cents_per_tx).toBe(0); + }); +}); + +describe("costMetrics.getUnitCostAnalytics", () => { + it("returns empty array when no data recorded", () => { + expect(costMetrics.getUnitCostAnalytics()).toEqual([]); + }); + + it("sorts records by total cost descending (most expensive first)", () => { + // mtn/deposit: 100 API calls → expensive + for (let i = 0; i < 100; i++) costMetrics.recordApiCall("mtn", "deposit", 50); + costMetrics.recordTransaction("mtn", "deposit"); + + // airtel/withdraw: 1 API call → cheap + costMetrics.recordApiCall("airtel", "withdraw", 50); + costMetrics.recordTransaction("airtel", "withdraw"); + + const records = costMetrics.getUnitCostAnalytics(); + expect(records[0].provider).toBe("mtn"); + expect(records[0].feature).toBe("deposit"); + }); + + it("keeps separate entries per provider+feature combination", () => { + costMetrics.recordApiCall("mtn", "deposit", 100); + costMetrics.recordApiCall("mtn", "withdraw", 100); + costMetrics.recordApiCall("airtel", "deposit", 100); + + const records = costMetrics.getUnitCostAnalytics(); + expect(records.length).toBe(3); + }); + + it("identifies expensive features by api_calls", () => { + for (let i = 0; i < 50; i++) costMetrics.recordApiCall("mtn", "kyc", 200); + costMetrics.recordTransaction("mtn", "kyc"); + + for (let i = 0; i < 5; i++) costMetrics.recordApiCall("mtn", "deposit", 100); + costMetrics.recordTransaction("mtn", "deposit"); + + const records = costMetrics.getUnitCostAnalytics(); + // KYC should be more expensive (50 API calls / 1 tx vs 5 / 1 tx) + const kyc = records.find((r) => r.feature === "kyc"); + const deposit = records.find((r) => r.feature === "deposit"); + expect(kyc!.estimated_api_cost_usd_cents_per_tx).toBeGreaterThan( + deposit!.estimated_api_cost_usd_cents_per_tx, + ); + }); +});