Pidoko257 · feyisaralawal · Jul 29, 2026
diff --git a/config/grafana/dashboards/provider_performance_comparison.json b/config/grafana/dashboards/provider_performance_comparison.json
@@ -0,0 +1,101 @@
+{
+  "annotations": {
+    "list": []
+  },
+  "editable": true,
+  "fiscalYearStartMonth": 0,
+  "graphTooltip": 1,
+  "id": null,
+  "title": "Mobile Money Provider Performance Comparison",
+  "tags": ["proxypay", "providers", "mtn", "airtel", "orange", "circuit-breaker"],
+  "timezone": "browser",
+  "refresh": "10s",
+  "schemaVersion": 38,
+  "panels": [
+    {
+      "id": 1,
+      "title": "Provider Success Rate Comparison (%)",
+      "type": "timeseries",
+      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 0 },
+      "targets": [
+        {
+          "datasource": "Prometheus",
+          "expr": "provider_success_rate",
+          "legendFormat": "{{provider}} - {{operation}}"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "custom": { "drawStyle": "line", "lineInterpolation": "smooth" },
+          "unit": "percent",
+          "min": 0,
+          "max": 100,
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              { "color": "red", "value": null },
+              { "color": "yellow", "value": 85 },
+              { "color": "green", "value": 95 }
+            ]
+          }
+        }
+      }
+    },
+    {
+      "id": 2,
+      "title": "Circuit Breaker State (0=Closed, 0.5=Half-Open, 1=Open)",
+      "type": "stat",
+      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 0 },
+      "targets": [
+        {
+          "datasource": "Prometheus",
+          "expr": "provider_circuit_breaker_state",
+          "legendFormat": "{{provider}} - {{operation}}"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              { "color": "green", "value": null },
+              { "color": "yellow", "value": 0.5 },
+              { "color": "red", "value": 1.0 }
+            ]
+          }
+        }
+      }
+    },
+    {
+      "id": 3,
+      "title": "Provider Response Latency P95 (seconds)",
+      "type": "timeseries",
+      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 8 },
+      "targets": [
+        {
+          "datasource": "Prometheus",
+          "expr": "histogram_quantile(0.95, sum(rate(provider_response_time_seconds_bucket[5m])) by (le, provider))",
+          "legendFormat": "{{provider}} P95"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "s"
+        }
+      }
+    },
+    {
+      "id": 4,
+      "title": "Provider Degradation & Failover Alerts Total",
+      "type": "timeseries",
+      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 8 },
+      "targets": [
+        {
+          "datasource": "Prometheus",
+          "expr": "rate(provider_degradation_alerts_total[5m])",
+          "legendFormat": "{{provider}} - {{reason}}"
+        }
+      ]
+    }
+  ]
+}
diff --git a/ingest-go/main.go b/ingest-go/main.go
@@ -227,7 +227,7 @@ func initMessaging() error {
 	return nil
 }
 
-func publish(p *CallbackPayload) error {
+func publish(p *CallbackPayload, traceID string) error {
 	data, err := json.Marshal(p)
 	if err != nil {
 		return err
@@ -242,6 +242,7 @@ func publish(p *CallbackPayload) error {
 				"event_type": p.EventType,
 				"provider":   p.Provider,
 				"reference":  p.Reference,
+				"trace_id":   traceID,
 				"data":       string(data),
 			},
 		}).Err(); err != nil {
@@ -268,6 +269,16 @@ func handleIngest(ctx *fasthttp.RequestCtx) {
 		return
 	}
 
+	// Distributed trace ID extraction / generation
+	traceID := string(ctx.Request.Header.Peek("X-Trace-Id"))
+	if traceID == "" {
+		traceID = string(ctx.Request.Header.Peek("X-Datadog-Trace-Id"))
+	}
+	if traceID == "" {
+		traceID = fmt.Sprintf("go-%d", time.Now().UnixNano())
+	}
+	ctx.Response.Header.Set("X-Trace-Id", traceID)
+
 	payload, err := parseCallbackPayload(ctx.PostBody())
 	if err != nil {
 		ctx.SetStatusCode(fasthttp.StatusBadRequest)
@@ -281,16 +292,16 @@ func handleIngest(ctx *fasthttp.RequestCtx) {
 		return
 	}
 
-	if err := publish(&payload); err != nil {
+	if err := publish(payload, traceID); err != nil {
 		sentry.CaptureException(err)
-		log.Printf("[ingest] publish error: %v", err)
+		log.Printf("[ingest] [trace_id=%s] publish error: %v", traceID, err)
 		ctx.SetStatusCode(fasthttp.StatusInternalServerError)
 		ctx.SetBodyString(`{"error":"publish failed"}`)
 		return
 	}
 
 	ctx.SetStatusCode(fasthttp.StatusAccepted)
-	fmt.Fprintf(ctx, `{"status":"accepted","reference":%q}`, payload.Reference)
+	fmt.Fprintf(ctx, `{"status":"accepted","reference":%q,"trace_id":%q}`, payload.Reference, traceID)
 }
 
 func handleHealth(ctx *fasthttp.RequestCtx) {

diff --git a/src/jobs/scheduler.ts b/src/jobs/scheduler.ts
@@ -150,6 +150,15 @@ const JOBS: JobConfig[] = [
     schedule: process.env.DATABASE_BACKUP_VERIFY_CRON || "0 3 * * *",
     handler: runDatabaseBackupVerifyJob,
   },
+  {
+    name: "synthetic-monitoring",
+    // Every minute - runs synthetic transaction flows (deposit, withdraw, dispute) and alerts on failure
+    schedule: process.env.SYNTHETIC_MONITORING_CRON || "* * * * *",
+    handler: async () => {
+      const { runSyntheticMonitoringJob } = await import("./syntheticMonitoringJob.js");
+      return runSyntheticMonitoringJob();
+    },
+  },
 ];
 
 async function runJob(job: JobConfig): Promise<void> {

diff --git a/src/jobs/syntheticMonitoringJob.ts b/src/jobs/syntheticMonitoringJob.ts
@@ -0,0 +1,14 @@
+import { syntheticMonitoringService } from "../services/syntheticMonitoringService";
+
+/**
+ * Job handler for running synthetic transaction tests for critical flows.
+ * Scheduled to run every minute.
+ */
+export async function runSyntheticMonitoringJob(): Promise<void> {
+  console.log("[synthetic-monitoring] Executing synthetic transaction flow checks...");
+  const results = await syntheticMonitoringService.runAllFlows();
+  const summary = results
+    .map((r) => `${r.flow}: ${r.success ? "PASS" : "FAIL"} (${r.durationMs}ms)`)
+    .join(" | ");
+  console.log(`[synthetic-monitoring] Completed checks: ${summary}`);
+}
diff --git a/src/middleware/tracing.ts b/src/middleware/tracing.ts
@@ -0,0 +1,51 @@
+import { Request, Response, NextFunction } from "express";
+import tracer, { extractTraceContext, injectTraceContext } from "../tracer";
+
+export interface TracedRequest extends Request {
+  traceId?: string;
+}
+
+/**
+ * Middleware that extracts trace headers from incoming requests (e.g. from Go ingest service or API Gateway),
+ * creates a root/child span for the request, and injects trace headers for outgoing requests.
+ */
+export function tracingMiddleware(req: TracedRequest, res: Response, next: NextFunction): void {
+  const childOf = extractTraceContext(req.headers as any);
+  const path = req.path || req.url;
+  const method = req.method;
+
+  const span = tracer.startSpan("express.request", {
+    childOf: childOf || undefined,
+    tags: {
+      "http.method": method,
+      "http.url": path,
+      "component": "express",
+      "service.name": "proxypay-express-api",
+    },
+  });
+
+  const traceId = (span.context() as any).toTraceId
+    ? (span.context() as any).toTraceId()
+    : req.headers["x-trace-id"] || `${Date.now()}`;
+
+  req.traceId = String(traceId);
+  res.setHeader("x-trace-id", String(traceId));
+
+  // Attach trace propagation helper to res.locals for client calls
+  res.locals.injectTraceContext = (headers: Record<string, string> = {}) => {
+    tracer.inject(span, "http_headers", headers);
+    return headers;
+  };
+
+  res.on("finish", () => {
+    span.setTag("http.status_code", res.statusCode);
+    if (res.statusCode >= 400) {
+      span.setTag("error", true);
+    }
+    span.finish();
+  });
+
+  tracer.scope().activate(span, () => {
+    next();
+  });
+}
diff --git a/src/services/providerMetricsService.ts b/src/services/providerMetricsService.ts
@@ -0,0 +1,98 @@
+import {
+  providerSuccessRateGauge,
+  providerRequestsTotal,
+  providerDegradationAlertTotal,
+  providerResponseTimeSeconds,
+} from "../utils/metrics";
+import { MonitoringService } from "./monitoringService";
+
+export type MobileMoneyProviderName = "MTN" | "Airtel" | "Orange" | string;
+
+interface ProviderWindowStats {
+  total: number;
+  successes: number;
+  failures: number;
+  lastUpdated: number;
+}
+
+class ProviderMetricsService {
+  // Sliding window statistics per provider + operation
+  private windowStats: Map<string, ProviderWindowStats> = new Map();
+  private readonly windowMs = 5 * 60 * 1000; // 5-minute sliding window
+  private readonly degradationThresholdPercent = 85.0; // Alert if success rate drops below 85%
+
+  private getKey(provider: MobileMoneyProviderName, operation: string): string {
+    return `${provider.toUpperCase()}:${operation}`;
+  }
+
+  /**
+   * Records a provider request outcome (success or failure) and updates success rate gauge and metrics.
+   */
+  public recordProviderCall(
+    provider: MobileMoneyProviderName,
+    operation: string,
+    durationMs: number,
+    success: boolean,
+    errorType?: string,
+  ): void {
+    const normProvider = provider.toUpperCase();
+    const statusLabel = success ? "success" : "failure";
+
+    // 1. Prometheus counter & latency histogram
+    providerRequestsTotal.inc({ provider: normProvider, operation, status: statusLabel });
+    providerResponseTimeSeconds.observe(
+      { provider: normProvider, operation, status: statusLabel },
+      durationMs / 1000,
+    );
+
+    // 2. Update sliding window stats for success rate calculation
+    const key = this.getKey(normProvider, operation);
+    const now = Date.now();
+    let stats = this.windowStats.get(key);
+
+    if (!stats || now - stats.lastUpdated > this.windowMs) {
+      stats = { total: 0, successes: 0, failures: 0, lastUpdated: now };
+    }
+
+    stats.total += 1;
+    if (success) {
+      stats.successes += 1;
+    } else {
+      stats.failures += 1;
+    }
+    stats.lastUpdated = now;
+    this.windowStats.set(key, stats);
+
+    // 3. Compute current success rate percentage
+    const successRate = (stats.successes / stats.total) * 100;
+    providerSuccessRateGauge.set({ provider: normProvider, operation }, parseFloat(successRate.toFixed(2)));
+
+    // 4. Check for provider degradation
+    if (stats.total >= 5 && successRate < this.degradationThresholdPercent) {
+      const reason = `Success rate dropped to ${successRate.toFixed(1)}% (below ${this.degradationThresholdPercent}% threshold)`;
+      providerDegradationAlertTotal.inc({ provider: normProvider, reason });
+
+      console.warn(
+        `[ProviderMetrics] DEGRADATION DETECTED for provider ${normProvider} (${operation}): ${reason}`,
+      );
+
+      MonitoringService.reportAlert(
+        `Provider Degradation Alert: ${normProvider} (${operation}) success rate is ${successRate.toFixed(1)}% over the last ${stats.total} requests.`,
+      ).catch((err) => {
+        console.error(`[ProviderMetrics] Failed to send degradation alert:`, err);
+      });
+    }
+  }
+
+  /**
+   * Helper to fetch current success rate for comparison.
+   */
+  public getProviderSuccessRate(provider: MobileMoneyProviderName, operation: string): number {
+    const key = this.getKey(provider, operation);
+    const stats = this.windowStats.get(key);
+    if (!stats || stats.total === 0) return 100.0;
+    return parseFloat(((stats.successes / stats.total) * 100).toFixed(2));
+  }
+}
+
+export const providerMetricsService = new ProviderMetricsService();