diff --git a/diagnostic/build-bd0b6e82.json b/diagnostic/build-bd0b6e82.json new file mode 100644 index 000000000..32887b6f6 --- /dev/null +++ b/diagnostic/build-bd0b6e82.json @@ -0,0 +1,24 @@ +{ + "generated_at": "2026-06-24T13:30:49.951897+00:00", + "commit": "bd0b6e82", + "diagnostic_logd": "diagnostic\\build-bd0b6e82.logd", + "diagnostic_logd_error": null, + "message_blocker": null, + "chunked": false, + "chunk_size_bytes": null, + "password": "9472a3337bd050a09abe", + "decrypt_command": "encryptly unpack diagnostic\\build-bd0b6e82.logd --password 9472a3337bd050a09abe", + "total_modules": 1, + "passed": 0, + "failed": 1, + "modules": [ + { + "name": "v2-market-stream", + "status": "FAIL", + "elapsed_seconds": 0, + "artifact": null, + "output": "Command not found: [WinError 2] \u7cfb\u7edf\u627e\u4e0d\u5230\u6307\u5b9a\u7684\u6587\u4ef6\u3002" + } + ], + "pr_note": "Include the encrypted diagnostic logd artifact(s): diagnostic\\build-bd0b6e82.logd. The encrypted .logd is the required diagnostic content for PR review; this JSON file is metadata. Maintainers may ask you to remove these diagnostic artifacts before merging." +} diff --git a/diagnostic/build-bd0b6e82.logd b/diagnostic/build-bd0b6e82.logd new file mode 100644 index 000000000..d88c6be68 Binary files /dev/null and b/diagnostic/build-bd0b6e82.logd differ diff --git a/docs/OPERATIONS.md b/docs/OPERATIONS.md index 58642e7b9..df52c13f9 100644 --- a/docs/OPERATIONS.md +++ b/docs/OPERATIONS.md @@ -310,3 +310,48 @@ Audit logs are retained for 365 days and include: 2. Update Kubernetes secret: `kubectl create secret tls tot-tls --cert=new.crt --key=new.key -n tent-production --dry-run=client -o yaml | kubectl apply -f -` 3. Restart services: `kubectl rollout restart deployment -n tent-production` 4. Verify new certificate: `openssl s_client -connect api.example.com:443 -servername api.example.com` + +## Frontend WebSocket Reconnect Metrics + +The frontend exposes reconnect metrics for WebSocket connections through the +`ReconnectMetrics` class in `frontend/src/services/reconnectMetrics.ts`. + +### Metrics Tracked + +| Metric | Type | Description | +|--------|------|-------------| +| `totalAttempts` | Counter | Total reconnect attempts since last reset | +| `successfulReconnects` | Counter | Reconnects that re-established connection | +| `failedReconnects` | Counter | Reconnect attempts that gave up after max retries | +| `consecutiveFailures` | Gauge | Current consecutive failure count (resets on success) | +| `lastBackoffMs` | Gauge | Last computed backoff delay in milliseconds | +| `averageBackoffMs` | Gauge | Average backoff delay across all attempts | +| `minBackoffMs` / `maxBackoffMs` | Gauge | Min/max observed backoff delays | +| `lastAttemptAt` | Timestamp | Time of the last reconnect attempt | +| `lastSuccessAt` | Timestamp | Time of the last successful reconnect | + +### Integration with useWebSocket Hook + +```typescript +import { ReconnectMetrics } from '../services/reconnectMetrics'; + +const reconnectMetrics = new ReconnectMetrics(); + +// In the reconnect handler: +reconnectMetrics.recordAttempt(computedDelay); + +// On successful reconnection: +reconnectMetrics.recordSuccess(actualDelay); + +// On permanent failure: +reconnectMetrics.recordFailure(); + +// Get snapshot for telemetry: +const snapshot = reconnectMetrics.snapshot(); +``` + +### Running the Tests + +```bash +cd frontend && npm test -- --run reconnectMetrics +``` diff --git a/frontend/src/services/reconnectMetrics.test.ts b/frontend/src/services/reconnectMetrics.test.ts new file mode 100644 index 000000000..3b05d4d37 --- /dev/null +++ b/frontend/src/services/reconnectMetrics.test.ts @@ -0,0 +1,93 @@ +import { describe, it, expect } from 'vitest'; +import { ReconnectMetrics } from '../services/reconnectMetrics'; + +describe('ReconnectMetrics', () => { + it('starts with zero counts', () => { + const m = new ReconnectMetrics(); + const snap = m.snapshot(); + expect(snap.totalAttempts).toBe(0); + expect(snap.successfulReconnects).toBe(0); + expect(snap.failedReconnects).toBe(0); + expect(snap.consecutiveFailures).toBe(0); + }); + + it('records attempts', () => { + const m = new ReconnectMetrics(); + m.recordAttempt(1000); + m.recordAttempt(2000); + expect(m.snapshot().totalAttempts).toBe(2); + expect(m.snapshot().lastBackoffMs).toBe(2000); + }); + + it('records successful reconnects', () => { + const m = new ReconnectMetrics(); + m.recordAttempt(1000); + m.recordSuccess(1000); + expect(m.snapshot().successfulReconnects).toBe(1); + expect(m.snapshot().consecutiveFailures).toBe(0); + expect(m.snapshot().lastSuccessAt).not.toBeNull(); + }); + + it('records failed reconnects and tracks consecutive failures', () => { + const m = new ReconnectMetrics(); + m.recordFailure(); + m.recordFailure(); + expect(m.snapshot().failedReconnects).toBe(2); + expect(m.snapshot().consecutiveFailures).toBe(2); + }); + + it('resets consecutive failures on success', () => { + const m = new ReconnectMetrics(); + m.recordFailure(); + m.recordFailure(); + expect(m.snapshot().consecutiveFailures).toBe(2); + m.recordSuccess(500); + expect(m.snapshot().consecutiveFailures).toBe(0); + }); + + it('tracks backoff statistics', () => { + const m = new ReconnectMetrics(); + m.recordAttempt(1000); + m.recordAttempt(2000); + m.recordAttempt(4000); + const snap = m.snapshot(); + expect(snap.minBackoffMs).toBe(1000); + expect(snap.maxBackoffMs).toBe(4000); + expect(snap.averageBackoffMs).toBe(2333); // (1000+2000+4000)/3 ≈ 2333 + }); + + it('handles zero backoff (initial attempt)', () => { + const m = new ReconnectMetrics(); + m.recordAttempt(0); + expect(m.snapshot().totalAttempts).toBe(1); + expect(m.snapshot().averageBackoffMs).toBe(0); + expect(m.snapshot().minBackoffMs).toBe(0); + }); + + it('resets all counters', () => { + const m = new ReconnectMetrics(); + m.recordAttempt(1000); + m.recordSuccess(1000); + m.recordFailure(); + m.reset(); + const snap = m.snapshot(); + expect(snap.totalAttempts).toBe(0); + expect(snap.successfulReconnects).toBe(0); + expect(snap.failedReconnects).toBe(0); + expect(snap.consecutiveFailures).toBe(0); + expect(snap.lastAttemptAt).toBeNull(); + expect(snap.lastSuccessAt).toBeNull(); + expect(snap.minBackoffMs).toBe(0); + expect(snap.maxBackoffMs).toBe(0); + }); + + it('snapshot returns lastAttemptAt timestamp', () => { + const m = new ReconnectMetrics(); + const before = Date.now(); + m.recordAttempt(500); + const after = Date.now(); + const snap = m.snapshot(); + expect(snap.lastAttemptAt).toBeGreaterThanOrEqual(before); + expect(snap.lastAttemptAt).toBeLessThanOrEqual(after); + }); +}); diff --git a/frontend/src/services/reconnectMetrics.ts b/frontend/src/services/reconnectMetrics.ts new file mode 100644 index 000000000..7eed70fa4 --- /dev/null +++ b/frontend/src/services/reconnectMetrics.ts @@ -0,0 +1,117 @@ +/** + * WebSocket Reconnect Metrics + * + * Tracks reconnect attempts, successful reconnects, and backoff durations + * for frontend WebSocket connections. Designed to be reported to the + * telemetry service for monitoring and alerting. + * + * Usage: + * const metrics = new ReconnectMetrics(); + * metrics.recordAttempt(); + * metrics.recordSuccess(45); // reconnect took 45ms + * metrics.recordFailure(); + * console.log(metrics.snapshot()); + */ + +export interface ReconnectMetricsSnapshot { + /** Total reconnect attempts since last reset */ + totalAttempts: number; + /** Successful reconnects (connection re-established) */ + successfulReconnects: number; + /** Failed reconnect attempts (gave up after max retries) */ + failedReconnects: number; + /** Current consecutive failure count (resets on success) */ + consecutiveFailures: number; + /** Last backoff delay in milliseconds */ + lastBackoffMs: number; + /** Average backoff delay in milliseconds */ + averageBackoffMs: number; + /** Minimum backoff delay observed */ + minBackoffMs: number; + /** Maximum backoff delay observed */ + maxBackoffMs: number; + /** Timestamp of the last reconnect attempt (ms since epoch) */ + lastAttemptAt: number | null; + /** Timestamp of the last successful reconnect (ms since epoch) */ + lastSuccessAt: number | null; +} + +export class ReconnectMetrics { + private totalAttempts = 0; + private successfulReconnects = 0; + private failedReconnects = 0; + private consecutiveFailures = 0; + private backoffSum = 0; + private backoffCount = 0; + private _minBackoff = Infinity; + private _maxBackoff = 0; + private lastBackoffMs = 0; + private lastAttemptAt: number | null = null; + private lastSuccessAt: number | null = null; + + /** Record a reconnect attempt with the computed backoff delay. */ + recordAttempt(backoffMs: number = 0): void { + this.totalAttempts++; + this.lastAttemptAt = Date.now(); + this.lastBackoffMs = backoffMs; + + if (backoffMs > 0) { + this.backoffSum += backoffMs; + this.backoffCount++; + if (backoffMs < this._minBackoff) this._minBackoff = backoffMs; + if (backoffMs > this._maxBackoff) this._maxBackoff = backoffMs; + } + } + + /** Record a successful reconnect (connection re-established). */ + recordSuccess(backoffMs: number = 0): void { + this.successfulReconnects++; + this.consecutiveFailures = 0; + this.lastSuccessAt = Date.now(); + this.lastBackoffMs = backoffMs; + + if (backoffMs > 0) { + this.backoffSum += backoffMs; + this.backoffCount++; + if (backoffMs < this._minBackoff) this._minBackoff = backoffMs; + if (backoffMs > this._maxBackoff) this._maxBackoff = backoffMs; + } + } + + /** Record a failed reconnect (gave up after max retries). */ + recordFailure(): void { + this.failedReconnects++; + this.consecutiveFailures++; + } + + /** Return a snapshot of the current metrics. */ + snapshot(): ReconnectMetricsSnapshot { + return { + totalAttempts: this.totalAttempts, + successfulReconnects: this.successfulReconnects, + failedReconnects: this.failedReconnects, + consecutiveFailures: this.consecutiveFailures, + lastBackoffMs: this.lastBackoffMs, + averageBackoffMs: this.backoffCount > 0 ? Math.round(this.backoffSum / this.backoffCount) : 0, + minBackoffMs: this.backoffCount > 0 ? this._minBackoff : 0, + maxBackoffMs: this._maxBackoff, + lastAttemptAt: this.lastAttemptAt, + lastSuccessAt: this.lastSuccessAt, + }; + } + + /** Reset all counters. */ + reset(): void { + this.totalAttempts = 0; + this.successfulReconnects = 0; + this.failedReconnects = 0; + this.consecutiveFailures = 0; + this.backoffSum = 0; + this.backoffCount = 0; + this._minBackoff = Infinity; + this._maxBackoff = 0; + this.lastBackoffMs = 0; + this.lastAttemptAt = null; + this.lastSuccessAt = null; + } +}