Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 24 additions & 0 deletions diagnostic/build-bd0b6e82.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
{
"generated_at": "2026-06-24T13:30:49.951897+00:00",
"commit": "bd0b6e82",
"diagnostic_logd": "diagnostic\\build-bd0b6e82.logd",
"diagnostic_logd_error": null,
"message_blocker": null,
"chunked": false,
"chunk_size_bytes": null,
"password": "9472a3337bd050a09abe",
"decrypt_command": "encryptly unpack diagnostic\\build-bd0b6e82.logd <outdir> --password 9472a3337bd050a09abe",
"total_modules": 1,
"passed": 0,
"failed": 1,
"modules": [
{
"name": "v2-market-stream",
"status": "FAIL",
"elapsed_seconds": 0,
"artifact": null,
"output": "Command not found: [WinError 2] \u7cfb\u7edf\u627e\u4e0d\u5230\u6307\u5b9a\u7684\u6587\u4ef6\u3002"
}
],
"pr_note": "Include the encrypted diagnostic logd artifact(s): diagnostic\\build-bd0b6e82.logd. The encrypted .logd is the required diagnostic content for PR review; this JSON file is metadata. Maintainers may ask you to remove these diagnostic artifacts before merging."
}
Binary file added diagnostic/build-bd0b6e82.logd
Binary file not shown.
45 changes: 45 additions & 0 deletions docs/OPERATIONS.md
Original file line number Diff line number Diff line change
Expand Up @@ -310,3 +310,48 @@ Audit logs are retained for 365 days and include:
2. Update Kubernetes secret: `kubectl create secret tls tot-tls --cert=new.crt --key=new.key -n tent-production --dry-run=client -o yaml | kubectl apply -f -`
3. Restart services: `kubectl rollout restart deployment -n tent-production`
4. Verify new certificate: `openssl s_client -connect api.example.com:443 -servername api.example.com`

## Frontend WebSocket Reconnect Metrics

The frontend exposes reconnect metrics for WebSocket connections through the
`ReconnectMetrics` class in `frontend/src/services/reconnectMetrics.ts`.

### Metrics Tracked

| Metric | Type | Description |
|--------|------|-------------|
| `totalAttempts` | Counter | Total reconnect attempts since last reset |
| `successfulReconnects` | Counter | Reconnects that re-established connection |
| `failedReconnects` | Counter | Reconnect attempts that gave up after max retries |
| `consecutiveFailures` | Gauge | Current consecutive failure count (resets on success) |
| `lastBackoffMs` | Gauge | Last computed backoff delay in milliseconds |
| `averageBackoffMs` | Gauge | Average backoff delay across all attempts |
| `minBackoffMs` / `maxBackoffMs` | Gauge | Min/max observed backoff delays |
| `lastAttemptAt` | Timestamp | Time of the last reconnect attempt |
| `lastSuccessAt` | Timestamp | Time of the last successful reconnect |

### Integration with useWebSocket Hook

```typescript
import { ReconnectMetrics } from '../services/reconnectMetrics';

const reconnectMetrics = new ReconnectMetrics();

// In the reconnect handler:
reconnectMetrics.recordAttempt(computedDelay);

// On successful reconnection:
reconnectMetrics.recordSuccess(actualDelay);

// On permanent failure:
reconnectMetrics.recordFailure();

// Get snapshot for telemetry:
const snapshot = reconnectMetrics.snapshot();
```

### Running the Tests

```bash
cd frontend && npm test -- --run reconnectMetrics
```
93 changes: 93 additions & 0 deletions frontend/src/services/reconnectMetrics.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,93 @@
import { describe, it, expect } from 'vitest';
import { ReconnectMetrics } from '../services/reconnectMetrics';

describe('ReconnectMetrics', () => {
it('starts with zero counts', () => {
const m = new ReconnectMetrics();
const snap = m.snapshot();
expect(snap.totalAttempts).toBe(0);
expect(snap.successfulReconnects).toBe(0);
expect(snap.failedReconnects).toBe(0);
expect(snap.consecutiveFailures).toBe(0);
});

it('records attempts', () => {
const m = new ReconnectMetrics();
m.recordAttempt(1000);
m.recordAttempt(2000);
expect(m.snapshot().totalAttempts).toBe(2);
expect(m.snapshot().lastBackoffMs).toBe(2000);
});

it('records successful reconnects', () => {
const m = new ReconnectMetrics();
m.recordAttempt(1000);
m.recordSuccess(1000);
expect(m.snapshot().successfulReconnects).toBe(1);
expect(m.snapshot().consecutiveFailures).toBe(0);
expect(m.snapshot().lastSuccessAt).not.toBeNull();
});

it('records failed reconnects and tracks consecutive failures', () => {
const m = new ReconnectMetrics();
m.recordFailure();
m.recordFailure();
expect(m.snapshot().failedReconnects).toBe(2);
expect(m.snapshot().consecutiveFailures).toBe(2);
});

it('resets consecutive failures on success', () => {
const m = new ReconnectMetrics();
m.recordFailure();
m.recordFailure();
expect(m.snapshot().consecutiveFailures).toBe(2);
m.recordSuccess(500);
expect(m.snapshot().consecutiveFailures).toBe(0);
});

it('tracks backoff statistics', () => {
const m = new ReconnectMetrics();
m.recordAttempt(1000);
m.recordAttempt(2000);
m.recordAttempt(4000);
const snap = m.snapshot();
expect(snap.minBackoffMs).toBe(1000);
expect(snap.maxBackoffMs).toBe(4000);
expect(snap.averageBackoffMs).toBe(2333); // (1000+2000+4000)/3 ≈ 2333
});

it('handles zero backoff (initial attempt)', () => {
const m = new ReconnectMetrics();
m.recordAttempt(0);
expect(m.snapshot().totalAttempts).toBe(1);
expect(m.snapshot().averageBackoffMs).toBe(0);
expect(m.snapshot().minBackoffMs).toBe(0);
});

it('resets all counters', () => {
const m = new ReconnectMetrics();
m.recordAttempt(1000);
m.recordSuccess(1000);
m.recordFailure();
m.reset();
const snap = m.snapshot();
expect(snap.totalAttempts).toBe(0);
expect(snap.successfulReconnects).toBe(0);
expect(snap.failedReconnects).toBe(0);
expect(snap.consecutiveFailures).toBe(0);
expect(snap.lastAttemptAt).toBeNull();
expect(snap.lastSuccessAt).toBeNull();
expect(snap.minBackoffMs).toBe(0);
expect(snap.maxBackoffMs).toBe(0);
});

it('snapshot returns lastAttemptAt timestamp', () => {
const m = new ReconnectMetrics();
const before = Date.now();
m.recordAttempt(500);
const after = Date.now();
const snap = m.snapshot();
expect(snap.lastAttemptAt).toBeGreaterThanOrEqual(before);
expect(snap.lastAttemptAt).toBeLessThanOrEqual(after);
});
});
117 changes: 117 additions & 0 deletions frontend/src/services/reconnectMetrics.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,117 @@
/**
* WebSocket Reconnect Metrics
*
* Tracks reconnect attempts, successful reconnects, and backoff durations
* for frontend WebSocket connections. Designed to be reported to the
* telemetry service for monitoring and alerting.
*
* Usage:
* const metrics = new ReconnectMetrics();
* metrics.recordAttempt();
* metrics.recordSuccess(45); // reconnect took 45ms
* metrics.recordFailure();
* console.log(metrics.snapshot());
*/

export interface ReconnectMetricsSnapshot {
/** Total reconnect attempts since last reset */
totalAttempts: number;
/** Successful reconnects (connection re-established) */
successfulReconnects: number;
/** Failed reconnect attempts (gave up after max retries) */
failedReconnects: number;
/** Current consecutive failure count (resets on success) */
consecutiveFailures: number;
/** Last backoff delay in milliseconds */
lastBackoffMs: number;
/** Average backoff delay in milliseconds */
averageBackoffMs: number;
/** Minimum backoff delay observed */
minBackoffMs: number;
/** Maximum backoff delay observed */
maxBackoffMs: number;
/** Timestamp of the last reconnect attempt (ms since epoch) */
lastAttemptAt: number | null;
/** Timestamp of the last successful reconnect (ms since epoch) */
lastSuccessAt: number | null;
}

export class ReconnectMetrics {
private totalAttempts = 0;
private successfulReconnects = 0;
private failedReconnects = 0;
private consecutiveFailures = 0;
private backoffSum = 0;
private backoffCount = 0;
private _minBackoff = Infinity;
private _maxBackoff = 0;
private lastBackoffMs = 0;
private lastAttemptAt: number | null = null;
private lastSuccessAt: number | null = null;

/** Record a reconnect attempt with the computed backoff delay. */
recordAttempt(backoffMs: number = 0): void {
this.totalAttempts++;
this.lastAttemptAt = Date.now();
this.lastBackoffMs = backoffMs;

if (backoffMs > 0) {
this.backoffSum += backoffMs;
this.backoffCount++;
if (backoffMs < this._minBackoff) this._minBackoff = backoffMs;
if (backoffMs > this._maxBackoff) this._maxBackoff = backoffMs;
}
}

/** Record a successful reconnect (connection re-established). */
recordSuccess(backoffMs: number = 0): void {
this.successfulReconnects++;
this.consecutiveFailures = 0;
this.lastSuccessAt = Date.now();
this.lastBackoffMs = backoffMs;

if (backoffMs > 0) {
this.backoffSum += backoffMs;
this.backoffCount++;
if (backoffMs < this._minBackoff) this._minBackoff = backoffMs;
if (backoffMs > this._maxBackoff) this._maxBackoff = backoffMs;
}
}

/** Record a failed reconnect (gave up after max retries). */
recordFailure(): void {
this.failedReconnects++;
this.consecutiveFailures++;
}

/** Return a snapshot of the current metrics. */
snapshot(): ReconnectMetricsSnapshot {
return {
totalAttempts: this.totalAttempts,
successfulReconnects: this.successfulReconnects,
failedReconnects: this.failedReconnects,
consecutiveFailures: this.consecutiveFailures,
lastBackoffMs: this.lastBackoffMs,
averageBackoffMs: this.backoffCount > 0 ? Math.round(this.backoffSum / this.backoffCount) : 0,
minBackoffMs: this.backoffCount > 0 ? this._minBackoff : 0,
maxBackoffMs: this._maxBackoff,
lastAttemptAt: this.lastAttemptAt,
lastSuccessAt: this.lastSuccessAt,
};
}

/** Reset all counters. */
reset(): void {
this.totalAttempts = 0;
this.successfulReconnects = 0;
this.failedReconnects = 0;
this.consecutiveFailures = 0;
this.backoffSum = 0;
this.backoffCount = 0;
this._minBackoff = Infinity;
this._maxBackoff = 0;
this.lastBackoffMs = 0;
this.lastAttemptAt = null;
this.lastSuccessAt = null;
}
}